Changeset 12003

Timestamp:

2006-07-04T15:36:20+12:00 (18 years ago)

Author:

davidb

Message:

Scripts upgraded to perform more efficiently with incremental addition.

Location:

trunk/gsdl/bin/script

Files:

• build (modified) (5 diffs)
• import.pl (modified) (7 diffs)
• lucene_passes.pl (modified) (1 diff)

trunk/gsdl/bin/script/build

@@ -155 +155 @@
     print STDOUT "   -optionfile file        Get options from file, useful on systems where\n";
     print STDOUT "                           long command lines may cause problems\n";
+    print STDOUT "   -indextype mg|mgpp|lucene \n";
+    print STDERR "                           Specify the type of indexer used in this collection\n";
+    print STDERR "                           If -append is used then -indextype is needed to \n";
+    print STDERR "                           determine how to run buildcol.pl as well as update\n";
+    print STDERR "                           'building' and 'index' according.\n";
     print STDOUT "   -append                 Add new files to existing collection\n";
+    print STDOUT "   -manifest               Use manifest.xml file to determine which files to process.\n";
     print STDOUT "   -remove_archives        Remove archives directory after successfully\n";
     print STDOUT "                           building the collection.\n";
@@ -386 +392 @@
     my $import_cmd = "perl -S import.pl";
     $import_cmd .= " -out \"$outfile.import\"" if $use_out;
-    $import_cmd .= " -removeold" unless $append;
+    if ($append) {
+    $import_cmd .= " -keepold";
+    } else {
+    $import_cmd .= " -removeold";
+    }
+
+    $import_cmd .= " -manifest manifest.xml" if ($manifest);
     $import_cmd .= " -collectdir \"$collectdir\"" if $collectdir =~ /\w/;
     $import_cmd .= " -statsfile \"$statsfile\"" if $statsfile =~ /\w/;
@@ -415 +427 @@
 
     my $build_cmd = "perl -S buildcol.pl";
+
+    my $removeold = 1;
+    if ($append) {
+    if ($indextype eq "lucene") {
+        $build_cmd .= " -keepold";
+        $removeold = 0;
+    }
+    else {
+        $build_cmd .= " -removeold";
+    }
+    }
+    else {
+    $build_cmd .= " -removeold";
+    }
+
     $build_cmd .= " -out \"$outfile.build\"" if $use_out;
     $build_cmd .= " -collectdir \"$collectdir\"" if $collectdir =~ /\w/;
@@ -437 +464 @@
     }
 
-    # replace old indexes with new ones
-    if (&has_content ($indexdir)) {
-    print $out "removing old indexes\n";
-    &util::rm_r ($indexdir);
-    }
-    rmdir ($indexdir) if -d $indexdir;
-    &File::Copy::move ($buildingdir, $indexdir);
+    if ($removeold) {
+    # replace old indexes with new ones
+    if (&has_content ($indexdir)) {
+        print $out "removing old indexes\n";
+        &util::rm_r ($indexdir);
+    }
+    rmdir ($indexdir) if -d $indexdir;
+    &File::Copy::move ($buildingdir, $indexdir);
+    }
+    else {
+    # Do nothing.  Assume index is symbolic link to building
+    }
 
     # remove the cached arhives
@@ -560 +592 @@
     if (!parsargv::parse($argref, 
              'optionfile/.*/', \$optionfile,
+             'indextype/^(mg|mgpp|lucene)$/mg', \$indextype,
              'append', \$append,
+             'manifest', \$manifest,
              'remove_archives', \$remove_archives,
              'remove_import', \$remove_import,

trunk/gsdl/bin/script/import.pl

@@ -44 +44 @@
 use plugin;
 use docprint;
+use manifest;
 use util;
 use scriptutil;
@@ -50 +51 @@
 use printusage;
 use parse2;
+
+
 
 use strict;
@@ -92 +95 @@
     # parsearg left "" as default
     #'deft' => &util::filename_cat ($ENV{'GSDLHOME'}, "collect"),
+    'deft' => "",
+    'reqd' => "no",
+        'hiddengli' => "yes" },
+      { 'name' => "manifest",
+    'desc' => "{import.manifest}",
+    'type' => "string",
     'deft' => "",
     'reqd' => "no",
@@ -219 +228 @@
 
 sub main {
-    my ($verbosity, $importdir, $archivedir, $keepold, 
+    my ($verbosity, $importdir, $archivedir, $manifest, $keepold, 
     $removeold, $saveas, $version,
     $gzip, $groupsize, $OIDtype, $debug, 
@@ -331 +340 @@
     $archivedir = $collectcfg->{'archivedir'};
     }
+    if (defined $collectcfg->{'manifest'} && $manifest eq "") {
+    $manifest = $collectcfg->{'manifest'};
+    }
 
     if (defined $collectcfg->{'gzip'} && !$gzip) {
@@ -403 +415 @@
     $archivedir =~ s/\/$//;
 
+    my $manifest_lookup = new manifest();
+    if ($manifest ne "") {  
+    my $manifest_filename = $manifest;
+
+    if ($manifest_filename !~ m/^[\\\/]/) {
+        $manifest_filename = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, $manifest_filename);
+    }
+
+    $manifest =~ s/[\\\/]+/\//g;
+    $manifest =~ s/\/$//;
+
+    $manifest_lookup->parse($manifest_filename);
+    }
+
+
     # load all the plugins
     $pluginfo = &plugin::load_plugins ($plugins, $verbosity, $out, $faillog, \@global_opts);
@@ -445 +472 @@
     &plugin::begin($pluginfo, $importdir, $processor, $maxdocs, $gli);
 
-    # process the import directory
-    &plugin::read ($pluginfo, $importdir, "", {}, $processor, $maxdocs, 0, $gli);
-    
+    if ($manifest eq "") {
+    # process the import directory
+    &plugin::read ($pluginfo, $importdir, "", {}, $processor, $maxdocs, 0, $gli);
+    }
+    else {
+
+    # process any new files
+    foreach my $file (keys %{$manifest_lookup->{'index'}}) {
+        &plugin::read ($pluginfo, $importdir, $file, {}, $processor, $maxdocs, 0, $gli);
+    }
+
+    # record files marked for deletion in arcinfo
+    foreach my $file (keys %{$manifest_lookup->{'delete'}}) {
+        # consider finding it?
+        # $archive_info->add_info($OID,$doc_xml_file,"D");
+    }
+    }
+
     &plugin::end($pluginfo, $processor);
 

trunk/gsdl/bin/script/lucene_passes.pl

@@ -127 +127 @@
         } elsif ($mode eq "index") {
         # notify lucene indexer
-        # print STDERR $doc_xml;
+
+        # SAX parser seems to be sensitive to blank lines
+        # => remove them
+        $doc_xml =~ s/\n+/\n/g;
+
+#        print STDERR $doc_xml;
+
 ##      print PIPEOUT "$output_filename\n";
+
         print PIPEOUT "$doc_xml";
+
+
         #save_xml_doc($full_textdir, "$output_filename.txt", $doc_xml);
         }