Changeset 20685

Timestamp:

2009-09-23T13:47:49+12:00 (15 years ago)

Author:

kjdon

Message:

removed classify lines from INC - we don't do classifying during import. while processing the import directory, do the prime_doc_oid_count and new_vs_old-import_diff when incremental_mode is onlyadd, as well as for incremental. Otherwise, if we do -keepold, we get no docs imported

File:

• gsdl/trunk/bin/script/import.pl (modified) (6 diffs)

gsdl/trunk/bin/script/import.pl

@@ -39 +39 @@
     unshift (@INC, "$ENV{'GSDLHOME'}/perllib/plugins");
     unshift (@INC, "$ENV{'GSDLHOME'}/perllib/plugouts");
-    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/classify");
 
     if (defined $ENV{'GSDLEXTS'}) {
@@ -50 +49 @@
         unshift (@INC, "$ext_prefix/perllib/plugins");
         unshift (@INC, "$ext_prefix/perllib/plugouts");
-        unshift (@INC, "$ext_prefix/perllib/classify");
     }
     }
@@ -198 +196 @@
     'desc' => "{import.OIDmetadata}",
     'type' => "string",
-    # 'type' => "metadata", #doesn't work properly in GLI
+     #'type' => "metadata", #doesn't work properly in GLI
     # parsearg left "" as default
     #'deft' => "dc.Identifier",
@@ -589 +587 @@
 
 
-    if ($incremental) {
-        # equivalent to saying ($keepold && ($incremental_mode eq "all"))
+    if ($incremental || $incremental_mode eq "onlyadd") {
 
         &inexport::prime_doc_oid_count($archivedir);
@@ -601 +598 @@
                           $archivedir,$verbosity,$incremental_mode);
         
-        my @deleted_files = sort keys %{$block_hash->{'deleted_files'}};
-        # Filter out any in gsdl/tmp area
-        my @filtered_deleted_files = ();
-        my $gsdl_tmp_area = &util::filename_cat($ENV{'GSDLHOME'}, "tmp");
-        my $collect_tmp_area = &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, "tmp");
-        $gsdl_tmp_area = &util::filename_to_regex($gsdl_tmp_area);
-        $collect_tmp_area = &util::filename_to_regex($collect_tmp_area);
-
-
-        foreach my $df (@deleted_files) {
-        next if ($df =~ m/^$gsdl_tmp_area/);
-        next if ($df =~ m/^$collect_tmp_area/);
-        
-        push(@filtered_deleted_files,$df);
-        }
-
-        
-        @deleted_files = @filtered_deleted_files;
-
-        if (scalar(@deleted_files>0)) {
-        print STDERR "Files deleted since last import:\n  ";
-        print STDERR join("\n  ",@deleted_files), "\n";
-        }
-        
         my @new_files = sort keys %{$block_hash->{'new_files'}};
         if (scalar(@new_files>0)) {
@@ -630 +603 @@
         print STDERR join("\n  ",@new_files), "\n";
         }
-        
-        &inexport::mark_docs_for_deletion($archive_info,$block_hash,\@deleted_files,
-                          $archivedir,$verbosity);
-
-        &inexport::mark_docs_for_reindex($archive_info,$block_hash,
-                         $archivedir,$verbosity);
-
-        my @reindex_files = sort keys %{$block_hash->{'reindex_files'}};
-
-        if (scalar(@reindex_files>0)) {
-        print STDERR "Files to reindex since last import:\n  ";
-        print STDERR join("\n  ",@reindex_files), "\n";
-        }
-
-
-        # not sure if the following will work -- will the metadata data-structure be correctly initialized
-        # in the right order?
+
+        if ($incremental) {
+               # only look for deletions if we are truely incremental
+        my @deleted_files = sort keys %{$block_hash->{'deleted_files'}};
+        # Filter out any in gsdl/tmp area
+        my @filtered_deleted_files = ();
+        my $gsdl_tmp_area = &util::filename_cat($ENV{'GSDLHOME'}, "tmp");
+        my $collect_tmp_area = &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, "tmp");
+        $gsdl_tmp_area = &util::filename_to_regex($gsdl_tmp_area);
+        $collect_tmp_area = &util::filename_to_regex($collect_tmp_area);
+        
+        
+        foreach my $df (@deleted_files) {
+            next if ($df =~ m/^$gsdl_tmp_area/);
+            next if ($df =~ m/^$collect_tmp_area/);
+            
+            push(@filtered_deleted_files,$df);
+        }
+        
+        
+        @deleted_files = @filtered_deleted_files;
+        
+        if (scalar(@deleted_files>0)) {
+            print STDERR "Files deleted since last import:\n  ";
+            print STDERR join("\n  ",@deleted_files), "\n";
+        }
+        
+        
+        &inexport::mark_docs_for_deletion($archive_info,$block_hash,\@deleted_files,
+                          $archivedir,$verbosity);
+        
+        &inexport::mark_docs_for_reindex($archive_info,$block_hash,
+                         $archivedir,$verbosity);
+        
+        my @reindex_files = sort keys %{$block_hash->{'reindex_files'}};
+        
+        if (scalar(@reindex_files>0)) {
+            print STDERR "Files to reindex since last import:\n  ";
+            print STDERR join("\n  ",@reindex_files), "\n";
+        }
+        
+        
+        # not sure if the following will work -- will the metadata data-structure be correctly initialized
+        # in the right order?
 #       foreach my $file (@new_files, @reindex_files) {
 #       &plugin::read ($pluginfo, $importdir, $file, $block_hash, $metadata, $processor, $maxdocs, 0, $gli);
 #       }
-
-
+        
+        }
+        
         # Play it safe, and run through the entire folder, only processing new or edited files
         &plugin::read ($pluginfo, $importdir, "", $block_hash, $metadata, $processor, $maxdocs, 0, $gli);