Changeset 20578

Timestamp:

2009-09-10T10:51:48+12:00 (15 years ago)

Author:

davidb

Message:

Changes to support new incremental_mode variable, and to track metadata.xml files more closely so incremental building works when a document is first build without any metadata attached, then then in a subsequent build has metadata from such a source as metadata.xml added.

Location:

gsdl/trunk/perllib

Files:

• inexport.pm (modified) (5 diffs)
• plugin.pm (modified) (2 diffs)
• scriptutil.pm (modified) (4 diffs)

gsdl/trunk/perllib/inexport.pm

@@ -79 +79 @@
 sub new_vs_old_import_diff
 {
-    my ($archive_info,$block_hash,$importdir) = @_;
+    my ($archive_info,$block_hash,$importdir,$archivedir,$verbosity,$incremental_mode) = @_;
 
     # First convert all files to absolute form
@@ -108 +108 @@
 
     # entry in 'all_files' is moved to either 'existing_files', 
-    # 'deleted_files', or 'new_files'
+    # 'deleted_files', 'new_files', or 'new_metadata_files'
 
     if (!&util::filename_is_absolute($curr_file)) {
@@ -129 +129 @@
     # figure out if new file or not
     if (defined $full_prev_all_files->{$full_curr_file}) {
+        
+        if ($incremental_mode eq "all") {
+
+        # had it before
+        $block_hash->{'existing_files'}->{$full_curr_file} = 1;
+        
+        # Now remove it, so by end of loop only the files
+        # that need deleting are left
         
-        # had it before
-        $block_hash->{'existing_files'}->{$full_curr_file} = 1;
-
-        # Now remove it, so by end of loop only the files
-        # that need deleting are left
-
-        delete $full_prev_all_files->{$full_curr_file};
-        }
+        delete $full_prev_all_files->{$full_curr_file};
+        }
+        else {
+        # Warning in "onlyadd" mode, but had it before!
+        print STDERR "Warning: File $full_curr_file previously imported.\n";
+        print STDERR "         Treating as new file\n";
+
+        $block_hash->{'new_files'}->{$full_curr_file} = 1;
+        delete $full_prev_all_files->{$full_curr_file};
+        }
+    }
     else {
-        $block_hash->{'new_files'}->{$full_curr_file} = 1;
-    }
+        if ($block_hash->{'metadata_files'}->{$full_curr_file}) {
+        # the new file is the special sort of file greenstone uses
+        # to attach metadata to src documents
+        # i.e metadata.xml 
+        # (but note, the filename used is not constrained in 
+        # Greenstone to always be this)
+
+##      print STDERR "***** Detected new metadata file: $full_curr_file\n";
+        $block_hash->{'new_metadata_files'}->{$full_curr_file} = 1;
+        }
+        else {
+        $block_hash->{'new_files'}->{$full_curr_file} = 1;
+        }
+    }
+
     
     delete $block_hash->{'all_files'}->{$curr_file};
     }
 
+
+    # Deal with complication of new metadata.xml files by forcing
+    # everything from this point down in the file hierarchy to
+    # be freshly imported.  
+    #
+    # This may mean files that have not changed are reindexed, but does
+    # guarantee by the end of processing all new metadata is correctly
+    # associated with the relevant document(s).
+
+    foreach my $new_mdf (keys %{$block_hash->{'new_metadata_files'}}) {
+    my ($fileroot,$situated_dir,$ext) = fileparse($new_mdf, "\\.[^\\.]+\$");
+
+    $situated_dir =~ s/[\\\/]+$//; # remove tailing slashes
+
+    # Go through existing_files, and mark anything that is contained
+    # within 'situated_dir' to be reindexed (in case some of the metadata
+    # attaches to one of these files)
+
+    my $reindex_files = [];
+
+    foreach my $existing_f (keys %{$block_hash->{'existing_files'}}) {
+        # need to protect windows slash \ in regular expression?
+
+        if ($existing_f =~ m/^$situated_dir/) {
+        push(@$reindex_files,$existing_f);
+        $block_hash->{'reindex_files'}->{$existing_f} = 1;
+
+        }
+    }
+
+    # Reindexing is accomplished by putting them in th list for reindexing (line above)
+    # and then tagging the arcinfo version as to be deleted.
+
+    _mark_docs_for_deletion($archive_info,$block_hash,$reindex_files,$archivedir,$verbosity, "reindex");
+    
+    # metadata file needs to be in new_files list so parsed by MetadataXMLPlug
+    # (or equivalent)
+    $block_hash->{'new_files'}->{$new_mdf} = 1; 
+
+    }
+
+    
     # By this point full_prev_all_files contains the files
     # mentioned in archiveinf-src.db but are not in the 'import'
@@ -153 +219 @@
     #
     # In building the final list of files to delete, we test to see if
-    # it exists on the filesystem and if it does (unusual for a file
-    # that's allegedly deleted!), supress it from going into the final
-    # list
+    # it exists on the filesystem and if it does (unusual for a "normal" 
+    # file in import, but possible in the case of 'tmp' files), 
+    # supress it from going into the final list
 
     my $collectdir = $ENV{'GSDLCOLLECTDIR'};
@@ -174 +240 @@
       }
       } @deleted_files;
+
+
+
 }
 

gsdl/trunk/perllib/plugin.pm

@@ -146 +146 @@
 sub load_plugins {
     my ($plugin_list) = shift @_;
-    my $incremental;
-    ($verbosity, $outhandle, $failhandle, $globaloptions, $incremental) = @_; # globals
+    my $incremental_mode;
+    ($verbosity, $outhandle, $failhandle, $globaloptions, $incremental_mode) = @_; # globals
     my @plugin_objects = ();
-    $incremental = 0 unless (defined $incremental && $incremental == 1);
     $verbosity = 2 unless defined $verbosity;
     $outhandle = 'STDERR' unless defined $outhandle;
@@ -184 +183 @@
     $plugobj->init($verbosity, $outhandle, $failhandle);
     
-    $plugobj->set_incremental($incremental);
+    $plugobj->set_incremental($incremental_mode);
 
     # add this object to the list

gsdl/trunk/perllib/scriptutil.pm

@@ -35 +35 @@
 sub check_removeold_and_keepold {
 
-    my ($removeold, $keepold, $incremental, $dir, $collectcfg) = @_;
+    my ($removeold, $keepold, $incremental, $dir, $collectcfg) = @_;    
+
+    # Incremental mode may be set to "none", "onlyadd" or "all"
+    # depending on status of -keepold and -incremental flags
+    my $incremental_mode = ($incremental) ? "all" : "none"; 
 
     if (($keepold && $removeold) || ($incremental && $removeold) ) {
     gsprintf(STDERR, "{scripts.both_old_options}\n", $dir);
     sleep(3); #just in case
-    return (1,0); 
+    return (1,0,0,"none"); 
     
     } 
@@ -50 +54 @@
     } elsif (defined $collectcfg->{'keepold'} && $collectcfg->{'keepold'} =~ /^true$/i) {
         $keepold = 1;
+        $incremental_mode = "onlyadd";
     } elsif (defined $collectcfg->{'incremental'} && $collectcfg->{'incremental'} =~ /^true$/i) {
         $incremental = 1;
+        $incremental_mode = "all";
     }
     }
@@ -58 +64 @@
     gsprintf(STDERR, "{scripts.no_old_options} \n", $dir);
     sleep(3); #just in case
-    return (1,0);
+    return (1,0,0,"none");
     }
     
@@ -65 +71 @@
     $keepold = 1;
     }
-    return ($removeold, $keepold, $incremental);
+    return ($removeold, $keepold, $incremental, $incremental_mode);
 
 }