Changeset 18469 for gsdl/trunk/perllib/inexport.pm

Timestamp:

2009-02-06T18:19:44+13:00 (15 years ago)

Author:

davidb

Message:

Support for reindexing a document added

File:

• gsdl/trunk/perllib/inexport.pm (modified) (8 diffs)

gsdl/trunk/perllib/inexport.pm

@@ -27 +27 @@
 
 use strict;
+
+use File::Basename;
 
 use util;
@@ -53 +55 @@
     }
 
+
     # Figure out which are the new files, existing files and so
     # by implication the files from the previous import that are not
@@ -68 +71 @@
     }
 
+    if (defined $block_hash->{'file_blocks'}->{$full_curr_file}) {
+        # If in block list, we want to ignore it
+        delete $block_hash->{'all_files'}->{$curr_file};
+
+        if (defined $full_prev_all_files->{$full_curr_file}) {
+        # also make sure it is gone from 'previous' list so
+        # not mistaken for a file that needs to be deleted
+        delete $full_prev_all_files->{$full_curr_file};
+        }
+        next;
+    }
+
     # figure of if new file or not
     if (defined $full_prev_all_files->{$full_curr_file}) {
+        
         # had it before
-        $block_hash->{'existing_files'}->{$curr_file} = 1;
+        $block_hash->{'existing_files'}->{$full_curr_file} = 1;
+
         # Now remove it, so by end of loop only the files
         # that need deleting are left
@@ -78 +95 @@
         }
     else {
-        $block_hash->{'new_files'}->{$curr_file} = 1;
+        $block_hash->{'new_files'}->{$full_curr_file} = 1;
     }
     
@@ -84 +101 @@
     }
 
-    # By this point full_prev_all_files contains only the files
-    # that are not in the current import folder => i.e. files
-    # to be deleted
+    # By this point full_prev_all_files contains the files
+    # mentioned in archiveinf-src.db but are not in the 'import'
+    # folder (or whatever was specified through -importdir ...)
+
+    # This list can contain files that were created in the 'tmp' or
+    # 'cache' areas (such as screen-size and thumbnail images).
     #
-    # The value in each key is its "local" import file name, which is what 
-    # we want to use
+    # In building the final list of files to delete, we test to see if
+    # it exists on the filesystem and if it does (unusual for a file
+    # that's allegedly deleted!) , supress it from going into the final
+    # list
+
+    my $collectdir = $ENV{'GSDLCOLLECTDIR'};
+
     my @deleted_files = values %$full_prev_all_files;
-    map { $block_hash->{'deleted_files'}->{$_} = 1 } @deleted_files;
+    map { my $curr_file = $_;
+      my $full_curr_file = $curr_file;
+
+      if (!&util::filename_is_absolute($curr_file)) {
+          # add in import dir to make absolute
+
+          $full_curr_file = &util::filename_cat($collectdir,$curr_file);
+      }
+
+
+      if (!-e $full_curr_file) {
+          $block_hash->{'deleted_files'}->{$curr_file} = 1;
+      }
+      } @deleted_files;
 }
 
@@ -110 +148 @@
     # this file is used in (note in most cases, it's just one OID)
     
-    # An improvement would be to have the record read
-    # into a hash array
     my $src_rec = GDBMUtils::gdbmRecordToHash($arcinfo_src_filename,$file);
     my $oids = $src_rec->{'oid'};
     foreach my $oid (@$oids) {
 
-        # find out if it's an assoc file or main doc
+        # Find out if it's an assoc file or main doc
 
         my $doc_rec = GDBMUtils::gdbmRecordToHash($arcinfo_doc_filename,$oid);
-##      print STDERR "file = $file\n";
-
         if ($doc_rec->{'src-file'}->[0] eq $file) {
-        # mark it for deletion
+        # It's the main doc
+        # => mark it for deletion
+    
         if ($verbosity>1) {
-            print STDERR "$oid marked to be deleted\n";
+            print STDERR "$oid marked to be deleted from index on next buildcol.pl\n";
         }
         $archive_info->set_status_info($oid,"D");
 
         my $val = &GDBMUtils::gdbmDatabaseGet($arcinfo_doc_filename,$oid);
-        $val =~ s/^<index-status>(.*)$/<index-status>D/m;
-        &GDBMUtils::gdbmDatabaseSet($arcinfo_doc_filename,$oid,$val);
+        my ($index_status) = ($val =~ m/^<index-status>(.*)$/m);
+        if ($index_status ne "D") {
+            $val =~ s/^<index-status>(.*)$/<index-status>D/m;
+            &GDBMUtils::gdbmDatabaseSet($arcinfo_doc_filename,$oid,$val);
+            my $doc_file = $doc_rec->{'doc-file'}->[0];
+
+            my $doc_filename = &util::filename_cat($archivedir,$doc_file);
+
+
+            my ($doc_tailname, $doc_dirname, $suffix) 
+            = File::Basename::fileparse($doc_filename, "\\.[^\\.]+\$");
+
+            print STDERR "Removing $doc_dirname\n" if ($verbosity>2);
+
+            &util::rm_r($doc_dirname);
+            
+        }
         }
         else {
@@ -149 +200 @@
         }
         }
+
+        GDBMUtils::gdbmDatabaseRemove($arcinfo_src_filename,$file);
     }
     }
@@ -155 +208 @@
 
 
+sub mark_docs_for_reindex
+{
+    my ($archive_info,$existing_files_ref,$archivedir,$verbosity) = @_;
+
+    # Reindexing is accomplished by deleting the previously indexed
+    # version of the document, and then allowing the new version to
+    # be indexed (as would a new document be indexed). 
+    # 
+    # The first step (marking for deletion) is implemented by this routine.
+    # 
+    # By default in Greenstone a new version of an index will hash to
+    # a new unique OID, and the above strategy of reindex=delete+add
+    # works fine.  A special case arises when a persistent OID is 
+    # allocated to a document (for instance through a metadata field),
+    # and the second step to reindexing (see XXXX) detects this and
+    # deals with it appropriately.
+
+    my $db_ext = &util::is_little_endian() ? ".ldb" : ".bdb";
+    my $doc_db = "archiveinf-doc$db_ext";
+    my $arcinfo_doc_filename = &util::filename_cat ($archivedir, $doc_db);
+
+
+    my $archiveinf_timestamp = -M $arcinfo_doc_filename;
+
+    my $reindex_files_ref = [];
+
+    foreach my $existing_filename (@$existing_files_ref) {
+    
+    if (-M $existing_filename < $archiveinf_timestamp) {
+        # file is newer than last build
+        
+        my $existing_file = $existing_filename;
+        my $collectdir = &util::filename_cat($ENV{'GSDLCOLLECTDIR'});
+
+        $existing_file =~ s/^$collectdir(\\|\/)?//;
+        
+        print STDERR "**** Deleting existing file: $existing_file\n";
+
+        push(@$reindex_files_ref,$existing_file);
+    }
+
+    }
+    
+    mark_docs_for_deletion($archive_info,$reindex_files_ref,$archivedir,$verbosity);
+
+    return @$reindex_files_ref;
+}
+
+
+
 1;