Changeset 26536 for main/trunk/greenstone2/perllib/doc.pm

Timestamp:

2012-11-28T11:59:17+13:00 (11 years ago)

Author:

davidb

Message:

Introduction of two new OIDtype values (hash_on_full_filename and full_filename) designed to help provide more stable document IDs for collections that are rebuilt over time, including rebuilt after the Greenstone install has been upgraded

File:

• main/trunk/greenstone2/perllib/doc.pm (modified) (4 diffs)

main/trunk/greenstone2/perllib/doc.pm

@@ -204 +204 @@
     my ($type, $metadata) = @_;
 
-    if (defined $type && $type =~ /^(hash|hash_on_file|hash_on_ga_xml|incremental|filename|dirname|assigned)$/) {
+    if (defined $type && $type =~ /^(hash|hash_on_file|hash_on_ga_xml|hash_on_full_filename|incremental|filename|dirname|full_filename|assigned)$/) {
     $self->{'OIDtype'} = $type;
     } else {
@@ -404 +404 @@
         my $filename = $self->get_source_filename();
         $OID = &File::Basename::fileparse($filename, qr/\.[^.]*/);
+        $OID = &util::tidy_up_oid($OID);
+    } elsif ($self->{'OIDtype'} eq "full_filename") {
+        my $source_filename = $self->get_source_filename();
+        my $dirsep = &util::get_os_dirsep();
+
+        $source_filename =~ s/^import$dirsep//;
+        $source_filename =~ s/$dirsep/-/g;
+        $source_filename =~ s/\./_/g;
+
+        $OID = $source_filename;
         $OID = &util::tidy_up_oid($OID);
     } elsif ($self->{'OIDtype'} eq "dirname") {
@@ -439 +449 @@
     if ($use_hash_oid) {
         my $hash_on_file = 1; 
+        my $hash_on_ga_xml = 0;
+
         if ($self->{'OIDtype'} eq "hash_on_ga_xml") {
         $hash_on_file = 0;
+        $hash_on_ga_xml = 1;
         }
+
+        if ($self->{'OIDtype'} eq "hash_on_full_filename") {
+        $hash_on_file = 0;
+        $hash_on_ga_xml = 0;
+
+        my $source_filename = $self->get_source_filename();
+        my $dirsep = &util::get_os_dirsep();
+
+        $source_filename =~ s/^import$dirsep//;
+        $source_filename =~ s/$dirsep/-/g;
+        $source_filename =~ s/\./_/g;
+        
+        # If the filename is very short then (handled naively)
+        # this can cause conjestion in the hash-values
+        # computed, leading documents sharing the same leading
+        # Hex values in the computed has.
+        #
+        # The solution taken here is to replace the name of
+        # the file name a sufficient number of times (up to
+        # the character limit defined in 'rep_limit' and
+        # make that the content that is hashed on
+
+        # *** Think twice before changing the following value
+        # as it will break backward compatability of computed
+        # document HASH values
+
+        my $rep_limit  = 256; 
+        my $hash_content = undef;
+
+        if (length($source_filename)<$rep_limit) {
+            my $rep_string = "$source_filename|";
+            my $rs_len = length($rep_string);
+
+            my $clone_times = int(($rep_limit-1)/$rs_len) +1;
+            
+            $hash_content = substr($rep_string x $clone_times, 0, $rep_limit);
+        }
+        else {
+            $hash_content = $source_filename;
+        }
+
+        my $filename = &util::get_tmp_filename();
+        if (!open (OUTFILE, ">:utf8", $filename)) {
+            print STDERR "doc::set_OID could not write to $filename\n";
+        } else {
+            print OUTFILE $hash_content;
+            close (OUTFILE);
+        }
+        $OID = $self->_calc_OID ($filename);
+
+        print STDERR "****!!! the computed hash for: '", $source_filename, "' is: ", $OID,"\n\n";
+
+        &util::rm ($filename);
+        }
+
         if ($hash_on_file) {
         # "hash" OID - feed file to hashfile.exe
@@ -450 +518 @@
             $OID = $self->_calc_OID ($filename);
         } else {
-            $hash_on_file = 0;
+            $hash_on_ga_xml = 1; # switch to back-up plan, and hash on GA file instead
         }
         }
-        if (!$hash_on_file) {
+
+        if ($hash_on_ga_xml) {
+        # In addition being asked to explicity calculate the has based on the GA file,
+        # can also end up coming into this block is doing 'hash_on_file' but the file
+        # itself is of zero bytes (as could be the case with 'doc.nul' file
+
         my $filename = &util::get_tmp_filename();
         if (!open (OUTFILE, ">:utf8", $filename)) {