Changeset 26536

Timestamp:

2012-11-28T11:59:17+13:00 (11 years ago)

Author:

davidb

Message:

Introduction of two new OIDtype values (hash_on_full_filename and full_filename) designed to help provide more stable document IDs for collections that are rebuilt over time, including rebuilt after the Greenstone install has been upgraded

Location:

main/trunk/greenstone2

Files:

• bin/script/import.pl (modified) (2 diffs)
• perllib/doc.pm (modified) (4 diffs)
• perllib/docproc.pm (modified) (1 diff)
• perllib/inexport.pm (modified) (2 diffs)
• perllib/plugins/BasePlugin.pm (modified) (2 diffs)
• perllib/plugouts/BasePlugout.pm (modified) (1 diff)
• perllib/strings.properties (modified) (2 diffs)

main/trunk/greenstone2/bin/script/import.pl

@@ -73 +73 @@
     [ { 'name' => "hash",
         'desc' => "{import.OIDtype.hash}" },
+      { 'name' => "hash_on_full_filename",
+        'desc' => "{import.OIDtype.hash_on_full_filename}" },
       { 'name' => "assigned",
         'desc' => "{import.OIDtype.assigned}" },
@@ -78 +80 @@
         'desc' => "{import.OIDtype.incremental}" },
       { 'name' => "dirname",
-        'desc' => "{import.OIDtype.dirname}" } ];
+        'desc' => "{import.OIDtype.dirname}" },
+      { 'name' => "full_filename",
+        'desc' => "{import.OIDtype.full_filename}" } ];
 
 

main/trunk/greenstone2/perllib/doc.pm

@@ -204 +204 @@
     my ($type, $metadata) = @_;
 
-    if (defined $type && $type =~ /^(hash|hash_on_file|hash_on_ga_xml|incremental|filename|dirname|assigned)$/) {
+    if (defined $type && $type =~ /^(hash|hash_on_file|hash_on_ga_xml|hash_on_full_filename|incremental|filename|dirname|full_filename|assigned)$/) {
     $self->{'OIDtype'} = $type;
     } else {
@@ -404 +404 @@
         my $filename = $self->get_source_filename();
         $OID = &File::Basename::fileparse($filename, qr/\.[^.]*/);
+        $OID = &util::tidy_up_oid($OID);
+    } elsif ($self->{'OIDtype'} eq "full_filename") {
+        my $source_filename = $self->get_source_filename();
+        my $dirsep = &util::get_os_dirsep();
+
+        $source_filename =~ s/^import$dirsep//;
+        $source_filename =~ s/$dirsep/-/g;
+        $source_filename =~ s/\./_/g;
+
+        $OID = $source_filename;
         $OID = &util::tidy_up_oid($OID);
     } elsif ($self->{'OIDtype'} eq "dirname") {
@@ -439 +449 @@
     if ($use_hash_oid) {
         my $hash_on_file = 1; 
+        my $hash_on_ga_xml = 0;
+
         if ($self->{'OIDtype'} eq "hash_on_ga_xml") {
         $hash_on_file = 0;
+        $hash_on_ga_xml = 1;
         }
+
+        if ($self->{'OIDtype'} eq "hash_on_full_filename") {
+        $hash_on_file = 0;
+        $hash_on_ga_xml = 0;
+
+        my $source_filename = $self->get_source_filename();
+        my $dirsep = &util::get_os_dirsep();
+
+        $source_filename =~ s/^import$dirsep//;
+        $source_filename =~ s/$dirsep/-/g;
+        $source_filename =~ s/\./_/g;
+        
+        # If the filename is very short then (handled naively)
+        # this can cause conjestion in the hash-values
+        # computed, leading documents sharing the same leading
+        # Hex values in the computed has.
+        #
+        # The solution taken here is to replace the name of
+        # the file name a sufficient number of times (up to
+        # the character limit defined in 'rep_limit' and
+        # make that the content that is hashed on
+
+        # *** Think twice before changing the following value
+        # as it will break backward compatability of computed
+        # document HASH values
+
+        my $rep_limit  = 256; 
+        my $hash_content = undef;
+
+        if (length($source_filename)<$rep_limit) {
+            my $rep_string = "$source_filename|";
+            my $rs_len = length($rep_string);
+
+            my $clone_times = int(($rep_limit-1)/$rs_len) +1;
+            
+            $hash_content = substr($rep_string x $clone_times, 0, $rep_limit);
+        }
+        else {
+            $hash_content = $source_filename;
+        }
+
+        my $filename = &util::get_tmp_filename();
+        if (!open (OUTFILE, ">:utf8", $filename)) {
+            print STDERR "doc::set_OID could not write to $filename\n";
+        } else {
+            print OUTFILE $hash_content;
+            close (OUTFILE);
+        }
+        $OID = $self->_calc_OID ($filename);
+
+        print STDERR "****!!! the computed hash for: '", $source_filename, "' is: ", $OID,"\n\n";
+
+        &util::rm ($filename);
+        }
+
         if ($hash_on_file) {
         # "hash" OID - feed file to hashfile.exe
@@ -450 +518 @@
             $OID = $self->_calc_OID ($filename);
         } else {
-            $hash_on_file = 0;
+            $hash_on_ga_xml = 1; # switch to back-up plan, and hash on GA file instead
         }
         }
-        if (!$hash_on_file) {
+
+        if ($hash_on_ga_xml) {
+        # In addition being asked to explicity calculate the has based on the GA file,
+        # can also end up coming into this block is doing 'hash_on_file' but the file
+        # itself is of zero bytes (as could be the case with 'doc.nul' file
+
         my $filename = &util::get_tmp_filename();
         if (!open (OUTFILE, ">:utf8", $filename)) {

main/trunk/greenstone2/perllib/docproc.pm

@@ -54 +54 @@
     my ($type, $metadata) = @_;
 
-    if ($type =~ /^(hash|incremental|dirname|assigned)$/) {
+    if ($type =~ /^(hash|hash_on_full_filename|incremental|dirname|full_filename|assigned)$/) {
     $self->{'OIDtype'} = $type;
     } else {

main/trunk/greenstone2/perllib/inexport.pm

@@ -297 +297 @@
 
     if (!defined $self->{'OIDtype'} 
-    || ($self->{'OIDtype'} !~ /^(hash|incremental|assigned|dirname)$/ )) {
+    || ($self->{'OIDtype'} !~ /^(hash|hash_on_full_filename|incremental|assigned|dirname|full_filename)$/ )) {
+    # OIDtype was either not defined on the command-line, or if it was not one of the recognized values
     if (defined $collectcfg->{'OIDtype'} 
-        && $collectcfg->{'OIDtype'} =~ /^(hash|incremental|assigned|dirname)$/) {
+        && $collectcfg->{'OIDtype'} =~ /^(hash|hash_on_full_filename|incremental|assigned|dirname|full_filename)$/) {
         $self->{'OIDtype'} = $collectcfg->{'OIDtype'};
     } else {
@@ -548 +549 @@
     $processor->setoutputdir ($archivedir);
     $processor->set_sortmeta ($sortmeta, $removeprefix, $removesuffix) if defined $sortmeta;
+
     $processor->set_OIDtype ($OIDtype, $OIDmetadata);
     

main/trunk/greenstone2/perllib/plugins/BasePlugin.pm

@@ -97 +97 @@
       { 'name' => "hash_on_ga_xml",
         'desc' => "{import.OIDtype.hash_on_ga_xml}" },
+      { 'name' => "hash_on_full_filename",
+        'desc' => "{import.OIDtype.hash_on_full_filename}" },
       { 'name' => "assigned",
         'desc' => "{import.OIDtype.assigned}" },
@@ -104 +106 @@
         'desc' => "{import.OIDtype.filename}" },
       { 'name' => "dirname",
-        'desc' => "{import.OIDtype.dirname}" } ];
+        'desc' => "{import.OIDtype.dirname}" },
+      { 'name' => "full_filename",
+        'desc' => "{import.OIDtype.full_filename}" } ];
 
 my $arguments =

main/trunk/greenstone2/perllib/plugouts/BasePlugout.pm

@@ -292 +292 @@
     my ($type, $metadata) = @_;
 
-    if ($type =~ /^(hash|incremental|dirname|assigned)$/) {
+    if ($type =~ /^(hash|hash_on_full_filename|incremental|dirname|full_filename|assigned)$/) {
     $self->{'OIDtype'} = $type;
     } else {

main/trunk/greenstone2/perllib/strings.properties

@@ -279 +279 @@
 import.OIDtype.hash:Hash the contents of the file. Document identifiers will be the same every time the collection is imported.
 import.OIDtype.hash_on_ga_xml:Hash the contents of the Greenstone Archive XML file. Document identifiers will be the same every time the collection is imported as long as the metadata does not change.
+import.OIDtype.hash_on_full_filename:Hash on the full filename to the document within the 'import' folder (and not its contents).  Helps make document identifiers more stable across upgrades of the software, although it means that duplicate documents contained in the collection are no longer detected automatically.
 
 import.OIDtype.incremental:Use a simple document count. Significantly faster than "hash", but does not necessarily assign the same identifier to the same document content if the collection is reimported.
@@ -285 +286 @@
 
 import.OIDtype.dirname:Use the parent directory name (preceded by 'J'). There should only be one document per directory, and directory names should be unique. E.g. import/b13as/h15ef/page.html will get an identifier of Jh15ef.
+
+import.OIDtype.filename:Use the tail file name.  Requires every filename across all the folders within 'import' to be unique.
+
+import.OIDtype.full_filename:Use the full file name within the 'import' folder as the identifier for the document (with _ and - substitutions made for symbols such as directory separators and the fullstop in a filename extension)
 
 import.OIDmetadata:Specifies the metadata element that hold's the document's unique identifier, for use with -OIDtype=assigned.