Changeset 18528 for gsdl/trunk/perllib

Timestamp:

2009-02-17T15:36:44+13:00 (15 years ago)

Author:

davidb

Message:

OIDmetadata wasn't supported in collect.cfg, but OIDtype was. Now rectified. Also introduced OIDcount as a file saved in the archives folder to help doc.pm use the correct value when working incrementally

Location:

gsdl/trunk/perllib

Files:

• colcfg.pm (modified) (2 diffs)
• doc.pm (modified) (1 diff)
• inexport.pm (modified) (2 diffs)
• plugins/ArchivesInfPlugin.pm (modified) (3 diffs)
• plugins/ReadXMLFile.pm (modified) (1 diff)
• plugouts/BasePlugout.pm (modified) (3 diffs)

gsdl/trunk/perllib/colcfg.pm

@@ -80 +80 @@
                     q/archivedir|cachedir|builddir|removeold|/ .
                     q/textcompress|buildtype|no_text|keepold|gzip|/ .
-                    q/verbosity|remove_empty_classifications|OIDtype|/ .
+                    q/verbosity|remove_empty_classifications|OIDtype|OIDmetadata|/ .
                     q/groupsize|maxdocs|debug|mode|saveas|/ .
                     q/sortmeta|removesuffix|removeprefix|create_images|/ .
@@ -114 +114 @@
                  q/archivedir|cachedir|builddir|removeold|/ .
                  q/textcompress|buildtype|no_text|keepold|gzip|/ .
-                 q/verbosity|remove_empty_classifications|OIDtype|/.
+                 q/verbosity|remove_empty_classifications|OIDtype|OIDmetadata|/.
                  q/groupsize|maxdocs|debug|mode|saveas|/ .
                  q/sortmeta|removesuffix|removeprefix|create_images|/ .

gsdl/trunk/perllib/doc.pm

@@ -45 +45 @@
 # classification
 
-my $OIDcount = 0;
+our $OIDcount = 0;
 
 sub new {

gsdl/trunk/perllib/inexport.pm

@@ -32 +32 @@
 use util;
 use GDBMUtils;
+
+
+sub prime_doc_oid_count
+{
+    my ($archivedir) = @_;
+    my $oid_count_filename = &util::filename_cat ($archivedir, "OIDcount");
+
+    if (-e $oid_count_filename) {
+    if (open(OIDIN,"<$oid_count_filename")) {
+        my $OIDcount = <OIDIN>;
+        chomp $OIDcount;        
+        close(OIDIN);
+
+        $doc::OIDcount = $OIDcount;     
+    }
+    else {
+        
+        print STDERR "Warning: unable to read document OID count from $oid_count_filename\n";
+        print STDERR "Setting value to 0\n";
+    }
+    }
+    
+}
+
+sub store_doc_oid_count
+{
+    # Use the file "OIDcount" in the archives directory to record
+    # what value doc.pm got up to
+
+    my ($archivedir) = @_;
+    my $oid_count_filename = &util::filename_cat ($archivedir, "OIDcount");
+
+
+    if (open(OIDOUT,">$oid_count_filename")) {
+    print OIDOUT $doc::OIDcount, "\n";
+        
+    close(OIDOUT);
+    }
+    else {
+    print STDERR "Warning: unable to store document OID count\n";
+    }
+}
+
+
 
 sub new_vs_old_import_diff
@@ -160 +204 @@
     
         if ($verbosity>1) {
-            print STDERR "$oid marked to be deleted from index on next buildcol.pl\n";
+            print STDERR "$oid ($file) marked to be deleted from index on next buildcol.pl\n";
         }
         $archive_info->set_status_info($oid,"D");

gsdl/trunk/perllib/plugins/ArchivesInfPlugin.pm

@@ -32 +32 @@
 
 use util;
+use doc;
 use PrintInfo;
 use plugin;
@@ -173 +174 @@
 }
 
-# we don't do any file blocking
 sub file_block_read {
 
     my $self = shift (@_);  
     my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $gli) = @_;
+
+    if ($file eq "OIDcount") {
+    my ($filename_full_path, $filename_no_path) 
+        = &util::get_full_filenames($base_dir, $file);
+    $block_hash->{'file_blocks'}->{$filename_full_path} = 1;
+    return 1;
+    }
+
+    # otherwise, we don't do any file blocking
 
     return undef;
@@ -287 +296 @@
     }
 
+
     # wasn't an archives directory, someone else will have to process it
     return undef;

gsdl/trunk/perllib/plugins/ReadXMLFile.pm

@@ -293 +293 @@
 }
 
+
 # Called at the beginning of the XML document.
 sub xml_start_document {

gsdl/trunk/perllib/plugouts/BasePlugout.pm

@@ -669 +669 @@
               'assoc-files' => [] };
     
-    my @reverse_lookups = ($source_filename);
+    my %reverse_lookups = ( $source_filename => "1" );
 
     foreach my $assoc_file_rec (@{$doc_obj->get_assoc_files()}) {
@@ -687 +687 @@
         }
 
-        push(@reverse_lookups,$real_filename);
+        $reverse_lookups{$real_filename} = 1;
 
         push(@{$oid_files->{'assoc-files'}},$full_afile);
@@ -715 +715 @@
     chomp($doc_db_text); # remove trailing \n
 
-    ## print STDERR "*** To set in db: \n\t$doc_db\n\t$oid\n\t$doc_db_text\n";
+    ##print STDERR "*** To set in db: \n\t$doc_db\n\t$oid\n\t$doc_db_text\n";
     &GDBMUtils::gdbmDatabaseSet($doc_db,$oid,$doc_db_text);
 
-    foreach my $rl (@reverse_lookups) {
+    foreach my $rl (keys %reverse_lookups) {
     &GDBMUtils::gdbmDatabaseAppend($src_db,$rl,"<oid>$oid\n");
     }