Changeset 22037

Timestamp:

2010-05-05T14:53:53+12:00 (13 years ago)

Author:

davidb

Message:

Manifest file processing upgraded to support OIDs. The code then uses the archiveinf-doc database to look up which files it uses.

Location:

main/trunk/greenstone2

Files:

• bin/script/export.pl (modified) (1 diff)
• bin/script/import.pl (modified) (4 diffs)
• perllib/manifest.pm (modified) (8 diffs)
• perllib/plugin.pm (modified) (1 diff)

main/trunk/greenstone2/bin/script/export.pl

@@ -488 +488 @@
     print STDERR "<export>\n" if $gli;
     
-    my $manifest_lookup = new manifest();
+    my $manifest_lookup = new manifest($collectcfg->{'infodbtype'},$archivedir);
     if ($manifest ne "") {  
     my $manifest_filename = $manifest;

main/trunk/greenstone2/bin/script/import.pl

@@ -491 +491 @@
     print STDERR "<Import>\n" if $gli;
     
-    my $manifest_lookup = new manifest();
+    my $manifest_lookup = new manifest($collectcfg->{'infodbtype'},$archivedir);
     if ($manifest ne "") {  
     my $manifest_filename = $manifest;
@@ -617 +617 @@
         $gsdl_tmp_area = &util::filename_to_regex($gsdl_tmp_area);
         $collect_tmp_area = &util::filename_to_regex($collect_tmp_area);
-        
-        
+                  
         foreach my $df (@deleted_files) {
             next if ($df =~ m/^$gsdl_tmp_area/);
@@ -624 +623 @@
             
             push(@filtered_deleted_files,$df);
-        }
+        }       
         
-        
+
         @deleted_files = @filtered_deleted_files;
         
@@ -661 +660 @@
     else
     {
-    # process any files marked for importing
-    foreach my $file (keys %{$manifest_lookup->{'import'}}) {
-        &plugin::read ($pluginfo, $importdir, $file, {}, {}, $processor, $maxdocs, 0, $gli);
-    }
+    # 
+    # 1. Process delete files first
+    # 
 
     my @deleted_files = keys %{$manifest_lookup->{'delete'}};
     my @full_deleted_files = ();
 
-    foreach my $df (@deleted_files) {
-        my $full_df = &util::filename_cat($importdir,$df);
+    # ensure all filenames are absolute
+    foreach my $df (@deleted_files) {       
+        my $full_df =
+        (&util::filename_is_absolute($df)) 
+        ? $df
+        : &util::filename_cat($importdir,$df);
+
         push(@full_deleted_files,$full_df);
     }
-
+    
+    &plugin::remove_some($pluginfo, $collectcfg->{'infodbtype'}, $archivedir, \@full_deleted_files);
     &inexport::mark_docs_for_deletion($archive_info,{},
                       \@full_deleted_files,
                       $archivedir, $verbosity, "delete");
+
+
+    # 
+    # 2. Now files for reindexing
+    # 
+
+    my @reindex_files = keys %{$manifest_lookup->{'reindex'}};
+    my @full_reindex_files = ();
+
+    # ensure all filenames are absolute
+    foreach my $rf (@reindex_files) {       
+        my $full_rf =
+        (&util::filename_is_absolute($rf)) 
+        ? $rf
+        : &util::filename_cat($importdir,$rf);
+
+        push(@full_reindex_files,$full_rf);
+    }
+    
+    &plugin::remove_some($pluginfo, $collectcfg->{'infodbtype'}, $archivedir, \@full_reindex_files);
+    &inexport::mark_docs_for_deletion($archive_info,{},\@full_reindex_files, $archivedir,$verbosity, "reindex");
+
+    # And now ensure the new version of the file processed by appropriate
+    # plugin
+    foreach my $full_rf (@full_reindex_files) {
+        &plugin::read ($pluginfo, "", $full_rf, {}, {}, $processor, $maxdocs, 0, $gli);
+    }
+
+
+    # 
+    # 3. Now finally any new files
+    # 
+
+    foreach my $file (keys %{$manifest_lookup->{'index'}}) {
+        &plugin::read ($pluginfo, $importdir, $file, {}, {}, $processor, $maxdocs, 0, $gli);
+    }
+
+
     }
 

main/trunk/greenstone2/perllib/manifest.pm

@@ -1 +1 @@
 package manifest;
 
-use XMLParser;
 use strict;
 no strict 'refs'; # allow filehandles to be variables and viceversa
 
+use XMLParser;
+use dbutil;
+
 our $self;
 
 sub new {
     my ($class) = shift (@_);
+    my ($infodbtype,$archivedir) = @_;
 
     $self = {} ;
@@ -15 +18 @@
     $self->{'reindex'} = {};
     $self->{'delete'} = {};
+
+    my $arcinfo_doc_filename 
+    = &dbutil::get_infodb_file_path($infodbtype, "archiveinf-doc", $archivedir);
+
+    if (-e $arcinfo_doc_filename) {
+    # Only store the infodb-doc filename if it exists
+    # If it doesn't exist then this means the collection has not been
+    #   built yet (or else the archives folder has been deleted).
+    #   Either way we have no way to look up which files
+    #   are associated with an OID.  If we we encounter an OID
+    #   tag later on, we will use the fact that this field is
+    #   not defined to issue a warning
+
+    $self->{'_arcinfo-doc-filename'} = $arcinfo_doc_filename;
+    $self->{'_infodbtype'} = $infodbtype;
+    }
 
     return bless $self, $class;
@@ -89 +108 @@
     my ($expat, $element) = @_;
 
-    if ($element eq "Filename")
-    {
-    $self->{'filename'} = "";
+    if (($element eq "Filename") || ($element eq "OID"))
+    {
+    $self->{'item-val'} = "";
     }
     elsif ($element eq "Manifest") {
@@ -99 +118 @@
     if (defined($self->{'file-type'}))
     {
-        print STDERR "Warning: Malformed XML manifest ($element nested inside " . $self->{'file-type'} . ")\n";
-    }
-
-    $self->{'file-type'} = $element;
+        print STDERR "Warning: Malformed XML manifest\n";
+        print STDERR "         Unrecognized element $element nested inside " . $self->{'file-type'} . ".\n";
+    }
+    else {
+        my $filetype = lc($element);
+        $self->{'file-type'} = $filetype;
+        if (!defined $self->{$filetype}) {
+        print STDERR "Warning: <$element> is not one of the registered tags for manifest format.\n";
+        }
+    }
+
     }
 }
@@ -114 +140 @@
     if ($element eq "Filename")
     {
-    $self->{lc($self->{'file-type'})}->{$self->{'filename'}} = 1;
-    $self->{'filename'} = undef;
+    my $filetype = $self->{'file-type'};
+    my $filename  = $self->{'item-val'};
+
+    $self->{$filetype}->{$filename} = 1;
+    $self->{'item-val'} = undef;
+    }
+    elsif ($element eq "OID") {
+    # look up src and assoc filenames used by this doc oid
+
+    my $filetype = $self->{'file-type'};
+    my $oid  = $self->{'item-val'};
+
+    if (defined $self->{'_infodbtype'}) {
+        
+
+        my $infodbtype = $self->{'_infodbtype'};
+        my $arcinfo_doc_filename = $self->{'_arcinfo-doc-filename'};
+        
+        my $doc_rec_string = &dbutil::read_infodb_entry($infodbtype, $arcinfo_doc_filename, $oid);
+
+        my $doc_rec = &dbutil::convert_infodb_string_to_hash($doc_rec_string);
+        
+        my $doc_source_file = $doc_rec->{'src-file'}->[0];
+        my $assoc_files = $doc_rec->{'assoc-file'};
+        my @all_files = ($doc_source_file,@$assoc_files);
+        
+        foreach my $filename (@all_files) {
+        
+        if (!&util::filename_is_absolute($filename)) {
+            $filename = &util::filename_cat($ENV{'GSDLCOLLECTDIR'},$filename);
+        }
+
+        $self->{$filetype}->{$filename} = 1;
+        }
+    }
+    else {
+        print STDERR "Warning: No archiveinf-doc database in archives directory.\n";
+        print STDERR "         Unable to look up source files that constitute document $oid.\n";
+    }
+
+    $self->{'item-val'} = undef;
     }
     else
@@ -129 +194 @@
     my ($expat) = @_;
 
-    if (defined $self->{'filename'}) {
+    if (defined $self->{'item-val'}) {
     my $text = $_;
     chomp($text);
@@ -136 +201 @@
     $text =~ s/\s+$//;  
     
-    $self->{'filename'} .= $text if ($text !~ m/^\s*$/);
+    $self->{'item-val'} .= $text if ($text !~ m/^\s*$/);
     }
 }
@@ -152 +217 @@
     my ($expat) = @_;
 
+    if (defined $self->{'import'}) {
+    print STDERR "Warning: <Import> tag is deprecated.\n";
+    print STDERR "         Processing data as if it were tagged as <Index>\n";
+    $self->{'index'} = $self->{'import'};
+    }
+
 }
 

main/trunk/greenstone2/perllib/plugin.pm

@@ -222 +222 @@
 sub remove_some {
     my ($pluginfo, $infodbtype, $archivedir, $deleted_files) = @_;
-    print STDERR "in remove some\n";
     return if (scalar(@$deleted_files)==0);
     my $arcinfo_src_filename = &dbutil::get_infodb_file_path($infodbtype, "archiveinf-src", $archivedir);