Changeset 23053

Timestamp:

2010-10-06T15:39:33+13:00 (14 years ago)

Author:

kjdon

Message:

reworking of manifest stuff. Now, instead of calling plugin::read on each file in the reindex/new_files lists, we add all the filenames into the standard block_hash. This is the equivalent of what the code for incremental build does, except that instead of inspecting the import folder, we just use what is listed in manifest file. If a directory is listed, then we look through the directory and add each file in there to the appropriate list. Once block_hash is set up, then we tell the plugins we are incremental, and just process the import folder as per usual. Only those docs in new_files and reindex lists in block_hash will be processed. But this gets a full metadata_read done, which we need in case there is metadata specified up a level for the files that are changed/new.

File:

• main/trunk/greenstone2/perllib/inexport.pm (modified) (6 diffs)

main/trunk/greenstone2/perllib/inexport.pm

@@ -375 +375 @@
     
     my $manifest_lookup = new manifest($collectcfg->{'infodbtype'},$archivedir);
-    if ($self->{'manifest'} ne "") {    
+    if ($self->{'manifest'} ne "") {
+    print STDERR "parsing manifest\n";
     my $manifest_filename = $self->{'manifest'};
 
@@ -396 +397 @@
     }
 
+    my $plugin_incr_mode = $incremental_mode;
+    if ($manifest ne "") {
+    # if we have a manifest file, then we pretend we are fully incremental for plugins
+    $plugin_incr_mode = "all";
+    }
     #some global options for the plugins
     my @global_opts = ();
 
-    my $pluginfo = &plugin::load_plugins ($plugins, $verbosity, $out, $faillog, \@global_opts, $incremental_mode);
+    my $pluginfo = &plugin::load_plugins ($plugins, $verbosity, $out, $faillog, \@global_opts, $plugin_incr_mode);
     if (scalar(@$pluginfo) == 0) {
     &gsprintf($out, "{import.no_plugins_loaded}\n");
@@ -424 +430 @@
 
     # read the archive information file
-##  my $arcinfo_doc_filename = &util::filename_cat ($archivedir, "archives.inf");
 
     # BACKWARDS COMPATIBILITY: Just in case there are old .ldb/.bdb files (won't do anything for other infodbtypes)
@@ -509 +514 @@
     &plugin::remove_all($pluginfo, $importdir, $processor, $maxdocs, $gli);
     }
-    if ($manifest eq "") {
-    # process the import directory
-    my $block_hash = {};
-    my $metadata = {};
-    # gobal blocking pass may set up some metadata
-    &plugin::file_block_read($pluginfo, $importdir, "", $block_hash, $metadata, $gli);
+
+    # process the import directory
+    my $block_hash = {};
+    $block_hash->{'new_files'} = {};
+    $block_hash->{'reindex_files'} = {};
+    my $metadata = {};
+    
+    # gobal blocking pass may set up some metadata
+    &plugin::file_block_read($pluginfo, $importdir, "", $block_hash, $metadata, $gli);
+    
+    if ($manifest ne "") {
+    # 
+    # 1. Process delete files first
+    # 
+
+    my @deleted_files = keys %{$manifest_lookup->{'delete'}};
+    my @full_deleted_files = ();
+
+    # ensure all filenames are absolute
+    foreach my $df (@deleted_files) {
+        #print STDERR "**delete file $df\n";
+        my $full_df =
+        (&util::filename_is_absolute($df)) 
+        ? $df
+        : &util::filename_cat($importdir,$df);
+
+        if (-d $full_df) {
+        &add_dir_contents_to_list($full_df, \@full_deleted_files);
+        } else {
+        push(@full_deleted_files,$full_df);
+        }
+    }
+    
+    &plugin::remove_some($pluginfo, $collectcfg->{'infodbtype'}, $archivedir, \@full_deleted_files);
+    mark_docs_for_deletion($archive_info,{},
+                   \@full_deleted_files,
+                   $archivedir, $verbosity, "delete");
+
+
+    # 
+    # 2. Now files for reindexing
+    # 
+
+    my @reindex_files = keys %{$manifest_lookup->{'reindex'}};
+    my @full_reindex_files = ();
+
+    # ensure all filenames are absolute
+    foreach my $rf (@reindex_files) {       
+        my $full_rf =
+        (&util::filename_is_absolute($rf)) 
+        ? $rf
+        : &util::filename_cat($importdir,$rf);
+
+        if (-d $full_rf) {
+        &add_dir_contents_to_list($full_rf, \@full_reindex_files);
+        } else {
+        push(@full_reindex_files,$full_rf);
+        }
+    }
+    
+    &plugin::remove_some($pluginfo, $collectcfg->{'infodbtype'}, $archivedir, \@full_reindex_files);
+    mark_docs_for_deletion($archive_info,{},\@full_reindex_files, $archivedir,$verbosity, "reindex");
+
+    # And now to ensure the new version of the file processed by 
+    # appropriate plugin, we need to add it to block_hash reindex list
+    foreach my $full_rf (@full_reindex_files) {
+        #print STDERR "***reindex file $full_rf\n";
+        $block_hash->{'reindex_files'}->{$full_rf} = 1;
+    }
+
+
+    # 
+    # 3. Now finally any new files - add to block_hash new_files list
+    # 
+
+    my @new_files = keys %{$manifest_lookup->{'index'}};
+    my @full_new_files = ();
+
+    foreach my $nf (@new_files) {
+        #print STDERR "***index file $nf\n";
+        # ensure filename is absolute
+        my $full_nf =
+        (&util::filename_is_absolute($nf)) 
+        ? $nf
+        : &util::filename_cat($importdir,$nf);
+
+        if (-d $full_nf) {
+        &add_dir_contents_to_list($full_nf, \@full_new_files);
+        } else {
+        push(@full_new_files,$full_nf);
+        }
+    }
+
+    foreach my $f (@full_new_files) {
+        $block_hash->{'new_files'}->{$f} = 1;
+    }
+    }
+    else {
+    # if incremental, we read through the import folder to see whats changed.
 
     if ($incremental || $incremental_mode eq "onlyadd") {
-
         prime_doc_oid_count($archivedir);
 
@@ -571 +668 @@
         }
                 
-        }
-        
-        # Play it safe, and run through the entire folder, only processing new or edited files
-        
-            if ((defined $jobs) && ($jobs > 1))
-            { 
-        # if jobs are set to >1, run in parallel using MPI helper
-        # [hs, 1 july 2010]
-        &ParallelInexport::farm_out_processes($jobs, $epoch, $importdir, $block_hash, 
-                              $self->{'collection'}, $self->{'site'}); 
-            }
-            else
-            {
-           &plugin::read ($pluginfo, $importdir, "", $block_hash, $metadata, $processor, $maxdocs, 0, $gli);
-            }
-    }
-    else {
-            if ((defined $jobs) && ($jobs > 1))
-            { 
-        # if jobs are set to >1, run in parallel using MPI helper
-        # [hs, 1 july 2010]
-        &ParallelInexport::farm_out_processes($jobs, $epoch, $importdir, $block_hash, 
-                              $self->{'collection'}, $self->{'site'}); 
-            }
-            else
-            {
-        &plugin::read ($pluginfo, $importdir, "", $block_hash, $metadata, $processor, $maxdocs, 0, $gli);
-            }   
-    }
-
+        }       
+    }
+    }
+    
+    # now, whichever mode we are in, we can process the entire import folder
+    if ((defined $jobs) && ($jobs > 1))
+    { 
+    # if jobs are set to >1, run in parallel using MPI helper
+    # [hs, 1 july 2010]
+    &ParallelInexport::farm_out_processes($jobs, $epoch, $importdir, $block_hash, 
+                          $self->{'collection'}, $self->{'site'}); 
     }
     else
     {
-    # 
-    # 1. Process delete files first
-    # 
-
-    my @deleted_files = keys %{$manifest_lookup->{'delete'}};
-    my @full_deleted_files = ();
-
-    # ensure all filenames are absolute
-    foreach my $df (@deleted_files) {       
-        my $full_df =
-        (&util::filename_is_absolute($df)) 
-        ? $df
-        : &util::filename_cat($importdir,$df);
-
-        push(@full_deleted_files,$full_df);
-    }
-    
-    &plugin::remove_some($pluginfo, $collectcfg->{'infodbtype'}, $archivedir, \@full_deleted_files);
-    mark_docs_for_deletion($archive_info,{},
-                   \@full_deleted_files,
-                   $archivedir, $verbosity, "delete");
-
-
-    # 
-    # 2. Now files for reindexing
-    # 
-
-    my @reindex_files = keys %{$manifest_lookup->{'reindex'}};
-    my @full_reindex_files = ();
-
-    # ensure all filenames are absolute
-    foreach my $rf (@reindex_files) {       
-        my $full_rf =
-        (&util::filename_is_absolute($rf)) 
-        ? $rf
-        : &util::filename_cat($importdir,$rf);
-
-        push(@full_reindex_files,$full_rf);
-    }
-    
-    &plugin::remove_some($pluginfo, $collectcfg->{'infodbtype'}, $archivedir, \@full_reindex_files);
-    mark_docs_for_deletion($archive_info,{},\@full_reindex_files, $archivedir,$verbosity, "reindex");
-
-    # And now ensure the new version of the file processed by appropriate
-    # plugin
-    foreach my $full_rf (@full_reindex_files) {
-        &plugin::read ($pluginfo, "", $full_rf, {}, {}, $processor, $maxdocs, 0, $gli);
-    }
-
-
-    # 
-    # 3. Now finally any new files
-    # 
-
-    foreach my $file (keys %{$manifest_lookup->{'index'}}) {
-        my $block_hash = {};
-        my $metadata = {};
-        &plugin::file_block_read($pluginfo, $importdir, $file, $block_hash, $metadata, $gli);
-        &plugin::read ($pluginfo, $importdir, $file, $block_hash, $metadata, $processor, $maxdocs, 0, $gli);
-    }
-
-
-    }
-
+    &plugin::read ($pluginfo, $importdir, "", $block_hash, $metadata, $processor, $maxdocs, 0, $gli);
+    }
+    
+    
     if ($saveas eq "FedoraMETS") {
     # create collection "doc obj" for Fedora that contains
@@ -1123 +1141 @@
 }
 
+sub add_dir_contents_to_list {
+
+    my ($dirname, $list) = @_;
+ 
+    # Recur over directory contents.
+    my (@dir, $subfile);
+    
+    # find all the files in the directory
+    if (!opendir (DIR, $dirname)) {
+    print STDERR "inexport: WARNING - couldn't read directory $dirname\n";
+    return -1; # error in processing
+    }
+    @dir = readdir (DIR);
+    closedir (DIR);
+    
+    for (my $i = 0; $i < scalar(@dir); $i++) {
+    my $subfile = $dir[$i];
+    next if ($subfile =~ m/^\.\.?$/);
+    my $full_file = &util::filename_cat($dirname, $subfile);
+    if (-d $full_file) {
+        &add_dir_contents_to_list($full_file, $list);
+    } else {
+        push (@$list, $full_file);
+    }
+    }
+    
+}