Changeset 24686 for gs2-extensions/parallel-building/trunk

Timestamp:

2011-09-28T13:22:52+13:00 (13 years ago)

Author:

jmt12

Message:

Several changes to ensure parallel importing plays nicely with manifest files and (simple) accompanying metadata.xml files. Also made it so initial calls to get_infodb_file_path run the GDBMServer (if necessary) to ensure it persists through parallel importing

File:

• gs2-extensions/parallel-building/trunk/src/perllib/inexport.pm (modified) (5 diffs)

gs2-extensions/parallel-building/trunk/src/perllib/inexport.pm

@@ -351 +351 @@
     $self->{'incremental'}      = $incremental;
     $self->{'incremental_mode'} = $incremental_mode;
+
+    # Since this wasted my morning, let's at least warn a user that manifest
+    # files now *only* work if keepold is set. [jmt12]
+    if ($self->{'manifest'} && !$self->{'keepold'})
+    {
+      print STDERR "Warning: -manifest flag should not be specified without also setting -keepold or -incremental. Ignoring.\n";
+    }
 }
 
@@ -470 +477 @@
     &util::rename_ldb_or_bdb_file(&util::filename_cat($archivedir, "archiveinf-src"));
 
-    my $arcinfo_doc_filename = &dbutil::get_infodb_file_path($collectcfg->{'infodbtype'}, "archiveinf-doc", $archivedir);
-    my $arcinfo_src_filename = &dbutil::get_infodb_file_path($collectcfg->{'infodbtype'}, "archiveinf-src", $archivedir);
-                            
+    # Warning! Black magic follows. When the following functions are called on
+    # the GDBMServer class they will actually prompt the running of the Server
+    # and attach themselves as a listener (even though they don't do anything)
+    # This is done so that, in parallel importing, the server will persist
+    # until the top level import.pl (which will be the first this that calls
+    # this function) completes.
+    my $arcinfo_doc_filename = &dbutil::get_infodb_file_path($collectcfg->{'infodbtype'}, "archiveinf-doc", $archivedir, 1);
+    my $arcinfo_src_filename = &dbutil::get_infodb_file_path($collectcfg->{'infodbtype'}, "archiveinf-src", $archivedir, 1);
+
     my $archive_info = new arcinfo ($collectcfg->{'infodbtype'});
     $archive_info->load_info ($arcinfo_doc_filename);
@@ -558 +571 @@
     $block_hash->{'new_files'} = {};
     $block_hash->{'reindex_files'} = {};
+    # All of these are set somewhere else, so it's kinda nice to define them
+    # here. [jmt12]
+    $block_hash->{'all_files'} = {};
+    $block_hash->{'deleted_files'} = {};
+    $block_hash->{'file_blocks'} = {};
+    $block_hash->{'metadata_files'} = {};
+    $block_hash->{'shared_fileroot'} = '';
+    # My new flag so we can tell we had a manifest way down in the plugins
+    # [jmt12]
+    $block_hash->{'manifest'} = 'false';
     my $metadata = {};
     
     # global blocking pass may set up some metadata
-    &plugin::file_block_read($pluginfo, $importdir, "", $block_hash, $metadata, $gli);
+    # - when we have a manifest file we don't do this -unless- the collection
+    #   configuration indicates this collection contains complex (inherited)
+    #   metadata. [jmt12]
+    if ($manifest eq '' || (defined $collectcfg->{'complexmeta'} && $collectcfg->{'complexmeta'} eq 'true'))
+    {
+      &plugin::file_block_read($pluginfo, $importdir, "", $block_hash, $metadata, $gli);
+    }
+    else
+    {
+      print "Skipping global file scan due to manifest and complexmeta configuration\n";
+    }
     
     if ($manifest ne "") {
+
+      $block_hash->{'manifest'} = 'true';
+
     # 
     # 1. Process delete files first
@@ -642 +678 @@
 
     my $arcinfo_src_filename = &dbutil::get_infodb_file_path($collectcfg->{'infodbtype'}, "archiveinf-src", $archivedir);
+      # need to check this file exists before trying to read it. [jmt12]
+      if (-e $arcinfo_src_filename)
+      {
     my $arcinfodb_map = {};
     &dbutil::read_infodb_file($collectcfg->{'infodbtype'}, $arcinfo_src_filename, $arcinfodb_map);
@@ -653 +692 @@
         }
     }
-
     undef $arcinfodb_map;
+      }
+      # no existing files - so we can just add all the ones that need adding.
+      # [jmt12]
+      else
+      {
+        foreach my $f (@full_new_files)
+        {
+          $block_hash->{'new_files'}->{$f} = 1;
+        }
+      }
+
+      # If we are not using complex inherited metadata (and thus have skipped
+      # the global file scan) we need to at least scan the directory of the
+      # files being indexed/reindexed. [jmt12]
+      if ($collectcfg->{'complexmeta'} ne 'true')
+      {
+        my @all_files_to_import = (keys %{$block_hash->{'reindex_files'}}, keys %{$block_hash->{'new_files'}});
+        foreach my $file_to_import (@all_files_to_import)
+        {
+          my $dir_to_import = $file_to_import;
+          $dir_to_import =~ s/[^\\\/]*$//;
+          # - one day we may need to manually scan this directory for child
+          #   directories and somehow explicitly block them from being
+          #   recursed.
+          if (-d $dir_to_import)
+          {
+            &plugin::file_block_read($pluginfo, $dir_to_import, '', $block_hash, $metadata, $gli);
+          }
+        }
+      }
     }
     else {