Changeset 29257

Timestamp:

2014-09-10T10:19:19+12:00 (10 years ago)

Author:

jmt12

Message:

Allow for collection configuration to be passed down to parallel import proc (so we can get infodbtype), and support infodbtypes that require post-import merging(such as TDBCluster

File:

• gs2-extensions/parallel-building/trunk/src/perllib/parallelbuildinginexport.pm (modified) (4 diffs)

gs2-extensions/parallel-building/trunk/src/perllib/parallelbuildinginexport.pm

@@ -154 +154 @@
 
 
+## @function
+#
+sub set_collection_options
+{
+  my $self = shift(@_);
+  my ($collectcfg) = @_;
+  $self->SUPER::set_collection_options($collectcfg);
+  $self->{'collectcfg'} = $collectcfg;
+}
+## set_collection_options() ##
+
+
 ## @function deinit()
 
@@ -202 +214 @@
     print STDERR " ***** CLUSTER MODE *****\n";
     $mpi_flags .= '-machinefile "' . $mpi_conf_path . '" ';
-    #$mpi_flags .= '-nolocal ';
+    $mpi_flags .= '-nolocal ';
   }
   else
@@ -209 +221 @@
   }
   $mpi_flags .= ' --show-progress --timestamp-output --verbose';
+  # fix for mpi binding to incorrect interface device (seems to have an
+  # unhealthy obsession with virbr0)
+  $mpi_flags .= ' --mca btl tcp,sm,self --mca btl_tcp_if_include eth0 ';
 
   # invoke the farmer to start processing the files
@@ -282 +297 @@
     $self->SUPER::perform_process_files($manifest, $pluginfo, $importdir, $file_to_import, $block_hash, $metadata, $processor, $maxdocs);
   }
+
+  # the individual parts into one single database
+  my $infodb_type = $self->{'collectcfg'}->{'infodbtype'};
+  my $mpi_conf_path = &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, 'mpi.conf');
+  if ( -f $mpi_conf_path && &dbutil::supportsMerge($infodb_type) )
+  {
+    print STDERR ' * Merging ' . $infodb_type . ' databases... ';
+    my @databases = ('archiveinf-src','archiveinf-doc');
+    foreach my $database (@databases)
+    {
+      # generate the path to the target database without any hostname suffix
+      my $archive_db_path = &dbutil::get_infodb_file_path($infodb_type, $database, $self->{'archivedir'}, 0, '');
+      #rint STDERR " - merging to: " . $archive_db_path . "\n";
+      open(MPIHOSTS, '<:utf8', $mpi_conf_path) or die("Failed to open mpi.conf for reading");
+      my $line;
+      while ($line = <MPIHOSTS>)
+      {
+        if ($line =~ /^([a-z0-9\-]+)/i)
+        {
+          my $hostname = $1;
+          ###rint STDERR " - searching for database for: " . $hostname . "\n";
+          my $mergable_db_path = &dbutil::get_infodb_file_path($infodb_type, $database, $self->{'archivedir'}, 0, $hostname);
+          ###rint STDERR "[debug] " . $mergable_db_path . "\n";
+          if (-f $mergable_db_path)
+          {
+            ###rint STDERR " - merge $mergable_db_path > $archive_db_path\n";
+            &dbutil::merge_databases($infodb_type, $mergable_db_path, $archive_db_path);
+          }
+        }
+      }
+      close(MPIHOSTS);
+    }
+    print "Done!\n";
+  }
+
 }
 ## perform_process_files() ##