Changeset 29257


Ignore:
Timestamp:
09/10/14 10:19:19 (6 years ago)
Author:
jmt12
Message:

Allow for collection configuration to be passed down to parallel import proc (so we can get infodbtype), and support infodbtypes that require post-import merging(such as TDBCluster

File:
1 edited

Legend:

Unmodified
Added
Removed
  • gs2-extensions/parallel-building/trunk/src/perllib/parallelbuildinginexport.pm

    r28779 r29257  
    154154
    155155
     156## @function
     157#
     158sub set_collection_options
     159{
     160  my $self = shift(@_);
     161  my ($collectcfg) = @_;
     162  $self->SUPER::set_collection_options($collectcfg);
     163  $self->{'collectcfg'} = $collectcfg;
     164}
     165## set_collection_options() ##
     166
     167
    156168## @function deinit()
    157169
     
    202214    print STDERR " ***** CLUSTER MODE *****\n";
    203215    $mpi_flags .= '-machinefile "' . $mpi_conf_path . '" ';
    204     #$mpi_flags .= '-nolocal ';
     216    $mpi_flags .= '-nolocal ';
    205217  }
    206218  else
     
    209221  }
    210222  $mpi_flags .= ' --show-progress --timestamp-output --verbose';
     223  # fix for mpi binding to incorrect interface device (seems to have an
     224  # unhealthy obsession with virbr0)
     225  $mpi_flags .= ' --mca btl tcp,sm,self --mca btl_tcp_if_include eth0 ';
    211226
    212227  # invoke the farmer to start processing the files
     
    282297    $self->SUPER::perform_process_files($manifest, $pluginfo, $importdir, $file_to_import, $block_hash, $metadata, $processor, $maxdocs);
    283298  }
     299
     300  # the individual parts into one single database
     301  my $infodb_type = $self->{'collectcfg'}->{'infodbtype'};
     302  my $mpi_conf_path = &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, 'mpi.conf');
     303  if ( -f $mpi_conf_path && &dbutil::supportsMerge($infodb_type) )
     304  {
     305    print STDERR ' * Merging ' . $infodb_type . ' databases... ';
     306    my @databases = ('archiveinf-src','archiveinf-doc');
     307    foreach my $database (@databases)
     308    {
     309      # generate the path to the target database without any hostname suffix
     310      my $archive_db_path = &dbutil::get_infodb_file_path($infodb_type, $database, $self->{'archivedir'}, 0, '');
     311      #rint STDERR " - merging to: " . $archive_db_path . "\n";
     312      open(MPIHOSTS, '<:utf8', $mpi_conf_path) or die("Failed to open mpi.conf for reading");
     313      my $line;
     314      while ($line = <MPIHOSTS>)
     315      {
     316        if ($line =~ /^([a-z0-9\-]+)/i)
     317        {
     318          my $hostname = $1;
     319          ###rint STDERR " - searching for database for: " . $hostname . "\n";
     320          my $mergable_db_path = &dbutil::get_infodb_file_path($infodb_type, $database, $self->{'archivedir'}, 0, $hostname);
     321          ###rint STDERR "[debug] " . $mergable_db_path . "\n";
     322          if (-f $mergable_db_path)
     323          {
     324            ###rint STDERR " - merge $mergable_db_path > $archive_db_path\n";
     325            &dbutil::merge_databases($infodb_type, $mergable_db_path, $archive_db_path);
     326          }
     327        }
     328      }
     329      close(MPIHOSTS);
     330    }
     331    print "Done!\n";
     332  }
     333
    284334}
    285335## perform_process_files() ##
Note: See TracChangeset for help on using the changeset viewer.