Changeset 29257

Show
Ignore:
Timestamp:
10.09.2014 10:19:19 (5 years ago)
Author:
jmt12
Message:

Allow for collection configuration to be passed down to parallel import proc (so we can get infodbtype), and support infodbtypes that require post-import merging(such as TDBCluster

Files:
1 modified

Legend:

Unmodified
Added
Removed
  • gs2-extensions/parallel-building/trunk/src/perllib/parallelbuildinginexport.pm

    r28779 r29257  
    154154 
    155155 
     156## @function 
     157# 
     158sub set_collection_options 
     159{ 
     160  my $self = shift(@_); 
     161  my ($collectcfg) = @_; 
     162  $self->SUPER::set_collection_options($collectcfg); 
     163  $self->{'collectcfg'} = $collectcfg; 
     164} 
     165## set_collection_options() ## 
     166 
     167 
    156168## @function deinit() 
    157169 
     
    202214    print STDERR " ***** CLUSTER MODE *****\n"; 
    203215    $mpi_flags .= '-machinefile "' . $mpi_conf_path . '" '; 
    204     #$mpi_flags .= '-nolocal '; 
     216    $mpi_flags .= '-nolocal '; 
    205217  } 
    206218  else 
     
    209221  } 
    210222  $mpi_flags .= ' --show-progress --timestamp-output --verbose'; 
     223  # fix for mpi binding to incorrect interface device (seems to have an 
     224  # unhealthy obsession with virbr0) 
     225  $mpi_flags .= ' --mca btl tcp,sm,self --mca btl_tcp_if_include eth0 '; 
    211226 
    212227  # invoke the farmer to start processing the files 
     
    282297    $self->SUPER::perform_process_files($manifest, $pluginfo, $importdir, $file_to_import, $block_hash, $metadata, $processor, $maxdocs); 
    283298  } 
     299 
     300  # the individual parts into one single database 
     301  my $infodb_type = $self->{'collectcfg'}->{'infodbtype'}; 
     302  my $mpi_conf_path = &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, 'mpi.conf'); 
     303  if ( -f $mpi_conf_path && &dbutil::supportsMerge($infodb_type) ) 
     304  { 
     305    print STDERR ' * Merging ' . $infodb_type . ' databases... '; 
     306    my @databases = ('archiveinf-src','archiveinf-doc'); 
     307    foreach my $database (@databases) 
     308    { 
     309      # generate the path to the target database without any hostname suffix 
     310      my $archive_db_path = &dbutil::get_infodb_file_path($infodb_type, $database, $self->{'archivedir'}, 0, ''); 
     311      #rint STDERR " - merging to: " . $archive_db_path . "\n"; 
     312      open(MPIHOSTS, '<:utf8', $mpi_conf_path) or die("Failed to open mpi.conf for reading"); 
     313      my $line; 
     314      while ($line = <MPIHOSTS>) 
     315      { 
     316        if ($line =~ /^([a-z0-9\-]+)/i) 
     317        { 
     318          my $hostname = $1; 
     319          ###rint STDERR " - searching for database for: " . $hostname . "\n"; 
     320          my $mergable_db_path = &dbutil::get_infodb_file_path($infodb_type, $database, $self->{'archivedir'}, 0, $hostname); 
     321          ###rint STDERR "[debug] " . $mergable_db_path . "\n"; 
     322          if (-f $mergable_db_path) 
     323          { 
     324            ###rint STDERR " - merge $mergable_db_path > $archive_db_path\n"; 
     325            &dbutil::merge_databases($infodb_type, $mergable_db_path, $archive_db_path); 
     326          } 
     327        } 
     328      } 
     329      close(MPIHOSTS); 
     330    } 
     331    print "Done!\n"; 
     332  } 
     333 
    284334} 
    285335## perform_process_files() ##