Changeset 26071

Timestamp:

2012-08-06T11:10:22+12:00 (12 years ago)

Author:

jmt12

Message:

Randomizing the order files are imported and restoring debugging flags to MPI call

File:

• gs2-extensions/parallel-building/trunk/src/perllib/ParallelInexport.pm (modified) (3 diffs)

gs2-extensions/parallel-building/trunk/src/perllib/ParallelInexport.pm

@@ -28 +28 @@
 use strict;
 
+# Randomize the order of files in the filelist
+use List::Util qw( shuffle );
 
 # index the files in parallel using MPI farmer to farm off multiple processes
@@ -42 +44 @@
 
    # create the list of files to import
+   my $overwrite = 1;
    my $tmp_filelist = &util::filename_cat($tmp_dir_path, "filelist.txt");
    # - if the file is already there (which is should be during testing) then
    #   don't regenerate. This is especially important for imports of 1 million
    #   documents as just the directory scan can take several hours.
-   if (!-f $tmp_filelist)
+   if ($overwrite || !-f $tmp_filelist)
    {
      open (my $filelist, ">$tmp_filelist");
-     foreach my $filename (sort keys %{$block_hash->{'all_files'}})
+     my @filenames = keys %{$block_hash->{'all_files'}};
+     @filenames = shuffle(@filenames);
+     foreach my $filename (@filenames)
      {
        my $full_filename = &util::filename_cat($importdir,$filename);
@@ -65 +70 @@
    my $gsdlhome = $ENV{'GSDLHOME'};
    my $farmer_exe = 'mpiimport'; # will be on PATH
-   my $mpi_cmd = "mpirun -n $jobs $farmer_exe $tmp_filelist $epoch $gsdlhome $collection $site";
-#   my $mpi_cmd = "mpirun --show-progress --timestamp-output --verbose --report-bindings --tag-output -n $jobs $farmer_exe $tmp_filelist $epoch $gsdlhome $collection $site";
+   #my $mpi_cmd = "mpirun -n $jobs $farmer_exe $tmp_filelist $epoch $gsdlhome $collection $site";
+   my $mpi_cmd = "mpirun --show-progress --timestamp-output --verbose --report-bindings --tag-output -n $jobs $farmer_exe $tmp_filelist $epoch $gsdlhome $collection $site";
    print STDERR "MPI Command: \"" . $mpi_cmd . "\"\n";
 #   system ($mpi_cmd);