Changeset 24839 for gs2-extensions/parallel-building/trunk/src

Timestamp:

2011-12-01T12:24:10+13:00 (12 years ago)

Author:

jmt12

Message:

If the filelist is already present in tmp don't regenerate (just for testing) and make the mpi launch in a pipe so we can continually trap output (rather than waiting until a system() call had completed

File:

• gs2-extensions/parallel-building/trunk/src/perllib/ParallelInexport.pm (modified) (2 diffs)

gs2-extensions/parallel-building/trunk/src/perllib/ParallelInexport.pm

@@ -41 +41 @@
    }
 
+   # create the list of files to import
    my $tmp_filelist = &util::filename_cat($tmp_dir_path, "filelist.txt");
-
-   # create the list of files to import
-   open (my $filelist, ">$tmp_filelist");
-   foreach my $filename (sort keys %{$block_hash->{'all_files'}})
+   # - if the file is already there (which is should be during testing) then
+   #   don't regenerate. This is especially important for imports of 1 million
+   #   documents as just the directory scan can take several hours.
+   if (!-f $tmp_filelist)
    {
+     open (my $filelist, ">$tmp_filelist");
+     foreach my $filename (sort keys %{$block_hash->{'all_files'}})
+     {
        my $full_filename = &util::filename_cat($importdir,$filename);
        if ((! exists $block_hash->{'file_blocks'}->{$full_filename}) 
        && ($filename !~ m/metadata\.xml$/))
        {
-       print $filelist "$filename\n";
+         print $filelist "$filename\n";
        }
+     }
+     close ($filelist); 
    }
-   close ($filelist); 
 
    # invoke the farmer to start processing the files
@@ -60 +65 @@
    my $gsdlhome = $ENV{'GSDLHOME'};
    my $farmer_exe = 'mpiimport'; # will be on PATH
-   my $mpi_cmd = "mpirun -np $jobs $farmer_exe $tmp_filelist $epoch $gsdlhome $collection $site";
-   system ($mpi_cmd);
+   my $mpi_cmd = "mpirun -n $jobs $farmer_exe $tmp_filelist $epoch $gsdlhome $collection $site";
+#   my $mpi_cmd = "mpirun --show-progress --timestamp-output --verbose --report-bindings --tag-output -n $jobs $farmer_exe $tmp_filelist $epoch $gsdlhome $collection $site";
+   print STDERR "MPI Command: \"" . $mpi_cmd . "\"\n";
+#   system ($mpi_cmd);
+   open(MPI, $mpi_cmd . " |") or die("Couldn't Execute MPI");
+   while ( defined( my $line = <MPI> )  )
+   {
+     chomp($line);
+     print "$line\n";
+   }
+   close(MPI);
 }