Changeset 26998 for gs2-extensions

Timestamp:

2013-03-05T11:07:26+13:00 (11 years ago)

Author:

jmt12

Message:

Adding maxdocs variable, lots of debug comments, added some tests for directories rather than just deleting them (causing non-fatal errors if the directories weren't there) and added some extra flags to ensure MPI bound to correct interface (as Medusa has some phantom virtual interfaces that occasionally interfere)

File:

• gs2-extensions/parallel-building/trunk/src/bin/script/parallel_terrier_fileindex.pl (modified) (6 diffs)

gs2-extensions/parallel-building/trunk/src/bin/script/parallel_terrier_fileindex.pl

@@ -90 +90 @@
     print STDERR 'Error! ' . $message . "\n";
   }
-  print STDERR 'Usage: parallel_terrier_fileindexer.pl -workers <int> -terrier <path> -collection <path> -batchsize <int> [-debug]' . "\n\n";
+  print STDERR 'Usage: parallel_terrier_fileindexer.pl -terrier <path> -collection <path> -workers <num> -batchsize <num> [-maxfiles <num>] [-debug]' . "\n\n";
   print '[' . time() . ']Parallel FileIndexer Complete: ' . localtime() . "\n";
   exit(0);
@@ -109 +109 @@
   my $batch_size = 0;
   my $debug = 0;
+  my $max_files = 0;
   # - parse arguments
   my $argument;
@@ -137 +138 @@
     {
       $debug = 1;
+    }
+    elsif ('-maxfiles' eq $argument)
+    {
+      $i++;
+      $max_files = $ARGV[$i];
     }
     else
@@ -189 +195 @@
       unlink($old_path);
     }
-  }
-  my $index_path = &fileCat($var_path, 'index');
-  my $delete_command = 'rm -rf "' . $index_path . '"';
-  &debugPrint($debug, 'command: ' . $delete_command . "\n");
-  `$delete_command`;
-  my $assoc_path = &fileCat($terrier_home, 'share', 'images', 'assoc');
-  $delete_command  = 'rm -rf "' . $assoc_path . '"';
-  &debugPrint($debug, 'command: ' . $delete_command . "\n");
-  `$delete_command`;
+    my $index_path = &fileCat($var_path, 'index');
+    if (-d $index_path)
+    {
+      my $delete_command = 'rm -rf "' . $index_path . '"';
+      &debugPrint($debug, 'command: ' . $delete_command . "\n");
+      `$delete_command`;
+    }
+    my $assoc_path = &fileCat($terrier_home, 'share', 'images', 'assoc');
+    if (-d $assoc_path)
+    {
+      my $delete_command2  = 'rm -rf "' . $assoc_path . '"';
+      &debugPrint($debug, 'command: ' . $delete_command2 . "\n");
+      `$delete_command2`;
+    }
+  }
 
   # 3. Prepare the collection for parallel indexing
@@ -205 +217 @@
     {
       $prepare_command .= ' -batchsize ' . $batch_size;
+    }
+    if (0 < $max_files)
+    {
+      $prepare_command .= ' -maxfiles ' . $max_files;
     }
     &debugPrint($debug, 'command: ' . $prepare_command . "\n");
@@ -238 +254 @@
     else
     {
-      print STDOUT '[SCRIPT] Index collection using parallel processing (' . $worker_count . " workers)";
+      print STDOUT "[SCRIPT] Index collection with parallel processing (" . $worker_count . " workers\n";
       my $mpi_flags = '--show-progress --verbose ';
+      # Excessive force! Ensure we bind to the correct network interface
+      $mpi_flags .= '--mca btl tcp,sm,self --mca btl_tcp_if_include eth0 ';
+      #$mpi_flags .= '-nolocal ';
       my $mpi_conf_path = &fileCat($terrier_home, 'mpi.conf');
       if (-f $mpi_conf_path)
       {
         print STDOUT "(cluster)\n";
-        $mpi_flags .= ' -nolocal -machinefile "' . $mpi_conf_path . '"';
+        $mpi_flags .= '-machinefile "' . $mpi_conf_path . '" ';
       }
       else