Index: /gs2-extensions/parallel-building/trunk/src/bin/script/batch-testing.pl
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/bin/script/batch-testing.pl	(revision 25117)
+++ /gs2-extensions/parallel-building/trunk/src/bin/script/batch-testing.pl	(revision 25118)
@@ -15,13 +15,27 @@
 
 # 0. Configuration
-#my $infodb_type = 'GDBM';
+my $infodb_type = 'GDBM';
 #my $infodb_type = 'GDBMServer';
 #my $infodb_type = 'SQLite';
-my $infodb_type = 'TDB';
-my @sizes = (100);
+#my $infodb_type = 'TDB';
+#my @sizes = (100);
+#my @sizes = (500);
+#my @sizes = (1000);
+#my @sizes = (5000);
+#my @sizes = (10000);
+#my @sizes = (50000);
+#my @sizes = (100, 500, 1000, 5000, 10000, 50000, 100000);
+my @sizes = (500000);
+#my @sizes = (1000000);
+#my @sizes = (500000, 1000000);
 #my @sizes = (100, 500, 1000, 5000, 10000, 50000, 100000, 500000, 1000000);
-my @threads = (1);
-my @batchsize = (100);
+#my @threads = (1,9);
+my @threads = (9);
+#my @threads = (1,2,3,4,5,6,7,8,9,10,11,12,13);
+#my @batchsizes = (0); # Non-parallel
+#my @batchsizes = (100);
+my @batchsizes = (1000);
 my $test_iterations = 1;
+#my $test_iterations = 3;
 #my $test_iterations = 9;
 
@@ -45,8 +59,8 @@
   $threads_str .= '-' . $threads[(scalar(@threads) - 1)];
 }
-my $batch_str = 'b' . $batchsize[0];
-if (scalar(@batchsize) > 1)
-{
-  $batch_str .= '-' . $batchsize[(scalar(@batchsize) - 1)];
+my $batch_str = 'b' . $batchsizes[0];
+if (scalar(@batchsizes) > 1)
+{
+  $batch_str .= '-' . $batchsizes[(scalar(@batchsizes) - 1)];
 }
 my $db_name = $machine_name . '_' . $os_name . '_' . $fs_name . '_' . $infodb_type . '_' . $sizes_str . '_' . $threads_str . '_' . $batch_str . '_i' . $test_iterations . '.db';
@@ -70,6 +84,6 @@
   print STDOUT " * Creating database tables\n";
   # create tests table
-  execSQL($db_path, 'CREATE TABLE IF NOT EXISTS tests (walltime INTEGER DEFAULT 0, collection TEXT, type TEXT, testrun INTEGER, realtime REAL DEFAULT 0, systime REAL DEFAULT 0, usertime REAL DEFAULT 0, PRIMARY KEY (collection, type, testrun))');
-  execSQL($db_path, 'CREATE TABLE IF NOT EXISTS testoutput (walltime INTEGER DEFAULT 0, collection TEXT, type TEXT, testrun INTEGER,  output TEXT, PRIMARY KEY (collection, type, testrun))');
+  execSQL($db_path, 'CREATE TABLE IF NOT EXISTS tests (walltime INTEGER DEFAULT 0, collection TEXT, type TEXT, threads INTEGER, batchsize INTEGER, testrun INTEGER, realtime REAL DEFAULT 0, systime REAL DEFAULT 0, usertime REAL DEFAULT 0, PRIMARY KEY (collection, type, testrun, threads, batchsize))');
+  execSQL($db_path, 'CREATE TABLE IF NOT EXISTS testoutput (walltime INTEGER DEFAULT 0, collection TEXT, type TEXT, testrun INTEGER, threads INTEGER, batchsize INTEGER, output TEXT, PRIMARY KEY (collection, type, testrun, threads, batchsize))');
 
   # populate with anticpated tests
@@ -82,16 +96,22 @@
     for (my $test_run = 1; $test_run <= $a_test_iterations; $test_run++)
     {
-       my $sql = "INSERT INTO tests (collection, type, testrun) VALUES ('" . $collection . "','rm'," . $test_run . ")";
-       print STDERR ' - sql: ' . $sql . "\n";
-       execSQL($db_path, $sql);
-       $sql = "INSERT INTO tests (collection, type, testrun) VALUES ('" . $collection . "','import'," . $test_run . ")";
-       print STDERR ' - sql: ' . $sql . "\n";
-       execSQL($db_path, $sql);
-       $sql = "INSERT INTO testoutput (collection, type, testrun) VALUES ('" . $collection . "','rm'," . $test_run . ")";
-       print STDERR ' - sql: ' . $sql . "\n";
-       execSQL($db_path, $sql);
-       $sql = "INSERT INTO testoutput (collection, type, testrun) VALUES ('" . $collection . "','import'," . $test_run . ")";
-       print STDERR ' - sql: ' . $sql . "\n";
-       execSQL($db_path, $sql);
+      foreach my $threads_run (@threads)
+      {
+        foreach my $batchsize_run (@batchsizes)
+        {
+          my $sql = "INSERT INTO tests (collection, type, testrun, threads, batchsize) VALUES ('" . $collection . "','rm'," . $test_run . "," . $threads_run . "," . $batchsize_run . ")";
+          print STDERR ' - sql: ' . $sql . "\n";
+          execSQL($db_path, $sql);
+          $sql = "INSERT INTO tests (collection, type, testrun, threads, batchsize) VALUES ('" . $collection . "','import'," . $test_run . "," . $threads_run . "," . $batchsize_run . ")";
+          print STDERR ' - sql: ' . $sql . "\n";
+          execSQL($db_path, $sql);
+          $sql = "INSERT INTO testoutput (collection, type, testrun, threads, batchsize) VALUES ('" . $collection . "','rm'," . $test_run . "," . $threads_run . "," . $batchsize_run . ")";
+          print STDERR ' - sql: ' . $sql . "\n";
+          execSQL($db_path, $sql);
+          $sql = "INSERT INTO testoutput (collection, type, testrun, threads, batchsize) VALUES ('" . $collection . "','import'," . $test_run . "," . $threads_run . "," . $batchsize_run . ")";
+          print STDERR ' - sql: ' . $sql . "\n";
+          execSQL($db_path, $sql);
+        }
+      }
     }
   }
@@ -111,6 +131,6 @@
   my $rtime; my $utime; my $stime;
   # 4. Pick a random test (thread count and epoch) and run and time it
-  my ($counter, $collection, $test_run) = getRecordSQL($db_path, 'SELECT _rowid_, collection, testrun FROM tests WHERE realtime=0 ORDER BY RANDOM() LIMIT 1');
-  print STDOUT '   - running test import for collection=' . $collection . ', and test_run=' . $test_run . "\n";
+  my ($counter, $collection, $test_run, $threads_run, $batchsize_run) = getRecordSQL($db_path, 'SELECT _rowid_, collection, testrun, threads, batchsize FROM tests WHERE realtime=0 ORDER BY _rowid_ LIMIT 1');
+  print STDOUT '   - running test import for collection=' . $collection . ', test_run=' . $test_run . ", and threads=" . $threads_run . "\n";
 
   # Command one: run rm_archives.pl and record information in database
@@ -133,8 +153,8 @@
     $stime = $1;
   }
-  $result =~ s/'//g;
+  $result =~ s/['"]//g;
   my $walltime = time();
-  execSQL($db_path, 'UPDATE tests SET walltime=' . $walltime . ', realtime=' . $rtime . ', usertime=' . $utime . ', systime=' . $stime . ' WHERE collection=\'' . $collection . '\' AND type=\'rm\' AND testrun=' . $test_run);
-  execSQL($db_path, 'UPDATE testoutput SET walltime=' . $walltime . ', output=\'' . $result . '\' WHERE collection=\'' . $collection . '\' AND type=\'rm\' AND testrun=' . $test_run);
+  execSQL($db_path, 'UPDATE tests SET walltime=' . $walltime . ', realtime=' . $rtime . ', usertime=' . $utime . ', systime=' . $stime . ' WHERE collection=\'' . $collection . '\' AND type=\'rm\' AND testrun=' . $test_run . ' AND threads=' . $threads_run . ' AND batchsize=' . $batchsize_run);
+  execSQL($db_path, 'UPDATE testoutput SET walltime=' . $walltime . ', output=\'' . $result . '\' WHERE collection=\'' . $collection . '\' AND type=\'rm\' AND testrun=' . $test_run . ' AND threads=' . $threads_run . ' AND batchsize=' . $batchsize_run);
 
   # Have a sleep to try and prevent any headaches caused by 'rm' influencing
@@ -169,7 +189,35 @@
 
   # Command three: run import.pl and record information in database
-  $cmd = 'time -p import.pl -removeold -verbosity 0 ' . $collection . ' 2>&1';
+  if ($batchsize_run == 0)
+  {
+    $cmd = 'time -p import.pl -removeold -verbosity 0 ' . $collection . ' 2>&1';
+  }
+  # Very super special case: if parallel processing, but number of threads is
+  # one (i.e. non-parallel) we call import but specify a manifest file for the
+  # entire collection (otherwise the import would do a pre-scan, drastically
+  # altering the timing)
+  elsif ($threads_run == 1)
+  {
+    $cmd = 'time -p import.pl -keepold -verbosity 0 -manifest manifest.xml ' . $collection . ' 2>&1';
+  }
+  else
+  {
+#    $cmd = 'time -p parallel_import.pl -removeold -verbosity 0 -epoch ' . $batchsize_run . ' -jobs ' . $threads_run . ' ' . $collection . ' 2>&1';
+    $cmd = 'time -p parallel_import.pl -removeold -verbosity 42 -epoch ' . $batchsize_run . ' -jobs ' . $threads_run . ' ' . $collection . ' 2>&1';
+  }
+
   print STDOUT '   - command: ' . $cmd . "\n";
-  $result = `$cmd`;
+#  $result = `$cmd`;
+  open(CMDINPUT, $cmd . ' |') or die("Failed to execute command: " . $cmd);
+  my $line = '';
+  my $line_counter = 0;
+  while (defined($line = <CMDINPUT>))
+  {
+    chomp($line);
+    print STDOUT "[" . $line_counter . "] " . $line . "\n";
+    $line_counter++;
+  }
+  close(CMDINPUT);
+
   $rtime = 0;
   if ($result =~ /real\s+(\d+\.\d+)/)
@@ -187,8 +235,8 @@
     $stime = $1;
   }
-  $result =~ s/'//g;
+  $result =~ s/['"]//g;
   $walltime = time();
-  execSQL($db_path, 'UPDATE tests SET walltime=' . $walltime . ', realtime=' . $rtime . ', usertime=' . $utime . ', systime=' . $stime . ' WHERE collection=\'' . $collection . '\' AND type=\'import\' AND testrun=' . $test_run);
-  execSQL($db_path, 'UPDATE testoutput SET walltime=' . $walltime . ', output=\'' . $result . '\' WHERE collection=\'' . $collection . '\' AND type=\'import\' AND testrun=' . $test_run);
+  execSQL($db_path, 'UPDATE tests SET walltime=' . $walltime . ', realtime=' . $rtime . ', usertime=' . $utime . ', systime=' . $stime . ' WHERE collection=\'' . $collection . '\' AND type=\'import\' AND testrun=' . $test_run . ' AND threads=' . $threads_run . ' AND batchsize=' . $batchsize_run);
+  execSQL($db_path, 'UPDATE testoutput SET walltime=' . $walltime . ', output=\'' . $result . '\' WHERE collection=\'' . $collection . '\' AND type=\'import\' AND testrun=' . $test_run . ' AND threads=' . $threads_run . ' AND batchsize=' . $batchsize_run);
 
   # Repeat until we have exhausted pending tests
@@ -235,5 +283,5 @@
   if ($result =~ /Error:/)
   {
-    die($result);
+    die("Fatal Error!\nSQL:" . $sql . "\nMsg:" . $result);
   }
   # trim
