source: gs2-extensions/parallel-building/trunk/src/bin/script/batch-testing.pl@ 30354

#!/usr/bin/perl

use strict;

use POSIX qw(strftime);

BEGIN
{
  die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
}

print STDOUT "===== Batch Testing =====\n";
print STDOUT "Runs a multitude of tests against a 'lorem' collection and making\n";
print STDOUT "use of parallel importing.\n\n";

# 0. Configuration
my $infodb_type = 'GDBM';
#my $infodb_type = 'GDBMServer';
#my $infodb_type = 'SQLite';
#my $infodb_type = 'TDB';
#my @sizes = (100);
#my @sizes = (500);
#my @sizes = (1000);
#my @sizes = (5000);
#my @sizes = (10000);
#my @sizes = (50000);
#my @sizes = (100, 500, 1000, 5000, 10000, 50000, 100000);
my @sizes = (500000);
#my @sizes = (1000000);
#my @sizes = (500000, 1000000);
#my @sizes = (100, 500, 1000, 5000, 10000, 50000, 100000, 500000, 1000000);
#my @threads = (1,9);
my @threads = (9);
#my @threads = (1,2,3,4,5,6,7,8,9,10,11,12,13);
#my @batchsizes = (0); # Non-parallel
#my @batchsizes = (100);
my @batchsizes = (1000);
my $test_iterations = 1;
#my $test_iterations = 3;
#my $test_iterations = 9;

# 1. Initialization
my $machine_name = `hostname -s`;
chomp($machine_name);
$machine_name = ucfirst($machine_name);
my $os_name = `lsb_release -i`;
$os_name =~ s/^Distributor ID:\s+(.*)\r?\n$/$1/i;
my $fs_name = `df -T $ENV{'GSDLHOME'}`;
$fs_name =~ s/^.*(ext2|ext3|ext4|xfs|zfs).*$/$1/is;
$fs_name = uc($fs_name);
my $sizes_str = 'd' . $sizes[0];
if (scalar(@sizes) > 1)
{
  $sizes_str .= '-' . $sizes[(scalar(@sizes) - 1)];
}
my $threads_str = 't' . $threads[0];
if (scalar(@threads) > 1)
{
  $threads_str .= '-' . $threads[(scalar(@threads) - 1)];
}
my $batch_str = 'b' . $batchsizes[0];
if (scalar(@batchsizes) > 1)
{
  $batch_str .= '-' . $batchsizes[(scalar(@batchsizes) - 1)];
}
my $db_name = $machine_name . '_' . $os_name . '_' . $fs_name . '_' . $infodb_type . '_' . $sizes_str . '_' . $threads_str . '_' . $batch_str . '_i' . $test_iterations . '.db';
my $db_path = $ENV{'GSDLHOME'} . '/collect/' . $db_name;

# 2. If we haven't already, initialize the database by creating the tables and
#    populating the pending tests queue
my $init_database = 0;

if (!-f $db_path)
{
  $init_database = 1;
}
elsif('0' eq getValueSQL($db_path, 'SELECT COUNT(*) FROM tests'))
{
  $init_database = 1;
}

if ($init_database > 0)
{
  print STDOUT " * Creating database tables\n";
  # create tests table
  execSQL($db_path, 'CREATE TABLE IF NOT EXISTS tests (walltime INTEGER DEFAULT 0, collection TEXT, type TEXT, threads INTEGER, batchsize INTEGER, testrun INTEGER, realtime REAL DEFAULT 0, systime REAL DEFAULT 0, usertime REAL DEFAULT 0, PRIMARY KEY (collection, type, testrun, threads, batchsize))');
  execSQL($db_path, 'CREATE TABLE IF NOT EXISTS testoutput (walltime INTEGER DEFAULT 0, collection TEXT, type TEXT, testrun INTEGER, threads INTEGER, batchsize INTEGER, output TEXT, PRIMARY KEY (collection, type, testrun, threads, batchsize))');

  # populate with anticpated tests
  print STDOUT " * Populating tests table\n";
  foreach my $size (@sizes)
  {
    my $collection = lc($infodb_type) . sprintf("%07d", $size);
    print STDOUT ' - generating tests for collection=' . $collection . "\n";
    my $a_test_iterations = $test_iterations;
    for (my $test_run = 1; $test_run <= $a_test_iterations; $test_run++)
    {
      foreach my $threads_run (@threads)
      {
        foreach my $batchsize_run (@batchsizes)
        {
          my $sql = "INSERT INTO tests (collection, type, testrun, threads, batchsize) VALUES ('" . $collection . "','rm'," . $test_run . "," . $threads_run . "," . $batchsize_run . ")";
          print STDERR ' - sql: ' . $sql . "\n";
          execSQL($db_path, $sql);
          $sql = "INSERT INTO tests (collection, type, testrun, threads, batchsize) VALUES ('" . $collection . "','import'," . $test_run . "," . $threads_run . "," . $batchsize_run . ")";
          print STDERR ' - sql: ' . $sql . "\n";
          execSQL($db_path, $sql);
          $sql = "INSERT INTO testoutput (collection, type, testrun, threads, batchsize) VALUES ('" . $collection . "','rm'," . $test_run . "," . $threads_run . "," . $batchsize_run . ")";
          print STDERR ' - sql: ' . $sql . "\n";
          execSQL($db_path, $sql);
          $sql = "INSERT INTO testoutput (collection, type, testrun, threads, batchsize) VALUES ('" . $collection . "','import'," . $test_run . "," . $threads_run . "," . $batchsize_run . ")";
          print STDERR ' - sql: ' . $sql . "\n";
          execSQL($db_path, $sql);
        }
      }
    }
  }
}

# 3. While there are still pending tests in the queue
my $total_count = getValueSQL($db_path, 'SELECT COUNT(*) FROM tests');
my $test_count = getValueSQL($db_path, 'SELECT COUNT(*) FROM tests WHERE realtime=0');
my $exit_file_path = $ENV{'GSDLHOME'} . '/collect/exit.now';
while ($test_count ne "0" && !-f $exit_file_path)
{
  my $x = $total_count - $test_count;
  my $now_string = strftime "%a %b %e %H:%M:%S %Y", localtime;
  print STDERR ' * [' . $now_string . '] Progress: ' . sprintf("%.0f",(($x/$total_count)*100)) . '% complete! [' . $test_count . " tests remaining]\n";
  my $cmd;
  my $result;
  my $rtime; my $utime; my $stime;
  # 4. Pick a random test (thread count and epoch) and run and time it
  my ($counter, $collection, $test_run, $threads_run, $batchsize_run) = getRecordSQL($db_path, 'SELECT _rowid_, collection, testrun, threads, batchsize FROM tests WHERE realtime=0 ORDER BY _rowid_ LIMIT 1');
  print STDOUT '   - running test import for collection=' . $collection . ', test_run=' . $test_run . ", and threads=" . $threads_run . "\n";

  # Command one: run rm_archives.pl and record information in database
  $cmd = 'time -p rm_archives.pl ' . $collection . ' 2>&1';
  print STDOUT '   - command: ' . $cmd . "\n";
  $result = `$cmd`;
  $rtime = 0;
  if ($result =~ /real\s+(\d+\.\d+)/)
  {
    $rtime = $1;
  }
  $utime = 0;
  if ($result =~ /user\s+(\d+\.\d+)/)
  {
    $utime = $1;
  }
  $stime = 0;
  if ($result =~ /sys\s+(\d+\.\d+)/)
  {
    $stime = $1;
  }
  $result =~ s/['"]//g;
  my $walltime = time();
  execSQL($db_path, 'UPDATE tests SET walltime=' . $walltime . ', realtime=' . $rtime . ', usertime=' . $utime . ', systime=' . $stime . ' WHERE collection=\'' . $collection . '\' AND type=\'rm\' AND testrun=' . $test_run . ' AND threads=' . $threads_run . ' AND batchsize=' . $batchsize_run);
  execSQL($db_path, 'UPDATE testoutput SET walltime=' . $walltime . ', output=\'' . $result . '\' WHERE collection=\'' . $collection . '\' AND type=\'rm\' AND testrun=' . $test_run . ' AND threads=' . $threads_run . ' AND batchsize=' . $batchsize_run);

  # Have a sleep to try and prevent any headaches caused by 'rm' influencing
  # later processes
  #sleep(5);

  # Command two: use sudoedit and other black magic to clear out the memory-
  # based disk cache (which is done by writing the number 3 to a certain
  # system file)
  print STDOUT "   - Synching file system... ";
  `sync`;
  print STDOUT "Done!\n";
  print STDOUT "   - Dropping memory disk cache... ";
  # - save our current default editor
  my $current_editor = $ENV{'EDITOR'};
  # - replace default editor with a script that simply clobbers the contents
  #   of any file it's handed with the number "3"
  $ENV{'EDITOR'} = 'reset_memcache_editor.sh';
  # - we now call sudoedit on the system file. How sudoedit works is that it
  #   starts by making a temp copy of the system file with appropriate
  #   permissions allowing the user to edit. It then passes the path to the
  #   temp file to the default editor - typically this would be an interactive
  #   editor like 'vi'. However, we've just replaced the editor with a custom
  #   script that just writes '3' as the content of the tmp file. Finally, when
  #   the editor exits, sudoedit copies the tmp file over the top of the system
  #   file, restoring appropriate root-level permissions
  `sudoedit /proc/sys/vm/drop_caches`;
  # - restore the default editor, just in case something in Greenstone
  #   depends on this being a reasonably value
  $ENV{'EDITOR'} = $current_editor;
  print STDOUT "Done!\n";

  # Command three: run import.pl and record information in database
  if ($batchsize_run == 0)
  {
    $cmd = 'time -p import.pl -removeold -verbosity 0 ' . $collection . ' 2>&1';
  }
  # Very super special case: if parallel processing, but number of threads is
  # one (i.e. non-parallel) we call import but specify a manifest file for the
  # entire collection (otherwise the import would do a pre-scan, drastically
  # altering the timing)
  elsif ($threads_run == 1)
  {
    $cmd = 'time -p import.pl -keepold -verbosity 0 -manifest manifest.xml ' . $collection . ' 2>&1';
  }
  else
  {
#    $cmd = 'time -p parallel_import.pl -removeold -verbosity 0 -epoch ' . $batchsize_run . ' -jobs ' . $threads_run . ' ' . $collection . ' 2>&1';
    $cmd = 'time -p parallel_import.pl -removeold -verbosity 42 -epoch ' . $batchsize_run . ' -jobs ' . $threads_run . ' ' . $collection . ' 2>&1';
  }

  print STDOUT '   - command: ' . $cmd . "\n";
#  $result = `$cmd`;
  open(CMDINPUT, $cmd . ' |') or die("Failed to execute command: " . $cmd);
  my $line = '';
  my $line_counter = 0;
  while (defined($line = <CMDINPUT>))
  {
    chomp($line);
    print STDOUT "[" . $line_counter . "] " . $line . "\n";
    $line_counter++;
  }
  close(CMDINPUT);

  $rtime = 0;
  if ($result =~ /real\s+(\d+\.\d+)/)
  {
    $rtime = $1;
  }
  $utime = 0;
  if ($result =~ /user\s+(\d+\.\d+)/)
  {
    $utime = $1;
  }
  $stime = 0;
  if ($result =~ /sys\s+(\d+\.\d+)/)
  {
    $stime = $1;
  }
  $result =~ s/['"]//g;
  $walltime = time();
  execSQL($db_path, 'UPDATE tests SET walltime=' . $walltime . ', realtime=' . $rtime . ', usertime=' . $utime . ', systime=' . $stime . ' WHERE collection=\'' . $collection . '\' AND type=\'import\' AND testrun=' . $test_run . ' AND threads=' . $threads_run . ' AND batchsize=' . $batchsize_run);
  execSQL($db_path, 'UPDATE testoutput SET walltime=' . $walltime . ', output=\'' . $result . '\' WHERE collection=\'' . $collection . '\' AND type=\'import\' AND testrun=' . $test_run . ' AND threads=' . $threads_run . ' AND batchsize=' . $batchsize_run);

  # Repeat until we have exhausted pending tests
  $test_count = getValueSQL($db_path, 'SELECT COUNT(*) FROM tests WHERE realtime=0');
}

# Remove any exit file
if (-f $exit_file_path)
{
  print STDOUT "   - Removing exit file... ";
  unlink($exit_file_path);
  print STDOUT "Done!\n";
}


print STDOUT "Complete!\n\n";

exit;

sub execSQL
{
  my ($db_path, $sql) = @_;
  # call getValueSQL but don't care about result
  getValueSQL($db_path, $sql);
}
# /** execSQL() **/

sub getRecordSQL
{
  my ($db_path, $sql) = @_;
  if ($sql !~ /LIMIT 1/i)
  {
    $sql .= ' LIMIT 1';
  }
  my $value = getValueSQL($db_path, $sql);
  return split(/\|/,$value);
}
# /** getRecordSQL() **/

sub getValueSQL
{
  my ($db_path, $sql) = @_;
  my $result = `sqlite3 "$db_path" "$sql" 2>&1`;
  if ($result =~ /Error:/)
  {
    die("Fatal Error!\nSQL:" . $sql . "\nMsg:" . $result);
  }
  # trim
  $result =~ s/^\s*|\s*$//g;
  return $result;
}
# /** getValueSQL() **/

sub printUsage
{
  my ($msg) = @_;
  # flush STDOUT
  select((select(STDOUT), $|=1)[0]);
  print STDOUT '';
  select((select(STDOUT), $|=0)[0]);
  # output any error message
  if (defined $msg)
  {
    print STDERR 'Error! ' . $msg . "\n";
  }
  # and finally the usage
  print STDERR "Usage: mpitesting.pl <path to database> <number docs> <worker threads>\n";
  print STDERR "\n";
  exit;
}

1;