Index: /gs2-extensions/parallel-building/trunk/src/bin/script/generate_gantt.pl
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/bin/script/generate_gantt.pl	(revision 27550)
+++ /gs2-extensions/parallel-building/trunk/src/bin/script/generate_gantt.pl	(revision 27551)
@@ -9,149 +9,70 @@
 print "\n===== Generate Timing (GANTT) =====\n";
 
+# 0. Configuration
+my $debug = 0;
+my $import_dir;
+
 # 1. Initialization
-#my $chart_width = 1024;
-my $chart_width = 1536;
-#my $chart_width = 2048;
+if (!defined $ARGV[0] || !-d $ARGV[0])
+{
+  &printUsage('Directory not provided or doesn\'t exist');
+}
+my $dir = $ARGV[0];
+my $timing_csv_path = &filenameCat($dir, 'timing.csv');
+if (!-e $timing_csv_path)
+{
+  &printUsage('Directory doesn\'t contain timing.csv: ' . $dir);
+}
+print 'Timing File: ' . $timing_csv_path . "\n";
+my $chart_width = 1024;
+if (defined $ARGV[1])
+{
+  if ($ARGV[1] !~ /^\d+$/)
+  {
+    &printUsage('Chart width not a number');
+  }
+  $chart_width = $ARGV[1];
+}
 print "Chart Width: " . $chart_width . "px\n";
-# 1.1 Store all information extracted in a cool data structure
-# - N = hostname, S = thread start, E = thread end
-my $timing_data = {'M' => {'N'=>'', 'S'=>0, 'E'=>0}};
-# 1.2 Check the file exists
-if (!-f $ARGV[0])
-{
-  die("Error! File can't be read: " . $ARGV[0]);
-}
-my $main_log_filename = $ARGV[0];
-# 1.2 From the filename we can parse in some information like the number of worker threads
-my $number_of_workers = 0;
-if ($main_log_filename =~ /\-W(\d+)E/)
-{
-  $number_of_workers = $1;
+print "===================================\n\n";
+
+# Read in timing.csv and parse information into data structure
+my $timing_data = {};
+my $id_2_worker_id = {};
+if (open(TIN, '<:utf8', $timing_csv_path))
+{
+  my $line;
+  while ($line = <TIN>)
+  {
+    my @parts = split(/,/, $line);
+    if ($parts[1] eq 'M0')
+    {
+      $timing_data->{'M'} = {'N'=>$parts[2], 'S'=>$parts[3], 'E'=>$parts[4]};
+    }
+    elsif ($parts[1] =~ /W\d+/)
+    {
+      $timing_data->{$parts[1]} = {'N'=>$parts[2], 'S'=>$parts[3], 'E'=>$parts[4], 'F'=>{}};
+      $id_2_worker_id->{$parts[0]} = $parts[1];
+    }
+    elsif ($parts[1] =~ /T\d+/)
+    {
+      my $worker_id = $id_2_worker_id->{$parts[7]};
+      my $stop = $parts[4];
+      my $filepath = $parts[8];
+      $import_dir = &longestCommonPath($filepath, $import_dir);
+      $timing_data->{$worker_id}->{'F'}->{$parts[3]} = {'FN'=>$filepath, 'PS'=>($stop - $parts[5]), 'PE'=>$stop, 'E'=>$stop};
+    }
+  }
+  close(TIN);
 }
 else
 {
-  die("Error! Malformed filename (expecting number of workers): " . $main_log_filename);
-}
-# 1.3 Initialize the data structure with the number of workers too (we don't
-#     know the number of files yet, so they'll have to be adding on the fly)
-for (my $i = 1; $i <= $number_of_workers; $i++)
-{
-  $timing_data->{'W' . $i} = {'N'=>'', 'S'=>0, 'E'=>0, 'F'=>{}};
-}
-
-# 2. Read in main log file
-print " * Reading main log: " . $main_log_filename . "\n";
-open(LOGIN, '<:utf8', $main_log_filename) or die("Error! Failed to open file for reading: " . $main_log_filename);
-my $line = '';
-my $currently_processing = {};
-while ($line = <LOGIN>)
-{
-  # 2.1 Parse in the Master thread start time
-  if ($line =~ /\[M\d?:(\d+)\] Starting on (.+)/)
-  {
-    $timing_data->{'M'}->{'S'} = $1;
-    $timing_data->{'M'}->{'N'} = $2;
-  }
-  elsif ($line =~ /\[(W\d+):(\d+)\] Starting on (.+)/)
-  {
-    my $worker_id = $1;
-    $timing_data->{$worker_id}->{'S'} = $2;
-    $timing_data->{$worker_id}->{'N'} = $3;
-  }
-  elsif ($line =~ /\[(W\d+):(\d+)\] Processing/)
-  {
-    my $worker_id = $1;
-    my $job_start_time = $2;
-    $timing_data->{$worker_id}->{'F'}->{$job_start_time} = {'FN'=>'', 'PS'=>0, 'PE'=>0, 'E'=>0};  
-    $currently_processing->{$worker_id} = $job_start_time;
-  }
-  # 2.3 Or we may parse in the starting times for each working thread
-  # 2.4 Or we may also parse (the last encountered) completion time for each
-  #     working thread
-  elsif ($line =~ /\[(W\d+):(\d+)\] Process complete/)
-  {
-    my $worker_id = $1;
-    my $job_end_time = $2;
-    $timing_data->{$worker_id}->{'E'} = $job_end_time;
-    my $job_start_time = $currently_processing->{$worker_id};
-    $timing_data->{$worker_id}->{'F'}->{$job_start_time}->{'E'} = $job_end_time;
-    delete($currently_processing->{$worker_id});
-  }
-  # 2.5 Finally, we may parse in the Master thread end time
-  elsif ($line =~ /\[M\d?:(\d+)\] Master will exit when workers complete/)
-  {
-    $timing_data->{'M'}->{'E'} = $1;
-  }
-}
-close(LOGIN);
-
-# 3. Read each of worker logs parsing in information about the files processed
-#    - each will be stored (in an associative array) against its start time
-print " * Reading worker logs";
-foreach my $worker_id (nsort keys %{$timing_data})
-{
-  my $jobs = $timing_data->{$worker_id}->{'F'};
-  my $counter = 1;
-  foreach my $job_start_time (sort keys %{$jobs})
-  {
-    my $log_filename = 'gsimport-' . $worker_id . '-' . $counter . '.log';
-    print ".";
-    open(WLOGIN, '<:utf8', $log_filename) or die("Error! Failed to open for reading: " . $log_filename);
-    my $wline = '';
-    while ($wline = <WLOGIN>)
-    {
-      if ($wline =~ /\[A:\d+\] SimpleVideoPlugin processing: (.+)/)
-      {
-        $timing_data->{$worker_id}->{'F'}->{$job_start_time}->{'FN'} = $1;
-      }
-      # Start of video processing (excluding as much IO as possible) 
-      elsif ($wline =~ /\[C1:(\d+)\]/)
-      {
-        $timing_data->{$worker_id}->{'F'}->{$job_start_time}->{'PS'} = $1;
-      }
-      # Immediately after processing video
-      elsif ($wline =~ /\[E2:(\d+)\]/)
-      {
-        $timing_data->{$worker_id}->{'F'}->{$job_start_time}->{'PE'} = $1;
-      }
-    }
-
-    if ($timing_data->{$worker_id}->{'F'}->{$job_start_time}->{'PE'} <= 0)
-    {
-      print "\n[Warning - bogus log: $log_filename]";
-    }
-
-    close(WLOGIN);
-    $counter++;
-  }
-}
-print " Done!\n";
-
-# 4. Produce CSV of information
-print " * Generating timing information as CSV... ";
-open(CSVOUT, '>:utf8', 'timing.csv') or die('Error! Failed to open file for writing: timing.csv');
-print CSVOUT "number,id,hostname,start,end,hierarchy\n";
-my $thread_counter = 1;
-foreach my $thread (nsort keys %{$timing_data})
-{
-  my $data = $timing_data->{$thread};
-  print CSVOUT $thread_counter . ',' . $thread . ',' . $data->{'N'} . ',' . strftime("%H:%M:%S", localtime($data->{'S'})) . ',' . strftime("%H:%M:%S", localtime($data->{'E'})) . ',';
-  if ($thread eq 'M')
-  {
-    print CSVOUT '0';
-  }
-  else
-  {
-    print CSVOUT '1';
-  }
-  print CSVOUT "\n";
-  $thread_counter++;
-}
-close(CSVOUT);
-print "Done!\n";
-
-# 5. Produce pretty HTML chart of timing information including jobs
+  die('Error! Failed to open file for reading: ' . $timing_csv_path);
+}
+my $number_of_workers = scalar(keys(%{$id_2_worker_id}));;
+
+# 3. Produce pretty HTML chart of timing information including jobs
 print " * Generating timing information as HTML... ";
-open(HTMLOUT, '>:utf8', 'gantt.html') or die('Error! Failed to open file for writing: gantt.html');
+open(HTMLOUT, '>:utf8', $dir . '/gantt.html') or die('Error! Failed to open file for writing: gantt.html');
 print HTMLOUT "<html>\n";
 print HTMLOUT '<head>' . "\n";
@@ -159,5 +80,5 @@
 print HTMLOUT 'div.thread {position:relative}' . "\n";
 print HTMLOUT 'div.master {border:1px solid gray;color:white;font-weight:bold}' . "\n";
-print HTMLOUT 'div.worker {background-color:green;color:white;font-weight:bold}' . "\n";
+print HTMLOUT 'div.worker {border:1px solid black;background-color:green;color:white;font-weight:bold}' . "\n";
 print HTMLOUT 'div.time {font-size:smaller;font-weight:normal}' . "\n";
 print HTMLOUT 'div.job {background-color:transparent;color:black;border:1px solid black;display:block;font-size:smaller;position:relative;text-align:center}' . "\n";
@@ -195,7 +116,7 @@
         my $process_duration = $process_end - $process_start;
         my $total_duration = $io_duration + $process_duration;
-        ###rint "[DEBUG] filename: " . $timing_data->{$worker_id}->{'F'}->{$job_start}->{'FN'} . "\n";
-        ###rint "[DEBUG] start: $job_start ps: $process_start pe: $process_end end: $job_end\n";
-        ###rint "[DEBUG] io: $io_duration process: $process_duration duration: $total_duration\n";
+        &debugPrint("filename: " . $timing_data->{$worker_id}->{'F'}->{$job_start}->{'FN'});
+        &debugPrint("start: $job_start ps: $process_start pe: $process_end end: $job_end");
+        &debugPrint("io: $io_duration process: $process_duration duration: $total_duration");
         # Running stats
         $total_io_time += $io_duration;
@@ -210,4 +131,6 @@
         }
       }
+      # Shorten filename
+      $timing_data->{$worker_id}->{'F'}->{$job_start}->{'FN'} = substr($timing_data->{$worker_id}->{'F'}->{$job_start}->{'FN'}, length($import_dir) + 1);
       $file_count++;
     }
@@ -216,4 +139,5 @@
 my $avg_processing_time = floor(($total_io_time + $total_process_time) / $file_count);
 
+print HTMLOUT "<tr><th>Import Directory:</th><td>" . $import_dir . "</td></tr>\n";
 print HTMLOUT "<tr><th>Processing Time:</th><td>" . &renderTime($total_duration) . "</td></tr>\n";
 print HTMLOUT "<tr><th>Processing Threads:</th><td>" . $number_of_workers . "</td></tr>\n";
@@ -247,9 +171,83 @@
 exit;
 
-# /**
-#  */
+
+## @function debugPrint()
+#
+sub debugPrint
+{
+  my $msg = shift(@_);
+  if ($debug)
+  {
+    print STDERR '[DEBUG] ' . $msg . "\n";
+  }
+}
+## debugPrint() ##
+
+
+## @function filenameCat
+#
+sub filenameCat
+{
+  my $path = join('/', @_);
+  $path =~ s/[\/\\]+/\//g;
+  # protocols
+  $path =~ s/^(HDFS|HDFSShell|HDThriftFS):\//$1:\/\//;
+  return $path;
+}
+## filenameCat() ##
+
+## @function printUsage()
+#
+sub printUsage
+{
+  my $msg = shift(@_);
+  if (defined $msg)
+  {
+    print 'Error! ' . $msg . "\n";
+  }
+  die("Usage: generate_gantt.pl <results dir> [<width in pixels>]\n\n");
+}
+## printUsage() ##
+
+
+## @function longestCommonPath
+#
+sub longestCommonPath
+{
+  my ($path_new, $path_current) = @_;
+  my $result = '';
+  if (defined $path_current)
+  {
+    my @path_new_parts = split(/\//, $path_new);
+    my @path_current_parts = split(/\//, $path_current);
+    my @path_parts;
+    for (my $i = 0; $i < scalar(@path_current_parts); $i++)
+    {
+      if ($path_current_parts[$i] eq $path_new_parts[$i])
+      {
+        push(@path_parts, $path_new_parts[$i]);
+      }
+      else
+      {
+        last;
+      }
+    }
+    $result = &filenameCat(@path_parts);
+  }
+  else
+  {
+    $result = $path_new;
+  }
+  return $result;
+}
+## longestCommonPath() ##
+
+
+## @function renderLine()
+#
 sub renderLine
 {
   my ($table_width, $start, $end, $class, $tname, $tstart, $tend, $jobs) = @_;
+  &debugPrint("renderLine($table_width, $start, $end, $class, $tname, $tstart, $tend, <jobs>)");
   # All timings need to be relative to 0 (relative start)
   my $duration = $end - $start;
@@ -311,6 +309,9 @@
   return $html;
 }
-# /** renderLine() **/
-
+## renderLine() ##
+
+
+## @function renderTime()
+#
 sub renderTime
 {
