Changeset 27752 for gs2-extensions/parallel-building

Timestamp:

2013-07-04T13:44:22+12:00 (11 years ago)

Author:

jmt12

Message:

Data locality file not being found is no longer fatal (HDFS-NFS-Proxy doesn't do locality very well) and added code to determine percentage of file HandbrakeCLI thinks it has parsed

File:

• gs2-extensions/parallel-building/trunk/src/bin/script/hadoop_report.pl (modified) (6 diffs)

gs2-extensions/parallel-building/trunk/src/bin/script/hadoop_report.pl

@@ -89 +89 @@
 
 # Read in data_locality.csv (will be matched to task logs)
-print ' * Reading and parsing "data_locality.csv"... ';
 my $data_was_local = {};
 my $data_locality_csv_path = $results_path . '/data_locality.csv';
-if (open(DLIN, '<:utf8', $data_locality_csv_path))
-{
-  while (my $line = <DLIN>)
-  {
-    if ($line =~ /(\d+),\d,(\d)/)
-    {
-      $data_was_local->{$1} = $2;
-    }
-  }
-  close(DLIN);
+if (-f $data_locality_csv_path)
+{
+  print ' * Reading and parsing "data_locality.csv"... ';
+  if (open(DLIN, '<:utf8', $data_locality_csv_path))
+  {
+    while (my $line = <DLIN>)
+    {
+      if ($line =~ /(\d+),\d,(\d)/)
+      {
+        $data_was_local->{$1} = $2;
+      }
+    }
+    close(DLIN);
+  }
+  else
+  {
+    die('Error! Failed to open file for reading: ' . $data_locality_csv_path);
+  }
+  print "Done!\n";
 }
 else
 {
-  die('Error! Failed to open file for reading: ' . $data_locality_csv_path);
-}
-print "Done!\n";
+  print " * Data locality not available or not applicable\n";
+}
 
 # Read in all task logs and parse task records
@@ -121 +128 @@
       my $job_no = $1;
       my $task_no = $2;
-      my $is_data_local = $data_was_local->{$task_no};
-      my $task_record = {'host'=>'', 'cpu'=>0, 'job' => $job_no, 'task' => $task_no, 'start'=>0, 'end'=>0, 'cpu_time'=>0, 'data_locality'=>$is_data_local, 'file'=>''};
+      my $is_data_local = 0;
+      if (defined ($data_was_local->{$task_no}))
+      {
+        $is_data_local = $data_was_local->{$task_no};
+      }
+      my $task_record = {'host'=>'', 'cpu'=>0, 'job' => $job_no, 'task' => $task_no, 'start'=>0, 'end'=>0, 'cpu_time'=>0, 'data_locality'=>$is_data_local, 'file'=>'', 'percom'=>'NA'};
       print ' - Reading and parsing "' . $file . '"... ';
       my $task_log_path = $results_path . '/' . $file;
@@ -169 +180 @@
         # Calculate CPU time (total time - IO time)
         $task_record->{'cpu_time'} = $task_record->{'end'} - $task_record->{'start'} - $io_time;
+
+        # We should now have the filename - use this and try and locate a
+        # convert log for this item (assuming it is multimedia, which it may
+        # not be)
+        if (defined $task_record->{'file'} && $task_record->{'file'} =~ /\/([^\/]+)\.ts/)
+        {
+          my $filename_sans_extension = $1;
+          my $convert_log = $results_path . '/convert-' . $filename_sans_extension . '.log';
+          if (-f $convert_log)
+          {
+            print '[Reading and parsing convert log]... ';
+            if (open(CLIN, '<:utf8', $convert_log))
+            {
+              my $max_percent = 0.00;
+              while (my $line = <CLIN>)
+              {
+                if ($line =~ /.*Encoding: task 1 of 1, (\d+\.\d\d) \%/)
+                {
+                  my $percent = $1;
+                  if ($percent > $max_percent)
+                  {
+                    $max_percent = $percent;
+                  }
+                }
+              }
+              close(CLIN);
+              $task_record->{'percom'} = $max_percent;
+            }
+            else
+            {
+              print STDERR "Warning! Failed to open log file for reading: " . $convert_log . "\n";
+            }
+          }
+        }
+
         # Store this record
         $task_records->{$task_no} = $task_record;
@@ -232 +278 @@
   my $row_counter = 1;
   # Header
-  print CSVOUT "id,name,hostname,start,end,cputime,dl,pid,filename\n";
+  print CSVOUT "id,name,hostname,start,end,cputime,dl,pid,filename,percom\n";
   # Master Record
-  print CSVOUT $row_counter . ',M0,' . $job_record->{'host'} . ',' . $job_record->{'start'} . ',' . $job_record->{'end'} . ',' . ($job_record->{'cpu_time'} / 1000) . ",0,0,NA\n";
+  print CSVOUT $row_counter . ',M0,' . $job_record->{'host'} . ',' . $job_record->{'start'} . ',' . $job_record->{'end'} . ',' . ($job_record->{'cpu_time'} / 1000) . ",0,0,NA,NA\n";
   $row_counter++;
   # For each compute node record
@@ -259 +305 @@
     }
     $known_workers->{$csv_worker_id} = 1;
-    print CSVOUT $node_id . ',' . $csv_worker_id . ',' . $node_record->{'host'} . ',' . $node_record->{'start'} . ',' . $node_record->{'end'} . ',' . $node_record->{'cpu_time'} . ",0,1,NA\n";
+    print CSVOUT $node_id . ',' . $csv_worker_id . ',' . $node_record->{'host'} . ',' . $node_record->{'start'} . ',' . $node_record->{'end'} . ',' . $node_record->{'cpu_time'} . ",0,1,NA,NA\n";
     # List the child task records
     foreach my $taskno (sort keys %{$task_records})
@@ -266 +312 @@
       if ($task_record->{'host'} . '#' . $task_record->{'cpu'} eq $worker_id)
       {
-        print CSVOUT $row_counter . ',T' . ($task_record->{'task'} + 0) . ',' . $task_record->{'host'} . ',' . $task_record->{'start'} . ',' . $task_record->{'end'} . ',' . $task_record->{'cpu_time'} . ',' . $task_record->{'data_locality'} . ',' . $node_id . ',' . $task_record->{'file'} . "\n";
+        print CSVOUT $row_counter . ',T' . ($task_record->{'task'} + 0) . ',' . $task_record->{'host'} . ',' . $task_record->{'start'} . ',' . $task_record->{'end'} . ',' . $task_record->{'cpu_time'} . ',' . $task_record->{'data_locality'} . ',' . $node_id . ',' . $task_record->{'file'} . ',' . $task_record->{'percom'} . "\n";
         $row_counter++;
       }