Changeset 27412 for gs2-extensions/parallel-building/trunk/src/bin/script/parse_task_info_from_hadoop_log.pl

Timestamp:

2013-05-24T09:22:13+12:00 (11 years ago)

Author:

jmt12

Message:

I obviously hadn't run this script on Karearea before - assumed all compute nodes to have a name ending '.local' when parsing. Yeah, so that won't happen.

File:

• gs2-extensions/parallel-building/trunk/src/bin/script/parse_task_info_from_hadoop_log.pl (modified) (4 diffs)

gs2-extensions/parallel-building/trunk/src/bin/script/parse_task_info_from_hadoop_log.pl

@@ -99 +99 @@
   {
     # Tips provide a match between task and file splits
-    if ($line =~ /tip:task_${job_id}(_m_\d+) has split on node:\/default-rack\/([^\.]+).local/)
+    if ($line =~ /tip:task_${job_id}(_m_\d+) has split on node:\/default-rack\/([^\.\r\n]+)/)
     {
       my $task_id = $job_id . $1;
@@ -147 +147 @@
 # 3. Write CSV of information
 print " * Writing Job Information... ";
-&debugPrint("AttemptID\tComputeNode\tSucceeded");
+&debugPrint("\nAttemptID\tComputeNode\tSucceeded");
 foreach my $attempt_id (keys %{$aid_2_node})
 {
@@ -157 +157 @@
   &debugPrint($task_id . "\t" . join(',', natsort(@{$tid_2_splits->{$task_id}})));
 }
+
 # - open the CSV file and write out the combined information from above
 if (open(CSVOUT, '>:utf8', $data_locality_report_path))
@@ -167 +168 @@
     my $compute_node = $aid_2_node->{$attempt_id}->{'compute_node'};
     my @splits = @{$tid_2_splits->{$task_id}};
-    my $data_local = 'N';
+    my $data_local = 0;
     if (grep($_ eq $compute_node, @splits))
     {
-      $data_local = 'Y';
-    }
-    print CSVOUT $task_number . "," . $attempt_number . "," . $data_local . "," . $compute_node . ",\"" . join(',', natsort(@splits)) . "\"\n";
+      $data_local = 1;
+    }
+    print CSVOUT $task_number . "," . $attempt_number . "," . $data_local . ",\"" . $compute_node . "\",\"" . join(',', natsort(@splits)) . "\"\n";
   }
   close(CSVOUT);