Changeset 27685 for gs2-extensions/parallel-building/trunk/src/bin/script/parse_task_info_from_hadoop_log.pl

Timestamp:

2013-06-21T12:24:54+12:00 (11 years ago)

Author:

jmt12

Message:

in the case of multiple attempts you need to retain the information about data location tips - whereas I was undefining it after the first task with that id (assuming they were unique). Also added in a check that data location tips are defined for a task before accessing them

File:

• gs2-extensions/parallel-building/trunk/src/bin/script/parse_task_info_from_hadoop_log.pl (modified) (1 diff)

gs2-extensions/parallel-building/trunk/src/bin/script/parse_task_info_from_hadoop_log.pl

@@ -246 +246 @@
     my $task_id = $job_id . '_m_' . $task_number;
     my $compute_node = $aid_2_node->{$attempt_id}->{'compute_node'};
-    my @splits = @{$tid_2_splits->{$task_id}};
-    my $data_local = 0;
-    if (grep($_ eq $compute_node, @splits))
-    {
-      $data_local = 1;
-    }
-    print CSVOUT $task_number . "," . $attempt_number . "," . $data_local . ",\"" . $compute_node . "\",\"" . join(',', natsort(@splits)) . "\"\n";
-    $tid_2_splits->{$task_id} = undef;
+    if (defined $tid_2_splits->{$task_id})
+    {
+      my @splits = @{$tid_2_splits->{$task_id}};
+      my $data_local = 0;
+      if (grep($_ eq $compute_node, @splits))
+      {
+        $data_local = 1;
+      }
+      print CSVOUT $task_number . "," . $attempt_number . "," . $data_local . ",\"" . $compute_node . "\",\"" . join(',', natsort(@splits)) . "\"\n";
+    }
+    else
+    {
+      print "Warning! Missing data location information for task: " . $task_id . "\n";
+    }
   }