Changeset 27586

Timestamp:

2013-06-10T11:15:36+12:00 (11 years ago)

Author:

jmt12

Message:

Updating script to date date of hadoop job into account when searching for the task tracker log to parse in - just in case log has been rolled

File:

• gs2-extensions/parallel-building/trunk/src/bin/script/parse_task_info_from_hadoop_log.pl (modified) (6 diffs)

gs2-extensions/parallel-building/trunk/src/bin/script/parse_task_info_from_hadoop_log.pl

@@ -52 +52 @@
 print " Results directory: " . $results_dir . "\n";
 
+# 1. Determine job ID
 my $hadoop_log_path = &fileCat($results_dir, 'hadoop.log');
 if (!-e $hadoop_log_path)
@@ -58 +59 @@
 }
 print " Hadoop log path: " . $hadoop_log_path . "\n";
-
+print " * Determine JobID: ";
+my $job_id;
+my $result = `grep "Running job:" "$hadoop_log_path"`;
+if ($result =~ /Running job: job_(\d+_\d+)/)
+{
+  $job_id = $1;
+}
+else
+{
+  die("Error! Failed to locate JobID\n");
+}
+print $job_id . "\n";
+# - we'll need the date to locate the appopriate log file
+my $log_date_suffix = '';
+if ($job_id =~ /^(\d\d\d\d)(\d\d)(\d\d)/)
+{
+  $log_date_suffix = '.' . $1 . '-' . $2 . '-' . $3;
+}
+
+# 2. Determine appropriate job tracker log
 my $username = `whoami`;
 chomp($username);
@@ -65 +85 @@
 chomp($hostname);
 print " Hostname: " . $hostname . "\n";
-my $jobtracker_log_path = &fileCat($ENV{'HADOOP_PREFIX'}, 'logs', 'hadoop-' . $username . '-jobtracker-' . $hostname . '.log');
+my $jobtracker_log_path = &fileCat($ENV{'HADOOP_PREFIX'}, 'logs', 'hadoop-' . $username . '-jobtracker-' . $hostname . '.log' . $log_date_suffix);
 if (!-e $jobtracker_log_path)
 {
-  die("Error! Hadoop JobTracker log file cannot be found: " . $jobtracker_log_path . "\n");
+  $jobtracker_log_path = &fileCat($ENV{'HADOOP_PREFIX'}, 'logs', 'hadoop-' . $username . '-jobtracker-' . $hostname . '.log');
+  if (!-e $jobtracker_log_path)
+  {
+    die("Error! Hadoop JobTracker log file cannot be found: " . $jobtracker_log_path . "\n");
+  }
 }
 print " Jobtracker log path: " . $jobtracker_log_path . "\n";
@@ -74 +98 @@
 print " Report path: " . $data_locality_report_path . "\n";
 
-# 1. Determine job ID
-print " * Determine JobID: ";
-my $job_id;
-my $result = `grep "Running job:" "$hadoop_log_path"`;
-if ($result =~ /Running job: job_(\d+_\d+)/)
-{
-  $job_id = $1;
-}
-else
-{
-  die("Error! Failed to locate JobID\n");
-}
-print $job_id . "\n";
-
-# 2. Parse log
+# 3. Parse log
 print " * Parse JobTracker Log... ";
 my $tid_2_splits = {};
@@ -145 +155 @@
 
 
-# 3. Write CSV of information
+# 4. Write CSV of information
 print " * Writing Job Information... ";
 &debugPrint("\nAttemptID\tComputeNode\tSucceeded");
@@ -183 +193 @@
 print "Done\n";
 
-# 4. Done
+# 5. Done
 print "===== Complete! =====\n\n";
 exit;