root/gs2-extensions/parallel-building/trunk/src/bin/script/hadoop_report.pl @ 27560

Revision 27560, 7.2 KB (checked in by jmt12, 7 years ago)

Fixing typo in regexp that meant filenames sometimes ignored

  • Property svn:executable set to *
Line 
1#!/usr/bin/perl
2
3use strict;
4use warnings;
5
6use Devel::Peek;
7use Time::Local;
8
9print "\n===== GS Hadoop Report =====\n";
10print "Generate a report (CSV) from the output of a parallel processing Green-\n";
11print "stone import, suitable for feeding into the Gantt chart generator.\n";
12print "============================\n\n";
13
14# Configuration
15if (!defined $ARGV[0] || !-d $ARGV[0])
16{
17  die("usage: hadoop_report.pl <path to results>\n");
18}
19my $results_path = $ARGV[0];
20
21# Read in hadoop.log and parse top level record
22print ' * Reading and parsing "hadoop.log"... ';
23my $job_record = {'host'=>'', 'job'=>'', 'start'=>0, 'end'=>0, 'cpu_time'=>0};
24my $hadoop_log_path = $results_path . '/hadoop.log';
25if (open(HLIN, '<:utf8', $hadoop_log_path))
26{
27  while (my $line = <HLIN>)
28  {
29    if ($line =~ /host:(.+)/)
30    {
31      $job_record->{'host'} = $1;
32    }
33    elsif ($line =~ /Running job: job_(\d+_\d+)/)
34    {
35      $job_record->{'job'} = $1;
36    }
37    elsif ($line =~ /CPU time spent \(ms\)=(\d+)/)
38    {
39      $job_record->{'cpu_time'} = $1;
40    }
41    elsif ($job_record->{'start'} == 0 && $line =~ /(\d\d)\/(\d\d)\/(\d\d) (\d\d):(\d\d):(\d\d)/)
42    {
43      $job_record->{'start'} = timelocal($6, $5, $4, $3, ($2 - 1), $1);
44    }
45    elsif ($line =~ /(\d\d)\/(\d\d)\/(\d\d) (\d\d):(\d\d):(\d\d)/)
46    {
47      my $end = timelocal($6, $5, $4, $3, ($2 - 1), $1);
48      if ($end > $job_record->{'end'})
49      {
50        $job_record->{'end'} = $end;
51      }
52    }
53  }
54  close(HLIN);
55  if ($job_record->{'start'} == 0 || $job_record->{'end'} == 0)
56  {
57    die('Error! Failed to parse timing information from log: ' . $hadoop_log_path);
58  }
59}
60else
61{
62  die('Error! Failed to open file for reading: ' . $hadoop_log_path);
63}
64print "Done!\n";
65
66# Read in data_locality.csv (will be matched to task logs)
67print ' * Reading and parsing "data_locality.csv"... ';
68my $data_was_local = {};
69my $data_locality_csv_path = $results_path . '/data_locality.csv';
70if (open(DLIN, '<:utf8', $data_locality_csv_path))
71{
72  while (my $line = <DLIN>)
73  {
74    if ($line =~ /(\d+),\d,(\d)/)
75    {
76      $data_was_local->{$1} = $2;
77    }
78  }
79  close(DLIN);
80}
81else
82{
83  die('Error! Failed to open file for reading: ' . $data_locality_csv_path);
84}
85print "Done!\n";
86
87# Read in all task logs and parse task records
88my $task_records;
89print " * Locating task logs...\n";
90if (opendir(DH, $results_path))
91{
92  my @files = readdir(DH);
93  foreach my $file (sort @files)
94  {
95    if ($file =~ /import-hadoop-(\d+_\d+)_m_(\d+)_\d+\.log/)
96    {
97      my $job_no = $1;
98      my $task_no = $2;
99      my $is_data_local = $data_was_local->{$task_no};
100      my $task_record = {'host'=>'', 'cpu'=>0, 'job' => $job_no, 'task' => $task_no, 'start'=>0, 'end'=>0, 'cpu_time'=>0, 'data_locality'=>$is_data_local, 'file'=>''};
101      print ' - Reading and parsing "' . $file . '"... ';
102      my $task_log_path = $results_path . '/' . $file;
103      my $io_time = 0;
104      if (open(TIN, '<:utf8', $task_log_path))
105      {
106        my $io_start_time = 0;
107        while (my $line = <TIN>)
108        {
109          if ($line =~ /\[Started:(\d+)\]/)
110          {
111            $task_record->{'start'} = $1;
112          }
113          elsif ($line =~ /\[Host:([^\]]+)\]/)
114          {
115            $task_record->{'host'} = $1;
116          }
117          elsif ($line =~ /\[CPU:(\d+)\]/)
118          {
119            $task_record->{'cpu'} = $1;
120          }
121          elsif ($line =~ /\[Map:([^\]]+)=>1\]/)
122          {
123            $task_record->{'file'} = $1;
124          }
125          elsif ($line =~ /\[IOS:(\d+)\]/)
126          {
127            $io_start_time = $1;
128          }
129          elsif ($io_start_time > 0 && $line =~ /\[IOE:(\d+)\]/)
130          {
131            $io_time += ($1 - $io_start_time);
132            $io_start_time = 0;
133          }
134          elsif ($line =~ /\[Completed:(\d+)\]/)
135          {
136            my $end_time = $1;
137            $task_record->{'end'} = $end_time;
138            if ($io_start_time > 0)
139            {
140              $io_time += ($end_time - $io_start_time);
141            }
142          }
143        }
144        close(TIN);
145        # Calculate CPU time (total time - IO time)
146        $task_record->{'cpu_time'} = $task_record->{'end'} - $task_record->{'start'} - $io_time;
147        # Store this record
148        $task_records->{$task_no} = $task_record;
149      }
150      else
151      {
152        die('Error! Failed to open file for reading: ' . $task_log_path);
153      }
154      print "Done!\n";
155    }
156  }
157  close(DH);
158}
159else
160{
161  die('Error! Failed to open directory for reading: ' . $results_path);
162}
163print ' - Processed ' . scalar(keys(%{$task_records})) . " records\n";
164
165# Generate compute-node records
166print ' * Generating compute node records... ';
167my $node_records;
168foreach my $taskno (sort keys %{$task_records})
169{
170  my $task_record = $task_records->{$taskno};
171  my $node_record = {'host'=>'', 'cpu'=>0, 'job' => '', 'start'=>0, 'end'=>0, 'cpu_time'=>0};
172  # - retrieve any existing record
173  my $worker_id = $task_record->{'host'} . '#' . $task_record->{'cpu'};
174  if (defined $node_records->{$worker_id})
175  {
176    $node_record = $node_records->{$worker_id};
177  }
178  if ($node_record->{'host'} eq '')
179  {
180    $node_record->{'host'} = $task_record->{'host'};
181  }
182  if ($node_record->{'cpu'} == 0)
183  {
184    $node_record->{'cpu'} = $task_record->{'cpu'};
185  }
186  if ($node_record->{'job'} eq '')
187  {
188    $node_record->{'job'} = $task_record->{'job'};
189  }
190  if ($node_record->{'start'} == 0 || $task_record->{'start'} < $node_record->{'start'})
191  {
192    $node_record->{'start'} = $task_record->{'start'};
193  }
194  if ($node_record->{'end'} == 0 || $task_record->{'end'} > $node_record->{'end'})
195  {
196    $node_record->{'end'} = $task_record->{'end'};
197  }
198  $node_record->{'cpu_time'} += $task_record->{'cpu_time'};
199  # - store it
200  $node_records->{$worker_id} = $node_record;
201}
202print "Done!\n";
203
204# Write out CSV of all information
205my $report_csv_path = $results_path . '/timing.csv';
206if (open(CSVOUT, '>:utf8', $report_csv_path))
207{
208  my $row_counter = 1;
209  # Header
210  print CSVOUT "id,name,hostname,start,end,cputime,dl,pid,filename\n";
211  # Master Record
212  print CSVOUT $row_counter . ',M0,' . $job_record->{'host'} . ',' . $job_record->{'start'} . ',' . $job_record->{'end'} . ',' . ($job_record->{'cpu_time'} / 1000) . ",0,0,NA\n";
213  $row_counter++;
214  # For each compute node record
215  foreach my $worker_id (sort keys %{$node_records})
216  {
217    my $node_record = $node_records->{$worker_id};
218    my $node_id = $row_counter;
219    $row_counter++;
220    print CSVOUT $node_id . ',W' . $node_record->{'cpu'} . ',' . $node_record->{'host'} . ',' . $node_record->{'start'} . ',' . $node_record->{'end'} . ',' . $node_record->{'cpu_time'} . ",0,1,NA\n";
221    # List the child task records
222    foreach my $taskno (sort keys %{$task_records})
223    {
224      my $task_record = $task_records->{$taskno};
225      if ($task_record->{'host'} . '#' . $task_record->{'cpu'} eq $worker_id)
226      {
227        print CSVOUT $row_counter . ',T' . ($task_record->{'task'} + 0) . ',' . $task_record->{'host'} . ',' . $task_record->{'start'} . ',' . $task_record->{'end'} . ',' . $task_record->{'cpu_time'} . ',' . $task_record->{'data_locality'} . ',' . $node_id . ',' . $task_record->{'file'} . "\n";
228        $row_counter++;
229      }
230    }
231  }
232  close(CSVOUT);
233}
234else
235{
236  die('Error! Failed to open file for writing: ' . $report_csv_path);
237}
238
239print "Complete!\n\n";
240exit;
241
Note: See TracBrowser for help on using the browser.