root/gs2-extensions/parallel-building/trunk/src/bin/script/generate_gantt.pl @ 27590

Revision 27590, 12.4 KB (checked in by jmt12, 6 years ago)

Adding statistics about data locality, and highlighting tasks where file was non-local

  • Property svn:executable set to *
Line 
1#!/usr/bin/perl
2
3use strict;
4use warnings;
5
6use Sort::Naturally;
7use POSIX qw(floor strftime);
8
9print "\n===== Generate Timing (GANTT) =====\n";
10
11# 0. Configuration
12my $debug = 0;
13my $import_dir;
14
15# 1. Initialization
16if (!defined $ARGV[0] || !-d $ARGV[0])
17{
18  &printUsage('Directory not provided or doesn\'t exist');
19}
20my $dir = $ARGV[0];
21my $timing_csv_path = &filenameCat($dir, 'timing.csv');
22if (!-e $timing_csv_path)
23{
24  &printUsage('Directory doesn\'t contain timing.csv: ' . $dir);
25}
26print 'Timing File: ' . $timing_csv_path . "\n";
27my $chart_width = 1024;
28if (defined $ARGV[1])
29{
30  if ($ARGV[1] !~ /^\d+$/)
31  {
32    &printUsage('Chart width not a number');
33  }
34  $chart_width = $ARGV[1];
35}
36print "Chart Width: " . $chart_width . "px\n";
37
38my ($epoc) = $dir =~ /(\d+)$/;
39my $gantt_path = $dir . '/' . $epoc . '-gantt.html';
40print "Gantt Chart: " . $gantt_path . "\n";
41
42print "===================================\n\n";
43
44# Read in timing.csv and parse information into data structure
45my $timing_data = {};
46my $id_2_worker_id = {};
47if (open(TIN, '<:utf8', $timing_csv_path))
48{
49  my $line;
50  while ($line = <TIN>)
51  {
52    my @parts = split(/,/, $line);
53    if ($parts[1] eq 'M0')
54    {
55      $timing_data->{'M'} = {'N'=>$parts[2], 'S'=>$parts[3], 'E'=>$parts[4]};
56    }
57    elsif ($parts[1] =~ /W\d+/)
58    {
59      $timing_data->{$parts[1]} = {'N'=>$parts[2], 'S'=>$parts[3], 'E'=>$parts[4], 'F'=>{}};
60      $id_2_worker_id->{$parts[0]} = $parts[1];
61    }
62    elsif ($parts[1] =~ /T\d+/)
63    {
64      my $worker_id = $id_2_worker_id->{$parts[7]};
65      my $stop = $parts[4];
66      my $filepath = $parts[8];
67      $filepath =~ s/^\s+|\s+$//g;
68      $import_dir = &longestCommonPath($filepath, $import_dir);
69      $timing_data->{$worker_id}->{'F'}->{$parts[3]} = {'FN'=>$filepath, 'PS'=>($stop - $parts[5]), 'PE'=>$stop, 'E'=>$stop, 'DL'=>$parts[6]};
70    }
71  }
72  close(TIN);
73}
74else
75{
76  die('Error! Failed to open file for reading: ' . $timing_csv_path);
77}
78my $number_of_workers = scalar(keys(%{$id_2_worker_id}));;
79
80# 3. Produce pretty HTML chart of timing information including jobs
81print " * Generating timing information as HTML... ";
82open(HTMLOUT, '>:utf8', $gantt_path) or die('Error! Failed to open file for writing: gantt.html');
83print HTMLOUT "<html>\n";
84print HTMLOUT '<head>' . "\n";
85print HTMLOUT '<style type="text/css">' . "\n";
86print HTMLOUT 'div.thread {position:relative}' . "\n";
87print HTMLOUT 'div.master {border:1px solid gray;color:white;font-weight:bold}' . "\n";
88print HTMLOUT 'div.worker {border:1px solid black;background-color:green;color:white;font-weight:bold}' . "\n";
89print HTMLOUT 'div.time {font-size:smaller;font-weight:normal}' . "\n";
90print HTMLOUT 'div.job {background-color:transparent;color:black;border:1px solid black;display:block;font-size:smaller;position:relative;text-align:center;overflow:hidden}' . "\n";
91print HTMLOUT 'span.process {z-index:-1;background-color:#C7C7C7;position:absolute}' . "\n";
92print HTMLOUT 'span.label {z-index:1;background-color:transparent;overflow:hidden;white-space:nowrap;}' . "\n";
93print HTMLOUT "th {text-align:left}\n";
94print HTMLOUT '</style>' . "\n";
95print HTMLOUT '</head>' . "\n";
96print HTMLOUT "<body>\n";
97print HTMLOUT "<h2>Statistics</h2>\n";
98print HTMLOUT "<table>\n";
99
100my $total_duration = $timing_data->{'M'}->{'E'} - $timing_data->{'M'}->{'S'};
101my $file_count = 0;
102my $data_locality = 0;
103my $total_io_time = 0;
104my $total_process_time = 0;
105my $fastest_file = 0;
106my $slowest_file = 0;
107my $problem_files = 0;
108foreach my $worker_id (keys %{$timing_data})
109{
110  if ($worker_id ne 'M')
111  {
112    foreach my $job_start ( keys %{$timing_data->{$worker_id}->{'F'}} )
113    {
114      my $process_start = $timing_data->{$worker_id}->{'F'}->{$job_start}->{'PS'};
115      my $process_end = $timing_data->{$worker_id}->{'F'}->{$job_start}->{'PE'};
116      my $job_end = $timing_data->{$worker_id}->{'F'}->{$job_start}->{'E'};
117      if ($process_start == 0 || $process_end == 0 || $job_end == 0)
118      {
119        $problem_files++;
120      }
121      else
122      {
123        my $io_duration = ($process_start - $job_start) + ($job_end - $process_end);
124        my $process_duration = $process_end - $process_start;
125        my $total_duration = $io_duration + $process_duration;
126        &debugPrint("filename: " . $timing_data->{$worker_id}->{'F'}->{$job_start}->{'FN'});
127        &debugPrint("start: $job_start ps: $process_start pe: $process_end end: $job_end");
128        &debugPrint("io: $io_duration process: $process_duration duration: $total_duration");
129        # Running stats
130        $total_io_time += $io_duration;
131        $total_process_time += $process_duration;
132        if ($fastest_file == 0 || $total_duration < $fastest_file)
133        {
134          $fastest_file = $total_duration;
135        }
136        if ($slowest_file == 0 || $total_duration > $slowest_file)
137        {
138          $slowest_file = $total_duration;
139        }
140      }
141      # Shorten filename
142      if (defined $timing_data->{$worker_id}->{'F'}->{$job_start}->{'FN'})
143      {
144        $timing_data->{$worker_id}->{'F'}->{$job_start}->{'FN'} = substr($timing_data->{$worker_id}->{'F'}->{$job_start}->{'FN'}, length($import_dir) + 1);
145      }
146      $file_count++;
147      if ($timing_data->{$worker_id}->{'F'}->{$job_start}->{'DL'} == 1)
148      {
149        $data_locality++;
150      }
151    }
152  }
153}
154my $avg_processing_time = floor(($total_io_time + $total_process_time) / $file_count);
155my @months = ("Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec");
156
157print HTMLOUT "<tr><th>Import Directory:</th><td>" . $import_dir . "</td></tr>\n";
158my ($sec, $min, $hour, $day, $month, $year) = (localtime($timing_data->{'M'}->{'S'}))[0,1,2,3,4,5];
159print HTMLOUT "<tr><th>Start Time:</th><td>" . sprintf('%04d%s%02d %02d:%02d:%02d', ($year+1900), $months[$month], $day, $hour, $min, $sec) . "</td></tr>\n";
160print HTMLOUT "<tr><th>Processing Time:</th><td>" . &renderTime($total_duration) . "</td></tr>\n";
161print HTMLOUT "<tr><th>Processing Threads:</th><td>" . $number_of_workers . "</td></tr>\n";
162print HTMLOUT "<tr><th>Files Processed:</th><td>" . $file_count . "</td></tr>\n";
163if ($data_locality > 0)
164{
165  print HTMLOUT "<tr><th>Data Locality:</th><td>" . sprintf('%d%% [%d out of %d]', (($data_locality / $file_count) * 100), $data_locality, $file_count) . "</td></tr>\n";
166}
167print HTMLOUT "<tr><th>Serial Processing Time:</th><td>" . &renderTime($total_process_time) . "</td></tr>\n";
168print HTMLOUT "<tr><th>Serial IO Time:</th><td>" . &renderTime($total_io_time) . "</td></tr>\n";
169print HTMLOUT "<tr><th>Average File Processing Time:</th><td>" . &renderTime($avg_processing_time) . "</td></tr>\n";
170print HTMLOUT "<tr><th>Fastest File:</th><td>" . &renderTime($fastest_file) . "</td></tr>\n";
171print HTMLOUT "<tr><th>Slowest File:</th><td>" . &renderTime($slowest_file) . "</td></tr>\n";
172print HTMLOUT "<tr><th>Problem Files:</th><td>" . $problem_files . "</td></tr>\n";
173
174print HTMLOUT "</table>\n";
175print HTMLOUT "<hr />\n";
176print HTMLOUT "<h2>Timing Chart (Gannt)</h2>\n";
177print HTMLOUT renderLine($chart_width, $timing_data->{'M'}->{'S'}, $timing_data->{'M'}->{'E'}, 'master', $timing_data->{'M'}->{'N'}, $timing_data->{'M'}->{'S'}, $timing_data->{'M'}->{'E'}, {});
178foreach my $worker_id (nsort keys %{$timing_data})
179{
180  if ($worker_id ne 'M')
181  {
182    my $data = $timing_data->{$worker_id};
183    print HTMLOUT renderLine($chart_width, $timing_data->{'M'}->{'S'}, $timing_data->{'M'}->{'E'}, 'worker', $worker_id . ' [' . $data->{'N'} . ']', $data->{'S'}, $data->{'E'}, $data->{'F'});
184  }
185}
186print HTMLOUT '<div>' . "\n";
187print HTMLOUT "</body>\n";
188print HTMLOUT "</html>";
189close(HTMLOUT);
190
191print "Done!\n";
192print "Complete!\n\n";
193exit;
194
195
196## @function debugPrint()
197#
198sub debugPrint
199{
200  my $msg = shift(@_);
201  if ($debug)
202  {
203    print STDERR '[DEBUG] ' . $msg . "\n";
204  }
205}
206## debugPrint() ##
207
208
209## @function filenameCat
210#
211sub filenameCat
212{
213  my $path = join('/', @_);
214  $path =~ s/[\/\\]+/\//g;
215  # protocols
216  $path =~ s/^(HDFS|HDFSShell|HDThriftFS):\//$1:\/\//;
217  return $path;
218}
219## filenameCat() ##
220
221## @function printUsage()
222#
223sub printUsage
224{
225  my $msg = shift(@_);
226  if (defined $msg)
227  {
228    print 'Error! ' . $msg . "\n";
229  }
230  die("Usage: generate_gantt.pl <results dir> [<width in pixels>]\n\n");
231}
232## printUsage() ##
233
234
235## @function longestCommonPath
236#
237sub longestCommonPath
238{
239  my ($path_new, $path_current) = @_;
240  my $result = '';
241  if (defined $path_current)
242  {
243    my @path_new_parts = split(/\//, $path_new);
244    my @path_current_parts = split(/\//, $path_current);
245    my @path_parts;
246    for (my $i = 0; $i < scalar(@path_current_parts); $i++)
247    {
248      if ($path_current_parts[$i] eq $path_new_parts[$i])
249      {
250        push(@path_parts, $path_new_parts[$i]);
251      }
252      else
253      {
254        last;
255      }
256    }
257    $result = &filenameCat(@path_parts);
258  }
259  else
260  {
261    $result = $path_new;
262  }
263  return $result;
264}
265## longestCommonPath() ##
266
267
268## @function renderLine()
269#
270sub renderLine
271{
272  my ($table_width, $start, $end, $class, $tname, $tstart, $tend, $jobs) = @_;
273  &debugPrint("renderLine($table_width, $start, $end, $class, $tname, $tstart, $tend, <jobs>)");
274  # All timings need to be relative to 0 (relative start)
275  my $duration = $end - $start;
276  my $rtstart = $tstart - $start;
277  my $rtend = $tend - $start;
278  # We need to scale these depending on the timing of this thread relative to
279  # the master thread
280  my $width = $chart_width;
281  my $left = 0;
282  if ($start != $tstart)
283  {
284    my $left_offset_percent = $rtstart / $duration;
285    $left = $left_offset_percent * $table_width;
286  }
287  # - subtract any left offset from width
288  $width = $width - $left;
289  # - right offset directly subtracted from width
290  if ($end != $tend)
291  {
292    my $right_offset_percent = ($duration - $rtend) / $duration;
293    my $right = $right_offset_percent * $table_width;
294    $width = $width - $right;
295  }
296  # Round things off (simple dutch rounding)
297  $left = int($left + 0.5);
298  $width = int($width + 0.5);
299  # Output the bar for this master/worker
300  my $html = '<div class="thread ' . $class . '" style="left:' . $left . 'px;width:' . $width . 'px;">';
301  if ($class eq 'master')
302  {
303    $html .= '<div style="background-color:blue">';
304  }
305  $html .= '<div class="time" style="display:table-cell">' . &renderTime($rtstart) . '</div><div style="display:table-cell;padding-left:20px;width:100%;">' . ucfirst($class) . ': ' . $tname . '</div><div class="time" style="display:table-cell">' . renderTime($rtend) . '</div></div>';
306  my $previous_jright = 0;
307  foreach my $jstart (sort keys %{$jobs})
308  {
309    my $rjstart = $jstart - $start;
310    my $rpstart = $jobs->{$jstart}->{'PS'} - $start;
311    my $rpend = $jobs->{$jstart}->{'PE'} - $start;
312    my $rjend = $jobs->{$jstart}->{'E'} - $start;
313    my $jduration = $jobs->{$jstart}->{'E'} - $jstart;
314    # Scale Job co-ordinates
315    my $jleft_percent = $rjstart / $duration;
316    my $jleft = int(($jleft_percent * $table_width) + 0.5);
317    my $jwidth_percent = $jduration / $duration;
318    my $jwidth = int(($jwidth_percent * $table_width) + 0.5);
319    if ($jleft + $jwidth > $left + $width)
320    {
321      $jwidth = ($left + $width) - $jleft;
322    }
323    # Then scale process timings within that!
324    my $rpleft_percent = ($rpstart - $rjstart) / $duration;
325    my $rpleft = int(($rpleft_percent * $table_width) + 0.5);
326    my $rpwidth = $jwidth - $rpleft;
327    my $cpu_percent = int((($rpwidth / $jwidth) * 100) + 0.5);
328    $html .= '<div class="job" style="left:' . $jleft . 'px;width:' . $jwidth . 'px;';
329    if ($jobs->{$jstart}->{'DL'} != 1)
330    {
331      $html .= 'border:1px dashed black;';
332    }
333    $html .= '" title="FN:' . $jobs->{$jstart}->{'FN'} . ', S:' . renderTime($rjstart) . ', E:' . renderTime($rjend) . ', CPU: ' . $cpu_percent . '%"><span class="process" style="left:' . $rpleft . 'px;width:' . $rpwidth . 'px">&nbsp;</span><span class="label"';
334    if ($jobs->{$jstart}->{'DL'} != 1)
335    {
336      $html .= ' style="color:#FF0000"';
337    }
338    $html .=  '>' . $jobs->{$jstart}->{'FN'};
339    if ($jobs->{$jstart}->{'DL'} != 1)
340    {
341      $html .= ' [NL]';
342    }
343    $html .= '</span></div>';
344  }
345  return $html;
346}
347## renderLine() ##
348
349
350## @function renderTime()
351#
352sub renderTime
353{
354  my ($seconds) = @_;
355  my $time_str = '';
356  # determine how many hours
357  my $an_hour = 60 * 60;
358  my $hours = floor($seconds / $an_hour);
359  $seconds = $seconds - ($hours * $an_hour);
360  my $a_minute = 60;
361  my $minutes = floor($seconds / $a_minute);
362  $seconds = $seconds - ($minutes * $a_minute);
363  if ($hours > 0)
364  {
365    $time_str = sprintf('%dh%02dm%02ds', $hours, $minutes, $seconds);
366  }
367  elsif ($minutes > 0)
368  {
369    $time_str = sprintf('%dm%02ds', $minutes, $seconds);
370  }
371  else
372  {
373    $time_str = $seconds . 's';
374  }
375  return $time_str;
376}
Note: See TracBrowser for help on using the browser.