source: gs2-extensions/parallel-building/trunk/src/bin/script/generate_gantt.pl@ 28189

Last change on this file since 28189 was 28189, checked in by jmt12, 11 years ago

Replace the newer (and faster) while(@file) loop with the older (and more reliable) foreach() loop
  • Property svn:executable set to *
File size: 16.8 KB
Line 
1#!/usr/bin/perl
2
3# Pragma
4use strict;
5use warnings;
6
7# Modules
8use Sort::Naturally;
9use POSIX qw(floor strftime);
10
11print "\n===== Generate Timing (GANTT) =====\n";
12
13# 0. Init
14# - configurables
15my $chart_width = 1600;
16my $debug = 0;
17# - globals
18my $chart_count = 0;
19my @months = ("Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec");
20
21# 1. Parse options
22while (defined $ARGV[0] && $ARGV[0] =~ /^-/)
23{
24 my $option = shift(@ARGV);
25 if ($option eq '-debug')
26 {
27 $debug = 1;
28 }
29 elsif ($option eq '-width')
30 {
31 if (!defined $ARGV[0])
32 {
33 &printUsage('Error! No width value specified');
34 }
35 my $value = shift(@ARGV);
36 if ($value !~ /^\d+$/)
37 {
38 &printUsage('Error! Chart width not a number');
39 }
40 $chart_width = $value;
41 }
42 else
43 {
44 &printUsage('Error! Unknown option: ' . $option);
45 }
46}
47print "Chart Width: " . $chart_width . "px\n";
48print "Debug? " . ($debug ? 'Yes' : 'No') . "\n";
49print "===================================\n\n";
50
51# 2. Search for valid directories (containing timing.csv)
52while (defined $ARGV[0])
53{
54 my $dir = shift(@ARGV);
55 if (!-d $dir)
56 {
57 &printUsage('Error! Not a directory: ' . $dir);
58 }
59 &searchForTimingCSV($dir);
60}
61
62# 3. Done
63print "Complete!\n\n";
64print "===================================\n";
65print 'Generated ' . $chart_count . " charts\n";
66print "===================================\n\n";
67exit;
68## main() ##
69
70
71## @function searchForTimingCSV()
72#
73sub searchForTimingCSV
74{
75 my $dir = shift(@_);
76 # For every directory where we find a timing.csv we generate a gantt chart
77 my $timing_path = &filenameCat($dir, 'timing.csv');
78 if (-e $timing_path)
79 {
80 &generateChart($dir, $timing_path);
81 }
82 # We also recursively search for other directories containing timing.csv's
83 opendir(my $dh, $dir) or &printError('Failed to open directory for reading: ' . $dir);
84 my @files = readdir($dh);
85 foreach my $file (@files)
86 {
87 if ($file !~ /^\./)
88 {
89 my $path = &filenameCat($dir, $file);
90 if (-d $path)
91 {
92 &searchForTimingCSV($path);
93 }
94 }
95 }
96}
97## searchForTimingCSV() ##
98
99
100## @function generateChart()
101#
102sub generateChart
103{
104 my $dir = shift(@_);
105 my $timing_csv_path = shift(@_);
106 my $import_dir;
107 my ($epoc) = $dir =~ /(\d+)$/;
108 my $gantt_path = $dir . '/' . $epoc . '-gantt.html';
109
110 print ' * Generating chart for: ' . $dir . "\n";
111 print ' - timing file: ' . $timing_csv_path . "\n";
112 print ' - gantt chart: ' . $gantt_path . "\n";
113
114 # Read in timing.csv and parse information into data structure
115 print ' - parsing timing.csv... ';
116 my $timing_data = {};
117 my $id_2_worker_id = {};
118 if (open(TIN, '<:utf8', $timing_csv_path))
119 {
120 my $line;
121 while ($line = <TIN>)
122 {
123 my @parts = split(/,/, $line);
124 if ($parts[1] eq 'M0')
125 {
126 $timing_data->{'M'} = {'N'=>$parts[2], 'S'=>$parts[3], 'E'=>$parts[4]};
127 }
128 elsif ($parts[1] =~ /W\d+/)
129 {
130 my $worker_id = $parts[1];
131 my $hostname = $parts[2];
132 # Alter the worker name for compute nodes so they can be naturally
133 # sorted
134 if ($hostname =~ /compute-0-(\d+)/)
135 {
136 $worker_id = 'W' . $1;
137 }
138 $timing_data->{$worker_id} = {'N'=>$hostname, 'S'=>$parts[3], 'E'=>$parts[4], 'F'=>{}};
139 $id_2_worker_id->{$parts[0]} = $worker_id;
140 }
141 elsif ($parts[1] =~ /T\d+/)
142 {
143 my $worker_id = $id_2_worker_id->{$parts[7]};
144 my $stop = $parts[4];
145 my $filepath = $parts[8];
146 $filepath =~ s/^\s+|\s+$//g;
147 my $percent_complete = $parts[9];
148 chomp($percent_complete);
149 $import_dir = &longestCommonPath($filepath, $import_dir);
150 my $start_time = $parts[3];
151 while (defined $timing_data->{$worker_id}->{'F'}->{$start_time})
152 {
153 $start_time += 0.000001;
154 }
155 $timing_data->{$worker_id}->{'F'}->{$start_time} = {'FN'=>$filepath, 'S'=>$parts[3], 'PS'=>($stop - $parts[5]), 'PE'=>$stop, 'E'=>$stop, 'DL'=>$parts[6], 'PC'=>$percent_complete};
156 }
157 }
158 close(TIN);
159 }
160 else
161 {
162 die('Error! Failed to open file for reading: ' . $timing_csv_path);
163 }
164 my $number_of_workers = scalar(keys(%{$id_2_worker_id}));;
165 print "Done\n";
166
167 # 3. Produce pretty HTML chart of timing information including jobs
168 print " - generating timing information as chart in HTML... ";
169 open(HTMLOUT, '>:utf8', $gantt_path) or die('Error! Failed to open file for writing: gantt.html');
170 print HTMLOUT "<html>\n";
171 print HTMLOUT '<head>' . "\n";
172 print HTMLOUT '<style type="text/css">' . "\n";
173 print HTMLOUT 'div.thread {position:relative}' . "\n";
174 print HTMLOUT 'div.master {border:1px solid gray;color:white;font-weight:bold}' . "\n";
175 print HTMLOUT 'div.worker {border:1px solid black;background-color:green;color:white;font-weight:bold;margin-bottom:1px;}' . "\n";
176 print HTMLOUT 'div.time {font-size:smaller;font-weight:normal}' . "\n";
177 print HTMLOUT 'div.job {background-color:transparent;color:black;border:1px solid black;display:block;font-size:smaller;position:relative;text-align:center;overflow:hidden;margin-bottom:1px;}' . "\n";
178 print HTMLOUT 'span.process {z-index:-1;background-color:#C7C7C7;position:absolute}' . "\n";
179 print HTMLOUT 'span.label {z-index:1;background-color:transparent;overflow:hidden;white-space:nowrap;}' . "\n";
180 print HTMLOUT "th {text-align:left}\n";
181 print HTMLOUT '</style>' . "\n";
182 print HTMLOUT '</head>' . "\n";
183 print HTMLOUT "<body>\n";
184 print HTMLOUT "<h2>Statistics</h2>\n";
185 print HTMLOUT "<table style=\"width:100%;\">\n";
186
187 my $total_duration = $timing_data->{'M'}->{'E'} - $timing_data->{'M'}->{'S'};
188 my $file_count = 0;
189 my $data_locality = 0;
190 my $total_io_time = 0;
191 my $total_process_time = 0;
192 my $fastest_file = 0;
193 my $slowest_file = 0;
194 my $problem_files = 0;
195 foreach my $worker_id (keys %{$timing_data})
196 {
197 if ($worker_id ne 'M')
198 {
199 foreach my $job_start ( keys %{$timing_data->{$worker_id}->{'F'}} )
200 {
201 my $process_start = $timing_data->{$worker_id}->{'F'}->{$job_start}->{'PS'};
202 my $process_end = $timing_data->{$worker_id}->{'F'}->{$job_start}->{'PE'};
203 my $job_end = $timing_data->{$worker_id}->{'F'}->{$job_start}->{'E'};
204 if ($process_start == 0 || $process_end == 0 || $job_end == 0)
205 {
206 $problem_files++;
207 }
208 else
209 {
210 my $io_duration = ($process_start - $job_start) + ($job_end - $process_end);
211 my $process_duration = $process_end - $process_start;
212 my $total_duration = $io_duration + $process_duration;
213 &debugPrint("filename: " . $timing_data->{$worker_id}->{'F'}->{$job_start}->{'FN'});
214 &debugPrint("start: $job_start ps: $process_start pe: $process_end end: $job_end");
215 &debugPrint("io: $io_duration process: $process_duration duration: $total_duration");
216 # Running stats
217 $total_io_time += $io_duration;
218 $total_process_time += $process_duration;
219 if ($fastest_file == 0 || $total_duration < $fastest_file)
220 {
221 $fastest_file = $total_duration;
222 }
223 if ($slowest_file == 0 || $total_duration > $slowest_file)
224 {
225 $slowest_file = $total_duration;
226 }
227 }
228 # Shorten filename
229 if (defined $timing_data->{$worker_id}->{'F'}->{$job_start}->{'FN'} && $timing_data->{$worker_id}->{'F'}->{$job_start}->{'FN'} ne '')
230 {
231 $timing_data->{$worker_id}->{'F'}->{$job_start}->{'FN'} = substr($timing_data->{$worker_id}->{'F'}->{$job_start}->{'FN'}, length($import_dir) + 1);
232 }
233 $file_count++;
234 if ($timing_data->{$worker_id}->{'F'}->{$job_start}->{'DL'} == 1)
235 {
236 $data_locality++;
237 }
238 }
239 }
240 }
241 if ($file_count <= 0)
242 {
243 $file_count = 1;
244 }
245 if ($total_process_time <= 0)
246 {
247 $total_process_time = 1;
248 }
249 my $avg_processing_time = floor(($total_io_time + $total_process_time) / $file_count);
250 my $avg_io_time = int(($total_io_time / $file_count) + 0.5);
251 my $avg_cpu_time = int(($total_process_time / $file_count) + 0.5);
252
253 print HTMLOUT "<tr>\n";
254 print HTMLOUT ' <th style="width:12%;">Import Directory:</th><td style="width:22%;" colspan="5">' . $import_dir . "</td>\n";
255 print HTMLOUT "</tr>\n";
256
257 print HTMLOUT "<tr>\n";
258 my ($sec, $min, $hour, $day, $month, $year) = (localtime($timing_data->{'M'}->{'S'}))[0,1,2,3,4,5];
259 print HTMLOUT ' <th style="width:11%;">Start Time:</th><td style="width:22%;">' . sprintf('%04d%s%02d %02d:%02d:%02d', ($year+1900), $months[$month], $day, $hour, $min, $sec) . "</td>\n";
260 ($sec, $min, $hour, $day, $month, $year) = (localtime($timing_data->{'M'}->{'E'}))[0,1,2,3,4,5];
261 print HTMLOUT ' <th style="width:11%;">End Time:</th><td style="width:22%;">' . sprintf('%04d%s%02d %02d:%02d:%02d', ($year+1900), $months[$month], $day, $hour, $min, $sec) . "</td>\n";
262 print HTMLOUT " <th>Processing Time:</th><td>" . &renderTime($total_duration) . "</td>\n";
263 print HTMLOUT "</tr>\n";
264
265 print HTMLOUT "<tr>\n";
266 print HTMLOUT " <th>Processing Threads:</th><td>" . $number_of_workers . "</td>\n";
267 print HTMLOUT " <th>Files Processed:</th><td>" . $file_count . "</td>\n";
268 print HTMLOUT " <th>Problem Files:</th><td>" . $problem_files . "</td>\n";
269 print HTMLOUT "</tr>\n";
270
271 print HTMLOUT "<tr>\n";
272 print HTMLOUT ' <th>Serial Processing Time:</th><td>' . &renderTime($total_process_time) . "</td>\n";
273 print HTMLOUT ' <th>Serial IO Time:</th><td>' . &renderTime($total_io_time) . "</td>\n";
274 print HTMLOUT ' <th>IO Percentage:</th><td>' . sprintf('%d%%', (($total_io_time / $total_process_time) * 100)) . "</td>\n";
275 print HTMLOUT "</tr>\n";
276
277 print HTMLOUT "<tr>\n";
278 print HTMLOUT " <th>Average Processing Time:</th><td>" . &renderTime($avg_processing_time) . "</td>\n";
279 print HTMLOUT " <th>Average File IO Time:</th><td>" . &renderTime($avg_io_time) . "</td>\n";
280 print HTMLOUT " <th>Average File CPU Time:</th><td>" . &renderTime($avg_cpu_time) . "</td>\n";
281 print HTMLOUT "</tr>\n";
282
283 print HTMLOUT "<tr>\n";
284 print HTMLOUT " <th>Fastest File:</th><td>" . &renderTime($fastest_file) . "</td>\n";
285 print HTMLOUT " <th>Slowest File:</th><td>" . &renderTime($slowest_file) . "</td>\n";
286 if ($data_locality > 0)
287 {
288 print HTMLOUT " <th>Data Locality:</th><td>" . sprintf('%d%% [%d out of %d]', (($data_locality / $file_count) * 100), $data_locality, $file_count) . "</td>\n";
289 }
290 else
291 {
292 print HTMLOUT " <th>Data Locality:</th><td><i>Not Applicable</i></td>\n";
293 }
294 print HTMLOUT "</tr>\n";
295
296 print HTMLOUT "</table>\n";
297 print HTMLOUT "<hr />\n";
298 print HTMLOUT "<h2>Timing Chart (Gantt)</h2>\n";
299 print HTMLOUT renderLine($chart_width, $timing_data->{'M'}->{'S'}, $timing_data->{'M'}->{'E'}, 'master', $timing_data->{'M'}->{'N'}, $timing_data->{'M'}->{'S'}, $timing_data->{'M'}->{'E'}, {}, $data_locality);
300 foreach my $worker_id (nsort keys %{$timing_data})
301 {
302 if ($worker_id ne 'M')
303 {
304 my $data = $timing_data->{$worker_id};
305 print HTMLOUT renderLine($chart_width, $timing_data->{'M'}->{'S'}, $timing_data->{'M'}->{'E'}, 'worker', $worker_id . ' [' . $data->{'N'} . ']', $data->{'S'}, $data->{'E'}, $data->{'F'}, $data_locality);
306 }
307 }
308 print HTMLOUT '<div>' . "\n";
309 print HTMLOUT "</body>\n";
310 print HTMLOUT "</html>";
311 close(HTMLOUT);
312 print "Done!\n\n";
313 $chart_count++;
314}
315## generateChart() ##
316
317
318## @function debugPrint()
319#
320sub debugPrint
321{
322 my $msg = shift(@_);
323 if ($debug)
324 {
325 print STDERR '[DEBUG] ' . $msg . "\n";
326 }
327}
328## debugPrint() ##
329
330
331## @function filenameCat
332#
333sub filenameCat
334{
335 my $path = join('/', @_);
336 $path =~ s/[\/\\]+/\//g;
337 return $path;
338}
339## filenameCat() ##
340
341
342## @function printError()
343#
344sub printError
345{
346 my $msg = shift(@_);
347 die('Error! ' . $msg . "\n\n");
348}
349## printError() ##
350
351
352## @function printUsage()
353#
354sub printUsage
355{
356 my $msg = shift(@_);
357 if (defined $msg)
358 {
359 print 'Error! ' . $msg . "\n";
360 }
361 die("Usage: generate_gantt.pl [-width <width in pixels>] <dir> [<dir> ...]\n\n");
362}
363## printUsage() ##
364
365
366## @function longestCommonPath
367#
368sub longestCommonPath
369{
370 my ($path_new, $path_current) = @_;
371 my $result = '';
372 if (defined $path_current)
373 {
374 # Hide protocol before we split by slash
375 $path_new =~ s/:\/\//:/;
376 $path_current =~ s/:\/\//:/;
377 my @path_new_parts = split(/\//, $path_new);
378 my @path_current_parts = split(/\//, $path_current);
379 my @path_parts;
380 for (my $i = 0; $i < scalar(@path_current_parts); $i++)
381 {
382 if ($path_current_parts[$i] eq $path_new_parts[$i])
383 {
384 push(@path_parts, $path_new_parts[$i]);
385 }
386 else
387 {
388 last;
389 }
390 }
391 $result = &filenameCat(@path_parts);
392 # Restore protocol
393 $result =~ s/:/:\/\//;
394 }
395 else
396 {
397 $result = $path_new;
398 }
399 return $result;
400}
401## longestCommonPath() ##
402
403
404## @function renderLine()
405#
406sub renderLine
407{
408 my ($table_width, $start, $end, $class, $tname, $tstart, $tend, $jobs, $data_locality) = @_;
409 &debugPrint("renderLine($table_width, $start, $end, $class, $tname, $tstart, $tend, <jobs>)");
410 # All timings need to be relative to 0 (relative start)
411 my $duration = $end - $start;
412 my $rtstart = $tstart - $start;
413 my $rtend = $tend - $start;
414 # We need to scale these depending on the timing of this thread relative to
415 # the master thread
416 my $width = $chart_width;
417 my $left = 0;
418 if ($start != $tstart)
419 {
420 my $left_offset_percent = $rtstart / $duration;
421 $left = $left_offset_percent * $table_width;
422 }
423 # - subtract any left offset from width
424 $width = $width - $left;
425 # - right offset directly subtracted from width
426 if ($end != $tend)
427 {
428 my $right_offset_percent = ($duration - $rtend) / $duration;
429 my $right = $right_offset_percent * $table_width;
430 $width = $width - $right;
431 }
432 # Round things off (simple dutch rounding)
433 $left = int($left + 0.5);
434 $width = int($width + 0.5);
435 # Output the bar for this master/worker
436 my $html = '<div class="thread ' . $class . '" style="left:' . $left . 'px;width:' . $width . 'px;">';
437 if ($class eq 'master')
438 {
439 $html .= '<div style="background-color:blue;margin-bottom:1px">';
440 }
441 $html .= '<div class="time" style="display:table-cell">' . &renderTime($rtstart) . '</div><div style="display:table-cell;padding-left:20px;width:100%;">' . ucfirst($class) . ': ' . $tname . '</div><div class="time" style="display:table-cell">' . renderTime($rtend) . '</div></div>';
442 my $previous_jright = 0;
443 foreach my $jstart (sort keys %{$jobs})
444 {
445 my $rjstart = $jstart - $start;
446 my $rpstart = $jobs->{$jstart}->{'PS'} - $start;
447 my $rpend = $jobs->{$jstart}->{'PE'} - $start;
448 my $rjend = $jobs->{$jstart}->{'E'} - $start;
449 my $jduration = $jobs->{$jstart}->{'E'} - $jstart;
450 my $io_duration = $rpstart - $rjstart;
451 my $cpu_duration = $rpend - $rpstart;
452 # Scale Job co-ordinates
453 my $jleft_percent = $rjstart / $duration;
454 my $jleft = int(($jleft_percent * $table_width) + 0.5);
455 my $jwidth_percent = $jduration / $duration;
456 # -2 for left and right 1 pixel border
457 my $jwidth = int(($jwidth_percent * $table_width) + 0.5) - 2;
458 if ($jleft + $jwidth > $left + $width)
459 {
460 $jwidth = ($left + $width) - $jleft;
461 }
462 # Then scale process timings within that!
463 my $rpleft_percent = ($rpstart - $rjstart) / $duration;
464 my $rpleft = int(($rpleft_percent * $table_width) + 0.5);
465 my $rpwidth = $jwidth - $rpleft;
466 my $cpu_percent = int((($rpwidth / $jwidth) * 100) + 0.5);
467 $html .= '<div class="job" style="left:' . $jleft . 'px;width:' . $jwidth . 'px;';
468 if ($data_locality > 1 && $jobs->{$jstart}->{'DL'} != 1)
469 {
470 $html .= 'border:1px dashed black;';
471 }
472 $html .= '" title="FN:' . $jobs->{$jstart}->{'FN'} . ', S:' . &renderTime($rjstart) . ', E:' . &renderTime($rjend) . ', CPU: ' . $cpu_percent . '% [' . &renderTime($io_duration) . ', ' . &renderTime($cpu_duration) . ', PC: ' . $jobs->{$jstart}->{'PC'} . '%]"><span class="process" style="left:' . $rpleft . 'px;width:' . $rpwidth . 'px">&nbsp;</span><span class="label"';
473 if ($data_locality > 1 && $jobs->{$jstart}->{'DL'} != 1)
474 {
475 $html .= ' style="color:#FF0000"';
476 }
477 $html .= '>' . $jobs->{$jstart}->{'FN'};
478 if ($jobs->{$jstart}->{'PC'} ne 'NA')
479 {
480 $html .= ' <small>[' . $jobs->{$jstart}->{'PC'} . '%]</small>';
481 }
482 if ($data_locality > 1 && $jobs->{$jstart}->{'DL'} != 1)
483 {
484 $html .= ' [NL]';
485 }
486 $html .= '</span></div>';
487 }
488 return $html;
489}
490## renderLine() ##
491
492
493## @function renderTime()
494#
495sub renderTime
496{
497 my ($seconds) = @_;
498 my $time_str = '';
499 # determine how many hours
500 my $an_hour = 60 * 60;
501 my $hours = floor($seconds / $an_hour);
502 $seconds = $seconds - ($hours * $an_hour);
503 my $a_minute = 60;
504 my $minutes = floor($seconds / $a_minute);
505 $seconds = $seconds - ($minutes * $a_minute);
506 if ($hours > 0)
507 {
508 $time_str = sprintf('%dh%02dm%02ds', $hours, $minutes, $seconds);
509 }
510 else
511 {
512 $time_str = sprintf('%dm%02ds', $minutes, $seconds);
513 }
514 return $time_str;
515}
Note: See TracBrowser for help on using the repository browser.