Changeset 29663 for gs2-extensions

Show
Ignore:
Timestamp:
19.12.2014 12:30:03 (5 years ago)
Author:
jmt12
Message:

Supporting grayscale printing, fixing mismatched tags and speechmarks, and data locality and incomplete import occurances more obvious

Files:
1 modified

Legend:

Unmodified
Added
Removed
  • gs2-extensions/parallel-building/trunk/src/bin/script/generate_gantt.pl

    r28358 r29663  
    1414# - configurables 
    1515my $chart_width = 1600; 
     16# - any video more than 95% complete is probably complete with rounding errors 
     17my $complete_threshold = 95; 
     18 
    1619my $debug = 0; 
     20my $color_master = 'blue'; 
     21my $color_worker = 'green'; 
     22my $color_nlocal = 'red'; 
     23my $disable_header = 0; 
     24my $max_worker_count = 0; 
    1725# - globals 
    1826my $chart_count = 0; 
     
    4048    $chart_width = $value; 
    4149  } 
     50  elsif ($option eq '-grayscale') 
     51  { 
     52    $color_master = '#1D1D1D'; 
     53    $color_worker = '#969696'; 
     54    $color_nlocal = '#4C4C4C'; 
     55  } 
     56  elsif ($option eq '-noheader') 
     57  { 
     58    $disable_header = 1; 
     59  } 
     60  elsif ($option eq '-maxworkers') 
     61  { 
     62    if (!defined $ARGV[0]) 
     63    { 
     64      &printUsage('Error! No maxworkers value specified'); 
     65    } 
     66    my $value = shift(@ARGV); 
     67    if ($value !~ /^\d+$/) 
     68    { 
     69      &printUsage('Error! Maxworkers not a number'); 
     70    } 
     71    $max_worker_count = $value; 
     72  } 
    4273  else 
    4374  { 
     
    4677} 
    4778print "Chart Width: " . $chart_width . "px\n"; 
     79print "Grayscale? " . ($debug ? 'Yes' : 'No') . "\n"; 
    4880print "Debug? " . ($debug ? 'Yes' : 'No') . "\n"; 
    4981print "===================================\n\n"; 
     
    149181        my $filepath = $parts[8]; 
    150182        $filepath =~ s/^\s+|\s+$//g; 
    151         my $percent_complete = $parts[9]; 
    152         chomp($percent_complete); 
     183        my $percent_complete = 'NA'; 
     184        if (defined($parts[9])) 
     185        { 
     186          $percent_complete = $parts[9]; 
     187          chomp($percent_complete); 
     188          if ($percent_complete >= $complete_threshold) 
     189          { 
     190            $percent_complete = 'NA'; 
     191          } 
     192        } 
    153193        $import_dir = &longestCommonPath($filepath, $import_dir); 
    154194        my $start_time = $parts[3]; 
     
    175215  print HTMLOUT '<head>' . "\n"; 
    176216  print HTMLOUT '<style type="text/css">' . "\n"; 
     217  print HTMLOUT "body {margin:0px;padding:4px}\n"; 
    177218  print HTMLOUT 'div.thread {position:relative}' . "\n"; 
    178219  print HTMLOUT 'div.master {border:1px solid gray;color:white;font-weight:bold}' . "\n"; 
    179   print HTMLOUT 'div.worker {border:1px solid black;background-color:green;color:white;font-weight:bold;margin-bottom:1px;}' . "\n"; 
     220  print HTMLOUT 'div.worker {border:1px solid black;background-color:' . $color_worker . ';color:white;font-weight:bold;margin-bottom:1px;}' . "\n"; 
    180221  print HTMLOUT 'div.time {font-size:smaller;font-weight:normal}' . "\n"; 
    181   print HTMLOUT 'div.job {background-color:transparent;color:black;border:1px solid black;display:block;font-size:smaller;position:relative;text-align:center;overflow:hidden;margin-bottom:1px;}' . "\n"; 
     222  print HTMLOUT 'div.job {background-color:transparent;color:black;border:1px solid black;display:block;font-size:smaller;font-weight:normal;position:relative;text-align:left;margin-bottom:1px;}' . "\n"; 
    182223  print HTMLOUT 'span.process {z-index:-1;background-color:#C7C7C7;position:absolute}' . "\n"; 
    183   print HTMLOUT 'span.label {z-index:1;background-color:transparent;overflow:hidden;white-space:nowrap;}' . "\n"; 
     224  print HTMLOUT 'div.label {z-index:1;background-color:transparent;white-space:nowrap;text-align:center}' . "\n"; 
    184225  print HTMLOUT "th {text-align:left}\n"; 
     226  print HTMLOUT "tr.toprule th,tr.toprule td {border-top:2px solid black;width:17%}\n"; 
     227  print HTMLOUT "tr.bottomrule th,tr.bottomrule td {border-bottom:2px solid black}\n"; 
    185228  print HTMLOUT '</style>' . "\n"; 
    186229  print HTMLOUT '</head>' . "\n"; 
    187230  print HTMLOUT "<body>\n"; 
    188   print HTMLOUT "<h2>Statistics</h2>\n"; 
    189   print HTMLOUT "<table style=\"width:100%;\">\n"; 
     231  ##print HTMLOUT "<h2>Parallel Import Timing Chart</h2>\n"; 
     232  print HTMLOUT "<table style=\"border-collapse:collapse;width:" . $chart_width . "px;"; 
     233  if ($disable_header) 
     234  { 
     235    print HTMLOUT "display:none;"; 
     236  } 
     237  print HTMLOUT "\">\n"; 
    190238 
    191239  my $total_duration = $timing_data->{'M'}->{'E'} - $timing_data->{'M'}->{'S'}; 
     
    206254        my $process_end = $timing_data->{$worker_id}->{'F'}->{$job_start}->{'PE'}; 
    207255        my $job_end = $timing_data->{$worker_id}->{'F'}->{$job_start}->{'E'}; 
    208         if ($process_start == 0 || $process_end == 0 || $job_end == 0) 
     256        my $percent_complete = $timing_data->{$worker_id}->{'F'}->{$job_start}->{'PC'}; 
     257        if ($process_start == 0 || $process_end == 0 || $job_end == 0 || ($percent_complete =~ /^\d+$/ && $percent_complete < $complete_threshold)) 
    209258        { 
    210259          $problem_files++; 
     
    255304  my $avg_cpu_time = int(($total_process_time / $file_count) + 0.5); 
    256305 
    257   print HTMLOUT "<tr>\n"; 
    258   print HTMLOUT ' <th style="width:12%;">Import Directory:</th><td style="width:22%;" colspan="5">' . $import_dir . "</td>\n"; 
     306  print HTMLOUT "<tr class=\"toprule\">\n"; 
     307  print HTMLOUT ' <th style="width:17%;">Import Directory:</th><td style="width:83%;" colspan="5">' . $import_dir . "</td>\n"; 
    259308  print HTMLOUT "</tr>\n"; 
    260309 
    261310  print HTMLOUT "<tr>\n"; 
    262311  my ($sec, $min, $hour, $day, $month, $year) = (localtime($timing_data->{'M'}->{'S'}))[0,1,2,3,4,5]; 
    263   print HTMLOUT ' <th style="width:11%;">Start Time:</th><td style="width:22%;">' . sprintf('%04d%s%02d %02d:%02d:%02d', ($year+1900), $months[$month], $day, $hour, $min, $sec) . "</td>\n"; 
     312  print HTMLOUT ' <th>Start Time:</th><td>' . sprintf('%04d%s%02d %02d:%02d:%02d', ($year+1900), $months[$month], $day, $hour, $min, $sec) . "</td>\n"; 
    264313  ($sec, $min, $hour, $day, $month, $year) = (localtime($timing_data->{'M'}->{'E'}))[0,1,2,3,4,5]; 
    265   print HTMLOUT ' <th style="width:11%;">End Time:</th><td style="width:22%;">' . sprintf('%04d%s%02d %02d:%02d:%02d', ($year+1900), $months[$month], $day, $hour, $min, $sec) . "</td>\n"; 
     314  print HTMLOUT ' <th>End Time:</th><td>' . sprintf('%04d%s%02d %02d:%02d:%02d', ($year+1900), $months[$month], $day, $hour, $min, $sec) . "</td>\n"; 
    266315  print HTMLOUT "  <th>Processing Time:</th><td>" . &renderTime($total_duration) . "</td>\n"; 
    267316  print HTMLOUT "</tr>\n"; 
     
    280329 
    281330  print HTMLOUT "<tr>\n"; 
    282   print HTMLOUT "  <th>Average Processing Time:</th><td>" . &renderTime($avg_processing_time) . "</td>\n"; 
    283   print HTMLOUT "  <th>Average File IO Time:</th><td>" . &renderTime($avg_io_time) . "</td>\n"; 
    284   print HTMLOUT "  <th>Average File CPU Time:</th><td>" . &renderTime($avg_cpu_time) . "</td>\n"; 
     331  print HTMLOUT "  <th>Avg Processing Time:</th><td>" . &renderTime($avg_processing_time) . "</td>\n"; 
     332  print HTMLOUT "  <th>Avg File IO Time:</th><td>" . &renderTime($avg_io_time) . "</td>\n"; 
     333  print HTMLOUT "  <th>Avg File CPU Time:</th><td>" . &renderTime($avg_cpu_time) . "</td>\n"; 
    285334  print HTMLOUT "</tr>\n"; 
    286335 
    287   print HTMLOUT "<tr>\n"; 
     336  print HTMLOUT "<tr class=\"bottomrule\">\n"; 
    288337  print HTMLOUT "  <th>Fastest File:</th><td>" . &renderTime($fastest_file) . "</td>\n"; 
    289338  print HTMLOUT "  <th>Slowest File:</th><td>" . &renderTime($slowest_file) . "</td>\n"; 
     
    299348 
    300349  print HTMLOUT "</table>\n"; 
    301   print HTMLOUT "<hr />\n"; 
    302   print HTMLOUT "<h2>Timing Chart (Gantt)</h2>\n"; 
    303350  print HTMLOUT renderLine($chart_width, $timing_data->{'M'}->{'S'}, $timing_data->{'M'}->{'E'}, 'master', $timing_data->{'M'}->{'N'}, $timing_data->{'M'}->{'S'}, $timing_data->{'M'}->{'E'}, {}, $data_locality); 
     351  my $worker_count = 0; 
    304352  foreach my $worker_id (nsort keys %{$timing_data}) 
    305353  { 
    306     if ($worker_id ne 'M') 
    307     { 
    308       my $data = $timing_data->{$worker_id}; 
    309       print HTMLOUT renderLine($chart_width, $timing_data->{'M'}->{'S'}, $timing_data->{'M'}->{'E'}, 'worker', $worker_id . ' [' . $data->{'N'} . ']', $data->{'S'}, $data->{'E'}, $data->{'F'}, 2); #$data_locality); 
    310     } 
    311   } 
    312   print HTMLOUT '<div>' . "\n"; 
     354    if ($max_worker_count < 1 || $worker_count <= $max_worker_count) 
     355    { 
     356      if ($worker_id ne 'M') 
     357      { 
     358        my $data = $timing_data->{$worker_id}; 
     359        print HTMLOUT renderLine($chart_width, $timing_data->{'M'}->{'S'}, $timing_data->{'M'}->{'E'}, 'worker', $worker_id . ' [' . $data->{'N'} . ']', $data->{'S'}, $data->{'E'}, $data->{'F'}, 2); #$data_locality); 
     360        $worker_count++; 
     361      } 
     362    } 
     363  } 
     364  print HTMLOUT '</div>' . "\n"; 
    313365  print HTMLOUT "</body>\n"; 
    314366  print HTMLOUT "</html>"; 
     
    441493  if ($class eq 'master') 
    442494  { 
    443     $html .= '<div style="background-color:blue;margin-bottom:1px">'; 
     495    $html .= '<div style="background-color:' . $color_master . ';margin-bottom:1px">'; 
    444496  } 
    445497  $html .= '<div class="time" style="display:table-cell">' . &renderTime($rtstart) . '</div><div style="display:table-cell;padding-left:20px;width:100%;">' . ucfirst($class) . ': ' . $tname . '</div><div class="time" style="display:table-cell">' . renderTime($rtend) . '</div></div>'; 
     
    470522    my $cpu_percent = int((($rpwidth / $jwidth) * 100) + 0.5); 
    471523    $html .= '<div class="job" style="left:' . $jleft . 'px;width:' . $jwidth . 'px;'; 
    472     print "Data Locality? " . $data_locality . " DL? " . $jobs->{$jstart}->{'DL'} . "\n"; 
     524    ###rint "Data Locality? " . $data_locality . " DL? " . $jobs->{$jstart}->{'DL'} . "\n"; 
    473525    if ($data_locality > 1 && $jobs->{$jstart}->{'DL'} != 1) 
    474526    { 
    475       $html .= 'border:1px dashed black;'; 
    476     } 
    477     $html .= '" title="FN:' . $jobs->{$jstart}->{'FN'} . ', S:' . &renderTime($rjstart) . ', E:' . &renderTime($rjend) . ', CPU: ' . $cpu_percent . '% [' . &renderTime($io_duration) . ', ' . &renderTime($cpu_duration) . ', PC: ' . $jobs->{$jstart}->{'PC'} . '%]"><span class="process" style="left:' . $rpleft . 'px;width:' . $rpwidth . 'px">&nbsp;</span><span class="label"'; 
     527      $html .= 'border:1px solid #C7C7C7;'; 
     528    } 
     529    $html .= '" title="FN:' . $jobs->{$jstart}->{'FN'} . ', S:' . &renderTime($rjstart) . ', E:' . &renderTime($rjend) . ', CPU: ' . $cpu_percent . '% [' . &renderTime($io_duration) . ', ' . &renderTime($cpu_duration) . ', PC: ' . $jobs->{$jstart}->{'PC'} . '%]"><span class="process" style="left:' . $rpleft . 'px;width:' . $rpwidth . 'px">&nbsp;</span><div class="label" style="width:' . $jwidth; 
    478530    if ($data_locality > 1 && $jobs->{$jstart}->{'DL'} != 1) 
    479531    { 
    480       $html .= ' style="color:#FF0000"'; 
    481     } 
    482     $html .=  '>' . $jobs->{$jstart}->{'FN'}; 
     532      $html .= ';color:' . $color_nlocal; 
     533    } 
     534    $html .=  '">' . $jobs->{$jstart}->{'FN'}; 
    483535    if ($jobs->{$jstart}->{'PC'} ne 'NA') 
    484536    { 
    485       $html .= ' <small>[' . $jobs->{$jstart}->{'PC'} . '%]</small>'; 
     537      $html .= ' <b>[Incomplete! ' . $jobs->{$jstart}->{'PC'} . '%]</b>'; 
    486538    } 
    487539    if ($data_locality > 1 && $jobs->{$jstart}->{'DL'} != 1) 
     
    489541      $html .= ' [NL]'; 
    490542    } 
    491     $html .= '</span></div>'; 
     543    $html .= '</div></div>'; 
    492544  } 
    493545  return $html;