Ignore:
Timestamp:
2013-06-10T16:27:49+12:00 (11 years ago)
Author:
jmt12
Message:

Adding statistics about data locality, and highlighting tasks where file was non-local

File:
1 edited

Legend:

Unmodified
Added
Removed

  • gs2-extensions/parallel-building/trunk/src/bin/script/generate_gantt.pl

    r27559 r27590  
    3535}
    3636print "Chart Width: " . $chart_width . "px\n";
     37
     38my ($epoc) = $dir =~ /(\d+)$/;
     39my $gantt_path = $dir . '/' . $epoc . '-gantt.html';
     40print "Gantt Chart: " . $gantt_path . "\n";
     41
    3742print "===================================\n\n";
    3843
    … …  
    6267      $filepath =~ s/^\s+|\s+$//g;
    6368      $import_dir = &longestCommonPath($filepath, $import_dir);
    64       $timing_data->{$worker_id}->{'F'}->{$parts[3]} = {'FN'=>$filepath, 'PS'=>($stop - $parts[5]), 'PE'=>$stop, 'E'=>$stop};
     69      $timing_data->{$worker_id}->{'F'}->{$parts[3]} = {'FN'=>$filepath, 'PS'=>($stop - $parts[5]), 'PE'=>$stop, 'E'=>$stop, 'DL'=>$parts[6]};
    6570    }
    6671  }
    … …  
    7580# 3. Produce pretty HTML chart of timing information including jobs
    7681print " * Generating timing information as HTML... ";
    77 open(HTMLOUT, '>:utf8', $dir . '/gantt.html') or die('Error! Failed to open file for writing: gantt.html');
     82open(HTMLOUT, '>:utf8', $gantt_path) or die('Error! Failed to open file for writing: gantt.html');
    7883print HTMLOUT "<html>\n";
    7984print HTMLOUT '<head>' . "\n";
    … …  
    95100my $total_duration = $timing_data->{'M'}->{'E'} - $timing_data->{'M'}->{'S'};
    96101my $file_count = 0;
     102my $data_locality = 0;
    97103my $total_io_time = 0;
    98104my $total_process_time = 0;
    … …  
    139145      }
    140146      $file_count++;
     147      if ($timing_data->{$worker_id}->{'F'}->{$job_start}->{'DL'} == 1)
     148      {
     149        $data_locality++;
     150      }
    141151    }
    142152  }
    143153}
    144154my $avg_processing_time = floor(($total_io_time + $total_process_time) / $file_count);
     155my @months = ("Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec");
    145156
    146157print HTMLOUT "<tr><th>Import Directory:</th><td>" . $import_dir . "</td></tr>\n";
     158my ($sec, $min, $hour, $day, $month, $year) = (localtime($timing_data->{'M'}->{'S'}))[0,1,2,3,4,5];
     159print HTMLOUT "<tr><th>Start Time:</th><td>" . sprintf('%04d%s%02d %02d:%02d:%02d', ($year+1900), $months[$month], $day, $hour, $min, $sec) . "</td></tr>\n";
    147160print HTMLOUT "<tr><th>Processing Time:</th><td>" . &renderTime($total_duration) . "</td></tr>\n";
    148161print HTMLOUT "<tr><th>Processing Threads:</th><td>" . $number_of_workers . "</td></tr>\n";
    149162print HTMLOUT "<tr><th>Files Processed:</th><td>" . $file_count . "</td></tr>\n";
    150 print HTMLOUT "<tr><th>Problem Files:</th><td>" . $problem_files . "</td></tr>\n";
     163if ($data_locality > 0)
     164{
     165  print HTMLOUT "<tr><th>Data Locality:</th><td>" . sprintf('%d%% [%d out of %d]', (($data_locality / $file_count) * 100), $data_locality, $file_count) . "</td></tr>\n";
     166}
    151167print HTMLOUT "<tr><th>Serial Processing Time:</th><td>" . &renderTime($total_process_time) . "</td></tr>\n";
    152168print HTMLOUT "<tr><th>Serial IO Time:</th><td>" . &renderTime($total_io_time) . "</td></tr>\n";
    … …  
    154170print HTMLOUT "<tr><th>Fastest File:</th><td>" . &renderTime($fastest_file) . "</td></tr>\n";
    155171print HTMLOUT "<tr><th>Slowest File:</th><td>" . &renderTime($slowest_file) . "</td></tr>\n";
     172print HTMLOUT "<tr><th>Problem Files:</th><td>" . $problem_files . "</td></tr>\n";
    156173
    157174print HTMLOUT "</table>\n";
    … …  
    309326    my $rpwidth = $jwidth - $rpleft;
    310327    my $cpu_percent = int((($rpwidth / $jwidth) * 100) + 0.5);
    311     $html .= '<div class="job" style="left:' . $jleft . 'px;width:' . $jwidth . 'px" title="FN:' . $jobs->{$jstart}->{'FN'} . ', S:' . renderTime($rjstart) . ', E:' . renderTime($rjend) . ', CPU: ' . $cpu_percent . '%"><span class="process" style="left:' . $rpleft . 'px;width:' . $rpwidth . 'px">&nbsp;</span><span class="label">' . $jobs->{$jstart}->{'FN'} . '</span></div>';
     328    $html .= '<div class="job" style="left:' . $jleft . 'px;width:' . $jwidth . 'px;';
     329    if ($jobs->{$jstart}->{'DL'} != 1)
     330    {
     331      $html .= 'border:1px dashed black;';
     332    }
     333    $html .= '" title="FN:' . $jobs->{$jstart}->{'FN'} . ', S:' . renderTime($rjstart) . ', E:' . renderTime($rjend) . ', CPU: ' . $cpu_percent . '%"><span class="process" style="left:' . $rpleft . 'px;width:' . $rpwidth . 'px">&nbsp;</span><span class="label"';
     334    if ($jobs->{$jstart}->{'DL'} != 1)
     335    {
     336      $html .= ' style="color:#FF0000"';
     337    }
     338    $html .=  '>' . $jobs->{$jstart}->{'FN'};
     339    if ($jobs->{$jstart}->{'DL'} != 1)
     340    {
     341      $html .= ' [NL]';
     342    }
     343    $html .= '</span></div>';
    312344  }
    313345  return $html;
Note: See TracChangeset for help on using the changeset viewer.