Changeset 27590

Show
Ignore:
Timestamp:
10.06.2013 16:27:49 (6 years ago)
Author:
jmt12
Message:

Adding statistics about data locality, and highlighting tasks where file was non-local

Files:
1 modified

Legend:

Unmodified
Added
Removed
  • gs2-extensions/parallel-building/trunk/src/bin/script/generate_gantt.pl

    r27559 r27590  
    3535} 
    3636print "Chart Width: " . $chart_width . "px\n"; 
     37 
     38my ($epoc) = $dir =~ /(\d+)$/; 
     39my $gantt_path = $dir . '/' . $epoc . '-gantt.html'; 
     40print "Gantt Chart: " . $gantt_path . "\n"; 
     41 
    3742print "===================================\n\n"; 
    3843 
     
    6267      $filepath =~ s/^\s+|\s+$//g; 
    6368      $import_dir = &longestCommonPath($filepath, $import_dir); 
    64       $timing_data->{$worker_id}->{'F'}->{$parts[3]} = {'FN'=>$filepath, 'PS'=>($stop - $parts[5]), 'PE'=>$stop, 'E'=>$stop}; 
     69      $timing_data->{$worker_id}->{'F'}->{$parts[3]} = {'FN'=>$filepath, 'PS'=>($stop - $parts[5]), 'PE'=>$stop, 'E'=>$stop, 'DL'=>$parts[6]}; 
    6570    } 
    6671  } 
     
    7580# 3. Produce pretty HTML chart of timing information including jobs 
    7681print " * Generating timing information as HTML... "; 
    77 open(HTMLOUT, '>:utf8', $dir . '/gantt.html') or die('Error! Failed to open file for writing: gantt.html'); 
     82open(HTMLOUT, '>:utf8', $gantt_path) or die('Error! Failed to open file for writing: gantt.html'); 
    7883print HTMLOUT "<html>\n"; 
    7984print HTMLOUT '<head>' . "\n"; 
     
    95100my $total_duration = $timing_data->{'M'}->{'E'} - $timing_data->{'M'}->{'S'}; 
    96101my $file_count = 0; 
     102my $data_locality = 0; 
    97103my $total_io_time = 0; 
    98104my $total_process_time = 0; 
     
    139145      } 
    140146      $file_count++; 
     147      if ($timing_data->{$worker_id}->{'F'}->{$job_start}->{'DL'} == 1) 
     148      { 
     149        $data_locality++; 
     150      } 
    141151    } 
    142152  } 
    143153} 
    144154my $avg_processing_time = floor(($total_io_time + $total_process_time) / $file_count); 
     155my @months = ("Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec"); 
    145156 
    146157print HTMLOUT "<tr><th>Import Directory:</th><td>" . $import_dir . "</td></tr>\n"; 
     158my ($sec, $min, $hour, $day, $month, $year) = (localtime($timing_data->{'M'}->{'S'}))[0,1,2,3,4,5]; 
     159print HTMLOUT "<tr><th>Start Time:</th><td>" . sprintf('%04d%s%02d %02d:%02d:%02d', ($year+1900), $months[$month], $day, $hour, $min, $sec) . "</td></tr>\n"; 
    147160print HTMLOUT "<tr><th>Processing Time:</th><td>" . &renderTime($total_duration) . "</td></tr>\n"; 
    148161print HTMLOUT "<tr><th>Processing Threads:</th><td>" . $number_of_workers . "</td></tr>\n"; 
    149162print HTMLOUT "<tr><th>Files Processed:</th><td>" . $file_count . "</td></tr>\n"; 
    150 print HTMLOUT "<tr><th>Problem Files:</th><td>" . $problem_files . "</td></tr>\n"; 
     163if ($data_locality > 0) 
     164{ 
     165  print HTMLOUT "<tr><th>Data Locality:</th><td>" . sprintf('%d%% [%d out of %d]', (($data_locality / $file_count) * 100), $data_locality, $file_count) . "</td></tr>\n"; 
     166} 
    151167print HTMLOUT "<tr><th>Serial Processing Time:</th><td>" . &renderTime($total_process_time) . "</td></tr>\n"; 
    152168print HTMLOUT "<tr><th>Serial IO Time:</th><td>" . &renderTime($total_io_time) . "</td></tr>\n"; 
     
    154170print HTMLOUT "<tr><th>Fastest File:</th><td>" . &renderTime($fastest_file) . "</td></tr>\n"; 
    155171print HTMLOUT "<tr><th>Slowest File:</th><td>" . &renderTime($slowest_file) . "</td></tr>\n"; 
     172print HTMLOUT "<tr><th>Problem Files:</th><td>" . $problem_files . "</td></tr>\n"; 
    156173 
    157174print HTMLOUT "</table>\n"; 
     
    309326    my $rpwidth = $jwidth - $rpleft; 
    310327    my $cpu_percent = int((($rpwidth / $jwidth) * 100) + 0.5); 
    311     $html .= '<div class="job" style="left:' . $jleft . 'px;width:' . $jwidth . 'px" title="FN:' . $jobs->{$jstart}->{'FN'} . ', S:' . renderTime($rjstart) . ', E:' . renderTime($rjend) . ', CPU: ' . $cpu_percent . '%"><span class="process" style="left:' . $rpleft . 'px;width:' . $rpwidth . 'px">&nbsp;</span><span class="label">' . $jobs->{$jstart}->{'FN'} . '</span></div>'; 
     328    $html .= '<div class="job" style="left:' . $jleft . 'px;width:' . $jwidth . 'px;'; 
     329    if ($jobs->{$jstart}->{'DL'} != 1) 
     330    { 
     331      $html .= 'border:1px dashed black;'; 
     332    } 
     333    $html .= '" title="FN:' . $jobs->{$jstart}->{'FN'} . ', S:' . renderTime($rjstart) . ', E:' . renderTime($rjend) . ', CPU: ' . $cpu_percent . '%"><span class="process" style="left:' . $rpleft . 'px;width:' . $rpwidth . 'px">&nbsp;</span><span class="label"'; 
     334    if ($jobs->{$jstart}->{'DL'} != 1) 
     335    { 
     336      $html .= ' style="color:#FF0000"'; 
     337    } 
     338    $html .=  '>' . $jobs->{$jstart}->{'FN'}; 
     339    if ($jobs->{$jstart}->{'DL'} != 1) 
     340    { 
     341      $html .= ' [NL]'; 
     342    } 
     343    $html .= '</span></div>'; 
    312344  } 
    313345  return $html;