| 1 | #!/usr/bin/perl
|
|---|
| 2 |
|
|---|
| 3 | use strict;
|
|---|
| 4 | use warnings;
|
|---|
| 5 |
|
|---|
| 6 | use Sort::Naturally;
|
|---|
| 7 | use POSIX qw(floor strftime);
|
|---|
| 8 |
|
|---|
| 9 | print "\n===== Generate Timing (GANTT) =====\n";
|
|---|
| 10 |
|
|---|
| 11 | # 0. Configuration
|
|---|
| 12 | my $debug = 0;
|
|---|
| 13 | my $import_dir;
|
|---|
| 14 |
|
|---|
| 15 | # 1. Initialization
|
|---|
| 16 | if (!defined $ARGV[0] || !-d $ARGV[0])
|
|---|
| 17 | {
|
|---|
| 18 | &printUsage('Directory not provided or doesn\'t exist');
|
|---|
| 19 | }
|
|---|
| 20 | my $dir = $ARGV[0];
|
|---|
| 21 | my $timing_csv_path = &filenameCat($dir, 'timing.csv');
|
|---|
| 22 | if (!-e $timing_csv_path)
|
|---|
| 23 | {
|
|---|
| 24 | &printUsage('Directory doesn\'t contain timing.csv: ' . $dir);
|
|---|
| 25 | }
|
|---|
| 26 | print 'Timing File: ' . $timing_csv_path . "\n";
|
|---|
| 27 | my $chart_width = 1024;
|
|---|
| 28 | if (defined $ARGV[1])
|
|---|
| 29 | {
|
|---|
| 30 | if ($ARGV[1] !~ /^\d+$/)
|
|---|
| 31 | {
|
|---|
| 32 | &printUsage('Chart width not a number');
|
|---|
| 33 | }
|
|---|
| 34 | $chart_width = $ARGV[1];
|
|---|
| 35 | }
|
|---|
| 36 | print "Chart Width: " . $chart_width . "px\n";
|
|---|
| 37 |
|
|---|
| 38 | my ($epoc) = $dir =~ /(\d+)$/;
|
|---|
| 39 | my $gantt_path = $dir . '/' . $epoc . '-gantt.html';
|
|---|
| 40 | print "Gantt Chart: " . $gantt_path . "\n";
|
|---|
| 41 |
|
|---|
| 42 | print "===================================\n\n";
|
|---|
| 43 |
|
|---|
| 44 | # Read in timing.csv and parse information into data structure
|
|---|
| 45 | my $timing_data = {};
|
|---|
| 46 | my $id_2_worker_id = {};
|
|---|
| 47 | if (open(TIN, '<:utf8', $timing_csv_path))
|
|---|
| 48 | {
|
|---|
| 49 | my $line;
|
|---|
| 50 | while ($line = <TIN>)
|
|---|
| 51 | {
|
|---|
| 52 | my @parts = split(/,/, $line);
|
|---|
| 53 | if ($parts[1] eq 'M0')
|
|---|
| 54 | {
|
|---|
| 55 | $timing_data->{'M'} = {'N'=>$parts[2], 'S'=>$parts[3], 'E'=>$parts[4]};
|
|---|
| 56 | }
|
|---|
| 57 | elsif ($parts[1] =~ /W\d+/)
|
|---|
| 58 | {
|
|---|
| 59 | $timing_data->{$parts[1]} = {'N'=>$parts[2], 'S'=>$parts[3], 'E'=>$parts[4], 'F'=>{}};
|
|---|
| 60 | $id_2_worker_id->{$parts[0]} = $parts[1];
|
|---|
| 61 | }
|
|---|
| 62 | elsif ($parts[1] =~ /T\d+/)
|
|---|
| 63 | {
|
|---|
| 64 | my $worker_id = $id_2_worker_id->{$parts[7]};
|
|---|
| 65 | my $stop = $parts[4];
|
|---|
| 66 | my $filepath = $parts[8];
|
|---|
| 67 | $filepath =~ s/^\s+|\s+$//g;
|
|---|
| 68 | $import_dir = &longestCommonPath($filepath, $import_dir);
|
|---|
| 69 | $timing_data->{$worker_id}->{'F'}->{$parts[3]} = {'FN'=>$filepath, 'PS'=>($stop - $parts[5]), 'PE'=>$stop, 'E'=>$stop, 'DL'=>$parts[6]};
|
|---|
| 70 | }
|
|---|
| 71 | }
|
|---|
| 72 | close(TIN);
|
|---|
| 73 | }
|
|---|
| 74 | else
|
|---|
| 75 | {
|
|---|
| 76 | die('Error! Failed to open file for reading: ' . $timing_csv_path);
|
|---|
| 77 | }
|
|---|
| 78 | my $number_of_workers = scalar(keys(%{$id_2_worker_id}));;
|
|---|
| 79 |
|
|---|
| 80 | # 3. Produce pretty HTML chart of timing information including jobs
|
|---|
| 81 | print " * Generating timing information as HTML... ";
|
|---|
| 82 | open(HTMLOUT, '>:utf8', $gantt_path) or die('Error! Failed to open file for writing: gantt.html');
|
|---|
| 83 | print HTMLOUT "<html>\n";
|
|---|
| 84 | print HTMLOUT '<head>' . "\n";
|
|---|
| 85 | print HTMLOUT '<style type="text/css">' . "\n";
|
|---|
| 86 | print HTMLOUT 'div.thread {position:relative}' . "\n";
|
|---|
| 87 | print HTMLOUT 'div.master {border:1px solid gray;color:white;font-weight:bold}' . "\n";
|
|---|
| 88 | print HTMLOUT 'div.worker {border:1px solid black;background-color:green;color:white;font-weight:bold}' . "\n";
|
|---|
| 89 | print HTMLOUT 'div.time {font-size:smaller;font-weight:normal}' . "\n";
|
|---|
| 90 | print HTMLOUT 'div.job {background-color:transparent;color:black;border:1px solid black;display:block;font-size:smaller;position:relative;text-align:center;overflow:hidden}' . "\n";
|
|---|
| 91 | print HTMLOUT 'span.process {z-index:-1;background-color:#C7C7C7;position:absolute}' . "\n";
|
|---|
| 92 | print HTMLOUT 'span.label {z-index:1;background-color:transparent;overflow:hidden;white-space:nowrap;}' . "\n";
|
|---|
| 93 | print HTMLOUT "th {text-align:left}\n";
|
|---|
| 94 | print HTMLOUT '</style>' . "\n";
|
|---|
| 95 | print HTMLOUT '</head>' . "\n";
|
|---|
| 96 | print HTMLOUT "<body>\n";
|
|---|
| 97 | print HTMLOUT "<h2>Statistics</h2>\n";
|
|---|
| 98 | print HTMLOUT "<table>\n";
|
|---|
| 99 |
|
|---|
| 100 | my $total_duration = $timing_data->{'M'}->{'E'} - $timing_data->{'M'}->{'S'};
|
|---|
| 101 | my $file_count = 0;
|
|---|
| 102 | my $data_locality = 0;
|
|---|
| 103 | my $total_io_time = 0;
|
|---|
| 104 | my $total_process_time = 0;
|
|---|
| 105 | my $fastest_file = 0;
|
|---|
| 106 | my $slowest_file = 0;
|
|---|
| 107 | my $problem_files = 0;
|
|---|
| 108 | foreach my $worker_id (keys %{$timing_data})
|
|---|
| 109 | {
|
|---|
| 110 | if ($worker_id ne 'M')
|
|---|
| 111 | {
|
|---|
| 112 | foreach my $job_start ( keys %{$timing_data->{$worker_id}->{'F'}} )
|
|---|
| 113 | {
|
|---|
| 114 | my $process_start = $timing_data->{$worker_id}->{'F'}->{$job_start}->{'PS'};
|
|---|
| 115 | my $process_end = $timing_data->{$worker_id}->{'F'}->{$job_start}->{'PE'};
|
|---|
| 116 | my $job_end = $timing_data->{$worker_id}->{'F'}->{$job_start}->{'E'};
|
|---|
| 117 | if ($process_start == 0 || $process_end == 0 || $job_end == 0)
|
|---|
| 118 | {
|
|---|
| 119 | $problem_files++;
|
|---|
| 120 | }
|
|---|
| 121 | else
|
|---|
| 122 | {
|
|---|
| 123 | my $io_duration = ($process_start - $job_start) + ($job_end - $process_end);
|
|---|
| 124 | my $process_duration = $process_end - $process_start;
|
|---|
| 125 | my $total_duration = $io_duration + $process_duration;
|
|---|
| 126 | &debugPrint("filename: " . $timing_data->{$worker_id}->{'F'}->{$job_start}->{'FN'});
|
|---|
| 127 | &debugPrint("start: $job_start ps: $process_start pe: $process_end end: $job_end");
|
|---|
| 128 | &debugPrint("io: $io_duration process: $process_duration duration: $total_duration");
|
|---|
| 129 | # Running stats
|
|---|
| 130 | $total_io_time += $io_duration;
|
|---|
| 131 | $total_process_time += $process_duration;
|
|---|
| 132 | if ($fastest_file == 0 || $total_duration < $fastest_file)
|
|---|
| 133 | {
|
|---|
| 134 | $fastest_file = $total_duration;
|
|---|
| 135 | }
|
|---|
| 136 | if ($slowest_file == 0 || $total_duration > $slowest_file)
|
|---|
| 137 | {
|
|---|
| 138 | $slowest_file = $total_duration;
|
|---|
| 139 | }
|
|---|
| 140 | }
|
|---|
| 141 | # Shorten filename
|
|---|
| 142 | if (defined $timing_data->{$worker_id}->{'F'}->{$job_start}->{'FN'})
|
|---|
| 143 | {
|
|---|
| 144 | $timing_data->{$worker_id}->{'F'}->{$job_start}->{'FN'} = substr($timing_data->{$worker_id}->{'F'}->{$job_start}->{'FN'}, length($import_dir) + 1);
|
|---|
| 145 | }
|
|---|
| 146 | $file_count++;
|
|---|
| 147 | if ($timing_data->{$worker_id}->{'F'}->{$job_start}->{'DL'} == 1)
|
|---|
| 148 | {
|
|---|
| 149 | $data_locality++;
|
|---|
| 150 | }
|
|---|
| 151 | }
|
|---|
| 152 | }
|
|---|
| 153 | }
|
|---|
| 154 | my $avg_processing_time = floor(($total_io_time + $total_process_time) / $file_count);
|
|---|
| 155 | my @months = ("Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec");
|
|---|
| 156 |
|
|---|
| 157 | print HTMLOUT "<tr><th>Import Directory:</th><td>" . $import_dir . "</td></tr>\n";
|
|---|
| 158 | my ($sec, $min, $hour, $day, $month, $year) = (localtime($timing_data->{'M'}->{'S'}))[0,1,2,3,4,5];
|
|---|
| 159 | print HTMLOUT "<tr><th>Start Time:</th><td>" . sprintf('%04d%s%02d %02d:%02d:%02d', ($year+1900), $months[$month], $day, $hour, $min, $sec) . "</td></tr>\n";
|
|---|
| 160 | print HTMLOUT "<tr><th>Processing Time:</th><td>" . &renderTime($total_duration) . "</td></tr>\n";
|
|---|
| 161 | print HTMLOUT "<tr><th>Processing Threads:</th><td>" . $number_of_workers . "</td></tr>\n";
|
|---|
| 162 | print HTMLOUT "<tr><th>Files Processed:</th><td>" . $file_count . "</td></tr>\n";
|
|---|
| 163 | if ($data_locality > 0)
|
|---|
| 164 | {
|
|---|
| 165 | print HTMLOUT "<tr><th>Data Locality:</th><td>" . sprintf('%d%% [%d out of %d]', (($data_locality / $file_count) * 100), $data_locality, $file_count) . "</td></tr>\n";
|
|---|
| 166 | }
|
|---|
| 167 | print HTMLOUT "<tr><th>Serial Processing Time:</th><td>" . &renderTime($total_process_time) . "</td></tr>\n";
|
|---|
| 168 | print HTMLOUT "<tr><th>Serial IO Time:</th><td>" . &renderTime($total_io_time) . "</td></tr>\n";
|
|---|
| 169 | print HTMLOUT "<tr><th>Average File Processing Time:</th><td>" . &renderTime($avg_processing_time) . "</td></tr>\n";
|
|---|
| 170 | print HTMLOUT "<tr><th>Fastest File:</th><td>" . &renderTime($fastest_file) . "</td></tr>\n";
|
|---|
| 171 | print HTMLOUT "<tr><th>Slowest File:</th><td>" . &renderTime($slowest_file) . "</td></tr>\n";
|
|---|
| 172 | print HTMLOUT "<tr><th>Problem Files:</th><td>" . $problem_files . "</td></tr>\n";
|
|---|
| 173 |
|
|---|
| 174 | print HTMLOUT "</table>\n";
|
|---|
| 175 | print HTMLOUT "<hr />\n";
|
|---|
| 176 | print HTMLOUT "<h2>Timing Chart (Gannt)</h2>\n";
|
|---|
| 177 | print HTMLOUT renderLine($chart_width, $timing_data->{'M'}->{'S'}, $timing_data->{'M'}->{'E'}, 'master', $timing_data->{'M'}->{'N'}, $timing_data->{'M'}->{'S'}, $timing_data->{'M'}->{'E'}, {});
|
|---|
| 178 | foreach my $worker_id (nsort keys %{$timing_data})
|
|---|
| 179 | {
|
|---|
| 180 | if ($worker_id ne 'M')
|
|---|
| 181 | {
|
|---|
| 182 | my $data = $timing_data->{$worker_id};
|
|---|
| 183 | print HTMLOUT renderLine($chart_width, $timing_data->{'M'}->{'S'}, $timing_data->{'M'}->{'E'}, 'worker', $worker_id . ' [' . $data->{'N'} . ']', $data->{'S'}, $data->{'E'}, $data->{'F'});
|
|---|
| 184 | }
|
|---|
| 185 | }
|
|---|
| 186 | print HTMLOUT '<div>' . "\n";
|
|---|
| 187 | print HTMLOUT "</body>\n";
|
|---|
| 188 | print HTMLOUT "</html>";
|
|---|
| 189 | close(HTMLOUT);
|
|---|
| 190 |
|
|---|
| 191 | print "Done!\n";
|
|---|
| 192 | print "Complete!\n\n";
|
|---|
| 193 | exit;
|
|---|
| 194 |
|
|---|
| 195 |
|
|---|
| 196 | ## @function debugPrint()
|
|---|
| 197 | #
|
|---|
| 198 | sub debugPrint
|
|---|
| 199 | {
|
|---|
| 200 | my $msg = shift(@_);
|
|---|
| 201 | if ($debug)
|
|---|
| 202 | {
|
|---|
| 203 | print STDERR '[DEBUG] ' . $msg . "\n";
|
|---|
| 204 | }
|
|---|
| 205 | }
|
|---|
| 206 | ## debugPrint() ##
|
|---|
| 207 |
|
|---|
| 208 |
|
|---|
| 209 | ## @function filenameCat
|
|---|
| 210 | #
|
|---|
| 211 | sub filenameCat
|
|---|
| 212 | {
|
|---|
| 213 | my $path = join('/', @_);
|
|---|
| 214 | $path =~ s/[\/\\]+/\//g;
|
|---|
| 215 | # protocols
|
|---|
| 216 | $path =~ s/^(HDFS|HDFSShell|HDThriftFS):\//$1:\/\//;
|
|---|
| 217 | return $path;
|
|---|
| 218 | }
|
|---|
| 219 | ## filenameCat() ##
|
|---|
| 220 |
|
|---|
| 221 | ## @function printUsage()
|
|---|
| 222 | #
|
|---|
| 223 | sub printUsage
|
|---|
| 224 | {
|
|---|
| 225 | my $msg = shift(@_);
|
|---|
| 226 | if (defined $msg)
|
|---|
| 227 | {
|
|---|
| 228 | print 'Error! ' . $msg . "\n";
|
|---|
| 229 | }
|
|---|
| 230 | die("Usage: generate_gantt.pl <results dir> [<width in pixels>]\n\n");
|
|---|
| 231 | }
|
|---|
| 232 | ## printUsage() ##
|
|---|
| 233 |
|
|---|
| 234 |
|
|---|
| 235 | ## @function longestCommonPath
|
|---|
| 236 | #
|
|---|
| 237 | sub longestCommonPath
|
|---|
| 238 | {
|
|---|
| 239 | my ($path_new, $path_current) = @_;
|
|---|
| 240 | my $result = '';
|
|---|
| 241 | if (defined $path_current)
|
|---|
| 242 | {
|
|---|
| 243 | my @path_new_parts = split(/\//, $path_new);
|
|---|
| 244 | my @path_current_parts = split(/\//, $path_current);
|
|---|
| 245 | my @path_parts;
|
|---|
| 246 | for (my $i = 0; $i < scalar(@path_current_parts); $i++)
|
|---|
| 247 | {
|
|---|
| 248 | if ($path_current_parts[$i] eq $path_new_parts[$i])
|
|---|
| 249 | {
|
|---|
| 250 | push(@path_parts, $path_new_parts[$i]);
|
|---|
| 251 | }
|
|---|
| 252 | else
|
|---|
| 253 | {
|
|---|
| 254 | last;
|
|---|
| 255 | }
|
|---|
| 256 | }
|
|---|
| 257 | $result = &filenameCat(@path_parts);
|
|---|
| 258 | }
|
|---|
| 259 | else
|
|---|
| 260 | {
|
|---|
| 261 | $result = $path_new;
|
|---|
| 262 | }
|
|---|
| 263 | return $result;
|
|---|
| 264 | }
|
|---|
| 265 | ## longestCommonPath() ##
|
|---|
| 266 |
|
|---|
| 267 |
|
|---|
| 268 | ## @function renderLine()
|
|---|
| 269 | #
|
|---|
| 270 | sub renderLine
|
|---|
| 271 | {
|
|---|
| 272 | my ($table_width, $start, $end, $class, $tname, $tstart, $tend, $jobs) = @_;
|
|---|
| 273 | &debugPrint("renderLine($table_width, $start, $end, $class, $tname, $tstart, $tend, <jobs>)");
|
|---|
| 274 | # All timings need to be relative to 0 (relative start)
|
|---|
| 275 | my $duration = $end - $start;
|
|---|
| 276 | my $rtstart = $tstart - $start;
|
|---|
| 277 | my $rtend = $tend - $start;
|
|---|
| 278 | # We need to scale these depending on the timing of this thread relative to
|
|---|
| 279 | # the master thread
|
|---|
| 280 | my $width = $chart_width;
|
|---|
| 281 | my $left = 0;
|
|---|
| 282 | if ($start != $tstart)
|
|---|
| 283 | {
|
|---|
| 284 | my $left_offset_percent = $rtstart / $duration;
|
|---|
| 285 | $left = $left_offset_percent * $table_width;
|
|---|
| 286 | }
|
|---|
| 287 | # - subtract any left offset from width
|
|---|
| 288 | $width = $width - $left;
|
|---|
| 289 | # - right offset directly subtracted from width
|
|---|
| 290 | if ($end != $tend)
|
|---|
| 291 | {
|
|---|
| 292 | my $right_offset_percent = ($duration - $rtend) / $duration;
|
|---|
| 293 | my $right = $right_offset_percent * $table_width;
|
|---|
| 294 | $width = $width - $right;
|
|---|
| 295 | }
|
|---|
| 296 | # Round things off (simple dutch rounding)
|
|---|
| 297 | $left = int($left + 0.5);
|
|---|
| 298 | $width = int($width + 0.5);
|
|---|
| 299 | # Output the bar for this master/worker
|
|---|
| 300 | my $html = '<div class="thread ' . $class . '" style="left:' . $left . 'px;width:' . $width . 'px;">';
|
|---|
| 301 | if ($class eq 'master')
|
|---|
| 302 | {
|
|---|
| 303 | $html .= '<div style="background-color:blue">';
|
|---|
| 304 | }
|
|---|
| 305 | $html .= '<div class="time" style="display:table-cell">' . &renderTime($rtstart) . '</div><div style="display:table-cell;padding-left:20px;width:100%;">' . ucfirst($class) . ': ' . $tname . '</div><div class="time" style="display:table-cell">' . renderTime($rtend) . '</div></div>';
|
|---|
| 306 | my $previous_jright = 0;
|
|---|
| 307 | foreach my $jstart (sort keys %{$jobs})
|
|---|
| 308 | {
|
|---|
| 309 | my $rjstart = $jstart - $start;
|
|---|
| 310 | my $rpstart = $jobs->{$jstart}->{'PS'} - $start;
|
|---|
| 311 | my $rpend = $jobs->{$jstart}->{'PE'} - $start;
|
|---|
| 312 | my $rjend = $jobs->{$jstart}->{'E'} - $start;
|
|---|
| 313 | my $jduration = $jobs->{$jstart}->{'E'} - $jstart;
|
|---|
| 314 | # Scale Job co-ordinates
|
|---|
| 315 | my $jleft_percent = $rjstart / $duration;
|
|---|
| 316 | my $jleft = int(($jleft_percent * $table_width) + 0.5);
|
|---|
| 317 | my $jwidth_percent = $jduration / $duration;
|
|---|
| 318 | my $jwidth = int(($jwidth_percent * $table_width) + 0.5);
|
|---|
| 319 | if ($jleft + $jwidth > $left + $width)
|
|---|
| 320 | {
|
|---|
| 321 | $jwidth = ($left + $width) - $jleft;
|
|---|
| 322 | }
|
|---|
| 323 | # Then scale process timings within that!
|
|---|
| 324 | my $rpleft_percent = ($rpstart - $rjstart) / $duration;
|
|---|
| 325 | my $rpleft = int(($rpleft_percent * $table_width) + 0.5);
|
|---|
| 326 | my $rpwidth = $jwidth - $rpleft;
|
|---|
| 327 | my $cpu_percent = int((($rpwidth / $jwidth) * 100) + 0.5);
|
|---|
| 328 | $html .= '<div class="job" style="left:' . $jleft . 'px;width:' . $jwidth . 'px;';
|
|---|
| 329 | if ($jobs->{$jstart}->{'DL'} != 1)
|
|---|
| 330 | {
|
|---|
| 331 | $html .= 'border:1px dashed black;';
|
|---|
| 332 | }
|
|---|
| 333 | $html .= '" title="FN:' . $jobs->{$jstart}->{'FN'} . ', S:' . renderTime($rjstart) . ', E:' . renderTime($rjend) . ', CPU: ' . $cpu_percent . '%"><span class="process" style="left:' . $rpleft . 'px;width:' . $rpwidth . 'px"> </span><span class="label"';
|
|---|
| 334 | if ($jobs->{$jstart}->{'DL'} != 1)
|
|---|
| 335 | {
|
|---|
| 336 | $html .= ' style="color:#FF0000"';
|
|---|
| 337 | }
|
|---|
| 338 | $html .= '>' . $jobs->{$jstart}->{'FN'};
|
|---|
| 339 | if ($jobs->{$jstart}->{'DL'} != 1)
|
|---|
| 340 | {
|
|---|
| 341 | $html .= ' [NL]';
|
|---|
| 342 | }
|
|---|
| 343 | $html .= '</span></div>';
|
|---|
| 344 | }
|
|---|
| 345 | return $html;
|
|---|
| 346 | }
|
|---|
| 347 | ## renderLine() ##
|
|---|
| 348 |
|
|---|
| 349 |
|
|---|
| 350 | ## @function renderTime()
|
|---|
| 351 | #
|
|---|
| 352 | sub renderTime
|
|---|
| 353 | {
|
|---|
| 354 | my ($seconds) = @_;
|
|---|
| 355 | my $time_str = '';
|
|---|
| 356 | # determine how many hours
|
|---|
| 357 | my $an_hour = 60 * 60;
|
|---|
| 358 | my $hours = floor($seconds / $an_hour);
|
|---|
| 359 | $seconds = $seconds - ($hours * $an_hour);
|
|---|
| 360 | my $a_minute = 60;
|
|---|
| 361 | my $minutes = floor($seconds / $a_minute);
|
|---|
| 362 | $seconds = $seconds - ($minutes * $a_minute);
|
|---|
| 363 | if ($hours > 0)
|
|---|
| 364 | {
|
|---|
| 365 | $time_str = sprintf('%dh%02dm%02ds', $hours, $minutes, $seconds);
|
|---|
| 366 | }
|
|---|
| 367 | elsif ($minutes > 0)
|
|---|
| 368 | {
|
|---|
| 369 | $time_str = sprintf('%dm%02ds', $minutes, $seconds);
|
|---|
| 370 | }
|
|---|
| 371 | else
|
|---|
| 372 | {
|
|---|
| 373 | $time_str = $seconds . 's';
|
|---|
| 374 | }
|
|---|
| 375 | return $time_str;
|
|---|
| 376 | }
|
|---|
