Changeset 25842 for gs2-extensions


Ignore:
Timestamp:
2012-06-28T10:41:32+12:00 (12 years ago)
Author:
jmt12
Message:

Adding in more timing information and allowing for clusters to copy files to a local filespace (/tmp) before processing, so we can separate the timing for IO

File:
1 edited

Legend:

Unmodified
Added
Removed

  • gs2-extensions/video-and-audio/trunk/src/perllib/plugins/SimpleVideoPlugin.pm

    r25780 r25842  
    7575                    'list' => [{'name' => "true", 'desc' => "{common.true}"},
    7676                               {'name' => "false", 'desc' => "{common.false}"}],
    77                     'deft' => "false",
    78                     'reqd' => "no" }
     77                    'deft' => "true",
     78                    'reqd' => "no" },
    7979                  { 'name' => "isCluster",
    8080                    'desc' => "Will the import be run on a cluster (multiple computers) or not (single computer - possibly multiple processors)",
    … …  
    8383                               {'name' => "false", 'desc' => "{common.false}"}],
    8484                    'deft' => "false",
     85                    'reqd' => "no" },
     86                  { 'name' => "separateIO",
     87                    'desc' => "copy and process the file locally (good for segregating IO cost)",
     88                    'type' => "enum",
     89                    'list' => [{'name' => "true", 'desc' => "{common.true}"},
     90                               {'name' => "false", 'desc' => "{common.false}"}],
     91                    'deft' => "false",
    8592                    'reqd' => "no" }
    8693                ];
    … …  
    120127  my ($pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
    121128
    122   my $ivideo_path = &util::filename_cat($base_dir, $file);
     129  print STDERR "[A:" . time() . "] SimpleVideoPlugin processing: " . $file . "\n";
     130
     131  # - I have to add some text (yay, back to needing dummy text) otherwise the
     132  #   DocumentText formatting is ignored (?!?)
    123133  my $topsection = $doc_obj->get_top_section();
     134  $doc_obj->add_utf8_text($topsection, "This is dummy text");
     135
    124136
    125137  $file =~ /[\/]?(.+)\.(?:ts)$/;
    126138  my $filename = $1;
    127   $filename =~ /(\d\d\d\d)-(\d\d)-(\d\d)/;
    128   my $date = $1 . $2 . $3;
    129   $filename =~ s/[^a-z0-9]+/_/ig;
    130   $filename =~ s/^_+|_+$//g;
    131 
    132   $doc_obj->add_utf8_metadata($topsection,"Date",$date);
    133 
    134   my $logs_dir = &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, "logs");
     139
     140
     141  # Optional date metadata (available on raw ReplayMe recordings)
     142  if ($filename =~ /(\d\d\d\d)-(\d\d)-(\d\d)/)
     143  {
     144    my $date = $1 . $2 . $3;
     145    $filename =~ s/[^a-z0-9]+/_/ig;
     146    $filename =~ s/^_+|_+$//g;
     147    $doc_obj->add_utf8_metadata($topsection,"Date",$date);
     148  }
     149
     150  my $process_dir = $ENV{'GSDLCOLLECTDIR'};
     151  # If we are in a cluster, then we don't want to be writing all the logs
     152  # etc to the shared file system. Instead, we write to the tmp drive
     153  my $separate_io = $self->{'separateIO'};
     154  if ($separate_io)
     155  {
     156    $process_dir = &util::filename_cat('/tmp', 'gsimport-' . $filename);
     157    if (!-d $process_dir)
     158    {
     159      mkdir($process_dir, 0775);
     160    }
     161  }
     162  my $logs_dir = &util::filename_cat($process_dir, "logs");
    135163  if (!-d $logs_dir)
    136164  {
    … …  
    139167  my $convert_log_path = &util::filename_cat($logs_dir, 'convert-' . $filename . '.log');
    140168  my $pass_log_path = &util::filename_cat($logs_dir, 'convert-' . $filename . '-pass');
    141   my $tmp_dir = &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, "cached");
     169  my $tmp_dir = &util::filename_cat($process_dir, "cached");
    142170  if (!-d $tmp_dir)
    143171  {
    … …  
    150178  }
    151179
     180  # If we are separating IO, then we also start by copying the file to
     181  # the process directory (local tmp) as well
     182  my $ivideo_path = &util::filename_cat($base_dir, $file);
     183  if ($separate_io)
     184  {
     185    print STDERR "[B1:" . time() . "] Creating local copy of file\n";
     186    my $local_ivideo_path = &util::filename_cat($process_dir, $file);
     187    my $copy_cmd = 'cp "' . $ivideo_path . '" "' . $local_ivideo_path . '"';
     188    `$copy_cmd`;
     189    $ivideo_path = $local_ivideo_path;
     190    print STDERR "[B2:" . time() . "] Complete\n";
     191  }
     192
    152193  # 1. Use MediaInfo to extract important metadata
     194  print STDERR "[C1:" . time() . "] Extracting metadata\n";
    153195  print " - Extracting metadata using MediaInfo\n";
    154196  my $mi_metadata = $self->getMetadata($ivideo_path);
    … …  
    158200  if (defined $mi_metadata->{'Video'}->{'Format_Info'} && defined $mi_metadata->{'Video'}->{'Format'})
    159201  {
    160     $doc_obj->add_utf8_metadata($topsection,"VideoFormat",$mi_metadata->{'Video'}->{'Format_Info'} . ' (' . $metadata->{'Video'}->{'Format'} . ')');
     202    $doc_obj->add_utf8_metadata($topsection,"VideoFormat",$mi_metadata->{'Video'}->{'Format_Info'} . ' (' . $mi_metadata->{'Video'}->{'Format'} . ')');
    161203  }
    162204  if (defined $mi_metadata->{'Audio'}->{'Format_Info'} && defined $mi_metadata->{'Audio'}->{'Format'})
    163205  {
    164     $doc_obj->add_utf8_metadata($topsection,"AudioFormat",$mi_metadata->{'Audio'}->{'Format_Info'} . ' (' . $metadata->{'Audio'}->{'Format'} . ')');
     206    $doc_obj->add_utf8_metadata($topsection,"AudioFormat",$mi_metadata->{'Audio'}->{'Format_Info'} . ' (' . $mi_metadata->{'Audio'}->{'Format'} . ')');
    165207  }
    166208  $doc_obj->add_utf8_metadata($topsection,"Width",$mi_metadata->{'Video'}->{'Width'});
    167209  $doc_obj->add_utf8_metadata($topsection,"Height",$mi_metadata->{'Video'}->{'Height'});
     210  print STDERR "[C2:" . time() . "] Complete\n";
    168211
    169212  # 2. Convert into FLV, reprocess to make seekable, and associate
    170213  # - generate a path for our temporary converted video file
     214  print STDERR "[D1:" . time() . "] Converting video to streamble format\n";
    171215  my $ovideo_path = &util::filename_cat($tmp_dir, 'gsv.mp4');
    172 
    173216  if (-f $ovideo_path)
    174217  {
    … …  
    209252    if ($is_parallel && !$is_cluster)
    210253    {
    211       $mencoder_options = ':threads=1';
     254      $mencoder_options .= ':threads=1';
    212255    }
    213256    my $cmd = 'HandBrakeCLI -i "' . $ivideo_path . '" -t 1 -c 1 -f mp4 -O -o "' . $ovideo_path . '" ' . $video_processing_parameters . ' -e x264 -b ' . $streaming_HQ_VideoBitrate . ' -a 1 -E faac -6 dpl2 -R Auto -B ' . $streaming_HQ_AudioBitrate . ' -D 0.0 -x ' . $mencoder_options . ' > "' . $convert_log_path . '" 2>&1';
    … …  
    219262    die("Fatal Error! Failed to convert video: " . $ovideo_path . "\nReason:" . $! . "\n");
    220263  }
    221   # - associate
    222   $doc_obj->associate_file($ovideo_path,'gsv.mp4','video/mp4',$topsection);
    223 
    224   # 3. Extract keyframes using hive, and associate
     264  print STDERR "[D2:" . time() . "] Complete\n";
     265
     266  # 3. Extract keyframes using hive
     267  print STDERR "[E1:" . time() . "] Extract keyframes\n";
    225268  my $oshots_path = &util::filename_cat($tmp_dir, 'shots.xml');
    226269  if (-f $oshots_path)
    … …  
    239282    die("Fatal Error! Failed to extract keyframe images: " . $oshots_path . "\nReason:" . $! . "\n");
    240283  }
     284  print STDERR "[E2:" . time() . "] Complete\n";
     285
     286
     287  # 4. Associate files (copies back to shared space if IO separated)
     288  print STDERR "[F1:" . time() . "] Associate derived files to doc_obj\n";
     289  # - associate streamable video
     290  $doc_obj->associate_file($ovideo_path,'gsv.mp4','video/mp4',$topsection);
    241291  # - associate all of the JPGs found in the temp directory
    242292  opendir(my $dh, $tmp_dir);
    … …  
    258308    }
    259309  }
    260 
    261   # - I have to add some text (yay, back to needing dummy text) otherwise the
    262   #   DocumentText formatting is ignored (?!?)
    263   $doc_obj->add_utf8_text($topsection, "This is dummy text");
    264 
    265   # 4. Done! Cleanup.
    266 
     310  print STDERR "[F2:" . time() . "] Complete\n";
     311
     312  # 5. Done! Cleanup.
     313  print STDERR "[G:" . time() . "] SimpleVideoPlugin: Complete!\n";
    267314  return 1;
    268315}
    … …  
    272319  my ($self, $ivideo_path) = @_;
    273320  my $cmd = 'mediainfo --Output=XML "' . $ivideo_path . '" 2>&1';
    274   ###print "Cmd: " . $cmd1 . "\n";
     321  ###print "Cmd: " . $cmd . "\n";
    275322  my $metadata_xml = `$cmd`;
    276323  my @lines = split(/\r?\n/, $metadata_xml);
    … …  
    279326  foreach my $line (@lines)
    280327  {
    281     if ($line =~ /<track type="(.+)">/)
     328   if ($line =~ /<track type="(.+)">/)
    282329    {
    283330      $metadata_type = $1;
Note: See TracChangeset for help on using the changeset viewer.