Changeset 25842

Show
Ignore:
Timestamp:
28.06.2012 10:41:32 (7 years ago)
Author:
jmt12
Message:

Adding in more timing information and allowing for clusters to copy files to a local filespace (/tmp) before processing, so we can separate the timing for IO

Files:
1 modified

Legend:

Unmodified
Added
Removed
  • gs2-extensions/video-and-audio/trunk/src/perllib/plugins/SimpleVideoPlugin.pm

    r25780 r25842  
    7575                    'list' => [{'name' => "true", 'desc' => "{common.true}"}, 
    7676                               {'name' => "false", 'desc' => "{common.false}"}], 
    77                     'deft' => "false", 
    78                     'reqd' => "no" } 
     77                    'deft' => "true", 
     78                    'reqd' => "no" }, 
    7979                  { 'name' => "isCluster", 
    8080                    'desc' => "Will the import be run on a cluster (multiple computers) or not (single computer - possibly multiple processors)", 
     
    8383                               {'name' => "false", 'desc' => "{common.false}"}], 
    8484                    'deft' => "false", 
     85                    'reqd' => "no" }, 
     86                  { 'name' => "separateIO", 
     87                    'desc' => "copy and process the file locally (good for segregating IO cost)", 
     88                    'type' => "enum", 
     89                    'list' => [{'name' => "true", 'desc' => "{common.true}"}, 
     90                               {'name' => "false", 'desc' => "{common.false}"}], 
     91                    'deft' => "false", 
    8592                    'reqd' => "no" } 
    8693                ]; 
     
    120127  my ($pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_; 
    121128 
    122   my $ivideo_path = &util::filename_cat($base_dir, $file); 
     129  print STDERR "[A:" . time() . "] SimpleVideoPlugin processing: " . $file . "\n"; 
     130 
     131  # - I have to add some text (yay, back to needing dummy text) otherwise the 
     132  #   DocumentText formatting is ignored (?!?) 
    123133  my $topsection = $doc_obj->get_top_section(); 
     134  $doc_obj->add_utf8_text($topsection, "This is dummy text"); 
     135 
    124136 
    125137  $file =~ /[\/]?(.+)\.(?:ts)$/; 
    126138  my $filename = $1; 
    127   $filename =~ /(\d\d\d\d)-(\d\d)-(\d\d)/; 
    128   my $date = $1 . $2 . $3; 
    129   $filename =~ s/[^a-z0-9]+/_/ig; 
    130   $filename =~ s/^_+|_+$//g; 
    131  
    132   $doc_obj->add_utf8_metadata($topsection,"Date",$date); 
    133  
    134   my $logs_dir = &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, "logs"); 
     139 
     140 
     141  # Optional date metadata (available on raw ReplayMe recordings) 
     142  if ($filename =~ /(\d\d\d\d)-(\d\d)-(\d\d)/) 
     143  { 
     144    my $date = $1 . $2 . $3; 
     145    $filename =~ s/[^a-z0-9]+/_/ig; 
     146    $filename =~ s/^_+|_+$//g; 
     147    $doc_obj->add_utf8_metadata($topsection,"Date",$date); 
     148  } 
     149 
     150  my $process_dir = $ENV{'GSDLCOLLECTDIR'}; 
     151  # If we are in a cluster, then we don't want to be writing all the logs 
     152  # etc to the shared file system. Instead, we write to the tmp drive 
     153  my $separate_io = $self->{'separateIO'}; 
     154  if ($separate_io) 
     155  { 
     156    $process_dir = &util::filename_cat('/tmp', 'gsimport-' . $filename); 
     157    if (!-d $process_dir) 
     158    { 
     159      mkdir($process_dir, 0775); 
     160    } 
     161  } 
     162  my $logs_dir = &util::filename_cat($process_dir, "logs"); 
    135163  if (!-d $logs_dir) 
    136164  { 
     
    139167  my $convert_log_path = &util::filename_cat($logs_dir, 'convert-' . $filename . '.log'); 
    140168  my $pass_log_path = &util::filename_cat($logs_dir, 'convert-' . $filename . '-pass'); 
    141   my $tmp_dir = &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, "cached"); 
     169  my $tmp_dir = &util::filename_cat($process_dir, "cached"); 
    142170  if (!-d $tmp_dir) 
    143171  { 
     
    150178  } 
    151179 
     180  # If we are separating IO, then we also start by copying the file to 
     181  # the process directory (local tmp) as well 
     182  my $ivideo_path = &util::filename_cat($base_dir, $file); 
     183  if ($separate_io) 
     184  { 
     185    print STDERR "[B1:" . time() . "] Creating local copy of file\n"; 
     186    my $local_ivideo_path = &util::filename_cat($process_dir, $file); 
     187    my $copy_cmd = 'cp "' . $ivideo_path . '" "' . $local_ivideo_path . '"'; 
     188    `$copy_cmd`; 
     189    $ivideo_path = $local_ivideo_path; 
     190    print STDERR "[B2:" . time() . "] Complete\n"; 
     191  } 
     192 
    152193  # 1. Use MediaInfo to extract important metadata 
     194  print STDERR "[C1:" . time() . "] Extracting metadata\n"; 
    153195  print " - Extracting metadata using MediaInfo\n"; 
    154196  my $mi_metadata = $self->getMetadata($ivideo_path); 
     
    158200  if (defined $mi_metadata->{'Video'}->{'Format_Info'} && defined $mi_metadata->{'Video'}->{'Format'}) 
    159201  { 
    160     $doc_obj->add_utf8_metadata($topsection,"VideoFormat",$mi_metadata->{'Video'}->{'Format_Info'} . ' (' . $metadata->{'Video'}->{'Format'} . ')'); 
     202    $doc_obj->add_utf8_metadata($topsection,"VideoFormat",$mi_metadata->{'Video'}->{'Format_Info'} . ' (' . $mi_metadata->{'Video'}->{'Format'} . ')'); 
    161203  } 
    162204  if (defined $mi_metadata->{'Audio'}->{'Format_Info'} && defined $mi_metadata->{'Audio'}->{'Format'}) 
    163205  { 
    164     $doc_obj->add_utf8_metadata($topsection,"AudioFormat",$mi_metadata->{'Audio'}->{'Format_Info'} . ' (' . $metadata->{'Audio'}->{'Format'} . ')'); 
     206    $doc_obj->add_utf8_metadata($topsection,"AudioFormat",$mi_metadata->{'Audio'}->{'Format_Info'} . ' (' . $mi_metadata->{'Audio'}->{'Format'} . ')'); 
    165207  } 
    166208  $doc_obj->add_utf8_metadata($topsection,"Width",$mi_metadata->{'Video'}->{'Width'}); 
    167209  $doc_obj->add_utf8_metadata($topsection,"Height",$mi_metadata->{'Video'}->{'Height'}); 
     210  print STDERR "[C2:" . time() . "] Complete\n"; 
    168211 
    169212  # 2. Convert into FLV, reprocess to make seekable, and associate 
    170213  # - generate a path for our temporary converted video file 
     214  print STDERR "[D1:" . time() . "] Converting video to streamble format\n"; 
    171215  my $ovideo_path = &util::filename_cat($tmp_dir, 'gsv.mp4'); 
    172  
    173216  if (-f $ovideo_path) 
    174217  { 
     
    209252    if ($is_parallel && !$is_cluster) 
    210253    { 
    211       $mencoder_options = ':threads=1'; 
     254      $mencoder_options .= ':threads=1'; 
    212255    } 
    213256    my $cmd = 'HandBrakeCLI -i "' . $ivideo_path . '" -t 1 -c 1 -f mp4 -O -o "' . $ovideo_path . '" ' . $video_processing_parameters . ' -e x264 -b ' . $streaming_HQ_VideoBitrate . ' -a 1 -E faac -6 dpl2 -R Auto -B ' . $streaming_HQ_AudioBitrate . ' -D 0.0 -x ' . $mencoder_options . ' > "' . $convert_log_path . '" 2>&1'; 
     
    219262    die("Fatal Error! Failed to convert video: " . $ovideo_path . "\nReason:" . $! . "\n"); 
    220263  } 
    221   # - associate 
    222   $doc_obj->associate_file($ovideo_path,'gsv.mp4','video/mp4',$topsection); 
    223  
    224   # 3. Extract keyframes using hive, and associate 
     264  print STDERR "[D2:" . time() . "] Complete\n"; 
     265 
     266  # 3. Extract keyframes using hive 
     267  print STDERR "[E1:" . time() . "] Extract keyframes\n"; 
    225268  my $oshots_path = &util::filename_cat($tmp_dir, 'shots.xml'); 
    226269  if (-f $oshots_path) 
     
    239282    die("Fatal Error! Failed to extract keyframe images: " . $oshots_path . "\nReason:" . $! . "\n"); 
    240283  } 
     284  print STDERR "[E2:" . time() . "] Complete\n"; 
     285 
     286 
     287  # 4. Associate files (copies back to shared space if IO separated) 
     288  print STDERR "[F1:" . time() . "] Associate derived files to doc_obj\n"; 
     289  # - associate streamable video 
     290  $doc_obj->associate_file($ovideo_path,'gsv.mp4','video/mp4',$topsection); 
    241291  # - associate all of the JPGs found in the temp directory 
    242292  opendir(my $dh, $tmp_dir); 
     
    258308    } 
    259309  } 
    260  
    261   # - I have to add some text (yay, back to needing dummy text) otherwise the 
    262   #   DocumentText formatting is ignored (?!?) 
    263   $doc_obj->add_utf8_text($topsection, "This is dummy text"); 
    264  
    265   # 4. Done! Cleanup. 
    266  
     310  print STDERR "[F2:" . time() . "] Complete\n"; 
     311 
     312  # 5. Done! Cleanup. 
     313  print STDERR "[G:" . time() . "] SimpleVideoPlugin: Complete!\n"; 
    267314  return 1; 
    268315} 
     
    272319  my ($self, $ivideo_path) = @_; 
    273320  my $cmd = 'mediainfo --Output=XML "' . $ivideo_path . '" 2>&1'; 
    274   ###print "Cmd: " . $cmd1 . "\n"; 
     321  ###print "Cmd: " . $cmd . "\n"; 
    275322  my $metadata_xml = `$cmd`; 
    276323  my @lines = split(/\r?\n/, $metadata_xml); 
     
    279326  foreach my $line (@lines) 
    280327  { 
    281     if ($line =~ /<track type="(.+)">/) 
     328   if ($line =~ /<track type="(.+)">/) 
    282329    { 
    283330      $metadata_type = $1;