Changeset 25842

Timestamp:

2012-06-28T10:41:32+12:00 (11 years ago)

Author:

jmt12

Message:

Adding in more timing information and allowing for clusters to copy files to a local filespace (/tmp) before processing, so we can separate the timing for IO

File:

• gs2-extensions/video-and-audio/trunk/src/perllib/plugins/SimpleVideoPlugin.pm (modified) (12 diffs)

gs2-extensions/video-and-audio/trunk/src/perllib/plugins/SimpleVideoPlugin.pm

@@ -75 +75 @@
                     'list' => [{'name' => "true", 'desc' => "{common.true}"},
                                {'name' => "false", 'desc' => "{common.false}"}],
-                    'deft' => "false",
-                    'reqd' => "no" }
+                    'deft' => "true",
+                    'reqd' => "no" },
                   { 'name' => "isCluster",
                     'desc' => "Will the import be run on a cluster (multiple computers) or not (single computer - possibly multiple processors)",
@@ -83 +83 @@
                                {'name' => "false", 'desc' => "{common.false}"}],
                     'deft' => "false",
+                    'reqd' => "no" },
+                  { 'name' => "separateIO",
+                    'desc' => "copy and process the file locally (good for segregating IO cost)",
+                    'type' => "enum",
+                    'list' => [{'name' => "true", 'desc' => "{common.true}"},
+                               {'name' => "false", 'desc' => "{common.false}"}],
+                    'deft' => "false",
                     'reqd' => "no" }
                 ];
@@ -120 +127 @@
   my ($pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
 
-  my $ivideo_path = &util::filename_cat($base_dir, $file);
+  print STDERR "[A:" . time() . "] SimpleVideoPlugin processing: " . $file . "\n";
+
+  # - I have to add some text (yay, back to needing dummy text) otherwise the
+  #   DocumentText formatting is ignored (?!?)
   my $topsection = $doc_obj->get_top_section();
+  $doc_obj->add_utf8_text($topsection, "This is dummy text");
+
 
   $file =~ /[\/]?(.+)\.(?:ts)$/;
   my $filename = $1;
-  $filename =~ /(\d\d\d\d)-(\d\d)-(\d\d)/;
-  my $date = $1 . $2 . $3;
-  $filename =~ s/[^a-z0-9]+/_/ig;
-  $filename =~ s/^_+|_+$//g;
-
-  $doc_obj->add_utf8_metadata($topsection,"Date",$date);
-
-  my $logs_dir = &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, "logs");
+
+
+  # Optional date metadata (available on raw ReplayMe recordings)
+  if ($filename =~ /(\d\d\d\d)-(\d\d)-(\d\d)/)
+  {
+    my $date = $1 . $2 . $3;
+    $filename =~ s/[^a-z0-9]+/_/ig;
+    $filename =~ s/^_+|_+$//g;
+    $doc_obj->add_utf8_metadata($topsection,"Date",$date);
+  }
+
+  my $process_dir = $ENV{'GSDLCOLLECTDIR'};
+  # If we are in a cluster, then we don't want to be writing all the logs
+  # etc to the shared file system. Instead, we write to the tmp drive
+  my $separate_io = $self->{'separateIO'};
+  if ($separate_io)
+  {
+    $process_dir = &util::filename_cat('/tmp', 'gsimport-' . $filename);
+    if (!-d $process_dir)
+    {
+      mkdir($process_dir, 0775);
+    }
+  }
+  my $logs_dir = &util::filename_cat($process_dir, "logs");
   if (!-d $logs_dir)
   {
@@ -139 +167 @@
   my $convert_log_path = &util::filename_cat($logs_dir, 'convert-' . $filename . '.log');
   my $pass_log_path = &util::filename_cat($logs_dir, 'convert-' . $filename . '-pass');
-  my $tmp_dir = &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, "cached");
+  my $tmp_dir = &util::filename_cat($process_dir, "cached");
   if (!-d $tmp_dir)
   {
@@ -150 +178 @@
   }
 
+  # If we are separating IO, then we also start by copying the file to
+  # the process directory (local tmp) as well
+  my $ivideo_path = &util::filename_cat($base_dir, $file);
+  if ($separate_io)
+  {
+    print STDERR "[B1:" . time() . "] Creating local copy of file\n";
+    my $local_ivideo_path = &util::filename_cat($process_dir, $file);
+    my $copy_cmd = 'cp "' . $ivideo_path . '" "' . $local_ivideo_path . '"';
+    `$copy_cmd`;
+    $ivideo_path = $local_ivideo_path;
+    print STDERR "[B2:" . time() . "] Complete\n";
+  }
+
   # 1. Use MediaInfo to extract important metadata
+  print STDERR "[C1:" . time() . "] Extracting metadata\n";
   print " - Extracting metadata using MediaInfo\n";
   my $mi_metadata = $self->getMetadata($ivideo_path);
@@ -158 +200 @@
   if (defined $mi_metadata->{'Video'}->{'Format_Info'} && defined $mi_metadata->{'Video'}->{'Format'})
   {
-    $doc_obj->add_utf8_metadata($topsection,"VideoFormat",$mi_metadata->{'Video'}->{'Format_Info'} . ' (' . $metadata->{'Video'}->{'Format'} . ')');
+    $doc_obj->add_utf8_metadata($topsection,"VideoFormat",$mi_metadata->{'Video'}->{'Format_Info'} . ' (' . $mi_metadata->{'Video'}->{'Format'} . ')');
   }
   if (defined $mi_metadata->{'Audio'}->{'Format_Info'} && defined $mi_metadata->{'Audio'}->{'Format'})
   {
-    $doc_obj->add_utf8_metadata($topsection,"AudioFormat",$mi_metadata->{'Audio'}->{'Format_Info'} . ' (' . $metadata->{'Audio'}->{'Format'} . ')');
+    $doc_obj->add_utf8_metadata($topsection,"AudioFormat",$mi_metadata->{'Audio'}->{'Format_Info'} . ' (' . $mi_metadata->{'Audio'}->{'Format'} . ')');
   }
   $doc_obj->add_utf8_metadata($topsection,"Width",$mi_metadata->{'Video'}->{'Width'});
   $doc_obj->add_utf8_metadata($topsection,"Height",$mi_metadata->{'Video'}->{'Height'});
+  print STDERR "[C2:" . time() . "] Complete\n";
 
   # 2. Convert into FLV, reprocess to make seekable, and associate
   # - generate a path for our temporary converted video file
+  print STDERR "[D1:" . time() . "] Converting video to streamble format\n";
   my $ovideo_path = &util::filename_cat($tmp_dir, 'gsv.mp4');
-
   if (-f $ovideo_path)
   {
@@ -209 +252 @@
     if ($is_parallel && !$is_cluster)
     {
-      $mencoder_options = ':threads=1';
+      $mencoder_options .= ':threads=1';
     }
     my $cmd = 'HandBrakeCLI -i "' . $ivideo_path . '" -t 1 -c 1 -f mp4 -O -o "' . $ovideo_path . '" ' . $video_processing_parameters . ' -e x264 -b ' . $streaming_HQ_VideoBitrate . ' -a 1 -E faac -6 dpl2 -R Auto -B ' . $streaming_HQ_AudioBitrate . ' -D 0.0 -x ' . $mencoder_options . ' > "' . $convert_log_path . '" 2>&1';
@@ -219 +262 @@
     die("Fatal Error! Failed to convert video: " . $ovideo_path . "\nReason:" . $! . "\n");
   }
-  # - associate
-  $doc_obj->associate_file($ovideo_path,'gsv.mp4','video/mp4',$topsection);
-
-  # 3. Extract keyframes using hive, and associate
+  print STDERR "[D2:" . time() . "] Complete\n";
+
+  # 3. Extract keyframes using hive
+  print STDERR "[E1:" . time() . "] Extract keyframes\n";
   my $oshots_path = &util::filename_cat($tmp_dir, 'shots.xml');
   if (-f $oshots_path)
@@ -239 +282 @@
     die("Fatal Error! Failed to extract keyframe images: " . $oshots_path . "\nReason:" . $! . "\n");
   }
+  print STDERR "[E2:" . time() . "] Complete\n";
+
+
+  # 4. Associate files (copies back to shared space if IO separated)
+  print STDERR "[F1:" . time() . "] Associate derived files to doc_obj\n";
+  # - associate streamable video
+  $doc_obj->associate_file($ovideo_path,'gsv.mp4','video/mp4',$topsection);
   # - associate all of the JPGs found in the temp directory
   opendir(my $dh, $tmp_dir);
@@ -258 +308 @@
     }
   }
-
-  # - I have to add some text (yay, back to needing dummy text) otherwise the
-  #   DocumentText formatting is ignored (?!?)
-  $doc_obj->add_utf8_text($topsection, "This is dummy text");
-
-  # 4. Done! Cleanup.
-
+  print STDERR "[F2:" . time() . "] Complete\n";
+
+  # 5. Done! Cleanup.
+  print STDERR "[G:" . time() . "] SimpleVideoPlugin: Complete!\n";
   return 1;
 }
@@ -272 +319 @@
   my ($self, $ivideo_path) = @_;
   my $cmd = 'mediainfo --Output=XML "' . $ivideo_path . '" 2>&1';
-  ###print "Cmd: " . $cmd1 . "\n";
+  ###print "Cmd: " . $cmd . "\n";
   my $metadata_xml = `$cmd`;
   my @lines = split(/\r?\n/, $metadata_xml);
@@ -279 +326 @@
   foreach my $line (@lines)
   {
-    if ($line =~ /<track type="(.+)">/)
+   if ($line =~ /<track type="(.+)">/)
     {
       $metadata_type = $1;