Changeset 26948 for gs2-extensions/video-and-audio/trunk/src/perllib/plugins/SimpleVideoPlugin.pm

Timestamp:

2013-02-26T10:41:13+13:00 (11 years ago)

Author:

jmt12

Message:

Adding 'fixedCore' argument to allow restricting HandbrakeCLI to a single CPU (threads=1 only affects encoding). Replace all built-in perl file tests with calls to util library to allow plugin to process files in HDFS. Coincidentally HDFS *only* supported by 'separateIO' functionality.

File:

• gs2-extensions/video-and-audio/trunk/src/perllib/plugins/SimpleVideoPlugin.pm (modified) (10 diffs)

gs2-extensions/video-and-audio/trunk/src/perllib/plugins/SimpleVideoPlugin.pm

@@ -90 +90 @@
                                {'name' => "false", 'desc' => "{common.false}"}],
                     'deft' => "false",
-                    'reqd' => "no" }
+                    'reqd' => "no" },
+                  { 'name' => "fixedCore",
+                    'desc' => "Restrict the execution of Handbrake to a single core (0 = no restriction, > 0 use value-1'th core)",
+                    'type' => "int",
+                    'deft' => "0",
+                    'reqd' => "no" },
                 ];
 
@@ -135 +140 @@
 
 
-  $file =~ /[\/]?(.+)\.(?:ts)$/;
+  $file =~ /[\/]?([^\/]+)\.(?:ts)$/;
   my $filename = $1;
-
 
   # Optional date metadata (available on raw ReplayMe recordings)
@@ -148 +152 @@
   }
 
+  # Special Case: HDFS *only* supported by separateIO flag (you need to move
+  # the file out of HDFS to local filespace to allow MediaInfo and Handbrake
+  # to be run on it.
+  my $separate_io = $self->{'separateIO'};
+  if (&util::isHDFS($file))
+  {
+    $separate_io = 'true';
+  }
+
   my $process_dir = $ENV{'GSDLCOLLECTDIR'};
   # If we are in a cluster, then we don't want to be writing all the logs
   # etc to the shared file system. Instead, we write to the tmp drive
-  my $separate_io = $self->{'separateIO'};
-  if ($separate_io)
+  if ($separate_io eq 'true')
   {
     $process_dir = &util::filename_cat('/tmp', 'gsimport-' . $filename);
-    if (!-d $process_dir)
+    if (!&util::dir_exists($process_dir))
     {
       mkdir($process_dir, 0775);
@@ -161 +173 @@
   }
   my $logs_dir = &util::filename_cat($process_dir, "logs");
-  if (!-d $logs_dir)
+  if (!&util::dir_exists($logs_dir))
   {
     mkdir($logs_dir, 0775);
@@ -168 +180 @@
   my $pass_log_path = &util::filename_cat($logs_dir, 'convert-' . $filename . '-pass');
   my $tmp_dir = &util::filename_cat($process_dir, "cached");
-  if (!-d $tmp_dir)
+  if (!&util::dir_exists($tmp_dir))
   {
     mkdir($tmp_dir, 0775);
   }
   $tmp_dir = &util::filename_cat($tmp_dir, $filename);
-  if (!-d $tmp_dir)
+  if (!&util::dir_exists($tmp_dir))
   {
     mkdir($tmp_dir, 0775);
@@ -181 +193 @@
   # the process directory (local tmp) as well
   my $ivideo_path = &util::filename_cat($base_dir, $file);
-  if ($separate_io)
+  if ($separate_io eq 'true')
   {
     print STDERR "[B1:" . time() . "] Creating local copy of file\n";
-    my $local_ivideo_path = &util::filename_cat($process_dir, $file);
-    my $copy_cmd = 'cp "' . $ivideo_path . '" "' . $local_ivideo_path . '"';
-    `$copy_cmd`;
+    my $local_ivideo_path = &util::filename_cat($process_dir, $filename . ".ts");
+    &util::cp($ivideo_path, $local_ivideo_path);
     $ivideo_path = $local_ivideo_path;
     print STDERR "[B2:" . time() . "] Complete\n";
@@ -214 +225 @@
   print STDERR "[D1:" . time() . "] Converting video to streamble format\n";
   my $ovideo_path = &util::filename_cat($tmp_dir, 'gsv.mp4');
-  if (-f $ovideo_path)
+  if (&util::file_exists($ovideo_path))
   {
     print " - Found existing converted video in cache\n";
@@ -249 +260 @@
     # causing all other threads to wait anyway). It will interesting to test
     # whether parallel processing or serial processing (with HandBrake parallel
-    # processing) is faster.
-    if ($is_parallel && !$is_cluster)
+    # processing) is faster. *update* threads=1 *only* controls the encoding and
+    # several other parts of Handbrake can run parallel (demuxing etc). I've
+    # had to include a 'taskset' command to truely make Handbrake serial
+    if ($is_parallel eq 'true'  && $is_cluster eq 'false')
     {
       $mencoder_options .= ':threads=1';
     }
-    my $cmd = 'HandBrakeCLI -i "' . $ivideo_path . '" -t 1 -c 1 -f mp4 -O -o "' . $ovideo_path . '" ' . $video_processing_parameters . ' -e x264 -b ' . $streaming_HQ_VideoBitrate . ' -a 1 -E faac -6 dpl2 -R Auto -B ' . $streaming_HQ_AudioBitrate . ' -D 0.0 -x ' . $mencoder_options . ' > "' . $convert_log_path . '" 2>&1';
+    # Banish HandbrakeCLI to the (fixedCore-1)'th CPU if necessary
+    my $cmd = '';
+    if (defined $self->{'fixedCore'} && $self->{'fixedCore'} > 0)
+    {
+      $cmd .= 'taskset -c ' . ($self->{'fixedCore'} - 1) . ' ';
+    }
+    $cmd .= 'HandBrakeCLI -i "' . $ivideo_path . '" -t 1 -c 1 -f mp4 -O -o "' . $ovideo_path . '" ' . $video_processing_parameters . ' -e x264 -b ' . $streaming_HQ_VideoBitrate . ' -a 1 -E faac -6 dpl2 -R Auto -B ' . $streaming_HQ_AudioBitrate . ' -D 0.0 -x ' . $mencoder_options . ' > "' . $convert_log_path . '" 2>&1';
     print "[DEBUG: " . $cmd . "]\n";
     `$cmd`;
   }
-  if (!-f $ovideo_path)
+  if (!&util::file_exists($ovideo_path))
   {
     die("Fatal Error! Failed to convert video: " . $ovideo_path . "\nReason:" . $! . "\n");
@@ -267 +286 @@
   print STDERR "[E1:" . time() . "] Extract keyframes\n";
   my $oshots_path = &util::filename_cat($tmp_dir, 'shots.xml');
-  if (-f $oshots_path)
+  if (&util::file_exists($oshots_path))
   {
     print " - Found existing keyframe images in cache\n";
@@ -278 +297 @@
     `$cmd`;
   }
-  if (!-f $oshots_path)
+  if (!&util::file_exists($oshots_path))
   {
     die("Fatal Error! Failed to extract keyframe images: " . $oshots_path . "\nReason:" . $! . "\n");