Changeset 29179


Ignore:
Timestamp:
2014-08-12T09:18:25+12:00 (10 years ago)
Author:
jmt12
Message:

Changes to support low and med cpu loads and to avoid excessive IO in high CPU mode

File:
1 edited

Legend:

Unmodified
Added
Removed

  • gs2-extensions/video-and-audio/trunk/src/perllib/plugins/SimpleVideoPlugin.pm

    r29028 r29179  
    9999                  { 'name' => "no_keyframes",
    100100                    'desc' => "Disable keyframe extraction",
    101                     'type' => "enum",
    102                     'list' => [{'name' => "true", 'desc' => "{common.true}"},
    103                                {'name' => "false", 'desc' => "{common.false}"}],
    104                     'deft' => "false",
     101                    'type' => "flag",
    105102                    'reqd' => "no" },
    106103                ];
    … …  
    262259  # 2. Convert into FLV, reprocess to make seekable, and associate
    263260  # - generate a path for our temporary converted video file
    264   print STDERR " - Converting video to streamble format...\n";
    265261  my $ovideo_path = &FileUtils::filenameConcatenate($tmp_dir, 'gsv.mp4');
    266   if (&FileUtils::fileExists($ovideo_path))
    267   {
    268     print "   - Found existing converted video in cache!\n";
     262  if ($self->{'streamingHQsize'} > 0)
     263  {
     264    print STDERR " - Converting video to streamble format...\n";
     265    if (&FileUtils::fileExists($ovideo_path))
     266    {
     267      print "   - Found existing converted video in cache!\n";
     268    }
     269    else
     270    {
     271      # - first conversion pass
     272      print "   - Convert using Handbrake\n";
     273      my $streaming_HQ_size    = $self->{'streamingHQsize'};
     274      my $streaming_HQ_VideoBitrate    = $self->{'streamingHQVideoBitrate'};
     275      my $streaming_HQ_AudioBitrate    = $self->{'streamingHQAudioBitrate'};
     276      my $deinterlace = $self->{'videoDeinterlacingFilter'};
     277      my $video_processing_parameters;
     278      if (!$streaming_HQ_size || $streaming_HQ_size eq "fullsize")
     279      {
     280        $video_processing_parameters = "--strict-anamorphic";
     281      }
     282      else
     283      {
     284        $video_processing_parameters = "-w $streaming_HQ_size --loose-anamorphic";
     285      }
     286      if ($deinterlace eq "true")
     287      {
     288        $video_processing_parameters .= " --decomb";
     289      }
     290      # Default MenCoder options for x264
     291      my $mencoder_options = 'ref=2:bframes=2:subq=6:mixed-refs=0:weightb=0:8x8dct=0:trellis=0';
     292      my $is_cluster = $self->{'isCluster'};
     293      my $is_parallel = $self->{'isParallel'};
     294      # If we are parallel processing on a single (presumably) multicore computer
     295      # then we need to limit the number of threads (and hence CPUs) HandBrake
     296      # will utilize in order to emulate true parallel processing (otherwise the
     297      # first thread to get to HandBrake conversion will take up most the CPUs
     298      # causing all other threads to wait anyway). It will interesting to test
     299      # whether parallel processing or serial processing (with HandBrake parallel
     300      # processing) is faster. *update* threads=1 *only* controls the encoding and
     301      # several other parts of Handbrake can run parallel (demuxing etc). I've
     302      # had to include a 'taskset' command to truely make Handbrake serial
     303      if ($is_parallel eq 'true'  && $is_cluster eq 'false')
     304      {
     305        $mencoder_options .= ':threads=1';
     306      }
     307      # Banish HandbrakeCLI to the (fixedCore-1)'th CPU if necessary
     308      my $cmd = '';
     309      if (defined $self->{'fixedCore'} && $self->{'fixedCore'} > 0)
     310      {
     311        $cmd .= 'taskset -c ' . ($self->{'fixedCore'} - 1) . ' ';
     312      }
     313      $cmd .= 'HandBrakeCLI -i "' . $ivideo_path . '" -t 1 -c 1 -f mp4 -O -o "' . $ovideo_path . '" ' . $video_processing_parameters . ' -e x264 -b ' . $streaming_HQ_VideoBitrate . ' -a 1 -E faac -6 dpl2 -R Auto -B ' . $streaming_HQ_AudioBitrate . ' -D 0.0 -x ' . $mencoder_options . ' > "' . $convert_log_path . '" 2>&1';
     314      my $attempt_count = 0;
     315      do
     316      {
     317        $attempt_count++;
     318        ###rint "[DEBUG: Video conversion attempt #" . $attempt_count . ": |" . $cmd . "|]\n";
     319        `$cmd`;
     320      }
     321      while ($attempt_count < 5 && !&FileUtils::fileExists($ovideo_path))
     322    }
     323    if (!&FileUtils::fileExists($ovideo_path))
     324    {
     325      die("Fatal Error! Failed to convert video: " . $ovideo_path . "\nReason:" . $! . "\n");
     326    }
     327    # Extra check - ensure the converted video is approximately the same duration
     328    # as the input video, given or take around 5 seconds
     329    my $output_raw_video_duration = &getDuration($ovideo_path);
     330    my $output_video_duration = &parseDurationAsSeconds($output_raw_video_duration);
     331    if (abs($input_video_duration - $output_video_duration) > 5)
     332    {
     333      print STDERR "!Warning! Output video does not have same duration as input video.\n";
     334    }
     335    print STDERR " - conversion done!\n";
    269336  }
    270337  else
    271338  {
    272     # - first conversion pass
    273     print "   - Convert using Handbrake\n";
    274     my $streaming_HQ_size    = $self->{'streamingHQsize'};
    275     my $streaming_HQ_VideoBitrate    = $self->{'streamingHQVideoBitrate'};
    276     my $streaming_HQ_AudioBitrate    = $self->{'streamingHQAudioBitrate'};
    277     my $deinterlace = $self->{'videoDeinterlacingFilter'};
    278     my $video_processing_parameters;
    279     if (!$streaming_HQ_size || $streaming_HQ_size eq "fullsize")
    280     {
    281       $video_processing_parameters = "--strict-anamorphic";
     339    print "   - skipping web-streamable conversion\n";
     340  }
     341
     342  # 3. Extract keyframes using hive
     343  my $generate_keyframes = 1;
     344  if (defined $self->{'no_keyframes'} && $self->{'no_keyframes'} == 1)
     345  {
     346    $generate_keyframes = 0;
     347    print "   - skipping keyframe generation\n";
     348  }
     349  else
     350  {
     351    print STDERR " - extract keyframes...\n";
     352    $tmp_dir = '/tmp/ramdrive';
     353    my $oshots_path = &FileUtils::filenameConcatenate($tmp_dir, 'shots.xml');
     354    if (&FileUtils::fileExists($oshots_path))
     355    {
     356      print "   - found existing keyframe images in cache\n";
    282357    }
    283358    else
    284359    {
    285       $video_processing_parameters = "-w $streaming_HQ_size --loose-anamorphic";
    286     }
    287     if ($deinterlace eq "true")
    288     {
    289       $video_processing_parameters .= " --decomb";
    290     }
    291     # Default MenCoder options for x264
    292     my $mencoder_options = 'ref=2:bframes=2:subq=6:mixed-refs=0:weightb=0:8x8dct=0:trellis=0';
    293     my $is_cluster = $self->{'isCluster'};
    294     my $is_parallel = $self->{'isParallel'};
    295     # If we are parallel processing on a single (presumably) multicore computer
    296     # then we need to limit the number of threads (and hence CPUs) HandBrake
    297     # will utilize in order to emulate true parallel processing (otherwise the
    298     # first thread to get to HandBrake conversion will take up most the CPUs
    299     # causing all other threads to wait anyway). It will interesting to test
    300     # whether parallel processing or serial processing (with HandBrake parallel
    301     # processing) is faster. *update* threads=1 *only* controls the encoding and
    302     # several other parts of Handbrake can run parallel (demuxing etc). I've
    303     # had to include a 'taskset' command to truely make Handbrake serial
    304     if ($is_parallel eq 'true'  && $is_cluster eq 'false')
    305     {
    306       $mencoder_options .= ':threads=1';
    307     }
    308     # Banish HandbrakeCLI to the (fixedCore-1)'th CPU if necessary
    309     my $cmd = '';
    310     if (defined $self->{'fixedCore'} && $self->{'fixedCore'} > 0)
    311     {
    312       $cmd .= 'taskset -c ' . ($self->{'fixedCore'} - 1) . ' ';
    313     }
    314     $cmd .= 'HandBrakeCLI -i "' . $ivideo_path . '" -t 1 -c 1 -f mp4 -O -o "' . $ovideo_path . '" ' . $video_processing_parameters . ' -e x264 -b ' . $streaming_HQ_VideoBitrate . ' -a 1 -E faac -6 dpl2 -R Auto -B ' . $streaming_HQ_AudioBitrate . ' -D 0.0 -x ' . $mencoder_options . ' > "' . $convert_log_path . '" 2>&1';
    315     my $attempt_count = 0;
    316     do
    317     {
    318       $attempt_count++;
    319       ###rint "[DEBUG: Video conversion attempt #" . $attempt_count . ": |" . $cmd . "|]\n";
     360      print "   - generating keyframe images using Hive2\n";
     361      my $cmd = 'hive2_ffmpegsvn -o "' . $oshots_path . '" -k "' . $tmp_dir . '" "' . $ovideo_path . '" >> "' . $convert_log_path . '" 2>&1';
     362      ###print "[cmd: " . $cmd . "]\n";
    320363      `$cmd`;
    321364    }
    322     while ($attempt_count < 5 && !&FileUtils::fileExists($ovideo_path))
    323   }
    324   if (!&FileUtils::fileExists($ovideo_path))
    325   {
    326     die("Fatal Error! Failed to convert video: " . $ovideo_path . "\nReason:" . $! . "\n");
    327   }
    328   # Extra check - ensure the converted video is approximately the same duration
    329   # as the input video, given or take around 5 seconds
    330   my $output_raw_video_duration = &getDuration($ovideo_path);
    331   my $output_video_duration = &parseDurationAsSeconds($output_raw_video_duration);
    332   if (abs($input_video_duration - $output_video_duration) > 5)
    333   {
    334     print STDERR "!Warning! Output video does not have same duration as input video.\n";
    335   }
    336   print STDERR " - conversion done!\n";
    337 
    338   # 3. Extract keyframes using hive
    339   print STDERR " - extract keyframes...\n";
    340   my $oshots_path = &FileUtils::filenameConcatenate($tmp_dir, 'shots.xml');
    341   if (&FileUtils::fileExists($oshots_path))
    342   {
    343     print "   - found existing keyframe images in cache\n";
    344   }
    345   else
    346   {
    347     print "   - generating keyframe images using Hive2\n";
    348     my $cmd = 'hive2_ffmpegsvn -o "' . $oshots_path . '" -k "' . $tmp_dir . '" "' . $ovideo_path . '" >> "' . $convert_log_path . '" 2>&1';
    349     ###print "[cmd: " . $cmd . "]\n";
    350     `$cmd`;
    351   }
    352   if (!&FileUtils::fileExists($oshots_path))
    353   {
    354     die("Fatal Error! Failed to extract keyframe images: " . $oshots_path . "\nReason:" . $! . "\n");
    355   }
    356   print STDERR "  - keyframes extracted!\n";
    357 
     365    if (!&FileUtils::fileExists($oshots_path))
     366    {
     367      die("Fatal Error! Failed to extract keyframe images: " . $oshots_path . "\nReason:" . $! . "\n");
     368    }
     369    print STDERR "  - keyframes extracted!\n";
     370  }
    358371
    359372  # 4. Associate files (copies back to shared space if IO separated)
    360373  print STDERR " - Associate derived files to doc_obj... ";
    361374  # - associate streamable video
    362   $doc_obj->associate_file($ovideo_path,'gsv.mp4','video/mp4',$topsection);
     375  if (-f $ovideo_path)
     376  {
     377    $doc_obj->associate_file($ovideo_path, 'gsv.mp4', 'video/mp4', $topsection);
     378  }
     379  else
     380  {
     381    $doc_obj->associate_file($ivideo_path, 'gsv.ts', 'video/ts', $topsection);
     382  }
    363383  # - associate all of the JPGs found in the temp directory
    364   opendir(my $dh, $tmp_dir);
    365   my @shots = readdir($dh);
    366   closedir($dh);
    367   my $thumbnail = 0;
    368   foreach my $shot (sort @shots)
    369   {
    370     my $shot_path = &FileUtils::filenameConcatenate($tmp_dir, $shot);
    371     if ($shot =~ /.jpg$/)
    372     {
    373       if (!$thumbnail)
    374       {
    375         $doc_obj->add_utf8_metadata($topsection,"Thumbnail",$shot);
    376         $thumbnail = 1;
    377       }
    378       $doc_obj->add_utf8_metadata($topsection,"Keyframe",$shot);
    379       $doc_obj->associate_file($shot_path,$shot,"image/jpeg",$topsection);
     384  if ($generate_keyframes)
     385  {
     386    opendir(my $dh, $tmp_dir);
     387    my @shots = readdir($dh);
     388    closedir($dh);
     389    my $thumbnail = 0;
     390    foreach my $shot (sort @shots)
     391    {
     392      my $shot_path = &FileUtils::filenameConcatenate($tmp_dir, $shot);
     393      if ($shot =~ /.jpg$/)
     394      {
     395        if (!$thumbnail)
     396        {
     397          $doc_obj->add_utf8_metadata($topsection,"Thumbnail",$shot);
     398          $thumbnail = 1;
     399        }
     400        $doc_obj->add_utf8_metadata($topsection,"Keyframe",$shot);
     401        #$doc_obj->associate_file($shot_path,$shot,"image/jpeg",$topsection);
     402      }
    380403    }
    381404  }
Note: See TracChangeset for help on using the changeset viewer.