Changeset 29179 for gs2-extensions

Show
Ignore:
Timestamp:
12.08.2014 09:18:25 (5 years ago)
Author:
jmt12
Message:

Changes to support low and med cpu loads and to avoid excessive IO in high CPU mode

Files:
1 modified

Legend:

Unmodified
Added
Removed
  • gs2-extensions/video-and-audio/trunk/src/perllib/plugins/SimpleVideoPlugin.pm

    r29028 r29179  
    9999                  { 'name' => "no_keyframes", 
    100100                    'desc' => "Disable keyframe extraction", 
    101                     'type' => "enum", 
    102                     'list' => [{'name' => "true", 'desc' => "{common.true}"}, 
    103                                {'name' => "false", 'desc' => "{common.false}"}], 
    104                     'deft' => "false", 
     101                    'type' => "flag", 
    105102                    'reqd' => "no" }, 
    106103                ]; 
     
    262259  # 2. Convert into FLV, reprocess to make seekable, and associate 
    263260  # - generate a path for our temporary converted video file 
    264   print STDERR " - Converting video to streamble format...\n"; 
    265261  my $ovideo_path = &FileUtils::filenameConcatenate($tmp_dir, 'gsv.mp4'); 
    266   if (&FileUtils::fileExists($ovideo_path)) 
    267   { 
    268     print "   - Found existing converted video in cache!\n"; 
     262  if ($self->{'streamingHQsize'} > 0) 
     263  { 
     264    print STDERR " - Converting video to streamble format...\n"; 
     265    if (&FileUtils::fileExists($ovideo_path)) 
     266    { 
     267      print "   - Found existing converted video in cache!\n"; 
     268    } 
     269    else 
     270    { 
     271      # - first conversion pass 
     272      print "   - Convert using Handbrake\n"; 
     273      my $streaming_HQ_size    = $self->{'streamingHQsize'}; 
     274      my $streaming_HQ_VideoBitrate    = $self->{'streamingHQVideoBitrate'}; 
     275      my $streaming_HQ_AudioBitrate    = $self->{'streamingHQAudioBitrate'}; 
     276      my $deinterlace = $self->{'videoDeinterlacingFilter'}; 
     277      my $video_processing_parameters; 
     278      if (!$streaming_HQ_size || $streaming_HQ_size eq "fullsize") 
     279      { 
     280        $video_processing_parameters = "--strict-anamorphic"; 
     281      } 
     282      else 
     283      { 
     284        $video_processing_parameters = "-w $streaming_HQ_size --loose-anamorphic"; 
     285      } 
     286      if ($deinterlace eq "true") 
     287      { 
     288        $video_processing_parameters .= " --decomb"; 
     289      } 
     290      # Default MenCoder options for x264 
     291      my $mencoder_options = 'ref=2:bframes=2:subq=6:mixed-refs=0:weightb=0:8x8dct=0:trellis=0'; 
     292      my $is_cluster = $self->{'isCluster'}; 
     293      my $is_parallel = $self->{'isParallel'}; 
     294      # If we are parallel processing on a single (presumably) multicore computer 
     295      # then we need to limit the number of threads (and hence CPUs) HandBrake 
     296      # will utilize in order to emulate true parallel processing (otherwise the 
     297      # first thread to get to HandBrake conversion will take up most the CPUs 
     298      # causing all other threads to wait anyway). It will interesting to test 
     299      # whether parallel processing or serial processing (with HandBrake parallel 
     300      # processing) is faster. *update* threads=1 *only* controls the encoding and 
     301      # several other parts of Handbrake can run parallel (demuxing etc). I've 
     302      # had to include a 'taskset' command to truely make Handbrake serial 
     303      if ($is_parallel eq 'true'  && $is_cluster eq 'false') 
     304      { 
     305        $mencoder_options .= ':threads=1'; 
     306      } 
     307      # Banish HandbrakeCLI to the (fixedCore-1)'th CPU if necessary 
     308      my $cmd = ''; 
     309      if (defined $self->{'fixedCore'} && $self->{'fixedCore'} > 0) 
     310      { 
     311        $cmd .= 'taskset -c ' . ($self->{'fixedCore'} - 1) . ' '; 
     312      } 
     313      $cmd .= 'HandBrakeCLI -i "' . $ivideo_path . '" -t 1 -c 1 -f mp4 -O -o "' . $ovideo_path . '" ' . $video_processing_parameters . ' -e x264 -b ' . $streaming_HQ_VideoBitrate . ' -a 1 -E faac -6 dpl2 -R Auto -B ' . $streaming_HQ_AudioBitrate . ' -D 0.0 -x ' . $mencoder_options . ' > "' . $convert_log_path . '" 2>&1'; 
     314      my $attempt_count = 0; 
     315      do 
     316      { 
     317        $attempt_count++; 
     318        ###rint "[DEBUG: Video conversion attempt #" . $attempt_count . ": |" . $cmd . "|]\n"; 
     319        `$cmd`; 
     320      } 
     321      while ($attempt_count < 5 && !&FileUtils::fileExists($ovideo_path)) 
     322    } 
     323    if (!&FileUtils::fileExists($ovideo_path)) 
     324    { 
     325      die("Fatal Error! Failed to convert video: " . $ovideo_path . "\nReason:" . $! . "\n"); 
     326    } 
     327    # Extra check - ensure the converted video is approximately the same duration 
     328    # as the input video, given or take around 5 seconds 
     329    my $output_raw_video_duration = &getDuration($ovideo_path); 
     330    my $output_video_duration = &parseDurationAsSeconds($output_raw_video_duration); 
     331    if (abs($input_video_duration - $output_video_duration) > 5) 
     332    { 
     333      print STDERR "!Warning! Output video does not have same duration as input video.\n"; 
     334    } 
     335    print STDERR " - conversion done!\n"; 
    269336  } 
    270337  else 
    271338  { 
    272     # - first conversion pass 
    273     print "   - Convert using Handbrake\n"; 
    274     my $streaming_HQ_size    = $self->{'streamingHQsize'}; 
    275     my $streaming_HQ_VideoBitrate    = $self->{'streamingHQVideoBitrate'}; 
    276     my $streaming_HQ_AudioBitrate    = $self->{'streamingHQAudioBitrate'}; 
    277     my $deinterlace = $self->{'videoDeinterlacingFilter'}; 
    278     my $video_processing_parameters; 
    279     if (!$streaming_HQ_size || $streaming_HQ_size eq "fullsize") 
    280     { 
    281       $video_processing_parameters = "--strict-anamorphic"; 
     339    print "   - skipping web-streamable conversion\n"; 
     340  } 
     341 
     342  # 3. Extract keyframes using hive 
     343  my $generate_keyframes = 1; 
     344  if (defined $self->{'no_keyframes'} && $self->{'no_keyframes'} == 1) 
     345  { 
     346    $generate_keyframes = 0; 
     347    print "   - skipping keyframe generation\n"; 
     348  } 
     349  else 
     350  { 
     351    print STDERR " - extract keyframes...\n"; 
     352    $tmp_dir = '/tmp/ramdrive'; 
     353    my $oshots_path = &FileUtils::filenameConcatenate($tmp_dir, 'shots.xml'); 
     354    if (&FileUtils::fileExists($oshots_path)) 
     355    { 
     356      print "   - found existing keyframe images in cache\n"; 
    282357    } 
    283358    else 
    284359    { 
    285       $video_processing_parameters = "-w $streaming_HQ_size --loose-anamorphic"; 
    286     } 
    287     if ($deinterlace eq "true") 
    288     { 
    289       $video_processing_parameters .= " --decomb"; 
    290     } 
    291     # Default MenCoder options for x264 
    292     my $mencoder_options = 'ref=2:bframes=2:subq=6:mixed-refs=0:weightb=0:8x8dct=0:trellis=0'; 
    293     my $is_cluster = $self->{'isCluster'}; 
    294     my $is_parallel = $self->{'isParallel'}; 
    295     # If we are parallel processing on a single (presumably) multicore computer 
    296     # then we need to limit the number of threads (and hence CPUs) HandBrake 
    297     # will utilize in order to emulate true parallel processing (otherwise the 
    298     # first thread to get to HandBrake conversion will take up most the CPUs 
    299     # causing all other threads to wait anyway). It will interesting to test 
    300     # whether parallel processing or serial processing (with HandBrake parallel 
    301     # processing) is faster. *update* threads=1 *only* controls the encoding and 
    302     # several other parts of Handbrake can run parallel (demuxing etc). I've 
    303     # had to include a 'taskset' command to truely make Handbrake serial 
    304     if ($is_parallel eq 'true'  && $is_cluster eq 'false') 
    305     { 
    306       $mencoder_options .= ':threads=1'; 
    307     } 
    308     # Banish HandbrakeCLI to the (fixedCore-1)'th CPU if necessary 
    309     my $cmd = ''; 
    310     if (defined $self->{'fixedCore'} && $self->{'fixedCore'} > 0) 
    311     { 
    312       $cmd .= 'taskset -c ' . ($self->{'fixedCore'} - 1) . ' '; 
    313     } 
    314     $cmd .= 'HandBrakeCLI -i "' . $ivideo_path . '" -t 1 -c 1 -f mp4 -O -o "' . $ovideo_path . '" ' . $video_processing_parameters . ' -e x264 -b ' . $streaming_HQ_VideoBitrate . ' -a 1 -E faac -6 dpl2 -R Auto -B ' . $streaming_HQ_AudioBitrate . ' -D 0.0 -x ' . $mencoder_options . ' > "' . $convert_log_path . '" 2>&1'; 
    315     my $attempt_count = 0; 
    316     do 
    317     { 
    318       $attempt_count++; 
    319       ###rint "[DEBUG: Video conversion attempt #" . $attempt_count . ": |" . $cmd . "|]\n"; 
     360      print "   - generating keyframe images using Hive2\n"; 
     361      my $cmd = 'hive2_ffmpegsvn -o "' . $oshots_path . '" -k "' . $tmp_dir . '" "' . $ovideo_path . '" >> "' . $convert_log_path . '" 2>&1'; 
     362      ###print "[cmd: " . $cmd . "]\n"; 
    320363      `$cmd`; 
    321364    } 
    322     while ($attempt_count < 5 && !&FileUtils::fileExists($ovideo_path)) 
    323   } 
    324   if (!&FileUtils::fileExists($ovideo_path)) 
    325   { 
    326     die("Fatal Error! Failed to convert video: " . $ovideo_path . "\nReason:" . $! . "\n"); 
    327   } 
    328   # Extra check - ensure the converted video is approximately the same duration 
    329   # as the input video, given or take around 5 seconds 
    330   my $output_raw_video_duration = &getDuration($ovideo_path); 
    331   my $output_video_duration = &parseDurationAsSeconds($output_raw_video_duration); 
    332   if (abs($input_video_duration - $output_video_duration) > 5) 
    333   { 
    334     print STDERR "!Warning! Output video does not have same duration as input video.\n"; 
    335   } 
    336   print STDERR " - conversion done!\n"; 
    337  
    338   # 3. Extract keyframes using hive 
    339   print STDERR " - extract keyframes...\n"; 
    340   my $oshots_path = &FileUtils::filenameConcatenate($tmp_dir, 'shots.xml'); 
    341   if (&FileUtils::fileExists($oshots_path)) 
    342   { 
    343     print "   - found existing keyframe images in cache\n"; 
    344   } 
    345   else 
    346   { 
    347     print "   - generating keyframe images using Hive2\n"; 
    348     my $cmd = 'hive2_ffmpegsvn -o "' . $oshots_path . '" -k "' . $tmp_dir . '" "' . $ovideo_path . '" >> "' . $convert_log_path . '" 2>&1'; 
    349     ###print "[cmd: " . $cmd . "]\n"; 
    350     `$cmd`; 
    351   } 
    352   if (!&FileUtils::fileExists($oshots_path)) 
    353   { 
    354     die("Fatal Error! Failed to extract keyframe images: " . $oshots_path . "\nReason:" . $! . "\n"); 
    355   } 
    356   print STDERR "  - keyframes extracted!\n"; 
    357  
     365    if (!&FileUtils::fileExists($oshots_path)) 
     366    { 
     367      die("Fatal Error! Failed to extract keyframe images: " . $oshots_path . "\nReason:" . $! . "\n"); 
     368    } 
     369    print STDERR "  - keyframes extracted!\n"; 
     370  } 
    358371 
    359372  # 4. Associate files (copies back to shared space if IO separated) 
    360373  print STDERR " - Associate derived files to doc_obj... "; 
    361374  # - associate streamable video 
    362   $doc_obj->associate_file($ovideo_path,'gsv.mp4','video/mp4',$topsection); 
     375  if (-f $ovideo_path) 
     376  { 
     377    $doc_obj->associate_file($ovideo_path, 'gsv.mp4', 'video/mp4', $topsection); 
     378  } 
     379  else 
     380  { 
     381    $doc_obj->associate_file($ivideo_path, 'gsv.ts', 'video/ts', $topsection); 
     382  } 
    363383  # - associate all of the JPGs found in the temp directory 
    364   opendir(my $dh, $tmp_dir); 
    365   my @shots = readdir($dh); 
    366   closedir($dh); 
    367   my $thumbnail = 0; 
    368   foreach my $shot (sort @shots) 
    369   { 
    370     my $shot_path = &FileUtils::filenameConcatenate($tmp_dir, $shot); 
    371     if ($shot =~ /.jpg$/) 
    372     { 
    373       if (!$thumbnail) 
    374       { 
    375         $doc_obj->add_utf8_metadata($topsection,"Thumbnail",$shot); 
    376         $thumbnail = 1; 
    377       } 
    378       $doc_obj->add_utf8_metadata($topsection,"Keyframe",$shot); 
    379       $doc_obj->associate_file($shot_path,$shot,"image/jpeg",$topsection); 
     384  if ($generate_keyframes) 
     385  { 
     386    opendir(my $dh, $tmp_dir); 
     387    my @shots = readdir($dh); 
     388    closedir($dh); 
     389    my $thumbnail = 0; 
     390    foreach my $shot (sort @shots) 
     391    { 
     392      my $shot_path = &FileUtils::filenameConcatenate($tmp_dir, $shot); 
     393      if ($shot =~ /.jpg$/) 
     394      { 
     395        if (!$thumbnail) 
     396        { 
     397          $doc_obj->add_utf8_metadata($topsection,"Thumbnail",$shot); 
     398          $thumbnail = 1; 
     399        } 
     400        $doc_obj->add_utf8_metadata($topsection,"Keyframe",$shot); 
     401        #$doc_obj->associate_file($shot_path,$shot,"image/jpeg",$topsection); 
     402      } 
    380403    } 
    381404  }