Changeset 26146

Show
Ignore:
Timestamp:
30.08.2012 22:29:12 (7 years ago)
Author:
davidb
Message:

Refinement to EmbeddedMetadataPlugin? that allows it to operate with maxdocs, for faster prototype building/rebuilding. Rather than pass one every file in metadata_read() this plugin now abides by the process expression it had, which has (in this mode) been changed from '*' back to a more conservative one that looks for images and PDF docs

Files:
1 modified

Legend:

Unmodified
Added
Removed
  • main/trunk/greenstone2/perllib/plugins/EmbeddedMetadataPlugin.pm

    r25332 r26146  
    140140} 
    141141 
     142sub begin { 
     143    my $self = shift (@_); 
     144    my ($pluginfo, $base_dir, $processor, $maxdocs) = @_; 
     145 
     146 
     147    # For most plugins, the metadata_read() phase either does not 
     148    # exist, or is very fast at processing the files, and so is 
     149    # not an undue burden on collection building. 
     150 
     151    # EmbeddedMetadataPlugin bucks this trend, as the ExifTool module 
     152    # it relies on needs to make a fairly detailed scan of the files 
     153    # that match the plugin's process expression.  This has the 
     154    # unfortunate side effect of hampering quick collection building 
     155    # with '-maxdocs'.  It is therefore worth a bit of non-standard 
     156    # "meddling" (as Anu would say) to help the special case of  
     157    # 'maxdocs' run more quickly.   
     158    # 
     159    # The approach is to notice how many files EmbeddedMetadtaPlugin 
     160    # has scanned, and once this reaches 'maxdocs' to then force the 
     161    # can_process_this_file_for_metadata() method to always return the 
     162    # answer 'not recognized' to prevent any further scanning. 
     163    # Bacause 'maxdocs' is not one of the standard parameters passed 
     164    # in to metadata_read() we need to store the value in the object 
     165    # using this method so it can be used at the relevant place in the 
     166    # code later on 
     167 
     168    $self->{'maxdocs'} = $maxdocs; 
     169    $self->{'exif_scanned_count'} = 0; 
     170 
     171} 
     172 
    142173 
    143174# Need to think some more about this 
    144175sub get_default_process_exp() 
    145176{ 
    146     return ".*"; 
    147     #q^(?i)\.(wma|wmv|jpe?g|gif)$^; 
     177##    return ".*"; 
     178    q^(?i)\.(jpe?g|gif|png|tiff|pdf)$^; 
    148179} 
    149180 
     
    164195sub can_process_this_file_for_metadata { 
    165196    my $self = shift(@_); 
    166  
    167     # this plugin will look for metadata in any file through its 
    168     # metadata_read(). Returning undef here means anything else further 
    169     # down the pipeline can do the same 
    170  
    171     return undef; 
     197    my ($filename) = (@_); 
     198 
     199    # Want this plugin to look for metadata in the named file using 
     200    # ExifTool through its metadata_read() function, as long as it 
     201    # matches the process expression. But first there are a few 
     202    # special cases to test for ... 
     203    # 
     204 
     205    if (-d $filename && !$self->{'can_process_directories'}) { 
     206    return 0; 
     207    } 
     208 
     209    if ($self->{'maxdocs'} != -1) { 
     210    $self->{'exif_scanned_count'}++; 
     211    if ($self->{'exif_scanned_count'} > $self->{'maxdocs'}) { 
     212        # Above the limit of files to scan 
     213        return 0; 
     214    } 
     215    } 
     216 
     217 
     218    if ($self->{'process_exp'} ne "" && $filename =~ /$self->{'process_exp'}/) { 
     219    # Even though we say yes to this here, because we are using a custom 
     220    # metadata_read() method in this plugin, we can also ensure the 
     221    # file is considered by other plugins in the pipeline 
     222 
     223    return 1; 
     224    } 
     225 
     226    # If we get to here then the answer is no for processing by this plugin 
     227    # Note :because this plugin has its own custom metadata_read(), even 
     228    #  though we return a 'no' here, this doesn't stop the file being 
     229    #  considered by other plugins in the pipleline for metadata_read().   
     230    #  This is needed to ensure a file like metadata.xml (which would 
     231    #  normally not be of interest to this plugin) is passed on to 
     232    #  the plugin that does need to read it (MetadataPlugin in this case). 
     233 
     234    return 0; 
    172235} 
    173236 
     
    187250    } 
    188251} 
     252 
     253sub filtered_add_metadata 
     254{ 
     255    my $self = shift(@_); 
     256    my ($field,$val,$exif_metadata_ref) = @_; 
     257 
     258    my $count = 0; 
     259 
     260    if ($self->checkAgainstFilters($field)) { 
     261    push (@{$exif_metadata_ref->{$field}}, $self->gsSafe($val)); 
     262    $count++; 
     263 
     264 
     265    if ($field =~ m/GPSPosition/) { 
     266        my ($lat,$long) = split(/\s+/,$val); 
     267         
     268        push (@{$exif_metadata_ref->{"Longitude"}}, $self->gsSafe($long)); 
     269        push (@{$exif_metadata_ref->{"Latitude"}}, $self->gsSafe($lat)); 
     270        # 'count' keeps track of the number of items extracted from the file 
     271        # so for these 'on the side' values set, don't include them in  
     272        # the count 
     273 
     274    } 
     275 
     276 
     277    if ($field =~ m/GPSDateTime/) { 
     278        my ($date,$time) = split(/\s+/,$val); 
     279 
     280        my ($yyyy,$mm,$dd) = ($date =~ m/^(\d{4}):(\d{2}):(\d{2})$/); 
     281 
     282        push (@{$exif_metadata_ref->{"Date"}}, $self->gsSafe("$yyyy$mm$dd")); 
     283        # as for Long/Lat don't need to increase 'count' 
     284 
     285    } 
     286 
     287     
     288    } 
     289 
     290    return $count; 
     291} 
     292 
    189293 
    190294sub extractEmbeddedMetadata() 
     
    257361                                foreach my $val (@vs) { 
    258362                                    if ($val =~ /\S/) { 
    259                                         push (@{$exif_metadata{$field}}, $self->gsSafe($val)) if $self->checkAgainstFilters($field); 
    260                                         ++$metadata_count; 
     363                                        $metadata_count += $self->filtered_add_metadata($field,$val,\%exif_metadata); 
    261364                                    } 
    262365                                } 
     
    264367                            else 
    265368                            { 
    266                                 push (@{$exif_metadata{$field}}, $self->gsSafe($v)) if $self->checkAgainstFilters($field); 
    267                                 ++$metadata_count; 
     369                                $metadata_count += $self->filtered_add_metadata($field,$v,\%exif_metadata); 
    268370                            } 
    269371                        } 
     
    281383                            foreach my $val (@vs) { 
    282384                                if ($val =~ /\S/) { 
    283                                     push (@{$exif_metadata{$field}}, $self->gsSafe($val)) if $self->checkAgainstFilters($field); 
    284                                     ++$metadata_count; 
     385                                        $metadata_count += $self->filtered_add_metadata($field,$val,\%exif_metadata); 
    285386                                } 
    286387                            } 
     
    288389                        else 
    289390                        { 
    290                             push (@{$exif_metadata{$field}}, $self->gsSafe($allvals)) if $self->checkAgainstFilters($field); 
    291                             ++$metadata_count; 
     391                            $metadata_count += $self->filtered_add_metadata($field,$allvals,\%exif_metadata); 
    292392                        } 
    293393                    } 
     
    300400                        foreach my $v (@vs) { 
    301401                            if ($v =~ /\S/) { 
    302                                 push (@{$exif_metadata{$field}}, $self->gsSafe($v)) if $self->checkAgainstFilters($field); 
    303                                 ++$metadata_count;  
     402                                $metadata_count += $self->filtered_add_metadata($field,$v,\%exif_metadata); 
    304403                            } 
    305404                        } 
     
    307406                } 
    308407                if (!$metadata_done) { 
    309                     push (@{$exif_metadata{$field}}, $self->gsSafe($value)) if $self->checkAgainstFilters($field); 
    310                     ++$metadata_count; 
     408                    $metadata_count += $self->filtered_add_metadata($field,$value,\%exif_metadata); 
    311409                } 
    312410            } 
     
    354452  
    355453    my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file); 
    356      
    357     # we don't want to process directories 
    358     if (!-f $filename_full_path) { 
     454  
     455# Now handled in the can_process_this_file_for_metadata method 
     456#    
     457#    # we don't want to process directories 
     458#    if (!-f $filename_full_path) { 
     459#   return undef; 
     460#    } 
     461 
     462    if (!$self->can_process_this_file_for_metadata($filename_full_path)) { 
     463 
     464    # Avoid scanning it with ExitTool ... 
     465    # ... but let any other plugin in metadata_read() passes pipeline 
     466    # consider it 
     467 
    359468    return undef; 
    360469    } 
     470 
     471 
    361472    print STDERR "\n<Processing n='$file' p='EmbeddedMetadataPlugin'>\n" if ($gli); 
    362473    print STDERR "EmbeddedMetadataPlugin: processing $file\n" if ($self->{'verbosity'}) > 1; 
     
    365476                   $extrametadata,$extrametakeys); 
    366477     
     478    # also want it considered by other plugins in the metadata_read() pipeline 
    367479    return undef; 
    368480}