Changeset 26146

Timestamp:

2012-08-30T22:29:12+12:00 (12 years ago)

Author:

davidb

Message:

Refinement to EmbeddedMetadataPlugin that allows it to operate with maxdocs, for faster prototype building/rebuilding. Rather than pass one every file in metadata_read() this plugin now abides by the process expression it had, which has (in this mode) been changed from '*' back to a more conservative one that looks for images and PDF docs

File:

• main/trunk/greenstone2/perllib/plugins/EmbeddedMetadataPlugin.pm (modified) (11 diffs)

main/trunk/greenstone2/perllib/plugins/EmbeddedMetadataPlugin.pm

@@ -140 +140 @@
 }
 
+sub begin {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $processor, $maxdocs) = @_;
+
+
+    # For most plugins, the metadata_read() phase either does not
+    # exist, or is very fast at processing the files, and so is
+    # not an undue burden on collection building.
+
+    # EmbeddedMetadataPlugin bucks this trend, as the ExifTool module
+    # it relies on needs to make a fairly detailed scan of the files
+    # that match the plugin's process expression.  This has the
+    # unfortunate side effect of hampering quick collection building
+    # with '-maxdocs'.  It is therefore worth a bit of non-standard
+    # "meddling" (as Anu would say) to help the special case of 
+    # 'maxdocs' run more quickly.  
+    #
+    # The approach is to notice how many files EmbeddedMetadtaPlugin
+    # has scanned, and once this reaches 'maxdocs' to then force the
+    # can_process_this_file_for_metadata() method to always return the
+    # answer 'not recognized' to prevent any further scanning.
+    # Bacause 'maxdocs' is not one of the standard parameters passed
+    # in to metadata_read() we need to store the value in the object
+    # using this method so it can be used at the relevant place in the
+    # code later on
+
+    $self->{'maxdocs'} = $maxdocs;
+    $self->{'exif_scanned_count'} = 0;
+
+}
+
 
 # Need to think some more about this
 sub get_default_process_exp()
 {
-    return ".*";
-    #q^(?i)\.(wma|wmv|jpe?g|gif)$^;
+##    return ".*";
+    q^(?i)\.(jpe?g|gif|png|tiff|pdf)$^;
 }
 
@@ -164 +195 @@
 sub can_process_this_file_for_metadata {
     my $self = shift(@_);
-
-    # this plugin will look for metadata in any file through its
-    # metadata_read(). Returning undef here means anything else further
-    # down the pipeline can do the same
-
-    return undef;
+    my ($filename) = (@_);
+
+    # Want this plugin to look for metadata in the named file using
+    # ExifTool through its metadata_read() function, as long as it
+    # matches the process expression. But first there are a few
+    # special cases to test for ...
+    #
+
+    if (-d $filename && !$self->{'can_process_directories'}) {
+    return 0;
+    }
+
+    if ($self->{'maxdocs'} != -1) {
+    $self->{'exif_scanned_count'}++;
+    if ($self->{'exif_scanned_count'} > $self->{'maxdocs'}) {
+        # Above the limit of files to scan
+        return 0;
+    }
+    }
+
+
+    if ($self->{'process_exp'} ne "" && $filename =~ /$self->{'process_exp'}/) {
+    # Even though we say yes to this here, because we are using a custom
+    # metadata_read() method in this plugin, we can also ensure the
+    # file is considered by other plugins in the pipeline
+
+    return 1;
+    }
+
+    # If we get to here then the answer is no for processing by this plugin
+    # Note :because this plugin has its own custom metadata_read(), even
+    #  though we return a 'no' here, this doesn't stop the file being
+    #  considered by other plugins in the pipleline for metadata_read().  
+    #  This is needed to ensure a file like metadata.xml (which would
+    #  normally not be of interest to this plugin) is passed on to
+    #  the plugin that does need to read it (MetadataPlugin in this case).
+
+    return 0;
 }
 
@@ -187 +250 @@
     }
 }
+
+sub filtered_add_metadata
+{
+    my $self = shift(@_);
+    my ($field,$val,$exif_metadata_ref) = @_;
+
+    my $count = 0;
+
+    if ($self->checkAgainstFilters($field)) {
+    push (@{$exif_metadata_ref->{$field}}, $self->gsSafe($val));
+    $count++;
+
+
+    if ($field =~ m/GPSPosition/) {
+        my ($lat,$long) = split(/\s+/,$val);
+        
+        push (@{$exif_metadata_ref->{"Longitude"}}, $self->gsSafe($long));
+        push (@{$exif_metadata_ref->{"Latitude"}}, $self->gsSafe($lat));
+        # 'count' keeps track of the number of items extracted from the file
+        # so for these 'on the side' values set, don't include them in 
+        # the count
+
+    }
+
+
+    if ($field =~ m/GPSDateTime/) {
+        my ($date,$time) = split(/\s+/,$val);
+
+        my ($yyyy,$mm,$dd) = ($date =~ m/^(\d{4}):(\d{2}):(\d{2})$/);
+
+        push (@{$exif_metadata_ref->{"Date"}}, $self->gsSafe("$yyyy$mm$dd"));
+        # as for Long/Lat don't need to increase 'count'
+
+    }
+
+    
+    }
+
+    return $count;
+}
+
 
 sub extractEmbeddedMetadata()
@@ -257 +361 @@
                                 foreach my $val (@vs) {
                                     if ($val =~ /\S/) {
-                                        push (@{$exif_metadata{$field}}, $self->gsSafe($val)) if $self->checkAgainstFilters($field);
-                                        ++$metadata_count;
+                                        $metadata_count += $self->filtered_add_metadata($field,$val,\%exif_metadata);
                                     }
                                 }
@@ -264 +367 @@
                             else
                             {
-                                push (@{$exif_metadata{$field}}, $self->gsSafe($v)) if $self->checkAgainstFilters($field);
-                                ++$metadata_count;
+                                $metadata_count += $self->filtered_add_metadata($field,$v,\%exif_metadata);
                             }
                         }
@@ -281 +383 @@
                             foreach my $val (@vs) {
                                 if ($val =~ /\S/) {
-                                    push (@{$exif_metadata{$field}}, $self->gsSafe($val)) if $self->checkAgainstFilters($field);
-                                    ++$metadata_count;
+                                        $metadata_count += $self->filtered_add_metadata($field,$val,\%exif_metadata);
                                 }
                             }
@@ -288 +389 @@
                         else
                         {
-                            push (@{$exif_metadata{$field}}, $self->gsSafe($allvals)) if $self->checkAgainstFilters($field);
-                            ++$metadata_count;
+                            $metadata_count += $self->filtered_add_metadata($field,$allvals,\%exif_metadata);
                         }
                     }
@@ -300 +400 @@
                         foreach my $v (@vs) {
                             if ($v =~ /\S/) {
-                                push (@{$exif_metadata{$field}}, $self->gsSafe($v)) if $self->checkAgainstFilters($field);
-                                ++$metadata_count; 
+                                $metadata_count += $self->filtered_add_metadata($field,$v,\%exif_metadata);
                             }
                         }
@@ -307 +406 @@
                 }
                 if (!$metadata_done) {
-                    push (@{$exif_metadata{$field}}, $self->gsSafe($value)) if $self->checkAgainstFilters($field);
-                    ++$metadata_count;
+                    $metadata_count += $self->filtered_add_metadata($field,$value,\%exif_metadata);
                 }
             }
@@ -354 +452 @@
  
     my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
-    
-    # we don't want to process directories
-    if (!-f $filename_full_path) {
+ 
+# Now handled in the can_process_this_file_for_metadata method
+#   
+#    # we don't want to process directories
+#    if (!-f $filename_full_path) {
+#   return undef;
+#    }
+
+    if (!$self->can_process_this_file_for_metadata($filename_full_path)) {
+
+    # Avoid scanning it with ExitTool ...
+    # ... but let any other plugin in metadata_read() passes pipeline
+    # consider it
+
     return undef;
     }
+
+
     print STDERR "\n<Processing n='$file' p='EmbeddedMetadataPlugin'>\n" if ($gli);
     print STDERR "EmbeddedMetadataPlugin: processing $file\n" if ($self->{'verbosity'}) > 1;
@@ -365 +476 @@
                    $extrametadata,$extrametakeys);
     
+    # also want it considered by other plugins in the metadata_read() pipeline
     return undef;
 }