source: main/trunk/greenstone2/perllib/plugins/EmbeddedMetadataPlugin.pm@ 24796

Last change on this file since 24796 was 24796, checked in by ak19, 12 years ago

EmbeddedMetadataPlugin now blocks *.oai files so that they can be processed by OAIPlugin even if this plugin comes later in the plugin pipeline than EmbeddedMetadataPlugin.
File size: 10.8 KB
Line 
1###########################################################################
2#
3# EmbeddedMetadataPlugin.pm -- A plugin for EXIF
4#
5# A component of the Greenstone digital library software
6# from the New Zealand Digital Library Project at the
7# University of Waikato, New Zealand.
8#
9# Copyright 2007 New Zealand Digital Library Project
10#
11# This program is free software; you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation; either version 2 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program; if not, write to the Free Software
23# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
24#
25###########################################################################
26
27
28package EmbeddedMetadataPlugin;
29
30use BasePlugin;
31
32use Encode;
33use Image::ExifTool qw(:Public);
34use strict;
35
36no strict 'refs'; # allow filehandles to be variables and viceversa
37
38
39sub BEGIN
40{
41 @EmbeddedMetadataPlugin::ISA = ('BasePlugin');
42 binmode(STDERR, ":utf8");
43}
44
45my $encoding_plus_auto_list = [{
46 'name' => "auto",
47 'desc' => "{ReadTextFile.input_encoding.auto}" }];
48push(@{$encoding_plus_auto_list},@{$BasePlugin::encoding_list});
49
50my $arguments = [{
51 'name' => "metadata_field_separator",
52 'desc' => "{HTMLPlugin.metadata_field_separator}",
53 'type' => "string",
54 'deft' => ""
55 },{
56 'name' => "input_encoding",
57 'desc' => "{ReadTextFile.input_encoding}",
58 'type' => "enum",
59 'list' => $encoding_plus_auto_list,
60 'reqd' => "no",
61 'deft' => "auto"
62 },{
63 'name' => "join_before_split",
64 'desc' => "{EmbeddedMetadataPlugin.join_before_split}",
65 'type' => "flag"
66 },{
67 'name' => "join_character",
68 'desc' => "{EmbeddedMetadataPlugin.join_character}",
69 'type' => "string",
70 'deft' => " "
71 },{
72 'name' => "trim_whitespace",
73 'desc' => "{EmbeddedMetadataPlugin.trim_whitespace}",
74 'type' => "enum",
75 'list' => [{'name' => "true", 'desc' => "{common.true}"}, {'name' => "false", 'desc' => "{common.false}"}],
76 'deft' => "true"
77 },{
78 'name' => "set_filter_list",
79 'desc' => "{EmbeddedMetadataPlugin.set_filter_list}",
80 'type' => "string"
81 },{
82 'name' => "set_filter_regexp",
83 'desc' => "{EmbeddedMetadataPlugin.set_filter_regexp}",
84 'type' => "string",
85 'deft' => ".*" #If changing this default, also need to update the constructor
86 }];
87
88my $options = {
89 'name' => "EmbeddedMetadataPlugin",
90 'desc' => "{EmbeddedMetadataPlugin.desc}",
91 'abstract' => "no",
92 'inherits' => "yes",
93 'args' => $arguments };
94
95sub new()
96{
97 my ($class) = shift (@_);
98 my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
99 push(@$pluginlist, $class);
100
101 if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
102 if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
103
104 my $self = new BasePlugin($pluginlist, $inputargs, $hashArgOptLists);
105
106 # Create a new Image::ExifTool object
107 my $exifTool = new Image::ExifTool;
108 $exifTool->Options(Duplicates => 0);
109 $exifTool->Options(PrintConv => 0);
110 $exifTool->Options(Unknown => 1);
111 $exifTool->Options('Verbose');
112 $self->{'exiftool'} = $exifTool;
113
114 my $setFilterList = $self->{'set_filter_list'};
115 my $setFilterRegexp = $self->{'set_filter_regexp'};
116 if ((defined $setFilterList) && ($setFilterList ne ""))
117 {
118 if ((defined $setFilterRegexp) && ($setFilterRegexp ne ".*") && ($setFilterRegexp ne ""))
119 {
120 my $outhandle = $self->{'outhandle'};
121 print $outhandle "Warning: can only specify 'set_filter_list' or 'set_filter_regexp'\n";
122 print $outhandle " defaulting to 'set_filter_list'\n";
123 }
124
125 my @sets = split(/,/,$setFilterList);
126 my @sets_bracketed;
127 foreach my $s (@sets)
128 {
129 $s =~ s/^(ex\.)?(.*)$/(ex.$2)/;
130 push (@sets_bracketed, $s);
131 }
132
133 my $setFilterRegexp = join("|",@sets_bracketed);
134 $self->{'set_filter_regexp'} = $setFilterRegexp;
135 }
136
137 return bless $self, $class;
138}
139
140
141# Need to think some more about this
142sub get_default_process_exp()
143{
144 return ".*";
145 #q^(?i)\.(wma|wmv|jpe?g|gif)$^;
146}
147
148
149# This plugin blocks *.oai files, so that they can be processed by OAIPlugin
150# even if OAIPlugin comes later in the pipeline than EmbeddedMetadataPlugin.
151sub get_default_block_exp()
152{
153 return q^(?i)\.(oai)$^;
154}
155
156# plugins that rely on more than process_exp (eg XML plugins) can override this method
157sub can_process_this_file {
158 my $self = shift(@_);
159
160 # we process metadata, not the file
161 return 0;
162}
163
164# Even if a plugin can extract metadata in its metadata_read pass,
165# make the default return 'undef' so processing of the file continues
166# down the pipeline, so other plugins can also have the opportunity to
167# locate metadata and set it up in the extrametakeys variables that
168# are passed around.
169
170sub can_process_this_file_for_metadata {
171 my $self = shift(@_);
172
173 # this plugin will look for metadata in any file through its
174 # metadata_read(). Returning undef here means anything else further
175 # down the pipeline can do the same
176
177 return undef;
178}
179
180sub checkAgainstFilters
181{
182 my $self = shift(@_);
183 my $name = shift(@_);
184
185 my $setFilterRegexp = $self->{'set_filter_regexp'};
186 if((defined $setFilterRegexp) && ($setFilterRegexp ne ""))
187 {
188 return ($name =~ m/($setFilterRegexp)/i);
189 }
190 else
191 {
192 return 1;
193 }
194}
195
196sub extractEmbeddedMetadata()
197{
198 my $self = shift(@_);
199 my ($file, $filename, $extrametadata, $extrametakeys) = @_;
200
201 my %exif_metadata = ();
202
203 my $verbosity = $self->{'verbosity'};
204 my $outhandle = $self->{'outhandle'};
205
206 my $metadata_count = 0;
207
208 my $separator = $self->{'metadata_field_separator'};
209 if ($separator eq "") {
210 undef $separator;
211 }
212
213 my @group_list = Image::ExifTool::GetAllGroups(0);
214 foreach my $group (@group_list) {
215## print STDERR "**** group = $group\n";
216
217 # Extract meta information from an image
218 $self->{'exiftool'}->Options(Group0 => [$group]);
219 $self->{'exiftool'}->ExtractInfo($filename);
220
221 # Get list of tags in the order they were found in the file
222 my @tag_list = $self->{'exiftool'}->GetFoundTags('File');
223 foreach my $tag (@tag_list) {
224
225 # Strip any numbering suffix
226 $tag =~ s/^([^\s]+)\s.*$/$1/i;
227 my $value = $self->{'exiftool'}->GetValue($tag);
228 if (defined $value && $value =~ /[a-z0-9]+/i) {
229 my $field = "ex.$group.$tag";
230
231 my $encoding = $self->{'input_encoding'};
232 if($encoding eq "auto")
233 {
234 $encoding = "utf8"
235 }
236
237 if (!defined $exif_metadata{$field})
238 {
239 $exif_metadata{$field} = [];
240 }
241
242 $field = Encode::decode($encoding,$field);
243 my $metadata_done = 0;
244 if (ref $value eq 'SCALAR') {
245 if ($$value =~ /^Binary data/) {
246 $value = "($$value)";
247 }
248 else {
249 my $len = length($$value);
250 $value = "(Binary data $len bytes)";
251 }
252 }
253 elsif (ref $value eq 'ARRAY') {
254 $metadata_done = 1;
255
256 my $allvals = "";
257 foreach my $v (@$value) {
258 $v = Encode::decode($encoding,$v);
259
260 if(!$self->{'join_before_split'}){
261 if (defined $separator) {
262 my @vs = split($separator, $v);
263 foreach my $val (@vs) {
264 if ($val =~ /\S/) {
265 push (@{$exif_metadata{$field}}, $self->gsSafe($val)) if $self->checkAgainstFilters($field);
266 ++$metadata_count;
267 }
268 }
269 }
270 else
271 {
272 push (@{$exif_metadata{$field}}, $self->gsSafe($v)) if $self->checkAgainstFilters($field);
273 ++$metadata_count;
274 }
275 }
276 else{
277 if($allvals ne ""){
278 $allvals = $allvals . $self->{'join_character'};
279 }
280 $allvals = $allvals . $v;
281 }
282 }
283
284 if($self->{'join_before_split'}){
285 if (defined $separator) {
286 my @vs = split($separator, $allvals);
287 foreach my $val (@vs) {
288 if ($val =~ /\S/) {
289 push (@{$exif_metadata{$field}}, $self->gsSafe($val)) if $self->checkAgainstFilters($field);
290 ++$metadata_count;
291 }
292 }
293 }
294 else
295 {
296 push (@{$exif_metadata{$field}}, $self->gsSafe($allvals)) if $self->checkAgainstFilters($field);
297 ++$metadata_count;
298 }
299 }
300 }
301 else {
302 $value = Encode::decode($encoding,$value);
303 if (defined $separator) {
304 my @vs = split($separator, $value);
305 $metadata_done = 1;
306 foreach my $v (@vs) {
307 if ($v =~ /\S/) {
308 push (@{$exif_metadata{$field}}, $self->gsSafe($v)) if $self->checkAgainstFilters($field);
309 ++$metadata_count;
310 }
311 }
312 }
313 }
314 if (!$metadata_done) {
315 push (@{$exif_metadata{$field}}, $self->gsSafe($value)) if $self->checkAgainstFilters($field);
316 ++$metadata_count;
317 }
318 }
319 }
320 }
321
322 if ($metadata_count > 0) {
323 print $outhandle " Extracted $metadata_count pieces of metadata from $filename EXIF block\n";
324 }
325
326 # Protect windows directory chars \
327 $file = &util::filename_to_regex($file);
328
329 # Associate the metadata now
330
331 if (defined $extrametadata->{$file}) {
332 print STDERR "\n**** Need to merge new metadata with existing stored metadata: file = $file\n" if $verbosity > 2;
333
334 my $file_metadata_table = $extrametadata->{$file};
335
336 foreach my $metaname (keys %exif_metadata) {
337 # will create new entry if one does not already exist
338 push(@{$file_metadata_table->{$metaname}}, @{$exif_metadata{$metaname}});
339 }
340
341 # no need to push $file on to $extrametakeys as it is already in the list
342 }
343 else {
344 $extrametadata->{$file} = \%exif_metadata;
345 push(@$extrametakeys, $file);
346 }
347
348}
349
350
351sub metadata_read
352{
353 my $self = shift (@_);
354 my ($pluginfo, $base_dir, $file, $block_hash,
355 $extrametakeys, $extrametadata, $extrametafile,
356 $processor, $gli, $aux) = @_;
357
358 my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
359
360 # we don't want to process directories
361 if (!-f $filename_full_path) {
362 return undef;
363 }
364 print STDERR "\n<Processing n='$file' p='EmbeddedMetadataPlugin'>\n" if ($gli);
365 print STDERR "EmbeddedMetadataPlugin: processing $file\n" if ($self->{'verbosity'}) > 1;
366
367 $self->extractEmbeddedMetadata($filename_no_path,$filename_full_path,
368 $extrametadata,$extrametakeys);
369
370 return undef;
371}
372
373sub read
374{
375 return undef;
376}
377
378sub process
379{
380 # not used
381 return undef;
382}
383
384sub gsSafe() {
385 my $self = shift(@_);
386 my ($text) = @_;
387
388 # Replace potentially problematic characters
389 $text =~ s/\(/&#40;/g;
390 $text =~ s/\)/&#41;/g;
391 $text =~ s/,/&#44;/g;
392 $text =~ s/\</&#60;/g;
393 $text =~ s/\>/&#62;/g;
394 $text =~ s/\[/&#91;/g;
395 $text =~ s/\]/&#93;/g;
396 $text =~ s/\{/&#123;/g;
397 $text =~ s/\}/&#125;/g;
398 # Done
399
400 if ($self->{'trim_whitespace'} eq "true"){
401 $text =~ s/^\s+//;
402 $text =~ s/\s+$//;
403 }
404
405 return $text;
406}
407
4081;
Note: See TracBrowser for help on using the repository browser.