source: main/trunk/greenstone2/perllib/plugins/EmbeddedMetadataPlugin.pm@ 24971

Last change on this file since 24971 was 24971, checked in by ak19, 12 years ago
  1. Introduced the util::filepath_to_url_format subroutine which will be used to convert filenames to URL style filenames to match the slashes used in the filename regex-es in extrameta keys used to index into extrameta data structures. 2. Fixed bug on windows where metadata.xml specifies filenames as regex with backslash in front of the file extension's period mark: DirectoryPlugin needed to unregex the filepath before calling fileparse on it, else the escaping backslash would interfere with perl's fileparse routine (only on windows, since backslash also represents a dirsep here). 3. Updated all those perl plugins where the new util::filepath_to_url_format needs to be called so that they use URL style filenames (thereafter regexed) to index into the extrameta data structures.
File size: 11.3 KB
Line 
1###########################################################################
2#
3# EmbeddedMetadataPlugin.pm -- A plugin for EXIF
4#
5# A component of the Greenstone digital library software
6# from the New Zealand Digital Library Project at the
7# University of Waikato, New Zealand.
8#
9# Copyright 2007 New Zealand Digital Library Project
10#
11# This program is free software; you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation; either version 2 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program; if not, write to the Free Software
23# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
24#
25###########################################################################
26
27
28package EmbeddedMetadataPlugin;
29
30use BasePlugin;
31use extrametautil;
32use util;
33
34use Encode;
35use Image::ExifTool qw(:Public);
36use strict;
37
38no strict 'refs'; # allow filehandles to be variables and viceversa
39
40
41sub BEGIN
42{
43 @EmbeddedMetadataPlugin::ISA = ('BasePlugin');
44 binmode(STDERR, ":utf8");
45}
46
47my $encoding_plus_auto_list = [{
48 'name' => "auto",
49 'desc' => "{ReadTextFile.input_encoding.auto}" }];
50push(@{$encoding_plus_auto_list},@{$BasePlugin::encoding_list});
51
52my $arguments = [{
53 'name' => "metadata_field_separator",
54 'desc' => "{HTMLPlugin.metadata_field_separator}",
55 'type' => "string",
56 'deft' => ""
57 },{
58 'name' => "input_encoding",
59 'desc' => "{ReadTextFile.input_encoding}",
60 'type' => "enum",
61 'list' => $encoding_plus_auto_list,
62 'reqd' => "no",
63 'deft' => "auto"
64 },{
65 'name' => "join_before_split",
66 'desc' => "{EmbeddedMetadataPlugin.join_before_split}",
67 'type' => "flag"
68 },{
69 'name' => "join_character",
70 'desc' => "{EmbeddedMetadataPlugin.join_character}",
71 'type' => "string",
72 'deft' => " "
73 },{
74 'name' => "trim_whitespace",
75 'desc' => "{EmbeddedMetadataPlugin.trim_whitespace}",
76 'type' => "enum",
77 'list' => [{'name' => "true", 'desc' => "{common.true}"}, {'name' => "false", 'desc' => "{common.false}"}],
78 'deft' => "true"
79 },{
80 'name' => "set_filter_list",
81 'desc' => "{EmbeddedMetadataPlugin.set_filter_list}",
82 'type' => "string"
83 },{
84 'name' => "set_filter_regexp",
85 'desc' => "{EmbeddedMetadataPlugin.set_filter_regexp}",
86 'type' => "string",
87 'deft' => ".*" #If changing this default, also need to update the constructor
88 }];
89
90my $options = {
91 'name' => "EmbeddedMetadataPlugin",
92 'desc' => "{EmbeddedMetadataPlugin.desc}",
93 'abstract' => "no",
94 'inherits' => "yes",
95 'args' => $arguments };
96
97sub new()
98{
99 my ($class) = shift (@_);
100 my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
101 push(@$pluginlist, $class);
102
103 if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
104 if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
105
106 my $self = new BasePlugin($pluginlist, $inputargs, $hashArgOptLists);
107
108 # Create a new Image::ExifTool object
109 my $exifTool = new Image::ExifTool;
110 $exifTool->Options(Duplicates => 0);
111 $exifTool->Options(PrintConv => 0);
112 $exifTool->Options(Unknown => 1);
113 $exifTool->Options('Verbose');
114 $self->{'exiftool'} = $exifTool;
115
116 my $setFilterList = $self->{'set_filter_list'};
117 my $setFilterRegexp = $self->{'set_filter_regexp'};
118 if ((defined $setFilterList) && ($setFilterList ne ""))
119 {
120 if ((defined $setFilterRegexp) && ($setFilterRegexp ne ".*") && ($setFilterRegexp ne ""))
121 {
122 my $outhandle = $self->{'outhandle'};
123 print $outhandle "Warning: can only specify 'set_filter_list' or 'set_filter_regexp'\n";
124 print $outhandle " defaulting to 'set_filter_list'\n";
125 }
126
127 my @sets = split(/,/,$setFilterList);
128 my @sets_bracketed;
129 foreach my $s (@sets)
130 {
131 $s =~ s/^(ex\.)?(.*)$/(ex.$2)/;
132 push (@sets_bracketed, $s);
133 }
134
135 my $setFilterRegexp = join("|",@sets_bracketed);
136 $self->{'set_filter_regexp'} = $setFilterRegexp;
137 }
138
139 return bless $self, $class;
140}
141
142
143# Need to think some more about this
144sub get_default_process_exp()
145{
146 return ".*";
147 #q^(?i)\.(wma|wmv|jpe?g|gif)$^;
148}
149
150
151# This plugin blocks *.oai files, so that they can be processed by OAIPlugin
152# even if OAIPlugin comes later in the pipeline than EmbeddedMetadataPlugin.
153sub get_default_block_exp()
154{
155 return q^(?i)\.(oai)$^;
156}
157
158# plugins that rely on more than process_exp (eg XML plugins) can override this method
159sub can_process_this_file {
160 my $self = shift(@_);
161
162 # we process metadata, not the file
163 return 0;
164}
165
166# Even if a plugin can extract metadata in its metadata_read pass,
167# make the default return 'undef' so processing of the file continues
168# down the pipeline, so other plugins can also have the opportunity to
169# locate metadata and set it up in the extrametakeys variables that
170# are passed around.
171
172sub can_process_this_file_for_metadata {
173 my $self = shift(@_);
174
175 # this plugin will look for metadata in any file through its
176 # metadata_read(). Returning undef here means anything else further
177 # down the pipeline can do the same
178
179 return undef;
180}
181
182sub checkAgainstFilters
183{
184 my $self = shift(@_);
185 my $name = shift(@_);
186
187 my $setFilterRegexp = $self->{'set_filter_regexp'};
188 if((defined $setFilterRegexp) && ($setFilterRegexp ne ""))
189 {
190 return ($name =~ m/($setFilterRegexp)/i);
191 }
192 else
193 {
194 return 1;
195 }
196}
197
198sub extractEmbeddedMetadata()
199{
200 my $self = shift(@_);
201 my ($file, $filename, $extrametadata, $extrametakeys) = @_;
202
203 my %exif_metadata = ();
204
205 my $verbosity = $self->{'verbosity'};
206 my $outhandle = $self->{'outhandle'};
207
208 my $metadata_count = 0;
209
210 my $separator = $self->{'metadata_field_separator'};
211 if ($separator eq "") {
212 undef $separator;
213 }
214
215 my @group_list = Image::ExifTool::GetAllGroups(0);
216 foreach my $group (@group_list) {
217## print STDERR "**** group = $group\n";
218
219 # Extract meta information from an image
220 $self->{'exiftool'}->Options(Group0 => [$group]);
221 $self->{'exiftool'}->ExtractInfo($filename);
222
223 # Get list of tags in the order they were found in the file
224 my @tag_list = $self->{'exiftool'}->GetFoundTags('File');
225 foreach my $tag (@tag_list) {
226
227 # Strip any numbering suffix
228 $tag =~ s/^([^\s]+)\s.*$/$1/i;
229 my $value = $self->{'exiftool'}->GetValue($tag);
230 if (defined $value && $value =~ /[a-z0-9]+/i) {
231 my $field = "ex.$group.$tag";
232
233 my $encoding = $self->{'input_encoding'};
234 if($encoding eq "auto")
235 {
236 $encoding = "utf8"
237 }
238
239 if (!defined $exif_metadata{$field})
240 {
241 $exif_metadata{$field} = [];
242 }
243
244 $field = Encode::decode($encoding,$field);
245 my $metadata_done = 0;
246 if (ref $value eq 'SCALAR') {
247 if ($$value =~ /^Binary data/) {
248 $value = "($$value)";
249 }
250 else {
251 my $len = length($$value);
252 $value = "(Binary data $len bytes)";
253 }
254 }
255 elsif (ref $value eq 'ARRAY') {
256 $metadata_done = 1;
257
258 my $allvals = "";
259 foreach my $v (@$value) {
260 $v = Encode::decode($encoding,$v);
261
262 if(!$self->{'join_before_split'}){
263 if (defined $separator) {
264 my @vs = split($separator, $v);
265 foreach my $val (@vs) {
266 if ($val =~ /\S/) {
267 push (@{$exif_metadata{$field}}, $self->gsSafe($val)) if $self->checkAgainstFilters($field);
268 ++$metadata_count;
269 }
270 }
271 }
272 else
273 {
274 push (@{$exif_metadata{$field}}, $self->gsSafe($v)) if $self->checkAgainstFilters($field);
275 ++$metadata_count;
276 }
277 }
278 else{
279 if($allvals ne ""){
280 $allvals = $allvals . $self->{'join_character'};
281 }
282 $allvals = $allvals . $v;
283 }
284 }
285
286 if($self->{'join_before_split'}){
287 if (defined $separator) {
288 my @vs = split($separator, $allvals);
289 foreach my $val (@vs) {
290 if ($val =~ /\S/) {
291 push (@{$exif_metadata{$field}}, $self->gsSafe($val)) if $self->checkAgainstFilters($field);
292 ++$metadata_count;
293 }
294 }
295 }
296 else
297 {
298 push (@{$exif_metadata{$field}}, $self->gsSafe($allvals)) if $self->checkAgainstFilters($field);
299 ++$metadata_count;
300 }
301 }
302 }
303 else {
304 $value = Encode::decode($encoding,$value);
305 if (defined $separator) {
306 my @vs = split($separator, $value);
307 $metadata_done = 1;
308 foreach my $v (@vs) {
309 if ($v =~ /\S/) {
310 push (@{$exif_metadata{$field}}, $self->gsSafe($v)) if $self->checkAgainstFilters($field);
311 ++$metadata_count;
312 }
313 }
314 }
315 }
316 if (!$metadata_done) {
317 push (@{$exif_metadata{$field}}, $self->gsSafe($value)) if $self->checkAgainstFilters($field);
318 ++$metadata_count;
319 }
320 }
321 }
322 }
323
324 if ($metadata_count > 0) {
325 print $outhandle " Extracted $metadata_count pieces of metadata from $filename EXIF block\n";
326 }
327
328 # Indexing into the extrameta data structures requires the filename's style of slashes to be in URL format
329 # Then need to convert the filename to a regex, no longer to protect windows directory chars \, but for
330 # protecting special characters like brackets in the filepath such as "C:\Program Files (x86)\Greenstone".
331 $file = &util::filepath_to_url_format($file);
332 $file = &util::filename_to_regex($file);
333
334 # Associate the metadata now
335
336 if (defined &extrametautil::getmetadata($extrametadata, $file)) {
337 print STDERR "\n**** Need to merge new metadata with existing stored metadata: file = $file\n" if $verbosity > 2;
338
339 my $file_metadata_table = &extrametautil::getmetadata($extrametadata, $file);
340
341 foreach my $metaname (keys %exif_metadata) {
342 # will create new entry if one does not already exist
343 push(@{$file_metadata_table->{$metaname}}, @{$exif_metadata{$metaname}});
344 }
345
346 # no need to push $file on to $extrametakeys as it is already in the list
347 }
348 else {
349 &extrametautil::setmetadata($extrametadata, $file, \%exif_metadata);
350 &extrametautil::addmetakey($extrametakeys, $file);
351 }
352
353}
354
355
356sub metadata_read
357{
358 my $self = shift (@_);
359 my ($pluginfo, $base_dir, $file, $block_hash,
360 $extrametakeys, $extrametadata, $extrametafile,
361 $processor, $gli, $aux) = @_;
362
363 my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
364
365 # we don't want to process directories
366 if (!-f $filename_full_path) {
367 return undef;
368 }
369 print STDERR "\n<Processing n='$file' p='EmbeddedMetadataPlugin'>\n" if ($gli);
370 print STDERR "EmbeddedMetadataPlugin: processing $file\n" if ($self->{'verbosity'}) > 1;
371
372 $self->extractEmbeddedMetadata($filename_no_path,$filename_full_path,
373 $extrametadata,$extrametakeys);
374
375 return undef;
376}
377
378sub read
379{
380 return undef;
381}
382
383sub process
384{
385 # not used
386 return undef;
387}
388
389sub gsSafe() {
390 my $self = shift(@_);
391 my ($text) = @_;
392
393 # Replace potentially problematic characters
394 $text =~ s/\(/&#40;/g;
395 $text =~ s/\)/&#41;/g;
396 $text =~ s/,/&#44;/g;
397 $text =~ s/\</&#60;/g;
398 $text =~ s/\>/&#62;/g;
399 $text =~ s/\[/&#91;/g;
400 $text =~ s/\]/&#93;/g;
401 $text =~ s/\{/&#123;/g;
402 $text =~ s/\}/&#125;/g;
403 # Done
404
405 if ($self->{'trim_whitespace'} eq "true"){
406 $text =~ s/^\s+//;
407 $text =~ s/\s+$//;
408 }
409
410 return $text;
411}
412
4131;
Note: See TracBrowser for help on using the repository browser.