source: gsdl/trunk/perllib/plugin.pm@ 19619

Last change on this file since 19619 was 19497, checked in by davidb, 15 years ago

Introduction of new extrametafile to track which metadata.xml file a piece of metadata came from

  • Property svn:keywords set to Author Date Id Revision
File size: 12.4 KB
Line 
1###########################################################################
2#
3# plugin.pm -- functions to handle using plugins
4# A component of the Greenstone digital library software
5# from the New Zealand Digital Library Project at the
6# University of Waikato, New Zealand.
7#
8# Copyright (C) 1999 New Zealand Digital Library Project
9#
10# This program is free software; you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation; either version 2 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program; if not, write to the Free Software
22# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
23#
24###########################################################################
25
26package plugin;
27
28use strict; # to pick up typos and undeclared variables...
29no strict 'refs'; # ...but allow filehandles to be variables and vice versa
30no strict 'subs';
31
32require util;
33use gsprintf 'gsprintf';
34
35# mapping from old plugin names to new ones for backwards compatibility
36# can remove at sometime in future when we no longer want to support old xxPlug names in the config file
37my $plugin_name_map = {
38 'GAPlug' => 'GreenstoneXMLPlugin',
39 'ArcPlug' => 'ArchivesInfPlugin',
40 'RecPlug' => 'DirectoryPlugin',
41 'TEXTPlug' => 'TextPlugin',
42 'XMLPlug' => 'ReadXMLFile',
43 'EMAILPlug' => 'EmailPlugin',
44 'SRCPlug' => 'SourceCodePlugin',
45 'NULPlug' => 'NulPlugin',
46 'W3ImgPlug' => 'HTMLImagePlugin',
47 'PagedImgPlug' => 'PagedImagePlugin',
48 'METSPlug' => 'GreenstoneMETSPlugin',
49 'PPTPlug' => 'PowerPointPlugin',
50 'PSPlug' => 'PostScriptPlugin',
51 'DBPlug' => 'DatabasePlugin'
52 };
53
54# global variables
55my $stats = {'num_processed' => 0,
56 'num_blocked' => 0,
57 'num_not_processed' => 0,
58 'num_not_recognised' => 0,
59 'num_archives' => 0
60 };
61
62#globaloptions contains any options that should be passed to all plugins
63my ($verbosity, $outhandle, $failhandle, $globaloptions);
64
65sub get_valid_pluginname {
66 my ($pluginname) = @_;
67 my $valid_name = $pluginname;
68 if (defined $plugin_name_map->{$pluginname}) {
69 $valid_name = $plugin_name_map->{$pluginname};
70 } elsif ($pluginname =~ /Plug$/) {
71 $valid_name =~ s/Plug/Plugin/;
72
73 }
74 return $valid_name;
75}
76sub load_plugin_require
77{
78 my ($pluginname) = @_;
79
80 my @check_list = ();
81
82 # pp_plugname shorthand for 'perllib' 'plugin' '$pluginname.pm'
83 my $pp_plugname
84 = &util::filename_cat('perllib', 'plugins', "${pluginname}.pm");
85 my $collectdir = $ENV{'GSDLCOLLECTDIR'};
86
87 # find the plugin
88 if (defined($ENV{'GSDLCOLLECTION'}))
89 {
90 my $customplugname
91 = &util::filename_cat($collectdir, "custom",$ENV{'GSDLCOLLECTION'},
92 $pp_plugname);
93 push(@check_list,$customplugname);
94 }
95
96 my $colplugname = &util::filename_cat($collectdir, $pp_plugname);
97 push(@check_list,$colplugname);
98
99 if (defined $ENV{'GSDLEXTS'}) {
100
101 my $ext_prefix = &util::filename_cat($ENV{'GSDLHOME'}, "ext");
102
103 my @extensions = split(/:/,$ENV{'GSDLEXTS'});
104 foreach my $e (@extensions) {
105 my $extplugname = &util::filename_cat($ext_prefix, $e, $pp_plugname);
106 push(@check_list,$extplugname);
107
108 }
109 }
110
111
112 my $mainplugname = &util::filename_cat($ENV{'GSDLHOME'}, $pp_plugname);
113 push(@check_list,$mainplugname);
114
115 my $success=0;
116 foreach my $plugname (@check_list) {
117 if (-e $plugname) {
118 require $plugname;
119 $success=1;
120 last;
121 }
122 }
123
124 if (!$success) {
125 &gsprintf(STDERR, "{plugin.could_not_find_plugin}\n",
126 $pluginname);
127 die "\n";
128 }
129}
130
131sub load_plugin_for_info {
132 my ($pluginname) = shift @_;
133 $pluginname = &get_valid_pluginname($pluginname);
134 load_plugin_require($pluginname);
135
136 # create a plugin object
137 my ($plugobj);
138 my $options = "-gsdlinfo";
139
140 eval ("\$plugobj = new \$pluginname([],[$options])");
141 die "$@" if $@;
142
143 return $plugobj;
144}
145
146sub load_plugins {
147 my ($plugin_list) = shift @_;
148 my $incremental;
149 ($verbosity, $outhandle, $failhandle, $globaloptions, $incremental) = @_; # globals
150 my @plugin_objects = ();
151 $incremental = 0 unless (defined $incremental && $incremental == 1);
152 $verbosity = 2 unless defined $verbosity;
153 $outhandle = 'STDERR' unless defined $outhandle;
154 $failhandle = 'STDERR' unless defined $failhandle;
155
156 my $colplugindir = &util::filename_cat($ENV{'GSDLCOLLECTDIR'},"perllib/plugins");
157 unshift (@INC, $colplugindir);
158
159 map { $_ = "\"$_\""; } @$globaloptions;
160 my $globals = join (",", @$globaloptions);
161
162 foreach my $pluginoptions (@$plugin_list) {
163 my $pluginname = shift @$pluginoptions;
164 next unless defined $pluginname;
165 $pluginname = &get_valid_pluginname($pluginname);
166 load_plugin_require($pluginname);
167
168 # create a plugin object
169 my ($plugobj);
170 map { $_ = "\"$_\""; } @$pluginoptions;
171 my $options = join (",", @$pluginoptions);
172 if ($globals) {
173 if (@$pluginoptions) {
174 $options .= ",";
175 }
176 $options .= "$globals";
177 }
178 $options =~ s/\$/\\\$/g;
179
180 eval ("\$plugobj = new \$pluginname([],[$options])");
181 die "$@" if $@;
182
183 # initialize plugin
184 $plugobj->init($verbosity, $outhandle, $failhandle);
185
186 $plugobj->set_incremental($incremental);
187
188 # add this object to the list
189 push (@plugin_objects, $plugobj);
190 }
191
192 return \@plugin_objects;
193}
194
195
196sub begin {
197 my ($pluginfo, $base_dir, $processor, $maxdocs, $gli) = @_;
198
199 map { $_->{'gli'} = $gli; } @$pluginfo;
200 map { $_->begin($pluginfo, $base_dir, $processor, $maxdocs); } @$pluginfo;
201}
202
203sub file_block_read {
204 my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $gli) = @_;
205
206
207 $gli = 0 unless defined $gli;
208
209 my $rv = 0;
210 my $glifile = $file;
211
212 $glifile =~ s/^[\/\\]+//; # file sometimes starts with a / so get rid of it
213
214 # Announce to GLI that we are handling a file
215 print STDERR "<File n='$glifile'>\n" if $gli;
216
217 # the .kill file is a handy (if not very elegant) way of aborting
218 # an import.pl or buildcol.pl process
219 if (-e &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, ".kill")) {
220 gsprintf($outhandle, "{plugin.kill_file}\n");
221 die "\n";
222 }
223
224 foreach my $plugobj (@$pluginfo) {
225
226 $rv = $plugobj->file_block_read($pluginfo, $base_dir, $file, $block_hash, $metadata, $gli);
227 #last if (defined $rv && $rv==1); # stop this file once we have found something to 'process' it
228 }
229
230}
231
232
233sub metadata_read {
234 my ($pluginfo, $base_dir, $file, $block_hash,
235 $extrametakeys, $extrametadata, $extrametafile,
236 $processor, $maxdocs, $gli, $aux) = @_;
237
238 $maxdocs = -1 unless defined $maxdocs && $maxdocs =~ /\d/;
239 $gli = 0 unless defined $gli;
240
241 my $rv = 0;
242 my $glifile = $file;
243
244 $glifile =~ s/^[\/\\]+//; # file sometimes starts with a / so get rid of it
245
246 # Announce to GLI that we are handling a file
247 print STDERR "<File n='$glifile'>\n" if $gli;
248
249 # the .kill file is a handy (if not very elegant) way of aborting
250 # an import.pl or buildcol.pl process
251 if (-e &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, ".kill")) {
252 gsprintf($outhandle, "{plugin.kill_file}\n");
253 die "\n";
254 }
255
256 my $had_error = 0;
257 # pass this file by each of the plugins in turn until one
258 # is found which will process it
259 # read must return:
260 # undef - could not recognise
261 # -1 - tried but error
262 # 0 - blocked
263 # anything else for successful processing
264
265 foreach my $plugobj (@$pluginfo) {
266
267 $rv = $plugobj->metadata_read($pluginfo, $base_dir, $file, $block_hash,
268 $extrametakeys, $extrametadata, $extrametafile,
269 $processor, $maxdocs, $gli, $aux);
270
271 if (defined $rv) {
272 if ($rv == -1) {
273 # an error has occurred
274 $had_error = 1;
275 print STDERR "<ProcessingError n='$glifile'>\n" if $gli;
276 } else {
277 return $rv;
278 }
279 } # else undefined - was not recognised by the plugin
280 }
281
282 return 0;
283}
284
285sub read {
286 my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $processor, $maxdocs, $total_count, $gli, $aux) = @_;
287
288 $maxdocs = -1 unless defined $maxdocs && $maxdocs =~ /\d/;
289 $total_count = 0 unless defined $total_count && $total_count =~ /\d/;
290 $gli = 0 unless defined $gli;
291
292 my $rv = 0;
293 my $glifile = $file;
294
295 $glifile =~ s/^[\/\\]+//; # file sometimes starts with a / so get rid of it
296
297 # Announce to GLI that we are handling a file
298 print STDERR "<File n='$glifile'>\n" if $gli;
299
300 # the .kill file is a handy (if not very elegant) way of aborting
301 # an import.pl or buildcol.pl process
302 if (-e &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, ".kill")) {
303 gsprintf($outhandle, "{plugin.kill_file}\n");
304 die "\n";
305 }
306
307 my $had_error = 0;
308 # pass this file by each of the plugins in turn until one
309 # is found which will process it
310 # read must return:
311 # undef - could not recognise
312 # -1 - tried but error
313 # 0 - blocked
314 # anything else for successful processing
315
316 foreach my $plugobj (@$pluginfo) {
317
318 $rv = $plugobj->read($pluginfo, $base_dir, $file,
319 $block_hash, $metadata, $processor, $maxdocs,
320 $total_count, $gli, $aux);
321
322 if (defined $rv) {
323 if ($rv == -1) {
324 # an error has occurred
325 $had_error = 1;
326 } else {
327 return $rv;
328 }
329 } # else undefined - was not recognised by the plugin
330 }
331
332 if ($had_error) {
333 # was recognised but couldn't be processed
334 if ($verbosity >= 2) {
335 gsprintf($outhandle, "{plugin.no_plugin_could_process}\n", $file);
336 }
337 # tell the GLI that it was not processed
338 print STDERR "<NonProcessedFile n='$glifile'>\n" if $gli;
339
340 gsprintf($failhandle, "$file: {plugin.no_plugin_could_process_this_file}\n");
341 $stats->{'num_not_processed'} ++;
342 } else {
343 # was not recognised
344 if ($verbosity >= 2) {
345 gsprintf($outhandle, "{plugin.no_plugin_could_recognise}\n",$file);
346 }
347 # tell the GLI that it was not processed
348 print STDERR "<NonRecognisedFile n='$glifile'>\n" if $gli;
349
350 gsprintf($failhandle, "$file: {plugin.no_plugin_could_recognise_this_file}\n");
351 $stats->{'num_not_recognised'} ++;
352 }
353 return 0;
354}
355
356# write out some general stats that the plugins have compiled - note that
357# the buildcol.pl process doesn't currently call this process so the stats
358# are only output after import.pl -
359sub write_stats {
360 my ($pluginfo, $statshandle, $faillog, $gli) = @_;
361
362 $gli = 0 unless defined $gli;
363
364 foreach my $plugobj (@$pluginfo) {
365 $plugobj->compile_stats($stats);
366 }
367
368 my $total = $stats->{'num_processed'} + $stats->{'num_blocked'} +
369 $stats->{'num_not_processed'} + $stats->{'num_not_recognised'};
370
371 print STDERR "<ImportComplete considered='$total' processed='$stats->{'num_processed'}' blocked='$stats->{'num_blocked'}' ignored='$stats->{'num_not_recognised'}' failed='$stats->{'num_not_processed'}'>\n" if $gli;
372
373 if ($total == 1) {
374 gsprintf($statshandle, "* {plugin.one_considered}\n");
375 } else {
376 gsprintf($statshandle, "* {plugin.n_considered}\n", $total);
377 }
378 if ($stats->{'num_archives'}) {
379 if ($stats->{'num_archives'} == 1) {
380 gsprintf($statshandle, " ({plugin.including_archive})\n");
381 }
382 else {
383 gsprintf($statshandle, " ({plugin.including_archives})\n",
384 $stats->{'num_archives'});
385 }
386 }
387 if ($stats->{'num_processed'} == 1) {
388 gsprintf($statshandle, "* {plugin.one_included}\n");
389 } else {
390 gsprintf($statshandle, "* {plugin.n_included}\n", $stats->{'num_processed'});
391 }
392 if ($stats->{'num_not_recognised'}) {
393 if ($stats->{'num_not_recognised'} == 1) {
394 gsprintf($statshandle, "* {plugin.one_unrecognised}\n");
395 } else {
396 gsprintf($statshandle, "* {plugin.n_unrecognised}\n",
397 $stats->{'num_not_recognised'});
398 }
399
400 }
401 if ($stats->{'num_not_processed'}) {
402 if ($stats->{'num_not_processed'} == 1) {
403 gsprintf($statshandle, "* {plugin.one_rejected}\n");
404 } else {
405 gsprintf($statshandle, "* {plugin.n_rejected}\n",
406 $stats->{'num_not_processed'});
407 }
408 }
409 if ($stats->{'num_not_processed'} || $stats->{'num_not_recognised'}) {
410 gsprintf($statshandle, " {plugin.see_faillog}\n", $faillog);
411 }
412}
413
414sub end {
415 my ($pluginfo, $processor) = @_;
416 map { $_->end($processor); } @$pluginfo;
417}
418
419sub deinit {
420 my ($pluginfo, $processor) = @_;
421
422
423 map { $_->deinit($processor); } @$pluginfo;
424}
425
4261;
Note: See TracBrowser for help on using the repository browser.