source: main/trunk/greenstone2/perllib/plugin.pm@ 21314

Last change on this file since 21314 was 21314, checked in by kjdon, 11 years ago

check return values for remove_one, and stop the pipeline once found something positive

  • Property svn:keywords set to Author Date Id Revision
File size: 13.6 KB
Line 
1###########################################################################
2#
3# plugin.pm -- functions to handle using plugins
4# A component of the Greenstone digital library software
5# from the New Zealand Digital Library Project at the
6# University of Waikato, New Zealand.
7#
8# Copyright (C) 1999 New Zealand Digital Library Project
9#
10# This program is free software; you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation; either version 2 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program; if not, write to the Free Software
22# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
23#
24###########################################################################
25
26package plugin;
27
28use inexport;
29
30use strict; # to pick up typos and undeclared variables...
31no strict 'refs'; # ...but allow filehandles to be variables and vice versa
32no strict 'subs';
33
34require util;
35use gsprintf 'gsprintf';
36
37# mapping from old plugin names to new ones for backwards compatibility
38# can remove at sometime in future when we no longer want to support old xxPlug names in the config file
39my $plugin_name_map = {
40 'GAPlug' => 'GreenstoneXMLPlugin',
41 'ArcPlug' => 'ArchivesInfPlugin',
42 'RecPlug' => 'DirectoryPlugin',
43 'TEXTPlug' => 'TextPlugin',
44 'XMLPlug' => 'ReadXMLFile',
45 'EMAILPlug' => 'EmailPlugin',
46 'SRCPlug' => 'SourceCodePlugin',
47 'NULPlug' => 'NulPlugin',
48 'W3ImgPlug' => 'HTMLImagePlugin',
49 'PagedImgPlug' => 'PagedImagePlugin',
50 'METSPlug' => 'GreenstoneMETSPlugin',
51 'PPTPlug' => 'PowerPointPlugin',
52 'PSPlug' => 'PostScriptPlugin',
53 'DBPlug' => 'DatabasePlugin'
54 };
55
56# global variables
57my $stats = {'num_processed' => 0,
58 'num_blocked' => 0,
59 'num_not_processed' => 0,
60 'num_not_recognised' => 0,
61 'num_archives' => 0
62 };
63
64#globaloptions contains any options that should be passed to all plugins
65my ($verbosity, $outhandle, $failhandle, $globaloptions);
66
67sub get_valid_pluginname {
68 my ($pluginname) = @_;
69 my $valid_name = $pluginname;
70 if (defined $plugin_name_map->{$pluginname}) {
71 $valid_name = $plugin_name_map->{$pluginname};
72 } elsif ($pluginname =~ /Plug$/) {
73 $valid_name =~ s/Plug/Plugin/;
74
75 }
76 return $valid_name;
77}
78
79sub load_plugin_require
80{
81 my ($pluginname) = @_;
82
83 my @check_list = ();
84
85 # pp_plugname shorthand for 'perllib' 'plugin' '$pluginname.pm'
86 my $pp_plugname
87 = &util::filename_cat('perllib', 'plugins', "${pluginname}.pm");
88 my $collectdir = $ENV{'GSDLCOLLECTDIR'};
89
90 # find the plugin
91 if (defined($ENV{'GSDLCOLLECTION'}))
92 {
93 my $customplugname
94 = &util::filename_cat($collectdir, "custom",$ENV{'GSDLCOLLECTION'},
95 $pp_plugname);
96 push(@check_list,$customplugname);
97 }
98
99 my $colplugname = &util::filename_cat($collectdir, $pp_plugname);
100 push(@check_list,$colplugname);
101
102 if (defined $ENV{'GSDLEXTS'}) {
103
104 my $ext_prefix = &util::filename_cat($ENV{'GSDLHOME'}, "ext");
105
106 my @extensions = split(/:/,$ENV{'GSDLEXTS'});
107 foreach my $e (@extensions) {
108 my $extplugname = &util::filename_cat($ext_prefix, $e, $pp_plugname);
109 push(@check_list,$extplugname);
110
111 }
112 }
113 if (defined $ENV{'GSDL3EXTS'}) {
114
115 my $ext_prefix = &util::filename_cat($ENV{'GSDL3SRCHOME'}, "ext");
116
117 my @extensions = split(/:/,$ENV{'GSDL3EXTS'});
118 foreach my $e (@extensions) {
119 my $extplugname = &util::filename_cat($ext_prefix, $e, $pp_plugname);
120 push(@check_list,$extplugname);
121
122 }
123 }
124
125
126 my $mainplugname = &util::filename_cat($ENV{'GSDLHOME'}, $pp_plugname);
127 push(@check_list,$mainplugname);
128
129 my $success=0;
130 foreach my $plugname (@check_list) {
131 if (-e $plugname) {
132 require $plugname;
133 $success=1;
134 last;
135 }
136 }
137
138 if (!$success) {
139 &gsprintf(STDERR, "{plugin.could_not_find_plugin}\n",
140 $pluginname);
141 die "\n";
142 }
143}
144
145sub load_plugin_for_info {
146 my ($pluginname) = shift @_;
147 $pluginname = &get_valid_pluginname($pluginname);
148 load_plugin_require($pluginname);
149
150 # create a plugin object
151 my ($plugobj);
152 my $options = "-gsdlinfo";
153
154 eval ("\$plugobj = new \$pluginname([],[$options])");
155 die "$@" if $@;
156
157 return $plugobj;
158}
159
160sub load_plugins {
161 my ($plugin_list) = shift @_;
162 my $incremental_mode;
163 ($verbosity, $outhandle, $failhandle, $globaloptions, $incremental_mode) = @_; # globals
164 my @plugin_objects = ();
165 $verbosity = 2 unless defined $verbosity;
166 $outhandle = 'STDERR' unless defined $outhandle;
167 $failhandle = 'STDERR' unless defined $failhandle;
168
169 my $colplugindir = &util::filename_cat($ENV{'GSDLCOLLECTDIR'},"perllib/plugins");
170 unshift (@INC, $colplugindir);
171
172 map { $_ = "\"$_\""; } @$globaloptions;
173 my $globals = join (",", @$globaloptions);
174
175 foreach my $pluginoptions (@$plugin_list) {
176 my $pluginname = shift @$pluginoptions;
177 next unless defined $pluginname;
178 $pluginname = &get_valid_pluginname($pluginname);
179 load_plugin_require($pluginname);
180
181 # create a plugin object
182 my ($plugobj);
183 map { $_ = "\"$_\""; } @$pluginoptions;
184 my $options = join (",", @$pluginoptions);
185 if ($globals) {
186 if (@$pluginoptions) {
187 $options .= ",";
188 }
189 $options .= "$globals";
190 }
191 # need to escape backslash before putting in to the eval
192 $options =~ s/\\/\\\\/g;
193 $options =~ s/\$/\\\$/g;
194 eval ("\$plugobj = new \$pluginname([],[$options])");
195 die "$@" if $@;
196
197 # initialize plugin
198 $plugobj->init($verbosity, $outhandle, $failhandle);
199
200 $plugobj->set_incremental($incremental_mode);
201
202 # add this object to the list
203 push (@plugin_objects, $plugobj);
204 }
205
206 return \@plugin_objects;
207}
208
209
210sub begin {
211 my ($pluginfo, $base_dir, $processor, $maxdocs, $gli) = @_;
212
213 map { $_->{'gli'} = $gli; } @$pluginfo;
214 map { $_->begin($pluginfo, $base_dir, $processor, $maxdocs); } @$pluginfo;
215}
216
217 sub remove_all {
218 my ($pluginfo, $base_dir, $processor, $maxdocs) = @_;
219
220 map { $_->remove_all($pluginfo, $base_dir, $processor, $maxdocs); } @$pluginfo;
221}
222
223sub remove_some {
224 my ($pluginfo, $archivedir, $deleted_files) = @_;
225 print STDERR "in remove some\n";
226 return if (scalar(@$deleted_files)==0);
227 my $arcinfo_src_filename = &inexport::src_db_file($archivedir);
228
229 foreach my $file (@$deleted_files) {
230 # use 'archiveinf-src' GDBM file to look up all the OIDs
231 # that this file is used in (note in most cases, it's just one OID)
232
233 my $src_rec = GDBMUtils::gdbmRecordToHash($arcinfo_src_filename,$file);
234 my $oids = $src_rec->{'oid'};
235 my $rv;
236 foreach my $plugobj (@$pluginfo) {
237
238 $rv = $plugobj->remove_one($file, $oids, $archivedir);
239 if (defined $rv && $rv != -1) {
240 return $rv;
241 } # else undefined (was not recognised by the plugin) or there was an error, try the next one
242 }
243 return 0;
244 }
245
246}
247sub file_block_read {
248 my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $gli) = @_;
249
250
251 $gli = 0 unless defined $gli;
252
253 my $rv = 0;
254 my $glifile = $file;
255
256 $glifile =~ s/^[\/\\]+//; # file sometimes starts with a / so get rid of it
257
258 # Announce to GLI that we are handling a file
259 print STDERR "<File n='$glifile'>\n" if $gli;
260
261 # the .kill file is a handy (if not very elegant) way of aborting
262 # an import.pl or buildcol.pl process
263 if (-e &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, ".kill")) {
264 gsprintf($outhandle, "{plugin.kill_file}\n");
265 die "\n";
266 }
267
268 foreach my $plugobj (@$pluginfo) {
269
270 $rv = $plugobj->file_block_read($pluginfo, $base_dir, $file, $block_hash, $metadata, $gli);
271 #last if (defined $rv && $rv==1); # stop this file once we have found something to 'process' it
272 }
273
274}
275
276
277sub metadata_read {
278 my ($pluginfo, $base_dir, $file, $block_hash,
279 $extrametakeys, $extrametadata, $extrametafile,
280 $processor, $maxdocs, $gli, $aux) = @_;
281
282 $maxdocs = -1 unless defined $maxdocs && $maxdocs =~ /\d/;
283 $gli = 0 unless defined $gli;
284
285 my $rv = 0;
286 my $glifile = $file;
287
288 $glifile =~ s/^[\/\\]+//; # file sometimes starts with a / so get rid of it
289
290 # Announce to GLI that we are handling a file
291 print STDERR "<File n='$glifile'>\n" if $gli;
292
293 # the .kill file is a handy (if not very elegant) way of aborting
294 # an import.pl or buildcol.pl process
295 if (-e &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, ".kill")) {
296 gsprintf($outhandle, "{plugin.kill_file}\n");
297 die "\n";
298 }
299
300 my $had_error = 0;
301 # pass this file by each of the plugins in turn until one
302 # is found which will process it
303 # read must return:
304 # undef - could not recognise
305 # -1 - tried but error
306 # 0 - blocked
307 # anything else for successful processing
308
309 foreach my $plugobj (@$pluginfo) {
310
311 $rv = $plugobj->metadata_read($pluginfo, $base_dir, $file, $block_hash,
312 $extrametakeys, $extrametadata, $extrametafile,
313 $processor, $maxdocs, $gli, $aux);
314
315 if (defined $rv) {
316 if ($rv == -1) {
317 # an error has occurred
318 $had_error = 1;
319 print STDERR "<ProcessingError n='$glifile'>\n" if $gli;
320 } else {
321 return $rv;
322 }
323 } # else undefined - was not recognised by the plugin
324 }
325
326 return 0;
327}
328
329sub read {
330 my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $processor, $maxdocs, $total_count, $gli, $aux) = @_;
331
332 $maxdocs = -1 unless defined $maxdocs && $maxdocs =~ /\d/;
333 $total_count = 0 unless defined $total_count && $total_count =~ /\d/;
334 $gli = 0 unless defined $gli;
335
336 my $rv = 0;
337 my $glifile = $file;
338
339 $glifile =~ s/^[\/\\]+//; # file sometimes starts with a / so get rid of it
340
341 # Announce to GLI that we are handling a file
342 print STDERR "<File n='$glifile'>\n" if $gli;
343
344 # the .kill file is a handy (if not very elegant) way of aborting
345 # an import.pl or buildcol.pl process
346 if (-e &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, ".kill")) {
347 gsprintf($outhandle, "{plugin.kill_file}\n");
348 die "\n";
349 }
350
351 my $had_error = 0;
352 # pass this file by each of the plugins in turn until one
353 # is found which will process it
354 # read must return:
355 # undef - could not recognise
356 # -1 - tried but error
357 # 0 - blocked
358 # anything else for successful processing
359
360 foreach my $plugobj (@$pluginfo) {
361
362 $rv = $plugobj->read($pluginfo, $base_dir, $file,
363 $block_hash, $metadata, $processor, $maxdocs,
364 $total_count, $gli, $aux);
365
366 if (defined $rv) {
367 if ($rv == -1) {
368 # an error has occurred
369 $had_error = 1;
370 } else {
371 return $rv;
372 }
373 } # else undefined - was not recognised by the plugin
374 }
375
376 if ($had_error) {
377 # was recognised but couldn't be processed
378 if ($verbosity >= 2) {
379 gsprintf($outhandle, "{plugin.no_plugin_could_process}\n", $file);
380 }
381 # tell the GLI that it was not processed
382 print STDERR "<NonProcessedFile n='$glifile'>\n" if $gli;
383
384 gsprintf($failhandle, "$file: {plugin.no_plugin_could_process_this_file}\n");
385 $stats->{'num_not_processed'} ++;
386 } else {
387 # was not recognised
388 if ($verbosity >= 2) {
389 gsprintf($outhandle, "{plugin.no_plugin_could_recognise}\n",$file);
390 }
391 # tell the GLI that it was not processed
392 print STDERR "<NonRecognisedFile n='$glifile'>\n" if $gli;
393
394 gsprintf($failhandle, "$file: {plugin.no_plugin_could_recognise_this_file}\n");
395 $stats->{'num_not_recognised'} ++;
396 }
397 return 0;
398}
399
400# write out some general stats that the plugins have compiled - note that
401# the buildcol.pl process doesn't currently call this process so the stats
402# are only output after import.pl -
403sub write_stats {
404 my ($pluginfo, $statshandle, $faillog, $gli) = @_;
405
406 $gli = 0 unless defined $gli;
407
408 foreach my $plugobj (@$pluginfo) {
409 $plugobj->compile_stats($stats);
410 }
411
412 my $total = $stats->{'num_processed'} + $stats->{'num_blocked'} +
413 $stats->{'num_not_processed'} + $stats->{'num_not_recognised'};
414
415 print STDERR "<ImportComplete considered='$total' processed='$stats->{'num_processed'}' blocked='$stats->{'num_blocked'}' ignored='$stats->{'num_not_recognised'}' failed='$stats->{'num_not_processed'}'>\n" if $gli;
416
417 if ($total == 1) {
418 gsprintf($statshandle, "* {plugin.one_considered}\n");
419 } else {
420 gsprintf($statshandle, "* {plugin.n_considered}\n", $total);
421 }
422 if ($stats->{'num_archives'}) {
423 if ($stats->{'num_archives'} == 1) {
424 gsprintf($statshandle, " ({plugin.including_archive})\n");
425 }
426 else {
427 gsprintf($statshandle, " ({plugin.including_archives})\n",
428 $stats->{'num_archives'});
429 }
430 }
431 if ($stats->{'num_processed'} == 1) {
432 gsprintf($statshandle, "* {plugin.one_included}\n");
433 } else {
434 gsprintf($statshandle, "* {plugin.n_included}\n", $stats->{'num_processed'});
435 }
436 if ($stats->{'num_not_recognised'}) {
437 if ($stats->{'num_not_recognised'} == 1) {
438 gsprintf($statshandle, "* {plugin.one_unrecognised}\n");
439 } else {
440 gsprintf($statshandle, "* {plugin.n_unrecognised}\n",
441 $stats->{'num_not_recognised'});
442 }
443
444 }
445 if ($stats->{'num_not_processed'}) {
446 if ($stats->{'num_not_processed'} == 1) {
447 gsprintf($statshandle, "* {plugin.one_rejected}\n");
448 } else {
449 gsprintf($statshandle, "* {plugin.n_rejected}\n",
450 $stats->{'num_not_processed'});
451 }
452 }
453 if ($stats->{'num_not_processed'} || $stats->{'num_not_recognised'}) {
454 gsprintf($statshandle, " {plugin.see_faillog}\n", $faillog);
455 }
456}
457
458sub end {
459 my ($pluginfo, $processor) = @_;
460 map { $_->end($processor); } @$pluginfo;
461}
462
463sub deinit {
464 my ($pluginfo, $processor) = @_;
465
466
467 map { $_->deinit($processor); } @$pluginfo;
468}
469
4701;
Note: See TracBrowser for help on using the repository browser.