source: main/trunk/greenstone2/bin/script/import.pl@ 32541

Last change on this file since 32541 was 32541, checked in by ak19, 5 years ago

Using proper parameters to GreenstoneSQLPlugin/Plugout instead of hardcoded values for params.
  • Property svn:executable set to *
  • Property svn:keywords set to Author Date Id Revision
File size: 12.0 KB
Line 
1#!/usr/bin/perl -w
2
3###########################################################################
4#
5# import.pl --
6# A component of the Greenstone digital library software
7# from the New Zealand Digital Library Project at the
8# University of Waikato, New Zealand.
9#
10# Copyright (C) 1999 New Zealand Digital Library Project
11#
12# This program is free software; you can redistribute it and/or modify
13# it under the terms of the GNU General Public License as published by
14# the Free Software Foundation; either version 2 of the License, or
15# (at your option) any later version.
16#
17# This program is distributed in the hope that it will be useful,
18# but WITHOUT ANY WARRANTY; without even the implied warranty of
19# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20# GNU General Public License for more details.
21#
22# You should have received a copy of the GNU General Public License
23# along with this program; if not, write to the Free Software
24# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
25#
26###########################################################################
27
28
29# This program will import a number of files into a particular collection
30
31package import;
32
33BEGIN {
34 die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
35 die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'};
36 unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
37 unshift (@INC, "$ENV{'GSDLHOME'}/perllib/cpan");
38 unshift (@INC, "$ENV{'GSDLHOME'}/perllib/plugins");
39 unshift (@INC, "$ENV{'GSDLHOME'}/perllib/plugouts");
40
41 if (defined $ENV{'GSDLEXTS'}) {
42 my @extensions = split(/:/,$ENV{'GSDLEXTS'});
43 foreach my $e (@extensions) {
44 my $ext_prefix = "$ENV{'GSDLHOME'}/ext/$e";
45
46 unshift (@INC, "$ext_prefix/perllib");
47 unshift (@INC, "$ext_prefix/perllib/cpan");
48 unshift (@INC, "$ext_prefix/perllib/plugins");
49 unshift (@INC, "$ext_prefix/perllib/plugouts");
50 }
51 }
52 if (defined $ENV{'GSDL3EXTS'}) {
53 my @extensions = split(/:/,$ENV{'GSDL3EXTS'});
54 foreach my $e (@extensions) {
55 my $ext_prefix = "$ENV{'GSDL3SRCHOME'}/ext/$e";
56
57 unshift (@INC, "$ext_prefix/perllib");
58 unshift (@INC, "$ext_prefix/perllib/cpan");
59 unshift (@INC, "$ext_prefix/perllib/plugins");
60 unshift (@INC, "$ext_prefix/perllib/plugouts");
61 }
62 }
63
64 if ((defined $ENV{'DEBUG_UNICODE'}) && (defined $ENV{'DEBUG_UNICODE'})) {
65 binmode(STDERR,":utf8");
66 }
67}
68
69# Pragma
70use strict;
71no strict 'subs'; # allow barewords (eg STDERR) as function arguments
72use warnings;
73
74# Modules
75use Symbol qw<qualify>; # Needed for runtime loading of modules [jmt12]
76
77# Greenstone Modules
78use FileUtils;
79use inexport;
80use util;
81use gsprintf 'gsprintf';
82
83
84# used to control output file format
85my $saveas_list =
86 [ { 'name' => "GreenstoneXML",
87 'desc' => "{export.saveas.GreenstoneXML}"},
88 { 'name' => "GreenstoneMETS",
89 'desc' => "{export.saveas.GreenstoneMETS}"},
90 { 'name' => "GreenstoneSQL",
91 'desc' => "{export.saveas.GreenstoneSQL}"}
92 ];
93
94
95# Possible attributes for each argument
96# name: The name of the argument
97# desc: A description (or more likely a reference to a description) for this argument
98# type: The type of control used to represent the argument. Options include: string, int, flag, regexp, metadata, language, enum etc
99# reqd: Is this argument required?
100# hiddengli: Is this argument hidden in GLI?
101# modegli: The lowest detail mode this argument is visible at in GLI
102
103my $saveas_argument
104 = { 'name' => "saveas",
105 'desc' => "{import.saveas}",
106 'type' => "enum",
107 'list' => $saveas_list,
108 'deft' => "GreenstoneXML",
109 'reqd' => "no",
110 'modegli' => "3" };
111
112
113my $arguments =
114 [
115 $saveas_argument,
116 { 'name' => "saveas_options",
117 'desc' => "{import.saveas_options}",
118 'type' => "string",
119 'reqd' => "no" },
120 { 'name' => "sortmeta",
121 'desc' => "{import.sortmeta}",
122 'type' => "string",
123 #'type' => "metadata", #doesn't work properly in GLI
124 'reqd' => "no",
125 'modegli' => "2" },
126 { 'name' => "removeprefix",
127 'desc' => "{BasClas.removeprefix}",
128 'type' => "regexp",
129 'deft' => "",
130 'reqd' => "no",
131 'modegli' => "3" },
132 { 'name' => "removesuffix",
133 'desc' => "{BasClas.removesuffix}",
134 'type' => "regexp",
135 'deft' => "",
136 'reqd' => "no",
137 'modegli' => "3" },
138 { 'name' => "groupsize",
139 'desc' => "{import.groupsize}",
140 'type' => "int",
141 'deft' => "1",
142 'reqd' => "no",
143 'modegli' => "2" },
144 { 'name' => "archivedir",
145 'desc' => "{import.archivedir}",
146 'type' => "string",
147 'reqd' => "no",
148 'deft' => "archives",
149 'hiddengli' => "yes" },
150 @$inexport::directory_arguments,
151 { 'name' => "gzip",
152 'desc' => "{import.gzip}",
153 'type' => "flag",
154 'reqd' => "no",
155 'modegli' => "3" },
156 @$inexport::arguments,
157 { 'name' => "NO_IMPORT",
158 'desc' => "{import.NO_IMPORT}",
159 'type' => "flag",
160 'reqd' => "no",
161 'modegli' => "3"}
162];
163
164my $options = { 'name' => "import.pl",
165 'desc' => "{import.desc}",
166 'args' => $arguments };
167
168my $function_to_inexport_subclass_mappings = {};
169
170sub main
171{
172 # Dynamically include arguments from any subclasses of inexport we find
173 # in the extensions directory
174 if (defined $ENV{'GSDLEXTS'})
175 {
176 &_scanForSubclasses($ENV{'GSDLHOME'}, $ENV{'GSDLEXTS'});
177 }
178 if (defined $ENV{'GSDL3EXTS'})
179 {
180 &_scanForSubclasses($ENV{'GSDL3SRCHOME'}, $ENV{'GSDL3EXTS'});
181 }
182
183 # Loop through arguments, checking to see if any depend on a specific
184 # subclass of InExport. Note that we load the first subclass we encounter
185 # so only support a single 'override' ATM.
186 my $inexport_subclass;
187 foreach my $argument (@ARGV)
188 {
189 if ($argument eq "-NO_IMPORT") {
190 &gsprintf(STDERR, "{import.NO_IMPORT_set}\n\n");
191 exit 0;
192 }
193 # proper arguments start with a hyphen
194 if ($argument =~ /^-/ && defined $function_to_inexport_subclass_mappings->{$argument})
195 {
196 my $required_inexport_subclass = $function_to_inexport_subclass_mappings->{$argument};
197 if (!defined $inexport_subclass)
198 {
199 $inexport_subclass = $required_inexport_subclass;
200 }
201 # Oh noes! The user has included specific arguments from two different
202 # inexport subclasses... this isn't supported
203 elsif ($inexport_subclass ne $required_inexport_subclass)
204 {
205 print STDERR "Error! You cannot specify arguments from two different extention specific inexport modules: " . $inexport_subclass . " != " . $required_inexport_subclass . "\n";
206 exit;
207 }
208 }
209 }
210
211 my $inexport;
212 if (defined $inexport_subclass)
213 {
214 print "* Loading Overriding InExport Module: " . $inexport_subclass . "\n";
215 require $inexport_subclass . '.pm';
216 $inexport = new $inexport_subclass("import",\@ARGV,$options);
217 }
218
219 # We don't have a overridden inexport, or the above command failed somehow
220 # so load the base inexport class
221 if (!defined $inexport)
222 {
223 $inexport = new inexport("import",\@ARGV,$options);
224 }
225
226 my $collection = $inexport->get_collection();
227
228 if (defined $collection)
229 {
230 my ($config_filename,$collect_cfg) = $inexport->read_collection_cfg($collection,$options);
231 if ($collect_cfg->{'NO_IMPORT'}) {
232 &gsprintf(STDERR, "{import.NO_IMPORT_set}\n\n");
233 exit 0;
234 }
235 #$inexport->set_collection_options($collect_cfg);
236 &set_collection_options($inexport, $collect_cfg);
237
238
239 my $pluginfo = $inexport->process_files($config_filename,$collect_cfg);
240
241 $inexport->generate_statistics($pluginfo);
242 }
243
244 $inexport->deinit();
245}
246# main()
247
248# @function _scanForSubclasses()
249# @param $dir The extension directory to look within
250# @param $exts A list of the available extensions (as a colon separated string)
251# @return The number of subclasses of InExport found as an Integer
252sub _scanForSubclasses
253{
254 my ($dir, $exts) = @_;
255 my $inexport_class_count = 0;
256 my $ext_prefix = &FileUtils::filenameConcatenate($dir, "ext");
257 my @extensions = split(/:/, $exts);
258 foreach my $e (@extensions)
259 {
260 # - any subclass of InExport must be prefixed with the name of the ext
261 my $package_name = $e . 'inexport';
262 $package_name =~ s/[^a-z]//gi; # package names have limited characters
263 my $inexport_filename = $package_name . '.pm';
264 my $inexport_path = &FileUtils::filenameConcatenate($ext_prefix, $e, 'perllib', $inexport_filename);
265 # see if we have a subclass of InExport lurking in that extension folder
266 if (-f $inexport_path)
267 {
268 # - note we load the filename (with pm) unlike normal modules
269 require $inexport_filename;
270 # - make call to the newly created package
271 my $symbol = qualify('getSupportedArguments', $package_name);
272 # - strict prevents strings being used as function calls, so temporarily
273 # disable that pragma
274 no strict;
275 # - lets check that the function we are about to call actually exists
276 if ( defined &{$symbol} )
277 {
278 my $extra_arguments = &{$symbol}();
279 foreach my $argument (@{$extra_arguments})
280 {
281 # - record a mapping from each extra arguments to the inexport class
282 # that supports it. We put the hyphen on here to make comparing
283 # with command line arguments even easier
284 $function_to_inexport_subclass_mappings->{'-' . $argument->{'name'}} = $package_name;
285 # - and them add them as acceptable arguments to import.pl
286 push(@{$options->{'args'}}, $argument);
287 }
288 $inexport_class_count++;
289 }
290 else
291 {
292 print "Warning! A subclass of InExport module (named '" . $inexport_filename . "') does not implement the required getSupportedArguments() function - ignoring. Found in: " . $inexport_path . "\n";
293 }
294 }
295 }
296 return $inexport_class_count;
297}
298# _scanForInExportModules()
299
300# look up collect.cfg for import options, then all inexport version for the
301# common ones
302sub set_collection_options
303{
304
305 my ($inexport, $collectcfg) = @_;
306 my $out = $inexport->{'out'};
307
308 # check all options for default_optname - this will be set if the parsing
309 # code has just set the value based on the arg default. In this case,
310 # check in collect.cfg for the option
311
312 # groupsize can only be defined for import, not export, and actually only
313 # applies to GreenstoneXML format.
314 if (defined $inexport->{'default_groupsize'}) {
315 if (defined $collectcfg->{'groupsize'} && $collectcfg->{'groupsize'} =~ /\d+/) {
316 $inexport->{'groupsize'} = $collectcfg->{'groupsize'};
317 }
318
319 }
320 if (defined $inexport->{'default_saveas'}) {
321 if (defined $collectcfg->{'saveas'}
322 && $collectcfg->{'saveas'} =~ /^(GreenstoneXML|GreenstoneMETS)$/) {
323 $inexport->{'saveas'} = $collectcfg->{'saveas'};
324 } else {
325 $inexport->{'saveas'} = "GreenstoneXML"; # the default
326 }
327 }
328 if (!defined $inexport->{'saveas_options'} || $inexport->{'saveas_options'} eq "") {
329 if (defined $collectcfg->{'saveas_options'} ){
330 $inexport->{'saveas_options'} = $collectcfg->{'saveas_options'};
331 }
332 }
333
334 my $sortmeta = $inexport->{'sortmeta'};
335 if (defined $collectcfg->{'sortmeta'} && $sortmeta eq "") {
336 $sortmeta = $collectcfg->{'sortmeta'};
337 }
338 # sortmeta cannot be used with group size
339 $sortmeta = undef unless defined $sortmeta && $sortmeta =~ /\S/;
340 if (defined $sortmeta && $inexport->{'groupsize'} > 1) {
341 &gsprintf($out, "{import.cannot_sort}\n\n");
342 $sortmeta = undef;
343 }
344 if (defined $sortmeta) {
345 &gsprintf($out, "{import.sortmeta_paired_with_ArchivesInfPlugin}\n\n");
346 }
347 $inexport->{'sortmeta'} = $sortmeta;
348
349 if (defined $collectcfg->{'removeprefix'} && $inexport->{'removeprefix'} eq "") {
350 $inexport->{'removeprefix'} = $collectcfg->{'removeprefix'};
351 }
352
353 if (defined $collectcfg->{'removesuffix'} && $inexport->{'removesuffix'} eq "") {
354 $inexport->{'removesuffix'} = $collectcfg->{'removesuffix'};
355 }
356
357 $inexport->set_collection_options($collectcfg);
358
359}
360&main();
Note: See TracBrowser for help on using the repository browser.