source: gsdl/trunk/bin/script/import.pl@ 15072

Last change on this file since 15072 was 15072, checked in by kjdon, 16 years ago

added -reversesort option to import.pl. Used with -sortmeta option, to sort in reverse order
  • Property svn:executable set to *
  • Property svn:keywords set to Author Date Id Revision
File size: 19.4 KB
Line 
1#!/usr/bin/perl -w
2
3###########################################################################
4#
5# import.pl --
6# A component of the Greenstone digital library software
7# from the New Zealand Digital Library Project at the
8# University of Waikato, New Zealand.
9#
10# Copyright (C) 1999 New Zealand Digital Library Project
11#
12# This program is free software; you can redistribute it and/or modify
13# it under the terms of the GNU General Public License as published by
14# the Free Software Foundation; either version 2 of the License, or
15# (at your option) any later version.
16#
17# This program is distributed in the hope that it will be useful,
18# but WITHOUT ANY WARRANTY; without even the implied warranty of
19# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20# GNU General Public License for more details.
21#
22# You should have received a copy of the GNU General Public License
23# along with this program; if not, write to the Free Software
24# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
25#
26###########################################################################
27
28
29# This program will import a number of files into a particular collection
30
31package import;
32
33BEGIN {
34 die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
35 die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'};
36 unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
37 unshift (@INC, "$ENV{'GSDLHOME'}/perllib/cpan");
38 unshift (@INC, "$ENV{'GSDLHOME'}/perllib/plugins");
39 unshift (@INC, "$ENV{'GSDLHOME'}/perllib/plugouts");
40 unshift (@INC, "$ENV{'GSDLHOME'}/perllib/classify");
41
42 if (defined $ENV{'GSDLEXTS'}) {
43 my @extensions = split(/:/,$ENV{'GSDLEXTS'});
44 foreach my $e (@extensions) {
45 my $ext_prefix = "$ENV{'GSDLHOME'}/ext/$e";
46
47 unshift (@INC, "$ext_prefix/perllib");
48 unshift (@INC, "$ext_prefix/perllib/cpan");
49 }
50 }
51}
52
53use strict;
54no strict 'refs'; # allow filehandles to be variables and vice versa
55no strict 'subs'; # allow barewords (eg STDERR) as function arguments
56
57use arcinfo;
58use colcfg;
59use plugin;
60use plugout;
61use manifest;
62use util;
63use scriptutil;
64use FileHandle;
65use gsprintf 'gsprintf';
66use printusage;
67use parse2;
68
69
70
71my $oidtype_list =
72 [ { 'name' => "hash",
73 'desc' => "{import.OIDtype.hash}" },
74 { 'name' => "assigned",
75 'desc' => "{import.OIDtype.assigned}" },
76 { 'name' => "incremental",
77 'desc' => "{import.OIDtype.incremental}" },
78 { 'name' => "dirname",
79 'desc' => "{import.OIDtype.dirname}" } ];
80
81
82# used to control output file format
83my $saveas_list =
84 [ { 'name' => "DSpace",
85 'desc' => "{export.saveas.DSpace}" },
86 { 'name' => "GreenstoneMETS",
87 'desc' => "{export.saveas.GreenstoneMETS}"},
88 { 'name' => "FedoraMETS",
89 'desc' => "{export.saveas.FedoraMETS}"},
90 { 'name' => "GA",
91 'desc' => "{export.saveas.GA}"},
92 { 'name' => "MARCXML",
93 'desc' => "{export.saveas.MARCXML}"}
94 ];
95
96
97# Possible attributes for each argument
98# name: The name of the argument
99# desc: A description (or more likely a reference to a description) for this argument
100# type: The type of control used to represent the argument. Options include: string, int, flag, regexp, metadata, language, enum etc
101# reqd: Is this argument required?
102# hiddengli: Is this argument hidden in GLI?
103# modegli: The lowest detail mode this argument is visible at in GLI
104
105my $saveas_argument
106 = { 'name' => "saveas",
107 'desc' => "{import.saveas}",
108 'type' => "enum",
109 'list' => $saveas_list,
110 'deft' => "GA",
111 'reqd' => "no",
112 'modegli' => "3" };
113
114
115my $arguments =
116 [
117 $saveas_argument,
118 { 'name' => "archivedir",
119 'desc' => "{import.archivedir}",
120 'type' => "string",
121 'reqd' => "no",
122 'hiddengli' => "yes" },
123 { 'name' => "importdir",
124 'desc' => "{import.importdir}",
125 'type' => "string",
126 'reqd' => "no",
127 'hiddengli' => "yes" },
128 { 'name' => "collectdir",
129 'desc' => "{import.collectdir}",
130 'type' => "string",
131 # parsearg left "" as default
132 #'deft' => &util::filename_cat ($ENV{'GSDLHOME'}, "collect"),
133 'deft' => "",
134 'reqd' => "no",
135 'hiddengli' => "yes" },
136 { 'name' => "site",
137 'desc' => "{import.site}",
138 'type' => "string",
139 'deft' => "",
140 'reqd' => "no",
141 'hiddengli' => "yes" },
142 { 'name' => "manifest",
143 'desc' => "{import.manifest}",
144 'type' => "string",
145 'deft' => "",
146 'reqd' => "no",
147 'hiddengli' => "yes" },
148 { 'name' => "debug",
149 'desc' => "{import.debug}",
150 'type' => "flag",
151 'reqd' => "no",
152 'hiddengli' => "yes" },
153 { 'name' => "faillog",
154 'desc' => "{import.faillog}",
155 'type' => "string",
156 # parsearg left "" as default
157 #'deft' => &util::filename_cat("<collectdir>", "colname", "etc", "fail.log"),
158 'deft' => "",
159 'reqd' => "no",
160 'modegli' => "4" },
161 { 'name' => "incremental",
162 'desc' => "{import.incremental}",
163 'type' => "flag",
164 'hiddengli' => "yes" },
165 { 'name' => "keepold",
166 'desc' => "{import.keepold}",
167 'type' => "flag",
168 'reqd' => "no",
169 'hiddengli' => "yes" },
170 { 'name' => "removeold",
171 'desc' => "{import.removeold}",
172 'type' => "flag",
173 'reqd' => "no",
174 'hiddengli' => "yes" },
175 { 'name' => "language",
176 'desc' => "{scripts.language}",
177 'type' => "string",
178 'reqd' => "no",
179 'hiddengli' => "yes" },
180 { 'name' => "maxdocs",
181 'desc' => "{import.maxdocs}",
182 'type' => "int",
183 'reqd' => "no",
184 # parsearg left "" as default
185 #'deft' => "-1",
186 'range' => "1,",
187 'modegli' => "1" },
188 { 'name' => "OIDtype",
189 'desc' => "{import.OIDtype}",
190 'type' => "enum",
191 'list' => $oidtype_list,
192 # parsearg left "" as default
193 #'deft' => "hash",
194 'reqd' => "no",
195 'modegli' => "2" },
196 { 'name' => "OIDmetadata",
197 'desc' => "{import.OIDmetadata}",
198 'type' => "metadata",
199 'deft' => "dc.Identifier",
200 'reqd' => "no",
201 'modegli' => "2" },
202 { 'name' => "out",
203 'desc' => "{import.out}",
204 'type' => "string",
205 'deft' => "STDERR",
206 'reqd' => "no",
207 'hiddengli' => "yes" },
208 { 'name' => "sortmeta",
209 'desc' => "{import.sortmeta}",
210 'type' => "metadata",
211 'reqd' => "no",
212 'modegli' => "3" },
213 { 'name' => "reversesort",
214 'desc' => "{import.reversesort}",
215 'type' => "flag",
216 'reqd' => "no",
217 'modegli' => "3" },
218 { 'name' => "removeprefix",
219 'desc' => "{BasClas.removeprefix}",
220 'type' => "regexp",
221 'deft' => "",
222 'reqd' => "no",
223 'modegli' => "3" },
224 { 'name' => "removesuffix",
225 'desc' => "{BasClas.removesuffix}",
226 'type' => "regexp",
227 'deft' => "",
228 'reqd' => "no",
229 'modegli' => "3" },
230 { 'name' => "groupsize",
231 'desc' => "{import.groupsize}",
232 'type' => "int",
233 'deft' => "1",
234 'reqd' => "no",
235 'modegli' => "3" },
236 { 'name' => "gzip",
237 'desc' => "{import.gzip}",
238 'type' => "flag",
239 'reqd' => "no",
240 'modegli' => "4" },
241 { 'name' => "statsfile",
242 'desc' => "{import.statsfile}",
243 'type' => "string",
244 'deft' => "STDERR",
245 'reqd' => "no",
246 'hiddengli' => "yes" },
247 { 'name' => "verbosity",
248 'desc' => "{import.verbosity}",
249 'type' => "int",
250 'range' => "0,",
251 # parsearg left "" as default
252 #'deft' => "2",
253 'reqd' => "no",
254 'modegli' => "4" },
255 { 'name' => "gli",
256 'desc' => "",
257 'type' => "flag",
258 'reqd' => "no",
259 'hiddengli' => "yes" },
260 { 'name' => "xml",
261 'desc' => "{scripts.xml}",
262 'type' => "flag",
263 'reqd' => "no",
264 'hiddengli' => "yes" }];
265
266my $options = { 'name' => "import.pl",
267 'desc' => "{import.desc}",
268 'args' => $arguments };
269
270
271&main();
272
273sub main {
274 # params
275 my ($language, $verbosity, $debug,
276 $importdir, $site, $manifest, $incremental, $keepold,
277 $removeold, $saveas,
278 $OIDtype, $OIDmetadata,
279 $maxdocs, $statsfile,
280 $out, $faillog, $collectdir, $gli,
281
282 $archivedir,
283 $gzip, $groupsize,
284 $sortmeta, $reversesort, $removeprefix, $removesuffix,
285 );
286
287 my $xml = 0;
288
289 # other vars
290 my ($configfilename, $collectcfg, $collection,
291 $archive_info_filename, $archive_info,
292 $gs_mode,
293 $processor, $pluginfo);
294
295 my $service = "import";
296
297 my $hashParsingResult = {};
298 # general options available to all plugins
299 my $intArgLeftinAfterParsing = parse2::parse(\@ARGV,$arguments,$hashParsingResult,"allow_extra_options");
300 # Parse returns -1 if something has gone wrong
301 if ($intArgLeftinAfterParsing == -1)
302 {
303 &PrintUsage::print_txt_usage($options, "{import.params}");
304 die "\n";
305 }
306
307 foreach my $strVariable (keys %$hashParsingResult)
308 {
309 eval "\$$strVariable = \$hashParsingResult->{\"\$strVariable\"}";
310 }
311
312 # If $language has been specified, load the appropriate resource bundle
313 # (Otherwise, the default resource bundle will be loaded automatically)
314 if ($language && $language =~ /\S/) {
315 &gsprintf::load_language_specific_resource_bundle($language);
316 }
317
318 if ($xml) {
319 &PrintUsage::print_xml_usage($options);
320 print "\n";
321 return;
322 }
323
324 if ($gli) { # the gli wants strings to be in UTF-8
325 &gsprintf::output_strings_in_UTF8;
326 }
327
328 # now check that we had exactly one leftover arg, which should be
329 # the collection name. We don't want to do this earlier, cos
330 # -xml arg doesn't need a collection name
331 # Or if the user specified -h, then we output the usage also
332 if ($intArgLeftinAfterParsing != 1 || (@ARGV && $ARGV[0] =~ /^\-+h/))
333 {
334 &PrintUsage::print_txt_usage($options, "{buildcol.params}");
335 die "\n";
336 }
337
338 my $close_out = 0;
339 if ($out !~ /^(STDERR|STDOUT)$/i) {
340 open (OUT, ">$out") ||
341 (&gsprintf(STDERR, "{common.cannot_open_output_file}: $!\n", $out) && die);
342 $out = 'import::OUT';
343 $close_out = 1;
344 }
345 $out->autoflush(1);
346
347 # get and check the collection name
348 if (($collection = &colcfg::use_collection($site, @ARGV, $collectdir)) eq "") {
349 &PrintUsage::print_txt_usage($options, "{import.params}");
350 die "\n";
351 }
352
353 # add collection's perllib dir into include path in
354 # case we have collection specific modules
355 unshift (@INC, "$ENV{'GSDLCOLLECTDIR'}/perllib");
356
357 # check that we can open the faillog
358 if ($faillog eq "") {
359 $faillog = &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, "etc", "fail.log");
360 }
361 open (FAILLOG, ">$faillog") ||
362 (&gsprintf(STDERR, "{import.cannot_open_fail_log}\n", $faillog) && die);
363
364
365 my $faillogname = $faillog;
366 $faillog = 'import::FAILLOG';
367 $faillog->autoflush(1);
368
369 # Read in the collection configuration file.
370 ($configfilename, $gs_mode) = &colcfg::get_collect_cfg_name($out);
371
372 if ($gs_mode eq "gs2") {
373 $collectcfg = &colcfg::read_collect_cfg ($configfilename);
374 } elsif ($gs_mode eq "gs3") {
375 $collectcfg = &colcfg::read_collection_cfg_xml ($configfilename);
376 }
377
378 if (defined $collectcfg->{'importdir'} && $importdir eq "") {
379 $importdir = $collectcfg->{'importdir'};
380 }
381 if (defined $collectcfg->{'archivedir'} && $archivedir eq "") {
382 $archivedir = $collectcfg->{'archivedir'};
383 }
384 # fill in the default import and archives directories if none
385 # were supplied, turn all \ into / and remove trailing /
386 $importdir = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, "import") if $importdir eq "";
387 $importdir =~ s/[\\\/]+/\//g;
388 $importdir =~ s/\/$//;
389 if (!-e $importdir) {
390 &gsprintf($out, "{import.no_import_dir}\n\n", $importdir);
391 die "\n";
392 }
393
394 $archivedir = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, "archives") if $archivedir eq "";
395 $archivedir =~ s/[\\\/]+/\//g;
396 $archivedir =~ s/\/$//;
397
398 my $plugins = [];
399 if (defined $collectcfg->{'plugin'}) {
400 $plugins = $collectcfg->{'plugin'};
401 }
402 #some global options for the plugins
403 my @global_opts = ();
404
405 if ($verbosity !~ /\d+/) {
406 if (defined $collectcfg->{'verbosity'} && $collectcfg->{'verbosity'} =~ /\d+/) {
407 $verbosity = $collectcfg->{'verbosity'};
408 } else {
409 $verbosity = 2; # the default
410 }
411 }
412 if (defined $collectcfg->{'manifest'} && $manifest eq "") {
413 $manifest = $collectcfg->{'manifest'};
414 }
415
416 if (defined $collectcfg->{'gzip'} && !$gzip) {
417 if ($collectcfg->{'gzip'} =~ /^true$/i) {
418 $gzip = 1;
419 }
420 }
421
422 if ($maxdocs !~ /\-?\d+/) {
423 if (defined $collectcfg->{'maxdocs'} && $collectcfg->{'maxdocs'} =~ /\-?\d+/) {
424 $maxdocs = $collectcfg->{'maxdocs'};
425 } else {
426 $maxdocs = -1; # the default
427 }
428 }
429 if ($groupsize == 1) {
430 if (defined $collectcfg->{'groupsize'} && $collectcfg->{'groupsize'} =~ /\d+/) {
431 $groupsize = $collectcfg->{'groupsize'};
432 }
433 }
434
435 if ($OIDtype !~ /^(hash|incremental|assigned|dirname)$/) {
436 if (defined $collectcfg->{'OIDtype'} && $collectcfg->{'OIDtype'} =~ /^(hash|incremental|assigned|dirname)$/) {
437 $OIDtype = $collectcfg->{'OIDtype'};
438 } else {
439 $OIDtype = "hash"; # the default
440 }
441 }
442
443 my ($plugout);
444 if (defined $collectcfg->{'plugout'}) {
445 $plugout = $collectcfg->{'plugout'};
446 }
447 else{
448 if ($saveas !~ /^(GA|.*METS|DSpace|MARCXML)$/) {
449 push @$plugout,"GAPlugout";
450 }
451 else{
452 push @$plugout,$saveas."Plugout";
453 }
454 }
455
456 if (defined $collectcfg->{'sortmeta'} && (!defined $sortmeta || $sortmeta eq "")) {
457 $sortmeta = $collectcfg->{'sortmeta'};
458 }
459 # sortmeta cannot be used with group size
460 $sortmeta = undef unless defined $sortmeta && $sortmeta =~ /\S/;
461 if (defined $sortmeta && $groupsize > 1) {
462 &gsprintf($out, "{import.cannot_sort}\n\n");
463 $sortmeta = undef;
464 }
465
466 if (defined $sortmeta) {
467 if (defined $collectcfg->{'reversesort'} && $collectcfg->{'reversesort'} =~ /^true$/i) {
468 $reversesort = 1;
469 }
470 } else {
471 # reversesort only valid with sortmeta
472 $reversesort = 0;
473 }
474 if (defined $collectcfg->{'removeprefix'} && $removeprefix eq "") {
475 $removeprefix = $collectcfg->{'removeprefix'};
476 }
477
478 if (defined $collectcfg->{'removesuffix'} && $removesuffix eq "") {
479 $removesuffix = $collectcfg->{'removesuffix'};
480 }
481 if (defined $collectcfg->{'debug'} && $collectcfg->{'debug'} =~ /^true$/i) {
482 $debug = 1;
483 }
484 if (defined $collectcfg->{'gli'} && $collectcfg->{'gli'} =~ /^true$/i) {
485 $gli = 1;
486 }
487
488
489 # global plugin stuff
490 if (defined $collectcfg->{'separate_cjk'} && $collectcfg->{'separate_cjk'} =~ /^true$/i) {
491 push @global_opts, "-separate_cjk";
492 }
493
494 # check keepold and removeold
495 ($removeold, $keepold, $incremental) = &scriptutil::check_removeold_and_keepold($removeold, $keepold, $incremental, "archives", $collectcfg);
496
497 $gli = 0 unless defined $gli;
498
499 print STDERR "<Import>\n" if $gli;
500
501 my $manifest_lookup = new manifest();
502 if ($manifest ne "") {
503 my $manifest_filename = $manifest;
504
505 if ($manifest_filename !~ m/^[\\\/]/) {
506 $manifest_filename = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, $manifest_filename);
507 }
508
509 $manifest =~ s/[\\\/]+/\//g;
510 $manifest =~ s/\/$//;
511
512 $manifest_lookup->parse($manifest_filename);
513 }
514
515
516 # load all the plugins
517 $pluginfo = &plugin::load_plugins ($plugins, $verbosity, $out, $faillog, \@global_opts, $incremental);
518 if (scalar(@$pluginfo) == 0) {
519 &gsprintf($out, "{import.no_plugins_loaded}\n");
520 die "\n";
521 }
522
523 # remove the old contents of the archives directory (and tmp directory) if needed
524 if ($removeold) {
525 if (-e $archivedir) {
526 &gsprintf($out, "{import.removing_archives}\n");
527 &util::rm_r ($archivedir);
528 }
529 my $tmpdir = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, "tmp");
530 $tmpdir =~ s/[\\\/]+/\//g;
531 $tmpdir =~ s/\/$//;
532 if (-e $tmpdir) {
533 &gsprintf($out, "{import.removing_tmpdir}\n");
534 &util::rm_r ($tmpdir);
535 }
536 }
537 # create the archives dir if needed
538 &util::mk_all_dir($archivedir);
539
540 # read the archive information file
541
542 # If saveas=DSpace, a "contents" file will be created, otherwise "archives.inf"
543
544 # the plugouts should be doing this!!
545 if ($saveas eq "DSpace"){
546 $archive_info_filename = &util::filename_cat ($archivedir, "contents");
547 } elsif ($saveas =~ m/^.*METS$/ || $saveas eq "GA" || $saveas eq "MARC" ) {
548 $archive_info_filename = &util::filename_cat ($archivedir, "archives.inf");
549 }
550
551 $archive_info = new arcinfo ();
552 $archive_info->load_info ($archive_info_filename);
553 if ($reversesort) {
554 $archive_info->reverse_sort();
555 }
556
557 ####Use Plugout####
558 push @$plugout,("-output_info",$archive_info) if (defined $archive_info);
559 push @$plugout,("-verbosity",$verbosity) if (defined $verbosity);
560 push @$plugout,("-gzip_output") if ($gzip);
561 push @$plugout,("-group_size",$groupsize) if (defined $groupsize);
562 push @$plugout,("-output_handle",$out) if (defined $out);
563 push @$plugout,("-debug") if ($debug);
564
565 $processor = &plugout::load_plugout($plugout);
566 $processor->setoutputdir ($archivedir);
567 $processor->set_sortmeta ($sortmeta, $removeprefix, $removesuffix) if defined $sortmeta;
568 $processor->set_OIDtype ($OIDtype, $OIDmetadata);
569
570 &plugin::begin($pluginfo, $importdir, $processor, $maxdocs, $gli);
571
572 if ($manifest eq "") {
573 # process the import directory
574 &plugin::read ($pluginfo, $importdir, "", {}, $processor, $maxdocs, 0, $gli);
575 }
576 else {
577
578 # process any new files
579 foreach my $file (keys %{$manifest_lookup->{'index'}}) {
580 &plugin::read ($pluginfo, $importdir, $file, {}, $processor, $maxdocs, 0, $gli);
581 }
582
583 # record files marked for deletion in arcinfo
584 foreach my $file (keys %{$manifest_lookup->{'delete'}}) {
585 # consider finding it?
586 # $archive_info->add_info($OID,$doc_xml_file,"D");
587 }
588 }
589
590 if ($saveas eq "FedoraMETS")
591 {
592 # This would be better (should?!) be done in Plugout!!
593
594 # create collection "doc obj" for Fedora that contains
595 # collection-level metadata
596
597 my $doc_obj = new doc($configfilename,"nonindexed_doc");
598 $doc_obj->set_OID("collection");
599
600 my $col_name = undef;
601 my $col_meta = $collectcfg->{'collectionmeta'};
602 if (defined $col_meta)
603 {
604 store_collectionmeta($col_meta,"collectionname",$doc_obj);
605 store_collectionmeta($col_meta,"collectioextra",$doc_obj);
606
607 }
608
609 $processor->process($doc_obj);
610 }
611
612
613 &plugin::end($pluginfo, $processor);
614
615 &plugin::deinit($pluginfo, $processor);
616
617 # write out the archive information file
618 $processor->close_file_output() if $groupsize > 1;
619 $processor->close_group_output() if $processor->is_group();
620
621# The following 'if' statement is in the export.pl version of the script,
622# but not (so far) the import.pl version. Why is this?
623## if ($saveas =~ m/^.*METS$/) {
624 # should we still do this in debug mode??
625
626 $archive_info->save_info($archive_info_filename);
627## }
628
629 # write out import stats
630 my $close_stats = 0;
631 if ($statsfile !~ /^(STDERR|STDOUT)$/i) {
632 if (open (STATS, ">$statsfile")) {
633 $statsfile = 'import::STATS';
634 $close_stats = 1;
635 } else {
636 &gsprintf($out, "{import.cannot_open_stats_file}", $statsfile);
637 &gsprintf($out, "{import.stats_backup}\n");
638 $statsfile = 'STDERR';
639 }
640 }
641
642 &gsprintf($out, "\n");
643 &gsprintf($out, "*********************************************\n");
644 &gsprintf($out, "{import.complete}\n");
645 &gsprintf($out, "*********************************************\n");
646
647 &plugin::write_stats($pluginfo, $statsfile, $faillogname, $gli);
648 if ($close_stats) {
649 close STATS;
650 }
651
652 close OUT if $close_out;
653 close FAILLOG;
654}
Note: See TracBrowser for help on using the repository browser.