Changeset 28768 for gs2-extensions/parallel-building
- Timestamp:
- 2013-12-18T10:53:15+13:00 (10 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
gs2-extensions/parallel-building/trunk/src/bin/script/parallel_import.pl
r25117 r28768 1 1 #!/usr/bin/perl -w 2 2 3 ########################################################################### 4 # 5 # import.pl -- 6 # A component of the Greenstone digital library software 7 # from the New Zealand Digital Library Project at the 8 # University of Waikato, New Zealand. 3 ############################################################################### 4 # 5 # import.pl -- A component of the Greenstone digital library software from the 6 # New Zealand Digital Library Project at the University of Waikato, New 7 # Zealand. 9 8 # 10 9 # Copyright (C) 1999 New Zealand Digital Library Project 11 10 # 12 # This program is free software; you can redistribute it and/or modify 13 # it under the terms of the GNU General Public License as published by 14 # the Free Software Foundation; either version 2 of the License, or 15 # (at your option) any later version. 16 # 17 # This program is distributed in the hope that it will be useful, 18 # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 20 # GNU General Public License for more details. 21 # 22 # You should have received a copy of the GNU General Public License 23 # along with this program; if not, write to the Free Software 24 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 25 # 26 ########################################################################### 27 11 # This program is free software; you can redistribute it and/or modify it under 12 # the terms of the GNU General Public License as published by the Free Software 13 # Foundation; either version 2 of the License, or (at your option) any later 14 # version. 15 # 16 # This program is distributed in the hope that it will be useful, but WITHOUT 17 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 18 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 19 # details. 20 # 21 # You should have received a copy of the GNU General Public License along with 22 # this program; if not, write to the Free Software Foundation, Inc., 675 Mass 23 # Ave, Cambridge, MA 02139, USA. 24 # 25 ############################################################################### 28 26 29 27 # This program will import a number of files into a particular collection … … 31 29 package parallel_import; 32 30 33 BEGIN { 34 die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'}; 35 die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'}; 36 unshift (@INC, "$ENV{'GSDLHOME'}/perllib"); 37 unshift (@INC, "$ENV{'GSDLHOME'}/perllib/cpan"); 38 unshift (@INC, "$ENV{'GSDLHOME'}/perllib/plugins"); 39 unshift (@INC, "$ENV{'GSDLHOME'}/perllib/plugouts"); 40 41 if (defined $ENV{'GSDLEXTS'}) { 42 my @extensions = split(/:/,$ENV{'GSDLEXTS'}); 43 foreach my $e (@extensions) { 44 my $ext_prefix = "$ENV{'GSDLHOME'}/ext/$e"; 45 46 unshift (@INC, "$ext_prefix/perllib"); 47 unshift (@INC, "$ext_prefix/perllib/cpan"); 48 unshift (@INC, "$ext_prefix/perllib/plugins"); 49 unshift (@INC, "$ext_prefix/perllib/plugouts"); 50 } 31 BEGIN 32 { 33 if (!defined $ENV{'GSDLHOME'}) 34 { 35 die("GSDLHOME not set\n"); 36 } 37 unshift (@INC, $ENV{'GSDLHOME'} . '/perllib'); 38 unshift (@INC, $ENV{'GSDLHOME'} . '/perllib/cpan'); 39 unshift (@INC, $ENV{'GSDLHOME'} . '/perllib/plugins'); 40 unshift (@INC, $ENV{'GSDLHOME'} . '/perllib/plugouts'); 41 42 if (defined $ENV{'GSDLEXTS'}) 43 { 44 my @extensions = split(/:/,$ENV{'GSDLEXTS'}); 45 foreach my $e (@extensions) 46 { 47 my $ext_prefix = $ENV{'GSDLHOME'} . '/ext/' . $e; 48 unshift (@INC, $ext_prefix . '/perllib'); 49 unshift (@INC, $ext_prefix . '/perllib/cpan'); 50 unshift (@INC, $ext_prefix . '/perllib/plugins'); 51 unshift (@INC, $ext_prefix . '/perllib/plugouts'); 51 52 } 52 if (defined $ENV{'GSDL3EXTS'}) { 53 my @extensions = split(/:/,$ENV{'GSDL3EXTS'}); 54 foreach my $e (@extensions) { 55 my $ext_prefix = "$ENV{'GSDL3SRCHOME'}/ext/$e"; 56 57 unshift (@INC, "$ext_prefix/perllib"); 58 unshift (@INC, "$ext_prefix/perllib/cpan"); 59 unshift (@INC, "$ext_prefix/perllib/plugins"); 60 unshift (@INC, "$ext_prefix/perllib/plugouts"); 61 } 53 } 54 # We may want to make use of GSDL3 Extensions too, such as Solr 55 if (defined $ENV{'GSDL3EXTS'}) 56 { 57 my @extensions = split(/:/,$ENV{'GSDL3EXTS'}); 58 foreach my $e (@extensions) 59 { 60 my $ext_prefix = $ENV{'GSDL3SRCHOME'} . '/ext/' . $e; 61 unshift (@INC, $ext_prefix . '/perllib'); 62 unshift (@INC, $ext_prefix . '/perllib/cpan'); 63 unshift (@INC, $ext_prefix . '/perllib/plugins'); 64 unshift (@INC, $ext_prefix . '/perllib/plugouts'); 62 65 } 66 } 63 67 } 64 68 65 69 use strict; 66 70 71 use Time::HiRes qw( gettimeofday tv_interval ); 67 72 use inexport; 68 73 … … 179 184 'modegli' => "1" }, 180 185 # don't set the default to hash - want to allow this to come from 181 # entry in collect.cfg but want to override it here 186 # entry in collect.cfg but want to override it here 182 187 { 'name' => "OIDtype", 183 188 'desc' => "{import.OIDtype}", … … 276 281 'args' => $arguments }; 277 282 278 279 280 281 282 sub main 283 ## @function main() 284 # 285 sub main 283 286 { 284 print "Parallel Import Started: " . `date` . "\n"; 285 286 my $inexport = new inexport("import",\@ARGV,$options); 287 288 my $collection = $inexport->get_collection(); 289 my ($config_filename,$collect_cfg) = $inexport->read_collection_cfg($collection,$options); 290 $inexport->set_collection_options($collect_cfg); 291 292 my $pluginfo = $inexport->process_files($config_filename,$collect_cfg); 293 294 $inexport->generate_statistics($pluginfo); 295 296 # Ensure all changes are pushed to disk 297 #`sync`; 298 299 print "Parallel Import Complete: " . `date` . "\n"; 287 my $start_time = [gettimeofday()]; 288 print "[" . @{$start_time}[0] . '.' . @{$start_time}[1] . "] Parallel Import Started\n"; 289 290 print "Warning! Script is obsolete - use: import.pl -workers <int> [-batchsize <int>] <collection>\n"; 291 292 my $inexport = new inexport("import", \@ARGV, $options); 293 294 # get the collection configuration (we'll need it to init the plugins) 295 my $collection = $inexport->get_collection(); 296 my ($config_filename,$collect_cfg) = $inexport->read_collection_cfg($collection,$options); 297 $inexport->set_collection_options($collect_cfg); 298 299 my $pluginfo = $inexport->process_files($config_filename,$collect_cfg); 300 301 $inexport->generate_statistics($pluginfo); 302 303 # Done 304 my $end_time = [gettimeofday()]; 305 my $duration = tv_interval($start_time, $end_time); 306 print "[" . @{$end_time}[0] . "." . @{$end_time}[1] . "] Parallel Import Complete in: " . sprintf('%0.6f', $duration) . " seconds\n"; 300 307 } 308 ## main() ## 301 309 302 310 &main(); 311 312 1;
Note:
See TracChangeset
for help on using the changeset viewer.