Changeset 28768 for gs2-extensions

Show
Ignore:
Timestamp:
18.12.2013 10:53:15 (6 years ago)
Author:
jmt12
Message:

Initially added microtime to this script, but then remembered it isn't actually used any more. Added message about being obsolete - and then changed file format to utf-8-unix, thus forever obscuring any other changes I've made. sigh. Oh well, I'm going to remove this as soon as I get it up-to-date

Files:
1 modified

Legend:

Unmodified
Added
Removed

  • gs2-extensions/parallel-building/trunk/src/bin/script/parallel_import.pl

    r25117 r28768  
    11#!/usr/bin/perl -w 
    22 
    3 ########################################################################### 
    4 # 
    5 # import.pl -- 
    6 # A component of the Greenstone digital library software 
    7 # from the New Zealand Digital Library Project at the  
    8 # University of Waikato, New Zealand. 
     3############################################################################### 
     4# 
     5# import.pl -- A component of the Greenstone digital library software from the 
     6# New Zealand Digital Library Project at the University of Waikato, New 
     7# Zealand. 
    98# 
    109# Copyright (C) 1999 New Zealand Digital Library Project 
    1110# 
    12 # This program is free software; you can redistribute it and/or modify 
    13 # it under the terms of the GNU General Public License as published by 
    14 # the Free Software Foundation; either version 2 of the License, or 
    15 # (at your option) any later version. 
    16 # 
    17 # This program is distributed in the hope that it will be useful, 
    18 # but WITHOUT ANY WARRANTY; without even the implied warranty of 
    19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
    20 # GNU General Public License for more details. 
    21 # 
    22 # You should have received a copy of the GNU General Public License 
    23 # along with this program; if not, write to the Free Software 
    24 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 
    25 # 
    26 ########################################################################### 
    27  
     11# This program is free software; you can redistribute it and/or modify it under 
     12# the terms of the GNU General Public License as published by the Free Software 
     13# Foundation; either version 2 of the License, or (at your option) any later 
     14# version. 
     15# 
     16# This program is distributed in the hope that it will be useful, but WITHOUT 
     17# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
     18# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more 
     19# details. 
     20# 
     21# You should have received a copy of the GNU General Public License along with 
     22# this program; if not, write to the Free Software Foundation, Inc., 675 Mass 
     23# Ave, Cambridge, MA 02139, USA. 
     24# 
     25############################################################################### 
    2826 
    2927# This program will import a number of files into a particular collection 
    … …  
    3129package parallel_import; 
    3230 
    33 BEGIN { 
    34     die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'}; 
    35     die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'}; 
    36     unshift (@INC, "$ENV{'GSDLHOME'}/perllib"); 
    37     unshift (@INC, "$ENV{'GSDLHOME'}/perllib/cpan"); 
    38     unshift (@INC, "$ENV{'GSDLHOME'}/perllib/plugins"); 
    39     unshift (@INC, "$ENV{'GSDLHOME'}/perllib/plugouts"); 
    40  
    41     if (defined $ENV{'GSDLEXTS'}) { 
    42     my @extensions = split(/:/,$ENV{'GSDLEXTS'}); 
    43     foreach my $e (@extensions) { 
    44         my $ext_prefix = "$ENV{'GSDLHOME'}/ext/$e"; 
    45  
    46         unshift (@INC, "$ext_prefix/perllib"); 
    47         unshift (@INC, "$ext_prefix/perllib/cpan"); 
    48         unshift (@INC, "$ext_prefix/perllib/plugins"); 
    49         unshift (@INC, "$ext_prefix/perllib/plugouts"); 
    50     } 
     31BEGIN 
     32{ 
     33  if (!defined $ENV{'GSDLHOME'}) 
     34  { 
     35    die("GSDLHOME not set\n"); 
     36  } 
     37  unshift (@INC, $ENV{'GSDLHOME'} . '/perllib'); 
     38  unshift (@INC, $ENV{'GSDLHOME'} . '/perllib/cpan'); 
     39  unshift (@INC, $ENV{'GSDLHOME'} . '/perllib/plugins'); 
     40  unshift (@INC, $ENV{'GSDLHOME'} . '/perllib/plugouts'); 
     41 
     42  if (defined $ENV{'GSDLEXTS'}) 
     43  { 
     44    my @extensions = split(/:/,$ENV{'GSDLEXTS'}); 
     45    foreach my $e (@extensions) 
     46    { 
     47      my $ext_prefix = $ENV{'GSDLHOME'} . '/ext/' . $e; 
     48      unshift (@INC, $ext_prefix . '/perllib'); 
     49      unshift (@INC, $ext_prefix . '/perllib/cpan'); 
     50      unshift (@INC, $ext_prefix . '/perllib/plugins'); 
     51      unshift (@INC, $ext_prefix . '/perllib/plugouts'); 
    5152    } 
    52     if (defined $ENV{'GSDL3EXTS'}) { 
    53     my @extensions = split(/:/,$ENV{'GSDL3EXTS'}); 
    54     foreach my $e (@extensions) { 
    55         my $ext_prefix = "$ENV{'GSDL3SRCHOME'}/ext/$e"; 
    56  
    57         unshift (@INC, "$ext_prefix/perllib"); 
    58         unshift (@INC, "$ext_prefix/perllib/cpan"); 
    59         unshift (@INC, "$ext_prefix/perllib/plugins"); 
    60         unshift (@INC, "$ext_prefix/perllib/plugouts"); 
    61     } 
     53  } 
     54  # We may want to make use of GSDL3 Extensions too, such as Solr 
     55  if (defined $ENV{'GSDL3EXTS'}) 
     56  { 
     57    my @extensions = split(/:/,$ENV{'GSDL3EXTS'}); 
     58    foreach my $e (@extensions) 
     59    { 
     60      my $ext_prefix = $ENV{'GSDL3SRCHOME'} . '/ext/' . $e; 
     61      unshift (@INC, $ext_prefix . '/perllib'); 
     62      unshift (@INC, $ext_prefix . '/perllib/cpan'); 
     63      unshift (@INC, $ext_prefix . '/perllib/plugins'); 
     64      unshift (@INC, $ext_prefix . '/perllib/plugouts'); 
    6265    } 
     66  } 
    6367} 
    6468 
    6569use strict; 
    6670 
     71use Time::HiRes  qw( gettimeofday tv_interval ); 
    6772use inexport; 
    6873 
    … …  
    179184    'modegli' => "1" }, 
    180185      # don't set the default to hash - want to allow this to come from 
    181       # entry in collect.cfg but want to override it here  
     186      # entry in collect.cfg but want to override it here 
    182187      { 'name' => "OIDtype", 
    183188    'desc' => "{import.OIDtype}", 
    … …  
    276281        'args' => $arguments }; 
    277282 
    278  
    279  
    280  
    281  
    282 sub main  
     283## @function main() 
     284# 
     285sub main 
    283286{ 
    284     print "Parallel Import Started: " . `date` . "\n"; 
    285  
    286     my $inexport = new inexport("import",\@ARGV,$options);    
    287  
    288     my $collection = $inexport->get_collection(); 
    289     my ($config_filename,$collect_cfg) = $inexport->read_collection_cfg($collection,$options);     
    290     $inexport->set_collection_options($collect_cfg); 
    291      
    292     my $pluginfo = $inexport->process_files($config_filename,$collect_cfg); 
    293  
    294     $inexport->generate_statistics($pluginfo); 
    295  
    296     # Ensure all changes are pushed to disk 
    297     #`sync`; 
    298  
    299     print "Parallel Import Complete: " . `date` . "\n"; 
     287  my $start_time = [gettimeofday()]; 
     288  print "[" . @{$start_time}[0] . '.' . @{$start_time}[1] . "] Parallel Import Started\n"; 
     289 
     290  print "Warning! Script is obsolete - use: import.pl -workers <int> [-batchsize <int>] <collection>\n"; 
     291 
     292  my $inexport = new inexport("import", \@ARGV, $options); 
     293 
     294  # get the collection configuration (we'll need it to init the plugins) 
     295  my $collection = $inexport->get_collection(); 
     296  my ($config_filename,$collect_cfg) = $inexport->read_collection_cfg($collection,$options); 
     297  $inexport->set_collection_options($collect_cfg); 
     298 
     299  my $pluginfo = $inexport->process_files($config_filename,$collect_cfg); 
     300 
     301  $inexport->generate_statistics($pluginfo); 
     302 
     303  # Done 
     304  my $end_time = [gettimeofday()]; 
     305  my $duration = tv_interval($start_time, $end_time); 
     306  print "[" . @{$end_time}[0] . "." . @{$end_time}[1] . "] Parallel Import Complete in: " . sprintf('%0.6f', $duration) . " seconds\n"; 
    300307} 
     308## main() ## 
    301309 
    302310&main(); 
     311 
     3121;