Ignore:
Timestamp:
2013-12-18T10:53:15+13:00 (10 years ago)
Author:
jmt12
Message:

Initially added microtime to this script, but then remembered it isn't actually used any more. Added message about being obsolete - and then changed file format to utf-8-unix, thus forever obscuring any other changes I've made. sigh. Oh well, I'm going to remove this as soon as I get it up-to-date

File:
1 edited

Legend:

Unmodified
Added
Removed

  • gs2-extensions/parallel-building/trunk/src/bin/script/parallel_import.pl

    r25117 r28768  
    11#!/usr/bin/perl -w
    22
    3 ###########################################################################
    4 #
    5 # import.pl --
    6 # A component of the Greenstone digital library software
    7 # from the New Zealand Digital Library Project at the
    8 # University of Waikato, New Zealand.
     3###############################################################################
     4#
     5# import.pl -- A component of the Greenstone digital library software from the
     6# New Zealand Digital Library Project at the University of Waikato, New
     7# Zealand.
    98#
    109# Copyright (C) 1999 New Zealand Digital Library Project
    1110#
    12 # This program is free software; you can redistribute it and/or modify
    13 # it under the terms of the GNU General Public License as published by
    14 # the Free Software Foundation; either version 2 of the License, or
    15 # (at your option) any later version.
    16 #
    17 # This program is distributed in the hope that it will be useful,
    18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
    19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    20 # GNU General Public License for more details.
    21 #
    22 # You should have received a copy of the GNU General Public License
    23 # along with this program; if not, write to the Free Software
    24 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
    25 #
    26 ###########################################################################
    27 
     11# This program is free software; you can redistribute it and/or modify it under
     12# the terms of the GNU General Public License as published by the Free Software
     13# Foundation; either version 2 of the License, or (at your option) any later
     14# version.
     15#
     16# This program is distributed in the hope that it will be useful, but WITHOUT
     17# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
     18# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
     19# details.
     20#
     21# You should have received a copy of the GNU General Public License along with
     22# this program; if not, write to the Free Software Foundation, Inc., 675 Mass
     23# Ave, Cambridge, MA 02139, USA.
     24#
     25###############################################################################
    2826
    2927# This program will import a number of files into a particular collection
    … …  
    3129package parallel_import;
    3230
    33 BEGIN {
    34     die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
    35     die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'};
    36     unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
    37     unshift (@INC, "$ENV{'GSDLHOME'}/perllib/cpan");
    38     unshift (@INC, "$ENV{'GSDLHOME'}/perllib/plugins");
    39     unshift (@INC, "$ENV{'GSDLHOME'}/perllib/plugouts");
    40 
    41     if (defined $ENV{'GSDLEXTS'}) {
    42     my @extensions = split(/:/,$ENV{'GSDLEXTS'});
    43     foreach my $e (@extensions) {
    44         my $ext_prefix = "$ENV{'GSDLHOME'}/ext/$e";
    45 
    46         unshift (@INC, "$ext_prefix/perllib");
    47         unshift (@INC, "$ext_prefix/perllib/cpan");
    48         unshift (@INC, "$ext_prefix/perllib/plugins");
    49         unshift (@INC, "$ext_prefix/perllib/plugouts");
    50     }
     31BEGIN
     32{
     33  if (!defined $ENV{'GSDLHOME'})
     34  {
     35    die("GSDLHOME not set\n");
     36  }
     37  unshift (@INC, $ENV{'GSDLHOME'} . '/perllib');
     38  unshift (@INC, $ENV{'GSDLHOME'} . '/perllib/cpan');
     39  unshift (@INC, $ENV{'GSDLHOME'} . '/perllib/plugins');
     40  unshift (@INC, $ENV{'GSDLHOME'} . '/perllib/plugouts');
     41
     42  if (defined $ENV{'GSDLEXTS'})
     43  {
     44    my @extensions = split(/:/,$ENV{'GSDLEXTS'});
     45    foreach my $e (@extensions)
     46    {
     47      my $ext_prefix = $ENV{'GSDLHOME'} . '/ext/' . $e;
     48      unshift (@INC, $ext_prefix . '/perllib');
     49      unshift (@INC, $ext_prefix . '/perllib/cpan');
     50      unshift (@INC, $ext_prefix . '/perllib/plugins');
     51      unshift (@INC, $ext_prefix . '/perllib/plugouts');
    5152    }
    52     if (defined $ENV{'GSDL3EXTS'}) {
    53     my @extensions = split(/:/,$ENV{'GSDL3EXTS'});
    54     foreach my $e (@extensions) {
    55         my $ext_prefix = "$ENV{'GSDL3SRCHOME'}/ext/$e";
    56 
    57         unshift (@INC, "$ext_prefix/perllib");
    58         unshift (@INC, "$ext_prefix/perllib/cpan");
    59         unshift (@INC, "$ext_prefix/perllib/plugins");
    60         unshift (@INC, "$ext_prefix/perllib/plugouts");
    61     }
     53  }
     54  # We may want to make use of GSDL3 Extensions too, such as Solr
     55  if (defined $ENV{'GSDL3EXTS'})
     56  {
     57    my @extensions = split(/:/,$ENV{'GSDL3EXTS'});
     58    foreach my $e (@extensions)
     59    {
     60      my $ext_prefix = $ENV{'GSDL3SRCHOME'} . '/ext/' . $e;
     61      unshift (@INC, $ext_prefix . '/perllib');
     62      unshift (@INC, $ext_prefix . '/perllib/cpan');
     63      unshift (@INC, $ext_prefix . '/perllib/plugins');
     64      unshift (@INC, $ext_prefix . '/perllib/plugouts');
    6265    }
     66  }
    6367}
    6468
    6569use strict;
    6670
     71use Time::HiRes  qw( gettimeofday tv_interval );
    6772use inexport;
    6873
    … …  
    179184    'modegli' => "1" },
    180185      # don't set the default to hash - want to allow this to come from
    181       # entry in collect.cfg but want to override it here 
     186      # entry in collect.cfg but want to override it here
    182187      { 'name' => "OIDtype",
    183188    'desc' => "{import.OIDtype}",
    … …  
    276281        'args' => $arguments };
    277282
    278 
    279 
    280 
    281 
    282 sub main
     283## @function main()
     284#
     285sub main
    283286{
    284     print "Parallel Import Started: " . `date` . "\n";
    285 
    286     my $inexport = new inexport("import",\@ARGV,$options);   
    287 
    288     my $collection = $inexport->get_collection();
    289     my ($config_filename,$collect_cfg) = $inexport->read_collection_cfg($collection,$options);   
    290     $inexport->set_collection_options($collect_cfg);
    291    
    292     my $pluginfo = $inexport->process_files($config_filename,$collect_cfg);
    293 
    294     $inexport->generate_statistics($pluginfo);
    295 
    296     # Ensure all changes are pushed to disk
    297     #`sync`;
    298 
    299     print "Parallel Import Complete: " . `date` . "\n";
     287  my $start_time = [gettimeofday()];
     288  print "[" . @{$start_time}[0] . '.' . @{$start_time}[1] . "] Parallel Import Started\n";
     289
     290  print "Warning! Script is obsolete - use: import.pl -workers <int> [-batchsize <int>] <collection>\n";
     291
     292  my $inexport = new inexport("import", \@ARGV, $options);
     293
     294  # get the collection configuration (we'll need it to init the plugins)
     295  my $collection = $inexport->get_collection();
     296  my ($config_filename,$collect_cfg) = $inexport->read_collection_cfg($collection,$options);
     297  $inexport->set_collection_options($collect_cfg);
     298
     299  my $pluginfo = $inexport->process_files($config_filename,$collect_cfg);
     300
     301  $inexport->generate_statistics($pluginfo);
     302
     303  # Done
     304  my $end_time = [gettimeofday()];
     305  my $duration = tv_interval($start_time, $end_time);
     306  print "[" . @{$end_time}[0] . "." . @{$end_time}[1] . "] Parallel Import Complete in: " . sprintf('%0.6f', $duration) . " seconds\n";
    300307}
     308## main() ##
    301309
    302310&main();
     311
     3121;
Note: See TracChangeset for help on using the changeset viewer.