############################################################################### # # parallelmgbuilder.pm -- inherits from the MGBuilder object but adds parallel # indexing capability by way of a function that generates the 'recipe' for # indexing. This recipe including information on precedence and can be used by # a controller (such as OpenMPI) to parallelize parts of the recipe. # # A component of the Greenstone digital library software # from the New Zealand Digital Library Project at the # University of Waikato, New Zealand. # # Copyright (C) 2013 New Zealand Digital Library Project # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. # ########################################################################### # @author John Thompson [jmt12], Waikato DL Research group package parallelmgbuilder; use mgbuilder; use parallelbasebuilder; use strict; BEGIN { # multiple inheritence FTW @parallelmgbuilder::ISA = ('mgbuilder', 'parallelbasebuilder'); } # /** @function new() # */ sub new { my $class = shift(@_); my $self = new mgbuilder(@_); return bless($self, $class); } # /** new() **/ # /** @function prepareIndexRecipe() # * MG is the most 'complex' recipe we currently support. While the compress # * text and infodb phases can be run in parallel, the indexes cannot be built # * until after the compress text phase is complete. # * @param $self # * @param $collection # * @param $recipe a reference to an array of recipe 'steps' # * @author jmt12 # */ sub prepareIndexRecipe { my ($self, $collection, $recipe) = @_; my $outhandle = $self->{'outhandle'}; my $verbosity = $self->{'verbosity'}; # 1. We start by building up the array of indexes, as we'll need to add them # as child tasks to the compress text phase my $index_tasks = (); my $indexes = $self->{'collect_cfg'}->{'indexes'}; foreach my $index (@{$indexes}) { push(@{$index_tasks}, {'command'=>'buildcol.pl -keepold -verbosity ' . $verbosity . ' -mode build_index -indexname ' . $index . ' ' . $collection}); } # 2. Compressing the text - this time with dependent tasks push(@{$recipe}, {'command'=>'buildcol.pl -keepold -verbosity ' . $verbosity . ' -mode compress_text ' . $collection, 'children'=>$index_tasks}); # 3. Info database building push(@{$recipe}, {'command'=>'buildcol.pl -keepold -verbosity ' . $verbosity . ' -mode infodb ' . $collection}); # Complete! } # /** prepareIndexRecipe() **/ 1;