Changeset 20647


Ignore:
Timestamp:
2009-09-21T12:06:23+12:00 (13 years ago)
Author:
kjdon
Message:

now pass incremental_mode to new method. check keepold/incremental based on self->is_incremental_capable, and set to 0 if not incremental capable. moved load plugins adn classifiers to init so that we can use subclass methods

File:
1 edited

Legend:

Unmodified
Added
Removed
  • gsdl/trunk/perllib/basebuilder.pm

    r20575 r20647  
    5656sub new {
    5757    my ($class, $collection, $source_dir, $build_dir, $verbosity,
    58     $maxdocs, $debug, $keepold, $incremental,
     58    $maxdocs, $debug, $keepold, $incremental, $incremental_mode,
    5959    $remove_empty_classifications,
    6060    $outhandle, $no_text, $failhandle, $gli, $disable_OAI) = @_;
     
    7373              'keepold'=>$keepold,
    7474              'incremental'=>$incremental,
     75              'incremental_mode'=>$incremental_mode,
    7576              'remove_empty_classifications'=>$remove_empty_classifications,
    7677              'outhandle'=>$outhandle,
     
    102103    $self->{'infodbtype'} = $self->{'collect_cfg'}->{'infodbtype'} || &dbutil::get_default_infodb_type();
    103104
    104     # get the list of plugins for this collection
    105     my $plugins = [];
    106     if (defined $self->{'collect_cfg'}->{'plugin'}) {
    107     $plugins = $self->{'collect_cfg'}->{'plugin'};
    108     }
    109    
    110     # load all the plugins
    111 
    112     #build up the extra global options for the plugins
    113     my @global_opts = ();
    114     if (defined $self->{'collect_cfg'}->{'separate_cjk'} && $self->{'collect_cfg'}->{'separate_cjk'} =~ /^true$/i) {
    115     push @global_opts, "-separate_cjk";
    116     }
    117     $self->{'pluginfo'} = &plugin::load_plugins ($plugins, $verbosity, $outhandle, $failhandle, \@global_opts, $keepold);
    118    
    119     if (scalar(@{$self->{'pluginfo'}}) == 0) {
    120     print $outhandle "No plugins were loaded.\n";
    121     die "\n";
    122     }
    123 
    124     # get the list of classifiers for this collection
    125     my $classifiers = [];
    126     if (defined $self->{'collect_cfg'}->{'classify'}) {
    127     $classifiers = $self->{'collect_cfg'}->{'classify'};
    128     }
    129 
    130     # load all the classifiers
    131     $self->{'classifiers'} = &classify::load_classifiers ($classifiers, $build_dir, $outhandle);
    132105
    133106    # load up any dontdb fields
     
    147120    my $self = shift(@_);
    148121   
     122    my $outhandle = $self->{'outhandle'};
     123    my $failhandle = $self->{'failhandle'};
     124
    149125    $self->generate_index_list();
    150126    my $indexes = $self->{'collect_cfg'}->{'indexes'};
     
    192168    }
    193169
     170    # check incremental against whether builder can cope or not.
     171    if (($self->{'keepold'} || $self->{'incremental'}) && !$self->is_incremental_capable()) {
     172    print $outhandle "WARNING: The indexer used is not capable of incremental building. Reverting to -removeold\n";
     173    $self->{'keepold'} = 0;
     174    $self->{'incremental'} = 0;
     175    $self->{'incremental_mode'} = "none";
     176   
     177    }
     178
     179
     180    # get the list of plugins for this collection
     181    my $plugins = [];
     182    if (defined $self->{'collect_cfg'}->{'plugin'}) {
     183    $plugins = $self->{'collect_cfg'}->{'plugin'};
     184    }
     185   
     186    # load all the plugins
     187
     188    #build up the extra global options for the plugins
     189    my @global_opts = ();
     190    if (defined $self->{'collect_cfg'}->{'separate_cjk'} && $self->{'collect_cfg'}->{'separate_cjk'} =~ /^true$/i) {
     191    push @global_opts, "-separate_cjk";
     192    }
     193    $self->{'pluginfo'} = &plugin::load_plugins ($plugins, $self->{'verbosity'}, $outhandle, $failhandle, \@global_opts, $self->{'incremental_mode'});
     194   
     195    if (scalar(@{$self->{'pluginfo'}}) == 0) {
     196    print $outhandle "No plugins were loaded.\n";
     197    die "\n";
     198    }
     199
     200    # get the list of classifiers for this collection
     201    my $classifiers = [];
     202    if (defined $self->{'collect_cfg'}->{'classify'}) {
     203    $classifiers = $self->{'collect_cfg'}->{'classify'};
     204    }
     205
     206    # load all the classifiers
     207    $self->{'classifiers'} = &classify::load_classifiers ($classifiers, $self->{'build_dir'}, $outhandle);
     208
    194209    # load up the document processor for building
    195210    # if a buildproc class has been created for this collection, use it
    196     # otherwise, use the mg buildproc
     211    # otherwise, use the default buildproc for the builder we are initialising
    197212    my ($buildprocdir, $buildproctype);
    198213    my $collection = $self->{'collection'};
     
    236251}
    237252
     253sub is_incremental_capable
     254{
     255    # By default we return 'no' as the answer
     256    # Safer to assume non-incremental to start with, and then override in
     257    # inherited classes that are.
     258
     259    return 0;
     260}
     261
    238262# implement this in subclass if want to do additional initialisation for an
    239263# incremental build
     
    386410    # the fork creates a clone of the *large* process image which (admittedly)
    387411    # is then quickly replaced in the execve() with the much smaller image for
    388     # 'txt2db'.  The trouble is, in that for a seismic second caused by
     412    # 'txt2db'.  The trouble is, in that seismic second caused by
    389413    # the fork(), the system really does need to have all that memory available
    390414    # even though it isn't ultimately used.  The result is an out of memory
Note: See TracChangeset for help on using the changeset viewer.