Changeset 486 for trunk


Ignore:
Timestamp:
1999-08-31T20:51:55+12:00 (25 years ago)
Author:
rjmcnab
Message:

Added code to keep track of indexes that aren't built and changed
the directory separator on windows to \ so the execution of programs
using the shell is more robust.

File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/gsdl/perllib/mgbuilder.pm

    r461 r486  
    3434              'verbosity'=>$verbosity,
    3535              'maxdocs'=>$maxdocs,
    36               'allclassifications'=>$allclassifications
     36              'allclassifications'=>$allclassifications,
     37              'notbuilt'=>[]    # indexes not built
    3738              }, $class;
    3839
    … …  
    153154    my $exedir = "$ENV{'GSDLHOME'}/bin/$ENV{'GSDLOS'}";
    154155    my $exe = &util::get_os_exe ();
     156    my $mg_passes_exe = &util::filename_cat($exedir, "mg_passes$exe");
     157    my $mg_compression_dict_exe = &util::filename_cat($exedir, "mg_compression_dict$exe");
    155158
    156159    &util::mk_all_dir (&util::filename_cat($self->{'build_dir'}, "text"));
    … …  
    176179    # -b $maxdocsize sets the maximum document size to be 12 meg
    177180    print STDERR "\n    collecting text statistics\n"  if ($self->{'verbosity'} >= 1);
    178     if (!-e "$exedir/mg_passes$exe" || !open (PIPEOUT,
    179             "| $exedir/mg_passes$exe -f $fulltextprefix -b $maxdocsize -T1 $osextra")) {
    180     die "mgbuilder::compress_text - couldn't run $exedir/mg_passes$exe\n";
     181    if (!-e "$mg_passes_exe" || !open (PIPEOUT,
     182            "| $mg_passes_exe -f $fulltextprefix -b $maxdocsize -T1 $osextra")) {
     183    die "mgbuilder::compress_text - couldn't run $mg_passes_exe\n";
    181184    }
    182185    $self->{'buildproc'}->reset();
    … …  
    191194    # words being put into the dictionary first (-2 -k 5120)
    192195    print STDERR "\n    creating the compression dictionary\n"  if ($self->{'verbosity'} >= 1);
    193     if (!-e "$exedir/mg_compression_dict$exe") {
    194     die "mgbuilder::compress_text - couldn't run $exedir/mg_compression_dict$exe\n";
    195     }
    196     system ("$exedir/mg_compression_dict$exe -f $fulltextprefix -S -H -2 -k 5120 $osextra");
     196    if (!-e "$mg_compression_dict_exe") {
     197    die "mgbuilder::compress_text - couldn't run $mg_compression_dict_exe\n";
     198    }
     199    system ("$mg_compression_dict_exe -f $fulltextprefix -S -H -2 -k 5120 $osextra");
    197200
    198201    # compress the text
    199202    # -b $maxdocsize sets the maximum document size to be 12 meg
    200203    print STDERR "\n    compressing the text\n"  if ($self->{'verbosity'} >= 1);
    201     if (!-e "$exedir/mg_passes$exe" || !open (PIPEOUT,
    202             "| $exedir/mg_passes$exe -f $fulltextprefix -b $maxdocsize -T2 $osextra")) {
    203     die "mgbuilder::compress_text - couldn't run $exedir/mg_passes$exe\n";
     204    if (!-e "$mg_passes_exe" || !open (PIPEOUT,
     205            "| $mg_passes_exe -f $fulltextprefix -b $maxdocsize -T2 $osextra")) {
     206    die "mgbuilder::compress_text - couldn't run $mg_passes_exe\n";
    204207    }
    205208    $self->{'buildproc'}->reset();
    … …  
    209212}
    210213
     214sub want_built {
     215    my $self = shift (@_);
     216    my ($index) = @_;
     217
     218    if (defined ($self->{'collect_cfg'}->{'dontbuild'})) {
     219    foreach $checkstr (@{$self->{'collect_cfg'}->{'dontbuild'}}) {
     220        if ($index =~ /^$checkstr$/) {
     221        push (@{$self->{'notbuilt'}}, $self->{'index_mapping'}->{$index});
     222        return 0;
     223        }
     224    }
     225    }
     226
     227    return 1;
     228}
     229
    211230sub build_indexes {
    212231    my $self = shift (@_);
    … …  
    219238    # build each of the indexes
    220239    foreach $index (@$indexes) {
    221     print STDERR "\n*** building index $index in subdirectory " .
    222         "$self->{'index_mapping'}->{$index}\n" if ($self->{'verbosity'} >= 1);
    223     $self->build_index($index);
     240    if ($self->want_built($index)) {
     241        print STDERR "\n*** building index $index in subdirectory " .
     242        "$self->{'index_mapping'}->{$index}\n" if ($self->{'verbosity'} >= 1);
     243        $self->build_index($index);
     244    } else {
     245        print STDERR "\n*** ignoring index $index\n" if ($self->{'verbosity'} >= 1);
     246    }
    224247    }
    225248}
    … …  
    358381    my $exedir = "$ENV{'GSDLHOME'}/bin/$ENV{'GSDLOS'}";
    359382    my $exe = &util::get_os_exe ();
     383    my $mg_passes_exe = &util::filename_cat($exedir, "mg_passes$exe");
     384    my $mg_perf_hash_build_exe =
     385    &util::filename_cat($exedir, "mg_perf_hash_build$exe");
     386    my $mg_weights_build_exe =
     387    &util::filename_cat ($exedir, "mg_weights_build$exe");
     388    my $mg_invf_dict_exe =
     389    &util::filename_cat ($exedir, "mg_invf_dict$exe");
     390    my $mg_stem_idx_exe =
     391    &util::filename_cat ($exedir, "mg_stem_idx$exe");
     392
    360393    my $osextra = "";
    361394    if ($ENV{'GSDLOS'} =~ /^windows$/i) {
    … …  
    403436    # Build index dictionary. Uses verbatim stem method
    404437    print STDERR "\n    creating index dictionary\n"  if ($self->{'verbosity'} >= 1);
    405     if (!-e "$exedir/mg_passes$exe" || !open (PIPEOUT,
    406         "| $exedir/mg_passes$exe -f $fullindexprefix -b $maxdocsize " .
     438    if (!-e "$mg_passes_exe" || !open (PIPEOUT,
     439        "| $mg_passes_exe -f $fullindexprefix -b $maxdocsize " .
    407440        "-$index_level -m 32 -s 0 -G -t 10 -N1 $osextra")) {
    408     die "mgbuilder::build_index - couldn't run $exedir/mg_passes$exe\n";
     441    die "mgbuilder::build_index - couldn't run $mg_passes_exe\n";
    409442    }
    410443    $self->{'buildproc'}->reset();
    … …  
    414447
    415448    # create the perfect hash function
    416     if (!-e "$exedir/mg_perf_hash_build$exe") {
    417     die "mgbuilder::build_index - couldn't run $exedir/mg_perf_hash_build$exe\n";
    418     }
    419     system ("$exedir/mg_perf_hash_build$exe -f $fullindexprefix $osextra");
     449    if (!-e "$mg_perf_hash_build_exe") {
     450    die "mgbuilder::build_index - couldn't run $mg_perf_hash_build_exe\n";
     451    }
     452    system ("$mg_perf_hash_build_exe -f $fullindexprefix $osextra");
    420453
    421454    # invert the text
    422455    print STDERR "\n    inverting the text\n"  if ($self->{'verbosity'} >= 1);
    423     if (!-e "$exedir/mg_passes$exe" || !open (PIPEOUT,
    424         "| $exedir/mg_passes$exe -f $fullindexprefix -b $maxdocsize " .
     456    if (!-e "$mg_passes_exe" || !open (PIPEOUT,
     457        "| $mg_passes_exe -f $fullindexprefix -b $maxdocsize " .
    425458        "-$index_level -c 3 -G -t 10 -N2 $osextra")) {
    426     die "mgbuilder::build_index - couldn't run $exedir/mg_passes$exe\n";
     459    die "mgbuilder::build_index - couldn't run $mg_passes_exe\n";
    427460    }
    428461    $self->{'buildproc'}->reset();
    … …  
    433466    # create the weights file
    434467    print STDERR "\n    create the weights file\n"  if ($self->{'verbosity'} >= 1);
    435     if (!-e "$exedir/mg_weights_build$exe") {
    436     die "mgbuilder::build_index - couldn't run $exedir/mg_weights_build$exe\n";
    437     }
    438     system ("$exedir/mg_weights_build$exe -f $fullindexprefix -t $fulltextprefix $osextra");
     468    if (!-e "$mg_weights_build_exe") {
     469    die "mgbuilder::build_index - couldn't run $mg_weights_build_exe\n";
     470    }
     471    system ("$mg_weights_build_exe -f $fullindexprefix -t $fulltextprefix $osextra");
    439472
    440473    # create 'on-disk' stemmed dictionary
    441474    print STDERR "\n    creating 'on-disk' stemmed dictionary\n"  if ($self->{'verbosity'} >= 1);
    442     if (!-e "$exedir/mg_invf_dict$exe") {
    443     die "mgbuilder::build_index - couldn't run $exedir/mg_invf_dict$exe\n";
    444     }
    445     system ("$exedir/mg_invf_dict$exe -f $fullindexprefix $osextra");
     475    if (!-e "$mg_invf_dict_exe") {
     476    die "mgbuilder::build_index - couldn't run $mg_invf_dict_exe\n";
     477    }
     478    system ("$mg_invf_dict_exe -f $fullindexprefix $osextra");
    446479
    447480
    448481    # creates stem index files for the various stemming methods
    449482    print STDERR "\n    creating stem indexes\n"  if ($self->{'verbosity'} >= 1);
    450     if (!-e "$exedir/mg_stem_idx$exe") {
    451     die "mgbuilder::build_index - couldn't run $exedir/mg_stem_idx$exe\n";
    452     }
    453     system ("$exedir/mg_stem_idx$exe -b 4096 -s1 -f $fullindexprefix $osextra");
    454     system ("$exedir/mg_stem_idx$exe -b 4096 -s2 -f $fullindexprefix $osextra");
    455     system ("$exedir/mg_stem_idx$exe -b 4096 -s3 -f $fullindexprefix $osextra");
     483    if (!-e "$mg_stem_idx_exe") {
     484    die "mgbuilder::build_index - couldn't run $mg_stem_idx_exe\n";
     485    }
     486    system ("$mg_stem_idx_exe -b 4096 -s1 -f $fullindexprefix $osextra");
     487    system ("$mg_stem_idx_exe -b 4096 -s2 -f $fullindexprefix $osextra");
     488    system ("$mg_stem_idx_exe -b 4096 -s3 -f $fullindexprefix $osextra");
    456489
    457490   
    … …  
    485518    my $exedir = "$ENV{'GSDLHOME'}/bin/$ENV{'GSDLOS'}";
    486519    my $exe = &util::get_os_exe ();
     520    my $txt2db_exe = &util::filename_cat($exedir, "txt2db$exe");
    487521
    488522    print STDERR "\n*** creating the info database\n" if ($self->{'verbosity'} >= 1);
    … …  
    498532   
    499533    # create the infodatabase
    500     if (!-e "$exedir/txt2db$exe" || !open (PIPEOUT,
    501             "| $exedir/txt2db$exe $fulldbname")) {
    502     die "mgbuilder::make_infodatabase - couldn't run $exedir/txt2db$exe\n";
     534    if (!-e "$txt2db_exe" || !open (PIPEOUT,
     535            "| $txt2db_exe $fulldbname")) {
     536    die "mgbuilder::make_infodatabase - couldn't run $txt2db_exe\n";
    503537    }
    504538    $self->{'buildproc'}->reset();
    … …  
    578612    $build_cfg->{'languagemap'} = \@languagemap if scalar (@languagemap);
    579613
     614    $build_cfg->{'notbuilt'} = $self->{'notbuilt'};
     615
    580616    # write out the build information
    581617    &cfgread::write_cfg_file("$self->{'build_dir'}/build.cfg", $build_cfg,
    582                  '^(builddate|numdocs|numbytes)$', '^(indexmap|subcollectionmap|languagemap)$');
     618                 '^(builddate|numdocs|numbytes)$', '^(indexmap|subcollectionmap|languagemap|notbuilt)$');
    583619
    584620}
Note: See TracChangeset for help on using the changeset viewer.