Ignore:
Timestamp:
2014-09-22T20:38:05+12:00 (10 years ago)
Author:
ak19
Message:

Better debugging: if verbosity is >= 4, then mgbuilder and mgppbuilder will print the mg(pp) cmds they are going to run. Related to commits 28119, 28121 which checked for errors after executing mg(pp) cmds

Location:
main/trunk/greenstone2/perllib
Files:
2 edited

Legend:

Unmodified
Added
Removed

  • main/trunk/greenstone2/perllib/mgbuilder.pm

    r28375 r29328  
    158158    }
    159159    else {
    160     if (!-e "$mg_passes_exe" ||
    161         !open($handle, "| mg_passes$exe -f \"$fulltextprefix\" -b $maxdocsize -T1 -M $maxnumeric $osextra")) {
     160    my $mgpasses_cmd = "mg_passes$exe -f \"$fulltextprefix\" -b $maxdocsize -T1 -M $maxnumeric $osextra";   
     161    #print STDERR "**** mg_passes$exe -f \"$fulltextprefix\" -b $maxdocsize -T1 -M $maxnumeric $osextra\n\n";
     162    print $outhandle "\ncmd: $mgpasses_cmd\n" if ($self->{'verbosity'} >= 4);
     163
     164    if (!-e "$mg_passes_exe" || !open($handle, "| $mgpasses_cmd")) {
    162165        print STDERR "<FatalError name='NoRunMGPasses'>\n</Stage>\n" if $self->{'gli'};
    163166        die "mgbuilder::compress_text - couldn't run $mg_passes_exe\n";
    … …  
    196199    if (!$self->{'debug'}) {
    197200    print $outhandle "\n    creating the compression dictionary\n"  if ($self->{'verbosity'} >= 1);
     201    my $compdict_cmd = "mg_compression_dict$exe -f \"$fulltextprefix\" -S\ -H -2 -k 5120 $osextra";
     202    print $outhandle "\ncmd: $compdict_cmd\n" if ($self->{'verbosity'} >= 4);
    198203    print STDERR "<Phase name='CreatingCompress'/>\n" if $self->{'gli'};
    199204    if (!-e "$mg_compression_dict_exe") {
    200205        die "mgbuilder::compress_text - couldn't run $mg_compression_dict_exe\n";
    201206    }
    202     my $comp_dict_status = system ("mg_compression_dict$exe -f \"$fulltextprefix\" -S -H -2 -k 5120 $osextra");
     207    my $comp_dict_status = system ($compdict_cmd);
    203208    if($comp_dict_status != 0) {
    204209        print $outhandle "\nmgbuilder::compress_text - Warning: there's no compressed text\n";
    … …  
    209214
    210215    # -b $maxdocsize sets the maximum document size to be 12 meg
    211     if (!-e "$mg_passes_exe" ||
    212         !open ($handle, "| mg_passes$exe -f \"$fulltextprefix\" -b $maxdocsize -T2 -M $maxnumeric $osextra")) {
     216    my $mgpasses_cmd = "mg_passes$exe -f \"$fulltextprefix\" -b $maxdocsize -T2 -M $maxnumeric $osextra" ;
     217    print $outhandle "\ncmd: $mgpasses_cmd\n" if ($self->{'verbosity'} >= 4);
     218
     219    if (!-e "$mg_passes_exe" || !open ($handle, "| $mgpasses_cmd")) {
    213220        print STDERR "<FatalError name='NoRunMGPasses'/>\n</Stage>\n" if $self->{'gli'};
    214221        die "mgbuilder::compress_text - couldn't run $mg_passes_exe\n";
    … …  
    415422    }
    416423    else {
    417     if (!-e "$mg_passes_exe" ||
    418         !open($handle, "| mg_passes$exe -f \"$fullindexprefix\" -b $maxdocsize " .
    419            "-$index_level -m 32 -s 0 -G -t 10 -N1 -M $maxnumeric $osextra")) {
     424        my $mgpasses_cmd = "mg_passes$exe -f \"$fullindexprefix\" -b $maxdocsize " .
     425           "-$index_level -m 32 -s 0 -G -t 10 -N1 -M $maxnumeric $osextra";
     426        print $outhandle "\ncmd: $mgpasses_cmd\n" if ($self->{'verbosity'} >= 4);
     427
     428    if (!-e "$mg_passes_exe" || !open($handle, "| $mgpasses_cmd")) {
    420429        print STDERR "<FatalError name='NoRunMGPasses'/>\n</Stage>\n" if $self->{'gli'};
    421430        die "mgbuilder::build_index - couldn't run $mg_passes_exe\n";
    … …  
    466475    }
    467476
    468     if (!-e "$mg_passes_exe" ||
    469         !open ($handle, "| mg_passes$exe -f \"$fullindexprefix\" -b $maxdocsize " .
    470            "-$index_level -c 3 -G -t 10 -N2 -M $maxnumeric $osextra")) {
     477        my $mgpasses_cmd = "mg_passes$exe -f \"$fullindexprefix\" -b $maxdocsize " .
     478            "-$index_level -c 3 -G -t 10 -N2 -M $maxnumeric $osextra";
     479        print $outhandle "\ncmd: $mgpasses_cmd\n" if ($self->{'verbosity'} >= 4);
     480    if (!-e "$mg_passes_exe" || !open ($handle, "| $mgpasses_cmd")) {
    471481        print STDERR "<FatalError name='NoRunMGPasses'/>\n</Stage>\n" if $self->{'gli'};
    472482        die "mgbuilder::build_index - couldn't run $mg_passes_exe\n";
    … …  
    514524    # create 'on-disk' stemmed dictionary
    515525    print $outhandle "\n    creating 'on-disk' stemmed dictionary\n"  if ($self->{'verbosity'} >= 1);
     526        my $invdict_cmd = "mg_invf_dict$exe -f \"$fullindexprefix\" $osextra";
     527    print $outhandle "\ncmd: $invdict_cmd\n" if ($self->{'verbosity'} >= 4);
     528
    516529    print STDERR "<Phase name='CreateStemmedDic'/>\n" if $self->{'gli'};
    517530    if (!-e "$mg_invf_dict_exe") {
    … …  
    519532        die "mgbuilder::build_index - couldn't run $mg_invf_dict_exe\n";
    520533    }
    521     my $invdict_status = system ("mg_invf_dict$exe -f \"$fullindexprefix\" $osextra");
     534    my $invdict_status = system ($invdict_cmd);
    522535    # check that it worked - if not, don't carry on
    523536    if ($invdict_status !=0) {
    … …  
    550563    # same as above: generate all the stem idexes. But don't bother stemming if
    551564    # casefolding failed, and don't try generating indexes for both if stemming failed
    552     my $stem_index_status = system ("mg_stem_idx$exe -b 4096 -s1 -f \"$fullindexprefix\" $osextra");
     565        my $stemindex_cmd = "mg_stem_idx$exe -b 4096 -s1 -f \"$fullindexprefix\" $osextra";
     566    print $outhandle "\ncmd: $stemindex_cmd\n" if ($self->{'verbosity'} >= 4);
     567
     568        my $stem_index_status = system ($stemindex_cmd);
    553569    if($stem_index_status != 0) {
    554570        print $outhandle "\nCase folding failed: mg_stem_idx exit status $stem_index_status\n" if ($self->{'verbosity'} >= 4);
    555571    } else {
    556         $stem_index_status = system ("mg_stem_idx$exe -b 4096 -s2 -f \"$fullindexprefix\" $osextra");
     572        $stemindex_cmd = "mg_stem_idx$exe -b 4096 -s2 -f \"$fullindexprefix\" $osextra";
     573        print $outhandle "\ncmd: $stemindex_cmd\n" if ($self->{'verbosity'} >= 4);
     574        $stem_index_status = system ($stemindex_cmd);
    557575
    558576        if($stem_index_status != 0) {
    559577        print $outhandle "\nStemming failed: mg_stem_idx exit status $stem_index_status\n" if ($self->{'verbosity'} >= 4);
    560578        } else {
    561         $stem_index_status = system ("mg_stem_idx$exe -b 4096 -s3 -f \"$fullindexprefix\" $osextra");
     579        $stemindex_cmd = "mg_stem_idx$exe -b 4096 -s3 -f \"$fullindexprefix\" $osextra";
     580        print $outhandle "\ncmd: $stemindex_cmd\n" if ($self->{'verbosity'} >= 4);
     581        $stem_index_status = system ($stemindex_cmd);
    562582
    563583        if($stem_index_status != 0) {
    … …  
    596616    my $collect_tail = &util::get_dirsep_tail($self->{'collection'});
    597617    my $input_file = &FileUtils::filenameConcatenate("text", $collect_tail);
    598     if (!-e "$mgstat_exe" || !open (PIPEIN, "mgstat$exe -d \"$self->{'build_dir'}\" -f \"$input_file\" |")) {
    599     my $outhandle = $self->{'outhandle'};
     618
     619    my $mgstat_cmd = "mgstat$exe -d \"$self->{'build_dir'}\" -f \"$input_file\"";
     620    my $outhandle = $self->{'outhandle'};
     621    print $outhandle "\ncmd: $mgstat_cmd\n" if ($self->{'verbosity'} >= 4);
     622    if (!-e "$mgstat_exe" || !open (PIPEIN, "$mgstat_cmd |")) {
     623
    600624    print $outhandle "Warning: Couldn't open pipe to $mgstat_exe to get additional stats\n";
    601625    } else {

  • main/trunk/greenstone2/perllib/mgppbuilder.pm

    r28375 r29328  
    208208    }
    209209    else {
    210     if (!-e "$mgpp_passes_exe" ||
    211         !open($handle, "| mgpp_passes$exe  -M $maxnumeric $mgpp_passes_sections -f \"$fulltextprefix\" -T1 $osextra")) {
     210    my $mgpp_passes_cmd = "mgpp_passes$exe  -M $maxnumeric $mgpp_passes_sections -f \"$fulltextprefix\" -T1 $osextra";
     211    print $outhandle "\ncmd: $mgpp_passes_cmd\n"  if ($self->{'verbosity'} >= 4);
     212    if (!-e "$mgpp_passes_exe" || !open($handle, "| $mgpp_passes_cmd")) {
    212213        print STDERR "<FatalError name='NoRunMGPasses'>\n</Stage>\n" if $self->{'gli'};
    213214        die "mgppbuilder::compress_text - couldn't run $mgpp_passes_exe\n";
    … …  
    242243    # note: these options are left over from mg version
    243244    if (!$self->{'debug'}) {
     245    my $compdict_cmd = "mgpp_compression_dict$exe -f \"$fulltextprefix\" -S -H -2 -k 5120 $osextra";   
    244246    print $outhandle "\n    creating the compression dictionary\n"  if ($self->{'verbosity'} >= 1);
     247    print $outhandle "\ncmd: $compdict_cmd\n"  if ($self->{'verbosity'} >= 4);
    245248    print STDERR "<Phase name='CreatingCompress'/>\n" if $self->{'gli'};
    246249    if (!-e "$mgpp_compression_dict_exe") {
    … …  
    248251        die "mgppbuilder::compress_text - couldn't run $mgpp_compression_dict_exe\n";
    249252    }
    250     my $comp_dict_status = system ("mgpp_compression_dict$exe -f \"$fulltextprefix\" -S -H -2 -k 5120 $osextra");
     253    my $comp_dict_status = system ($compdict_cmd);
    251254    if($comp_dict_status != 0) {
    252255        print $outhandle "\nmgppbuilder::compress_text - Warning: there's no compressed text\n";
    … …  
    257260
    258261    if (!$self->{'debug'}) {
    259         if (!-e "$mgpp_passes_exe" ||
    260         !open ($handle, "| mgpp_passes$exe  -M $maxnumeric $mgpp_passes_sections -f \"$fulltextprefix\" -T2 $osextra")) {
     262        my $mgpp_passes_cmd = "mgpp_passes$exe -M $maxnumeric $mgpp_passes_sections -f \"$fulltextprefix\" -T2 $osextra";
     263        print $outhandle "\ncmd: $mgpp_passes_cmd\n"  if ($self->{'verbosity'} >= 4);
     264
     265        if (!-e "$mgpp_passes_exe" || !open ($handle, "| $mgpp_passes_cmd")) {
    261266        print STDERR "<FatalError name='NoRunMGPasses'/>\n</Stage>\n" if $self->{'gli'};
    262267        die "mgppbuilder::compress_text - couldn't run $mgpp_passes_exe\n";
    … …  
    474479    }
    475480    else {
    476     if (!-e "$mgpp_passes_exe" ||
    477         !open($handle, "| mgpp_passes$exe -M $maxnumeric $mgpp_passes_sections -f \"$fullindexprefix\" -I1 $osextra")) {
     481        my $mgpp_passes_cmd = "mgpp_passes$exe -M $maxnumeric $mgpp_passes_sections -f \"$fullindexprefix\" -I1 $osextra";
     482        print $outhandle "\ncmd: $mgpp_passes_cmd\n"  if ($self->{'verbosity'} >= 4);
     483    if (!-e "$mgpp_passes_exe" || !open($handle, "| $mgpp_passes_cmd")) {
    478484        print STDERR "<FatalError name='NoRunMGPasses'/>\n</Stage>\n" if $self->{'gli'};
    479485        die "mgppbuilder::build_index - couldn't run $mgpp_passes_exe\n";
    … …  
    530536       
    531537    }
    532    
    533     if (!-e "$mgpp_passes_exe" ||
    534         !open ($handle, "| mgpp_passes$exe -M $maxnumeric $mgpp_passes_sections -f \"$fullindexprefix\" -I2 $osextra")) {
     538
     539    my $mgpp_passes_cmd = "mgpp_passes$exe -M $maxnumeric $mgpp_passes_sections -f \"$fullindexprefix\" -I2 $osextra";
     540        print $outhandle "\ncmd: $mgpp_passes_cmd\n"  if ($self->{'verbosity'} >= 4);
     541    if (!-e "$mgpp_passes_exe" || !open ($handle, "| $mgpp_passes_cmd")) {
    535542        print STDERR "<FatalError name='NoRunMGPasses'/>\n</Stage>\n" if $self->{'gli'};
    536543        die "mgppbuilder::build_index - couldn't run $mgpp_passes_exe\n";
    … …  
    581588        die "mgppbuilder::build_index - couldn't run $mgpp_invf_dict_exe\n";
    582589    }
    583     my $invdict_status = system ("mgpp_invf_dict$exe -f \"$fullindexprefix\" $osextra" );
     590        my $invdict_cmd = "mgpp_invf_dict$exe -f \"$fullindexprefix\" $osextra";
     591    print $outhandle "\ncmd: $invdict_cmd\n"  if ($self->{'verbosity'} >= 4);
     592        my $invdict_status = system ($invdict_cmd);
    584593    # check that it worked - if not, don't carry on
    585594    if ($invdict_status !=0) {
    … …  
    600609    my $accent_folding_enabled = 1;
    601610    if ($self->{'accentfold'}) {
     611        my $accentfold_cmd = "mgpp_stem_idx$exe -b 4096 -s4 -f \"$fullindexprefix\" $osextra";
     612        print $outhandle "\ncmd: $accentfold_cmd\n"  if ($self->{'verbosity'} >= 4);
    602613        # the first time we do this, we test for accent folding enabled
    603         my $accent_status = system ("mgpp_stem_idx$exe -b 4096 -s4 -f \"$fullindexprefix\" $osextra");
     614        my $accent_status = system ($accentfold_cmd);
    604615        if ($accent_status == 2) {
    605616        # accent folding has not been enabled in mgpp
    … …  
    614625    }
    615626    if ($self->{'casefold'}) {
    616         my $casefold_status = system ("mgpp_stem_idx$exe -b 4096 -s1 -f \"$fullindexprefix\" $osextra");
     627        my $casefold_cmd = "mgpp_stem_idx$exe -b 4096 -s1 -f \"$fullindexprefix\" $osextra";
     628        print $outhandle "\ncmd: $casefold_cmd\n"  if ($self->{'verbosity'} >= 4);
     629        my $casefold_status = system ($casefold_cmd);
    617630        if ($casefold_status != 0) {
    618631        print $outhandle "\nCase folding failed: mgpp_stem_idx exit status $casefold_status\n" if ($self->{'verbosity'} >= 4);
    … …  
    622635       
    623636        elsif ($accent_folding_enabled && $self->{'accentfold'}) {
    624         my $status = system ("mgpp_stem_idx$exe -b 4096 -s5 -f \"$fullindexprefix\" $osextra");
     637            my $accent_casefold_cmd = "mgpp_stem_idx$exe -b 4096 -s5 -f \"$fullindexprefix\" $osextra";
     638            print $outhandle "\ncmd: $accent_casefold_cmd\n"  if ($self->{'verbosity'} >= 4);
     639        my $status = system ($accent_casefold_cmd);
    625640        if($status != 0) {
    626641            print $outhandle "\nAccent folding (with casefolding) failed: mgpp_stem_idx exit status $status\n" if ($self->{'verbosity'} >= 4);
    … …  
    631646    }
    632647    if ($self->{'stem'}) {
    633         my $stem_status = system ("mgpp_stem_idx$exe -b 4096 -s2 -f \"$fullindexprefix\" $osextra");
     648            my $stem_cmd = "mgpp_stem_idx$exe -b 4096 -s2 -f \"$fullindexprefix\" $osextra";
     649            print $outhandle "\ncmd: $stem_cmd\n"  if ($self->{'verbosity'} >= 4);
     650        my $stem_status = system ($stem_cmd);
    634651        if ($stem_status != 0) {
    635652        print $outhandle "\nStemming failed: mgpp_stem_idx exit status $stem_status\n" if ($self->{'verbosity'} >= 4);
    … …  
    638655        }
    639656        elsif ($accent_folding_enabled && $self->{'accentfold'}) {
    640         my $status = system ("mgpp_stem_idx$exe -b 4096 -s6 -f \"$fullindexprefix\" $osextra");
     657                my $accent_stem_cmd = "mgpp_stem_idx$exe -b 4096 -s6 -f \"$fullindexprefix\" $osextra";
     658                print $outhandle "\ncmd: $accent_stem_cmd\n"  if ($self->{'verbosity'} >= 4);
     659            my $status = system ($accent_stem_cmd);
    641660        if($status != 0) {
    642661            print $outhandle "\nAccent folding (with stemming) failed: mgpp_stem_idx exit status $status\n" if ($self->{'verbosity'} >= 4);
    … …  
    647666    }
    648667    if ($self->{'casefold'} && $self->{'stem'}) {
    649         my $case_and_stem_status = system ("mgpp_stem_idx$exe -b 4096 -s3 -f \"$fullindexprefix\" $osextra");
     668            my $case_stem_cmd = "mgpp_stem_idx$exe -b 4096 -s3 -f \"$fullindexprefix\" $osextra";
     669            print $outhandle "\ncmd: $case_stem_cmd\n"  if ($self->{'verbosity'} >= 4);
     670        my $case_and_stem_status = system ($case_stem_cmd);
    650671        if ($case_and_stem_status != 0) {
    651672        print $outhandle "\nCasefolding and stemming failed: mgpp_stem_idx exit status $case_and_stem_status\n" if ($self->{'verbosity'} >= 4);
    … …  
    655676        }
    656677        elsif ($accent_folding_enabled && $self->{'accentfold'}) {
    657         my $status = system ("mgpp_stem_idx$exe -b 4096 -s7 -f \"$fullindexprefix\" $osextra");
     678        my $accent_case_stem_cmd = "mgpp_stem_idx$exe -b 4096 -s7 -f \"$fullindexprefix\" $osextra";
     679                print $outhandle "\ncmd: $accent_case_stem_cmd\n"  if ($self->{'verbosity'} >= 4);
     680        my $status = system ($accent_case_stem_cmd);
    658681        if($status != 0) {
    659682            print $outhandle "\nAccent folding (with stemming and casefolding) failed: mgpp_stem_idx exit status $status\n" if ($self->{'verbosity'} >= 4);
Note: See TracChangeset for help on using the changeset viewer.