Changeset 29328

Show
Ignore:
Timestamp:
22.09.2014 20:38:05 (5 years ago)
Author:
ak19
Message:

Better debugging: if verbosity is >= 4, then mgbuilder and mgppbuilder will print the mg(pp) cmds they are going to run. Related to commits 28119, 28121 which checked for errors after executing mg(pp) cmds

Location:
main/trunk/greenstone2/perllib
Files:
2 modified

Legend:

Unmodified
Added
Removed
  • main/trunk/greenstone2/perllib/mgbuilder.pm

    r28375 r29328  
    158158    } 
    159159    else { 
    160     if (!-e "$mg_passes_exe" ||  
    161         !open($handle, "| mg_passes$exe -f \"$fulltextprefix\" -b $maxdocsize -T1 -M $maxnumeric $osextra")) { 
     160    my $mgpasses_cmd = "mg_passes$exe -f \"$fulltextprefix\" -b $maxdocsize -T1 -M $maxnumeric $osextra";    
     161    #print STDERR "**** mg_passes$exe -f \"$fulltextprefix\" -b $maxdocsize -T1 -M $maxnumeric $osextra\n\n"; 
     162    print $outhandle "\ncmd: $mgpasses_cmd\n" if ($self->{'verbosity'} >= 4); 
     163 
     164    if (!-e "$mg_passes_exe" || !open($handle, "| $mgpasses_cmd")) { 
    162165        print STDERR "<FatalError name='NoRunMGPasses'>\n</Stage>\n" if $self->{'gli'}; 
    163166        die "mgbuilder::compress_text - couldn't run $mg_passes_exe\n"; 
     
    196199    if (!$self->{'debug'}) { 
    197200    print $outhandle "\n    creating the compression dictionary\n"  if ($self->{'verbosity'} >= 1); 
     201    my $compdict_cmd = "mg_compression_dict$exe -f \"$fulltextprefix\" -S\ -H -2 -k 5120 $osextra"; 
     202    print $outhandle "\ncmd: $compdict_cmd\n" if ($self->{'verbosity'} >= 4); 
    198203    print STDERR "<Phase name='CreatingCompress'/>\n" if $self->{'gli'}; 
    199204    if (!-e "$mg_compression_dict_exe") { 
    200205        die "mgbuilder::compress_text - couldn't run $mg_compression_dict_exe\n"; 
    201206    } 
    202     my $comp_dict_status = system ("mg_compression_dict$exe -f \"$fulltextprefix\" -S -H -2 -k 5120 $osextra"); 
     207    my $comp_dict_status = system ($compdict_cmd); 
    203208    if($comp_dict_status != 0) { 
    204209        print $outhandle "\nmgbuilder::compress_text - Warning: there's no compressed text\n"; 
     
    209214 
    210215    # -b $maxdocsize sets the maximum document size to be 12 meg 
    211     if (!-e "$mg_passes_exe" ||  
    212         !open ($handle, "| mg_passes$exe -f \"$fulltextprefix\" -b $maxdocsize -T2 -M $maxnumeric $osextra")) { 
     216    my $mgpasses_cmd = "mg_passes$exe -f \"$fulltextprefix\" -b $maxdocsize -T2 -M $maxnumeric $osextra" ; 
     217    print $outhandle "\ncmd: $mgpasses_cmd\n" if ($self->{'verbosity'} >= 4); 
     218 
     219    if (!-e "$mg_passes_exe" || !open ($handle, "| $mgpasses_cmd")) { 
    213220        print STDERR "<FatalError name='NoRunMGPasses'/>\n</Stage>\n" if $self->{'gli'}; 
    214221        die "mgbuilder::compress_text - couldn't run $mg_passes_exe\n"; 
     
    415422    } 
    416423    else { 
    417     if (!-e "$mg_passes_exe" ||  
    418         !open($handle, "| mg_passes$exe -f \"$fullindexprefix\" -b $maxdocsize " . 
    419            "-$index_level -m 32 -s 0 -G -t 10 -N1 -M $maxnumeric $osextra")) { 
     424        my $mgpasses_cmd = "mg_passes$exe -f \"$fullindexprefix\" -b $maxdocsize " . 
     425           "-$index_level -m 32 -s 0 -G -t 10 -N1 -M $maxnumeric $osextra"; 
     426        print $outhandle "\ncmd: $mgpasses_cmd\n" if ($self->{'verbosity'} >= 4); 
     427 
     428    if (!-e "$mg_passes_exe" || !open($handle, "| $mgpasses_cmd")) { 
    420429        print STDERR "<FatalError name='NoRunMGPasses'/>\n</Stage>\n" if $self->{'gli'}; 
    421430        die "mgbuilder::build_index - couldn't run $mg_passes_exe\n"; 
     
    466475    } 
    467476 
    468     if (!-e "$mg_passes_exe" ||  
    469         !open ($handle, "| mg_passes$exe -f \"$fullindexprefix\" -b $maxdocsize " . 
    470            "-$index_level -c 3 -G -t 10 -N2 -M $maxnumeric $osextra")) { 
     477        my $mgpasses_cmd = "mg_passes$exe -f \"$fullindexprefix\" -b $maxdocsize " . 
     478            "-$index_level -c 3 -G -t 10 -N2 -M $maxnumeric $osextra"; 
     479        print $outhandle "\ncmd: $mgpasses_cmd\n" if ($self->{'verbosity'} >= 4); 
     480    if (!-e "$mg_passes_exe" || !open ($handle, "| $mgpasses_cmd")) { 
    471481        print STDERR "<FatalError name='NoRunMGPasses'/>\n</Stage>\n" if $self->{'gli'}; 
    472482        die "mgbuilder::build_index - couldn't run $mg_passes_exe\n"; 
     
    514524    # create 'on-disk' stemmed dictionary 
    515525    print $outhandle "\n    creating 'on-disk' stemmed dictionary\n"  if ($self->{'verbosity'} >= 1); 
     526        my $invdict_cmd = "mg_invf_dict$exe -f \"$fullindexprefix\" $osextra"; 
     527    print $outhandle "\ncmd: $invdict_cmd\n" if ($self->{'verbosity'} >= 4); 
     528 
    516529    print STDERR "<Phase name='CreateStemmedDic'/>\n" if $self->{'gli'}; 
    517530    if (!-e "$mg_invf_dict_exe") { 
     
    519532        die "mgbuilder::build_index - couldn't run $mg_invf_dict_exe\n"; 
    520533    } 
    521     my $invdict_status = system ("mg_invf_dict$exe -f \"$fullindexprefix\" $osextra"); 
     534    my $invdict_status = system ($invdict_cmd); 
    522535    # check that it worked - if not, don't carry on 
    523536    if ($invdict_status !=0) { 
     
    550563    # same as above: generate all the stem idexes. But don't bother stemming if 
    551564    # casefolding failed, and don't try generating indexes for both if stemming failed 
    552     my $stem_index_status = system ("mg_stem_idx$exe -b 4096 -s1 -f \"$fullindexprefix\" $osextra"); 
     565        my $stemindex_cmd = "mg_stem_idx$exe -b 4096 -s1 -f \"$fullindexprefix\" $osextra"; 
     566    print $outhandle "\ncmd: $stemindex_cmd\n" if ($self->{'verbosity'} >= 4); 
     567 
     568        my $stem_index_status = system ($stemindex_cmd); 
    553569    if($stem_index_status != 0) { 
    554570        print $outhandle "\nCase folding failed: mg_stem_idx exit status $stem_index_status\n" if ($self->{'verbosity'} >= 4); 
    555571    } else { 
    556         $stem_index_status = system ("mg_stem_idx$exe -b 4096 -s2 -f \"$fullindexprefix\" $osextra"); 
     572        $stemindex_cmd = "mg_stem_idx$exe -b 4096 -s2 -f \"$fullindexprefix\" $osextra"; 
     573        print $outhandle "\ncmd: $stemindex_cmd\n" if ($self->{'verbosity'} >= 4); 
     574        $stem_index_status = system ($stemindex_cmd); 
    557575 
    558576        if($stem_index_status != 0) { 
    559577        print $outhandle "\nStemming failed: mg_stem_idx exit status $stem_index_status\n" if ($self->{'verbosity'} >= 4); 
    560578        } else { 
    561         $stem_index_status = system ("mg_stem_idx$exe -b 4096 -s3 -f \"$fullindexprefix\" $osextra"); 
     579        $stemindex_cmd = "mg_stem_idx$exe -b 4096 -s3 -f \"$fullindexprefix\" $osextra"; 
     580        print $outhandle "\ncmd: $stemindex_cmd\n" if ($self->{'verbosity'} >= 4); 
     581        $stem_index_status = system ($stemindex_cmd); 
    562582 
    563583        if($stem_index_status != 0) { 
     
    596616    my $collect_tail = &util::get_dirsep_tail($self->{'collection'}); 
    597617    my $input_file = &FileUtils::filenameConcatenate("text", $collect_tail); 
    598     if (!-e "$mgstat_exe" || !open (PIPEIN, "mgstat$exe -d \"$self->{'build_dir'}\" -f \"$input_file\" |")) { 
    599     my $outhandle = $self->{'outhandle'}; 
     618 
     619    my $mgstat_cmd = "mgstat$exe -d \"$self->{'build_dir'}\" -f \"$input_file\""; 
     620    my $outhandle = $self->{'outhandle'}; 
     621    print $outhandle "\ncmd: $mgstat_cmd\n" if ($self->{'verbosity'} >= 4); 
     622    if (!-e "$mgstat_exe" || !open (PIPEIN, "$mgstat_cmd |")) { 
     623 
    600624    print $outhandle "Warning: Couldn't open pipe to $mgstat_exe to get additional stats\n"; 
    601625    } else { 
  • main/trunk/greenstone2/perllib/mgppbuilder.pm

    r28375 r29328  
    208208    } 
    209209    else { 
    210     if (!-e "$mgpp_passes_exe" ||  
    211         !open($handle, "| mgpp_passes$exe  -M $maxnumeric $mgpp_passes_sections -f \"$fulltextprefix\" -T1 $osextra")) { 
     210    my $mgpp_passes_cmd = "mgpp_passes$exe  -M $maxnumeric $mgpp_passes_sections -f \"$fulltextprefix\" -T1 $osextra"; 
     211    print $outhandle "\ncmd: $mgpp_passes_cmd\n"  if ($self->{'verbosity'} >= 4); 
     212    if (!-e "$mgpp_passes_exe" || !open($handle, "| $mgpp_passes_cmd")) { 
    212213        print STDERR "<FatalError name='NoRunMGPasses'>\n</Stage>\n" if $self->{'gli'}; 
    213214        die "mgppbuilder::compress_text - couldn't run $mgpp_passes_exe\n"; 
     
    242243    # note: these options are left over from mg version 
    243244    if (!$self->{'debug'}) { 
     245    my $compdict_cmd = "mgpp_compression_dict$exe -f \"$fulltextprefix\" -S -H -2 -k 5120 $osextra";     
    244246    print $outhandle "\n    creating the compression dictionary\n"  if ($self->{'verbosity'} >= 1); 
     247    print $outhandle "\ncmd: $compdict_cmd\n"  if ($self->{'verbosity'} >= 4); 
    245248    print STDERR "<Phase name='CreatingCompress'/>\n" if $self->{'gli'}; 
    246249    if (!-e "$mgpp_compression_dict_exe") { 
     
    248251        die "mgppbuilder::compress_text - couldn't run $mgpp_compression_dict_exe\n"; 
    249252    } 
    250     my $comp_dict_status = system ("mgpp_compression_dict$exe -f \"$fulltextprefix\" -S -H -2 -k 5120 $osextra"); 
     253    my $comp_dict_status = system ($compdict_cmd); 
    251254    if($comp_dict_status != 0) { 
    252255        print $outhandle "\nmgppbuilder::compress_text - Warning: there's no compressed text\n"; 
     
    257260 
    258261    if (!$self->{'debug'}) { 
    259         if (!-e "$mgpp_passes_exe" ||  
    260         !open ($handle, "| mgpp_passes$exe  -M $maxnumeric $mgpp_passes_sections -f \"$fulltextprefix\" -T2 $osextra")) { 
     262        my $mgpp_passes_cmd = "mgpp_passes$exe -M $maxnumeric $mgpp_passes_sections -f \"$fulltextprefix\" -T2 $osextra"; 
     263        print $outhandle "\ncmd: $mgpp_passes_cmd\n"  if ($self->{'verbosity'} >= 4); 
     264 
     265        if (!-e "$mgpp_passes_exe" || !open ($handle, "| $mgpp_passes_cmd")) { 
    261266        print STDERR "<FatalError name='NoRunMGPasses'/>\n</Stage>\n" if $self->{'gli'}; 
    262267        die "mgppbuilder::compress_text - couldn't run $mgpp_passes_exe\n"; 
     
    474479    } 
    475480    else { 
    476     if (!-e "$mgpp_passes_exe" ||  
    477         !open($handle, "| mgpp_passes$exe -M $maxnumeric $mgpp_passes_sections -f \"$fullindexprefix\" -I1 $osextra")) { 
     481        my $mgpp_passes_cmd = "mgpp_passes$exe -M $maxnumeric $mgpp_passes_sections -f \"$fullindexprefix\" -I1 $osextra"; 
     482        print $outhandle "\ncmd: $mgpp_passes_cmd\n"  if ($self->{'verbosity'} >= 4); 
     483    if (!-e "$mgpp_passes_exe" || !open($handle, "| $mgpp_passes_cmd")) { 
    478484        print STDERR "<FatalError name='NoRunMGPasses'/>\n</Stage>\n" if $self->{'gli'}; 
    479485        die "mgppbuilder::build_index - couldn't run $mgpp_passes_exe\n"; 
     
    530536         
    531537    } 
    532      
    533     if (!-e "$mgpp_passes_exe" ||  
    534         !open ($handle, "| mgpp_passes$exe -M $maxnumeric $mgpp_passes_sections -f \"$fullindexprefix\" -I2 $osextra")) { 
     538 
     539    my $mgpp_passes_cmd = "mgpp_passes$exe -M $maxnumeric $mgpp_passes_sections -f \"$fullindexprefix\" -I2 $osextra"; 
     540        print $outhandle "\ncmd: $mgpp_passes_cmd\n"  if ($self->{'verbosity'} >= 4); 
     541    if (!-e "$mgpp_passes_exe" || !open ($handle, "| $mgpp_passes_cmd")) { 
    535542        print STDERR "<FatalError name='NoRunMGPasses'/>\n</Stage>\n" if $self->{'gli'}; 
    536543        die "mgppbuilder::build_index - couldn't run $mgpp_passes_exe\n"; 
     
    581588        die "mgppbuilder::build_index - couldn't run $mgpp_invf_dict_exe\n"; 
    582589    } 
    583     my $invdict_status = system ("mgpp_invf_dict$exe -f \"$fullindexprefix\" $osextra" ); 
     590        my $invdict_cmd = "mgpp_invf_dict$exe -f \"$fullindexprefix\" $osextra"; 
     591    print $outhandle "\ncmd: $invdict_cmd\n"  if ($self->{'verbosity'} >= 4); 
     592        my $invdict_status = system ($invdict_cmd); 
    584593    # check that it worked - if not, don't carry on 
    585594    if ($invdict_status !=0) { 
     
    600609    my $accent_folding_enabled = 1; 
    601610    if ($self->{'accentfold'}) { 
     611        my $accentfold_cmd = "mgpp_stem_idx$exe -b 4096 -s4 -f \"$fullindexprefix\" $osextra"; 
     612        print $outhandle "\ncmd: $accentfold_cmd\n"  if ($self->{'verbosity'} >= 4); 
    602613        # the first time we do this, we test for accent folding enabled 
    603         my $accent_status = system ("mgpp_stem_idx$exe -b 4096 -s4 -f \"$fullindexprefix\" $osextra"); 
     614        my $accent_status = system ($accentfold_cmd); 
    604615        if ($accent_status == 2) { 
    605616        # accent folding has not been enabled in mgpp 
     
    614625    } 
    615626    if ($self->{'casefold'}) { 
    616         my $casefold_status = system ("mgpp_stem_idx$exe -b 4096 -s1 -f \"$fullindexprefix\" $osextra"); 
     627        my $casefold_cmd = "mgpp_stem_idx$exe -b 4096 -s1 -f \"$fullindexprefix\" $osextra"; 
     628        print $outhandle "\ncmd: $casefold_cmd\n"  if ($self->{'verbosity'} >= 4); 
     629        my $casefold_status = system ($casefold_cmd); 
    617630        if ($casefold_status != 0) { 
    618631        print $outhandle "\nCase folding failed: mgpp_stem_idx exit status $casefold_status\n" if ($self->{'verbosity'} >= 4); 
     
    622635         
    623636        elsif ($accent_folding_enabled && $self->{'accentfold'}) { 
    624         my $status = system ("mgpp_stem_idx$exe -b 4096 -s5 -f \"$fullindexprefix\" $osextra"); 
     637            my $accent_casefold_cmd = "mgpp_stem_idx$exe -b 4096 -s5 -f \"$fullindexprefix\" $osextra"; 
     638            print $outhandle "\ncmd: $accent_casefold_cmd\n"  if ($self->{'verbosity'} >= 4); 
     639        my $status = system ($accent_casefold_cmd); 
    625640        if($status != 0) { 
    626641            print $outhandle "\nAccent folding (with casefolding) failed: mgpp_stem_idx exit status $status\n" if ($self->{'verbosity'} >= 4); 
     
    631646    } 
    632647    if ($self->{'stem'}) { 
    633         my $stem_status = system ("mgpp_stem_idx$exe -b 4096 -s2 -f \"$fullindexprefix\" $osextra"); 
     648            my $stem_cmd = "mgpp_stem_idx$exe -b 4096 -s2 -f \"$fullindexprefix\" $osextra"; 
     649            print $outhandle "\ncmd: $stem_cmd\n"  if ($self->{'verbosity'} >= 4); 
     650        my $stem_status = system ($stem_cmd); 
    634651        if ($stem_status != 0) { 
    635652        print $outhandle "\nStemming failed: mgpp_stem_idx exit status $stem_status\n" if ($self->{'verbosity'} >= 4); 
     
    638655        } 
    639656        elsif ($accent_folding_enabled && $self->{'accentfold'}) { 
    640         my $status = system ("mgpp_stem_idx$exe -b 4096 -s6 -f \"$fullindexprefix\" $osextra"); 
     657                my $accent_stem_cmd = "mgpp_stem_idx$exe -b 4096 -s6 -f \"$fullindexprefix\" $osextra"; 
     658                print $outhandle "\ncmd: $accent_stem_cmd\n"  if ($self->{'verbosity'} >= 4); 
     659            my $status = system ($accent_stem_cmd); 
    641660        if($status != 0) { 
    642661            print $outhandle "\nAccent folding (with stemming) failed: mgpp_stem_idx exit status $status\n" if ($self->{'verbosity'} >= 4); 
     
    647666    } 
    648667    if ($self->{'casefold'} && $self->{'stem'}) { 
    649         my $case_and_stem_status = system ("mgpp_stem_idx$exe -b 4096 -s3 -f \"$fullindexprefix\" $osextra"); 
     668            my $case_stem_cmd = "mgpp_stem_idx$exe -b 4096 -s3 -f \"$fullindexprefix\" $osextra"; 
     669            print $outhandle "\ncmd: $case_stem_cmd\n"  if ($self->{'verbosity'} >= 4); 
     670        my $case_and_stem_status = system ($case_stem_cmd); 
    650671        if ($case_and_stem_status != 0) { 
    651672        print $outhandle "\nCasefolding and stemming failed: mgpp_stem_idx exit status $case_and_stem_status\n" if ($self->{'verbosity'} >= 4); 
     
    655676        } 
    656677        elsif ($accent_folding_enabled && $self->{'accentfold'}) { 
    657         my $status = system ("mgpp_stem_idx$exe -b 4096 -s7 -f \"$fullindexprefix\" $osextra"); 
     678        my $accent_case_stem_cmd = "mgpp_stem_idx$exe -b 4096 -s7 -f \"$fullindexprefix\" $osextra"; 
     679                print $outhandle "\ncmd: $accent_case_stem_cmd\n"  if ($self->{'verbosity'} >= 4); 
     680        my $status = system ($accent_case_stem_cmd); 
    658681        if($status != 0) { 
    659682            print $outhandle "\nAccent folding (with stemming and casefolding) failed: mgpp_stem_idx exit status $status\n" if ($self->{'verbosity'} >= 4);