Changeset 29328 for main/trunk/greenstone2/perllib
- Timestamp:
- 2014-09-22T20:38:05+12:00 (10 years ago)
- Location:
- main/trunk/greenstone2/perllib
- Files:
-
- 2 edited
Legend:
- Unmodified
- Added
- Removed
-
main/trunk/greenstone2/perllib/mgbuilder.pm
r28375 r29328 158 158 } 159 159 else { 160 if (!-e "$mg_passes_exe" || 161 !open($handle, "| mg_passes$exe -f \"$fulltextprefix\" -b $maxdocsize -T1 -M $maxnumeric $osextra")) { 160 my $mgpasses_cmd = "mg_passes$exe -f \"$fulltextprefix\" -b $maxdocsize -T1 -M $maxnumeric $osextra"; 161 #print STDERR "**** mg_passes$exe -f \"$fulltextprefix\" -b $maxdocsize -T1 -M $maxnumeric $osextra\n\n"; 162 print $outhandle "\ncmd: $mgpasses_cmd\n" if ($self->{'verbosity'} >= 4); 163 164 if (!-e "$mg_passes_exe" || !open($handle, "| $mgpasses_cmd")) { 162 165 print STDERR "<FatalError name='NoRunMGPasses'>\n</Stage>\n" if $self->{'gli'}; 163 166 die "mgbuilder::compress_text - couldn't run $mg_passes_exe\n"; … … 196 199 if (!$self->{'debug'}) { 197 200 print $outhandle "\n creating the compression dictionary\n" if ($self->{'verbosity'} >= 1); 201 my $compdict_cmd = "mg_compression_dict$exe -f \"$fulltextprefix\" -S\ -H -2 -k 5120 $osextra"; 202 print $outhandle "\ncmd: $compdict_cmd\n" if ($self->{'verbosity'} >= 4); 198 203 print STDERR "<Phase name='CreatingCompress'/>\n" if $self->{'gli'}; 199 204 if (!-e "$mg_compression_dict_exe") { 200 205 die "mgbuilder::compress_text - couldn't run $mg_compression_dict_exe\n"; 201 206 } 202 my $comp_dict_status = system ( "mg_compression_dict$exe -f \"$fulltextprefix\" -S -H -2 -k 5120 $osextra");207 my $comp_dict_status = system ($compdict_cmd); 203 208 if($comp_dict_status != 0) { 204 209 print $outhandle "\nmgbuilder::compress_text - Warning: there's no compressed text\n"; … … 209 214 210 215 # -b $maxdocsize sets the maximum document size to be 12 meg 211 if (!-e "$mg_passes_exe" || 212 !open ($handle, "| mg_passes$exe -f \"$fulltextprefix\" -b $maxdocsize -T2 -M $maxnumeric $osextra")) { 216 my $mgpasses_cmd = "mg_passes$exe -f \"$fulltextprefix\" -b $maxdocsize -T2 -M $maxnumeric $osextra" ; 217 print $outhandle "\ncmd: $mgpasses_cmd\n" if ($self->{'verbosity'} >= 4); 218 219 if (!-e "$mg_passes_exe" || !open ($handle, "| $mgpasses_cmd")) { 213 220 print STDERR "<FatalError name='NoRunMGPasses'/>\n</Stage>\n" if $self->{'gli'}; 214 221 die "mgbuilder::compress_text - couldn't run $mg_passes_exe\n"; … … 415 422 } 416 423 else { 417 if (!-e "$mg_passes_exe" || 418 !open($handle, "| mg_passes$exe -f \"$fullindexprefix\" -b $maxdocsize " . 419 "-$index_level -m 32 -s 0 -G -t 10 -N1 -M $maxnumeric $osextra")) { 424 my $mgpasses_cmd = "mg_passes$exe -f \"$fullindexprefix\" -b $maxdocsize " . 425 "-$index_level -m 32 -s 0 -G -t 10 -N1 -M $maxnumeric $osextra"; 426 print $outhandle "\ncmd: $mgpasses_cmd\n" if ($self->{'verbosity'} >= 4); 427 428 if (!-e "$mg_passes_exe" || !open($handle, "| $mgpasses_cmd")) { 420 429 print STDERR "<FatalError name='NoRunMGPasses'/>\n</Stage>\n" if $self->{'gli'}; 421 430 die "mgbuilder::build_index - couldn't run $mg_passes_exe\n"; … … 466 475 } 467 476 468 if (!-e "$mg_passes_exe" || 469 !open ($handle, "| mg_passes$exe -f \"$fullindexprefix\" -b $maxdocsize " . 470 "-$index_level -c 3 -G -t 10 -N2 -M $maxnumeric $osextra")) { 477 my $mgpasses_cmd = "mg_passes$exe -f \"$fullindexprefix\" -b $maxdocsize " . 478 "-$index_level -c 3 -G -t 10 -N2 -M $maxnumeric $osextra"; 479 print $outhandle "\ncmd: $mgpasses_cmd\n" if ($self->{'verbosity'} >= 4); 480 if (!-e "$mg_passes_exe" || !open ($handle, "| $mgpasses_cmd")) { 471 481 print STDERR "<FatalError name='NoRunMGPasses'/>\n</Stage>\n" if $self->{'gli'}; 472 482 die "mgbuilder::build_index - couldn't run $mg_passes_exe\n"; … … 514 524 # create 'on-disk' stemmed dictionary 515 525 print $outhandle "\n creating 'on-disk' stemmed dictionary\n" if ($self->{'verbosity'} >= 1); 526 my $invdict_cmd = "mg_invf_dict$exe -f \"$fullindexprefix\" $osextra"; 527 print $outhandle "\ncmd: $invdict_cmd\n" if ($self->{'verbosity'} >= 4); 528 516 529 print STDERR "<Phase name='CreateStemmedDic'/>\n" if $self->{'gli'}; 517 530 if (!-e "$mg_invf_dict_exe") { … … 519 532 die "mgbuilder::build_index - couldn't run $mg_invf_dict_exe\n"; 520 533 } 521 my $invdict_status = system ( "mg_invf_dict$exe -f \"$fullindexprefix\" $osextra");534 my $invdict_status = system ($invdict_cmd); 522 535 # check that it worked - if not, don't carry on 523 536 if ($invdict_status !=0) { … … 550 563 # same as above: generate all the stem idexes. But don't bother stemming if 551 564 # casefolding failed, and don't try generating indexes for both if stemming failed 552 my $stem_index_status = system ("mg_stem_idx$exe -b 4096 -s1 -f \"$fullindexprefix\" $osextra"); 565 my $stemindex_cmd = "mg_stem_idx$exe -b 4096 -s1 -f \"$fullindexprefix\" $osextra"; 566 print $outhandle "\ncmd: $stemindex_cmd\n" if ($self->{'verbosity'} >= 4); 567 568 my $stem_index_status = system ($stemindex_cmd); 553 569 if($stem_index_status != 0) { 554 570 print $outhandle "\nCase folding failed: mg_stem_idx exit status $stem_index_status\n" if ($self->{'verbosity'} >= 4); 555 571 } else { 556 $stem_index_status = system ("mg_stem_idx$exe -b 4096 -s2 -f \"$fullindexprefix\" $osextra"); 572 $stemindex_cmd = "mg_stem_idx$exe -b 4096 -s2 -f \"$fullindexprefix\" $osextra"; 573 print $outhandle "\ncmd: $stemindex_cmd\n" if ($self->{'verbosity'} >= 4); 574 $stem_index_status = system ($stemindex_cmd); 557 575 558 576 if($stem_index_status != 0) { 559 577 print $outhandle "\nStemming failed: mg_stem_idx exit status $stem_index_status\n" if ($self->{'verbosity'} >= 4); 560 578 } else { 561 $stem_index_status = system ("mg_stem_idx$exe -b 4096 -s3 -f \"$fullindexprefix\" $osextra"); 579 $stemindex_cmd = "mg_stem_idx$exe -b 4096 -s3 -f \"$fullindexprefix\" $osextra"; 580 print $outhandle "\ncmd: $stemindex_cmd\n" if ($self->{'verbosity'} >= 4); 581 $stem_index_status = system ($stemindex_cmd); 562 582 563 583 if($stem_index_status != 0) { … … 596 616 my $collect_tail = &util::get_dirsep_tail($self->{'collection'}); 597 617 my $input_file = &FileUtils::filenameConcatenate("text", $collect_tail); 598 if (!-e "$mgstat_exe" || !open (PIPEIN, "mgstat$exe -d \"$self->{'build_dir'}\" -f \"$input_file\" |")) { 599 my $outhandle = $self->{'outhandle'}; 618 619 my $mgstat_cmd = "mgstat$exe -d \"$self->{'build_dir'}\" -f \"$input_file\""; 620 my $outhandle = $self->{'outhandle'}; 621 print $outhandle "\ncmd: $mgstat_cmd\n" if ($self->{'verbosity'} >= 4); 622 if (!-e "$mgstat_exe" || !open (PIPEIN, "$mgstat_cmd |")) { 623 600 624 print $outhandle "Warning: Couldn't open pipe to $mgstat_exe to get additional stats\n"; 601 625 } else { -
main/trunk/greenstone2/perllib/mgppbuilder.pm
r28375 r29328 208 208 } 209 209 else { 210 if (!-e "$mgpp_passes_exe" || 211 !open($handle, "| mgpp_passes$exe -M $maxnumeric $mgpp_passes_sections -f \"$fulltextprefix\" -T1 $osextra")) { 210 my $mgpp_passes_cmd = "mgpp_passes$exe -M $maxnumeric $mgpp_passes_sections -f \"$fulltextprefix\" -T1 $osextra"; 211 print $outhandle "\ncmd: $mgpp_passes_cmd\n" if ($self->{'verbosity'} >= 4); 212 if (!-e "$mgpp_passes_exe" || !open($handle, "| $mgpp_passes_cmd")) { 212 213 print STDERR "<FatalError name='NoRunMGPasses'>\n</Stage>\n" if $self->{'gli'}; 213 214 die "mgppbuilder::compress_text - couldn't run $mgpp_passes_exe\n"; … … 242 243 # note: these options are left over from mg version 243 244 if (!$self->{'debug'}) { 245 my $compdict_cmd = "mgpp_compression_dict$exe -f \"$fulltextprefix\" -S -H -2 -k 5120 $osextra"; 244 246 print $outhandle "\n creating the compression dictionary\n" if ($self->{'verbosity'} >= 1); 247 print $outhandle "\ncmd: $compdict_cmd\n" if ($self->{'verbosity'} >= 4); 245 248 print STDERR "<Phase name='CreatingCompress'/>\n" if $self->{'gli'}; 246 249 if (!-e "$mgpp_compression_dict_exe") { … … 248 251 die "mgppbuilder::compress_text - couldn't run $mgpp_compression_dict_exe\n"; 249 252 } 250 my $comp_dict_status = system ( "mgpp_compression_dict$exe -f \"$fulltextprefix\" -S -H -2 -k 5120 $osextra");253 my $comp_dict_status = system ($compdict_cmd); 251 254 if($comp_dict_status != 0) { 252 255 print $outhandle "\nmgppbuilder::compress_text - Warning: there's no compressed text\n"; … … 257 260 258 261 if (!$self->{'debug'}) { 259 if (!-e "$mgpp_passes_exe" || 260 !open ($handle, "| mgpp_passes$exe -M $maxnumeric $mgpp_passes_sections -f \"$fulltextprefix\" -T2 $osextra")) { 262 my $mgpp_passes_cmd = "mgpp_passes$exe -M $maxnumeric $mgpp_passes_sections -f \"$fulltextprefix\" -T2 $osextra"; 263 print $outhandle "\ncmd: $mgpp_passes_cmd\n" if ($self->{'verbosity'} >= 4); 264 265 if (!-e "$mgpp_passes_exe" || !open ($handle, "| $mgpp_passes_cmd")) { 261 266 print STDERR "<FatalError name='NoRunMGPasses'/>\n</Stage>\n" if $self->{'gli'}; 262 267 die "mgppbuilder::compress_text - couldn't run $mgpp_passes_exe\n"; … … 474 479 } 475 480 else { 476 if (!-e "$mgpp_passes_exe" || 477 !open($handle, "| mgpp_passes$exe -M $maxnumeric $mgpp_passes_sections -f \"$fullindexprefix\" -I1 $osextra")) { 481 my $mgpp_passes_cmd = "mgpp_passes$exe -M $maxnumeric $mgpp_passes_sections -f \"$fullindexprefix\" -I1 $osextra"; 482 print $outhandle "\ncmd: $mgpp_passes_cmd\n" if ($self->{'verbosity'} >= 4); 483 if (!-e "$mgpp_passes_exe" || !open($handle, "| $mgpp_passes_cmd")) { 478 484 print STDERR "<FatalError name='NoRunMGPasses'/>\n</Stage>\n" if $self->{'gli'}; 479 485 die "mgppbuilder::build_index - couldn't run $mgpp_passes_exe\n"; … … 530 536 531 537 } 532 533 if (!-e "$mgpp_passes_exe" || 534 !open ($handle, "| mgpp_passes$exe -M $maxnumeric $mgpp_passes_sections -f \"$fullindexprefix\" -I2 $osextra")) { 538 539 my $mgpp_passes_cmd = "mgpp_passes$exe -M $maxnumeric $mgpp_passes_sections -f \"$fullindexprefix\" -I2 $osextra"; 540 print $outhandle "\ncmd: $mgpp_passes_cmd\n" if ($self->{'verbosity'} >= 4); 541 if (!-e "$mgpp_passes_exe" || !open ($handle, "| $mgpp_passes_cmd")) { 535 542 print STDERR "<FatalError name='NoRunMGPasses'/>\n</Stage>\n" if $self->{'gli'}; 536 543 die "mgppbuilder::build_index - couldn't run $mgpp_passes_exe\n"; … … 581 588 die "mgppbuilder::build_index - couldn't run $mgpp_invf_dict_exe\n"; 582 589 } 583 my $invdict_status = system ("mgpp_invf_dict$exe -f \"$fullindexprefix\" $osextra" ); 590 my $invdict_cmd = "mgpp_invf_dict$exe -f \"$fullindexprefix\" $osextra"; 591 print $outhandle "\ncmd: $invdict_cmd\n" if ($self->{'verbosity'} >= 4); 592 my $invdict_status = system ($invdict_cmd); 584 593 # check that it worked - if not, don't carry on 585 594 if ($invdict_status !=0) { … … 600 609 my $accent_folding_enabled = 1; 601 610 if ($self->{'accentfold'}) { 611 my $accentfold_cmd = "mgpp_stem_idx$exe -b 4096 -s4 -f \"$fullindexprefix\" $osextra"; 612 print $outhandle "\ncmd: $accentfold_cmd\n" if ($self->{'verbosity'} >= 4); 602 613 # the first time we do this, we test for accent folding enabled 603 my $accent_status = system ( "mgpp_stem_idx$exe -b 4096 -s4 -f \"$fullindexprefix\" $osextra");614 my $accent_status = system ($accentfold_cmd); 604 615 if ($accent_status == 2) { 605 616 # accent folding has not been enabled in mgpp … … 614 625 } 615 626 if ($self->{'casefold'}) { 616 my $casefold_status = system ("mgpp_stem_idx$exe -b 4096 -s1 -f \"$fullindexprefix\" $osextra"); 627 my $casefold_cmd = "mgpp_stem_idx$exe -b 4096 -s1 -f \"$fullindexprefix\" $osextra"; 628 print $outhandle "\ncmd: $casefold_cmd\n" if ($self->{'verbosity'} >= 4); 629 my $casefold_status = system ($casefold_cmd); 617 630 if ($casefold_status != 0) { 618 631 print $outhandle "\nCase folding failed: mgpp_stem_idx exit status $casefold_status\n" if ($self->{'verbosity'} >= 4); … … 622 635 623 636 elsif ($accent_folding_enabled && $self->{'accentfold'}) { 624 my $status = system ("mgpp_stem_idx$exe -b 4096 -s5 -f \"$fullindexprefix\" $osextra"); 637 my $accent_casefold_cmd = "mgpp_stem_idx$exe -b 4096 -s5 -f \"$fullindexprefix\" $osextra"; 638 print $outhandle "\ncmd: $accent_casefold_cmd\n" if ($self->{'verbosity'} >= 4); 639 my $status = system ($accent_casefold_cmd); 625 640 if($status != 0) { 626 641 print $outhandle "\nAccent folding (with casefolding) failed: mgpp_stem_idx exit status $status\n" if ($self->{'verbosity'} >= 4); … … 631 646 } 632 647 if ($self->{'stem'}) { 633 my $stem_status = system ("mgpp_stem_idx$exe -b 4096 -s2 -f \"$fullindexprefix\" $osextra"); 648 my $stem_cmd = "mgpp_stem_idx$exe -b 4096 -s2 -f \"$fullindexprefix\" $osextra"; 649 print $outhandle "\ncmd: $stem_cmd\n" if ($self->{'verbosity'} >= 4); 650 my $stem_status = system ($stem_cmd); 634 651 if ($stem_status != 0) { 635 652 print $outhandle "\nStemming failed: mgpp_stem_idx exit status $stem_status\n" if ($self->{'verbosity'} >= 4); … … 638 655 } 639 656 elsif ($accent_folding_enabled && $self->{'accentfold'}) { 640 my $status = system ("mgpp_stem_idx$exe -b 4096 -s6 -f \"$fullindexprefix\" $osextra"); 657 my $accent_stem_cmd = "mgpp_stem_idx$exe -b 4096 -s6 -f \"$fullindexprefix\" $osextra"; 658 print $outhandle "\ncmd: $accent_stem_cmd\n" if ($self->{'verbosity'} >= 4); 659 my $status = system ($accent_stem_cmd); 641 660 if($status != 0) { 642 661 print $outhandle "\nAccent folding (with stemming) failed: mgpp_stem_idx exit status $status\n" if ($self->{'verbosity'} >= 4); … … 647 666 } 648 667 if ($self->{'casefold'} && $self->{'stem'}) { 649 my $case_and_stem_status = system ("mgpp_stem_idx$exe -b 4096 -s3 -f \"$fullindexprefix\" $osextra"); 668 my $case_stem_cmd = "mgpp_stem_idx$exe -b 4096 -s3 -f \"$fullindexprefix\" $osextra"; 669 print $outhandle "\ncmd: $case_stem_cmd\n" if ($self->{'verbosity'} >= 4); 670 my $case_and_stem_status = system ($case_stem_cmd); 650 671 if ($case_and_stem_status != 0) { 651 672 print $outhandle "\nCasefolding and stemming failed: mgpp_stem_idx exit status $case_and_stem_status\n" if ($self->{'verbosity'} >= 4); … … 655 676 } 656 677 elsif ($accent_folding_enabled && $self->{'accentfold'}) { 657 my $status = system ("mgpp_stem_idx$exe -b 4096 -s7 -f \"$fullindexprefix\" $osextra"); 678 my $accent_case_stem_cmd = "mgpp_stem_idx$exe -b 4096 -s7 -f \"$fullindexprefix\" $osextra"; 679 print $outhandle "\ncmd: $accent_case_stem_cmd\n" if ($self->{'verbosity'} >= 4); 680 my $status = system ($accent_case_stem_cmd); 658 681 if($status != 0) { 659 682 print $outhandle "\nAccent folding (with stemming and casefolding) failed: mgpp_stem_idx exit status $status\n" if ($self->{'verbosity'} >= 4);
Note:
See TracChangeset
for help on using the changeset viewer.