Changeset 6332


Ignore:
Timestamp:
2003-12-19T15:16:29+13:00 (20 years ago)
Author:
jmt12
Message:

When -gli argument is provided to calling script these modules will now output gli specific, non-language nor verbosity specific, messages

Location:
trunk/gsdl/perllib
Files:
25 edited

Legend:

Unmodified
Added
Removed

  • trunk/gsdl/perllib/classify.pm

    r5682 r6332  
    115115# to the gdbm
    116116sub output_classify_info {
    117     my ($classifiers, $handle, $allclassifications) = @_;
     117    my ($classifiers, $handle, $allclassifications, $gli) = @_;
    118118#    $handle = "main::STDOUT";
     119
     120    $gli = 0 unless defined $gli;
    119121
    120122    # create a classification containing all the info
    … …  
    124126    # get each of the classifications
    125127    foreach $classobj (@$classifiers) {
    126     my $tempinfo = $classobj->get_classify_info();
     128    my $tempinfo = $classobj->get_classify_info($gli);
    127129    $tempinfo->{'classifyOID'} = "CL$next_classify_num";
    128130    $next_classify_num++;

  • trunk/gsdl/perllib/mgbuilder.pm

    r5768 r6332  
    6262    my ($class, $collection, $source_dir, $build_dir, $verbosity,
    6363    $maxdocs, $debug, $keepold, $allclassifications,
    64     $outhandle, $no_text, $failhandle) = @_;
     64    $outhandle, $no_text, $failhandle, $gli) = @_;
    6565
    6666    $outhandle = STDERR unless defined $outhandle;
    … …  
    8080              'no_text'=>$no_text,
    8181              'failhandle'=>$failhandle,
    82               'notbuilt'=>{}    # indexes not built
     82              'notbuilt'=>{},    # indexes not built
     83              'gli'=>$gli
    8384              }, $class;
    8485
     86    $self->{'gli'} = 0 unless defined $self->{'gli'};
    8587
    8688    # read in the collection configuration file
    … …  
    233235
    234236    print $outhandle "\n*** creating the compressed text\n" if ($self->{'verbosity'} >= 1);
     237    print STDERR "<Stage name='CompressText'>\n" if $self->{'gli'};
    235238
    236239    # collect the statistics for the text
    237240    # -b $maxdocsize sets the maximum document size to be 12 meg
    238241    print $outhandle "\n    collecting text statistics\n"  if ($self->{'verbosity'} >= 1);
     242    print STDERR "<Phase name='CollectTextStats'>\n" if $self->{'gli'};
    239243
    240244    my ($handle);
    … …  
    276280    if (!$self->{'debug'}) {
    277281    print $outhandle "\n    creating the compression dictionary\n"  if ($self->{'verbosity'} >= 1);
     282    print STDERR "<Phase name='CreatingCompress'>\n" if $self->{'gli'};
    278283    if (!-e "$mg_compression_dict_exe") {
    279284        die "mgbuilder::compress_text - couldn't run $mg_compression_dict_exe\n";
    … …  
    287292    }
    288293    }
     294    else {
     295    print STDERR "<Phase name='SkipCreatingComp'>\n" if $self->{'gli'};
     296    }
    289297
    290298    $self->{'buildproc'}->reset();
    291299    # compress the text
    292300    print $outhandle "\n    compressing the text\n"  if ($self->{'verbosity'} >= 1);
     301    print STDERR "<Phase name='CompressingText'>\n" if $self->{'gli'};
     302
    293303    &plugin::read ($self->{'pluginfo'}, $self->{'source_dir'},
    294304           "", {}, $self->{'buildproc'}, $self->{'maxdocs'});
    … …  
    336346        print $outhandle "\n*** building index $index in subdirectory " .
    337347        "$self->{'index_mapping'}->{$index}\n" if ($self->{'verbosity'} >= 1);
     348        print STDERR "<Phase name='Index' source='$index'>\n" if $self->{'gli'};
    338349        $self->build_index($index);
    339350    } else {
    … …  
    558569    # Build index dictionary. Uses verbatim stem method
    559570    print $outhandle "\n    creating index dictionary\n"  if ($self->{'verbosity'} >= 1);
     571    print STDERR "<Stage name='CreatingIndexDic'>\n" if $self->{'gli'};
    560572    my ($handle);
    561573    if ($self->{'debug'}) {
    … …  
    608620    # invert the text
    609621    print $outhandle "\n    inverting the text\n"  if ($self->{'verbosity'} >= 1);
    610 
     622    print STDERR "<Stage name='InvertingText'>\n" if $self->{'gli'};
    611623    $self->{'buildproc'}->reset();
    612624    &plugin::read ($self->{'pluginfo'}, $self->{'source_dir'},
    … …  
    621633    # create the weights file
    622634    print $outhandle "\n    create the weights file\n"  if ($self->{'verbosity'} >= 1);
     635    print STDERR "<Stage name='CreateTheWeights'>\n" if $self->{'gli'};
    623636    if (!-e "$mg_weights_build_exe") {
    624637        die "mgbuilder::build_index - couldn't run $mg_weights_build_exe\n";
    … …  
    628641    # create 'on-disk' stemmed dictionary
    629642    print $outhandle "\n    creating 'on-disk' stemmed dictionary\n"  if ($self->{'verbosity'} >= 1);
     643    print STDERR "<Stage name='CreateStemmedDic'>\n" if $self->{'gli'};
    630644    if (!-e "$mg_invf_dict_exe") {
    631645        die "mgbuilder::build_index - couldn't run $mg_invf_dict_exe\n";
    … …  
    636650    # creates stem index files for the various stemming methods
    637651    print $outhandle "\n    creating stem indexes\n"  if ($self->{'verbosity'} >= 1);
     652    print STDERR "<Stage name='CreatingStemIndx'>\n" if $self->{'gli'};
    638653    if (!-e "$mg_stem_idx_exe") {
    639654        die "mgbuilder::build_index - couldn't run $mg_stem_idx_exe\n";
    … …  
    681696    print $outhandle "\n*** creating the info database and processing associated files\n"
    682697    if ($self->{'verbosity'} >= 1);
     698    print STDERR "<Phase name='CreateInfoData'>\n" if $self->{'gli'};
    683699
    684700    # init all the classifiers
    … …  
    774790    # output classification information
    775791    &classify::output_classify_info ($self->{'classifiers'}, $handle,
    776                      $self->{'allclassifications'});
    777 
     792                     $self->{'allclassifications'},
     793                     $self->{'gli'});
    778794
    779795
    … …  
    803819
    804820    print $outhandle "\n*** creating auxiliary files \n" if ($self->{'verbosity'} >= 1);
     821    print STDERR "<Phase name='CreatingAuxilary'>\n" if $self->{'gli'};
    805822
    806823    # get the text directory

  • trunk/gsdl/perllib/plugin.pm

    r5682 r6332  
    9292
    9393sub read {
    94     my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $aux) = @_;
     94    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $gli, $aux) = @_;
    9595
    9696    $maxdocs = -1 unless defined $maxdocs && $maxdocs =~ /\d/;
     97    $gli = 0 unless defined $gli;
     98
    9799    my $rv = 0;
     100
     101    # Announce to GLI that we are handling a file
     102    print STDERR "<File n='$file'>\n" if $gli;
    98103
    99104    # the .kill file is a handy (if not very elegant) way of aborting
    … …  
    109114    foreach $plugobj (@$pluginfo) {
    110115    $rv = $plugobj->read($pluginfo, $base_dir, $file,
    111                  $metadata, $processor, $maxdocs, $aux);
     116                 $metadata, $processor, $maxdocs, $gli, $aux);
    112117    return $rv if defined $rv;
    113118    }
    … …  
    130135# are only output after import.pl -
    131136sub write_stats {
    132     my ($pluginfo, $statshandle, $faillog) = @_;
     137    my ($pluginfo, $statshandle, $faillog, $gli) = @_;
     138
     139    $gli = 0 unless defined $gli;
    133140
    134141    foreach $plugobj (@$pluginfo) {
    … …  
    138145    my $total = $stats->{'num_processed'} + $stats->{'num_blocked'} +
    139146    $stats->{'num_not_processed'};
     147
     148    print STDERR "<ImportComplete c='$stats->{'num_processed'}' p='$stats->{'num_processed'}'>\n" if $gli;
    140149
    141150    if ($total == 1) {

  • trunk/gsdl/perllib/plugins/ArcPlug.pm

    r5680 r6332  
    8585
    8686    # found an archives.inf file
    87     &gsprintf($outhandle, "ArcPlug: {common.processing} $archive_info_filename\n");
     87    &gsprintf($outhandle, "ArcPlug: {common.processing} $archive_info_filename\n") if $self->{'verbosity'} > 1;
    8888
    8989    # read in the archives information file

  • trunk/gsdl/perllib/plugins/BasPlug.pm

    r5924 r6332  
    7777    'desc' => "{BasPlug.default_encoding}",
    7878    'type' => "enum",
     79    'list' => $unicode_list,
    7980    'reqd' => "no",
    8081        'deft' => "utf8" },
    … …  
    490491    my $self = shift (@_); 
    491492 
    492     my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs) = @_;
     493    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $gli) = @_;
    493494
    494495    if ($self->is_recursive()) {
    … …  
    547548
    548549    # do plugin specific processing of doc_obj
    549     return undef unless defined ($self->process (\$text, $pluginfo, $base_dir, $file, $metadata, $doc_obj));
     550    return undef unless defined ($self->process (\$text, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli));
    550551   
    551552    # do any automatic metadata extraction

  • trunk/gsdl/perllib/plugins/BibTexPlug.pm

    r5924 r6332  
    8989sub process {
    9090    my $self = shift (@_);
    91     my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj) = @_;
     91    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
    9292    my $outhandle = $self->{'outhandle'};
    9393
    … …  
    101101
    102102    # Report that we're processing the file
     103    print STDERR "<Processing n='$file' p='BibTexPlug'>\n" if ($gli);
    103104    print $outhandle "BibTexPlug: processing $file\n"
    104105    if ($self->{'verbosity'}) > 1;

  • trunk/gsdl/perllib/plugins/BookPlug.pm

    r5924 r6332  
    103103sub process {
    104104    my $self = shift (@_);
    105     my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj) = @_;
     105    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
    106106    my $outhandle = $self->{'outhandle'};
    107107
     108    print STDERR "<Processing n='$file' p='BookPlug'>\n" if ($gli);
    108109    print $outhandle "BookPlug: processing $file\n"
    109110    if $self->{'verbosity'} > 1;

  • trunk/gsdl/perllib/plugins/EMAILPlug.pm

    r6079 r6332  
    159159
    160160    my $self = shift (@_);
    161     my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj) = @_;
     161    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
    162162    my $outhandle = $self->{'outhandle'};
    163163
    … …  
    172172    }
    173173
     174
     175    print STDERR "<Processing n='$file' p='EMAILPlug'>\n" if ($gli);
    174176
    175177    print $outhandle "EMAILPlug: processing $file\n"

  • trunk/gsdl/perllib/plugins/FOXPlug.pm

    r5924 r6332  
    7272sub read {
    7373    my $self = shift (@_);
    74     my ($pluginfo, $base_dir, $file, $metadata, $processor) = @_;
     74    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $gli) = @_;
    7575    my $fullname = &util::filename_cat ($base_dir, $file);
    7676
    … …  
    8888    return undef;
    8989    }
     90
     91    print STDERR "<Processing n='$file' p='FOXPlug'>\n" if ($gli);
    9092   
    9193    print STDERR "FOXPlug: processing $file\n";

  • trunk/gsdl/perllib/plugins/GAPlug.pm

    r5680 r6332  
    7575
    7676    my $outhandle = $self->{'outhandle'};
    77     print $outhandle "GAPLug: processing $self->{'file'}\n";
     77    print $outhandle "GAPLug: processing $self->{'file'}\n" if $self->{'verbosity'} > 1;
    7878}
    7979

  • trunk/gsdl/perllib/plugins/GMLPlug.pm

    r5680 r6332  
    6464sub read {
    6565    my $self = shift (@_);
    66     my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs) = @_;
     66    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $gli) = @_;
    6767    my $outhandle = $self->{'outhandle'};
    6868
    … …  
    7676    $file =~ s/^[\/\\]+//; # $file often begins with / so we'll tidy it up
    7777
     78    print STDERR "<Processing n='$file' p='GMLPlug'>\n" if ($gli);
    7879    print $outhandle "GMLPlug: processing $file\n";
    7980

  • trunk/gsdl/perllib/plugins/HBPlug.pm

    r5924 r6332  
    227227sub read {
    228228    my $self = shift (@_);
    229     my ($pluginfo, $base_dir, $file, $metadata, $processor) = @_;
     229    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $gli) = @_;
    230230    my $outhandle = $self->{'outhandle'};
    231231
    … …  
    239239    return undef unless -e $htmlfile;
    240240
     241    print STDERR "<Processing n='$file' p='HBPlug'>\n" if ($gli);
    241242    print $outhandle "HBPlug: processing $file\n";
    242243

  • trunk/gsdl/perllib/plugins/HTMLPlug.pm

    r5924 r6332  
    194194sub process {
    195195    my $self = shift (@_);
    196     my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj) = @_;
     196    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
    197197    my $outhandle = $self->{'outhandle'};
     198
     199    print STDERR "<Processing n='$file' p='HTMLPlug'>\n" if ($gli);
     200
    198201    print $outhandle "HTMLPlug: processing $file\n"
    199202    if $self->{'verbosity'} > 1;
    … …  
    648651        if (defined $1) {
    649652            my $title = $1;
     653            # Arg. This allows only ascii value characters in titles
    650654            if ($title =~ /\w/) {
    651655            $title =~ s/<[^>]*>/ /g;

  • trunk/gsdl/perllib/plugins/ISISPlug.pm

    r6138 r6332  
    135135{
    136136    my $self = shift (@_);
    137     my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj) = @_;
     137    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
    138138    my $outhandle = $self->{'outhandle'};
    139139
    … …  
    142142
    143143    # Report that we're processing the file
     144    print STDERR "<Processing n='$file' p='ISISPlug'>\n" if ($gli);
    144145    print $outhandle "IsisPlug: processing $file\n"
    145146    if ($self->{'verbosity'}) > 1;

  • trunk/gsdl/perllib/plugins/ImagePlug.pm

    r5924 r6332  
    364364sub read {
    365365    my $self = shift (@_);
    366     my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs) = @_;
     366    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $gli) = @_;
    367367
    368368    my $outhandle = $self->{'outhandle'};
    … …  
    373373    return undef;
    374374    }
     375
     376    print STDERR "<Processing n='$file' p='ImagePlug'>\n" if ($gli);
    375377    print $outhandle "ImagePlug processing \"$filename\"\n"
    376378        if $self->{'verbosity'} > 1;

  • trunk/gsdl/perllib/plugins/IndexPlug.pm

    r5919 r6332  
    9090sub read {
    9191    my $self = shift (@_);
    92     my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs) = @_;
     92    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $gli) = @_;
    9393    my $outhandle = $self->{'outhandle'};
    9494
    … …  
    100100
    101101    # found an index.txt file
     102    print STDERR "<Processing n='$file' p='IndexPlug'>\n" if ($gli);
    102103    print $outhandle "IndexPlug: processing $indexfile\n";
    103104

  • trunk/gsdl/perllib/plugins/MACROPlug.pm

    r5680 r6332  
    141141sub process {
    142142    my $self = shift (@_);
    143     my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj) = @_;
     143    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
    144144    my $outhandle = $self->{'outhandle'};
    145145
     146    print STDERR "<Processing n='$file' p='MACROPlug'>\n" if ($gli);
    146147    print $outhandle "MACROPlug: processing $file\n"
    147148    if $self->{'verbosity'} > 1;

  • trunk/gsdl/perllib/plugins/MARCPlug.pm

    r5924 r6332  
    5252require MARC::Record; 
    5353require MARC::Batch; 
     54#use MARC::Record; 
     55#use MARC::Batch;
    5456
    5557#  sub print_usage {
    … …  
    211213sub process {
    212214    my $self = shift (@_);
    213     my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj) = @_;
     215    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
    214216    my $outhandle = $self->{'outhandle'};
    215217
    … …  
    219221    return undef;
    220222    }
     223
     224    print STDERR "<Processing n='$file' p='MARCPlug'>\n" if ($gli);
    221225    print $outhandle "MARCPlug: processing $file\n"
    222226    if $self->{'verbosity'} > 1;

  • trunk/gsdl/perllib/plugins/OAIPlug.pm

    r6132 r6332  
    162162sub process {
    163163    my $self = shift (@_);
    164     my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj) = @_;
     164    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
    165165    my $outhandle = $self->{'outhandle'};
    166166
     167    print STDERR "<Processing n='$file' p='OAIPlug'>\n" if ($gli);
    167168    print $outhandle "OAIPlug: processing $file\n"
    168169    if $self->{'verbosity'} > 1;

  • trunk/gsdl/perllib/plugins/RecPlug.pm

    r5680 r6332  
    146146    if (!parsargv::parse(\@_,
    147147             q^use_metadata_files^, \$self->{'use_metadata_files'},
    148              q^show_progress^, \$self->{'show_progress'}, # Undocumented (for GLI)
    149148             "allow_extra_options")) {
    150149    print STDERR "\nRecPlug uses an incorrect option.\n";
    … …  
    192191sub read {
    193192    my $self = shift (@_);
    194     my ($pluginfo, $base_dir, $file, $in_metadata, $processor, $maxdocs) = @_;
     193    my ($pluginfo, $base_dir, $file, $in_metadata, $processor, $maxdocs, $gli) = @_;
    195194   
    196195    my $outhandle = $self->{'outhandle'};
    … …  
    232231    my (@dir, $subfile);
    233232    my $count = 0;
     233   
    234234    print $outhandle "RecPlug: getting directory $dirname\n" if ($verbosity);
    235235   
    … …  
    308308    # Recursively read each $subfile
    309309    print $outhandle "RecPlug recurring: $subfile\n" if ($verbosity > 2);
    310     print $outhandle "RecPlug - $subfile\n" if ($self->{'show_progress'});
    311310
    312311    $count += &plugin::read ($pluginfo, $base_dir,
    313312                 &util::filename_cat($file, $subfile),
    314                  $out_metadata, $processor, $maxdocs);
     313                 $out_metadata, $processor, $maxdocs, $gli);
    315314    }
    316315    return $count;

  • trunk/gsdl/perllib/plugins/ReferPlug.pm

    r5924 r6332  
    110110sub process {
    111111    my $self = shift (@_);
    112     my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj) = @_;
     112    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
    113113    my $outhandle = $self->{'outhandle'};
    114114
    … …  
    117117
    118118    # Report that we're processing the file
     119    print STDERR "<Processing n='$file' p='ReferPlug'>\n" if ($gli);
    119120    print $outhandle "ReferPlug: processing $file\n"
    120121    if ($self->{'verbosity'}) > 1;

  • trunk/gsdl/perllib/plugins/RogPlug.pm

    r5924 r6332  
    211211sub read {
    212212    my $self = shift (@_);
    213     my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs) = @_;
     213    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $gli) = @_;
    214214
    215215    my $filename = &util::filename_cat($base_dir, $file);
    … …  
    219219    my $gz = (defined $3) ? 1: 0;
    220220
     221        print STDERR "<Processing n='$file' p='RogPlug'>\n" if ($gli);
    221222    print STDERR "RogPlug: processing $filename\n" if $processor->{'verbosity'};
    222223   

  • trunk/gsdl/perllib/plugins/SRCPlug.pm

    r5924 r6332  
    118118sub process {
    119119    my $self = shift (@_);
    120     my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj) = @_;
     120    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
    121121    my $outhandle = $self->{'outhandle'};
    122122   
     123        print STDERR "<Processing n='$file' p='SRCPlug'>\n" if ($gli);
    123124    print $outhandle "SRCPlug: processing $file\n"
    124125    if $self->{'verbosity'} > 1;

  • trunk/gsdl/perllib/plugins/TEXTPlug.pm

    r5924 r6332  
    9595sub process {
    9696    my $self = shift (@_);
    97     my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj) = @_;
     97    my ($textref, $pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
    9898    my $outhandle = $self->{'outhandle'};
    9999
     100    print STDERR "<Processing n='$file' p='TEXTPlug'>\n" if ($gli);
    100101    print $outhandle "TEXTPlug: processing $file\n"
    101102    if $self->{'verbosity'} > 1;

  • trunk/gsdl/perllib/plugins/UnknownPlug.pm

    r6214 r6332  
    170170sub read {
    171171    my $self = shift (@_);
    172     my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs) = @_;
     172    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs, $gli) = @_;
    173173
    174174    my $outhandle = $self->{'outhandle'};
    … …  
    180180    return undef;
    181181    }
     182    print STDERR "<Processing n='$file' p='UnknownPlug'>\n" if ($gli);
    182183    print $outhandle "UnknownPlug processing \"$filename\"\n"
    183184        if $self->{'verbosity'} > 1;
Note: See TracChangeset for help on using the changeset viewer.