Ignore:
Timestamp:
2001-10-09T17:21:29+13:00 (23 years ago)
Author:
sjboddie
Message:

The build process now creates a summary of how many files were included,
which were rejected, etc. A link to a page containing this summary is
provided from the final page of the collector (once the collection is built
successfully) and from the default "about this collection" text for
collections built by the collector.

Also did a little bit of tidying in a couple of places

Location:
trunk/gsdl/perllib/plugins
Files:
7 edited

Legend:

Unmodified
Added
Removed

  • trunk/gsdl/perllib/plugins/BasPlug.pm

    r2755 r2785  
    142142
    143143    $self->{'textcat'} = new textcat();
     144
     145    $self->{'num_processed'} = 0;
     146    $self->{'num_not_processed'} = 0;
     147    $self->{'num_blocked'} = 0;
     148    $self->{'num_archives'} = 0;
    144149   
    145150    # general options available to all plugins
    … …  
    175180sub init {
    176181    my $self = shift (@_);
    177     my ($verbosity, $outhandle, $faillogname) = @_;
     182    my ($verbosity, $outhandle, $failhandle) = @_;
    178183
    179184    # verbosity is passed through from the processor
    180185    $self->{'verbosity'} = $verbosity;
    181186
    182     # as is the outhandle ...
     187    # as are the outhandle and failhandle
    183188    $self->{'outhandle'} = $outhandle if defined $outhandle;
    184     $self->{'faillogname'} = $faillogname;
     189    $self->{'failhandle'} = $failhandle;
    185190
    186191    # set process_exp and block_exp to defaults unless they were
    … …  
    258263
    259264    my $filename = &util::filename_cat($base_dir, $file);
    260     return 0 if $self->{'block_exp'} ne "" && $filename =~ /$self->{'block_exp'}/;
     265    if ($self->{'block_exp'} ne "" && $filename =~ /$self->{'block_exp'}/) {
     266    $self->{'num_blocked'} ++;
     267    return 0;
     268    }
    261269    if ($filename !~ /$self->{'process_exp'}/ || !-f $filename) {
    262270    return undef;
    … …  
    298306    if (!length ($text)) {
    299307    print $outhandle "$plugin_name: ERROR: $file contains no text\n" if $self->{'verbosity'};
     308
     309    my $failhandle = $self->{'failhandle'};
     310    print $failhandle "$file: " . ref($self) . ": file contains no text\n";
     311    $self->{'num_not_processed'} ++;
     312
    300313    return 0;
    301314    }
    … …  
    316329    # process the document
    317330    $processor->process($doc_obj);
     331
     332    $self->{'num_processed'} ++;
    318333
    319334    return 1; # processed the file
    … …  
    671686}
    672687
     688sub compile_stats {
     689    my $self = shift(@_);
     690    my ($stats) = @_;
     691
     692    $stats->{'num_processed'} += $self->{'num_processed'};
     693    $stats->{'num_not_processed'} += $self->{'num_not_processed'};
     694
     695}
     696
     697
    6736981;

  • trunk/gsdl/perllib/plugins/ConvertToPlug.pm

    r2755 r2785  
    4848#    @ISA = ('BasPlug'); #, 'HTMLPlug', 'TEXTPlug');
    4949}
    50 
    51 # use strict;  # this breaks 'print $outhandle ' error msgs.
    5250
    5351sub print_usage {
    … …  
    146144    my $outhandle = $self->{'outhandle'};
    147145    my $convert_to = $self->{'convert_to'};
     146    my $failhandle = $self->{'failhandle'};
    148147
    149148    # softlink to collection tmp dir
    … …  
    184183    if ($output_type eq "fail") {
    185184    print $outhandle "Could not convert $tailname$suffix to $convert_to format\n";
    186     if ($self->{'faillogname'} ne "" && -s "$errlog") {
    187         open(SAVELOG, ">>$self->{'faillogname'}");
     185    print $failhandle "$tailname$suffix: " . ref($self) . " failed to convert to $convert_to\n";
     186    $self->{'num_not_processed'} ++;
     187    if (-s "$errlog") {
    188188        open(ERRLOG, "$errlog");
    189         print SAVELOG "$tailname$suffix (converting to $convert_to) failed:\n";
    190189        while (<ERRLOG>) {
    191         print SAVELOG "$_";
     190        print $outhandle "$_";
    192191        }
     192        print $outhandle "\n";
    193193        close ERRLOG;
    194         print SAVELOG "\n";
    195         close SAVELOG;
    196194    }
    197195    &util::rm("$errlog") if (-e "$errlog");
    … …  
    236234
    237235    my $filename = &util::filename_cat($base_dir, $file);
    238     return 0 if $self->{'block_exp'} ne "" && $filename =~ /$self->{'block_exp'}/;
     236    if ($self->{'block_exp'} ne "" && $filename =~ /$self->{'block_exp'}/) {
     237    $self->{'num_blocked'} ++;
     238    return 0;
     239    }
    239240    if ($filename !~ /$self->{'process_exp'}/ || !-f $filename) {
    240241        return undef;
    … …  
    299300    $self->cleanup_tmp_area();
    300301
     302    $self->{'num_processed'} ++;
    301303
    302304    return 1;

  • trunk/gsdl/perllib/plugins/PDFPlug.pm

    r2661 r2785  
    6363
    6464    my $outhandle = $self->{'outhandle'};
    65     print $outhandle "PDFPlug: passing $_[3] onto $self->{'convert_to'} Plug\n"
     65    print $outhandle "PDFPlug: passing $_[3] on to $self->{'convert_to'}Plug\n"
    6666    if $self->{'verbosity'} > 1;
    6767   
    … …  
    6969}
    7070
    71 
    72711;
    73 
    74 
    75 
    76 

  • trunk/gsdl/perllib/plugins/PSPlug.pm

    r2451 r2785  
    3333}
    3434
    35 use strict;
    36 
    37 
    3835sub new {
    3936    my $class = shift (@_);
    … …  
    6562    print STDERR "using the PRESCRIPT package, which is available for download at\n   http://www.nzdl.org/html/software.html\n\n";
    6663}
    67 
    6864
    6965sub get_default_block_exp {
    … …  
    147143}
    148144
    149 
    150145# do plugin specific processing of doc_obj for HTML type
    151146sub process {
    … …  
    153148    my ($trash, $trash2, $path, $file, $trash3, $doc) = @_;
    154149
    155     print STDERR "PSPlug: passing $_[3] onto $self->{'convert_to'} Plug\n"
     150    my $outhandle = $self->{'outhandle'};
     151    print $outhandle "PSPlug: passing $_[3] on to $self->{'convert_to'}Plug\n"
    156152    if $self->{'verbosity'} > 1;
    157153   

  • trunk/gsdl/perllib/plugins/RTFPlug.pm

    r2564 r2785  
    3333}
    3434
    35 use strict;
    36 
    3735sub get_default_process_exp {
    3836    my $self = shift (@_);
    … …  
    4240sub process {
    4341    my $self = shift (@_);
    44     print STDERR "RTFPlug: passing $_[3] onto $self->{'convert_to'} Plug\n"
     42
     43    my $outhandle = $self->{'outhandle'};
     44    print $outhandle "RTFPlug: passing $_[3] on to $self->{'convert_to'}Plug\n"
    4545    if $self->{'verbosity'} > 1;
     46
    4647    return ConvertToPlug::process_type($self,"rtf",@_);
    4748}

  • trunk/gsdl/perllib/plugins/WordPlug.pm

    r2515 r2785  
    4545
    4646    my $outhandle = $self->{'outhandle'};   
    47     print $outhandle "WordPlug: passing $_[3] onto $self->{'convert_to'} Plug\n"
     47    print $outhandle "WordPlug: passing $_[3] on to $self->{'convert_to'}Plug\n"
    4848    if $self->{'verbosity'} > 1;
    4949   
    … …  
    5151}
    5252
    53 
    54531;
    55 

  • trunk/gsdl/perllib/plugins/ZIPPlug.pm

    r2082 r2785  
    119119    my $numdocs = &plugin::read ($pluginfo, "", $tmpdir, $metadata, $processor, $maxdocs);
    120120    &util::rm_r ($tmpdir);
     121
     122    $self->{'num_archives'} ++;
     123
    121124    return $numdocs;
    122125
Note: See TracChangeset for help on using the changeset viewer.