Changeset 2785 for trunk/gsdl/bin/script


Ignore:
Timestamp:
2001-10-09T17:21:29+13:00 (23 years ago)
Author:
sjboddie
Message:

The build process now creates a summary of how many files were included,
which were rejected, etc. A link to a page containing this summary is
provided from the final page of the collector (once the collection is built
successfully) and from the default "about this collection" text for
collections built by the collector.

Also did a little bit of tidying in a couple of places

Location:
trunk/gsdl/bin/script
Files:
8 edited

Legend:

Unmodified
Added
Removed

  • trunk/gsdl/bin/script/build

    r2774 r2785  
    132132&main();
    133133
    134 close OUT if $use_out;
     134if ($use_out) {
     135    close OUT;
     136
     137    # if we've created a build log we'll copy it to the collection's etc
     138    # directory
     139    my ($etcdir);
     140    if ($dontinstall) {
     141    $etcdir = &util::filename_cat($collectdir, "etc", "build.log");
     142    } else {
     143    $etcdir = &util::filename_cat($ENV{'GSDLHOME'}, "collect", $collection, "etc", "build.log");
     144    }
     145   
     146    &util::cp($outfile, $etcdir);
     147}
    135148
    136149sub print_usage {
    … …  
    168181    print STDOUT "   -out                    Filename or handle to print output status to.\n";
    169182    print STDOUT "                           The default is STDERR\n";
     183    print STDOUT "   -statsfile name         Filename or handle to print import statistics to.\n";
     184    print STDOUT "                           The default is STDERR\n";
    170185    print STDOUT "   -make_writable          If set build will make the collection and any\n";
    171186    print STDOUT "                           temporary files it created globally writable after\n";
    … …  
    373388    $import_cmd .= " -removeold" unless $append;
    374389    $import_cmd .= " -collectdir \"$collectdir\"" if $collectdir =~ /\w/;
     390    $import_cmd .= " -statsfile \"$statsfile\"" if $statsfile =~ /\w/;
    375391    $import_cmd .= " $maxdocs $collection";
     392
     393    print STDERR "\n**import_cmd: $import_cmd\n";
     394
    376395    system ($import_cmd);
    377396    # if using output directory append the import output to it
    … …  
    558577             'email_events/.*/', \$email_events,
    559578             'mail_server/.*/', \$mail_server,
     579             'statsfile/.*/STDERR', \$statsfile,
    560580             'event_header/.*/', \$event_header)) {
    561581   

  • trunk/gsdl/bin/script/buildcol.pl

    r2524 r2785  
    7676    print STDOUT "   -no_strip_html        Do not strip the html tags from the indexed text\n";
    7777    print STDOUT "                         (only used for mgpp collections).\n\n";
     78    print STDOUT "   -faillog name         Fail log filename. This log receives the filenames\n";
     79    print STDOUT "                         of any files which fail to be processed (defaults.\n";
     80    print STDOUT "                         to " .
     81    &util::filename_cat("<collectdir>", "colname", "etc", "fail.log") . ")\n";
    7882    print STDOUT "  [Type \"perl -S buildcol.pl | more\" if this help text scrolled off your screen]";
    7983    print STDOUT "\n" unless $ENV{'GSDLOS'} =~ /^windows$/i;
    … …  
    8690    $debug, $mode, $indexname, $keepold, $allclassifications,
    8791    $create_images, $collectdir, $out, $buildtype, $textindex,
    88     $no_strip_html, $no_text);
     92    $no_strip_html, $no_text, $faillog);
    8993
    9094    # note that no defaults are passed for most options as they're set
    … …  
    106110             'out/.*/STDERR', \$out,
    107111             'no_strip_html', \$no_strip_html,
    108              'buildtype/^(mg|mgpp)$/', \$buildtype)) {
     112             'buildtype/^(mg|mgpp)$/', \$buildtype,
     113             'faillog/.*/', \$faillog)) {
    109114    &print_usage();
    110115    die "\n";
    … …  
    125130    die "\n";
    126131    }
     132
     133    if ($faillog eq "") {
     134    $faillog = &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, "etc", "fail.log");
     135    }
     136    # note that we're appending to the faillog here (import.pl clears it each time)
     137    # this could potentially create a situation where the faillog keeps being added
     138    # to over multiple builds (if the import process is being skipped)
     139    open (FAILLOG, ">>$faillog") || die "Couldn't open fail log $faillog\n";
     140    $faillog = 'buildcol::FAILLOG';
     141    $faillog->autoflush(1);
    127142
    128143    # read the configuration file
    … …  
    268283     "\$realarchivedir, \$realbuilddir, \$verbosity, " .
    269284     "\$maxdocs, \$debug, \$keepold, \$allclassifications, " .
    270      "\$out, \$no_text)");
     285     "\$out, \$no_text, \$faillog)");
    271286    die "$@" if $@;
    272287
    … …  
    301316
    302317    close OUT if $close_out;
     318    close FAILLOG;
    303319}
    304320

  • trunk/gsdl/bin/script/classinfo.pl

    r1970 r2785  
    3535}
    3636
    37 use strict;
    3837use classify;
    3938use util;

  • trunk/gsdl/bin/script/import.pl

    r2766 r2785  
    8181    &util::filename_cat ($ENV{'GSDLHOME'}, "collect") . ")\n";
    8282    print STDOUT "   -out name              Filename or handle to print output status to.\n";
    83     print STDOUT "   -faillog name          Filename to log import failures to.\n";
    84     print STDOUT "                          The default is <collectdir>/colname/etc/fail.log\n\n";
     83    print STDOUT "                          The default is STDERR\n";
     84    print STDOUT "   -statsfile name        Filename or handle to print import statistics to.\n";
     85    print STDOUT "                          The default is STDERR\n";
     86    print STDOUT "   -faillog name          Fail log filename. This log receives the filenames\n";
     87    print STDOUT "                          of any files which fail to be processed (defaults.\n";
     88    print STDOUT "                          to " .
     89    &util::filename_cat("<collectdir>", "colname", "etc", "fail.log") . ")\n";
    8590    print STDOUT "  [Type \"perl -S import.pl | more\" if this help text scrolled off your screen]";
    8691    print STDOUT "\n" unless $ENV{'GSDLOS'} =~ /^windows$/i;
    … …  
    9398    $removeold, $gzip, $groupsize, $OIDtype, $debug,
    9499    $maxdocs, $collection, $configfilename, $collectcfg,
    95     $pluginfo, $sortmeta, $archive_info_filename,
    96     $archive_info, $processor, $out, $faillogname, $collectdir);
     100    $pluginfo, $sortmeta, $archive_info_filename, $statsfile,
     101    $archive_info, $processor, $out, $faillog, $collectdir);
    97102
    98103    # note that no defaults are passed for most options as they're set
    … …  
    112117             'collectdir/.*/', \$collectdir,
    113118             'out/.*/STDERR', \$out,
    114              'faillog/.*/', \$faillogname)) {
     119             'statsfile/.*/STDERR', \$statsfile,
     120             'faillog/.*/', \$faillog)) {
    115121    &print_usage();
    116122    die "\n";
    … …  
    133139    die "\n";
    134140    }
    135    
    136     # check and/or set fail log file
    137     if ($faillogname eq "") {
    138     $faillogname="$ENV{GSDLCOLLECTDIR}/etc/fail.log";
    139     } else {
    140     my $can_open=1;
    141     open (TESTOPEN, ">$faillogname") || ($can_open=0);
    142     close (TESTOPEN);
    143     if ($can_open==0) {
    144         warn "fail.log - cannot write to \"$faillogname\", using default\n \"$ENV{GSDLCOLLECTDIR}/etc/fail.log\" instead.\n";
    145         $faillogname="$ENV{GSDLCOLLECTDIR}/etc/fail.log";
    146     }
    147     }
    148     # test that default is writable...
    149     if ($faillogname eq "$ENV{GSDLCOLLECTDIR}/etc/fail.log") {
    150     my $can_open=1;
    151     open (TESTOPEN, ">$faillogname") || ($can_open=0);
    152     close (TESTOPEN);
    153     if ($can_open==0) {
    154         warn "warning - cannot write to \"$faillogname\".\n";
    155         $faillogname="";
    156     }
    157     }
    158 
     141
     142    if ($faillog eq "") {
     143    $faillog = &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, "etc", "fail.log");
     144    }
     145    open (FAILLOG, ">$faillog") || die "Couldn't open fail log $faillog\n";
     146    $faillog = 'import::FAILLOG';
     147    $faillog->autoflush(1);
    159148
    160149    # check sortmeta
    … …  
    256245
    257246    # load all the plugins
    258     $pluginfo = &plugin::load_plugins ($plugins, $verbosity, $out, $faillogname);
     247    $pluginfo = &plugin::load_plugins ($plugins, $verbosity, $out, $faillog);
    259248    if (scalar(@$pluginfo) == 0) {
    260249    print $out "No plugins were loaded.\n";
    … …  
    290279   
    291280    &plugin::end($pluginfo, $processor);
    292    
     281
    293282    # write out the archive information file
    294283    if (!$debug) {
    … …  
    296285    $archive_info->save_info($archive_info_filename);
    297286    }
     287   
     288    # write out import stats
     289    my $close_stats = 0;
     290    if ($statsfile !~ /^(STDERR|STDOUT)$/i) {
     291    if (open (STATS, ">$statsfile")) {
     292        $statsfile = 'import::STATS';
     293        $close_stats = 1;
     294    } else {
     295        print $out "WARNING: couldn't open stats file $statsfile\n";
     296        print $out "         will print stats to STDERR instead\n";
     297        $statsfile = 'STDERR';
     298    }
     299    }
     300
     301    print $out "\n";
     302    print $out "*********************************************\n";
     303    print $out "Import Complete\n";
     304    print $out "*********************************************\n";
     305
     306    &plugin::write_stats($pluginfo, $statsfile);
     307    if ($close_stats) {
     308    close STATS;
     309    }
     310
    298311    close OUT if $close_out;
     312    close FAILLOG;
    299313}
     314

  • trunk/gsdl/bin/script/mirror.pl

    r2472 r2785  
    4848}
    4949
    50 use strict;
    5150use arcinfo;
    5251use colcfg;

  • trunk/gsdl/bin/script/nightly.pl

    r1970 r2785  
    4141}
    4242
    43 use strict;
    4443use arcinfo;
    4544use colcfg;

  • trunk/gsdl/bin/script/pluginfo.pl

    r1970 r2785  
    3636}
    3737
    38 use strict;
    3938use plugin;
    4039use util;

  • trunk/gsdl/bin/script/update.pl

    r1970 r2785  
    4646}
    4747
    48 use strict;
    4948use arcinfo;
    5049use colcfg;
Note: See TracChangeset for help on using the changeset viewer.