Changeset 18528 for gsdl


Ignore:
Timestamp:
02/17/09 15:36:44 (12 years ago)
Author:
davidb
Message:

OIDmetadata wasn't supported in collect.cfg, but OIDtype was. Now rectified. Also introduced OIDcount as a file saved in the archives folder to help doc.pm use the correct value when working incrementally

Location:
gsdl/trunk
Files:
7 edited

Legend:

Unmodified
Added
Removed
  • gsdl/trunk/bin/script/import.pl

    r18507 r18528  
    197197    'desc' => "{import.OIDmetadata}",
    198198    'type' => "metadata",
    199     'deft' => "dc.Identifier",
     199    # parsearg left "" as default
     200    #'deft' => "dc.Identifier",
    200201    'reqd' => "no",
    201202    'modegli' => "2" },
     
    440441    }
    441442
     443    if ((!defined $OIDmetadata) || ($OIDmetadata eq "")) {
     444    if (defined $collectcfg->{'OIDmetadata'}) {
     445        $OIDmetadata = $collectcfg->{'OIDmetadata'};
     446    } else {
     447        $OIDmetadata = "dc.Identifier"; # the default
     448    }
     449    }
     450
    442451    if (defined $collectcfg->{'sortmeta'} && (!defined $sortmeta || $sortmeta eq "")) {
    443452    $sortmeta = $collectcfg->{'sortmeta'};
     
    575584
    576585    if (!$removeold) {
     586
     587        &inexport::prime_doc_oid_count($archivedir);
     588
     589
    577590        # Can now work out which files were new, already existed, and have
    578591        # been deleted
     
    620633    &plugin::deinit($pluginfo, $processor);
    621634
     635    # Store the value of OIDCount (used in doc.pm) so it can be
     636    # restored correctly to this value on an incremental build
     637    &inexport::store_doc_oid_count($archivedir);
     638
    622639    # write out the archive information file
    623640    $processor->close_file_output() if $groupsize > 1;
  • gsdl/trunk/perllib/colcfg.pm

    r17109 r18528  
    8080                    q/archivedir|cachedir|builddir|removeold|/ .
    8181                    q/textcompress|buildtype|no_text|keepold|gzip|/ .
    82                     q/verbosity|remove_empty_classifications|OIDtype|/ .
     82                    q/verbosity|remove_empty_classifications|OIDtype|OIDmetadata|/ .
    8383                    q/groupsize|maxdocs|debug|mode|saveas|/ .
    8484                    q/sortmeta|removesuffix|removeprefix|create_images|/ .
     
    114114                 q/archivedir|cachedir|builddir|removeold|/ .
    115115                 q/textcompress|buildtype|no_text|keepold|gzip|/ .
    116                  q/verbosity|remove_empty_classifications|OIDtype|/.
     116                 q/verbosity|remove_empty_classifications|OIDtype|OIDmetadata|/.
    117117                 q/groupsize|maxdocs|debug|mode|saveas|/ .
    118118                 q/sortmeta|removesuffix|removeprefix|create_images|/ .
  • gsdl/trunk/perllib/doc.pm

    r18508 r18528  
    4545# classification
    4646
    47 my $OIDcount = 0;
     47our $OIDcount = 0;
    4848
    4949sub new {
  • gsdl/trunk/perllib/inexport.pm

    r18508 r18528  
    3232use util;
    3333use GDBMUtils;
     34
     35
     36sub prime_doc_oid_count
     37{
     38    my ($archivedir) = @_;
     39    my $oid_count_filename = &util::filename_cat ($archivedir, "OIDcount");
     40
     41    if (-e $oid_count_filename) {
     42    if (open(OIDIN,"<$oid_count_filename")) {
     43        my $OIDcount = <OIDIN>;
     44        chomp $OIDcount;       
     45        close(OIDIN);
     46
     47        $doc::OIDcount = $OIDcount;     
     48    }
     49    else {
     50       
     51        print STDERR "Warning: unable to read document OID count from $oid_count_filename\n";
     52        print STDERR "Setting value to 0\n";
     53    }
     54    }
     55   
     56}
     57
     58sub store_doc_oid_count
     59{
     60    # Use the file "OIDcount" in the archives directory to record
     61    # what value doc.pm got up to
     62
     63    my ($archivedir) = @_;
     64    my $oid_count_filename = &util::filename_cat ($archivedir, "OIDcount");
     65
     66
     67    if (open(OIDOUT,">$oid_count_filename")) {
     68    print OIDOUT $doc::OIDcount, "\n";
     69       
     70    close(OIDOUT);
     71    }
     72    else {
     73    print STDERR "Warning: unable to store document OID count\n";
     74    }
     75}
     76
     77
    3478
    3579sub new_vs_old_import_diff
     
    160204   
    161205        if ($verbosity>1) {
    162             print STDERR "$oid marked to be deleted from index on next buildcol.pl\n";
     206            print STDERR "$oid ($file) marked to be deleted from index on next buildcol.pl\n";
    163207        }
    164208        $archive_info->set_status_info($oid,"D");
  • gsdl/trunk/perllib/plugins/ArchivesInfPlugin.pm

    r18509 r18528  
    3232
    3333use util;
     34use doc;
    3435use PrintInfo;
    3536use plugin;
     
    173174}
    174175
    175 # we don't do any file blocking
    176176sub file_block_read {
    177177
    178178    my $self = shift (@_); 
    179179    my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $gli) = @_;
     180
     181    if ($file eq "OIDcount") {
     182    my ($filename_full_path, $filename_no_path)
     183        = &util::get_full_filenames($base_dir, $file);
     184    $block_hash->{'file_blocks'}->{$filename_full_path} = 1;
     185    return 1;
     186    }
     187
     188    # otherwise, we don't do any file blocking
    180189
    181190    return undef;
     
    287296    }
    288297
     298
    289299    # wasn't an archives directory, someone else will have to process it
    290300    return undef;
  • gsdl/trunk/perllib/plugins/ReadXMLFile.pm

    r18327 r18528  
    293293}
    294294
     295
    295296# Called at the beginning of the XML document.
    296297sub xml_start_document {
  • gsdl/trunk/perllib/plugouts/BasePlugout.pm

    r18463 r18528  
    669669              'assoc-files' => [] };
    670670   
    671     my @reverse_lookups = ($source_filename);
     671    my %reverse_lookups = ( $source_filename => "1" );
    672672
    673673    foreach my $assoc_file_rec (@{$doc_obj->get_assoc_files()}) {
     
    687687        }
    688688
    689         push(@reverse_lookups,$real_filename);
     689        $reverse_lookups{$real_filename} = 1;
    690690
    691691        push(@{$oid_files->{'assoc-files'}},$full_afile);
     
    715715    chomp($doc_db_text); # remove trailing \n
    716716
    717     ## print STDERR "*** To set in db: \n\t$doc_db\n\t$oid\n\t$doc_db_text\n";
     717    ##print STDERR "*** To set in db: \n\t$doc_db\n\t$oid\n\t$doc_db_text\n";
    718718    &GDBMUtils::gdbmDatabaseSet($doc_db,$oid,$doc_db_text);
    719719
    720     foreach my $rl (@reverse_lookups) {
     720    foreach my $rl (keys %reverse_lookups) {
    721721    &GDBMUtils::gdbmDatabaseAppend($src_db,$rl,"<oid>$oid\n");
    722722    }
Note: See TracChangeset for help on using the changeset viewer.