Changeset 18528 for gsdl

Show
Ignore:
Timestamp:
17.02.2009 15:36:44 (11 years ago)
Author:
davidb
Message:

OIDmetadata wasn't supported in collect.cfg, but OIDtype was. Now rectified. Also introduced OIDcount as a file saved in the archives folder to help doc.pm use the correct value when working incrementally

Location:
gsdl/trunk
Files:
7 modified

Legend:

Unmodified
Added
Removed
  • gsdl/trunk/bin/script/import.pl

    r18507 r18528  
    197197    'desc' => "{import.OIDmetadata}", 
    198198    'type' => "metadata", 
    199     'deft' => "dc.Identifier", 
     199    # parsearg left "" as default 
     200    #'deft' => "dc.Identifier", 
    200201    'reqd' => "no", 
    201202    'modegli' => "2" }, 
     
    440441    } 
    441442 
     443    if ((!defined $OIDmetadata) || ($OIDmetadata eq "")) { 
     444    if (defined $collectcfg->{'OIDmetadata'}) { 
     445        $OIDmetadata = $collectcfg->{'OIDmetadata'}; 
     446    } else { 
     447        $OIDmetadata = "dc.Identifier"; # the default 
     448    } 
     449    } 
     450 
    442451    if (defined $collectcfg->{'sortmeta'} && (!defined $sortmeta || $sortmeta eq "")) { 
    443452    $sortmeta = $collectcfg->{'sortmeta'}; 
     
    575584 
    576585    if (!$removeold) { 
     586 
     587        &inexport::prime_doc_oid_count($archivedir); 
     588 
     589 
    577590        # Can now work out which files were new, already existed, and have 
    578591        # been deleted 
     
    620633    &plugin::deinit($pluginfo, $processor); 
    621634 
     635    # Store the value of OIDCount (used in doc.pm) so it can be 
     636    # restored correctly to this value on an incremental build 
     637    &inexport::store_doc_oid_count($archivedir); 
     638 
    622639    # write out the archive information file 
    623640    $processor->close_file_output() if $groupsize > 1; 
  • gsdl/trunk/perllib/colcfg.pm

    r17109 r18528  
    8080                    q/archivedir|cachedir|builddir|removeold|/ . 
    8181                    q/textcompress|buildtype|no_text|keepold|gzip|/ . 
    82                     q/verbosity|remove_empty_classifications|OIDtype|/ . 
     82                    q/verbosity|remove_empty_classifications|OIDtype|OIDmetadata|/ . 
    8383                    q/groupsize|maxdocs|debug|mode|saveas|/ . 
    8484                    q/sortmeta|removesuffix|removeprefix|create_images|/ . 
     
    114114                 q/archivedir|cachedir|builddir|removeold|/ . 
    115115                 q/textcompress|buildtype|no_text|keepold|gzip|/ . 
    116                  q/verbosity|remove_empty_classifications|OIDtype|/. 
     116                 q/verbosity|remove_empty_classifications|OIDtype|OIDmetadata|/. 
    117117                 q/groupsize|maxdocs|debug|mode|saveas|/ . 
    118118                 q/sortmeta|removesuffix|removeprefix|create_images|/ . 
  • gsdl/trunk/perllib/doc.pm

    r18508 r18528  
    4545# classification 
    4646 
    47 my $OIDcount = 0; 
     47our $OIDcount = 0; 
    4848 
    4949sub new { 
  • gsdl/trunk/perllib/inexport.pm

    r18508 r18528  
    3232use util; 
    3333use GDBMUtils; 
     34 
     35 
     36sub prime_doc_oid_count 
     37{ 
     38    my ($archivedir) = @_; 
     39    my $oid_count_filename = &util::filename_cat ($archivedir, "OIDcount"); 
     40 
     41    if (-e $oid_count_filename) { 
     42    if (open(OIDIN,"<$oid_count_filename")) { 
     43        my $OIDcount = <OIDIN>; 
     44        chomp $OIDcount;         
     45        close(OIDIN); 
     46 
     47        $doc::OIDcount = $OIDcount;      
     48    } 
     49    else { 
     50         
     51        print STDERR "Warning: unable to read document OID count from $oid_count_filename\n"; 
     52        print STDERR "Setting value to 0\n"; 
     53    } 
     54    } 
     55     
     56} 
     57 
     58sub store_doc_oid_count 
     59{ 
     60    # Use the file "OIDcount" in the archives directory to record 
     61    # what value doc.pm got up to 
     62 
     63    my ($archivedir) = @_; 
     64    my $oid_count_filename = &util::filename_cat ($archivedir, "OIDcount"); 
     65 
     66 
     67    if (open(OIDOUT,">$oid_count_filename")) { 
     68    print OIDOUT $doc::OIDcount, "\n"; 
     69         
     70    close(OIDOUT); 
     71    } 
     72    else { 
     73    print STDERR "Warning: unable to store document OID count\n"; 
     74    } 
     75} 
     76 
     77 
    3478 
    3579sub new_vs_old_import_diff 
     
    160204     
    161205        if ($verbosity>1) { 
    162             print STDERR "$oid marked to be deleted from index on next buildcol.pl\n"; 
     206            print STDERR "$oid ($file) marked to be deleted from index on next buildcol.pl\n"; 
    163207        } 
    164208        $archive_info->set_status_info($oid,"D"); 
  • gsdl/trunk/perllib/plugins/ArchivesInfPlugin.pm

    r18509 r18528  
    3232 
    3333use util; 
     34use doc; 
    3435use PrintInfo; 
    3536use plugin; 
     
    173174} 
    174175 
    175 # we don't do any file blocking 
    176176sub file_block_read { 
    177177 
    178178    my $self = shift (@_);   
    179179    my ($pluginfo, $base_dir, $file, $block_hash, $metadata, $gli) = @_; 
     180 
     181    if ($file eq "OIDcount") { 
     182    my ($filename_full_path, $filename_no_path)  
     183        = &util::get_full_filenames($base_dir, $file); 
     184    $block_hash->{'file_blocks'}->{$filename_full_path} = 1; 
     185    return 1; 
     186    } 
     187 
     188    # otherwise, we don't do any file blocking 
    180189 
    181190    return undef; 
     
    287296    } 
    288297 
     298 
    289299    # wasn't an archives directory, someone else will have to process it 
    290300    return undef; 
  • gsdl/trunk/perllib/plugins/ReadXMLFile.pm

    r18327 r18528  
    293293} 
    294294 
     295 
    295296# Called at the beginning of the XML document. 
    296297sub xml_start_document { 
  • gsdl/trunk/perllib/plugouts/BasePlugout.pm

    r18463 r18528  
    669669              'assoc-files' => [] }; 
    670670     
    671     my @reverse_lookups = ($source_filename); 
     671    my %reverse_lookups = ( $source_filename => "1" ); 
    672672 
    673673    foreach my $assoc_file_rec (@{$doc_obj->get_assoc_files()}) { 
     
    687687        } 
    688688 
    689         push(@reverse_lookups,$real_filename); 
     689        $reverse_lookups{$real_filename} = 1; 
    690690 
    691691        push(@{$oid_files->{'assoc-files'}},$full_afile); 
     
    715715    chomp($doc_db_text); # remove trailing \n 
    716716 
    717     ## print STDERR "*** To set in db: \n\t$doc_db\n\t$oid\n\t$doc_db_text\n"; 
     717    ##print STDERR "*** To set in db: \n\t$doc_db\n\t$oid\n\t$doc_db_text\n"; 
    718718    &GDBMUtils::gdbmDatabaseSet($doc_db,$oid,$doc_db_text); 
    719719 
    720     foreach my $rl (@reverse_lookups) { 
     720    foreach my $rl (keys %reverse_lookups) { 
    721721    &GDBMUtils::gdbmDatabaseAppend($src_db,$rl,"<oid>$oid\n"); 
    722722    }