Changeset 23939


Ignore:
Timestamp:
04/20/11 21:27:19 (10 years ago)
Author:
ak19
Message:

GS3's OAIserver passes final official oaiserver validation tests: to do with earliestDatestamp. 1. Perl code (inexport, basebuilder, colcfg, buildconfigxml.pm perl files) write out the earliestDatestamp into GS3's buildconfig.xml. Whenever a full-build is performed, the archives directory is recreated. At this stage, inexport creates a new file in archives called earliestDatestamp containing the current time. Whenever an incremental build is performed, this file already exists in archive, so it is left untouched, preserving the time of the full-build (which is the earliestDatestamp). The other perl files are concerned with obtaining this value from the archives directory and writing it out to the build config file. 2. doc.pm and BasePlugout.pm write out the current date and time for each document processed under the new fields oailastmodified and oailastmodifieddate. Changes made in this commit are related to GS3 java src code changes that work in tandem.

Location:
main/trunk/greenstone2/perllib
Files:
6 edited

Legend:

Unmodified
Added
Removed
  • main/trunk/greenstone2/perllib/basebuilder.pm

    r23172 r23939  
    560560
    561561    $build_cfg->{'infodbtype'} = $self->{'infodbtype'};
     562   
     563    # write out the earliestDatestamp information needed for OAI
     564    my $archivedir = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, "archives");
     565    if(!-d $archivedir) {
     566    $archivedir = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, "export");
     567    }   
     568    if(-d $archivedir) {
     569    my $earliestDatestampFile = &util::filename_cat ($archivedir, "earliestDatestamp");
     570    open(FIN,"<$earliestDatestampFile") || die "{common.cannot_open} $earliestDatestampFile: $!\n";
     571    my $earliestDatestamp;
     572    {
     573        # slurp in file as a single line
     574        local $/ = undef;
     575        $earliestDatestamp = <FIN>;
     576        #&unicode::ensure_utf8(\$earliestDatestamp); # turn any high bytes that aren't valid utf-8 into utf-8.
     577    }
     578    close(FIN);
     579    $build_cfg->{'earliestDatestamp'} = $earliestDatestamp;
     580    }
    562581
    563582    $self->build_cfg_extra($build_cfg);
  • main/trunk/greenstone2/perllib/buildConfigxml.pm

    r23895 r23939  
    226226    &write_line('COLCFG', ["<metadata name=\"infodbType\">", $buildcfg->{"infodbtype"}, "</metadata>"]);
    227227    }
     228    if (defined $buildcfg->{'earliestDatestamp'}) {
     229    &write_line('COLCFG', ["<metadata name=\"earliestDatestamp\">", $buildcfg->{"earliestDatestamp"}, "</metadata>"]);
     230    }
     231
    228232    &write_line('COLCFG', ["</metadataList>"]);
    229233
  • main/trunk/greenstone2/perllib/colcfg.pm

    r21785 r23939  
    225225
    226226    return &cfgread::read_cfg_file ($filename,
    227            q/^(infodbtype|builddate|buildtype|numdocs|numsections|numwords|numbytes|maxnumeric|textlevel|indexstem|stemindexes|separate_cjk)$/,
     227           q/^(earliestdatestamp|infodbtype|builddate|buildtype|numdocs|numsections|numwords|numbytes|maxnumeric|textlevel|indexstem|stemindexes|separate_cjk)$/,
    228228           q/^(indexmap|subcollectionmap|languagemap|notbuilt|indexfields|indexfieldmap|indexlevels|levelmap)$/);
    229229                   
     
    235235
    236236    &cfgread::write_cfg_file($filename, $data,
    237            q/^(infodbtype|builddate|buildtype|numdocs|numsections|numwords|numbytes|maxnumeric|textlevel|indexstem|stemindexes|separate_cjk)$/,
     237           q/^(earliestdatestamp|infodbtype|builddate|buildtype|numdocs|numsections|numwords|numbytes|maxnumeric|textlevel|indexstem|stemindexes|separate_cjk)$/,
    238238           q/^(indexmap|subcollectionmap|languagemap|notbuilt|indexfields|indexfieldmap|indexlevels|levelmap)$/);           
    239239}
  • main/trunk/greenstone2/perllib/doc.pm

    r23923 r23939  
    126126
    127127# set lastmodified for OAI purposes, added by GRB, moved by kjdon
     128sub set_oailastmodified {
     129    my $self = shift (@_);
     130
     131    my $source_path = $self->{'terse_source_path'};
     132   
     133    if (defined $source_path && (-e $source_path)) {
     134    my $current_time = time;
     135
     136    my ($seconds, $minutes, $hours, $day_of_month, $month, $year,
     137        $wday, $yday, $isdst) = localtime($current_time);
     138
     139    my $date_modified = sprintf("%d%02d%02d",1900+$year,$month+1,$day_of_month);
     140
     141    $self->add_utf8_metadata($self->get_top_section(), "oailastmodified", $current_time);
     142    $self->add_utf8_metadata($self->get_top_section(), "oailastmodifieddate", $date_modified);
     143    }
     144}
     145
     146# no longer used for OAI purposes, since lastmodified is not what we want as the
     147# Datestamp of a document. This doc metadata may be useful for general purposes.
    128148sub set_lastmodified {
    129149    my $self = shift (@_);
  • main/trunk/greenstone2/perllib/inexport.pm

    r23825 r23939  
    708708    }
    709709    }
    710    
     710
     711    # Check for existence of the file that's to contain earliestDateStamp in archivesdir
     712    # Do nothing if the file already exists (file exists on incremental build).
     713    # If the file doesn't exist, as happens on full build, create it and write out the current datestamp into it
     714    # In buildcol, read the file's contents and set the earliestdateStamp in GS2's build.cfg / GS3's buildconfig.xml
     715    # In doc.pm have set_oaiLastModified similar to set_lastmodified, and create the doc fields
     716    # oailastmodified and oailastmodifieddate
     717    my $earliestDatestampFile = &util::filename_cat($archivedir, "earliestDatestamp");
     718    if (!-f $earliestDatestampFile) {
     719    my $current_time_in_seconds = time; # in seconds
     720    open(FOUT, ">$earliestDatestampFile") || die "{common.cannot_open} $earliestDatestampFile: $!\n";
     721    #    (&gsprintf(STDERR, "{common.cannot_open}: $!\n", $earliestDatestampFile) && die);
     722    print FOUT $current_time_in_seconds;
     723    close(FOUT);
     724    }
     725
    711726    # now, whichever mode we are in, we can process the entire import folder
    712727    if ((defined $jobs) && ($jobs > 1))
  • main/trunk/greenstone2/perllib/plugouts/BasePlugout.pm

    r23824 r23939  
    424424    my ($doc_obj) = @_;
    425425   
     426    # for OAI purposes
    426427    $doc_obj->set_lastmodified();
     428    $doc_obj->set_oailastmodified();
    427429
    428430     if ($self->{'group_size'} > 1) {
Note: See TracChangeset for help on using the changeset viewer.