Changeset 23939 for main

Timestamp:

2011-04-20T21:27:19+12:00 (13 years ago)

Author:

ak19

Message:

GS3's OAIserver passes final official oaiserver validation tests: to do with earliestDatestamp. 1. Perl code (inexport, basebuilder, colcfg, buildconfigxml.pm perl files) write out the earliestDatestamp into GS3's buildconfig.xml. Whenever a full-build is performed, the archives directory is recreated. At this stage, inexport creates a new file in archives called earliestDatestamp containing the current time. Whenever an incremental build is performed, this file already exists in archive, so it is left untouched, preserving the time of the full-build (which is the earliestDatestamp). The other perl files are concerned with obtaining this value from the archives directory and writing it out to the build config file. 2. doc.pm and BasePlugout.pm write out the current date and time for each document processed under the new fields oailastmodified and oailastmodifieddate. Changes made in this commit are related to GS3 java src code changes that work in tandem.

Location:

main/trunk/greenstone2/perllib

Files:

• basebuilder.pm (modified) (1 diff)
• buildConfigxml.pm (modified) (1 diff)
• colcfg.pm (modified) (2 diffs)
• doc.pm (modified) (1 diff)
• inexport.pm (modified) (1 diff)
• plugouts/BasePlugout.pm (modified) (1 diff)

main/trunk/greenstone2/perllib/basebuilder.pm

@@ -560 +560 @@
 
     $build_cfg->{'infodbtype'} = $self->{'infodbtype'};
+    
+    # write out the earliestDatestamp information needed for OAI
+    my $archivedir = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, "archives");
+    if(!-d $archivedir) {
+    $archivedir = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, "export");
+    }    
+    if(-d $archivedir) {
+    my $earliestDatestampFile = &util::filename_cat ($archivedir, "earliestDatestamp");
+    open(FIN,"<$earliestDatestampFile") || die "{common.cannot_open} $earliestDatestampFile: $!\n";
+    my $earliestDatestamp;
+    {
+        # slurp in file as a single line
+        local $/ = undef;
+        $earliestDatestamp = <FIN>;
+        #&unicode::ensure_utf8(\$earliestDatestamp); # turn any high bytes that aren't valid utf-8 into utf-8.
+    }
+    close(FIN); 
+    $build_cfg->{'earliestDatestamp'} = $earliestDatestamp;
+    }
 
     $self->build_cfg_extra($build_cfg);

main/trunk/greenstone2/perllib/buildConfigxml.pm

@@ -226 +226 @@
     &write_line('COLCFG', ["<metadata name=\"infodbType\">", $buildcfg->{"infodbtype"}, "</metadata>"]);
     }
+    if (defined $buildcfg->{'earliestDatestamp'}) {
+    &write_line('COLCFG', ["<metadata name=\"earliestDatestamp\">", $buildcfg->{"earliestDatestamp"}, "</metadata>"]);
+    }
+
     &write_line('COLCFG', ["</metadataList>"]);
 

main/trunk/greenstone2/perllib/colcfg.pm

@@ -225 +225 @@
 
     return &cfgread::read_cfg_file ($filename, 
-           q/^(infodbtype|builddate|buildtype|numdocs|numsections|numwords|numbytes|maxnumeric|textlevel|indexstem|stemindexes|separate_cjk)$/,
+           q/^(earliestdatestamp|infodbtype|builddate|buildtype|numdocs|numsections|numwords|numbytes|maxnumeric|textlevel|indexstem|stemindexes|separate_cjk)$/,
            q/^(indexmap|subcollectionmap|languagemap|notbuilt|indexfields|indexfieldmap|indexlevels|levelmap)$/);
                     
@@ -235 +235 @@
 
     &cfgread::write_cfg_file($filename, $data,
-           q/^(infodbtype|builddate|buildtype|numdocs|numsections|numwords|numbytes|maxnumeric|textlevel|indexstem|stemindexes|separate_cjk)$/,
+           q/^(earliestdatestamp|infodbtype|builddate|buildtype|numdocs|numsections|numwords|numbytes|maxnumeric|textlevel|indexstem|stemindexes|separate_cjk)$/,
            q/^(indexmap|subcollectionmap|languagemap|notbuilt|indexfields|indexfieldmap|indexlevels|levelmap)$/);            
 }

main/trunk/greenstone2/perllib/doc.pm

@@ -126 +126 @@
 
 # set lastmodified for OAI purposes, added by GRB, moved by kjdon
+sub set_oailastmodified {
+    my $self = shift (@_);
+
+    my $source_path = $self->{'terse_source_path'};
+   
+    if (defined $source_path && (-e $source_path)) {
+    my $current_time = time;
+
+    my ($seconds, $minutes, $hours, $day_of_month, $month, $year,
+        $wday, $yday, $isdst) = localtime($current_time);
+
+    my $date_modified = sprintf("%d%02d%02d",1900+$year,$month+1,$day_of_month);
+
+    $self->add_utf8_metadata($self->get_top_section(), "oailastmodified", $current_time);
+    $self->add_utf8_metadata($self->get_top_section(), "oailastmodifieddate", $date_modified);
+    } 
+}
+
+# no longer used for OAI purposes, since lastmodified is not what we want as the
+# Datestamp of a document. This doc metadata may be useful for general purposes.
 sub set_lastmodified {
     my $self = shift (@_);

main/trunk/greenstone2/perllib/inexport.pm

@@ -708 +708 @@
     }
     }
-    
+
+    # Check for existence of the file that's to contain earliestDateStamp in archivesdir
+    # Do nothing if the file already exists (file exists on incremental build).
+    # If the file doesn't exist, as happens on full build, create it and write out the current datestamp into it
+    # In buildcol, read the file's contents and set the earliestdateStamp in GS2's build.cfg / GS3's buildconfig.xml
+    # In doc.pm have set_oaiLastModified similar to set_lastmodified, and create the doc fields 
+    # oailastmodified and oailastmodifieddate
+    my $earliestDatestampFile = &util::filename_cat($archivedir, "earliestDatestamp");
+    if (!-f $earliestDatestampFile) {
+    my $current_time_in_seconds = time; # in seconds
+    open(FOUT, ">$earliestDatestampFile") || die "{common.cannot_open} $earliestDatestampFile: $!\n";
+    #    (&gsprintf(STDERR, "{common.cannot_open}: $!\n", $earliestDatestampFile) && die);
+    print FOUT $current_time_in_seconds;
+    close(FOUT);
+    }
+
     # now, whichever mode we are in, we can process the entire import folder
     if ((defined $jobs) && ($jobs > 1))

main/trunk/greenstone2/perllib/plugouts/BasePlugout.pm

@@ -424 +424 @@
     my ($doc_obj) = @_;
     
+    # for OAI purposes
     $doc_obj->set_lastmodified();
+    $doc_obj->set_oailastmodified();
 
      if ($self->{'group_size'} > 1) {