Changeset 31915

Timestamp:

2017-08-23T18:58:38+12:00 (6 years ago)

Author:

ak19

Message:

Dr Bainbridge thought about it and decided that the correct solution is that, since a collection will always have an oai-inf db from now on, the earliest datestamp of a collection should not fall back to either buildconfig's earliestdatestamp field or else buildconfig's lastmodified. However, the latter are used as the publishing date by the RSS service, and so still stored as Collection.java's earliestDatestamp. Now OAICollection has a new additional field, earliestOAIDatestamp which contains the earliest timestamp in oai-inf db. The OAIReceptionist now determines the earliestDatestamp of the entire OAIRepository solely based on the earliestOAIDatestamp values across all OAICollections, also with no fallbacks on Collections' earliestDatestamp or lastModified fields.

Location:

main/trunk/greenstone3/src/java/org/greenstone/gsdl3

Files:

• collection/Collection.java (modified) (3 diffs)
• collection/OAICollection.java (modified) (3 diffs)
• core/OAIMessageRouter.java (modified) (1 diff)
• core/OAIReceptionist.java (modified) (1 diff)
• util/OAIXML.java (modified) (1 diff)

main/trunk/greenstone3/src/java/org/greenstone/gsdl3/collection/Collection.java

@@ -73 +73 @@
     /** time when this collection was built Used by RSS */
     protected long lastmodified = 0;
-    /** earliestDatestamp of this collection. Used by OAI/RSS */
+    /** earliestDatestamp of this collection. Used by RSS and as fallback by OAI */
         protected long earliestDatestamp = 0;
 
@@ -187 +187 @@
     }
 
-    // Used by OAI Receptionist and RSSRetrieve 
+    // Used by OAI Receptionist (as second fallback) and RSSRetrieve 
     public long getLastmodified()
     {
@@ -193 +193 @@
     }
 
-    // used by the OAIReceptionist and RSSRetrieve
+    // used by the OAIReceptionist (as fallback) and RSSRetrieve
     public long getEarliestDatestamp()
     {

main/trunk/greenstone3/src/java/org/greenstone/gsdl3/collection/OAICollection.java

@@ -61 +61 @@
     protected boolean has_oai = false;
 
-  /** a reference to the OAIPMH service rack */
-  protected OAIPMH oai_service_rack = null;
+    /** earliest datestamp of an OAI collection. Also used to work out the earliest datetimestamp of the entire OAI repository */
+    protected long earliestOAIDatestamp = 0;
+
+    /** a reference to the OAIPMH service rack */
+    protected OAIPMH oai_service_rack = null;
 
     /**
@@ -120 +123 @@
     }
 
+    /**
+     * The earliesttimestamp entry in the oai-inf.db representing when the collection was created.
+     * Used by the OAIReceptionist
+    */   
+    public long getEarliestOAIDatestamp()
+    {
+        return earliestOAIDatestamp;
+    }
+    
   /** add any extra info for collection from OAIConfig.xml */
   public boolean configureOAI(Element oai_config) {
@@ -152 +164 @@
 
           // once we've configured the OAIPMH service, we can use the OAIPMH service to
-          // retrieve the earliest timestamp of the collection from the oai-inf db and
-          // overwrite Collection.earliestDatestamp with it.
+          // retrieve the earliest timestamp of this OAI collection from the oai-inf db
           long earliestTimestamp = this.oai_service_rack.getEarliestTimestamp();
           if(earliestTimestamp == -1) {
-          //this.earliestDatestamp = -1;
-          logger.warn("No OAI timestamp for collection " + this.cluster_name
-                  + ". Falling back to using its earliestDatestamp from build config: " + this.earliestDatestamp);
+          this.earliestOAIDatestamp = -1;
+          logger.warn("No OAI timestamp for collection " + this.cluster_name);
+          //logger.warn("No OAI timestamp for collection " + this.cluster_name
+          //+ ". Falling back to using its earliestDatestamp from build config: " + this.earliestDatestamp);
           } else {
-          this.earliestDatestamp = earliestTimestamp; // milliseconds
+          this.earliestOAIDatestamp = earliestTimestamp; // milliseconds
           }
           

main/trunk/greenstone3/src/java/org/greenstone/gsdl3/core/OAIMessageRouter.java

@@ -149 +149 @@
         e.setAttribute(OAIXML.LASTMODIFIED, "" + c.getLastmodified());
         e.setAttribute(OAIXML.EARLIEST_DATESTAMP, "" + c.getEarliestDatestamp());
+        e.setAttribute(OAIXML.EARLIEST_OAI_DATESTAMP, "" + c.getEarliestOAIDatestamp());
         this.collection_list.appendChild(e);
         return true;

main/trunk/greenstone3/src/java/org/greenstone/gsdl3/core/OAIReceptionist.java

@@ -1038 +1038 @@
     for(int i=0; i<oai_coll_size; i++) {
     String collName = collection_name_list.get(i);
-    long coll_earliestDatestamp = Long.parseLong(((Element)oai_coll.item(i)).getAttribute(OAIXML.EARLIEST_DATESTAMP)); // Taken from oai-inf db's OAI_EARLIEST_TIMESTAMP_OID entry, else falls back to earliest datestamp field in buildcfg
-      if (coll_earliestDatestamp == 0) {
-    // try last modified
-    coll_earliestDatestamp = Long.parseLong(((Element)oai_coll.item(i)).getAttribute(OAIXML.LAST_MODIFIED));
-    //logger.info("@@@ Falling back to using collection " + collName + "'s lastmodified date as its earliest timestamp: " + coll_earliestDatestamp);
-      }
-      if (coll_earliestDatestamp > 0) {
-    earliestDatestamp = (earliestDatestamp > coll_earliestDatestamp)? coll_earliestDatestamp : earliestDatestamp;
-      }
-    }
+    long coll_earliestDatestamp = Long.parseLong(((Element)oai_coll.item(i)).getAttribute(OAIXML.EARLIEST_OAI_DATESTAMP)); // Taken from oai-inf db's OAI_EARLIEST_TIMESTAMP_OID entry, -1 if not found
+
+    if (coll_earliestDatestamp > 0 && earliestDatestamp > coll_earliestDatestamp) {
+        earliestDatestamp = coll_earliestDatestamp;
+        //logger.info("@@@ Found earlier timestamp: " + earliestDatestamp + " ms");
+    }
+    }
+
+    // we're no longer trying fallbacks for earliestDatestamp (other than the extreme fallback of
+    // unix epoch time) because, going forward, all collections will have oai-inf db containing
+    // an entry for earliesttimestamp. And all OAICollections will moreover have them stored and
+    // will return them upon calling getEarliestOAIDatestamp().
+    /*    
+    if(earliestDatestamp == current_time) {
+    logger.info("Can't determine earliesttimestamp from oai-inf.db for any OAI collection. Trying timestamps in build config...");
+    for(int i=0; i<oai_coll_size; i++) {
+        String collName = collection_name_list.get(i);
+        long coll_earliestDatestamp = Long.parseLong(((Element)oai_coll.item(i)).getAttribute(OAIXML.EARLIEST_DATESTAMP)); // Taken from the earliest datestamp field in buildcfg
+        if (coll_earliestDatestamp == 0) {
+        // try last modified
+        coll_earliestDatestamp = Long.parseLong(((Element)oai_coll.item(i)).getAttribute(OAIXML.LAST_MODIFIED));
+        //logger.info("@@@ Falling back to using collection " + collName + "'s lastmodified date as its earliest timestamp: " + coll_earliestDatestamp);
+        }
+        if (coll_earliestDatestamp > 0) {
+        earliestDatestamp = (earliestDatestamp > coll_earliestDatestamp)? coll_earliestDatestamp : earliestDatestamp;
+        }
+    }
+    }
+    */
+    
     if (earliestDatestamp == current_time) {
       logger.info("no collection had a real datestamp, using value from OAIConfig");

main/trunk/greenstone3/src/java/org/greenstone/gsdl3/util/OAIXML.java

@@ -89 +89 @@
   public static final String DELETED_RECORD = "deletedRecord";
   public static final String DESCRIPTION = "description";
-  public static final String EARLIEST_DATESTAMP = "earliestDatestamp";
+  public static final String EARLIEST_DATESTAMP = "earliestDatestamp"; // taken from buildconfig used as publishing date by RSS service
+  public static final String EARLIEST_OAI_DATESTAMP = "earliestOAIDatestamp"; // earliest timestamp of an OAI collection stored in oai-inf db
   public static final String GRANULARITY = "granularity";
   public static final String LAST_MODIFIED = "lastmodified";