Changeset 31903

Timestamp:

2017-08-18T19:28:42+12:00 (6 years ago)

Author:

ak19

Message:

I hope these are all the changes necessary on the runtime side of GS2 to get the OAI server validation working for GS2: instead of working out the earliest datetime stamp of the OAI repository by comparing the builddate in index/build.cfg of each OAI collection and selecting the earliest, the oai-inf.db is now storing the special earliesttimestamp record. The timestamp of this record represents its collection's earliest timestamp. And the earliest of these among all OAI collections is now the earliest datetime of the OAI repository.

Location:

main/trunk/greenstone2/runtime-src/src

Files:

• colservr/browsefilter.cpp (modified) (2 diffs)
• colservr/collectserver.cpp (modified) (2 diffs)
• colservr/comtypes.h (modified) (2 diffs)
• oaiservr/listidsaction.cpp (modified) (1 diff)
• oaiservr/oaiaction.cpp (modified) (5 diffs)
• protocol/recptprototools.cpp (modified) (2 diffs)

main/trunk/greenstone2/runtime-src/src/colservr/browsefilter.cpp

@@ -143 +143 @@
     if(parentnode == "oai") { // doing an OAI listidentifiers request
 
-      // open up the oai-inf db, if it exists, and return all IDs
-      // if it doesn't exist, proceed as usual
+      // open up the oai-inf db if it exists, and return all IDs *except* the special OID=OAI_EARLIESTTIMESTAMP_OID
+      // if oai-inf db doesn't exist, proceed as usual
       success = get_oaiinf_db_entries(response, err, logout); //adds any stuff in oai-inf db for the current OID to resultdoc.metadata
 
@@ -377 +377 @@
     
     resultdoc.OID = (*key_here);
-   
+    
+    // OAI_EARLIESTTIMESTAMP_OID is the OID of a special record that we'll ignore
+    // here in browsefilter.cpp, since it's not a doc.
+    // When the *metadata* for this special OID is requested, source.cpp will handle it
+    if(resultdoc.OID == OAI_EARLIESTTIMESTAMP_OID) {
+      ++key_here;
+      continue;
+    }
+
     if(!oaidb_ptr->getinfo(resultdoc.OID, oai_info)) {
       logout << text_t2ascii

main/trunk/greenstone2/runtime-src/src/colservr/collectserver.cpp

@@ -535 +535 @@
       // first check for oai metadata from the oai_db, if asked for it (if FROAI is set)
       if(((*source_here).s != NULL) &&
-         request.filterResultOptions & FROAI &&
+         (request.filterResultOptions & FROAI) &&
          ((*source_here).s->get_oai_metadata(request.requestParams, request.refParams,
                          request.getParents, request.fields, 
@@ -546 +546 @@
       }
 
+      // if OID is the special OAI specific OAI_EARLIESTTIMESTAMP_OID,
+      // then we'd have got its OAI meta above if we were requested to do so.
+      // Either way, we won't be additionally getting regular meta for this OID,
+      // as it's not a real doc OID, so we stop processing this OID here.
+      if((*resultdoc_here).OID == OAI_EARLIESTTIMESTAMP_OID) {
+        ++source_here;
+        if(processed == true) break;
+        else continue;
+      }
+  
       // We may or may not have got oai_meta (depends on if FROAI was set).
       // If we didn't get oai_meta, then deleted_status would still be "".

main/trunk/greenstone2/runtime-src/src/colservr/comtypes.h

@@ -48 +48 @@
 #endif
 
+
+const text_t OAI_EARLIESTTIMESTAMP_OID = "earliesttimestamp";
+         // https://www.tutorialspoint.com/cplusplus/cpp_constants_literals.htm
 
 enum comerror_t {noError, authenticationFailure, protocolError, 
@@ -113 +116 @@
   text_tmap      dynamic_classifiers;
   text_t         earliestDatestamp;   // For OAIserver validation. Time since linux Epoch, expressed as (long) integer value in string form
+  // This field stored the builddate value from a collection's index/build.cfg file.
+  // This field earliestDatestamp was used when oaiaction.calcEarliestDatestamp() needed to
+  // determine the earliest timestamp of the repository, by selecting the earliestDatestamp among
+  // all oai collections. Now it's no longer used by OAI, as that resorts to consulting oai-inf.db
+  // of each collection, which has a collection's earliest timestamp stored, to work out the
+  // earliest timestamp among all collections. But the build.cfg builddate stored in the
+  // earliestDatestamp field may still be useful in future for some purpose, so leaving it alone.
+  
 };
 

main/trunk/greenstone2/runtime-src/src/oaiservr/listidsaction.cpp

@@ -237 +237 @@
   text_tset        metadata;
   ofstream         logout("oai.log", ios::app);
+    // logout closed automatically by object destructor at this method's end:
+    // https://stackoverflow.com/questions/748014/do-i-need-to-manually-close-an-ifstream
   text_t           lastModified = "";
   text_t           deleted_status = "";

main/trunk/greenstone2/runtime-src/src/oaiservr/oaiaction.cpp

@@ -551 +551 @@
 }
 
+// works out the earliest date/timestamp of the OAI repository,
+// which compares and then selects the earliest timestamp value among all collections.
+// In the past, the earliest date/timestamp used to be the unix epoch,
+// later it became the earliest among each collection's builddate in build.cfg,
+// and now, after implementing the OAI deletion policy, it needed to become the earliest
+// among each collection's OAI_EARLIESTTIMESTAMP_OID record's timestamp value.
+
 text_t oaiaction::calcEarliestDatestamp(recptproto *protocol, oaiargs &params) {
     
@@ -570 +577 @@
     // (the earliestDatestamp for a collection has already been extracted from 
     // their build.cfg file at this point by collectserver::configure. The field
-    // is declared in comtypes.h)   
-    
+    // is declared in comtypes.h)
+    
+    // We've changed this again to grab the timestamp field of the OAI_EARLIESTTIMESTAMP_OID
+    // record of each collection's oai-inf.db (representing the time that collection was
+    // built for the very first time) and choosing the earliest among all of them.
     
     // Get a list of the OAI-enabled collections available
@@ -597 +607 @@
             collection_iterator++;
         }
+
+        ofstream         logout("oai.log", ios::app);
         
         // Now loop through the remaining collections
@@ -604 +616 @@
             collection_name = (*collection_iterator);
             
+            text_tset        metadata;
+            FilterResponse_t response;
+
+            // get timestamps from etc/oai-inf.<db> now, no longer from build.cfg
+            // request the special record with OID=OAI_EARLIESTTIMESTAMP_OID for the collection
+            bool status_ok = get_oai_info(OAI_EARLIESTTIMESTAMP_OID, collection_name, "", metadata, false, protocol, response, logout);
+            
+            if(status_ok) {
+              ResultDocInfo_t doc_info = response.docInfo[0];
+              text_t collEarliestTimestamp;
+              // the timestamp we want lives in the "lastmodified" field of the OAI_EARLIESTTIMESTAMP_OID record
+              this->getLastModifiedDate(doc_info, collEarliestTimestamp);
+              
+              if(earliestDatestamp == "") { // first earliestdatestamp we've seen for an oai collection
+                earliestDatestamp = collEarliestTimestamp;
+              } else if(collEarliestTimestamp < earliestDatestamp) {
+                earliestDatestamp = collEarliestTimestamp;
+              }
+            } /*else { // collection may not have oai-inf.db (yet), check build.cfg?
+                // No: no need to use the old way as fallback
+               
             ColInfoResponse_t cinfo;
             comerror_t err; 
@@ -618 +651 @@
                 }
             }
+              }
+              */
             collection_iterator++;
             
-        }       
+        }
+        logout.close();
     }
         

main/trunk/greenstone2/runtime-src/src/protocol/recptprototools.cpp

@@ -34 +34 @@
 // THIS FILE IS A CANDIDATE FOR REFACTORING: merge get_info methods
 
+// overloaded, to allow "custom" filter options.
 bool get_info (const text_t &OID, const text_t &collection, const text_t &lang,
            const text_tset &metadata, bool getParents,
@@ -76 +77 @@
 }
 
-// overloaded, to allow "custom" filter options.
 bool get_info (const text_t &OID, const text_t &collection, const text_t &lang,
            const text_tset &metadata, const OptionValue_tarray &options,