Changeset 20590 for gsdl/trunk/runtime-src/src/oaiservr/listsetsaction.cpp

Timestamp:

2009-09-11T11:54:17+12:00 (15 years ago)

Author:

mdewsnip

Message:

Completely rewrote the resumption token support, as its buginess finally tipped the "I can't stand it any more" scale...

File:

• gsdl/trunk/runtime-src/src/oaiservr/listsetsaction.cpp (modified) (3 diffs)

gsdl/trunk/runtime-src/src/oaiservr/listsetsaction.cpp

@@ -1 +1 @@
 #include "listsetsaction.h"
 
-#if defined(GSDL_USE_STL_H)
-#include <fstream.h>
-#else
-#include <fstream>
-#endif
-
+#include "resumptiontoken.h"
 #include "recptprototools.h"
 #include "oaitools.h"
+
 
 bool listsetsaction::validateAction(recptproto *protocol, oaiargs &params)
@@ -56 +52 @@
     // Check the resumption token is valid
     ResumptionToken token(params["resumptionToken"]);
-    if (true)  // TO DO: Fix this (the token.isValid() function is useless for ListSets)
+    if (token.isValid())
     {
       // Everything is fine, and we don't continue further because this is an exclusive argument
@@ -90 +86 @@
 bool listsetsaction::output_content(ostream &output, recptproto *protocol, oaiargs &params)
 {
-  // output the total list of classifier points
-
-  // variables required
-  text_t browseOID = "browse";
-  FilterResponse_t response;
-  comerror_t       err;
-  text_tarray &    collections = this->configuration->getCollectionsList();
-  text_tset        metadata;
-  ofstream         logout("oai.log", ios::app);
-  
-  // get a list of the collections available
-  //  protocol->get_collection_list(collections, err, output); 
-  if (collections.size() == 0) {
-    logout << "Found *no* OAI collections - check main.cfg for oaicollection items and read the OAI documentation.\n";
-  }
-
-  // check resumption token
-  int startSet = 0;
-  if (params["resumptionToken"] != "") {
-    ResumptionToken token(params["resumptionToken"]);
-    startSet = token.getPosition() - 1; // first document is said to be 1..
-  }
-  this->replyToken = NULL;
-
-  this->setNumber = 0;
+  // Reset variables
   this->setsOutput = 0;
-  for(int current_col = 0; current_col < collections.size(); ++current_col) {
-    // output the collection as a set, first, then its children
-    text_t gsdlCollect = collections[current_col];
-
+
+  text_t collection = "";
+
+  // Process the resumptionToken if there is one
+  if (params["resumptionToken"] != "")
+  {
+    ResumptionToken resumption_token(params["resumptionToken"]);
+    collection = resumption_token.getSet();
+  }
+
+  // Get a list of the collections available
+  text_tarray& collections = this->configuration->getCollectionsList();
+  if (collections.size() == 0)
+  {
+    return false;
+  }
+
+  // Find the starting collection
+  text_tarray::iterator collection_iterator = collections.begin();
+  while (collection_iterator != collections.end())
+  {
+    if (collection == "" || collection == *collection_iterator)
+    {
+      break;
+    }
+
+    collection_iterator++;
+  }
+
+  // Now loop through the remaining collections
+  while (collection_iterator != collections.end())
+  {
+    collection = (*collection_iterator);
+
+    // If we've output the desired number of records return a resumptionToken and we're done
     if (this->setsOutput == this->configuration->resumeAfter())
     {
-      this->replyToken = new ResumptionToken("", "", "");
-      this->replyToken->setPosition("", this->setNumber+1);
-      break;
-    }
-
-    if (this->setNumber >= startSet)
-    {
+      // Get the buildDate from the build.cfg file
+      ColInfoResponse_t cinfo;
+      comerror_t err;
+      protocol->get_collectinfo(collection, cinfo, err, cerr);
+
+      ResumptionToken resumption_token(cinfo.buildDate, collection, "", "", "", "");
+
+      // Don't add any whitespace around the resumption token as it can confuse harvesters/validators
+      output << "  <resumptionToken>" << resumption_token.getResumptionTokenString() << "</resumptionToken>" << endl;
+      return true;
+    }
+
+    // If output_content_for_col() returns false a resumption token has been output, so it's time to stop
+    if (output_content_for_col(output, protocol, params, collection) == false)
+    {
+      return true;
+    }
+
+    collection_iterator++;
+  }
+
+  return true;
+}
+
+
+bool listsetsaction::output_content_for_col(ostream &output, recptproto *protocol, oaiargs &params, text_t collection)
+{
+  text_t position = "";
+
+  // Process the resumptionToken if there is one
+  if (params["resumptionToken"] != "")
+  {
+    ResumptionToken resumption_token(params["resumptionToken"]);
+    position = resumption_token.getPosition();
+  }
+
+  // Get the list of sets in this collection
+  // Collections should not contain too many sets otherwise this will use a lot of time and memory
+  text_tset metadata;  // Must be empty for efficiency
+  FilterResponse_t sets_response;
+  get_children("browse", collection, "", metadata, false, protocol, sets_response, *this->logout);
+
+  // Find the starting position, if necessary
+  ResultDocInfo_tarray::iterator set_iterator = sets_response.docInfo.begin();
+  if (this->setsOutput == 0)
+  {
+    while (set_iterator != sets_response.docInfo.end())
+    {
+      if (position == "" || position == (*set_iterator).OID)
+      {
+    break;
+      }
+
+      set_iterator++;
+    }
+  }
+
+  // Output the collection as a set
+  if (position == "")
+  {
+    output << "  <set>" << endl;
+    output << "    <setSpec>" << collection << "</setSpec>" << endl;
+    output << "    <setName>" << collection << "</setName>" << endl;
+    output << "  </set>" << endl;
+    this->setsOutput++;
+  }
+
+  // Now loop through displaying the next matching records
+  while (set_iterator != sets_response.docInfo.end())
+  {
+    text_t set = (*set_iterator).OID;
+
+    // Only classifiers with supportsmemberof become OAI sets, for reasons I don't really understand
+    text_tset set_metadata;
+    set_metadata.insert("supportsmemberof");
+    set_metadata.insert("Title");
+    FilterResponse_t set_response;
+    get_info(set, collection, "", set_metadata, false, protocol, set_response, *this->logout);
+
+    if (set_response.docInfo[0].metadata["supportsmemberof"].values.size() > 0 && set_response.docInfo[0].metadata["supportsmemberof"].values[0] == "true")
+    {
+      // If we've output the desired number of records return a resumptionToken and we're done
+      if (this->setsOutput == this->configuration->resumeAfter())
+      {
+    // Get the buildDate from the build.cfg file
+    ColInfoResponse_t cinfo;
+    comerror_t err;
+    protocol->get_collectinfo(collection, cinfo, err, cerr);
+
+    ResumptionToken resumption_token(cinfo.buildDate, collection, "", "", "", set);
+
+    // Don't add any whitespace around the resumption token as it can confuse harvesters/validators
+    output << "  <resumptionToken>" << resumption_token.getResumptionTokenString() << "</resumptionToken>" << endl;
+    return false;
+      }
+
+      // Otherwise output this set and increment the count
+      text_t set_title = set_response.docInfo[0].metadata["Title"].values[0];
       output << "  <set>" << endl;
-      output << "    <setSpec>" << gsdlCollect << "</setSpec>" << endl;;
-      output << "    <setName>" << gsdlCollect << "</setName>" << endl;
+      output << "    <setSpec>" << collection << ":" << set << "</setSpec>" << endl;
+      output << "    <setName>" << collection << ":" << set_title << "</setName>" << endl;
       output << "  </set>" << endl;
       this->setsOutput++;
     }
-    setNumber++;
-
-    // get all the children of the (relevant) classifier data structures
-    get_children(browseOID, gsdlCollect, "", metadata, false, protocol, response, logout);
-    // and send them to the "recurse_content" list
-    for (int c = 0; c < response.numDocs; ++c) {
-      this->recurse_content(output, protocol, gsdlCollect, response.docInfo[c].OID, gsdlCollect, startSet);
-    }
-  }
-
-  // do a resumption token if required; errors cancel a token...
-  if (this->replyToken != NULL && this->errorType == "") {
-    // Don't add any whitespace around the resumption token as it can confuse harvesters/validators
-    output << "  <resumptionToken>" << this->replyToken->getToken() << "</resumptionToken>" << endl;
+
+    set_iterator++;
   }
 
   return true;
 }
-
-void listsetsaction::recurse_content(ostream &output, recptproto *protocol, text_t &collection, 
-                     const text_t &classifier, text_t setHierarchy, int startSet)
-{
-  // metadata for this call
-  FilterResponse_t response;
-  text_tset        metadata;
-  ofstream         logout("oai.log", ios::app);
-
-  if (this->setsOutput == this->configuration->resumeAfter())
-  {
-    this->replyToken = new ResumptionToken("", "", "");
-    this->replyToken->setPosition("", this->setNumber+1);
-    return;
-  }
-
-  metadata.insert("contains");
-  metadata.insert("Title");
-  metadata.insert("supportsmemberof");
-
-  // get the document information
-  if (!get_info(classifier, collection, "", metadata, false, protocol, response, logout)) {
-    //cerr << "recurse content: Bad identifier or protocol " << classifier << endl;
-    return;
-  }
-
-  // check for top-level classifiers, check if the set name includes a '.'; if 
-  // not, it is a top-level classifier: check for memberof support.  Those without
-  // memberof support will not be supported on OAI
-  if (findchar(classifier.begin(), classifier.end(), '.') == classifier.end()) {
-    if (response.docInfo[0].metadata["supportsmemberof"].values.size() > 0) {
-      text_t memberOf = response.docInfo[0].metadata["supportsmemberof"].values[0];
-      if (memberOf != "true") {
-    return;
-      }
-    }
-    else {
-      return;
-    }
-  }
-
-  MetadataInfo_tmap::iterator here = response.docInfo[0].metadata.begin();
-  MetadataInfo_tmap::iterator end  = response.docInfo[0].metadata.end();
-  text_t title;
-
-  while (here != end)
-  {
-    // Each set should only have one title - hence we only output one title here
-    // (it is a set title, not a collection)
-    if (here->first == "Title" && here->second.values.size() > 0) { 
-      title = here->second.values[0];
-    }
-
-    ++here;
-  }
-
-  // output the xml for this set; use the classifier id for the name
-  // if the title is blank
-  // curSet holds the colon-separated sequence of parent sets of the current set
-  text_t curSet;
-  if (this->setNumber >= startSet)
-  {
-    output << "  <set>" << endl;
-    text_t oai_classifier = classifier;
-    oaiclassifier::toOAI(collection, oai_classifier);
-    output << "    <setSpec>" << oai_classifier << "</setSpec>" << endl;
-    output << "    <setName>";
-    if (!title.empty()) {
-      curSet = setHierarchy + ":" + title;
-    }
-    else {
-      curSet = classifier; // Pretty much never gets here (shouldn't, at least)
-    }
-    output << curSet;
-    output << "</setName>" << endl;
-    output << "  </set>" << endl;
-    this->setsOutput++;
-  }
-  this->setNumber++;
-
-  // get the children of this classifier and iterate them
-  get_children(classifier, collection, "", metadata, false, protocol, response, logout);
-  for (int c = 0; c < response.numDocs; ++c) {
-    text_t child = response.docInfo[c].OID;
-
-    if (child == classifier)
-      continue;
-
-    // check for non classifier items and exclude them
-    text_t childHead;
-    text_t::const_iterator start = child.begin();
-    text_t::const_iterator here  = child.begin();
-    here += 2;
-    childHead = substr(start, here);
-
-    if (childHead != "CL")
-      continue;
-
-    // Recurse for "proper" classifier children. Pass curSet, the colon-separated list of 
-    // parent sets. curSet is pass-by-value, so that as we step out of recursion we remember
-    // old set hierarchies.
-    this->recurse_content(output, protocol, collection, child, curSet, startSet);
-  }
-  
-  return;
-}