Context Navigation

← Previous Change
Next Change →

abstractlistaction.cpp

Timestamp:

2009-09-11T11:54:17+12:00 (15 years ago)

Author:

mdewsnip

Message:

Completely rewrote the resumption token support, as its buginess finally tipped the "I can't stand it any more" scale...

File:

: 1 edited

gsdl/trunk/runtime-src/src/oaiservr/abstractlistaction.cpp (modified) (2 diffs)

Legend:

: Unmodified
: Added
: Removed

gsdl/trunk/runtime-src/src/oaiservr/abstractlistaction.cpp

-              r16835
+              r20590
 #include "abstractlistaction.h"
+#include "OIDtools.h"
 #include "recptprototools.h"
 #include "oaitools.h"
+//--------------------------------------------------------------------------------------------------
+bool abstractlistaction::check_classifier(recptproto *protocol, const text_t &collection, const text_t &set_name)
+{
+  text_tset metadata;
+  FilterResponse_t response;
+  return get_info(set_name, collection, "", metadata, false, protocol, response, *this->logout);
+}
 bool abstractlistaction::output_content(ostream &output, recptproto *protocol, oaiargs &params)
+{
+  bool   prevDocSeen;
+  ResumptionToken *token = NULL;
+  // start the call; clear down the total number of output documents
+  this->outputDocs = 0;
+  // We don't actually handle resumptionTokens yet; if we get one, ignore it
+  if (params["resumptionToken"] != "") {
+    token = new ResumptionToken(params["resumptionToken"]);
+  }
+  this->replyToken = NULL;
+  // if we've been asked for a set, then use it!
+  if (params["set"] != "") {
+    // get the children of this set
+    text_t gsdlSet = params["set"];
+    text_t gsdlCollect = "";
+    // given 'demo:CL2', toGSDL returns 'demo' in gsdlCollect and 'CL2' in gsdlSet. If there is no further
+    // set specified after the name of the collection however, then gsdlSet is empty.
+    oaiclassifier::toGSDL(gsdlCollect, gsdlSet);
+    // If gsdlSet is empty, then the user is requesting all the identifiers for the collection, so
+    // we simply output all docs via their oai_id tag. But if a specific subset IS requested, then
+    // use recurse_set() to traverse any sub classifiers to find the relevant docs.
+    if(gsdlSet == ""){
+      ColInfoResponse_t cinfo;
+      comerror_t err;
+      this->output_content_for_col(output, protocol, gsdlCollect, cinfo, err, params);
+    }
+    else {
+      if (this->check_classifier(protocol, gsdlCollect, gsdlSet)) {
+    this->recurse_set(output, protocol, gsdlCollect, gsdlSet, params, token);
+      }
+    }
+  }
+  // output all records in all hierarchies
+  else {
+    this->output_content_for_all(output, protocol, params);
+  }
+  // If - regardless of set required - no documents have been seen, throw an error.
+  if (this->configuration->getOAIVersion() >= 200 && this->prevDocSeen == false) {
+  // Reset variables
+  this->output_docs = 0;
+  text_t set_name = params["set"];
+  text_t position = params["position"];
+  // Process the resumptionToken if there is one
+  if (params["resumptionToken"] != "")
+  {
+    ResumptionToken resumption_token(params["resumptionToken"]);
+    set_name = resumption_token.getSet();
+    position = resumption_token.getPosition();
+  }
+  // Case for "set" argument present -- output just the records in the specified set
+  if (set_name != "")
+  {
+    // Separate the collection name and Greenstone classifier OID from the set name
+    text_t collection_name = "";
+    text_t gsdl_classifier_OID = set_name;
+    oaiclassifier::toGSDL(collection_name, gsdl_classifier_OID);
+    // If output_content_for_set() returns false a resumption token has been output, so it's time to stop
+    if (output_content_for_set(output, protocol, params, collection_name, gsdl_classifier_OID, set_name) == false)
+    {
+      return true;
+    }
+  }
+  // Case for no "set" argument present -- output all records in all collections
+  else
+  {
+    // Get a list of the collections available
+    text_tarray& collections = this->configuration->getCollectionsList();
+    if (collections.size() == 0)
+    {
+      return false;
+    }
+    // Get the current collection from the position value
+    text_t collection_name = "";
+    oaiclassifier::toGSDL(collection_name, position);
+    // Find the starting collection
+    text_tarray::iterator collection_iterator = collections.begin();
+    while (collection_iterator != collections.end())
+    {
+      if (collection_name == "" || collection_name == *collection_iterator)
+      {
+    break;
+      }
+      collection_iterator++;
+    }
+    // Now loop through the remaining collections
+    while (collection_iterator != collections.end())
+    {
+      // If output_content_for_set() returns false a resumption token has been output, so it's time to stop
+      if (output_content_for_set(output, protocol, params, *collection_iterator, "", "") == false)
+      {
+    return true;
+      }
+      collection_iterator++;
+    }
+  }
+  // If no records were output throw an error
+  if (this->configuration->getOAIVersion() >= 200 && this->output_docs == 0)
+  {
     errorType = "noRecordsMatch";
     this->output_error(output, errorType);
+    return false;
+  }
+  // do a resumption token if required; errors cancel a token...
+  if (this->replyToken != NULL && this->errorType == "") {
+    // Don't add any whitespace around the resumption token as it can confuse harvesters/validators
+    output << "  <resumptionToken>" << this->replyToken->getToken() << "</resumptionToken>" << endl;
+    return false;
+  }
 …
+}
+//--------------------------------------------------------------------------------------------------
+void abstractlistaction::output_content_for_col(ostream &output, recptproto *protocol, text_t &gsdlCollect,
+                        ColInfoResponse_t &cinfo, comerror_t &err, oaiargs &params)
+{ int startDoc = 0;
+  text_t metadataPrefix = params["metadataPrefix"];
+  // check resumption token
+  if (params["resumptionToken"] != "") {
+    ResumptionToken token(params["resumptionToken"]);
+    if (token.getCollection() == gsdlCollect) {
+      startDoc = token.getPosition() - 1; // first document is said to be 1..
+      metadataPrefix = "oai_dc";  // TO DO: This should come from the resumption token
+    }
+  }
+  // Get the OAI nodes from the info db file
+  text_t oai_root_node = "oai";
+bool abstractlistaction::output_content_for_set(ostream &output, recptproto *protocol, oaiargs &params, text_t collection_name, text_t gsdl_classifier_OID, text_t set_name)
+{
+  // Check if the set is actually a collection
+  if (gsdl_classifier_OID == "")
+  {
+    gsdl_classifier_OID = "oai";
+  }
+  text_t metadata_prefix = params["metadataPrefix"];
+  text_t from = params["from"];
+  text_t until = params["until"];
+  text_t position = "";
+  // Process the resumptionToken if there is one
+  if (params["resumptionToken"] != "")
+  {
+    ResumptionToken resumption_token(params["resumptionToken"]);
+    metadata_prefix = resumption_token.getMetadataPrefix();
+    from = resumption_token.getFrom();
+    until = resumption_token.getUntil();
+    position = resumption_token.getPosition();
+  }
+  // Get the list of identifiers in this collection
+  // Collections should not contain too many identifiers otherwise this will use a lot of time and memory
   text_tset metadata;  // Must be empty for efficiency
+  FilterResponse_t identifiers_response;
+  get_children(gsdl_classifier_OID, collection_name, "", metadata, false, protocol, identifiers_response, *this->logout);
+  // Find the starting position, if necessary
+  ResultDocInfo_tarray::iterator identifier_iterator = identifiers_response.docInfo.begin();
+  if (output_docs == 0)
+  {
+    while (identifier_iterator != identifiers_response.docInfo.end())
+    {
+      if (position == "" || position == (collection_name + ":" + (*identifier_iterator).OID))
+      {
+    break;
+      }
+      identifier_iterator++;
+    }
+  }
+  // Now loop through displaying the next matching records
+  while (identifier_iterator != identifiers_response.docInfo.end())
+  {
+    position = (*identifier_iterator).OID;
+    text_t document_OID = position;
+    if (starts_with(document_OID, "oai."))
+    {
+      document_OID = oaiclassifier::getGSDL_OID(collection_name, document_OID, protocol, *this->logout);
+    }
+    // Check this OID is in the (optional) date range specified
+    if (this->in_date_range(output, protocol, params, collection_name, document_OID, from, until))
+    {
+      // If we've output the desired number of records return a resumptionToken and we're done
+      if (this->output_docs == this->configuration->resumeAfter())
+      {
+    // Get the buildDate from the build.cfg file
+    ColInfoResponse_t cinfo;
+    comerror_t err;
+    protocol->get_collectinfo(collection_name, cinfo, err, cerr);
+    ResumptionToken resumption_token(cinfo.buildDate, set_name, metadata_prefix, from, until, collection_name + ":" + position);
+    // Don't add any whitespace around the resumption token as it can confuse harvesters/validators
+    output << "  <resumptionToken>" << resumption_token.getResumptionTokenString() << "</resumptionToken>" << endl;
+    return false;
+      }
+      // Otherwise output this record and increment the count
+      this->output_document(output, protocol, collection_name, document_OID, metadata_prefix);
+      this->output_docs++;
+    }
+    identifier_iterator++;
+  }
+  return true;
+}
+bool abstractlistaction::in_date_range(ostream &output, recptproto *protocol, oaiargs &params,
+                       text_t& collection, text_t oai_OID, text_t from, text_t until)
+{
+  // If no "from" or "until" value is specified every record matches, so we don't need to go any further
+  if (from == "" && until == "")
+  {
+    return true;
+  }
+  // Get the datestamp from the document as sections do not have this metadata
+  text_t document_OID;
+  get_top(oai_OID, document_OID);
+  // Request the lastmodified value for this document
+  text_tset metadata;
+  metadata.insert("lastmodified");
   FilterResponse_t response;
+  get_children(oai_root_node, gsdlCollect, "", metadata, false, protocol, response, *this->logout);
+  // If numDocs is 0, do nothing - this->prevDocSeen will stay false if this is the only collection
+  // looked at, or will keep whatever value it had prior to this col (ensures that if the flag has
+  // been set to true by a previous collection that this won't overwrite it to be false).
+  if (response.docInfo.size() > 0) {
+    int errorCount      = 0; // Count the number of errors found in the given collection
+    for (long i = startDoc; i < response.docInfo.size(); ++i) {
+      if (errorCount > 3) { // If num errors reaches the cut-off value, bail.
+    cerr << "Error: too many records(" << errorCount << ") in the " << gsdlCollect
+         << " collection have invalid or non-existant oai_ids - skipping remainder of collection.\n";
+    return;
+      }
+      text_t oai_id = "oai.";
+      oai_id += i;
+      text_t gsdl_id = oaiclassifier::getGSDL_OID(gsdlCollect, oai_id, protocol, *this->logout);
+      if (gsdl_id == "") { // If the string is empty, then the document didn't have an oai_id, so
+    ++errorCount;     // increase error count
+    continue;
+      }
+      // Check that the item with the 0ID 'gsdl_id' has a lastmodified field that fits within
+      // the required date range (if specified).
+      if (this->inDateRange(params["from"], params["until"], gsdlCollect, gsdl_id, protocol, output)) {
+    if (this->output_document(output, protocol, gsdlCollect, gsdl_id, metadataPrefix)) {
+      // this should be an IF statement, where prevDocSeen is only set to true if the above
+      // function call returns true (indicating that the doc supported the metadata prefix) but
+      // for some reason this is always false. This means that if no doc in the requested set supports
+      // the metadata format, the "no records match" error that should be thrown won't be...
+      //
+      // GRB: the above comment is no longer true; proper checks are made
+      this->prevDocSeen = true;
+      ++this->outputDocs;
+    }
+      }
+      // if we've output the number of resumption documents; prepare a resumptionToken
+      if (this->outputDocs == this->configuration->resumeAfter()) {
+    this->replyToken = new ResumptionToken(gsdlCollect, "", "");
+    this->replyToken->setPosition("", i+2);
+    break;
+      }
+    }
+    cinfo.clear(); // Clear for next collection to use (if there is one).
+  }
+}
+//--------------------------------------------------------------------------------------------
+// Returns true if at least one document record is found
+void abstractlistaction::output_content_for_all(ostream &output, recptproto *protocol, oaiargs &params)
+{
+  ColInfoResponse_t cinfo;
+  comerror_t        err;
+  text_tarray       collections;
+  text_t            gsdlCollect = "";
+  ResumptionToken   *token = NULL;
+  // get a list of the collections available
+  collections = this->configuration->getCollectionsList();
+  //  protocol->get_collection_list(collections, err, output);
+  if (params["resumptionToken"] != "") {
+    token = new ResumptionToken(params["resumptionToken"]);
+  }
+  for(int current_col = 0; current_col < collections.size(); ++current_col){
+    gsdlCollect = collections[current_col];
+    // ignore all leading collections before the one that matches the resumptiontoken
+    if (token != NULL &&
+    token->getCollection() != gsdlCollect)
+    { continue;
+    }
+    this->output_content_for_col(output, protocol, gsdlCollect, cinfo, err, params);
+    // once we've output at least one collection, continue
+    // outputting all others until the resumption total hits
+    token = NULL;
+    if (this->outputDocs == this->configuration->resumeAfter()) {
+      break;
+    }
+  }
+}
+//-------------------------------------------------------------------------------------------------
+// Check that the requested from/until dates don't include a time, as this would be asking for too
+// fine a level of granularity, one that greenstone doesn't support. An OAI error must be thrown.
+/*
+bool abstractlistaction::granularityTooFine(text_t &from, text_t &until)
+{
+  if (from != "" && from.){
+  }
+}
+*/
+//-------------------------------------------------------------------------------------------------
+bool abstractlistaction::check_classifier(recptproto *protocol, const text_t &collection,
+                      const text_t &classifier)
+{ text_t topClass;
+  FilterResponse_t response;
+  text_tset        metadata;
+  ofstream         logout("oai.log", ios::app);
+  // exclude false children of a top-level classifier immediately...
+  if (!get_info(classifier, collection, "", metadata, false, protocol, response, logout)) {
+    return false;
+  }
+  // now check the top-level parent
+  metadata.insert("supportsmemberof");
+  text_t::const_iterator dot = findchar(classifier.begin(), classifier.end(), '.');
+  if (dot != classifier.end()) {
+    topClass = substr(classifier.begin(), dot);
+  }
+  else {
+    topClass = classifier;
+  }
+  if (!get_info(topClass, collection, "", metadata, false, protocol, response, logout)) {
+    return false;
+  }
+  if (response.docInfo[0].metadata["supportsmemberof"].values.size() == 0) {
+    return false;
+  }
+  if (response.docInfo[0].metadata["supportsmemberof"].values[0] != "true") {
+    return false;
+  }
+  if (!get_info(document_OID, collection, "", metadata, false, protocol, response, *this->logout))
+  {
+    return false;
+  }
+  text_t last_modified_date;
+  this->getLastModifiedDate(response.docInfo[0], last_modified_date);
+  // Check this record is not before the "from" value, if it exists
+  if (from != "" && last_modified_date < from)
+  {
+    // Too early
+    return false;
+  }
+  // Check this record is not after the "until" value, if it exists
+  if (until != "" && last_modified_date > until)
+  {
+    // Too late
+    return false;
+  }
+  // Just right
   return true;
+}
-void abstractlistaction::recurse_set(ostream &output, recptproto *protocol, const text_t &collection,
-                     const text_t &classifier, oaiargs &params, ResumptionToken *resumptionToken)
+{
-  // metadata for this call
-  FilterResponse_t response;
-  text_tset        metadata;
-  ofstream         logout("oai.log", ios::app);
-  text_t           from = params["from"];
-  text_t           until = params["until"];
-  text_t           metadataPrefix = params["metadataPrefix"];
-  //  ResumptionToken  resumptionToken(params["resumptionToken"]);
-  int              startPos = 0;
-  // This is a recursive function, and so just because the current set is empty doesn't mean we necessarily
-  // want to throw a 'noRecordsMatch' error; another set (parent/sibling/child) may have had documents. It
-  // is therefore not enough to check that the response object in the current iteration has no docs - we
-  // must also verify that NO OTHER set has had any documents. This is done with the 'prevDocSeen' flag.
-  // It is set to FALSE initially, but as soon as we see a set that isn't empty, it is set to TRUE. The
-  // 'noRecordsMatch' error will only be thrown if, after all appropriate sets have been recursed into,
-  // the 'prevDocSeen' flag is still FALSE. The function returns false if no docs were seen, allowing us to
-  // throw the noRecordsMatch error.
-  //  bool prevDocSeen = false;
-  get_children(classifier, collection, "", metadata, false, protocol, response, *this->logout);
-  if (params["resumptionToken"] != "") {
-    // if we're at a resumptionToken
-    if (classifier == resumptionToken->getNode()) {
-      startPos = resumptionToken->getPosition();
+    }
-    else {
-      text_t fullNode = resumptionToken->getNode();
-      text_t::iterator leafIter = fullNode.begin() + classifier.size();
-      // if the next character isn't a dot, blow up!
-      if (*leafIter != '.') {
-    // fatal error;
-    exit(1);
+      }
-      // get the first '.' after the current classifier point;
-      text_t::iterator separator = findchar(leafIter + 1, fullNode.end(), '.');
-      // now, create a new subpath
-      text_t nextNode = substr(fullNode.begin(), separator);
-      // seek forward; TODO: improve performance of this
-      for (int c = 0; c < response.numDocs; ++c) {
-    if (response.docInfo[c].OID == nextNode) {
-      startPos = c;
-      break;
+    }
+      }
+    }
-    // We need to subtract one from the startPos value to turn it into an index value
-    startPos--;
+  }
-  for (int c = startPos; c < response.numDocs; ++c) {
-    text_t child = response.docInfo[c].OID;
-    // distinguish classifiers and documents by checking whether OID
-    // starts with CL or not
-    text_t childHead;
-    text_t::const_iterator start = child.begin();
-    text_t::const_iterator here  = child.begin();
-    here += 2;
-    childHead = substr(start, here);
-    // documents we output now
-    if (childHead != "CL") {
-      // Check that the item with the 0ID 'gsdl_id' has a lastmodified field that fits
-      // within the required date range (if specified)
-      if (this->inDateRange(from, until, collection, child, protocol, output)) {
-    // TODO: check that the document can be disseminated in the required metadataPrefix
-    if (this->output_document(output, protocol, collection, child, metadataPrefix)) {
-      this->prevDocSeen = true;
-      ++this->outputDocs;
+    }
+      }
+    }
-    // children which are classifiers are recursed
-    else {
-      if (resumptionToken != NULL) {
-    int depth = countchar(classifier.begin(), classifier.end(), '.');
-    resumptionToken->setOffset(depth, c+2);
+      }
-      this->recurse_set(output, protocol, collection, child, params, resumptionToken);
+    }
-    if (this->outputDocs == this->configuration->resumeAfter()) {
-      this->replyToken = new ResumptionToken(collection, params["set"], "");
-      this->replyToken->setPosition(classifier, c+2);
-      break;
+    }
+  }
+}

Note: See TracChangeset for help on using the changeset viewer.

Context Navigation

Changeset 20590 for gsdl/trunk/runtime-src/src/oaiservr/abstractlistaction.cpp

Legend:

gsdl/trunk/runtime-src/src/oaiservr/abstractlistaction.cpp

Download in other formats: