Changeset 31387 for main/trunk/greenstone2

Timestamp:

2017-02-08T18:31:18+13:00 (7 years ago)

Author:

ak19

Message:

Round 1 of commits for getting OAI deletion policy to work with GS2 (server end). The perl code writing out the OAI db and the GS3 server code implementing the deletion policy had already been completed earlier (end 2016).

Location:

main/trunk/greenstone2/runtime-src/src

Files:

• colservr/browsefilter.cpp (modified) (8 diffs)
• colservr/browsefilter.h (modified) (3 diffs)
• colservr/collectserver.cpp (modified) (3 diffs)
• colservr/collectset.cpp (modified) (9 diffs)
• colservr/comtypes.h (modified) (1 diff)
• colservr/filter.cpp (modified) (2 diffs)
• colservr/filter.h (modified) (1 diff)
• colservr/source.cpp (modified) (9 diffs)
• colservr/source.h (modified) (4 diffs)
• oaiservr/abstractlistaction.cpp (modified) (2 diffs)
• oaiservr/identifyaction.cpp (modified) (1 diff)
• oaiservr/listidsaction.cpp (modified) (2 diffs)
• oaiservr/metaformatsaction.cpp (modified) (1 diff)
• oaiservr/oaiaction.cpp (modified) (4 diffs)
• oaiservr/oaiaction.h (modified) (2 diffs)
• oaiservr/oaitools.cpp (modified) (1 diff)
• oaiservr/recordaction.cpp (modified) (5 diffs)
• protocol/recptprototools.cpp (modified) (5 diffs)
• protocol/recptprototools.h (modified) (2 diffs)

main/trunk/greenstone2/runtime-src/src/colservr/browsefilter.cpp

@@ -26 +26 @@
 #include "browsefilter.h"
 #include "fileutil.h"
+#include <assert.h>
 
 
 browsefilterclass::browsefilterclass () {
   db_ptr = NULL;
+  oaidb_ptr = NULL;
 
   // -- onePerQuery StartResults   integer
@@ -60 +62 @@
 }
 
-browsefilterclass::~browsefilterclass () {
-}
+browsefilterclass::~browsefilterclass () {}
 
 void browsefilterclass::configure (const text_t &key, const text_tarray &cfgline) {
@@ -75 +76 @@
   if (!filterclass::init(logout)) return false;
 
-  if (db_ptr == NULL) {
+  if (db_ptr == NULL || oaidb_ptr == NULL) {
     // most likely a configuration problem
     logout << text_t2ascii 
@@ -93 +94 @@
   }
 
+  oaidb_filename = resolve_oaidb_filename(oaidb_ptr->getfileextension());
+
   return true;
 }
@@ -104 +107 @@
   response.clear ();
   err = noError;
-  if (db_ptr == NULL) {
-    // most likely a configuration problem
-    logout << text_t2ascii 
-       << "configuration error: browsefilter contains a null dbclass\n\n";
-    err = configurationError;
-    return;
-  }
-
-  // open the database
-  db_ptr->setlogout(&logout);
-  if (!db_ptr->opendatabase (db_filename, DB_READER, 100, false)) {
-    // most likely a system problem (we have already checked that the database exists)
-    logout << text_t2ascii
-       << "system problem: open on database \""
-       << db_filename << "\" failed\n\n";
-    err = systemProblem;
-    return;
-  }
 
   // get the browse parameters
@@ -146 +131 @@
   }
 
+  // if we're only working on oai, open oai_db, no need to work with index_db in browsefilter.cpp
+  // (but source.cpp uses both oai-inf.db and index.db to get metadata for OAI request)
+  // If we can't open the oai-inf db, this can be because it didn't exist in older versions of GS
+  // in that case, proceed as usual, using the index db.
+  if((request.filterResultOptions & FROAI)) { // OAI request    
+    bool success = false;
+
+    if(parentnode == "oai") { // doing an OAI listidentifiers request
+
+      // open up the oai-inf db, if it exists, and return all IDs
+      // if it doesn't exist, proceed as usual
+      success = get_oaiinf_db_entries(response, err, logout); //adds any stuff in oai-inf db for the current OID to resultdoc.metadata
+
+      response.numDocs = response.docInfo.size();
+      response.isApprox = Exact;
+    }
+    if (success) return; // oai request successfully completed with oai-inf.db, no need to open index_db
+  } 
+
+  // Since we're here, it means we're not doing anything oai (or oai-inf.db did not exist/open)
+  // So we don't need to work with oai_db. Instead, work with index_db:
+
+  if (db_ptr == NULL) {
+    // most likely a configuration problem
+    logout << text_t2ascii 
+       << "configuration error: browsefilter contains a null index dbclass\n\n";
+    err = configurationError;
+    return;
+  }
+
+  // open the database
+  db_ptr->setlogout(&logout);
+  if (!db_ptr->opendatabase (db_filename, DB_READER, 100, false)) {
+    // most likely a system problem (we have already checked that the database exists)
+    logout << text_t2ascii
+       << "system problem: open on database \""
+       << db_filename << "\" failed\n\n";
+    err = systemProblem;
+    return;
+  }
+
   infodbclass info;
 
@@ -158 +184 @@
       (request.filterResultOptions & FRmetadata)) {
     if (!db_ptr->getinfo(parentnode, info)) {
-      // didn't find the node
+      // didn't find the node in index db
       logout << text_t2ascii
          << "warning: lookup for node \"" << parentnode 
@@ -308 +334 @@
   response.isApprox = Exact;
 }
+
+bool browsefilterclass::get_oaiinf_db_entries(FilterResponse_t &response, 
+                      comerror_t &err, ostream &logout) 
+{
+  outconvertclass text_t2ascii;
+  
+  //logout << text_t2ascii << "browsefilterclass::get_oaiinf_db_entries\n";   
+
+  // ONLY if we're doing any OAI stuff (FROAI will be set then) will we be here
+  // So next try to open the oai-inf db if it exists for this collection
+
+  
+  if (!file_exists(oaidb_filename)) { // if the oaidb file doesn't even exist, let's not bother with oaidb
+    
+    logout << text_t2ascii
+       << "warning: collection's oai-inf database \"" << oaidb_filename << "\" does not exist\n\n";    
+    return false;
+
+  } else { // let's try opening the oaidb file
+    oaidb_ptr->setlogout(&logout);
+    if (!oaidb_ptr->opendatabase (oaidb_filename, DB_READER, 100, false)) {
+      // most likely a system problem (we have already checked that the database exists just above)
+      logout << text_t2ascii
+         << "system problem: open on database \""
+         << oaidb_filename << "\" failed\n\n";
+      err = systemProblem;
+      return false;
+    } // now we've opened the oai-inf db file successfully
+  }
+
+  infodbclass oai_info;
+  ResultDocInfo_t resultdoc;
+
+  text_tarray keys = oaidb_ptr->getkeys();
+
+  text_tarray::iterator key_here = keys.begin();
+  text_tarray::iterator key_end = keys.end();
+  while (key_here != key_end) {    
+    
+    resultdoc.OID = (*key_here);
+   
+    if(!oaidb_ptr->getinfo(resultdoc.OID, oai_info)) {
+      logout << text_t2ascii
+         << "warning: lookup for node \"" << resultdoc.OID
+         << "\" in etc/oai-inf db failed for browsefilter.\n\n";
+    } else {
+      //logout << text_t2ascii << "@@@@ found node \"" << resultdoc.OID << "\" in etc/oai-inf db.\n\n";
+      
+      resultdoc.metadata["oaiinf.status"].isRef = false;
+      resultdoc.metadata["oaiinf.status"].values.push_back(oai_info["status"]);
+      resultdoc.metadata["oaiinf.timestamp"].isRef = false;
+      resultdoc.metadata["oaiinf.timestamp"].values.push_back(oai_info["timestamp"]);
+      resultdoc.metadata["oaiinf.datestamp"].isRef = false;
+      resultdoc.metadata["oaiinf.datestamp"].values.push_back(oai_info["datestamp"]);
+    }
+    
+    response.docInfo.push_back(resultdoc);
+    ++key_here;    
+    
+  }
+
+  // we're done with oai-inf db
+
+  oaidb_ptr->closedatabase(); // don't leave files open
+
+  return true;
+}

main/trunk/greenstone2/runtime-src/src/colservr/browsefilter.h

@@ -41 +41 @@
   text_t indexstem;
   dbclass *db_ptr;
+  text_t oaidb_filename;
+  dbclass *oaidb_ptr;
 
 public:
@@ -50 +52 @@
   virtual void configure (const text_t &key, const text_tarray &cfgline);
 
-  // the db ptr remains the responsability of the calling code and
-  // should be destroyed after this browsefilter is destroyed
+  // the db ptr and oaidb ptr remain the responsibility of the calling code and should
+  // be destroyed after this browsefilter is destroyed (source.h class destroys these)
   void set_db_ptr (dbclass *db_ptr_arg) { db_ptr = db_ptr_arg; }
+  void set_oaidb_ptr (dbclass *oaidb_ptr_arg) { oaidb_ptr = oaidb_ptr_arg; }
 
   bool init (ostream &logout);
@@ -59 +62 @@
            FilterResponse_t &response,
            comerror_t &err, ostream &logout);
+
+ protected:
+  bool get_oaiinf_db_entries(FilterResponse_t &response, 
+                 comerror_t &err, ostream &logout);
 };
 

main/trunk/greenstone2/runtime-src/src/colservr/collectserver.cpp

@@ -515 +515 @@
     thisfilter->filter (request, response, err, logout);
     if (err != noError) return;
+
     // fill in the metadata for each of the OIDs (if it is requested)
     if (request.filterResultOptions & FRmetadata) {
+
       bool processed = false;
       ResultDocInfo_tarray::iterator resultdoc_here = response.docInfo.begin();
       ResultDocInfo_tarray::iterator resultdoc_end = response.docInfo.end();
       while (resultdoc_here != resultdoc_end) {
+
+    text_t deleted_status = "";
+    bool append_metadata = (request.filterResultOptions & FROAI) ? true : false;
+
     // try each of the sources in turn
     sourcelistclass::iterator source_here = sources.begin();
@@ -526 +532 @@
     while (source_here != source_end) {
       assert ((*source_here).s != NULL);
+
+      // first check for oai metadata from the oai_db, if asked for it (if FROAI is set)
+      if(((*source_here).s != NULL) &&
+         request.filterResultOptions & FROAI &&
+         ((*source_here).s->get_oai_metadata(request.requestParams, request.refParams,
+                         request.getParents, request.fields, 
+                         (*resultdoc_here).OID, deleted_status, (*resultdoc_here).metadata, 
+                         err, logout))) {
+        
+        if (err != noError) return;
+
+        processed = true;
+      }
+
+      // We may or may not have got oai_meta (depends on if FROAI was set).
+      // If we didn't get oai_meta, then deleted_status would still be "".
+      // If we did get oai_meta, and if the deleted_status for the OID was D for deleted entry,
+      // don't bother getting any other metadata, as there will be no entry for that OID in index db.
+      
+      // Note that if we did get oai_meta and OID marked as existing, we're in append_mode:
+      // don't let get_metadata() clear the metadata list, as there's already stuff in there
+      //if(deleted_status == "E") append_metadata = true;
+
       if (((*source_here).s != NULL) &&
+          deleted_status != "D" &&
           ((*source_here).s->get_metadata(request.requestParams, request.refParams,
                           request.getParents, request.fields, 
                           (*resultdoc_here).OID, (*resultdoc_here).metadata, 
-                          err, logout))) {
-        if (err != noError) return;
-        processed = true;
-        break;
+                          err, logout, append_metadata))) {
+        if (err != noError) return; // check for errors again     
+        
+        processed = processed || true; // processed would not have been set yet if not doing FROAI. Set now.
+        // OR-ing isn't necessary, but indicates some consideration of both get oai meta & get meta success
       }
+      
+      if(processed) break;
+
       ++source_here;
     }
@@ -548 +582 @@
       }
     }  
-  
+    
     err = noError;
   }

main/trunk/greenstone2/runtime-src/src/colservr/collectset.cpp

@@ -335 +335 @@
 
     // Create a dbclass of the correct type
-    dbclass *db_ptr = NULL;
+    dbclass *db_ptr = NULL; // index database in index/text
+    dbclass *oaidb_ptr = NULL; // etc/oai-inf database
 
     if (infodbtype == "sqlite")
@@ -342 +343 @@
     sqlitedbclass *sql_db_ptr = new sqlitedbclass(gsdlhome);
     db_ptr = sql_db_ptr;
+
+    sqlitedbclass *sql_oaidb_ptr = new sqlitedbclass(gsdlhome);
+    oaidb_ptr = sql_oaidb_ptr;
 
     // add a sql browse filter
@@ -364 +368 @@
     mssqldbclass *mssql_db_ptr = new mssqldbclass(gsdlhome);
     db_ptr = mssql_db_ptr;
+
+    mssqldbclass *mssql_oaidb_ptr = new mssqldbclass(gsdlhome);
+    oaidb_ptr = mssql_oaidb_ptr;
 
     // add a sql browse filter
@@ -380 +387 @@
     jdbmnaiveclass *jdbm_db_ptr = new jdbmnaiveclass(gsdlhome);
     db_ptr = jdbm_db_ptr;
+
+    jdbmnaiveclass *jdbm_oaidb_ptr = new jdbmnaiveclass(gsdlhome);
+    oaidb_ptr = jdbm_oaidb_ptr;
 #else
     cerr << "Warning: infodbtype of 'jdbm' was not compiled in to " << endl;
@@ -392 +402 @@
 #ifdef USE_GDBM
       db_ptr = new gdbmclass(gsdlhome);
+      oaidb_ptr = new gdbmclass(gsdlhome);
 #else
     cerr << "Warning: infodbtype of 'gdbm' was not compiled in to " << endl;
@@ -405 +416 @@
     browsefilterclass *browsefilter = new browsefilterclass();
     browsefilter->set_db_ptr(db_ptr);
+    // set the oaidb ptr for the browse filter. As with the db_ptr, the oaidb_ptr
+    // is not managed and deleted by the browsefilter class, but by the sourceclass
+    browsefilter->set_oaidb_ptr(oaidb_ptr);
     cserver->add_filter (browsefilter);  
 
@@ -420 +434 @@
       mgsourceclass *mgsource = new mgsourceclass ();
       mgsource->set_db_ptr(db_ptr);
+      mgsource->set_oaidb_ptr(oaidb_ptr);
       mgsource->set_textsearchptr (mgsearch);
       cserver->add_source (mgsource);
@@ -439 +454 @@
       mgppsourceclass *mgppsource = new mgppsourceclass ();
       mgppsource->set_db_ptr(db_ptr);
+      mgppsource->set_oaidb_ptr(oaidb_ptr);
       mgppsource->set_textsearchptr (mgppsearch);
       cserver->add_source (mgppsource);
@@ -459 +475 @@
       lucenesourceclass *lucenesource = new lucenesourceclass ();
       lucenesource->set_db_ptr(db_ptr);
+      lucenesource->set_oaidb_ptr(oaidb_ptr);
       lucenesource->set_textsearchptr (lucenesearch);
       cserver->add_source (lucenesource);

main/trunk/greenstone2/runtime-src/src/colservr/comtypes.h

@@ -243 +243 @@
 // used to indicate a full text browse query for mgpp
 #define FRfullTextBrowse  128
+#define FROAI  256
 
 struct FilterRequest_t {

main/trunk/greenstone2/runtime-src/src/colservr/filter.cpp

@@ -61 +61 @@
 }
 
+text_t filterclass::getcollectionpath()
+{
+  text_t resolved_filename;
+  
+  if (gsdlhome==dbhome) {
+    // dbhome has defaulted to gsdlhome which we take to means the 
+    // database has been specifically moved out of gsdlhome area.
+    // => it should be whereever collecthome is set to
+    resolved_filename = filename_cat(collecthome, collection);
+  }
+  else {
+    // dbhome is explicitly set to something other than gsdlhome
+    // => use dbhome
+    resolved_filename = filename_cat(dbhome, "collect", collection);
+  }
+
+  return resolved_filename;
+}
+
 text_t filterclass::resolve_db_filename(const text_t& idx,
                     const text_t& file_ext)
 {
+  /*
   text_t resolved_filename;
 
@@ -77 +97 @@
     resolved_filename = filename_cat(dbhome, "collect", collection, "index", "text", idx);
   }
+  */
+  
+  text_t resolved_filename = filename_cat(getcollectionpath(), "index", "text", idx);
+  resolved_filename += file_ext;
+
+  return resolved_filename;
+}
+
+text_t filterclass::resolve_oaidb_filename(const text_t& file_ext)
+{
+  text_t resolved_filename = filename_cat(getcollectionpath(), "etc", "oai-inf");
 
   resolved_filename += file_ext;

main/trunk/greenstone2/runtime-src/src/colservr/filter.h

@@ -69 +69 @@
  protected:
   text_t resolve_db_filename(const text_t& idx,const text_t& file_ext);
-
+  text_t resolve_oaidb_filename(const text_t& file_ext);
+  text_t getcollectionpath();
 
 };

main/trunk/greenstone2/runtime-src/src/colservr/source.cpp

@@ -33 +33 @@
 {
   db_ptr = NULL;
+  oaidb_ptr = NULL;
   textsearchptr = NULL;
   classname = "source";
@@ -41 +42 @@
   if (db_ptr != NULL) delete db_ptr;
   if (textsearchptr != NULL) delete textsearchptr;
+
+  // http://stackoverflow.com/questions/677620/do-i-need-to-explicitly-call-the-base-virtual-destructor
+  // Answer: "No, destructors are called automatically [by the derived destructor] in the reverse
+  // order of construction. (Base classes last). Do not call base class destructors."
+  // The link also explains why the destructors should be declared as virtual in the header files.
+
+  // now delete the oaidb object pointed to by the oaidb_ptr too
+  if (oaidb_ptr != NULL) delete oaidb_ptr;
+  oaidb_ptr = NULL;
 }
 
@@ -78 +88 @@
 }
 
+text_t sourceclass::getcollectionpath()
+{
+  text_t resolved_filename;
+  
+  if (gsdlhome==dbhome) {
+    // dbhome has defaulted to gsdlhome which we take to means the 
+    // database has been specifically moved out of gsdlhome area.
+    // => it should be whereever collecthome is set to
+    resolved_filename = filename_cat(collecthome, collection);
+  }
+  else {
+    // dbhome is explicitly set to something other than gsdlhome
+    // => use dbhome
+    resolved_filename = filename_cat(dbhome, "collect", collection);
+  }
+
+  return resolved_filename;
+}
+
 text_t sourceclass::resolve_db_filename(const text_t& idx,
                     const text_t& file_ext)
@@ -91 +120 @@
   // are also drawn from one object
 
+  /*
   text_t resolved_filename;
 
@@ -105 +135 @@
     resolved_filename = filename_cat(dbhome, "collect", collection, "index", "text", idx);
   }
-
+  */
+
+  text_t resolved_filename = filename_cat(getcollectionpath(), "index", "text", idx);
   resolved_filename += file_ext;
 
@@ -111 +143 @@
 }
 
-
+text_t sourceclass::resolve_oaidb_filename(const text_t& file_ext)
+{
+  text_t resolved_filename = filename_cat(getcollectionpath(), "etc", "oai-inf");
+
+  resolved_filename += file_ext;
+
+  return resolved_filename;
+}
 
 // init should be called after all the configuration is done but
@@ -172 +211 @@
   }
   
+  oaidb_filename = resolve_oaidb_filename(oaidb_ptr->getfileextension());
+
   return true;
 }
@@ -207 +248 @@
 }
 
+// get_oai_metadata fills out the oai metadata from oaidb if possible
+// if it is not responsible for the given OID then it will return false.
+bool sourceclass::get_oai_metadata (const text_t &requestParams, const text_t &refParams, 
+                bool getParents, const text_tset &fields, 
+                const text_t &OID, text_t &deleted_status, MetadataInfo_tmap &metadata, 
+                comerror_t &err, ostream &logout)
+{
+  outconvertclass text_t2ascii;
+  metadata.erase(metadata.begin(), metadata.end());
+  err = noError;
+
+  //logout << text_t2ascii << "sourceclass::get_oai_meta for OID: " << OID << "\n";   
+
+// ONLY if we're doing any OAI stuff (FROAI will be set then) will we even be in here
+// So next try to open the oai-inf db if it exists for this collection
+  if (!file_exists(oaidb_filename)) { // if the oaidb file doesn't even exist, let's not bother with oaidb
+    
+    logout << text_t2ascii
+       << "warning: collection's oai-inf database \"" << oaidb_filename << "\" does not exist\n\n";    
+    return false;
+    
+  } else { // let's try opening the oaidb file
+    oaidb_ptr->setlogout(&logout);
+    if (!oaidb_ptr->opendatabase (oaidb_filename, DB_READER, 100, false)) {
+      // most likely a system problem (we have already checked that the database exists just above)
+      logout << text_t2ascii
+         << "system problem: open on database \""
+         << oaidb_filename << "\" failed\n\n";
+      err = systemProblem;
+      return false;
+    } // now we've opened the oai-inf db file successfully
+  }
+
+  // get the metadata (deleted status, timestamp, datestamp) for OID, and pushback onto the metadata array
+  infodbclass oai_info;
+  if (!oaidb_ptr->getinfo(OID, oai_info)) { // OID not found in oai_inf db
+    logout << text_t2ascii
+         << "warning: lookup for OID \"" << OID
+         << "\" in etc/oai-inf db failed when getting metadata (in sourceclass source.cpp).\n\n";
+    
+    return false; 
+  } else {
+    //logout << text_t2ascii << "@@@@ getting metadata for OID \"" << OID << "\" from etc/oai-inf db.\n\n";
+    //logout << text_t2ascii << "@@@@ timestamp: " << oai_info["timestamp"] << "\n\n";
+
+    metadata["oaiinf.status"].isRef = false;
+    metadata["oaiinf.status"].values.push_back(oai_info["status"]);
+    metadata["oaiinf.timestamp"].isRef = false;
+    metadata["oaiinf.timestamp"].values.push_back(oai_info["timestamp"]);
+    metadata["oaiinf.datestamp"].isRef = false;
+    metadata["oaiinf.datestamp"].values.push_back(oai_info["datestamp"]);
+  }
+
+   // we're done with oai-inf db
+
+  oaidb_ptr->closedatabase(); // don't leave files open
+
+  return true;
+}
+
 
 // get_metadata fills out the metadata if possible, if it is not responsible
@@ -213 +314 @@
                 bool getParents, const text_tset &fields, 
                 const text_t &OID, MetadataInfo_tmap &metadata, 
-                comerror_t &err, ostream &logout)
+                comerror_t &err, ostream &logout, bool append)
 {
   outconvertclass text_t2ascii;
 
-  metadata.erase(metadata.begin(), metadata.end());
+  if(!append) {
+    metadata.erase(metadata.begin(), metadata.end());
+  }
 
   err = noError;

main/trunk/greenstone2/runtime-src/src/colservr/source.h

@@ -62 +62 @@
   text_t db_filename;
   dbclass *db_ptr;
+  text_t oaidb_filename;
+  dbclass *oaidb_ptr;
 
   searchclass *textsearchptr;
@@ -69 +71 @@
   virtual ~sourceclass ();
 
-  // the DB ptr remains the responsibility of the calling code
+  // the DB ptrs remain the responsibility of the calling code
+  // but as the db_ptr is deleted by this source object, the oaidb_ptr will be deleted by this object too
   void set_db_ptr (dbclass *db_ptr_arg) { db_ptr = db_ptr_arg; }
+  void set_oaidb_ptr (dbclass *oaidb_ptr_arg) { oaidb_ptr = oaidb_ptr_arg; }
 
   // the textsearchptr remains the responsibility of the calling code
@@ -86 +90 @@
                   comerror_t &err, ostream &logout);
 
-  // get_metadata fills out the metadata if possible, if it is not responsable
-  // for the given OID then it will return false.
+  // get_metadata fills out the metadata if possible.
+  // If it is not responsible for the given OID then it will return false.
   virtual bool get_metadata (const text_t &requestParams, const text_t &refParams, 
                  bool getParents, const text_tset &fields,
                  const text_t &OID, MetadataInfo_tmap &metadata, 
-                 comerror_t &err, ostream &logout);
+                 comerror_t &err, ostream &logout, bool append);
+
+  // get_oai_metadata fills out the metadata from the oai_db if possible
+  // if it is not responsible for the given OID then it will return false.
+  virtual bool get_oai_metadata (const text_t &requestParams, const text_t &refParams, 
+                 bool getParents, const text_tset &fields,
+                 const text_t &OID, text_t &deleted_status,
+                 MetadataInfo_tmap &metadata, comerror_t &err, ostream &logout);
 
   virtual bool get_document (const text_t &OID, text_t &doc, 
@@ -101 +112 @@
   text_t resolve_db_filename(const text_t& idx,
                  const text_t& file_ext);
+  text_t resolve_oaidb_filename(const text_t& file_ext);
+  text_t getcollectionpath();
 
 };

main/trunk/greenstone2/runtime-src/src/oaiservr/abstractlistaction.cpp

@@ -170 +170 @@
   text_tset metadata;  // Must be empty for efficiency
   FilterResponse_t identifiers_response;
-  get_children(gsdl_classifier_OID, collection_name, "", metadata, false, protocol, identifiers_response, *this->logout);
+  get_children(gsdl_classifier_OID, collection_name, "", metadata, false, protocol, identifiers_response, *this->logout, FROAI);
 
   // Find the starting position, if necessary
@@ -251 +251 @@
   metadata.insert("gs.OAIDateStamp");
   FilterResponse_t response;
-  if (!get_info(document_OID, collection, "", metadata, false, protocol, response, *this->logout))
+  if (!get_oai_info(document_OID, collection, "", metadata, false, protocol, response, *this->logout))
   {
     return false;

main/trunk/greenstone2/runtime-src/src/oaiservr/identifyaction.cpp

@@ -117 +117 @@
     
     output << utf8convert << "  <earliestDatestamp>"<< this->mEarliestDatestamp <<"</earliestDatestamp>\n";
-    output << utf8convert << "  <deletedRecord>no</deletedRecord>\n";
+    output << utf8convert << "  <deletedRecord>persistent</deletedRecord>\n";
     output << utf8convert << "  <granularity>YYYY-MM-DD</granularity>\n";
   }

main/trunk/greenstone2/runtime-src/src/oaiservr/listidsaction.cpp

@@ -237 +237 @@
   text_tset        metadata;
   ofstream         logout("oai.log", ios::app);
-  text_t           lastModified;
+  text_t           lastModified = "";
+  text_t           deleted_status = "";
   int              oaiVersion = this->configuration->getOAIVersion();
   text_t repos_id = this->configuration->getRepositoryId();
-  get_info(OID, collection, "", metadata, false, protocol, response, logout);
+  get_oai_info(OID, collection, "", metadata, false, protocol, response, logout);
   doc_info = response.docInfo[0];
   this->getLastModifiedDate(doc_info, lastModified);
+  this->getMeta(doc_info, "oaiinf.status", deleted_status);
   
   // output the record for this document
@@ -251 +253 @@
     output << "  <identifier>" << oaiLabel << "</identifier>\n";
   else
-    this->output_record_header(output, oaiLabel, lastModified, 
+    this->output_record_header(output, oaiLabel, lastModified, deleted_status,
                    doc_info.metadata["memberof"].values, oaiVersion);
 

main/trunk/greenstone2/runtime-src/src/oaiservr/metaformatsaction.cpp

@@ -97 +97 @@
     text_tset metadata; 
     
-    if (!get_info(identifier, collection, "", metadata, false, protocol, this->gsdlResponse, *logout))
+    if (!get_oai_info(identifier, collection, "", metadata, false, protocol, this->gsdlResponse, *logout))
     {
       this->errorType = "idDoesNotExist";

main/trunk/greenstone2/runtime-src/src/oaiservr/oaiaction.cpp

@@ -403 +403 @@
 
 void oaiaction::output_record_header(ostream &output, const text_t &oaiLabel, const text_t &lastModified, 
-                     const text_tarray &memberOf, int oaiVersion)
-{
+                     const text_t &deleted_status, const text_tarray &memberOf, int oaiVersion)
+{
+  if(deleted_status == "D") { // record marked as deleted
+    output << "    <header status=\"deleted\">" << endl;
+  } else { // deleted_status is "" or E for exists
     output << "    <header>" << endl;
+  }
     output << "      <identifier>" << oaiLabel     << "</identifier>" << endl;
     output << "      <datestamp>"  << lastModified << "</datestamp>" << endl;
@@ -443 +447 @@
 }
 
+// Method that looks for the requested metaname in the doc_info, and if found, sets the value
+// in the argument 'metavalue'.
+void oaiaction::getMeta(ResultDocInfo_t &doc_info, const text_t &metaname, text_t &metavalue)
+{
+  text_t current_metaname;
+
+  //ofstream         logout("oai.log", ios::app);
+
+  MetadataInfo_tmap::iterator current = doc_info.metadata.begin();
+  MetadataInfo_tmap::iterator end = doc_info.metadata.end();
+
+  while(current != end){
+    current_metaname = current->first;
+    lc(current_metaname); // lowercased for string comparison below
+
+    if (current_metaname == metaname && current->second.values.size() >= 1) { // found match with a value
+      metavalue = current->second.values[0];
+
+      //logout << "Looking for metaname = " << current_metaname << endl;
+      //logout << "\t\t\t Found value = " << metavalue << endl;
+      return; // done
+    }
+
+    ++current;
+  }
+
+  //logout.close();
+}
+
 void oaiaction::getLastModifiedDate(ResultDocInfo_t &doc_info, text_t &lastModified)
 {
@@ -460 +493 @@
     }
     else{ 
-      if (temp == "oailastmodified" && lastModified == "" && current->second.values.size() >= 1) {
-    lastModified = current->second.values[0];
-    time_t raw_time = (time_t)lastModified.getint();
-    lastModified = this->parseDatestamp(raw_time);
+      if(lastModified == "" && current->second.values.size() >= 1) {
+
+    if (temp == "oaiinf.timestamp") { // new way is to store oai timestamp in oai db and get it from there
+      // check if there was a timestamp for the doc in the etc/oai-inf database
+      lastModified = current->second.values[0];
+    
+    } else if (temp == "oailastmodified") { // old way, being phased out
+      // check if there was an oailastmodified timestamp for the doc in the collection index db
+      lastModified = current->second.values[0];
+    }
+
+    if(lastModified != "") { // if we've now set the lastModified value, convert it for display and return
+      time_t raw_time = (time_t)lastModified.getint();
+      lastModified = this->parseDatestamp(raw_time);
+    
+      return;
+    }
       }
-    }
+    } // else keep looking for oai timestamp
     ++current;
   }  
+
 }
 
@@ -475 +522 @@
   FilterResponse_t response;
   text_tset        metadata;
-  bool status_ok = get_info(OID, collection, "", metadata, false, protocol, response, logout);
+  bool status_ok = get_oai_info(OID, collection, "", metadata, false, protocol, response, logout);
+     // get timestamp from etc/oai-inf.<db> now, no longer from index.db
   bool not_too_early = false, not_too_recent = false;
 

main/trunk/greenstone2/runtime-src/src/oaiservr/oaiaction.h

@@ -59 +59 @@
   void   output_action_tag(ostream &output, bool openTag);
   void   output_record_header(ostream &output, const text_t &oaiLabel, const text_t &lastModified, 
-                  const text_tarray &memberOf, int oaiVersion);
+                  const text_t &deleted_status, const text_tarray &memberOf, int oaiVersion);
   void   output_error(ostream &output, text_t &errorType); // Output any error conditions
   bool   formatNotSupported(text_t &metaFormat);           // Return true if the supplied metadataPrefix
@@ -66 +66 @@
   text_t getErrorType(){return this->errorType;};
   void   getLastModifiedDate(ResultDocInfo_t &doc_info, text_t &lastModified);
+  void   getMeta(ResultDocInfo_t &doc_info, const text_t &metaname, text_t &metavalue);
   bool   inDateRange(const text_t &from, const text_t &until, const text_t &collection, 
              const text_t &OID, recptproto *protocol, ostream &logout);

main/trunk/greenstone2/runtime-src/src/oaiservr/oaitools.cpp

@@ -52 +52 @@
 
   bool status_ok = get_children(oai_id, collection, "", metadata, false, protocol, response, logout);
+       // OLD: gets the ID in the 'contains' field of the index_db for OID key of type [oai.x] where x is numeric
+       // Now we check the etc/oai-inf.<db> instead.
   text_t gsdl_id = "";
   if (status_ok) {

main/trunk/greenstone2/runtime-src/src/oaiservr/recordaction.cpp

@@ -148 +148 @@
   // Check a document with the specified identifier exists
   text_tset metadata;
-  if (!get_info(identifier, collection, "", metadata, false, protocol, this->gsdlResponse, *logout))
+  if (!get_oai_info(identifier, collection, "", metadata, false, protocol, this->gsdlResponse, *logout))
   {
     this->errorType = "idDoesNotExist";
@@ -200 +200 @@
   
   // get the document information
-  if (!get_info(OID, collection, "", metadata, false, protocol, this->gsdlResponse, logout)) {
+  if (!get_oai_info(OID, collection, "", metadata, false, protocol, this->gsdlResponse, logout)) {
 
     this->errorType = "idDoesNotExist";
@@ -208 +208 @@
       return false;
     }
+  }
+  else {    
+    this->errorType = "";
   }
 
@@ -232 +235 @@
   ResultDocInfo_t doc_info = this->gsdlResponse.docInfo[0];
   text_t lastModified = "";
+  text_t deleted_status = "";
 
   // Fills lastModified with the date from the document in doc_info, in the format YYYY-MM-DD
   this->getLastModifiedDate(doc_info, lastModified);
+  this->getMeta(doc_info, "oaiinf.status", deleted_status);
 
   // If the ID exists, output record for oai response (OAI v1.1)
@@ -246 +251 @@
     
     // output header part of oai response
-    this->output_record_header(output, oaiLabel, lastModified, 
+    this->output_record_header(output, oaiLabel, lastModified, deleted_status,
                    doc_info.metadata["memberof"].values, oaiVersion);
     
-    if (this->errorType != "cannotDisseminateFormat"){
+    // only output <metadata /> section if record is not marked as deleted
+    if (this->errorType != "cannotDisseminateFormat" && deleted_status != "D") {
       if (this->formatMap[metadataPrefix].get_class()->output_metadata(output, collection, doc_info)) {
     //  output 'about' part of oai response - we probably won't ever use this

main/trunk/greenstone2/runtime-src/src/protocol/recptprototools.cpp

@@ -32 +32 @@
 // metadata fields are to be requested.
 
+// THIS FILE IS A CANDIDATE FOR REFACTORING: merge get_info methods
+
 bool get_info (const text_t &OID, const text_t &collection, const text_t &lang,
            const text_tset &metadata, bool getParents,
            recptproto *collectproto, FilterResponse_t &response, 
-           ostream &logout)
+           ostream &logout, int filterResultOptions)
 {
   response.clear();
@@ -45 +47 @@
   request.filterName = "NullFilter";
   request.filterLang = lang;
-  request.filterResultOptions = FRmetadata;
+  request.filterResultOptions = filterResultOptions | FRmetadata;
   request.getParents = getParents;
   request.fields = metadata;
@@ -65 +67 @@
 }
 
+bool get_oai_info (const text_t &OID, const text_t &collection, const text_t &lang,
+           const text_tset &metadata, bool getParents,
+           recptproto *collectproto, FilterResponse_t &response, 
+           ostream &logout)
+{
+  // set filteroption to FROAI
+  return get_info(OID, collection, lang, metadata, getParents, collectproto, response, logout, FROAI);
+}
 
 // overloaded, to allow "custom" filter options.
@@ -162 +172 @@
            const text_tset &metadata, bool getParents,
            recptproto *collectproto, FilterResponse_t &response, 
-           ostream &logout)
+           ostream &logout, int filterResultOptions)
 {
   response.clear();
@@ -175 +185 @@
   request.filterName = "BrowseFilter";
   request.filterLang = lang;
-  request.filterResultOptions = FROID;
+  request.filterResultOptions = filterResultOptions | FROID;
 
   // Efficiency improvement: only get the filter to retrieve metadata if some has been requested

main/trunk/greenstone2/runtime-src/src/protocol/recptprototools.h

@@ -38 +38 @@
            const text_tset &metadata, bool getParents,
            recptproto *collectproto, FilterResponse_t &response, 
+           ostream &logout, int filterResultOptions=0);
+bool get_oai_info (const text_t &OID, const text_t &collection, const text_t &lang,
+           const text_tset &metadata, bool getParents,
+           recptproto *collectproto, FilterResponse_t &response, 
            ostream &logout);
 bool get_info (const text_t &OID, const text_t &collection, const text_t &lang,
@@ -59 +63 @@
            const text_tset &metadata, bool getParents,
            recptproto *collectproto, FilterResponse_t &response, 
-           ostream &logout);
+           ostream &logout, int filterResultOptions=0);
 
 // get_contents returns OIDs and metadata of all contents