Changeset 31387

Show
Ignore:
Timestamp:
08.02.2017 18:31:18 (3 years ago)
Author:
ak19
Message:

Round 1 of commits for getting OAI deletion policy to work with GS2 (server end). The perl code writing out the OAI db and the GS3 server code implementing the deletion policy had already been completed earlier (end 2016).

Location:
main/trunk/greenstone2/runtime-src/src
Files:
19 modified

Legend:

Unmodified
Added
Removed
  • main/trunk/greenstone2/runtime-src/src/colservr/browsefilter.cpp

    r16310 r31387  
    2626#include "browsefilter.h" 
    2727#include "fileutil.h" 
     28#include <assert.h> 
    2829 
    2930 
    3031browsefilterclass::browsefilterclass () { 
    3132  db_ptr = NULL; 
     33  oaidb_ptr = NULL; 
    3234 
    3335  // -- onePerQuery StartResults   integer 
     
    6062} 
    6163 
    62 browsefilterclass::~browsefilterclass () { 
    63 } 
     64browsefilterclass::~browsefilterclass () {} 
    6465 
    6566void browsefilterclass::configure (const text_t &key, const text_tarray &cfgline) { 
     
    7576  if (!filterclass::init(logout)) return false; 
    7677 
    77   if (db_ptr == NULL) { 
     78  if (db_ptr == NULL || oaidb_ptr == NULL) { 
    7879    // most likely a configuration problem 
    7980    logout << text_t2ascii  
     
    9394  } 
    9495 
     96  oaidb_filename = resolve_oaidb_filename(oaidb_ptr->getfileextension()); 
     97 
    9598  return true; 
    9699} 
     
    104107  response.clear (); 
    105108  err = noError; 
    106   if (db_ptr == NULL) { 
    107     // most likely a configuration problem 
    108     logout << text_t2ascii  
    109        << "configuration error: browsefilter contains a null dbclass\n\n"; 
    110     err = configurationError; 
    111     return; 
    112   } 
    113  
    114   // open the database 
    115   db_ptr->setlogout(&logout); 
    116   if (!db_ptr->opendatabase (db_filename, DB_READER, 100, false)) { 
    117     // most likely a system problem (we have already checked that the database exists) 
    118     logout << text_t2ascii 
    119        << "system problem: open on database \"" 
    120        << db_filename << "\" failed\n\n"; 
    121     err = systemProblem; 
    122     return; 
    123   } 
    124109 
    125110  // get the browse parameters 
     
    146131  } 
    147132 
     133  // if we're only working on oai, open oai_db, no need to work with index_db in browsefilter.cpp 
     134  // (but source.cpp uses both oai-inf.db and index.db to get metadata for OAI request) 
     135  // If we can't open the oai-inf db, this can be because it didn't exist in older versions of GS 
     136  // in that case, proceed as usual, using the index db. 
     137  if((request.filterResultOptions & FROAI)) { // OAI request     
     138    bool success = false; 
     139 
     140    if(parentnode == "oai") { // doing an OAI listidentifiers request 
     141 
     142      // open up the oai-inf db, if it exists, and return all IDs 
     143      // if it doesn't exist, proceed as usual 
     144      success = get_oaiinf_db_entries(response, err, logout); //adds any stuff in oai-inf db for the current OID to resultdoc.metadata 
     145 
     146      response.numDocs = response.docInfo.size(); 
     147      response.isApprox = Exact; 
     148    } 
     149    if (success) return; // oai request successfully completed with oai-inf.db, no need to open index_db 
     150  }  
     151 
     152  // Since we're here, it means we're not doing anything oai (or oai-inf.db did not exist/open) 
     153  // So we don't need to work with oai_db. Instead, work with index_db: 
     154 
     155  if (db_ptr == NULL) { 
     156    // most likely a configuration problem 
     157    logout << text_t2ascii  
     158       << "configuration error: browsefilter contains a null index dbclass\n\n"; 
     159    err = configurationError; 
     160    return; 
     161  } 
     162 
     163  // open the database 
     164  db_ptr->setlogout(&logout); 
     165  if (!db_ptr->opendatabase (db_filename, DB_READER, 100, false)) { 
     166    // most likely a system problem (we have already checked that the database exists) 
     167    logout << text_t2ascii 
     168       << "system problem: open on database \"" 
     169       << db_filename << "\" failed\n\n"; 
     170    err = systemProblem; 
     171    return; 
     172  } 
     173 
    148174  infodbclass info; 
    149175 
     
    158184      (request.filterResultOptions & FRmetadata)) { 
    159185    if (!db_ptr->getinfo(parentnode, info)) { 
    160       // didn't find the node 
     186      // didn't find the node in index db 
    161187      logout << text_t2ascii 
    162188         << "warning: lookup for node \"" << parentnode  
     
    308334  response.isApprox = Exact; 
    309335} 
     336 
     337bool browsefilterclass::get_oaiinf_db_entries(FilterResponse_t &response,  
     338                      comerror_t &err, ostream &logout)  
     339{ 
     340  outconvertclass text_t2ascii; 
     341   
     342  //logout << text_t2ascii << "browsefilterclass::get_oaiinf_db_entries\n";    
     343 
     344  // ONLY if we're doing any OAI stuff (FROAI will be set then) will we be here 
     345  // So next try to open the oai-inf db if it exists for this collection 
     346 
     347   
     348  if (!file_exists(oaidb_filename)) { // if the oaidb file doesn't even exist, let's not bother with oaidb 
     349     
     350    logout << text_t2ascii 
     351       << "warning: collection's oai-inf database \"" << oaidb_filename << "\" does not exist\n\n";     
     352    return false; 
     353 
     354  } else { // let's try opening the oaidb file 
     355    oaidb_ptr->setlogout(&logout); 
     356    if (!oaidb_ptr->opendatabase (oaidb_filename, DB_READER, 100, false)) { 
     357      // most likely a system problem (we have already checked that the database exists just above) 
     358      logout << text_t2ascii 
     359         << "system problem: open on database \"" 
     360         << oaidb_filename << "\" failed\n\n"; 
     361      err = systemProblem; 
     362      return false; 
     363    } // now we've opened the oai-inf db file successfully 
     364  } 
     365 
     366  infodbclass oai_info; 
     367  ResultDocInfo_t resultdoc; 
     368 
     369  text_tarray keys = oaidb_ptr->getkeys(); 
     370 
     371  text_tarray::iterator key_here = keys.begin(); 
     372  text_tarray::iterator key_end = keys.end(); 
     373  while (key_here != key_end) {     
     374     
     375    resultdoc.OID = (*key_here); 
     376    
     377    if(!oaidb_ptr->getinfo(resultdoc.OID, oai_info)) { 
     378      logout << text_t2ascii 
     379         << "warning: lookup for node \"" << resultdoc.OID 
     380         << "\" in etc/oai-inf db failed for browsefilter.\n\n"; 
     381    } else { 
     382      //logout << text_t2ascii << "@@@@ found node \"" << resultdoc.OID << "\" in etc/oai-inf db.\n\n"; 
     383       
     384      resultdoc.metadata["oaiinf.status"].isRef = false; 
     385      resultdoc.metadata["oaiinf.status"].values.push_back(oai_info["status"]); 
     386      resultdoc.metadata["oaiinf.timestamp"].isRef = false; 
     387      resultdoc.metadata["oaiinf.timestamp"].values.push_back(oai_info["timestamp"]); 
     388      resultdoc.metadata["oaiinf.datestamp"].isRef = false; 
     389      resultdoc.metadata["oaiinf.datestamp"].values.push_back(oai_info["datestamp"]); 
     390    } 
     391     
     392    response.docInfo.push_back(resultdoc); 
     393    ++key_here;     
     394     
     395  } 
     396 
     397  // we're done with oai-inf db 
     398 
     399  oaidb_ptr->closedatabase(); // don't leave files open 
     400 
     401  return true; 
     402} 
  • main/trunk/greenstone2/runtime-src/src/colservr/browsefilter.h

    r15558 r31387  
    4141  text_t indexstem; 
    4242  dbclass *db_ptr; 
     43  text_t oaidb_filename; 
     44  dbclass *oaidb_ptr; 
    4345 
    4446public: 
     
    5052  virtual void configure (const text_t &key, const text_tarray &cfgline); 
    5153 
    52   // the db ptr remains the responsability of the calling code and 
    53   // should be destroyed after this browsefilter is destroyed 
     54  // the db ptr and oaidb ptr remain the responsibility of the calling code and should 
     55  // be destroyed after this browsefilter is destroyed (source.h class destroys these) 
    5456  void set_db_ptr (dbclass *db_ptr_arg) { db_ptr = db_ptr_arg; } 
     57  void set_oaidb_ptr (dbclass *oaidb_ptr_arg) { oaidb_ptr = oaidb_ptr_arg; } 
    5558 
    5659  bool init (ostream &logout); 
     
    5962           FilterResponse_t &response, 
    6063           comerror_t &err, ostream &logout); 
     64 
     65 protected: 
     66  bool get_oaiinf_db_entries(FilterResponse_t &response,  
     67                 comerror_t &err, ostream &logout); 
    6168}; 
    6269 
  • main/trunk/greenstone2/runtime-src/src/colservr/collectserver.cpp

    r29100 r31387  
    515515    thisfilter->filter (request, response, err, logout); 
    516516    if (err != noError) return; 
     517 
    517518    // fill in the metadata for each of the OIDs (if it is requested) 
    518519    if (request.filterResultOptions & FRmetadata) { 
     520 
    519521      bool processed = false; 
    520522      ResultDocInfo_tarray::iterator resultdoc_here = response.docInfo.begin(); 
    521523      ResultDocInfo_tarray::iterator resultdoc_end = response.docInfo.end(); 
    522524      while (resultdoc_here != resultdoc_end) { 
     525 
     526    text_t deleted_status = ""; 
     527    bool append_metadata = (request.filterResultOptions & FROAI) ? true : false; 
     528 
    523529    // try each of the sources in turn 
    524530    sourcelistclass::iterator source_here = sources.begin(); 
     
    526532    while (source_here != source_end) { 
    527533      assert ((*source_here).s != NULL); 
     534 
     535      // first check for oai metadata from the oai_db, if asked for it (if FROAI is set) 
     536      if(((*source_here).s != NULL) && 
     537         request.filterResultOptions & FROAI && 
     538         ((*source_here).s->get_oai_metadata(request.requestParams, request.refParams, 
     539                         request.getParents, request.fields,  
     540                         (*resultdoc_here).OID, deleted_status, (*resultdoc_here).metadata,  
     541                         err, logout))) { 
     542         
     543        if (err != noError) return; 
     544 
     545        processed = true; 
     546      } 
     547 
     548      // We may or may not have got oai_meta (depends on if FROAI was set). 
     549      // If we didn't get oai_meta, then deleted_status would still be "". 
     550      // If we did get oai_meta, and if the deleted_status for the OID was D for deleted entry, 
     551      // don't bother getting any other metadata, as there will be no entry for that OID in index db. 
     552       
     553      // Note that if we did get oai_meta and OID marked as existing, we're in append_mode: 
     554      // don't let get_metadata() clear the metadata list, as there's already stuff in there 
     555      //if(deleted_status == "E") append_metadata = true; 
     556 
    528557      if (((*source_here).s != NULL) && 
     558          deleted_status != "D" && 
    529559          ((*source_here).s->get_metadata(request.requestParams, request.refParams, 
    530560                          request.getParents, request.fields,  
    531561                          (*resultdoc_here).OID, (*resultdoc_here).metadata,  
    532                           err, logout))) { 
    533         if (err != noError) return; 
    534         processed = true; 
    535         break; 
     562                          err, logout, append_metadata))) { 
     563        if (err != noError) return; // check for errors again      
     564         
     565        processed = processed || true; // processed would not have been set yet if not doing FROAI. Set now. 
     566        // OR-ing isn't necessary, but indicates some consideration of both get oai meta & get meta success 
    536567      } 
     568       
     569      if(processed) break; 
     570 
    537571      ++source_here; 
    538572    } 
     
    548582      } 
    549583    }   
    550    
     584     
    551585    err = noError; 
    552586  } 
  • main/trunk/greenstone2/runtime-src/src/colservr/collectset.cpp

    r25235 r31387  
    335335 
    336336    // Create a dbclass of the correct type 
    337     dbclass *db_ptr = NULL; 
     337    dbclass *db_ptr = NULL; // index database in index/text 
     338    dbclass *oaidb_ptr = NULL; // etc/oai-inf database 
    338339 
    339340    if (infodbtype == "sqlite") 
     
    342343    sqlitedbclass *sql_db_ptr = new sqlitedbclass(gsdlhome); 
    343344    db_ptr = sql_db_ptr; 
     345 
     346    sqlitedbclass *sql_oaidb_ptr = new sqlitedbclass(gsdlhome); 
     347    oaidb_ptr = sql_oaidb_ptr; 
    344348 
    345349    // add a sql browse filter 
     
    364368    mssqldbclass *mssql_db_ptr = new mssqldbclass(gsdlhome); 
    365369    db_ptr = mssql_db_ptr; 
     370 
     371    mssqldbclass *mssql_oaidb_ptr = new mssqldbclass(gsdlhome); 
     372    oaidb_ptr = mssql_oaidb_ptr; 
    366373 
    367374    // add a sql browse filter 
     
    380387    jdbmnaiveclass *jdbm_db_ptr = new jdbmnaiveclass(gsdlhome); 
    381388    db_ptr = jdbm_db_ptr; 
     389 
     390    jdbmnaiveclass *jdbm_oaidb_ptr = new jdbmnaiveclass(gsdlhome); 
     391    oaidb_ptr = jdbm_oaidb_ptr; 
    382392#else 
    383393    cerr << "Warning: infodbtype of 'jdbm' was not compiled in to " << endl; 
     
    392402#ifdef USE_GDBM 
    393403      db_ptr = new gdbmclass(gsdlhome); 
     404      oaidb_ptr = new gdbmclass(gsdlhome); 
    394405#else 
    395406    cerr << "Warning: infodbtype of 'gdbm' was not compiled in to " << endl; 
     
    405416    browsefilterclass *browsefilter = new browsefilterclass(); 
    406417    browsefilter->set_db_ptr(db_ptr); 
     418    // set the oaidb ptr for the browse filter. As with the db_ptr, the oaidb_ptr 
     419    // is not managed and deleted by the browsefilter class, but by the sourceclass 
     420    browsefilter->set_oaidb_ptr(oaidb_ptr); 
    407421    cserver->add_filter (browsefilter);   
    408422 
     
    420434      mgsourceclass *mgsource = new mgsourceclass (); 
    421435      mgsource->set_db_ptr(db_ptr); 
     436      mgsource->set_oaidb_ptr(oaidb_ptr); 
    422437      mgsource->set_textsearchptr (mgsearch); 
    423438      cserver->add_source (mgsource); 
     
    439454      mgppsourceclass *mgppsource = new mgppsourceclass (); 
    440455      mgppsource->set_db_ptr(db_ptr); 
     456      mgppsource->set_oaidb_ptr(oaidb_ptr); 
    441457      mgppsource->set_textsearchptr (mgppsearch); 
    442458      cserver->add_source (mgppsource); 
     
    459475      lucenesourceclass *lucenesource = new lucenesourceclass (); 
    460476      lucenesource->set_db_ptr(db_ptr); 
     477      lucenesource->set_oaidb_ptr(oaidb_ptr); 
    461478      lucenesource->set_textsearchptr (lucenesearch); 
    462479      cserver->add_source (lucenesource); 
  • main/trunk/greenstone2/runtime-src/src/colservr/comtypes.h

    r24114 r31387  
    243243// used to indicate a full text browse query for mgpp 
    244244#define FRfullTextBrowse  128 
     245#define FROAI  256 
    245246 
    246247struct FilterRequest_t { 
  • main/trunk/greenstone2/runtime-src/src/colservr/filter.cpp

    r16310 r31387  
    6161} 
    6262 
     63text_t filterclass::getcollectionpath() 
     64{ 
     65  text_t resolved_filename; 
     66   
     67  if (gsdlhome==dbhome) { 
     68    // dbhome has defaulted to gsdlhome which we take to means the  
     69    // database has been specifically moved out of gsdlhome area. 
     70    // => it should be whereever collecthome is set to 
     71    resolved_filename = filename_cat(collecthome, collection); 
     72  } 
     73  else { 
     74    // dbhome is explicitly set to something other than gsdlhome 
     75    // => use dbhome 
     76    resolved_filename = filename_cat(dbhome, "collect", collection); 
     77  } 
     78 
     79  return resolved_filename; 
     80} 
     81 
    6382text_t filterclass::resolve_db_filename(const text_t& idx, 
    6483                    const text_t& file_ext) 
    6584{ 
     85  /* 
    6686  text_t resolved_filename; 
    6787 
     
    7797    resolved_filename = filename_cat(dbhome, "collect", collection, "index", "text", idx); 
    7898  } 
     99  */ 
     100   
     101  text_t resolved_filename = filename_cat(getcollectionpath(), "index", "text", idx); 
     102  resolved_filename += file_ext; 
     103 
     104  return resolved_filename; 
     105} 
     106 
     107text_t filterclass::resolve_oaidb_filename(const text_t& file_ext) 
     108{ 
     109  text_t resolved_filename = filename_cat(getcollectionpath(), "etc", "oai-inf"); 
    79110 
    80111  resolved_filename += file_ext; 
  • main/trunk/greenstone2/runtime-src/src/colservr/filter.h

    r16310 r31387  
    6969 protected: 
    7070  text_t resolve_db_filename(const text_t& idx,const text_t& file_ext); 
    71  
     71  text_t resolve_oaidb_filename(const text_t& file_ext); 
     72  text_t getcollectionpath(); 
    7273 
    7374}; 
  • main/trunk/greenstone2/runtime-src/src/colservr/source.cpp

    r21772 r31387  
    3333{ 
    3434  db_ptr = NULL; 
     35  oaidb_ptr = NULL; 
    3536  textsearchptr = NULL; 
    3637  classname = "source"; 
     
    4142  if (db_ptr != NULL) delete db_ptr; 
    4243  if (textsearchptr != NULL) delete textsearchptr; 
     44 
     45  // http://stackoverflow.com/questions/677620/do-i-need-to-explicitly-call-the-base-virtual-destructor 
     46  // Answer: "No, destructors are called automatically [by the derived destructor] in the reverse 
     47  // order of construction. (Base classes last). Do not call base class destructors." 
     48  // The link also explains why the destructors should be declared as virtual in the header files. 
     49 
     50  // now delete the oaidb object pointed to by the oaidb_ptr too 
     51  if (oaidb_ptr != NULL) delete oaidb_ptr; 
     52  oaidb_ptr = NULL; 
    4353} 
    4454 
     
    7888} 
    7989 
     90text_t sourceclass::getcollectionpath() 
     91{ 
     92  text_t resolved_filename; 
     93   
     94  if (gsdlhome==dbhome) { 
     95    // dbhome has defaulted to gsdlhome which we take to means the  
     96    // database has been specifically moved out of gsdlhome area. 
     97    // => it should be whereever collecthome is set to 
     98    resolved_filename = filename_cat(collecthome, collection); 
     99  } 
     100  else { 
     101    // dbhome is explicitly set to something other than gsdlhome 
     102    // => use dbhome 
     103    resolved_filename = filename_cat(dbhome, "collect", collection); 
     104  } 
     105 
     106  return resolved_filename; 
     107} 
     108 
    80109text_t sourceclass::resolve_db_filename(const text_t& idx, 
    81110                    const text_t& file_ext) 
     
    91120  // are also drawn from one object 
    92121 
     122  /* 
    93123  text_t resolved_filename; 
    94124 
     
    105135    resolved_filename = filename_cat(dbhome, "collect", collection, "index", "text", idx); 
    106136  } 
    107  
     137  */ 
     138 
     139  text_t resolved_filename = filename_cat(getcollectionpath(), "index", "text", idx); 
    108140  resolved_filename += file_ext; 
    109141 
     
    111143} 
    112144 
    113  
     145text_t sourceclass::resolve_oaidb_filename(const text_t& file_ext) 
     146{ 
     147  text_t resolved_filename = filename_cat(getcollectionpath(), "etc", "oai-inf"); 
     148 
     149  resolved_filename += file_ext; 
     150 
     151  return resolved_filename; 
     152} 
    114153 
    115154// init should be called after all the configuration is done but 
     
    172211  } 
    173212   
     213  oaidb_filename = resolve_oaidb_filename(oaidb_ptr->getfileextension()); 
     214 
    174215  return true; 
    175216} 
     
    207248} 
    208249 
     250// get_oai_metadata fills out the oai metadata from oaidb if possible 
     251// if it is not responsible for the given OID then it will return false. 
     252bool sourceclass::get_oai_metadata (const text_t &requestParams, const text_t &refParams,  
     253                bool getParents, const text_tset &fields,  
     254                const text_t &OID, text_t &deleted_status, MetadataInfo_tmap &metadata,  
     255                comerror_t &err, ostream &logout) 
     256{ 
     257  outconvertclass text_t2ascii; 
     258  metadata.erase(metadata.begin(), metadata.end()); 
     259  err = noError; 
     260 
     261  //logout << text_t2ascii << "sourceclass::get_oai_meta for OID: " << OID << "\n";    
     262 
     263// ONLY if we're doing any OAI stuff (FROAI will be set then) will we even be in here 
     264// So next try to open the oai-inf db if it exists for this collection 
     265  if (!file_exists(oaidb_filename)) { // if the oaidb file doesn't even exist, let's not bother with oaidb 
     266     
     267    logout << text_t2ascii 
     268       << "warning: collection's oai-inf database \"" << oaidb_filename << "\" does not exist\n\n";     
     269    return false; 
     270     
     271  } else { // let's try opening the oaidb file 
     272    oaidb_ptr->setlogout(&logout); 
     273    if (!oaidb_ptr->opendatabase (oaidb_filename, DB_READER, 100, false)) { 
     274      // most likely a system problem (we have already checked that the database exists just above) 
     275      logout << text_t2ascii 
     276         << "system problem: open on database \"" 
     277         << oaidb_filename << "\" failed\n\n"; 
     278      err = systemProblem; 
     279      return false; 
     280    } // now we've opened the oai-inf db file successfully 
     281  } 
     282 
     283  // get the metadata (deleted status, timestamp, datestamp) for OID, and pushback onto the metadata array 
     284  infodbclass oai_info; 
     285  if (!oaidb_ptr->getinfo(OID, oai_info)) { // OID not found in oai_inf db 
     286    logout << text_t2ascii 
     287         << "warning: lookup for OID \"" << OID 
     288         << "\" in etc/oai-inf db failed when getting metadata (in sourceclass source.cpp).\n\n"; 
     289     
     290    return false;  
     291  } else { 
     292    //logout << text_t2ascii << "@@@@ getting metadata for OID \"" << OID << "\" from etc/oai-inf db.\n\n"; 
     293    //logout << text_t2ascii << "@@@@ timestamp: " << oai_info["timestamp"] << "\n\n"; 
     294 
     295    metadata["oaiinf.status"].isRef = false; 
     296    metadata["oaiinf.status"].values.push_back(oai_info["status"]); 
     297    metadata["oaiinf.timestamp"].isRef = false; 
     298    metadata["oaiinf.timestamp"].values.push_back(oai_info["timestamp"]); 
     299    metadata["oaiinf.datestamp"].isRef = false; 
     300    metadata["oaiinf.datestamp"].values.push_back(oai_info["datestamp"]); 
     301  } 
     302 
     303   // we're done with oai-inf db 
     304 
     305  oaidb_ptr->closedatabase(); // don't leave files open 
     306 
     307  return true; 
     308} 
     309 
    209310 
    210311// get_metadata fills out the metadata if possible, if it is not responsible 
     
    213314                bool getParents, const text_tset &fields,  
    214315                const text_t &OID, MetadataInfo_tmap &metadata,  
    215                 comerror_t &err, ostream &logout) 
     316                comerror_t &err, ostream &logout, bool append) 
    216317{ 
    217318  outconvertclass text_t2ascii; 
    218319 
    219   metadata.erase(metadata.begin(), metadata.end()); 
     320  if(!append) { 
     321    metadata.erase(metadata.begin(), metadata.end()); 
     322  } 
    220323 
    221324  err = noError; 
  • main/trunk/greenstone2/runtime-src/src/colservr/source.h

    r16310 r31387  
    6262  text_t db_filename; 
    6363  dbclass *db_ptr; 
     64  text_t oaidb_filename; 
     65  dbclass *oaidb_ptr; 
    6466 
    6567  searchclass *textsearchptr; 
     
    6971  virtual ~sourceclass (); 
    7072 
    71   // the DB ptr remains the responsibility of the calling code 
     73  // the DB ptrs remain the responsibility of the calling code 
     74  // but as the db_ptr is deleted by this source object, the oaidb_ptr will be deleted by this object too 
    7275  void set_db_ptr (dbclass *db_ptr_arg) { db_ptr = db_ptr_arg; } 
     76  void set_oaidb_ptr (dbclass *oaidb_ptr_arg) { oaidb_ptr = oaidb_ptr_arg; } 
    7377 
    7478  // the textsearchptr remains the responsibility of the calling code 
     
    8690                  comerror_t &err, ostream &logout); 
    8791 
    88   // get_metadata fills out the metadata if possible, if it is not responsable 
    89   // for the given OID then it will return false. 
     92  // get_metadata fills out the metadata if possible. 
     93  // If it is not responsible for the given OID then it will return false. 
    9094  virtual bool get_metadata (const text_t &requestParams, const text_t &refParams,  
    9195                 bool getParents, const text_tset &fields, 
    9296                 const text_t &OID, MetadataInfo_tmap &metadata,  
    93                  comerror_t &err, ostream &logout); 
     97                 comerror_t &err, ostream &logout, bool append); 
     98 
     99  // get_oai_metadata fills out the metadata from the oai_db if possible 
     100  // if it is not responsible for the given OID then it will return false. 
     101  virtual bool get_oai_metadata (const text_t &requestParams, const text_t &refParams,  
     102                 bool getParents, const text_tset &fields, 
     103                 const text_t &OID, text_t &deleted_status, 
     104                 MetadataInfo_tmap &metadata, comerror_t &err, ostream &logout); 
    94105 
    95106  virtual bool get_document (const text_t &OID, text_t &doc,  
     
    101112  text_t resolve_db_filename(const text_t& idx, 
    102113                 const text_t& file_ext); 
     114  text_t resolve_oaidb_filename(const text_t& file_ext); 
     115  text_t getcollectionpath(); 
    103116 
    104117}; 
  • main/trunk/greenstone2/runtime-src/src/oaiservr/abstractlistaction.cpp

    r27534 r31387  
    170170  text_tset metadata;  // Must be empty for efficiency 
    171171  FilterResponse_t identifiers_response; 
    172   get_children(gsdl_classifier_OID, collection_name, "", metadata, false, protocol, identifiers_response, *this->logout); 
     172  get_children(gsdl_classifier_OID, collection_name, "", metadata, false, protocol, identifiers_response, *this->logout, FROAI); 
    173173 
    174174  // Find the starting position, if necessary 
     
    251251  metadata.insert("gs.OAIDateStamp"); 
    252252  FilterResponse_t response; 
    253   if (!get_info(document_OID, collection, "", metadata, false, protocol, response, *this->logout)) 
     253  if (!get_oai_info(document_OID, collection, "", metadata, false, protocol, response, *this->logout)) 
    254254  { 
    255255    return false; 
  • main/trunk/greenstone2/runtime-src/src/oaiservr/identifyaction.cpp

    r24412 r31387  
    117117     
    118118    output << utf8convert << "  <earliestDatestamp>"<< this->mEarliestDatestamp <<"</earliestDatestamp>\n"; 
    119     output << utf8convert << "  <deletedRecord>no</deletedRecord>\n"; 
     119    output << utf8convert << "  <deletedRecord>persistent</deletedRecord>\n"; 
    120120    output << utf8convert << "  <granularity>YYYY-MM-DD</granularity>\n"; 
    121121  } 
  • main/trunk/greenstone2/runtime-src/src/oaiservr/listidsaction.cpp

    r27534 r31387  
    237237  text_tset        metadata; 
    238238  ofstream         logout("oai.log", ios::app); 
    239   text_t           lastModified; 
     239  text_t           lastModified = ""; 
     240  text_t           deleted_status = ""; 
    240241  int              oaiVersion = this->configuration->getOAIVersion(); 
    241242  text_t repos_id = this->configuration->getRepositoryId(); 
    242   get_info(OID, collection, "", metadata, false, protocol, response, logout); 
     243  get_oai_info(OID, collection, "", metadata, false, protocol, response, logout); 
    243244  doc_info = response.docInfo[0]; 
    244245  this->getLastModifiedDate(doc_info, lastModified); 
     246  this->getMeta(doc_info, "oaiinf.status", deleted_status); 
    245247   
    246248  // output the record for this document 
     
    251253    output << "  <identifier>" << oaiLabel << "</identifier>\n"; 
    252254  else 
    253     this->output_record_header(output, oaiLabel, lastModified,  
     255    this->output_record_header(output, oaiLabel, lastModified, deleted_status, 
    254256                   doc_info.metadata["memberof"].values, oaiVersion); 
    255257 
  • main/trunk/greenstone2/runtime-src/src/oaiservr/metaformatsaction.cpp

    r24108 r31387  
    9797    text_tset metadata;  
    9898     
    99     if (!get_info(identifier, collection, "", metadata, false, protocol, this->gsdlResponse, *logout)) 
     99    if (!get_oai_info(identifier, collection, "", metadata, false, protocol, this->gsdlResponse, *logout)) 
    100100    { 
    101101      this->errorType = "idDoesNotExist"; 
  • main/trunk/greenstone2/runtime-src/src/oaiservr/oaiaction.cpp

    r27534 r31387  
    403403 
    404404void oaiaction::output_record_header(ostream &output, const text_t &oaiLabel, const text_t &lastModified,  
    405                      const text_tarray &memberOf, int oaiVersion) 
    406 { 
     405                     const text_t &deleted_status, const text_tarray &memberOf, int oaiVersion) 
     406{ 
     407  if(deleted_status == "D") { // record marked as deleted 
     408    output << "    <header status=\"deleted\">" << endl; 
     409  } else { // deleted_status is "" or E for exists 
    407410    output << "    <header>" << endl; 
     411  } 
    408412    output << "      <identifier>" << oaiLabel     << "</identifier>" << endl; 
    409413    output << "      <datestamp>"  << lastModified << "</datestamp>" << endl; 
     
    443447} 
    444448 
     449// Method that looks for the requested metaname in the doc_info, and if found, sets the value 
     450// in the argument 'metavalue'. 
     451void oaiaction::getMeta(ResultDocInfo_t &doc_info, const text_t &metaname, text_t &metavalue) 
     452{ 
     453  text_t current_metaname; 
     454 
     455  //ofstream         logout("oai.log", ios::app); 
     456 
     457  MetadataInfo_tmap::iterator current = doc_info.metadata.begin(); 
     458  MetadataInfo_tmap::iterator end = doc_info.metadata.end(); 
     459 
     460  while(current != end){ 
     461    current_metaname = current->first; 
     462    lc(current_metaname); // lowercased for string comparison below 
     463 
     464    if (current_metaname == metaname && current->second.values.size() >= 1) { // found match with a value 
     465      metavalue = current->second.values[0]; 
     466 
     467      //logout << "Looking for metaname = " << current_metaname << endl; 
     468      //logout << "\t\t\t Found value = " << metavalue << endl; 
     469      return; // done 
     470    } 
     471 
     472    ++current; 
     473  } 
     474 
     475  //logout.close(); 
     476} 
     477 
    445478void oaiaction::getLastModifiedDate(ResultDocInfo_t &doc_info, text_t &lastModified) 
    446479{ 
     
    460493    } 
    461494    else{  
    462       if (temp == "oailastmodified" && lastModified == "" && current->second.values.size() >= 1) { 
    463     lastModified = current->second.values[0]; 
    464     time_t raw_time = (time_t)lastModified.getint(); 
    465     lastModified = this->parseDatestamp(raw_time); 
     495      if(lastModified == "" && current->second.values.size() >= 1) { 
     496 
     497    if (temp == "oaiinf.timestamp") { // new way is to store oai timestamp in oai db and get it from there 
     498      // check if there was a timestamp for the doc in the etc/oai-inf database 
     499      lastModified = current->second.values[0]; 
     500     
     501    } else if (temp == "oailastmodified") { // old way, being phased out 
     502      // check if there was an oailastmodified timestamp for the doc in the collection index db 
     503      lastModified = current->second.values[0]; 
     504    } 
     505 
     506    if(lastModified != "") { // if we've now set the lastModified value, convert it for display and return 
     507      time_t raw_time = (time_t)lastModified.getint(); 
     508      lastModified = this->parseDatestamp(raw_time); 
     509     
     510      return; 
     511    } 
    466512      } 
    467     } 
     513    } // else keep looking for oai timestamp 
    468514    ++current; 
    469515  }   
     516 
    470517} 
    471518 
     
    475522  FilterResponse_t response; 
    476523  text_tset        metadata; 
    477   bool status_ok = get_info(OID, collection, "", metadata, false, protocol, response, logout); 
     524  bool status_ok = get_oai_info(OID, collection, "", metadata, false, protocol, response, logout); 
     525     // get timestamp from etc/oai-inf.<db> now, no longer from index.db 
    478526  bool not_too_early = false, not_too_recent = false; 
    479527 
  • main/trunk/greenstone2/runtime-src/src/oaiservr/oaiaction.h

    r24412 r31387  
    5959  void   output_action_tag(ostream &output, bool openTag); 
    6060  void   output_record_header(ostream &output, const text_t &oaiLabel, const text_t &lastModified,  
    61                   const text_tarray &memberOf, int oaiVersion); 
     61                  const text_t &deleted_status, const text_tarray &memberOf, int oaiVersion); 
    6262  void   output_error(ostream &output, text_t &errorType); // Output any error conditions 
    6363  bool   formatNotSupported(text_t &metaFormat);           // Return true if the supplied metadataPrefix 
     
    6666  text_t getErrorType(){return this->errorType;}; 
    6767  void   getLastModifiedDate(ResultDocInfo_t &doc_info, text_t &lastModified); 
     68  void   getMeta(ResultDocInfo_t &doc_info, const text_t &metaname, text_t &metavalue); 
    6869  bool   inDateRange(const text_t &from, const text_t &until, const text_t &collection,  
    6970             const text_t &OID, recptproto *protocol, ostream &logout); 
  • main/trunk/greenstone2/runtime-src/src/oaiservr/oaitools.cpp

    r24096 r31387  
    5252 
    5353  bool status_ok = get_children(oai_id, collection, "", metadata, false, protocol, response, logout); 
     54       // OLD: gets the ID in the 'contains' field of the index_db for OID key of type [oai.x] where x is numeric 
     55       // Now we check the etc/oai-inf.<db> instead. 
    5456  text_t gsdl_id = ""; 
    5557  if (status_ok) { 
  • main/trunk/greenstone2/runtime-src/src/oaiservr/recordaction.cpp

    r24286 r31387  
    148148  // Check a document with the specified identifier exists 
    149149  text_tset metadata; 
    150   if (!get_info(identifier, collection, "", metadata, false, protocol, this->gsdlResponse, *logout)) 
     150  if (!get_oai_info(identifier, collection, "", metadata, false, protocol, this->gsdlResponse, *logout)) 
    151151  { 
    152152    this->errorType = "idDoesNotExist"; 
     
    200200   
    201201  // get the document information 
    202   if (!get_info(OID, collection, "", metadata, false, protocol, this->gsdlResponse, logout)) { 
     202  if (!get_oai_info(OID, collection, "", metadata, false, protocol, this->gsdlResponse, logout)) { 
    203203 
    204204    this->errorType = "idDoesNotExist"; 
     
    208208      return false; 
    209209    } 
     210  } 
     211  else {     
     212    this->errorType = ""; 
    210213  } 
    211214 
     
    232235  ResultDocInfo_t doc_info = this->gsdlResponse.docInfo[0]; 
    233236  text_t lastModified = ""; 
     237  text_t deleted_status = ""; 
    234238 
    235239  // Fills lastModified with the date from the document in doc_info, in the format YYYY-MM-DD 
    236240  this->getLastModifiedDate(doc_info, lastModified); 
     241  this->getMeta(doc_info, "oaiinf.status", deleted_status); 
    237242 
    238243  // If the ID exists, output record for oai response (OAI v1.1) 
     
    246251     
    247252    // output header part of oai response 
    248     this->output_record_header(output, oaiLabel, lastModified,  
     253    this->output_record_header(output, oaiLabel, lastModified, deleted_status, 
    249254                   doc_info.metadata["memberof"].values, oaiVersion); 
    250255     
    251     if (this->errorType != "cannotDisseminateFormat"){ 
     256    // only output <metadata /> section if record is not marked as deleted 
     257    if (this->errorType != "cannotDisseminateFormat" && deleted_status != "D") { 
    252258      if (this->formatMap[metadataPrefix].get_class()->output_metadata(output, collection, doc_info)) { 
    253259    //  output 'about' part of oai response - we probably won't ever use this 
  • main/trunk/greenstone2/runtime-src/src/protocol/recptprototools.cpp

    r22142 r31387  
    3232// metadata fields are to be requested. 
    3333 
     34// THIS FILE IS A CANDIDATE FOR REFACTORING: merge get_info methods 
     35 
    3436bool get_info (const text_t &OID, const text_t &collection, const text_t &lang, 
    3537           const text_tset &metadata, bool getParents, 
    3638           recptproto *collectproto, FilterResponse_t &response,  
    37            ostream &logout) 
     39           ostream &logout, int filterResultOptions) 
    3840{ 
    3941  response.clear(); 
     
    4547  request.filterName = "NullFilter"; 
    4648  request.filterLang = lang; 
    47   request.filterResultOptions = FRmetadata; 
     49  request.filterResultOptions = filterResultOptions | FRmetadata; 
    4850  request.getParents = getParents; 
    4951  request.fields = metadata; 
     
    6567} 
    6668 
     69bool get_oai_info (const text_t &OID, const text_t &collection, const text_t &lang, 
     70           const text_tset &metadata, bool getParents, 
     71           recptproto *collectproto, FilterResponse_t &response,  
     72           ostream &logout) 
     73{ 
     74  // set filteroption to FROAI 
     75  return get_info(OID, collection, lang, metadata, getParents, collectproto, response, logout, FROAI); 
     76} 
    6777 
    6878// overloaded, to allow "custom" filter options. 
     
    162172           const text_tset &metadata, bool getParents, 
    163173           recptproto *collectproto, FilterResponse_t &response,  
    164            ostream &logout) 
     174           ostream &logout, int filterResultOptions) 
    165175{ 
    166176  response.clear(); 
     
    175185  request.filterName = "BrowseFilter"; 
    176186  request.filterLang = lang; 
    177   request.filterResultOptions = FROID; 
     187  request.filterResultOptions = filterResultOptions | FROID; 
    178188 
    179189  // Efficiency improvement: only get the filter to retrieve metadata if some has been requested 
  • main/trunk/greenstone2/runtime-src/src/protocol/recptprototools.h

    r16180 r31387  
    3838           const text_tset &metadata, bool getParents, 
    3939           recptproto *collectproto, FilterResponse_t &response,  
     40           ostream &logout, int filterResultOptions=0); 
     41bool get_oai_info (const text_t &OID, const text_t &collection, const text_t &lang, 
     42           const text_tset &metadata, bool getParents, 
     43           recptproto *collectproto, FilterResponse_t &response,  
    4044           ostream &logout); 
    4145bool get_info (const text_t &OID, const text_t &collection, const text_t &lang, 
     
    5963           const text_tset &metadata, bool getParents, 
    6064           recptproto *collectproto, FilterResponse_t &response,  
    61            ostream &logout); 
     65           ostream &logout, int filterResultOptions=0); 
    6266 
    6367// get_contents returns OIDs and metadata of all contents