source: main/trunk/greenstone2/runtime-src/src/colservr/browsefilter.cpp@ 31903

/**********************************************************************
 *
 * browsefilter.cpp -- 
 * Copyright (C) 1999  The New Zealand Digital Library Project
 *
 * A component of the Greenstone digital library software
 * from the New Zealand Digital Library Project at the
 * University of Waikato, New Zealand.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 *
 *********************************************************************/

#include "browsefilter.h"
#include "colservertools.h"
#include "fileutil.h"
#include <assert.h>


browsefilterclass::browsefilterclass () {
  db_ptr = NULL;
  oaidb_ptr = NULL;

  // -- onePerQuery StartResults   integer
  FilterOption_t filtopt;
  filtopt.name = "StartResults";
  filtopt.type = FilterOption_t::integert;
  filtopt.repeatable = FilterOption_t::onePerQuery;
  filtopt.defaultValue = "1";
  filtopt.validValues.push_back("1");
  filtopt.validValues.push_back("10000");
  filterOptions["StartResults"] = filtopt;

  // -- onePerQuery EndResults     integer
  filtopt.clear();
  filtopt.name = "EndResults";
  filtopt.type = FilterOption_t::integert;
  filtopt.repeatable = FilterOption_t::onePerQuery;
  filtopt.defaultValue = "-1";
  filtopt.validValues.push_back("-1");
  filtopt.validValues.push_back("10000");
  filterOptions["EndResults"] = filtopt;

  // -- onePerQuery ParentNode     string ("" will return the browsing available)
  filtopt.clear();
  filtopt.name = "ParentNode";
  filtopt.type = FilterOption_t::stringt;
  filtopt.repeatable = FilterOption_t::onePerQuery;
  filtopt.defaultValue = g_EmptyText;
  filterOptions["ParentNode"] = filtopt;
}

browsefilterclass::~browsefilterclass () {}

void browsefilterclass::configure (const text_t &key, const text_tarray &cfgline) {
  filterclass::configure (key, cfgline);
  if (key == "indexstem") {
    indexstem = cfgline[0];
  }
}

bool browsefilterclass::init (ostream &logout) {
  outconvertclass text_t2ascii;

  if (!filterclass::init(logout)) return false;

  if (db_ptr == NULL || oaidb_ptr == NULL) {
    // most likely a configuration problem
    logout << text_t2ascii 
       << "configuration error: browsefilter contains a null dbclass\n\n";
    return false;
  }

  if (indexstem.empty()) {
    indexstem = collection;
  }

  db_filename = resolve_db_filename(gsdlhome, dbhome, collecthome, collection,
                    indexstem,db_ptr->getfileextension());
  if (!file_exists(db_filename)) {
    logout << text_t2ascii
       << "warning: database \"" << db_filename << "\" does not exist\n\n";
    //    return false;
  }

  oaidb_filename = resolve_oaidb_filename(gsdlhome, dbhome, collecthome, collection,
                      oaidb_ptr->getfileextension());

  return true;
}

void browsefilterclass::filter (const FilterRequest_t &request,
                FilterResponse_t &response,
                comerror_t &err, ostream &logout) {
  int numDocs = 0;
  outconvertclass text_t2ascii;

  response.clear ();
  err = noError;

  // get the browse parameters
  int startresults = filterOptions["StartResults"].defaultValue.getint();
  int endresults = filterOptions["EndResults"].defaultValue.getint();
  text_t parentnode = filterOptions["ParentNode"].defaultValue;
  OptionValue_tarray::const_iterator options_here = request.filterOptions.begin();
  OptionValue_tarray::const_iterator options_end = request.filterOptions.end();
  while (options_here != options_end) {
    if ((*options_here).name == "StartResults") 
      startresults = (*options_here).value.getint();
    else if ((*options_here).name == "EndResults")
      endresults = (*options_here).value.getint();
    else if ((*options_here).name == "ParentNode")
      parentnode = (*options_here).value;
    else {
      logout << text_t2ascii
         << "warning: unknown browsefilter option \""
         << (*options_here).name
         << "\" ignored.\n\n";
    }

    ++options_here;
  }

  // if we're only working on oai, open oai_db, no need to work with index_db in browsefilter.cpp
  // (but source.cpp uses both oai-inf.db and index.db to get metadata for OAI request)
  // If we can't open the oai-inf db, this can be because it didn't exist in older versions of GS
  // in that case, proceed as usual, using the index db.
  if((request.filterResultOptions & FROAI)) { // OAI request    
    bool success = false;

    if(parentnode == "oai") { // doing an OAI listidentifiers request

      // open up the oai-inf db if it exists, and return all IDs *except* the special OID=OAI_EARLIESTTIMESTAMP_OID
      // if oai-inf db doesn't exist, proceed as usual
      success = get_oaiinf_db_entries(response, err, logout); //adds any stuff in oai-inf db for the current OID to resultdoc.metadata

      response.numDocs = response.docInfo.size();
      response.isApprox = Exact;
    }
    if (success) return; // oai request successfully completed with oai-inf.db, no need to open index_db
  } 

  // Since we're here, it means we're not doing anything oai (or oai-inf.db did not exist/open)
  // So we don't need to work with oai_db. Instead, work with index_db:

  if (db_ptr == NULL) {
    // most likely a configuration problem
    logout << text_t2ascii 
       << "configuration error: browsefilter contains a null index dbclass\n\n";
    err = configurationError;
    return;
  }

  // open the database
  db_ptr->setlogout(&logout);
  if (!db_ptr->opendatabase (db_filename, DB_READER, 100, false)) {
    // most likely a system problem (we have already checked that the database exists)
    logout << text_t2ascii
       << "system problem: open on database \""
       << db_filename << "\" failed\n\n";
    err = systemProblem;
    return;
  }

  infodbclass info;

  // translate any ".fc", ".pr" etc. stuff in the parentnode
  parentnode = db_ptr->translate_OID (parentnode, info);

  // adjust topmost browsing node
  if (parentnode.empty()) parentnode = "browse";

  // get the node
  if ((request.filterResultOptions & FROID) ||
      (request.filterResultOptions & FRmetadata)) {
    if (!db_ptr->getinfo(parentnode, info)) {
      // didn't find the node in index db
      logout << text_t2ascii
         << "warning: lookup for node \"" << parentnode 
         << "\" failed for browsefilter.\n\n";
    } else {
      // found the node

      // replace " with the parent node name and split the contains string
      // into the result set
      text_tarray resultset;
      text_t tmptext;
      text_t &contains = info["contains"];
      text_t::iterator contains_here = contains.begin();
      text_t::iterator contains_end = contains.end();
      while (contains_here != contains_end) {
    if (*contains_here == '"') tmptext += parentnode;
    else if (*contains_here == ';') {
      if (!tmptext.empty()) resultset.push_back (tmptext);
      tmptext.clear();
    } else tmptext.push_back(*contains_here);
    
    ++contains_here;
      }
      // insert the last result in the set
      if (!tmptext.empty()) resultset.push_back (tmptext);

      text_tarray offset_resultset;
      text_t &md_type = info["mdtype"];
      if (!md_type.empty())
    {
      text_t &md_offset = info["mdoffset"];
      if (!md_offset.empty())
        {
          text_t offsettext;

          text_t::iterator offset_here = md_offset.begin();
          text_t::iterator offset_end = md_offset.end();
          while (offset_here != offset_end) 
        {
          if (*offset_here == ';')
            {
              if (offsettext.empty())
            {
              offset_resultset.push_back ("0");
            }
              else
            {
              offset_resultset.push_back (offsettext);
            }
              offsettext.clear();
            } 
          else 
            {
              offsettext.push_back(*offset_here);
            }
          
          ++offset_here;
        }
          // insert the last result in the set
          if (offsettext.empty())
        {
          offset_resultset.push_back ("0");
        }
          else
        {
          offset_resultset.push_back (offsettext);
        }
        }
      else
        {
          // add 0 offset for each 'contains' entry
          text_tarray::iterator result_here = resultset.begin();
          text_tarray::iterator result_end = resultset.end();
          while (result_here != result_end) {
        offset_resultset.push_back("0");
        ++result_here;
          }
        }

      // do an intersection with the input set
      if (!request.docSet.empty()) {

        text_tarray intersect_resultset;
        text_tarray intersect_offset_resultset;
        
        text_tarray::const_iterator resultset_here = resultset.begin();
        text_tarray::const_iterator resultset_end = resultset.end();
        text_tarray::const_iterator offset_resultset_here = offset_resultset.begin();
        
        while (resultset_here != resultset_end) {
          if (in_set (request.docSet, *resultset_here))
        {
          intersect_resultset.push_back (*resultset_here);
          intersect_offset_resultset.push_back (*offset_resultset_here);
        }
          ++resultset_here;
          ++offset_resultset_here;
        }
        resultset = intersect_resultset;
        offset_resultset = intersect_offset_resultset;
      }
    }
      else
    {
      // do an intersection with the input set
      if (!request.docSet.empty()) {
        intersect (resultset, request.docSet);
      }

      // add 0 offset for each 'contains' entry
      text_tarray::iterator result_here = resultset.begin();
      text_tarray::iterator result_end = resultset.end();
      while (result_here != result_end) {
        offset_resultset.push_back("0");
        ++result_here;
      }
    }

      // create the response
      numDocs = resultset.size();
      int resultnum = 1;
      ResultDocInfo_t resultdoc;
      text_tarray::iterator result_here = resultset.begin();
      text_tarray::iterator result_end = resultset.end();
      text_tarray::iterator offset_result_here = offset_resultset.begin();

      while (result_here != result_end) {
    // if endresults is -1 get all results
    if ((endresults != -1) && (resultnum > endresults)) break;
    if (resultnum >= startresults) {
      resultdoc.OID = (*result_here);
      if (!md_type.empty())
        {
          resultdoc.classifier_metadata_type = md_type;
          resultdoc.classifier_metadata_offset = offset_result_here->getint();
        }
      response.docInfo.push_back(resultdoc);
    }

    ++resultnum;
    ++result_here;
    if (!md_type.empty()) ++offset_result_here;
      }
    }
  }

  db_ptr->closedatabase();  // Important that local library doesn't leave any files open
  response.numDocs = numDocs;
  response.isApprox = Exact;
}

bool browsefilterclass::get_oaiinf_db_entries(FilterResponse_t &response, 
                      comerror_t &err, ostream &logout) 
{
  outconvertclass text_t2ascii;
  
  //logout << text_t2ascii << "browsefilterclass::get_oaiinf_db_entries\n";   

  // ONLY if we're doing any OAI stuff (FROAI will be set then) will we be here
  // So next try to open the oai-inf db if it exists for this collection

  
  if (!file_exists(oaidb_filename)) { // if the oaidb file doesn't even exist, let's not bother with oaidb
    
    logout << text_t2ascii
       << "warning: collection's oai-inf database \"" << oaidb_filename << "\" does not exist\n\n";    
    return false;

  } else { // let's try opening the oaidb file
    oaidb_ptr->setlogout(&logout);
    if (!oaidb_ptr->opendatabase (oaidb_filename, DB_READER, 100, false)) {
      // most likely a system problem (we have already checked that the database exists just above)
      logout << text_t2ascii
         << "system problem: open on database \""
         << oaidb_filename << "\" failed\n\n";
      err = systemProblem;
      return false;
    } // now we've opened the oai-inf db file successfully
  }

  infodbclass oai_info;
  ResultDocInfo_t resultdoc;

  text_tarray keys = oaidb_ptr->getkeys();

  text_tarray::iterator key_here = keys.begin();
  text_tarray::iterator key_end = keys.end();
  while (key_here != key_end) {    
    
    resultdoc.OID = (*key_here);
    
    // OAI_EARLIESTTIMESTAMP_OID is the OID of a special record that we'll ignore
    // here in browsefilter.cpp, since it's not a doc.
    // When the *metadata* for this special OID is requested, source.cpp will handle it
    if(resultdoc.OID == OAI_EARLIESTTIMESTAMP_OID) {
      ++key_here;
      continue;
    }

    if(!oaidb_ptr->getinfo(resultdoc.OID, oai_info)) {
      logout << text_t2ascii
         << "warning: lookup for node \"" << resultdoc.OID
         << "\" in etc/oai-inf db failed for browsefilter.\n\n";
    }
    // We don't need to get the oai metadata from oai-inf.db at this stage. That will be
    // handled by a separate metadata request. See collectserver::filter() and source.cpp's get_oai_metadata().
    /*
    else {
      //logout << text_t2ascii << "@@@@ found node \"" << resultdoc.OID << "\" in etc/oai-inf db.\n\n";
      
      resultdoc.metadata["oaiinf.status"].isRef = false;
      resultdoc.metadata["oaiinf.status"].values.push_back(oai_info["status"]);
      resultdoc.metadata["oaiinf.timestamp"].isRef = false;
      resultdoc.metadata["oaiinf.timestamp"].values.push_back(oai_info["timestamp"]);
      resultdoc.metadata["oaiinf.datestamp"].isRef = false;
      resultdoc.metadata["oaiinf.datestamp"].values.push_back(oai_info["datestamp"]);
    }
    */
    
    response.docInfo.push_back(resultdoc);
    ++key_here;    
    
  }

  // we're done with oai-inf db

  oaidb_ptr->closedatabase(); // don't leave files open

  return true;
}