source: trunk/gsdl/src/colservr/queryfilter.cpp@ 334

/**********************************************************************
 *
 * queryfilter.cpp -- 
 * Copyright (C) 1999  The New Zealand Digital Library Project
 *
 * PUT COPYRIGHT NOTICE HERE
 *
 * $Id: queryfilter.cpp 334 1999-07-01 09:29:21Z rjmcnab $
 *
 *********************************************************************/

/*
   $Log$
   Revision 1.9  1999/07/01 09:29:20  rjmcnab
   Changes for better reporting of number documents which match a query. Changes
   should still work as before with older versions of mg.

   Revision 1.8  1999/07/01 03:59:54  rjmcnab
   reduced MAXDOCS to 200 (more reasonable ???). I also added a virtual
   method for post-processing the query.

   Revision 1.7  1999/06/30 04:04:13  rjmcnab
   made stemming functions available from mgsearch and made the stems
   for the query terms available in queryinfo

   Revision 1.6  1999/06/29 22:06:23  rjmcnab
   Added a couple of fields to queryinfo to handle a special version
   of mg.

   Revision 1.5  1999/06/27 22:08:48  sjboddie
   now check for defaultindex, defaultsubcollection, and defaultlanguage
   entries in config files

   Revision 1.4  1999/06/16 02:03:25  sjboddie
   fixed bug in isApprox and set MAXDOCS to always be 500

   Revision 1.3  1999/04/19 23:56:09  rjmcnab
   Finished the gdbm metadata stuff

   Revision 1.2  1999/04/12 03:45:03  rjmcnab
   Finished the query filter.

   Revision 1.1  1999/04/06 22:22:09  rjmcnab
   Initial revision.

 */


#include "queryfilter.h"
#include "fileutil.h"
#include "queryinfo.h"

#define MAXDOCS 200 // note that maxdocs must be at least as large
                    // as the highest possible value of EndResults 

// some useful functions

// translate will return true if successful
static bool translate (gdbmclass *gdbmptr, int docnum, text_t &trans_OID) {
  infodbclass info;

  trans_OID.clear();

  // get the info
  if (gdbmptr == NULL) return false;
  if (!gdbmptr->getinfo(docnum, info)) return false;

  // translate
  if (info["section"].empty()) return false;

  trans_OID = info["section"];
  return true;
}


// do aditional query processing
void queryfilterclass::post_process (const queryparamclass &/*queryparams*/, const text_t &/*index*/,
                     queryresultsclass &/*queryresults*/) {
}



queryfilterclass::queryfilterclass () {
  gdbmptr = NULL;
  mgsearchptr = NULL;

  // -- onePerQuery StartResults   integer
  FilterOption_t filtopt;
  filtopt.name = "StartResults";
  filtopt.type = FilterOption_t::integert;
  filtopt.repeatable = FilterOption_t::onePerQuery;
  filtopt.defaultValue = "1";
  filtopt.validValues.push_back("1");
  filtopt.validValues.push_back("1000");
  filterOptions["StartResults"] = filtopt;

  // -- onePerQuery EndResults     integer
  filtopt.clear();
  filtopt.name = "EndResults";
  filtopt.type = FilterOption_t::integert;
  filtopt.repeatable = FilterOption_t::onePerQuery;
  filtopt.defaultValue = "10";
  filtopt.validValues.push_back("1");
  filtopt.validValues.push_back("1000");
  filterOptions["EndResults"] = filtopt;

  // -- onePerQuery QueryType      enumerated (boolean, ranked)
  filtopt.clear();
  filtopt.name = "QueryType";
  filtopt.type = FilterOption_t::enumeratedt;
  filtopt.repeatable = FilterOption_t::onePerQuery;
  filtopt.defaultValue = "ranked";
  filtopt.validValues.push_back("boolean");
  filtopt.validValues.push_back("ranked");
  filterOptions["QueryType"] = filtopt;

  // -- onePerTerm  Term           string ???
  filtopt.clear();
  filtopt.name = "Term";
  filtopt.type = FilterOption_t::stringt;
  filtopt.repeatable = FilterOption_t::onePerTerm;
  filtopt.defaultValue = "";
  filterOptions["Term"] = filtopt;

  // -- onePerTerm  Casefold       boolean
  filtopt.clear();
  filtopt.name = "Casefold";
  filtopt.type = FilterOption_t::booleant;
  filtopt.repeatable = FilterOption_t::onePerTerm;
  filtopt.defaultValue = "true";
  filtopt.validValues.push_back("false");
  filtopt.validValues.push_back("true");
  filterOptions["Casefold"] = filtopt;

  // -- onePerTerm  Stem           boolean
  filtopt.clear();
  filtopt.name = "Stem";
  filtopt.type = FilterOption_t::booleant;
  filtopt.repeatable = FilterOption_t::onePerTerm;
  filtopt.defaultValue = "false";
  filtopt.validValues.push_back("false");
  filtopt.validValues.push_back("true");
  filterOptions["Stem"] = filtopt;

  // -- onePerTerm  Index          enumerated
  filtopt.clear();
  filtopt.name = "Index";
  filtopt.type = FilterOption_t::enumeratedt;
  filtopt.repeatable = FilterOption_t::onePerTerm;
  filtopt.defaultValue = "";
  filterOptions["Index"] = filtopt;

  // -- onePerTerm  Subcollection  enumerated
  filtopt.clear();
  filtopt.name = "Subcollection";
  filtopt.type = FilterOption_t::enumeratedt;
  filtopt.repeatable = FilterOption_t::onePerTerm;
  filtopt.defaultValue = "";
  filterOptions["Subcollection"] = filtopt;

  // -- onePerTerm  Language  enumerated
  filtopt.clear();
  filtopt.name = "Language";
  filtopt.type = FilterOption_t::enumeratedt;
  filtopt.repeatable = FilterOption_t::onePerTerm;
  filtopt.defaultValue = "";
  filterOptions["Language"] = filtopt;
}

queryfilterclass::~queryfilterclass () {
}

void queryfilterclass::configure (const text_t &key, const text_tarray &cfgline) {
  filterclass::configure (key, cfgline);

  if (key == "indexmap") {
    indexmap.importmap (cfgline);
    
    // update the list of indexes in the filter information
    text_tarray options;
    indexmap.gettoarray (options);
    filterOptions["Index"].validValues = options;

  } else if (key == "defaultindex") {
    indexmap.from2to (cfgline[0], filterOptions["Index"].defaultValue);

  } else if (key == "subcollectionmap") {
    subcollectionmap.importmap (cfgline);

    // update the list of subcollections in the filter information
    text_tarray options;
    subcollectionmap.gettoarray (options);
    filterOptions["Subcollection"].validValues = options;

  } else if (key == "defaultsubcollection") {
    subcollectionmap.from2to (cfgline[0], filterOptions["Subcollection"].defaultValue);

  } else if (key == "languagemap") {
    languagemap.importmap (cfgline);

    // update the list of languages in the filter information
    text_tarray options;
    languagemap.gettoarray (options);
    filterOptions["Language"].validValues = options;

  } else if (key == "defaultlanguage") 
    languagemap.from2to (cfgline[0], filterOptions["Language"].defaultValue);
}

bool queryfilterclass::init (ostream &logout) {
  outconvertclass text_t2ascii;

  if (!filterclass::init(logout)) return false;

  // get the filename for the database and make sure it exists
  gdbm_filename = filename_cat(collectdir,"index","text",collection);
#ifdef _LITTLE_ENDIAN
  gdbm_filename += ".ldb";
#else
  gdbm_filename += ".bdb";
#endif
  if (!file_exists(gdbm_filename)) {
    logout << text_t2ascii
       << "error: gdbm database \""
       << gdbm_filename << "\" does not exist\n\n";
    return false;
  }

  return true;
}

void queryfilterclass::filter (const FilterRequest_t &request,
                   FilterResponse_t &response,
                   comerror_t &err, ostream &logout) {
  outconvertclass text_t2ascii;

  response.clear ();
  err = noError;
  if (gdbmptr == NULL) {
    // most likely a configuration problem
    logout << text_t2ascii 
       << "configuration error: queryfilter contains a null gdbmclass\n\n";
    err = configurationError;
    return;
  }
  if (mgsearchptr == NULL) {
    // most likely a configuration problem
    logout << text_t2ascii 
       << "configuration error: queryfilter contains a null mgsearchclass\n\n";
    err = configurationError;
    return;
  }

  // open the database
  gdbmptr->setlogout(&logout);
  if (!gdbmptr->opendatabase (gdbm_filename)) {
    // most likely a system problem (we have already checked that the
    // gdbm database exists)
    logout << text_t2ascii
       << "system problem: open on gdbm database \""
       << gdbm_filename << "\" failed\n\n";
    err = systemProblem;
    return;
  }

  // get the query parameters
  int startresults = filterOptions["StartResults"].defaultValue.getint();
  int endresults = filterOptions["EndResults"].defaultValue.getint();
  text_t index = filterOptions["Index"].defaultValue;
  text_t subcollection = filterOptions["Subcollection"].defaultValue;
  text_t language = filterOptions["Language"].defaultValue;
  queryparamclass queryparams;
  queryparams.collection = collection;
  queryparams.search_type = (filterOptions["QueryType"].defaultValue == "ranked");
  queryparams.casefolding = (filterOptions["Casefold"].defaultValue == "true");
  queryparams.stemming = (filterOptions["Stem"].defaultValue == "true");

  OptionValue_tarray::const_iterator options_here = request.filterOptions.begin();
  OptionValue_tarray::const_iterator options_end = request.filterOptions.end();
  while (options_here != options_end) {
    if ((*options_here).name == "StartResults") {
      startresults = (*options_here).value.getint();
    } else if ((*options_here).name == "EndResults") {
      endresults = (*options_here).value.getint();
    } else if ((*options_here).name == "QueryType") {
      queryparams.search_type = ((*options_here).value == "ranked");
    } else if ((*options_here).name == "Term") {
      queryparams.querystring = (*options_here).value;
    } else if ((*options_here).name == "Casefold") {
      queryparams.casefolding = ((*options_here).value == "true");
    } else if ((*options_here).name == "Stem") {
      queryparams.stemming = ((*options_here).value == "true");
    } else if ((*options_here).name == "Index") {
      index = (*options_here).value;
    } else if ((*options_here).name == "Subcollection") {
      subcollection = (*options_here).value;
    } else if ((*options_here).name == "Language") {
      language = (*options_here).value;
    } else {
      logout << text_t2ascii
         << "warning: unknown queryfilter option \""
         << (*options_here).name
         << "\" ignored.\n\n";
    }

    options_here++;
  }

  queryparams.search_index = index+subcollection+language;
  queryparams.maxdocs = MAXDOCS;

  // do query
  queryresultsclass queryresults;
  mgsearchptr->setcollectdir (collectdir);
  if (!mgsearchptr->search(queryparams, queryresults)) {
    // most likely a system problem
    logout << text_t2ascii
       << "system problem: could not do search with mg for index \""
       << queryparams.search_index << "\".\n\n";
    err = systemProblem;
    return;
  }

  // assemble document results
  if ((request.filterResultOptions & FROID) || (request.filterResultOptions & FRranking) ||
      (request.filterResultOptions & FRmetadata)) {

    // post-process the results if needed
    if (!queryresults.postprocessed && queryresults.orgterms.size() > 1 &&
    !queryresults.docs.empty()) {
      post_process (queryparams, index, queryresults);
      queryresults.postprocessed = true;
    }
  
    int resultnum = 1;
    ResultDocInfo_t resultdoc;
    text_t trans_OID;
    vector<docresultclass>::iterator docs_here = queryresults.docs.begin();
    vector<docresultclass>::iterator docs_end = queryresults.docs.end();
    
    while (docs_here != docs_end) {
      if (resultnum > endresults) break;
      
      // translate the document number
      if (!translate(gdbmptr, (*docs_here).docnum, trans_OID)) {
    logout << text_t2ascii
           << "warning: could not translate mg document number \""
           << (*docs_here).docnum << "\"to OID.\n\n";
    
      } else {
    // see if it is in the set (or the set is empty)
    if (request.docSet.empty() || in_set(request.docSet, trans_OID)) {
      if (resultnum >= startresults) {
        // add this document
        resultdoc.OID = trans_OID;
        resultdoc.ranking = (int)((*docs_here).docweight * 10000.0 + 0.5);

        // these next two are not available on all versions of mg
        resultdoc.num_terms_matched = (*docs_here).num_query_terms_matched; 
        resultdoc.query_phrase_match = (*docs_here).query_phrase_match;
        
        response.docInfo.push_back (resultdoc);
      }
      
      resultnum++;
    }
      }
      
      docs_here++;
    }
  }

  // assemble the term results
  if ((request.filterResultOptions & FRtermFreq) || (request.filterResultOptions & FRmatchTerms)) {
    // note: the terms have already been sorted and uniqued

    TermInfo_t terminfo;
    bool terms_first = true;
    vector<termfreqclass>::iterator terms_here = queryresults.terms.begin();
    vector<termfreqclass>::iterator terms_end = queryresults.terms.end();

    while (terms_here != terms_end) {
      terminfo.clear();
      terminfo.term = (*terms_here).termstr;
      terminfo.freq = (*terms_here).termfreq;
      if (terms_first) terminfo.matchTerms = queryresults.termvariants;
      terms_first = false;

      response.termInfo.push_back (terminfo);

      terms_here++;
    }
  }

  response.numDocs = queryresults.docs_matched;
  response.isApprox = queryresults.is_approx;
}