source: trunk/gsdl/src/colservr/queryfilter.cpp@ 501

/**********************************************************************
 *
 * queryfilter.cpp -- 
 * Copyright (C) 1999  The New Zealand Digital Library Project
 *
 * PUT COPYRIGHT NOTICE HERE
 *
 * $Id: queryfilter.cpp 501 1999-08-31 22:47:09Z rjmcnab $
 *
 *********************************************************************/

/*
   $Log$
   Revision 1.15  1999/08/31 22:47:09  rjmcnab
   Added matchmode option for some and all.

   Revision 1.14  1999/07/16 03:42:21  sjboddie
   changed isApprox

   Revision 1.13  1999/07/16 00:17:06  sjboddie
   got using phrasesearch for post-processing

   Revision 1.12  1999/07/09 02:19:43  rjmcnab
   Fixed a couple of compiler conflicts

   Revision 1.11  1999/07/08 20:49:44  rjmcnab
   Added result_num to the ResultDocInto_t structure.

   Revision 1.10  1999/07/07 06:19:46  rjmcnab
   Added ability to combine two or more independant queries.

   Revision 1.9  1999/07/01 09:29:20  rjmcnab
   Changes for better reporting of number documents which match a query. Changes
   should still work as before with older versions of mg.

   Revision 1.8  1999/07/01 03:59:54  rjmcnab
   reduced MAXDOCS to 200 (more reasonable ???). I also added a virtual
   method for post-processing the query.

   Revision 1.7  1999/06/30 04:04:13  rjmcnab
   made stemming functions available from mgsearch and made the stems
   for the query terms available in queryinfo

   Revision 1.6  1999/06/29 22:06:23  rjmcnab
   Added a couple of fields to queryinfo to handle a special version
   of mg.

   Revision 1.5  1999/06/27 22:08:48  sjboddie
   now check for defaultindex, defaultsubcollection, and defaultlanguage
   entries in config files

   Revision 1.4  1999/06/16 02:03:25  sjboddie
   fixed bug in isApprox and set MAXDOCS to always be 500

   Revision 1.3  1999/04/19 23:56:09  rjmcnab
   Finished the gdbm metadata stuff

   Revision 1.2  1999/04/12 03:45:03  rjmcnab
   Finished the query filter.

   Revision 1.1  1999/04/06 22:22:09  rjmcnab
   Initial revision.

 */


#include "queryfilter.h"
#include "fileutil.h"
#include "queryinfo.h"
#include "phrasesearch.h"
#include <assert.h>

#define MAXDOCS 50000 // note that maxdocs must be at least as large
                      // as the highest possible value of EndResults 

// some useful functions

// translate will return true if successful
static bool translate (gdbmclass *gdbmptr, int docnum, text_t &trans_OID) {
  infodbclass info;

  trans_OID.clear();

  // get the info
  if (gdbmptr == NULL) return false;
  if (!gdbmptr->getinfo(docnum, info)) return false;

  // translate
  if (info["section"].empty()) return false;

  trans_OID = info["section"];
  return true;
}


// whether document results are needed
static bool need_matching_docs (int filterResultOptions) {
  return ((filterResultOptions & FROID) || (filterResultOptions & FRranking) ||
      (filterResultOptions & FRmetadata));
}

// whether term information is needed
static bool need_term_info (int filterResultOptions) {
  return ((filterResultOptions & FRtermFreq) || (filterResultOptions & FRmatchTerms));
}

///////////////////////////////
// methods for resultsorderer_t
///////////////////////////////

resultsorderer_t::resultsorderer_t() {
  clear ();
}

void resultsorderer_t::clear() {
  compare_phrase_match = false;
  compare_terms_match = false;
  compare_doc_weight = true;

  docset = NULL;
}

bool resultsorderer_t::operator()(const int &t1, const int &t2) const {
  if (docset == NULL) return t1>t2;

  docresultmap::iterator t1_here = docset->find(t1);
  docresultmap::iterator t2_here = docset->find(t2);
  docresultmap::iterator end = docset->end();

  // sort all the document numbers not in the document set to
  // the end of the list
  if (t1_here == end) {
    if (t2_here == end) return t1>t2;
    else return true;
  } else if (t2_here == end) return false;
  
  if (compare_phrase_match) {
    if ((*t1_here).second.num_phrase_match > (*t2_here).second.num_phrase_match) return true;
    if ((*t1_here).second.num_phrase_match < (*t2_here).second.num_phrase_match) return false;
  }

  if (compare_terms_match) {
    if ((*t1_here).second.num_query_terms_matched > (*t2_here).second.num_query_terms_matched) return true;
    if ((*t1_here).second.num_query_terms_matched < (*t2_here).second.num_query_terms_matched) return false;
  }

  if (compare_doc_weight) {
    if ((*t1_here).second.docweight > (*t2_here).second.docweight) return true;
    if ((*t1_here).second.docweight < (*t2_here).second.docweight) return false;
  }

  return t1>t2;
}




/////////////////////////////////
// functions for queryfilterclass
/////////////////////////////////

// loads up phrases data structure with any phrases (that's the quoted bits)
// occuring in the querystring
void queryfilterclass::get_phrase_terms (const text_t &querystring, 
                     const termfreqclassarray &orgterms,
                     vector<termfreqclassarray> &phrases) {

  text_t::const_iterator here = querystring.begin();
  text_t::const_iterator end = querystring.end();

  termfreqclassarray tmpterms;

  int termcount = 0;
  bool foundquote = false;
  bool foundbreak = false;
  bool start = true;
  while (here != end) {
    if (*here == '\"') {
      if (foundquote) {
    if (!foundbreak && !start) {
      tmpterms.push_back (orgterms[termcount]);
      termcount ++;
    }
    if (tmpterms.size() > 1) {
      phrases.push_back (tmpterms);
      tmpterms.erase (tmpterms.begin(), tmpterms.end());
    }
    foundquote = false;
    foundbreak = true;
      } else foundquote = true;
    } else if (!is_unicode_letdig(*here)) {
      // found a break between terms
      if (!foundbreak && !start) {
    if (foundquote)
      tmpterms.push_back (orgterms[termcount]);
    termcount ++;
      }
      foundbreak = true;
    } else {
      start = false;
      foundbreak = false;
    }      
    here++;
  }
}

// do aditional query processing
void queryfilterclass::post_process (const queryparamclass &queryparams, 
                     queryresultsclass &queryresults) {

  // post-process the results if needed
  if (queryresults.orgterms.size() > 1 && !queryresults.docs.docset.empty()) {

    // get the terms between quotes (if any)
    vector<termfreqclassarray> phrases;
    get_phrase_terms (queryparams.querystring, queryresults.orgterms, phrases);

    if (phrases.size() > 0) {

      // get the long version of the index
      text_t longindex;
      indexmap.to2from (queryparams.index, longindex);
      
      vector<termfreqclassarray>::const_iterator this_phrase = phrases.begin();
      vector<termfreqclassarray>::const_iterator end_phrase = phrases.end();

      while (this_phrase != end_phrase) {

    // process each of the matched documents
    docresultmap::iterator docs_here = queryresults.docs.docset.begin();
    docresultmap::iterator docs_end = queryresults.docs.docset.end();
    while (docs_here != docs_end) {
      if (OID_phrase_search (*mgsearchptr, *gdbmptr, queryparams.index,
                 queryparams.subcollection, queryparams.language, 
                 longindex, queryparams.collection, *this_phrase,
                 (*docs_here).second.docnum)) {
        (*docs_here).second.num_phrase_match++;
      }
    
      docs_here++;
    }
    this_phrase++;
      }
    }
  }
}

// get the query parameters
void queryfilterclass::parse_query_params (const FilterRequest_t &request,
                       vector<queryparamclass> &query_params,
                       int &startresults,
                       int &endresults,
                       ostream &logout) {
  outconvertclass text_t2ascii;

  // set defaults for the return parameters
  query_params.erase(query_params.begin(), query_params.end());
  startresults = filterOptions["StartResults"].defaultValue.getint();
  endresults = filterOptions["EndResults"].defaultValue.getint();

  // set defaults for query parameters
  queryparamclass query;
  query.combinequery = "or"; // first one must be "or"
  query.collection = collection;
  query.index = filterOptions["Index"].defaultValue;
  query.subcollection = filterOptions["Subcollection"].defaultValue;
  query.language = filterOptions["Language"].defaultValue;
  query.querystring.clear();
  query.search_type = (filterOptions["QueryType"].defaultValue == "ranked");
  query.match_mode = (filterOptions["MatchMode"].defaultValue == "all");
  query.casefolding = (filterOptions["Casefold"].defaultValue == "true");
  query.stemming = (filterOptions["Stem"].defaultValue == "true");
  query.maxdocs = MAXDOCS; // default for single query

  OptionValue_tarray::const_iterator options_here = request.filterOptions.begin();
  OptionValue_tarray::const_iterator options_end = request.filterOptions.end();
  while (options_here != options_end) {
    if ((*options_here).name == "CombineQuery") {
      // add this query
      
      // "all", needed when combining queries where the document results are needed
      if (need_matching_docs (request.filterResultOptions)) query.maxdocs = -1; 
      query_params.push_back (query);

      // start on next query
      query.clear();
      query.combinequery = (*options_here).value;

      // set defaults for query parameters
      query.collection = collection;
      query.index = filterOptions["Index"].defaultValue;
      query.subcollection = filterOptions["Subcollection"].defaultValue;
      query.language = filterOptions["Language"].defaultValue;
      query.querystring.clear();
      query.search_type = (filterOptions["QueryType"].defaultValue == "ranked");
      query.match_mode = (filterOptions["MatchMode"].defaultValue == "all");
      query.casefolding = (filterOptions["Casefold"].defaultValue == "true");
      query.stemming = (filterOptions["Stem"].defaultValue == "true");
      
      // "all", needed when combining queries where the document results are needed
      if (need_matching_docs (request.filterResultOptions)) query.maxdocs = -1; 
      else query.maxdocs = MAXDOCS; // "all"
      
    } else if ((*options_here).name == "StartResults") {
      startresults = (*options_here).value.getint();
    } else if ((*options_here).name == "EndResults") {
      endresults = (*options_here).value.getint();
    } else if ((*options_here).name == "QueryType") {
      query.search_type = ((*options_here).value == "ranked");
    } else if ((*options_here).name == "MatchMode") {
      query.match_mode = ((*options_here).value == "all");
      if (query.match_mode == 1) query.maxdocs = -1;
    } else if ((*options_here).name == "Term") {
      query.querystring = (*options_here).value;
    } else if ((*options_here).name == "Casefold") {
      query.casefolding = ((*options_here).value == "true");
    } else if ((*options_here).name == "Stem") {
      query.stemming = ((*options_here).value == "true");
    } else if ((*options_here).name == "Index") {
      query.index = (*options_here).value;
    } else if ((*options_here).name == "Subcollection") {
      query.subcollection = (*options_here).value;
    } else if ((*options_here).name == "Language") {
      query.language = (*options_here).value;
    } else {
      logout << text_t2ascii
         << "warning: unknown queryfilter option \""
         << (*options_here).name
         << "\" ignored.\n\n";
    }

    options_here++;
  }

  // add the last query
  query_params.push_back (query);
}



// do query that might involve multiple sub queries
// mgsearchptr and gdbmptr are assumed to be valid
void queryfilterclass::do_multi_query (const FilterRequest_t &request,
                       const vector<queryparamclass> &query_params,
                       queryresultsclass &multiresults,
                       comerror_t &err, ostream &logout) {
  outconvertclass text_t2ascii;

  err = noError;
  mgsearchptr->setcollectdir (collectdir);
  multiresults.clear();
  
  vector<queryparamclass>::const_iterator query_here = query_params.begin();
  vector<queryparamclass>::const_iterator query_end = query_params.end();
  while (query_here != query_end) {
    queryresultsclass thisqueryresults;
    
    if (!mgsearchptr->search(*query_here, thisqueryresults)) {
      // most likely a system problem
      logout << text_t2ascii
         << "system problem: could not do search with mg for index \""
         << (*query_here).index << (*query_here).subcollection
         << (*query_here).language << "\".\n\n";
      err = systemProblem;
      return;
    }

    // combine the results
    if (need_matching_docs (request.filterResultOptions)) {
      // post-process the results if needed
      if (!thisqueryresults.postprocessed && thisqueryresults.orgterms.size() > 1 &&
      !thisqueryresults.docs.docset.empty()) {
    post_process (*query_here, thisqueryresults);
    thisqueryresults.postprocessed = true;
    multiresults.postprocessed = true;
      }
      
      if (query_params.size() == 1) {
    multiresults.docs = thisqueryresults.docs; // just one set of results
    multiresults.docs_matched = thisqueryresults.docs_matched;
    multiresults.is_approx = thisqueryresults.is_approx;
    
      } else {
    if ((*query_here).combinequery == "and") {
      multiresults.docs.combine_and (thisqueryresults.docs);
    } else if ((*query_here).combinequery == "or") {
      multiresults.docs.combine_or (thisqueryresults.docs);
    } else if ((*query_here).combinequery == "not") {
      multiresults.docs.combine_not (thisqueryresults.docs);
    }
    multiresults.docs_matched = multiresults.docs.docset.size();
    multiresults.is_approx = Exact;
      }
    }

    // combine the term information 
    if (need_term_info (request.filterResultOptions)) {
      // append the terms
      multiresults.orgterms.insert(multiresults.orgterms.end(),
                   thisqueryresults.orgterms.begin(),
                   thisqueryresults.orgterms.end());

      // add the term variants
      text_tset::iterator termvar_here = thisqueryresults.termvariants.begin();
      text_tset::iterator termvar_end = thisqueryresults.termvariants.end();
      while (termvar_here != termvar_end) {
    multiresults.termvariants.insert(*termvar_here);
    termvar_here++;
      }
    }
   
    query_here++;
  }

  // sort and unique the query terms
  multiresults.sortuniqqueryterms ();
}


void queryfilterclass::sort_doc_results (const FilterRequest_t &/*request*/,
                     docresultsclass &docs) {
  resultsorderer_t resultsorderer;
  resultsorderer.compare_phrase_match = true;
  resultsorderer.docset = &(docs.docset);

  // first get a list of document numbers
  docs.docnum_order();

  sort (docs.docorder.begin(), docs.docorder.end(), resultsorderer);
}



queryfilterclass::queryfilterclass () {
  gdbmptr = NULL;
  mgsearchptr = NULL;

  FilterOption_t filtopt;
  filtopt.name = "CombineQuery";
  filtopt.type = FilterOption_t::enumeratedt;
  filtopt.repeatable = FilterOption_t::onePerQuery;
  filtopt.defaultValue = "and";
  filtopt.validValues.push_back("and");
  filtopt.validValues.push_back("or");
  filtopt.validValues.push_back("not");
  filterOptions["CombineQuery"] = filtopt;

  // -- onePerQuery StartResults   integer
  filtopt.clear();
  filtopt.name = "StartResults";
  filtopt.type = FilterOption_t::integert;
  filtopt.repeatable = FilterOption_t::onePerQuery;
  filtopt.defaultValue = "1";
  filtopt.validValues.push_back("1");
  filtopt.validValues.push_back("1000");
  filterOptions["StartResults"] = filtopt;

  // -- onePerQuery EndResults     integer
  filtopt.clear();
  filtopt.name = "EndResults";
  filtopt.type = FilterOption_t::integert;
  filtopt.repeatable = FilterOption_t::onePerQuery;
  filtopt.defaultValue = "10";
  filtopt.validValues.push_back("1");
  filtopt.validValues.push_back("1000");
  filterOptions["EndResults"] = filtopt;

  // -- onePerQuery QueryType      enumerated (boolean, ranked)
  filtopt.clear();
  filtopt.name = "QueryType";
  filtopt.type = FilterOption_t::enumeratedt;
  filtopt.repeatable = FilterOption_t::onePerQuery;
  filtopt.defaultValue = "ranked";
  filtopt.validValues.push_back("boolean");
  filtopt.validValues.push_back("ranked");
  filterOptions["QueryType"] = filtopt;

  // -- onePerQuery MatchMode      enumerated (some, all)
  filtopt.clear();
  filtopt.name = "MatchMode";
  filtopt.type = FilterOption_t::enumeratedt;
  filtopt.repeatable = FilterOption_t::onePerQuery;
  filtopt.defaultValue = "some";
  filtopt.validValues.push_back("some");
  filtopt.validValues.push_back("all");
  filterOptions["QueryType"] = filtopt;

  // -- onePerTerm  Term           string ???
  filtopt.clear();
  filtopt.name = "Term";
  filtopt.type = FilterOption_t::stringt;
  filtopt.repeatable = FilterOption_t::onePerTerm;
  filtopt.defaultValue = "";
  filterOptions["Term"] = filtopt;

  // -- onePerTerm  Casefold       boolean
  filtopt.clear();
  filtopt.name = "Casefold";
  filtopt.type = FilterOption_t::booleant;
  filtopt.repeatable = FilterOption_t::onePerTerm;
  filtopt.defaultValue = "true";
  filtopt.validValues.push_back("false");
  filtopt.validValues.push_back("true");
  filterOptions["Casefold"] = filtopt;

  // -- onePerTerm  Stem           boolean
  filtopt.clear();
  filtopt.name = "Stem";
  filtopt.type = FilterOption_t::booleant;
  filtopt.repeatable = FilterOption_t::onePerTerm;
  filtopt.defaultValue = "false";
  filtopt.validValues.push_back("false");
  filtopt.validValues.push_back("true");
  filterOptions["Stem"] = filtopt;

  // -- onePerTerm  Index          enumerated
  filtopt.clear();
  filtopt.name = "Index";
  filtopt.type = FilterOption_t::enumeratedt;
  filtopt.repeatable = FilterOption_t::onePerTerm;
  filtopt.defaultValue = "";
  filterOptions["Index"] = filtopt;

  // -- onePerTerm  Subcollection  enumerated
  filtopt.clear();
  filtopt.name = "Subcollection";
  filtopt.type = FilterOption_t::enumeratedt;
  filtopt.repeatable = FilterOption_t::onePerTerm;
  filtopt.defaultValue = "";
  filterOptions["Subcollection"] = filtopt;

  // -- onePerTerm  Language  enumerated
  filtopt.clear();
  filtopt.name = "Language";
  filtopt.type = FilterOption_t::enumeratedt;
  filtopt.repeatable = FilterOption_t::onePerTerm;
  filtopt.defaultValue = "";
  filterOptions["Language"] = filtopt;
}

queryfilterclass::~queryfilterclass () {
}

void queryfilterclass::configure (const text_t &key, const text_tarray &cfgline) {
  filterclass::configure (key, cfgline);

  if (key == "indexmap") {
    indexmap.importmap (cfgline);
    
    // update the list of indexes in the filter information
    text_tarray options;
    indexmap.gettoarray (options);
    filterOptions["Index"].validValues = options;

  } else if (key == "defaultindex") {
    indexmap.from2to (cfgline[0], filterOptions["Index"].defaultValue);

  } else if (key == "subcollectionmap") {
    subcollectionmap.importmap (cfgline);

    // update the list of subcollections in the filter information
    text_tarray options;
    subcollectionmap.gettoarray (options);
    filterOptions["Subcollection"].validValues = options;

  } else if (key == "defaultsubcollection") {
    subcollectionmap.from2to (cfgline[0], filterOptions["Subcollection"].defaultValue);

  } else if (key == "languagemap") {
    languagemap.importmap (cfgline);

    // update the list of languages in the filter information
    text_tarray options;
    languagemap.gettoarray (options);
    filterOptions["Language"].validValues = options;

  } else if (key == "defaultlanguage") 
    languagemap.from2to (cfgline[0], filterOptions["Language"].defaultValue);
}

bool queryfilterclass::init (ostream &logout) {
  outconvertclass text_t2ascii;

  if (!filterclass::init(logout)) return false;

  // get the filename for the database and make sure it exists


  // yet another hack for niupepa
  if (collection == "niupepa") 
    gdbm_filename = filename_cat(collectdir,"index.new","text",collection);
  else
    gdbm_filename = filename_cat(collectdir,"index","text",collection);

#ifdef _LITTLE_ENDIAN
  gdbm_filename += ".ldb";
#else
  gdbm_filename += ".bdb";
#endif
  if (!file_exists(gdbm_filename)) {
    logout << text_t2ascii
       << "error: gdbm database \""
       << gdbm_filename << "\" does not exist\n\n";
    return false;
  }

  return true;
}

void queryfilterclass::filter (const FilterRequest_t &request,
                   FilterResponse_t &response,
                   comerror_t &err, ostream &logout) {
  outconvertclass text_t2ascii;

  response.clear ();
  err = noError;
  if (gdbmptr == NULL) {
    // most likely a configuration problem
    logout << text_t2ascii 
       << "configuration error: queryfilter contains a null gdbmclass\n\n";
    err = configurationError;
    return;
  }
  if (mgsearchptr == NULL) {
    // most likely a configuration problem
    logout << text_t2ascii 
       << "configuration error: queryfilter contains a null mgsearchclass\n\n";
    err = configurationError;
    return;
  }

  // open the database
  gdbmptr->setlogout(&logout);
  if (!gdbmptr->opendatabase (gdbm_filename, GDBM_READER, 100, false)) {
    // most likely a system problem (we have already checked that the
    // gdbm database exists)
    logout << text_t2ascii
       << "system problem: open on gdbm database \""
       << gdbm_filename << "\" failed\n\n";
    err = systemProblem;
    return;
  }

  // get the query parameters
  int startresults = filterOptions["StartResults"].defaultValue.getint();
  int endresults = filterOptions["EndResults"].defaultValue.getint();
  vector<queryparamclass> queryfilterparams;
  parse_query_params (request, queryfilterparams, startresults, endresults, logout);  
  
  // do query
  queryresultsclass queryresults;
  do_multi_query (request, queryfilterparams, queryresults, err, logout);
  if (err != noError) return;
  
  // assemble document results
  if (need_matching_docs (request.filterResultOptions)) {
    // sort the query results
    sort_doc_results (request, queryresults.docs);
    
    int resultnum = 1;
    ResultDocInfo_t resultdoc;
    text_t trans_OID;
    vector<int>::iterator docorder_here = queryresults.docs.docorder.begin();
    vector<int>::iterator docorder_end = queryresults.docs.docorder.end();

    while (docorder_here != docorder_end) {
      if (resultnum > endresults) break;
      
      // translate the document number
      if (!translate(gdbmptr, *docorder_here, trans_OID)) {
    logout << text_t2ascii
           << "warning: could not translate mg document number \""
           << *docorder_here << "\"to OID.\n\n";
    
      } else {
    docresultmap::iterator docset_here = queryresults.docs.docset.find (*docorder_here);
    
    // see if there is a result for this number,
    // if it is in the request set (or the request set is empty)
    if (docset_here != queryresults.docs.docset.end() &&
        (request.docSet.empty() || in_set(request.docSet, trans_OID))) {
      if (resultnum >= startresults) {
        // add this document
        resultdoc.OID = trans_OID;
        resultdoc.result_num = resultnum;
        resultdoc.ranking = (int)((*docset_here).second.docweight * 10000.0 + 0.5);

        // these next two are not available on all versions of mg
        resultdoc.num_terms_matched = (*docset_here).second.num_query_terms_matched; 
        resultdoc.num_phrase_match = (*docset_here).second.num_phrase_match;
        
        response.docInfo.push_back (resultdoc);
      }
      
      resultnum++;
    }
      }
      
      docorder_here++;
    }
  }

  // assemble the term results
  if (need_term_info(request.filterResultOptions)) {
    // note: the terms have already been sorted and uniqued

    TermInfo_t terminfo;
    bool terms_first = true;
    termfreqclassarray::iterator terms_here = queryresults.terms.begin();
    termfreqclassarray::iterator terms_end = queryresults.terms.end();

    while (terms_here != terms_end) {
      terminfo.clear();
      terminfo.term = (*terms_here).termstr;
      terminfo.freq = (*terms_here).termfreq;
      if (terms_first) {
    text_tset::iterator termvariants_here = queryresults.termvariants.begin();
    text_tset::iterator termvariants_end = queryresults.termvariants.end();
    while (termvariants_here != termvariants_end) {
      terminfo.matchTerms.push_back (*termvariants_here);
      termvariants_here++;
    }
      }
      terms_first = false;

      response.termInfo.push_back (terminfo);

      terms_here++;
    }
  }

  response.numDocs = queryresults.docs_matched;
  response.isApprox = queryresults.is_approx;
}