source: trunk/gsdl/src/colservr/mgsearch.cpp@ 301

/**********************************************************************
 *
 * mgsearch.cpp -- 
 * Copyright (C) 1999  The New Zealand Digital Library Project
 *
 * PUT COPYRIGHT NOTICE HERE
 *
 * $Id: mgsearch.cpp 301 1999-06-27 22:07:27Z sjboddie $
 *
 *********************************************************************/

/*
   $Log$
   Revision 1.7  1999/06/27 22:07:27  sjboddie
   got rid of all the old functions for dealing with dir indexes

   Revision 1.6  1999/06/09 00:41:32  sjboddie
   phrase searching now uses case-folding if it's turned on

   Revision 1.5  1999/02/21 22:31:35  rjmcnab

   Removed locateinfo.

   Revision 1.4  1999/02/03 01:13:27  sjboddie

   Got interface to handle subcollections and language subcollections -
   committed changes made to some of the collections

   Revision 1.3  1999/01/19 01:38:17  rjmcnab

   Made the source more portable.

   Revision 1.2  1999/01/12 01:51:02  rjmcnab

   Standard header.

   Revision 1.1  1999/01/08 09:02:16  rjmcnab

   Moved from src/library.

 */


#include "gsdlconf.h"
#include "mgsearch.h"
#include "fileutil.h"

#include <string.h>
#include <stdio.h>
#include <stdlib.h>
#include <ctype.h>

#if defined(GSDL_USE_OBJECTSPACE)
#  include <ospace\std\iostream>
#elif defined(GSDL_USE_IOS_H)
#  include <iostream.h>
#else
#  include <iostream>
#endif

#if defined(__WIN32__)
// gdbm stuff
#  include "autoconf.h"
#  include "systems.h"
#  include "gdbmconst.h"
#  include "gdbm.h"
#else
#  include <gdbm.h>
#endif

  
#include <assert.h>

#include "mgq.h"
// #include "locateinfo.h"
#include "gsdlunicode.h"
#include "unitool.h"


/////////////
// globals //
/////////////

static char *quotedquery = NULL;
static int casefold;


////////////////////////
// callback functions //
////////////////////////

// This routine is called for each document found in a search
// it assumes that cache_num is set up correctly to point to
// a suitable result cache
int ourquerycallback(char *UDoc, int /*ULen*/, int DocNum, 
             float Weight, void *info) {

  
  queryresultsclass *queryresults = (queryresultsclass * )info;

  // check the returned document for the presence of the
  // quoted part of the query, if there was one
 
  //  if (UDoc != NULL && quotedquery != NULL &&
  //    quotedquery[0] != '\0' && strstr (UDoc, quotedquery) == NULL) return 0;
  
  
  if (UDoc != NULL && quotedquery != NULL && quotedquery[0] != '\0') {

      if (casefold) {
          int len;
          for (len = 0; quotedquery[len] != '\0'; len ++)
              quotedquery[len] = tolower (quotedquery[len]);
          for (len = 0; UDoc[len] != '\0'; len ++)
              UDoc[len] = tolower (UDoc[len]);
      }
      if (strstr (UDoc, quotedquery) == NULL) return 0;
  }

  // append this entry to the document results
  docresultclass docresult;
  docresult.docnum = DocNum;
  docresult.docweight = Weight;

  queryresults->docs.push_back(docresult);
  
  return 0;
}

// This callback is called once for each term in the query
int termfreqcallback(char *Word, int ULen,  int Freq, 
             float /*Weight*/,  void *info) {
  queryresultsclass *queryresults = (queryresultsclass *)info;

  text_t term;
  term.setcarr(Word, ULen);
  termfreqclass termfreq;
  termfreq.termstr = to_uni(term);
  termfreq.termfreq = Freq;
  queryresults->terms.push_back(termfreq);
  
  return 0;
}

// this callback is called once for each variation of each term
int termscallback(char *Word, int ULen, int /*Freq*/,
          float /*Weight*/, void *info) {

  text_t term;
  term.setcarr(Word, ULen);
  queryresultsclass *queryresults = (queryresultsclass *)info;
  queryresults->termvariants.push_back(to_uni(term));

  return 0;
}

// This callback is for getting document text
int doctextcallback(char *Word, int ULen,  int /*Freq*/, 
            float /*Weight*/,  void *info) {
  text_t *output = (text_t *)info;
  if (output == NULL) return 0;
  output->clear();

  utf8inconvertclass inconvert;
  convertclass::status_t status;
  inconvert.reset ();
  inconvert.setinput (Word, ULen);
  inconvert.convert (*output, status);
  
  // replace all control-Cs with spaces
  text_t::iterator here = output->begin();
  text_t::iterator end = output->end();
  while (here != end) {
    if (*here == '\x3') *here = ' ';
    here++;
  }
  
  return 0;
}


static text_t getindexsuffix (const text_t &collection, 
                  const text_t &index) {
  text_t indexsuffix = "index";
  indexsuffix = filename_cat (indexsuffix, index);
  indexsuffix = filename_cat (indexsuffix, collection);
  return indexsuffix;
}




////////////////////
// mgsearch class //
////////////////////

mgsearchclass::mgsearchclass ()
{
  cache = new querycache (RESULTCACHESIZE);
}

mgsearchclass::~mgsearchclass () 
{
  if (cache != NULL) 
    {
      delete cache;
      cache = NULL;
    }
}


void mgsearchclass::setcollectdir (const text_t &thecollectdir)
{
  collectdir = thecollectdir;
}


bool mgsearchclass::search(const queryparamclass &queryparams, 
               queryresultsclass &queryresults)
{
  bool databaseloaded = true;

  assert (cache != NULL);

  queryresults.clear();

  // first check the cache
  if (cache->find(queryparams, queryresults))
    return true;

  // make sure there is a query to be processed
  text_t::const_iterator queryhere = queryparams.querystring.begin();
  text_t::const_iterator queryend = queryparams.querystring.end();
  while (queryhere != queryend) {
    if (is_unicode_letdig (*queryhere)) break;
    queryhere++;
  }

  // if we reached the end of the query string without finding
  // any alphanumeric characters then return no results (and say
  // the database was loaded)
  if (queryhere == queryend) return true;

  casefold = queryparams.casefolding;

  // get the names of the collection, index and text suffixes
  char *ccollection = queryparams.collection.getcstr();
  assert (ccollection != NULL);
  char *idxsuffix = (getindexsuffix (queryparams.collection, 
                     queryparams.search_index)).getcstr();
  assert (idxsuffix != NULL);
  char *txtsuffix = (getindexsuffix (queryparams.collection, "text")).getcstr();
  assert (txtsuffix != NULL);

#ifdef __WIN32__
  char *ccollectdir = (collectdir+"\\").getcstr(); assert (ccollectdir != NULL);
#else
  char *ccollectdir = collectdir.getcstr(); assert (ccollectdir != NULL);
#endif

  if (load_database(ccollection, ccollectdir, idxsuffix, txtsuffix))
    {
      setsearchmode (queryparams);
      submitquery (queryparams);
      getresults (queryresults);
    }
  else databaseloaded = false;

  // free up the c strings
  delete ccollection;
  delete idxsuffix;
  delete txtsuffix;
  delete ccollectdir;

  return databaseloaded;
}


void mgsearchclass::setsearchmode (const queryparamclass &queryparams)
{
  mgq_ask(".set expert true");
  mgq_ask(".set accumulator_method list");
  mgq_ask(".set max_accumulators 50000");
  mgq_ask(".set verbatim true");
  mgq_ask(".unset skip_dump");
  mgq_ask(".set mode docnums");

  switch (queryparams.search_type) 
    {
    case 0: mgq_ask(".set query boolean");  break;
    case 1:  mgq_ask(".set query ranked"); break;
    }
  switch (queryparams.casefolding) 
    {
    case 1: mgq_ask(".set casefold on");  break;
    case 0: mgq_ask(".set casefold off"); break;
    }
  switch (queryparams.stemming) 
    {
    case 1: mgq_ask(".set stem on");  break;
    case 0: mgq_ask(".set stem off"); break;
    }
  mgq_ask(".set heads_length 150");
  
  char maxdocstr[32];
  sprintf(maxdocstr, ".set maxdocs %i", queryparams.maxdocs);
  mgq_ask(maxdocstr);
}


void mgsearchclass::submitquery (const queryparamclass &queryparams)
{
  // sort out the query string
  text_t ttquerystring = queryparams.querystring;
  text_t ttquotedquery;
  extractquoted (ttquerystring, ttquotedquery);
  filterquery (ttquerystring);
  
  // turn the strings into c strings for mg
  if (quotedquery != NULL) // quotedquery is a global
    {
      delete quotedquery;
      quotedquery = NULL;
    }

  // quotedquery will be deleted on the next call to this function
  quotedquery = to_utf8(ttquotedquery).getcstr ();
  char *querystring = to_utf8(ttquerystring).getcstr();
  
  // submit the query
  mgq_ask(querystring);

  delete querystring;
}


void mgsearchclass::getresults (queryresultsclass &queryresults)
{
  if (quotedquery[0] == '\0') 
    { 
      // don't need the text
      mgq_results(result_docnums, 0, MAXNUMDOCS, 
          ourquerycallback, (void *)(&queryresults));
    }
  else
    {
      // we need the text for this one
      mgq_results(result_docs, 0, MAXNUMDOCS, 
          ourquerycallback, (void *)(&queryresults));
    }
  
  // get the term frequencies
  mgq_results(result_termfreqs, 0, MAXNUMTERMS, 
          termfreqcallback, (void *)(&queryresults));
  mgq_results(result_terms, 0, MAXNUMTERMS,
              termscallback, (void *)(&queryresults));
  queryresults.sortqueryterms();
  queryresults.uniqqueryterms();
}


void mgsearchclass::extractquoted (text_t &ttquerystring, text_t &ttquotedquery)
{
  ttquotedquery.clear();

  text_t::iterator ithere = ttquerystring.begin ();
  text_t::iterator itend = ttquerystring.end ();

  bool inquote = false;

  while (ithere != itend)
    {
      if ((*ithere) == '\"')
    {
      if (!inquote) ttquotedquery.clear ();
      inquote = !inquote;
      *ithere = ' '; // delete the quote
    }
      else if (inquote)
    {
      ttquotedquery.push_back(*ithere);
      *ithere = ' ';
    }
      
      ithere++;
    }
}


void mgsearchclass::filterquery (text_t &ttquerystring) {
  text_t::iterator ithere = ttquerystring.begin ();
  text_t::iterator itend = ttquerystring.end ();
  
  // remove all non alphanumeric characters
  while (ithere != itend) {
    if (!is_unicode_letdig(*ithere)) (*ithere) = ' ';
    ithere++;
  }
}


// the document text for 'docnum' is placed in 'output'
// docTargetDocument returns 'true' if it was able to
// try to get a document
// collection is needed to see if an index from the 
// collection is loaded. If no index has been loaded
// defaultindex is needed to load one
bool mgsearchclass::docTargetDocument(const text_t &defaultindex, 
                      const text_t &collection,
                      int docnum,
                      text_t &output)
{
  int databaseloaded = 0;

  output.clear();

  char *ccollection = collection.getcstr();
  assert (ccollection != NULL);

  // see if we can make an appropriate database current
  databaseloaded = load_text_database (ccollection);

  // try and load the database
  if (!databaseloaded)
    {
      // get the names of the index and text suffixes
      char *idxsuffix = (getindexsuffix (collection, 
                     defaultindex)).getcstr();
      assert (idxsuffix != NULL);
      char *txtsuffix = (getindexsuffix (collection, "text")).getcstr();
      assert (txtsuffix != NULL);

#ifdef __WIN32__
      char *ccollectdir = (collectdir+"\\").getcstr(); assert (ccollectdir != NULL);
#else
      char *ccollectdir = collectdir.getcstr(); assert (ccollectdir != NULL);
#endif
      
      databaseloaded = load_database(ccollection, ccollectdir, idxsuffix, txtsuffix);
      
      // free up the c strings
      delete idxsuffix;
      delete txtsuffix;
      delete ccollectdir;
    }

  // free up the c collection string
  delete ccollection;
  
  if (databaseloaded)
    {
      // retrieve the document from mg
      char docstr[32];
      sprintf(docstr, "%i", docnum);

      mgq_ask(".set mode text");
      mgq_ask(".set query docnums");
      mgq_ask(docstr);
      mgq_results (result_docs, 0, 1, doctextcallback, (void *)&output);
    }

  return databaseloaded;
}