Changeset 325 for trunk/gsdl/src/colservr/mgsearch.cpp

Timestamp:

1999-07-01T15:54:49+12:00 (25 years ago)

Author:

rjmcnab

Message:

Added code to plug in the equivalent terms of each of the query terms.
Also added a function to get a raw utf8 encoded mg document (for speeding
up a phrase matching function)

File:

• trunk/gsdl/src/colservr/mgsearch.cpp (modified) (7 diffs)

trunk/gsdl/src/colservr/mgsearch.cpp

@@ -12 +12 @@
 /*
    $Log$
+   Revision 1.9  1999/07/01 03:54:48  rjmcnab
+   Added code to plug in the equivalent terms of each of the query terms.
+   Also added a function to get a raw utf8 encoded mg document (for speeding
+   up a phrase matching function)
+
    Revision 1.8  1999/06/30 04:04:12  rjmcnab
    made stemming functions available from mgsearch and made the stems
@@ -89 +94 @@
 static int casefold;
 
+static char *tempdoc = NULL;
+static int templen = 0;
+
 
 //////////////////////
@@ -119 +127 @@
   tempstr.setcarr((char *)(&word_stem[1]), word_stem[0]);
 
+  delete [] word_stem;
+  
   return tempstr;
 }
@@ -167 +177 @@
 }
 
+int termequivcallback(char *Word, int ULen,  int /*Freq*/, 
+              float /*Weight*/,  void *info) {
+  text_tset *equivterms = (text_tset *)info;
+  if (equivterms == NULL) return 0;
+
+  text_t thisterm;
+  thisterm.setcarr(Word, ULen);
+
+  equivterms->insert(thisterm);
+  
+  return 0;
+}
+
+
+void mgsearch_equivterms (const text_t &word, text_tset &equivterms) {
+  // allocate working stem space
+  int maxstemlen = mgq_getmaxstemlen ();
+  unsigned char *word_stem = new unsigned char [maxstemlen + 2];
+  if (word_stem == NULL) return;
+
+  // copy word to word_stem
+  int len = 0;
+  text_t::const_iterator here = word.begin();
+  text_t::const_iterator end = word.end();
+  while (len < maxstemlen && here != end) {
+    word_stem[len+1] = (unsigned char)(*here);
+    len++; here++;
+  }
+  word_stem[len+1] = '\0';
+  word_stem[0] = len;
+
+  // get the equivalent terms
+  mgq_equivterms (word_stem, termequivcallback, (void *)(&equivterms));
+  
+  delete [] word_stem;
+
+  return;
+}
+
+  text_tset utf8equivterms; // kept as utf8 string for fast matching
+
+
 // This callback is called once for each term in the query
 int termfreqcallback(char *Word, int ULen,  int Freq, 
              float /*Weight*/,  void *info) {
   queryresultsclass *queryresults = (queryresultsclass *)info;
+  if (queryresults == NULL) return 0;
 
   text_t term;
   term.setcarr(Word, ULen);
   termfreqclass termfreq;
+
   termfreq.termstr = to_uni(term);
-  termfreq.termstemstr = to_uni (mgsearch_stemword (term));
+  text_t utf8termstem = mgsearch_stemword (term);
+  termfreq.termstemstr = to_uni (utf8termstem);
+
+  mgsearch_equivterms (utf8termstem, termfreq.utf8equivterms);
+  
   termfreq.termfreq = Freq;
   queryresults->orgterms.push_back(termfreq);
@@ -196 +254 @@
 
 // This callback is for getting document text
-int doctextcallback(char *Word, int ULen,  int /*Freq*/, 
-            float /*Weight*/,  void *info) {
-  text_t *output = (text_t *)info;
-  if (output == NULL) return 0;
-  output->clear();
-
-  utf8inconvertclass inconvert;
-  convertclass::status_t status;
-  inconvert.reset ();
-  inconvert.setinput (Word, ULen);
-  inconvert.convert (*output, status);
-  
-  // replace all control-Cs with spaces
-  text_t::iterator here = output->begin();
-  text_t::iterator end = output->end();
-  while (here != end) {
-    if (*here == '\x3') *here = ' ';
-    here++;
-  }
+int doctextcallback(char *Doc, int ULen,  int /*Freq*/, 
+            float /*Weight*/,  void */*info*/) {
+  tempdoc = Doc;
+  templen = ULen;
   
   return 0;
@@ -302 +345 @@
   return to_uni (mgsearch_stemword (to_utf8 (word)));
 }
-  
+
+text_t mgsearchclass::stemword (text_t::const_iterator here, text_t::const_iterator end) {
+  return to_uni (mgsearch_stemword (to_utf8 (here, end)));
+}
+
 
 bool mgsearchclass::search(const queryparamclass &queryparams, 
@@ -473 +520 @@
                       const text_t &collection,
                       int docnum,
-                      text_t &output)
-{
-  int databaseloaded = 0;
-
+                      text_t &output) {
   output.clear();
 
+  // get the mg version of the document
+  char *mgdoc = NULL;
+  int doclen = 0;
+  if (!mgdocument (defaultindex, collection, docnum, mgdoc, doclen)) return false;
+  if (mgdoc == NULL) return false;
+
+  // replace all control-Cs with spaces
+  char *mgdoc_here = mgdoc;
+  char *mgdoc_end = mgdoc + doclen;
+  while (mgdoc_here < mgdoc_end) {
+    if (*mgdoc_here == '\x3') *mgdoc_here = ' ';
+    mgdoc_here++;
+  }
+
+  // convert this document to unicode
+  utf8inconvertclass inconvert;
+  convertclass::status_t status;
+  inconvert.reset ();
+  inconvert.setinput (mgdoc, doclen);
+  inconvert.convert (output, status);
+
+  return true;
+}
+
+
+bool mgsearchclass::mgdocument (const text_t &defaultindex, 
+                const text_t &collection,
+                int docnum,
+                char *&UDoc, int &ULen) {
+  bool databaseloaded = 0;
+
+  UDoc = NULL; ULen = 0;
+  
+  // see if we can make an appropriate database current
   char *ccollection = collection.getcstr();
   assert (ccollection != NULL);
-
-  // see if we can make an appropriate database current
   databaseloaded = load_text_database (ccollection);
-
+  delete ccollection;
+  
   // try and load the database
-  if (!databaseloaded)
-    {
-      // get the names of the index and text suffixes
-      char *idxsuffix = (getindexsuffix (collection, 
-                     defaultindex)).getcstr();
-      assert (idxsuffix != NULL);
-      char *txtsuffix = (getindexsuffix (collection, "text")).getcstr();
-      assert (txtsuffix != NULL);
-
-#ifdef __WIN32__
-      char *ccollectdir = (collectdir+"\\").getcstr(); assert (ccollectdir != NULL);
-#else
-      char *ccollectdir = collectdir.getcstr(); assert (ccollectdir != NULL);
-#endif
-      
-      databaseloaded = load_database(ccollection, ccollectdir, idxsuffix, txtsuffix);
-      
-      // free up the c strings
-      delete idxsuffix;
-      delete txtsuffix;
-      delete ccollectdir;
-    }
-
-  // free up the c collection string
-  delete ccollection;
-  
-  if (databaseloaded)
-    {
-      // retrieve the document from mg
-      char docstr[32];
-      sprintf(docstr, "%i", docnum);
-
-      mgq_ask(".set mode text");
-      mgq_ask(".set query docnums");
-      mgq_ask(docstr);
-      mgq_results (result_docs, 0, 1, doctextcallback, (void *)&output);
-    }
+  if (!databaseloaded) databaseloaded = makeindexcurrent (defaultindex, collection);
+  
+  if (databaseloaded) {
+    // retrieve the document from mg
+    char docstr[32];
+    sprintf(docstr, "%i", docnum);
+    
+    mgq_ask(".set mode text");
+    mgq_ask(".set query docnums");
+    mgq_ask(docstr);
+
+    tempdoc = NULL;
+    templen = 0;
+    mgq_results (result_docs, 0, 1, doctextcallback, (void *)NULL);
+    UDoc = tempdoc;
+    ULen = templen;
+  }
 
   return databaseloaded;