Changeset 319 for trunk/gsdl/src/colservr/mgsearch.cpp

Timestamp:

1999-06-30T16:04:14+12:00 (25 years ago)

Author:

rjmcnab

Message:

made stemming functions available from mgsearch and made the stems
for the query terms available in queryinfo

File:

• trunk/gsdl/src/colservr/mgsearch.cpp (modified) (10 diffs)

trunk/gsdl/src/colservr/mgsearch.cpp

@@ -12 +12 @@
 /*
    $Log$
+   Revision 1.8  1999/06/30 04:04:12  rjmcnab
+   made stemming functions available from mgsearch and made the stems
+   for the query terms available in queryinfo
+
    Revision 1.7  1999/06/27 22:07:27  sjboddie
    got rid of all the old functions for dealing with dir indexes
@@ -86 +90 @@
 
 
+//////////////////////
+// useful functions //
+//////////////////////
+
+
+// input and output are in utf8
+text_t mgsearch_stemword (const text_t &word) {
+  // allocate working stem space
+  int maxstemlen = mgq_getmaxstemlen ();
+  unsigned char *word_stem = new unsigned char [maxstemlen + 2];
+  if (word_stem == NULL) return "";
+
+  // copy word to word_stem
+  int len = 0;
+  text_t::const_iterator here = word.begin();
+  text_t::const_iterator end = word.end();
+  while (len < maxstemlen && here != end) {
+    word_stem[len+1] = (unsigned char)(*here);
+    len++; here++;
+  }
+  word_stem[len+1] = '\0';
+  word_stem[0] = len;
+
+  mgq_stemword (word_stem);
+
+  // copy word_stem back to tempstr
+  text_t tempstr;
+  tempstr.setcarr((char *)(&word_stem[1]), word_stem[0]);
+
+  return tempstr;
+}
+
+
+
 ////////////////////////
 // callback functions //
@@ -121 +159 @@
   docresultclass docresult;
   docresult.docnum = DocNum;
-  docresult.docweight = Weight;
-
+  docresult.num_query_terms_matched = (int)(Weight/100.0); // will always be 0 on some versions of mg...
+  docresult.docweight = Weight - docresult.num_query_terms_matched*100;
+  
   queryresults->docs.push_back(docresult);
   
@@ -137 +176 @@
   termfreqclass termfreq;
   termfreq.termstr = to_uni(term);
+  termfreq.termstemstr = to_uni (mgsearch_stemword (term));
   termfreq.termfreq = Freq;
-  queryresults->terms.push_back(termfreq);
+  queryresults->orgterms.push_back(termfreq);
   
   return 0;
@@ -144 +184 @@
 
 // this callback is called once for each variation of each term
-int termscallback(char *Word, int ULen, int /*Freq*/,
-          float /*Weight*/, void *info) {
+int termvariantscallback(char *Word, int ULen, int /*Freq*/,
+             float /*Weight*/, void *info) {
 
   text_t term;
@@ -215 +255 @@
 }
 
+// you only need to use this function before doing any stemming
+// casefolding and stemming will be set if values for them are
+// provided (0 or 1).
+// makeindexcurrent returns true if it was able to load the database
+bool mgsearchclass::makeindexcurrent (const text_t &index,
+                      const text_t &collection,
+                      int casefolding,
+                      int stemming) {
+  bool databaseloaded = true;
+
+  // get the names of the collection, index and text suffixes
+  char *ccollection = collection.getcstr();
+  assert (ccollection != NULL);
+  char *idxsuffix = (getindexsuffix (collection, index)).getcstr();
+  assert (idxsuffix != NULL);
+  char *txtsuffix = (getindexsuffix (collection, "text")).getcstr();
+  assert (txtsuffix != NULL);
+
+#ifdef __WIN32__
+  char *ccollectdir = (collectdir+"\\").getcstr(); assert (ccollectdir != NULL);
+#else
+  char *ccollectdir = collectdir.getcstr(); assert (ccollectdir != NULL);
+#endif
+
+  if (load_database(ccollection, ccollectdir, idxsuffix, txtsuffix)) {
+    if (casefolding == 0) mgq_ask(".set casefold off");
+    else if (casefolding > 0) mgq_ask(".set casefold on");
+    if (stemming == 0) mgq_ask(".set stem off");
+    else if (stemming > 0) mgq_ask(".set stem on");
+    
+  } else databaseloaded = false;
+
+  // free up the c strings
+  delete ccollection;
+  delete idxsuffix;
+  delete txtsuffix;
+  delete ccollectdir;
+
+  return databaseloaded;
+}
+
+
+// stem word uses the values set in the last call to makeindexcurrent
+// to stem the word. It is assumed that word is in unicode
+text_t mgsearchclass::stemword (const text_t &word) {
+  return to_uni (mgsearch_stemword (to_utf8 (word)));
+}
+  
 
 bool mgsearchclass::search(const queryparamclass &queryparams, 
-               queryresultsclass &queryresults)
-{
-  bool databaseloaded = true;
-
+               queryresultsclass &queryresults) {
   assert (cache != NULL);
 
@@ -226 +311 @@
 
   // first check the cache
-  if (cache->find(queryparams, queryresults))
-    return true;
+  if (cache->find(queryparams, queryresults)) return true;
 
   // make sure there is a query to be processed
@@ -244 +328 @@
   casefold = queryparams.casefolding;
 
-  // get the names of the collection, index and text suffixes
-  char *ccollection = queryparams.collection.getcstr();
-  assert (ccollection != NULL);
-  char *idxsuffix = (getindexsuffix (queryparams.collection, 
-                     queryparams.search_index)).getcstr();
-  assert (idxsuffix != NULL);
-  char *txtsuffix = (getindexsuffix (queryparams.collection, "text")).getcstr();
-  assert (txtsuffix != NULL);
-
-#ifdef __WIN32__
-  char *ccollectdir = (collectdir+"\\").getcstr(); assert (ccollectdir != NULL);
-#else
-  char *ccollectdir = collectdir.getcstr(); assert (ccollectdir != NULL);
-#endif
-
-  if (load_database(ccollection, ccollectdir, idxsuffix, txtsuffix))
-    {
-      setsearchmode (queryparams);
-      submitquery (queryparams);
-      getresults (queryresults);
-    }
-  else databaseloaded = false;
-
-  // free up the c strings
-  delete ccollection;
-  delete idxsuffix;
-  delete txtsuffix;
-  delete ccollectdir;
-
-  return databaseloaded;
+  if (makeindexcurrent (queryparams.search_index, queryparams.collection)) {
+    setsearchmode (queryparams);
+    submitquery (queryparams);
+    getresults (queryresults);
+    return true;
+  }
+
+  return false;
 }
 
@@ -280 +342 @@
 {
   mgq_ask(".set expert true");
+  mgq_ask(".set sorted_terms true");
   mgq_ask(".set accumulator_method list");
   mgq_ask(".set max_accumulators 50000");
@@ -353 +416 @@
   mgq_results(result_termfreqs, 0, MAXNUMTERMS, 
           termfreqcallback, (void *)(&queryresults));
+  queryresults.sortuniqqueryterms();
+
+  // get term variants 
   mgq_results(result_terms, 0, MAXNUMTERMS,
-              termscallback, (void *)(&queryresults));
-  queryresults.sortqueryterms();
-  queryresults.uniqqueryterms();
+              termvariantscallback, (void *)(&queryresults));
 }