Changeset 319 for trunk/gsdl/src/colservr

Timestamp:

1999-06-30T16:04:14+12:00 (25 years ago)

Author:

rjmcnab

Message:

made stemming functions available from mgsearch and made the stems
for the query terms available in queryinfo

Location:

trunk/gsdl/src/colservr

Files:

• mgq.c (modified) (3 diffs)
• mgq.h (modified) (1 diff)
• mgsearch.cpp (modified) (10 diffs)
• mgsearch.h (modified) (1 diff)
• queryfilter.cpp (modified) (2 diffs)
• queryinfo.cpp (modified) (10 diffs)
• queryinfo.h (modified) (3 diffs)

trunk/gsdl/src/colservr/mgq.c

@@ -12 +12 @@
 /*
    $Log$
+   Revision 1.5  1999/06/30 04:04:11  rjmcnab
+   made stemming functions available from mgsearch and made the stems
+   for the query terms available in queryinfo
+
    Revision 1.4  1999/06/28 08:56:29  rjmcnab
    A bit of hacking to remove the restriction that the index to get
@@ -60 +64 @@
 #include "term_lists.h"
 #include "local_strings.h"
+
+#include "words.h"
+#include "stemmer.h"
   
 #ifdef __cplusplus
@@ -561 +568 @@
 
 
+/* use mgq_getmaxstemlen to determine the length of the word stems to pass */
+/* to mgq_stemword */
+int mgq_getmaxstemlen () {
+  return MAXSTEMLEN;
+}
+
+/* note: the stemming method and the stemmer come from the last query */
+/* "word" should be at least maxstemlen+1 long and it is a string that */
+/* starts with the string length */
+void mgq_stemword (unsigned char *word) {
+  int stem_method = 0;
+  query_data *qd = NULL;
+
+  if (cur_cachenum == -1) return;
+  qd = dbcache[cur_cachenum].qd;
+  if (qd == NULL || word == NULL) return;
+
+  if (qd->sd->sdh.indexed) {
+    stem_method = BooleanEnv(GetEnv("casefold"),0) | (BooleanEnv(GetEnv("stem"),0) << 1);
+  } else {
+    stem_method = qd->sd->sdh.stem_method;
+  }
+
+  stemmer (stem_method, qd->sd->sdh.stemmer_num, word);
+}
+
+
+
 int is_dbcache_full (void) {
   init_dbcache ();

trunk/gsdl/src/colservr/mgq.h

@@ -18 +18 @@
 #endif
 
+  
 enum result_kinds {
-    result_docs,      /* Return the documents found in last search */
-    result_docnums,   /* Return document id numbers and weights */
-    result_termfreqs, /* Return terms and frequencies */
-    result_terms      /* Return matching query terms */
-    };
+  result_docs,      /* Return the documents found in last search */
+  result_docnums,   /* Return document id numbers and weights */
+  result_termfreqs, /* Return terms and frequencies */
+  result_terms      /* Return matching query terms */
+};
 
 int mgq_ask(char *line);
-int mgq_results(enum result_kinds kind,int skip, int howmany, int (*sender)(char *, int, int, float, void *), void *ptr);
+int mgq_results(enum result_kinds kind,int skip, int howmany,
+        int (*sender)(char *, int, int, float, void *), void *ptr);
 int mgq_numdocs(void);
 int mgq_numterms(void);
 
+/* use mgq_getmaxstemlen to determine the length of the word stems to pass */
+/* to mgq_stemword */
+int mgq_getmaxstemlen ();
 
+/* note: the stemming method and the stemmer come from the last query */
+/* "word" should be at least maxstemlen+1 long and it is a string that */
+/* starts with the string length */
+void mgq_stemword (unsigned char *word);
+
+  
 int is_dbcache_full (void);
 int load_database (char *collection, char *mgdir, char *gensuffix, char *textsuffix);

trunk/gsdl/src/colservr/mgsearch.cpp

@@ -12 +12 @@
 /*
    $Log$
+   Revision 1.8  1999/06/30 04:04:12  rjmcnab
+   made stemming functions available from mgsearch and made the stems
+   for the query terms available in queryinfo
+
    Revision 1.7  1999/06/27 22:07:27  sjboddie
    got rid of all the old functions for dealing with dir indexes
@@ -86 +90 @@
 
 
+//////////////////////
+// useful functions //
+//////////////////////
+
+
+// input and output are in utf8
+text_t mgsearch_stemword (const text_t &word) {
+  // allocate working stem space
+  int maxstemlen = mgq_getmaxstemlen ();
+  unsigned char *word_stem = new unsigned char [maxstemlen + 2];
+  if (word_stem == NULL) return "";
+
+  // copy word to word_stem
+  int len = 0;
+  text_t::const_iterator here = word.begin();
+  text_t::const_iterator end = word.end();
+  while (len < maxstemlen && here != end) {
+    word_stem[len+1] = (unsigned char)(*here);
+    len++; here++;
+  }
+  word_stem[len+1] = '\0';
+  word_stem[0] = len;
+
+  mgq_stemword (word_stem);
+
+  // copy word_stem back to tempstr
+  text_t tempstr;
+  tempstr.setcarr((char *)(&word_stem[1]), word_stem[0]);
+
+  return tempstr;
+}
+
+
+
 ////////////////////////
 // callback functions //
@@ -121 +159 @@
   docresultclass docresult;
   docresult.docnum = DocNum;
-  docresult.docweight = Weight;
-
+  docresult.num_query_terms_matched = (int)(Weight/100.0); // will always be 0 on some versions of mg...
+  docresult.docweight = Weight - docresult.num_query_terms_matched*100;
+  
   queryresults->docs.push_back(docresult);
   
@@ -137 +176 @@
   termfreqclass termfreq;
   termfreq.termstr = to_uni(term);
+  termfreq.termstemstr = to_uni (mgsearch_stemword (term));
   termfreq.termfreq = Freq;
-  queryresults->terms.push_back(termfreq);
+  queryresults->orgterms.push_back(termfreq);
   
   return 0;
@@ -144 +184 @@
 
 // this callback is called once for each variation of each term
-int termscallback(char *Word, int ULen, int /*Freq*/,
-          float /*Weight*/, void *info) {
+int termvariantscallback(char *Word, int ULen, int /*Freq*/,
+             float /*Weight*/, void *info) {
 
   text_t term;
@@ -215 +255 @@
 }
 
+// you only need to use this function before doing any stemming
+// casefolding and stemming will be set if values for them are
+// provided (0 or 1).
+// makeindexcurrent returns true if it was able to load the database
+bool mgsearchclass::makeindexcurrent (const text_t &index,
+                      const text_t &collection,
+                      int casefolding,
+                      int stemming) {
+  bool databaseloaded = true;
+
+  // get the names of the collection, index and text suffixes
+  char *ccollection = collection.getcstr();
+  assert (ccollection != NULL);
+  char *idxsuffix = (getindexsuffix (collection, index)).getcstr();
+  assert (idxsuffix != NULL);
+  char *txtsuffix = (getindexsuffix (collection, "text")).getcstr();
+  assert (txtsuffix != NULL);
+
+#ifdef __WIN32__
+  char *ccollectdir = (collectdir+"\\").getcstr(); assert (ccollectdir != NULL);
+#else
+  char *ccollectdir = collectdir.getcstr(); assert (ccollectdir != NULL);
+#endif
+
+  if (load_database(ccollection, ccollectdir, idxsuffix, txtsuffix)) {
+    if (casefolding == 0) mgq_ask(".set casefold off");
+    else if (casefolding > 0) mgq_ask(".set casefold on");
+    if (stemming == 0) mgq_ask(".set stem off");
+    else if (stemming > 0) mgq_ask(".set stem on");
+    
+  } else databaseloaded = false;
+
+  // free up the c strings
+  delete ccollection;
+  delete idxsuffix;
+  delete txtsuffix;
+  delete ccollectdir;
+
+  return databaseloaded;
+}
+
+
+// stem word uses the values set in the last call to makeindexcurrent
+// to stem the word. It is assumed that word is in unicode
+text_t mgsearchclass::stemword (const text_t &word) {
+  return to_uni (mgsearch_stemword (to_utf8 (word)));
+}
+  
 
 bool mgsearchclass::search(const queryparamclass &queryparams, 
-               queryresultsclass &queryresults)
-{
-  bool databaseloaded = true;
-
+               queryresultsclass &queryresults) {
   assert (cache != NULL);
 
@@ -226 +311 @@
 
   // first check the cache
-  if (cache->find(queryparams, queryresults))
-    return true;
+  if (cache->find(queryparams, queryresults)) return true;
 
   // make sure there is a query to be processed
@@ -244 +328 @@
   casefold = queryparams.casefolding;
 
-  // get the names of the collection, index and text suffixes
-  char *ccollection = queryparams.collection.getcstr();
-  assert (ccollection != NULL);
-  char *idxsuffix = (getindexsuffix (queryparams.collection, 
-                     queryparams.search_index)).getcstr();
-  assert (idxsuffix != NULL);
-  char *txtsuffix = (getindexsuffix (queryparams.collection, "text")).getcstr();
-  assert (txtsuffix != NULL);
-
-#ifdef __WIN32__
-  char *ccollectdir = (collectdir+"\\").getcstr(); assert (ccollectdir != NULL);
-#else
-  char *ccollectdir = collectdir.getcstr(); assert (ccollectdir != NULL);
-#endif
-
-  if (load_database(ccollection, ccollectdir, idxsuffix, txtsuffix))
-    {
-      setsearchmode (queryparams);
-      submitquery (queryparams);
-      getresults (queryresults);
-    }
-  else databaseloaded = false;
-
-  // free up the c strings
-  delete ccollection;
-  delete idxsuffix;
-  delete txtsuffix;
-  delete ccollectdir;
-
-  return databaseloaded;
+  if (makeindexcurrent (queryparams.search_index, queryparams.collection)) {
+    setsearchmode (queryparams);
+    submitquery (queryparams);
+    getresults (queryresults);
+    return true;
+  }
+
+  return false;
 }
 
@@ -280 +342 @@
 {
   mgq_ask(".set expert true");
+  mgq_ask(".set sorted_terms true");
   mgq_ask(".set accumulator_method list");
   mgq_ask(".set max_accumulators 50000");
@@ -353 +416 @@
   mgq_results(result_termfreqs, 0, MAXNUMTERMS, 
           termfreqcallback, (void *)(&queryresults));
+  queryresults.sortuniqqueryterms();
+
+  // get term variants 
   mgq_results(result_terms, 0, MAXNUMTERMS,
-              termscallback, (void *)(&queryresults));
-  queryresults.sortqueryterms();
-  queryresults.uniqqueryterms();
+              termvariantscallback, (void *)(&queryresults));
 }
 

trunk/gsdl/src/colservr/mgsearch.h

@@ -26 +26 @@
   mgsearchclass ();
   virtual ~mgsearchclass ();
-  
+
   // the index directory must be set before any searching
   // is done
   void setcollectdir (const text_t &thecollectdir);
+
+  // you only need to use this function before doing any stemming
+  // casefolding and stemming will be set if values for them are
+  // provided (0 or 1).
+  // makeindexcurrent returns true if it was able to load the database
+  bool makeindexcurrent (const text_t &index, const text_t &collection,
+             int casefolding = -1, int stemming = -1);
+
+  // stem word uses the values set in the last call to makeindexcurrent
+  // to stem the word. It is assumed that word is in unicode
+  text_t stemword (const text_t &word);
   
   // the search results are returned in queryresults

trunk/gsdl/src/colservr/queryfilter.cpp

@@ -12 +12 @@
 /*
    $Log$
+   Revision 1.7  1999/06/30 04:04:13  rjmcnab
+   made stemming functions available from mgsearch and made the stems
+   for the query terms available in queryinfo
+
    Revision 1.6  1999/06/29 22:06:23  rjmcnab
    Added a couple of fields to queryinfo to handle a special version
@@ -347 +351 @@
   // assemble the term results
   if ((request.filterResultOptions & FRtermFreq) || (request.filterResultOptions & FRmatchTerms)) {
-    queryresults.sortqueryterms();
-    queryresults.uniqqueryterms();
+    // note: the terms have already been sorted and uniqued
 
     TermInfo_t terminfo;

trunk/gsdl/src/colservr/queryinfo.cpp

@@ -12 +12 @@
 /*
    $Log$
+   Revision 1.4  1999/06/30 04:04:13  rjmcnab
+   made stemming functions available from mgsearch and made the stems
+   for the query terms available in queryinfo
+
    Revision 1.3  1999/06/29 22:06:23  rjmcnab
    Added a couple of fields to queryinfo to handle a special version
@@ -88 +92 @@
 {
   termstr = t.termstr;
+  termstemstr = t.termstemstr;
   termfreq = t.termfreq;
 
@@ -96 +101 @@
 {
   return ((x.termstr == y.termstr) &&
+      (x.termstemstr == y.termstemstr) &&
       (x.termfreq == y.termfreq));
 }
@@ -108 +114 @@
 {
   return ((x.termfreq < y.termfreq) ||
-      ((x.termfreq == y.termfreq) && (x.termstr < y.termstr)));
-
+      ((x.termfreq == y.termfreq) && (x.termstemstr < y.termstemstr)) ||
+      ((x.termfreq == y.termfreq) && (x.termstemstr == y.termstemstr) && (x.termstr < y.termstr)));
 }
 
@@ -115 +121 @@
 {
   return ((x.termfreq > y.termfreq) ||
-      ((x.termfreq == y.termfreq) && (x.termstr > y.termstr)));
-
+      ((x.termfreq == y.termfreq) && (x.termstemstr > y.termstemstr)) ||
+      ((x.termfreq == y.termfreq) && (x.termstemstr == y.termstemstr) && (x.termstr > y.termstr)));
 }
 
@@ -125 +131 @@
 
   outs << text_t2ascii << " t:\"" << t.termstr << "\"";
+  outs << text_t2ascii << " s:\"" << t.termstemstr << "\"";
   outs << " f:" << t.termfreq << "\n";
 
@@ -145 +152 @@
 // query results
 
-void queryresultsclass::clear ()
-{
+void queryresultsclass::clear () {
   docs_matched_set = false;;
   docs_matched = 0;
@@ -152 +158 @@
   
   docs.erase(docs.begin(),docs.end());
+  orgterms.erase(orgterms.begin(),orgterms.end());
   terms.erase(terms.begin(),terms.end());
 }
@@ -168 +175 @@
 }
 
-void queryresultsclass::sortqueryterms()
-{
+void queryresultsclass::sortuniqqueryterms() {
+  terms = orgterms;
+
+  // sort the terms
   sort (terms.begin(), terms.end());
-}
-
-void queryresultsclass::uniqqueryterms()
-{
+
+  // and then unique them
   vector<termfreqclass>::iterator new_end = unique (terms.begin(), terms.end());
   terms.erase(new_end, terms.end());
 }
-
 
 
@@ -192 +198 @@
     outs << (*docshere);
     docshere++;
+  }
+
+  outs << "orgterms\n";
+  vector<termfreqclass>::iterator orgtermshere = q.orgterms.begin();
+  vector<termfreqclass>::iterator orgtermsend = q.orgterms.end();
+  while (orgtermshere != orgtermsend) {
+    outs << (*orgtermshere);
+    orgtermshere++;
   }
 

trunk/gsdl/src/colservr/queryinfo.h

@@ -63 +63 @@
 public:
   text_t termstr;
+  text_t termstemstr;
   unsigned int termfreq;
   
@@ -110 +111 @@
   
   vector<docresultclass> docs;
+  vector<termfreqclass> orgterms; // terms before they are sorted and uniqued
   vector<termfreqclass> terms;
   text_tarray termvariants;
@@ -122 +124 @@
   int getnumterms () {return terms.size();}
   
-  void sortqueryterms();
-  void uniqqueryterms();
+  void sortuniqqueryterms();
 };