Changeset 13477 for trunk/mgpp/text/Terms.cpp

Timestamp:

2006-12-11T11:22:20+13:00 (17 years ago)

Author:

shaoqun

Message:

added code for accentfolding

File:

• trunk/mgpp/text/Terms.cpp (modified) (6 diffs)

trunk/mgpp/text/Terms.cpp

@@ -210 +210 @@
               vector<unsigned long> &equivWords) {
   equivWords.erase (equivWords.begin(), equivWords.end());
-  
-  if (stemMethod == 0 || stemMethod==4 || stemMethod==5) {
+
+  // if the stem method specified is not a valid one (i.e. there was no appropriate stem index, then we set it to 0)
+  // unless we have partial matching, in which case we are not doing stem indexes anyway.
+  if (!(stemMethod & STEM_PARTIAL_MATCH) && indexData.stemFile[stemMethod-1] == NULL) {
+    cerr << "Stem index for method "<<stemMethod<< " was not built, so not doing stemming\n";
+    stemMethod = 0;
+  }
+  /* [JFG - Mar 06: Accent folding patch] */
+  /* use flag PARTIAL_MATCH */  
+  if (stemMethod == 0 || (stemMethod & STEM_PARTIAL_MATCH)) {
     // don't need to stem the word,
     // find the word number(s) for this term
@@ -218 +226 @@
     word_block_dict_el wordDictEl;
     wordDictEl.SetNumLevels (numLevels);
-    if (stemMethod ==0) {
+    if (stemMethod == 0) {
       if (SearchWordBlockDictEl (indexData.dictFile, indexData.biWords,
                  indexData.bdh.entries_per_wblk,
@@ -228 +236 @@
     } else {
       // partial matching,
-      PartialMatchSearchWordBlockDictEl (indexData.dictFile, indexData.biWords, indexData.bdh.entries_per_wblk, indexData.bdh.word_dict_size, numLevels, term, wordDictEl, equivWords, (stemMethod==5?true:false) );
+      PartialMatchSearchWordBlockDictEl (indexData.dictFile, indexData.biWords, indexData.bdh.entries_per_wblk, indexData.bdh.word_dict_size, numLevels, term, wordDictEl, equivWords, (stemMethod & STEM_CaseFolding)? true : false);
+      // TODO: Accent Folding is not handled here!!
       return;
     }
@@ -234 +243 @@
               
   // need to stem this word and find it in the blocked stem index
-  
-  unsigned char  mgWord[MAXSTEMLEN + 1];
+  unsigned char mgWord[MAXSTEMLEN + 1];
   UCArray stemTerm;
   unsigned long stemmerNum = 0;
-  if (stemMethod == 1) stemmerNum = indexData.sih1.stemmer_num;
-  else if (stemMethod == 2) stemmerNum = indexData.sih2.stemmer_num;
-  else if (stemMethod == 3) stemmerNum = indexData.sih3.stemmer_num;
-   
+
+  /* [JFG - Mar 06: Accent folding patch] */
+  if(stemMethod > STEM_MAX) {
+    return;
+    //TODO: throw an error here
+  }
+
+  stemmerNum = indexData.sih[stemMethod-1].stemmer_num;
+  
   // convert the word to an "mg word"
   mgWord[0] = term.size();
@@ -247 +260 @@
   
   // stem the word
-  stemmer (stemMethod, stemmerNum, mgWord);
-
+  mgpp_stemmer (stemMethod, stemmerNum, mgWord);
   // convert the result back to a UCArray
   stemTerm.insert (stemTerm.end(), &mgWord[1], &mgWord[1] + mgWord[0]);
@@ -256 +268 @@
   unsigned long stemElNum;
   bool result = false;
-  if (stemMethod == 1) {
-    result = SearchStemBlockDictEl (indexData.stem1File,
-               indexData.sii1,
-               indexData.sih1.entries_per_block,
-               indexData.sih1.dict_size,
+  
+  /* [JFG - Mar 06: Accent folding patch] */
+  result = SearchStemBlockDictEl (indexData.stemFile[stemMethod-1],
+               indexData.sii[stemMethod-1],
+               indexData.sih[stemMethod-1].entries_per_block,
+               indexData.sih[stemMethod-1].dict_size,
                stemTerm,
                stemDictEl,
                stemElNum);
-
-  } else if (stemMethod == 2) {
-    result = SearchStemBlockDictEl (indexData.stem2File,
-               indexData.sii2,
-               indexData.sih2.entries_per_block,
-               indexData.sih2.dict_size,
-               stemTerm,
-               stemDictEl,
-               stemElNum);
-
-  } else if (stemMethod == 3) {
-    result = SearchStemBlockDictEl (indexData.stem3File,
-               indexData.sii3,
-               indexData.sih3.entries_per_block,
-               indexData.sih3.dict_size,
-               stemTerm,
-               stemDictEl,
-               stemElNum);
-  }
-
+  
   if (result) {
     equivWords = stemDictEl.equivWords;