Changeset 13477 for trunk/indexers/mgpp/text/stemmer.cpp

Timestamp:

2006-12-11T11:22:20+13:00 (17 years ago)

Author:

shaoqun

Message:

added code for accentfolding

File:

• trunk/indexers/mgpp/text/stemmer.cpp (modified) (4 diffs)

trunk/indexers/mgpp/text/stemmer.cpp

@@ -22 +22 @@
 #include "sysfuncs.h"
 #include "stemmer.h"
-
 #include "lovinstem.h"
 #include "simplefrenchstem.h"
 #include "unitool.h"
+
+#ifdef ENABLE_ACCENTFOLD
+/* [JFG - Mar 06: Accent folding patch] */
+#include "unac.h"
+#endif
 
 #define LOVINSTEMMER        0
@@ -34 +38 @@
  * making sure the final length doesn't exceed the original
  * length */
-static void unicode_casefold (u_char *word) {
+static void mgpp_unicode_casefold (u_char *word) {
   unsigned short out[256]; /* temp space */
   int i;
@@ -52 +56 @@
 }
 
+#ifdef ENABLE_ACCENTFOLD
+/* [JFG - Mar 06: Accent folding patch] */
+/* =========================================================================
+ * Function: unicode_accentfold
+ 
+ * Description: remove accents from characters
+ * Input: a word string with the length in the first byte
+ * Output: the unaccented word
+ * ========================================================================= */
+void mgpp_unicode_accentfold (unsigned char *word) {      
+  size_t unac_size = 0;
+  char *unac = NULL;
 
-int stemmernumber (u_char *stemmerdescription) {
+
+  unac_string("utf-8", (char*)word+1, word[0], &unac, &unac_size);
+  strncpy((char*)word+1, unac, word[0]+1);
+  word[0] = unac_size;
+  
+  free(unac);
+  return;
+}
+#endif
+      
+int mgpp_stemmernumber (u_char *stemmerdescription) {
   u_char descript[MAX_STEM_DESCRIPTION_LEN];
   int i;
@@ -85 +111 @@
  * Method 2 - Stem.
  * Method 3 - Case fold and stem.
- *
+ * Method 4 - Accent fold
+ * Method 5 - Accent fold and case fold
+ * Method 6 - Accent fold and stem
+ * Method 7 - Accent fold, stem and case fold
+
  * The stemmer number should be obtained using
  * the stemmernumber function above.
  */
 void 
-stemmer (int method, int stemmer, u_char *word) {
-  if (method & 1) {
-    unicode_casefold (word);
+mgpp_stemmer (int method, int stemmer, u_char *word) {
+  if (method & STEM_CaseFolding) {
+    mgpp_unicode_casefold (word);
   }
 
-  if (method & 2) {
+#ifdef ENABLE_ACCENTFOLD
+  if (method & STEM_AccentFolding) {
+    mgpp_unicode_accentfold (word);
+  }
+#endif
+
+  if (method & STEM_Stemming) {
     switch (stemmer) {
     case LOVINSTEMMER: lovinstem (word);