Changeset 12879

Timestamp:

2006-09-28T10:23:36+12:00 (18 years ago)

Author:

kjdon

Message:

Accent Folding patch thanks to Juan Grigera. mgpp_stemmer now does accent folding too if needed. stemmernumber and stemmer functions renamed to mgpp_stemmer and mgpp_stemmernumber so that they don't clash with mg versions when linked in to the library

Location:

trunk/gsdl/src/mgpp/text

Files:

• stemmer.cpp (modified) (4 diffs)
• stemmer.h (modified) (3 diffs)

trunk/gsdl/src/mgpp/text/stemmer.cpp

@@ -22 +22 @@
 #include "sysfuncs.h"
 #include "stemmer.h"
-
+#include "mg_files.h"
 #include "lovinstem.h"
 #include "simplefrenchstem.h"
 #include "unitool.h"
+
+/* [JFG - Mar 06: Accent folding patch] */
+#include "unac.h"
 
 #define LOVINSTEMMER        0
@@ -34 +37 @@
  * making sure the final length doesn't exceed the original
  * length */
-static void unicode_casefold (u_char *word) {
+static void mgpp_unicode_casefold (u_char *word) {
   unsigned short out[256]; /* temp space */
   int i;
@@ -52 +55 @@
 }
 
+/* [JFG - Mar 06: Accent folding patch] */
+/* =========================================================================
+ * Function: unicode_accentfold
+ 
+ * Description: remove accents from characters
+ * Input: a word string with the length in the first byte
+ * Output: the unaccented word
+ * ========================================================================= */
+void mgpp_unicode_accentfold (unsigned char *word) {      
+  size_t unac_size = 0;
+  char *unac = NULL;
 
-int stemmernumber (u_char *stemmerdescription) {
+
+  unac_string("utf-8", (char*)word+1, word[0], &unac, &unac_size);
+  strncpy((char*)word+1, unac, word[0]+1);
+  word[0] = unac_size;
+  
+  free(unac);
+  return;
+}
+
+      
+int mgpp_stemmernumber (u_char *stemmerdescription) {
   u_char descript[MAX_STEM_DESCRIPTION_LEN];
   int i;
@@ -85 +109 @@
  * Method 2 - Stem.
  * Method 3 - Case fold and stem.
- *
+ * Method 4 - Accent fold
+ * Method 5 - Accent fold and case fold
+ * Method 6 - Accent fold and stem
+ * Method 7 - Accent fold, stem and case fold
+
  * The stemmer number should be obtained using
  * the stemmernumber function above.
  */
 void 
-stemmer (int method, int stemmer, u_char *word) {
-  if (method & 1) {
-    unicode_casefold (word);
+mgpp_stemmer (int method, int stemmer, u_char *word) {
+  if (method & STEM_CaseFolding) {
+    mgpp_unicode_casefold (word);
   }
 
-  if (method & 2) {
+  if (method & STEM_AccentFolding) {
+    mgpp_unicode_accentfold (word);
+  }
+
+  if (method & STEM_Stemming) {
     switch (stemmer) {
     case LOVINSTEMMER: lovinstem (word);

trunk/gsdl/src/mgpp/text/stemmer.h

@@ -47 +47 @@
  * stemmer description.
  */
-int stemmernumber (u_char *stemmerdescription);
+int mgpp_stemmernumber (u_char *stemmerdescription);
 
 /*
@@ -54 +54 @@
  * Method 2 - Stem.
  * Method 3 - Case fold and stem.
- *
+ * Method 4 - Accent fold
+ * Method 5 - Case fold and accent fold
+ * Method 6 - Stem and accent fold
+ * Method 7 - Case fold, stem and accent fold
  * The stemmer number should be obtained using function
  * stemmernumber above.
@@ -61 +64 @@
 extern "C" 
 #endif
-void stemmer (int method, int stemmer, u_char * word);
+void mgpp_stemmer (int method, int stemmer, u_char * word);
 
 #endif