Changeset 13477 for trunk/indexers

Timestamp:

2006-12-11T11:22:20+13:00 (17 years ago)

Author:

shaoqun

Message:

added code for accentfolding

Location:

trunk/indexers/mgpp/text

Files:

• IndexData.cpp (modified) (3 diffs)
• IndexData.h (modified) (2 diffs)
• Queryer.cpp (modified) (6 diffs)
• Terms.cpp (modified) (6 diffs)
• mg_files.cpp (modified) (1 diff)
• mg_files.h (modified) (3 diffs)
• mgpp_stem_idx.cpp (modified) (3 diffs)
• stemmer.cpp (modified) (4 diffs)
• stemmer.h (modified) (4 diffs)

trunk/indexers/mgpp/text/IndexData.cpp

@@ -30 +30 @@
   dictFile = NULL;
 
-  stem1File = NULL;
-  stem2File = NULL;
-  stem3File = NULL;
+  /* [JFG - Mar 06: Accent folding patch] */
+  for(int i=STEM_MIN;i <= STEM_MAX;i++) 
+    stemFile[i-1] = NULL;
 
   invfFile = NULL;
@@ -79 +79 @@
   if (!ReadBlockIdx (dictFile, biTags)) { UnloadData (); return false; }
 
-  // blocked stem index 1
-  stem1File = open_file (filename, INVF_DICT_BLOCKED_1_SUFFIX,
-             "rb", MAGIC_STEM_1, MG_ABORT);
-  if (!sih1.Read (stem1File)) { UnloadData (); return false; }
-
-  fseek (stem1File, sih1.block_idx_start, SEEK_SET);
-  if (!ReadBlockIdx (stem1File, sii1)) { UnloadData (); return false; }
-  
-  // blocked stem index 2
-  stem2File = open_file (filename, INVF_DICT_BLOCKED_2_SUFFIX,
-             "rb", MAGIC_STEM_2, MG_ABORT);
-  if (!sih2.Read (stem2File)) { UnloadData (); return false; }
-
-  fseek (stem2File, sih2.block_idx_start, SEEK_SET);
-  if (!ReadBlockIdx (stem2File, sii2)) { UnloadData (); return false; }
-  
-  // blocked stem index 3
-  stem3File = open_file (filename, INVF_DICT_BLOCKED_3_SUFFIX,
-             "rb", MAGIC_STEM_3, MG_ABORT);
-  if (!sih3.Read (stem3File)) { UnloadData (); return false; }
-
-  fseek (stem3File, sih3.block_idx_start, SEEK_SET);
-  if (!ReadBlockIdx (stem3File, sii3)) { UnloadData (); return false; }
-
+  /* [JFG - Mar 06: Accent folding patch] */
+  // read stem indexes
+  // [KJD - optional stemming patch]
+  // allow no stem indexes
+  for(int stem = STEM_MIN; stem <= STEM_MAX; stem++) { 
+    char *suffix = make_suffix (INVF_DICT_BLOCKED_SUFFIX_PAT, stem, NULL);
+    stemFile[stem-1] = open_file (filename, suffix,
+                  "rb", MAGIC_STEM_GEN(stem + '0'), MG_MESSAGE);
+    if (stemFile[stem-1]!= NULL) {
+      if (!sih[stem-1].Read (stemFile[stem-1])) { 
+    fclose (stemFile[stem-1]);
+    stemFile[stem-1] = NULL;
+    //UnloadData (); return false; 
+      }
+      
+      fseek (stemFile[stem-1], sih[stem-1].block_idx_start, SEEK_SET);
+      if (!ReadBlockIdx (stemFile[stem-1], sii[stem-1])) { 
+    fclose (stemFile[stem-1]);
+    stemFile[stem-1] = NULL;
+    //UnloadData (); return false; 
+      }
+    }
+  }
+  
+  
   // inverted file
   invfFile = open_file (filename, INVF_SUFFIX, "rb",
@@ -125 +126 @@
   }
 
-  if (stem1File != NULL) {
-    fclose (stem1File); stem1File = NULL;
-  }
-  if (stem2File != NULL) {
-    fclose (stem2File); stem2File = NULL;
-  }
-  if (stem3File != NULL) {
-    fclose (stem3File); stem3File = NULL;
+  for(int i=STEM_MIN;i <= STEM_MAX;i++) {
+    if (stemFile[i-1] != NULL) {
+      fclose (stemFile[i-1]); stemFile[i-1] = NULL;
+    }
   }
 

trunk/indexers/mgpp/text/IndexData.h

@@ -28 +28 @@
 #include "FragLevelConvert.h"
 #include "Weights.h"
-
+#include "stemmer.h" 
 
 class IndexData {
@@ -43 +43 @@
   block_idx biTags;
 
+  /* [JFG - Mar 06: Accent folding patch] */
   // stem indexes
-  FILE *stem1File;
-  FILE *stem2File;
-  FILE *stem3File;
-  stem_idx_header sih1;
-  stem_idx_header sih2;
-  stem_idx_header sih3;
-  block_idx sii1;
-  block_idx sii2;
-  block_idx sii3;
-
+  FILE *stemFile[STEM_MAX]; 
+  stem_idx_header sih[STEM_MAX];
+  block_idx sii[STEM_MAX];
+ 
   // inverted file
   FILE *invfFile;

trunk/indexers/mgpp/text/Queryer.cpp

@@ -50 +50 @@
        << "\t.c0/.c1\t\tcasefolding off/on\n"
        << "\t.s0/.s1\t\tstemming off/on\n"
+#ifdef ENABLE_ACCENTFOLD
+       << "\t.a0/.a1\t\taccentfolding off/on\n"
+#endif
        << "\t.o0/.o1\t\tshort output off/on\n"
        << "\t.m\t\tset maxnumeric (enter the number at the prompt)\n\n"
@@ -91 +94 @@
   
   // init the text system
-    TextData textData;
+  TextData textData;
   if (!textData.LoadData (basePath, textfilename)) {
     FatalError (1, "Couldn't load text information for \"%s\"", textfilename);
@@ -134 +137 @@
   //SetCStr(level, "");
    
-  int defaultStemMethod = 0; // uncasefolded, unstemmed
+  int defaultStemMethod = 0; // uncasefolded, unstemmed, unaccentfolded
   int defaultBoolCombine = 0; // OR
   bool shortOutput = false;
@@ -207 +210 @@
       }
       else if (queryArray[1] == 'c') { // casefolding - on/off
-    if (queryArray[2] == '1') defaultStemMethod |= 1;
-    else if (queryArray[2] == '0') defaultStemMethod &= 0xe;
+    if (queryArray[2] == '1') defaultStemMethod |= STEM_CaseFolding;
+    else if (queryArray[2] == '0') defaultStemMethod &= (~STEM_CaseFolding);
     else {
       cout << "Error: please enter .c0 (case sensitive) or .c1 (casefolded)\n";
@@ -214 +217 @@
       }
       else if (queryArray[1] == 's') { // stemming - on/off
-    if (queryArray[2] == '1') defaultStemMethod |=2;
-    else if (queryArray[2] == '0') defaultStemMethod &=0xd;
+    if (queryArray[2] == '1') defaultStemMethod |= STEM_Stemming;
+    else if (queryArray[2] == '0') defaultStemMethod &= (~STEM_Stemming);
     else {
       cout << "Error: please enter .s0 (unstemmed) or .s1 (stemmed)\n";
     }
       }      
+#ifdef ENABLE_ACCENTFOLD
+      else if (queryArray[1] == 'a') { // accentfolding - on/off
+    if (queryArray[2] == '1') defaultStemMethod |= STEM_AccentFolding;
+    else if (queryArray[2] == '0') defaultStemMethod &= (~STEM_AccentFolding);
+    else {
+      cout << "Error: please enter .a0 (accent sensitive) or .a1 (accentfolded)\n";
+    }
+      }
+#endif
       else if (queryArray[1] == 'o') { // output - short/long
     if (queryArray[2] == '1') shortOutput = true;
@@ -274 +286 @@
 
   // clean up, everybody clean up
-    textData.UnloadData ();
+  textData.UnloadData ();
   indexData.UnloadData ();
 

trunk/indexers/mgpp/text/Terms.cpp

@@ -210 +210 @@
               vector<unsigned long> &equivWords) {
   equivWords.erase (equivWords.begin(), equivWords.end());
-  
-  if (stemMethod == 0 || stemMethod==4 || stemMethod==5) {
+
+  // if the stem method specified is not a valid one (i.e. there was no appropriate stem index, then we set it to 0)
+  // unless we have partial matching, in which case we are not doing stem indexes anyway.
+  if (!(stemMethod & STEM_PARTIAL_MATCH) && indexData.stemFile[stemMethod-1] == NULL) {
+    cerr << "Stem index for method "<<stemMethod<< " was not built, so not doing stemming\n";
+    stemMethod = 0;
+  }
+  /* [JFG - Mar 06: Accent folding patch] */
+  /* use flag PARTIAL_MATCH */  
+  if (stemMethod == 0 || (stemMethod & STEM_PARTIAL_MATCH)) {
     // don't need to stem the word,
     // find the word number(s) for this term
@@ -218 +226 @@
     word_block_dict_el wordDictEl;
     wordDictEl.SetNumLevels (numLevels);
-    if (stemMethod ==0) {
+    if (stemMethod == 0) {
       if (SearchWordBlockDictEl (indexData.dictFile, indexData.biWords,
                  indexData.bdh.entries_per_wblk,
@@ -228 +236 @@
     } else {
       // partial matching,
-      PartialMatchSearchWordBlockDictEl (indexData.dictFile, indexData.biWords, indexData.bdh.entries_per_wblk, indexData.bdh.word_dict_size, numLevels, term, wordDictEl, equivWords, (stemMethod==5?true:false) );
+      PartialMatchSearchWordBlockDictEl (indexData.dictFile, indexData.biWords, indexData.bdh.entries_per_wblk, indexData.bdh.word_dict_size, numLevels, term, wordDictEl, equivWords, (stemMethod & STEM_CaseFolding)? true : false);
+      // TODO: Accent Folding is not handled here!!
       return;
     }
@@ -234 +243 @@
               
   // need to stem this word and find it in the blocked stem index
-  
-  unsigned char  mgWord[MAXSTEMLEN + 1];
+  unsigned char mgWord[MAXSTEMLEN + 1];
   UCArray stemTerm;
   unsigned long stemmerNum = 0;
-  if (stemMethod == 1) stemmerNum = indexData.sih1.stemmer_num;
-  else if (stemMethod == 2) stemmerNum = indexData.sih2.stemmer_num;
-  else if (stemMethod == 3) stemmerNum = indexData.sih3.stemmer_num;
-   
+
+  /* [JFG - Mar 06: Accent folding patch] */
+  if(stemMethod > STEM_MAX) {
+    return;
+    //TODO: throw an error here
+  }
+
+  stemmerNum = indexData.sih[stemMethod-1].stemmer_num;
+  
   // convert the word to an "mg word"
   mgWord[0] = term.size();
@@ -247 +260 @@
   
   // stem the word
-  stemmer (stemMethod, stemmerNum, mgWord);
-
+  mgpp_stemmer (stemMethod, stemmerNum, mgWord);
   // convert the result back to a UCArray
   stemTerm.insert (stemTerm.end(), &mgWord[1], &mgWord[1] + mgWord[0]);
@@ -256 +268 @@
   unsigned long stemElNum;
   bool result = false;
-  if (stemMethod == 1) {
-    result = SearchStemBlockDictEl (indexData.stem1File,
-               indexData.sii1,
-               indexData.sih1.entries_per_block,
-               indexData.sih1.dict_size,
+  
+  /* [JFG - Mar 06: Accent folding patch] */
+  result = SearchStemBlockDictEl (indexData.stemFile[stemMethod-1],
+               indexData.sii[stemMethod-1],
+               indexData.sih[stemMethod-1].entries_per_block,
+               indexData.sih[stemMethod-1].dict_size,
                stemTerm,
                stemDictEl,
                stemElNum);
-
-  } else if (stemMethod == 2) {
-    result = SearchStemBlockDictEl (indexData.stem2File,
-               indexData.sii2,
-               indexData.sih2.entries_per_block,
-               indexData.sih2.dict_size,
-               stemTerm,
-               stemDictEl,
-               stemElNum);
-
-  } else if (stemMethod == 3) {
-    result = SearchStemBlockDictEl (indexData.stem3File,
-               indexData.sii3,
-               indexData.sih3.entries_per_block,
-               indexData.sih3.dict_size,
-               stemTerm,
-               stemDictEl,
-               stemElNum);
-  }
-
+  
   if (result) {
     equivWords = stemDictEl.equivWords;  

trunk/indexers/mgpp/text/mg_files.cpp

@@ -93 +93 @@
 }
 
+
+/* [JFG - Mar 06: Accent folding patch] */
+/* This generates a suffixe for a file name. It places the name in the
+   buffer specified or if that is NULL, it uses a static buffer.
+   Please do not specify buffers under 512 or the data to be written. */
+char *
+make_suffix (const char *suffix_format, const char suffix_arg, char *buffer)
+{
+  static char suffix[512]; 
+  if (!buffer)
+    buffer = suffix;
+  sprintf (buffer, suffix_format, suffix_arg);
+  return buffer;
+}
+ 
 
 

trunk/indexers/mgpp/text/mg_files.h

@@ -50 +50 @@
 #define MAGIC_WGHT_APPROX   GEN_MAGIC('M','G','w', 0 )
 #define MAGIC_PARAGRAPH     GEN_MAGIC('M','G','P', 0 )
-/* [RPAP - Jan 97: Stem Index Change] */
-#define MAGIC_STEM_1            GEN_MAGIC('M','G','s','1')
-#define MAGIC_STEM_2            GEN_MAGIC('M','G','s','2')
-#define MAGIC_STEM_3            GEN_MAGIC('M','G','s','3')
-
+#define MAGIC_STEM_GEN(x)   GEN_MAGIC('M', 'G', 's', x)
 #define IS_MAGIC(a) ((((u_long)(a)) & 0xffff0000) == MAGIC_XXXX)
 
@@ -219 +215 @@
 
 /* [RPAP - Jan 97: Stem Index Change] */
+/* [JFG - Mar 06: Accent folding patch] 
+ * Use the pattern with make_suffix */
 /* The casefolded index into the stemmed dictionary */
 #ifdef SHORT_SUFFIX
-# define INVF_DICT_BLOCKED_1_SUFFIX ".ib1"
-#else
-# define INVF_DICT_BLOCKED_1_SUFFIX ".invf.dict.blocked.1"
-#endif
-
-/* [RPAP - Jan 97: Stem Index Change] */
-/* The stemmed index into the stemmed dictionary */
-#ifdef SHORT_SUFFIX
-# define INVF_DICT_BLOCKED_2_SUFFIX ".ib2"
-#else
-# define INVF_DICT_BLOCKED_2_SUFFIX ".invf.dict.blocked.2"
-#endif
-
-/* [RPAP - Jan 97: Stem Index Change] */
-/* The casefolded and stemmed  index into the stemmed dictionary */
-#ifdef SHORT_SUFFIX
-# define INVF_DICT_BLOCKED_3_SUFFIX ".ib3"
-#else
-# define INVF_DICT_BLOCKED_3_SUFFIX ".invf.dict.blocked.3"
-#endif
+# define INVF_DICT_BLOCKED_SUFFIX_PAT ".ib%d"
+#else
+# define INVF_DICT_BLOCKED_SUFFIX_PAT ".invf.dict.blocked.%d"
+#endif
+
 
 /* [RPAP - Feb 97: WIN32 Port] */
@@ -264 +247 @@
 char *make_name (const char *name, const char *suffix, char *buffer);
 
-
+/* [JFG - Mar 06: Accent folding patch] */
+/* This generates the suffix of a file. It places the name in the buffer
+   specified or if that is NULL it uses a static buffer. */
+char *make_suffix (const char *suffix_format, const char suffix_arg, char *buffer);
 
 

trunk/indexers/mgpp/text/mgpp_stem_idx.cpp

@@ -80 +80 @@
 
     // stem the word
-    stemmer (stemMethod, stemmerNum, mgWord);
+    mgpp_stemmer (stemMethod, stemmerNum, mgWord);
 
     // convert the result back to a UCArray
@@ -101 +101 @@
             int stemmerNum,
             unsigned long entriesPerBlock) {
+  
+  /* [JFG - Mar 06: Accent folding patch] */
   // Create appropriate stem index file
   FILE *stemDictFile = NULL;
-  if (stemMethod == 1) {
-    stemDictFile = create_file (filename, INVF_DICT_BLOCKED_1_SUFFIX,
-                "wb", MAGIC_STEM_1, MG_ABORT);
-  } else if (stemMethod == 2) {
-    stemDictFile = create_file (filename, INVF_DICT_BLOCKED_2_SUFFIX,
-                "wb", MAGIC_STEM_2, MG_ABORT);
-  } else if (stemMethod == 3) {
-    stemDictFile = create_file (filename, INVF_DICT_BLOCKED_3_SUFFIX,
-                "wb", MAGIC_STEM_3, MG_ABORT);
-  } else {
+  if (stemMethod >= STEM_MIN && stemMethod <= STEM_MAX) {
+    char *suffix = make_suffix (INVF_DICT_BLOCKED_SUFFIX_PAT, stemMethod, NULL);
+    stemDictFile = create_file (filename, suffix,
+                "wb", MAGIC_STEM_GEN(stemMethod + '0'), MG_ABORT);  
+  }
+  else {
     FatalError (1, "Unknown stem method %d", stemMethod);
   }
@@ -213 +211 @@
       break;
     case 'a':
-      stemmerNum = stemmernumber ((unsigned char *) optarg);
+      stemmerNum = mgpp_stemmernumber ((unsigned char *) optarg);
       break;
     case 'h':
     case '?':
       fprintf (stderr, "usage: %s [-d directory] "
-           "[-b entries-per-block] [-h] -s 1|2|3 "
-           "[-a stemmer-method] -f name\n", argv[0]);
+           "[-b entries-per-block] [-h] -s 1|2|3", argv[0]);
+#ifdef ENABLE_ACCENTFOLD
+      fprintf (stderr, "|4|5|6|7");
+#endif
+      fprintf (stderr, " [-a stemmer-method] -f name\n");
       exit (1);
     }
   }
   
-  if (stemMethod < 1 || stemMethod > 3)
-    FatalError (1, "Stem method must be 1, 2 or 3");
-
+  /* [JFG - Mar 06: Accent folding patch] */
+  if (stemMethod < STEM_MIN || stemMethod > STEM_MAX) 
+    FatalError (1, "Stem method must be between %d and %d", STEM_MIN, STEM_MAX);
+#ifndef ENABLE_ACCENTFOLD
+  if (stemMethod & STEM_AccentFolding) {
+    // accent folding not enabled
+    return -1;
+  }
+#endif
   // read in the dictionary and create the in memory dictionary
   StemMapDict stemDict;

trunk/indexers/mgpp/text/stemmer.cpp

@@ -22 +22 @@
 #include "sysfuncs.h"
 #include "stemmer.h"
-
 #include "lovinstem.h"
 #include "simplefrenchstem.h"
 #include "unitool.h"
+
+#ifdef ENABLE_ACCENTFOLD
+/* [JFG - Mar 06: Accent folding patch] */
+#include "unac.h"
+#endif
 
 #define LOVINSTEMMER        0
@@ -34 +38 @@
  * making sure the final length doesn't exceed the original
  * length */
-static void unicode_casefold (u_char *word) {
+static void mgpp_unicode_casefold (u_char *word) {
   unsigned short out[256]; /* temp space */
   int i;
@@ -52 +56 @@
 }
 
+#ifdef ENABLE_ACCENTFOLD
+/* [JFG - Mar 06: Accent folding patch] */
+/* =========================================================================
+ * Function: unicode_accentfold
+ 
+ * Description: remove accents from characters
+ * Input: a word string with the length in the first byte
+ * Output: the unaccented word
+ * ========================================================================= */
+void mgpp_unicode_accentfold (unsigned char *word) {      
+  size_t unac_size = 0;
+  char *unac = NULL;
 
-int stemmernumber (u_char *stemmerdescription) {
+
+  unac_string("utf-8", (char*)word+1, word[0], &unac, &unac_size);
+  strncpy((char*)word+1, unac, word[0]+1);
+  word[0] = unac_size;
+  
+  free(unac);
+  return;
+}
+#endif
+      
+int mgpp_stemmernumber (u_char *stemmerdescription) {
   u_char descript[MAX_STEM_DESCRIPTION_LEN];
   int i;
@@ -85 +111 @@
  * Method 2 - Stem.
  * Method 3 - Case fold and stem.
- *
+ * Method 4 - Accent fold
+ * Method 5 - Accent fold and case fold
+ * Method 6 - Accent fold and stem
+ * Method 7 - Accent fold, stem and case fold
+
  * The stemmer number should be obtained using
  * the stemmernumber function above.
  */
 void 
-stemmer (int method, int stemmer, u_char *word) {
-  if (method & 1) {
-    unicode_casefold (word);
+mgpp_stemmer (int method, int stemmer, u_char *word) {
+  if (method & STEM_CaseFolding) {
+    mgpp_unicode_casefold (word);
   }
 
-  if (method & 2) {
+#ifdef ENABLE_ACCENTFOLD
+  if (method & STEM_AccentFolding) {
+    mgpp_unicode_accentfold (word);
+  }
+#endif
+
+  if (method & STEM_Stemming) {
     switch (stemmer) {
     case LOVINSTEMMER: lovinstem (word);

trunk/indexers/mgpp/text/stemmer.h

@@ -23 +23 @@
 #define STEMMER_H
 
-#include "sysfuncs.h"
+/* [RPAP - Jan 97: Stem Index Change] */
+/* [JFG - Mar 06: Accent folding patch] */
+enum stemMethods {
+    STEM_None       =   0,
+    STEM_CaseFolding    =   0x1,
+    STEM_Stemming       =   0x2,
+    STEM_AccentFolding  =   0x4,
+};
+
+/* This is for the QueryParser */
+#define CHAR_FLAG_STEM_CaseFold    'i'  // ignore case
+#define CHAR_FLAG_STEM_NoCaseFold  'c'  // case sensitive
+#define CHAR_FLAG_STEM_Stemming    's'  // stem words
+#define CHAR_FLAG_STEM_NoStemming  'u'  // do not stem words
+#define CHAR_FLAG_STEM_AccentFold  'f'  // accent fold
+#define CHAR_FLAG_STEM_NoAccentFold  'a'    // do no accent folding
+#define CHAR_FLAG_STEM_Validator  "icsufa"  // all of the above
+
+
+#define STEM_MIN 1
+#define STEM_MAX (STEM_CaseFolding | STEM_Stemming | STEM_AccentFolding)
+#define STEM_PARTIAL_MATCH  (STEM_MAX+1)
+#define STEM_INVALID        (STEM_MAX+2)
 
 #define STEMMER_MASK 3
@@ -47 +69 @@
  * stemmer description.
  */
-int stemmernumber (u_char *stemmerdescription);
+int mgpp_stemmernumber (unsigned char *stemmerdescription);
 
 /*
@@ -54 +76 @@
  * Method 2 - Stem.
  * Method 3 - Case fold and stem.
- *
+ * Method 4 - Accent fold
+ * Method 5 - Case fold and accent fold
+ * Method 6 - Stem and accent fold
+ * Method 7 - Case fold, stem and accent fold
  * The stemmer number should be obtained using function
  * stemmernumber above.
@@ -61 +86 @@
 extern "C" 
 #endif
-void stemmer (int method, int stemmer, u_char * word);
+void mgpp_stemmer (int method, int stemmer, unsigned char * word);
 
 #endif