source: trunk/gsdl/src/mgpp/text/mg_stem_idx.cpp@ 856

Last change on this file since 856 was 856, checked in by sjboddie, 22 years ago

Rodgers new C++ mg

  • Property svn:keywords set to Author Date Id Revision
File size: 6.5 KB
Line 
1/**************************************************************************
2 *
3 * mg_stem_idx.cpp -- stem index builder
4 * Copyright (C) 1999 Rodger McNab
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
19 *
20 **************************************************************************/
21
22
23#include "sysfuncs.h"
24#include "messages.h"
25
26#include "mg_files.h"
27#include "invf.h"
28#include "UCArray.h"
29#include "words.h"
30
31#include "stemmer.h"
32
33
34#if defined(GSDL_USE_OBJECTSPACE)
35# include <ospace\std\map>
36# include <ospace\std\vector>
37#elif defined(GSDL_USE_STL_H)
38# include <map.h>
39# include <vector.h>
40#else
41# include <map>
42# include <vector>
43#endif
44
45
46/*
47 $Log$
48 Revision 1.1 2000/01/14 02:26:20 sjboddie
49 Rodgers new C++ mg
50
51 */
52
53
54typedef vector<unsigned long> WordNumList;
55typedef map<UCArray, WordNumList, DictLTUCArray> StemMapDict;
56
57
58void CreateStemDict (char *filename,
59 StemMapDict &stemDict,
60 int stemMethod,
61 int stemmerNum) {
62 stemDict.erase (stemDict.begin(), stemDict.end());
63
64 // open the dictionary
65 FILE *dictFile = open_file (filename, INVF_DICT_SUFFIX, "rb",
66 MAGIC_STEM_BUILD, MG_ABORT);
67 invf_dict_header idh;
68 idh.Read (dictFile);
69
70 fseek (dictFile, idh.word_dict_start, SEEK_SET);
71
72 unsigned long wordNum;
73 u_char mgWord[MAXSTEMLEN + 1];
74 word_dict_el wordEl;
75 UCArray stemEl;
76 wordEl.SetNumLevels (idh.num_levels);
77 for (wordNum=0; wordNum<idh.word_dict_size; wordNum++) {
78 // read in the next word
79 wordEl.Read (dictFile, idh.num_levels);
80
81 // convert the word to an "mg word"
82 mgWord[0] = wordEl.el.size();
83 bcopy ((char *)wordEl.el.begin(), (char *)&mgWord[1], wordEl.el.size());
84
85 // stem the word
86 stemmer (stemMethod, stemmerNum, mgWord);
87
88 // convert the result back to a UCArray
89 stemEl.erase (stemEl.begin(), stemEl.end());
90 stemEl.insert (stemEl.end(), &mgWord[1], &mgWord[1] + mgWord[0]);
91
92// cout << "\"" << stemEl << "\" -> \"" << wordEl.el << "\"\n";
93
94 // add this word number to the list of word numbers for this word
95 stemDict[stemEl].push_back (wordNum);
96 }
97
98 fclose (dictFile);
99}
100
101
102void WriteStemDict (char *filename,
103 StemMapDict &stemDict,
104 int stemMethod,
105 int stemmerNum,
106 unsigned long entriesPerBlock) {
107 // Create appropriate stem index file
108 FILE *stemDictFile;
109 if (stemMethod == 1) {
110 stemDictFile = create_file (filename, INVF_DICT_BLOCKED_1_SUFFIX,
111 "wb", MAGIC_STEM_1, MG_ABORT);
112 } else if (stemMethod == 2) {
113 stemDictFile = create_file (filename, INVF_DICT_BLOCKED_2_SUFFIX,
114 "wb", MAGIC_STEM_2, MG_ABORT);
115 } else if (stemMethod == 3) {
116 stemDictFile = create_file (filename, INVF_DICT_BLOCKED_3_SUFFIX,
117 "wb", MAGIC_STEM_3, MG_ABORT);
118 } else {
119 FatalError (1, "Unknown stem method %d", stemMethod);
120 }
121
122 stem_idx_header sih;
123 sih.lookback = 0;
124 sih.dict_size = stemDict.size();
125 sih.entries_per_block = entriesPerBlock;
126 sih.max_block_size = 0;
127
128 sih.stemmer_num = stemmerNum;
129 sih.stem_method = stemMethod;
130
131 // write out a place-holder version of the header
132 sih.Write (stemDictFile);
133
134 sih.blocks_start = ftell (stemDictFile);
135
136 block_idx stemIdx;
137 unsigned long stemNum = 0;
138 stem_block_dict_el stemEl;
139 UCArray lastEl;
140
141 StemMapDict::const_iterator here = stemDict.begin();
142 StemMapDict::const_iterator end = stemDict.end();
143 while (here != end) {
144 // remember this stem (and position) if this is the start
145 // of a new block
146 if (stemNum % entriesPerBlock == 0) {
147 block_idx_info elIdx;
148 elIdx.el = (*here).first;
149 elIdx.block_ptr = ftell (stemDictFile);
150
151 // see if this block is the longest so far
152 if (stemIdx.size() > 0) {
153 unsigned long blockLen = elIdx.block_ptr -
154 (*(stemIdx.end()-1)).block_ptr;
155 if (blockLen > sih.max_block_size) sih.max_block_size = blockLen;
156 }
157
158 stemIdx.push_back (elIdx);
159 lastEl.erase (lastEl.begin(), lastEl.end()); // output full word
160 }
161
162 // copy the information for this stem
163 stemEl.el = (*here).first;
164 stemEl.equivWords = (*here).second;
165
166 // write out the stem
167 stemEl.Write (stemDictFile, &lastEl);
168
169 here++; stemNum++;
170 }
171
172
173 // write out the element indexes
174 sih.num_blocks = stemIdx.size();
175 sih.block_idx_start = ftell (stemDictFile);
176 WriteBlockIdx (stemDictFile, stemIdx);
177
178 // write out the stem dictionary header
179 fseek (stemDictFile, sizeof(unsigned long), SEEK_SET);
180 sih.Write (stemDictFile);
181
182
183 // close open files
184 fclose (stemDictFile);
185
186 // print out information
187 Message ("Num word stems = %d\n", sih.dict_size);
188 Message ("Max stem block size = %d\n", sih.max_block_size);
189 Message ("Number of stem blocks written = %d\n", sih.num_blocks);
190}
191
192
193int main (int argc, char **argv) {
194 unsigned long entriesPerBlock = 16;
195 char *filename = "";
196 int ch;
197 int stemMethod = 0; // illegal value (no translation)
198 int stemmerNum = 0; // English stemmer
199 msg_prefix = argv[0];
200 opterr = 0;
201
202 while ((ch = getopt (argc, argv, "f:d:b:s:h")) != -1) {
203 switch (ch) {
204 case 'f': // input file
205 filename = optarg;
206 break;
207 case 'd':
208 set_basepath (optarg);
209 break;
210 case 'b':
211 entriesPerBlock = atoi (optarg);
212 break;
213 case 's':
214 stemMethod = atoi (optarg);
215 break;
216 case 'a':
217 stemmerNum = stemmernumber ((unsigned char *) optarg);
218 break;
219 case 'h':
220 case '?':
221 fprintf (stderr, "usage: %s [-d directory] "
222 "[-b entries-per-block] [-h] -s 1|2|3 "
223 "[-a stemmer-method] -f name\n", argv[0]);
224 exit (1);
225 }
226 }
227
228 if (stemMethod < 1 || stemMethod > 3)
229 FatalError (1, "Stem method must be 1, 2 or 3");
230
231 // read in the dictionary and create the in memory dictionary
232 StemMapDict stemDict;
233 CreateStemDict (filename, stemDict, stemMethod, stemmerNum);
234
235 // write out the dictionary as a blocked file
236 WriteStemDict (filename, stemDict, stemMethod, stemmerNum, entriesPerBlock);
237
238 return 0;
239}
Note: See TracBrowser for help on using the repository browser.