1 | /**************************************************************************
|
---|
2 | *
|
---|
3 | * mgpass.cpp -- Driver for the various passes -
|
---|
4 |
|
---|
5 | V1 - removed all the pipe processing and replaced with
|
---|
6 | code to directly explore a directory of files.
|
---|
7 | V2 - rebuilt to extract non text files from web browser
|
---|
8 | 'catch' file. Also to display progress count
|
---|
9 | GH/WJR
|
---|
10 |
|
---|
11 | * Copyright (C) 1994 Neil Sharman, ..
|
---|
12 | *
|
---|
13 | * This program is free software; you can redistribute it and/or modify
|
---|
14 | * it under the terms of the GNU General Public License as published by
|
---|
15 | * the Free Software Foundation; either version 2 of the License, or
|
---|
16 | * (at your option) any later version.
|
---|
17 | *
|
---|
18 | * This program is distributed in the hope that it will be useful,
|
---|
19 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
|
---|
20 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
---|
21 | * GNU General Public License for more details.
|
---|
22 | *
|
---|
23 | * You should have received a copy of the GNU General Public License
|
---|
24 | * along with this program; if not, write to the Free Software
|
---|
25 | * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
|
---|
26 | *
|
---|
27 | * $Id: MGPASS.C 13 1998-11-17 09:36:00Z rjmcnab $
|
---|
28 | *
|
---|
29 | **************************************************************************/
|
---|
30 |
|
---|
31 | #include "sysfuncs.h"
|
---|
32 |
|
---|
33 | #include <malloc.h>
|
---|
34 | #include <stdlib.h>
|
---|
35 | #include <stdio.h>
|
---|
36 | #include <io.h>
|
---|
37 | #include <fcntl.h>
|
---|
38 | #include <string.h>
|
---|
39 |
|
---|
40 | #include "memlib.h"
|
---|
41 | #include "messages.h"
|
---|
42 |
|
---|
43 | #include "mg_files.h"
|
---|
44 | #include "mg.h"
|
---|
45 | #include "build.h"
|
---|
46 | #include "text.h"
|
---|
47 | #include "stemmer.h"
|
---|
48 |
|
---|
49 | /*
|
---|
50 | $Log$
|
---|
51 | Revision 1.1 1998/11/17 09:34:12 rjmcnab
|
---|
52 | *** empty log message ***
|
---|
53 |
|
---|
54 | * Revision 1.3 1994/10/20 03:56:57 tes
|
---|
55 | * I have rewritten the boolean query optimiser and abstracted out the
|
---|
56 | * components of the boolean query.
|
---|
57 | *
|
---|
58 | * Revision 1.2 1994/09/20 04:41:52 tes
|
---|
59 | * For version 1.1
|
---|
60 | *
|
---|
61 | */
|
---|
62 |
|
---|
63 | static char *RCSID = "$Id: MGPASS.C 13 1998-11-17 09:36:00Z rjmcnab $";
|
---|
64 |
|
---|
65 | #define MAX_PASSES 5
|
---|
66 |
|
---|
67 | #define SPECIAL 1
|
---|
68 | #define TEXT_PASS_1 2
|
---|
69 | #define TEXT_PASS_2 4
|
---|
70 | #define IVF_PASS_1 8
|
---|
71 | #define IVF_PASS_2 16
|
---|
72 |
|
---|
73 | #define MIN_BUF 8192
|
---|
74 | #define path_length 256
|
---|
75 |
|
---|
76 | unsigned long buf_size = 3 * 1024 * 1024; /* 3Mb */
|
---|
77 | unsigned long invf_buffer_size = 5 * 1024 * 1024; /* 5Mb */
|
---|
78 | unsigned long ChunkLimit = 0;
|
---|
79 | char InvfLevel = 2;
|
---|
80 | char SkipSGML = 0;
|
---|
81 | char MakeWeights = 0;
|
---|
82 | FILE *Comp_Stats = NULL;
|
---|
83 | int comp_stat_point = 0;
|
---|
84 | u_long bytes_processed = 0;
|
---|
85 | int num_docs = 0;
|
---|
86 | u_long bytes_received = 0;
|
---|
87 | int stem_method = 0;
|
---|
88 |
|
---|
89 | static char Passes = 0;
|
---|
90 | static unsigned long trace = 0;
|
---|
91 | static int Dump = 0;
|
---|
92 | static char **files = NULL;
|
---|
93 | static int num_files = 0;
|
---|
94 | static char *trace_name = NULL;
|
---|
95 |
|
---|
96 | static char dirname[path_length], wildname[path_length];
|
---|
97 | static int by_para = 0, recurse = 0, html_catch = 0;
|
---|
98 | static char *buffer;
|
---|
99 | char *line_start, *data_end, *base, *scan;
|
---|
100 |
|
---|
101 | typedef struct pass_data
|
---|
102 | {
|
---|
103 | char *name;
|
---|
104 | int (*init) (char *);
|
---|
105 | int (*process) (u_char *, int);
|
---|
106 | int (*done) (char *);
|
---|
107 | }
|
---|
108 | pass_data;
|
---|
109 |
|
---|
110 | static pass_data PassData[MAX_PASSES] =
|
---|
111 | {
|
---|
112 | {"special", init_special, process_special, done_special},
|
---|
113 | {"text.pass1", init_text_1, process_text_1, done_text_1},
|
---|
114 | {"text.pass2", init_text_2, process_text_2, done_text_2},
|
---|
115 | {"ivf.pass1", init_ivf_1, process_ivf_1, done_ivf_1},
|
---|
116 | {"ivf.pass2", init_ivf_2, process_ivf_2, done_ivf_2},
|
---|
117 | };
|
---|
118 |
|
---|
119 | static char *usage_str = "\nUSAGE:\n"
|
---|
120 | " %s [-h] [-G] [-D] [-1|-2|-3] [-T1] [-T2] [-I1] [-I2] [-N1]\n"
|
---|
121 | " %*s [-N2] [-W] [-S] [-b buffer-size] [-d dictionary-directory]\n"
|
---|
122 | " %*s [-t trace-point Mb] [-m invf-memory] [-c chunk-limit]\n"
|
---|
123 | " %*s [-n trace-name] [-C comp-stat-size] [-s stem_method] -f doc-collection-name\n"
|
---|
124 | " %*s [source directory\\] [source file]\n";
|
---|
125 |
|
---|
126 | static void usage (char *err)
|
---|
127 | {
|
---|
128 | if (err) Message (err);
|
---|
129 | fprintf (stderr, usage_str, msg_prefix, strlen (msg_prefix), "",
|
---|
130 | strlen (msg_prefix), "", strlen (msg_prefix), "");
|
---|
131 | exit (1);
|
---|
132 | }
|
---|
133 |
|
---|
134 | void do_process(char *buffer, int num)
|
---|
135 | {
|
---|
136 | int pass;
|
---|
137 | if (num == 0) Message ("Warning : Processing zero length document");
|
---|
138 | num_docs++;
|
---|
139 | bytes_processed += num;
|
---|
140 | for (pass = 0; pass < MAX_PASSES; pass++)
|
---|
141 | if (Passes & (1 << pass))
|
---|
142 | if (PassData[pass].process((u_char *)buffer, num) == COMPERROR)
|
---|
143 | FatalError(1, "Error during file processing");
|
---|
144 | }
|
---|
145 |
|
---|
146 | int refill(int in_fd)
|
---|
147 | {
|
---|
148 | int num, bitleft;
|
---|
149 | bitleft = data_end - base;
|
---|
150 | memmove(buffer, base, bitleft);
|
---|
151 | if (buf_size - bitleft < MIN_BUF)
|
---|
152 | FatalError(1, "Paragraph too big for buffer");
|
---|
153 | num = read(in_fd, &buffer[bitleft], buf_size - bitleft);
|
---|
154 | line_start -= (base - buffer);
|
---|
155 | scan -= (base - buffer);
|
---|
156 | base = buffer;
|
---|
157 | data_end = buffer + bitleft + num;
|
---|
158 | if (num > 0) return 1; else return 0;
|
---|
159 | }
|
---|
160 |
|
---|
161 | char *scanpara(int in_fd)
|
---|
162 | {
|
---|
163 | int num, blank, in_blank, at_end;
|
---|
164 | num = read(in_fd,buffer,buf_size);
|
---|
165 | in_blank = 1; base = buffer; line_start = buffer; scan = buffer;
|
---|
166 | data_end = buffer + num; at_end = 0;
|
---|
167 | for (;;) {
|
---|
168 | blank = 1; line_start = scan; // Get a line
|
---|
169 | while (scan < data_end && *scan != '\n')
|
---|
170 | if (*scan++ > ' ') blank = 0;
|
---|
171 | if (scan >= data_end) {
|
---|
172 | if (refill(in_fd)) {
|
---|
173 | while (scan < data_end && *scan != '\n')
|
---|
174 | if (*scan++ > ' ') blank = 0;
|
---|
175 | if (scan < data_end) scan++;
|
---|
176 | }
|
---|
177 | else at_end = 1;
|
---|
178 | }
|
---|
179 | else scan++;
|
---|
180 | if (line_start < scan) { // If we have a line
|
---|
181 | if (in_blank) {
|
---|
182 | if (!blank) { in_blank = 0; base = line_start; }
|
---|
183 | }
|
---|
184 | else {
|
---|
185 | if (blank) {
|
---|
186 | do_process(base, line_start - base);
|
---|
187 | in_blank = 1; base = line_start;
|
---|
188 | }
|
---|
189 | }
|
---|
190 | }
|
---|
191 | if (at_end) break;
|
---|
192 | }
|
---|
193 | if (in_blank) base = scan;
|
---|
194 | if (scan + 2 <= buffer + buf_size) {
|
---|
195 | *scan++ = 26; *scan++ = 10;
|
---|
196 | }
|
---|
197 | if (base < scan) do_process(base, scan - base);
|
---|
198 | return NULL;
|
---|
199 | }
|
---|
200 |
|
---|
201 | char *scanfile(int in_fd)
|
---|
202 | {
|
---|
203 | int num = read(in_fd,buffer,buf_size); /*expect to read the whole file*/
|
---|
204 | if (num < 0) return "file locked";
|
---|
205 | if (num >= buf_size-1) return "file too long";
|
---|
206 | do_process(buffer, num);
|
---|
207 | return NULL;
|
---|
208 | }
|
---|
209 |
|
---|
210 | void search(char *dname, char *fname)
|
---|
211 | {
|
---|
212 | long dirtag; struct _finddata_t dirinfo; int in_fd; char *res;
|
---|
213 | char search_name[path_length], found_name[path_length];
|
---|
214 |
|
---|
215 | sprintf(search_name, "%s%s", dname, fname); /*Scan files*/
|
---|
216 | dirtag = _findfirst(search_name, &dirinfo);
|
---|
217 | if (dirtag >= 0) {
|
---|
218 | do {
|
---|
219 | if ((dirinfo.attrib & (_A_SUBDIR | _A_HIDDEN | _A_SYSTEM)) == 0) {
|
---|
220 | sprintf(found_name,"%s%s",dname,dirinfo.name);
|
---|
221 | in_fd = open(found_name,O_RDONLY|O_BINARY);
|
---|
222 | if (in_fd >= 0) {
|
---|
223 | if (by_para)
|
---|
224 | res = scanpara(in_fd);
|
---|
225 | else
|
---|
226 | res = scanfile(in_fd);
|
---|
227 | if (res != NULL) {
|
---|
228 | Message("Error %s in processing file %s\n", res, found_name);
|
---|
229 | exit(1);
|
---|
230 | }
|
---|
231 | close(in_fd);
|
---|
232 | }
|
---|
233 | }
|
---|
234 | } while (_findnext(dirtag, &dirinfo) == 0);
|
---|
235 | _findclose(dirtag);
|
---|
236 | }
|
---|
237 |
|
---|
238 | if (recurse == 0) return;
|
---|
239 |
|
---|
240 | sprintf(search_name, "%s*.*", dname); /*Look for subdirs*/
|
---|
241 | dirtag = _findfirst(search_name, &dirinfo);
|
---|
242 | if (dirtag < 0) return;
|
---|
243 | do {
|
---|
244 | if ( ((dirinfo.attrib & (_A_HIDDEN | _A_SYSTEM)) == 0) &&
|
---|
245 | ((dirinfo.attrib & _A_SUBDIR) != 0) &&
|
---|
246 | strcmp(dirinfo.name,".") != 0 &&
|
---|
247 | strcmp(dirinfo.name,"..") != 0) {
|
---|
248 | sprintf(found_name,"%s%s",dname,dirinfo.name);
|
---|
249 | strcat(found_name,"\\");
|
---|
250 | search(found_name,fname);
|
---|
251 | }
|
---|
252 | } while (_findnext(dirtag, &dirinfo) == 0);
|
---|
253 | _findclose(dirtag);
|
---|
254 | }
|
---|
255 |
|
---|
256 | static int toobig(int n)
|
---|
257 | {
|
---|
258 | if (n > path_length) {
|
---|
259 | printf("Cannot handle urls > %d characters in length\n", path_length);
|
---|
260 | exit(1);
|
---|
261 | }
|
---|
262 | return 0;
|
---|
263 | }
|
---|
264 |
|
---|
265 | void scan_catch(char *dname, char *fname)
|
---|
266 | {
|
---|
267 | int in_fd, urllen, conlen, filesize; char catch_name[path_length];
|
---|
268 | int filecount = 0;
|
---|
269 | enum { filekind_redirected, filekind_text, filekind_other } filekind;
|
---|
270 | sprintf(catch_name, "%s%s", dname, fname);
|
---|
271 | in_fd = open(catch_name,O_RDONLY|O_BINARY);
|
---|
272 | if (in_fd < 0)
|
---|
273 | FatalError(1, "Couldn't open catch file \"%s\"", catch_name);
|
---|
274 | for (;;) {
|
---|
275 | filecount++; if (filecount%100 == 0) { printf("%d\r", filecount); fflush(stdout); }
|
---|
276 | if (read(in_fd, &urllen, sizeof(int)) != sizeof(int) || toobig(urllen) ||
|
---|
277 | read(in_fd, catch_name, urllen) != urllen ||
|
---|
278 | read(in_fd, &conlen, sizeof(int)) != sizeof(int) || toobig(conlen) ||
|
---|
279 | read(in_fd, catch_name, conlen) != conlen) break;
|
---|
280 | if (conlen >= 1 && *catch_name == '@')
|
---|
281 | filekind = filekind_redirected;
|
---|
282 | else {
|
---|
283 | if (conlen >= 4 && strncmp(catch_name, "text", 4) == 0)
|
---|
284 | filekind = filekind_text;
|
---|
285 | else
|
---|
286 | filekind = filekind_other;
|
---|
287 | if (read(in_fd, &filesize, sizeof(int)) != sizeof(int))
|
---|
288 | FatalError(1, "File read failed for size field");
|
---|
289 | if (filesize > buf_size)
|
---|
290 | FatalError(1, "File too large (%d > %d)", filesize, buf_size);
|
---|
291 | if (read(in_fd, buffer, filesize) != filesize)
|
---|
292 | FatalError(1, "Failed to read file data");
|
---|
293 | }
|
---|
294 | if (filekind == filekind_text) do_process(buffer, filesize);
|
---|
295 | }
|
---|
296 | close(in_fd);
|
---|
297 | }
|
---|
298 |
|
---|
299 | static void driver (FILE * Trace, char *file_name)
|
---|
300 | {
|
---|
301 | int pass;
|
---|
302 |
|
---|
303 | buffer = (char *)Xmalloc (buf_size);
|
---|
304 |
|
---|
305 | for (pass = 0; pass < MAX_PASSES; pass++)
|
---|
306 | if (Passes & (1 << pass))
|
---|
307 | {
|
---|
308 | if (PassData[pass].init (file_name) == COMPERROR)
|
---|
309 | FatalError (1, "Error during init of \"%s\"",PassData[pass].name);
|
---|
310 | }
|
---|
311 |
|
---|
312 | if (html_catch == 0) search(dirname, wildname);
|
---|
313 | else scan_catch(dirname, wildname);
|
---|
314 |
|
---|
315 | for (pass = 0; pass < MAX_PASSES; pass++)
|
---|
316 | if (Passes & (1 << pass))
|
---|
317 | {
|
---|
318 | if (PassData[pass].done (file_name) == COMPERROR)
|
---|
319 | FatalError (1, "Error during done of \"%s\"", PassData[pass].name);
|
---|
320 | }
|
---|
321 |
|
---|
322 |
|
---|
323 | free (buffer);
|
---|
324 | }
|
---|
325 |
|
---|
326 | void main (int argc, char **argv)
|
---|
327 | {
|
---|
328 | int ch;
|
---|
329 | char *filename = NULL;
|
---|
330 | FILE *Trace = NULL;
|
---|
331 |
|
---|
332 | msg_prefix = argv[0];
|
---|
333 |
|
---|
334 | opterr = 0;
|
---|
335 | while ((ch = getopt (argc, argv, "hC:WHGpSD123f:d:b:T:I:t:m:N:c:n:s:")) != -1)
|
---|
336 | {
|
---|
337 | switch (ch)
|
---|
338 | {
|
---|
339 | case 'H':
|
---|
340 | html_catch = 1;
|
---|
341 | break;
|
---|
342 | case 'G':
|
---|
343 | SkipSGML = 1;
|
---|
344 | break;
|
---|
345 | case 'p':
|
---|
346 | by_para = 1;
|
---|
347 | break;
|
---|
348 | case 'S':
|
---|
349 | Passes |= SPECIAL;
|
---|
350 | break;
|
---|
351 | case '1':
|
---|
352 | InvfLevel = 1;
|
---|
353 | break;
|
---|
354 | case '2':
|
---|
355 | InvfLevel = 2;
|
---|
356 | break;
|
---|
357 | case '3':
|
---|
358 | InvfLevel = 3;
|
---|
359 | break;
|
---|
360 | case 'f':
|
---|
361 | filename = optarg;
|
---|
362 | break;
|
---|
363 | case 'n':
|
---|
364 | trace_name = optarg;
|
---|
365 | break;
|
---|
366 | case 'D':
|
---|
367 | Dump = 1;
|
---|
368 | break;
|
---|
369 | case 'W':
|
---|
370 | MakeWeights = 1;
|
---|
371 | break;
|
---|
372 | case 'd':
|
---|
373 | set_basepath (optarg);
|
---|
374 | break;
|
---|
375 | case 's':
|
---|
376 | stem_method = atoi (optarg) & STEMMER_MASK;
|
---|
377 | break;
|
---|
378 | case 'b':
|
---|
379 | buf_size = atoi (optarg) * 1024;
|
---|
380 | break;
|
---|
381 | case 'C':
|
---|
382 | comp_stat_point = atoi (optarg) * 1024;
|
---|
383 | break;
|
---|
384 | case 'c':
|
---|
385 | ChunkLimit = atoi (optarg);
|
---|
386 | break;
|
---|
387 | case 'm':
|
---|
388 | invf_buffer_size = (int) (atof (optarg) * 1024 * 1024);
|
---|
389 | break;
|
---|
390 | case 'I':
|
---|
391 | case 'N': /* N kept for compatability */
|
---|
392 | if (*optarg == '1')
|
---|
393 | Passes |= IVF_PASS_1;
|
---|
394 | else if (*optarg == '2')
|
---|
395 | Passes |= IVF_PASS_2;
|
---|
396 | else
|
---|
397 | usage ("Invalid pass number");
|
---|
398 | break;
|
---|
399 | case 'T':
|
---|
400 | if (*optarg == '1')
|
---|
401 | Passes |= TEXT_PASS_1;
|
---|
402 | else if (*optarg == '2')
|
---|
403 | Passes |= TEXT_PASS_2;
|
---|
404 | else
|
---|
405 | usage ("Invalid pass number");
|
---|
406 | break;
|
---|
407 | case 't':
|
---|
408 | trace = (unsigned long) (atof (optarg) * 1024 * 1024);
|
---|
409 | break;
|
---|
410 | case 'h':
|
---|
411 | case '?':
|
---|
412 | usage (NULL);
|
---|
413 | }
|
---|
414 | }
|
---|
415 |
|
---|
416 | if (!filename || *filename == '\0')
|
---|
417 | FatalError (1, "A document collection name must be specified.");
|
---|
418 |
|
---|
419 | if (buf_size < MIN_BUF)
|
---|
420 | FatalError (1, "The buffer size must exceed 1024 bytes.");
|
---|
421 |
|
---|
422 | if ((Passes & (IVF_PASS_1 | IVF_PASS_2)) == (IVF_PASS_1 | IVF_PASS_2))
|
---|
423 | FatalError (1, "I1 and I2 cannot be done simultaneously.");
|
---|
424 |
|
---|
425 | if ((Passes & (TEXT_PASS_1 | TEXT_PASS_2)) == (TEXT_PASS_1 | TEXT_PASS_2))
|
---|
426 | FatalError (1, "T1 and T2 cannot be done simultaneously.");
|
---|
427 |
|
---|
428 | if (!Passes)
|
---|
429 | FatalError (1, "S, T1, T2, I1 or I2 must be specified.");
|
---|
430 |
|
---|
431 | if (argc - optind == 1) {
|
---|
432 | strcpy(dirname,"");
|
---|
433 | strcpy(wildname,argv[optind]);
|
---|
434 | }
|
---|
435 | else if (argc - optind == 2) {
|
---|
436 | strcpy(dirname,argv[optind]);
|
---|
437 | strcpy(wildname,argv[optind+1]);
|
---|
438 | }
|
---|
439 | else FatalError(1, "Finder code requires directory and filespec.");
|
---|
440 |
|
---|
441 | if (strrchr(wildname,'*') != NULL || strrchr(wildname,'?') != NULL)
|
---|
442 | recurse = 1;
|
---|
443 |
|
---|
444 | if (trace)
|
---|
445 | {
|
---|
446 | if (!trace_name)
|
---|
447 | trace_name = make_name (filename, TRACE_SUFFIX, NULL);
|
---|
448 | if (!(Trace = fopen (trace_name, "a")))
|
---|
449 | Message ("Unable to open \"%s\". No tracing will be done.", trace_name);
|
---|
450 | else
|
---|
451 | setbuf (Trace, NULL);
|
---|
452 | }
|
---|
453 | else
|
---|
454 | Trace = NULL;
|
---|
455 |
|
---|
456 | if (comp_stat_point)
|
---|
457 | {
|
---|
458 | char *name = make_name (filename, COMPRESSION_STATS_SUFFIX, NULL);
|
---|
459 | if (!(Comp_Stats = fopen (name, "wb")))
|
---|
460 | Message ("Unable to open \"%s\". No comp. stats. will be generated.",
|
---|
461 | name);
|
---|
462 | }
|
---|
463 |
|
---|
464 |
|
---|
465 | if (Trace)
|
---|
466 | {
|
---|
467 | int i;
|
---|
468 | fprintf (Trace, "\n\n\t\t-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\n\n");
|
---|
469 | for (i = 0; i < argc; i++)
|
---|
470 | fprintf (Trace, "%s ", argv[i]);
|
---|
471 | fprintf (Trace, "\n\n");
|
---|
472 | }
|
---|
473 |
|
---|
474 | driver (Trace, filename);
|
---|
475 |
|
---|
476 | if (Trace)
|
---|
477 | fclose (Trace);
|
---|
478 |
|
---|
479 | if (Comp_Stats)
|
---|
480 | fclose (Comp_Stats);
|
---|
481 |
|
---|
482 | exit (0);
|
---|
483 | }
|
---|
484 | |
---|