Context Navigation

← Previous Revision
Latest Revision
Next Revision →
Blame
Revision Log

source: trunk/gsdl/packages/mg/src/text/mg_fast_comp_dict.c@ 1014

Last change on this file since 1014 was 439, checked in by sjboddie, 25 years ago
renamed mg-1.3d directory mg
Property svn:executable set to ``* Property svn:keywords set to `Author Date Id Revision`
File size: 20.2 KB

Line
1	/**************************************************************************
2	*
3	* mg_fast_comp_dict.c -- Program to generate a fast compression dictionary
4	* Copyright (C) 1994 Neil Sharman
5	*
6	* This program is free software; you can redistribute it and/or modify
7	* it under the terms of the GNU General Public License as published by
8	* the Free Software Foundation; either version 2 of the License, or
9	* (at your option) any later version.
10	*
11	* This program is distributed in the hope that it will be useful,
12	* but WITHOUT ANY WARRANTY; without even the implied warranty of
13	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14	* GNU General Public License for more details.
15	*
16	* You should have received a copy of the GNU General Public License
17	* along with this program; if not, write to the Free Software
18	* Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
19	*
20	* $Id: mg_fast_comp_dict.c 439 1999-08-10 21:23:37Z sjboddie $
21	*
22	**************************************************************************/
23
24
25	/*
26	$Log$
27	Revision 1.1 1999/08/10 21:18:06 sjboddie
28	renamed mg-1.3d directory mg
29
30	Revision 1.2 1998/11/25 07:55:44 rjmcnab
31
32	Modified mg to that you can specify the stemmer you want
33	to use via a command line option. You specify it to
34	mg_passes during the build process. The number of the
35	stemmer that you used is stored within the inverted
36	dictionary header and the stemmed dictionary header so
37	the correct stemmer is used in later stages of building
38	and querying.
39
40	Revision 1.1 1998/11/17 09:34:57 rjmcnab
41	* empty log message *
42
43	* Revision 1.3 1994/10/20 03:56:55 tes
44	* I have rewritten the boolean query optimiser and abstracted out the
45	* components of the boolean query.
46	*
47	* Revision 1.2 1994/09/20 04:41:47 tes
48	* For version 1.1
49	*
50	*/
51
52	static char *RCSID = "$Id: mg_fast_comp_dict.c 439 1999-08-10 21:23:37Z sjboddie $";
53
54	#include "sysfuncs.h"
55
56	#include "filestats.h"
57	#include "huffman.h"
58	#include "timing.h"
59	#include "messages.h"
60	#include "memlib.h"
61	#include "netorder.h" /* [RPAP - Jan 97: Endian Ordering] */
62
63	#include "local_strings.h"
64	#include "mg.h"
65	#include "text.h"
66	#include "invf.h"
67	#include "lists.h"
68	#include "backend.h"
69	#include "mg_files.h"
70	#include "locallib.h"
71	#include "words.h"
72
73
74
75	#define ALIGN_SIZE(p,t) (((p) + (sizeof(t)-1)) & (~(sizeof(t)-1)))
76
77	#define WORDNO(p, base) ((((char)(p))-((char)(base)))/sizeof(u_char*))
78	#define FIXUP(p) (fixup[WORDNO(p,buffer)/8] \|= (1<<(WORDNO(p, buffer) & 7)))
79
80	#define IS_FIXUP(p) ((fixup[WORDNO(p,buffer)/8] & (1<<(WORDNO(p, buffer) & 7))) != 0)
81
82	#define FIXUP_VALS(vals) do { \
83	int i; \
84	for (i=0; i < MAX_HUFFCODE_LEN+1; i++) \
85	FIXUP(&vals[i]); \
86	} while(0)
87
88
89
90	static u_long mem_for_comp_dict (char *filename);
91	static void load_comp_dict (char *filename);
92	static void save_fast_dict (char *filename);
93	static void unfixup_buffer (void);
94
95
96	static void *buffer;
97	static void *cur;
98	static u_char *fixup;
99	static u_long mem, fixup_mem;
100
101	int main (int argc, char **argv)
102	{
103	ProgTime StartTime;
104	int ch;
105	char *filename = "";
106	opterr = 0;
107	msg_prefix = argv[0];
108	while ((ch = getopt (argc, argv, "f:d:h")) != -1)
109	switch (ch)
110	{
111	case 'f': /* input file */
112	filename = optarg;
113	break;
114	case 'd':
115	set_basepath (optarg);
116	break;
117	case 'h':
118	case '?':
119	fprintf (stderr, "usage: %s [-f input_file] [-d data directory] [-h]\n",
120	argv[0]);
121	exit (1);
122	}
123	GetTime (&StartTime);
124
125	mem = mem_for_comp_dict (filename);
126
127	fixup_mem = (ALIGN_SIZE (mem, u_char ) / sizeof (u_char ) + 7) / 8;
128
129	cur = buffer = Xmalloc (mem);
130	bzero (buffer, mem);
131	fixup = Xmalloc (fixup_mem);
132	bzero (fixup, fixup_mem);
133
134	Message ("Estimated memory for fast_dict %u", mem);
135	Message ("Estimated memory for fixups %u", fixup_mem);
136
137	load_comp_dict (filename);
138
139	Message ("Actual memory for fast_dict %u", (char ) cur - (char ) buffer);
140
141	if ((u_long) cur > (u_long) buffer + mem)
142	FatalError (1, "The buffer was not big enough for the dictionary");
143
144	{
145	/* [RPAP - Jan 97: Endian Ordering] */
146	compression_dict cd = (compression_dict ) buffer;
147	int i, which;
148
149	/* cfh */
150	for (which = 0; which <= 1; which++)
151	{
152	int j;
153
154	HTONSI(cd->cfh[which]->hd.num_codes);
155	HTONSI(cd->cfh[which]->hd.mincodelen);
156	HTONSI(cd->cfh[which]->hd.maxcodelen);
157	for (j = 0; j < MAX_HUFFCODE_LEN + 1; j++)
158	{
159	HTONSI(cd->cfh[which]->hd.lencount[j]);
160	HTONUL(cd->cfh[which]->hd.min_code[j]);
161	}
162	HTONUL(cd->cfh[which]->uncompressed_size);
163	for (j = 0; j < MAX_HUFFCODE_LEN + 1; j++)
164	HTONUL(cd->cfh[which]->huff_words_size[j]);
165	}
166	HTONUL(cd->MemForCompDict);
167	/* ad */
168	if (cd->cdh.novel_method == MG_NOVEL_BINARY \|\|
169	cd->cdh.novel_method == MG_NOVEL_DELTA \|\|
170	cd->cdh.novel_method == MG_NOVEL_HYBRID \|\|
171	cd->cdh.novel_method == MG_NOVEL_HYBRID_MTF)
172	for (which = 0; which <= 1; which++)
173	{
174	int j;
175
176	HTONUL(cd->ad->afh[which].num_frags);
177	HTONUL(cd->ad->afh[which].mem_for_frags);
178	for (j = 0; j < 33; j++)
179	{
180	HTONSI(cd->ad->blk_start[which][j]);
181	HTONSI(cd->ad->blk_end[which][j]);
182	}
183	}
184	/* cdh */
185	HTONUL(cd->cdh.dict_type);
186	HTONUL(cd->cdh.novel_method);
187	for (i = 0; i < TEXT_PARAMS; i++)
188	HTONUL(cd->cdh.params[which]);
189	HTONUL(cd->cdh.num_words[0]);
190	HTONUL(cd->cdh.num_words[1]);
191	HTONUL(cd->cdh.num_word_chars[0]);
192	HTONUL(cd->cdh.num_word_chars[1]);
193	HTONUL(cd->cdh.lookback);
194	HTONSI(cd->fast_loaded);
195	}
196
197	unfixup_buffer ();
198
199	save_fast_dict (filename);
200
201	Message ("%s", ElapsedTime (&StartTime, NULL));
202	return 0;
203	}
204
205
206
207	static void
208	unfixup_buffer ()
209	{
210	u_long *p;
211	for (p = buffer; (u_long) p < (u_long) cur; p++)
212	{
213	if (IS_FIXUP (p))
214	p = p - (u_long) buffer;
215	}
216	}
217
218
219
220
221	static u_long
222	mem_for_aux_dict (compression_dict_header * cdh, char *filename)
223	{
224	int i;
225	u_long mem = sizeof (auxiliary_dict);
226	FILE *aux;
227
228	aux = open_file (filename, TEXT_DICT_AUX_SUFFIX, "rb",
229	MAGIC_AUX_DICT, MG_ABORT); /* [RPAP - Feb 97: WIN32 Port] */
230
231	for (i = 0; i <= 1; i++)
232	{
233	aux_frags_header afh;
234	fread (&afh, sizeof (afh), 1, aux);
235	NTOHUL(afh.num_frags); /* [RPAP - Jan 97: Endian Ordering] */
236	NTOHUL(afh.mem_for_frags); /* [RPAP - Jan 97: Endian Ordering] */
237	mem += afh.num_frags * sizeof (u_char *);
238	mem = ALIGN_SIZE (mem + afh.mem_for_frags, u_char *);
239	fseek (aux, afh.mem_for_frags, SEEK_CUR);
240	}
241	fclose (aux);
242
243	return mem;
244	}
245
246
247
248	static u_long
249	mem_for_words (FILE * dict, compression_dict_header * cdh,
250	comp_frags_header * cfh)
251	{
252	u_long mem = 0;
253	u_long i, lookback;
254	int ptrs_reqd = 0;
255	int mem_reqd = 0;
256
257	lookback = cdh->lookback;
258
259	for (i = cfh->hd.mincodelen; i <= cfh->hd.maxcodelen; i++)
260	{
261	ptrs_reqd += (cfh->hd.lencount[i] + ((1 << lookback) - 1)) >> lookback;
262	mem_reqd += cfh->huff_words_size[i];
263	}
264
265	mem += ptrs_reqd * sizeof (u_char *);
266	mem += (MAX_HUFFCODE_LEN + 1) * sizeof (u_char **);
267	mem += mem_reqd;
268
269	for (i = 0; i < cfh->hd.num_codes; i++)
270	{
271	register int val;
272	for (val = getc (dict) & 0xf; val; val--)
273	getc (dict);
274	}
275	return ALIGN_SIZE (mem, u_char *);
276	}
277
278
279
280
281	static u_long
282	mem_for_comp_dict (char *filename)
283	{
284	u_long mem = sizeof (compression_dict);
285	compression_dict_header cdh;
286	comp_frags_header cfh;
287	FILE *dict;
288	int which;
289	int i; /* [RPAP - Jan 97: Endian Ordering] */
290
291	dict = open_file (filename, TEXT_DICT_SUFFIX, "rb",
292	MAGIC_DICT, MG_ABORT); /* [RPAP - Feb 97: WIN32 Port] */
293
294	fread (&cdh, sizeof (cdh), 1, dict);
295	/* [RPAP - Jan 97: Endian Ordering] */
296	NTOHUL(cdh.dict_type);
297	NTOHUL(cdh.novel_method);
298	for (i = 0; i < TEXT_PARAMS; i++)
299	NTOHUL(cdh.params[i]);
300	NTOHUL(cdh.num_words[0]);
301	NTOHUL(cdh.num_words[1]);
302	NTOHUL(cdh.num_word_chars[0]);
303	NTOHUL(cdh.num_word_chars[1]);
304	NTOHUL(cdh.lookback);
305
306	for (which = 0; which < 2; which++)
307	switch (cdh.dict_type)
308	{
309	case MG_COMPLETE_DICTIONARY:
310	{
311	mem += sizeof (comp_frags_header);
312	Read_cfh (dict, &cfh, NULL, NULL);
313
314	/* Don't need to count the space for the clens of the huffman data */
315
316	mem += mem_for_words (dict, &cdh, &cfh);
317	if (cfh.hd.clens)
318	Xfree (cfh.hd.clens);
319	}
320	break;
321	case MG_PARTIAL_DICTIONARY:
322	{
323	huff_data hd;
324	if (cdh.num_words[which])
325	{
326	mem += sizeof (comp_frags_header);
327	Read_cfh (dict, &cfh, NULL, NULL);
328
329	/* Don't need to count the space for the clens of the
330	huffman data */
331
332	mem += mem_for_words (dict, &cdh, &cfh);
333	if (cfh.hd.clens)
334	Xfree (cfh.hd.clens);
335
336	}
337
338	mem += sizeof (hd);
339	Read_Huffman_Data (dict, &hd, NULL, NULL);
340	if (hd.clens)
341	Xfree (hd.clens);
342	mem += hd.num_codes * sizeof (unsigned long);
343	mem += (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *);
344
345	mem += sizeof (hd);
346	Read_Huffman_Data (dict, &hd, NULL, NULL);
347	if (hd.clens)
348	Xfree (hd.clens);
349	mem += hd.num_codes * sizeof (unsigned long);
350	mem += (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *);
351	}
352	break;
353	case MG_SEED_DICTIONARY:
354	{
355	huff_data hd;
356	if (cdh.num_words[which])
357	{
358	mem += sizeof (comp_frags_header);
359	Read_cfh (dict, &cfh, NULL, NULL);
360
361	/* Don't need to count the space for the clens of the
362	huffman data */
363
364	mem += mem_for_words (dict, &cdh, &cfh);
365	if (cfh.hd.clens)
366	Xfree (cfh.hd.clens);
367
368	}
369	switch (cdh.novel_method)
370	{
371	case MG_NOVEL_HUFFMAN_CHARS:
372	mem += sizeof (hd);
373	Read_Huffman_Data (dict, &hd, NULL, NULL);
374	if (hd.clens)
375	Xfree (hd.clens);
376	mem += hd.num_codes * sizeof (unsigned long);
377	mem += (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *);
378
379	mem += sizeof (hd);
380	Read_Huffman_Data (dict, &hd, NULL, NULL);
381	if (hd.clens)
382	Xfree (hd.clens);
383	mem += hd.num_codes * sizeof (unsigned long);
384	mem += (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *);
385	break;
386	case MG_NOVEL_BINARY:
387	break;
388	case MG_NOVEL_DELTA:
389	break;
390	case MG_NOVEL_HYBRID:
391	break;
392	case MG_NOVEL_HYBRID_MTF:
393	break;
394	}
395	break;
396	}
397	}
398	fclose (dict);
399
400	if (cdh.novel_method == MG_NOVEL_BINARY \|\|
401	cdh.novel_method == MG_NOVEL_DELTA \|\|
402	cdh.novel_method == MG_NOVEL_HYBRID \|\|
403	cdh.novel_method == MG_NOVEL_HYBRID_MTF)
404	mem += mem_for_aux_dict (&cdh, filename);
405
406	return ALIGN_SIZE (mem, u_char *);
407	}
408
409
410
411	void *
412	getmem (u_long size, int align)
413	{
414	void *res;
415	cur = (void *) (((u_long) cur + (align - 1)) & (~(align - 1)));
416	res = cur;
417	cur = (char *) cur + size;
418	if ((long) cur > (long) buffer + mem)
419	FatalError (1, "The buffer was not big enough for the dictionary");
420	return res;
421	}
422
423
424
425
426
427
428
429
430
431
432
433	static auxiliary_dict *
434	LoadAuxDict (compression_dict_header * cdh,
435	char *filename)
436	{
437	auxiliary_dict *ad;
438	int i;
439	FILE *aux;
440
441	aux = open_file (filename, TEXT_DICT_AUX_SUFFIX, "rb",
442	MAGIC_AUX_DICT, MG_ABORT); /* [RPAP - Feb 97: WIN32 Port] */
443
444	ad = getmem (sizeof (auxiliary_dict), sizeof (u_char *));
445
446	for (i = 0; i <= 1; i++)
447	{
448	int j;
449	u_char *pos;
450
451	fread (&ad->afh[i], sizeof (aux_frags_header), 1, aux);
452
453	/* [RPAP - Jan 97: Endian Ordering] */
454	NTOHUL(ad->afh[i].num_frags);
455	NTOHUL(ad->afh[i].mem_for_frags);
456
457	ad->word_data[i] = getmem (ad->afh[i].mem_for_frags, 1);
458	FIXUP (&ad->word_data[i]);
459
460	ad->words[i] = getmem (ad->afh[i].num_frags * sizeof (u_char *),
461	sizeof (u_char *));
462	FIXUP (&ad->words[i]);
463
464	fread (ad->word_data[i], ad->afh[i].mem_for_frags, sizeof (u_char), aux);
465
466	pos = ad->word_data[i];
467	for (j = 0; j < ad->afh[i].num_frags; j++)
468	{
469	ad->words[i][j] = pos;
470	FIXUP (&ad->words[i][j]);
471	pos += *pos + 1;
472	}
473	if (cdh->novel_method == MG_NOVEL_HYBRID \|\|
474	cdh->novel_method == MG_NOVEL_HYBRID_MTF)
475	{
476	int num;
477	num = 1;
478	ad->blk_start[i][0] = 0;
479	ad->blk_end[i][0] = cdh->num_words[i] - 1;
480	while (num < 33)
481	{
482	ad->blk_start[i][num] = ad->blk_end[i][num - 1] + 1;
483	ad->blk_end[i][num] = ad->blk_start[i][num] +
484	(ad->blk_end[i][num - 1] - ad->blk_start[i][num - 1]) * 2;
485	num++;
486	}
487	}
488	}
489	fclose (aux);
490	return (ad);
491	}
492
493
494
495
496
497	static u_char ***
498	ReadInWords (FILE * dict, compression_dict * cd,
499	comp_frags_header * cfh, u_char ** escape)
500	{
501	int i, lookback;
502	int ptrs_reqd = 0;
503	int mem_reqd = 0;
504	int num_set[MAX_HUFFCODE_LEN + 1];
505	u_char *next_word[MAX_HUFFCODE_LEN + 1];
506	u_char **vals;
507	u_char ***values;
508	u_char word[MAXWORDLEN + 1];
509	u_char last_word[MAX_HUFFCODE_LEN + 1][MAXWORDLEN + 1];
510
511	lookback = cd->cdh.lookback;
512
513	for (i = cfh->hd.mincodelen; i <= cfh->hd.maxcodelen; i++)
514	{
515	ptrs_reqd += (cfh->hd.lencount[i] + ((1 << lookback) - 1)) >> lookback;
516	mem_reqd += cfh->huff_words_size[i];
517	}
518
519	vals = getmem (ptrs_reqd * sizeof (vals), sizeof (u_char ));
520
521	values = getmem ((MAX_HUFFCODE_LEN + 1) * sizeof (u_char ), sizeof (u_char ));
522
523	next_word[0] = getmem (mem_reqd, sizeof (char));
524
525	cd->MemForCompDict += ptrs_reqd * sizeof (*vals) +
526	(MAX_HUFFCODE_LEN + 1) * sizeof (u_char **) +
527	mem_reqd;
528
529	values[0] = vals;
530	FIXUP (&values[0]);
531	values[0][0] = next_word[0];
532	FIXUP (&values[0][0]);
533	for (i = 1; i <= cfh->hd.maxcodelen; i++)
534	{
535	int next_start = (values[i - 1] - vals) +
536	((cfh->hd.lencount[i - 1] + ((1 << lookback) - 1)) >> lookback);
537	values[i] = &vals[next_start];
538	FIXUP (&values[i]);
539	next_word[i] = next_word[i - 1] + cfh->huff_words_size[i - 1];
540	values[i][0] = next_word[i];
541	FIXUP (&values[i][0]);
542	}
543
544	bzero ((char *) num_set, sizeof (num_set));
545
546	for (i = 0; i < cfh->hd.num_codes; i++)
547	{
548	register int val, copy;
549	register int len = cfh->hd.clens[i];
550	val = getc (dict);
551	copy = (val >> 4) & 0xf;
552	val &= 0xf;
553
554	fread (word + copy + 1, sizeof (u_char), val, dict);
555	*word = val + copy;
556
557	if ((num_set[len] & ((1 << lookback) - 1)) == 0)
558	{
559	values[len][num_set[len] >> lookback] = next_word[len];
560	FIXUP (&values[len][num_set[len] >> lookback]);
561	memcpy (next_word[len], word, *word + 1);
562	if (escape && i == cfh->hd.num_codes - 1)
563	{
564	*escape = next_word[len];
565	FIXUP (escape);
566	}
567	next_word[len] += *word + 1;
568	}
569	else
570	{
571	copy = prefixlen (last_word[len], word);
572	memcpy (next_word[len] + 1, word + copy + 1, *word - copy);
573	next_word[len] = (copy << 4) + (word - copy);
574	if (escape && i == cfh->hd.num_codes - 1)
575	{
576	*escape = next_word[len];
577	FIXUP (escape);
578	}
579	next_word[len] += (*word - copy) + 1;
580	}
581	memcpy (last_word[len], word, *word + 1);
582	num_set[len]++;
583	}
584	if (cfh->hd.clens)
585	Xfree (cfh->hd.clens);
586	cfh->hd.clens = NULL;
587	return values;
588	}
589
590
591	static unsigned long **
592	Generate_Fast_Huffman_Vals (huff_data * data, u_long * mem)
593	{
594	int i;
595	unsigned long *fcode[MAX_HUFFCODE_LEN + 1];
596	unsigned long **values;
597	unsigned long *vals;
598
599	if (!data)
600	return (NULL);
601	vals = getmem (data->num_codes * sizeof (vals), sizeof (long ));
602	values = getmem ((MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *),
603	sizeof (long *));
604
605	bzero ((char ) values, (MAX_HUFFCODE_LEN + 1) sizeof (unsigned long *));
606
607	if (mem)
608	mem += data->num_codes sizeof (*vals) +
609	(MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *);
610
611	fcode[0] = values[0] = &vals[0];
612	FIXUP (&values[0]);
613	for (i = 1; i <= data->maxcodelen; i++)
614	{
615	fcode[i] = values[i] = &vals[(values[i - 1] - vals) + data->lencount[i - 1]];
616	FIXUP (&values[i]);
617	}
618
619	for (i = 0; i < data->num_codes; i++)
620	if (data->clens[i])
621	*fcode[(int) (data->clens[i])]++ = i;
622	return (values);
623	}
624
625
626
627	static void
628	load_comp_dict (char *filename)
629	{
630	FILE *dict;
631	int which;
632	compression_dict *cd;
633
634	cd = getmem (sizeof (compression_dict), sizeof (u_char *));
635	cd->fast_loaded = 1;
636
637	dict = open_file (filename, TEXT_DICT_SUFFIX, "rb",
638	MAGIC_DICT, MG_ABORT); /* [RPAP - Feb 97: WIN32 Port] */
639
640	Read_cdh (dict, &cd->cdh, NULL, NULL);
641
642	for (which = 0; which < 2; which++)
643	switch (cd->cdh.dict_type)
644	{
645	case MG_COMPLETE_DICTIONARY:
646	{
647	cd->cfh[which] = getmem (sizeof (cd->cfh[which]), sizeof (u_char ));
648	cd->MemForCompDict += sizeof (*cd->cfh[which]);
649	Read_cfh (dict, cd->cfh[which], &cd->MemForCompDict, NULL);
650
651	cd->values[which] = ReadInWords (dict, cd, cd->cfh[which], NULL);
652	FIXUP (&cd->cfh[which]);
653	FIXUP (&cd->values[which]);
654	cd->escape[which] = NULL;
655	}
656	break;
657	case MG_PARTIAL_DICTIONARY:
658	{
659	huff_data *hd;
660	u_long **vals;
661	if (cd->cdh.num_words[which])
662	{
663	cd->cfh[which] = getmem (sizeof (cd->cfh[which]), sizeof (u_char ));
664	cd->MemForCompDict += sizeof (*cd->cfh[which]);
665	Read_cfh (dict, cd->cfh[which], &cd->MemForCompDict, NULL);
666	cd->values[which] = ReadInWords (dict, cd, cd->cfh[which],
667	&cd->escape[which]);
668	FIXUP (&cd->cfh[which]);
669	FIXUP (&cd->values[which]);
670	FIXUP (&cd->escape[which]);
671	}
672
673	hd = getmem (sizeof (huff_data), sizeof (char *));
674	cd->MemForCompDict += sizeof (huff_data);
675	Read_Huffman_Data (dict, hd, &cd->MemForCompDict, NULL);
676	vals = Generate_Fast_Huffman_Vals (hd, &cd->MemForCompDict);
677	cd->chars_huff[which] = hd;
678	FIXUP (&cd->chars_huff[which]);
679	cd->chars_vals[which] = vals;
680	FIXUP (&cd->chars_vals[which]);
681	if (hd->clens)
682	Xfree (hd->clens);
683	hd->clens = NULL;
684
685	hd = getmem (sizeof (huff_data), sizeof (char *));
686	cd->MemForCompDict += sizeof (huff_data);
687	Read_Huffman_Data (dict, hd, &cd->MemForCompDict, NULL);
688	vals = Generate_Fast_Huffman_Vals (hd, &cd->MemForCompDict);
689	cd->lens_huff[which] = hd;
690	FIXUP (&cd->lens_huff[which]);
691	cd->lens_vals[which] = vals;
692	FIXUP (&cd->lens_vals[which]);
693	if (hd->clens)
694	Xfree (hd->clens);
695	hd->clens = NULL;
696	}
697	break;
698	case MG_SEED_DICTIONARY:
699	{
700	huff_data *hd;
701	u_long **vals;
702	if (cd->cdh.num_words[which])
703	{
704	cd->cfh[which] = getmem (sizeof (cd->cfh[which]), sizeof (u_char ));
705	cd->MemForCompDict += sizeof (*cd->cfh[which]);
706	Read_cfh (dict, cd->cfh[which], &cd->MemForCompDict, NULL);
707	cd->values[which] = ReadInWords (dict, cd, cd->cfh[which],
708	&cd->escape[which]);
709	FIXUP (&cd->cfh[which]);
710	FIXUP (&cd->values[which]);
711	FIXUP (&cd->escape[which]);
712	}
713	switch (cd->cdh.novel_method)
714	{
715	case MG_NOVEL_HUFFMAN_CHARS:
716	hd = getmem (sizeof (huff_data), sizeof (char *));
717	cd->MemForCompDict += sizeof (huff_data);
718	Read_Huffman_Data (dict, hd, &cd->MemForCompDict, NULL);
719	vals = Generate_Fast_Huffman_Vals (hd, &cd->MemForCompDict);
720	cd->chars_huff[which] = hd;
721	FIXUP (&cd->chars_huff[which]);
722	cd->chars_vals[which] = vals;
723	FIXUP (&cd->chars_vals[which]);
724	if (hd->clens)
725	Xfree (hd->clens);
726	hd->clens = NULL;
727
728	hd = getmem (sizeof (huff_data), sizeof (char *));
729	cd->MemForCompDict += sizeof (huff_data);
730	Read_Huffman_Data (dict, hd, &cd->MemForCompDict, NULL);
731	vals = Generate_Fast_Huffman_Vals (hd, &cd->MemForCompDict);
732	cd->lens_huff[which] = hd;
733	FIXUP (&cd->lens_huff[which]);
734	cd->lens_vals[which] = vals;
735	FIXUP (&cd->lens_vals[which]);
736	if (hd->clens)
737	Xfree (hd->clens);
738	hd->clens = NULL;
739	break;
740	case MG_NOVEL_BINARY:
741	break;
742	case MG_NOVEL_DELTA:
743	break;
744	case MG_NOVEL_HYBRID:
745	break;
746	case MG_NOVEL_HYBRID_MTF:
747	break;
748	}
749	break;
750	}
751	}
752	fclose (dict);
753
754
755	if (cd->cdh.novel_method == MG_NOVEL_BINARY \|\|
756	cd->cdh.novel_method == MG_NOVEL_DELTA \|\|
757	cd->cdh.novel_method == MG_NOVEL_HYBRID \|\|
758	cd->cdh.novel_method == MG_NOVEL_HYBRID_MTF)
759	{
760	cd->ad = LoadAuxDict (&cd->cdh, filename);
761	FIXUP (&cd->ad);
762	}
763	}
764
765
766	static void
767	save_fast_dict (char *filename)
768	{
769	FILE *fdict;
770
771	fdict = create_file (filename, TEXT_DICT_FAST_SUFFIX, "wb",
772	MAGIC_FAST_DICT, MG_ABORT); /* [RPAP - Feb 97: WIN32 Port] */
773
774	{
775	u_long *p;
776	for (p = buffer; (u_long) p < (u_long) cur; p++)
777	{
778	if (IS_FIXUP (p))
779	HTONUL(*p);
780	}
781	}
782
783	/* [RPAP - Jan 97: Endian Ordering] */
784	HTONUL(mem);
785	HTONUL(fixup_mem);
786
787	fwrite (&mem, sizeof (mem), 1, fdict);
788	fwrite (&fixup_mem, sizeof (fixup_mem), 1, fdict);
789
790	/* [RPAP - Jan 97: Endian Ordering] */
791	NTOHUL(mem);
792	NTOHUL(fixup_mem);
793
794	fwrite (buffer, sizeof (u_char), mem, fdict);
795	fwrite (fixup, sizeof (u_char), fixup_mem, fdict);
796
797	fclose (fdict);
798	}

Note: See TracBrowser for help on using the repository browser.

Download in other formats: