Index: /trunk/gsdl/src/mgpp/text/Makefile.in
===================================================================
--- /trunk/gsdl/src/mgpp/text/Makefile.in	(revision 856)
+++ /trunk/gsdl/src/mgpp/text/Makefile.in	(revision 856)
@@ -0,0 +1,326 @@
+###########################################################################
+#
+# Makefile.in -- Makefile for the MG system
+# Copyright (C) 1994  Neil Sharman;  1995 Tim Shimmin
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)Makefile.in	1.10 22 Mar 1994
+#
+###########################################################################
+
+PACKAGE = @PACKAGE@
+VERSION = @VERSION@
+
+SHELL = /bin/sh
+VPATH = @srcdir@
+exec_prefix = @exec_prefix@
+prefix = @prefix@
+srcdir = @srcdir@
+subdir = text
+top_srcdir = @top_srcdir@
+transform = @program_transform_name@
+
+AR = ar
+CC = @CC@
+CPP = @CXX@
+ifneq ($(NZDL),1)
+CFLAGS = @CFLAGS@ -ansi -g -Wall -Wunused -pedantic -W -Woverloaded-virtual
+CPPFLAGS = @CPPFLAGS@ -ansi -g -Wall -Wunused -pedantic -W -Woverloaded-virtual
+else
+CFLAGS = @CFLAGS@ -ansi -Wall -Wunused -pedantic -W -Woverloaded-virtual -g -DNZDL -DQUIET -DSHORT_SUFFIX -DPARADOCNUM
+CPPFLAGS = @CPPFLAGS@ -ansi -Wall -Wunused -pedantic -W -Woverloaded-virtual -g -DNZDL -DQUIET -DSHORT_SUFFIX -DPARADOCNUM
+endif
+DEFS = @DEFS@
+RANLIB = @RANLIB@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+LDFLAGS = @LDFLAGS@
+LIBS = ../lib/libmg.a -lstdc++ @LIBS@
+TOUCH = echo timestamp >
+AWK = @AWK@
+
+##########################################
+# George Buchanan - August 1999
+# 
+# The below section is now irrelevant - 
+# see the ifndef on CFLAGS above; simply
+# compiling with make NZDL=1 will auto-
+# matically ensure that the NZDL compile
+# mode is used
+#
+##########################################
+#
+# Uncomment for compiling for the nzdl project
+#
+#CFLAGS = @CFLAGS@ -DNZDL -DQUIET -DSHORT_SUFFIX -DPARADOCNUM
+##########################################
+
+
+bindir = $(exec_prefix)/bin
+datadir = $(prefix)/share
+libexecdir = $(exec_prefix)/libexec
+localedir = $(datadir)/locale
+mandir = $(prefix)/man/man1
+
+COMPILE = $(CC) -c $(DEFS) $(TREC_MODE) $(READLINE) $(INCLUDES) $(CPPFLAGS) $(CFLAGS)
+CPPCOMPILE = $(CPP) -c $(DEFS) $(TREC_MODE) $(READLINE) $(INCLUDES) $(CPPFLAGS) $(CFLAGS)
+LINK = $(CC) $(LDFLAGS) -o $@
+
+.SUFFIXES:
+.SUFFIXES: .c .o
+.c.o:
+	$(COMPILE) $<
+
+.SUFFIXES: .cpp .o
+.cpp.o:
+	$(CPPCOMPILE) -D__cplusplus $<
+
+
+ANSI2KNR = @ANSI2KNR@
+o = .@U@o
+
+.SUFFIXES: ._c ._o
+.c._c:
+	./ansi2knr $< > $*.tmp && mv $*.tmp $@
+._c._o:
+	@echo $(COMPILE) $<
+	@rm -f _$*.c
+	@ln $< _$*.c && $(COMPILE) _$*.c && mv _$*.o $@ && rm _$*.c
+.c._o:
+	./ansi2knr $< > $*.tmp && mv $*.tmp $*._c
+	@echo $(COMPILE) $*._c
+	@rm -f _$*.c
+	@ln $*._c _$*.c && $(COMPILE) _$*.c && mv _$*.o $@ && rm _$*.c
+
+INCLUDES = -I../ -I../lib -I.
+
+#############################################################################
+#
+# Uncomment (and possibly edit) the following lines if you wish
+# to use the GNU readline library for the input in mgquery
+# 
+#READLINE = -DGNU_READLINE -I/opt/gnu/include/readline -L/opt/gnu/lib
+#READLINE_LIBS = -lreadline -ltermcap
+#
+#
+#############################################################################
+
+SOURCES = \
+  FIvfLevelInfo.cpp        ivf.pass1.cpp            mg_invf_dump.cpp       \
+  FText.cpp                ivf.pass2.cpp            mg_passes.cpp          \
+  FileBuf.cpp              locallib.cpp             mg_perf_hash_build.cpp \
+  FragLevelConvert.cpp     mg.special.cpp           mg_stem_idx.cpp        \
+  IndexData.cpp            mg_compression_dict.cpp  mg_weights_build.cpp   \
+  TagInfo.cpp              mg_decompress_text.cpp   stemmer.cpp            \
+  TextEl.cpp               mg_errors.cpp            text.cpp               \
+  TextGet.cpp              mg_fast_comp_dict.cpp    text.pass1.cpp         \
+  UCArray.cpp              mg_files.cpp             text.pass2.cpp         \
+  comp_dict.cpp            mg_invf_dict.cpp         words.cpp              \
+  invf.cpp                 mg_invf_dict_dump.cpp    Weights.cpp            \
+  MGQuery.cpp              Terms.cpp                QueryTester.cpp        \
+  QueryLex.cpp             QueryParser.cpp          Queryer.cpp
+
+
+HEADERS = \
+  FIvfLevelInfo.h     TextEl.h            hash.h              stemmer.h  \
+  FText.h             TextGet.h           invf.h              text.h     \
+  FileBuf.h           UCArray.h           locallib.h          words.h    \
+  FragLevelConvert.h  WordData.h          mg.h                           \
+  IndexData.h         build.h             mg_errors.h                    \
+  TagInfo.h           comp_dict.h         mg_files.h          Weights.h  \
+  MGQuery.h           Terms.h             QueryLex.h          QueryParser.h
+
+
+MISC          = Makefile.in 
+
+ALLFILES      =	$(SOURCES) $(HEADERS) $(MISC) $(MAN)
+
+DISTFILES     = $(ALLFILES) 
+
+EXEC          = mg_passes mg_decompress_text mg_perf_hash_build \
+                mg_invf_dump mg_weights_build mg_invf_dict mg_stem_idx \
+                mg_invf_dict_dump mg_compression_dict mg_fast_comp_dict \
+                QueryTester Queryer
+
+MAN           =  mg_weights_build.1  mg_invf_dict.1 \
+                 mg_passes.1 mg_perf_hash_build.1 \
+                 mg_compression_dict.1  mg_fast_comp_dict.1 \
+                 mg_stem_idx.1
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+
+all:	$(EXEC) libtextin.a
+
+depend:
+	makedepend -Y -- $(DEFS) $(INCLUDES) $(CPPFLAGS) $(CFLAGS) -- $(SOURCES)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+
+PASSES_OBJS = \
+	mg_passes$o text.pass1$o comp_dict$o \
+	text.pass2$o locallib$o \
+	ivf.pass1$o ivf.pass2$o mg.special$o mg_files$o \
+	words$o invf$o FileBuf$o TextEl$o UCArray$o TagInfo$o \
+        FText$o text$o FIvfLevelInfo$o
+
+mg_passes: $(PASSES_OBJS)
+	$(LINK) $(PASSES_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+QUERYER_OBJS = Queryer$o mg_files$o mg_errors$o locallib$o invf$o UCArray$o \
+               FIvfLevelInfo$o FragLevelConvert$o Terms$o MGQuery$o \
+               IndexData$o stemmer$o Weights$o TextGet$o text$o FText$o \
+               QueryParser$o QueryLex$o words$o
+
+Queryer: $(QUERYER_OBJS)
+	$(LINK) $(QUERYER_OBJS) $(LIBS)
+
+QUERY_TESTER_OBJS = QueryTester$o mg_files$o invf$o UCArray$o \
+                    FIvfLevelInfo$o FragLevelConvert$o Terms$o MGQuery$o \
+                    IndexData$o stemmer$o Weights$o
+
+QueryTester: $(QUERY_TESTER_OBJS)
+	$(LINK) $(QUERY_TESTER_OBJS) $(LIBS)
+
+WEIGHTS_BUILD_OBJS = mg_weights_build$o mg_files$o invf$o UCArray$o \
+                     FIvfLevelInfo$o FragLevelConvert$o
+
+mg_weights_build: $(WEIGHTS_BUILD_OBJS)
+	$(LINK) $(WEIGHTS_BUILD_OBJS) $(LIBS)
+
+COMP_DICT_OBJS = mg_compression_dict$o mg_files$o locallib$o
+
+mg_compression_dict: $(COMP_DICT_OBJS)
+	$(LINK) $(COMP_DICT_OBJS) $(LIBS)
+
+FAST_COMP_OBJS = mg_fast_comp_dict$o mg_files$o locallib$o
+
+mg_fast_comp_dict: $(FAST_COMP_OBJS)
+	$(LINK) $(FAST_COMP_OBJS) $(LIBS)
+
+INVF_DUMP_OBJS = mg_invf_dump$o locallib$o mg_files$o invf$o UCArray$o
+
+mg_invf_dump: $(INVF_DUMP_OBJS)
+	$(LINK) $(INVF_DUMP_OBJS) $(LIBS)
+
+INVF_DICT_OBJS = mg_invf_dict$o mg_files$o invf$o UCArray$o
+
+mg_invf_dict: $(INVF_DICT_OBJS)
+	$(LINK) $(INVF_DICT_OBJS) $(LIBS)
+
+INVF_DICT_DUMP_OBJS = mg_invf_dict_dump$o mg_files$o invf$o UCArray$o
+
+mg_invf_dict_dump: $(INVF_DICT_DUMP_OBJS)
+	$(LINK) $(INVF_DICT_DUMP_OBJS) $(LIBS)
+
+PERF_HASH_OBJS = mg_perf_hash_build$o mg_files$o invf$o UCArray$o
+
+mg_perf_hash_build: $(PERF_HASH_OBJS)
+	$(LINK) $(PERF_HASH_OBJS) $(LIBS)
+
+STEM_IDX_OBJS = mg_stem_idx$o mg_files$o stemmer$o \
+                invf$o UCArray$o
+
+mg_stem_idx: $(STEM_IDX_OBJS)
+	$(LINK) $(STEM_IDX_OBJS) $(LIBS)
+
+DECOMPRESS_TEXT_OBJS = \
+                mg_decompress_text$o mg_files$o mg_errors$o \
+                locallib$o text$o UCArray$o TextGet$o FText$o
+
+mg_decompress_text: $(DECOMPRESS_TEXT_OBJS)
+	$(LINK) $(DECOMPRESS_TEXT_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+ansi2knr: ansi2knr.o
+	$(LINK) ansi2knr.o $(LIBS)
+
+install: execinstall maninstall
+uninstall: execuninstall manuninstall
+
+execinstall: $(EXEC)
+	$(top_srcdir)/mkinstalldirs $(bindir); \
+	for name in $(EXEC); do \
+	  $(INSTALL_PROGRAM) $$name \
+	  $(bindir)/`echo $$name | sed '$(transform)'`; \
+	done
+
+execuninstall:
+	for name in $(EXEC); do \
+	 rm -f $(bindir)/`echo $$name | sed '$(transform)'`; \
+	done
+
+maninstall: $(MAN)
+	$(top_srcdir)/mkinstalldirs $(mandir); \
+	for f in $(MAN); do \
+	  $(INSTALL_DATA) $$f $(mandir)/$$f ;\
+	done
+
+manuninstall: $(MAN)
+	for f in $(MAN); do \
+	  rm -f $(mandir)/$$f ;\
+	done
+
+
+LIB_OBJS = Queryer$o mg_files$o mg_errors$o locallib$o invf$o UCArray$o \
+           FIvfLevelInfo$o FragLevelConvert$o Terms$o MGQuery$o \
+           IndexData$o stemmer$o Weights$o TextGet$o text$o FText$o \
+           QueryParser$o QueryLex$o words$o
+
+libtextin.a: $(LIB_OBJS)
+	rm -f libtextin.a
+	$(AR) cru libtextin.a $(LIB_OBJS)
+	$(RANLIB) libtextin.a
+
+
+mostlyclean:
+	rm -f *$o _*.c _*.o *._c *._o core core.*
+
+clean: mostlyclean
+	rm -f $(EXEC)
+
+distclean: clean
+	rm -f ansi2knr
+	rm -f Makefile
+
+maintainer-clean: distclean
+	@echo "This command is intended only for maintainers to use;"
+	@echo "rebuilding the deleted files may require special tools."
+
+dist: $(DISTFILES)
+	for file in $(DISTFILES); do \
+	  ln $(srcdir)/$$file $(distdir) 2> /dev/null \
+	  || cp -p $(srcdir)/$$file $(distdir); \
+	done
+
+movclean:
+	for file in $(SOURCES); do \
+          test = $$file; \
+          echo $("test.c"##*.c); \
+        done 
+
+Makefile: Makefile.in ../config.status
+	cd .. && CONFIG_FILES=$(subdir)/$@ CONFIG_HEADERS= ./config.status
+
+# Tell versions [3.59,3.63) of GNU make not to export all variables.
+# Otherwise a system limit (for SysV at least) may be exc
+
+
Index: /trunk/gsdl/src/mgpp/text/WIN32.MAK
===================================================================
--- /trunk/gsdl/src/mgpp/text/WIN32.MAK	(revision 856)
+++ /trunk/gsdl/src/mgpp/text/WIN32.MAK	(revision 856)
@@ -0,0 +1,240 @@
+#PACKAGE = mg
+#VERSION = 1.3
+
+#SHELL = /bin/sh
+#exec_prefix = ${prefix}
+#prefix = /image/httpd/mg-1.2
+#srcdir = .
+#subdir = src/text
+#top_srcdir = ../..
+#transform = s,x,x,
+
+CC = cl
+#CC = cl /Z7  this gives symbol and line number debugging (set /DEBUG on link)
+#CFLAGS = -g -O
+CPPFLAGS =
+DEFS = -DNZDL -DQUIET -DSHORT_SUFFIX -DPARADOCNUM -DHAVE_CONFIG_H -D__WIN32__ -D_LITTLE_ENDIAN
+#INSTALL = /usr/local/gnu/bin/install -c
+#INSTALL_DATA = ${INSTALL} -m 644
+#INSTALL_PROGRAM = ${INSTALL}
+LDFLAGS =
+LIBS = ..\..\lib\libmg.lib
+#TOUCH = echo timestamp >
+#AWK = gawk
+
+#bindir = $(exec_prefix)/bin
+#datadir = $(prefix)/share
+#libexecdir = $(exec_prefix)/libexec
+#localedir = $(datadir)/locale
+#mandir = $(prefix)/man/man1
+
+COMPILE = $(CC) -c $(DEFS) -I. -I..\.. -I..\..\lib
+LINK = $(CC) $(LDFLAGS)
+
+.SUFFIXES:
+.SUFFIXES: .c .obj .cpp
+.c.obj:
+	$(COMPILE) $<
+.cpp.obj:
+	$(COMPILE) $<
+
+o = .obj
+e = .exe
+
+#############################################################################
+#
+# Uncomment (and possibly edit) the following lines if you wish
+# to use the GNU readline library for the input in mgquery
+#
+#READLINE = -DGNU_READLINE -I/opt/gnu/include/readline -L/opt/gnu/lib
+#READLINE_LIBS = -lreadline -ltermcap
+#
+#
+#############################################################################
+
+
+SOURCES = \
+  FIvfLevelInfo.cpp        ivf.pass1.cpp            mg_invf_dump.cpp       \
+  FText.cpp                ivf.pass2.cpp            mg_passes.cpp          \
+  FileBuf.cpp              locallib.cpp             mg_perf_hash_build.cpp \
+  FragLevelConvert.cpp     mg.special.cpp           mg_stem_idx.cpp        \
+  IndexData.cpp            mg_compression_dict.cpp  mg_weights_build.cpp   \
+  TagInfo.cpp              mg_decompress_text.cpp   stemmer.cpp            \
+  TextEl.cpp               mg_errors.cpp            text.cpp               \
+  TextGet.cpp              mg_fast_comp_dict.cpp    text.pass1.cpp         \
+  UCArray.cpp              mg_files.cpp             text.pass2.cpp         \
+  comp_dict.cpp            mg_invf_dict.cpp         words.cpp              \
+  invf.cpp                 mg_invf_dict_dump.cpp    Weights.cpp            \
+  MGQuery.cpp              Terms.cpp                QueryTester.cpp        \
+  QueryLex.cpp             QueryParser.cpp          Queryer.cpp
+
+
+HEADERS = \
+  FIvfLevelInfo.h     TextEl.h            hash.h              stemmer.h  \
+  FText.h             TextGet.h           invf.h              text.h     \
+  FileBuf.h           UCArray.h           locallib.h          words.h    \
+  FragLevelConvert.h  WordData.h          mg.h                           \
+  IndexData.h         build.h             mg_errors.h                    \
+  TagInfo.h           comp_dict.h         mg_files.h          Weights.h  \
+  MGQuery.h           Terms.h             QueryLex.h          QueryParser.h
+
+#MISC          = Makefile.in 
+
+ALLFILES      = $(SOURCES) $(HEADERS) $(MISC) $(MAN)
+
+DISTFILES     = $(ALLFILES)
+
+EXEC          = mg_passes$e mg_decompress_text$e mg_perf_hash_build$e \
+                mg_invf_dump$e mg_weights_build$e mg_invf_dict$e mg_stem_idx$e \
+                mg_invf_dict_dump$e mg_compression_dict$e mg_fast_comp_dict$e \
+                QueryTester$e Queryer$e
+
+MAN           =  mg_weights_build.1  mg_invf_dict.1 \
+                 mg_passes.1 mg_perf_hash_build.1 \
+                 mg_compression_dict.1  mg_fast_comp_dict.1 \
+                 mg_stem_idx.1
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+
+all:    $(EXEC) libtextin.lib
+
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+HILITE_OBJS = mg_hilite_words$o stemmer$o locallib$o words$o
+
+mg_hilite_words$e: $(HILITE_OBJS)
+	$(LINK) $(HILITE_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+PASSES_OBJS = \
+	mg_passes$o text.pass1$o comp_dict$o \
+	text.pass2$o locallib$o \
+	ivf.pass1$o ivf.pass2$o mg.special$o mg_files$o \
+	words$o invf$o FileBuf$o TextEl$o UCArray$o TagInfo$o \
+        FText$o text$o FIvfLevelInfo$o
+
+mg_passes: $(PASSES_OBJS)
+	$(LINK) $(PASSES_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+QUERYER_OBJS = Queryer$o mg_files$o mg_errors$o locallib$o invf$o UCArray$o \
+               FIvfLevelInfo$o FragLevelConvert$o Terms$o MGQuery$o \
+               IndexData$o stemmer$o Weights$o TextGet$o text$o FText$o \
+               QueryParser$o QueryLex$o words$o
+
+Queryer: $(QUERYER_OBJS)
+	$(LINK) $(QUERYER_OBJS) $(LIBS)
+
+QUERY_TESTER_OBJS = QueryTester$o mg_files$o invf$o UCArray$o \
+                    FIvfLevelInfo$o FragLevelConvert$o Terms$o MGQuery$o \
+                    IndexData$o stemmer$o Weights$o
+
+QueryTester: $(QUERY_TESTER_OBJS)
+	$(LINK) $(QUERY_TESTER_OBJS) $(LIBS)
+
+WEIGHTS_BUILD_OBJS = mg_weights_build$o mg_files$o invf$o UCArray$o \
+                     FIvfLevelInfo$o FragLevelConvert$o
+
+mg_weights_build: $(WEIGHTS_BUILD_OBJS)
+	$(LINK) $(WEIGHTS_BUILD_OBJS) $(LIBS)
+
+COMP_DICT_OBJS = mg_compression_dict$o mg_files$o locallib$o
+
+mg_compression_dict: $(COMP_DICT_OBJS)
+	$(LINK) $(COMP_DICT_OBJS) $(LIBS)
+
+FAST_COMP_OBJS = mg_fast_comp_dict$o mg_files$o locallib$o
+
+mg_fast_comp_dict: $(FAST_COMP_OBJS)
+	$(LINK) $(FAST_COMP_OBJS) $(LIBS)
+
+INVF_DUMP_OBJS = mg_invf_dump$o locallib$o mg_files$o invf$o UCArray$o
+
+mg_invf_dump: $(INVF_DUMP_OBJS)
+	$(LINK) $(INVF_DUMP_OBJS) $(LIBS)
+
+INVF_DICT_OBJS = mg_invf_dict$o mg_files$o invf$o UCArray$o
+
+mg_invf_dict: $(INVF_DICT_OBJS)
+	$(LINK) $(INVF_DICT_OBJS) $(LIBS)
+
+INVF_DICT_DUMP_OBJS = mg_invf_dict_dump$o mg_files$o invf$o UCArray$o
+
+mg_invf_dict_dump: $(INVF_DICT_DUMP_OBJS)
+	$(LINK) $(INVF_DICT_DUMP_OBJS) $(LIBS)
+
+PERF_HASH_OBJS = mg_perf_hash_build$o mg_files$o invf$o UCArray$o
+
+mg_perf_hash_build: $(PERF_HASH_OBJS)
+	$(LINK) $(PERF_HASH_OBJS) $(LIBS)
+
+STEM_IDX_OBJS = mg_stem_idx$o mg_files$o stemmer$o \
+                invf$o UCArray$o
+
+mg_stem_idx: $(STEM_IDX_OBJS)
+	$(LINK) $(STEM_IDX_OBJS) $(LIBS)
+
+DECOMPRESS_TEXT_OBJS = \
+                mg_decompress_text$o mg_files$o mg_errors$o \
+                locallib$o text$o UCArray$o TextGet$o FText$o
+
+mg_decompress_text: $(DECOMPRESS_TEXT_OBJS)
+	$(LINK) $(DECOMPRESS_TEXT_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+install: execinstall maninstall
+uninstall: execuninstall manuninstall
+
+maninstall: $(MAN)
+	$(top_srcdir)/mkinstalldirs $(mandir); \
+	for f in $(MAN); do \
+	  $(INSTALL_DATA) $$f $(mandir)/$$f ;\
+	done
+
+manuninstall: $(MAN)
+	for f in $(MAN); do \
+	  rm -f $(mandir)/$$f ;\
+	done
+
+
+LIB_OBJS = Queryer$o mg_files$o mg_errors$o locallib$o invf$o UCArray$o \
+           FIvfLevelInfo$o FragLevelConvert$o Terms$o MGQuery$o \
+           IndexData$o stemmer$o Weights$o TextGet$o text$o FText$o \
+           QueryParser$o QueryLex$o words$o
+
+libtextin.lib: $(LIB_OBJS)
+	del libtextin.lib
+	lib /out:libtextin.lib $(LIB_OBJS)
+
+
+mostlyclean:
+	rm -f *$o _*.c _*.o *._c *._o core core.*
+ 
+clean: mostlyclean
+	rm -f $(EXEC)
+ 
+distclean: clean
+	rm -f ansi2knr
+	rm -f Makefile
+ 
+maintainer-clean: distclean
+	@echo "This command is intended only for maintainers to use;"
+	@echo "rebuilding the deleted files may require special tools."
+ 
+#dist: $(DISTFILES)
+#        for file in $(DISTFILES); do \
+#          ln $(srcdir)/$$file $(distdir) 2> /dev/null \
+#          || cp -p $(srcdir)/$$file $(distdir); \
+#        done
+ 
+#Makefile: Makefile.in ../../config.status
+#        cd ../.. && CONFIG_FILES=$(subdir)/$@ CONFIG_HEADERS= ./config.status
+ 
+# Tell versions [3.59,3.63) of GNU make not to export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+#.NOEXPORT:
Index: /trunk/gsdl/src/mgpp/text/WordData.h
===================================================================
--- /trunk/gsdl/src/mgpp/text/WordData.h	(revision 856)
+++ /trunk/gsdl/src/mgpp/text/WordData.h	(revision 856)
@@ -0,0 +1,49 @@
+/**************************************************************************
+ *
+ * WordData.h -- 
+ * Copyright (C) 1999  NZDL
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+class WordData
+{
+
+ public:
+  unsigned long docCount;
+  unsigned long wordCount;
+
+  inline void read(FILE *f)
+    {
+      fread((char *) &docCount, sizeof(docCount), 1, f);
+      fread((char *) &wordCount, sizeof(wordCount), 1, f);
+
+      NTOHUL(docCount);
+      NTOHUL(wordCount);
+    }
+
+  inline unsigned long documents()
+    {
+      return docCount;
+    }
+
+  inline int words()
+    {
+      return wordCount;
+    }
+};
Index: /trunk/gsdl/src/mgpp/text/build.h
===================================================================
--- /trunk/gsdl/src/mgpp/text/build.h	(revision 856)
+++ /trunk/gsdl/src/mgpp/text/build.h	(revision 856)
@@ -0,0 +1,88 @@
+/**************************************************************************
+ *
+ * build.h -- Global information for the passes of mg_passes
+ * Copyright (C) 1994  Neil Sharman, Alistair Moffat and Lachlan Andrew
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef H_BUILD
+#define H_BUILD
+
+#include "longlong.h"
+
+#include "TextEl.h"
+#include "TagInfo.h"
+
+
+int init_special (const TagInfo &tagInfo, char *file_name);
+int init_text_1 (const TagInfo &tagInfo, char *file_name);
+int init_text_2 (const TagInfo &tagInfo, char *file_name);
+int init_ivf_1 (const TagInfo &tagInfo, char *file_name);
+int init_ivf_2 (const TagInfo &tagInfo, char *file_name);
+/*
+ * file_name  IN      The name of the dictionary file
+ * 
+ * returns      COMPALLOK for all ok,
+ *              COMPERROR for any error.  e.g. cannot read file
+ */
+
+
+
+
+
+int process_special (const TagInfo &tagInfo, const TextElArray &doc);
+int process_text_1 (const TagInfo &tagInfo, const TextElArray &doc);
+int process_text_2 (const TagInfo &tagInfo, const TextElArray &doc);
+int process_ivf_1 (const TagInfo &tagInfo, const TextElArray &doc);
+int process_ivf_2 (const TagInfo &tagInfo, const TextElArray &doc);
+/*
+ * returns      COMPALLOK for all ok,
+ *              COMPERROR for any error.  e.g. cannot read file
+ *
+ * The calling routine is responsible for ensuring that s_out is long 
+ * enough.
+ */
+
+
+
+
+
+int done_special (const TagInfo &tagInfo, char *filename);
+int done_text_1 (const TagInfo &tagInfo, char *filename);
+int done_text_2 (const TagInfo &tagInfo, char *filename);
+int done_ivf_1 (const TagInfo &tagInfo, char *filename);
+int done_ivf_2 (const TagInfo &tagInfo, char *filename);
+/*
+ * returns    COMPALLOK for all ok,
+ *              COMPERROR for any error.  e.g. cannot write file
+ */
+
+
+
+
+extern unsigned long invf_buffer_size;
+/*
+ * The amount of memory to allocate to the invertion buffer.
+ */
+
+
+
+#endif
Index: /trunk/gsdl/src/mgpp/text/comp_dict.cpp
===================================================================
--- /trunk/gsdl/src/mgpp/text/comp_dict.cpp	(revision 856)
+++ /trunk/gsdl/src/mgpp/text/comp_dict.cpp	(revision 856)
@@ -0,0 +1,255 @@
+/**************************************************************************
+ *
+ * comp_dict.cpp -- Functions for loading the compression dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "huffman.h"
+#include "local_strings.h"
+#include "memlib.h"
+#include "messages.h"
+
+#include "mg.h"
+#include "hash.h"
+#include "text.h"
+#include "comp_dict.h"
+#include "locallib.h"
+#include "mg_files.h"
+
+/*
+   $Log$
+   Revision 1.1  2000/01/14 02:26:04  sjboddie
+   Rodgers new C++ mg
+
+   Revision 1.1  1999/10/11 02:57:10  cs025
+   Base install of MG-PP
+
+   Revision 1.1  1999/08/10 21:17:48  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:34:34  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:41  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:24  tes
+   * For version 1.1
+   *
+ */
+
+
+compression_dict_header cdh;
+compressed_text_header cth;
+comp_frags_header cfh[2];
+
+dict_hash_table *ht[2];
+
+huff_data char_huff[2];
+huff_data lens_huff[2];
+u_long *char_codes[2], *lens_codes[2];
+u_long Words_disk = 0;
+u_long Chars_disk = 0;
+
+
+static dict_hash_table *
+ReadInWords (FILE * dict, comp_frags_header * cfh,
+	     int esc)
+{
+  int i;
+  u_char *allwords, *prev = NULL;
+  dict_hash_table *ht;
+  u_char **words;
+  u_long ht_size;
+
+  ht_size = prime (cfh->hd.num_codes * HASH_RATIO);
+  if (!(ht = (dict_hash_table *) Xmalloc (sizeof (dict_hash_table) +
+		                            (ht_size - 1) * sizeof (ht->table[0]))))
+    {
+      Message ("no memory for hash_table\n");
+      return NULL;
+    }
+
+  ht->size = ht_size;
+  ht->hd = &cfh->hd;
+
+  if (!(ht->codes = Generate_Huffman_Codes (&cfh->hd, NULL)))
+    {
+      Message ("no memory for huffman codes\n");
+      return NULL;
+    }
+
+  if (!(ht->words = (u_char **) Xmalloc (sizeof (u_char *) * cfh->hd.num_codes)))
+    {
+      Message ("no memory for word pointers\n");
+      return NULL;
+    }
+  words = ht->words;
+
+  bzero ((char *) ht->table, ht_size * sizeof (ht->table[0]));
+
+  if (!(allwords = (u_char *) Xmalloc (sizeof (u_char) * cfh->uncompressed_size)))
+    {
+      Message ("no memory for words\n");
+      return NULL;
+    }
+
+  for (i = 0; i < cfh->hd.num_codes; i++, words++)
+    {
+      register int val, copy;
+      val = fgetc (dict);
+      copy = (val >> 4) & 0xf;
+      val &= 0xf;
+
+      *words = allwords;
+
+      memcpy (allwords + 1, prev + 1, copy);
+      fread (allwords + copy + 1, sizeof (u_char), val, dict);
+      *allwords = val + copy;
+
+      Words_disk += val + 1;
+
+      /* insert into the hash table */
+      if (i < cfh->hd.num_codes - esc)
+	{
+	  register u_char **wptr;
+	  register unsigned long tsize = ht->size;
+	  register unsigned long hashval, step;
+
+	  HASH (hashval, step, allwords, tsize);
+
+	  wptr = ht->table[hashval];
+	  for (; wptr;)
+	    {
+	      hashval += step;
+	      if (hashval >= tsize)
+		hashval -= tsize;
+	      wptr = ht->table[hashval];
+	    }
+	  ht->table[hashval] = words;
+	}
+      prev = allwords;
+      allwords += *allwords + 1;
+    }
+  return ht;
+}
+
+inline int LoadWords(FILE *dict, int which)
+{
+  if (Read_cfh (dict, &cfh[which], NULL, &Words_disk) == -1)
+    return 1;
+  
+  if (!(ht[which] = ReadInWords (dict, &cfh[which], 0)))
+    return 1;
+
+  return 0;
+}
+
+inline int LoadHuffmanCodes(FILE *dict, int which)
+{
+  if (Read_Huffman_Data (dict, &char_huff[which], NULL,
+			 &Chars_disk) == -1)
+    return 1;
+  if (!(char_codes[which] =
+	Generate_Huffman_Codes (&char_huff[which], NULL)))
+    return 2;
+  if (Read_Huffman_Data (dict, &lens_huff[which], NULL,
+			 &Chars_disk) == -1)
+    return 3;
+  if (!(lens_codes[which] =
+	Generate_Huffman_Codes (&lens_huff[which], NULL)))
+    return 4;
+
+  return 0;
+}
+
+int 
+LoadCompressionDictionary (char *dict_file_name)
+{
+  FILE *dict;
+  int which;
+  if (!(dict = open_named_file (dict_file_name, "rb", MAGIC_DICT, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return COMPERROR;
+
+  Words_disk = sizeof (u_long);
+
+  if (Read_cdh (dict, &cdh, NULL, &Words_disk) == -1)
+    goto error;
+
+  for (which = 0; which < 2; which++)
+    switch (cdh.dict_type)
+      {
+      case MG_COMPLETE_DICTIONARY:
+	{
+	  if (LoadWords(dict, which) != 0)
+	    goto error;
+	}
+	break;
+      case MG_PARTIAL_DICTIONARY:
+	{
+	  if (cdh.num_words[which])
+	    {
+	      if (LoadWords(dict, which) != 0)
+		goto error;
+	    }
+	  else
+	    ht[which] = NULL;
+	  if (LoadHuffmanCodes(dict, which) != 0)
+	    goto error;
+	}
+	break;
+      case MG_SEED_DICTIONARY:
+	{
+	  if (cdh.num_words[which])
+	    {
+	      if (LoadWords(dict, which) != 0)
+		goto error;
+	    }
+	  else
+	    ht[which] = NULL;
+	  switch (cdh.novel_method)
+	    {
+	    case MG_NOVEL_HUFFMAN_CHARS:
+	      if (LoadHuffmanCodes(dict, which) != 0)
+		goto error;
+	      break;
+	    case MG_NOVEL_DELTA:
+	      break;
+	    case MG_NOVEL_HYBRID:
+	      break;
+	    default:
+	      FatalError (1, "Bad novel method");
+	    }
+	}
+	break;
+      default:
+	FatalError (1, "Bad dictionary kind\n");
+      }
+
+  return (COMPALLOK);
+
+
+error:
+  fclose (dict);
+  return (COMPERROR);
+}
Index: /trunk/gsdl/src/mgpp/text/comp_dict.h
===================================================================
--- /trunk/gsdl/src/mgpp/text/comp_dict.h	(revision 856)
+++ /trunk/gsdl/src/mgpp/text/comp_dict.h	(revision 856)
@@ -0,0 +1,59 @@
+/**************************************************************************
+ *
+ * comp_dict.h -- Functions for loading the compression dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+
+
+#define HASH_RATIO 2
+
+
+
+
+typedef struct dict_hash_table
+  {
+    u_long size;
+    huff_data *hd;
+    u_long *codes;
+    u_char **words;
+    u_char **table[1];
+  }
+dict_hash_table;
+
+
+
+
+extern compression_dict_header cdh;
+extern compressed_text_header cth;
+extern comp_frags_header cfh[2];
+
+extern dict_hash_table *ht[2];
+
+extern huff_data char_huff[2];
+extern huff_data lens_huff[2];
+extern u_long *char_codes[2], *lens_codes[2];
+extern u_long Words_disk;
+extern u_long Chars_disk;
+
+
+
+int LoadCompressionDictionary (char *dict_file_name);
Index: /trunk/gsdl/src/mgpp/text/hash.h
===================================================================
--- /trunk/gsdl/src/mgpp/text/hash.h	(revision 856)
+++ /trunk/gsdl/src/mgpp/text/hash.h	(revision 856)
@@ -0,0 +1,44 @@
+/**************************************************************************
+ *
+ * hash.h -- Hash function
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef H_HASH
+#define H_HASH
+
+
+
+#define HASH(h, step, wd, tablesize)			\
+	do {						\
+		register u_char *p=wd;			\
+		register long len = *p++;		\
+		register unsigned long hval;		\
+		if (len>25) len = 25;			\
+		for (hval=0; len; len--)		\
+		  hval = (hval << 5) - hval + *p++;	\
+		step = hval % (tablesize - 2) + 1;	\
+		h = hval % tablesize;			\
+	} while (0)
+
+
+
+#endif
Index: /trunk/gsdl/src/mgpp/text/invf.cpp
===================================================================
--- /trunk/gsdl/src/mgpp/text/invf.cpp	(revision 856)
+++ /trunk/gsdl/src/mgpp/text/invf.cpp	(revision 856)
@@ -0,0 +1,690 @@
+/**************************************************************************
+ *
+ * invf.cpp -- Data structures for inverted files
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "invf.h"
+#include "UCArray.h"
+
+
+invf_dict_header::invf_dict_header () {
+  Clear();
+}
+
+invf_dict_header::~invf_dict_header () {
+}
+
+void invf_dict_header::Clear() {
+  lookback = 0;
+  word_dict_start = 0;
+  word_dict_size = 0;
+  tag_dict_start = 0;
+  tag_dict_size = 0;
+  num_docs = 0;
+  num_frags = 0;
+  num_words = 0;
+  total_bytes = 0;
+  index_string_bytes = 0;
+  num_levels = 0;
+}
+
+bool invf_dict_header::Read (FILE *f) {
+  return (ReadUL (f, lookback) &&
+	  ReadUL (f, word_dict_start) &&
+	  ReadUL (f, word_dict_size) &&
+	  ReadUL (f, tag_dict_start) &&
+	  ReadUL (f, tag_dict_size) &&
+	  ReadUL (f, num_docs) &&
+	  ReadUL (f, num_frags) &&
+	  ReadUL (f, num_words) &&
+	  ReadUL (f, total_bytes) &&
+	  ReadUL (f, index_string_bytes) &&
+	  ReadUL (f, num_levels));
+}
+
+bool invf_dict_header::Write (FILE *f) const {
+  return (WriteUL (f, lookback) &&
+	  WriteUL (f, word_dict_start) &&
+	  WriteUL (f, word_dict_size) &&
+	  WriteUL (f, tag_dict_start) &&
+	  WriteUL (f, tag_dict_size) &&
+	  WriteUL (f, num_docs) &&
+	  WriteUL (f, num_frags) &&
+	  WriteUL (f, num_words) &&
+	  WriteUL (f, total_bytes) &&
+	  WriteUL (f, index_string_bytes) &&
+	  WriteUL (f, num_levels));
+}
+
+
+
+void dict_el::Clear () {
+  el.erase (el.begin(), el.end());
+  frag_occur = 0;
+  freq = 0;
+}
+
+bool dict_el::Read (FILE *f) {
+  return (ReadPreSufStr (f, el) &&
+	  ReadUL (f, frag_occur) &&
+	  ReadUL (f, freq));
+}
+
+bool dict_el::Write (FILE *f, const UCArray *lastEl) const {
+  return (WritePreSufStr (f, lastEl, el) &&
+	  WriteUL (f, frag_occur) &&
+	  WriteUL (f, freq));
+}
+
+
+void word_dict_el::Clear () {
+  dict_el::Clear();
+  if (levelFreqs != NULL) delete [] levelFreqs;
+  levelFreqs = NULL;
+}
+
+word_dict_el::~word_dict_el () {
+  if (levelFreqs != NULL) delete [] levelFreqs;
+}
+
+void word_dict_el::SetNumLevels (unsigned long numLevels) {
+  if (levelFreqs != NULL) delete [] levelFreqs;
+  levelFreqs = new unsigned long [numLevels];
+}
+  
+bool word_dict_el::Read (FILE *f, unsigned long numLevels) {
+  if (!dict_el::Read (f)) return false;
+
+  if (levelFreqs == NULL) return false;
+  
+  unsigned long i;
+  for (i=0; i<numLevels; i++) {
+    if (!ReadUL (f, levelFreqs[i])) return false;
+  }
+
+  return true;
+}
+
+bool word_dict_el::Write (FILE *f, const UCArray *lastEl,
+			  unsigned long numLevels) const {
+  if (!dict_el::Write (f, lastEl)) return false;
+
+  if (levelFreqs == NULL) return false;
+
+  unsigned long i;
+  for (i=0; i<numLevels; i++) {
+    if (!WriteUL (f, levelFreqs[i])) return false;
+  }
+
+  return true;
+}
+
+
+
+block_dict_header::block_dict_header () {
+  Clear();
+}
+
+void block_dict_header::Clear () {
+  invf_dict_header::Clear();
+
+  entries_per_wblk = 0;
+  num_wblks = 0;
+  max_wblk_size = 0;
+  wblk_start = 0;
+  wblk_idx_start = 0;
+  
+  entries_per_tblk = 0;
+  num_tblks = 0;
+  max_tblk_size = 0;
+  tblk_start = 0;
+  tblk_idx_start = 0;
+}
+
+bool block_dict_header::Read (FILE *f) {
+  return (invf_dict_header::Read (f) &&
+
+	  ReadUL (f, entries_per_wblk) &&
+	  ReadUL (f, num_wblks) &&
+	  ReadUL (f, max_wblk_size) &&
+	  ReadUL (f, wblk_start) &&
+	  ReadUL (f, wblk_idx_start) &&
+  
+	  ReadUL (f, entries_per_tblk) &&
+	  ReadUL (f, num_tblks) &&
+	  ReadUL (f, max_tblk_size) &&
+	  ReadUL (f, tblk_start) &&
+	  ReadUL (f, tblk_idx_start));
+}
+
+bool block_dict_header::Write (FILE *f) const {
+  return (invf_dict_header::Write (f) &&
+
+	  WriteUL (f, entries_per_wblk) &&
+	  WriteUL (f, num_wblks) &&
+	  WriteUL (f, max_wblk_size) &&
+	  WriteUL (f, wblk_start) &&
+	  WriteUL (f, wblk_idx_start) &&
+  
+	  WriteUL (f, entries_per_tblk) &&
+	  WriteUL (f, num_tblks) &&
+	  WriteUL (f, max_tblk_size) &&
+	  WriteUL (f, tblk_start) &&
+	  WriteUL (f, tblk_idx_start));
+}
+
+
+
+
+
+void block_dict_el::Clear () {
+  el.erase (el.begin(), el.end());
+  frag_occur = 0;
+  freq = 0;
+  invf_ptr = 0;
+}
+
+bool block_dict_el::Read (FILE *f) {
+  return (ReadPreSufStr (f, el) &&
+	  ReadUL (f, frag_occur) &&
+	  ReadUL (f, freq) &&
+	  ReadUL (f, invf_ptr));
+}
+
+bool block_dict_el::Write (FILE *f, const UCArray *lastEl) const {
+  return (WritePreSufStr (f, lastEl, el) &&
+	  WriteUL (f, frag_occur) &&
+	  WriteUL (f, freq) &&
+	  WriteUL (f, invf_ptr));
+}
+
+
+
+
+
+void word_block_dict_el::Clear () {
+  block_dict_el::Clear();
+  if (levelFreqs != NULL) delete [] levelFreqs;
+  levelFreqs = NULL;
+}
+
+word_block_dict_el::~word_block_dict_el () {
+  if (levelFreqs != NULL) delete [] levelFreqs;
+}
+
+void word_block_dict_el::SetNumLevels (unsigned long numLevels) {
+  if (levelFreqs != NULL) delete [] levelFreqs;
+  levelFreqs = new unsigned long [numLevels];
+}
+
+bool word_block_dict_el::Read (FILE *f, unsigned long numLevels) {
+  if (!block_dict_el::Read (f)) return false;
+  
+  if (levelFreqs == NULL) return false;
+  
+  unsigned long i;
+  for (i=0; i<numLevels; i++) {
+    if (!ReadUL (f, levelFreqs[i])) return false;
+  }
+
+  return true;
+}
+
+bool word_block_dict_el::Write (FILE *f, const UCArray *lastEl,
+				unsigned long numLevels) const {
+  if (!block_dict_el::Write (f, lastEl)) return false;
+  
+  if (levelFreqs == NULL) return false;
+  
+  unsigned long i;
+  for (i=0; i<numLevels; i++) {
+    if (!WriteUL (f, levelFreqs[i])) return false;
+  }
+  
+  return true;
+}
+
+
+
+
+block_idx_info::block_idx_info () {
+  Clear ();
+}
+
+void block_idx_info::Clear () {
+  el.erase (el.begin(), el.end());
+  block_ptr = 0;
+}
+
+bool block_idx_info::Read (FILE *f) {
+  return (ReadUCArray (f, el) &&
+	  ReadUL (f, block_ptr));
+}
+
+bool block_idx_info::Write (FILE *f) const {
+  return (WriteUCArray (f, el) &&
+	  WriteUL (f, block_ptr));
+}
+
+
+bool ReadBlockIdx (FILE *f, block_idx &blockIdx) {
+  blockIdx.erase (blockIdx.begin(), blockIdx.end());
+  
+  // read in the array size
+  unsigned long arraySize = 0;
+  if (!ReadVarLenUL (f, arraySize)) return false;
+  
+  // read in the array
+  block_idx_info bi;
+  while (arraySize > 0) {
+    if (!bi.Read (f)) return false;
+    blockIdx.push_back (bi);
+    
+    arraySize--;
+  }
+  
+  return true;
+}
+
+bool WriteBlockIdx (FILE *f, const block_idx &blockIdx) {
+  // write out the array size
+  if (!WriteVarLenUL (f, blockIdx.size())) return false;
+
+  block_idx::const_iterator here = blockIdx.begin();
+  block_idx::const_iterator end = blockIdx.end();
+  while (here != end) {
+    if (!(*here).Write (f)) return false;
+    here++;
+  }
+
+  return true;
+}
+
+
+
+
+stem_idx_header::stem_idx_header () {
+  Clear ();
+}
+
+void stem_idx_header::Clear () {
+  lookback = 0;
+  dict_size = 0;
+
+  entries_per_block = 0;
+  num_blocks = 0;
+  max_block_size = 0;
+  blocks_start = 0;
+  block_idx_start = 0;
+  
+  stemmer_num = 0;
+  stem_method = 0;
+}
+
+bool stem_idx_header::Read (FILE *f) {
+  return (ReadUL (f, lookback) &&
+	  ReadUL (f, dict_size) &&
+	  
+	  ReadUL (f, entries_per_block) &&
+	  ReadUL (f, num_blocks) &&
+	  ReadUL (f, max_block_size) &&
+	  ReadUL (f, blocks_start) &&
+	  ReadUL (f, block_idx_start) &&
+	  
+	  ReadUL (f, stemmer_num) &&
+	  ReadUL (f, stem_method));
+}
+
+bool stem_idx_header::Write (FILE *f) const {
+  return (WriteUL (f, lookback) &&
+	  WriteUL (f, dict_size) &&
+	  
+	  WriteUL (f, entries_per_block) &&
+	  WriteUL (f, num_blocks) &&
+	  WriteUL (f, max_block_size) &&
+	  WriteUL (f, blocks_start) &&
+	  WriteUL (f, block_idx_start) &&
+	  
+	  WriteUL (f, stemmer_num) &&
+	  WriteUL (f, stem_method));
+}
+
+
+
+stem_block_dict_el::stem_block_dict_el () {
+  Clear ();
+}
+
+void stem_block_dict_el::Clear () {
+  el.erase (el.begin(), el.end());
+  equivWords.erase (equivWords.begin(), equivWords.end());
+}
+
+bool stem_block_dict_el::Read (FILE *f) {
+  equivWords.erase (equivWords.begin(), equivWords.end());
+  
+  if (!ReadPreSufStr (f, el)) return false;
+  
+  // read in the array size
+  unsigned long arraySize = 0;
+  if (!ReadVarLenUL (f, arraySize)) return false;
+  
+  // read in the array
+  unsigned long wordNum;
+  while (arraySize > 0) {
+    if (!ReadUL (f, wordNum)) return false;
+    equivWords.push_back (wordNum);
+    
+    arraySize--;
+  }
+  
+  return true;
+}
+
+bool stem_block_dict_el::Write (FILE *f, const UCArray *lastEl) const {
+  if (!WritePreSufStr (f, lastEl, el)) return false;
+
+  // write out the array size
+  if (!WriteVarLenUL (f, equivWords.size())) return false;
+
+  vector<unsigned long>::const_iterator here = equivWords.begin();
+  vector<unsigned long>::const_iterator end = equivWords.end();
+  while (here != end) {
+    if (!WriteUL (f, (*here))) return false;
+    here++;
+  }
+
+  return true;
+}
+
+
+
+
+invf_file_header::invf_file_header () {
+  Clear ();
+}
+
+void invf_file_header::Clear () {
+  no_of_words = 0;
+  no_of_tags = 0;
+  skip_mode = SKIP_MODE_NO_SKIPS;
+  word_level_index = 0;
+  
+  int i;
+  for (i=0; i<16; i++) params[i] = 0;
+}
+  
+bool invf_file_header::Read (FILE *f) {
+  if (!ReadUL (f, no_of_words) ||
+      !ReadUL (f, no_of_tags) ||
+      !ReadUL (f, skip_mode) ||
+      !ReadUL (f, word_level_index)) return false;
+
+  int i;
+  for (i=0; i<16; i++) {
+    if (!ReadUL (f, params[i])) return false;
+  }
+
+  return true;
+}
+
+bool invf_file_header::Write (FILE *f) const {
+  if (!WriteUL (f, no_of_words) ||
+      !WriteUL (f, no_of_tags) ||
+      !WriteUL (f, skip_mode) ||
+      !WriteUL (f, word_level_index)) return false;
+
+  int i;
+  for (i=0; i<16; i++) {
+    if (!WriteUL (f, params[i])) return false;
+  }
+
+  return true;
+}
+
+
+
+
+
+bool SearchElNum (const block_idx &bIdx,
+		  unsigned long entriesPerBlock,
+		  unsigned long elNum,
+		  unsigned long &blockIdxNum,
+		  unsigned long &blockStartElNum) {
+  blockIdxNum = 0;
+  blockStartElNum = 0;
+
+  // make sure the element number isn't out of range
+  if (elNum > bIdx.size()*entriesPerBlock) return false;
+
+  blockIdxNum = elNum / entriesPerBlock;
+  blockStartElNum = blockIdxNum * entriesPerBlock;
+  
+  return true;
+}
+
+bool SearchEl (const block_idx &bIdx,
+	       unsigned long entriesPerBlock,
+	       const UCArray &el,
+	       unsigned long &blockIdxNum,
+	       unsigned long &blockStartElNum) {
+  blockIdxNum = 0;
+  blockStartElNum = 0;
+
+  unsigned long begin = 0;
+  unsigned long bIdxEnd = bIdx.size();
+  unsigned long end = bIdxEnd;
+  unsigned long mid;
+  while (begin < end) {
+    mid = (begin+end)/2;
+    if (DictCompare (el, bIdx[mid].el) < 0) {
+      end = mid;
+    } else if (mid+1 < bIdxEnd &&
+	       DictCompare (el, bIdx[mid+1].el) >= 0) {
+      begin = mid+1;
+    } else {
+      blockIdxNum = mid;
+      blockStartElNum = blockIdxNum * entriesPerBlock;
+      return true;
+    }
+  }
+  
+  return false;
+}
+
+
+
+
+bool SearchBlockDictElNum (FILE *dictFile,
+			   const block_idx &bIdx,
+			   unsigned long entriesPerBlock,
+			   unsigned long dictSize,
+			   unsigned long elNum,
+			   block_dict_el &dictEl) {
+  UCArrayClear (dictEl.el);
+  if (elNum >= dictSize) return false;
+
+  // find the block that contains the element
+  unsigned long blockIdxNum, curElNum;
+  if (!SearchElNum (bIdx, entriesPerBlock, elNum,
+		    blockIdxNum, curElNum))
+    return false;
+
+  // look for the block
+  fseek (dictFile, bIdx[blockIdxNum].block_ptr, SEEK_SET);
+  do {
+    dictEl.Read (dictFile);
+  } while (curElNum++ < elNum);
+
+  return true;
+}
+
+bool SearchBlockDictEl (FILE *dictFile,
+			const block_idx &bIdx,
+			unsigned long entriesPerBlock,
+			unsigned long dictSize,
+			const UCArray &el,
+			block_dict_el &dictEl,
+			unsigned long &elNum) {
+  UCArrayClear (dictEl.el);
+
+  // find the block that contains the element
+  unsigned long blockIdxNum;
+  if (!SearchEl (bIdx, entriesPerBlock, el,
+		 blockIdxNum, elNum))
+    return false;
+
+  unsigned long blockEndElNum = elNum + entriesPerBlock;
+  if (blockEndElNum > dictSize) blockEndElNum = dictSize;
+
+  // look for the block
+  fseek (dictFile, bIdx[blockIdxNum].block_ptr, SEEK_SET);
+  while (elNum < blockEndElNum) {
+    dictEl.Read (dictFile);
+    int res = DictCompare (dictEl.el, el);
+    if (res == 0) return true; // found it
+    else if (res > 0) break; // not here
+    
+    elNum++;
+  }
+
+  return false;
+}
+
+
+
+
+bool SearchWordBlockDictElNum (FILE *dictFile,
+			       const block_idx &bIdx,
+			       unsigned long entriesPerBlock,
+			       unsigned long dictSize,
+			       unsigned long numLevels,
+			       unsigned long elNum,
+			       word_block_dict_el &dictEl) {
+  UCArrayClear (dictEl.el);
+  if (elNum >= dictSize) return false;
+
+  // find the block that contains the element
+  unsigned long blockIdxNum, curElNum;
+  if (!SearchElNum (bIdx, entriesPerBlock, elNum,
+		    blockIdxNum, curElNum))
+    return false;
+
+  // look for the block
+  fseek (dictFile, bIdx[blockIdxNum].block_ptr, SEEK_SET);
+  do {
+    dictEl.Read (dictFile, numLevels);
+  } while (curElNum++ < elNum);
+
+  return true;
+}
+
+bool SearchWordBlockDictEl (FILE *dictFile,
+			    const block_idx &bIdx,
+			    unsigned long entriesPerBlock,
+			    unsigned long dictSize,
+			    unsigned long numLevels,
+			    const UCArray &el,
+			    word_block_dict_el &dictEl,
+			    unsigned long &elNum) {
+  UCArrayClear (dictEl.el);
+
+  // find the block that contains the element
+  unsigned long blockIdxNum;
+  if (!SearchEl (bIdx, entriesPerBlock, el,
+		 blockIdxNum, elNum))
+    return false;
+
+  unsigned long blockEndElNum = elNum + entriesPerBlock;
+  if (blockEndElNum > dictSize) blockEndElNum = dictSize;
+
+  // look for the block
+  fseek (dictFile, bIdx[blockIdxNum].block_ptr, SEEK_SET);
+  while (elNum < blockEndElNum) {
+    dictEl.Read (dictFile, numLevels);
+    int res = DictCompare (dictEl.el, el);
+    if (res == 0) return true; // found it
+    else if (res > 0) break; // not here
+    
+    elNum++;
+  }
+
+  return false;
+}
+
+
+
+
+bool SearchStemBlockDictElNum (FILE *dictFile,
+			       const block_idx &bIdx,
+			       unsigned long entriesPerBlock,
+			       unsigned long dictSize,
+			       unsigned long elNum,
+			       stem_block_dict_el &dictEl) {
+  UCArrayClear (dictEl.el);
+  if (elNum >= dictSize) return false;
+
+  // find the block that contains the element
+  unsigned long blockIdxNum, curElNum;
+  if (!SearchElNum (bIdx, entriesPerBlock, elNum,
+		    blockIdxNum, curElNum))
+    return false;
+
+  // look for the block
+  fseek (dictFile, bIdx[blockIdxNum].block_ptr, SEEK_SET);
+  do {
+    dictEl.Read (dictFile);
+  } while (curElNum++ < elNum);
+
+  return true;
+}
+
+bool SearchStemBlockDictEl (FILE *dictFile,
+			    const block_idx &bIdx,
+			    unsigned long entriesPerBlock,
+			    unsigned long dictSize,
+			    const UCArray &el,
+			    stem_block_dict_el &dictEl,
+			    unsigned long &elNum) {
+  UCArrayClear (dictEl.el);
+
+  // find the block that contains the element
+  unsigned long blockIdxNum;
+  if (!SearchEl (bIdx, entriesPerBlock, el,
+		 blockIdxNum, elNum))
+    return false;
+
+  unsigned long blockEndElNum = elNum + entriesPerBlock;
+  if (blockEndElNum > dictSize) blockEndElNum = dictSize;
+
+  // look for the block
+  fseek (dictFile, bIdx[blockIdxNum].block_ptr, SEEK_SET);
+  while (elNum < blockEndElNum) {
+    dictEl.Read (dictFile);
+    int res = DictCompare (dictEl.el, el);
+    if (res == 0) return true; // found it
+    else if (res > 0) break; // not here
+    
+    elNum++;
+  }
+
+  return false;
+}
Index: /trunk/gsdl/src/mgpp/text/invf.h
===================================================================
--- /trunk/gsdl/src/mgpp/text/invf.h	(revision 856)
+++ /trunk/gsdl/src/mgpp/text/invf.h	(revision 856)
@@ -0,0 +1,292 @@
+/**************************************************************************
+ *
+ * invf.h -- Data structures for inverted files
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef H_INVF
+#define H_INVF
+
+#include <stdio.h>
+
+#include "UCArray.h"
+
+// NOTE: This does not include the magic number
+struct invf_dict_header {
+  unsigned long lookback;
+  unsigned long word_dict_start;
+  unsigned long word_dict_size;
+  unsigned long tag_dict_start;
+  unsigned long tag_dict_size;
+  unsigned long num_docs;
+  unsigned long num_frags;
+  unsigned long num_words;
+  unsigned long total_bytes;
+  unsigned long index_string_bytes;
+  unsigned long num_levels;
+
+  invf_dict_header ();
+  virtual ~invf_dict_header ();
+  virtual void Clear();
+
+  virtual bool Read (FILE *f);
+  virtual bool Write (FILE *f) const;
+};
+
+
+struct dict_el {
+  UCArray el; // word or tag
+  unsigned long frag_occur;
+  unsigned long freq;
+
+  virtual void Clear ();
+  dict_el () { Clear (); }
+  virtual ~dict_el () { }
+
+  // Read assumes that the last word is in el
+  bool Read (FILE *f);
+  bool Write (FILE *f, const UCArray *lastEl) const;
+};
+
+
+struct word_dict_el : public dict_el {
+  unsigned long *levelFreqs;
+
+  void Clear ();
+  word_dict_el () { levelFreqs = NULL; Clear (); }
+  ~word_dict_el ();
+
+  void SetNumLevels (unsigned long numLevels);
+
+  // SetNumLevels should be called before either
+  // reading or writing using Read and Write
+  
+  // Read assumes that the last word is in el
+  bool Read (FILE *f, unsigned long numLevels);
+  bool Write (FILE *f, const UCArray *lastEl,
+	      unsigned long numLevels) const;
+};
+
+
+// wblk = word block
+// tblk = tag block
+// this version of the blocked dictionary uses a fixed number
+// of entries per block, not a fixed block size
+struct block_dict_header : public invf_dict_header {
+  // note: word_dict_start and tag_dict_start are undefined
+  // for blocked dictionaries
+
+  unsigned long entries_per_wblk;
+  unsigned long num_wblks;
+  unsigned long max_wblk_size;
+  unsigned long wblk_start;
+  unsigned long wblk_idx_start;
+  
+  unsigned long entries_per_tblk;
+  unsigned long num_tblks;
+  unsigned long max_tblk_size;
+  unsigned long tblk_start;
+  unsigned long tblk_idx_start;
+  
+  block_dict_header ();
+  void Clear ();
+  
+  bool Read (FILE *f);
+  bool Write (FILE *f) const;
+};
+
+
+struct block_dict_el {
+  UCArray el; // word or tag
+  unsigned long frag_occur; // # entries in invf file
+  unsigned long freq;
+  unsigned long invf_ptr;
+
+  virtual void Clear ();
+  block_dict_el () { Clear (); }
+  virtual ~block_dict_el () { }
+
+  // Read assumes that the last word is in el
+  // set lastEl = NULL when no lookback is wanted (eg
+  // for the start of a block
+  bool Read (FILE *f);
+  bool Write (FILE *f, const UCArray *lastEl) const;
+};
+
+struct word_block_dict_el : public block_dict_el {
+  unsigned long *levelFreqs;
+
+  void Clear ();
+  word_block_dict_el () { levelFreqs = NULL; Clear (); }
+  ~word_block_dict_el ();
+
+  void SetNumLevels (unsigned long numLevels);
+
+  // SetNumLevels should be called before either
+  // reading or writing using Read and Write
+  
+  // Read assumes that the last word is in el
+  bool Read (FILE *f, unsigned long numLevels);
+  bool Write (FILE *f, const UCArray *lastEl,
+	      unsigned long numLevels) const;
+};
+
+
+
+struct block_idx_info {
+  UCArray el;
+  unsigned long block_ptr;
+
+  block_idx_info ();
+  void Clear ();
+
+  bool Read (FILE *f);
+  bool Write (FILE *f) const;
+};
+
+// used for an index into the word and tag blocks
+typedef vector<block_idx_info> block_idx;
+
+bool ReadBlockIdx (FILE *f, block_idx &blockIdx);
+bool WriteBlockIdx (FILE *f, const block_idx &blockIdx);
+
+
+
+struct stem_idx_header {
+  unsigned long lookback;
+  unsigned long dict_size;
+
+  unsigned long entries_per_block;
+  unsigned long num_blocks;
+  unsigned long max_block_size;
+  unsigned long blocks_start;
+  unsigned long block_idx_start;
+  
+  unsigned long stemmer_num;
+  unsigned long stem_method;
+
+  stem_idx_header ();
+  void Clear ();
+
+  bool Read (FILE *f);
+  bool Write (FILE *f) const;
+};
+
+struct stem_block_dict_el {
+  UCArray el; // word or tag
+  vector<unsigned long> equivWords;
+  
+  stem_block_dict_el ();
+  void Clear ();
+
+  // Read assumes that the last word is in el
+  // set lastEl = NULL when no lookback is wanted (eg
+  // for the start of a block
+  bool Read (FILE *f);
+  bool Write (FILE *f, const UCArray *lastEl) const;
+};
+
+
+
+#define SKIP_MODE_NO_SKIPS 0
+
+struct invf_file_header {
+  unsigned long no_of_words;
+  unsigned long no_of_tags;
+  unsigned long skip_mode;
+  unsigned long word_level_index; // 1 if word level index
+  unsigned long params[16];
+
+  invf_file_header ();
+  void Clear ();
+  
+  bool Read (FILE *f);
+  bool Write (FILE *f) const;
+};
+
+
+
+
+
+
+// the search functions returns true if a block that could
+// satisfy the request is found. these functions assume that
+// the block index is sorted by DictCompare (or DictLTUCArray)
+bool SearchElNum (const block_idx &bIdx,
+		  unsigned long entriesPerBlock,
+		  unsigned long elNum,
+		  unsigned long &blockIdxNum,
+		  unsigned long &blockStartElNum);
+bool SearchEl (const block_idx &bIdx,
+	       unsigned long entriesPerBlock,
+	       const UCArray &el,
+	       unsigned long &blockIdxNum,
+	       unsigned long &blockStartElNum);
+	       
+bool SearchBlockDictElNum (FILE *dictFile,
+			   const block_idx &bIdx,
+			   unsigned long entriesPerBlock,
+			   unsigned long dictSize,
+			   unsigned long elNum,
+			   block_dict_el &dictEl);
+bool SearchBlockDictEl (FILE *dictFile,
+			const block_idx &bIdx,
+			unsigned long entriesPerBlock,
+			unsigned long dictSize,
+			const UCArray &el,
+			block_dict_el &dictEl,
+			unsigned long &elNum);
+
+// assumes the numLevels has been set for dictEl
+bool SearchWordBlockDictElNum (FILE *dictFile,
+			       const block_idx &bIdx,
+			       unsigned long entriesPerBlock,
+			       unsigned long dictSize,
+			       unsigned long numLevels,
+			       unsigned long elNum,
+			       word_block_dict_el &dictEl);
+bool SearchWordBlockDictEl (FILE *dictFile,
+			    const block_idx &bIdx,
+			    unsigned long entriesPerBlock,
+			    unsigned long dictSize,
+			    unsigned long numLevels,
+			    const UCArray &el,
+			    word_block_dict_el &dictEl,
+			    unsigned long &elNum);
+
+bool SearchStemBlockDictElNum (FILE *dictFile,
+			       const block_idx &bIdx,
+			       unsigned long entriesPerBlock,
+			       unsigned long dictSize,
+			       unsigned long elNum,
+			       stem_block_dict_el &dictEl);
+bool SearchStemBlockDictEl (FILE *dictFile,
+			    const block_idx &bIdx,
+			    unsigned long entriesPerBlock,
+			    unsigned long dictSize,
+			    const UCArray &el,
+			    stem_block_dict_el &dictEl,
+			    unsigned long &elNum);
+
+
+
+
+#endif
Index: /trunk/gsdl/src/mgpp/text/ivf.pass1.cpp
===================================================================
--- /trunk/gsdl/src/mgpp/text/ivf.pass1.cpp	(revision 856)
+++ /trunk/gsdl/src/mgpp/text/ivf.pass1.cpp	(revision 856)
@@ -0,0 +1,789 @@
+/**************************************************************************
+ *
+ * ivf.pass1.cpp -- Memory efficient pass 1 inversion
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "mg_files.h"
+#include "invf.h"
+#include "mg.h"
+#include "build.h"
+#include "locallib.h"
+#include "UCArray.h"
+#include "bitio_m_stdio.h"
+#include "bitio_gen.h"
+#include <stdio.h>
+#include "words.h"
+#include "messages.h"
+#include "netorder.h"
+#include "FIvfLevelInfo.h"
+
+#include "longlong.h"
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\map>
+#elif defined(GSDL_USE_STL_H)
+#  include <map.h>
+#else
+#  include <map>
+#endif
+
+
+/*
+   $Log$
+   Revision 1.1  2000/01/14 02:26:07  sjboddie
+   Rodgers new C++ mg
+
+   *
+ */
+
+
+
+// a fragment corresponds to the index level (word-level is the
+// minimum index level)
+
+
+// structure to determine level information
+struct LevelWorker {
+  unsigned long lastLevelDocNum;
+  unsigned long count;
+
+  LevelWorker () {
+    lastLevelDocNum = count = 0;
+  }
+};
+
+
+// note: the word is stored in the map
+struct IvfWordInfo {
+  unsigned long wordCount;      // word frequency
+  unsigned long fragCount;      // number of fragments that contain the word
+  unsigned long lastFragNum;    // last fragment to contain the word
+  unsigned long chunkWordCount; // word frequency within this chunk
+  unsigned long chunkFragCount; // number of fragments within this chunk that
+                                //   contain the word
+
+  LevelWorker *levels;          // level info for this word
+  
+  IvfWordInfo ();
+  ~IvfWordInfo ();
+  void Clear ();                // will delete levels
+};
+
+typedef map<UCArray, IvfWordInfo, DictLTUCArray> IvfWordInfoMap;
+typedef vector<IvfWordInfoMap::iterator> IvfWordInfoItArray;
+
+// tags don't require as much information
+struct IvfTagInfo {
+  unsigned long tagCount;       // tag frequency
+  unsigned long fragCount;      // number of fragments that contain the tag
+  unsigned long lastFragNum;    // last fragment to contain the tag
+  unsigned long chunkFragCount; // number of fragments within this chunk that
+                                //   contain the tag
+
+  IvfTagInfo ();
+  void Clear ();
+};
+
+typedef map<UCArray, IvfTagInfo, DictLTUCArray> IvfTagInfoMap;
+typedef vector<IvfTagInfoMap::iterator> IvfTagInfoItArray;
+
+
+#define INIT_CHECK_FRAC 0.10
+#define CHECK_FRAC 0.75
+#define CHECK_CLOSE 0.999
+#define CHECK_DIV 1.5
+
+
+static FILE *ic;		// the invf chunk file
+static stdio_bitio_buffer icb;
+
+IvfWordInfoMap ivfWordInfo;
+IvfWordInfoItArray ivfWordInfoOccurOrder;
+IvfTagInfoMap ivfTagInfo;
+IvfTagInfoItArray ivfTagInfoOccurOrder;
+
+static unsigned long chunksWritten;
+static unsigned long maxMemNeeded;
+
+static unsigned long numDocs;
+static unsigned long numChunkDocs;
+
+static unsigned long numFrags;
+static unsigned long numChunkFrags;
+
+static unsigned long numWords;
+
+// the number of document numbers in the inverted file
+static unsigned long numChunkEntries;
+
+// next entry in the inverted file to check memory
+// requirements for the current chunk
+static unsigned long entryCheckPoint;
+
+// information about all the different levels
+static FIvfLevel ivfLevel;
+
+
+
+IvfWordInfo::IvfWordInfo () {
+  levels = NULL;
+  Clear();
+}
+
+IvfWordInfo::~IvfWordInfo () {
+  if (levels != NULL) delete [] levels;
+}
+
+void IvfWordInfo::Clear () {
+  wordCount = 0;
+  chunkWordCount = 0;
+  lastFragNum = 0;
+  fragCount = 0;
+  chunkFragCount = 0;
+
+  if (levels != NULL) {
+    delete [] levels;
+    levels = NULL;
+  }
+}
+
+
+IvfTagInfo::IvfTagInfo () {
+  Clear();
+}
+
+void IvfTagInfo::Clear () {
+  tagCount = 0;
+  lastFragNum = 0;
+  fragCount = 0;
+  chunkFragCount = 0;
+}
+
+
+
+int init_ivf_1 (const TagInfo &tagInfo, char *file_name) {
+  // set up the chunk file
+  if (!(ic = create_file (file_name, INVF_CHUNK_SUFFIX, "wb",
+			  MAGIC_CHUNK, MG_MESSAGE)))
+    return COMPERROR;
+  fwrite ("    ", sizeof (u_long), 1, ic);	// Space for the maxmem
+  icb.attachFile (ic);
+  icb.encodeStart();
+
+  // reset global variables
+  ivfWordInfo.erase (ivfWordInfo.begin(), ivfWordInfo.end());
+  ivfWordInfoOccurOrder.erase (ivfWordInfoOccurOrder.begin(), ivfWordInfoOccurOrder.end());
+  ivfTagInfo.erase (ivfTagInfo.begin(), ivfTagInfo.end());
+  ivfTagInfoOccurOrder.erase (ivfTagInfoOccurOrder.begin(), ivfTagInfoOccurOrder.end());
+  
+  chunksWritten = 0;
+  maxMemNeeded = 0;
+ 
+  numDocs = 0;
+  numChunkDocs = 0;
+
+  numFrags = 0;
+  numChunkFrags = 0;
+
+  numWords = 0;
+
+  numChunkEntries = 0;
+  entryCheckPoint = (unsigned long) ((invf_buffer_size * INIT_CHECK_FRAC) / CHECK_DIV);
+
+  // init the level information
+  ivfLevel.Clear();
+  ivfLevel.docTag = tagInfo.docTag;
+  ivfLevel.indexLevel = tagInfo.indexLevel;
+  IvfLevelInfo blankLevel;
+  UCArraySet::const_iterator levelHere = tagInfo.levelTags.begin();
+  UCArraySet::const_iterator levelEnd = tagInfo.levelTags.end();
+  while (levelHere != levelEnd) {
+    blankLevel.levelTag = *levelHere;
+    ivfLevel.levelInfo[*levelHere] = blankLevel;
+    levelHere++;
+  }
+  
+  return COMPALLOK;
+}
+
+static void ProcessOpenTag (const TagInfo &tagInfo, const TextEl &el,
+			    bool &inFrag) {
+  bool wordLevelIndex = tagInfo.indexLevel.empty();
+
+  // check for start of next fragment
+  if (!wordLevelIndex && el.tagName == tagInfo.indexLevel) {
+    numFrags++;
+    numChunkFrags++;
+    inFrag = true;
+  }
+  
+  // update tag stats
+  IvfTagInfo &i = ivfTagInfo[el.tagName];
+  if (i.tagCount == 0) {
+    // new tag, add to list of iterators
+    IvfTagInfoMap::iterator iIt = ivfTagInfo.find (el.tagName);
+    ivfTagInfoOccurOrder.push_back (iIt);
+  }
+  i.tagCount++;
+
+  // all open tags count as new tags
+  numChunkEntries++;
+  i.fragCount++;
+  i.chunkFragCount++;
+  i.lastFragNum = numFrags;
+  
+  // update level information
+  IvfLevelInfoMap::iterator levelIt = ivfLevel.levelInfo.find (el.tagName);
+  if (levelIt != ivfLevel.levelInfo.end()) {
+    // is a level tag
+    (*levelIt).second.numEntries++;
+    (*levelIt).second.workInLevel = true;
+  }
+}
+
+static void ProcessCloseTag (const TagInfo &tagInfo, const TextEl &el,
+			     bool &inFrag) {
+  bool wordLevelIndex = tagInfo.indexLevel.empty();
+
+  // check for end of fragment
+  if (!wordLevelIndex && el.tagName == tagInfo.indexLevel) {
+    inFrag = false;
+  }
+
+  // update level information
+  IvfLevelInfoMap::iterator levelIt = ivfLevel.levelInfo.find (el.tagName);
+  if (levelIt != ivfLevel.levelInfo.end()) {
+    // is a level tag
+    (*levelIt).second.workInLevel = false;
+  }
+}
+
+static void ProcessText (const TagInfo &tagInfo, const TextEl &el,
+			 bool &inFrag) {
+  bool wordLevelIndex = tagInfo.indexLevel.empty();
+
+  // make sure this text is to be indexed
+  if (!wordLevelIndex && !inFrag) return;
+
+  const unsigned char *textHere = el.text.begin();
+  const unsigned char *textEnd = el.text.end() - 1;
+  UCArray word;
+  
+  if (!inaword (textHere, textEnd))
+    ParseNonindexWord (textHere, textEnd);
+
+  
+  // Alternately parse off words and non-words from the input
+  // Each token is then inserted into the set if it does
+  // not exist or has it's frequency count incremented if it does.
+
+  while (textHere <= textEnd) {
+    textHere = ParseIndexWord (textHere, textEnd, word);
+    textHere = ParseNonindexWord (textHere, textEnd);
+
+    if (!word.empty()) {
+      numWords++;
+
+      if (wordLevelIndex) {
+	numFrags++;
+	numChunkFrags++;
+      }
+      
+      // update word stats
+
+      IvfWordInfo &i = ivfWordInfo[word];
+      if (i.wordCount == 0) {
+	// new word
+	// add to list of iterators
+	IvfWordInfoMap::iterator iIt = ivfWordInfo.find (word);
+	ivfWordInfoOccurOrder.push_back (iIt);
+
+	// add level information array
+	if (ivfLevel.levelInfo.size() > 0) 
+	  i.levels = new LevelWorker [ivfLevel.levelInfo.size()];
+      }
+
+      i.wordCount++;
+      i.chunkWordCount++;
+      if (numFrags > i.lastFragNum) {
+	numChunkEntries++;
+	i.fragCount++;
+	i.chunkFragCount++;
+	i.lastFragNum = numFrags;
+      }
+
+      // update level information for this word
+      if (i.levels != NULL) {
+	IvfLevelInfoMap::iterator levelHere = ivfLevel.levelInfo.begin();
+	IvfLevelInfoMap::iterator levelEnd = ivfLevel.levelInfo.end();
+	LevelWorker *levelWorkerPtr = i.levels;
+	while (levelHere != levelEnd) {
+
+	  // check to make sure the level encompases this fragment
+	  if (!(*levelHere).second.workInLevel) {
+	    cerr << "Level tag <" << (*levelHere).first
+		 << "> does not encompass all fragments\n";
+	    exit (1);
+	  }
+	  
+	  if ((*levelHere).second.numEntries > (*levelWorkerPtr).lastLevelDocNum) {
+	    (*levelWorkerPtr).lastLevelDocNum = (*levelHere).second.numEntries;
+	    (*levelWorkerPtr).count ++;
+	  }
+	  
+	  levelHere++;
+	  levelWorkerPtr++;
+	}
+      }
+    }
+  }
+}
+
+
+static unsigned long MemoryRequired (bool wordLevelIndex) {
+  register unsigned long total = 0;
+
+  // add memory required for word entries
+  IvfWordInfoMap::const_iterator wordHere = ivfWordInfo.begin();
+  IvfWordInfoMap::const_iterator wordEnd = ivfWordInfo.end();
+  while (wordHere != wordEnd) {
+    register const IvfWordInfo &info = (*wordHere).second;
+    if (info.chunkFragCount > 0) {
+      total += BIO_Bblock_Bound (numChunkFrags, info.chunkFragCount);
+      if (!wordLevelIndex) {
+	total += info.chunkWordCount;
+      }
+    }
+    
+    wordHere++;
+  }
+
+  // add memory required for tag entries
+  IvfTagInfoMap::const_iterator tagHere = ivfTagInfo.begin();
+  IvfTagInfoMap::const_iterator tagEnd = ivfTagInfo.end();
+  while (tagHere != tagEnd) {
+    register const IvfTagInfo &info = (*tagHere).second;
+    if (info.chunkFragCount > 0) {
+      // two d entries for each frag entry
+      unsigned long pTag = info.chunkFragCount*2;
+      total += BIO_Bblock_Bound (numChunkFrags+pTag, pTag);
+    }
+    
+    tagHere++;
+  }
+  
+  total = (total + 7) >> 3;
+  return total;
+}
+
+
+/*
+static void PrintChunkInfo (unsigned long mem) {
+  cout << "Chunk Number: " << chunksWritten << "\n";
+  cout << "numChunkDocs " << numChunkDocs << "\n";
+  cout << "numChunkFrags " << numChunkFrags << "\n";
+  cout << "mem " << mem << "\n";
+  cout << "numWords " << ivfWordInfo.size() << "\n";
+  cout << "numTags " << ivfTagInfo.size() << "\n\n";
+
+  // output debug tag information in dictionary order
+  IvfTagInfoMap::iterator tagMapHere = ivfTagInfo.begin();
+  IvfTagInfoMap::iterator tagMapEnd = ivfTagInfo.end();
+  unsigned long tagNum = 0;
+  while (tagMapHere != tagMapEnd) {
+    cout << (*tagMapHere).first << " " << tagNum
+	 << " " << (*tagMapHere).second.chunkFragCount << "\n";
+    tagNum++;
+    tagMapHere++;
+  }
+}
+*/
+
+static void OutputChunkInfo (unsigned long mem, bool /*wordLevelIndex*/) {
+  chunksWritten++;
+
+  // sanity check
+  if (ivfWordInfo.size() != ivfWordInfoOccurOrder.size()) {
+    Message ("ERROR: Word information size mismatch: %u vs %u\n",
+	     (unsigned int)ivfWordInfo.size(),
+	     (unsigned int)ivfWordInfoOccurOrder.size());
+    exit (1);
+  }
+  if (ivfTagInfo.size() != ivfTagInfoOccurOrder.size()) {
+    Message ("ERROR: Tag information size mismatch: %u vs %u\n",
+	     (unsigned int)ivfTagInfo.size(),
+	     (unsigned int)ivfTagInfoOccurOrder.size());
+    exit (1);
+  }
+
+  icb.gamma_encode (numChunkDocs + 1, NULL);
+  icb.gamma_encode (numChunkFrags + 1, NULL);
+  icb.gamma_encode (mem + 1, NULL);
+  icb.gamma_encode (ivfWordInfo.size() + 1, NULL);
+  icb.gamma_encode (ivfTagInfo.size() + 1, NULL);
+
+  /*  PrintChunkInfo (mem);*/
+
+  // output word information in occurance order
+  IvfWordInfoItArray::iterator wordHere = ivfWordInfoOccurOrder.begin();
+  IvfWordInfoItArray::iterator wordEnd = ivfWordInfoOccurOrder.end();
+  while (wordHere != wordEnd) {
+    register IvfWordInfo &ivfWordInfo = (*(*wordHere)).second;
+    
+    icb.gamma_encode (ivfWordInfo.chunkWordCount + 1, NULL);
+    if (ivfWordInfo.chunkWordCount >= 2) {
+      icb.gamma_encode (ivfWordInfo.chunkFragCount, NULL);
+    }
+
+    ivfWordInfo.lastFragNum = 0;
+    ivfWordInfo.chunkWordCount = 0;
+    ivfWordInfo.chunkFragCount = 0;
+    
+    wordHere++;
+  }
+
+  // output tag information in occurance order
+  IvfTagInfoItArray::iterator tagHere = ivfTagInfoOccurOrder.begin();
+  IvfTagInfoItArray::iterator tagEnd = ivfTagInfoOccurOrder.end();
+  while (tagHere != tagEnd) {
+    register IvfTagInfo &ivfTagInfo = (*(*tagHere)).second;
+
+    icb.gamma_encode (ivfTagInfo.chunkFragCount + 1, NULL);
+
+    ivfTagInfo.lastFragNum = 0;
+    ivfTagInfo.chunkFragCount = 0;
+    
+    tagHere++;
+  }
+
+  numChunkDocs = 0;
+  numChunkFrags = 0;
+  numChunkEntries = 0;
+}
+
+
+int process_ivf_1 (const TagInfo &tagInfo, const TextElArray &doc) {
+  bool wordLevelIndex = tagInfo.indexLevel.empty();
+  bool inFrag = false;
+  if (wordLevelIndex) inFrag = true; // unconditional
+  
+  numDocs++;
+  numChunkDocs++;
+
+  // process each text element in this document
+  TextElArray::const_iterator here = doc.begin();
+  TextElArray::const_iterator end = doc.end();
+  while (here != end) {
+    if ((*here).elType == OpenTagE)
+      ProcessOpenTag (tagInfo, *here, inFrag);
+    else if ((*here).elType == CloseTagE)
+      ProcessCloseTag (tagInfo, *here, inFrag);
+    else if ((*here).elType == TextE)
+      ProcessText (tagInfo, *here, inFrag);
+    
+    here++;
+  }
+
+  // check the amount of memory needed for this chunk
+  if (numChunkEntries >= entryCheckPoint) {
+    unsigned long mem = MemoryRequired (wordLevelIndex);
+    if (mem >= invf_buffer_size * CHECK_CLOSE) {
+      if (mem > maxMemNeeded) maxMemNeeded = mem;
+      OutputChunkInfo (mem, wordLevelIndex);
+      entryCheckPoint = (unsigned long)
+	((invf_buffer_size * INIT_CHECK_FRAC) / CHECK_DIV);
+      
+    } else {
+      entryCheckPoint = (unsigned long)
+	(entryCheckPoint * ((CHECK_FRAC * (invf_buffer_size - mem)) / mem) +
+	 entryCheckPoint);
+      if (entryCheckPoint <= numChunkEntries)
+	entryCheckPoint = numChunkEntries + 1;
+    }
+  }
+  
+  return COMPALLOK;
+}
+
+
+static void CalcInvfDictSize (unsigned long &totalBytes,
+			      unsigned long &indexStringBytes) {
+  totalBytes = 0;	// The sum of the length of all words, including 
+			// the length byte
+  indexStringBytes = 0; // The amount of space required to store the 
+			// words in the diction, this takes into account 
+			// the prefixes
+  const UCArray *lastWord = NULL;
+  
+  // calculate size of word information
+  IvfWordInfoMap::iterator wordHere = ivfWordInfo.begin();
+  IvfWordInfoMap::iterator wordEnd = ivfWordInfo.end();
+  while (wordHere != wordEnd) {
+    unsigned long wordSize = (*wordHere).first.size();
+    totalBytes += wordSize + 1;
+    indexStringBytes += wordSize + 2;
+    if (lastWord != NULL)
+      indexStringBytes -= PrefixLen (*lastWord, (*wordHere).first);
+    lastWord = &((*wordHere).first);
+    
+    wordHere++;
+  }
+
+  // calculate size of tag information
+  lastWord = NULL;
+  IvfTagInfoMap::iterator tagHere = ivfTagInfo.begin();
+  IvfTagInfoMap::iterator tagEnd = ivfTagInfo.end();
+  while (tagHere != tagEnd) {
+    unsigned long tagSize = (*tagHere).first.size();
+    totalBytes += tagSize + 1;
+    indexStringBytes += tagSize + 2;
+    if (lastWord != NULL)
+      indexStringBytes -= PrefixLen (*lastWord, (*tagHere).first);
+    lastWord = &((*tagHere).first);
+    
+    tagHere++;
+  }
+}
+
+
+//      OutputInvfDict ():
+//                writes out the stemmed dictionary file
+//                in the following format 
+//                        lookback value (int)
+//                        totalbytes value (int)
+//                        indexstringbytes (int)
+//                        for each word 
+//                          wordlen (4 bits)
+//                          prefix match (4 bits)
+//                          word (wordlen bytes)
+//                          word frequency (int)
+//                          word count (int)
+//
+//        Accesses outside variables:     
+//
+//        Return value...:                
+static void OutputInvfDict (char *filename) {
+  // create the dictionary header
+  invf_dict_header idh;
+  idh.word_dict_size = ivfWordInfo.size();
+  idh.tag_dict_size = ivfTagInfo.size();
+  idh.num_docs = numDocs;
+  idh.num_frags = numFrags;
+  idh.num_words = numWords;
+  idh.num_levels = ivfLevel.levelInfo.size();
+  CalcInvfDictSize (idh.total_bytes, idh.index_string_bytes);
+
+  // create the inverted dictionary file
+  FILE *sp;
+  if (!(sp = create_file (filename, INVF_DICT_SUFFIX, "wb", MAGIC_STEM_BUILD,
+			  MG_MESSAGE)))
+    return;
+
+  // write out the dictionary header
+  if (!idh.Write (sp)) { fclose (sp); return; }
+
+  // remember where the word dictionary starts
+  idh.word_dict_start = ftell (sp);
+  
+  // output the word dictionary
+  const UCArray *lastWord = NULL;
+  IvfWordInfoMap::iterator wordHere = ivfWordInfo.begin();
+  IvfWordInfoMap::iterator wordEnd = ivfWordInfo.end();
+  while (wordHere != wordEnd) {
+    // get the prefix and suffix lengths
+    const UCArray &thisWord = (*wordHere).first;
+    WritePreSufStr (sp, lastWord, thisWord);
+
+    // output the number of fragments the word appeared in and the
+    // number of times the word appeared
+    WriteUL (sp, (*wordHere).second.fragCount);
+    WriteUL (sp, (*wordHere).second.wordCount);
+
+    // output frequency information for each level
+    // note that we are expecting every word to have
+    // level information
+    LevelWorker *lwHere = (*wordHere).second.levels;
+    LevelWorker *lwEnd = lwHere + idh.num_levels;
+    while (lwHere != lwEnd) {
+      WriteUL (sp, (*lwHere).count);
+      lwHere++;
+    }
+    
+    lastWord = &thisWord;
+    wordHere++;
+  }
+
+  // remember where the tag dictionary starts
+  idh.tag_dict_start = ftell (sp);
+  
+  // output the tag dictionary
+  const UCArray *lastTag = NULL;
+  IvfTagInfoMap::iterator tagHere = ivfTagInfo.begin();
+  IvfTagInfoMap::iterator tagEnd = ivfTagInfo.end();
+  while (tagHere != tagEnd) {
+    // get the prefix and suffix lengths
+    const UCArray &thisTag = (*tagHere).first;
+    WritePreSufStr (sp, lastTag, thisTag);
+
+    // output the number of fragments the tag appeared in and the
+    // number of times the tag appeared
+    WriteUL (sp, (*tagHere).second.fragCount);
+    WriteUL (sp, (*tagHere).second.tagCount);
+    
+    lastTag = &thisTag;
+    tagHere++;
+  }
+
+  // write out the updated header
+  fseek (sp, sizeof (u_long), SEEK_SET);
+  if (!idh.Write (sp)) { fclose (sp); return; }
+  
+  fclose (sp);
+}
+
+static void OutputLevelFile (char *filename) {
+  // create the level file
+  FILE *f;
+  if (!(f = create_file (filename, INVF_LEVEL_SUFFIX, "wb", MAGIC_INVF_LEVELS,
+			 MG_MESSAGE)))
+    return;
+
+  // write out the information
+  ivfLevel.Write (f);
+  
+  // close the file
+  fclose (f);
+}
+
+static void OutputTransFile (char *filename) {
+  // save the word number (in lastFragNum :-/)
+  int i = 0;
+  IvfWordInfoMap::iterator wordHere = ivfWordInfo.begin();
+  IvfWordInfoMap::iterator wordEnd = ivfWordInfo.end();
+  while (wordHere != wordEnd) {
+    (*wordHere).second.lastFragNum = i;
+    i++; wordHere++;
+  }
+
+  // save the tag number (in lastFragNum :-/)
+  i = 0;
+  IvfTagInfoMap::iterator tagHere = ivfTagInfo.begin();
+  IvfTagInfoMap::iterator tagEnd = ivfTagInfo.end();
+  while (tagHere != tagEnd) {
+    (*tagHere).second.lastFragNum = i;
+    i++; tagHere++;
+  }
+  
+  // create the translation file
+  FILE *f;
+  if (!(f = create_file (filename, INVF_CHUNK_TRANS_SUFFIX, "wb",
+			 MAGIC_CHUNK_TRANS, MG_MESSAGE)))
+    return;
+
+  stdio_bitio_buffer buffer(f);
+  buffer.encodeStart();
+
+  // write out the word translation table
+  unsigned long wordDictSize = ivfWordInfoOccurOrder.size();
+  IvfWordInfoItArray::iterator wordItHere = ivfWordInfoOccurOrder.begin();
+  IvfWordInfoItArray::iterator wordItEnd = ivfWordInfoOccurOrder.end();
+  unsigned long oN = 0;
+  while (wordItHere != wordItEnd) {
+    register IvfWordInfo &ivfWordInfo = (*(*wordItHere)).second;
+    buffer.binary_encode (ivfWordInfo.lastFragNum + 1, wordDictSize + 1, NULL);
+    oN++;
+    wordItHere++;
+  }
+
+  // write out the tag translation table
+  unsigned long tagDictSize = ivfTagInfoOccurOrder.size();
+  IvfTagInfoItArray::iterator tagItHere = ivfTagInfoOccurOrder.begin();
+  IvfTagInfoItArray::iterator tagItEnd = ivfTagInfoOccurOrder.end();
+  while (tagItHere != tagItEnd) {
+    register IvfTagInfo &ivfTagInfo = (*(*tagItHere)).second;
+    buffer.binary_encode (ivfTagInfo.lastFragNum + 1, tagDictSize + 1, NULL);
+    oN++;
+    tagItHere++;
+  }
+
+  // finish encoding and close the file
+  buffer.encodeDone();
+  fclose (f);
+}
+
+#ifndef SILENT
+static void PrintStats () {
+  Message ("Inverted buffer size:          %8u bytes\n", invf_buffer_size);
+  Message ("Max memory needed for 1 chunk: %8u bytes\n", maxMemNeeded);
+
+  Message ("Number of chunks written:      %8u\n", chunksWritten);
+  Message ("Number of documents:           %8u\n", numDocs);
+  Message ("Number of fragments:           %8u\n", numFrags);
+  Message ("Number of words:               %8u\n", numWords);
+
+  Message ("Size of word dictionary:       %8u\n", ivfWordInfo.size());
+  Message ("Size of tag dictionary:        %8u\n", ivfTagInfo.size());
+}
+#endif
+
+int done_ivf_1 (const TagInfo &tagInfo, char * filename) {
+  bool wordLevelIndex = tagInfo.indexLevel.empty();
+
+  char *temp_str = msg_prefix;
+  msg_prefix = "ivf.pass1";
+
+  // output the last chunk
+  if (numChunkDocs > 0) {
+    unsigned long mem = MemoryRequired (wordLevelIndex);
+    OutputChunkInfo (mem, wordLevelIndex);
+    if (mem > maxMemNeeded) maxMemNeeded = mem;
+  }
+  
+  // write out and encoded 1 to say there are no more chunks
+  icb.gamma_encode (1, NULL);
+  icb.encodeDone ();
+
+  // write out the maximum memory required and close the file
+  fseek (ic, sizeof (long), 0);
+  WriteUL (ic, maxMemNeeded);
+  fclose (ic);
+
+  // output the inverted dictionary
+  OutputInvfDict (filename);
+
+  // write out the translation file
+  OutputTransFile (filename);
+
+  // output the level information
+  OutputLevelFile (filename);
+  
+  // output statistics
+#ifndef SILENT
+  PrintStats ();
+#endif
+  
+  msg_prefix = temp_str;
+
+  return COMPALLOK;
+}
+
Index: /trunk/gsdl/src/mgpp/text/ivf.pass2.cpp
===================================================================
--- /trunk/gsdl/src/mgpp/text/ivf.pass2.cpp	(revision 856)
+++ /trunk/gsdl/src/mgpp/text/ivf.pass2.cpp	(revision 856)
@@ -0,0 +1,1116 @@
+/**************************************************************************
+ *
+ * ivf.pass2.cpp -- Memory efficient pass 2 inversion
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  2000/01/14 02:26:07  sjboddie
+   Rodgers new C++ mg
+
+ */
+
+#include <stdio.h>
+#include <unistd.h>
+
+#include "non_ansi.h"
+
+#include "sysfuncs.h"
+
+#include "mg_files.h"
+#include "invf.h"
+#include "mg.h"
+#include "build.h"
+#include "locallib.h"
+#include "UCArray.h"
+#include "bitio_m_random.h"
+#include "bitio_m_stdio.h"
+#include "bitio_m_mems.h"
+#include "bitio_gen.h"
+#include <stdio.h>
+#include "words.h"
+#include "messages.h"
+#include "netorder.h"
+#include "FIvfLevelInfo.h"
+#include "perf_hash.h"
+#include "string.h"
+
+#include "longlong.h"
+
+#ifdef __WIN32_
+#include <io.h>
+#endif
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\map>
+#elif defined(GSDL_USE_STL_H)
+#  include <map.h>
+#else
+#  include <map>
+#endif
+
+
+#ifdef USE_LONG_LONG
+#define SEEK_X seek_LL
+#define TELL_X tell_LL
+#else
+#define SEEK_X seek
+#define TELL_X tell
+#endif
+
+#ifndef RND_BUF_SIZE
+#define RND_BUF_SIZE 8*1024
+#endif
+
+
+static unsigned long numDocs = 0;
+static unsigned long numChunkDocs = 0;
+static unsigned long numDocsInChunk = 0;
+
+static unsigned long numFrags = 0;
+static unsigned long numFragsInChunk = 0;
+static unsigned long chunkStartFragNum = 0;
+
+
+
+
+struct BitPtr {
+  unsigned long start;
+  unsigned long here;
+  unsigned long lastFragNum;
+  unsigned long lgB;
+  
+  void Clear () { start = here = lastFragNum = lgB = 0; }
+  BitPtr () { Clear(); }
+};
+
+class WordBitPtrs {
+protected:
+  unsigned long numWords;
+  unsigned long numTags;
+  unsigned long size;
+  BitPtr *wordBitPtrs;
+
+  void CheckBufOverrun (unsigned long num) {
+    if (wordBitPtrs[num].here > wordBitPtrs[num+1].start) {
+      cerr << "numDocs: " << numDocs << "\n";
+      cerr << "numChunkDocs: " << numChunkDocs << "\n";
+      cerr << "numDocsInChunk: " << numDocsInChunk << "\n";
+      cerr << "numFrags: " << numFrags << "\n";
+      cerr << "numFragsInChunk: " << numFragsInChunk << "\n";
+      cerr << "chunkStartFragNum: " << chunkStartFragNum << "\n";
+      cerr << "num: " << num << "\n";
+      cerr << "[num].start: " << wordBitPtrs[num].start << "\n";
+      cerr << "[num].here: " << wordBitPtrs[num].here << "\n";
+      cerr << "[num+1].start: " << wordBitPtrs[num+1].start << "\n";
+      FatalError (1, "Bit buffer overrun");
+    }
+  }
+  
+public:
+  void Clear ();
+  WordBitPtrs () { wordBitPtrs = NULL; Clear(); }
+  ~WordBitPtrs ();
+  void SetSize (unsigned long _numWords,
+		unsigned long _numTags);
+
+  void ResetPtrs () {
+    if (wordBitPtrs == NULL) return;
+    unsigned long i;
+    for (i=0; i<size; i++) wordBitPtrs[i].Clear();
+  }
+  
+  BitPtr &GetWordBitPtr (unsigned long wordNum)
+    { return wordBitPtrs[wordNum]; }
+  unsigned long &GetWordStart (unsigned long wordNum)
+    { return wordBitPtrs[wordNum].start; }
+  unsigned long &GetWordHere (unsigned long wordNum)
+    { return wordBitPtrs[wordNum].here; }
+  void CheckWordBufOverrun (unsigned long wordNum)
+    { CheckBufOverrun (wordNum); }
+
+  BitPtr &GetTagBitPtr (unsigned long tagNum)
+    { return wordBitPtrs[tagNum + numWords]; }
+  unsigned long &GetTagStart (unsigned long tagNum)
+    { return wordBitPtrs[tagNum + numWords].start; }
+  unsigned long &GetTagHere (unsigned long tagNum)
+    { return wordBitPtrs[tagNum + numWords].here; }
+  void CheckTagBufOverrun (unsigned long tagNum)
+    { CheckBufOverrun (tagNum + numWords); }
+
+  BitPtr &GetEndBitPtr ()
+    { return wordBitPtrs[size-1]; }
+  unsigned long &GetEndStart ()
+    { return wordBitPtrs[size-1].start; }
+  unsigned long &GetEndHere ()
+    { return wordBitPtrs[size-1].here; }
+};
+
+
+struct IP2TagInfo {
+  bool inTag;
+  unsigned long startFrag;
+  unsigned long tagNum;
+
+  IP2TagInfo () {
+    inTag = false;
+    startFrag = 0;
+    tagNum = 0;
+  }
+};
+
+// maps tags to tag information
+typedef map<UCArray, IP2TagInfo, DictLTUCArray> TagMapDict;
+
+
+// class to handle the translation of occurrence order
+// to dictionary order for words and tags
+class OccurToDictConverter {
+protected:
+  unsigned long pos;
+  unsigned long val;
+  FILE *transFile;
+  random_bitio_buffer rbs;
+
+  unsigned long wordDictSize;
+  unsigned long tagDictSize;
+
+  void SeekStart ();
+  unsigned long TranslateNum (unsigned long num);
+  
+public:
+  OccurToDictConverter ();
+  ~OccurToDictConverter ();
+
+  void Open (char *filename, unsigned long _wordDictSize,
+	     unsigned long _tagDictSize);
+
+  // Close frees all allocated memory
+  void Close ();
+
+  unsigned long TranslateWord (unsigned long occurNum)
+    { return TranslateNum (occurNum); }
+  unsigned long TranslateTag (unsigned long occurNum)
+    { return TranslateNum (occurNum+wordDictSize); }
+};
+
+
+struct InvfStateRec {
+  mg_ullong start;
+  mg_ullong here;
+  unsigned long lastFragNum;
+  unsigned long B;
+
+  void Clear () {
+    start = here = 0;
+    lastFragNum = B = 0;
+  }
+  InvfStateRec () { Clear (); }
+};
+
+
+#define ISR_SIZE 1024
+
+class InvfStateCache {
+protected:
+  InvfStateRec recCache [ISR_SIZE];
+  unsigned long startNum;
+
+  FILE *stateFile;
+
+  void ClearCache () {
+    unsigned int i = 0;
+    for (i=0; i<ISR_SIZE; i++) recCache[i].Clear();
+  }
+  
+public:
+  InvfStateCache ();
+  ~InvfStateCache ();
+  
+  void Open (char *filename);
+  void Close ();
+
+  // previous references to state records may be
+  // invalidated calling GetRec
+  InvfStateRec &GetRec (unsigned long num);
+};
+
+
+static invf_dict_header idh;
+static WordBitPtrs bitPtrs;
+
+static FILE *chunkFile = NULL;
+static stdio_bitio_buffer chunkBuf;
+
+static unsigned long ivfMemBufSize = 0;
+static char *ivfMemBuf = NULL;
+
+// word and tag dictionaries. a map is used for the tag dictionary
+// as it should never be very big (and the perfect hash function
+// sometimes has trouble with small values).
+static perf_hash_data *wordHashDict = NULL;
+static TagMapDict tagMapDict;
+
+// information about all the different levels
+static FIvfLevel ivfLevel;
+
+static OccurToDictConverter occurConvert;
+
+// information about the state of the inverted file
+static InvfStateCache invfState;
+
+static char collectFilename[512];
+
+
+void WordBitPtrs::Clear () {
+  numWords = 0;
+  numTags = 0;
+  size=0;
+  if (wordBitPtrs != NULL) delete [] wordBitPtrs;
+  wordBitPtrs = NULL;
+}
+
+WordBitPtrs::~WordBitPtrs () {
+  if (wordBitPtrs != NULL) delete [] wordBitPtrs;
+}
+
+void WordBitPtrs::SetSize (unsigned long _numWords,
+			   unsigned long _numTags){
+  Clear();
+  numWords = _numWords;
+  numTags = _numTags;
+  size = numWords + numTags + 1;
+  wordBitPtrs = new BitPtr [size];
+}
+
+
+void OccurToDictConverter::SeekStart () {
+  if (transFile == NULL) return;
+  rbs.SEEK_X (sizeof (unsigned long) * 8);
+  pos = 0;
+}
+
+unsigned long OccurToDictConverter::TranslateNum (unsigned long num) {
+  if (num < pos) SeekStart ();
+  while (pos <= num) {
+    if (pos < wordDictSize)
+      val = rbs.binary_decode (wordDictSize + 1, NULL) - 1;
+    else 
+      val = rbs.binary_decode (tagDictSize + 1, NULL) - 1;
+    pos++;
+  }
+  return val;
+}
+
+OccurToDictConverter::OccurToDictConverter () {
+  pos = 0;
+  val = 0;
+  transFile = NULL;
+  wordDictSize = 0;
+  tagDictSize = 0;
+}
+
+OccurToDictConverter::~OccurToDictConverter () {
+  if (transFile != NULL) Close ();
+}
+
+void OccurToDictConverter::Open (char *filename, unsigned long _wordDictSize,
+				 unsigned long _tagDictSize) {
+  if (transFile != NULL) Close ();
+
+  wordDictSize = _wordDictSize;
+  tagDictSize = _tagDictSize;
+
+  transFile = open_file (filename, INVF_CHUNK_TRANS_SUFFIX, "rb",
+			 MAGIC_CHUNK_TRANS, MG_ABORT);
+  rbs.attachFile (transFile, RND_BUF_SIZE);
+  SeekStart ();
+  val = 0;
+}
+
+void OccurToDictConverter::Close () {
+  if (transFile == NULL) return;
+
+  rbs.done ();
+  fclose (transFile);
+  transFile = NULL;
+  pos = 0;
+  val = 0;
+
+  wordDictSize = 0;
+  tagDictSize = 0;
+}
+
+
+
+
+InvfStateCache::InvfStateCache () {
+  startNum = 0;
+  stateFile = NULL;
+}
+
+InvfStateCache::~InvfStateCache () {
+  if (stateFile != NULL) Close ();
+}
+  
+void InvfStateCache::Open (char *filename) {
+  if (stateFile != NULL) Close();
+
+  // open the state file
+  char path[512];
+  sprintf (path, FILE_NAME_FORMAT ".%ld", get_basepath (), filename,
+	   ".invf.state", (long) getpid ());
+  if (!(stateFile = fopen (path, "wb+"))) {
+    Message ("Unable to create \"%s\"", path);
+    exit (1);
+  }
+  unlink (path); // file will be deleted after it is closed
+  
+  // reset the buffer
+  startNum = 0;
+  ClearCache();
+}
+
+void InvfStateCache::Close () {
+  if (stateFile == NULL) return;
+  fclose (stateFile);
+  stateFile = NULL;
+  startNum = 0;
+}
+
+InvfStateRec &InvfStateCache::GetRec (unsigned long num) {
+  // see if cached
+  if ((num >= startNum) && (num < startNum + ISR_SIZE))
+    return recCache[num-startNum];
+
+  // not cached, write out this lot of records and read in
+  fseek (stateFile, startNum*sizeof (InvfStateRec), SEEK_SET);
+  fwrite ((char *) recCache, sizeof (InvfStateRec), ISR_SIZE, stateFile);
+
+  // read in the new set of records
+  ClearCache ();
+  startNum = num - (num % ISR_SIZE);
+  fseek (stateFile, startNum*sizeof (InvfStateRec), SEEK_SET);
+  fread ((char *) recCache, sizeof (InvfStateRec), ISR_SIZE, stateFile);
+
+  return recCache[num-startNum];
+}
+
+
+
+static void ClearCharBuf (char *buf, unsigned long size) {
+  char *end = buf + size;
+  while (buf != end) *buf++ = 0;
+}
+
+static void ReadWordDict (char *filename) {
+  // read in the perfect hash function for the word dictionary
+  FILE *wordHashFile = open_file (filename, INVF_DICT_HASH_SUFFIX, "rb",
+				  MAGIC_HASH, MG_ABORT);
+  if (!(wordHashDict = read_perf_hash_data (wordHashFile))) {
+    FatalError (1, "Unable to read in hash data for word dictionary");
+  }
+  fclose (wordHashFile);
+}
+
+static void ReadTagDict (char *filename, invf_dict_header &_idh) {
+  // open the file
+  FILE *dictFile = open_file (filename, INVF_DICT_SUFFIX, "rb",
+			      MAGIC_STEM_BUILD, MG_ABORT);
+
+  // seek to the start of the tag dictionary
+  fseek (dictFile, _idh.tag_dict_start, SEEK_SET);
+  
+  unsigned long tagNum;
+  dict_el thisEl;
+  for (tagNum = 0; tagNum < _idh.tag_dict_size; tagNum++) {
+    thisEl.Read (dictFile);
+    tagMapDict[thisEl.el].tagNum = tagNum;
+  }
+  
+  fclose (dictFile);
+}
+
+static void ReadLevelFile (char *filename) {
+  FILE *f;
+  f = open_file (filename, INVF_LEVEL_SUFFIX, "rb",
+		 MAGIC_INVF_LEVELS, MG_ABORT);
+  ivfLevel.Read (f);
+  fclose (f);
+}
+
+void CheckIntOverflow (mg_ullong totalIBits, mg_ullong lastTotalIBits) {
+  if (totalIBits < lastTotalIBits) {
+    fprintf(stderr, "ERROR: The totalIBits counter (%d byte unsigned integer) has overflowed.\n", sizeof (mg_ullong));
+    if (sizeof (mg_ullong) < 8) {
+      fprintf(stderr, "       Try compiling with GCC to enable use of 8 bytes for this counter.\n");
+    }
+    fprintf(stderr, "       Build aborted.\n");
+    exit(1);
+  }
+}
+
+// assumes the inverted file state file has been opened
+static void InitInvfState (char *filename,
+			   invf_dict_header &_idh,
+			   InvfStateCache &_invfState,
+			   mg_ullong &totalIBits,
+			   bool wordLevelIndex) {
+  // read in the dictionary, setting inverted state information
+  FILE *dictFile = open_file (filename, INVF_DICT_SUFFIX, "rb",
+			      MAGIC_STEM_BUILD, MG_ABORT);
+
+  // seek to the start of the word dictionary
+  fseek (dictFile, _idh.word_dict_start, SEEK_SET);
+
+  // add the word entries
+  word_dict_el wordEl;
+  wordEl.SetNumLevels (_idh.num_levels);
+  unsigned long dictWordNum, p;
+  mg_ullong lastTotalIBits;
+  unsigned long N = _idh.num_frags;
+  for (dictWordNum=0; dictWordNum<_idh.word_dict_size; dictWordNum++) {
+    // lastTotalIBits is used to detect integer overflow
+    lastTotalIBits = totalIBits;
+
+    // read the next word and associated information
+    wordEl.Read (dictFile, _idh.num_levels);
+
+    // update the state record
+    p = wordEl.frag_occur;
+    InvfStateRec &wisr = _invfState.GetRec (dictWordNum);
+    wisr.start = totalIBits;
+    wisr.here = totalIBits;
+    wisr.B = BIO_Bblock_Init (N, p);
+    
+    // add the length of the fragment numbers
+    totalIBits += BIO_Bblock_Bound_b (N, p, wisr.B);
+
+    // if needed, add the length of the fragment frequency information
+    if (!wordLevelIndex)
+      totalIBits += BIO_Gamma_Bound (wordEl.freq, wordEl.frag_occur);
+
+    // align next byte
+#ifdef USE_LONG_LONG
+    totalIBits = (totalIBits + 7ull) & 0xfffffffffffffff8ull;
+#else
+    totalIBits = (totalIBits + 7ul) & 0xfffffff8ul;
+#endif
+
+    CheckIntOverflow (totalIBits, lastTotalIBits);
+  }
+
+  
+  // seek to the start of the tag dictionary
+  fseek (dictFile, _idh.tag_dict_start, SEEK_SET);
+
+  // add the tag entries
+  dict_el tagEl;
+  unsigned long dictTagNum;
+  N = _idh.num_frags;
+  for (dictTagNum=0; dictTagNum<_idh.tag_dict_size; dictTagNum++) {
+    // lastTotalIBits is used to detect integer overflow
+    lastTotalIBits = totalIBits;
+
+    // read the next tag and associated information
+    tagEl.Read (dictFile);
+
+    // update the state record
+    p = tagEl.frag_occur*2;
+    InvfStateRec &tisr = _invfState.GetRec (dictTagNum + _idh.word_dict_size);
+    tisr.start = totalIBits;
+    tisr.here = totalIBits;
+    tisr.B = BIO_Bblock_Init (N+p, p);
+    
+    // add the length of the fragment numbers (two numbers for each
+    // tag, one for start and one for end)
+    totalIBits += BIO_Bblock_Bound_b (N+p, p, tisr.B);
+
+    // align next byte
+#ifdef USE_LONG_LONG
+    totalIBits = (totalIBits + 7ull) & 0xfffffffffffffff8ull;
+#else
+    totalIBits = (totalIBits + 7ul) & 0xfffffff8ul;
+#endif
+
+    CheckIntOverflow (totalIBits, lastTotalIBits);
+  }
+
+  fclose (dictFile);
+}
+
+/*
+// assumes the chunk tag information has been placed in .first
+static void PrintChunkInfo (unsigned long chunkMem,
+			    unsigned long numChunkWords,
+			    unsigned long numChunkTags) {
+  static unsigned long chunksRead = 0;
+  chunksRead++;
+  cout << "Chunk Number: " << chunksRead << "\n";
+  cout << "numChunkDocs " << numDocsInChunk << "\n";
+  cout << "numChunkFrags " << numFragsInChunk << "\n";
+  cout << "mem " << chunkMem << "\n";
+  cout << "numWords " << numChunkWords << "\n";
+  cout << "numTags " << numChunkTags << "\n\n";
+
+  TagMapDict::iterator tagMapHere = tagMapDict.begin();
+  TagMapDict::iterator tagMapEnd = tagMapDict.end();
+  while (tagMapHere != tagMapEnd) {
+    unsigned long tagMapNum = (*tagMapHere).second.tagNum;
+    cout << (*tagMapHere).first << " " << tagMapNum << " "
+	 << bitPtrs.GetTagBitPtr(tagMapNum).here << "\n";
+    tagMapHere++;
+  }
+}
+*/
+
+void ReadChunk (invf_dict_header &_idh, bool wordLevelIndex) {
+  // reset globals
+  numChunkDocs = 0;
+  chunkStartFragNum = numFrags;
+
+  // read in information about this chunk
+  numDocsInChunk = chunkBuf.gamma_decode (NULL) - 1;
+  if (numDocsInChunk == 0)
+    FatalError (1, "The number of docs in the current chunk is 0");
+
+  numFragsInChunk = chunkBuf.gamma_decode (NULL) - 1;
+  unsigned long chunkMem = chunkBuf.gamma_decode (NULL) - 1;
+
+  if (chunkMem > ivfMemBufSize)
+    FatalError (1, "Chunk memory size is greater than maximum");
+  
+  unsigned long numChunkWords = chunkBuf.gamma_decode (NULL) - 1;
+  unsigned long numChunkTags = chunkBuf.gamma_decode (NULL) - 1;
+
+  
+  // reset stuff
+  ClearCharBuf (ivfMemBuf, ivfMemBufSize);
+  bitPtrs.ResetPtrs();  
+
+  // read in the entries in occurrence order storing the
+  // "chunkWordCount" in "start" and the "chunkFragCount"
+  // in "here"
+  unsigned long numOccur;
+  unsigned long wordNum;
+  for (numOccur=0; numOccur<numChunkWords; numOccur++) {
+    wordNum = occurConvert.TranslateWord (numOccur);
+    BitPtr &wordPtr = bitPtrs.GetWordBitPtr (wordNum);
+    wordPtr.start = chunkBuf.gamma_decode (NULL) - 1;
+    if (wordPtr.start >= 2)
+      wordPtr.here = chunkBuf.gamma_decode (NULL);
+    else wordPtr.here = wordPtr.start;
+  }
+  unsigned long tagNum;
+  for (numOccur=0; numOccur<numChunkTags; numOccur++) {
+    tagNum = occurConvert.TranslateTag (numOccur);
+    BitPtr &tagPtr = bitPtrs.GetTagBitPtr (tagNum);
+    // only chunkFragCount is encoded for tags
+    tagPtr.start = chunkBuf.gamma_decode (NULL) - 1;
+    tagPtr.here = tagPtr.start;
+  }
+
+  /*  PrintChunkInfo (chunkMem, numChunkWords, numChunkTags);*/
+  
+  // create the bit ptrs in dictionary order
+  unsigned long totalIBits = 0; // only dealing with memory
+  unsigned long chunkWordCount, chunkFragCount;
+  for (wordNum=0; wordNum<_idh.word_dict_size; wordNum++) {
+    BitPtr &wordPtr = bitPtrs.GetWordBitPtr (wordNum);
+    chunkWordCount = wordPtr.start;
+    chunkFragCount = wordPtr.here;
+    wordPtr.start = totalIBits;
+    wordPtr.here = totalIBits;
+    wordPtr.lastFragNum = chunkStartFragNum;
+    wordPtr.lgB = 0;
+    if (chunkWordCount > 0) {
+      wordPtr.lgB = floorlog_2 (BIO_Bblock_Init_W (numFragsInChunk,
+						   chunkFragCount));
+      totalIBits += BIO_Bblock_Bound (numFragsInChunk, chunkFragCount);
+      // use unary encoding for memory buffer encoding of fragment freq
+      if (!wordLevelIndex) {
+	totalIBits += chunkWordCount;
+      }
+    }
+  }
+  for (tagNum=0; tagNum<_idh.tag_dict_size; tagNum++) {
+    BitPtr &tagPtr = bitPtrs.GetTagBitPtr (tagNum);
+    chunkFragCount = tagPtr.here;
+    tagPtr.start = totalIBits;
+    tagPtr.here = totalIBits;
+    tagPtr.lastFragNum = chunkStartFragNum;
+    tagPtr.lgB = 0;
+    if (chunkFragCount > 0) {
+      unsigned long pTag = chunkFragCount*2;
+      tagPtr.lgB = floorlog_2 (BIO_Bblock_Init_W (numFragsInChunk+pTag,
+						  pTag));
+      unsigned long bLen = BIO_Bblock_Bound (numFragsInChunk+pTag,
+					     pTag);
+//        cout << tagNum + _idh.word_dict_size << " ";
+//        cout << "numFrags: " << numFragsInChunk
+//    	   << " chunkFragCount: " << chunkFragCount
+//    	   << " B: " << 1 << tagPtr.lgB
+//    	   << " blen: " << blen << "\n";
+      totalIBits += bLen;
+    }
+  }
+  bitPtrs.GetEndStart() = totalIBits;
+  bitPtrs.GetEndHere() = totalIBits;
+
+  if ((totalIBits + 7ul) >> 3ul > chunkMem) {
+    cerr << "totalIBits: " << totalIBits << "\n";
+    cerr << "bytes: " << ((totalIBits + 7ul) >> 3ul) << "\n";
+    cerr << "chunkMem: " << chunkMem << "\n";
+    FatalError (1, "Pointers exceed buffer size");
+  }
+}
+
+
+
+
+int init_ivf_2 (const TagInfo &/*tagInfo*/, char *filename) {
+  // read in compressed dictionary header
+  FILE *dictFile = open_file (filename, INVF_DICT_SUFFIX, "rb",
+			      MAGIC_STEM_BUILD, MG_ABORT);
+  idh.Read (dictFile);
+  fclose (dictFile);
+
+  // set the size of the bit ptrs
+  bitPtrs.SetSize (idh.word_dict_size, idh.tag_dict_size);
+
+  // open the chunk file and read in the maximum memory needed
+  // for the inverted memory buffer
+  chunkFile = open_file (filename, INVF_CHUNK_SUFFIX, "rb",
+			 MAGIC_CHUNK, MG_ABORT);
+  ReadUL (chunkFile, ivfMemBufSize);
+  chunkBuf.attachFile (chunkFile);
+
+  // allocate memory for the inverted buffer
+  ivfMemBuf = new char [ivfMemBufSize];
+  ClearCharBuf (ivfMemBuf, ivfMemBufSize);
+
+  // read in the word dictionary
+  ReadWordDict (filename);
+
+  // read in the tag dictionary
+  ReadTagDict (filename, idh);
+
+  // read in the level information
+  ReadLevelFile (filename);
+  bool wordLevelIndex = ivfLevel.indexLevel.empty();
+  
+  // set up the translation table file
+  occurConvert.Open (filename, idh.word_dict_size, idh.tag_dict_size);
+
+  // reset some globals
+  numDocs = 0;
+  numChunkDocs = 0;
+  numDocsInChunk = 0;
+  numFrags = 0;
+  numFragsInChunk = 0;
+  chunkStartFragNum = 0;
+
+  strcpy (collectFilename, filename);
+  
+  
+  // create the inverted file
+  mg_ullong totalIBits = 0;
+  FILE *invfFile = create_file (filename, INVF_SUFFIX, "wb",
+				MAGIC_INVF, MG_ABORT);
+  totalIBits += sizeof (unsigned long) * 8; // magic number
+  totalIBits += 8 * 200;                    // 200 byte gap -- why??????
+  fclose (invfFile);
+
+  // init the inverted file state cache
+  invfState.Open (filename);
+  InitInvfState (filename, idh, invfState, totalIBits, wordLevelIndex);
+
+  return COMPALLOK;
+}
+
+static void CloseTextTag (IP2TagInfo &tInfo, const UCArray &/*tagName*/) {
+  if (!tInfo.inTag) return;
+
+    // add this tag to the inverted list
+  BitPtr &tagBitPtr = bitPtrs.GetTagBitPtr (tInfo.tagNum);
+  unsigned long endFrag = numFrags;
+  int b = 1 << tagBitPtr.lgB;
+
+  /*
+  cout << (tInfo.tagNum+idh.word_dict_size) << " \"<" << tagName << ">\" "
+       << tInfo.startFrag << " " << endFrag << "\n";
+  */
+    
+  mems_bitio_buffer buffer ((u_char *) ivfMemBuf, tagBitPtr.here);
+  buffer.bblock_encode (tInfo.startFrag - tagBitPtr.lastFragNum + 1,
+			b, NULL);
+  buffer.bblock_encode (endFrag - tInfo.startFrag + 1, b, NULL);
+  tagBitPtr.lastFragNum = endFrag;
+  tagBitPtr.here = buffer.position();
+  buffer.encodeDone();
+  
+  // check for buffer overrun
+  bitPtrs.CheckTagBufOverrun (tInfo.tagNum);
+  
+  // reset information about this tag
+  tInfo.inTag = false;
+  tInfo.startFrag = 0;
+}
+
+static void ProcessOpenTag (const TextEl &el, bool &inFrag) {
+  // close tag if open
+  IP2TagInfo &tInfo = tagMapDict[el.tagName];
+  if (tInfo.inTag) CloseTextTag (tInfo, el.tagName);
+
+  // open this tag
+  tInfo.inTag = true;
+  tInfo.startFrag = numFrags;
+
+  // check for start of next fragment
+  bool wordLevelIndex = ivfLevel.indexLevel.empty();
+  if (!wordLevelIndex && el.tagName == ivfLevel.indexLevel) {
+    numFrags++;
+    inFrag = true;
+  }
+}
+
+static void ProcessCloseTag (const TextEl &el, bool &inFrag) {
+  // check for end of fragment
+  bool wordLevelIndex = ivfLevel.indexLevel.empty();
+  if (!wordLevelIndex && el.tagName == ivfLevel.indexLevel) {
+    inFrag = false;
+  }
+
+  IP2TagInfo &tInfo = tagMapDict[el.tagName];
+  CloseTextTag (tInfo, el.tagName);
+}
+
+static void ProcessText (const TextEl &el, bool &inFrag) {
+  // make sure this text is to be indexed
+  bool wordLevelIndex = ivfLevel.indexLevel.empty();
+  if (!wordLevelIndex && !inFrag) return;
+
+  const unsigned char *textHere = el.text.begin();
+  const unsigned char *textEnd = el.text.end() - 1;
+  unsigned char mgWord[MAXSTEMLEN + 1];
+  
+  if (!inaword (textHere, textEnd))
+    ParseNonindexWord (textHere, textEnd);
+
+  
+  // Alternately parse off words and non-words from the input
+
+  while (textHere <= textEnd) {
+    textHere = ParseIndexMGWord (textHere, textEnd, mgWord);
+    textHere = ParseNonindexWord (textHere, textEnd);
+
+    if (mgWord[0] > 0) {
+      if (wordLevelIndex) numFrags++;
+
+      unsigned long wordNum = perf_hash (wordHashDict, mgWord);
+
+      /*
+      cout << wordNum << " \"";
+      cout.write (mgWord+1, *mgWord);
+      cout << "\" " << numFrags << "\n";
+      */
+      
+      // add this word to the inverted list
+      BitPtr &wordBitPtr = bitPtrs.GetWordBitPtr (wordNum);
+      unsigned long fragNum = numFrags;
+      int b = 1 << wordBitPtr.lgB;
+  
+      mems_bitio_buffer buffer ((u_char *) ivfMemBuf, wordBitPtr.here);
+
+      // note: this assumes that fragments don't carry over between
+      // chunks (which they don't because all tags are closed at the
+      // end of each document and chunks are based on document
+      // boundaries), i.e. the first fragment number must be greater
+      // than the starting fragment number of the chunk.
+      if (fragNum > wordBitPtr.lastFragNum) {
+	buffer.bblock_encode ((fragNum - wordBitPtr.lastFragNum - 1) + 1,
+			      b, NULL);
+	if (!wordLevelIndex) buffer.encodeBit (1); // freq = 1
+	
+      } else if (!wordLevelIndex) {
+	// add one to the frequency count for this word
+	buffer.seek (buffer.position()-1);
+	buffer.encodeBit (0); // unary encoding -- last = 1
+	buffer.encodeBit (1);
+      }
+      
+      wordBitPtr.lastFragNum = fragNum;
+      wordBitPtr.here = buffer.position();
+      buffer.encodeDone();
+      
+      // check for buffer overrun
+      bitPtrs.CheckWordBufOverrun (wordNum);
+    }
+  }
+}
+
+// combine the in memory inverted buffer with the disk
+// based inverted file
+static void DiskMerge (char *filename) {
+  bool wordLevelIndex = ivfLevel.indexLevel.empty();
+
+      // make sure we have something to process
+  if (numChunkDocs <= 0) return;
+    
+  // open the inverted file
+  FILE *invfFile = open_file (filename, INVF_SUFFIX, "rb+",
+			      MAGIC_INVF, MG_ABORT);
+  random_bitio_buffer invfOutBuf (invfFile);
+
+  // set up to decode the entries in memory
+  mems_bitio_buffer memInBuf ((u_char *) ivfMemBuf, 0);
+
+  // write out the word information
+  unsigned long wordNum;
+  int b;
+  unsigned long currFragNum;
+  unsigned long delta;
+  unsigned long currFreq;
+  for (wordNum=0; wordNum<idh.word_dict_size; wordNum++) {
+    // go to the end of the last inverted file entry
+    InvfStateRec &wordDiskState = invfState.GetRec (wordNum);
+    invfOutBuf.SEEK_X (wordDiskState.here);
+
+    // go to the start of the inverted chunk info in memory
+    BitPtr &wordBitPtr = bitPtrs.GetWordBitPtr (wordNum);
+    memInBuf.seek(wordBitPtr.start);
+
+    // decode each entry and re-write to disk
+    currFragNum = chunkStartFragNum;
+    while (memInBuf.position() < wordBitPtr.here) {
+      // decode word entry
+      b = 1 << wordBitPtr.lgB;
+      delta = memInBuf.bblock_decode (b, NULL);
+      currFragNum += delta;
+      if (!wordLevelIndex) currFreq = memInBuf.unary_decode (NULL);
+      else currFreq = 1;
+
+      // recode on disk
+      invfOutBuf.bblock_encode (currFragNum-wordDiskState.lastFragNum,
+				wordDiskState.B, NULL);
+      if (!wordLevelIndex) invfOutBuf.gamma_encode (currFreq, NULL);
+      wordDiskState.lastFragNum = currFragNum;
+    }
+
+    wordDiskState.here = invfOutBuf.TELL_X();
+  }
+
+  // write out the tag information
+  unsigned long tagNum;
+  unsigned long currTagStart;
+  unsigned long currTagEnd;
+  for (tagNum=0; tagNum<idh.tag_dict_size; tagNum++) {
+    // go to the end of the last inverted file entry
+    InvfStateRec &tagDiskState = invfState.GetRec (tagNum+idh.word_dict_size);
+    invfOutBuf.SEEK_X (tagDiskState.here);
+
+    // go to the start of the inverted chunk info in memory
+    BitPtr &tagBitPtr = bitPtrs.GetTagBitPtr (tagNum);
+    memInBuf.seek(tagBitPtr.start);
+
+    // decode each entry and re-write to disk
+    currTagEnd = chunkStartFragNum;
+    while (memInBuf.position() < tagBitPtr.here) {
+      // decode tag entry
+      b = 1 << tagBitPtr.lgB;
+      delta = memInBuf.bblock_decode (b, NULL) - 1;
+      currTagStart = currTagEnd + delta;
+      delta = memInBuf.bblock_decode (b, NULL) - 1;
+      currTagEnd = currTagStart + delta;
+
+      // recode on disk
+      invfOutBuf.bblock_encode (currTagStart-tagDiskState.lastFragNum+1,
+				tagDiskState.B, NULL);
+      invfOutBuf.bblock_encode (currTagEnd-currTagStart+1,
+				tagDiskState.B, NULL);
+
+      tagDiskState.lastFragNum = currTagEnd;
+    }
+
+    tagDiskState.here = invfOutBuf.TELL_X();
+  }
+
+  memInBuf.done();
+
+  invfOutBuf.encodeDone();
+  fclose (invfFile);
+}
+
+
+int process_ivf_2 (const TagInfo &/*tagInfo*/, const TextElArray &doc) {
+  bool wordLevelIndex = ivfLevel.indexLevel.empty();
+  bool inFrag = false;
+  if (wordLevelIndex) inFrag = true; // unconditional
+  
+  // get next chunk information if need to. the chunk information
+  // is needed before the first document is processed
+  if (numChunkDocs >= numDocsInChunk) ReadChunk (idh, wordLevelIndex);
+
+  // process each text element
+  TextElArray::const_iterator here = doc.begin();
+  TextElArray::const_iterator end = doc.end();
+  while (here != end) {
+    // process this element
+    if ((*here).elType == OpenTagE) ProcessOpenTag (*here, inFrag);
+    else if ((*here).elType == CloseTagE) ProcessCloseTag (*here, inFrag);
+    else ProcessText (*here, inFrag);
+    
+    here++;
+  }
+
+  // close off any unclosed tags
+  TagMapDict::iterator tdHere = tagMapDict.begin();
+  TagMapDict::iterator tdEnd = tagMapDict.end();
+  while (tdHere != tdEnd) {
+    CloseTextTag ((*tdHere).second, (*tdHere).first);
+    tdHere++;
+  }
+  
+  // we've processed one more document
+  numDocs++;
+  numChunkDocs++;
+
+  // merge the memory based inverted file with the one on
+  // disk if this is the end of this chunk
+  if (numChunkDocs >= numDocsInChunk) DiskMerge (collectFilename);
+  
+  return COMPALLOK;
+}
+
+
+static void CondenseInvfFile (char *filename, unsigned long &bytesOutput) {
+  FILE *inInvfFile = open_file (filename, INVF_SUFFIX, "rb",
+				MAGIC_INVF, MG_ABORT);
+  FILE *outInvfFile = open_file (filename, INVF_SUFFIX, "rb+",
+				 MAGIC_INVF, MG_ABORT);
+
+  // skip the magic number
+  fseek (outInvfFile, sizeof (unsigned long), SEEK_SET);
+  
+  // write the inverted file header -- use defaults for most things
+  invf_file_header ifh;
+  ifh.no_of_words = idh.word_dict_size;
+  ifh.no_of_tags = idh.tag_dict_size;
+  ifh.word_level_index = (ivfLevel.indexLevel.empty()) ? 1 : 0;
+  ifh.Write (outInvfFile);
+
+  bytesOutput = ftell (outInvfFile);
+
+  // process each meaningful byte in the file
+  unsigned long numEntries = ifh.no_of_words + ifh.no_of_tags;
+  unsigned long entryNum;
+  mg_ullong lastStart = 0;
+  for (entryNum = 0; entryNum < numEntries; entryNum++) {
+    InvfStateRec &stateRec = invfState.GetRec (entryNum);
+
+    // overrun check
+    if (stateRec.start < lastStart)
+      FatalError (1, "Inverted file Buffer overrun");
+    lastStart = stateRec.start;
+    
+    unsigned long oldEntryStart = stateRec.start >> 3;
+    unsigned long oldEntryStartOver = stateRec.start & 7; // should be 0
+    unsigned long oldEntryEnd = (stateRec.here + 7) >> 3;  // byte after end
+    unsigned long oldEntryEndOver = stateRec.here & 7;
+
+    fseek (inInvfFile, oldEntryStart, SEEK_SET);
+
+    stateRec.here -= stateRec.start;
+    stateRec.start = bytesOutput * 8 + oldEntryStartOver;
+    stateRec.here += stateRec.start;
+    while (oldEntryStart < oldEntryEnd) {
+      unsigned char c = getc (inInvfFile);
+      if (oldEntryStart == oldEntryEnd - 1) {
+	u_char ands[8] =
+	{0xff, 0x80, 0xc0, 0xe0, 0xf0, 0xf8, 0xfc, 0xfe};
+	c &= ands[oldEntryEndOver];
+      }
+      putc (c, outInvfFile);
+      bytesOutput++;
+      oldEntryStart++;
+    }
+  }  
+
+  fclose (inInvfFile);
+
+#ifdef __WIN32__
+  if (!(_chsize (_fileno (outInvfFile), bytesOutput)))
+    Message ("Could not truncate invf.");
+#else
+  ftruncate (fileno (outInvfFile), bytesOutput);
+#endif
+
+  fclose (outInvfFile);
+}
+
+static void OutputInvfIdx (char *filename, unsigned long invfNumBytes) {
+  FILE *invfIdxFile = create_file (filename, INVF_IDX_SUFFIX, "wb",
+				   MAGIC_INVI, MG_ABORT);
+
+  // process each meaningful byte in the file
+  unsigned long numEntries = idh.word_dict_size + idh.tag_dict_size;
+  unsigned long entryNum;
+  for (entryNum = 0; entryNum < numEntries; entryNum++) {
+    InvfStateRec &stateRec = invfState.GetRec (entryNum);
+
+    // assumes that inverted entries start at beginning of each byte
+    if (!WriteUL (invfIdxFile, (stateRec.start >> 3))) break;
+  }
+
+  WriteUL (invfIdxFile, invfNumBytes);
+
+  fclose (invfIdxFile);
+}
+
+
+int done_ivf_2 (const TagInfo &/*tagInfo*/, char *filename) {
+  // close most open files
+  if (chunkFile != NULL) {
+    chunkBuf.done();
+    fclose (chunkFile);
+    chunkFile = NULL;
+  }
+  occurConvert.Close();
+  
+  // free allocated memory
+  bitPtrs.Clear();
+  if (ivfMemBuf != NULL) { delete [] ivfMemBuf; ivfMemBuf = NULL; }
+  free_perf_hash (wordHashDict);
+  wordHashDict = NULL;
+  tagMapDict.erase (tagMapDict.begin(), tagMapDict.end());
+
+  // condense the inverted file and truncate it
+  // this function also writes out the inverted header
+  unsigned long invfNumBytes = 0;
+  CondenseInvfFile (filename, invfNumBytes);
+
+  OutputInvfIdx (filename, invfNumBytes);
+
+  // close the rest of the open files
+  invfState.Close ();
+  
+  return COMPALLOK;
+}
Index: /trunk/gsdl/src/mgpp/text/locallib.cpp
===================================================================
--- /trunk/gsdl/src/mgpp/text/locallib.cpp	(revision 856)
+++ /trunk/gsdl/src/mgpp/text/locallib.cpp	(revision 856)
@@ -0,0 +1,244 @@
+/**************************************************************************
+ *
+ * locallib.cpp -- Misc functions
+ * Copyright (C) 1994  Gary Eddy, Alistair Moffat and Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "bitio_gen.h"
+#include "huffman.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "locallib.h"
+#include "text.h"
+
+/*
+   $Log$
+   Revision 1.1  2000/01/14 02:26:08  sjboddie
+   Rodgers new C++ mg
+
+   Revision 1.1  1999/10/11 02:57:43  cs025
+   Base install of MG-PP
+
+   Revision 1.1  1999/08/10 21:18:02  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:34:49  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:51  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:38  tes
+   * For version 1.1
+   *
+ */
+
+
+int 
+vecentropy (int *A, int n)
+{
+  int total = 0, i;
+  double bits = 0.0;
+
+  for (i = 0; i < n; i++)
+    total += A[i];
+  for (i = 0; i < n; i++)
+    if (A[i])
+      bits += A[i] * log2 (1.0 * total / A[i]);
+  return ((int) (bits + 0.9999));
+
+}
+
+static void 
+siftdown (int *A, int i, int n)
+{
+  int ms, t;
+
+  while (2 * i + 1 <= n - 1)
+    {
+      if (2 * i + 1 == n - 1 || A[2 * i + 1] < A[2 * i + 2])
+	ms = 2 * i + 1;
+      else
+	ms = 2 * i + 2;
+      if (A[i] > A[ms])
+	{
+	  t = A[i];
+	  A[i] = A[ms];
+	  A[ms] = t;
+	}
+      i = ms;
+    }
+}
+
+static void 
+buildheap (int *A, int n)
+{
+  int i;
+  for (i = n / 2 - 1; i >= 0; i--)
+    siftdown (A, i, n);
+}
+
+int 
+huffcodebits (unsigned long *A, int n)
+{
+  int i, j, tot = 0, bits = 0, v1, v2, *B;
+
+  B = (int *) Xmalloc (n * sizeof (int));
+  j = 0;
+  for (i = 0; i < n; i++)
+    {
+      if (A[i])
+	{
+	  tot += (B[j++] = A[i]);
+	}
+    }
+  n = j;
+
+  if (n == 0 || tot == 0)
+    return (0);
+
+  buildheap (B, n);
+  while (n > 1)
+    {
+      v1 = B[0];
+      B[0] = B[n - 1];
+      n--;
+      siftdown (B, 0, n);
+      v2 = B[0];
+      B[0] = v1 + v2;
+      siftdown (B, 0, n);
+      bits += v1 + v2;
+    }
+  Xfree (B);
+  return (bits);
+}
+
+
+int 
+modelbits (unsigned long *A, int n)
+{
+  int i, bits = 0, last, N = 0;
+
+  last = -1;
+  for (i = 0; i < n; i++)
+    {
+      if (A[i])
+	{
+	  bits += BIO_Gamma_Length (i - last) + BIO_Gamma_Length (A[i]);
+	  last = i;
+	  N++;
+	}
+    }
+  if (N == 0)
+    return (0);
+  bits += BIO_Gamma_Length (N);
+  return (bits);
+}
+
+
+
+
+
+/* Find the next prime number larger than p */
+int 
+prime (int p)
+{
+  int limit;
+  p += (p & 1) + 1;
+  limit = (int) sqrt ((double) p) + 1;
+  do
+    {
+      int j;
+      while (limit * limit < p)
+	limit++;
+      for (j = 3; j <= limit && p % j; j += 2)
+	;			/* NULLBODY */
+      if (j > limit)
+	break;
+      p += 2;
+    }
+  while (1);
+  return (p);
+}
+
+
+
+int Read_cdh (FILE * f, compression_dict_header * cdh, u_long * /*mem*/,
+	      u_long * disk) {
+  if (disk)
+    (*disk) += sizeof (*cdh);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  if (fread (cdh, sizeof (*cdh), 1, f) == 1)
+    {
+      int i;
+      NTOHUL(cdh->dict_type);
+      NTOHUL(cdh->novel_method);
+      for (i = 0; i < TEXT_PARAMS; i++)
+	NTOHUL(cdh->params[i]);
+      NTOHUL(cdh->num_words[0]);
+      NTOHUL(cdh->num_words[1]);
+      NTOHUL(cdh->num_word_chars[0]);
+      NTOHUL(cdh->num_word_chars[1]);
+      NTOHUL(cdh->lookback);
+      return 0;
+    }
+  else
+    return -1;
+}
+
+
+int 
+Read_cfh (FILE * f, comp_frags_header * cfh, u_long * mem, u_long * disk)
+{
+  int i;  /* [RPAP - Jan 97: Endian Ordering] */
+
+  if (Read_Huffman_Data (f, &cfh->hd, mem, disk) == -1)
+    return -1;
+  if (fread (&cfh->uncompressed_size,
+	     sizeof (cfh->uncompressed_size), 1, f) != 1)
+    return -1;
+  NTOHUL(cfh->uncompressed_size);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  if (disk)
+    (*disk) += sizeof (cfh->uncompressed_size);
+
+  bzero ((char *) cfh->huff_words_size, sizeof (cfh->huff_words_size));
+
+  if ((int)fread (cfh->huff_words_size + cfh->hd.mincodelen,
+	     sizeof (*cfh->huff_words_size),
+	     cfh->hd.maxcodelen - cfh->hd.mincodelen + 1, f) !=
+      cfh->hd.maxcodelen - cfh->hd.mincodelen + 1)
+    return -1;
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = cfh->hd.mincodelen; i < cfh->hd.maxcodelen + 1; i++)
+    NTOHUL(cfh->huff_words_size[i]);
+
+  if (disk)
+    (*disk) += sizeof (*cfh->huff_words_size) *
+      (cfh->hd.maxcodelen - cfh->hd.mincodelen + 1);
+  return 0;
+}
+
Index: /trunk/gsdl/src/mgpp/text/locallib.h
===================================================================
--- /trunk/gsdl/src/mgpp/text/locallib.h	(revision 856)
+++ /trunk/gsdl/src/mgpp/text/locallib.h	(revision 856)
@@ -0,0 +1,47 @@
+/**************************************************************************
+ *
+ * locallib.h -- Misc functions
+ * Copyright (C) 1994  Gary Eddy, Alistair Moffat and Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef LOCALLIB_H
+#define LOCALLIB_H
+
+#include "sysfuncs.h"
+
+#include "text.h"
+
+#define NUMOF(a) (sizeof(a)/sizeof((a)[0]))
+
+int vecentropy (int *A, int n);
+
+int huffcodebits (unsigned long *A, int n);
+
+int modelbits (unsigned long *A, int n);
+
+int prime (int p);
+
+
+int Read_cdh (FILE * f, compression_dict_header * cdh, u_long * mem, u_long * disk);
+
+int Read_cfh (FILE * f, comp_frags_header * cfh, u_long * mem, u_long * disk);
+
+
+#endif
Index: /trunk/gsdl/src/mgpp/text/mg.h
===================================================================
--- /trunk/gsdl/src/mgpp/text/mg.h	(revision 856)
+++ /trunk/gsdl/src/mgpp/text/mg.h	(revision 856)
@@ -0,0 +1,42 @@
+/**************************************************************************
+ *
+ * mg.h -- General header file
+ * Copyright (C) 1994  Neil Sharman, Alistair Moffat and Lachlan Andrew
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef H_MG
+#define H_MG
+
+
+#define COMPALLOK	(0)
+#define COMPERROR	(-1)
+
+#define MAXSTRING       3000000
+#define BUFFERSIZE      (MAXSTRING+30000)
+
+
+
+
+#define	MAXLN	65536
+#define WORDS           1
+#define NONWORDS        0
+
+#endif
Index: /trunk/gsdl/src/mgpp/text/mg.special.cpp
===================================================================
--- /trunk/gsdl/src/mgpp/text/mg.special.cpp	(revision 856)
+++ /trunk/gsdl/src/mgpp/text/mg.special.cpp	(revision 856)
@@ -0,0 +1,81 @@
+/**************************************************************************
+ *
+ * mg.special.cpp -- Special pass for mg_passes
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "mg.h"
+#include "build.h"
+
+
+/*
+   $Log$
+   Revision 1.1  2000/01/14 02:26:10  sjboddie
+   Rodgers new C++ mg
+
+   Revision 1.1  1999/10/11 02:57:45  cs025
+   Base install of MG-PP
+
+   Revision 1.1  1999/08/10 21:18:03  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:34:51  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:53  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:44  tes
+   * For version 1.1
+   *
+ */
+
+
+int init_special (const TagInfo &/*tagInfo*/, char * /*FileName*/) {
+  return COMPALLOK;
+}
+
+int process_special (const TagInfo &/*tagInfo*/, const TextElArray &doc) {
+  // print out the document
+  TextElArray::const_iterator here = doc.begin();
+  TextElArray::const_iterator end = doc.end();
+  while (here != end) {
+    if ((*here).elType == OpenTagE) {
+      cout << "** OpenTag  \"" << (*here).tagName << "\"\tText \""
+	   << (*here).text << "\"\n"; 
+    } else if ((*here).elType == CloseTagE) {
+      cout << "** CloseTag \"" << (*here).tagName << "\"\tText \""
+	   << (*here).text << "\"\n";
+    } else {
+      cout << "** Text     \"" << (*here).text << "\"\n";
+    }
+    
+    here++;
+  }
+  
+  return COMPALLOK;
+}
+
+int done_special (const TagInfo &/*tagInfo*/, char * /*FileName*/) {
+  return COMPALLOK;
+}
Index: /trunk/gsdl/src/mgpp/text/mg_compression_dict.1
===================================================================
--- /trunk/gsdl/src/mgpp/text/mg_compression_dict.1	(revision 856)
+++ /trunk/gsdl/src/mgpp/text/mg_compression_dict.1	(revision 856)
@@ -0,0 +1,189 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_compression_dict 1 \*(Dt CITRI
+.SH NAME
+mg_compression_dict \- build a compression dictionary.
+.SH SYNOPSIS
+.B mg_compression_dict
+[
+.B \-h
+]
+[
+.BR \-C " |"
+.BR \-P " |"
+.B  \-S
+]
+.if n .ti +9n
+[
+.BR \-0 " |"
+.BR \-1 " |"
+.BR \-2 " |"
+.B  \-3
+]
+[
+.BR \-H " |"
+.BR \-B " |"
+.BR \-D " |"
+.BR \-Y " |"
+]
+.if n .ti +9n
+.if t .ti +.5i
+[
+.BI \-l " lookback"
+]
+[
+.BI \-k " mem"
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_compression_dict
+builds a compression dictionary based on the statistics gathered
+during the first pass over the text.  The options to the program are
+mainly concerned with limiting the amount of memory the dictionary
+will use and with how the text compressor will cope with any novel
+words found during the compression phase.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-C
+Build a complete dictionary from the statistics file.  If during the
+text compression phase a novel word is found, then the compressor will
+produce an error message and stop.
+.TP
+.B \-P
+Build a partial dictionary from the statistics file.  This dictionary
+assumes that the statistics file are based on the entire text.  The
+statistics of words not includes in the dictionary are used to
+calculate the escape probability.  If novel words are being coded
+character by character, then there may not be a Huffman code for every
+possible character.  This means that the compressor may fail if a novel
+word contains a novel character.
+.TP
+.B \-S
+Build a seed dictionary from the statistics file.  This dictionary
+assumes that the statistics file is based on only a portion of the
+text to be compressed.  The probability of a novel word is based on the
+number of words that have only occurred once.  If novel words are being
+coded character by character, then the Huffman codes for characters are
+based on the frequency of characters in the dictionary.
+.TP
+.B \-0
+All words from the statistics file are included in the built
+dictionary.
+.TP
+.B \-1
+Words are included in the dictionary until the dictionary reaches the
+desired size.  Words are selected for the dictionary based on the order
+they occurred in the source text.
+.TP
+.B \-2
+Words are included in the dictionary until the dictionary reaches the
+desired size.  The most frequent words are included in the dictionary
+first; where there is a tie for frequency, the shortest word is
+included first.
+.TP
+.B \-3
+Words are included in the dictionary until the dictionary reaches the
+desired size.  The most frequent words are included in the dictionary
+first; where there is a tie for frequency, the shortest word is
+included first.  Words are the shuffled back and forth between the
+`keep' and `discard' lists to find the `optimal' set of words that
+should be in the dictionary.
+.TP
+.B \-H
+This specifies that novel words will be coded character by character
+using Huffman codes.
+.TP
+.B \-B
+This specifies that an auxiliary dictionary will be built by the
+compressor.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using binary codes.  The binary code represents their occurrence
+position in the auxiliary dictionary.
+.TP
+.B \-D
+This specifies that an auxiliary dictionary will be built by the
+compressor.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using delta codes.  The delta code represents their occurrence position
+in the auxiliary dictionary.
+.TP
+.B \-Y
+This specifies that an auxiliary dictionary will be built by the
+compressor.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using a combination of gamma and binary codes.  The code represents
+their occurrence position in the auxiliary dictionary.  This generally
+produces better compression than
+.B \-B
+or
+.BR \-D .
+.TP
+.BI \-l " lookback"
+The generated dictionary is designed to be front coded when it is
+loaded into memory.  Under normal circumstances, a front-coded
+dictionary would require scanning from the beginning in order to find
+any particular word.  However, every
+.I lookback
+words in the dictionary, the whole word is stored and a pointer to that
+word maintained.  E.g., if
+.I lookback
+is 4, then every fourth word is stored in its entirety.
+.TP
+.BI \-k " mem"
+This limits the amount of memory to use for the generated
+dictionary.  Words are selected for the dictionary based of the text
+statistics, and whether
+.BR \-0 , " \-1" , " \-2"
+or
+.B \-3
+is specified.  The memory is calculated assuming a lookback of 0,
+irrespective of what actual lookback is specified.  This means that if
+a non-zero lookback is given, the dictionary will actually occupy
+less space than specified by
+.BR \-k .
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the mg
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.text.stats
+Statistics about the source text.
+.TP
+.B *.text.dict
+Compression dictionary for the source text.
+.SH "SEE ALSO"
+.na
+.BR mg_fast_comp_dict (1),
+.BR mg_invf_dict (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_stem_idx (1),
+.BR mg_weights_build (1)
Index: /trunk/gsdl/src/mgpp/text/mg_compression_dict.cpp
===================================================================
--- /trunk/gsdl/src/mgpp/text/mg_compression_dict.cpp	(revision 856)
+++ /trunk/gsdl/src/mgpp/text/mg_compression_dict.cpp	(revision 856)
@@ -0,0 +1,1054 @@
+/**************************************************************************
+ *
+ * mg_compression_dict.cpp -- Routines for creating compression dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "local_strings.h"
+#include "bitio_gen.h"
+#include "bitio_m_stdio.h"
+#include "mgheap.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "locallib.h"
+#include "invf.h"
+#include "text.h"
+#include "words.h"
+#include "mg.h"
+#include "WordData.h"
+
+#define MAXBITS (sizeof(unsigned long) * 8)
+
+/*
+   $Log$
+   Revision 1.1  2000/01/14 02:26:11  sjboddie
+   Rodgers new C++ mg
+
+   Revision 1.3  1999/10/17 23:43:25  cs025
+   Changes to eradicate Xmalloc
+
+   Revision 1.2  1999/10/11 22:03:03  cs025
+   Changed mistaken use of NTOHUL on a double to NTOHD as per report
+   from Craig Neville-Manning of problems installing mg/gsdl.
+
+   Revision 1.1  1999/10/11 02:57:46  cs025
+   Base install of MG-PP
+
+   Revision 1.1  1999/08/10 21:18:04  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.4  1999/01/15 03:05:51  rjmcnab
+
+   Renamed lib/heap to be lib/mgheap (it was causing some problems with
+   some versions of the STL libraries which contained a heap.h)
+
+   Revision 1.3  1998/12/17 09:12:52  rjmcnab
+
+   Altered mg to process utf-8 encoded Unicode. The main changes
+   are in the parsing of the input, the casefolding, and the stemming.
+
+   Revision 1.2  1998/11/25 07:55:44  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:34:52  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:54  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:45  tes
+   * For version 1.1
+   *
+ */
+
+/* #define DEBUG1 */
+
+/* #define DEBUG */
+
+#define is_power_of_two(a) ((a) != 0 && (((a) & ((a)-1)) == 0))
+
+#define MAX_RECALCULATIONS 100
+
+typedef struct DictWordData : public WordData
+  {
+    float saving;
+    float char_bit_cost;
+    u_long num_trans;
+    u_char *word;
+  }
+DictWordData;
+
+static DictWordData *Words[2];
+static u_long Num[2];
+static u_long chars[2];
+
+#define KIND(p) (((p) >= Words[0] && (p) < Words[0] + Num[0]) ? 0 : 1)
+#define IsWord(p) (((p) >= Words[1] && (p) < Words[1] + Num[1]) ? 1 : 0)
+#define IsNonWord(p) (((p) >= Words[0] && (p) < Words[0] + Num[0]) ? 1 : 0)
+
+
+typedef struct DictData
+  {
+    DictWordData **wd;
+    u_long num_wds;
+    u_long chars;
+  }
+DictData;
+
+typedef DictData DictInfo[2];
+
+static DictInfo keep, discard, all;
+static compression_stats_header csh;
+
+static char *file_name = "";
+static u_long novel_method = MG_NOVEL_HUFFMAN_CHARS;
+
+
+static void ReadInWords (char *);
+static void Select_on (int k, heap_comp hc);
+static void Method3 (int k);
+static void Select_all (void);
+static u_long WriteOutWords (char *, u_long, int);
+static int DecFreqIncWL (void *a, void *b);
+static int OccuranceOrder (void *a, void *b);
+
+
+
+int main (int argc, char **argv)
+{
+  int ch;
+  char type = 'C';
+  char mode = '0';
+  int lookback = 2;
+  double k = 0;
+  u_long mem_reqd;
+  opterr = 0;
+  msg_prefix = argv[0];
+  while ((ch = getopt (argc, argv, "0123CPSf:d:l:hk:HDY")) != -1)
+    switch (ch)
+      {
+      case 'H':
+	novel_method = MG_NOVEL_HUFFMAN_CHARS;
+	break;
+      case 'D':
+	novel_method = MG_NOVEL_DELTA;
+	break;
+      case 'Y':
+	novel_method = MG_NOVEL_HYBRID;
+	break;
+      case 'f':		/* input file */
+	file_name = optarg;
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 'C':
+      case 'P':
+      case 'S':
+	type = ch;
+	break;
+      case '0':
+      case '1':
+      case '2':
+      case '3':
+	mode = ch;
+	break;
+      case 'l':
+	lookback = atoi (optarg);
+	if (!is_power_of_two (lookback))
+	  FatalError (1, "The lookback value must be a power of 2");
+	lookback = floorlog_2 (lookback);
+	break;
+      case 'k':
+	k = atof (optarg) * 1024;
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-l lookback] [-f input_file]"
+	"[-d data directory] [-h] [-0|-1|-2|-3] [-C|-P|-S] [-k mem (Kb)]\n",
+		 argv[0]);
+	exit (1);
+      }
+
+  ReadInWords (file_name);
+
+  if (type == 'C')
+    {
+      Select_all ();
+      mem_reqd = WriteOutWords (file_name, MG_COMPLETE_DICTIONARY, lookback);
+    }
+  else
+    {
+      switch (mode)
+	{
+	case '0':
+	  Select_all ();
+	  break;
+	case '1':
+	  Message ("Dictionary limit of %.2f Kb", k / 1024);
+	  Select_on ((int) k, OccuranceOrder);
+	  break;
+	case '2':
+	  Message ("Dictionary limit of %.2f Kb", k / 1024);
+	  Select_on ((int) k, DecFreqIncWL);
+	  break;
+	case '3':
+	  Message ("Dictionary limit of %.2f Kb", k / 1024);
+	  Method3 ((int) k);
+	  break;
+	}
+      if (type == 'P')
+	{
+	  mem_reqd = WriteOutWords (file_name, MG_PARTIAL_DICTIONARY, lookback);
+	}
+      else
+	{
+	  mem_reqd = WriteOutWords (file_name, MG_SEED_DICTIONARY, lookback);
+	}
+    }
+
+  Message ("Num words           : %8u -> %8u\n", Num[1], keep[1].num_wds);
+  Message ("Num non-words       : %8u -> %8u\n", Num[0], keep[0].num_wds);
+  Message ("Chars of words      : %8u -> %8u\n", chars[1], keep[1].chars);
+  Message ("Chars of non-words  : %8u -> %8u\n", chars[0], keep[0].chars);
+  Message ("Mem usage           : %8u -> %8u\n",
+	   (Num[0] + Num[1]) * sizeof (char *) + chars[0] + chars[1],
+	     (keep[0].num_wds + keep[1].num_wds) * sizeof (char *) +
+	   keep[0].chars + keep[1].chars);
+  Message ("Actual mem required : %8u\n", mem_reqd);
+
+  exit (0);
+}
+
+
+
+
+static void 
+ReadInWords (char *filename)
+{
+  FILE *f;
+  unsigned long i;
+  f = open_file (filename, TEXT_STATS_DICT_SUFFIX, "rb",
+		 MAGIC_STATS_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  fread (&csh, sizeof (csh), 1, f);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(csh.num_docs);
+  NTOHD(csh.num_bytes);
+
+  for (i = 0; i < 2; i++)
+    {
+      frags_stats_header fsh;
+      char *buf;
+      DictWordData *wd;
+      chars[i] = 0;
+
+      fread (&fsh, sizeof (fsh), 1, f);
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fsh.num_frags);
+      NTOHUL(fsh.mem_for_frags);
+
+      Num[i] = all[i].num_wds = fsh.num_frags;
+
+      /* The +1 on the following line is to leave room for the esc code. */
+      all[i].wd = (DictWordData **) Xmalloc (sizeof (DictWordData *) * Num[i] + 1);
+
+      buf = (char *) Xmalloc (fsh.mem_for_frags);
+      wd = Words[i] = (DictWordData *) Xmalloc (sizeof (DictWordData) * Num[i]);
+      unsigned int j;
+      for (j = 0; j < Num[i]; j++, wd++)
+	{
+	  int len;
+
+	  // read docCount and wordCount
+	  wd->read(f);
+
+	  // read an mgString
+	  len = fgetc (f);
+	  wd->word = (u_char *) buf;
+	  *buf++ = len;
+	  fread (buf, len, 1, f);
+	  buf += len;
+	  all[i].wd[j] = wd;
+	}
+      chars[i] = fsh.mem_for_frags - fsh.num_frags;
+    }
+  fclose (f);
+}
+
+
+static void 
+Alloc_keep_discard (void)
+{
+  keep[0].num_wds = 0;
+  keep[0].wd = (DictWordData **) Xmalloc ((Num[0] + 1) * sizeof (DictWordData *));
+  keep[1].num_wds = 0;
+  keep[1].wd = (DictWordData **) Xmalloc ((Num[1] + 1) * sizeof (DictWordData *));
+  discard[0].num_wds = 0;
+  discard[0].wd = (DictWordData **) Xmalloc ((Num[0] + 1) * sizeof (DictWordData *));
+  discard[1].num_wds = 0;
+  discard[1].wd = (DictWordData **) Xmalloc ((Num[1] + 1) * sizeof (DictWordData *));
+}
+
+
+static int 
+sort_comp (const void *a, const void *b)
+{
+  DictWordData *aa = *(DictWordData **) a;
+  DictWordData *bb = *(DictWordData **) b;
+  return casecompare (aa->word, bb->word);  /* [RPAP - Jan 97: Stem Index Change] */
+}
+
+
+
+static void 
+SortAndCount_DictData (DictData * dd)
+{
+  unsigned int i;
+  DictWordData **wd;
+  qsort (dd->wd, dd->num_wds, sizeof (DictWordData *), sort_comp);
+  dd->chars = 0;
+  wd = dd->wd;
+  for (i = 0; i < dd->num_wds; i++, wd++)
+    dd->chars += (*wd)->word[0];
+}
+
+
+static void 
+Select_all (void)
+{
+  unsigned int i;
+  Alloc_keep_discard ();
+  keep[0].num_wds = Num[0];
+  for (i = 0; i < Num[0]; i++)
+    keep[0].wd[i] = Words[0] + i;
+  keep[1].num_wds = Num[1];
+  for (i = 0; i < Num[1]; i++)
+    keep[1].wd[i] = Words[1] + i;
+  SortAndCount_DictData (&keep[0]);
+  SortAndCount_DictData (&keep[1]);
+}
+
+
+
+
+static int 
+DecFreqIncWL (void *a, void *b)
+{
+  DictWordData *aa = *(DictWordData **) a;
+  DictWordData *bb = *(DictWordData **) b;
+  if (aa->docCount > bb->docCount)
+    return -1;
+  else if (aa->docCount < bb->docCount)
+    return 1;
+  else
+    return bb->word[0] - aa->word[0];
+}
+
+
+static int 
+OccuranceOrder (void *a, void *b)
+{
+  DictWordData *aa = *(DictWordData **) a;
+  DictWordData *bb = *(DictWordData **) b;
+  if (aa->words() > bb->words())
+    return 1;
+  else if (aa->words() < bb->words())
+    return -1;
+  else
+    return 0;
+}
+
+
+static void 
+Select_on (int k, heap_comp hc)
+{
+  int i, num, mem;
+  DictWordData **wd;
+
+  Alloc_keep_discard ();
+
+  num = Num[0] + Num[1];
+  wd = (DictWordData **) Xmalloc (num * sizeof (DictWordData *));
+  for (i = 0; (unsigned int)i < Num[0]; i++)
+    wd[i] = Words[0] + i;
+  for (i = 0; (unsigned int)i < Num[1]; i++)
+    wd[i + Num[0]] = Words[1] + i;
+
+  heap_build (wd, sizeof (*wd), num, hc);
+
+  mem = 0;
+  while (k > mem && num)
+    {
+      int idx;
+      DictWordData *word = wd[0];
+#ifdef DEBUG
+      fprintf (stderr, "%4d:%4d:%8d :%8d :%8d : \"%s\"\n",
+	       keep[0].num_wds, keep[1].num_wds,
+	       mem, word->documents(), word->word, word2str (word->word));
+#endif
+      mem += sizeof (u_char *) + word->word[0];
+      heap_deletehead (wd, sizeof (*wd), &num, hc);
+      idx = KIND (word);
+      keep[idx].wd[keep[idx].num_wds++] = word;
+    }
+
+  for (i = 0; i < num; i++)
+    {
+      DictWordData *word = wd[i];
+      int idx = KIND (word);
+      discard[idx].wd[discard[idx].num_wds++] = word;
+    }
+  SortAndCount_DictData (&keep[0]);
+  SortAndCount_DictData (&keep[1]);
+  SortAndCount_DictData (&discard[0]);
+  SortAndCount_DictData (&discard[1]);
+
+  assert (keep[0].num_wds + discard[0].num_wds == Num[0]);
+  assert (keep[1].num_wds + discard[1].num_wds == Num[1]);
+
+}
+
+
+
+static int 
+BigSaving (void *a, void *b)
+{
+  DictWordData *aa = *(DictWordData **) a;
+  DictWordData *bb = *(DictWordData **) b;
+  return (aa->saving > bb->saving) ? -1 : (aa->saving < bb->saving);
+}
+
+static int 
+SmallSaving (void *a, void *b)
+{
+  DictWordData *aa = *(DictWordData **) a;
+  DictWordData *bb = *(DictWordData **) b;
+  return (aa->saving < bb->saving) ? -1 : (aa->saving > bb->saving);
+}
+
+
+static void 
+CalcCharCounts (DictWordData ** wd, int num,
+		char *char_lens[2], char *len_lens[2],
+		u_long escape[2])
+{
+  long char_freqs[2][256];
+  long len_freqs[2][16];
+  huff_data hd;
+  int i;
+  escape[0] = 0;
+  escape[1] = 0;
+  bzero ((char *) char_freqs, sizeof (char_freqs));
+  bzero ((char *) len_freqs, sizeof (len_freqs));
+  for (i = 0; i < num; i++, wd++)
+    {
+      u_long freq = (*wd)->documents();
+      u_char *buf = (*wd)->word;
+      int len = *buf++;
+      int idx = KIND (*wd);
+      len_freqs[idx][len] += freq;
+      escape[idx] += freq;
+      for (; len; len--, buf++)
+	char_freqs[idx][(u_long) (*buf)] += freq;
+    }
+  Generate_Huffman_Data (256, char_freqs[0], &hd, NULL);
+  char_lens[0] = hd.clens;
+  Generate_Huffman_Data (256, char_freqs[1], &hd, NULL);
+  char_lens[1] = hd.clens;
+
+  Generate_Huffman_Data (16, len_freqs[0], &hd, NULL);
+  len_lens[0] = hd.clens;
+  Generate_Huffman_Data (16, len_freqs[1], &hd, NULL);
+  len_lens[1] = hd.clens;
+}
+
+
+
+
+
+inline void CalcBitCostForWordData(DictWordData **word, int num,
+				   double freqs_trans[2], u_long escape[2],
+				   char * char_lens[2], char *len_lens[2], 
+				   double esc[2], int num_trans)
+{ 
+  int j;
+
+  for (j = 0; j < num; j++, word++)
+    {
+      float   cbc, wbc;
+      u_char *buf = (*word)->word;
+      int     len = *buf++;
+      u_long  freq = (*word)->documents();
+      int     idx = KIND (*word);
+
+      cbc = len_lens[idx][len];
+      for (; len; len--, buf++)
+	cbc += char_lens[idx][(u_long) (*buf)];
+
+      (*word)->char_bit_cost = (cbc + esc[idx]) * freq;
+
+      wbc = -log2 (freq / (freqs_trans[idx] + escape[idx])) * freq;
+
+      (*word)->saving = ((*word)->char_bit_cost - wbc) /
+	(sizeof (char *) + (*word)->word[0]);
+
+      (*word)->num_trans = num_trans;
+    }
+}
+
+
+
+void 
+CalcBitCost (DictWordData ** discard_word, int discard_num,
+	     DictWordData ** keep_word, int keep_num, double freqs_trans[2],
+	     u_long escape[2], int num_trans)
+{
+  char *char_lens[2];
+  char *len_lens[2];
+  double esc[2];
+  CalcCharCounts (discard_word, discard_num, char_lens, len_lens, escape);
+  esc[0] = -log2 (escape[0] / (freqs_trans[0] + escape[0]));
+  esc[1] = -log2 (escape[1] / (freqs_trans[1] + escape[1]));
+  CalcBitCostForWordData(discard_word, discard_num, freqs_trans, escape,
+			 char_lens, len_lens, esc, num_trans);
+  CalcBitCostForWordData(keep_word, keep_num, freqs_trans, escape,
+			 char_lens, len_lens, esc, num_trans);
+  Xfree (char_lens[0]);
+  Xfree (char_lens[1]);
+  Xfree (len_lens[0]);
+  Xfree (len_lens[1]);
+}
+
+inline void m3_transferWord(DictWordData **toHeap, int *toNum, heap_comp toSaving,
+			    DictWordData **fromHeap, int *fromNum, heap_comp fromSaving,
+			    int *num_trans, double freqs_trans[], int *mem, double *total,
+			    int *count)
+{
+  /* Transfer the word at the top of the keep heap to the
+     discard heap. */
+  DictWordData *word = fromHeap[0];
+  int idx = KIND (word);
+  heap_deletehead (fromHeap, sizeof (word), fromNum, fromSaving);
+  toHeap[*toNum] = word;
+  heap_additem (toHeap, sizeof (word), toNum, toSaving);
+  freqs_trans[idx] -= word->documents();
+  *mem = (*mem) - sizeof (u_char *) + word->word[0];
+  *num_trans += 1;
+  *total += word->saving;
+  *count += 1;
+#ifdef DEBUG
+  fprintf (stderr,
+	   "KEEP -> DISCARD %8d :%8d :%8.0f :%8.0f : \"%s\"\n",
+	   *mem, word->documents(), word->char_bit_cost,
+	   word->saving, word2str (word->word));
+#endif
+}
+
+inline void m3_storeWord(DictWordData **wordHeap, int wordNum, int num_trans, 
+			 double freqs_trans[], u_long escape[], heap_comp hc)
+{
+  DictWordData *word = wordHeap[0];
+  int idx = KIND (word);
+  float wbc;
+#ifdef DEBUG1
+  fprintf (stderr, "KEEP \"%s\" %.2f ->", word2str (word->word),
+	   word->saving);
+#endif
+  wbc = -log2 (word->documents() / (freqs_trans[idx] + escape[idx])) *
+    word->documents();
+  word->saving = (word->char_bit_cost - wbc) /
+    (sizeof (char *) + word->word[0]);
+#ifdef DEBUG1
+  fprintf (stderr, " %.2f\n", word->saving);
+#endif
+  word->num_trans = num_trans;
+  heap_changedhead (wordHeap, sizeof (word), wordNum, hc);
+}
+
+
+static void 
+Method3 (int k)
+{
+  int i, keep_num, discard_num, mem, num_trans, recalc_reqd;
+  int keep_to_discard = 0;
+  int discard_to_keep = 0;
+  int recalcs = 0;
+  double freqs_trans[2], total;
+  u_long escape[2];
+  DictWordData **keep_heap, **discard_heap;
+
+  freqs_trans[0] = freqs_trans[1] = 0;
+  num_trans = 0;
+
+  discard_num = Num[0] + Num[1];
+  discard_heap = (DictWordData **) Xmalloc (discard_num * sizeof (DictWordData *));
+
+  keep_num = 0;
+  keep_heap = (DictWordData **) Xmalloc (discard_num * sizeof (DictWordData *));
+
+
+  for (i = 0; (unsigned int)i < Num[0]; i++)
+    discard_heap[i] = Words[0] + i;
+  for (i = 0; (unsigned int)i < Num[1]; i++)
+    discard_heap[i + Num[0]] = Words[1] + i;
+
+  heap_build (discard_heap, sizeof (*discard_heap), discard_num, DecFreqIncWL);
+
+  mem = 0;
+  while (k > mem && discard_num)
+    {
+      DictWordData *word = discard_heap[0];
+      mem += sizeof (u_char *) + word->word[0];
+      heap_deletehead (discard_heap, sizeof (word), &discard_num, DecFreqIncWL);
+      keep_heap[keep_num++] = word;
+      freqs_trans[KIND (word)] += word->documents();
+      num_trans++;
+    }
+
+  CalcBitCost (discard_heap, discard_num, keep_heap, keep_num,
+	       freqs_trans, escape, num_trans);
+  heap_build (discard_heap, sizeof (*discard_heap), discard_num, BigSaving);
+  heap_build (keep_heap, sizeof (*keep_heap), keep_num, SmallSaving);
+
+  total = 0;
+  recalc_reqd = 0;
+  while (keep_num && discard_num)
+    {
+      if ((keep_num && keep_heap[0]->num_trans < (unsigned)num_trans) ||
+	  (discard_num && discard_heap[0]->num_trans < (unsigned)num_trans))
+	{
+	  if (keep_num && keep_heap[0]->num_trans < (unsigned)num_trans)
+	    {
+	      m3_storeWord(keep_heap, keep_num, num_trans, freqs_trans, escape, SmallSaving);
+	    }
+
+	  if (discard_num && discard_heap[0]->num_trans < (unsigned)num_trans)
+	    {
+	      m3_storeWord(discard_heap, discard_num, num_trans, freqs_trans, escape, BigSaving);
+	    }
+	}
+      else if (keep_heap[0]->saving < discard_heap[0]->saving)
+	{
+	  assert (keep_num && discard_num);
+	  if (keep_num && mem + sizeof (char *) + discard_heap[0]->word[0] > (unsigned)k)
+	    {
+	      /* Transfer the word at the top of the keep heap to the
+	         discard heap. */
+	      m3_transferWord(discard_heap, &discard_num, BigSaving,
+			      keep_heap, &keep_num, SmallSaving,
+			      &num_trans, freqs_trans, &mem, &total, &keep_to_discard);
+	    }
+	  else
+	    {
+	      /* Transfer the word at the top of the discard heap to the
+	         keep heap. */
+	      m3_transferWord(keep_heap, &keep_num, SmallSaving,
+			      discard_heap, &discard_num, BigSaving,
+			      &num_trans, freqs_trans, &mem, &total, &discard_to_keep);
+	    }
+
+	  recalc_reqd = 1;
+
+	}
+      else
+	{
+	  if (recalc_reqd == 0)
+	    break;
+#ifdef DEBUG1
+	  fprintf (stderr, "--------------\n");
+#endif
+	  if (recalcs == MAX_RECALCULATIONS)
+	    break;
+	  CalcBitCost (discard_heap, discard_num, keep_heap, keep_num,
+		       freqs_trans, escape, num_trans);
+	  heap_build (discard_heap, sizeof (*discard_heap),
+		      discard_num, BigSaving);
+	  heap_build (keep_heap, sizeof (keep_heap), keep_num, SmallSaving);
+	  recalc_reqd = 0;
+	  recalcs++;
+	}
+    }
+
+  Alloc_keep_discard ();
+
+  for (i = 0; i < discard_num; i++)
+    {
+      DictWordData *word = discard_heap[i];
+      int idx = KIND (word);
+      assert (IsWord (word) || IsNonWord (word));
+      discard[idx].wd[discard[idx].num_wds++] = word;
+    }
+  for (i = 0; i < keep_num; i++)
+    {
+      DictWordData *word = keep_heap[i];
+      int idx = KIND (word);
+      assert (IsWord (word) || IsNonWord (word));
+      keep[idx].wd[keep[idx].num_wds++] = word;
+    }
+  SortAndCount_DictData (&keep[0]);
+  SortAndCount_DictData (&keep[1]);
+  SortAndCount_DictData (&discard[0]);
+  SortAndCount_DictData (&discard[1]);
+
+  assert (keep[0].num_wds + discard[0].num_wds == Num[0]);
+  assert (keep[1].num_wds + discard[1].num_wds == Num[1]);
+  Message ("Keep -> Discard        : %8d", keep_to_discard);
+  Message ("Discard -> Keep        : %8d", discard_to_keep);
+  Message ("Huffman Recalculations : %8d", recalcs);
+  if (recalcs == MAX_RECALCULATIONS)
+    Message ("WARNING: The number of recalculations == %d", MAX_RECALCULATIONS);
+}
+
+
+
+
+
+
+
+
+
+
+/****************************************************************************
+ *****									*****
+ *****			Dictionary saving code				*****
+ *****									*****
+ ****************************************************************************/
+
+
+
+static void 
+Write_cdh (FILE * f, compression_dict_header * cdh)
+{
+  /* [RPAP - Jan 97: Endian Ordering] */
+  int i;
+  compression_dict_header tmp = *cdh;
+  HTONUL(tmp.dict_type);
+  HTONUL(tmp.novel_method);
+  for (i = 0; i < TEXT_PARAMS; i++)
+    HTONUL(tmp.params[i]);
+  HTONUL(tmp.num_words[0]);
+  HTONUL(tmp.num_words[1]);
+  HTONUL(tmp.num_word_chars[0]);
+  HTONUL(tmp.num_word_chars[1]);
+  HTONUL(tmp.lookback);
+
+  fwrite (&tmp, sizeof (tmp), 1, f);
+}
+
+
+static void 
+Write_words (FILE * f, DictData * dd)
+{
+  unsigned int i;
+  u_char *curr, *prev = NULL;
+  for (i = 0; i < dd->num_wds; i++)
+    {
+      int len;
+      curr = dd->wd[i]->word;
+      if (prev)
+	/* look for prefix match with prev string */
+	len = prefixlen (prev, curr);
+      else
+	len = 0;
+      fputc ((len << 4) + (curr[0] - len), f);
+      fwrite (curr + len + 1, sizeof (u_char), curr[0] - len, f);
+      prev = curr;
+    }
+
+}
+
+
+static int 
+Uncompressed_size (DictData * dd)
+{
+  unsigned int i, us;
+  for (us = i = 0; i < dd->num_wds; i++)
+    us += dd->wd[i]->word[0];
+  return us;
+}
+
+
+static u_long 
+Write_data (FILE * f, DictData * dd, int lookback)
+{
+  u_long mem_reqd;
+  huff_data *hd;
+  int i; 
+  u_long us = dd->num_wds;
+  long *freqs;
+  u_long huff_words_size[MAX_HUFFCODE_LEN + 1];
+  u_long lencounts[MAX_HUFFCODE_LEN + 1];
+  u_char *lastword[MAX_HUFFCODE_LEN + 1];
+
+  if (!(freqs = new long [dd->num_wds]))
+    FatalError (1, "Unable to allocate memory for freqs");
+
+  for (i = 0; (unsigned)i < dd->num_wds; i++)
+    {
+      freqs[i] = dd->wd[i]->documents();
+      us += dd->wd[i]->word[0];
+    }
+
+  if (!(hd = Generate_Huffman_Data (dd->num_wds, freqs, NULL, NULL)))
+    FatalError (1, "Unable to allocate memory for huffman data");
+
+  delete (freqs);
+  freqs = NULL;
+
+  if (Write_Huffman_Data (f, hd) == -1)
+    FatalError (1, "Unable to write huffman data");
+
+  HTONUL(us);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite (&us, sizeof (us), 1, f);
+  NTOHUL(us);  /* [RPAP - Jan 97: Endian Ordering] */
+
+
+/* Calculate the amount of memory that will be required to store the text for
+   each different huffman code len. Every 1<<lookback words for each different
+   codelen length will not be prefixed by previous strings. */
+
+
+  bzero ((char *) &huff_words_size, sizeof (huff_words_size));
+  bzero ((char *) &lencounts, sizeof (lencounts));
+
+  mem_reqd = 0;
+
+  for (i = 0; (unsigned)i < dd->num_wds; i++)
+    {
+      int codelen = hd->clens[i];
+      u_char *word = dd->wd[i]->word;
+
+      if (!codelen)
+	FatalError (1, "The length of a code for a word was zero");
+
+      huff_words_size[codelen] += word[0] + 1;
+      mem_reqd += word[0] + (lookback != 0);
+#if 0
+      if ((lencounts[codelen] & ((1 << lookback) - 1)) == 0)
+	lastword[codelen] = word;
+      else
+	huff_words_size[codelen] -= prefixlen (lastword[codelen], word);
+#else
+      if ((lencounts[codelen] & ((1 << lookback) - 1)) != 0)
+	{
+	  int save = prefixlen (lastword[codelen], word);
+	  mem_reqd -= save;
+	  huff_words_size[codelen] -= save;
+	}
+      else
+	{
+	  mem_reqd += sizeof (u_char *);
+	}
+      lastword[codelen] = word;
+#endif
+      lencounts[codelen]++;
+    }
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = hd->mincodelen; i < hd->maxcodelen + 1; i++)
+    HTONUL(huff_words_size[i]);
+
+  fwrite (huff_words_size + hd->mincodelen, sizeof (*huff_words_size),
+	  hd->maxcodelen - hd->mincodelen + 1, f);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = hd->mincodelen; i < hd->maxcodelen + 1; i++)
+    NTOHUL(huff_words_size[i]);
+
+  Write_words (f, dd);
+
+  delete hd->clens;
+  delete hd;
+
+  return mem_reqd;
+}
+
+
+
+static void 
+Write_charfreqs (FILE * f, DictData * dd, int words,
+		 int zero_freq_permitted)
+{
+  unsigned int j;
+  long freqs[256];
+  DictWordData **wd = dd->wd;
+  huff_data *hd;
+
+  bzero ((char *) freqs, sizeof (freqs));
+
+  for (j = 0; j < dd->num_wds; j++, wd++)
+    {
+      u_char *buf = (*wd)->word;
+      int len = *buf++;
+      for (; len; len--, buf++)
+	freqs[(u_long) (*buf)] += (*wd)->documents();
+    }
+
+  if (!zero_freq_permitted)
+    for (j = 0; j < 256; j++)
+      if (!freqs[j] && PESINAWORD (j) == words)
+	freqs[j] = 1;
+
+  if (!(hd = Generate_Huffman_Data (256, freqs, NULL, NULL)))
+    FatalError (1, "Unable to allocate memory for huffman data");
+
+  if (Write_Huffman_Data (f, hd) == -1)
+    FatalError (1, "Unable to write huffman data");
+
+  delete hd->clens;
+  delete hd;
+}
+
+
+
+
+static void 
+Write_wordlens (FILE * f, DictData * dd, int zero_freq_permitted)
+{
+  unsigned int j;
+  long freqs[16];
+  DictWordData **wd = dd->wd;
+  huff_data *hd;
+
+  bzero ((char *) freqs, sizeof (freqs));
+
+  for (j = 0; j < dd->num_wds; j++, wd++)
+    freqs[(*wd)->word[0]] += (*wd)->documents();
+
+  if (!zero_freq_permitted)
+    for (j = 0; j < 16; j++)
+      if (!freqs[j])
+	freqs[j] = 1;
+
+  if (!(hd = Generate_Huffman_Data (16, freqs, NULL, NULL)))
+    FatalError (1, "Unable to allocate memory for huffman data");
+
+  if (Write_Huffman_Data (f, hd) == -1)
+    FatalError (1, "Unable to write huffman data");
+
+
+  delete hd->clens;
+  delete hd;
+}
+
+
+
+static u_long 
+WriteOutWords (char *file_name, u_long type, int lookback)
+{
+  FILE *f;
+  int i;
+  u_long mem_reqd = 0;
+
+  compression_dict_header cdh;
+  f = create_file (file_name, TEXT_DICT_SUFFIX, "w+b",
+		   MAGIC_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  bzero((char *) &cdh, sizeof(cdh));
+
+  cdh.dict_type = type;
+  cdh.novel_method = (type == MG_SEED_DICTIONARY) ? novel_method :
+    MG_NOVEL_HUFFMAN_CHARS;
+
+  cdh.num_words[1] = keep[1].num_wds;
+  cdh.num_words[0] = keep[0].num_wds;
+  cdh.num_word_chars[1] = Uncompressed_size (&keep[1]);
+  cdh.num_word_chars[0] = Uncompressed_size (&keep[0]);
+  cdh.lookback = lookback;
+
+  Write_cdh (f, &cdh);
+
+  for (i = 0; i < 2; i++)
+    switch (type)
+      {
+      case MG_COMPLETE_DICTIONARY:
+	{
+	  mem_reqd += Write_data (f, &keep[i], lookback);
+	}
+	break;
+      case MG_PARTIAL_DICTIONARY:
+	{
+	  if (keep[i].num_wds)
+	    {
+	      int j;
+	      DictWordData esc;
+	      esc.docCount = 0;
+	      esc.word = (u_char *) "";
+	      keep[i].wd[keep[i].num_wds++] = &esc;
+	      for (j = 0; (unsigned)j < discard[i].num_wds; j++)
+		esc.docCount += discard[i].wd[j]->documents();
+	      if (!esc.docCount)
+		esc.docCount++;
+	      mem_reqd += Write_data (f, &keep[i], lookback);
+	    }
+	  Write_charfreqs (f, &discard[i], i, 1);
+	  Write_wordlens (f, &discard[i], 1);
+	}
+	break;
+      case MG_SEED_DICTIONARY:
+	{
+	  if (keep[i].num_wds)
+	    {
+	      int j;
+	      DictWordData esc;
+	      esc.docCount = 0;
+	      esc.word = (u_char *) "";
+	      keep[i].wd[keep[i].num_wds++] = &esc;
+	      for (j = 0; (unsigned)j < all[i].num_wds; j++)
+		if (all[i].wd[j]->documents() == 1)
+		  esc.docCount++;
+	      if (!esc.docCount)
+		esc.docCount++;
+	      mem_reqd += Write_data (f, &keep[i], lookback);
+	    }
+	  switch (novel_method)
+	    {
+	    case MG_NOVEL_HUFFMAN_CHARS:
+	      Write_charfreqs (f, &all[i], i, 0);
+	      Write_wordlens (f, &all[i], 0);
+	      break;
+	    case MG_NOVEL_DELTA:
+	      break;
+	    case MG_NOVEL_HYBRID:
+	      break;
+	    default:
+	      FatalError (1, "Bad novel method");
+	    }
+	}
+	break;
+      }
+  fclose (f);
+  return mem_reqd;
+}
Index: /trunk/gsdl/src/mgpp/text/mg_decompress_text.cpp
===================================================================
--- /trunk/gsdl/src/mgpp/text/mg_decompress_text.cpp	(revision 856)
+++ /trunk/gsdl/src/mgpp/text/mg_decompress_text.cpp	(revision 856)
@@ -0,0 +1,91 @@
+/**************************************************************************
+ *
+ * mg_decompress_text.cpp -- 
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "TextGet.h"
+#include "UCArray.h"
+
+#include "messages.h"
+#include "mg_files.h"
+
+
+int main (int argc, char **argv) {
+  int ch;
+  char *filename = "";
+  UCArray level;
+  SetCStr (level, "Document");
+
+  opterr = 0;
+  msg_prefix = argv[0];
+
+  // process the command line arguments
+  while ((ch = getopt (argc, argv, "f:d:K:h")) != -1) {
+    switch (ch) {
+    case 'f':		/* input file */
+      filename = optarg;
+      break;
+    case 'd':
+      set_basepath (optarg);
+      break;
+    case 'K':
+      SetCStr (level, optarg);
+      break;
+    case 'h':
+    case '?':
+      fprintf (stderr, "usage: %s [-h] [-K level] [-d directory] -f name\n",
+	       argv[0]);
+      exit (1);
+    }
+  }
+
+  // load up the text information
+  TextData td;
+  if (!td.LoadData (filename)) {
+    FatalError (1, "Couldn't load text information for \"%s\"", filename);
+  }
+  
+  // output each document in the level
+//    cout << td.levels << "\n";
+  
+  TextLevelInfo levelInfo = td.levels.levelInfo[level];
+  unsigned long docNum = 1;
+  UCArray docText;
+  while (docNum <= levelInfo.numEntries) {
+//      TextIdx docIdx;
+//      if (GetDocIdx (td, level, docNum, docIdx)) {
+//        cout << "doc: " << docNum << "\n";
+//        cout << docIdx;
+//      }
+    
+    if (!GetDocText (td, level, docNum, docText)) {
+      FatalError (1, "Error while trying to get document %u", docNum);
+    }
+
+    cout << docText << "\n";
+    
+    docNum++;
+  }
+  
+  return 0;
+}
Index: /trunk/gsdl/src/mgpp/text/mg_errors.cpp
===================================================================
--- /trunk/gsdl/src/mgpp/text/mg_errors.cpp	(revision 856)
+++ /trunk/gsdl/src/mgpp/text/mg_errors.cpp	(revision 856)
@@ -0,0 +1,73 @@
+/**************************************************************************
+ *
+ * mg_errors.cpp -- Error related stuff for mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "memlib.h"
+
+#include "mg_errors.h"
+
+
+
+int mg_errno = MG_NOERROR;
+
+char *mg_errorstrs[] =
+{
+  "No error",
+  "Out of memory",
+  "File \"%s\" not found",
+  "Bad magic number in \"%s\"",
+  "Error reading \"%s\"",
+  "MG_BUFTOOSMALL",
+  "Files required for level 2 and 3 inversion are missing"};
+
+
+/* [RJM 07/98: Memory Leak] -- rest of file */
+
+static char null_data[] = "";
+char *mg_error_data = null_data;
+
+
+void MgErrorData (char *s)
+{
+  /* free the current error string, unless it is the null string */
+  if ((mg_error_data != NULL) && (mg_error_data != null_data)) {
+    delete mg_error_data;
+    mg_error_data = null_data;
+  }
+
+  /* make a copy of the string */
+  mg_error_data = Xstrdup (s);
+
+  /* if the alloc failed set the error to the null string */
+  if (!mg_error_data) mg_error_data = null_data;
+}
+
+
+void MgErrorDeinit (void)
+{
+  /* free the current error string, unless it is the null string */
+  if ((mg_error_data != NULL) && (mg_error_data != null_data)) {
+    delete mg_error_data;
+    mg_error_data = null_data;
+  }
+}
Index: /trunk/gsdl/src/mgpp/text/mg_errors.h
===================================================================
--- /trunk/gsdl/src/mgpp/text/mg_errors.h	(revision 856)
+++ /trunk/gsdl/src/mgpp/text/mg_errors.h	(revision 856)
@@ -0,0 +1,39 @@
+/**************************************************************************
+ *
+ * mg_errors.h -- Error related stuff for mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+extern int mg_errno;
+extern char *mg_error_data;
+extern char *mg_errorstrs[];
+
+
+#define MG_NOERROR 0
+#define MG_NOMEM 1
+#define MG_NOFILE 2
+#define MG_BADMAGIC 3
+#define MG_READERR 4
+#define MG_BUFTOOSMALL 5
+#define MG_INVERSION 6
+
+void MgErrorData (char *s);
+void MgErrorDeinit (void);
Index: /trunk/gsdl/src/mgpp/text/mg_fast_comp_dict.1
===================================================================
--- /trunk/gsdl/src/mgpp/text/mg_fast_comp_dict.1	(revision 856)
+++ /trunk/gsdl/src/mgpp/text/mg_fast_comp_dict.1	(revision 856)
@@ -0,0 +1,70 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_fast_comp_dict 1 \*(Dt CITRI
+.SH NAME
+mg_fast_comp_dict \- build a fast loading compression dictionary
+.SH SYNOPSIS
+.B mg_fast_comp_dict
+[
+.B \-h
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_fast_comp_dict
+builds a fast-loading compression dictionary from the standard
+compression dictionary.  The new dictionary has the property that
+.BR mgquery (1)
+can load it in very quickly, making the startup time for
+.BR mgquery (1)
+much shorter.  The new dictionary is larger on disk than the original,
+but they are no different when it comes to memory usage.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the mg
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.text.dict
+Compression dictionary.
+.TP
+.B *.text.dict.fast
+Fast loading compression dictionary.
+.TP
+.B *.text.dict.aux
+Auxiliary compression dictionary.
+.SH "SEE ALSO"
+.na
+.BR mg_compression_dict (1),
+.BR mg_invf_dict (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_stem_idx (1),
+.BR mg_weights_build (1)
Index: /trunk/gsdl/src/mgpp/text/mg_fast_comp_dict.cpp
===================================================================
--- /trunk/gsdl/src/mgpp/text/mg_fast_comp_dict.cpp	(revision 856)
+++ /trunk/gsdl/src/mgpp/text/mg_fast_comp_dict.cpp	(revision 856)
@@ -0,0 +1,729 @@
+/**************************************************************************
+ *
+ * mg_fast_comp_dict.cpp -- Program to generate a fast compression dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+/*
+   $Log$
+   Revision 1.1  2000/01/14 02:26:13  sjboddie
+   Rodgers new C++ mg
+
+   Revision 1.2  1999/10/17 23:43:26  cs025
+   Changes to eradicate Xmalloc
+
+   Revision 1.1  1999/10/11 02:57:50  cs025
+   Base install of MG-PP
+
+   Revision 1.1  1999/08/10 21:18:06  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:44  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:34:57  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:55  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:47  tes
+   * For version 1.1
+   *
+ */
+
+#include "sysfuncs.h"
+
+#include "huffman.h"
+#include "messages.h"
+#include "memlib.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "local_strings.h"
+#include "mg.h"
+#include "text.h"
+#include "invf.h"
+#include "mg_files.h"
+#include "locallib.h"
+#include "words.h"
+
+
+
+#define ALIGN_SIZE(p,t) (((p) + (sizeof(t)-1)) & (~(sizeof(t)-1)))
+
+#define WORDNO(p, base) ((((char*)(p))-((char*)(base)))/sizeof(u_char*))
+#define FIXUP(p) (fixup[WORDNO(p,buffer)/8] |= (1<<(WORDNO(p, buffer) & 7)))
+
+#define IS_FIXUP(p) ((fixup[WORDNO(p,buffer)/8] & (1<<(WORDNO(p, buffer) & 7))) != 0)
+
+#define FIXUP_VALS(vals) do {						\
+	int i;								\
+	for (i=0; i < MAX_HUFFCODE_LEN+1; i++)				\
+	  FIXUP(&vals[i]);						\
+      } while(0)
+
+
+
+static u_long mem_for_comp_dict (char *filename);
+static void load_comp_dict (char *filename);
+static void save_fast_dict (char *filename);
+static void unfixup_buffer (void);
+
+
+static void *buffer;
+static void *cur;
+static u_char *fixup;
+static u_long mem, fixup_mem;
+
+int main (int argc, char **argv)
+{
+  int ch;
+  char *filename = "";
+  opterr = 0;
+  msg_prefix = argv[0];
+  while ((ch = getopt (argc, argv, "f:d:h")) != -1)
+    switch (ch)
+      {
+      case 'f':		/* input file */
+	filename = optarg;
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-f input_file] [-d data directory] [-h]\n",
+		 argv[0]);
+	exit (1);
+      }
+
+  mem = mem_for_comp_dict (filename);
+
+  fixup_mem = (ALIGN_SIZE (mem, u_char *) / sizeof (u_char *) + 7) / 8;
+
+  cur = buffer = Xmalloc (mem);
+  bzero (buffer, mem);
+  fixup = (u_char *) Xmalloc (fixup_mem);
+  bzero (fixup, fixup_mem);
+
+  Message ("Estimated memory for fast_dict %u", mem);
+  Message ("Estimated memory for fixups %u", fixup_mem);
+
+  load_comp_dict (filename);
+
+  Message ("Actual memory for fast_dict %u", (char *) cur - (char *) buffer);
+
+  if ((u_long) cur > (u_long) buffer + mem)
+    FatalError (1, "The buffer was not big enough for the dictionary");
+
+  {
+    /* [RPAP - Jan 97: Endian Ordering] */
+    compression_dict *cd = (compression_dict *) buffer;
+    int i, which;
+
+    /* cfh */
+    for (which = 0; which <= 1; which++)
+      {
+	int j;
+	
+	HTONSI(cd->cfh[which]->hd.num_codes);
+	HTONSI(cd->cfh[which]->hd.mincodelen);
+	HTONSI(cd->cfh[which]->hd.maxcodelen);
+	for (j = 0; j < MAX_HUFFCODE_LEN + 1; j++)
+	  {
+	    HTONSI(cd->cfh[which]->hd.lencount[j]);
+	    HTONUL(cd->cfh[which]->hd.min_code[j]);
+	  }
+	HTONUL(cd->cfh[which]->uncompressed_size);
+	for (j = 0; j < MAX_HUFFCODE_LEN + 1; j++)
+	  HTONUL(cd->cfh[which]->huff_words_size[j]);
+      }
+    HTONUL(cd->MemForCompDict);
+    /* ad */
+    if (cd->cdh.novel_method == MG_NOVEL_DELTA ||
+	cd->cdh.novel_method == MG_NOVEL_HYBRID)
+      for (which = 0; which <= 1; which++)
+	{
+	  int j;
+	  
+	  HTONUL(cd->ad->afh[which].num_frags);
+	  HTONUL(cd->ad->afh[which].mem_for_frags);
+	  for (j = 0; j < 33; j++)
+	    {
+	      HTONSI(cd->ad->blk_start[which][j]);
+	      HTONSI(cd->ad->blk_end[which][j]);
+	    }
+	}
+    /* cdh */
+    HTONUL(cd->cdh.dict_type);
+    HTONUL(cd->cdh.novel_method);
+    for (i = 0; i < TEXT_PARAMS; i++)
+      HTONUL(cd->cdh.params[which]);
+    HTONUL(cd->cdh.num_words[0]);
+    HTONUL(cd->cdh.num_words[1]);
+    HTONUL(cd->cdh.num_word_chars[0]);
+    HTONUL(cd->cdh.num_word_chars[1]);
+    HTONUL(cd->cdh.lookback);
+    HTONSI(cd->fast_loaded);
+  }
+
+  unfixup_buffer ();
+
+  save_fast_dict (filename);
+
+  exit (0);
+}
+
+
+
+static void 
+unfixup_buffer ()
+{
+  u_long *p;
+  for (p = (u_long *) buffer; (u_long) p < (u_long) cur; p++)
+    {
+      if (IS_FIXUP (p))
+	*p = *p - (u_long) buffer;
+    }
+}
+
+
+
+
+static u_long 
+mem_for_aux_dict (compression_dict_header * /*cdh*/, char *filename)
+{
+  int i;
+  u_long mem = sizeof (auxiliary_dict);
+  FILE *aux;
+
+  aux = open_file (filename, TEXT_DICT_AUX_SUFFIX, "rb",
+		   MAGIC_AUX_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  for (i = 0; i <= 1; i++)
+    {
+      aux_frags_header afh;
+      fread (&afh, sizeof (afh), 1, aux);
+      NTOHUL(afh.num_frags);  /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(afh.mem_for_frags);  /* [RPAP - Jan 97: Endian Ordering] */
+      mem += afh.num_frags * sizeof (u_char *);
+      mem = ALIGN_SIZE (mem + afh.mem_for_frags, u_char *);
+      fseek (aux, afh.mem_for_frags, SEEK_CUR);
+    }
+  fclose (aux);
+
+  return mem;
+}
+
+
+
+static u_long 
+mem_for_words (FILE * dict, compression_dict_header * cdh,
+	       comp_frags_header * cfh)
+{
+  u_long mem = 0;
+  long i, lookback;
+  int ptrs_reqd = 0;
+  int mem_reqd = 0;
+
+  lookback = cdh->lookback;
+
+  for (i = cfh->hd.mincodelen; i <= cfh->hd.maxcodelen; i++)
+    {
+      ptrs_reqd += (cfh->hd.lencount[i] + ((1 << lookback) - 1)) >> lookback;
+      mem_reqd += cfh->huff_words_size[i];
+    }
+
+  mem += ptrs_reqd * sizeof (u_char *);
+  mem += (MAX_HUFFCODE_LEN + 1) * sizeof (u_char **);
+  mem += mem_reqd;
+
+  for (i = 0; i < cfh->hd.num_codes; i++)
+    {
+      register int val;
+      for (val = getc (dict) & 0xf; val; val--)
+	getc (dict);
+    }
+  return ALIGN_SIZE (mem, u_char *);
+}
+
+static u_long mem_skip_hd(FILE *dict, u_long mem)
+{ huff_data hd;
+
+  mem += sizeof (hd);
+  Read_Huffman_Data (dict, &hd, NULL, NULL);
+  if (hd.clens)
+    delete hd.clens;
+  mem += hd.num_codes * sizeof (unsigned long);
+  mem += (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *);
+  return mem;
+}
+
+static u_long mem_read_cfh(FILE *dict, compression_dict_header *cdh, comp_frags_header *cfh, 
+			   u_long mem)
+{
+  mem += sizeof (comp_frags_header);
+  Read_cfh (dict, cfh, NULL, NULL);
+
+  /* Don't need to count the space for the clens of the huffman data */
+
+  mem += mem_for_words (dict, cdh, cfh);
+  if (cfh->hd.clens)
+    delete cfh->hd.clens;
+
+  return mem;
+}
+
+
+static u_long 
+mem_for_comp_dict (char *filename)
+{
+  u_long mem = sizeof (compression_dict);
+  compression_dict_header cdh;
+  comp_frags_header cfh;
+  FILE *dict;
+  int which;
+  int i;  /* [RPAP - Jan 97: Endian Ordering] */
+
+  dict = open_file (filename, TEXT_DICT_SUFFIX, "rb",
+		    MAGIC_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  fread (&cdh, sizeof (cdh), 1, dict);
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(cdh.dict_type);
+  NTOHUL(cdh.novel_method);
+  for (i = 0; i < TEXT_PARAMS; i++)
+    NTOHUL(cdh.params[i]);
+  NTOHUL(cdh.num_words[0]);
+  NTOHUL(cdh.num_words[1]);
+  NTOHUL(cdh.num_word_chars[0]);
+  NTOHUL(cdh.num_word_chars[1]);
+  NTOHUL(cdh.lookback);
+
+  for (which = 0; which < 2; which++)
+    switch (cdh.dict_type)
+      {
+      case MG_COMPLETE_DICTIONARY:
+	{
+	  mem = mem_read_cfh(dict, &cdh, &cfh, mem);
+	}
+	break;
+      case MG_PARTIAL_DICTIONARY:
+	{
+	  // huff_data hd;
+	  if (cdh.num_words[which])
+	    {
+	      mem = mem_read_cfh(dict, &cdh, &cfh, mem);
+	    }
+
+	  mem = mem_skip_hd(dict, mem);
+	  mem = mem_skip_hd(dict, mem);
+	}
+	break;
+      case MG_SEED_DICTIONARY:
+	{
+	  // huff_data hd;
+	  if (cdh.num_words[which])
+	    {
+	      mem = mem_read_cfh(dict, &cdh, &cfh, mem);
+	    }
+	  switch (cdh.novel_method)
+	    {
+	    case MG_NOVEL_HUFFMAN_CHARS:
+	      mem = mem_skip_hd(dict, mem);
+	      mem = mem_skip_hd(dict, mem);
+	      break;
+	    case MG_NOVEL_DELTA:
+	      break;
+	    case MG_NOVEL_HYBRID:
+	      break;
+	    }
+	  break;
+	}
+      }
+  fclose (dict);
+
+  if (cdh.novel_method == MG_NOVEL_DELTA ||
+      cdh.novel_method == MG_NOVEL_HYBRID)
+    mem += mem_for_aux_dict (&cdh, filename);
+
+  return ALIGN_SIZE (mem, u_char *);
+}
+
+
+
+void *
+getmem (u_long size, int align)
+{
+  void *res;
+  cur = (void *) (((u_long) cur + (align - 1)) & (~(align - 1)));
+  res = cur;
+  cur = (char *) cur + size;
+  if ((u_long) cur > (u_long)buffer + mem)
+    FatalError (1, "The buffer was not big enough for the dictionary");
+  return res;
+}
+
+
+
+
+
+
+
+
+
+
+
+static auxiliary_dict *
+LoadAuxDict (compression_dict_header * cdh,
+	     char *filename)
+{
+  auxiliary_dict *ad;
+  int i;
+  FILE *aux;
+
+  aux = open_file (filename, TEXT_DICT_AUX_SUFFIX, "rb",
+		   MAGIC_AUX_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  ad = (auxiliary_dict *) getmem (sizeof (auxiliary_dict), sizeof (u_char *));
+
+  for (i = 0; i <= 1; i++)
+    {
+      unsigned int j;
+      u_char *pos;
+
+      fread (&ad->afh[i], sizeof (aux_frags_header), 1, aux);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(ad->afh[i].num_frags);
+      NTOHUL(ad->afh[i].mem_for_frags);
+
+      ad->word_data[i] = (u_char *) getmem (ad->afh[i].mem_for_frags, 1);
+      FIXUP (&ad->word_data[i]);
+
+      ad->words[i] = (u_char **) getmem (ad->afh[i].num_frags * sizeof (u_char *),
+			     sizeof (u_char *));
+      FIXUP (&ad->words[i]);
+
+      fread (ad->word_data[i], ad->afh[i].mem_for_frags, sizeof (u_char), aux);
+
+      pos = ad->word_data[i];
+      for (j = 0; j < ad->afh[i].num_frags; j++)
+	{
+	  ad->words[i][j] = pos;
+	  FIXUP (&ad->words[i][j]);
+	  pos += *pos + 1;
+	}
+      if (cdh->novel_method == MG_NOVEL_HYBRID)
+	{
+	  int num;
+	  num = 1;
+	  ad->blk_start[i][0] = 0;
+	  ad->blk_end[i][0] = cdh->num_words[i] - 1;
+	  while (num < 33)
+	    {
+	      ad->blk_start[i][num] = ad->blk_end[i][num - 1] + 1;
+	      ad->blk_end[i][num] = ad->blk_start[i][num] +
+		(ad->blk_end[i][num - 1] - ad->blk_start[i][num - 1]) * 2;
+	      num++;
+	    }
+	}
+    }
+  fclose (aux);
+  return (ad);
+}
+
+
+
+
+
+static u_char ***
+ReadInWords (FILE * dict, compression_dict * cd,
+	     comp_frags_header * cfh, u_char ** escape)
+{
+  int i, lookback;
+  int ptrs_reqd = 0;
+  int mem_reqd = 0;
+  int num_set[MAX_HUFFCODE_LEN + 1];
+  u_char *next_word[MAX_HUFFCODE_LEN + 1];
+  u_char **vals;
+  u_char ***values;
+  u_char word[MAXWORDLEN + 1];
+  u_char last_word[MAX_HUFFCODE_LEN + 1][MAXWORDLEN + 1];
+
+  lookback = cd->cdh.lookback;
+
+  for (i = cfh->hd.mincodelen; i <= cfh->hd.maxcodelen; i++)
+    {
+      ptrs_reqd += (cfh->hd.lencount[i] + ((1 << lookback) - 1)) >> lookback;
+      mem_reqd += cfh->huff_words_size[i];
+    }
+
+  vals = (u_char **) getmem (ptrs_reqd * sizeof (*vals), sizeof (u_char *));
+
+  values = (u_char ***) getmem ((MAX_HUFFCODE_LEN + 1) * sizeof (u_char **), sizeof (u_char **));
+
+  next_word[0] = (u_char *) getmem (mem_reqd, sizeof (char));
+
+  cd->MemForCompDict += ptrs_reqd * sizeof (*vals) +
+    (MAX_HUFFCODE_LEN + 1) * sizeof (u_char **) +
+    mem_reqd;
+
+  values[0] = vals;
+  FIXUP (&values[0]);
+  values[0][0] = next_word[0];
+  FIXUP (&values[0][0]);
+  for (i = 1; i <= cfh->hd.maxcodelen; i++)
+    {
+      int next_start = (values[i - 1] - vals) +
+      ((cfh->hd.lencount[i - 1] + ((1 << lookback) - 1)) >> lookback);
+      values[i] = &vals[next_start];
+      FIXUP (&values[i]);
+      next_word[i] = next_word[i - 1] + cfh->huff_words_size[i - 1];
+      values[i][0] = next_word[i];
+      FIXUP (&values[i][0]);
+    }
+
+  bzero ((char *) num_set, sizeof (num_set));
+
+  for (i = 0; i < cfh->hd.num_codes; i++)
+    {
+      register int val, copy;
+      register int len = cfh->hd.clens[i];
+      val = getc (dict);
+      copy = (val >> 4) & 0xf;
+      val &= 0xf;
+
+      fread (word + copy + 1, sizeof (u_char), val, dict);
+      *word = val + copy;
+
+      if ((num_set[len] & ((1 << lookback) - 1)) == 0)
+	{
+	  values[len][num_set[len] >> lookback] = next_word[len];
+	  FIXUP (&values[len][num_set[len] >> lookback]);
+	  memcpy (next_word[len], word, *word + 1);
+	  if (escape && i == cfh->hd.num_codes - 1)
+	    {
+	      *escape = next_word[len];
+	      FIXUP (escape);
+	    }
+	  next_word[len] += *word + 1;
+	}
+      else
+	{
+	  copy = prefixlen (last_word[len], word);
+	  memcpy (next_word[len] + 1, word + copy + 1, *word - copy);
+	  *next_word[len] = (copy << 4) + (*word - copy);
+	  if (escape && i == cfh->hd.num_codes - 1)
+	    {
+	      *escape = next_word[len];
+	      FIXUP (escape);
+	    }
+	  next_word[len] += (*word - copy) + 1;
+	}
+      memcpy (last_word[len], word, *word + 1);
+      num_set[len]++;
+    }
+  if (cfh->hd.clens)
+    delete cfh->hd.clens;
+  cfh->hd.clens = NULL;
+  return values;
+}
+
+
+static unsigned long **
+Generate_Fast_Huffman_Vals (huff_data * data, u_long * mem)
+{
+  int i;
+  unsigned long *fcode[MAX_HUFFCODE_LEN + 1];
+  unsigned long **values;
+  unsigned long *vals;
+
+  if (!data)
+    return (NULL);
+  vals = (unsigned long *) getmem (data->num_codes * sizeof (*vals), sizeof (long *));
+  values = (unsigned long **) getmem ((MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *),
+		   sizeof (long *));
+
+  bzero ((char *) values, (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *));
+
+  if (mem)
+    *mem += data->num_codes * sizeof (*vals) +
+      (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *);
+
+  fcode[0] = values[0] = &vals[0];
+  FIXUP (&values[0]);
+  for (i = 1; i <= data->maxcodelen; i++)
+    {
+      fcode[i] = values[i] = &vals[(values[i - 1] - vals) + data->lencount[i - 1]];
+      FIXUP (&values[i]);
+    }
+
+  for (i = 0; i < data->num_codes; i++)
+    if (data->clens[i])
+      *fcode[(int) (data->clens[i])]++ = i;
+  return (values);
+}
+
+
+static void load_read_huffman(FILE *dict, compression_dict *cd, int which)
+{
+  huff_data *hd;
+  u_long   **vals;
+
+  hd = (huff_data *) getmem (sizeof (huff_data), sizeof (char *));
+  cd->MemForCompDict += sizeof (huff_data);
+  Read_Huffman_Data (dict, hd, &cd->MemForCompDict, NULL);
+  vals = Generate_Fast_Huffman_Vals (hd, &cd->MemForCompDict);
+  cd->chars_huff[which] = hd;
+  FIXUP (&cd->chars_huff[which]);
+  cd->chars_vals[which] = vals;
+  FIXUP (&cd->chars_vals[which]);
+  if (hd->clens)
+    delete hd->clens;
+  hd->clens = NULL;
+}
+
+static void load_read_words(FILE *dict, compression_dict *cd, int which, int fixescape)
+{
+  cd->cfh[which] = (comp_frags_header *) getmem (sizeof (*cd->cfh[which]), sizeof (u_char *));
+  cd->MemForCompDict += sizeof (*cd->cfh[which]);
+  Read_cfh (dict, cd->cfh[which], &cd->MemForCompDict, NULL);
+
+  cd->values[which] = ReadInWords (dict, cd, cd->cfh[which], NULL);
+  FIXUP (&cd->cfh[which]);
+  FIXUP (&cd->values[which]);
+  if (fixescape)
+    { FIXUP(&cd->escape[which]);
+    }
+  else
+    {
+      cd->escape[which] = NULL;
+    }
+}
+
+static void 
+load_comp_dict (char *filename)
+{
+  FILE *dict;
+  int which;
+  compression_dict *cd;
+
+  cd = (compression_dict *) getmem (sizeof (compression_dict), sizeof (u_char *));
+  cd->fast_loaded = 1;
+
+  dict = open_file (filename, TEXT_DICT_SUFFIX, "rb",
+		    MAGIC_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  Read_cdh (dict, &cd->cdh, NULL, NULL);
+
+  for (which = 0; which < 2; which++)
+    switch (cd->cdh.dict_type)
+      {
+      case MG_COMPLETE_DICTIONARY:
+	{
+	  load_read_words(dict, cd, which, 0);
+	}
+	break;
+      case MG_PARTIAL_DICTIONARY:
+	{
+	  if (cd->cdh.num_words[which])
+	    {
+	      load_read_words(dict, cd, which, 1);
+	    }
+
+	  load_read_huffman(dict, cd, which);
+	  load_read_huffman(dict, cd, which);
+	}
+	break;
+      case MG_SEED_DICTIONARY:
+	{
+	  if (cd->cdh.num_words[which])
+	    {
+	      load_read_words(dict, cd, which, 1);
+	    }
+	  switch (cd->cdh.novel_method)
+	    {
+	    case MG_NOVEL_HUFFMAN_CHARS:
+	      load_read_huffman(dict, cd, which);
+	      load_read_huffman(dict, cd, which);
+	      break;
+	    case MG_NOVEL_DELTA:
+	      break;
+	    case MG_NOVEL_HYBRID:
+	      break;
+	    }
+	  break;
+	}
+      }
+  fclose (dict);
+
+
+  if (cd->cdh.novel_method == MG_NOVEL_DELTA ||
+      cd->cdh.novel_method == MG_NOVEL_HYBRID)
+    {
+      cd->ad = LoadAuxDict (&cd->cdh, filename);
+      FIXUP (&cd->ad);
+    }
+}
+
+
+static void 
+save_fast_dict (char *filename)
+{
+  FILE *fdict;
+
+  fdict = create_file (filename, TEXT_DICT_FAST_SUFFIX, "wb",
+		       MAGIC_FAST_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  {
+    u_long *p;
+    for (p = (u_long *) buffer; (u_long) p < (u_long) cur; p++)
+      {
+	if (IS_FIXUP (p))
+	  HTONUL(*p);
+      }
+  }
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(mem);
+  HTONUL(fixup_mem);
+
+  fwrite (&mem, sizeof (mem), 1, fdict);
+  fwrite (&fixup_mem, sizeof (fixup_mem), 1, fdict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(mem);
+  NTOHUL(fixup_mem);
+
+  fwrite (buffer, sizeof (u_char), mem, fdict);
+  fwrite (fixup, sizeof (u_char), fixup_mem, fdict);
+
+  fclose (fdict);
+}
Index: /trunk/gsdl/src/mgpp/text/mg_files.cpp
===================================================================
--- /trunk/gsdl/src/mgpp/text/mg_files.cpp	(revision 856)
+++ /trunk/gsdl/src/mgpp/text/mg_files.cpp	(revision 856)
@@ -0,0 +1,261 @@
+/**************************************************************************
+ *
+ * mg_files.cpp -- Routines for handling files for the auxillary programs
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "memlib.h"
+#include "messages.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+
+
+/* This must contain a valid path without a trailing slash */
+static char *basepath = NULL;
+
+
+
+/* This sets the base path for all file operations */
+void 
+set_basepath (const char *bp)
+{
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+  basepath = "";
+#else
+  char *s;
+  /* Free the memory for the base path if it has already been allocated */
+  if (basepath)
+    {
+      delete basepath;
+      basepath = NULL;
+    }
+
+  if (!bp || *bp == '\0')
+    bp = ".";
+
+  s = strrchr (bp, '/');
+  if (s && *(s + 1) == '\0')
+    {
+      basepath = new char[strlen (bp) + 2];
+      if (!basepath)
+	return;
+      strcpy (basepath, bp);
+      strcat (basepath, ".");
+    }
+  else
+    basepath = Xstrdup (bp);
+#endif
+}
+
+
+
+/* return the currently defined basepath */
+char *
+get_basepath (void)
+{
+  if (!basepath)
+    set_basepath (getenv ("MGDATA"));
+  return basepath;
+}
+
+
+
+/* This generates the name of a file. It places the name in the buffer
+   specified or if that is NULL it uses a static buffer. */
+char *
+make_name (const char *name, const char *suffix, char *buffer)
+{
+  static char path[512];
+  if (!buffer)
+    buffer = path;
+  if (!basepath)
+    set_basepath (getenv ("MGDATA"));
+  sprintf (buffer, FILE_NAME_FORMAT, basepath, name, suffix);  /* [RPAP - Feb 97: WIN32 Port] */
+  return buffer;
+}
+
+
+
+/* This will open the specified file and check its magic number.
+   Mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *
+open_named_file (const char *name, const char *mode,
+		 u_long magic_num, int err_mode)
+{
+  unsigned long magic;
+  FILE *f = NULL;
+  char *err;
+  f = fopen (name, mode);
+
+  if (!f)
+    {
+      err = "Unable to open \"%s\"";
+      goto error;
+    }
+
+  if (magic_num)
+    {
+      if (fread (&magic, sizeof (magic), 1, f) != 1)
+	{
+	  err = "No magic number \"%s\"";
+	  goto error;
+	}
+
+      NTOHUL(magic);  /* [RPAP - Jan 97: Endian Ordering] */
+
+      if (!IS_MAGIC (magic))
+	{
+	  err = "No MG magic number \"%s\"";
+	  goto error;
+	}
+
+      if (magic != magic_num)
+	{
+	  err = "Wrong MG magic number \"%s\"";
+	  goto error;
+	}
+    }
+  return f;
+
+error:
+  if (f)
+    fclose (f);
+  if (err_mode == MG_ABORT)
+    FatalError (1, err, name);
+  if (err_mode == MG_MESSAGE)
+    Message (err, name);
+  return NULL;
+}
+
+
+
+/* This will open the specified file and check its magic number.
+   Mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *
+open_file (const char *name, const char *suffix, const char *mode,
+	   u_long magic_num, int err_mode)
+{
+  char path[512];
+  if (!basepath)
+    set_basepath (getenv ("MGDATA"));
+  sprintf (path, FILE_NAME_FORMAT, basepath, name, suffix);  /* [RPAP - Feb 97: WIN32 Port] */
+  return open_named_file (path, mode, magic_num, err_mode);
+}
+
+
+
+
+
+
+/* This will create the specified file and set its magic number.
+
+   Mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *
+create_named_file (const char *name, const char *mode,
+		   u_long magic_num, int err_mode)
+{
+  FILE *f = NULL;
+  char *err;
+  f = fopen (name, mode);
+
+  if (!f)
+    {
+      err = "Unable to open \"%s\"";
+      goto error;
+    }
+
+  if (magic_num)
+    HTONUL(magic_num);  /* [RPAP - Jan 97: Endian Ordering] */
+    if (fwrite (&magic_num, sizeof (magic_num), 1, f) != 1)
+      {
+	err = "Couldn't write magic number \"%s\"";
+	goto error;
+      }
+
+  return f;
+
+error:
+  if (f)
+    fclose (f);
+  if (err_mode == MG_ABORT)
+    FatalError (1, err, name);
+  if (err_mode == MG_MESSAGE)
+    Message (err, name);
+  return NULL;
+}
+
+
+
+
+
+
+
+/* This will create the specified file and set its magic number.
+
+   Mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *
+create_file (const char *name, const char *suffix, const char *mode,
+	     u_long magic_num, int err_mode)
+{
+  char path[512];
+  if (!basepath)
+    set_basepath (getenv ("MGDATA"));
+  sprintf (path, FILE_NAME_FORMAT, basepath, name, suffix);  /* [RPAP - Feb 97: WIN32 Port] */
+  return create_named_file (path, mode, magic_num, err_mode);
+}
Index: /trunk/gsdl/src/mgpp/text/mg_files.h
===================================================================
--- /trunk/gsdl/src/mgpp/text/mg_files.h	(revision 856)
+++ /trunk/gsdl/src/mgpp/text/mg_files.h	(revision 856)
@@ -0,0 +1,345 @@
+/**************************************************************************
+ *
+ * mg_files.h -- Routines for handling files for the auxillary programs
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef MG_FILES_H
+#define MG_FILES_H
+
+#include "sysfuncs.h"
+
+/* Magic numbers for the different types of files */
+
+#define GEN_MAGIC(a,b,c,d) ((unsigned long)(((a)<<24) + ((b)<<16) + \
+					    ((c)<<8) + (d)))
+
+#define MAGIC_XXXX		GEN_MAGIC('M','G', 0 , 0)
+#define MAGIC_STATS_DICT	GEN_MAGIC('M','G','S','D')
+#define MAGIC_AUX_DICT		GEN_MAGIC('M','G','A','D')
+#define MAGIC_FAST_DICT		GEN_MAGIC('M','G','F','D')
+#define MAGIC_DICT		GEN_MAGIC('M','G','D', 0 )
+#define MAGIC_STEM_BUILD	GEN_MAGIC('M','G','S', 0 )
+#define MAGIC_HASH		GEN_MAGIC('M','G','H', 0 )
+#define MAGIC_STEM		GEN_MAGIC('M','G','s', 0 )
+#define MAGIC_CHUNK		GEN_MAGIC('M','G','C', 0 )
+#define MAGIC_CHUNK_TRANS	GEN_MAGIC('M','G','c', 0 )
+#define MAGIC_TEXT		GEN_MAGIC('M','G','T', 0 )
+#define MAGIC_TEXI		GEN_MAGIC('M','G','t', 0 )
+#define MAGIC_TEXT_LEVELS	GEN_MAGIC('M','G','t','L')
+#define MAGIC_TEXI_WGT		GEN_MAGIC('M','G','t','W')
+#define MAGIC_INVF		GEN_MAGIC('M','G','I', 0 )
+#define MAGIC_INVF_LEVELS	GEN_MAGIC('M','G','I','L')
+#define MAGIC_INVI		GEN_MAGIC('M','G','i', 0 )
+#define MAGIC_WGHT		GEN_MAGIC('M','G','W', 0 )
+#define MAGIC_WGHT_APPROX	GEN_MAGIC('M','G','w', 0 )
+#define MAGIC_PARAGRAPH		GEN_MAGIC('M','G','P', 0 )
+/* [RPAP - Jan 97: Stem Index Change] */
+#define MAGIC_STEM_1            GEN_MAGIC('M','G','s','1')
+#define MAGIC_STEM_2            GEN_MAGIC('M','G','s','2')
+#define MAGIC_STEM_3            GEN_MAGIC('M','G','s','3')
+
+#define IS_MAGIC(a) ((((u_long)(a)) & 0xffff0000) == MAGIC_XXXX)
+
+
+/* err_mode values for open_file and create_file */
+#define MG_ABORT 0
+#define MG_MESSAGE 1
+#define MG_CONTINUE 2
+
+
+
+
+
+/* File suffixes */
+
+
+/* The compression dictionary built by txt.pass1 */
+#ifdef SHORT_SUFFIX
+# define TEXT_STATS_DICT_SUFFIX	 ".tsd"
+#else
+# define TEXT_STATS_DICT_SUFFIX	 ".text.stats"
+#endif
+
+/* The compression dictionary built by text.pass1 and comp_dict.process */
+#ifdef SHORT_SUFFIX
+# define TEXT_DICT_SUFFIX	 ".td"
+#else
+# define TEXT_DICT_SUFFIX	 ".text.dict"
+#endif
+
+/* The compression dictionary built by mg_make_fast_dict */
+#ifdef SHORT_SUFFIX
+# define TEXT_DICT_FAST_SUFFIX	 ".tdf"
+#else
+# define TEXT_DICT_FAST_SUFFIX	 ".text.dict.fast"
+#endif
+
+/* The auxilary dictionary built by text.pass2 */
+#ifdef SHORT_SUFFIX
+# define TEXT_DICT_AUX_SUFFIX	 ".tda"
+#else
+# define TEXT_DICT_AUX_SUFFIX	 ".text.dict.aux"
+#endif
+
+/* The compressed text build by text.pass2 */
+#ifdef SHORT_SUFFIX
+# define TEXT_SUFFIX		 ".t"
+#else
+# define TEXT_SUFFIX		 ".text"
+#endif
+
+/* The combined compressed text index and document weight file */
+#ifdef SHORT_SUFFIX
+# define TEXT_IDX_WGT_SUFFIX	 ".tiw"
+#else
+# define TEXT_IDX_WGT_SUFFIX	 ".text.idx.wgt"
+#endif
+
+/* The compressed text index file */
+#ifdef SHORT_SUFFIX
+# define TEXT_IDX_SUFFIX	 	".ti"
+#else
+# define TEXT_IDX_SUFFIX	 	".text.idx"
+#endif
+
+/* The document levels in the text */
+#ifdef SHORT_SUFFIX
+# define TEXT_LEVEL_SUFFIX	 	".tl"
+#else
+# define TEXT_LEVEL_SUFFIX	 	".text.level"
+#endif
+
+/* The dictionary of stemmed words build by invf.pass1 and ivf.pass1 */
+#ifdef SHORT_SUFFIX
+# define INVF_DICT_SUFFIX         ".id"
+#else
+# define INVF_DICT_SUFFIX         ".invf.dict"
+#endif
+
+/* The dictionary of stemmed words build by stem.process */
+#ifdef SHORT_SUFFIX
+# define INVF_DICT_BLOCKED_SUFFIX ".idb"
+#else
+# define INVF_DICT_BLOCKED_SUFFIX ".invf.dict.blocked"
+#endif
+
+/* The exact document weights file build by make.weights, invf.pass2,
+   or ivf.pass2 */
+#ifdef SHORT_SUFFIX
+# define WEIGHTS_SUFFIX           ".w"
+#else
+# define WEIGHTS_SUFFIX           ".weight"
+#endif
+
+/* The approximate weights file built by make.weights */
+#ifdef SHORT_SUFFIX
+# define APPROX_WEIGHTS_SUFFIX    ".wa"
+#else
+# define APPROX_WEIGHTS_SUFFIX    ".weight.approx"
+#endif
+
+/* The inverted file build by invf.pass2 or ivf.pass2 */
+#ifdef SHORT_SUFFIX
+# define INVF_SUFFIX              ".i"
+#else
+# define INVF_SUFFIX              ".invf"
+#endif
+
+/* The level information for the inverted text */
+#ifdef SHORT_SUFFIX
+# define INVF_LEVEL_SUFFIX              ".il"
+#else
+# define INVF_LEVEL_SUFFIX              ".invf.level"
+#endif
+
+/* The inverted file index build by invf.pass2 or ivf.pass2 */
+#ifdef SHORT_SUFFIX
+# define INVF_IDX_SUFFIX          ".ii"
+#else
+# define INVF_IDX_SUFFIX          ".invf.idx"
+#endif
+
+/* The inverted file chunk descriptor built by ivf.pass1 */
+#ifdef SHORT_SUFFIX
+# define INVF_CHUNK_SUFFIX        ".ic"
+#else
+# define INVF_CHUNK_SUFFIX        ".invf.chunk"
+#endif
+
+/* The word index translation file built by ivf.pass1 */
+#ifdef SHORT_SUFFIX
+# define INVF_CHUNK_TRANS_SUFFIX  ".ict"
+#else
+# define INVF_CHUNK_TRANS_SUFFIX  ".invf.chunk.trans"
+#endif
+
+/* The hashed stemmed dictionary built by make.perf_hash */
+#ifdef SHORT_SUFFIX
+# define INVF_DICT_HASH_SUFFIX    ".idh"
+#else
+# define INVF_DICT_HASH_SUFFIX    ".invf.dict.hash"
+#endif
+
+/* The paragraph descriptior file built by invf.pass1 or ivf.pass1 */
+#ifdef SHORT_SUFFIX
+# define INVF_PARAGRAPH_SUFFIX    ".ip"
+#else
+# define INVF_PARAGRAPH_SUFFIX    ".invf.paragraph"
+#endif
+
+/* The trace file build by mg.builder. */
+#ifdef SHORT_SUFFIX
+# define TRACE_SUFFIX             ".trc"
+#else
+# define TRACE_SUFFIX             ".trace"
+#endif
+
+/* The compression stats file build by mg.builder. */
+#ifdef SHORT_SUFFIX
+# define COMPRESSION_STATS_SUFFIX ".cs"
+#else
+# define COMPRESSION_STATS_SUFFIX ".compression.stats"
+#endif
+
+/* [RPAP - Jan 97: Stem Index Change] */
+/* The casefolded index into the stemmed dictionary */
+#ifdef SHORT_SUFFIX
+# define INVF_DICT_BLOCKED_1_SUFFIX ".ib1"
+#else
+# define INVF_DICT_BLOCKED_1_SUFFIX ".invf.dict.blocked.1"
+#endif
+
+/* [RPAP - Jan 97: Stem Index Change] */
+/* The stemmed index into the stemmed dictionary */
+#ifdef SHORT_SUFFIX
+# define INVF_DICT_BLOCKED_2_SUFFIX ".ib2"
+#else
+# define INVF_DICT_BLOCKED_2_SUFFIX ".invf.dict.blocked.2"
+#endif
+
+/* [RPAP - Jan 97: Stem Index Change] */
+/* The casefolded and stemmed  index into the stemmed dictionary */
+#ifdef SHORT_SUFFIX
+# define INVF_DICT_BLOCKED_3_SUFFIX ".ib3"
+#else
+# define INVF_DICT_BLOCKED_3_SUFFIX ".invf.dict.blocked.3"
+#endif
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+#define FILE_NAME_FORMAT "%s%s%s"
+#else
+#define FILE_NAME_FORMAT "%s/%s%s"
+#endif
+
+
+/* This sets the base path for all file operations */
+void set_basepath (const char *bp);
+
+
+/* return the currently defined basepath */
+char *get_basepath (void);
+
+
+
+
+/* This generates the name of a file. It places the name in the buffer
+   specified or if that is NULL it uses a static buffer. */
+char *make_name (const char *name, const char *suffix, char *buffer);
+
+
+
+
+
+
+
+/* This will open the specified file and check its magic number.
+   Mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *open_named_file (const char *name, const char *mode,
+		       u_long magic_num, int err_mode);
+
+
+
+
+/* This will open the specified file and check its magic number.
+
+   err_mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *open_file (const char *name, const char *suffix, const char *mode,
+		 u_long magic_num, int err_mode);
+
+
+
+
+
+/* This will create the specified file and set its magic number.
+
+   Mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *create_named_file (const char *name, const char *mode,
+			 u_long magic_num, int err_mode);
+
+
+
+/* This will create the specified file and set its magic number.
+
+   err_mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *create_file (const char *name, const char *suffix, const char *mode,
+		   u_long magic_num, int err_mode);
+
+
+
+
+#endif
Index: /trunk/gsdl/src/mgpp/text/mg_invf_dict.1
===================================================================
--- /trunk/gsdl/src/mgpp/text/mg_invf_dict.1	(revision 856)
+++ /trunk/gsdl/src/mgpp/text/mg_invf_dict.1	(revision 856)
@@ -0,0 +1,80 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_invf_dict 1 \*(Dt CITRI
+.SH NAME
+mg_invf_dict \- build an `on-disk' stemmed dictionary
+.SH SYNOPSIS
+.B mg_invf_dict
+[
+.B \-h
+]
+[
+.BI \-b " entries-per-block"
+]
+.if n .ti +13n
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_invf_dict
+builds an `on-disk' stemmed dictionary from the
+compressed stemmed dictionary that can be used during querying.
+Each entry in the dictionary contains the disk
+pointer to the associated inverted file entry, and various other pieces
+of information about the stemmed word. The dictionary is blocked with
+each block containing the same number of entries (not the same size).
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectoryyyyyyyy\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-b " entries-per-block"
+The dictionary is stored in blocks on disk; this option is used to 
+specify the number of entries per block. The default is 16.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the mg
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.dict
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict.blocked
+The `on-disk' stemmed dictionary.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.SH "SEE ALSO"
+.na
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_stem_idx (1),
+.BR mg_weights_build (1)
Index: /trunk/gsdl/src/mgpp/text/mg_invf_dict.cpp
===================================================================
--- /trunk/gsdl/src/mgpp/text/mg_invf_dict.cpp	(revision 856)
+++ /trunk/gsdl/src/mgpp/text/mg_invf_dict.cpp	(revision 856)
@@ -0,0 +1,231 @@
+/**************************************************************************
+ *
+ * mg_invf_dict.cpp -- Program to build the blocked stemmed dictionary
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "messages.h"
+
+#include "mg_files.h"
+#include "invf.h"
+
+/*
+   $Log$
+   Revision 1.1  2000/01/14 02:26:16  sjboddie
+   Rodgers new C++ mg
+
+ */
+
+
+static void process_files (char *filename, unsigned long entriesPerBlock) {
+  // open the dictionary
+  FILE *dictFile = open_file (filename, INVF_DICT_SUFFIX, "rb",
+			      MAGIC_STEM_BUILD, MG_ABORT);
+  invf_dict_header idh;
+  idh.Read (dictFile);
+
+  // open the inverted index file
+  FILE *invfIdxFile = open_file (filename, INVF_IDX_SUFFIX, "rb",
+				 MAGIC_INVI, MG_ABORT);
+
+  // create the blocked dictionary
+  FILE *blockDictFile = create_file (filename, INVF_DICT_BLOCKED_SUFFIX, "wb",
+				     MAGIC_STEM, MG_ABORT);
+  block_dict_header bdh;
+  bdh.lookback = idh.lookback;
+  bdh.word_dict_start = idh.word_dict_start;
+  bdh.word_dict_size = idh.word_dict_size;
+  bdh.tag_dict_start = idh.tag_dict_start;
+  bdh.tag_dict_size = idh.tag_dict_size;
+  bdh.num_docs = idh.num_docs;
+  bdh.num_frags = idh.num_frags;
+  bdh.num_words = idh.num_words;
+  bdh.total_bytes = idh.total_bytes;
+  bdh.index_string_bytes = idh.index_string_bytes;
+  bdh.num_levels = idh.num_levels;
+  bdh.Write (blockDictFile);
+
+  
+  // write out the word part of the dictionary
+
+  bdh.entries_per_wblk = entriesPerBlock;
+  bdh.max_wblk_size = 0;
+  bdh.wblk_start = ftell (blockDictFile);
+  
+  fseek (dictFile, idh.word_dict_start, SEEK_SET);
+
+  block_idx wordIdx;
+  word_block_dict_el wordBlockEl;
+  wordBlockEl.SetNumLevels (idh.num_levels);
+  
+  
+  unsigned long wordNum;
+  unsigned long wordInvfPtr;
+  UCArray lastEl;
+  word_dict_el wordEl;
+  wordEl.SetNumLevels (idh.num_levels);
+  for (wordNum=0; wordNum<idh.word_dict_size; wordNum++) {
+    // read in the next word and inverted file pointer
+    wordEl.Read (dictFile, idh.num_levels);
+    ReadUL (invfIdxFile, wordInvfPtr);
+
+    // remember this word (and position) if this is the start
+    // of a new block
+    if (wordNum % entriesPerBlock == 0) {
+      block_idx_info elIdx;
+      elIdx.el = wordEl.el;
+      elIdx.block_ptr = ftell (blockDictFile);
+
+      // see if this block is the longest so far
+      if (wordIdx.size() > 0) {
+	unsigned long blockLen = elIdx.block_ptr -
+	  (*(wordIdx.end()-1)).block_ptr;
+	if (blockLen > bdh.max_wblk_size) bdh.max_wblk_size = blockLen;
+      }
+      
+      wordIdx.push_back (elIdx);
+      lastEl.erase (lastEl.begin(), lastEl.end()); // output full word
+    }
+
+    // copy the information for this word
+    wordBlockEl.el = wordEl.el;
+    wordBlockEl.frag_occur = wordEl.frag_occur;
+    wordBlockEl.freq = wordEl.freq;
+    wordBlockEl.invf_ptr = wordInvfPtr;
+    unsigned long tempI;
+    for (tempI=0; tempI<idh.num_levels; tempI++)
+      wordBlockEl.levelFreqs[tempI] = wordEl.levelFreqs[tempI];
+
+    // write out the word
+    wordBlockEl.Write (blockDictFile, &lastEl, idh.num_levels);
+
+    lastEl = wordBlockEl.el;
+  }
+
+
+  // write out the tag part of the dictionary
+
+  bdh.entries_per_tblk = entriesPerBlock;
+  bdh.max_tblk_size = 0;
+  bdh.tblk_start = ftell (blockDictFile);
+  
+  fseek (dictFile, idh.tag_dict_start, SEEK_SET);
+
+  block_idx tagIdx;
+  block_dict_el tagBlockEl;
+  
+  unsigned long tagNum;
+  unsigned long tagInvfPtr;
+  dict_el tagEl;
+  lastEl.erase (lastEl.begin(), lastEl.end());
+  for (tagNum=0; tagNum<idh.tag_dict_size; tagNum++) {
+    // read in the next tag and inverted file pointer
+    tagEl.Read (dictFile);
+    ReadUL (invfIdxFile, tagInvfPtr);
+
+    // remember this tag (and position) if this is the start
+    // of a new block
+    if (tagNum % entriesPerBlock == 0) {
+      block_idx_info elIdx;
+      elIdx.el = tagEl.el;
+      elIdx.block_ptr = ftell (blockDictFile);
+
+      // see if this block is the longest so far
+      if (tagIdx.size() > 0) {
+	unsigned long blockLen = elIdx.block_ptr -
+	  (*(tagIdx.end()-1)).block_ptr;
+	if (blockLen > bdh.max_tblk_size) bdh.max_tblk_size = blockLen;
+      }
+      
+      tagIdx.push_back (elIdx);
+      lastEl.erase (lastEl.begin(), lastEl.end()); // output full word
+    }
+
+    // copy the information for this tag
+    tagBlockEl.el = tagEl.el;
+    tagBlockEl.frag_occur = tagEl.frag_occur;
+    tagBlockEl.freq = tagEl.freq;
+    tagBlockEl.invf_ptr = tagInvfPtr;
+
+    // write out the tag
+    tagBlockEl.Write (blockDictFile, &lastEl);
+
+    lastEl = tagBlockEl.el;
+  }
+
+
+  // write out the element indexes
+  bdh.num_wblks = wordIdx.size();
+  bdh.wblk_idx_start = ftell (blockDictFile);
+  WriteBlockIdx (blockDictFile, wordIdx);
+  
+  bdh.num_tblks = tagIdx.size();
+  bdh.tblk_idx_start = ftell (blockDictFile);
+  WriteBlockIdx (blockDictFile, tagIdx);
+
+  // write out the blocked dictionary header
+  fseek (blockDictFile, sizeof(unsigned long), SEEK_SET);
+  bdh.Write (blockDictFile);
+
+
+  // close open files
+  fclose (blockDictFile);
+  fclose (invfIdxFile);
+  fclose (dictFile);
+  
+  // print out information
+  Message ("Max word block size = %d\n", bdh.max_wblk_size);
+  Message ("Max tag block size = %d\n", bdh.max_tblk_size);
+  Message ("Number of word blocks written = %d\n", bdh.num_wblks);
+  Message ("Number of tag blocks written = %d\n", bdh.num_tblks);
+}
+
+
+int main (int argc, char **argv) {
+  unsigned long entriesPerBlock = 16;
+  char *filename = "";
+  int ch;
+  msg_prefix = argv[0];
+  opterr = 0;
+
+  while ((ch = getopt (argc, argv, "f:d:b:h")) != -1) {
+    switch (ch) {
+    case 'f':		// input file
+      filename = optarg;
+      break;
+    case 'd':
+      set_basepath (optarg);
+      break;
+    case 'b':
+      entriesPerBlock = atoi (optarg);
+      break;
+    case 'h':
+    case '?':
+      fprintf (stderr, "usage: %s [-f input_file] "
+	       "[-d data directory] [-b entries-per-block] "
+	       "[-h]\n", argv[0]);
+      exit (1);
+    }
+  }
+
+  process_files (filename, entriesPerBlock);
+  return 0;
+}
Index: /trunk/gsdl/src/mgpp/text/mg_invf_dump.cpp
===================================================================
--- /trunk/gsdl/src/mgpp/text/mg_invf_dump.cpp	(revision 856)
+++ /trunk/gsdl/src/mgpp/text/mg_invf_dump.cpp	(revision 856)
@@ -0,0 +1,232 @@
+/**************************************************************************
+ *
+ * mg_invf_dump.cpp -- Program to dump uot an inverted fil
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "messages.h"
+#include "bitio_m_stdio.h"
+#include "bitio_gen.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "locallib.h"
+#include "words.h"
+#include "invf.h"
+#include "WordData.h"
+
+/*
+   $Log$
+   Revision 1.1  2000/01/14 02:26:17  sjboddie
+   Rodgers new C++ mg
+
+   Revision 1.1  1999/10/11 02:57:55  cs025
+   Base install of MG-PP
+
+   Revision 1.1  1999/08/10 21:18:09  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:46  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:05  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/11/29  00:32:01  tes
+   * Committing the new merged files and changes.
+   *
+   * Revision 1.2  1994/09/20  04:41:50  tes
+   * For version 1.1
+   *
+ */
+
+
+static void PrintInvfWord (FILE *invfFile,
+			   invf_dict_header &idh,
+			   invf_file_header &ifh,
+			   word_dict_el &wordEl,
+			   unsigned long wordStart) {
+  cout << wordEl.frag_occur << " \"" << wordEl.el << "\"\n";
+  
+  // seek to the appropriate place in the inverted file
+  fseek (invfFile, wordStart, SEEK_SET);
+
+  stdio_bitio_buffer buffer(invfFile);
+  
+  unsigned long B = BIO_Bblock_Init (idh.num_frags, wordEl.frag_occur);
+  unsigned long fragNum = 0;
+  unsigned long i;
+  for (i=0; i<wordEl.frag_occur; i++) {
+    unsigned long delta = buffer.bblock_decode (B, NULL);
+    fragNum += delta;
+    cout << " " << fragNum;
+    
+    if (!ifh.word_level_index ) {
+      unsigned long count = buffer.gamma_decode (NULL);
+      cout << "(" << count << ")";
+    } else {
+      cout << "(1)";
+    }
+  }
+
+  cout << "\n";
+  
+  buffer.done();
+}
+
+static void PrintInvfTag (FILE *invfFile,
+			  invf_dict_header &idh,
+			  invf_file_header &/*ifh*/,
+			  dict_el &tagEl,
+			  unsigned long tagStart) {
+  cout << tagEl.frag_occur << " \"<" << tagEl.el << ">\"\n";
+  
+  // seek to the appropriate place in the inverted file
+  fseek (invfFile, tagStart, SEEK_SET);
+
+  stdio_bitio_buffer buffer(invfFile);
+  
+  unsigned long pTag = tagEl.frag_occur*2;
+  unsigned long B = BIO_Bblock_Init (idh.num_frags+pTag, pTag);
+  unsigned long fragNum = 0;
+  unsigned long i;
+  for (i=0; i<tagEl.frag_occur; i++) {
+    unsigned long delta = buffer.bblock_decode (B, NULL)-1;
+    fragNum += delta;
+    cout << " " << fragNum;
+    cout << "-";
+    delta = buffer.bblock_decode (B, NULL)-1;
+    fragNum += delta;
+    cout << fragNum;
+  }
+
+  cout << "\n";
+  
+  buffer.done();
+}
+
+static void PrintHeaderInfo (invf_dict_header &idh,
+			     invf_file_header &ifh) {
+  cerr << "Lookback:         " << idh.lookback << "\n";
+  cerr << "Word Dict Size:   " << idh.word_dict_size << "\n";
+  cerr << "Tag Dict Size:    " << idh.tag_dict_size << "\n";
+  cerr << "Num Documents:    " << idh.num_docs << "\n";
+  cerr << "Num Fragments:    " << idh.num_frags << "\n";
+  cerr << "Num Words:        " << idh.num_words << "\n";
+
+  cerr << "Skip Mode:        " << ifh.skip_mode << "\n";
+  cerr << "Word Level Index: " << ifh.word_level_index << "\n";
+
+  cerr << "\n";
+}
+
+
+static void process_files (char *filename) {
+  // open the dictionary
+  FILE *dictFile = open_file (filename, INVF_DICT_SUFFIX, "rb",
+			MAGIC_STEM_BUILD, MG_ABORT);
+  invf_dict_header idh;
+  idh.Read (dictFile);
+
+  // open the inverted file
+  FILE *invfFile = open_file (filename, INVF_SUFFIX, "rb",
+			MAGIC_INVF, MG_ABORT);
+  
+  invf_file_header ifh;
+  ifh.Read (invfFile);
+
+  if (ifh.skip_mode != SKIP_MODE_NO_SKIPS)
+    FatalError (1, "The invf file contains skips. Unable to dump.");
+
+  // print out header information
+  PrintHeaderInfo (idh, ifh);
+  
+  // open the inverted index
+  FILE *invfIdxFile = open_file (filename, INVF_IDX_SUFFIX, "rb",
+				 MAGIC_INVI, MG_ABORT);
+
+  // go to the start of the word dictionary
+  fseek (dictFile, idh.word_dict_start, SEEK_SET);
+
+  // process all the words
+  unsigned long wordNum;
+  unsigned long wordStart;
+  word_dict_el wordEl;
+  wordEl.SetNumLevels (idh.num_levels);
+  for (wordNum=0; wordNum<idh.word_dict_size; wordNum++) {
+    wordEl.Read (dictFile, idh.num_levels);
+    ReadUL (invfIdxFile, wordStart);
+    PrintInvfWord (invfFile, idh, ifh, wordEl, wordStart);
+  }
+
+  // process all the tags
+  unsigned long tagNum;
+  unsigned long tagStart;
+  dict_el tagEl;
+  for (tagNum=0; tagNum<idh.tag_dict_size; tagNum++) {
+    tagEl.Read (dictFile);
+    ReadUL (invfIdxFile, tagStart);
+    PrintInvfTag (invfFile, idh, ifh, tagEl, tagStart);
+  }
+  
+  // close the open files
+  fclose (invfIdxFile);
+  fclose (invfFile);
+  fclose (dictFile);
+}
+
+
+int main (int argc, char **argv) {
+  char *dir_name, *filename = "";
+  int ch;
+  msg_prefix = argv[0];
+  dir_name = getenv ("MGDATA");
+  opterr = 0;
+  
+  msg_prefix = argv[0];
+  while ((ch = getopt (argc, argv, "hbwtf:d:")) != -1) {
+    switch (ch) {
+    case 'f':		// input file
+      filename = optarg;
+      break;
+    case 'd':
+      set_basepath(optarg);
+      break;
+    case 'h':
+    case '?':
+      fprintf (stderr, "usage: %s [-h] [-b] [-w] [-t] [-f input_file]"
+	       "[-d data directory]\n", argv[0]);
+      exit (1);
+    }
+  }
+  
+  process_files (filename);
+
+  return 0;
+}
Index: /trunk/gsdl/src/mgpp/text/mg_passes.1
===================================================================
--- /trunk/gsdl/src/mgpp/text/mg_passes.1	(revision 856)
+++ /trunk/gsdl/src/mgpp/text/mg_passes.1	(revision 856)
@@ -0,0 +1,324 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_passes 1 \*(Dt CITRI
+.SH NAME
+mg_passes \- builds mg databases
+.SH SYNOPSIS
+.B mg_passes
+[
+.BI \-J " doc-tag"
+]
+[
+.BI \-K " level-tag"
+]
+.if n .ti +10n
+[
+.BI \-L " index-level"
+]
+[
+.BI \-m " invf-mem-buffer"
+]
+.if n .ti +10n
+[
+.B \-T1
+]
+[
+.B \-T2
+]
+[
+.B \-I1
+]
+[
+.B \-I2
+]
+[
+.B \-S
+]
+[
+.B \-C
+]
+.if n .ti +10n
+[
+.B \-h
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+[
+.I filename(s)
+]
+.SH DESCRIPTION
+.B mg_passes
+is the program that does most of the work when building mg
+database systems.  The input documents can come from either
+.I stdin
+or from a list of files on the command line.  In general,
+.B mg_passes
+must be run twice to build a database, first with the
+.B \-T1
+and
+.B \-I1
+options, and second with the
+.B \-T2
+and
+.B \-I2
+options.  Several other programs must be run in order to get an
+mg database.  The
+.SB EXAMPLE
+section below gives an example of how to build a complete
+mg database.
+.SH OPTIONS
+Options may appear in any order, but the
+.IR filename(s) ,
+if specified, must be last.
+.TP "\w'\fB\-C\fP \fIcompstatpointt\fP'u+2n"
+.BI \-J " doc-tag"
+Specifies the SGML tag that encloses each document.  Text appearing
+outside this tag is ignored.  The document tag defines the highest
+level document that can be queried and printed.  The default document
+tag is 'Document'.
+.TP
+.BI \-K " level-tag"
+Specifies the SGML tag of a sub document level.  A level tag must
+enclose all text enclosed by the document tag.  Levels can be
+queried and printed as if they were separate documents.  Multiple
+document levels can be specified (the document tag is always
+added as a document level).
+.TP
+.BI \-L " index-level"
+Specifies the SGML tag enclosing the smallest indexed element.  The
+index level should be no larger than the smallest document 
+level.  An empty string can be used to specify a word level index 
+(which is the default).
+.TP
+.BI \-m " invf-mem-buffer"
+Maximum amount of memory to use for the pass-2 file inversion in
+megabytes.  This option is only useful when used in conjunction with
+the option
+.BR \-I1 .
+The larger this value, the faster the pass-2 inversion will proceed.
+The default value is 5 MB.
+.TP
+.B \-T1
+Generate the
+.I *.text.stats
+file.
+.TP
+.B \-T2
+Generate the
+.IR *.text ,
+.IR *.text.idx ,
+.IR *.text.level ,
+and possibly the
+.I *.text.dict.aux
+files.  Using this option requires that the
+.I *.text.dict
+file be present.
+.TP
+.B \-I1
+Generate the
+.IR *.invf.dict ,
+.IR *.invf.level ,
+.IR *.invf.chunk ,
+and
+.I *.invf.chunk.trans
+files.
+.TP
+.B \-I2
+Generate the
+.I *.invf
+and
+.I *.invf.idx
+files.  Using this option requires
+that the
+.IR *.invf.dict.hash ,
+.IR *.invf.level ,
+.IR *.invf.chunk ,
+and
+.I *.invf.chunk.trans
+files be present.  The
+.I *.invf.dict.hash
+file is generated by
+.BR mg_perf_hash_build (1)
+from the
+.I *.invf.dict
+file.
+.TP
+.B \-S
+This option causes a special pass to be executed.  It is up to a user
+to modify
+.I mg.special.c
+in the source code to do something with the documents it is given.
+.TP 
+.B \-C
+This activates the compatibility parsing mode.  When using this
+mode documents are separated by control-B and paragraphs are separated
+by control-C.  Internally these are converted to documents surrounded
+by 'Document' tags and paragraphs surrounded by 'Paragraph' tags.
+.TP
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection is to be
+written.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection that will be
+created.
+.TP
+.I filename(s)
+This specifies the source text. If this is not specified, then the
+program expects the source text from
+.IR stdin .
+.SH EXAMPLE
+What follows is a UNIX
+.BR csh (1)
+script as an example of how to build an mg document collection.
+.LP
+.nf
+.DT
+.ft B
+.I #! /bin/csh
+.I
+# The first argument on the command line specifies the
+.I
+#   source of the text
+set source = ($1)
+.PP
+.I
+# The second argument is the name of the collection
+set text = ($2)
+.PP
+.I
+# Create *.text.stats, *.invf.dict, *.invf.level
+.I
+#   *.invf.chunk and *.invf.chunks.trans
+${source} | mg_passes -T1 -I1 -f ${text}
+.PP
+.I
+# Create *.text.dict
+mg_compression_dict -f ${text}
+.PP
+.I
+# Create *.invf.dict.hash
+mg_perf_hash_build -f ${text}
+.PP
+.I
+# Create *.text, *.text.idx, *.text.level
+.I
+#   *.invf and *.invf.idx
+${source} | mg_passes -T2 -I2 -f ${text}
+.PP
+.I
+# Create *.text.weight and *.weight.approx
+mg_weights_build -f ${text}
+.PP
+.I
+# Create *.invf.dict.blocked
+mg_invf_dict -f ${text}
+.PP
+.I
+# Create *.invf.dict.blocked.1
+mg_stem_idx -s 1 -f ${text}
+.PP
+.I
+# Create *.invf.dict.blocked.2
+mg_stem_idx -s 2 -f ${text}
+.PP
+.I
+# Create *.invf.dict.blocked.3
+mg_stem_idx -s 3 -f ${text}
+.PP
+.I
+# Create *.text.dict.fast
+mg_fast_comp_dict -f ${text}
+.ft R
+.fi
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the mg
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 22
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.chunk
+Inverted file chunk descriptor file.  When the inverted file is
+created it is created in chunks that use no more than a set amount of
+memory.  This file describes those chunks.
+.TP
+.B *.invf.chunk.trans
+Word-occurrence-order to lexical-order translation file.  The
+.B *.invf.chunk
+file is written in word-occurrence order but is required by
+.B \-I2
+to be in lexical order.
+.TP
+.B *.invf.dict
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict.blocked
+Compressed stemmed dictionary with index into the dictionary.
+.TP
+.B *.invf.dict.blocked.n
+Transformation dictionary from words stemmed with method 
+.B n
+to unstemmed words.
+.TP
+.B *.invf.dict.hash
+Data for an order-preserving perfect hash function.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.TP
+.B *.invf.level
+Information about the document levels needed for querying.
+.TP
+.B *.text
+Compressed text.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.text.dict.fast
+A fast loading version of the compressed compression dictionary.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.text.level
+Information about the document levels needed for text decompression.
+.TP
+.B *.text.stats
+Statistics about the text.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.B *.weight.approx
+The approximate weights file.
+.SH "SEE ALSO"
+.na
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_invf_dict (1),
+.BR mg_perf_hash_build (1),
+.BR mg_stem_idx (1),
+.BR mg_weights_build (1)
Index: /trunk/gsdl/src/mgpp/text/mg_passes.cpp
===================================================================
--- /trunk/gsdl/src/mgpp/text/mg_passes.cpp	(revision 856)
+++ /trunk/gsdl/src/mgpp/text/mg_passes.cpp	(revision 856)
@@ -0,0 +1,291 @@
+/**************************************************************************
+ *
+ * mg_passes.cpp -- Driver for the various passes
+ * Copyright (C) 1994  Neil Sharman
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#ifdef HAVE_MALLINFO
+# include <malloc.h> 
+#endif
+
+#include "memlib.h"
+#include "messages.h"
+
+#include "longlong.h"
+
+#include "mg_files.h"
+#include "mg.h"
+#include "build.h"
+#include "text.h"
+#include "stemmer.h"
+
+#include "FileBuf.h"
+#include "TextEl.h"
+#include "TagInfo.h"
+
+
+/*
+   $Log$
+   Revision 1.1  2000/01/14 02:26:18  sjboddie
+   Rodgers new C++ mg
+
+   Revision 1.1  1999/10/11 02:58:00  cs025
+   Base install of MG-PP
+
+   Revision 1.1  1999/08/10 21:18:12  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.3  1998/12/17 09:12:53  rjmcnab
+
+   Altered mg to process utf-8 encoded Unicode. The main changes
+   are in the parsing of the input, the casefolding, and the stemming.
+
+   Revision 1.2  1998/11/25 07:55:47  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:13  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:57  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:52  tes
+   * For version 1.1
+   *
+ */
+
+#define MAX_PASSES 5
+
+#define SPECIAL 1
+#define TEXT_PASS_1 2
+#define TEXT_PASS_2 4
+#define IVF_PASS_1 8
+#define IVF_PASS_2 16
+
+#define MIN_BUF 8192
+
+
+unsigned long invf_buffer_size = 5 * 1024 * 1024;	/* 5Mb */
+
+static char Passes = 0;
+static char **files = NULL;
+static int num_files = 0;
+
+
+struct pass_data {
+  char *name;
+  int (*init) (const TagInfo &tagInfo, char *);
+  int (*process) (const TagInfo &tagInfo, const TextElArray &doc);
+  int (*done) (const TagInfo &tagInfo, char *);
+};
+
+
+static pass_data PassData[MAX_PASSES] = {
+  {"special", init_special, process_special, done_special},
+  {"text.pass1", init_text_1, process_text_1, done_text_1},
+  {"text.pass2", init_text_2, process_text_2, done_text_2},
+  {"ivf.pass1", init_ivf_1, process_ivf_1, done_ivf_1},
+  {"ivf.pass2", init_ivf_2, process_ivf_2, done_ivf_2},
+};
+
+static char *usage_str = "\nUSAGE:\n"
+"  %s [-J doc-tag] [-K level-tag] [-L index-level]\n"
+"  %*s [-m invf-memory] [-T1] [-T2] [-I1] [-I2] [-S]\n"
+"  %*s [-C] [-h] [-d directory] -f name\n\n";
+
+
+
+static void Usage (char *err) {
+  if (err) Message (err);
+  fprintf (stderr, usage_str, msg_prefix, strlen (msg_prefix), "",
+	   strlen (msg_prefix), "");
+  exit (1);
+}
+
+
+
+
+int OpenNextFile (int in_fd) {
+  if (in_fd > 0) close (in_fd);
+  if (num_files == 0) return (-1);
+  if ((in_fd = open (files[0], O_RDONLY)) == -1)
+    FatalError (1, "Cannot open %s", files[0]);
+  files++;
+  num_files--;
+  return (in_fd);
+}
+
+
+static void Driver (int in_fd, char *file_name,
+		    const TagInfo &tagInfo, bool compatMode) {
+  //  cout << tagInfo;
+
+  int pass;
+
+  unsigned long numBytes = 0;
+  unsigned long numDocs = 0;
+
+  // initialise all the passes
+  for (pass = 0; pass < MAX_PASSES; pass++) {
+    if (Passes & (1 << pass)) {
+      if (PassData[pass].init (tagInfo, file_name) == COMPERROR)
+	FatalError (1, "Error during init of \"%s\"", PassData[pass].name);
+    }
+  }
+
+  
+  // set up various variables
+  FileBuf buf;
+  TextElArray doc;
+  unsigned long docLen = 0;
+
+  // read and process each file (start with an open file)
+  do {
+    
+    // read and process each document in this file
+    buf.SetFD (in_fd);
+    while (ReadDoc (buf, tagInfo.docTag, doc, docLen, compatMode)) {
+
+      // give this document to each pass
+      for (pass = 0; pass < MAX_PASSES; pass++) {
+	if (Passes & (1 << pass)) {
+	  if (PassData[pass].process (tagInfo, doc) == COMPERROR)
+	    FatalError(1,"Error during processing of \"%s\"",PassData[pass].name);
+	}
+      }
+      
+      // another document has been processed
+      numBytes += docLen;
+      numDocs++;
+    }
+    
+  } while ((in_fd = OpenNextFile (in_fd)) > 0);
+
+  
+  // do done for each pass
+  for (pass = 0; pass < MAX_PASSES; pass++) {
+    if (Passes & (1 << pass)) {
+      if (PassData[pass].done (tagInfo, file_name) == COMPERROR)
+	FatalError (1, "Error during done of \"%s\"", PassData[pass].name);
+    }
+  }
+}
+
+
+
+int main (int argc, char **argv) {
+  int ch, in_fd;
+  char *filename = NULL;
+  bool compatMode = false;
+  TagInfo tagInfo;
+  tagInfo.SetDocTag ("Document");
+  
+  msg_prefix = argv[0];
+
+  opterr = 0;
+  while ((ch=getopt(argc, argv, "J:K:L:f:d:m:I:T:SCh"))!=-1){
+    switch (ch) {
+    case 'J':
+      tagInfo.SetDocTag (optarg);
+      break;
+    case 'K':
+      tagInfo.AddLevelTag (optarg);
+      break;
+    case 'L':
+      tagInfo.SetIndexLevel (optarg);
+      break;
+    case 'f':
+      filename = optarg;
+      break;
+    case 'd':
+      set_basepath (optarg);
+      break;
+    case 'm':
+      invf_buffer_size = (int) (atof (optarg) * 1024 * 1024);
+      break;
+    case 'I':
+      if (*optarg == '1')
+	Passes |= IVF_PASS_1;
+      else if (*optarg == '2')
+	Passes |= IVF_PASS_2;
+      else
+	Usage ("Invalid pass number");
+      break;
+    case 'T':
+      if (*optarg == '1')
+	Passes |= TEXT_PASS_1;
+      else if (*optarg == '2')
+	Passes |= TEXT_PASS_2;
+      else
+	Usage ("Invalid pass number");
+      break;
+    case 'S':
+      Passes |= SPECIAL;
+      break;
+    case 'C':
+      compatMode = true;
+      break;
+    case 'h':
+    case '?':
+      Usage (NULL);
+    }
+  }
+
+  if (!filename || *filename == '\0')
+    FatalError (1, "A document collection name must be specified.");
+
+  if ((Passes & (IVF_PASS_1 | IVF_PASS_2)) == (IVF_PASS_1 | IVF_PASS_2))
+    FatalError (1, "I1 and I2 cannot be done simultaneously.");
+
+  if ((Passes & (TEXT_PASS_1 | TEXT_PASS_2)) == (TEXT_PASS_1 | TEXT_PASS_2))
+    FatalError (1, "T1 and T2 cannot be done simultaneously.");
+
+  if (!Passes)
+    FatalError (1, "S, T1, T2, I1 or I2 must be specified.");
+
+  if (optind < argc) {
+    if ((in_fd = open (argv[optind], O_RDONLY)) == -1)
+      FatalError (1, "Cannot open %s", argv[optind]);
+    files = &argv[optind + 1];
+    num_files = argc - (optind + 1);
+    
+  } else in_fd = 0;  // stdin
+
+  
+  if (compatMode) tagInfo.SetDocTag ("Document");
+
+  // a document tag is also a level tag
+  tagInfo.levelTags.insert (tagInfo.docTag); 
+  
+  Driver (in_fd, filename, tagInfo, compatMode);
+
+  exit (0);
+}
Index: /trunk/gsdl/src/mgpp/text/mg_perf_hash_build.1
===================================================================
--- /trunk/gsdl/src/mgpp/text/mg_perf_hash_build.1	(revision 856)
+++ /trunk/gsdl/src/mgpp/text/mg_perf_hash_build.1	(revision 856)
@@ -0,0 +1,73 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_perf_hash_build 1 \*(Dt CITRI
+.SH NAME
+mg_perf_hash_build \- generate an order-preserving hash function for the stemmed dictionary
+.SH SYNOPSIS
+.B mg_perf_hash_build
+[
+.B \-h
+]
+[
+.BI \-r " rndseed"
+]
+[
+.BI \-d  " directory"
+]
+.if n .ti +9n
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_perf_hash_build
+generates an order-preserving hashing function from the compressed
+stemmed dictionary and writes it out to disk.
+.BR mg_passes (1)
+will make use of the hash function when it builds the inverted file.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-r " rndseed"
+This specifies the random seed to be used in the generation of the
+hash function; it must be an integer.  If this is not specified, the
+current time will be used as the random seed.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the mg
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf.dict
+Stemmed dictionary.
+.TP
+.B *.invf.dict.hash
+Data for a order-preserving perfect hash function.
+.SH "SEE ALSO"
+.na
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_invf_dict (1),
+.BR mg_passes (1),
+.BR mg_stem_idx (1),
+.BR mg_weights_build (1)
Index: /trunk/gsdl/src/mgpp/text/mg_perf_hash_build.cpp
===================================================================
--- /trunk/gsdl/src/mgpp/text/mg_perf_hash_build.cpp	(revision 856)
+++ /trunk/gsdl/src/mgpp/text/mg_perf_hash_build.cpp	(revision 856)
@@ -0,0 +1,164 @@
+/**************************************************************************
+ *
+ * mg_perf_hash_build.cpp -- Program to build a perfect hash function
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "memlib.h"
+#include "messages.h"
+#include "local_strings.h"
+#include "perf_hash.h"
+#include "netorder.h"
+
+#include "mg_files.h"
+#include "invf.h"
+#include "locallib.h"
+#include "words.h"
+#include "mg.h"
+
+/*
+   $Log$
+   Revision 1.1  2000/01/14 02:26:19  sjboddie
+   Rodgers new C++ mg
+
+   Revision 1.2  1999/10/17 23:43:27  cs025
+   Changes to eradicate Xmalloc
+
+   Revision 1.1  1999/10/11 02:58:01  cs025
+   Base install of MG-PP
+
+   Revision 1.1  1999/08/10 21:18:13  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:47  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:15  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:58  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:53  tes
+   * For version 1.1
+   *
+ */
+
+
+
+#define POOL_SIZE 1024*1024
+
+
+static void ProcessFiles (char *filename, int r) {
+  FILE *dictFile, *hashFile;
+  unsigned long i;
+  invf_dict_header idh;
+  perf_hash_data *phd;
+  u_char *pool;
+  unsigned long pool_left;
+  u_char **starts;
+
+  // read in the dictionary
+  dictFile = open_file (filename, INVF_DICT_SUFFIX, "rb",
+			MAGIC_STEM_BUILD, MG_ABORT);
+  idh.Read (dictFile);
+
+  // go to the start of the word dictionary
+  fseek (dictFile, idh.word_dict_start, SEEK_SET);
+  
+  if (!(pool = (u_char *) Xmalloc (POOL_SIZE)))
+    FatalError (1, "Out of memory");
+  pool_left = POOL_SIZE;
+
+  if (!(starts = (u_char **) Xmalloc (sizeof (u_char *) * idh.word_dict_size)))
+    FatalError (1, "Out of memory");
+
+  word_dict_el wordEl;
+  wordEl.SetNumLevels (idh.num_levels);
+  for (i = 0; i < idh.word_dict_size; i++) {
+    // read the next word and associated information
+    wordEl.Read (dictFile, idh.num_levels);
+
+    // push string onto pool data
+    register unsigned long l;
+    l = wordEl.el.size() + 1;
+    if (pool_left < l) {
+      pool = (u_char *) Xmalloc (POOL_SIZE);
+      pool_left = POOL_SIZE;
+    }
+    starts[i] = pool;
+    
+    *pool++ = wordEl.el.size();
+    bcopy ((char *) wordEl.el.begin(), (char *) pool, wordEl.el.size());
+    pool += l;
+    pool_left -= l;
+  }
+  fclose (dictFile);
+
+  // create perfect hash file
+  hashFile = create_file (filename, INVF_DICT_HASH_SUFFIX, "wb",
+			  MAGIC_HASH, MG_ABORT);
+  if (!(phd = gen_hash_func (idh.word_dict_size, starts, r)))
+    FatalError (1, "Unable to generate hash function");
+  if (write_perf_hash_data (hashFile, phd) == -1)
+    FatalError (1, "Unable to write hash function");
+  fclose (hashFile);
+}
+
+
+
+int main (int argc, char **argv) {
+  int r = -1;
+  char *filename = "";
+  int ch;
+  msg_prefix = argv[0];
+  opterr = 0;
+
+  while ((ch = getopt (argc, argv, "f:d:r:h")) != -1) {
+    switch (ch) {
+    case 'f':		// input file
+      filename = optarg;
+      break;
+    case 'd':
+      set_basepath (optarg);
+      break;
+    case 'r':
+      r = atoi (optarg);
+      break;
+    case 'h':
+    case '?':
+      fprintf (stderr, "usage: %s [-f input_file]"
+	       "[-d data directory] [-r random seed] [-h]\n", argv[0]);
+      exit (1);
+    }
+  }
+
+  ProcessFiles (filename, r);
+  return 0;
+}
Index: /trunk/gsdl/src/mgpp/text/mg_stem_idx.1
===================================================================
--- /trunk/gsdl/src/mgpp/text/mg_stem_idx.1	(revision 856)
+++ /trunk/gsdl/src/mgpp/text/mg_stem_idx.1	(revision 856)
@@ -0,0 +1,96 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_stem_idx 1 \*(Dt CITRI
+.SH NAME
+mg_stem_idx \- builds a stem index file
+.SH SYNOPSIS
+.B mg_stem_idx
+[
+.B \-h
+]
+[
+.BI \-b " entries-per-block"
+]
+.if n .ti +12n
+[
+.BI \-a " stemmer"
+]
+[
+.BI \-d " directory"
+]
+.if n .ti +12n
+.B \-s 1|2|3
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_stem_idx
+generates a stem index file for a collection.
+This program should be called three times: once for each
+.B -s
+parameter.  It uses the stemmed dictionary to create the stem index which
+contains pointers into the stemmed dictionary.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectoryyyyyyyy\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-b " entries-per-block"
+The dictionary is stored in blocks on disk; this option is used to set
+the number of entries per block. The default is 16.
+.TP
+.BI \-a " stemmer"
+The name of the stemmer to use, the default is the Lovin stemmer.
+.TP
+.B -s 1|2|3
+The stem method to apply for the stem index.
+.br
+1 = casefolded and non-stemmed
+.br
+2 = non-casefolded and stemmed
+.br
+3 = casefolded and stemmed
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the mg
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 22
+.B *.invf.dict
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict.blocked.1
+Stem index with stem index method 1.
+.TP
+.B *.invf.dict.blocked.2
+Stem index with stem index method 2.
+.TP
+.B *.invf.dict.blocked.3
+Stem index with stem index method 3.
+.SH "SEE ALSO"
+.na
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_invf_dict (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_weights_build (1)
Index: /trunk/gsdl/src/mgpp/text/mg_stem_idx.cpp
===================================================================
--- /trunk/gsdl/src/mgpp/text/mg_stem_idx.cpp	(revision 856)
+++ /trunk/gsdl/src/mgpp/text/mg_stem_idx.cpp	(revision 856)
@@ -0,0 +1,239 @@
+/**************************************************************************
+ *
+ * mg_stem_idx.cpp -- stem index builder
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+
+#include "sysfuncs.h"
+#include "messages.h"
+
+#include "mg_files.h"
+#include "invf.h"
+#include "UCArray.h"
+#include "words.h"
+
+#include "stemmer.h"
+
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\map>
+#  include <ospace\std\vector>
+#elif defined(GSDL_USE_STL_H)
+#  include <map.h>
+#  include <vector.h>
+#else
+#  include <map>
+#  include <vector>
+#endif
+
+
+/*
+   $Log$
+   Revision 1.1  2000/01/14 02:26:20  sjboddie
+   Rodgers new C++ mg
+
+ */
+
+
+typedef vector<unsigned long> WordNumList;
+typedef map<UCArray, WordNumList, DictLTUCArray> StemMapDict;
+
+
+void CreateStemDict (char *filename,
+		     StemMapDict &stemDict,
+		     int stemMethod,
+		     int stemmerNum) {
+  stemDict.erase (stemDict.begin(), stemDict.end());
+
+  // open the dictionary
+  FILE *dictFile = open_file (filename, INVF_DICT_SUFFIX, "rb",
+			      MAGIC_STEM_BUILD, MG_ABORT);
+  invf_dict_header idh;
+  idh.Read (dictFile);
+
+  fseek (dictFile, idh.word_dict_start, SEEK_SET);
+
+  unsigned long wordNum;
+  u_char mgWord[MAXSTEMLEN + 1];
+  word_dict_el wordEl;
+  UCArray stemEl;
+  wordEl.SetNumLevels (idh.num_levels);
+  for (wordNum=0; wordNum<idh.word_dict_size; wordNum++) {
+    // read in the next word
+    wordEl.Read (dictFile, idh.num_levels);
+
+    // convert the word to an "mg word"
+    mgWord[0] = wordEl.el.size();
+    bcopy ((char *)wordEl.el.begin(), (char *)&mgWord[1], wordEl.el.size());
+
+    // stem the word
+    stemmer (stemMethod, stemmerNum, mgWord);
+
+    // convert the result back to a UCArray
+    stemEl.erase (stemEl.begin(), stemEl.end());
+    stemEl.insert (stemEl.end(), &mgWord[1], &mgWord[1] + mgWord[0]);
+
+//      cout << "\"" << stemEl << "\" -> \"" << wordEl.el << "\"\n";
+    
+    // add this word number to the list of word numbers for this word
+    stemDict[stemEl].push_back (wordNum);
+  }
+
+  fclose (dictFile);
+}
+
+
+void WriteStemDict (char *filename,
+		    StemMapDict &stemDict,
+		    int stemMethod,
+		    int stemmerNum,
+		    unsigned long entriesPerBlock) {
+  // Create appropriate stem index file
+  FILE *stemDictFile;
+  if (stemMethod == 1) {
+    stemDictFile = create_file (filename, INVF_DICT_BLOCKED_1_SUFFIX,
+				"wb", MAGIC_STEM_1, MG_ABORT);
+  } else if (stemMethod == 2) {
+    stemDictFile = create_file (filename, INVF_DICT_BLOCKED_2_SUFFIX,
+				"wb", MAGIC_STEM_2, MG_ABORT);
+  } else if (stemMethod == 3) {
+    stemDictFile = create_file (filename, INVF_DICT_BLOCKED_3_SUFFIX,
+				"wb", MAGIC_STEM_3, MG_ABORT);
+  } else {
+    FatalError (1, "Unknown stem method %d", stemMethod);
+  }
+
+  stem_idx_header sih;
+  sih.lookback = 0;
+  sih.dict_size = stemDict.size();
+  sih.entries_per_block = entriesPerBlock;
+  sih.max_block_size = 0;
+  
+  sih.stemmer_num = stemmerNum;
+  sih.stem_method = stemMethod;
+
+  // write out a place-holder version of the header
+  sih.Write (stemDictFile);
+
+  sih.blocks_start = ftell (stemDictFile);
+
+  block_idx stemIdx;
+  unsigned long stemNum = 0;
+  stem_block_dict_el stemEl;
+  UCArray lastEl;
+
+  StemMapDict::const_iterator here = stemDict.begin();
+  StemMapDict::const_iterator end = stemDict.end();
+  while (here != end) {
+    // remember this stem (and position) if this is the start
+    // of a new block
+    if (stemNum % entriesPerBlock == 0) {
+      block_idx_info elIdx;
+      elIdx.el = (*here).first;
+      elIdx.block_ptr = ftell (stemDictFile);
+
+      // see if this block is the longest so far
+      if (stemIdx.size() > 0) {
+	unsigned long blockLen = elIdx.block_ptr -
+	  (*(stemIdx.end()-1)).block_ptr;
+	if (blockLen > sih.max_block_size) sih.max_block_size = blockLen;
+      }
+      
+      stemIdx.push_back (elIdx);
+      lastEl.erase (lastEl.begin(), lastEl.end()); // output full word
+    }
+
+    // copy the information for this stem
+    stemEl.el = (*here).first;
+    stemEl.equivWords = (*here).second;
+
+    // write out the stem
+    stemEl.Write (stemDictFile, &lastEl);
+    
+    here++; stemNum++;
+  }
+
+
+  // write out the element indexes
+  sih.num_blocks = stemIdx.size();
+  sih.block_idx_start = ftell (stemDictFile);
+  WriteBlockIdx (stemDictFile, stemIdx);
+  
+  // write out the stem dictionary header
+  fseek (stemDictFile, sizeof(unsigned long), SEEK_SET);
+  sih.Write (stemDictFile);
+
+
+  // close open files
+  fclose (stemDictFile);
+  
+  // print out information
+  Message ("Num word stems = %d\n", sih.dict_size);
+  Message ("Max stem block size = %d\n", sih.max_block_size);
+  Message ("Number of stem blocks written = %d\n", sih.num_blocks);
+}
+
+
+int main (int argc, char **argv) {
+  unsigned long entriesPerBlock = 16;
+  char *filename = "";
+  int ch;
+  int stemMethod = 0; // illegal value (no translation)
+  int stemmerNum = 0; // English stemmer
+  msg_prefix = argv[0];
+  opterr = 0;
+
+  while ((ch = getopt (argc, argv, "f:d:b:s:h")) != -1) {
+    switch (ch) {
+    case 'f':		// input file
+      filename = optarg;
+      break;
+    case 'd':
+      set_basepath (optarg);
+      break;
+    case 'b':
+      entriesPerBlock = atoi (optarg);
+      break;
+    case 's':
+      stemMethod = atoi (optarg);
+      break;
+    case 'a':
+      stemmerNum = stemmernumber ((unsigned char *) optarg);
+      break;
+    case 'h':
+    case '?':
+      fprintf (stderr, "usage: %s [-d directory] "
+	       "[-b entries-per-block] [-h] -s 1|2|3 "
+	       "[-a stemmer-method] -f name\n", argv[0]);
+      exit (1);
+    }
+  }
+  
+  if (stemMethod < 1 || stemMethod > 3)
+    FatalError (1, "Stem method must be 1, 2 or 3");
+
+  // read in the dictionary and create the in memory dictionary
+  StemMapDict stemDict;
+  CreateStemDict (filename, stemDict, stemMethod, stemmerNum);
+  
+  // write out the dictionary as a blocked file
+  WriteStemDict (filename, stemDict, stemMethod, stemmerNum, entriesPerBlock);
+  
+  return 0;
+}
Index: /trunk/gsdl/src/mgpp/text/mg_weights_build.1
===================================================================
--- /trunk/gsdl/src/mgpp/text/mg_weights_build.1	(revision 856)
+++ /trunk/gsdl/src/mgpp/text/mg_weights_build.1	(revision 856)
@@ -0,0 +1,80 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_weights_build 1 \*(Dt CITRI
+.SH NAME
+mg_weights_build \- builds the approximate weights file
+.SH SYNOPSIS
+.B mg_weights_build
+[
+.B \-h
+]
+[
+.BI \-b " num-bits"
+]
+.if n .ti +17n
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_weights_build
+is the program that generates the exact and approximate weights 
+files.  To do this, it uses the inverted file
+and dictionary.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-b " num-bits"
+This specifies the number of bits of precision that should be used for
+the approximate weights.  If not specified, it defaults to 8.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection is.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the mg
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.dict
+Compressed stemmed dictionary.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.B *.weight.approx
+Approximate document weights.
+.SH "SEE ALSO"
+.na
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_invf_dict (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_stem_idx (1)
Index: /trunk/gsdl/src/mgpp/text/mg_weights_build.cpp
===================================================================
--- /trunk/gsdl/src/mgpp/text/mg_weights_build.cpp	(revision 856)
+++ /trunk/gsdl/src/mgpp/text/mg_weights_build.cpp	(revision 856)
@@ -0,0 +1,426 @@
+/**************************************************************************
+ *
+ * mg_weights_build.cpp -- Program to build the document weights file
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "memlib.h"
+#include "messages.h"
+#include "local_strings.h"
+#include "bitio_gen.h"
+#include "bitio_m_stdio.h"
+
+#include "mg_files.h"
+#include "locallib.h"
+#include "invf.h"
+// #include "WordData.h"
+#include "UCArray.h"
+#include "FIvfLevelInfo.h"
+#include "FragLevelConvert.h"
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\map>
+#elif defined(GSDL_USE_STL_H)
+#  include <map.h>
+#else
+#  include <map>
+#endif
+
+
+/*
+   $Log$
+   Revision 1.1  2000/01/14 02:26:21  sjboddie
+   Rodgers new C++ mg
+
+   Revision 1.1  1999/10/11 02:58:06  cs025
+   Base install of MG-PP
+
+   Revision 1.1  1999/08/10 21:18:16  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:49  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:22  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.4  1994/11/29  00:32:05  tes
+   * Committing the new merged files and changes.
+   *
+   * Revision 1.3  1994/10/20  03:57:00  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:55  tes
+   * For version 1.1
+   *
+ */
+
+#define MAXBITS (sizeof(unsigned long) * 8)
+
+struct WBTagPtr {
+  unsigned long tagNum;
+  unsigned long tagPtr;
+  unsigned long fragOccur;
+  
+  WBTagPtr () {
+    tagNum = 0;
+    tagPtr = 0;
+    fragOccur = 0;
+  }
+};
+
+// maps tags to tag information
+typedef map<UCArray, WBTagPtr, DictLTUCArray> WBTagDict;
+
+typedef vector<float> Weights;
+
+
+static void ReadTagDict (const invf_dict_header &idh,
+			 FILE *dictFile,
+			 FILE *invfIdxFile,
+			 WBTagDict &tagDict) {
+  tagDict.erase (tagDict.begin(), tagDict.end());
+
+  // seek to the start of the tag information
+  fseek (dictFile, idh.tag_dict_start, SEEK_SET);
+  fseek (invfIdxFile, sizeof(unsigned long) +
+	 idh.word_dict_size*sizeof(unsigned long), SEEK_SET);
+  
+  unsigned long tagNum;
+  unsigned long tagPtr;
+  dict_el thisEl;
+  for (tagNum = 0; tagNum < idh.tag_dict_size; tagNum++) {
+    thisEl.Read (dictFile);
+    ReadUL (invfIdxFile, tagPtr);
+    tagDict[thisEl.el].tagNum = tagNum;
+    tagDict[thisEl.el].tagPtr = tagPtr;
+    tagDict[thisEl.el].fragOccur = thisEl.frag_occur;
+  }
+}
+
+static void AddWeight (Weights &w,
+		       unsigned long levelDocNum,
+		       unsigned long termFreq,
+		       float idf) {
+  double weight = termFreq * idf;
+  w[levelDocNum-1] += weight * weight;
+}
+
+static void GenerateLevelWeights (const invf_dict_header &idh,
+				  const invf_file_header &ifh,
+				  unsigned long numLevelDocs,
+				  unsigned long levelNum,
+				  FILE *dictFile,
+				  FILE *invfFile,
+				  FILE *invfIdxFile,
+				  const FragLevelConvert &fragLevelConvert,
+				  Weights &w) {
+  // pre-allocate the right number of weights
+  w.erase (w.begin(), w.end());
+  w.insert (w.end(), (Weights::size_type)numLevelDocs, (float)0.0);
+
+  double logN = log ((double) numLevelDocs);
+
+  // reset the files
+  fseek (dictFile, idh.word_dict_start, SEEK_SET);
+  fseek (invfIdxFile, sizeof (unsigned long), SEEK_SET);
+  
+  // process each word adding its contributions to the document weights
+  unsigned long wordNum;
+  unsigned long wordStart;
+  word_dict_el wordEl;
+  wordEl.SetNumLevels (idh.num_levels);
+  for (wordNum=0; wordNum<idh.word_dict_size; wordNum++) {
+    // give a little feedback every 4096 words
+    if ((wordNum & 0xfff) == 0) fprintf (stderr, ".");
+    
+    wordEl.Read (dictFile, idh.num_levels);
+    ReadUL (invfIdxFile, wordStart);
+
+    float idf = logN - log ((double) wordEl.levelFreqs[levelNum]);
+
+    // seek to the appropriate place in the inverted file
+    fseek (invfFile, wordStart, SEEK_SET);
+    stdio_bitio_buffer buffer (invfFile);
+    
+    unsigned long B = BIO_Bblock_Init (idh.num_frags, wordEl.frag_occur);
+    unsigned long fragNum = 0;
+    unsigned long levelDocNum = 0;
+    unsigned long lastLevelDocNum = 0;
+    unsigned long termFreq = 0;
+    unsigned long checkLevelFreq = 0;
+    
+    unsigned long count, i;
+    for (i=0; i<wordEl.frag_occur; i++) {
+      fragNum += buffer.bblock_decode (B, NULL);
+      if (!ifh.word_level_index) count = buffer.gamma_decode (NULL);
+      else count = 1;
+
+      if (fragNum > idh.num_frags)
+	FatalError (1, "fragNum = %d, "
+		    "number of fragments = %d\n"
+		    "wordNum = %d\n"
+		    "i = %d, frag_occur = %d\n",
+		    fragNum, idh.num_frags, wordNum, i, wordEl.frag_occur);
+
+      if (!fragLevelConvert.FragToLevel (fragNum, levelDocNum))
+	FatalError (1, "could not convert fragment number %d", fragNum);
+
+      if (levelDocNum == 0 || levelDocNum > numLevelDocs)
+	FatalError (1, "bad level document number %d", levelDocNum);
+      
+      if (levelDocNum != lastLevelDocNum) {
+	// new level document
+	if (lastLevelDocNum > 0) {
+	  AddWeight (w, lastLevelDocNum, termFreq, idf);
+	  checkLevelFreq++;
+	}
+	lastLevelDocNum = levelDocNum;
+	termFreq = 0;
+      }
+      termFreq += count;
+    }
+
+    if (lastLevelDocNum > 0) {
+      AddWeight (w, lastLevelDocNum, termFreq, idf);
+      checkLevelFreq++;
+    }
+
+    if (checkLevelFreq != wordEl.levelFreqs[levelNum]) {
+      cerr << "bad level freq " << checkLevelFreq << " != "
+	   << wordEl.levelFreqs[levelNum] << ", word \""
+	   << wordEl.el << "\" (" << wordNum << ")\n";
+      exit (1);
+    }
+    
+    buffer.done();
+  }
+
+  if (w.size() != numLevelDocs) 
+    FatalError (1, "wrong number of weights created %d != %d",
+		w.size(), numLevelDocs);
+}
+
+static void WriteExactWeights (FILE *weightsFile,
+			       unsigned long &diskPtr,
+			       const Weights &w) {
+  diskPtr = ftell(weightsFile);
+
+  Weights::const_iterator here = w.begin();
+  Weights::const_iterator end = w.end();
+  while (here != end) {
+//      cout << *here << "\n";
+    WriteF (weightsFile, *here);
+    here++;
+  }
+}
+
+static void WriteApproxWeights (FILE *approxWeightsFile,
+				unsigned long &diskPtr,
+				const Weights &w,
+				unsigned char bits) {
+  diskPtr = ftell(approxWeightsFile);
+
+  // calculate L, U and B
+  double L = 1e300;
+  double U = 0;
+  float wgt;
+  Weights::const_iterator here = w.begin();
+  Weights::const_iterator end = w.end();
+  while (here != end) {
+    wgt = sqrt (*here);
+    if (wgt > U) U = wgt;
+    if (wgt > 0 && wgt < L) L = wgt;
+    here++;
+  }
+
+  double B = pow (U / L, pow (2.0, -(double) bits));
+
+  fprintf (stderr, "L = %f\n", L);
+  fprintf (stderr, "U = %f\n", U);
+  fprintf (stderr, "B = %f\n", B);
+
+  WriteUC (approxWeightsFile, bits);
+  WriteD (approxWeightsFile, L);
+  WriteD (approxWeightsFile, B);
+  
+
+  unsigned long max = (bits == 32) ? 0xffffffff : (1 << bits) - 1;;
+  unsigned long i=0, buf=0, pos=0;
+  here = w.begin();
+  end = w.end();
+  while (here != end) {
+    unsigned long fx;
+    wgt = sqrt (*here);
+    if (wgt == 0) {
+      wgt = L;
+#ifndef QUIET
+      Message ("Warning: Document %d had a weight of 0.", i);
+#endif
+    }
+    fx = (unsigned long) floor (log (wgt / L) / log (B));
+
+    if (fx > max) fx = max;
+
+    buf |= (fx << pos);
+    pos += bits;
+
+    if (pos >= MAXBITS) {
+      WriteUL (approxWeightsFile, buf);
+      buf = fx >> (bits - (pos - MAXBITS));
+      pos = pos - MAXBITS;
+    }
+
+    here++; i++;
+  }
+
+  // write out the last bits
+  if (pos > 0) WriteUL (approxWeightsFile, buf);
+}
+
+int main (int argc, char **argv) {
+  unsigned char bits = 8;
+  char *filename = "";
+  int ch;
+  opterr = 0;
+  msg_prefix = argv[0];
+
+  while ((ch = getopt (argc, argv, "f:d:b:h")) != -1) {
+    switch (ch) {
+    case 'f':		// input file
+      filename = optarg;
+      break;
+    case 'd':
+      set_basepath (optarg);
+      break;
+    case 'b':
+      bits = atoi (optarg);
+      if (bits > 32) {
+	fprintf (stderr, "b may only take values 0-32\n");
+	exit (1);
+      }
+      break;
+    case 'h':
+    case '?':
+      fprintf (stderr, "usage: %s [-f input_file]"
+	       "[-d data directory] [-b bits] [-h]\n", argv[0]);
+      exit (1);
+    }
+  }
+
+  
+  // open the dictionary
+  FILE *dictFile = open_file (filename, INVF_DICT_SUFFIX, "rb",
+			      MAGIC_STEM_BUILD, MG_ABORT);
+  invf_dict_header idh;
+  idh.Read (dictFile);
+
+  // open the inverted file
+  FILE *invfFile = open_file (filename, INVF_SUFFIX, "rb",
+			MAGIC_INVF, MG_ABORT);
+  invf_file_header ifh;
+  ifh.Read (invfFile);
+  if (ifh.skip_mode != SKIP_MODE_NO_SKIPS)
+    FatalError (1, "The invf file contains skips. Unable to create weights.");
+
+  // open the inverted index file
+  FILE *invfIdxFile = open_file (filename, INVF_IDX_SUFFIX, "rb",
+				 MAGIC_INVI, MG_ABORT);
+  
+  // read the level information
+  FILE *levelFile = open_file (filename, INVF_LEVEL_SUFFIX, "rb",
+			       MAGIC_INVF_LEVELS, MG_ABORT);
+  FIvfLevel ivfLevel;
+  ivfLevel.Read (levelFile);
+  fclose (levelFile);
+
+  // read in the tag dictionary and inverted file pointers
+  WBTagDict tagDict;
+  ReadTagDict (idh, dictFile, invfIdxFile, tagDict);
+
+  
+  // create the weights file
+  FILE *weightsFile = create_file (filename, WEIGHTS_SUFFIX, "wb",
+				   MAGIC_WGHT, MG_ABORT);
+
+  // create the approx weights file
+  FILE *approxWeightsFile = create_file (filename, APPROX_WEIGHTS_SUFFIX, "wb",
+					 MAGIC_WGHT_APPROX, MG_ABORT);
+
+  
+  // create weights for each document level
+  FragLevelConvert fragLevelConvert;
+  Weights w;
+  IvfLevelInfoMap::iterator levelHere = ivfLevel.levelInfo.begin();
+  IvfLevelInfoMap::iterator levelEnd = ivfLevel.levelInfo.end();
+  unsigned long levelNum = 0;
+  while (levelHere != levelEnd) {
+    const UCArray &levelName = (*levelHere).first;
+    
+    // read the tag information about this level
+    fragLevelConvert.Read(invfFile, tagDict[levelName].tagPtr,
+			  idh.num_frags, tagDict[levelName].fragOccur);
+    
+    // create the weights for this level
+    GenerateLevelWeights (idh, ifh, tagDict[levelName].fragOccur,
+			  levelNum, dictFile, invfFile, invfIdxFile,
+			  fragLevelConvert, w);
+
+    // write out the exact weights
+    WriteExactWeights (weightsFile,
+		       (*levelHere).second.exactWeightsDiskPtr,
+		       w);
+    
+    // write out the approximate weights
+    WriteApproxWeights (approxWeightsFile,
+			(*levelHere).second.approxWeightsDiskPtr,
+			w, bits);
+    
+    levelHere++; levelNum++;
+  }
+  
+
+  // close input files
+  fclose (dictFile);
+  fclose (invfFile);
+  fclose (invfIdxFile);
+
+  // update the level information
+  levelFile = create_file (filename, INVF_LEVEL_SUFFIX, "wb",
+			   MAGIC_INVF_LEVELS, MG_ABORT);
+  ivfLevel.Write (levelFile);
+  fclose (levelFile);
+
+  // close output files
+  fclose (weightsFile);
+  fclose (approxWeightsFile);
+
+
+  return 0;
+}
Index: /trunk/gsdl/src/mgpp/text/stemmer.cpp
===================================================================
--- /trunk/gsdl/src/mgpp/text/stemmer.cpp	(revision 856)
+++ /trunk/gsdl/src/mgpp/text/stemmer.cpp	(revision 856)
@@ -0,0 +1,146 @@
+/**************************************************************************
+ *
+ * stemmer.cpp -- The stemmer/case folder
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "stemmer.h"
+
+#include "lovinstem.h"
+#include "simplefrenchstem.h"
+#include "unitool.h"
+
+/*
+   $Log$
+   Revision 1.1  2000/01/14 02:26:22  sjboddie
+   Rodgers new C++ mg
+
+   Revision 1.1  1999/10/11 02:58:34  cs025
+   Base install of MG-PP
+
+   Revision 1.1  1999/08/10 21:18:23  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.3  1998/12/17 09:12:54  rjmcnab
+
+   Altered mg to process utf-8 encoded Unicode. The main changes
+   are in the parsing of the input, the casefolding, and the stemming.
+
+   Revision 1.2  1998/11/25 07:55:51  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:42  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:57:05  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:42:10  tes
+   * For version 1.1
+   *
+ */
+
+
+#define LOVINSTEMMER        0
+#define SIMPLEFRENCHSTEMMER 1
+
+
+/* decode the utf-8 encoded unicode, casefold and then recode 
+ * making sure the final length doesn't exceed the original
+ * length */
+static void unicode_casefold (u_char *word) {
+  unsigned short out[256]; /* temp space */
+  int i;
+  int len;
+
+  /* decode */
+  utf8_word_to_unicode (word, out, 255);
+  len = out[0];
+
+  /* casefold and simplify-fold */
+  for (i=0; i<len; i++) {
+    out[i+1] = unicode_tosimplified(unicode_tolower(out[i+1]));
+  }
+
+  /* re-code */
+  unicode_to_utf8_word (out, word, word[0]+1);
+}
+
+
+int stemmernumber (u_char *stemmerdescription) {
+  u_char descript[MAX_STEM_DESCRIPTION_LEN];
+  int i;
+
+  /* copy and case-fold the description */
+  for (i=0; (stemmerdescription[i] != '\0') && 
+	 (i < MAX_STEM_DESCRIPTION_LEN-1); i++)
+    descript[i] = tolower (stemmerdescription[i]);
+  descript[i] = '\0';
+
+  /* map the description to its number */
+
+  if ((strcmp ((char *) descript,  "0") == 0) ||
+      (strcmp ((char *) descript, "english") == 0) ||
+      (strcmp ((char *) descript, "lovin") == 0))
+    return LOVINSTEMMER;
+
+  if ((strcmp ((char *) descript, "1") == 1) ||
+      (strcmp ((char *) descript, "french") == 0) ||
+      (strcmp ((char *) descript, "simplefrench") == 0))
+    return SIMPLEFRENCHSTEMMER;
+
+  return -1;
+}
+
+
+
+/*
+ * Method 0 - Do not stem or case fold.
+ * Method 1 - Case fold.
+ * Method 2 - Stem.
+ * Method 3 - Case fold and stem.
+ *
+ * The stemmer number should be obtained using
+ * the stemmernumber function above.
+ */
+void 
+stemmer (int method, int stemmer, u_char *word) {
+  if (method & 1) {
+    unicode_casefold (word);
+  }
+
+  if (method & 2) {
+    switch (stemmer) {
+    case LOVINSTEMMER: lovinstem (word);
+      break;
+    case SIMPLEFRENCHSTEMMER: simplefrenchstem (word);
+      break;
+    }
+  }
+}
Index: /trunk/gsdl/src/mgpp/text/stemmer.h
===================================================================
--- /trunk/gsdl/src/mgpp/text/stemmer.h	(revision 856)
+++ /trunk/gsdl/src/mgpp/text/stemmer.h	(revision 856)
@@ -0,0 +1,67 @@
+/**************************************************************************
+ *
+ * stemmer.h -- The stemmer/case folder
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef STEMMER_H
+#define STEMMER_H
+
+#include "sysfuncs.h"
+
+#define STEMMER_MASK 3
+#define MAX_STEM_DESCRIPTION_LEN 16
+
+/* stemmernumber will return the stemmer for
+ * a description of the stemmer. Stemmer descriptions
+ * are not case sensitive. Valid descriptions are:
+ *
+ *   'English'
+ *   'Lovin'
+ *   'French'
+ *   'SimpleFrench'
+ *
+ * More than one description might result in the same
+ * stemmer number (for example, for stemming 'English'
+ * we currently use the 'Lovin' stemmer).
+ *
+ * stemmerdescription is a normal C, null-terminated,
+ * string.
+ *
+ * stemmernumber will return -1 if it doesn't know the
+ * stemmer description.
+ */
+int stemmernumber (u_char *stemmerdescription);
+
+/*
+ * Method 0 - Do not stem or case fold.
+ * Method 1 - Case fold.
+ * Method 2 - Stem.
+ * Method 3 - Case fold and stem.
+ *
+ * The stemmer number should be obtained using function
+ * stemmernumber above.
+ */
+#ifdef __cplusplus
+extern "C" 
+#endif
+void stemmer (int method, int stemmer, u_char * word);
+
+#endif
Index: /trunk/gsdl/src/mgpp/text/text.h
===================================================================
--- /trunk/gsdl/src/mgpp/text/text.h	(revision 856)
+++ /trunk/gsdl/src/mgpp/text/text.h	(revision 856)
@@ -0,0 +1,195 @@
+/**************************************************************************
+ *
+ * text.h -- Header file for compression related stuff
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+
+#ifndef H_TEXT
+#define H_TEXT
+
+#include "huffman.h"
+#include <stdio.h>
+
+
+
+
+/*****************************************************************************
+ *
+ * There are several different methods of compressing the text of the
+ * database. The following defines list the different methods of text
+ * compression
+ *
+ */
+
+
+/* The dictionary contains all the fragments that occur in the collection
+   i.e. escapes are not possible */
+#define MG_COMPLETE_DICTIONARY 0
+
+
+/* Certain words have been deleted from the dictionary. The words deleted
+   have been used to create the frequency huffman codes of the characters. 
+   This dictionary has an escape code and may be used to compress novel
+   words. This dictionary may fail if there is a novel character. */
+#define MG_PARTIAL_DICTIONARY 1
+
+
+/* This dictionary has an escape so that novel words and non-words can be 
+   coded. The method for coding the novel words and non-words is determined
+   by a dictionary parameter. */
+#define MG_SEED_DICTIONARY 2
+
+
+
+
+
+
+
+
+
+/*****************************************************************************
+ *
+ * With a seed dictionary there are several methods for coding the novel
+ * words and non-words the following defined values specify the different
+ * methods of coding.
+ *
+ */
+
+
+/* Code novel words and non-words character by character using huffman codes.
+   The huffman codes for the word and non-word lengths and characters are 
+   generated from the distribution of lengths and characters in the
+   dictionary. */
+#define MG_NOVEL_HUFFMAN_CHARS 0
+
+
+/* This method codes novel words using delta codes. The novel words are stored
+   in a auxillary dictionary which is built by pass two. */
+#define MG_NOVEL_DELTA 2
+
+
+/* This method codes novel words using hybrid version of delta. The novel
+   words are stored in a auxillary dictionary which is built by pass two. */
+#define MG_NOVEL_HYBRID 3
+
+
+
+
+
+
+/* This specified an amount of extra space allocated in the
+   compression_dict_header for adding new parameters. As new
+   parameters are added this should be decreased. */
+#define TEXT_PARAMS 15
+
+
+
+struct compression_dict_header {
+  u_long dict_type;
+  u_long novel_method;
+  u_long params[TEXT_PARAMS];
+  u_long num_words[2];
+  u_long num_word_chars[2];
+  u_long lookback;
+};
+
+
+struct comp_frags_header {
+  huff_data hd;
+  u_long uncompressed_size;
+  u_long huff_words_size[MAX_HUFFCODE_LEN + 1];
+};
+
+// BOGUSTEXTLEN is used to replace the ratio
+// in the compressed_text_header while preparing
+// for UCArray
+#define BOGUSTEXTLEN 1000000
+
+struct compressed_text_header {
+  u_long num_of_docs;
+  u_long num_of_words; // number of words in collection
+  double num_of_bytes;
+
+  compressed_text_header ();
+  void Clear ();
+
+  // you must seek to the appropriate place before calling
+  // Read or Write
+  bool Read (FILE *f);
+  bool Write (FILE *f) const;
+};
+
+
+struct compression_stats_header {
+  u_long num_docs;
+  u_long dummy;     // alignment
+  double num_bytes; 
+};
+
+
+struct frags_stats_header {
+  u_long num_frags;
+  u_long mem_for_frags;
+};
+
+
+struct aux_frags_header {
+  u_long num_frags;
+  u_long mem_for_frags;
+};
+
+
+struct auxiliary_dict {
+  aux_frags_header afh[2];
+  u_char *word_data[2];
+  u_char **words[2];
+  int blk_start[2][33], blk_end[2][33];	/* blk_start and blk_end are required
+					   for the hybrid methods */
+
+  auxiliary_dict ();
+  ~auxiliary_dict ();
+
+  void Clear ();
+};
+
+
+struct compression_dict {
+  compression_dict_header cdh;
+  comp_frags_header *cfh[2];
+  unsigned long MemForCompDict;
+  u_char ***values[2];
+  u_char *escape[2];
+  huff_data *chars_huff[2];
+  u_long **chars_vals[2];
+  huff_data *lens_huff[2];
+  u_long **lens_vals[2];
+  auxiliary_dict *ad;
+  int fast_loaded;
+
+  compression_dict ();
+  ~compression_dict ();
+
+  void Clear ();
+};
+
+
+#endif
Index: /trunk/gsdl/src/mgpp/text/text.pass1.cpp
===================================================================
--- /trunk/gsdl/src/mgpp/text/text.pass1.cpp	(revision 856)
+++ /trunk/gsdl/src/mgpp/text/text.pass1.cpp	(revision 856)
@@ -0,0 +1,409 @@
+/**************************************************************************
+ *
+ * text.pass1.cpp -- Text compression (Pass 1)
+ * Copyright (C) 1994  Neil Sharman, Gary Eddy and Alistair Moffat
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "huffman.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+
+#include "mg_files.h"
+#include "mg.h"
+#include "build.h"
+#include "locallib.h"
+#include "words.h"
+#include "text.h"
+#include "hash.h"
+#include "local_strings.h"
+
+#include "TextEl.h"
+
+
+/*
+   $Log$
+   Revision 1.1  2000/01/14 02:26:23  sjboddie
+   Rodgers new C++ mg
+
+   Revision 1.1  1999/10/11 02:58:37  cs025
+   Base install of MG-PP
+
+   Revision 1.1  1999/08/10 21:18:25  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/12/17 09:12:54  rjmcnab
+
+   Altered mg to process utf-8 encoded Unicode. The main changes
+   are in the parsing of the input, the casefolding, and the stemming.
+
+   Revision 1.1  1998/11/17 09:35:46  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.4  1994/11/25  03:47:47  tes
+   * Committing files before adding the merge stuff.
+   *
+   * Revision 1.3  1994/10/20  03:57:09  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:42:13  tes
+   * For version 1.1
+   *
+ */
+
+
+#define POOL_SIZE 1024*1024
+#define INITIAL_HASH_SIZE 7927
+
+
+
+
+
+
+
+typedef struct hash_rec
+  {
+    unsigned long wcnt;		/* word frequency */
+    unsigned long occurance_num;
+    u_char *word;
+  }
+hash_rec;
+
+typedef struct dict_data
+  {
+    hash_rec *HashTable;
+    unsigned long HashSize;
+    unsigned long HashUsed;
+    unsigned long wordnum;
+    unsigned long words_read;
+    unsigned long bytes_diff;
+    huff_data hd;
+  }
+dict_data;
+
+
+
+static unsigned long longestDoc = 0;
+static unsigned long occurance_num = 0;
+static dict_data DictData[2];
+
+static u_char *Pool;
+static int PoolLeft;
+static double inputbytes = 0; /* [RJM 07/97: 4G limit] */
+static unsigned long MaxMemInUse = 0;
+static unsigned long MemInUse = 0;
+static compression_stats_header csh = {0, 0, 0.0}; /* [RJM 07/97: 4G limit] */
+
+
+static void ChangeMem (int Change) {
+  MemInUse += Change;
+  if (MemInUse > MaxMemInUse) MaxMemInUse = MemInUse;
+}
+
+
+int init_text_1 (const TagInfo &/*tagInfo*/, char * /*FileName*/) {
+  int which;
+  
+  if (!(Pool = (u_char *) Xmalloc (POOL_SIZE))) {
+    Message ("Unable to allocate memory for pool");
+    return (COMPERROR);
+  }
+  PoolLeft = POOL_SIZE;
+  ChangeMem (POOL_SIZE);
+  
+  for (which = 1; which >= 0; which--) {
+    u_char *word;
+    hash_rec *ent;
+    dict_data *dd = &DictData[which];
+    
+    dd->wordnum = 0;
+    dd->words_read = 0;
+    dd->bytes_diff = 0;
+    dd->HashSize = INITIAL_HASH_SIZE;
+    dd->HashUsed = 0;
+    
+    if (!(dd->HashTable = (hash_rec *) Xmalloc (sizeof (hash_rec) * dd->HashSize))) {
+      Message ("Unable to allocate memory for table");
+      return (COMPERROR);
+    }
+    ChangeMem (sizeof (hash_rec) * dd->HashSize);
+    bzero ((char *) (dd->HashTable), sizeof (hash_rec) * dd->HashSize);
+
+    word = Pool;
+    *Pool++ = '\0';
+    PoolLeft--;
+    {
+      register u_char *wptr;
+      register int hsize = dd->HashSize;
+      register unsigned long hashval, step;
+      
+      HASH (hashval, step, word, hsize);
+      wptr = (dd->HashTable + hashval)->word;
+      while (wptr) {
+	hashval += step;
+	if (hashval >= (unsigned long)hsize)
+	  hashval -= hsize;
+	wptr = (dd->HashTable + hashval)->word;
+      }
+      ent = dd->HashTable + hashval;
+    }
+    ent->wcnt = 1;
+    ent->word = word;
+    dd->HashUsed = 1;
+  }
+  return (COMPALLOK);
+}
+
+
+static int process_text_element (const u_char *s_in, int l_in) {
+  const u_char *end = s_in + l_in - 1;
+  
+   /*
+   ** Alternately parse off words and non-words from the input
+   ** stream beginning with a non-word. Each token is then
+   ** inserted into the set if it does not exist or has it's
+   ** frequency count incremented if it does.
+   */
+  
+  bool which = false; // non-word
+  for (; s_in <= end; which = !which) {
+    u_char Word[MAXWORDLEN + 1];
+    dict_data *dd = &DictData[which];
+    
+    /* First parse a word or non-word out of the string */
+    if (which) PARSE_WORD (Word, s_in, end);
+    else PARSE_NON_WORD (Word, s_in, end);
+    
+    dd->wordnum++;
+    inputbytes += *Word;
+    dd->words_read++;
+    
+    /* Search the hash table for Word */
+    {
+      register unsigned long hashval, step;
+      register int hsize = dd->HashSize;
+      HASH (hashval, step, Word, hsize);
+      for (;;) {
+	register u_char *s1;
+	register u_char *s2;
+	register int len;
+	register hash_rec *ent;
+	ent = dd->HashTable + hashval;
+	if (!ent->word) {
+	  int len = *Word + 1;
+	  if (len > PoolLeft) {
+	    if (!(Pool = (u_char *) Xmalloc (POOL_SIZE))) {
+	      Message ("Unable to allocate memory for pool");
+	      return (COMPERROR);
+	    }
+	    PoolLeft = POOL_SIZE;
+	    ChangeMem (POOL_SIZE);
+	  }
+	  ent->occurance_num = occurance_num++;
+	  ent->wcnt = 1;
+	  ent->word = Pool;
+	  memcpy (Pool, Word, len);
+	  Pool += len;
+	  PoolLeft -= len;
+	  dd->HashUsed++;
+	  dd->bytes_diff += Word[0];
+	  break;
+	}
+	
+	/* Compare the words */
+	s1 = Word;
+	s2 = ent->word;
+	len = *s1 + 1;
+	for (; len; len--)
+	  if (*s1++ != *s2++) break;
+	
+	if (len) {
+	  hashval = (hashval + step);
+	  if (hashval >= (unsigned long)hsize) hashval -= hsize;
+	} else {
+	  ent->wcnt++;
+	  break;
+	}
+      }
+    }
+
+    
+    if (dd->HashUsed >= dd->HashSize >> 1) {
+      hash_rec *ht;
+      unsigned long size;
+      unsigned long i;
+      size = prime (dd->HashSize * 2);
+      if (!(ht = (hash_rec *) Xmalloc (sizeof (hash_rec) * size))) {
+	Message ("Unable to allocate memory for table");
+	return (COMPERROR);
+      }
+      ChangeMem (sizeof (hash_rec) * size);
+      bzero ((char *) ht, sizeof (hash_rec) * size);
+
+      for (i = 0; i < dd->HashSize; i++)
+	if (dd->HashTable[i].word) {
+	  register u_char *wptr;
+	  register unsigned long hashval, step;
+	  
+	  wptr = dd->HashTable[i].word;
+	  HASH (hashval, step, wptr, size);
+	  wptr = (ht + hashval)->word;
+	  while (wptr) {
+	    hashval += step;
+	    if (hashval >= size) hashval -= size;
+	    wptr = (ht + hashval)->word;
+	  }
+	  ht[hashval] = dd->HashTable[i];
+	}
+      Xfree (dd->HashTable);
+      ChangeMem (-sizeof (hash_rec) * dd->HashSize);
+      dd->HashTable = ht;
+      dd->HashSize = size;
+    }
+  }
+
+  return COMPALLOK;
+}
+
+
+int process_text_1 (const TagInfo &/*tagInfo*/, const TextElArray &doc) {
+  unsigned long textLen = 0;
+  unsigned long docLen = 0;
+  int retValue;
+
+  // process each text element in this document
+  TextElArray::const_iterator here = doc.begin();
+  TextElArray::const_iterator end = doc.end();
+  while (here != end) {
+    textLen = (*here).text.size();
+    docLen += textLen;
+
+    retValue = process_text_element ((*here).text.begin(), textLen);
+    if (retValue != COMPALLOK) return retValue;
+    
+    here++;
+  }
+
+  // get max document length
+  if (docLen > longestDoc) longestDoc = docLen;
+
+  // update header information
+  csh.num_docs++;
+  csh.num_bytes += docLen;
+  
+  return COMPALLOK;
+}
+
+
+static int PackHashTable (dict_data * dd) {
+  int s, d;
+  for (s = d = 0; (unsigned int)s < dd->HashSize; s++)
+    if (dd->HashTable[s].word)
+      dd->HashTable[d++] = dd->HashTable[s];
+  
+  ChangeMem (-sizeof (hash_rec) * dd->HashSize);
+  ChangeMem (sizeof (hash_rec) * dd->HashUsed);
+  
+  if (!(dd->HashTable = (hash_rec *) Xrealloc (dd->HashTable,
+			   sizeof (hash_rec) * dd->HashUsed))) {
+    Message ("Out of memory");
+    return COMPERROR;
+  }
+  dd->HashSize = dd->HashUsed;
+  return COMPALLOK;
+}
+
+
+static int ent_comp (const void *s1, const void *s2) {
+  return casecompare (((hash_rec *) s1)->word, ((hash_rec *) s2)->word);
+}
+
+
+static void WriteHashTable (FILE * fp, dict_data * dd) {
+  frags_stats_header fsh;
+  u_long j = 0;
+  u_char *curr;
+
+  if (PackHashTable (dd) == COMPERROR) return;
+  
+  qsort (dd->HashTable, dd->HashUsed, sizeof (hash_rec), ent_comp);
+  
+  fsh.num_frags = dd->HashSize;
+  fsh.mem_for_frags = dd->HashSize;
+  for (j = 0; j < dd->HashSize; j++)
+    fsh.mem_for_frags += dd->HashTable[j].word[0];
+  
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(fsh.num_frags);
+  HTONUL(fsh.mem_for_frags);
+  
+  fwrite (&fsh, sizeof (fsh), 1, fp);
+  
+  for (j = 0; j < dd->HashSize; j++) {
+    curr = dd->HashTable[j].word;
+    
+    /* [RPAP - Jan 97: Endian Ordering] */
+    HTONUL(dd->HashTable[j].wcnt);
+    HTONUL(dd->HashTable[j].occurance_num);
+    
+    fwrite (&dd->HashTable[j].wcnt, sizeof (dd->HashTable[j].wcnt), 1, fp);
+    fwrite (&dd->HashTable[j].occurance_num,
+	    sizeof (dd->HashTable[j].occurance_num), 1, fp);
+    
+    /* [RPAP - Jan 97: Endian Ordering] */
+    NTOHUL(dd->HashTable[j].wcnt);
+    NTOHUL(dd->HashTable[j].occurance_num);
+    
+    fwrite (curr, sizeof (u_char), curr[0] + 1, fp);
+  }
+}
+
+
+int done_text_1 (const TagInfo &/*tagInfo*/, char *file_name) {
+  char *temp_str;
+  FILE *fp;
+  
+  if (!(fp = create_file (file_name, TEXT_STATS_DICT_SUFFIX, "wb",
+			  MAGIC_STATS_DICT, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return COMPERROR;
+  
+  temp_str = msg_prefix;
+  msg_prefix = "text.pass1";
+  
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(csh.num_docs);
+  HTOND(csh.num_bytes); /* [RJM 07/97: 4G limit] */
+
+  fwrite (&csh, sizeof (csh), 1, fp);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(csh.num_docs);
+  NTOHD(csh.num_bytes); /* [RJM 07/97: 4G limit] */
+
+  WriteHashTable (fp, &DictData[0]);
+  WriteHashTable (fp, &DictData[1]);
+  msg_prefix = temp_str;
+  return COMPALLOK;
+}
Index: /trunk/gsdl/src/mgpp/text/text.pass2.cpp
===================================================================
--- /trunk/gsdl/src/mgpp/text/text.pass2.cpp	(revision 856)
+++ /trunk/gsdl/src/mgpp/text/text.pass2.cpp	(revision 856)
@@ -0,0 +1,605 @@
+/**************************************************************************
+ *
+ * text.pass2.cpp -- Text compression (Pass 2)
+ * Copyright (C) 1994  Neil Sharman, Gary Eddy and Alistair Moffat
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "local_strings.h"
+#include "bitio_m_stdio.h"
+#include "huffman.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg.h"
+#include "mg_files.h"
+#include "build.h"
+#include "words.h"
+#include "text.h"
+#include "hash.h"
+#include "locallib.h"
+#include "comp_dict.h"
+
+#include "FText.h"
+
+
+/*
+   $Log$
+   Revision 1.1  2000/01/14 02:26:24  sjboddie
+   Rodgers new C++ mg
+
+   Revision 1.1  1999/10/11 02:58:38  cs025
+   Base install of MG-PP
+
+   Revision 1.1  1999/08/10 21:18:25  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/12/17 09:12:54  rjmcnab
+
+   Altered mg to process utf-8 encoded Unicode. The main changes
+   are in the parsing of the input, the casefolding, and the stemming.
+
+   Revision 1.1  1998/11/17 09:35:47  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:57:10  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:42:14  tes
+   * For version 1.1
+   *
+ */
+
+#define POOL_SIZE 1024*256
+
+struct char_pool {
+  struct char_pool *next;
+  u_long left;
+  u_char *ptr;
+  u_char pool[POOL_SIZE];
+};
+
+struct novel_hash_rec {
+  u_long ordinal_num;
+  u_char *word;
+};
+
+
+#define INITIAL_HASH_SIZE 7927
+
+struct novel_hash_table {
+  novel_hash_rec *HashTable;
+  u_long HashSize, HashUsed;
+  char_pool *first_pool;
+  char_pool *pool;
+  u_long next_num, binary_start;
+  novel_hash_rec **code_to_nhr;
+};
+
+
+static FILE *text;
+static stdio_bitio_buffer textOutBuf;
+static u_long headLen;
+
+static novel_hash_table nht[2];
+
+int blk_start[2][33], blk_end[2][33];
+
+// 0 = non-word, 1 = word
+unsigned char whichWordType = 0;
+
+CompressTextInfoMap textInfoMap;
+
+
+static char_pool *new_pool (char_pool * pool) {
+  char_pool *p = (char_pool *) Xmalloc (sizeof (char_pool));
+  if (!p) FatalError (1, "Unable to allocate memory for pool");
+  
+  if (pool) pool->next = p;
+  p->next = NULL;
+  p->left = POOL_SIZE;
+  p->ptr = p->pool;
+  return p;
+}
+
+
+int init_text_2 (const TagInfo &/*tagInfo*/, char *fileName) {
+  char path[512];
+
+  // load the compression dictionary
+  if (LoadCompressionDictionary (make_name (fileName, TEXT_DICT_SUFFIX,
+					    path)) == COMPERROR)
+    return COMPERROR;
+
+  // create the compressed text file and attach it to the text buffer
+  if (!(text = create_file (fileName, TEXT_SUFFIX, "w+b",
+			    MAGIC_TEXT, MG_MESSAGE)))
+    return COMPERROR;
+  textOutBuf.attachFile (text);
+  textOutBuf.encodeStart ();
+  
+  // write out the compressed text header
+  bzero ((char *) &cth, sizeof (cth));
+  if (fwrite (&cth, sizeof (cth), 1, text) != 1)
+    return COMPERROR;
+
+  // the length of the compressed header is the size of the
+  // magic number plus the size of the compressed text header
+  headLen = sizeof (u_long) + sizeof (cth);
+
+  // start off with non-word
+  whichWordType = 0;
+
+  // clear the text pointers
+  textInfoMap.erase (textInfoMap.begin(), textInfoMap.end());
+  
+  // set up the hash table of novel words
+  int i;
+  if (cdh.novel_method != MG_NOVEL_HUFFMAN_CHARS)
+    for (i = 0; i <= 1; i++)
+      {
+	nht[i].HashSize = INITIAL_HASH_SIZE;
+	nht[i].HashTable = (novel_hash_rec *) Xmalloc (sizeof (novel_hash_rec) * nht[i].HashSize);
+	bzero ((char *) nht[i].HashTable,
+	       sizeof (novel_hash_rec) * nht[i].HashSize);
+	nht[i].HashUsed = 0;
+	nht[i].HashSize = INITIAL_HASH_SIZE;
+	nht[i].pool = nht[i].first_pool = new_pool (NULL);
+	nht[i].next_num = 1;
+	nht[i].binary_start = 1;
+	nht[i].code_to_nhr = NULL;
+	if (cdh.novel_method == MG_NOVEL_HYBRID)
+	  {
+	    int num;
+	    num = 1;
+	    blk_start[i][0] = 0;
+	    blk_end[i][0] = cdh.num_words[i] - 1;
+	    while (num < 33)
+	      {
+		blk_start[i][num] = blk_end[i][num - 1] + 1;
+		blk_end[i][num] = blk_start[i][num] +
+		  (blk_end[i][num - 1] - blk_start[i][num - 1]) * 2;
+		num++;
+	      }
+	  }
+      }
+
+  return (COMPALLOK);
+}
+
+
+static int compress_text (bitio_buffer &buffer, const u_char *s_in,
+			  int l_in, unsigned char &which,
+			  unsigned long &numWords) {
+  const u_char *end = s_in + l_in - 1;
+
+  for (; s_in <= end; which = !which) {
+    u_char Word[MAXWORDLEN + 1];
+    int res;
+    
+    if (which) numWords++;
+
+    /* First parse a word or non-word out of the string */
+    if (which) PARSE_WORD (Word, s_in, end);
+    else PARSE_NON_WORD (Word, s_in, end);
+
+    /* Search the hash table for Word */
+    if (ht[which]) {
+      register unsigned long hashval, step;
+      register int tsize = ht[which]->size;
+      register u_char **wptr;
+      HASH (hashval, step, Word, tsize);
+      for (;;) {
+	register u_char *s1;
+	register u_char *s2;
+	register int len;
+	wptr = ht[which]->table[hashval];
+	if (wptr == NULL) {
+	  res = COMPERROR;
+	  break;
+	}
+
+	/* Compare the words */
+	s1 = Word;
+	s2 = *wptr;
+	len = *s1 + 1;
+	for (; len; len--)
+	  if (*s1++ != *s2++) break;
+	
+	if (len) {
+	  hashval += step;
+	  if (hashval >= (unsigned long)tsize) hashval -= tsize;
+	  
+	} else {
+	  res = ht[which]->table[hashval] - ht[which]->words;
+	  break;
+	}
+      }
+    }
+    else
+      res = COMPERROR;
+    /* Check that the word was found in the dictionary */
+    if (res == COMPERROR)
+      {
+	if (cdh.dict_type == MG_COMPLETE_DICTIONARY)
+	  {
+	    Message ("Unknown word \"%.*s\"\n", *Word, Word + 1);
+	    return (COMPERROR);
+	  }
+	if (cdh.dict_type == MG_PARTIAL_DICTIONARY)
+	  {
+	    u_long i;
+	    if (ht[which])
+	      {
+		res = ht[which]->hd->num_codes - 1;
+		buffer.huff_encode (res, ht[which]->codes, ht[which]->hd->clens, NULL);
+	      }
+	    buffer.huff_encode (Word[0], lens_codes[which], lens_huff[which].clens, NULL);
+	    for (i = 0; i < Word[0]; i++)
+	      buffer.huff_encode (Word[i + 1], char_codes[which],
+				  char_huff[which].clens, NULL);
+	  }
+	if (cdh.dict_type == MG_SEED_DICTIONARY)
+	  {
+	    if (ht[which])
+	      {
+		res = ht[which]->hd->num_codes - 1;
+		buffer.huff_encode (res, ht[which]->codes, ht[which]->hd->clens, NULL);
+	      }
+	    switch (cdh.novel_method)
+	      {
+	      case MG_NOVEL_HUFFMAN_CHARS:
+		{
+		  u_long i;
+		  buffer.huff_encode (Word[0], lens_codes[which],
+				      lens_huff[which].clens, NULL);
+		  for (i = 0; i < Word[0]; i++)
+		    buffer.huff_encode (Word[i + 1], char_codes[which],
+					char_huff[which].clens, NULL);
+		}
+		break;
+	      case MG_NOVEL_DELTA:
+	      case MG_NOVEL_HYBRID:
+		{
+		  register unsigned long hashval, step;
+		  register novel_hash_table *h = &nht[which];
+		  register int hsize = h->HashSize;
+		  register novel_hash_rec *ent;
+		  HASH (hashval, step, Word, hsize);
+		  for (;;)
+		    {
+		      register u_char *s1, *s2;
+		      register int len;
+		      ent = h->HashTable + hashval;
+		      if (!ent->word)
+			{
+			  int len = *Word + 1;
+			  if ((unsigned)len > h->pool->left)
+			    h->pool = new_pool (h->pool);
+			  ent->word = h->pool->ptr;
+			  ent->ordinal_num = h->next_num++;
+			  memcpy (h->pool->ptr, Word, len);
+			  h->pool->ptr += len;
+			  h->pool->left -= len;
+			  h->HashUsed++;
+			  break;
+			}
+		      /* Compare the words */
+		      s1 = Word;
+		      s2 = ent->word;
+		      len = *s1 + 1;
+		      for (; len; len--)
+			if (*s1++ != *s2++)
+			  break;
+		      
+		      if (!len)
+			break;
+		      
+		      hashval = (hashval + step);
+		      if (hashval >= (unsigned)hsize) hashval -= hsize;
+		    }
+		  
+		  switch (cdh.novel_method) {
+		  case MG_NOVEL_DELTA:
+		    buffer.delta_encode (ent->ordinal_num, NULL);
+		    break;
+		  case MG_NOVEL_HYBRID:
+		    int k = 0;
+		    int j = ent->ordinal_num - 1;
+		    while (j > blk_end[which][k])
+		      k++;
+		    assert (j - blk_start[which][k] + 1 >= 1 &&
+			    j - blk_start[which][k] + 1 <=
+			    blk_end[which][k] - blk_start[which][k] + 1);
+		    
+		    buffer.gamma_encode (k + 1, NULL);
+		    buffer.binary_encode (j - blk_start[which][k] + 1,
+					  blk_end[which][k] - blk_start[which][k] + 1,
+					  NULL);
+		    break;
+		  }
+		  
+		  if (h->HashUsed >= h->HashSize >> 1)
+		    {
+		      novel_hash_rec *ht;
+		      unsigned long size;
+		      unsigned long i;
+		      size = prime (h->HashSize * 2);
+		      if (!(ht = (novel_hash_rec *) Xmalloc (sizeof (novel_hash_rec) * size)))
+			{
+			  Message ("Unable to allocate memory for table");
+			  return (COMPERROR);
+			}
+		      bzero ((char *) ht, sizeof (novel_hash_rec) * size);
+		      
+		      for (i = 0; i < h->HashSize; i++)
+			if (h->HashTable[i].word)
+			  {
+			    register u_char *wptr;
+			    register unsigned long hashval, step;
+			    
+			    wptr = h->HashTable[i].word;
+			    HASH (hashval, step, wptr, size);
+			    wptr = (ht + hashval)->word;
+			    while (wptr)
+			      {
+				hashval += step;
+				if (hashval >= size)
+				  hashval -= size;
+				wptr = (ht + hashval)->word;
+			      }
+			    ht[hashval] = h->HashTable[i];
+			  }
+		      Xfree (h->HashTable);
+		      h->HashTable = ht;
+		      h->HashSize = size;
+		    }
+		}
+		break;
+	      }
+	  }
+      }
+    else
+      {
+	buffer.huff_encode (res, ht[which]->codes, ht[which]->hd->clens, NULL);
+      }
+  }
+  
+  return COMPALLOK;
+}
+
+
+int process_text_2 (const TagInfo &tagInfo, const TextElArray &doc) {
+  // we need to remember where the last text element ends
+  unsigned char endBit = textOutBuf.GetBtg (); // 8=high bit, 1=low bit
+  unsigned long endPos = textOutBuf.GetBytesWritten () + headLen;
+
+  // process each text element
+  TextElArray::const_iterator here = doc.begin();
+  TextElArray::const_iterator end = doc.end();
+  while (here != end) {
+    // remember current place in compressed text
+    unsigned char startWhich = whichWordType;
+    unsigned char startBit = textOutBuf.GetBtg (); // 8=high bit, 1=low bit
+    unsigned long startPos = textOutBuf.GetBytesWritten () + headLen;
+
+    // compress the text
+    if (compress_text (textOutBuf,
+		       (*here).text.begin(),
+		       (*here).text.size(),
+		       whichWordType,
+		       cth.num_of_words) != COMPALLOK)
+      return COMPERROR;
+
+    endBit = textOutBuf.GetBtg (); // 8=high bit, 1=low bit
+    endPos = textOutBuf.GetBytesWritten () + headLen;
+    
+    // update the pointers if this is a level tag
+    if (tagInfo.levelTags.find ((*here).tagName) != tagInfo.levelTags.end()) {
+      if ((*here).elType == OpenTagE) {
+	// set the start of the document
+	textInfoMap[(*here).tagName].SetStart (startPos, startBit, startWhich);
+	
+      } else if ((*here).elType == CloseTagE) {
+	// set the end of the document
+	textInfoMap[(*here).tagName].SetEnd (endPos, endBit);
+	
+      } else {
+	// TextE, nothing needs to be done
+      }
+    }
+    
+    cth.num_of_bytes += (*here).text.size();
+    here++;
+  }
+
+  // close off any unclosed tags
+  CompressTextInfoMap::iterator tiHere = textInfoMap.begin ();
+  CompressTextInfoMap::iterator tiEnd = textInfoMap.end ();
+  while (tiHere != tiEnd) {
+    if ((*tiHere).second.inDoc) (*tiHere).second.SetEnd (endPos, endBit);
+    tiHere++;
+  }
+  
+  // we've processed one more document
+  cth.num_of_docs++;
+
+  return COMPALLOK;
+}
+
+
+static int write_aux_dict (char *fileName) {
+  int i;
+  FILE *aux;
+  if (!(aux = create_file (fileName, TEXT_DICT_AUX_SUFFIX, "wb",
+			   MAGIC_AUX_DICT, MG_MESSAGE)))
+    return COMPERROR;
+  
+  for (i = 0; i <= 1; i++)
+    {
+      aux_frags_header afh;
+      char_pool *cp;
+      
+      afh.num_frags = nht[i].HashUsed;
+      afh.mem_for_frags = 0;
+      for (cp = nht[i].first_pool; cp; cp = cp->next)
+	afh.mem_for_frags += POOL_SIZE - cp->left;
+      
+      /* [RPAP - Jan 97: Endian Ordering] */
+      HTONUL(afh.num_frags);
+      HTONUL(afh.mem_for_frags);
+      
+      fwrite (&afh, sizeof (afh), 1, aux);
+      
+      for (cp = nht[i].first_pool; cp; cp = cp->next)
+	fwrite (cp->pool, POOL_SIZE - cp->left, sizeof (u_char), aux);
+    }
+  fclose (aux);
+  return COMPALLOK;
+}
+
+
+static void estimate_compressed_aux_dict (void) {
+  int i;
+  u_long aux_compressed = 0, total_uncomp = 0;
+  for (i = 0; i <= 1; i++)
+    {
+      int j;
+      long chars[256], fchars[256];
+      long lens[16], flens[16];
+      char_pool *cp;
+      bzero ((char *) chars, sizeof (chars));
+      bzero ((char *) lens, sizeof (lens));
+      for (cp = nht[i].first_pool; cp; cp = cp->next)
+	{
+	  u_char *buf = cp->pool;
+	  while (buf != cp->ptr)
+	    {
+	      int len = *buf++;
+	      lens[len]++;
+	      total_uncomp += len + 4;
+	      for (; len; len--)
+		chars[*buf++]++;
+	    }
+	}
+      for (j = 0; j < 256; j++)
+	if (!chars[j] && PESINAWORD (j) == i)
+	  fchars[j] = 1;
+	else
+	  fchars[j] = chars[j];
+      for (j = 0; j < 16; j++)
+	if (!lens[j])
+	  flens[j] = 1;
+	else
+	  flens[j] = lens[j];
+      
+      aux_compressed += (long unsigned int) ((Calculate_Huffman_Size (16, flens, lens) +
+					      Calculate_Huffman_Size (256, fchars, chars)) / 8);
+      
+    }
+}
+
+
+
+static bool WriteTextIdx (char *fileName, FTextLevel &textLevels) {
+  // create the text index
+  FILE *textIdxFile;
+  if (!(textIdxFile = create_file (fileName, TEXT_IDX_SUFFIX, "w+b",
+				MAGIC_TEXI, MG_MESSAGE)))
+    return false;
+
+  // output each level
+  CompressTextInfoMap::iterator tiHere = textInfoMap.begin ();
+  CompressTextInfoMap::iterator tiEnd = textInfoMap.end ();
+  while (tiHere != tiEnd) {
+    // remember the level information
+    TextLevelInfo levelInfo;
+    levelInfo.levelTag = (*tiHere).first;
+    levelInfo.textIdxPtr = ftell (textIdxFile);
+    levelInfo.numEntries = (*tiHere).second.docPtrs.size();
+    textLevels.levelInfo[levelInfo.levelTag] = levelInfo;
+
+    // write out the index array for this level
+    if (!WriteTextIdxArray (textIdxFile, (*tiHere).second.docPtrs))
+      return false;
+    
+    tiHere++;
+  }
+
+  // close the file
+  fclose (textIdxFile);
+
+  return true;
+}
+
+
+bool WriteTextLevels (char *fileName, const FTextLevel &textLevels) {
+  //  cout << textLevels;
+  
+  // create the text index
+  FILE *textLevelFile;
+  if (!(textLevelFile = create_file (fileName, TEXT_LEVEL_SUFFIX, "w+b",
+				     MAGIC_TEXT_LEVELS, MG_MESSAGE)))
+    return false;
+
+  // output each level
+  if (!textLevels.Write (textLevelFile)) return false;
+  
+  // close the file
+  fclose (textLevelFile);
+
+  return true;
+}
+
+
+int done_text_2 (const TagInfo &/*tagInfo*/, char *fileName) {
+  // write out any remaining bits
+  textOutBuf.encodeDone();
+
+  // write out the compressed text header (with updated statistics)
+  if (fseek (text, sizeof (u_long), SEEK_SET) == -1 || !cth.Write (text))
+    return COMPERROR;
+  fclose (text);
+
+  // write out the text index file and get the level information
+  FTextLevel textLevels;
+  if (!WriteTextIdx (fileName, textLevels)) return COMPERROR;
+  
+  // write out the text level information
+  if (!WriteTextLevels (fileName, textLevels)) return COMPERROR;
+
+  // write out the auxiliary dictionary
+  if (cdh.dict_type != MG_COMPLETE_DICTIONARY &&
+      (cdh.novel_method == MG_NOVEL_DELTA ||
+       cdh.novel_method == MG_NOVEL_HYBRID)) {
+    if (write_aux_dict (fileName) == COMPERROR) return COMPERROR;
+    estimate_compressed_aux_dict ();
+    
+  } else {
+    unlink (make_name (fileName, TEXT_DICT_AUX_SUFFIX, NULL));
+  }
+
+  return (COMPALLOK);
+}
Index: /trunk/gsdl/src/mgpp/text/words.cpp
===================================================================
--- /trunk/gsdl/src/mgpp/text/words.cpp	(revision 856)
+++ /trunk/gsdl/src/mgpp/text/words.cpp	(revision 856)
@@ -0,0 +1,97 @@
+/**************************************************************************
+ *
+ * words.cpp -- Functions for parsing out words from the source text
+ * Copyright (C) 1999  Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "words.h"
+
+
+/* Takes the place of the old INAWORD macro. It determines
+   whether a given place in a UTF-8 encoded Unicode string
+   is part of a word. */
+int inaword (const u_char *here, const u_char *end) {
+  unsigned short c;
+  if (parse_utf8_char(here, end, &c) > 0) return is_unicode_letdig (c);
+  return 0;
+}
+
+const unsigned char *ParseIndexWord (const unsigned char *textHere,
+				     const unsigned char *textEnd,
+				     UCArray &word) {
+  word.erase (word.begin(), word.end());
+
+  register int charlength = 0;
+  register int length = 0;
+  register int numeric = 0;
+  unsigned short c;
+  
+  charlength = parse_utf8_char (textHere, textEnd, &c);
+  
+  while (length+charlength <= MAXSTEMLEN && charlength > 0 &&
+	 (is_unicode_letter(c) || (is_unicode_digit(c) &&
+				   ++numeric <= MAXNUMERIC))) {
+    while (charlength-- > 0) {
+      word.push_back (*textHere++); length++;
+    }
+    charlength = parse_utf8_char (textHere, textEnd, &c);
+  }
+  
+  return textHere;
+}
+
+const unsigned char *ParseIndexMGWord (const unsigned char *textHere,
+				       const unsigned char *textEnd,
+				       unsigned char *mgWord) {
+  register int charlength = 0;
+  register int length = 0;
+  register int numeric = 0;
+  unsigned short c;
+  
+  charlength = parse_utf8_char (textHere, textEnd, &c);
+  
+  while (length+charlength <= MAXSTEMLEN && charlength > 0 &&
+	 (is_unicode_letter(c) || (is_unicode_digit(c) &&
+				   ++numeric <= MAXNUMERIC))) {
+    while (charlength-- > 0) {
+      mgWord[++length] = *textHere++;
+    }
+    charlength = parse_utf8_char (textHere, textEnd, &c);
+  }
+
+  mgWord[0] = length;
+  
+  return textHere;
+}
+
+const unsigned char *ParseNonindexWord (const unsigned char *textHere,
+					const unsigned char *textEnd) {
+  register int charlength = 0;
+  unsigned short c;
+  
+  charlength = parse_utf8_char(textHere, textEnd, &c);
+  
+  while (charlength > 0 && !is_unicode_letdig(c)) {
+    textHere += charlength;
+    charlength = parse_utf8_char (textHere, textEnd, &c);
+  }
+  
+  return textHere;
+}
Index: /trunk/gsdl/src/mgpp/text/words.h
===================================================================
--- /trunk/gsdl/src/mgpp/text/words.h	(revision 856)
+++ /trunk/gsdl/src/mgpp/text/words.h	(revision 856)
@@ -0,0 +1,157 @@
+/**************************************************************************
+ *
+ * words.h -- Macros for parsing out words from the source text
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "unitool.h"
+#include "UCArray.h"
+
+
+/*
+ * This has been cleaned up by Tim Shimmin. 
+ */
+
+/*
+ * ---NOTE---
+ *
+ * "WORD" refers to a word in the compressed text.
+ * "STEM" or "STEM_WORD" refers to a word for indexing on
+ *
+ */
+
+#define MAXWORDLEN	15
+	/* Maximum length in bytes of any word or non-word. Note that
+	   variations to MAXWORDLEN may have dramatic effects on the rest
+	   of the program, as the length and the prefix match are packed
+	   together into a four bit nibble, and there is not check that
+	   this is possible, i.e., leave MAXWORDLEN alone... */
+
+#define MAXSTEMLEN	255
+	/* Maximum length in bytes of any stem. Note that
+	   variations to MAXSTEMLEN may have dramatic effects on the rest
+	   of the program, , i.e., leave MAXSTEMLEN alone... */
+
+#define MAXNUMERIC	4
+	/* Maximum number of numeric characters permitted in a word.
+	   This avoids long sequences of numbers creating just one
+	   word occurrence for each number. At most 10,000 all numeric
+	   words will be permitted. */
+
+/* [RPAP - Jan 97: Stem Index Change] */
+#define MAXPARAMLEN     20
+        /* Maximum number of bytes to read for a parameter value for a
+	   term in a query. */
+#define WEIGHTPARAM     '/'
+#define STEMPARAM       '#'
+
+/* [RJM 07/97: Ranked Required Terms] */
+#define MUSTMATCHPARAM     '+'
+
+
+#define PESINAWORD(c)      (isalnum(c) || ((c) >= 0x80 && (c) <= 0xff))
+        /* The definition of what characters are permitted in a word.
+	   This macro is pessimistic, you cannot tell from a particular
+	   byte above 0x80 whether it is a character or not. This function
+	   is needed by various functions relating to huffman coding
+	   where frequency counts need to be primed, it should not be
+	   used in parsing the UTF-8 encoded input. */
+
+#ifdef __cplusplus
+extern "C"
+#endif
+int inaword (const u_char *here, const u_char *end);
+        /* Takes the place of the old INAWORD macro. It determines
+	   whether a given place in a UTF-8 encoded Unicode string
+	   is part of a word. */
+
+
+const unsigned char *ParseIndexWord (const unsigned char *textHere,
+				     const unsigned char *textEnd,
+				     UCArray &word);
+const unsigned char *ParseIndexMGWord (const unsigned char *textHere,
+				       const unsigned char *textEnd,
+				       unsigned char *mgWord);
+const unsigned char *ParseNonindexWord (const unsigned char *textHere,
+					const unsigned char *textEnd);
+
+
+
+/* =========================================================================
+ * Macro: PARSE_WORD
+ * Description: 
+ *      Extract a word out for compressing text
+ * Input: 
+ *      s_in = string start in buffer
+ *      end = string end in buffer
+ * Output: 
+ *      Word = extracted word with length in 1st byte
+ *      s_in = ptr to next character in buffer yet to be processed
+ * ========================================================================= */
+#define PARSE_WORD(Word, s_in, end)                                \
+  do {                                                             \
+    register int charlength = 0;                                   \
+    register u_char *wptr = (Word)+1;                              \
+    register int length = 0;                                       \
+    register int numeric = 0;                                      \
+    unsigned short c;                                              \
+                                                                   \
+    charlength = parse_utf8_char((s_in),(end),&c);                 \
+                                                                   \
+    while (length+charlength <= MAXWORDLEN && charlength > 0 &&    \
+	   (is_unicode_letter(c) || (is_unicode_digit(c) &&        \
+				     ++numeric <= MAXNUMERIC))) {  \
+      while (charlength-- > 0) {                                   \
+        *wptr++ = *(s_in)++; length++;                             \
+      }                                                            \
+      charlength = parse_utf8_char((s_in),(end),&c);               \
+    }                                                              \
+    *(Word) = length;                                              \
+  }while(0)
+
+
+
+/* =========================================================================
+ * Macro: PARSE_NON_WORD
+ * Description: 
+ *      Extract a non-word out for storing compressed text
+ * Input: as above
+ * Output: as above
+ * ========================================================================= */
+#define PARSE_NON_WORD(Word, s_in, end)                            \
+  do {                                                             \
+    register int charlength = 0;                                   \
+    register u_char *wptr = (Word)+1;                              \
+    register int length = 0;                                       \
+    unsigned short c;                                              \
+                                                                   \
+    charlength = parse_utf8_char((s_in),(end),&c);                 \
+                                                                   \
+    while (length+charlength <= MAXWORDLEN && charlength > 0 &&    \
+	   !is_unicode_letdig(c)) {                                \
+      while (charlength-- > 0) {                                   \
+        *wptr++ = *(s_in)++; length++;                             \
+      }                                                            \
+      charlength = parse_utf8_char((s_in),(end),&c);               \
+    }                                                              \
+    *(Word) = length;                                              \
+  }while(0)
