Index: /tags/gsdl-21/gsdl/Makefile.in
===================================================================
--- /tags/gsdl-21/gsdl/Makefile.in	(revision 699)
+++ /tags/gsdl-21/gsdl/Makefile.in	(revision 699)
@@ -0,0 +1,91 @@
+###########################################################################
+#
+# Makefile -- gsdl
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999  The New Zealand Digital Library Project
+#
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+PACKAGE = @PACKAGE@
+VERSION = @VERSION@
+
+@SET_MAKE@
+CC = @CC@
+CPP = @CXX@
+CFLAGS = @CFLAGS@
+LIBS   = @LIBS@
+MDEFINES =
+TOPSRC = `pwd`
+
+COMPILE = $(CC) -c
+
+SOURCES = 
+
+OBJECTS = 
+
+ALLFILES = $(OBJECTS)
+
+INSTALLDIRS = src/txt2db src/db2txt src/hashfile
+
+OTHERDIRS = packages/mg lib src/colservr src/recpt collect/cstr/src/recpt collect/niupepa/src/recpt
+
+all:
+	sed s^XXXX^$(TOPSRC)^ lib/gsdlhome.h.example > lib/gsdlhome.h
+
+	for odir in $(OTHERDIRS); do \
+	  echo making $@ in $$odir; \
+	  (cd $$odir && $(MAKE) $(MDEFINES) $@) || exit 1; \
+	done
+
+	for idir in $(INSTALLDIRS); do \
+	  echo making $@ in $$idir; \
+	  (cd $$idir && $(MAKE) $(MDEFINES) $@) || exit 1; \
+	  echo installing $$idir; \
+	  (cd $$idir && $(MAKE) install) || exit 1; \
+	done
+
+clean:
+	rm -f Makefile config.cache config.h config.log stamp-h
+	rm lib/gsdlhome.h
+	for subdir in $(INSTALLDIRS) $(OTHERDIRS); do \
+	  echo cleaning $@ in $$subdir; \
+	  (cd $$subdir && $(MAKE) clean) || exit 1; \
+	done
+
+# For an explanation of the following Makefile rules, see node
+# `Automatic Remaking' in GNU Autoconf documentation.
+Makefile: Makefile.in config.status
+	CONFIG_FILES=$@ CONFIG_HEADERS= ./config.status
+config.status: configure
+	./config.status --recheck
+configure: configure.in aclocal.m4
+	cd $(srcdir) && autoconf
+
+config.h: stamp-h
+stamp-h: config.h.in config.status
+	CONFIG_FILES= CONFIG_HEADERS=config.h ./config.status
+config.h.in: stamp-h.in
+stamp-h.in: configure.in aclocal.m4 acconfig.h
+	cd $(srcdir) && autoheader
+	$(TOUCH) $(srcdir)/stamp-h.in
+
+
+
Index: /tags/gsdl-21/gsdl/acconfig.h
===================================================================
--- /tags/gsdl-21/gsdl/acconfig.h	(revision 699)
+++ /tags/gsdl-21/gsdl/acconfig.h	(revision 699)
@@ -0,0 +1,43 @@
+/* Special definitions for mg, processed by autoheader.
+   Created by [TS:Aug/95] 
+   by grepping for AC_DEFINE on configure.in and using the appropriate defs 
+   of acconfig.h used by GNU tar.
+*/
+
+/* Define to the name of the distribution.  */
+#undef PACKAGE
+
+/* Define to the version of the distribution.  */
+#undef VERSION
+
+/* Define if `union wait' is the type of the first arg to wait functions.  */
+#undef HAVE_UNION_WAIT
+
+/* Define to 1 if you have the valloc function.  */
+#undef HAVE_VALLOC
+
+/* Define to 1 if ANSI function prototypes are usable.  */
+#undef PROTOTYPES
+ 
+/* Define to 1 for better use of the debugging malloc library.  See
+   site ftp.antaire.com in antaire/src, file dmalloc/dmalloc.tar.gz.  */
+#undef WITH_DMALLOC
+ 
+/* Define to 1 if GNU regex should be used instead of GNU rx.  */
+#undef WITH_REGEX
+
+/* Some braindead header files do not have the requ'd function declarations */
+#undef HAVE_FGETC_DECL
+#undef HAVE_FREAD_DECL
+
+/* See if have this field in the prstatus_t structure */
+/* It stores the size of the process heap */
+#undef HAVE_PR_BRKSIZE
+
+/* Define FASTCGI if needsbe */
+#undef USE_FASTCGI
+
+
+
+
+
Index: /tags/gsdl-21/gsdl/bin/script/buildcol.pl
===================================================================
--- /tags/gsdl-21/gsdl/bin/script/buildcol.pl	(revision 699)
+++ /tags/gsdl-21/gsdl/bin/script/buildcol.pl	(revision 699)
@@ -0,0 +1,162 @@
+#!/usr/local/bin/perl5 -w
+
+###########################################################################
+#
+# buildcol.pl -- This program will build a particular collection
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/plugins");
+}
+
+use colcfg;
+use parsargv;
+use util;
+
+&main();
+
+sub print_usage {
+    print STDERR "\n  usage: $0 [options] collection-name\n\n";
+    print STDERR "  options:\n";
+    print STDERR "   -verbosity number     0=none, 3=lots\n";
+    print STDERR "   -archivedir directory Where the archives live\n";
+    print STDERR "   -cachedir directory   Where to cache the archives\n";
+    print STDERR "   -builddir directory   Where to put the built indexes\n";
+    print STDERR "   -maxdocs number       Maximum number of documents to build\n";
+    print STDERR "   -allclassifications   Don't remove empty classifications\n\n";
+}
+
+
+sub main
+{
+    if (!parsargv::parse(\@ARGV, 
+			 'verbosity/\d+/2', \$verbosity,
+			 'archivedir/.*/', \$archivedir,
+			 'cachedir/.*/', \$cachedir,
+			 'builddir/.*/', \$builddir,
+			 'maxdocs/\d+/', \$maxdocs,
+			 'allclassifications', \$allclassifications)) {
+	&print_usage();
+	die "\n";
+    }
+
+    # get and check the collection
+    if (($collection = &util::use_collection(@ARGV)) eq "") {
+	&print_usage();
+	die "\n";
+    }
+
+    # only here to prevent a perl warning
+    $maxdocs = "" if $maxdocs !~ /\d/; 
+    $allclassifications = 0 unless $allclassifications == 1;
+
+    # read the configuration file
+    $textindex = "section:text";
+    $configfilename = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, "etc/collect.cfg");
+    if (-e $configfilename) {
+	$collectcfg = &colcfg::read_collect_cfg ($configfilename);
+	if (defined $collectcfg->{'archivedir'} && $archivedir eq "") {
+	    $archivedir = $collectcfg->{'archivedir'};
+	}
+	if (defined $collectcfg->{'cachedir'} && $cachedir eq "") {
+	    $cachedir = $collectcfg->{'cachedir'};
+	}
+	if (defined $collectcfg->{'builddir'} && $builddir eq "") {
+	    $builddir = $collectcfg->{'builddir'};
+	}
+	if (defined $collectcfg->{'textindex'}) {
+	    $textindex = $collectcfg->{'textindex'};
+	}
+    } else {
+	die "Couldn't find the configuration file $configfilename\n";
+    }
+    
+    # fill in the default archives and building directories if none
+    # were supplied, turn all \ into / and remove trailing /
+    $archivedir = "$ENV{'GSDLCOLLECTDIR'}/archives" if $archivedir eq "";
+    $archivedir =~ s/[\\\/]+/\//g;
+    $archivedir =~ s/\/$//;
+    $builddir = "$ENV{'GSDLCOLLECTDIR'}/building" if $builddir eq "";
+    $builddir =~ s/[\\\/]+/\//g;
+    $builddir =~ s/\/$//;
+
+    # update the archive cache if needed
+    if ($cachedir) {
+	print STDERR "Updating archive cache\n" if ($verbosity >= 1);
+
+	$cachedir =~ s/[\\\/]+$//;
+	$cachedir .= "/collect/$collection" unless 
+	    $cachedir =~ /collect\/$collection/;
+
+	$realarchivedir = "$cachedir/archives";
+	$realbuilddir = "$cachedir/building";
+	&util::mk_all_dir ($realarchivedir);
+	&util::mk_all_dir ($realbuilddir);
+	&util::cachedir ($archivedir, $realarchivedir, $verbosity);
+
+    } else {
+	$realarchivedir = $archivedir;
+	$realbuilddir = $builddir;
+    }
+
+    # build it in realbuilddir
+    &util::mk_all_dir ($realbuilddir);
+
+
+    # if a builder class has been created for this collection, use it
+    # otherwise, use the mg builder
+    if (-e "$ENV{'GSDLCOLLECTDIR'}/perllib/${collection}builder.pm") {
+	$builderdir = "$ENV{'GSDLCOLLECTDIR'}/perllib";
+	$buildertype = "${collection}builder";
+    } else {
+	$builderdir = "$ENV{'GSDLHOME'}/perllib";
+	$buildertype = "mgbuilder";
+    }
+	
+    require "$builderdir/$buildertype.pm";
+
+    eval("\$builder = new $buildertype(\$collection, " .
+	 "\$realarchivedir, \$realbuilddir, \$verbosity, " .
+	 "\$maxdocs, \$allclassifications)");
+    die "$@" if $@;
+
+    $builder->init();
+    $builder->compress_text($textindex);
+    $builder->build_indexes();
+    $builder->make_infodatabase();
+    $builder->collect_specific();
+    $builder->make_auxiliary_files();
+    $builder->deinit();
+
+
+    if ($realbuilddir ne $builddir) {
+	print STDERR "Copying back the cached build\n" if ($verbosity >= 1);
+	&util::rm_r ($builddir);
+	&util::cp_r ($realbuilddir, $builddir);
+    }
+}
+
+
Index: /tags/gsdl-21/gsdl/bin/script/clean_archives.pl
===================================================================
--- /tags/gsdl-21/gsdl/bin/script/clean_archives.pl	(revision 699)
+++ /tags/gsdl-21/gsdl/bin/script/clean_archives.pl	(revision 699)
@@ -0,0 +1,109 @@
+#!/usr/local/bin/perl5
+
+###########################################################################
+#
+# clean_archives.pl
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+
+# clean_archives.pl removes any doc.gml, doc.gml.gz, and empty
+# directories from the directory passed on the command line.
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/plugins");
+}
+
+use parsargv;
+use util;
+
+sub print_usage {
+    print STDERR "\n  usage: $0 [options] archive-dir\n\n";
+    print STDERR "  oprions:\n";
+    print STDERR "   -verbose\n\n";
+}
+
+
+&main ();
+
+sub main {
+    
+    if (!parsargv::parse(\@ARGV, 
+			 'verbose', \$verbose)) {
+	&print_usage();
+	die "\n";
+    }
+
+    if (scalar (@ARGV) != 1) {
+	&print_usage();
+	die "\n";
+    }
+
+    my $dir = $ARGV[0];
+    
+    if (!-d $dir) {
+	&print_usage();
+	die "\n";
+    }
+	
+    &recurse_dir ($dir);
+}
+
+sub recurse_dir {
+    my ($dir) = @_;
+
+    opendir (DIR, $dir) || die;
+    my @files = readdir DIR;
+    closedir DIR;
+
+    foreach $file (@files) {
+	next if $file =~ /^\.+$/;
+	my $filename = &util::filename_cat($dir, $file);
+	
+	if (-d $filename) {
+	    &recurse_dir ($filename);
+	} elsif ($file =~ /^doc\.gml(\.gz)?$/) {
+
+	    unlink ($filename);
+	    print STDERR "removing $filename\n" if $verbose;
+	}
+    }
+
+    # remove the directory if it's now empty
+    opendir (DIR, $dir) || die;
+    @files = readdir DIR;
+    closedir DIR;
+
+    if (scalar(@files) == 0) {
+	rmdir ($dir);
+	print STDERR "removing empty directory $dir\n" if $verbosity;
+    } elsif (scalar(@files) == 2) {
+	my $empty = 1;
+	map { if ($_ !~ /^\.+$/) {$empty = 0;} } @files;
+	if ($empty) {
+	    rmdir ($dir);
+	    print STDERR "removing empty directory $dir\n" if $verbosity;
+	}
+    }
+}
Index: /tags/gsdl-21/gsdl/bin/script/gettext.pl
===================================================================
--- /tags/gsdl-21/gsdl/bin/script/gettext.pl	(revision 699)
+++ /tags/gsdl-21/gsdl/bin/script/gettext.pl	(revision 699)
@@ -0,0 +1,131 @@
+#!/usr/local/bin/perl5 -w
+
+###########################################################################
+#
+# gettext.pl --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+
+# This program will output the text for a particular index
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/plugins");
+}
+
+use parsargv;
+use plugin;
+use colcfg;
+use util;
+
+sub print_usage {
+    print STDERR "\n  usage: $0 collection-name\n";
+    print STDERR "  options:\n";
+    print STDERR "   -archivedir directory Where the archives live\n";
+    print STDERR "   -mode infodb for gdbm database, defaults to text\n";
+    print STDERR "   -index index   The index to output\n\n";
+}
+
+
+&main ();
+
+sub main {
+
+    if (!parsargv::parse(\@ARGV, 
+			 'archivedir/.*/', \$archivedir,
+			 'index/.*/section:text', \$index,
+			 'mode/.*/text', \$mode)) {
+	&print_usage();
+	die "\n";
+    }
+
+    # get and check the collection name
+   if (($collection = &util::use_collection(@ARGV)) eq "") {
+	&print_usage();
+	die "\n";
+    }
+    
+    # get the list of plugins for this collection
+    @plugins = ();
+    $configfilename = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, "etc/collect.cfg");
+    if (-e $configfilename) {
+	$collectcfg = &colcfg::read_collect_cfg ($configfilename);
+	if (defined $collectcfg->{'plugins'}) {
+	    @plugins = @{$collectcfg->{'plugins'}};
+	}
+	if (defined $collectcfg->{'archivedir'} && $archivedir eq "") {
+	    $archivedir = $collectcfg->{'archivedir'};
+	}
+    } else {
+	die "Couldn't find the configuration file $configfilename\n";
+    }
+
+    # fill in the default archives directories if none
+    # was supplied, turn all \ into / and remove trailing /
+    $archivedir = "$ENV{'GSDLCOLLECTDIR'}/archives" if $archivedir eq "";
+    $archivedir =~ s/[\\\/]+/\//g;
+    $archivedir =~ s/\/$//;
+
+    # load all the plugins
+    $pluginfo = &plugin::load_plugins ($collection, \@plugins);
+    if (scalar(@$pluginfo) == 0) {
+	print STDERR "No plugins were loaded.\n";
+	die "\n";
+    }
+
+    # load up the document processor for building
+    # if a buildproc class has been created for this collection, use it
+    # otherwise, use the mg buildproc
+    # if a builder class has been created for this collection, use it
+    # otherwise, use the mg builder
+    # load up the document processor for building
+    # if a buildproc class has been created for this collection, use it
+    # otherwise, use the mg buildproc
+    my ($buildprocdir, $buildproctype);
+    if (-e "$ENV{'GSDLCOLLECTDIR'}/perllib/${collection}buildproc.pm") {
+	$buildprocdir = "$ENV{'GSDLCOLLECTDIR'}/perllib";
+	$buildproctype = "${collection}buildproc";
+    } else {
+	$buildprocdir = "$ENV{'GSDLHOME'}/perllib";
+	$buildproctype = "mgbuildproc";
+    }
+    require "$buildprocdir/$buildproctype.pm";
+
+    eval("\$buildproc = new $buildproctype(\$collection, " .
+	 "\$archivedir, \"\$ENV{'GSDLCOLLECTDIR'}/building\", 1)");
+    die "$@" if $@;
+
+    # set up the document processor
+    $buildproc->set_output_handle ('STDOUT');
+    $buildproc->set_mode ($mode);
+    $buildproc->set_index ($index);
+
+    $buildproc->reset();
+
+    # process the import directory
+    &plugin::read ($pluginfo, $archivedir, 
+		   "", {}, $buildproc);
+
+}
Index: /tags/gsdl-21/gsdl/bin/script/gsdlinfo.pl
===================================================================
--- /tags/gsdl-21/gsdl/bin/script/gsdlinfo.pl	(revision 699)
+++ /tags/gsdl-21/gsdl/bin/script/gsdlinfo.pl	(revision 699)
@@ -0,0 +1,135 @@
+#!/usr/local/bin/perl5 -w
+
+###########################################################################
+#
+# gsdlinfo.pl --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# This program will look to see what collections are installed
+# under this file system
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+}
+
+use parsargv;
+use colcfg;
+
+
+# the collection information is stored as an array of hashes
+# (one hash for each collection). The hash contains all the 
+# information in the collect and build configuration files
+
+
+# read in all the collection configuration information
+
+opendir (DIR, "$ENV{'GSDLHOME'}/collect") ||
+    die "ERROR - couldn't read the directory $ENV{'GSDLHOME'}/collect\n";
+@dir = readdir (DIR);
+closedir (DIR);
+
+@collections = ();
+foreach $dir (@dir) {
+    if ($dir =~ /^(\..*|modelcol|CVS)$/) {
+	# do nothing
+
+    } else {
+	# read in the collection configuration file
+	my $collectcfg = {};
+	if (-e "$ENV{'GSDLHOME'}/collect/$dir/etc/collect.cfg") {
+	    $collectcfg = &colcfg::read_collect_cfg ("$ENV{'GSDLHOME'}/collect/$dir/etc/collect.cfg");
+	}
+	$collectcfg->{'collection'} = $dir;
+
+	# read in the build configuration file
+	my $buildcfg = {};
+	if (-e "$ENV{'GSDLHOME'}/collect/$dir/index/build.cfg") {
+	    $buildcfg = &colcfg::read_build_cfg ("$ENV{'GSDLHOME'}/collect/$dir/index/build.cfg");
+	}
+
+	# add the merged configuration files to the current list of collections
+	push (@collections, {%$collectcfg, %$buildcfg});
+    }
+}
+
+
+# do any sorting requested
+
+
+# print out the collection information
+
+# {'creator'}->string
+# {'maintainer'}->array of strings
+# {'public'}->string
+# {'beta'}->string
+# {'key'}->string
+# {'indexes'}->array of strings
+# {'defaultindex'}->string
+# {'builddate'}->string
+# {'metadata'}->array of strings
+# {'languages'}->array of strings
+# {'numdocs'}->string
+# {'numwords'}->string
+# {'numbytes'}->string
+
+print "\n";
+foreach $info (@collections) {
+    if (defined $info->{'collection'}) {
+	$collection = $info->{'collection'};
+	if (defined $info->{'public'}) {
+	    $public = $info->{'public'};
+	} else {
+	    $public = "?";
+	}
+	if (defined $info->{'beta'}) {
+	    $beta = $info->{'beta'};
+	} else {
+	    $beta = "?";
+	}
+	if (defined $info->{'numdocs'}) {
+	    $numdocs = $info->{'numdocs'};
+	} else {
+	    $numdocs = "?";
+	}
+	if (defined $info->{'numbytes'}) {
+	    $numMbytes = int(($info->{'numbytes'}/1024.0/1024.0)*1000)/1000.0;
+	} else {
+	    $numMbytes = "?";
+	}
+	write;
+    }
+}
+print "\n";
+
+exit;
+
+format STDOUT_TOP = 
+  name   public  beta   documents  size (Mbytes)
+-------- ------ ------ ----------- -------------
+.
+
+format STDOUT = 
+@>>>>>>> @>>>>> @>>>>> @>>>>>>>>>> @>>>>>>>>>>>>
+$collection, $public, $beta, $numdocs, $numMbytes
+.
Index: /tags/gsdl-21/gsdl/bin/script/import.pl
===================================================================
--- /tags/gsdl-21/gsdl/bin/script/import.pl	(revision 699)
+++ /tags/gsdl-21/gsdl/bin/script/import.pl	(revision 699)
@@ -0,0 +1,149 @@
+#!/usr/local/bin/perl5 -w
+
+###########################################################################
+#
+# import.pl --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+
+# This program will import a number of files into a particular collection
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/plugins");
+}
+
+use arcinfo;
+use colcfg;
+use plugin;
+use docsave;
+use util;
+use parsargv;
+
+sub print_usage {
+    print STDERR "\n  usage: $0 [options] collection-name\n\n";
+    print STDERR "  options:\n";
+    print STDERR "   -verbosity number      0=none, 3=lots\n";
+    print STDERR "   -importdir directory   Where the original material lives\n";
+    print STDERR "   -archivedir directory  Where the converted material ends up\n";
+    print STDERR "   -keepold               Will not destroy the current contents of the\n";
+    print STDERR "                          archives directory (the default)\n";
+    print STDERR "   -removeold             Will remove the old contents of the archives\n";
+    print STDERR "                          directory -- use with care\n";
+    print STDERR "   -gzip                  Use gzip to compress resulting gml documents\n";
+    print STDERR "   -maxdocs number        Maximum number of documents to import\n\n";
+}
+
+
+&main ();
+
+sub main {
+    if (!parsargv::parse(\@ARGV, 
+			 'verbosity/\d+/2', \$verbosity,
+			 'importdir/.*/', \$importdir,
+			 'archivedir/.*/', \$archivedir,
+			 'keepold', \$keepold,
+			 'removeold', \$removeold,
+			 'gzip', \$gzip,
+			 'maxdocs/\d+/', \$maxdocs)) {
+	&print_usage();
+	die "\n";
+    }
+
+    # set removeold to false if it has been defined
+    $removeold = 0 if ($keepold);
+
+    # get and check the collection name
+    if (($collection = &util::use_collection(@ARGV)) eq "") {
+	&print_usage();
+	die "\n";
+    }
+
+    # get the list of plugins for this collection
+    @plugins = ();
+    $configfilename = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, "etc/collect.cfg");
+    if (-e $configfilename) {
+	$collectcfg = &colcfg::read_collect_cfg ($configfilename);
+	if (defined $collectcfg->{'plugins'}) {
+	    @plugins = @{$collectcfg->{'plugins'}};
+	}
+	if (defined $collectcfg->{'importdir'} && $importdir eq "") {
+	    $importdir = $collectcfg->{'importdir'};
+	}
+	if (defined $collectcfg->{'archivedir'} && $archivedir eq "") {
+	    $archivedir = $collectcfg->{'archivedir'};
+	}
+	if (defined $collectcfg->{'removeold'}) {
+	    if ($collectcfg->{'removeold'} =~ /^true$/i && !$keepold) {
+		$removeold = 1;
+	    }
+	    if ($collectcfg->{'removeold'} =~ /^false$/i && !$removeold) {
+		$removeold = 0;
+	    }
+	}
+    } else {
+	die "Couldn't find the configuration file $configfilename\n";
+    }
+    
+    # fill in the default import and archives directories if none
+    # were supplied, turn all \ into / and remove trailing /
+    $importdir = "$ENV{'GSDLCOLLECTDIR'}/import" if $importdir eq "";
+    $importdir =~ s/[\\\/]+/\//g;
+    $importdir =~ s/\/$//;
+    $archivedir = "$ENV{'GSDLCOLLECTDIR'}/archives" if $archivedir eq "";
+    $archivedir =~ s/[\\\/]+/\//g;
+    $archivedir =~ s/\/$//;
+
+    # load all the plugins
+    $pluginfo = &plugin::load_plugins ($collection, \@plugins);
+    if (scalar(@$pluginfo) == 0) {
+	print STDERR "No plugins were loaded.\n";
+	die "\n";
+    }
+    
+    # remove the old contents of the archives directory if needed
+    if ($removeold && -e $archivedir) {
+	print STDERR "Warning - removing current contents of the archives directory\n";
+	print STDERR "          in preparation for the import\n";
+	sleep(5); # just in case...
+	&util::rm_r ($archivedir);
+    }
+
+    # read the archive information file
+    $archive_info_filename = &util::filename_cat ($archivedir, "archives.inf");
+    $archive_info = new arcinfo ();
+    $archive_info->load_info ($archive_info_filename);
+
+    # create a docsave object to process the documents
+    $processor = new docsave ($collection, $archive_info, $verbosity, $gzip);
+    $processor->setarchivedir ($archivedir);
+
+    # process the import directory
+    &plugin::read ($pluginfo, $importdir, 
+		   "", {}, $processor, $maxdocs);
+    
+    # write out the archive information file
+    $archive_info->save_info($archive_info_filename);
+}
Index: /tags/gsdl-21/gsdl/bin/script/mkcol.pl
===================================================================
--- /tags/gsdl-21/gsdl/bin/script/mkcol.pl	(revision 699)
+++ /tags/gsdl-21/gsdl/bin/script/mkcol.pl	(revision 699)
@@ -0,0 +1,191 @@
+#!/usr/local/bin/perl5 -w
+
+###########################################################################
+#
+# mkcol.pl --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+
+# This program will setup a new collection from a model one. It does this by
+# copying the model, moving files to have the correct names, and replacing
+# text within the files to match the parameters.
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+}
+
+use parsargv;
+
+sub print_usage {
+    print STDERR "\n  usage: $0 [options] collection-name\n\n";
+    print STDERR "  options:\n";
+    print STDERR "   -creator email      Your email address\n";
+    print STDERR "   -maintainer email   The current maintainer's email address\n";
+    print STDERR "   -public true|false  If this collection has anonymous access\n";
+    print STDERR "   -beta true|false    If this collection is still under development\n";
+#    print STDERR "   -index type         The indexes which should be made\n";
+#    print STDERR "   -indextext name     The index description\n";
+#    print STDERR "   -defaultindex type  The index to use if no others are supplied\n\n";
+}
+
+sub traverse_dir
+{
+    my ($modeldir, $coldir) = @_;
+    my ($newfile, @filetext);
+
+    if (!(-e $coldir)) {
+	if (!mkdir ($coldir, 0775)) {
+	    die "$!";
+	}
+    }
+
+    opendir(DIR, $modeldir) || die "Can't read $modeldir";
+    my @files = grep(!/^(\.\.?|CVS)$/, readdir(DIR));
+    closedir(DIR);
+
+    foreach $file (@files)
+    {
+	if (-d "$modeldir/$file") {
+	    traverse_dir ("$modeldir/$file", "$coldir/$file");
+
+	} else {
+	    my $destfile = $file;
+	    $destfile =~ s/^\modelcol/$collection/;
+	    $destfile =~ s/^\MODELCOL/$capcollection/;
+
+	    print STDERR "doing replacements for $modeldir/$file\n";
+	    open (INFILE, "$modeldir/$file") || 
+		die "Can't read file $modeldir/$file";
+	    open (OUTFILE, ">$coldir/$destfile") ||
+		die "Can't create file $coldir/destfile";
+
+	    while (defined ($line = <INFILE>)) {
+		$line =~ s/\*\*collection\*\*/$collection/g;
+		$line =~ s/\*\*COLLECTION\*\*/$capcollection/g;
+		$line =~ s/\*\*creator\*\*/$creator/g;
+		$line =~ s/\*\*maintainer\*\*/$maintainer/g;
+		$line =~ s/\*\*public\*\*/$public/g;
+		$line =~ s/\*\*beta\*\*/$beta/g;
+		$line =~ s/\*\*key\*\*/$key/g;
+		$line =~ s/\*\*indexes\*\*/$indexesstr/g;
+		$line =~ s/\*\*indexestext\*\*/$indexestextstr/g;
+		$line =~ s/\*\*defaultindex\*\*/$defaultindex/g;
+
+		print OUTFILE $line;
+	    }
+	    
+	    close (OUTFILE);
+	    close (INFILE);
+	}
+    }
+}
+
+
+my (@indexes, @indexestext);
+
+# get and check options
+if (!&parsargv::parse(\@ARGV,
+		       'creator/\w+\@[\w\.]+/', \$creator,
+		       'maintainer/\w+\@[\w\.]+/', \$maintainer,
+		       'public/true|false/true', \$public,
+		       'beta/true|false/true', \$beta,
+		       'index/.*/document:all', \@indexes,
+		       'indextext/\.*/Terms must appear within the same document', \@indexestext,
+		       'defaultindex/.*/document:all', \$defaultindex
+		       )) {
+    &print_usage();
+    die "\n";
+}
+    
+# get and check the collection name
+($collection) = @ARGV;
+if (!defined($collection)) {
+    &print_usage();
+    die "\n";
+}
+
+if (length($collection) > 8) {
+    print STDERR "The collection name must be less than 8 characters\n";
+    print STDERR "so compatibility with earlier filesystems can be\n";
+    print STDERR "maintained.\n";
+    die "\n";
+}
+
+if ($collection eq "modelcol") {
+    print STDERR "No collection can be named modelcol as this is the\n";
+    print STDERR "name of the model collection.\n";
+    die "\n";
+}
+
+if ($collection eq "CVS") {
+    print STDERR "No collection can be named CVS as this may interfere\n";
+    print STDERR "with directories created by the CVS versioning system\n";
+    die "\n";
+}
+
+if (!defined($creator) || $creator eq "") {
+    print STDERR "The creator was not defined. This variable is\n";
+    print STDERR "needed to recognise duplicate collection names.\n";
+    die "\n";
+}
+
+if (!defined($maintainer) || $maintainer eq "") {
+    $maintainer = $creator;
+}
+
+$public = "true" unless defined $public;
+$beta = "true" unless defined $beta;
+
+srand(time);
+$key = int(rand()*100000000.0) + int(rand()*100000000.0);
+
+# get capitalised version of the collection
+$capcollection = $collection;
+$capcollection =~ tr/a-z/A-Z/;
+
+
+# get the strings to include.
+$indexesstr = join ("\t", @indexes);
+$indexestextstr = "";
+for ($i=0; $i<scalar(@indexestext); $i++) {
+    $indexestextstr .= "_$indexes[$i]_[] {$indexestext[$i]}\n";
+}
+
+
+# make sure the model collection exists
+die "Cannot find the model collection $ENV{'GSDLHOME'}/collect/modelcol" unless
+    (-d "$ENV{'GSDLHOME'}/collect/modelcol");
+
+# make sure this collection does not already exist
+if (-e "$ENV{'GSDLHOME'}/collect/$collection") {
+    print STDERR "This collection already exists\n";
+    die "\n";
+}
+
+
+# start creating the collection
+print STDERR "Creating the collection $collection\n\n";
+&traverse_dir ("$ENV{'GSDLHOME'}/collect/modelcol", "$ENV{'GSDLHOME'}/collect/$collection");
+print STDERR "\n\nThe new collection is in $ENV{'GSDLHOME'}/collect/$collection.\n\n";
+
Index: /tags/gsdl-21/gsdl/bin/script/newsrc.pl
===================================================================
--- /tags/gsdl-21/gsdl/bin/script/newsrc.pl	(revision 699)
+++ /tags/gsdl-21/gsdl/bin/script/newsrc.pl	(revision 699)
@@ -0,0 +1,126 @@
+#!/usr/local/bin/perl5 -w
+
+###########################################################################
+#
+# newsrc.pl -- creates new blank source files
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+foreach $filename (@ARGV) {
+    if (-e "$filename.cpp") {
+	print STDERR "$filename.cpp already exists\n";
+	next;
+    }
+    if (-e "$filename.h") {
+	print STDERR "$filename.h already exists\n";
+	next;
+    }
+
+    $upperfilename = $filename;
+    $upperfilename =~ tr/a-z/A-Z/;
+    $upperfilename =~ s/[^A-Z]/_/g;
+
+    print STDERR "Creating $filename.cpp\n";
+
+    open (CPPFILE, ">$filename.cpp") || die "Couldn't write $filename.cpp";
+    
+    print CPPFILE "/**********************************************************************\n";
+    print CPPFILE " *\n";
+    print CPPFILE " * $filename.cpp -- \n";
+    print CPPFILE " * A component of the Greenstone digital library software\n";
+    print CPPFILE " * from the New Zealand Digital Library Project at the\n";
+    print CPPFILE " * University of Waikato, New Zealand.\n";
+    print CPPFILE " *\n";
+    print CPPFILE " * Copyright (C) 1999  The New Zealand Digital Library Project\n";
+    print CPPFILE " *\n";
+    print CPPFILE " * This program is free software; you can redistribute it and/or modify\n";
+    print CPPFILE " * it under the terms of the GNU General Public License as published by\n";
+    print CPPFILE " * the Free Software Foundation; either version 2 of the License, or\n";
+    print CPPFILE " * (at your option) any later version.\n";
+    print CPPFILE " *\n";
+    print CPPFILE " * This program is distributed in the hope that it will be useful,\n";
+    print CPPFILE " * but WITHOUT ANY WARRANTY; without even the implied warranty of\n";
+    print CPPFILE " * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n";
+    print CPPFILE " * GNU General Public License for more details.\n";
+    print CPPFILE " *\n";
+    print CPPFILE " * You should have received a copy of the GNU General Public License\n";
+    print CPPFILE " * along with this program; if not, write to the Free Software\n";
+    print CPPFILE " * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.\n";
+    print CPPFILE " *\n";
+    print CPPFILE " * \$" . "Id: Exp \$\n";
+    print CPPFILE " *\n";
+    print CPPFILE " *********************************************************************/\n";
+    print CPPFILE "\n";
+    print CPPFILE "/*\n";
+    print CPPFILE "   \$" . "Log: newsrc.pl,v \$\n";
+    print CPPFILE " */\n";
+    print CPPFILE "\n";
+    print CPPFILE "\n";
+    print CPPFILE "#include \"$filename.h\"\n";
+    print CPPFILE "\n";
+
+    close (CPPFILE);
+
+
+    print STDERR "Creating $filename.h\n";
+
+    open (HFILE, ">$filename.h") || die "Couldn't write $filename.h";
+
+    print HFILE "/**********************************************************************\n";
+    print HFILE " *\n";
+    print HFILE " * $filename.h -- \n";
+    print HFILE " * A component of the Greenstone digital library software\n";
+    print HFILE " * from the New Zealand Digital Library Project at the\n";
+    print HFILE " * University of Waikato, New Zealand.\n";
+    print HFILE " *\n";
+    print HFILE " * Copyright (C) 1999  The New Zealand Digital Library Project\n";
+    print HFILE " *\n";
+    print HFILE " * This program is free software; you can redistribute it and/or modify\n";
+    print HFILE " * it under the terms of the GNU General Public License as published by\n";
+    print HFILE " * the Free Software Foundation; either version 2 of the License, or\n";
+    print HFILE " * (at your option) any later version.\n";
+    print HFILE " *\n";
+    print HFILE " * This program is distributed in the hope that it will be useful,\n";
+    print HFILE " * but WITHOUT ANY WARRANTY; without even the implied warranty of\n";
+    print HFILE " * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n";
+    print HFILE " * GNU General Public License for more details.\n";
+    print HFILE " *\n";
+    print HFILE " * You should have received a copy of the GNU General Public License\n";
+    print HFILE " * along with this program; if not, write to the Free Software\n";
+    print HFILE " * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.\n";
+    print HFILE " *\n";
+    print HFILE " * \$" . "Id: Exp \$\n";
+    print HFILE " *\n";
+    print HFILE " *********************************************************************/\n";
+    print HFILE "\n";
+    print HFILE "\n";
+    print HFILE "\#ifndef ${upperfilename}_H\n";
+    print HFILE "\#define ${upperfilename}_H\n";
+    print HFILE "\n";
+    print HFILE "\n";
+    print HFILE "\n";
+    print HFILE "\n";
+    print HFILE "\#endif\n";
+
+    close (HFILE);
+}
+
Index: /tags/gsdl-21/gsdl/bin/script/perlcalc.pl
===================================================================
--- /tags/gsdl-21/gsdl/bin/script/perlcalc.pl	(revision 699)
+++ /tags/gsdl-21/gsdl/bin/script/perlcalc.pl	(revision 699)
@@ -0,0 +1,5 @@
+#! /usr/local/bin/perl5 -w
+
+while (defined($line=<STDIN>)) {
+	print eval($line), "\n";
+}
Index: /tags/gsdl-21/gsdl/bin/script/togb.pl
===================================================================
--- /tags/gsdl-21/gsdl/bin/script/togb.pl	(revision 699)
+++ /tags/gsdl-21/gsdl/bin/script/togb.pl	(revision 699)
@@ -0,0 +1,58 @@
+#!/usr/local/bin/perl5 -w
+
+###########################################################################
+#
+# togb.pl -- converts to GB
+#
+# Copyright (C) 1999 DigiLib Systems Limited, NZ.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+}
+
+use unicode;
+use multiread;
+use parsargv;
+
+
+$encoding = "utf8";
+if (!parsargv::parse(\@ARGV, 
+		     'unicode', \$unicode,
+		     'extended', \$extended,
+		     'gb', \$gb)) {
+    print STDERR "\n  usage: $0 [options]\n\n";
+    print STDERR "  options:\n";
+    print STDERR "   -utf8     input is in utf-8 or unicode (default)\n";
+    print STDERR "   -extended input is in extended ascii\n";
+    print STDERR "   -gb       input is in GB or GBK\n\n";
+    die "\n";
+}
+
+$encoding = "utf8" if $unicode;
+$encoding = "extended" if $extended;
+$encoding = "gb" if $gb;
+
+$reader = new multiread ();
+$reader->set_handle ('main::STDIN');
+$reader->set_encoding ($encoding);
+$line = "";
+while (defined ($line = $reader->read_line())) {
+    print &gb::unicode2gb (&unicode::utf82unicode ($line));
+}
Index: /tags/gsdl-21/gsdl/bin/script/touc.pl
===================================================================
--- /tags/gsdl-21/gsdl/bin/script/touc.pl	(revision 699)
+++ /tags/gsdl-21/gsdl/bin/script/touc.pl	(revision 699)
@@ -0,0 +1,72 @@
+#!/usr/local/bin/perl5 -w
+
+###########################################################################
+#
+# touc.pl -- converts to unicode
+#
+# Copyright (C) 1999 DigiLib Systems Limited, NZ.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+}
+
+use unicode;
+use multiread;
+use parsargv;
+
+
+$encoding = "utf8";
+if (!parsargv::parse(\@ARGV, 
+		     'unicode', \$unicode,
+		     'extended', \$extended,
+		     'gb', \$gb)) {
+    print STDERR "\n  usage: $0 [options]\n\n";
+    print STDERR "  options:\n";
+    print STDERR "   -utf8     input is in utf-8 or unicode (default)\n";
+    print STDERR "   -extended input is in extended ascii\n";
+    print STDERR "   -gb       input is in GB or GBK\n\n";
+    die "\n";
+}
+
+$encoding = "utf8" if $unicode;
+$encoding = "extended" if $extended;
+$encoding = "gb" if $gb;
+
+
+if ($ENV{'GSDLOS'} =~ /windows/i) {
+    binmode (STDOUT); # silly windows
+}
+
+# output in little endian
+print "\xff\xfe";
+
+$reader = new multiread ();
+$reader->set_handle ('main::STDIN');
+$reader->set_encoding ($encoding);
+$line = "";
+$ucline = [];
+while (defined ($line = $reader->read_line())) {
+    $ucline = &unicode::utf82unicode ($line);
+    foreach $c (@$ucline) {
+	$c1 = chr (int ($c / 256));
+	$c2 = chr (int ($c % 256));
+	print "$c2$c1";
+    }
+}
Index: /tags/gsdl-21/gsdl/bin/script/toutf8.pl
===================================================================
--- /tags/gsdl-21/gsdl/bin/script/toutf8.pl	(revision 699)
+++ /tags/gsdl-21/gsdl/bin/script/toutf8.pl	(revision 699)
@@ -0,0 +1,58 @@
+#!/usr/local/bin/perl5 -w
+
+###########################################################################
+#
+# toutf8.pl -- converts to utf8
+#
+# Copyright (C) 1999 DigiLib Systems Limited, NZ.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+}
+
+use unicode;
+use multiread;
+use parsargv;
+
+
+$encoding = "utf8";
+if (!parsargv::parse(\@ARGV, 
+		     'unicode', \$unicode,
+		     'extended', \$extended,
+		     'gb', \$gb)) {
+    print STDERR "\n  usage: $0 [options]\n\n";
+    print STDERR "  options:\n";
+    print STDERR "   -utf8     input is in utf-8 or unicode (default)\n";
+    print STDERR "   -extended input is in extended ascii\n";
+    print STDERR "   -gb       input is in GB or GBK\n\n";
+    die "\n";
+}
+
+$encoding = "utf8" if $unicode;
+$encoding = "extended" if $extended;
+$encoding = "gb" if $gb;
+
+$reader = new multiread ();
+$reader->set_handle ('main::STDIN');
+$reader->set_encoding ($encoding);
+$line = "";
+while (defined ($line = $reader->read_line())) {
+    print $line;
+}
Index: /tags/gsdl-21/gsdl/bin/script/unindex.pl
===================================================================
--- /tags/gsdl-21/gsdl/bin/script/unindex.pl	(revision 699)
+++ /tags/gsdl-21/gsdl/bin/script/unindex.pl	(revision 699)
@@ -0,0 +1,425 @@
+#!/usr/local/bin/perl5 -w
+
+###########################################################################
+#
+# unindex.pl --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# this program will decompress all the text from a built index
+# and return it to gml format - may be slightly broken at present
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+    $FileHandle = 'FH000';
+}
+
+use doc;
+use docsave;
+use util;
+use parsargv;
+use GDBM_File;
+use FileHandle;
+use English;
+
+select STDERR; $| = 1;
+select STDOUT; $| = 1;
+
+
+# globals
+$collection = "";
+$index = "";
+$textdir = "";
+$classinfo = {};
+$mgread = ++$FileHandle;
+$mgwrite = ++$FileHandle;
+
+
+
+sub print_usage {
+    print STDERR "\n  usage: $0 [options]\n\n";
+    print STDERR "  options:\n";
+    print STDERR "   -verbosity number      0=none, 3=lots\n";
+    print STDERR "   -indexdir directory    The index to be decompressed (defaults to ./index)\n";
+    print STDERR "   -archivedir directory  Where the converted material ends up (defaults to ./archives\n";
+    print STDERR "   -removeold             Will remove the old contents of the archives\n";
+    print STDERR "                          directory -- use with care\n\n";
+}
+
+&main ();
+
+sub main {
+    if (!parsargv::parse(\@ARGV, 
+			 'verbosity/\d+/2', \$verbosity,
+			 'indexdir/.*/index', \$indexdir,
+			 'archivedir/.*/archives', \$archivedir,
+			 'removeold', \$removeold)) {
+	&print_usage();
+	die "\n";
+    }
+
+    die "indexdir $indexdir does not exist\n\n" unless (-d $indexdir);
+    $indexdir =~ s/\/$//;
+    if (-d $archivedir) {
+	if ($removeold) {
+	    print STDERR "Warning - removing current contents of the archives directory $archivedir\n";
+	    print STDERR "          in preparation for the import\n";
+	    sleep(5); # just in case...
+	    &util::rm_r ($archivedir);
+	}
+    } else {
+	&util::mk_all_dir ($archivedir);
+    }
+
+    my $gdbmfile = &get_gdbmfile ($indexdir);
+    &set_index ();
+    &get_classinfo ($gdbmfile);
+    &openmg ();
+
+    # read the archive information file
+    my $archive_info_filename = &util::filename_cat ($archivedir, "archives.inf");
+    my $archive_info = new arcinfo ();
+    $archive_info->load_info ($archive_info_filename);
+
+    # create a docsave object to process the documents
+    my $processor = new docsave ("", $archive_info, $verbosity);
+    $processor->setarchivedir ($archivedir);
+
+    # create a document object for the classification file
+    my $clsudoc_obj = new doc($gdbmfile, "classification");
+    $clsudoc_obj->set_OID ("CLSU");
+
+    my ($doc_obj);
+    foreach $classification (keys %$classinfo) {
+	my $count = 0;
+	foreach $section (sort numerically keys %{$classinfo->{$classification}}) {
+	    print STDERR "\n$classification.$section" if $verbosity >= 2;
+	    if ($classification =~ /^B$/i) {
+		last if $count > 10;
+		# create a new document
+		$doc_obj = new doc ($classinfo->{$classification}->{$section}->{'9999999'}->{'jobnum'}, 
+				    "indexed_doc");
+		$doc_obj->set_OID();
+		my $cursection = $doc_obj->get_top_section();
+		&add_section_content ($doc_obj, $cursection,
+				      $classinfo->{$classification}->{$section});
+		&recurse_classinfo ($doc_obj, $classinfo->{$classification}->{$section}, 
+				    $doc_obj->get_end_child($cursection), 0);
+		$processor->process($doc_obj);
+	    } else {
+		my $classifier = &int_classification ($classification);
+		$clsudoc_obj->create_named_section("$classifier.$section");
+		&add_section_content ($clsudoc_obj, "$classifier.$section",
+				      $classinfo->{$classification}->{$section});
+		&recurse_classinfo ($clsudoc_obj, $classinfo->{$classification}->{$section}, 
+				    "", "$classifier.$section");
+	    }
+	    $count ++;
+	}
+    }
+    $processor->process($clsudoc_obj);
+    &closemg();
+
+    # write out the archive information file
+    $archive_info->save_info($archive_info_filename);
+}
+
+# returns the path to the gdbm info database - also
+# sets the $collection and $textdir global variable
+sub get_gdbmfile {
+    my ($indexdir) = @_;
+    my ($gdbmfile);
+
+    opendir (DIR, $indexdir) || die "Couldn't open directory $indexdir\n\n";
+    my @conts = readdir DIR;
+    close DIR;
+
+    foreach $file (@conts) {
+	if ($file =~ /text$/) {
+	    $textdir = $file;
+	    last;
+	}
+    }
+    die "No text directory found in $indexdir\n\n" 
+	unless defined $textdir && $textdir =~ /text$/;
+
+    $gdbmfile = &util::filename_cat ($indexdir, $textdir);
+
+    opendir (DIR, $gdbmfile) || die "Couldn't open directory $gdbmfile\n\n";
+    @conts = readdir DIR;
+    close DIR;
+
+    foreach $file (@conts) {
+	if ($file =~ /^(.*?)\.(?:ldb|bdb)$/) {
+	    $collection = $1;
+	    $gdbmfile = &util::filename_cat ($gdbmfile, $file);
+	    last;
+	}
+    }
+    
+    if (defined $collection && $collection =~ /\w/) {
+	$textdir = &util::filename_cat ($textdir, $collection);
+    } else {
+	die "collection global wasn't set\n";
+    }
+    return $gdbmfile if (-e $gdbmfile);
+    die "Couldn't find gdbm info database in $indexdir\n\n";
+}
+
+sub get_classinfo {
+    my ($gdbmfile) = @_;
+    my ($class);
+
+    open (DB2TXT, "db2txt $gdbmfile |") || die "couldn't open pipe to db2txt\n";
+
+    print STDERR "Generating classification table from $gdbmfile\n" if $verbosity >= 2;
+    my $count = 0;
+
+    $/ = '-' x 70;
+    my $entry = "";
+    while (defined ($entry = <DB2TXT>)) {
+	next unless $entry =~ /\w/;
+	$entry =~ s/\n+/\\n/g;
+	my ($key, $value) = $entry =~ /\[([^\]]*)\](.*)/;
+
+	print STDERR "." if (($count % 100) == 99 && $verbosity >= 2);
+
+	next if ($key !~ /\./); # ignore the docnums and top level of classifications
+
+	die "Badly formatted key $key in $gdbmfile\n\n" 
+	    unless (($class, $section) = $key =~ /^(.)\.(.*)$/);	
+	$classinfo->{$class} = {} unless defined $classinfo->{$class};
+	my @sections = split /\./, $section;
+
+	my $hashptr = $classinfo->{$class};
+	foreach $sec (@sections) {
+	    $hashptr->{$sec} = {} unless defined $hashptr->{$sec};
+	    $hashptr = $hashptr->{$sec};
+	}
+	&get_metadata ($key, \$value, $hashptr);
+
+	$count ++;
+    }
+
+    $/ = "\n";
+    print STDERR "\n" if $verbosity >= 2;
+}
+
+sub get_metadata {
+    my ($key, $gdb_str_ref, $hashsection) = @_;
+    
+    my ($title) = $$gdb_str_ref =~ /(?:<t>|<title>)(.*?)(?:\\n|$)/i; 
+    my ($docnum) = $$gdb_str_ref =~ /(?:<d>|<docnum>)(.*?)(?:\\n|$)/i; 
+    my ($jobnum) = $$gdb_str_ref =~ /(?:<j>|<jobnumber>)(.*?)(?:\\n|$)/i; 
+    my ($classifications) = $$gdb_str_ref =~ /(?:<x>|<classification>)(.*?)(?:\\n|$)/i; 
+    my $hastext = 1;
+    $hastext = 0 if ($$gdb_str_ref =~ /(?:<c>|<contains>)(.*?)(?:\\n|$)/i);
+    my ($creator) = $$gdb_str_ref =~ /<a>(.*?)(?:\\n|$)/i; 
+    my ($date) = $$gdb_str_ref =~ / <i>(.*?)(?:\\n|$)/i; 
+			
+    # just in case there are empty classifications			     
+    if ($hastext && !defined $docnum) {
+	print STDERR "\nwarning: $key entry has no contents\n" if $verbosity;
+	if ($verbosity >= 2) {
+	    my $valuestr = $$gdb_str_ref;
+	    $valuestr =~ s/\\n/\n/g;
+	    print STDERR "$valuestr\n";
+	}
+	$hastext = 0;
+    }
+
+    # shove metadata in 9999999 to keep it numeric and prevent 
+    # sorting from being a pain. Watch out for documents with
+    # 9999999 subsections ;-)
+    if (defined $hashsection->{'9999999'}) {
+	print STDERR "\nwarning: $key appears multiple times\n" if $verbosity;
+    } else {
+	$hashsection->{'9999999'}->{'Title'} = $title if defined $title;
+	$hashsection->{'9999999'}->{'docnum'} = $docnum if defined $docnum;
+	$hashsection->{'9999999'}->{'jobnum'} = $jobnum if defined $jobnum;
+	$hashsection->{'9999999'}->{'classifications'} = $classifications if defined $classifications;
+	$hashsection->{'9999999'}->{'hastext'} = $hastext;
+	$hashsection->{'9999999'}->{'Creator'} = $creator if defined $creator;
+	$hashsection->{'9999999'}->{'Date'} = $date if defined $date;
+    }
+}
+
+sub recurse_classinfo {
+    my ($doc_obj, $hashsection, $cursection, $class) = @_;
+    foreach $section (sort numerically keys %$hashsection) {
+	next if $section == 9999999;
+	if ($class) {
+	    $doc_obj->create_named_section("$class.$section");
+	    &add_section_content ($doc_obj, "$class.$section", $hashsection->{$section});
+	    &recurse_classinfo ($doc_obj, $hashsection->{$section}, "", "$class.$section");
+	} else {
+	    $cursection = 
+		$doc_obj->insert_section($doc_obj->get_end_child($doc_obj->get_parent_section($cursection)));
+	    &add_section_content ($doc_obj, $cursection, $hashsection->{$section});
+	    &recurse_classinfo ($doc_obj, $hashsection->{$section}, $doc_obj->get_end_child($cursection));
+	}
+    }
+}
+
+sub add_section_content {
+    my ($doc_obj, $cursection, $hashsection) = @_;
+
+    $doc_obj->add_metadata ($cursection, "Title", $hashsection->{'9999999'}->{'Title'})
+	if defined $hashsection->{'9999999'}->{'Title'};
+    $doc_obj->add_metadata ($cursection, "Creator", $hashsection->{'9999999'}->{'Creator'})
+	if defined $hashsection->{'9999999'}->{'Creator'};
+    $doc_obj->add_metadata ($cursection, "Date", $hashsection->{'9999999'}->{'Date'})
+	if defined $hashsection->{'9999999'}->{'Date'};
+
+    if (defined $hashsection->{'9999999'}->{'classifications'}) {
+	my @classifications = split /:/, $hashsection->{'9999999'}->{'classifications'};
+	map {$doc_obj->add_metadata ($cursection, 'Subject', $_); } @classifications;
+    }
+
+    $doc_obj->add_text ($cursection, &get_text ($hashsection->{'9999999'}->{'docnum'}))
+	if ($hashsection->{'9999999'}->{'hastext'});
+}
+
+sub set_index {
+    # check that $collection has been set
+    die "collection global was not set\n"
+	unless defined $collection && $collection =~ /\w/;
+
+    # find an index (just use first non-text directory we come across in $indexdir)
+    opendir (INDEXDIR, $indexdir) || die "couldn't open directory $indexdir\n";
+    my @indexes = readdir INDEXDIR;
+    close INDEXDIR;
+    foreach $i (@indexes) {
+	next if $i =~ /text$/i || $i =~ /\./ ;
+	$index = &util::filename_cat ($i, $collection);
+	last;
+    }
+}
+
+sub get_text {
+    my ($docnum) = @_;
+
+    print STDERR "." if $verbosity >= 2;
+    &mgcommand ($docnum);
+
+    <$mgread>;	# eat the document separator
+
+    my $text = "";
+    my $line = "";
+
+    while (defined ($line = <$mgread>))
+    {
+	last if $line =~ /^<\/mg>/;
+	$text .= $line;
+    }
+
+    # Read in the last statement, which should be:
+    #  "dd documents retrieved."
+    <$mgread>;
+
+    return $text;
+}
+
+sub numerically {$a <=> $b;}
+
+# converts leading letter of a classification into its ascii equivalent
+# i.e C.2.4 becomes 67.2.4
+sub int_classification {
+    my ($classification) = @_;
+    my $c = ord($classification);
+    $classification =~ s/^./$c/;
+
+    return $classification;
+}
+
+
+# mg stuff
+
+sub openmg {
+    
+    die "Unable to start mgquery." unless
+	openpipe($mgread, $mgwrite,
+		 "mgquery -d $indexdir -f $index -t $textdir");
+
+    $mgwrite->autoflush();
+
+    &mgcommand('.set expert true');
+    &mgcommand('.set terminator "</mg>\n"');
+    &mgcommand('.set mode text');
+    &mgcommand('.set query docnums');
+    &mgcommand('.set term_freq off');
+}
+
+sub closemg {
+    &mgcommand (".quit");
+    close($mgread);
+    close($mgwrite);
+}
+
+sub mgcommand {
+    my ($command) = @_;
+
+    return if $command =~ /^\s*$/;
+    print $mgwrite "$command\n";
+
+    # eat up the command executed which is echoed
+    <$mgread>;
+}
+
+# openpipe(READ, WRITE, CMD)
+# 
+# Like open2, except CMD's stderr is also redirected.
+# 
+sub openpipe
+{
+    my ($read, $write, $cmd) = @_;
+    my ($child_read, $child_write);
+
+    $child_read = ++$FileHandle;
+    $child_write = ++$FileHandle;
+
+    pipe($read, $child_write) || die "Failed pipe($read, $child_write): $!";
+    pipe($child_read, $write) || die "Failed pipe($child_read, $write): $!";
+    my $pid;
+
+    if (($pid = fork) < 0) {
+        die "Failed fork: $!";
+    } elsif ($pid == 0) {
+        close($read);
+        close($write);
+        open(STDIN, "<&$child_read");
+        open(STDOUT, ">&$child_write");
+        open(STDERR, ">&$child_write");
+        exec($cmd);
+        die "Failed exec $cmd: $!";
+    }
+
+    close($child_read);
+    close($child_write);
+
+    $write->autoflush();
+    $read->autoflush();
+
+    return 1;
+}
Index: /tags/gsdl-21/gsdl/collect/demo/etc/mags.txt
===================================================================
--- /tags/gsdl-21/gsdl/collect/demo/etc/mags.txt	(revision 699)
+++ /tags/gsdl-21/gsdl/collect/demo/etc/mags.txt	(revision 699)
@@ -0,0 +1,15 @@
+CERES                                                              1  CERES
+"Food and Nutrition Bulletin"                                      2  "Food and Nutrition Bulletin"
+"The Courier"                                                      3  "The Courier"
+"SPORE Bulletin"                                                   4  "SPORE Bulletin"
+"Boiling Point"                                                    5  "Boiling Point"
+"Developing Ideas"                                                 6  "Developing Ideas"
+"GATE Magazine"                                                    7  "GATE Magazine"
+"Go Between"                                                       8  "Go Between"
+"Basin - News"                                                     9  "Basin - News"
+"HDD-(PHN)FLASH, newsletter on Population, Health and Nutrition"  10  "HDD-(PHN)FLASH, newsletter on Population, Health and Nutrition"
+"IDRC Reports"                                                    11  "IDRC Reports"
+"New and Noteworthy in Nutrition"                                 12  "New and Noteworthy in Nutrition"
+Seedling                                                          13  Seedling
+"Soft Energy Worldwide"                                           14  "Soft Energy Worldwide"
+"Sustainable energy News"                                         15  "Sustainable energy News"
Index: /tags/gsdl-21/gsdl/collect/demo/etc/org.txt
===================================================================
--- /tags/gsdl-21/gsdl/collect/demo/etc/org.txt	(revision 699)
+++ /tags/gsdl-21/gsdl/collect/demo/etc/org.txt	(revision 699)
@@ -0,0 +1,60 @@
+accu	1	ACCU
+ag21	2	"Agenda 21"
+bgz	3	"BASIN - GTZ - SKAT"
+bostid	4	BOSTID
+cps	5	CPS
+cpas	6	"CTA Spore"
+cf	7	"Commonwealth Foundation"
+csu	8	"Computer Science Unplugged"
+dcs	9	"Development Consultancy Services"
+1echo	0	E.C.H.O
+ecc	11	"EC Courier"
+ecdg8	12	"EC DG8"
+echo	13	"Educational Concerns for Hunger Organization"
+fao	14	FAO
+faobfs	15	"FAO Better Farming series"
+ceres	16	"FAO Ceres"
+ff	17	"Food First"
+ghp	18	GHP
+grain	19	GRAIN
+gret	20	GRET
+gtz	21	GTZ
+gate	22	"GTZ GATE Magazine"
+habitat	23	Habitat
+idrc	24	IDRC
+iisd	25	IISD
+irma	26	IRMA
+bp	27	"ITDG Boiling Point"
+itdg	28	"ITDG Food Chain"
+iis	29	"Indian Institute of Sciences"
+chandra	30	"Individual Contributor Chandrakant"
+alkire	31	"Individual Contributor S. Alkire"
+inforse	32	Inforse
+iirr	33	"International Institute for Rural Reconstruction (IIRR)"
+msf	34	MSF
+ngls	35	NGLS
+norad	36	NORAD
+nri	37	NRI
+nsc	38	"North South Centre"
+op	39	"Orbi Pharma"
+oxfam	40	Oxfam
+pace	41	PACE
+paho	42	PAHO
+pc	43	"Peace Corps"
+pelum	44	Pelum
+sida	45	SIDA
+skat	46	SKAT
+tool	47	Tool
+un	48	UN
+unesco	49	UNESCO
+unicef	50	UNICEF
+fnb	51	"UNU Food and Nutrition Bulletin"
+unu	52	"UNU United Nations University"
+vso	53	VSO
+who	54	WHO
+wwf	55	WWF
+wwsf	56	WWSF
+wa	57	"Water Aid"
+winrock	58	Winrock
+ws	59	Workshop
+wb	60	"World Bank"
Index: /tags/gsdl-21/gsdl/collect/demo/etc/sub.txt
===================================================================
--- /tags/gsdl-21/gsdl/collect/demo/etc/sub.txt	(revision 699)
+++ /tags/gsdl-21/gsdl/collect/demo/etc/sub.txt	(revision 699)
@@ -0,0 +1,245 @@
+1        1        "General reference"
+1.2      1.2      "Dictionaries, glossaries, language courses, terminology (all languages)"
+2        2        "Sustainable Development, International cooperation, Projects; NGO, Organizations, Poverty and Hunger Alleviation, Basic Human Needs"
+2.1      2.1      "Development policy and theory, international cooperation, national planning, national plans"
+2.2      2.2      "Development, national planning, national plans"
+2.3      2.3      "Project planning and evaluation (incl. project management and dissemination strategies)"
+2.4      2.4      "Regional development and planning incl. regional profiles"
+2.5      2.5      "Nongovernmental organizations (NGOs) in general, self- help organizations (their role in development). For specific organizations such as women's or peasants' see respective class within 06."
+2.6      2.6      "Organizations, institutions, United Nations (general, directories, yearbooks, annual reports, etc.)"
+2.6.1    2.6.1    "United Nations"
+2.6.2    2.6.2    "International organizations"
+2.6.3    2.6.3    "Regional organizations"
+2.6.5    2.6.5    "European Community - European Union"
+2.7      2.7      "Sustainable Development, Development models and examples; Best practices (general)"
+2.8      2.8      "Basic Human Needs"
+2.9      2.9      "Hunger and Poverty Alleviation"
+3        3        "Economics, Economic policies, Finance, Trade, Structural Adjustment"
+3.2      3.2      "Economic and social situation, economic policy, country reports (incl. national statistical yearbooks)"
+3.4      3.4      "Industrial economics, industrial development, foreign investment (incl. investment policy, investment promotion) incl.multinational corporations"
+3.6     3.6       "Trade and commerce, Export policy and reglementations"
+3.7     3.7       "Tourism (economic and social aspects)"
+3.8     3.8       "Labour economics"
+3.8.1   3.8.1     "Employment policy"
+3.10    3.10      "Structural Adjustment"
+4       4         "Business, Micro-entreprises, Management, Marketing, Finance, Microfinance, Cooperatives"
+4.1     4.1       "Business economics (general works, textbooks)"
+4.2     4.2       "Microeconomic planning techniques, Micro-entreprises"
+4.3     4.3       "Management and personnel management"
+4.4     4.4       "Business organization and administration, office management"
+4.5     4.5       "Marketing and commercialisation, export (from businesses viewpoint)"
+4.6     4.6       "Accounting"
+4.7     4.7       "Finance and microfinance / microcredit"
+4.8     4.8       "Cooperatives, cooperative movement"
+4.8.1   4.8.1     "Agricultural cooperatives"
+4.8.4   4.8.4     "Housing cooperatives"
+4.8.5   4.8.5     "Other specific cooperatives"
+5       5         "Politics, Public administration, Law and Legislation, Peace, Human Rights"
+5.2     5.2       "Public administration, Tasks of councillors"
+5.4     5.4       "Human rights, humanitarian law"
+5.4     5.5       "Disarmament, Peace, Conflict and Conflict Resolutions"
+6       6         "Society, Culture, Community, Woman, Youth, Population"
+6.1     6.1       "Social sciences, sociology (works comprising several subgroups) incl. participatory research and training"
+6.2     6.2       "Society, social structure, social change, social services, social movements, social groups, Minorities"
+6.3     6.3       "Community development, community organizing"
+6.4     6.4       "Women, gender and development, women's organizations"
+6.5     6.5       "Children and youth (youth centres)"
+6.6     6.6       "Demography and population"
+6.7     6.7       "Philosophy"
+6.8     6.8       "History, Cultural Heritage and Language"
+6.9     6.9       "Psychology"
+6.10    6.10      "Anthropology, ethnology"
+6.11    6.11      "Religion - Spirituality - Yoga"
+6.12    6.12      "Art, Literature, Creativity, Traditional Cultures, Fashion"
+7       7         "Education and Training"
+7.1     7.1       "Education and training (general works, theories, methods)"
+7.3     7.3       "Teaching methods and aids"
+7.4     7.4       "Adult education, Literacy training"
+7.5     7.5       "Early education, Primary education, Secondary education"
+7.6     7.6       "Specific courses or subjects, educational games"
+7.7     7.7       "Higher education"
+7.8     7.8       "Vocational, professional and practical training - for specific training- curricula, textbooks- see specific subject."
+7.9     7.9       "Health Education and Training (see under health and Nutrition)"
+7.10    7.10      "Environmental education, Global education, Psychological and assertiveness education, Non-formal education etc"
+7.11    7.11      "Language education"
+7.12    7.12      "Specialised education: see also under the subject"
+7.13    7.13      "Teacher Training"
+8       8         "Communication, Information and Documentation"
+8.1     8.1       "Communication, telecommunication, mass communication, mass media, film-making"
+8.2     8.2       "Information for development, Information transfer, Information policy"
+8.3     8.3       "Documentation, Library science and management, see also RAMP studies under 08.06"
+8.4     8.4       "Information technology, Computer science, Computer use, Use of computers for developing countries"
+8.5     8.5       "Internet, e-mail"
+8.6     8.6       "Records and Archives Management Programme (RAMP) of  UNESCO, Archive Management"
+9       9         "Transport"
+9.1     9.1       "Transport (rural and urban) (general works)"
+9.2     9.2       "Land transport"
+10      10        "Science, Research and Technology"
+10.1    10.1      "Natural science, technology, engineering, research and development, research methods and theory (general works, directories, encyclopedias)"
+10.2    10.2      "Transfer of technology and scientific cooperation"
+10.2.1  10.2.1    "Technology transfer and scientific cooperation for developing countries"
+10.3    10.3      "Appropriate/ Alternative/ Applied technology (AT), traditional, intermediate technology,  economic and social aspect, directories"
+10.8    10.8      "Biology"
+10.9    10.9      "Biotechnology, genetic engineering"
+10.10   10.10     "Medicine and Health : science and research"
+11      11        "Ecology, Biodiversity, Natural resources, Environment, Waste Disposal"
+11.1    11.1      "Natural resources, geography"
+11.1.1  11.1.1    "Climate, weather, meteorology . Climatologic Changes. Global warming"
+11.2    11.2      "Ecology, environmental policy, eco- development studies"
+11.3    11.3      "Resources conservation, environmental protection, biodiversity"
+11.3.1  11.3.1    "Arid zones, desertification"
+11.3.2  11.3.2    "Nature reserve, game reserve"
+11.3.3  11.3.3    "Rainforests and rainforest conservation"
+11.3.4  11.3.4    "Biodiversity conservation"
+11.4    11.4      "Erosion and erosion control - for techniques see agriculture and forestry."
+11.5    11.5      "Pollution, Pollution control, Pesticides, Biological monitoring, Asbestos"
+11.5.2  11.5.2    "Water pollution"
+11.5.4  11.5.4    "Pesticide pollution"
+11.5.5  11.5.5    "Asbestos"
+11.6    11.6      "Solid waste disposal"
+11.7    11.7      "Environmental Handbook of GTZ-BMZ"
+12      12        "Water, Sanitation and Sewage Disposal"
+12.1    12.1      "Water and sanitation, hydrology"
+12.2    12.2      "Water resources (surface water, groundwater)"
+12.3    12.3      "Hydraulic structures (dams, intakes, wells, ponds, cisterns)"
+12.4    12.4      "Pumps and other water- lifting devices"
+12.5    12.5      "Water supply and storage"
+12.6    12.6      "Water treatment and purification (biological, chemical, physical)"
+12.7    12.7      "Sanitation. Waste water and sewage disposal"
+13      13        "Health, Nutrition, Medicine"
+13.1    13.1      "Health (general works, incl. health economics and sociology, health statistics, health and community development)"
+13.1.1  13.1.1    "Health planning and policy"
+13.1.6  13.1.6    "Disasters, medicine in humanitarian emergencies"
+13.2    13.2      "Food, Nutrition and Malnutrition"
+13.3    13.3      "Family health,  Mother and Child care, Family planning, Pregnancy and Childbirth"
+13.4    13.4      "Community Health, Health Care Delivery, Extension workers and training"
+13.5    13.5      "Medicine, Clinical Care, Diseases, Hygiene, Emergencies (treatment of specific diseases, injuries, accidents)"
+13.6    13.6      "Health Education, Promotion, Prevention, AIDS Prevention, Safety, Books covering multiple health dimensions"
+13.7    13.7      "Production and use of medicines, essential drugs"
+13.7.1  13.7.1    "Pharmaceuticals and essential drugs"
+13.7.2  13.7.2"   "Herbal medicines, medicinal plants"
+13.8    13.8      "Health material and equipment; aids for disabled people"
+13.9    13.9      "Medical an nutrition research papers, medical and nutrition scientific studies, specialised tropical medicine publications"
+14      14        "Agriculture and Food Processing"
+14.1    14.1      "Agriculture (general works), Economics,  Policies, Development, Research, Management"
+14.1.1  14.1.1    "Land reform, Settlements, Rural Development and related topics"
+14.1.2  14.1.2    "Agricultural training and education"
+14.1.3  14.1.3    "Agricultural Science and Research"
+14.1.4  14.1.4    "Rural and Farm management"
+14.1.5  14.1.5    "Policies and Planning in agriculture"
+14.1.6  14.1.6    "Biotechnology and agriculture"
+14.1.7  14.1.7    "Agricultural finance systems"
+14.2    14.2      "Agricultural production systems (shifting cultivation, wet- land farming, dry- land farming, agroforestry), Sustainable Agriculture"
+14.3    14.3      "Organic farming, Eco-farming"
+14.4    14.4      "Agricultural implements, tools and machinery (incl. mechanization, agricultural transport) - for implements specific to particular plants see respective plant."
+14.5    14.5      "Soil and land management, improvement and treatment (incl. terracing, erosion control techniques, rangeland improvement)"
+14.6    14.6      "Irrigation and drainage - for hydraulic structures see 12.03."
+14.7    14.7      "Fertilization and composting"
+14.8    14.8      "Storage and packaging of agricultural products, Post-Harvest management"
+14.9    14.9      "Agricultural waste processing - for energy use see 20.06."
+14.10   14.10     "Food processing and preservation"
+14.10.2 14.10.2   "Food drying (all kinds)"
+14.10.3 14.10.3   "Fermented Foods"
+14.10.4 14.10.4   "Oil production and processing"
+14.12   14.12     "Better Farming series of FAO and INADES"
+15      15        "Plant and Crop Production, Protection, and Processing"
+15.1    15.1      "Plant/crop production and processing, vegetable gardening, botany (general works, publications comprising several plants and practical how to books, intercropping)"
+15.2    15.2      "Plant protection (incl. insecticidinal plants), Pests and Pest Control, Integrated Pest Management"
+15.4    15.4      "Cereals (rice, corn/maize, millet, sorghum, wheat, etc.)"
+15.5    15.5      "Starchy crops and root crops (cassava, potatoes, etc.)"
+15.6    15.6      "Oil- producing plants, vegetable oil, nuts"
+15.7    15.7      "Garden vegetables, legumes, mushrooms"
+15.8    15.8      "Fruits and wine (bananas, mango's, apple's etc)"
+15.10   15.10     "Coffee, Tea, Cocoa, Coca and other stimulants"
+15.12   15.12     "Rubber, resin, wax plants"
+15.14   15.14     "Fodder and pasture plants"
+15.15   15.15     "Other plants/crops, intercropping (for intercropping, see also under 15.01)"
+16      16        "Animal Husbandry and Animal Product Processing"
+16.1    16.1      "Animal husbandry (general,works) incl. animal feed, works covering several aspects and animals, zoology, see also other animals 6.11"
+16.2    16.2      "Veterinary medicine, animal diseases"
+16.3    16.3      "Animal traction (excl. implements), animal training"
+16.5    16.5      "Cattle"
+16.6    16.6      "Goats, sheep, camels"
+16.8    16.8      "Rabbits and guinea pigs"
+16.9    16.9      "Poultry, ducks and other birds"
+16.10   16.10     "Beekeeping and honey extraction"
+16.11   16.11     "Other animals (micro-livestock, little known animals, silkworms, reptiles, frogs, snails, game, etc.)"
+16.12   16.12     "Animal product processing - for processing of wool and silk see 19.08 - for processing of leather see 19.09"
+16.12.2 16.12.2   "Milk and egg processing (dairy, cheese)"
+17      17        "Fishery and Aquaculture"
+17.1    17.1      "Fishery and aquaculture (general works), fishery resources"
+17.2    17.2      "Fishing techniques and equipment"
+17.3    17.3      "Aquaculture, fish farming, fish production (fish ponds, feeding, etc.)"
+17.6    17.6      "Fishery product processing, storage, and marketing"
+18      18        "Forestry and Agroforestry"
+18.1    18.1      "Forestry, general works, Forest conservation"
+18.2    18.2      "Reforestation - for erosion control see 11.04 and 14.05."
+18.3    18.3      "Cultivation and conservation (nurseries), protection and pest resistance"
+18.4    18.4      "Forests, Cloud Forests"
+18.5    18.5      "Specific trees and other forestry plants (incl. bamboo)"
+18.6    18.6      "Agroforestry"
+18.7    18.7      "Sustainable exploitation and commercialisation of forest products"
+19      19        "Industry, Manufacture and Services"
+19.1    19.1      "Industry (general works, textbooks, directories) incl. industrial planning"
+19.2    19.2      "Industrial training, consulting - for vocational training see 07.08."
+19.3    19.3      "Small- scale industry, handicraft, home industry"
+19.4    19.4      "Industrial profiles and studies covering several sectors"
+19.5    19.5      "Recycling and solid waste services"
+19.7    19.7      "Mining and quarrying"
+19.8    19.8      "Textile industry, manufacture of wearing apparel (incl. processing of natural fibres)"
+19.11   19.11     "Pulp and paper products (incl. printing and bookbinding)"
+19.12   19.12     "Chemical and chemical products industry incl. soap and candle production, salt"
+19.13   19.13     "Pottery, ceramics, glass and glass products (incl. jewellery, buttons, chalk)"
+19.14   19.14     "Metal industry"
+19.14.1 19.14.1   "Forging and blacksmithing"
+19.17   19.17     "Packing Industry"
+20      20        "Energy , Renewable energies, Household energy"
+20.1    20.1      "Energy and power (general works) incl. energy per country, energy savings, and publications covering several sources of energy, rural energy"
+20.2    20.2      "Transformation, distribution and storage of energy and electricity"
+20.2.1  20.2.1    "Rural electrification"
+20.3    20.3      "Conventional energy resources (oil, petroleum, coal, natural gas, peat, nuclear energy)"
+20.4    20.4      "Renewable energy resources"
+20.4.1  20.4.1    "Bio-mass (general works)"
+20.5    20.5      "Wood and charcoal (incl. kilns and activated charcoal)"
+20.6    20.6      "Solid fuels from agricultural wastes and other organic material (pellets, eco- bricks, briquets)"
+20.7    20.7      "Biogas (incl. biogas equipment)"
+20.8    20.8      "Cookstoves, Fuel-saving cookstoves, Household Energy"
+20.10   20.10     "Alcohol and other liquid fuels"
+20.11   20.11     "Solar energy, photovoltaic energy"
+20.12   20.12     "Wind energy"
+20.13   20.13     "Hydropower"
+20.14   20.14     "New Energy technology and research, Planetary Association for Clean Energy publications"
+21      21        "Settlements, Housing, Building -  Infrastructure Construction (Roads etc)"
+21.1    21.1      "Settlements and housing: general works incl. low- cost housing, planning techniques, surveying, etc."
+21.2    21.2      "Urban settlements (urban housing programmes, urbanization, squatters, slums, slum improvement)"
+21.3    21.3      "Rural settlements"
+21.4    21.4      "Architecture and Building design"
+21.4.2  21.4.2    "Solar and climate responsive architecture"
+21.5    21.5      "Construction materials"
+21.6    21.6      "Building machinery and equipment"
+21.7    21.7      "Building and construction (building components, construction techniques)"
+21.7.1  21.7.1    "Roofing"
+21.9    21.9      "Civil engineering (roads, bridges, etc.) - Road Construction"
+22      22        "Disasters, Disaster Mitigation, Humanitarian and Food Aid, Humanitarian Interventions, Refugees"
+22.1    22.1      "Disasters and Disaster Mitigation, Organisation"
+22.2    22.2      "Humanitarian Aid, Food Aid"
+22.3    22.3      "Humanitarian Interventions and Conflicts"
+22.5    22.5      "Refugees"
+23      23        "Development Periodicals and Magazines"
+23.1    23.1      "BASIN-NEWS - Building Advisory Service and Information Network by GTZ- GATE, SKAT,  ITDG and CRATerre"
+23.2    23.2      "Boiling Point Newsletters on Household Energy of Intermediate Technology Development Group.and GTZ"
+23.3    23.3      "CERES - FAO Review on Agriculture and Development: 1976 - 1997"
+23.4    23.4      "Developing Ideas - Bi-monthly Digests by the International Institute for Sustainable Development"
+23.5    23.5      "ECHO's Human Development Notes by the Educational Concerns for Hunger network"
+23.6    23.6      "Food and Nutrition Bulletins 1978-1996 - United Nations University"
+23.7    23.7      "Food Chain, newsletters on Food Processing of Intermediate Technology Development Group."
+23.8    23.8      "GATE Development magazine of GTZ"
+23.9    23.9      "HDD-FLASH, newsletter on Population, Health and Nutrition of the Worldbank Human Development Department"
+23.10   23.10     "IDRC monthly and weekly reports 1993 - 1997"
+23.11   23.11     "New and Noteworthy in Nutrition - World Bank"
+23.12   23.12     "SEEDLING: bi-montly publication of GRAIN - Genetic Resources International - Spain"
+23.13   23.13     "SPORE Bulletins 1986-1996 of the CTA - Technical and Agricultural Centre of the European Union"
+23.14   23.14     "Sustainable Energy Newsletters of the INFORSE network"
+23.15   23.15     "The Courier ACP 1990 - 1996 Africa-Caribbean-Pacific - European Union"
+23.16   23.16     "The NGLS Magazine Go Between"
+24      24        "Resources, Product Lists, Catalogues, Internet Sites"
Index: /tags/gsdl-21/gsdl/collect/demo/perllib/classify/HDLList.pm
===================================================================
--- /tags/gsdl-21/gsdl/collect/demo/perllib/classify/HDLList.pm	(revision 699)
+++ /tags/gsdl-21/gsdl/collect/demo/perllib/classify/HDLList.pm	(revision 699)
@@ -0,0 +1,253 @@
+###########################################################################
+#
+# HDLList.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+
+# this is kind of a mutated version of AZList, written
+# specifically for the hdl collection. It's the same as
+# an AZList classification except it contains a Hierarcical
+# 'Magazines' classification as well as the standard 
+# alphabetical ones
+
+# There's also no need to provide a metadata element
+# to sort by - it always uses Title
+
+# magazines are sorted by filename
+
+package HDLList;
+
+use sorttools;
+
+sub new {
+    my ($class) = @_;
+
+    return bless {
+	'list'=>{},
+	'magclassifier'=>""
+	}, $class;
+}
+
+sub init {
+    my $self = shift (@_);
+
+    $self->{'list'} = {};
+
+    # load up a Hierarchy classifier for the 'magazines' sub-section
+    my $hrname = &util::filename_cat($ENV{'GSDLHOME'},"perllib/classify/Hierarchy.pm"); 
+    if (-e $hrname) { require $hrname; }
+    else { die "HDLList ERROR - couldn't find classifier \"$hrname\"\n"; }
+
+    # create the Hierarchy classifier object
+    my ($hrclassobj);
+    eval ("\$hrclassobj = new Hierarchy(\"hfile=mags.txt\", \"metadata=Magazine\", \"sort=Filename\", \"title=_magazines_\")");
+    die "$@" if $@;
+
+    $hrclassobj->init();
+    $self->{'magclassifier'} = $hrclassobj;
+}
+
+sub classify {
+    my $self = shift (@_);
+    my ($doc_obj) = @_;
+
+    my $doc_OID = $doc_obj->get_OID();
+
+    my $magazine = $doc_obj->get_metadata_element ($doc_obj->get_top_section(),
+						   'Magazine');
+
+    my $title = $doc_obj->get_metadata_element ($doc_obj->get_top_section(),
+						'Title');
+
+    # if this document contains <Magazine> metadata we want to shove it into
+    # the Hierarchical magazine classification (and not include it in the
+    # alphabetical list), otherwise shove it in the AZList bit (as long as
+    # it has a <Title>
+    if (defined $magazine && $magazine =~ /\w/) {
+	$self->{'magclassifier'}->classify ($doc_obj);
+
+    } elsif (defined $title && $title =~ /\w/) {
+	&sorttools::format_string_english (\$title);
+	
+	if (defined $self->{'list'}->{$doc_OID}) {
+	    print STDERR "WARNING: HDLList::classify called multiple times for $doc_OID\n";
+	} 
+	$self->{'list'}->{$doc_OID} = $title;
+    }
+}
+
+
+sub get_classify_info {
+    my $self = shift (@_);
+
+    my @classlist = sort {$self->{'list'}->{$a} cmp $self->{'list'}->{$b};} keys %{$self->{'list'}};
+
+    my $classifyinfo = $self->splitlist (\@classlist);
+
+    # shove in the magazine bit
+    push (@{$classifyinfo->{'contains'}}, $self->{'magclassifier'}->get_classify_info());
+    
+    return $classifyinfo;
+}
+
+
+sub get_entry {
+    my $self = shift (@_);
+    my ($title, $childtype, $thistype) = @_;
+    
+    # organise into classification structure
+    my %classifyinfo = ('childtype'=>$childtype,
+			'Title'=>$title,
+			'contains'=>[]);
+    $classifyinfo{'thistype'} = $thistype 
+	if defined $thistype && $thistype =~ /\w/;
+
+    return \%classifyinfo;
+}
+
+# splitlist takes an ordered list of classifications (@$classlistref) and splits it
+# up into alphabetical sub-sections.
+sub splitlist {
+    my $self = shift (@_);
+    my ($classlistref) = @_;
+    my $classhash = {};
+
+    # top level
+    my $classifyinfo = $self->get_entry ("Title", "HList", "Invisible");
+
+    # don't need to do any splitting if there are less than 20 classifications
+    # we have a A-Z section containing everything if there aren't many 
+    # classifications (since we've already got a 'Magazines' section)
+    if ((scalar @$classlistref) <= 20) {
+	my $allinfo = $self->get_entry ("A-Z", "VList");
+	foreach $subOID (@$classlistref) {
+	    push (@{$allinfo->{'contains'}}, {'OID'=>$subOID});
+	}
+	push (@{$classifyinfo->{'contains'}}, $allinfo);
+	return $classifyinfo;
+    }
+	
+    # first split up the list into separate A-Z and 0-9 classifications
+    foreach $classification (@$classlistref) {
+	my $title = $self->{'list'}->{$classification};
+	$title =~ s/^(.).*$/$1/;
+	$title =~ tr/[a-z]/[A-Z]/;
+	if ($title =~ /^[0-9]$/) {$title = '0-9';}
+	elsif ($title !~ /^[A-Z]$/) {
+	    print STDERR "HDLList: WARNING $classification has badly formatted title ($title)\n";
+	}
+	$classhash->{$title} = [] unless defined $classhash->{$title};
+	push (@{$classhash->{$title}}, $classification);
+    }
+    $classhash = $self->compactlist ($classhash);
+
+    my @tmparr = ();
+    foreach $subsection (sort keys (%$classhash)) {
+	push (@tmparr, $subsection);
+    }
+    
+    # if there's a 0-9 section it will have been sorted to the beginning
+    # but we want it at the end
+    if ($tmparr[0] eq '0-9') {
+	shift @tmparr;
+	push (@tmparr, '0-9');
+    }
+
+    foreach $subclass (@tmparr) {
+	my $tempclassify = $self->get_entry($subclass, "VList");
+	foreach $subsubOID (@{$classhash->{$subclass}}) {
+	    push (@{$tempclassify->{'contains'}}, {'OID'=>$subsubOID});
+	}
+	push (@{$classifyinfo->{'contains'}}, $tempclassify);
+    }
+
+    return $classifyinfo;
+}
+
+sub compactlist {
+    my $self = shift (@_);
+    my ($classhashref) = @_;
+    my $compactedhash = {};
+    my @currentOIDs = ();
+    my $currentfirstletter = "";
+    my $currentlastletter = "";
+    my $lastkey = "";
+
+    # minimum and maximum documents to be displayed per page.
+    # the actual maximum will be max + (min-1).
+    # the smallest sub-section is a single letter at present
+    # so in this case there may be many times max documents
+    # displayed on a page.
+    my $min = 10; 
+    my $max = 30;
+
+    foreach $subsection (sort keys %$classhashref) {
+	if ($subsection eq '0-9') {
+	    @{$compactedhash->{$subsection}} = @{$classhashref->{$subsection}};
+	    next;
+	}
+	$currentfirstletter = $subsection if $currentfirstletter eq "";
+	if ((scalar (@currentOIDs) < $min) ||
+	    ((scalar (@currentOIDs) + scalar (@{$classhashref->{$subsection}})) <= $max)) {
+	    push (@currentOIDs, @{$classhashref->{$subsection}});
+	    $currentlastletter = $subsection;
+	} else {
+
+	    if ($currentfirstletter eq $currentlastletter) {
+		@{$compactedhash->{$currentfirstletter}} = @currentOIDs;
+		$lastkey = $currentfirstletter;
+	    } else {
+		@{$compactedhash->{"$currentfirstletter-$currentlastletter"}} = @currentOIDs;
+		$lastkey = "$currentfirstletter-$currentlastletter";
+	    } 
+	    if (scalar (@{$classhashref->{$subsection}}) >= $max) {
+		$compactedhash->{$subsection} = $classhashref->{$subsection};
+		@currentOIDs = ();
+		$currentfirstletter = "";
+	    } else {
+		@currentOIDs = @{$classhashref->{$subsection}};
+		$currentfirstletter = $subsection;
+		$currentlastletter = $subsection;
+	    }
+	}
+    }
+
+    # add final OIDs to last sub-classification if there aren't many otherwise
+    # add final sub-classification
+    if (scalar (@currentOIDs) < $min) {
+	my ($newkey) = $lastkey =~ /^(.)/;
+	@currentOIDs = (@{$compactedhash->{$lastkey}}, @currentOIDs);
+	delete $compactedhash->{$lastkey};
+	@{$compactedhash->{"$newkey-$currentlastletter"}} = @currentOIDs;	
+    } else {
+	if ($currentfirstletter eq $currentlastletter) {
+	    @{$compactedhash->{$currentfirstletter}} = @currentOIDs;
+	} else {
+	    @{$compactedhash->{"$currentfirstletter-$currentlastletter"}} = @currentOIDs;
+	} 
+    }
+
+    return $compactedhash;
+}
+
+1;
Index: /tags/gsdl-21/gsdl/collect/modelcol/etc/collect.cfg
===================================================================
--- /tags/gsdl-21/gsdl/collect/modelcol/etc/collect.cfg	(revision 699)
+++ /tags/gsdl-21/gsdl/collect/modelcol/etc/collect.cfg	(revision 699)
@@ -0,0 +1,14 @@
+creator       **creator**
+maintainer    **maintainer**
+public        **public**
+beta          **beta**
+key           **key**
+
+indexes        document:text
+defaultindex   document:text
+plugins        GMLPlug TEXTPlug ArcPlug RecPlug
+
+classify       AZList metadata=Title
+
+collectionmeta collectionname    "**collection**"
+collectionmeta .document:text    "documents"
Index: /tags/gsdl-21/gsdl/config.h.in
===================================================================
--- /tags/gsdl-21/gsdl/config.h.in	(revision 699)
+++ /tags/gsdl-21/gsdl/config.h.in	(revision 699)
@@ -0,0 +1,171 @@
+/* config.h.in.  Generated automatically from configure.in by autoheader.  */
+
+/* Define if on AIX 3.
+   System headers sometimes define this.
+   We just want to avoid a redefinition error message.  */
+#ifndef _ALL_SOURCE
+#undef _ALL_SOURCE
+#endif
+
+/* Define if using alloca.c.  */
+#undef C_ALLOCA
+
+/* Define to empty if the keyword does not work.  */
+#undef const
+
+/* Define to one of _getb67, GETB67, getb67 for Cray-2 and Cray-YMP systems.
+   This function is required for alloca.c support on those systems.  */
+#undef CRAY_STACKSEG_END
+
+/* Define if you have alloca, as a function or macro.  */
+#undef HAVE_ALLOCA
+
+/* Define if you have <alloca.h> and it should be used (not on Ultrix).  */
+#undef HAVE_ALLOCA_H
+
+/* Define if you don't have vprintf but do have _doprnt.  */
+#undef HAVE_DOPRNT
+
+/* Define if you have the vprintf function.  */
+#undef HAVE_VPRINTF
+
+/* Define if on MINIX.  */
+#undef _MINIX
+
+/* Define to `long' if <sys/types.h> doesn't define.  */
+#undef off_t
+
+/* Define if the system does not provide POSIX.1 features except
+   with this defined.  */
+#undef _POSIX_1_SOURCE
+
+/* Define if you need to in order for stat and other things to work.  */
+#undef _POSIX_SOURCE
+
+/* Define as the return type of signal handlers (int or void).  */
+#undef RETSIGTYPE
+
+/* Define to `unsigned' if <sys/types.h> doesn't define.  */
+#undef size_t
+
+/* If using the C implementation of alloca, define if you know the
+   direction of stack growth for your system; otherwise it will be
+   automatically deduced at run-time.
+ STACK_DIRECTION > 0 => grows toward higher addresses
+ STACK_DIRECTION < 0 => grows toward lower addresses
+ STACK_DIRECTION = 0 => direction of growth unknown
+ */
+#undef STACK_DIRECTION
+
+/* Define if you have the ANSI C header files.  */
+#undef STDC_HEADERS
+
+/* Define if you can safely include both <sys/time.h> and <time.h>.  */
+#undef TIME_WITH_SYS_TIME
+
+/* Define if your <sys/time.h> declares struct tm.  */
+#undef TM_IN_SYS_TIME
+
+/* Define to the name of the distribution.  */
+#undef PACKAGE
+
+/* Define to the version of the distribution.  */
+#undef VERSION
+
+/* Define to 1 if you have the valloc function.  */
+#undef HAVE_VALLOC
+
+/* Some braindead header files do not have the requ'd function declarations */
+#undef HAVE_FGETC_DECL
+#undef HAVE_FREAD_DECL
+
+/* See if have this field in the prstatus_t structure */
+/* It stores the size of the process heap */
+#undef HAVE_PR_BRKSIZE
+
+/* Define FASTCGI if needsbe */
+#undef USE_FASTCGI
+
+/* Define if you have the ftime function.  */
+#undef HAVE_FTIME
+
+/* Define if you have the ftruncate function.  */
+#undef HAVE_FTRUNCATE
+
+/* Define if you have the getpagesize function.  */
+#undef HAVE_GETPAGESIZE
+
+/* Define if you have the getrusage function.  */
+#undef HAVE_GETRUSAGE
+
+/* Define if you have the mallinfo function.  */
+#undef HAVE_MALLINFO
+
+/* Define if you have the select function.  */
+#undef HAVE_SELECT
+
+/* Define if you have the setbuffer function.  */
+#undef HAVE_SETBUFFER
+
+/* Define if you have the strcasecmp function.  */
+#undef HAVE_STRCASECMP
+
+/* Define if you have the strftime function.  */
+#undef HAVE_STRFTIME
+
+/* Define if you have the strstr function.  */
+#undef HAVE_STRSTR
+
+/* Define if you have the strtol function.  */
+#undef HAVE_STRTOL
+
+/* Define if you have the times function.  */
+#undef HAVE_TIMES
+
+/* Define if you have the valloc function.  */
+#undef HAVE_VALLOC
+
+/* Define if you have the <dirent.h> header file.  */
+#undef HAVE_DIRENT_H
+
+/* Define if you have the <fcntl.h> header file.  */
+#undef HAVE_FCNTL_H
+
+/* Define if you have the <limits.h> header file.  */
+#undef HAVE_LIMITS_H
+
+/* Define if you have the <memory.h> header file.  */
+#undef HAVE_MEMORY_H
+
+/* Define if you have the <ndir.h> header file.  */
+#undef HAVE_NDIR_H
+
+/* Define if you have the <string.h> header file.  */
+#undef HAVE_STRING_H
+
+/* Define if you have the <sys/dir.h> header file.  */
+#undef HAVE_SYS_DIR_H
+
+/* Define if you have the <sys/ndir.h> header file.  */
+#undef HAVE_SYS_NDIR_H
+
+/* Define if you have the <sys/procfs.h> header file.  */
+#undef HAVE_SYS_PROCFS_H
+
+/* Define if you have the <sys/time.h> header file.  */
+#undef HAVE_SYS_TIME_H
+
+/* Define if you have the <unistd.h> header file.  */
+#undef HAVE_UNISTD_H
+
+/* Define if you have the g library (-lg).  */
+#undef HAVE_LIBG
+
+/* Define if you have the gdbm library (-lgdbm).  */
+#undef HAVE_LIBGDBM
+
+/* Define if you have the m library (-lm).  */
+#undef HAVE_LIBM
+
+/* Define if you have the stdc library (-lstdc).  */
+#undef HAVE_LIBSTDC
Index: /tags/gsdl-21/gsdl/configtest.pl
===================================================================
--- /tags/gsdl-21/gsdl/configtest.pl	(revision 699)
+++ /tags/gsdl-21/gsdl/configtest.pl	(revision 699)
@@ -0,0 +1,1 @@
+print $]
Index: /tags/gsdl-21/gsdl/configure
===================================================================
--- /tags/gsdl-21/gsdl/configure	(revision 699)
+++ /tags/gsdl-21/gsdl/configure	(revision 699)
@@ -0,0 +1,3295 @@
+#! /bin/sh
+
+# Guess values for system-dependent variables and create Makefiles.
+# Generated automatically using autoconf version 2.13 
+# Copyright (C) 1992, 93, 94, 95, 96 Free Software Foundation, Inc.
+#
+# This configure script is free software; the Free Software Foundation
+# gives unlimited permission to copy, distribute and modify it.
+
+# Defaults:
+ac_help=
+ac_default_prefix=/usr/local
+# Any additions from configure.in:
+
+# Initialize some variables set by options.
+# The variables have the same names as the options, with
+# dashes changed to underlines.
+build=NONE
+cache_file=./config.cache
+exec_prefix=NONE
+host=NONE
+no_create=
+nonopt=NONE
+no_recursion=
+prefix=NONE
+program_prefix=NONE
+program_suffix=NONE
+program_transform_name=s,x,x,
+silent=
+site=
+srcdir=
+target=NONE
+verbose=
+x_includes=NONE
+x_libraries=NONE
+bindir='${exec_prefix}/bin'
+sbindir='${exec_prefix}/sbin'
+libexecdir='${exec_prefix}/libexec'
+datadir='${prefix}/share'
+sysconfdir='${prefix}/etc'
+sharedstatedir='${prefix}/com'
+localstatedir='${prefix}/var'
+libdir='${exec_prefix}/lib'
+includedir='${prefix}/include'
+oldincludedir='/usr/include'
+infodir='${prefix}/info'
+mandir='${prefix}/man'
+
+# Initialize some other variables.
+subdirs=
+MFLAGS= MAKEFLAGS=
+SHELL=${CONFIG_SHELL-/bin/sh}
+# Maximum number of lines to put in a shell here document.
+ac_max_here_lines=12
+
+ac_prev=
+for ac_option
+do
+
+  # If the previous option needs an argument, assign it.
+  if test -n "$ac_prev"; then
+    eval "$ac_prev=\$ac_option"
+    ac_prev=
+    continue
+  fi
+
+  case "$ac_option" in
+  -*=*) ac_optarg=`echo "$ac_option" | sed 's/[-_a-zA-Z0-9]*=//'` ;;
+  *) ac_optarg= ;;
+  esac
+
+  # Accept the important Cygnus configure options, so we can diagnose typos.
+
+  case "$ac_option" in
+
+  -bindir | --bindir | --bindi | --bind | --bin | --bi)
+    ac_prev=bindir ;;
+  -bindir=* | --bindir=* | --bindi=* | --bind=* | --bin=* | --bi=*)
+    bindir="$ac_optarg" ;;
+
+  -build | --build | --buil | --bui | --bu)
+    ac_prev=build ;;
+  -build=* | --build=* | --buil=* | --bui=* | --bu=*)
+    build="$ac_optarg" ;;
+
+  -cache-file | --cache-file | --cache-fil | --cache-fi \
+  | --cache-f | --cache- | --cache | --cach | --cac | --ca | --c)
+    ac_prev=cache_file ;;
+  -cache-file=* | --cache-file=* | --cache-fil=* | --cache-fi=* \
+  | --cache-f=* | --cache-=* | --cache=* | --cach=* | --cac=* | --ca=* | --c=*)
+    cache_file="$ac_optarg" ;;
+
+  -datadir | --datadir | --datadi | --datad | --data | --dat | --da)
+    ac_prev=datadir ;;
+  -datadir=* | --datadir=* | --datadi=* | --datad=* | --data=* | --dat=* \
+  | --da=*)
+    datadir="$ac_optarg" ;;
+
+  -disable-* | --disable-*)
+    ac_feature=`echo $ac_option|sed -e 's/-*disable-//'`
+    # Reject names that are not valid shell variable names.
+    if test -n "`echo $ac_feature| sed 's/[-a-zA-Z0-9_]//g'`"; then
+      { echo "configure: error: $ac_feature: invalid feature name" 1>&2; exit 1; }
+    fi
+    ac_feature=`echo $ac_feature| sed 's/-/_/g'`
+    eval "enable_${ac_feature}=no" ;;
+
+  -enable-* | --enable-*)
+    ac_feature=`echo $ac_option|sed -e 's/-*enable-//' -e 's/=.*//'`
+    # Reject names that are not valid shell variable names.
+    if test -n "`echo $ac_feature| sed 's/[-_a-zA-Z0-9]//g'`"; then
+      { echo "configure: error: $ac_feature: invalid feature name" 1>&2; exit 1; }
+    fi
+    ac_feature=`echo $ac_feature| sed 's/-/_/g'`
+    case "$ac_option" in
+      *=*) ;;
+      *) ac_optarg=yes ;;
+    esac
+    eval "enable_${ac_feature}='$ac_optarg'" ;;
+
+  -exec-prefix | --exec_prefix | --exec-prefix | --exec-prefi \
+  | --exec-pref | --exec-pre | --exec-pr | --exec-p | --exec- \
+  | --exec | --exe | --ex)
+    ac_prev=exec_prefix ;;
+  -exec-prefix=* | --exec_prefix=* | --exec-prefix=* | --exec-prefi=* \
+  | --exec-pref=* | --exec-pre=* | --exec-pr=* | --exec-p=* | --exec-=* \
+  | --exec=* | --exe=* | --ex=*)
+    exec_prefix="$ac_optarg" ;;
+
+  -gas | --gas | --ga | --g)
+    # Obsolete; use --with-gas.
+    with_gas=yes ;;
+
+  -help | --help | --hel | --he)
+    # Omit some internal or obsolete options to make the list less imposing.
+    # This message is too long to be a string in the A/UX 3.1 sh.
+    cat << EOF
+Usage: configure [options] [host]
+Options: [defaults in brackets after descriptions]
+Configuration:
+  --cache-file=FILE       cache test results in FILE
+  --help                  print this message
+  --no-create             do not create output files
+  --quiet, --silent       do not print \`checking...' messages
+  --version               print the version of autoconf that created configure
+Directory and file names:
+  --prefix=PREFIX         install architecture-independent files in PREFIX
+                          [$ac_default_prefix]
+  --exec-prefix=EPREFIX   install architecture-dependent files in EPREFIX
+                          [same as prefix]
+  --bindir=DIR            user executables in DIR [EPREFIX/bin]
+  --sbindir=DIR           system admin executables in DIR [EPREFIX/sbin]
+  --libexecdir=DIR        program executables in DIR [EPREFIX/libexec]
+  --datadir=DIR           read-only architecture-independent data in DIR
+                          [PREFIX/share]
+  --sysconfdir=DIR        read-only single-machine data in DIR [PREFIX/etc]
+  --sharedstatedir=DIR    modifiable architecture-independent data in DIR
+                          [PREFIX/com]
+  --localstatedir=DIR     modifiable single-machine data in DIR [PREFIX/var]
+  --libdir=DIR            object code libraries in DIR [EPREFIX/lib]
+  --includedir=DIR        C header files in DIR [PREFIX/include]
+  --oldincludedir=DIR     C header files for non-gcc in DIR [/usr/include]
+  --infodir=DIR           info documentation in DIR [PREFIX/info]
+  --mandir=DIR            man documentation in DIR [PREFIX/man]
+  --srcdir=DIR            find the sources in DIR [configure dir or ..]
+  --program-prefix=PREFIX prepend PREFIX to installed program names
+  --program-suffix=SUFFIX append SUFFIX to installed program names
+  --program-transform-name=PROGRAM
+                          run sed PROGRAM on installed program names
+EOF
+    cat << EOF
+Host type:
+  --build=BUILD           configure for building on BUILD [BUILD=HOST]
+  --host=HOST             configure for HOST [guessed]
+  --target=TARGET         configure for TARGET [TARGET=HOST]
+Features and packages:
+  --disable-FEATURE       do not include FEATURE (same as --enable-FEATURE=no)
+  --enable-FEATURE[=ARG]  include FEATURE [ARG=yes]
+  --with-PACKAGE[=ARG]    use PACKAGE [ARG=yes]
+  --without-PACKAGE       do not use PACKAGE (same as --with-PACKAGE=no)
+  --x-includes=DIR        X include files are in DIR
+  --x-libraries=DIR       X library files are in DIR
+EOF
+    if test -n "$ac_help"; then
+      echo "--enable and --with options recognized:$ac_help"
+    fi
+    exit 0 ;;
+
+  -host | --host | --hos | --ho)
+    ac_prev=host ;;
+  -host=* | --host=* | --hos=* | --ho=*)
+    host="$ac_optarg" ;;
+
+  -includedir | --includedir | --includedi | --included | --include \
+  | --includ | --inclu | --incl | --inc)
+    ac_prev=includedir ;;
+  -includedir=* | --includedir=* | --includedi=* | --included=* | --include=* \
+  | --includ=* | --inclu=* | --incl=* | --inc=*)
+    includedir="$ac_optarg" ;;
+
+  -infodir | --infodir | --infodi | --infod | --info | --inf)
+    ac_prev=infodir ;;
+  -infodir=* | --infodir=* | --infodi=* | --infod=* | --info=* | --inf=*)
+    infodir="$ac_optarg" ;;
+
+  -libdir | --libdir | --libdi | --libd)
+    ac_prev=libdir ;;
+  -libdir=* | --libdir=* | --libdi=* | --libd=*)
+    libdir="$ac_optarg" ;;
+
+  -libexecdir | --libexecdir | --libexecdi | --libexecd | --libexec \
+  | --libexe | --libex | --libe)
+    ac_prev=libexecdir ;;
+  -libexecdir=* | --libexecdir=* | --libexecdi=* | --libexecd=* | --libexec=* \
+  | --libexe=* | --libex=* | --libe=*)
+    libexecdir="$ac_optarg" ;;
+
+  -localstatedir | --localstatedir | --localstatedi | --localstated \
+  | --localstate | --localstat | --localsta | --localst \
+  | --locals | --local | --loca | --loc | --lo)
+    ac_prev=localstatedir ;;
+  -localstatedir=* | --localstatedir=* | --localstatedi=* | --localstated=* \
+  | --localstate=* | --localstat=* | --localsta=* | --localst=* \
+  | --locals=* | --local=* | --loca=* | --loc=* | --lo=*)
+    localstatedir="$ac_optarg" ;;
+
+  -mandir | --mandir | --mandi | --mand | --man | --ma | --m)
+    ac_prev=mandir ;;
+  -mandir=* | --mandir=* | --mandi=* | --mand=* | --man=* | --ma=* | --m=*)
+    mandir="$ac_optarg" ;;
+
+  -nfp | --nfp | --nf)
+    # Obsolete; use --without-fp.
+    with_fp=no ;;
+
+  -no-create | --no-create | --no-creat | --no-crea | --no-cre \
+  | --no-cr | --no-c)
+    no_create=yes ;;
+
+  -no-recursion | --no-recursion | --no-recursio | --no-recursi \
+  | --no-recurs | --no-recur | --no-recu | --no-rec | --no-re | --no-r)
+    no_recursion=yes ;;
+
+  -oldincludedir | --oldincludedir | --oldincludedi | --oldincluded \
+  | --oldinclude | --oldinclud | --oldinclu | --oldincl | --oldinc \
+  | --oldin | --oldi | --old | --ol | --o)
+    ac_prev=oldincludedir ;;
+  -oldincludedir=* | --oldincludedir=* | --oldincludedi=* | --oldincluded=* \
+  | --oldinclude=* | --oldinclud=* | --oldinclu=* | --oldincl=* | --oldinc=* \
+  | --oldin=* | --oldi=* | --old=* | --ol=* | --o=*)
+    oldincludedir="$ac_optarg" ;;
+
+  -prefix | --prefix | --prefi | --pref | --pre | --pr | --p)
+    ac_prev=prefix ;;
+  -prefix=* | --prefix=* | --prefi=* | --pref=* | --pre=* | --pr=* | --p=*)
+    prefix="$ac_optarg" ;;
+
+  -program-prefix | --program-prefix | --program-prefi | --program-pref \
+  | --program-pre | --program-pr | --program-p)
+    ac_prev=program_prefix ;;
+  -program-prefix=* | --program-prefix=* | --program-prefi=* \
+  | --program-pref=* | --program-pre=* | --program-pr=* | --program-p=*)
+    program_prefix="$ac_optarg" ;;
+
+  -program-suffix | --program-suffix | --program-suffi | --program-suff \
+  | --program-suf | --program-su | --program-s)
+    ac_prev=program_suffix ;;
+  -program-suffix=* | --program-suffix=* | --program-suffi=* \
+  | --program-suff=* | --program-suf=* | --program-su=* | --program-s=*)
+    program_suffix="$ac_optarg" ;;
+
+  -program-transform-name | --program-transform-name \
+  | --program-transform-nam | --program-transform-na \
+  | --program-transform-n | --program-transform- \
+  | --program-transform | --program-transfor \
+  | --program-transfo | --program-transf \
+  | --program-trans | --program-tran \
+  | --progr-tra | --program-tr | --program-t)
+    ac_prev=program_transform_name ;;
+  -program-transform-name=* | --program-transform-name=* \
+  | --program-transform-nam=* | --program-transform-na=* \
+  | --program-transform-n=* | --program-transform-=* \
+  | --program-transform=* | --program-transfor=* \
+  | --program-transfo=* | --program-transf=* \
+  | --program-trans=* | --program-tran=* \
+  | --progr-tra=* | --program-tr=* | --program-t=*)
+    program_transform_name="$ac_optarg" ;;
+
+  -q | -quiet | --quiet | --quie | --qui | --qu | --q \
+  | -silent | --silent | --silen | --sile | --sil)
+    silent=yes ;;
+
+  -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb)
+    ac_prev=sbindir ;;
+  -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \
+  | --sbi=* | --sb=*)
+    sbindir="$ac_optarg" ;;
+
+  -sharedstatedir | --sharedstatedir | --sharedstatedi \
+  | --sharedstated | --sharedstate | --sharedstat | --sharedsta \
+  | --sharedst | --shareds | --shared | --share | --shar \
+  | --sha | --sh)
+    ac_prev=sharedstatedir ;;
+  -sharedstatedir=* | --sharedstatedir=* | --sharedstatedi=* \
+  | --sharedstated=* | --sharedstate=* | --sharedstat=* | --sharedsta=* \
+  | --sharedst=* | --shareds=* | --shared=* | --share=* | --shar=* \
+  | --sha=* | --sh=*)
+    sharedstatedir="$ac_optarg" ;;
+
+  -site | --site | --sit)
+    ac_prev=site ;;
+  -site=* | --site=* | --sit=*)
+    site="$ac_optarg" ;;
+
+  -srcdir | --srcdir | --srcdi | --srcd | --src | --sr)
+    ac_prev=srcdir ;;
+  -srcdir=* | --srcdir=* | --srcdi=* | --srcd=* | --src=* | --sr=*)
+    srcdir="$ac_optarg" ;;
+
+  -sysconfdir | --sysconfdir | --sysconfdi | --sysconfd | --sysconf \
+  | --syscon | --sysco | --sysc | --sys | --sy)
+    ac_prev=sysconfdir ;;
+  -sysconfdir=* | --sysconfdir=* | --sysconfdi=* | --sysconfd=* | --sysconf=* \
+  | --syscon=* | --sysco=* | --sysc=* | --sys=* | --sy=*)
+    sysconfdir="$ac_optarg" ;;
+
+  -target | --target | --targe | --targ | --tar | --ta | --t)
+    ac_prev=target ;;
+  -target=* | --target=* | --targe=* | --targ=* | --tar=* | --ta=* | --t=*)
+    target="$ac_optarg" ;;
+
+  -v | -verbose | --verbose | --verbos | --verbo | --verb)
+    verbose=yes ;;
+
+  -version | --version | --versio | --versi | --vers)
+    echo "configure generated by autoconf version 2.13"
+    exit 0 ;;
+
+  -with-* | --with-*)
+    ac_package=`echo $ac_option|sed -e 's/-*with-//' -e 's/=.*//'`
+    # Reject names that are not valid shell variable names.
+    if test -n "`echo $ac_package| sed 's/[-_a-zA-Z0-9]//g'`"; then
+      { echo "configure: error: $ac_package: invalid package name" 1>&2; exit 1; }
+    fi
+    ac_package=`echo $ac_package| sed 's/-/_/g'`
+    case "$ac_option" in
+      *=*) ;;
+      *) ac_optarg=yes ;;
+    esac
+    eval "with_${ac_package}='$ac_optarg'" ;;
+
+  -without-* | --without-*)
+    ac_package=`echo $ac_option|sed -e 's/-*without-//'`
+    # Reject names that are not valid shell variable names.
+    if test -n "`echo $ac_package| sed 's/[-a-zA-Z0-9_]//g'`"; then
+      { echo "configure: error: $ac_package: invalid package name" 1>&2; exit 1; }
+    fi
+    ac_package=`echo $ac_package| sed 's/-/_/g'`
+    eval "with_${ac_package}=no" ;;
+
+  --x)
+    # Obsolete; use --with-x.
+    with_x=yes ;;
+
+  -x-includes | --x-includes | --x-include | --x-includ | --x-inclu \
+  | --x-incl | --x-inc | --x-in | --x-i)
+    ac_prev=x_includes ;;
+  -x-includes=* | --x-includes=* | --x-include=* | --x-includ=* | --x-inclu=* \
+  | --x-incl=* | --x-inc=* | --x-in=* | --x-i=*)
+    x_includes="$ac_optarg" ;;
+
+  -x-libraries | --x-libraries | --x-librarie | --x-librari \
+  | --x-librar | --x-libra | --x-libr | --x-lib | --x-li | --x-l)
+    ac_prev=x_libraries ;;
+  -x-libraries=* | --x-libraries=* | --x-librarie=* | --x-librari=* \
+  | --x-librar=* | --x-libra=* | --x-libr=* | --x-lib=* | --x-li=* | --x-l=*)
+    x_libraries="$ac_optarg" ;;
+
+  -*) { echo "configure: error: $ac_option: invalid option; use --help to show usage" 1>&2; exit 1; }
+    ;;
+
+  *)
+    if test -n "`echo $ac_option| sed 's/[-a-z0-9.]//g'`"; then
+      echo "configure: warning: $ac_option: invalid host type" 1>&2
+    fi
+    if test "x$nonopt" != xNONE; then
+      { echo "configure: error: can only configure for one host and one target at a time" 1>&2; exit 1; }
+    fi
+    nonopt="$ac_option"
+    ;;
+
+  esac
+done
+
+if test -n "$ac_prev"; then
+  { echo "configure: error: missing argument to --`echo $ac_prev | sed 's/_/-/g'`" 1>&2; exit 1; }
+fi
+
+trap 'rm -fr conftest* confdefs* core core.* *.core $ac_clean_files; exit 1' 1 2 15
+
+# File descriptor usage:
+# 0 standard input
+# 1 file creation
+# 2 errors and warnings
+# 3 some systems may open it to /dev/tty
+# 4 used on the Kubota Titan
+# 6 checking for... messages and results
+# 5 compiler messages saved in config.log
+if test "$silent" = yes; then
+  exec 6>/dev/null
+else
+  exec 6>&1
+fi
+exec 5>./config.log
+
+echo "\
+This file contains any messages produced by compilers while
+running configure, to aid debugging if configure makes a mistake.
+" 1>&5
+
+# Strip out --no-create and --no-recursion so they do not pile up.
+# Also quote any args containing shell metacharacters.
+ac_configure_args=
+for ac_arg
+do
+  case "$ac_arg" in
+  -no-create | --no-create | --no-creat | --no-crea | --no-cre \
+  | --no-cr | --no-c) ;;
+  -no-recursion | --no-recursion | --no-recursio | --no-recursi \
+  | --no-recurs | --no-recur | --no-recu | --no-rec | --no-re | --no-r) ;;
+  *" "*|*"	"*|*[\[\]\~\#\$\^\&\*\(\)\{\}\\\|\;\<\>\?]*)
+  ac_configure_args="$ac_configure_args '$ac_arg'" ;;
+  *) ac_configure_args="$ac_configure_args $ac_arg" ;;
+  esac
+done
+
+# NLS nuisances.
+# Only set these to C if already set.  These must not be set unconditionally
+# because not all systems understand e.g. LANG=C (notably SCO).
+# Fixing LC_MESSAGES prevents Solaris sh from translating var values in `set'!
+# Non-C LC_CTYPE values break the ctype check.
+if test "${LANG+set}"   = set; then LANG=C;   export LANG;   fi
+if test "${LC_ALL+set}" = set; then LC_ALL=C; export LC_ALL; fi
+if test "${LC_MESSAGES+set}" = set; then LC_MESSAGES=C; export LC_MESSAGES; fi
+if test "${LC_CTYPE+set}"    = set; then LC_CTYPE=C;    export LC_CTYPE;    fi
+
+# confdefs.h avoids OS command line length limits that DEFS can exceed.
+rm -rf conftest* confdefs.h
+# AIX cpp loses on an empty file, so make sure it contains at least a newline.
+echo > confdefs.h
+
+# A filename unique to this package, relative to the directory that
+# configure is in, which we can look for to find out if srcdir is correct.
+ac_unique_file=lib/text_t.h
+
+# Find the source files, if location was not specified.
+if test -z "$srcdir"; then
+  ac_srcdir_defaulted=yes
+  # Try the directory containing this script, then its parent.
+  ac_prog=$0
+  ac_confdir=`echo $ac_prog|sed 's%/[^/][^/]*$%%'`
+  test "x$ac_confdir" = "x$ac_prog" && ac_confdir=.
+  srcdir=$ac_confdir
+  if test ! -r $srcdir/$ac_unique_file; then
+    srcdir=..
+  fi
+else
+  ac_srcdir_defaulted=no
+fi
+if test ! -r $srcdir/$ac_unique_file; then
+  if test "$ac_srcdir_defaulted" = yes; then
+    { echo "configure: error: can not find sources in $ac_confdir or .." 1>&2; exit 1; }
+  else
+    { echo "configure: error: can not find sources in $srcdir" 1>&2; exit 1; }
+  fi
+fi
+srcdir=`echo "${srcdir}" | sed 's%\([^/]\)/*$%\1%'`
+
+# Prefer explicitly selected file to automatically selected ones.
+if test -z "$CONFIG_SITE"; then
+  if test "x$prefix" != xNONE; then
+    CONFIG_SITE="$prefix/share/config.site $prefix/etc/config.site"
+  else
+    CONFIG_SITE="$ac_default_prefix/share/config.site $ac_default_prefix/etc/config.site"
+  fi
+fi
+for ac_site_file in $CONFIG_SITE; do
+  if test -r "$ac_site_file"; then
+    echo "loading site script $ac_site_file"
+    . "$ac_site_file"
+  fi
+done
+
+if test -r "$cache_file"; then
+  echo "loading cache $cache_file"
+  . $cache_file
+else
+  echo "creating cache $cache_file"
+  > $cache_file
+fi
+
+ac_ext=c
+# CFLAGS is not in ac_cpp because -g, -O, etc. are not valid cpp options.
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='${CC-cc} -c $CFLAGS $CPPFLAGS conftest.$ac_ext 1>&5'
+ac_link='${CC-cc} -o conftest${ac_exeext} $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS 1>&5'
+cross_compiling=$ac_cv_prog_cc_cross
+
+ac_exeext=
+ac_objext=o
+if (echo "testing\c"; echo 1,2,3) | grep c >/dev/null; then
+  # Stardent Vistra SVR4 grep lacks -e, says ghazi@caip.rutgers.edu.
+  if (echo -n testing; echo 1,2,3) | sed s/-n/xn/ | grep xn >/dev/null; then
+    ac_n= ac_c='
+' ac_t='	'
+  else
+    ac_n=-n ac_c= ac_t=
+  fi
+else
+  ac_n= ac_c='\c' ac_t=
+fi
+
+
+
+if test "$program_transform_name" = s,x,x,; then
+  program_transform_name=
+else
+  # Double any \ or $.  echo might interpret backslashes.
+  cat <<\EOF_SED > conftestsed
+s,\\,\\\\,g; s,\$,$$,g
+EOF_SED
+  program_transform_name="`echo $program_transform_name|sed -f conftestsed`"
+  rm -f conftestsed
+fi
+test "$program_prefix" != NONE &&
+  program_transform_name="s,^,${program_prefix},; $program_transform_name"
+# Use a double $ so make ignores it.
+test "$program_suffix" != NONE &&
+  program_transform_name="s,\$\$,${program_suffix},; $program_transform_name"
+
+# sed with no file args requires a program.
+test "$program_transform_name" = "" && program_transform_name="s,x,x,"
+
+
+PACKAGE=gsdl
+VERSION=2.1
+cat >> confdefs.h <<EOF
+#define PACKAGE "$PACKAGE"
+EOF
+
+cat >> confdefs.h <<EOF
+#define VERSION "$VERSION"
+EOF
+
+
+
+
+USE_FASTCGI=0
+if test USE_FASTCGI = 1; then
+cat >> confdefs.h <<\EOF
+#define USE_FASTCGI 1
+EOF
+
+
+fi
+
+CFLAGS=-g
+
+
+# Extract the first word of "gcc", so it can be a program name with args.
+set dummy gcc; ac_word=$2
+echo $ac_n "checking for $ac_word""... $ac_c" 1>&6
+echo "configure:574: checking for $ac_word" >&5
+if eval "test \"`echo '$''{'ac_cv_prog_CC'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  if test -n "$CC"; then
+  ac_cv_prog_CC="$CC" # Let the user override the test.
+else
+  IFS="${IFS= 	}"; ac_save_ifs="$IFS"; IFS=":"
+  ac_dummy="$PATH"
+  for ac_dir in $ac_dummy; do
+    test -z "$ac_dir" && ac_dir=.
+    if test -f $ac_dir/$ac_word; then
+      ac_cv_prog_CC="gcc"
+      break
+    fi
+  done
+  IFS="$ac_save_ifs"
+fi
+fi
+CC="$ac_cv_prog_CC"
+if test -n "$CC"; then
+  echo "$ac_t""$CC" 1>&6
+else
+  echo "$ac_t""no" 1>&6
+fi
+
+if test -z "$CC"; then
+  # Extract the first word of "cc", so it can be a program name with args.
+set dummy cc; ac_word=$2
+echo $ac_n "checking for $ac_word""... $ac_c" 1>&6
+echo "configure:604: checking for $ac_word" >&5
+if eval "test \"`echo '$''{'ac_cv_prog_CC'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  if test -n "$CC"; then
+  ac_cv_prog_CC="$CC" # Let the user override the test.
+else
+  IFS="${IFS= 	}"; ac_save_ifs="$IFS"; IFS=":"
+  ac_prog_rejected=no
+  ac_dummy="$PATH"
+  for ac_dir in $ac_dummy; do
+    test -z "$ac_dir" && ac_dir=.
+    if test -f $ac_dir/$ac_word; then
+      if test "$ac_dir/$ac_word" = "/usr/ucb/cc"; then
+        ac_prog_rejected=yes
+	continue
+      fi
+      ac_cv_prog_CC="cc"
+      break
+    fi
+  done
+  IFS="$ac_save_ifs"
+if test $ac_prog_rejected = yes; then
+  # We found a bogon in the path, so make sure we never use it.
+  set dummy $ac_cv_prog_CC
+  shift
+  if test $# -gt 0; then
+    # We chose a different compiler from the bogus one.
+    # However, it has the same basename, so the bogon will be chosen
+    # first if we set CC to just the basename; use the full file name.
+    shift
+    set dummy "$ac_dir/$ac_word" "$@"
+    shift
+    ac_cv_prog_CC="$@"
+  fi
+fi
+fi
+fi
+CC="$ac_cv_prog_CC"
+if test -n "$CC"; then
+  echo "$ac_t""$CC" 1>&6
+else
+  echo "$ac_t""no" 1>&6
+fi
+
+  if test -z "$CC"; then
+    case "`uname -s`" in
+    *win32* | *WIN32*)
+      # Extract the first word of "cl", so it can be a program name with args.
+set dummy cl; ac_word=$2
+echo $ac_n "checking for $ac_word""... $ac_c" 1>&6
+echo "configure:655: checking for $ac_word" >&5
+if eval "test \"`echo '$''{'ac_cv_prog_CC'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  if test -n "$CC"; then
+  ac_cv_prog_CC="$CC" # Let the user override the test.
+else
+  IFS="${IFS= 	}"; ac_save_ifs="$IFS"; IFS=":"
+  ac_dummy="$PATH"
+  for ac_dir in $ac_dummy; do
+    test -z "$ac_dir" && ac_dir=.
+    if test -f $ac_dir/$ac_word; then
+      ac_cv_prog_CC="cl"
+      break
+    fi
+  done
+  IFS="$ac_save_ifs"
+fi
+fi
+CC="$ac_cv_prog_CC"
+if test -n "$CC"; then
+  echo "$ac_t""$CC" 1>&6
+else
+  echo "$ac_t""no" 1>&6
+fi
+ ;;
+    esac
+  fi
+  test -z "$CC" && { echo "configure: error: no acceptable cc found in \$PATH" 1>&2; exit 1; }
+fi
+
+echo $ac_n "checking whether the C compiler ($CC $CFLAGS $LDFLAGS) works""... $ac_c" 1>&6
+echo "configure:687: checking whether the C compiler ($CC $CFLAGS $LDFLAGS) works" >&5
+
+ac_ext=c
+# CFLAGS is not in ac_cpp because -g, -O, etc. are not valid cpp options.
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='${CC-cc} -c $CFLAGS $CPPFLAGS conftest.$ac_ext 1>&5'
+ac_link='${CC-cc} -o conftest${ac_exeext} $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS 1>&5'
+cross_compiling=$ac_cv_prog_cc_cross
+
+cat > conftest.$ac_ext << EOF
+
+#line 698 "configure"
+#include "confdefs.h"
+
+main(){return(0);}
+EOF
+if { (eval echo configure:703: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
+  ac_cv_prog_cc_works=yes
+  # If we can't run a trivial program, we are probably using a cross compiler.
+  if (./conftest; exit) 2>/dev/null; then
+    ac_cv_prog_cc_cross=no
+  else
+    ac_cv_prog_cc_cross=yes
+  fi
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  ac_cv_prog_cc_works=no
+fi
+rm -fr conftest*
+ac_ext=c
+# CFLAGS is not in ac_cpp because -g, -O, etc. are not valid cpp options.
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='${CC-cc} -c $CFLAGS $CPPFLAGS conftest.$ac_ext 1>&5'
+ac_link='${CC-cc} -o conftest${ac_exeext} $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS 1>&5'
+cross_compiling=$ac_cv_prog_cc_cross
+
+echo "$ac_t""$ac_cv_prog_cc_works" 1>&6
+if test $ac_cv_prog_cc_works = no; then
+  { echo "configure: error: installation or configuration problem: C compiler cannot create executables." 1>&2; exit 1; }
+fi
+echo $ac_n "checking whether the C compiler ($CC $CFLAGS $LDFLAGS) is a cross-compiler""... $ac_c" 1>&6
+echo "configure:729: checking whether the C compiler ($CC $CFLAGS $LDFLAGS) is a cross-compiler" >&5
+echo "$ac_t""$ac_cv_prog_cc_cross" 1>&6
+cross_compiling=$ac_cv_prog_cc_cross
+
+echo $ac_n "checking whether we are using GNU C""... $ac_c" 1>&6
+echo "configure:734: checking whether we are using GNU C" >&5
+if eval "test \"`echo '$''{'ac_cv_prog_gcc'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  cat > conftest.c <<EOF
+#ifdef __GNUC__
+  yes;
+#endif
+EOF
+if { ac_try='${CC-cc} -E conftest.c'; { (eval echo configure:743: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; }; } | egrep yes >/dev/null 2>&1; then
+  ac_cv_prog_gcc=yes
+else
+  ac_cv_prog_gcc=no
+fi
+fi
+
+echo "$ac_t""$ac_cv_prog_gcc" 1>&6
+
+if test $ac_cv_prog_gcc = yes; then
+  GCC=yes
+else
+  GCC=
+fi
+
+ac_test_CFLAGS="${CFLAGS+set}"
+ac_save_CFLAGS="$CFLAGS"
+CFLAGS=
+echo $ac_n "checking whether ${CC-cc} accepts -g""... $ac_c" 1>&6
+echo "configure:762: checking whether ${CC-cc} accepts -g" >&5
+if eval "test \"`echo '$''{'ac_cv_prog_cc_g'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  echo 'void f(){}' > conftest.c
+if test -z "`${CC-cc} -g -c conftest.c 2>&1`"; then
+  ac_cv_prog_cc_g=yes
+else
+  ac_cv_prog_cc_g=no
+fi
+rm -f conftest*
+
+fi
+
+echo "$ac_t""$ac_cv_prog_cc_g" 1>&6
+if test "$ac_test_CFLAGS" = set; then
+  CFLAGS="$ac_save_CFLAGS"
+elif test $ac_cv_prog_cc_g = yes; then
+  if test "$GCC" = yes; then
+    CFLAGS="-g -O2"
+  else
+    CFLAGS="-g"
+  fi
+else
+  if test "$GCC" = yes; then
+    CFLAGS="-O2"
+  else
+    CFLAGS=
+  fi
+fi
+
+for ac_prog in $CCC c++ g++ gcc CC cxx cc++ cl
+do
+# Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+echo $ac_n "checking for $ac_word""... $ac_c" 1>&6
+echo "configure:798: checking for $ac_word" >&5
+if eval "test \"`echo '$''{'ac_cv_prog_CXX'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  if test -n "$CXX"; then
+  ac_cv_prog_CXX="$CXX" # Let the user override the test.
+else
+  IFS="${IFS= 	}"; ac_save_ifs="$IFS"; IFS=":"
+  ac_dummy="$PATH"
+  for ac_dir in $ac_dummy; do
+    test -z "$ac_dir" && ac_dir=.
+    if test -f $ac_dir/$ac_word; then
+      ac_cv_prog_CXX="$ac_prog"
+      break
+    fi
+  done
+  IFS="$ac_save_ifs"
+fi
+fi
+CXX="$ac_cv_prog_CXX"
+if test -n "$CXX"; then
+  echo "$ac_t""$CXX" 1>&6
+else
+  echo "$ac_t""no" 1>&6
+fi
+
+test -n "$CXX" && break
+done
+test -n "$CXX" || CXX="gcc"
+
+
+echo $ac_n "checking whether the C++ compiler ($CXX $CXXFLAGS $LDFLAGS) works""... $ac_c" 1>&6
+echo "configure:830: checking whether the C++ compiler ($CXX $CXXFLAGS $LDFLAGS) works" >&5
+
+ac_ext=C
+# CXXFLAGS is not in ac_cpp because -g, -O, etc. are not valid cpp options.
+ac_cpp='$CXXCPP $CPPFLAGS'
+ac_compile='${CXX-g++} -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext 1>&5'
+ac_link='${CXX-g++} -o conftest${ac_exeext} $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS 1>&5'
+cross_compiling=$ac_cv_prog_cxx_cross
+
+cat > conftest.$ac_ext << EOF
+
+#line 841 "configure"
+#include "confdefs.h"
+
+int main(){return(0);}
+EOF
+if { (eval echo configure:846: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
+  ac_cv_prog_cxx_works=yes
+  # If we can't run a trivial program, we are probably using a cross compiler.
+  if (./conftest; exit) 2>/dev/null; then
+    ac_cv_prog_cxx_cross=no
+  else
+    ac_cv_prog_cxx_cross=yes
+  fi
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  ac_cv_prog_cxx_works=no
+fi
+rm -fr conftest*
+ac_ext=c
+# CFLAGS is not in ac_cpp because -g, -O, etc. are not valid cpp options.
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='${CC-cc} -c $CFLAGS $CPPFLAGS conftest.$ac_ext 1>&5'
+ac_link='${CC-cc} -o conftest${ac_exeext} $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS 1>&5'
+cross_compiling=$ac_cv_prog_cc_cross
+
+echo "$ac_t""$ac_cv_prog_cxx_works" 1>&6
+if test $ac_cv_prog_cxx_works = no; then
+  { echo "configure: error: installation or configuration problem: C++ compiler cannot create executables." 1>&2; exit 1; }
+fi
+echo $ac_n "checking whether the C++ compiler ($CXX $CXXFLAGS $LDFLAGS) is a cross-compiler""... $ac_c" 1>&6
+echo "configure:872: checking whether the C++ compiler ($CXX $CXXFLAGS $LDFLAGS) is a cross-compiler" >&5
+echo "$ac_t""$ac_cv_prog_cxx_cross" 1>&6
+cross_compiling=$ac_cv_prog_cxx_cross
+
+echo $ac_n "checking whether we are using GNU C++""... $ac_c" 1>&6
+echo "configure:877: checking whether we are using GNU C++" >&5
+if eval "test \"`echo '$''{'ac_cv_prog_gxx'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  cat > conftest.C <<EOF
+#ifdef __GNUC__
+  yes;
+#endif
+EOF
+if { ac_try='${CXX-g++} -E conftest.C'; { (eval echo configure:886: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; }; } | egrep yes >/dev/null 2>&1; then
+  ac_cv_prog_gxx=yes
+else
+  ac_cv_prog_gxx=no
+fi
+fi
+
+echo "$ac_t""$ac_cv_prog_gxx" 1>&6
+
+if test $ac_cv_prog_gxx = yes; then
+  GXX=yes
+else
+  GXX=
+fi
+
+ac_test_CXXFLAGS="${CXXFLAGS+set}"
+ac_save_CXXFLAGS="$CXXFLAGS"
+CXXFLAGS=
+echo $ac_n "checking whether ${CXX-g++} accepts -g""... $ac_c" 1>&6
+echo "configure:905: checking whether ${CXX-g++} accepts -g" >&5
+if eval "test \"`echo '$''{'ac_cv_prog_cxx_g'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  echo 'void f(){}' > conftest.cc
+if test -z "`${CXX-g++} -g -c conftest.cc 2>&1`"; then
+  ac_cv_prog_cxx_g=yes
+else
+  ac_cv_prog_cxx_g=no
+fi
+rm -f conftest*
+
+fi
+
+echo "$ac_t""$ac_cv_prog_cxx_g" 1>&6
+if test "$ac_test_CXXFLAGS" = set; then
+  CXXFLAGS="$ac_save_CXXFLAGS"
+elif test $ac_cv_prog_cxx_g = yes; then
+  if test "$GXX" = yes; then
+    CXXFLAGS="-g -O2"
+  else
+    CXXFLAGS="-g"
+  fi
+else
+  if test "$GXX" = yes; then
+    CXXFLAGS="-O2"
+  else
+    CXXFLAGS=
+  fi
+fi
+
+for ac_prog in mawk gawk nawk awk
+do
+# Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+echo $ac_n "checking for $ac_word""... $ac_c" 1>&6
+echo "configure:941: checking for $ac_word" >&5
+if eval "test \"`echo '$''{'ac_cv_prog_AWK'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  if test -n "$AWK"; then
+  ac_cv_prog_AWK="$AWK" # Let the user override the test.
+else
+  IFS="${IFS= 	}"; ac_save_ifs="$IFS"; IFS=":"
+  ac_dummy="$PATH"
+  for ac_dir in $ac_dummy; do
+    test -z "$ac_dir" && ac_dir=.
+    if test -f $ac_dir/$ac_word; then
+      ac_cv_prog_AWK="$ac_prog"
+      break
+    fi
+  done
+  IFS="$ac_save_ifs"
+fi
+fi
+AWK="$ac_cv_prog_AWK"
+if test -n "$AWK"; then
+  echo "$ac_t""$AWK" 1>&6
+else
+  echo "$ac_t""no" 1>&6
+fi
+
+test -n "$AWK" && break
+done
+
+for ac_prog in 'bison -y' byacc
+do
+# Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+echo $ac_n "checking for $ac_word""... $ac_c" 1>&6
+echo "configure:975: checking for $ac_word" >&5
+if eval "test \"`echo '$''{'ac_cv_prog_YACC'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  if test -n "$YACC"; then
+  ac_cv_prog_YACC="$YACC" # Let the user override the test.
+else
+  IFS="${IFS= 	}"; ac_save_ifs="$IFS"; IFS=":"
+  ac_dummy="$PATH"
+  for ac_dir in $ac_dummy; do
+    test -z "$ac_dir" && ac_dir=.
+    if test -f $ac_dir/$ac_word; then
+      ac_cv_prog_YACC="$ac_prog"
+      break
+    fi
+  done
+  IFS="$ac_save_ifs"
+fi
+fi
+YACC="$ac_cv_prog_YACC"
+if test -n "$YACC"; then
+  echo "$ac_t""$YACC" 1>&6
+else
+  echo "$ac_t""no" 1>&6
+fi
+
+test -n "$YACC" && break
+done
+test -n "$YACC" || YACC="yacc"
+
+echo $ac_n "checking whether ln -s works""... $ac_c" 1>&6
+echo "configure:1006: checking whether ln -s works" >&5
+if eval "test \"`echo '$''{'ac_cv_prog_LN_S'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  rm -f conftestdata
+if ln -s X conftestdata 2>/dev/null
+then
+  rm -f conftestdata
+  ac_cv_prog_LN_S="ln -s"
+else
+  ac_cv_prog_LN_S=ln
+fi
+fi
+LN_S="$ac_cv_prog_LN_S"
+if test "$ac_cv_prog_LN_S" = "ln -s"; then
+  echo "$ac_t""yes" 1>&6
+else
+  echo "$ac_t""no" 1>&6
+fi
+
+echo $ac_n "checking whether ${MAKE-make} sets \${MAKE}""... $ac_c" 1>&6
+echo "configure:1027: checking whether ${MAKE-make} sets \${MAKE}" >&5
+set dummy ${MAKE-make}; ac_make=`echo "$2" | sed 'y%./+-%__p_%'`
+if eval "test \"`echo '$''{'ac_cv_prog_make_${ac_make}_set'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  cat > conftestmake <<\EOF
+all:
+	@echo 'ac_maketemp="${MAKE}"'
+EOF
+# GNU make sometimes prints "make[1]: Entering...", which would confuse us.
+eval `${MAKE-make} -f conftestmake 2>/dev/null | grep temp=`
+if test -n "$ac_maketemp"; then
+  eval ac_cv_prog_make_${ac_make}_set=yes
+else
+  eval ac_cv_prog_make_${ac_make}_set=no
+fi
+rm -f conftestmake
+fi
+if eval "test \"`echo '$ac_cv_prog_make_'${ac_make}_set`\" = yes"; then
+  echo "$ac_t""yes" 1>&6
+  SET_MAKE=
+else
+  echo "$ac_t""no" 1>&6
+  SET_MAKE="MAKE=${MAKE-make}"
+fi
+
+# Extract the first word of "ranlib", so it can be a program name with args.
+set dummy ranlib; ac_word=$2
+echo $ac_n "checking for $ac_word""... $ac_c" 1>&6
+echo "configure:1056: checking for $ac_word" >&5
+if eval "test \"`echo '$''{'ac_cv_prog_RANLIB'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  if test -n "$RANLIB"; then
+  ac_cv_prog_RANLIB="$RANLIB" # Let the user override the test.
+else
+  IFS="${IFS= 	}"; ac_save_ifs="$IFS"; IFS=":"
+  ac_dummy="$PATH"
+  for ac_dir in $ac_dummy; do
+    test -z "$ac_dir" && ac_dir=.
+    if test -f $ac_dir/$ac_word; then
+      ac_cv_prog_RANLIB="ranlib"
+      break
+    fi
+  done
+  IFS="$ac_save_ifs"
+  test -z "$ac_cv_prog_RANLIB" && ac_cv_prog_RANLIB=":"
+fi
+fi
+RANLIB="$ac_cv_prog_RANLIB"
+if test -n "$RANLIB"; then
+  echo "$ac_t""$RANLIB" 1>&6
+else
+  echo "$ac_t""no" 1>&6
+fi
+
+
+
+echo $ac_n "checking how to run the C preprocessor""... $ac_c" 1>&6
+echo "configure:1086: checking how to run the C preprocessor" >&5
+# On Suns, sometimes $CPP names a directory.
+if test -n "$CPP" && test -d "$CPP"; then
+  CPP=
+fi
+if test -z "$CPP"; then
+if eval "test \"`echo '$''{'ac_cv_prog_CPP'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+    # This must be in double quotes, not single quotes, because CPP may get
+  # substituted into the Makefile and "${CC-cc}" will confuse make.
+  CPP="${CC-cc} -E"
+  # On the NeXT, cc -E runs the code through the compiler's parser,
+  # not just through cpp.
+  cat > conftest.$ac_ext <<EOF
+#line 1101 "configure"
+#include "confdefs.h"
+#include <assert.h>
+Syntax Error
+EOF
+ac_try="$ac_cpp conftest.$ac_ext >/dev/null 2>conftest.out"
+{ (eval echo configure:1107: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; }
+ac_err=`grep -v '^ *+' conftest.out | grep -v "^conftest.${ac_ext}\$"`
+if test -z "$ac_err"; then
+  :
+else
+  echo "$ac_err" >&5
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  CPP="${CC-cc} -E -traditional-cpp"
+  cat > conftest.$ac_ext <<EOF
+#line 1118 "configure"
+#include "confdefs.h"
+#include <assert.h>
+Syntax Error
+EOF
+ac_try="$ac_cpp conftest.$ac_ext >/dev/null 2>conftest.out"
+{ (eval echo configure:1124: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; }
+ac_err=`grep -v '^ *+' conftest.out | grep -v "^conftest.${ac_ext}\$"`
+if test -z "$ac_err"; then
+  :
+else
+  echo "$ac_err" >&5
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  CPP="${CC-cc} -nologo -E"
+  cat > conftest.$ac_ext <<EOF
+#line 1135 "configure"
+#include "confdefs.h"
+#include <assert.h>
+Syntax Error
+EOF
+ac_try="$ac_cpp conftest.$ac_ext >/dev/null 2>conftest.out"
+{ (eval echo configure:1141: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; }
+ac_err=`grep -v '^ *+' conftest.out | grep -v "^conftest.${ac_ext}\$"`
+if test -z "$ac_err"; then
+  :
+else
+  echo "$ac_err" >&5
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  CPP=/lib/cpp
+fi
+rm -f conftest*
+fi
+rm -f conftest*
+fi
+rm -f conftest*
+  ac_cv_prog_CPP="$CPP"
+fi
+  CPP="$ac_cv_prog_CPP"
+else
+  ac_cv_prog_CPP="$CPP"
+fi
+echo "$ac_t""$CPP" 1>&6
+
+echo $ac_n "checking for AIX""... $ac_c" 1>&6
+echo "configure:1166: checking for AIX" >&5
+cat > conftest.$ac_ext <<EOF
+#line 1168 "configure"
+#include "confdefs.h"
+#ifdef _AIX
+  yes
+#endif
+
+EOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  egrep "yes" >/dev/null 2>&1; then
+  rm -rf conftest*
+  echo "$ac_t""yes" 1>&6; cat >> confdefs.h <<\EOF
+#define _ALL_SOURCE 1
+EOF
+
+else
+  rm -rf conftest*
+  echo "$ac_t""no" 1>&6
+fi
+rm -f conftest*
+
+
+echo $ac_n "checking for POSIXized ISC""... $ac_c" 1>&6
+echo "configure:1190: checking for POSIXized ISC" >&5
+if test -d /etc/conf/kconfig.d &&
+  grep _POSIX_VERSION /usr/include/sys/unistd.h >/dev/null 2>&1
+then
+  echo "$ac_t""yes" 1>&6
+  ISC=yes # If later tests want to check for ISC.
+  cat >> confdefs.h <<\EOF
+#define _POSIX_SOURCE 1
+EOF
+
+  if test "$GCC" = yes; then
+    CC="$CC -posix"
+  else
+    CC="$CC -Xp"
+  fi
+else
+  echo "$ac_t""no" 1>&6
+  ISC=
+fi
+
+ac_safe=`echo "minix/config.h" | sed 'y%./+-%__p_%'`
+echo $ac_n "checking for minix/config.h""... $ac_c" 1>&6
+echo "configure:1212: checking for minix/config.h" >&5
+if eval "test \"`echo '$''{'ac_cv_header_$ac_safe'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  cat > conftest.$ac_ext <<EOF
+#line 1217 "configure"
+#include "confdefs.h"
+#include <minix/config.h>
+EOF
+ac_try="$ac_cpp conftest.$ac_ext >/dev/null 2>conftest.out"
+{ (eval echo configure:1222: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; }
+ac_err=`grep -v '^ *+' conftest.out | grep -v "^conftest.${ac_ext}\$"`
+if test -z "$ac_err"; then
+  rm -rf conftest*
+  eval "ac_cv_header_$ac_safe=yes"
+else
+  echo "$ac_err" >&5
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  eval "ac_cv_header_$ac_safe=no"
+fi
+rm -f conftest*
+fi
+if eval "test \"`echo '$ac_cv_header_'$ac_safe`\" = yes"; then
+  echo "$ac_t""yes" 1>&6
+  MINIX=yes
+else
+  echo "$ac_t""no" 1>&6
+MINIX=
+fi
+
+if test "$MINIX" = yes; then
+  cat >> confdefs.h <<\EOF
+#define _POSIX_SOURCE 1
+EOF
+
+  cat >> confdefs.h <<\EOF
+#define _POSIX_1_SOURCE 2
+EOF
+
+  cat >> confdefs.h <<\EOF
+#define _MINIX 1
+EOF
+
+fi
+
+fp_C_PROTOTYPES
+echo $ac_n "checking for working const""... $ac_c" 1>&6
+echo "configure:1261: checking for working const" >&5
+if eval "test \"`echo '$''{'ac_cv_c_const'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  cat > conftest.$ac_ext <<EOF
+#line 1266 "configure"
+#include "confdefs.h"
+
+int main() {
+
+/* Ultrix mips cc rejects this.  */
+typedef int charset[2]; const charset x;
+/* SunOS 4.1.1 cc rejects this.  */
+char const *const *ccp;
+char **p;
+/* NEC SVR4.0.2 mips cc rejects this.  */
+struct point {int x, y;};
+static struct point const zero = {0,0};
+/* AIX XL C 1.02.0.0 rejects this.
+   It does not let you subtract one const X* pointer from another in an arm
+   of an if-expression whose if-part is not a constant expression */
+const char *g = "string";
+ccp = &g + (g ? g-g : 0);
+/* HPUX 7.0 cc rejects these. */
+++ccp;
+p = (char**) ccp;
+ccp = (char const *const *) p;
+{ /* SCO 3.2v4 cc rejects this.  */
+  char *t;
+  char const *s = 0 ? (char *) 0 : (char const *) 0;
+
+  *t++ = 0;
+}
+{ /* Someone thinks the Sun supposedly-ANSI compiler will reject this.  */
+  int x[] = {25, 17};
+  const int *foo = &x[0];
+  ++foo;
+}
+{ /* Sun SC1.0 ANSI compiler rejects this -- but not the above. */
+  typedef const int *iptr;
+  iptr p = 0;
+  ++p;
+}
+{ /* AIX XL C 1.02.0.0 rejects this saying
+     "k.c", line 2.27: 1506-025 (S) Operand must be a modifiable lvalue. */
+  struct s { int j; const int *ap[3]; };
+  struct s *b; b->j = 5;
+}
+{ /* ULTRIX-32 V3.1 (Rev 9) vcc rejects this */
+  const int foo = 10;
+}
+
+; return 0; }
+EOF
+if { (eval echo configure:1315: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then
+  rm -rf conftest*
+  ac_cv_c_const=yes
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  ac_cv_c_const=no
+fi
+rm -f conftest*
+fi
+
+echo "$ac_t""$ac_cv_c_const" 1>&6
+if test $ac_cv_c_const = no; then
+  cat >> confdefs.h <<\EOF
+#define const 
+EOF
+
+fi
+
+echo $ac_n "checking for ANSI C header files""... $ac_c" 1>&6
+echo "configure:1336: checking for ANSI C header files" >&5
+if eval "test \"`echo '$''{'ac_cv_header_stdc'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  cat > conftest.$ac_ext <<EOF
+#line 1341 "configure"
+#include "confdefs.h"
+#include <stdlib.h>
+#include <stdarg.h>
+#include <string.h>
+#include <float.h>
+EOF
+ac_try="$ac_cpp conftest.$ac_ext >/dev/null 2>conftest.out"
+{ (eval echo configure:1349: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; }
+ac_err=`grep -v '^ *+' conftest.out | grep -v "^conftest.${ac_ext}\$"`
+if test -z "$ac_err"; then
+  rm -rf conftest*
+  ac_cv_header_stdc=yes
+else
+  echo "$ac_err" >&5
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  ac_cv_header_stdc=no
+fi
+rm -f conftest*
+
+if test $ac_cv_header_stdc = yes; then
+  # SunOS 4.x string.h does not declare mem*, contrary to ANSI.
+cat > conftest.$ac_ext <<EOF
+#line 1366 "configure"
+#include "confdefs.h"
+#include <string.h>
+EOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  egrep "memchr" >/dev/null 2>&1; then
+  :
+else
+  rm -rf conftest*
+  ac_cv_header_stdc=no
+fi
+rm -f conftest*
+
+fi
+
+if test $ac_cv_header_stdc = yes; then
+  # ISC 2.0.2 stdlib.h does not declare free, contrary to ANSI.
+cat > conftest.$ac_ext <<EOF
+#line 1384 "configure"
+#include "confdefs.h"
+#include <stdlib.h>
+EOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  egrep "free" >/dev/null 2>&1; then
+  :
+else
+  rm -rf conftest*
+  ac_cv_header_stdc=no
+fi
+rm -f conftest*
+
+fi
+
+if test $ac_cv_header_stdc = yes; then
+  # /bin/cc in Irix-4.0.5 gets non-ANSI ctype macros unless using -ansi.
+if test "$cross_compiling" = yes; then
+  :
+else
+  cat > conftest.$ac_ext <<EOF
+#line 1405 "configure"
+#include "confdefs.h"
+#include <ctype.h>
+#define ISLOWER(c) ('a' <= (c) && (c) <= 'z')
+#define TOUPPER(c) (ISLOWER(c) ? 'A' + ((c) - 'a') : (c))
+#define XOR(e, f) (((e) && !(f)) || (!(e) && (f)))
+int main () { int i; for (i = 0; i < 256; i++)
+if (XOR (islower (i), ISLOWER (i)) || toupper (i) != TOUPPER (i)) exit(2);
+exit (0); }
+
+EOF
+if { (eval echo configure:1416: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext} && (./conftest; exit) 2>/dev/null
+then
+  :
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -fr conftest*
+  ac_cv_header_stdc=no
+fi
+rm -fr conftest*
+fi
+
+fi
+fi
+
+echo "$ac_t""$ac_cv_header_stdc" 1>&6
+if test $ac_cv_header_stdc = yes; then
+  cat >> confdefs.h <<\EOF
+#define STDC_HEADERS 1
+EOF
+
+fi
+
+echo $ac_n "checking for off_t""... $ac_c" 1>&6
+echo "configure:1440: checking for off_t" >&5
+if eval "test \"`echo '$''{'ac_cv_type_off_t'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  cat > conftest.$ac_ext <<EOF
+#line 1445 "configure"
+#include "confdefs.h"
+#include <sys/types.h>
+#if STDC_HEADERS
+#include <stdlib.h>
+#include <stddef.h>
+#endif
+EOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  egrep "(^|[^a-zA-Z_0-9])off_t[^a-zA-Z_0-9]" >/dev/null 2>&1; then
+  rm -rf conftest*
+  ac_cv_type_off_t=yes
+else
+  rm -rf conftest*
+  ac_cv_type_off_t=no
+fi
+rm -f conftest*
+
+fi
+echo "$ac_t""$ac_cv_type_off_t" 1>&6
+if test $ac_cv_type_off_t = no; then
+  cat >> confdefs.h <<\EOF
+#define off_t long
+EOF
+
+fi
+
+echo $ac_n "checking for size_t""... $ac_c" 1>&6
+echo "configure:1473: checking for size_t" >&5
+if eval "test \"`echo '$''{'ac_cv_type_size_t'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  cat > conftest.$ac_ext <<EOF
+#line 1478 "configure"
+#include "confdefs.h"
+#include <sys/types.h>
+#if STDC_HEADERS
+#include <stdlib.h>
+#include <stddef.h>
+#endif
+EOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  egrep "(^|[^a-zA-Z_0-9])size_t[^a-zA-Z_0-9]" >/dev/null 2>&1; then
+  rm -rf conftest*
+  ac_cv_type_size_t=yes
+else
+  rm -rf conftest*
+  ac_cv_type_size_t=no
+fi
+rm -f conftest*
+
+fi
+echo "$ac_t""$ac_cv_type_size_t" 1>&6
+if test $ac_cv_type_size_t = no; then
+  cat >> confdefs.h <<\EOF
+#define size_t unsigned
+EOF
+
+fi
+
+echo $ac_n "checking whether time.h and sys/time.h may both be included""... $ac_c" 1>&6
+echo "configure:1506: checking whether time.h and sys/time.h may both be included" >&5
+if eval "test \"`echo '$''{'ac_cv_header_time'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  cat > conftest.$ac_ext <<EOF
+#line 1511 "configure"
+#include "confdefs.h"
+#include <sys/types.h>
+#include <sys/time.h>
+#include <time.h>
+int main() {
+struct tm *tp;
+; return 0; }
+EOF
+if { (eval echo configure:1520: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then
+  rm -rf conftest*
+  ac_cv_header_time=yes
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  ac_cv_header_time=no
+fi
+rm -f conftest*
+fi
+
+echo "$ac_t""$ac_cv_header_time" 1>&6
+if test $ac_cv_header_time = yes; then
+  cat >> confdefs.h <<\EOF
+#define TIME_WITH_SYS_TIME 1
+EOF
+
+fi
+
+echo $ac_n "checking whether struct tm is in sys/time.h or time.h""... $ac_c" 1>&6
+echo "configure:1541: checking whether struct tm is in sys/time.h or time.h" >&5
+if eval "test \"`echo '$''{'ac_cv_struct_tm'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  cat > conftest.$ac_ext <<EOF
+#line 1546 "configure"
+#include "confdefs.h"
+#include <sys/types.h>
+#include <time.h>
+int main() {
+struct tm *tp; tp->tm_sec;
+; return 0; }
+EOF
+if { (eval echo configure:1554: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then
+  rm -rf conftest*
+  ac_cv_struct_tm=time.h
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  ac_cv_struct_tm=sys/time.h
+fi
+rm -f conftest*
+fi
+
+echo "$ac_t""$ac_cv_struct_tm" 1>&6
+if test $ac_cv_struct_tm = sys/time.h; then
+  cat >> confdefs.h <<\EOF
+#define TM_IN_SYS_TIME 1
+EOF
+
+fi
+
+
+if test "$ac_cv_prog_cc_stdc" = '-Xc'; then
+cat > conftest.$ac_ext <<EOF
+#line 1577 "configure"
+#include "confdefs.h"
+#include <stdio.h>
+
+int main() {
+ FILE *rubbish = popen("yes", "r"); 
+; return 0; }
+EOF
+if { (eval echo configure:1585: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then
+  :
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  CC="`echo $CC | sed 's/-Xc/-Xa/'`"    ac_cv_prog_cc_stdc='-Xa'
+fi
+rm -f conftest*
+fi
+
+
+echo $ac_n "checking for main in -lg""... $ac_c" 1>&6
+echo "configure:1598: checking for main in -lg" >&5
+ac_lib_var=`echo g'_'main | sed 'y%./+-%__p_%'`
+if eval "test \"`echo '$''{'ac_cv_lib_$ac_lib_var'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  ac_save_LIBS="$LIBS"
+LIBS="-lg  $LIBS"
+cat > conftest.$ac_ext <<EOF
+#line 1606 "configure"
+#include "confdefs.h"
+
+int main() {
+main()
+; return 0; }
+EOF
+if { (eval echo configure:1613: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
+  rm -rf conftest*
+  eval "ac_cv_lib_$ac_lib_var=yes"
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  eval "ac_cv_lib_$ac_lib_var=no"
+fi
+rm -f conftest*
+LIBS="$ac_save_LIBS"
+
+fi
+if eval "test \"`echo '$ac_cv_lib_'$ac_lib_var`\" = yes"; then
+  echo "$ac_t""yes" 1>&6
+    ac_tr_lib=HAVE_LIB`echo g | sed -e 's/[^a-zA-Z0-9_]/_/g' \
+    -e 'y/abcdefghijklmnopqrstuvwxyz/ABCDEFGHIJKLMNOPQRSTUVWXYZ/'`
+  cat >> confdefs.h <<EOF
+#define $ac_tr_lib 1
+EOF
+
+  LIBS="-lg $LIBS"
+
+else
+  echo "$ac_t""no" 1>&6
+fi
+
+echo $ac_n "checking for main in -lgdbm""... $ac_c" 1>&6
+echo "configure:1641: checking for main in -lgdbm" >&5
+ac_lib_var=`echo gdbm'_'main | sed 'y%./+-%__p_%'`
+if eval "test \"`echo '$''{'ac_cv_lib_$ac_lib_var'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  ac_save_LIBS="$LIBS"
+LIBS="-lgdbm  $LIBS"
+cat > conftest.$ac_ext <<EOF
+#line 1649 "configure"
+#include "confdefs.h"
+
+int main() {
+main()
+; return 0; }
+EOF
+if { (eval echo configure:1656: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
+  rm -rf conftest*
+  eval "ac_cv_lib_$ac_lib_var=yes"
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  eval "ac_cv_lib_$ac_lib_var=no"
+fi
+rm -f conftest*
+LIBS="$ac_save_LIBS"
+
+fi
+if eval "test \"`echo '$ac_cv_lib_'$ac_lib_var`\" = yes"; then
+  echo "$ac_t""yes" 1>&6
+    ac_tr_lib=HAVE_LIB`echo gdbm | sed -e 's/[^a-zA-Z0-9_]/_/g' \
+    -e 'y/abcdefghijklmnopqrstuvwxyz/ABCDEFGHIJKLMNOPQRSTUVWXYZ/'`
+  cat >> confdefs.h <<EOF
+#define $ac_tr_lib 1
+EOF
+
+  LIBS="-lgdbm $LIBS"
+
+else
+  echo "$ac_t""no" 1>&6
+fi
+
+echo $ac_n "checking for main in -lm""... $ac_c" 1>&6
+echo "configure:1684: checking for main in -lm" >&5
+ac_lib_var=`echo m'_'main | sed 'y%./+-%__p_%'`
+if eval "test \"`echo '$''{'ac_cv_lib_$ac_lib_var'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  ac_save_LIBS="$LIBS"
+LIBS="-lm  $LIBS"
+cat > conftest.$ac_ext <<EOF
+#line 1692 "configure"
+#include "confdefs.h"
+
+int main() {
+main()
+; return 0; }
+EOF
+if { (eval echo configure:1699: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
+  rm -rf conftest*
+  eval "ac_cv_lib_$ac_lib_var=yes"
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  eval "ac_cv_lib_$ac_lib_var=no"
+fi
+rm -f conftest*
+LIBS="$ac_save_LIBS"
+
+fi
+if eval "test \"`echo '$ac_cv_lib_'$ac_lib_var`\" = yes"; then
+  echo "$ac_t""yes" 1>&6
+    ac_tr_lib=HAVE_LIB`echo m | sed -e 's/[^a-zA-Z0-9_]/_/g' \
+    -e 'y/abcdefghijklmnopqrstuvwxyz/ABCDEFGHIJKLMNOPQRSTUVWXYZ/'`
+  cat >> confdefs.h <<EOF
+#define $ac_tr_lib 1
+EOF
+
+  LIBS="-lm $LIBS"
+
+else
+  echo "$ac_t""no" 1>&6
+fi
+
+echo $ac_n "checking for main in -lstdc""... $ac_c" 1>&6
+echo "configure:1727: checking for main in -lstdc" >&5
+ac_lib_var=`echo stdc'_'main | sed 'y%./+-%__p_%'`
+if eval "test \"`echo '$''{'ac_cv_lib_$ac_lib_var'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  ac_save_LIBS="$LIBS"
+LIBS="-lstdc  $LIBS"
+cat > conftest.$ac_ext <<EOF
+#line 1735 "configure"
+#include "confdefs.h"
+
+int main() {
+main()
+; return 0; }
+EOF
+if { (eval echo configure:1742: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
+  rm -rf conftest*
+  eval "ac_cv_lib_$ac_lib_var=yes"
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  eval "ac_cv_lib_$ac_lib_var=no"
+fi
+rm -f conftest*
+LIBS="$ac_save_LIBS"
+
+fi
+if eval "test \"`echo '$ac_cv_lib_'$ac_lib_var`\" = yes"; then
+  echo "$ac_t""yes" 1>&6
+    ac_tr_lib=HAVE_LIB`echo stdc | sed -e 's/[^a-zA-Z0-9_]/_/g' \
+    -e 'y/abcdefghijklmnopqrstuvwxyz/ABCDEFGHIJKLMNOPQRSTUVWXYZ/'`
+  cat >> confdefs.h <<EOF
+#define $ac_tr_lib 1
+EOF
+
+  LIBS="-lstdc $LIBS"
+
+else
+  echo "$ac_t""no" 1>&6
+fi
+
+
+ac_header_dirent=no
+for ac_hdr in dirent.h sys/ndir.h sys/dir.h ndir.h
+do
+ac_safe=`echo "$ac_hdr" | sed 'y%./+-%__p_%'`
+echo $ac_n "checking for $ac_hdr that defines DIR""... $ac_c" 1>&6
+echo "configure:1775: checking for $ac_hdr that defines DIR" >&5
+if eval "test \"`echo '$''{'ac_cv_header_dirent_$ac_safe'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  cat > conftest.$ac_ext <<EOF
+#line 1780 "configure"
+#include "confdefs.h"
+#include <sys/types.h>
+#include <$ac_hdr>
+int main() {
+DIR *dirp = 0;
+; return 0; }
+EOF
+if { (eval echo configure:1788: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then
+  rm -rf conftest*
+  eval "ac_cv_header_dirent_$ac_safe=yes"
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  eval "ac_cv_header_dirent_$ac_safe=no"
+fi
+rm -f conftest*
+fi
+if eval "test \"`echo '$ac_cv_header_dirent_'$ac_safe`\" = yes"; then
+  echo "$ac_t""yes" 1>&6
+    ac_tr_hdr=HAVE_`echo $ac_hdr | sed 'y%abcdefghijklmnopqrstuvwxyz./-%ABCDEFGHIJKLMNOPQRSTUVWXYZ___%'`
+  cat >> confdefs.h <<EOF
+#define $ac_tr_hdr 1
+EOF
+ ac_header_dirent=$ac_hdr; break
+else
+  echo "$ac_t""no" 1>&6
+fi
+done
+# Two versions of opendir et al. are in -ldir and -lx on SCO Xenix.
+if test $ac_header_dirent = dirent.h; then
+echo $ac_n "checking for opendir in -ldir""... $ac_c" 1>&6
+echo "configure:1813: checking for opendir in -ldir" >&5
+ac_lib_var=`echo dir'_'opendir | sed 'y%./+-%__p_%'`
+if eval "test \"`echo '$''{'ac_cv_lib_$ac_lib_var'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  ac_save_LIBS="$LIBS"
+LIBS="-ldir  $LIBS"
+cat > conftest.$ac_ext <<EOF
+#line 1821 "configure"
+#include "confdefs.h"
+/* Override any gcc2 internal prototype to avoid an error.  */
+/* We use char because int might match the return type of a gcc2
+    builtin and then its argument prototype would still apply.  */
+char opendir();
+
+int main() {
+opendir()
+; return 0; }
+EOF
+if { (eval echo configure:1832: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
+  rm -rf conftest*
+  eval "ac_cv_lib_$ac_lib_var=yes"
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  eval "ac_cv_lib_$ac_lib_var=no"
+fi
+rm -f conftest*
+LIBS="$ac_save_LIBS"
+
+fi
+if eval "test \"`echo '$ac_cv_lib_'$ac_lib_var`\" = yes"; then
+  echo "$ac_t""yes" 1>&6
+  LIBS="$LIBS -ldir"
+else
+  echo "$ac_t""no" 1>&6
+fi
+
+else
+echo $ac_n "checking for opendir in -lx""... $ac_c" 1>&6
+echo "configure:1854: checking for opendir in -lx" >&5
+ac_lib_var=`echo x'_'opendir | sed 'y%./+-%__p_%'`
+if eval "test \"`echo '$''{'ac_cv_lib_$ac_lib_var'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  ac_save_LIBS="$LIBS"
+LIBS="-lx  $LIBS"
+cat > conftest.$ac_ext <<EOF
+#line 1862 "configure"
+#include "confdefs.h"
+/* Override any gcc2 internal prototype to avoid an error.  */
+/* We use char because int might match the return type of a gcc2
+    builtin and then its argument prototype would still apply.  */
+char opendir();
+
+int main() {
+opendir()
+; return 0; }
+EOF
+if { (eval echo configure:1873: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
+  rm -rf conftest*
+  eval "ac_cv_lib_$ac_lib_var=yes"
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  eval "ac_cv_lib_$ac_lib_var=no"
+fi
+rm -f conftest*
+LIBS="$ac_save_LIBS"
+
+fi
+if eval "test \"`echo '$ac_cv_lib_'$ac_lib_var`\" = yes"; then
+  echo "$ac_t""yes" 1>&6
+  LIBS="$LIBS -lx"
+else
+  echo "$ac_t""no" 1>&6
+fi
+
+fi
+
+echo $ac_n "checking for ANSI C header files""... $ac_c" 1>&6
+echo "configure:1896: checking for ANSI C header files" >&5
+if eval "test \"`echo '$''{'ac_cv_header_stdc'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  cat > conftest.$ac_ext <<EOF
+#line 1901 "configure"
+#include "confdefs.h"
+#include <stdlib.h>
+#include <stdarg.h>
+#include <string.h>
+#include <float.h>
+EOF
+ac_try="$ac_cpp conftest.$ac_ext >/dev/null 2>conftest.out"
+{ (eval echo configure:1909: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; }
+ac_err=`grep -v '^ *+' conftest.out | grep -v "^conftest.${ac_ext}\$"`
+if test -z "$ac_err"; then
+  rm -rf conftest*
+  ac_cv_header_stdc=yes
+else
+  echo "$ac_err" >&5
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  ac_cv_header_stdc=no
+fi
+rm -f conftest*
+
+if test $ac_cv_header_stdc = yes; then
+  # SunOS 4.x string.h does not declare mem*, contrary to ANSI.
+cat > conftest.$ac_ext <<EOF
+#line 1926 "configure"
+#include "confdefs.h"
+#include <string.h>
+EOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  egrep "memchr" >/dev/null 2>&1; then
+  :
+else
+  rm -rf conftest*
+  ac_cv_header_stdc=no
+fi
+rm -f conftest*
+
+fi
+
+if test $ac_cv_header_stdc = yes; then
+  # ISC 2.0.2 stdlib.h does not declare free, contrary to ANSI.
+cat > conftest.$ac_ext <<EOF
+#line 1944 "configure"
+#include "confdefs.h"
+#include <stdlib.h>
+EOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  egrep "free" >/dev/null 2>&1; then
+  :
+else
+  rm -rf conftest*
+  ac_cv_header_stdc=no
+fi
+rm -f conftest*
+
+fi
+
+if test $ac_cv_header_stdc = yes; then
+  # /bin/cc in Irix-4.0.5 gets non-ANSI ctype macros unless using -ansi.
+if test "$cross_compiling" = yes; then
+  :
+else
+  cat > conftest.$ac_ext <<EOF
+#line 1965 "configure"
+#include "confdefs.h"
+#include <ctype.h>
+#define ISLOWER(c) ('a' <= (c) && (c) <= 'z')
+#define TOUPPER(c) (ISLOWER(c) ? 'A' + ((c) - 'a') : (c))
+#define XOR(e, f) (((e) && !(f)) || (!(e) && (f)))
+int main () { int i; for (i = 0; i < 256; i++)
+if (XOR (islower (i), ISLOWER (i)) || toupper (i) != TOUPPER (i)) exit(2);
+exit (0); }
+
+EOF
+if { (eval echo configure:1976: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext} && (./conftest; exit) 2>/dev/null
+then
+  :
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -fr conftest*
+  ac_cv_header_stdc=no
+fi
+rm -fr conftest*
+fi
+
+fi
+fi
+
+echo "$ac_t""$ac_cv_header_stdc" 1>&6
+if test $ac_cv_header_stdc = yes; then
+  cat >> confdefs.h <<\EOF
+#define STDC_HEADERS 1
+EOF
+
+fi
+
+for ac_hdr in fcntl.h limits.h sys/time.h unistd.h string.h memory.h sys/procfs.h
+do
+ac_safe=`echo "$ac_hdr" | sed 'y%./+-%__p_%'`
+echo $ac_n "checking for $ac_hdr""... $ac_c" 1>&6
+echo "configure:2003: checking for $ac_hdr" >&5
+if eval "test \"`echo '$''{'ac_cv_header_$ac_safe'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  cat > conftest.$ac_ext <<EOF
+#line 2008 "configure"
+#include "confdefs.h"
+#include <$ac_hdr>
+EOF
+ac_try="$ac_cpp conftest.$ac_ext >/dev/null 2>conftest.out"
+{ (eval echo configure:2013: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; }
+ac_err=`grep -v '^ *+' conftest.out | grep -v "^conftest.${ac_ext}\$"`
+if test -z "$ac_err"; then
+  rm -rf conftest*
+  eval "ac_cv_header_$ac_safe=yes"
+else
+  echo "$ac_err" >&5
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  eval "ac_cv_header_$ac_safe=no"
+fi
+rm -f conftest*
+fi
+if eval "test \"`echo '$ac_cv_header_'$ac_safe`\" = yes"; then
+  echo "$ac_t""yes" 1>&6
+    ac_tr_hdr=HAVE_`echo $ac_hdr | sed 'y%abcdefghijklmnopqrstuvwxyz./-%ABCDEFGHIJKLMNOPQRSTUVWXYZ___%'`
+  cat >> confdefs.h <<EOF
+#define $ac_tr_hdr 1
+EOF
+ 
+else
+  echo "$ac_t""no" 1>&6
+fi
+done
+
+cat > conftest.$ac_ext <<EOF
+#line 2040 "configure"
+#include "confdefs.h"
+#include <stdio.h>
+EOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  egrep "fread" >/dev/null 2>&1; then
+  rm -rf conftest*
+  cat >> confdefs.h <<\EOF
+#define HAVE_FREAD_DECL 1
+EOF
+
+fi
+rm -f conftest*
+
+cat > conftest.$ac_ext <<EOF
+#line 2055 "configure"
+#include "confdefs.h"
+#include <stdio.h>
+EOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  egrep "fgetc" >/dev/null 2>&1; then
+  rm -rf conftest*
+  cat >> confdefs.h <<\EOF
+#define HAVE_FGETC_DECL 1
+EOF
+
+fi
+rm -f conftest*
+
+cat > conftest.$ac_ext <<EOF
+#line 2070 "configure"
+#include "confdefs.h"
+#include <sys/procfs.h>
+EOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  egrep "pr_brksize" >/dev/null 2>&1; then
+  rm -rf conftest*
+  cat >> confdefs.h <<\EOF
+#define HAVE_PR_BRKSIZE 1
+EOF
+
+fi
+rm -f conftest*
+
+
+# The Ultrix 4.2 mips builtin alloca declared by alloca.h only works
+# for constant arguments.  Useless!
+echo $ac_n "checking for working alloca.h""... $ac_c" 1>&6
+echo "configure:2088: checking for working alloca.h" >&5
+if eval "test \"`echo '$''{'ac_cv_header_alloca_h'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  cat > conftest.$ac_ext <<EOF
+#line 2093 "configure"
+#include "confdefs.h"
+#include <alloca.h>
+int main() {
+char *p = alloca(2 * sizeof(int));
+; return 0; }
+EOF
+if { (eval echo configure:2100: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
+  rm -rf conftest*
+  ac_cv_header_alloca_h=yes
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  ac_cv_header_alloca_h=no
+fi
+rm -f conftest*
+fi
+
+echo "$ac_t""$ac_cv_header_alloca_h" 1>&6
+if test $ac_cv_header_alloca_h = yes; then
+  cat >> confdefs.h <<\EOF
+#define HAVE_ALLOCA_H 1
+EOF
+
+fi
+
+echo $ac_n "checking for alloca""... $ac_c" 1>&6
+echo "configure:2121: checking for alloca" >&5
+if eval "test \"`echo '$''{'ac_cv_func_alloca_works'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  cat > conftest.$ac_ext <<EOF
+#line 2126 "configure"
+#include "confdefs.h"
+
+#ifdef __GNUC__
+# define alloca __builtin_alloca
+#else
+# ifdef _MSC_VER
+#  include <malloc.h>
+#  define alloca _alloca
+# else
+#  if HAVE_ALLOCA_H
+#   include <alloca.h>
+#  else
+#   ifdef _AIX
+ #pragma alloca
+#   else
+#    ifndef alloca /* predefined by HP cc +Olibcalls */
+char *alloca ();
+#    endif
+#   endif
+#  endif
+# endif
+#endif
+
+int main() {
+char *p = (char *) alloca(1);
+; return 0; }
+EOF
+if { (eval echo configure:2154: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
+  rm -rf conftest*
+  ac_cv_func_alloca_works=yes
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  ac_cv_func_alloca_works=no
+fi
+rm -f conftest*
+fi
+
+echo "$ac_t""$ac_cv_func_alloca_works" 1>&6
+if test $ac_cv_func_alloca_works = yes; then
+  cat >> confdefs.h <<\EOF
+#define HAVE_ALLOCA 1
+EOF
+
+fi
+
+if test $ac_cv_func_alloca_works = no; then
+  # The SVR3 libPW and SVR4 libucb both contain incompatible functions
+  # that cause trouble.  Some versions do not even contain alloca or
+  # contain a buggy version.  If you still want to use their alloca,
+  # use ar to extract alloca.o from them instead of compiling alloca.c.
+  ALLOCA=alloca.${ac_objext}
+  cat >> confdefs.h <<\EOF
+#define C_ALLOCA 1
+EOF
+
+
+echo $ac_n "checking whether alloca needs Cray hooks""... $ac_c" 1>&6
+echo "configure:2186: checking whether alloca needs Cray hooks" >&5
+if eval "test \"`echo '$''{'ac_cv_os_cray'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  cat > conftest.$ac_ext <<EOF
+#line 2191 "configure"
+#include "confdefs.h"
+#if defined(CRAY) && ! defined(CRAY2)
+webecray
+#else
+wenotbecray
+#endif
+
+EOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  egrep "webecray" >/dev/null 2>&1; then
+  rm -rf conftest*
+  ac_cv_os_cray=yes
+else
+  rm -rf conftest*
+  ac_cv_os_cray=no
+fi
+rm -f conftest*
+
+fi
+
+echo "$ac_t""$ac_cv_os_cray" 1>&6
+if test $ac_cv_os_cray = yes; then
+for ac_func in _getb67 GETB67 getb67; do
+  echo $ac_n "checking for $ac_func""... $ac_c" 1>&6
+echo "configure:2216: checking for $ac_func" >&5
+if eval "test \"`echo '$''{'ac_cv_func_$ac_func'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  cat > conftest.$ac_ext <<EOF
+#line 2221 "configure"
+#include "confdefs.h"
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char $ac_func(); below.  */
+#include <assert.h>
+/* Override any gcc2 internal prototype to avoid an error.  */
+/* We use char because int might match the return type of a gcc2
+    builtin and then its argument prototype would still apply.  */
+char $ac_func();
+
+int main() {
+
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_$ac_func) || defined (__stub___$ac_func)
+choke me
+#else
+$ac_func();
+#endif
+
+; return 0; }
+EOF
+if { (eval echo configure:2244: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
+  rm -rf conftest*
+  eval "ac_cv_func_$ac_func=yes"
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  eval "ac_cv_func_$ac_func=no"
+fi
+rm -f conftest*
+fi
+
+if eval "test \"`echo '$ac_cv_func_'$ac_func`\" = yes"; then
+  echo "$ac_t""yes" 1>&6
+  cat >> confdefs.h <<EOF
+#define CRAY_STACKSEG_END $ac_func
+EOF
+
+  break
+else
+  echo "$ac_t""no" 1>&6
+fi
+
+done
+fi
+
+echo $ac_n "checking stack direction for C alloca""... $ac_c" 1>&6
+echo "configure:2271: checking stack direction for C alloca" >&5
+if eval "test \"`echo '$''{'ac_cv_c_stack_direction'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  if test "$cross_compiling" = yes; then
+  ac_cv_c_stack_direction=0
+else
+  cat > conftest.$ac_ext <<EOF
+#line 2279 "configure"
+#include "confdefs.h"
+find_stack_direction ()
+{
+  static char *addr = 0;
+  auto char dummy;
+  if (addr == 0)
+    {
+      addr = &dummy;
+      return find_stack_direction ();
+    }
+  else
+    return (&dummy > addr) ? 1 : -1;
+}
+main ()
+{
+  exit (find_stack_direction() < 0);
+}
+EOF
+if { (eval echo configure:2298: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext} && (./conftest; exit) 2>/dev/null
+then
+  ac_cv_c_stack_direction=1
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -fr conftest*
+  ac_cv_c_stack_direction=-1
+fi
+rm -fr conftest*
+fi
+
+fi
+
+echo "$ac_t""$ac_cv_c_stack_direction" 1>&6
+cat >> confdefs.h <<EOF
+#define STACK_DIRECTION $ac_cv_c_stack_direction
+EOF
+
+fi
+
+if test $ac_cv_prog_gcc = yes; then
+    echo $ac_n "checking whether ${CC-cc} needs -traditional""... $ac_c" 1>&6
+echo "configure:2321: checking whether ${CC-cc} needs -traditional" >&5
+if eval "test \"`echo '$''{'ac_cv_prog_gcc_traditional'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+    ac_pattern="Autoconf.*'x'"
+  cat > conftest.$ac_ext <<EOF
+#line 2327 "configure"
+#include "confdefs.h"
+#include <sgtty.h>
+Autoconf TIOCGETP
+EOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  egrep "$ac_pattern" >/dev/null 2>&1; then
+  rm -rf conftest*
+  ac_cv_prog_gcc_traditional=yes
+else
+  rm -rf conftest*
+  ac_cv_prog_gcc_traditional=no
+fi
+rm -f conftest*
+
+
+  if test $ac_cv_prog_gcc_traditional = no; then
+    cat > conftest.$ac_ext <<EOF
+#line 2345 "configure"
+#include "confdefs.h"
+#include <termio.h>
+Autoconf TCGETA
+EOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  egrep "$ac_pattern" >/dev/null 2>&1; then
+  rm -rf conftest*
+  ac_cv_prog_gcc_traditional=yes
+fi
+rm -f conftest*
+
+  fi
+fi
+
+echo "$ac_t""$ac_cv_prog_gcc_traditional" 1>&6
+  if test $ac_cv_prog_gcc_traditional = yes; then
+    CC="$CC -traditional"
+  fi
+fi
+
+echo $ac_n "checking return type of signal handlers""... $ac_c" 1>&6
+echo "configure:2367: checking return type of signal handlers" >&5
+if eval "test \"`echo '$''{'ac_cv_type_signal'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  cat > conftest.$ac_ext <<EOF
+#line 2372 "configure"
+#include "confdefs.h"
+#include <sys/types.h>
+#include <signal.h>
+#ifdef signal
+#undef signal
+#endif
+#ifdef __cplusplus
+extern "C" void (*signal (int, void (*)(int)))(int);
+#else
+void (*signal ()) ();
+#endif
+
+int main() {
+int i;
+; return 0; }
+EOF
+if { (eval echo configure:2389: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then
+  rm -rf conftest*
+  ac_cv_type_signal=void
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  ac_cv_type_signal=int
+fi
+rm -f conftest*
+fi
+
+echo "$ac_t""$ac_cv_type_signal" 1>&6
+cat >> confdefs.h <<EOF
+#define RETSIGTYPE $ac_cv_type_signal
+EOF
+
+
+echo $ac_n "checking for vprintf""... $ac_c" 1>&6
+echo "configure:2408: checking for vprintf" >&5
+if eval "test \"`echo '$''{'ac_cv_func_vprintf'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  cat > conftest.$ac_ext <<EOF
+#line 2413 "configure"
+#include "confdefs.h"
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char vprintf(); below.  */
+#include <assert.h>
+/* Override any gcc2 internal prototype to avoid an error.  */
+/* We use char because int might match the return type of a gcc2
+    builtin and then its argument prototype would still apply.  */
+char vprintf();
+
+int main() {
+
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_vprintf) || defined (__stub___vprintf)
+choke me
+#else
+vprintf();
+#endif
+
+; return 0; }
+EOF
+if { (eval echo configure:2436: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
+  rm -rf conftest*
+  eval "ac_cv_func_vprintf=yes"
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  eval "ac_cv_func_vprintf=no"
+fi
+rm -f conftest*
+fi
+
+if eval "test \"`echo '$ac_cv_func_'vprintf`\" = yes"; then
+  echo "$ac_t""yes" 1>&6
+  cat >> confdefs.h <<\EOF
+#define HAVE_VPRINTF 1
+EOF
+
+else
+  echo "$ac_t""no" 1>&6
+fi
+
+if test "$ac_cv_func_vprintf" != yes; then
+echo $ac_n "checking for _doprnt""... $ac_c" 1>&6
+echo "configure:2460: checking for _doprnt" >&5
+if eval "test \"`echo '$''{'ac_cv_func__doprnt'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  cat > conftest.$ac_ext <<EOF
+#line 2465 "configure"
+#include "confdefs.h"
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char _doprnt(); below.  */
+#include <assert.h>
+/* Override any gcc2 internal prototype to avoid an error.  */
+/* We use char because int might match the return type of a gcc2
+    builtin and then its argument prototype would still apply.  */
+char _doprnt();
+
+int main() {
+
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub__doprnt) || defined (__stub____doprnt)
+choke me
+#else
+_doprnt();
+#endif
+
+; return 0; }
+EOF
+if { (eval echo configure:2488: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
+  rm -rf conftest*
+  eval "ac_cv_func__doprnt=yes"
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  eval "ac_cv_func__doprnt=no"
+fi
+rm -f conftest*
+fi
+
+if eval "test \"`echo '$ac_cv_func_'_doprnt`\" = yes"; then
+  echo "$ac_t""yes" 1>&6
+  cat >> confdefs.h <<\EOF
+#define HAVE_DOPRNT 1
+EOF
+
+else
+  echo "$ac_t""no" 1>&6
+fi
+
+fi
+
+for ac_func in ftime select strftime strtol getrusage times mallinfo setbuffer getpagesize
+do
+echo $ac_n "checking for $ac_func""... $ac_c" 1>&6
+echo "configure:2515: checking for $ac_func" >&5
+if eval "test \"`echo '$''{'ac_cv_func_$ac_func'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  cat > conftest.$ac_ext <<EOF
+#line 2520 "configure"
+#include "confdefs.h"
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char $ac_func(); below.  */
+#include <assert.h>
+/* Override any gcc2 internal prototype to avoid an error.  */
+/* We use char because int might match the return type of a gcc2
+    builtin and then its argument prototype would still apply.  */
+char $ac_func();
+
+int main() {
+
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_$ac_func) || defined (__stub___$ac_func)
+choke me
+#else
+$ac_func();
+#endif
+
+; return 0; }
+EOF
+if { (eval echo configure:2543: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
+  rm -rf conftest*
+  eval "ac_cv_func_$ac_func=yes"
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  eval "ac_cv_func_$ac_func=no"
+fi
+rm -f conftest*
+fi
+
+if eval "test \"`echo '$ac_cv_func_'$ac_func`\" = yes"; then
+  echo "$ac_t""yes" 1>&6
+    ac_tr_func=HAVE_`echo $ac_func | tr 'abcdefghijklmnopqrstuvwxyz' 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'`
+  cat >> confdefs.h <<EOF
+#define $ac_tr_func 1
+EOF
+ 
+else
+  echo "$ac_t""no" 1>&6
+fi
+done
+
+for ac_func in ftruncate strstr strcasecmp
+do
+echo $ac_n "checking for $ac_func""... $ac_c" 1>&6
+echo "configure:2570: checking for $ac_func" >&5
+if eval "test \"`echo '$''{'ac_cv_func_$ac_func'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  cat > conftest.$ac_ext <<EOF
+#line 2575 "configure"
+#include "confdefs.h"
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char $ac_func(); below.  */
+#include <assert.h>
+/* Override any gcc2 internal prototype to avoid an error.  */
+/* We use char because int might match the return type of a gcc2
+    builtin and then its argument prototype would still apply.  */
+char $ac_func();
+
+int main() {
+
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_$ac_func) || defined (__stub___$ac_func)
+choke me
+#else
+$ac_func();
+#endif
+
+; return 0; }
+EOF
+if { (eval echo configure:2598: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
+  rm -rf conftest*
+  eval "ac_cv_func_$ac_func=yes"
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  eval "ac_cv_func_$ac_func=no"
+fi
+rm -f conftest*
+fi
+
+if eval "test \"`echo '$ac_cv_func_'$ac_func`\" = yes"; then
+  echo "$ac_t""yes" 1>&6
+    ac_tr_func=HAVE_`echo $ac_func | tr 'abcdefghijklmnopqrstuvwxyz' 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'`
+  cat >> confdefs.h <<EOF
+#define $ac_tr_func 1
+EOF
+ 
+else
+  echo "$ac_t""no" 1>&6
+LIBOBJS="$LIBOBJS ${ac_func}.${ac_objext}"
+fi
+done
+
+
+
+
+# *** Custom checking (based on GNU tar configure.in) ***
+# ---------------------------------------------------------------------------
+echo $ac_n "checking for HP-UX needing gmalloc""... $ac_c" 1>&6
+echo "configure:2629: checking for HP-UX needing gmalloc" >&5
+if test "`(uname -s) 2> /dev/null`" = 'HP-UX'; then
+  echo "$ac_t""yes" 1>&6
+  LIBOBJS="$LIBOBJS gmalloc.o"
+  cat >> confdefs.h <<\EOF
+#define HAVE_VALLOC 1
+EOF
+
+else
+  echo "$ac_t""no" 1>&6
+  for ac_func in valloc
+do
+echo $ac_n "checking for $ac_func""... $ac_c" 1>&6
+echo "configure:2642: checking for $ac_func" >&5
+if eval "test \"`echo '$''{'ac_cv_func_$ac_func'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  cat > conftest.$ac_ext <<EOF
+#line 2647 "configure"
+#include "confdefs.h"
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char $ac_func(); below.  */
+#include <assert.h>
+/* Override any gcc2 internal prototype to avoid an error.  */
+/* We use char because int might match the return type of a gcc2
+    builtin and then its argument prototype would still apply.  */
+char $ac_func();
+
+int main() {
+
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_$ac_func) || defined (__stub___$ac_func)
+choke me
+#else
+$ac_func();
+#endif
+
+; return 0; }
+EOF
+if { (eval echo configure:2670: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
+  rm -rf conftest*
+  eval "ac_cv_func_$ac_func=yes"
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  eval "ac_cv_func_$ac_func=no"
+fi
+rm -f conftest*
+fi
+
+if eval "test \"`echo '$ac_cv_func_'$ac_func`\" = yes"; then
+  echo "$ac_t""yes" 1>&6
+    ac_tr_func=HAVE_`echo $ac_func | tr 'abcdefghijklmnopqrstuvwxyz' 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'`
+  cat >> confdefs.h <<EOF
+#define $ac_tr_func 1
+EOF
+ 
+else
+  echo "$ac_t""no" 1>&6
+fi
+done
+
+fi
+
+fp_WITH_DMALLOC
+fp_WITH_REGEX
+
+# ---------------------------------------------------------------------------
+if test "$ac_cv_func_alloca" = 'no'; then
+  LIBOBJS="$LIBOBJS xmalloc.o error.o"
+fi
+
+
+# ---------------------------------------------------------------------------
+
+ac_ext=C
+# CXXFLAGS is not in ac_cpp because -g, -O, etc. are not valid cpp options.
+ac_cpp='$CXXCPP $CPPFLAGS'
+ac_compile='${CXX-g++} -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext 1>&5'
+ac_link='${CXX-g++} -o conftest${ac_exeext} $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS 1>&5'
+cross_compiling=$ac_cv_prog_cxx_cross
+
+
+echo $ac_n "checking "that the GDBM library is available"""... $ac_c" 1>&6
+echo "configure:2716: checking "that the GDBM library is available"" >&5
+cat > conftest.$ac_ext <<EOF
+#line 2718 "configure"
+#include "confdefs.h"
+#include <gdbm.h>
+
+int main() {
+
+; return 0; }
+EOF
+if { (eval echo configure:2726: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then
+  rm -rf conftest*
+  success=yes
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  success=no
+
+fi
+rm -f conftest*
+echo "$ac_t""$success" 1>&6
+if test $success = "no"; then
+{ echo "configure: error: "GDBM Library not available - cannot install"" 1>&2; exit 1; }
+fi
+
+echo $ac_n "checking "that Perl 5 is available"""... $ac_c" 1>&6
+echo "configure:2743: checking "that Perl 5 is available"" >&5
+success="no"
+pl_path="$PATH"
+IFS="${IFS=   }"; pl_save_ifs="$IFS"; IFS=":"
+for pl_dir in $pl_path; do
+  test -z "$pl_dir" && pl_dir=.
+  if test -x $pl_dir/perl; then
+    success="yes"
+    break
+  fi
+done
+IFS="$pl_save_ifs"
+
+if test $success = "yes"; then
+try=`perl configtest.pl` || try="0"
+try=`expr $try \> 5.000`
+if test $try = "1"; then
+echo "$ac_t"""yes"" 1>&6
+else
+success="no"
+fi
+fi
+
+if test $success = "no"; then
+echo "$ac_t"""no"" 1>&6
+{ echo "configure: error: "Perl 5 not available - cannot install"" 1>&2; exit 1; }
+fi
+
+success=no
+echo $ac_n "checking "whether STL library has known faults"""... $ac_c" 1>&6
+echo "configure:2773: checking "whether STL library has known faults"" >&5
+
+cat > conftest.$ac_ext <<EOF
+#line 2776 "configure"
+#include "confdefs.h"
+#include <vector>
+
+int main() {
+
+; return 0; }
+EOF
+if { (eval echo configure:2784: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then
+  rm -rf conftest*
+  success=yes
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+fi
+rm -f conftest*
+
+if test $success = "no"; then
+cat > conftest.$ac_ext <<EOF
+#line 2795 "configure"
+#include "confdefs.h"
+#include <vector.h>
+
+int main() {
+
+; return 0; }
+EOF
+if { (eval echo configure:2803: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then
+  rm -rf conftest*
+  success="yes"
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+fi
+rm -f conftest*
+fi
+
+if test $success = "no"; then
+cat > conftest.$ac_ext <<EOF
+#line 2815 "configure"
+#include "confdefs.h"
+#include <ospace\\std\\vector>
+
+int main() {
+
+; return 0; }
+EOF
+if { (eval echo configure:2823: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then
+  rm -rf conftest*
+  success="yes"
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+fi
+rm -f conftest*
+fi
+
+if test $success = yes; then
+echo "$ac_t"""no"" 1>&6
+else
+echo "$ac_t"""yes"" 1>&6
+{ echo "configure: error: "STL Broken - Obtain newer version of GNU C Compiler"" 1>&2; exit 1; }
+fi
+
+gsdlos=`uname | tr A-Z a-z`
+
+
+echo $ac_n "checking for HP-UX needing gmalloc""... $ac_c" 1>&6
+echo "configure:2844: checking for HP-UX needing gmalloc" >&5
+if test "`(uname -s) 2> /dev/null`" = 'HP-UX'; then
+  echo "$ac_t""yes" 1>&6
+  LIBOBJS="$LIBOBJS gmalloc.o"
+  cat >> confdefs.h <<\EOF
+#define HAVE_VALLOC 1
+EOF
+
+else
+  echo "$ac_t""no" 1>&6
+  for ac_func in valloc
+do
+echo $ac_n "checking for $ac_func""... $ac_c" 1>&6
+echo "configure:2857: checking for $ac_func" >&5
+if eval "test \"`echo '$''{'ac_cv_func_$ac_func'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  cat > conftest.$ac_ext <<EOF
+#line 2862 "configure"
+#include "confdefs.h"
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char $ac_func(); below.  */
+#include <assert.h>
+/* Override any gcc2 internal prototype to avoid an error.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+/* We use char because int might match the return type of a gcc2
+    builtin and then its argument prototype would still apply.  */
+char $ac_func();
+
+int main() {
+
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_$ac_func) || defined (__stub___$ac_func)
+choke me
+#else
+$ac_func();
+#endif
+
+; return 0; }
+EOF
+if { (eval echo configure:2888: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest${ac_exeext}; then
+  rm -rf conftest*
+  eval "ac_cv_func_$ac_func=yes"
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  eval "ac_cv_func_$ac_func=no"
+fi
+rm -f conftest*
+fi
+
+if eval "test \"`echo '$ac_cv_func_'$ac_func`\" = yes"; then
+  echo "$ac_t""yes" 1>&6
+    ac_tr_func=HAVE_`echo $ac_func | tr 'abcdefghijklmnopqrstuvwxyz' 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'`
+  cat >> confdefs.h <<EOF
+#define $ac_tr_func 1
+EOF
+ 
+else
+  echo "$ac_t""no" 1>&6
+fi
+done
+
+fi
+
+fp_WITH_DMALLOC
+fp_WITH_REGEX
+
+# ---------------------------------------------------------------------------
+if test "$ac_cv_func_alloca" = 'no'; then
+  LIBOBJS="$LIBOBJS xmalloc.o error.o"
+fi
+
+
+# ---------------------------------------------------------------------------
+
+trap '' 1 2 15
+cat > confcache <<\EOF
+# This file is a shell script that caches the results of configure
+# tests run on this system so they can be shared between configure
+# scripts and configure runs.  It is not useful on other systems.
+# If it contains results you don't want to keep, you may remove or edit it.
+#
+# By default, configure uses ./config.cache as the cache file,
+# creating it if it does not exist already.  You can give configure
+# the --cache-file=FILE option to use a different cache file; that is
+# what configure does when it calls configure scripts in
+# subdirectories, so they share the cache.
+# Giving --cache-file=/dev/null disables caching, for debugging configure.
+# config.status only pays attention to the cache file if you give it the
+# --recheck option to rerun configure.
+#
+EOF
+# The following way of writing the cache mishandles newlines in values,
+# but we know of no workaround that is simple, portable, and efficient.
+# So, don't put newlines in cache variables' values.
+# Ultrix sh set writes to stderr and can't be redirected directly,
+# and sets the high bit in the cache file unless we assign to the vars.
+(set) 2>&1 |
+  case `(ac_space=' '; set | grep ac_space) 2>&1` in
+  *ac_space=\ *)
+    # `set' does not quote correctly, so add quotes (double-quote substitution
+    # turns \\\\ into \\, and sed turns \\ into \).
+    sed -n \
+      -e "s/'/'\\\\''/g" \
+      -e "s/^\\([a-zA-Z0-9_]*_cv_[a-zA-Z0-9_]*\\)=\\(.*\\)/\\1=\${\\1='\\2'}/p"
+    ;;
+  *)
+    # `set' quotes correctly as required by POSIX, so do not add quotes.
+    sed -n -e 's/^\([a-zA-Z0-9_]*_cv_[a-zA-Z0-9_]*\)=\(.*\)/\1=${\1=\2}/p'
+    ;;
+  esac >> confcache
+if cmp -s $cache_file confcache; then
+  :
+else
+  if test -w $cache_file; then
+    echo "updating cache $cache_file"
+    cat confcache > $cache_file
+  else
+    echo "not updating unwritable cache $cache_file"
+  fi
+fi
+rm -f confcache
+
+trap 'rm -fr conftest* confdefs* core core.* *.core $ac_clean_files; exit 1' 1 2 15
+
+test "x$prefix" = xNONE && prefix=$ac_default_prefix
+# Let make expand exec_prefix.
+test "x$exec_prefix" = xNONE && exec_prefix='${prefix}'
+
+# Any assignment to VPATH causes Sun make to only execute
+# the first set of double-colon rules, so remove it if not needed.
+# If there is a colon in the path, we need to keep it.
+if test "x$srcdir" = x.; then
+  ac_vpsub='/^[ 	]*VPATH[ 	]*=[^:]*$/d'
+fi
+
+trap 'rm -f $CONFIG_STATUS conftest*; exit 1' 1 2 15
+
+DEFS=-DHAVE_CONFIG_H
+
+# Without the "./", some shells look in PATH for config.status.
+: ${CONFIG_STATUS=./config.status}
+
+echo creating $CONFIG_STATUS
+rm -f $CONFIG_STATUS
+cat > $CONFIG_STATUS <<EOF
+#! /bin/sh
+# Generated automatically by configure.
+# Run this file to recreate the current configuration.
+# This directory was configured as follows,
+# on host `(hostname || uname -n) 2>/dev/null | sed 1q`:
+#
+# $0 $ac_configure_args
+#
+# Compiler output produced by configure, useful for debugging
+# configure, is in ./config.log if it exists.
+
+ac_cs_usage="Usage: $CONFIG_STATUS [--recheck] [--version] [--help]"
+for ac_option
+do
+  case "\$ac_option" in
+  -recheck | --recheck | --rechec | --reche | --rech | --rec | --re | --r)
+    echo "running \${CONFIG_SHELL-/bin/sh} $0 $ac_configure_args --no-create --no-recursion"
+    exec \${CONFIG_SHELL-/bin/sh} $0 $ac_configure_args --no-create --no-recursion ;;
+  -version | --version | --versio | --versi | --vers | --ver | --ve | --v)
+    echo "$CONFIG_STATUS generated by autoconf version 2.13"
+    exit 0 ;;
+  -help | --help | --hel | --he | --h)
+    echo "\$ac_cs_usage"; exit 0 ;;
+  *) echo "\$ac_cs_usage"; exit 1 ;;
+  esac
+done
+
+ac_given_srcdir=$srcdir
+
+trap 'rm -fr `echo "packages/mg/Makefile packages/mg/src/text/Makefile packages/mg/src/images/Makefile 
+          packages/mg/src/scripts/Makefile packages/mg/lib/Makefile
+          src/hashfile/Makefile src/colservr/Makefile src/recpt/Makefile src/txt2db/Makefile 
+          src/db2txt/Makefile lib/Makefile collect/cstr/src/recpt/Makefile 
+          collect/niupepa/src/recpt/Makefile Makefile config.h" | sed "s/:[^ ]*//g"` conftest*; exit 1' 1 2 15
+EOF
+cat >> $CONFIG_STATUS <<EOF
+
+# Protect against being on the right side of a sed subst in config.status.
+sed 's/%@/@@/; s/@%/@@/; s/%g\$/@g/; /@g\$/s/[\\\\&%]/\\\\&/g;
+ s/@@/%@/; s/@@/@%/; s/@g\$/%g/' > conftest.subs <<\\CEOF
+$ac_vpsub
+$extrasub
+s%@SHELL@%$SHELL%g
+s%@CFLAGS@%$CFLAGS%g
+s%@CPPFLAGS@%$CPPFLAGS%g
+s%@CXXFLAGS@%$CXXFLAGS%g
+s%@FFLAGS@%$FFLAGS%g
+s%@DEFS@%$DEFS%g
+s%@LDFLAGS@%$LDFLAGS%g
+s%@LIBS@%$LIBS%g
+s%@exec_prefix@%$exec_prefix%g
+s%@prefix@%$prefix%g
+s%@program_transform_name@%$program_transform_name%g
+s%@bindir@%$bindir%g
+s%@sbindir@%$sbindir%g
+s%@libexecdir@%$libexecdir%g
+s%@datadir@%$datadir%g
+s%@sysconfdir@%$sysconfdir%g
+s%@sharedstatedir@%$sharedstatedir%g
+s%@localstatedir@%$localstatedir%g
+s%@libdir@%$libdir%g
+s%@includedir@%$includedir%g
+s%@oldincludedir@%$oldincludedir%g
+s%@infodir@%$infodir%g
+s%@mandir@%$mandir%g
+s%@PACKAGE@%$PACKAGE%g
+s%@VERSION@%$VERSION%g
+s%@USE_FASTCGI@%$USE_FASTCGI%g
+s%@CC@%$CC%g
+s%@CXX@%$CXX%g
+s%@AWK@%$AWK%g
+s%@YACC@%$YACC%g
+s%@LN_S@%$LN_S%g
+s%@SET_MAKE@%$SET_MAKE%g
+s%@RANLIB@%$RANLIB%g
+s%@CPP@%$CPP%g
+s%@ALLOCA@%$ALLOCA%g
+s%@LIBOBJS@%$LIBOBJS%g
+s%@gsdlos@%$gsdlos%g
+
+CEOF
+EOF
+
+cat >> $CONFIG_STATUS <<\EOF
+
+# Split the substitutions into bite-sized pieces for seds with
+# small command number limits, like on Digital OSF/1 and HP-UX.
+ac_max_sed_cmds=90 # Maximum number of lines to put in a sed script.
+ac_file=1 # Number of current file.
+ac_beg=1 # First line for current file.
+ac_end=$ac_max_sed_cmds # Line after last line for current file.
+ac_more_lines=:
+ac_sed_cmds=""
+while $ac_more_lines; do
+  if test $ac_beg -gt 1; then
+    sed "1,${ac_beg}d; ${ac_end}q" conftest.subs > conftest.s$ac_file
+  else
+    sed "${ac_end}q" conftest.subs > conftest.s$ac_file
+  fi
+  if test ! -s conftest.s$ac_file; then
+    ac_more_lines=false
+    rm -f conftest.s$ac_file
+  else
+    if test -z "$ac_sed_cmds"; then
+      ac_sed_cmds="sed -f conftest.s$ac_file"
+    else
+      ac_sed_cmds="$ac_sed_cmds | sed -f conftest.s$ac_file"
+    fi
+    ac_file=`expr $ac_file + 1`
+    ac_beg=$ac_end
+    ac_end=`expr $ac_end + $ac_max_sed_cmds`
+  fi
+done
+if test -z "$ac_sed_cmds"; then
+  ac_sed_cmds=cat
+fi
+EOF
+
+cat >> $CONFIG_STATUS <<EOF
+
+CONFIG_FILES=\${CONFIG_FILES-"packages/mg/Makefile packages/mg/src/text/Makefile packages/mg/src/images/Makefile 
+          packages/mg/src/scripts/Makefile packages/mg/lib/Makefile
+          src/hashfile/Makefile src/colservr/Makefile src/recpt/Makefile src/txt2db/Makefile 
+          src/db2txt/Makefile lib/Makefile collect/cstr/src/recpt/Makefile 
+          collect/niupepa/src/recpt/Makefile Makefile"}
+EOF
+cat >> $CONFIG_STATUS <<\EOF
+for ac_file in .. $CONFIG_FILES; do if test "x$ac_file" != x..; then
+  # Support "outfile[:infile[:infile...]]", defaulting infile="outfile.in".
+  case "$ac_file" in
+  *:*) ac_file_in=`echo "$ac_file"|sed 's%[^:]*:%%'`
+       ac_file=`echo "$ac_file"|sed 's%:.*%%'` ;;
+  *) ac_file_in="${ac_file}.in" ;;
+  esac
+
+  # Adjust a relative srcdir, top_srcdir, and INSTALL for subdirectories.
+
+  # Remove last slash and all that follows it.  Not all systems have dirname.
+  ac_dir=`echo $ac_file|sed 's%/[^/][^/]*$%%'`
+  if test "$ac_dir" != "$ac_file" && test "$ac_dir" != .; then
+    # The file is in a subdirectory.
+    test ! -d "$ac_dir" && mkdir "$ac_dir"
+    ac_dir_suffix="/`echo $ac_dir|sed 's%^\./%%'`"
+    # A "../" for each directory in $ac_dir_suffix.
+    ac_dots=`echo $ac_dir_suffix|sed 's%/[^/]*%../%g'`
+  else
+    ac_dir_suffix= ac_dots=
+  fi
+
+  case "$ac_given_srcdir" in
+  .)  srcdir=.
+      if test -z "$ac_dots"; then top_srcdir=.
+      else top_srcdir=`echo $ac_dots|sed 's%/$%%'`; fi ;;
+  /*) srcdir="$ac_given_srcdir$ac_dir_suffix"; top_srcdir="$ac_given_srcdir" ;;
+  *) # Relative path.
+    srcdir="$ac_dots$ac_given_srcdir$ac_dir_suffix"
+    top_srcdir="$ac_dots$ac_given_srcdir" ;;
+  esac
+
+
+  echo creating "$ac_file"
+  rm -f "$ac_file"
+  configure_input="Generated automatically from `echo $ac_file_in|sed 's%.*/%%'` by configure."
+  case "$ac_file" in
+  *Makefile*) ac_comsub="1i\\
+# $configure_input" ;;
+  *) ac_comsub= ;;
+  esac
+
+  ac_file_inputs=`echo $ac_file_in|sed -e "s%^%$ac_given_srcdir/%" -e "s%:% $ac_given_srcdir/%g"`
+  sed -e "$ac_comsub
+s%@configure_input@%$configure_input%g
+s%@srcdir@%$srcdir%g
+s%@top_srcdir@%$top_srcdir%g
+" $ac_file_inputs | (eval "$ac_sed_cmds") > $ac_file
+fi; done
+rm -f conftest.s*
+
+# These sed commands are passed to sed as "A NAME B NAME C VALUE D", where
+# NAME is the cpp macro being defined and VALUE is the value it is being given.
+#
+# ac_d sets the value in "#define NAME VALUE" lines.
+ac_dA='s%^\([ 	]*\)#\([ 	]*define[ 	][ 	]*\)'
+ac_dB='\([ 	][ 	]*\)[^ 	]*%\1#\2'
+ac_dC='\3'
+ac_dD='%g'
+# ac_u turns "#undef NAME" with trailing blanks into "#define NAME VALUE".
+ac_uA='s%^\([ 	]*\)#\([ 	]*\)undef\([ 	][ 	]*\)'
+ac_uB='\([ 	]\)%\1#\2define\3'
+ac_uC=' '
+ac_uD='\4%g'
+# ac_e turns "#undef NAME" without trailing blanks into "#define NAME VALUE".
+ac_eA='s%^\([ 	]*\)#\([ 	]*\)undef\([ 	][ 	]*\)'
+ac_eB='$%\1#\2define\3'
+ac_eC=' '
+ac_eD='%g'
+
+if test "${CONFIG_HEADERS+set}" != set; then
+EOF
+cat >> $CONFIG_STATUS <<EOF
+  CONFIG_HEADERS="config.h"
+EOF
+cat >> $CONFIG_STATUS <<\EOF
+fi
+for ac_file in .. $CONFIG_HEADERS; do if test "x$ac_file" != x..; then
+  # Support "outfile[:infile[:infile...]]", defaulting infile="outfile.in".
+  case "$ac_file" in
+  *:*) ac_file_in=`echo "$ac_file"|sed 's%[^:]*:%%'`
+       ac_file=`echo "$ac_file"|sed 's%:.*%%'` ;;
+  *) ac_file_in="${ac_file}.in" ;;
+  esac
+
+  echo creating $ac_file
+
+  rm -f conftest.frag conftest.in conftest.out
+  ac_file_inputs=`echo $ac_file_in|sed -e "s%^%$ac_given_srcdir/%" -e "s%:% $ac_given_srcdir/%g"`
+  cat $ac_file_inputs > conftest.in
+
+EOF
+
+# Transform confdefs.h into a sed script conftest.vals that substitutes
+# the proper values into config.h.in to produce config.h.  And first:
+# Protect against being on the right side of a sed subst in config.status.
+# Protect against being in an unquoted here document in config.status.
+rm -f conftest.vals
+cat > conftest.hdr <<\EOF
+s/[\\&%]/\\&/g
+s%[\\$`]%\\&%g
+s%#define \([A-Za-z_][A-Za-z0-9_]*\) *\(.*\)%${ac_dA}\1${ac_dB}\1${ac_dC}\2${ac_dD}%gp
+s%ac_d%ac_u%gp
+s%ac_u%ac_e%gp
+EOF
+sed -n -f conftest.hdr confdefs.h > conftest.vals
+rm -f conftest.hdr
+
+# This sed command replaces #undef with comments.  This is necessary, for
+# example, in the case of _POSIX_SOURCE, which is predefined and required
+# on some systems where configure will not decide to define it.
+cat >> conftest.vals <<\EOF
+s%^[ 	]*#[ 	]*undef[ 	][ 	]*[a-zA-Z_][a-zA-Z_0-9]*%/* & */%
+EOF
+
+# Break up conftest.vals because some shells have a limit on
+# the size of here documents, and old seds have small limits too.
+
+rm -f conftest.tail
+while :
+do
+  ac_lines=`grep -c . conftest.vals`
+  # grep -c gives empty output for an empty file on some AIX systems.
+  if test -z "$ac_lines" || test "$ac_lines" -eq 0; then break; fi
+  # Write a limited-size here document to conftest.frag.
+  echo '  cat > conftest.frag <<CEOF' >> $CONFIG_STATUS
+  sed ${ac_max_here_lines}q conftest.vals >> $CONFIG_STATUS
+  echo 'CEOF
+  sed -f conftest.frag conftest.in > conftest.out
+  rm -f conftest.in
+  mv conftest.out conftest.in
+' >> $CONFIG_STATUS
+  sed 1,${ac_max_here_lines}d conftest.vals > conftest.tail
+  rm -f conftest.vals
+  mv conftest.tail conftest.vals
+done
+rm -f conftest.vals
+
+cat >> $CONFIG_STATUS <<\EOF
+  rm -f conftest.frag conftest.h
+  echo "/* $ac_file.  Generated automatically by configure.  */" > conftest.h
+  cat conftest.in >> conftest.h
+  rm -f conftest.in
+  if cmp -s $ac_file conftest.h 2>/dev/null; then
+    echo "$ac_file is unchanged"
+    rm -f conftest.h
+  else
+    # Remove last slash and all that follows it.  Not all systems have dirname.
+      ac_dir=`echo $ac_file|sed 's%/[^/][^/]*$%%'`
+      if test "$ac_dir" != "$ac_file" && test "$ac_dir" != .; then
+      # The file is in a subdirectory.
+      test ! -d "$ac_dir" && mkdir "$ac_dir"
+    fi
+    rm -f $ac_file
+    mv conftest.h $ac_file
+  fi
+fi; done
+
+EOF
+cat >> $CONFIG_STATUS <<EOF
+
+EOF
+cat >> $CONFIG_STATUS <<\EOF
+
+exit 0
+EOF
+chmod +x $CONFIG_STATUS
+rm -fr confdefs* $ac_clean_files
+test "$no_create" = yes || ${CONFIG_SHELL-/bin/sh} $CONFIG_STATUS || exit 1
+
+
+
+
+
Index: /tags/gsdl-21/gsdl/configure.in
===================================================================
--- /tags/gsdl-21/gsdl/configure.in	(revision 699)
+++ /tags/gsdl-21/gsdl/configure.in	(revision 699)
@@ -0,0 +1,248 @@
+dnl Process this file with autoconf to produce a configure script.
+AC_INIT(lib/text_t.h)
+AC_CONFIG_HEADER(config.h)
+AC_ARG_PROGRAM
+
+PACKAGE=gsdl
+VERSION=2.1
+AC_DEFINE_UNQUOTED(PACKAGE, "$PACKAGE")
+AC_DEFINE_UNQUOTED(VERSION, "$VERSION")
+AC_SUBST(PACKAGE)
+AC_SUBST(VERSION)
+
+USE_FASTCGI=0
+if test USE_FASTCGI = 1; then
+AC_DEFINE(USE_FASTCGI)
+AC_SUBST(USE_FASTCGI)
+fi
+
+dnl Checks for programs.
+CFLAGS=-g
+AC_SUBST(CFLAGS)
+
+AC_PROG_CC
+AC_PROG_CXX
+AC_PROG_AWK
+AC_PROG_YACC
+dnl AC_PROG_INSTALL
+AC_PROG_LN_S
+AC_PROG_MAKE_SET
+AC_PROG_RANLIB
+
+
+dnl Checks for typedefs, structures, and compiler characteristics.
+AC_AIX
+AC_ISC_POSIX
+AC_MINIX
+fp_C_PROTOTYPES
+AC_C_CONST
+AC_TYPE_OFF_T
+AC_TYPE_SIZE_T
+AC_HEADER_TIME
+AC_STRUCT_TM
+
+dnl --- This should be in the stdc checking -----------------
+if test "$ac_cv_prog_cc_stdc" = '-Xc'; then
+dnl The Pyramids running SMP_DC.OSx on MIPS r3000 chips with
+dnl /usr/opt/cc3.11/bin/cc treat -Xc as full ANSI mode - no POSIX or
+dnl XOPEN extensions (like popen).
+dnl Unfortunately -D_{XOPEN,POSIX}_SOURCE can't provide all that mg
+dnl needs. So drop back to -Xa
+AC_TRY_COMPILE(
+[#include <stdio.h>
+], [ FILE *rubbish = popen("yes", "r"); ],,
+[CC="`echo $CC | sed 's/-Xc/-Xa/'`"    ac_cv_prog_cc_stdc='-Xa'])
+fi
+dnl ----------------------
+
+
+dnl Checks for libraries.
+dnl Replace `main' with a function in -lg:
+AC_CHECK_LIB(g, main)
+dnl Replace `main' with a function in -lgdbm:
+AC_CHECK_LIB(gdbm, main)
+dnl Replace `main' with a function in -lm:
+AC_CHECK_LIB(m, main)
+dnl Replace `main' with a function in -lstdc:
+AC_CHECK_LIB(stdc, main)
+
+dnl Checks for header files.
+AC_HEADER_DIRENT
+AC_HEADER_STDC
+AC_CHECK_HEADERS(fcntl.h limits.h sys/time.h unistd.h string.h memory.h sys/procfs.h)
+AC_EGREP_HEADER(fread, stdio.h, AC_DEFINE(HAVE_FREAD_DECL))
+AC_EGREP_HEADER(fgetc, stdio.h, AC_DEFINE(HAVE_FGETC_DECL))
+AC_EGREP_HEADER(pr_brksize, sys/procfs.h, AC_DEFINE(HAVE_PR_BRKSIZE))
+
+dnl Checks for library functions.
+AC_FUNC_ALLOCA
+AC_PROG_GCC_TRADITIONAL
+AC_TYPE_SIGNAL
+AC_FUNC_VPRINTF
+AC_CHECK_FUNCS(ftime select strftime strtol getrusage times mallinfo setbuffer getpagesize)
+AC_REPLACE_FUNCS(ftruncate strstr strcasecmp)
+
+
+# *** Custom checking (based on GNU tar configure.in) ***
+# ---------------------------------------------------------------------------
+AC_MSG_CHECKING(for HP-UX needing gmalloc)
+if test "`(uname -s) 2> /dev/null`" = 'HP-UX'; then
+  AC_MSG_RESULT(yes)
+  LIBOBJS="$LIBOBJS gmalloc.o"
+  AC_DEFINE(HAVE_VALLOC)
+else
+  AC_MSG_RESULT(no)
+  AC_CHECK_FUNCS(valloc)
+fi
+
+fp_WITH_DMALLOC
+fp_WITH_REGEX
+
+# ---------------------------------------------------------------------------
+if test "$ac_cv_func_alloca" = 'no'; then
+dnl    support functions for alloca.c
+  LIBOBJS="$LIBOBJS xmalloc.o error.o"
+fi
+
+
+# ---------------------------------------------------------------------------
+
+dnl
+dnl Switch into C++
+dnl
+AC_LANG_CPLUSPLUS
+
+dnl
+dnl Check that the GDBM library is available
+dnl
+AC_MSG_CHECKING("that the GDBM library is available")
+AC_TRY_COMPILE(
+[#include <gdbm.h>
+],
+[],
+success=yes,
+success=no
+)
+AC_MSG_RESULT($success)
+if test $success = "no"; then
+AC_MSG_ERROR("GDBM Library not available - cannot install")
+fi
+
+dnl
+dnl Now check that Perl is actually around, and can be found
+dnl
+AC_MSG_CHECKING("that Perl 5 is available")
+success="no"
+pl_path="$PATH"
+IFS="${IFS=   }"; pl_save_ifs="$IFS"; IFS=":"
+for pl_dir in $pl_path; do
+  test -z "$pl_dir" && pl_dir=.
+  if test -x $pl_dir/perl; then
+    success="yes"
+    break
+  fi
+done
+IFS="$pl_save_ifs"
+
+dnl
+dnl Perl actually can be run; now we see if it is version 5 by
+dnl running a sample script (which returns a "clean" version number
+dnl and then tests the result using expr.  If perl errors in some
+dnl way (eg. broken Perl executable, missing script), we set a failure
+dnl value ("0") and then proceed as best we can
+dnl
+if test $success = "yes"; then
+try=`perl configtest.pl` || try="0"
+try=`expr $try \> 5.000`
+if test $try = "1"; then
+AC_MSG_RESULT("yes")
+else
+success="no"
+fi
+fi
+
+if test $success = "no"; then
+AC_MSG_RESULT("no")
+AC_MSG_ERROR("Perl 5 not available - cannot install")
+fi
+
+dnl
+dnl Check if the STL library has any known faults;
+dnl In GCC 2.7.2.x series, some STL library headers are missing
+dnl a file called "new" which is referred to by other headers
+dnl (eg. vector/vector.h); if this applies, then all use of the
+dnl STL will cause a compile fail, so we'd better let the user
+dnl know now, before things really go wrong!
+dnl
+success=no
+AC_MSG_CHECKING("whether STL library has known faults")
+
+dnl - first shot, try simple good old vector on its own
+AC_TRY_COMPILE(
+[#include <vector>
+], 
+[],
+success=yes,
+)
+
+dnl - try using the .h equivalent
+if test $success = "no"; then
+AC_TRY_COMPILE(
+[#include <vector.h>
+], 
+[],
+success="yes",
+)
+fi
+
+dnl - now try for the objectspace headers
+if test $success = "no"; then
+AC_TRY_COMPILE(
+[#include <ospace\\std\\vector>
+], 
+[],
+success="yes",
+)
+fi
+
+if test $success = yes; then
+AC_MSG_RESULT("no")
+else
+AC_MSG_RESULT("yes")
+AC_MSG_ERROR("STL Broken - Obtain newer version of GNU C Compiler")
+fi
+
+gsdlos=`uname | tr A-Z a-z`
+AC_SUBST(gsdlos)
+
+AC_MSG_CHECKING(for HP-UX needing gmalloc)
+if test "`(uname -s) 2> /dev/null`" = 'HP-UX'; then
+  AC_MSG_RESULT(yes)
+  LIBOBJS="$LIBOBJS gmalloc.o"
+  AC_DEFINE(HAVE_VALLOC)
+else
+  AC_MSG_RESULT(no)
+  AC_CHECK_FUNCS(valloc)
+fi
+
+fp_WITH_DMALLOC
+fp_WITH_REGEX
+
+# ---------------------------------------------------------------------------
+if test "$ac_cv_func_alloca" = 'no'; then
+dnl    support functions for alloca.c
+  LIBOBJS="$LIBOBJS xmalloc.o error.o"
+fi
+
+
+# ---------------------------------------------------------------------------
+
+AC_OUTPUT(packages/mg/Makefile packages/mg/src/text/Makefile packages/mg/src/images/Makefile 
+          packages/mg/src/scripts/Makefile packages/mg/lib/Makefile
+          src/hashfile/Makefile src/colservr/Makefile src/recpt/Makefile src/txt2db/Makefile 
+          src/db2txt/Makefile lib/Makefile collect/cstr/src/recpt/Makefile 
+          collect/niupepa/src/recpt/Makefile Makefile)
+
+
+
+
Index: /tags/gsdl-21/gsdl/etc/site.cfg.example
===================================================================
--- /tags/gsdl-21/gsdl/etc/site.cfg.example	(revision 699)
+++ /tags/gsdl-21/gsdl/etc/site.cfg.example	(revision 699)
@@ -0,0 +1,29 @@
+# You should save this file as site.cfg and edit it
+# to suit your site's configuration. This file may
+# be placed in the GSDLHOME/etc directory or in a 
+# particular collection's etc directory.
+
+
+# points to $GSDLHOME. This is only needed if it is
+# not the same as GSDL_GSDLHOME defined in site.h
+#gsdlhome    /home/rjmcnab/gsdl
+
+# used for cgi programs derived from the libinterface module
+httpprefix  /gsdl
+
+# used for cgi programs derived from the receptionist module.
+# It should contain the http address of the directory which
+# contains the images for the interface.
+httpimg     /gsdl/images
+
+# should contain the http address of this cgi script. This 
+# is not needed if the http server sets the environment variable 
+# SCRIPT_NAME
+#gwcgi       /cgi-bin/recpt
+
+# maxrequests is the most requests a fastcgi process
+# will serve before it exits. This can be set to a
+# low figure (like 1) while debugging and then set
+# to a high figure (like 10000) when everything is
+# working well.
+maxrequests 1
Index: /tags/gsdl-21/gsdl/lib/Makefile.in
===================================================================
--- /tags/gsdl-21/gsdl/lib/Makefile.in	(revision 699)
+++ /tags/gsdl-21/gsdl/lib/Makefile.in	(revision 699)
@@ -0,0 +1,76 @@
+###########################################################################
+#
+# Makefile -- lib
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999  The New Zealand Digital Library Project
+#
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+
+AR = ar
+CC = @CC@
+CCFLAGS = @CFLAGS@ -Wall -Wunused -pedantic -W -Woverloaded-virtual
+DEFS = @DEFS@
+INCLUDES = -I../packages/mg/lib
+RANLIB = @RANLIB@
+
+COMPILE = $(CC) -c $(CCFLAGS) $(DEFS) $(INCLUDES)
+
+.SUFFIXES:
+.SUFFIXES: .c .o
+.c.o:
+	$(COMPILE) $<
+
+.SUFFIXES: .cpp .o
+.cpp.o:
+	$(COMPILE) $< 
+
+
+HEADERS = \
+  cfgread.h      fileutil.h     gsdlunicode.h  text_t.h \
+  display.h      gsdlconf.h     gsdltimes.h
+
+SOURCES = \
+  cfgread.cpp      fileutil.cpp     text_t.cpp \
+  display.cpp      gsdlunicode.cpp  gsdltimes.cpp
+
+OBJECTS = \
+  cfgread.o        fileutil.o       text_t.o   \
+  display.o        gsdlunicode.o    gsdltimes.o
+
+
+all: gsdllib.a
+
+
+GSDLLIBOBJECTS = $(OBJECTS)
+
+gsdllib.a: $(GSDLLIBOBJECTS)
+	rm -f gsdllib.a
+	$(AR) cru gsdllib.a $(GSDLLIBOBJECTS)
+	$(RANLIB) gsdllib.a
+
+clean:
+	rm -f $(OBJECTS) gsdllib.a
+
+install:
+
+depend:
+	makedepend -Y -- $(DEFS) $(INCLUDES) $(CCFLAGS) -- $(SOURCES)
Index: /tags/gsdl-21/gsdl/lib/cfgread.cpp
===================================================================
--- /tags/gsdl-21/gsdl/lib/cfgread.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/lib/cfgread.cpp	(revision 699)
@@ -0,0 +1,120 @@
+/**********************************************************************
+ *
+ * cfgread.cpp -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.5  1999/09/07 04:57:41  sjboddie
+   added gpl notice
+
+   Revision 1.4  1999/02/02 09:54:26  rjmcnab
+
+   Changed the configuration file reader so that any space can be used to
+   seperate items and so that quotes can be used to contain items. I also
+   allowed a '\' to indicate that the line continues on the next line.
+
+   Revision 1.3  1999/01/12 01:50:57  rjmcnab
+
+   Standard header.
+
+   Revision 1.2  1999/01/08 02:33:13  rjmcnab
+
+   Added standard header to source files.
+
+ */
+
+
+#include "cfgread.h"
+
+
+// returns 0 on success, -1 on failure
+int read_cfg_line (ifstream &filein, text_tarray &values) {
+  //  outconvertclass text_t2ascii;
+
+  values.erase(values.begin(), values.end());
+
+  if (!filein.good()) return -1;
+
+  text_t curvalue;
+  char c, quote;
+
+  bool linecontinues = true;
+  filein.get(c);
+  while (!filein.eof() && linecontinues) {
+    // eat up all return characters
+    while (!filein.eof() && (c=='\n' || c=='\r')) { filein.get(c); }
+
+    // ignore comments
+    if (c == '#') {
+      while (!filein.eof() && c!='\n' && c!='\r') { filein.get(c); }
+    }
+
+    // deal with all the records on this line
+    while (!filein.eof() && c!='\n' && c!='\r') {
+      // ignore white space
+      while (!filein.eof() && (c==' ' || c=='\t')) { filein.get(c); }
+
+      // get the next record
+      if (c!='\n' && c!='\r') {
+	curvalue.clear();
+	quote = 0;
+
+	// see if this is a quoted record
+	if (c=='\'' || c=='\"') {
+	  quote = c;
+	  filein.get(c);
+	}
+
+	// get the value
+	while (!filein.eof() && c!='\n' && c!='\r') {
+	  if (quote != 0 && c == quote) { // end of quote
+	    filein.get(c);
+	    break;
+	  } else if (quote == 0 && (c == ' ' || c == '\t')) {
+	    break; // space seperator
+	  }
+	  curvalue.push_back(c);
+	  filein.get(c);
+	}
+
+	// see if the record continues on the next line
+	if ((c=='\n' || c=='\r') && curvalue=="\\") {
+	  linecontinues = true;
+	} else {
+	  //	  cerr << text_t2ascii << "\"" << curvalue << "\" ";
+	  
+	  values.push_back(curvalue);
+	  curvalue.clear();
+	  linecontinues = false;
+	}
+      }
+    }
+  }
+
+  //  cerr << "\n";
+
+  return 0;
+}
Index: /tags/gsdl-21/gsdl/lib/cfgread.h
===================================================================
--- /tags/gsdl-21/gsdl/lib/cfgread.h	(revision 699)
+++ /tags/gsdl-21/gsdl/lib/cfgread.h	(revision 699)
@@ -0,0 +1,47 @@
+/**********************************************************************
+ *
+ * cfgread.h -- reads in configuration files
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef CFGREAD_H
+#define CFGREAD_H
+
+#include "gsdlconf.h"
+#include "text_t.h"
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\fstream>
+#elif defined(GSDL_USE_IOS_H)
+#  include <fstream.h>
+#else
+#  include <fstream>
+#endif
+
+
+// returns 0 on success, -1 on failure
+int read_cfg_line (ifstream &filein, text_tarray &values);
+
+#endif
Index: /tags/gsdl-21/gsdl/lib/display.cpp
===================================================================
--- /tags/gsdl-21/gsdl/lib/display.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/lib/display.cpp	(revision 699)
@@ -0,0 +1,1702 @@
+/**********************************************************************
+ *
+ * display.cpp -- Context sensitive macro language
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.15  1999/09/24 02:25:35  rjmcnab
+   removed limitation on the number of arguments that a macro can have
+
+   Revision 1.14  1999/09/07 04:57:41  sjboddie
+   added gpl notice
+
+   Revision 1.13  1999/08/31 07:59:11  rjmcnab
+   Generalised spaces to unicode spaces, added ability to automatically detect
+   whether the input file is Unicode or UTF-8 and read the file in the
+   appropriate way, and improved the error messages slightly.
+
+   Revision 1.12  1999/07/21 20:46:12  rjmcnab
+   fixed small bug
+
+   Revision 1.11  1999/07/21 07:26:30  rjmcnab
+   Added more operators to _If_, changed quoting of parameters so that
+   ' can be used to quote but it is removed, and made slight optimisation
+   so that macros that don't contain other macros aren't expanded.
+
+   Revision 1.10  1999/03/01 20:39:54  sjboddie
+
+   Added eq and ne functionality to _If_
+
+   Revision 1.9  1999/03/01 01:18:09  sjboddie
+
+   Fixed bug in _If_ when value being tested was only one character
+   long (e.g. "0" or "1")
+
+   Revision 1.8  1999/02/28 23:15:29  rjmcnab
+
+   Made _If_ recognise 'false' and '0' as being false.
+
+   Revision 1.7  1999/02/08 01:26:11  rjmcnab
+
+   Improved the error reporting.
+
+   Revision 1.6  1999/01/19 08:30:23  rjmcnab
+
+   Added a method to determine whether a macro has been defined.
+
+   Revision 1.5  1999/01/19 01:38:12  rjmcnab
+
+   Made the source more portable.
+
+   Revision 1.4  1999/01/12 01:50:57  rjmcnab
+
+   Standard header.
+
+   Revision 1.3  1999/01/08 02:33:13  rjmcnab
+
+   Added standard header to source files.
+
+ */
+
+
+#include "display.h"
+#include "gsdlunicode.h"
+#include "unitool.h"
+#include <assert.h>
+
+// include to get NULL
+#include <stdlib.h>
+
+
+
+/////////////////////////////////////
+// misc classes
+/////////////////////////////////////
+
+
+// precedencetype defines a 'precedence value' for each parameter
+typedef map<text_t, double, lttext_t> precedencetype;
+
+
+// macro value structure
+struct mvalue
+{
+  text_t filename;
+  text_t value;
+};
+
+inline bool operator==(const mvalue &x, const mvalue &y) {
+  return (x.filename==y.filename && x.value==y.value);
+}
+
+inline bool operator<(const mvalue &x, const mvalue &y) {
+  return (x.filename<y.filename ||
+	  (x.filename==y.filename && x.filename<y.filename));
+}
+
+
+
+/////////////////////////////////////
+// stuff for defaultmacros_t
+/////////////////////////////////////
+  
+typedef map<text_t, mvalue, lttext_t> defparammap;
+typedef map<text_t, defparammap, lttext_t> defmacromap;
+typedef map<text_t, defmacromap, lttext_t> defpackagemap;
+
+// all methods in defaultmacros_t assume the parameters
+// and packages in a standard form and not empty
+class defaultmacros_t
+{
+protected:
+  defpackagemap macros;
+
+public:
+  typedef defpackagemap::const_iterator const_package_iterator;
+  typedef defpackagemap::size_type package_size_type;
+
+  // setmacro returns 0 if there was no error,
+  //            -1 if it redefined a macro
+  //            -2 if it hid a Global macro
+  //            -3 if it redefined a macro and hid a Global macro
+  //            -4 if either a package, macroname, or params were not supplied
+  int setmacro(const text_t &package,
+	  const text_t &macroname, 
+	  const text_t &params, 
+	  const text_t &filename, 
+	  const text_t &macrovalue);
+
+  void delmacro(const text_t &package,
+	      const text_t &macroname, 
+	      const text_t &params);
+
+  void clear () {macros.erase(macros.begin(), macros.end());}
+
+  // top level stuff
+  const_package_iterator package_begin () const {return macros.begin();}
+  const_package_iterator package_end () const {return macros.end();}
+  bool package_empty () const {return macros.empty();}
+  package_size_type package_size() const {return macros.size();}
+
+  // methods to find things
+  defmacromap *package_find (const text_t &packagename);
+  defparammap *macro_find (const text_t &packagename, 
+			   const text_t &macroname);
+  mvalue *parameter_find (const text_t &packagename, 
+			  const text_t &macroname,
+			  const text_t &parametername);
+};
+
+
+// setmacro returns 0 if there was no error,
+//            -1 if it redefined a macro
+//            -2 if it hid a Global macro
+//            -3 if it redefined a macro and hid a Global macro
+//            -4 if either a package, macroname, or params were not supplied
+int defaultmacros_t::setmacro(const text_t &package,
+			 const text_t &macroname, 
+			 const text_t &params, 
+			 const text_t &filename, 
+			 const text_t &macrovalue)
+{
+  paramhashtype paramhash;
+
+  defmacromap *macromapptr;
+  defparammap *parammapptr;
+  mvalue *mvalueptr;
+
+  int warning = 0;
+
+  // check to see if we would be hiding a Global macro with the 
+  // same name. Note: it doesn't matter what parameters are used
+  // a Global macro will still be hid by a non-Global one with
+  // the same name.
+  if ((package != "Global") && 
+      (macro_find ("Global", macroname) != NULL))
+    {
+      warning -= 2; // found the macroname
+    }
+
+  // define this macro
+  macromapptr = &(macros[package]); // -- package
+  parammapptr = &((*macromapptr)[macroname]); // -- macro name
+  // -- parameters
+  if ((*parammapptr).find(params) != (*parammapptr).end()) {
+    warning -= 1; // found the parameters
+  }
+  mvalueptr = &((*parammapptr)[params]);
+
+  // -- value
+  (*mvalueptr).filename = filename;
+  (*mvalueptr).value = macrovalue;
+
+  return warning;
+}
+
+void defaultmacros_t::delmacro(const text_t &package,
+			     const text_t &macroname, 
+			     const text_t &params)
+{
+  // make sure everything was supplied
+  if (package.empty() || macroname.empty() || params.empty()) return;
+
+  // find the package and macroname
+  defparammap *parammapptr = macro_find(package, macroname);
+  if (parammapptr == NULL) return;
+
+  // find the parameters
+  defparammap::iterator parammapit = parammapptr->find(params);
+  if (parammapit == parammapptr->end()) return;
+
+  // finally delete this element
+  parammapptr->erase(parammapit);
+}
+
+defmacromap *defaultmacros_t::package_find (const text_t &packagename)
+{
+  if (packagename.empty()) return NULL;
+
+  defpackagemap::iterator it = macros.find(packagename);
+  if (it == macros.end()) return NULL;
+
+  return &((*it).second);
+}
+
+defparammap *defaultmacros_t::macro_find (const text_t &packagename, 
+					       const text_t &macroname)
+{
+  defmacromap *macromapptr = package_find(packagename);
+  if (macromapptr == NULL || macroname.empty()) return NULL;
+
+  defmacromap::iterator it = (*macromapptr).find(macroname);
+  if (it == (*macromapptr).end()) return NULL;
+
+  return &((*it).second);
+}
+
+mvalue *defaultmacros_t::parameter_find (const text_t &packagename, 
+					 const text_t &macroname,
+					 const text_t &parametername)
+{
+  defparammap *parammapptr = macro_find(packagename, macroname);
+  if (parammapptr == NULL || parametername.empty()) return NULL;
+
+  defparammap::iterator it = (*parammapptr).find(parametername);
+  if (it == (*parammapptr).end()) return NULL;
+
+  return &((*it).second);
+}
+
+
+
+/////////////////////////////////////
+// stuff for currentmacros_t
+/////////////////////////////////////
+
+typedef map<text_t, mvalue, lttext_t> curmacromap;
+typedef map<text_t, curmacromap, lttext_t> curpackagemap;
+
+// all methods in currentmacros_t assume the parameters
+// and packages in a standard form and not empty
+class currentmacros_t 
+{
+protected:
+  curpackagemap macros;
+
+public:
+  typedef curpackagemap::const_iterator const_package_iterator;
+  typedef curpackagemap::size_type package_size_type;
+
+  // setmacro returns 0 if there was no error,
+  //            -1 if it redefined a macro
+  //            -4 if either a package, or macroname were not supplied
+  int setmacro(const text_t &package,
+	  const text_t &macroname, 
+	  const text_t &filename, 
+	  const text_t &macrovalue);
+
+  void delmacro(const text_t &package,
+		const text_t &macroname);
+
+  void clear () {macros.erase(macros.begin(), macros.end());}
+
+  // top level stuff
+  const_package_iterator package_begin () const {return macros.begin();}
+  const_package_iterator package_end () const {return macros.end();}
+  bool package_empty () const {return macros.empty();}
+  package_size_type package_size() const {return macros.size();}
+
+  // methods to find things
+  curmacromap *package_find (const text_t &packagename);
+  mvalue *macro_find (const text_t &packagename, 
+		      const text_t &macroname);
+};
+
+
+// setmacro returns 0 if there was no error,
+//            -1 if it redefined a macro
+//            -4 if either a package, or macroname were not supplied
+int currentmacros_t::setmacro(const text_t &package,
+			 const text_t &macroname, 
+			 const text_t &filename, 
+			 const text_t &macrovalue)
+{
+  int warning = 0;
+
+  // make sure everything was supplied
+  if (package.empty() || macroname.empty()) return -4;
+
+  // define this macro
+  curmacromap *macromapptr = &(macros[package]);
+  if ((*macromapptr).find(macroname) != (*macromapptr).end()) 
+    {
+      warning -= 1; // found the macroname
+    }
+  mvalue *mvalueptr = &((*macromapptr)[macroname]);
+
+  // -- value
+  (*mvalueptr).filename = filename;
+  (*mvalueptr).value = macrovalue;
+
+  return warning;
+}
+
+void currentmacros_t::delmacro(const text_t &package,
+			     const text_t &macroname)
+{
+  // make sure everything was supplied
+  if (package.empty() || macroname.empty()) return;
+
+  // find the package
+  curmacromap *macromapptr = package_find(package);
+  if (macromapptr == NULL) return;
+
+  // find the macroname
+  curmacromap::iterator macromapit = macromapptr->find(macroname);
+  if (macromapit == macromapptr->end()) return;
+
+  // finally delete this element
+  macromapptr->erase(macromapit);
+}
+
+curmacromap *currentmacros_t::package_find (const text_t &packagename)
+{
+  if (packagename.empty()) return NULL;
+
+  curpackagemap::iterator it = macros.find(packagename);
+  if (it == macros.end()) return NULL;
+
+  return &((*it).second);
+}
+
+mvalue *currentmacros_t::macro_find (const text_t &packagename, 
+				     const text_t &macroname)
+{
+  curmacromap *macromapptr = package_find(packagename);
+  if (macromapptr == NULL || macroname.empty()) return NULL;
+
+  curmacromap::iterator it = (*macromapptr).find(macroname);
+  if (it == (*macromapptr).end()) return NULL;
+
+  return &((*it).second);
+}
+
+
+
+
+/////////////////////////////////////
+// support functions
+/////////////////////////////////////
+
+// this calculation of precedence will make one occurance of a high
+// precedence item override many occurances of lower precedence items
+void calcprecedence (const text_t &precedstr, precedencetype &precedhash) 
+{
+  text_t param;
+  double multiple = 5.0;
+  double nextprec = multiple;
+
+  text_t::const_iterator here, end;
+
+  // set precedhash to an empty hash
+  precedhash.erase(precedhash.begin(), precedhash.end());
+
+  // iterate through paramstring ignoring space and extracting
+  // out key value pairs.
+  here = precedstr.begin();
+  end = precedstr.end();
+  while (here != end) 
+    {
+      // get the next parameter
+      param.clear(); // set param to an empty string
+      
+      while (here != end) 
+	{
+	  if (*here == ',') 
+	    {
+	      // found the end of the parameter
+	      here++;
+	      break;
+	    } 
+	  else if (*here == ' ') 
+	    {
+	      // do nothing (ignore spaces)
+	    } 
+	  else 
+	    {
+	      // character in parameter
+	      param.push_back (*here); // copy this character
+	    }
+	  
+	  here++;
+	}
+
+      // if a parameter was found insert its value
+      if (!param.empty()) 
+	{
+	  precedhash[param] = nextprec;
+	  nextprec *= multiple;
+	}
+    }
+}
+
+
+// decides how specific any given parameter is to the page parameters. Returns 
+// negative if any options are different or else the sum of the precedence for
+// the options which match. 
+double getspecificness(const paramhashtype &pageparams, 
+		       const paramhashtype &theseparams, 
+		       const precedencetype &precedhash) 
+{
+  double specificness = 0.0;
+  paramhashtype::const_iterator here, pagevalueit;
+  precedencetype::const_iterator precedit;
+  text_t thesekey, thesevalue, constignore("ignore");
+  
+  here = theseparams.begin();
+  while (here != theseparams.end()) 
+    {
+      thesekey = (*here).first;
+      thesevalue = (*here).second;
+      
+      if (thesekey == constignore) 
+	{
+	  // do nothing, this is a placeholder
+	} 
+      else 
+	{
+	  pagevalueit = pageparams.find(thesekey);
+	  if ((pagevalueit != pageparams.end()) && 
+	      ((*pagevalueit).second == thesevalue)) 
+	    {
+	      // this parameter fits, add its precedence value
+	      precedit = precedhash.find(thesekey);
+	      if (precedit == precedhash.end()) 
+		specificness += 1.0; // no precedence defined
+	      else 
+		  specificness += (*precedit).second;
+	    }
+	  else 
+	    {
+	      return -1.0; // did not match
+	    }
+	}
+
+      here++;
+    }
+  
+  return specificness;
+}
+
+
+void splitparams (const text_t &paramstring, paramhashtype &paramhash)
+{
+  text_t key;
+  text_t value;
+
+  text_t::const_iterator here, end;
+
+  // set paramhash to an empty hash
+  paramhash.erase(paramhash.begin(), paramhash.end());
+
+  // iterate through paramstring ignoring space and extracting
+  // out key value pairs.
+  here = paramstring.begin();
+  end  = paramstring.end();
+  while (here != end) 
+    {
+      // reset key and value
+      key.clear();
+      value.clear();
+
+      // get key
+      while (here != end) 
+	{
+	  if (*here == ',') 
+	    {
+	      // have a key with no value
+	      break;
+	    } 
+	  else if (*here == '=') 
+	    {
+	      // found the end of the key
+	      here ++; 
+	      break;
+	    } 
+	  else if (*here == ' ') 
+	    {
+	      // do nothing (ignore spaces)
+	    } 
+	  else 
+	    {
+	      // character in key
+	      key.push_back (*here); // copy this character
+	    }
+	  here ++;
+	}
+
+      // get value
+      while (here != end) 
+	{
+	  if (*here == '=') 
+	    {
+	      // multiple values for one key, ignore previous
+	      // values
+	      value.clear();
+	    } 
+	  else if (*here == ',') 
+	    {
+	      // found the end of the value
+	      here ++;
+	      break;
+	    } 
+	  else if (*here == ' ') 
+	    {
+	      // do nothing (ignore spaces)
+	    } 
+	  else 
+	    {
+	      // character in value
+	      value.push_back (*here); // copy this character
+	    }
+	  here ++;
+	}
+    
+      // if a key was found insert the key/value pair in 
+      // the map 
+      if (!key.empty()) 
+	{
+	  paramhash[key] = value;
+	}
+    }
+}
+
+
+void joinparams (const paramhashtype &paramhash, text_t &paramstring)
+{
+  //set paramstring to an empty string
+  paramstring.clear();
+
+  //iterate through the paramhash
+  paramhashtype::const_iterator thepair;
+  int firstparam = 1;
+
+  for (thepair=paramhash.begin();thepair!=paramhash.end(); thepair++)
+    {
+      if (!firstparam) paramstring.push_back(',');
+      firstparam = 0;
+      
+      // insert pairs of "key=value"
+      text_t thevalue;
+      paramstring.append((*thepair).first);
+      paramstring.push_back('=');
+      paramstring.append((*thepair).second);
+    }
+}
+
+
+
+/////////////////////////////////////
+// parsing support functions
+/////////////////////////////////////
+
+inline int my_isalpha (unsigned short c) 
+{
+  return ((c >= 'A' && c <= 'Z') ||
+	  (c >= 'a' && c <= 'z'));
+} 
+
+
+
+inline unsigned short my_ttnextchar (text_t::const_iterator &here, 
+				     text_t::const_iterator &end) 
+{
+  if (here != end) 
+    {
+      here++;
+      if (here != end) return (*here);
+    }
+  return '\0';
+}
+
+
+
+/////////////////////////////////////
+// methods for display
+/////////////////////////////////////
+
+// public methods for display
+
+displayclass::displayclass ()
+{
+  defaultmacros = new defaultmacros_t;
+  defaultfiles = new fileinfomap;
+  
+  orderparamlist = new paramspeclist;
+  currentmacros = new currentmacros_t;
+
+  outc = NULL;
+
+  logout = &cerr;
+}
+
+
+displayclass::~displayclass ()
+{
+  delete defaultmacros;
+  delete defaultfiles;
+  
+  delete orderparamlist;
+  delete currentmacros;
+}
+  
+// setdefaultmacro adds an entry to the list of default macros 
+// returns 0 if there was no error,
+//        -1 if it redefined a macro
+//        -2 if it hid a Global macro
+//        -3 if it redefined a macro and hid a Global macro
+//        -4 if no macroname was supplied
+int displayclass::setdefaultmacro (text_t package, const text_t &macroname, 
+			      text_t params, const text_t &macrovalue) 
+{
+  return setdefaultmacro (macroname, package, params, macrovalue, "memory");
+}
+
+
+
+// as we are using one character lookahead the
+// value of line might be off by one.
+// the input file must be in the utf-8 or unicode format
+// initially for each file isunicode should be set to 0 and 
+// bigendian should be set to 1
+// 0 will be returned when the end of the file has been found
+unsigned short my_uni_get (istream &fin, int &line, 
+			   int &isunicode, int &bigendian) {
+  unsigned short c = 0;
+  
+  if (isunicode) {
+    // unicode text
+    // get the next two characters
+    unsigned char c1 = 0, c2 = 0;
+    if (!fin.eof()) fin.get(c1);
+    if (!fin.eof()) fin.get(c2);
+    else c1 = 0;
+
+    // if they indicate the order get the next character
+    // otherwise just get these characters
+    if (c1 == 0xff && c2 == 0xfe) {
+      bigendian = 0;
+      c = my_uni_get (fin, line, isunicode, bigendian);
+    } else if (c1 == 0xfe && c2 == 0xff) {
+      bigendian = 1;
+      c = my_uni_get (fin, line, isunicode, bigendian);
+    } else c = (bigendian) ? (c1*256+c2) : (c2*256+c1);
+    
+  } else {
+    // utf-8 text
+    // how many characters we get depends on what we find
+    unsigned char c1 = 0, c2 = 0, c3 = 0;
+    while (!fin.eof()) {
+      fin.get(c1);
+      if (c1 == 0xfe || c1 == 0xff) {
+	// switch to unicode
+	isunicode = 1;
+	if (!fin.eof()) fin.get(c2);
+
+	if (c1 == 0xff && c2 == 0xfe) bigendian = 0;
+	else bigendian = 1;
+
+	c = my_uni_get (fin, line, isunicode, bigendian);
+	break;
+
+      } else if (c1 <= 0x7f) {
+	// one byte character
+	c = c1;
+	break;
+
+      } else if (c1 >= 0xc0 && c1 <= 0xdf) {
+	// two byte character
+	if (!fin.eof()) fin.get(c2);
+	c = ((c1 & 0x1f) << 6) + (c2 & 0x3f);
+	break;
+
+      } else if (c1 >= 0xe0 && c1 <= 0xef) {
+	// three byte character
+	if (!fin.eof()) fin.get(c2);
+	if (!fin.eof()) fin.get(c3);
+	c = ((c1 & 0xf) << 12) + ((c2 & 0x3f) << 6) + (c3 & 0x3f);
+	break;
+      }
+
+      // if we get here there was an error in the file, we should
+      // be able to recover from it however, maybe the file is in
+      // another encoding
+    }
+  }
+
+  if (c == '\n') line++;
+  return c;
+}
+
+
+
+// loads a default macro file (if it isn't already loaded)
+// returns 0 if didn't need to load the file (it was already loaded)
+//         1 if was (re)loaded
+//        -1 an error occurred while trying to load the file
+int displayclass::loaddefaultmacros (text_t thisfilename) {
+  // convert the filename to a C string
+  char *filenamestr = thisfilename.getcstr();
+  outconvertclass text_t2ascii;
+
+  // see if we need to open this file -- this isn't implemented yet
+  
+  // open the file
+  ifstream fin(filenamestr);
+  if (fin.fail()) return -1; // read failed
+
+  text_t package = "Global";
+  int line = 1;
+  int isunicode = 0, bigendian = 1;
+
+  // pre-fetch the next character
+  unsigned short c = my_uni_get(fin, line, isunicode, bigendian);
+
+  text_t macropackage, macroname, macroparameters, macrovalue;
+  int err; // for keeping track of whether an error occurred somewhere
+
+  while (!fin.eof()) {
+    // expect: white space, comment, "package", or macroname
+    if (is_unicode_space(c)) {
+      // found some white-space
+      c = my_uni_get(fin, line, isunicode, bigendian);
+
+    } else if (c == '#') {
+      // found the start of a comment
+      // skip all characters up to the end of the line
+      c = my_uni_get(fin, line, isunicode, bigendian); // skip the '#'
+      while (!fin.eof ()) {
+	if (c == '\n') break;
+	c = my_uni_get(fin, line, isunicode, bigendian);
+      }
+
+    } else if (c == 'p') {
+      // found the start of 'package' (hopefully)
+      // get everything up to the next space
+      text_t tmp;
+      while (!fin.eof() && my_isalpha(c)) {
+	tmp.push_back(c);
+	c = my_uni_get(fin, line, isunicode, bigendian);
+      }
+      // see if we have a package name
+      if (tmp == "package") {
+	// skip all white space
+	while (!fin.eof() && is_unicode_space(c)) 
+	  c = my_uni_get(fin, line, isunicode, bigendian);
+	
+	// get the package name
+	tmp.clear(); // init tmp
+	while (!fin.eof() && my_isalpha(c)) {
+	  tmp.push_back(c);
+	  c = my_uni_get(fin, line, isunicode, bigendian);
+	}
+	package = tmp;
+	if (package.empty()) package = "Global";
+
+      } else {
+	// error
+	if (logout != NULL) {
+	  (*logout) << text_t2ascii << "Expected 'package' on line " << line 
+		    << " of " << thisfilename << "\n";
+	}
+      }
+      
+    } else if (c == '_') {
+      // found the start of a macro (hopefully)
+      c = my_uni_get(fin, line, isunicode, bigendian); // skip the _
+
+      // init variables
+      err = 0;
+      macropackage = package;
+      macroname.clear(); // init macroname
+      macroparameters.clear(); // init macroname
+      macrovalue.clear(); // init macroname
+      
+      // get the macro name
+      while ((!fin.eof()) && (!is_unicode_space(c)) && 
+	     (c != '\\') && (c != '_') &&(c != ':') && 
+	     (macroname.size() < 80)) {
+	macroname.push_back(c);
+	c = my_uni_get(fin, line, isunicode, bigendian);
+      }
+	  
+      if (c == ':') {
+	// we actually had the macro package
+	c = my_uni_get(fin, line, isunicode, bigendian); // skip :
+	macropackage = macroname;
+	macroname.clear ();
+	
+	// get the macro name (honest!)
+	while ((!fin.eof()) && (!is_unicode_space(c)) && 
+	       (c != '\\') && (c != '_') &&(c != ':') && 
+	       (macroname.size() < 80)) {
+	  macroname.push_back(c);
+	  c = my_uni_get(fin, line, isunicode, bigendian);
+	}
+      }
+      
+      if (!err && c == '_') {
+	c = my_uni_get(fin, line, isunicode, bigendian); // skip the _
+	
+	// skip all white space
+	while (!fin.eof() && is_unicode_space(c)) c = my_uni_get(fin, line, isunicode, bigendian);
+      } else if (!err) err = 1;
+
+      // get the macro parameters (optional)
+      if (!err && c == '[') {
+	c = my_uni_get(fin, line, isunicode, bigendian); // skip the [
+	while ((!fin.eof()) && (c != '\n') && (c != '\\') && (c != ']')) {
+	  macroparameters.push_back(c);
+	  c = my_uni_get(fin, line, isunicode, bigendian);
+	}
+	
+	if (c == ']') {
+	  c = my_uni_get(fin, line, isunicode, bigendian); // skip the ]
+	  
+	  // skip all white space
+	  while (!fin.eof() && is_unicode_space(c)) c = my_uni_get(fin, line, isunicode, bigendian);
+	} 
+	else if (!err) err = 2;
+      }
+
+      // get the macro value
+      if (!err && c == '{') {
+	c = my_uni_get(fin, line, isunicode, bigendian); // skip the {
+	while ((!fin.eof()) && (c != '}')) {
+	  if (c == '\\') {
+	    macrovalue.push_back(c); // keep the '\'
+	    c = my_uni_get(fin, line, isunicode, bigendian); // store the *next* value regardless
+	    if (!fin.eof()) macrovalue.push_back(c);
+	    c = my_uni_get(fin, line, isunicode, bigendian);
+	  }
+	  macrovalue.push_back(c);
+	  c = my_uni_get(fin, line, isunicode, bigendian);
+	}
+	
+	if (c == '}') {
+	  c = my_uni_get(fin, line, isunicode, bigendian); // skip the }
+	  
+	  // define the macro
+	  err = setdefaultmacro (macropackage, macroname, macroparameters,
+				 thisfilename, macrovalue);
+	  if ((err == -1 || err == -3) && logout != NULL) {
+	    (*logout) << text_t2ascii << "Warning: redefinition of _" << 
+	      package << ":" << macroname << "_[" << macroparameters <<
+	      "] on line ";
+	    (*logout) << line;
+	    (*logout) << text_t2ascii << " of " << thisfilename << "\n";
+	    
+	  } else if (err == -2 && logout != NULL) {
+	    (*logout) << text_t2ascii << "Warning: _" << 
+	      package << ":" << macroname << "_[" << macroparameters <<
+	      "] on line ";
+	    (*logout) << line;
+	    (*logout) << text_t2ascii << " of " << 
+	      thisfilename << " hides a Global macro with the same name\n";
+
+	  } else if (err == -4 && logout != NULL) {
+	    (*logout) << text_t2ascii << "Error: macro name expected on line ";
+	    (*logout) << line ;
+	    (*logout) << text_t2ascii << " of " << thisfilename << "\n";
+	  }		  
+	  
+	  err = 0; // for the test below
+	}
+	else if (!err) err = 3;
+      }
+      else if (!err) err = 4;
+      
+      if (err) {
+	// found an error, skip to the end of the line
+	if (logout != NULL) {
+	  (*logout) << text_t2ascii << "Error: ";
+	  if (err == 1) (*logout) << text_t2ascii << "'_'";
+	  else if (err == 2) (*logout) << text_t2ascii << "']'";
+	  else if (err == 3) (*logout) << text_t2ascii << "'}'";
+	  else if (err == 4) (*logout) << text_t2ascii << "'{'";
+	  (*logout) << text_t2ascii << " expected on line ";
+	  (*logout) << line ;
+	  (*logout) << text_t2ascii << " of " << thisfilename << "\n";
+	}
+	while (!fin.eof ()) {
+	  if (c == '\n') break;
+	  c = my_uni_get(fin, line, isunicode, bigendian);
+	}
+      }
+      
+    } else {
+      // found an error, skip to the end of the line
+      if (logout != NULL) {
+	(*logout) << text_t2ascii << "Error: Unexpected input on line " << line 
+		  << " of " << thisfilename << "\n";
+      }
+      while (!fin.eof ()) {
+	if (c == '\n') break;
+	c = my_uni_get(fin, line, isunicode, bigendian);
+      }
+      
+    }
+  }
+  
+  fin.close ();
+  
+  // free up memory
+  delete filenamestr;
+  return 0;
+}
+
+
+// prepares to create a page.
+void displayclass::openpage (const text_t &thispageparams, 
+			const text_t &thisprecedence)
+{
+  // init variables
+  currentmacros->clear();
+
+  // reload any default macro files which have changed.
+  if (checkdefaultmacrofiles() < 0) 
+    {
+      // print error message
+    }
+
+  setpageparams (thispageparams, thisprecedence);
+}
+
+
+// changes the parameters for the current page.
+void displayclass::setpageparams (text_t thispageparams, 
+			     text_t thisprecedence)
+{
+  paramhashtype thissplitparams, splittheseparams;
+  precedencetype precedhash;
+  text_tset::iterator text_tsetit;
+  paramspec tmpps;
+
+  precedence = thisprecedence;
+  calcprecedence(thisprecedence, precedhash);
+
+  splitparams(thispageparams, thissplitparams);
+  joinparams(thissplitparams, thispageparams);
+  params = thispageparams;
+
+  // get a list of parameters with their specificness
+  orderparamlist->erase(orderparamlist->begin(), orderparamlist->end());
+  for (text_tsetit=allparams.begin();
+       text_tsetit!=allparams.end();
+       text_tsetit++) 
+    {
+      tmpps.param = (*text_tsetit);
+      splitparams(tmpps.param, splittheseparams);
+      tmpps.spec = getspecificness(thissplitparams, splittheseparams, precedhash);
+      if (tmpps.spec >= 0) 
+	{
+	  orderparamlist->push_back (tmpps);
+	}
+    }
+
+  // sort the list
+  sort(orderparamlist->begin(), orderparamlist->end());
+
+//   paramspeclist::iterator pshere = orderparamlist->begin();
+//   paramspeclist::iterator psend = orderparamlist->end();
+//   while (pshere != psend) 
+//     {
+//       cerr << text_t2ascii << "param=" << (*pshere).param;
+//       cerr << " spec=" << (int)((*pshere).spec) << "\n";
+//       pshere++;
+//     }
+}
+
+
+// overrides (or sets) a macro for the current page.
+// returns 0 if there was no error,
+//        -1 if it redefined a macro
+//        -4 if no macroname was supplied
+int displayclass::setmacro (const text_t &macroname, 
+		       text_t package, 
+		       const text_t &macrovalue)
+{
+  // make sure a macroname was supplied
+  if (macroname.empty()) return -4;
+
+  // make package "Global" if it doesn't point to anything yet
+  if (package.empty()) package = "Global";
+
+  // set the macro
+  return currentmacros->setmacro(package, macroname, "memory", macrovalue);
+}
+
+
+void displayclass::expandstring (const text_t &inputtext, text_t &outputtext)
+{
+  expandstring("", inputtext, outputtext);
+}
+
+
+void displayclass::expandstring (text_t package, const text_t &inputtext,
+				 text_t &outputtext, int recursiondepth)
+{
+  text_t macroname, macropackage, macroargs;
+  text_t::const_iterator tthere = inputtext.begin();
+  text_t::const_iterator ttend = inputtext.end();
+  unsigned short c;
+
+  if (package.empty()) package = "Global";
+
+  outputtext.clear();
+      
+  // use one-character lookahead
+  if (tthere != ttend) c = (*tthere);
+  while (tthere != ttend) 
+    {
+      if (c == '\\')
+	{
+	  // found an escape character
+	  c = my_ttnextchar (tthere, ttend); // skip the escape character
+	  if (tthere != ttend) 
+	    {
+	      if (c == 'n') outputtext.push_back('\n');
+	      else if (c == 't') outputtext.push_back('\t');
+	      else outputtext.push_back(c);
+	    }
+	  c = my_ttnextchar (tthere, ttend);
+	  
+	} 
+      else if (c == '_') 
+	{
+	  // might have found the start of a macro
+      
+	  // embedded macros (macros within the names
+	  // of other macros) are no longer supported -- Rodger.
+      
+	  // save the current state (in case we need to back out)
+	  text_t::const_iterator savedtthere = tthere;
+	  unsigned short savedc = c;
+	  
+	  macroname.clear();
+	  macropackage = package;
+	  macroargs.clear();
+      
+	  c = my_ttnextchar (tthere, ttend); // skip the '_'
+	  
+	  // get the macroname
+	  while (tthere != ttend && (!is_unicode_space(c)) && 
+		 (c != '\\') && (c != '_') &&(c != ':') && 
+		 (macroname.size() < 80)) 
+	    {
+	      macroname.push_back((unsigned char)c);
+	      c = my_ttnextchar (tthere, ttend);
+	    }
+      
+	  if (c == ':') 
+	    {
+	      // we actually had the macro package
+	      c = my_ttnextchar (tthere, ttend);
+	      macropackage = macroname;
+	      macroname.clear ();
+	
+	      // get the macro name (honest!)
+	      while ((tthere != ttend) && (!is_unicode_space(c)) && 
+		     (c != '\\') && (c != '_') &&(c != ':') && 
+		     (macroname.size() < 80)) 
+		{
+		  macroname.push_back((unsigned char)c);
+		  c = my_ttnextchar (tthere, ttend);
+		}
+	    }
+      
+	  if ((tthere != ttend) && (c == '_') &&
+	      (macroname.size() > 0)) 
+	    {
+	      // found a macro!!!! (maybe)
+	
+	      c = my_ttnextchar (tthere, ttend); // skip the '_'
+	
+	      // get the parameters (if there are any)
+	      if ((tthere != ttend) && (c == '(')) 
+		{
+		  c = my_ttnextchar (tthere, ttend); // skip '('
+	  
+		  // have to be careful of quotes
+		  unsigned short quote = '\0';
+		  while (tthere != ttend) 
+		    {
+		      if (c == '\\') 
+			{
+			  // have an escape character, save escape 
+			  // character and next character as well
+			  macroargs.push_back (c);
+			  c = my_ttnextchar (tthere, ttend); 
+	      
+			} 
+		      else if (quote == '\0') 
+			{
+			  // not in a quote at the moment
+			  if (c == ')') 
+			    {
+			      // found end of the arguments
+			      c = my_ttnextchar (tthere, ttend); // skip ')'
+			      break;
+		
+			    } 
+			  else if (c == '\'' || c == '\"') 
+			    {
+			      // found a quote
+			      quote = c;
+			    }
+	      
+			} 
+		      else 
+			{
+			  // we are in a quote
+			  if (c == quote) quote = '\0';
+			}
+	    
+		      // save this character
+		      if (tthere != ttend) macroargs.push_back (c);
+		      c = my_ttnextchar (tthere, ttend);
+		    }
+		}
+	
+	      //
+	      // we now have the macropackage, macroname, and macroargs
+	      //
+	
+	      // try to expand out this macro
+	      text_t expandedmacro;
+	      if (macro (macroname, macropackage, macroargs, expandedmacro, recursiondepth)) 
+		{
+		  // append the expanded macro
+		  outputtext += expandedmacro;
+		} 
+	      else
+		{
+		  // wasn't a macro
+		  tthere = savedtthere;
+		  c = savedc;
+		  outputtext.push_back(c); // the '_'
+		  c = my_ttnextchar (tthere, ttend);
+		}
+	
+
+	    }
+	  else
+	    {
+	      // wasn't a macro
+	      tthere = savedtthere;
+	      c = savedc;
+	      outputtext.push_back(c); // the '_'
+	      c = my_ttnextchar (tthere, ttend);
+	    }
+
+	}
+      else 
+	{
+	  // nothing of interest
+	  outputtext.push_back(c); // the '_'
+	  c = my_ttnextchar (tthere, ttend);
+	}
+    }
+}
+
+
+// protected methods for display
+
+// reloads any default macro files which have changed
+// returns 0 no errors occurred
+//        -1 an error occurred while trying to load one of the files
+int displayclass::checkdefaultmacrofiles ()
+{
+  // this isn't implemented yet
+  return 0;
+}
+
+// isdefaultmacro sees if there is an entry in the list of
+// default macros with the given package and macro name
+// returns 0 if no macros in the package or in the global package
+//           were found
+//         1 if no macros in the package were found but a macro
+//           in the global package was found
+//         2 if a macro in the given package was found
+int displayclass::isdefaultmacro (text_t package, const text_t &macroname) {
+  // make sure a macroname was supplied
+  if (macroname.empty()) return 0;
+
+  // make package "Global" if it doesn't point to anything yet
+  if (package.empty()) package = "Global";
+
+  // check the given package
+  if (defaultmacros->macro_find(package, macroname) != NULL) return 2;
+
+  // check the Global package
+  if (defaultmacros->macro_find("Global", macroname) != NULL) return 1;
+
+  return 0;
+}
+
+
+// setdefaultmacro adds an entry to the list of default macros 
+// returns 0 if there was no error,
+//        -1 if it redefined a macro
+//        -2 if it hid a Global macro
+//        -3 if it redefined a macro and hid a Global macro
+//        -4 if no macroname was supplied
+int displayclass::setdefaultmacro (text_t package, const text_t &macroname, 
+			      text_t params, const text_t &filename,
+			      const text_t &macrovalue) 
+{
+  // make sure a macroname was supplied
+  if (macroname.empty()) return -4;
+
+  // put the parameters in a standard form
+  paramhashtype paramhash;
+  if (params.empty()) params = "ignore=yes";
+  splitparams (params, paramhash);
+  joinparams (paramhash, params);
+
+  // make package "Global" if it doesn't point to anything yet
+  if (package.empty()) package = "Global";
+
+  // remember these parameters
+  allparams.insert (params);
+
+  // remember this filename (this part isn't finished yet -- Rodger).
+
+  // set the macro
+  return defaultmacros->setmacro(package, macroname, params, filename, macrovalue);
+}
+
+
+// evaluates a boolean expression
+// returns false if expr equals "" or "0". 
+// otherwise returns true *unless* expr is 
+// format "XXXX" eq/ne "dddd" in which case
+// the two quoted strings are compared
+bool displayclass::boolexpr (text_t package, const text_t &expr, int recursiondepth) {
+
+  if (expr.empty()) return false;
+
+  text_t expexpr;
+  if (package.empty()) package = "Global";  
+  expandstring (package, expr, expexpr, recursiondepth);
+
+  if (expexpr.empty() || expexpr == "0") return false;
+  if (expr[0] != '\"') return true;
+
+  // don't use expexpr while separating quoted parts of
+  // expression just in case expanded out macros contain
+  // quotes
+  text_t::const_iterator here = expr.begin();
+  text_t::const_iterator end = expr.end();
+
+  int quotecount = 0;
+  text_t string1, expstring1;
+  text_t string2, expstring2;
+  text_t op;
+  text_t combineop;
+  bool result = false; // an empty string is false
+  bool result2 = false;
+
+  while (here != end) {
+    // get a comparison
+    quotecount = 0;
+    string1.clear();
+    string2.clear();
+    op.clear();
+    while (here != end) {
+      if (*here == '"') quotecount++;
+      else if (quotecount == 1) string1.push_back(*here);
+      else if ((quotecount == 2) && !is_unicode_space (*here))
+	op.push_back(*here);
+      else if (quotecount == 3) string2.push_back(*here);
+      else if (quotecount >= 4) break;
+      here ++;
+    }
+    
+    expandstring (package, string1, expstring1, recursiondepth);
+    expandstring (package, string2, expstring2, recursiondepth);
+
+    // get next result
+    result2 = true; // any badly formatted string will return true
+    if (op == "eq") result2 = (expstring1 == expstring2);
+    else if (op == "ne") result2 = (expstring1 != expstring2);
+    else if (op == "gt") result2 = (expstring1 > expstring2);
+    else if (op == "ge") result2 = (expstring1 >= expstring2);
+    else if (op == "lt") result2 = (expstring1 < expstring2);
+    else if (op == "le") result2 = (expstring1 <= expstring2);
+    else if (op == "==") result2 = (expstring1.getint() == expstring2.getint());
+    else if (op == "!=") result2 = (expstring1.getint() != expstring2.getint());
+    else if (op == ">") result2 = (expstring1.getint() > expstring2.getint());
+    else if (op == ">=") result2 = (expstring1.getint() >= expstring2.getint());
+    else if (op == "<") result2 = (expstring1.getint() < expstring2.getint());
+    else if (op == "<=") result2 = (expstring1.getint() <= expstring2.getint());
+
+    // combine the results
+    if (combineop == "&&") result = (result && result2);
+    else if (combineop == "||") result = (result || result2);
+    else result = result2;
+
+    // get next combination operator
+    combineop.clear();
+    while (here != end && *here != '"') {
+      if (!is_unicode_space(*here)) combineop.push_back(*here);
+      here++;
+    }
+  }
+
+  return result;
+}
+
+
+// (recursively) expand out a macro
+// returns true  if the macro was found
+//         false if the macro was not found
+bool displayclass::macro (const text_t &macroname, text_t macropackage, 
+		     const text_t &macroparam, text_t &outputtext,
+		     int recursiondepth)
+{
+  outconvertclass text_t2ascii;
+  text_tlist splitmacroparam;
+  text_t::const_iterator hereit, endit;
+  text_t aparam;
+  unsigned short c, quote;
+
+  //  cerr << "r: " << recursiondepth << "\n";
+
+  // check for deep recursion
+  if (recursiondepth >= MAXRECURSIONDEPTH)
+    {
+      if (logout != NULL)
+	(*logout) << "Warning: deep recursion, limiting macro expansion\n";
+      return false;
+    }
+
+  // check the macropackage
+  if (macropackage.empty()) macropackage = "Global";
+
+  // get the parameters (but don't expand them)
+  if (macroparam.size() > 0) {
+    // set up for loop
+    hereit = macroparam.begin();
+    endit = macroparam.end();
+    if (hereit != endit) c = (*hereit);
+
+    while (hereit != endit) {
+      // get the next parameter
+      aparam.clear();
+      quote = '\0'; // not-quoted unless proven quoted
+      
+      // ignore initial whitespace
+      while ((hereit!=endit)&&is_unicode_space(c)) c=my_ttnextchar(hereit,endit);
+      
+      // look for the end of the parameter
+      while (hereit != endit) {
+	if (c == '\\') {
+	  // found escape character, also get next character
+	  aparam.push_back(c);
+	  c = my_ttnextchar (hereit, endit);
+	  if (hereit != endit) aparam.push_back(c);
+	  
+	} else if (quote=='\0' && (c=='\'' /*|| c=='"'*/)) {
+	  // found a quoted section
+	  quote = c;
+	  //	  aparam.push_back(c);
+
+	} else if (quote!='\0' && c==quote) {
+	  // found the end of a quote
+	  quote = '\0';
+	  //	  aparam.push_back(c);
+
+	} else if (quote=='\0' && c==',') {
+	  // found the end of a parameter
+	  c = my_ttnextchar (hereit, endit);
+	  break;
+
+	} else {
+	  // ordinary character
+	  aparam.push_back(c);
+	}
+
+	c = my_ttnextchar (hereit, endit);
+      }
+
+      // add this parameter to the list
+      splitmacroparam.push_back(aparam);
+    }
+  }
+
+  if (macroname == "If") {
+    // get the condition, then clause and else clause
+    text_tlist::iterator paramcond = splitmacroparam.begin();
+    text_tlist::iterator paramend = splitmacroparam.end();
+    text_tlist::iterator paramthen = paramend;
+    text_tlist::iterator paramelse = paramend;
+    
+    if (paramcond != paramend) {
+      paramthen = paramcond;
+      paramthen++;
+    }
+    if (paramthen != paramend) {
+      paramelse = paramthen;
+      paramelse++;
+    }
+    
+    // will always output something
+    outputtext.clear();
+    
+    // expand out the first parameter
+    if (paramcond != paramend) {
+      text_t tmpoutput;
+      expandstring (macropackage, *paramcond, tmpoutput, recursiondepth+1);
+      lc (tmpoutput);
+      
+      // test the expanded string
+      if (boolexpr (macropackage, *paramcond, recursiondepth+1)) {
+	//(tmpoutput.size()) && (tmpoutput != "false") && (tmpoutput != "0")) {
+	// true
+	if (paramthen != paramend)
+	  expandstring (macropackage, *paramthen, outputtext, recursiondepth+1);
+
+      } else {
+	// false
+	if (paramelse != paramend)
+	  expandstring (macropackage, *paramelse, outputtext, recursiondepth+1);
+      }
+    }
+    
+    return true;
+  }
+
+  // try and find this macro
+
+  // this list might be replaced by something a little more
+  // sophisticated in the future.
+  text_tlist packagelist;
+  packagelist.push_back (macropackage);
+  packagelist.push_back ("Global");
+
+  paramspeclist::iterator paramhere, paramend;
+  text_tlist::iterator packagehere = packagelist.begin();
+  text_tlist::iterator packageend = packagelist.end();
+  mvalue *macrovalue = NULL;
+  while ((macrovalue == NULL) && (packagehere != packageend)) 
+    {
+      // first look in the currentmacros
+      macrovalue = currentmacros->macro_find(*packagehere, macroname);
+      if (macrovalue == NULL)
+	macrovalue = currentmacros->macro_find("Style", macroname);
+
+      // look in the default macros
+      if (macrovalue == NULL) 
+	{
+	  // next look in the defaultmacros
+	  if (defaultmacros->macro_find (*packagehere, macroname) != NULL)
+	    {
+	      paramhere = orderparamlist->begin(); paramend = orderparamlist->end();
+	      while ((macrovalue == NULL) && (paramhere != paramend))
+		{
+// 		  cerr << text_t2ascii << "\npackage: " << *packagehere << "\n";
+// 		  cerr << text_t2ascii << "macroname: " << macroname << "\n";
+// 		  cerr << text_t2ascii << "param: " << (*paramhere).param << "\n";
+// 		  cerr << "spec: " << (int)((*paramhere).spec) << "\n";
+		  macrovalue = defaultmacros->parameter_find(*packagehere, macroname,
+							     (*paramhere).param);
+		  paramhere++;
+		}
+	    }
+	}
+
+      // and again in the package "Style"
+      if (macrovalue == NULL) 
+	{
+	  // next look in the defaultmacros
+	  if (defaultmacros->macro_find ("Style", macroname) != NULL)
+	    {
+	      paramhere = orderparamlist->begin(); paramend = orderparamlist->end();
+	      while ((macrovalue == NULL) && (paramhere != paramend))
+		{
+		  macrovalue = defaultmacros->parameter_find("Style", macroname,
+							     (*paramhere).param);
+		  paramhere++;
+		}
+	    }
+	}
+      if (macrovalue == NULL) packagehere++;
+    }
+
+  if (macrovalue != NULL)
+    {
+      // we found a macro
+
+      // replace all the option macros (_1_, _2_, ...) in the value of this macro
+      // and decide if we need to expand the value of this macro
+      bool needsexpansion = false;
+      text_t tempmacrovalue;
+      tempmacrovalue.clear();
+      hereit = macrovalue->value.begin();
+      endit = macrovalue->value.end();
+      if (hereit != endit) c = (*hereit);
+      while (hereit != endit)
+	{
+	  if (c == '\\')
+	    {
+	      // found an escape character
+	      needsexpansion = true;
+	      tempmacrovalue.push_back(c);
+	      c = my_ttnextchar (hereit, endit);
+	      if (hereit != endit) tempmacrovalue.push_back(c);
+	      c = my_ttnextchar (hereit, endit);
+	    }
+	  else if (c == '_')
+	    {
+	      text_t::const_iterator savedhereit = hereit;
+
+	      // attempt to get a number
+	      int argnum = 0;
+	      unsigned short digc = my_ttnextchar (hereit, endit);
+	      while (digc >= '0' && digc <= '9') {
+		argnum = argnum*10 + digc - '0';
+		digc = my_ttnextchar (hereit, endit);
+	      }
+	      if (digc == '_' && argnum > 0) {
+		// found an option macro, append the appropriate text
+		text_tlist::iterator ttlisthere = splitmacroparam.begin();
+		text_tlist::iterator ttlistend = splitmacroparam.end();
+		while ((argnum > 1) && (ttlisthere != ttlistend)) {
+		  argnum--;
+		  ttlisthere++;
+		}
+		if (ttlisthere != ttlistend) {
+		  tempmacrovalue.append(*ttlisthere);
+		  if (findchar((*ttlisthere).begin(), (*ttlisthere).end(), '_') != (*ttlisthere).end())
+		    needsexpansion = true;
+		}
+
+		c = my_ttnextchar (hereit, endit);
+
+	      } else {
+		// wasn't a option macro
+		needsexpansion = true;
+		tempmacrovalue.push_back(c);
+		hereit = savedhereit;
+		c = my_ttnextchar (hereit, endit);
+	      }
+	    }
+	  else
+	    {
+	      tempmacrovalue.push_back(c);
+	      c = my_ttnextchar (hereit, endit);
+	    }
+	}
+      
+      // recursively replace this macro (if we need to)
+      if (needsexpansion) {
+	expandstring (*packagehere, tempmacrovalue, outputtext, recursiondepth+1);
+      } else {
+	outputtext += tempmacrovalue;
+      }
+
+      return true;
+    }
+
+  if (logout != NULL) {
+    (*logout) << text_t2ascii << "Warning: _" << 
+      macropackage << ":" << macroname << "_ not found\n";
+  }
+  
+  return false; // didn't find the macro
+}
+
+
+void displayclass::printdefaultmacros ()
+{
+  defpackagemap::const_iterator packagemapit;
+  defmacromap::const_iterator macromapit;
+  defparammap::const_iterator parammapit;
+  const mvalue *mvalueptr;
+  outconvertclass text_t2ascii;
+
+  text_t packagename, macroname, macroparam;
+  
+  for (packagemapit = defaultmacros->package_begin();
+       packagemapit != defaultmacros->package_end();
+       packagemapit++) 
+    {
+      packagename = (*packagemapit).first;
+      //     cerr << "***package   : \"" << packagename << "\"\n";
+    
+      for (macromapit = (*packagemapit).second.begin();
+	   macromapit != (*packagemapit).second.end();
+	   macromapit++) 
+	{
+	  macroname = (*macromapit).first;
+	  //       cerr << "***macroname : \"" << macroname << "\"\n";
+
+	  for (parammapit = (*macromapit).second.begin();
+	       parammapit != (*macromapit).second.end();
+	       parammapit++) 
+	    {
+	      macroparam = (*parammapit).first;
+	      mvalueptr = &((*parammapit).second);
+	      cerr << text_t2ascii << "package   : \"" << packagename << "\"\n";
+	      cerr << text_t2ascii << "macroname : \"" << macroname << "\"\n";
+	      cerr << text_t2ascii << "parameters: [" << macroparam << "]\n";
+	      cerr << text_t2ascii << "filename  : \"" << mvalueptr->filename << "\"\n";
+	      cerr << text_t2ascii << "value     : {" << mvalueptr->value << "}\n\n";
+	    }
+	}
+    }
+}
+
+void displayclass::printallparams ()
+{
+  text_tset::iterator text_tsetit;
+  outconvertclass text_t2ascii;
+
+  cerr << text_t2ascii << "** allparams\n";
+  for (text_tsetit=allparams.begin();
+       text_tsetit!=allparams.end();
+       text_tsetit++) {
+    cerr << text_t2ascii << (*text_tsetit) << "\n";
+  }
+  cerr << text_t2ascii << "\n";
+}
+
+
+
+/////////////////////////////////////
+// stuff to do tricky output
+/////////////////////////////////////
+
+displayclass &operator<< (outconvertclass &theoutc, displayclass &display)
+{
+  display.setconvertclass (&theoutc);
+  return display;
+}
+
+displayclass &operator<< (displayclass &display, const text_t &t)
+{
+  outconvertclass *theoutc = display.getconvertclass();
+
+  if (theoutc == NULL) return display;
+  
+  text_t output;
+  display.expandstring (t, output);
+  (*theoutc) << output;
+
+  return display;
+}
+
Index: /tags/gsdl-21/gsdl/lib/display.h
===================================================================
--- /tags/gsdl-21/gsdl/lib/display.h	(revision 699)
+++ /tags/gsdl-21/gsdl/lib/display.h	(revision 699)
@@ -0,0 +1,265 @@
+/**********************************************************************
+ *
+ * display.h -- Context sensitive macro language
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef DISPLAY_H
+#define DISPLAY_H
+
+#include "gsdlconf.h"
+#include "text_t.h"
+
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\vector>
+#  include <ospace\std\list>
+#  include <ospace\std\set>
+#  include <ospace\std\map>
+#  include <ospace\std\algorithm>
+#elif defined(GSDL_USE_STL_H)
+#  include <vector.h>
+#  include <list.h>
+#  include <set.h>
+#  include <map.h>
+#  if defined(GSDL_USE_ALGO_H)
+#    include <algo.h>
+#  else
+#    include <algorithm.h>
+#  endif
+#else
+#  include <vector>
+#  include <list>
+#  include <set>
+#  include <map>
+#  include <algorithm>
+#endif
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\iostream>
+#  include <ospace\std\fstream>
+#elif defined(GSDL_USE_IOS_H)
+#  include <iostream.h>
+#  include <fstream.h>
+#else
+#  include <iostream>
+#  include <fstream>
+#endif
+
+
+// use the standard namespace
+#if defined(GSDL_USE_OBJECTSPACE)
+using namespace ospace::std;
+#elif !defined(GSDL_NAMESPACE_BROKEN)
+using namespace std;
+#endif
+
+
+
+// MAXRECURSIONDEPTH is a cutoff to catch
+// cyclic macros (a includes b and b includes a)
+#define MAXRECURSIONDEPTH 30
+
+
+// class prototypes
+class defaultmacros_t;
+class currentmacros_t;
+
+// a few supporting types
+// fileelement isn't finished yet, I have to find out
+// more about getting information about files
+struct fileinfoelement 
+{
+  int otherinfo;
+};
+
+inline bool operator==(const fileinfoelement &x, const fileinfoelement &y) {
+  return (x.otherinfo==y.otherinfo);
+}
+
+inline bool operator<(const fileinfoelement &x, const fileinfoelement &y) {
+  return (x.otherinfo<y.otherinfo);
+}
+
+typedef map<text_t, fileinfoelement, lttext_t> fileinfomap;
+
+
+typedef map<text_t, text_t, lttext_t> paramhashtype;
+
+
+struct paramspec 
+{
+  double spec;
+  text_t param;
+};
+
+inline bool operator==(const paramspec& x, const paramspec& y) 
+{
+  return ((x.spec == y.spec) && (x.param == y.param));
+}
+
+inline bool operator!=(const paramspec& x, const paramspec& y) 
+{
+  return ((x.spec != y.spec) || (x.param != y.param));
+}
+
+// note: paramspec is sorted by reverse spec and then param
+
+inline bool operator<(const paramspec& x, const paramspec& y) 
+{
+  return ((x.spec > y.spec) ||
+	  ((x.spec == y.spec) && (x.param < y.param)));
+}
+
+inline bool operator>(const paramspec& x, const paramspec& y) 
+{
+  return ((x.spec < y.spec) ||
+	  ((x.spec == y.spec) && (x.param > y.param)));
+}
+
+typedef vector<paramspec> paramspeclist;
+
+
+// NOTE: when macros are used within text they should have
+// underscores '_' on both sides, however, in calls to
+// setdefaultmacro and setmacro they should not have the
+// underscores. So you might have
+//
+// setmacro ("aname", "query", " some text ");
+// expandstring ("_aname_", expandedtext);
+//
+// because the input to expandstring is a block of text, not
+// a macroname
+
+class displayclass 
+{
+public:
+  displayclass ();
+  ~displayclass ();
+  
+  // isdefaultmacro sees if there is an entry in the list of
+  // default macros with the given package and macro name
+  // returns 0 if no macros in the package or in the global package
+  //           were found
+  //         1 if no macros in the package were found but a macro
+  //           in the global package was found
+  //         2 if a macro in the given package was found
+  int isdefaultmacro (text_t package, const text_t &macroname);
+
+  // setdefaultmacro adds an entry to the list of default macros 
+  // returns 0 if there was no error,
+  //        -1 if it redefined a macro
+  //        -2 if it hid a Global macro
+  //        -3 if it redefined a macro and hid a Global macro
+  //        -4 if no macroname was supplied
+  int setdefaultmacro (text_t package, const text_t &macroname, 
+		       text_t params, const text_t &macrovalue);
+  
+  // loads a default macro file (if it isn't already loaded)
+  // returns 0 if didn't need to load the file (it was already loaded)
+  //         1 if was (re)loaded
+  //        -1 an error occurred while trying to load the file
+  int loaddefaultmacros (text_t thisfilename);
+  
+  // prepares to create a page: deletes all macros set with
+  // 'setmacro', checks all the default macro files to see
+  // if any need reloading, and sets the page parameters.
+  void openpage (const text_t &thispageparams, 
+		 const text_t &thisprecedence);
+  
+  // changes the parameters for the current page.
+  void setpageparams (text_t thispageparams, 
+		      text_t thisprecedence);
+  
+  // overrides (or sets) a macro for the current page.
+  // returns 0 if there was no error,
+  //        -1 if it redefined a macro
+  //        -4 if no macroname was supplied
+  int setmacro (const text_t &macroname, 
+		text_t package, 
+		const text_t &macrovalue);
+  
+  void expandstring (const text_t &inputtext, text_t &outputtext);
+  void expandstring (text_t package, const text_t &inputtext,
+		     text_t &outputtext, int recursiondepth = 0);
+
+  // these functions are not ment to be used directly, they are used to permit 
+  // concise stream output like:
+  // cout << text_t2ascii << display << "_amacro_" << "_anothermacro_";
+  void setconvertclass (outconvertclass *theoutc) {outc = theoutc;}
+  outconvertclass *getconvertclass () {return outc;}
+
+  // say where any error logging goes, this can be NULL
+  // if no error logging is desired
+  void setlogout (ostream *thelogout) {logout = thelogout;}
+
+  // debug stuff
+  void printdefaultmacros ();
+  void printallparams ();
+
+protected:
+  // special variables to permit trickly output
+  outconvertclass *outc;
+
+  // general variables
+  text_tset allparams;  // possible parameter combinations
+  defaultmacros_t *defaultmacros;
+  fileinfomap *defaultfiles;
+  
+  // variables to do with a page expansion
+  text_t params;
+  text_t precedence;
+  paramspeclist *orderparamlist; // ordered by specificness
+  currentmacros_t *currentmacros; // only macros set by setmacro
+
+  // logging variable
+  ostream *logout;
+
+  // reloads any default macro files which have changed
+  // returns 0 no errors occurred
+  //        -1 an error occurred while trying to load one of the files
+  int checkdefaultmacrofiles ();
+
+  int setdefaultmacro (text_t package, const text_t &macroname, 
+		       text_t params, const text_t &filename,
+		       const text_t &macrovalue); 
+
+  // evaluates a boolean expression
+  bool boolexpr (text_t package, const text_t &expr, int recursiondepth);
+
+  // (recursively) expand out a macro
+  // returns true  if the macro was found
+  //         false if the macro was not found
+  bool macro (const text_t &macroname, text_t macropackage, 
+	      const text_t &macroparam, text_t &outputtext,
+	      int recursiondepth);
+};
+
+
+displayclass &operator<< (outconvertclass &theoutc, displayclass &display);
+displayclass &operator<< (displayclass &display, const text_t &t);
+
+#endif
+
Index: /tags/gsdl-21/gsdl/lib/fileutil.cpp
===================================================================
--- /tags/gsdl-21/gsdl/lib/fileutil.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/lib/fileutil.cpp	(revision 699)
@@ -0,0 +1,135 @@
+/**********************************************************************
+ *
+ * fileutil.cpp -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.8  1999/09/07 04:57:42  sjboddie
+   added gpl notice
+
+   Revision 1.7  1999/08/31 08:00:43  rjmcnab
+   Fixed a windows specific error in the file_exists function.
+
+   Revision 1.6  1999/04/06 22:16:49  rjmcnab
+   Added more options for filename_cat.
+
+   Revision 1.5  1999/01/19 01:38:14  rjmcnab
+
+   Made the source more portable.
+
+   Revision 1.4  1999/01/12 01:50:59  rjmcnab
+
+   Standard header.
+
+   Revision 1.3  1999/01/08 02:33:14  rjmcnab
+
+   Added standard header to source files.
+
+ */
+
+
+#include "fileutil.h"
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\iostream>
+#  include <ospace\std\fstream>
+#elif defined(GSDL_USE_IOS_H)
+#  include <iostream.h>
+#  include <fstream.h>
+#else
+#  include <iostream>
+#  include <fstream>
+#endif
+
+
+// returns the proper concatenation of the two paths
+text_t filename_cat (text_t path1, text_t path2) {
+  text_t::iterator here;
+  text_t::iterator begin;
+  text_t::iterator end;
+
+  // make sure there is just one slash, of the correct type,
+  // at the end of path1 (unless path1 is an empty string).
+  if (!path1.empty()) {
+    // remove all trailing slashes
+    here = path1.end();
+    here--;
+    begin = path1.begin();
+    while (here != begin && (*here == '/' || *here == '\\')) {
+      here--;
+    }
+    here++;
+    path1.erase(here,path1.end());
+
+    // add one final slash
+#ifdef __WIN32__
+    path1 += "\\";
+#else
+    path1 += "/";
+#endif 
+  }
+
+  // remove all slashes from the start of path2
+  here = path2.begin();
+  end = path2.end();
+  while (here != end && (*here == '/' || *here == '\\')) {
+    here++;
+  }
+  path2.erase (path2.begin(), here);
+  
+  // return the concatenation of the two strings
+  return path1 + path2;
+}
+
+text_t filename_cat (text_t path1, text_t path2, text_t path3) {
+  return filename_cat(filename_cat(path1,path2),path3);
+}
+
+text_t filename_cat (text_t path1, text_t path2, text_t path3, text_t path4) {
+  return filename_cat(filename_cat(filename_cat(path1,path2),path3),path4);
+}
+
+text_t filename_cat (text_t path1, text_t path2, text_t path3, text_t path4, 
+		     text_t path5) {
+  return filename_cat(filename_cat(filename_cat(filename_cat(path1,path2),path3),
+				   path4),path5);
+}
+
+// returns true if filename can be opened
+bool file_exists (const text_t &filename) {
+  char *cstr = filename.getcstr();
+  ifstream filestream (cstr, ios::in | ios::nocreate);
+  delete cstr;
+
+  if (filestream) {
+    // file exists
+    filestream.close ();
+    return true;
+  }
+
+  // file does not exist
+  return false;
+}
Index: /tags/gsdl-21/gsdl/lib/fileutil.h
===================================================================
--- /tags/gsdl-21/gsdl/lib/fileutil.h	(revision 699)
+++ /tags/gsdl-21/gsdl/lib/fileutil.h	(revision 699)
@@ -0,0 +1,46 @@
+/**********************************************************************
+ *
+ * fileutil.h -- a few useful file utilities
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef FILEUTIL_H
+#define FILEUTIL_H
+
+#include "text_t.h"
+
+
+// returns the proper concatenation of the two paths
+text_t filename_cat (text_t path1, text_t path2);
+text_t filename_cat (text_t path1, text_t path2, text_t path3);
+text_t filename_cat (text_t path1, text_t path2, text_t path3, text_t path4);
+text_t filename_cat (text_t path1, text_t path2, text_t path3, text_t path4, 
+		     text_t path5);
+
+// returns true if filename can be opened
+bool file_exists (const text_t &filename);
+
+
+#endif
Index: /tags/gsdl-21/gsdl/lib/gsdlconf.h
===================================================================
--- /tags/gsdl-21/gsdl/lib/gsdlconf.h	(revision 699)
+++ /tags/gsdl-21/gsdl/lib/gsdlconf.h	(revision 699)
@@ -0,0 +1,135 @@
+/**********************************************************************
+ *
+ * gsdlconf.h -- system specific includes
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef GSDLCONF_H
+#define GSDLCONF_H
+
+
+// Note: all macros used in the gsdl software (except the _H macros) should
+// now start with GSDL_
+
+// GSDL_NEED_STRINGS_H should be defined if some string functions
+// needed (like strcasecmp) are in strings.h
+#if defined(__IBMCPP__)
+#  define GSDL_NEED_STRINGS_H
+#endif
+
+// GSDL_USE_IOS_H should be defined if the .h extension needs to
+// be used for the io stream headers
+#if defined(__GNUG__) || defined(__IBMCPP__)
+#  define GSDL_USE_IOS_H 1
+#endif
+
+// GSDL_USE_STL_H should be defined if the .h extension needs to
+// be used for the stl header files
+#if defined(__IBMCPP__)
+#  define GSDL_USE_STL_H 1
+#endif
+
+// GSDL_USE_ALGO_H should be defined if algo.h should be used 
+// instead of algorithm.h
+#if defined(__IBMCPP__)
+#  define GSDL_USE_ALGO_H 1
+#endif
+
+// GSDL_NAMESPACE_BROKEN should be defined if namespaces should
+// not be used
+#if defined(__GNUG__) || defined(__IBMCPP__)
+#  define GSDL_NAMESPACE_BROKEN 1
+#endif
+
+// GSDL_USE_OBJECTSPACE should now be used instead of USE_OBJECTSPACE
+#if defined(USE_OBJECTSPACE)
+#  define GSDL_USE_OBJECTSPACE
+#endif
+
+// GSDL_NEED_DESTROY_USHORT should be defined if the compiler needs
+// a definition for void destroy(short unsigned int *)
+#if defined(__IBMCPP__)
+# define GSDL_NEED_DESTROY_USHORT
+#endif
+
+
+// file locking
+
+#if defined(__WIN32__)
+#include <io.h>
+#include <sys/locking.h>
+#define GSDL_GET_FILEDESC(str) str.fd()
+#define GSDL_UNLOCK_FILE(fd) _locking(fd, LK_UNLCK, 200)
+#define GSDL_LOCK_FILE(fd) lock_val = _locking(fd, LK_NBLCK, 200)
+
+#else
+#define GSDL_GET_FILEDESC(str) str.filedesc()
+
+#if GSDL_USE_FLOCK
+#include <sys/lock.h>
+
+#ifndef LOCK_SH
+#define LOCK_SH	1
+#endif
+
+#ifndef LOCK_EX
+#define LOCK_EX	2
+#endif
+
+#ifndef LOCK_NB
+#define LOCK_NB 4
+#endif
+
+#ifndef LOCK_UN
+#define LOCK_UN 8
+#endif
+
+#define GSDL_UNLOCK_FILE(fd) flock (fd, LOCK_UN)
+#define GSDL_LOCK_FILE(fd) lock_val = flock (fd, LOCK_EX + LOCK_NB)
+
+#else
+// use fcntl
+#include <fcntl.h>
+#define GSDL_UNLOCK_FILE(fd) \
+	{					\
+	  struct flock flock;			\
+	  flock.l_type = F_UNLCK;		\
+	  flock.l_whence = SEEK_SET;		\
+	  flock.l_start = flock.l_len = 0L;	\
+	  fcntl (fd, F_SETLK, &flock);	\
+	}
+#define GSDL_LOCK_FILE(fd) \
+	{					\
+	  struct flock flock;			\
+	  flock.l_type = F_WRLCK;		\
+	  flock.l_whence = SEEK_SET;			\
+	  flock.l_start = flock.l_len = 0L;	\
+	  lock_val = fcntl (fd, F_SETLK, &flock);	\
+	}
+
+#endif
+#endif
+
+#endif
Index: /tags/gsdl-21/gsdl/lib/gsdlhome.h.example
===================================================================
--- /tags/gsdl-21/gsdl/lib/gsdlhome.h.example	(revision 699)
+++ /tags/gsdl-21/gsdl/lib/gsdlhome.h.example	(revision 699)
@@ -0,0 +1,9 @@
+#ifndef GSDLHOME_H
+#define GSDLHOME_H
+
+// this define is now in its own header so that
+// we end up with less dependences on it.
+#define GSDL_GSDLHOME "XXXX"
+
+#endif
+
Index: /tags/gsdl-21/gsdl/lib/gsdltimes.cpp
===================================================================
--- /tags/gsdl-21/gsdl/lib/gsdltimes.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/lib/gsdltimes.cpp	(revision 699)
@@ -0,0 +1,128 @@
+/**********************************************************************
+ *
+ * gsdltimes.cpp -- functions to deal with time
+ * Copyright (C) 1999  DigiLib Systems Limited, New Zealand
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.2  1999/09/07 04:57:42  sjboddie
+   added gpl notice
+
+   Revision 1.1  1999/07/11 10:52:56  rjmcnab
+   Initial revision.
+
+ */
+
+
+#include "gsdltimes.h"
+
+
+// returns "" if an error occurs
+text_t time2text (time_t time) {
+  tm *tm_ptr = gmtime(&time);
+  if (tm_ptr == NULL) return "";
+
+  // year
+  text_t timestr(tm_ptr->tm_year+1900);
+  timestr.push_back('/');
+
+  // month (note the +1)
+  if ((tm_ptr->tm_mon+1) < 10) timestr.push_back('0');
+  timestr += (tm_ptr->tm_mon+1);
+  timestr.push_back('/');
+
+  // day
+  if (tm_ptr->tm_mday < 10) timestr.push_back('0');
+  timestr += tm_ptr->tm_mday;
+  timestr.push_back(' ');
+
+  // hour
+  if (tm_ptr->tm_hour < 10) timestr.push_back('0');
+  timestr += tm_ptr->tm_hour;
+  timestr.push_back(':');
+
+  // minutes
+  if (tm_ptr->tm_min < 10) timestr.push_back('0');
+  timestr += tm_ptr->tm_min;
+  timestr.push_back(':');
+
+  // seconds
+  if (tm_ptr->tm_sec < 10) timestr.push_back('0');
+  timestr += tm_ptr->tm_sec;
+
+  return timestr;
+}
+
+// returns -1 if an error occurs
+time_t text2time (const text_t &timestr) {
+  tm timetm;
+  text_t tempstr;
+  
+  text_t::const_iterator time_here = timestr.begin();
+  text_t::const_iterator time_end = timestr.end();
+
+  // year
+  if (time_here == time_end) return (time_t) -1;
+  time_here = getdelimitstr (time_here, time_end, '/', tempstr);
+  timetm.tm_year = tempstr.getint() - 1900;
+  if (timetm.tm_year < 0) return (time_t) -1;
+
+  // month (note the -1)
+  if (time_here == time_end) return (time_t) -1;
+  time_here = getdelimitstr (time_here, time_end, '/', tempstr);
+  timetm.tm_mon = tempstr.getint() - 1;
+  if (timetm.tm_mon < 0 || timetm.tm_mon > 11)
+    return (time_t) -1;
+  
+  // day
+  if (time_here == time_end) return (time_t) -1;
+  time_here = getdelimitstr (time_here, time_end, ' ', tempstr);
+  timetm.tm_mday = tempstr.getint();
+  if (timetm.tm_mday < 1 || timetm.tm_mday > 31)
+    return (time_t) -1;
+  
+  // hour
+  if (time_here == time_end) return (time_t) -1;
+  time_here = getdelimitstr (time_here, time_end, ':', tempstr);
+  timetm.tm_hour = tempstr.getint();
+  if (timetm.tm_hour < 0 || timetm.tm_hour > 23)
+    return (time_t) -1;
+  
+  // min
+  if (time_here == time_end) return (time_t) -1;
+  time_here = getdelimitstr (time_here, time_end, ':', tempstr);
+  timetm.tm_min = tempstr.getint();
+  if (timetm.tm_min < 0 || timetm.tm_min > 59)
+    return (time_t) -1;
+  
+  // sec
+  if (time_here == time_end) return (time_t) -1;
+  time_here = getdelimitstr (time_here, time_end, ':', tempstr);
+  timetm.tm_sec = tempstr.getint();
+  if (timetm.tm_sec < 0 || timetm.tm_sec > 59)
+    return (time_t) -1;
+  
+  return mktime (&timetm);
+}
Index: /tags/gsdl-21/gsdl/lib/gsdltimes.h
===================================================================
--- /tags/gsdl-21/gsdl/lib/gsdltimes.h	(revision 699)
+++ /tags/gsdl-21/gsdl/lib/gsdltimes.h	(revision 699)
@@ -0,0 +1,43 @@
+/**********************************************************************
+ *
+ * gsdltimes.h -- functions to deal with time
+ * Copyright (C) 1999  DigiLib Systems Limited, New Zealand
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef GSDLTIMES_H
+#define GSDLTIMES_H
+
+#include <time.h>
+#include "text_t.h"
+
+// dates are stored in the format YYYY/MM/DD hh:mm:ss
+
+// returns "" if an error occurs
+text_t time2text (time_t time);
+
+// returns -1 if an error occurs
+time_t text2time (const text_t &timestr);
+
+#endif
Index: /tags/gsdl-21/gsdl/lib/gsdlunicode.cpp
===================================================================
--- /tags/gsdl-21/gsdl/lib/gsdlunicode.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/lib/gsdlunicode.cpp	(revision 699)
@@ -0,0 +1,548 @@
+/**********************************************************************
+ *
+ * gsdlunicode.cpp -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.10  1999/09/07 04:57:43  sjboddie
+   added gpl notice
+
+   Revision 1.9  1999/07/21 07:23:17  rjmcnab
+   Added setmapfile function to map conversion utilities so the map file
+   does not need to be loaded when map conversion object is created.
+
+   Revision 1.8  1999/07/01 04:03:45  rjmcnab
+   Optimised utf8inconvertclass::convert slightly.
+
+   Revision 1.7  1999/06/30 04:59:03  rjmcnab
+   Added a to_utf8 function that takes iterators as input.
+
+   Revision 1.6  1999/06/26 01:05:04  rjmcnab
+   No real changes.
+
+   Revision 1.5  1999/01/12 01:50:59  rjmcnab
+
+   Standard header.
+
+   Revision 1.4  1999/01/08 02:33:15  rjmcnab
+
+   Added standard header to source files.
+
+ */
+
+
+#include "gsdlunicode.h"
+
+
+// unitool is currently in mg, if mg is not being used it should
+// be moved into GSDLHOME/lib 
+#include "unitool.h"
+
+#include "fileutil.h"
+
+#include <stdio.h>
+
+
+
+// converts a unicode encode text_t string to a utf-8
+// encoded text_t string
+text_t to_utf8 (text_t::const_iterator here, text_t::const_iterator end) {
+  text_t out;
+
+  unsigned char thischar[MAXUTF8CHARLEN];
+  int i, charlen;
+
+  while (here != end) {
+    charlen = output_utf8_char (*here, thischar, &thischar[MAXUTF8CHARLEN-1]);
+    for (i=0; i<charlen; i++) out.push_back(thischar[i]);
+    here++;
+  }
+
+  return out;
+}
+
+// converts a utf-8 encoded text_t string to a unicode
+// encoded text_t string
+text_t to_uni (const text_t &in) {
+  text_t out;
+  unsigned char *in_cstr = (unsigned char *)in.getcstr();
+  unsigned char *here = in_cstr;
+  unsigned char *end = in_cstr;
+
+  unsigned short unichar;
+  int charlen = 0;
+
+  // get the last valid character in the string
+  while (*end != '\0') end++;
+  end--;
+
+  while ((charlen = parse_utf8_char (here, end, &unichar)) > 0) {
+    out.push_back(unichar);
+    here += charlen;
+  }
+
+  delete in_cstr;
+
+  return out;
+}
+
+
+
+utf8inconvertclass::utf8inconvertclass () {
+  utf8buflen = 0;
+}
+
+void utf8inconvertclass::reset () {
+  start = NULL;
+  len = 0;
+  utf8buflen=0;
+}
+
+void utf8inconvertclass::convert (text_t &output, status_t &status) {
+  output.clear();
+  output.reserve (len/3);
+  
+  if (start == NULL || len == 0) {
+    if (utf8buflen == 0) status = finished;
+    else status = stopped;
+    return;
+  }
+
+  // don't want any funny sign conversions happening
+  unsigned char *here = (unsigned char *)start;
+  unsigned char *end = here+len-1;
+  unsigned short c;
+  size_t realcharlen;
+
+  size_t charlen = getutf8charlen ();
+  while (len > 0) {
+    if (charlen == 0) {
+      // start parsing a new character
+      utf8buflen = 0;
+
+      // fast common case
+      while (len > 3) {
+  	realcharlen = parse_utf8_char (here, end, &c);
+  	output.push_back (c);
+  	here += realcharlen;
+  	len -= realcharlen;
+      }
+
+      utf8buf[utf8buflen++] = *here;
+      ++here;
+      --len;
+      charlen = getutf8charlen ();
+
+    } else if (utf8buflen < charlen) {
+      // assumes charlen is always less than MAXUTF8CHARLEN
+      utf8buf[utf8buflen++] = *here;
+      ++here;
+      --len;
+    }
+
+    if (utf8buflen == charlen) {
+      // got a complete character
+      realcharlen = parse_utf8_char (utf8buf, &utf8buf[utf8buflen-1], &c);
+      output.push_back (c);
+      
+      // move any unparsed characters. If an error occurred some of
+      // the characters might be unused.
+      int i;
+      int diff = utf8buflen - realcharlen;
+      for (i=0; i < diff; i++) utf8buf[i] = utf8buf[i+diff];
+      utf8buflen = diff;
+      charlen = getutf8charlen ();
+    }
+  }
+
+  start = (char *)here; // save current position
+
+  if (utf8buflen == 0) status = finished;
+  else status = stopped;
+}
+
+
+// returns the length that the current contents of the 
+// utf8buf should be
+size_t utf8inconvertclass::getutf8charlen () {
+  if (utf8buflen == 0) return 0;
+
+  // one byte character
+  if (utf8buf[0] < 0x80) return 1;
+
+  // error, is not the start of a utf-8 character
+  if (utf8buf[0] < 0xc0) return 1;
+
+  // two bute character
+  if (utf8buf[0] < 0xe0) return 2;
+
+  // three byte character
+  if (utf8buf[0] < 0xf0) return 3;
+
+  // error, character too long for unicode
+  return 1;
+}
+
+
+void utf8outconvertclass::reset () {
+  input = NULL;
+  outs = NULL;
+  utf8buflen = 0;
+  utf8bufhere = 0;
+}
+
+// note that convert does not null-terminate the
+// output array of characters
+void utf8outconvertclass::convert (char *output, size_t maxlen, 
+				   size_t &len, status_t &status) {
+  if (input == NULL || output == NULL) {
+    if (utf8buflen == 0) status = finished;
+    else status = unfinished;
+    return;
+  }
+
+  // don't want any funny sign conversions happening
+  unsigned char *uoutput = (unsigned char *)output;
+  text_t::iterator textend = input->end();
+  len = 0;
+  while (len < maxlen) {
+    // empty the contents of the internal buffer
+    if (utf8buflen > 0) {
+      while (len < maxlen && utf8bufhere < utf8buflen) {
+	*uoutput = utf8buf[utf8bufhere];
+	uoutput++;
+	len++;
+	utf8bufhere++;
+      }
+
+      if (utf8bufhere == utf8buflen) {
+	utf8bufhere = 0;
+	utf8buflen = 0;
+      }
+    }
+
+    // fill up the buffer with the next character
+    if (utf8buflen == 0) {
+      if (texthere == textend) break; // finished!
+      if (!rzws || (*texthere != 0x200b))
+	utf8buflen = output_utf8_char (*texthere, utf8buf, 
+				       &utf8buf[MAXUTF8CHARLEN-1]);
+      texthere++;
+      utf8bufhere = 0;
+    }
+  }
+  
+  if (texthere == textend && utf8buflen == 0) status = finished;
+  else status = unfinished;
+}
+
+
+
+
+
+
+mapdata_t::mapdata_t () {
+  int i;
+
+  // reset all the map ptrs to be NULL
+  for (i=0; i<256; i++) {
+    ptrs[i] = (unsigned short *)NULL;
+  }
+
+  // say nothing has been loaded
+  loaded = false;
+}
+
+
+mapconvert::mapconvert () {
+  absentc = 0;
+}
+
+// setmapfile will cause loadmapfile to be called when conversion is
+// needed
+bool mapconvert::setmapfile (const text_t &thegsdlhome, const text_t &theencoding, 
+			     unsigned short theabsentc) {
+  // check to see if the mapfile has been already loaded
+  if (mapdata.loaded && gsdlhome == thegsdlhome &&
+      encoding == theencoding && absentc == theabsentc) 
+    return true;
+
+  unloadmapfile ();
+  gsdlhome = thegsdlhome;
+  encoding = theencoding;
+  absentc = theabsentc;
+  
+  return true;
+}
+
+
+
+// loadmapfile should be called before any conversion is done
+bool mapconvert::loadmapfile (const text_t &thegsdlhome, 
+			      const text_t &theencoding, 
+			      unsigned short theabsentc) {
+  FILE *mapfilein = (FILE *)NULL;
+
+  // check to see if the mapfile has been already loaded
+  if (mapdata.loaded && gsdlhome == thegsdlhome &&
+      encoding == theencoding && absentc == theabsentc) 
+    return true;
+
+  unloadmapfile ();
+  gsdlhome = thegsdlhome;
+  encoding = theencoding;
+  absentc = theabsentc;
+
+  // open the map file
+  text_t filename = filename_cat (gsdlhome, "unicode");
+  filename = filename_cat (filename, encoding);
+  filename += ".ump";
+  char *cfilename = filename.getcstr();
+  if (cfilename == (char *)NULL) return false;
+  mapfilein = fopen(cfilename, "rb");
+  delete cfilename;
+
+  if (mapfilein == (FILE *)NULL) return false;
+
+  unsigned char c, n1, n2;
+  unsigned short *arrptr;
+  int i;
+  c = fgetc (mapfilein);
+  while (!feof (mapfilein)) {
+    if (mapdata.ptrs[c] == (unsigned short *)NULL) {
+      // allocate a new array
+      arrptr = new unsigned short[256];
+      mapdata.ptrs[c] = arrptr;
+    } else arrptr = mapdata.ptrs[c];
+
+    // clear the array
+    for (i=0; i<256; i++) arrptr[i] = 0;
+    
+    // read in this block
+    n1 = fgetc (mapfilein);
+    n2 = fgetc (mapfilein);
+    i=0;
+    while (!feof (mapfilein)) {
+      arrptr[i] = ((unsigned short)n1 << 8) | (unsigned short)n2;
+
+      i++;
+      if (i >= 256) break;
+      n1 = fgetc (mapfilein);
+      n2 = fgetc (mapfilein);
+    }
+
+    c = fgetc (mapfilein);
+  }
+
+  mapdata.loaded = true;
+
+  return true;
+}
+
+void mapconvert::unloadmapfile () {
+  if (!mapdata.loaded) return;
+
+  int i;
+  for (i=0; i<256; i++) {
+    if (mapdata.ptrs[i] != (unsigned short *)NULL) {
+      delete [] mapdata.ptrs[i];
+      mapdata.ptrs[i] = (unsigned short *)NULL;
+    }
+  }
+
+  mapdata.loaded = false;
+}
+
+
+unsigned short mapconvert::convert (unsigned short c) {
+  if (!mapdata.loaded) {
+    if (!gsdlhome.empty() && !encoding.empty() &&
+	loadmapfile (gsdlhome, encoding, absentc)) {
+      // do nothing, successfully loaded database
+    } else return absentc;
+  }
+
+  if (c == 0) return 0; // 0 always maps to 0...
+
+  unsigned short n1 = c >> 8;
+  unsigned short n2 = c & 0xff;
+
+  unsigned short *arrptr = mapdata.ptrs[n1];
+  if (arrptr == (unsigned short *)NULL) return absentc;
+
+  if (arrptr[n2] == 0) return absentc;
+  return arrptr[n2];
+}
+
+text_t mapconvert::convert (const text_t &instr) {
+  if (!mapdata.loaded) return absentc;
+
+  text_t outstr;
+  text_t::const_iterator here = instr.begin();
+  text_t::const_iterator end = instr.end();
+
+  while (here != end) {
+    outstr.push_back(this->convert(*here));
+    here++;
+  }
+  
+  return outstr;
+}
+
+
+
+
+mapinconvertclass::mapinconvertclass () {
+  mapbuflen = 0;
+}
+
+void mapinconvertclass::reset () {
+  start = NULL;
+  len = 0;
+  mapbuflen=0;
+}
+
+void mapinconvertclass::convert (text_t &output, status_t &status) {
+  output.clear();
+
+  if (start == NULL || len == 0) {
+    if (mapbuflen == 0) status = finished;
+    else status = stopped;
+    return;
+  }
+
+  // don't want any funny sign conversions happening
+  unsigned char *here = (unsigned char *)start;
+
+  size_t charlen = getmapcharlen ();
+  while (len > 0) {
+    if (charlen == 0) {
+      // start parsing a new character
+      mapbuflen = 0;
+      mapbuf[mapbuflen++] = *here;
+      ++here;
+      --len;
+      charlen = getmapcharlen ();
+
+    } else if (mapbuflen < charlen) {
+      // assumes charlen is always less than MAXMAPCHARLEN
+      mapbuf[mapbuflen++] = *here;
+      ++here;
+      --len;
+    }
+
+    if (mapbuflen == charlen) {
+      // got a complete character
+      if (charlen == 1) {
+	// ascii character
+	output.push_back (mapbuf[0]);
+
+      } else {
+	// two byte character
+	output.push_back (converter.convert(((unsigned short)mapbuf[0] << 8) |
+					    (unsigned short)mapbuf[1]));
+      }
+
+      mapbuflen = 0;
+      charlen = 0;
+    }
+  }
+
+  start = (char *)here; // save current position
+
+  if (mapbuflen == 0) status = finished;
+  else status = stopped;
+}
+
+
+
+mapoutconvertclass::mapoutconvertclass () {
+  mapbuflen=0; 
+  mapbufhere=0;
+}
+
+void mapoutconvertclass::reset () {
+  input = NULL;
+  outs = NULL;
+  mapbuflen = 0;
+  mapbufhere = 0;
+}
+
+// note that convert does not null-terminate the
+// output array of characters
+void mapoutconvertclass::convert (char *output, size_t maxlen, 
+				 size_t &len, status_t &status) {
+  unsigned short outc;
+
+  if (input == NULL || output == NULL) {
+    if (mapbuflen == 0) status = finished;
+    else status = unfinished;
+    return;
+  }
+
+  // don't want any funny sign conversions happening
+  unsigned char *uoutput = (unsigned char *)output;
+  text_t::iterator textend = input->end();
+  len = 0;
+  while (len < maxlen) {
+    // empty the contents of the internal buffer
+    if (mapbuflen > 0) {
+      while (len < maxlen && mapbufhere < mapbuflen) {
+	*uoutput = mapbuf[mapbufhere];
+	uoutput++;
+	len++;
+	mapbufhere++;
+      }
+
+      if (mapbufhere == mapbuflen) {
+	mapbufhere = 0;
+	mapbuflen = 0;
+      }
+    }
+
+    // fill up the buffer with the next character
+    if (mapbuflen == 0) {
+      if (texthere == textend) break; // finished!
+      if (!rzws || (*texthere != 0x200b)) {
+	if (*texthere < 0x80) {
+	  mapbuf[0] = (unsigned char)*texthere;
+	  mapbuflen = 1;
+	} else {
+	  outc = converter.convert (*texthere);
+	  mapbuf[0] = (unsigned char)(outc >> 8);
+	  mapbuf[1] = (unsigned char)(outc & 0xff);
+	  mapbuflen = 2;
+	}
+      }
+
+      texthere++;
+      mapbufhere = 0;
+    }
+  }
+  
+  if (texthere == textend && mapbuflen == 0) status = finished;
+  else status = unfinished;
+}
Index: /tags/gsdl-21/gsdl/lib/gsdlunicode.h
===================================================================
--- /tags/gsdl-21/gsdl/lib/gsdlunicode.h	(revision 699)
+++ /tags/gsdl-21/gsdl/lib/gsdlunicode.h	(revision 699)
@@ -0,0 +1,217 @@
+/**********************************************************************
+ *
+ * gsdlunicode.h -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef GSDLUNICODE_H
+#define GSDLUNICODE_H
+
+#include "text_t.h"
+
+
+// converts a unicode encode text_t string to a utf-8
+// encoded text_t string
+text_t to_utf8 (text_t::const_iterator here, text_t::const_iterator end);
+inline text_t to_utf8 (const text_t &in) {return to_utf8 (in.begin(), in.end());}
+
+// converts a utf-8 encoded text_t string to a unicode
+// encoded text_t string
+text_t to_uni (const text_t &in);
+
+
+
+#define MAXUTF8CHARLEN 3
+
+// convert from a utf-8 char stream to the text_t class
+class utf8inconvertclass : public inconvertclass {
+public:
+  utf8inconvertclass();
+  void reset ();
+  void convert (text_t &output, status_t &status);
+
+protected:
+  // buffer to hold unconverted characters in a stream
+  unsigned char utf8buf[MAXUTF8CHARLEN];
+  size_t utf8buflen;
+
+  // returns the length that the current contents of the 
+  // utf8buf should be
+  size_t getutf8charlen ();
+};
+
+
+// This class provides the option of removing zero width
+// spaces (U+200B) during the output. By default this
+// option is turned off. The functionality is actually
+// implemented by the sub-classes, this class just provides
+// the framework for these classes. 
+//
+// Note: by convention reset() should not reset the rzws flag.
+class rzwsoutconvertclass : public outconvertclass {
+public:
+  rzwsoutconvertclass () {rzws = 0;};
+  void set_rzws (int new_rzws) {rzws = new_rzws;};
+
+protected:
+  int rzws;
+};
+
+
+// Convert from a text_t class to a utf-8 char stream
+class utf8outconvertclass : public rzwsoutconvertclass {
+public:
+  utf8outconvertclass () {utf8buflen=0; utf8bufhere=0;};
+  void reset ();
+  // note that convert does not null-terminate the
+  // output array of characters
+  void convert (char *output, size_t maxlen, 
+		size_t &len, status_t &status);
+
+protected:
+  unsigned char utf8buf[MAXUTF8CHARLEN];
+  size_t utf8buflen;
+  size_t utf8bufhere;
+};
+
+
+// mapdata_t is used by mapconvert to hold the map file data
+class mapdata_t {
+public:
+  mapdata_t();
+  bool loaded;
+  unsigned short *ptrs[256];
+};
+
+// mapconvert is used in situations where conversion is best
+// done using a map file. The mapfile should reside in 
+// gsdlhome/unicode.
+class mapconvert {
+public:
+  mapconvert ();
+  ~mapconvert () {unloadmapfile();};
+
+  // setmapfile will cause loadmapfile to be called when conversion is
+  // needed
+  bool setmapfile (const text_t &thegsdlhome, const text_t &theencoding, 
+		   unsigned short theabsentc);
+
+  // loadmapfile should be called before any conversion is done
+  bool loadmapfile (const text_t &thegsdlhome, const text_t &theencoding, 
+		    unsigned short theabsentc);
+  void unloadmapfile ();
+
+  unsigned short convert (unsigned short c);
+
+  // note that this version of convert has different semantics to
+  // the convertclass version.
+  text_t convert (const text_t &instr);
+
+protected:
+  text_t gsdlhome;
+  text_t encoding;
+  unsigned short absentc;
+  mapdata_t mapdata;
+};
+
+
+
+#define MAXMAPCHARLEN 2
+
+// convert from a gb char stream to the unicode text_t class
+class mapinconvertclass : public inconvertclass {
+public:
+  mapinconvertclass();
+  virtual ~mapinconvertclass() {};
+
+  // setmapfile will cause loadmapfile to be called when conversion is needed
+  bool setmapfile (const text_t &thegsdlhome, const text_t &theencoding, 
+		    unsigned short theabsentc) {
+    return converter.setmapfile (thegsdlhome, theencoding, theabsentc);
+  };
+
+  // loadmapfile should be called before any conversion takes
+  // place
+  bool loadmapfile (const text_t &thegsdlhome, const text_t &theencoding, 
+		    unsigned short theabsentc) {
+    return converter.loadmapfile (thegsdlhome, theencoding, theabsentc);
+  };
+
+  void reset ();
+  void convert (text_t &output, status_t &status);
+
+protected:
+  // buffer to hold unconverted characters in a stream
+  unsigned char mapbuf[MAXMAPCHARLEN];
+  size_t mapbuflen;
+
+  // note: multiple instances of mapinconvert class are expensive
+  // as each will have its own copy of the map file data. This
+  // could be reduced by making map2unimap static, but then it
+  // wouldn't be thread safe.
+  mapconvert converter;
+
+  // returns the length that the current contents of the 
+  // mapbuf should be
+  inline size_t getmapcharlen () {
+    if (mapbuflen == 0) return 0;
+    if (mapbuf[0] < 0x80) return 1;
+    return 2;
+  }
+};
+
+
+// Convert from a text_t class to a map char stream
+class mapoutconvertclass : public rzwsoutconvertclass {
+public:
+  mapoutconvertclass ();
+  virtual ~mapoutconvertclass() {};
+
+  // setmapfile will cause loadmapfile to be called when conversion is needed
+  bool setmapfile (const text_t &thegsdlhome, const text_t &theencoding, 
+		    unsigned short theabsentc) {
+    return converter.setmapfile (thegsdlhome, theencoding, theabsentc);
+  };
+
+  // loadmapfile should be called before any conversion takes
+  // place
+  bool loadmapfile (const text_t &thegsdlhome, const text_t &theencoding, 
+		    unsigned short theabsentc) {
+    return converter.loadmapfile (thegsdlhome, theencoding, theabsentc);
+  };
+
+  void reset ();
+  void convert (char *output, size_t maxlen, 
+		size_t &len, status_t &status);
+
+protected:
+  unsigned char mapbuf[MAXMAPCHARLEN];
+  size_t mapbuflen;
+  size_t mapbufhere;
+
+  mapconvert converter;
+};
+
+#endif
Index: /tags/gsdl-21/gsdl/lib/text_t.cpp
===================================================================
--- /tags/gsdl-21/gsdl/lib/text_t.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/lib/text_t.cpp	(revision 699)
@@ -0,0 +1,770 @@
+/**********************************************************************
+ *
+ * text_t.cpp -- a simple 16-bit character string class
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.15  1999/10/14 22:52:39  sjboddie
+   joinchar can join using text_t string now too
+
+   Revision 1.14  1999/09/24 02:30:03  rjmcnab
+   added function has_unicode_letdig
+
+   Revision 1.13  1999/09/07 04:57:43  sjboddie
+   added gpl notice
+
+   Revision 1.12  1999/08/31 08:04:41  rjmcnab
+   Fixed a small but hard to find bug in getcarr
+
+   Revision 1.11  1999/07/01 04:05:09  rjmcnab
+   Optimised append functions slightly and added a reserve function.
+
+   Revision 1.10  1999/04/26 03:58:03  sjboddie
+   added is_number function
+
+   Revision 1.9  1999/04/06 22:17:24  rjmcnab
+   Added splits and joins using text_tset.
+
+   Revision 1.8  1999/02/28 23:14:41  rjmcnab
+
+   Added uc and lc to convert to uppercase and lowercase.
+
+   Revision 1.7  1999/02/21 22:26:39  rjmcnab
+
+   Made getint() a constant function.
+
+   Revision 1.6  1999/02/03 01:13:26  sjboddie
+
+   Got interface to handle subcollections and language subcollections -
+   committed changes made to some of the collections
+
+   Revision 1.5  1999/01/19 01:38:14  rjmcnab
+
+   Made the source more portable.
+
+   Revision 1.4  1999/01/12 01:51:00  rjmcnab
+
+   Standard header.
+
+   Revision 1.3  1999/01/08 02:33:16  rjmcnab
+
+   Added standard header to source files.
+
+ */
+
+
+#include "text_t.h"
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\algorithm>
+#elif defined(GSDL_USE_STL_H)
+#  if defined(GSDL_USE_ALGO_H)
+#    include <algo.h>
+#  else
+#    include <algorithm.h>
+#  endif
+#else
+#  include <algorithm>
+#endif
+
+
+#include "unitool.h"
+
+////////////////////////////////////
+// text_t methods
+////////////////////////////////////
+
+text_t::text_t () 
+{
+  setencoding(0);
+  clear ();
+}
+
+text_t::text_t (int i) 
+{
+  setencoding(0);
+  clear ();
+  appendint (i);
+}
+
+text_t::text_t (char *s) 
+{ 
+  setencoding(0);
+  clear ();
+  appendcstr (s);
+}
+
+void text_t::append (const text_t &t) 
+{
+  text.insert(text.end(), t.begin(), t.end());
+  //  const_iterator here, end=t.end();
+  //  for (here=t.begin(); here!=end;here++)
+  //    {
+  //      text.push_back(*here);
+  //    }
+}
+
+void text_t::appendrange (iterator first, iterator last) 
+{
+  text.insert(text.end(), first, last);
+  //  while (first != last) 
+  //  {
+  //    text.push_back (*first);
+  //    first++;
+  //  }
+}
+
+void text_t::appendrange (const_iterator first, const_iterator last) 
+{
+  text.insert(text.end(), first, last);
+  //  while (first != last) 
+  //  {
+  //    text.push_back (*first);
+  //    first++;
+  //  }
+}
+
+void text_t::appendint (int i)
+{
+  // deal with zeros and negatives
+  if (i == 0) 
+    {
+      text.push_back('0');
+      return;
+    }
+  else if (i < 0)
+    {
+      text.push_back('-');
+      i *= -1;
+    }
+
+  // get a buffer for the conversion
+  int maxbuflen = sizeof(int)*3;
+  char *buf = new char[maxbuflen];
+  int len = 0;
+  
+  // get the number in reverse
+  while (i > 0)
+    {
+      buf[len++] = '0'+ (i%10);
+      i = i/10;
+    }
+
+  // reverse the number
+  while (len > 0)
+    {
+      text.push_back(buf[--len]);
+    }
+
+  delete buf;
+}
+
+int text_t::getint () const
+{
+  int i = 0;
+  int mult = 1; // become -1 for negative numbers
+
+  const_iterator here = text.begin();
+  const_iterator end = text.end();
+  
+  // do plus and minus signs
+  if (here != end)
+    {
+      if (*here == '-')
+	{
+	  mult = -1;
+	  here++;
+	}
+      else if (*here == '+')
+	{
+	  mult = 1;
+	  here++;
+	}
+    }
+
+  // deal with the number
+  while ((here != end) && (*here >= '0') && (*here <= '9'))
+    {
+      i = 10*i + (*here - '0');
+      here++;
+    }
+
+  i *= mult;
+  return i;
+}
+
+
+
+void text_t::appendcarr (char *s, size_type len)
+{
+  unsigned char *us = (unsigned char *)s;
+  while (len > 0) 
+    {
+      text.push_back (*us); // append this character
+      us++;
+      len--;
+    }
+}
+
+void text_t::appendcstr (char *s) 
+{
+  unsigned char *us = (unsigned char *)s;
+  while (*us != '\0') 
+    {
+      text.push_back (*us); // append this character
+      us++;
+    }
+}
+
+
+// strings returned from getcarr and getcstr become the callers 
+// responsibility and should be deallocated with "delete"
+
+char *text_t::getcarr(size_type &len) const
+{
+  unsigned char *cstr = new unsigned char[size()];
+  len = 0;
+
+  const_iterator ithere = begin();
+  const_iterator itend = end();
+  while (ithere != itend)
+    {
+      if (*ithere < 256) cstr[len] = (unsigned char)(*ithere);
+      else {
+	// put a space or a question mark depending on what
+	// the character is. Question marks tell the user that
+	// they are missing some information.
+	if (is_unicode_space (*ithere)) cstr[len] = ' ';
+	else cstr[len] = '?';
+      }
+      len++;
+      ithere++;
+    }
+
+  return (char *)cstr;
+}
+
+char *text_t::getcstr() const
+{
+  unsigned char *cstr = new unsigned char[size() + 1];
+  const_iterator ithere = begin();
+  const_iterator itend = end();
+  int len = 0;
+
+  while (ithere != itend)
+    {
+      if (*ithere < 256) cstr[len] = (unsigned char)(*ithere);
+      else {
+	// put a space or a question mark depending on what
+	// the character is. Question marks tell the user that
+	// they are missing some information.
+	if (is_unicode_space (*ithere)) cstr[len] = ' ';
+	else cstr[len] = '?';
+      }
+      len++;
+      ithere++;
+    }
+
+  cstr[len] = '\0';
+
+  return (char *)cstr;
+}
+
+
+// general functions which work on text_ts
+
+// find a character within a range
+text_t::const_iterator findchar (text_t::const_iterator first, text_t::const_iterator last, 
+				 unsigned short c)
+{
+  while (first != last)
+    {
+      if (*first == c) break;
+      first++;
+    }
+  return first;
+}
+
+text_t::iterator findchar (text_t::iterator first, text_t::iterator last, 
+			   unsigned short c)
+{
+  while (first != last)
+    {
+      if (*first == c) break;
+      first++;
+    }
+  return first;
+}
+
+// get a string up to the next delimiter (which is skipped)
+text_t::const_iterator getdelimitstr (text_t::const_iterator first, 
+				      text_t::const_iterator last,
+				      unsigned short c, text_t &outstr)
+{
+  text_t::const_iterator here = first;
+  here = findchar (first, last, c);
+  outstr.clear();
+  outstr.appendrange (first, here);
+  if (here != last) here++; // skip c
+  return here;
+}
+
+text_t::iterator getdelimitstr (text_t::iterator first, text_t::iterator last,
+				unsigned short c, text_t &outstr)
+{
+  text_t::iterator here = first;
+  here = findchar (first, last, c);
+  outstr.clear();
+  outstr.appendrange (first, here);
+  if (here != last) here++; // skip c
+  return here;
+}
+
+// split a string with a character
+void splitchar (text_t::const_iterator first, text_t::const_iterator last,
+		unsigned short c, text_tset &outlist)
+{
+  outlist.erase(outlist.begin(), outlist.end());
+
+  text_t t;
+
+  while (first != last)
+    {
+      first = getdelimitstr (first, last, c, t);
+      outlist.insert (t);
+    }
+}
+
+void splitchar (text_t::const_iterator first, text_t::const_iterator last,
+		unsigned short c, text_tlist &outlist)
+{
+  outlist.erase(outlist.begin(), outlist.end());
+
+  text_t t;
+
+  while (first != last)
+    {
+      first = getdelimitstr (first, last, c, t);
+      outlist.push_back (t);
+    }
+}
+
+void splitchar (text_t::const_iterator first, text_t::const_iterator last,
+		unsigned short c, text_tarray &outlist)
+{
+  outlist.erase(outlist.begin(), outlist.end());
+
+  text_t t;
+
+  while (first != last)
+    {
+      first = getdelimitstr (first, last, c, t);
+      outlist.push_back (t);
+    }
+}
+
+// join a string using a character
+void joinchar (const text_tset &inlist, unsigned short c, text_t &outtext)
+{
+  outtext.clear ();
+
+  text_tset::const_iterator here = inlist.begin ();
+  text_tset::const_iterator end = inlist.end ();
+  bool first = true;
+  while (here != end)
+    {
+      if (!first) outtext.push_back (c);
+      first = false;
+      outtext += *here;
+      here++;
+    }
+}
+
+void joinchar (const text_tlist &inlist, unsigned short c, text_t &outtext)
+{
+  outtext.clear ();
+
+  text_tlist::const_iterator here = inlist.begin ();
+  text_tlist::const_iterator end = inlist.end ();
+  bool first = true;
+  while (here != end)
+    {
+      if (!first) outtext.push_back (c);
+      first = false;
+      outtext += *here;
+      here++;
+    }
+}
+
+void joinchar (const text_tarray &inlist, unsigned short c, text_t &outtext)
+{
+  outtext.clear ();
+
+  text_tarray::const_iterator here = inlist.begin ();
+  text_tarray::const_iterator end = inlist.end ();
+  bool first = true;
+  while (here != end)
+    {
+      if (!first) outtext.push_back (c);
+      first = false;
+      outtext += *here;
+      here++;
+    }
+}
+
+void joinchar (const text_tarray &inlist, text_t c, text_t &outtext)
+{
+  outtext.clear ();
+
+  text_tarray::const_iterator here = inlist.begin ();
+  text_tarray::const_iterator end = inlist.end ();
+  bool first = true;
+  while (here != end)
+    {
+      if (!first) outtext += c;
+      first = false;
+      outtext += *here;
+      here++;
+    }
+}
+
+// count the occurances of a character within a range
+int countchar (text_t::const_iterator first, text_t::const_iterator last,
+	       unsigned short c)
+{
+  int count = 0;
+  while (first != last) {
+    if (*first == c) count ++;
+    first ++;
+  }
+  return count;
+}
+
+// return a substring of string from first up to but not including last
+text_t substr (text_t::const_iterator first, text_t::const_iterator last) {
+
+  text_t substr;
+  while (first != last) {
+    substr.push_back(*first);
+    first ++;
+  }
+  return substr;
+}
+
+
+// convert to lowercase
+void lc (text_t::iterator first, text_t::iterator last) {
+  while (first != last) {
+    *first = unicode_tolower(*first);
+    first++;
+  }
+}
+
+// convert to uppercase
+void uc (text_t::iterator first, text_t::iterator last) {
+  while (first != last) {
+    *first = unicode_toupper(*first);
+    first++;
+  }
+}
+
+
+// checks to see if it is a number (i.e. contains only 0-9)
+bool is_number (const text_t &text) {
+
+  text_t::const_iterator here = text.begin();
+  text_t::const_iterator end = text.end();
+
+  while (here != end) {
+    if ((*here!='0') && (*here!='1') && (*here!='2') &&
+	(*here!='3') && (*here!='4') && (*here!='5') &&
+	(*here!='6') && (*here!='7') && (*here!='8') &&
+	(*here!='9')) return false;
+    here ++;
+  }
+  return true;
+}
+
+
+// checks to see if the text has any letters or digits
+bool has_unicode_letdig (const text_t &text) {
+  if (text.empty()) return false;
+  
+  text_t::const_iterator here = text.begin();
+  text_t::const_iterator end = text.end();
+  while (here != end) {
+    if (is_unicode_letdig (*here)) return true;
+    here++;
+  }
+
+  return false;
+}
+
+
+
+////////////////////////////////////
+// convertclass methods
+////////////////////////////////////
+
+// conversion classes used for getting information in to and out of
+// the text_t class.
+
+convertclass::convertclass () 
+{
+  // nothing to do
+}
+
+void convertclass::reset ()
+{
+  // nothing to do
+}
+
+
+////////////////////////////////////
+// inconvertclass methods
+////////////////////////////////////
+
+// convert from a char stream to the text_t class
+// the default version assumes the input is a ascii
+// character array
+
+inconvertclass::inconvertclass () 
+{
+  start = NULL;
+  len = 0;
+}
+
+
+void inconvertclass::reset ()
+{
+  start = NULL;
+  len = 0;
+}
+
+void inconvertclass::setinput (char *thestart, size_t thelen)
+{
+  start = thestart;
+  len = thelen;
+}
+
+void inconvertclass::convert (text_t &output, status_t &status)
+{
+  output.clear();
+
+  if (start == NULL || len == 0)
+    {
+      status = finished;
+      return;
+    }
+
+  // don't want any funny sign conversions happening
+  unsigned char *here = (unsigned char *)start;
+  while (len > 0) 
+    {
+      output.push_back (*here); // append this character
+      ++here;
+      --len;
+    }
+
+  start = (char *)here; // save current position
+  status = finished;
+}
+
+// will treat the text_t as a 8-bit string and convert
+// it to a 16-bit string using the about convert method.
+text_t inconvertclass::convert (const text_t &t) {
+  text_t out;
+  text_t tmpout;
+  status_t status;
+  text_t::const_iterator here = t.begin();
+  text_t::const_iterator end = t.end();
+  unsigned char cbuf[256];
+  size_t cbuflen = 0;
+ 
+  while (here != end) {
+    while (here != end && cbuflen < 256) {
+      cbuf[cbuflen++] = (unsigned char)(*here & 0xff);
+      here++;
+    }
+
+    if (cbuflen > 0) {
+      setinput ((char *)cbuf, cbuflen);
+      status = unfinished;
+      while (status == unfinished) {
+	convert (tmpout, status);
+	out += tmpout;
+      }
+      cbuflen = 0;
+    }
+  }
+
+  out.setencoding (0); // unicode
+
+  return out;
+}
+
+// an instance of the default inconvertclass to do simple
+// conversions. Note that any functions that use this are
+// not reentrant. If a function needs to be reentrant it
+// should declare its own instance.
+inconvertclass ascii2text_t;
+
+
+////////////////////////////////////
+// outconvertclass methods
+////////////////////////////////////
+
+// Convert from a text_t class to a char stream
+// This default version assumes the output is a ascii
+// character array. If you set the output stream you
+// can use this class to output to a stream using the
+// << operator. The << operator can also be conveniently
+// used to set the output stream by doing something like
+//
+// cout << text_t2ascii << text_tstr << anothertext_tstr;
+//
+outconvertclass::outconvertclass ()
+{
+  input = NULL;
+  outs = NULL;
+}
+
+void outconvertclass::reset ()
+{
+  input = NULL;
+  outs = NULL;
+}
+
+void outconvertclass::setinput (text_t *theinput)
+{
+  input = theinput;
+  if (input != NULL) texthere = input->begin();
+}
+
+void outconvertclass::convert (char *output, size_t maxlen, 
+		      size_t &len, status_t &status)
+{
+  if (input == NULL || output == NULL)
+    {
+      status = finished;
+      return;
+    }
+
+  // don't want any funny sign conversions happening
+  unsigned char *uoutput = (unsigned char *)output;
+  text_t::iterator textend = input->end();
+  len = 0;
+  while ((len < maxlen) && (texthere != textend)) 
+    {
+      if (*texthere < 256) *uoutput = (unsigned char)(*texthere);
+      else {
+	// put a space or a question mark depending on what
+	// the character is. Question marks tell the user that
+	// they are missing some information.
+	if (is_unicode_space (*texthere)) *uoutput = ' ';
+	else *uoutput = '?';
+      }
+      ++uoutput;
+      ++len;
+      ++texthere;
+    }
+  
+  if (texthere == textend) status = finished;
+  else status = unfinished;
+}
+
+// will convert the 16-bit string to a 8-bit stream
+// and place the result in a text_t. This method uses
+// the above convert function.
+text_t outconvertclass::convert (const text_t &t) {
+  text_t out;
+  unsigned char cbuf[256];
+  size_t cbuflen = 0;
+  status_t status = unfinished;
+
+  setinput ((text_t *)&t); // discard constant
+  while (status == unfinished) {
+    convert ((char *)cbuf, 256, cbuflen, status);
+    out.appendcarr ((char *)cbuf, cbuflen);
+  }
+
+  out.setencoding (1); // other encoding
+  
+  return out;
+}
+
+
+void outconvertclass::setostream (ostream *theouts)
+{
+  outs = theouts;
+}
+
+ostream *outconvertclass::getostream ()
+{
+  return outs;
+}
+
+
+
+
+// an instance of the default outconvertclass to do simple
+// conversions
+outconvertclass text_t2ascii;
+
+
+
+// stream operators for the output class
+
+outconvertclass &operator<< (ostream &theouts, outconvertclass &outconverter)
+{
+  outconverter.setostream(&theouts);
+  return outconverter;
+}
+
+
+#define STREAMBUFSIZE 256
+outconvertclass &operator<< (outconvertclass &outconverter, const text_t &t)
+{
+  ostream *outstream = outconverter.getostream();
+
+  if (outstream == NULL) return outconverter;
+
+  char outbuf[STREAMBUFSIZE];
+  size_t len;
+  outconvertclass::status_t status = outconvertclass::unfinished;
+
+  // assume that there is no data needing converting
+  // left in the converter
+  outconverter.setinput ((text_t *)(&t)); // note the const -> nonconst conversion
+
+  while (status == outconvertclass::unfinished)
+    {
+      outconverter.convert (outbuf, STREAMBUFSIZE, len, status);
+      if (len > 0) outstream->write(outbuf, len);
+    }
+
+  return outconverter;
+}
Index: /tags/gsdl-21/gsdl/lib/text_t.h
===================================================================
--- /tags/gsdl-21/gsdl/lib/text_t.h	(revision 699)
+++ /tags/gsdl-21/gsdl/lib/text_t.h	(revision 699)
@@ -0,0 +1,358 @@
+/**********************************************************************
+ *
+ * text_t.h -- a simple 16-bit character string class
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef TEXT_T_H
+#define TEXT_T_H
+
+#include "gsdlconf.h"
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\vector>
+#  include <ospace\std\list>
+#  include <ospace\std\set>
+#  include <ospace\std\map>
+#elif defined(GSDL_USE_STL_H)
+#  include <vector.h>
+#  include <list.h>
+#  include <set.h>
+#  include <map.h>
+#else
+#  include <vector>
+#  include <list>
+#  include <set>
+#  include <map>
+#endif
+
+// use the standard namespace
+#if !defined (GSDL_NAMESPACE_BROKEN)
+#if defined(GSDL_USE_OBJECTSPACE)
+using namespace ospace::std;
+#else
+using namespace std;
+#endif
+#endif
+
+// class prototypes
+class inconvertclass;
+class outconvertclass;
+
+
+// for those stupid compilers which need it
+#if defined(GSDL_NEED_DESTROY_USHORT)
+inline void destroy(unsigned short *) {};
+inline void destroy(int *) {};
+#endif
+
+typedef vector<unsigned short> usvector;
+
+// The class text_t can handle long strings which may contain
+// null characters. It uses unsigned shorts to represent up to
+// 64K character values.
+class text_t {
+  public:
+  //type support for ucvector
+  typedef usvector::iterator iterator;
+  typedef usvector::const_iterator const_iterator;
+  typedef usvector::reference reference;
+  typedef usvector::const_reference const_reference;
+  typedef usvector::size_type size_type;
+  typedef usvector::difference_type difference_type;
+  typedef usvector::const_reverse_iterator const_reverse_iterator;
+  typedef usvector::reverse_iterator reverse_iterator;
+  
+protected:
+  usvector text;
+  unsigned short encoding; // 0 = unicode, 1 = other
+  
+public:
+  // constructors
+  text_t ();
+  text_t (int i);
+  text_t (char *s); // assumed to be a normal c string
+
+  void setencoding (unsigned short theencoding) {encoding=theencoding;};
+  unsigned short getencoding () {return encoding;};
+
+  // basic container support
+  iterator begin () {return text.begin();}
+  const_iterator begin () const {return text.begin();}
+  iterator end () {return text.end();}
+  const_iterator end () const {return text.end();}
+
+  void erase(iterator pos) {text.erase(pos);}
+  void erase(iterator first, iterator last) {text.erase(first, last);}
+  void push_back(unsigned short c) {text.push_back(c);}
+  void pop_back() {text.pop_back();}
+  text_t &operator=(const text_t &x) {text=x.text; encoding=x.encoding; return *this;}
+  reference operator[](size_type n) {return text[n];};
+  const_reference operator[](size_type n) const {return text[n];};
+
+  void reserve (size_type n) {text.reserve(n);}
+  
+  bool empty () const {return text.empty();}
+  size_type size() const {return text.size();}
+  friend inline bool operator!=(const text_t& x, const text_t& y)
+	{return (x.text != y.text);}
+  friend inline bool operator==(const text_t& x, const text_t& y) 
+	{return (x.text == y.text);}
+  friend inline bool operator<(const text_t& x, const text_t& y) 
+	{return (x.text < y.text);}
+  friend inline bool operator>(const text_t& x, const text_t& y) 
+	{return (x.text > y.text);}
+  friend inline bool operator>=(const text_t& x, const text_t& y) 
+	{return (x.text >= y.text);}
+  friend inline bool operator<=(const text_t& x, const text_t& y) 
+	{return (x.text <= y.text);}
+  
+  // added functionality
+  void clear () {text.erase(text.begin(),text.end());}
+  void append (const text_t &t);
+  void appendrange (iterator first, iterator last);
+  void appendrange (const_iterator first, const_iterator last);
+  text_t &operator+= (const text_t &t) {append(t);return *this;}
+
+  // support for integers
+  void appendint (int i);
+  void setint (int i) {clear();appendint(i);}
+  text_t &operator=(int i) {setint (i);return *this;}
+  text_t &operator+= (int i) {appendint(i);return *this;}
+  int getint () const;
+
+  // support for arrays of chars
+  void appendcarr (char *s, size_type len);
+  void setcarr (char *s, size_type len) {clear();appendcarr(s,len);}
+
+  // support for null-terminated C strings
+  void appendcstr (char *s);
+  void setcstr (char *s) {clear();appendcstr(s);}
+  text_t &operator= (char *s) {setcstr(s);return *this;} // c string
+  text_t &operator+= (char *s) {appendcstr(s);return *this;} // c string
+
+  // support for const null-terminated C string
+  void appendcstr (const char *s);
+  void setcstr (const char *s) {clear();appendcstr((char *) s);}
+  text_t &operator= (const char *s) {setcstr((char *) s);return *this;} // c string
+  text_t &operator+= (const char *s) {appendcstr((char *) s);return *this;} // c string
+
+  // strings returned from getcarr and getcstr become the callers 
+  // responsibility and should be deallocated with "delete"
+  char *getcarr(size_type &len) const;
+  char *getcstr() const;
+};
+
+
+inline text_t operator+(const text_t &t1, const text_t &t2)
+{
+  text_t tnew = t1;
+  tnew.append(t2);
+  return tnew;
+}
+
+inline text_t operator+(const text_t &t1, int i1)
+{
+  text_t tnew = t1;
+  tnew.appendint(i1);
+  return tnew;
+}
+
+inline text_t operator+(const text_t &t1, char *s1)
+{
+  text_t tnew = t1;
+  tnew.appendcstr(s1);
+  return tnew;
+}
+
+
+
+
+struct eqtext_t 
+{
+  bool operator()(const text_t &t1, const text_t &t2) const
+  { return t1 == t2; }
+};
+
+struct lttext_t 
+{
+  bool operator()(const text_t &t1, const text_t &t2) const
+  { return t1 < t2; }
+};
+
+
+// frequently used derived types
+typedef set<text_t,lttext_t> text_tset;
+typedef list<text_t> text_tlist;  // more efficient for insertions/deletions
+typedef vector<text_t> text_tarray; // more space efficient than text_tlist
+typedef map<text_t, text_t, lttext_t> text_tmap;
+
+
+// general functions which work on text_ts
+
+// find a character within a range
+text_t::const_iterator findchar (text_t::const_iterator first, text_t::const_iterator last, 
+				 unsigned short c);
+text_t::iterator findchar (text_t::iterator first, text_t::iterator last, 
+			   unsigned short c);
+
+// get a string up to the next delimiter (which is skipped)
+text_t::const_iterator getdelimitstr (text_t::const_iterator first, 
+				      text_t::const_iterator last,
+				      unsigned short c, text_t &outstr);
+text_t::iterator getdelimitstr (text_t::iterator first, text_t::iterator last,
+				unsigned short c, text_t &outstr);
+
+// split a string with a character
+void splitchar (text_t::const_iterator first, text_t::const_iterator last,
+		unsigned short c, text_tset &outlist);
+void splitchar (text_t::const_iterator first, text_t::const_iterator last,
+		unsigned short c, text_tlist &outlist);
+void splitchar (text_t::const_iterator first, text_t::const_iterator last,
+		unsigned short c, text_tarray &outlist);
+
+// join a string using a character
+void joinchar (const text_tset &inlist, unsigned short c, text_t &outtext);
+void joinchar (const text_tlist &inlist, unsigned short c, text_t &outtext);
+void joinchar (const text_tarray &inlist, unsigned short c, text_t &outtext);
+void joinchar (const text_tarray &inlist, text_t c, text_t &outtext);
+
+// count the occurances of a character within a range
+int countchar (text_t::const_iterator first, text_t::const_iterator last,
+	       unsigned short c);
+
+// return a substring of string from first up to but not including last
+text_t substr (text_t::const_iterator first, text_t::const_iterator last);
+
+// convert to lowercase
+void lc (text_t::iterator first, text_t::iterator last);
+inline void lc (text_t &t) {lc (t.begin(), t.end());}
+
+// convert to uppercase
+void uc (text_t::iterator first, text_t::iterator last);
+inline void uc (text_t &t) {uc (t.begin(), t.end());}
+
+// checks to see if it is a number (i.e. contains only 0-9)
+bool is_number (const text_t &text);
+
+// checks to see if the text has any letters or digits
+bool has_unicode_letdig (const text_t &text);
+
+
+// conversion classes used for getting information in to and out of
+// the text_t class.
+
+class convertclass 
+{
+public:
+  enum status_t {finished, stopped, unfinished};
+  
+  convertclass ();
+  virtual void reset ();
+};
+
+
+
+// convert from a char stream to the text_t class
+// the default version assumes the input is a ascii
+// character array
+class inconvertclass : public convertclass
+{
+public:
+  inconvertclass ();
+  void reset ();
+  void setinput (char *thestart, size_t thelen);
+
+  // output will be cleared before the conversion
+  virtual void convert (text_t &output, status_t &status);
+
+  // will treat the text_t as a 8-bit string and convert
+  // it to a 16-bit string using the about convert method.
+  text_t convert (const text_t &t);
+
+protected:
+  char *start;
+  size_t len;
+};
+
+// to get something which will do the conversion
+// to ascii declare a (non global!) instance like
+// this
+// inconvertclass ascii2text_t;
+
+#if defined(GSDL_USE_IOS_H)
+#include <ostream.h>
+#else
+#include <ostream>
+#endif
+
+// Convert from a text_t class to a char stream
+// This default version assumes the output is a ascii
+// character array. If you set the output stream you
+// can use this class to output to a stream using the
+// << operator. The << operator can also be conveniently
+// used to set the output stream by doing something like
+//
+// cout << text_t2ascii << textstr << anothertextstr;
+//
+// this class assumes that the input text doesn't change
+// while the conversion takes place
+class outconvertclass : public convertclass
+{
+public:
+  outconvertclass ();
+  void reset ();
+  void setinput (text_t *theinput);
+  // note that convert does not null-terminate the
+  // output array of characters
+  virtual void convert (char *output, size_t maxlen, 
+			size_t &len, status_t &status);
+
+  // will convert the 16-bit string to a 8-bit stream
+  // and place the result in a text_t. This method uses
+  // the above convert function.
+  text_t convert (const text_t &t);
+
+  void setostream (ostream *theouts);
+  ostream *getostream ();
+
+protected:
+  text_t *input;
+  text_t::iterator texthere; // only valid if input is valid
+
+  ostream *outs;
+};
+
+// to get something which will do the conversion
+// to text_t declare a (non global!) instance like
+// this
+// outconvertclass text_t2ascii;
+
+
+// stream operators for the output class
+outconvertclass &operator<< (ostream &theouts, outconvertclass &outconverter);
+outconvertclass &operator<< (outconvertclass &outconverter, const text_t &t);
+
+#endif
Index: /tags/gsdl-21/gsdl/macros/about.dm
===================================================================
--- /tags/gsdl-21/gsdl/macros/about.dm	(revision 699)
+++ /tags/gsdl-21/gsdl/macros/about.dm	(revision 699)
@@ -0,0 +1,61 @@
+package about
+
+#######################################################################
+# Icons
+#######################################################################
+
+_imagethispage_ {<img src="_httpiconhabout_" width=_widthhabout_ height=_heighthabout_>}
+
+# don't want link to 'about' page
+_imagecollection_ {<img src="_iconcollection_" border=0>}
+
+#######################################################################
+# English language text 
+#######################################################################
+
+_texthelpexpl_ {For help at any time, press this button at the top of the page}
+
+_textretabout_ {To return to this "about" page, click the "collection"
+icon at the top left of the page.}
+
+_textabcol_ {About this collection}
+
+_textabout_ {
+<h3>_textabcol_</h3>
+_Global:collectionextra_
+}
+
+#######################################################################
+# Maori language text 
+#######################################################################
+
+_texthelpexpl_ [l=mi] {Kia titiro ki ng&auml; kupu &auml;whina p&auml;whiritia t&euml;nei
+p&auml;tene kei runga r&auml; o te wh&auml;rangi}
+
+_textabcol_ [l=mi] {He kupu whakam&auml;rama i te kohikohinga niupepa}
+
+##############
+# about page #
+##############
+
+_pagetitle_ {_collectionname_: About this collection}
+
+_content_ {
+<center>
+_navigationbar_
+</center>
+_query:queryform_
+<p>
+_iconblankbar_<br>
+<i>_textretabout_</i>
+<table cellpadding=3 cellspacing=0 border=0>
+<tr valign=bottom><td><img src="_httpiconchelpon_" width=_widthchelpx_ height=_heightchelpx_></td>
+<td><i>_texthelpexpl_</i>
+</td></tr></table>
+_iconblankbar_
+<p>
+_textabout_
+<h3>_help:textsimplehelpheading_</h3>
+_help:simplehelp_
+}
+
Index: /tags/gsdl-21/gsdl/macros/authen.dm
===================================================================
--- /tags/gsdl-21/gsdl/macros/authen.dm	(revision 699)
+++ /tags/gsdl-21/gsdl/macros/authen.dm	(revision 699)
@@ -0,0 +1,60 @@
+package authen
+
+
+#######################################################################
+# page content                     
+#######################################################################
+
+_pagetitle_ {Greenstone Digital Library}
+_imagethispage_ {_imagegreenstone_}
+
+_content_ {
+_messagestatus_
+
+<form name="login" method="post" action="_gwcgi_">
+<input type=hidden name="e" value="_If_(_cgiarger_,_cgiarger_,_compressedoptions_)">
+_hiddenargs_
+
+<table border=0>
+<tr><td>username</td><td><input type="text" name="un" value="_cgiargun_" size=10></td></tr>
+<tr><td>password</td><td><input type="password" name="pw" size=10></td></tr>
+<tr><td></td><td><input type="submit" value="sign in"></td></tr>
+</table>
+</form>
+}
+
+
+_messageinvalid_ {The page you have requested requires you to sign in.
+Please enter your username
+and password.}
+
+_messagefailed_ {Either your user name or password was incorrect.}
+
+_messageenabled_ {}
+
+_messagedisabled_ {Sorry, you account has been disabled. Please contact
+the webmaster for this site.}
+
+_messagepermissiondenied_ {Sorry, you do not have permission to access this page.}
+
+_messagestalekey_ {The link you have followed is now stale. 
+Please enter your password to access this page.}
+
+
+#################################
+# when authentication is accepted
+#################################
+
+package authenok
+
+
+#######################################################################
+# page content                     
+#######################################################################
+
+_pagetitle_ {Greenstone Digital Library}
+_imagethispage_ {_imagegreenstone_}
+
+_content_ {
+_authen:messagestatus_
+}
Index: /tags/gsdl-21/gsdl/macros/base.dm
===================================================================
--- /tags/gsdl-21/gsdl/macros/base.dm	(revision 699)
+++ /tags/gsdl-21/gsdl/macros/base.dm	(revision 699)
@@ -0,0 +1,833 @@
+#######################################################################
+# GLOBAL MACROS 
+#######################################################################
+
+package Global
+
+##########
+
+# this is a hack for the chinese collection and should be removed once 
+# the collection metadata is fixed up 
+_documents_ [c=chinese] {document sections}
+_documents_ [c=chinese,l=zh] {ææ¬}
+_titles_ [c=chinese] {titles}
+_titles_ [c=chinese,l=zh] {é¢å}
+_chinese_ {Chinese demonstration collection}
+
+##########
+
+_collectionextra_ {This collection contains _about:numdocs_ documents. 
+It was last built _about:builddate_ days ago.}
+
+
+#######################################################################
+# page content
+#
+# these should always be overridden for each page/collection 
+#######################################################################
+
+_content_ {<p><h2>oops</h2>
+The requested page could not be found. Please use your browsers 'back' button
+or the above home button to return to the Greenstone Digital Library.}
+_pagetitle_ {GSDL Error}
+_imagethispage_ {}
+_iconcollection_ {}
+_imagecollection_ {_If_("_iconcollection_" ne "",<a href="_httppagex_(about)"><img src="_iconcollection_" border=0></a>)}
+
+
+#######################################################################
+# these width macros are read in by the server when calculating 
+# width of navigation bar etc. There should be one for each 
+# classification that this receptionist supports, one for the 
+# search button, and the _pagewidth_ macro which is the total width
+# of the page
+#######################################################################
+
+# width of tabs when displaying tables of contents
+_tabwidth_ {25}
+
+_pagewidth_ {537}
+# defaultwidth is the width buttons default to if not included in this list
+_defaultwidth_ {87}
+_searchwidth_ {_widthtsrchx_}
+_Titlewidth_ {_widthttitlx_}
+_Listwidth_ {_widthtlistx_}
+_Creatorwidth_ {_widthtauthx_}
+_Serieswidth_ {_widthtserx_}
+_Datewidth_ {_widthtdatex_}
+_Subjectwidth_ {_widthtsubjx_}
+_Organizationwidth_ {_widthtorgx_}
+_Howtowidth_ {_widththowx_}
+_Topicwidth_ {_widthttopicx_}
+_Browsewidth_ {_widthtbrwsex_}
+_Peoplewidth_ {_widthtpeopx_}
+
+#######################################################################
+# Macros whose values are set from within the server at runtime. These 
+# are here only for reference and to set default values if required.           
+#######################################################################
+
+_navigationbar_ {}
+_javaimagesnavbar_ {}
+_javaimagescontent_ {}
+_widthtspace_ {2}
+
+# all cgi args are set as _cgiargX_ macros - those I've put here 
+# are those that need to default to something
+_cgiargd_ {}
+_cgiargcl_ {}
+
+# set from within the query action
+_hselection_ {}
+_jselection_ {}
+_nselection_ {}
+
+#######################################################################
+# navigation bar images
+#
+#######################################################################
+
+# the spacer image - the width of this is set from server
+_imagespacer_ {<img 
+src="_httpicontspace_" width="_widthtspace_" height="_heighttspace_">}
+_imagespacer_[v=1] {<br>
+}
+
+# image macros for all the classifications currently supported by
+# this receptionist. i.e. title, author, subject, series, date,
+# howto, and organization (and search)
+
+_imagesearch_ {<a href="_httpquery_" 
+    onMouseover = "img\_on('search')"
+    onMouseout = "img\_off('search')"><img name="search"
+        src="_httpicontsrchof_" width="_widthtsrchx_" height="_heighttsrchx_" 
+        border="0" alt="_textimagesearch_"></a>}
+_imagesearch_[v=1] {<a href="_httpquery_">_textimagesearch_</a><br>
+}
+
+_imageTitle_ {<a href="_httpbrowseTitle_"
+    onMouseover = "img\_on('titles')" 
+    onMouseout = "img\_off('titles')"><img name="titles" 
+        src="_httpiconttitlof_" width=_widthttitlx_ height=_heightttitlx_ 
+        border=0 alt="_textimageTitle_"></a>}
+_imageTitle_[v=1] {<a href="_httpbrowseTitle_">_textimageTitle_</a><br>
+}
+
+_imageList_ {<a href="_httpbrowseList_"
+    onMouseover = "img\_on('list')" 
+    onMouseout = "img\_off('list')"><img name="list" 
+        src="_httpicontlistof_" width=_widthtlistx_ height=_heighttlistx_ 
+        border=0 alt="_textimageList_"></a>}
+_imageList_[v=1] {<a href="_httpbrowseList_">_textimageList_</a><br>
+}
+
+
+_imageCreator_ {<a href="_httpbrowseCreator_"
+    onMouseover = "img\_on('authors')" 
+    onMouseout = "img\_off('authors')"><img name="authors" 
+        src="_httpicontauthof_" width=_widthtauthx_ height=_heighttauthx_ 
+        border=0 alt="_textimageCreator_"></a>}
+_imageCreator_[v=1] {<a href="_httpbrowseCreator_">_textimageCreator_</a><br>
+}
+
+_imageSeries_ {<a href="_httpbrowseSeries_"
+    onMouseover = "img\_on('series')" 
+    onMouseout = "img\_off('series')"><img name="series" 
+        src="_httpicontserof_" width="_widthtserx_" height="_heighttserx_" 
+        border="0" alt="_textimageSeries_"></a>}
+_imageSeries_[v=1] {<a href="_httpbrowseSeries_">_textimageSeries_</a><br>
+}
+
+_imageDate_ {<a href="_httpbrowseDate_"
+    onMouseover = "img\_on('dates')" 
+    onMouseout = "img\_off('dates')"><img name="dates" 
+        src="_httpicontdateof_" width="_widthtdatex_" height="_heighttdatex_" 
+        border="0" alt="_textimageDate_"></a>}
+_imageDate_[v=1] {<a href="_httpbrowseDate_">_textimageDate_</a><br>
+}
+
+_imageSubject_ {<a href="_httpbrowseSubject_"
+    onMouseover = "img\_on('subjects')" 
+    onMouseout = "img\_off('subjects')"><img name="subjects" 
+        src="_httpicontsubjof_" width="_widthtsubjx_" height="_heighttsubjx_" 
+        border="0" alt="_textimageSubject_"></a>}
+_imageSubject_[v=1] {<a href="_httpbrowseSubject_">_textimageSubject_</a><br>
+}
+
+_imageOrganization_ {<a href="_httpbrowseOrganization_"
+    onMouseover = "img\_on('org')" 
+    onMouseout = "img\_off('org')"><img name="org" 
+        src="_httpicontorgof_" width="_widthtorgx_" height="_heighttorgx_" 
+        border="0" alt="_textimageOrganization_"></a>}
+_imageOrganization_[v=1] {<a href="_httpbrowseOrganization_">_textimageOrganization_</a><br>
+}
+
+_imageHowto_ {<a href="_httpbrowseHowto_"
+    onMouseover = "img\_on('how')" 
+    onMouseout = "img\_off('how')"><img name="how" 
+        src="_httpiconthowof_" width="_widththowx_" height="_heightthowx_" 
+        border="0" alt="_textimageHowto_"></a>}
+_imageHowto_[v=1] {<a href="_httpbrowseHowto_">_textimageHowto_</a><br>
+}
+
+_imageTopic_ {<a href="_httpbrowseTopic_"
+    onMouseover = "img\_on('topic')" 
+    onMouseout = "img\_off('topic')"><img name="topic" 
+        src="_httpiconttopicof_" width="_widthttopicx_" height="_heightttopicx_" 
+        border="0" alt="_textimageTopic_"></a>}
+_imageTopic_[v=1] {<a href="_httpbrowseTopic_">_textimageTopic_</a><br>
+}
+
+_imageBrowse_ {<a href="_httpbrowseBrowse_"
+    onMouseover = "img\_on('browse')" 
+    onMouseout = "img\_off('browse')"><img name="browse" 
+        src="_httpicontbrwseof_" width="_widthtbrwsex_" height="_heighttbrwsex_" 
+        border="0" alt="_textimageBrowse_"></a>}
+_imageBrowse_[v=1] {<a href="_httpbrowseBrowse_">_textimageBrowse_</a><br>
+}
+
+_imagePeople_ {<a href="_httpbrowsePeople_"
+    onMouseover = "img\_on('People')"
+    onMouseout = "img\_off('People')"><img name="People"
+        src="_httpicontpeopof_" width=_widthtpeopx_ height=_heighttpeopx_
+        border=0 alt="_textimagePeople_"></a>}
+_imagePeople_[v=1] {<a href="_httpbrowsePeople_">_textimagePeople_</a><br>
+}
+
+
+#######################################################################
+# java images/scripts
+#######################################################################
+
+# the _javalinks_ macros are the flashy image links at the top right of
+# the page. 
+
+_javalinks_ {_imagehome__imagehelp__imagepref_}
+_javalinks_ [v=1] {
+_imagehome_<br>
+_imagehelp_<br>
+_imagepref_<br>
+}
+
+# the javaimages macros contain the javascript code required
+# by each flashy javascript image. _javaimagesheader_ contains those
+# images at the top right of the page, _javaimagesnavbar_ (set
+# from within the server) contains those images in the navigation bar,
+# and _javaimagescontent_ (also set in server) contains any other 
+# javascript images that occur in the page. 
+# the javaimages macros should contain _javaX_ macros for each javascript
+# image link within the page.
+
+_javaimagesheader_ {_javahome__javahelp__javapref_}
+_javaimagesheader_ [v=1] {}
+
+_javaabout_ {
+	about\_on = new Image(_widthcabtx_, _heightcabtx_);
+	about\_on.src = "_httpiconcabton_";
+	about\_off = new Image(_widthcabtx_, _heightcabtx_);
+	about\_off.src = "_httpiconcabtof_";
+}
+
+_javahome_ {
+	home\_on = new Image(_widthchomex_, _heightchomex_);          
+	home\_on.src = "_httpiconchomeon_";		        
+	home\_off = new Image(_widthchomex_, _heightchomex_);          
+	home\_off.src = "_httpiconchomeof_";             
+}
+
+_javahelp_ {
+	help\_on = new Image(_widthchelpx_, _heightchelpx_);
+	help\_on.src = "_httpiconchelpon_";
+	help\_off = new Image(_widthchelpx_, _heightchelpx_);
+	help\_off.src = "_httpiconchelpof_";
+}
+
+_javapref_ {
+	pref\_on = new Image(_widthcprefx_, _heightcprefx_);          
+	pref\_on.src = "_httpiconcprefon_";
+	pref\_off = new Image(_widthcprefx_, _heightcprefx_);
+	pref\_off.src = "_httpiconcprefof_"; 
+}
+
+_javasearch_ {
+	search\_on = new Image(_widthtsrchx_, _heighttsrchx_);          
+	search\_on.src = "_httpicontsrchon_";
+	search\_off = new Image(_widthtsrchx_, _heighttsrchx_);          
+	search\_off.src = "_httpicontsrchof_";
+}
+
+_javaTitle_ {
+	titles\_on = new Image(_widthttitlx_, _heightttitlx_);
+	titles\_on.src = "_httpiconttitlon_";
+	titles\_off = new Image(_widthttitlx_, _heightttitlx_);
+	titles\_off.src = "_httpiconttitlof_";
+}
+
+_javaList_ {
+	list\_on = new Image(_widthtlistx_, _heighttlistx_);
+	list\_on.src = "_httpicontliston_";
+	list\_off = new Image(_widthtlistx_, _heighttlistx_);
+	list\_off.src = "_httpicontlistof_";
+}
+
+_javaCreator_ {
+	authors\_on = new Image(_widthtauthx_, _heighttauthx_);
+	authors\_on.src = "_httpicontauthon_";
+	authors\_off = new Image(_widthtauthx_, _heighttauthx_);
+	authors\_off.src = "_httpicontauthof_";
+}
+
+_javaSeries_ {
+	series\_on = new Image(_widthtserx_, _heighttserx_);          
+	series\_on.src = "_httpicontseron_";
+	series\_off = new Image(_widthtserx_, _heighttserx_);          
+	series\_off.src = "_httpicontserof_";
+}
+
+_javaDate_ {
+	dates\_on = new Image(_widthtdatex_, _heighttdatex_);          
+	dates\_on.src = "_httpicontdateon_";
+	dates\_off = new Image(_widthtdatex_, _heighttdatex_);          
+	dates\_off.src = "_httpicontdateof_";	 		
+}
+
+_javaSubject_ {
+	subjects\_on = new Image(_widthtsubjx_, _heighttsubjx_);          
+	subjects\_on.src = "_httpicontsubjon_";
+	subjects\_off = new Image(_widthtsubjx_, _heighttsubjx_);          
+	subjects\_off.src = "_httpicontsubjof_";	 		
+}
+
+_javaHowto_ {
+	how\_on = new Image(_widththowx_, _heightthowx_);          
+	how\_on.src = "_httpiconthowon_";
+	how\_off = new Image(_widththowx_, _heightthowx_);          
+	how\_off.src = "_httpiconthowof_";	 		
+}
+
+_javaOrganization_ {
+	org\_on = new Image(_widthtorgx_, _heighttorgx_);          
+	org\_on.src = "_httpicontorgon_";
+	org\_off = new Image(_widthtorgx_, _heighttorgx_);          
+	org\_off.src = "_httpicontorgof_";	 		
+}
+
+_javaTopic_ {
+	topic\_on = new Image(_widthttopicx_, _heightttopicx_);          
+	topic\_on.src = "_httpiconttopicon_";
+	topic\_off = new Image(_widthttopicx_, _heightttopicx_);          
+	topic\_off.src = "_httpiconttopicof_";	 		
+}
+
+_javaBrowse_ {
+	browse\_on = new Image(_widthtbrwsex_, _heighttbrwsex_);          
+	browse\_on.src = "_httpicontbrwseon_";
+	browse\_off = new Image(_widthtbrwsex_, _heighttbrwsex_);          
+	browse\_off.src = "_httpicontbrwseof_";	 		
+}
+
+_javaPeople_ {
+        People\_on = new Image(_widthtpeopx_, _heighttpeopx_);
+        People\_on.src = "_httpicontpeopon_";
+        People\_off = new Image(_widthtpeopx_, _heighttpeopx_);
+        People\_off.src = "_httpicontpeopof_";
+}
+
+
+#######################################################################
+# general web macros 
+#######################################################################
+
+_mailaddr_ {nzdl@nzdl.org}
+
+
+
+#######################################################################
+# http macros 
+#
+# These contain the url without any quotes
+#######################################################################
+
+_httpcollection_{_httpprefix_/collect/_cgiargc_}
+
+_httppagex_ {_gwcgi_?e=_compressedoptions_&a=p&p=_1_}
+_httppagestatus_ {_gwcgi_?e=_compressedoptions_&a=status}
+_httppageabout_ {_httppagex_(about)}
+#_httppagehome_ {_httppagex_(home)}
+#_httppagehome_ {_gwcgi_?a=p&p=home}
+_httppagehome_ {http://www.nzdl.org/cgi-bin/sblibrary?a=p&p=home}
+_httppagehelp_ {_httppagex_(help)}
+_httppagepref_ {_httppagex_(preferences)}
+
+_httpgreenstone_ {_httppagex_(gsdl)}
+
+_httpcurrentdocument_ {_gwcgi_?e=_compressedoptions_&cl=_cgiargcl_&d=_cgiargd_}
+_httpquery_ {_gwcgi_?e=_compressedoptions_&a=q}
+_httpdocument_ {_gwcgi_?e=_compressedoptions_&a=d}
+
+_httpiconchalk_ {_httpimg_/chalk.gif}
+_widthchalk_ {2000}
+_heightchalk_ {10}
+
+_httpiconchelpof_ {_httpimg_/chelpof.gif}
+_httpiconchelpof_ [l=mi] {_httpimg_/mchelpof.gif}
+_httpiconchelpon_ {_httpimg_/chelpon.gif}
+_httpiconchelpon_ [l=mi] {_httpimg_/mchelpon.gif}
+_widthchelpx_ {36}
+_widthchelpx_ [l=mi] {57}
+_heightchelpx_ {20}
+
+_httpiconchomeof_ {_httpimg_/chomeof.gif}
+_httpiconchomeof_ [l=mi] {_httpimg_/mchomeof.gif}
+_httpiconchomeon_ {_httpimg_/chomeon.gif}
+_httpiconchomeon_ [l=mi] {_httpimg_/mchomeon.gif}
+_widthchomex_ {43}
+_widthchomex_ [l=mi] {54}
+_heightchomex_ {20}
+
+_httpiconcprefof_ {_httpimg_/cprefof.gif}
+_httpiconcprefof_ [l=mi] {_httpimg_/mcprefof.gif}
+_httpiconcprefon_ {_httpimg_/cprefon.gif}
+_httpiconcprefon_ [l=mi] {_httpimg_/mcprefon.gif}
+_widthcprefx_ {87}
+_widthcprefx_ [l=mi] {103}
+_heightcprefx_ {20}
+
+_httpiconcabtof_ {_httpimg_/cabtof.gif}
+_httpiconcabton_ {_httpimg_/cabton.gif}
+_widthcabtx_ {50}
+_heightcabtx_ {20}
+
+_httpicondivb_ {_httpimg_/divb.gif}
+_widthdivb_ {_pagewidth_}
+_heightdivb_ {17}
+
+_httpicongsdl_ {_httpimg_/gsdl.gif}
+_widthgsdl_ {140}
+_heightgsdl_ {77}
+
+_httpiconhhelp_ {_httpimg_/h\_help.gif}
+_widthhhelp_ {200}
+_heighthhelp_ {57}
+
+_httpiconitext_ {_httpimg_/itext.gif}
+_widthitext_ {16}
+_heightitext_ {21}
+
+_httpiconibtext_ {_httpimg_/ibtext.gif}
+_widthibtext_ {16}
+_heightibtext_ {21}
+
+_httpiconless_ {_httpimg_/less.gif}
+_widthless_ {30}
+_heightless_ {16}
+
+_httpiconmore_ {_httpimg_/more.gif}
+_widthmore_ {30}
+_heightmore_ {16}
+
+_httpiconqryresb_ {_httpimg_/qryresb.gif}
+_httpiconqryresb_ [l=mi] {_httpimg_/mqryresb.gif}
+_widthqryresb_ {_pagewidth_}
+_heightqryresb_ {17}
+
+_httpiconspacer_ {_httpimg_/spacer.gif}
+_widthspacer_ {42}
+_heightspacer_ {4}
+
+_httpicontabspace_ {_httpimg_/tabspace.gif}
+_widthtabspace_ {23}
+_heighttabspace_ {1}
+
+_httpicontauthgr_ {_httpimg_/tauthgr.gif}
+_httpicontauthof_ {_httpimg_/tauthof.gif}
+_httpicontauthon_ {_httpimg_/tauthon.gif}
+_widthtauthx_ {110}
+_heighttauthx_ {17}
+
+_httpicontsergr_ {_httpimg_/tsergr.gif}
+_httpicontsergr_ [l=mi] {_httpimg_/mtsergr.gif}
+_httpicontserof_ {_httpimg_/tserof.gif}
+_httpicontserof_ [l=mi] {_httpimg_/mtserof.gif}
+_httpicontseron_ {_httpimg_/tseron.gif}
+_httpicontseron_ [l=mi] {_httpimg_/mtseron.gif}
+_widthtserx_ {87}
+_widthtserx_ [l=mi] {192}
+_heighttserx_ {17}
+
+_httpicontdategr_ {_httpimg_/tdategr.gif}
+_httpicontdategr_ [l=mi] {_httpimg_/mtdategr.gif}
+_httpicontdateof_ {_httpimg_/tdateof.gif}
+_httpicontdateof_ [l=mi] {_httpimg_/mtdateof.gif}
+_httpicontdateon_ {_httpimg_/tdateon.gif}
+_httpicontdateon_ [l=mi] {_httpimg_/mtdateon.gif}
+_widthtdatex_ {87}
+_heighttdatex_ {17}
+
+_httpicontsubjgr_ {_httpimg_/tsubjgr.gif}
+_httpicontsubjon_ {_httpimg_/tsubjon.gif}
+_httpicontsubjof_ {_httpimg_/tsubjof.gif}
+_widthtsubjx_ {87}
+_heighttsubjx_ {17}
+
+_httpicontorggr_{_httpimg_/torggr.gif}
+_httpicontorgon_{_httpimg_/torgon.gif}
+_httpicontorgof_{_httpimg_/torgof.gif}
+_widthtorgx_ {109}
+_heighttorgx_ {17}
+
+_httpiconthowgr_{_httpimg_/thowgr.gif}
+_httpiconthowon_{_httpimg_/thowon.gif}
+_httpiconthowof_{_httpimg_/thowof.gif}
+_widththowx_ {87}
+_heightthowx_ {17}
+
+_httpiconttopicgr_{_httpimg_/ttopicgr.gif}
+_httpiconttopicon_{_httpimg_/ttopicon.gif}
+_httpiconttopicof_{_httpimg_/ttopicof.gif}
+_widthttopicx_ {87}
+_heightttopicx_ {17}
+
+_httpicontbrwsegr_{_httpimg_/tbrwsegr.gif}
+_httpicontbrwseon_{_httpimg_/tbrwseon.gif}
+_httpicontbrwseof_{_httpimg_/tbrwseof.gif}
+_widthtbrwsex_ {87}
+_heighttbrwsex_ {17}
+
+_httpicontsrchgr_ {_httpimg_/tsrchgr.gif}
+_httpicontsrchgr_ [l=mi] {_httpimg_/mtsrchgr.gif}
+_httpicontsrchof_ {_httpimg_/tsrchof.gif}
+_httpicontsrchof_ [l=mi] {_httpimg_/mtsrchof.gif}
+_httpicontsrchon_ {_httpimg_/tsrchon.gif}
+_httpicontsrchon_ [l=mi] {_httpimg_/mtsrchon.gif}
+_widthtsrchx_ {87}
+_heighttsrchx_ {17}
+
+_httpicontlistgr_ {_httpimg_/tlistgr.gif}
+_httpicontlistof_ {_httpimg_/tlistof.gif}
+_httpicontliston_ {_httpimg_/tliston.gif}
+_widthtlistx_ {87}
+_heighttlistx_ {17}
+
+_httpiconttitlgr_ {_httpimg_/ttitlgr.gif}
+_httpiconttitlof_ {_httpimg_/ttitlof.gif}
+_httpiconttitlon_ {_httpimg_/ttitlon.gif}
+_widthttitlx_ {87}
+_heightttitlx_ {17}
+
+_httpicontpeopgr_ {_httpimg/tpeopgr.gif}
+_httpicontpeopof_ {_httpimg/tpeopof.gif}
+_httpicontpeopon_ {_httpimg/tpeopon.gif}
+_widthtpeopx_ {97}
+_heighttpeopx_ {17}
+
+_httpicontspace_ {_httpimg_/tspace.gif}
+_heighttspace_ {17}
+
+_httpiconwarning_ {_httpimg_/warning.gif}
+_widthwarning_ {30}
+_heightwarning_ {29}
+
+_httpiconhabout_ {_httpimg_/h\_about.gif}
+_httpiconhabout_ [l=mi] {_httpimg_/mh\_about.gif}
+_widthhabout_ {200}
+_widthhabout_ [l=mi] {262}
+_heighthabout_ {57}
+
+_httpiconhhome_ {_httpimg_/h\_home.gif}
+_widthhhome_ {200}
+_heighthhome_ {57}
+
+_httpiconhname_ {_httpimg_/h\_name.gif}
+_widthhname_ {200}
+_heighthname_ {57}
+
+_httpicontnamegr_ {_httpimg_/tnamegr.gif}
+_httpicontnameof_ {_httpimg_/tnameof.gif}
+_httpicontnameon_ {_httpimg_/tnameon.gif}
+_widthtnamex_ {95}
+_heighttnamex_ {17}
+
+_httpiconarrrght_ {_httpimg_/arrrght.gif}
+_widtharrrght_ {23}
+_heightarrrght_ {15}
+
+_httpiconopenbook_ {_httpimg_/openbook.gif}
+_widthopenbook_ {28}
+_heightopenbook_ {23}
+
+_httpiconaopenbk_ {_httpimg_/aopenbk.gif}
+_widthaopenbk_ {28}
+_heightaopenbk_ {23}
+
+_httpiconopenfldr_ {_httpimg_/openfldr.gif}
+_widthopenfldr_ {23}
+_heightopenfldr_ {15}
+
+_httpiconaopenfdr_ {_httpimg_/aopenfdr.gif}
+_widthaopenfdr_ {23}
+_heightaopenfdr_ {15}
+
+_httpiconbook_ {_httpimg_/book.gif}
+_widthbook_ {18}
+_heightbook_ {11}
+
+_httpiconabook_ {_httpimg_/abook.gif}
+_widthabook_ {18}
+_heightabook_ {11}
+
+_httpiconbshelf_ {_httpimg_/bshelf.gif}
+_widthbshelf_ {20}
+_heightbshelf_ {16}
+
+_httpiconabshelf_ {_httpimg_/abshelf.gif}
+_widthabshelf_ {20}
+_heightabshelf_ {16}
+
+_httpiconsmtext_ {_httpimg_/smtext.gif}
+_widthsmtext_ {23}
+_heightsmtext_ {15}
+
+_httpiconasmtext_ {_httpimg_/asmtext.gif}
+_widthasmtext_ {23}
+_heightasmtext_ {15}
+
+_httpiconclsdfldr_ {_httpimg_/clsdfldr.gif}
+_widthclsdfldr_ {23}
+_heightclsdfldr_ {15}
+
+_httpiconaclsdfdr_ {_httpimg_/aclsdfdr.gif}
+_widthaclsdfdr_ {23}
+_heightaclsdfdr_ {15}
+
+
+#######################################################################
+# Icons 
+#
+# Must not include links (so they can be used as links). If you want to
+# include links use _image
+#######################################################################
+
+_iconnext_{<img src="_httpiconmore_" align=absbottom 
+width=_widthmore_ height=_heightmore_ border=0 alt="_texticonnext_">}
+_iconnext_[v=1] {_texticonnext_}
+
+_iconprev_{<img src="_httpiconless_" align=absbottom 
+width=_widthless_ height=_heightless_ border=0 alt="_texticonprev_">}
+_iconprev_[v=1] {_texticonprev_}
+
+_icontabsearchgreen_ {<img 
+src="_httpicontsrchgr_" height=_heighttsrchx_ width=_widthtsrchx_ border=0>}
+_icontabsearchgreen_[v=1] {_texticontabsearchgreen_}
+
+_icontabListgreen_ {<img
+src="_httpicontlistgr_" height=_heighttlistx_ width=_widthtlistx_ border=0>}
+_icontablistgreen_ [v=1] {_texticontablistgreen_}
+
+_icontabTitlegreen_ {<img 
+src="_httpiconttitlgr_" height=_heightttitlx_ width=_widthttitlx_ border=0>}
+_icontabTitlegreen_[v=1] {_texticontabtitlegreen_}
+
+_icontabCreatorgreen_ {<img 
+src="_httpicontauthgr_" height=_heighttauthx_ width=_widthtauthx_ border=0>}
+_icontabCreatorgreen_[v=1] {_texticontabauthorgreen_}
+
+_icontabSeriesgreen_ {<img 
+src="_httpicontsergr_" height=_heighttserx_ width=_widthtserx_ border=0>}
+_icontabSeriesgreen_[v=1] {_texticontabseriesgreen_}
+
+_icontabDategreen_ {<img 
+src="_httpicontdategr_" height=_heighttdatex_ width=_widthtdatex_ border=0>}
+_icontabDategreen_[v=1] {_texticontabdategreen_}
+
+_icontabSubjectgreen_ {<img 
+src="_httpicontsubjgr_" height=_heighttsubjx_ width=widthtsubjx_ border=0>}
+_icontabSubjectgreen_[v=1] {_texticontabsubjectgreen_}
+
+_icontabHowtogreen_ {<img 
+src="_httpiconthowgr_" height=_heightthowx_ width=_widththowx_ border=0>}
+_icontabHowtogreen_[v=1] {_texticontabhowgreen_}
+
+_icontabTopicgreen_ {<img 
+src="_httpiconttopicgr_" height=_heightttopicx_ width=_widthttopicx_ border=0>}
+_icontabTopicgreen_[v=1] {_texticontabtopicgreen_}
+
+_icontabBrowsegreen_ {<img 
+src="_httpicontbrwsegr_" height=_heighttbrwsex_ width=_widthtbrwsex_ border=0>}
+_icontabBrowsegreen_[v=1] {_texticontabbrwsegreen_}
+
+_icontabOrganizationgreen_ {<img 
+src="_httpicontorggr_" height=_heighttorggr_ width=_widthtorggr_ border=0>}
+_icontabOrganizationgreen_[v=1] {_texticontaborggreen_}
+
+_icontabPeoplegreen_ {<img
+src="_httpicontpeopgr_" height=_heighttpeopx_ width=_widthtpeopx_ border=0>}
+_icontabPeoplegreen_[v=1] {_texticontabPeoplegreen_}
+
+_iconblankbar_ {<center><img src="_httpicondivb_" width=_widthdivb_ height=_heightdivb_></center>}
+_iconblankbar_[v=1] {<br><hr><br>}
+
+_icontext_ {<img border=0 align=absmiddle src="_httpiconitext_" width=_widthitext_ height=_heightitext_ alt="_texticontext_">}
+_icontext_[v=1] {_texticontext_}
+
+_iconblanktext_ {<img border=0 align=absmiddle src="_httpiconibtext_" width=_widthibtext_ height=_heightibtext_>}
+_iconblanktext_[v=1] {}
+
+_iconclosedbook_ {<img src="_httpiconbook_" width=_widthbook_ height=_heightbook_ border=0 alt="_texticonclosedbook_">}
+_iconclosedbook_ [v=1] {<small><b>_texticonclosedbook_: </b></small>}
+
+
+#######################################################################
+# Image links 
+#
+# These might include a link (not like an _icon_
+#######################################################################
+
+
+_imageabout_ {<a href="_httppageabout_" 
+    onMouseover = "img\_on('about')" 
+    onMouseout = "img\_off('about')"><img name="about"
+        src="_httpiconcabtof_" width=_widthcabtx_ height=_heightcabtx_
+        border=0 alt="_textimageabout_"></a>}
+
+_imageabout_[v=1] {
+<a href="_httppageabout_">_textimageabout_</a>
+}
+
+_imagehome_ {<a href="_httppagehome_" 
+    onMouseover = "img\_on('home')" 
+    onMouseout = "img\_off('home')"><img name="home"
+        src="_httpiconchomeof_" width="_widthchomex_" height="_heightchomex_"
+        border="0" alt="_textimagehome_"></a>}
+
+_imagehome_[v=1] {
+<a href="_httppagehome_">_textimagehome_</a>
+}
+
+
+_imagehelp_ {<a href="_httppagehelp_"
+    onMouseover = "img\_on('help')" 
+    onMouseout = "img\_off('help')"><img name="help" 
+        src="_httpiconchelpof_" width="_widthchelpx_" height="_heightchelpx_" 
+        border="0" alt="_textimagehelp_"></a>}
+
+_imagehelp_[v=1] {
+<a href="_httppagehelp_">_textimagehelp_</a>
+}
+
+
+_imagepref_ {<a href="_httppagepref_"
+    onMouseover = "img\_on('pref')" 
+    onMouseout = "img\_off('pref')"><img name="pref" 
+        src="_httpiconcprefof_" width="_widthcprefx_" height="_heightcprefx_" border="0" alt="_textimagepref_"></a>}
+
+_imagepref_[v=1] {
+<a href="_httppagepref_">_textimagepref_</a>
+}
+
+_imagegreenstone_ {<img src="_httpicongsdl_" width="_widthgsdl_" height="_heightgsdl_" border="0" alt="_textimagegreenstone_" hspace=0>}
+
+_imagegreenstone_[v=1] {_textimagegreenstone_}
+
+
+################################
+# English Macros               #
+################################
+
+_textimagecollection_ {}
+_textimageabout_ {About page}
+_textimagehome_ {Home page}
+_textimagehelp_ {Help page}
+_textimagepref_ {Preferences page}
+_textimagegreenstone_ {Greenstone Digital Library Software}
+
+_textimagesearch_ {Search for specific terms}
+_textimageTitle_ {Browse alphabetical list of titles}
+_textimageList_ {Browse document listing}
+_textimageCreator_ {Browse alphabetical list of authors}
+_textimageSeries_ {Browse by issue}
+_textimageDate_ {Browse by date}
+_textimageSubject_ {Browse by subject category}
+_textimageOrganization_ {Browse by organization}
+_textimageHowto_ {Browse how to categories}
+_textimageTopic_ {Browse special topic issues}
+_textimageBrowse_ {Browse}
+_textimagePeople_ {Browse alphabetical list of People}
+
+_texticontabsearchgreen_{Search}
+_texticontabdategreen_{Dates}
+_texticontabseriesgreen_{Series}
+_texticontabauthorgreen_{Authors A-Z}
+_texticontabtitlegreen_{Titles A-Z}
+_texticontablistgreen_ {Listing}
+_texticontabsubjectgreen_{Subjects}
+_texticontaborggreen_{Organization}
+_texticontabhowgreen_{How to}
+_texticontabtopicgreen_{Topics}
+_texticontabbrwsegreen_{Browse}
+_texticontabPeoplegreen_{People A-Z}
+
+_texticontext_ {View the document}
+_texticonclosedbook_ {open this document and view contents}
+_texticonnext_ {to next section}
+_texticonprev_ {to previous section}
+
+_page_ {page }
+_of_ { of }
+_vol_ {Volume}
+_num_ {Number}
+
+_textmonth00_ {}
+_textmonth01_ {January}
+_textmonth02_ {February}
+_textmonth03_ {March}
+_textmonth04_ {April}
+_textmonth05_ {May}
+_textmonth06_ {June}
+_textmonth07_ {July}
+_textmonth08_ {August}
+_textmonth09_ {September}
+_textmonth10_ {October}
+_textmonth11_ {November}
+_textmonth12_ {December}
+
+_magazines_ {Magazines}
+
+################################
+# Maori Macros                 #
+################################
+
+_textimagehelp_ [l=mi] {Te wh&auml;rangi whakam&auml;rama}
+_textimagepref_ [l=mi] {Te wh&auml;rangi o ng&auml; whiriwhiringa}
+_textimagehome_ [l=mi] {Te wh&auml;rangi k&auml;inga}
+_textimagesearch_ [l=mi] {rapua he kupu}
+_textimagebrowseseries_ [l=mi] {Tirohia te whakar&auml;rangi o ng&auml; niupepa}
+_textimagebrowsedates_ [l=mi] {Tirohia te w&auml; i puta mai ai ng&auml; niupepa}
+
+_page_ [l=mi] {wh&auml;rangi }
+_vol_ [l=mi] {Pukapuka}
+_num_ [l=mi] {Nama}
+_textmonth01_ [l=mi] {Hanuere}
+_textmonth02_ [l=mi] {Pepuere}
+_textmonth03_ [l=mi] {Maehe}
+_textmonth04_ [l=mi] {Aperira}
+_textmonth05_ [l=mi] {Mei}
+_textmonth06_ [l=mi] {Hune}
+_textmonth07_ [l=mi] {Hurae}
+_textmonth08_ [l=mi] {Akuhata}
+_textmonth09_ [l=mi] {Hepetema}
+_textmonth10_ [l=mi] {Oketopa}
+_textmonth11_ [l=mi] {Noema}
+_textmonth12_ [l=mi] {Tihema}
+_of_ [l=mi] { i te }
+
+
+_texticontabsearchgreen_ [l=mi] {rapu}
+_texticontabdatesgreen_ [l=mi] {w&auml;taka}
+_texticontabseriesgreen_ [l=mi] {whakar&auml;rangi niupepa}
+_texticonnext_ [l=mi] {titiro ki te wh&auml;rangi o muri}
+_texticonprev_ [l=mi] {titiro ki te wh&auml;rangi o mua}
+_texticontext_ [l=mi] {Tirohia te wh&auml;rangi}
+
+_textimageSeries_ [l=mi] {Tirohia te whakar&auml;rangi taitara}
+_textimageDate_ [l=mi] {Tirohia te whakar&auml;rangi w&auml;taka}
Index: /tags/gsdl-21/gsdl/macros/document.dm
===================================================================
--- /tags/gsdl-21/gsdl/macros/document.dm	(revision 699)
+++ /tags/gsdl-21/gsdl/macros/document.dm	(revision 699)
@@ -0,0 +1,515 @@
+package document
+
+#######################################################################
+# macros set from within the server
+#######################################################################
+
+_classificationlinks_ {}
+_imagethispage_ {}
+_httpprevarrow_ {}
+_httpnextarrow_ {}
+_pagetitle_ {}
+
+#######################################################################
+# page content
+#######################################################################
+
+_content_ {
+<center>
+_navigationbar_
+<p>
+_classificationlinks_
+</center>
+}
+
+
+#######################################################################
+# javascript macros
+#
+# there should be one of these for each flashy javascript button
+# that may appear within a document page
+#######################################################################
+
+# javaextras allows specific collections to include extra java macros
+# without having to override their document action
+_javaextras_ {}
+
+_javadetach_ {
+	detach\_on = new Image(_widthedtchx_, _heightedtchx_); 
+	detach\_on.src = "_httpiconedtchon_";
+	detach\_off = new Image(_widthedtchx_, _heightedtchx_); 
+	detach\_off.src = "_httpiconedtchof_";
+}
+
+_javaexpandcontents_ {
+	expcon\_on = new Image(_widtheexpcx_, _heighteexpcx_);
+	expcon\_on.src = "_httpiconeexpcon_";
+	expcon\_off = new Image(_widtheexpcx_, _heighteexpcx_);
+	expcon\_off.src = "_httpiconeexpcof_";
+}
+
+_javacontractcontents_ {
+	concon\_on = new Image(_widtheconcx_, _heighteconcx_);
+	concon\_on.src = "_httpiconeconcon_";
+	concon\_off = new Image(_widtheconcx_, _heighteconcx_);
+	concon\_off.src = "_httpiconeconcof_";
+}
+
+_javaexpandtext_ {
+	alltext\_on = new Image(_widthealltx_, _heightealltx_);
+	alltext\_on.src = "_httpiconeallton_";
+	alltext\_off = new Image(_widthealltx_, _heightealltx_);
+	alltext\_off.src = "_httpiconealltof_";
+}
+
+_javacontracttext_ {
+	thissec\_on = new Image(_widthetsecx_, _heightetsecx_);
+	thissec\_on.src = "_httpiconetsecon_";
+	thissec\_off = new Image(_widthetsecx_, _heightetsecx_);
+	thissec\_off.src = "_httpiconetsecof_";
+}
+
+_javahighlighting_ {
+	hl\_on = new Image(_widthehlx_, _heightehlx_);
+	hl\_on.src = "_httpiconehlon_";
+	hl\_off = new Image(_widthehlx_, _heightehlx_);
+	hl\_off.src = "_httpiconehlof_";
+}
+
+_javanohighlighting_ {
+	nhl\_on = new Image(_widthenhlx_, _heightenhlx_);
+	nhl\_on.src = "_httpiconenhlon_";
+	nhl\_off = new Image(_widthenhlx_, _heightenhlx_);
+	nhl\_off.src = "_httpiconenhlof_";
+}
+
+_javacontinue_ {
+	cont\_on = new Image(_widthcontx_, _heightcontx_);
+	cont\_on.src = "_httpiconconton_";
+	cont\_off = new Image(_widthcontx_, _heightcontx_);
+	cont\_off.src = "_httpiconcontoff_";
+}
+
+#######################################################################
+# navigation arrows
+#
+#######################################################################
+
+_navarrows_ {<center>
+<table width=_pagewidth_ cellpadding=0 cellspacing=0 border=0>
+<tr><td align=left valign=top>
+_prevarrow_
+</td><td align=right valign=top>
+_nextarrow_
+</td></tr></table>
+</center>
+}
+
+_prevarrow_ {_If_(_httpprevarrow_,<a href="_httpprevarrow_">_iconprev_</a>)}
+_nextarrow_ {_If_(_httpnextarrow_,<a href="_httpnextarrow_">_iconnext_</a>)}
+
+_navarrows_ [v=1] {<p>
+_prevarrow_<br>
+_nextarrow_
+}
+
+#######################################################################
+# the goto form
+#######################################################################
+
+_gotoform_ {
+<form name="GotoForm" method="get" action="_gwcgi_">
+<input type=hidden name="e" value="_compressedoptions_">
+<input type=hidden name="d" value="_cgiargd_">
+<input type=hidden name="cl" value="_cgiargcl_">
+<input type="text" name="gp" size="3" maxlength="4">
+<input type="submit" value="go to page">
+</form>
+}
+
+_textnumpages_ {&nbsp;<i>(_1_ pages)</i>}
+_textpage_ {page }
+_textintro_ { <i>(introductory text)</i>}
+_parentarrow_ {}
+
+
+#######################################################################
+# icons
+#######################################################################
+
+# title images - there should be one of these for each classification
+# supported by this receptionist
+
+_iconTitlepage_ {<img src="_httpiconhtitle_" width=_widthhtitle_
+height=_heighthtitle_}
+_iconTitlepage_ [v=1] {<h2>_texticonhtitle_</h2>}
+
+_iconListpage_ {<img src="_httpiconhlist_" width=_widthhlist_
+height=_heighthlist_}
+_iconListpage_ [v=1] {<h2>_texticonhlist_</h2>}
+
+_iconCreatorpage_ {<img src="_httpiconhauth_" width=_widthhauth_
+height=_heighthauth_}
+_iconCreatorpage_ [v=1] {<h2>_texticonhauth_</h2>}
+
+_iconSubjectpage_ {<img src="_httpiconhsubj_" width=_widthhsubj_
+height=_heighthsubj_}
+_iconSubjectpage_ [v=1] {<h2>_texticonhsubj_</h2>}
+
+_iconSeriespage_ {<img src="_httpiconhser_" width=_widthhser_
+height=_heighthser_}
+_iconSeriespage_ [v=1] {<h2>_texticonhser_</h2>}
+
+_iconDatepage_ {<img src="_httpiconhdate_" width=_widthhdate_
+height=_heighthdate_}
+_iconDatepage_ [v=1] {<h2>_texticonhdate_</h2>}
+
+_iconHowtopage_ {<img src="_httpiconhhow_" width=_widthhhow_
+height=_heighthhow_}
+_iconHowtopage_ [v=1] {<h2>_texticonhhow_</h2>}
+
+_iconTopicpage_ {<img src="_httpiconhtopic_" width=_widthhtopic_
+height=_heighthtopic_}
+_iconTopicpage_ [v=1] {<h2>_texticonhtopic_</h2>}
+
+_iconOrganizationpage_ {<img src="_httpiconhorg_" width=_widthhorg_
+height=_heighthorg_}
+_iconOrganizationpage_ [v=1] {<h2>_texticonhorg_</h2>}
+
+_iconBrowsepage_ {<img src="_httpiconhbrwse_" width=_widthhbrwse_
+height=_heighthbrwse_}
+_iconBrowsepage_ [v=1] {<h2>_texticonhbrwse_</h2>}
+
+_iconPeoplepage_ {<img src="_httpiconhpeople_" width=_widthhpeople_
+height=_heighthpeople_}
+_iconPeoplepage_ [v=1] {<h2>_texticonhpeople_</h2>}
+
+
+_iconsmalltext_ {<img src="_httpiconsmtext_" border=0 width=_widthsmtext_ height=_heightsmtext_ alt="_texticonsmalltext_">}
+_iconsmalltext_ [v=1] {<small><b>_texticonsmalltext2_</b></small>}
+
+_iconarrowsmalltext_ {<img src="_httpiconasmtext_" border=0 width=_widthasmtext_ height=_heightasmtext_ alt="_texticonsmalltext_">}
+_iconarrowsmalltext_ [v=1] {<small><b>->_texticonsmalltext2_</b></small>}
+
+_iconclosedfolder_ {<img src="_httpiconclsdfldr_" border=0 width=_widthclsdfldr_ height=_heightclsdfldr_ alt="_texticonclosedfolder_">}
+_iconclosedfolder_ [v=1] {<small><b>_texticonclosedfolder2_</b></small>}
+
+_iconarrowclosedfolder_ {<img src=_httpiconaclsdfdr_" border=0 width=_widthaclsdfdr_ height=_heightaclsdfdr_ alt="_texticonclosedfolder_">}
+_iconarrowclosedfolder_ [v=1] {<small><b>->_texticonclosedfolder2_</b></small>}
+
+_iconopenfolder_ {<img src="_httpiconopenfldr_" border=0 width=_widthopenfldr_ height=_heightopenfldr_ alt="_texticonopenfolder_">}
+_iconopenfolder_ [v=1] {<small><b>_texticonopenfolder2_</b></small>}
+
+_iconarrowopenfolder_ {<img src="_httpiconaopenfdr_" border=0 width=_widthaopenfdr_ height=_heightaopenfdr_ alt="_texticonopenfolder_">}
+_iconarrowopenfolder_ [v=1] {<small><b>->_texticonopenfolder2_</b></small>}
+
+_iconarrowclosedbook_ {<img src="_httpiconabook_" width=_widthabook_ height=_heightabook_ border=0 alt="_texticonclosedbook_">}
+_iconarrowclosedbook_ [v=1] {<small><b>->_texticonclosedbook_: </b></small>}
+
+_iconopenbook_ {<img src="_httpiconopenbook_" width=_widthopenbook_ height=_heightopenbook_ border=0 alt="_texticonopenbook_">}
+_iconopenbook_ [v=1] {<small><b>_texticonopenbook_: </b></small>}
+
+_iconarrowopenbook_ {<img src="_httpiconaopenbk_" width=_widthaopenbk_ height=_heightaopenbk_ border=0 alt="_texticonopenbook_">}
+_iconarrowopenbook_ [v=1] {<small><b>->_texticonopenbook_: </b></small>}
+
+_iconopenbookshelf_ {<img src="_httpiconbshelf_" border=0 width=_widthbshelf_ height=_heightbshelf_ alt="_texticonopenbookshelf_">}
+_iconopenbookshelf_ [v=1] {<small><b>_texticonopenbookshelf_: </b></small>}
+
+_iconarrowopenbookshelf_ {<img src="_httpiconabshelf_" border=0 width=_widthabshelf_ height=_heightabshelf_ alt="_texticonopenbookshelf_">}
+_iconarrowopenbookshelf_ [v=1] {<small><b>->_texticonopenbookshelf_: </b></small>}
+
+_iconclosedbookshelf_ {<img src="_httpiconbshelf_" border=0 width=_widthbshelf_ height=_heightbshelf_ alt="_texticonclosedbookshelf_">}
+_iconclosedbookshelf_ [v=1] {<small><b>_texticonclosedbookshelf_: </b></small>}
+
+_iconarrowclosedbookshelf_ {<img src="_httpiconabshelf_" border=0 width=_widthabshelf_ height=_heightabshelf_ alt="_texticonclosedbookshelf_">}
+_iconarrowclosedbookshelf_ [v=1] {<small><b>->_texticonclosedbookshelf_: </b></small>}
+
+_iconpointer_ {<img src="_httpiconarrrght_" border=0 width=_widtharrrght_ height=_heightarrrght_>}
+_iconpointer_ [v=1] {<small><b>_texticonpointer_-> </b></small>}
+
+_icontab_ {<img src="_httpicontabspace_" width=_widthtabspace_ height=_heighttabspace_>}
+_tab_ {<td>_icontab_</td>}
+
+_iconcontracttoc_ {<img name="concon" src="_httpiconeconcof_" width=_widtheconcx_ height=_heighteconcof_ alt="_texticoncontracttoc_" border=0>}
+_iconcontracttoc_[v=1] {_texticoncontracttoc_}
+
+_iconexpandtoc_ {<img name="expcon" src="_httpiconeexpcof_" width=_widtheexpx_ height=_heighteexpcx_ alt="_texticonexpandtoc_" border=0>}
+_iconexpandtoc_[v=1] {_texticonexpandtoc_}
+
+_iconcontracttext_ {<img name="thissec" src="_httpiconetsecof_" width=_widthetsecx_ height=_heightetsecx_ alt="_texticoncontracttext_" border=0>}
+_iconcontracttext_[v=1] {_texticoncontracttext_}
+
+_iconexpandtext_ {<img name="alltext" src="_httpiconealltof_" width=_widthealltx_ height=_heightealltx_ alt="_texticonexpandtext_" border=0>}
+_iconexpandtext_[v=1] {_texticonexpandtext_}
+
+_icondetach_ {<img name="detach" src="_httpiconedtchof_" width=_widthedtchx_ height=_heightedtchx_ alt="_texticondetach_" border=0>}
+_icondetach_[v=1] {_texticondetach_}
+
+_iconhighlight_ {<img name="hl" src="_httpiconehlof_" width=_widthehlx_ height=_heightehlx_ alt="_texticonhighlight_" border=0>}
+_iconhighlight_ [v=1] {_texticonhighlight_}
+
+_iconnohighlight_ {<img name="nhl" src="_httpiconenhlof_" width=_widthenhlx_ height=_heightenhlx_ alt="_texticonnohighlight_" border=0>}
+_iconnohighlight_ [v=1] {_texticonnohighlight_}
+
+_iconwarning_ {<img src="_httpiconwarning_" width=_widthwarning_ height=_heightwarning_ border=0 align=left>}
+_iconwarning_ [v=1] {_texticonwarning_}
+
+_iconcont_ {<img name="cont" src="_httpiconcontoff_" width=_widthcontx_ height=_heightcontx_ border=0>}
+_iconcont_ [v=1] {_texticoncont_}
+
+
+#######################################################################
+# images
+#######################################################################
+
+_imagedetach_ {<a href="_httpcurrentdocument_&x=1" target=\_blank onMouseover = "img\_on('detach')"
+onMouseout = "img\_off('detach')">_icondetach_</a>}
+_imagedetach_[v=1] {<a href="_httpcurrentdocument_&x=1" target=\_blank>_texticondetach_</a><br>}
+
+_imagehighlight_ {<a href="_httpcurrentdocument_&hl=1&gc=_cgiarggc_&amp;gt=_cgiarggt_" onMouseover = "img\_on('hl')"
+onMouseout = "img\_off('hl')">_iconhighlight_</a>}
+_imagehighlight_ [v=1] {<a href="_httpcurrentdocument_&hl=1&gc=_cgiarggc_&amp;gt=_cgiarggt_">_texticonhighlight_</a><br>}
+
+_imagenohighlight_ {<a href="_httpcurrentdocument_&hl=0&gc=_cgiarggc_&amp;gt=_cgiarggt_" onMouseover = "img\_on('nhl')"
+onMouseout = "img\_off('nhl')">_iconnohighlight_</a>}
+_imagenohighlight_ [v=1] {<a href="_httpcurrentdocument_&hl=0&gc=_cgiarggc_&amp;gt=_cgiarggt_">_texticonnohighlight_</a><br>}
+
+_imagecontracttoc_ {<a href="_httpcurrentdocument_&gc=0" onMouseover = "img\_on('concon')"
+onMouseout = "img\_off('concon')">_iconcontracttoc_</a>}
+_imagecontracttoc_[v=1] {"<a href="_httpcurrentdocument_&gc=0">_texticoncontracttoc_</a><br>}
+
+_imageexpandtoc_ {<a href="_httpcurrentdocument_&gc=1" onMouseover = "img\_on('expcon')"
+onMouseout = "img\_off('expcon')">_iconexpandtoc_</a>}
+_imageexpandtoc_ [v=1] {"<a href="_httpcurrentdocument_&gc=1">_texticonexpandtoc_</a><br>}
+
+_imagecontracttext_ {<a href="_httpcurrentdocument_&amp;gt=0" onMouseover = "img\_on('thissec')"
+onMouseout = "img\_off('thissec')">_iconcontracttext_</a>}
+_imagecontracttext_[v=1] {<a href="_httpcurrentdocument_&amp;gt=0">display only current section</a><br>}
+
+_imageexpandtext_ {<a href="_httpcurrentdocument_&amp;gt=1" onMouseover = "img\_on('alltext')"
+onMouseout = "img\_off('alltext')">_iconexpandtext_</a>}
+_imageexpandtext_[v=1] {<a href="_httpcurrentdocument_&amp;gt=1">display all text</a><br>}
+
+_imagecont_ {<a href="_httpcurrentdocument_&amp;gt=2" onMouseover = "img\_on('cont')"
+onMouseout = "img\_off('cont')">_iconcont_</a>}
+_imagecont_[v=1] {<a href="_httpcurrentdocument&amp;gt=2">_texticoncont_</a><br>}
+
+
+#######################################################################
+# https
+#
+# These contain the url without any quotes
+#######################################################################
+
+_httpiconhtitle_ {_httpimg_/h\_title.gif}
+_widthhtitle_ {200}
+_heighthtitle_ {57}
+
+_httpiconhlist_ {_httpimg_/h\_list.gif}
+_widthhlist_ {200}
+_heighthlist_ {57}
+
+_httpiconhauth_ {_httpimg_/h\_auth.gif}
+_widthhauth_ {200}
+_heighthauth_ {57}
+
+_httpiconhsubj_ {_httpimg_/h\_subj.gif}
+_widthhsubj_ {200}
+_heighthsubj_ {57}
+
+_httpiconhser_ {_httpimg_/h\_ser.gif}
+_httpiconhser_ [l=mi] {_httpimg_/mh\_ser.gif}
+_widthhser_ {200}
+_widthhser_ [l=mi] {310}
+_heighthser_ {57}
+
+_httpiconhdate_ {_httpimg_/h\_date.gif}
+_httpiconhdate_ [l=mi] {_httpimg_/mh\_date.gif}
+_widthhdate_ {200}
+_heighthdate_ {57}
+
+_httpiconhhow_ {_httpimg_/h\_how.gif}
+_widthhhow_ {200}
+_heighthhow_ {57}
+
+_httpiconhtopic_ {_httpimg_/h\_topic.gif}
+_widthhtopic_ {200}
+_heighthtopic_ {57}
+
+_httpiconhorg_ {_httpimg_/h\_org.gif}
+_widthhorg_ {250}
+_heighthorg_ {57}
+
+_httpiconhbrwse_ {_httpimg_/h\_brwse.gif}
+_widthhbrwse_ {200}
+_heighthbrwse_ {57}
+
+_httpiconhpeople_ {_httpimg_/h\_people.gif}
+_widthhpeople_ {200}
+_heighthpeople_ {57}
+
+_httpiconcontoff_ {_httpimg_/contoff.gif}
+_httpiconconton_ {_httpimg_/conton.gif}
+_widthcontx_ {76}
+_heightcontx_ {17}
+
+_httpiconealltof_ {_httpimg_/ealltof.gif}
+_httpiconealltof_ [l=mi] {_httpimg_/mealltof.gif}
+_httpiconeallton_ {_httpimg_/eallton.gif}
+_httpiconeallton_ [l=mi] {_httpimg_/meallton.gif}
+_widthealltx_ {46}
+_widthealltx_ [l=mi] {42}
+_heightealltx_ {30}
+
+_httpiconeconcof_ {_httpimg_/econcof.gif}
+_httpiconeconcon_ {_httpimg_/econcon.gif}
+_widtheconcx_ {63}
+_heighteconcx_ {30}
+
+_httpiconedtchof_ {_httpimg_/edtchof.gif}
+_httpiconedtchof_ [l=mi] {_httpimg_/medtchof.gif}
+_httpiconedtchon_ {_httpimg_/edtchon.gif}
+_httpiconedtchon_ [l=mi] {_httpimg_/medtchon.gif}
+_widthedtchx_ {47}
+_widthedtchx_ [l=mi] {56}
+_heightedtchx_ {30}
+
+_httpiconeexpcof_ {_httpimg_/eexpcof.gif}
+_httpiconeexpcon_ {_httpimg_/eexpcon.gif}
+_widtheexpcx_ {60}
+_heighteexpcx_ {30}
+
+_httpiconetsecof_ {_httpimg_/etsecof.gif}
+_httpiconetsecof_ [l=mi] {_httpimg_/metsecof.gif}
+_httpiconetsecon_ {_httpimg_/etsecon.gif}
+_httpiconetsecon_ [l=mi] {_httpimg_/metsecon.gif}
+_widthetsecx_ {63}
+_widthetsecx_ [l=mi] {53}
+_heightetsecx_ {30}
+
+_httpiconehlof_ {_httpimg_/ehlof.gif}
+_httpiconehlof_ [l=mi] {_httpimg_/mehlof.gif}
+_httpiconehlon_ {_httpimg_/ehlon.gif}
+_httpiconehlon_ [l=mi] {_httpimg_/mehlon.gif}
+_widthehlx_ {53}
+_widthehlx_ [l=mi] {47}
+_heightehlx_ {30}
+
+_httpiconenhlof_ {_httpimg_/enhlof.gif}
+_httpiconenhlof_ [l=mi] {_httpimg_/menhlof.gif}
+_httpiconenhlon_ {_httpimg_/enhlon.gif}
+_httpiconenhlon_ [l=mi] {_httpimg_/menhlon.gif}
+_widthenhlx_ {57}
+_widthenhlx_ [l=mi] {56}
+_heightenhlx_ {30}
+
+
+#######################################################################
+# headers/footers
+#######################################################################
+
+
+# header overridden for text pages
+_textheader_ {_cgihead_
+_htmlhead_
+<center>
+<table width=_pagewidth_><tr><td align=right>
+_javalinks_</td></tr></table>
+</center>
+}
+_textheader_ [v=1] {_cgihead_
+_htmlhead_
+_javalinks_
+}
+
+_footer_ {
+<p>
+_navarrows_
+</table>
+_endspacer__htmlfooter_
+}
+
+
+#######################################################################
+# English language text macros
+#######################################################################
+
+_textTitlepage_ {_texticonhtitle_}
+_textCreatorpage_ {_texticonhauth_}
+_textSubjectpage_ {_texticonhsubj_}
+_textSeriespage_ {_texticonhser_}
+_textDatepage_ {_texticonhdate_}
+_textHowtopage_ {_texticonhhow_}
+_textOrganizationpage_ {_texticonhorg_}
+_textBrowsepage_ {_texticonhbrwse_}
+_textPeoplepage_ {_texticonhpeople_}
+
+_texticonhtitle_ {Titles A-Z}
+_texticonhauth_ {Authors A-Z}
+_texticonhsubj_ {Subjects}
+_texticonhser_ {Series}
+_texticonhdate_ {Dates}
+_texticonhhow_ {How to}
+_texticonhorg_ {Organizations}
+_texticonhbrwse_ {Browse}
+_texticonhpeople_ {People A-Z}
+
+_texticonopenbookshelf_ {close this section of the library}
+_texticonclosedbookshelf_ {open this section of the library and view contents}
+_texticonopenbook_ {close this book}
+_texticonclosedfolder_ {Open this folder and view contents}
+_texticonclosedfolder2_ {open sub-section: }
+_texticonopenfolder_ {close this folder}
+_texticonopenfolder2_ {close sub-section: }
+_texticonsmalltext_ {View this section of the text}
+_texticonsmalltext2_ {view text: }
+_texticonpointer_ {current section}
+_texticondetach_ {Open this page in a new window}
+_texticonhighlight_ {Highlight search terms}
+_texticonnohighlight_ {Don't highlight search terms}
+_texticoncontracttoc_ {Collapse table of contents}
+_texticonexpandtoc_ {Expand table of contents}
+_texticonexpandtext_ {Display all text}
+_texticoncontracttext_ {Display text only for currently selected section}
+_texticonwarning_ {<b>Warning: </b>}
+_texticoncont_ {continue?}
+
+_textltwarning_ {
+<p><center>
+<table cellspacing=0 cellpadding=0 width=_pagewidth_>
+<tr valign=top><td>_iconwarning_Expanding the text here will generate a large 
+amount of data for your browser to display _imagecont_
+</td></tr></table></center>
+}
+
+
+#######################################################################
+# Maori text macros
+#######################################################################
+
+_texticondetach_ [l=mi] {Huakina mai he matapihi hou m&ouml; t&euml;nei k&ouml;rero}
+_texticonexpandtext_ [l=mi] {Titiro ki ng&auml; kupu katoa o t&euml;nei niupepa}
+_texticoncontracttext_ [l=mi] {Titiro ki ng&auml; kupu o t&euml;nei wh&auml;rangi anake}
+_texticonopenbookshelf_ [l=mi] {k&auml;tia}
+_texticonclosedbookshelf_ [l=mi] {whakat&uuml;whera mai}
+_texticonclosedbook_ [l=mi] {whakat&uuml;whera mai}
+_texticonopenbook_ [l=mi] {k&auml;tia}
+_texticonpointer_ [l=mi] {w&auml;hanga e tiro ana}
+_texticonhighlight_ [l=mi] {Kia p&uuml;kaha ng&auml; kupu rapu}
+_texticonnohighlight_ [l=mi] {Kia kore he p&uuml;kaha}
+_texticonsmalltext_ [l=mi] {He wh&auml;rangi niupepa}
+_texticonclosedfolder_ [l=mi] {He k&ouml;paki}
+
+
+
+#######################################################################
+# Chinese text macros
+#######################################################################
+
+
+_texticonsmalltext_ [l=zh] {ãæµè§è¯¥éšä»œææ¬}
+_texticonclosedfolder_ [l=zh] {ãæåŒæ­€æä»¶å€¹ä»¥æµè§ç®åœ}
+_texticondetach_ [l=zh] {ãæåŒäžäžªæ°ççªå£æŸç€ºæ¬é¡µ}
+_texticonexpandtoc_ [l=zh] {ãæ©å±ä¹Šåœè¡š}
+_texticonexpandtext_ [l=zh] {ãæŸç€ºææææ¬}
+_texticonhighlight_ [l=zh] {ãæ¥è¯¢é¡¹ä»¥ç²äœæŸç€º}
Index: /tags/gsdl-21/gsdl/macros/help.dm
===================================================================
--- /tags/gsdl-21/gsdl/macros/help.dm	(revision 699)
+++ /tags/gsdl-21/gsdl/macros/help.dm	(revision 699)
@@ -0,0 +1,612 @@
+package help
+
+_imagethispage_ {_iconthispage_}
+_iconthispage_ {<img src="_httpimg_/h_help.gif" alt="Help" width="200" height="57">}
+_iconthispage_ [l=mi] {<img src="_httpimg_/mh_help.gif" alt="&Auml;whina" width="200" height="57">}
+
+
+_simplehelp_ {
+_textnumbrowseoptions_
+_textbrowseoptions_
+<p><br>_iconexplain_
+}
+
+_textSearchshort_ {search for particular words}
+_textSearchshort_ [l=mi] {rapua he kupu}
+_textSeriesshort_ {access publications by series}
+_textSeriesshort_ [l=mi] {tirohia te whakar&auml;rangi taitara}
+_textDateshort_ {access publications by date}
+_textDateshort_ [l=mi] {tirohia te whakar&auml;rangi e p&auml; ana ki te w&auml;}
+_textSubjectshort_ {access publications by subject}
+_textTitleshort_ {access publications by title}
+_textBrowseshort_ {browse publications}
+_textCreatorshort_ {access publications by author}
+_textOrganizationshort_ {access publications by organization}
+_textHowtoshort_ {access publications by how to listing}
+_textTopicshort_ {access publications by topic}
+_textPeopleshort_ {access publications by people}
+_textUnknownshort_ {unknown browse option}
+
+_textSearchlong_ {
+<p>You can <i>search for particular words</i> that appear in the text from the
+"search" page. This is the first page that comes up when you begin, and can be
+reached from other pages by pressing the <i>search</i> button.
+}
+
+_textSearchlong_ [l=mi] {
+<p>Ko te wh&auml;rangi rapu te wh&auml;rangi <i>e rapu atu ai ng&auml; kupu</i> i roto
+i ng&auml; tuhinga. Kia puta mai ai t&euml;nei wh&auml;rangi, i a koe e titiro ana ki 
+te kohinga niupepa nei p&auml;whiritia te p&auml;tene <i>rapu</i>.
+}
+
+_textTitlelong_ {
+<p>You can <i>access publications by title</i> by pressing the <i>titles a-z</i>
+button.  This brings up a list of books in alphabetic order.
+}
+
+_textOrganizationlong_ {
+<p>You can <i>access publications by organization</i> by pressing the <i>organization</i>
+button.  This brings up a list of organizations.
+}
+
+_textHowtolong_ {
+<p>You can <i>access publications by how to listing</i> by pressing the <i>how to</i>
+button.  This brings up a list of how to strings.
+}
+
+_textCreatorlong_ {
+<p>You can <i>access publications by author</i> by pressing the <i>authors a-z</i>
+button.  This brings up a list of books, sorted by author name.
+}
+
+_textTopiclong_ {
+<p>You can <i>access publications by topic</i> by pressing the <i>topic</i>
+button.  This brings up a list of topics to browse.
+}
+
+_textSubjectlong_ {
+<p>You can <i>access publications by subject</i> by pressing the <i>subjects</i>
+button.  This brings up a list of subjects, represented by bookshelves.
+}
+
+_textSerieslong_ {
+<p>You can <i>access publications by series</i> by pressing the <i>series</i> button. 
+This brings up a list of those series which are currently in the collection. 
+}
+
+_textSerieslong_ [l=mi] {
+<p>Kia <i>tirohia te whakar&auml;rangi taitara</i> p&auml;whiritia te p&auml;tene 
+<i>whakar&auml;rangi taitara</i>. Ka puta mai te whakar&auml;rangi o ng&auml;
+niupepa e p&auml; ana ki ng&auml; taitara.
+}
+
+_textDatelong_ {
+<p>You can <i>access publications by date</i> by pressing the <i>dates</i> button. 
+This brings up a list of all the issues, sorted chronologically.
+}
+
+_textDatelong_ [l=mi] {<p>Kia <i>tirohia te whakar&auml;rangi e p&auml; ana ki te w&auml;</i> p&auml;whiritia te
+p&auml;tene <i>w&auml;taka</i>. Ka puta mai te whakar&auml;rangi o ng&auml; niupepa
+e p&auml; ana ki te w&auml; i puta mai ai aua niupepa.
+}
+
+_textBrowselong_ {
+<p>You can <i>browse publications</i> by pressing the <i>browse</i> button.
+}
+
+_textPeoplelong_ {
+<p>You can <i>access publications by people</i> by pressing the 
+   <i>people a-z</i> button.  This brings up a list of entries, sorted by 
+     surname.
+}
+
+_textUnknownlong_ {
+<p>Click on the <i>unknown</i> button to browse publications (listing is sorted
+by an unknown field).
+}
+
+
+_iconexplain_ {
+<table cellspacing="0" cellpadding="2" border="0">
+<tr valign="top"><td><img src="_httpimg_/book.gif" width="18" height="11" border="0"></td>
+<td>Click on a book icon to read the corresponding book</td></tr></table>
+<table cellspacing="0" cellpadding="2" border="0">
+<tr valign="top"><td><img src="_httpimg_/bshelf.gif" border="0" width=20 height=16></td>
+<td>Click on a bookshelf icon to look at books on that subject</td></tr>
+</table>
+}
+
+_iconexplain_ [l=mi] {
+<table cellspacing="0" cellpadding="2" border="0">
+<tr valign="top"><td><img src="_httpimg_/book.gif" width="18" height="11" border="0"></td>
+<td>Pat&ouml;hia te p&auml;tene pukapuka kia puta mai ai taua pukapuka</td></tr></table>
+<table cellspacing="0" cellpadding="2" border="0">
+<tr valign="top"><td><img src="_httpimg_/bshelf.gif" border="0" width=20 height=16></td>
+<td>Pat&ouml;hia te p&auml;tene kohinga kia puta mai ai ng&auml; niupepa o taua kohinga</td></tr>
+</table>
+}
+
+_iconexplain_ [v=1] {
+<p>
+Click on <b><u>open book</u></b> to read corresponding book.<br>
+Click on <b><u>open section</u></b> to look at books on that subject.
+}
+
+_iconexplain_ [l=mi,v=1] {
+<p>
+Pat&ouml;hia te <b><u>pukapuka</u></b> kia puta mai ai taua pukapuka<br>
+Pat&ouml;hia te <b><u>kohinga</u></b> kia puta mai ai ng&auml; niupepa o taua kohinga
+}
+
+_content_ {
+<center>_navigationbar_</center>
+_texthelptopics_
+<a name=finding-information>
+_iconblankbar_<br>
+<h2>_textsimplehelpheading_</h2>
+_simplehelp_
+<p>
+<a name=reading-books>
+_iconblankbar_<br>
+_texthelpreadingbooks_
+_textexplainbuttons_
+<p>
+<a name=searching>
+_iconblankbar_<br>
+_texthelpsearching_
+<p>
+_iconblankbar_<br>
+}
+
+################################
+# English text                 #
+################################
+
+_texthelptopics_ {
+<h2 align=left>Topics</h2>
+<ul>
+  <li><a href="\#finding-information">_textsimplehelpheading_</a>
+  <li><a href="\#reading-books">How to read the books</a>
+  <li><a href="\#searching">How to search for particular words</a>
+<ul>
+  <li><a href="\#query-terms">Search terms</a>
+  <li><a href="\#query-type">Query type</a>
+  <li><a href="\#scope-of-queries">Scope of queries</a>
+  <li><a href="\#stemming">Case sensitivity and stemming</a>
+</ul>
+</ul>
+}
+
+_texthelpreadingbooks_ {
+<h2>How to read the books</h2>
+
+<p>You can tell when you have arrived at an individual book because there is a
+photograph of its front cover.  Beside the photograph is a table of contents
+with an arrow marking where you are.  This table is expandable: click on the
+folders to open them or close them.  Click on the open book at the top to close
+it.
+
+<p>Underneath is the text of the current section.  When you have read through
+it, there are arrows at the bottom to take you on to the next section or back
+to the previous one.
+
+<p>Below the photograph are three buttons.  Click on <i>detach</i> to make a
+new browser window for this book.  (This is useful if you want to compare
+books, or read two at once.)  Click on <i>expand contents</i> to expand out the
+whole table of contents so that you can see the titles of all chapters and
+subsections.  Click on <i>expand text</i> to expand out the whole text of the
+current section, or book.  If the book is large, this could take a long time
+and use a lot of memory!
+}
+
+_textnextbutton_ {Click on the arrow to go to the next section ...}
+_textprevbutton_ {... or back to the previous section}
+
+_httpicondtchdemo_ {_httpimg_/edtchof.gif}
+_httpicondtchdemo_ [l=mi] {_httpimg_/medtchof.gif}
+
+_httpiconeexpdemo_ {_httpimg_/eexpcof.gif}
+_httpiconeexpdemo_ [l=mi] {_httpimg_/meexpcof.gif}
+
+_httpiconealltdemo_ {_httpimg_/ealltof.gif}
+_httpiconealltdemo_ [l=mi] {_httpimg_/mealltof.gif}
+
+_textexplainbuttons_ {
+<p>
+<table cellspacing="0" cellpadding="2" border="0">
+<tr><td><img src="_httpimg_/smtext.gif" border="0"></td><td>_document:texticonsmalltext_</td></tr>
+<tr><td><img src="_httpimg_/clsdfldr.gif" border="0"></td><td>_document:texticonclosedfolder_</td></tr>
+<tr><td><img src="_httpimg_/more.gif"></td><td>_textnextbutton_</td></tr>
+<tr><td><img src="_httpimg_/less.gif"></td><td>_textprevbutton_</td></tr>
+<tr><td><img src="_httpicondtchdemo_"></td><td valign="center">_document:texticondetach_</td></tr>
+<tr><td><img src="_httpiconeexpdemo_"></td><td valign="center">_document:texticonexpandtoc_</td></tr>
+<tr><td><img src="_httpiconealltdemo_"></td><td valign="center">_document:texticonexpandtext_</td></tr>
+<tr><td><img src="_document:httpiconehlof_"></td><td valign="center">_document:texticonhighlight_</td></tr>
+</table>
+}
+
+_texthelpsearching_ {
+<h2>How to search for particular words</h2>
+<p>
+  From the search page, you make a query in these simple steps:<p>
+  
+  <ol><li>Specify what documents you want to search
+      <li>Say whether you want to search for all or just some of the words
+      <li>Type in the words you want to search for
+      <li>Click the <i>Begin Search</i> button
+  </ol>
+
+<p>When you make a query, the titles of twenty matching documents will be shown.
+There is a button at the end to take you on to the next twenty documents.  From
+there you will find buttons to take you on to the third twenty or back to the
+first twenty, and so on.  Click the title of any document, or the little button
+beside it, to see it.
+
+<p>A maximum of 100 is imposed on the number of
+  documents returned.  You can change this number by clicking the
+  <i>preferences</i> button at the top of the page.<p>
+
+<p>
+<a name=query-terms>
+_iconblankbar_<br>
+
+<h3>Search terms</h3>
+
+<p>The stuff that you type into the query box is interpreted as a list of words
+called "search terms."  Each term contains nothing but alphabetic characters
+and digits.  Terms are separated by white space.  If any other characters such
+as punctuation appear, they serve to separate terms just as though they were
+spaces.  And then they are ignored.  You can't search for words that include
+punctuation.
+
+<p>For example, the query<p>
+    <ul><kbd>Agro-forestry in the Pacific Islands: Systems for Sustainability (1993)</kbd></ul>
+      <p>will be treated the same as<p>
+      <ul><kbd>Agro forestry in the Pacific Islands  Systems for Sustainability  1993 </kbd></ul><p>
+<p>
+<a name=query-type>
+_iconblankbar_<br>
+
+<h3>Query type</h3>
+
+<p><ul>There are two different kinds of query.<p>
+
+  <li>Queries for <b>all</b> of the words.  These look for chapters (or
+      titles) that 
+      contain all the words you have specified.  Documents that satisfy the
+      query are displayed, in alphabetical order.<p>
+
+  <li>Queries for <b>some</b> of the words.  Just list some terms that are
+      likely to appear in the documents you are looking for.  Documents are
+      displayed in order of how closely they match the query.  When determining
+
+      the degree of match,
+
+      <p><ul>
+        <li> the more search terms a document contains, the closer it matches;
+        <li> rare terms are more important than common ones;
+        <li> short documents match better than long ones.
+      </ul>
+
+      <p>Use as many search terms as you like--a whole sentence, or even a
+      whole paragraph.  If you specify only
+      one term, documents will be ordered by its frequency of occurrence.<p>
+      
+</ul>
+
+<p>
+<a name=other-options><a name=scope-of-queries>
+_iconblankbar_<br>
+_texthelpscope_
+<h3 align=left>Case sensitivity and stemming</h3>
+
+<p>When you click the <i>preferences</i> button at the top of the page you will
+see two pairs of buttons that control the case sensitivity and stemming of your
+queries. The first set of buttons controls whether upper and lower case must
+match (case sensitivity).  The second set controls whether to ignore word
+endings or not (stemming).
+
+<p>For example, if the buttons <i>ignore case differences</i> and
+<i>ignore word endings</i> are selected, the query<p>
+    <ul><kbd>African building</kbd></ul>
+      <p>will be treated the same as<p>
+      <ul><kbd>africa builds</kbd></ul><p>
+      because the uppercase letter in "African" will be transformed to
+      lowercase, and  the suffixes "n" and "ing" will be removed from
+      "African" and "building" respectively (also, "s" would be removed from
+      "builds").
+}
+
+_textnumbrowseoptions_ {<p>There _If_("_numbrowseoptions_" == "1",is 1 way,are _numbrowseoptions_ ways) to 
+find information in this collection:}
+
+_textnumbrowseoptions_ [l=mi] {<p>E _numbrowseoptions_ ng&auml; huarahi hei rapu k&ouml;rero i te kohikohinga niupepa}
+
+_textsimplehelpheading_ {How to find information}
+
+_texthelpscope_ {
+<h3 align=left>Scope of queries</h3>
+
+<p>
+You can choose the "granularity" at which your search is done, that
+is, the size of the text against which the query is matched.
+You can choose <i>title</i> or
+<i>chapter</i>.
+
+<p>Regardless of how you search, the results are always displayed in terms of a
+complete book, opened at the appropriate place.
+
+<p>
+<a name=stemming>
+_iconblankbar_<br>
+}
+
+
+################################
+# Maori text                   #
+################################
+
+
+_texthelptopics_ [l=mi] {
+<h2 align=left>Kaupapa</h2>
+<ul>
+  <li><a href="\#finding-information">Ng&auml; huarahi hei rapu k&ouml;rero</a>
+  <li><a href="\#reading-books">Te p&auml;nui niupepa/kupu whakataki</a>
+  <li><a href="\#searching">Te rapu kupu</a>
+<ul>
+  <li><a href="\#query-terms">Ng&auml; kupu hei rapu</a>
+  <li><a href="\#query-type">Ng&auml; t&uuml;momo rapu</a>
+  <li><a href="\#scope-of-queries">Te wh&auml;nui o te rapu</a>
+  <li><a href="\#stemming">Ng&auml; p&uuml; matua, p&uuml; iti me ng&auml; k&uuml;muri hoki</a>
+</ul>
+</ul>
+}
+
+_texthelpsearching_ [l=mi] {
+<h2>Te rapu kupu</h2>
+<p>
+  Ko te wh&auml;rangi rapu te wh&auml;rangi hei rapu kupu. Ka puta mai taua wh&auml;rangi ka
+  whai mai &euml;nei k&ouml;kiri kia rapu ai i (t)&euml;tehi kupu:<p>	
+
+  <ol><li>K&ouml;whiritia ko ng&auml; niupepa, ko ng&auml; kupu whakataki r&auml;nei, 
+	ko ng&auml; r&auml;popoto reo P&auml;keh&auml;, ko aua mea e toru r&auml;nei
+      <li>K&ouml;whiritia ko &euml;tehi o ng&auml; kupu rapu, ko te katoa r&auml;nei o ng&auml;
+          kupu rapu
+      <li>Tuhia ng&auml; kupu rapu
+      <li>Pat&ouml;hia te p&auml;tene <i>Rapua</i>
+  </ol>
+<p>Ka tukua te rapu ka hoki mai ng&auml; pukapuka rua tekau e p&auml; ana ki &auml;u kupu rapu.
+He p&auml;tene kei raro r&auml; o te wh&auml;rangi hei titiro ai ki ng&auml; pukapuka 21 ki te
+40 e p&auml; ana ki &auml;u kupu rapu. M&auml; &euml;nei p&auml;tene kei raro r&auml; ka taea
+te tirohia te katoa o ng&auml; wh&auml;rangi e p&auml; ana ki t&auml;u i rapu ai.
+
+<p>Ka kore e nuku atu i te 100 o ng&auml; pukapuka ka kitea i t&euml;tehi rapunga. Heoi an&auml;
+ka taea te whakarerek&euml; t&euml;nei nama i te wh&auml;rangi whiriwhiringa.
+
+<p>
+<a name=query-terms>
+_iconblankbar_<br>
+
+<h3>Ng&auml; kupu hei rapu</h3>
+
+<p>Koia nei ng&auml; kupu ka tuhia ki roto i te pouaka rapu. Me tuhi ko ng&auml; reta me ng&auml;
+nama anak&euml;. Kaua e tuhia ng&auml; tohu t&ouml;, he uaua te kite i ng&auml; tohu t&ouml; i roto 
+i ng&auml; niupepa. Me noho te mokow&auml; (w&auml;hi w&auml;tea) i waenganui i ng&auml; kupu.
+Kaua e tuhi ng&auml; tuhutuhi (p&euml;r&auml; i te "?" me te "/"), ka tangohia aua tohu e te
+rorohiko nei, ka whakauru ko te mokow&auml;.
+
+<p>Hei tauira, ki te tuku atu i &euml;nei kupu rapu;
+    <ul><kbd>huihuinga-a-motu / Whanganui (1860-1885)</kbd></ul>
+       <p>ka rapua k&euml;tia e te rorohiko &euml;nei kupu;
+    <ul><kbd>huihuinga a motu Whanganui 1860 1885</kbd></ul><p>
+
+<p>
+<a name=query-type>
+_iconblankbar_<br>
+
+<h3>Ng&auml; T&uuml;momo Rapu</h3>
+
+<p><ul>E rua ng&auml; t&uuml;momo rapu<p>
+
+  <li>Ka rapua te <b>katoa</b> o &auml;u kupu rapu. Ka rapua ng&auml; pukapuka e mau ana i te katoa
+      o ng&auml; kupu rapu kua tuhia e koe. Ka whakaatu mai aua pukapuka e p&auml; ana ki te 
+      whakar&auml;rangi-&auml;-p&uuml;.
+
+  <li>Ka rapua <b>&euml;tehi</b> o &auml;u kupu rapu. N&ouml; reira, me tuhi noa he kupu e p&auml;
+      ana ki te kaupapa e hiahia ana koe. Ka whakar&auml;rangitia ng&auml; pukapuka e p&auml; ana
+      ki te ritenga ki &auml;u kupu rapu. Ar&auml;, ka puta tuatahitia mai ng&auml; pukapuka;	 
+
+      <p><ul>
+        <li> e mau ana i te maha o &auml;u kupu rapu;
+        <li> e mau ana i ng&auml; kupu uaua te kite o &auml;u kupu rapu;
+        <li> e iti ana, ka whai mai ko ng&auml; pukapuka nui.
+      </ul>
+
+      <p>Me tuhi i te maha o ng&auml; kupu rapu kia arahina te rapu ki t&ouml; ake hiahia.
+</ul>
+
+<p>
+<a name=other-options><a name=scope-of-queries>
+_iconblankbar_<br>
+
+_texthelpscope_
+
+<h3 align=left>Ng&auml; p&uuml; matua, p&uuml; iti me ng&auml; k&uuml;muri hoki</h3>
+
+<p>Pat&ouml;hia te p&auml;tene whiriwhiringa kia puta mai ai te wh&auml;rangi e p&auml;
+ana ki ng&auml; p&uuml; matua, p&uuml; iti me ng&auml; k&uuml;muri hoki. Ka h&auml;ngai
+te whiriwhiringa tuatahi ki ng&auml; p&uuml; matua me ng&auml; p&uuml; iti. &Auml;e
+r&auml;nei, me tika te p&uuml; matua me te p&uuml; iti, k&auml;ore r&auml;nei, hei aha
+te &auml;ta titiro ki ng&auml; p&uuml; matua me ng&auml; p&uuml; iti. Ka h&auml;ngai te
+whiriwhiringa tuarua ki ng&auml; k&uuml;muri o te kupu. &Auml;e r&auml;nei, me tino
+&ouml;rite te k&uuml;muri, k&auml;ore r&auml;nei, hei aha te &auml;ta titiro ki ng&auml; 
+k&uuml;muri o ng&auml; kupu. M&ouml; ng&auml; kupu P&auml;keh&auml; anak&euml; te 
+whiriwhiringa k&uuml;muri.
+
+<p>Hei tauira, mehemea ka k&ouml;whiritia te <i>hei aha te p&uuml; matua me te p&uuml;
+iti</i>, &auml;, ka k&ouml;whiritia hoki te <i>hei aha te k&uuml;muri o te kupu</i>
+k&auml;tahi ka tukua &euml;nei kupu hei rapu;
+<p>
+  <ul><kbd>Aotearoa building</kbd></ul>
+    <p>ka &ouml;rite te whakahoki mai o ng&auml; pukapuka ki &euml;nei kupu rapu
+  <ul><kbd>aotearoa builds</kbd></ul>
+<p>ka p&euml;nei i te mea, k&auml;ore te rapu i te &auml;ta titiro mehemea he p&uuml;
+matua, he p&uuml; iti r&auml;nei, &auml, ka tangohia ng&auml; k&uuml;muri (p&euml;r&auml;
+i te "ing", me te "s") i t&euml;nei whiriwhiringa.
+
+}
+
+_textsimplehelpheading_ [l=mi] {Ng&auml; huarahi hei rapu k&ouml;rero}
+
+_textnextbutton_ [l=mi] {Haere whakamua}
+_textprevbutton_ [l=mi] {Haere whakamuri}
+
+
+
+################################
+# Chinese text                 #
+################################
+
+_texthelptopics_ [l=zh] {
+<h2 align=left>ç®åœ</h2>
+<ul>
+  <li><a href="\#finding-information">_textsimplehelpheading_</a>
+  <li><a href="\#reading-books">ãåŠäœé
+è¯»ä¹Šç®</a>
+  <li><a href="\#searching">ãåŠäœæç¹å®å­æ¥è¯¢</a>
+<ul>
+  <li><a href="\#query-terms">ããæ£çŽ¢é¡¹</a>
+  <li><a href="\#query-type">ããæ¥è¯¢ç±»å</a>
+  <li><a href="\#scope-of-queries">ããæ¥è¯¢èåŽ</a>
+</ul>
+</ul>
+}
+
+_textsimplehelpheading_ [l=zh] {åŠäœæ¥æŸä¿¡æ¯}
+
+
+_texthelpreadingbooks_ [l=zh] {
+<h2>åŠäœé
+è¯»ä¹Šç®</h2>
+
+<p>ããåœè¿å
+¥æäžä¹Šç®æ¶ïŒéè¿å°é¢ïŒäœ å¯åŸç¥å±å¹æ­£åšæŸç€ºåªäžäžªä¹Šç®ïŒå°é¢æèŸ¹æ¯ç®åœè¡šïŒç®­å€Žæææ­€æ¶æå€äœçœ®ãè¯¥ç®åœè¡šæ¯å¯æ©å±çïŒåå»æ­€ç®åœè¡šïŒå¯å®ç°æåŒæå
+³é­ïŒåå»é¡¶éšæåŒçä¹Šç®ä»¥å
+³é­å®ã
+
+<p>ç®åœè¡šäžé¢æ¯åœåéšä»œçææ¬ïŒåšæ­€ä¹åæäž€äžªç®­å€Žåå«æåäžäžéšä»œæåäžéšä»œææ¬ã
+
+<p>åšå°é¢äžæ¹æäžäžªæé®ïŒåå»âåçŠ»âïŒåæåŒåŠäžæµè§çªæŸç€ºè¯¥å±å¹ãïŒè¿äžªåèœåšæ³èŠæ¯èŸäžåçä¹Šæåæ¶é
+è¯»äž€æ¬ä¹Šæ¶æ¯æåž®å©çïŒïŒåå»âæ©å±å
+å®¹âä»¥åŸå°æŽäžªä¹Šç®è¡šïŒä»èçå°ææç« èçç®åœïŒåå»âæ©å±ææ¬âå¯è·åŸåœåç« èæä¹Šç®çå
+šéšææ¬ïŒåŠææ¯äžæ¬åŸé¿çä¹ŠïŒå¯èœäŒéèŠèŸé¿çæ¶éŽä»¥åå€§éçå
+å­ã
+}
+
+_textnextbutton_ [l=zh] {ãåå»ç®­å€ŽïŒè¿å
+¥äžäžéšä»œ}
+_textprevbutton_ [l=zh] {ãæè
+è¿ååé¡µ}
+
+_texthelpsearching_ [l=zh] {
+<h2>åŠäœæç¹å®å­æ¥è¯¢</h2>
+<p>
+ä»æ¥è¯¢é¡µïŒæä»¥äžç®åçæ­¥éª€èŸå
+¥æ¥è¯¢é¡¹ïŒ<p>
+  
+  <ol><li>æææ³èŠæ¥è¯¢çæä»¶
+      <li>éæ©æ¯æ¥è¯¢ææå­è¿æ¯éšä»œå­
+      <li>èŸå
+¥èŠæ¥è¯¢çå­
+      <li>åå»âåŒå§æ¥è¯¢â
+  </ol>
+
+<p>è¿è¡æ¥è¯¢åïŒç¬¬äžç»ïŒïŒäžªå¹é
+äžçè®°åœçé¢åäŒæŸç€ºåºæ¥ïŒåšæ­€ïŒïŒäžªé¢åä¹åæäžæé®å¯ä»¥æåäžäžç»ïŒïŒäžªè®°åœçé¢åïŒä»è¯¥å±å¹äœ å¯çå°äž€äžªæé®ïŒäžäžªæé®è¿äžæ­¥æåç¬¬äžç»ïŒïŒäžªè®°åœçé¢åïŒåŠå€äžäžªæé®çšæ¥è¿åäžäžç»é¢åãåå»ä»»äžä¹Šç®çé¢åæå
+¶æèŸ¹çå°æé®ïŒå°±å¯è¿è¡é
+è¯»è¿æ¡è®°åœçè¯Šç»å
+å®¹ã
+
+<p>æå€è¿åïŒïŒïŒæ¡è®°åœïŒåŠèŠæ¹åè¯¥æ°ç®ïŒååå»é¡µé¢å³äžæ¹çâéé¡¹â
+
+<p>
+<a name=query-terms>
+_iconblankbar_<br>
+
+<h3>æ£çŽ¢é¡¹</h3>
+
+<p>èŸå
+¥å°æ¥è¯¢æ¡äž­çå
+å®¹ïŒå°è¢«èœ¬åæäžäžªå«åæ¥è¯¢é¡¹çåå­éïŒæ¥è¯¢é¡¹å¯ä»¥æ¯æ±å­ãé¿æäŒ¯å­æ¯ææ°å­ïŒæ¥è¯¢é¡¹ä¹éŽå¯ä»¥æç©ºæ ŒïŒä¹å¯ä»¥æ²¡æãåŠæéå°å
+¶ä»çç¬Šå·ïŒäŸåŠæ ç¹ç¬Šå·ïŒåäŒè¢«å¿œç¥ïŒå¹¶å°å®ä»¬çäœç©ºæ Œå€çïŒå æ­€äžèœæ¥è¯¢å
+å«æ ç¹ç¬Šå·çè¯ã
+
+<p>äŸåŠïŒæ¥è¯¢<p>
+    <ul><kbd>ããå
+å®¹å
+æ¬ïŒèœ¯ä»¶å·¥çšïŒ1999ïŒ</kbd></ul>
+      <p>äžäžé¢çæ¥è¯¢é¡¹å€çæ¯çžåçã<p>
+      <ul><kbd>ããå
+å®¹å
+æ¬èœ¯ä»¶å·¥çš1999</kbd></ul><p>
+
+<a name=query-type>
+_iconblankbar_<br>
+
+<h3>æ¥è¯¢ç±»å</h3>
+
+<p><ul>å¯ä»¥æäž€ç§äžåçæ¹åŒæ¥è¯¢ã<p>
+
+  <li>æ¥è¯¢ææçå­ãåšè¿ç§æ
+åµäžç³»ç»å°æ¥æŸå
+æ¬ææåšæ¥è¯¢é¡¹æå®çå­çè®°åœïŒæé¢åïŒãç¬Šåæ¡ä»¶çè®°åœå°è¢«æŸç€ºåšå±å¹äžã
+
+  <li>æ¥è¯¢éšä»œå­ãç³»ç»ä»
+æ¥æŸå
+å«æ¥è¯¢é¡¹äž­ä»»æäžäžªæå äžªå­çè®°åœïŒæä»¶æå
+¶äžæ¥è¯¢å
+å®¹æå¹é
+ççšåºŠæ¥æŸç€ºïŒåšå³å®å¹é
+çšåºŠæ¶ïŒ
+
+      <p><ul>
+        <li> ãå
+å«è¶å€çæ¥è¯¢é¡¹çè®°åœïŒå
+¶å¹é
+çšåºŠè¶é«ïŒ
+        <li> ãå°è§çæ¥è¯¢é¡¹æ¯åžžè§çæ¥è¯¢é¡¹æŽéèŠïŒ
+        <li> ãç­è®°åœæ¯é¿è®°åœçå¹é
+çšåºŠé«ã
+      </ul>
+
+      <p>åšå¯èœçæ
+åµäžïŒäœ¿çšå°œå¯èœå€çæ¥è¯¢é¡¹äžºå¥œïŒäžäžªå¥å­ïŒæè
+äžäžªæ®µèœãåŠæåªç»äžäžªæ¥è¯¢é¡¹ïŒæ¥è¯¢ç»æå°æç
+§å
+¶åšè®°åœäž­åºç°çé¢çè¿è¡æåºã
+      
+</ul>
+
+<p>
+<a name=other-options><a name=scope-of-queries>
+_iconblankbar_<br>
+_texthelpscope_
+}
+
+_texthelpscope_ [l=zh] {
+<h3 align=left>æ¥è¯¢èåŽ</h3>
+
+<p>
+å¯ä»¥éæ©åšä»ä¹èåŽå
+è¿è¡æ¥è¯¢ïŒæ¢æ¥è¯¢é¡¹åšä»ä¹èåŽå
+è¿è¡å¹é
+ãå¯ä»¥éæ©é¢åæç« èã
+
+<p>æ è®ºåšä»ä¹èåŽå
+è¿è¡æ¥è¯¢ïŒæ¥è¯¢ç»æéœå°æ¯æŽæ¡è®°åœã
+}
+
+
+
+
+
Index: /tags/gsdl-21/gsdl/macros/home.dm
===================================================================
--- /tags/gsdl-21/gsdl/macros/home.dm	(revision 699)
+++ /tags/gsdl-21/gsdl/macros/home.dm	(revision 699)
@@ -0,0 +1,213 @@
+package home
+
+_httpmusiclibrary_ {http://www.nzdl.org/cgi-bin/gw?c=meldex&a=page&p=coltitle}
+_httpchineselibrary_ {_gwcgi_?a=p&p=about&c=chinese&l=zh&nw=u}
+
+#######################################################################
+# java images/scripts
+#######################################################################
+
+# the _javalinks_ macros are the flashy image links at the top right of
+# the page. this is overridden here as we don't want 'home' 
+# links on this page
+
+_javalinks_ {}
+_javalinks_ [v=1] {}
+
+
+
+#######################################################################
+# icons
+#######################################################################
+
+_iconnzdl_ {
+<p><img src="_httpimg_/new.gif">Welcome to our new look (still experimental). The old system is
+still up at <a href="http://www.nzdl.org/old">http://www.nzdl.org/old</a>
+<p><br>
+<img src="_httpiconnzdl_" width=_widthnzdl_ height=_heightnzdl_>}
+_icongbull_ {<img src="_httpicongbull_">}
+_iconpdf_ {<img src="_httpiconpdf_">}
+_iconselectcollection_ {<img src="_httpiconselcolgr_" width=_widthselcolgr_ height=_heightselcolgr_>}
+_iconmusiclibrary_ {<img src="_httpicontmusic_" border=1>}
+
+#######################################################################
+# http macros 
+#
+# These contain the url without any quotes
+#######################################################################
+
+#_httpiconnzdl_ {_httpimg_/nzdl.gif}
+#_widthnzdl_ {400}
+#_heightnzdl_ {112}
+_httpiconnzdl_ {_httpimg_/nzdl2gr.gif}
+_widthnzdl_ {457}
+_heightnzdl_ {181}
+
+_httpicongbull_ {_httpimg_/gbull.gif}
+
+_httpiconpdf_ {_httpimg_/pdf.gif}
+
+_httpiconselcolgr_ {_httpimg_/selcolgr.gif}
+_widthselcolgr_ {537}
+_heightselcolgr_ {17}
+
+_httpicontmusic_ {_httpimg_/meldexsm.gif}
+
+#######################################################################
+# page content                     
+#######################################################################
+
+_pagetitle_ {Greenstone Digital Library}
+_imagethispage_ {}
+_imagecollection_ {<center>_iconnzdl_</center>}
+
+_content_ {
+<center>_iconselectcollection_</center><br>
+_homeextra_
+<center>_iconblankbar_</center>
+
+<p>
+<table>
+<tr><td colspan=2><center><h2>_textprojhead_</h2></center></td></tr>
+<tr valign=top><td>
+
+<table border=0 cellpadding=5>
+<tr valign=top><td rowspan=10 width=32> </td>
+<tr valign=top><td>_icongbull_</td><td><a href="mailto:_mailaddr_">_textfb_</a></td></tr>
+<tr valign=top><td>_icongbull_</td><td><a href="_httppagex_(people)">_textpeople_</a><br></td></tr>
+<tr valign=top><td>_icongbull_</td><td><a href="_httpprefix_/html/pub.htm">_textpub_</a><br></td></tr>
+<tr valign=top><td>_icongbull_</td><td><a href="_httppagex_(rw)">_textrw_</a><br></td></tr>
+<tr valign=top><td>_icongbull_</td><td><a href="_httppagex_(technology)">_texttec_</a><br></td></tr>
+<tr valign=top><td>_iconpdf_</td><td><a href="http://www.nzdl.org/publications/NZDLtext.pdf">_textinfosheet_</a></td></tr>
+<tr valign=top><td>_iconpdf_</td><td><a href="http://www.nzdl.org/publications/NZDLpictures.pdf">_textscreenshots_</a></td></tr>
+<tr valign=top><td>_iconpdf_</td><td><a href="http://www.nzdl.org/publications/Niupepa.pdf">_textnpepainfosheet_</a></td></tr>
+<tr valign=top><td colspan=2><a href="_httppagestatus_"><img src="_httpimg_/tabspace.gif" width=60 height=20 border=0></a></td></tr>
+
+</table>
+
+</td>
+<td>
+
+<table border=0 cellpadding=5>
+<tr><td>
+_textprojinfo_
+</td></tr></table>
+</td></tr></table>
+_iconblankbar_
+<table><tr valign=top>
+<td>_textpoem_</td>
+<td>_imagegreenstone_</td>
+</tr>
+<tr><td colspan=2>_textgreenstone_</td>
+</tr>
+</table>
+}
+
+
+#######################################################################
+# English language text macros
+#######################################################################
+
+_documents_ { documents. }
+_lastupdate_ {Last updated}
+_ago_ {days ago.}
+
+_textpeople_ {People}
+_textfb_ {Feedback}
+_textpub_ {Publications}
+_texttec_ {Technology}
+_textrw_ {Related Work}
+_textinfosheet_ {Info Sheet}
+_textscreenshots_ {Screen Shots}
+_textnpepainfosheet_ {Niupepa Info Sheet}
+
+_textprojhead_ {The New Zealand Digital Library Project}
+
+_textprojinfo_ {
+<h4>The NZDL system</h4>
+
+<p> The New Zealand Digital Library system comprises several demonstration
+collections -- computer science technical reports and bibliographies, literary
+works, humanitarian and development information, magazines -- and makes them
+available over the Web through full-text interfaces.  Behind the query
+interface lies a huge collection providing gigabytes of information.  We hope
+you find what you want, or at least something intriguing!
+
+<h4>The Greenstone software</h4>
+
+<p> The Greenstone Digital Library software provides a new way of organizing
+information and making it available over the Internet.  A <i>collection</i> of
+information comprises several (typically several thousand, or even several
+million) <i>documents</i>, which share a uniform searching and browsing
+interface.  The collections in a library are organized in a different
+way--though they share a strong family resemblance.  Although primarily
+designed for access over the Web, Greenstone collections can be made available,
+in precisely the same form, on CD-ROM for standalone PCs.  Greenstone is
+open-source software, available under the terms of the Gnu public linense.
+ 
+<h4>The research</h4>
+
+<p> The goal of our research program is to explore the potential of
+internet-based digital libraries.  Our vision is to develop systems that
+automatically impose structure on anarchic, uncatalogued, distributed
+repositories of information, thereby providing information consumers with
+effective tools to locate what they need and to peruse it conveniently and
+comfortably.  Our research objectives are to
+
+<ul>
+  <li> develop technology for creating and automatically
+      maintaining collections;
+  <li> monitor usage to study library users' needs;
+  <li> look at novel interfaces that cater to a wide spectrum of users;
+  <li> find ways to abstract layout and bibliographic information
+      from document files;
+  <li> use this information to enhance presentation and for
+      bibliometric research;
+  <li> assess potential subject areas for public-domain collections;
+  <li> survey and critique other digital library projects.
+</ul>
+
+<h4>DigiLib Systems Limited</h4>
+
+<p><a href="http://www.digilibs.com/">DigiLib Systems Limited</a> is an
+innovative software company that creates international digital libraries.
+As a major contributor to the Greenstone Digital Library Software they are
+able to build, customize, and extend digital libraries to meet exacting needs.
+Please <a href="mailto:contact@digilibs.com">contact</a> them for an obligation
+free quote.
+
+}
+
+_textpoem_ {
+<br><h2 align=left>Kia papapounamu te moana</h2>
+
+        <p>kia hora te marino,
+        <br>kia tere te karohirohi,
+        <br>kia papapounamu te moana
+
+        <p>may peace and calmness surround you,
+        <br>may you reside in the warmth of a summer's haze,
+        <br>may the ocean of your travels be as smooth as the polished greenstone.
+}
+
+_textgreenstone_ {
+<p><br>
+Greenstone is a semi-precious stone that (like this software) is sourced in
+New Zealand.  In traditional Maori society it was the most highly prized and
+sought after of all substances.  It can absorb and hold <i>wairua</i>, which is
+a spirit or life force, and is endowed with traditional virtues that make it an
+appropriate emblem for a public-domain digital library project.  Its lustre
+shows charity; its translucence, honesty; its toughness, courage; and the sharp
+edge it can take, justice.  The carved piece used in the Greenstone Digital
+Library Software logo is a <i>patu</i> or fighting club, and is a family
+heirloom of one of our project members.  In hand-to-hand combat its delivery is
+very quick, very accurate, and very complete.  We like to think these qualities
+also apply to our software, the razor sharp edge of the <i>patu</i> symbolizing
+the leading edge of technology.
+
+<p><a href="mailto:greenstone@cs.waikato.ac.nz">Greenstone Digital Library Software</a>
+<br><a href="http://www.cs.waikato.ac.nz/cs">Computer Science Department</a>, 
+<a href="http://www.waikato.ac.nz">University of Waikato</a>, New Zealand
+
+<br>October 1999
+}
Index: /tags/gsdl-21/gsdl/macros/html.dm
===================================================================
--- /tags/gsdl-21/gsdl/macros/html.dm	(revision 699)
+++ /tags/gsdl-21/gsdl/macros/html.dm	(revision 699)
@@ -0,0 +1,44 @@
+package html
+
+
+_header_ {<html>
+<head></head>
+}
+
+_footer_ {</html>}
+
+_pagetitle_ {_collectionname_}
+
+_content_ {
+<frameset rows="50,*" noresize border=0>
+  <frame frameborder=0 src="_gwcgi_?e=_compressedoptions_&a=p&p=nav">
+  <frame frameborder=0 src="_cgiarghp_">
+  <noframes>
+  <p>You must have a frame enabled browser to view this.</p>
+  </noframes>
+</frameset>
+}
+
+
+package nav
+
+_header_ {<html>
+<head>
+<base target="_top">
+_globalscripts_
+</head>
+<body bgcolor="\#ffffff" text="\#000000" link="\#006666"
+      alink="\#cc9900" vlink="\#666633">
+}
+
+_footer_ {</body>
+</html>
+}
+
+_content_ {
+<center>
+<br>
+_navigationbar_
+</center>
+}
+
Index: /tags/gsdl-21/gsdl/macros/pref.dm
===================================================================
--- /tags/gsdl-21/gsdl/macros/pref.dm	(revision 699)
+++ /tags/gsdl-21/gsdl/macros/pref.dm	(revision 699)
@@ -0,0 +1,292 @@
+package preferences
+
+
+#######################################################################
+# java images/scripts
+#######################################################################
+
+# the _javalinks_ macros are the flashy image links at the top right of
+# the page. this is overridden here as we don't want a 'preferences' 
+# link on this page
+
+_javalinks_ {_imagehome__imagehelp_}
+_javalinks_ [v=1] {
+_imagehome_<br>
+_imagehelp_<br>
+}
+
+
+#######################################################################
+# scripts
+# note that changing either the "m" or the "o" argument
+# from the preferences page necessitates reseting the "r"
+# argument to 1
+#######################################################################
+
+_pagescriptextra_ {
+// preferences scripts generated from \_preferences:pagescriptextra\_
+
+	savedhrefs = new Object();
+
+	argk = "";
+	args = "";		
+	argm = "";
+	argo = "";
+	argb = "";
+
+	function initialize () \{
+		for (i = 0; i < document.links.length; i++)
+			savedhrefs[i] = document.links[i].href;
+	\}
+
+	function updatehrefs () \{
+		for (i = 0; i < document.links.length; i++)
+			document.links[i].href = savedhrefs[i] + argk + args + argm + argo + argb;
+	\}
+
+	function updatek (value) \{
+		if (value != _cgiargk_) argk = "&k=" + value;
+		else argk = "";
+		updatehrefs();
+	\}
+
+	function updates (value) \{
+		if (value != _cgiargs_) args = "&s=" + value;
+		else args = "";
+		updatehrefs();
+	\}
+
+	function updatem () \{
+		value = document.PrefForm.m.options[document.PrefForm.m.selectedIndex].value;
+		if (value != _cgiargm_) argm = "&r=1&m=" + value;
+		else argm = "";
+		updatehrefs();
+	\}
+
+	function updateo () \{
+		value = document.PrefForm.o.options[document.PrefForm.o.selectedIndex].value;
+		if (value != _cgiargo_) argo = "&r=1&o=" + value;
+		else argo = "";
+		updatehrefs();
+	\}
+
+	function updateb (value) \{
+		if (value != _cgiargb_) argb = "&b=" + value;
+		else argb = "";
+		updatehrefs();
+	\}
+
+	function updatel () \{
+		value = document.PrefForm.l.options[document.PrefForm.l.selectedIndex].value;
+		location.href = location.href + argk + args + argm + argo + argb + "&l=" + value;
+	\}
+
+	function updatev () \{
+		value = document.PrefForm.v.options[document.PrefForm.v.selectedIndex].value;
+		location.href = location.href + argk + args + argm + argo + argb + "&v=" + value;
+	\}
+
+	function updatew () \{
+		value = document.PrefForm.w.options[document.PrefForm.w.selectedIndex].value;
+		location.href = location.href + argk + args + argm + argo + argb + "&nw=" + value;
+	\}
+
+}
+
+#######################################################################
+# headers
+# these are overridden so we can put an onLoad event handler
+# in the <body> tag of this page
+#######################################################################
+
+
+_header_ {_cgihead_
+_htmlhead_(background="_httpiconchalk_" onLoad="initialize();")_startspacer__pagebanner_
+}
+
+# this declaration ends up being the same as style=restrict, never mind
+_header_[v=1] {_cgihead_
+_htmlhead_(onLoad="initialize();")_pagebanner_
+}
+
+
+#######################################################################
+# page content
+#######################################################################
+
+_pagetitle_ {_collectionname_: _textpreferences_}
+_imagethispage_ {_iconthispage_}
+
+_caseoption_ {
+<input type=radio name=k value=1 onClick="updatek(1);"
+		_If_(_cgiargk_, checked)>_textignorecase_<br>
+<input type=radio name=k value=0 onClick="updatek(0);"
+		_If_(_cgiargk_,, checked)>_textmatchcase_
+}
+
+_caseoption_ [l=zh] {}
+
+_stemoption_ {
+<input type=radio name=s value=1 onClick="updates(1);"
+		_If_(_cgiargs_, checked)>_textstem_<br>
+<input type=radio name=s value=0 onClick="updates(0);"
+		_If_(_cgiargs_,, checked)>_textnostem_
+}
+
+_stemoption_ [l=zh] {}
+
+_maxdocoption_ {
+<select name="m" onChange="updatem();">
+  <option value="50"_If_("_cgiargm_" eq "50", selected)>50
+  <option value="100"_If_("_cgiargm_" eq "100", selected)>100
+  <option value="200"_If_("_cgiargm_" eq "200", selected)>200
+</select>
+}
+
+_hitsperpageoption_ {
+<select name="o" onChange="updateo();">
+  <option value="10"_If_("_cgiargo_" eq "10", selected)>10  
+  <option value="20"_If_("_cgiargo_" eq "20", selected)>20
+  <option value="50"_If_("_cgiargo_" eq "50", selected)>50
+  <option value="100")_If_("_cgiargo_" eq "100", selected)>all
+</select>
+}
+
+_modeoption_ {
+<input type=radio name=b value=0 onClick="updateb(0);"
+		_If_(_cgiargb_,, checked)>_textsimplemode_<br>
+<input type=radio name=b value=1 onClick="updateb(1);"
+		_If_(_cgiargb_, checked)>_textadvancedmode_
+}
+
+_languageoption_ {
+_textlanguage_
+<select name="l" onChange="updatel();">
+  <option value="en"_If_("_cgiargl_",, selected)_If_("_cgiargl_" eq "en", selected)>English
+  <option value="mi"_If_("_cgiargl_" eq "mi", selected)>M&auml;ori
+  <option value="zh"_If_("_cgiargl_" eq "zh", selected)>Chinese
+</select>
+}
+
+_encodingoption_ {
+_textencoding_
+<select name="w" onChange="updatew();">
+  <option value="w" _If_("_cgiargw_" eq "w",selected)>Western (ISO-8859-1)
+  <option value="u" _If_("_cgiargw_" eq "u",selected)>UTF-8
+  <option value="g" _If_("_cgiargw_" eq "g",selected)>GBK
+</select>
+}
+
+_formatoption_ {
+_textformat_
+<select name="v" onChange="updatev();">
+  <option value="0"_If_("_cgiargv_" eq "0", selected)>_textgraphical_
+  <option value="1"_If_("_cgiargv_" eq "1", selected)>_texttextual_
+</select>
+}
+
+_content_ {
+<center>
+_navigationbar_
+</center>
+<blockquote>
+
+<form name=PrefForm method=get action="_gwcgi_">
+
+<input type="hidden" name="e" value="_compressedoptions_">
+
+<p>_caseoption_
+
+<p>_stemoption_
+
+<p>
+_textprefop_
+
+<p>
+_modeoption_
+
+<p>
+_languageoption_
+
+<p>
+_encodingoption_
+
+<p>
+_formatoption_
+
+</blockquote>
+</form>
+}
+
+
+#######################################################################
+# https
+#######################################################################
+
+_httpiconhpref_ {_httpimg_/h\_pref.gif}
+_httpiconhpref_ [l=mi] {_httpimg_/mh\_pref.gif}
+_widthhpref_ {200}
+_widthhpref_ [l=mi] {220}
+_heighthpref_ {57}
+
+
+#######################################################################
+# icons
+#######################################################################
+
+_iconthispage_ {<img src="_httpiconhpref_" alt="_textpreferences_" width="_widthhpref_" height="_heighthpref_">}
+_iconthispage_ [l=mi] {<img src="_httpimg_/mh\_pref.gif" alt="" width="220" height="57">}
+
+
+#######################################################################
+# english macros
+#######################################################################
+
+_textpreferences_ {Preferences}
+_textignorecase_ { ignore case differences}
+_textmatchcase_ { upper/lower case must match}
+_textstem_ { ignore word endings}
+_textnostem_ { whole word must match}
+_textprefop_ {Return up to _maxdocoption_ hits with _hitsperpageoption_ hits per page.}
+_textlanguage_ {Interface language: }
+_textencoding_ {Encoding: }
+_textformat_ {Interface format: }
+_textsimplemode_ {simple query mode}
+_textadvancedmode_ {advanced query mode (allows boolean searching using !, &, |, and parenthesis)}
+_textgraphical_ {Graphical}
+_texttextual_ {Textual}
+
+
+#######################################################################
+# chinese macros
+#######################################################################
+
+_textprefop_ [l=zh] {æ¥è¯¢æå€è¿åã
+ã _maxdocoption_ è®°åœïŒæ¯é¡µè¿å
+_hitsperpageoption_ è®°åœ}
+_textlanguage_ [l=zh] {çé¢è¯­èšïŒ}
+_textencoding_ [l=zh] {çŒç æ¹åŒïŒ}
+_textformat_ [l=zh] {çé¢æ ŒåŒïŒ}
+
+#######################################################################
+# maori macros
+#######################################################################
+
+_textpreferences_ [l=mi] {Whiriwhiringa}
+_textignorecase_ [l=mi] { hei aha te p&uuml; matua me te p&uuml; iti}
+_textmatchcase_ [l=mi] { me tika te p&uuml; matua me te p&uuml; iti}
+_textstem_ [l=mi] { hei aha te k&uuml;muri}
+_textnostem_ [l=mi] { me tika te k&uuml;muri}
+_textprefop_ [l=mi] {Whakahokia mai te _maxdocoption_ kupu, &auml;, kia _hitsperpageoption_ kupu ki ia wh&auml;rangi.}
+_textlanguage_ [l=mi] {Reo: }
+_textencoding_ [l=mi] {Whakatakotoranga p&uuml;: }
+_textformat_ [l=mi] {Whakatakotoranga wh&auml;rangi: }
+_textsimplemode_ [l=mi] {te rapu wh&auml;iti}
+_textadvancedmode_ [l=mi] {te rapu wh&auml;nui (ka taea te whakamahia ng&auml; p&uuml; boolean, 
+p&euml;r&auml; i te !, &, |, me te "()" hoki)}
+_textgraphical_ [l=mi] {Whakaahua}
+_texttextual_ [l=mi] {K&auml;ore he whakaahua}
+
+
+
+
Index: /tags/gsdl-21/gsdl/macros/query.dm
===================================================================
--- /tags/gsdl-21/gsdl/macros/query.dm	(revision 699)
+++ /tags/gsdl-21/gsdl/macros/query.dm	(revision 699)
@@ -0,0 +1,175 @@
+package query
+
+
+#######################################################################
+# Macros whose values are set from within the server at runtime. These 
+# are here only for reference and to set default values if required.           
+#######################################################################
+
+_quotedquery_ {}
+_freqmsg_ {}
+_resultline_ {}
+_thisfirst_ {}
+_thislast_ {}
+_nextfirst_ {}
+_nextlast_ {}
+_prevfirst_ {}
+_prevlast_ {}
+
+#######################################################################
+# https
+#######################################################################
+
+_httpiconthispage_ {_httpiconhsearch_}
+_widthiconthispage_ {_widthiconhsearch_}
+_heighticonthispage_ {_heighticonhsearch_}
+
+_httpiconhsearch_ {_httpimg_/h\_search.gif}
+_httpiconhsearch_ [l=mi] {_httpimg_/mh\_srch.gif}
+_widthhsearch_ {200}
+_heighthsearch_ {57}
+
+
+#######################################################################
+# icons
+#######################################################################
+
+_iconthispage_ {<img src="_httpiconthispage_" alt="_texticonthispage_" width="_widthiconthispage_" height="_heighticonthispage_">}
+_iconthispage_ [v=1] {<h2>_texticonthispage_</h2>}
+
+_iconqueryresultsbar_ {<img src="_httpiconqryresb_" width="_widthqryresb_" height="_heightqryresb_" alt="_texticonqueryresultsbar_">}
+_iconqueryresultsbar_[v=1] {_texticonqueryresultsbar_}
+
+# _iconnext_ and _iconprev_ are overridden in this package as we
+# don't want alt text here
+_iconnext_{<img src="_httpiconmore_" width=_widthmore_ height=_heightmore_ border=0 align=top>}
+_iconprev_{<img src="_httpiconless_" width=_widthless_ height=_heightless_ border=0 align=top>}
+
+#######################################################################
+# images
+#######################################################################
+
+_imagethispage_ {_iconthispage_}
+
+
+#######################################################################
+# page content
+#######################################################################
+
+_pagetitle_ {_If_(_cgiargq_,_textquerytitle_,_textnoquerytitle_)}
+
+_content_ {
+<center>
+_navigationbar_
+</center>
+_queryform_
+_If_(_cgiargq_,<center>_iconqueryresultsbar_</center><br>
+<small>
+_freqmsg_
+_textpostprocess_</small><br>
+_resultline_
+,<center>_iconblankbar_</center>)<br>
+}
+
+_queryform_ {
+<!-- query form -->
+<form name=QueryForm method=get action="_gwcgi_">
+
+<input type=hidden name="a" value="q">
+<input type=hidden name="e" value="_compressedoptions_">
+<input type=hidden name="r" value="1">
+
+<center>
+<table><tr><td>
+<nobr>
+_textselect_<br>
+</nobr>
+</td></tr><tr><td>
+<nobr>
+<input type="text" name="q" value="_cgiargq_" size="50">
+<input type="submit" value="_textbeginsearch_">
+</nobr>
+</td></tr></table>
+</center>
+</form>
+<!-- end of query form -->
+}
+
+# we want to put the links to previous/next pages of results
+# in the footer
+_pagefooterextra_ {
+<center>
+<table cellspacing=0 cellpadding=0 width=_pagewidth_>
+<tr>
+<td align=left>_If_(_prevfirst_,<a href="_httpquery_&r=_prevfirst_">_iconprev__textmatches__prevfirst_ - _prevlast_</a>)</td>
+<td align=right>_If_(_nextfirst_,<a href="_httpquery_&r=_nextfirst_">_textmatches__nextfirst_ - _nextlast__iconnext_</a>)</td>
+</tr></table>
+</center>
+}
+
+#######################################################################
+# English text 
+#######################################################################
+
+_textquerytitle_ {_If_(_thislast_,results _thisfirst_ - _thislast_ for query: _cgiargq_,No matches for query: _cgiargq_}
+_textnoquerytitle_ {Search page}
+
+_texticonthispage_ {search}
+_textsome_ {_If_(_cgiargb_,ranked,some)}
+_textall_ {_If_(_cgiargb_,boolean,all)}
+_texticonqueryresultsbar_ {}
+_textfreqmsg1_ {Word count: }
+_textpostprocess_ {_If_(_quotedquery_,<br><i>post-processed to find _quotedquery_</i>
+)}
+
+_textmorethan_ {More than }
+_textapprox_ {About }
+_textnodocs_ {No documents matched the query.}
+_text1doc_ {1 document matched the query.}
+_textlotsdocs_ {documents matched the query.}
+_textmatches_ {Matches }
+_textbeginsearch_ {Begin Search}
+
+# the textselect macro may need to be overridden for some collections
+# _hselection_, _jselection_ and _nselection_ are set from within the server
+# some or all of them may remain unset
+_textselect_ {_If_(_cgiargb_,_textadvancedsearch_,_textsimplesearch_)}
+
+_textsimplesearch_ {Search for _If_(_hselection_, _hselection_)_If_(_jselection_,_textjselect_)_If_(_nselection_, in _nselection_ language) 
+which contain _querytypeselection_ of the words}
+
+_textadvancedsearch_ {Search_If_(_hselection_, _hselection_)_If_(_jselection_,_textjselect_)_If_(_nselection_, in _nselection_ language) 
+using _querytypeselection_ query}
+
+_textjselect_ {_If_(_hselection_, of) _jselection_}
+
+_querytypeselection_ {
+<select name="t">
+<option value="1"_If_(_cgiargt_, selected)>_textsome_
+<option value="0"_If_(_cgiargt_,, selected)>_textall_
+</select>
+}
+
+_textstemon_ { (ignoring word endings)}
+
+
+#######################################################################
+# Maori text		
+#######################################################################
+
+_texticonthispage_ [l=mi] {rapu}
+_textsome_ [l=mi] {&euml;tehi}
+_textall_ [l=mi] {_If_(_cgiargb_,te boolean,katoa)}
+_textbeginsearch_ [l=mi] {Rapua}
+_textfreqmsg1_ [l=mi] {Kia hia ng&auml; kupu: }
+_textmorethan_ [l=mi] {Neke atu i te }
+_textnodocs_ [l=mi] {k&auml;ore he pukapuka e p&auml; ana ki &auml;u kupu rapu.}
+_text1doc_ [l=mi] {1 te pukapuka e p&auml; ana ki &auml;u kupu rapu.}
+_textlotsdocs_ [l=mi] {ng&auml; pukapuka e p&auml; ana ki &auml;u kupu rapu.}
+_textmatches_ [l=mi] {Kitenga }
+_textstemon_ [l=mi] { (hei aha te k&uuml;muri)}
+
+_textsimplesearch_ [l=mi] {Tirohia ki _If_(_hselection_, _hselection_)_If_(_jselection_,_textjselect_)_If_(_nselection_, in _nselection_ language) m&ouml; _querytypeselection_ o ng&auml; kupu}
+
+_textadvancedsearch_ [l=mi] {Tirohia ki_If_(_hselection_, _hselection_)_If_(_jselection_,_textjselect_)_If_(_nselection_, in _nselection_ language) 
+m&ouml; _querytypeselection_ o ng&auml; kupu}
Index: /tags/gsdl-21/gsdl/macros/status.dm
===================================================================
--- /tags/gsdl-21/gsdl/macros/status.dm	(revision 699)
+++ /tags/gsdl-21/gsdl/macros/status.dm	(revision 699)
@@ -0,0 +1,56 @@
+package status
+
+_header_ {<html>
+<head>
+<title>_1_</title>
+</head>
+<body bgcolor="\#ffffff" text="\#000000" link="\#006666"
+      alink="\#cc9900" vlink="\#666633">
+}
+
+_footer_ {</body>
+</html>
+}
+
+
+_frameset_ {
+<html>
+<head>
+<title>Configuration and Status Information</title>
+</head>
+
+<frameset cols="170,*">
+  <noframes><body bgcolor="#ffffff">
+  <p>You must have a frame enabled browser to view this.</p>
+  </body>
+  </noframes>
+  <frame src="_gwcgi_?e=_compressedoptions_&a=status&sp=select">
+  <frame src="_gwcgi_?e=_compressedoptions_&a=status&sp=generalinfo" name=infoframe>
+</frameset>
+
+</html>
+}
+
+_select_ {
+<img src="_httpimg_/gsdl.gif">
+
+_If_("_cgiarguma_" ne "\_cgiarguma\_", 
+<p><b>User Management</b><br>
+<a href="_gwcgi_?e=_compressedoptions_&a=um&uma=listusers" target=infoframe>list users</a><br>
+<a href="_gwcgi_?e=_compressedoptions_&a=um&uma=adduser" target=infoframe>add a new user</a><br>
+)
+
+<p><b>Information</b><br>
+<a href="_gwcgi_?e=_compressedoptions_&a=status&sp=generalinfo" target=infoframe>general</a><br>
+<a href="_gwcgi_?e=_compressedoptions_&a=status&sp=argumentinfo" target=infoframe>arguments</a><br>
+<a href="_gwcgi_?e=_compressedoptions_&a=status&sp=actioninfo" target=infoframe>actions</a><br>
+<a href="_gwcgi_?e=_compressedoptions_&a=status&sp=protocolinfo" target=infoframe>protocols</a>
+
+
+<p><b>Logs</b><br>
+<a href="_gwcgi_?e=_compressedoptions_&a=status&sp=initlog" target=infoframe>init log</a><br>
+<a href="_gwcgi_?e=_compressedoptions_&a=status&sp=errorlog" target=infoframe>error log</a>
+
+<p>
+<a href="_httppagehome_" target=_top>Return to NZDL</a>
+}
Index: /tags/gsdl-21/gsdl/macros/style.dm
===================================================================
--- /tags/gsdl-21/gsdl/macros/style.dm	(revision 699)
+++ /tags/gsdl-21/gsdl/macros/style.dm	(revision 699)
@@ -0,0 +1,176 @@
+#######################################################################
+# PAGE STYLES 
+#######################################################################
+
+package Style
+
+# to use this style system output
+# _header_
+# all your page content, then
+# _footer_
+
+# use the page parameter 'style' to choose the appropriate style
+
+# the style system uses
+# _pagetitle_  - what gets displayed at the top of the browser window
+# _pagescriptextra_ - any extra javascript you want included in the header
+# _pagebannerextra_ - anything extra you want displayed in the page banner
+# _pagefooterextra_ - anything extra you want displayed in the footer
+
+# defaults for the above macros
+_pagetitle_ {_collectionname_}
+_pagescriptextra_ {}
+_pagebannerextra_ {}
+_pagefooterextra_ {}
+
+# it also relies on lots of Globals, the most important of these are:
+# _cookie_ - put in the cgi header
+# _globalscripts_ - javascript stuff
+# _httpiconchalk_ - the image down the left of the page
+# _imagecollection_
+# _imagehome_
+# _imagehelp_
+# _imagepref_
+# _imagethispage_
+# _linkotherversion_
+
+_header_ {_cgihead_
+_htmlhead_(background="_httpiconchalk_")_startspacer__pagebanner_
+}
+
+_header_[v=1] {_cgihead_
+_htmlhead__pagebanner_
+}
+
+# _cgihead_ {Content-type: text/html
+# _cookie_
+#
+# }	
+_cgihead_{}
+
+
+# htmlhead uses:
+# _1_ - extra parameters for the body tag
+# _pagetitle_
+# _globalscripts_
+_htmlhead_ {
+<html>
+<head>
+<title>_pagetitle_</title>
+_globalscripts_
+</head>
+
+<body bgcolor="\#ffffff" text="\#000000" link="\#006666" 
+      alink="\#cc9900" vlink="\#666633" _1_>
+}
+
+
+# _startspacer_ is a spacer to get past the 10010 graphic. It contains an open
+# <table> tag which must eventually be closed by _endspacer_.
+_startspacer_ {
+<table border=0 cellspacing=0 cellpadding=0 width="100%">
+<tr><td valign=top width=65><img src="_httpimg_/spacer.gif" width="65" height="1" alt="" border="0"></td>
+<td><center><table width="80%"><tr><td>
+}
+
+
+# pagebanner uses :
+# _imagecollection_ 
+# _imagehome_
+# _imagehelp_
+# _imagepref_
+# _imagethispage_
+# _pagebannerextra_
+_pagebanner_ {
+<!-- page banner (\_style:pagebanner\_) -->
+<center>
+<table width=_pagewidth_ cellspacing=0 cellpadding=0>
+  <tr valign=top>
+    <td rowspan=2 align=left>_imagecollection_</td>
+    <td align=right>_javalinks_</td>
+  </tr>
+
+  <tr>
+    <td align=right>_imagethispage_</td>
+  </tr>
+
+  <tr>
+    <td colspan=2>_pagebannerextra_</td>
+  </tr>
+</table>
+</center>
+<!-- end of page banner -->
+}
+
+_pagebanner_[v=1] {
+<!-- page banner - text version [v=1] (\_style:pagebanner\_) -->
+<center><h2><b><u>_imagecollection_</u></b></h2></center><p>
+_javalinks_
+_pagebannerextra_
+<p>
+<!-- end of page banner -->
+}
+
+_footer_ {
+<!-- page footer (\_style:footer\_) -->
+_pagefooterextra_
+</table>
+_endspacer__htmlfooter_
+}
+
+_endspacer_ {</center>
+</td></tr></table>
+}
+
+_htmlfooter_ {
+</body>
+</html>
+}
+
+_globalscripts_{
+<script>
+<!--
+    function img\_on(imgName) \{
+        if (version == "n3") \{
+            imgOn = eval(imgName + "\_on.src");
+            document [imgName].src = imgOn;
+        \}
+    \}
+
+    function img\_off(imgName) \{
+        if (version == "n3") \{
+            imgOff = eval(imgName + "\_off.src");
+            document [imgName].src = imgOff;
+        \}
+    \}
+    _If_(_cgiargx_,_scriptdetach_)
+    browserName = navigator.appName;          
+    browserVer = parseInt(navigator.appVersion);
+    if ((browserName == "Netscape" || browserName == "Microsoft Internet Explorer")
+        && browserVer >= 3) version = "n3";
+    else version = "n2";
+              
+    if (version == "n3") \{
+	_javaimagesheader_
+	_javaimagesnavbar_
+	_javaimagescontent_
+    \}
+    _pagescriptextra_
+// -->
+</script>
+}
+
+_globalscripts_ [v=1] {
+<script>
+<!--
+_If_(_cgiargx_,_scriptdetach_)
+_pagescriptextra_
+// -->
+</script>
+}
+
+_scriptdetach_ {
+    function close\_detach() \{
+	close();
+    \}
+}
Index: /tags/gsdl-21/gsdl/macros/users.dm
===================================================================
--- /tags/gsdl-21/gsdl/macros/users.dm	(revision 699)
+++ /tags/gsdl-21/gsdl/macros/users.dm	(revision 699)
@@ -0,0 +1,194 @@
+package userslistusers
+
+_header_ {
+<html>
+<head>
+<title>List of current users</title>
+</head>
+<body bgcolor="#ffffff" text="#000000" link="#006666"
+      alink="#cc9900" vlink="#666633">
+}
+
+_content_ {}
+
+_contentstart_ {
+<h2>List of current users</h2>
+
+
+<table border=0 cellspacing=1 cellpadding=3 width=100%>
+<tr>
+  <th bgcolor="\#d0d0d0">user</th>
+  <th bgcolor="\#d0d0d0">account status</th>
+  <th bgcolor="\#d0d0d0">groups</th>
+  <th bgcolor="\#d0d0d0">comment</th>
+  <th>&nbsp;</th>
+</tr>
+}
+
+_contentend_ {
+</table>
+}
+
+_textadduser_ {add a new user}
+_textedituser_ {edit}
+_textdeleteuser_ {delete}
+
+_footer_ {
+</body>
+</html>
+}
+
+
+
+package usersedituser
+
+_header_ {
+<html>
+<head>
+<title>_If_("_cgiarguma_" eq "edituser",Edit user information,Add a new user)</title>
+</head>
+<body bgcolor="#ffffff" text="#000000" link="#006666"
+      alink="#cc9900" vlink="#666633">
+}
+
+_content_ {
+<h2>_If_("_cgiarguma_" eq "edituser",Edit user information,Add a new user)</h2>
+
+
+_users:messagestatus_
+
+<form name="edituser" method="post" action="_gwcgi_">
+<input type=hidden name="e" value="_compressedoptions_">
+<input type=hidden name="a" value="um">
+<input type=hidden name="uma" value="_cgiarguma_">
+
+<table border=0>
+<tr><td>username</td><td><input type="text" name="umun" value="_users:usersargun_" size=15></td>
+<td><font color=gray>Usernames must be between
+2 and 30 characters long. They can
+contain alphanumeric characters, '.', and '_'.
+</font></td>
+</tr>
+<tr><td>password</td><td><input type="text" name="umpw" value="_users:usersargpw_" size=9></td>
+<td><font color=gray>
+Passwords must be between 3 and 8 characters long. They can contain 
+any normal printable ASCII characters. 
+_If_("_cgiarguma_" eq "edituser",If this field is blank the old password will be kept.)
+</font></td></tr>
+<tr><td>account status</td><td>
+<select name="umus">
+<option value="enabled">enabled
+<option value="disabled"_If_("_users:usersargus_" ne "enabled", selected)>disabled
+</select>
+</td></tr>
+<tr><td>groups</td><td><input type="text" name="umug" value="_users:usersargug_" size=15></td>
+<td><font color=gray>
+Groups is a comma separated list, do not put spaces after the commas.
+</font></td></tr>
+<tr><td>comment</td><td colspan=2><input type="text" name="umc" value="_users:usersargc_" size=50></td></tr>
+<tr></td><td><td><input type="submit" name=beu value="submit"></td></tr>
+</table>
+</form>
+}
+
+_footer_ {
+</body>
+</html>
+}
+
+
+
+
+package usersdeleteuser
+
+_header_ {
+<html>
+<head>
+<title>Delete a user</title>
+</head>
+<body bgcolor="#ffffff" text="#000000" link="#006666"
+      alink="#cc9900" vlink="#666633">
+}
+
+_content_ {
+
+<p>
+<table border=0>
+<tr><td>
+<form name="deleteuser" method="post" action="_gwcgi_">
+<input type=hidden name="e" value="_compressedoptions_">
+<input type=hidden name="a" value="um">
+<input type=hidden name="uma" value="_cgiarguma_">
+<input type=hidden name="umun" value="_cgiargumun_">
+<font color=red>Do you really want to permanently remove user <b>_cgiargumun_</b>?</font>
+<input type="submit" name=cm value="no">
+<input type="submit" name=cm value="yes">
+</form>
+</td></tr>
+</table>
+}
+
+_footer_ {
+</body>
+</html>
+}
+
+
+
+package userschangepasswd
+
+_pagetitle_ {Change password}
+_imagethispage_ {_imagegreenstone_}
+
+_content_ {
+<h2>Change password</h2>
+<p><font color=gray>Passwords must be between 3 and 8 characters long. 
+They can contain any normal printable ASCII characters.</font>
+
+<p>_users:messagestatus_
+
+<p>
+<form name="changepasswd" method="post" action="_gwcgi_">
+<input type=hidden name="e" value="_compressedoptions_">
+<input type=hidden name="a" value="um">
+<input type=hidden name="uma" value="_cgiarguma_">
+<table border=0>
+<tr><td align=right>old password</td><td><input type="text" name="umpw" size=9></td></tr>
+<tr><td align=right>new password</td><td><input type="password" name="umnpw1" size=9></td></tr>
+<tr><td align=right>retype new password</td><td><input type="password" name="umnpw2" size=9></td></tr>
+<tr><td></td><td><input type="submit" name=bcp value="submit"></td></tr>
+</table>
+</table>
+</form>
+
+}
+
+
+package userschangepasswdok
+
+_pagetitle_ {Change password}
+_imagethispage_ {_imagegreenstone_}
+
+_content_ {
+<h2>Change password</h2>
+
+<p>Your password was successfully changed.
+}
+
+
+
+package users
+
+_messageinvalidusername_ {<font color=red>The username is invalid.</font> }
+_messageinvalidpassword_ {<font color=red>The password is invalid.</font> }
+_messageemptypassword_ {<font color=red>Please enter an initial password for this user.</font> }
+_messageuserexists_ {<font color=red>This user already exists, 
+please enter in another username.</font> }
+
+_messageusernameempty_ {<font color=red>Please enter your username.</font> }
+_messagepasswordempty_ {<font color=red>You must enter your old password.</font> }
+_messagenewpass1empty_ {<font color=red>Enter your new password and then retype it.</font> }
+_messagenewpass2empty_ {_messagenewpass1empty_}
+_messagenewpassmismatch_ {<font color=red>The two versions of your new password did not match.</font> }
+_messagenewinvalidpassword_ {<font color=red>You entered an invalid password.</font> }
+_messagefailed_ {<font color=red>Either your username or password was incorrect.</font> }
Index: /tags/gsdl-21/gsdl/macros/zhbase.dm
===================================================================
--- /tags/gsdl-21/gsdl/macros/zhbase.dm	(revision 699)
+++ /tags/gsdl-21/gsdl/macros/zhbase.dm	(revision 699)
@@ -0,0 +1,88 @@
+package Global
+
+_httpiconchelpof_[l=zh] {_httpimg_/zhchelpof.gif}
+_httpiconchelpon_[l=zh] {_httpimg_/zhchelpon.gif}
+_widthchelpx_[l=zh] {41}
+_heightchelpx_[l=zh] {20}
+
+_httpiconchomeof_[l=zh] {_httpimg_/zhchomeof.gif}
+_httpiconchomeon_[l=zh] {_httpimg_/zhchomeon.gif}
+_widthchomex_[l=zh] {41}
+_heightchomex_[l=zh] {20}
+
+_httpiconcontoff_[l=zh] {_httpimg_/zhcontoff.gif}
+_httpiconconton_[l=zh] {_httpimg_/zhconton.gif}
+_widthcontx_[l=zh] {76}
+_heightcontx_[l=zh] {17}
+
+_httpiconcprefof_[l=zh] {_httpimg_/zhcprefof.gif}
+_httpiconcprefon_[l=zh] {_httpimg_/zhcprefon.gif}
+_widthcprefx_[l=zh] {41}
+_heightcprefx_[l=zh] {20}
+
+_httpiconhauth_[l=zh] {_httpimg_/zhh\_auth.gif}
+_httpiconhhelp_[l=zh] {_httpimg_/zhh\_help.gif}
+_widthhauth_[l=zh] {200}
+_heighthauth_[l=zh] {57}
+
+_httpiconhpref_[l=zh] {_httpimg_/zhh\_pref.gif}
+_widthhpref_[l=zh] {200}
+_heighthpref_[l=zh] {57}
+
+_httpiconhsearch_[l=zh] {_httpimg_/zhh\_sear.gif}
+_widthhsearch_[l=zh] {200}
+_heighthsearch_[l=zh] {57}
+
+_httpiconhtitle_[l=zh] {_httpimg_/zhh\_title.gif}
+_widthhtitle_[l=zh] {200}
+_heighthtitle_[l=zh] {57}
+
+_httpiconqryresb_[l=zh] {_httpimg_/zhqryresb.gif}
+_widthqryresb_[l=zh] {537}
+_heightqryresb_[l=zh] {17}
+
+_httpicontauthgr_[l=zh] {_httpimg_/zhtauthgr.gif}
+_httpicontauthof_[l=zh] {_httpimg_/zhtauthof.gif}
+_httpicontauthon_[l=zh] {_httpimg_/zhtauthon.gif}
+_widthtauthx_[l=zh] {110}
+_heighttauthx_[l=zh] {17}
+
+_httpicontsrchgr_[l=zh] {_httpimg_/zhtsrchgr.gif}
+_httpicontsrchof_[l=zh] {_httpimg_/zhtsrchof.gif}
+_httpicontsrchon_[l=zh] {_httpimg_/zhtsrchon.gif}
+_widthtsrchx_[l=zh] {87}
+_heighttsrchx_[l=zh] {17}
+
+_httpiconttitlgr_[l=zh] {_httpimg_/zhttitlgr.gif}
+_httpiconttitlof_[l=zh] {_httpimg_/zhttitlof.gif}
+_httpiconttitlon_[l=zh] {_httpimg_/zhttitlon.gif}
+_widthttitlx_[l=zh] {87}
+_heightttitlx_[l=zh] {17}
+
+
+_iconclosedbook_ [v=1,l=zh]{<small><_font_><b>æåŒè®°åœ: </b></font></small>}
+
+_textimageabout_[l=zh] {äž»é¡µ}
+_textimagehelp_[l=zh]{åž®å©é¡µ}
+_textimagepref_[l=zh]{éé¡¹é¡µ}
+_textimagegreenstone_[l=zh] {ç»¿ç³æ°å­ååŸä¹ŠéŠèœ¯ä»¶}
+
+_textimagesearch_[l=zh]{ç¹å®èåŽæ£çŽ¢}
+_textimagebrowsetitles_[l=zh] {æé¢åçé¿æäŒ¯å­æ¯é¡ºåºæµè§}
+_textimagebrowseauthors_[l=zh]{æäœè
+åå­çé¿æäŒ¯å­æ¯é¡ºåºæµè§ }
+
+_textgotext_[l=zh] {æææ¬æ¹åŒæµè§æ­€é¡µ}
+_textgographics_[l=zh]{æåŸåœ¢æ¹åŒæµè§æ­€é¡µ}
+
+_texticontabsearchgreen_[l=zh]{æ¥è¯¢}
+_texticontabauthorgreen_[l=zh]{äœè
+ A-Z}
+_texticontabtitlegreen_[l=zh]{é¢å A-Z}
+_texticontext_[l=zh] {æµè§æç« å
+å®¹}
+
+_texticondetach_[l=zh] {æåŒäžäžªæ°ççªå£æµè§æ­€é¡µ}
+
+_texticonwarning_[l=zh] {<b>è­Šå: </b>}
+_texticoncont_[l=zh] {ç»§ç»­?}
Index: /tags/gsdl-21/gsdl/macros/zhhelp.dm
===================================================================
--- /tags/gsdl-21/gsdl/macros/zhhelp.dm	(revision 699)
+++ /tags/gsdl-21/gsdl/macros/zhhelp.dm	(revision 699)
@@ -0,0 +1,31 @@
+package help
+
+_simplehelp_[l=zh]{
+<p>æäžç§æ¹æ³æ¥æŸæå
+³_collectionname_çä¿¡æ¯:
+<ul>
+  <li>æç¹å®çè¯æ¥è¯¢
+  <li>æé¢åçæŒé³é¡ºåºæµè§
+  <li>æäœè
+åçæŒé³é¡ºåºæµè§
+</ul>
+
+<p>äœ å¯ä»¥éè¿æ¥è¯¢é¡µ<b>æç¹å®çè¯æ¥è¯¢</b>å
+å«è¯¥è¯çè®°åœãæ¥è¯¢é¡µæ¯è¿å
+¥ç³»ç»æ¶çéŠé¡µïŒäœ ä¹å¯ä»¥éè¿æå
+¶ä»é¡µé¢äžæ¹ç<b>æ¥è¯¢</b>æé®è¿å
+¥è¿äžé¡µã
+
+<p>äœ å¯ä»¥éè¿æé¡µé¢äžæ¹ç<b>é¢å a-z </b>æé®<b>æé¢åçæŒé³é¡ºåºæµè§</b>å
+šéšè®°åœã<br>  
+
+<p>äœ å¯ä»¥éè¿æé¡µé¢äžæ¹ç<b>äœè
+ a-z </b>æé®<b>æäœè
+åçæŒé³é¡ºåºæµè§</b>å
+šéšè®°åœã<br>
+
+<p>
+_iconexplain_
+}
+
+
Index: /tags/gsdl-21/gsdl/macros/zhquery.dm
===================================================================
--- /tags/gsdl-21/gsdl/macros/zhquery.dm	(revision 699)
+++ /tags/gsdl-21/gsdl/macros/zhquery.dm	(revision 699)
@@ -0,0 +1,22 @@
+package query
+
+_textfm1_[l=zh] {è¯é¢: }
+_textmt2_[l=zh] {è¶
+è¿}
+_textdmtq3_[l=zh] {å¹é
+æé®çè®°åœ}
+
+_textmatches_[l=zh] {å¹é
+ }
+
+_textbeginsearch_[l=zh] {åŒå§æ¥è¯¢}
+_textsome_[l=zh] {éšå}
+_textall_[l=zh] {å
+šéš}
+
+_textsimplesearch_[l=zh] {æ _If_(_hselection_, _hselection_)_If_(_jselection_,_textjselect_)_If_(_nselection_, in _nselection_ language) 
+æ¥è¯¢å
+å« _querytypeselection_ æé®è¯çè®°åœ}
+
+# these can be the same until translation is done
+_textadvancedsearch_[l=zh] {_textsimplesearch_}
Index: /tags/gsdl-21/gsdl/packages/mg/COPYING
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/COPYING	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/COPYING	(revision 699)
@@ -0,0 +1,344 @@
+		    GNU GENERAL PUBLIC LICENSE
+		       Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.
+                          675 Mass Ave, Cambridge, MA 02139, USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+			    Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it.  (Some other Free Software Foundation software is covered by
+the GNU Library General Public License instead.)  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+  To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have.  You must make sure that they, too, receive or can get the
+source code.  And you must show them these terms so they know their
+rights.
+
+  We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+  Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+  Finally, any free program is threatened constantly by software
+patents.  We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary.  To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+
+		    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+			    NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+		     END OF TERMS AND CONDITIONS
+
+
+	    How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) 19yy  <name of author>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program; if not, write to the Free Software
+    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+    Gnomovision version 69, Copyright (C) 19yy name of author
+    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+  `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+  <signature of Ty Coon>, 1 April 1989
+  Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs.  If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library.  If this is what you want to do, use the GNU Library General
+Public License instead of this License.
Index: /tags/gsdl-21/gsdl/packages/mg/INSTALL
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/INSTALL	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/INSTALL	(revision 699)
@@ -0,0 +1,176 @@
+Basic Installation
+==================
+
+   These are generic installation instructions.
+
+   The `configure' shell script attempts to guess correct values for
+various system-dependent variables used during compilation.  It uses
+those values to create a `Makefile' in each directory of the package.
+It may also create one or more `.h' files containing system-dependent
+definitions.  Finally, it creates a shell script `config.status' that
+you can run in the future to recreate the current configuration, a file
+`config.cache' that saves the results of its tests to speed up
+reconfiguring, and a file `config.log' containing compiler output
+(useful mainly for debugging `configure').
+
+   If you need to do unusual things to compile the package, please try
+to figure out how `configure' could check whether to do them, and mail
+diffs or instructions to the address given in the `README' so they can
+be considered for the next release.  If at some point `config.cache'
+contains results you don't want to keep, you may remove or edit it.
+
+   The file `configure.in' is used to create `configure' by a program
+called `autoconf'.  You only need `configure.in' if you want to change
+it or regenerate `configure' using a newer version of `autoconf'.
+
+The simplest way to compile this package is:
+
+  1. `cd' to the directory containing the package's source code and type
+     `./configure' to configure the package for your system.  If you're
+     using `csh' on an old version of System V, you might need to type
+     `sh ./configure' instead to prevent `csh' from trying to execute
+     `configure' itself.
+
+     Running `configure' takes awhile.  While running, it prints some
+     messages telling which features it is checking for.
+
+  2. Type `make' to compile the package.
+
+  3. Optionally, type `make check' to run any self-tests that come with
+     the package.
+
+  4. Type `make install' to install the programs and any data files and
+     documentation.
+
+  5. You can remove the program binaries and object files from the
+     source code directory by typing `make clean'.  To also remove the
+     files that `configure' created (so you can compile the package for
+     a different kind of computer), type `make distclean'.  There is
+     also a `make maintainer-clean' target, but that is intended mainly
+     for the package's developers.  If you use it, you may have to get
+     all sorts of other programs in order to regenerate files that came
+     with the distribution.
+
+Compilers and Options
+=====================
+
+   Some systems require unusual options for compilation or linking that
+the `configure' script does not know about.  You can give `configure'
+initial values for variables by setting them in the environment.  Using
+a Bourne-compatible shell, you can do that on the command line like
+this:
+     CC=c89 CFLAGS=-O2 LIBS=-lposix ./configure
+
+Or on systems that have the `env' program, you can do it like this:
+     env CPPFLAGS=-I/usr/local/include LDFLAGS=-s ./configure
+
+Compiling For Multiple Architectures
+====================================
+
+   You can compile the package for more than one kind of computer at the
+same time, by placing the object files for each architecture in their
+own directory.  To do this, you must use a version of `make' that
+supports the `VPATH' variable, such as GNU `make'.  `cd' to the
+directory where you want the object files and executables to go and run
+the `configure' script.  `configure' automatically checks for the
+source code in the directory that `configure' is in and in `..'.
+
+   If you have to use a `make' that does not supports the `VPATH'
+variable, you have to compile the package for one architecture at a time
+in the source code directory.  After you have installed the package for
+one architecture, use `make distclean' before reconfiguring for another
+architecture.
+
+Installation Names
+==================
+
+   By default, `make install' will install the package's files in
+`/usr/local/bin', `/usr/local/man', etc.  You can specify an
+installation prefix other than `/usr/local' by giving `configure' the
+option `--prefix=PATH'.
+
+   You can specify separate installation prefixes for
+architecture-specific files and architecture-independent files.  If you
+give `configure' the option `--exec-prefix=PATH', the package will use
+PATH as the prefix for installing programs and libraries.
+Documentation and other data files will still use the regular prefix.
+
+   If the package supports it, you can cause programs to be installed
+with an extra prefix or suffix on their names by giving `configure' the
+option `--program-prefix=PREFIX' or `--program-suffix=SUFFIX'.
+
+Optional Features
+=================
+
+   Some packages pay attention to `--enable-FEATURE' options to
+`configure', where FEATURE indicates an optional part of the package.
+They may also pay attention to `--with-PACKAGE' options, where PACKAGE
+is something like `gnu-as' or `x' (for the X Window System).  The
+`README' should mention any `--enable-' and `--with-' options that the
+package recognizes.
+
+   For packages that use the X Window System, `configure' can usually
+find the X include and library files automatically, but if it doesn't,
+you can use the `configure' options `--x-includes=DIR' and
+`--x-libraries=DIR' to specify their locations.
+
+Specifying the System Type
+==========================
+
+   There may be some features `configure' can not figure out
+automatically, but needs to determine by the type of host the package
+will run on.  Usually `configure' can figure that out, but if it prints
+a message saying it can not guess the host type, give it the
+`--host=TYPE' option.  TYPE can either be a short name for the system
+type, such as `sun4', or a canonical name with three fields:
+     CPU-COMPANY-SYSTEM
+
+See the file `config.sub' for the possible values of each field.  If
+`config.sub' isn't included in this package, then this package doesn't
+need to know the host type.
+
+   If you are building compiler tools for cross-compiling, you can also
+use the `--target=TYPE' option to select the type of system they will
+produce code for and the `--build=TYPE' option to select the type of
+system on which you are compiling the package.
+
+Sharing Defaults
+================
+
+   If you want to set default values for `configure' scripts to share,
+you can create a site shell script called `config.site' that gives
+default values for variables like `CC', `cache_file', and `prefix'.
+`configure' looks for `PREFIX/share/config.site' if it exists, then
+`PREFIX/etc/config.site' if it exists.  Or, you can set the
+`CONFIG_SITE' environment variable to the location of the site script.
+A warning: not all `configure' scripts look for a site script.
+
+Operation Controls
+==================
+
+   `configure' recognizes the following options to control how it
+operates.
+
+`--cache-file=FILE'
+     Use and save the results of the tests in FILE instead of
+     `./config.cache'.  Set FILE to `/dev/null' to disable caching, for
+     debugging `configure'.
+
+`--help'
+     Print a summary of the options to `configure', and exit.
+
+`--quiet'
+`--silent'
+`-q'
+     Do not print messages saying which checks are being made.
+
+`--srcdir=DIR'
+     Look for the package's source code in directory DIR.  Usually
+     `configure' can determine that directory automatically.
+
+`--version'
+     Print the version of Autoconf used to generate the `configure'
+     script, and exit.
+
+`configure' also accepts some other, not widely useful, options.
+
Index: /tags/gsdl-21/gsdl/packages/mg/INSTALL.mg
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/INSTALL.mg	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/INSTALL.mg	(revision 699)
@@ -0,0 +1,38 @@
+                     BUILDING AND INSTALLATION OF MG
+                     ===============================
+ 
+This file explains how to build the mg system.
+ 
+(1) Follow the general GNU install instructions in the INSTALL file.
+    NOTE: this procedure has changed from version mg-1.1 which did
+    not use a GNU configure script.
+    e.g.
+      cd ~/mg-1.2
+      ./configure --prefix=`pwd`
+      make
+      make check
+      make install
+    Report portability problems to tes@kbs.citri.edu.au .
+ 
+(2) You will need to set the environment variable MGDATA which 
+    indicates where the mg data files will be created. You will also
+    need to create the directory you specify.
+ 
+    e.g.
+        setenv MGDATA ~/mgdata
+	mkdir ~/mgdata
+
+(4) You may also need to set the environment variable MGSAMPLE which
+    is used by mg_get and indicates where the sample data included
+    with this package is located.
+    If this is not specified it defaults to ``./SampleData''
+
+(4) Included with mg is an X11 interface to mgquery called xmg. 
+    Xmg is a ``wish'' script which uses the Tcl/Tk packages available
+    by anonymous ftp from allspice.berkeley.edu [128.32.150.27].
+ 
+(5) Another environment variable that you may wish to set is
+    MGIMAGEVIEWER, this variable sets the image viewer to be used
+    to display images. The image viewer must take the image from stdin.
+    If this is not specified it defaults to ``xv -''.
+ 
Index: /tags/gsdl-21/gsdl/packages/mg/MODIFICATIONS
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/MODIFICATIONS	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/MODIFICATIONS	(revision 699)
@@ -0,0 +1,1023 @@
+TITLE
+  Parsing of Long Words
+APPLICATION
+ mg-1, mg-2
+TYPE
+  bug
+REPORT
+  tim@cosc.canterbury.ac.nz - May 11th 1994
+FIX
+  tes@kbs.citri.edu.au - August 9th 1994
+CLAIM
+  Mg didn't handle long words properly; it crashed.
+PROBLEM
+  Invf passes calls PARSE_LONG_WORD [words.h] which uses a limit of
+  MAXLONGWORD on iterating thru the string and storing into 
+  a word. MAXLONGWORD = 8192.
+  However, mg strings generally store the length in the first
+  byte limiting them to 255 characters. The word which was passed
+  to PARSE_LONG_WORD was an allocated string of MAXSTEMLEN = 255,
+  which is as large as we should get anyway. Thus when accessing
+  a larger word than 255 chars, PARSE_LONG_WORD would allow it
+  (less than 8192) and would try storing beyond the array limit.
+SOLUTION
+  The author can't remember why PARSE_LONG_WORD was used and what
+  the significance of MAXLONGWORD = 8192 is.
+  So PARSE_LONG_WORD has been changed to PARSE_STEM_WORD which
+  uses MAXSTEMLEN as its limit.
+FILES
+ *   words.h
+ *   invf.pass1.c
+ *   invf.pass2.c
+ *   ivf.pass1.c
+ *   ivf.pass2.c
+ *   query.ranked.c
+*************************************************************
+TITLE
+  Use of Lovins stemmer
+APPLICATION
+  mg-1
+TYPE
+  improve
+REPORT
+  local - 1994
+FIX
+  linh@kbs.citri.edu.au - 1994
+CLAIM
+  Stemming was done naively.
+PROBLEM
+  Only a few types of words and their endings
+  were considered.
+SOLUTION
+  Replacement with a more elaborate "known" stemmer by Lovins.
+  The algorithm is described in:
+    J.B. Lovins, "Development of a Stemming Algorithm",
+    Mechanical Translation and Computational Linguistics, Vol 11,1968.
+FILES
+ *   stem.c
+ *   stem.h
+*************************************************************
+TITLE
+  Different term parsing
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  tes@kbs.citri.edu.au - 23 Aug 1994
+FIX
+  tes@kbs.citri.edu.au - 23 Aug 1994
+CLAIM
+  Boolean queries did not extract words/terms using the
+  same method as is done at inverted-file creation and
+  as is used for rank query parsing.
+PROBLEM
+  The hand-written lex. analyser, query_lex, which is called by
+  the boolean query parser was not calling a common
+  word-extraction routine as used by the rest of mg. 
+  This would be ok if the code did the same things - but they didn't.
+  Query_lex, for instance, did NOT place any limit on the
+  number of digits in a term.
+  Of even more concern, it would allow arbitrary sized words
+  although it used Pascal style strings which store the length
+  in the first byte and can therefore only be 255 characters in length.
+SOLUTION
+  Query_lex in "query.bool.y", was modified to call the routine
+  PARSE_STEM_WORD which is also used by text-inversion routines and
+  ranking query routines.
+  Now all terms are extracted by the same routine.
+  To do this, the end of the line buffer had to be noted as
+  PARSE_STEM_WORD requires a pointer to the end - which is the
+  safe thing to do (don't want to run over the end).
+  This meant I had to find the length of the query line buffer.
+  This was allocated in the file "read_line.c" by the routine,
+  "readline". Its size was the literal number 1024. 
+  This was changed to a constant and placed in "read_line.h".
+  The definition for PARSE_STEM_WORD can be found in "words.h".
+FILES
+ *   query.bool.y
+ *   query.bool.c (by bison)
+ *   read_line.c
+ *   read_line.h
+*************************************************************
+TITLE
+  Highlighting of query terms
+APPLICATION
+  mg-1
+TYPE
+  extend
+REPORT
+  tes@kbs.citri.edu.au - Aug 94
+FIX
+  tes@kbs.citri.edu.au - Sep 94
+CLAIM
+  Difficult to feel happy that the query-result returned is
+  satisfying the query - need to look hard to find the queried words.
+  Need to show words in results using some highlighting method.
+PROBLEM
+  No highlighting of query terms in results.
+SOLUTION
+  Mgquery was previously outputting the decompressed text to a pager
+  such as "less(1)" or "more(1)".
+  (Except when redirected or piped elsewhere :)
+  So what was needed was some sort of highlight pager that instead of
+  displaying the text would also use some means for highlighting the
+  stemmed query words. 
+  Two common forms of highlighting were chosen: underline and bolding.
+  These are supported by "less(1)" and possibly by "more(1)" by
+  using the backspace character.
+  A highlight pager will also need to know which words need to be
+  highlighted. Therefore, the code was modified to build up a
+  string of the stemmed query words for passing to the highlight pager.
+  Design Options:
+  ---------------
+    * Could do text filtering in mgquery before passing out to pager.
+      Instead I pipe to a separate process, the "hilite_words" pager,
+      which filters and pipes into less/more.
+    * Could do different highlighting or a combination.
+    * Could use a different structure for storing the query words other
+      than the hash-table I used.
+FILES
+ *   Makefile - to include hilite_words target
+ *   mg_hilite_words.c
+ *   mgquery.c
+ *   mgquery.1
+ *   query.bool.y
+ *   query.ranked.c
+ *   environment.c
+ *   environment.h
+ *   backend.h
+*************************************************************
+TITLE
+  Mg_compression_dict did premature free
+APPLICATION
+  mg-1
+TYPE
+  bug 
+REPORT
+  R.Sosic@cit.gu.edu.au - 23 Sep 94
+FIX
+  R.Sosic@cit.gu.edu.au - 23 Sep 94
+CLAIM
+  mg_compression_dict dumped core in 
+	file: mg_compression_dict.c
+	function: Write_data
+	line: int codelen = hd->clens[i];
+PROBLEM
+  Huffman data, hd, was freed *before* it was accessed again. 
+SOLUTION
+  The freeing of hd has been moved to after all accesses
+  (just before returning).
+FILES
+ *   mg_compression_dict.c
+*************************************************************
+TITLE
+  Boolean tree optimising rewrite
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  sosic@kurango.cit.gu.edu.au - 23 Sep 94
+FIX
+  tes@kbs.citri.edu.au - Oct 94
+CLAIM
+  "I am still getting core dump in "and" queries in mgquery, 
+   where the first word does not exist, but the second one does."
+PROBLEM
+  Having freed a particular node, it tried to refree it and 
+  access one of its fields.
+
+  I.e. code-fragment...
+
+       FreeNode(curr); /* where curr = CHILD(base) for 1st term in list */
+       FreeNodes(next);
+       FreeNodes(CHILD(base)); 
+              /* but CHILD(base) has already been freed above */
+              /* if the node was the first one in the list */
+
+SOLUTION
+  A number of things in the code seemed a bit dubious to me.
+  So I have rewritten the boolean optimising stage and abstracted out
+  the various stages - each file starts with "bool".
+  Boolean query optimising seems to be a tricky problem. 
+  It is not clear that putting an expression into a certain form will
+  actually simplify it and whether simplification means faster querying.
+  I have converted a given boolean expression into DNF 
+  (Disjunctive Normal Form).  "And not" nodes, which are readily apparent 
+  in DNF, are converted to "diff" nodes.  I have only applied the idempotency 
+  laws involving TRUE and FALSE, and not the ones requiring matching of 
+  expressions - it is a potentially more complicated problem.
+  The optimiser has been tested by playing with "bool_tester", and if you are
+  having a crash or problem in a boolean query it would be worth testing the
+  query on the "bool_tester." The token "*" stands for TRUE (or all documents) 
+  and the token "_" stands for FALSE (or no documents). This should show the 
+  expression before and after optimisation in an ascii tree bracketting format.
+FILES
+ *   bool_tree.c
+ *   bool_parser.y
+ *   bool_optimiser.c
+ *   bool_query.c 
+ *   bool_tester.c
+ *   term_lists.c
+*************************************************************
+TITLE
+  Mgtic pixel placement
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  Bruce McKenzie - bruce@cosc.canterbury.ac.nz (21st Oct 1994)
+FIX
+  singlis@cs.waikato.ac.nz
+CLAIM
+  mgtic crashed on certain files.
+PROBLEM
+  Placing pixels outside of bitmap.
+SOLUTION
+  Changed the putpixel routine to truncate at borders of the image.
+FILES
+ *   mgtic.c
+*************************************************************
+TITLE
+  Improved boolean tree optimising
+APPLICATION
+  mg-1
+TYPE
+  improve
+REPORT
+  tes@kbs.citri.edu.au - 12/Dec/94
+FIX
+  tes@kbs.citri.edu.au - 21/Dec/94, 14/Mar/95
+CLAIM
+  Optimising by conversion to DNF is not necessarily such
+  a good idea - can actually slow things down.
+PROBLEM
+  The distributive law used in converting to DNF
+  duplicates expressions.
+SOLUTION
+  Introduce a query environment variable, optimise_type = 0 | 1 | 2.
+  Type 0 does nothing to the parse tree. 
+  Type 2 does the DNF conversion.
+  Type 1 is the new default and does the following...
+    Do simple tree rearrangement like flattening. 
+    Optimise for CNF queries.
+FILES
+ *   bool_query.c, .h
+ *   bool_optimiser.c
+ *   environment.c
+ *   invf_get.c
+ *   bool_tree.c, .h
+ *   bool_tester.c
+ *   lists.h
+*************************************************************
+TITLE
+  Similarity variants
+APPLICATION
+  mg-2
+TYPE
+  extend
+REPORT
+  jz@kbs.citri.edu.au/alistair - June 1994
+FIX
+  tes@kbs.citri.edu.au - July 1994 .. Feb 1995
+CLAIM
+  Can only use one type of similarity measure - the
+  standard cosine measure.
+PROBLEM
+  See CITRI/TR-95-3 for more details.
+  The standard measure can be broken up into 7 components.
+  The 7 components are  
+  Each one of these components has a number of alternatives.
+  The overall measure, S_qd, can also be altered.
+  Thus the particular similarity measure used can be specified
+  by an 8 dimensional vector.
+  What is desired is to be able to specify to mgquery an option
+  and a 8-digit string representing this vector (assuming that
+  any one alternative can have at most 9 (not using zero) variants).
+SOLUTION
+  The programs which had to be modified were:
+  (i) mgquery,
+  (ii) mg_weights_build.
+  The other mg programs in existence store the text, indexing info,
+  and the basic statistics such as N, n, ft, fdt.  
+  Other programs which had to be created were:
+  (i) mg_fmd_build,
+  (ii) mg_wt_build.
+  Mg_fmd_build will create the file to store the f_md statistic, 
+  where f_md is the largest (maximum) f_dt of any term in document, d.
+  Mg_wt_build will create the file to store the w_t primitive.
+  It only creates this for the w_t variants 6-9 which would require
+  extra passes of invf at query time if they were not stored here.
+  For details on similarity changes for mgquery and mg_weights_build,
+  please see the other modification entries.  
+FILES
+ *   mg_fmd_build.c
+ *   mg_wt_build.c
+ *   build_lib.c, build_lib.h
+*************************************************************
+TITLE
+  Similarity variants for mgquery
+APPLICATION
+  mg-2
+TYPE
+  extend
+REPORT
+  jz@kbs.citri.edu.au/alistair - June 1994
+FIX
+  tes@kbs.citri.edu.au - July 1994 .. Feb 1995
+CLAIM
+  "mgquery" needs to be altered to allow modification of
+  the similarity measure. 
+PROBLEM
+  See CITRI/TR-95-3 for more details.
+SOLUTION
+
+  Most of the similarity measures, Sqd, are of the 
+  form:   Aqd
+         -----
+          Bqd
+  where Bqd is an expression involving Wd and possibly Wq,
+  where Aqd is a sum over the common document/query terms
+  of w_qt and w_dt.
+  Building of Aqd
+  ===============
+  The calculation of Aqd is done in the file build_Aqd.c .
+  The functions for doing this used to be in the file invf_get.c .
+  Build_Aqd.c contains 4 different functions for building Aqd, each
+  of them building a different data structure:
+    (i) Array, (ii) Splay Tree, (iii) Hash Table, (iv) List Table.
+  Each of these routines seems to have been construction by modifications
+  to duplicated code. This is often the easiest way to construct variants
+  but is quite difficult to maintain consistency. 
+  As the aim of the exercise was to try out different sim. measures for
+  retrieval effectiveness, I only modified the code that constructed
+  an array. This routine was called "CosineDecode"; I changed it to
+  "build_Aqd_Array." This change reflects the fact that we are only 
+  calculating Aqd and this need not be used for the Cosine measure.
+  The other routines: "CosineDecodeSplay," "CosineDecodeHash," and
+  "CosineDecodeList" have been left unaltered - they need to be updated
+  in the future which would be best be done by abstracting out common code.
+  By the stage of building Aqd, the query terms have been looked up in
+  the inverted file dictionary and put into a list.
+  This list of common terms is traversed to lookup the corresponding
+  invf entries. Before the invf entry is processed, all query and term
+  relevant statistics and primitive quantities are calculated.
+  For example, fqt, ft, wt, rqt, wqt, Wq-partial-sum.
+  To save unnecessary calculations, there is a test for each value
+  to see whether it is needed e.g. "if (sim->variant_needs & NEEDS_wt) ... ".  
+  Aside: Variant Needs
+  --------------------
+  The idea behind the "variant_needs" field is to be able to have all
+  the code in the one place for each possible variant and this code would
+  get the information at the correct time/place only when it is needed.
+  The overhead is a "bit-and" and "test" for each component.
+  The important concern though, is that the "variant_needs" must be
+  accurate i.e. it should be carefully maintained.
+  Each possible need is stored as a bit position in a constant/macro
+  of the form "NEEDS_component" e.g. NEEDS_wt, NEEDS_nt 
+  More recently (Jan/95), I have found that the _need_ing of a component
+  may be relevant to a particular purpose, that is, it may be needed in
+  one situation and not another. This was the case for Wd. 
+  For rdt#6, Wd was needed, however, it might not be needed for Bqd, the
+  denominator of Sqd. So I changed from NEEDS_Wd to NEEDS_Wd_for_Sqd and
+  NEEDS_Wd_for_rdt. 
+  --------------------
+  To handle the different calculations of the variants, I wrote macros
+  based around a switch statement. All these macros are stored in the file
+  "similarity.h" . The point of doing this is to centralise most of the changes
+  and cut down on the number of files which have to be altered if a new
+  way of calculating a primitive is to be added.  
+  This is achieved by having a data record called, "Similarity_variant" whose
+  fields includes all the statistics and the similarity primitives.
+  So the standard procedure is to see if something is needed and if so, then
+  extract it from an mg file or calculate it using a macro - most of the
+  input and output to the macro is done via the "Similarity_variant" structure.
+  As well as calculating Aqd, it is also a convenient place to calculate Wq.
+  Previously, for Cosine measure, Wq was not needed because each Aqd was directly
+  divided by Wq and thus would not change the ordering. However, there are
+  some Sqd's which divide by a sum involving Wq and thus need Wq. 
+  Calculating Sqd and Ordering Documents
+  ====================================== 
+  The file query.ranked.c contains the code for calculating Sqd using the
+  approximate Wd in order to do the ranking of the documents.
+  The mg-1 file was cleaned up and modified slightly.
+  All the heap data structures and routines were taken out and placed in 
+  their own file, heap_weights.c/.h .
+  The major components/steps in the ranking process were abstracted into
+  macros and functions:
+    calc_MaxParas
+    insert_heap_entry
+    insert_greater_heap_entry
+    approx_guided_insert
+    fill_initial_heap
+    add_heap_remainders
+    change_heap_to_exact
+    add_remainder_exact_weights
+    Make_Exact_Root
+    build_doc_list
+  Aside: Zeroing of an Aqd Element
+  -------------------------
+  One interesting change concerned the use of zeroing out an Aqd element
+  so as to mark it as being used in the heap.
+  In the heap, the approx. Sqd is stored based on Aqd/Wd-approx.
+  Later when Aqd is required, it can be extracted from the approx. Sqd
+  by muliplication of Wd-approx i.e. Aqd = Sqd-approx * Wd-approx.
+  This, however, is not always possible for the various Sqd variants.
+  So, instead of zeroing out Aqd, I decided just to make it negative.
+  -------------------------
+  In some Sqd cases, Wd and Wd-approx. are not required. In which case,
+  step 3 which calls on "change_heap_to_exact", "Heap_Build" and
+  "add_remainder_exact_weights" is not required.
+ 
+FILES
+ *    build_Aqd.c
+ *    query.ranked.c
+ *    similarity.c/.h (in libmg)
+ *    heap_weights.c/.h
+ *    backend.c/.h
+*************************************************************
+TITLE
+  Similarity variants for mg_weights_build
+APPLICATION
+  mg-2
+TYPE
+  extend
+REPORT
+  jz@kbs.citri.edu.au/alistair - June 1994
+FIX
+  tes@kbs.citri.edu.au - July 1994 .. Feb 1995
+CLAIM
+  "mg_weights_build" needs to be altered to allow modification of
+  the similarity measure.
+PROBLEM
+  See CITRI/TR-95-3 for more details.
+SOLUTION
+  The weight files which are generated for a particular similarity
+  measure have their names extended by a suffix.
+  In the case of Wd#1 no weights are generated.
+  In the case of the standard cosine weights, Wd#2, a 3 letter suffix
+  is used to represent .
+  In the case of the other Wd variants, a one letter suffix is used
+  to represent which Wd variant it is.
+  In each case, the variant input (e.g. -q 22222222) should be the whole
+  similarity variant string and the relevant fields will be extracted out.
+  This is done for consistency in code and interface. 
+  The code is fairly similar to the original.
+  A dependency check has been added so that the dates of files and the
+  type of needed files is verified before building.
+  The dependencies include, invf dictionary, invf index, invf, fmd and wt files.
+  The major change here, is the possible use of fmd and/or wt files.
+  Later when I was having to write mg_fmd_build and mg_wt_build,
+  I decided to abstract out some macros, namely:
+    Get_ft, Get_ft_Ft, loop_invf_entry,
+  which were put into "build_lib.c" .
+  "loop_invf_entry" takes a function/macro name as a parameter and applies
+  it to the sim record (with field fdt set), current doc number and modifies
+  the return value. 
+FILES
+ *   mg_weights_build.c
+ *   build_lib.c
+*************************************************************
+TITLE
+  Mgstat with non-existent files
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  beebe@math.utah.edu - 16 May 1994
+FIX
+  tes@kbs.citri.edu.au - 10 Aug 1994
+CLAIM
+  NaNs and Infinites would be printed out by mgstat
+  if unable to open .text or .text.dict file.
+PROBLEM
+  The NaNs etc. were output in the column stating
+  the percentage size of the file compared with the
+  number of input bytes of the source text data.
+  If it couldn't read the .text file with its
+  header describing the number of source text bytes, then
+  in working out the percentage it would divide by zero.
+  Also due to some bad control flow, it wouldn't attempt to
+  open the .text file if it failed when opening 
+  the .text.dict file.
+SOLUTION
+  Only printout the percentage if we can read the header
+  from the .text file.
+  Read in text header irrespective of text dictionary file.
+FILES
+ *   mgstat.c
+*************************************************************
+TITLE
+  Boolean tree optimisations
+APPLICATION
+  mg-2
+TYPE
+  extend
+REPORT
+  (i) tes@kbs.citri.edu.au - 28/Sep/94
+  (ii) tes@kbs.citri.edu.au - 12/Dec/94
+FIX
+  (i) tes@kbs.citri.edu.au - 18/Oct/94
+  (ii) tes@kbs.citri.edu.au - 21/Dec/94
+CLAIM
+  The initial prompt for investigating the optimisation of
+  boolean queries is noted in the mg-1 mod14.txt. 
+  The code for optimising seemed to have a number of faults.
+PROBLEM
+  Boolean optimisation was unreliable.
+SOLUTION
+  Initially (in case (i) above, see mg-1/mod14.txt), I rewrote 
+  all the boolean tree and optimising code. I converted the boolean
+  expression into DNF. I did this after reading some notes about 
+  the steps involved in optimisation and they suggested standardising
+  in some normal form. I thought that DNF would be appropriate so that
+  all the terms are converted to be part of "and" expressions and be
+  evaluated quickly using skipping.
+  This, however, can suffer quite badly if the distributive law is
+  applied to often and the query expands in size. If there was
+  some sort of cacheing of invf entries, then it might not be so
+  bad otherwise there is quite an overhead on reading the same
+  invf entry more than once.
+
+  As it happens, CNF queries are reasonably common, where the user
+  queries with a conjunction of disjunctions of similes:
+    e.g. (car | automobile | vehicle) & (fast | quick | speedy)
+  This sort of CNF query expands a hell of a lot !
+  So after speaking with Justin who wanted to benchmark Atlas with Mg on
+  these sort of queries, I looked up the MG book for other ideas.
+
+  The method that I implemented was the following:
+
+  -----------------------------------------------------------------
+  Steps of tree modifications:
+    Gets literals by pushing the nots down, detecting T/F at leaves
+    and collapsing the tree by detecting  'and' of 'and's and 'or' of
+    'or's.
+    Next it looks at the or nodes and if all the children are terms
+    then it marks the or-node as such.
+    Finally, the or-term-nodes are sorted by using the sum of their
+    ft's for comparison.
+  Steps at query evaluation:
+    If it comes across an 'and' of 'or-terms' then the evaluation is
+    done noting the distributive law.
+    I.e. a & (b | c | d) = (a & b) | (a & c) | (a & d)
+    Assuming 'a' is the c-set of documents.
+    All of 'a' is tested against 'b' and matching ones are marked.
+    Next, all the unmarked members of 'a' are tested against 'c'.
+    Likewise for 'd'.
+    Now all the marked members of our c-set are kept.
+    When we do the testing, we can use the skipping in the invf entries.
+  -----------------------------------------------------------------
+
+
+  After doing this, I added the choice of which type of optimisation
+  the user wanted by adding query-environment-variable, "optimise_type".
+  Type 0 = no parse tree modification. 
+  Type 1 = Or-term recognition and CNF query evaluation optimisation. 
+  Type 2 = Put into DNF form. [generally not recommended]
+FILES
+ *   bool_tree.c
+ *   bool_parser.y
+ *   bool_optimiser.c
+ *   bool_query.c 
+ *   bool_tester.c
+ *   term_lists.c
+ *   query_env.c
+ *   invf_get.c [GetDocsOp]
+*************************************************************
+TITLE
+  nonexistent HOME bug  
+APPLICATION
+  mg-1, mg-2
+TYPE
+  bug 
+REPORT
+  cgn@totara.cs.waikato.ac.nz - 2/May/95
+FIX
+  tes@kbs.citri.edu.au - 2/May/95
+CLAIM
+"The big problem was that mgquery crashes when the HOME environment
+ variable is not set, which is the case when it is run by the www server."
+ [...] "I expect it happens when looking for $HOME/.mgrc."
+PROBLEM
+  The result of getenv("HOME")" was used directly in
+  a sprintf call. If the environment variable HOME
+  was not in existence then null would be used.
+  In some C libraries sprintf will convert the 0
+  string into the string "(null)" on others it will core dump.
+  (For example, Solaris seems to core dump, sunos 4 seems ok).
+SOLUTION
+  The result from getenv("HOME")" is tested before
+  being used.
+FILES
+ *   commands.c
+*************************************************************
+TITLE
+  mgquery collection name preference  
+APPLICATION
+  mg-1, mg-2
+TYPE
+  improve
+REPORT
+  cgn@totara.cs.waikato.ac.nz - 2/May/95
+FIX
+  tes@kbs.citri.edu.au - 4/May/95
+CLAIM
+ Surely something must override mquery's preference for ./bib. 
+ If MGDATA is set correctly, I think it should prefer that collection, 
+ and -d should definitely override it. 
+ I could always say -d . if I really wanted ./bib.
+PROBLEM
+Currently the priority is:
+1.  Check if ./name is a directory,
+     If so then use it as the collection directory.
+2.  Check if ./name.text is a file,
+     If so then use ./ as the collection directory.
+3.  Check if mgdir/name is a directory,
+     If so then use mgdir/name as the collection directory.
+4.  Otherwise,
+     Use mgdir/name as the database file prefix.
+     This would be the case if one used "-f alice/alice".
+     However, one would then not specify a final name argument
+     and we'd never get here. Go figure ???
+SOLUTION
+Moved step 3 to the top instead.
+FILES
+ *   mgquery.c [search_for_collection()]
+*************************************************************
+TITLE
+  Printout of query terms
+APPLICATION
+  mg-1, mg-2
+TYPE
+  extend
+REPORT
+  tes@kbs.citri.edu.au - April 95
+FIX
+  tes@kbs.citri.edu.au - April 95
+CLAIM
+  No easy way to find out the parsed and stemmed words
+  used in the query. Would like to know these words
+  so I can call a separate highlighting program to 
+  highlight these words.
+PROBLEM
+  No facility available.
+SOLUTION
+  A ".queryterms" mgquery command was added which lists
+  out the parsed/stemmed queryterms of the last query.
+FILES
+ *    commands.c (added CmdQueryTerms)
+*************************************************************
+TITLE
+  mg_getrc  
+APPLICATION
+  mg-1, mg-2
+TYPE
+  extend
+REPORT
+  bruce@cosc.canterbury.ac.nz - 2/May/95
+FIX
+   - 
+CLAIM
+  Repeated code had to be written for different named
+  gets but really the same type of parsing required.
+  E.g. one might want to use a standard method for inserting
+  ^Bs between paragraphs for different books. One doesn't
+  want to write duplicate code for each different named book,
+  rather note that each book should be filtered "book" style.
+PROBLEM
+  There was no way of abstracting out types of filters from
+  the name of an instance of a collection.
+SOLUTION
+  Allow information to be given with <name, type, files>.
+  This extra info can be provided in a mg_getrc file.
+  See man page for mg_get for details.
+FILES
+ *    mg_get.sh
+*************************************************************
+TITLE
+  TREC DocNo file   
+APPLICATION
+  mg-2
+TYPE
+  improve
+REPORT
+  tes@kbs.citri.edu.au - 26/May/95
+FIX
+  tes@kbs.citri.edu.au - 26/May/95
+CLAIM
+  MG has problems dealing with trec docnos for trec disk 3.
+PROBLEM
+  Trec DocNos file didn't have a wide enough field
+  to handle disk 3. 
+SOLUTION
+  Allow different width fields for file.
+  It is still fixed width but a number in the header
+  says how wide the field is.
+FILES
+ *   passes/mg.special.c
+ *   query/mgquery.c
+*************************************************************
+TITLE
+  Boolean optimiser #1 with `!'  
+APPLICATION
+  mg-1, mg-2
+TYPE
+  bug 
+REPORT
+  triffid@iconz.co.nz - 20/7/95
+FIX
+  tes@kbs.citri.edu.au - 27/7/95
+CLAIM
+  Complained about not-nodes. 
+  e.g. complained about "croquet & !hedgehog"
+PROBLEM
+  Boolean optimiser type#1 didn't convert
+  "and not"s into diff nodes.
+SOLUTION
+  Added code to convert '&!' to '-'.
+FILES
+ *   mg/bool_optimiser.c [mg-1]
+ *   query/bool_optimiser.c [mg-2]
+*************************************************************
+TITLE
+   Autoconfiguring mg-1  
+APPLICATION
+  mg-1
+TYPE
+  improve
+REPORT
+  many people - 94/95
+FIX
+  tes@kbs.citri.edu.au - Aug/95
+CLAIM
+  Portability is limited by setting up c-macros just for particular
+  machines and operating systems.
+  People had to make changes for HP, Next, Linux, Dec Alpha, ...
+PROBLEM
+  Porting was only targetting at the machines that the author had
+  access to.
+SOLUTION
+  Use GNU's autoconfigure program.
+  This allows checking of the systems features/characteristics.
+  It also allows some checking for specific machines/OS - although
+  I have not utilised this option.
+  I used GNU's tar-1.11.8 as an example to base my changes on.
+  I also used autoscan to generate the initial "configure.in".
+  The "Makefile.in"s were done very similarly to GNU tar's.
+  The "config.h" and "sysfuncs.h" files were scrapped and 
+  rewritten. The new "config.h" file is generated by the configure
+  script - it contains all the #define's for the system features.
+  The "sysfuncs.h" file wraps up a number of system headers. 
+  For example, some systems use , while some use ;
+  which one is included is decided in "sysfuncs.h".
+  I have also used GNU tar's use of ansi2knr in its Makefiles.
+  This should hopefully allow the package to work on a system with
+  only a K&R C compiler.
+  However, there are probably problems with what I have done.
+  I am concerned about <stdarg.h> for example.
+  I also noticed that "ansi2knr" require function definitions as
+  the GNU coding style recommends ie. with function name the first
+  string on the line. This prompted me to run all the package's code
+  thru GNU's indent.
+  Setting up the configure changes is difficult. It really seems
+  necessary to try the package out on many target machines so one
+  can know what is necessary.
+  A simple check target for the main Makefile has been written.
+  It is used to see if the installation is working - it does
+  not test much of the functionality of mg.
+  It does cmp's on data files and diff's on query/result files.
+FILES
+  Most of the files in the distribution.
+*************************************************************
+TITLE
+  Consistent use of stderr
+APPLICATION
+  mg-1
+TYPE
+  improve
+REPORT
+  beebe@math.utah.edu - 16 May 1994
+FIX
+  tes@kbs.citri.edu.au - 11 August 1994
+CLAIM
+  Inconsistent use of stdout/stderr in usage messages.
+PROBLEM
+  Sometimes used "printf" and sometimes used "fprintf(stderr"
+  in usage messages.
+SOLUTION
+  All should now use "fprintf(stderr" in usage messages.
+FILES
+ *   mg_compression_dict.c
+ *   mg_compression_dict.1
+ *   mg_fast_comp_dict.c
+ *   mg_fast_comp_dict.1
+ *   mg_invf_dict.c
+ *   mg_invf_dict.1
+ *   mg_invf_dump.c
+ *   mg_invf_dump.1
+ *   mg_invf_rebuild.c
+ *   mg_invf_rebuild.1
+ *   mg_perf_hash_build.c
+ *   mg_perf_hash_build.1
+ *   mg_text_estimate.c
+ *   mg_text_estimate.1
+ *   mg_weights_build.c
+ *   mg_weights_build.1
+*************************************************************
+TITLE
+  xmg bug
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  cgn@cpsc.ucalgary.ca - 22 April 1994
+FIX
+  cgn@cpsc.ucalgary.ca - 22 April 1994
+CLAIM
+  "Serious problem in xmg, which I fear occurs whenever a query
+   doesn't return anything."
+PROBLEM
+  ??
+SOLUTION
+  [xmg.sh 201] set rank 0
+FILES
+ *   xmg.sh
+*************************************************************
+TITLE
+  Unnecessary loading of text
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  tes@kbs.citri.edu.au - ?? August 1994
+FIX
+  tes@kbs.citri.edu.au - 12 August 1994
+CLAIM
+  Mg was loading and uncompressing text when the
+  query did not require the text.
+PROBLEM
+  There was no test for the query mode
+  before loading and uncompressing the text.
+SOLUTION
+  Only load/uncompress text if query mode
+  is for text, headers or silent(for timing).
+FILES
+ *   mgquery.c
+*************************************************************
+TITLE
+  Man page errors
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  beebe@math.utah.edu - 16 May 1994
+FIX
+  beebe@math.utah.edu - 16 May 1994
+CLAIM
+  Man page errors.
+PROBLEM
+  See below.
+SOLUTION
+ "The mg_make_fast_dict.1 file has been renamed mg_fast_comp_dict.1,
+  and all mg_make_fast_dict strings changed to mg_fast_comp_dict in all
+  man pages.
+  A large number of errors of spelling, typography, spacing, fonts,
+  grammar, omitted words, slang, punctuation, missing man page
+  cross-references, and man-page style have been corrected."
+FILES
+ *    mg_compression_dict.1
+ *    mg_fast_comp_dict.1
+ *    mg_get.1
+ *    mg_invf_dict.1
+ *    mg_invf_dump.1
+ *    mg_invf_rebuild.1
+ *    mg_passes.1
+ *    mg_perf_hash_build.1
+ *    mg_text_estimate.1
+ *    mg_weights_build.1
+ *    mgbilevel.1
+ *    mgbuild.1
+ *    mgdictlist.1
+ *    mgfelics.1
+ *    mgquery.1
+ *    mgstat.1
+ *    mgtic.1
+ *    mgticbuild.1
+ *    mgticdump.1
+ *    mgticprune.1
+ *    mgticstat.1
+ *    xmg.1
+*************************************************************
+TITLE
+  Man page overview
+APPLICATION
+  mg-1
+TYPE
+  extend
+REPORT
+  beebe@math.utah.edu - 
+FIX
+  tes@kbs.citri.edu.au - 17 August 1994
+CLAIM
+  "Write new mg.1 file to give a brief overview of mg, with samples
+   of how to use it.  Otherwise, users are likely to be completely
+   overwhelmed by the number of programs (about 20) which might need to
+   be used, when in reality, only 2 or 3 are likely to be run by end
+   users."
+SOLUTION
+  It was thought that mg.1, written by Nelson Beebe, was very useful
+  but a bit too comprehensive for an introduction.
+  Therefore, two man files, mgintro.1 and mgintro++.1 were written
+  with the basic stuff in mgintro.1 and slightly more advanced stuff
+  in mgintro++.1 . 
+FILES
+ *   mg.1
+ *   mgintro.1
+ *   mgintro++.1 
+*************************************************************
+TITLE
+  Parse errors not bus errors
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  tim@cosc.canterbury.ac.nz - 2 Jun 94
+FIX
+  tes@kbs.citri.edu.au - 19 Aug 94
+CLAIM
+  "These two queries 
+   (which I typed in before I knew what I was doing!!)
+      > The Queen of Hearts, she made some tarts
+      > "The Queen of Hearts" and "she made some tarts"
+      produced the following result:
+      mgquery : parse error
+      Bus error
+  "
+PROBLEM
+  What is expected to happen under boolean querying:
+    Query1:
+    > The Queen of Hearts, she made some tarts
+    will produce a parse error due to the comma which
+    is not a valid TERM.
+    Query2:
+    > "The Queen of Hearts" and "she made some tarts"
+    will store a post-processing string
+    of ''The Queen of Hearts" and "she made some tarts'' and
+    will have a main boolean query of the empty string.
+    This is because the postprocessing string takes in
+    everything between the first quote and the last one.
+    An empty string is illegal for the boolean grammar and
+    hence a parse error.
+  The problem stems from the fact that the processing of
+  the parse tree is carried out, even though we have a 
+  parse error. In the case of using an empty string to build
+  a parse tree, it is likely to leave the parse tree undefined.
+SOLUTION
+  As soon as we find out that there is a parse-error,
+  we abandon any processing of the parse tree.
+FILES
+ *   query.bool.y
+ *   query.bool.c (generated from query.bool.y)
+*************************************************************
+TITLE
+  Perfect hashing on small vocab
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  tes@kbs.citri.edu.au - July 1994
+FIX
+  alistair@cs.mu.oz.au - July 1994
+CLAIM
+  Mg could not handle small collections in the case
+  where there was only a small number of unique words.
+  The perfect hash function would report an error.
+PROBLEM
+  Rounding of the arithmetic during the calculation of the
+  parameters of the perfect hash function was resulting in a
+  combination of values such that the probability of a hash
+  function being found was very small. This led to the limit
+  on the generation loop being exceeded, and eventual
+  failure.
+SOLUTION
+  By using ceiling rather than floor when converting from a
+  floating point value to an integer parameter, the arithmetic
+  is now correct for all lexicon sizes, and the probability of
+  each iteration successfully generating a hash function is
+  sufficiently great that with _very_ high probability the
+  execution loop counter will not be exceeded unless there
+  genuinely is no hash function (for example, if the lexicon
+  contains two words the same there cannot be  a hash
+  function).
+FILES
+ *   perf_hash.c
+*************************************************************
Index: /tags/gsdl-21/gsdl/packages/mg/Makefile.in
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/Makefile.in	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/Makefile.in	(revision 699)
@@ -0,0 +1,160 @@
+# Main Makefile for GNU tar.
+# Copyright (C) 1994, 1995 Free Software Foundation, Inc.
+ 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+ 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+ 
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ 
+PACKAGE = @PACKAGE@
+VERSION = @VERSION@
+ 
+@SET_MAKE@
+SHELL = /bin/sh
+VPATH = @srcdir@
+exec_prefix = @exec_prefix@
+prefix = @prefix@
+srcdir = @srcdir@
+ 
+
+CC = @CC@
+CFLAGS = @CFLAGS@ 
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+LDFLAGS = @LDFLAGS@
+LIBS = @LIBS@  
+TOUCH = echo timestamp >
+ 
+bindir = $(exec_prefix)/bin
+infodir = $(prefix)/info
+libexecdir = $(exec_prefix)/libexec
+ 
+MDEFINES = 
+#CC='$(CC)' CFLAGS='$(CFLAGS)' LDFLAGS='$(LDFLAGS)' LIBS='$(LIBS)' \
+#prefix='$(prefix)' exec_prefix='$(exec_prefix)' \
+#bindir='$(bindir)' libexecdir='$(libexecdir)' \
+#infodir='$(infodir)' infodir='$(infodir)' libexecdir='$(libexecdir)'
+
+SUBDIRS = lib src/text src/images src/scripts
+
+
+.SUFFIXES:
+
+all install uninstall execinstall execuninstall maninstall manuninstall: ../../config.h
+	for subdir in $(SUBDIRS); do \
+	  echo making $@ in $$subdir; \
+	  (cd $$subdir && $(MAKE) $(MDEFINES) $@) || exit 1; \
+	done
+
+# Have not written any texinfo files for the documentation yet
+#info dvi:
+#	cd doc && $(MAKE) $@
+
+check: all
+	echo making $@ in ./test directory
+	(cd ./test && $(MAKE) all) || exit 1
+	./test/testmg `pwd`
+
+mostlyclean: mostlyclean-recursive mostlyclean-local
+
+clean: clean-recursive clean-local
+
+distclean: distclean-recursive distclean-local
+	rm config.status
+
+maintainer-clean: maintainer-clean-recursive maintainer-clean-local
+	rm config.status
+
+mostlyclean-recursive clean-recursive distclean-recursive \
+maintainer-clean-recursive:
+	for subdir in $(SUBDIRS); do \
+	  target=`echo $@ | sed 's/-recursive//'`; \
+	  echo making $$target in $$subdir; \
+	  (cd $$subdir && $(MAKE) $$target) || exit 1; \
+	done
+
+mostlyclean-local:
+
+clean-local: mostlyclean-local
+
+distclean-local: clean-local
+	rm -f Makefile stamp-h
+
+maintainer-clean-local: distclean-local
+	@echo "This command is intended only for maintainers to use;"
+	@echo "rebuilding the deleted files may require special tools."
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+distdir = $(PACKAGE)-$(VERSION)
+
+distname = $(distdir)
+
+DISTFILES = \
+INSTALL.mg INSTALL README README.port COPYING MODIFICATIONS about_mg.html \
+Makefile.in acconfig.h aclocal.m4 configure.in install-sh mkinstalldirs \
+config.guess config.sub \
+config.h.in configure stamp-h.in sysfuncs.h
+
+dist: $(DISTFILES)
+#-----------------------------------------------------------------
+	#create distribution directory
+	-rm -rf $(distdir)
+	mkdir $(distdir)
+#
+	#link in local distribution files
+	ln $(DISTFILES) $(distdir)
+#
+	#link in SampleData
+	mkdir $(distdir)/SampleData
+	(cd SampleData; $(MAKE) "distdir=../$(distdir)/SampleData" dist)
+	# link in davinci stuff
+	mkdir $(distdir)/SampleData/davinci
+	(cd SampleData/davinci; \
+	$(MAKE) "distdir=../../$(distdir)/SampleData/davinci" dist)
+	# link in bible stuff
+	mkdir $(distdir)/SampleData/bible
+	(cd SampleData/bible; \
+	$(MAKE) "distdir=../../$(distdir)/SampleData/bible" dist)
+#
+	#link in tests
+	mkdir $(distdir)/test
+	(cd test; $(MAKE) "distdir=../$(distdir)/test" dist)
+#
+	#link in docs directory
+	mkdir $(distdir)/docs
+	(cd docs; $(MAKE) "distdir=../$(distdir)/docs" dist)
+#
+	#link in lib directory
+	mkdir $(distdir)/lib
+	(cd lib; $(MAKE) "distdir=../$(distdir)/lib" dist)
+#
+	#link in src distribution files
+	mkdir $(distdir)/src
+	for dir in src/text src/images src/scripts; do \
+	  echo makeing $@ in $$dir ; \
+	  mkdir $(distdir)/$$dir; \
+	  (cd $$dir; $(MAKE) "distdir=../../$(distdir)/$$dir" $@) ; \
+	done
+#
+	# archive and compress
+	tar -chf - $(distdir) | gzip -9 > $(distname).tar.gz
+	tar -chf - $(distdir) | compress > $(distname).tar.Z
+#
+	# cleanup
+	rm -rf $(distdir)
+#-----------------------------------------------------------------
+
+# Tell versions [3.59,3.63) of GNU make not to export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:
Index: /tags/gsdl-21/gsdl/packages/mg/README
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/README	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/README	(revision 699)
@@ -0,0 +1,156 @@
+                 MG INFORMATION RETRIEVAL SYSTEM
+                 ===============================
+
+The MG system is a suite of programs for compressing and indexing text
+and images. Most of the functionality implemented in the suite is as
+described in the book ``Managing Gigabytes:  Compressing and Indexing
+Documents and Images'', I.H. Witten, A.  Moffat, and T.C. Bell; Van
+Nostrand Reinhold, New York, 1994, ISBN 0-442-01863-0; US $54.95; call
+1 (800) 544-0550 to order.
+
+These features include:
+
+-- text compression using a Huffman-coded semi-static word-based scheme
+-- two-level context-based compression of bi-level images
+-- FELICS lossless compression of gray-scale images
+-- combined lossy/lossless compression for textual images
+-- indexing algorithms for large volumes of text in limited main memory
+-- index compression
+-- a retrieval system that processes Boolean and ranked queries
+-- an X windows interface to the retrieval system
+
+As one example, a collection of 2 Gb of text (1,700,000 documents) can
+be indexed (on a SPARC 10 Model 512) in about four hours and compressed
+in a further four hours to make a database that in total occupies less
+than 800 Mb, or 40% of the original size. This includes a full index to
+every word and number in the original text. Boolean queries such as
+``managing AND gigabytes'' run in a few seconds, and ranked queries of
+30--50 terms are evaluated in 10--30 seconds.
+
+Details of these methods and further performance results appear in the
+MG book.
+
+The MG system comes with ABSOLUTELY NO WARRANTY; for details see the
+file COPYING.
+
+Instructions on how to build and install mg are in the file INSTALL.
+
+
+** CHANGES FROM BOOK
+
+For copyright reasons the stemmer used in this distribution of MG is
+not the same as the one illustrated in Figure 3.8 on page 108 of the MG
+book.  This means that the numbers generated by the command ``mgstat
+alice'' will not match those numbers in Figure A.1 on page 394.
+Another stemmer was initially written as a simple stopgap for version
+1.0.  That stemmer has been replaced by a stemmer based on the Lovin's
+stemming algorithm for mg-1.1.
+
+The output format of ``mgstat'' has changed since Figure A.1 (page 394)
+was prepared. The same information is displayed but formatted
+differently.
+
+** MG VERSIONS
+
+The current version is mg-1.2, September 1995. The changes from earlier 
+versions are listed in the file MODIFICATIONS. This can be accessed
+with mg by building a database using ``mgbuild mods'' and can also be
+accessed from the mg web page (see below). 
+
+The mg-1.2 extensions include:
+
+-- Source modifications for use of GNU's autoconf.
+
+The mg-1.1 extensions include:
+
+-- A new highlighting mode.
+   The output mode ``hilite'' will highlight the query terms in the
+   retrieved text documents. The variable ``hilite_style'' can be set
+   to ``bold'' or ``underline''. It works best with the pager
+   ``less''.  A .mgrc to use would include:
+	.set pager less
+	.set mode hilite
+	.set hilite_style bold
+
+-- A web site containing manual pages, documentation, and a
+   mgquery demo page (utilising cgi scripts). 
+   See: http://www.kbs.citri.edu.au/mg
+   One of these pages ``about_mg.html'' is included in this
+   distribution.
+
+-- A revised mg_get script which uses a .mg_getrc file to map
+   specific collection names to filter types.  (Modifications by Bruce
+   McKenzie). See mg_get.1 for more details.
+
+-- Code to perform merging of existing databases. This code
+   was created by Shane Hudson and is documented in the mgmerge.README
+   file found in the docs subdirectory.  This code is maintained by
+   Shane Hudson (shane@cosc.canterbury.ac.nz).
+
+-- Revised man pages, including some new entries (thanks to Nelson
+   Beebe). See mg.1, mgintro.1, mgintro++.1.
+
+-- A real (rather than toy) stemmer.
+
+** PORTABILITY
+
+Please refer to "README.port".
+
+** CREDITS
+ 
+The MG development is largely the result of research collaboration
+between:
+
+        Tim C. Bell            <tim@cosc.canterbury.ac.nz>
+        Ian Witten             <ihw@cs.waikato.ac.nz>
+        Alistair Moffat        <alistair@cs.mu.oz.au>
+        Justin Zobel           <jz@cs.rmit.oz.au>
+
+The bulk of the programming work has been carried out by:
+
+        Stuart Inglis          <singlis@cs.waikato.ac.nz>
+        Craig Nevill-Manning   <cgn@cs.waikato.ac.nz>
+        Neil Sharman           <neil@kbs.citri.edu.au>
+        Tim Shimmin            <tes@kbs.citri.edu.au>
+
+In addition to these, the following people have contributed to the
+development of the MG software:
+
+        Lachlan Andrew         <lha@kbs.citri.edu.au>
+        Gary Eddy              <gary@cs.mu.oz.au>
+        Hugh Emberson          <hugh@cosc.canterbury.ac.nz>
+        Kerry Guise            <kerry@cs.waikato.ac.nz>
+	Shane Hudson           <shane@cosc.canterbury.ac.nz>
+        Linh Huynh             <linh@kbs.citri.edu.au>
+        Bohdan S. Majewski     <bohdan@cs.uq.oz.au>
+        Bruce McKenzie         <bruce@cosc.canterbury.ac.nz>
+
+In addition to these, the following people have submitted bug reports
+and suggestions/fixes:
+
+        Rex Barzee             <rex@python.cs.byu.edu>
+        Tim A.H. Bell          <bhat@cs.mu.oz.au>
+        Tim C. Bell            <tim@cosc.canterbury.ac.nz>
+        Nelson Beebe           <beebe@math.utah.edu>
+	Rodney Brown           <RBrown@cmutual.com.au>
+        Rok Sosic              <R.Sosic@cit.gu.edu.au>
+        Carl Staelin           <staelin@cello.hpl.hp.com>
+
+Development of the MG system was supported by the Australian Research
+Council; the Universities of Melbourne, Waikato, Canterbury, and
+Calgary; RMIT; and the Collaborative Information Technology Research
+Institute (Melbourne).
+
+** BUG REPORTS
+
+Send bug reports to <alistair@cs.mu.oz.au> and <tes@kbs.citri.edu.au>.
+Back-traces from gdb are always welcome but not mandatory :-)
+
+** FURTHER READING
+
+A bibliography of MG related research work appears in the files
+MG.Bibliography.ps and MG.Bibliography.bib on the ftp site and
+is accessible through the Web page.
+
+
+
Index: /tags/gsdl-21/gsdl/packages/mg/README.port
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/README.port	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/README.port	(revision 699)
@@ -0,0 +1,125 @@
+P O R T A B I L I T Y
+=====================
+
+The main development of MG is done under Solaris on a Sun sparc.
+However, we would also like MG to be reasonably portable.
+Therefore, MG-1.2 now uses a configure stript generated by GNU's autoconf.
+Below, is a list of the machines that the s/ware has been ported to. 
+Most only need the configure script and some require manual modifications 
+mentioned in the "Problems" field.
+
+------------------------------------------------------------------------- 
+** Systems which have successfully been built.
+
+Tester:		tes@kbs.citri.edu.au
+Date:
+Machine:	Sun
+System:		SunOS 4.1.2 8 sun4c
+Compiler:	gcc version 2.3.3
+Problems:	none
+
+Tester:		tes@kbs.citri.edu.au
+Date:
+Machine:	Sun
+System:		SunOS 4.1.2_DB 14 sun4c
+Compiler:	gcc version 2.3.3
+Problems:	none
+
+Tester:		tes@kbs.citri.edu.au
+Date:
+Machine:	Sun
+System:		SunOS 4.1.3_U1 1 sun4c
+Compiler:	gcc version 2.5.7
+Problems:	none
+
+Tester:		tes@kbs.citri.edu.au
+Date:
+Machine:	Sun
+System:		SunOS 5.3 Generic_Patch sun4m sparc
+Compiler:	gcc version 2.4.3
+Problems:	none
+
+Tester:		tes@kbs.citri.edu.au
+Date:
+Machine:	Silicon Graphics
+System:		IRIX 5.2 02282014 IP7
+Compiler:	gcc version 2.6.3
+Problems:	none
+
+Tester:		tes@kbs.citri.edu.au
+Date:
+Machine:	Sun
+System:		SunOS 5.4 Generic_101945-27 sun4c sparc
+Compiler:	cc: SC3.0.1 13 Jul 1994
+Problems:	none
+
+Tester:		RBrown@cmutual.com.au
+Date:
+Machine:
+System:		SCO 3.2v4.1
+Compiler:	gcc 2.6.1
+Problems:	requires RANLIB=true, use of tcsh
+
+Tester:		RBrown@cmutual.com.au
+Date:
+Machine:	HP
+System:		HP-UX A.09.04 D 9000/837 415303291 64-user license
+Compiler:	HP92453-01 A.09.73 HP C Compiler
+Problems:
+
+Tester:		RBrown@cmutual.com.au
+Date:
+Machine:	Data General
+System:		dgux 5.4R3.00 generic AViiON mc88110 
+Compiler:	gcc 2.4.5 (dg supplied)
+Problems:	none
+
+Tester:		RBrown@cmutual.com.au
+Date:
+Machine:	Pyramid MIPS R3000 box
+System:		
+Compiler:	cc (vendor supplied)
+Problems:	none
+
+Tester:		RBrown@cmutual.com.au
+Date:
+Machine:	IBM RS6000
+System:		AIX 3.2
+Compiler:	AIX XL C Compiler/6000
+Problems:	none
+
+Tester:		RBrown@cmutual.com.au
+Date:
+Machine:	Linux 1.2.9
+System:		
+Compiler:	gcc 2.6.3
+Problems:	none
+
+-----------------------------------------------------------------------
+** Systems which have NOT successfully been built.
+
+Tester:		RBrown@cmutual.com.au, 
+		Lloyd.Parkes@vuw.ac.nz
+Date:		Oct, Nov, 1995
+Machine:	Dec Alpha OSF/1
+System:
+Compiler:	cc
+Problems: 	Many - see below.
+
+* Problems with Dec Alpha
+  - 64 bit length longs as opposed to 32 bit longs.
+  - Lloyd.Parkes@vuw.ac.nz says that there is some bit encoding problems 
+    but the decoding seems to work.
+  - RBrown@cmutual.com.au says that it will compile but will core dump
+    when  building the test database.
+  - tes.kbs.citri.edu.au says that he needs to look into the repercussions
+    of 64 bit longs, however, he does not have access to such a machine.
+		
+-----------------------------------------------------------------------
+** Other systems. 
+
+Systems other than the above may have installation problems.
+If this is the case, please send me (tes@kbs.citri.edu.au) a log of the output
+and I will modify the configuration files in an effort to fix the problems.
+
+-----------------------------------------------------------------------
Index: /tags/gsdl-21/gsdl/packages/mg/SampleData/MODIFICATIONS
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/SampleData/MODIFICATIONS	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/SampleData/MODIFICATIONS	(revision 699)
@@ -0,0 +1,664 @@
+TITLE
+  Parsing of Long Words
+APPLICATION
+ mg-1, mg-2
+TYPE
+  bug
+REPORT
+  tim@cosc.canterbury.ac.nz - May 11th 1994
+FIX
+  tes@kbs.citri.edu.au - August 9th 1994
+CLAIM
+  Mg didn't handle long words properly; it crashed.
+PROBLEM
+  Invf passes calls PARSE_LONG_WORD [words.h] which uses a limit of
+  MAXLONGWORD on iterating thru the string and storing into 
+  a word. MAXLONGWORD = 8192.
+  However, mg strings generally store the length in the first
+  byte limiting them to 255 characters. The word which was passed
+  to PARSE_LONG_WORD was an allocated string of MAXSTEMLEN = 255,
+  which is as large as we should get anyway. Thus when accessing
+  a larger word than 255 chars, PARSE_LONG_WORD would allow it
+  (less than 8192) and would try storing beyond the array limit.
+SOLUTION
+  The author can't remember why PARSE_LONG_WORD was used and what
+  the significance of MAXLONGWORD = 8192 is.
+  So PARSE_LONG_WORD has been changed to PARSE_STEM_WORD which
+  uses MAXSTEMLEN as its limit.
+FILES
+ *   words.h
+ *   invf.pass1.c
+ *   invf.pass2.c
+ *   ivf.pass1.c
+ *   ivf.pass2.c
+ *   query.ranked.c
+*************************************************************
+TITLE
+  Use of Lovins stemmer
+APPLICATION
+  mg-1
+TYPE
+  improve
+REPORT
+  local - 1994
+FIX
+  linh@kbs.citri.edu.au - 1994
+CLAIM
+  Stemming was done naively.
+PROBLEM
+  Only a few types of words and their endings
+  were considered.
+SOLUTION
+  Replacement with a more elaborate "known" stemmer by Lovins.
+  The algorithm is described in:
+    J.B. Lovins, "Development of a Stemming Algorithm",
+    Mechanical Translation and Computational Linguistics, Vol 11,1968.
+FILES
+ *   stem.c
+ *   stem.h
+*************************************************************
+TITLE
+  Different term parsing
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  tes@kbs.citri.edu.au - 23 Aug 1994
+FIX
+  tes@kbs.citri.edu.au - 23 Aug 1994
+CLAIM
+  Boolean queries did not extract words/terms using the
+  same method as is done at inverted-file creation and
+  as is used for rank query parsing.
+PROBLEM
+  The hand-written lex. analyser, query_lex, which is called by
+  the boolean query parser was not calling a common
+  word-extraction routine as used by the rest of mg. 
+  This would be ok if the code did the same things - but they didn't.
+  Query_lex, for instance, did NOT place any limit on the
+  number of digits in a term.
+  Of even more concern, it would allow arbitrary sized words
+  although it used Pascal style strings which store the length
+  in the first byte and can therefore only be 255 characters in length.
+SOLUTION
+  Query_lex in "query.bool.y", was modified to call the routine
+  PARSE_STEM_WORD which is also used by text-inversion routines and
+  ranking query routines.
+  Now all terms are extracted by the same routine.
+  To do this, the end of the line buffer had to be noted as
+  PARSE_STEM_WORD requires a pointer to the end - which is the
+  safe thing to do (don't want to run over the end).
+  This meant I had to find the length of the query line buffer.
+  This was allocated in the file "read_line.c" by the routine,
+  "readline". Its size was the literal number 1024. 
+  This was changed to a constant and placed in "read_line.h".
+  The definition for PARSE_STEM_WORD can be found in "words.h".
+FILES
+ *   query.bool.y
+ *   query.bool.c (by bison)
+ *   read_line.c
+ *   read_line.h
+*************************************************************
+TITLE
+  Highlighting of query terms
+APPLICATION
+  mg-1
+TYPE
+  extend
+REPORT
+  tes@kbs.citri.edu.au - Aug 94
+FIX
+  tes@kbs.citri.edu.au - Sep 94
+CLAIM
+  Difficult to feel happy that the query-result returned is
+  satisfying the query - need to look hard to find the queried words.
+  Need to show words in results using some highlighting method.
+PROBLEM
+  No highlighting of query terms in results.
+SOLUTION
+  Mgquery was previously outputting the decompressed text to a pager
+  such as "less(1)" or "more(1)".
+  (Except when redirected or piped elsewhere :)
+  So what was needed was some sort of highlight pager that instead of
+  displaying the text would also use some means for highlighting the
+  stemmed query words. 
+  Two common forms of highlighting were chosen: underline and bolding.
+  These are supported by "less(1)" and possibly by "more(1)" by
+  using the backspace character.
+  A highlight pager will also need to know which words need to be
+  highlighted. Therefore, the code was modified to build up a
+  string of the stemmed query words for passing to the highlight pager.
+  Design Options:
+  ---------------
+    * Could do text filtering in mgquery before passing out to pager.
+      Instead I pipe to a separate process, the "hilite_words" pager,
+      which filters and pipes into less/more.
+    * Could do different highlighting or a combination.
+    * Could use a different structure for storing the query words other
+      than the hash-table I used.
+FILES
+ *   Makefile - to include hilite_words target
+ *   mg_hilite_words.c
+ *   mgquery.c
+ *   mgquery.1
+ *   query.bool.y
+ *   query.ranked.c
+ *   environment.c
+ *   environment.h
+ *   backend.h
+*************************************************************
+TITLE
+  Mg_compression_dict did premature free
+APPLICATION
+  mg-1
+TYPE
+  bug 
+REPORT
+  R.Sosic@cit.gu.edu.au - 23 Sep 94
+FIX
+  R.Sosic@cit.gu.edu.au - 23 Sep 94
+CLAIM
+  mg_compression_dict dumped core in 
+	file: mg_compression_dict.c
+	function: Write_data
+	line: int codelen = hd->clens[i];
+PROBLEM
+  Huffman data, hd, was freed *before* it was accessed again. 
+SOLUTION
+  The freeing of hd has been moved to after all accesses
+  (just before returning).
+FILES
+ *   mg_compression_dict.c
+*************************************************************
+TITLE
+  Boolean tree optimising rewrite
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  sosic@kurango.cit.gu.edu.au - 23 Sep 94
+FIX
+  tes@kbs.citri.edu.au - Oct 94
+CLAIM
+  "I am still getting core dump in "and" queries in mgquery, 
+   where the first word does not exist, but the second one does."
+PROBLEM
+  Having freed a particular node, it tried to refree it and 
+  access one of its fields.
+
+  I.e. code-fragment...
+
+       FreeNode(curr); /* where curr = CHILD(base) for 1st term in list */
+       FreeNodes(next);
+       FreeNodes(CHILD(base)); 
+              /* but CHILD(base) has already been freed above */
+              /* if the node was the first one in the list */
+
+SOLUTION
+  A number of things in the code seemed a bit dubious to me.
+  So I have rewritten the boolean optimising stage and abstracted out
+  the various stages - each file starts with "bool".
+  Boolean query optimising seems to be a tricky problem. 
+  It is not clear that putting an expression into a certain form will
+  actually simplify it and whether simplification means faster querying.
+  I have converted a given boolean expression into DNF 
+  (Disjunctive Normal Form).  "And not" nodes, which are readily apparent 
+  in DNF, are converted to "diff" nodes.  I have only applied the idempotency 
+  laws involving TRUE and FALSE, and not the ones requiring matching of 
+  expressions - it is a potentially more complicated problem.
+  The optimiser has been tested by playing with "bool_tester", and if you are
+  having a crash or problem in a boolean query it would be worth testing the
+  query on the "bool_tester." The token "*" stands for TRUE (or all documents) 
+  and the token "_" stands for FALSE (or no documents). This should show the 
+  expression before and after optimisation in an ascii tree bracketting format.
+FILES
+ *   bool_tree.c
+ *   bool_parser.y
+ *   bool_optimiser.c
+ *   bool_query.c 
+ *   bool_tester.c
+ *   term_lists.c
+*************************************************************
+TITLE
+  Mgtic pixel placement
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  Bruce McKenzie - bruce@cosc.canterbury.ac.nz (21st Oct 1994)
+FIX
+  singlis@cs.waikato.ac.nz
+CLAIM
+  mgtic crashed on certain files.
+PROBLEM
+  Placing pixels outside of bitmap.
+SOLUTION
+  Changed the putpixel routine to truncate at borders of the image.
+FILES
+ *   mgtic.c
+*************************************************************
+TITLE
+  Improved boolean tree optimising
+APPLICATION
+  mg-1
+TYPE
+  improve
+REPORT
+  tes@kbs.citri.edu.au - 12/Dec/94
+FIX
+  tes@kbs.citri.edu.au - 21/Dec/94, 14/Mar/95
+CLAIM
+  Optimising by conversion to DNF is not necessarily such
+  a good idea - can actually slow things down.
+PROBLEM
+  The distributive law used in converting to DNF
+  duplicates expressions.
+SOLUTION
+  Introduce a query environment variable, optimise_type = 0 | 1 | 2.
+  Type 0 does nothing to the parse tree. 
+  Type 2 does the DNF conversion.
+  Type 1 is the new default and does the following...
+    Do simple tree rearrangement like flattening. 
+    Optimise for CNF queries.
+FILES
+ *   bool_query.c, .h
+ *   bool_optimiser.c
+ *   environment.c
+ *   invf_get.c
+ *   bool_tree.c, .h
+ *   bool_tester.c
+ *   lists.h
+*************************************************************
+TITLE
+  Mgstat with non-existent files
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  beebe@math.utah.edu - 16 May 1994
+FIX
+  tes@kbs.citri.edu.au - 10 Aug 1994
+CLAIM
+  NaNs and Infinites would be printed out by mgstat
+  if unable to open .text or .text.dict file.
+PROBLEM
+  The NaNs etc. were output in the column stating
+  the percentage size of the file compared with the
+  number of input bytes of the source text data.
+  If it couldn't read the .text file with its
+  header describing the number of source text bytes, then
+  in working out the percentage it would divide by zero.
+  Also due to some bad control flow, it wouldn't attempt to
+  open the .text file if it failed when opening 
+  the .text.dict file.
+SOLUTION
+  Only printout the percentage if we can read the header
+  from the .text file.
+  Read in text header irrespective of text dictionary file.
+FILES
+ *   mgstat.c
+*************************************************************
+TITLE
+  nonexistent HOME bug  
+APPLICATION
+  mg-1, mg-2
+TYPE
+  bug 
+REPORT
+  cgn@totara.cs.waikato.ac.nz - 2/May/95
+FIX
+  tes@kbs.citri.edu.au - 2/May/95
+CLAIM
+"The big problem was that mgquery crashes when the HOME environment
+ variable is not set, which is the case when it is run by the www server."
+ [...] "I expect it happens when looking for $HOME/.mgrc."
+PROBLEM
+  The result of getenv("HOME")" was used directly in
+  a sprintf call. If the environment variable HOME
+  was not in existence then null would be used.
+  In some C libraries sprintf will convert the 0
+  string into the string "(null)" on others it will core dump.
+  (For example, Solaris seems to core dump, sunos 4 seems ok).
+SOLUTION
+  The result from getenv("HOME")" is tested before
+  being used.
+FILES
+ *   commands.c
+*************************************************************
+TITLE
+  mgquery collection name preference  
+APPLICATION
+  mg-1, mg-2
+TYPE
+  improve
+REPORT
+  cgn@totara.cs.waikato.ac.nz - 2/May/95
+FIX
+  tes@kbs.citri.edu.au - 4/May/95
+CLAIM
+ Surely something must override mquery's preference for ./bib. 
+ If MGDATA is set correctly, I think it should prefer that collection, 
+ and -d should definitely override it. 
+ I could always say -d . if I really wanted ./bib.
+PROBLEM
+Currently the priority is:
+1.  Check if ./name is a directory,
+     If so then use it as the collection directory.
+2.  Check if ./name.text is a file,
+     If so then use ./ as the collection directory.
+3.  Check if mgdir/name is a directory,
+     If so then use mgdir/name as the collection directory.
+4.  Otherwise,
+     Use mgdir/name as the database file prefix.
+     This would be the case if one used "-f alice/alice".
+     However, one would then not specify a final name argument
+     and we'd never get here. Go figure ???
+SOLUTION
+Moved step 3 to the top instead.
+FILES
+ *   mgquery.c [search_for_collection()]
+*************************************************************
+TITLE
+  Printout of query terms
+APPLICATION
+  mg-1, mg-2
+TYPE
+  extend
+REPORT
+  tes@kbs.citri.edu.au - April 95
+FIX
+  tes@kbs.citri.edu.au - April 95
+CLAIM
+  No easy way to find out the parsed and stemmed words
+  used in the query. Would like to know these words
+  so I can call a separate highlighting program to 
+  highlight these words.
+PROBLEM
+  No facility available.
+SOLUTION
+  A ".queryterms" mgquery command was added which lists
+  out the parsed/stemmed queryterms of the last query.
+FILES
+ *    commands.c (added CmdQueryTerms)
+*************************************************************
+TITLE
+  mg_getrc  
+APPLICATION
+  mg-1, mg-2
+TYPE
+  extend
+REPORT
+  bruce@cosc.canterbury.ac.nz - 2/May/95
+FIX
+   - 
+CLAIM
+  Repeated code had to be written for different named
+  gets but really the same type of parsing required.
+  E.g. one might want to use a standard method for inserting
+  ^Bs between paragraphs for different books. One doesn't
+  want to write duplicate code for each different named book,
+  rather note that each book should be filtered "book" style.
+PROBLEM
+  There was no way of abstracting out types of filters from
+  the name of an instance of a collection.
+SOLUTION
+  Allow information to be given with <name, type, files>.
+  This extra info can be provided in a mg_getrc file.
+  See man page for mg_get for details.
+FILES
+ *    mg_get.sh
+*************************************************************
+TITLE
+  Boolean optimiser #1 with `!'  
+APPLICATION
+  mg-1, mg-2
+TYPE
+  bug 
+REPORT
+  triffid@iconz.co.nz - 20/7/95
+FIX
+  tes@kbs.citri.edu.au - 27/7/95
+CLAIM
+  Complained about not-nodes. 
+  e.g. complained about "croquet & !hedgehog"
+PROBLEM
+  Boolean optimiser type#1 didn't convert
+  "and not"s into diff nodes.
+SOLUTION
+  Added code to convert '&!' to '-'.
+FILES
+ *   mg/bool_optimiser.c [mg-1]
+ *   query/bool_optimiser.c [mg-2]
+*************************************************************
+TITLE
+  Consistent use of stderr
+APPLICATION
+  mg-1
+TYPE
+  improve
+REPORT
+  beebe@math.utah.edu - 16 May 1994
+FIX
+  tes@kbs.citri.edu.au - 11 August 1994
+CLAIM
+  Inconsistent use of stdout/stderr in usage messages.
+PROBLEM
+  Sometimes used "printf" and sometimes used "fprintf(stderr"
+  in usage messages.
+SOLUTION
+  All should now use "fprintf(stderr" in usage messages.
+FILES
+ *   mg_compression_dict.c
+ *   mg_compression_dict.1
+ *   mg_fast_comp_dict.c
+ *   mg_fast_comp_dict.1
+ *   mg_invf_dict.c
+ *   mg_invf_dict.1
+ *   mg_invf_dump.c
+ *   mg_invf_dump.1
+ *   mg_invf_rebuild.c
+ *   mg_invf_rebuild.1
+ *   mg_perf_hash_build.c
+ *   mg_perf_hash_build.1
+ *   mg_text_estimate.c
+ *   mg_text_estimate.1
+ *   mg_weights_build.c
+ *   mg_weights_build.1
+*************************************************************
+TITLE
+  xmg bug
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  cgn@cpsc.ucalgary.ca - 22 April 1994
+FIX
+  cgn@cpsc.ucalgary.ca - 22 April 1994
+CLAIM
+  "Serious problem in xmg, which I fear occurs whenever a query
+   doesn't return anything."
+PROBLEM
+  ??
+SOLUTION
+  [xmg.sh 201] set rank 0
+FILES
+ *   xmg.sh
+*************************************************************
+TITLE
+  Unnecessary loading of text
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  tes@kbs.citri.edu.au - ?? August 1994
+FIX
+  tes@kbs.citri.edu.au - 12 August 1994
+CLAIM
+  Mg was loading and uncompressing text when the
+  query did not require the text.
+PROBLEM
+  There was no test for the query mode
+  before loading and uncompressing the text.
+SOLUTION
+  Only load/uncompress text if query mode
+  is for text, headers or silent(for timing).
+FILES
+ *   mgquery.c
+*************************************************************
+TITLE
+  Man page errors
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  beebe@math.utah.edu - 16 May 1994
+FIX
+  beebe@math.utah.edu - 16 May 1994
+CLAIM
+  Man page errors.
+PROBLEM
+  See below.
+SOLUTION
+ "The mg_make_fast_dict.1 file has been renamed mg_fast_comp_dict.1,
+  and all mg_make_fast_dict strings changed to mg_fast_comp_dict in all
+  man pages.
+  A large number of errors of spelling, typography, spacing, fonts,
+  grammar, omitted words, slang, punctuation, missing man page
+  cross-references, and man-page style have been corrected."
+FILES
+ *    mg_compression_dict.1
+ *    mg_fast_comp_dict.1
+ *    mg_get.1
+ *    mg_invf_dict.1
+ *    mg_invf_dump.1
+ *    mg_invf_rebuild.1
+ *    mg_passes.1
+ *    mg_perf_hash_build.1
+ *    mg_text_estimate.1
+ *    mg_weights_build.1
+ *    mgbilevel.1
+ *    mgbuild.1
+ *    mgdictlist.1
+ *    mgfelics.1
+ *    mgquery.1
+ *    mgstat.1
+ *    mgtic.1
+ *    mgticbuild.1
+ *    mgticdump.1
+ *    mgticprune.1
+ *    mgticstat.1
+ *    xmg.1
+*************************************************************
+TITLE
+  Man page overview
+APPLICATION
+  mg-1
+TYPE
+  extend
+REPORT
+  beebe@math.utah.edu - 
+FIX
+  tes@kbs.citri.edu.au - 17 August 1994
+CLAIM
+  "Write new mg.1 file to give a brief overview of mg, with samples
+   of how to use it.  Otherwise, users are likely to be completely
+   overwhelmed by the number of programs (about 20) which might need to
+   be used, when in reality, only 2 or 3 are likely to be run by end
+   users."
+SOLUTION
+  It was thought that mg.1, written by Nelson Beebe, was very useful
+  but a bit too comprehensive for an introduction.
+  Therefore, two man files, mgintro.1 and mgintro++.1 were written
+  with the basic stuff in mgintro.1 and slightly more advanced stuff
+  in mgintro++.1 . 
+FILES
+ *   mg.1
+ *   mgintro.1
+ *   mgintro++.1 
+*************************************************************
+TITLE
+  Parse errors not bus errors
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  tim@cosc.canterbury.ac.nz - 2 Jun 94
+FIX
+  tes@kbs.citri.edu.au - 19 Aug 94
+CLAIM
+  "These two queries 
+   (which I typed in before I knew what I was doing!!)
+      > The Queen of Hearts, she made some tarts
+      > "The Queen of Hearts" and "she made some tarts"
+      produced the following result:
+      mgquery : parse error
+      Bus error
+  "
+PROBLEM
+  What is expected to happen under boolean querying:
+    Query1:
+    > The Queen of Hearts, she made some tarts
+    will produce a parse error due to the comma which
+    is not a valid TERM.
+    Query2:
+    > "The Queen of Hearts" and "she made some tarts"
+    will store a post-processing string
+    of ''The Queen of Hearts" and "she made some tarts'' and
+    will have a main boolean query of the empty string.
+    This is because the postprocessing string takes in
+    everything between the first quote and the last one.
+    An empty string is illegal for the boolean grammar and
+    hence a parse error.
+  The problem stems from the fact that the processing of
+  the parse tree is carried out, even though we have a 
+  parse error. In the case of using an empty string to build
+  a parse tree, it is likely to leave the parse tree undefined.
+SOLUTION
+  As soon as we find out that there is a parse-error,
+  we abandon any processing of the parse tree.
+FILES
+ *   query.bool.y
+ *   query.bool.c (generated from query.bool.y)
+*************************************************************
+TITLE
+  Perfect hashing on small vocab
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  tes@kbs.citri.edu.au - July 1994
+FIX
+  alistair@cs.mu.oz.au - July 1994
+CLAIM
+  Mg could not handle small collections in the case
+  where there was only a small number of unique words.
+  The perfect hash function would report an error.
+PROBLEM
+  Rounding of the arithmetic during the calculation of the
+  parameters of the perfect hash function was resulting in a
+  combination of values such that the probability of a hash
+  function being found was very small. This led to the limit
+  on the generation loop being exceeded, and eventual
+  failure.
+SOLUTION
+  By using ceiling rather than floor when converting from a
+  floating point value to an integer parameter, the arithmetic
+  is now correct for all lexicon sizes, and the probability of
+  each iteration successfully generating a hash function is
+  sufficiently great that with _very_ high probability the
+  execution loop counter will not be exceeded unless there
+  genuinely is no hash function (for example, if the lexicon
+  contains two words the same there cannot be  a hash
+  function).
+FILES
+ *   perf_hash.c
+*************************************************************
Index: /tags/gsdl-21/gsdl/packages/mg/SampleData/Makefile
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/SampleData/Makefile	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/SampleData/Makefile	(revision 699)
@@ -0,0 +1,4 @@
+DISTFILES = Makefile alice13a.txt.Z MODIFICATIONS 
+
+dist:
+	ln $(DISTFILES) $(distdir)
Index: /tags/gsdl-21/gsdl/packages/mg/WIN32cfg.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/WIN32cfg.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/WIN32cfg.h	(revision 699)
@@ -0,0 +1,163 @@
+/* config.h.  Generated automatically by configure.  */
+/* config.h.in.  Generated automatically from configure.in by autoheader.  */
+
+/* Define if on AIX 3.
+   System headers sometimes define this.
+   We just want to avoid a redefinition error message.  */
+#ifndef _ALL_SOURCE
+/* #undef _ALL_SOURCE */
+#endif
+
+/* Define if using alloca.c.  */
+/* #undef C_ALLOCA */
+
+/* Define to empty if the keyword does not work.  */
+/* #undef const */
+
+/* Define to one of _getb67, GETB67, getb67 for Cray-2 and Cray-YMP systems.
+   This function is required for alloca.c support on those systems.  */
+/* #undef CRAY_STACKSEG_END */
+
+/* Define if you have alloca, as a function or macro.  */
+#define HAVE_ALLOCA 1
+
+/* Define if you have <alloca.h> and it should be used (not on Ultrix).  */
+/* #define HAVE_ALLOCA_H 1 */
+
+/* Define if you don't have vprintf but do have _doprnt.  */
+/* #undef HAVE_DOPRNT */
+
+/* Define if you have the vprintf function.  */
+#define HAVE_VPRINTF 1
+
+/* Define if on MINIX.  */
+/* #undef _MINIX */
+
+/* Define to `long' if <sys/types.h> doesn't define.  */
+/* #undef off_t */
+
+/* Define if the system does not provide POSIX.1 features except
+   with this defined.  */
+/* #undef _POSIX_1_SOURCE */
+
+/* Define if you need to in order for stat and other things to work.  */
+/* #undef _POSIX_SOURCE */
+
+/* Define as the return type of signal handlers (int or void).  */
+#define RETSIGTYPE void
+
+/* Define to `unsigned' if <sys/types.h> doesn't define.  */
+/* #undef size_t */
+
+/* If using the C implementation of alloca, define if you know the
+   direction of stack growth for your system; otherwise it will be
+   automatically deduced at run-time.
+	STACK_DIRECTION > 0 => grows toward higher addresses
+	STACK_DIRECTION < 0 => grows toward lower addresses
+	STACK_DIRECTION = 0 => direction of growth unknown
+ */
+/* #undef STACK_DIRECTION */
+
+/* Define if you have the ANSI C header files.  */
+#define STDC_HEADERS 1
+
+/* Define if you can safely include both <sys/time.h> and <time.h>.  */
+/* #define TIME_WITH_SYS_TIME 1 */
+
+/* Define if your <sys/time.h> declares struct tm.  */
+/* #undef TM_IN_SYS_TIME */
+
+/* Define to the name of the distribution.  */
+#define PACKAGE "mg"
+
+/* Define to the version of the distribution.  */
+#define VERSION "1.2"
+
+/* Define to 1 if you have the valloc function.  */
+/* #define HAVE_VALLOC 1 */
+
+/* Define to 1 if ANSI function prototypes are usable.  */
+#define PROTOTYPES 1
+
+ 
+/* Define to 1 for better use of the debugging malloc library.  See
+   site ftp.antaire.com in antaire/src, file dmalloc/dmalloc.tar.gz.  */
+/* #undef WITH_DMALLOC */
+
+ 
+/* Define to 1 if GNU regex should be used instead of GNU rx.  */
+/* #undef WITH_REGEX */
+
+/* Some braindead header files do not have the requ'd function declarations */
+#define HAVE_FGETC_DECL 1
+#define HAVE_FREAD_DECL 1
+
+/* See if have this field in the prstatus_t structure */
+/* It stores the size of the process heap */
+/* #define HAVE_PR_BRKSIZE 1 */
+
+/* Define if you have the ftime function.  */
+/* #undef HAVE_FTIME */
+
+/* Define if you have the getpagesize function.  */
+/* #undef HAVE_GETPAGESIZE */
+
+/* Define if you have the getrusage function.  */
+/* #undef HAVE_GETRUSAGE */
+
+/* Define if you have the mallinfo function.  */
+/* #undef HAVE_MALLINFO */
+
+/* Define if you have the select function.  */
+#define HAVE_SELECT 1
+
+/* Define if you have the setbuffer function.  */
+/* #undef HAVE_SETBUFFER */
+
+/* Define if you have the strftime function.  */
+#define HAVE_STRFTIME 1
+
+/* Define if you have the strtol function.  */
+#define HAVE_STRTOL 1
+
+/* Define if you have the times function.  */
+/* #define HAVE_TIMES 1 */
+
+/* Define if you have the valloc function.  */
+/* #define HAVE_VALLOC 1 */
+
+/* Define if you have the <dirent.h> header file.  */
+/* #define HAVE_DIRENT_H 1 */
+
+/* Define if you have the <fcntl.h> header file.  */
+#define HAVE_FCNTL_H 1
+
+/* Define if you have the <limits.h> header file.  */
+#define HAVE_LIMITS_H 1
+
+/* Define if you have the <memory.h> header file.  */
+#define HAVE_MEMORY_H 1
+
+/* Define if you have the <ndir.h> header file.  */
+/* #undef HAVE_NDIR_H */
+
+/* Define if you have the <string.h> header file.  */
+#define HAVE_STRING_H 1
+
+/* Define if you have the <sys/dir.h> header file.  */
+/* #undef HAVE_SYS_DIR_H */
+
+/* Define if you have the <sys/ndir.h> header file.  */
+/* #undef HAVE_SYS_NDIR_H */
+
+/* Define if you have the <sys/procfs.h> header file.  */
+/* #define HAVE_SYS_PROCFS_H 1 */
+
+/* Define if you have the <sys/time.h> header file.  */
+/* #define HAVE_SYS_TIME_H 1 */
+
+/* Define if you have the <unistd.h> header file.  */
+/* #define HAVE_UNISTD_H 1 */
+
+/* Define if you have the m library (-lm).  */
+#define HAVE_LIBM 1
Index: /tags/gsdl-21/gsdl/packages/mg/about_mg.html
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/about_mg.html	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/about_mg.html	(revision 699)
@@ -0,0 +1,75 @@
+<HTML>
+<HEAD>
+<TITLE>About MG</TITLE>
+</HEAD>
+<BODY>
+<H1>About MG</H1>
+ 
+<HR SIZE=4>
+
+MG is a public domain
+indexing and retrieval system for text, images, and textual images.
+MG is covered by a 
+<a href="http://deimos.kbs.citri.edu.au/mg/intro/copying.html">GNU 
+public licence</a>.
+<P>
+MG is available for ftp <a href="ftp://munnari.oz.au/pub/mg">here</a>
+
+<H2>Developers</H2>
+
+The MG system is an embodiment of ideas developed primarily by:
+<ul>
+<li> <a href="http://www.cosc.canterbury.ac.nz/~tim">Tim C. Bell</a>,
+	University of Canterbury;
+<li> <a href="http://www.cs.mu.oz.au/~alistair">Alistair Moffat</a>,
+	University of Melbourne;
+<li> <a href="http://www.cs.waikato.ac.nz/~ihw">Ian Witten</a>,
+	University of Waikato; and
+<li> <a href="http://www.cs.rmit.edu.au/~jz">Justin Zobel</a>,
+	RMIT.
+</ul>
+
+<H2>Programmers</H2>
+
+The bulk of the programming work has been undertaken by:
+<ul>
+<li><A HREF="http://www.cs.waikato.ac.nz/~singlis/">Stuart Inglis</A>,
+       University of Waikato;
+<li><A HREF="http://www.cs.waikato.ac.nz/~cgn/">Craig Nevill-Manning</A>,
+       University of Waikato;
+<li> Neil Sharman, University of Melbourne; and
+<li> Tim Shimmin, Collaborative Information Technology Research Institute (CITRI).
+</ul>
+A full list of contributors appears
+<A HREF="http://deimos.kbs.citri.edu.au/mg/intro/mgintro.html#xtocid2063010">
+here</a>.
+
+<H2>Institutional Support</H2>
+
+The MG project has been supported by the following organisations:
+<ul>
+<li>	The Australian Research Council;
+<li>	<a HREF="http://www.citri.edu.au/">CITRI</a>, Melbourne;
+<li>	The <a HREF="http://www.cpsc.ucalgary.ca/">University
+		of Calgary</a>, Canada;
+<li>	The <a HREF="http://www.cosc.canterbury.ac.nz/">University
+		of Canterbury</a>, New Zealand;
+<li>	The <a HREF="http://www.cs.mu.oz.au/">University
+		of Melbourne</a>, Australia;
+<li>	<a HREF="http://www.cs.rmit.edu.au/">RMIT</a>, Australia; and
+<li>	The <a HREF="http://www.cs.waikato.ac.nz/">University
+		of Waikato</a>, New Zealand.
+</ul>
+
+<H2>Backpointer</H2>
+
+If you didn't just come from there, the MG home page is accessible
+<a href="http://www.kbs.citri.edu.au/mg/">here</a>.
+
+<HR SIZE=4>
+<ADDRESS>alistair@cs.mu.oz.au</ADDRESS><BR>
+ 
+</BODY>
+</HTML>
+
+
Index: /tags/gsdl-21/gsdl/packages/mg/acconfig.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/acconfig.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/acconfig.h	(revision 699)
@@ -0,0 +1,35 @@
+/* Special definitions for mg, processed by autoheader.
+   Created by [TS:Aug/95] 
+   by grepping for AC_DEFINE on configure.in and using the appropriate defs 
+   of acconfig.h used by GNU tar.
+*/
+
+/* Define to the name of the distribution.  */
+#undef PACKAGE
+
+/* Define to the version of the distribution.  */
+#undef VERSION
+
+/* Define if `union wait' is the type of the first arg to wait functions.  */
+#undef HAVE_UNION_WAIT
+
+/* Define to 1 if you have the valloc function.  */
+#undef HAVE_VALLOC
+
+/* Define to 1 if ANSI function prototypes are usable.  */
+#undef PROTOTYPES
+ 
+/* Define to 1 for better use of the debugging malloc library.  See
+   site ftp.antaire.com in antaire/src, file dmalloc/dmalloc.tar.gz.  */
+#undef WITH_DMALLOC
+ 
+/* Define to 1 if GNU regex should be used instead of GNU rx.  */
+#undef WITH_REGEX
+
+/* Some braindead header files do not have the requ'd function declarations */
+#undef HAVE_FGETC_DECL
+#undef HAVE_FREAD_DECL
+
+/* See if have this field in the prstatus_t structure */
+/* It stores the size of the process heap */
+#undef HAVE_PR_BRKSIZE
Index: /tags/gsdl-21/gsdl/packages/mg/aclocal.m4
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/aclocal.m4	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/aclocal.m4	(revision 699)
@@ -0,0 +1,141 @@
+# Local additions to Autoconf macros.
+# Copyright (C) 1992, 1994, 1995 Free Software Foundation, Inc.
+# François Pinard <pinard@iro.umontreal.ca>, 1992.
+
+# [TS:Aug/95] - chopped out the NLS stuff which I don't support
+
+## ------------------------------- ##
+## Check for function prototypes.  ##
+## ------------------------------- ##
+
+AC_DEFUN(fp_C_PROTOTYPES,
+[AC_REQUIRE([fp_PROG_CC_STDC])
+AC_MSG_CHECKING([for function prototypes])
+if test "$ac_cv_prog_cc_stdc" != no; then
+  AC_MSG_RESULT(yes)
+  AC_DEFINE(PROTOTYPES)
+  U= ANSI2KNR=
+else
+  AC_MSG_RESULT(no)
+  U=_ ANSI2KNR=ansi2knr
+fi
+AC_SUBST(U)dnl
+AC_SUBST(ANSI2KNR)dnl
+])
+
+## ----------------------------------------- ##
+## ANSIfy the C compiler whenever possible.  ##
+## ----------------------------------------- ##
+
+# @defmac AC_PROG_CC_STDC
+# @maindex PROG_CC_STDC
+# @ovindex CC
+# If the C compiler in not in ANSI C mode by default, try to add an option
+# to output variable @code{CC} to make it so.  This macro tries various
+# options that select ANSI C on some system or another.  It considers the
+# compiler to be in ANSI C mode if it defines @code{__STDC__} to 1 and
+# handles function prototypes correctly.
+#
+# If you use this macro, you should check after calling it whether the C
+# compiler has been set to accept ANSI C; if not, the shell variable
+# @code{ac_cv_prog_cc_stdc} is set to @samp{no}.  If you wrote your source
+# code in ANSI C, you can make an un-ANSIfied copy of it by using the
+# program @code{ansi2knr}, which comes with Ghostscript.
+# @end defmac
+
+AC_DEFUN(fp_PROG_CC_STDC,
+[AC_MSG_CHECKING(for ${CC-cc} option to accept ANSI C)
+AC_CACHE_VAL(ac_cv_prog_cc_stdc,
+[ac_cv_prog_cc_stdc=no
+ac_save_CFLAGS="$CFLAGS"
+# Don't try gcc -ansi; that turns off useful extensions and
+# breaks some systems' header files.
+# AIX			-qlanglvl=ansi
+# Ultrix and OSF/1	-std1
+# HP-UX			-Aa -D_HPUX_SOURCE
+# SVR4			-Xc
+for ac_arg in "" -qlanglvl=ansi -std1 "-Aa -D_HPUX_SOURCE" -Xc
+do
+  CFLAGS="$ac_save_CFLAGS $ac_arg"
+  AC_TRY_COMPILE(
+[#if !defined(__STDC__) || __STDC__ != 1
+choke me
+#endif	
+], [int test (int i, double x);
+struct s1 {int (*f) (int a);};
+struct s2 {int (*f) (double a);};],
+[ac_cv_prog_cc_stdc="$ac_arg"; break])
+done
+CFLAGS="$ac_save_CFLAGS"
+])
+AC_MSG_RESULT($ac_cv_prog_cc_stdc)
+case "x$ac_cv_prog_cc_stdc" in
+  x|xno) ;;
+  *) CC="$CC $ac_cv_prog_cc_stdc" ;;
+esac
+])
+
+## --------------------------------------------------------- ##
+## Use AC_PROG_INSTALL, supplementing it with INSTALL_SCRIPT ##
+## substitution.					     ##
+## --------------------------------------------------------- ##
+
+AC_DEFUN(fp_PROG_INSTALL,
+[AC_PROG_INSTALL
+test -z "$INSTALL_SCRIPT" && INSTALL_SCRIPT='${INSTALL} -m 755'
+AC_SUBST(INSTALL_SCRIPT)dnl
+])
+
+## ----------------------------------- ##
+## Check if --with-dmalloc was given.  ##
+## ----------------------------------- ##
+
+# I just checked, and GNU rx seems to work fine with a slightly
+# modified GNU m4.  So, I put out the test below in my aclocal.m4,
+# and will try to use it in my things.  The idea is to distribute
+# rx.[hc] and regex.[hc] together, for a while.  The WITH_REGEX symbol
+# (which should also be documented in acconfig.h) is used to decide
+# which of regex.h or rx.h should be included in the application.
+#
+# If `./configure --with-regex' is given, the package will use
+# the older regex.  Else, a check is made to see if rx is already
+# installed, as with newer Linux'es.  If not found, the package will
+# use the rx from the distribution.  If found, the package will use
+# the system's rx which, on Linux at least, will result in a smaller
+# executable file.
+
+AC_DEFUN(fp_WITH_DMALLOC,
+[AC_MSG_CHECKING(if malloc debugging is wanted)
+AC_ARG_WITH(dmalloc,
+[  --with-dmalloc          use dmalloc, as in
+                          ftp://ftp.letters.com/src/dmalloc/dmalloc.tar.gz],
+[if test "$withval" = yes; then
+  AC_MSG_RESULT(yes)
+  AC_DEFINE(WITH_DMALLOC)
+  LIBS="$LIBS -ldmalloc"
+  LDFLAGS="$LDFLAGS -g"
+else
+  AC_MSG_RESULT(no)
+fi], [AC_MSG_RESULT(no)])
+])
+
+## --------------------------------- ##
+## Check if --with-regex was given.  ##
+## --------------------------------- ##
+
+AC_DEFUN(fp_WITH_REGEX,
+[AC_MSG_CHECKING(which of rx or regex is wanted)
+AC_ARG_WITH(regex,
+[  --with-regex            use older regex in lieu of GNU rx for matching],
+[if test "$withval" = yes; then
+  ac_with_regex=1
+  AC_MSG_RESULT(regex)
+  AC_DEFINE(WITH_REGEX)
+  LIBOBJS="$LIBOBJS regex.o"
+fi])
+if test -z "$ac_with_regex"; then
+  AC_MSG_RESULT(rx)
+  AC_CHECK_FUNC(re_rx_search, , [LIBOBJS="$LIBOBJS rx.o"])
+fi
+AC_SUBST(LIBOBJS)dnl
+])
Index: /tags/gsdl-21/gsdl/packages/mg/build_options.in
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/build_options.in	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/build_options.in	(revision 699)
@@ -0,0 +1,2 @@
+set stem_method = 3
+set do_indexes = 1
Index: /tags/gsdl-21/gsdl/packages/mg/config.guess
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/config.guess	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/config.guess	(revision 699)
@@ -0,0 +1,470 @@
+#!/bin/sh
+# Attempt to guess a canonical system name.
+#   Copyright (C) 1992, 1993, 1994 Free Software Foundation, Inc.
+#
+# This file is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+# As a special exception to the GNU General Public License, if you
+# distribute this file as part of a program that contains a
+# configuration script generated by Autoconf, you may include it under
+# the same distribution terms that you use for the rest of that program.
+
+# Written by Per Bothner <bothner@cygnus.com>.
+# The master version of this file is at the FSF in /home/gd/gnu/lib.
+#
+# This script attempts to guess a canonical system name similar to
+# config.sub.  If it succeeds, it prints the system name on stdout, and
+# exits with 0.  Otherwise, it exits with 1.
+#
+# The plan is that this can be called by configure scripts if you
+# don't specify an explicit system type (host/target name).
+#
+# Only a few systems have been added to this list; please add others
+# (but try to keep the structure clean).
+#
+
+# This is needed to find uname on a Pyramid OSx when run in the BSD universe.
+# (ghazi@noc.rutgers.edu 8/24/94.)
+if (test -f /.attbin/uname) >/dev/null 2>&1 ; then
+	PATH=$PATH:/.attbin ; export PATH
+fi
+
+UNAME_MACHINE=`(uname -m) 2>/dev/null` || UNAME_MACHINE=unknown
+UNAME_RELEASE=`(uname -r) 2>/dev/null` || UNAME_RELEASE=unknown
+UNAME_SYSTEM=`(uname -s) 2>/dev/null` || UNAME_SYSTEM=unknown
+UNAME_VERSION=`(uname -v) 2>/dev/null` || UNAME_VERSION=unknown
+
+trap 'rm -f dummy.c dummy.o dummy; exit 1' 1 2 15
+
+# Note: order is significant - the case branches are not exclusive.
+
+case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
+    alpha:OSF1:V*:*)
+	# After 1.2, OSF1 uses "V1.3" for uname -r.
+	echo alpha-dec-osf`echo ${UNAME_RELEASE} | sed -e 's/^V//'`
+	exit 0 ;;
+    alpha:OSF1:*:*)
+	# 1.2 uses "1.2" for uname -r.
+	echo alpha-dec-osf${UNAME_RELEASE}
+        exit 0 ;;
+    arm:RISC*:1.[012]*:*|arm:riscix:1.[012]*:*)
+	echo arm-acorn-riscix${UNAME_RELEASE}
+	exit 0;;
+    Pyramid*:OSx*:*:*)
+	if test "`(/bin/universe) 2>/dev/null`" = att ; then
+		echo pyramid-pyramid-sysv3
+	else
+		echo pyramid-pyramid-bsd
+	fi
+	exit 0 ;;
+    sun4*:SunOS:5.*:*)
+	echo sparc-sun-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'`
+	exit 0 ;;
+    sun4*:SunOS:6*:*)
+	# According to config.sub, this is the proper way to canonicalize
+	# SunOS6.  Hard to guess exactly what SunOS6 will be like, but
+	# it's likely to be more like Solaris than SunOS4.
+	echo sparc-sun-solaris3`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'`
+	exit 0 ;;
+    sun4*:SunOS:*:*)
+	# Japanese Language versions have a version number like `4.1.3-JL'.
+	echo sparc-sun-sunos`echo ${UNAME_RELEASE}|sed -e 's/-/_/'`
+	exit 0 ;;
+    sun3*:SunOS:*:*)
+	echo m68k-sun-sunos${UNAME_RELEASE}
+	exit 0 ;;
+    RISC*:ULTRIX:*:*)
+	echo mips-dec-ultrix${UNAME_RELEASE}
+	exit 0 ;;
+    VAX*:ULTRIX*:*:*)
+	echo vax-dec-ultrix${UNAME_RELEASE}
+	exit 0 ;;
+    mips:*:5*:RISCos)
+	echo mips-mips-riscos${UNAME_RELEASE}
+	exit 0 ;;
+    m88k:CX/UX:7*:*)
+	echo m88k-harris-cxux7
+	exit 0 ;;
+    m88k:*:4*:R4*)
+	echo m88k-motorola-sysv4
+	exit 0 ;;
+    m88k:*:3*:R3*)
+	echo m88k-motorola-sysv3
+	exit 0 ;;
+    AViiON:dgux:*:*)
+	if [ ${TARGET_BINARY_INTERFACE}x = m88kdguxelfx \
+	     -o ${TARGET_BINARY_INTERFACE}x = x ] ; then
+		echo m88k-dg-dgux${UNAME_RELEASE}
+	else
+		echo m88k-dg-dguxbcs${UNAME_RELEASE}
+	fi
+ 	exit 0 ;;
+    M88*:DolphinOS:*:*)	# DolphinOS (SVR3)
+	echo m88k-dolphin-sysv3
+	exit 0 ;;
+    M88*:*:R3*:*)
+	# Delta 88k system running SVR3
+	echo m88k-motorola-sysv3
+	exit 0 ;;
+    XD88*:*:*:*) # Tektronix XD88 system running UTekV (SVR3)
+	echo m88k-tektronix-sysv3
+	exit 0 ;;
+    Tek43[0-9][0-9]:UTek:*:*) # Tektronix 4300 system running UTek (BSD)
+	echo m68k-tektronix-bsd
+	exit 0 ;;
+    *:IRIX:*:*)
+	echo mips-sgi-irix${UNAME_RELEASE}
+	exit 0 ;;
+    i[34]86:AIX:*:*)
+	echo i386-ibm-aix
+	exit 0 ;;
+    *:AIX:2:3)
+	if grep bos325 /usr/include/stdio.h >/dev/null 2>&1; then
+		sed 's/^		//' << EOF >dummy.c
+		#include <sys/systemcfg.h>
+
+		main()
+			{
+			if (!__power_pc())
+				exit(1);
+			puts("powerpc-ibm-aix3.2.5");
+			exit(0);
+			}
+EOF
+		${CC-cc} dummy.c -o dummy && ./dummy && rm dummy.c dummy && exit 0
+		rm -f dummy.c dummy
+		echo rs6000-ibm-aix3.2.5
+	elif grep bos324 /usr/include/stdio.h >/dev/null 2>&1; then
+		echo rs6000-ibm-aix3.2.4
+	else
+		echo rs6000-ibm-aix3.2
+	fi
+	exit 0 ;;
+    *:AIX:*:4)
+	if /usr/sbin/lsattr -EHl proc0 | grep POWER >/dev/null 2>&1; then
+		IBM_ARCH=rs6000
+	else
+		IBM_ARCH=powerpc
+	fi
+	if grep bos410 /usr/include/stdio.h >/dev/null 2>&1; then
+		IBM_REV=4.1
+	elif grep bos411 /usr/include/stdio.h >/dev/null 2>&1; then
+		IBM_REV=4.1.1
+	else
+		IBM_REV=4.${UNAME_RELEASE}
+	fi
+	echo ${IBM_ARCH}-ibm-aix${IBM_REV}
+	exit 0 ;;
+    *:AIX:*:*)
+	echo rs6000-ibm-aix
+	exit 0 ;;
+    *:BOSX:*:*)
+	echo rs6000-bull-bosx
+	exit 0 ;;
+    DPX/2?00:B.O.S.:*:*)
+	echo m68k-bull-sysv3
+	exit 0 ;;
+    9000/[34]??:4.3bsd:1.*:*)
+	echo m68k-hp-bsd
+	exit 0 ;;
+    hp300:4.4BSD:*:* | 9000/[34]??:4.3bsd:2.*:*)
+	echo m68k-hp-bsd4.4
+	exit 0 ;;
+    9000/[3478]??:HP-UX:*:*)
+	case "${UNAME_MACHINE}" in
+	    9000/31? )            HP_ARCH=m68000 ;;
+	    9000/[34]?? )         HP_ARCH=m68k ;;
+	    9000/7?? | 9000/8?7 ) HP_ARCH=hppa1.1 ;;
+	    9000/8?? )            HP_ARCH=hppa1.0 ;;
+	esac
+	HPUX_REV=`echo ${UNAME_RELEASE}|sed -e 's/[^.]*.[0B]*//'`
+	echo ${HP_ARCH}-hp-hpux${HPUX_REV}
+	exit 0 ;;
+    3050*:HI-UX:*:*)
+	sed 's/^	//' << EOF >dummy.c
+	#include <unistd.h>
+	int
+	main ()
+	{
+	  long cpu = sysconf (_SC_CPU_VERSION);
+	  /* The order matters, because CPU_IS_HP_MC68K erroneously returns
+	     true for CPU_PA_RISC1_0.  CPU_IS_PA_RISC returns correct
+	     results, however.  */
+	  if (CPU_IS_PA_RISC (cpu))
+	    {
+	      switch (cpu)
+		{
+		  case CPU_PA_RISC1_0: puts ("hppa1.0-hitachi-hiuxwe2"); break;
+		  case CPU_PA_RISC1_1: puts ("hppa1.1-hitachi-hiuxwe2"); break;
+		  case CPU_PA_RISC2_0: puts ("hppa2.0-hitachi-hiuxwe2"); break;
+		  default: puts ("hppa-hitachi-hiuxwe2"); break;
+		}
+	    }
+	  else if (CPU_IS_HP_MC68K (cpu))
+	    puts ("m68k-hitachi-hiuxwe2");
+	  else puts ("unknown-hitachi-hiuxwe2");
+	  exit (0);
+	}
+EOF
+	${CC-cc} dummy.c -o dummy && ./dummy && rm dummy.c dummy && exit 0
+	rm -f dummy.c dummy
+	echo unknown-hitachi-hiuxwe2
+	exit 0 ;;
+    9000/7??:4.3bsd:*:* | 9000/8?7:4.3bsd:*:* )
+	echo hppa1.1-hp-bsd
+	exit 0 ;;
+    9000/8??:4.3bsd:*:*)
+	echo hppa1.0-hp-bsd
+	exit 0 ;;
+    hp7??:OSF1:*:* | hp8?7:OSF1:*:* )
+	echo hppa1.1-hp-osf
+	exit 0 ;;
+    hp8??:OSF1:*:*)
+	echo hppa1.0-hp-osf
+	exit 0 ;;
+    C1*:ConvexOS:*:* | convex:ConvexOS:C1*:*)
+	echo c1-convex-bsd
+        exit 0 ;;
+    C2*:ConvexOS:*:* | convex:ConvexOS:C2*:*)
+	if getsysinfo -f scalar_acc
+	then echo c32-convex-bsd
+	else echo c2-convex-bsd
+	fi
+        exit 0 ;;
+    C34*:ConvexOS:*:* | convex:ConvexOS:C34*:*)
+	echo c34-convex-bsd
+        exit 0 ;;
+    C38*:ConvexOS:*:* | convex:ConvexOS:C38*:*)
+	echo c38-convex-bsd
+        exit 0 ;;
+    C4*:ConvexOS:*:* | convex:ConvexOS:C4*:*)
+	echo c4-convex-bsd
+        exit 0 ;;
+    CRAY*X-MP:UNICOS:*:*)
+	echo xmp-cray-unicos
+        exit 0 ;;
+    CRAY*Y-MP:UNICOS:*:*)
+	echo ymp-cray-unicos
+        exit 0 ;;
+    CRAY-2:UNICOS:*:*)
+	echo cray2-cray-unicos
+        exit 0 ;;
+    hp3[0-9][05]:NetBSD:*:*)
+	echo m68k-hp-netbsd${UNAME_RELEASE}
+	exit 0 ;;
+    i[34]86:BSD/386:*:*)
+	echo ${UNAME_MACHINE}-unknown-bsdi${UNAME_RELEASE}
+	exit 0 ;;
+    *:FreeBSD:*:*)
+	echo ${UNAME_MACHINE}-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`
+	exit 0 ;;
+    *:NetBSD:*:*)
+	echo ${UNAME_MACHINE}-unknown-netbsd`echo ${UNAME_RELEASE}|sed -e 's/[-_].*/\./'`
+	exit 0 ;;
+    *:GNU:*:*)
+	echo `echo ${UNAME_MACHINE}|sed -e 's,/.*$,,'`-unknown-gnu`echo ${UNAME_RELEASE}|sed -e 's,/.*$,,'`
+	exit 0 ;;
+    *:Linux:*:*)
+	echo ${UNAME_MACHINE}-unknown-linux
+	exit 0 ;;
+# ptx 4.0 does uname -s correctly, with DYNIX/ptx in there.  earlier versions
+# are messed up and put the nodename in both sysname and nodename.
+    i[34]86:DYNIX/ptx:4*:*)
+	echo i386-sequent-sysv4
+	exit 0 ;;
+    i[34]86:*:4.*:* | i[34]86:SYSTEM_V:4.*:*)
+	if grep Novell /usr/include/link.h >/dev/null 2>/dev/null; then
+		echo ${UNAME_MACHINE}-univel-sysv${UNAME_RELEASE}
+	else
+		echo ${UNAME_MACHINE}-unknown-sysv${UNAME_RELEASE}
+	fi
+	exit 0 ;;
+    i[34]86:*:3.2:*)
+	if /bin/uname -X 2>/dev/null >/dev/null ; then
+		UNAME_REL=`(/bin/uname -X|egrep Release|sed -e 's/.*= //')`
+		(/bin/uname -X|egrep i80486 >/dev/null) && UNAME_MACHINE=i486
+		echo ${UNAME_MACHINE}-unknown-sco$UNAME_REL
+	elif test -f /usr/options/cb.name; then
+		UNAME_REL=`sed -n 's/.*Version //p' </usr/options/cb.name`
+		echo ${UNAME_MACHINE}-unknown-isc$UNAME_REL
+	else
+		echo ${UNAME_MACHINE}-unknown-sysv32
+	fi
+	exit 0 ;;
+    Intel:Mach:3*:*)
+	echo i386-unknown-mach3
+	exit 0 ;;
+    i860:*:4.*:*) # i860-SVR4
+	if grep Stardent /usr/include/sys/uadmin.h >/dev/null 2>&1 ; then
+	  echo i860-stardent-sysv${UNAME_RELEASE} # Stardent Vistra i860-SVR4
+	else # Add other i860-SVR4 vendors below as they are discovered.
+	  echo i860-unknown-sysv${UNAME_RELEASE}  # Unknown i860-SVR4
+	fi
+	exit 0 ;;
+    mini*:CTIX:SYS*5:*)
+	# "miniframe"
+	echo m68010-convergent-sysv
+	exit 0 ;;
+    M680[234]0:*:R3V[567]*:*)
+	test -r /sysV68 && echo 'm68k-motorola-sysv' && exit 0 ;;
+    3[34]??:*:4.0:3.0 | 3[34]??,*:*:4.0:3.0)
+        uname -p 2>/dev/null | grep 86 >/dev/null \
+          && echo i486-ncr-sysv4.3 && exit 0 ;;
+    3[34]??:*:4.0:* | 3[34]??,*:*:4.0:*)
+        uname -p 2>/dev/null | grep 86 >/dev/null \
+          && echo i486-ncr-sysv4 && exit 0 ;;
+    m680[234]0:LynxOS:2.2*:*)
+	echo m68k-lynx-lynxos${UNAME_RELEASE}
+	exit 0 ;;
+    mc68030:UNIX_System_V:4.*:*)
+	echo m68k-atari-sysv4
+	exit 0 ;;
+    i[34]86:LynxOS:2.2*:*)
+	echo i386-lynx-lynxos${UNAME_RELEASE}
+	exit 0 ;;
+    TSUNAMI:LynxOS:2.2*:*)
+	echo sparc-lynx-lynxos${UNAME_RELEASE}
+	exit 0 ;;
+    rs6000:LynxOS:2.2*:*)
+	echo rs6000-lynx-lynxos${UNAME_RELEASE}
+	exit 0 ;;
+    RM*:SINIX-*:*:*)
+	echo mips-sni-sysv4
+	exit 0 ;;
+    *:SINIX-*:*:*)
+	if uname -p 2>/dev/null >/dev/null ; then
+		UNAME_MACHINE=`(uname -p) 2>/dev/null`
+		echo ${UNAME_MACHINE}-sni-sysv4
+	else
+		echo ns32k-sni-sysv
+	fi
+	exit 0 ;;
+esac
+
+#echo '(No uname command or uname output not recognized.)' 1>&2
+#echo "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" 1>&2
+
+cat >dummy.c <<EOF
+main ()
+{
+#if defined (sony)
+#if defined (MIPSEB)
+  /* BFD wants "bsd" instead of "newsos".  Perhaps BFD should be changed,
+     I don't know....  */
+  printf ("mips-sony-bsd\n"); exit (0);
+#else
+  printf ("m68k-sony-newsos\n"); exit (0);
+#endif
+#endif
+
+#if defined (__arm) && defined (__acorn) && defined (__unix)
+  printf ("arm-acorn-riscix"); exit (0);
+#endif
+
+#if defined (hp300) && !defined (hpux)
+  printf ("m68k-hp-bsd\n"); exit (0);
+#endif
+
+#if defined (NeXT)
+#if !defined (__ARCHITECTURE__)
+#define __ARCHITECTURE__ "m68k"
+#endif
+  int version;
+  version=`(hostinfo | sed -n 's/.*NeXT Mach \([0-9]*\).*/\1/p') 2>/dev/null`;
+  printf ("%s-next-nextstep%s\n", __ARCHITECTURE__,  version==2 ? "2" : "3");
+  exit (0);
+#endif
+
+#if defined (MULTIMAX) || defined (n16)
+#if defined (UMAXV)
+  printf ("ns32k-encore-sysv\n"); exit (0);
+#else
+#if defined (CMU)
+  printf ("ns32k-encore-mach\n"); exit (0);
+#else
+  printf ("ns32k-encore-bsd\n"); exit (0);
+#endif
+#endif
+#endif
+
+#if defined (__386BSD__)
+  printf ("i386-unknown-bsd\n"); exit (0);
+#endif
+
+#if defined (sequent)
+#if defined (i386)
+  printf ("i386-sequent-dynix\n"); exit (0);
+#endif
+#if defined (ns32000)
+  printf ("ns32k-sequent-dynix\n"); exit (0);
+#endif
+#endif
+
+#if defined (_SEQUENT_)
+  printf ("i386-sequent-ptx\n"); exit (0);
+#endif
+
+#if defined (vax)
+#if !defined (ultrix)
+  printf ("vax-dec-bsd\n"); exit (0);
+#else
+  printf ("vax-dec-ultrix\n"); exit (0);
+#endif
+#endif
+
+#if defined (alliant) && defined (i860)
+  printf ("i860-alliant-bsd\n"); exit (0);
+#endif
+
+  exit (1);
+}
+EOF
+
+${CC-cc} dummy.c -o dummy 2>/dev/null && ./dummy && rm dummy.c dummy && exit 0
+rm -f dummy.c dummy
+
+# Apollos put the system type in the environment.
+
+test -d /usr/apollo && { echo ${ISP}-apollo-${SYSTYPE}; exit 0; }
+
+# Convex versions that predate uname can use getsysinfo(1)
+
+if [ -x /usr/convex/getsysinfo ]
+then
+    case `getsysinfo -f cpu_type` in
+    c1*)
+	echo c1-convex-bsd
+	exit 0 ;;
+    c2*)
+	if getsysinfo -f scalar_acc
+	then echo c32-convex-bsd
+	else echo c2-convex-bsd
+	fi
+	exit 0 ;;
+    c34*)
+	echo c34-convex-bsd
+	exit 0 ;;
+    c38*)
+	echo c38-convex-bsd
+	exit 0 ;;
+    c4*)
+	echo c4-convex-bsd
+	exit 0 ;;
+    esac
+fi
+
+#echo '(Unable to guess system type)' 1>&2
+
+exit 1
Index: /tags/gsdl-21/gsdl/packages/mg/config.h.in
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/config.h.in	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/config.h.in	(revision 699)
@@ -0,0 +1,171 @@
+/* config.h.in.  Generated automatically from configure.in by autoheader.  */
+
+/* Define if on AIX 3.
+   System headers sometimes define this.
+   We just want to avoid a redefinition error message.  */
+#ifndef _ALL_SOURCE
+#undef _ALL_SOURCE
+#endif
+
+/* Define if using alloca.c.  */
+#undef C_ALLOCA
+
+/* Define to empty if the keyword does not work.  */
+#undef const
+
+/* Define to one of _getb67, GETB67, getb67 for Cray-2 and Cray-YMP systems.
+   This function is required for alloca.c support on those systems.  */
+#undef CRAY_STACKSEG_END
+
+/* Define if you have alloca, as a function or macro.  */
+#undef HAVE_ALLOCA
+
+/* Define if you have <alloca.h> and it should be used (not on Ultrix).  */
+#undef HAVE_ALLOCA_H
+
+/* Define if you don't have vprintf but do have _doprnt.  */
+#undef HAVE_DOPRNT
+
+/* Define if you have the vprintf function.  */
+#undef HAVE_VPRINTF
+
+/* Define if on MINIX.  */
+#undef _MINIX
+
+/* Define to `long' if <sys/types.h> doesn't define.  */
+#undef off_t
+
+/* Define if the system does not provide POSIX.1 features except
+   with this defined.  */
+#undef _POSIX_1_SOURCE
+
+/* Define if you need to in order for stat and other things to work.  */
+#undef _POSIX_SOURCE
+
+/* Define as the return type of signal handlers (int or void).  */
+#undef RETSIGTYPE
+
+/* Define to `unsigned' if <sys/types.h> doesn't define.  */
+#undef size_t
+
+/* If using the C implementation of alloca, define if you know the
+   direction of stack growth for your system; otherwise it will be
+   automatically deduced at run-time.
+ STACK_DIRECTION > 0 => grows toward higher addresses
+ STACK_DIRECTION < 0 => grows toward lower addresses
+ STACK_DIRECTION = 0 => direction of growth unknown
+ */
+#undef STACK_DIRECTION
+
+/* Define if you have the ANSI C header files.  */
+#undef STDC_HEADERS
+
+/* Define if you can safely include both <sys/time.h> and <time.h>.  */
+#undef TIME_WITH_SYS_TIME
+
+/* Define if your <sys/time.h> declares struct tm.  */
+#undef TM_IN_SYS_TIME
+
+/* Define to the name of the distribution.  */
+#undef PACKAGE
+
+/* Define to the version of the distribution.  */
+#undef VERSION
+
+/* Define to 1 if you have the valloc function.  */
+#undef HAVE_VALLOC
+
+/* Define to 1 if ANSI function prototypes are usable.  */
+#undef PROTOTYPES
+
+ 
+/* Define to 1 for better use of the debugging malloc library.  See
+   site ftp.antaire.com in antaire/src, file dmalloc/dmalloc.tar.gz.  */
+#undef WITH_DMALLOC
+
+ 
+/* Define to 1 if GNU regex should be used instead of GNU rx.  */
+#undef WITH_REGEX
+
+/* Some braindead header files do not have the requ'd function declarations */
+#undef HAVE_FGETC_DECL
+#undef HAVE_FREAD_DECL
+
+/* See if have this field in the prstatus_t structure */
+/* It stores the size of the process heap */
+#undef HAVE_PR_BRKSIZE
+
+/* Define if you have the ftime function.  */
+#undef HAVE_FTIME
+
+/* Define if you have the ftruncate function.  */
+#undef HAVE_FTRUNCATE
+
+/* Define if you have the getpagesize function.  */
+#undef HAVE_GETPAGESIZE
+
+/* Define if you have the getrusage function.  */
+#undef HAVE_GETRUSAGE
+
+/* Define if you have the mallinfo function.  */
+#undef HAVE_MALLINFO
+
+/* Define if you have the select function.  */
+#undef HAVE_SELECT
+
+/* Define if you have the setbuffer function.  */
+#undef HAVE_SETBUFFER
+
+/* Define if you have the strcasecmp function.  */
+#undef HAVE_STRCASECMP
+
+/* Define if you have the strftime function.  */
+#undef HAVE_STRFTIME
+
+/* Define if you have the strstr function.  */
+#undef HAVE_STRSTR
+
+/* Define if you have the strtol function.  */
+#undef HAVE_STRTOL
+
+/* Define if you have the times function.  */
+#undef HAVE_TIMES
+
+/* Define if you have the valloc function.  */
+#undef HAVE_VALLOC
+
+/* Define if you have the <dirent.h> header file.  */
+#undef HAVE_DIRENT_H
+
+/* Define if you have the <fcntl.h> header file.  */
+#undef HAVE_FCNTL_H
+
+/* Define if you have the <limits.h> header file.  */
+#undef HAVE_LIMITS_H
+
+/* Define if you have the <memory.h> header file.  */
+#undef HAVE_MEMORY_H
+
+/* Define if you have the <ndir.h> header file.  */
+#undef HAVE_NDIR_H
+
+/* Define if you have the <string.h> header file.  */
+#undef HAVE_STRING_H
+
+/* Define if you have the <sys/dir.h> header file.  */
+#undef HAVE_SYS_DIR_H
+
+/* Define if you have the <sys/ndir.h> header file.  */
+#undef HAVE_SYS_NDIR_H
+
+/* Define if you have the <sys/procfs.h> header file.  */
+#undef HAVE_SYS_PROCFS_H
+
+/* Define if you have the <sys/time.h> header file.  */
+#undef HAVE_SYS_TIME_H
+
+/* Define if you have the <unistd.h> header file.  */
+#undef HAVE_UNISTD_H
+
+/* Define if you have the m library (-lm).  */
+#undef HAVE_LIBM
Index: /tags/gsdl-21/gsdl/packages/mg/config.sub
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/config.sub	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/config.sub	(revision 699)
@@ -0,0 +1,793 @@
+#!/bin/sh
+# Configuration validation subroutine script, version 1.1.
+#   Copyright (C) 1991, 1992, 1993, 1994 Free Software Foundation, Inc.
+# This file is (in principle) common to ALL GNU software.
+# The presence of a machine in this file suggests that SOME GNU software
+# can handle that machine.  It does not imply ALL GNU software can. 
+#
+# This file is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+# As a special exception to the GNU General Public License, if you
+# distribute this file as part of a program that contains a
+# configuration script generated by Autoconf, you may include it under
+# the same distribution terms that you use for the rest of that program.
+
+# Configuration subroutine to validate and canonicalize a configuration type.
+# Supply the specified configuration type as an argument.
+# If it is invalid, we print an error message on stderr and exit with code 1.
+# Otherwise, we print the canonical config type on stdout and succeed.
+
+# This file is supposed to be the same for all GNU packages
+# and recognize all the CPU types, system types and aliases
+# that are meaningful with *any* GNU software.
+# Each package is responsible for reporting which valid configurations
+# it does not support.  The user should be able to distinguish
+# a failure to support a valid configuration from a meaningless
+# configuration.
+
+# The goal of this file is to map all the various variations of a given
+# machine specification into a single specification in the form:
+#	CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM
+# It is wrong to echo any other type of specification.
+
+# First pass through any local machine types.
+case $1 in
+	*local*)
+		echo $1
+		exit 0
+		;;
+	*)
+	;;
+esac
+
+# Separate what the user gave into CPU-COMPANY and OS (if any).
+basic_machine=`echo $1 | sed 's/-[^-]*$//'`
+if [ $basic_machine != $1 ]
+then os=`echo $1 | sed 's/.*-/-/'`
+else os=; fi
+
+### Let's recognize common machines as not being operating systems so
+### that things like config.sub decstation-3100 work.  We also
+### recognize some manufacturers as not being operating systems, so we
+### can provide default operating systems below.
+case $os in
+	-sun*os*)
+		# Prevent following clause from handling this invalid input.
+		;;
+	-dec* | -mips* | -sequent* | -encore* | -pc532* | -sgi* | -sony* | \
+	-att* | -7300* | -3300* | -delta* | -motorola* | -sun[234]* | \
+	-unicom* | -ibm* | -next | -hp | -isi* | -apollo | -altos* | \
+	-convergent* | -ncr* | -news | -32* | -3600* | -3100* | -hitachi* |\
+	-c[123]* | -convex* | -sun | -crds | -omron* | -dg | -ultra | -tti* | \
+	-harris | -dolphin | -highlevel | -gould | -cbm | -ns | -masscomp )
+		os=
+		basic_machine=$1
+		;;
+	-hiux*)
+		os=-hiuxwe2
+		;;
+	-sco4)
+		os=-sco3.2v4
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-unknown/'`
+		;;
+	-sco3.2.[4-9]*)
+		os=`echo $os | sed -e 's/sco3.2./sco3.2v/'`
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-unknown/'`
+		;;
+	-sco3.2v[4-9]*)
+		# Don't forget version if it is 3.2v4 or newer.
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-unknown/'`
+		;;
+	-sco*)
+		os=-sco3.2v2
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-unknown/'`
+		;;
+	-isc)
+		os=-isc2.2
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-unknown/'`
+		;;
+	-clix*)
+		basic_machine=clipper-intergraph
+		;;
+	-isc*)
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-unknown/'`
+		;;
+	-lynx)
+		os=-lynxos
+		;;
+	-ptx*)
+		basic_machine=`echo $1 | sed -e 's/86-.*/86-sequent/'`
+		;;
+	-windowsnt*)
+		os=`echo $os | sed -e 's/windowsnt/winnt/'`
+		;;
+esac
+
+# Decode aliases for certain CPU-COMPANY combinations.
+case $basic_machine in
+	# Recognize the basic CPU types without company name.
+	# Some are omitted here because they have special meanings below.
+	tahoe | i[345]86 | i860 | m68k | m68000 | m88k | ns32k | arm | pyramid \
+		| tron | a29k | 580 | i960 | h8300 | hppa1.0 | hppa1.1 \
+		| alpha | we32k | ns16k | clipper | sparclite | i370 | sh \
+		| powerpc | sparc64 | 1750a | dsp16xx | mips64 | mipsel \
+		| pdp11 | mips64el | mips64orion | mips64orionel )
+		basic_machine=$basic_machine-unknown
+		;;
+	# Object if more than one company name word.
+	*-*-*)
+		echo Invalid configuration \`$1\': machine \`$basic_machine\' not recognized 1>&2
+		exit 1
+		;;
+	# Recognize the basic CPU types with company name.
+	vax-* | tahoe-* | i[345]86-* | i860-* | m68k-* | m68000-* | m88k-* \
+	      | sparc-* | ns32k-* | fx80-* | arm-* | c[123]* \
+	      | mips-* | pyramid-* | tron-* | a29k-* | romp-* | rs6000-* \
+	      | none-* | 580-* | cray2-* | h8300-* | i960-* | xmp-* | ymp-* \
+	      | hppa1.0-* | hppa1.1-* | alpha-* | we32k-* | cydra-* | ns16k-* \
+	      | pn-* | np1-* | xps100-* | clipper-* | orion-* | sparclite-* \
+	      | pdp11-* | sh-* | powerpc-* | sparc64-* | mips64-* | mipsel-* \
+	      | mips64el-* | mips64orion-* | mips64orionel-* )
+		;;
+	# Recognize the various machine names and aliases which stand
+	# for a CPU type and a company and sometimes even an OS.
+	3b1 | 7300 | 7300-att | att-7300 | pc7300 | safari | unixpc)
+		basic_machine=m68000-att
+		;;
+	3b*)
+		basic_machine=we32k-att
+		;;
+	alliant | fx80)
+		basic_machine=fx80-alliant
+		;;
+	altos | altos3068)
+		basic_machine=m68k-altos
+		;;
+	am29k)
+		basic_machine=a29k-none
+		os=-bsd
+		;;
+	amdahl)
+		basic_machine=580-amdahl
+		os=-sysv
+		;;
+	amiga | amiga-*)
+		basic_machine=m68k-cbm
+		;;
+	amigados)
+		basic_machine=m68k-cbm
+		os=-amigados
+		;;
+	amigaunix | amix)
+		basic_machine=m68k-cbm
+		os=-sysv4
+		;;
+	apollo68)
+		basic_machine=m68k-apollo
+		os=-sysv
+		;;
+	balance)
+		basic_machine=ns32k-sequent
+		os=-dynix
+		;;
+	convex-c1)
+		basic_machine=c1-convex
+		os=-bsd
+		;;
+	convex-c2)
+		basic_machine=c2-convex
+		os=-bsd
+		;;
+	convex-c32)
+		basic_machine=c32-convex
+		os=-bsd
+		;;
+	convex-c34)
+		basic_machine=c34-convex
+		os=-bsd
+		;;
+	convex-c38)
+		basic_machine=c38-convex
+		os=-bsd
+		;;
+	cray | ymp)
+		basic_machine=ymp-cray
+		os=-unicos
+		;;
+	cray2)
+		basic_machine=cray2-cray
+		os=-unicos
+		;;
+	crds | unos)
+		basic_machine=m68k-crds
+		;;
+	da30 | da30-*)
+		basic_machine=m68k-da30
+		;;
+	decstation | decstation-3100 | pmax | pmax-* | pmin | dec3100 | decstatn)
+		basic_machine=mips-dec
+		;;
+	delta | 3300 | motorola-3300 | motorola-delta \
+	      | 3300-motorola | delta-motorola)
+		basic_machine=m68k-motorola
+		;;
+	delta88)
+		basic_machine=m88k-motorola
+		os=-sysv3
+		;;
+	dpx20 | dpx20-*)
+		basic_machine=rs6000-bull
+		os=-bosx
+		;;
+	dpx2* | dpx2*-bull)
+		basic_machine=m68k-bull
+		os=-sysv3
+		;;
+	ebmon29k)
+		basic_machine=a29k-amd
+		os=-ebmon
+		;;
+	elxsi)
+		basic_machine=elxsi-elxsi
+		os=-bsd
+		;;
+	encore | umax | mmax)
+		basic_machine=ns32k-encore
+		;;
+	fx2800)
+		basic_machine=i860-alliant
+		;;
+	genix)
+		basic_machine=ns32k-ns
+		;;
+	gmicro)
+		basic_machine=tron-gmicro
+		os=-sysv
+		;;
+	h3050r* | hiux*)
+		basic_machine=hppa1.1-hitachi
+		os=-hiuxwe2
+		;;
+	h8300hms)
+		basic_machine=h8300-hitachi
+		os=-hms
+		;;
+	harris)
+		basic_machine=m88k-harris
+		os=-sysv3
+		;;
+	hp300-*)
+		basic_machine=m68k-hp
+		;;
+	hp300bsd)
+		basic_machine=m68k-hp
+		os=-bsd
+		;;
+	hp300hpux)
+		basic_machine=m68k-hp
+		os=-hpux
+		;;
+	hp9k2[0-9][0-9] | hp9k31[0-9])
+		basic_machine=m68000-hp
+		;;
+	hp9k3[2-9][0-9])
+		basic_machine=m68k-hp
+		;;
+	hp9k7[0-9][0-9] | hp7[0-9][0-9] | hp9k8[0-9]7 | hp8[0-9]7)
+		basic_machine=hppa1.1-hp
+		;;
+	hp9k8[0-9][0-9] | hp8[0-9][0-9])
+		basic_machine=hppa1.0-hp
+		;;
+	i370-ibm* | ibm*)
+		basic_machine=i370-ibm
+		os=-mvs
+		;;
+# I'm not sure what "Sysv32" means.  Should this be sysv3.2?
+	i[345]86v32)
+		basic_machine=`echo $1 | sed -e 's/86.*/86-unknown/'`
+		os=-sysv32
+		;;
+	i[345]86v4*)
+		basic_machine=`echo $1 | sed -e 's/86.*/86-unknown/'`
+		os=-sysv4
+		;;
+	i[345]86v)
+		basic_machine=`echo $1 | sed -e 's/86.*/86-unknown/'`
+		os=-sysv
+		;;
+	i[345]86sol2)
+		basic_machine=`echo $1 | sed -e 's/86.*/86-unknown/'`
+		os=-solaris2
+		;;
+	iris | iris4d)
+		basic_machine=mips-sgi
+		case $os in
+		    -irix*)
+			;;
+		    *)
+			os=-irix4
+			;;
+		esac
+		;;
+	isi68 | isi)
+		basic_machine=m68k-isi
+		os=-sysv
+		;;
+	m88k-omron*)
+		basic_machine=m88k-omron
+		;;
+	magnum | m3230)
+		basic_machine=mips-mips
+		os=-sysv
+		;;
+	merlin)
+		basic_machine=ns32k-utek
+		os=-sysv
+		;;
+	miniframe)
+		basic_machine=m68000-convergent
+		;;
+	mips3*-*)
+		basic_machine=`echo $basic_machine | sed -e 's/mips3/mips64/'`
+		;;
+	mips3*)
+		basic_machine=`echo $basic_machine | sed -e 's/mips3/mips64/'`-unknown
+		;;
+	ncr3000)
+		basic_machine=i486-ncr
+		os=-sysv4
+		;;
+	news | news700 | news800 | news900)
+		basic_machine=m68k-sony
+		os=-newsos
+		;;
+	news1000)
+		basic_machine=m68030-sony
+		os=-newsos
+		;;
+	news-3600 | risc-news)
+		basic_machine=mips-sony
+		os=-newsos
+		;;
+	next | m*-next )
+		basic_machine=m68k-next
+		case $os in
+		    -nextstep* )
+			;;
+		    -ns2*)
+		      os=-nextstep2
+			;;
+		    *)
+		      os=-nextstep3
+			;;
+		esac
+		;;
+	nh3000)
+		basic_machine=m68k-harris
+		os=-cxux
+		;;
+	nh[45]000)
+		basic_machine=m88k-harris
+		os=-cxux
+		;;
+	nindy960)
+		basic_machine=i960-intel
+		os=-nindy
+		;;
+	np1)
+		basic_machine=np1-gould
+		;;
+	pa-hitachi)
+		basic_machine=hppa1.1-hitachi
+		os=-hiuxwe2
+		;;
+	paragon)
+		basic_machine=i860-intel
+		os=-osf
+		;;
+	pbd)
+		basic_machine=sparc-tti
+		;;
+	pbb)
+		basic_machine=m68k-tti
+		;;
+        pc532 | pc532-*)
+		basic_machine=ns32k-pc532
+		;;
+	pentium-*)
+		# We will change tis to say i586 once there has been
+		# time for various packages to start to recognize that.
+		basic_machine=i486-`echo $basic_machine | sed 's/^[^-]*-//'`
+		;;
+	pn)
+		basic_machine=pn-gould
+		;;
+	ps2)
+		basic_machine=i386-ibm
+		;;
+	rtpc | rtpc-*)
+		basic_machine=romp-ibm
+		;;
+	sequent)
+		basic_machine=i386-sequent
+		;;
+	sh)
+		basic_machine=sh-hitachi
+		os=-hms
+		;;
+	sps7)
+		basic_machine=m68k-bull
+		os=-sysv2
+		;;
+	spur)
+		basic_machine=spur-unknown
+		;;
+	sun2)
+		basic_machine=m68000-sun
+		;;
+	sun2os3)
+		basic_machine=m68000-sun
+		os=-sunos3
+		;;
+	sun2os4)
+		basic_machine=m68000-sun
+		os=-sunos4
+		;;
+	sun3os3)
+		basic_machine=m68k-sun
+		os=-sunos3
+		;;
+	sun3os4)
+		basic_machine=m68k-sun
+		os=-sunos4
+		;;
+	sun4os3)
+		basic_machine=sparc-sun
+		os=-sunos3
+		;;
+	sun4os4)
+		basic_machine=sparc-sun
+		os=-sunos4
+		;;
+	sun3 | sun3-*)
+		basic_machine=m68k-sun
+		;;
+	sun4)
+		basic_machine=sparc-sun
+		;;
+	sun386 | sun386i | roadrunner)
+		basic_machine=i386-sun
+		;;
+	symmetry)
+		basic_machine=i386-sequent
+		os=-dynix
+		;;
+	tower | tower-32)
+		basic_machine=m68k-ncr
+		;;
+	ultra3)
+		basic_machine=a29k-nyu
+		os=-sym1
+		;;
+	vaxv)
+		basic_machine=vax-dec
+		os=-sysv
+		;;
+	vms)
+		basic_machine=vax-dec
+		os=-vms
+		;;
+	vxworks960)
+		basic_machine=i960-wrs
+		os=-vxworks
+		;;
+	vxworks68)
+		basic_machine=m68k-wrs
+		os=-vxworks
+		;;
+	xmp)
+		basic_machine=xmp-cray
+		os=-unicos
+		;;
+        xps | xps100)
+		basic_machine=xps100-honeywell
+		;;
+	none)
+		basic_machine=none-none
+		os=-none
+		;;
+
+# Here we handle the default manufacturer of certain CPU types.  It is in
+# some cases the only manufacturer, in others, it is the most popular.
+	mips)
+		basic_machine=mips-mips
+		;;
+	romp)
+		basic_machine=romp-ibm
+		;;
+	rs6000)
+		basic_machine=rs6000-ibm
+		;;
+	vax)
+		basic_machine=vax-dec
+		;;
+	pdp11)
+		basic_machine=pdp11-dec
+		;;
+	we32k)
+		basic_machine=we32k-att
+		;;
+	sparc)
+		basic_machine=sparc-sun
+		;;
+        cydra)
+		basic_machine=cydra-cydrome
+		;;
+	orion)
+		basic_machine=orion-highlevel
+		;;
+	orion105)
+		basic_machine=clipper-highlevel
+		;;
+	*)
+		echo Invalid configuration \`$1\': machine \`$basic_machine\' not recognized 1>&2
+		exit 1
+		;;
+esac
+
+# Here we canonicalize certain aliases for manufacturers.
+case $basic_machine in
+	*-digital*)
+		basic_machine=`echo $basic_machine | sed 's/digital.*/dec/'`
+		;;
+	*-commodore*)
+		basic_machine=`echo $basic_machine | sed 's/commodore.*/cbm/'`
+		;;
+	*)
+		;;
+esac
+
+# Decode manufacturer-specific aliases for certain operating systems.
+
+if [ x"$os" != x"" ]
+then
+case $os in
+	# -solaris* is a basic system type, with this one exception.
+	-solaris1 | -solaris1.*)
+		os=`echo $os | sed -e 's|solaris1|sunos4|'`
+		;;
+	-solaris)
+		os=-solaris2
+		;;
+	-gnu/linux*)
+		os=`echo $os | sed -e 's|gnu/linux|linux|'`
+		;;
+	# First accept the basic system types.
+	# The portable systems comes first.
+	# Each alternative must end in a *, to match a version number.
+	# -sysv* is not here because it comes later, after sysvr4.
+	-gnu* | -bsd* | -mach* | -minix* | -genix* | -ultrix* | -irix* \
+	      | -vms* | -sco* | -esix* | -isc* | -aix* | -sunos | -sunos[345]* \
+	      | -hpux* | -unos* | -osf* | -luna* | -dgux* | -solaris* | -sym* \
+	      | -amigados* | -msdos* | -newsos* | -unicos* | -aos* \
+	      | -nindy* | -vxworks* | -ebmon* | -hms* | -mvs* | -clix* \
+	      | -riscos* | -linux* | -uniplus* | -iris* | -rtu* | -xenix* \
+	      | -hiux* | -386bsd* | -netbsd* | -freebsd* | -riscix* \
+	      | -lynxos* | -bosx* | -nextstep* | -cxux* | -aout* | -elf* \
+	      | -ptx* | -coff* | -winnt*)
+		;;
+	-sunos5*)
+		os=`echo $os | sed -e 's|sunos5|solaris2|'`
+		;;
+	-sunos6*)
+		os=`echo $os | sed -e 's|sunos6|solaris3|'`
+		;;
+	-osfrose*)
+		os=-osfrose
+		;;
+	-osf*)
+		os=-osf
+		;;
+	-utek*)
+		os=-bsd
+		;;
+	-dynix*)
+		os=-bsd
+		;;
+	-acis*)
+		os=-aos
+		;;
+	-ctix* | -uts*)
+		os=-sysv
+		;;
+	-triton*)
+		os=-sysv3
+		;;
+	-oss*)
+		os=-sysv3
+		;;
+	-svr4)
+		os=-sysv4
+		;;
+	-svr3)
+		os=-sysv3
+		;;
+	-sysvr4)
+		os=-sysv4
+		;;
+	# This must come after -sysvr4.
+	-sysv*)
+		;;
+	-xenix)
+		os=-xenix
+		;;
+	-none)
+		;;
+	*)
+		# Get rid of the `-' at the beginning of $os.
+		os=`echo $os | sed 's/[^-]*-//'`
+		echo Invalid configuration \`$1\': system \`$os\' not recognized 1>&2
+		exit 1
+		;;
+esac
+else
+
+# Here we handle the default operating systems that come with various machines.
+# The value should be what the vendor currently ships out the door with their
+# machine or put another way, the most popular os provided with the machine.
+
+# Note that if you're going to try to match "-MANUFACTURER" here (say,
+# "-sun"), then you have to tell the case statement up towards the top
+# that MANUFACTURER isn't an operating system.  Otherwise, code above
+# will signal an error saying that MANUFACTURER isn't an operating
+# system, and we'll never get to this point.
+
+case $basic_machine in
+	*-acorn)
+		os=-riscix1.2
+		;;
+        pdp11-*)
+		os=-none
+		;;
+	*-dec | vax-*)
+		os=-ultrix4.2
+		;;
+	i386-sun)
+		os=-sunos4.0.2
+		;;
+	m68000-sun)
+		os=-sunos3
+		# This also exists in the configure program, but was not the
+		# default.
+		# os=-sunos4
+		;;
+	*-tti)	# must be before sparc entry or we get the wrong os.
+		os=-sysv3
+		;;
+	sparc-* | *-sun)
+		os=-sunos4.1.1
+		;;
+	*-ibm)
+		os=-aix
+		;;
+	*-hp)
+		os=-hpux
+		;;
+	*-hitachi)
+		os=-hiux
+		;;
+	i860-* | *-att | *-ncr | *-altos | *-motorola | *-convergent)
+		os=-sysv
+		;;
+	*-cbm)
+		os=-amigados
+		;;
+	*-dg)
+		os=-dgux
+		;;
+	*-dolphin)
+		os=-sysv3
+		;;
+	m68k-ccur)
+		os=-rtu
+		;;
+	m88k-omron*)
+		os=-luna
+		;;
+	*-sequent)
+		os=-ptx
+		;;
+	*-crds)
+		os=-unos
+		;;
+	*-ns)
+		os=-genix
+		;;
+	i370-*)
+		os=-mvs
+		;;
+	*-next)
+		os=-nextstep3
+		;;
+        *-gould)
+		os=-sysv
+		;;
+        *-highlevel)
+		os=-bsd
+		;;
+	*-encore)
+		os=-bsd
+		;;
+        *-sgi)
+		os=-irix
+		;;
+	*-masscomp)
+		os=-rtu
+		;;
+	*)
+		os=-none
+		;;
+esac
+fi
+
+# Here we handle the case where we know the os, and the CPU type, but not the
+# manufacturer.  We pick the logical manufacturer.
+vendor=unknown
+case $basic_machine in
+	*-unknown)
+		case $os in
+			-riscix*)
+				vendor=acorn
+				;;
+			-sunos*)
+				vendor=sun
+				;;
+			-lynxos*)
+				vendor=lynx
+				;;
+			-aix*)
+				vendor=ibm
+				;;
+			-hpux*)
+				vendor=hp
+				;;
+			-hiux*)
+				vendor=hitachi
+				;;
+			-unos*)
+				vendor=crds
+				;;
+			-dgux*)
+				vendor=dg
+				;;
+			-luna*)
+				vendor=omron
+				;;
+			-genix*)
+				vendor=ns
+				;;
+			-mvs*)
+				vendor=ibm
+				;;
+			-ptx*)
+				vendor=sequent
+				;;
+		esac
+		basic_machine=`echo $basic_machine | sed "s/unknown/$vendor/"`
+		;;
+esac
+
+echo $basic_machine$os
Index: /tags/gsdl-21/gsdl/packages/mg/configure
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/configure	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/configure	(revision 699)
@@ -0,0 +1,3073 @@
+#! /bin/sh
+
+# Guess values for system-dependent variables and create Makefiles.
+# Generated automatically using autoconf version 2.12 
+# Copyright (C) 1992, 93, 94, 95, 96 Free Software Foundation, Inc.
+#
+# This configure script is free software; the Free Software Foundation
+# gives unlimited permission to copy, distribute and modify it.
+
+# Defaults:
+ac_help=
+ac_default_prefix=/usr/local
+# Any additions from configure.in:
+ac_help="$ac_help
+  --with-dmalloc          use dmalloc, as in
+                          ftp://ftp.letters.com/src/dmalloc/dmalloc.tar.gz"
+ac_help="$ac_help
+  --with-regex            use older regex in lieu of GNU rx for matching"
+
+# Initialize some variables set by options.
+# The variables have the same names as the options, with
+# dashes changed to underlines.
+build=NONE
+cache_file=./config.cache
+exec_prefix=NONE
+host=NONE
+no_create=
+nonopt=NONE
+no_recursion=
+prefix=NONE
+program_prefix=NONE
+program_suffix=NONE
+program_transform_name=s,x,x,
+silent=
+site=
+srcdir=
+target=NONE
+verbose=
+x_includes=NONE
+x_libraries=NONE
+bindir='${exec_prefix}/bin'
+sbindir='${exec_prefix}/sbin'
+libexecdir='${exec_prefix}/libexec'
+datadir='${prefix}/share'
+sysconfdir='${prefix}/etc'
+sharedstatedir='${prefix}/com'
+localstatedir='${prefix}/var'
+libdir='${exec_prefix}/lib'
+includedir='${prefix}/include'
+oldincludedir='/usr/include'
+infodir='${prefix}/info'
+mandir='${prefix}/man'
+
+# Initialize some other variables.
+subdirs=
+MFLAGS= MAKEFLAGS=
+# Maximum number of lines to put in a shell here document.
+ac_max_here_lines=12
+
+ac_prev=
+for ac_option
+do
+
+  # If the previous option needs an argument, assign it.
+  if test -n "$ac_prev"; then
+    eval "$ac_prev=\$ac_option"
+    ac_prev=
+    continue
+  fi
+
+  case "$ac_option" in
+  -*=*) ac_optarg=`echo "$ac_option" | sed 's/[-_a-zA-Z0-9]*=//'` ;;
+  *) ac_optarg= ;;
+  esac
+
+  # Accept the important Cygnus configure options, so we can diagnose typos.
+
+  case "$ac_option" in
+
+  -bindir | --bindir | --bindi | --bind | --bin | --bi)
+    ac_prev=bindir ;;
+  -bindir=* | --bindir=* | --bindi=* | --bind=* | --bin=* | --bi=*)
+    bindir="$ac_optarg" ;;
+
+  -build | --build | --buil | --bui | --bu)
+    ac_prev=build ;;
+  -build=* | --build=* | --buil=* | --bui=* | --bu=*)
+    build="$ac_optarg" ;;
+
+  -cache-file | --cache-file | --cache-fil | --cache-fi \
+  | --cache-f | --cache- | --cache | --cach | --cac | --ca | --c)
+    ac_prev=cache_file ;;
+  -cache-file=* | --cache-file=* | --cache-fil=* | --cache-fi=* \
+  | --cache-f=* | --cache-=* | --cache=* | --cach=* | --cac=* | --ca=* | --c=*)
+    cache_file="$ac_optarg" ;;
+
+  -datadir | --datadir | --datadi | --datad | --data | --dat | --da)
+    ac_prev=datadir ;;
+  -datadir=* | --datadir=* | --datadi=* | --datad=* | --data=* | --dat=* \
+  | --da=*)
+    datadir="$ac_optarg" ;;
+
+  -disable-* | --disable-*)
+    ac_feature=`echo $ac_option|sed -e 's/-*disable-//'`
+    # Reject names that are not valid shell variable names.
+    if test -n "`echo $ac_feature| sed 's/[-a-zA-Z0-9_]//g'`"; then
+      { echo "configure: error: $ac_feature: invalid feature name" 1>&2; exit 1; }
+    fi
+    ac_feature=`echo $ac_feature| sed 's/-/_/g'`
+    eval "enable_${ac_feature}=no" ;;
+
+  -enable-* | --enable-*)
+    ac_feature=`echo $ac_option|sed -e 's/-*enable-//' -e 's/=.*//'`
+    # Reject names that are not valid shell variable names.
+    if test -n "`echo $ac_feature| sed 's/[-_a-zA-Z0-9]//g'`"; then
+      { echo "configure: error: $ac_feature: invalid feature name" 1>&2; exit 1; }
+    fi
+    ac_feature=`echo $ac_feature| sed 's/-/_/g'`
+    case "$ac_option" in
+      *=*) ;;
+      *) ac_optarg=yes ;;
+    esac
+    eval "enable_${ac_feature}='$ac_optarg'" ;;
+
+  -exec-prefix | --exec_prefix | --exec-prefix | --exec-prefi \
+  | --exec-pref | --exec-pre | --exec-pr | --exec-p | --exec- \
+  | --exec | --exe | --ex)
+    ac_prev=exec_prefix ;;
+  -exec-prefix=* | --exec_prefix=* | --exec-prefix=* | --exec-prefi=* \
+  | --exec-pref=* | --exec-pre=* | --exec-pr=* | --exec-p=* | --exec-=* \
+  | --exec=* | --exe=* | --ex=*)
+    exec_prefix="$ac_optarg" ;;
+
+  -gas | --gas | --ga | --g)
+    # Obsolete; use --with-gas.
+    with_gas=yes ;;
+
+  -help | --help | --hel | --he)
+    # Omit some internal or obsolete options to make the list less imposing.
+    # This message is too long to be a string in the A/UX 3.1 sh.
+    cat << EOF
+Usage: configure [options] [host]
+Options: [defaults in brackets after descriptions]
+Configuration:
+  --cache-file=FILE       cache test results in FILE
+  --help                  print this message
+  --no-create             do not create output files
+  --quiet, --silent       do not print \`checking...' messages
+  --version               print the version of autoconf that created configure
+Directory and file names:
+  --prefix=PREFIX         install architecture-independent files in PREFIX
+                          [$ac_default_prefix]
+  --exec-prefix=EPREFIX   install architecture-dependent files in EPREFIX
+                          [same as prefix]
+  --bindir=DIR            user executables in DIR [EPREFIX/bin]
+  --sbindir=DIR           system admin executables in DIR [EPREFIX/sbin]
+  --libexecdir=DIR        program executables in DIR [EPREFIX/libexec]
+  --datadir=DIR           read-only architecture-independent data in DIR
+                          [PREFIX/share]
+  --sysconfdir=DIR        read-only single-machine data in DIR [PREFIX/etc]
+  --sharedstatedir=DIR    modifiable architecture-independent data in DIR
+                          [PREFIX/com]
+  --localstatedir=DIR     modifiable single-machine data in DIR [PREFIX/var]
+  --libdir=DIR            object code libraries in DIR [EPREFIX/lib]
+  --includedir=DIR        C header files in DIR [PREFIX/include]
+  --oldincludedir=DIR     C header files for non-gcc in DIR [/usr/include]
+  --infodir=DIR           info documentation in DIR [PREFIX/info]
+  --mandir=DIR            man documentation in DIR [PREFIX/man]
+  --srcdir=DIR            find the sources in DIR [configure dir or ..]
+  --program-prefix=PREFIX prepend PREFIX to installed program names
+  --program-suffix=SUFFIX append SUFFIX to installed program names
+  --program-transform-name=PROGRAM
+                          run sed PROGRAM on installed program names
+EOF
+    cat << EOF
+Host type:
+  --build=BUILD           configure for building on BUILD [BUILD=HOST]
+  --host=HOST             configure for HOST [guessed]
+  --target=TARGET         configure for TARGET [TARGET=HOST]
+Features and packages:
+  --disable-FEATURE       do not include FEATURE (same as --enable-FEATURE=no)
+  --enable-FEATURE[=ARG]  include FEATURE [ARG=yes]
+  --with-PACKAGE[=ARG]    use PACKAGE [ARG=yes]
+  --without-PACKAGE       do not use PACKAGE (same as --with-PACKAGE=no)
+  --x-includes=DIR        X include files are in DIR
+  --x-libraries=DIR       X library files are in DIR
+EOF
+    if test -n "$ac_help"; then
+      echo "--enable and --with options recognized:$ac_help"
+    fi
+    exit 0 ;;
+
+  -host | --host | --hos | --ho)
+    ac_prev=host ;;
+  -host=* | --host=* | --hos=* | --ho=*)
+    host="$ac_optarg" ;;
+
+  -includedir | --includedir | --includedi | --included | --include \
+  | --includ | --inclu | --incl | --inc)
+    ac_prev=includedir ;;
+  -includedir=* | --includedir=* | --includedi=* | --included=* | --include=* \
+  | --includ=* | --inclu=* | --incl=* | --inc=*)
+    includedir="$ac_optarg" ;;
+
+  -infodir | --infodir | --infodi | --infod | --info | --inf)
+    ac_prev=infodir ;;
+  -infodir=* | --infodir=* | --infodi=* | --infod=* | --info=* | --inf=*)
+    infodir="$ac_optarg" ;;
+
+  -libdir | --libdir | --libdi | --libd)
+    ac_prev=libdir ;;
+  -libdir=* | --libdir=* | --libdi=* | --libd=*)
+    libdir="$ac_optarg" ;;
+
+  -libexecdir | --libexecdir | --libexecdi | --libexecd | --libexec \
+  | --libexe | --libex | --libe)
+    ac_prev=libexecdir ;;
+  -libexecdir=* | --libexecdir=* | --libexecdi=* | --libexecd=* | --libexec=* \
+  | --libexe=* | --libex=* | --libe=*)
+    libexecdir="$ac_optarg" ;;
+
+  -localstatedir | --localstatedir | --localstatedi | --localstated \
+  | --localstate | --localstat | --localsta | --localst \
+  | --locals | --local | --loca | --loc | --lo)
+    ac_prev=localstatedir ;;
+  -localstatedir=* | --localstatedir=* | --localstatedi=* | --localstated=* \
+  | --localstate=* | --localstat=* | --localsta=* | --localst=* \
+  | --locals=* | --local=* | --loca=* | --loc=* | --lo=*)
+    localstatedir="$ac_optarg" ;;
+
+  -mandir | --mandir | --mandi | --mand | --man | --ma | --m)
+    ac_prev=mandir ;;
+  -mandir=* | --mandir=* | --mandi=* | --mand=* | --man=* | --ma=* | --m=*)
+    mandir="$ac_optarg" ;;
+
+  -nfp | --nfp | --nf)
+    # Obsolete; use --without-fp.
+    with_fp=no ;;
+
+  -no-create | --no-create | --no-creat | --no-crea | --no-cre \
+  | --no-cr | --no-c)
+    no_create=yes ;;
+
+  -no-recursion | --no-recursion | --no-recursio | --no-recursi \
+  | --no-recurs | --no-recur | --no-recu | --no-rec | --no-re | --no-r)
+    no_recursion=yes ;;
+
+  -oldincludedir | --oldincludedir | --oldincludedi | --oldincluded \
+  | --oldinclude | --oldinclud | --oldinclu | --oldincl | --oldinc \
+  | --oldin | --oldi | --old | --ol | --o)
+    ac_prev=oldincludedir ;;
+  -oldincludedir=* | --oldincludedir=* | --oldincludedi=* | --oldincluded=* \
+  | --oldinclude=* | --oldinclud=* | --oldinclu=* | --oldincl=* | --oldinc=* \
+  | --oldin=* | --oldi=* | --old=* | --ol=* | --o=*)
+    oldincludedir="$ac_optarg" ;;
+
+  -prefix | --prefix | --prefi | --pref | --pre | --pr | --p)
+    ac_prev=prefix ;;
+  -prefix=* | --prefix=* | --prefi=* | --pref=* | --pre=* | --pr=* | --p=*)
+    prefix="$ac_optarg" ;;
+
+  -program-prefix | --program-prefix | --program-prefi | --program-pref \
+  | --program-pre | --program-pr | --program-p)
+    ac_prev=program_prefix ;;
+  -program-prefix=* | --program-prefix=* | --program-prefi=* \
+  | --program-pref=* | --program-pre=* | --program-pr=* | --program-p=*)
+    program_prefix="$ac_optarg" ;;
+
+  -program-suffix | --program-suffix | --program-suffi | --program-suff \
+  | --program-suf | --program-su | --program-s)
+    ac_prev=program_suffix ;;
+  -program-suffix=* | --program-suffix=* | --program-suffi=* \
+  | --program-suff=* | --program-suf=* | --program-su=* | --program-s=*)
+    program_suffix="$ac_optarg" ;;
+
+  -program-transform-name | --program-transform-name \
+  | --program-transform-nam | --program-transform-na \
+  | --program-transform-n | --program-transform- \
+  | --program-transform | --program-transfor \
+  | --program-transfo | --program-transf \
+  | --program-trans | --program-tran \
+  | --progr-tra | --program-tr | --program-t)
+    ac_prev=program_transform_name ;;
+  -program-transform-name=* | --program-transform-name=* \
+  | --program-transform-nam=* | --program-transform-na=* \
+  | --program-transform-n=* | --program-transform-=* \
+  | --program-transform=* | --program-transfor=* \
+  | --program-transfo=* | --program-transf=* \
+  | --program-trans=* | --program-tran=* \
+  | --progr-tra=* | --program-tr=* | --program-t=*)
+    program_transform_name="$ac_optarg" ;;
+
+  -q | -quiet | --quiet | --quie | --qui | --qu | --q \
+  | -silent | --silent | --silen | --sile | --sil)
+    silent=yes ;;
+
+  -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb)
+    ac_prev=sbindir ;;
+  -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \
+  | --sbi=* | --sb=*)
+    sbindir="$ac_optarg" ;;
+
+  -sharedstatedir | --sharedstatedir | --sharedstatedi \
+  | --sharedstated | --sharedstate | --sharedstat | --sharedsta \
+  | --sharedst | --shareds | --shared | --share | --shar \
+  | --sha | --sh)
+    ac_prev=sharedstatedir ;;
+  -sharedstatedir=* | --sharedstatedir=* | --sharedstatedi=* \
+  | --sharedstated=* | --sharedstate=* | --sharedstat=* | --sharedsta=* \
+  | --sharedst=* | --shareds=* | --shared=* | --share=* | --shar=* \
+  | --sha=* | --sh=*)
+    sharedstatedir="$ac_optarg" ;;
+
+  -site | --site | --sit)
+    ac_prev=site ;;
+  -site=* | --site=* | --sit=*)
+    site="$ac_optarg" ;;
+
+  -srcdir | --srcdir | --srcdi | --srcd | --src | --sr)
+    ac_prev=srcdir ;;
+  -srcdir=* | --srcdir=* | --srcdi=* | --srcd=* | --src=* | --sr=*)
+    srcdir="$ac_optarg" ;;
+
+  -sysconfdir | --sysconfdir | --sysconfdi | --sysconfd | --sysconf \
+  | --syscon | --sysco | --sysc | --sys | --sy)
+    ac_prev=sysconfdir ;;
+  -sysconfdir=* | --sysconfdir=* | --sysconfdi=* | --sysconfd=* | --sysconf=* \
+  | --syscon=* | --sysco=* | --sysc=* | --sys=* | --sy=*)
+    sysconfdir="$ac_optarg" ;;
+
+  -target | --target | --targe | --targ | --tar | --ta | --t)
+    ac_prev=target ;;
+  -target=* | --target=* | --targe=* | --targ=* | --tar=* | --ta=* | --t=*)
+    target="$ac_optarg" ;;
+
+  -v | -verbose | --verbose | --verbos | --verbo | --verb)
+    verbose=yes ;;
+
+  -version | --version | --versio | --versi | --vers)
+    echo "configure generated by autoconf version 2.12"
+    exit 0 ;;
+
+  -with-* | --with-*)
+    ac_package=`echo $ac_option|sed -e 's/-*with-//' -e 's/=.*//'`
+    # Reject names that are not valid shell variable names.
+    if test -n "`echo $ac_package| sed 's/[-_a-zA-Z0-9]//g'`"; then
+      { echo "configure: error: $ac_package: invalid package name" 1>&2; exit 1; }
+    fi
+    ac_package=`echo $ac_package| sed 's/-/_/g'`
+    case "$ac_option" in
+      *=*) ;;
+      *) ac_optarg=yes ;;
+    esac
+    eval "with_${ac_package}='$ac_optarg'" ;;
+
+  -without-* | --without-*)
+    ac_package=`echo $ac_option|sed -e 's/-*without-//'`
+    # Reject names that are not valid shell variable names.
+    if test -n "`echo $ac_package| sed 's/[-a-zA-Z0-9_]//g'`"; then
+      { echo "configure: error: $ac_package: invalid package name" 1>&2; exit 1; }
+    fi
+    ac_package=`echo $ac_package| sed 's/-/_/g'`
+    eval "with_${ac_package}=no" ;;
+
+  --x)
+    # Obsolete; use --with-x.
+    with_x=yes ;;
+
+  -x-includes | --x-includes | --x-include | --x-includ | --x-inclu \
+  | --x-incl | --x-inc | --x-in | --x-i)
+    ac_prev=x_includes ;;
+  -x-includes=* | --x-includes=* | --x-include=* | --x-includ=* | --x-inclu=* \
+  | --x-incl=* | --x-inc=* | --x-in=* | --x-i=*)
+    x_includes="$ac_optarg" ;;
+
+  -x-libraries | --x-libraries | --x-librarie | --x-librari \
+  | --x-librar | --x-libra | --x-libr | --x-lib | --x-li | --x-l)
+    ac_prev=x_libraries ;;
+  -x-libraries=* | --x-libraries=* | --x-librarie=* | --x-librari=* \
+  | --x-librar=* | --x-libra=* | --x-libr=* | --x-lib=* | --x-li=* | --x-l=*)
+    x_libraries="$ac_optarg" ;;
+
+  -*) { echo "configure: error: $ac_option: invalid option; use --help to show usage" 1>&2; exit 1; }
+    ;;
+
+  *)
+    if test -n "`echo $ac_option| sed 's/[-a-z0-9.]//g'`"; then
+      echo "configure: warning: $ac_option: invalid host type" 1>&2
+    fi
+    if test "x$nonopt" != xNONE; then
+      { echo "configure: error: can only configure for one host and one target at a time" 1>&2; exit 1; }
+    fi
+    nonopt="$ac_option"
+    ;;
+
+  esac
+done
+
+if test -n "$ac_prev"; then
+  { echo "configure: error: missing argument to --`echo $ac_prev | sed 's/_/-/g'`" 1>&2; exit 1; }
+fi
+
+trap 'rm -fr conftest* confdefs* core core.* *.core $ac_clean_files; exit 1' 1 2 15
+
+# File descriptor usage:
+# 0 standard input
+# 1 file creation
+# 2 errors and warnings
+# 3 some systems may open it to /dev/tty
+# 4 used on the Kubota Titan
+# 6 checking for... messages and results
+# 5 compiler messages saved in config.log
+if test "$silent" = yes; then
+  exec 6>/dev/null
+else
+  exec 6>&1
+fi
+exec 5>./config.log
+
+echo "\
+This file contains any messages produced by compilers while
+running configure, to aid debugging if configure makes a mistake.
+" 1>&5
+
+# Strip out --no-create and --no-recursion so they do not pile up.
+# Also quote any args containing shell metacharacters.
+ac_configure_args=
+for ac_arg
+do
+  case "$ac_arg" in
+  -no-create | --no-create | --no-creat | --no-crea | --no-cre \
+  | --no-cr | --no-c) ;;
+  -no-recursion | --no-recursion | --no-recursio | --no-recursi \
+  | --no-recurs | --no-recur | --no-recu | --no-rec | --no-re | --no-r) ;;
+  *" "*|*"	"*|*[\[\]\~\#\$\^\&\*\(\)\{\}\\\|\;\<\>\?]*)
+  ac_configure_args="$ac_configure_args '$ac_arg'" ;;
+  *) ac_configure_args="$ac_configure_args $ac_arg" ;;
+  esac
+done
+
+# NLS nuisances.
+# Only set these to C if already set.  These must not be set unconditionally
+# because not all systems understand e.g. LANG=C (notably SCO).
+# Fixing LC_MESSAGES prevents Solaris sh from translating var values in `set'!
+# Non-C LC_CTYPE values break the ctype check.
+if test "${LANG+set}"   = set; then LANG=C;   export LANG;   fi
+if test "${LC_ALL+set}" = set; then LC_ALL=C; export LC_ALL; fi
+if test "${LC_MESSAGES+set}" = set; then LC_MESSAGES=C; export LC_MESSAGES; fi
+if test "${LC_CTYPE+set}"    = set; then LC_CTYPE=C;    export LC_CTYPE;    fi
+
+# confdefs.h avoids OS command line length limits that DEFS can exceed.
+rm -rf conftest* confdefs.h
+# AIX cpp loses on an empty file, so make sure it contains at least a newline.
+echo > confdefs.h
+
+# A filename unique to this package, relative to the directory that
+# configure is in, which we can look for to find out if srcdir is correct.
+ac_unique_file=src/text/mgquery.c
+
+# Find the source files, if location was not specified.
+if test -z "$srcdir"; then
+  ac_srcdir_defaulted=yes
+  # Try the directory containing this script, then its parent.
+  ac_prog=$0
+  ac_confdir=`echo $ac_prog|sed 's%/[^/][^/]*$%%'`
+  test "x$ac_confdir" = "x$ac_prog" && ac_confdir=.
+  srcdir=$ac_confdir
+  if test ! -r $srcdir/$ac_unique_file; then
+    srcdir=..
+  fi
+else
+  ac_srcdir_defaulted=no
+fi
+if test ! -r $srcdir/$ac_unique_file; then
+  if test "$ac_srcdir_defaulted" = yes; then
+    { echo "configure: error: can not find sources in $ac_confdir or .." 1>&2; exit 1; }
+  else
+    { echo "configure: error: can not find sources in $srcdir" 1>&2; exit 1; }
+  fi
+fi
+srcdir=`echo "${srcdir}" | sed 's%\([^/]\)/*$%\1%'`
+
+# Prefer explicitly selected file to automatically selected ones.
+if test -z "$CONFIG_SITE"; then
+  if test "x$prefix" != xNONE; then
+    CONFIG_SITE="$prefix/share/config.site $prefix/etc/config.site"
+  else
+    CONFIG_SITE="$ac_default_prefix/share/config.site $ac_default_prefix/etc/config.site"
+  fi
+fi
+for ac_site_file in $CONFIG_SITE; do
+  if test -r "$ac_site_file"; then
+    echo "loading site script $ac_site_file"
+    . "$ac_site_file"
+  fi
+done
+
+if test -r "$cache_file"; then
+  echo "loading cache $cache_file"
+  . $cache_file
+else
+  echo "creating cache $cache_file"
+  > $cache_file
+fi
+
+ac_ext=c
+# CFLAGS is not in ac_cpp because -g, -O, etc. are not valid cpp options.
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='${CC-cc} -c $CFLAGS $CPPFLAGS conftest.$ac_ext 1>&5'
+ac_link='${CC-cc} -o conftest $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS 1>&5'
+cross_compiling=$ac_cv_prog_cc_cross
+
+if (echo "testing\c"; echo 1,2,3) | grep c >/dev/null; then
+  # Stardent Vistra SVR4 grep lacks -e, says ghazi@caip.rutgers.edu.
+  if (echo -n testing; echo 1,2,3) | sed s/-n/xn/ | grep xn >/dev/null; then
+    ac_n= ac_c='
+' ac_t='	'
+  else
+    ac_n=-n ac_c= ac_t=
+  fi
+else
+  ac_n= ac_c='\c' ac_t=
+fi
+
+
+
+if test "$program_transform_name" = s,x,x,; then
+  program_transform_name=
+else
+  # Double any \ or $.  echo might interpret backslashes.
+  cat <<\EOF_SED > conftestsed
+s,\\,\\\\,g; s,\$,$$,g
+EOF_SED
+  program_transform_name="`echo $program_transform_name|sed -f conftestsed`"
+  rm -f conftestsed
+fi
+test "$program_prefix" != NONE &&
+  program_transform_name="s,^,${program_prefix},; $program_transform_name"
+# Use a double $ so make ignores it.
+test "$program_suffix" != NONE &&
+  program_transform_name="s,\$\$,${program_suffix},; $program_transform_name"
+
+# sed with no file args requires a program.
+test "$program_transform_name" = "" && program_transform_name="s,x,x,"
+
+
+PACKAGE=mg
+VERSION=1.3e
+
+cat >> confdefs.h <<EOF
+#define PACKAGE "$PACKAGE"
+EOF
+
+cat >> confdefs.h <<EOF
+#define VERSION "$VERSION"
+EOF
+
+
+
+
+for ac_prog in $CCC c++ g++ gcc CC cxx cc++
+do
+# Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+echo $ac_n "checking for $ac_word""... $ac_c" 1>&6
+echo "configure:567: checking for $ac_word" >&5
+if eval "test \"`echo '$''{'ac_cv_prog_CXX'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  if test -n "$CXX"; then
+  ac_cv_prog_CXX="$CXX" # Let the user override the test.
+else
+  IFS="${IFS= 	}"; ac_save_ifs="$IFS"; IFS="${IFS}:"
+  for ac_dir in $PATH; do
+    test -z "$ac_dir" && ac_dir=.
+    if test -f $ac_dir/$ac_word; then
+      ac_cv_prog_CXX="$ac_prog"
+      break
+    fi
+  done
+  IFS="$ac_save_ifs"
+fi
+fi
+CXX="$ac_cv_prog_CXX"
+if test -n "$CXX"; then
+  echo "$ac_t""$CXX" 1>&6
+else
+  echo "$ac_t""no" 1>&6
+fi
+
+test -n "$CXX" && break
+done
+test -n "$CXX" || CXX="gcc"
+
+
+echo $ac_n "checking whether the C++ compiler ($CXX $CXXFLAGS $LDFLAGS) works""... $ac_c" 1>&6
+echo "configure:598: checking whether the C++ compiler ($CXX $CXXFLAGS $LDFLAGS) works" >&5
+
+ac_ext=C
+# CXXFLAGS is not in ac_cpp because -g, -O, etc. are not valid cpp options.
+ac_cpp='$CXXCPP $CPPFLAGS'
+ac_compile='${CXX-g++} -c $CXXFLAGS $CPPFLAGS conftest.$ac_ext 1>&5'
+ac_link='${CXX-g++} -o conftest $CXXFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS 1>&5'
+cross_compiling=$ac_cv_prog_cxx_cross
+
+cat > conftest.$ac_ext <<EOF
+#line 608 "configure"
+#include "confdefs.h"
+main(){return(0);}
+EOF
+if { (eval echo configure:612: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest; then
+  ac_cv_prog_cxx_works=yes
+  # If we can't run a trivial program, we are probably using a cross compiler.
+  if (./conftest; exit) 2>/dev/null; then
+    ac_cv_prog_cxx_cross=no
+  else
+    ac_cv_prog_cxx_cross=yes
+  fi
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  ac_cv_prog_cxx_works=no
+fi
+rm -fr conftest*
+ac_ext=c
+# CFLAGS is not in ac_cpp because -g, -O, etc. are not valid cpp options.
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='${CC-cc} -c $CFLAGS $CPPFLAGS conftest.$ac_ext 1>&5'
+ac_link='${CC-cc} -o conftest $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS 1>&5'
+cross_compiling=$ac_cv_prog_cc_cross
+
+echo "$ac_t""$ac_cv_prog_cxx_works" 1>&6
+if test $ac_cv_prog_cxx_works = no; then
+  { echo "configure: error: installation or configuration problem: C++ compiler cannot create executables." 1>&2; exit 1; }
+fi
+echo $ac_n "checking whether the C++ compiler ($CXX $CXXFLAGS $LDFLAGS) is a cross-compiler""... $ac_c" 1>&6
+echo "configure:638: checking whether the C++ compiler ($CXX $CXXFLAGS $LDFLAGS) is a cross-compiler" >&5
+echo "$ac_t""$ac_cv_prog_cxx_cross" 1>&6
+cross_compiling=$ac_cv_prog_cxx_cross
+
+echo $ac_n "checking whether we are using GNU C++""... $ac_c" 1>&6
+echo "configure:643: checking whether we are using GNU C++" >&5
+if eval "test \"`echo '$''{'ac_cv_prog_gxx'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  cat > conftest.C <<EOF
+#ifdef __GNUC__
+  yes;
+#endif
+EOF
+if { ac_try='${CXX-g++} -E conftest.C'; { (eval echo configure:652: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; }; } | egrep yes >/dev/null 2>&1; then
+  ac_cv_prog_gxx=yes
+else
+  ac_cv_prog_gxx=no
+fi
+fi
+
+echo "$ac_t""$ac_cv_prog_gxx" 1>&6
+
+if test $ac_cv_prog_gxx = yes; then
+  GXX=yes
+  ac_test_CXXFLAGS="${CXXFLAGS+set}"
+  ac_save_CXXFLAGS="$CXXFLAGS"
+  CXXFLAGS=
+  echo $ac_n "checking whether ${CXX-g++} accepts -g""... $ac_c" 1>&6
+echo "configure:667: checking whether ${CXX-g++} accepts -g" >&5
+if eval "test \"`echo '$''{'ac_cv_prog_cxx_g'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  echo 'void f(){}' > conftest.cc
+if test -z "`${CXX-g++} -g -c conftest.cc 2>&1`"; then
+  ac_cv_prog_cxx_g=yes
+else
+  ac_cv_prog_cxx_g=no
+fi
+rm -f conftest*
+
+fi
+
+echo "$ac_t""$ac_cv_prog_cxx_g" 1>&6
+  if test "$ac_test_CXXFLAGS" = set; then
+    CXXFLAGS="$ac_save_CXXFLAGS"
+  elif test $ac_cv_prog_cxx_g = yes; then
+    CXXFLAGS="-g -O2"
+  else
+    CXXFLAGS="-O2"
+  fi
+else
+  GXX=
+  test "${CXXFLAGS+set}" = set || CXXFLAGS="-g"
+fi
+
+for ac_prog in mawk gawk nawk awk
+do
+# Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+echo $ac_n "checking for $ac_word""... $ac_c" 1>&6
+echo "configure:699: checking for $ac_word" >&5
+if eval "test \"`echo '$''{'ac_cv_prog_AWK'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  if test -n "$AWK"; then
+  ac_cv_prog_AWK="$AWK" # Let the user override the test.
+else
+  IFS="${IFS= 	}"; ac_save_ifs="$IFS"; IFS="${IFS}:"
+  for ac_dir in $PATH; do
+    test -z "$ac_dir" && ac_dir=.
+    if test -f $ac_dir/$ac_word; then
+      ac_cv_prog_AWK="$ac_prog"
+      break
+    fi
+  done
+  IFS="$ac_save_ifs"
+fi
+fi
+AWK="$ac_cv_prog_AWK"
+if test -n "$AWK"; then
+  echo "$ac_t""$AWK" 1>&6
+else
+  echo "$ac_t""no" 1>&6
+fi
+
+test -n "$AWK" && break
+done
+
+for ac_prog in 'bison -y' byacc
+do
+# Extract the first word of "$ac_prog", so it can be a program name with args.
+set dummy $ac_prog; ac_word=$2
+echo $ac_n "checking for $ac_word""... $ac_c" 1>&6
+echo "configure:732: checking for $ac_word" >&5
+if eval "test \"`echo '$''{'ac_cv_prog_YACC'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  if test -n "$YACC"; then
+  ac_cv_prog_YACC="$YACC" # Let the user override the test.
+else
+  IFS="${IFS= 	}"; ac_save_ifs="$IFS"; IFS="${IFS}:"
+  for ac_dir in $PATH; do
+    test -z "$ac_dir" && ac_dir=.
+    if test -f $ac_dir/$ac_word; then
+      ac_cv_prog_YACC="$ac_prog"
+      break
+    fi
+  done
+  IFS="$ac_save_ifs"
+fi
+fi
+YACC="$ac_cv_prog_YACC"
+if test -n "$YACC"; then
+  echo "$ac_t""$YACC" 1>&6
+else
+  echo "$ac_t""no" 1>&6
+fi
+
+test -n "$YACC" && break
+done
+test -n "$YACC" || YACC="yacc"
+
+# Extract the first word of "gcc", so it can be a program name with args.
+set dummy gcc; ac_word=$2
+echo $ac_n "checking for $ac_word""... $ac_c" 1>&6
+echo "configure:764: checking for $ac_word" >&5
+if eval "test \"`echo '$''{'ac_cv_prog_CC'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  if test -n "$CC"; then
+  ac_cv_prog_CC="$CC" # Let the user override the test.
+else
+  IFS="${IFS= 	}"; ac_save_ifs="$IFS"; IFS="${IFS}:"
+  for ac_dir in $PATH; do
+    test -z "$ac_dir" && ac_dir=.
+    if test -f $ac_dir/$ac_word; then
+      ac_cv_prog_CC="gcc"
+      break
+    fi
+  done
+  IFS="$ac_save_ifs"
+fi
+fi
+CC="$ac_cv_prog_CC"
+if test -n "$CC"; then
+  echo "$ac_t""$CC" 1>&6
+else
+  echo "$ac_t""no" 1>&6
+fi
+
+if test -z "$CC"; then
+  # Extract the first word of "cc", so it can be a program name with args.
+set dummy cc; ac_word=$2
+echo $ac_n "checking for $ac_word""... $ac_c" 1>&6
+echo "configure:793: checking for $ac_word" >&5
+if eval "test \"`echo '$''{'ac_cv_prog_CC'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  if test -n "$CC"; then
+  ac_cv_prog_CC="$CC" # Let the user override the test.
+else
+  IFS="${IFS= 	}"; ac_save_ifs="$IFS"; IFS="${IFS}:"
+  ac_prog_rejected=no
+  for ac_dir in $PATH; do
+    test -z "$ac_dir" && ac_dir=.
+    if test -f $ac_dir/$ac_word; then
+      if test "$ac_dir/$ac_word" = "/usr/ucb/cc"; then
+        ac_prog_rejected=yes
+	continue
+      fi
+      ac_cv_prog_CC="cc"
+      break
+    fi
+  done
+  IFS="$ac_save_ifs"
+if test $ac_prog_rejected = yes; then
+  # We found a bogon in the path, so make sure we never use it.
+  set dummy $ac_cv_prog_CC
+  shift
+  if test $# -gt 0; then
+    # We chose a different compiler from the bogus one.
+    # However, it has the same basename, so the bogon will be chosen
+    # first if we set CC to just the basename; use the full file name.
+    shift
+    set dummy "$ac_dir/$ac_word" "$@"
+    shift
+    ac_cv_prog_CC="$@"
+  fi
+fi
+fi
+fi
+CC="$ac_cv_prog_CC"
+if test -n "$CC"; then
+  echo "$ac_t""$CC" 1>&6
+else
+  echo "$ac_t""no" 1>&6
+fi
+
+  test -z "$CC" && { echo "configure: error: no acceptable cc found in \$PATH" 1>&2; exit 1; }
+fi
+
+echo $ac_n "checking whether the C compiler ($CC $CFLAGS $LDFLAGS) works""... $ac_c" 1>&6
+echo "configure:841: checking whether the C compiler ($CC $CFLAGS $LDFLAGS) works" >&5
+
+ac_ext=c
+# CFLAGS is not in ac_cpp because -g, -O, etc. are not valid cpp options.
+ac_cpp='$CPP $CPPFLAGS'
+ac_compile='${CC-cc} -c $CFLAGS $CPPFLAGS conftest.$ac_ext 1>&5'
+ac_link='${CC-cc} -o conftest $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS 1>&5'
+cross_compiling=$ac_cv_prog_cc_cross
+
+cat > conftest.$ac_ext <<EOF
+#line 851 "configure"
+#include "confdefs.h"
+main(){return(0);}
+EOF
+if { (eval echo configure:855: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest; then
+  ac_cv_prog_cc_works=yes
+  # If we can't run a trivial program, we are probably using a cross compiler.
+  if (./conftest; exit) 2>/dev/null; then
+    ac_cv_prog_cc_cross=no
+  else
+    ac_cv_prog_cc_cross=yes
+  fi
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  ac_cv_prog_cc_works=no
+fi
+rm -fr conftest*
+
+echo "$ac_t""$ac_cv_prog_cc_works" 1>&6
+if test $ac_cv_prog_cc_works = no; then
+  { echo "configure: error: installation or configuration problem: C compiler cannot create executables." 1>&2; exit 1; }
+fi
+echo $ac_n "checking whether the C compiler ($CC $CFLAGS $LDFLAGS) is a cross-compiler""... $ac_c" 1>&6
+echo "configure:875: checking whether the C compiler ($CC $CFLAGS $LDFLAGS) is a cross-compiler" >&5
+echo "$ac_t""$ac_cv_prog_cc_cross" 1>&6
+cross_compiling=$ac_cv_prog_cc_cross
+
+echo $ac_n "checking whether we are using GNU C""... $ac_c" 1>&6
+echo "configure:880: checking whether we are using GNU C" >&5
+if eval "test \"`echo '$''{'ac_cv_prog_gcc'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  cat > conftest.c <<EOF
+#ifdef __GNUC__
+  yes;
+#endif
+EOF
+if { ac_try='${CC-cc} -E conftest.c'; { (eval echo configure:889: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; }; } | egrep yes >/dev/null 2>&1; then
+  ac_cv_prog_gcc=yes
+else
+  ac_cv_prog_gcc=no
+fi
+fi
+
+echo "$ac_t""$ac_cv_prog_gcc" 1>&6
+
+if test $ac_cv_prog_gcc = yes; then
+  GCC=yes
+  ac_test_CFLAGS="${CFLAGS+set}"
+  ac_save_CFLAGS="$CFLAGS"
+  CFLAGS=
+  echo $ac_n "checking whether ${CC-cc} accepts -g""... $ac_c" 1>&6
+echo "configure:904: checking whether ${CC-cc} accepts -g" >&5
+if eval "test \"`echo '$''{'ac_cv_prog_cc_g'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  echo 'void f(){}' > conftest.c
+if test -z "`${CC-cc} -g -c conftest.c 2>&1`"; then
+  ac_cv_prog_cc_g=yes
+else
+  ac_cv_prog_cc_g=no
+fi
+rm -f conftest*
+
+fi
+
+echo "$ac_t""$ac_cv_prog_cc_g" 1>&6
+  if test "$ac_test_CFLAGS" = set; then
+    CFLAGS="$ac_save_CFLAGS"
+  elif test $ac_cv_prog_cc_g = yes; then
+    CFLAGS="-g -O2"
+  else
+    CFLAGS="-O2"
+  fi
+else
+  GCC=
+  test "${CFLAGS+set}" = set || CFLAGS="-g"
+fi
+
+ac_aux_dir=
+for ac_dir in $srcdir $srcdir/.. $srcdir/../..; do
+  if test -f $ac_dir/install-sh; then
+    ac_aux_dir=$ac_dir
+    ac_install_sh="$ac_aux_dir/install-sh -c"
+    break
+  elif test -f $ac_dir/install.sh; then
+    ac_aux_dir=$ac_dir
+    ac_install_sh="$ac_aux_dir/install.sh -c"
+    break
+  fi
+done
+if test -z "$ac_aux_dir"; then
+  { echo "configure: error: can not find install-sh or install.sh in $srcdir $srcdir/.. $srcdir/../.." 1>&2; exit 1; }
+fi
+ac_config_guess=$ac_aux_dir/config.guess
+ac_config_sub=$ac_aux_dir/config.sub
+ac_configure=$ac_aux_dir/configure # This should be Cygnus configure.
+
+# Find a good install program.  We prefer a C program (faster),
+# so one script is as good as another.  But avoid the broken or
+# incompatible versions:
+# SysV /etc/install, /usr/sbin/install
+# SunOS /usr/etc/install
+# IRIX /sbin/install
+# AIX /bin/install
+# AFS /usr/afsws/bin/install, which mishandles nonexistent args
+# SVR4 /usr/ucb/install, which tries to use the nonexistent group "staff"
+# ./install, which can be erroneously created by make from ./install.sh.
+echo $ac_n "checking for a BSD compatible install""... $ac_c" 1>&6
+echo "configure:961: checking for a BSD compatible install" >&5
+if test -z "$INSTALL"; then
+if eval "test \"`echo '$''{'ac_cv_path_install'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+    IFS="${IFS= 	}"; ac_save_IFS="$IFS"; IFS="${IFS}:"
+  for ac_dir in $PATH; do
+    # Account for people who put trailing slashes in PATH elements.
+    case "$ac_dir/" in
+    /|./|.//|/etc/*|/usr/sbin/*|/usr/etc/*|/sbin/*|/usr/afsws/bin/*|/usr/ucb/*) ;;
+    *)
+      # OSF1 and SCO ODT 3.0 have their own names for install.
+      for ac_prog in ginstall installbsd scoinst install; do
+        if test -f $ac_dir/$ac_prog; then
+	  if test $ac_prog = install &&
+            grep dspmsg $ac_dir/$ac_prog >/dev/null 2>&1; then
+	    # AIX install.  It has an incompatible calling convention.
+	    # OSF/1 installbsd also uses dspmsg, but is usable.
+	    :
+	  else
+	    ac_cv_path_install="$ac_dir/$ac_prog -c"
+	    break 2
+	  fi
+	fi
+      done
+      ;;
+    esac
+  done
+  IFS="$ac_save_IFS"
+
+fi
+  if test "${ac_cv_path_install+set}" = set; then
+    INSTALL="$ac_cv_path_install"
+  else
+    # As a last resort, use the slow shell script.  We don't cache a
+    # path for INSTALL within a source directory, because that will
+    # break other packages using the cache if that directory is
+    # removed, or if the path is relative.
+    INSTALL="$ac_install_sh"
+  fi
+fi
+echo "$ac_t""$INSTALL" 1>&6
+
+# Use test -z because SunOS4 sh mishandles braces in ${var-val}.
+# It thinks the first close brace ends the variable substitution.
+test -z "$INSTALL_PROGRAM" && INSTALL_PROGRAM='${INSTALL}'
+
+test -z "$INSTALL_DATA" && INSTALL_DATA='${INSTALL} -m 644'
+
+echo $ac_n "checking whether ln -s works""... $ac_c" 1>&6
+echo "configure:1011: checking whether ln -s works" >&5
+if eval "test \"`echo '$''{'ac_cv_prog_LN_S'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  rm -f conftestdata
+if ln -s X conftestdata 2>/dev/null
+then
+  rm -f conftestdata
+  ac_cv_prog_LN_S="ln -s"
+else
+  ac_cv_prog_LN_S=ln
+fi
+fi
+LN_S="$ac_cv_prog_LN_S"
+if test "$ac_cv_prog_LN_S" = "ln -s"; then
+  echo "$ac_t""yes" 1>&6
+else
+  echo "$ac_t""no" 1>&6
+fi
+
+echo $ac_n "checking whether ${MAKE-make} sets \${MAKE}""... $ac_c" 1>&6
+echo "configure:1032: checking whether ${MAKE-make} sets \${MAKE}" >&5
+set dummy ${MAKE-make}; ac_make=`echo "$2" | sed 'y%./+-%__p_%'`
+if eval "test \"`echo '$''{'ac_cv_prog_make_${ac_make}_set'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  cat > conftestmake <<\EOF
+all:
+	@echo 'ac_maketemp="${MAKE}"'
+EOF
+# GNU make sometimes prints "make[1]: Entering...", which would confuse us.
+eval `${MAKE-make} -f conftestmake 2>/dev/null | grep temp=`
+if test -n "$ac_maketemp"; then
+  eval ac_cv_prog_make_${ac_make}_set=yes
+else
+  eval ac_cv_prog_make_${ac_make}_set=no
+fi
+rm -f conftestmake
+fi
+if eval "test \"`echo '$ac_cv_prog_make_'${ac_make}_set`\" = yes"; then
+  echo "$ac_t""yes" 1>&6
+  SET_MAKE=
+else
+  echo "$ac_t""no" 1>&6
+  SET_MAKE="MAKE=${MAKE-make}"
+fi
+
+# Extract the first word of "ranlib", so it can be a program name with args.
+set dummy ranlib; ac_word=$2
+echo $ac_n "checking for $ac_word""... $ac_c" 1>&6
+echo "configure:1061: checking for $ac_word" >&5
+if eval "test \"`echo '$''{'ac_cv_prog_RANLIB'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  if test -n "$RANLIB"; then
+  ac_cv_prog_RANLIB="$RANLIB" # Let the user override the test.
+else
+  IFS="${IFS= 	}"; ac_save_ifs="$IFS"; IFS="${IFS}:"
+  for ac_dir in $PATH; do
+    test -z "$ac_dir" && ac_dir=.
+    if test -f $ac_dir/$ac_word; then
+      ac_cv_prog_RANLIB="ranlib"
+      break
+    fi
+  done
+  IFS="$ac_save_ifs"
+  test -z "$ac_cv_prog_RANLIB" && ac_cv_prog_RANLIB=":"
+fi
+fi
+RANLIB="$ac_cv_prog_RANLIB"
+if test -n "$RANLIB"; then
+  echo "$ac_t""$RANLIB" 1>&6
+else
+  echo "$ac_t""no" 1>&6
+fi
+
+
+echo $ac_n "checking how to run the C preprocessor""... $ac_c" 1>&6
+echo "configure:1089: checking how to run the C preprocessor" >&5
+# On Suns, sometimes $CPP names a directory.
+if test -n "$CPP" && test -d "$CPP"; then
+  CPP=
+fi
+if test -z "$CPP"; then
+if eval "test \"`echo '$''{'ac_cv_prog_CPP'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+    # This must be in double quotes, not single quotes, because CPP may get
+  # substituted into the Makefile and "${CC-cc}" will confuse make.
+  CPP="${CC-cc} -E"
+  # On the NeXT, cc -E runs the code through the compiler's parser,
+  # not just through cpp.
+  cat > conftest.$ac_ext <<EOF
+#line 1104 "configure"
+#include "confdefs.h"
+#include <assert.h>
+Syntax Error
+EOF
+ac_try="$ac_cpp conftest.$ac_ext >/dev/null 2>conftest.out"
+{ (eval echo configure:1110: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; }
+ac_err=`grep -v '^ *+' conftest.out`
+if test -z "$ac_err"; then
+  :
+else
+  echo "$ac_err" >&5
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  CPP="${CC-cc} -E -traditional-cpp"
+  cat > conftest.$ac_ext <<EOF
+#line 1121 "configure"
+#include "confdefs.h"
+#include <assert.h>
+Syntax Error
+EOF
+ac_try="$ac_cpp conftest.$ac_ext >/dev/null 2>conftest.out"
+{ (eval echo configure:1127: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; }
+ac_err=`grep -v '^ *+' conftest.out`
+if test -z "$ac_err"; then
+  :
+else
+  echo "$ac_err" >&5
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  CPP=/lib/cpp
+fi
+rm -f conftest*
+fi
+rm -f conftest*
+  ac_cv_prog_CPP="$CPP"
+fi
+  CPP="$ac_cv_prog_CPP"
+else
+  ac_cv_prog_CPP="$CPP"
+fi
+echo "$ac_t""$CPP" 1>&6
+
+echo $ac_n "checking for AIX""... $ac_c" 1>&6
+echo "configure:1150: checking for AIX" >&5
+cat > conftest.$ac_ext <<EOF
+#line 1152 "configure"
+#include "confdefs.h"
+#ifdef _AIX
+  yes
+#endif
+
+EOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  egrep "yes" >/dev/null 2>&1; then
+  rm -rf conftest*
+  echo "$ac_t""yes" 1>&6; cat >> confdefs.h <<\EOF
+#define _ALL_SOURCE 1
+EOF
+
+else
+  rm -rf conftest*
+  echo "$ac_t""no" 1>&6
+fi
+rm -f conftest*
+
+
+echo $ac_n "checking for POSIXized ISC""... $ac_c" 1>&6
+echo "configure:1174: checking for POSIXized ISC" >&5
+if test -d /etc/conf/kconfig.d &&
+  grep _POSIX_VERSION /usr/include/sys/unistd.h >/dev/null 2>&1
+then
+  echo "$ac_t""yes" 1>&6
+  ISC=yes # If later tests want to check for ISC.
+  cat >> confdefs.h <<\EOF
+#define _POSIX_SOURCE 1
+EOF
+
+  if test "$GCC" = yes; then
+    CC="$CC -posix"
+  else
+    CC="$CC -Xp"
+  fi
+else
+  echo "$ac_t""no" 1>&6
+  ISC=
+fi
+
+ac_safe=`echo "minix/config.h" | sed 'y%./+-%__p_%'`
+echo $ac_n "checking for minix/config.h""... $ac_c" 1>&6
+echo "configure:1196: checking for minix/config.h" >&5
+if eval "test \"`echo '$''{'ac_cv_header_$ac_safe'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  cat > conftest.$ac_ext <<EOF
+#line 1201 "configure"
+#include "confdefs.h"
+#include <minix/config.h>
+EOF
+ac_try="$ac_cpp conftest.$ac_ext >/dev/null 2>conftest.out"
+{ (eval echo configure:1206: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; }
+ac_err=`grep -v '^ *+' conftest.out`
+if test -z "$ac_err"; then
+  rm -rf conftest*
+  eval "ac_cv_header_$ac_safe=yes"
+else
+  echo "$ac_err" >&5
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  eval "ac_cv_header_$ac_safe=no"
+fi
+rm -f conftest*
+fi
+if eval "test \"`echo '$ac_cv_header_'$ac_safe`\" = yes"; then
+  echo "$ac_t""yes" 1>&6
+  MINIX=yes
+else
+  echo "$ac_t""no" 1>&6
+MINIX=
+fi
+
+if test "$MINIX" = yes; then
+  cat >> confdefs.h <<\EOF
+#define _POSIX_SOURCE 1
+EOF
+
+  cat >> confdefs.h <<\EOF
+#define _POSIX_1_SOURCE 2
+EOF
+
+  cat >> confdefs.h <<\EOF
+#define _MINIX 1
+EOF
+
+fi
+
+echo $ac_n "checking for ${CC-cc} option to accept ANSI C""... $ac_c" 1>&6
+echo "configure:1244: checking for ${CC-cc} option to accept ANSI C" >&5
+if eval "test \"`echo '$''{'ac_cv_prog_cc_stdc'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  ac_cv_prog_cc_stdc=no
+ac_save_CFLAGS="$CFLAGS"
+# Don't try gcc -ansi; that turns off useful extensions and
+# breaks some systems' header files.
+# AIX			-qlanglvl=ansi
+# Ultrix and OSF/1	-std1
+# HP-UX			-Aa -D_HPUX_SOURCE
+# SVR4			-Xc
+for ac_arg in "" -qlanglvl=ansi -std1 "-Aa -D_HPUX_SOURCE" -Xc
+do
+  CFLAGS="$ac_save_CFLAGS $ac_arg"
+  cat > conftest.$ac_ext <<EOF
+#line 1260 "configure"
+#include "confdefs.h"
+#if !defined(__STDC__) || __STDC__ != 1
+choke me
+#endif	
+
+int main() {
+int test (int i, double x);
+struct s1 {int (*f) (int a);};
+struct s2 {int (*f) (double a);};
+; return 0; }
+EOF
+if { (eval echo configure:1272: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then
+  rm -rf conftest*
+  ac_cv_prog_cc_stdc="$ac_arg"; break
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+fi
+rm -f conftest*
+done
+CFLAGS="$ac_save_CFLAGS"
+
+fi
+
+echo "$ac_t""$ac_cv_prog_cc_stdc" 1>&6
+case "x$ac_cv_prog_cc_stdc" in
+  x|xno) ;;
+  *) CC="$CC $ac_cv_prog_cc_stdc" ;;
+esac
+
+
+echo $ac_n "checking for function prototypes""... $ac_c" 1>&6
+echo "configure:1293: checking for function prototypes" >&5
+if test "$ac_cv_prog_cc_stdc" != no; then
+  echo "$ac_t""yes" 1>&6
+  cat >> confdefs.h <<\EOF
+#define PROTOTYPES 1
+EOF
+
+  U= ANSI2KNR=
+else
+  echo "$ac_t""no" 1>&6
+  U=_ ANSI2KNR=ansi2knr
+fi
+
+echo $ac_n "checking for working const""... $ac_c" 1>&6
+echo "configure:1307: checking for working const" >&5
+if eval "test \"`echo '$''{'ac_cv_c_const'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  cat > conftest.$ac_ext <<EOF
+#line 1312 "configure"
+#include "confdefs.h"
+
+int main() {
+
+/* Ultrix mips cc rejects this.  */
+typedef int charset[2]; const charset x;
+/* SunOS 4.1.1 cc rejects this.  */
+char const *const *ccp;
+char **p;
+/* NEC SVR4.0.2 mips cc rejects this.  */
+struct point {int x, y;};
+static struct point const zero = {0,0};
+/* AIX XL C 1.02.0.0 rejects this.
+   It does not let you subtract one const X* pointer from another in an arm
+   of an if-expression whose if-part is not a constant expression */
+const char *g = "string";
+ccp = &g + (g ? g-g : 0);
+/* HPUX 7.0 cc rejects these. */
+++ccp;
+p = (char**) ccp;
+ccp = (char const *const *) p;
+{ /* SCO 3.2v4 cc rejects this.  */
+  char *t;
+  char const *s = 0 ? (char *) 0 : (char const *) 0;
+
+  *t++ = 0;
+}
+{ /* Someone thinks the Sun supposedly-ANSI compiler will reject this.  */
+  int x[] = {25, 17};
+  const int *foo = &x[0];
+  ++foo;
+}
+{ /* Sun SC1.0 ANSI compiler rejects this -- but not the above. */
+  typedef const int *iptr;
+  iptr p = 0;
+  ++p;
+}
+{ /* AIX XL C 1.02.0.0 rejects this saying
+     "k.c", line 2.27: 1506-025 (S) Operand must be a modifiable lvalue. */
+  struct s { int j; const int *ap[3]; };
+  struct s *b; b->j = 5;
+}
+{ /* ULTRIX-32 V3.1 (Rev 9) vcc rejects this */
+  const int foo = 10;
+}
+
+; return 0; }
+EOF
+if { (eval echo configure:1361: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then
+  rm -rf conftest*
+  ac_cv_c_const=yes
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  ac_cv_c_const=no
+fi
+rm -f conftest*
+fi
+
+echo "$ac_t""$ac_cv_c_const" 1>&6
+if test $ac_cv_c_const = no; then
+  cat >> confdefs.h <<\EOF
+#define const 
+EOF
+
+fi
+
+echo $ac_n "checking for ANSI C header files""... $ac_c" 1>&6
+echo "configure:1382: checking for ANSI C header files" >&5
+if eval "test \"`echo '$''{'ac_cv_header_stdc'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  cat > conftest.$ac_ext <<EOF
+#line 1387 "configure"
+#include "confdefs.h"
+#include <stdlib.h>
+#include <stdarg.h>
+#include <string.h>
+#include <float.h>
+EOF
+ac_try="$ac_cpp conftest.$ac_ext >/dev/null 2>conftest.out"
+{ (eval echo configure:1395: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; }
+ac_err=`grep -v '^ *+' conftest.out`
+if test -z "$ac_err"; then
+  rm -rf conftest*
+  ac_cv_header_stdc=yes
+else
+  echo "$ac_err" >&5
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  ac_cv_header_stdc=no
+fi
+rm -f conftest*
+
+if test $ac_cv_header_stdc = yes; then
+  # SunOS 4.x string.h does not declare mem*, contrary to ANSI.
+cat > conftest.$ac_ext <<EOF
+#line 1412 "configure"
+#include "confdefs.h"
+#include <string.h>
+EOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  egrep "memchr" >/dev/null 2>&1; then
+  :
+else
+  rm -rf conftest*
+  ac_cv_header_stdc=no
+fi
+rm -f conftest*
+
+fi
+
+if test $ac_cv_header_stdc = yes; then
+  # ISC 2.0.2 stdlib.h does not declare free, contrary to ANSI.
+cat > conftest.$ac_ext <<EOF
+#line 1430 "configure"
+#include "confdefs.h"
+#include <stdlib.h>
+EOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  egrep "free" >/dev/null 2>&1; then
+  :
+else
+  rm -rf conftest*
+  ac_cv_header_stdc=no
+fi
+rm -f conftest*
+
+fi
+
+if test $ac_cv_header_stdc = yes; then
+  # /bin/cc in Irix-4.0.5 gets non-ANSI ctype macros unless using -ansi.
+if test "$cross_compiling" = yes; then
+  :
+else
+  cat > conftest.$ac_ext <<EOF
+#line 1451 "configure"
+#include "confdefs.h"
+#include <ctype.h>
+#define ISLOWER(c) ('a' <= (c) && (c) <= 'z')
+#define TOUPPER(c) (ISLOWER(c) ? 'A' + ((c) - 'a') : (c))
+#define XOR(e, f) (((e) && !(f)) || (!(e) && (f)))
+int main () { int i; for (i = 0; i < 256; i++)
+if (XOR (islower (i), ISLOWER (i)) || toupper (i) != TOUPPER (i)) exit(2);
+exit (0); }
+
+EOF
+if { (eval echo configure:1462: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest && (./conftest; exit) 2>/dev/null
+then
+  :
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -fr conftest*
+  ac_cv_header_stdc=no
+fi
+rm -fr conftest*
+fi
+
+fi
+fi
+
+echo "$ac_t""$ac_cv_header_stdc" 1>&6
+if test $ac_cv_header_stdc = yes; then
+  cat >> confdefs.h <<\EOF
+#define STDC_HEADERS 1
+EOF
+
+fi
+
+echo $ac_n "checking for off_t""... $ac_c" 1>&6
+echo "configure:1486: checking for off_t" >&5
+if eval "test \"`echo '$''{'ac_cv_type_off_t'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  cat > conftest.$ac_ext <<EOF
+#line 1491 "configure"
+#include "confdefs.h"
+#include <sys/types.h>
+#if STDC_HEADERS
+#include <stdlib.h>
+#include <stddef.h>
+#endif
+EOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  egrep "off_t[^a-zA-Z_0-9]" >/dev/null 2>&1; then
+  rm -rf conftest*
+  ac_cv_type_off_t=yes
+else
+  rm -rf conftest*
+  ac_cv_type_off_t=no
+fi
+rm -f conftest*
+
+fi
+echo "$ac_t""$ac_cv_type_off_t" 1>&6
+if test $ac_cv_type_off_t = no; then
+  cat >> confdefs.h <<\EOF
+#define off_t long
+EOF
+
+fi
+
+echo $ac_n "checking for size_t""... $ac_c" 1>&6
+echo "configure:1519: checking for size_t" >&5
+if eval "test \"`echo '$''{'ac_cv_type_size_t'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  cat > conftest.$ac_ext <<EOF
+#line 1524 "configure"
+#include "confdefs.h"
+#include <sys/types.h>
+#if STDC_HEADERS
+#include <stdlib.h>
+#include <stddef.h>
+#endif
+EOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  egrep "size_t[^a-zA-Z_0-9]" >/dev/null 2>&1; then
+  rm -rf conftest*
+  ac_cv_type_size_t=yes
+else
+  rm -rf conftest*
+  ac_cv_type_size_t=no
+fi
+rm -f conftest*
+
+fi
+echo "$ac_t""$ac_cv_type_size_t" 1>&6
+if test $ac_cv_type_size_t = no; then
+  cat >> confdefs.h <<\EOF
+#define size_t unsigned
+EOF
+
+fi
+
+echo $ac_n "checking whether time.h and sys/time.h may both be included""... $ac_c" 1>&6
+echo "configure:1552: checking whether time.h and sys/time.h may both be included" >&5
+if eval "test \"`echo '$''{'ac_cv_header_time'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  cat > conftest.$ac_ext <<EOF
+#line 1557 "configure"
+#include "confdefs.h"
+#include <sys/types.h>
+#include <sys/time.h>
+#include <time.h>
+int main() {
+struct tm *tp;
+; return 0; }
+EOF
+if { (eval echo configure:1566: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then
+  rm -rf conftest*
+  ac_cv_header_time=yes
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  ac_cv_header_time=no
+fi
+rm -f conftest*
+fi
+
+echo "$ac_t""$ac_cv_header_time" 1>&6
+if test $ac_cv_header_time = yes; then
+  cat >> confdefs.h <<\EOF
+#define TIME_WITH_SYS_TIME 1
+EOF
+
+fi
+
+echo $ac_n "checking whether struct tm is in sys/time.h or time.h""... $ac_c" 1>&6
+echo "configure:1587: checking whether struct tm is in sys/time.h or time.h" >&5
+if eval "test \"`echo '$''{'ac_cv_struct_tm'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  cat > conftest.$ac_ext <<EOF
+#line 1592 "configure"
+#include "confdefs.h"
+#include <sys/types.h>
+#include <time.h>
+int main() {
+struct tm *tp; tp->tm_sec;
+; return 0; }
+EOF
+if { (eval echo configure:1600: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then
+  rm -rf conftest*
+  ac_cv_struct_tm=time.h
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  ac_cv_struct_tm=sys/time.h
+fi
+rm -f conftest*
+fi
+
+echo "$ac_t""$ac_cv_struct_tm" 1>&6
+if test $ac_cv_struct_tm = sys/time.h; then
+  cat >> confdefs.h <<\EOF
+#define TM_IN_SYS_TIME 1
+EOF
+
+fi
+
+
+if test "$ac_cv_prog_cc_stdc" = '-Xc'; then
+cat > conftest.$ac_ext <<EOF
+#line 1623 "configure"
+#include "confdefs.h"
+#include <stdio.h>
+
+int main() {
+ FILE *rubbish = popen("yes", "r"); 
+; return 0; }
+EOF
+if { (eval echo configure:1631: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then
+  :
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  CC="`echo $CC | sed 's/-Xc/-Xa/'`"    ac_cv_prog_cc_stdc='-Xa'
+fi
+rm -f conftest*
+fi
+
+
+echo $ac_n "checking for main in -lm""... $ac_c" 1>&6
+echo "configure:1644: checking for main in -lm" >&5
+ac_lib_var=`echo m'_'main | sed 'y%./+-%__p_%'`
+if eval "test \"`echo '$''{'ac_cv_lib_$ac_lib_var'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  ac_save_LIBS="$LIBS"
+LIBS="-lm  $LIBS"
+cat > conftest.$ac_ext <<EOF
+#line 1652 "configure"
+#include "confdefs.h"
+
+int main() {
+main()
+; return 0; }
+EOF
+if { (eval echo configure:1659: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest; then
+  rm -rf conftest*
+  eval "ac_cv_lib_$ac_lib_var=yes"
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  eval "ac_cv_lib_$ac_lib_var=no"
+fi
+rm -f conftest*
+LIBS="$ac_save_LIBS"
+
+fi
+if eval "test \"`echo '$ac_cv_lib_'$ac_lib_var`\" = yes"; then
+  echo "$ac_t""yes" 1>&6
+    ac_tr_lib=HAVE_LIB`echo m | sed -e 's/[^a-zA-Z0-9_]/_/g' \
+    -e 'y/abcdefghijklmnopqrstuvwxyz/ABCDEFGHIJKLMNOPQRSTUVWXYZ/'`
+  cat >> confdefs.h <<EOF
+#define $ac_tr_lib 1
+EOF
+
+  LIBS="-lm $LIBS"
+
+else
+  echo "$ac_t""no" 1>&6
+fi
+
+
+ac_header_dirent=no
+for ac_hdr in dirent.h sys/ndir.h sys/dir.h ndir.h
+do
+ac_safe=`echo "$ac_hdr" | sed 'y%./+-%__p_%'`
+echo $ac_n "checking for $ac_hdr that defines DIR""... $ac_c" 1>&6
+echo "configure:1692: checking for $ac_hdr that defines DIR" >&5
+if eval "test \"`echo '$''{'ac_cv_header_dirent_$ac_safe'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  cat > conftest.$ac_ext <<EOF
+#line 1697 "configure"
+#include "confdefs.h"
+#include <sys/types.h>
+#include <$ac_hdr>
+int main() {
+DIR *dirp = 0;
+; return 0; }
+EOF
+if { (eval echo configure:1705: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then
+  rm -rf conftest*
+  eval "ac_cv_header_dirent_$ac_safe=yes"
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  eval "ac_cv_header_dirent_$ac_safe=no"
+fi
+rm -f conftest*
+fi
+if eval "test \"`echo '$ac_cv_header_dirent_'$ac_safe`\" = yes"; then
+  echo "$ac_t""yes" 1>&6
+    ac_tr_hdr=HAVE_`echo $ac_hdr | sed 'y%abcdefghijklmnopqrstuvwxyz./-%ABCDEFGHIJKLMNOPQRSTUVWXYZ___%'`
+  cat >> confdefs.h <<EOF
+#define $ac_tr_hdr 1
+EOF
+ ac_header_dirent=$ac_hdr; break
+else
+  echo "$ac_t""no" 1>&6
+fi
+done
+# Two versions of opendir et al. are in -ldir and -lx on SCO Xenix.
+if test $ac_header_dirent = dirent.h; then
+echo $ac_n "checking for opendir in -ldir""... $ac_c" 1>&6
+echo "configure:1730: checking for opendir in -ldir" >&5
+ac_lib_var=`echo dir'_'opendir | sed 'y%./+-%__p_%'`
+if eval "test \"`echo '$''{'ac_cv_lib_$ac_lib_var'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  ac_save_LIBS="$LIBS"
+LIBS="-ldir  $LIBS"
+cat > conftest.$ac_ext <<EOF
+#line 1738 "configure"
+#include "confdefs.h"
+/* Override any gcc2 internal prototype to avoid an error.  */
+/* We use char because int might match the return type of a gcc2
+    builtin and then its argument prototype would still apply.  */
+char opendir();
+
+int main() {
+opendir()
+; return 0; }
+EOF
+if { (eval echo configure:1749: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest; then
+  rm -rf conftest*
+  eval "ac_cv_lib_$ac_lib_var=yes"
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  eval "ac_cv_lib_$ac_lib_var=no"
+fi
+rm -f conftest*
+LIBS="$ac_save_LIBS"
+
+fi
+if eval "test \"`echo '$ac_cv_lib_'$ac_lib_var`\" = yes"; then
+  echo "$ac_t""yes" 1>&6
+  LIBS="$LIBS -ldir"
+else
+  echo "$ac_t""no" 1>&6
+fi
+
+else
+echo $ac_n "checking for opendir in -lx""... $ac_c" 1>&6
+echo "configure:1771: checking for opendir in -lx" >&5
+ac_lib_var=`echo x'_'opendir | sed 'y%./+-%__p_%'`
+if eval "test \"`echo '$''{'ac_cv_lib_$ac_lib_var'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  ac_save_LIBS="$LIBS"
+LIBS="-lx  $LIBS"
+cat > conftest.$ac_ext <<EOF
+#line 1779 "configure"
+#include "confdefs.h"
+/* Override any gcc2 internal prototype to avoid an error.  */
+/* We use char because int might match the return type of a gcc2
+    builtin and then its argument prototype would still apply.  */
+char opendir();
+
+int main() {
+opendir()
+; return 0; }
+EOF
+if { (eval echo configure:1790: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest; then
+  rm -rf conftest*
+  eval "ac_cv_lib_$ac_lib_var=yes"
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  eval "ac_cv_lib_$ac_lib_var=no"
+fi
+rm -f conftest*
+LIBS="$ac_save_LIBS"
+
+fi
+if eval "test \"`echo '$ac_cv_lib_'$ac_lib_var`\" = yes"; then
+  echo "$ac_t""yes" 1>&6
+  LIBS="$LIBS -lx"
+else
+  echo "$ac_t""no" 1>&6
+fi
+
+fi
+
+echo $ac_n "checking for ANSI C header files""... $ac_c" 1>&6
+echo "configure:1813: checking for ANSI C header files" >&5
+if eval "test \"`echo '$''{'ac_cv_header_stdc'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  cat > conftest.$ac_ext <<EOF
+#line 1818 "configure"
+#include "confdefs.h"
+#include <stdlib.h>
+#include <stdarg.h>
+#include <string.h>
+#include <float.h>
+EOF
+ac_try="$ac_cpp conftest.$ac_ext >/dev/null 2>conftest.out"
+{ (eval echo configure:1826: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; }
+ac_err=`grep -v '^ *+' conftest.out`
+if test -z "$ac_err"; then
+  rm -rf conftest*
+  ac_cv_header_stdc=yes
+else
+  echo "$ac_err" >&5
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  ac_cv_header_stdc=no
+fi
+rm -f conftest*
+
+if test $ac_cv_header_stdc = yes; then
+  # SunOS 4.x string.h does not declare mem*, contrary to ANSI.
+cat > conftest.$ac_ext <<EOF
+#line 1843 "configure"
+#include "confdefs.h"
+#include <string.h>
+EOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  egrep "memchr" >/dev/null 2>&1; then
+  :
+else
+  rm -rf conftest*
+  ac_cv_header_stdc=no
+fi
+rm -f conftest*
+
+fi
+
+if test $ac_cv_header_stdc = yes; then
+  # ISC 2.0.2 stdlib.h does not declare free, contrary to ANSI.
+cat > conftest.$ac_ext <<EOF
+#line 1861 "configure"
+#include "confdefs.h"
+#include <stdlib.h>
+EOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  egrep "free" >/dev/null 2>&1; then
+  :
+else
+  rm -rf conftest*
+  ac_cv_header_stdc=no
+fi
+rm -f conftest*
+
+fi
+
+if test $ac_cv_header_stdc = yes; then
+  # /bin/cc in Irix-4.0.5 gets non-ANSI ctype macros unless using -ansi.
+if test "$cross_compiling" = yes; then
+  :
+else
+  cat > conftest.$ac_ext <<EOF
+#line 1882 "configure"
+#include "confdefs.h"
+#include <ctype.h>
+#define ISLOWER(c) ('a' <= (c) && (c) <= 'z')
+#define TOUPPER(c) (ISLOWER(c) ? 'A' + ((c) - 'a') : (c))
+#define XOR(e, f) (((e) && !(f)) || (!(e) && (f)))
+int main () { int i; for (i = 0; i < 256; i++)
+if (XOR (islower (i), ISLOWER (i)) || toupper (i) != TOUPPER (i)) exit(2);
+exit (0); }
+
+EOF
+if { (eval echo configure:1893: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest && (./conftest; exit) 2>/dev/null
+then
+  :
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -fr conftest*
+  ac_cv_header_stdc=no
+fi
+rm -fr conftest*
+fi
+
+fi
+fi
+
+echo "$ac_t""$ac_cv_header_stdc" 1>&6
+if test $ac_cv_header_stdc = yes; then
+  cat >> confdefs.h <<\EOF
+#define STDC_HEADERS 1
+EOF
+
+fi
+
+for ac_hdr in fcntl.h limits.h sys/time.h unistd.h string.h memory.h sys/procfs.h
+do
+ac_safe=`echo "$ac_hdr" | sed 'y%./+-%__p_%'`
+echo $ac_n "checking for $ac_hdr""... $ac_c" 1>&6
+echo "configure:1920: checking for $ac_hdr" >&5
+if eval "test \"`echo '$''{'ac_cv_header_$ac_safe'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  cat > conftest.$ac_ext <<EOF
+#line 1925 "configure"
+#include "confdefs.h"
+#include <$ac_hdr>
+EOF
+ac_try="$ac_cpp conftest.$ac_ext >/dev/null 2>conftest.out"
+{ (eval echo configure:1930: \"$ac_try\") 1>&5; (eval $ac_try) 2>&5; }
+ac_err=`grep -v '^ *+' conftest.out`
+if test -z "$ac_err"; then
+  rm -rf conftest*
+  eval "ac_cv_header_$ac_safe=yes"
+else
+  echo "$ac_err" >&5
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  eval "ac_cv_header_$ac_safe=no"
+fi
+rm -f conftest*
+fi
+if eval "test \"`echo '$ac_cv_header_'$ac_safe`\" = yes"; then
+  echo "$ac_t""yes" 1>&6
+    ac_tr_hdr=HAVE_`echo $ac_hdr | sed 'y%abcdefghijklmnopqrstuvwxyz./-%ABCDEFGHIJKLMNOPQRSTUVWXYZ___%'`
+  cat >> confdefs.h <<EOF
+#define $ac_tr_hdr 1
+EOF
+ 
+else
+  echo "$ac_t""no" 1>&6
+fi
+done
+
+cat > conftest.$ac_ext <<EOF
+#line 1957 "configure"
+#include "confdefs.h"
+#include <stdio.h>
+EOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  egrep "fread" >/dev/null 2>&1; then
+  rm -rf conftest*
+  cat >> confdefs.h <<\EOF
+#define HAVE_FREAD_DECL 1
+EOF
+
+fi
+rm -f conftest*
+
+cat > conftest.$ac_ext <<EOF
+#line 1972 "configure"
+#include "confdefs.h"
+#include <stdio.h>
+EOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  egrep "fgetc" >/dev/null 2>&1; then
+  rm -rf conftest*
+  cat >> confdefs.h <<\EOF
+#define HAVE_FGETC_DECL 1
+EOF
+
+fi
+rm -f conftest*
+
+cat > conftest.$ac_ext <<EOF
+#line 1987 "configure"
+#include "confdefs.h"
+#include <sys/procfs.h>
+EOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  egrep "pr_brksize" >/dev/null 2>&1; then
+  rm -rf conftest*
+  cat >> confdefs.h <<\EOF
+#define HAVE_PR_BRKSIZE 1
+EOF
+
+fi
+rm -f conftest*
+
+
+# The Ultrix 4.2 mips builtin alloca declared by alloca.h only works
+# for constant arguments.  Useless!
+echo $ac_n "checking for working alloca.h""... $ac_c" 1>&6
+echo "configure:2005: checking for working alloca.h" >&5
+if eval "test \"`echo '$''{'ac_cv_header_alloca_h'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  cat > conftest.$ac_ext <<EOF
+#line 2010 "configure"
+#include "confdefs.h"
+#include <alloca.h>
+int main() {
+char *p = alloca(2 * sizeof(int));
+; return 0; }
+EOF
+if { (eval echo configure:2017: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest; then
+  rm -rf conftest*
+  ac_cv_header_alloca_h=yes
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  ac_cv_header_alloca_h=no
+fi
+rm -f conftest*
+fi
+
+echo "$ac_t""$ac_cv_header_alloca_h" 1>&6
+if test $ac_cv_header_alloca_h = yes; then
+  cat >> confdefs.h <<\EOF
+#define HAVE_ALLOCA_H 1
+EOF
+
+fi
+
+echo $ac_n "checking for alloca""... $ac_c" 1>&6
+echo "configure:2038: checking for alloca" >&5
+if eval "test \"`echo '$''{'ac_cv_func_alloca_works'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  cat > conftest.$ac_ext <<EOF
+#line 2043 "configure"
+#include "confdefs.h"
+
+#ifdef __GNUC__
+# define alloca __builtin_alloca
+#else
+# if HAVE_ALLOCA_H
+#  include <alloca.h>
+# else
+#  ifdef _AIX
+ #pragma alloca
+#  else
+#   ifndef alloca /* predefined by HP cc +Olibcalls */
+char *alloca ();
+#   endif
+#  endif
+# endif
+#endif
+
+int main() {
+char *p = (char *) alloca(1);
+; return 0; }
+EOF
+if { (eval echo configure:2066: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest; then
+  rm -rf conftest*
+  ac_cv_func_alloca_works=yes
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  ac_cv_func_alloca_works=no
+fi
+rm -f conftest*
+fi
+
+echo "$ac_t""$ac_cv_func_alloca_works" 1>&6
+if test $ac_cv_func_alloca_works = yes; then
+  cat >> confdefs.h <<\EOF
+#define HAVE_ALLOCA 1
+EOF
+
+fi
+
+if test $ac_cv_func_alloca_works = no; then
+  # The SVR3 libPW and SVR4 libucb both contain incompatible functions
+  # that cause trouble.  Some versions do not even contain alloca or
+  # contain a buggy version.  If you still want to use their alloca,
+  # use ar to extract alloca.o from them instead of compiling alloca.c.
+  ALLOCA=alloca.o
+  cat >> confdefs.h <<\EOF
+#define C_ALLOCA 1
+EOF
+
+
+echo $ac_n "checking whether alloca needs Cray hooks""... $ac_c" 1>&6
+echo "configure:2098: checking whether alloca needs Cray hooks" >&5
+if eval "test \"`echo '$''{'ac_cv_os_cray'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  cat > conftest.$ac_ext <<EOF
+#line 2103 "configure"
+#include "confdefs.h"
+#if defined(CRAY) && ! defined(CRAY2)
+webecray
+#else
+wenotbecray
+#endif
+
+EOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  egrep "webecray" >/dev/null 2>&1; then
+  rm -rf conftest*
+  ac_cv_os_cray=yes
+else
+  rm -rf conftest*
+  ac_cv_os_cray=no
+fi
+rm -f conftest*
+
+fi
+
+echo "$ac_t""$ac_cv_os_cray" 1>&6
+if test $ac_cv_os_cray = yes; then
+for ac_func in _getb67 GETB67 getb67; do
+  echo $ac_n "checking for $ac_func""... $ac_c" 1>&6
+echo "configure:2128: checking for $ac_func" >&5
+if eval "test \"`echo '$''{'ac_cv_func_$ac_func'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  cat > conftest.$ac_ext <<EOF
+#line 2133 "configure"
+#include "confdefs.h"
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char $ac_func(); below.  */
+#include <assert.h>
+/* Override any gcc2 internal prototype to avoid an error.  */
+/* We use char because int might match the return type of a gcc2
+    builtin and then its argument prototype would still apply.  */
+char $ac_func();
+
+int main() {
+
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_$ac_func) || defined (__stub___$ac_func)
+choke me
+#else
+$ac_func();
+#endif
+
+; return 0; }
+EOF
+if { (eval echo configure:2156: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest; then
+  rm -rf conftest*
+  eval "ac_cv_func_$ac_func=yes"
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  eval "ac_cv_func_$ac_func=no"
+fi
+rm -f conftest*
+fi
+
+if eval "test \"`echo '$ac_cv_func_'$ac_func`\" = yes"; then
+  echo "$ac_t""yes" 1>&6
+  cat >> confdefs.h <<EOF
+#define CRAY_STACKSEG_END $ac_func
+EOF
+
+  break
+else
+  echo "$ac_t""no" 1>&6
+fi
+
+done
+fi
+
+echo $ac_n "checking stack direction for C alloca""... $ac_c" 1>&6
+echo "configure:2183: checking stack direction for C alloca" >&5
+if eval "test \"`echo '$''{'ac_cv_c_stack_direction'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  if test "$cross_compiling" = yes; then
+  ac_cv_c_stack_direction=0
+else
+  cat > conftest.$ac_ext <<EOF
+#line 2191 "configure"
+#include "confdefs.h"
+find_stack_direction ()
+{
+  static char *addr = 0;
+  auto char dummy;
+  if (addr == 0)
+    {
+      addr = &dummy;
+      return find_stack_direction ();
+    }
+  else
+    return (&dummy > addr) ? 1 : -1;
+}
+main ()
+{
+  exit (find_stack_direction() < 0);
+}
+EOF
+if { (eval echo configure:2210: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest && (./conftest; exit) 2>/dev/null
+then
+  ac_cv_c_stack_direction=1
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -fr conftest*
+  ac_cv_c_stack_direction=-1
+fi
+rm -fr conftest*
+fi
+
+fi
+
+echo "$ac_t""$ac_cv_c_stack_direction" 1>&6
+cat >> confdefs.h <<EOF
+#define STACK_DIRECTION $ac_cv_c_stack_direction
+EOF
+
+fi
+
+if test $ac_cv_prog_gcc = yes; then
+    echo $ac_n "checking whether ${CC-cc} needs -traditional""... $ac_c" 1>&6
+echo "configure:2233: checking whether ${CC-cc} needs -traditional" >&5
+if eval "test \"`echo '$''{'ac_cv_prog_gcc_traditional'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+    ac_pattern="Autoconf.*'x'"
+  cat > conftest.$ac_ext <<EOF
+#line 2239 "configure"
+#include "confdefs.h"
+#include <sgtty.h>
+Autoconf TIOCGETP
+EOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  egrep "$ac_pattern" >/dev/null 2>&1; then
+  rm -rf conftest*
+  ac_cv_prog_gcc_traditional=yes
+else
+  rm -rf conftest*
+  ac_cv_prog_gcc_traditional=no
+fi
+rm -f conftest*
+
+
+  if test $ac_cv_prog_gcc_traditional = no; then
+    cat > conftest.$ac_ext <<EOF
+#line 2257 "configure"
+#include "confdefs.h"
+#include <termio.h>
+Autoconf TCGETA
+EOF
+if (eval "$ac_cpp conftest.$ac_ext") 2>&5 |
+  egrep "$ac_pattern" >/dev/null 2>&1; then
+  rm -rf conftest*
+  ac_cv_prog_gcc_traditional=yes
+fi
+rm -f conftest*
+
+  fi
+fi
+
+echo "$ac_t""$ac_cv_prog_gcc_traditional" 1>&6
+  if test $ac_cv_prog_gcc_traditional = yes; then
+    CC="$CC -traditional"
+  fi
+fi
+
+echo $ac_n "checking return type of signal handlers""... $ac_c" 1>&6
+echo "configure:2279: checking return type of signal handlers" >&5
+if eval "test \"`echo '$''{'ac_cv_type_signal'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  cat > conftest.$ac_ext <<EOF
+#line 2284 "configure"
+#include "confdefs.h"
+#include <sys/types.h>
+#include <signal.h>
+#ifdef signal
+#undef signal
+#endif
+#ifdef __cplusplus
+extern "C" void (*signal (int, void (*)(int)))(int);
+#else
+void (*signal ()) ();
+#endif
+
+int main() {
+int i;
+; return 0; }
+EOF
+if { (eval echo configure:2301: \"$ac_compile\") 1>&5; (eval $ac_compile) 2>&5; }; then
+  rm -rf conftest*
+  ac_cv_type_signal=void
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  ac_cv_type_signal=int
+fi
+rm -f conftest*
+fi
+
+echo "$ac_t""$ac_cv_type_signal" 1>&6
+cat >> confdefs.h <<EOF
+#define RETSIGTYPE $ac_cv_type_signal
+EOF
+
+
+echo $ac_n "checking for vprintf""... $ac_c" 1>&6
+echo "configure:2320: checking for vprintf" >&5
+if eval "test \"`echo '$''{'ac_cv_func_vprintf'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  cat > conftest.$ac_ext <<EOF
+#line 2325 "configure"
+#include "confdefs.h"
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char vprintf(); below.  */
+#include <assert.h>
+/* Override any gcc2 internal prototype to avoid an error.  */
+/* We use char because int might match the return type of a gcc2
+    builtin and then its argument prototype would still apply.  */
+char vprintf();
+
+int main() {
+
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_vprintf) || defined (__stub___vprintf)
+choke me
+#else
+vprintf();
+#endif
+
+; return 0; }
+EOF
+if { (eval echo configure:2348: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest; then
+  rm -rf conftest*
+  eval "ac_cv_func_vprintf=yes"
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  eval "ac_cv_func_vprintf=no"
+fi
+rm -f conftest*
+fi
+
+if eval "test \"`echo '$ac_cv_func_'vprintf`\" = yes"; then
+  echo "$ac_t""yes" 1>&6
+  cat >> confdefs.h <<\EOF
+#define HAVE_VPRINTF 1
+EOF
+
+else
+  echo "$ac_t""no" 1>&6
+fi
+
+if test "$ac_cv_func_vprintf" != yes; then
+echo $ac_n "checking for _doprnt""... $ac_c" 1>&6
+echo "configure:2372: checking for _doprnt" >&5
+if eval "test \"`echo '$''{'ac_cv_func__doprnt'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  cat > conftest.$ac_ext <<EOF
+#line 2377 "configure"
+#include "confdefs.h"
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char _doprnt(); below.  */
+#include <assert.h>
+/* Override any gcc2 internal prototype to avoid an error.  */
+/* We use char because int might match the return type of a gcc2
+    builtin and then its argument prototype would still apply.  */
+char _doprnt();
+
+int main() {
+
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub__doprnt) || defined (__stub____doprnt)
+choke me
+#else
+_doprnt();
+#endif
+
+; return 0; }
+EOF
+if { (eval echo configure:2400: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest; then
+  rm -rf conftest*
+  eval "ac_cv_func__doprnt=yes"
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  eval "ac_cv_func__doprnt=no"
+fi
+rm -f conftest*
+fi
+
+if eval "test \"`echo '$ac_cv_func_'_doprnt`\" = yes"; then
+  echo "$ac_t""yes" 1>&6
+  cat >> confdefs.h <<\EOF
+#define HAVE_DOPRNT 1
+EOF
+
+else
+  echo "$ac_t""no" 1>&6
+fi
+
+fi
+
+for ac_func in ftime select strftime strtol getrusage times mallinfo setbuffer getpagesize
+do
+echo $ac_n "checking for $ac_func""... $ac_c" 1>&6
+echo "configure:2427: checking for $ac_func" >&5
+if eval "test \"`echo '$''{'ac_cv_func_$ac_func'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  cat > conftest.$ac_ext <<EOF
+#line 2432 "configure"
+#include "confdefs.h"
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char $ac_func(); below.  */
+#include <assert.h>
+/* Override any gcc2 internal prototype to avoid an error.  */
+/* We use char because int might match the return type of a gcc2
+    builtin and then its argument prototype would still apply.  */
+char $ac_func();
+
+int main() {
+
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_$ac_func) || defined (__stub___$ac_func)
+choke me
+#else
+$ac_func();
+#endif
+
+; return 0; }
+EOF
+if { (eval echo configure:2455: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest; then
+  rm -rf conftest*
+  eval "ac_cv_func_$ac_func=yes"
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  eval "ac_cv_func_$ac_func=no"
+fi
+rm -f conftest*
+fi
+
+if eval "test \"`echo '$ac_cv_func_'$ac_func`\" = yes"; then
+  echo "$ac_t""yes" 1>&6
+    ac_tr_func=HAVE_`echo $ac_func | tr 'abcdefghijklmnopqrstuvwxyz' 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'`
+  cat >> confdefs.h <<EOF
+#define $ac_tr_func 1
+EOF
+ 
+else
+  echo "$ac_t""no" 1>&6
+fi
+done
+
+for ac_func in ftruncate strstr strcasecmp
+do
+echo $ac_n "checking for $ac_func""... $ac_c" 1>&6
+echo "configure:2482: checking for $ac_func" >&5
+if eval "test \"`echo '$''{'ac_cv_func_$ac_func'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  cat > conftest.$ac_ext <<EOF
+#line 2487 "configure"
+#include "confdefs.h"
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char $ac_func(); below.  */
+#include <assert.h>
+/* Override any gcc2 internal prototype to avoid an error.  */
+/* We use char because int might match the return type of a gcc2
+    builtin and then its argument prototype would still apply.  */
+char $ac_func();
+
+int main() {
+
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_$ac_func) || defined (__stub___$ac_func)
+choke me
+#else
+$ac_func();
+#endif
+
+; return 0; }
+EOF
+if { (eval echo configure:2510: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest; then
+  rm -rf conftest*
+  eval "ac_cv_func_$ac_func=yes"
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  eval "ac_cv_func_$ac_func=no"
+fi
+rm -f conftest*
+fi
+
+if eval "test \"`echo '$ac_cv_func_'$ac_func`\" = yes"; then
+  echo "$ac_t""yes" 1>&6
+    ac_tr_func=HAVE_`echo $ac_func | tr 'abcdefghijklmnopqrstuvwxyz' 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'`
+  cat >> confdefs.h <<EOF
+#define $ac_tr_func 1
+EOF
+ 
+else
+  echo "$ac_t""no" 1>&6
+LIBOBJS="$LIBOBJS ${ac_func}.o"
+fi
+done
+
+
+
+
+# *** Custom checking (based on GNU tar configure.in) ***
+# ---------------------------------------------------------------------------
+echo $ac_n "checking for HP-UX needing gmalloc""... $ac_c" 1>&6
+echo "configure:2541: checking for HP-UX needing gmalloc" >&5
+if test "`(uname -s) 2> /dev/null`" = 'HP-UX'; then
+  echo "$ac_t""yes" 1>&6
+  LIBOBJS="$LIBOBJS gmalloc.o"
+  cat >> confdefs.h <<\EOF
+#define HAVE_VALLOC 1
+EOF
+
+else
+  echo "$ac_t""no" 1>&6
+  for ac_func in valloc
+do
+echo $ac_n "checking for $ac_func""... $ac_c" 1>&6
+echo "configure:2554: checking for $ac_func" >&5
+if eval "test \"`echo '$''{'ac_cv_func_$ac_func'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  cat > conftest.$ac_ext <<EOF
+#line 2559 "configure"
+#include "confdefs.h"
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char $ac_func(); below.  */
+#include <assert.h>
+/* Override any gcc2 internal prototype to avoid an error.  */
+/* We use char because int might match the return type of a gcc2
+    builtin and then its argument prototype would still apply.  */
+char $ac_func();
+
+int main() {
+
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_$ac_func) || defined (__stub___$ac_func)
+choke me
+#else
+$ac_func();
+#endif
+
+; return 0; }
+EOF
+if { (eval echo configure:2582: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest; then
+  rm -rf conftest*
+  eval "ac_cv_func_$ac_func=yes"
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  eval "ac_cv_func_$ac_func=no"
+fi
+rm -f conftest*
+fi
+
+if eval "test \"`echo '$ac_cv_func_'$ac_func`\" = yes"; then
+  echo "$ac_t""yes" 1>&6
+    ac_tr_func=HAVE_`echo $ac_func | tr 'abcdefghijklmnopqrstuvwxyz' 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'`
+  cat >> confdefs.h <<EOF
+#define $ac_tr_func 1
+EOF
+ 
+else
+  echo "$ac_t""no" 1>&6
+fi
+done
+
+fi
+
+echo $ac_n "checking if malloc debugging is wanted""... $ac_c" 1>&6
+echo "configure:2609: checking if malloc debugging is wanted" >&5
+# Check whether --with-dmalloc or --without-dmalloc was given.
+if test "${with_dmalloc+set}" = set; then
+  withval="$with_dmalloc"
+  if test "$withval" = yes; then
+  echo "$ac_t""yes" 1>&6
+  cat >> confdefs.h <<\EOF
+#define WITH_DMALLOC 1
+EOF
+
+  LIBS="$LIBS -ldmalloc"
+  LDFLAGS="$LDFLAGS -g"
+else
+  echo "$ac_t""no" 1>&6
+fi
+else
+  echo "$ac_t""no" 1>&6
+fi
+
+
+echo $ac_n "checking which of rx or regex is wanted""... $ac_c" 1>&6
+echo "configure:2630: checking which of rx or regex is wanted" >&5
+# Check whether --with-regex or --without-regex was given.
+if test "${with_regex+set}" = set; then
+  withval="$with_regex"
+  if test "$withval" = yes; then
+  ac_with_regex=1
+  echo "$ac_t""regex" 1>&6
+  cat >> confdefs.h <<\EOF
+#define WITH_REGEX 1
+EOF
+
+  LIBOBJS="$LIBOBJS regex.o"
+fi
+fi
+
+if test -z "$ac_with_regex"; then
+  echo "$ac_t""rx" 1>&6
+  echo $ac_n "checking for re_rx_search""... $ac_c" 1>&6
+echo "configure:2648: checking for re_rx_search" >&5
+if eval "test \"`echo '$''{'ac_cv_func_re_rx_search'+set}'`\" = set"; then
+  echo $ac_n "(cached) $ac_c" 1>&6
+else
+  cat > conftest.$ac_ext <<EOF
+#line 2653 "configure"
+#include "confdefs.h"
+/* System header to define __stub macros and hopefully few prototypes,
+    which can conflict with char re_rx_search(); below.  */
+#include <assert.h>
+/* Override any gcc2 internal prototype to avoid an error.  */
+/* We use char because int might match the return type of a gcc2
+    builtin and then its argument prototype would still apply.  */
+char re_rx_search();
+
+int main() {
+
+/* The GNU C library defines this for functions which it implements
+    to always fail with ENOSYS.  Some functions are actually named
+    something starting with __ and the normal name is an alias.  */
+#if defined (__stub_re_rx_search) || defined (__stub___re_rx_search)
+choke me
+#else
+re_rx_search();
+#endif
+
+; return 0; }
+EOF
+if { (eval echo configure:2676: \"$ac_link\") 1>&5; (eval $ac_link) 2>&5; } && test -s conftest; then
+  rm -rf conftest*
+  eval "ac_cv_func_re_rx_search=yes"
+else
+  echo "configure: failed program was:" >&5
+  cat conftest.$ac_ext >&5
+  rm -rf conftest*
+  eval "ac_cv_func_re_rx_search=no"
+fi
+rm -f conftest*
+fi
+
+if eval "test \"`echo '$ac_cv_func_'re_rx_search`\" = yes"; then
+  echo "$ac_t""yes" 1>&6
+  :
+else
+  echo "$ac_t""no" 1>&6
+LIBOBJS="$LIBOBJS rx.o"
+fi
+
+fi
+
+
+# ---------------------------------------------------------------------------
+if test "$ac_cv_func_alloca" = 'no'; then
+  LIBOBJS="$LIBOBJS xmalloc.o error.o"
+fi
+
+
+# ---------------------------------------------------------------------------
+
+trap '' 1 2 15
+cat > confcache <<\EOF
+# This file is a shell script that caches the results of configure
+# tests run on this system so they can be shared between configure
+# scripts and configure runs.  It is not useful on other systems.
+# If it contains results you don't want to keep, you may remove or edit it.
+#
+# By default, configure uses ./config.cache as the cache file,
+# creating it if it does not exist already.  You can give configure
+# the --cache-file=FILE option to use a different cache file; that is
+# what configure does when it calls configure scripts in
+# subdirectories, so they share the cache.
+# Giving --cache-file=/dev/null disables caching, for debugging configure.
+# config.status only pays attention to the cache file if you give it the
+# --recheck option to rerun configure.
+#
+EOF
+# The following way of writing the cache mishandles newlines in values,
+# but we know of no workaround that is simple, portable, and efficient.
+# So, don't put newlines in cache variables' values.
+# Ultrix sh set writes to stderr and can't be redirected directly,
+# and sets the high bit in the cache file unless we assign to the vars.
+(set) 2>&1 |
+  case `(ac_space=' '; set) 2>&1` in
+  *ac_space=\ *)
+    # `set' does not quote correctly, so add quotes (double-quote substitution
+    # turns \\\\ into \\, and sed turns \\ into \).
+    sed -n \
+      -e "s/'/'\\\\''/g" \
+      -e "s/^\\([a-zA-Z0-9_]*_cv_[a-zA-Z0-9_]*\\)=\\(.*\\)/\\1=\${\\1='\\2'}/p"
+    ;;
+  *)
+    # `set' quotes correctly as required by POSIX, so do not add quotes.
+    sed -n -e 's/^\([a-zA-Z0-9_]*_cv_[a-zA-Z0-9_]*\)=\(.*\)/\1=${\1=\2}/p'
+    ;;
+  esac >> confcache
+if cmp -s $cache_file confcache; then
+  :
+else
+  if test -w $cache_file; then
+    echo "updating cache $cache_file"
+    cat confcache > $cache_file
+  else
+    echo "not updating unwritable cache $cache_file"
+  fi
+fi
+rm -f confcache
+
+trap 'rm -fr conftest* confdefs* core core.* *.core $ac_clean_files; exit 1' 1 2 15
+
+test "x$prefix" = xNONE && prefix=$ac_default_prefix
+# Let make expand exec_prefix.
+test "x$exec_prefix" = xNONE && exec_prefix='${prefix}'
+
+# Any assignment to VPATH causes Sun make to only execute
+# the first set of double-colon rules, so remove it if not needed.
+# If there is a colon in the path, we need to keep it.
+if test "x$srcdir" = x.; then
+  ac_vpsub='/^[ 	]*VPATH[ 	]*=[^:]*$/d'
+fi
+
+trap 'rm -f $CONFIG_STATUS conftest*; exit 1' 1 2 15
+
+DEFS=-DHAVE_CONFIG_H
+
+# Without the "./", some shells look in PATH for config.status.
+: ${CONFIG_STATUS=./config.status}
+
+echo creating $CONFIG_STATUS
+rm -f $CONFIG_STATUS
+cat > $CONFIG_STATUS <<EOF
+#! /bin/sh
+# Generated automatically by configure.
+# Run this file to recreate the current configuration.
+# This directory was configured as follows,
+# on host `(hostname || uname -n) 2>/dev/null | sed 1q`:
+#
+# $0 $ac_configure_args
+#
+# Compiler output produced by configure, useful for debugging
+# configure, is in ./config.log if it exists.
+
+ac_cs_usage="Usage: $CONFIG_STATUS [--recheck] [--version] [--help]"
+for ac_option
+do
+  case "\$ac_option" in
+  -recheck | --recheck | --rechec | --reche | --rech | --rec | --re | --r)
+    echo "running \${CONFIG_SHELL-/bin/sh} $0 $ac_configure_args --no-create --no-recursion"
+    exec \${CONFIG_SHELL-/bin/sh} $0 $ac_configure_args --no-create --no-recursion ;;
+  -version | --version | --versio | --versi | --vers | --ver | --ve | --v)
+    echo "$CONFIG_STATUS generated by autoconf version 2.12"
+    exit 0 ;;
+  -help | --help | --hel | --he | --h)
+    echo "\$ac_cs_usage"; exit 0 ;;
+  *) echo "\$ac_cs_usage"; exit 1 ;;
+  esac
+done
+
+ac_given_srcdir=$srcdir
+ac_given_INSTALL="$INSTALL"
+
+trap 'rm -fr `echo "Makefile src/text/Makefile src/images/Makefile 
+          src/scripts/Makefile lib/Makefile  config.h" | sed "s/:[^ ]*//g"` conftest*; exit 1' 1 2 15
+EOF
+cat >> $CONFIG_STATUS <<EOF
+
+# Protect against being on the right side of a sed subst in config.status.
+sed 's/%@/@@/; s/@%/@@/; s/%g\$/@g/; /@g\$/s/[\\\\&%]/\\\\&/g;
+ s/@@/%@/; s/@@/@%/; s/@g\$/%g/' > conftest.subs <<\\CEOF
+$ac_vpsub
+$extrasub
+s%@CFLAGS@%$CFLAGS%g
+s%@CPPFLAGS@%$CPPFLAGS%g
+s%@CXXFLAGS@%$CXXFLAGS%g
+s%@DEFS@%$DEFS%g
+s%@LDFLAGS@%$LDFLAGS%g
+s%@LIBS@%$LIBS%g
+s%@exec_prefix@%$exec_prefix%g
+s%@prefix@%$prefix%g
+s%@program_transform_name@%$program_transform_name%g
+s%@bindir@%$bindir%g
+s%@sbindir@%$sbindir%g
+s%@libexecdir@%$libexecdir%g
+s%@datadir@%$datadir%g
+s%@sysconfdir@%$sysconfdir%g
+s%@sharedstatedir@%$sharedstatedir%g
+s%@localstatedir@%$localstatedir%g
+s%@libdir@%$libdir%g
+s%@includedir@%$includedir%g
+s%@oldincludedir@%$oldincludedir%g
+s%@infodir@%$infodir%g
+s%@mandir@%$mandir%g
+s%@PACKAGE@%$PACKAGE%g
+s%@VERSION@%$VERSION%g
+s%@CXX@%$CXX%g
+s%@AWK@%$AWK%g
+s%@YACC@%$YACC%g
+s%@CC@%$CC%g
+s%@INSTALL_PROGRAM@%$INSTALL_PROGRAM%g
+s%@INSTALL_DATA@%$INSTALL_DATA%g
+s%@LN_S@%$LN_S%g
+s%@SET_MAKE@%$SET_MAKE%g
+s%@RANLIB@%$RANLIB%g
+s%@CPP@%$CPP%g
+s%@U@%$U%g
+s%@ANSI2KNR@%$ANSI2KNR%g
+s%@ALLOCA@%$ALLOCA%g
+s%@LIBOBJS@%$LIBOBJS%g
+
+CEOF
+EOF
+
+cat >> $CONFIG_STATUS <<\EOF
+
+# Split the substitutions into bite-sized pieces for seds with
+# small command number limits, like on Digital OSF/1 and HP-UX.
+ac_max_sed_cmds=90 # Maximum number of lines to put in a sed script.
+ac_file=1 # Number of current file.
+ac_beg=1 # First line for current file.
+ac_end=$ac_max_sed_cmds # Line after last line for current file.
+ac_more_lines=:
+ac_sed_cmds=""
+while $ac_more_lines; do
+  if test $ac_beg -gt 1; then
+    sed "1,${ac_beg}d; ${ac_end}q" conftest.subs > conftest.s$ac_file
+  else
+    sed "${ac_end}q" conftest.subs > conftest.s$ac_file
+  fi
+  if test ! -s conftest.s$ac_file; then
+    ac_more_lines=false
+    rm -f conftest.s$ac_file
+  else
+    if test -z "$ac_sed_cmds"; then
+      ac_sed_cmds="sed -f conftest.s$ac_file"
+    else
+      ac_sed_cmds="$ac_sed_cmds | sed -f conftest.s$ac_file"
+    fi
+    ac_file=`expr $ac_file + 1`
+    ac_beg=$ac_end
+    ac_end=`expr $ac_end + $ac_max_sed_cmds`
+  fi
+done
+if test -z "$ac_sed_cmds"; then
+  ac_sed_cmds=cat
+fi
+EOF
+
+cat >> $CONFIG_STATUS <<EOF
+
+CONFIG_FILES=\${CONFIG_FILES-"Makefile src/text/Makefile src/images/Makefile 
+          src/scripts/Makefile lib/Makefile "}
+EOF
+cat >> $CONFIG_STATUS <<\EOF
+for ac_file in .. $CONFIG_FILES; do if test "x$ac_file" != x..; then
+  # Support "outfile[:infile[:infile...]]", defaulting infile="outfile.in".
+  case "$ac_file" in
+  *:*) ac_file_in=`echo "$ac_file"|sed 's%[^:]*:%%'`
+       ac_file=`echo "$ac_file"|sed 's%:.*%%'` ;;
+  *) ac_file_in="${ac_file}.in" ;;
+  esac
+
+  # Adjust a relative srcdir, top_srcdir, and INSTALL for subdirectories.
+
+  # Remove last slash and all that follows it.  Not all systems have dirname.
+  ac_dir=`echo $ac_file|sed 's%/[^/][^/]*$%%'`
+  if test "$ac_dir" != "$ac_file" && test "$ac_dir" != .; then
+    # The file is in a subdirectory.
+    test ! -d "$ac_dir" && mkdir "$ac_dir"
+    ac_dir_suffix="/`echo $ac_dir|sed 's%^\./%%'`"
+    # A "../" for each directory in $ac_dir_suffix.
+    ac_dots=`echo $ac_dir_suffix|sed 's%/[^/]*%../%g'`
+  else
+    ac_dir_suffix= ac_dots=
+  fi
+
+  case "$ac_given_srcdir" in
+  .)  srcdir=.
+      if test -z "$ac_dots"; then top_srcdir=.
+      else top_srcdir=`echo $ac_dots|sed 's%/$%%'`; fi ;;
+  /*) srcdir="$ac_given_srcdir$ac_dir_suffix"; top_srcdir="$ac_given_srcdir" ;;
+  *) # Relative path.
+    srcdir="$ac_dots$ac_given_srcdir$ac_dir_suffix"
+    top_srcdir="$ac_dots$ac_given_srcdir" ;;
+  esac
+
+  case "$ac_given_INSTALL" in
+  [/$]*) INSTALL="$ac_given_INSTALL" ;;
+  *) INSTALL="$ac_dots$ac_given_INSTALL" ;;
+  esac
+
+  echo creating "$ac_file"
+  rm -f "$ac_file"
+  configure_input="Generated automatically from `echo $ac_file_in|sed 's%.*/%%'` by configure."
+  case "$ac_file" in
+  *Makefile*) ac_comsub="1i\\
+# $configure_input" ;;
+  *) ac_comsub= ;;
+  esac
+
+  ac_file_inputs=`echo $ac_file_in|sed -e "s%^%$ac_given_srcdir/%" -e "s%:% $ac_given_srcdir/%g"`
+  sed -e "$ac_comsub
+s%@configure_input@%$configure_input%g
+s%@srcdir@%$srcdir%g
+s%@top_srcdir@%$top_srcdir%g
+s%@INSTALL@%$INSTALL%g
+" $ac_file_inputs | (eval "$ac_sed_cmds") > $ac_file
+fi; done
+rm -f conftest.s*
+
+# These sed commands are passed to sed as "A NAME B NAME C VALUE D", where
+# NAME is the cpp macro being defined and VALUE is the value it is being given.
+#
+# ac_d sets the value in "#define NAME VALUE" lines.
+ac_dA='s%^\([ 	]*\)#\([ 	]*define[ 	][ 	]*\)'
+ac_dB='\([ 	][ 	]*\)[^ 	]*%\1#\2'
+ac_dC='\3'
+ac_dD='%g'
+# ac_u turns "#undef NAME" with trailing blanks into "#define NAME VALUE".
+ac_uA='s%^\([ 	]*\)#\([ 	]*\)undef\([ 	][ 	]*\)'
+ac_uB='\([ 	]\)%\1#\2define\3'
+ac_uC=' '
+ac_uD='\4%g'
+# ac_e turns "#undef NAME" without trailing blanks into "#define NAME VALUE".
+ac_eA='s%^\([ 	]*\)#\([ 	]*\)undef\([ 	][ 	]*\)'
+ac_eB='$%\1#\2define\3'
+ac_eC=' '
+ac_eD='%g'
+
+if test "${CONFIG_HEADERS+set}" != set; then
+EOF
+cat >> $CONFIG_STATUS <<EOF
+  CONFIG_HEADERS="config.h"
+EOF
+cat >> $CONFIG_STATUS <<\EOF
+fi
+for ac_file in .. $CONFIG_HEADERS; do if test "x$ac_file" != x..; then
+  # Support "outfile[:infile[:infile...]]", defaulting infile="outfile.in".
+  case "$ac_file" in
+  *:*) ac_file_in=`echo "$ac_file"|sed 's%[^:]*:%%'`
+       ac_file=`echo "$ac_file"|sed 's%:.*%%'` ;;
+  *) ac_file_in="${ac_file}.in" ;;
+  esac
+
+  echo creating $ac_file
+
+  rm -f conftest.frag conftest.in conftest.out
+  ac_file_inputs=`echo $ac_file_in|sed -e "s%^%$ac_given_srcdir/%" -e "s%:% $ac_given_srcdir/%g"`
+  cat $ac_file_inputs > conftest.in
+
+EOF
+
+# Transform confdefs.h into a sed script conftest.vals that substitutes
+# the proper values into config.h.in to produce config.h.  And first:
+# Protect against being on the right side of a sed subst in config.status.
+# Protect against being in an unquoted here document in config.status.
+rm -f conftest.vals
+cat > conftest.hdr <<\EOF
+s/[\\&%]/\\&/g
+s%[\\$`]%\\&%g
+s%#define \([A-Za-z_][A-Za-z0-9_]*\) *\(.*\)%${ac_dA}\1${ac_dB}\1${ac_dC}\2${ac_dD}%gp
+s%ac_d%ac_u%gp
+s%ac_u%ac_e%gp
+EOF
+sed -n -f conftest.hdr confdefs.h > conftest.vals
+rm -f conftest.hdr
+
+# This sed command replaces #undef with comments.  This is necessary, for
+# example, in the case of _POSIX_SOURCE, which is predefined and required
+# on some systems where configure will not decide to define it.
+cat >> conftest.vals <<\EOF
+s%^[ 	]*#[ 	]*undef[ 	][ 	]*[a-zA-Z_][a-zA-Z_0-9]*%/* & */%
+EOF
+
+# Break up conftest.vals because some shells have a limit on
+# the size of here documents, and old seds have small limits too.
+
+rm -f conftest.tail
+while :
+do
+  ac_lines=`grep -c . conftest.vals`
+  # grep -c gives empty output for an empty file on some AIX systems.
+  if test -z "$ac_lines" || test "$ac_lines" -eq 0; then break; fi
+  # Write a limited-size here document to conftest.frag.
+  echo '  cat > conftest.frag <<CEOF' >> $CONFIG_STATUS
+  sed ${ac_max_here_lines}q conftest.vals >> $CONFIG_STATUS
+  echo 'CEOF
+  sed -f conftest.frag conftest.in > conftest.out
+  rm -f conftest.in
+  mv conftest.out conftest.in
+' >> $CONFIG_STATUS
+  sed 1,${ac_max_here_lines}d conftest.vals > conftest.tail
+  rm -f conftest.vals
+  mv conftest.tail conftest.vals
+done
+rm -f conftest.vals
+
+cat >> $CONFIG_STATUS <<\EOF
+  rm -f conftest.frag conftest.h
+  echo "/* $ac_file.  Generated automatically by configure.  */" > conftest.h
+  cat conftest.in >> conftest.h
+  rm -f conftest.in
+  if cmp -s $ac_file conftest.h 2>/dev/null; then
+    echo "$ac_file is unchanged"
+    rm -f conftest.h
+  else
+    # Remove last slash and all that follows it.  Not all systems have dirname.
+      ac_dir=`echo $ac_file|sed 's%/[^/][^/]*$%%'`
+      if test "$ac_dir" != "$ac_file" && test "$ac_dir" != .; then
+      # The file is in a subdirectory.
+      test ! -d "$ac_dir" && mkdir "$ac_dir"
+    fi
+    rm -f $ac_file
+    mv conftest.h $ac_file
+  fi
+fi; done
+
+EOF
+cat >> $CONFIG_STATUS <<EOF
+
+EOF
+cat >> $CONFIG_STATUS <<\EOF
+test -z "$CONFIG_HEADERS" || echo timestamp > stamp-h
+exit 0
+EOF
+chmod +x $CONFIG_STATUS
+rm -fr confdefs* $ac_clean_files
+test "$no_create" = yes || ${CONFIG_SHELL-/bin/sh} $CONFIG_STATUS || exit 1
+
Index: /tags/gsdl-21/gsdl/packages/mg/configure.in
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/configure.in	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/configure.in	(revision 699)
@@ -0,0 +1,98 @@
+dnl Process this file with autoconf to produce a configure script.
+AC_INIT(src/text/mgquery.c)
+AC_CONFIG_HEADER(config.h)
+AC_ARG_PROGRAM
+
+dnl
+PACKAGE=mg
+VERSION=1.2
+
+AC_DEFINE_UNQUOTED(PACKAGE, "$PACKAGE")
+AC_DEFINE_UNQUOTED(VERSION, "$VERSION")
+AC_SUBST(PACKAGE)
+AC_SUBST(VERSION)
+
+dnl Checks for programs.
+AC_PROG_CXX
+AC_PROG_AWK
+AC_PROG_YACC
+AC_PROG_CC
+AC_PROG_INSTALL
+AC_PROG_LN_S
+AC_PROG_MAKE_SET
+AC_PROG_RANLIB
+
+dnl Checks for typedefs, structures, and compiler characteristics.
+AC_AIX
+AC_ISC_POSIX
+AC_MINIX
+fp_C_PROTOTYPES
+AC_C_CONST
+AC_TYPE_OFF_T
+AC_TYPE_SIZE_T
+AC_HEADER_TIME
+AC_STRUCT_TM
+
+dnl --- This should be in the stdc checking -----------------
+if test "$ac_cv_prog_cc_stdc" = '-Xc'; then
+dnl The Pyramids running SMP_DC.OSx on MIPS r3000 chips with
+dnl /usr/opt/cc3.11/bin/cc treat -Xc as full ANSI mode - no POSIX or
+dnl XOPEN extensions (like popen).
+dnl Unfortunately -D_{XOPEN,POSIX}_SOURCE can't provide all that mg
+dnl needs. So drop back to -Xa
+AC_TRY_COMPILE(
+[#include <stdio.h>
+], [ FILE *rubbish = popen("yes", "r"); ],,
+[CC="`echo $CC | sed 's/-Xc/-Xa/'`"    ac_cv_prog_cc_stdc='-Xa'])
+fi
+dnl ----------------------
+
+
+dnl Checks for libraries.
+dnl Replace `main' with a function in -lm:
+AC_CHECK_LIB(m, main)
+
+dnl Checks for header files.
+AC_HEADER_DIRENT
+AC_HEADER_STDC
+AC_CHECK_HEADERS(fcntl.h limits.h sys/time.h unistd.h string.h memory.h sys/procfs.h)
+AC_EGREP_HEADER(fread, stdio.h, AC_DEFINE(HAVE_FREAD_DECL))
+AC_EGREP_HEADER(fgetc, stdio.h, AC_DEFINE(HAVE_FGETC_DECL))
+AC_EGREP_HEADER(pr_brksize, sys/procfs.h, AC_DEFINE(HAVE_PR_BRKSIZE))
+
+dnl Checks for library functions.
+AC_FUNC_ALLOCA
+AC_PROG_GCC_TRADITIONAL
+AC_TYPE_SIGNAL
+AC_FUNC_VPRINTF
+AC_CHECK_FUNCS(ftime select strftime strtol getrusage times mallinfo setbuffer getpagesize)
+AC_REPLACE_FUNCS(ftruncate strstr strcasecmp)
+
+
+# *** Custom checking (based on GNU tar configure.in) ***
+# ---------------------------------------------------------------------------
+AC_MSG_CHECKING(for HP-UX needing gmalloc)
+if test "`(uname -s) 2> /dev/null`" = 'HP-UX'; then
+  AC_MSG_RESULT(yes)
+  LIBOBJS="$LIBOBJS gmalloc.o"
+  AC_DEFINE(HAVE_VALLOC)
+else
+  AC_MSG_RESULT(no)
+  AC_CHECK_FUNCS(valloc)
+fi
+
+fp_WITH_DMALLOC
+fp_WITH_REGEX
+
+# ---------------------------------------------------------------------------
+if test "$ac_cv_func_alloca" = 'no'; then
+dnl    support functions for alloca.c
+  LIBOBJS="$LIBOBJS xmalloc.o error.o"
+fi
+
+
+# ---------------------------------------------------------------------------
+
+AC_OUTPUT([Makefile src/text/Makefile src/images/Makefile 
+          src/scripts/Makefile lib/Makefile ], 
+[test -z "$CONFIG_HEADERS" || echo timestamp > stamp-h])
Index: /tags/gsdl-21/gsdl/packages/mg/docs/Makefile
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/docs/Makefile	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/docs/Makefile	(revision 699)
@@ -0,0 +1,5 @@
+DISTFILES = Makefile mgmerge.README mg_invf_dump.changes \
+  README.man mg.1 mgintro.1 mgintro++.1
+
+dist:
+	ln $(DISTFILES) $(distdir)
Index: /tags/gsdl-21/gsdl/packages/mg/docs/README.man
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/docs/README.man	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/docs/README.man	(revision 699)
@@ -0,0 +1,1 @@
+The rest of the man files are in the respective src directories.
Index: /tags/gsdl-21/gsdl/packages/mg/docs/mg.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/docs/mg.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/docs/mg.1	(revision 699)
@@ -0,0 +1,462 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg 1 \*(Dt CITRI
+.\"=====================================================================
+.\" Author:
+.\"     Nelson H. F. Beebe
+.\"     Center for Scientific Computing
+.\"     Department of Mathematics
+.\"     University of Utah
+.\"     Salt Lake City, UT 84112
+.\"     USA
+.\"     Email: beebe@math.utah.edu (Internet)
+.\"=====================================================================
+.if t .ds Bi B\s-2IB\s+2T\\h'-0.1667m'\\v'0.20v'E\\v'-0.20v'\\h'-0.125m'X
+.if n .ds Bi BibTeX
+.if t .ds Te T\\h'-0.1667m'\\v'0.20v'E\\v'-0.20v'\\h'-0.125m'X
+.if n .ds Te TeX
+.\"=====================================================================
+.SH NAME
+mg \- full-text inverted index support
+.\"=====================================================================
+.SH DESCRIPTION
+.B mg
+is a suite of programs that can be used to create
+and query full-text inverted indexes for a
+collection of documents.
+.PP
+An inverted index is essentially a list of
+pointers to occurrences of every word in a
+collection of documents, so that, for example, in
+a collection of Shakespeare's works, one can pose
+questions like
+.I "How often is a fool mentioned in the plays?"
+and
+.I "Are Romeo and Juliet ever mentioned in the same sentence?"
+While search utilities like the UNIX
+.BR grep (1)
+family can be used for simple queries, they suffer
+from several limitations:
+.TP \w'\(bu'u+2n
+\(bu
+Each invocation requires a separate pass over the
+documents, which becomes impossibly slow if the
+document collection is very large.
+.TP
+\(bu
+It is difficult to compose Boolean queries like
+.IR "Caesar AND Brutus AND NOT Antony" .
+.TP
+\(bu
+They provide no mechanism for ranking matches
+according to importance, which is extremely
+important when queries are complex and numerous
+matches are found.
+.TP
+\(bu
+They provide no mechanism for displaying
+surrounding context (although the Free Software
+Foundation
+implementations for the GNU Project remedy this
+with their
+.BI \-A " num"
+(after)
+and
+.BI \-B " num"
+(before)
+switches).
+.TP
+\(bu
+They provide no mechanism for searching for
+occurrences in the same paragraph, unless
+preprocessing is done on the files to wrap
+paragraphs into long lines.  Even that will often
+fail, because input buffer sizes may be limited to
+a few hundred characters.
+.TP
+\(bu
+They provide no easy way to deal with grammatical
+word-ending variants, except by explicit
+enumeration, such as searching for
+.IR compress ,
+.IR compressed ,
+.IR compresses ,
+.IR compressing ,
+and
+.IR compression .
+.PP
+Most computer users have been faced with the
+problem of finding information from a large
+collection of files, such as electronic mail,
+on-line documentation, or source code.  Inverted
+indices provide a highly-effective solution to
+this problem.
+.PP
+The
+.B mg
+software is described in Appendix A of the book
+.RS
+.nf
+Ian H. Witten, Alistair Moffat, and Timothy C. Bell
+.I "Managing Gigabytes: Compressing and Indexing Documents and Images"
+Van Nostrand Reinhold
+1994
+xiv + 429 pages
+US$54.95
+ISBN 0-442-01863-0
+Library of Congress catalog number TA1637 .W58 1994
+.fi
+.RE
+.PP
+Many of the algorithms implemented in
+.B mg
+represent significant advances over previous work,
+both in speed, and in storage requirements.  On a
+fast workstation, in tens of minutes, or a few
+hours,
+.BR mgbuild (1)
+can create an index to
+.I all
+of the words in hundreds of thousands of documents
+occupying hundreds of megabytes, or even more than
+a gigabyte, of disk space.
+.BR mgquery (1)
+can then be used to answer complex queries, with
+responses often returned in a second or less.
+.B mg
+also contains algorithms to deal with images, so
+that with a small amount of descriptive text for
+each image, it is possible to do searches in
+collections of images, and to have retrievals
+display the images using a viewer like
+.BR xv (1).
+.PP
+.B mg
+can deal with compressed text and image files and
+surprisingly, it usually runs faster than it would
+if the files were not compressed!  Thus, the
+considerable disk space savings possible from
+file compression are not lost because of the need
+for fast document search and retrieval.
+.PP
+The Free Software Foundation GNU Project
+compression utilities
+.BR gzip (1)
+and
+.BR gunzip (1)
+are recommended for general use over older
+alternatives, like
+.BR compress (1),
+because of their speed and high compression
+ratios.
+.\"=====================================================================
+.SH AVAILABILITY
+The
+.B mg
+software can be obtained via anonymous ftp to the
+Australian archive host
+.I "munnari.oz.au [128.250.1.21]"
+from the directory
+.IR "/pub/mg" .
+.\"=====================================================================
+.SH "TYPICAL USE OF mg"
+Although
+.B mg
+consists of more than 20 separate programs, many
+of which have complicated command-line options,
+take heart: most users require only two or three
+of these programs, and nothing more than a
+document name on the command line.
+.PP
+A
+.I document
+for
+.B mg
+is a fragment of text suitable for retrieval as a
+unit when it is found to contain a requested word,
+or words.  In a collection of poetry, a document
+might be a stanza, while in a novel, it could be a
+paragraph.  In an index of first lines of poems, a
+document would likely be just a single line.
+.PP
+Just what constitutes a document is decided by a
+user-modifiable UNIX shell script,
+.BR mg_get (1).
+The default script provided with the
+.B mg
+source distribution knows about these named
+document collections:
+.TP \w'mailfiles'u+2n
+.I alice
+Lewis Carroll's
+.I "Alice in Wonderland"
+book.
+.TP
+.I allfiles
+all mail files in the directory tree
+.IR $HOME/Mail ,
+including all of its nested subdirectories.
+.TP
+.I mailfiles
+individual mail messages in
+.IR $HOME/mbox
+and
+.IR $HOME/.sentmail.
+.TP
+.I davinci
+A small collection of text and images from the
+work of Leonardo da Vinci.
+.PP
+A document collection name is used by
+.B mg
+as a
+.BR csh (1)
+.I case
+statement selector, and as a subdirectory name in the
+.B $MGDATA
+directory, or the current directory, if
+.B MGDATA
+is not defined.
+.\"=====================================================================
+.SH "EXTENDING mg_get"
+This section describes how to extend
+.BR mg_get (1)
+to handle a new document collection.
+.PP
+Let us take two examples: all \*(Bi
+.I .bib
+files, and all \*(Te files, contained in
+subdirectories under the login directory.
+.PP
+For \*(Bi, each bibliographic entry will be
+considered a separate document.  In order to
+facilitate easy identification of entries, we
+shall require them to begin at the start of a
+line; the
+.BR bibclean (1)
+utility can be used to standardize the format of
+.I .bib
+files, and to validate their string values, so
+that this requirement is met.
+.PP
+For \*(Te, each paragraph will be a separate
+document, and we assume that paragraphs are
+separated by blank lines.  We assume that files
+with extensions
+.IR .atx ,
+.IR .ltx ,
+.IR .stx ,
+and
+.I .tex
+contain input to common \*(Te macro package
+variants.
+.PP
+Make a personal copy of the
+.I mg_get
+script, using the one in the
+.B mg
+source distribution
+.RI ( mg-1.0/mg/mg_get.sh ),
+or the one in the local binary program directory,
+at many sites called
+.IR /usr/local/bin/mg_get .
+.PP
+Examination of the
+.I mg_get
+script shows that each document collection name is
+used in a
+.BR csh (1)
+.I case
+statement selector, and that most of work is done
+by very simple
+.BR awk (1)
+programs that extract documents from files.  In
+your private copy of the
+.I mg_get
+file, after the line
+.PP
+.nf
+  breaksw #davinci
+.fi
+.PP
+and before the line
+.PP
+.nf
+  default:
+.fi
+.PP
+insert this new code:
+.PP
+.nf
+  case bibfiles:
+# Takes a list of files that contain BibTeX entries, and splits them up
+#   by putting ^B after each entry. Assumes that each entry
+#   begins with a line '^@'.
+  switch ($flag)
+    case '-init':
+    breaksw
+
+    case '-text':
+      find $HOME -name '*.bib' -print | \e
+        sort | \e
+        xargs -l100 awk \e
+          '/^@/&&NR!=1{print "^B"} {print $0} END{print "^B"}'
+    breaksw #-text
+
+    case '-cleanup':
+    breaksw #-cleanup
+
+  endsw #flag
+  breaksw #bibfiles
+
+  case texfiles:
+# Takes a list of TeX files and split them up
+#   by putting ^B after each paragraph. Assumes that each entry
+#   begins with a line '^@'.
+  switch ($flag)
+    case '-init':
+    breaksw
+
+    case '-text':
+      find $HOME -name '*x' -print | \e
+        egrep '[.]tex$|[.]ltx$|[.]atx$|[.]stx$' | \e
+        sort | \e
+        xargs -l100 nawk '   /^ *$/ {if (b!=1) printf "^B";b=1} \e
+            \e!/^ *$/ {print;b=0} \e
+            END {printf "^B"}'
+    breaksw #-text
+
+    case '-cleanup':
+    breaksw #-cleanup
+
+  endsw #flag
+  breaksw #texfiles
+.fi
+.PP
+The ^B characters here are Control-B characters,
+.I not
+caret-B pairs.
+.PP
+If you have a large number of \*(Bi or \*(Te
+files, it is likely that a list of them would be
+too long for the UNIX shell to hold in a single
+variable, or on a single command line.  Thus,
+instead of storing the output of
+.BR find (1)
+in a variable, we proceed more cautiously, and
+employ it to produce a list of the required files,
+then pipe them to
+.BR xargs (1),
+which in turn passes up to 100 filenames at a time
+to
+.BR nawk (1)
+for document selection.
+.PP
+Install this modified
+.I mg_get
+script in your private directory for executable
+programs (e.g.
+.IR $HOME/bin ),
+create a directory
+.I $HOME/mgdata
+to hold the index, issue a
+.B rehash
+command if you are using
+.BR csh (1)
+or
+.BR tcsh (1),
+ensure that
+.I mg_get
+occurs in your search path
+.I before
+any system-wide one (the command
+.BI which " mg_get"
+will tell you which version will be selected),
+then create the inverted indexes by
+.BI mgbuild " bibfiles"
+and
+.B mgbuild
+.IR texfiles .
+These commands may take several minutes to run if
+you have a lot of \*(Bi or \*(Te files, or a large
+home directory tree.  Once they are complete, you
+can then query the index with the commands
+.BI mgquery " bibfiles"
+and
+.B mgquery
+.IR texfiles .
+These should respond very rapidly.
+.PP
+In order to keep your index up-to-date, you should
+arrange for it to be recreated automatically and
+regularly, probably every night.  You can do this
+with
+.BR cron (1).
+Use the command
+.B "crontab \-e"
+to edit your
+.I crontab
+file and add two lines like this:
+.PP
+.nf
+00 04 * * * mgbuild bibfiles >$HOME/mgdata/bibfiles.log 2>&1
+15 04 * * * mgbuild texfiles >$HOME/mgdata/texfiles.log 2>&1
+.fi
+.PP
+Save the file and exit the editor.  Now, every
+night at 4am and 4:15am,
+.BR mgbuild (1)
+will reconstruct your inverted indexes, and the
+results of the builds will be saved in log files
+in your
+.I $HOME/mgdata
+directory.
+.\"=====================================================================
+.SH "SEE ALSO"
+.na
+.BR awk (1),
+.BR bibclean (1),
+.BR bibtex (1),
+.BR compress (1),
+.BR csh (1),
+.BR grep (1),
+.BR gunzip (1),
+.BR gzip (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1),
+.BR nawk (1),
+.BR tcsh (1),
+.BR tex (1),
+.BR xargs (1),
+.BR xmg (1),
+.BR xv (1).
+.\"=====================================================================
+.\" This is for GNU Emacs file-specific customization:
+.\" Local Variables:
+.\" fill-column: 50
+.\" End:
Index: /tags/gsdl-21/gsdl/packages/mg/docs/mgintro++.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/docs/mgintro++.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/docs/mgintro++.1	(revision 699)
@@ -0,0 +1,212 @@
+.\"------------------------------------------------------------  
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4 
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.ds r \&\s-1MG\s0
+.if n .ds - \%--
+.if t .ds - \(em
+.\"------------------------------------------------------------  
+.am SS
+.LP
+..
+.\"------------------------------------------------------------
+.TH MGINTRO++ 1 \*(Dt CITRI
+.\"--------------------------------------------------------------
+.SH NAME
+mgintro++ \- extended introduction to the MG system
+.\"-------------------------------------------------------------
+.SH DESCRIPTION
+This manual assumes the reader has already read
+.BR mgintro (1).
+.\"-------------------------------------------------------------
+.SS Creating Different Databases
+If a user wants to build databases other than for some
+predefined ones, such as "alice", "davinci", "mailfiles", "allfiles",
+then the user has a couple of choices.
+Ultimately (s)he must produce a text file with control-Bs 
+terminating the documents. 
+To do this one can produce one or more such files, or write a
+"get" command (typically in the form of a script or c program).
+.\"-------------------------------------------------------------
+.SS Using Input Files for mgbuild
+If you don't want to write a "get" script and just want to use
+one or more text files as input, then you must first generate
+the file with control-Bs. For a simple example, you could take
+any text file(s) such as "test1.txt" and "test2.txt", and use 
+.BR vi (1)
+to insert control-Bs by typing "control-V b".
+Next you should create a file with "set" statements 
+in the following form:
+.PP
+.IP
+\fBset pipe = 0 # do not use pipe - use file instead 
+.br
+\fBset input_files = 'test1.txt test2.txt'
+.LP
+Let's call this file, "build_options".
+Now issue the command:
+.IP
+.B mgbuild -s build_options test
+.LP
+This should build a database called "test" in the $MGDATA directory,
+based on the source data of "test1.txt" and "test2.txt".
+The build_options file is simply sourced by 
+.BR mgbuild (1)
+after it has set up its variables. 
+Therefore, any settings one makes in the
+build_options file will override the standard settings.
+See 
+.BR mgbuild (1) 
+for more information.
+.\"-------------------------------------------------------------
+.SS Writing A Get Program
+Instead of using files as input, it is often more convenient to 
+write a "get" program. This program is called by 
+.BR mgbuild (1)
+to get the text data with control-Bs as document terminators.
+It should take three options:
+.br
+(i) -init; (ii) -text; (iii) -cleanup.
+.br
+Get will be called with "init" first and with "cleanup" at the end.
+It will call get with "text" when it wants the text and it should
+write the text to stdout.
+.br
+See 
+.BR mg_get (1)
+for an example.
+.\"-------------------------------------------------------------
+.SS Regular Builds
+The MG system provides a static database; 
+there are no update commands.
+So if one wants to keep one's database reasonably up-to-date
+then one can have this done automatically on a regular basis by 
+.BR cron (1).
+A crontab file can be created using:
+crontab -e
+A crontab file contains lines of the form:
+.nf
+.IP
+\fBminute hour day-of-month month day-of-week shell-command.
+.LP
+.fi
+See
+.BR crontab (1)
+for more information.
+.nf
+An example crontab entry is:
+.IP
+\fB15 02 * * * mgbuild allfiles >$MGDATA/allfiles/allfiles.log 2>&1
+.LP
+.fi
+This will build up the mg database for "allfiles", your mail in
+the folders, every morning at 2:15am. 
+.\"
+.\"-------------------------------------------------------------
+.SS Command Structure
+There are 22 commands that make up the mg system. However,
+a user may only need to be aware of a few: 
+.BR mgbuild (1),
+.BR mgquery (1), 
+and perhaps
+.BR mg_get (1). 
+Many of the commands are called by 
+.BR mgbuild(1).
+The commands can be broken up into a hierarchy.
+.PP
+--------------------------------------
+.br
+MG--+--image compression
+    |  |
+    |  +--mgbilevel
+    |  |
+    |  +--mgfelics
+    |  |
+    |  +--mgtic
+    |  |
+    |  +--mgticbuild
+    |  |
+    |  +--mgticdump
+    |  |
+    |  +--mgticprune
+    |  |
+    |  +--mgticstat
+    |
+    +--text 
+       |
+       +--compression
+       |  |
+       |  +--mg_passes -T1
+       |  |
+       |  +--mg_passes -T2
+       |  |
+       |  +--mg_compression_dict
+       |  |
+       |  +--mg_fast_comp_dict
+       |
+       +--indexing
+       |  |
+       |  +--mg_passes -N1
+       |  |
+       |  +--mg_passes -N2
+       |  |
+       |  +--mg_perf_hash_build
+       |  |
+       |  +--mg_invf_dict
+       |  |
+       |  +--mg_invf_rebuild
+       |  
+       +--weights
+       |  |
+       |  +--mg_weights_build
+       |
+       +--query
+       |  |
+       |  +--mgquery
+       |
+       +--tools
+          |
+          +--mg_invf_dump
+          |
+          +--mg_text_estimate
+          |
+          +--mgdictlist
+          |
+          +--mgstat
+.br
+--------------------------------------
+.PP
+.nf
+.BR mgbuild (1)
+calls the following commands:
+.RS
+.BR mg_passes (1), mg_compression_dict (1)
+.BR mg_perf_hash_build (1), mg_invf_dict (1), mg_invf_rebuild (1)
+.BR mg_weights_build (1)
+.RE
+.fi
+.\"--------------------------------------------
+.SH SEE ALSO
+.BR mgintro (1),
+.BR mgbuild (1),
+.BR mg_get (1)
+.br
+"Guide To The \*r System", in Appendix A of the book:
+.PP
+.RS
+.nf
+Ian H. Witten, Alistair Moffat, and Timothy C. Bell
+.I "Managing Gigabytes: Compressing and Indexing Documents and Images"
+Van Nostrand Reinhold
+1994
+xiv + 429 pages
+US$54.95 
+ISBN 0-442-01863-0
+Library of Congress catalog number TA1637 .W58 1994.
+.fi
+.RE
+
Index: /tags/gsdl-21/gsdl/packages/mg/docs/mgintro.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/docs/mgintro.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/docs/mgintro.1	(revision 699)
@@ -0,0 +1,155 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.ds r \&\s-1MG\s0
+.if n .ds - \%--
+.if t .ds - \(em
+.\"------------------------------------------------------------
+.am SS
+.LP
+..
+.\"------------------------------------------------------------
+.TH MGINTRO 1 \*(Dt CITRI
+.\"--------------------------------------------------------------
+.SH NAME
+mgintro \- introduction to the MG system
+.\"--------------------------------------------------------------
+.SH DESCRIPTION
+The MG (Managing Gigabytes) system is a collection of
+programs which comprise a full-text retrieval system.
+A full-text retrieval system allows one to create a
+database out of some given documents and then do queries
+upon it to retrieve any relevant documents.
+It is "full-text" in the sense that every word in the
+text is indexed and the query operates only on this index 
+to do the searching.
+.PP
+For example, one could have a database on the book,
+"Alice in Wonderland." A document could be represented by
+each paragraph in the book.
+Having built up the "Alice" database, one could do queries
+such as "cat alice grin" and retrieve any paragraphs which
+match the query. The matching could either be boolean, that
+is the retrieved paragraphs contain a boolean expression of
+the query terms e.g. "cat alice grin"; or the matching
+could be ranked i.e. the most relevant documents to the query
+in relevance order, using some standard heuristic measure.
+.\"--------------------------------------------------------------
+.SS Motivation
+If one wants to find some particular information which
+is stored in a computer text file then one has a few alternative
+courses of action. One can operate directly on the text files
+with utilities such as grep or can process the text files into
+some form of database. Grep is generally limited to identifying
+lines by matching on regular expressions. If the collection 
+of files which grep operates on becomes large, then continual
+passes over the entire text on each query becomes expensive.
+However, its usage is simple as no auxiliary files must be created.
+.PP
+A database consists of some data and indexes into that data. By having
+indexes one can query a large database quickly. Standard databases
+divide the data up into records of fields. This means that the granularity
+of search is a field. In a full-text system, such as \*r,
+there are no fields 
+(or there is an arbitrary sized list of word fields per document) 
+and instead every word is indexed. 
+Using this method, we can except free-form information and yet be fast on searches.
+The next question is what is the overhead of this database.
+In \*r most files which are produced are in a compressed form. The
+two notable compressed files being the given data and the index, called
+an "inverted file". By compressing the files it is possible to have the
+size of the database smaller than the size of the source data.
+.\"--------------------------------------------------------------
+.SS Typical Usage
+The most common use for \*r
+has been as a search database on unix mail files.
+However, any set of text data can be used, one just needs to determine
+what constitutes a document (see 
+.BR mgintro++ (1)
+).
+\*r has also been used on large collections such
+as Comact (Commonwealth Acts of Australia) which is around 132 megabytes 
+and also on sizes up to around 2 gigabytes for TREC 
+(a mixture of collections such as the Wall Street Journal 
+and Associated Press).
+.\"--------------------------------------------------------------
+.SS Getting Started with \*r
+The first thing to do is install the package; 
+please follow the INSTALL instructions.
+Having done this, it is necessary to set a couple of environment variables.
+MGDATA should be set to a directory which is to hold subdirectories for
+each database that you build. For example: 
+.IP
+.B mkdir ~/mgdata; setenv MGDATA ~/mgdata.
+.LP
+If you want to try out building some sample databases then there is
+some sample data such as the "Alice In Wonderland" book. To make sure
+this is accessible you should set the environment variable MGSAMPLE.
+For example: 
+.IP
+.B setenv MGSAMPLE ~/mg/SampleData
+.LP
+Here, "~/mg/SampleData" should contain alice.z .
+.PP
+To build the Alice database (to be contained in $MGDATA/alice subdirectory), 
+type the command 
+.IP
+.B mgbuild alice 
+.LP
+Assuming all went well and some status messages 
+were printed indicating the build was completed, then type 
+.IP
+.B mgquery alice 
+.LP
+to query the database. 
+You can type a few words at the prompt, hit return and
+some relevant documents, Alice paragraphs, should be retrieved.
+Type ".set query ranked" to do ranking queries. Please refer to the 
+.BR mgquery (1)
+man-page for more information on the commands and options of 
+.BR mgquery (1).
+.PP
+The next thing to do is to use \*r
+on a more personal database. If you have
+your mail stored in subdirectories of ~/Mail, such as is done if you use the
+typical set up of 
+.BR elm (1), 
+then type 
+.IP
+.B mgbuild allfiles 
+.LP
+If, however, you keep all your mail in ~/mbox or ~/sentmail, then type 
+.IP
+.B mgbuild mailfiles
+.LP
+.\"--------------------------------------------------------------
+.SH AVAILABILITY
+The \*r software for SunOS 4, Solaris, HPUX, and MIPS,
+can be ftped from: munnari.oz.au [128.250.1.21] in the directory
+/pub/mg. 
+.\"--------------------------------------------------------------
+.SH SEE ALSO
+.na
+.BR mgintro++ (1),
+.BR mgbuild (1),
+.BR mgquery (1)
+.br
+"Guide To The \*r System", in Appendix A of the book:
+.PP
+.RS
+.nf
+Ian H. Witten, Alistair Moffat, and Timothy C. Bell
+.I "Managing Gigabytes: Compressing and Indexing Documents and Images"
+Van Nostrand Reinhold
+1994
+xiv + 429 pages
+US$54.95
+ISBN 0-442-01863-0
+Library of Congress catalog number TA1637 .W58 1994.
+.fi
+.RE
Index: /tags/gsdl-21/gsdl/packages/mg/docs/mgmerge.README
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/docs/mgmerge.README	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/docs/mgmerge.README	(revision 699)
@@ -0,0 +1,230 @@
+A CRASH COURSE GUIDE TO MGMERGE
+===============================
+
+Shane Hudson
+15 November 1994
+
+
+This document is intended as a note to the maintainers of mgmerge
+outlining the mgmerge utility, the changes
+that had to be made to the existing mg code to make mgmerge
+work, and the new source code for mgmerge.
+
+
+****  NOTE:    ****
+All the NEW/CHANGED files are in the file "new_mg.tar" which
+you should have received with this; files that were
+not changed are NOT in the tar file  so it will have to be
+extracted "on top of" the existing code. You'll figure it out :)
+
+
+****  DESCRIPTION:   ****
+mgmerge adds documents to an existing mg database
+without the need to rebuild the database from scratch.
+It works by building a temporary new database from the
+new documents, then merging the "old" and "new" databases
+to give one merged database.
+
+
+The source consists of:
+    mgmerge.sh		The main script, a lot like mgbuild.sh
+    mg_get_merge.sh	Script to retrieve new documents, a lot like
+			mg_get.sh
+    mg_merge.h		header file used by mg_invf_merge.c
+			and mg_text_merge.c
+    mg_text_merge.c	Program to append one compressed text file
+			to another
+    mg_invf_merge.c	Program to merge two inverted files and
+			stemmed dictionaries
+
+See the man pages for mgmerge, mg_get_merge, mg_text_merge and
+mg_invf_merge for a brief overview.
+
+mg_get_merge.sh should really be put into mg_get.sh, with a "-merge"
+option if it is being called by the mgmerge utility, but I was
+too lazy to change it to that.
+
+
+The files updated by mgmerge are:
+
+*.text			with mg_text_merge
+*.text.idx		with mg_text_merge
+*.text.idx.wgt		with mg_weights_build (after merging)
+*.invf			with mg_invf_merge
+*.invf.idx		with      "
+*.invf.dict		with      "
+*.weight		with mg_invf_merge or mg_weights_build
+*.invf.dict.blocked	with mg_invf_dict (after merging)
+
+The *.text.stats file and *.text.dict file remain the same.
+Any other database files (ie, *.invf.dict.hash)
+will be out of date after a merge and will need to be recomputed.
+
+--------------------------------------------------------------------------
+
+MG_TEXT_MERGE
+=============
+
+The two "merging" utilities are mg_text_merge and mg_invf_merge
+
+mg_text_merge simply appends the compressed text for the new
+documents to the old compressed text file.
+For this to succeed, the two files should have been created
+with the same parameters to mg_passes and the same compression
+model must be used. So *.text.dict from the old (static) database
+is used as the model for compressing the new documents.
+The default "-C" option to mg_compression_dict in mgbuild was
+changed to "-S" so novel words in the new documents can be coded.
+The option must be the same in mgmerge -- it is "-S" there too.
+
+---------------
+
+EFFECT on FILE COMPRESSION PERFORMANCE
+
+Since the compression model is not being updated,compression
+performance for the text will slowly degrade. The only solution
+is a periodic rebuild from scratch.
+
+---------------------------------------------------------------------------
+
+MG_INVF_MERGE
+=============
+
+mg_invf_merge updates the stemmed dictionary and
+inverted file.
+For reasons of simplicity, the stemmed dictionaries merged
+are in the ".invf.dict" format, NOT the ".invf.dict.blocked"
+format. 
+So if a database is to be merged the .invf.dict file should
+not be deleted after the .invf.dict.blocked file used by mgquery
+has been created.
+Also for simplicity, level 3 inverted files are not supported, nor
+are skipped format files.
+
+
+
+Merging the inverted files requires every entry in each file
+to be decoded and merged if an entry for the same term appears
+in the other inverted file.
+This can be a slow process if the old inverted file is very large.
+Especially since each entry is decoded bit-by-bit with
+the bit-level I/O rouutines.
+As was noted on page 94 of the book "Managing Gigabytes" (hereafter 
+called "MG"), switching from bernoulli coding (called "Bblock" in
+the mg source code) to gamma or delta coding for the
+inter-document gaps would be an advantage since they require no
+parameters.
+With Bblock coding, N (the number of documents) is a parameter and
+as N changes whenever documents are added, every entry must
+be decoded and recoded.
+My approach was to keep Bblock coding, but keep the N parameter
+constant by recording the artificial N  used to code document
+gaps (called "Nstatic") as well as the real number of documents.
+
+Then, any entry in the old inverted file (called "IFold") that
+is not merged with an entry in the new inverted file (called "IFnew")
+can be copied directly to the merged inverted file (called "IFmerge").
+This can give a significant increase in speed, since usually the
+size of IFnew is very small if only a few documents are being added.
+
+To store Nstatic, a field "static_num_of_docs" was added to
+the invf_dict_header and stem_dict_header structs defined
+in invf.h
+Any program that decodes inverted file entries also had to be changed.
+(The existing mg source code that I changed has been returned with
+this document.)
+Most changes were simply altering a line that
+called BIO_Bblock_Init() so it used the static_num_of_docs
+value rather than num_of_docs.
+
+So since the file format has changed, any existing collection will
+have to be rebuilt first even using mgmerge with it is not
+intended.
+
+-------------
+
+EFFECT on INVERTED FILE COMPRESSION PERFORMANCE
+
+As more merges are accumulatively performed on a database, the
+compression performance will decline.
+The solution is an option in mg_invf_merge to do a "slow"
+merge where every inverted file entry is decoded and recoded using
+the real value of N.
+This is not a lot slower than a fast merge when compared to the
+cost of completely rebuilding the collection from scratch, and 
+the option can be used periodically (when, say, 40% of the
+inverted file has been added since the previous slow merge).
+
+-----------------
+
+THE EXACT WEIGHTS FILE
+
+Recomputing exact document weights requires a complete scan over the
+merged inverted file. This is a waste since the weights for old
+documents will (hopefully) not change much anyway, and the weights
+for new documents can be computed exactly when merging the inverted files,
+at no extra time cost.
+So mg_invf_merge computes the weights for new documents and
+updates the .weight file.
+The weights for old documents are left untouched.
+mgmerge has an option to recompute the weights file.
+This is not much more expensive in terms of time, and is only
+periodically needed.
+
+The effect of leaving the old weights unchanged is difficult to
+asess since the "correct" ranking of documents for a query is
+subjective.
+The frequency with which a rebuild of the weights file with the
+"-w" option should be done is hard to guess.
+But if most merges involves only a few documents, it makes sense
+to not bother recomputing it since the change in old weights values
+is very small and the weights only stray from their "true"
+values slowly over accumulated merging.
+
+-------------------------------------------------------------------------
+
+EXPERIMENTAL RESULTS
+====================
+
+With some small test collections (only a few Mb of text)
+and a slow merge, and a weights file rebuild,
+mgmerge typically took under 20% of the time to completely
+rebuild. Using the default options (fast merge, dont rebuild weights)
+took around 10% or more.
+For larger collections, the savings were greater, especially if the
+added text is very small.
+For example, when the last 6Kb of text of the GNUBib collection was
+added to the rest of it (14Mb) using mgmerge, the total mgmerge time
+was around 20 seconds whereas a complete mgbuild took 280 seconds.
+Only one larger collection was tested (resource contraints prevented
+testing on any really large collections): two short (one-line)
+documents were added to the Gutenberg collection, which comprised
+of nearly 74Mb of source text and had an inverted file size of 9Mb.
+mgmerge took 42 seconds (or 142 seconds with a slow merge and rebuilding the
+weights file), and I'd estimate that an mgbuild on the same machine would 
+have taken at least 20 minutes.
+
+To summarise, mgmerge is not as fast as some sophisticated method
+such as the fixed-length blocks described in section 5.7 of "MG",
+but its huge advantage is that it required almost no changes to
+the existing mg code and is still far better than using mgbuild
+every time a document needs to be added to a collection.
+
+----------------------------------------------------------------------------
+
+OTHER NOTES
+===========
+
+Since it works like mgbuild, mgmerge needs mg_get_merge
+to return the SAME text each time "mg_get_merge -text .." is called.
+
+----
+
+One other thing discovered (but not fixed) was a bug in "invf.pass2.c":
+It crashes on parsed words longer than 128 characters.
+So if any text with words >128 chars is piped to mg_passes,
+the "-N1" and "-N2" options (the memory-efficient inversion method)
+had better be used and not the "-I1"/"-I2" option (which would only
+work on small collections anyway, being the memory-inefficient
+method).
+
Index: /tags/gsdl-21/gsdl/packages/mg/install-sh
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/install-sh	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/install-sh	(revision 699)
@@ -0,0 +1,238 @@
+#! /bin/sh
+#
+# install - install a program, script, or datafile
+# This comes from X11R5.
+#
+# Calling this script install-sh is preferred over install.sh, to prevent
+# `make' implicit rules from creating a file called install from it
+# when there is no Makefile.
+#
+# This script is compatible with the BSD install script, but was written
+# from scratch.
+#
+
+
+# set DOITPROG to echo to test this script
+
+# Don't use :- since 4.3BSD and earlier shells don't like it.
+doit="${DOITPROG-}"
+
+
+# put in absolute paths if you don't have them in your path; or use env. vars.
+
+mvprog="${MVPROG-mv}"
+cpprog="${CPPROG-cp}"
+chmodprog="${CHMODPROG-chmod}"
+chownprog="${CHOWNPROG-chown}"
+chgrpprog="${CHGRPPROG-chgrp}"
+stripprog="${STRIPPROG-strip}"
+rmprog="${RMPROG-rm}"
+mkdirprog="${MKDIRPROG-mkdir}"
+
+tranformbasename=""
+transform_arg=""
+instcmd="$mvprog"
+chmodcmd="$chmodprog 0755"
+chowncmd=""
+chgrpcmd=""
+stripcmd=""
+rmcmd="$rmprog -f"
+mvcmd="$mvprog"
+src=""
+dst=""
+dir_arg=""
+
+while [ x"$1" != x ]; do
+    case $1 in
+	-c) instcmd="$cpprog"
+	    shift
+	    continue;;
+
+	-d) dir_arg=true
+	    shift
+	    continue;;
+
+	-m) chmodcmd="$chmodprog $2"
+	    shift
+	    shift
+	    continue;;
+
+	-o) chowncmd="$chownprog $2"
+	    shift
+	    shift
+	    continue;;
+
+	-g) chgrpcmd="$chgrpprog $2"
+	    shift
+	    shift
+	    continue;;
+
+	-s) stripcmd="$stripprog"
+	    shift
+	    continue;;
+
+	-t=*) transformarg=`echo $1 | sed 's/-t=//'`
+	    shift
+	    continue;;
+
+	-b=*) transformbasename=`echo $1 | sed 's/-b=//'`
+	    shift
+	    continue;;
+
+	*)  if [ x"$src" = x ]
+	    then
+		src=$1
+	    else
+		# this colon is to work around a 386BSD /bin/sh bug
+		:
+		dst=$1
+	    fi
+	    shift
+	    continue;;
+    esac
+done
+
+if [ x"$src" = x ]
+then
+	echo "install:	no input file specified"
+	exit 1
+else
+	true
+fi
+
+if [ x"$dir_arg" != x ]; then
+	dst=$src
+	src=""
+	
+	if [ -d $dst ]; then
+		instcmd=:
+	else
+		instcmd=mkdir
+	fi
+else
+
+# Waiting for this to be detected by the "$instcmd $src $dsttmp" command
+# might cause directories to be created, which would be especially bad 
+# if $src (and thus $dsttmp) contains '*'.
+
+	if [ -f $src -o -d $src ]
+	then
+		true
+	else
+		echo "install:  $src does not exist"
+		exit 1
+	fi
+	
+	if [ x"$dst" = x ]
+	then
+		echo "install:	no destination specified"
+		exit 1
+	else
+		true
+	fi
+
+# If destination is a directory, append the input filename; if your system
+# does not like double slashes in filenames, you may need to add some logic
+
+	if [ -d $dst ]
+	then
+		dst="$dst"/`basename $src`
+	else
+		true
+	fi
+fi
+
+## this sed command emulates the dirname command
+dstdir=`echo $dst | sed -e 's,[^/]*$,,;s,/$,,;s,^$,.,'`
+
+# Make sure that the destination directory exists.
+#  this part is taken from Noah Friedman's mkinstalldirs script
+
+# Skip lots of stat calls in the usual case.
+if [ ! -d "$dstdir" ]; then
+defaultIFS='	
+'
+IFS="${IFS-${defaultIFS}}"
+
+oIFS="${IFS}"
+# Some sh's can't handle IFS=/ for some reason.
+IFS='%'
+set - `echo ${dstdir} | sed -e 's@/@%@g' -e 's@^%@/@'`
+IFS="${oIFS}"
+
+pathcomp=''
+
+while [ $# -ne 0 ] ; do
+	pathcomp="${pathcomp}${1}"
+	shift
+
+	if [ ! -d "${pathcomp}" ] ;
+        then
+		$mkdirprog "${pathcomp}"
+	else
+		true
+	fi
+
+	pathcomp="${pathcomp}/"
+done
+fi
+
+if [ x"$dir_arg" != x ]
+then
+	$doit $instcmd $dst &&
+
+	if [ x"$chowncmd" != x ]; then $doit $chowncmd $dst; else true ; fi &&
+	if [ x"$chgrpcmd" != x ]; then $doit $chgrpcmd $dst; else true ; fi &&
+	if [ x"$stripcmd" != x ]; then $doit $stripcmd $dst; else true ; fi &&
+	if [ x"$chmodcmd" != x ]; then $doit $chmodcmd $dst; else true ; fi
+else
+
+# If we're going to rename the final executable, determine the name now.
+
+	if [ x"$transformarg" = x ] 
+	then
+		dstfile=`basename $dst`
+	else
+		dstfile=`basename $dst $transformbasename | 
+			sed $transformarg`$transformbasename
+	fi
+
+# don't allow the sed command to completely eliminate the filename
+
+	if [ x"$dstfile" = x ] 
+	then
+		dstfile=`basename $dst`
+	else
+		true
+	fi
+
+# Make a temp file name in the proper directory.
+
+	dsttmp=$dstdir/#inst.$$#
+
+# Move or copy the file name to the temp name
+
+	$doit $instcmd $src $dsttmp &&
+
+	trap "rm -f ${dsttmp}" 0 &&
+
+# and set any options; do chmod last to preserve setuid bits
+
+# If any of these fail, we abort the whole thing.  If we want to
+# ignore errors from any of these, just make sure not to ignore
+# errors from the above "$doit $instcmd $src $dsttmp" command.
+
+	if [ x"$chowncmd" != x ]; then $doit $chowncmd $dsttmp; else true;fi &&
+	if [ x"$chgrpcmd" != x ]; then $doit $chgrpcmd $dsttmp; else true;fi &&
+	if [ x"$stripcmd" != x ]; then $doit $stripcmd $dsttmp; else true;fi &&
+	if [ x"$chmodcmd" != x ]; then $doit $chmodcmd $dsttmp; else true;fi &&
+
+# Now rename the file to the real destination.
+
+	$doit $rmcmd -f $dstdir/$dstfile &&
+	$doit $mvcmd $dsttmp $dstdir/$dstfile 
+
+fi &&
+
+
+exit 0
Index: /tags/gsdl-21/gsdl/packages/mg/lib/Makefile.in
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/Makefile.in	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/Makefile.in	(revision 699)
@@ -0,0 +1,156 @@
+###########################################################################
+#
+# Makefile.in -- Makefile for the general purpose library
+# Copyright (C) 1994  Neil Sharman; (C) 1995 Tim Shimmin
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)Makefile.in	1.3 16 Mar 1994
+#
+###########################################################################
+
+
+PACKAGE = @PACKAGE@
+VERSION = @VERSION@
+ 
+SHELL = /bin/sh
+srcdir = @srcdir@
+subdir = lib
+VPATH = @srcdir@
+ 
+AR = ar
+CC = @CC@
+CFLAGS = @CFLAGS@ 
+CPPFLAGS = @CPPFLAGS@
+DEFS = @DEFS@
+RANLIB = @RANLIB@
+
+##########################################
+# Uncomment for compiling for the nzdl project
+#
+CFLAGS = @CFLAGS@ -DQUIET
+##########################################
+
+ 
+prefix = @prefix@
+datadir = $(prefix)/share
+localedir = $(datadir)/locale
+ 
+COMPILE = $(CC) -c $(DEFS) $(INCLUDES) $(CPPFLAGS) $(CFLAGS)
+ 
+.SUFFIXES:
+.SUFFIXES: .c .o
+.c.o:
+	$(COMPILE) $<
+
+ANSI2KNR = @ANSI2KNR@
+o = .@U@o
+ 
+.SUFFIXES: ._c ._o
+.c._c:
+	./ansi2knr $< > $*.tmp && mv $*.tmp $@
+._c._o:
+	@echo $(COMPILE) $<
+	@rm -f _$*.c
+	@ln $< _$*.c && $(COMPILE) _$*.c && mv _$*.o $@ && rm _$*.c
+.c._o:
+	./ansi2knr $< > $*.tmp && mv $*.tmp $*._c
+	@echo $(COMPILE) $*._c
+	@rm -f _$*.c
+	@ln $*._c _$*.c && $(COMPILE) _$*.c && mv _$*.o $@ && rm _$*.c
+
+ 
+INCLUDES = -I../../.. -I.. -I$(srcdir)
+ 
+HEADERS = \
+    bitio_m_random.h   filestats.h        local_strings.h    lovinstem.h \
+    bitio_m_stdio.h    getopt.h           memlib.h \
+    bitio_gen.h        bitio_mem.h        mgheap.h             messages.h \
+    bitio_m.h          bitio_mems.h       huffman.h          timing.h \
+    bitio_m_mem.h      bitio_random.h     huffman_mem.h      perf_hash.h \
+    bitio_m_mems.h     bitio_stdio.h      huffman_stdio.h    sptree.h \
+    pathmax.h          getpagesize.h \
+    random.h           simplefrenchstem.h unitool.h
+# regex.h            rx.h               
+
+
+SOURCES = \
+    bitio_random.c    mgheap.c            memlib.c \
+    bitio_stdio.c     huffman.c         messages.c \
+    bitio_gen.c       filestats.c       huffman_mem.c     perf_hash.c \
+    bitio_mem.c       getopt.c          huffman_stdio.c   sptree.c \
+    bitio_mems.c      getopt1.c         local_strings.c   lovinstem.c \
+    timing.c  \
+    alloca.c          error.c           xmalloc.c         strstr.c \
+    gmalloc.c         ftruncate.c       strcasecmp.c      random.c \
+    simplefrenchstem.c                  unitool.c
+# regex.c           rx.c              
+
+OBJECTS = @ALLOCA@ @LIBOBJS@ \
+    bitio_random$o    mgheap$o            memlib$o \
+    bitio_stdio$o     huffman$o         messages$o \
+    bitio_gen$o       filestats$o       huffman_mem$o     perf_hash$o \
+    bitio_mem$o       getopt$o          huffman_stdio$o   sptree$o \
+    bitio_mems$o      getopt1$o         local_strings$o   lovinstem$o \
+    timing$o \
+    error$o           xmalloc$o \
+    random$o          simplefrenchstem$o \
+    unitool$o
+#   gmalloc$o regex$o		rx$o 
+
+DISTFILES = Makefile.in $(HEADERS) $(SOURCES)
+ 
+all : libmg.a
+
+libmg.a : $(OBJECTS)
+	rm -f libmg.a
+	$(AR) cru libmg.a $(OBJECTS)
+	$(RANLIB) libmg.a
+
+$(OBJECTS): ../../../config.h
+
+install: 
+installdirs:
+uninstall:
+execinstall:
+execuninstall:
+maninstall:
+manuninstall:
+ 
+dist: $(DISTFILES)
+	for file in $(DISTFILES); do \
+	  ln $$file $(distdir) 2> /dev/null \
+	  || cp -p $$file $(distdir); \
+	done
+
+mostlyclean:
+	rm -f *.o *_.o
+ 
+clean: mostlyclean
+	rm -f libmg.a
+ 
+distclean: clean
+	rm -f Makefile
+ 
+maintainer-clean: distclean
+	@echo "This command is intended only for maintainers to use;"
+	@echo "rebuilding the deleted files may require special tools."
+
+Makefile: Makefile.in ../../../config.status
+	cd ../../.. && CONFIG_FILES=$(subdir)/$@ CONFIG_HEADERS= ./config.status
+ 
+# Tell versions [3.59,3.63) of GNU make not to export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:
Index: /tags/gsdl-21/gsdl/packages/mg/lib/WIN32.MAK
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/WIN32.MAK	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/WIN32.MAK	(revision 699)
@@ -0,0 +1,122 @@
+# Generated automatically from Makefile.in by configure.
+###########################################################################
+#
+# Makefile.in -- Makefile for the general purpose library
+# Copyright (C) 1994  Neil Sharman; (C) 1995 Tim Shimmin
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)Makefile.in 1.3 16 Mar 1994
+#
+###########################################################################
+
+
+PACKAGE = mg
+VERSION = 1.2
+ 
+AR = lib
+CC = cl
+#CFLAGS = -g -O 
+CPPFLAGS = 
+DEFS = -DQUIET -DHAVE_CONFIG_H -D__WIN32__ -D_LITTLE_ENDIAN
+ 
+COMPILE = $(CC) -c $(DEFS) -I. -I..
+ 
+.SUFFIXES:
+.SUFFIXES: .c .obj
+.c.obj:
+	$(COMPILE) $<
+
+ANSI2KNR = 
+o = .obj
+ 
+ 
+INCLUDES = -I.. 
+ 
+HEADERS = \
+    bitio_m_random.h   filestats.h        local_strings.h    lovinstem.h \
+    bitio_m_stdio.h    getopt.h           memlib.h \
+    bitio_gen.h        bitio_mem.h        mgheap.h             messages.h \
+    bitio_m.h          bitio_mems.h       huffman.h          timing.h \
+    bitio_m_mem.h      bitio_random.h     huffman_mem.h      perf_hash.h \
+    bitio_m_mems.h     bitio_stdio.h      huffman_stdio.h    sptree.h \
+                       rx.h               pathmax.h          getpagesize.h \
+    random.h           win32in.h          simplefrenchstem.h unitool.h
+
+SOURCES = \
+    bitio_random.c    mgheap.c            memlib.c \
+    bitio_stdio.c     huffman.c         messages.c \
+    bitio_gen.c       filestats.c       huffman_mem.c     perf_hash.c \
+    bitio_mem.c       getopt.c          huffman_stdio.c   sptree.c \
+    bitio_mems.c      getopt1.c         local_strings.c   lovinstem.c \
+    timing.c                            rx.c              \
+    alloca.c          error.c           xmalloc.c         strstr.c \
+    ftruncate.c       strcasecmp.c      random.c          win32in.c \
+    simplefrenchstem.c                  unitool.h
+
+OBJECTS =   rx$o \
+    bitio_random$o    mgheap$o            memlib$o \
+    bitio_stdio$o     huffman$o         messages$o \
+    bitio_gen$o       filestats$o       huffman_mem$o     perf_hash$o \
+    bitio_mem$o       getopt$o          huffman_stdio$o   sptree$o \
+    bitio_mems$o      getopt1$o         local_strings$o   lovinstem$o \
+    timing$o           \
+    error$o           xmalloc$o \
+    random$o          win32in$o         simplefrenchstem$o \
+    unitool$o
+ 
+DISTFILES = Makefile.in $(HEADERS) $(SOURCES)
+ 
+all : libmg.lib
+
+libmg.lib : $(OBJECTS)
+	del libmg.lib
+	$(AR) /out:libmg.lib $(OBJECTS)
+
+$(OBJECTS): ..\win32cfg.h
+
+install: 
+installdirs:
+uninstall:
+execinstall:
+execuninstall:
+maninstall:
+manuninstall:
+ 
+#dist: $(DISTFILES)
+#        for file in $(DISTFILES); do \
+#          ln $$file $(distdir) 2> /dev/null \
+#         || cp -p $$file $(distdir); \
+#       done
+
+mostlyclean:
+	del *.obj
+ 
+clean: mostlyclean
+	del libmg.lib
+ 
+distclean: clean
+	rm -f Makefile
+ 
+maintainer-clean: distclean
+	@echo "This command is intended only for maintainers to use;"
+	@echo "rebuilding the deleted files may require special tools."
+
+#Makefile: Makefile.in ../config.status
+#        cd .. && CONFIG_FILES=$(subdir)/$@ CONFIG_HEADERS= ./config.status
+ 
+# Tell versions [3.59,3.63) of GNU make not to export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+#.NOEXPORT:
Index: /tags/gsdl-21/gsdl/packages/mg/lib/alloca.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/alloca.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/alloca.c	(revision 699)
@@ -0,0 +1,496 @@
+/* alloca.c -- allocate automatically reclaimed memory
+   (Mostly) portable public-domain implementation -- D A Gwyn
+
+   This implementation of the PWB library alloca function,
+   which is used to allocate space off the run-time stack so
+   that it is automatically reclaimed upon procedure exit,
+   was inspired by discussions with J. Q. Johnson of Cornell.
+   J.Otto Tennant <jot@cray.com> contributed the Cray support.
+
+   There are some preprocessor constants that can
+   be defined when compiling for your specific system, for
+   improved efficiency; however, the defaults should be okay.
+
+   The general concept of this implementation is to keep
+   track of all alloca-allocated blocks, and reclaim any
+   that are found to be deeper in the stack than the current
+   invocation.  This heuristic does not reclaim storage as
+   soon as it becomes invalid, but it will do so eventually.
+
+   As a special case, alloca(0) reclaims storage without
+   allocating any.  It is a good idea to use alloca(0) in
+   your main control loop, etc. to force garbage collection.  */
+
+#ifdef HAVE_CONFIG_H
+# ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+#  include <win32cfg.h>
+# else
+#  include <sysfuncs.h>
+# endif
+#endif
+
+#ifdef emacs
+#include "blockinput.h"
+#endif
+
+/* If compiling with GCC 2, this file's not needed.  */
+#if !defined (__GNUC__) || __GNUC__ < 2
+
+/* If someone has defined alloca as a macro,
+   there must be some other way alloca is supposed to work.  */
+#ifndef alloca
+
+#ifdef emacs
+#ifdef static
+/* actually, only want this if static is defined as ""
+   -- this is for usg, in which emacs must undefine static
+   in order to make unexec workable
+   */
+#ifndef STACK_DIRECTION
+you
+lose
+-- must know STACK_DIRECTION at compile-time
+#endif /* STACK_DIRECTION undefined */
+#endif /* static */
+#endif /* emacs */
+
+/* If your stack is a linked list of frames, you have to
+   provide an "address metric" ADDRESS_FUNCTION macro.  */
+
+#if defined (CRAY) && defined (CRAY_STACKSEG_END)
+long i00afunc ();
+#define ADDRESS_FUNCTION(arg) (char *) i00afunc (&(arg))
+#else
+#define ADDRESS_FUNCTION(arg) &(arg)
+#endif
+
+#if __STDC__
+typedef void *pointer;
+#else
+typedef char *pointer;
+#endif
+
+#define	NULL	0
+
+/* Different portions of Emacs need to call different versions of
+   malloc.  The Emacs executable needs alloca to call xmalloc, because
+   ordinary malloc isn't protected from input signals.  On the other
+   hand, the utilities in lib-src need alloca to call malloc; some of
+   them are very simple, and don't have an xmalloc routine.
+
+   Non-Emacs programs expect this to call use xmalloc.
+
+   Callers below should use malloc.  */
+
+#ifndef emacs
+#define malloc xmalloc
+#endif
+extern pointer malloc ();
+
+/* Define STACK_DIRECTION if you know the direction of stack
+   growth for your system; otherwise it will be automatically
+   deduced at run-time.
+
+   STACK_DIRECTION > 0 => grows toward higher addresses
+   STACK_DIRECTION < 0 => grows toward lower addresses
+   STACK_DIRECTION = 0 => direction of growth unknown  */
+
+#ifndef STACK_DIRECTION
+#define	STACK_DIRECTION	0	/* Direction unknown.  */
+#endif
+
+#if STACK_DIRECTION != 0
+
+#define	STACK_DIR	STACK_DIRECTION	/* Known at compile-time.  */
+
+#else /* STACK_DIRECTION == 0; need run-time code.  */
+
+static int stack_dir;		/* 1 or -1 once known.  */
+#define	STACK_DIR	stack_dir
+
+static void
+find_stack_direction ()
+{
+  static char *addr = NULL;	/* Address of first `dummy', once known.  */
+  auto char dummy;		/* To get stack address.  */
+
+  if (addr == NULL)
+    {				/* Initial entry.  */
+      addr = ADDRESS_FUNCTION (dummy);
+
+      find_stack_direction ();	/* Recurse once.  */
+    }
+  else
+    {
+      /* Second entry.  */
+      if (ADDRESS_FUNCTION (dummy) > addr)
+	stack_dir = 1;		/* Stack grew upward.  */
+      else
+	stack_dir = -1;		/* Stack grew downward.  */
+    }
+}
+
+#endif /* STACK_DIRECTION == 0 */
+
+/* An "alloca header" is used to:
+   (a) chain together all alloca'ed blocks;
+   (b) keep track of stack depth.
+
+   It is very important that sizeof(header) agree with malloc
+   alignment chunk size.  The following default should work okay.  */
+
+#ifndef	ALIGN_SIZE
+#define	ALIGN_SIZE	sizeof(double)
+#endif
+
+typedef union hdr
+{
+  char align[ALIGN_SIZE];	/* To force sizeof(header).  */
+  struct
+    {
+      union hdr *next;		/* For chaining headers.  */
+      char *deep;		/* For stack depth measure.  */
+    } h;
+} header;
+
+static header *last_alloca_header = NULL;	/* -> last alloca header.  */
+
+/* Return a pointer to at least SIZE bytes of storage,
+   which will be automatically reclaimed upon exit from
+   the procedure that called alloca.  Originally, this space
+   was supposed to be taken from the current stack frame of the
+   caller, but that method cannot be made to work for some
+   implementations of C, for example under Gould's UTX/32.  */
+
+pointer
+alloca (size)
+     unsigned size;
+{
+  auto char probe;		/* Probes stack depth: */
+  register char *depth = ADDRESS_FUNCTION (probe);
+
+#if STACK_DIRECTION == 0
+  if (STACK_DIR == 0)		/* Unknown growth direction.  */
+    find_stack_direction ();
+#endif
+
+  /* Reclaim garbage, defined as all alloca'd storage that
+     was allocated from deeper in the stack than currently. */
+
+  {
+    register header *hp;	/* Traverses linked list.  */
+
+#ifdef emacs
+    BLOCK_INPUT;
+#endif
+
+    for (hp = last_alloca_header; hp != NULL;)
+      if ((STACK_DIR > 0 && hp->h.deep > depth)
+	  || (STACK_DIR < 0 && hp->h.deep < depth))
+	{
+	  register header *np = hp->h.next;
+
+	  free ((pointer) hp);	/* Collect garbage.  */
+
+	  hp = np;		/* -> next header.  */
+	}
+      else
+	break;			/* Rest are not deeper.  */
+
+    last_alloca_header = hp;	/* -> last valid storage.  */
+
+#ifdef emacs
+    UNBLOCK_INPUT;
+#endif
+  }
+
+  if (size == 0)
+    return NULL;		/* No allocation required.  */
+
+  /* Allocate combined header + user data storage.  */
+
+  {
+    register pointer new = malloc (sizeof (header) + size);
+    /* Address of header.  */
+
+    ((header *) new)->h.next = last_alloca_header;
+    ((header *) new)->h.deep = depth;
+
+    last_alloca_header = (header *) new;
+
+    /* User storage begins just after header.  */
+
+    return (pointer) ((char *) new + sizeof (header));
+  }
+}
+
+#if defined (CRAY) && defined (CRAY_STACKSEG_END)
+
+#ifdef DEBUG_I00AFUNC
+#include <stdio.h>
+#endif
+
+#ifndef CRAY_STACK
+#define CRAY_STACK
+#ifndef CRAY2
+/* Stack structures for CRAY-1, CRAY X-MP, and CRAY Y-MP */
+struct stack_control_header
+  {
+    long shgrow:32;		/* Number of times stack has grown.  */
+    long shaseg:32;		/* Size of increments to stack.  */
+    long shhwm:32;		/* High water mark of stack.  */
+    long shsize:32;		/* Current size of stack (all segments).  */
+  };
+
+/* The stack segment linkage control information occurs at
+   the high-address end of a stack segment.  (The stack
+   grows from low addresses to high addresses.)  The initial
+   part of the stack segment linkage control information is
+   0200 (octal) words.  This provides for register storage
+   for the routine which overflows the stack.  */
+
+struct stack_segment_linkage
+  {
+    long ss[0200];		/* 0200 overflow words.  */
+    long sssize:32;		/* Number of words in this segment.  */
+    long ssbase:32;		/* Offset to stack base.  */
+    long:32;
+    long sspseg:32;		/* Offset to linkage control of previous
+				   segment of stack.  */
+    long:32;
+    long sstcpt:32;		/* Pointer to task common address block.  */
+    long sscsnm;		/* Private control structure number for
+				   microtasking.  */
+    long ssusr1;		/* Reserved for user.  */
+    long ssusr2;		/* Reserved for user.  */
+    long sstpid;		/* Process ID for pid based multi-tasking.  */
+    long ssgvup;		/* Pointer to multitasking thread giveup.  */
+    long sscray[7];		/* Reserved for Cray Research.  */
+    long ssa0;
+    long ssa1;
+    long ssa2;
+    long ssa3;
+    long ssa4;
+    long ssa5;
+    long ssa6;
+    long ssa7;
+    long sss0;
+    long sss1;
+    long sss2;
+    long sss3;
+    long sss4;
+    long sss5;
+    long sss6;
+    long sss7;
+  };
+
+#else /* CRAY2 */
+/* The following structure defines the vector of words
+   returned by the STKSTAT library routine.  */
+struct stk_stat
+  {
+    long now;			/* Current total stack size.  */
+    long maxc;			/* Amount of contiguous space which would
+				   be required to satisfy the maximum
+				   stack demand to date.  */
+    long high_water;		/* Stack high-water mark.  */
+    long overflows;		/* Number of stack overflow ($STKOFEN) calls.  */
+    long hits;			/* Number of internal buffer hits.  */
+    long extends;		/* Number of block extensions.  */
+    long stko_mallocs;		/* Block allocations by $STKOFEN.  */
+    long underflows;		/* Number of stack underflow calls ($STKRETN).  */
+    long stko_free;		/* Number of deallocations by $STKRETN.  */
+    long stkm_free;		/* Number of deallocations by $STKMRET.  */
+    long segments;		/* Current number of stack segments.  */
+    long maxs;			/* Maximum number of stack segments so far.  */
+    long pad_size;		/* Stack pad size.  */
+    long current_address;	/* Current stack segment address.  */
+    long current_size;		/* Current stack segment size.  This
+				   number is actually corrupted by STKSTAT to
+				   include the fifteen word trailer area.  */
+    long initial_address;	/* Address of initial segment.  */
+    long initial_size;		/* Size of initial segment.  */
+  };
+
+/* The following structure describes the data structure which trails
+   any stack segment.  I think that the description in 'asdef' is
+   out of date.  I only describe the parts that I am sure about.  */
+
+struct stk_trailer
+  {
+    long this_address;		/* Address of this block.  */
+    long this_size;		/* Size of this block (does not include
+				   this trailer).  */
+    long unknown2;
+    long unknown3;
+    long link;			/* Address of trailer block of previous
+				   segment.  */
+    long unknown5;
+    long unknown6;
+    long unknown7;
+    long unknown8;
+    long unknown9;
+    long unknown10;
+    long unknown11;
+    long unknown12;
+    long unknown13;
+    long unknown14;
+  };
+
+#endif /* CRAY2 */
+#endif /* not CRAY_STACK */
+
+#ifdef CRAY2
+/* Determine a "stack measure" for an arbitrary ADDRESS.
+   I doubt that "lint" will like this much. */
+
+static long
+i00afunc (long *address)
+{
+  struct stk_stat status;
+  struct stk_trailer *trailer;
+  long *block, size;
+  long result = 0;
+
+  /* We want to iterate through all of the segments.  The first
+     step is to get the stack status structure.  We could do this
+     more quickly and more directly, perhaps, by referencing the
+     $LM00 common block, but I know that this works.  */
+
+  STKSTAT (&status);
+
+  /* Set up the iteration.  */
+
+  trailer = (struct stk_trailer *) (status.current_address
+				    + status.current_size
+				    - 15);
+
+  /* There must be at least one stack segment.  Therefore it is
+     a fatal error if "trailer" is null.  */
+
+  if (trailer == 0)
+    abort ();
+
+  /* Discard segments that do not contain our argument address.  */
+
+  while (trailer != 0)
+    {
+      block = (long *) trailer->this_address;
+      size = trailer->this_size;
+      if (block == 0 || size == 0)
+	abort ();
+      trailer = (struct stk_trailer *) trailer->link;
+      if ((block <= address) && (address < (block + size)))
+	break;
+    }
+
+  /* Set the result to the offset in this segment and add the sizes
+     of all predecessor segments.  */
+
+  result = address - block;
+
+  if (trailer == 0)
+    {
+      return result;
+    }
+
+  do
+    {
+      if (trailer->this_size <= 0)
+	abort ();
+      result += trailer->this_size;
+      trailer = (struct stk_trailer *) trailer->link;
+    }
+  while (trailer != 0);
+
+  /* We are done.  Note that if you present a bogus address (one
+     not in any segment), you will get a different number back, formed
+     from subtracting the address of the first block.  This is probably
+     not what you want.  */
+
+  return (result);
+}
+
+#else /* not CRAY2 */
+/* Stack address function for a CRAY-1, CRAY X-MP, or CRAY Y-MP.
+   Determine the number of the cell within the stack,
+   given the address of the cell.  The purpose of this
+   routine is to linearize, in some sense, stack addresses
+   for alloca.  */
+
+static long
+i00afunc (long address)
+{
+  long stkl = 0;
+
+  long size, pseg, this_segment, stack;
+  long result = 0;
+
+  struct stack_segment_linkage *ssptr;
+
+  /* Register B67 contains the address of the end of the
+     current stack segment.  If you (as a subprogram) store
+     your registers on the stack and find that you are past
+     the contents of B67, you have overflowed the segment.
+
+     B67 also points to the stack segment linkage control
+     area, which is what we are really interested in.  */
+
+  stkl = CRAY_STACKSEG_END ();
+  ssptr = (struct stack_segment_linkage *) stkl;
+
+  /* If one subtracts 'size' from the end of the segment,
+     one has the address of the first word of the segment.
+
+     If this is not the first segment, 'pseg' will be
+     nonzero.  */
+
+  pseg = ssptr->sspseg;
+  size = ssptr->sssize;
+
+  this_segment = stkl - size;
+
+  /* It is possible that calling this routine itself caused
+     a stack overflow.  Discard stack segments which do not
+     contain the target address.  */
+
+  while (!(this_segment <= address && address <= stkl))
+    {
+#ifdef DEBUG_I00AFUNC
+      fprintf (stderr, "%011o %011o %011o\n", this_segment, address, stkl);
+#endif
+      if (pseg == 0)
+	break;
+      stkl = stkl - pseg;
+      ssptr = (struct stack_segment_linkage *) stkl;
+      size = ssptr->sssize;
+      pseg = ssptr->sspseg;
+      this_segment = stkl - size;
+    }
+
+  result = address - this_segment;
+
+  /* If you subtract pseg from the current end of the stack,
+     you get the address of the previous stack segment's end.
+     This seems a little convoluted to me, but I'll bet you save
+     a cycle somewhere.  */
+
+  while (pseg != 0)
+    {
+#ifdef DEBUG_I00AFUNC
+      fprintf (stderr, "%011o %011o\n", pseg, size);
+#endif
+      stkl = stkl - pseg;
+      ssptr = (struct stack_segment_linkage *) stkl;
+      size = ssptr->sssize;
+      pseg = ssptr->sspseg;
+      result += size;
+    }
+  return (result);
+}
+
+#endif /* not CRAY2 */
+#endif /* CRAY */
+
+#endif /* no alloca */
+#endif /* not GCC version 2 */
Index: /tags/gsdl-21/gsdl/packages/mg/lib/bitio_gen.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/bitio_gen.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/bitio_gen.c	(revision 699)
@@ -0,0 +1,153 @@
+/**************************************************************************
+ *
+ * bitio_gen.c -- General supoport routines for bitio
+ * Copyright (C) 1994  Neil Sharman and Alistair Moffat
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:16:42  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:31:40  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/08/22  00:24:38  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+#include "bitio_m.h"
+
+
+
+
+int fprintf (FILE *, const char *,...);
+
+unsigned long 
+BIO_Unary_Length (unsigned long val)
+{
+  register unsigned long num;
+  UNARY_LENGTH (val, num);
+  return (num);
+}
+
+
+unsigned long 
+BIO_Binary_Length (unsigned long val, unsigned long b)
+{
+  register unsigned long num;
+  BINARY_LENGTH (val, b, num);
+  return (num);
+}
+
+
+unsigned long 
+BIO_Gamma_Length (unsigned long val)
+{
+  register unsigned long num;
+  GAMMA_LENGTH (val, num);
+  return (num);
+}
+
+
+unsigned long 
+BIO_Delta_Length (unsigned long val)
+{
+  register unsigned long num;
+  DELTA_LENGTH (val, num);
+  return (num);
+}
+
+
+unsigned long 
+BIO_Elias_Length (unsigned long val, unsigned long b, double s)
+{
+  register unsigned long num;
+  ELIAS_LENGTH (val, b, s, num);
+  return (num);
+}
+
+unsigned long 
+BIO_Bblock_Length (unsigned long val, unsigned long b)
+{
+  register unsigned long num;
+  BBLOCK_LENGTH (val, b, num);
+  return (num);
+}
+
+
+int 
+BIO_Bblock_Init (int N, int p)
+{
+  int b;
+  b = (int) (0.5 + 0.6931471 * N / p);
+  return (b ? b : 1);
+}
+
+
+int 
+BIO_Bblock_Init_W (int N, int p)
+{
+  int logb;
+  FLOORLOG_2 ((N - p) / p, logb);
+  return (logb < 0 ? 1 : (1 << logb));
+}
+
+int 
+BIO_Bblock_Bound_b (int N, int p, int b)
+{
+  int clogb;
+  CEILLOG_2 (b, clogb);
+  return (p * (1 + clogb) + (N - p * ((1 << clogb) - b + 1)) / b);
+}
+
+int 
+BIO_Bblock_Bound (int N, int p)
+{
+  int b;
+  b = BIO_Bblock_Init_W (N, p);
+  return (BIO_Bblock_Bound_b (N, p, b));
+}
+
+int 
+BIO_Gamma_Bound (int N, int p)
+{
+  return ((int) (p * (2 * log2 ((double) N / p) + 1)));
+}
+
+int 
+floorlog_2 (int b)
+{
+  int logb;
+  FLOORLOG_2 (b, logb);
+  return logb;
+}
+
+int 
+ceillog_2 (int b)
+{
+  int logb;
+  CEILLOG_2 (b, logb);
+  return logb;
+}
Index: /tags/gsdl-21/gsdl/packages/mg/lib/bitio_gen.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/bitio_gen.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/bitio_gen.h	(revision 699)
@@ -0,0 +1,63 @@
+/**************************************************************************
+ *
+ * bitio_gen.h -- General supoport routines for bitio
+ * Copyright (C) 1994  Neil Sharman and Alistair Moffat
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ *  This file contains function definitions for doing general bitio operations.
+ *
+ *  This file defines functions for calculation the length in bits of a number
+ *  coded unary, binary, delta, gamma, elias, and  bblock coding. It defines 
+ *  routines for calculating the parameter and bounds for bblock coding. 
+ *  Routines for calculating ceil and floor lg are also defined.
+ *
+ *
+ **************************************************************************/
+
+#ifndef H_BITIO_GEN
+#define H_BITIO_GEN
+
+unsigned long BIO_Unary_Length (unsigned long val);
+
+unsigned long BIO_Binary_Length (unsigned long val, unsigned long b);
+
+unsigned long BIO_Gamma_Length (unsigned long val);
+
+unsigned long BIO_Delta_Length (unsigned long val);
+
+unsigned long BIO_Elias_Length (unsigned long val, unsigned long b, double s);
+
+unsigned long BIO_Bblock_Length (unsigned long val, unsigned long b);
+
+int BIO_Bblock_Init (int N, int p);
+
+int BIO_Bblock_Init_W (int N, int p);
+
+int BIO_Bblock_Bound_b (int N, int p, int b);
+
+int BIO_Bblock_Bound (int N, int p);
+
+int BIO_Gamma_Bound (int N, int p);
+
+int floorlog_2 (int b);
+
+int ceillog_2 (int b);
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/lib/bitio_m.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/bitio_m.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/bitio_m.h	(revision 699)
@@ -0,0 +1,543 @@
+/**************************************************************************
+ *
+ * bitio_m.h -- Macros for bitio
+ * Copyright (C) 1994  Neil Sharman and Alistair Moffat
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef H_BITIO_M
+#define H_BITIO_M
+
+/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
+
+#ifndef BIO_ENCODE_PROLOGUE
+#define BIO_ENCODE_PROLOGUE
+#endif
+
+#ifndef BIO_DECODE_PROLOGUE
+#define BIO_DECODE_PROLOGUE
+#endif
+
+#ifndef BIO_ENCODE_EPILOGUE
+#define BIO_ENCODE_EPILOGUE
+#endif
+
+#ifndef BIO_DECODE_EPILOGUE
+#define BIO_DECODE_EPILOGUE
+#endif
+
+#ifndef DECODE_ADD
+#define DECODE_ADD(b) (b) += (b) + DECODE_BIT
+#endif
+
+/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
+
+#define POSITIVE(f, x)							\
+  if ((x)<=0) 								\
+    fprintf(stderr,"Error: Cannot "#f" encode %lu\n",(unsigned long)x),exit(1);
+
+
+#define CEILLOG_2(x,v)							\
+do {									\
+  register int _B_x  = (x) - 1;						\
+  (v) = 0;								\
+  for (; _B_x ; _B_x>>=1, (v)++);					\
+} while(0)
+
+
+#define FLOORLOG_2(x,v)							\
+do {									\
+  register int _B_x  = (x);						\
+  (v) = -1;								\
+  for (; _B_x ; _B_x>>=1, (v)++);					\
+} while(0)
+
+
+/****************************************************************************/
+
+
+
+#define UNARY_ENCODE(x)							\
+do {									\
+  register unsigned long _B_x = (x);					\
+  BIO_ENCODE_PROLOGUE;							\
+  POSITIVE(unary, _B_x);						\
+  while(--_B_x) ENCODE_BIT(0);						\
+  ENCODE_BIT(1);							\
+  BIO_ENCODE_EPILOGUE;							\
+} while(0)
+
+#define UNARY_ENCODE_L(x, count)					\
+do {									\
+  register unsigned long _B_x = (x);					\
+  BIO_ENCODE_PROLOGUE;							\
+  POSITIVE(unary, _B_x);						\
+  (count) += _B_x;							\
+  while(--_B_x) ENCODE_BIT(0);						\
+  ENCODE_BIT(1);							\
+  BIO_ENCODE_EPILOGUE;							\
+} while(0)
+
+#define UNARY_DECODE(x)							\
+do {									\
+  BIO_DECODE_PROLOGUE;							\
+  (x) = 1;								\
+  while (!DECODE_BIT) (x)++;						\
+  BIO_DECODE_EPILOGUE;							\
+} while(0)
+
+#define UNARY_DECODE_L(x, count)					\
+do {									\
+  BIO_DECODE_PROLOGUE;							\
+  (x) = 1;								\
+  while (!DECODE_BIT) (x)++;						\
+  (count) += (x);							\
+  BIO_DECODE_EPILOGUE;							\
+} while(0)
+
+
+#define UNARY_LENGTH(x, count)						\
+do {									\
+  POSITIVE(unary, x);							\
+  (count) = (x);							\
+} while(0)
+
+
+/****************************************************************************/
+
+
+#define BINARY_ENCODE(x, b)						\
+do {									\
+  register unsigned long _B_x = (x);					\
+  register unsigned long _B_b = (b);					\
+  register int _B_nbits, _B_logofb, _B_thresh;				\
+  BIO_ENCODE_PROLOGUE;							\
+  POSITIVE(binary, _B_x);						\
+  CEILLOG_2(_B_b, _B_logofb);						\
+  _B_thresh = (1<<_B_logofb) - _B_b;					\
+  if (--_B_x < _B_thresh)						\
+    _B_nbits = _B_logofb-1;						\
+  else									\
+    {									\
+      _B_nbits = _B_logofb;						\
+      _B_x += _B_thresh;						\
+    }									\
+  while (--_B_nbits>=0)							\
+    ENCODE_BIT((_B_x>>_B_nbits) & 0x1);					\
+  BIO_ENCODE_EPILOGUE;							\
+} while(0)
+
+#define BINARY_ENCODE_L(x, b, count)					\
+do {									\
+  register unsigned long _B_x = (x);					\
+  register unsigned long _B_b = (b);					\
+  register int _B_nbits, _B_logofb, _B_thresh;				\
+  BIO_ENCODE_PROLOGUE;							\
+  POSITIVE(binary, _B_x);						\
+  CEILLOG_2(_B_b, _B_logofb);						\
+  _B_thresh = (1<<_B_logofb) - _B_b;					\
+  if (--_B_x < _B_thresh)						\
+    _B_nbits = _B_logofb-1;						\
+  else									\
+    {									\
+      _B_nbits = _B_logofb;						\
+      _B_x += _B_thresh;						\
+    }									\
+  (count) += _B_nbits;							\
+  while (--_B_nbits>=0)							\
+    ENCODE_BIT((_B_x>>_B_nbits) & 0x1);					\
+  BIO_ENCODE_EPILOGUE;							\
+} while(0)
+
+
+#define BINARY_DECODE(x, b)						\
+do {									\
+  register unsigned long _B_x = 0;					\
+  register unsigned long _B_b = (b);					\
+  register int _B_i, _B_logofb, _B_thresh;				\
+  BIO_DECODE_PROLOGUE;							\
+  if (_B_b != 1)							\
+    {									\
+      CEILLOG_2(_B_b, _B_logofb);					\
+      _B_thresh = (1<<_B_logofb) - _B_b;				\
+      _B_logofb--;							\
+      for (_B_i=0; _B_i < _B_logofb; _B_i++)			 	\
+        DECODE_ADD(_B_x);						\
+      if (_B_x >= _B_thresh)						\
+        {								\
+          DECODE_ADD(_B_x);						\
+          _B_x -= _B_thresh;						\
+	}								\
+      (x) = _B_x+1;							\
+    }									\
+  else									\
+    (x) = 1;								\
+  BIO_DECODE_EPILOGUE;							\
+} while(0)
+
+#define BINARY_DECODE_L(x, b, count)					\
+do {									\
+  register unsigned long _B_x = 0;					\
+  register unsigned long _B_b = (b);					\
+  register int _B_i, _B_logofb, _B_thresh;				\
+  BIO_DECODE_PROLOGUE;							\
+  if (_B_b != 1)							\
+    {									\
+      CEILLOG_2(_B_b, _B_logofb);					\
+      _B_thresh = (1<<_B_logofb) - _B_b;				\
+      _B_logofb--;							\
+      (count) += _B_logofb;					       	\
+      for (_B_i=0; _B_i < _B_logofb; _B_i++)			 	\
+        DECODE_ADD(_B_x);						\
+      if (_B_x >= _B_thresh)						\
+        {								\
+          DECODE_ADD(_B_x);						\
+          _B_x -= _B_thresh;						\
+          (count)++;							\
+	}								\
+      (x) = _B_x+1;							\
+    }									\
+  else									\
+    (x) = 1;								\
+  BIO_DECODE_EPILOGUE;							\
+} while(0)
+
+#define BINARY_LENGTH(x, b, count)					\
+do {									\
+  register unsigned long _B_x = (x);					\
+  register unsigned long _B_b = (b);					\
+  register int _B_nbits, _B_logofb, _B_thresh;				\
+  POSITIVE(binary, _B_x);						\
+  CEILLOG_2(_B_b, _B_logofb);						\
+  _B_thresh = (1<<_B_logofb) - _B_b;					\
+  if (--_B_x < _B_thresh)						\
+    _B_nbits = _B_logofb-1;						\
+  else									\
+    _B_nbits = _B_logofb;						\
+  (count) = _B_nbits;							\
+} while(0)
+
+/****************************************************************************/
+
+
+
+
+#define GAMMA_ENCODE(x)							\
+do {									\
+  register unsigned long _B_xx = (x);					\
+  register int _B_logofb;						\
+  register int _B_nbits;						\
+  BIO_ENCODE_PROLOGUE;							\
+  POSITIVE(gamma, _B_xx);						\
+  FLOORLOG_2(_B_xx, _B_logofb);						\
+  _B_nbits = _B_logofb+1;						\
+  while(_B_logofb--) ENCODE_BIT(0);					\
+  while (--_B_nbits>=0)							\
+    ENCODE_BIT((_B_xx>>_B_nbits) & 0x1);				\
+  BIO_ENCODE_EPILOGUE;							\
+} while (0)
+
+#define GAMMA_ENCODE_L(x, count)					\
+do {									\
+  register unsigned long _B_xx = (x);					\
+  register int _B_logofb;						\
+  register int _B_nbits;						\
+  BIO_ENCODE_PROLOGUE;							\
+  POSITIVE(gamma, _B_xx);						\
+  FLOORLOG_2(_B_xx, _B_logofb);						\
+  _B_nbits = _B_logofb+1;						\
+  (count) += _B_logofb*2+1;						\
+  while(_B_logofb--) ENCODE_BIT(0);					\
+  while (--_B_nbits>=0)							\
+    ENCODE_BIT((_B_xx>>_B_nbits) & 0x1);				\
+  BIO_ENCODE_EPILOGUE;							\
+} while (0)
+
+#define GAMMA_DECODE(x)							\
+do {									\
+  register unsigned long _B_xx = 1;					\
+  register int _B_nbits = 0;						\
+  BIO_DECODE_PROLOGUE;							\
+  while(!DECODE_BIT) _B_nbits++;					\
+  while (_B_nbits-- > 0)						\
+    DECODE_ADD(_B_xx);							\
+  (x) = _B_xx;								\
+  BIO_DECODE_EPILOGUE;							\
+} while (0)
+
+#define GAMMA_DECODE_L(x, count)					\
+do {									\
+  register unsigned long _B_xx = 1;					\
+  register int _B_nbits = 0;						\
+  BIO_DECODE_PROLOGUE;							\
+  while(!DECODE_BIT) _B_nbits++;					\
+  (count) += _B_nbits*2+1;						\
+  while (_B_nbits-- > 0)						\
+    DECODE_ADD(_B_xx);							\
+  (x) = _B_xx;								\
+  BIO_DECODE_EPILOGUE;							\
+} while (0)
+
+#define GAMMA_LENGTH(x, count)						\
+do {									\
+  register unsigned long _B_xx = (x);					\
+  register int _B_logofb;						\
+  POSITIVE(gamma, _B_xx);						\
+  FLOORLOG_2(_B_xx, _B_logofb);						\
+  (count) = _B_logofb*2+1;						\
+} while (0)
+
+
+
+/****************************************************************************/
+
+
+#define DELTA_ENCODE(x)							\
+do {									\
+  register unsigned long _B_xxx = (x);					\
+  register int _B_logx;							\
+  FLOORLOG_2(_B_xxx, _B_logx);						\
+  GAMMA_ENCODE(_B_logx+1);						\
+  BINARY_ENCODE(_B_xxx+1, 1<<_B_logx);					\
+} while (0)
+
+#define DELTA_ENCODE_L(x, count)					\
+do {									\
+  register unsigned long _B_xxx = (x);					\
+  register int _B_logx;							\
+  FLOORLOG_2(_B_xxx, _B_logx);						\
+  GAMMA_ENCODE_L(_B_logx+1, count);					\
+  BINARY_ENCODE_L(_B_xxx+1, 1<<_B_logx, count);				\
+} while (0)
+
+
+#define DELTA_DECODE(x)							\
+do {									\
+  register unsigned long _B_xxx;					\
+  register int _B_logx;							\
+  GAMMA_DECODE(_B_logx); _B_logx--;					\
+  BINARY_DECODE(_B_xxx, 1<<_B_logx); _B_xxx--;				\
+  (x) = _B_xxx + (1<<_B_logx);						\
+} while (0)
+
+#define DELTA_DECODE_L(x, count)					\
+do {									\
+  register unsigned long _B_xxx;					\
+  register int _B_logx;							\
+  GAMMA_DECODE_L(_B_logx, count); _B_logx--;				\
+  BINARY_DECODE_L(_B_xxx, 1<<_B_logx, count); _B_xxx--;			\
+  (x) = _B_xxx + (1<<_B_logx);						\
+} while (0)
+
+#define DELTA_LENGTH(x, count)						\
+do {									\
+  register unsigned long _B_xxx = (x);					\
+  register int _B_logx, _B_dcount;					\
+  FLOORLOG_2(_B_xxx, _B_logx);						\
+  GAMMA_LENGTH(_B_logx+1, count);					\
+  BINARY_LENGTH(_B_xxx+1, 1<<_B_logx, _B_dcount);			\
+  (count) += _B_dcount;							\
+} while (0)
+
+
+/****************************************************************************/
+
+
+
+
+#define ELIAS_ENCODE(x, b, s)						\
+do {									\
+  register unsigned long _B_xx = (x);					\
+  register unsigned long _B_b = (b);					\
+  register double _B_s = (s);						\
+  register int _B_lower=1, _B_upper=1;					\
+  register int _B_k=0;							\
+  register double _B_pow=1.0;						\
+  POSITIVE(elias, _B_xx);						\
+  while (_B_xx>_B_upper) 						\
+    {									\
+      _B_k++;								\
+      _B_lower = _B_upper+1;						\
+      _B_pow *= _B_s;							\
+      _B_upper += _B_b*_B_pow;						\
+    }									\
+  UNARY_ENCODE(_B_k+1);							\
+  BINARY_ENCODE(_B_xx-_B_lower+1, _B_upper-_B_lower+1);			\
+} while (0)
+
+#define ELIAS_ENCODE_L(x, b, s, count)					\
+do {									\
+  register unsigned long _B_xx = (x);					\
+  register unsigned long _B_b = (b);					\
+  register double _B_s = (s);						\
+  register int _B_lower=1, _B_upper=1;					\
+  register int _B_k=0;							\
+  register double _B_pow=1.0;						\
+  POSITIVE(elias, _B_xx);						\
+  while (_B_xx>_B_upper) 						\
+    {									\
+      _B_k++;								\
+      _B_lower = _B_upper+1;						\
+      _B_pow *= _B_s;							\
+      _B_upper += _B_b*_B_pow;						\
+    }									\
+  UNARY_ENCODE_L(_B_k+1, count);					\
+  BINARY_ENCODE_L(_B_xx-_B_lower+1, _B_upper-_B_lower+1, count);	\
+} while (0)
+
+#define ELIAS_DECODE(x, b, s)						\
+do {									\
+  register unsigned long _B_xx;						\
+  register unsigned long _B_b = (b);					\
+  register double _B_s = (s);						\
+  register int _B_lower=1, _B_upper=1;					\
+  register int _B_k;							\
+  register double _B_pow=1.0;						\
+  UNARY_DECODE(_B_k); _B_k--;						\
+  while (_B_k--)							\
+    {									\
+      _B_lower = _B_upper+1;						\
+      _B_pow *= _B_s;							\
+      _B_upper += _B_b*_B_pow;						\
+    }									\
+  BINARY_DECODE(_B_xx, _B_upper-_B_lower+1); _B_xx--;			\
+  POSITIVE(gamma, _B_xx+_B_lower);				  	\
+  (x) = _B_xx+_B_lower;							\
+} while (0)
+
+#define ELIAS_DECODE_L(x, b, s, count)					\
+do {									\
+  register unsigned long _B_xx;						\
+  register unsigned long _B_b = (b);					\
+  register double _B_s = (s);						\
+  register int _B_lower=1, _B_upper=1;					\
+  register int _B_k;							\
+  register double _B_pow=1.0;						\
+  UNARY_DECODE_L(_B_k, count); _B_k--;					\
+  while (_B_k--)							\
+    {									\
+      _B_lower = _B_upper+1;						\
+      _B_pow *= _B_s;							\
+      _B_upper += _B_b*_B_pow;						\
+    }									\
+  BINARY_DECODE_L(_B_xx, _B_upper-_B_lower+1, count); _B_xx--;		\
+  POSITIVE(gamma, _B_xx+_B_lower);				  	\
+  (x) = _B_xx+_B_lower;							\
+} while (0)
+
+#define ELIAS_LENGTH(x, b, s, count)					\
+do {									\
+  register unsigned long _B_xx = (x);					\
+  register unsigned long _B_b = (b);					\
+  register double _B_s = (s);						\
+  register int _B_lower=1, _B_upper=1;					\
+  register int _B_k=0, _B_ecount;					\
+  register double _B_pow=1.0;						\
+  POSITIVE(gamma, _B_xx);						\
+  while (_B_xx>_B_upper) 						\
+    {									\
+      _B_k++;								\
+      _B_lower = _B_upper+1;						\
+      _B_pow *= _B_s;							\
+      _B_upper += _B_b*_B_pow;						\
+    }									\
+  UNARY_LENGTH(_B_k+1, count);						\
+  BINARY_LENGTH(_B_xx-_B_lower+1, _B_upper-_B_lower+1, _B_ecount);	\
+  (count) += _B_ecount;							\
+} while (0)
+
+
+/****************************************************************************/
+
+
+#define BBLOCK_ENCODE(x, b)						\
+do {									\
+  register unsigned long _B_xx = (x);					\
+  register unsigned long _B_bb = (b);					\
+  register int _B_xdivb = 0;						\
+  POSITIVE(bblock, _B_xx);						\
+  _B_xx--;								\
+  while (_B_xx >= _B_bb) 						\
+    {									\
+      _B_xdivb++;							\
+      _B_xx -= _B_bb;							\
+    }									\
+  UNARY_ENCODE(_B_xdivb+1);						\
+  BINARY_ENCODE(_B_xx+1, _B_bb);					\
+} while (0)
+
+#define BBLOCK_ENCODE_L(x, b, count)					\
+do {									\
+  register unsigned long _B_xx = (x);					\
+  register unsigned long _B_bb = (b);					\
+  register int _B_xdivb = 0;						\
+  POSITIVE(bblock, _B_xx);						\
+  _B_xx--;								\
+  while (_B_xx >= _B_bb) 						\
+    {									\
+      _B_xdivb++;							\
+      _B_xx -= _B_bb;							\
+    }									\
+  UNARY_ENCODE_L(_B_xdivb+1, count);					\
+  BINARY_ENCODE_L(_B_xx+1, _B_bb, count);				\
+} while (0)
+
+#define BBLOCK_DECODE(x, b)						\
+do {									\
+  register unsigned long _B_x1, _B_xx = 0;				\
+  register unsigned long _B_bb = (b);					\
+  register int _B_xdivb;						\
+  UNARY_DECODE(_B_xdivb); _B_xdivb--;					\
+  while (_B_xdivb--)							\
+    _B_xx += _B_bb;							\
+  BINARY_DECODE(_B_x1, _B_bb);						\
+  (x) = _B_xx+_B_x1;							\
+} while (0)
+
+#define BBLOCK_DECODE_L(x, b, count)					\
+do {									\
+  register unsigned long _B_x1, _B_xx = 0;				\
+  register unsigned long _B_bb = (b);					\
+  register int _B_xdivb;						\
+  UNARY_DECODE_L(_B_xdivb, count); _B_xdivb--;				\
+  while (_B_xdivb--)							\
+    _B_xx += _B_bb;							\
+  BINARY_DECODE_L(_B_x1, _B_bb, count);					\
+  (x) = _B_xx+_B_x1;							\
+} while (0)
+
+#define BBLOCK_LENGTH(x, b, count)					\
+do {									\
+  register unsigned long _B_bcount, _B_xx = (x);			\
+  register unsigned long _B_bb = (b);					\
+  register int _B_xdivb = 0;						\
+  POSITIVE(bblock, _B_xx);						\
+  _B_xx--;								\
+  while (_B_xx >= _B_bb) 						\
+    {									\
+      _B_xdivb++;							\
+      _B_xx -= _B_bb;							\
+    }									\
+  UNARY_LENGTH(_B_xdivb+1, count);					\
+  BINARY_LENGTH(_B_xx+1, _B_bb, _B_bcount);				\
+  (count) += _B_bcount;  						\
+} while (0)
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/lib/bitio_m_mem.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/bitio_m_mem.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/bitio_m_mem.h	(revision 699)
@@ -0,0 +1,191 @@
+/**************************************************************************
+ *
+ * bitio_m_mem.h -- Macros for bitio to memory
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ *  This file contains macros for doing bitwise input and output on an array 
+ *  of chars. These routines are faster than the ones in "mems" files. But 
+ *  with these routines you cannot mix reads and writes on the array of chars,
+ *  or multiple write, at the same time and guarantee them to work, also you 
+ *  cannot seek to a point and do a write. The decode routine can detect when
+ *  you run off the end of the array and will produce an approate error 
+ *  message, and the encode routine will stop when it gets to the end of the 
+ *  character array. 
+ *
+ **************************************************************************/
+
+#ifndef H_BITIO_M_MEM
+#define H_BITIO_M_MEM
+
+typedef struct mem_bitio_state
+  {
+    unsigned char *Base;
+    unsigned char *Pos;
+    int Remaining;
+    unsigned char Buff;
+    unsigned char Btg;
+  }
+mem_bitio_state;
+
+
+
+
+#ifndef DECODE_ERROR
+#define DECODE_ERROR (fprintf(stderr,"Unexpected EOF in \"%s\" on line %d\n",\
+				__FILE__, __LINE__), exit(1))
+#endif
+
+
+
+#define ENCODE_START(p,r)						\
+  {									\
+    register unsigned char *__pos = p;					\
+    register unsigned char *__base = p;					\
+    register int __remaining = r;					\
+    register unsigned char __buff = 0;					\
+    register unsigned char __btg = sizeof(__buff)*8;
+
+#define ENCODE_CONTINUE(b)						\
+  {									\
+    register unsigned char *__pos = (b).Pos;				\
+    register unsigned char *__base = (b).Base;				\
+    register int __remaining = (b).Remaining;				\
+    register unsigned char __buff = (b).Buff;				\
+    register unsigned char __btg = (b).Btg;
+
+#define ENCODE_BIT(b)							\
+  do {									\
+    __btg--;								\
+    if (b) __buff |= (1 << __btg);					\
+    if (!__btg)								\
+      {									\
+	if (__remaining)						\
+	  {								\
+	    *__pos++ = __buff;						\
+	    __remaining--;						\
+	  }  								\
+	__buff = 0;							\
+	__btg = sizeof(__buff)*8;					\
+      }									\
+  } while(0)
+
+#define ENCODE_PAUSE(b)							\
+    (b).Pos = __pos;							\
+    (b).Base = __base;							\
+    (b).Remaining = __remaining;					\
+    (b).Buff = __buff;							\
+    (b).Btg =  __btg;							\
+  }
+
+#define ENCODE_FLUSH							\
+  do {									\
+    if (__btg != sizeof(__buff)*8)					\
+      if (__remaining)							\
+	{								\
+          *__pos++ = __buff;						\
+          __remaining--;						\
+	}  								\
+    __btg = sizeof(__buff)*8;						\
+  } while (0)
+
+#define ENCODE_DONE							\
+    ENCODE_FLUSH;							\
+  }
+
+
+#define DECODE_START(p,r)						\
+  {									\
+    register unsigned char *__pos = p;					\
+    register unsigned char *__base = p;					\
+    register int __remaining = r;					\
+    register unsigned char __buff = 0;					\
+    register unsigned char __btg = 0;
+
+#define DECODE_CONTINUE(b)						\
+  {									\
+    register unsigned char *__pos = (b).Pos;				\
+    register unsigned char *__base = (b).Base;				\
+    register int __remaining = (b).Remaining;				\
+    register unsigned char __buff = (b).Buff;				\
+    register unsigned char __btg = (b).Btg;
+
+#define DECODE_ADD_FF(b)						\
+  do {									\
+    if (!__btg)								\
+      {									\
+	if (__remaining)						\
+	  {								\
+  	    __buff = *__pos++;						\
+	    __remaining--;						\
+	  }								\
+        else								\
+          __buff = 0xff;						\
+	__btg = sizeof(__buff)*8;					\
+      }									\
+    (b) += (b) + ((__buff >> --__btg) & 1);				\
+  } while(0)
+
+#define DECODE_ADD_00(b)						\
+  do {									\
+    if (!__btg)								\
+      {									\
+	if (__remaining)						\
+	  {								\
+  	    __buff = *__pos++;						\
+	    __remaining--;						\
+	  }								\
+        else								\
+          __buff = 0x00;						\
+	__btg = sizeof(__buff)*8;					\
+      }									\
+    (b) += (b) + ((__buff >> --__btg) & 1);				\
+  } while(0)
+
+#define DECODE_BIT (__btg ? (__buff >> --__btg) & 1 :			\
+   (!__remaining ? 							\
+    (DECODE_ERROR, 0) :							\
+    (__buff = *__pos++, __remaining--, __btg = sizeof(__buff)*8, 	\
+     (__buff >> --__btg) & 1)))
+
+#define DECODE_DONE	;						\
+  }
+
+#define DECODE_PAUSE(b)							\
+    (b).Pos = __pos;							\
+    (b).Base = __base;							\
+    (b).Remaining = __remaining;					\
+    (b).Buff = __buff;							\
+    (b).Btg =  __btg;							\
+  }
+
+#define DECODE_SEEK(pos)						\
+  do {									\
+    register long __new_pos = (pos);					\
+    __pos = __base + (__new_pos >> 3);					\
+    __buff = *__pos++;							\
+    __btg = 8 - (__new_pos&7);						\
+    }									\
+  while(0)
+
+
+
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/lib/bitio_m_mems.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/bitio_m_mems.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/bitio_m_mems.h	(revision 699)
@@ -0,0 +1,114 @@
+/**************************************************************************
+ *
+ * bitio_m_mems.h -- Macros for bitio to memory (random access)
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ *  This file contains macros for doing bitwise input and output on an array 
+ *  of chars. These routines are slower than the ones in "mem" files. but 
+ *  with these routines you can mix reads and writes, or multiple writes,  on
+ *  the array of chars at the same time and guarantee them to work, also you 
+ *  can seek to a point and do a write. The decode and encode macros cannot 
+ *  detect when the end off the character is reached and just continue 
+ *  processing.
+ *
+ **************************************************************************/
+
+#ifndef H_BITIO_M_MEMS
+#define H_BITIO_M_MEMS
+
+typedef struct mems_bitio_state
+  {
+    unsigned char *Base;
+    unsigned long pos;
+  }
+mems_bitio_state;
+
+
+#define ENCODE_START(b,p)						\
+  {									\
+    register unsigned char *__base = b;					\
+    register unsigned long __pos = p;
+
+#define ENCODE_CONTINUE(b)						\
+  {									\
+    register unsigned char *__base = (b).Base;				\
+    register unsigned long __pos = (b).pos;
+
+#define ENCODE_BIT(b)							\
+  do {									\
+    if (b)								\
+      __base[__pos>>3] |= 0x80 >> (__pos&7);				\
+    else								\
+      __base[__pos>>3] &= 0xff7f >> (__pos&7);				\
+    __pos++;								\
+  } while(0)
+
+#define ENCODE_PAUSE(b)							\
+    (b).Base = __base;							\
+    (b).pos = __pos;							\
+  }
+
+#define ENCODE_FLUSH
+
+
+#define ENCODE_DONE							\
+    ENCODE_FLUSH;							\
+  }
+
+
+#define DECODE_START(b,p)						\
+  {									\
+    register unsigned char *__base = b;					\
+    register unsigned long __pos = p;
+
+#define DECODE_CONTINUE(b)						\
+  {									\
+    register unsigned char *__base = (b).Base;				\
+    register unsigned long __pos = (b).pos;
+
+#define DECODE_ADD_FF(b)						\
+  do {									\
+    (b) += (b) + (__base[__pos>>3] & (0x80 >> (__pos&7)) != 0);		\
+    __pos++;								\
+  } while(0)
+
+#define DECODE_ADD_00(b) DECODE_ADD_FF(b)
+
+#define DECODE_BIT 							\
+    (__pos++, ((__base[(__pos-1)>>3] & (0x80 >> ((__pos-1)&7))) != 0))
+
+#define DECODE_DONE	;						\
+  }
+
+#define DECODE_PAUSE(b)							\
+    (b).Base = __base;							\
+    (b).pos = __pos;							\
+  }
+
+#define DECODE_SEEK(pos) __pos = (pos)
+
+#define ENCODE_SEEK(pos) __pos = (pos)
+
+
+
+
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/lib/bitio_m_random.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/bitio_m_random.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/bitio_m_random.h	(revision 699)
@@ -0,0 +1,168 @@
+/**************************************************************************
+ *
+ * bitio_m_random.h -- Macros for bitio to a file (random access)
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ *  This file contains macros for doing bitwise input and output on an array 
+ *  of chars. These routines are slower than the ones in "mem" files. but 
+ *  with these routines you can mix reads and writes, or multiple writes,  on
+ *  the array of chars at the same time and guarantee them to work, also you 
+ *  can seek to a point and do a write. The decode and encode macros cannot 
+ *  detect when the end off the character is reached and just continue 
+ *  processing.
+ *
+ *
+ **************************************************************************/
+
+#ifndef H_BITIO_M_RANDOM
+#define H_BITIO_M_RANDOM
+
+typedef struct random_bitio_state
+  {
+    FILE *File;
+    unsigned char *Buf;
+    unsigned long Base;
+    unsigned long Used;
+    unsigned long pos;
+    unsigned long len;
+    unsigned long sft;
+  }
+random_bitio_state;
+
+
+#define ENCODE_START(f,l)						\
+  {									\
+    register FILE *__file = f;						\
+    register unsigned char *__buf;					\
+    register unsigned long __pos = 0;					\
+    register unsigned long __base = 0;					\
+    register unsigned long __used = 0;					\
+    register unsigned long __len = (l)-1;				\
+    register unsigned long __sft = 0;					\
+    while (__len) { __sft++; __len >>=1; }				\
+    __len = 1<<__sft;							\
+    __buf = Xmalloc(__len);						\
+    fseek(__file, 0, 0);						\
+    fread(__buf, 1, __len, __file);
+
+#define ENCODE_CONTINUE(b)						\
+  {									\
+    register FILE *__file = (b).File;					\
+    register unsigned char *__buf = (b).Buf;				\
+    register unsigned long __pos = (b).pos;				\
+    register unsigned long __base = (b).Base;				\
+    register unsigned long __used = (b).Used;				\
+    register unsigned long __len = (b).len;				\
+    register unsigned long __sft = (b).sft;
+
+#define SEEK fprintf(stderr, "Seek to %d\n",__base)
+#define READ fprintf(stderr, "Read of %d\n",__len)
+#define WRITE fprintf(stderr, "Write of %d\n",__used)
+
+#define WRITE_READ 							\
+	(__used ? (fseek(__file, __base, 0),				\
+                   fwrite(__buf, 1, __len, __file)) : 0,		\
+        __base += __len,						\
+	fseek(__file, __base, 0),					\
+        fread(__buf, 1, __len, __file),					\
+	__pos = 0, __used = 0)
+
+#define ENCODE_BIT(b)							\
+  do {									\
+    if (b)								\
+      __buf[__pos>>3] |= 0x80 >> (__pos&7);				\
+    else								\
+      __buf[__pos>>3] &= 0xff7f >> (__pos&7);				\
+    __used = 1;								\
+    __pos++;								\
+    if ((__pos>>3) >= __len)						\
+      (void)WRITE_READ;							\
+  } while(0)
+
+#define ENCODE_PAUSE(b)							\
+    (b).File = __file;							\
+    (b).Buf = __buf;							\
+    (b).pos = __pos;							\
+    (b).Base = __base;							\
+    (b).Used = __used;							\
+    (b).len = __len;							\
+    (b).sft = __sft;							\
+  }
+
+#define ENCODE_FLUSH							\
+  if (__used)								\
+    {									\
+      fseek(__file, __base, 0);						\
+      fwrite(__buf, 1, __len, __file);					\
+      __used = 0;							\
+    }
+
+
+
+#define ENCODE_DONE							\
+    ENCODE_FLUSH;							\
+    free(__buf);							\
+  }
+
+
+#define DECODE_START(f,l) ENCODE_START(f,l)
+
+#define DECODE_CONTINUE(b) ENCODE_CONTINUE(b)
+
+#define DECODE_BIT 							\
+    (__pos++, (((__pos-1)>>3) >= __len ? WRITE_READ, __pos=1 : 0),	\
+     ((__buf[(__pos-1)>>3] & (0x80 >> ((__pos-1)&7))) != 0))
+
+#define DECODE_ADD_FF(b) ((b) += (b) + DECODE_BIT)
+
+#define DECODE_ADD_00(b) DECODE_ADD_FF(b)
+
+#define DECODE_DONE ENCODE_DONE
+
+#define DECODE_PAUSE(b)	ENCODE_PAUSE(b)
+
+#define ENCODE_SEEK(pos) 						\
+  do {									\
+    if ((((pos)>>3) >= __base) && ((((pos)+7)>>3) < (__base+__len)))   	\
+      __pos = (pos)-(__base << 3); 					\
+    else								\
+      {									\
+        ENCODE_FLUSH;							\
+        __base = (pos) >> 3;						\
+        __base = ((pos)  >> (__sft+3)) << __sft;			\
+        fseek(__file, __base, 0);					\
+        fread(__buf, 1, __len, __file);					\
+        __pos = (pos)&7; 						\
+        __pos = (pos)&((8 << __sft)-1);					\
+      }									\
+  } while(0)
+
+
+#define DECODE_SEEK(pos) ENCODE_SEEK(pos)
+
+#define ENCODE_TELL ((__base << 3) + __pos)
+
+#define DECODE_TELL ENCODE_TELL
+
+
+
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/lib/bitio_m_stdio.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/bitio_m_stdio.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/bitio_m_stdio.h	(revision 699)
@@ -0,0 +1,147 @@
+/**************************************************************************
+ *
+ * bitio_m_stdio.h -- Macros for bitio to a file
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ *  This file contains macros for doing bitwise input and output on a FILE*.
+ *  With these routines you cannot mix reads and writes on the FILE,
+ *  or multiple writes, at the same time and guarantee them to work, also you 
+ *  cannot seek to a point and do a write. The decode routine can detect when
+ *  you run off the end of the file and will produce an approate error message.
+ *
+ *
+ **************************************************************************/
+
+#ifndef H_BITIO_M_STDIO
+#define H_BITIO_M_STDIO
+
+typedef struct stdio_bitio_state
+  {
+    FILE *File;
+    unsigned char Buff;
+    unsigned char Btg;
+  }
+stdio_bitio_state;
+
+
+
+#ifndef DECODE_ERROR
+#define DECODE_ERROR (fprintf(stderr,"Unexpected EOF in \"%s\" on line %d\n",\
+				__FILE__, __LINE__), exit(1))
+#endif
+
+
+
+#define ENCODE_START(f)							\
+  {									\
+    FILE *__outfile = f;						\
+    register unsigned char __buff = 0;					\
+    register unsigned char __btg = sizeof(__buff)*8;
+
+#define ENCODE_CONTINUE(b)						\
+  {									\
+    FILE *__outfile = (b).File;						\
+    register unsigned char __buff = (b).Buff;				\
+    register unsigned char __btg = (b).Btg;
+
+#define ENCODE_BIT(b)							\
+  do {									\
+    __btg--;								\
+    if (b) __buff |= (1 << __btg);					\
+    if (!__btg)								\
+      {									\
+	putc(__buff, __outfile);					\
+	__buff = 0;							\
+	__btg = sizeof(__buff)*8;					\
+      }									\
+  } while(0)
+
+#define ENCODE_PAUSE(b)							\
+    (b).File = __outfile;						\
+    (b).Buff = __buff;							\
+    (b).Btg =  __btg;							\
+  }
+
+#define ENCODE_DONE							\
+    if (__btg != sizeof(__buff)*8)					\
+      putc(__buff,__outfile);						\
+  }
+
+
+#define DECODE_START(f)							\
+  {									\
+    FILE *__infile = f;							\
+    register unsigned char __buff = 0;					\
+    register unsigned char __btg = 0;
+
+#define DECODE_CONTINUE(b)						\
+  {									\
+    FILE *__infile = (b).File;						\
+    register unsigned char __buff = (b).Buff;				\
+    register unsigned char __btg = (b).Btg;
+
+#define DECODE_ADD_FF(b)						\
+  do {									\
+    if (!__btg)								\
+      {									\
+	register int val = getc(__infile);				\
+	__buff = val == EOF ? 0xff : val;				\
+	__btg = sizeof(__buff)*8;					\
+      }									\
+    (b) += (b) + ((__buff >> --__btg) & 1);				\
+  } while(0)
+
+#define DECODE_ADD_00(b)						\
+  do {									\
+    if (!__btg)								\
+      {									\
+	register int val = getc(__infile);				\
+	__buff = val == EOF ? 0xff : val;				\
+	__btg = sizeof(__buff)*8;					\
+      }									\
+    (b) += (b) + ((__buff >> --__btg) & 1);				\
+  } while(0)
+
+#define DECODE_BIT (__btg ? (__buff >> --__btg) & 1 :			\
+   (__buff = getc(__infile), (feof(__infile) ?				\
+    (DECODE_ERROR, 0) :							\
+    (__btg = sizeof(__buff)*8, (__buff >> --__btg) & 1))))
+
+#define DECODE_DONE	;						\
+  }
+
+#define DECODE_PAUSE(b)							\
+    (b).File = __infile;						\
+    (b).Buff = __buff;							\
+    (b).Btg =  __btg;							\
+  }
+
+
+#define DECODE_SEEK(pos)						\
+  do {									\
+    register long __new_pos = (pos);					\
+    fseek(__infile, __new_pos >> 3, 0);					\
+    __buff = getc(__infile);						\
+    __btg = 8 - (__new_pos&7);						\
+    }									\
+  while(0)
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/lib/bitio_mem.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/bitio_mem.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/bitio_mem.c	(revision 699)
@@ -0,0 +1,256 @@
+/**************************************************************************
+ *
+ * bitio_mem.c -- Functions for bitio to memory
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:16:45  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:31:47  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/08/22  00:24:39  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+
+#include "sysfuncs.h"
+
+#include "bitio_m_mem.h"
+#include "bitio_m.h"
+
+int fprintf (FILE *, const char *,...);
+
+
+
+void 
+BIO_Mem_Encode_Start (void *buf, int rem, mem_bitio_state * bs)
+{
+  ENCODE_START (buf, rem)
+    ENCODE_PAUSE (*bs)
+}
+
+void 
+BIO_Mem_Encode_Done (mem_bitio_state * bs)
+{
+  ENCODE_CONTINUE (*bs)
+    ENCODE_DONE
+}
+
+
+void 
+BIO_Mem_Decode_Start (void *buf, int rem, mem_bitio_state * bs)
+{
+  DECODE_START (buf, rem)
+    DECODE_PAUSE (*bs)
+}
+
+
+
+
+void 
+BIO_Mem_Unary_Encode (unsigned long val, mem_bitio_state * bs,
+		      unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    UNARY_ENCODE_L (val, *bits);
+  else
+    UNARY_ENCODE (val);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Mem_Unary_Decode (mem_bitio_state * bs,
+		      unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    UNARY_DECODE_L (val, *bits);
+  else
+    UNARY_DECODE (val);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+
+
+
+void 
+BIO_Mem_Binary_Encode (unsigned long val, unsigned long b,
+		       mem_bitio_state * bs, unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    BINARY_ENCODE_L (val, b, *bits);
+  else
+    BINARY_ENCODE (val, b);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Mem_Binary_Decode (unsigned long b, mem_bitio_state * bs,
+		       unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    BINARY_DECODE_L (val, b, *bits);
+  else
+    BINARY_DECODE (val, b);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+
+
+
+void 
+BIO_Mem_Gamma_Encode (unsigned long val, mem_bitio_state * bs,
+		      unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    GAMMA_ENCODE_L (val, *bits);
+  else
+    GAMMA_ENCODE (val);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Mem_Gamma_Decode (mem_bitio_state * bs, unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    GAMMA_DECODE_L (val, *bits);
+  else
+    GAMMA_DECODE (val);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+void 
+BIO_Mem_Delta_Encode (unsigned long val, mem_bitio_state * bs,
+		      unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    DELTA_ENCODE_L (val, *bits);
+  else
+    DELTA_ENCODE (val);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Mem_Delta_Decode (mem_bitio_state * bs, unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    DELTA_DECODE_L (val, *bits);
+  else
+    DELTA_DECODE (val);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+void 
+BIO_Mem_Elias_Encode (unsigned long val, unsigned long b, double s,
+		      mem_bitio_state * bs, unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    ELIAS_ENCODE_L (val, b, s, *bits);
+  else
+    ELIAS_ENCODE (val, b, s);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Mem_Elias_Decode (unsigned long b, double s,
+		      mem_bitio_state * bs, unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    ELIAS_DECODE_L (val, b, s, *bits);
+  else
+    ELIAS_DECODE (val, b, s);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+void 
+BIO_Mem_Bblock_Encode (unsigned long val, unsigned long b,
+		       mem_bitio_state * bs, unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    BBLOCK_ENCODE_L (val, b, *bits);
+  else
+    BBLOCK_ENCODE (val, b);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Mem_Bblock_Decode (unsigned long b, mem_bitio_state * bs,
+		       unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    BBLOCK_DECODE_L (val, b, *bits);
+  else
+    BBLOCK_DECODE (val, b);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+void 
+BIO_Mem_Decode_Seek (unsigned long pos, mem_bitio_state * bs)
+{
+  DECODE_CONTINUE (*bs)
+    DECODE_SEEK (pos);
+  DECODE_PAUSE (*bs)
+}
Index: /tags/gsdl-21/gsdl/packages/mg/lib/bitio_mem.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/bitio_mem.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/bitio_mem.h	(revision 699)
@@ -0,0 +1,105 @@
+/**************************************************************************
+ *
+ * bitio_mem.h -- Functions for bitio to memory
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ *  This file contains function definitions for doing bitwise input and output
+ *  of numbers on an array of chars. These routines are faster than the ones 
+ *  in "mems" files. But with these routines you cannot mix reads and writes 
+ *  on the array of chars, or multiple write, at the same time and guarantee
+ *  them to work, also you cannot seek to a point and do a write. The decode
+ *  routine can detect when you run off the end of the array and will produce
+ *  an approate error message, and the encode routine will stop when it gets
+ *  to the end of the character array. 
+ *
+ *
+ **************************************************************************/
+
+
+#ifndef H_BITIO_MEM
+#define H_BITIO_MEM
+
+
+
+typedef struct mem_bitio_state
+  {
+    unsigned char *Base;
+    unsigned char *Pos;
+    int Remaining;
+    unsigned char Buff;
+    unsigned char Btg;
+  }
+mem_bitio_state;
+
+
+/* NOTE : All bytes are filled high bit first */
+
+
+void BIO_Mem_Encode_Start (void *buf, int rem, mem_bitio_state * bs);
+void BIO_Mem_Encode_Done (mem_bitio_state * bs);
+
+
+
+void BIO_Mem_Decode_Start (void *buf, int rem, mem_bitio_state * bs);
+
+
+
+void BIO_Mem_Unary_Encode (unsigned long val, mem_bitio_state * bs,
+			   unsigned long *bits);
+unsigned long BIO_Mem_Unary_Decode (mem_bitio_state * bs,
+				    unsigned long *bits);
+
+
+
+void BIO_Mem_Binary_Encode (unsigned long val, unsigned long b,
+			    mem_bitio_state * bs, unsigned long *bits);
+unsigned long BIO_Mem_Binary_Decode (unsigned long b, mem_bitio_state * bs,
+				     unsigned long *bits);
+
+
+
+void BIO_Mem_Gamma_Encode (unsigned long val, mem_bitio_state * bs,
+			   unsigned long *bits);
+unsigned long BIO_Mem_Gamma_Decode (mem_bitio_state * bs, unsigned long *bits);
+
+
+
+void BIO_Mem_Delta_Encode (unsigned long val, mem_bitio_state * bs,
+			   unsigned long *bits);
+unsigned long BIO_Mem_Delta_Decode (mem_bitio_state * bs, unsigned long *bits);
+
+
+void BIO_Mem_Elias_Encode (unsigned long val, unsigned long b, double s,
+			   mem_bitio_state * bs, unsigned long *bits);
+unsigned long BIO_Mem_Elias_Decode (unsigned long b, double s,
+				 mem_bitio_state * bs, unsigned long *bits);
+
+
+void BIO_Mem_Bblock_Encode (unsigned long val, unsigned long b,
+			    mem_bitio_state * bs, unsigned long *bits);
+unsigned long BIO_Mem_Bblock_Decode (unsigned long b, mem_bitio_state * bs,
+				     unsigned long *bits);
+
+
+void BIO_Mem_Decode_Seek (unsigned long pos, mem_bitio_state * bs);
+
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/lib/bitio_mems.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/bitio_mems.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/bitio_mems.c	(revision 699)
@@ -0,0 +1,265 @@
+/**************************************************************************
+ *
+ * bitio_mems.c -- Functions for bitio to memory (random access)
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:16:45  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:31:48  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/08/22  00:24:39  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+#include "sysfuncs.h"
+
+#include "bitio_m_mems.h"
+#include "bitio_m.h"
+
+int fprintf (FILE *, const char *,...);
+
+
+
+void 
+BIO_Mems_Encode_Start (unsigned char *buf, unsigned long pos,
+		       mems_bitio_state * bs)
+{
+  ENCODE_START (buf, pos)
+    ENCODE_PAUSE (*bs)
+}
+
+void 
+BIO_Mems_Encode_Done (mems_bitio_state * bs)
+{
+  ENCODE_CONTINUE (*bs)
+    ENCODE_DONE
+}
+
+
+void 
+BIO_Mems_Decode_Start (unsigned char *buf, unsigned long pos,
+		       mems_bitio_state * bs)
+{
+  DECODE_START (buf, pos)
+    DECODE_PAUSE (*bs)
+}
+
+
+
+
+void 
+BIO_Mems_Unary_Encode (unsigned long val, mems_bitio_state * bs,
+		       unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    UNARY_ENCODE_L (val, *bits);
+  else
+    UNARY_ENCODE (val);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Mems_Unary_Decode (mems_bitio_state * bs,
+		       unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    UNARY_DECODE_L (val, *bits);
+  else
+    UNARY_DECODE (val);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+
+
+
+void 
+BIO_Mems_Binary_Encode (unsigned long val, unsigned long b,
+			mems_bitio_state * bs, unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    BINARY_ENCODE_L (val, b, *bits);
+  else
+    BINARY_ENCODE (val, b);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Mems_Binary_Decode (unsigned long b, mems_bitio_state * bs,
+			unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    BINARY_DECODE_L (val, b, *bits);
+  else
+    BINARY_DECODE (val, b);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+
+
+
+void 
+BIO_Mems_Gamma_Encode (unsigned long val, mems_bitio_state * bs,
+		       unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    GAMMA_ENCODE_L (val, *bits);
+  else
+    GAMMA_ENCODE (val);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Mems_Gamma_Decode (mems_bitio_state * bs, unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    GAMMA_DECODE_L (val, *bits);
+  else
+    GAMMA_DECODE (val);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+void 
+BIO_Mems_Delta_Encode (unsigned long val, mems_bitio_state * bs,
+		       unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    DELTA_ENCODE_L (val, *bits);
+  else
+    DELTA_ENCODE (val);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Mems_Delta_Decode (mems_bitio_state * bs, unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    DELTA_DECODE_L (val, *bits);
+  else
+    DELTA_DECODE (val);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+void 
+BIO_Mems_Elias_Encode (unsigned long val, unsigned long b, double s,
+		       mems_bitio_state * bs, unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    ELIAS_ENCODE_L (val, b, s, *bits);
+  else
+    ELIAS_ENCODE (val, b, s);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Mems_Elias_Decode (unsigned long b, double s,
+		       mems_bitio_state * bs, unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    ELIAS_DECODE_L (val, b, s, *bits);
+  else
+    ELIAS_DECODE (val, b, s);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+void 
+BIO_Mems_Bblock_Encode (unsigned long val, unsigned long b,
+			mems_bitio_state * bs, unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    BBLOCK_ENCODE_L (val, b, *bits);
+  else
+    BBLOCK_ENCODE (val, b);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Mems_Bblock_Decode (unsigned long b, mems_bitio_state * bs,
+			unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    BBLOCK_DECODE_L (val, b, *bits);
+  else
+    BBLOCK_DECODE (val, b);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+void 
+BIO_Mems_Decode_Seek (unsigned long pos, mems_bitio_state * bs)
+{
+  DECODE_CONTINUE (*bs)
+    DECODE_SEEK (pos);
+  DECODE_PAUSE (*bs)
+}
+
+void 
+BIO_Mems_Encode_Seek (unsigned long pos, mems_bitio_state * bs)
+{
+  ENCODE_CONTINUE (*bs)
+    ENCODE_SEEK (pos);
+  ENCODE_PAUSE (*bs)
+}
Index: /tags/gsdl-21/gsdl/packages/mg/lib/bitio_mems.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/bitio_mems.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/bitio_mems.h	(revision 699)
@@ -0,0 +1,99 @@
+/**************************************************************************
+ *
+ * bitio_mems.h -- Functions for bitio to memory (random access)
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ *  This file contains function definitions for doing bitwise input and output
+ *  of numbers on an array of chars. These routines are slower than the ones 
+ *  in "mem" files. but with these routines you can mix reads and writes, or 
+ *  multiple writes,  on the array of chars at the same time and guarantee
+ *  them to work, also you can seek to a point and do a write. The decode and 
+ *  encode functions cannot detect when the end off the character is reached
+ *  and just continue processing.
+ *
+ **************************************************************************/
+
+#ifndef H_BITIO_MEMS
+#define H_BITIO_MEMS
+
+
+
+typedef struct mems_bitio_state
+  {
+    unsigned char *Base;
+    unsigned long pos;
+  }
+mems_bitio_state;
+
+
+/* NOTE : All bytes are filled high bit first */
+
+
+void BIO_Mems_Encode_Start (char *buf, unsigned long pos, mems_bitio_state * bs);
+void BIO_Mems_Encode_Done (mems_bitio_state * bs);
+
+
+
+void BIO_Mems_Decode_Start (void *buf, unsigned long pos, mems_bitio_state * bs);
+
+
+
+void BIO_Mems_Unary_Encode (unsigned long val, mems_bitio_state * bs,
+			    unsigned long *bits);
+unsigned long BIO_Mems_Unary_Decode (mems_bitio_state * bs,
+				     unsigned long *bits);
+
+
+
+void BIO_Mems_Binary_Encode (unsigned long val, unsigned long b,
+			     mems_bitio_state * bs, unsigned long *bits);
+unsigned long BIO_Mems_Binary_Decode (unsigned long b, mems_bitio_state * bs,
+				      unsigned long *bits);
+
+
+
+void BIO_Mems_Gamma_Encode (unsigned long val, mems_bitio_state * bs,
+			    unsigned long *bits);
+unsigned long BIO_Mems_Gamma_Decode (mems_bitio_state * bs, unsigned long *bits);
+
+
+
+void BIO_Mems_Delta_Encode (unsigned long val, mems_bitio_state * bs,
+			    unsigned long *bits);
+unsigned long BIO_Mems_Delta_Decode (mems_bitio_state * bs, unsigned long *bits);
+
+
+void BIO_Mems_Elias_Encode (unsigned long val, unsigned long b, double s,
+			    mems_bitio_state * bs, unsigned long *bits);
+unsigned long BIO_Mems_Elias_Decode (unsigned long b, double s,
+				mems_bitio_state * bs, unsigned long *bits);
+
+
+void BIO_Mems_Bblock_Encode (unsigned long val, unsigned long b,
+			     mems_bitio_state * bs, unsigned long *bits);
+unsigned long BIO_Mems_Bblock_Decode (unsigned long b, mems_bitio_state * bs,
+				      unsigned long *bits);
+
+
+void BIO_Mems_Decode_Seek (unsigned long pos, mems_bitio_state * bs);
+
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/lib/bitio_random.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/bitio_random.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/bitio_random.c	(revision 699)
@@ -0,0 +1,285 @@
+/**************************************************************************
+ *
+ * bitio_random.c -- Functions for bitio to a file (random access)
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:16:46  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:31:50  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/08/22  00:24:40  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+
+#include "sysfuncs.h"
+#include "memlib.h"
+
+#include "bitio_m_random.h"
+#include "bitio_m.h"
+
+
+void 
+BIO_Random_Start (FILE * f, unsigned long len,
+		  random_bitio_state * bs)
+{
+  ENCODE_START (f, len)
+    ENCODE_PAUSE (*bs)
+}
+
+void 
+BIO_Random_Done (random_bitio_state * bs)
+{
+  ENCODE_CONTINUE (*bs)
+    ENCODE_DONE
+}
+
+void 
+BIO_Random_Encode_Bit (int bit, random_bitio_state * bs)
+{
+  ENCODE_CONTINUE (*bs)
+    ENCODE_BIT (bit);
+  ENCODE_PAUSE (*bs)
+}
+
+int 
+BIO_Random_Decode_Bit (random_bitio_state * bs)
+{
+  register int val;
+  DECODE_CONTINUE (*bs)
+    val = DECODE_BIT;
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+void 
+BIO_Random_Unary_Encode (unsigned long val, random_bitio_state * bs,
+			 unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    UNARY_ENCODE_L (val, *bits);
+  else
+    UNARY_ENCODE (val);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Random_Unary_Decode (random_bitio_state * bs,
+			 unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    UNARY_DECODE_L (val, *bits);
+  else
+    UNARY_DECODE (val);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+
+
+
+void 
+BIO_Random_Binary_Encode (unsigned long val, unsigned long b,
+			  random_bitio_state * bs, unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    BINARY_ENCODE_L (val, b, *bits);
+  else
+    BINARY_ENCODE (val, b);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Random_Binary_Decode (unsigned long b,
+			  random_bitio_state * bs,
+			  unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    BINARY_DECODE_L (val, b, *bits);
+  else
+    BINARY_DECODE (val, b);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+
+
+
+void 
+BIO_Random_Gamma_Encode (unsigned long val, random_bitio_state * bs,
+			 unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    GAMMA_ENCODE_L (val, *bits);
+  else
+    GAMMA_ENCODE (val);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Random_Gamma_Decode (random_bitio_state * bs,
+			 unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    GAMMA_DECODE_L (val, *bits);
+  else
+    GAMMA_DECODE (val);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+void 
+BIO_Random_Delta_Encode (unsigned long val, random_bitio_state * bs,
+			 unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    DELTA_ENCODE_L (val, *bits);
+  else
+    DELTA_ENCODE (val);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Random_Delta_Decode (random_bitio_state * bs,
+			 unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    DELTA_DECODE_L (val, *bits);
+  else
+    DELTA_DECODE (val);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+void 
+BIO_Random_Elias_Encode (unsigned long val, unsigned long b, double s,
+			 random_bitio_state * bs, unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    ELIAS_ENCODE_L (val, b, s, *bits);
+  else
+    ELIAS_ENCODE (val, b, s);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Random_Elias_Decode (unsigned long b, double s,
+			 random_bitio_state * bs,
+			 unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    ELIAS_DECODE_L (val, b, s, *bits);
+  else
+    ELIAS_DECODE (val, b, s);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+void 
+BIO_Random_Bblock_Encode (unsigned long val, unsigned long b,
+			  random_bitio_state * bs, unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    BBLOCK_ENCODE_L (val, b, *bits);
+  else
+    BBLOCK_ENCODE (val, b);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Random_Bblock_Decode (unsigned long b,
+			  random_bitio_state * bs,
+			  unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    BBLOCK_DECODE_L (val, b, *bits);
+  else
+    BBLOCK_DECODE (val, b);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+void 
+BIO_Random_Seek (unsigned long pos, random_bitio_state * bs)
+{
+  ENCODE_CONTINUE (*bs)
+    ENCODE_SEEK (pos);
+  ENCODE_PAUSE (*bs)
+}
+void 
+BIO_Random_Flush (random_bitio_state * bs)
+{
+  ENCODE_CONTINUE (*bs)
+    ENCODE_FLUSH;
+  ENCODE_PAUSE (*bs)
+}
+
+unsigned long 
+BIO_Random_Tell (random_bitio_state * bs)
+{
+  register unsigned long t;
+  ENCODE_CONTINUE (*bs)
+    t = ENCODE_TELL;
+  ENCODE_PAUSE (*bs)
+    return (t);
+}
Index: /tags/gsdl-21/gsdl/packages/mg/lib/bitio_random.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/bitio_random.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/bitio_random.h	(revision 699)
@@ -0,0 +1,117 @@
+/**************************************************************************
+ *
+ * bitio_random.h -- Functions for bitio to a file (random access)
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ *  This file contains function definitions for doing bitwise input and output
+ *  of numbers on an array of chars. These routines are slower than the ones 
+ *  in "mem" files. but with these routines you can mix reads and writes, or 
+ *  multiple writes,  on the array of chars at the same time and guarantee
+ *  them to work, also you can seek to a point and do a write. The decode and 
+ *  encode functions cannot detect when the end off the character is reached
+ *  and just continue processing.
+ *
+ **************************************************************************/
+
+#ifndef H_BITIO_RANDOM
+#define H_BITIO_RANDOM
+
+
+
+typedef struct random_bitio_state
+  {
+    FILE *File;
+    unsigned char *Buf;
+    unsigned long Base;
+    unsigned long Used;
+    unsigned long pos;
+    unsigned long len;
+    unsigned long sft;
+  }
+random_bitio_state;
+
+
+/* NOTE : All bytes are filled high bit first */
+
+
+void BIO_Random_Start (FILE * f, unsigned long len,
+		       random_bitio_state * bs);
+void BIO_Random_Done (random_bitio_state * bs);
+
+
+
+void BIO_Random_Decode_Start (void *buf, unsigned long pos,
+			      random_bitio_state * bs);
+
+void BIO_Random_Encode_Bit (int bit, random_bitio_state * bs);
+
+int BIO_Random_Decode_Bit (random_bitio_state * bs);
+
+
+void BIO_Random_Unary_Encode (unsigned long val, random_bitio_state * bs,
+			      unsigned long *bits);
+unsigned long BIO_Random_Unary_Decode (random_bitio_state * bs,
+				       unsigned long *bits);
+
+
+
+void BIO_Random_Binary_Encode (unsigned long val, unsigned long b,
+			       random_bitio_state * bs, unsigned long *bits);
+unsigned long BIO_Random_Binary_Decode (unsigned long b, random_bitio_state * bs,
+					unsigned long *bits);
+
+
+
+void BIO_Random_Gamma_Encode (unsigned long val, random_bitio_state * bs,
+			      unsigned long *bits);
+unsigned long BIO_Random_Gamma_Decode (random_bitio_state * bs,
+				       unsigned long *bits);
+
+
+
+void BIO_Random_Delta_Encode (unsigned long val, random_bitio_state * bs,
+			      unsigned long *bits);
+unsigned long BIO_Random_Delta_Decode (random_bitio_state * bs,
+				       unsigned long *bits);
+
+
+void BIO_Random_Elias_Encode (unsigned long val, unsigned long b, double s,
+			      random_bitio_state * bs, unsigned long *bits);
+unsigned long BIO_Random_Elias_Decode (unsigned long b, double s,
+				       random_bitio_state * bs,
+				       unsigned long *bits);
+
+
+void BIO_Random_Bblock_Encode (unsigned long val, unsigned long b,
+			       random_bitio_state * bs, unsigned long *bits);
+unsigned long BIO_Random_Bblock_Decode (unsigned long b,
+					random_bitio_state * bs,
+					unsigned long *bits);
+
+
+void BIO_Random_Seek (unsigned long pos, random_bitio_state * bs);
+
+void BIO_Random_Flush (random_bitio_state * bs);
+
+unsigned long BIO_Random_Tell (random_bitio_state * bs);
+
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/lib/bitio_stdio.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/bitio_stdio.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/bitio_stdio.c	(revision 699)
@@ -0,0 +1,281 @@
+/**************************************************************************
+ *
+ * bitio_stdio.c -- Functions for bitio to a file
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:16:47  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:31:52  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/08/22  00:24:41  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+
+#include "sysfuncs.h"
+
+#include "bitio_m_stdio.h"
+#include "bitio_m.h"
+
+
+
+void 
+BIO_Stdio_Encode_Start (FILE * f, stdio_bitio_state * bs)
+{
+  ENCODE_START (f)
+    ENCODE_PAUSE (*bs)
+}
+
+void 
+BIO_Stdio_Encode_Done (stdio_bitio_state * bs)
+{
+  ENCODE_CONTINUE (*bs)
+    ENCODE_DONE
+}
+
+
+void 
+BIO_Stdio_Decode_Start (FILE * f, stdio_bitio_state * bs)
+{
+  DECODE_START (f)
+    DECODE_PAUSE (*bs)
+}
+
+void 
+BIO_Stdio_Encode_Bit (int bit, stdio_bitio_state * bs)
+{
+  ENCODE_CONTINUE (*bs)
+    ENCODE_BIT (bit);
+  ENCODE_PAUSE (*bs)
+}
+
+
+int 
+BIO_Stdio_Decode_Bit (stdio_bitio_state * bs)
+{
+  register int val;
+  DECODE_CONTINUE (*bs)
+    val = DECODE_BIT;
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+void 
+BIO_Stdio_Unary_Encode (unsigned long val, stdio_bitio_state * bs,
+			unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    UNARY_ENCODE_L (val, *bits);
+  else
+    UNARY_ENCODE (val);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Stdio_Unary_Decode (stdio_bitio_state * bs,
+			unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    UNARY_DECODE_L (val, *bits);
+  else
+    UNARY_DECODE (val);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+
+
+
+void 
+BIO_Stdio_Binary_Encode (unsigned long val, unsigned long b,
+			 stdio_bitio_state * bs, unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    BINARY_ENCODE_L (val, b, *bits);
+  else
+    BINARY_ENCODE (val, b);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Stdio_Binary_Decode (unsigned long b, stdio_bitio_state * bs,
+			 unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    BINARY_DECODE_L (val, b, *bits);
+  else
+    BINARY_DECODE (val, b);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+
+
+
+void 
+BIO_Stdio_Gamma_Encode (unsigned long val, stdio_bitio_state * bs,
+			unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    GAMMA_ENCODE_L (val, *bits);
+  else
+    GAMMA_ENCODE (val);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Stdio_Gamma_Decode (stdio_bitio_state * bs,
+			unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    GAMMA_DECODE_L (val, *bits);
+  else
+    GAMMA_DECODE (val);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+void 
+BIO_Stdio_Delta_Encode (unsigned long val, stdio_bitio_state * bs,
+			unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    DELTA_ENCODE_L (val, *bits);
+  else
+    DELTA_ENCODE (val);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Stdio_Delta_Decode (stdio_bitio_state * bs,
+			unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    DELTA_DECODE_L (val, *bits);
+  else
+    DELTA_DECODE (val);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+
+void 
+BIO_Stdio_Elias_Encode (unsigned long val, unsigned long b, double s,
+			stdio_bitio_state * bs, unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    ELIAS_ENCODE_L (val, b, s, *bits);
+  else
+    ELIAS_ENCODE (val, b, s);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Stdio_Elias_Decode (unsigned long b, double s,
+			stdio_bitio_state * bs,
+			unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    ELIAS_DECODE_L (val, b, s, *bits);
+  else
+    ELIAS_DECODE (val, b, s);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+
+
+void 
+BIO_Stdio_Bblock_Encode (unsigned long val, unsigned long b,
+			 stdio_bitio_state * bs, unsigned long *bits)
+{
+  ENCODE_CONTINUE (*bs)
+    if (bits)
+    BBLOCK_ENCODE_L (val, b, *bits);
+  else
+    BBLOCK_ENCODE (val, b);
+  ENCODE_PAUSE (*bs)
+}
+
+
+unsigned long 
+BIO_Stdio_Bblock_Decode (unsigned long b, stdio_bitio_state * bs,
+			 unsigned long *bits)
+{
+  register unsigned long val;
+  DECODE_CONTINUE (*bs)
+    if (bits)
+    BBLOCK_DECODE_L (val, b, *bits);
+  else
+    BBLOCK_DECODE (val, b);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
+
+void 
+BIO_Stdio_Decode_Seek (unsigned long pos, stdio_bitio_state * bs)
+{
+  DECODE_CONTINUE (*bs)
+    DECODE_SEEK (pos);
+  DECODE_PAUSE (*bs)
+}
Index: /tags/gsdl-21/gsdl/packages/mg/lib/bitio_stdio.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/bitio_stdio.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/bitio_stdio.h	(revision 699)
@@ -0,0 +1,106 @@
+/**************************************************************************
+ *
+ * bitio_stdio.h -- Functions for bitio to a file
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ *  This file contains function definitions for doing bitwise input and output
+ *  of number on a FILE*. With these function you cannot mix reads and writes
+ *  on the FILE, or multiple writes, at the same time and guarantee them to 
+ *  work, also you  cannot seek to a point and do a write. The decode function
+ *  can detect when you run off the end of the file and will produce an 
+ *  approate error message.
+ *
+ *
+ **************************************************************************/
+
+#ifndef H_BITIO_STDIO
+#define H_BITIO_STDIO
+
+
+typedef struct stdio_bitio_state
+  {
+    FILE *File;
+    unsigned char Buff;
+    unsigned char Btg;
+  }
+stdio_bitio_state;
+
+
+/* NOTE : All bytes are filled high bit first */
+
+
+void BIO_Stdio_Encode_Start (FILE * f, stdio_bitio_state * bs);
+void BIO_Stdio_Encode_Done (stdio_bitio_state * bs);
+
+
+
+void BIO_Stdio_Decode_Start (FILE * f, stdio_bitio_state * bs);
+
+
+
+void BIO_Stdio_Encode_Bit (int bit, stdio_bitio_state * bs);
+int BIO_Stdio_Decode_Bit (stdio_bitio_state * bs);
+
+
+void BIO_Stdio_Unary_Encode (unsigned long val, stdio_bitio_state * bs,
+			     unsigned long *bits);
+unsigned long BIO_Stdio_Unary_Decode (stdio_bitio_state * bs,
+				      unsigned long *bits);
+
+
+
+void BIO_Stdio_Binary_Encode (unsigned long val, unsigned long b,
+			      stdio_bitio_state * bs, unsigned long *bits);
+unsigned long BIO_Stdio_Binary_Decode (unsigned long b, stdio_bitio_state * bs,
+				       unsigned long *bits);
+
+
+
+void BIO_Stdio_Gamma_Encode (unsigned long val, stdio_bitio_state * bs,
+			     unsigned long *bits);
+unsigned long BIO_Stdio_Gamma_Decode (stdio_bitio_state * bs,
+				      unsigned long *bits);
+
+
+
+void BIO_Stdio_Delta_Encode (unsigned long val, stdio_bitio_state * bs,
+			     unsigned long *bits);
+unsigned long BIO_Stdio_Delta_Decode (stdio_bitio_state * bs,
+				      unsigned long *bits);
+
+
+
+void BIO_Stdio_Elias_Encode (unsigned long val, unsigned long b, double s,
+			     stdio_bitio_state * bs, unsigned long *bits);
+unsigned long BIO_Stdio_Elias_Decode (unsigned long b, double s,
+				      stdio_bitio_state * bs,
+				      unsigned long *bits);
+
+
+void BIO_Stdio_Bblock_Encode (unsigned long val, unsigned long b,
+			      stdio_bitio_state * bs, unsigned long *bits);
+unsigned long BIO_Stdio_Bblock_Decode (unsigned long b, stdio_bitio_state * bs,
+				       unsigned long *bits);
+
+
+void BIO_Stdio_Decode_Seek (unsigned long pos, stdio_bitio_state * bs);
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/lib/error.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/error.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/error.c	(revision 699)
@@ -0,0 +1,127 @@
+/* error.c -- error handler for noninteractive utilities
+   Copyright (C) 1990, 91, 92, 93, 94 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+/* Written by David MacKenzie <djm@gnu.ai.mit.edu>.  */
+
+#ifdef HAVE_CONFIG_H
+# ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+#  include <win32cfg.h>
+# else
+#  include <sysfuncs.h>
+# endif
+#endif
+
+#include <stdio.h>
+
+#if HAVE_VPRINTF || HAVE_DOPRNT
+# if __STDC__
+#  include <stdarg.h>
+#  define VA_START(args, lastarg) va_start(args, lastarg)
+# else
+#  include <varargs.h>
+#  define VA_START(args, lastarg) va_start(args)
+# endif
+#else
+# define va_alist a1, a2, a3, a4, a5, a6, a7, a8
+# define va_dcl char *a1, *a2, *a3, *a4, *a5, *a6, *a7, *a8;
+#endif
+
+#if STDC_HEADERS
+# include <stdlib.h>
+# include <string.h>
+#else
+void exit ();
+#endif
+
+/* If NULL, error will flush stdout, then print on stderr the program
+   name, a colon and a space.  Otherwise, error will call this
+   function without parameters instead.  */
+void (*error_print_progname) () = NULL;
+
+/* The calling program should define program_name and set it to the
+   name of the executing program.  */
+#if 0 /* [RB/TS:10/95] do not use program_name */
+extern char *program_name;
+#endif
+
+#if HAVE_STRERROR
+char *strerror ();
+#else
+static char *
+private_strerror (errnum)
+     int errnum;
+{
+  extern char *sys_errlist[];
+  extern int sys_nerr;
+
+  if (errnum > 0 && errnum <= sys_nerr)
+    return sys_errlist[errnum];
+  return "Unknown system error";
+}
+#define strerror private_strerror
+#endif
+
+/* Print the program name and error message MESSAGE, which is a printf-style
+   format string with optional args.
+   If ERRNUM is nonzero, print its corresponding system error message.
+   Exit with status STATUS if it is nonzero.  */
+/* VARARGS */
+
+void
+#if defined(VA_START) && __STDC__
+error (int status, int errnum, const char *message, ...)
+#else
+error (status, errnum, message, va_alist)
+     int status;
+     int errnum;
+     char *message;
+     va_dcl
+#endif
+{
+#ifdef VA_START
+  va_list args;
+#endif
+
+  if (error_print_progname)
+    (*error_print_progname) ();
+  else
+    {
+      fflush (stdout);
+#if 0 /* [RB/TS:10/95] do not use program_name */
+      fprintf (stderr, "%s: ", program_name);
+#endif
+    }
+
+#ifdef VA_START
+  VA_START (args, message);
+# if HAVE_VPRINTF
+  vfprintf (stderr, message, args);
+# else
+  _doprnt (message, args, stderr);
+# endif
+  va_end (args);
+#else
+  fprintf (stderr, message, a1, a2, a3, a4, a5, a6, a7, a8);
+#endif
+
+  if (errnum)
+    fprintf (stderr, ": %s", strerror (errnum));
+  putc ('\n', stderr);
+  fflush (stderr);
+  if (status)
+    exit (status);
+}
Index: /tags/gsdl-21/gsdl/packages/mg/lib/filestats.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/filestats.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/filestats.c	(revision 699)
@@ -0,0 +1,133 @@
+/**************************************************************************
+ *
+ * filestats.c -- Functions for keeping stats on file accesses
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:16:48  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:31:54  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/08/22  00:24:42  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "filestats.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+
+File *
+Fopen (char *name, char *mode, unsigned long magic)
+{
+  FILE *f;
+  File *F;
+  unsigned long m;
+  if (!(f = fopen (name, mode)))
+    return (NULL);
+  if (magic)
+    switch (*mode)
+      {
+      case 'r':
+	fread ((char *) &m, sizeof (m), 1, f);
+	NTOHUL(m);  /* [RPAP - Jan 97: Endian Ordering] */
+	if (m != magic)
+	  {
+	    fclose (f);
+	    return (NULL);
+	  }
+	break;
+      case 'w':
+	HTONUL(magic);  /* [RPAP - Jan 97: Endian Ordering] */
+	fwrite ((char *) &magic, sizeof (magic), 1, f);
+      }
+  if (!(F = Xmalloc (sizeof (File))))
+    {
+      fclose (f);
+      return (NULL);
+    }
+  F->pathname = Xstrdup (name);
+  F->name = strrchr (F->pathname, '/');
+  F->name = F->name ? F->name + 1 : F->pathname;
+  F->f = f;
+  F->Current.NumSeeks = F->Current.NumReads = F->Current.NumBytes = 0;
+  F->Cumulative = F->Current;
+  return (F);
+}
+
+
+int 
+Fclose (File * F)
+{
+  int num;
+  if (!F)
+    return (0);
+  num = fclose (F->f);
+  if (F->pathname)
+    Xfree (F->pathname);
+  Xfree (F);
+  return (num);
+}
+
+size_t 
+Fread (void *ptr, size_t size, size_t nitems, File * F)
+{
+  int num;
+  num = fread ((char *) ptr, size, nitems, F->f);
+  F->Current.NumReads++;
+  F->Current.NumBytes += num * size;
+  return (num);
+}
+
+int 
+Fseek (File * F, long offset, int ptrname)
+{
+  int num;
+  num = fseek (F->f, offset, ptrname);
+  F->Current.NumSeeks++;
+  return (num);
+}
+
+void 
+Rewind (File * F)
+{
+  rewind (F->f);
+  F->Current.NumSeeks++;
+}
+
+
+void 
+ZeroFileStats (File * F)
+{
+  F->Cumulative.NumSeeks += F->Current.NumSeeks;
+  F->Cumulative.NumReads += F->Current.NumReads;
+  F->Cumulative.NumBytes += F->Current.NumBytes;
+  F->Current.NumSeeks = F->Current.NumReads = F->Current.NumBytes = 0;
+}
Index: /tags/gsdl-21/gsdl/packages/mg/lib/filestats.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/filestats.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/filestats.h	(revision 699)
@@ -0,0 +1,67 @@
+/**************************************************************************
+ *
+ * filestats.h -- Functions for keeping stats on file accesses
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef H_FILESTATS
+#define H_FILESTATS
+
+#include "sysfuncs.h"
+
+typedef struct FileStats
+  {
+    unsigned long NumSeeks;
+    unsigned long NumReads;
+    unsigned long NumBytes;
+  }
+FileStats;
+
+
+
+typedef struct
+  {
+    FILE *f;
+    char *pathname;
+    char *name;
+    FileStats Current;
+    FileStats Cumulative;
+  }
+File;
+
+/* if magic is 0 no magic number is read or written */
+File *Fopen (char *name, char *mode, unsigned long magic);
+
+size_t Fread (void *ptr, size_t size, size_t nitems, File * F);
+
+int Fseek (File * F, long offset, int ptrname);
+
+void Rewind (File * F);
+
+int Fclose (File * F);
+
+#define Getc(F) (F->Current.NumReads++, F->Current.NumBytes++, getc(F->f))
+
+/* This adds the Current file stats to the Cumulative stats and then zeros
+   the Current stats */
+void ZeroFileStats (File * F);
+
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/lib/ftruncate.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/ftruncate.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/ftruncate.c	(revision 699)
@@ -0,0 +1,107 @@
+/* ftruncate emulations that work on some System V's.
+   This file is in the public domain.  */
+
+#ifdef HAVE_CONFIG_H
+# ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+#  include <win32cfg.h>
+# else
+#  include <sysfuncs.h>
+# endif
+#endif
+
+#include <sys/types.h>
+#include <fcntl.h>
+
+#ifdef F_CHSIZE
+
+int
+ftruncate (fd, length)
+     int fd;
+     off_t length;
+{
+  return fcntl (fd, F_CHSIZE, length);
+}
+
+#else /* not F_CHSIZE */
+#ifdef F_FREESP
+
+/* By William Kucharski <kucharsk@netcom.com>.  */
+
+#include <sys/stat.h>
+#include <errno.h>
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+int
+ftruncate (fd, length)
+     int fd;
+     off_t length;
+{
+  struct flock fl;
+  struct stat filebuf;
+
+  if (fstat (fd, &filebuf) < 0)
+    return -1;
+
+  if (filebuf.st_size < length)
+    {
+      /* Extend file length. */
+      if (lseek (fd, (length - 1), SEEK_SET) < 0)
+	return -1;
+
+      /* Write a "0" byte. */
+      if (write (fd, "", 1) != 1)
+	return -1;
+    }
+  else
+    {
+
+      /* Truncate length. */
+
+      fl.l_whence = 0;
+      fl.l_len = 0;
+      fl.l_start = length;
+      fl.l_type = F_WRLCK;	/* write lock on file space */
+
+      /* This relies on the *undocumented* F_FREESP argument to fcntl,
+	 which truncates the file so that it ends at the position
+	 indicated by fl.l_start.  Will minor miracles never cease?  */
+
+      if (fcntl (fd, F_FREESP, &fl) < 0)
+	return -1;
+    }
+
+  return 0;
+}
+
+#else /* not F_CHSIZE nor F_FREESP */
+#ifdef HAVE_CHSIZE
+
+int
+ftruncate (fd, length)
+     int fd;
+     off_t length;
+{
+  return chsize (fd, length);
+}
+
+#else /* not F_CHSIZE nor F_FREESP nor HAVE_CHSIZE */
+
+#include <errno.h>
+#ifndef errno
+extern int errno;
+#endif
+
+int
+ftruncate (fd, length)
+     int fd;
+     off_t length;
+{
+  errno = EIO;
+  return -1;
+}
+
+#endif /* not HAVE_CHSIZE */
+#endif /* not F_FREESP */
+#endif /* not F_CHSIZE */
Index: /tags/gsdl-21/gsdl/packages/mg/lib/getopt.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/getopt.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/getopt.c	(revision 699)
@@ -0,0 +1,750 @@
+/* Getopt for GNU.
+   NOTE: getopt is now part of the C library, so if you don't know what
+   "Keep this file name-space clean" means, talk to roland@gnu.ai.mit.edu
+   before changing it!
+
+   Copyright (C) 1987, 88, 89, 90, 91, 92, 1993
+   Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the
+   Free Software Foundation; either version 2, or (at your option) any
+   later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+
+#ifdef HAVE_CONFIG_H
+#include "sysfuncs.h"
+#endif
+
+#if !__STDC__ && !defined(const) && IN_GCC
+#define const
+#endif
+
+/* This tells Alpha OSF/1 not to define a getopt prototype in <stdio.h>.  */
+#ifndef _NO_PROTO
+#define _NO_PROTO
+#endif
+
+#include <stdio.h>
+
+/* Comment out all this code if we are using the GNU C Library, and are not
+   actually compiling the library itself.  This code is part of the GNU C
+   Library, but also included in many other GNU distributions.  Compiling
+   and linking in this code is a waste when using the GNU C library
+   (especially if it is a shared library).  Rather than having every GNU
+   program understand `configure --with-gnu-libc' and omit the object files,
+   it is simpler to just do this in the source for each such file.  */
+
+#if defined (_LIBC) || !defined (__GNU_LIBRARY__)
+
+
+/* This needs to come after some library #include
+   to get __GNU_LIBRARY__ defined.  */
+#ifdef	__GNU_LIBRARY__
+/* Don't include stdlib.h for non-GNU C libraries because some of them
+   contain conflicting prototypes for getopt.  */
+#include <stdlib.h>
+#endif /* GNU C library.  */
+
+/* If GETOPT_COMPAT is defined, `+' as well as `--' can introduce a
+   long-named option.  Because this is not POSIX.2 compliant, it is
+   being phased out.  */
+/* #define GETOPT_COMPAT */
+
+/* This version of `getopt' appears to the caller like standard Unix `getopt'
+   but it behaves differently for the user, since it allows the user
+   to intersperse the options with the other arguments.
+
+   As `getopt' works, it permutes the elements of ARGV so that,
+   when it is done, all the options precede everything else.  Thus
+   all application programs are extended to handle flexible argument order.
+
+   Setting the environment variable POSIXLY_CORRECT disables permutation.
+   Then the behavior is completely standard.
+
+   GNU application programs can use a third alternative mode in which
+   they can distinguish the relative order of options and other arguments.  */
+
+#include "getopt.h"
+
+/* For communication from `getopt' to the caller.
+   When `getopt' finds an option that takes an argument,
+   the argument value is returned here.
+   Also, when `ordering' is RETURN_IN_ORDER,
+   each non-option ARGV-element is returned here.  */
+
+char *optarg = 0;
+
+/* Index in ARGV of the next element to be scanned.
+   This is used for communication to and from the caller
+   and for communication between successive calls to `getopt'.
+
+   On entry to `getopt', zero means this is the first call; initialize.
+
+   When `getopt' returns EOF, this is the index of the first of the
+   non-option elements that the caller should itself scan.
+
+   Otherwise, `optind' communicates from one call to the next
+   how much of ARGV has been scanned so far.  */
+
+/* XXX 1003.2 says this must be 1 before any call.  */
+int optind = 0;
+
+/* The next char to be scanned in the option-element
+   in which the last option character we returned was found.
+   This allows us to pick up the scan where we left off.
+
+   If this is zero, or a null string, it means resume the scan
+   by advancing to the next ARGV-element.  */
+
+static char *nextchar;
+
+/* Callers store zero here to inhibit the error message
+   for unrecognized options.  */
+
+int opterr = 1;
+
+/* Set to an option character which was unrecognized.
+   This must be initialized on some systems to avoid linking in the
+   system's own getopt implementation.  */
+
+int optopt = '?';
+
+/* Describe how to deal with options that follow non-option ARGV-elements.
+
+   If the caller did not specify anything,
+   the default is REQUIRE_ORDER if the environment variable
+   POSIXLY_CORRECT is defined, PERMUTE otherwise.
+
+   REQUIRE_ORDER means don't recognize them as options;
+   stop option processing when the first non-option is seen.
+   This is what Unix does.
+   This mode of operation is selected by either setting the environment
+   variable POSIXLY_CORRECT, or using `+' as the first character
+   of the list of option characters.
+
+   PERMUTE is the default.  We permute the contents of ARGV as we scan,
+   so that eventually all the non-options are at the end.  This allows options
+   to be given in any order, even with programs that were not written to
+   expect this.
+
+   RETURN_IN_ORDER is an option available to programs that were written
+   to expect options and other ARGV-elements in any order and that care about
+   the ordering of the two.  We describe each non-option ARGV-element
+   as if it were the argument of an option with character code 1.
+   Using `-' as the first character of the list of option characters
+   selects this mode of operation.
+
+   The special argument `--' forces an end of option-scanning regardless
+   of the value of `ordering'.  In the case of RETURN_IN_ORDER, only
+   `--' can cause `getopt' to return EOF with `optind' != ARGC.  */
+
+static enum
+  {
+    REQUIRE_ORDER, PERMUTE, RETURN_IN_ORDER
+  }
+ordering;
+
+
+#ifdef	__GNU_LIBRARY__
+/* We want to avoid inclusion of string.h with non-GNU libraries
+   because there are many ways it can cause trouble.
+   On some systems, it contains special magic macros that don't work
+   in GCC.  */
+#include <string.h>
+#define	my_index	strchr
+#else
+
+/* Avoid depending on library functions or files
+   whose names are inconsistent.  */
+
+char *getenv ();
+
+static char *
+my_index (str, chr)
+     const char *str;
+     int chr;
+{
+  while (*str)
+    {
+      if (*str == chr)
+	return (char *) str;
+      str++;
+    }
+  return 0;
+}
+
+/* If using GCC, we can safely declare strlen this way.
+   If not using GCC, it is ok not to declare it.  */
+#ifdef __GNUC__
+#ifdef IN_GCC
+#include "gstddef.h"
+#else
+#include <stddef.h>
+#endif
+extern size_t strlen (const char *);
+#endif
+
+#endif /* GNU C library.  */
+
+
+/* Handle permutation of arguments.  */
+
+/* Describe the part of ARGV that contains non-options that have
+   been skipped.  `first_nonopt' is the index in ARGV of the first of them;
+   `last_nonopt' is the index after the last of them.  */
+
+static int first_nonopt;
+static int last_nonopt;
+
+/* Exchange two adjacent subsequences of ARGV.
+   One subsequence is elements [first_nonopt,last_nonopt)
+   which contains all the non-options that have been skipped so far.
+   The other is elements [last_nonopt,optind), which contains all
+   the options processed since those non-options were skipped.
+
+   `first_nonopt' and `last_nonopt' are relocated so that they describe
+   the new indices of the non-options in ARGV after they are moved.  */
+
+static void
+exchange (argv)
+     char **argv;
+{
+  int bottom = first_nonopt;
+  int middle = last_nonopt;
+  int top = optind;
+  char *tem;
+
+  /* Exchange the shorter segment with the far end of the longer segment.
+     That puts the shorter segment into the right place.
+     It leaves the longer segment in the right place overall,
+     but it consists of two parts that need to be swapped next.  */
+
+  while (top > middle && middle > bottom)
+    {
+      if (top - middle > middle - bottom)
+	{
+	  /* Bottom segment is the short one.  */
+	  int len = middle - bottom;
+	  register int i;
+
+	  /* Swap it with the top part of the top segment.  */
+	  for (i = 0; i < len; i++)
+	    {
+	      tem = argv[bottom + i];
+	      argv[bottom + i] = argv[top - (middle - bottom) + i];
+	      argv[top - (middle - bottom) + i] = tem;
+	    }
+	  /* Exclude the moved bottom segment from further swapping.  */
+	  top -= len;
+	}
+      else
+	{
+	  /* Top segment is the short one.  */
+	  int len = top - middle;
+	  register int i;
+
+	  /* Swap it with the bottom part of the bottom segment.  */
+	  for (i = 0; i < len; i++)
+	    {
+	      tem = argv[bottom + i];
+	      argv[bottom + i] = argv[middle + i];
+	      argv[middle + i] = tem;
+	    }
+	  /* Exclude the moved top segment from further swapping.  */
+	  bottom += len;
+	}
+    }
+
+  /* Update records for the slots the non-options now occupy.  */
+
+  first_nonopt += (optind - last_nonopt);
+  last_nonopt = optind;
+}
+
+
+/* Scan elements of ARGV (whose length is ARGC) for option characters
+   given in OPTSTRING.
+
+   If an element of ARGV starts with '-', and is not exactly "-" or "--",
+   then it is an option element.  The characters of this element
+   (aside from the initial '-') are option characters.  If `getopt'
+   is called repeatedly, it returns successively each of the option characters
+   from each of the option elements.
+
+   If `getopt' finds another option character, it returns that character,
+   updating `optind' and `nextchar' so that the next call to `getopt' can
+   resume the scan with the following option character or ARGV-element.
+
+   If there are no more option characters, `getopt' returns `EOF'.
+   Then `optind' is the index in ARGV of the first ARGV-element
+   that is not an option.  (The ARGV-elements have been permuted
+   so that those that are not options now come last.)
+
+   OPTSTRING is a string containing the legitimate option characters.
+   If an option character is seen that is not listed in OPTSTRING,
+   return '?' after printing an error message.  If you set `opterr' to
+   zero, the error message is suppressed but we still return '?'.
+
+   If a char in OPTSTRING is followed by a colon, that means it wants an arg,
+   so the following text in the same ARGV-element, or the text of the following
+   ARGV-element, is returned in `optarg'.  Two colons mean an option that
+   wants an optional arg; if there is text in the current ARGV-element,
+   it is returned in `optarg', otherwise `optarg' is set to zero.
+
+   If OPTSTRING starts with `-' or `+', it requests different methods of
+   handling the non-option ARGV-elements.
+   See the comments about RETURN_IN_ORDER and REQUIRE_ORDER, above.
+
+   Long-named options begin with `--' instead of `-'.
+   Their names may be abbreviated as long as the abbreviation is unique
+   or is an exact match for some defined option.  If they have an
+   argument, it follows the option name in the same ARGV-element, separated
+   from the option name by a `=', or else the in next ARGV-element.
+   When `getopt' finds a long-named option, it returns 0 if that option's
+   `flag' field is nonzero, the value of the option's `val' field
+   if the `flag' field is zero.
+
+   The elements of ARGV aren't really const, because we permute them.
+   But we pretend they're const in the prototype to be compatible
+   with other systems.
+
+   LONGOPTS is a vector of `struct option' terminated by an
+   element containing a name which is zero.
+
+   LONGIND returns the index in LONGOPT of the long-named option found.
+   It is only valid when a long-named option has been found by the most
+   recent call.
+
+   If LONG_ONLY is nonzero, '-' as well as '--' can introduce
+   long-named options.  */
+
+int
+_getopt_internal (argc, argv, optstring, longopts, longind, long_only)
+     int argc;
+     char *const *argv;
+     const char *optstring;
+     const struct option *longopts;
+     int *longind;
+     int long_only;
+{
+  int option_index;
+
+  optarg = 0;
+
+  /* Initialize the internal data when the first call is made.
+     Start processing options with ARGV-element 1 (since ARGV-element 0
+     is the program name); the sequence of previously skipped
+     non-option ARGV-elements is empty.  */
+
+  if (optind == 0)
+    {
+      first_nonopt = last_nonopt = optind = 1;
+
+      nextchar = NULL;
+
+      /* Determine how to handle the ordering of options and nonoptions.  */
+
+      if (optstring[0] == '-')
+	{
+	  ordering = RETURN_IN_ORDER;
+	  ++optstring;
+	}
+      else if (optstring[0] == '+')
+	{
+	  ordering = REQUIRE_ORDER;
+	  ++optstring;
+	}
+      else if (getenv ("POSIXLY_CORRECT") != NULL)
+	ordering = REQUIRE_ORDER;
+      else
+	ordering = PERMUTE;
+    }
+
+  if (nextchar == NULL || *nextchar == '\0')
+    {
+      if (ordering == PERMUTE)
+	{
+	  /* If we have just processed some options following some non-options,
+	     exchange them so that the options come first.  */
+
+	  if (first_nonopt != last_nonopt && last_nonopt != optind)
+	    exchange ((char **) argv);
+	  else if (last_nonopt != optind)
+	    first_nonopt = optind;
+
+	  /* Now skip any additional non-options
+	     and extend the range of non-options previously skipped.  */
+
+	  while (optind < argc
+		 && (argv[optind][0] != '-' || argv[optind][1] == '\0')
+#ifdef GETOPT_COMPAT
+		 && (longopts == NULL
+		     || argv[optind][0] != '+' || argv[optind][1] == '\0')
+#endif /* GETOPT_COMPAT */
+	    )
+	    optind++;
+	  last_nonopt = optind;
+	}
+
+      /* Special ARGV-element `--' means premature end of options.
+         Skip it like a null option,
+         then exchange with previous non-options as if it were an option,
+         then skip everything else like a non-option.  */
+
+      if (optind != argc && !strcmp (argv[optind], "--"))
+	{
+	  optind++;
+
+	  if (first_nonopt != last_nonopt && last_nonopt != optind)
+	    exchange ((char **) argv);
+	  else if (first_nonopt == last_nonopt)
+	    first_nonopt = optind;
+	  last_nonopt = argc;
+
+	  optind = argc;
+	}
+
+      /* If we have done all the ARGV-elements, stop the scan
+         and back over any non-options that we skipped and permuted.  */
+
+      if (optind == argc)
+	{
+	  /* Set the next-arg-index to point at the non-options
+	     that we previously skipped, so the caller will digest them.  */
+	  if (first_nonopt != last_nonopt)
+	    optind = first_nonopt;
+	  return EOF;
+	}
+
+      /* If we have come to a non-option and did not permute it,
+         either stop the scan or describe it to the caller and pass it by.  */
+
+      if ((argv[optind][0] != '-' || argv[optind][1] == '\0')
+#ifdef GETOPT_COMPAT
+	  && (longopts == NULL
+	      || argv[optind][0] != '+' || argv[optind][1] == '\0')
+#endif /* GETOPT_COMPAT */
+	)
+	{
+	  if (ordering == REQUIRE_ORDER)
+	    return EOF;
+	  optarg = argv[optind++];
+	  return 1;
+	}
+
+      /* We have found another option-ARGV-element.
+         Start decoding its characters.  */
+
+      nextchar = (argv[optind] + 1
+		  + (longopts != NULL && argv[optind][1] == '-'));
+    }
+
+  if (longopts != NULL
+      && ((argv[optind][0] == '-'
+	   && (argv[optind][1] == '-' || long_only))
+#ifdef GETOPT_COMPAT
+	  || argv[optind][0] == '+'
+#endif /* GETOPT_COMPAT */
+      ))
+    {
+      const struct option *p;
+      char *s = nextchar;
+      int exact = 0;
+      int ambig = 0;
+      const struct option *pfound = NULL;
+      int indfound;
+
+      while (*s && *s != '=')
+	s++;
+
+      /* Test all options for either exact match or abbreviated matches.  */
+      for (p = longopts, option_index = 0; p->name;
+	   p++, option_index++)
+	if (!strncmp (p->name, nextchar, s - nextchar))
+	  {
+	    if (s - nextchar == strlen (p->name))
+	      {
+		/* Exact match found.  */
+		pfound = p;
+		indfound = option_index;
+		exact = 1;
+		break;
+	      }
+	    else if (pfound == NULL)
+	      {
+		/* First nonexact match found.  */
+		pfound = p;
+		indfound = option_index;
+	      }
+	    else
+	      /* Second nonexact match found.  */
+	      ambig = 1;
+	  }
+
+      if (ambig && !exact)
+	{
+	  if (opterr)
+	    fprintf (stderr, "%s: option `%s' is ambiguous\n",
+		     argv[0], argv[optind]);
+	  nextchar += strlen (nextchar);
+	  optind++;
+	  return '?';
+	}
+
+      if (pfound != NULL)
+	{
+	  option_index = indfound;
+	  optind++;
+	  if (*s)
+	    {
+	      /* Don't test has_arg with >, because some C compilers don't
+	         allow it to be used on enums.  */
+	      if (pfound->has_arg)
+		optarg = s + 1;
+	      else
+		{
+		  if (opterr)
+		    {
+		      if (argv[optind - 1][1] == '-')
+			/* --option */
+			fprintf (stderr,
+			    "%s: option `--%s' doesn't allow an argument\n",
+				 argv[0], pfound->name);
+		      else
+			/* +option or -option */
+			fprintf (stderr,
+			    "%s: option `%c%s' doesn't allow an argument\n",
+				 argv[0], argv[optind - 1][0], pfound->name);
+		    }
+		  nextchar += strlen (nextchar);
+		  return '?';
+		}
+	    }
+	  else if (pfound->has_arg == 1)
+	    {
+	      if (optind < argc)
+		optarg = argv[optind++];
+	      else
+		{
+		  if (opterr)
+		    fprintf (stderr, "%s: option `%s' requires an argument\n",
+			     argv[0], argv[optind - 1]);
+		  nextchar += strlen (nextchar);
+		  return optstring[0] == ':' ? ':' : '?';
+		}
+	    }
+	  nextchar += strlen (nextchar);
+	  if (longind != NULL)
+	    *longind = option_index;
+	  if (pfound->flag)
+	    {
+	      *(pfound->flag) = pfound->val;
+	      return 0;
+	    }
+	  return pfound->val;
+	}
+      /* Can't find it as a long option.  If this is not getopt_long_only,
+         or the option starts with '--' or is not a valid short
+         option, then it's an error.
+         Otherwise interpret it as a short option.  */
+      if (!long_only || argv[optind][1] == '-'
+#ifdef GETOPT_COMPAT
+	  || argv[optind][0] == '+'
+#endif /* GETOPT_COMPAT */
+	  || my_index (optstring, *nextchar) == NULL)
+	{
+	  if (opterr)
+	    {
+	      if (argv[optind][1] == '-')
+		/* --option */
+		fprintf (stderr, "%s: unrecognized option `--%s'\n",
+			 argv[0], nextchar);
+	      else
+		/* +option or -option */
+		fprintf (stderr, "%s: unrecognized option `%c%s'\n",
+			 argv[0], argv[optind][0], nextchar);
+	    }
+	  nextchar = (char *) "";
+	  optind++;
+	  return '?';
+	}
+    }
+
+  /* Look at and handle the next option-character.  */
+
+  {
+    char c = *nextchar++;
+    char *temp = my_index (optstring, c);
+
+    /* Increment `optind' when we start to process its last character.  */
+    if (*nextchar == '\0')
+      ++optind;
+
+    if (temp == NULL || c == ':')
+      {
+	if (opterr)
+	  {
+#if 0
+	    if (c < 040 || c >= 0177)
+	      fprintf (stderr, "%s: unrecognized option, character code 0%o\n",
+		       argv[0], c);
+	    else
+	      fprintf (stderr, "%s: unrecognized option `-%c'\n", argv[0], c);
+#else
+	    /* 1003.2 specifies the format of this message.  */
+	    fprintf (stderr, "%s: illegal option -- %c\n", argv[0], c);
+#endif
+	  }
+	optopt = c;
+	return '?';
+      }
+    if (temp[1] == ':')
+      {
+	if (temp[2] == ':')
+	  {
+	    /* This is an option that accepts an argument optionally.  */
+	    if (*nextchar != '\0')
+	      {
+		optarg = nextchar;
+		optind++;
+	      }
+	    else
+	      optarg = 0;
+	    nextchar = NULL;
+	  }
+	else
+	  {
+	    /* This is an option that requires an argument.  */
+	    if (*nextchar != '\0')
+	      {
+		optarg = nextchar;
+		/* If we end this ARGV-element by taking the rest as an arg,
+		   we must advance to the next element now.  */
+		optind++;
+	      }
+	    else if (optind == argc)
+	      {
+		if (opterr)
+		  {
+#if 0
+		    fprintf (stderr, "%s: option `-%c' requires an argument\n",
+			     argv[0], c);
+#else
+		    /* 1003.2 specifies the format of this message.  */
+		    fprintf (stderr, "%s: option requires an argument -- %c\n",
+			     argv[0], c);
+#endif
+		  }
+		optopt = c;
+		if (optstring[0] == ':')
+		  c = ':';
+		else
+		  c = '?';
+	      }
+	    else
+	      /* We already incremented `optind' once;
+	         increment it again when taking next ARGV-elt as argument.  */
+	      optarg = argv[optind++];
+	    nextchar = NULL;
+	  }
+      }
+    return c;
+  }
+}
+
+int
+getopt (argc, argv, optstring)
+     int argc;
+     char *const *argv;
+     const char *optstring;
+{
+  return _getopt_internal (argc, argv, optstring,
+			   (const struct option *) 0,
+			   (int *) 0,
+			   0);
+}
+
+#endif /* _LIBC or not __GNU_LIBRARY__.  */
+
+
+#ifdef TEST
+
+/* Compile with -DTEST to make an executable for use in testing
+   the above definition of `getopt'.  */
+
+int
+main (argc, argv)
+     int argc;
+     char **argv;
+{
+  int c;
+  int digit_optind = 0;
+
+  while (1)
+    {
+      int this_option_optind = optind ? optind : 1;
+
+      c = getopt (argc, argv, "abc:d:0123456789");
+      if (c == EOF)
+	break;
+
+      switch (c)
+	{
+	case '0':
+	case '1':
+	case '2':
+	case '3':
+	case '4':
+	case '5':
+	case '6':
+	case '7':
+	case '8':
+	case '9':
+	  if (digit_optind != 0 && digit_optind != this_option_optind)
+	    printf ("digits occur in two different argv-elements.\n");
+	  digit_optind = this_option_optind;
+	  printf ("option %c\n", c);
+	  break;
+
+	case 'a':
+	  printf ("option a\n");
+	  break;
+
+	case 'b':
+	  printf ("option b\n");
+	  break;
+
+	case 'c':
+	  printf ("option c with value `%s'\n", optarg);
+	  break;
+
+	case '?':
+	  break;
+
+	default:
+	  printf ("?? getopt returned character code 0%o ??\n", c);
+	}
+    }
+
+  if (optind < argc)
+    {
+      printf ("non-option ARGV-elements: ");
+      while (optind < argc)
+	printf ("%s ", argv[optind++]);
+      printf ("\n");
+    }
+
+  exit (0);
+}
+
+#endif /* TEST */
Index: /tags/gsdl-21/gsdl/packages/mg/lib/getopt.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/getopt.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/getopt.h	(revision 699)
@@ -0,0 +1,130 @@
+/* Declarations for getopt.
+   Copyright (C) 1989, 1990, 1991, 1992, 1993 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the
+   Free Software Foundation; either version 2, or (at your option) any
+   later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+#ifndef _GETOPT_H
+#define _GETOPT_H 1
+
+#ifdef	__cplusplus
+extern "C"
+{
+#endif
+
+/* For communication from `getopt' to the caller.
+   When `getopt' finds an option that takes an argument,
+   the argument value is returned here.
+   Also, when `ordering' is RETURN_IN_ORDER,
+   each non-option ARGV-element is returned here.  */
+
+  extern char *optarg;
+
+/* Index in ARGV of the next element to be scanned.
+   This is used for communication to and from the caller
+   and for communication between successive calls to `getopt'.
+
+   On entry to `getopt', zero means this is the first call; initialize.
+
+   When `getopt' returns EOF, this is the index of the first of the
+   non-option elements that the caller should itself scan.
+
+   Otherwise, `optind' communicates from one call to the next
+   how much of ARGV has been scanned so far.  */
+
+  extern int optind;
+
+/* Callers store zero here to inhibit the error message `getopt' prints
+   for unrecognized options.  */
+
+  extern int opterr;
+
+/* Set to an option character which was unrecognized.  */
+
+  extern int optopt;
+
+/* Describe the long-named options requested by the application.
+   The LONG_OPTIONS argument to getopt_long or getopt_long_only is a vector
+   of `struct option' terminated by an element containing a name which is
+   zero.
+
+   The field `has_arg' is:
+   no_argument          (or 0) if the option does not take an argument,
+   required_argument    (or 1) if the option requires an argument,
+   optional_argument    (or 2) if the option takes an optional argument.
+
+   If the field `flag' is not NULL, it points to a variable that is set
+   to the value given in the field `val' when the option is found, but
+   left unchanged if the option is not found.
+
+   To have a long-named option do something other than set an `int' to
+   a compiled-in constant, such as set a value from `optarg', set the
+   option's `flag' field to zero and its `val' field to a nonzero
+   value (the equivalent single-letter option character, if there is
+   one).  For long options that have a zero `flag' field, `getopt'
+   returns the contents of the `val' field.  */
+
+  struct option
+    {
+#if	__STDC__
+      const char *name;
+#else
+      char *name;
+#endif
+      /* has_arg can't be an enum because some compilers complain about
+         type mismatches in all the code that assumes it is an int.  */
+      int has_arg;
+      int *flag;
+      int val;
+    };
+
+/* Names for the values of the `has_arg' field of `struct option'.  */
+
+#define	no_argument		0
+#define required_argument	1
+#define optional_argument	2
+
+#if __STDC__
+#if defined(__GNU_LIBRARY__)
+/* Many other libraries have conflicting prototypes for getopt, with
+   differences in the consts, in stdlib.h.  To avoid compilation
+   errors, only prototype getopt for the GNU C library.  */
+  extern int getopt (int argc, char *const *argv, const char *shortopts);
+#else				/* not __GNU_LIBRARY__ */
+  extern int getopt ();
+#endif				/* not __GNU_LIBRARY__ */
+  extern int getopt_long (int argc, char *const *argv, const char *shortopts,
+			  const struct option *longopts, int *longind);
+  extern int getopt_long_only (int argc, char *const *argv,
+			       const char *shortopts,
+			       const struct option *longopts, int *longind);
+
+/* Internal only.  Users should not call this directly.  */
+  extern int _getopt_internal (int argc, char *const *argv,
+			       const char *shortopts,
+			       const struct option *longopts, int *longind,
+			       int long_only);
+#else				/* not __STDC__ */
+  extern int getopt ();
+  extern int getopt_long ();
+  extern int getopt_long_only ();
+
+  extern int _getopt_internal ();
+#endif				/* not __STDC__ */
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif				/* _GETOPT_H */
Index: /tags/gsdl-21/gsdl/packages/mg/lib/getopt1.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/getopt1.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/getopt1.c	(revision 699)
@@ -0,0 +1,178 @@
+/* getopt_long and getopt_long_only entry points for GNU getopt.
+   Copyright (C) 1987, 88, 89, 90, 91, 92, 1993
+   Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the
+   Free Software Foundation; either version 2, or (at your option) any
+   later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+
+#ifdef HAVE_CONFIG_H
+#include "sysfuncs.h"
+#endif
+
+#include "getopt.h"
+
+#if !__STDC__ && !defined(const) && IN_GCC
+#define const
+#endif
+
+#include <stdio.h>
+
+/* Comment out all this code if we are using the GNU C Library, and are not
+   actually compiling the library itself.  This code is part of the GNU C
+   Library, but also included in many other GNU distributions.  Compiling
+   and linking in this code is a waste when using the GNU C library
+   (especially if it is a shared library).  Rather than having every GNU
+   program understand `configure --with-gnu-libc' and omit the object files,
+   it is simpler to just do this in the source for each such file.  */
+
+#if defined (_LIBC) || !defined (__GNU_LIBRARY__)
+
+
+/* This needs to come after some library #include
+   to get __GNU_LIBRARY__ defined.  */
+#ifdef __GNU_LIBRARY__
+#include <stdlib.h>
+#else
+char *getenv ();
+#endif
+
+#ifndef	NULL
+#define NULL 0
+#endif
+
+int
+getopt_long (argc, argv, options, long_options, opt_index)
+     int argc;
+     char *const *argv;
+     const char *options;
+     const struct option *long_options;
+     int *opt_index;
+{
+  return _getopt_internal (argc, argv, options, long_options, opt_index, 0);
+}
+
+/* Like getopt_long, but '-' as well as '--' can indicate a long option.
+   If an option that starts with '-' (not '--') doesn't match a long option,
+   but does match a short option, it is parsed as a short option
+   instead.  */
+
+int
+getopt_long_only (argc, argv, options, long_options, opt_index)
+     int argc;
+     char *const *argv;
+     const char *options;
+     const struct option *long_options;
+     int *opt_index;
+{
+  return _getopt_internal (argc, argv, options, long_options, opt_index, 1);
+}
+
+
+#endif /* _LIBC or not __GNU_LIBRARY__.  */
+
+
+#ifdef TEST
+
+#include <stdio.h>
+
+int
+main (argc, argv)
+     int argc;
+     char **argv;
+{
+  int c;
+  int digit_optind = 0;
+
+  while (1)
+    {
+      int this_option_optind = optind ? optind : 1;
+      int option_index = 0;
+      static struct option long_options[] =
+      {
+	{"add", 1, 0, 0},
+	{"append", 0, 0, 0},
+	{"delete", 1, 0, 0},
+	{"verbose", 0, 0, 0},
+	{"create", 0, 0, 0},
+	{"file", 1, 0, 0},
+	{0, 0, 0, 0}
+      };
+
+      c = getopt_long (argc, argv, "abc:d:0123456789",
+		       long_options, &option_index);
+      if (c == EOF)
+	break;
+
+      switch (c)
+	{
+	case 0:
+	  printf ("option %s", long_options[option_index].name);
+	  if (optarg)
+	    printf (" with arg %s", optarg);
+	  printf ("\n");
+	  break;
+
+	case '0':
+	case '1':
+	case '2':
+	case '3':
+	case '4':
+	case '5':
+	case '6':
+	case '7':
+	case '8':
+	case '9':
+	  if (digit_optind != 0 && digit_optind != this_option_optind)
+	    printf ("digits occur in two different argv-elements.\n");
+	  digit_optind = this_option_optind;
+	  printf ("option %c\n", c);
+	  break;
+
+	case 'a':
+	  printf ("option a\n");
+	  break;
+
+	case 'b':
+	  printf ("option b\n");
+	  break;
+
+	case 'c':
+	  printf ("option c with value `%s'\n", optarg);
+	  break;
+
+	case 'd':
+	  printf ("option d with value `%s'\n", optarg);
+	  break;
+
+	case '?':
+	  break;
+
+	default:
+	  printf ("?? getopt returned character code 0%o ??\n", c);
+	}
+    }
+
+  if (optind < argc)
+    {
+      printf ("non-option ARGV-elements: ");
+      while (optind < argc)
+	printf ("%s ", argv[optind++]);
+      printf ("\n");
+    }
+
+  exit (0);
+}
+
+#endif /* TEST */
Index: /tags/gsdl-21/gsdl/packages/mg/lib/getpagesize.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/getpagesize.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/getpagesize.h	(revision 699)
@@ -0,0 +1,38 @@
+/* Emulate getpagesize on systems that lack it.  */
+
+#ifndef HAVE_GETPAGESIZE
+
+#ifdef VMS
+#define getpagesize() 512
+#endif
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#ifdef _SC_PAGESIZE
+#define getpagesize() sysconf(_SC_PAGESIZE)
+#else
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifndef __WIN32__
+#include <sys/param.h>
+#endif
+
+#ifdef EXEC_PAGESIZE
+#define getpagesize() EXEC_PAGESIZE
+#else
+#ifdef NBPG
+#define getpagesize() NBPG * CLSIZE
+#ifndef CLSIZE
+#define CLSIZE 1
+#endif /* no CLSIZE */
+#else /* no NBPG */
+#ifdef NBPC
+#define getpagesize() NBPC
+#endif /* NBPC */
+#endif /* no NBPG */
+#endif /* no EXEC_PAGESIZE */
+#endif /* no _SC_PAGESIZE */
+
+#endif /* not HAVE_GETPAGESIZE */
Index: /tags/gsdl-21/gsdl/packages/mg/lib/gmalloc.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/gmalloc.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/gmalloc.c	(revision 699)
@@ -0,0 +1,1318 @@
+/* DO NOT EDIT THIS FILE -- it is automagically generated.  -*- C -*- */
+
+#define _MALLOC_INTERNAL
+
+/* The malloc headers and source files from the C library follow here.  */
+
+/* Declarations for `malloc' and friends.
+   Copyright 1990, 1991, 1992, 1993 Free Software Foundation, Inc.
+		  Written May 1989 by Mike Haertel.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.
+
+   The author may be reached (Email) at the address mike@ai.mit.edu,
+   or (US mail) as Mike Haertel c/o Free Software Foundation.  */
+
+#ifndef _MALLOC_H
+
+#define _MALLOC_H	1
+
+#ifdef _MALLOC_INTERNAL
+
+#ifdef	HAVE_CONFIG_H
+#include <sysfuncs.h>
+#endif
+
+#if	defined(_LIBC) || defined(STDC_HEADERS) || defined(USG)
+#include <string.h>
+#else
+#ifndef memset
+#define	memset(s, zero, n)	bzero ((s), (n))
+#endif
+#ifndef memcpy
+#define	memcpy(d, s, n)		bcopy ((s), (d), (n))
+#endif
+#endif
+
+#if	defined(__GNU_LIBRARY__) || __STDC__
+#include <limits.h>
+#else
+#define	CHAR_BIT	8
+#endif
+
+#ifdef	HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#endif	/* _MALLOC_INTERNAL.  */
+
+
+#ifdef	__cplusplus
+extern "C"
+{
+#endif
+
+#if defined (__cplusplus) || __STDC__
+#undef	__P
+#define	__P(args)	args
+#undef	__ptr_t
+#define	__ptr_t		void *
+#else /* Not C++ or ANSI C.  */
+#undef	__P
+#define	__P(args)	()
+#undef	const
+#define	const
+#undef	__ptr_t
+#define	__ptr_t		char *
+#endif /* C++ or ANSI C.  */
+
+#if	__STDC__
+#include <stddef.h>
+#else
+#undef	size_t
+#define	size_t		unsigned int
+#undef	ptrdiff_t
+#define	ptrdiff_t	int
+#endif
+
+#ifndef	NULL
+#define	NULL	0
+#endif
+
+
+/* Allocate SIZE bytes of memory.  */
+extern __ptr_t malloc __P ((size_t __size));
+/* Re-allocate the previously allocated block
+   in __ptr_t, making the new block SIZE bytes long.  */
+extern __ptr_t realloc __P ((__ptr_t __ptr, size_t __size));
+/* Allocate NMEMB elements of SIZE bytes each, all initialized to 0.  */
+extern __ptr_t calloc __P ((size_t __nmemb, size_t __size));
+/* Free a block allocated by `malloc', `realloc' or `calloc'.  */
+extern void free __P ((__ptr_t __ptr));
+
+/* Allocate SIZE bytes allocated to ALIGNMENT bytes.  */
+extern __ptr_t memalign __P ((size_t __alignment, size_t __size));
+
+/* Allocate SIZE bytes on a page boundary.  */
+extern __ptr_t valloc __P ((size_t __size));
+
+
+#ifdef _MALLOC_INTERNAL
+
+/* The allocator divides the heap into blocks of fixed size; large
+   requests receive one or more whole blocks, and small requests
+   receive a fragment of a block.  Fragment sizes are powers of two,
+   and all fragments of a block are the same size.  When all the
+   fragments in a block have been freed, the block itself is freed.  */
+#define INT_BIT		(CHAR_BIT * sizeof(int))
+#define BLOCKLOG	(INT_BIT > 16 ? 12 : 9)
+#define BLOCKSIZE	(1 << BLOCKLOG)
+#define BLOCKIFY(SIZE)	(((SIZE) + BLOCKSIZE - 1) / BLOCKSIZE)
+
+/* Determine the amount of memory spanned by the initial heap table
+   (not an absolute limit).  */
+#define HEAP		(INT_BIT > 16 ? 4194304 : 65536)
+
+/* Number of contiguous free blocks allowed to build up at the end of
+   memory before they will be returned to the system.  */
+#define FINAL_FREE_BLOCKS	8
+
+/* Data structure giving per-block information.  */
+typedef union
+  {
+    /* Heap information for a busy block.  */
+    struct
+      {
+	/* Zero for a large block, or positive giving the
+	   logarithm to the base two of the fragment size.  */
+	int type;
+	union
+	  {
+	    struct
+	      {
+		size_t nfree;	/* Free fragments in a fragmented block.  */
+		size_t first;	/* First free fragment of the block.  */
+	      } frag;
+	    /* Size (in blocks) of a large cluster.  */
+	    size_t size;
+	  } info;
+      } busy;
+    /* Heap information for a free block
+       (that may be the first of a free cluster).  */
+    struct
+      {
+	size_t size;		/* Size (in blocks) of a free cluster.  */
+	size_t next;		/* Index of next free cluster.  */
+	size_t prev;		/* Index of previous free cluster.  */
+      } free;
+  } malloc_info;
+
+/* Pointer to first block of the heap.  */
+extern char *_heapbase;
+
+/* Table indexed by block number giving per-block information.  */
+extern malloc_info *_heapinfo;
+
+/* Address to block number and vice versa.  */
+#define BLOCK(A)	(((char *) (A) - _heapbase) / BLOCKSIZE + 1)
+#define ADDRESS(B)	((__ptr_t) (((B) - 1) * BLOCKSIZE + _heapbase))
+
+/* Current search index for the heap table.  */
+extern size_t _heapindex;
+
+/* Limit of valid info table indices.  */
+extern size_t _heaplimit;
+
+/* Doubly linked lists of free fragments.  */
+struct list
+  {
+    struct list *next;
+    struct list *prev;
+  };
+
+/* Free list headers for each fragment size.  */
+extern struct list _fraghead[];
+
+/* List of blocks allocated with `memalign' (or `valloc').  */
+struct alignlist
+  {
+    struct alignlist *next;
+    __ptr_t aligned;		/* The address that memaligned returned.  */
+    __ptr_t exact;		/* The address that malloc returned.  */
+  };
+extern struct alignlist *_aligned_blocks;
+
+/* Instrumentation.  */
+extern size_t _chunks_used;
+extern size_t _bytes_used;
+extern size_t _chunks_free;
+extern size_t _bytes_free;
+
+/* Internal version of `free' used in `morecore' (malloc.c). */
+extern void _free_internal __P ((__ptr_t __ptr));
+
+#endif /* _MALLOC_INTERNAL.  */
+
+/* Underlying allocation function; successive calls should
+   return contiguous pieces of memory.  */
+extern __ptr_t (*__morecore) __P ((ptrdiff_t __size));
+
+/* Default value of `__morecore'.  */
+extern __ptr_t __default_morecore __P ((ptrdiff_t __size));
+
+/* If not NULL, this function is called after each time
+   `__morecore' is called to increase the data size.  */
+extern void (*__after_morecore_hook) __P ((void));
+
+/* Nonzero if `malloc' has been called and done its initialization.  */
+extern int __malloc_initialized;
+
+/* Hooks for debugging versions.  */
+extern void (*__free_hook) __P ((__ptr_t __ptr));
+extern __ptr_t (*__malloc_hook) __P ((size_t __size));
+extern __ptr_t (*__realloc_hook) __P ((__ptr_t __ptr, size_t __size));
+
+/* Return values for `mprobe': these are the kinds of inconsistencies that
+   `mcheck' enables detection of.  */
+enum mcheck_status
+  {
+    MCHECK_DISABLED = -1,	/* Consistency checking is not turned on.  */
+    MCHECK_OK,			/* Block is fine.  */
+    MCHECK_FREE,		/* Block freed twice.  */
+    MCHECK_HEAD,		/* Memory before the block was clobbered.  */
+    MCHECK_TAIL			/* Memory after the block was clobbered.  */
+  };
+
+/* Activate a standard collection of debugging hooks.  This must be called
+   before `malloc' is ever called.  ABORTFUNC is called with an error code
+   (see enum above) when an inconsistency is detected.  If ABORTFUNC is
+   null, the standard function prints on stderr and then calls `abort'.  */
+extern int mcheck __P ((void (*__abortfunc) __P ((enum mcheck_status))));
+
+/* Check for aberrations in a particular malloc'd block.  You must have
+   called `mcheck' already.  These are the same checks that `mcheck' does
+   when you free or reallocate a block.  */
+extern enum mcheck_status mprobe __P ((__ptr_t __ptr));
+
+/* Activate a standard collection of tracing hooks.  */
+extern void mtrace __P ((void));
+
+/* Statistics available to the user.  */
+struct mstats
+  {
+    size_t bytes_total;		/* Total size of the heap. */
+    size_t chunks_used;		/* Chunks allocated by the user. */
+    size_t bytes_used;		/* Byte total of user-allocated chunks. */
+    size_t chunks_free;		/* Chunks in the free list. */
+    size_t bytes_free;		/* Byte total of chunks in the free list. */
+  };
+
+/* Pick up the current statistics. */
+extern struct mstats mstats __P ((void));
+
+/* Call WARNFUN with a warning message when memory usage is high.  */
+extern void memory_warnings __P ((__ptr_t __start,
+				  void (*__warnfun) __P ((const char *))));
+
+
+/* Relocating allocator.  */
+
+/* Allocate SIZE bytes, and store the address in *HANDLEPTR.  */
+extern __ptr_t r_alloc __P ((__ptr_t *__handleptr, size_t __size));
+
+/* Free the storage allocated in HANDLEPTR.  */
+extern void r_alloc_free __P ((__ptr_t *__handleptr));
+
+/* Adjust the block at HANDLEPTR to be SIZE bytes long.  */
+extern __ptr_t r_re_alloc __P ((__ptr_t *__handleptr, size_t __size));
+
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif /* malloc.h  */
+/* Allocate memory on a page boundary.
+   Copyright (C) 1991, 1992, 1993 Free Software Foundation, Inc.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.
+
+   The author may be reached (Email) at the address mike@ai.mit.edu,
+   or (US mail) as Mike Haertel c/o Free Software Foundation.  */
+
+#if defined (__GNU_LIBRARY__) || defined (_LIBC)
+#include <stddef.h>
+#include <sys/cdefs.h>
+extern size_t __getpagesize __P ((void));
+#else
+#include "getpagesize.h"
+#define	 __getpagesize()	getpagesize()
+#endif
+
+#ifndef	_MALLOC_INTERNAL
+#define	_MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+static size_t pagesize;
+
+__ptr_t
+valloc (size)
+     size_t size;
+{
+  if (pagesize == 0)
+    pagesize = __getpagesize ();
+
+  return memalign (pagesize, size);
+}
+/* Memory allocator `malloc'.
+   Copyright 1990, 1991, 1992, 1993, 1994 Free Software Foundation
+		  Written May 1989 by Mike Haertel.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.
+
+   The author may be reached (Email) at the address mike@ai.mit.edu,
+   or (US mail) as Mike Haertel c/o Free Software Foundation.  */
+
+#ifndef	_MALLOC_INTERNAL
+#define _MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+/* How to really get more memory.  */
+__ptr_t (*__morecore) __P ((ptrdiff_t __size)) = __default_morecore;
+
+/* Debugging hook for `malloc'.  */
+__ptr_t (*__malloc_hook) __P ((size_t __size));
+
+/* Pointer to the base of the first block.  */
+char *_heapbase;
+
+/* Block information table.  Allocated with align/__free (not malloc/free).  */
+malloc_info *_heapinfo;
+
+/* Number of info entries.  */
+static size_t heapsize;
+
+/* Search index in the info table.  */
+size_t _heapindex;
+
+/* Limit of valid info table indices.  */
+size_t _heaplimit;
+
+/* Free lists for each fragment size.  */
+struct list _fraghead[BLOCKLOG];
+
+/* Instrumentation.  */
+size_t _chunks_used;
+size_t _bytes_used;
+size_t _chunks_free;
+size_t _bytes_free;
+
+/* Are you experienced?  */
+int __malloc_initialized;
+
+void (*__after_morecore_hook) __P ((void));
+
+/* Aligned allocation.  */
+static __ptr_t align __P ((size_t));
+static __ptr_t
+align (size)
+     size_t size;
+{
+  __ptr_t result;
+  unsigned long int adj;
+
+  result = (*__morecore) (size);
+  adj = (unsigned long int) ((unsigned long int) ((char *) result -
+						  (char *) NULL)) % BLOCKSIZE;
+  if (adj != 0)
+    {
+      adj = BLOCKSIZE - adj;
+      (void) (*__morecore) (adj);
+      result = (char *) result + adj;
+    }
+
+  if (__after_morecore_hook)
+    (*__after_morecore_hook) ();
+
+  return result;
+}
+
+/* Set everything up and remember that we have.  */
+static int initialize __P ((void));
+static int
+initialize ()
+{
+  heapsize = HEAP / BLOCKSIZE;
+  _heapinfo = (malloc_info *) align (heapsize * sizeof (malloc_info));
+  if (_heapinfo == NULL)
+    return 0;
+  memset (_heapinfo, 0, heapsize * sizeof (malloc_info));
+  _heapinfo[0].free.size = 0;
+  _heapinfo[0].free.next = _heapinfo[0].free.prev = 0;
+  _heapindex = 0;
+  _heapbase = (char *) _heapinfo;
+
+  /* Account for the _heapinfo block itself in the statistics.  */
+  _bytes_used = heapsize * sizeof (malloc_info);
+  _chunks_used = 1;
+
+  __malloc_initialized = 1;
+  return 1;
+}
+
+/* Get neatly aligned memory, initializing or
+   growing the heap info table as necessary. */
+static __ptr_t morecore __P ((size_t));
+static __ptr_t
+morecore (size)
+     size_t size;
+{
+  __ptr_t result;
+  malloc_info *newinfo, *oldinfo;
+  size_t newsize;
+
+  result = align (size);
+  if (result == NULL)
+    return NULL;
+
+  /* Check if we need to grow the info table.  */
+  if ((size_t) BLOCK ((char *) result + size) > heapsize)
+    {
+      newsize = heapsize;
+      while ((size_t) BLOCK ((char *) result + size) > newsize)
+	newsize *= 2;
+      newinfo = (malloc_info *) align (newsize * sizeof (malloc_info));
+      if (newinfo == NULL)
+	{
+	  (*__morecore) (-size);
+	  return NULL;
+	}
+      memcpy (newinfo, _heapinfo, heapsize * sizeof (malloc_info));
+      memset (&newinfo[heapsize], 0,
+	      (newsize - heapsize) * sizeof (malloc_info));
+      oldinfo = _heapinfo;
+      newinfo[BLOCK (oldinfo)].busy.type = 0;
+      newinfo[BLOCK (oldinfo)].busy.info.size
+	= BLOCKIFY (heapsize * sizeof (malloc_info));
+      _heapinfo = newinfo;
+      /* Account for the _heapinfo block itself in the statistics.  */
+      _bytes_used += newsize * sizeof (malloc_info);
+      ++_chunks_used;
+      _free_internal (oldinfo);
+      heapsize = newsize;
+    }
+
+  _heaplimit = BLOCK ((char *) result + size);
+  return result;
+}
+
+/* Allocate memory from the heap.  */
+__ptr_t
+malloc (size)
+     size_t size;
+{
+  __ptr_t result;
+  size_t block, blocks, lastblocks, start;
+  register size_t i;
+  struct list *next;
+
+  /* ANSI C allows `malloc (0)' to either return NULL, or to return a
+     valid address you can realloc and free (though not dereference).
+
+     It turns out that some extant code (sunrpc, at least Ultrix's version)
+     expects `malloc (0)' to return non-NULL and breaks otherwise.
+     Be compatible.  */
+
+#if	0
+  if (size == 0)
+    return NULL;
+#endif
+
+  if (__malloc_hook != NULL)
+    return (*__malloc_hook) (size);
+
+  if (!__malloc_initialized)
+    if (!initialize ())
+      return NULL;
+
+  if (size < sizeof (struct list))
+      size = sizeof (struct list);
+
+#ifdef SUNOS_LOCALTIME_BUG
+  if (size < 16)
+    size = 16;
+#endif
+
+  /* Determine the allocation policy based on the request size.  */
+  if (size <= BLOCKSIZE / 2)
+    {
+      /* Small allocation to receive a fragment of a block.
+	 Determine the logarithm to base two of the fragment size. */
+      register size_t log = 1;
+      --size;
+      while ((size /= 2) != 0)
+	++log;
+
+      /* Look in the fragment lists for a
+	 free fragment of the desired size. */
+      next = _fraghead[log].next;
+      if (next != NULL)
+	{
+	  /* There are free fragments of this size.
+	     Pop a fragment out of the fragment list and return it.
+	     Update the block's nfree and first counters. */
+	  result = (__ptr_t) next;
+	  next->prev->next = next->next;
+	  if (next->next != NULL)
+	    next->next->prev = next->prev;
+	  block = BLOCK (result);
+	  if (--_heapinfo[block].busy.info.frag.nfree != 0)
+	    _heapinfo[block].busy.info.frag.first = (unsigned long int)
+	      ((unsigned long int) ((char *) next->next - (char *) NULL)
+	       % BLOCKSIZE) >> log;
+
+	  /* Update the statistics.  */
+	  ++_chunks_used;
+	  _bytes_used += 1 << log;
+	  --_chunks_free;
+	  _bytes_free -= 1 << log;
+	}
+      else
+	{
+	  /* No free fragments of the desired size, so get a new block
+	     and break it into fragments, returning the first.  */
+	  result = malloc (BLOCKSIZE);
+	  if (result == NULL)
+	    return NULL;
+
+	  /* Link all fragments but the first into the free list.  */
+	  for (i = 1; i < (size_t) (BLOCKSIZE >> log); ++i)
+	    {
+	      next = (struct list *) ((char *) result + (i << log));
+	      next->next = _fraghead[log].next;
+	      next->prev = &_fraghead[log];
+	      next->prev->next = next;
+	      if (next->next != NULL)
+		next->next->prev = next;
+	    }
+
+	  /* Initialize the nfree and first counters for this block.  */
+	  block = BLOCK (result);
+	  _heapinfo[block].busy.type = log;
+	  _heapinfo[block].busy.info.frag.nfree = i - 1;
+	  _heapinfo[block].busy.info.frag.first = i - 1;
+
+	  _chunks_free += (BLOCKSIZE >> log) - 1;
+	  _bytes_free += BLOCKSIZE - (1 << log);
+	  _bytes_used -= BLOCKSIZE - (1 << log);
+	}
+    }
+  else
+    {
+      /* Large allocation to receive one or more blocks.
+	 Search the free list in a circle starting at the last place visited.
+	 If we loop completely around without finding a large enough
+	 space we will have to get more memory from the system.  */
+      blocks = BLOCKIFY (size);
+      start = block = _heapindex;
+      while (_heapinfo[block].free.size < blocks)
+	{
+	  block = _heapinfo[block].free.next;
+	  if (block == start)
+	    {
+	      /* Need to get more from the system.  Check to see if
+		 the new core will be contiguous with the final free
+		 block; if so we don't need to get as much.  */
+	      block = _heapinfo[0].free.prev;
+	      lastblocks = _heapinfo[block].free.size;
+	      if (_heaplimit != 0 && block + lastblocks == _heaplimit &&
+		  (*__morecore) (0) == ADDRESS (block + lastblocks) &&
+		  (morecore ((blocks - lastblocks) * BLOCKSIZE)) != NULL)
+		{
+ 		  /* Which block we are extending (the `final free
+ 		     block' referred to above) might have changed, if
+ 		     it got combined with a freed info table.  */
+ 		  block = _heapinfo[0].free.prev;
+  		  _heapinfo[block].free.size += (blocks - lastblocks);
+		  _bytes_free += (blocks - lastblocks) * BLOCKSIZE;
+		  continue;
+		}
+	      result = morecore (blocks * BLOCKSIZE);
+	      if (result == NULL)
+		return NULL;
+	      block = BLOCK (result);
+	      _heapinfo[block].busy.type = 0;
+	      _heapinfo[block].busy.info.size = blocks;
+	      ++_chunks_used;
+	      _bytes_used += blocks * BLOCKSIZE;
+	      return result;
+	    }
+	}
+
+      /* At this point we have found a suitable free list entry.
+	 Figure out how to remove what we need from the list. */
+      result = ADDRESS (block);
+      if (_heapinfo[block].free.size > blocks)
+	{
+	  /* The block we found has a bit left over,
+	     so relink the tail end back into the free list. */
+	  _heapinfo[block + blocks].free.size
+	    = _heapinfo[block].free.size - blocks;
+	  _heapinfo[block + blocks].free.next
+	    = _heapinfo[block].free.next;
+	  _heapinfo[block + blocks].free.prev
+	    = _heapinfo[block].free.prev;
+	  _heapinfo[_heapinfo[block].free.prev].free.next
+	    = _heapinfo[_heapinfo[block].free.next].free.prev
+	    = _heapindex = block + blocks;
+	}
+      else
+	{
+	  /* The block exactly matches our requirements,
+	     so just remove it from the list. */
+	  _heapinfo[_heapinfo[block].free.next].free.prev
+	    = _heapinfo[block].free.prev;
+	  _heapinfo[_heapinfo[block].free.prev].free.next
+	    = _heapindex = _heapinfo[block].free.next;
+	  --_chunks_free;
+	}
+
+      _heapinfo[block].busy.type = 0;
+      _heapinfo[block].busy.info.size = blocks;
+      ++_chunks_used;
+      _bytes_used += blocks * BLOCKSIZE;
+      _bytes_free -= blocks * BLOCKSIZE;
+    }
+
+  return result;
+}
+
+
+#ifndef _LIBC
+
+/* On some ANSI C systems, some libc functions call _malloc, _free
+   and _realloc.  Make them use the GNU functions.  */
+
+__ptr_t
+_malloc (size)
+     size_t size;
+{
+  return malloc (size);
+}
+
+void
+_free (ptr)
+     __ptr_t ptr;
+{
+  free (ptr);
+}
+
+__ptr_t
+_realloc (ptr, size)
+     __ptr_t ptr;
+     size_t size;
+{
+  return realloc (ptr, size);
+}
+
+#endif
+/* Free a block of memory allocated by `malloc'.
+   Copyright 1990, 1991, 1992 Free Software Foundation
+		  Written May 1989 by Mike Haertel.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.
+
+   The author may be reached (Email) at the address mike@ai.mit.edu,
+   or (US mail) as Mike Haertel c/o Free Software Foundation.  */
+
+#ifndef	_MALLOC_INTERNAL
+#define _MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+/* Debugging hook for free.  */
+void (*__free_hook) __P ((__ptr_t __ptr));
+
+/* List of blocks allocated by memalign.  */
+struct alignlist *_aligned_blocks = NULL;
+
+/* Return memory to the heap.
+   Like `free' but don't call a __free_hook if there is one.  */
+void
+_free_internal (ptr)
+     __ptr_t ptr;
+{
+  int type;
+  size_t block, blocks;
+  register size_t i;
+  struct list *prev, *next;
+
+  block = BLOCK (ptr);
+
+  type = _heapinfo[block].busy.type;
+  switch (type)
+    {
+    case 0:
+      /* Get as many statistics as early as we can.  */
+      --_chunks_used;
+      _bytes_used -= _heapinfo[block].busy.info.size * BLOCKSIZE;
+      _bytes_free += _heapinfo[block].busy.info.size * BLOCKSIZE;
+
+      /* Find the free cluster previous to this one in the free list.
+	 Start searching at the last block referenced; this may benefit
+	 programs with locality of allocation.  */
+      i = _heapindex;
+      if (i > block)
+	while (i > block)
+	  i = _heapinfo[i].free.prev;
+      else
+	{
+	  do
+	    i = _heapinfo[i].free.next;
+	  while (i > 0 && i < block);
+	  i = _heapinfo[i].free.prev;
+	}
+
+      /* Determine how to link this block into the free list.  */
+      if (block == i + _heapinfo[i].free.size)
+	{
+	  /* Coalesce this block with its predecessor.  */
+	  _heapinfo[i].free.size += _heapinfo[block].busy.info.size;
+	  block = i;
+	}
+      else
+	{
+	  /* Really link this block back into the free list.  */
+	  _heapinfo[block].free.size = _heapinfo[block].busy.info.size;
+	  _heapinfo[block].free.next = _heapinfo[i].free.next;
+	  _heapinfo[block].free.prev = i;
+	  _heapinfo[i].free.next = block;
+	  _heapinfo[_heapinfo[block].free.next].free.prev = block;
+	  ++_chunks_free;
+	}
+
+      /* Now that the block is linked in, see if we can coalesce it
+	 with its successor (by deleting its successor from the list
+	 and adding in its size).  */
+      if (block + _heapinfo[block].free.size == _heapinfo[block].free.next)
+	{
+	  _heapinfo[block].free.size
+	    += _heapinfo[_heapinfo[block].free.next].free.size;
+	  _heapinfo[block].free.next
+	    = _heapinfo[_heapinfo[block].free.next].free.next;
+	  _heapinfo[_heapinfo[block].free.next].free.prev = block;
+	  --_chunks_free;
+	}
+
+      /* Now see if we can return stuff to the system.  */
+      blocks = _heapinfo[block].free.size;
+      if (blocks >= FINAL_FREE_BLOCKS && block + blocks == _heaplimit
+	  && (*__morecore) (0) == ADDRESS (block + blocks))
+	{
+	  register size_t bytes = blocks * BLOCKSIZE;
+	  _heaplimit -= blocks;
+	  (*__morecore) (-bytes);
+	  _heapinfo[_heapinfo[block].free.prev].free.next
+	    = _heapinfo[block].free.next;
+	  _heapinfo[_heapinfo[block].free.next].free.prev
+	    = _heapinfo[block].free.prev;
+	  block = _heapinfo[block].free.prev;
+	  --_chunks_free;
+	  _bytes_free -= bytes;
+	}
+
+      /* Set the next search to begin at this block.  */
+      _heapindex = block;
+      break;
+
+    default:
+      /* Do some of the statistics.  */
+      --_chunks_used;
+      _bytes_used -= 1 << type;
+      ++_chunks_free;
+      _bytes_free += 1 << type;
+
+      /* Get the address of the first free fragment in this block.  */
+      prev = (struct list *) ((char *) ADDRESS (block) +
+			   (_heapinfo[block].busy.info.frag.first << type));
+
+      if (_heapinfo[block].busy.info.frag.nfree == (BLOCKSIZE >> type) - 1)
+	{
+	  /* If all fragments of this block are free, remove them
+	     from the fragment list and free the whole block.  */
+	  next = prev;
+	  for (i = 1; i < (size_t) (BLOCKSIZE >> type); ++i)
+	    next = next->next;
+	  prev->prev->next = next;
+	  if (next != NULL)
+	    next->prev = prev->prev;
+	  _heapinfo[block].busy.type = 0;
+	  _heapinfo[block].busy.info.size = 1;
+
+	  /* Keep the statistics accurate.  */
+	  ++_chunks_used;
+	  _bytes_used += BLOCKSIZE;
+	  _chunks_free -= BLOCKSIZE >> type;
+	  _bytes_free -= BLOCKSIZE;
+
+	  free (ADDRESS (block));
+	}
+      else if (_heapinfo[block].busy.info.frag.nfree != 0)
+	{
+	  /* If some fragments of this block are free, link this
+	     fragment into the fragment list after the first free
+	     fragment of this block. */
+	  next = (struct list *) ptr;
+	  next->next = prev->next;
+	  next->prev = prev;
+	  prev->next = next;
+	  if (next->next != NULL)
+	    next->next->prev = next;
+	  ++_heapinfo[block].busy.info.frag.nfree;
+	}
+      else
+	{
+	  /* No fragments of this block are free, so link this
+	     fragment into the fragment list and announce that
+	     it is the first free fragment of this block. */
+	  prev = (struct list *) ptr;
+	  _heapinfo[block].busy.info.frag.nfree = 1;
+	  _heapinfo[block].busy.info.frag.first = (unsigned long int)
+	    ((unsigned long int) ((char *) ptr - (char *) NULL)
+	     % BLOCKSIZE >> type);
+	  prev->next = _fraghead[type].next;
+	  prev->prev = &_fraghead[type];
+	  prev->prev->next = prev;
+	  if (prev->next != NULL)
+	    prev->next->prev = prev;
+	}
+      break;
+    }
+}
+
+/* Return memory to the heap.  */
+void
+free (ptr)
+     __ptr_t ptr;
+{
+  register struct alignlist *l;
+
+  if (ptr == NULL)
+    return;
+
+  for (l = _aligned_blocks; l != NULL; l = l->next)
+    if (l->aligned == ptr)
+      {
+	l->aligned = NULL;	/* Mark the slot in the list as free.  */
+	ptr = l->exact;
+	break;
+      }
+
+  if (__free_hook != NULL)
+    (*__free_hook) (ptr);
+  else
+    _free_internal (ptr);
+}
+/* Copyright (C) 1991, 1993, 1994 Free Software Foundation, Inc.
+This file is part of the GNU C Library.
+
+The GNU C Library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+The GNU C Library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with the GNU C Library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.  */
+
+#ifndef	_MALLOC_INTERNAL
+#define _MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+#ifdef _LIBC
+
+#include <ansidecl.h>
+#include <gnu-stabs.h>
+
+#undef	cfree
+
+function_alias(cfree, free, void, (ptr),
+	       DEFUN(cfree, (ptr), PTR ptr))
+
+#else
+
+void
+cfree (ptr)
+     __ptr_t ptr;
+{
+  free (ptr);
+}
+
+#endif
+/* Change the size of a block allocated by `malloc'.
+   Copyright 1990, 1991, 1992, 1993, 1994 Free Software Foundation, Inc.
+		     Written May 1989 by Mike Haertel.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.
+
+   The author may be reached (Email) at the address mike@ai.mit.edu,
+   or (US mail) as Mike Haertel c/o Free Software Foundation.  */
+
+#ifndef	_MALLOC_INTERNAL
+#define _MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+#if  (defined (MEMMOVE_MISSING) || \
+      !defined(_LIBC) && !defined(STDC_HEADERS) && !defined(USG))
+
+/* Snarfed directly from Emacs src/dispnew.c:
+   XXX Should use system bcopy if it handles overlap.  */
+#ifndef emacs
+
+/* Like bcopy except never gets confused by overlap.  */
+
+static void
+safe_bcopy (from, to, size)
+     char *from, *to;
+     int size;
+{
+  if (size <= 0 || from == to)
+    return;
+
+  /* If the source and destination don't overlap, then bcopy can
+     handle it.  If they do overlap, but the destination is lower in
+     memory than the source, we'll assume bcopy can handle that.  */
+  if (to < from || from + size <= to)
+    bcopy (from, to, size);
+
+  /* Otherwise, we'll copy from the end.  */
+  else
+    {
+      register char *endf = from + size;
+      register char *endt = to + size;
+
+      /* If TO - FROM is large, then we should break the copy into
+	 nonoverlapping chunks of TO - FROM bytes each.  However, if
+	 TO - FROM is small, then the bcopy function call overhead
+	 makes this not worth it.  The crossover point could be about
+	 anywhere.  Since I don't think the obvious copy loop is too
+	 bad, I'm trying to err in its favor.  */
+      if (to - from < 64)
+	{
+	  do
+	    *--endt = *--endf;
+	  while (endf != from);
+	}
+      else
+	{
+	  for (;;)
+	    {
+	      endt -= (to - from);
+	      endf -= (to - from);
+
+	      if (endt < to)
+		break;
+
+	      bcopy (endf, endt, to - from);
+	    }
+
+	  /* If SIZE wasn't a multiple of TO - FROM, there will be a
+	     little left over.  The amount left over is
+	     (endt + (to - from)) - to, which is endt - from.  */
+	  bcopy (from, to, endt - from);
+	}
+    }
+}     
+#endif	/* Not emacs.  */
+
+#define memmove(to, from, size) safe_bcopy ((from), (to), (size))
+
+#endif
+
+
+#define min(A, B) ((A) < (B) ? (A) : (B))
+
+/* Debugging hook for realloc.  */
+__ptr_t (*__realloc_hook) __P ((__ptr_t __ptr, size_t __size));
+
+/* Resize the given region to the new size, returning a pointer
+   to the (possibly moved) region.  This is optimized for speed;
+   some benchmarks seem to indicate that greater compactness is
+   achieved by unconditionally allocating and copying to a
+   new region.  This module has incestuous knowledge of the
+   internals of both free and malloc. */
+__ptr_t
+realloc (ptr, size)
+     __ptr_t ptr;
+     size_t size;
+{
+  __ptr_t result;
+  int type;
+  size_t block, blocks, oldlimit;
+
+  if (size == 0)
+    {
+      free (ptr);
+      return malloc (0);
+    }
+  else if (ptr == NULL)
+    return malloc (size);
+
+  if (__realloc_hook != NULL)
+    return (*__realloc_hook) (ptr, size);
+
+  block = BLOCK (ptr);
+
+  type = _heapinfo[block].busy.type;
+  switch (type)
+    {
+    case 0:
+      /* Maybe reallocate a large block to a small fragment.  */
+      if (size <= BLOCKSIZE / 2)
+	{
+	  result = malloc (size);
+	  if (result != NULL)
+	    {
+	      memcpy (result, ptr, size);
+	      _free_internal (ptr);
+	      return result;
+	    }
+	}
+
+      /* The new size is a large allocation as well;
+	 see if we can hold it in place. */
+      blocks = BLOCKIFY (size);
+      if (blocks < _heapinfo[block].busy.info.size)
+	{
+	  /* The new size is smaller; return
+	     excess memory to the free list. */
+	  _heapinfo[block + blocks].busy.type = 0;
+	  _heapinfo[block + blocks].busy.info.size
+	    = _heapinfo[block].busy.info.size - blocks;
+	  _heapinfo[block].busy.info.size = blocks;
+	  /* We have just created a new chunk by splitting a chunk in two.
+	     Now we will free this chunk; increment the statistics counter
+	     so it doesn't become wrong when _free_internal decrements it.  */
+	  ++_chunks_used;
+	  _free_internal (ADDRESS (block + blocks));
+	  result = ptr;
+	}
+      else if (blocks == _heapinfo[block].busy.info.size)
+	/* No size change necessary.  */
+	result = ptr;
+      else
+	{
+	  /* Won't fit, so allocate a new region that will.
+	     Free the old region first in case there is sufficient
+	     adjacent free space to grow without moving. */
+	  blocks = _heapinfo[block].busy.info.size;
+	  /* Prevent free from actually returning memory to the system.  */
+	  oldlimit = _heaplimit;
+	  _heaplimit = 0;
+	  _free_internal (ptr);
+	  _heaplimit = oldlimit;
+	  result = malloc (size);
+	  if (result == NULL)
+	    {
+	      /* Now we're really in trouble.  We have to unfree
+		 the thing we just freed.  Unfortunately it might
+		 have been coalesced with its neighbors.  */
+	      if (_heapindex == block)
+	        (void) malloc (blocks * BLOCKSIZE);
+	      else
+		{
+		  __ptr_t previous = malloc ((block - _heapindex) * BLOCKSIZE);
+		  (void) malloc (blocks * BLOCKSIZE);
+		  _free_internal (previous);
+		}
+	      return NULL;
+	    }
+	  if (ptr != result)
+	    memmove (result, ptr, blocks * BLOCKSIZE);
+	}
+      break;
+
+    default:
+      /* Old size is a fragment; type is logarithm
+	 to base two of the fragment size.  */
+      if (size > (size_t) (1 << (type - 1)) && size <= (size_t) (1 << type))
+	/* The new size is the same kind of fragment.  */
+	result = ptr;
+      else
+	{
+	  /* The new size is different; allocate a new space,
+	     and copy the lesser of the new size and the old. */
+	  result = malloc (size);
+	  if (result == NULL)
+	    return NULL;
+	  memcpy (result, ptr, min (size, (size_t) 1 << type));
+	  free (ptr);
+	}
+      break;
+    }
+
+  return result;
+}
+/* Copyright (C) 1991, 1992 Free Software Foundation, Inc.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.
+
+   The author may be reached (Email) at the address mike@ai.mit.edu,
+   or (US mail) as Mike Haertel c/o Free Software Foundation.  */
+
+#ifndef	_MALLOC_INTERNAL
+#define	_MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+/* Allocate an array of NMEMB elements each SIZE bytes long.
+   The entire array is initialized to zeros.  */
+__ptr_t
+calloc (nmemb, size)
+     register size_t nmemb;
+     register size_t size;
+{
+  register __ptr_t result = malloc (nmemb * size);
+
+  if (result != NULL)
+    (void) memset (result, 0, nmemb * size);
+
+  return result;
+}
+/* Copyright (C) 1991, 1992, 1993 Free Software Foundation, Inc.
+This file is part of the GNU C Library.
+
+The GNU C Library is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2, or (at your option)
+any later version.
+
+The GNU C Library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with the GNU C Library; see the file COPYING.  If not, write to
+the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+#ifndef	_MALLOC_INTERNAL
+#define	_MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+#ifndef	__GNU_LIBRARY__
+#define	__sbrk	sbrk
+#endif
+
+#ifdef __GNU_LIBRARY__
+/* It is best not to declare this and cast its result on foreign operating
+   systems with potentially hostile include files.  */
+extern __ptr_t __sbrk __P ((int increment));
+#endif
+
+#ifndef NULL
+#define NULL 0
+#endif
+
+/* Allocate INCREMENT more bytes of data space,
+   and return the start of data space, or NULL on errors.
+   If INCREMENT is negative, shrink data space.  */
+__ptr_t
+__default_morecore (increment)
+     ptrdiff_t increment;
+{
+  __ptr_t result = (__ptr_t) __sbrk ((int) increment);
+  if (result == (__ptr_t) -1)
+    return NULL;
+  return result;
+}
+/* Copyright (C) 1991, 1992, 1993 Free Software Foundation, Inc.
+
+This library is free software; you can redistribute it and/or
+modify it under the terms of the GNU Library General Public License as
+published by the Free Software Foundation; either version 2 of the
+License, or (at your option) any later version.
+
+This library is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this library; see the file COPYING.LIB.  If
+not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.  */
+
+#ifndef	_MALLOC_INTERNAL
+#define _MALLOC_INTERNAL
+#include <malloc.h>
+#endif
+
+__ptr_t
+memalign (alignment, size)
+     size_t alignment;
+     size_t size;
+{
+  __ptr_t result;
+  unsigned long int adj;
+
+  size = ((size + alignment - 1) / alignment) * alignment;
+
+  result = malloc (size);
+  if (result == NULL)
+    return NULL;
+  adj = (unsigned long int) ((unsigned long int) ((char *) result -
+						  (char *) NULL)) % alignment;
+  if (adj != 0)
+    {
+      struct alignlist *l;
+      for (l = _aligned_blocks; l != NULL; l = l->next)
+	if (l->aligned == NULL)
+	  /* This slot is free.  Use it.  */
+	  break;
+      if (l == NULL)
+	{
+	  l = (struct alignlist *) malloc (sizeof (struct alignlist));
+	  if (l == NULL)
+	    {
+	      free (result);
+	      return NULL;
+	    }
+	  l->next = _aligned_blocks;
+	  _aligned_blocks = l;
+	}
+      l->exact = result;
+      result = l->aligned = (char *) result + alignment - adj;
+    }
+
+  return result;
+}
Index: /tags/gsdl-21/gsdl/packages/mg/lib/huffman.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/huffman.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/huffman.c	(revision 699)
@@ -0,0 +1,423 @@
+/**************************************************************************
+ *
+ * huffman.c -- Huffman coding functions
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:16:51  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:32:03  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/08/22  00:24:44  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+#include "memlib.h"
+#include "huffman.h"
+#include "messages.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+/* ->data on success,  
+   NULL on error 
+ */
+huff_data *
+Generate_Huffman_Data (int num, long *freqs, huff_data * data,
+		       u_long * mem)
+{
+  int HNum, i, count;
+  unsigned long *heap;
+  huff_data *hd = data;
+  if (!hd)
+    {
+      if (!(hd = Xmalloc (sizeof (huff_data))))
+	goto error0;
+    }
+  if (!(heap = Xmalloc (num * 2 * sizeof (*heap))))
+    goto error1;
+  if (!(hd->clens = Xmalloc (num * sizeof (char))))
+      goto error2;
+  hd->num_codes = num;
+  if (mem)
+    {
+      if (hd != data)
+	*mem += sizeof (huff_data);
+      *mem += num * sizeof (char);
+    }
+  bcopy ((char *) freqs, (char *) &heap[num], num * sizeof (*heap));
+  bzero ((char *) heap, num * sizeof (*heap));
+
+  /* Initialise the pointers to the leaves */
+  for (count = i = 0; i < num; i++)
+    if (heap[num + i])
+      heap[count++] = num + i;
+
+  /* Reorganise the pointers so that it is a heap */
+  HNum = count;
+  for (i = HNum / 2; i > 0; i--)
+    {
+      register int curr, child;
+      curr = i;
+      child = curr * 2;
+      while (child <= HNum)
+	{
+	  if (child < HNum && heap[heap[child]] < heap[heap[child - 1]])
+	    child++;
+	  if (heap[heap[curr - 1]] > heap[heap[child - 1]])
+	    {
+	      register u_long temp;
+	      temp = heap[child - 1];
+	      heap[child - 1] = heap[curr - 1];
+	      heap[curr - 1] = temp;
+	      curr = child;
+	      child = 2 * curr;
+	    }
+	  else
+	    break;
+	}
+    }
+
+
+  /* Form the huffman tree */
+  while (HNum > 1)
+    {
+      int pos[2];
+
+      for (i = 0; i < 2; i++)
+	{
+	  register int curr, child;
+	  pos[i] = heap[0];
+	  heap[0] = heap[--HNum];
+	  curr = 1;
+	  child = 2;
+	  while (child <= HNum)
+	    {
+	      if (child < HNum &&
+		  heap[heap[child]] < heap[heap[child - 1]])
+		child++;
+	      if (heap[heap[curr - 1]] > heap[heap[child - 1]])
+		{
+		  register int temp;
+		  temp = heap[child - 1];
+		  heap[child - 1] = heap[curr - 1];
+		  heap[curr - 1] = temp;
+		  curr = child;
+		  child = 2 * curr;
+		}
+	      else
+		break;
+	    }
+	}
+
+      heap[HNum + 1] = heap[pos[0]] + heap[pos[1]];
+      heap[pos[0]] = heap[pos[1]] = HNum + 1;
+
+      heap[HNum] = HNum + 1;
+
+      {
+	register int parent, curr;
+	HNum++;
+	curr = HNum;
+	parent = curr >> 1;
+	while (parent && heap[heap[parent - 1]] > heap[heap[curr - 1]])
+	  {
+	    register int temp;
+	    temp = heap[parent - 1];
+	    heap[parent - 1] = heap[curr - 1];
+	    heap[curr - 1] = temp;
+	    curr = parent;
+	    parent = curr >> 1;
+	  }
+      }
+    }
+
+
+  /* Calculate the code lens */
+  heap[0] = -1UL;
+  heap[1] = 0;
+  for (i = 2; i < num * 2; i++)
+    heap[i] = heap[heap[i]] + 1;
+
+  /* zero the lencount's array */
+  bzero ((char *) hd->lencount, sizeof (hd->lencount));
+  hd->maxcodelen = 0;
+  hd->mincodelen = 100;
+
+  /* Set the code length of each leaf in the huffman tree */
+  for (i = 0; i < num; i++)
+    {
+      register u_long codelen = heap[i + num];
+      hd->clens[i] = (char) codelen;
+      if (!codelen)
+	continue;
+      if (codelen > hd->maxcodelen)
+	hd->maxcodelen = codelen;
+      if (codelen < hd->mincodelen)
+	hd->mincodelen = codelen;
+      hd->lencount[codelen]++;
+    }
+
+
+  if (hd->maxcodelen == 0 && hd->mincodelen == 100)
+    {
+      hd->num_codes = 0;
+    }
+  else
+    {
+      /* Calculate the current codes for each different code length */
+      hd->min_code[hd->maxcodelen] = 0;
+      for (i = hd->maxcodelen - 1; i>=0; i--)
+	hd->min_code[i] = (hd->min_code[i + 1] + hd->lencount[i + 1]) >> 1;
+    }
+  free (heap);
+  return (hd);
+
+error2:
+  free (heap);
+error1:
+  if (!data)
+    free (hd);
+error0:
+  return (NULL);
+
+}
+
+unsigned long *
+Generate_Huffman_Codes (huff_data * data, u_long * mem)
+{
+  int i;
+  unsigned long *codes;
+  unsigned long mc[MAX_HUFFCODE_LEN + 1];
+
+  if (!data)
+    return (NULL);
+  if (!(codes = Xmalloc (data->num_codes * sizeof (*codes))))
+    return (NULL);
+  if (mem)
+    *mem += data->num_codes * sizeof (*codes);
+  bcopy ((char *) data->min_code, (char *) mc, sizeof (mc));
+  for (i = 0; i < data->num_codes; i++)
+    if (data->clens[i])
+      codes[i] = mc[(int) (data->clens[i])]++;
+
+  return (codes);
+}
+
+
+unsigned long **
+Generate_Huffman_Vals (huff_data * data, u_long * mem)
+{
+  int i;
+  unsigned long *fcode[MAX_HUFFCODE_LEN + 1];
+  unsigned long **values;
+  unsigned long *vals;
+
+  if (!data)
+    return (NULL);
+  if (!(vals = Xmalloc (data->num_codes * sizeof (*vals))))
+    return (NULL);
+  if (!(values = Xmalloc ((MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *))))
+    {
+      free (vals);
+      return (NULL);
+    }
+
+  bzero ((char *) values, (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *));
+
+  if (mem)
+    *mem += data->num_codes * sizeof (*vals) +
+      (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *);
+
+  fcode[0] = values[0] = &vals[0];
+  for (i = 1; i <= data->maxcodelen; i++)
+    fcode[i] = values[i] = &vals[(values[i - 1] - vals) + data->lencount[i - 1]];
+
+  for (i = 0; i < data->num_codes; i++)
+    if (data->clens[i])
+      *fcode[(int) (data->clens[i])]++ = i;
+  return (values);
+}
+
+
+
+double 
+Calculate_Huffman_Size (int num, long *freqs, long *counts)
+{
+  double size = 0;
+  huff_data hd;
+  int i;
+  if (!Generate_Huffman_Data (num, freqs, &hd, NULL))
+    return -1;
+  for (i = 0; i < num; i++)
+    size += counts[i] * hd.clens[i];
+  Xfree (hd.clens);
+  return size;
+}
+
+
+
+int 
+Write_Huffman_Data (FILE * f, huff_data * hd)
+{
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONSI(hd->num_codes);
+  HTONSI(hd->mincodelen);
+  HTONSI(hd->maxcodelen);
+
+  if (fwrite (&hd->num_codes, sizeof (hd->num_codes), 1, f) != 1)
+    return -1;
+
+  if (fwrite (&hd->mincodelen, sizeof (hd->mincodelen), 1, f) != 1)
+    return -1;
+
+  if (fwrite (&hd->maxcodelen, sizeof (hd->maxcodelen), 1, f) != 1)
+    return -1;
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHSI(hd->num_codes);
+  NTOHSI(hd->mincodelen);
+  NTOHSI(hd->maxcodelen);
+
+  if (hd->num_codes)
+    {
+      /* [RPAP - Jan 97: Endian Ordering] */
+      int i;
+      for (i = hd->mincodelen; i < hd->maxcodelen + 1; i++)
+	HTONSI(hd->lencount[i]);
+      for (i = 0; i < hd->maxcodelen + 1; i++)
+	HTONUL(hd->min_code[i]);
+
+      if (fwrite (hd->lencount + hd->mincodelen, sizeof (*hd->lencount),
+		  hd->maxcodelen - hd->mincodelen + 1, f) !=
+	  hd->maxcodelen - hd->mincodelen + 1)
+	return -1;
+
+      if (fwrite (hd->min_code, sizeof (*hd->min_code), hd->maxcodelen + 1, f) !=
+	  hd->maxcodelen + 1)
+	return -1;
+
+      if (fwrite (hd->clens, sizeof (*hd->clens), hd->num_codes, f) != hd->num_codes)
+	return -1;
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (i = hd->mincodelen; i < hd->maxcodelen + 1; i++)
+	NTOHSI(hd->lencount[i]);
+      for (i = 0; i < hd->maxcodelen + 1; i++)
+	NTOHUL(hd->min_code[i]);
+    }
+  return 0;
+}
+
+
+
+
+static int 
+General_Read_Huffman_Data (size_t (*rd) (), void *f,
+			   huff_data * hd, u_long * mem, u_long * disk)
+{
+  if (rd (&hd->num_codes, sizeof (hd->num_codes), 1, f) != 1)
+    return -1;
+
+  if (rd (&hd->mincodelen, sizeof (hd->mincodelen), 1, f) != 1)
+    return -1;
+
+  if (rd (&hd->maxcodelen, sizeof (hd->maxcodelen), 1, f) != 1)
+    return -1;
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHSI(hd->num_codes);
+  NTOHSI(hd->mincodelen);
+  NTOHSI(hd->maxcodelen);
+
+  if (disk)
+    (*disk) += sizeof (hd->num_codes) + sizeof (hd->mincodelen) +
+      sizeof (hd->maxcodelen);
+
+  hd->clens = NULL;
+
+  if (hd->num_codes)
+    {
+      int i;  /* [RPAP - Jan 97: Endian Ordering] */
+
+      bzero ((char *) (hd->lencount), sizeof (hd->lencount));
+      if (rd (hd->lencount + hd->mincodelen, sizeof (*hd->lencount),
+	      hd->maxcodelen - hd->mincodelen + 1, f) !=
+	  hd->maxcodelen - hd->mincodelen + 1)
+	return -1;
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (i = hd->mincodelen; i < hd->maxcodelen + 1; i++)
+	NTOHSI(hd->lencount[i]);
+
+      if (disk)
+	(*disk) += sizeof (*hd->lencount) * (hd->maxcodelen - hd->mincodelen + 1);
+
+      bzero ((char *) (hd->min_code), sizeof (hd->min_code));
+
+      if (rd (hd->min_code, sizeof (*hd->min_code), hd->maxcodelen + 1, f) !=
+	  hd->maxcodelen + 1)
+	return -1;
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (i = 0; i < hd->maxcodelen + 1; i++)
+	NTOHUL(hd->min_code[i]);
+
+      if (disk)
+	(*disk) += sizeof (*hd->min_code) * (hd->maxcodelen + 1);
+
+      if (!(hd->clens = Xmalloc (sizeof (*hd->clens) * hd->num_codes)))
+	return -1;
+
+
+      if (rd (hd->clens, sizeof (*hd->clens), hd->num_codes, f) != hd->num_codes)
+	return -1;
+
+      if (disk)
+	(*disk) += sizeof (*hd->clens) * hd->num_codes;
+    }
+
+  if (mem)
+    *mem += sizeof (*hd->clens) * hd->num_codes;
+  return 0;
+}
+
+
+
+int 
+Read_Huffman_Data (FILE * f, huff_data * hd, u_long * mem, u_long * disk)
+{
+  return General_Read_Huffman_Data (fread, f, hd, mem, disk);
+}
+
+
+
+
+int 
+F_Read_Huffman_Data (File * f, huff_data * hd, u_long * mem, u_long * disk)
+{
+  return General_Read_Huffman_Data (Fread, f, hd, mem, disk);
+}
Index: /tags/gsdl-21/gsdl/packages/mg/lib/huffman.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/huffman.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/huffman.h	(revision 699)
@@ -0,0 +1,114 @@
+/**************************************************************************
+ *
+ * huffman.h -- Huffman coding functions
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef H_HUFFMAN
+#define H_HUFFMAN
+
+#include "sysfuncs.h"
+
+#include "filestats.h"
+
+
+#define MAX_HUFFCODE_LEN 31
+
+typedef struct huff_data
+  {
+    int num_codes;
+    int mincodelen;
+    int maxcodelen;
+    int lencount[MAX_HUFFCODE_LEN + 1];
+    unsigned long min_code[MAX_HUFFCODE_LEN + 1];
+    char *clens;
+  }
+huff_data;
+
+
+huff_data *Generate_Huffman_Data (int num, long *freqs, huff_data * data,
+				  u_long * mem);
+
+unsigned long *Generate_Huffman_Codes (huff_data * data, u_long * mem);
+
+unsigned long **Generate_Huffman_Vals (huff_data * data, u_long * mem);
+
+int Write_Huffman_Data (FILE * f, huff_data * hd);
+
+int Read_Huffman_Data (FILE * f, huff_data * hd, u_long * mem, u_long * disk);
+
+int F_Read_Huffman_Data (File * f, huff_data * hd, u_long * mem, u_long * disk);
+
+/* Calculate the number of bits required to code the data with the
+   specified frequencies. Normally freqs and counts should point to
+   the same array. */
+double Calculate_Huffman_Size (int num, long *freqs, long *counts);
+
+
+
+#define HUFF_ENCODE(x, codes, lens)					\
+  do {									\
+    register int __i;							\
+    register int __clen = (lens)[x];					\
+    register unsigned long __code = (codes)[x];				\
+    for (__i=__clen-1; __i>=0; __i--)					\
+      ENCODE_BIT((__code >> __i) & 1);					\
+  } while(0)
+
+#define HUFF_ENCODE_L(x, codes, lens, count)				\
+  do {									\
+    HUFF_ENCODE(x, codes, lens);		   			\
+    (count) += (lens)[x];						\
+  } while(0)
+
+
+#define HUFF_DECODE(x, mcodes, values)					\
+  do {									\
+    register unsigned long *__min_code = (mcodes);			\
+    register unsigned long *__mclen = __min_code;			\
+    register unsigned long **__values = (values);			\
+    register unsigned long __code = 0;					\
+    do									\
+      {									\
+        DECODE_ADD(__code);						\
+      }									\
+    while (__code < *++__mclen);					\
+    (x) = __values[__mclen - __min_code][__code - *__mclen];		\
+  } while(0);
+
+
+#define HUFF_DECODE_L(x, mcodes, values, count)				\
+  do {									\
+    register unsigned long *__min_code = (mcodes);			\
+    register unsigned long *__mclen = __min_code;			\
+    register unsigned long **__values = (values);			\
+    register unsigned long __code = 0;					\
+    do									\
+      {									\
+        DECODE_ADD(__code);						\
+	(count)++;							\
+      }									\
+    while (__code < *++__mclen);					\
+    (x) = __values[__mclen - __min_code][__code - *__mclen];		\
+  } while(0);
+
+
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/lib/huffman_mem.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/huffman_mem.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/huffman_mem.c	(revision 699)
@@ -0,0 +1,66 @@
+/**************************************************************************
+ *
+ * huffman_mem.c -- Huffman coding functions to memory
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:16:51  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:32:05  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/08/22  00:24:45  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+#include "bitio_m.h"
+#include "bitio_m_mem.h"
+#include "huffman.h"
+
+#if 0
+int fprintf (FILE *, const char *,...);
+#endif
+
+void 
+BIO_Mem_Huff_Encode (unsigned long val, unsigned long *codes,
+		     char *clens, mem_bitio_state * bs)
+{
+  ENCODE_CONTINUE (*bs)
+    HUFF_ENCODE (val, codes, clens);
+  ENCODE_PAUSE (*bs)
+}
+
+unsigned long 
+BIO_Mem_Huff_Decode (unsigned long *mincodes,
+		     unsigned long **values, mem_bitio_state * bs)
+{
+  unsigned long val;
+  DECODE_CONTINUE (*bs)
+    HUFF_DECODE (val, mincodes, values);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
Index: /tags/gsdl-21/gsdl/packages/mg/lib/huffman_mem.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/huffman_mem.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/huffman_mem.h	(revision 699)
@@ -0,0 +1,36 @@
+/**************************************************************************
+ *
+ * huffman_mem.h -- Huffman coding functions to memory
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef H_HUFFMAN_MEM
+#define H_HUFFMAN_MEM
+
+
+
+void BIO_Mem_Huff_Encode (unsigned long val, unsigned long *codes,
+			  char *clens, mem_bitio_state * bs);
+
+unsigned long BIO_Mem_Huff_Decode (unsigned long *mincodes,
+			      unsigned long **values, mem_bitio_state * bs);
+
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/lib/huffman_stdio.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/huffman_stdio.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/huffman_stdio.c	(revision 699)
@@ -0,0 +1,70 @@
+/**************************************************************************
+ *
+ * huffman_stdio.c -- Huffman coding function to a file
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:16:52  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:32:07  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/08/22  00:24:46  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+#include "bitio_m.h"
+#include "bitio_m_stdio.h"
+#include "huffman.h"
+
+#if 0
+int fprintf (FILE *, const char *,...);
+int _filbuf (FILE *);
+int _flsbuf (unsigned, FILE *);
+#endif
+
+void 
+BIO_Stdio_Huff_Encode (unsigned long val, unsigned long *codes,
+		       char *clens, stdio_bitio_state * bs)
+{
+  ENCODE_CONTINUE (*bs)
+    HUFF_ENCODE (val, codes, clens);
+  ENCODE_PAUSE (*bs)
+}
+
+unsigned long 
+BIO_Stdio_Huff_Decode (unsigned long *mincodes,
+		       unsigned long **values,
+		       stdio_bitio_state * bs)
+{
+  unsigned long val;
+  DECODE_CONTINUE (*bs)
+    HUFF_DECODE (val, mincodes, values);
+  DECODE_PAUSE (*bs)
+    return (val);
+}
Index: /tags/gsdl-21/gsdl/packages/mg/lib/huffman_stdio.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/huffman_stdio.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/huffman_stdio.h	(revision 699)
@@ -0,0 +1,38 @@
+/**************************************************************************
+ *
+ * huffman_stdio.h -- Huffman coding function to a file
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef H_HUFFMAN_STDIO
+#define H_HUFFMAN_STDIO
+
+
+
+void BIO_Stdio_Huff_Encode (unsigned long val, unsigned long *codes,
+			    char *clens, stdio_bitio_state * bs);
+
+
+unsigned long BIO_Stdio_Huff_Decode (unsigned long *mincodes,
+				     unsigned long **values,
+				     stdio_bitio_state * bs);
+
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/lib/local_strings.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/local_strings.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/local_strings.c	(revision 699)
@@ -0,0 +1,415 @@
+/**************************************************************************
+ *
+ * local_strings -- provides some extra string routines
+ * Copyright (C) 1994  Authors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+#include "local_strings.h"
+#include "memlib.h"
+
+/* =========================================================================
+ * Function: arg_atoi
+ * Description: 
+ *      Converts an ascii string to an integer
+ *      but allows a scale string immediately following.
+ *      i.e. "10Mb" or "20Kb" or "40Megabytes"
+ *      There must be no gap between the last digit and the scale.
+ *      Only the fist letter is significant.
+ *      K = Kilo
+ *      M = Mega
+ *      G = Giga
+ * Input: 
+ *      string of form: xxxxxxK or xxxxxxxM or xxxxxxxG
+ *      where x is a digit
+ * Output: 
+ *      a scaled integer
+ * ========================================================================= */
+
+#define KILO 	1024
+#define MEGA	1024*1024
+#define GIGA	1024*1024*1024
+
+int 
+arg_atoi (char *str)
+{
+  char *str_ptr = str;
+  char ch = '\0';
+  long scale = 1;
+
+  /* find any scale letter immediately after digits */
+  ch = *str_ptr;
+  while (!isalpha (ch) && (ch != '\0'))
+    ch = *str_ptr++;
+
+  /* replace scale letter by end-of-string character */
+  *str_ptr = '\0';
+
+  /* set scale appropriately */
+  switch (ch)
+    {
+    case 'K':
+      scale = KILO;
+      break;
+    case 'M':
+      scale = MEGA;
+      break;
+    case 'G':
+      scale = GIGA;
+      break;
+    default:
+      scale = 1;
+      break;
+    }
+
+  return (atoi (str) * scale);
+
+}
+
+/* ===============================================================================
+ * Function: compare
+ * Description: Compare two strings. 
+ * Input: Two binary strings, first byte of string indicates length. 
+ * Output: Like standard strcmp.
+ *      = 0 if s1 = s2
+ *      < 0 if s1 < s2
+ *      > 0 if s1 > s2
+ * =============================================================================== */
+int 
+compare (u_char * s1, u_char * s2)
+{
+  int l1 = *s1, l2 = *s2;
+  int l = (l1 < l2) ? l1 : l2;
+
+  while (l--)
+    if (*++s1 != *++s2)
+      return (*s1 - *s2);
+  return (l1 - l2);
+
+}
+
+/*
+ * This array is designed for mapping upper and lower case letter
+ * together for a case independent comparison.  The mappings are
+ * based upon ascii character sequences.
+ */
+static unsigned char casecharmap[] = {
+	'\000', '\001', '\002', '\003', '\004', '\005', '\006', '\007',
+	'\010', '\011', '\012', '\013', '\014', '\015', '\016', '\017',
+	'\020', '\021', '\022', '\023', '\024', '\025', '\026', '\027',
+	'\030', '\031', '\032', '\033', '\034', '\035', '\036', '\037',
+	'\040', '\041', '\042', '\043', '\044', '\045', '\046', '\047',
+	'\050', '\051', '\052', '\053', '\054', '\055', '\056', '\057',
+	'\060', '\061', '\062', '\063', '\064', '\065', '\066', '\067',
+	'\070', '\071', '\072', '\073', '\074', '\075', '\076', '\077',
+	'\100', '\141', '\142', '\143', '\144', '\145', '\146', '\147',
+	'\150', '\151', '\152', '\153', '\154', '\155', '\156', '\157',
+	'\160', '\161', '\162', '\163', '\164', '\165', '\166', '\167',
+	'\170', '\171', '\172', '\133', '\134', '\135', '\136', '\137',
+	'\140', '\141', '\142', '\143', '\144', '\145', '\146', '\147',
+	'\150', '\151', '\152', '\153', '\154', '\155', '\156', '\157',
+	'\160', '\161', '\162', '\163', '\164', '\165', '\166', '\167',
+	'\170', '\171', '\172', '\173', '\174', '\175', '\176', '\177',
+	'\200', '\201', '\202', '\203', '\204', '\205', '\206', '\207',
+	'\210', '\211', '\212', '\213', '\214', '\215', '\216', '\217',
+	'\220', '\221', '\222', '\223', '\224', '\225', '\226', '\227',
+	'\230', '\231', '\232', '\233', '\234', '\235', '\236', '\237',
+	'\240', '\241', '\242', '\243', '\244', '\245', '\246', '\247',
+	'\250', '\251', '\252', '\253', '\254', '\255', '\256', '\257',
+	'\260', '\261', '\262', '\263', '\264', '\265', '\266', '\267',
+	'\270', '\271', '\272', '\273', '\274', '\275', '\276', '\277',
+	'\300', '\341', '\342', '\343', '\344', '\345', '\346', '\347',
+	'\350', '\351', '\352', '\353', '\354', '\355', '\356', '\357',
+	'\360', '\361', '\362', '\363', '\364', '\365', '\366', '\367',
+	'\370', '\371', '\372', '\333', '\334', '\335', '\336', '\337',
+	'\340', '\341', '\342', '\343', '\344', '\345', '\346', '\347',
+	'\350', '\351', '\352', '\353', '\354', '\355', '\356', '\357',
+	'\360', '\361', '\362', '\363', '\364', '\365', '\366', '\367',
+	'\370', '\371', '\372', '\373', '\374', '\375', '\376', '\377',
+};
+
+/* ===============================================================================
+ * Function: casecompare   [RPAP - Jan 97: Stem Index Change]
+ * Description: Compare two strings in dictionary order
+ * Input: Two binary strings, first byte of string indicates length. 
+ * Output:
+ *      = 0 if s1 = s2
+ *      < 0 if s1 < s2
+ *      > 0 if s1 > s2
+ * =============================================================================== */
+int 
+casecompare (u_char * s1, u_char * s2)
+{
+  int l1 = *s1, l2 = *s2;
+  int l = (l1 < l2) ? l1 : l2;
+  int pos = 0;
+  register int diff = 0;
+
+  while (l--)
+    if ((diff = casecharmap[*++s1] - casecharmap[*++s2]) != 0)
+      return (diff);
+    else if ((diff = *s1 - *s2) != 0)
+      if (!pos)
+	pos = diff;
+
+  return ((l1 - l2) ? (l1 - l2) : (pos));
+
+}
+
+/* =========================================================================
+ * Function: copy_string
+ * Description: 
+ *      Copies a string with the len in 1st byte
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+u_char *
+copy_string (u_char * src_str)
+{
+  u_char *copy = Xmalloc (src_str[0] + 1);
+
+  if (!copy)
+    return NULL;
+
+  bcopy ((char *) src_str, (char *) (copy), src_str[0] + 1);
+  return copy;
+
+}
+
+
+/* ===============================================================================
+ * Function: prefixlen
+ * Description: Returns length of common prefix string between s1 and s2.
+ * Input: two binary strings
+ * Output: prefix length
+ * =============================================================================== */
+int 
+prefixlen (u_char * s1, u_char * s2)
+{
+  int l = (*s1 < *s2) ? *s1 : *s2;
+  int i = 0;
+
+  while (i < l && *++s1 == *++s2)
+    i++;
+  return (i);
+}
+
+
+/* ===============================================================================
+ * Function: char2str
+ * Description: 
+ *      Ensures that non-printable characters are converted to a string
+ *      which is printable i.e. it indentifies the character.
+ *      e.g. carriage-return = "\n".
+ * Input: character (may be non-printable)
+ * Output: pointer to static string
+ * =============================================================================== */
+
+char *
+char2str (u_char c)
+{
+  static char buf[10];
+  switch (c)
+    {
+    case '\n':
+      sprintf (buf, "\\n");
+      break;
+    case '\b':
+      sprintf (buf, "\\b");
+      break;
+    case '\f':
+      sprintf (buf, "\\f");
+      break;
+    case '\t':
+      sprintf (buf, "\\t");
+      break;
+    default:
+	if (c >= ' ' && c <= 126)
+	switch (c)
+	  {
+	  case '\\':
+	    sprintf (buf, "\\\\");
+	    break;
+	  case '\"':
+	    sprintf (buf, "\\\"");
+	    break;
+	  case '\'':
+	    sprintf (buf, "\\\'");
+	    break;
+	  default:
+	    buf[0] = c;
+	    buf[1] = '\0';
+	    break;
+	  }
+      else
+	sprintf (buf, "\\x%02x", c);
+      break;
+    }
+  return buf;
+}
+
+/* ===============================================================================
+ * Function: word2str
+ * Description: 
+ *      Converts a string with possible non-printable characters into a string
+ *      consisting entirely of printable characters.
+ * Input: string, length stored in first byte
+ * Output: pointer to static string
+ * =============================================================================== */
+
+char *
+word2str (u_char * s)
+{
+  static char buf[1024];
+  char *p = buf;
+  int i, len = (int) *s++;
+
+  bzero (buf, sizeof (buf));
+  for (i = 0; i < len; i++)
+    {
+      strcpy (p, char2str (s[i]));
+      p = strchr (p, '\0');
+    }
+  return buf;
+}
+
+
+/* =========================================================================
+ * Function: de_escape_string
+ * Description: 
+ *      Turn a string containing \ escapes into real characters
+ *      Note this destroys the string s in the process
+ * Input: string to de-escape 
+ * Output: the de-escaped string
+ * ========================================================================= */
+char *
+de_escape_string (char *s)
+{
+  char *org = s;
+  char *d = s;
+  while (*s)
+    {
+      if (*s == '\\')
+	{
+	  s++;
+	  switch (*s++)
+	    {
+	    case '\0':
+	      *d++ = '\\';
+	      s--;
+	      break;
+	    case '\\':
+	      *d++ = '\\';
+	      break;
+	    case 'b':
+	      *d++ = '\b';
+	      break;
+	    case 'f':
+	      *d++ = '\f';
+	      break;
+	    case 'n':
+	      *d++ = '\n';
+	      break;
+	    case 'r':
+	      *d++ = '\r';
+	      break;
+	    case 't':
+	      *d++ = '\r';
+	      break;
+	    case '\"':
+	      *d++ = '\"';
+	      break;
+	    case '\'':
+	      *d++ = '\'';
+	      break;
+	    case 'x':
+	      {
+		int num = 0;
+		if (isxdigit (*s))
+		  {
+		    char ch = toupper (*s++);
+		    num = ch <= '9' ? ch - '0' : ch - 'A' + 10;
+		  }
+		if (isxdigit (*s))
+		  {
+		    char ch = toupper (*s++);
+		    num = (num << 4) + (ch <= '9' ? ch - '0' : ch - 'A' + 10);
+		  }
+		*d++ = num;
+		break;
+	      }
+	    case '0':
+	    case '1':
+	    case '2':
+	    case '3':
+	    case '4':
+	    case '5':
+	    case '6':
+	    case '7':
+	      {
+		int num = *(s - 1) - '0';
+		if (*s >= '0' && *s <= '7')
+		  num = (num << 3) + *s++ - '0';
+		if (*s >= '0' && *s <= '7')
+		  num = (num << 3) + *s++ - '0';
+		*d++ = num;
+		break;
+	      }
+	    }
+	}
+      else
+	*d++ = *s++;
+    }
+  *d = '\0';
+  return (org);
+}
+
+/* =========================================================================
+ * Function: str255_to_string 
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+char *
+str255_to_string (u_char * str255, char *str)
+{
+  static char buf[256];
+  char *str_base = NULL;
+  int len = *str255++;
+  int i = 0;
+
+  /* if haven't been given a str to write to then use local buffer */
+  if (!str)
+    str = &(buf[0]);
+
+  str_base = str;
+  for (i = 0; i < len; i++)
+    {
+      *str++ = *str255++;
+    }
+  *str = '\0';
+
+  return str_base;
+}
Index: /tags/gsdl-21/gsdl/packages/mg/lib/local_strings.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/local_strings.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/local_strings.h	(revision 699)
@@ -0,0 +1,39 @@
+/**************************************************************************
+ *
+ * local_strings.h -- string routines
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef LOCAL_STRINGS_H
+#define LOCAL_STRINGS_H
+
+#include "sysfuncs.h"
+
+int arg_atoi (char *str);
+int compare (u_char * s1, u_char * s2);
+int casecompare (u_char * s1, u_char * s2);   /* [RPAP - Jan 97: Stem Index Change] */
+u_char *copy_string (u_char * src_str);
+char *str255_to_string (u_char * str255, char *str);
+int prefixlen (u_char * s1, u_char * s2);
+char *char2str (u_char c);
+char *word2str (u_char * s);
+char *de_escape_string (char *s);
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/lib/lovinstem.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/lovinstem.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/lovinstem.c	(revision 699)
@@ -0,0 +1,1187 @@
+/**************************************************************************
+ *
+ * lovinstem.c -- Stemming code
+ * Copyright (C) 1994  Linh Huynh (linh@kbs.citri.edu.au)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+
+/*****************************************************************************
+ *
+ *   Transformational Rules used in Recoding Stem Terminations
+ *
+ *   Parameter in functions:
+ *       ch : the second last character of the old ending
+ *            (after removing one of double consonants) 
+ *
+ ****************************************************************************/
+
+
+/************ Conditional rules associated with transformations **************/
+
+static int aio ();
+static int s ();
+static int pt ();
+static int m ();
+static int n ();
+
+static int 
+aio (ch)			/* Rule number 9     */
+     char ch;
+{
+  return ((ch != 'a') && (ch != 'i') && (ch != 'o'));
+}
+
+static int 
+s (ch)				/* Rule Number 24    */
+     char ch;
+{
+  return (ch != 's');
+}
+
+static int 
+pt (ch)				/* Rule number 28    */
+     char ch;
+{
+  return ((ch != 'p') && (ch != 't'));
+}
+
+static int 
+m (ch)				/* Rule number 30    */
+     char ch;
+{
+  return (ch != 'm');
+}
+
+static int 
+n (ch)				/* Rule number 32    */
+     char ch;
+{
+  return (ch != 'n');
+}
+
+
+/**************************************************************************
+ *
+ *  Context-sensitive rules associated with certain endings.
+ *
+ *  The notations of the rules are the same as in Lovins' paper
+ *  except that rule A is replaced by a NULL in the data structure.
+ *
+ *  Function parameters:
+ *      - stem_length: possible stem length
+ *      - end        : pointer points to the end of the possible stem.
+ *
+ *************************************************************************/
+
+
+/********* Context-sensitive rule function declarations ******************/
+
+static int B ();
+static int C ();
+static int D ();
+static int E ();
+static int F ();
+static int G ();
+static int H ();
+static int I ();
+static int J ();
+static int K ();
+static int L ();
+static int M ();
+static int N ();
+static int O ();
+static int P ();
+static int Q ();
+static int R ();
+static int S ();
+static int T ();
+static int U ();
+static int V ();
+static int W ();
+static int X ();
+static int Y ();
+static int Z ();
+static int AA ();
+static int BB ();
+static int CC ();
+
+static int 
+B (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  return (stem_length >= 3);
+}
+
+static int 
+C (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  return (stem_length >= 4);
+}
+
+static int 
+D (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  return (stem_length >= 5);
+}
+
+static int 
+E (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  return (*end != 'e');
+}
+
+static int 
+F (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  return ((stem_length >= 3) && (*end != 'e'));
+}
+
+static int 
+G (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  return ((stem_length >= 3) && (*end == 'f'));
+}
+
+static int 
+H (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1, p2;
+  p1 = *end;
+  p2 = *(end - 1);
+  return ((p1 == 't') || ((p1 == 'l') && (p2 == 'l')));
+}
+
+static int 
+I (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1;
+  p1 = *end;
+  return ((p1 != 'o') && (p1 != 'e'));
+}
+
+static int 
+J (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1;
+  p1 = *end;
+  return ((p1 != 'a') && (p1 != 'e'));
+}
+
+static int 
+K (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1;
+  p1 = *end;
+  return ((stem_length >= 3) &&
+	  ((p1 == 'l') || (p1 == 'i') ||
+	   ((p1 == 'e') && (*(end - 2) == 'u'))));
+}
+
+static int 
+L (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1, p2;
+  p1 = *end;
+  p2 = *(end - 1);
+  if (p1 == 's')
+    return (p2 == 'o');
+  else
+    return ((p1 != 'u') && (p1 != 'x'));
+}
+
+static int 
+M (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1;
+  p1 = *end;
+  return ((p1 != 'a') && (p1 != 'c') &&
+	  (p1 != 'e') && (p1 != 'm'));
+}
+
+static int 
+N (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  if (stem_length >= 3)
+    {
+      if (*(end - 2) == 's')
+	return (stem_length >= 4);
+      else
+	return 1;
+    }
+  else
+    return 0;
+}
+
+static int 
+O (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1;
+  p1 = *end;
+  return ((p1 == 'l') || (p1 == 'i'));
+}
+
+static int 
+P (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  return (*end != 'c');
+}
+
+static int 
+Q (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1;
+  p1 = *end;
+
+  return ((stem_length >= 3) && (p1 != 'l') && (p1 != 'n'));
+}
+
+static int 
+R (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1;
+  p1 = *end;
+  return ((p1 == 'n') || (p1 == 'r'));
+}
+
+static int 
+S (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1, p2;
+  p1 = *end;
+  p2 = *(end - 1);
+
+  if (p1 == 't')
+    return (p2 != 't');
+  else
+    return ((p1 == 'r') && (p2 == 'd'));
+}
+
+static int 
+T (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1, p2;
+  p1 = *end;
+  p2 = *(end - 1);
+
+  if (p1 == 't')
+    return (p2 != 'o');
+  else
+    return (p1 == 's');
+}
+
+static int 
+U (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1;
+  p1 = *end;
+
+  return ((p1 == 'l') || (p1 == 'm') ||
+	  (p1 == 'n') || (p1 == 'r'));
+}
+
+static int 
+V (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  return (*end == 'c');
+}
+
+static int 
+W (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1;
+  p1 = *end;
+
+  return ((p1 != 's') && (p1 != 'u'));
+}
+
+static int 
+X (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1;
+  p1 = *end;
+
+  if (p1 == 'e')
+    return ((stem_length >= 3) && (*(end - 2) == 'u'));
+  else
+    return ((p1 == 'l') || (p1 == 'i'));
+}
+
+static int 
+Y (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  return ((*end == 'n') && (*(end - 1) == 'i'));
+}
+
+static int 
+Z (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  return (*end != 'f');
+}
+
+static int 
+AA (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  char p1, p2;
+  p1 = *end;
+  p2 = *(end - 1);
+
+  if (p1 == 'h')
+    return ((p2 == 'p') || (p2 == 't'));
+  else if (p1 == 'r')
+    return ((p2 == 'e') || (p2 == 'o'));
+  else if (p1 == 's')
+    return (p2 == 'e');
+  else
+    return ((p1 == 'd') || (p1 == 'f') ||
+	    (p1 == 'l') || (p1 == 't'));
+}
+
+static int 
+BB (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  if (stem_length >= 3)
+    {
+      if (stem_length >= 4)
+	return (strncmp (end - 3, "ryst", 4) != 0);
+      else
+	return (strncmp (end - 2, "met", 3) != 0);
+    }
+  else
+    return 0;
+}
+
+static int 
+CC (stem_length, end)
+     int stem_length;
+     char *end;
+{
+  return (*end == 'l');
+}
+
+/**************************************************************************/
+
+#define MIN_STEM_LENGTH 2
+#define MAX_ENDING_SIZE 11
+#define PREDEFINED_SIZE (MAX_ENDING_SIZE + MIN_STEM_LENGTH)
+#define EOS     '\0'
+/* [RPAP - Jan 97: Stem Index Change] */
+#define MAX_STEM_LENGTH 255
+
+static char *remove_ending (char *word, int w_len);
+static void recode_stem (char *stem_end);
+
+/* =========================================================================
+ * Function: stem
+ * Description: 
+ *      Implemetation of the Lovins' stemming algorithm described in
+ *      J.B. Lovins, "Development of a Stemming Algorithm",
+ *      Mechanical Translation and Computational Linguistics, Vol 11,1968.
+ * Input: a word string with the length in the first byte
+ * Output: the stemmed word
+ * ========================================================================= */
+
+void
+lovinstem (u_char * word)
+{
+  char *word_start;		/* points to first letter of word */
+  char *stem_end;		/* points to last character of stem portion */
+  int old_length;		/* length of word */
+  int new_length;		/* length after stemming */
+
+  /* [RPAP - Jan 97: Stem Index Change] */
+  u_char copy[MAX_STEM_LENGTH + 2];
+  bcopy ((char *) word, (char *) copy, *word + 1);
+
+  old_length = *copy; /* [RPAP - Jan 97: Stem Index Change] */
+
+  /* The word must be at least MIN_STEM_LENGTH characters long. */
+  if (old_length <= MIN_STEM_LENGTH)
+    return;
+
+  word_start = (char *) copy + 1;	/* jump over length byte */ /* [RPAP - Jan 97: Stem Index Change] */
+  copy[old_length + 1] = '\0';	/* null terminate string */ /* [RPAP - Jan 97: Stem Index Change] */
+
+  stem_end = remove_ending (word_start, old_length);
+  recode_stem (stem_end);
+
+  /* fix up new length */
+  /* have to use strlen because have no other way of telling length */
+  new_length = strlen (word_start);
+  *copy = new_length;  /* [RPAP - Jan 97: Stem Index Change] */
+
+  /* [RPAP - Jan 97: Stem Index Change] */
+  bcopy ((char *) copy, (char *) word, *copy + 1);
+
+}				/*stem_complex */
+
+/* =========================================================================
+
+ *   Data structure for recoding and the method for recoding the stem. 
+ *
+ * ========================================================================= */
+
+typedef struct
+{
+  char *old_end;		/* old ending */
+  char *new_end;		/* new ending */
+  char old_offset;		/* length of the old ending - 1 */
+  int (*cond) ();		/* condition rule */
+  char end_group;		/* signal the end of the group */
+}
+Recode_Rules;
+
+/*  
+   Stem terminations are divided into groups having the same last
+   character
+ */
+
+static Recode_Rules Rules[] =
+{
+  "uad", "uas", 2, NULL, 0,
+  "vad", "vas", 2, NULL, 0,
+  "cid", "cis", 2, NULL, 0,
+  "lid", "lis", 2, NULL, 0,
+  "erid", "eris", 3, NULL, 0,
+  "pand", "pans", 3, NULL, 0,
+  "end", "ens", 2, s, 0,
+  "end", "ens", 2, m, 0,
+  "ond", "ons", 2, NULL, 0,
+  "lud", "lus", 2, NULL, 0,
+  "rud", "rus", 2, NULL, 1,
+
+  "ul", "l", 1, aio, 1,
+
+  "istr", "ister", 3, NULL, 0,
+  "metr", "meter", 3, NULL, 0,
+  "her", "hes", 2, pt, 1,
+
+  "urs", "ur", 2, NULL, 1,
+
+  "uct", "uc", 2, NULL, 0,
+  "umpt", "um", 3, NULL, 0,
+  "rpt", "rb", 2, NULL, 0,
+  "mit", "mis", 2, NULL, 0,
+  "ert", "ers", 2, NULL, 0,
+  "et", "es", 1, n, 0,
+  "yt", "ys", 1, NULL, 1,
+
+  "iev", "ief", 2, NULL, 0,
+  "olv", "olut", 2, NULL, 1,
+
+  "bex", "bic", 2, NULL, 0,
+  "dex", "dic", 2, NULL, 0,
+  "pex", "pic", 2, NULL, 0,
+  "tex", "tic", 2, NULL, 0,
+  "ax", "ac", 1, NULL, 0,
+  "ex", "ec", 1, NULL, 0,
+  "ix", "ic", 1, NULL, 0,
+  "lux", "luc", 2, NULL, 1,
+
+  "yz", "ys", 1, NULL, 1
+};
+
+typedef struct last
+  {
+    char c;			/* The last character */
+    struct last *left;		/* used in balanced   */
+    struct last *right;		/* binary tree        */
+    Recode_Rules *pt;		/* location of approriate group */
+  }
+Last_Char_Node;
+
+/*------------
+       s
+     /   \
+    l     x
+   / \   / \
+  d   r t   z
+         \
+          v
+---------------*/
+static Last_Char_Node pr[] =
+{
+  'd', NULL, NULL, Rules,
+  'l', pr, pr + 2, Rules + 11,
+  'r', NULL, NULL, Rules + 12,
+  's', pr + 1, pr + 6, Rules + 15,
+  't', NULL, pr + 5, Rules + 16,
+  'v', NULL, NULL, Rules + 23,
+  'x', pr + 4, pr + 7, Rules + 25,
+  'z', NULL, NULL, Rules + 33,
+};
+
+
+/*****************************************************************************
+ *
+ *  Recode the stem according to the transformation rules.
+ *
+ *****************************************************************************/
+
+static void
+recode_stem (char *stem_end)
+{
+  static Last_Char_Node *root = pr + 3;
+  Last_Char_Node *p_last = root;
+  Recode_Rules *rule;		/* points to the transformation list */
+  char *h,			/* points to start of possible ending */
+    ch,				/* last character of the old ending  */
+    ch2;
+
+  h = stem_end - 1;
+  ch = *stem_end;
+  if (*h == ch)			/* Check for double consonant        */
+    {
+      if (strchr ("bdglmnprst", ch) != NULL)
+	{
+	  *stem_end = EOS;
+	  stem_end = h;
+	}
+    }
+
+  do				/* Check for the last character       */
+    {
+      ch2 = p_last->c;
+      if (ch == ch2)
+	break;
+      else if (ch > ch2)
+	p_last = p_last->right;
+      else
+	p_last = p_last->left;
+    }
+  while (p_last != NULL);
+
+  if (p_last != NULL)		/* Check for the rest of suffix list */
+    {
+      rule = p_last->pt;
+      for (;;)
+	{
+	  h = stem_end - rule->old_offset;
+	  if (strcmp (h, rule->old_end) == 0)
+	    {
+	      if (!rule->cond || (*rule->cond) (*(h - 1)))
+		{
+		  (void) strcpy (h, rule->new_end);
+		  break;
+		}
+	    }
+	  if (rule->end_group)
+	    break;
+	  rule++;
+	}
+    }
+}
+
+/* ======================================================================
+
+ *    List of endings and the method to remove the ending 
+ *
+ * ======================================================================*/
+
+/************ Data structures for list of endings  ********************/
+
+typedef struct
+{
+  char *ending;			/* old ending                */
+  int (*cond) ();		/* conditional rule          */
+  signed char left_offset;	/* used to find the siblings */
+  signed char right_offset;	/* in balanced binary tree   */
+}
+Ending_List;
+
+/*
+   The original list of endings is re-organised into groups having
+   the same length and the same first character.
+ */
+
+static Ending_List List[] =
+{
+  "a", NULL, 0, 0,
+
+  "ae", NULL, 0, 0,
+  "al", BB, -1, 2,
+  "ar", X, 0, 0,
+  "as", B, -1, 0,
+
+  "acy", NULL, 0, 1,
+  "age", B, 0, 0,
+  "aic", NULL, -2, 1,
+  "als", BB, 0, 0,
+  "ant", B, -2, 2,
+  "ars", O, 0, 0,
+  "ary", F, -1, 2,
+  "ata", NULL, 0, 0,
+  "ate", NULL, -1, 0,
+
+  "able", NULL, 0, 1,
+  "ably", NULL, 0, 0,
+  "ages", B, -2, 2,
+  "ally", B, 0, 0,
+  "ance", B, -1, 1,
+  "ancy", B, 0, 0,
+  "ants", B, -4, 4,
+  "aric", NULL, 0, 0,
+  "arly", K, -1, 1,
+  "ated", I, 0, 0,
+  "ates", NULL, -2, 2,
+  "atic", B, 0, 0,
+  "ator", NULL, -1, 0,
+
+  "acies", NULL, 0, 0,
+  "acity", NULL, -1, 1,
+  "aging", B, 0, 0,
+  "aical", NULL, -2, 2,
+  "alist", NULL, 0, 0,
+  "alism", B, -1, 0,
+  "ality", NULL, -3, 3,
+  "alize", NULL, 0, 1,
+  "allic", BB, 0, 0,
+  "anced", B, -2, 2,
+  "ances", B, 0, 0,
+  "antic", C, -1, 0,
+  "arial", NULL, -6, 6,
+  "aries", NULL, 0, 1,
+  "arily", NULL, 0, 0,
+  "arity", B, -2, 2,
+  "arize", NULL, 0, 0,
+  "aroid", NULL, -1, 0,
+  "ately", NULL, -3, 3,
+  "ating", I, 0, 1,
+  "ation", B, 0, 0,
+  "ative", NULL, -2, 2,
+  "ators", NULL, 0, 0,
+  "atory", NULL, -1, 1,
+  "ature", E, 0, 0,
+
+  "aceous", NULL, 0, 1,
+  "acious", B, 0, 0,
+  "action", G, -2, 2,
+  "alness", NULL, 0, 0,
+  "ancial", NULL, -1, 1,
+  "ancies", NULL, 0, 0,
+  "ancing", B, -4, 4,
+  "ariser", NULL, 0, 0,
+  "arized", NULL, -1, 1,
+  "arizer", NULL, 0, 0,
+  "atable", NULL, -2, 2,
+  "ations", B, 0, 0,
+  "atives", NULL, -1, 0,
+
+  "ability", NULL, 0, 1,
+  "aically", NULL, 0, 0,
+  "alistic", B, -2, 2,
+  "alities", NULL, 0, 0,
+  "ariness", E, -1, 0,
+  "aristic", NULL, -3, 3,
+  "arizing", NULL, 0, 1,
+  "ateness", NULL, 0, 0,
+  "atingly", NULL, -2, 2,
+  "ational", B, 0, 0,
+  "atively", NULL, -1, 1,
+  "ativism", NULL, 0, 0,
+
+  "ableness", NULL, 0, 1,
+  "arizable", NULL, 0, 0,
+
+  "allically", C, 0, 0,
+  "antaneous", NULL, -1, 1,
+  "antiality", NULL, 0, 0,
+  "arisation", NULL, -2, 2,
+  "arization", NULL, 0, 0,
+  "ationally", B, -1, 1,
+  "ativeness", NULL, 0, 0,
+
+  "antialness", NULL, 0, 0,
+  "arisations", NULL, -1, 1,
+  "arizations", NULL, 0, 0,
+
+  "alistically", B, 0, 1,
+  "arizability", NULL, 0, 0,
+
+  "e", NULL, 0, 0,
+
+  "ed", E, 0, 0,
+  "en", F, -1, 1,
+  "es", E, 0, 0,
+
+  "eal", Y, 0, 0,
+  "ear", Y, -1, 1,
+  "ely", E, 0, 0,
+  "ene", E, -2, 2,
+  "ent", C, 0, 0,
+  "ery", E, -1, 1,
+  "ese", NULL, 0, 0,
+
+  "ealy", Y, 0, 1,
+  "edly", E, 0, 0,
+  "eful", NULL, -2, 1,
+  "eity", NULL, 0, 0,
+  "ence", NULL, -2, 2,
+  "ency", NULL, 0, 0,
+  "ened", E, -1, 2,
+  "enly", E, 0, 0,
+  "eous", NULL, -1, 0,
+
+  "early", Y, 0, 1,
+  "ehood", NULL, 0, 0,
+  "eless", NULL, -2, 2,
+  "elity", NULL, 0, 0,
+  "ement", NULL, -1, 0,
+  "enced", NULL, -3, 3,
+  "ences", NULL, 0, 1,
+  "eness", E, 0, 0,
+  "ening", E, -2, 2,
+  "ental", NULL, 0, 0,
+  "ented", C, -1, 1,
+  "ently", NULL, 0, 0,
+
+  "eature", Z, 0, 0,
+  "efully", NULL, -1, 1,
+  "encies", NULL, 0, 0,
+  "encing", NULL, -2, 2,
+  "ential", NULL, 0, 0,
+  "enting", C, -1, 1,
+  "entist", NULL, 0, 1,
+  "eously", NULL, 0, 0,
+
+  "elihood", E, 0, 1,
+  "encible", NULL, 0, 0,
+  "entally", NULL, -2, 2,
+  "entials", NULL, 0, 0,
+  "entiate", NULL, -1, 1,
+  "entness", NULL, 0, 0,
+
+  "entation", NULL, 0, 0,
+  "entially", NULL, -1, 1,
+  "eousness", NULL, 0, 0,
+
+  "eableness", E, 0, 1,
+  "entations", NULL, 0, 0,
+  "entiality", NULL, -2, 2,
+  "entialize", NULL, 0, 0,
+  "entiation", NULL, -1, 0,
+
+  "entialness", NULL, 0, 0,
+
+  "ful", NULL, 0, 0,
+
+  "fully", NULL, 0, 0,
+
+  "fulness", NULL, 0, 0,
+
+  "hood", NULL, 0, 0,
+
+  "i", NULL, 0, 0,
+
+  "ia", NULL, 0, 0,
+  "ic", NULL, -1, 1,
+  "is", NULL, 0, 0,
+
+  "ial", NULL, 0, 0,
+  "ian", NULL, -1, 1,
+  "ics", NULL, 0, 1,
+  "ide", L, 0, 0,
+  "ied", NULL, -3, 3,
+  "ier", NULL, 0, 0,
+  "ies", P, -1, 0,
+  "ily", NULL, -1, 1,
+  "ine", M, 0, 0,
+  "ing", N, -5, 5,
+  "ion", Q, 0, 0,
+  "ish", C, -1, 1,
+  "ism", B, 0, 1,
+  "ist", NULL, 0, 0,
+  "ite", AA, -3, 3,
+  "ity", NULL, 0, 0,
+  "ium", NULL, -1, 0,
+  "ive", NULL, -1, 1,
+  "ize", F, 0, 0,
+
+  "ials", NULL, 0, 0,
+  "ians", NULL, -1, 0,
+  "ible", NULL, -1, 1,
+  "ibly", NULL, 0, 0,
+  "ical", NULL, -2, 2,
+  "ides", L, 0, 0,
+  "iers", NULL, -1, 1,
+  "iful", NULL, 0, 0,
+  "ines", M, -4, 4,
+  "ings", N, 0, 0,
+  "ions", B, -1, 1,
+  "ious", NULL, 0, 0,
+  "isms", B, -2, 2,
+  "ists", NULL, 0, 0,
+  "itic", H, -1, 1,
+  "ized", F, 0, 1,
+  "izer", F, 0, 0,
+
+  "ially", NULL, 0, 0,
+  "icant", NULL, -1, 1,
+  "ician", NULL, 0, 0,
+  "icide", NULL, -2, 2,
+  "icism", NULL, 0, 0,
+  "icist", NULL, -1, 0,
+  "icity", NULL, -3, 3,
+  "idine", I, 0, 1,
+  "iedly", NULL, 0, 0,
+  "ihood", NULL, -2, 2,
+  "inate", NULL, 0, 0,
+  "iness", NULL, -1, 0,
+  "ingly", B, -6, 6,
+  "inism", J, 0, 1,
+  "inity", CC, 0, 0,
+  "ional", NULL, -2, 2,
+  "ioned", NULL, 0, 0,
+  "ished", NULL, -1, 0,
+  "istic", NULL, -3, 3,
+  "ities", NULL, 0, 1,
+  "itous", NULL, 0, 0,
+  "ively", NULL, -2, 2,
+  "ivity", NULL, 0, 0,
+  "izers", F, -1, 1,
+  "izing", F, 0, 0,
+
+  "ialist", NULL, 0, 0,
+  "iality", NULL, -1, 1,
+  "ialize", NULL, 0, 0,
+  "ically", NULL, -2, 2,
+  "icance", NULL, 0, 0,
+  "icians", NULL, -1, 1,
+  "icists", NULL, 0, 0,
+  "ifully", NULL, -4, 4,
+  "ionals", NULL, 0, 0,
+  "ionate", D, -1, 1,
+  "ioning", NULL, 0, 0,
+  "ionist", NULL, -2, 2,
+  "iously", NULL, 0, 0,
+  "istics", NULL, -1, 1,
+  "izable", E, 0, 0,
+
+  "ibility", NULL, 0, 0,
+  "icalism", NULL, -1, 1,
+  "icalist", NULL, 0, 1,
+  "icality", NULL, 0, 0,
+  "icalize", NULL, -3, 3,
+  "ication", G, 0, 0,
+  "icianry", NULL, -1, 0,
+  "ination", NULL, -1, 1,
+  "ingness", NULL, 0, 0,
+  "ionally", NULL, -5, 5,
+  "isation", NULL, 0, 0,
+  "ishness", NULL, -1, 1,
+  "istical", NULL, 0, 1,
+  "iteness", NULL, 0, 0,
+  "iveness", NULL, -3, 3,
+  "ivistic", NULL, 0, 0,
+  "ivities", NULL, -1, 0,
+  "ization", F, -1, 1,
+  "izement", NULL, 0, 0,
+
+  "ibleness", NULL, 0, 0,
+  "icalness", NULL, -1, 1,
+  "ionalism", NULL, 0, 0,
+  "ionality", NULL, -2, 2,
+  "ionalize", NULL, 0, 0,
+  "iousness", NULL, -1, 1,
+  "izations", NULL, 0, 0,
+
+  "ionalness", NULL, 0, 1,
+  "istically", NULL, 0, 0,
+  "itousness", NULL, -2, 2,
+  "izability", NULL, 0, 0,
+  "izational", NULL, -1, 0,
+
+  "izationally", B, 0, 0,
+
+  "ly", B, 0, 0,
+
+  "less", NULL, 0, 1,
+  "lily", NULL, 0, 0,
+
+  "lessly", NULL, 0, 0,
+
+  "lessness", NULL, 0, 0,
+
+  "ness", NULL, 0, 0,
+
+  "nesses", NULL, 0, 0,
+
+  "o", NULL, 0, 0,
+
+  "on", S, 0, 1,
+  "or", T, 0, 0,
+
+  "oid", NULL, 0, 0,
+  "one", R, -1, 1,
+  "ous", NULL, 0, 0,
+
+  "ogen", NULL, 0, 0,
+
+  "oidal", NULL, 0, 0,
+  "oides", NULL, -1, 2,
+  "otide", NULL, 0, 0,
+  "ously", NULL, -1, 0,
+
+  "oidism", NULL, 0, 0,
+
+  "oidally", NULL, 0, 1,
+  "ousness", NULL, 0, 0,
+
+  "s", W, 0, 0,
+
+  "s'", NULL, 0, 0,
+
+  "um", U, 0, 1,
+  "us", V, 0, 0,
+
+  "ward", NULL, 0, 1,
+  "wise", NULL, 0, 0,
+
+  "y", B, 0, 0,
+
+  "yl", R, 0, 0,
+
+  "ying", B, 0, 1,
+  "yish", NULL, 0, 0,
+
+  "'s", NULL, 0, 0,
+};
+
+typedef struct node
+  {
+    char c;			/* First character  */
+    struct node *left;		/* used in balanced */
+    struct node *right;		/* binary tree      */
+    Ending_List *ptr[11];	/* the approriate location */
+  }
+First_Char_Node;
+
+static First_Char_Node First[] =
+{
+  '\'', NULL, NULL,
+  {NULL,
+   List + 293, NULL, NULL, NULL, NULL,
+   NULL, NULL, NULL, NULL, NULL},
+
+  'a', First, NULL,
+  {List,
+   List + 2, List + 9, List + 20, List + 39, List + 58,
+   List + 70, List + 77, List + 82, List + 87, List + 89},
+
+  'e', First + 1, First + 4,
+  {List + 91,
+   List + 93, List + 98, List + 106, List + 116, List + 126,
+   List + 133, List + 138, List + 142, List + 145, NULL},
+
+  'f', NULL, NULL,
+  {NULL,
+   NULL, List + 146, NULL, List + 147, NULL,
+   List + 148, NULL, NULL, NULL, NULL},
+
+  'h', First + 3, First + 5,
+  {NULL,
+   NULL, NULL, List + 149, NULL, NULL,
+   NULL, NULL, NULL, NULL, NULL},
+
+  'i', NULL, NULL,
+  {List + 150,
+   List + 152, List + 163, List + 181, List + 202, List + 222,
+   List + 239, List + 252, List + 258, NULL, List + 261},
+
+  'l', First + 2, First + 10,
+  {NULL,
+   List + 262, NULL, List + 263, NULL, List + 265,
+   NULL, List + 266, NULL, NULL, NULL},
+
+  'n', NULL, NULL,
+  {NULL,
+   NULL, NULL, List + 267, NULL, List + 268,
+   NULL, NULL, NULL, NULL, NULL},
+
+  'o', First + 7, First + 9,
+  {List + 269,
+   List + 270, List + 273, List + 275, List + 277, List + 280,
+   List + 281, NULL, NULL, NULL, NULL},
+
+  's', NULL, NULL,
+  {List + 283,
+   List + 284, NULL, NULL, NULL, NULL,
+   NULL, NULL, NULL, NULL, NULL},
+
+  'u', First + 8, First + 12,
+  {NULL,
+   List + 285, NULL, NULL, NULL, NULL,
+   NULL, NULL, NULL, NULL, NULL},
+
+  'w', NULL, NULL,
+  {NULL,
+   NULL, NULL, List + 287, NULL, NULL,
+   NULL, NULL, NULL, NULL, NULL},
+
+  'y', First + 11, NULL,
+  {List + 289,
+   List + 290, NULL, List + 291, NULL, NULL,
+   NULL, NULL, NULL, NULL, NULL},
+};
+
+
+/******************************************************************************
+ *
+ *  Look for a match in the suffix list.
+ *  Return the pointer to the end of the new stem if there is a match.
+ *  Otherwise, return the pointer to the end of the original word.
+ *
+ *  Method:
+ *      * Search for the first character in the suffix list.
+ *      * If there is match, search for the rest of the suffix list.
+ *         
+ *****************************************************************************/
+
+static char *
+remove_ending (char *word, int w_len)
+{
+  static First_Char_Node *root = First + 6;	/* the root of binary tree is 'l' */
+  First_Char_Node *p_first;	/* Points to the first character      */
+  /* of the possible suffix             */
+  Ending_List *p_list;		/* Points to the List of Endings      */
+  char *suffix_start,		/* Points to start of possible suffix */
+   *stem_end,			/* Points to the end of possible stem */
+    ch1, ch2;
+  int s_len,			/* Possible stem length               */
+    e_offset,			/* Offset from the end to the start of */
+  /* possible suffix                    */
+    not_done = 1, cmp;
+
+  stem_end = word + w_len - 1;
+  if (w_len > PREDEFINED_SIZE)	/* find the start of suffix   */
+    suffix_start = word + w_len - MAX_ENDING_SIZE;
+  else
+    suffix_start = word + MIN_STEM_LENGTH;
+
+  ch1 = *suffix_start;
+  do				/* Search for the first character     */
+    {
+      p_first = root;
+      do
+	{
+	  ch2 = p_first->c;
+	  if (ch1 == ch2)
+	    break;
+	  else if (ch1 > ch2)
+	    p_first = p_first->right;
+	  else
+	    p_first = p_first->left;
+	}
+      while (p_first != NULL);
+
+      if (p_first != NULL)	/* Search for the rest */
+	{
+	  e_offset = stem_end - suffix_start;
+	  if ((p_list = p_first->ptr[e_offset]) != NULL)
+	    {
+	      for (;;)		/* no need to compare the first char  */
+		{
+		  cmp = strcmp (suffix_start + 1, p_list->ending + 1);
+		  if (cmp > 0)
+		    if (p_list->right_offset)
+		      p_list += p_list->right_offset;
+		    else
+		      break;
+		  else if (cmp < 0)
+		    if (p_list->left_offset)
+		      p_list += p_list->left_offset;
+		    else
+		      break;
+		  else
+		    {
+		      s_len = suffix_start - word;
+		      if (!p_list->cond ||
+			  (*p_list->cond) (s_len, suffix_start - 1))
+			{
+			  *suffix_start = EOS;
+			  stem_end = suffix_start - 1;
+			  not_done = 0;
+			}
+		      break;
+		    }
+		}
+	    }
+	}
+      suffix_start++;
+    }
+  while (not_done && ((ch1 = *suffix_start) != EOS));
+  return (stem_end);
+}
Index: /tags/gsdl-21/gsdl/packages/mg/lib/lovinstem.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/lovinstem.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/lovinstem.h	(revision 699)
@@ -0,0 +1,53 @@
+/**************************************************************************
+ *
+ * lovinstem.h -- Lovins' stemmer header
+ * Copyright (C) 1994  Linh Huynh, 
+ *                     (glued into library by tes@kbs.citri.edu.au)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:16:54  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/25 07:55:36  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:32:47  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.2  1994/09/20  04:20:10  tes
+   * Changes made for 1.1
+   *
+   *
+ */
+#ifndef LOVINSTEM_H
+#define LOVINSTEM_H
+
+void lovinstem (unsigned char *word);
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/lib/memlib.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/memlib.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/memlib.c	(revision 699)
@@ -0,0 +1,59 @@
+/**************************************************************************
+ *
+ * memlib.c -- Malloc wrappers
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:16:54  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:32:10  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/08/22  00:24:47  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+#include "sysfuncs.h"
+#include "memlib.h"
+
+/* Defined as strdup is not an ANSI function */
+char *
+my_strdup(const char *str)
+{
+  char *ret_str = malloc(strlen(str)+1);
+  if (ret_str) return strcpy(ret_str, str);
+  else return (char*) 0;
+}
+
+
+Malloc_func Xmalloc = malloc;
+
+Realloc_func Xrealloc = realloc;
+
+Free_func Xfree = free;
+
+Strdup_func Xstrdup = my_strdup;
Index: /tags/gsdl-21/gsdl/packages/mg/lib/memlib.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/memlib.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/memlib.h	(revision 699)
@@ -0,0 +1,51 @@
+/**************************************************************************
+ *
+ * memlib.h -- Malloc wrappers
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef H_MEMLIB
+#define H_MEMLIB
+
+#ifdef STD_MEM
+
+#define Xmalloc malloc
+#define Xrealloc realloc
+#define Xfree free
+#define Xstrdup strdup
+
+#else
+
+
+#include "sysfuncs.h"
+
+typedef void *(*Malloc_func) (size_t);
+typedef void *(*Realloc_func) (void *, size_t);
+typedef void (*Free_func) (void *);
+typedef char *(*Strdup_func) (const char *);
+
+extern Malloc_func Xmalloc;
+extern Realloc_func Xrealloc;
+extern Free_func Xfree;
+extern Strdup_func Xstrdup;
+#endif
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/lib/messages.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/messages.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/messages.c	(revision 699)
@@ -0,0 +1,96 @@
+/**************************************************************************
+ *
+ * messages.c -- Message and error functions
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:16:55  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/23 03:38:06  rjmcnab
+
+   combined unix and windows versions of mg
+
+   Revision 1.1  1998/11/19 03:00:32  sjboddie
+   Added windows gdbm and mg versions
+
+   * Revision 1.1  1994/08/22  00:24:48  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+#include <stdarg.h>
+#include "messages.h"
+
+char *msg_prefix = "";
+
+
+void VOLATILE 
+FatalError (int ExitCode, const char *fmt,...)
+{
+  char buf[1024];
+  char *s, *pfx;
+  va_list args;
+  va_start (args, fmt);
+  vsprintf (buf, fmt, args);
+  s = strrchr (buf, '\n');
+  if (!s || *(s + 1) != '\0')
+    strcat (buf, "\n");
+  pfx = strrchr (msg_prefix, '/');
+  pfx = pfx ? pfx + 1 : msg_prefix;
+  fprintf (stderr, "%s%s%s", pfx, *pfx ? " : " : "", buf);
+  exit (ExitCode);
+}
+
+
+
+/*
+ * This function writes messages to stderr. Due to the fact that I can't 
+ * guarantee that the fprintf call is monatomic I have to implement a 
+ * semaphore system.
+ */
+void 
+Message (const char *fmt,...)
+{
+  char buf[1024];
+  char *s, *pfx;
+  va_list args;
+
+#ifdef QUIET
+  return; /* stop those messages!!!! */
+#endif  
+
+  va_start (args, fmt);
+  vsprintf (buf, fmt, args);
+  s = strrchr (buf, '\n');
+  if (!s || *(s + 1) != '\0')
+    strcat (buf, "\n");
+
+  pfx = strrchr (msg_prefix, '/');
+  pfx = pfx ? pfx + 1 : msg_prefix;
+  fprintf (stderr, "%s%s%s", pfx, *pfx ? " : " : "", buf);
+
+}
Index: /tags/gsdl-21/gsdl/packages/mg/lib/messages.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/messages.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/messages.h	(revision 699)
@@ -0,0 +1,39 @@
+/**************************************************************************
+ *
+ * messages.h -- Message and error functions
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef H_MESSAGES
+#define H_MESSAGES
+
+#ifdef __GNUC__
+#define VOLATILE volatile
+#else
+#define VOLATILE
+#endif
+
+extern char *msg_prefix;
+
+void VOLATILE FatalError (int ExitCode, const char *fmt,...);
+
+void Message (const char *fmt,...);
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/lib/mgheap.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/mgheap.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/mgheap.c	(revision 699)
@@ -0,0 +1,170 @@
+/**************************************************************************
+ *
+ * mgheap.c -- Heap routines
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:16:55  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1999/01/15 03:05:50  rjmcnab
+
+   Renamed lib/heap to be lib/mgheap (it was causing some problems with
+   some versions of the STL libraries which contained a heap.h)
+
+   Revision 1.1  1998/11/17 09:32:01  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/08/22  00:24:43  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "mgheap.h"
+
+#define SWAP(s, a, b) 						\
+do {								\
+     register int __i;						\
+     register char *__a = (a), *__b = (b);			\
+     for (__i = (s); __i; __i--)				\
+       {							\
+         register char __t = *__a;				\
+	 *__a++ = *__b; *__b++ = __t;				\
+       }							\
+} while(0)
+
+
+
+#define COPY(s, dst, src) 					\
+do {								\
+     register int __i;						\
+     register char *__dst = (dst), *__src = (src);		\
+     for (__i = (s); __i; __i--)				\
+       *__dst++ = *__src++;					\
+} while(0)
+
+
+
+static void 
+heap_heapify (void *heap, int size, int num, int curr, heap_comp hc)
+{
+  register int child = curr * 2;
+  while (child <= num)
+    {
+      if (child < num && hc ((char *) heap + child * size,
+			     (char *) heap + child * size - size) < 0)
+	child++;
+      if (hc ((char *) heap + (curr - 1) * size, (char *) heap + (child - 1) * size) > 0)
+	{
+	  SWAP (size, (char *) heap + (curr - 1) * size,
+		(char *) heap + (child - 1) * size);
+	  curr = child;
+	  child = child * 2;
+	}
+      else
+	break;
+    }
+}
+
+static void 
+heap_pullup (void *heap, int size, int num, heap_comp hc)
+{
+  register int curr = num;
+  register int parent = curr >> 1;
+  while (parent &&
+	 hc ((char *) heap + (curr - 1) * size, (char *) heap + (parent - 1) * size) < 0)
+    {
+      SWAP (size, (char *) heap + (curr - 1) * size,
+	    (char *) heap + (parent - 1) * size);
+      curr = parent;
+      parent = curr >> 1;
+    }
+}
+
+/************************************************************************
+ *
+ * NOTE: If you choose to change the comparison function the first thing 
+ *       you do after changing the function is call heap_build.
+ *
+ */
+void 
+heap_build (void *heap, int size, int num, heap_comp hc)
+{
+  register int i;
+  for (i = num / 2; i > 0; i--)
+    heap_heapify (heap, size, num, i, hc);
+}
+
+
+
+/****************************************************
+ *
+ * NOTE : The heap must be built before heap_sort is called. 
+ *        This has the effect of reversing the order of the array.
+ *		e.g. if your comparison function is designed to pull the
+ *                   biggest thing off the heap first then the result of
+ *		     sorting with this function will be to put the bigest
+ *                   thing at the end of the array.
+ *
+ */
+void 
+heap_sort (void *heap, int size, int num, heap_comp hc)
+{
+  register int i;
+  for (i = num; i > 1; i--)
+    {
+      SWAP (size, heap, (char *) heap + (i - 1) * size);
+      heap_heapify (heap, size, i - 1, 1, hc);
+    }
+}
+
+
+void 
+heap_deletehead (void *heap, int size, int *num, heap_comp hc)
+{
+  (*num)--;
+  SWAP (size, heap, (char *) heap + *num * size);
+  heap_heapify (heap, size, *num, 1, hc);
+}
+
+
+void 
+heap_changedhead (void *heap, int size, int num, heap_comp hc)
+{
+  heap_heapify (heap, size, num, 1, hc);
+}
+
+
+/***********************************************************************
+ *
+ * This assumes that the item has been added to the end of the
+ * array that is the heap. But that num has not been changed. 
+ *
+ */
+void 
+heap_additem (void *heap, int size, int *num, heap_comp hc)
+{
+  (*num)++;
+  heap_pullup (heap, size, *num, hc);
+}
Index: /tags/gsdl-21/gsdl/packages/mg/lib/mgheap.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/mgheap.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/mgheap.h	(revision 699)
@@ -0,0 +1,84 @@
+/**************************************************************************
+ *
+ * mgheap.h -- Heap routines
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/* Care should be taken when using these routines as NO error checking is
+ * done.
+ */
+
+
+
+/* The heap comparison routine should return 
+ * <0 : If a should be before b
+ *  0 : If a and b are equal
+ * >0 : If a should be after b
+ */
+typedef int (*heap_comp) (void *a, void *b);
+
+
+/************************************************************************
+ *
+ * NOTE: If you choose to change the comparison function the first thing 
+ *       you do after changing the function is call heap_build.
+ *
+ */
+void heap_build (void *heap, int size, int num, heap_comp hc);
+
+
+
+/************************************************************************
+ *
+ * NOTE : The heap must be built before heap_sort is called.
+ *        This has the effect of reversing the order of the array.
+ *		e.g. if your comparison function is designed to pull the
+ *                   biggest thing off the heap first then the result of
+ *		     sorting with this function will be to put the bigest
+ *                   thing at the end of the array.
+ *
+ */
+void heap_sort (void *heap, int size, int num, heap_comp hc);
+
+/***********************************************************************
+ *
+ * NOTE: After deleting the head the heap will be one item shorter.
+ *       The deleted item will be placed at the end of the heap.
+ *
+ */
+void heap_deletehead (void *heap, int size, int *num, heap_comp hc);
+
+
+/***********************************************************************
+ *
+ * If you change the "value" of the root node of the heap then you
+ * should call this to re-pritorize the heap.
+ *
+ */
+void heap_changedhead (void *heap, int size, int num, heap_comp hc);
+
+
+/***********************************************************************
+ *
+ * This assumes that the item has been added to the end of the
+ * array that is the heap. But that num has not been changed. 
+ *
+ */
+void heap_additem (void *heap, int size, int *num, heap_comp hc);
Index: /tags/gsdl-21/gsdl/packages/mg/lib/netorder.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/netorder.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/netorder.h	(revision 699)
@@ -0,0 +1,189 @@
+#ifndef NETORDER_H
+#define NETORDER_H
+
+#include "sysfuncs.h"
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+#include "win32in.h"
+#else
+# include <netinet/in.h>
+#endif
+
+#if defined (BIG_ENDIAN) || defined (_BIG_ENDIAN) || defined (__BIG_ENDIAN)
+# ifndef _BIG_ENDIAN
+#  define _BIG_ENDIAN
+# endif
+#endif
+
+#if defined (LITTLE_ENDIAN) || defined (_LITTLE_ENDIAN) || defined(__LITTLE_ENDIAN)
+# ifndef _LITTLE_ENDIAN
+#  define _LITTLE_ENDIAN
+# endif
+#endif
+
+#ifdef _LITTLE_ENDIAN
+
+/* double */
+#define HTOND(d)                                                                                  \
+        do {                                                                                      \
+             unsigned long tmph, tmpl;                                                            \
+	     bcopy ((char *) &d, (char *) &tmph, sizeof(double) >> 1);                            \
+	     bcopy ((char *) &d + (sizeof(double) >> 1), (char *) &tmpl, sizeof (double) >> 1);   \
+	     tmph = htonl (tmph);                                                                 \
+	     tmpl = htonl (tmpl);                                                                 \
+	     bcopy ((char *) &tmpl, (char *) &d, sizeof (double) >> 1);                           \
+	     bcopy ((char *) &tmph, (char *) &d + (sizeof(double) >> 1), sizeof (double) >> 1);   \
+	}while(0)
+#define NTOHD(d)                                                                                  \
+        do {                                                                                      \
+             unsigned long tmph, tmpl;                                                            \
+	     bcopy ((char *) &d, (char *) &tmph, sizeof(double) >> 1);                            \
+	     bcopy ((char *) &d + (sizeof(double) >> 1), (char *) &tmpl, sizeof (double) >> 1);   \
+	     tmph = ntohl (tmph);                                                                 \
+	     tmpl = ntohl (tmpl);                                                                 \
+	     bcopy ((char *) &tmpl, (char *) &d, sizeof (double) >> 1);                           \
+	     bcopy ((char *) &tmph, (char *) &d + (sizeof(double) >> 1), sizeof (double) >> 1);   \
+        }while(0)
+#define HTOND2(hd, nd)                                                                            \
+        do {                                                                                      \
+             unsigned long tmph, tmpl;                                                            \
+	     bcopy ((char *) &hd, (char *) &tmph, sizeof(double) >> 1);                           \
+	     bcopy ((char *) &hd + (sizeof(double) >> 1), (char *) &tmpl, sizeof (double) >> 1);  \
+	     tmph = htonl (tmph);                                                                 \
+	     tmpl = htonl (tmpl);                                                                 \
+	     bcopy ((char *) &tmpl, (char *) &nd, sizeof (double) >> 1);                          \
+	     bcopy ((char *) &tmph, (char *) &nd + (sizeof(double) >> 1), sizeof (double) >> 1);  \
+	}while(0)
+#define NTOHD2(nd, hd)                                                                            \
+        do {                                                                                      \
+             unsigned long tmph, tmpl;                                                            \
+	     bcopy ((char *) &nd, (char *) &tmph, sizeof(double) >> 1);                           \
+	     bcopy ((char *) &nd + (sizeof(double) >> 1), (char *) &tmpl, sizeof (double) >> 1);  \
+	     tmph = ntohl (tmph);                                                                 \
+	     tmpl = ntohl (tmpl);                                                                 \
+	     bcopy ((char *) &tmpl, (char *) &hd, sizeof (double) >> 1);                          \
+	     bcopy ((char *) &tmph, (char *) &hd + (sizeof(double) >> 1), sizeof (double) >> 1);  \
+        }while(0)
+
+/* float */
+#define HTONF(f)                                                                   \
+        do {                                                                       \
+             unsigned long tmp;                                                    \
+             bcopy ((char *) &(f), (char *) &tmp, sizeof (float));                 \
+             HTONUL (tmp);                                                         \
+	     bcopy ((char *) &tmp, (char *) &(f), sizeof (float));                 \
+	}while(0)
+#define NTOHF(f)                                                                   \
+        do {                                                                       \
+             unsigned long tmp;                                                    \
+             bcopy ((char *) &(f), (char *) &tmp, sizeof (float));                 \
+	     NTOHUL (tmp);                                                         \
+	     bcopy ((char *) &tmp, (char *) &(f), sizeof (float));                 \
+	}while(0)
+#define HTONF2(hf, nf)                                                             \
+        do {                                                                       \
+             unsigned long tmp;                                                    \
+             bcopy ((char *) &(hf), (char *) &tmp, sizeof (float));                \
+             HTONUL (tmp);                                                         \
+	     bcopy ((char *) &tmp, (char *) &(nf), sizeof (float));                \
+	}while(0)
+#define NTOHF2(nf, hf)                                                             \
+        do {                                                                       \
+             unsigned long tmp;                                                    \
+             bcopy ((char *) &(nf), (char *) &tmp, sizeof (float));                \
+	     NTOHUL (tmp);                                                         \
+	     bcopy ((char *) &tmp, (char *) &(hf), sizeof (float));                \
+	}while(0)
+
+/* pointers */
+#define HTONP(p)          ((p) = (void *) htonl ((unsigned long) p))
+#define NTOHP(p)          ((p) = (void *) ntohl ((unsigned long) p))
+#define HTONP2(hp, np)    ((np) = (void *) htonl ((unsigned long) hp))
+#define NTOHP2(np, hp)    ((hp) = (void *) ntohl ((unsigned long) np))
+
+/* unsigned long */
+#define HTONUL(l)         ((l) = htonl((l)))
+#define NTOHUL(l)         ((l) = ntohl((l)))
+#define HTONUL2(hl, nl)   ((nl) = htonl ((hl)))
+#define NTOHUL2(nl, hl)   ((hl) = ntohl ((nl)))
+
+/* signed long */
+#define HTONSL(l)         ((l) = (long) htonl ((unsigned long) (l)))
+#define NTOHSL(l)         ((l) = (long) ntohl ((unsigned long) (l)))
+#define HTONSL2(hl, nl)   ((nl) = (long) htonl ((unsigned long) (hl)))
+#define NTOHSL2(nl, hl)   ((hl) = (long) ntohl ((unsigned long) (nl)))
+
+/* unsigned int */
+#define HTONUI(i)         ((i) = (unsigned int) htonl ((unsigned long) (i)))
+#define NTOHUI(i)         ((i) = (unsigned int) ntohl ((unsigned long) (i)))
+#define HTONUI2(hi, ni)   ((ni) = (unsigned int) htonl ((unsigned long) (hi)))
+#define NTOHUI2(ni, hi)   ((hi) = (unsigned int) ntohl ((unsigned long) (ni)))
+
+/* signed int */
+#define HTONSI(i)         ((i) = (int) htonl ((unsigned long) (i)))
+#define NTOHSI(i)         ((i) = (int) ntohl ((unsigned long) (i)))
+#define HTONSI2(hi, ni)   ((ni) = (int) htonl ((unsigned long) (hi)))
+#define NTOHSI2(ni, hi)   ((hi) = (int) ntohl ((unsigned long) (ni)))
+
+/* unsigned short */
+#define HTONUS(s)         ((s) = htons((s)))
+#define NTOHUS(s)         ((s) = ntohs((s)))
+#define HTONUS2(hs, ns)   ((ns) = htons((hs)))
+#define NTOHUS2(ns, hs)   ((hs) = ntohs((ns)))
+
+#else   /* _BIG_ENDIAN */
+
+/* double */
+#define HTOND(d)          (d)
+#define NTOHD(d)          (d)
+#define HTOND2(hd, nd)    ((hd) = (nd))
+#define NTOHD2(nd, hd)    ((nd) = (hd))
+
+/* float */
+#define HTONF(f)          (f)
+#define NTOHF(f)          (f)
+#define HTONF2(hf, nf)    ((nf) = (hf))
+#define NTOHF2(nf, hf)    ((hf) = (nf))
+
+/* pointers */
+#define HTONP(p)          (p)
+#define NTOHP(p)          (p)
+#define HTONP2(hp, np)    ((np) = (hp))
+#define NTOHP2(np, hp)    ((hp) = (np))
+
+/* unsigned long */
+#define HTONUL(l)         (l)
+#define NTOHUL(l)         (l)
+#define HTONUL2(hl, nl)   ((nl) = (hl))
+#define NTOHUL2(nl, hl)   ((hl) = (nl))
+
+/* signed long */
+#define HTONSL(l)         (l)
+#define NTOHSL(l)         (l)
+#define HTONSL2(hl, nl)   ((nl) = (hl))
+#define NTOHSL2(nl, hl)   ((hl) = (nl))
+
+/* unsigned int */
+#define HTONUI(i)         (i)
+#define NTOHUI(i)         (i)
+#define HTONUI2(hi, ni)   ((ni) = (hi))
+#define NTOHUI2(ni, hi)   ((hi) = (ni))
+
+/* signed int */
+#define HTONSI(i)         (i)
+#define NTOHSI(i)         (i)
+#define HTONSI2(hi, ni)   ((ni) = (hi))
+#define NTOHSI2(ni, hi)   ((hi) = (ni))
+
+/* unsigned short */
+#define HTONUS(s)         (s)
+#define NTOHUS(s)         (s)
+#define HTONUS2(hs, ns)   ((ns) = (hs))
+#define NTOHUS2(ns, hs)   ((hs) = (ns))
+
+
+
+#endif
+
+#endif /* netorder.h */
Index: /tags/gsdl-21/gsdl/packages/mg/lib/pathmax.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/pathmax.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/pathmax.h	(revision 699)
@@ -0,0 +1,53 @@
+/* Define PATH_MAX somehow.  Requires sys/types.h.
+   Copyright (C) 1992 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+#ifndef _PATHMAX_H
+#define _PATHMAX_H
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+/* Non-POSIX BSD systems might have gcc's limits.h, which doesn't define
+   PATH_MAX but might cause redefinition warnings when sys/param.h is
+   later included (as on MORE/BSD 4.3).  */
+#if defined(_POSIX_VERSION) || (defined(HAVE_LIMITS_H) && !defined(__GNUC__))
+#include <limits.h>
+#endif
+
+#ifndef _POSIX_PATH_MAX
+#define _POSIX_PATH_MAX 255
+#endif
+
+#if !defined(PATH_MAX) && defined(_PC_PATH_MAX)
+#define PATH_MAX (pathconf ("/", _PC_PATH_MAX) < 1 ? 1024 : pathconf ("/", _PC_PATH_MAX))
+#endif
+
+/* Don't include sys/param.h if it already has been.  */
+#if defined(HAVE_SYS_PARAM_H) && !defined(PATH_MAX) && !defined(MAXPATHLEN)
+#include <sys/param.h>
+#endif
+
+#if !defined(PATH_MAX) && defined(MAXPATHLEN)
+#define PATH_MAX MAXPATHLEN
+#endif
+
+#ifndef PATH_MAX
+#define PATH_MAX _POSIX_PATH_MAX
+#endif
+
+#endif /* _PATHMAX_H */
Index: /tags/gsdl-21/gsdl/packages/mg/lib/perf_hash.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/perf_hash.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/perf_hash.c	(revision 699)
@@ -0,0 +1,1094 @@
+/**************************************************************************
+ *
+ * perf_hash.c -- Perfect hashing functions
+ * Copyright (C) 1992  Bohdan S. Majewski
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ * 1994 Neil Sharman
+ *      Small modifications were made to the interface so that it fitted in
+ *      with mg.
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:16:57  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:32:17  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.2  1994/08/22  00:26:30  tes
+   * Made perf_hash take the ceiling of 1.23 * num
+   * in lieu of the floor.
+   * This was done so that perf_hash works ok on
+   * a small number of keys.
+   *
+   * Revision 1.1  1994/08/22  00:24:49  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+#define STRUCT
+
+#include "sysfuncs.h"
+
+#include "local_strings.h"
+#include "memlib.h"
+#include "messages.h"
+#include "perf_hash.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#define FALSE 0
+#define TRUE  1
+
+#define STATIC 0
+
+/* Random Number stuff */
+static long seed[] = {0, 0};
+#define RANDOM() irandm(seed)
+#define SEED_RANDOM(the_seed) do{ seed[0] = the_seed; }while(0)
+
+/* Max number of keys - must be a power of 2 minus 1 */
+static int MAX_M;
+
+
+static int MAX_CH;
+
+
+/* the width of the mapping tables  */
+static int MAX_L;
+
+
+
+#define c     1.23
+static int MAX_N;
+
+
+
+
+static u_char *translate;
+
+#define MEMBER 0
+#define ERASED 1
+
+
+
+/* direction constants for the edge oriented 3-graph representation */
+#define A              0
+#define B        (MAX_M)
+#define C          (2*B)
+
+/* NODEa, NODEb, NODEc, FIRST and NEXT are used to represent a 3-graph */
+static int *NODEa, *NODEb, *NODEc;
+static int *FIRST;
+static int *NEXT;
+
+
+#define norm(e) ((e) % MAX_M)
+/* in general should be ((e) % MAX_M) but MAX_M = 2^k - 1 */
+
+
+static char *mk;		/* for marking edges/vertices as removed */
+static int *g;			/* array g defines a minimal perfect hash function */
+
+
+
+/* stack of edges, created during the independency */
+/* test, used in the assignment step               */
+static struct
+  {
+    int sp;
+    int *st;
+  }
+S;
+#define push(s, e) s.st[s.sp++] = (e)
+#define pop(s, e)  e = s.st[--s.sp]
+
+#ifndef STRUCT
+static long **tb0, **tb1, **tb2;
+#else
+struct tb_entry **tb;
+#endif
+
+
+static int 
+unique (int v)
+{				/* checks if vertex v belongs to only one edge */
+  return (FIRST[v] != 0 && NEXT[FIRST[v]] == 0) ? FIRST[v] : 0;
+}				/*unique */
+
+
+
+
+
+
+
+
+static void 
+delete (int v, int e)
+{				/* deletes edge e from list of v */
+
+  int b;
+
+  b = FIRST[v];
+  assert (norm (b) != 0);
+  if (norm (b) == e)
+    FIRST[v] = NEXT[FIRST[v]];
+  else
+    {
+      while (norm (NEXT[b]) != e)
+	{
+	  b = NEXT[b];
+	  assert (norm (b) != 0);
+	}
+      NEXT[b] = NEXT[NEXT[b]];
+    }
+}				/*delete */
+
+
+
+
+
+
+#if 0
+/* recursively removes the edges of a 3-graph which have at least */
+/* one unique vertex. The removed edges are placed on stack S     */
+static void 
+ph_remove (int e, int v)
+{
+  int b;
+  mk[e] = ERASED;
+  push (S, e);
+
+  if (NODEa[e] != v)
+    delete (NODEa[e], e);
+  if (NODEb[e] != v)
+    delete (NODEb[e], e);
+  if (NODEc[e] != v)
+    delete (NODEc[e], e);
+
+  if (NODEa[e] != v && mk[b = norm (unique (NODEa[e]))] == MEMBER)
+    ph_remove (b, NODEa[e]);
+  if (NODEb[e] != v && mk[b = norm (unique (NODEb[e]))] == MEMBER)
+    ph_remove (b, NODEb[e]);
+  if (NODEc[e] != v && mk[b = norm (unique (NODEc[e]))] == MEMBER)
+    ph_remove (b, NODEc[e]);
+}
+#else
+
+
+static int StackPos, StackMax;
+static struct Stack
+  {
+    int e, v, m;
+  }
+ *Stack = NULL;
+
+#define StackAdd(_e, _v, _m)						\
+  do {									\
+    if (StackPos == StackMax)						\
+      {									\
+        StackMax <<= 1;							\
+	if (!(Stack = Xrealloc(Stack, sizeof(*Stack) * StackMax)))	\
+	  return(-1);							\
+      }									\
+    Stack[StackPos].e = (_e);						\
+    Stack[StackPos].v = (_v);						\
+    Stack[StackPos++].m = (_m);						\
+  } while(0)
+
+
+static int 
+ph_remove (int e, int v)
+{
+  if (!Stack)
+    {
+      StackMax = 256;
+      if (!(Stack = Xmalloc (sizeof (*Stack) * StackMax)))
+	return (-1);
+      StackPos = 0;
+    }
+  Stack[StackPos].e = e;
+  Stack[StackPos].v = v;
+  Stack[StackPos].m = 0;
+  StackPos++;
+  while (StackPos)
+    {
+      int m, b;
+      StackPos--;
+      e = Stack[StackPos].e;
+      v = Stack[StackPos].v;
+      m = Stack[StackPos].m;
+
+
+      switch (m)
+	{
+	case 0:
+	  {
+	    mk[e] = ERASED;
+	    push (S, e);
+
+	    if (NODEa[e] != v)
+	      delete (NODEa[e], e);
+	    if (NODEb[e] != v)
+	      delete (NODEb[e], e);
+	    if (NODEc[e] != v)
+	      delete (NODEc[e], e);
+
+	    if (NODEa[e] != v && mk[b = norm (unique (NODEa[e]))] == MEMBER)
+	      {
+		Stack[StackPos++].m = 1;
+		StackAdd (b, NODEa[e], 0);
+		break;
+	      }
+	  }
+	case 1:
+	  {
+	    if (NODEb[e] != v && mk[b = norm (unique (NODEb[e]))] == MEMBER)
+	      {
+		Stack[StackPos++].m = 2;
+		StackAdd (b, NODEb[e], 0);
+		break;
+	      }
+	  }
+	case 2:
+	  if (NODEc[e] != v && mk[b = norm (unique (NODEc[e]))] == MEMBER)
+	    StackAdd (b, NODEc[e], 0);
+	}
+    }
+  return (0);
+}
+#endif
+
+
+
+
+
+/* returns TRUE if and only if set of edges created in the tree_builder */
+/* procedure form a 3-graph with independent edges. Edges of a 3-graph  */
+/* are independent iff repeated deletion of edges containing vertices   */
+/* of degree 1 results in a graph with no edges in it.                  */
+static int 
+acyclic (int m, int n)
+{
+
+  int v, e;
+  /* build list of edges for each node */
+  for (v = 1; v <= n; FIRST[v++] = 0);
+  for (e = 1; e <= m; e++)
+    {
+      NEXT[A + e] = FIRST[NODEa[e]];
+      FIRST[NODEa[e]] = A + e;
+      NEXT[B + e] = FIRST[NODEb[e]];
+      FIRST[NODEb[e]] = B + e;
+      NEXT[C + e] = FIRST[NODEc[e]];
+      FIRST[NODEc[e]] = C + e;
+
+      /* mark edge e as belonging to the hypergraph */
+      mk[e] = MEMBER;
+    }
+
+
+  S.sp = 0;			/* empty the stack */
+  mk[0] = ERASED;		/* a sentinel for the test below */
+  for (v = 1; v <= n; v++)
+    {
+      if (mk[e = norm (unique (v))] == MEMBER)
+	ph_remove (e, v);
+    }
+
+  /* check if any edges were left in the graph */
+  return S.sp == m;
+}				/*acyclic */
+
+
+
+
+
+
+
+
+/* repeatedly maps words into edges of a 3-graph until the edges */
+/* are independent.                                              */
+static int 
+tree_builder (int num, u_char ** keys, int *n)
+{
+
+  int e, k = 0;
+  int i, j;
+  u_char *w;
+
+  do
+    {
+      *n = (int) (num * c + 1);
+      /* initialize tables of random integers */
+      for (i = 0; i < MAX_CH; i++)
+	for (j = 0; j < MAX_L; j++)
+	  {
+#ifndef STRUCT
+	    tb0[i][j] = RANDOM () % *n;
+	    tb1[i][j] = RANDOM () % *n;
+	    tb2[i][j] = RANDOM () % *n;
+#else
+	    struct tb_entry *tbp = &tb[i][j];
+	    tbp->tb0 = RANDOM () % *n;
+	    tbp->tb1 = RANDOM () % *n;
+	    tbp->tb2 = RANDOM () % *n;
+#endif
+	  }
+
+      /* compute an edge (NODEa[e], NODEb[e], NODEc[e]) for each word */
+      for (e = 1; e <= num; e++)
+	{
+	  /*generate an edge corresponding to the ith word */
+	  int x = 0, y = 0, z = 0, l;
+	  w = keys[e - 1];
+
+	  l = *w++;
+	  j = (l - 1) % MAX_L;
+
+	  do
+	    {
+#ifndef STRUCT
+	      x += tb0[translate[*w]][j];
+	      y += tb1[translate[*w]][j];
+	      z += tb2[translate[*w]][j];
+#else
+	      struct tb_entry *tbp = &tb[translate[*w]][j];
+	      x += tbp->tb0;
+	      y += tbp->tb1;
+	      z += tbp->tb2;
+#endif
+	      if (++j >= MAX_L)
+		j = 0;
+	      w++;
+	    }
+	  while (--l);
+	  x = (x % *n) + 1;
+	  y = (y % *n) + 1;
+	  z = (z % *n) + 1;
+	  if (y == x && ++y > *n)
+	    y = 1;
+	  if (z == x && ++z > *n)
+	    z = 1;
+	  if (z == y)
+	    {
+	      if (++z > *n)
+		z = 1;
+	      if (z == x && ++z > *n)
+		z = 1;
+	    }
+
+	  NODEa[e] = x;
+	  NODEb[e] = y;
+	  NODEc[e] = z;
+	}
+      if (++k > 50)		/* make sure we will not wait for ever */
+	FatalError (1, "Unable to generate the perfect hash function. "
+		    "This is probably because there aren't enough words");
+    }
+  while (!acyclic (num, *n));
+  return k;
+}				/*tree_builder */
+
+
+
+
+
+
+
+
+
+/* calculates values of the g function/array so that i-th word is */
+/* placed at the (i-1)st position of a hash table.                */
+static void 
+assign (int n)
+{
+
+  int v, w = 0, e;
+
+  /* mark all the vertices of the 3-graph as unassigned */
+  for (v = 1; v <= n; mk[v++] = FALSE);
+
+  while (S.sp > 0)
+    {
+      int xor = 0;
+      pop (S, e);
+      v = NODEa[e];
+      if (mk[v])
+	xor ^= g[v];
+      else
+	{
+	  g[w = v] = 0;
+	  mk[v] = TRUE;
+	}
+      v = NODEb[e];
+      if (mk[v])
+	xor ^= g[v];
+      else
+	{
+	  g[w = v] = 0;
+	  mk[v] = TRUE;
+	}
+      v = NODEc[e];
+      if (mk[v])
+	xor ^= g[v];
+      else
+	{
+	  g[w = v] = 0;
+	  mk[v] = TRUE;
+	}
+      g[w] = (e - 1) ^ xor;
+    }
+}				/*assign */
+
+
+
+
+
+
+
+
+
+/*check returns 0 if no error in mphf is detected, else returns error code */
+static int 
+check (int m)
+{
+
+  int e, err_code = 0;
+
+  /*for each word compute its place in hash table and check if correct */
+  for (e = 1; e <= m; e++)
+    {
+      if ((e - 1) != (g[NODEa[e]] ^ g[NODEb[e]] ^ g[NODEc[e]]))
+	{
+	  (void) fprintf (stderr, "Error for %dth word detected.\n", e);
+	  err_code = e;
+	}
+    }
+  return err_code;
+}				/*check */
+
+
+
+
+
+
+
+#if 0
+void 
+outputh (int m, int n)
+{
+
+  int i, j, w = (int) ceil (log10 ((double) n + 1.0)) + 1;
+
+#if 0
+  char *PRG =
+  "int h(char *s)\n"
+  "{\n"
+  "  int u, v, w, i;\n"
+  "i = (strlen(s) - 1) % _HXL;\n"
+  "  u = v = w = 0;\n"
+  "  while(*s)\n"
+  "    {\n"
+  "      u += mt1[(*s) - _HFC][i];\n"
+  "      v += mt2[(*s) - _HFC][i];\n"
+  "      w += mt3[(*s++) - _HFC][i];\n"
+  "      if(++i >= _HXL) i = 0;\n"
+  "    }\n"
+  "  u %= _HFN; v %= _HFN; w %= _HFN;\n"
+  "  if(u == v && ++v >= _HFN) v = 0;\n"
+  "  if(w == u && ++w >= _HFN) w = 0;\n"
+  "  if(w == v && ++w >= _HFN) w = 0;\n"
+  "  if(w == u && ++w >= _HFN) w = 0;\n"
+  "  return (g[u] ^ g[v] ^ g[w]) % _HFM;\n"
+  "};\n\n";
+#endif
+
+  char *PRG =
+  "int h(char *s)\n"
+  "{\n"
+  "  int u, v, w, i;\n"
+  "  i = (strlen(s) - 1) % _HXL;\n"
+  "  u = v = w = 0;\n"
+  "  while(*s)\n"
+  "    {\n"
+  "      if((u += mt1[(*s) - _HFC][i]) >= _HFN) u -= _HFN;\n"
+  "      if((v += mt2[(*s) - _HFC][i]) >= _HFN) v -= _HFN;\n"
+  "      if((w += mt3[(*s++) - _HFC][i]) >= _HFN) w -= _HFN;\n"
+  "      if(++i >= _HXL) i = 0;\n"
+  "    }\n"
+  "  if(u == v && ++v >= _HFN) v = 0;\n"
+  "  if(w == v) \n"
+  "    {\n"
+  "      if(++w >= _HFN) w = 0;\n"
+  "      if(w == u && ++w >= _HFN) w = 0;\n"
+  "    }\n"
+  "  else\n"
+  "    if(w == u)\n"
+  "      {\n"
+  "	     if(++w >= _HFN) w = 0;\n"
+  "	     if(w == v && ++w >= _HFN) w = 0;\n"
+  "      }\n"
+  "  return ((g[u] ^ g[v] ^ g[w]) > _HFM)? 0 : (g[u] ^ g[v] ^ g[w]);\n"
+  "}\n\n";
+
+  printf ("#define _HFN %d\n#define _HFC %d\n", n, fc);
+  printf ("#define _HFM %d\n#define _HXL %d\n", m, MAX_L);
+  printf ("#define _HFA %d\n\nstatic int g[_HFN] = {\n", lc - fc + 1);
+  for (i = 1; i < n; i++)
+    {
+      if (i % 8 == 0)
+	putchar ('\n');
+      printf ("%5d, ", g[i]);
+    }
+  printf ("%5d};\n\n", g[n]);
+
+  printf ("static int mt1[_HFA][_HXL] = {\n");
+  for (j = fc; j < lc; j++)
+    {
+      printf ("{");
+      for (i = 0; i < MAX_L - 1; i++)
+	printf ("%*d,", w, tb0[j][i]);
+      printf ("%*d},\n", w, tb0[j][MAX_L - 1]);
+    }
+  printf ("{");
+  for (i = 0; i < MAX_L - 1; i++)
+    printf ("%*d,", w, tb0[lc][i]);
+  printf ("%*d}\n", w, tb0[lc][MAX_L - 1]);
+  printf ("};\n\n");
+
+  printf ("static int mt2[_HFA][_HXL] = {\n");
+  for (j = fc; j < lc; j++)
+    {
+      printf ("{");
+      for (i = 0; i < MAX_L - 1; i++)
+	printf ("%*d,", w, tb1[j][i]);
+      printf ("%*d},\n", w, tb1[j][MAX_L - 1]);
+    }
+  printf ("{");
+  for (i = 0; i < MAX_L - 1; i++)
+    printf ("%*d,", w, tb1[lc][i]);
+  printf ("%*d}\n", w, tb1[lc][MAX_L - 1]);
+  printf ("};\n\n");
+
+  printf ("static int mt3[_HFA][_HXL] = {\n");
+  for (j = fc; j < lc; j++)
+    {
+      printf ("{");
+      for (i = 0; i < MAX_L - 1; i++)
+	printf ("%*d,", w, tb2[j][i]);
+      printf ("%*d},\n", w, tb2[j][MAX_L - 1]);
+    }
+  printf ("{");
+  for (i = 0; i < MAX_L - 1; i++)
+    printf ("%*d,", w, tb2[lc][i]);
+  printf ("%*d}\n", w, tb2[lc][MAX_L - 1]);
+  printf ("};\n\n");
+
+  printf ("%s", PRG);
+}				/*outputh */
+#endif
+
+
+
+
+
+
+
+static void 
+make_trans_func (int num, u_char ** keys)
+{
+  int i, j;
+
+  bzero ((char *) translate, sizeof (translate));
+
+  MAX_L = 1;
+  for (i = 0; i < num; i++)
+    {
+      unsigned len = keys[i][0];
+      u_char *s = keys[i] + 1;
+      for (; len; len--, s++)
+	translate[*s] = 1;
+      if (i)
+	{
+	  int l;
+	  l = prefixlen (keys[i - 1], keys[i]);
+	  if (l + 1 > MAX_L)
+	    MAX_L = l + 1;
+	}
+    }
+  j = 0;
+  for (i = 0; i < 256; i++)
+    if (translate[i])
+      translate[i] = j++;
+  MAX_CH = j;
+}
+
+
+static void 
+temp_free (void)
+{
+  if (NODEa)
+    Xfree (NODEa);
+  if (NODEb)
+    Xfree (NODEb);
+  if (NODEc)
+    Xfree (NODEc);
+  if (FIRST)
+    Xfree (FIRST);
+  if (NEXT)
+    Xfree (NEXT);
+  if (S.st)
+    Xfree (S.st);
+  if (mk)
+    Xfree (mk);
+}
+
+static void 
+all_free (void)
+{
+  int i;
+  temp_free ();
+  if (translate)
+    Xfree (translate);
+  if (g)
+    Xfree (g);
+#ifndef STRUCT
+  for (i = 0; i < MAX_CH; i++)
+    {
+      if (tb0 && tb0[i])
+	Xfree (tb0[i]);
+      if (tb1 && tb1[i])
+	Xfree (tb1[i]);
+      if (tb2 && tb2[i])
+	Xfree (tb2[i]);
+    }
+  if (tb0)
+    Xfree (tb0);
+  if (tb1)
+    Xfree (tb1);
+  if (tb2)
+    Xfree (tb2);
+#else
+  for (i = 0; i < MAX_CH; i++)
+    if (tb && tb[i])
+      Xfree (tb[i]);
+  if (tb)
+    Xfree (tb);
+#endif
+}
+
+static int 
+allocate_memory (void)
+{
+  int i, ok = 1;
+  NODEa = Xmalloc (sizeof (int) * (MAX_M + 1));
+  NODEb = Xmalloc (sizeof (int) * (MAX_M + 1));
+  NODEc = Xmalloc (sizeof (int) * (MAX_M + 1));
+  FIRST = Xmalloc (sizeof (int) * (MAX_N + 1));
+  NEXT = Xmalloc (sizeof (int) * (MAX_M + 1) * 3);
+  S.st = Xmalloc (sizeof (int) * MAX_M);
+  mk = Xmalloc (sizeof (char) * (MAX_N + 1));
+  g = Xmalloc (sizeof (int) * (MAX_N + 1));
+#ifndef STRUCT
+  tb0 = Xmalloc (sizeof (int *) * MAX_CH);
+  tb1 = Xmalloc (sizeof (int *) * MAX_CH);
+  tb2 = Xmalloc (sizeof (int *) * MAX_CH);
+  for (i = 0; i < MAX_CH; i++)
+    {
+      if (tb0)
+	if (!(tb0[i] = Xmalloc (sizeof (long) * MAX_L)))
+	    ok = 0;
+      if (tb1)
+	if (!(tb1[i] = Xmalloc (sizeof (long) * MAX_L)))
+	    ok = 0;
+      if (tb2)
+	if (!(tb2[i] = Xmalloc (sizeof (long) * MAX_L)))
+	    ok = 0;
+    }
+
+  if (!(NODEa && NODEb && NODEc && FIRST && NEXT && S.st && mk && g &&
+	tb0 && tb1 && tb2 && ok))
+    {
+      all_free ();
+      return (-1);
+    }
+#else
+  tb = Xmalloc (sizeof (struct tb_entry *) * MAX_CH);
+  for (i = 0; i < MAX_CH; i++)
+    if (tb)
+      if (!(tb[i] = Xmalloc (sizeof (struct tb_entry) * MAX_L)))
+	  ok = 0;
+
+  if (!(NODEa && NODEb && NODEc && FIRST && NEXT && S.st && mk && g &&
+	tb && ok))
+    {
+      all_free ();
+      return (-1);
+    }
+#endif
+  return (0);
+}
+
+
+perf_hash_data *
+gen_hash_func (int num, u_char ** keys, int r)
+{
+  int n;
+  perf_hash_data *phd;
+  translate = Xmalloc (sizeof (u_char) * 256);
+  bzero ((char *) translate, sizeof (u_char) * 256);  /* [RPAP - Feb 97: WIN32 Port] */
+  make_trans_func (num, keys);
+
+  if (r <= 0)
+    SEED_RANDOM ((long) time ((time_t *) NULL));
+  else
+    SEED_RANDOM (r);
+
+  MAX_M = num + 1;
+
+  MAX_N = MAX_M + MAX_M / 4;
+
+  if (allocate_memory () == -1)
+    return (NULL);
+
+  /* construct an "acyclic" hypergraph */
+  tree_builder (num, keys, &n);
+  /* compute function g */
+  assign (n);
+
+  if (check (num) != 0)
+    FatalError (1, "An error has been detected in the mphf.");
+  temp_free ();
+
+  if (!(phd = Xmalloc (sizeof (perf_hash_data))))
+    return (NULL);
+
+  bcopy ((char *) &g[1], (char *) g, n * sizeof (int));
+
+  phd->MAX_L = MAX_L;
+  phd->MAX_N = n;
+  phd->MAX_M = num;
+  phd->MAX_CH = MAX_CH;
+  phd->translate = translate;
+  phd->g = g;
+#ifndef STRUCT
+  phd->tb0 = tb0;
+  phd->tb1 = tb1;
+  phd->tb2 = tb2;
+#else
+  phd->tb = tb;
+#endif
+  return (phd);
+}
+
+int 
+write_perf_hash_data (FILE * f, perf_hash_data * phd)
+{
+  int tot, i;
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONSI(phd->MAX_L);
+  HTONSI(phd->MAX_N);
+  HTONSI(phd->MAX_M);
+  HTONSI(phd->MAX_CH);
+
+  tot = fwrite ((char *) &phd->MAX_L, sizeof (int), 1, f) * sizeof (int);
+  tot += fwrite ((char *) &phd->MAX_N, sizeof (int), 1, f) * sizeof (int);
+  tot += fwrite ((char *) &phd->MAX_M, sizeof (int), 1, f) * sizeof (int);
+  tot += fwrite ((char *) &phd->MAX_CH, sizeof (int), 1, f) * sizeof (int);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHSI(phd->MAX_L);
+  NTOHSI(phd->MAX_N);
+  NTOHSI(phd->MAX_M);
+  NTOHSI(phd->MAX_CH);
+
+  tot += fwrite ((char *) phd->translate, sizeof (u_char), 256, f);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = 0; i < phd->MAX_N + 1; i++)
+    HTONSI(phd->g[i]);
+
+  tot += fwrite ((char *) phd->g, sizeof (int), phd->MAX_N + 1, f) * sizeof (int);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = 0; i < phd->MAX_N + 1; i++)
+    NTOHSI(phd->g[i]);
+
+#ifndef STRUCT
+  for (i = 0; i < phd->MAX_CH; i++)
+    {
+      /* [RPAP - Jan 97: Endian Ordering] */
+      int j;
+      for (j = 0; j < phd->MAX_L; j++)
+	{
+	  HTONSI(phd->tb0[i][j]);
+	  HTONSI(phd->tb1[i][j]);
+	  HTONSI(phd->tb2[i][j]);
+	}
+
+      tot += fwrite ((char *) phd->tb0[i], sizeof (int), phd->MAX_L, f) *
+        sizeof (int);
+      tot += fwrite ((char *) phd->tb1[i], sizeof (int), phd->MAX_L, f) *
+        sizeof (int);
+      tot += fwrite ((char *) phd->tb2[i], sizeof (int), phd->MAX_L, f) *
+        sizeof (int);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (j = 0; j < phd->MAX_L; j++)
+	{
+	  NTOHSI(phd->tb0[i][j]);
+	  NTOHSI(phd->tb1[i][j]);
+	  NTOHSI(phd->tb2[i][j]);
+	}
+    }
+#else
+  for (i = 0; i < phd->MAX_CH; i++)
+    {
+      /* [RPAP - Jan 97: Endian Ordering] */
+      int j;
+      for (j = 0; j < phd->MAX_L; j++)
+	{
+	  HTONSL(phd->tb[i][j].tb0);
+	  HTONSL(phd->tb[i][j].tb1);
+	  HTONSL(phd->tb[i][j].tb2);
+	}
+
+      tot += fwrite ((char *) phd->tb[i], sizeof (struct tb_entry), phd->MAX_L, f) *
+	sizeof (struct tb_entry);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (j = 0; j < phd->MAX_L; j++)
+	{
+	  NTOHSL(phd->tb[i][j].tb0);
+	  NTOHSL(phd->tb[i][j].tb1);
+	  NTOHSL(phd->tb[i][j].tb2);
+	}
+    }
+#endif
+  return (tot);
+}
+
+
+perf_hash_data *
+read_perf_hash_data (FILE * f)
+{
+  perf_hash_data *phd;
+  int i, tot, ok = 1;
+  if (!(phd = Xmalloc (sizeof (perf_hash_data))))
+    return (NULL);
+  tot = fread ((char *) &phd->MAX_L, sizeof (int), 1, f) * sizeof (int);
+  tot += fread ((char *) &phd->MAX_N, sizeof (int), 1, f) * sizeof (int);
+  tot += fread ((char *) &phd->MAX_M, sizeof (int), 1, f) * sizeof (int);
+  tot += fread ((char *) &phd->MAX_CH, sizeof (int), 1, f) * sizeof (int);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHSI(phd->MAX_L);
+  NTOHSI(phd->MAX_N);
+  NTOHSI(phd->MAX_M);
+  NTOHSI(phd->MAX_CH);
+
+  if (tot != 4 * sizeof (int))
+      return (NULL);
+  phd->translate = Xmalloc (sizeof (u_char) * 256);
+  phd->g = Xmalloc (sizeof (int) * (phd->MAX_N + 1));
+#ifndef STRUCT
+  phd->tb0 = Xmalloc (sizeof (int *) * phd->MAX_CH);
+  phd->tb1 = Xmalloc (sizeof (int *) * phd->MAX_CH);
+  phd->tb2 = Xmalloc (sizeof (int *) * phd->MAX_CH);
+  for (i = 0; i < phd->MAX_CH; i++)
+    {
+      if (phd->tb0)
+	if (!(phd->tb0[i] = Xmalloc (sizeof (long) * phd->MAX_L)))
+	    ok = 0;
+      if (phd->tb1)
+	if (!(phd->tb1[i] = Xmalloc (sizeof (long) * phd->MAX_L)))
+	    ok = 0;
+      if (phd->tb2)
+	if (!(phd->tb2[i] = Xmalloc (sizeof (long) * phd->MAX_L)))
+	    ok = 0;
+    }
+  if (!(phd->translate && phd->g && phd->tb0 && phd->tb1 && phd->tb2 && ok))
+    {
+      if (phd->translate)
+	Xfree (phd->translate);
+      if (phd->g)
+	Xfree (phd->g);
+      for (i = 0; i < MAX_CH; i++)
+	{
+	  if (phd->tb0 && phd->tb0[i])
+	    Xfree (phd->tb0[i]);
+	  if (phd->tb1 && phd->tb1[i])
+	    Xfree (phd->tb1[i]);
+	  if (phd->tb2 && phd->tb2[i])
+	    Xfree (phd->tb2[i]);
+	}
+      if (phd->tb0)
+	Xfree (phd->tb0);
+      if (phd->tb1)
+	Xfree (phd->tb1);
+      if (phd->tb2)
+	Xfree (phd->tb2);
+      Xfree (phd);
+      return (NULL);
+    }
+  tot += fread ((char *) phd->translate, sizeof (u_char), 256, f);
+  tot += fread ((char *) phd->g, sizeof (int), phd->MAX_N + 1, f) * sizeof (int);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = 0; i < phd->MAX_N + 1; i++)
+    NTOHSI(phd->g[i]);
+
+  for (i = 0; i < phd->MAX_CH; i++)
+    {
+      int j;
+
+      tot += fread ((char *) phd->tb0[i], sizeof (long), phd->MAX_L, f) *
+        sizeof (int);
+      tot += fread ((char *) phd->tb1[i], sizeof (long), phd->MAX_L, f) *
+        sizeof (int);
+      tot += fread ((char *) phd->tb2[i], sizeof (long), phd->MAX_L, f) *
+        sizeof (int);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (j = 0; j < phd->MAX_L; j++)
+	{
+	  NTOHSI(phd->tb0[i][j]);
+	  NTOHSI(phd->tb1[i][j]);
+	  NTOHSI(phd->tb2[i][j]);
+	}
+    }
+#else
+  phd->tb = Xmalloc (sizeof (struct tb_entry *) * phd->MAX_CH);
+  for (i = 0; i < phd->MAX_CH; i++)
+    if (phd->tb)
+      if (!(phd->tb[i] = Xmalloc (sizeof (struct tb_entry) * phd->MAX_L)))
+	  ok = 0;
+  if (!(phd->translate && phd->g && phd->tb && ok))
+    {
+      if (phd->translate)
+	Xfree (phd->translate);
+      if (phd->g)
+	Xfree (phd->g);
+      for (i = 0; i < MAX_CH; i++)
+	if (phd->tb && phd->tb[i])
+	  Xfree (phd->tb[i]);
+      if (phd->tb)
+	Xfree (phd->tb);
+      Xfree (phd);
+      return (NULL);
+    }
+  tot += fread ((char *) phd->translate, sizeof (u_char), 256, f);
+  tot += fread ((char *) phd->g, sizeof (int), phd->MAX_N + 1, f) * sizeof (int);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = 0; i < phd->MAX_N + 1; i++)
+    NTOHSI(phd->g[i]);
+
+  for (i = 0; i < phd->MAX_CH; i++)
+    {
+      int j;
+
+      tot += fread ((char *) phd->tb[i], sizeof (struct tb_entry), phd->MAX_L, f) *
+	sizeof (struct tb_entry);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (j = 0; j < phd->MAX_L; j++)
+	{
+	  NTOHSL(phd->tb[i][j].tb0);
+	  NTOHSL(phd->tb[i][j].tb1);
+	  NTOHSL(phd->tb[i][j].tb2);
+	}
+    }
+#endif
+  return (phd);
+}
+
+void 
+free_perf_hash (perf_hash_data * phd)
+{
+  int i;
+  if (phd->translate)
+    Xfree (phd->translate);
+  if (phd->g)
+    Xfree (phd->g);
+  for (i = 0; i < phd->MAX_CH; i++)
+    if (phd->tb && phd->tb[i])
+      Xfree (phd->tb[i]);
+  if (phd->tb)
+    Xfree (phd->tb);
+  Xfree (phd);
+}
+
+
+int 
+perf_hash (perf_hash_data * phd, u_char * s)
+{
+  int u, v, w, i, l;
+  l = *s++;
+  i = (l - 1) % phd->MAX_L;
+  u = v = w = 0;
+  while (l)
+    {
+#ifndef STRUCT
+      if ((u += phd->tb0[phd->translate[*s]][i]) >= phd->MAX_N)
+	u -= phd->MAX_N;
+      if ((v += phd->tb1[phd->translate[*s]][i]) >= phd->MAX_N)
+	v -= phd->MAX_N;
+      if ((w += phd->tb2[phd->translate[*s]][i]) >= phd->MAX_N)
+	w -= phd->MAX_N;
+#else
+      struct tb_entry *tbp = &phd->tb[phd->translate[*s]][i];
+      if ((u += tbp->tb0) >= phd->MAX_N)
+	u -= phd->MAX_N;
+      if ((v += tbp->tb1) >= phd->MAX_N)
+	v -= phd->MAX_N;
+      if ((w += tbp->tb2) >= phd->MAX_N)
+	w -= phd->MAX_N;
+#endif
+      if (++i >= phd->MAX_L)
+	i = 0;
+      l--;
+      s++;
+    }
+  if (u == v && ++v >= phd->MAX_N)
+    v = 0;
+  if (w == v)
+    {
+      if (++w >= phd->MAX_N)
+	w = 0;
+      if (w == u && ++w >= phd->MAX_N)
+	w = 0;
+    }
+  else if (w == u)
+    {
+      if (++w >= phd->MAX_N)
+	w = 0;
+      if (w == v && ++w >= phd->MAX_N)
+	w = 0;
+    }
+  return ((phd->g[u] ^ phd->g[v] ^ phd->g[w]) > phd->MAX_M) ? 0 :
+    (phd->g[u] ^ phd->g[v] ^ phd->g[w]);
+}
Index: /tags/gsdl-21/gsdl/packages/mg/lib/perf_hash.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/perf_hash.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/perf_hash.h	(revision 699)
@@ -0,0 +1,57 @@
+/**************************************************************************
+ *
+ * perf_hash.h -- Perfect hashing functions
+ * Copyright (C) 1994  Bohdan S. Majewski and Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef H_PERF_HASH
+#define H_PERF_HASH
+
+
+
+
+struct tb_entry
+  {
+    long tb0, tb1, tb2;
+  };
+
+typedef struct
+  {
+    int MAX_L;
+    int MAX_N;
+    int MAX_M;
+    int MAX_CH;
+    u_char *translate;
+    int *g;
+    struct tb_entry **tb;
+  }
+perf_hash_data;
+
+perf_hash_data *gen_hash_func (int num, u_char ** keys, int r);
+
+int perf_hash (perf_hash_data * phd, u_char * s);
+
+int write_perf_hash_data (FILE * f, perf_hash_data * phd);
+
+void free_perf_hash (perf_hash_data * phd);
+
+perf_hash_data *read_perf_hash_data (FILE * f);
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/lib/random.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/random.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/random.c	(revision 699)
@@ -0,0 +1,225 @@
+/**************************************************************************
+ *
+ * random.c -- pseudo random number generator
+ * Copyright (C) 1994  Chris Wallace (csw@bruce.cs.monash.edu.au)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+$Log$
+Revision 1.1  1999/08/10 21:16:57  sjboddie
+renamed mg-1.3d directory mg
+
+Revision 1.1  1998/11/17 09:32:20  rjmcnab
+*** empty log message ***
+
+*/
+
+static char *RCSID = "$Id$";
+
+/*
+ *	A random number generator called as a function by
+ *	random (iseed)	or	irandm (iseed)
+ *	The parameter should be a pointer to a 2-element long vector.
+ *	The first function returns a double uniform in 0 .. 1.
+ *	The second returns a long integer uniform in 0 .. 2**31-1
+ *	Both update iseed[] in exactly the same way.
+ *	iseed[] must be a 2-element integer vector.
+ *	The initial value of the second element may be anything.
+ *
+ *	The period of the random sequence is 2**32 * (2**32-1)
+ *	The table mt[0:127] is defined by mt[i] = 69069 ** (128-i)
+ */
+
+#define MASK ((long) 593970775)
+/*	or in hex, 23674657	*/
+
+#define SCALE ((double) 1.0 / (1024.0 * 1024.0 * 1024.0 * 2.0))
+/*	i.e. 2 to power -31	*/
+
+static long mt [128] =   {
+      902906369,
+     2030498053,
+     -473499623,
+     1640834941,
+      723406961,
+     1993558325,
+     -257162999,
+    -1627724755,
+      913952737,
+      278845029,
+     1327502073,
+    -1261253155,
+      981676113,
+    -1785280363,
+     1700077033,
+      366908557,
+    -1514479167,
+     -682799163,
+      141955545,
+     -830150595,
+      317871153,
+     1542036469,
+     -946413879,
+    -1950779155,
+      985397153,
+      626515237,
+      530871481,
+      783087261,
+    -1512358895,
+     1031357269,
+    -2007710807,
+    -1652747955,
+    -1867214463,
+      928251525,
+     1243003801,
+    -2132510467,
+     1874683889,
+     -717013323,
+      218254473,
+    -1628774995,
+    -2064896159,
+       69678053,
+      281568889,
+    -2104168611,
+     -165128239,
+     1536495125,
+      -39650967,
+      546594317,
+     -725987007,
+     1392966981,
+     1044706649,
+      687331773,
+    -2051306575,
+     1544302965,
+     -758494647,
+    -1243934099,
+      -75073759,
+      293132965,
+    -1935153095,
+      118929437,
+      807830417,
+    -1416222507,
+    -1550074071,
+      -84903219,
+     1355292929,
+     -380482555,
+    -1818444007,
+     -204797315,
+      170442609,
+    -1636797387,
+      868931593,
+     -623503571,
+     1711722209,
+      381210981,
+     -161547783,
+     -272740131,
+    -1450066095,
+     2116588437,
+     1100682473,
+      358442893,
+    -1529216831,
+     2116152005,
+     -776333095,
+     1265240893,
+     -482278607,
+     1067190005,
+      333444553,
+       86502381,
+      753481377,
+       39000101,
+     1779014585,
+      219658653,
+     -920253679,
+     2029538901,
+     1207761577,
+    -1515772851,
+     -236195711,
+      442620293,
+      423166617,
+    -1763648515,
+     -398436623,
+    -1749358155,
+     -538598519,
+     -652439379,
+      430550625,
+    -1481396507,
+     2093206905,
+    -1934691747,
+     -962631983,
+     1454463253,
+    -1877118871,
+     -291917555,
+    -1711673279,
+      201201733,
+     -474645415,
+      -96764739,
+    -1587365199,
+     1945705589,
+     1303896393,
+     1744831853,
+      381957665,
+     2135332261,
+      -55996615,
+    -1190135011,
+     1790562961,
+    -1493191723,
+      475559465,
+          69069
+		};
+
+double 
+random (long is [2])
+{
+	long it, leh, nit;
+
+	it = is [0];
+	leh = is [1];
+	if (it <= 0)	
+		it = (it + it) ^ MASK;
+	else
+		it = it + it;
+	nit = it - 1;
+/*	to ensure all-ones pattern omitted    */
+	leh = leh * mt[nit & 127] + nit;
+	is [0] = it;    is [1] = leh;
+	if (leh < 0) leh = ~leh;
+	return (SCALE * ((long) (leh | 1)));
+}
+
+
+
+long 
+irandm (long is [2])
+{
+	long it, leh, nit;
+
+	it = is [0];
+	leh = is [1];
+	if (it <= 0)	
+		it = (it + it) ^ MASK;
+	else
+		it = it + it;
+	nit = it - 1;
+/*	to ensure all-ones pattern omitted    */
+	leh = leh * mt[nit & 127] + nit;
+	is [0] = it;    is [1] = leh;
+	if (leh < 0) leh = ~leh;
+	return (leh);
+}
Index: /tags/gsdl-21/gsdl/packages/mg/lib/random.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/random.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/random.h	(revision 699)
@@ -0,0 +1,54 @@
+/**************************************************************************
+ *
+ * random.h -- pseudo random number generator
+ * Copyright (C) 1994  Chris Wallace (csw@bruce.cs.monash.edu.au)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+$Log$
+Revision 1.1  1999/08/10 21:16:58  sjboddie
+renamed mg-1.3d directory mg
+
+Revision 1.1  1998/11/17 09:32:21  rjmcnab
+*** empty log message ***
+
+*/
+
+#ifndef RANDOM_H
+#define RANDOM_H
+
+/*
+ *	A random number generator called as a function by
+ *	random (iseed)	or	irandm (iseed)
+ *	The parameter should be a pointer to a 2-element long vector.
+ *	The first function returns a double uniform in 0 .. 1.
+ *	The second returns a long integer uniform in 0 .. 2**31-1
+ *	Both update iseed[] in exactly the same way.
+ *	iseed[] must be a 2-element integer vector.
+ *	The initial value of the second element may be anything.
+ *
+ *	The period of the random sequence is 2**32 * (2**32-1)
+ *	The table mt[0:127] is defined by mt[i] = 69069 ** (128-i)
+ */
+
+double random (long is [2]);
+long irandm (long is [2]);
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/lib/regex.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/regex.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/regex.c	(revision 699)
@@ -0,0 +1,5426 @@
+/* Extended regular expression matching and search library,
+   version 0.12.
+   (Implements POSIX draft P10003.2/D11.2, except for
+   internationalization features.)
+
+   Copyright (C) 1993, 1994, 1995 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+/* AIX requires this to be the first thing in the file. */
+#if defined (_AIX) && !defined (REGEX_MALLOC)
+  #pragma alloca
+#endif
+
+#define _GNU_SOURCE
+
+#ifdef HAVE_CONFIG_H
+# ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+#  include <win32cfg.h>
+# else
+#  include <sysfuncs.h>
+# endif
+#endif
+
+/* We need this for `regex.h', and perhaps for the Emacs include files.  */
+#include <sys/types.h>
+
+/* This is for other GNU distributions with internationalized messages.  */
+#if HAVE_LIBINTL_H || defined (_LIBC)
+# include <libintl.h>
+#else
+# define gettext(msgid) (msgid)
+#endif
+
+/* The `emacs' switch turns on certain matching commands
+   that make sense only in Emacs. */
+#ifdef emacs
+
+#include "lisp.h"
+#include "buffer.h"
+#include "syntax.h"
+
+#else  /* not emacs */
+
+/* If we are not linking with Emacs proper,
+   we can't use the relocating allocator
+   even if sysfuncs.h says that we can.  */
+#undef REL_ALLOC
+
+#if defined (STDC_HEADERS) || defined (_LIBC)
+#include <stdlib.h>
+#else
+char *malloc ();
+char *realloc ();
+#endif
+
+/* We used to test for `BSTRING' here, but only GCC and Emacs define
+   `BSTRING', as far as I know, and neither of them use this code.  */
+#ifndef INHIBIT_STRING_HEADER
+#if HAVE_STRING_H || STDC_HEADERS || defined (_LIBC)
+#include <string.h>
+#ifndef bcmp
+#define bcmp(s1, s2, n)	memcmp ((s1), (s2), (n))
+#endif
+#ifndef bcopy
+#define bcopy(s, d, n)	memcpy ((d), (s), (n))
+#endif
+#ifndef bzero
+#define bzero(s, n)	memset ((s), 0, (n))
+#endif
+#else
+#include <strings.h>
+#endif
+#endif
+
+/* Define the syntax stuff for \<, \>, etc.  */
+
+/* This must be nonzero for the wordchar and notwordchar pattern
+   commands in re_match_2.  */
+#ifndef Sword 
+#define Sword 1
+#endif
+
+#ifdef SWITCH_ENUM_BUG
+#define SWITCH_ENUM_CAST(x) ((int)(x))
+#else
+#define SWITCH_ENUM_CAST(x) (x)
+#endif
+
+#ifdef SYNTAX_TABLE
+
+extern char *re_syntax_table;
+
+#else /* not SYNTAX_TABLE */
+
+/* How many characters in the character set.  */
+#define CHAR_SET_SIZE 256
+
+static char re_syntax_table[CHAR_SET_SIZE];
+
+static void
+init_syntax_once ()
+{
+   register int c;
+   static int done = 0;
+
+   if (done)
+     return;
+
+   bzero (re_syntax_table, sizeof re_syntax_table);
+
+   for (c = 'a'; c <= 'z'; c++)
+     re_syntax_table[c] = Sword;
+
+   for (c = 'A'; c <= 'Z'; c++)
+     re_syntax_table[c] = Sword;
+
+   for (c = '0'; c <= '9'; c++)
+     re_syntax_table[c] = Sword;
+
+   re_syntax_table['_'] = Sword;
+
+   done = 1;
+}
+
+#endif /* not SYNTAX_TABLE */
+
+#define SYNTAX(c) re_syntax_table[c]
+
+#endif /* not emacs */
+
+
+/* Get the interface, including the syntax bits.  */
+#include "regex.h"
+
+/* isalpha etc. are used for the character classes.  */
+#include <ctype.h>
+
+/* Jim Meyering writes:
+
+   "... Some ctype macros are valid only for character codes that
+   isascii says are ASCII (SGI's IRIX-4.0.5 is one such system --when
+   using /bin/cc or gcc but without giving an ansi option).  So, all
+   ctype uses should be through macros like ISPRINT...  If
+   STDC_HEADERS is defined, then autoconf has verified that the ctype
+   macros don't need to be guarded with references to isascii. ...
+   Defining isascii to 1 should let any compiler worth its salt
+   eliminate the && through constant folding."  */
+
+#ifndef ISASCII
+#  if defined (STDC_HEADERS) || (!defined (isascii) && !defined (HAVE_ISASCII))
+#    define ISASCII(c) 1
+#  else
+#    define ISASCII(c) isascii(c)
+#  endif
+#endif
+
+#ifdef isblank
+#define ISBLANK(c) (ISASCII (c) && isblank (c))
+#else
+#define ISBLANK(c) ((c) == ' ' || (c) == '\t')
+#endif
+#ifdef isgraph
+#define ISGRAPH(c) (ISASCII (c) && isgraph (c))
+#else
+#define ISGRAPH(c) (ISASCII (c) && isprint (c) && !isspace (c))
+#endif
+
+#define ISPRINT(c) (ISASCII (c) && isprint (c))
+#define ISDIGIT(c) (ISASCII (c) && isdigit (c))
+#define ISALNUM(c) (ISASCII (c) && isalnum (c))
+#define ISALPHA(c) (ISASCII (c) && isalpha (c))
+#define ISCNTRL(c) (ISASCII (c) && iscntrl (c))
+#define ISLOWER(c) (ISASCII (c) && islower (c))
+#define ISPUNCT(c) (ISASCII (c) && ispunct (c))
+#define ISSPACE(c) (ISASCII (c) && isspace (c))
+#define ISUPPER(c) (ISASCII (c) && isupper (c))
+#define ISXDIGIT(c) (ISASCII (c) && isxdigit (c))
+
+#ifndef NULL
+#define NULL 0
+#endif
+
+/* We remove any previous definition of `SIGN_EXTEND_CHAR',
+   since ours (we hope) works properly with all combinations of
+   machines, compilers, `char' and `unsigned char' argument types.
+   (Per Bothner suggested the basic approach.)  */
+#undef SIGN_EXTEND_CHAR
+#if __STDC__
+#define SIGN_EXTEND_CHAR(c) ((signed char) (c))
+#else  /* not __STDC__ */
+/* As in Harbison and Steele.  */
+#define SIGN_EXTEND_CHAR(c) ((((unsigned char) (c)) ^ 128) - 128)
+#endif
+
+
+/* Should we use malloc or alloca?  If REGEX_MALLOC is not defined, we
+   use `alloca' instead of `malloc'.  This is because using malloc in
+   re_search* or re_match* could cause memory leaks when C-g is used in
+   Emacs; also, malloc is slower and causes storage fragmentation.  On
+   the other hand, malloc is more portable, and easier to debug.  
+   
+   Because we sometimes use alloca, some routines have to be macros,
+   not functions -- `alloca'-allocated space disappears at the end of the
+   function it is called in.  */
+
+#ifdef REGEX_MALLOC
+
+#define REGEX_ALLOCATE malloc
+#define REGEX_REALLOCATE(source, osize, nsize) realloc (source, nsize)
+#define REGEX_FREE free
+
+#else /* not REGEX_MALLOC  */
+
+/* Emacs already defines alloca, sometimes.  */
+#ifndef alloca
+
+/* Make alloca work the best possible way.  */
+#ifdef __GNUC__
+#define alloca __builtin_alloca
+#else /* not __GNUC__ */
+#if HAVE_ALLOCA_H
+#include <alloca.h>
+#else /* not __GNUC__ or HAVE_ALLOCA_H */
+#ifndef _AIX /* Already did AIX, up at the top.  */
+char *alloca ();
+#endif /* not _AIX */
+#endif /* not HAVE_ALLOCA_H */ 
+#endif /* not __GNUC__ */
+
+#endif /* not alloca */
+
+#define REGEX_ALLOCATE alloca
+
+/* Assumes a `char *destination' variable.  */
+#define REGEX_REALLOCATE(source, osize, nsize)				\
+  (destination = (char *) alloca (nsize),				\
+   bcopy (source, destination, osize),					\
+   destination)
+
+/* No need to do anything to free, after alloca.  */
+#define REGEX_FREE(arg) ((void)0) /* Do nothing!  But inhibit gcc warning.  */
+
+#endif /* not REGEX_MALLOC */
+
+/* Define how to allocate the failure stack.  */
+
+#ifdef REL_ALLOC
+#define REGEX_ALLOCATE_STACK(size)				\
+  r_alloc (&failure_stack_ptr, (size))
+#define REGEX_REALLOCATE_STACK(source, osize, nsize)		\
+  r_re_alloc (&failure_stack_ptr, (nsize))
+#define REGEX_FREE_STACK(ptr)					\
+  r_alloc_free (&failure_stack_ptr)
+
+#else /* not REL_ALLOC */
+
+#ifdef REGEX_MALLOC
+
+#define REGEX_ALLOCATE_STACK malloc
+#define REGEX_REALLOCATE_STACK(source, osize, nsize) realloc (source, nsize)
+#define REGEX_FREE_STACK free
+
+#else /* not REGEX_MALLOC */
+
+#define REGEX_ALLOCATE_STACK alloca
+
+#define REGEX_REALLOCATE_STACK(source, osize, nsize)			\
+   REGEX_REALLOCATE (source, osize, nsize)
+/* No need to explicitly free anything.  */
+#define REGEX_FREE_STACK(arg)
+
+#endif /* not REGEX_MALLOC */
+#endif /* not REL_ALLOC */
+
+
+/* True if `size1' is non-NULL and PTR is pointing anywhere inside
+   `string1' or just past its end.  This works if PTR is NULL, which is
+   a good thing.  */
+#define FIRST_STRING_P(ptr) 					\
+  (size1 && string1 <= (ptr) && (ptr) <= string1 + size1)
+
+/* (Re)Allocate N items of type T using malloc, or fail.  */
+#define TALLOC(n, t) ((t *) malloc ((n) * sizeof (t)))
+#define RETALLOC(addr, n, t) ((addr) = (t *) realloc (addr, (n) * sizeof (t)))
+#define RETALLOC_IF(addr, n, t) \
+  if (addr) RETALLOC((addr), (n), t); else (addr) = TALLOC ((n), t)
+#define REGEX_TALLOC(n, t) ((t *) REGEX_ALLOCATE ((n) * sizeof (t)))
+
+#define BYTEWIDTH 8 /* In bits.  */
+
+#define STREQ(s1, s2) ((strcmp (s1, s2) == 0))
+
+#undef MAX
+#undef MIN
+#define MAX(a, b) ((a) > (b) ? (a) : (b))
+#define MIN(a, b) ((a) < (b) ? (a) : (b))
+
+typedef char boolean;
+#define false 0
+#define true 1
+
+static int re_match_2_internal ();
+
+
+/* These are the command codes that appear in compiled regular
+   expressions.  Some opcodes are followed by argument bytes.  A
+   command code can specify any interpretation whatsoever for its
+   arguments.  Zero bytes may appear in the compiled regular expression.  */
+
+typedef enum
+{
+  no_op = 0,
+
+  /* Succeed right away--no more backtracking.  */
+  succeed,
+
+        /* Followed by one byte giving n, then by n literal bytes.  */
+  exactn,
+
+        /* Matches any (more or less) character.  */
+  anychar,
+
+        /* Matches any one char belonging to specified set.  First
+           following byte is number of bitmap bytes.  Then come bytes
+           for a bitmap saying which chars are in.  Bits in each byte
+           are ordered low-bit-first.  A character is in the set if its
+           bit is 1.  A character too large to have a bit in the map is
+           automatically not in the set.  */
+  charset,
+
+        /* Same parameters as charset, but match any character that is
+           not one of those specified.  */
+  charset_not,
+
+        /* Start remembering the text that is matched, for storing in a
+           register.  Followed by one byte with the register number, in
+           the range 0 to one less than the pattern buffer's re_nsub
+           field.  Then followed by one byte with the number of groups
+           inner to this one.  (This last has to be part of the
+           start_memory only because we need it in the on_failure_jump
+           of re_match_2.)  */
+  start_memory,
+
+        /* Stop remembering the text that is matched and store it in a
+           memory register.  Followed by one byte with the register
+           number, in the range 0 to one less than `re_nsub' in the
+           pattern buffer, and one byte with the number of inner groups,
+           just like `start_memory'.  (We need the number of inner
+           groups here because we don't have any easy way of finding the
+           corresponding start_memory when we're at a stop_memory.)  */
+  stop_memory,
+
+        /* Match a duplicate of something remembered. Followed by one
+           byte containing the register number.  */
+  duplicate,
+
+        /* Fail unless at beginning of line.  */
+  begline,
+
+        /* Fail unless at end of line.  */
+  endline,
+
+        /* Succeeds if at beginning of buffer (if emacs) or at beginning
+           of string to be matched (if not).  */
+  begbuf,
+
+        /* Analogously, for end of buffer/string.  */
+  endbuf,
+ 
+        /* Followed by two byte relative address to which to jump.  */
+  jump, 
+
+	/* Same as jump, but marks the end of an alternative.  */
+  jump_past_alt,
+
+        /* Followed by two-byte relative address of place to resume at
+           in case of failure.  */
+  on_failure_jump,
+	
+        /* Like on_failure_jump, but pushes a placeholder instead of the
+           current string position when executed.  */
+  on_failure_keep_string_jump,
+  
+        /* Throw away latest failure point and then jump to following
+           two-byte relative address.  */
+  pop_failure_jump,
+
+        /* Change to pop_failure_jump if know won't have to backtrack to
+           match; otherwise change to jump.  This is used to jump
+           back to the beginning of a repeat.  If what follows this jump
+           clearly won't match what the repeat does, such that we can be
+           sure that there is no use backtracking out of repetitions
+           already matched, then we change it to a pop_failure_jump.
+           Followed by two-byte address.  */
+  maybe_pop_jump,
+
+        /* Jump to following two-byte address, and push a dummy failure
+           point. This failure point will be thrown away if an attempt
+           is made to use it for a failure.  A `+' construct makes this
+           before the first repeat.  Also used as an intermediary kind
+           of jump when compiling an alternative.  */
+  dummy_failure_jump,
+
+	/* Push a dummy failure point and continue.  Used at the end of
+	   alternatives.  */
+  push_dummy_failure,
+
+        /* Followed by two-byte relative address and two-byte number n.
+           After matching N times, jump to the address upon failure.  */
+  succeed_n,
+
+        /* Followed by two-byte relative address, and two-byte number n.
+           Jump to the address N times, then fail.  */
+  jump_n,
+
+        /* Set the following two-byte relative address to the
+           subsequent two-byte number.  The address *includes* the two
+           bytes of number.  */
+  set_number_at,
+
+  wordchar,	/* Matches any word-constituent character.  */
+  notwordchar,	/* Matches any char that is not a word-constituent.  */
+
+  wordbeg,	/* Succeeds if at word beginning.  */
+  wordend,	/* Succeeds if at word end.  */
+
+  wordbound,	/* Succeeds if at a word boundary.  */
+  notwordbound	/* Succeeds if not at a word boundary.  */
+
+#ifdef emacs
+  ,before_dot,	/* Succeeds if before point.  */
+  at_dot,	/* Succeeds if at point.  */
+  after_dot,	/* Succeeds if after point.  */
+
+	/* Matches any character whose syntax is specified.  Followed by
+           a byte which contains a syntax code, e.g., Sword.  */
+  syntaxspec,
+
+	/* Matches any character whose syntax is not that specified.  */
+  notsyntaxspec
+#endif /* emacs */
+} re_opcode_t;
+
+
+/* Common operations on the compiled pattern.  */
+
+/* Store NUMBER in two contiguous bytes starting at DESTINATION.  */
+
+#define STORE_NUMBER(destination, number)				\
+  do {									\
+    (destination)[0] = (number) & 0377;					\
+    (destination)[1] = (number) >> 8;					\
+  } while (0)
+
+/* Same as STORE_NUMBER, except increment DESTINATION to
+   the byte after where the number is stored.  Therefore, DESTINATION
+   must be an lvalue.  */
+
+#define STORE_NUMBER_AND_INCR(destination, number)			\
+  do {									\
+    STORE_NUMBER (destination, number);					\
+    (destination) += 2;							\
+  } while (0)
+
+/* Put into DESTINATION a number stored in two contiguous bytes starting
+   at SOURCE.  */
+
+#define EXTRACT_NUMBER(destination, source)				\
+  do {									\
+    (destination) = *(source) & 0377;					\
+    (destination) += SIGN_EXTEND_CHAR (*((source) + 1)) << 8;		\
+  } while (0)
+
+#ifdef DEBUG
+static void
+extract_number (dest, source)
+    int *dest;
+    unsigned char *source;
+{
+  int temp = SIGN_EXTEND_CHAR (*(source + 1)); 
+  *dest = *source & 0377;
+  *dest += temp << 8;
+}
+
+#ifndef EXTRACT_MACROS /* To debug the macros.  */
+#undef EXTRACT_NUMBER
+#define EXTRACT_NUMBER(dest, src) extract_number (&dest, src)
+#endif /* not EXTRACT_MACROS */
+
+#endif /* DEBUG */
+
+/* Same as EXTRACT_NUMBER, except increment SOURCE to after the number.
+   SOURCE must be an lvalue.  */
+
+#define EXTRACT_NUMBER_AND_INCR(destination, source)			\
+  do {									\
+    EXTRACT_NUMBER (destination, source);				\
+    (source) += 2; 							\
+  } while (0)
+
+#ifdef DEBUG
+static void
+extract_number_and_incr (destination, source)
+    int *destination;
+    unsigned char **source;
+{ 
+  extract_number (destination, *source);
+  *source += 2;
+}
+
+#ifndef EXTRACT_MACROS
+#undef EXTRACT_NUMBER_AND_INCR
+#define EXTRACT_NUMBER_AND_INCR(dest, src) \
+  extract_number_and_incr (&dest, &src)
+#endif /* not EXTRACT_MACROS */
+
+#endif /* DEBUG */
+
+
+/* If DEBUG is defined, Regex prints many voluminous messages about what
+   it is doing (if the variable `debug' is nonzero).  If linked with the
+   main program in `iregex.c', you can enter patterns and strings
+   interactively.  And if linked with the main program in `main.c' and
+   the other test files, you can run the already-written tests.  */
+
+#ifdef DEBUG
+
+/* We use standard I/O for debugging.  */
+#include <stdio.h>
+
+/* It is useful to test things that ``must'' be true when debugging.  */
+#include <assert.h>
+
+static int debug = 0;
+
+#define DEBUG_STATEMENT(e) e
+#define DEBUG_PRINT1(x) if (debug) printf (x)
+#define DEBUG_PRINT2(x1, x2) if (debug) printf (x1, x2)
+#define DEBUG_PRINT3(x1, x2, x3) if (debug) printf (x1, x2, x3)
+#define DEBUG_PRINT4(x1, x2, x3, x4) if (debug) printf (x1, x2, x3, x4)
+#define DEBUG_PRINT_COMPILED_PATTERN(p, s, e) 				\
+  if (debug) print_partial_compiled_pattern (s, e)
+#define DEBUG_PRINT_DOUBLE_STRING(w, s1, sz1, s2, sz2)			\
+  if (debug) print_double_string (w, s1, sz1, s2, sz2)
+
+
+/* Print the fastmap in human-readable form.  */
+
+void
+print_fastmap (fastmap)
+    char *fastmap;
+{
+  unsigned was_a_range = 0;
+  unsigned i = 0;  
+  
+  while (i < (1 << BYTEWIDTH))
+    {
+      if (fastmap[i++])
+	{
+	  was_a_range = 0;
+          putchar (i - 1);
+          while (i < (1 << BYTEWIDTH)  &&  fastmap[i])
+            {
+              was_a_range = 1;
+              i++;
+            }
+	  if (was_a_range)
+            {
+              printf ("-");
+              putchar (i - 1);
+            }
+        }
+    }
+  putchar ('\n'); 
+}
+
+
+/* Print a compiled pattern string in human-readable form, starting at
+   the START pointer into it and ending just before the pointer END.  */
+
+void
+print_partial_compiled_pattern (start, end)
+    unsigned char *start;
+    unsigned char *end;
+{
+  int mcnt, mcnt2;
+  unsigned char *p = start;
+  unsigned char *pend = end;
+
+  if (start == NULL)
+    {
+      printf ("(null)\n");
+      return;
+    }
+    
+  /* Loop over pattern commands.  */
+  while (p < pend)
+    {
+      printf ("%d:\t", p - start);
+
+      switch ((re_opcode_t) *p++)
+	{
+        case no_op:
+          printf ("/no_op");
+          break;
+
+	case exactn:
+	  mcnt = *p++;
+          printf ("/exactn/%d", mcnt);
+          do
+	    {
+              putchar ('/');
+	      putchar (*p++);
+            }
+          while (--mcnt);
+          break;
+
+	case start_memory:
+          mcnt = *p++;
+          printf ("/start_memory/%d/%d", mcnt, *p++);
+          break;
+
+	case stop_memory:
+          mcnt = *p++;
+	  printf ("/stop_memory/%d/%d", mcnt, *p++);
+          break;
+
+	case duplicate:
+	  printf ("/duplicate/%d", *p++);
+	  break;
+
+	case anychar:
+	  printf ("/anychar");
+	  break;
+
+	case charset:
+        case charset_not:
+          {
+            register int c, last = -100;
+	    register int in_range = 0;
+
+	    printf ("/charset [%s",
+	            (re_opcode_t) *(p - 1) == charset_not ? "^" : "");
+            
+            assert (p + *p < pend);
+
+            for (c = 0; c < 256; c++)
+	      if (c / 8 < *p
+		  && (p[1 + (c/8)] & (1 << (c % 8))))
+		{
+		  /* Are we starting a range?  */
+		  if (last + 1 == c && ! in_range)
+		    {
+		      putchar ('-');
+		      in_range = 1;
+		    }
+		  /* Have we broken a range?  */
+		  else if (last + 1 != c && in_range)
+              {
+		      putchar (last);
+		      in_range = 0;
+		    }
+                
+		  if (! in_range)
+		    putchar (c);
+
+		  last = c;
+              }
+
+	    if (in_range)
+	      putchar (last);
+
+	    putchar (']');
+
+	    p += 1 + *p;
+	  }
+	  break;
+
+	case begline:
+	  printf ("/begline");
+          break;
+
+	case endline:
+          printf ("/endline");
+          break;
+
+	case on_failure_jump:
+          extract_number_and_incr (&mcnt, &p);
+  	  printf ("/on_failure_jump to %d", p + mcnt - start);
+          break;
+
+	case on_failure_keep_string_jump:
+          extract_number_and_incr (&mcnt, &p);
+  	  printf ("/on_failure_keep_string_jump to %d", p + mcnt - start);
+          break;
+
+	case dummy_failure_jump:
+          extract_number_and_incr (&mcnt, &p);
+  	  printf ("/dummy_failure_jump to %d", p + mcnt - start);
+          break;
+
+	case push_dummy_failure:
+          printf ("/push_dummy_failure");
+          break;
+          
+        case maybe_pop_jump:
+          extract_number_and_incr (&mcnt, &p);
+  	  printf ("/maybe_pop_jump to %d", p + mcnt - start);
+	  break;
+
+        case pop_failure_jump:
+	  extract_number_and_incr (&mcnt, &p);
+  	  printf ("/pop_failure_jump to %d", p + mcnt - start);
+	  break;          
+          
+        case jump_past_alt:
+	  extract_number_and_incr (&mcnt, &p);
+  	  printf ("/jump_past_alt to %d", p + mcnt - start);
+	  break;          
+          
+        case jump:
+	  extract_number_and_incr (&mcnt, &p);
+  	  printf ("/jump to %d", p + mcnt - start);
+	  break;
+
+        case succeed_n: 
+          extract_number_and_incr (&mcnt, &p);
+          extract_number_and_incr (&mcnt2, &p);
+	  printf ("/succeed_n to %d, %d times", p + mcnt - start, mcnt2);
+          break;
+        
+        case jump_n: 
+          extract_number_and_incr (&mcnt, &p);
+          extract_number_and_incr (&mcnt2, &p);
+	  printf ("/jump_n to %d, %d times", p + mcnt - start, mcnt2);
+          break;
+        
+        case set_number_at: 
+          extract_number_and_incr (&mcnt, &p);
+          extract_number_and_incr (&mcnt2, &p);
+	  printf ("/set_number_at location %d to %d", p + mcnt - start, mcnt2);
+          break;
+        
+        case wordbound:
+	  printf ("/wordbound");
+	  break;
+
+	case notwordbound:
+	  printf ("/notwordbound");
+          break;
+
+	case wordbeg:
+	  printf ("/wordbeg");
+	  break;
+          
+	case wordend:
+	  printf ("/wordend");
+          
+#ifdef emacs
+	case before_dot:
+	  printf ("/before_dot");
+          break;
+
+	case at_dot:
+	  printf ("/at_dot");
+          break;
+
+	case after_dot:
+	  printf ("/after_dot");
+          break;
+
+	case syntaxspec:
+          printf ("/syntaxspec");
+	  mcnt = *p++;
+	  printf ("/%d", mcnt);
+          break;
+	  
+	case notsyntaxspec:
+          printf ("/notsyntaxspec");
+	  mcnt = *p++;
+	  printf ("/%d", mcnt);
+	  break;
+#endif /* emacs */
+
+	case wordchar:
+	  printf ("/wordchar");
+          break;
+	  
+	case notwordchar:
+	  printf ("/notwordchar");
+          break;
+
+	case begbuf:
+	  printf ("/begbuf");
+          break;
+
+	case endbuf:
+	  printf ("/endbuf");
+          break;
+
+        default:
+          printf ("?%d", *(p-1));
+	}
+
+      putchar ('\n');
+    }
+
+  printf ("%d:\tend of pattern.\n", p - start);
+}
+
+
+void
+print_compiled_pattern (bufp)
+    struct re_pattern_buffer *bufp;
+{
+  unsigned char *buffer = bufp->buffer;
+
+  print_partial_compiled_pattern (buffer, buffer + bufp->used);
+  printf ("%d bytes used/%d bytes allocated.\n", bufp->used, bufp->allocated);
+
+  if (bufp->fastmap_accurate && bufp->fastmap)
+    {
+      printf ("fastmap: ");
+      print_fastmap (bufp->fastmap);
+    }
+
+  printf ("re_nsub: %d\t", bufp->re_nsub);
+  printf ("regs_alloc: %d\t", bufp->regs_allocated);
+  printf ("can_be_null: %d\t", bufp->can_be_null);
+  printf ("newline_anchor: %d\n", bufp->newline_anchor);
+  printf ("no_sub: %d\t", bufp->no_sub);
+  printf ("not_bol: %d\t", bufp->not_bol);
+  printf ("not_eol: %d\t", bufp->not_eol);
+  printf ("syntax: %d\n", bufp->syntax);
+  /* Perhaps we should print the translate table?  */
+}
+
+
+void
+print_double_string (where, string1, size1, string2, size2)
+    const char *where;
+    const char *string1;
+    const char *string2;
+    int size1;
+    int size2;
+{
+  unsigned this_char;
+  
+  if (where == NULL)
+    printf ("(null)");
+  else
+    {
+      if (FIRST_STRING_P (where))
+        {
+          for (this_char = where - string1; this_char < size1; this_char++)
+            putchar (string1[this_char]);
+
+          where = string2;    
+        }
+
+      for (this_char = where - string2; this_char < size2; this_char++)
+        putchar (string2[this_char]);
+    }
+}
+
+#else /* not DEBUG */
+
+#undef assert
+#define assert(e)
+
+#define DEBUG_STATEMENT(e)
+#define DEBUG_PRINT1(x)
+#define DEBUG_PRINT2(x1, x2)
+#define DEBUG_PRINT3(x1, x2, x3)
+#define DEBUG_PRINT4(x1, x2, x3, x4)
+#define DEBUG_PRINT_COMPILED_PATTERN(p, s, e)
+#define DEBUG_PRINT_DOUBLE_STRING(w, s1, sz1, s2, sz2)
+
+#endif /* not DEBUG */
+
+
+/* Set by `re_set_syntax' to the current regexp syntax to recognize.  Can
+   also be assigned to arbitrarily: each pattern buffer stores its own
+   syntax, so it can be changed between regex compilations.  */
+/* This has no initializer because initialized variables in Emacs
+   become read-only after dumping.  */
+reg_syntax_t re_syntax_options;
+
+
+/* Specify the precise syntax of regexps for compilation.  This provides
+   for compatibility for various utilities which historically have
+   different, incompatible syntaxes.
+
+   The argument SYNTAX is a bit mask comprised of the various bits
+   defined in regex.h.  We return the old syntax.  */
+
+reg_syntax_t
+re_set_syntax (syntax)
+    reg_syntax_t syntax;
+{
+  reg_syntax_t ret = re_syntax_options;
+  
+  re_syntax_options = syntax;
+  return ret;
+}
+
+
+/* This table gives an error message for each of the error codes listed
+   in regex.h.  Obviously the order here has to be same as there.
+   POSIX doesn't require that we do anything for REG_NOERROR,
+   but why not be nice?  */
+
+static const char *re_error_msgid[] =
+  { "Success",					/* REG_NOERROR */
+    "No match",					/* REG_NOMATCH */
+    "Invalid regular expression",		/* REG_BADPAT */
+    "Invalid collation character",		/* REG_ECOLLATE */
+    "Invalid character class name",		/* REG_ECTYPE */
+    "Trailing backslash",			/* REG_EESCAPE */
+    "Invalid back reference",			/* REG_ESUBREG */
+    "Unmatched [ or [^",			/* REG_EBRACK */
+    "Unmatched ( or \\(",			/* REG_EPAREN */
+    "Unmatched \\{",				/* REG_EBRACE */
+    "Invalid content of \\{\\}",		/* REG_BADBR */
+    "Invalid range end",			/* REG_ERANGE */
+    "Memory exhausted",				/* REG_ESPACE */
+    "Invalid preceding regular expression",	/* REG_BADRPT */
+    "Premature end of regular expression",	/* REG_EEND */
+    "Regular expression too big",		/* REG_ESIZE */
+    "Unmatched ) or \\)",			/* REG_ERPAREN */
+  };
+
+
+/* Avoiding alloca during matching, to placate r_alloc.  */
+
+/* Define MATCH_MAY_ALLOCATE unless we need to make sure that the
+   searching and matching functions should not call alloca.  On some
+   systems, alloca is implemented in terms of malloc, and if we're
+   using the relocating allocator routines, then malloc could cause a
+   relocation, which might (if the strings being searched are in the
+   ralloc heap) shift the data out from underneath the regexp
+   routines.
+
+   Here's another reason to avoid allocation: Emacs 
+   processes input from X in a signal handler; processing X input may
+   call malloc; if input arrives while a matching routine is calling
+   malloc, then we're scrod.  But Emacs can't just block input while
+   calling matching routines; then we don't notice interrupts when
+   they come in.  So, Emacs blocks input around all regexp calls
+   except the matching calls, which it leaves unprotected, in the
+   faith that they will not malloc.  */
+
+/* Normally, this is fine.  */
+#define MATCH_MAY_ALLOCATE
+
+/* When using GNU C, we are not REALLY using the C alloca, no matter
+   what sysfuncs.h may say.  So don't take precautions for it.  */
+#ifdef __GNUC__
+#undef C_ALLOCA
+#endif
+
+/* The match routines may not allocate if (1) they would do it with malloc
+   and (2) it's not safe for them to use malloc.
+   Note that if REL_ALLOC is defined, matching would not use malloc for the
+   failure stack, but we would still use it for the register vectors;
+   so REL_ALLOC should not affect this.  */
+#if (defined (C_ALLOCA) || defined (REGEX_MALLOC)) && defined (emacs)
+#undef MATCH_MAY_ALLOCATE
+#endif
+
+
+
+/* Failure stack declarations and macros; both re_compile_fastmap and
+   re_match_2 use a failure stack.  These have to be macros because of
+   REGEX_ALLOCATE_STACK.  */
+   
+
+/* Number of failure points for which to initially allocate space
+   when matching.  If this number is exceeded, we allocate more
+   space, so it is not a hard limit.  */
+#ifndef INIT_FAILURE_ALLOC
+#define INIT_FAILURE_ALLOC 5
+#endif
+
+/* Roughly the maximum number of failure points on the stack.  Would be
+   exactly that if always used MAX_FAILURE_SPACE each time we failed.
+   This is a variable only so users of regex can assign to it; we never
+   change it ourselves.  */
+#if defined (MATCH_MAY_ALLOCATE)
+int re_max_failures = 200000;
+#else
+int re_max_failures = 2000;
+#endif
+
+union fail_stack_elt
+{
+  unsigned char *pointer;
+  int integer;
+};
+
+typedef union fail_stack_elt fail_stack_elt_t;
+
+typedef struct
+{
+  fail_stack_elt_t *stack;
+  unsigned size;
+  unsigned avail;			/* Offset of next open position.  */
+} fail_stack_type;
+
+#define FAIL_STACK_EMPTY()     (fail_stack.avail == 0)
+#define FAIL_STACK_PTR_EMPTY() (fail_stack_ptr->avail == 0)
+#define FAIL_STACK_FULL()      (fail_stack.avail == fail_stack.size)
+
+
+/* Define macros to initialize and free the failure stack.
+   Do `return -2' if the alloc fails.  */
+
+#ifdef MATCH_MAY_ALLOCATE
+#define INIT_FAIL_STACK()						\
+  do {									\
+    fail_stack.stack = (fail_stack_elt_t *)				\
+      REGEX_ALLOCATE_STACK (INIT_FAILURE_ALLOC * sizeof (fail_stack_elt_t));	\
+									\
+    if (fail_stack.stack == NULL)					\
+      return -2;							\
+									\
+    fail_stack.size = INIT_FAILURE_ALLOC;				\
+    fail_stack.avail = 0;						\
+  } while (0)
+
+#define RESET_FAIL_STACK()  REGEX_FREE_STACK (fail_stack.stack)
+#else
+#define INIT_FAIL_STACK()						\
+  do {									\
+    fail_stack.avail = 0;						\
+  } while (0)
+
+#define RESET_FAIL_STACK()
+#endif
+
+
+/* Double the size of FAIL_STACK, up to approximately `re_max_failures' items.
+
+   Return 1 if succeeds, and 0 if either ran out of memory
+   allocating space for it or it was already too large.  
+   
+   REGEX_REALLOCATE_STACK requires `destination' be declared.   */
+
+#define DOUBLE_FAIL_STACK(fail_stack)					\
+  ((fail_stack).size > re_max_failures * MAX_FAILURE_ITEMS		\
+   ? 0									\
+   : ((fail_stack).stack = (fail_stack_elt_t *)				\
+        REGEX_REALLOCATE_STACK ((fail_stack).stack, 			\
+          (fail_stack).size * sizeof (fail_stack_elt_t),		\
+          ((fail_stack).size << 1) * sizeof (fail_stack_elt_t)),	\
+									\
+      (fail_stack).stack == NULL					\
+      ? 0								\
+      : ((fail_stack).size <<= 1, 					\
+         1)))
+
+
+/* Push pointer POINTER on FAIL_STACK. 
+   Return 1 if was able to do so and 0 if ran out of memory allocating
+   space to do so.  */
+#define PUSH_PATTERN_OP(POINTER, FAIL_STACK)				\
+  ((FAIL_STACK_FULL ()							\
+    && !DOUBLE_FAIL_STACK (FAIL_STACK))					\
+   ? 0									\
+   : ((FAIL_STACK).stack[(FAIL_STACK).avail++].pointer = POINTER,	\
+      1))
+
+/* Push a pointer value onto the failure stack.
+   Assumes the variable `fail_stack'.  Probably should only
+   be called from within `PUSH_FAILURE_POINT'.  */
+#define PUSH_FAILURE_POINTER(item)					\
+  fail_stack.stack[fail_stack.avail++].pointer = (unsigned char *) (item)
+
+/* This pushes an integer-valued item onto the failure stack.
+   Assumes the variable `fail_stack'.  Probably should only
+   be called from within `PUSH_FAILURE_POINT'.  */
+#define PUSH_FAILURE_INT(item)					\
+  fail_stack.stack[fail_stack.avail++].integer = (item)
+
+/* Push a fail_stack_elt_t value onto the failure stack.
+   Assumes the variable `fail_stack'.  Probably should only
+   be called from within `PUSH_FAILURE_POINT'.  */
+#define PUSH_FAILURE_ELT(item)					\
+  fail_stack.stack[fail_stack.avail++] =  (item)
+
+/* These three POP... operations complement the three PUSH... operations.
+   All assume that `fail_stack' is nonempty.  */
+#define POP_FAILURE_POINTER() fail_stack.stack[--fail_stack.avail].pointer
+#define POP_FAILURE_INT() fail_stack.stack[--fail_stack.avail].integer
+#define POP_FAILURE_ELT() fail_stack.stack[--fail_stack.avail]
+
+/* Used to omit pushing failure point id's when we're not debugging.  */
+#ifdef DEBUG
+#define DEBUG_PUSH PUSH_FAILURE_INT
+#define DEBUG_POP(item_addr) *(item_addr) = POP_FAILURE_INT ()
+#else
+#define DEBUG_PUSH(item)
+#define DEBUG_POP(item_addr)
+#endif
+
+
+/* Push the information about the state we will need
+   if we ever fail back to it.  
+   
+   Requires variables fail_stack, regstart, regend, reg_info, and
+   num_regs be declared.  DOUBLE_FAIL_STACK requires `destination' be
+   declared.
+   
+   Does `return FAILURE_CODE' if runs out of memory.  */
+
+#define PUSH_FAILURE_POINT(pattern_place, string_place, failure_code)	\
+  do {									\
+    char *destination;							\
+    /* Must be int, so when we don't save any registers, the arithmetic	\
+       of 0 + -1 isn't done as unsigned.  */				\
+    int this_reg;							\
+    									\
+    DEBUG_STATEMENT (failure_id++);					\
+    DEBUG_STATEMENT (nfailure_points_pushed++);				\
+    DEBUG_PRINT2 ("\nPUSH_FAILURE_POINT #%u:\n", failure_id);		\
+    DEBUG_PRINT2 ("  Before push, next avail: %d\n", (fail_stack).avail);\
+    DEBUG_PRINT2 ("                     size: %d\n", (fail_stack).size);\
+									\
+    DEBUG_PRINT2 ("  slots needed: %d\n", NUM_FAILURE_ITEMS);		\
+    DEBUG_PRINT2 ("     available: %d\n", REMAINING_AVAIL_SLOTS);	\
+									\
+    /* Ensure we have enough space allocated for what we will push.  */	\
+    while (REMAINING_AVAIL_SLOTS < NUM_FAILURE_ITEMS)			\
+      {									\
+        if (!DOUBLE_FAIL_STACK (fail_stack))				\
+          return failure_code;						\
+									\
+        DEBUG_PRINT2 ("\n  Doubled stack; size now: %d\n",		\
+		       (fail_stack).size);				\
+        DEBUG_PRINT2 ("  slots available: %d\n", REMAINING_AVAIL_SLOTS);\
+      }									\
+									\
+    /* Push the info, starting with the registers.  */			\
+    DEBUG_PRINT1 ("\n");						\
+									\
+    for (this_reg = lowest_active_reg; this_reg <= highest_active_reg;	\
+         this_reg++)							\
+      {									\
+	DEBUG_PRINT2 ("  Pushing reg: %d\n", this_reg);			\
+        DEBUG_STATEMENT (num_regs_pushed++);				\
+									\
+	DEBUG_PRINT2 ("    start: 0x%x\n", regstart[this_reg]);		\
+        PUSH_FAILURE_POINTER (regstart[this_reg]);			\
+                                                                        \
+	DEBUG_PRINT2 ("    end: 0x%x\n", regend[this_reg]);		\
+        PUSH_FAILURE_POINTER (regend[this_reg]);			\
+									\
+	DEBUG_PRINT2 ("    info: 0x%x\n      ", reg_info[this_reg]);	\
+        DEBUG_PRINT2 (" match_null=%d",					\
+                      REG_MATCH_NULL_STRING_P (reg_info[this_reg]));	\
+        DEBUG_PRINT2 (" active=%d", IS_ACTIVE (reg_info[this_reg]));	\
+        DEBUG_PRINT2 (" matched_something=%d",				\
+                      MATCHED_SOMETHING (reg_info[this_reg]));		\
+        DEBUG_PRINT2 (" ever_matched=%d",				\
+                      EVER_MATCHED_SOMETHING (reg_info[this_reg]));	\
+	DEBUG_PRINT1 ("\n");						\
+        PUSH_FAILURE_ELT (reg_info[this_reg].word);			\
+      }									\
+									\
+    DEBUG_PRINT2 ("  Pushing  low active reg: %d\n", lowest_active_reg);\
+    PUSH_FAILURE_INT (lowest_active_reg);				\
+									\
+    DEBUG_PRINT2 ("  Pushing high active reg: %d\n", highest_active_reg);\
+    PUSH_FAILURE_INT (highest_active_reg);				\
+									\
+    DEBUG_PRINT2 ("  Pushing pattern 0x%x: ", pattern_place);		\
+    DEBUG_PRINT_COMPILED_PATTERN (bufp, pattern_place, pend);		\
+    PUSH_FAILURE_POINTER (pattern_place);				\
+									\
+    DEBUG_PRINT2 ("  Pushing string 0x%x: `", string_place);		\
+    DEBUG_PRINT_DOUBLE_STRING (string_place, string1, size1, string2,   \
+				 size2);				\
+    DEBUG_PRINT1 ("'\n");						\
+    PUSH_FAILURE_POINTER (string_place);				\
+									\
+    DEBUG_PRINT2 ("  Pushing failure id: %u\n", failure_id);		\
+    DEBUG_PUSH (failure_id);						\
+  } while (0)
+
+/* This is the number of items that are pushed and popped on the stack
+   for each register.  */
+#define NUM_REG_ITEMS  3
+
+/* Individual items aside from the registers.  */
+#ifdef DEBUG
+#define NUM_NONREG_ITEMS 5 /* Includes failure point id.  */
+#else
+#define NUM_NONREG_ITEMS 4
+#endif
+
+/* We push at most this many items on the stack.  */
+#define MAX_FAILURE_ITEMS ((num_regs - 1) * NUM_REG_ITEMS + NUM_NONREG_ITEMS)
+
+/* We actually push this many items.  */
+#define NUM_FAILURE_ITEMS						\
+  ((highest_active_reg - lowest_active_reg + 1) * NUM_REG_ITEMS 	\
+    + NUM_NONREG_ITEMS)
+
+/* How many items can still be added to the stack without overflowing it.  */
+#define REMAINING_AVAIL_SLOTS ((fail_stack).size - (fail_stack).avail)
+
+
+/* Pops what PUSH_FAIL_STACK pushes.
+
+   We restore into the parameters, all of which should be lvalues:
+     STR -- the saved data position.
+     PAT -- the saved pattern position.
+     LOW_REG, HIGH_REG -- the highest and lowest active registers.
+     REGSTART, REGEND -- arrays of string positions.
+     REG_INFO -- array of information about each subexpression.
+   
+   Also assumes the variables `fail_stack' and (if debugging), `bufp',
+   `pend', `string1', `size1', `string2', and `size2'.  */
+
+#define POP_FAILURE_POINT(str, pat, low_reg, high_reg, regstart, regend, reg_info)\
+{									\
+  DEBUG_STATEMENT (fail_stack_elt_t failure_id;)			\
+  int this_reg;								\
+  const unsigned char *string_temp;					\
+									\
+  assert (!FAIL_STACK_EMPTY ());					\
+									\
+  /* Remove failure points and point to how many regs pushed.  */	\
+  DEBUG_PRINT1 ("POP_FAILURE_POINT:\n");				\
+  DEBUG_PRINT2 ("  Before pop, next avail: %d\n", fail_stack.avail);	\
+  DEBUG_PRINT2 ("                    size: %d\n", fail_stack.size);	\
+									\
+  assert (fail_stack.avail >= NUM_NONREG_ITEMS);			\
+									\
+  DEBUG_POP (&failure_id);						\
+  DEBUG_PRINT2 ("  Popping failure id: %u\n", failure_id);		\
+									\
+  /* If the saved string location is NULL, it came from an		\
+     on_failure_keep_string_jump opcode, and we want to throw away the	\
+     saved NULL, thus retaining our current position in the string.  */	\
+  string_temp = POP_FAILURE_POINTER ();					\
+  if (string_temp != NULL)						\
+    str = (const char *) string_temp;					\
+									\
+  DEBUG_PRINT2 ("  Popping string 0x%x: `", str);			\
+  DEBUG_PRINT_DOUBLE_STRING (str, string1, size1, string2, size2);	\
+  DEBUG_PRINT1 ("'\n");							\
+									\
+  pat = (unsigned char *) POP_FAILURE_POINTER ();			\
+  DEBUG_PRINT2 ("  Popping pattern 0x%x: ", pat);			\
+  DEBUG_PRINT_COMPILED_PATTERN (bufp, pat, pend);			\
+									\
+  /* Restore register info.  */						\
+  high_reg = (unsigned) POP_FAILURE_INT ();				\
+  DEBUG_PRINT2 ("  Popping high active reg: %d\n", high_reg);		\
+									\
+  low_reg = (unsigned) POP_FAILURE_INT ();				\
+  DEBUG_PRINT2 ("  Popping  low active reg: %d\n", low_reg);		\
+									\
+  for (this_reg = high_reg; this_reg >= low_reg; this_reg--)		\
+    {									\
+      DEBUG_PRINT2 ("    Popping reg: %d\n", this_reg);			\
+									\
+      reg_info[this_reg].word = POP_FAILURE_ELT ();			\
+      DEBUG_PRINT2 ("      info: 0x%x\n", reg_info[this_reg]);		\
+									\
+      regend[this_reg] = (const char *) POP_FAILURE_POINTER ();		\
+      DEBUG_PRINT2 ("      end: 0x%x\n", regend[this_reg]);		\
+									\
+      regstart[this_reg] = (const char *) POP_FAILURE_POINTER ();	\
+      DEBUG_PRINT2 ("      start: 0x%x\n", regstart[this_reg]);		\
+    }									\
+									\
+  set_regs_matched_done = 0;						\
+  DEBUG_STATEMENT (nfailure_points_popped++);				\
+} /* POP_FAILURE_POINT */
+
+
+
+
+/* Structure for per-register (a.k.a. per-group) information.
+   Other register information, such as the
+   starting and ending positions (which are addresses), and the list of
+   inner groups (which is a bits list) are maintained in separate
+   variables.  
+   
+   We are making a (strictly speaking) nonportable assumption here: that
+   the compiler will pack our bit fields into something that fits into
+   the type of `word', i.e., is something that fits into one item on the
+   failure stack.  */
+
+typedef union
+{
+  fail_stack_elt_t word;
+  struct
+  {
+      /* This field is one if this group can match the empty string,
+         zero if not.  If not yet determined,  `MATCH_NULL_UNSET_VALUE'.  */
+#define MATCH_NULL_UNSET_VALUE 3
+    unsigned match_null_string_p : 2;
+    unsigned is_active : 1;
+    unsigned matched_something : 1;
+    unsigned ever_matched_something : 1;
+  } bits;
+} register_info_type;
+
+#define REG_MATCH_NULL_STRING_P(R)  ((R).bits.match_null_string_p)
+#define IS_ACTIVE(R)  ((R).bits.is_active)
+#define MATCHED_SOMETHING(R)  ((R).bits.matched_something)
+#define EVER_MATCHED_SOMETHING(R)  ((R).bits.ever_matched_something)
+
+
+/* Call this when have matched a real character; it sets `matched' flags
+   for the subexpressions which we are currently inside.  Also records
+   that those subexprs have matched.  */
+#define SET_REGS_MATCHED()						\
+  do									\
+    {									\
+      if (!set_regs_matched_done)					\
+	{								\
+	  unsigned r;							\
+	  set_regs_matched_done = 1;					\
+	  for (r = lowest_active_reg; r <= highest_active_reg; r++)	\
+	    {								\
+	      MATCHED_SOMETHING (reg_info[r])				\
+		= EVER_MATCHED_SOMETHING (reg_info[r])			\
+		= 1;							\
+	    }								\
+	}								\
+    }									\
+  while (0)
+
+/* Registers are set to a sentinel when they haven't yet matched.  */
+static char reg_unset_dummy;
+#define REG_UNSET_VALUE (&reg_unset_dummy)
+#define REG_UNSET(e) ((e) == REG_UNSET_VALUE)
+
+
+/* Subroutine declarations and macros for regex_compile.  */
+
+static void store_op1 (), store_op2 ();
+static void insert_op1 (), insert_op2 ();
+static boolean at_begline_loc_p (), at_endline_loc_p ();
+static boolean group_in_compile_stack ();
+static reg_errcode_t compile_range ();
+
+/* Fetch the next character in the uncompiled pattern---translating it 
+   if necessary.  Also cast from a signed character in the constant
+   string passed to us by the user to an unsigned char that we can use
+   as an array index (in, e.g., `translate').  */
+#define PATFETCH(c)							\
+  do {if (p == pend) return REG_EEND;					\
+    c = (unsigned char) *p++;						\
+    if (translate) c = translate[c]; 					\
+  } while (0)
+
+/* Fetch the next character in the uncompiled pattern, with no
+   translation.  */
+#define PATFETCH_RAW(c)							\
+  do {if (p == pend) return REG_EEND;					\
+    c = (unsigned char) *p++; 						\
+  } while (0)
+
+/* Go backwards one character in the pattern.  */
+#define PATUNFETCH p--
+
+
+/* If `translate' is non-null, return translate[D], else just D.  We
+   cast the subscript to translate because some data is declared as
+   `char *', to avoid warnings when a string constant is passed.  But
+   when we use a character as a subscript we must make it unsigned.  */
+#define TRANSLATE(d) (translate ? translate[(unsigned char) (d)] : (d))
+
+
+/* Macros for outputting the compiled pattern into `buffer'.  */
+
+/* If the buffer isn't allocated when it comes in, use this.  */
+#define INIT_BUF_SIZE  32
+
+/* Make sure we have at least N more bytes of space in buffer.  */
+#define GET_BUFFER_SPACE(n)						\
+    while (b - bufp->buffer + (n) > bufp->allocated)			\
+      EXTEND_BUFFER ()
+
+/* Make sure we have one more byte of buffer space and then add C to it.  */
+#define BUF_PUSH(c)							\
+  do {									\
+    GET_BUFFER_SPACE (1);						\
+    *b++ = (unsigned char) (c);						\
+  } while (0)
+
+
+/* Ensure we have two more bytes of buffer space and then append C1 and C2.  */
+#define BUF_PUSH_2(c1, c2)						\
+  do {									\
+    GET_BUFFER_SPACE (2);						\
+    *b++ = (unsigned char) (c1);					\
+    *b++ = (unsigned char) (c2);					\
+  } while (0)
+
+
+/* As with BUF_PUSH_2, except for three bytes.  */
+#define BUF_PUSH_3(c1, c2, c3)						\
+  do {									\
+    GET_BUFFER_SPACE (3);						\
+    *b++ = (unsigned char) (c1);					\
+    *b++ = (unsigned char) (c2);					\
+    *b++ = (unsigned char) (c3);					\
+  } while (0)
+
+
+/* Store a jump with opcode OP at LOC to location TO.  We store a
+   relative address offset by the three bytes the jump itself occupies.  */
+#define STORE_JUMP(op, loc, to) \
+  store_op1 (op, loc, (to) - (loc) - 3)
+
+/* Likewise, for a two-argument jump.  */
+#define STORE_JUMP2(op, loc, to, arg) \
+  store_op2 (op, loc, (to) - (loc) - 3, arg)
+
+/* Like `STORE_JUMP', but for inserting.  Assume `b' is the buffer end.  */
+#define INSERT_JUMP(op, loc, to) \
+  insert_op1 (op, loc, (to) - (loc) - 3, b)
+
+/* Like `STORE_JUMP2', but for inserting.  Assume `b' is the buffer end.  */
+#define INSERT_JUMP2(op, loc, to, arg) \
+  insert_op2 (op, loc, (to) - (loc) - 3, arg, b)
+
+
+/* This is not an arbitrary limit: the arguments which represent offsets
+   into the pattern are two bytes long.  So if 2^16 bytes turns out to
+   be too small, many things would have to change.  */
+#define MAX_BUF_SIZE (1L << 16)
+
+
+/* Extend the buffer by twice its current size via realloc and
+   reset the pointers that pointed into the old block to point to the
+   correct places in the new one.  If extending the buffer results in it
+   being larger than MAX_BUF_SIZE, then flag memory exhausted.  */
+#define EXTEND_BUFFER()							\
+  do { 									\
+    unsigned char *old_buffer = bufp->buffer;				\
+    if (bufp->allocated == MAX_BUF_SIZE) 				\
+      return REG_ESIZE;							\
+    bufp->allocated <<= 1;						\
+    if (bufp->allocated > MAX_BUF_SIZE)					\
+      bufp->allocated = MAX_BUF_SIZE; 					\
+    bufp->buffer = (unsigned char *) realloc (bufp->buffer, bufp->allocated);\
+    if (bufp->buffer == NULL)						\
+      return REG_ESPACE;						\
+    /* If the buffer moved, move all the pointers into it.  */		\
+    if (old_buffer != bufp->buffer)					\
+      {									\
+        b = (b - old_buffer) + bufp->buffer;				\
+        begalt = (begalt - old_buffer) + bufp->buffer;			\
+        if (fixup_alt_jump)						\
+          fixup_alt_jump = (fixup_alt_jump - old_buffer) + bufp->buffer;\
+        if (laststart)							\
+          laststart = (laststart - old_buffer) + bufp->buffer;		\
+        if (pending_exact)						\
+          pending_exact = (pending_exact - old_buffer) + bufp->buffer;	\
+      }									\
+  } while (0)
+
+
+/* Since we have one byte reserved for the register number argument to
+   {start,stop}_memory, the maximum number of groups we can report
+   things about is what fits in that byte.  */
+#define MAX_REGNUM 255
+
+/* But patterns can have more than `MAX_REGNUM' registers.  We just
+   ignore the excess.  */
+typedef unsigned regnum_t;
+
+
+/* Macros for the compile stack.  */
+
+/* Since offsets can go either forwards or backwards, this type needs to
+   be able to hold values from -(MAX_BUF_SIZE - 1) to MAX_BUF_SIZE - 1.  */
+typedef int pattern_offset_t;
+
+typedef struct
+{
+  pattern_offset_t begalt_offset;
+  pattern_offset_t fixup_alt_jump;
+  pattern_offset_t inner_group_offset;
+  pattern_offset_t laststart_offset;  
+  regnum_t regnum;
+} compile_stack_elt_t;
+
+
+typedef struct
+{
+  compile_stack_elt_t *stack;
+  unsigned size;
+  unsigned avail;			/* Offset of next open position.  */
+} compile_stack_type;
+
+
+#define INIT_COMPILE_STACK_SIZE 32
+
+#define COMPILE_STACK_EMPTY  (compile_stack.avail == 0)
+#define COMPILE_STACK_FULL  (compile_stack.avail == compile_stack.size)
+
+/* The next available element.  */
+#define COMPILE_STACK_TOP (compile_stack.stack[compile_stack.avail])
+
+
+/* Set the bit for character C in a list.  */
+#define SET_LIST_BIT(c)                               \
+  (b[((unsigned char) (c)) / BYTEWIDTH]               \
+   |= 1 << (((unsigned char) c) % BYTEWIDTH))
+
+
+/* Get the next unsigned number in the uncompiled pattern.  */
+#define GET_UNSIGNED_NUMBER(num) 					\
+  { if (p != pend)							\
+     {									\
+       PATFETCH (c); 							\
+       while (ISDIGIT (c)) 						\
+         { 								\
+           if (num < 0)							\
+              num = 0;							\
+           num = num * 10 + c - '0'; 					\
+           if (p == pend) 						\
+              break; 							\
+           PATFETCH (c);						\
+         } 								\
+       } 								\
+    }		
+
+#define CHAR_CLASS_MAX_LENGTH  6 /* Namely, `xdigit'.  */
+
+#define IS_CHAR_CLASS(string)						\
+   (STREQ (string, "alpha") || STREQ (string, "upper")			\
+    || STREQ (string, "lower") || STREQ (string, "digit")		\
+    || STREQ (string, "alnum") || STREQ (string, "xdigit")		\
+    || STREQ (string, "space") || STREQ (string, "print")		\
+    || STREQ (string, "punct") || STREQ (string, "graph")		\
+    || STREQ (string, "cntrl") || STREQ (string, "blank"))
+
+
+#ifndef MATCH_MAY_ALLOCATE
+
+/* If we cannot allocate large objects within re_match_2_internal,
+   we make the fail stack and register vectors global.
+   The fail stack, we grow to the maximum size when a regexp
+   is compiled.
+   The register vectors, we adjust in size each time we
+   compile a regexp, according to the number of registers it needs.  */
+
+static fail_stack_type fail_stack;
+
+/* Size with which the following vectors are currently allocated.
+   That is so we can make them bigger as needed,
+   but never make them smaller.  */
+static int regs_allocated_size;
+
+static const char **     regstart, **     regend;
+static const char ** old_regstart, ** old_regend;
+static const char **best_regstart, **best_regend;
+static register_info_type *reg_info; 
+static const char **reg_dummy;
+static register_info_type *reg_info_dummy;
+
+/* Make the register vectors big enough for NUM_REGS registers,
+   but don't make them smaller.  */
+
+static
+regex_grow_registers (num_regs)
+     int num_regs;
+{
+  if (num_regs > regs_allocated_size)
+    {
+      RETALLOC_IF (regstart,	 num_regs, const char *);
+      RETALLOC_IF (regend,	 num_regs, const char *);
+      RETALLOC_IF (old_regstart, num_regs, const char *);
+      RETALLOC_IF (old_regend,	 num_regs, const char *);
+      RETALLOC_IF (best_regstart, num_regs, const char *);
+      RETALLOC_IF (best_regend,	 num_regs, const char *);
+      RETALLOC_IF (reg_info,	 num_regs, register_info_type);
+      RETALLOC_IF (reg_dummy,	 num_regs, const char *);
+      RETALLOC_IF (reg_info_dummy, num_regs, register_info_type);
+
+      regs_allocated_size = num_regs;
+    }
+}
+
+#endif /* not MATCH_MAY_ALLOCATE */
+
+
+/* `regex_compile' compiles PATTERN (of length SIZE) according to SYNTAX.
+   Returns one of error codes defined in `regex.h', or zero for success.
+
+   Assumes the `allocated' (and perhaps `buffer') and `translate'
+   fields are set in BUFP on entry.
+
+   If it succeeds, results are put in BUFP (if it returns an error, the
+   contents of BUFP are undefined):
+     `buffer' is the compiled pattern;
+     `syntax' is set to SYNTAX;
+     `used' is set to the length of the compiled pattern;
+     `fastmap_accurate' is zero;
+     `re_nsub' is the number of subexpressions in PATTERN;
+     `not_bol' and `not_eol' are zero;
+   
+   The `fastmap' and `newline_anchor' fields are neither
+   examined nor set.  */
+
+/* Return, freeing storage we allocated.  */
+#define FREE_STACK_RETURN(value)		\
+  return (free (compile_stack.stack), value)
+
+static reg_errcode_t
+regex_compile (pattern, size, syntax, bufp)
+     const char *pattern;
+     int size;
+     reg_syntax_t syntax;
+     struct re_pattern_buffer *bufp;
+{
+  /* We fetch characters from PATTERN here.  Even though PATTERN is
+     `char *' (i.e., signed), we declare these variables as unsigned, so
+     they can be reliably used as array indices.  */
+  register unsigned char c, c1;
+  
+  /* A random temporary spot in PATTERN.  */
+  const char *p1;
+
+  /* Points to the end of the buffer, where we should append.  */
+  register unsigned char *b;
+  
+  /* Keeps track of unclosed groups.  */
+  compile_stack_type compile_stack;
+
+  /* Points to the current (ending) position in the pattern.  */
+  const char *p = pattern;
+  const char *pend = pattern + size;
+  
+  /* How to translate the characters in the pattern.  */
+  char *translate = bufp->translate;
+
+  /* Address of the count-byte of the most recently inserted `exactn'
+     command.  This makes it possible to tell if a new exact-match
+     character can be added to that command or if the character requires
+     a new `exactn' command.  */
+  unsigned char *pending_exact = 0;
+
+  /* Address of start of the most recently finished expression.
+     This tells, e.g., postfix * where to find the start of its
+     operand.  Reset at the beginning of groups and alternatives.  */
+  unsigned char *laststart = 0;
+
+  /* Address of beginning of regexp, or inside of last group.  */
+  unsigned char *begalt;
+
+  /* Place in the uncompiled pattern (i.e., the {) to
+     which to go back if the interval is invalid.  */
+  const char *beg_interval;
+                
+  /* Address of the place where a forward jump should go to the end of
+     the containing expression.  Each alternative of an `or' -- except the
+     last -- ends with a forward jump of this sort.  */
+  unsigned char *fixup_alt_jump = 0;
+
+  /* Counts open-groups as they are encountered.  Remembered for the
+     matching close-group on the compile stack, so the same register
+     number is put in the stop_memory as the start_memory.  */
+  regnum_t regnum = 0;
+
+#ifdef DEBUG
+  DEBUG_PRINT1 ("\nCompiling pattern: ");
+  if (debug)
+    {
+      unsigned debug_count;
+      
+      for (debug_count = 0; debug_count < size; debug_count++)
+        putchar (pattern[debug_count]);
+      putchar ('\n');
+    }
+#endif /* DEBUG */
+
+  /* Initialize the compile stack.  */
+  compile_stack.stack = TALLOC (INIT_COMPILE_STACK_SIZE, compile_stack_elt_t);
+  if (compile_stack.stack == NULL)
+    return REG_ESPACE;
+
+  compile_stack.size = INIT_COMPILE_STACK_SIZE;
+  compile_stack.avail = 0;
+
+  /* Initialize the pattern buffer.  */
+  bufp->syntax = syntax;
+  bufp->fastmap_accurate = 0;
+  bufp->not_bol = bufp->not_eol = 0;
+
+  /* Set `used' to zero, so that if we return an error, the pattern
+     printer (for debugging) will think there's no pattern.  We reset it
+     at the end.  */
+  bufp->used = 0;
+  
+  /* Always count groups, whether or not bufp->no_sub is set.  */
+  bufp->re_nsub = 0;				
+
+#if !defined (emacs) && !defined (SYNTAX_TABLE)
+  /* Initialize the syntax table.  */
+   init_syntax_once ();
+#endif
+
+  if (bufp->allocated == 0)
+    {
+      if (bufp->buffer)
+	{ /* If zero allocated, but buffer is non-null, try to realloc
+             enough space.  This loses if buffer's address is bogus, but
+             that is the user's responsibility.  */
+          RETALLOC (bufp->buffer, INIT_BUF_SIZE, unsigned char);
+        }
+      else
+        { /* Caller did not allocate a buffer.  Do it for them.  */
+          bufp->buffer = TALLOC (INIT_BUF_SIZE, unsigned char);
+        }
+      if (!bufp->buffer) FREE_STACK_RETURN (REG_ESPACE);
+
+      bufp->allocated = INIT_BUF_SIZE;
+    }
+
+  begalt = b = bufp->buffer;
+
+  /* Loop through the uncompiled pattern until we're at the end.  */
+  while (p != pend)
+    {
+      PATFETCH (c);
+
+      switch (c)
+        {
+        case '^':
+          {
+            if (   /* If at start of pattern, it's an operator.  */
+                   p == pattern + 1
+                   /* If context independent, it's an operator.  */
+                || syntax & RE_CONTEXT_INDEP_ANCHORS
+                   /* Otherwise, depends on what's come before.  */
+                || at_begline_loc_p (pattern, p, syntax))
+              BUF_PUSH (begline);
+            else
+              goto normal_char;
+          }
+          break;
+
+
+        case '$':
+          {
+            if (   /* If at end of pattern, it's an operator.  */
+                   p == pend 
+                   /* If context independent, it's an operator.  */
+                || syntax & RE_CONTEXT_INDEP_ANCHORS
+                   /* Otherwise, depends on what's next.  */
+                || at_endline_loc_p (p, pend, syntax))
+               BUF_PUSH (endline);
+             else
+               goto normal_char;
+           }
+           break;
+
+
+	case '+':
+        case '?':
+          if ((syntax & RE_BK_PLUS_QM)
+              || (syntax & RE_LIMITED_OPS))
+            goto normal_char;
+        handle_plus:
+        case '*':
+          /* If there is no previous pattern... */
+          if (!laststart)
+            {
+              if (syntax & RE_CONTEXT_INVALID_OPS)
+                FREE_STACK_RETURN (REG_BADRPT);
+              else if (!(syntax & RE_CONTEXT_INDEP_OPS))
+                goto normal_char;
+            }
+
+          {
+            /* Are we optimizing this jump?  */
+            boolean keep_string_p = false;
+            
+            /* 1 means zero (many) matches is allowed.  */
+            char zero_times_ok = 0, many_times_ok = 0;
+
+            /* If there is a sequence of repetition chars, collapse it
+               down to just one (the right one).  We can't combine
+               interval operators with these because of, e.g., `a{2}*',
+               which should only match an even number of `a's.  */
+
+            for (;;)
+              {
+                zero_times_ok |= c != '+';
+                many_times_ok |= c != '?';
+
+                if (p == pend)
+                  break;
+
+                PATFETCH (c);
+
+                if (c == '*'
+                    || (!(syntax & RE_BK_PLUS_QM) && (c == '+' || c == '?')))
+                  ;
+
+                else if (syntax & RE_BK_PLUS_QM  &&  c == '\\')
+                  {
+                    if (p == pend) FREE_STACK_RETURN (REG_EESCAPE);
+
+                    PATFETCH (c1);
+                    if (!(c1 == '+' || c1 == '?'))
+                      {
+                        PATUNFETCH;
+                        PATUNFETCH;
+                        break;
+                      }
+
+                    c = c1;
+                  }
+                else
+                  {
+                    PATUNFETCH;
+                    break;
+                  }
+
+                /* If we get here, we found another repeat character.  */
+               }
+
+            /* Star, etc. applied to an empty pattern is equivalent
+               to an empty pattern.  */
+            if (!laststart)  
+              break;
+
+            /* Now we know whether or not zero matches is allowed
+               and also whether or not two or more matches is allowed.  */
+            if (many_times_ok)
+              { /* More than one repetition is allowed, so put in at the
+                   end a backward relative jump from `b' to before the next
+                   jump we're going to put in below (which jumps from
+                   laststart to after this jump).  
+
+                   But if we are at the `*' in the exact sequence `.*\n',
+                   insert an unconditional jump backwards to the .,
+                   instead of the beginning of the loop.  This way we only
+                   push a failure point once, instead of every time
+                   through the loop.  */
+                assert (p - 1 > pattern);
+
+                /* Allocate the space for the jump.  */
+                GET_BUFFER_SPACE (3);
+
+                /* We know we are not at the first character of the pattern,
+                   because laststart was nonzero.  And we've already
+                   incremented `p', by the way, to be the character after
+                   the `*'.  Do we have to do something analogous here
+                   for null bytes, because of RE_DOT_NOT_NULL?  */
+                if (TRANSLATE (*(p - 2)) == TRANSLATE ('.')
+		    && zero_times_ok
+                    && p < pend && TRANSLATE (*p) == TRANSLATE ('\n')
+                    && !(syntax & RE_DOT_NEWLINE))
+                  { /* We have .*\n.  */
+                    STORE_JUMP (jump, b, laststart);
+                    keep_string_p = true;
+                  }
+                else
+                  /* Anything else.  */
+                  STORE_JUMP (maybe_pop_jump, b, laststart - 3);
+
+                /* We've added more stuff to the buffer.  */
+                b += 3;
+              }
+
+            /* On failure, jump from laststart to b + 3, which will be the
+               end of the buffer after this jump is inserted.  */
+            GET_BUFFER_SPACE (3);
+            INSERT_JUMP (keep_string_p ? on_failure_keep_string_jump
+                                       : on_failure_jump,
+                         laststart, b + 3);
+            pending_exact = 0;
+            b += 3;
+
+            if (!zero_times_ok)
+              {
+                /* At least one repetition is required, so insert a
+                   `dummy_failure_jump' before the initial
+                   `on_failure_jump' instruction of the loop. This
+                   effects a skip over that instruction the first time
+                   we hit that loop.  */
+                GET_BUFFER_SPACE (3);
+                INSERT_JUMP (dummy_failure_jump, laststart, laststart + 6);
+                b += 3;
+              }
+            }
+	  break;
+
+
+	case '.':
+          laststart = b;
+          BUF_PUSH (anychar);
+          break;
+
+
+        case '[':
+          {
+            boolean had_char_class = false;
+
+            if (p == pend) FREE_STACK_RETURN (REG_EBRACK);
+
+            /* Ensure that we have enough space to push a charset: the
+               opcode, the length count, and the bitset; 34 bytes in all.  */
+	    GET_BUFFER_SPACE (34);
+
+            laststart = b;
+
+            /* We test `*p == '^' twice, instead of using an if
+               statement, so we only need one BUF_PUSH.  */
+            BUF_PUSH (*p == '^' ? charset_not : charset); 
+            if (*p == '^')
+              p++;
+
+            /* Remember the first position in the bracket expression.  */
+            p1 = p;
+
+            /* Push the number of bytes in the bitmap.  */
+            BUF_PUSH ((1 << BYTEWIDTH) / BYTEWIDTH);
+
+            /* Clear the whole map.  */
+            bzero (b, (1 << BYTEWIDTH) / BYTEWIDTH);
+
+            /* charset_not matches newline according to a syntax bit.  */
+            if ((re_opcode_t) b[-2] == charset_not
+                && (syntax & RE_HAT_LISTS_NOT_NEWLINE))
+              SET_LIST_BIT ('\n');
+
+            /* Read in characters and ranges, setting map bits.  */
+            for (;;)
+              {
+                if (p == pend) FREE_STACK_RETURN (REG_EBRACK);
+
+                PATFETCH (c);
+
+                /* \ might escape characters inside [...] and [^...].  */
+                if ((syntax & RE_BACKSLASH_ESCAPE_IN_LISTS) && c == '\\')
+                  {
+                    if (p == pend) FREE_STACK_RETURN (REG_EESCAPE);
+
+                    PATFETCH (c1);
+                    SET_LIST_BIT (c1);
+                    continue;
+                  }
+
+                /* Could be the end of the bracket expression.  If it's
+                   not (i.e., when the bracket expression is `[]' so
+                   far), the ']' character bit gets set way below.  */
+                if (c == ']' && p != p1 + 1)
+                  break;
+
+                /* Look ahead to see if it's a range when the last thing
+                   was a character class.  */
+                if (had_char_class && c == '-' && *p != ']')
+                  FREE_STACK_RETURN (REG_ERANGE);
+
+                /* Look ahead to see if it's a range when the last thing
+                   was a character: if this is a hyphen not at the
+                   beginning or the end of a list, then it's the range
+                   operator.  */
+                if (c == '-' 
+                    && !(p - 2 >= pattern && p[-2] == '[') 
+                    && !(p - 3 >= pattern && p[-3] == '[' && p[-2] == '^')
+                    && *p != ']')
+                  {
+                    reg_errcode_t ret
+                      = compile_range (&p, pend, translate, syntax, b);
+                    if (ret != REG_NOERROR) FREE_STACK_RETURN (ret);
+                  }
+
+                else if (p[0] == '-' && p[1] != ']')
+                  { /* This handles ranges made up of characters only.  */
+                    reg_errcode_t ret;
+
+		    /* Move past the `-'.  */
+                    PATFETCH (c1);
+                    
+                    ret = compile_range (&p, pend, translate, syntax, b);
+                    if (ret != REG_NOERROR) FREE_STACK_RETURN (ret);
+                  }
+
+                /* See if we're at the beginning of a possible character
+                   class.  */
+
+                else if (syntax & RE_CHAR_CLASSES && c == '[' && *p == ':')
+                  { /* Leave room for the null.  */
+                    char str[CHAR_CLASS_MAX_LENGTH + 1];
+
+                    PATFETCH (c);
+                    c1 = 0;
+
+                    /* If pattern is `[[:'.  */
+                    if (p == pend) FREE_STACK_RETURN (REG_EBRACK);
+
+                    for (;;)
+                      {
+                        PATFETCH (c);
+                        if (c == ':' || c == ']' || p == pend
+                            || c1 == CHAR_CLASS_MAX_LENGTH)
+                          break;
+                        str[c1++] = c;
+                      }
+                    str[c1] = '\0';
+
+                    /* If isn't a word bracketed by `[:' and:`]':
+                       undo the ending character, the letters, and leave 
+                       the leading `:' and `[' (but set bits for them).  */
+                    if (c == ':' && *p == ']')
+                      {
+                        int ch;
+                        boolean is_alnum = STREQ (str, "alnum");
+                        boolean is_alpha = STREQ (str, "alpha");
+                        boolean is_blank = STREQ (str, "blank");
+                        boolean is_cntrl = STREQ (str, "cntrl");
+                        boolean is_digit = STREQ (str, "digit");
+                        boolean is_graph = STREQ (str, "graph");
+                        boolean is_lower = STREQ (str, "lower");
+                        boolean is_print = STREQ (str, "print");
+                        boolean is_punct = STREQ (str, "punct");
+                        boolean is_space = STREQ (str, "space");
+                        boolean is_upper = STREQ (str, "upper");
+                        boolean is_xdigit = STREQ (str, "xdigit");
+                        
+                        if (!IS_CHAR_CLASS (str))
+			  FREE_STACK_RETURN (REG_ECTYPE);
+
+                        /* Throw away the ] at the end of the character
+                           class.  */
+                        PATFETCH (c);					
+
+                        if (p == pend) FREE_STACK_RETURN (REG_EBRACK);
+
+                        for (ch = 0; ch < 1 << BYTEWIDTH; ch++)
+                          {
+			    /* This was split into 3 if's to
+			       avoid an arbitrary limit in some compiler.  */
+                            if (   (is_alnum  && ISALNUM (ch))
+                                || (is_alpha  && ISALPHA (ch))
+                                || (is_blank  && ISBLANK (ch))
+                                || (is_cntrl  && ISCNTRL (ch)))
+			      SET_LIST_BIT (ch);
+			    if (   (is_digit  && ISDIGIT (ch))
+                                || (is_graph  && ISGRAPH (ch))
+                                || (is_lower  && ISLOWER (ch))
+                                || (is_print  && ISPRINT (ch)))
+			      SET_LIST_BIT (ch);
+			    if (   (is_punct  && ISPUNCT (ch))
+                                || (is_space  && ISSPACE (ch))
+                                || (is_upper  && ISUPPER (ch))
+                                || (is_xdigit && ISXDIGIT (ch)))
+			      SET_LIST_BIT (ch);
+                          }
+                        had_char_class = true;
+                      }
+                    else
+                      {
+                        c1++;
+                        while (c1--)    
+                          PATUNFETCH;
+                        SET_LIST_BIT ('[');
+                        SET_LIST_BIT (':');
+                        had_char_class = false;
+                      }
+                  }
+                else
+                  {
+                    had_char_class = false;
+                    SET_LIST_BIT (c);
+                  }
+              }
+
+            /* Discard any (non)matching list bytes that are all 0 at the
+               end of the map.  Decrease the map-length byte too.  */
+            while ((int) b[-1] > 0 && b[b[-1] - 1] == 0) 
+              b[-1]--; 
+            b += b[-1];
+          }
+          break;
+
+
+	case '(':
+          if (syntax & RE_NO_BK_PARENS)
+            goto handle_open;
+          else
+            goto normal_char;
+
+
+        case ')':
+          if (syntax & RE_NO_BK_PARENS)
+            goto handle_close;
+          else
+            goto normal_char;
+
+
+        case '\n':
+          if (syntax & RE_NEWLINE_ALT)
+            goto handle_alt;
+          else
+            goto normal_char;
+
+
+	case '|':
+          if (syntax & RE_NO_BK_VBAR)
+            goto handle_alt;
+          else
+            goto normal_char;
+
+
+        case '{':
+           if (syntax & RE_INTERVALS && syntax & RE_NO_BK_BRACES)
+             goto handle_interval;
+           else
+             goto normal_char;
+
+
+        case '\\':
+          if (p == pend) FREE_STACK_RETURN (REG_EESCAPE);
+
+          /* Do not translate the character after the \, so that we can
+             distinguish, e.g., \B from \b, even if we normally would
+             translate, e.g., B to b.  */
+          PATFETCH_RAW (c);
+
+          switch (c)
+            {
+            case '(':
+              if (syntax & RE_NO_BK_PARENS)
+                goto normal_backslash;
+
+            handle_open:
+              bufp->re_nsub++;
+              regnum++;
+
+              if (COMPILE_STACK_FULL)
+                { 
+                  RETALLOC (compile_stack.stack, compile_stack.size << 1,
+                            compile_stack_elt_t);
+                  if (compile_stack.stack == NULL) return REG_ESPACE;
+
+                  compile_stack.size <<= 1;
+                }
+
+              /* These are the values to restore when we hit end of this
+                 group.  They are all relative offsets, so that if the
+                 whole pattern moves because of realloc, they will still
+                 be valid.  */
+              COMPILE_STACK_TOP.begalt_offset = begalt - bufp->buffer;
+              COMPILE_STACK_TOP.fixup_alt_jump 
+                = fixup_alt_jump ? fixup_alt_jump - bufp->buffer + 1 : 0;
+              COMPILE_STACK_TOP.laststart_offset = b - bufp->buffer;
+              COMPILE_STACK_TOP.regnum = regnum;
+
+              /* We will eventually replace the 0 with the number of
+                 groups inner to this one.  But do not push a
+                 start_memory for groups beyond the last one we can
+                 represent in the compiled pattern.  */
+              if (regnum <= MAX_REGNUM)
+                {
+                  COMPILE_STACK_TOP.inner_group_offset = b - bufp->buffer + 2;
+                  BUF_PUSH_3 (start_memory, regnum, 0);
+                }
+                
+              compile_stack.avail++;
+
+              fixup_alt_jump = 0;
+              laststart = 0;
+              begalt = b;
+	      /* If we've reached MAX_REGNUM groups, then this open
+		 won't actually generate any code, so we'll have to
+		 clear pending_exact explicitly.  */
+	      pending_exact = 0;
+              break;
+
+
+            case ')':
+              if (syntax & RE_NO_BK_PARENS) goto normal_backslash;
+
+              if (COMPILE_STACK_EMPTY)
+                if (syntax & RE_UNMATCHED_RIGHT_PAREN_ORD)
+                  goto normal_backslash;
+                else
+                  FREE_STACK_RETURN (REG_ERPAREN);
+
+            handle_close:
+              if (fixup_alt_jump)
+                { /* Push a dummy failure point at the end of the
+                     alternative for a possible future
+                     `pop_failure_jump' to pop.  See comments at
+                     `push_dummy_failure' in `re_match_2'.  */
+                  BUF_PUSH (push_dummy_failure);
+                  
+                  /* We allocated space for this jump when we assigned
+                     to `fixup_alt_jump', in the `handle_alt' case below.  */
+                  STORE_JUMP (jump_past_alt, fixup_alt_jump, b - 1);
+                }
+
+              /* See similar code for backslashed left paren above.  */
+              if (COMPILE_STACK_EMPTY)
+                if (syntax & RE_UNMATCHED_RIGHT_PAREN_ORD)
+                  goto normal_char;
+                else
+                  FREE_STACK_RETURN (REG_ERPAREN);
+
+              /* Since we just checked for an empty stack above, this
+                 ``can't happen''.  */
+              assert (compile_stack.avail != 0);
+              {
+                /* We don't just want to restore into `regnum', because
+                   later groups should continue to be numbered higher,
+                   as in `(ab)c(de)' -- the second group is #2.  */
+                regnum_t this_group_regnum;
+
+                compile_stack.avail--;		
+                begalt = bufp->buffer + COMPILE_STACK_TOP.begalt_offset;
+                fixup_alt_jump
+                  = COMPILE_STACK_TOP.fixup_alt_jump
+                    ? bufp->buffer + COMPILE_STACK_TOP.fixup_alt_jump - 1 
+                    : 0;
+                laststart = bufp->buffer + COMPILE_STACK_TOP.laststart_offset;
+                this_group_regnum = COMPILE_STACK_TOP.regnum;
+		/* If we've reached MAX_REGNUM groups, then this open
+		   won't actually generate any code, so we'll have to
+		   clear pending_exact explicitly.  */
+		pending_exact = 0;
+
+                /* We're at the end of the group, so now we know how many
+                   groups were inside this one.  */
+                if (this_group_regnum <= MAX_REGNUM)
+                  {
+                    unsigned char *inner_group_loc
+                      = bufp->buffer + COMPILE_STACK_TOP.inner_group_offset;
+                    
+                    *inner_group_loc = regnum - this_group_regnum;
+                    BUF_PUSH_3 (stop_memory, this_group_regnum,
+                                regnum - this_group_regnum);
+                  }
+              }
+              break;
+
+
+            case '|':					/* `\|'.  */
+              if (syntax & RE_LIMITED_OPS || syntax & RE_NO_BK_VBAR)
+                goto normal_backslash;
+            handle_alt:
+              if (syntax & RE_LIMITED_OPS)
+                goto normal_char;
+
+              /* Insert before the previous alternative a jump which
+                 jumps to this alternative if the former fails.  */
+              GET_BUFFER_SPACE (3);
+              INSERT_JUMP (on_failure_jump, begalt, b + 6);
+              pending_exact = 0;
+              b += 3;
+
+              /* The alternative before this one has a jump after it
+                 which gets executed if it gets matched.  Adjust that
+                 jump so it will jump to this alternative's analogous
+                 jump (put in below, which in turn will jump to the next
+                 (if any) alternative's such jump, etc.).  The last such
+                 jump jumps to the correct final destination.  A picture:
+                          _____ _____ 
+                          |   | |   |   
+                          |   v |   v 
+                         a | b   | c   
+
+                 If we are at `b', then fixup_alt_jump right now points to a
+                 three-byte space after `a'.  We'll put in the jump, set
+                 fixup_alt_jump to right after `b', and leave behind three
+                 bytes which we'll fill in when we get to after `c'.  */
+
+              if (fixup_alt_jump)
+                STORE_JUMP (jump_past_alt, fixup_alt_jump, b);
+
+              /* Mark and leave space for a jump after this alternative,
+                 to be filled in later either by next alternative or
+                 when know we're at the end of a series of alternatives.  */
+              fixup_alt_jump = b;
+              GET_BUFFER_SPACE (3);
+              b += 3;
+
+              laststart = 0;
+              begalt = b;
+              break;
+
+
+            case '{': 
+              /* If \{ is a literal.  */
+              if (!(syntax & RE_INTERVALS)
+                     /* If we're at `\{' and it's not the open-interval 
+                        operator.  */
+                  || ((syntax & RE_INTERVALS) && (syntax & RE_NO_BK_BRACES))
+                  || (p - 2 == pattern  &&  p == pend))
+                goto normal_backslash;
+
+            handle_interval:
+              {
+                /* If got here, then the syntax allows intervals.  */
+
+                /* At least (most) this many matches must be made.  */
+                int lower_bound = -1, upper_bound = -1;
+
+                beg_interval = p - 1;
+
+                if (p == pend)
+                  {
+                    if (syntax & RE_NO_BK_BRACES)
+                      goto unfetch_interval;
+                    else
+                      FREE_STACK_RETURN (REG_EBRACE);
+                  }
+
+                GET_UNSIGNED_NUMBER (lower_bound);
+
+                if (c == ',')
+                  {
+                    GET_UNSIGNED_NUMBER (upper_bound);
+                    if (upper_bound < 0) upper_bound = RE_DUP_MAX;
+                  }
+                else
+                  /* Interval such as `{1}' => match exactly once. */
+                  upper_bound = lower_bound;
+
+                if (lower_bound < 0 || upper_bound > RE_DUP_MAX
+                    || lower_bound > upper_bound)
+                  {
+                    if (syntax & RE_NO_BK_BRACES)
+                      goto unfetch_interval;
+                    else 
+                      FREE_STACK_RETURN (REG_BADBR);
+                  }
+
+                if (!(syntax & RE_NO_BK_BRACES)) 
+                  {
+                    if (c != '\\') FREE_STACK_RETURN (REG_EBRACE);
+
+                    PATFETCH (c);
+                  }
+
+                if (c != '}')
+                  {
+                    if (syntax & RE_NO_BK_BRACES)
+                      goto unfetch_interval;
+                    else 
+                      FREE_STACK_RETURN (REG_BADBR);
+                  }
+
+                /* We just parsed a valid interval.  */
+
+                /* If it's invalid to have no preceding re.  */
+                if (!laststart)
+                  {
+                    if (syntax & RE_CONTEXT_INVALID_OPS)
+                      FREE_STACK_RETURN (REG_BADRPT);
+                    else if (syntax & RE_CONTEXT_INDEP_OPS)
+                      laststart = b;
+                    else
+                      goto unfetch_interval;
+                  }
+
+                /* If the upper bound is zero, don't want to succeed at
+                   all; jump from `laststart' to `b + 3', which will be
+                   the end of the buffer after we insert the jump.  */
+                 if (upper_bound == 0)
+                   {
+                     GET_BUFFER_SPACE (3);
+                     INSERT_JUMP (jump, laststart, b + 3);
+                     b += 3;
+                   }
+
+                 /* Otherwise, we have a nontrivial interval.  When
+                    we're all done, the pattern will look like:
+                      set_number_at <jump count> <upper bound>
+                      set_number_at <succeed_n count> <lower bound>
+                      succeed_n <after jump addr> <succeed_n count>
+                      <body of loop>
+                      jump_n <succeed_n addr> <jump count>
+                    (The upper bound and `jump_n' are omitted if
+                    `upper_bound' is 1, though.)  */
+                 else 
+                   { /* If the upper bound is > 1, we need to insert
+                        more at the end of the loop.  */
+                     unsigned nbytes = 10 + (upper_bound > 1) * 10;
+
+                     GET_BUFFER_SPACE (nbytes);
+
+                     /* Initialize lower bound of the `succeed_n', even
+                        though it will be set during matching by its
+                        attendant `set_number_at' (inserted next),
+                        because `re_compile_fastmap' needs to know.
+                        Jump to the `jump_n' we might insert below.  */
+                     INSERT_JUMP2 (succeed_n, laststart,
+                                   b + 5 + (upper_bound > 1) * 5,
+                                   lower_bound);
+                     b += 5;
+
+                     /* Code to initialize the lower bound.  Insert 
+                        before the `succeed_n'.  The `5' is the last two
+                        bytes of this `set_number_at', plus 3 bytes of
+                        the following `succeed_n'.  */
+                     insert_op2 (set_number_at, laststart, 5, lower_bound, b);
+                     b += 5;
+
+                     if (upper_bound > 1)
+                       { /* More than one repetition is allowed, so
+                            append a backward jump to the `succeed_n'
+                            that starts this interval.
+                            
+                            When we've reached this during matching,
+                            we'll have matched the interval once, so
+                            jump back only `upper_bound - 1' times.  */
+                         STORE_JUMP2 (jump_n, b, laststart + 5,
+                                      upper_bound - 1);
+                         b += 5;
+
+                         /* The location we want to set is the second
+                            parameter of the `jump_n'; that is `b-2' as
+                            an absolute address.  `laststart' will be
+                            the `set_number_at' we're about to insert;
+                            `laststart+3' the number to set, the source
+                            for the relative address.  But we are
+                            inserting into the middle of the pattern --
+                            so everything is getting moved up by 5.
+                            Conclusion: (b - 2) - (laststart + 3) + 5,
+                            i.e., b - laststart.
+                            
+                            We insert this at the beginning of the loop
+                            so that if we fail during matching, we'll
+                            reinitialize the bounds.  */
+                         insert_op2 (set_number_at, laststart, b - laststart,
+                                     upper_bound - 1, b);
+                         b += 5;
+                       }
+                   }
+                pending_exact = 0;
+                beg_interval = NULL;
+              }
+              break;
+
+            unfetch_interval:
+              /* If an invalid interval, match the characters as literals.  */
+               assert (beg_interval);
+               p = beg_interval;
+               beg_interval = NULL;
+
+               /* normal_char and normal_backslash need `c'.  */
+               PATFETCH (c);	
+
+               if (!(syntax & RE_NO_BK_BRACES))
+                 {
+                   if (p > pattern  &&  p[-1] == '\\')
+                     goto normal_backslash;
+                 }
+               goto normal_char;
+
+#ifdef emacs
+            /* There is no way to specify the before_dot and after_dot
+               operators.  rms says this is ok.  --karl  */
+            case '=':
+              BUF_PUSH (at_dot);
+              break;
+
+            case 's':	
+              laststart = b;
+              PATFETCH (c);
+              BUF_PUSH_2 (syntaxspec, syntax_spec_code[c]);
+              break;
+
+            case 'S':
+              laststart = b;
+              PATFETCH (c);
+              BUF_PUSH_2 (notsyntaxspec, syntax_spec_code[c]);
+              break;
+#endif /* emacs */
+
+
+            case 'w':
+              laststart = b;
+              BUF_PUSH (wordchar);
+              break;
+
+
+            case 'W':
+              laststart = b;
+              BUF_PUSH (notwordchar);
+              break;
+
+
+            case '<':
+              BUF_PUSH (wordbeg);
+              break;
+
+            case '>':
+              BUF_PUSH (wordend);
+              break;
+
+            case 'b':
+              BUF_PUSH (wordbound);
+              break;
+
+            case 'B':
+              BUF_PUSH (notwordbound);
+              break;
+
+            case '`':
+              BUF_PUSH (begbuf);
+              break;
+
+            case '\'':
+              BUF_PUSH (endbuf);
+              break;
+
+            case '1': case '2': case '3': case '4': case '5':
+            case '6': case '7': case '8': case '9':
+              if (syntax & RE_NO_BK_REFS)
+                goto normal_char;
+
+              c1 = c - '0';
+
+              if (c1 > regnum)
+                FREE_STACK_RETURN (REG_ESUBREG);
+
+              /* Can't back reference to a subexpression if inside of it.  */
+              if (group_in_compile_stack (compile_stack, c1))
+                goto normal_char;
+
+              laststart = b;
+              BUF_PUSH_2 (duplicate, c1);
+              break;
+
+
+            case '+':
+            case '?':
+              if (syntax & RE_BK_PLUS_QM)
+                goto handle_plus;
+              else
+                goto normal_backslash;
+
+            default:
+            normal_backslash:
+              /* You might think it would be useful for \ to mean
+                 not to translate; but if we don't translate it
+                 it will never match anything.  */
+              c = TRANSLATE (c);
+              goto normal_char;
+            }
+          break;
+
+
+	default:
+        /* Expects the character in `c'.  */
+	normal_char:
+	      /* If no exactn currently being built.  */
+          if (!pending_exact 
+
+              /* If last exactn not at current position.  */
+              || pending_exact + *pending_exact + 1 != b
+              
+              /* We have only one byte following the exactn for the count.  */
+	      || *pending_exact == (1 << BYTEWIDTH) - 1
+
+              /* If followed by a repetition operator.  */
+              || *p == '*' || *p == '^'
+	      || ((syntax & RE_BK_PLUS_QM)
+		  ? *p == '\\' && (p[1] == '+' || p[1] == '?')
+		  : (*p == '+' || *p == '?'))
+	      || ((syntax & RE_INTERVALS)
+                  && ((syntax & RE_NO_BK_BRACES)
+		      ? *p == '{'
+                      : (p[0] == '\\' && p[1] == '{'))))
+	    {
+	      /* Start building a new exactn.  */
+              
+              laststart = b;
+
+	      BUF_PUSH_2 (exactn, 0);
+	      pending_exact = b - 1;
+            }
+            
+	  BUF_PUSH (c);
+          (*pending_exact)++;
+	  break;
+        } /* switch (c) */
+    } /* while p != pend */
+
+  
+  /* Through the pattern now.  */
+  
+  if (fixup_alt_jump)
+    STORE_JUMP (jump_past_alt, fixup_alt_jump, b);
+
+  if (!COMPILE_STACK_EMPTY) 
+    FREE_STACK_RETURN (REG_EPAREN);
+
+  /* If we don't want backtracking, force success
+     the first time we reach the end of the compiled pattern.  */
+  if (syntax & RE_NO_POSIX_BACKTRACKING)
+    BUF_PUSH (succeed);
+
+  free (compile_stack.stack);
+
+  /* We have succeeded; set the length of the buffer.  */
+  bufp->used = b - bufp->buffer;
+
+#ifdef DEBUG
+  if (debug)
+    {
+      DEBUG_PRINT1 ("\nCompiled pattern: \n");
+      print_compiled_pattern (bufp);
+    }
+#endif /* DEBUG */
+
+#ifndef MATCH_MAY_ALLOCATE
+  /* Initialize the failure stack to the largest possible stack.  This
+     isn't necessary unless we're trying to avoid calling alloca in
+     the search and match routines.  */
+  {
+    int num_regs = bufp->re_nsub + 1;
+
+    /* Since DOUBLE_FAIL_STACK refuses to double only if the current size
+       is strictly greater than re_max_failures, the largest possible stack
+       is 2 * re_max_failures failure points.  */
+    if (fail_stack.size < (2 * re_max_failures * MAX_FAILURE_ITEMS))
+      {
+	fail_stack.size = (2 * re_max_failures * MAX_FAILURE_ITEMS);
+
+#ifdef emacs
+	if (! fail_stack.stack)
+	  fail_stack.stack
+	    = (fail_stack_elt_t *) xmalloc (fail_stack.size 
+					    * sizeof (fail_stack_elt_t));
+	else
+	  fail_stack.stack
+	    = (fail_stack_elt_t *) xrealloc (fail_stack.stack,
+					     (fail_stack.size
+					      * sizeof (fail_stack_elt_t)));
+#else /* not emacs */
+	if (! fail_stack.stack)
+	  fail_stack.stack
+	    = (fail_stack_elt_t *) malloc (fail_stack.size 
+					   * sizeof (fail_stack_elt_t));
+	else
+	  fail_stack.stack
+	    = (fail_stack_elt_t *) realloc (fail_stack.stack,
+					    (fail_stack.size
+					     * sizeof (fail_stack_elt_t)));
+#endif /* not emacs */
+      }
+
+    regex_grow_registers (num_regs);
+  }
+#endif /* not MATCH_MAY_ALLOCATE */
+
+  return REG_NOERROR;
+} /* regex_compile */
+
+
+/* Subroutines for `regex_compile'.  */
+
+/* Store OP at LOC followed by two-byte integer parameter ARG.  */
+
+static void
+store_op1 (op, loc, arg)
+    re_opcode_t op;
+    unsigned char *loc;
+    int arg;
+{
+  *loc = (unsigned char) op;
+  STORE_NUMBER (loc + 1, arg);
+}
+
+
+/* Like `store_op1', but for two two-byte parameters ARG1 and ARG2.  */
+
+static void
+store_op2 (op, loc, arg1, arg2)
+    re_opcode_t op;
+    unsigned char *loc;
+    int arg1, arg2;
+{
+  *loc = (unsigned char) op;
+  STORE_NUMBER (loc + 1, arg1);
+  STORE_NUMBER (loc + 3, arg2);
+}
+
+
+/* Copy the bytes from LOC to END to open up three bytes of space at LOC
+   for OP followed by two-byte integer parameter ARG.  */
+
+static void
+insert_op1 (op, loc, arg, end)
+    re_opcode_t op;
+    unsigned char *loc;
+    int arg;
+    unsigned char *end;    
+{
+  register unsigned char *pfrom = end;
+  register unsigned char *pto = end + 3;
+
+  while (pfrom != loc)
+    *--pto = *--pfrom;
+    
+  store_op1 (op, loc, arg);
+}
+
+
+/* Like `insert_op1', but for two two-byte parameters ARG1 and ARG2.  */
+
+static void
+insert_op2 (op, loc, arg1, arg2, end)
+    re_opcode_t op;
+    unsigned char *loc;
+    int arg1, arg2;
+    unsigned char *end;    
+{
+  register unsigned char *pfrom = end;
+  register unsigned char *pto = end + 5;
+
+  while (pfrom != loc)
+    *--pto = *--pfrom;
+    
+  store_op2 (op, loc, arg1, arg2);
+}
+
+
+/* P points to just after a ^ in PATTERN.  Return true if that ^ comes
+   after an alternative or a begin-subexpression.  We assume there is at
+   least one character before the ^.  */
+
+static boolean
+at_begline_loc_p (pattern, p, syntax)
+    const char *pattern, *p;
+    reg_syntax_t syntax;
+{
+  const char *prev = p - 2;
+  boolean prev_prev_backslash = prev > pattern && prev[-1] == '\\';
+  
+  return
+       /* After a subexpression?  */
+       (*prev == '(' && (syntax & RE_NO_BK_PARENS || prev_prev_backslash))
+       /* After an alternative?  */
+    || (*prev == '|' && (syntax & RE_NO_BK_VBAR || prev_prev_backslash));
+}
+
+
+/* The dual of at_begline_loc_p.  This one is for $.  We assume there is
+   at least one character after the $, i.e., `P < PEND'.  */
+
+static boolean
+at_endline_loc_p (p, pend, syntax)
+    const char *p, *pend;
+    int syntax;
+{
+  const char *next = p;
+  boolean next_backslash = *next == '\\';
+  const char *next_next = p + 1 < pend ? p + 1 : NULL;
+  
+  return
+       /* Before a subexpression?  */
+       (syntax & RE_NO_BK_PARENS ? *next == ')'
+        : next_backslash && next_next && *next_next == ')')
+       /* Before an alternative?  */
+    || (syntax & RE_NO_BK_VBAR ? *next == '|'
+        : next_backslash && next_next && *next_next == '|');
+}
+
+
+/* Returns true if REGNUM is in one of COMPILE_STACK's elements and 
+   false if it's not.  */
+
+static boolean
+group_in_compile_stack (compile_stack, regnum)
+    compile_stack_type compile_stack;
+    regnum_t regnum;
+{
+  int this_element;
+
+  for (this_element = compile_stack.avail - 1;  
+       this_element >= 0; 
+       this_element--)
+    if (compile_stack.stack[this_element].regnum == regnum)
+      return true;
+
+  return false;
+}
+
+
+/* Read the ending character of a range (in a bracket expression) from the
+   uncompiled pattern *P_PTR (which ends at PEND).  We assume the
+   starting character is in `P[-2]'.  (`P[-1]' is the character `-'.)
+   Then we set the translation of all bits between the starting and
+   ending characters (inclusive) in the compiled pattern B.
+   
+   Return an error code.
+   
+   We use these short variable names so we can use the same macros as
+   `regex_compile' itself.  */
+
+static reg_errcode_t
+compile_range (p_ptr, pend, translate, syntax, b)
+    const char **p_ptr, *pend;
+    char *translate;
+    reg_syntax_t syntax;
+    unsigned char *b;
+{
+  unsigned this_char;
+
+  const char *p = *p_ptr;
+  int range_start, range_end;
+  
+  if (p == pend)
+    return REG_ERANGE;
+
+  /* Even though the pattern is a signed `char *', we need to fetch
+     with unsigned char *'s; if the high bit of the pattern character
+     is set, the range endpoints will be negative if we fetch using a
+     signed char *.
+
+     We also want to fetch the endpoints without translating them; the 
+     appropriate translation is done in the bit-setting loop below.  */
+  /* The SVR4 compiler on the 3B2 had trouble with unsigned const char *.  */
+  range_start = ((const unsigned char *) p)[-2];
+  range_end   = ((const unsigned char *) p)[0];
+
+  /* Have to increment the pointer into the pattern string, so the
+     caller isn't still at the ending character.  */
+  (*p_ptr)++;
+
+  /* If the start is after the end, the range is empty.  */
+  if (range_start > range_end)
+    return syntax & RE_NO_EMPTY_RANGES ? REG_ERANGE : REG_NOERROR;
+
+  /* Here we see why `this_char' has to be larger than an `unsigned
+     char' -- the range is inclusive, so if `range_end' == 0xff
+     (assuming 8-bit characters), we would otherwise go into an infinite
+     loop, since all characters <= 0xff.  */
+  for (this_char = range_start; this_char <= range_end; this_char++)
+    {
+      SET_LIST_BIT (TRANSLATE (this_char));
+    }
+  
+  return REG_NOERROR;
+}
+
+
+/* re_compile_fastmap computes a ``fastmap'' for the compiled pattern in
+   BUFP.  A fastmap records which of the (1 << BYTEWIDTH) possible
+   characters can start a string that matches the pattern.  This fastmap
+   is used by re_search to skip quickly over impossible starting points.
+
+   The caller must supply the address of a (1 << BYTEWIDTH)-byte data
+   area as BUFP->fastmap.
+   
+   We set the `fastmap', `fastmap_accurate', and `can_be_null' fields in
+   the pattern buffer.
+
+   Returns 0 if we succeed, -2 if an internal error.   */
+
+int
+re_compile_fastmap (bufp)
+     struct re_pattern_buffer *bufp;
+{
+  int j, k;
+#ifdef MATCH_MAY_ALLOCATE
+  fail_stack_type fail_stack;
+#endif
+#ifndef REGEX_MALLOC
+  char *destination;
+#endif
+  /* We don't push any register information onto the failure stack.  */
+  unsigned num_regs = 0;
+  
+  register char *fastmap = bufp->fastmap;
+  unsigned char *pattern = bufp->buffer;
+  unsigned long size = bufp->used;
+  unsigned char *p = pattern;
+  register unsigned char *pend = pattern + size;
+
+  /* This holds the pointer to the failure stack, when
+     it is allocated relocatably.  */
+  fail_stack_elt_t *failure_stack_ptr;
+
+  /* Assume that each path through the pattern can be null until
+     proven otherwise.  We set this false at the bottom of switch
+     statement, to which we get only if a particular path doesn't
+     match the empty string.  */
+  boolean path_can_be_null = true;
+
+  /* We aren't doing a `succeed_n' to begin with.  */
+  boolean succeed_n_p = false;
+
+  assert (fastmap != NULL && p != NULL);
+  
+  INIT_FAIL_STACK ();
+  bzero (fastmap, 1 << BYTEWIDTH);  /* Assume nothing's valid.  */
+  bufp->fastmap_accurate = 1;	    /* It will be when we're done.  */
+  bufp->can_be_null = 0;
+      
+  while (1)
+    {
+      if (p == pend || *p == succeed)
+	{
+	  /* We have reached the (effective) end of pattern.  */
+	  if (!FAIL_STACK_EMPTY ())
+	    {
+	      bufp->can_be_null |= path_can_be_null;
+
+	      /* Reset for next path.  */
+	      path_can_be_null = true;
+
+	      p = fail_stack.stack[--fail_stack.avail].pointer;
+
+	      continue;
+	    }
+	  else
+	    break;
+	}
+
+      /* We should never be about to go beyond the end of the pattern.  */
+      assert (p < pend);
+      
+      switch (SWITCH_ENUM_CAST ((re_opcode_t) *p++))
+	{
+
+        /* I guess the idea here is to simply not bother with a fastmap
+           if a backreference is used, since it's too hard to figure out
+           the fastmap for the corresponding group.  Setting
+           `can_be_null' stops `re_search_2' from using the fastmap, so
+           that is all we do.  */
+	case duplicate:
+	  bufp->can_be_null = 1;
+          goto done;
+
+
+      /* Following are the cases which match a character.  These end
+         with `break'.  */
+
+	case exactn:
+          fastmap[p[1]] = 1;
+	  break;
+
+
+        case charset:
+          for (j = *p++ * BYTEWIDTH - 1; j >= 0; j--)
+	    if (p[j / BYTEWIDTH] & (1 << (j % BYTEWIDTH)))
+              fastmap[j] = 1;
+	  break;
+
+
+	case charset_not:
+	  /* Chars beyond end of map must be allowed.  */
+	  for (j = *p * BYTEWIDTH; j < (1 << BYTEWIDTH); j++)
+            fastmap[j] = 1;
+
+	  for (j = *p++ * BYTEWIDTH - 1; j >= 0; j--)
+	    if (!(p[j / BYTEWIDTH] & (1 << (j % BYTEWIDTH))))
+              fastmap[j] = 1;
+          break;
+
+
+	case wordchar:
+	  for (j = 0; j < (1 << BYTEWIDTH); j++)
+	    if (SYNTAX (j) == Sword)
+	      fastmap[j] = 1;
+	  break;
+
+
+	case notwordchar:
+	  for (j = 0; j < (1 << BYTEWIDTH); j++)
+	    if (SYNTAX (j) != Sword)
+	      fastmap[j] = 1;
+	  break;
+
+
+        case anychar:
+	  {
+	    int fastmap_newline = fastmap['\n'];
+
+	    /* `.' matches anything ...  */
+	    for (j = 0; j < (1 << BYTEWIDTH); j++)
+	      fastmap[j] = 1;
+
+	    /* ... except perhaps newline.  */
+	    if (!(bufp->syntax & RE_DOT_NEWLINE))
+	      fastmap['\n'] = fastmap_newline;
+
+	    /* Return if we have already set `can_be_null'; if we have,
+	       then the fastmap is irrelevant.  Something's wrong here.  */
+	    else if (bufp->can_be_null)
+	      goto done;
+
+	    /* Otherwise, have to check alternative paths.  */
+	    break;
+	  }
+
+#ifdef emacs
+        case syntaxspec:
+	  k = *p++;
+	  for (j = 0; j < (1 << BYTEWIDTH); j++)
+	    if (SYNTAX (j) == (enum syntaxcode) k)
+	      fastmap[j] = 1;
+	  break;
+
+
+	case notsyntaxspec:
+	  k = *p++;
+	  for (j = 0; j < (1 << BYTEWIDTH); j++)
+	    if (SYNTAX (j) != (enum syntaxcode) k)
+	      fastmap[j] = 1;
+	  break;
+
+
+      /* All cases after this match the empty string.  These end with
+         `continue'.  */
+
+
+	case before_dot:
+	case at_dot:
+	case after_dot:
+          continue;
+#endif /* not emacs */
+
+
+        case no_op:
+        case begline:
+        case endline:
+	case begbuf:
+	case endbuf:
+	case wordbound:
+	case notwordbound:
+	case wordbeg:
+	case wordend:
+        case push_dummy_failure:
+          continue;
+
+
+	case jump_n:
+        case pop_failure_jump:
+	case maybe_pop_jump:
+	case jump:
+        case jump_past_alt:
+	case dummy_failure_jump:
+          EXTRACT_NUMBER_AND_INCR (j, p);
+	  p += j;	
+	  if (j > 0)
+	    continue;
+            
+          /* Jump backward implies we just went through the body of a
+             loop and matched nothing.  Opcode jumped to should be
+             `on_failure_jump' or `succeed_n'.  Just treat it like an
+             ordinary jump.  For a * loop, it has pushed its failure
+             point already; if so, discard that as redundant.  */
+          if ((re_opcode_t) *p != on_failure_jump
+	      && (re_opcode_t) *p != succeed_n)
+	    continue;
+
+          p++;
+          EXTRACT_NUMBER_AND_INCR (j, p);
+          p += j;		
+	  
+          /* If what's on the stack is where we are now, pop it.  */
+          if (!FAIL_STACK_EMPTY () 
+	      && fail_stack.stack[fail_stack.avail - 1].pointer == p)
+            fail_stack.avail--;
+
+          continue;
+
+
+        case on_failure_jump:
+        case on_failure_keep_string_jump:
+	handle_on_failure_jump:
+          EXTRACT_NUMBER_AND_INCR (j, p);
+
+          /* For some patterns, e.g., `(a?)?', `p+j' here points to the
+             end of the pattern.  We don't want to push such a point,
+             since when we restore it above, entering the switch will
+             increment `p' past the end of the pattern.  We don't need
+             to push such a point since we obviously won't find any more
+             fastmap entries beyond `pend'.  Such a pattern can match
+             the null string, though.  */
+          if (p + j < pend)
+            {
+              if (!PUSH_PATTERN_OP (p + j, fail_stack))
+		{
+		  RESET_FAIL_STACK ();
+		  return -2;
+		}
+            }
+          else
+            bufp->can_be_null = 1;
+
+          if (succeed_n_p)
+            {
+              EXTRACT_NUMBER_AND_INCR (k, p);	/* Skip the n.  */
+              succeed_n_p = false;
+	    }
+
+          continue;
+
+
+	case succeed_n:
+          /* Get to the number of times to succeed.  */
+          p += 2;		
+
+          /* Increment p past the n for when k != 0.  */
+          EXTRACT_NUMBER_AND_INCR (k, p);
+          if (k == 0)
+	    {
+              p -= 4;
+  	      succeed_n_p = true;  /* Spaghetti code alert.  */
+              goto handle_on_failure_jump;
+            }
+          continue;
+
+
+	case set_number_at:
+          p += 4;
+          continue;
+
+
+	case start_memory:
+        case stop_memory:
+	  p += 2;
+	  continue;
+
+
+	default:
+          abort (); /* We have listed all the cases.  */
+        } /* switch *p++ */
+
+      /* Getting here means we have found the possible starting
+         characters for one path of the pattern -- and that the empty
+         string does not match.  We need not follow this path further.
+         Instead, look at the next alternative (remembered on the
+         stack), or quit if no more.  The test at the top of the loop
+         does these things.  */
+      path_can_be_null = false;
+      p = pend;
+    } /* while p */
+
+  /* Set `can_be_null' for the last path (also the first path, if the
+     pattern is empty).  */
+  bufp->can_be_null |= path_can_be_null;
+
+ done:
+  RESET_FAIL_STACK ();
+  return 0;
+} /* re_compile_fastmap */
+
+
+/* Set REGS to hold NUM_REGS registers, storing them in STARTS and
+   ENDS.  Subsequent matches using PATTERN_BUFFER and REGS will use
+   this memory for recording register information.  STARTS and ENDS
+   must be allocated using the malloc library routine, and must each
+   be at least NUM_REGS * sizeof (regoff_t) bytes long.
+
+   If NUM_REGS == 0, then subsequent matches should allocate their own
+   register data.
+
+   Unless this function is called, the first search or match using
+   PATTERN_BUFFER will allocate its own register data, without
+   freeing the old data.  */
+
+void
+re_set_registers (bufp, regs, num_regs, starts, ends)
+    struct re_pattern_buffer *bufp;
+    struct re_registers *regs;
+    unsigned num_regs;
+    regoff_t *starts, *ends;
+{
+  if (num_regs)
+    {
+      bufp->regs_allocated = REGS_REALLOCATE;
+      regs->num_regs = num_regs;
+      regs->start = starts;
+      regs->end = ends;
+    }
+  else
+    {
+      bufp->regs_allocated = REGS_UNALLOCATED;
+      regs->num_regs = 0;
+      regs->start = regs->end = (regoff_t *) 0;
+    }
+}
+
+
+/* Searching routines.  */
+
+/* Like re_search_2, below, but only one string is specified, and
+   doesn't let you say where to stop matching. */
+
+int
+re_search (bufp, string, size, startpos, range, regs)
+     struct re_pattern_buffer *bufp;
+     const char *string;
+     int size, startpos, range;
+     struct re_registers *regs;
+{
+  return re_search_2 (bufp, NULL, 0, string, size, startpos, range, 
+		      regs, size);
+}
+
+
+/* Using the compiled pattern in BUFP->buffer, first tries to match the
+   virtual concatenation of STRING1 and STRING2, starting first at index
+   STARTPOS, then at STARTPOS + 1, and so on.
+   
+   STRING1 and STRING2 have length SIZE1 and SIZE2, respectively.
+   
+   RANGE is how far to scan while trying to match.  RANGE = 0 means try
+   only at STARTPOS; in general, the last start tried is STARTPOS +
+   RANGE.
+   
+   In REGS, return the indices of the virtual concatenation of STRING1
+   and STRING2 that matched the entire BUFP->buffer and its contained
+   subexpressions.
+   
+   Do not consider matching one past the index STOP in the virtual
+   concatenation of STRING1 and STRING2.
+
+   We return either the position in the strings at which the match was
+   found, -1 if no match, or -2 if error (such as failure
+   stack overflow).  */
+
+int
+re_search_2 (bufp, string1, size1, string2, size2, startpos, range, regs, stop)
+     struct re_pattern_buffer *bufp;
+     const char *string1, *string2;
+     int size1, size2;
+     int startpos;
+     int range;
+     struct re_registers *regs;
+     int stop;
+{
+  int val;
+  register char *fastmap = bufp->fastmap;
+  register char *translate = bufp->translate;
+  int total_size = size1 + size2;
+  int endpos = startpos + range;
+
+  /* Check for out-of-range STARTPOS.  */
+  if (startpos < 0 || startpos > total_size)
+    return -1;
+    
+  /* Fix up RANGE if it might eventually take us outside
+     the virtual concatenation of STRING1 and STRING2.  */
+  if (endpos < -1)
+    range = -1 - startpos;
+  else if (endpos > total_size)
+    range = total_size - startpos;
+
+  /* If the search isn't to be a backwards one, don't waste time in a
+     search for a pattern that must be anchored.  */
+  if (bufp->used > 0 && (re_opcode_t) bufp->buffer[0] == begbuf && range > 0)
+    {
+      if (startpos > 0)
+	return -1;
+      else
+	range = 1;
+    }
+
+  /* Update the fastmap now if not correct already.  */
+  if (fastmap && !bufp->fastmap_accurate)
+    if (re_compile_fastmap (bufp) == -2)
+      return -2;
+  
+  /* Loop through the string, looking for a place to start matching.  */
+  for (;;)
+    { 
+      /* If a fastmap is supplied, skip quickly over characters that
+         cannot be the start of a match.  If the pattern can match the
+         null string, however, we don't need to skip characters; we want
+         the first null string.  */
+      if (fastmap && startpos < total_size && !bufp->can_be_null)
+	{
+	  if (range > 0)	/* Searching forwards.  */
+	    {
+	      register const char *d;
+	      register int lim = 0;
+	      int irange = range;
+
+              if (startpos < size1 && startpos + range >= size1)
+                lim = range - (size1 - startpos);
+
+	      d = (startpos >= size1 ? string2 - size1 : string1) + startpos;
+   
+              /* Written out as an if-else to avoid testing `translate'
+                 inside the loop.  */
+	      if (translate)
+                while (range > lim
+                       && !fastmap[(unsigned char)
+				   translate[(unsigned char) *d++]])
+                  range--;
+	      else
+                while (range > lim && !fastmap[(unsigned char) *d++])
+                  range--;
+
+	      startpos += irange - range;
+	    }
+	  else				/* Searching backwards.  */
+	    {
+	      register char c = (size1 == 0 || startpos >= size1
+                                 ? string2[startpos - size1] 
+                                 : string1[startpos]);
+
+	      if (!fastmap[(unsigned char) TRANSLATE (c)])
+		goto advance;
+	    }
+	}
+
+      /* If can't match the null string, and that's all we have left, fail.  */
+      if (range >= 0 && startpos == total_size && fastmap
+          && !bufp->can_be_null)
+	return -1;
+
+      val = re_match_2_internal (bufp, string1, size1, string2, size2,
+				 startpos, regs, stop);
+#ifndef REGEX_MALLOC
+#ifdef C_ALLOCA
+      alloca (0);
+#endif
+#endif
+
+      if (val >= 0)
+	return startpos;
+        
+      if (val == -2)
+	return -2;
+
+    advance:
+      if (!range) 
+        break;
+      else if (range > 0) 
+        {
+          range--; 
+          startpos++;
+        }
+      else
+        {
+          range++; 
+          startpos--;
+        }
+    }
+  return -1;
+} /* re_search_2 */
+
+
+/* Declarations and macros for re_match_2.  */
+
+static int bcmp_translate ();
+static boolean alt_match_null_string_p (),
+               common_op_match_null_string_p (),
+               group_match_null_string_p ();
+
+/* This converts PTR, a pointer into one of the search strings `string1'
+   and `string2' into an offset from the beginning of that string.  */
+#define POINTER_TO_OFFSET(ptr)			\
+  (FIRST_STRING_P (ptr)				\
+   ? ((regoff_t) ((ptr) - string1))		\
+   : ((regoff_t) ((ptr) - string2 + size1)))
+
+/* Macros for dealing with the split strings in re_match_2.  */
+
+#define MATCHING_IN_FIRST_STRING  (dend == end_match_1)
+
+/* Call before fetching a character with *d.  This switches over to
+   string2 if necessary.  */
+#define PREFETCH()							\
+  while (d == dend)						    	\
+    {									\
+      /* End of string2 => fail.  */					\
+      if (dend == end_match_2) 						\
+        goto fail;							\
+      /* End of string1 => advance to string2.  */ 			\
+      d = string2;						        \
+      dend = end_match_2;						\
+    }
+
+
+/* Test if at very beginning or at very end of the virtual concatenation
+   of `string1' and `string2'.  If only one string, it's `string2'.  */
+#define AT_STRINGS_BEG(d) ((d) == (size1 ? string1 : string2) || !size2)
+#define AT_STRINGS_END(d) ((d) == end2)	
+
+
+/* Test if D points to a character which is word-constituent.  We have
+   two special cases to check for: if past the end of string1, look at
+   the first character in string2; and if before the beginning of
+   string2, look at the last character in string1.  */
+#define WORDCHAR_P(d)							\
+  (SYNTAX ((d) == end1 ? *string2					\
+           : (d) == string2 - 1 ? *(end1 - 1) : *(d))			\
+   == Sword)
+
+/* Test if the character before D and the one at D differ with respect
+   to being word-constituent.  */
+#define AT_WORD_BOUNDARY(d)						\
+  (AT_STRINGS_BEG (d) || AT_STRINGS_END (d)				\
+   || WORDCHAR_P (d - 1) != WORDCHAR_P (d))
+
+
+/* Free everything we malloc.  */
+#ifdef MATCH_MAY_ALLOCATE
+#define FREE_VAR(var) if (var) REGEX_FREE (var); var = NULL
+#define FREE_VARIABLES()						\
+  do {									\
+    REGEX_FREE_STACK (fail_stack.stack);				\
+    FREE_VAR (regstart);						\
+    FREE_VAR (regend);							\
+    FREE_VAR (old_regstart);						\
+    FREE_VAR (old_regend);						\
+    FREE_VAR (best_regstart);						\
+    FREE_VAR (best_regend);						\
+    FREE_VAR (reg_info);						\
+    FREE_VAR (reg_dummy);						\
+    FREE_VAR (reg_info_dummy);						\
+  } while (0)
+#else
+#define FREE_VARIABLES() ((void)0) /* Do nothing!  But inhibit gcc warning.  */
+#endif /* not MATCH_MAY_ALLOCATE */
+
+/* These values must meet several constraints.  They must not be valid
+   register values; since we have a limit of 255 registers (because
+   we use only one byte in the pattern for the register number), we can
+   use numbers larger than 255.  They must differ by 1, because of
+   NUM_FAILURE_ITEMS above.  And the value for the lowest register must
+   be larger than the value for the highest register, so we do not try
+   to actually save any registers when none are active.  */
+#define NO_HIGHEST_ACTIVE_REG (1 << BYTEWIDTH)
+#define NO_LOWEST_ACTIVE_REG (NO_HIGHEST_ACTIVE_REG + 1)
+
+
+/* Matching routines.  */
+
+#ifndef emacs   /* Emacs never uses this.  */
+/* re_match is like re_match_2 except it takes only a single string.  */
+
+int
+re_match (bufp, string, size, pos, regs)
+     struct re_pattern_buffer *bufp;
+     const char *string;
+     int size, pos;
+     struct re_registers *regs;
+{
+  int result = re_match_2_internal (bufp, NULL, 0, string, size,
+				    pos, regs, size);
+  alloca (0);
+  return result;
+}
+#endif /* not emacs */
+
+
+/* re_match_2 matches the compiled pattern in BUFP against the
+   the (virtual) concatenation of STRING1 and STRING2 (of length SIZE1
+   and SIZE2, respectively).  We start matching at POS, and stop
+   matching at STOP.
+   
+   If REGS is non-null and the `no_sub' field of BUFP is nonzero, we
+   store offsets for the substring each group matched in REGS.  See the
+   documentation for exactly how many groups we fill.
+
+   We return -1 if no match, -2 if an internal error (such as the
+   failure stack overflowing).  Otherwise, we return the length of the
+   matched substring.  */
+
+int
+re_match_2 (bufp, string1, size1, string2, size2, pos, regs, stop)
+     struct re_pattern_buffer *bufp;
+     const char *string1, *string2;
+     int size1, size2;
+     int pos;
+     struct re_registers *regs;
+     int stop;
+{
+  int result = re_match_2_internal (bufp, string1, size1, string2, size2,
+				    pos, regs, stop);
+  alloca (0);
+  return result;
+}
+
+/* This is a separate function so that we can force an alloca cleanup
+   afterwards.  */
+static int
+re_match_2_internal (bufp, string1, size1, string2, size2, pos, regs, stop)
+     struct re_pattern_buffer *bufp;
+     const char *string1, *string2;
+     int size1, size2;
+     int pos;
+     struct re_registers *regs;
+     int stop;
+{
+  /* General temporaries.  */
+  int mcnt;
+  unsigned char *p1;
+
+  /* Just past the end of the corresponding string.  */
+  const char *end1, *end2;
+
+  /* Pointers into string1 and string2, just past the last characters in
+     each to consider matching.  */
+  const char *end_match_1, *end_match_2;
+
+  /* Where we are in the data, and the end of the current string.  */
+  const char *d, *dend;
+  
+  /* Where we are in the pattern, and the end of the pattern.  */
+  unsigned char *p = bufp->buffer;
+  register unsigned char *pend = p + bufp->used;
+
+  /* Mark the opcode just after a start_memory, so we can test for an
+     empty subpattern when we get to the stop_memory.  */
+  unsigned char *just_past_start_mem = 0;
+
+  /* We use this to map every character in the string.  */
+  char *translate = bufp->translate;
+
+  /* Failure point stack.  Each place that can handle a failure further
+     down the line pushes a failure point on this stack.  It consists of
+     restart, regend, and reg_info for all registers corresponding to
+     the subexpressions we're currently inside, plus the number of such
+     registers, and, finally, two char *'s.  The first char * is where
+     to resume scanning the pattern; the second one is where to resume
+     scanning the strings.  If the latter is zero, the failure point is
+     a ``dummy''; if a failure happens and the failure point is a dummy,
+     it gets discarded and the next next one is tried.  */
+#ifdef MATCH_MAY_ALLOCATE /* otherwise, this is global.  */
+  fail_stack_type fail_stack;
+#endif
+#ifdef DEBUG
+  static unsigned failure_id = 0;
+  unsigned nfailure_points_pushed = 0, nfailure_points_popped = 0;
+#endif
+
+  /* This holds the pointer to the failure stack, when
+     it is allocated relocatably.  */
+  fail_stack_elt_t *failure_stack_ptr;
+
+  /* We fill all the registers internally, independent of what we
+     return, for use in backreferences.  The number here includes
+     an element for register zero.  */
+  unsigned num_regs = bufp->re_nsub + 1;
+  
+  /* The currently active registers.  */
+  unsigned lowest_active_reg = NO_LOWEST_ACTIVE_REG;
+  unsigned highest_active_reg = NO_HIGHEST_ACTIVE_REG;
+
+  /* Information on the contents of registers. These are pointers into
+     the input strings; they record just what was matched (on this
+     attempt) by a subexpression part of the pattern, that is, the
+     regnum-th regstart pointer points to where in the pattern we began
+     matching and the regnum-th regend points to right after where we
+     stopped matching the regnum-th subexpression.  (The zeroth register
+     keeps track of what the whole pattern matches.)  */
+#ifdef MATCH_MAY_ALLOCATE /* otherwise, these are global.  */
+  const char **regstart, **regend;
+#endif
+
+  /* If a group that's operated upon by a repetition operator fails to
+     match anything, then the register for its start will need to be
+     restored because it will have been set to wherever in the string we
+     are when we last see its open-group operator.  Similarly for a
+     register's end.  */
+#ifdef MATCH_MAY_ALLOCATE /* otherwise, these are global.  */
+  const char **old_regstart, **old_regend;
+#endif
+
+  /* The is_active field of reg_info helps us keep track of which (possibly
+     nested) subexpressions we are currently in. The matched_something
+     field of reg_info[reg_num] helps us tell whether or not we have
+     matched any of the pattern so far this time through the reg_num-th
+     subexpression.  These two fields get reset each time through any
+     loop their register is in.  */
+#ifdef MATCH_MAY_ALLOCATE /* otherwise, this is global.  */
+  register_info_type *reg_info; 
+#endif
+
+  /* The following record the register info as found in the above
+     variables when we find a match better than any we've seen before. 
+     This happens as we backtrack through the failure points, which in
+     turn happens only if we have not yet matched the entire string. */
+  unsigned best_regs_set = false;
+#ifdef MATCH_MAY_ALLOCATE /* otherwise, these are global.  */
+  const char **best_regstart, **best_regend;
+#endif
+  
+  /* Logically, this is `best_regend[0]'.  But we don't want to have to
+     allocate space for that if we're not allocating space for anything
+     else (see below).  Also, we never need info about register 0 for
+     any of the other register vectors, and it seems rather a kludge to
+     treat `best_regend' differently than the rest.  So we keep track of
+     the end of the best match so far in a separate variable.  We
+     initialize this to NULL so that when we backtrack the first time
+     and need to test it, it's not garbage.  */
+  const char *match_end = NULL;
+
+  /* This helps SET_REGS_MATCHED avoid doing redundant work.  */
+  int set_regs_matched_done = 0;
+
+  /* Used when we pop values we don't care about.  */
+#ifdef MATCH_MAY_ALLOCATE /* otherwise, these are global.  */
+  const char **reg_dummy;
+  register_info_type *reg_info_dummy;
+#endif
+
+#ifdef DEBUG
+  /* Counts the total number of registers pushed.  */
+  unsigned num_regs_pushed = 0; 	
+#endif
+
+  DEBUG_PRINT1 ("\n\nEntering re_match_2.\n");
+  
+  INIT_FAIL_STACK ();
+  
+#ifdef MATCH_MAY_ALLOCATE
+  /* Do not bother to initialize all the register variables if there are
+     no groups in the pattern, as it takes a fair amount of time.  If
+     there are groups, we include space for register 0 (the whole
+     pattern), even though we never use it, since it simplifies the
+     array indexing.  We should fix this.  */
+  if (bufp->re_nsub)
+    {
+      regstart = REGEX_TALLOC (num_regs, const char *);
+      regend = REGEX_TALLOC (num_regs, const char *);
+      old_regstart = REGEX_TALLOC (num_regs, const char *);
+      old_regend = REGEX_TALLOC (num_regs, const char *);
+      best_regstart = REGEX_TALLOC (num_regs, const char *);
+      best_regend = REGEX_TALLOC (num_regs, const char *);
+      reg_info = REGEX_TALLOC (num_regs, register_info_type);
+      reg_dummy = REGEX_TALLOC (num_regs, const char *);
+      reg_info_dummy = REGEX_TALLOC (num_regs, register_info_type);
+
+      if (!(regstart && regend && old_regstart && old_regend && reg_info 
+            && best_regstart && best_regend && reg_dummy && reg_info_dummy)) 
+        {
+          FREE_VARIABLES ();
+          return -2;
+        }
+    }
+  else
+    {
+      /* We must initialize all our variables to NULL, so that
+         `FREE_VARIABLES' doesn't try to free them.  */
+      regstart = regend = old_regstart = old_regend = best_regstart
+        = best_regend = reg_dummy = NULL;
+      reg_info = reg_info_dummy = (register_info_type *) NULL;
+    }
+#endif /* MATCH_MAY_ALLOCATE */
+
+  /* The starting position is bogus.  */
+  if (pos < 0 || pos > size1 + size2)
+    {
+      FREE_VARIABLES ();
+      return -1;
+    }
+    
+  /* Initialize subexpression text positions to -1 to mark ones that no
+     start_memory/stop_memory has been seen for. Also initialize the
+     register information struct.  */
+  for (mcnt = 1; mcnt < num_regs; mcnt++)
+    {
+      regstart[mcnt] = regend[mcnt] 
+        = old_regstart[mcnt] = old_regend[mcnt] = REG_UNSET_VALUE;
+        
+      REG_MATCH_NULL_STRING_P (reg_info[mcnt]) = MATCH_NULL_UNSET_VALUE;
+      IS_ACTIVE (reg_info[mcnt]) = 0;
+      MATCHED_SOMETHING (reg_info[mcnt]) = 0;
+      EVER_MATCHED_SOMETHING (reg_info[mcnt]) = 0;
+    }
+  
+  /* We move `string1' into `string2' if the latter's empty -- but not if
+     `string1' is null.  */
+  if (size2 == 0 && string1 != NULL)
+    {
+      string2 = string1;
+      size2 = size1;
+      string1 = 0;
+      size1 = 0;
+    }
+  end1 = string1 + size1;
+  end2 = string2 + size2;
+
+  /* Compute where to stop matching, within the two strings.  */
+  if (stop <= size1)
+    {
+      end_match_1 = string1 + stop;
+      end_match_2 = string2;
+    }
+  else
+    {
+      end_match_1 = end1;
+      end_match_2 = string2 + stop - size1;
+    }
+
+  /* `p' scans through the pattern as `d' scans through the data. 
+     `dend' is the end of the input string that `d' points within.  `d'
+     is advanced into the following input string whenever necessary, but
+     this happens before fetching; therefore, at the beginning of the
+     loop, `d' can be pointing at the end of a string, but it cannot
+     equal `string2'.  */
+  if (size1 > 0 && pos <= size1)
+    {
+      d = string1 + pos;
+      dend = end_match_1;
+    }
+  else
+    {
+      d = string2 + pos - size1;
+      dend = end_match_2;
+    }
+
+  DEBUG_PRINT1 ("The compiled pattern is: ");
+  DEBUG_PRINT_COMPILED_PATTERN (bufp, p, pend);
+  DEBUG_PRINT1 ("The string to match is: `");
+  DEBUG_PRINT_DOUBLE_STRING (d, string1, size1, string2, size2);
+  DEBUG_PRINT1 ("'\n");
+  
+  /* This loops over pattern commands.  It exits by returning from the
+     function if the match is complete, or it drops through if the match
+     fails at this starting point in the input data.  */
+  for (;;)
+    {
+      DEBUG_PRINT2 ("\n0x%x: ", p);
+
+      if (p == pend)
+	{ /* End of pattern means we might have succeeded.  */
+          DEBUG_PRINT1 ("end of pattern ... ");
+          
+	  /* If we haven't matched the entire string, and we want the
+             longest match, try backtracking.  */
+          if (d != end_match_2)
+	    {
+	      /* 1 if this match ends in the same string (string1 or string2)
+		 as the best previous match.  */
+	      boolean same_str_p = (FIRST_STRING_P (match_end) 
+				    == MATCHING_IN_FIRST_STRING);
+	      /* 1 if this match is the best seen so far.  */
+	      boolean best_match_p;
+
+	      /* AIX compiler got confused when this was combined
+		 with the previous declaration.  */
+	      if (same_str_p)
+		best_match_p = d > match_end;
+	      else
+		best_match_p = !MATCHING_IN_FIRST_STRING;
+
+              DEBUG_PRINT1 ("backtracking.\n");
+              
+              if (!FAIL_STACK_EMPTY ())
+                { /* More failure points to try.  */
+
+                  /* If exceeds best match so far, save it.  */
+                  if (!best_regs_set || best_match_p)
+                    {
+                      best_regs_set = true;
+                      match_end = d;
+                      
+                      DEBUG_PRINT1 ("\nSAVING match as best so far.\n");
+                      
+                      for (mcnt = 1; mcnt < num_regs; mcnt++)
+                        {
+                          best_regstart[mcnt] = regstart[mcnt];
+                          best_regend[mcnt] = regend[mcnt];
+                        }
+                    }
+                  goto fail;	       
+                }
+
+              /* If no failure points, don't restore garbage.  And if
+                 last match is real best match, don't restore second
+                 best one. */
+              else if (best_regs_set && !best_match_p)
+                {
+  	        restore_best_regs:
+                  /* Restore best match.  It may happen that `dend ==
+                     end_match_1' while the restored d is in string2.
+                     For example, the pattern `x.*y.*z' against the
+                     strings `x-' and `y-z-', if the two strings are
+                     not consecutive in memory.  */
+                  DEBUG_PRINT1 ("Restoring best registers.\n");
+                  
+                  d = match_end;
+                  dend = ((d >= string1 && d <= end1)
+		           ? end_match_1 : end_match_2);
+
+		  for (mcnt = 1; mcnt < num_regs; mcnt++)
+		    {
+		      regstart[mcnt] = best_regstart[mcnt];
+		      regend[mcnt] = best_regend[mcnt];
+		    }
+                }
+            } /* d != end_match_2 */
+
+	succeed_label:
+          DEBUG_PRINT1 ("Accepting match.\n");
+
+          /* If caller wants register contents data back, do it.  */
+          if (regs && !bufp->no_sub)
+	    {
+              /* Have the register data arrays been allocated?  */
+              if (bufp->regs_allocated == REGS_UNALLOCATED)
+                { /* No.  So allocate them with malloc.  We need one
+                     extra element beyond `num_regs' for the `-1' marker
+                     GNU code uses.  */
+                  regs->num_regs = MAX (RE_NREGS, num_regs + 1);
+                  regs->start = TALLOC (regs->num_regs, regoff_t);
+                  regs->end = TALLOC (regs->num_regs, regoff_t);
+                  if (regs->start == NULL || regs->end == NULL)
+		    {
+		      FREE_VARIABLES ();
+		      return -2;
+		    }
+                  bufp->regs_allocated = REGS_REALLOCATE;
+                }
+              else if (bufp->regs_allocated == REGS_REALLOCATE)
+                { /* Yes.  If we need more elements than were already
+                     allocated, reallocate them.  If we need fewer, just
+                     leave it alone.  */
+                  if (regs->num_regs < num_regs + 1)
+                    {
+                      regs->num_regs = num_regs + 1;
+                      RETALLOC (regs->start, regs->num_regs, regoff_t);
+                      RETALLOC (regs->end, regs->num_regs, regoff_t);
+                      if (regs->start == NULL || regs->end == NULL)
+			{
+			  FREE_VARIABLES ();
+			  return -2;
+			}
+                    }
+                }
+              else
+		{
+		  /* These braces fend off a "empty body in an else-statement"
+		     warning under GCC when assert expands to nothing.  */
+		  assert (bufp->regs_allocated == REGS_FIXED);
+		}
+
+              /* Convert the pointer data in `regstart' and `regend' to
+                 indices.  Register zero has to be set differently,
+                 since we haven't kept track of any info for it.  */
+              if (regs->num_regs > 0)
+                {
+                  regs->start[0] = pos;
+                  regs->end[0] = (MATCHING_IN_FIRST_STRING
+				  ? ((regoff_t) (d - string1))
+			          : ((regoff_t) (d - string2 + size1)));
+                }
+              
+              /* Go through the first `min (num_regs, regs->num_regs)'
+                 registers, since that is all we initialized.  */
+	      for (mcnt = 1; mcnt < MIN (num_regs, regs->num_regs); mcnt++)
+		{
+                  if (REG_UNSET (regstart[mcnt]) || REG_UNSET (regend[mcnt]))
+                    regs->start[mcnt] = regs->end[mcnt] = -1;
+                  else
+                    {
+		      regs->start[mcnt]
+			= (regoff_t) POINTER_TO_OFFSET (regstart[mcnt]);
+                      regs->end[mcnt]
+			= (regoff_t) POINTER_TO_OFFSET (regend[mcnt]);
+                    }
+		}
+              
+              /* If the regs structure we return has more elements than
+                 were in the pattern, set the extra elements to -1.  If
+                 we (re)allocated the registers, this is the case,
+                 because we always allocate enough to have at least one
+                 -1 at the end.  */
+              for (mcnt = num_regs; mcnt < regs->num_regs; mcnt++)
+                regs->start[mcnt] = regs->end[mcnt] = -1;
+	    } /* regs && !bufp->no_sub */
+
+          DEBUG_PRINT4 ("%u failure points pushed, %u popped (%u remain).\n",
+                        nfailure_points_pushed, nfailure_points_popped,
+                        nfailure_points_pushed - nfailure_points_popped);
+          DEBUG_PRINT2 ("%u registers pushed.\n", num_regs_pushed);
+
+          mcnt = d - pos - (MATCHING_IN_FIRST_STRING 
+			    ? string1 
+			    : string2 - size1);
+
+          DEBUG_PRINT2 ("Returning %d from re_match_2.\n", mcnt);
+
+          FREE_VARIABLES ();
+          return mcnt;
+        }
+
+      /* Otherwise match next pattern command.  */
+      switch (SWITCH_ENUM_CAST ((re_opcode_t) *p++))
+	{
+        /* Ignore these.  Used to ignore the n of succeed_n's which
+           currently have n == 0.  */
+        case no_op:
+          DEBUG_PRINT1 ("EXECUTING no_op.\n");
+          break;
+
+	case succeed:
+          DEBUG_PRINT1 ("EXECUTING succeed.\n");
+	  goto succeed_label;
+
+        /* Match the next n pattern characters exactly.  The following
+           byte in the pattern defines n, and the n bytes after that
+           are the characters to match.  */
+	case exactn:
+	  mcnt = *p++;
+          DEBUG_PRINT2 ("EXECUTING exactn %d.\n", mcnt);
+
+          /* This is written out as an if-else so we don't waste time
+             testing `translate' inside the loop.  */
+          if (translate)
+	    {
+	      do
+		{
+		  PREFETCH ();
+		  if (translate[(unsigned char) *d++] != (char) *p++)
+                    goto fail;
+		}
+	      while (--mcnt);
+	    }
+	  else
+	    {
+	      do
+		{
+		  PREFETCH ();
+		  if (*d++ != (char) *p++) goto fail;
+		}
+	      while (--mcnt);
+	    }
+	  SET_REGS_MATCHED ();
+          break;
+
+
+        /* Match any character except possibly a newline or a null.  */
+	case anychar:
+          DEBUG_PRINT1 ("EXECUTING anychar.\n");
+
+          PREFETCH ();
+
+          if ((!(bufp->syntax & RE_DOT_NEWLINE) && TRANSLATE (*d) == '\n')
+              || (bufp->syntax & RE_DOT_NOT_NULL && TRANSLATE (*d) == '\000'))
+	    goto fail;
+
+          SET_REGS_MATCHED ();
+          DEBUG_PRINT2 ("  Matched `%d'.\n", *d);
+          d++;
+	  break;
+
+
+	case charset:
+	case charset_not:
+	  {
+	    register unsigned char c;
+	    boolean not = (re_opcode_t) *(p - 1) == charset_not;
+
+            DEBUG_PRINT2 ("EXECUTING charset%s.\n", not ? "_not" : "");
+
+	    PREFETCH ();
+	    c = TRANSLATE (*d); /* The character to match.  */
+
+            /* Cast to `unsigned' instead of `unsigned char' in case the
+               bit list is a full 32 bytes long.  */
+	    if (c < (unsigned) (*p * BYTEWIDTH)
+		&& p[1 + c / BYTEWIDTH] & (1 << (c % BYTEWIDTH)))
+	      not = !not;
+
+	    p += 1 + *p;
+
+	    if (!not) goto fail;
+            
+	    SET_REGS_MATCHED ();
+            d++;
+	    break;
+	  }
+
+
+        /* The beginning of a group is represented by start_memory.
+           The arguments are the register number in the next byte, and the
+           number of groups inner to this one in the next.  The text
+           matched within the group is recorded (in the internal
+           registers data structure) under the register number.  */
+        case start_memory:
+	  DEBUG_PRINT3 ("EXECUTING start_memory %d (%d):\n", *p, p[1]);
+
+          /* Find out if this group can match the empty string.  */
+	  p1 = p;		/* To send to group_match_null_string_p.  */
+          
+          if (REG_MATCH_NULL_STRING_P (reg_info[*p]) == MATCH_NULL_UNSET_VALUE)
+            REG_MATCH_NULL_STRING_P (reg_info[*p]) 
+              = group_match_null_string_p (&p1, pend, reg_info);
+
+          /* Save the position in the string where we were the last time
+             we were at this open-group operator in case the group is
+             operated upon by a repetition operator, e.g., with `(a*)*b'
+             against `ab'; then we want to ignore where we are now in
+             the string in case this attempt to match fails.  */
+          old_regstart[*p] = REG_MATCH_NULL_STRING_P (reg_info[*p])
+                             ? REG_UNSET (regstart[*p]) ? d : regstart[*p]
+                             : regstart[*p];
+	  DEBUG_PRINT2 ("  old_regstart: %d\n", 
+			 POINTER_TO_OFFSET (old_regstart[*p]));
+
+          regstart[*p] = d;
+	  DEBUG_PRINT2 ("  regstart: %d\n", POINTER_TO_OFFSET (regstart[*p]));
+
+          IS_ACTIVE (reg_info[*p]) = 1;
+          MATCHED_SOMETHING (reg_info[*p]) = 0;
+
+	  /* Clear this whenever we change the register activity status.  */
+	  set_regs_matched_done = 0;
+          
+          /* This is the new highest active register.  */
+          highest_active_reg = *p;
+          
+          /* If nothing was active before, this is the new lowest active
+             register.  */
+          if (lowest_active_reg == NO_LOWEST_ACTIVE_REG)
+            lowest_active_reg = *p;
+
+          /* Move past the register number and inner group count.  */
+          p += 2;
+	  just_past_start_mem = p;
+
+          break;
+
+
+        /* The stop_memory opcode represents the end of a group.  Its
+           arguments are the same as start_memory's: the register
+           number, and the number of inner groups.  */
+	case stop_memory:
+	  DEBUG_PRINT3 ("EXECUTING stop_memory %d (%d):\n", *p, p[1]);
+             
+          /* We need to save the string position the last time we were at
+             this close-group operator in case the group is operated
+             upon by a repetition operator, e.g., with `((a*)*(b*)*)*'
+             against `aba'; then we want to ignore where we are now in
+             the string in case this attempt to match fails.  */
+          old_regend[*p] = REG_MATCH_NULL_STRING_P (reg_info[*p])
+                           ? REG_UNSET (regend[*p]) ? d : regend[*p]
+			   : regend[*p];
+	  DEBUG_PRINT2 ("      old_regend: %d\n", 
+			 POINTER_TO_OFFSET (old_regend[*p]));
+
+          regend[*p] = d;
+	  DEBUG_PRINT2 ("      regend: %d\n", POINTER_TO_OFFSET (regend[*p]));
+
+          /* This register isn't active anymore.  */
+          IS_ACTIVE (reg_info[*p]) = 0;
+
+	  /* Clear this whenever we change the register activity status.  */
+	  set_regs_matched_done = 0;
+
+          /* If this was the only register active, nothing is active
+             anymore.  */
+          if (lowest_active_reg == highest_active_reg)
+            {
+              lowest_active_reg = NO_LOWEST_ACTIVE_REG;
+              highest_active_reg = NO_HIGHEST_ACTIVE_REG;
+            }
+          else
+            { /* We must scan for the new highest active register, since
+                 it isn't necessarily one less than now: consider
+                 (a(b)c(d(e)f)g).  When group 3 ends, after the f), the
+                 new highest active register is 1.  */
+              unsigned char r = *p - 1;
+              while (r > 0 && !IS_ACTIVE (reg_info[r]))
+                r--;
+              
+              /* If we end up at register zero, that means that we saved
+                 the registers as the result of an `on_failure_jump', not
+                 a `start_memory', and we jumped to past the innermost
+                 `stop_memory'.  For example, in ((.)*) we save
+                 registers 1 and 2 as a result of the *, but when we pop
+                 back to the second ), we are at the stop_memory 1.
+                 Thus, nothing is active.  */
+	      if (r == 0)
+                {
+                  lowest_active_reg = NO_LOWEST_ACTIVE_REG;
+                  highest_active_reg = NO_HIGHEST_ACTIVE_REG;
+                }
+              else
+                highest_active_reg = r;
+            }
+          
+          /* If just failed to match something this time around with a
+             group that's operated on by a repetition operator, try to
+             force exit from the ``loop'', and restore the register
+             information for this group that we had before trying this
+             last match.  */
+          if ((!MATCHED_SOMETHING (reg_info[*p])
+               || just_past_start_mem == p - 1)
+	      && (p + 2) < pend)              
+            {
+              boolean is_a_jump_n = false;
+              
+              p1 = p + 2;
+              mcnt = 0;
+              switch ((re_opcode_t) *p1++)
+                {
+                  case jump_n:
+		    is_a_jump_n = true;
+                  case pop_failure_jump:
+		  case maybe_pop_jump:
+		  case jump:
+		  case dummy_failure_jump:
+                    EXTRACT_NUMBER_AND_INCR (mcnt, p1);
+		    if (is_a_jump_n)
+		      p1 += 2;
+                    break;
+                  
+                  default:
+                    /* do nothing */ ;
+                }
+	      p1 += mcnt;
+        
+              /* If the next operation is a jump backwards in the pattern
+	         to an on_failure_jump right before the start_memory
+                 corresponding to this stop_memory, exit from the loop
+                 by forcing a failure after pushing on the stack the
+                 on_failure_jump's jump in the pattern, and d.  */
+              if (mcnt < 0 && (re_opcode_t) *p1 == on_failure_jump
+                  && (re_opcode_t) p1[3] == start_memory && p1[4] == *p)
+		{
+                  /* If this group ever matched anything, then restore
+                     what its registers were before trying this last
+                     failed match, e.g., with `(a*)*b' against `ab' for
+                     regstart[1], and, e.g., with `((a*)*(b*)*)*'
+                     against `aba' for regend[3].
+                     
+                     Also restore the registers for inner groups for,
+                     e.g., `((a*)(b*))*' against `aba' (register 3 would
+                     otherwise get trashed).  */
+                     
+                  if (EVER_MATCHED_SOMETHING (reg_info[*p]))
+		    {
+		      unsigned r; 
+        
+                      EVER_MATCHED_SOMETHING (reg_info[*p]) = 0;
+                      
+		      /* Restore this and inner groups' (if any) registers.  */
+                      for (r = *p; r < *p + *(p + 1); r++)
+                        {
+                          regstart[r] = old_regstart[r];
+
+                          /* xx why this test?  */
+                          if (old_regend[r] >= regstart[r])
+                            regend[r] = old_regend[r];
+                        }     
+                    }
+		  p1++;
+                  EXTRACT_NUMBER_AND_INCR (mcnt, p1);
+                  PUSH_FAILURE_POINT (p1 + mcnt, d, -2);
+
+                  goto fail;
+                }
+            }
+          
+          /* Move past the register number and the inner group count.  */
+          p += 2;
+          break;
+
+
+	/* \<digit> has been turned into a `duplicate' command which is
+           followed by the numeric value of <digit> as the register number.  */
+        case duplicate:
+	  {
+	    register const char *d2, *dend2;
+	    int regno = *p++;   /* Get which register to match against.  */
+	    DEBUG_PRINT2 ("EXECUTING duplicate %d.\n", regno);
+
+	    /* Can't back reference a group which we've never matched.  */
+            if (REG_UNSET (regstart[regno]) || REG_UNSET (regend[regno]))
+              goto fail;
+              
+            /* Where in input to try to start matching.  */
+            d2 = regstart[regno];
+            
+            /* Where to stop matching; if both the place to start and
+               the place to stop matching are in the same string, then
+               set to the place to stop, otherwise, for now have to use
+               the end of the first string.  */
+
+            dend2 = ((FIRST_STRING_P (regstart[regno]) 
+		      == FIRST_STRING_P (regend[regno]))
+		     ? regend[regno] : end_match_1);
+	    for (;;)
+	      {
+		/* If necessary, advance to next segment in register
+                   contents.  */
+		while (d2 == dend2)
+		  {
+		    if (dend2 == end_match_2) break;
+		    if (dend2 == regend[regno]) break;
+
+                    /* End of string1 => advance to string2. */
+                    d2 = string2;
+                    dend2 = regend[regno];
+		  }
+		/* At end of register contents => success */
+		if (d2 == dend2) break;
+
+		/* If necessary, advance to next segment in data.  */
+		PREFETCH ();
+
+		/* How many characters left in this segment to match.  */
+		mcnt = dend - d;
+                
+		/* Want how many consecutive characters we can match in
+                   one shot, so, if necessary, adjust the count.  */
+                if (mcnt > dend2 - d2)
+		  mcnt = dend2 - d2;
+                  
+		/* Compare that many; failure if mismatch, else move
+                   past them.  */
+		if (translate 
+                    ? bcmp_translate (d, d2, mcnt, translate) 
+                    : bcmp (d, d2, mcnt))
+		  goto fail;
+		d += mcnt, d2 += mcnt;
+
+		/* Do this because we've match some characters.  */
+		SET_REGS_MATCHED ();
+	      }
+	  }
+	  break;
+
+
+        /* begline matches the empty string at the beginning of the string
+           (unless `not_bol' is set in `bufp'), and, if
+           `newline_anchor' is set, after newlines.  */
+	case begline:
+          DEBUG_PRINT1 ("EXECUTING begline.\n");
+          
+          if (AT_STRINGS_BEG (d))
+            {
+              if (!bufp->not_bol) break;
+            }
+          else if (d[-1] == '\n' && bufp->newline_anchor)
+            {
+              break;
+            }
+          /* In all other cases, we fail.  */
+          goto fail;
+
+
+        /* endline is the dual of begline.  */
+	case endline:
+          DEBUG_PRINT1 ("EXECUTING endline.\n");
+
+          if (AT_STRINGS_END (d))
+            {
+              if (!bufp->not_eol) break;
+            }
+          
+          /* We have to ``prefetch'' the next character.  */
+          else if ((d == end1 ? *string2 : *d) == '\n'
+                   && bufp->newline_anchor)
+            {
+              break;
+            }
+          goto fail;
+
+
+	/* Match at the very beginning of the data.  */
+        case begbuf:
+          DEBUG_PRINT1 ("EXECUTING begbuf.\n");
+          if (AT_STRINGS_BEG (d))
+            break;
+          goto fail;
+
+
+	/* Match at the very end of the data.  */
+        case endbuf:
+          DEBUG_PRINT1 ("EXECUTING endbuf.\n");
+	  if (AT_STRINGS_END (d))
+	    break;
+          goto fail;
+
+
+        /* on_failure_keep_string_jump is used to optimize `.*\n'.  It
+           pushes NULL as the value for the string on the stack.  Then
+           `pop_failure_point' will keep the current value for the
+           string, instead of restoring it.  To see why, consider
+           matching `foo\nbar' against `.*\n'.  The .* matches the foo;
+           then the . fails against the \n.  But the next thing we want
+           to do is match the \n against the \n; if we restored the
+           string value, we would be back at the foo.
+           
+           Because this is used only in specific cases, we don't need to
+           check all the things that `on_failure_jump' does, to make
+           sure the right things get saved on the stack.  Hence we don't
+           share its code.  The only reason to push anything on the
+           stack at all is that otherwise we would have to change
+           `anychar's code to do something besides goto fail in this
+           case; that seems worse than this.  */
+        case on_failure_keep_string_jump:
+          DEBUG_PRINT1 ("EXECUTING on_failure_keep_string_jump");
+          
+          EXTRACT_NUMBER_AND_INCR (mcnt, p);
+          DEBUG_PRINT3 (" %d (to 0x%x):\n", mcnt, p + mcnt);
+
+          PUSH_FAILURE_POINT (p + mcnt, NULL, -2);
+          break;
+
+
+	/* Uses of on_failure_jump:
+        
+           Each alternative starts with an on_failure_jump that points
+           to the beginning of the next alternative.  Each alternative
+           except the last ends with a jump that in effect jumps past
+           the rest of the alternatives.  (They really jump to the
+           ending jump of the following alternative, because tensioning
+           these jumps is a hassle.)
+
+           Repeats start with an on_failure_jump that points past both
+           the repetition text and either the following jump or
+           pop_failure_jump back to this on_failure_jump.  */
+	case on_failure_jump:
+        on_failure:
+          DEBUG_PRINT1 ("EXECUTING on_failure_jump");
+
+          EXTRACT_NUMBER_AND_INCR (mcnt, p);
+          DEBUG_PRINT3 (" %d (to 0x%x)", mcnt, p + mcnt);
+
+          /* If this on_failure_jump comes right before a group (i.e.,
+             the original * applied to a group), save the information
+             for that group and all inner ones, so that if we fail back
+             to this point, the group's information will be correct.
+             For example, in \(a*\)*\1, we need the preceding group,
+             and in \(\(a*\)b*\)\2, we need the inner group.  */
+
+          /* We can't use `p' to check ahead because we push
+             a failure point to `p + mcnt' after we do this.  */
+          p1 = p;
+
+          /* We need to skip no_op's before we look for the
+             start_memory in case this on_failure_jump is happening as
+             the result of a completed succeed_n, as in \(a\)\{1,3\}b\1
+             against aba.  */
+          while (p1 < pend && (re_opcode_t) *p1 == no_op)
+            p1++;
+
+          if (p1 < pend && (re_opcode_t) *p1 == start_memory)
+            {
+              /* We have a new highest active register now.  This will
+                 get reset at the start_memory we are about to get to,
+                 but we will have saved all the registers relevant to
+                 this repetition op, as described above.  */
+              highest_active_reg = *(p1 + 1) + *(p1 + 2);
+              if (lowest_active_reg == NO_LOWEST_ACTIVE_REG)
+                lowest_active_reg = *(p1 + 1);
+            }
+
+          DEBUG_PRINT1 (":\n");
+          PUSH_FAILURE_POINT (p + mcnt, d, -2);
+          break;
+
+
+        /* A smart repeat ends with `maybe_pop_jump'.
+	   We change it to either `pop_failure_jump' or `jump'.  */
+        case maybe_pop_jump:
+          EXTRACT_NUMBER_AND_INCR (mcnt, p);
+          DEBUG_PRINT2 ("EXECUTING maybe_pop_jump %d.\n", mcnt);
+          {
+	    register unsigned char *p2 = p;
+
+            /* Compare the beginning of the repeat with what in the
+               pattern follows its end. If we can establish that there
+               is nothing that they would both match, i.e., that we
+               would have to backtrack because of (as in, e.g., `a*a')
+               then we can change to pop_failure_jump, because we'll
+               never have to backtrack.
+               
+               This is not true in the case of alternatives: in
+               `(a|ab)*' we do need to backtrack to the `ab' alternative
+               (e.g., if the string was `ab').  But instead of trying to
+               detect that here, the alternative has put on a dummy
+               failure point which is what we will end up popping.  */
+
+	    /* Skip over open/close-group commands.
+	       If what follows this loop is a ...+ construct,
+	       look at what begins its body, since we will have to
+	       match at least one of that.  */
+	    while (1)
+	      {
+		if (p2 + 2 < pend
+		    && ((re_opcode_t) *p2 == stop_memory
+			|| (re_opcode_t) *p2 == start_memory))
+		  p2 += 3;
+		else if (p2 + 6 < pend
+			 && (re_opcode_t) *p2 == dummy_failure_jump)
+		  p2 += 6;
+		else
+		  break;
+	      }
+
+	    p1 = p + mcnt;
+	    /* p1[0] ... p1[2] are the `on_failure_jump' corresponding
+	       to the `maybe_finalize_jump' of this case.  Examine what 
+	       follows.  */
+
+            /* If we're at the end of the pattern, we can change.  */
+            if (p2 == pend)
+	      {
+		/* Consider what happens when matching ":\(.*\)"
+		   against ":/".  I don't really understand this code
+		   yet.  */
+  	        p[-3] = (unsigned char) pop_failure_jump;
+                DEBUG_PRINT1
+                  ("  End of pattern: change to `pop_failure_jump'.\n");
+              }
+
+            else if ((re_opcode_t) *p2 == exactn
+		     || (bufp->newline_anchor && (re_opcode_t) *p2 == endline))
+	      {
+		register unsigned char c
+                  = *p2 == (unsigned char) endline ? '\n' : p2[2];
+
+                if ((re_opcode_t) p1[3] == exactn && p1[5] != c)
+                  {
+  		    p[-3] = (unsigned char) pop_failure_jump;
+                    DEBUG_PRINT3 ("  %c != %c => pop_failure_jump.\n",
+                                  c, p1[5]);
+                  }
+                  
+		else if ((re_opcode_t) p1[3] == charset
+			 || (re_opcode_t) p1[3] == charset_not)
+		  {
+		    int not = (re_opcode_t) p1[3] == charset_not;
+                    
+		    if (c < (unsigned char) (p1[4] * BYTEWIDTH)
+			&& p1[5 + c / BYTEWIDTH] & (1 << (c % BYTEWIDTH)))
+		      not = !not;
+
+                    /* `not' is equal to 1 if c would match, which means
+                        that we can't change to pop_failure_jump.  */
+		    if (!not)
+                      {
+  		        p[-3] = (unsigned char) pop_failure_jump;
+                        DEBUG_PRINT1 ("  No match => pop_failure_jump.\n");
+                      }
+		  }
+	      }
+            else if ((re_opcode_t) *p2 == charset)
+	      {
+#ifdef DEBUG
+		register unsigned char c
+                  = *p2 == (unsigned char) endline ? '\n' : p2[2];
+#endif
+
+                if ((re_opcode_t) p1[3] == exactn
+		    && ! ((int) p2[1] * BYTEWIDTH > (int) p1[4]
+			  && (p2[1 + p1[4] / BYTEWIDTH]
+			      & (1 << (p1[4] % BYTEWIDTH)))))
+                  {
+  		    p[-3] = (unsigned char) pop_failure_jump;
+                    DEBUG_PRINT3 ("  %c != %c => pop_failure_jump.\n",
+                                  c, p1[5]);
+                  }
+                  
+		else if ((re_opcode_t) p1[3] == charset_not)
+		  {
+		    int idx;
+		    /* We win if the charset_not inside the loop
+		       lists every character listed in the charset after.  */
+		    for (idx = 0; idx < (int) p2[1]; idx++)
+		      if (! (p2[2 + idx] == 0
+			     || (idx < (int) p1[4]
+				 && ((p2[2 + idx] & ~ p1[5 + idx]) == 0))))
+			break;
+
+		    if (idx == p2[1])
+                      {
+  		        p[-3] = (unsigned char) pop_failure_jump;
+                        DEBUG_PRINT1 ("  No match => pop_failure_jump.\n");
+                      }
+		  }
+		else if ((re_opcode_t) p1[3] == charset)
+		  {
+		    int idx;
+		    /* We win if the charset inside the loop
+		       has no overlap with the one after the loop.  */
+		    for (idx = 0;
+			 idx < (int) p2[1] && idx < (int) p1[4];
+			 idx++)
+		      if ((p2[2 + idx] & p1[5 + idx]) != 0)
+			break;
+
+		    if (idx == p2[1] || idx == p1[4])
+                      {
+  		        p[-3] = (unsigned char) pop_failure_jump;
+                        DEBUG_PRINT1 ("  No match => pop_failure_jump.\n");
+                      }
+		  }
+	      }
+	  }
+	  p -= 2;		/* Point at relative address again.  */
+	  if ((re_opcode_t) p[-1] != pop_failure_jump)
+	    {
+	      p[-1] = (unsigned char) jump;
+              DEBUG_PRINT1 ("  Match => jump.\n");
+	      goto unconditional_jump;
+	    }
+        /* Note fall through.  */
+
+
+	/* The end of a simple repeat has a pop_failure_jump back to
+           its matching on_failure_jump, where the latter will push a
+           failure point.  The pop_failure_jump takes off failure
+           points put on by this pop_failure_jump's matching
+           on_failure_jump; we got through the pattern to here from the
+           matching on_failure_jump, so didn't fail.  */
+        case pop_failure_jump:
+          {
+            /* We need to pass separate storage for the lowest and
+               highest registers, even though we don't care about the
+               actual values.  Otherwise, we will restore only one
+               register from the stack, since lowest will == highest in
+               `pop_failure_point'.  */
+            unsigned dummy_low_reg, dummy_high_reg;
+            unsigned char *pdummy;
+            const char *sdummy;
+
+            DEBUG_PRINT1 ("EXECUTING pop_failure_jump.\n");
+            POP_FAILURE_POINT (sdummy, pdummy,
+                               dummy_low_reg, dummy_high_reg,
+                               reg_dummy, reg_dummy, reg_info_dummy);
+          }
+          /* Note fall through.  */
+
+          
+        /* Unconditionally jump (without popping any failure points).  */
+        case jump:
+	unconditional_jump:
+	  EXTRACT_NUMBER_AND_INCR (mcnt, p);	/* Get the amount to jump.  */
+          DEBUG_PRINT2 ("EXECUTING jump %d ", mcnt);
+	  p += mcnt;				/* Do the jump.  */
+          DEBUG_PRINT2 ("(to 0x%x).\n", p);
+	  break;
+
+	
+        /* We need this opcode so we can detect where alternatives end
+           in `group_match_null_string_p' et al.  */
+        case jump_past_alt:
+          DEBUG_PRINT1 ("EXECUTING jump_past_alt.\n");
+          goto unconditional_jump;
+
+
+        /* Normally, the on_failure_jump pushes a failure point, which
+           then gets popped at pop_failure_jump.  We will end up at
+           pop_failure_jump, also, and with a pattern of, say, `a+', we
+           are skipping over the on_failure_jump, so we have to push
+           something meaningless for pop_failure_jump to pop.  */
+        case dummy_failure_jump:
+          DEBUG_PRINT1 ("EXECUTING dummy_failure_jump.\n");
+          /* It doesn't matter what we push for the string here.  What
+             the code at `fail' tests is the value for the pattern.  */
+          PUSH_FAILURE_POINT (0, 0, -2);
+          goto unconditional_jump;
+
+
+        /* At the end of an alternative, we need to push a dummy failure
+           point in case we are followed by a `pop_failure_jump', because
+           we don't want the failure point for the alternative to be
+           popped.  For example, matching `(a|ab)*' against `aab'
+           requires that we match the `ab' alternative.  */
+        case push_dummy_failure:
+          DEBUG_PRINT1 ("EXECUTING push_dummy_failure.\n");
+          /* See comments just above at `dummy_failure_jump' about the
+             two zeroes.  */
+          PUSH_FAILURE_POINT (0, 0, -2);
+          break;
+
+        /* Have to succeed matching what follows at least n times.
+           After that, handle like `on_failure_jump'.  */
+        case succeed_n: 
+          EXTRACT_NUMBER (mcnt, p + 2);
+          DEBUG_PRINT2 ("EXECUTING succeed_n %d.\n", mcnt);
+
+          assert (mcnt >= 0);
+          /* Originally, this is how many times we HAVE to succeed.  */
+          if (mcnt > 0)
+            {
+               mcnt--;
+	       p += 2;
+               STORE_NUMBER_AND_INCR (p, mcnt);
+               DEBUG_PRINT3 ("  Setting 0x%x to %d.\n", p, mcnt);
+            }
+	  else if (mcnt == 0)
+            {
+              DEBUG_PRINT2 ("  Setting two bytes from 0x%x to no_op.\n", p+2);
+	      p[2] = (unsigned char) no_op;
+              p[3] = (unsigned char) no_op;
+              goto on_failure;
+            }
+          break;
+        
+        case jump_n: 
+          EXTRACT_NUMBER (mcnt, p + 2);
+          DEBUG_PRINT2 ("EXECUTING jump_n %d.\n", mcnt);
+
+          /* Originally, this is how many times we CAN jump.  */
+          if (mcnt)
+            {
+               mcnt--;
+               STORE_NUMBER (p + 2, mcnt);
+	       goto unconditional_jump;	     
+            }
+          /* If don't have to jump any more, skip over the rest of command.  */
+	  else      
+	    p += 4;		     
+          break;
+        
+	case set_number_at:
+	  {
+            DEBUG_PRINT1 ("EXECUTING set_number_at.\n");
+
+            EXTRACT_NUMBER_AND_INCR (mcnt, p);
+            p1 = p + mcnt;
+            EXTRACT_NUMBER_AND_INCR (mcnt, p);
+            DEBUG_PRINT3 ("  Setting 0x%x to %d.\n", p1, mcnt);
+	    STORE_NUMBER (p1, mcnt);
+            break;
+          }
+
+        case wordbound:
+          DEBUG_PRINT1 ("EXECUTING wordbound.\n");
+          if (AT_WORD_BOUNDARY (d))
+	    break;
+          goto fail;
+
+	case notwordbound:
+          DEBUG_PRINT1 ("EXECUTING notwordbound.\n");
+	  if (AT_WORD_BOUNDARY (d))
+	    goto fail;
+          break;
+
+	case wordbeg:
+          DEBUG_PRINT1 ("EXECUTING wordbeg.\n");
+	  if (WORDCHAR_P (d) && (AT_STRINGS_BEG (d) || !WORDCHAR_P (d - 1)))
+	    break;
+          goto fail;
+
+	case wordend:
+          DEBUG_PRINT1 ("EXECUTING wordend.\n");
+	  if (!AT_STRINGS_BEG (d) && WORDCHAR_P (d - 1)
+              && (!WORDCHAR_P (d) || AT_STRINGS_END (d)))
+	    break;
+          goto fail;
+
+#ifdef emacs
+  	case before_dot:
+          DEBUG_PRINT1 ("EXECUTING before_dot.\n");
+ 	  if (PTR_CHAR_POS ((unsigned char *) d) >= point)
+  	    goto fail;
+  	  break;
+  
+  	case at_dot:
+          DEBUG_PRINT1 ("EXECUTING at_dot.\n");
+ 	  if (PTR_CHAR_POS ((unsigned char *) d) != point)
+  	    goto fail;
+  	  break;
+  
+  	case after_dot:
+          DEBUG_PRINT1 ("EXECUTING after_dot.\n");
+          if (PTR_CHAR_POS ((unsigned char *) d) <= point)
+  	    goto fail;
+  	  break;
+#if 0 /* not emacs19 */
+	case at_dot:
+          DEBUG_PRINT1 ("EXECUTING at_dot.\n");
+	  if (PTR_CHAR_POS ((unsigned char *) d) + 1 != point)
+	    goto fail;
+	  break;
+#endif /* not emacs19 */
+
+	case syntaxspec:
+          DEBUG_PRINT2 ("EXECUTING syntaxspec %d.\n", mcnt);
+	  mcnt = *p++;
+	  goto matchsyntax;
+
+        case wordchar:
+          DEBUG_PRINT1 ("EXECUTING Emacs wordchar.\n");
+	  mcnt = (int) Sword;
+        matchsyntax:
+	  PREFETCH ();
+	  /* Can't use *d++ here; SYNTAX may be an unsafe macro.  */
+	  d++;
+	  if (SYNTAX (d[-1]) != (enum syntaxcode) mcnt)
+	    goto fail;
+          SET_REGS_MATCHED ();
+	  break;
+
+	case notsyntaxspec:
+          DEBUG_PRINT2 ("EXECUTING notsyntaxspec %d.\n", mcnt);
+	  mcnt = *p++;
+	  goto matchnotsyntax;
+
+        case notwordchar:
+          DEBUG_PRINT1 ("EXECUTING Emacs notwordchar.\n");
+	  mcnt = (int) Sword;
+        matchnotsyntax:
+	  PREFETCH ();
+	  /* Can't use *d++ here; SYNTAX may be an unsafe macro.  */
+	  d++;
+	  if (SYNTAX (d[-1]) == (enum syntaxcode) mcnt)
+	    goto fail;
+	  SET_REGS_MATCHED ();
+          break;
+
+#else /* not emacs */
+	case wordchar:
+          DEBUG_PRINT1 ("EXECUTING non-Emacs wordchar.\n");
+	  PREFETCH ();
+          if (!WORDCHAR_P (d))
+            goto fail;
+	  SET_REGS_MATCHED ();
+          d++;
+	  break;
+	  
+	case notwordchar:
+          DEBUG_PRINT1 ("EXECUTING non-Emacs notwordchar.\n");
+	  PREFETCH ();
+	  if (WORDCHAR_P (d))
+            goto fail;
+          SET_REGS_MATCHED ();
+          d++;
+	  break;
+#endif /* not emacs */
+          
+        default:
+          abort ();
+	}
+      continue;  /* Successfully executed one pattern command; keep going.  */
+
+
+    /* We goto here if a matching operation fails. */
+    fail:
+      if (!FAIL_STACK_EMPTY ())
+	{ /* A restart point is known.  Restore to that state.  */
+          DEBUG_PRINT1 ("\nFAIL:\n");
+          POP_FAILURE_POINT (d, p,
+                             lowest_active_reg, highest_active_reg,
+                             regstart, regend, reg_info);
+
+          /* If this failure point is a dummy, try the next one.  */
+          if (!p)
+	    goto fail;
+
+          /* If we failed to the end of the pattern, don't examine *p.  */
+	  assert (p <= pend);
+          if (p < pend)
+            {
+              boolean is_a_jump_n = false;
+              
+              /* If failed to a backwards jump that's part of a repetition
+                 loop, need to pop this failure point and use the next one.  */
+              switch ((re_opcode_t) *p)
+                {
+                case jump_n:
+                  is_a_jump_n = true;
+                case maybe_pop_jump:
+                case pop_failure_jump:
+                case jump:
+                  p1 = p + 1;
+                  EXTRACT_NUMBER_AND_INCR (mcnt, p1);
+                  p1 += mcnt;	
+
+                  if ((is_a_jump_n && (re_opcode_t) *p1 == succeed_n)
+                      || (!is_a_jump_n
+                          && (re_opcode_t) *p1 == on_failure_jump))
+                    goto fail;
+                  break;
+                default:
+                  /* do nothing */ ;
+                }
+            }
+
+          if (d >= string1 && d <= end1)
+	    dend = end_match_1;
+        }
+      else
+        break;   /* Matching at this starting point really fails.  */
+    } /* for (;;) */
+
+  if (best_regs_set)
+    goto restore_best_regs;
+
+  FREE_VARIABLES ();
+
+  return -1;         			/* Failure to match.  */
+} /* re_match_2 */
+
+
+/* Subroutine definitions for re_match_2.  */
+
+
+/* We are passed P pointing to a register number after a start_memory.
+   
+   Return true if the pattern up to the corresponding stop_memory can
+   match the empty string, and false otherwise.
+   
+   If we find the matching stop_memory, sets P to point to one past its number.
+   Otherwise, sets P to an undefined byte less than or equal to END.
+
+   We don't handle duplicates properly (yet).  */
+
+static boolean
+group_match_null_string_p (p, end, reg_info)
+    unsigned char **p, *end;
+    register_info_type *reg_info;
+{
+  int mcnt;
+  /* Point to after the args to the start_memory.  */
+  unsigned char *p1 = *p + 2;
+  
+  while (p1 < end)
+    {
+      /* Skip over opcodes that can match nothing, and return true or
+	 false, as appropriate, when we get to one that can't, or to the
+         matching stop_memory.  */
+      
+      switch ((re_opcode_t) *p1)
+        {
+        /* Could be either a loop or a series of alternatives.  */
+        case on_failure_jump:
+          p1++;
+          EXTRACT_NUMBER_AND_INCR (mcnt, p1);
+          
+          /* If the next operation is not a jump backwards in the
+	     pattern.  */
+
+	  if (mcnt >= 0)
+	    {
+              /* Go through the on_failure_jumps of the alternatives,
+                 seeing if any of the alternatives cannot match nothing.
+                 The last alternative starts with only a jump,
+                 whereas the rest start with on_failure_jump and end
+                 with a jump, e.g., here is the pattern for `a|b|c':
+
+                 /on_failure_jump/0/6/exactn/1/a/jump_past_alt/0/6
+                 /on_failure_jump/0/6/exactn/1/b/jump_past_alt/0/3
+                 /exactn/1/c						
+
+                 So, we have to first go through the first (n-1)
+                 alternatives and then deal with the last one separately.  */
+
+
+              /* Deal with the first (n-1) alternatives, which start
+                 with an on_failure_jump (see above) that jumps to right
+                 past a jump_past_alt.  */
+
+              while ((re_opcode_t) p1[mcnt-3] == jump_past_alt)
+                {
+                  /* `mcnt' holds how many bytes long the alternative
+                     is, including the ending `jump_past_alt' and
+                     its number.  */
+
+                  if (!alt_match_null_string_p (p1, p1 + mcnt - 3, 
+				                      reg_info))
+                    return false;
+
+                  /* Move to right after this alternative, including the
+		     jump_past_alt.  */
+                  p1 += mcnt;	
+
+                  /* Break if it's the beginning of an n-th alternative
+                     that doesn't begin with an on_failure_jump.  */
+                  if ((re_opcode_t) *p1 != on_failure_jump)
+                    break;
+		
+		  /* Still have to check that it's not an n-th
+		     alternative that starts with an on_failure_jump.  */
+		  p1++;
+                  EXTRACT_NUMBER_AND_INCR (mcnt, p1);
+                  if ((re_opcode_t) p1[mcnt-3] != jump_past_alt)
+                    {
+		      /* Get to the beginning of the n-th alternative.  */
+                      p1 -= 3;
+                      break;
+                    }
+                }
+
+              /* Deal with the last alternative: go back and get number
+                 of the `jump_past_alt' just before it.  `mcnt' contains
+                 the length of the alternative.  */
+              EXTRACT_NUMBER (mcnt, p1 - 2);
+
+              if (!alt_match_null_string_p (p1, p1 + mcnt, reg_info))
+                return false;
+
+              p1 += mcnt;	/* Get past the n-th alternative.  */
+            } /* if mcnt > 0 */
+          break;
+
+          
+        case stop_memory:
+	  assert (p1[1] == **p);
+          *p = p1 + 2;
+          return true;
+
+        
+        default: 
+          if (!common_op_match_null_string_p (&p1, end, reg_info))
+            return false;
+        }
+    } /* while p1 < end */
+
+  return false;
+} /* group_match_null_string_p */
+
+
+/* Similar to group_match_null_string_p, but doesn't deal with alternatives:
+   It expects P to be the first byte of a single alternative and END one
+   byte past the last. The alternative can contain groups.  */
+   
+static boolean
+alt_match_null_string_p (p, end, reg_info)
+    unsigned char *p, *end;
+    register_info_type *reg_info;
+{
+  int mcnt;
+  unsigned char *p1 = p;
+  
+  while (p1 < end)
+    {
+      /* Skip over opcodes that can match nothing, and break when we get 
+         to one that can't.  */
+      
+      switch ((re_opcode_t) *p1)
+        {
+	/* It's a loop.  */
+        case on_failure_jump:
+          p1++;
+          EXTRACT_NUMBER_AND_INCR (mcnt, p1);
+          p1 += mcnt;
+          break;
+          
+	default: 
+          if (!common_op_match_null_string_p (&p1, end, reg_info))
+            return false;
+        }
+    }  /* while p1 < end */
+
+  return true;
+} /* alt_match_null_string_p */
+
+
+/* Deals with the ops common to group_match_null_string_p and
+   alt_match_null_string_p.  
+   
+   Sets P to one after the op and its arguments, if any.  */
+
+static boolean
+common_op_match_null_string_p (p, end, reg_info)
+    unsigned char **p, *end;
+    register_info_type *reg_info;
+{
+  int mcnt;
+  boolean ret;
+  int reg_no;
+  unsigned char *p1 = *p;
+
+  switch ((re_opcode_t) *p1++)
+    {
+    case no_op:
+    case begline:
+    case endline:
+    case begbuf:
+    case endbuf:
+    case wordbeg:
+    case wordend:
+    case wordbound:
+    case notwordbound:
+#ifdef emacs
+    case before_dot:
+    case at_dot:
+    case after_dot:
+#endif
+      break;
+
+    case start_memory:
+      reg_no = *p1;
+      assert (reg_no > 0 && reg_no <= MAX_REGNUM);
+      ret = group_match_null_string_p (&p1, end, reg_info);
+      
+      /* Have to set this here in case we're checking a group which
+         contains a group and a back reference to it.  */
+
+      if (REG_MATCH_NULL_STRING_P (reg_info[reg_no]) == MATCH_NULL_UNSET_VALUE)
+        REG_MATCH_NULL_STRING_P (reg_info[reg_no]) = ret;
+
+      if (!ret)
+        return false;
+      break;
+          
+    /* If this is an optimized succeed_n for zero times, make the jump.  */
+    case jump:
+      EXTRACT_NUMBER_AND_INCR (mcnt, p1);
+      if (mcnt >= 0)
+        p1 += mcnt;
+      else
+        return false;
+      break;
+
+    case succeed_n:
+      /* Get to the number of times to succeed.  */
+      p1 += 2;		
+      EXTRACT_NUMBER_AND_INCR (mcnt, p1);
+
+      if (mcnt == 0)
+        {
+          p1 -= 4;
+          EXTRACT_NUMBER_AND_INCR (mcnt, p1);
+          p1 += mcnt;
+        }
+      else
+        return false;
+      break;
+
+    case duplicate: 
+      if (!REG_MATCH_NULL_STRING_P (reg_info[*p1]))
+        return false;
+      break;
+
+    case set_number_at:
+      p1 += 4;
+
+    default:
+      /* All other opcodes mean we cannot match the empty string.  */
+      return false;
+  }
+
+  *p = p1;
+  return true;
+} /* common_op_match_null_string_p */
+
+
+/* Return zero if TRANSLATE[S1] and TRANSLATE[S2] are identical for LEN
+   bytes; nonzero otherwise.  */
+   
+static int
+bcmp_translate (s1, s2, len, translate)
+     unsigned char *s1, *s2;
+     register int len;
+     char *translate;
+{
+  register unsigned char *p1 = s1, *p2 = s2;
+  while (len)
+    {
+      if (translate[*p1++] != translate[*p2++]) return 1;
+      len--;
+    }
+  return 0;
+}
+
+
+/* Entry points for GNU code.  */
+
+/* re_compile_pattern is the GNU regular expression compiler: it
+   compiles PATTERN (of length SIZE) and puts the result in BUFP.
+   Returns 0 if the pattern was valid, otherwise an error string.
+   
+   Assumes the `allocated' (and perhaps `buffer') and `translate' fields
+   are set in BUFP on entry.
+   
+   We call regex_compile to do the actual compilation.  */
+
+const char *
+re_compile_pattern (pattern, length, bufp)
+     const char *pattern;
+     int length;
+     struct re_pattern_buffer *bufp;
+{
+  reg_errcode_t ret;
+  
+  /* GNU code is written to assume at least RE_NREGS registers will be set
+     (and at least one extra will be -1).  */
+  bufp->regs_allocated = REGS_UNALLOCATED;
+  
+  /* And GNU code determines whether or not to get register information
+     by passing null for the REGS argument to re_match, etc., not by
+     setting no_sub.  */
+  bufp->no_sub = 0;
+  
+  /* Match anchors at newline.  */
+  bufp->newline_anchor = 1;
+  
+  ret = regex_compile (pattern, length, re_syntax_options, bufp);
+
+  if (!ret)
+    return NULL;
+  return gettext (re_error_msgid[(int) ret]);
+}     
+
+
+/* Entry points compatible with 4.2 BSD regex library.  We don't define
+   them unless specifically requested.  */
+
+#ifdef _REGEX_RE_COMP
+
+/* BSD has one and only one pattern buffer.  */
+static struct re_pattern_buffer re_comp_buf;
+
+char *
+re_comp (s)
+    const char *s;
+{
+  reg_errcode_t ret;
+  
+  if (!s)
+    {
+      if (!re_comp_buf.buffer)
+	return gettext ("No previous regular expression");
+      return 0;
+    }
+
+  if (!re_comp_buf.buffer)
+    {
+      re_comp_buf.buffer = (unsigned char *) malloc (200);
+      if (re_comp_buf.buffer == NULL)
+        return gettext (re_error_msgid[(int) REG_ESPACE]);
+      re_comp_buf.allocated = 200;
+
+      re_comp_buf.fastmap = (char *) malloc (1 << BYTEWIDTH);
+      if (re_comp_buf.fastmap == NULL)
+	return gettext (re_error_msgid[(int) REG_ESPACE]);
+    }
+
+  /* Since `re_exec' always passes NULL for the `regs' argument, we
+     don't need to initialize the pattern buffer fields which affect it.  */
+
+  /* Match anchors at newlines.  */
+  re_comp_buf.newline_anchor = 1;
+
+  ret = regex_compile (s, strlen (s), re_syntax_options, &re_comp_buf);
+  
+  if (!ret)
+    return NULL;
+
+  /* Yes, we're discarding `const' here if !HAVE_LIBINTL.  */
+  return (char *) gettext (re_error_msgid[(int) ret]);
+}
+
+
+int
+re_exec (s)
+    const char *s;
+{
+  const int len = strlen (s);
+  return
+    0 <= re_search (&re_comp_buf, s, len, 0, len, (struct re_registers *) 0);
+}
+#endif /* _REGEX_RE_COMP */
+
+
+/* POSIX.2 functions.  Don't define these for Emacs.  */
+
+#ifndef emacs
+
+/* regcomp takes a regular expression as a string and compiles it.
+
+   PREG is a regex_t *.  We do not expect any fields to be initialized,
+   since POSIX says we shouldn't.  Thus, we set
+
+     `buffer' to the compiled pattern;
+     `used' to the length of the compiled pattern;
+     `syntax' to RE_SYNTAX_POSIX_EXTENDED if the
+       REG_EXTENDED bit in CFLAGS is set; otherwise, to
+       RE_SYNTAX_POSIX_BASIC;
+     `newline_anchor' to REG_NEWLINE being set in CFLAGS;
+     `fastmap' and `fastmap_accurate' to zero;
+     `re_nsub' to the number of subexpressions in PATTERN.
+
+   PATTERN is the address of the pattern string.
+
+   CFLAGS is a series of bits which affect compilation.
+
+     If REG_EXTENDED is set, we use POSIX extended syntax; otherwise, we
+     use POSIX basic syntax.
+
+     If REG_NEWLINE is set, then . and [^...] don't match newline.
+     Also, regexec will try a match beginning after every newline.
+
+     If REG_ICASE is set, then we considers upper- and lowercase
+     versions of letters to be equivalent when matching.
+
+     If REG_NOSUB is set, then when PREG is passed to regexec, that
+     routine will report only success or failure, and nothing about the
+     registers.
+
+   It returns 0 if it succeeds, nonzero if it doesn't.  (See regex.h for
+   the return codes and their meanings.)  */
+
+int
+regcomp (preg, pattern, cflags)
+    regex_t *preg;
+    const char *pattern; 
+    int cflags;
+{
+  reg_errcode_t ret;
+  unsigned syntax
+    = (cflags & REG_EXTENDED) ?
+      RE_SYNTAX_POSIX_EXTENDED : RE_SYNTAX_POSIX_BASIC;
+
+  /* regex_compile will allocate the space for the compiled pattern.  */
+  preg->buffer = 0;
+  preg->allocated = 0;
+  preg->used = 0;
+  
+  /* Don't bother to use a fastmap when searching.  This simplifies the
+     REG_NEWLINE case: if we used a fastmap, we'd have to put all the
+     characters after newlines into the fastmap.  This way, we just try
+     every character.  */
+  preg->fastmap = 0;
+  
+  if (cflags & REG_ICASE)
+    {
+      unsigned i;
+      
+      preg->translate = (char *) malloc (CHAR_SET_SIZE);
+      if (preg->translate == NULL)
+        return (int) REG_ESPACE;
+
+      /* Map uppercase characters to corresponding lowercase ones.  */
+      for (i = 0; i < CHAR_SET_SIZE; i++)
+        preg->translate[i] = ISUPPER (i) ? tolower (i) : i;
+    }
+  else
+    preg->translate = NULL;
+
+  /* If REG_NEWLINE is set, newlines are treated differently.  */
+  if (cflags & REG_NEWLINE)
+    { /* REG_NEWLINE implies neither . nor [^...] match newline.  */
+      syntax &= ~RE_DOT_NEWLINE;
+      syntax |= RE_HAT_LISTS_NOT_NEWLINE;
+      /* It also changes the matching behavior.  */
+      preg->newline_anchor = 1;
+    }
+  else
+    preg->newline_anchor = 0;
+
+  preg->no_sub = !!(cflags & REG_NOSUB);
+
+  /* POSIX says a null character in the pattern terminates it, so we 
+     can use strlen here in compiling the pattern.  */
+  ret = regex_compile (pattern, strlen (pattern), syntax, preg);
+  
+  /* POSIX doesn't distinguish between an unmatched open-group and an
+     unmatched close-group: both are REG_EPAREN.  */
+  if (ret == REG_ERPAREN) ret = REG_EPAREN;
+  
+  return (int) ret;
+}
+
+
+/* regexec searches for a given pattern, specified by PREG, in the
+   string STRING.
+   
+   If NMATCH is zero or REG_NOSUB was set in the cflags argument to
+   `regcomp', we ignore PMATCH.  Otherwise, we assume PMATCH has at
+   least NMATCH elements, and we set them to the offsets of the
+   corresponding matched substrings.
+   
+   EFLAGS specifies `execution flags' which affect matching: if
+   REG_NOTBOL is set, then ^ does not match at the beginning of the
+   string; if REG_NOTEOL is set, then $ does not match at the end.
+   
+   We return 0 if we find a match and REG_NOMATCH if not.  */
+
+int
+regexec (preg, string, nmatch, pmatch, eflags)
+    const regex_t *preg;
+    const char *string; 
+    size_t nmatch; 
+    regmatch_t pmatch[]; 
+    int eflags;
+{
+  int ret;
+  struct re_registers regs;
+  regex_t private_preg;
+  int len = strlen (string);
+  boolean want_reg_info = !preg->no_sub && nmatch > 0;
+
+  private_preg = *preg;
+  
+  private_preg.not_bol = !!(eflags & REG_NOTBOL);
+  private_preg.not_eol = !!(eflags & REG_NOTEOL);
+  
+  /* The user has told us exactly how many registers to return
+     information about, via `nmatch'.  We have to pass that on to the
+     matching routines.  */
+  private_preg.regs_allocated = REGS_FIXED;
+  
+  if (want_reg_info)
+    {
+      regs.num_regs = nmatch;
+      regs.start = TALLOC (nmatch, regoff_t);
+      regs.end = TALLOC (nmatch, regoff_t);
+      if (regs.start == NULL || regs.end == NULL)
+        return (int) REG_NOMATCH;
+    }
+
+  /* Perform the searching operation.  */
+  ret = re_search (&private_preg, string, len,
+                   /* start: */ 0, /* range: */ len,
+                   want_reg_info ? &regs : (struct re_registers *) 0);
+  
+  /* Copy the register information to the POSIX structure.  */
+  if (want_reg_info)
+    {
+      if (ret >= 0)
+        {
+          unsigned r;
+
+          for (r = 0; r < nmatch; r++)
+            {
+              pmatch[r].rm_so = regs.start[r];
+              pmatch[r].rm_eo = regs.end[r];
+            }
+        }
+
+      /* If we needed the temporary register info, free the space now.  */
+      free (regs.start);
+      free (regs.end);
+    }
+
+  /* We want zero return to mean success, unlike `re_search'.  */
+  return ret >= 0 ? (int) REG_NOERROR : (int) REG_NOMATCH;
+}
+
+
+/* Returns a message corresponding to an error code, ERRCODE, returned
+   from either regcomp or regexec.   We don't use PREG here.  */
+
+size_t
+regerror (errcode, preg, errbuf, errbuf_size)
+    int errcode;
+    const regex_t *preg;
+    char *errbuf;
+    size_t errbuf_size;
+{
+  const char *msg;
+  size_t msg_size;
+
+  if (errcode < 0
+      || errcode >= (sizeof (re_error_msgid) / sizeof (re_error_msgid[0])))
+    /* Only error codes returned by the rest of the code should be passed 
+       to this routine.  If we are given anything else, or if other regex
+       code generates an invalid error code, then the program has a bug.
+       Dump core so we can fix it.  */
+    abort ();
+
+  msg = gettext (re_error_msgid[errcode]);
+
+  msg_size = strlen (msg) + 1; /* Includes the null.  */
+  
+  if (errbuf_size != 0)
+    {
+      if (msg_size > errbuf_size)
+        {
+          strncpy (errbuf, msg, errbuf_size - 1);
+          errbuf[errbuf_size - 1] = 0;
+        }
+      else
+        strcpy (errbuf, msg);
+    }
+
+  return msg_size;
+}
+
+
+/* Free dynamically allocated space used by PREG.  */
+
+void
+regfree (preg)
+    regex_t *preg;
+{
+  if (preg->buffer != NULL)
+    free (preg->buffer);
+  preg->buffer = NULL;
+  
+  preg->allocated = 0;
+  preg->used = 0;
+
+  if (preg->fastmap != NULL)
+    free (preg->fastmap);
+  preg->fastmap = NULL;
+  preg->fastmap_accurate = 0;
+
+  if (preg->translate != NULL)
+    free (preg->translate);
+  preg->translate = NULL;
+}
+
+#endif /* not emacs  */
+
+
+/*
+Local variables:
+make-backup-files: t
+version-control: t
+trim-versions-without-asking: nil
+End:
+*/
Index: /tags/gsdl-21/gsdl/packages/mg/lib/regex.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/regex.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/regex.h	(revision 699)
@@ -0,0 +1,497 @@
+/* Definitions for data structures and routines for the regular
+   expression library, version 0.12.
+
+   Copyright (C) 1985, 89, 90, 91, 92, 93, 95 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+#ifndef __REGEXP_LIBRARY_H__
+#define __REGEXP_LIBRARY_H__
+
+/* POSIX says that <sys/types.h> must be included (by the caller) before
+   <regex.h>.  */
+
+#if !defined (_POSIX_C_SOURCE) && !defined (_POSIX_SOURCE) && defined (VMS)
+/* VMS doesn't have `size_t' in <sys/types.h>, even though POSIX says it
+   should be there.  */
+#include <stddef.h>
+#endif
+
+
+/* The following bits are used to determine the regexp syntax we
+   recognize.  The set/not-set meanings are chosen so that Emacs syntax
+   remains the value 0.  The bits are given in alphabetical order, and
+   the definitions shifted by one from the previous bit; thus, when we
+   add or remove a bit, only one other definition need change.  */
+typedef unsigned reg_syntax_t;
+
+/* If this bit is not set, then \ inside a bracket expression is literal.
+   If set, then such a \ quotes the following character.  */
+#define RE_BACKSLASH_ESCAPE_IN_LISTS (1)
+
+/* If this bit is not set, then + and ? are operators, and \+ and \? are
+     literals. 
+   If set, then \+ and \? are operators and + and ? are literals.  */
+#define RE_BK_PLUS_QM (RE_BACKSLASH_ESCAPE_IN_LISTS << 1)
+
+/* If this bit is set, then character classes are supported.  They are:
+     [:alpha:], [:upper:], [:lower:],  [:digit:], [:alnum:], [:xdigit:],
+     [:space:], [:print:], [:punct:], [:graph:], and [:cntrl:].
+   If not set, then character classes are not supported.  */
+#define RE_CHAR_CLASSES (RE_BK_PLUS_QM << 1)
+
+/* If this bit is set, then ^ and $ are always anchors (outside bracket
+     expressions, of course).
+   If this bit is not set, then it depends:
+        ^  is an anchor if it is at the beginning of a regular
+           expression or after an open-group or an alternation operator;
+        $  is an anchor if it is at the end of a regular expression, or
+           before a close-group or an alternation operator.  
+
+   This bit could be (re)combined with RE_CONTEXT_INDEP_OPS, because
+   POSIX draft 11.2 says that * etc. in leading positions is undefined.
+   We already implemented a previous draft which made those constructs
+   invalid, though, so we haven't changed the code back.  */
+#define RE_CONTEXT_INDEP_ANCHORS (RE_CHAR_CLASSES << 1)
+
+/* If this bit is set, then special characters are always special
+     regardless of where they are in the pattern.
+   If this bit is not set, then special characters are special only in
+     some contexts; otherwise they are ordinary.  Specifically, 
+     * + ? and intervals are only special when not after the beginning,
+     open-group, or alternation operator.  */
+#define RE_CONTEXT_INDEP_OPS (RE_CONTEXT_INDEP_ANCHORS << 1)
+
+/* If this bit is set, then *, +, ?, and { cannot be first in an re or
+     immediately after an alternation or begin-group operator.  */
+#define RE_CONTEXT_INVALID_OPS (RE_CONTEXT_INDEP_OPS << 1)
+
+/* If this bit is set, then . matches newline.
+   If not set, then it doesn't.  */
+#define RE_DOT_NEWLINE (RE_CONTEXT_INVALID_OPS << 1)
+
+/* If this bit is set, then . doesn't match NUL.
+   If not set, then it does.  */
+#define RE_DOT_NOT_NULL (RE_DOT_NEWLINE << 1)
+
+/* If this bit is set, nonmatching lists [^...] do not match newline.
+   If not set, they do.  */
+#define RE_HAT_LISTS_NOT_NEWLINE (RE_DOT_NOT_NULL << 1)
+
+/* If this bit is set, either \{...\} or {...} defines an
+     interval, depending on RE_NO_BK_BRACES. 
+   If not set, \{, \}, {, and } are literals.  */
+#define RE_INTERVALS (RE_HAT_LISTS_NOT_NEWLINE << 1)
+
+/* If this bit is set, +, ? and | aren't recognized as operators.
+   If not set, they are.  */
+#define RE_LIMITED_OPS (RE_INTERVALS << 1)
+
+/* If this bit is set, newline is an alternation operator.
+   If not set, newline is literal.  */
+#define RE_NEWLINE_ALT (RE_LIMITED_OPS << 1)
+
+/* If this bit is set, then `{...}' defines an interval, and \{ and \}
+     are literals.
+  If not set, then `\{...\}' defines an interval.  */
+#define RE_NO_BK_BRACES (RE_NEWLINE_ALT << 1)
+
+/* If this bit is set, (...) defines a group, and \( and \) are literals.
+   If not set, \(...\) defines a group, and ( and ) are literals.  */
+#define RE_NO_BK_PARENS (RE_NO_BK_BRACES << 1)
+
+/* If this bit is set, then \<digit> matches <digit>.
+   If not set, then \<digit> is a back-reference.  */
+#define RE_NO_BK_REFS (RE_NO_BK_PARENS << 1)
+
+/* If this bit is set, then | is an alternation operator, and \| is literal. 
+   If not set, then \| is an alternation operator, and | is literal.  */
+#define RE_NO_BK_VBAR (RE_NO_BK_REFS << 1)
+
+/* If this bit is set, then an ending range point collating higher
+     than the starting range point, as in [z-a], is invalid.
+   If not set, then when ending range point collates higher than the
+     starting range point, the range is ignored.  */
+#define RE_NO_EMPTY_RANGES (RE_NO_BK_VBAR << 1)
+
+/* If this bit is set, then an unmatched ) is ordinary.
+   If not set, then an unmatched ) is invalid.  */
+#define RE_UNMATCHED_RIGHT_PAREN_ORD (RE_NO_EMPTY_RANGES << 1)
+
+/* If this bit is set, succeed as soon as we match the whole pattern,
+   without further backtracking.  */
+#define RE_NO_POSIX_BACKTRACKING (RE_UNMATCHED_RIGHT_PAREN_ORD << 1)
+
+/* This global variable defines the particular regexp syntax to use (for
+   some interfaces).  When a regexp is compiled, the syntax used is
+   stored in the pattern buffer, so changing this does not affect
+   already-compiled regexps.  */
+extern reg_syntax_t re_syntax_options;
+
+
+/* Define combinations of the above bits for the standard possibilities.
+   (The [[[ comments delimit what gets put into the Texinfo file, so
+   don't delete them!)  */ 
+/* [[[begin syntaxes]]] */
+#define RE_SYNTAX_EMACS 0
+
+#define RE_SYNTAX_AWK							\
+  (RE_BACKSLASH_ESCAPE_IN_LISTS | RE_DOT_NOT_NULL			\
+   | RE_NO_BK_PARENS            | RE_NO_BK_REFS				\
+   | RE_NO_BK_VBAR               | RE_NO_EMPTY_RANGES			\
+   | RE_UNMATCHED_RIGHT_PAREN_ORD)
+
+#define RE_SYNTAX_POSIX_AWK 						\
+  (RE_SYNTAX_POSIX_EXTENDED | RE_BACKSLASH_ESCAPE_IN_LISTS)
+
+#define RE_SYNTAX_GREP							\
+  (RE_BK_PLUS_QM              | RE_CHAR_CLASSES				\
+   | RE_HAT_LISTS_NOT_NEWLINE | RE_INTERVALS				\
+   | RE_NEWLINE_ALT)
+
+#define RE_SYNTAX_EGREP							\
+  (RE_CHAR_CLASSES        | RE_CONTEXT_INDEP_ANCHORS			\
+   | RE_CONTEXT_INDEP_OPS | RE_HAT_LISTS_NOT_NEWLINE			\
+   | RE_NEWLINE_ALT       | RE_NO_BK_PARENS				\
+   | RE_NO_BK_VBAR)
+
+#define RE_SYNTAX_POSIX_EGREP						\
+  (RE_SYNTAX_EGREP | RE_INTERVALS | RE_NO_BK_BRACES)
+
+/* P1003.2/D11.2, section 4.20.7.1, lines 5078ff.  */
+#define RE_SYNTAX_ED RE_SYNTAX_POSIX_BASIC
+
+#define RE_SYNTAX_SED RE_SYNTAX_POSIX_BASIC
+
+/* Syntax bits common to both basic and extended POSIX regex syntax.  */
+#define _RE_SYNTAX_POSIX_COMMON						\
+  (RE_CHAR_CLASSES | RE_DOT_NEWLINE      | RE_DOT_NOT_NULL		\
+   | RE_INTERVALS  | RE_NO_EMPTY_RANGES)
+
+#define RE_SYNTAX_POSIX_BASIC						\
+  (_RE_SYNTAX_POSIX_COMMON | RE_BK_PLUS_QM)
+
+/* Differs from ..._POSIX_BASIC only in that RE_BK_PLUS_QM becomes
+   RE_LIMITED_OPS, i.e., \? \+ \| are not recognized.  Actually, this
+   isn't minimal, since other operators, such as \`, aren't disabled.  */
+#define RE_SYNTAX_POSIX_MINIMAL_BASIC					\
+  (_RE_SYNTAX_POSIX_COMMON | RE_LIMITED_OPS)
+
+#define RE_SYNTAX_POSIX_EXTENDED					\
+  (_RE_SYNTAX_POSIX_COMMON | RE_CONTEXT_INDEP_ANCHORS			\
+   | RE_CONTEXT_INDEP_OPS  | RE_NO_BK_BRACES				\
+   | RE_NO_BK_PARENS       | RE_NO_BK_VBAR				\
+   | RE_UNMATCHED_RIGHT_PAREN_ORD)
+
+/* Differs from ..._POSIX_EXTENDED in that RE_CONTEXT_INVALID_OPS
+   replaces RE_CONTEXT_INDEP_OPS and RE_NO_BK_REFS is added.  */
+#define RE_SYNTAX_POSIX_MINIMAL_EXTENDED				\
+  (_RE_SYNTAX_POSIX_COMMON  | RE_CONTEXT_INDEP_ANCHORS			\
+   | RE_CONTEXT_INVALID_OPS | RE_NO_BK_BRACES				\
+   | RE_NO_BK_PARENS        | RE_NO_BK_REFS				\
+   | RE_NO_BK_VBAR	    | RE_UNMATCHED_RIGHT_PAREN_ORD)
+/* [[[end syntaxes]]] */
+
+
+/* Maximum number of duplicates an interval can allow.  Some systems
+   (erroneously) define this in other header files, but we want our
+   value, so remove any previous define.  */
+#ifdef RE_DUP_MAX
+#undef RE_DUP_MAX
+#endif
+#define RE_DUP_MAX ((1 << 15) - 1) 
+
+
+/* POSIX `cflags' bits (i.e., information for `regcomp').  */
+
+/* If this bit is set, then use extended regular expression syntax.
+   If not set, then use basic regular expression syntax.  */
+#define REG_EXTENDED 1
+
+/* If this bit is set, then ignore case when matching.
+   If not set, then case is significant.  */
+#define REG_ICASE (REG_EXTENDED << 1)
+ 
+/* If this bit is set, then anchors do not match at newline
+     characters in the string.
+   If not set, then anchors do match at newlines.  */
+#define REG_NEWLINE (REG_ICASE << 1)
+
+/* If this bit is set, then report only success or fail in regexec.
+   If not set, then returns differ between not matching and errors.  */
+#define REG_NOSUB (REG_NEWLINE << 1)
+
+
+/* POSIX `eflags' bits (i.e., information for regexec).  */
+
+/* If this bit is set, then the beginning-of-line operator doesn't match
+     the beginning of the string (presumably because it's not the
+     beginning of a line).
+   If not set, then the beginning-of-line operator does match the
+     beginning of the string.  */
+#define REG_NOTBOL 1
+
+/* Like REG_NOTBOL, except for the end-of-line.  */
+#define REG_NOTEOL (1 << 1)
+
+
+/* If any error codes are removed, changed, or added, update the
+   `re_error_msg' table in regex.c.  */
+typedef enum
+{
+  REG_NOERROR = 0,	/* Success.  */
+  REG_NOMATCH,		/* Didn't find a match (for regexec).  */
+
+  /* POSIX regcomp return error codes.  (In the order listed in the
+     standard.)  */
+  REG_BADPAT,		/* Invalid pattern.  */
+  REG_ECOLLATE,		/* Not implemented.  */
+  REG_ECTYPE,		/* Invalid character class name.  */
+  REG_EESCAPE,		/* Trailing backslash.  */
+  REG_ESUBREG,		/* Invalid back reference.  */
+  REG_EBRACK,		/* Unmatched left bracket.  */
+  REG_EPAREN,		/* Parenthesis imbalance.  */ 
+  REG_EBRACE,		/* Unmatched \{.  */
+  REG_BADBR,		/* Invalid contents of \{\}.  */
+  REG_ERANGE,		/* Invalid range end.  */
+  REG_ESPACE,		/* Ran out of memory.  */
+  REG_BADRPT,		/* No preceding re for repetition op.  */
+
+  /* Error codes we've added.  */
+  REG_EEND,		/* Premature end.  */
+  REG_ESIZE,		/* Compiled pattern bigger than 2^16 bytes.  */
+  REG_ERPAREN		/* Unmatched ) or \); not returned from regcomp.  */
+} reg_errcode_t;
+
+
+/* This data structure represents a compiled pattern.  Before calling
+   the pattern compiler, the fields `buffer', `allocated', `fastmap',
+   `translate', and `no_sub' can be set.  After the pattern has been
+   compiled, the `re_nsub' field is available.  All other fields are
+   private to the regex routines.  */
+
+struct re_pattern_buffer
+{
+/* [[[begin pattern_buffer]]] */
+	/* Space that holds the compiled pattern.  It is declared as
+          `unsigned char *' because its elements are
+           sometimes used as array indexes.  */
+  unsigned char *buffer;
+
+	/* Number of bytes to which `buffer' points.  */
+  unsigned long allocated;
+
+	/* Number of bytes actually used in `buffer'.  */
+  unsigned long used;	
+
+        /* Syntax setting with which the pattern was compiled.  */
+  reg_syntax_t syntax;
+
+        /* Pointer to a fastmap, if any, otherwise zero.  re_search uses
+           the fastmap, if there is one, to skip over impossible
+           starting points for matches.  */
+  char *fastmap;
+
+        /* Either a translate table to apply to all characters before
+           comparing them, or zero for no translation.  The translation
+           is applied to a pattern when it is compiled and to a string
+           when it is matched.  */
+  char *translate;
+
+	/* Number of subexpressions found by the compiler.  */
+  size_t re_nsub;
+
+        /* Zero if this pattern cannot match the empty string, one else.
+           Well, in truth it's used only in `re_search_2', to see
+           whether or not we should use the fastmap, so we don't set
+           this absolutely perfectly; see `re_compile_fastmap' (the
+           `duplicate' case).  */
+  unsigned can_be_null : 1;
+
+        /* If REGS_UNALLOCATED, allocate space in the `regs' structure
+             for `max (RE_NREGS, re_nsub + 1)' groups.
+           If REGS_REALLOCATE, reallocate space if necessary.
+           If REGS_FIXED, use what's there.  */
+#define REGS_UNALLOCATED 0
+#define REGS_REALLOCATE 1
+#define REGS_FIXED 2
+  unsigned regs_allocated : 2;
+
+        /* Set to zero when `regex_compile' compiles a pattern; set to one
+           by `re_compile_fastmap' if it updates the fastmap.  */
+  unsigned fastmap_accurate : 1;
+
+        /* If set, `re_match_2' does not return information about
+           subexpressions.  */
+  unsigned no_sub : 1;
+
+        /* If set, a beginning-of-line anchor doesn't match at the
+           beginning of the string.  */ 
+  unsigned not_bol : 1;
+
+        /* Similarly for an end-of-line anchor.  */
+  unsigned not_eol : 1;
+
+        /* If true, an anchor at a newline matches.  */
+  unsigned newline_anchor : 1;
+
+/* [[[end pattern_buffer]]] */
+};
+
+typedef struct re_pattern_buffer regex_t;
+
+
+/* Type for byte offsets within the string.  POSIX mandates this.  */
+typedef int regoff_t;
+
+
+/* This is the structure we store register match data in.  See
+   regex.texinfo for a full description of what registers match.  */
+struct re_registers
+{
+  unsigned num_regs;
+  regoff_t *start;
+  regoff_t *end;
+};
+
+
+/* If `regs_allocated' is REGS_UNALLOCATED in the pattern buffer,
+   `re_match_2' returns information about at least this many registers
+   the first time a `regs' structure is passed.  */
+#ifndef RE_NREGS
+#define RE_NREGS 30
+#endif
+
+
+/* POSIX specification for registers.  Aside from the different names than
+   `re_registers', POSIX uses an array of structures, instead of a
+   structure of arrays.  */
+typedef struct
+{
+  regoff_t rm_so;  /* Byte offset from string's start to substring's start.  */
+  regoff_t rm_eo;  /* Byte offset from string's start to substring's end.  */
+} regmatch_t;
+
+
+/* Declarations for routines.  */
+
+/* To avoid duplicating every routine declaration -- once with a
+   prototype (if we are ANSI), and once without (if we aren't) -- we
+   use the following macro to declare argument types.  This
+   unfortunately clutters up the declarations a bit, but I think it's
+   worth it.  */
+
+#if __STDC__
+
+#define _RE_ARGS(args) args
+
+#else /* not __STDC__ */
+
+#define _RE_ARGS(args) ()
+
+#endif /* not __STDC__ */
+
+/* Sets the current default syntax to SYNTAX, and return the old syntax.
+   You can also simply assign to the `re_syntax_options' variable.  */
+extern reg_syntax_t re_set_syntax _RE_ARGS ((reg_syntax_t syntax));
+
+/* Compile the regular expression PATTERN, with length LENGTH
+   and syntax given by the global `re_syntax_options', into the buffer
+   BUFFER.  Return NULL if successful, and an error string if not.  */
+extern const char *re_compile_pattern
+  _RE_ARGS ((const char *pattern, int length,
+             struct re_pattern_buffer *buffer));
+
+
+/* Compile a fastmap for the compiled pattern in BUFFER; used to
+   accelerate searches.  Return 0 if successful and -2 if was an
+   internal error.  */
+extern int re_compile_fastmap _RE_ARGS ((struct re_pattern_buffer *buffer));
+
+
+/* Search in the string STRING (with length LENGTH) for the pattern
+   compiled into BUFFER.  Start searching at position START, for RANGE
+   characters.  Return the starting position of the match, -1 for no
+   match, or -2 for an internal error.  Also return register
+   information in REGS (if REGS and BUFFER->no_sub are nonzero).  */
+extern int re_search
+  _RE_ARGS ((struct re_pattern_buffer *buffer, const char *string,
+            int length, int start, int range, struct re_registers *regs));
+
+
+/* Like `re_search', but search in the concatenation of STRING1 and
+   STRING2.  Also, stop searching at index START + STOP.  */
+extern int re_search_2
+  _RE_ARGS ((struct re_pattern_buffer *buffer, const char *string1,
+             int length1, const char *string2, int length2,
+             int start, int range, struct re_registers *regs, int stop));
+
+
+/* Like `re_search', but return how many characters in STRING the regexp
+   in BUFFER matched, starting at position START.  */
+extern int re_match
+  _RE_ARGS ((struct re_pattern_buffer *buffer, const char *string,
+             int length, int start, struct re_registers *regs));
+
+
+/* Relates to `re_match' as `re_search_2' relates to `re_search'.  */
+extern int re_match_2 
+  _RE_ARGS ((struct re_pattern_buffer *buffer, const char *string1,
+             int length1, const char *string2, int length2,
+             int start, struct re_registers *regs, int stop));
+
+
+/* Set REGS to hold NUM_REGS registers, storing them in STARTS and
+   ENDS.  Subsequent matches using BUFFER and REGS will use this memory
+   for recording register information.  STARTS and ENDS must be
+   allocated with malloc, and must each be at least `NUM_REGS * sizeof
+   (regoff_t)' bytes long.
+
+   If NUM_REGS == 0, then subsequent matches should allocate their own
+   register data.
+
+   Unless this function is called, the first search or match using
+   PATTERN_BUFFER will allocate its own register data, without
+   freeing the old data.  */
+extern void re_set_registers
+  _RE_ARGS ((struct re_pattern_buffer *buffer, struct re_registers *regs,
+             unsigned num_regs, regoff_t *starts, regoff_t *ends));
+
+#ifdef _REGEX_RE_COMP
+/* 4.2 bsd compatibility.  */
+extern char *re_comp _RE_ARGS ((const char *));
+extern int re_exec _RE_ARGS ((const char *));
+#endif
+
+/* POSIX compatibility.  */
+extern int regcomp _RE_ARGS ((regex_t *preg, const char *pattern, int cflags));
+extern int regexec
+  _RE_ARGS ((const regex_t *preg, const char *string, size_t nmatch,
+             regmatch_t pmatch[], int eflags));
+extern size_t regerror
+  _RE_ARGS ((int errcode, const regex_t *preg, char *errbuf,
+             size_t errbuf_size));
+extern void regfree _RE_ARGS ((regex_t *preg));
+
+#endif /* not __REGEXP_LIBRARY_H__ */
+
+
+/*
+Local variables:
+make-backup-files: t
+version-control: t
+trim-versions-without-asking: nil
+End:
+*/
Index: /tags/gsdl-21/gsdl/packages/mg/lib/rx.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/rx.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/rx.c	(revision 699)
@@ -0,0 +1,7271 @@
+/*	Copyright (C) 1992, 1993, 1994, 1995 Free Software Foundation, Inc.
+
+This file is part of the librx library.
+
+Librx is free software; you can redistribute it and/or modify it under
+the terms of the GNU Library General Public License as published by
+the Free Software Foundation; either version 2, or (at your option)
+any later version.
+
+Librx is distributed in the hope that it will be useful, but WITHOUT
+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this software; see the file COPYING.LIB.  If not,
+write to the Free Software Foundation, 675 Mass Ave, Cambridge, MA
+02139, USA.  */
+
+/* NOTE!!!  AIX is so losing it requires this to be the first thing in the 
+ * file. 
+ * Do not put ANYTHING before it!  
+ */
+#if !defined (__GNUC__) && defined (_AIX)
+ #pragma alloca
+#endif
+
+/* To make linux happy? */
+#ifndef	_GNU_SOURCE
+#define	_GNU_SOURCE
+#endif
+
+#if HAVE_CONFIG_H
+# ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+#  include <win32cfg.h>
+#  include <malloc.h>
+# else
+#  include <sysfuncs.h>
+# endif
+#endif
+
+
+
+char rx_version_string[] = "GNU Rx version 0.07.1";
+
+			/* ``Too hard!''
+			 *	    -- anon.
+			 */
+
+
+
+#include <stdio.h>
+#include <ctype.h>
+#ifndef isgraph
+#define isgraph(c) (isprint (c) && !isspace (c))
+#endif
+#ifndef isblank
+#define isblank(c) ((c) == ' ' || (c) == '\t')
+#endif
+
+#include <sys/types.h>
+
+#undef MAX
+#undef MIN
+#define MAX(a, b) ((a) > (b) ? (a) : (b))
+#define MIN(a, b) ((a) < (b) ? (a) : (b))
+
+typedef char boolean;
+#define false 0
+#define true 1
+
+#ifndef __GCC__
+#undef __inline__
+#define __inline__
+#endif
+
+/* Emacs already defines alloca, sometimes.  */
+#ifndef alloca
+
+/* Make alloca work the best possible way.  */
+#ifdef __GNUC__
+#define alloca __builtin_alloca
+#else /* not __GNUC__ */
+#if HAVE_ALLOCA_H
+#include <alloca.h>
+#else /* not __GNUC__ or HAVE_ALLOCA_H */
+#ifndef _AIX /* Already did AIX, up at the top.  */
+char *alloca ();
+#endif /* not _AIX */
+#endif /* not HAVE_ALLOCA_H */ 
+#endif /* not __GNUC__ */
+
+#endif /* not alloca */
+
+/* Memory management and stuff for emacs. */
+
+#define CHARBITS 8
+#define remalloc(M, S) (M ? realloc (M, S) : malloc (S))
+
+
+/* Should we use malloc or alloca?  If REGEX_MALLOC is not defined, we
+ * use `alloca' instead of `malloc' for the backtracking stack.
+ *
+ * Emacs will die miserably if we don't do this.
+ */
+
+#ifdef REGEX_MALLOC
+#define REGEX_ALLOCATE malloc
+#else /* not REGEX_MALLOC  */
+#ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+#define REGEX_ALLOCATE _alloca
+#else
+#define REGEX_ALLOCATE alloca
+#endif /* __WIN32__ */
+#endif /* not REGEX_MALLOC */
+
+
+#ifdef RX_WANT_RX_DEFS
+#define RX_DECL extern
+#define RX_DEF_QUAL 
+#else
+#define RX_WANT_RX_DEFS
+#define RX_DECL static
+#define RX_DEF_QUAL static
+#endif
+#include "rx.h"
+#undef RX_DECL
+#define RX_DECL RX_DEF_QUAL
+
+
+#ifndef emacs
+
+#ifdef SYNTAX_TABLE
+extern char *re_syntax_table;
+#else /* not SYNTAX_TABLE */
+
+#ifdef __STDC__
+static void
+init_syntax_once (void)
+#else
+static void
+init_syntax_once ()
+#endif
+{
+   register int c;
+   static int done = 0;
+
+   if (done)
+     return;
+
+   bzero (re_syntax_table, sizeof re_syntax_table);
+
+   for (c = 'a'; c <= 'z'; c++)
+     re_syntax_table[c] = Sword;
+
+   for (c = 'A'; c <= 'Z'; c++)
+     re_syntax_table[c] = Sword;
+
+   for (c = '0'; c <= '9'; c++)
+     re_syntax_table[c] = Sword;
+
+   re_syntax_table['_'] = Sword;
+
+   done = 1;
+}
+#endif /* not SYNTAX_TABLE */
+#endif /* not emacs */
+
+
+/* Compile with `-DRX_DEBUG' and use the following flags.
+ *
+ * Debugging flags:
+ *   	rx_debug - print information as a regexp is compiled
+ * 	rx_debug_trace - print information as a regexp is executed
+ */
+
+#ifdef RX_DEBUG
+
+int rx_debug_compile = 0;
+int rx_debug_trace = 0;
+static struct re_pattern_buffer * dbug_rxb = 0;
+
+#ifdef __STDC__
+typedef void (*side_effect_printer) (struct rx *, void *, FILE *);
+#else
+typedef void (*side_effect_printer) ();
+#endif
+
+#ifdef __STDC__
+static void print_cset (struct rx *rx, rx_Bitset cset, FILE * fp);
+#else
+static void print_cset ();
+#endif
+
+#ifdef __STDC__
+static void
+print_rexp (struct rx *rx,
+	    struct rexp_node *node, int depth,
+	    side_effect_printer seprint, FILE * fp)
+#else
+static void
+print_rexp (rx, node, depth, seprint, fp)
+     struct rx *rx;
+     struct rexp_node *node;
+     int depth;
+     side_effect_printer seprint;
+     FILE * fp;
+#endif
+{
+  if (!node)
+    return;
+  else
+    {
+      switch (node->type)
+	{
+	case r_cset:
+	  {
+	    fprintf (fp, "%*s", depth, "");
+	    print_cset (rx, node->params.cset, fp);
+	    fputc ('\n', fp);
+	    break;
+	  }
+
+ 	case r_opt:
+	case r_star:
+	  fprintf (fp, "%*s%s\n", depth, "",
+		   node->type == r_opt ? "opt" : "star");
+	  print_rexp (rx, node->params.pair.left, depth + 3, seprint, fp);
+	  break;
+
+	case r_2phase_star:
+	  fprintf (fp, "%*s2phase star\n", depth, "");
+	  print_rexp (rx, node->params.pair.right, depth + 3, seprint, fp);
+	  print_rexp (rx, node->params.pair.left, depth + 3, seprint, fp);
+	  break;
+
+
+	case r_alternate:
+	case r_concat:
+	  fprintf (fp, "%*s%s\n", depth, "",
+		   node->type == r_alternate ? "alt" : "concat");
+	  print_rexp (rx, node->params.pair.left, depth + 3, seprint, fp);
+	  print_rexp (rx, node->params.pair.right, depth + 3, seprint, fp);
+	  break;
+	case r_side_effect:
+	  fprintf (fp, "%*sSide effect: ", depth, "");
+	  seprint (rx, node->params.side_effect, fp);
+	  fputc ('\n', fp);
+	}
+    }
+}
+
+#ifdef __STDC__
+static void
+print_nfa (struct rx * rx,
+	   struct rx_nfa_state * n,
+	   side_effect_printer seprint, FILE * fp)
+#else
+static void
+print_nfa (rx, n, seprint, fp)
+     struct rx * rx;
+     struct rx_nfa_state * n;
+     side_effect_printer seprint;
+     FILE * fp;
+#endif
+{
+  while (n)
+    {
+      struct rx_nfa_edge *e = n->edges;
+      struct rx_possible_future *ec = n->futures;
+      fprintf (fp, "node %d %s\n", n->id,
+	       n->is_final ? "final" : (n->is_start ? "start" : ""));
+      while (e)
+	{
+	  fprintf (fp, "   edge to %d, ", e->dest->id);
+	  switch (e->type)
+	    {
+	    case ne_epsilon:
+	      fprintf (fp, "epsilon\n");
+	      break;
+	    case ne_side_effect:
+	      fprintf (fp, "side effect ");
+	      seprint (rx, e->params.side_effect, fp);
+	      fputc ('\n', fp);
+	      break;
+	    case ne_cset:
+	      fprintf (fp, "cset ");
+	      print_cset (rx, e->params.cset, fp);
+	      fputc ('\n', fp);
+	      break;
+	    }
+	  e = e->next;
+	}
+
+      while (ec)
+	{
+	  int x;
+	  struct rx_nfa_state_set * s;
+	  struct rx_se_list * l;
+	  fprintf (fp, "   eclosure to {");
+	  for (s = ec->destset; s; s = s->cdr)
+	    fprintf (fp, "%d ", s->car->id);
+	  fprintf (fp, "} (");
+	  for (l = ec->effects; l; l = l->cdr)
+	    {
+	      seprint (rx, l->car, fp);
+	      fputc (' ', fp);
+	    }
+	  fprintf (fp, ")\n");
+	  ec = ec->next;
+	}
+      n = n->next;
+    }
+}
+
+static char * efnames [] =
+{
+  "bogon",
+  "re_se_try",
+  "re_se_pushback",
+  "re_se_push0",
+  "re_se_pushpos",
+  "re_se_chkpos",
+  "re_se_poppos",
+  "re_se_at_dot",
+  "re_se_syntax",
+  "re_se_not_syntax",
+  "re_se_begbuf",
+  "re_se_hat",
+  "re_se_wordbeg",
+  "re_se_wordbound",
+  "re_se_notwordbound",
+  "re_se_wordend",
+  "re_se_endbuf",
+  "re_se_dollar",
+  "re_se_fail",
+};
+
+static char * efnames2[] =
+{
+  "re_se_win",
+  "re_se_lparen",
+  "re_se_rparen",
+  "re_se_backref",
+  "re_se_iter",
+  "re_se_end_iter",
+  "re_se_tv"
+};
+
+static char * inx_names[] = 
+{
+  "rx_backtrack_point",
+  "rx_do_side_effects",
+  "rx_cache_miss",
+  "rx_next_char",
+  "rx_backtrack",
+  "rx_error_inx",
+  "rx_num_instructions"
+};
+
+
+#ifdef __STDC__
+static void
+re_seprint (struct rx * rx, void * effect, FILE * fp)
+#else
+static void
+re_seprint (rx, effect, fp)
+     struct rx * rx;
+     void * effect;
+     FILE * fp;
+#endif
+{
+  if ((int)effect < 0)
+    fputs (efnames[-(int)effect], fp);
+  else if (dbug_rxb)
+    {
+      struct re_se_params * p = &dbug_rxb->se_params[(int)effect];
+      fprintf (fp, "%s(%d,%d)", efnames2[p->se], p->op1, p->op2);
+    }
+  else
+    fprintf (fp, "[complex op # %d]", (int)effect);
+}
+
+
+/* These are so the regex.c regression tests will compile. */
+void
+print_compiled_pattern (rxb)
+     struct re_pattern_buffer * rxb;
+{
+}
+
+void
+print_fastmap (fm)
+     char * fm;
+{
+}
+
+#endif /* RX_DEBUG */
+
+
+
+
+/* This page: Bitsets.  Completely unintersting. */
+
+#ifdef __STDC__
+RX_DECL int
+rx_bitset_is_equal (int size, rx_Bitset a, rx_Bitset b)
+#else
+RX_DECL int
+rx_bitset_is_equal (size, a, b)
+     int size;
+     rx_Bitset a;
+     rx_Bitset b;
+#endif
+{
+  int x;
+  RX_subset s = b[0];
+  b[0] = ~a[0];
+
+  for (x = rx_bitset_numb_subsets(size) - 1; a[x] == b[x]; --x)
+    ;
+
+  b[0] = s;
+  return !x && s == a[0];
+}
+
+#ifdef __STDC__
+RX_DECL int
+rx_bitset_is_subset (int size, rx_Bitset a, rx_Bitset b)
+#else
+RX_DECL int
+rx_bitset_is_subset (size, a, b)
+     int size;
+     rx_Bitset a;
+     rx_Bitset b;
+#endif
+{
+  int x = rx_bitset_numb_subsets(size) - 1;
+  while (x-- && (a[x] & b[x]) == a[x]);
+  return x == -1;
+}
+
+
+#ifdef __STDC__
+RX_DECL int
+rx_bitset_empty (int size, rx_Bitset set)
+#else
+RX_DECL int
+rx_bitset_empty (size, set)
+     int size;
+     rx_Bitset set;
+#endif
+{
+  int x;
+  RX_subset s = set[0];
+  set[0] = 1;
+  for (x = rx_bitset_numb_subsets(size) - 1; !set[x]; --x)
+    ;
+  set[0] = s;
+  return !s;
+}
+
+#ifdef __STDC__
+RX_DECL void
+rx_bitset_null (int size, rx_Bitset b)
+#else
+RX_DECL void
+rx_bitset_null (size, b)
+     int size;
+     rx_Bitset b;
+#endif
+{
+  bzero (b, rx_sizeof_bitset(size));
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_bitset_universe (int size, rx_Bitset b)
+#else
+RX_DECL void
+rx_bitset_universe (size, b)
+     int size;
+     rx_Bitset b;
+#endif
+{
+  int x = rx_bitset_numb_subsets (size);
+  while (x--)
+    *b++ = ~(RX_subset)0;
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_bitset_complement (int size, rx_Bitset b)
+#else
+RX_DECL void
+rx_bitset_complement (size, b)
+     int size;
+     rx_Bitset b;
+#endif
+{
+  int x = rx_bitset_numb_subsets (size);
+  while (x--)
+    {
+      *b = ~*b;
+      ++b;
+    }
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_bitset_assign (int size, rx_Bitset a, rx_Bitset b)
+#else
+RX_DECL void
+rx_bitset_assign (size, a, b)
+     int size;
+     rx_Bitset a;
+     rx_Bitset b;
+#endif
+{
+  int x;
+  for (x = rx_bitset_numb_subsets(size) - 1; x >=0; --x)
+    a[x] = b[x];
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_bitset_union (int size, rx_Bitset a, rx_Bitset b)
+#else
+RX_DECL void
+rx_bitset_union (size, a, b)
+     int size;
+     rx_Bitset a;
+     rx_Bitset b;
+#endif
+{
+  int x;
+  for (x = rx_bitset_numb_subsets(size) - 1; x >=0; --x)
+    a[x] |= b[x];
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_bitset_intersection (int size,
+			rx_Bitset a, rx_Bitset b)
+#else
+RX_DECL void
+rx_bitset_intersection (size, a, b)
+     int size;
+     rx_Bitset a;
+     rx_Bitset b;
+#endif
+{
+  int x;
+  for (x = rx_bitset_numb_subsets(size) - 1; x >=0; --x)
+    a[x] &= b[x];
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_bitset_difference (int size, rx_Bitset a, rx_Bitset b)
+#else
+RX_DECL void
+rx_bitset_difference (size, a, b)
+     int size;
+     rx_Bitset a;
+     rx_Bitset b;
+#endif
+{
+  int x;
+  for (x = rx_bitset_numb_subsets(size) - 1; x >=0; --x)
+    a[x] &=  ~ b[x];
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_bitset_revdifference (int size,
+			 rx_Bitset a, rx_Bitset b)
+#else
+RX_DECL void
+rx_bitset_revdifference (size, a, b)
+     int size;
+     rx_Bitset a;
+     rx_Bitset b;
+#endif
+{
+  int x;
+  for (x = rx_bitset_numb_subsets(size) - 1; x >=0; --x)
+    a[x] = ~a[x] & b[x];
+}
+
+#ifdef __STDC__
+RX_DECL void
+rx_bitset_xor (int size, rx_Bitset a, rx_Bitset b)
+#else
+RX_DECL void
+rx_bitset_xor (size, a, b)
+     int size;
+     rx_Bitset a;
+     rx_Bitset b;
+#endif
+{
+  int x;
+  for (x = rx_bitset_numb_subsets(size) - 1; x >=0; --x)
+    a[x] ^= b[x];
+}
+
+
+#ifdef __STDC__
+RX_DECL unsigned long
+rx_bitset_hash (int size, rx_Bitset b)
+#else
+RX_DECL unsigned long
+rx_bitset_hash (size, b)
+     int size;
+     rx_Bitset b;
+#endif
+{
+  int x;
+  unsigned long hash = (unsigned long)rx_bitset_hash;
+
+  for (x = rx_bitset_numb_subsets(size) - 1; x >= 0; --x)
+    hash ^= rx_bitset_subset_val(b, x);
+
+  return hash;
+}
+
+
+RX_DECL RX_subset rx_subset_singletons [RX_subset_bits] = 
+{
+  0x1,
+  0x2,
+  0x4,
+  0x8,
+  0x10,
+  0x20,
+  0x40,
+  0x80,
+  0x100,
+  0x200,
+  0x400,
+  0x800,
+  0x1000,
+  0x2000,
+  0x4000,
+  0x8000,
+  0x10000,
+  0x20000,
+  0x40000,
+  0x80000,
+  0x100000,
+  0x200000,
+  0x400000,
+  0x800000,
+  0x1000000,
+  0x2000000,
+  0x4000000,
+  0x8000000,
+  0x10000000,
+  0x20000000,
+  0x40000000,
+  0x80000000
+};
+
+#ifdef RX_DEBUG
+
+#ifdef __STDC__
+static void
+print_cset (struct rx *rx, rx_Bitset cset, FILE * fp)
+#else
+static void
+print_cset (rx, cset, fp)
+     struct rx *rx;
+     rx_Bitset cset;
+     FILE * fp;
+#endif
+{
+  int x;
+  fputc ('[', fp);
+  for (x = 0; x < rx->local_cset_size; ++x)
+    if (RX_bitset_member (cset, x))
+      {
+	if (isprint(x))
+	  fputc (x, fp);
+	else
+	  fprintf (fp, "\\0%o ", x);
+      }
+  fputc (']', fp);
+}
+
+#endif /*  RX_DEBUG */
+
+
+
+
+static unsigned long rx_hash_masks[4] =
+{
+  0x12488421,
+  0x96699669,
+  0xbe7dd7eb,
+  0xffffffff
+};
+
+
+/* Hash tables */
+#ifdef __STDC__
+RX_DECL struct rx_hash_item * 
+rx_hash_find (struct rx_hash * table,
+	      unsigned long hash,
+	      void * value,
+	      struct rx_hash_rules * rules)
+#else
+RX_DECL struct rx_hash_item * 
+rx_hash_find (table, hash, value, rules)
+     struct rx_hash * table;
+     unsigned long hash;
+     void * value;
+     struct rx_hash_rules * rules;
+#endif
+{
+  rx_hash_eq eq = rules->eq;
+  int maskc = 0;
+  long mask = rx_hash_masks [0];
+  int bucket = (hash & mask) % 13;
+
+  while (table->children [bucket])
+    {
+      table = table->children [bucket];
+      ++maskc;
+      mask = rx_hash_masks[maskc];
+      bucket = (hash & mask) % 13;
+    }
+
+  {
+    struct rx_hash_item * it = table->buckets[bucket];
+    while (it)
+      if (eq (it->data, value))
+	return it;
+      else
+	it = it->next_same_hash;
+  }
+
+  return 0;
+}
+
+
+#ifdef __STDC__
+RX_DECL struct rx_hash_item *
+rx_hash_store (struct rx_hash * table,
+	       unsigned long hash,
+	       void * value,
+	       struct rx_hash_rules * rules)
+#else
+RX_DECL struct rx_hash_item *
+rx_hash_store (table, hash, value, rules)
+     struct rx_hash * table;
+     unsigned long hash;
+     void * value;
+     struct rx_hash_rules * rules;
+#endif
+{
+  rx_hash_eq eq = rules->eq;
+  int maskc = 0;
+  long mask = rx_hash_masks[0];
+  int bucket = (hash & mask) % 13;
+  int depth = 0;
+  
+  while (table->children [bucket])
+    {
+      table = table->children [bucket];
+      ++maskc;
+      mask = rx_hash_masks[maskc];
+      bucket = (hash & mask) % 13;
+      ++depth;
+    }
+  
+  {
+    struct rx_hash_item * it = table->buckets[bucket];
+    while (it)
+      if (eq (it->data, value))
+	return it;
+      else
+	it = it->next_same_hash;
+  }
+  
+  {
+    if (   (depth < 3)
+	&& (table->bucket_size [bucket] >= 4))
+      {
+	struct rx_hash * newtab = ((struct rx_hash *)
+				   rules->hash_alloc (rules));
+	if (!newtab)
+	  goto add_to_bucket;
+	bzero (newtab, sizeof (*newtab));
+	newtab->parent = table;
+	{
+	  struct rx_hash_item * them = table->buckets[bucket];
+	  unsigned long newmask = rx_hash_masks[maskc + 1];
+	  while (them)
+	    {
+	      struct rx_hash_item * save = them->next_same_hash;
+	      int new_buck = (them->hash & newmask) % 13;
+	      them->next_same_hash = newtab->buckets[new_buck];
+	      newtab->buckets[new_buck] = them;
+	      them->table = newtab;
+	      them = save;
+	      ++newtab->bucket_size[new_buck];
+	      ++newtab->refs;
+	    }
+	  table->refs = (table->refs - table->bucket_size[bucket] + 1);
+	  table->bucket_size[bucket] = 0;
+	  table->buckets[bucket] = 0;
+	  table->children[bucket] = newtab;
+	  table = newtab;
+	  bucket = (hash & newmask) % 13;
+	}
+      }
+  }
+ add_to_bucket:
+  {
+    struct rx_hash_item  * it = ((struct rx_hash_item *)
+				 rules->hash_item_alloc (rules, value));
+    if (!it)
+      return 0;
+    it->hash = hash;
+    it->table = table;
+    /* DATA and BINDING are to be set in hash_item_alloc */
+    it->next_same_hash = table->buckets [bucket];
+    table->buckets[bucket] = it;
+    ++table->bucket_size [bucket];
+    ++table->refs;
+    return it;
+  }
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_hash_free (struct rx_hash_item * it, struct rx_hash_rules * rules)
+#else
+RX_DECL void
+rx_hash_free (it, rules)
+     struct rx_hash_item * it;
+     struct rx_hash_rules * rules;
+#endif
+{
+  if (it)
+    {
+      struct rx_hash * table = it->table;
+      unsigned long hash = it->hash;
+      int depth = (table->parent
+		   ? (table->parent->parent
+		      ? (table->parent->parent->parent
+			 ? 3
+			 : 2)
+		      : 1)
+		   : 0);
+      int bucket = (hash & rx_hash_masks [depth]) % 13;
+      struct rx_hash_item ** pos = &table->buckets [bucket];
+      
+      while (*pos != it)
+	pos = &(*pos)->next_same_hash;
+      *pos = it->next_same_hash;
+      rules->free_hash_item (it, rules);
+      --table->bucket_size[bucket];
+      --table->refs;
+      while (!table->refs && depth)
+	{
+	  struct rx_hash * save = table;
+	  table = table->parent;
+	  --depth;
+	  bucket = (hash & rx_hash_masks [depth]) % 13;
+	  --table->refs;
+	  table->children[bucket] = 0;
+	  rules->free_hash (save, rules);
+	}
+    }
+}
+
+#ifdef __STDC__
+RX_DECL void
+rx_free_hash_table (struct rx_hash * tab, rx_hash_freefn freefn,
+		    struct rx_hash_rules * rules)
+#else
+RX_DECL void
+rx_free_hash_table (tab, freefn, rules)
+     struct rx_hash * tab;
+     rx_hash_freefn freefn;
+     struct rx_hash_rules * rules;
+#endif
+{
+  int x;
+
+  for (x = 0; x < 13; ++x)
+    if (tab->children[x])
+      {
+	rx_free_hash_table (tab->children[x], freefn, rules);
+	rules->free_hash (tab->children[x], rules);
+      }
+    else
+      {
+	struct rx_hash_item * them = tab->buckets[x];
+	while (them)
+	  {
+	    struct rx_hash_item * that = them;
+	    them = that->next_same_hash;
+	    freefn (that);
+	    rules->free_hash_item (that, rules);
+	  }
+      }
+}
+
+
+
+
+/* Utilities for manipulating bitset represntations of characters sets. */
+
+#ifdef __STDC__
+RX_DECL rx_Bitset
+rx_cset (struct rx *rx)
+#else
+RX_DECL rx_Bitset
+rx_cset (rx)
+     struct rx *rx;
+#endif
+{
+  rx_Bitset b = (rx_Bitset) malloc (rx_sizeof_bitset (rx->local_cset_size));
+  if (b)
+    rx_bitset_null (rx->local_cset_size, b);
+  return b;
+}
+
+
+#ifdef __STDC__
+RX_DECL rx_Bitset
+rx_copy_cset (struct rx *rx, rx_Bitset a)
+#else
+RX_DECL rx_Bitset
+rx_copy_cset (rx, a)
+     struct rx *rx;
+     rx_Bitset a;
+#endif
+{
+  rx_Bitset cs = rx_cset (rx);
+
+  if (cs)
+    rx_bitset_union (rx->local_cset_size, cs, a);
+
+  return cs;
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_free_cset (struct rx * rx, rx_Bitset c)
+#else
+RX_DECL void
+rx_free_cset (rx, c)
+     struct rx * rx;
+     rx_Bitset c;
+#endif
+{
+  if (c)
+    free ((char *)c);
+}
+
+
+
+/* Hash table memory allocation policy for the regexp compiler */
+
+#ifdef __STDC__
+static struct rx_hash *
+compiler_hash_alloc (struct rx_hash_rules * rules)
+#else
+static struct rx_hash *
+compiler_hash_alloc (rules)
+     struct rx_hash_rules * rules;
+#endif
+{
+  return (struct rx_hash *)malloc (sizeof (struct rx_hash));
+}
+
+
+#ifdef __STDC__
+static struct rx_hash_item *
+compiler_hash_item_alloc (struct rx_hash_rules * rules, void * value)
+#else
+static struct rx_hash_item *
+compiler_hash_item_alloc (rules, value)
+     struct rx_hash_rules * rules;
+     void * value;
+#endif
+{
+  struct rx_hash_item * it;
+  it = (struct rx_hash_item *)malloc (sizeof (*it));
+  if (it)
+    {
+      it->data = value;
+      it->binding = 0;
+    }
+  return it;
+}
+
+
+#ifdef __STDC__
+static void
+compiler_free_hash (struct rx_hash * tab,
+		    struct rx_hash_rules * rules)
+#else
+static void
+compiler_free_hash (tab, rules)
+     struct rx_hash * tab;
+     struct rx_hash_rules * rules;
+#endif
+{
+  free ((char *)tab);
+}
+
+
+#ifdef __STDC__
+static void
+compiler_free_hash_item (struct rx_hash_item * item,
+			 struct rx_hash_rules * rules)
+#else
+static void
+compiler_free_hash_item (item, rules)
+     struct rx_hash_item * item;
+     struct rx_hash_rules * rules;
+#endif
+{
+  free ((char *)item);
+}
+
+
+
+/* This page: REXP_NODE (expression tree) structures. */
+
+#ifdef __STDC__
+RX_DECL struct rexp_node *
+rexp_node (struct rx *rx,
+	   enum rexp_node_type type)
+#else
+RX_DECL struct rexp_node *
+rexp_node (rx, type)
+     struct rx *rx;
+     enum rexp_node_type type;
+#endif
+{
+  struct rexp_node *n;
+
+  n = (struct rexp_node *)malloc (sizeof (*n));
+  bzero (n, sizeof (*n));
+  if (n)
+    n->type = type;
+  return n;
+}
+
+
+/* free_rexp_node assumes that the bitset passed to rx_mk_r_cset
+ * can be freed using rx_free_cset.
+ */
+#ifdef __STDC__
+RX_DECL struct rexp_node *
+rx_mk_r_cset (struct rx * rx,
+	      rx_Bitset b)
+#else
+RX_DECL struct rexp_node *
+rx_mk_r_cset (rx, b)
+     struct rx * rx;
+     rx_Bitset b;
+#endif
+{
+  struct rexp_node * n = rexp_node (rx, r_cset);
+  if (n)
+    n->params.cset = b;
+  return n;
+}
+
+
+#ifdef __STDC__
+RX_DECL struct rexp_node *
+rx_mk_r_concat (struct rx * rx,
+		struct rexp_node * a,
+		struct rexp_node * b)
+#else
+RX_DECL struct rexp_node *
+rx_mk_r_concat (rx, a, b)
+     struct rx * rx;
+     struct rexp_node * a;
+     struct rexp_node * b;
+#endif
+{
+  struct rexp_node * n = rexp_node (rx, r_concat);
+  if (n)
+    {
+      n->params.pair.left = a;
+      n->params.pair.right = b;
+    }
+  return n;
+}
+
+
+#ifdef __STDC__
+RX_DECL struct rexp_node *
+rx_mk_r_alternate (struct rx * rx,
+		   struct rexp_node * a,
+		   struct rexp_node * b)
+#else
+RX_DECL struct rexp_node *
+rx_mk_r_alternate (rx, a, b)
+     struct rx * rx;
+     struct rexp_node * a;
+     struct rexp_node * b;
+#endif
+{
+  struct rexp_node * n = rexp_node (rx, r_alternate);
+  if (n)
+    {
+      n->params.pair.left = a;
+      n->params.pair.right = b;
+    }
+  return n;
+}
+
+
+#ifdef __STDC__
+RX_DECL struct rexp_node *
+rx_mk_r_opt (struct rx * rx,
+	     struct rexp_node * a)
+#else
+RX_DECL struct rexp_node *
+rx_mk_r_opt (rx, a)
+     struct rx * rx;
+     struct rexp_node * a;
+#endif
+{
+  struct rexp_node * n = rexp_node (rx, r_opt);
+  if (n)
+    {
+      n->params.pair.left = a;
+      n->params.pair.right = 0;
+    }
+  return n;
+}
+
+
+#ifdef __STDC__
+RX_DECL struct rexp_node *
+rx_mk_r_star (struct rx * rx,
+	      struct rexp_node * a)
+#else
+RX_DECL struct rexp_node *
+rx_mk_r_star (rx, a)
+     struct rx * rx;
+     struct rexp_node * a;
+#endif
+{
+  struct rexp_node * n = rexp_node (rx, r_star);
+  if (n)
+    {
+      n->params.pair.left = a;
+      n->params.pair.right = 0;
+    }
+  return n;
+}
+
+
+#ifdef __STDC__
+RX_DECL struct rexp_node *
+rx_mk_r_2phase_star (struct rx * rx,
+		     struct rexp_node * a,
+		     struct rexp_node * b)
+#else
+RX_DECL struct rexp_node *
+rx_mk_r_2phase_star (rx, a, b)
+     struct rx * rx;
+     struct rexp_node * a;
+     struct rexp_node * b;
+#endif
+{
+  struct rexp_node * n = rexp_node (rx, r_2phase_star);
+  if (n)
+    {
+      n->params.pair.left = a;
+      n->params.pair.right = b;
+    }
+  return n;
+}
+
+
+#ifdef __STDC__
+RX_DECL struct rexp_node *
+rx_mk_r_side_effect (struct rx * rx,
+		     rx_side_effect a)
+#else
+RX_DECL struct rexp_node *
+rx_mk_r_side_effect (rx, a)
+     struct rx * rx;
+     rx_side_effect a;
+#endif
+{
+  struct rexp_node * n = rexp_node (rx, r_side_effect);
+  if (n)
+    {
+      n->params.side_effect = a;
+      n->params.pair.right = 0;
+    }
+  return n;
+}
+
+
+#ifdef __STDC__
+RX_DECL struct rexp_node *
+rx_mk_r_data  (struct rx * rx,
+	       void * a)
+#else
+RX_DECL struct rexp_node *
+rx_mk_r_data  (rx, a)
+     struct rx * rx;
+     void * a;
+#endif
+{
+  struct rexp_node * n = rexp_node (rx, r_data);
+  if (n)
+    {
+      n->params.pair.left = a;
+      n->params.pair.right = 0;
+    }
+  return n;
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_free_rexp (struct rx * rx, struct rexp_node * node)
+#else
+RX_DECL void
+rx_free_rexp (rx, node)
+     struct rx * rx;
+     struct rexp_node * node;
+#endif
+{
+  if (node)
+    {
+      switch (node->type)
+	{
+	case r_cset:
+	  if (node->params.cset)
+	    rx_free_cset (rx, node->params.cset);
+
+	case r_side_effect:
+	  break;
+	  
+	case r_concat:
+	case r_alternate:
+	case r_2phase_star:
+	case r_opt:
+	case r_star:
+	  rx_free_rexp (rx, node->params.pair.left);
+	  rx_free_rexp (rx, node->params.pair.right);
+	  break;
+
+	case r_data:
+	  /* This shouldn't occur. */
+	  break;
+	}
+      free ((char *)node);
+    }
+}
+
+
+#ifdef __STDC__
+RX_DECL struct rexp_node * 
+rx_copy_rexp (struct rx *rx,
+	   struct rexp_node *node)
+#else
+RX_DECL struct rexp_node * 
+rx_copy_rexp (rx, node)
+     struct rx *rx;
+     struct rexp_node *node;
+#endif
+{
+  if (!node)
+    return 0;
+  else
+    {
+      struct rexp_node *n = rexp_node (rx, node->type);
+      if (!n)
+	return 0;
+      switch (node->type)
+	{
+	case r_cset:
+	  n->params.cset = rx_copy_cset (rx, node->params.cset);
+	  if (!n->params.cset)
+	    {
+	      rx_free_rexp (rx, n);
+	      return 0;
+	    }
+	  break;
+
+	case r_side_effect:
+	  n->params.side_effect = node->params.side_effect;
+	  break;
+
+	case r_concat:
+	case r_alternate:
+	case r_opt:
+	case r_2phase_star:
+	case r_star:
+	  n->params.pair.left =
+	    rx_copy_rexp (rx, node->params.pair.left);
+	  n->params.pair.right =
+	    rx_copy_rexp (rx, node->params.pair.right);
+	  if (   (node->params.pair.left && !n->params.pair.left)
+	      || (node->params.pair.right && !n->params.pair.right))
+	    {
+	      rx_free_rexp  (rx, n);
+	      return 0;
+	    }
+	  break;
+	case r_data:
+	  /* shouldn't happen */
+	  break;
+	}
+      return n;
+    }
+}
+
+
+
+
+/* This page: functions to build and destroy graphs that describe nfa's */
+
+/* Constructs a new nfa node. */
+#ifdef __STDC__
+RX_DECL struct rx_nfa_state *
+rx_nfa_state (struct rx *rx)
+#else
+RX_DECL struct rx_nfa_state *
+rx_nfa_state (rx)
+     struct rx *rx;
+#endif
+{
+  struct rx_nfa_state * n = (struct rx_nfa_state *)malloc (sizeof (*n));
+  if (!n)
+    return 0;
+  bzero (n, sizeof (*n));
+  n->next = rx->nfa_states;
+  rx->nfa_states = n;
+  return n;
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_free_nfa_state (struct rx_nfa_state * n)
+#else
+RX_DECL void
+rx_free_nfa_state (n)
+  struct rx_nfa_state * n;
+#endif
+{
+  free ((char *)n);
+}
+
+
+/* This looks up an nfa node, given a numeric id.  Numeric id's are
+ * assigned after the nfa has been built.
+ */
+#ifdef __STDC__
+RX_DECL struct rx_nfa_state * 
+rx_id_to_nfa_state (struct rx * rx,
+		    int id)
+#else
+RX_DECL struct rx_nfa_state * 
+rx_id_to_nfa_state (rx, id)
+     struct rx * rx;
+     int id;
+#endif
+{
+  struct rx_nfa_state * n;
+  for (n = rx->nfa_states; n; n = n->next)
+    if (n->id == id)
+      return n;
+  return 0;
+}
+
+
+/* This adds an edge between two nodes, but doesn't initialize the 
+ * edge label.
+ */
+
+#ifdef __STDC__
+RX_DECL struct rx_nfa_edge * 
+rx_nfa_edge (struct rx *rx,
+	     enum rx_nfa_etype type,
+	     struct rx_nfa_state *start,
+	     struct rx_nfa_state *dest)
+#else
+RX_DECL struct rx_nfa_edge * 
+rx_nfa_edge (rx, type, start, dest)
+     struct rx *rx;
+     enum rx_nfa_etype type;
+     struct rx_nfa_state *start;
+     struct rx_nfa_state *dest;
+#endif
+{
+  struct rx_nfa_edge *e;
+  e = (struct rx_nfa_edge *)malloc (sizeof (*e));
+  if (!e)
+    return 0;
+  e->next = start->edges;
+  start->edges = e;
+  e->type = type;
+  e->dest = dest;
+  return e;
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_free_nfa_edge (struct rx_nfa_edge * e)
+#else
+RX_DECL void
+rx_free_nfa_edge (e)
+     struct rx_nfa_edge * e;
+#endif
+{
+  free ((char *)e);
+}
+
+
+/* This constructs a POSSIBLE_FUTURE, which is a kind epsilon-closure
+ * of an NFA.  These are added to an nfa automaticly by eclose_nfa.
+ */  
+
+#ifdef __STDC__
+static struct rx_possible_future * 
+rx_possible_future (struct rx * rx,
+		 struct rx_se_list * effects)
+#else
+static struct rx_possible_future * 
+rx_possible_future (rx, effects)
+     struct rx * rx;
+     struct rx_se_list * effects;
+#endif
+{
+  struct rx_possible_future *ec;
+  ec = (struct rx_possible_future *) malloc (sizeof (*ec));
+  if (!ec)
+    return 0;
+  ec->destset = 0;
+  ec->next = 0;
+  ec->effects = effects;
+  return ec;
+}
+
+
+#ifdef __STDC__
+static void
+rx_free_possible_future (struct rx_possible_future * pf)
+#else
+static void
+rx_free_possible_future (pf)
+     struct rx_possible_future * pf;
+#endif
+{
+  free ((char *)pf);
+}
+
+
+#ifdef __STDC__
+RX_DECL void
+rx_free_nfa (struct rx *rx)
+#else
+RX_DECL void
+rx_free_nfa (rx)
+     struct rx *rx;
+#endif
+{
+  while (rx->nfa_states)
+    {
+      while (rx->nfa_states->edges)
+	{
+	  switch (rx->nfa_states->edges->type)
+	    {
+	    case ne_cset:
+	      rx_free_cset (rx, rx->nfa_states->edges->params.cset);
+	      break;
+	    default:
+	      break;
+	    }
+	  {
+	    struct rx_nfa_edge * e;
+	    e = rx->nfa_states->edges;
+	    rx->nfa_states->edges = rx->nfa_states->edges->next;
+	    rx_free_nfa_edge (e);
+	  }
+	} /* while (rx->nfa_states->edges) */
+      {
+	/* Iterate over the partial epsilon closures of rx->nfa_states */
+	struct rx_possible_future * pf = rx->nfa_states->futures;
+	while (pf)
+	  {
+	    struct rx_possible_future * pft = pf;
+	    pf = pf->next;
+	    rx_free_possible_future (pft);
+	  }
+      }
+      {
+	struct rx_nfa_state *n;
+	n = rx->nfa_states;
+	rx->nfa_states = rx->nfa_states->next;
+	rx_free_nfa_state (n);
+      }
+    }
+}
+
+
+
+
+/* This page: translating a pattern expression into an nfa and doing the 
+ * static part of the nfa->super-nfa translation.
+ */
+
+/* This is the thompson regexp->nfa algorithm. 
+ * It is modified to allow for `side-effect epsilons.'  Those are
+ * edges that are taken whenever a similar epsilon edge would be,
+ * but which imply that some side effect occurs when the edge 
+ * is taken.
+ *
+ * Side effects are used to model parts of the pattern langauge 
+ * that are not regular (in the formal sense).
+ */
+
+#ifdef __STDC__
+RX_DECL int
+rx_build_nfa (struct rx *rx,
+	      struct rexp_node *rexp,
+	      struct rx_nfa_state **start,
+	      struct rx_nfa_state **end)
+#else
+RX_DECL int
+rx_build_nfa (rx, rexp, start, end)
+     struct rx *rx;
+     struct rexp_node *rexp;
+     struct rx_nfa_state **start;
+     struct rx_nfa_state **end;
+#endif
+{
+  struct rx_nfa_edge *edge;
+
+  /* Start & end nodes may have been allocated by the caller. */
+  *start = *start ? *start : rx_nfa_state (rx);
+
+  if (!*start)
+    return 0;
+
+  if (!rexp)
+    {
+      *end = *start;
+      return 1;
+    }
+
+  *end = *end ? *end : rx_nfa_state (rx);
+
+  if (!*end)
+    {
+      rx_free_nfa_state (*start);
+      return 0;
+    }
+
+  switch (rexp->type)
+    {
+    case r_data:
+      return 0;
+
+    case r_cset:
+      edge = rx_nfa_edge (rx, ne_cset, *start, *end);
+      if (!edge)
+	return 0;
+      edge->params.cset = rx_copy_cset (rx, rexp->params.cset);
+      if (!edge->params.cset)
+	{
+	  rx_free_nfa_edge (edge);
+	  return 0;
+	}
+      return 1;
+ 
+    case r_opt:
+      return (rx_build_nfa (rx, rexp->params.pair.left, start, end)
+	      && rx_nfa_edge (rx, ne_epsilon, *start, *end));
+
+    case r_star:
+      {
+	struct rx_nfa_state * star_start = 0;
+	struct rx_nfa_state * star_end = 0;
+	return (rx_build_nfa (rx, rexp->params.pair.left,
+			      &star_start, &star_end)
+		&& star_start
+		&& star_end
+		&& rx_nfa_edge (rx, ne_epsilon, star_start, star_end)
+		&& rx_nfa_edge (rx, ne_epsilon, *start, star_start)
+		&& rx_nfa_edge (rx, ne_epsilon, star_end, *end)
+
+		&& rx_nfa_edge (rx, ne_epsilon, star_end, star_start));
+      }
+
+    case r_2phase_star:
+      {
+	struct rx_nfa_state * star_start = 0;
+	struct rx_nfa_state * star_end = 0;
+	struct rx_nfa_state * loop_exp_start = 0;
+	struct rx_nfa_state * loop_exp_end = 0;
+
+	return (rx_build_nfa (rx, rexp->params.pair.left,
+			      &star_start, &star_end)
+		&& rx_build_nfa (rx, rexp->params.pair.right,
+				 &loop_exp_start, &loop_exp_end)
+		&& star_start
+		&& star_end
+		&& loop_exp_end
+		&& loop_exp_start
+		&& rx_nfa_edge (rx, ne_epsilon, star_start, *end)
+		&& rx_nfa_edge (rx, ne_epsilon, *start, star_start)
+		&& rx_nfa_edge (rx, ne_epsilon, star_end, *end)
+
+		&& rx_nfa_edge (rx, ne_epsilon, star_end, loop_exp_start)
+		&& rx_nfa_edge (rx, ne_epsilon, loop_exp_end, star_start));
+      }
+
+
+    case r_concat:
+      {
+	struct rx_nfa_state *shared = 0;
+	return
+	  (rx_build_nfa (rx, rexp->params.pair.left, start, &shared)
+	   && rx_build_nfa (rx, rexp->params.pair.right, &shared, end));
+      }
+
+    case r_alternate:
+      {
+	struct rx_nfa_state *ls = 0;
+	struct rx_nfa_state *le = 0;
+	struct rx_nfa_state *rs = 0;
+	struct rx_nfa_state *re = 0;
+	return (rx_build_nfa (rx, rexp->params.pair.left, &ls, &le)
+		&& rx_build_nfa (rx, rexp->params.pair.right, &rs, &re)
+		&& rx_nfa_edge (rx, ne_epsilon, *start, ls)
+		&& rx_nfa_edge (rx, ne_epsilon, *start, rs)
+		&& rx_nfa_edge (rx, ne_epsilon, le, *end)
+		&& rx_nfa_edge (rx, ne_epsilon, re, *end));
+      }
+
+    case r_side_effect:
+      edge = rx_nfa_edge (rx, ne_side_effect, *start, *end);
+      if (!edge)
+	return 0;
+      edge->params.side_effect = rexp->params.side_effect;
+      return 1;
+    }
+
+  /* this should never happen */
+  return 0;
+}
+
+
+/* RX_NAME_NFA_STATES identifies all nodes with outgoing non-epsilon
+ * transitions.  Only these nodes can occur in super-states.  
+ * All nodes are given an integer id. 
+ * The id is non-negative if the node has non-epsilon out-transitions, negative
+ * otherwise (this is because we want the non-negative ids to be used as 
+ * array indexes in a few places).
+ */
+
+#ifdef __STDC__
+RX_DECL void
+rx_name_nfa_states (struct rx *rx)
+#else
+RX_DECL void
+rx_name_nfa_states (rx)
+     struct rx *rx;
+#endif
+{
+  struct rx_nfa_state *n = rx->nfa_states;
+
+  rx->nodec = 0;
+  rx->epsnodec = -1;
+
+  while (n)
+    {
+      struct rx_nfa_edge *e = n->edges;
+
+      if (n->is_start)
+	n->eclosure_needed = 1;
+
+      while (e)
+	{
+	  switch (e->type)
+	    {
+	    case ne_epsilon:
+	    case ne_side_effect:
+	      break;
+
+	    case ne_cset:
+	      n->id = rx->nodec++;
+	      {
+		struct rx_nfa_edge *from_n = n->edges;
+		while (from_n)
+		  {
+		    from_n->dest->eclosure_needed = 1;
+		    from_n = from_n->next;
+		  }
+	      }
+	      goto cont;
+	    }
+	  e = e->next;
+	}
+      n->id = rx->epsnodec--;
+    cont:
+      n = n->next;
+    }
+  rx->epsnodec = -rx->epsnodec;
+}
+
+
+
+/* This page: data structures for the static part of the nfa->supernfa
+ * translation.
+ *
+ * There are side effect lists -- lists of side effects occuring
+ * along an uninterrupted, acyclic path of side-effect epsilon edges.
+ * Such paths are collapsed to single edges in the course of computing
+ * epsilon closures.  Such single edges are labled with a list of all
+ * the side effects entailed in crossing them.  Like lists of side
+ * effects are made == by the constructors below.
+ *
+ * There are also nfa state sets.  These are used to hold a list of all
+ * states reachable from a starting state for a given type of transition
+ * and side effect list.   These are also hash-consed.
+ */
+
+/* The next several functions compare, construct, etc. lists of side
+ * effects.  See ECLOSE_NFA (below) for details.
+ */
+
+/* Ordering of rx_se_list
+ * (-1, 0, 1 return value convention).
+ */
+
+#ifdef __STDC__
+static int 
+se_list_cmp (void * va, void * vb)
+#else
+static int 
+se_list_cmp (va, vb)
+     void * va;
+     void * vb;
+#endif
+{
+  struct rx_se_list * a = (struct rx_se_list *)va;
+  struct rx_se_list * b = (struct rx_se_list *)vb;
+
+  return ((va == vb)
+	  ? 0
+	  : (!va
+	     ? -1
+	     : (!vb
+		? 1
+		: ((long)a->car < (long)b->car
+		   ? 1
+		   : ((long)a->car > (long)b->car
+		      ? -1
+		      : se_list_cmp ((void *)a->cdr, (void *)b->cdr))))));
+}
+
+
+#ifdef __STDC__
+static int 
+se_list_equal (void * va, void * vb)
+#else
+static int 
+se_list_equal (va, vb)
+     void * va;
+     void * vb;
+#endif
+{
+  return !(se_list_cmp (va, vb));
+}
+
+static struct rx_hash_rules se_list_hash_rules =
+{
+  se_list_equal,
+  compiler_hash_alloc,
+  compiler_free_hash,
+  compiler_hash_item_alloc,
+  compiler_free_hash_item
+};
+
+
+#ifdef __STDC__
+static struct rx_se_list * 
+side_effect_cons (struct rx * rx,
+		  void * se, struct rx_se_list * list)
+#else
+static struct rx_se_list * 
+side_effect_cons (rx, se, list)
+     struct rx * rx;
+     void * se;
+     struct rx_se_list * list;
+#endif
+{
+  struct rx_se_list * l;
+  l = ((struct rx_se_list *) malloc (sizeof (*l)));
+  if (!l)
+    return 0;
+  l->car = se;
+  l->cdr = list;
+  return l;
+}
+
+
+#ifdef __STDC__
+static struct rx_se_list *
+hash_cons_se_prog (struct rx * rx,
+		   struct rx_hash * memo,
+		   void * car, struct rx_se_list * cdr)
+#else
+static struct rx_se_list *
+hash_cons_se_prog (rx, memo, car, cdr)
+     struct rx * rx;
+     struct rx_hash * memo;
+     void * car;
+     struct rx_se_list * cdr;
+#endif
+{
+  long hash = (long)car ^ (long)cdr;
+  struct rx_se_list template;
+
+  template.car = car;
+  template.cdr = cdr;
+  {
+    struct rx_hash_item * it = rx_hash_store (memo, hash,
+					      (void *)&template,
+					      &se_list_hash_rules);
+    if (!it)
+      return 0;
+    if (it->data == (void *)&template)
+      {
+	struct rx_se_list * consed;
+	consed = (struct rx_se_list *) malloc (sizeof (*consed));
+	*consed = template;
+	it->data = (void *)consed;
+      }
+    return (struct rx_se_list *)it->data;
+  }
+}
+     
+
+#ifdef __STDC__
+static struct rx_se_list *
+hash_se_prog (struct rx * rx, struct rx_hash * memo, struct rx_se_list * prog)
+#else
+static struct rx_se_list *
+hash_se_prog (rx, memo, prog)
+     struct rx * rx;
+     struct rx_hash * memo;
+     struct rx_se_list * prog;
+#endif
+{
+  struct rx_se_list * answer = 0;
+  while (prog)
+    {
+      answer = hash_cons_se_prog (rx, memo, prog->car, answer);
+      if (!answer)
+	return 0;
+      prog = prog->cdr;
+    }
+  return answer;
+}
+
+#ifdef __STDC__
+static int 
+nfa_set_cmp (void * va, void * vb)
+#else
+static int 
+nfa_set_cmp (va, vb)
+     void * va;
+     void * vb;
+#endif
+{
+  struct rx_nfa_state_set * a = (struct rx_nfa_state_set *)va;
+  struct rx_nfa_state_set * b = (struct rx_nfa_state_set *)vb;
+
+  return ((va == vb)
+	  ? 0
+	  : (!va
+	     ? -1
+	     : (!vb
+		? 1
+		: (a->car->id < b->car->id
+		   ? 1
+		   : (a->car->id > b->car->id
+		      ? -1
+		      : nfa_set_cmp ((void *)a->cdr, (void *)b->cdr))))));
+}
+
+#ifdef __STDC__
+static int 
+nfa_set_equal (void * va, void * vb)
+#else
+static int 
+nfa_set_equal (va, vb)
+     void * va;
+     void * vb;
+#endif
+{
+  return !nfa_set_cmp (va, vb);
+}
+
+static struct rx_hash_rules nfa_set_hash_rules =
+{
+  nfa_set_equal,
+  compiler_hash_alloc,
+  compiler_free_hash,
+  compiler_hash_item_alloc,
+  compiler_free_hash_item
+};
+
+
+#ifdef __STDC__
+static struct rx_nfa_state_set * 
+nfa_set_cons (struct rx * rx,
+	      struct rx_hash * memo, struct rx_nfa_state * state,
+	      struct rx_nfa_state_set * set)
+#else
+static struct rx_nfa_state_set * 
+nfa_set_cons (rx, memo, state, set)
+     struct rx * rx;
+     struct rx_hash * memo;
+     struct rx_nfa_state * state;
+     struct rx_nfa_state_set * set;
+#endif
+{
+  struct rx_nfa_state_set template;
+  struct rx_hash_item * node;
+  template.car = state;
+  template.cdr = set;
+  node = rx_hash_store (memo,
+			(((long)state) >> 8) ^ (long)set,
+			&template, &nfa_set_hash_rules);
+  if (!node)
+    return 0;
+  if (node->data == &template)
+    {
+      struct rx_nfa_state_set * l;
+      l = (struct rx_nfa_state_set *) malloc (sizeof (*l));
+      node->data = (void *) l;
+      if (!l)
+	return 0;
+      *l = template;
+    }
+  return (struct rx_nfa_state_set *)node->data;
+}
+
+
+#ifdef __STDC__
+static struct rx_nfa_state_set * 
+nfa_set_enjoin (struct rx * rx,
+		struct rx_hash * memo, struct rx_nfa_state * state,
+		struct rx_nfa_state_set * set)
+#else
+static struct rx_nfa_state_set * 
+nfa_set_enjoin (rx, memo, state, set)
+     struct rx * rx;
+     struct rx_hash * memo;
+     struct rx_nfa_state * state;
+     struct rx_nfa_state_set * set;
+#endif
+{
+  if (!set || state->id < set->car->id)
+    return nfa_set_cons (rx, memo, state, set);
+  if (state->id == set->car->id)
+    return set;
+  else
+    {
+      struct rx_nfa_state_set * newcdr
+	= nfa_set_enjoin (rx, memo, state, set->cdr);
+      if (newcdr != set->cdr)
+	set = nfa_set_cons (rx, memo, set->car, newcdr);
+      return set;
+    }
+}
+
+
+
+
+/* This page: computing epsilon closures.  The closures aren't total.
+ * Each node's closures are partitioned according to the side effects entailed
+ * along the epsilon edges.  Return true on success.
+ */ 
+
+struct eclose_frame
+{
+  struct rx_se_list *prog_backwards;
+};
+
+
+#ifdef __STDC__
+static int 
+eclose_node (struct rx *rx, struct rx_nfa_state *outnode,
+	     struct rx_nfa_state *node, struct eclose_frame *frame)
+#else
+static int 
+eclose_node (rx, outnode, node, frame)
+     struct rx *rx;
+     struct rx_nfa_state *outnode;
+     struct rx_nfa_state *node;
+     struct eclose_frame *frame;
+#endif
+{
+  struct rx_nfa_edge *e = node->edges;
+
+  /* For each node, we follow all epsilon paths to build the closure.
+   * The closure omits nodes that have only epsilon edges.
+   * The closure is split into partial closures -- all the states in
+   * a partial closure are reached by crossing the same list of
+   * of side effects (though not necessarily the same path).
+   */
+  if (node->mark)
+    return 1;
+  node->mark = 1;
+
+  if (node->id >= 0 || node->is_final)
+    {
+      struct rx_possible_future **ec;
+      struct rx_se_list * prog_in_order
+	= ((struct rx_se_list *)hash_se_prog (rx,
+					      &rx->se_list_memo,
+					      frame->prog_backwards));
+      int cmp;
+
+      ec = &outnode->futures;
+
+      while (*ec)
+	{
+	  cmp = se_list_cmp ((void *)(*ec)->effects, (void *)prog_in_order);
+	  if (cmp <= 0)
+	    break;
+	  ec = &(*ec)->next;
+	}
+      if (!*ec || (cmp < 0))
+	{
+	  struct rx_possible_future * saved = *ec;
+	  *ec = rx_possible_future (rx, prog_in_order);
+	  (*ec)->next = saved;
+	  if (!*ec)
+	    return 0;
+	}
+      if (node->id >= 0)
+	{
+	  (*ec)->destset = nfa_set_enjoin (rx, &rx->set_list_memo,
+					   node, (*ec)->destset);
+	  if (!(*ec)->destset)
+	    return 0;
+	}
+    }
+
+  while (e)
+    {
+      switch (e->type)
+	{
+	case ne_epsilon:
+	  if (!eclose_node (rx, outnode, e->dest, frame))
+	    return 0;
+	  break;
+	case ne_side_effect:
+	  {
+	    frame->prog_backwards = side_effect_cons (rx, 
+						      e->params.side_effect,
+						      frame->prog_backwards);
+	    if (!frame->prog_backwards)
+	      return 0;
+	    if (!eclose_node (rx, outnode, e->dest, frame))
+	      return 0;
+	    {
+	      struct rx_se_list * dying = frame->prog_backwards;
+	      frame->prog_backwards = frame->prog_backwards->cdr;
+	      free ((char *)dying);
+	    }
+	    break;
+	  }
+	default:
+	  break;
+	}
+      e = e->next;
+    }
+  node->mark = 0;
+  return 1;
+}
+
+
+#ifdef __STDC__
+RX_DECL int 
+rx_eclose_nfa (struct rx *rx)
+#else
+RX_DECL int 
+rx_eclose_nfa (rx)
+     struct rx *rx;
+#endif
+{
+  struct rx_nfa_state *n = rx->nfa_states;
+  struct eclose_frame frame;
+  static int rx_id = 0;
+  
+  frame.prog_backwards = 0;
+  rx->rx_id = rx_id++;
+  bzero (&rx->se_list_memo, sizeof (rx->se_list_memo));
+  bzero (&rx->set_list_memo, sizeof (rx->set_list_memo));
+  while (n)
+    {
+      n->futures = 0;
+      if (n->eclosure_needed && !eclose_node (rx, n, n, &frame))
+	return 0;
+      /* clear_marks (rx); */
+      n = n->next;
+    }
+  return 1;
+}
+
+
+/* This deletes epsilon edges from an NFA.  After running eclose_node,
+ * we have no more need for these edges.  They are removed to simplify
+ * further operations on the NFA.
+ */
+
+#ifdef __STDC__
+RX_DECL void 
+rx_delete_epsilon_transitions (struct rx *rx)
+#else
+RX_DECL void 
+rx_delete_epsilon_transitions (rx)
+     struct rx *rx;
+#endif
+{
+  struct rx_nfa_state *n = rx->nfa_states;
+  struct rx_nfa_edge **e;
+
+  while (n)
+    {
+      e = &n->edges;
+      while (*e)
+	{
+	  struct rx_nfa_edge *t;
+	  switch ((*e)->type)
+	    {
+	    case ne_epsilon:
+	    case ne_side_effect:
+	      t = *e;
+	      *e = t->next;
+	      rx_free_nfa_edge (t);
+	      break;
+
+	    default:
+	      e = &(*e)->next;
+	      break;
+	    }
+	}
+      n = n->next;
+    }
+}
+
+
+
+/* This page: storing the nfa in a contiguous region of memory for
+ * subsequent conversion to a super-nfa.
+ */
+
+/* This is for qsort on an array of nfa_states. The order
+ * is based on state ids and goes 
+ *		[0...MAX][MIN..-1] where (MAX>=0) and (MIN<0)
+ * This way, positive ids double as array indices.
+ */
+
+#ifdef __STDC__
+static int 
+nfacmp (void * va, void * vb)
+#else
+static int 
+nfacmp (va, vb)
+     void * va;
+     void * vb;
+#endif
+{
+  struct rx_nfa_state **a = (struct rx_nfa_state **)va;
+  struct rx_nfa_state **b = (struct rx_nfa_state **)vb;
+  return (*a == *b		/* &&&& 3.18 */
+	  ? 0
+	  : (((*a)->id < 0) == ((*b)->id < 0)
+	     ? (((*a)->id  < (*b)->id) ? -1 : 1)
+	     : (((*a)->id < 0)
+		? 1 : -1)));
+}
+
+#ifdef __STDC__
+static int 
+count_hash_nodes (struct rx_hash * st)
+#else
+static int 
+count_hash_nodes (st)
+     struct rx_hash * st;
+#endif
+{
+  int x;
+  int count = 0;
+  for (x = 0; x < 13; ++x)
+    count += ((st->children[x])
+	      ? count_hash_nodes (st->children[x])
+	      : st->bucket_size[x]);
+  
+  return count;
+}
+
+
+#ifdef __STDC__
+static void 
+se_memo_freer (struct rx_hash_item * node)
+#else
+static void 
+se_memo_freer (node)
+     struct rx_hash_item * node;
+#endif
+{
+  free ((char *)node->data);
+}
+
+
+#ifdef __STDC__
+static void 
+nfa_set_freer (struct rx_hash_item * node)
+#else
+static void 
+nfa_set_freer (node)
+     struct rx_hash_item * node;
+#endif
+{
+  free ((char *)node->data);
+}
+
+
+/* This copies an entire NFA into a single malloced block of memory.
+ * Mostly this is for compatability with regex.c, though it is convenient
+ * to have the nfa nodes in an array.
+ */
+
+#ifdef __STDC__
+RX_DECL int 
+rx_compactify_nfa (struct rx *rx,
+		   void **mem, unsigned long *size)
+#else
+RX_DECL int 
+rx_compactify_nfa (rx, mem, size)
+     struct rx *rx;
+     void **mem;
+     unsigned long *size;
+#endif
+{
+  int total_nodec;
+  struct rx_nfa_state *n;
+  int edgec = 0;
+  int eclosec = 0;
+  int se_list_consc = count_hash_nodes (&rx->se_list_memo);
+  int nfa_setc = count_hash_nodes (&rx->set_list_memo);
+  unsigned long total_size;
+
+  /* This takes place in two stages.   First, the total size of the
+   * nfa is computed, then structures are copied.  
+   */   
+  n = rx->nfa_states;
+  total_nodec = 0;
+  while (n)
+    {
+      struct rx_nfa_edge *e = n->edges;
+      struct rx_possible_future *ec = n->futures;
+      ++total_nodec;
+      while (e)
+	{
+	  ++edgec;
+	  e = e->next;
+	}
+      while (ec)
+	{
+	  ++eclosec;
+	  ec = ec->next;
+	}
+      n = n->next;
+    }
+
+  total_size = (total_nodec * sizeof (struct rx_nfa_state)
+		+ edgec * rx_sizeof_bitset (rx->local_cset_size)
+		+ edgec * sizeof (struct rx_nfa_edge)
+		+ nfa_setc * sizeof (struct rx_nfa_state_set)
+		+ eclosec * sizeof (struct rx_possible_future)
+		+ se_list_consc * sizeof (struct rx_se_list)
+		+ rx->reserved);
+
+  if (total_size > *size)
+    {
+      *mem = remalloc (*mem, total_size);
+      if (*mem)
+	*size = total_size;
+      else
+	return 0;
+    }
+  /* Now we've allocated the memory; this copies the NFA. */
+  {
+    static struct rx_nfa_state **scratch = 0;
+    static int scratch_alloc = 0;
+    struct rx_nfa_state *state_base = (struct rx_nfa_state *) * mem;
+    struct rx_nfa_state *new_state = state_base;
+    struct rx_nfa_edge *new_edge =
+      (struct rx_nfa_edge *)
+	((char *) state_base + total_nodec * sizeof (struct rx_nfa_state));
+    struct rx_se_list * new_se_list =
+      (struct rx_se_list *)
+	((char *)new_edge + edgec * sizeof (struct rx_nfa_edge));
+    struct rx_possible_future *new_close =
+      ((struct rx_possible_future *)
+       ((char *) new_se_list
+	+ se_list_consc * sizeof (struct rx_se_list)));
+    struct rx_nfa_state_set * new_nfa_set =
+      ((struct rx_nfa_state_set *)
+       ((char *)new_close + eclosec * sizeof (struct rx_possible_future)));
+    char *new_bitset =
+      ((char *) new_nfa_set + nfa_setc * sizeof (struct rx_nfa_state_set));
+    int x;
+    struct rx_nfa_state *n;
+
+    if (scratch_alloc < total_nodec)
+      {
+	scratch = ((struct rx_nfa_state **)
+		   remalloc (scratch, total_nodec * sizeof (*scratch)));
+	if (scratch)
+	  scratch_alloc = total_nodec;
+	else
+	  {
+	    scratch_alloc = 0;
+	    return 0;
+	  }
+      }
+
+    for (x = 0, n = rx->nfa_states; n; n = n->next)
+      scratch[x++] = n;
+
+    qsort (scratch, total_nodec,
+	   sizeof (struct rx_nfa_state *), (int (*)())nfacmp);
+
+    for (x = 0; x < total_nodec; ++x)
+      {
+	struct rx_possible_future *eclose = scratch[x]->futures;
+	struct rx_nfa_edge *edge = scratch[x]->edges;
+	struct rx_nfa_state *cn = new_state++;
+	cn->futures = 0;
+	cn->edges = 0;
+	cn->next = (x == total_nodec - 1) ? 0 : (cn + 1);
+	cn->id = scratch[x]->id;
+	cn->is_final = scratch[x]->is_final;
+	cn->is_start = scratch[x]->is_start;
+	cn->mark = 0;
+	while (edge)
+	  {
+	    int indx = (edge->dest->id < 0
+			 ? (total_nodec + edge->dest->id)
+			 : edge->dest->id);
+	    struct rx_nfa_edge *e = new_edge++;
+	    rx_Bitset cset = (rx_Bitset) new_bitset;
+	    new_bitset += rx_sizeof_bitset (rx->local_cset_size);
+	    rx_bitset_null (rx->local_cset_size, cset);
+	    rx_bitset_union (rx->local_cset_size, cset, edge->params.cset);
+	    e->next = cn->edges;
+	    cn->edges = e;
+	    e->type = edge->type;
+	    e->dest = state_base + indx;
+	    e->params.cset = cset;
+	    edge = edge->next;
+	  }
+	while (eclose)
+	  {
+	    struct rx_possible_future *ec = new_close++;
+	    struct rx_hash_item * sp;
+	    struct rx_se_list ** sepos;
+	    struct rx_se_list * sesrc;
+	    struct rx_nfa_state_set * destlst;
+	    struct rx_nfa_state_set ** destpos;
+	    ec->next = cn->futures;
+	    cn->futures = ec;
+	    for (sepos = &ec->effects, sesrc = eclose->effects;
+		 sesrc;
+		 sesrc = sesrc->cdr, sepos = &(*sepos)->cdr)
+	      {
+		sp = rx_hash_find (&rx->se_list_memo,
+				   (long)sesrc->car ^ (long)sesrc->cdr,
+				   sesrc, &se_list_hash_rules);
+		if (sp->binding)
+		  {
+		    sesrc = (struct rx_se_list *)sp->binding;
+		    break;
+		  }
+		*new_se_list = *sesrc;
+		sp->binding = (void *)new_se_list;
+		*sepos = new_se_list;
+		++new_se_list;
+	      }
+	    *sepos = sesrc;
+	    for (destpos = &ec->destset, destlst = eclose->destset;
+		 destlst;
+		 destpos = &(*destpos)->cdr, destlst = destlst->cdr)
+	      {
+		sp = rx_hash_find (&rx->set_list_memo,
+				   ((((long)destlst->car) >> 8)
+				    ^ (long)destlst->cdr),
+				   destlst, &nfa_set_hash_rules);
+		if (sp->binding)
+		  {
+		    destlst = (struct rx_nfa_state_set *)sp->binding;
+		    break;
+		  }
+		*new_nfa_set = *destlst;
+		new_nfa_set->car = state_base + destlst->car->id;
+		sp->binding = (void *)new_nfa_set;
+		*destpos = new_nfa_set;
+		++new_nfa_set;
+	      }
+	    *destpos = destlst;
+	    eclose = eclose->next;
+	  }
+      }
+  }
+  rx_free_hash_table (&rx->se_list_memo, se_memo_freer, &se_list_hash_rules);
+  bzero (&rx->se_list_memo, sizeof (rx->se_list_memo));
+  rx_free_hash_table (&rx->set_list_memo, nfa_set_freer, &nfa_set_hash_rules);
+  bzero (&rx->set_list_memo, sizeof (rx->set_list_memo));
+
+  rx_free_nfa (rx);
+  rx->nfa_states = (struct rx_nfa_state *)*mem;
+  return 1;
+}
+
+
+
+/* The functions in the next several pages define the lazy-NFA-conversion used
+ * by matchers.  The input to this construction is an NFA such as 
+ * is built by compactify_nfa (rx.c).  The output is the superNFA.
+ */
+
+/* Match engines can use arbitrary values for opcodes.  So, the parse tree 
+ * is built using instructions names (enum rx_opcode), but the superstate
+ * nfa is populated with mystery opcodes (void *).
+ *
+ * For convenience, here is an id table.  The opcodes are == to their inxs
+ *
+ * The lables in re_search_2 would make good values for instructions.
+ */
+
+void * rx_id_instruction_table[rx_num_instructions] =
+{
+  (void *) rx_backtrack_point,
+  (void *) rx_do_side_effects,
+  (void *) rx_cache_miss,
+  (void *) rx_next_char,
+  (void *) rx_backtrack,
+  (void *) rx_error_inx
+};
+
+
+
+
+/* Memory mgt. for superstate graphs. */
+
+#ifdef __STDC__
+static char *
+rx_cache_malloc (struct rx_cache * cache, int bytes)
+#else
+static char *
+rx_cache_malloc (cache, bytes)
+     struct rx_cache * cache;
+     int bytes;
+#endif
+{
+  while (cache->bytes_left < bytes)
+    {
+      if (cache->memory_pos)
+	cache->memory_pos = cache->memory_pos->next;
+      if (!cache->memory_pos)
+	{
+	  cache->morecore (cache);
+	  if (!cache->memory_pos)
+	    return 0;
+	}
+      cache->bytes_left = cache->memory_pos->bytes;
+      cache->memory_addr = ((char *)cache->memory_pos
+			    + sizeof (struct rx_blocklist));
+    }
+  cache->bytes_left -= bytes;
+  {
+    char * addr = cache->memory_addr;
+    cache->memory_addr += bytes;
+    return addr;
+  }
+}
+
+#ifdef __STDC__
+static void
+rx_cache_free (struct rx_cache * cache,
+	       struct rx_freelist ** freelist, char * mem)
+#else
+static void
+rx_cache_free (cache, freelist, mem)
+     struct rx_cache * cache;
+     struct rx_freelist ** freelist;
+     char * mem;
+#endif
+{
+  struct rx_freelist * it = (struct rx_freelist *)mem;
+  it->next = *freelist;
+  *freelist = it;
+}
+
+
+/* The partially instantiated superstate graph has a transition 
+ * table at every node.  There is one entry for every character.
+ * This fills in the transition for a set.
+ */
+#ifdef __STDC__
+static void 
+install_transition (struct rx_superstate *super,
+		    struct rx_inx *answer, rx_Bitset trcset) 
+#else
+static void 
+install_transition (super, answer, trcset)
+     struct rx_superstate *super;
+     struct rx_inx *answer;
+     rx_Bitset trcset;
+#endif
+{
+  struct rx_inx * transitions = super->transitions;
+  int chr;
+  for (chr = 0; chr < 256; )
+    if (!*trcset)
+      {
+	++trcset;
+	chr += 32;
+      }
+    else
+      {
+	RX_subset sub = *trcset;
+	RX_subset mask = 1;
+	int bound = chr + 32;
+	while (chr < bound)
+	  {
+	    if (sub & mask)
+	      transitions [chr] = *answer;
+	    ++chr;
+	    mask <<= 1;
+	  }
+	++trcset;
+      }
+}
+
+
+#ifdef __STDC__
+static int
+qlen (struct rx_superstate * q)
+#else
+static int
+qlen (q)
+     struct rx_superstate * q;
+#endif
+{
+  int count = 1;
+  struct rx_superstate * it;
+  if (!q)
+    return 0;
+  for (it = q->next_recyclable; it != q; it = it->next_recyclable)
+    ++count;
+  return count;
+}
+
+#ifdef __STDC__
+static void
+check_cache (struct rx_cache * cache)
+#else
+static void
+check_cache (cache)
+     struct rx_cache * cache;
+#endif
+{
+  struct rx_cache * you_fucked_up = 0;
+  int total = cache->superstates;
+  int semi = cache->semifree_superstates;
+  if (semi != qlen (cache->semifree_superstate))
+    check_cache (you_fucked_up);
+  if ((total - semi) != qlen (cache->lru_superstate))
+    check_cache (you_fucked_up);
+}
+
+/* When a superstate is old and neglected, it can enter a 
+ * semi-free state.  A semi-free state is slated to die.
+ * Incoming transitions to a semi-free state are re-written
+ * to cause an (interpreted) fault when they are taken.
+ * The fault handler revives the semi-free state, patches
+ * incoming transitions back to normal, and continues.
+ *
+ * The idea is basicly to free in two stages, aborting 
+ * between the two if the state turns out to be useful again.
+ * When a free is aborted, the rescued superstate is placed
+ * in the most-favored slot to maximize the time until it
+ * is next semi-freed.
+ */
+
+#ifdef __STDC__
+static void
+semifree_superstate (struct rx_cache * cache)
+#else
+static void
+semifree_superstate (cache)
+     struct rx_cache * cache;
+#endif
+{
+  int disqualified = cache->semifree_superstates;
+  if (disqualified == cache->superstates)
+    return;
+  while (cache->lru_superstate->locks)
+    {
+      cache->lru_superstate = cache->lru_superstate->next_recyclable;
+      ++disqualified;
+      if (disqualified == cache->superstates)
+	return;
+    }
+  {
+    struct rx_superstate * it = cache->lru_superstate;
+    it->next_recyclable->prev_recyclable = it->prev_recyclable;
+    it->prev_recyclable->next_recyclable = it->next_recyclable;
+    cache->lru_superstate = (it == it->next_recyclable
+			     ? 0
+			     : it->next_recyclable);
+    if (!cache->semifree_superstate)
+      {
+	cache->semifree_superstate = it;
+	it->next_recyclable = it;
+	it->prev_recyclable = it;
+      }
+    else
+      {
+	it->prev_recyclable = cache->semifree_superstate->prev_recyclable;
+	it->next_recyclable = cache->semifree_superstate;
+	it->prev_recyclable->next_recyclable = it;
+	it->next_recyclable->prev_recyclable = it;
+      }
+    {
+      struct rx_distinct_future *df;
+      it->is_semifree = 1;
+      ++cache->semifree_superstates;
+      df = it->transition_refs;
+      if (df)
+	{
+	  df->prev_same_dest->next_same_dest = 0;
+	  for (df = it->transition_refs; df; df = df->next_same_dest)
+	    {
+	      df->future_frame.inx = cache->instruction_table[rx_cache_miss];
+	      df->future_frame.data = 0;
+	      df->future_frame.data_2 = (void *) df;
+	      /* If there are any NEXT-CHAR instruction frames that
+	       * refer to this state, we convert them to CACHE-MISS frames.
+	       */
+	      if (!df->effects
+		  && (df->edge->options->next_same_super_edge[0]
+		      == df->edge->options))
+		install_transition (df->present, &df->future_frame,
+				    df->edge->cset);
+	    }
+	  df = it->transition_refs;
+	  df->prev_same_dest->next_same_dest = df;
+	}
+    }
+  }
+}
+
+
+#ifdef __STDC__
+static void 
+refresh_semifree_superstate (struct rx_cache * cache,
+			     struct rx_superstate * super)
+#else
+static void 
+refresh_semifree_superstate (cache, super)
+     struct rx_cache * cache;
+     struct rx_superstate * super;
+#endif
+{
+  struct rx_distinct_future *df;
+
+  if (super->transition_refs)
+    {
+      super->transition_refs->prev_same_dest->next_same_dest = 0; 
+      for (df = super->transition_refs; df; df = df->next_same_dest)
+	{
+	  df->future_frame.inx = cache->instruction_table[rx_next_char];
+	  df->future_frame.data = (void *) super->transitions;
+	  /* CACHE-MISS instruction frames that refer to this state,
+	   * must be converted to NEXT-CHAR frames.
+	   */
+	  if (!df->effects
+	      && (df->edge->options->next_same_super_edge[0]
+		  == df->edge->options))
+	    install_transition (df->present, &df->future_frame,
+				df->edge->cset);
+	}
+      super->transition_refs->prev_same_dest->next_same_dest
+	= super->transition_refs;
+    }
+  if (cache->semifree_superstate == super)
+    cache->semifree_superstate = (super->prev_recyclable == super
+				  ? 0
+				  : super->prev_recyclable);
+  super->next_recyclable->prev_recyclable = super->prev_recyclable;
+  super->prev_recyclable->next_recyclable = super->next_recyclable;
+
+  if (!cache->lru_superstate)
+    (cache->lru_superstate
+     = super->next_recyclable
+     = super->prev_recyclable
+     = super);
+  else
+    {
+      super->next_recyclable = cache->lru_superstate;
+      super->prev_recyclable = cache->lru_superstate->prev_recyclable;
+      super->next_recyclable->prev_recyclable = super;
+      super->prev_recyclable->next_recyclable = super;
+    }
+  super->is_semifree = 0;
+  --cache->semifree_superstates;
+}
+
+#ifdef __STDC__
+static void
+rx_refresh_this_superstate (struct rx_cache * cache, struct rx_superstate * superstate)
+#else
+static void
+rx_refresh_this_superstate (cache, superstate)
+     struct rx_cache * cache;
+     struct rx_superstate * superstate;
+#endif
+{
+  if (superstate->is_semifree)
+    refresh_semifree_superstate (cache, superstate);
+  else if (cache->lru_superstate == superstate)
+    cache->lru_superstate = superstate->next_recyclable;
+  else if (superstate != cache->lru_superstate->prev_recyclable)
+    {
+      superstate->next_recyclable->prev_recyclable
+	= superstate->prev_recyclable;
+      superstate->prev_recyclable->next_recyclable
+	= superstate->next_recyclable;
+      superstate->next_recyclable = cache->lru_superstate;
+      superstate->prev_recyclable = cache->lru_superstate->prev_recyclable;
+      superstate->next_recyclable->prev_recyclable = superstate;
+      superstate->prev_recyclable->next_recyclable = superstate;
+    }
+}
+
+#ifdef __STDC__
+static void 
+release_superset_low (struct rx_cache * cache,
+		     struct rx_superset *set)
+#else
+static void 
+release_superset_low (cache, set)
+     struct rx_cache * cache;
+     struct rx_superset *set;
+#endif
+{
+  if (!--set->refs)
+    {
+      if (set->cdr)
+	release_superset_low (cache, set->cdr);
+
+      set->starts_for = 0;
+
+      rx_hash_free
+	(rx_hash_find
+	 (&cache->superset_table,
+	  (unsigned long)set->car ^ set->id ^ (unsigned long)set->cdr,
+	  (void *)set,
+	  &cache->superset_hash_rules),
+	 &cache->superset_hash_rules);
+      rx_cache_free (cache, &cache->free_supersets, (char *)set);
+    }
+}
+
+#ifdef __STDC__
+RX_DECL void 
+rx_release_superset (struct rx *rx,
+		     struct rx_superset *set)
+#else
+RX_DECL void 
+rx_release_superset (rx, set)
+     struct rx *rx;
+     struct rx_superset *set;
+#endif
+{
+  release_superset_low (rx->cache, set);
+}
+
+/* This tries to add a new superstate to the superstate freelist.
+ * It might, as a result, free some edge pieces or hash tables.
+ * If nothing can be freed because too many locks are being held, fail.
+ */
+
+#ifdef __STDC__
+static int
+rx_really_free_superstate (struct rx_cache * cache)
+#else
+static int
+rx_really_free_superstate (cache)
+     struct rx_cache * cache;
+#endif
+{
+  int locked_superstates = 0;
+  struct rx_superstate * it;
+
+  if (!cache->superstates)
+    return 0;
+
+  {
+    /* This is a total guess.  The idea is that we should expect as
+     * many misses as we've recently experienced.  I.e., cache->misses
+     * should be the same as cache->semifree_superstates.
+     */
+    while ((cache->hits + cache->misses) > cache->superstates_allowed)
+      {
+	cache->hits >>= 1;
+	cache->misses >>= 1;
+      }
+    if (  ((cache->hits + cache->misses) * cache->semifree_superstates)
+	< (cache->superstates		 * cache->misses))
+      {
+	semifree_superstate (cache);
+	semifree_superstate (cache);
+      }
+  }
+
+  while (cache->semifree_superstate && cache->semifree_superstate->locks)
+    {
+      refresh_semifree_superstate (cache, cache->semifree_superstate);
+      ++locked_superstates;
+      if (locked_superstates == cache->superstates)
+	return 0;
+    }
+
+  if (cache->semifree_superstate)
+    {
+      it = cache->semifree_superstate;
+      it->next_recyclable->prev_recyclable = it->prev_recyclable;
+      it->prev_recyclable->next_recyclable = it->next_recyclable;
+      cache->semifree_superstate = ((it == it->next_recyclable)
+				    ? 0
+				    : it->next_recyclable);
+      --cache->semifree_superstates;
+    }
+  else
+    {
+      while (cache->lru_superstate->locks)
+	{
+	  cache->lru_superstate = cache->lru_superstate->next_recyclable;
+	  ++locked_superstates;
+	  if (locked_superstates == cache->superstates)
+	    return 0;
+	}
+      it = cache->lru_superstate;
+      it->next_recyclable->prev_recyclable = it->prev_recyclable;
+      it->prev_recyclable->next_recyclable = it->next_recyclable;
+      cache->lru_superstate = ((it == it->next_recyclable)
+				    ? 0
+				    : it->next_recyclable);
+    }
+
+  if (it->transition_refs)
+    {
+      struct rx_distinct_future *df;
+      for (df = it->transition_refs,
+	   df->prev_same_dest->next_same_dest = 0;
+	   df;
+	   df = df->next_same_dest)
+	{
+	  df->future_frame.inx = cache->instruction_table[rx_cache_miss];
+	  df->future_frame.data = 0;
+	  df->future_frame.data_2 = (void *) df;
+	  df->future = 0;
+	}
+      it->transition_refs->prev_same_dest->next_same_dest =
+	it->transition_refs;
+    }
+  {
+    struct rx_super_edge *tc = it->edges;
+    while (tc)
+      {
+	struct rx_distinct_future * df;
+	struct rx_super_edge *tct = tc->next;
+	df = tc->options;
+	df->next_same_super_edge[1]->next_same_super_edge[0] = 0;
+	while (df)
+	  {
+	    struct rx_distinct_future *dft = df;
+	    df = df->next_same_super_edge[0];
+	    
+	    
+	    if (dft->future && dft->future->transition_refs == dft)
+	      {
+		dft->future->transition_refs = dft->next_same_dest;
+		if (dft->future->transition_refs == dft)
+		  dft->future->transition_refs = 0;
+	      }
+	    dft->next_same_dest->prev_same_dest = dft->prev_same_dest;
+	    dft->prev_same_dest->next_same_dest = dft->next_same_dest;
+	    rx_cache_free (cache, &cache->free_discernable_futures,
+			   (char *)dft);
+	  }
+	rx_cache_free (cache, &cache->free_transition_classes, (char *)tc);
+	tc = tct;
+      }
+  }
+  
+  if (it->contents->superstate == it)
+    it->contents->superstate = 0;
+  release_superset_low (cache, it->contents);
+  rx_cache_free (cache, &cache->free_superstates, (char *)it);
+  --cache->superstates;
+  return 1;
+}
+
+#ifdef __STDC__
+static char *
+rx_cache_get (struct rx_cache * cache,
+	      struct rx_freelist ** freelist)
+#else
+static char *
+rx_cache_get (cache, freelist)
+     struct rx_cache * cache;
+     struct rx_freelist ** freelist;
+#endif
+{
+  while (!*freelist && rx_really_free_superstate (cache))
+    ;
+  if (!*freelist)
+    return 0;
+  {
+    struct rx_freelist * it = *freelist;
+    *freelist = it->next;
+    return (char *)it;
+  }
+}
+
+#ifdef __STDC__
+static char *
+rx_cache_malloc_or_get (struct rx_cache * cache,
+			struct rx_freelist ** freelist, int bytes)
+#else
+static char *
+rx_cache_malloc_or_get (cache, freelist, bytes)
+     struct rx_cache * cache;
+     struct rx_freelist ** freelist;
+     int bytes;
+#endif
+{
+  if (!*freelist)
+    {
+      char * answer = rx_cache_malloc (cache, bytes);
+      if (answer)
+	return answer;
+    }
+
+  return rx_cache_get (cache, freelist);
+}
+
+#ifdef __STDC__
+static char *
+rx_cache_get_superstate (struct rx_cache * cache)
+#else
+static char *
+rx_cache_get_superstate (cache)
+	  struct rx_cache * cache;
+#endif
+{
+  char * answer;
+  int bytes = (   sizeof (struct rx_superstate)
+	       +  cache->local_cset_size * sizeof (struct rx_inx));
+  if (!cache->free_superstates
+      && (cache->superstates < cache->superstates_allowed))
+    {
+      answer = rx_cache_malloc (cache, bytes);
+      if (answer)
+	{
+	  ++cache->superstates;
+	  return answer;
+	}
+    }
+  answer = rx_cache_get (cache, &cache->free_superstates);
+  if (!answer)
+    {
+      answer = rx_cache_malloc (cache, bytes);
+      if (answer)
+	++cache->superstates_allowed;
+    }
+  ++cache->superstates;
+  return answer;
+}
+
+
+
+
+#ifdef __STDC__
+static int
+supersetcmp (void * va, void * vb)
+#else
+static int
+supersetcmp (va, vb)
+     void * va;
+     void * vb;
+#endif
+{
+  struct rx_superset * a = (struct rx_superset *)va;
+  struct rx_superset * b = (struct rx_superset *)vb;
+  return (   (a == b)
+	  || (a && b && (a->car == b->car) && (a->cdr == b->cdr)));
+}
+
+#ifdef __STDC__
+static struct rx_hash_item *
+superset_allocator (struct rx_hash_rules * rules, void * val)
+#else
+static struct rx_hash_item *
+superset_allocator (rules, val)
+     struct rx_hash_rules * rules;
+     void * val;
+#endif
+{
+  struct rx_cache * cache
+    = ((struct rx_cache *)
+       ((char *)rules
+	- (unsigned long)(&((struct rx_cache *)0)->superset_hash_rules)));
+  struct rx_superset * template = (struct rx_superset *)val;
+  struct rx_superset * newset
+    = ((struct rx_superset *)
+       rx_cache_malloc_or_get (cache,
+			       &cache->free_supersets,
+			       sizeof (*template)));
+  if (!newset)
+    return 0;
+  newset->refs = 0;
+  newset->car = template->car;
+  newset->id = template->car->id;
+  newset->cdr = template->cdr;
+  newset->superstate = 0;
+  rx_protect_superset (rx, template->cdr);
+  newset->hash_item.data = (void *)newset;
+  newset->hash_item.binding = 0;
+  return &newset->hash_item;
+}
+
+#ifdef __STDC__
+static struct rx_hash * 
+super_hash_allocator (struct rx_hash_rules * rules)
+#else
+static struct rx_hash * 
+super_hash_allocator (rules)
+     struct rx_hash_rules * rules;
+#endif
+{
+  struct rx_cache * cache
+    = ((struct rx_cache *)
+       ((char *)rules
+	- (unsigned long)(&((struct rx_cache *)0)->superset_hash_rules)));
+  return ((struct rx_hash *)
+	  rx_cache_malloc_or_get (cache,
+				  &cache->free_hash, sizeof (struct rx_hash)));
+}
+
+
+#ifdef __STDC__
+static void
+super_hash_liberator (struct rx_hash * hash, struct rx_hash_rules * rules)
+#else
+static void
+super_hash_liberator (hash, rules)
+     struct rx_hash * hash;
+     struct rx_hash_rules * rules;
+#endif
+{
+  struct rx_cache * cache
+    = ((struct rx_cache *)
+       (char *)rules - (long)(&((struct rx_cache *)0)->superset_hash_rules));
+  rx_cache_free (cache, &cache->free_hash, (char *)hash);
+}
+
+#ifdef __STDC__
+static void
+superset_hash_item_liberator (struct rx_hash_item * it,
+			      struct rx_hash_rules * rules)
+#else
+static void
+superset_hash_item_liberator (it, rules) /* Well, it does ya know. */
+     struct rx_hash_item * it;
+     struct rx_hash_rules * rules;
+#endif
+{
+}
+
+int rx_cache_bound = 128;
+static int rx_default_cache_got = 0;
+
+#ifdef __STDC__
+static int
+bytes_for_cache_size (int supers, int cset_size)
+#else
+static int
+bytes_for_cache_size (supers, cset_size)
+     int supers;
+     int cset_size;
+#endif
+{
+  /* What the hell is this? !!!*/
+  return (int)
+    ((float)supers *
+     (  (1.03 * (float) (  rx_sizeof_bitset (cset_size)
+			 + sizeof (struct rx_super_edge)))
+      + (1.80 * (float) sizeof (struct rx_possible_future))
+      + (float) (  sizeof (struct rx_superstate)
+		 + cset_size * sizeof (struct rx_inx))));
+}
+
+#ifdef __STDC__
+static void
+rx_morecore (struct rx_cache * cache)
+#else
+static void
+rx_morecore (cache)
+     struct rx_cache * cache;
+#endif
+{
+  if (rx_default_cache_got >= rx_cache_bound)
+    return;
+
+  rx_default_cache_got += 16;
+  cache->superstates_allowed = rx_cache_bound;
+  {
+    struct rx_blocklist ** pos = &cache->memory;
+    int size = bytes_for_cache_size (16, cache->local_cset_size);
+    while (*pos)
+      pos = &(*pos)->next;
+    *pos = ((struct rx_blocklist *)
+	    malloc (size + sizeof (struct rx_blocklist))); 
+    if (!*pos)
+      return;
+
+    (*pos)->next = 0;
+    (*pos)->bytes = size;
+    cache->memory_pos = *pos;
+    cache->memory_addr = (char *)*pos + sizeof (**pos);
+    cache->bytes_left = size;
+  }
+}
+
+static struct rx_cache default_cache = 
+{
+  {
+    supersetcmp,
+    super_hash_allocator,
+    super_hash_liberator,
+    superset_allocator,
+    superset_hash_item_liberator,
+  },
+  0,
+  0,
+  0,
+  0,
+  rx_morecore,
+
+  0,
+  0,
+  0,
+  0,
+  0,
+
+  0,
+  0,
+
+  0,
+
+  0,
+  0,
+  0,
+  0,
+  128,
+
+  256,
+  rx_id_instruction_table,
+
+  {
+    0,
+    0,
+    {0},
+    {0},
+    {0}
+  }
+};
+
+/* This adds an element to a superstate set.  These sets are lists, such
+ * that lists with == elements are ==.  The empty set is returned by
+ * superset_cons (rx, 0, 0) and is NOT equivelent to 
+ * (struct rx_superset)0.
+ */
+
+#ifdef __STDC__
+RX_DECL struct rx_superset *
+rx_superset_cons (struct rx * rx,
+		  struct rx_nfa_state *car, struct rx_superset *cdr)
+#else
+RX_DECL struct rx_superset *
+rx_superset_cons (rx, car, cdr)
+     struct rx * rx;
+     struct rx_nfa_state *car;
+     struct rx_superset *cdr;
+#endif
+{
+  struct rx_cache * cache = rx->cache;
+  if (!car && !cdr)
+    {
+      if (!cache->empty_superset)
+	{
+	  cache->empty_superset
+	    = ((struct rx_superset *)
+	       rx_cache_malloc_or_get (cache, &cache->free_supersets,
+				       sizeof (struct rx_superset)));
+	  if (!cache->empty_superset)
+	    return 0;
+	  bzero (cache->empty_superset, sizeof (struct rx_superset));
+	  cache->empty_superset->refs = 1000;
+	}
+      return cache->empty_superset;
+    }
+  {
+    struct rx_superset template;
+    struct rx_hash_item * hit;
+    template.car = car;
+    template.cdr = cdr;
+    template.id = car->id;
+    hit = rx_hash_store (&cache->superset_table,
+			 (unsigned long)car ^ car->id ^ (unsigned long)cdr,
+			 (void *)&template,
+			 &cache->superset_hash_rules);
+    return (hit
+	    ?  (struct rx_superset *)hit->data
+	    : 0);
+  }
+}
+
+/* This computes a union of two NFA state sets.  The sets do not have the
+ * same representation though.  One is a RX_SUPERSET structure (part
+ * of the superstate NFA) and the other is an NFA_STATE_SET (part of the NFA).
+ */
+
+#ifdef __STDC__
+RX_DECL struct rx_superset *
+rx_superstate_eclosure_union
+  (struct rx * rx, struct rx_superset *set, struct rx_nfa_state_set *ecl) 
+#else
+RX_DECL struct rx_superset *
+rx_superstate_eclosure_union (rx, set, ecl)
+     struct rx * rx;
+     struct rx_superset *set;
+     struct rx_nfa_state_set *ecl;
+#endif
+{
+  if (!ecl)
+    return set;
+
+  if (!set->car)
+    return rx_superset_cons (rx, ecl->car,
+			     rx_superstate_eclosure_union (rx, set, ecl->cdr));
+  if (set->car == ecl->car)
+    return rx_superstate_eclosure_union (rx, set, ecl->cdr);
+
+  {
+    struct rx_superset * tail;
+    struct rx_nfa_state * first;
+
+    if (set->car > ecl->car)
+      {
+	tail = rx_superstate_eclosure_union (rx, set->cdr, ecl);
+	first = set->car;
+      }
+    else
+      {
+	tail = rx_superstate_eclosure_union (rx, set, ecl->cdr);
+	first = ecl->car;
+      }
+    if (!tail)
+      return 0;
+    else
+      {
+	struct rx_superset * answer;
+	answer = rx_superset_cons (rx, first, tail);
+	if (!answer)
+	  {
+	    rx_protect_superset (rx, tail);
+	    rx_release_superset (rx, tail);
+	    return 0;
+	  }
+	else
+	  return answer;
+      }
+  }
+}
+
+
+
+
+
+/*
+ * This makes sure that a list of rx_distinct_futures contains
+ * a future for each possible set of side effects in the eclosure
+ * of a given state.  This is some of the work of filling in a
+ * superstate transition. 
+ */
+
+#ifdef __STDC__
+static struct rx_distinct_future *
+include_futures (struct rx *rx,
+		 struct rx_distinct_future *df, struct rx_nfa_state
+		 *state, struct rx_superstate *superstate) 
+#else
+static struct rx_distinct_future *
+include_futures (rx, df, state, superstate)
+     struct rx *rx;
+     struct rx_distinct_future *df;
+     struct rx_nfa_state *state;
+     struct rx_superstate *superstate;
+#endif
+{
+  struct rx_possible_future *future;
+  struct rx_cache * cache = rx->cache;
+  for (future = state->futures; future; future = future->next)
+    {
+      struct rx_distinct_future *dfp;
+      struct rx_distinct_future *insert_before = 0;
+      if (df)
+	df->next_same_super_edge[1]->next_same_super_edge[0] = 0;
+      for (dfp = df; dfp; dfp = dfp->next_same_super_edge[0])
+	if (dfp->effects == future->effects)
+	  break;
+	else
+	  {
+	    int order = rx->se_list_cmp (rx, dfp->effects, future->effects);
+	    if (order > 0)
+	      {
+		insert_before = dfp;
+		dfp = 0;
+		break;
+	      }
+	  }
+      if (df)
+	df->next_same_super_edge[1]->next_same_super_edge[0] = df;
+      if (!dfp)
+	{
+	  dfp
+	    = ((struct rx_distinct_future *)
+	       rx_cache_malloc_or_get (cache, &cache->free_discernable_futures,
+				       sizeof (struct rx_distinct_future)));
+	  if (!dfp)
+	    return 0;
+	  if (!df)
+	    {
+	      df = insert_before = dfp;
+	      df->next_same_super_edge[0] = df->next_same_super_edge[1] = df;
+	    }
+	  else if (!insert_before)
+	    insert_before = df;
+	  else if (insert_before == df)
+	    df = dfp;
+
+	  dfp->next_same_super_edge[0] = insert_before;
+	  dfp->next_same_super_edge[1]
+	    = insert_before->next_same_super_edge[1];
+	  dfp->next_same_super_edge[1]->next_same_super_edge[0] = dfp;
+	  dfp->next_same_super_edge[0]->next_same_super_edge[1] = dfp;
+	  dfp->next_same_dest = dfp->prev_same_dest = dfp;
+	  dfp->future = 0;
+	  dfp->present = superstate;
+	  dfp->future_frame.inx = rx->instruction_table[rx_cache_miss];
+	  dfp->future_frame.data = 0;
+	  dfp->future_frame.data_2 = (void *) dfp;
+	  dfp->side_effects_frame.inx
+	    = rx->instruction_table[rx_do_side_effects];
+	  dfp->side_effects_frame.data = 0;
+	  dfp->side_effects_frame.data_2 = (void *) dfp;
+	  dfp->effects = future->effects;
+	}
+    }
+  return df;
+}
+
+
+
+
+/* This constructs a new superstate from its state set.  The only 
+ * complexity here is memory management.
+ */
+#ifdef __STDC__
+RX_DECL struct rx_superstate *
+rx_superstate (struct rx *rx,
+	       struct rx_superset *set)
+#else
+RX_DECL struct rx_superstate *
+rx_superstate (rx, set)
+     struct rx *rx;
+     struct rx_superset *set;
+#endif
+{
+  struct rx_cache * cache = rx->cache;
+  struct rx_superstate * superstate = 0;
+
+  /* Does the superstate already exist in the cache? */
+  if (set->superstate)
+    {
+      if (set->superstate->rx_id != rx->rx_id)
+	{
+	  /* Aha.  It is in the cache, but belongs to a superstate
+	   * that refers to an NFA that no longer exists.
+	   * (We know it no longer exists because it was evidently
+	   *  stored in the same region of memory as the current nfa
+	   *  yet it has a different id.)
+	   */
+	  superstate = set->superstate;
+	  if (!superstate->is_semifree)
+	    {
+	      if (cache->lru_superstate == superstate)
+		{
+		  cache->lru_superstate = superstate->next_recyclable;
+		  if (cache->lru_superstate == superstate)
+		    cache->lru_superstate = 0;
+		}
+	      {
+		superstate->next_recyclable->prev_recyclable
+		  = superstate->prev_recyclable;
+		superstate->prev_recyclable->next_recyclable
+		  = superstate->next_recyclable;
+		if (!cache->semifree_superstate)
+		  {
+		    (cache->semifree_superstate
+		     = superstate->next_recyclable
+		     = superstate->prev_recyclable
+		     = superstate);
+		  }
+		else
+		  {
+		    superstate->next_recyclable = cache->semifree_superstate;
+		    superstate->prev_recyclable
+		      = cache->semifree_superstate->prev_recyclable;
+		    superstate->next_recyclable->prev_recyclable
+		      = superstate;
+		    superstate->prev_recyclable->next_recyclable
+		      = superstate;
+		    cache->semifree_superstate = superstate;
+		  }
+		++cache->semifree_superstates;
+	      }
+	    }
+	  set->superstate = 0;
+	  goto handle_cache_miss;
+	}
+      ++cache->hits;
+      superstate = set->superstate;
+
+      rx_refresh_this_superstate (cache, superstate);
+      return superstate;
+    }
+
+ handle_cache_miss:
+
+  /* This point reached only for cache misses. */
+  ++cache->misses;
+#if RX_DEBUG
+  if (rx_debug_trace > 1)
+    {
+      struct rx_superset * setp = set;
+      fprintf (stderr, "Building a superstet %d(%d): ", rx->rx_id, set);
+      while (setp)
+	{
+	  fprintf (stderr, "%d ", setp->id);
+	  setp = setp->cdr;
+	}
+      fprintf (stderr, "(%d)\n", set);
+    }
+#endif
+  superstate = (struct rx_superstate *)rx_cache_get_superstate (cache);
+  if (!superstate)
+    return 0;
+
+  if (!cache->lru_superstate)
+    (cache->lru_superstate
+     = superstate->next_recyclable
+     = superstate->prev_recyclable
+     = superstate);
+  else
+    {
+      superstate->next_recyclable = cache->lru_superstate;
+      superstate->prev_recyclable = cache->lru_superstate->prev_recyclable;
+      (  superstate->prev_recyclable->next_recyclable
+       = superstate->next_recyclable->prev_recyclable
+       = superstate);
+    }
+  superstate->rx_id = rx->rx_id;
+  superstate->transition_refs = 0;
+  superstate->locks = 0;
+  superstate->is_semifree = 0;
+  set->superstate = superstate;
+  superstate->contents = set;
+  rx_protect_superset (rx, set);
+  superstate->edges = 0;
+  {
+    int x;
+    /* None of the transitions from this superstate are known yet. */
+    for (x = 0; x < rx->local_cset_size; ++x) /* &&&&& 3.8 % */
+      {
+	struct rx_inx * ifr = &superstate->transitions[x];
+	ifr->inx = rx->instruction_table [rx_cache_miss];
+	ifr->data = ifr->data_2 = 0;
+      }
+  }
+  return superstate;
+}
+
+
+
+/* This computes the destination set of one edge of the superstate NFA.
+ * Note that a RX_DISTINCT_FUTURE is a superstate edge.
+ * Returns 0 on an allocation failure.
+ */
+
+#ifdef __STDC__
+static int 
+solve_destination (struct rx *rx, struct rx_distinct_future *df)
+#else
+static int 
+solve_destination (rx, df)
+     struct rx *rx;
+     struct rx_distinct_future *df;
+#endif
+{
+  struct rx_super_edge *tc = df->edge;
+  struct rx_superset *nfa_state;
+  struct rx_superset *nil_set = rx_superset_cons (rx, 0, 0);
+  struct rx_superset *solution = nil_set;
+  struct rx_superstate *dest;
+
+  rx_protect_superset (rx, solution);
+  /* Iterate over all NFA states in the state set of this superstate. */
+  for (nfa_state = df->present->contents;
+       nfa_state->car;
+       nfa_state = nfa_state->cdr)
+    {
+      struct rx_nfa_edge *e;
+      /* Iterate over all edges of each NFA state. */
+      for (e = nfa_state->car->edges; e; e = e->next)
+        /* If we find an edge that is labeled with 
+	 * the characters we are solving for.....
+	 */
+	if (rx_bitset_is_subset (rx->local_cset_size,
+				 tc->cset, e->params.cset))
+	  {
+	    struct rx_nfa_state *n = e->dest;
+	    struct rx_possible_future *pf;
+	    /* ....search the partial epsilon closures of the destination
+	     * of that edge for a path that involves the same set of
+	     * side effects we are solving for.
+	     * If we find such a RX_POSSIBLE_FUTURE, we add members to the
+	     * stateset we are computing.
+	     */
+	    for (pf = n->futures; pf; pf = pf->next)
+	      if (pf->effects == df->effects)
+		{
+		  struct rx_superset * old_sol;
+		  old_sol = solution;
+		  solution = rx_superstate_eclosure_union (rx, solution,
+							   pf->destset);
+		  if (!solution)
+		    return 0;
+		  rx_protect_superset (rx, solution);
+		  rx_release_superset (rx, old_sol);
+		}
+	  }
+    }
+  /* It is possible that the RX_DISTINCT_FUTURE we are working on has 
+   * the empty set of NFA states as its definition.  In that case, this
+   * is a failure point.
+   */
+  if (solution == nil_set)
+    {
+      df->future_frame.inx = (void *) rx_backtrack;
+      df->future_frame.data = 0;
+      df->future_frame.data_2 = 0;
+      return 1;
+    }
+  dest = rx_superstate (rx, solution);
+  rx_release_superset (rx, solution);
+  if (!dest)
+    return 0;
+
+  {
+    struct rx_distinct_future *dft;
+    dft = df;
+    df->prev_same_dest->next_same_dest = 0;
+    while (dft)
+      {
+	dft->future = dest;
+	dft->future_frame.inx = rx->instruction_table[rx_next_char];
+	dft->future_frame.data = (void *) dest->transitions;
+	dft = dft->next_same_dest;
+      }
+    df->prev_same_dest->next_same_dest = df;
+  }
+  if (!dest->transition_refs)
+    dest->transition_refs = df;
+  else
+    {
+      struct rx_distinct_future *dft = dest->transition_refs->next_same_dest;
+      dest->transition_refs->next_same_dest = df->next_same_dest;
+      df->next_same_dest->prev_same_dest = dest->transition_refs;
+      df->next_same_dest = dft;
+      dft->prev_same_dest = df;
+    }
+  return 1;
+}
+
+
+/* This takes a superstate and a character, and computes some edges
+ * from the superstate NFA.  In particular, this computes all edges
+ * that lead from SUPERSTATE given CHR.   This function also 
+ * computes the set of characters that share this edge set.
+ * This returns 0 on allocation error.
+ * The character set and list of edges are returned through 
+ * the paramters CSETOUT and DFOUT.
+} */
+
+#ifdef __STDC__
+static int 
+compute_super_edge (struct rx *rx, struct rx_distinct_future **dfout,
+			  rx_Bitset csetout, struct rx_superstate *superstate,
+			  unsigned char chr)  
+#else
+static int 
+compute_super_edge (rx, dfout, csetout, superstate, chr)
+     struct rx *rx;
+     struct rx_distinct_future **dfout;
+     rx_Bitset csetout;
+     struct rx_superstate *superstate;
+     unsigned char chr;
+#endif
+{
+  struct rx_superset *stateset = superstate->contents;
+
+  /* To compute the set of characters that share edges with CHR, 
+   * we start with the full character set, and subtract.
+   */
+  rx_bitset_universe (rx->local_cset_size, csetout);
+  *dfout = 0;
+
+  /* Iterate over the NFA states in the superstate state-set. */
+  while (stateset->car)
+    {
+      struct rx_nfa_edge *e;
+      for (e = stateset->car->edges; e; e = e->next)
+	if (RX_bitset_member (e->params.cset, chr))
+	  {
+	    /* If we find an NFA edge that applies, we make sure there
+	     * are corresponding edges in the superstate NFA.
+	     */
+	    {
+	      struct rx_distinct_future * saved;
+	      saved = *dfout;
+	      *dfout = include_futures (rx, *dfout, e->dest, superstate);
+	      if (!*dfout)
+		{
+		  struct rx_distinct_future * df;
+		  df = saved;
+		  df->next_same_super_edge[1]->next_same_super_edge[0] = 0;
+		  while (df)
+		    {
+		      struct rx_distinct_future *dft;
+		      dft = df;
+		      df = df->next_same_super_edge[0];
+
+		      if (dft->future && dft->future->transition_refs == dft)
+			{
+			  dft->future->transition_refs = dft->next_same_dest;
+			  if (dft->future->transition_refs == dft)
+			    dft->future->transition_refs = 0;
+			}
+		      dft->next_same_dest->prev_same_dest = dft->prev_same_dest;
+		      dft->prev_same_dest->next_same_dest = dft->next_same_dest;
+		      rx_cache_free (rx->cache,
+				     &rx->cache->free_discernable_futures,
+				     (char *)dft);
+		    }
+		  return 0;
+		}
+	    }
+	    /* We also trim the character set a bit. */
+	    rx_bitset_intersection (rx->local_cset_size,
+				    csetout, e->params.cset);
+	  }
+	else
+	  /* An edge that doesn't apply at least tells us some characters
+	   * that don't share the same edge set as CHR.
+	   */
+	  rx_bitset_difference (rx->local_cset_size, csetout, e->params.cset);
+      stateset = stateset->cdr;
+    }
+  return 1;
+}
+
+
+
+/* This is a constructor for RX_SUPER_EDGE structures.  These are
+ * wrappers for lists of superstate NFA edges that share character sets labels.
+ * If a transition class contains more than one rx_distinct_future (superstate
+ * edge), then it represents a non-determinism in the superstate NFA.
+ */
+
+#ifdef __STDC__
+static struct rx_super_edge *
+rx_super_edge (struct rx *rx,
+	       struct rx_superstate *super, rx_Bitset cset,
+	       struct rx_distinct_future *df) 
+#else
+static struct rx_super_edge *
+rx_super_edge (rx, super, cset, df)
+     struct rx *rx;
+     struct rx_superstate *super;
+     rx_Bitset cset;
+     struct rx_distinct_future *df;
+#endif
+{
+  struct rx_super_edge *tc =
+    (struct rx_super_edge *)rx_cache_malloc_or_get
+      (rx->cache, &rx->cache->free_transition_classes,
+       sizeof (struct rx_super_edge) + rx_sizeof_bitset (rx->local_cset_size));
+
+  if (!tc)
+    return 0;
+  tc->next = super->edges;
+  super->edges = tc;
+  tc->rx_backtrack_frame.inx = rx->instruction_table[rx_backtrack_point];
+  tc->rx_backtrack_frame.data = 0;
+  tc->rx_backtrack_frame.data_2 = (void *) tc;
+  tc->options = df;
+  tc->cset = (rx_Bitset) ((char *) tc + sizeof (*tc));
+  rx_bitset_assign (rx->local_cset_size, tc->cset, cset);
+  if (df)
+    {
+      struct rx_distinct_future * dfp = df;
+      df->next_same_super_edge[1]->next_same_super_edge[0] = 0;
+      while (dfp)
+	{
+	  dfp->edge = tc;
+	  dfp = dfp->next_same_super_edge[0];
+	}
+      df->next_same_super_edge[1]->next_same_super_edge[0] = df;
+    }
+  return tc;
+}
+
+
+/* There are three kinds of cache miss.  The first occurs when a
+ * transition is taken that has never been computed during the
+ * lifetime of the source superstate.  That cache miss is handled by
+ * calling COMPUTE_SUPER_EDGE.  The second kind of cache miss
+ * occurs when the destination superstate of a transition doesn't
+ * exist.  SOLVE_DESTINATION is used to construct the destination superstate.
+ * Finally, the third kind of cache miss occurs when the destination
+ * superstate of a transition is in a `semi-free state'.  That case is
+ * handled by UNFREE_SUPERSTATE.
+ *
+ * The function of HANDLE_CACHE_MISS is to figure out which of these
+ * cases applies.
+ */
+
+#ifdef __STDC__
+static void
+install_partial_transition  (struct rx_superstate *super,
+			     struct rx_inx *answer,
+			     RX_subset set, int offset)
+#else
+static void
+install_partial_transition  (super, answer, set, offset)
+     struct rx_superstate *super;
+     struct rx_inx *answer;
+     RX_subset set;
+     int offset;
+#endif
+{
+  int start = offset;
+  int end = start + 32;
+  RX_subset pos = 1;
+  struct rx_inx * transitions = super->transitions;
+  
+  while (start < end)
+    {
+      if (set & pos)
+	transitions[start] = *answer;
+      pos <<= 1;
+      ++start;
+    }
+}
+
+
+#ifdef __STDC__
+RX_DECL struct rx_inx *
+rx_handle_cache_miss
+  (struct rx *rx, struct rx_superstate *super, unsigned char chr, void *data) 
+#else
+RX_DECL struct rx_inx *
+rx_handle_cache_miss (rx, super, chr, data)
+     struct rx *rx;
+     struct rx_superstate *super;
+     unsigned char chr;
+     void *data;
+#endif
+{
+  int offset = chr / RX_subset_bits;
+  struct rx_distinct_future *df = data;
+
+  if (!df)			/* must be the shared_cache_miss_frame */
+    {
+      /* Perhaps this is just a transition waiting to be filled. */
+      struct rx_super_edge *tc;
+      RX_subset mask = rx_subset_singletons [chr % RX_subset_bits];
+
+      for (tc = super->edges; tc; tc = tc->next)
+	if (tc->cset[offset] & mask)
+	  {
+	    struct rx_inx * answer;
+	    df = tc->options;
+	    answer = ((tc->options->next_same_super_edge[0] != tc->options)
+		      ? &tc->rx_backtrack_frame
+		      : (df->effects
+			 ? &df->side_effects_frame
+			 : &df->future_frame));
+	    install_partial_transition (super, answer,
+					tc->cset [offset], offset * 32);
+	    return answer;
+	  }
+      /* Otherwise, it's a flushed or  newly encountered edge. */
+      {
+	char cset_space[1024];	/* this limit is far from unreasonable */
+	rx_Bitset trcset;
+	struct rx_inx *answer;
+
+	if (rx_sizeof_bitset (rx->local_cset_size) > sizeof (cset_space))
+	  return 0;		/* If the arbitrary limit is hit, always fail */
+				/* cleanly. */
+	trcset = (rx_Bitset)cset_space;
+	rx_lock_superstate (rx, super);
+	if (!compute_super_edge (rx, &df, trcset, super, chr))
+	  {
+	    rx_unlock_superstate (rx, super);
+	    return 0;
+	  }
+	if (!df)		/* We just computed the fail transition. */
+	  {
+	    static struct rx_inx
+	      shared_fail_frame = { 0, 0, (void *)rx_backtrack, 0 };
+	    answer = &shared_fail_frame;
+	  }
+	else
+	  {
+	    tc = rx_super_edge (rx, super, trcset, df);
+	    if (!tc)
+	      {
+		rx_unlock_superstate (rx, super);
+		return 0;
+	      }
+	    answer = ((tc->options->next_same_super_edge[0] != tc->options)
+		      ? &tc->rx_backtrack_frame
+		      : (df->effects
+			 ? &df->side_effects_frame
+			 : &df->future_frame));
+	  }
+	install_partial_transition (super, answer,
+				    trcset[offset], offset * 32);
+	rx_unlock_superstate (rx, super);
+	return answer;
+      }
+    }
+  else if (df->future) /* A cache miss on an edge with a future? Must be
+			* a semi-free destination. */
+    {				
+      if (df->future->is_semifree)
+	refresh_semifree_superstate (rx->cache, df->future);
+      return &df->future_frame;
+    }
+  else
+    /* no future superstate on an existing edge */
+    {
+      rx_lock_superstate (rx, super);
+      if (!solve_destination (rx, df))
+	{
+	  rx_unlock_superstate (rx, super);
+	  return 0;
+	}
+      if (!df->effects
+	  && (df->edge->options->next_same_super_edge[0] == df->edge->options))
+	install_partial_transition (super, &df->future_frame,
+				    df->edge->cset[offset], offset * 32);
+      rx_unlock_superstate (rx, super);
+      return &df->future_frame;
+    }
+}
+
+
+
+
+
+/* The rest of the code provides a regex.c compatable interface. */
+
+
+__const__ char *re_error_msg[] =
+{
+  0,						/* REG_NOUT */
+  "No match",					/* REG_NOMATCH */
+  "Invalid regular expression",			/* REG_BADPAT */
+  "Invalid collation character",		/* REG_ECOLLATE */
+  "Invalid character class name",		/* REG_ECTYPE */
+  "Trailing backslash",				/* REG_EESCAPE */
+  "Invalid back reference",			/* REG_ESUBREG */
+  "Unmatched [ or [^",				/* REG_EBRACK */
+  "Unmatched ( or \\(",				/* REG_EPAREN */
+  "Unmatched \\{",				/* REG_EBRACE */
+  "Invalid content of \\{\\}",			/* REG_BADBR */
+  "Invalid range end",				/* REG_ERANGE */
+  "Memory exhausted",				/* REG_ESPACE */
+  "Invalid preceding regular expression",	/* REG_BADRPT */
+  "Premature end of regular expression",	/* REG_EEND */
+  "Regular expression too big",			/* REG_ESIZE */
+  "Unmatched ) or \\)",				/* REG_ERPAREN */
+};
+
+
+
+
+/* 
+ * Macros used while compiling patterns.
+ *
+ * By convention, PEND points just past the end of the uncompiled pattern,
+ * P points to the read position in the pattern.  `translate' is the name
+ * of the translation table (`TRANSLATE' is the name of a macro that looks
+ * things up in `translate').
+ */
+
+
+/*
+ * Fetch the next character in the uncompiled pattern---translating it 
+ * if necessary. *Also cast from a signed character in the constant
+ * string passed to us by the user to an unsigned char that we can use
+ * as an array index (in, e.g., `translate').
+ */
+#define PATFETCH(c)							\
+ do {if (p == pend) return REG_EEND;					\
+    c = (unsigned char) *p++;						\
+    c = translate[c];		 					\
+ } while (0)
+
+/* 
+ * Fetch the next character in the uncompiled pattern, with no
+ * translation.
+ */
+#define PATFETCH_RAW(c)							\
+  do {if (p == pend) return REG_EEND;					\
+    c = (unsigned char) *p++; 						\
+  } while (0)
+
+/* Go backwards one character in the pattern.  */
+#define PATUNFETCH p--
+
+
+#define TRANSLATE(d) translate[(unsigned char) (d)]
+
+typedef unsigned regnum_t;
+
+/* Since offsets can go either forwards or backwards, this type needs to
+ * be able to hold values from -(MAX_BUF_SIZE - 1) to MAX_BUF_SIZE - 1.
+ */
+typedef int pattern_offset_t;
+
+typedef struct
+{
+  struct rexp_node ** top_expression; /* was begalt */
+  struct rexp_node ** last_expression; /* was laststart */
+  pattern_offset_t inner_group_offset;
+  regnum_t regnum;
+} compile_stack_elt_t;
+
+typedef struct
+{
+  compile_stack_elt_t *stack;
+  unsigned size;
+  unsigned avail;			/* Offset of next open position.  */
+} compile_stack_type;
+
+
+#define INIT_COMPILE_STACK_SIZE 32
+
+#define COMPILE_STACK_EMPTY  (compile_stack.avail == 0)
+#define COMPILE_STACK_FULL  (compile_stack.avail == compile_stack.size)
+
+/* The next available element.  */
+#define COMPILE_STACK_TOP (compile_stack.stack[compile_stack.avail])
+
+
+/* Set the bit for character C in a list.  */
+#define SET_LIST_BIT(c)                               \
+  (b[((unsigned char) (c)) / CHARBITS]               \
+   |= 1 << (((unsigned char) c) % CHARBITS))
+
+/* Get the next unsigned number in the uncompiled pattern.  */
+#define GET_UNSIGNED_NUMBER(num) 					\
+  { if (p != pend)							\
+     {									\
+       PATFETCH (c); 							\
+       while (isdigit (c)) 						\
+         { 								\
+           if (num < 0)							\
+              num = 0;							\
+           num = num * 10 + c - '0'; 					\
+           if (p == pend) 						\
+              break; 							\
+           PATFETCH (c);						\
+         } 								\
+       } 								\
+    }		
+
+#define CHAR_CLASS_MAX_LENGTH  6 /* Namely, `xdigit'.  */
+
+#define IS_CHAR_CLASS(string)						\
+   (!strcmp (string, "alpha") || !strcmp (string, "upper")		\
+    || !strcmp (string, "lower") || !strcmp (string, "digit")		\
+    || !strcmp (string, "alnum") || !strcmp (string, "xdigit")		\
+    || !strcmp (string, "space") || !strcmp (string, "print")		\
+    || !strcmp (string, "punct") || !strcmp (string, "graph")		\
+    || !strcmp (string, "cntrl") || !strcmp (string, "blank"))
+
+
+
+/* These predicates are used in regex_compile. */
+
+/* P points to just after a ^ in PATTERN.  Return true if that ^ comes
+ * after an alternative or a begin-subexpression.  We assume there is at
+ * least one character before the ^.  
+ */
+
+#ifdef __STDC__
+static boolean
+at_begline_loc_p (__const__ char *pattern, __const__ char * p, reg_syntax_t syntax)
+#else
+static boolean
+at_begline_loc_p (pattern, p, syntax)
+     __const__ char *pattern;
+     __const__ char * p;
+     reg_syntax_t syntax;
+#endif
+{
+  __const__ char *prev = p - 2;
+  boolean prev_prev_backslash = ((prev > pattern) && (prev[-1] == '\\'));
+  
+    return
+      
+      (/* After a subexpression?  */
+       ((*prev == '(') && ((syntax & RE_NO_BK_PARENS) || prev_prev_backslash))
+       ||
+       /* After an alternative?  */
+       ((*prev == '|') && ((syntax & RE_NO_BK_VBAR) || prev_prev_backslash))
+       );
+}
+
+/* The dual of at_begline_loc_p.  This one is for $.  We assume there is
+ * at least one character after the $, i.e., `P < PEND'.
+ */
+
+#ifdef __STDC__
+static boolean
+at_endline_loc_p (__const__ char *p, __const__ char *pend, int syntax)
+#else
+static boolean
+at_endline_loc_p (p, pend, syntax)
+     __const__ char *p;
+     __const__ char *pend;
+     int syntax;
+#endif
+{
+  __const__ char *next = p;
+  boolean next_backslash = (*next == '\\');
+  __const__ char *next_next = (p + 1 < pend) ? (p + 1) : 0;
+  
+  return
+    (
+     /* Before a subexpression?  */
+     ((syntax & RE_NO_BK_PARENS)
+      ? (*next == ')')
+      : (next_backslash && next_next && (*next_next == ')')))
+    ||
+     /* Before an alternative?  */
+     ((syntax & RE_NO_BK_VBAR)
+      ? (*next == '|')
+      : (next_backslash && next_next && (*next_next == '|')))
+     );
+}
+
+
+
+unsigned char rx_id_translation[256] =
+{
+  0,  1,  2,  3,  4,  5,  6,  7,  8,  9,
+ 10, 11, 12, 13, 14, 15, 16, 17, 18, 19,
+ 20, 21, 22, 23, 24, 25, 26, 27, 28, 29,
+ 30, 31, 32, 33, 34, 35, 36, 37, 38, 39,
+ 40, 41, 42, 43, 44, 45, 46, 47, 48, 49,
+ 50, 51, 52, 53, 54, 55, 56, 57, 58, 59,
+ 60, 61, 62, 63, 64, 65, 66, 67, 68, 69,
+ 70, 71, 72, 73, 74, 75, 76, 77, 78, 79,
+ 80, 81, 82, 83, 84, 85, 86, 87, 88, 89,
+ 90, 91, 92, 93, 94, 95, 96, 97, 98, 99,
+
+ 100, 101, 102, 103, 104, 105, 106, 107, 108, 109,
+ 110, 111, 112, 113, 114, 115, 116, 117, 118, 119,
+ 120, 121, 122, 123, 124, 125, 126, 127, 128, 129,
+ 130, 131, 132, 133, 134, 135, 136, 137, 138, 139,
+ 140, 141, 142, 143, 144, 145, 146, 147, 148, 149,
+ 150, 151, 152, 153, 154, 155, 156, 157, 158, 159,
+ 160, 161, 162, 163, 164, 165, 166, 167, 168, 169,
+ 170, 171, 172, 173, 174, 175, 176, 177, 178, 179,
+ 180, 181, 182, 183, 184, 185, 186, 187, 188, 189,
+ 190, 191, 192, 193, 194, 195, 196, 197, 198, 199,
+
+ 200, 201, 202, 203, 204, 205, 206, 207, 208, 209,
+ 210, 211, 212, 213, 214, 215, 216, 217, 218, 219,
+ 220, 221, 222, 223, 224, 225, 226, 227, 228, 229,
+ 230, 231, 232, 233, 234, 235, 236, 237, 238, 239,
+ 240, 241, 242, 243, 244, 245, 246, 247, 248, 249,
+ 250, 251, 252, 253, 254, 255
+};
+
+/* The compiler keeps an inverted translation table.
+ * This looks up/inititalize elements.
+ * VALID is an array of booleans that validate CACHE.
+ */
+
+#ifdef __STDC__
+static rx_Bitset
+inverse_translation (struct re_pattern_buffer * rxb,
+		     char * valid, rx_Bitset cache,
+		     unsigned char * translate, int c)
+#else
+static rx_Bitset
+inverse_translation (rxb, valid, cache, translate, c)
+     struct re_pattern_buffer * rxb;
+     char * valid;
+     rx_Bitset cache;
+     unsigned char * translate;
+     int c;
+#endif
+{
+  rx_Bitset cs
+    = cache + c * rx_bitset_numb_subsets (rxb->rx.local_cset_size); 
+
+  if (!valid[c])
+    {
+      int x;
+      int c_tr = TRANSLATE(c);
+      rx_bitset_null (rxb->rx.local_cset_size, cs);
+      for (x = 0; x < 256; ++x)	/* &&&& 13.37 */
+	if (TRANSLATE(x) == c_tr)
+	  RX_bitset_enjoin (cs, x);
+      valid[c] = 1;
+    }
+  return cs;
+}
+
+
+
+
+
+/* More subroutine declarations and macros for regex_compile.  */
+
+/* Returns true if REGNUM is in one of COMPILE_STACK's elements and 
+   false if it's not.  */
+
+#ifdef __STDC__
+static boolean
+group_in_compile_stack (compile_stack_type compile_stack, regnum_t regnum)
+#else
+static boolean
+group_in_compile_stack (compile_stack, regnum)
+    compile_stack_type compile_stack;
+    regnum_t regnum;
+#endif
+{
+  int this_element;
+
+  for (this_element = compile_stack.avail - 1;  
+       this_element >= 0; 
+       this_element--)
+    if (compile_stack.stack[this_element].regnum == regnum)
+      return true;
+
+  return false;
+}
+
+
+/*
+ * Read the ending character of a range (in a bracket expression) from the
+ * uncompiled pattern *P_PTR (which ends at PEND).  We assume the
+ * starting character is in `P[-2]'.  (`P[-1]' is the character `-'.)
+ * Then we set the translation of all bits between the starting and
+ * ending characters (inclusive) in the compiled pattern B.
+ * 
+ * Return an error code.
+ * 
+ * We use these short variable names so we can use the same macros as
+ * `regex_compile' itself.  
+ */
+
+#ifdef __STDC__
+static reg_errcode_t
+compile_range (struct re_pattern_buffer * rxb, rx_Bitset cs,
+	       __const__ char ** p_ptr, __const__ char * pend,
+	       unsigned char * translate, reg_syntax_t syntax,
+	       rx_Bitset inv_tr,  char * valid_inv_tr)
+#else
+static reg_errcode_t
+compile_range (rxb, cs, p_ptr, pend, translate, syntax, inv_tr, valid_inv_tr)
+     struct re_pattern_buffer * rxb;
+     rx_Bitset cs;
+     __const__ char ** p_ptr;
+     __const__ char * pend;
+     unsigned char * translate;
+     reg_syntax_t syntax;
+     rx_Bitset inv_tr;
+     char * valid_inv_tr;
+#endif
+{
+  unsigned this_char;
+
+  __const__ char *p = *p_ptr;
+
+  unsigned char range_end;
+  unsigned char range_start = TRANSLATE(p[-2]);
+
+  if (p == pend)
+    return REG_ERANGE;
+
+  PATFETCH (range_end);
+
+  (*p_ptr)++;
+
+  if (range_start > range_end)
+    return syntax & RE_NO_EMPTY_RANGES ? REG_ERANGE : REG_NOERROR;
+
+  for (this_char = range_start; this_char <= range_end; this_char++)
+    {
+      rx_Bitset it =
+	inverse_translation (rxb, valid_inv_tr, inv_tr, translate, this_char);
+      rx_bitset_union (rxb->rx.local_cset_size, cs, it);
+    }
+  
+  return REG_NOERROR;
+}
+
+
+
+/* This searches a regexp for backreference side effects.
+ * It fills in the array OUT with 1 at the index of every register pair
+ * referenced by a backreference.
+ *
+ * This is used to help optimize patterns for searching.  The information is
+ * useful because, if the caller doesn't want register values, backreferenced
+ * registers are the only registers for which we need rx_backtrack.
+ */
+
+#ifdef __STDC__
+static void
+find_backrefs (char * out, struct rexp_node * rexp,
+	       struct re_se_params * params)
+#else
+static void
+find_backrefs (out, rexp, params)
+     char * out;
+     struct rexp_node * rexp;
+     struct re_se_params * params;
+#endif
+{
+  if (rexp)
+    switch (rexp->type)
+      {
+      case r_cset:
+      case r_data:
+	return;
+      case r_alternate:
+      case r_concat:
+      case r_opt:
+      case r_star:
+      case r_2phase_star:
+	find_backrefs (out, rexp->params.pair.left, params);
+	find_backrefs (out, rexp->params.pair.right, params);
+	return;
+      case r_side_effect:
+	if (   ((long)rexp->params.side_effect >= 0)
+	    && (params [(long)rexp->params.side_effect].se == re_se_backref))
+	  out[ params [(long)rexp->params.side_effect].op1] = 1;
+	return;
+      }
+}
+
+
+
+
+/* Returns 0 unless the pattern can match the empty string. */
+
+#ifdef __STDC__
+static int
+compute_fastset (struct re_pattern_buffer * rxb, struct rexp_node * rexp)
+#else
+static int
+compute_fastset (rxb, rexp)
+     struct re_pattern_buffer * rxb;
+     struct rexp_node * rexp;
+#endif
+{
+  if (!rexp)
+    return 1;
+  switch (rexp->type)
+    {
+    case r_data:
+      return 1;
+    case r_cset:
+      {
+	rx_bitset_union (rxb->rx.local_cset_size,
+			 rxb->fastset, rexp->params.cset);
+      }
+      return 0;
+    case r_concat:
+      return (compute_fastset (rxb, rexp->params.pair.left)
+	      && compute_fastset (rxb, rexp->params.pair.right));
+    case r_2phase_star:
+      compute_fastset (rxb, rexp->params.pair.left);
+      /* compute_fastset (rxb, rexp->params.pair.right);  nope... */
+      return 1;
+    case r_alternate:
+      return !!(compute_fastset (rxb, rexp->params.pair.left)
+		+ compute_fastset (rxb, rexp->params.pair.right));
+    case r_opt:
+    case r_star:
+      compute_fastset (rxb, rexp->params.pair.left);
+      return 1;
+    case r_side_effect:
+      return 1;
+    }
+
+  /* this should never happen */
+  return 0;
+}
+
+
+
+/* returns
+ *  1 -- yes, definately anchored by the given side effect.
+ *  2 -- maybe anchored, maybe the empty string.
+ *  0 -- definately not anchored
+ *  There is simply no other possibility.
+ */
+
+#ifdef __STDC__
+static int
+is_anchored (struct rexp_node * rexp, rx_side_effect se)
+#else
+static int
+is_anchored (rexp, se)
+     struct rexp_node * rexp;
+     rx_side_effect se;
+#endif
+{
+  if (!rexp)
+    return 2;
+  switch (rexp->type)
+    {
+    case r_cset:
+    case r_data:
+      return 0;
+    case r_concat:
+    case r_2phase_star:
+      {
+	int l = is_anchored (rexp->params.pair.left, se);
+	return (l == 2 ? is_anchored (rexp->params.pair.right, se) : l);
+      }
+    case r_alternate:
+      {
+	int l = is_anchored (rexp->params.pair.left, se);
+	int r = l ? is_anchored (rexp->params.pair.right, se) : 0;
+
+	if (l == r)
+	  return l;
+	else if ((l == 0) || (r == 0))
+	  return 0;
+	else
+	  return 2;
+      }
+    case r_opt:
+    case r_star:
+      return is_anchored (rexp->params.pair.left, se) ? 2 : 0;
+      
+    case r_side_effect:
+      return ((rexp->params.side_effect == se)
+	      ? 1 : 2);
+    }
+
+  /* this should never happen */
+  return 0;
+}
+
+
+
+/* This removes register assignments that aren't required by backreferencing.
+ * This can speed up explore_future, especially if it eliminates
+ * non-determinism in the superstate NFA.
+ * 
+ * NEEDED is an array of characters, presumably filled in by FIND_BACKREFS.
+ * The non-zero elements of the array indicate which register assignments
+ * can NOT be removed from the expression.
+ */
+
+#ifdef __STDC__
+static struct rexp_node *
+remove_unecessary_side_effects (struct rx * rx, char * needed,
+				struct rexp_node * rexp,
+				struct re_se_params * params)
+#else
+static struct rexp_node *
+remove_unecessary_side_effects (rx, needed, rexp, params)
+     struct rx * rx;
+     char * needed;
+     struct rexp_node * rexp;
+     struct re_se_params * params;
+#endif
+{
+  struct rexp_node * l;
+  struct rexp_node * r;
+  if (!rexp)
+    return 0;
+  else
+    switch (rexp->type)
+      {
+      case r_cset:
+      case r_data:
+	return rexp;
+      case r_alternate:
+      case r_concat:
+      case r_2phase_star:
+	l = remove_unecessary_side_effects (rx, needed,
+					    rexp->params.pair.left, params);
+	r = remove_unecessary_side_effects (rx, needed,
+					    rexp->params.pair.right, params);
+	if ((l && r) || (rexp->type != r_concat))
+	  {
+	    rexp->params.pair.left = l;
+	    rexp->params.pair.right = r;
+	    return rexp;
+	  }
+	else
+	  {
+	    rexp->params.pair.left = rexp->params.pair.right = 0;
+	    rx_free_rexp (rx, rexp);
+	    return l ? l : r;
+	  }
+      case r_opt:
+      case r_star:
+	l = remove_unecessary_side_effects (rx, needed,
+					    rexp->params.pair.left, params);
+	if (l)
+	  {
+	    rexp->params.pair.left = l;
+	    return rexp;
+	  }
+	else
+	  {
+	    rexp->params.pair.left = 0;
+	    rx_free_rexp (rx, rexp);
+	    return 0;
+	  }
+      case r_side_effect:
+	{
+	  int se = (long)rexp->params.side_effect;
+	  if (   (se >= 0)
+	      && (   ((enum re_side_effects)params[se].se == re_se_lparen)
+		  || ((enum re_side_effects)params[se].se == re_se_rparen))
+	      && (params [se].op1 > 0)
+	      && (!needed [params [se].op1]))
+	    {
+	      rx_free_rexp (rx, rexp);
+	      return 0;
+	    }
+	  else
+	    return rexp;
+	}
+      }
+
+  /* this should never happen */
+  return 0;
+}
+
+
+
+
+#ifdef __STDC__
+static int
+pointless_if_repeated (struct rexp_node * node, struct re_se_params * params)
+#else
+static int
+pointless_if_repeated (node, params)
+     struct rexp_node * node;
+     struct re_se_params * params;
+#endif
+{
+  if (!node)
+    return 1;
+  switch (node->type)
+    {
+    case r_cset:
+      return 0;
+    case r_alternate:
+    case r_concat:
+    case r_2phase_star:
+      return (pointless_if_repeated (node->params.pair.left, params)
+	      && pointless_if_repeated (node->params.pair.right, params));
+    case r_opt:
+    case r_star:
+      return pointless_if_repeated (node->params.pair.left, params);
+    case r_side_effect:
+      switch ((enum re_side_effects) (((long)node->params.side_effect < 0)
+	      ? (enum re_side_effects) (long) node->params.side_effect
+	      : (enum re_side_effects)params[(long)node->params.side_effect].se))
+	{
+	case re_se_try:
+	case re_se_at_dot:
+	case re_se_begbuf:
+	case re_se_hat:
+	case re_se_wordbeg:
+	case re_se_wordbound:
+	case re_se_notwordbound:
+	case re_se_wordend:
+	case re_se_endbuf:
+	case re_se_dollar:
+	case re_se_fail:
+	case re_se_win:
+	  return 1;
+	case re_se_lparen:
+	case re_se_rparen:
+	case re_se_iter:
+	case re_se_end_iter:
+	case re_se_syntax:
+	case re_se_not_syntax:
+	case re_se_backref:
+	  return 0;
+	}
+    case r_data:
+    default:
+      return 0;
+    }
+}
+
+
+
+
+#ifdef __STDC__
+static int
+registers_on_stack (struct re_pattern_buffer * rxb,
+		    struct rexp_node * rexp, int in_danger,
+		    struct re_se_params * params)
+#else
+static int
+registers_on_stack (rxb, rexp, in_danger, params)
+     struct re_pattern_buffer * rxb;
+     struct rexp_node * rexp;
+     int in_danger;
+     struct re_se_params * params;
+#endif
+{
+  if (!rexp)
+    return 0;
+  else
+    switch (rexp->type)
+      {
+      case r_cset:
+      case r_data:
+	return 0;
+      case r_alternate:
+      case r_concat:
+	return (   registers_on_stack (rxb, rexp->params.pair.left,
+				       in_danger, params)
+		|| (registers_on_stack
+		    (rxb, rexp->params.pair.right,
+		     in_danger, params)));
+      case r_opt:
+	return registers_on_stack (rxb, rexp->params.pair.left, 0, params);
+      case r_star:
+	return registers_on_stack (rxb, rexp->params.pair.left, 1, params);
+      case r_2phase_star:
+	return
+	  (   registers_on_stack (rxb, rexp->params.pair.left, 1, params)
+	   || registers_on_stack (rxb, rexp->params.pair.right, 1, params));
+      case r_side_effect:
+	{
+	  int se = (long)rexp->params.side_effect;
+	  if (   in_danger
+	      && (se >= 0)
+	      && (params [se].op1 > 0)
+	      && (   ((enum re_side_effects)params[se].se == re_se_lparen)
+		  || ((enum re_side_effects)params[se].se == re_se_rparen)))
+	    return 1;
+	  else
+	    return 0;
+	}
+      }
+
+  /* this should never happen */
+  return 0;
+}
+
+
+
+
+static char idempotent_complex_se[] =
+{
+#define RX_WANT_SE_DEFS 1
+#undef RX_DEF_SE
+#undef RX_DEF_CPLX_SE
+#define RX_DEF_SE(IDEM, NAME, VALUE)	      
+#define RX_DEF_CPLX_SE(IDEM, NAME, VALUE)     IDEM,
+#include "rx.h"
+#undef RX_DEF_SE
+#undef RX_DEF_CPLX_SE
+#undef RX_WANT_SE_DEFS
+  23
+};
+
+static char idempotent_se[] =
+{
+  13,
+#define RX_WANT_SE_DEFS 1
+#undef RX_DEF_SE
+#undef RX_DEF_CPLX_SE
+#define RX_DEF_SE(IDEM, NAME, VALUE)	      IDEM,
+#define RX_DEF_CPLX_SE(IDEM, NAME, VALUE)     
+#include "rx.h"
+#undef RX_DEF_SE
+#undef RX_DEF_CPLX_SE
+#undef RX_WANT_SE_DEFS
+  42
+};
+
+
+
+
+
+#ifdef __STDC__
+static int
+has_any_se (struct rx * rx,
+	    struct rexp_node * rexp)
+#else
+static int
+has_any_se (rx, rexp)
+     struct rx * rx;
+     struct rexp_node * rexp;
+#endif
+{
+  if (!rexp)
+    return 0;
+
+  switch (rexp->type)
+    {
+    case r_cset:
+    case r_data:
+      return 0;
+
+    case r_side_effect:
+      return 1;
+      
+    case r_2phase_star:
+    case r_concat:
+    case r_alternate:
+      return
+	(   has_any_se (rx, rexp->params.pair.left)
+	 || has_any_se (rx, rexp->params.pair.right));
+
+    case r_opt:
+    case r_star:
+      return has_any_se (rx, rexp->params.pair.left);
+    }
+
+  /* this should never happen */
+  return 0;
+}
+
+
+
+
+/* This must be called AFTER `convert_hard_loops' for a given REXP. */
+#ifdef __STDC__
+static int
+has_non_idempotent_epsilon_path (struct rx * rx,
+				 struct rexp_node * rexp,
+				 struct re_se_params * params)
+#else
+static int
+has_non_idempotent_epsilon_path (rx, rexp, params)
+     struct rx * rx;
+     struct rexp_node * rexp;
+     struct re_se_params * params;
+#endif
+{
+  if (!rexp)
+    return 0;
+
+  switch (rexp->type)
+    {
+    case r_cset:
+    case r_data:
+    case r_star:
+      return 0;
+
+    case r_side_effect:
+      return
+	!((long)rexp->params.side_effect > 0
+	  ? idempotent_complex_se [ params [(long)rexp->params.side_effect].se ]
+	  : idempotent_se [-(long)rexp->params.side_effect]);
+      
+    case r_alternate:
+      return
+	(   has_non_idempotent_epsilon_path (rx,
+					     rexp->params.pair.left, params)
+	 || has_non_idempotent_epsilon_path (rx,
+					     rexp->params.pair.right, params));
+
+    case r_2phase_star:
+    case r_concat:
+      return
+	(   has_non_idempotent_epsilon_path (rx,
+					     rexp->params.pair.left, params)
+	 && has_non_idempotent_epsilon_path (rx,
+					     rexp->params.pair.right, params));
+
+    case r_opt:
+      return has_non_idempotent_epsilon_path (rx,
+					      rexp->params.pair.left, params);
+    }
+
+  /* this should never happen */
+  return 0;
+}
+
+
+
+
+/* This computes rougly what it's name suggests.   It can (and does) go wrong 
+ * in the direction of returning spurious 0 without causing disasters.
+ */
+#ifdef __STDC__
+static int
+begins_with_complex_se (struct rx * rx, struct rexp_node * rexp)
+#else
+static int
+begins_with_complex_se (rx, rexp)
+     struct rx * rx;
+     struct rexp_node * rexp;
+#endif
+{
+  if (!rexp)
+    return 0;
+
+  switch (rexp->type)
+    {
+    case r_cset:
+    case r_data:
+      return 0;
+
+    case r_side_effect:
+      return ((long)rexp->params.side_effect >= 0);
+      
+    case r_alternate:
+      return
+	(   begins_with_complex_se (rx, rexp->params.pair.left)
+	 && begins_with_complex_se (rx, rexp->params.pair.right));
+
+
+    case r_concat:
+      return has_any_se (rx, rexp->params.pair.left);
+    case r_opt:
+    case r_star:
+    case r_2phase_star:
+      return 0;
+    }
+
+  /* this should never happen */
+  return 0;
+}
+
+
+
+/* This destructively removes some of the re_se_tv side effects from 
+ * a rexp tree.  In particular, during parsing re_se_tv was inserted on the
+ * right half of every | to guarantee that posix path preference could be 
+ * honored.  This function removes some which it can be determined aren't 
+ * needed.  
+ */
+
+#ifdef __STDC__
+static void
+speed_up_alt (struct rx * rx,
+	      struct rexp_node * rexp,
+	      int unposix)
+#else
+static void
+speed_up_alt (rx, rexp, unposix)
+     struct rx * rx;
+     struct rexp_node * rexp;
+     int unposix;
+#endif
+{
+  if (!rexp)
+    return;
+
+  switch (rexp->type)
+    {
+    case r_cset:
+    case r_data:
+    case r_side_effect:
+      return;
+
+    case r_opt:
+    case r_star:
+      speed_up_alt (rx, rexp->params.pair.left, unposix);
+      return;
+
+    case r_2phase_star:
+    case r_concat:
+      speed_up_alt (rx, rexp->params.pair.left, unposix);
+      speed_up_alt (rx, rexp->params.pair.right, unposix);
+      return;
+
+    case r_alternate:
+      /* the right child is guaranteed to be (concat re_se_tv <subexp>) */
+
+      speed_up_alt (rx, rexp->params.pair.left, unposix);
+      speed_up_alt (rx, rexp->params.pair.right->params.pair.right, unposix);
+      
+      if (   unposix
+	  || (begins_with_complex_se
+	      (rx, rexp->params.pair.right->params.pair.right))
+	  || !(   has_any_se (rx, rexp->params.pair.right->params.pair.right)
+	       || has_any_se (rx, rexp->params.pair.left)))
+	{
+	  struct rexp_node * conc = rexp->params.pair.right;
+	  rexp->params.pair.right = conc->params.pair.right;
+	  conc->params.pair.right = 0;
+	  rx_free_rexp (rx, conc);
+	}
+    }
+}
+
+
+
+
+
+
+/* `regex_compile' compiles PATTERN (of length SIZE) according to SYNTAX.
+   Returns one of error codes defined in `regex.h', or zero for success.
+
+   Assumes the `allocated' (and perhaps `buffer') and `translate'
+   fields are set in BUFP on entry.
+
+   If it succeeds, results are put in BUFP (if it returns an error, the
+   contents of BUFP are undefined):
+     `buffer' is the compiled pattern;
+     `syntax' is set to SYNTAX;
+     `used' is set to the length of the compiled pattern;
+     `fastmap_accurate' is set to zero;
+     `re_nsub' is set to the number of groups in PATTERN;
+     `not_bol' and `not_eol' are set to zero.
+   
+   The `fastmap' and `newline_anchor' fields are neither
+   examined nor set.  */
+
+
+
+#ifdef __STDC__
+RX_DECL reg_errcode_t
+rx_compile (__const__ char *pattern, int size,
+	    reg_syntax_t syntax,
+	    struct re_pattern_buffer * rxb) 
+#else
+RX_DECL reg_errcode_t
+rx_compile (pattern, size, syntax, rxb)
+     __const__ char *pattern;
+     int size;
+     reg_syntax_t syntax;
+     struct re_pattern_buffer * rxb;
+#endif
+{
+  RX_subset
+    inverse_translate [CHAR_SET_SIZE * rx_bitset_numb_subsets(CHAR_SET_SIZE)];
+  char
+    validate_inv_tr [CHAR_SET_SIZE * rx_bitset_numb_subsets(CHAR_SET_SIZE)];
+
+  /* We fetch characters from PATTERN here.  Even though PATTERN is
+     `char *' (i.e., signed), we declare these variables as unsigned, so
+     they can be reliably used as array indices.  */
+  register unsigned char c, c1;
+  
+  /* A random tempory spot in PATTERN.  */
+  __const__ char *p1;
+  
+  /* Keeps track of unclosed groups.  */
+  compile_stack_type compile_stack;
+
+  /* Points to the current (ending) position in the pattern.  */
+  __const__ char *p = pattern;
+  __const__ char *pend = pattern + size;
+  
+  /* How to translate the characters in the pattern.  */
+  unsigned char *translate = (rxb->translate
+			      ? rxb->translate
+			      : rx_id_translation);
+
+  /* When parsing is done, this will hold the expression tree. */
+  struct rexp_node * rexp = 0;
+
+  /* In the midst of compilation, this holds onto the regexp 
+   * first parst while rexp goes on to aquire additional constructs.
+   */
+  struct rexp_node * orig_rexp = 0;
+  struct rexp_node * fewer_side_effects = 0;
+
+  /* This and top_expression are saved on the compile stack. */
+  struct rexp_node ** top_expression = &rexp;
+  struct rexp_node ** last_expression = top_expression;
+  
+  /* Parameter to `goto append_node' */
+  struct rexp_node * append;
+
+  /* Counts open-groups as they are encountered.  This is the index of the
+   * innermost group being compiled.
+   */
+  regnum_t regnum = 0;
+
+  /* Place in the uncompiled pattern (i.e., the {) to
+   * which to go back if the interval is invalid.  
+   */
+  __const__ char *beg_interval;
+
+  struct re_se_params * params = 0;
+  int paramc = 0;		/* How many complex side effects so far? */
+
+  rx_side_effect side;		/* param to `goto add_side_effect' */
+
+  bzero (validate_inv_tr, sizeof (validate_inv_tr));
+
+  rxb->rx.instruction_table = rx_id_instruction_table;
+
+
+  /* Initialize the compile stack.  */
+  compile_stack.stack =  (( compile_stack_elt_t *) malloc ((INIT_COMPILE_STACK_SIZE) * sizeof ( compile_stack_elt_t)));
+  if (compile_stack.stack == 0)
+    return REG_ESPACE;
+
+  compile_stack.size = INIT_COMPILE_STACK_SIZE;
+  compile_stack.avail = 0;
+
+  /* Initialize the pattern buffer.  */
+  rxb->rx.cache = &default_cache;
+  rxb->syntax = syntax;
+  rxb->fastmap_accurate = 0;
+  rxb->not_bol = rxb->not_eol = 0;
+  rxb->least_subs = 0;
+  
+  /* Always count groups, whether or not rxb->no_sub is set.  
+   * The whole pattern is implicitly group 0, so counting begins
+   * with 1.
+   */
+  rxb->re_nsub = 0;
+
+#if !defined (emacs) && !defined (SYNTAX_TABLE)
+  /* Initialize the syntax table.  */
+   init_syntax_once ();
+#endif
+
+  /* Loop through the uncompiled pattern until we're at the end.  */
+  while (p != pend)
+    {
+      PATFETCH (c);
+
+      switch (c)
+        {
+        case '^':
+          {
+            if (   /* If at start of pattern, it's an operator.  */
+                   p == pattern + 1
+                   /* If context independent, it's an operator.  */
+                || syntax & RE_CONTEXT_INDEP_ANCHORS
+                   /* Otherwise, depends on what's come before.  */
+                || at_begline_loc_p (pattern, p, syntax))
+	      {
+		struct rexp_node * n
+		  = rx_mk_r_side_effect (&rxb->rx, (rx_side_effect)re_se_hat);
+		if (!n)
+		  return REG_ESPACE;
+		append = n;
+		goto append_node;
+	      }
+            else
+              goto normal_char;
+          }
+          break;
+
+
+        case '$':
+          {
+            if (   /* If at end of pattern, it's an operator.  */
+                   p == pend 
+                   /* If context independent, it's an operator.  */
+                || syntax & RE_CONTEXT_INDEP_ANCHORS
+                   /* Otherwise, depends on what's next.  */
+                || at_endline_loc_p (p, pend, syntax))
+	      {
+		struct rexp_node * n
+		  = rx_mk_r_side_effect (&rxb->rx, (rx_side_effect)re_se_dollar);
+		if (!n)
+		  return REG_ESPACE;
+		append = n;
+		goto append_node;
+	      }
+             else
+               goto normal_char;
+           }
+           break;
+
+
+	case '+':
+        case '?':
+          if ((syntax & RE_BK_PLUS_QM)
+              || (syntax & RE_LIMITED_OPS))
+            goto normal_char;
+
+        handle_plus:
+        case '*':
+          /* If there is no previous pattern... */
+          if (pointless_if_repeated (*last_expression, params))
+            {
+              if (syntax & RE_CONTEXT_INVALID_OPS)
+                return REG_BADRPT;
+              else if (!(syntax & RE_CONTEXT_INDEP_OPS))
+                goto normal_char;
+            }
+
+          {
+            /* 1 means zero (many) matches is allowed.  */
+            char zero_times_ok = 0, many_times_ok = 0;
+
+            /* If there is a sequence of repetition chars, collapse it
+               down to just one (the right one).  We can't combine
+               interval operators with these because of, e.g., `a{2}*',
+               which should only match an even number of `a's.  */
+
+            for (;;)
+              {
+                zero_times_ok |= c != '+';
+                many_times_ok |= c != '?';
+
+                if (p == pend)
+                  break;
+
+                PATFETCH (c);
+
+                if (c == '*'
+                    || (!(syntax & RE_BK_PLUS_QM) && (c == '+' || c == '?')))
+                  ;
+
+                else if (syntax & RE_BK_PLUS_QM  &&  c == '\\')
+                  {
+                    if (p == pend) return REG_EESCAPE;
+
+                    PATFETCH (c1);
+                    if (!(c1 == '+' || c1 == '?'))
+                      {
+                        PATUNFETCH;
+                        PATUNFETCH;
+                        break;
+                      }
+
+                    c = c1;
+                  }
+                else
+                  {
+                    PATUNFETCH;
+                    break;
+                  }
+
+                /* If we get here, we found another repeat character.  */
+               }
+
+            /* Star, etc. applied to an empty pattern is equivalent
+               to an empty pattern.  */
+            if (!last_expression)
+              break;
+
+	    /* Now we know whether or not zero matches is allowed
+	     * and also whether or not two or more matches is allowed.
+	     */
+
+	    {
+	      struct rexp_node * inner_exp = *last_expression;
+	      int need_sync = 0;
+
+	      if (many_times_ok
+		  && has_non_idempotent_epsilon_path (&rxb->rx,
+						      inner_exp, params))
+		{
+		  struct rexp_node * pusher
+		    = rx_mk_r_side_effect (&rxb->rx,
+					   (rx_side_effect)re_se_pushpos);
+		  struct rexp_node * checker
+		    = rx_mk_r_side_effect (&rxb->rx,
+					   (rx_side_effect)re_se_chkpos);
+		  struct rexp_node * pushback
+		    = rx_mk_r_side_effect (&rxb->rx,
+					   (rx_side_effect)re_se_pushback);
+		  rx_Bitset cs = rx_cset (&rxb->rx);
+		  struct rexp_node * lit_t = rx_mk_r_cset (&rxb->rx, cs);
+		  struct rexp_node * fake_state
+		    = rx_mk_r_concat (&rxb->rx, pushback, lit_t);
+		  struct rexp_node * phase2
+		    = rx_mk_r_concat (&rxb->rx, checker, fake_state);
+		  struct rexp_node * popper
+		    = rx_mk_r_side_effect (&rxb->rx,
+					   (rx_side_effect)re_se_poppos);
+		  struct rexp_node * star
+		    = rx_mk_r_2phase_star (&rxb->rx, inner_exp, phase2);
+		  struct rexp_node * a
+		    = rx_mk_r_concat (&rxb->rx, pusher, star);
+		  struct rexp_node * whole_thing
+		    = rx_mk_r_concat (&rxb->rx, a, popper);
+		  if (!(pusher && star && pushback && lit_t && fake_state
+			&& lit_t && phase2 && checker && popper
+			&& a && whole_thing))
+		    return REG_ESPACE;
+		  RX_bitset_enjoin (cs, 't');
+		  *last_expression = whole_thing;
+		}
+	      else
+		{
+		  struct rexp_node * star =
+		    (many_times_ok ? rx_mk_r_star : rx_mk_r_opt)
+		      (&rxb->rx, *last_expression);
+		  if (!star)
+		    return REG_ESPACE;
+		  *last_expression = star;
+		  need_sync = has_any_se (&rxb->rx, *last_expression);
+		}
+	      if (!zero_times_ok)
+		{
+		  struct rexp_node * concat
+		    = rx_mk_r_concat (&rxb->rx, inner_exp,
+				      rx_copy_rexp (&rxb->rx,
+						    *last_expression));
+		  if (!concat)
+		    return REG_ESPACE;
+		  *last_expression = concat;
+		}
+	      if (need_sync)
+		{
+		  int sync_se = paramc;
+		  params = (params
+			    ? ((struct re_se_params *)
+			       realloc (params,
+					sizeof (*params) * (1 + paramc)))
+			    : ((struct re_se_params *)
+			       malloc (sizeof (*params))));
+		  if (!params)
+		    return REG_ESPACE;
+		  ++paramc;
+		  params [sync_se].se = re_se_tv;
+		  side = (rx_side_effect)sync_se;
+		  goto add_side_effect;
+		}
+	    }
+	    /* The old regex.c used to optimize `.*\n'.  
+	     * Maybe rx should too?
+	     */
+	  }
+	  break;
+
+
+	case '.':
+	  {
+	    rx_Bitset cs = rx_cset (&rxb->rx);
+	    struct rexp_node * n = rx_mk_r_cset (&rxb->rx, cs);
+	    if (!(cs && n))
+	      return REG_ESPACE;
+
+	    rx_bitset_universe (rxb->rx.local_cset_size, cs);
+	    if (!(rxb->syntax & RE_DOT_NEWLINE))
+	      RX_bitset_remove (cs, '\n');
+	    if (!(rxb->syntax & RE_DOT_NOT_NULL))
+	      RX_bitset_remove (cs, 0);
+
+	    append = n;
+	    goto append_node;
+	    break;
+	  }
+
+
+        case '[':
+	  if (p == pend) return REG_EBRACK;
+          {
+            boolean had_char_class = false;
+	    rx_Bitset cs = rx_cset (&rxb->rx);
+	    struct rexp_node * node = rx_mk_r_cset (&rxb->rx, cs);
+	    int is_inverted = *p == '^';
+	    
+	    if (!(node && cs))
+	      return REG_ESPACE;
+	    
+	    /* This branch of the switch is normally exited with
+	     *`goto append_node'
+	     */
+	    append = node;
+	    
+            if (is_inverted)
+	      p++;
+	    
+            /* Remember the first position in the bracket expression.  */
+            p1 = p;
+	    
+            /* Read in characters and ranges, setting map bits.  */
+            for (;;)
+              {
+                if (p == pend) return REG_EBRACK;
+		
+                PATFETCH (c);
+		
+                /* \ might escape characters inside [...] and [^...].  */
+                if ((syntax & RE_BACKSLASH_ESCAPE_IN_LISTS) && c == '\\')
+                  {
+                    if (p == pend) return REG_EESCAPE;
+		    
+                    PATFETCH (c1);
+		    {
+		      rx_Bitset it = inverse_translation (rxb, 
+							  validate_inv_tr,
+							  inverse_translate,
+							  translate,
+							  c1);
+		      rx_bitset_union (rxb->rx.local_cset_size, cs, it);
+		    }
+                    continue;
+                  }
+		
+                /* Could be the end of the bracket expression.  If it's
+                   not (i.e., when the bracket expression is `[]' so
+                   far), the ']' character bit gets set way below.  */
+                if (c == ']' && p != p1 + 1)
+                  goto finalize_class_and_append;
+		
+                /* Look ahead to see if it's a range when the last thing
+                   was a character class.  */
+                if (had_char_class && c == '-' && *p != ']')
+                  return REG_ERANGE;
+		
+                /* Look ahead to see if it's a range when the last thing
+                   was a character: if this is a hyphen not at the
+                   beginning or the end of a list, then it's the range
+                   operator.  */
+                if (c == '-' 
+                    && !(p - 2 >= pattern && p[-2] == '[') 
+                    && !(p - 3 >= pattern && p[-3] == '[' && p[-2] == '^')
+                    && *p != ']')
+                  {
+                    reg_errcode_t ret
+                      = compile_range (rxb, cs, &p, pend, translate, syntax,
+				       inverse_translate, validate_inv_tr);
+                    if (ret != REG_NOERROR) return ret;
+                  }
+		
+                else if (p[0] == '-' && p[1] != ']')
+                  { /* This handles ranges made up of characters only.  */
+                    reg_errcode_t ret;
+		    
+		    /* Move past the `-'.  */
+                    PATFETCH (c1);
+                    
+                    ret = compile_range (rxb, cs, &p, pend, translate, syntax,
+					 inverse_translate, validate_inv_tr);
+                    if (ret != REG_NOERROR) return ret;
+                  }
+		
+                /* See if we're at the beginning of a possible character
+                   class.  */
+		
+		else if ((syntax & RE_CHAR_CLASSES)
+			 && (c == '[') && (*p == ':'))
+                  {
+                    char str[CHAR_CLASS_MAX_LENGTH + 1];
+		    
+                    PATFETCH (c);
+                    c1 = 0;
+		    
+                    /* If pattern is `[[:'.  */
+                    if (p == pend) return REG_EBRACK;
+		    
+                    for (;;)
+                      {
+                        PATFETCH (c);
+                        if (c == ':' || c == ']' || p == pend
+                            || c1 == CHAR_CLASS_MAX_LENGTH)
+			  break;
+                        str[c1++] = c;
+                      }
+                    str[c1] = '\0';
+		    
+                    /* If isn't a word bracketed by `[:' and:`]':
+                       undo the ending character, the letters, and leave 
+                       the leading `:' and `[' (but set bits for them).  */
+                    if (c == ':' && *p == ']')
+                      {
+                        int ch;
+                        boolean is_alnum = !strcmp (str, "alnum");
+                        boolean is_alpha = !strcmp (str, "alpha");
+                        boolean is_blank = !strcmp (str, "blank");
+                        boolean is_cntrl = !strcmp (str, "cntrl");
+                        boolean is_digit = !strcmp (str, "digit");
+                        boolean is_graph = !strcmp (str, "graph");
+                        boolean is_lower = !strcmp (str, "lower");
+                        boolean is_print = !strcmp (str, "print");
+                        boolean is_punct = !strcmp (str, "punct");
+                        boolean is_space = !strcmp (str, "space");
+                        boolean is_upper = !strcmp (str, "upper");
+                        boolean is_xdigit = !strcmp (str, "xdigit");
+                        
+                        if (!IS_CHAR_CLASS (str)) return REG_ECTYPE;
+			
+                        /* Throw away the ] at the end of the character
+                           class.  */
+                        PATFETCH (c);					
+			
+                        if (p == pend) return REG_EBRACK;
+			
+                        for (ch = 0; ch < 1 << CHARBITS; ch++)
+                          {
+                            if (   (is_alnum  && isalnum (ch))
+                                || (is_alpha  && isalpha (ch))
+                                || (is_blank  && isblank (ch))
+                                || (is_cntrl  && iscntrl (ch))
+                                || (is_digit  && isdigit (ch))
+                                || (is_graph  && isgraph (ch))
+                                || (is_lower  && islower (ch))
+                                || (is_print  && isprint (ch))
+                                || (is_punct  && ispunct (ch))
+                                || (is_space  && isspace (ch))
+                                || (is_upper  && isupper (ch))
+                                || (is_xdigit && isxdigit (ch)))
+			      {
+				rx_Bitset it =
+				  inverse_translation (rxb, 
+						       validate_inv_tr,
+						       inverse_translate,
+						       translate,
+						       ch);
+				rx_bitset_union (rxb->rx.local_cset_size,
+						 cs, it);
+			      }
+                          }
+                        had_char_class = true;
+                      }
+                    else
+                      {
+                        c1++;
+                        while (c1--)    
+                          PATUNFETCH;
+			{
+			  rx_Bitset it =
+			    inverse_translation (rxb, 
+						 validate_inv_tr,
+						 inverse_translate,
+						 translate,
+						 '[');
+			  rx_bitset_union (rxb->rx.local_cset_size,
+					   cs, it);
+			}
+			{
+			  rx_Bitset it =
+			    inverse_translation (rxb, 
+						 validate_inv_tr,
+						 inverse_translate,
+						 translate,
+						 ':');
+			  rx_bitset_union (rxb->rx.local_cset_size,
+					   cs, it);
+			}
+                        had_char_class = false;
+                      }
+                  }
+                else
+                  {
+                    had_char_class = false;
+		    {
+		      rx_Bitset it = inverse_translation (rxb, 
+							  validate_inv_tr,
+							  inverse_translate,
+							  translate,
+							  c);
+		      rx_bitset_union (rxb->rx.local_cset_size, cs, it);
+		    }
+                  }
+              }
+
+	  finalize_class_and_append:
+	    if (is_inverted)
+	      {
+		rx_bitset_complement (rxb->rx.local_cset_size, cs);
+		if (syntax & RE_HAT_LISTS_NOT_NEWLINE)
+		  RX_bitset_remove (cs, '\n');
+	      }
+	    goto append_node;
+          }
+          break;
+
+
+	case '(':
+          if (syntax & RE_NO_BK_PARENS)
+            goto handle_open;
+          else
+            goto normal_char;
+
+
+        case ')':
+          if (syntax & RE_NO_BK_PARENS)
+            goto handle_close;
+          else
+            goto normal_char;
+
+
+        case '\n':
+          if (syntax & RE_NEWLINE_ALT)
+            goto handle_alt;
+          else
+            goto normal_char;
+
+
+	case '|':
+          if (syntax & RE_NO_BK_VBAR)
+            goto handle_alt;
+          else
+            goto normal_char;
+
+
+        case '{':
+	  if ((syntax & RE_INTERVALS) && (syntax & RE_NO_BK_BRACES))
+	    goto handle_interval;
+	  else
+	    goto normal_char;
+
+
+        case '\\':
+          if (p == pend) return REG_EESCAPE;
+
+          /* Do not translate the character after the \, so that we can
+             distinguish, e.g., \B from \b, even if we normally would
+             translate, e.g., B to b.  */
+          PATFETCH_RAW (c);
+
+          switch (c)
+            {
+            case '(':
+              if (syntax & RE_NO_BK_PARENS)
+                goto normal_backslash;
+
+            handle_open:
+              rxb->re_nsub++;
+              regnum++;
+              if (COMPILE_STACK_FULL)
+                { 
+                  ((compile_stack.stack) =
+		   (compile_stack_elt_t *) realloc (compile_stack.stack, ( compile_stack.size << 1) * sizeof (
+													      compile_stack_elt_t)));
+                  if (compile_stack.stack == 0) return REG_ESPACE;
+
+                  compile_stack.size <<= 1;
+                }
+
+	      if (*last_expression)
+		{
+		  struct rexp_node * concat
+		    = rx_mk_r_concat (&rxb->rx, *last_expression, 0);
+		  if (!concat)
+		    return REG_ESPACE;
+		  *last_expression = concat;
+		  last_expression = &concat->params.pair.right;
+		}
+
+              /*
+	       * These are the values to restore when we hit end of this
+               * group.  
+	       */
+	      COMPILE_STACK_TOP.top_expression = top_expression;
+	      COMPILE_STACK_TOP.last_expression = last_expression;
+              COMPILE_STACK_TOP.regnum = regnum;
+	      
+              compile_stack.avail++;
+	      
+	      top_expression = last_expression;
+	      break;
+
+
+            case ')':
+              if (syntax & RE_NO_BK_PARENS) goto normal_backslash;
+
+            handle_close:
+              /* See similar code for backslashed left paren above.  */
+              if (COMPILE_STACK_EMPTY)
+                if (syntax & RE_UNMATCHED_RIGHT_PAREN_ORD)
+                  goto normal_char;
+                else
+                  return REG_ERPAREN;
+
+              /* Since we just checked for an empty stack above, this
+                 ``can't happen''.  */
+
+              {
+                /* We don't just want to restore into `regnum', because
+                   later groups should continue to be numbered higher,
+                   as in `(ab)c(de)' -- the second group is #2.  */
+                regnum_t this_group_regnum;
+		struct rexp_node ** inner = top_expression;
+
+                compile_stack.avail--;
+		top_expression = COMPILE_STACK_TOP.top_expression;
+		last_expression = COMPILE_STACK_TOP.last_expression;
+                this_group_regnum = COMPILE_STACK_TOP.regnum;
+		{
+		  int left_se = paramc;
+		  int right_se = paramc + 1;
+
+		  params = (params
+			    ? ((struct re_se_params *)
+			       realloc (params,
+					(paramc + 2) * sizeof (params[0])))
+			    : ((struct re_se_params *)
+			       malloc (2 * sizeof (params[0]))));
+		  if (!params)
+		    return REG_ESPACE;
+		  paramc += 2;
+
+		  params[left_se].se = re_se_lparen;
+		  params[left_se].op1 = this_group_regnum;
+		  params[right_se].se = re_se_rparen;
+		  params[right_se].op1 = this_group_regnum;
+		  {
+		    struct rexp_node * left
+		      = rx_mk_r_side_effect (&rxb->rx,
+					     (rx_side_effect)left_se);
+		    struct rexp_node * right
+		      = rx_mk_r_side_effect (&rxb->rx,
+					     (rx_side_effect)right_se);
+		    struct rexp_node * c1
+		      = (*inner
+			 ? rx_mk_r_concat (&rxb->rx, left, *inner) : left);
+		    struct rexp_node * c2
+		      = rx_mk_r_concat (&rxb->rx, c1, right);
+		    if (!(left && right && c1 && c2))
+		      return REG_ESPACE;
+		    *inner = c2;
+		  }
+		}
+		break;
+	      }
+
+            case '|':					/* `\|'.  */
+              if ((syntax & RE_LIMITED_OPS) || (syntax & RE_NO_BK_VBAR))
+                goto normal_backslash;
+            handle_alt:
+              if (syntax & RE_LIMITED_OPS)
+                goto normal_char;
+
+	      {
+		struct rexp_node * alt
+		  = rx_mk_r_alternate (&rxb->rx, *top_expression, 0);
+		if (!alt)
+		  return REG_ESPACE;
+		*top_expression = alt;
+		last_expression = &alt->params.pair.right;
+		{
+		  int sync_se = paramc;
+
+		  params = (params
+			    ? ((struct re_se_params *)
+			       realloc (params,
+					(paramc + 1) * sizeof (params[0])))
+			    : ((struct re_se_params *)
+			       malloc (sizeof (params[0]))));
+		  if (!params)
+		    return REG_ESPACE;
+		  ++paramc;
+
+		  params[sync_se].se = re_se_tv;
+		  {
+		    struct rexp_node * sync
+		      = rx_mk_r_side_effect (&rxb->rx,
+					     (rx_side_effect)sync_se);
+		    struct rexp_node * conc
+		      = rx_mk_r_concat (&rxb->rx, sync, 0);
+
+		    if (!sync || !conc)
+		      return REG_ESPACE;
+
+		    *last_expression = conc;
+		    last_expression = &conc->params.pair.right;
+		  }
+		}
+	      }
+              break;
+
+
+            case '{': 
+              /* If \{ is a literal.  */
+              if (!(syntax & RE_INTERVALS)
+                     /* If we're at `\{' and it's not the open-interval 
+                        operator.  */
+                  || ((syntax & RE_INTERVALS) && (syntax & RE_NO_BK_BRACES))
+                  || (p - 2 == pattern  &&  p == pend))
+                goto normal_backslash;
+
+            handle_interval:
+              {
+                /* If got here, then the syntax allows intervals.  */
+
+                /* At least (most) this many matches must be made.  */
+                int lower_bound = -1, upper_bound = -1;
+
+                beg_interval = p - 1;
+
+                if (p == pend)
+                  {
+                    if (syntax & RE_NO_BK_BRACES)
+                      goto unfetch_interval;
+                    else
+                      return REG_EBRACE;
+                  }
+
+                GET_UNSIGNED_NUMBER (lower_bound);
+
+                if (c == ',')
+                  {
+                    GET_UNSIGNED_NUMBER (upper_bound);
+                    if (upper_bound < 0) upper_bound = RE_DUP_MAX;
+                  }
+                else
+                  /* Interval such as `{1}' => match exactly once. */
+                  upper_bound = lower_bound;
+
+                if (lower_bound < 0 || upper_bound > RE_DUP_MAX
+                    || lower_bound > upper_bound)
+                  {
+                    if (syntax & RE_NO_BK_BRACES)
+                      goto unfetch_interval;
+                    else 
+                      return REG_BADBR;
+                  }
+
+                if (!(syntax & RE_NO_BK_BRACES)) 
+                  {
+                    if (c != '\\') return REG_EBRACE;
+                    PATFETCH (c);
+                  }
+
+                if (c != '}')
+                  {
+                    if (syntax & RE_NO_BK_BRACES)
+                      goto unfetch_interval;
+                    else 
+                      return REG_BADBR;
+                  }
+
+                /* We just parsed a valid interval.  */
+
+                /* If it's invalid to have no preceding re.  */
+                if (pointless_if_repeated (*last_expression, params))
+                  {
+                    if (syntax & RE_CONTEXT_INVALID_OPS)
+                      return REG_BADRPT;
+                    else if (!(syntax & RE_CONTEXT_INDEP_OPS))
+                      goto unfetch_interval;
+		    /* was: else laststart = b; */
+                  }
+
+                /* If the upper bound is zero, don't want to iterate
+                 * at all.
+		 */
+                 if (upper_bound == 0)
+		   {
+		     if (*last_expression)
+		       {
+			 rx_free_rexp (&rxb->rx, *last_expression);
+			 *last_expression = 0;
+		       }
+		   }
+		else
+		  /* Otherwise, we have a nontrivial interval. */
+		  {
+		    int iter_se = paramc;
+		    int end_se = paramc + 1;
+		    params = (params
+			      ? ((struct re_se_params *)
+				 realloc (params,
+					  sizeof (*params) * (2 + paramc)))
+			      : ((struct re_se_params *)
+				 malloc (2 * sizeof (*params))));
+		    if (!params)
+		      return REG_ESPACE;
+		    paramc += 2;
+		    params [iter_se].se = re_se_iter;
+		    params [iter_se].op1 = lower_bound;
+		    params[iter_se].op2 = upper_bound;
+
+		    params[end_se].se = re_se_end_iter;
+		    params[end_se].op1 = lower_bound;
+		    params[end_se].op2 = upper_bound;
+		    {
+		      struct rexp_node * push0
+			= rx_mk_r_side_effect (&rxb->rx,
+					       (rx_side_effect)re_se_push0);
+		      struct rexp_node * start_one_iter
+			= rx_mk_r_side_effect (&rxb->rx,
+					       (rx_side_effect)iter_se);
+		      struct rexp_node * phase1
+			= rx_mk_r_concat (&rxb->rx, start_one_iter,
+					  *last_expression);
+		      struct rexp_node * pushback
+			= rx_mk_r_side_effect (&rxb->rx,
+					       (rx_side_effect)re_se_pushback);
+		      rx_Bitset cs = rx_cset (&rxb->rx);
+		      struct rexp_node * lit_t
+			= rx_mk_r_cset (&rxb->rx, cs);
+		      struct rexp_node * phase2
+			= rx_mk_r_concat (&rxb->rx, pushback, lit_t);
+		      struct rexp_node * loop
+			= rx_mk_r_2phase_star (&rxb->rx, phase1, phase2);
+		      struct rexp_node * push_n_loop
+			= rx_mk_r_concat (&rxb->rx, push0, loop);
+		      struct rexp_node * final_test
+			= rx_mk_r_side_effect (&rxb->rx,
+					       (rx_side_effect)end_se);
+		      struct rexp_node * full_exp
+			= rx_mk_r_concat (&rxb->rx, push_n_loop, final_test);
+
+		      if (!(push0 && start_one_iter && phase1
+			    && pushback && lit_t && phase2
+			    && loop && push_n_loop && final_test && full_exp))
+			return REG_ESPACE;
+
+		      RX_bitset_enjoin(cs, 't');
+
+		      *last_expression = full_exp;
+		    }
+		  }
+                beg_interval = 0;
+              }
+              break;
+
+            unfetch_interval:
+              /* If an invalid interval, match the characters as literals.  */
+               p = beg_interval;
+               beg_interval = 0;
+
+               /* normal_char and normal_backslash need `c'.  */
+               PATFETCH (c);	
+
+               if (!(syntax & RE_NO_BK_BRACES))
+                 {
+                   if (p > pattern  &&  p[-1] == '\\')
+                     goto normal_backslash;
+                 }
+               goto normal_char;
+
+#ifdef emacs
+            /* There is no way to specify the before_dot and after_dot
+               operators.  rms says this is ok.  --karl  */
+            case '=':
+	      side = (rx_side_effect)rx_se_at_dot;
+	      goto add_side_effect;
+              break;
+
+            case 's':
+	    case 'S':
+	      {
+		rx_Bitset cs = rx_cset (&rxb->rx);
+		struct rexp_node * set = rx_mk_r_cset (&rxb->rx, cs);
+		if (!(cs && set))
+		  return REG_ESPACE;
+		if (c == 'S')
+		  rx_bitset_universe (rxb->rx.local_cset_size, cs);
+
+		PATFETCH (c);
+		{
+		  int x;
+		  enum syntaxcode code = syntax_spec_code [c];
+		  for (x = 0; x < 256; ++x)
+		    {
+		      
+		      if (SYNTAX (x) == code)
+			{
+			  rx_Bitset it =
+			    inverse_translation (rxb, validate_inv_tr,
+						 inverse_translate,
+						 translate, x);
+			  rx_bitset_xor (rxb->rx.local_cset_size, cs, it);
+			}
+		    }
+		}
+		append = set;
+		goto append_node;
+	      }
+              break;
+#endif /* emacs */
+
+
+            case 'w':
+            case 'W':
+	      {
+		rx_Bitset cs = rx_cset (&rxb->rx);
+		struct rexp_node * n = (cs ? rx_mk_r_cset (&rxb->rx, cs) : 0);
+		if (!(cs && n))
+		  return REG_ESPACE;
+		if (c == 'W')
+		  rx_bitset_universe (rxb->rx.local_cset_size ,cs);
+		{
+		  int x;
+		  for (x = rxb->rx.local_cset_size - 1; x > 0; --x)
+		    if (SYNTAX(x) & Sword)
+		      RX_bitset_toggle (cs, x);
+		}
+		append = n;
+		goto append_node;
+	      }
+              break;
+
+/* With a little extra work, some of these side effects could be optimized
+ * away (basicly by looking at what we already know about the surrounding
+ * chars).  
+ */
+            case '<':
+	      side = (rx_side_effect)re_se_wordbeg;
+	      goto add_side_effect;
+              break;
+
+            case '>':
+              side = (rx_side_effect)re_se_wordend;
+	      goto add_side_effect;
+              break;
+
+            case 'b':
+              side = (rx_side_effect)re_se_wordbound;
+	      goto add_side_effect;
+              break;
+
+            case 'B':
+              side = (rx_side_effect)re_se_notwordbound;
+	      goto add_side_effect;
+              break;
+
+            case '`':
+	      side = (rx_side_effect)re_se_begbuf;
+	      goto add_side_effect;
+	      break;
+	      
+            case '\'':
+	      side = (rx_side_effect)re_se_endbuf;
+	      goto add_side_effect;
+              break;
+
+	    add_side_effect:
+	      {
+		struct rexp_node * se
+		  = rx_mk_r_side_effect (&rxb->rx, side);
+		if (!se)
+		  return REG_ESPACE;
+		append = se;
+		goto append_node;
+	      }
+	      break;
+
+            case '1': case '2': case '3': case '4': case '5':
+            case '6': case '7': case '8': case '9':
+              if (syntax & RE_NO_BK_REFS)
+                goto normal_char;
+
+              c1 = c - '0';
+
+              if (c1 > regnum)
+                return REG_ESUBREG;
+
+              /* Can't back reference to a subexpression if inside of it.  */
+              if (group_in_compile_stack (compile_stack, c1))
+		return REG_ESUBREG;
+
+	      {
+		int backref_se = paramc;
+		params = (params
+			  ? ((struct re_se_params *)
+			     realloc (params,
+				      sizeof (*params) * (1 + paramc)))
+			  : ((struct re_se_params *)
+			     malloc (sizeof (*params))));
+		if (!params)
+		  return REG_ESPACE;
+		++paramc;
+		params[backref_se].se = re_se_backref;
+		params[backref_se].op1 = c1;
+		side = (rx_side_effect)backref_se;
+		goto add_side_effect;
+	      }
+              break;
+
+            case '+':
+            case '?':
+              if (syntax & RE_BK_PLUS_QM)
+                goto handle_plus;
+              else
+                goto normal_backslash;
+
+            default:
+            normal_backslash:
+              /* You might think it would be useful for \ to mean
+                 not to translate; but if we don't translate it
+                 it will never match anything.  */
+              c = TRANSLATE (c);
+              goto normal_char;
+            }
+          break;
+
+
+	default:
+        /* Expects the character in `c'.  */
+	normal_char:
+	    {
+	      rx_Bitset cs = rx_cset(&rxb->rx);
+	      struct rexp_node * match = rx_mk_r_cset (&rxb->rx, cs);
+	      rx_Bitset it;
+	      if (!(cs && match))
+		return REG_ESPACE;
+	      it = inverse_translation (rxb, validate_inv_tr,
+					inverse_translate, translate, c);
+	      rx_bitset_union (CHAR_SET_SIZE, cs, it);
+	      append = match;
+
+	    append_node:
+	      /* This genericly appends the rexp APPEND to *LAST_EXPRESSION
+	       * and then parses the next character normally.
+	       */
+	      if (*last_expression)
+		{
+		  struct rexp_node * concat
+		    = rx_mk_r_concat (&rxb->rx, *last_expression, append);
+		  if (!concat)
+		    return REG_ESPACE;
+		  *last_expression = concat;
+		  last_expression = &concat->params.pair.right;
+		}
+	      else
+		*last_expression = append;
+	    }
+	} /* switch (c) */
+    } /* while p != pend */
+
+  
+  {
+    int win_se = paramc;
+    params = (params
+	      ? ((struct re_se_params *)
+		 realloc (params,
+			  sizeof (*params) * (1 + paramc)))
+	      : ((struct re_se_params *)
+		 malloc (sizeof (*params))));
+    if (!params)
+      return REG_ESPACE;
+    ++paramc;
+    params[win_se].se = re_se_win;
+    {
+      struct rexp_node * se
+	= rx_mk_r_side_effect (&rxb->rx, (rx_side_effect)win_se);
+      struct rexp_node * concat
+	= rx_mk_r_concat (&rxb->rx, rexp, se);
+      if (!(se && concat))
+	return REG_ESPACE;
+      rexp = concat;
+    }
+  }
+
+
+  /* Through the pattern now.  */
+
+  if (!COMPILE_STACK_EMPTY) 
+    return REG_EPAREN;
+
+      free (compile_stack.stack);
+
+  orig_rexp = rexp;
+#ifdef RX_DEBUG
+  if (rx_debug_compile)
+    {
+      dbug_rxb = rxb;
+      fputs ("\n\nCompiling ", stdout);
+      fwrite (pattern, 1, size, stdout);
+      fputs (":\n", stdout);
+      rxb->se_params = params;
+      print_rexp (&rxb->rx, orig_rexp, 2, re_seprint, stdout);
+    }
+#endif
+  {
+    rx_Bitset cs = rx_cset(&rxb->rx);
+    rx_Bitset cs2 = rx_cset(&rxb->rx);
+#ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+    char * se_map = (char *) _alloca (paramc);
+#else
+    char * se_map = (char *) alloca (paramc);
+#endif
+    struct rexp_node * new_rexp = 0;
+
+
+    bzero (se_map, paramc);
+    find_backrefs (se_map, rexp, params);
+    fewer_side_effects =
+      remove_unecessary_side_effects (&rxb->rx, se_map,
+				      rx_copy_rexp (&rxb->rx, rexp), params);
+
+    speed_up_alt (&rxb->rx, rexp, 0);
+    speed_up_alt (&rxb->rx, fewer_side_effects, 1);
+
+    {
+      char * syntax_parens = rxb->syntax_parens;
+      if (syntax_parens == (char *)0x1)
+	rexp = remove_unecessary_side_effects
+	  (&rxb->rx, se_map, rexp, params);
+      else if (syntax_parens)
+	{
+	  int x;
+	  for (x = 0; x < paramc; ++x)
+	    if ((   (params[x].se == re_se_lparen)
+		 || (params[x].se == re_se_rparen))
+		&& (!syntax_parens [params[x].op1]))
+	      se_map [x] = 1;
+	  rexp = remove_unecessary_side_effects
+	    (&rxb->rx, se_map, rexp, params);
+	}
+    }
+
+    /* At least one more optimization would be nice to have here but i ran out 
+     * of time.  The idea would be to delay side effects.  
+     * For examle, `(abc)' is the same thing as `abc()' except that the
+     * left paren is offset by 3 (which we know at compile time).
+     * (In this comment, write that second pattern `abc(:3:)' 
+     * where `(:3:' is a syntactic unit.)
+     *
+     * Trickier:  `(abc|defg)'  is the same as `(abc(:3:|defg(:4:))'
+     * (The paren nesting may be hard to follow -- that's an alternation
+     *	of `abc(:3:' and `defg(:4:' inside (purely syntactic) parens
+     *  followed by the closing paren from the original expression.)
+     *
+     * Neither the expression tree representation nor the the nfa make
+     * this very easy to write. :(
+     */
+
+  /* What we compile is different than what the parser returns.
+   * Suppose the parser returns expression R.
+   * Let R' be R with unnecessary register assignments removed 
+   * (see REMOVE_UNECESSARY_SIDE_EFFECTS, above).
+   *
+   * What we will compile is the expression:
+   *
+   *    m{try}R{win}\|s{try}R'{win}
+   *
+   * {try} and {win} denote side effect epsilons (see EXPLORE_FUTURE).
+   * 
+   * When trying a match, we insert an `m' at the beginning of the 
+   * string if the user wants registers to be filled, `s' if not.
+   */
+    new_rexp =
+      rx_mk_r_alternate
+	(&rxb->rx,
+	 rx_mk_r_concat (&rxb->rx, rx_mk_r_cset (&rxb->rx, cs2), rexp),
+	 rx_mk_r_concat (&rxb->rx,
+			 rx_mk_r_cset (&rxb->rx, cs), fewer_side_effects));
+
+    if (!(new_rexp && cs && cs2))
+      return REG_ESPACE;
+    RX_bitset_enjoin (cs2, '\0'); /* prefixed to the rexp used for matching. */
+    RX_bitset_enjoin (cs, '\1'); /* prefixed to the rexp used for searching. */
+    rexp = new_rexp;
+  }
+
+#ifdef RX_DEBUG
+  if (rx_debug_compile)
+    {
+      fputs ("\n...which is compiled as:\n", stdout);
+      print_rexp (&rxb->rx, rexp, 2, re_seprint, stdout);
+    }
+#endif
+  {
+    struct rx_nfa_state *start = 0;
+    struct rx_nfa_state *end = 0;
+
+    if (!rx_build_nfa (&rxb->rx, rexp, &start, &end))
+      return REG_ESPACE;	/*  */
+    else
+      {
+	void * mem = (void *)rxb->buffer;
+	unsigned long size = rxb->allocated;
+	int start_id;
+	char * perm_mem;
+	int iterator_size = paramc * sizeof (params[0]);
+
+	end->is_final = 1;
+	start->is_start = 1;
+	rx_name_nfa_states (&rxb->rx);
+	start_id = start->id;
+#ifdef RX_DEBUG
+	if (rx_debug_compile)
+	  {
+	    fputs ("...giving the NFA: \n", stdout);
+	    dbug_rxb = rxb;
+	    print_nfa (&rxb->rx, rxb->rx.nfa_states, re_seprint, stdout);
+	  }
+#endif
+	if (!rx_eclose_nfa (&rxb->rx))
+	  return REG_ESPACE;
+	else
+	  {
+	    rx_delete_epsilon_transitions (&rxb->rx);
+	    
+	    /* For compatability reasons, we need to shove the
+	     * compiled nfa into one chunk of malloced memory.
+	     */
+	    rxb->rx.reserved = (   sizeof (params[0]) * paramc
+				+  rx_sizeof_bitset (rxb->rx.local_cset_size));
+#ifdef RX_DEBUG
+	    if (rx_debug_compile)
+	      {
+		dbug_rxb = rxb;
+		fputs ("...which cooks down (uncompactified) to: \n", stdout);
+		print_nfa (&rxb->rx, rxb->rx.nfa_states, re_seprint, stdout);
+	      }
+#endif
+	    if (!rx_compactify_nfa (&rxb->rx, &mem, &size))
+	      return REG_ESPACE;
+	    rxb->buffer = mem;
+	    rxb->allocated = size;
+	    rxb->rx.buffer = mem;
+	    rxb->rx.allocated = size;
+	    perm_mem = ((char *)rxb->rx.buffer
+			+ rxb->rx.allocated - rxb->rx.reserved);
+	    rxb->se_params = ((struct re_se_params *)perm_mem);
+	    bcopy (params, rxb->se_params, iterator_size);
+	    perm_mem += iterator_size;
+	    rxb->fastset = (rx_Bitset) perm_mem;
+	    rxb->start = rx_id_to_nfa_state (&rxb->rx, start_id);
+	  }
+	rx_bitset_null (rxb->rx.local_cset_size, rxb->fastset);
+	rxb->can_match_empty = compute_fastset (rxb, orig_rexp);
+	rxb->match_regs_on_stack =
+	  registers_on_stack (rxb, orig_rexp, 0, params); 
+	rxb->search_regs_on_stack =
+	  registers_on_stack (rxb, fewer_side_effects, 0, params);
+	if (rxb->can_match_empty)
+	  rx_bitset_universe (rxb->rx.local_cset_size, rxb->fastset);
+	rxb->is_anchored = is_anchored (orig_rexp, (rx_side_effect) re_se_hat);
+	rxb->begbuf_only = is_anchored (orig_rexp,
+					(rx_side_effect) re_se_begbuf);
+      }
+    rx_free_rexp (&rxb->rx, rexp);
+    if (params)
+      free (params);
+#ifdef RX_DEBUG
+    if (rx_debug_compile)
+      {
+	dbug_rxb = rxb;
+	fputs ("...which cooks down to: \n", stdout);
+	print_nfa (&rxb->rx, rxb->rx.nfa_states, re_seprint, stdout);
+      }
+#endif
+  }
+  return REG_NOERROR;
+}
+
+
+
+
+/* This table gives an error message for each of the error codes listed
+   in regex.h.  Obviously the order here has to be same as there.  */
+
+__const__ char * rx_error_msg[] =
+{ 0,						/* REG_NOERROR */
+    "No match",					/* REG_NOMATCH */
+    "Invalid regular expression",		/* REG_BADPAT */
+    "Invalid collation character",		/* REG_ECOLLATE */
+    "Invalid character class name",		/* REG_ECTYPE */
+    "Trailing backslash",			/* REG_EESCAPE */
+    "Invalid back reference",			/* REG_ESUBREG */
+    "Unmatched [ or [^",			/* REG_EBRACK */
+    "Unmatched ( or \\(",			/* REG_EPAREN */
+    "Unmatched \\{",				/* REG_EBRACE */
+    "Invalid content of \\{\\}",		/* REG_BADBR */
+    "Invalid range end",			/* REG_ERANGE */
+    "Memory exhausted",				/* REG_ESPACE */
+    "Invalid preceding regular expression",	/* REG_BADRPT */
+    "Premature end of regular expression",	/* REG_EEND */
+    "Regular expression too big",		/* REG_ESIZE */
+    "Unmatched ) or \\)",			/* REG_ERPAREN */
+};
+
+
+
+
+
+char rx_slowmap [256] =
+{
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+};
+
+#ifdef __STDC__
+RX_DECL void
+rx_blow_up_fastmap (struct re_pattern_buffer * rxb)
+#else
+RX_DECL void
+rx_blow_up_fastmap (rxb)
+     struct re_pattern_buffer * rxb;
+#endif
+{
+  int x;
+  for (x = 0; x < 256; ++x)	/* &&&& 3.6 % */
+    rxb->fastmap [x] = !!RX_bitset_member (rxb->fastset, x);
+  rxb->fastmap_accurate = 1;
+}
+
+
+
+
+
+#if !defined(REGEX_MALLOC) && !defined(__GNUC__)
+#define RE_SEARCH_2_FN	inner_re_search_2
+#define RE_S2_QUAL static
+#else
+#define RE_SEARCH_2_FN	re_search_2
+#define RE_S2_QUAL 
+#endif
+
+struct re_search_2_closure
+{
+  __const__ char * string1;
+  int size1;
+  __const__ char * string2;
+  int size2;
+};
+
+
+static __inline__ enum rx_get_burst_return
+re_search_2_get_burst (pos, vclosure, stop)
+     struct rx_string_position * pos;
+     void * vclosure;
+     int stop;
+{
+  struct re_search_2_closure * closure;
+  closure = (struct re_search_2_closure *)vclosure;
+  if (!closure->string2)
+    {
+      int inset;
+
+      inset = pos->pos - pos->string;
+      if ((inset < -1) || (inset > closure->size1))
+	return rx_get_burst_no_more;
+      else
+	{
+	  pos->pos = (__const__ unsigned char *) closure->string1 + inset;
+	  pos->string = (__const__ unsigned char *) closure->string1;
+	  pos->size = closure->size1;
+	  pos->end = ((__const__ unsigned char *)
+		      MIN(closure->string1 + closure->size1,
+			  closure->string1 + stop));
+	  pos->offset = 0;
+	  return ((pos->pos < pos->end)
+		  ? rx_get_burst_ok
+		  :  rx_get_burst_no_more);
+	}
+    }
+  else if (!closure->string1)
+    {
+      int inset;
+
+      inset = pos->pos - pos->string;
+      pos->pos = (__const__ unsigned char *) closure->string2 + inset;
+      pos->string = (__const__ unsigned char *) closure->string2;
+      pos->size = closure->size2;
+      pos->end = ((__const__ unsigned char *)
+		  MIN(closure->string2 + closure->size2,
+		      closure->string2 + stop));
+      pos->offset = 0;
+      return ((pos->pos < pos->end)
+	      ? rx_get_burst_ok
+	      :  rx_get_burst_no_more);
+    }
+  else
+    {
+      int inset;
+
+      inset = pos->pos - pos->string + pos->offset;
+      if (inset < closure->size1)
+	{
+	  pos->pos = (__const__ unsigned char *) closure->string1 + inset;
+	  pos->string = (__const__ unsigned char *) closure->string1;
+	  pos->size = closure->size1;
+	  pos->end = ((__const__ unsigned char *)
+		      MIN(closure->string1 + closure->size1,
+			  closure->string1 + stop));
+	  pos->offset = 0;
+	  return rx_get_burst_ok;
+	}
+      else
+	{
+	  pos->pos = ((__const__ unsigned char *)
+		      closure->string2 + inset - closure->size1);
+	  pos->string = (__const__ unsigned char *) closure->string2;
+	  pos->size = closure->size2;
+	  pos->end = ((__const__ unsigned char *)
+		      MIN(closure->string2 + closure->size2,
+			  closure->string2 + stop - closure->size1));
+	  pos->offset = closure->size1;
+	  return ((pos->pos < pos->end)
+		  ? rx_get_burst_ok
+		  :  rx_get_burst_no_more);
+	}
+    }
+}
+
+
+static __inline__ enum rx_back_check_return
+re_search_2_back_check (pos, lparen, rparen, translate, vclosure, stop)
+     struct rx_string_position * pos;
+     int lparen;
+     int rparen;
+     unsigned char * translate;
+     void * vclosure;
+     int stop;
+{
+  struct rx_string_position there;
+  struct rx_string_position past;
+
+  there = *pos;
+  there.pos = there.string + lparen - there.offset;
+  re_search_2_get_burst (&there, vclosure, stop);
+
+  past = *pos;
+  past.pos = past.string + rparen - there.offset;
+  re_search_2_get_burst (&past, vclosure, stop);
+
+  ++pos->pos;
+  re_search_2_get_burst (pos, vclosure, stop);
+
+  while (   (there.pos != past.pos)
+	 && (pos->pos != pos->end))
+    if (TRANSLATE(*there.pos) != TRANSLATE(*pos->pos))
+      return rx_back_check_fail;
+    else
+      {
+	++there.pos;
+	++pos->pos;
+	if (there.pos == there.end)
+	  re_search_2_get_burst (&there, vclosure, stop);
+	if (pos->pos == pos->end)
+	  re_search_2_get_burst (pos, vclosure, stop);
+      }
+
+  if (there.pos != past.pos)
+    return rx_back_check_fail;
+  --pos->pos;
+  re_search_2_get_burst (pos, vclosure, stop);
+  return rx_back_check_pass;
+}
+
+static __inline__ int
+re_search_2_fetch_char (pos, offset, app_closure, stop)
+     struct rx_string_position * pos;
+     int offset;
+     void * app_closure;
+     int stop;
+{
+  struct re_search_2_closure * closure;
+  closure = (struct re_search_2_closure *)app_closure;
+  if (offset == 0)
+    {
+      if (pos->pos >= pos->string)
+	return *pos->pos;
+      else
+	{
+	  if (   (pos->string == (__const__ unsigned char *) closure->string2)
+	      && (closure->string1)
+	      && (closure->size1))
+	    return closure->string1[closure->size1 - 1];
+	  else
+	    return 0;		/* sure, why not. */
+	}
+    }
+  if (pos->pos == pos->end)
+    return *closure->string2;
+  else
+    return pos->pos[1];
+}
+     
+
+#ifdef __STDC__
+RE_S2_QUAL int
+RE_SEARCH_2_FN (struct re_pattern_buffer *rxb,
+		__const__ char * string1, int size1,
+		__const__ char * string2, int size2,
+		int startpos, int range,
+		struct re_registers *regs,
+		int stop)
+#else
+RE_S2_QUAL int
+RE_SEARCH_2_FN (rxb,
+		string1, size1, string2, size2, startpos, range, regs, stop)
+     struct re_pattern_buffer *rxb;
+     __const__ char * string1;
+     int size1;
+     __const__ char * string2;
+     int size2;
+     int startpos;
+     int range;
+     struct re_registers *regs;
+     int stop;
+#endif
+{
+  int answer;
+  struct re_search_2_closure closure;
+  closure.string1 = string1;
+  closure.size1 = size1;
+  closure.string2 = string2;
+  closure.size2 = size2;
+  answer = rx_search (rxb, startpos, range, stop, size1 + size2,
+		      re_search_2_get_burst,
+		      re_search_2_back_check,
+		      re_search_2_fetch_char,
+		      (void *)&closure,
+		      regs,
+		      0,
+		      0);
+  switch (answer)
+    {
+    case rx_search_continuation:
+      abort ();
+    case rx_search_error:
+      return -2;
+    case rx_search_soft_fail:
+    case rx_search_fail:
+      return -1;
+    default:
+      return answer;
+    }
+}
+
+/* Export rx_search to callers outside this file.  */
+
+int
+re_rx_search (rxb, startpos, range, stop, total_size,
+	      get_burst, back_check, fetch_char,
+	      app_closure, regs, resume_state, save_state)
+     struct re_pattern_buffer * rxb;
+     int startpos;
+     int range;
+     int stop;
+     int total_size;
+     rx_get_burst_fn get_burst;
+     rx_back_check_fn back_check;
+     rx_fetch_char_fn fetch_char;
+     void * app_closure;
+     struct re_registers * regs;
+     struct rx_search_state * resume_state;
+     struct rx_search_state * save_state;
+{
+  return rx_search (rxb, startpos, range, stop, total_size,
+		    get_burst, back_check, fetch_char, app_closure,
+		    regs, resume_state, save_state);
+}
+
+#if !defined(REGEX_MALLOC) && !defined(__GNUC__)
+#ifdef __STDC__
+int
+re_search_2 (struct re_pattern_buffer *rxb,
+	     __const__ char * string1, int size1,
+	     __const__ char * string2, int size2,
+	     int startpos, int range,
+	     struct re_registers *regs,
+	     int stop)
+#else
+int
+re_search_2 (rxb, string1, size1, string2, size2, startpos, range, regs, stop)
+     struct re_pattern_buffer *rxb;
+     __const__ char * string1;
+     int size1;
+     __const__ char * string2;
+     int size2;
+     int startpos;
+     int range;
+     struct re_registers *regs;
+     int stop;
+#endif
+{
+  int ret;
+  ret = inner_re_search_2 (rxb, string1, size1, string2, size2, startpos,
+			   range, regs, stop);
+#ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+  _alloca (0);
+#else
+  alloca (0);
+#endif
+  return ret;
+}
+#endif
+
+
+/* Like re_search_2, above, but only one string is specified, and
+ * doesn't let you say where to stop matching.
+ */
+
+#ifdef __STDC__
+int
+re_search (struct re_pattern_buffer * rxb, __const__ char *string,
+	   int size, int startpos, int range,
+	   struct re_registers *regs)
+#else
+int
+re_search (rxb, string, size, startpos, range, regs)
+     struct re_pattern_buffer * rxb;
+     __const__ char * string;
+     int size;
+     int startpos;
+     int range;
+     struct re_registers *regs;
+#endif
+{
+  return re_search_2 (rxb, 0, 0, string, size, startpos, range, regs, size);
+}
+
+#ifdef __STDC__
+int
+re_match_2 (struct re_pattern_buffer * rxb,
+	    __const__ char * string1, int size1,
+	    __const__ char * string2, int size2,
+	    int pos, struct re_registers *regs, int stop)
+#else
+int
+re_match_2 (rxb, string1, size1, string2, size2, pos, regs, stop)
+     struct re_pattern_buffer * rxb;
+     __const__ char * string1;
+     int size1;
+     __const__ char * string2;
+     int size2;
+     int pos;
+     struct re_registers *regs;
+     int stop;
+#endif
+{
+  struct re_registers some_regs;
+  regoff_t start;
+  regoff_t end;
+  int srch;
+  int save = rxb->regs_allocated;
+  struct re_registers * regs_to_pass = regs;
+
+  if (!regs)
+    {
+      some_regs.start = &start;
+      some_regs.end = &end;
+      some_regs.num_regs = 1;
+      regs_to_pass = &some_regs;
+      rxb->regs_allocated = REGS_FIXED;
+    }
+
+  srch = re_search_2 (rxb, string1, size1, string2, size2,
+		      pos, 1, regs_to_pass, stop);
+  if (regs_to_pass != regs)
+    rxb->regs_allocated = save;
+  if (srch < 0)
+    return srch;
+  return regs_to_pass->end[0] - regs_to_pass->start[0];
+}
+
+/* re_match is like re_match_2 except it takes only a single string.  */
+
+#ifdef __STDC__
+int
+re_match (struct re_pattern_buffer * rxb,
+	  __const__ char * string,
+	  int size, int pos,
+	  struct re_registers *regs)
+#else
+int
+re_match (rxb, string, size, pos, regs)
+     struct re_pattern_buffer * rxb;
+     __const__ char *string;
+     int size;
+     int pos;
+     struct re_registers *regs;
+#endif
+{
+  return re_match_2 (rxb, string, size, 0, 0, pos, regs, size);
+}
+
+
+
+
+/* Set by `re_set_syntax' to the current regexp syntax to recognize.  Can
+   also be assigned to arbitrarily: each pattern buffer stores its own
+   syntax, so it can be changed between regex compilations.  */
+reg_syntax_t re_syntax_options = RE_SYNTAX_EMACS;
+
+
+/* Specify the precise syntax of regexps for compilation.  This provides
+   for compatibility for various utilities which historically have
+   different, incompatible syntaxes.
+
+   The argument SYNTAX is a bit mask comprised of the various bits
+   defined in regex.h.  We return the old syntax.  */
+
+#ifdef __STDC__
+reg_syntax_t
+re_set_syntax (reg_syntax_t syntax)
+#else
+reg_syntax_t
+re_set_syntax (syntax)
+    reg_syntax_t syntax;
+#endif
+{
+  reg_syntax_t ret = re_syntax_options;
+
+  re_syntax_options = syntax;
+  return ret;
+}
+
+
+
+/* Set REGS to hold NUM_REGS registers, storing them in STARTS and
+   ENDS.  Subsequent matches using PATTERN_BUFFER and REGS will use
+   this memory for recording register information.  STARTS and ENDS
+   must be allocated using the malloc library routine, and must each
+   be at least NUM_REGS * sizeof (regoff_t) bytes long.
+
+   If NUM_REGS == 0, then subsequent matches should allocate their own
+   register data.
+
+   Unless this function is called, the first search or match using
+   PATTERN_BUFFER will allocate its own register data, without
+   freeing the old data.  */
+
+#ifdef __STDC__
+void
+re_set_registers (struct re_pattern_buffer *bufp,
+		  struct re_registers *regs,
+		  unsigned num_regs,
+		  regoff_t * starts, regoff_t * ends)
+#else
+void
+re_set_registers (bufp, regs, num_regs, starts, ends)
+     struct re_pattern_buffer *bufp;
+     struct re_registers *regs;
+     unsigned num_regs;
+     regoff_t * starts;
+     regoff_t * ends;
+#endif
+{
+  if (num_regs)
+    {
+      bufp->regs_allocated = REGS_REALLOCATE;
+      regs->num_regs = num_regs;
+      regs->start = starts;
+      regs->end = ends;
+    }
+  else
+    {
+      bufp->regs_allocated = REGS_UNALLOCATED;
+      regs->num_regs = 0;
+      regs->start = regs->end = (regoff_t) 0;
+    }
+}
+
+
+
+
+
+#ifdef __STDC__
+static int 
+cplx_se_sublist_len (struct rx_se_list * list)
+#else
+static int 
+cplx_se_sublist_len (list)
+     struct rx_se_list * list;
+#endif
+{
+  int x = 0;
+  while (list)
+    {
+      if ((long)list->car >= 0)
+	++x;
+      list = list->cdr;
+    }
+  return x;
+}
+
+
+/* For rx->se_list_cmp */
+
+#ifdef __STDC__
+static int 
+posix_se_list_order (struct rx * rx,
+		     struct rx_se_list * a, struct rx_se_list * b)
+#else
+static int 
+posix_se_list_order (rx, a, b)
+     struct rx * rx;
+     struct rx_se_list * a;
+     struct rx_se_list * b;
+#endif
+{
+  int al = cplx_se_sublist_len (a);
+  int bl = cplx_se_sublist_len (b);
+
+  if (!al && !bl)
+    return ((a == b)
+	    ? 0
+	    : ((a < b) ? -1 : 1));
+  
+  else if (!al)
+    return -1;
+
+  else if (!bl)
+    return 1;
+
+  else
+    {
+#ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+      rx_side_effect * av = ((rx_side_effect *)
+			     _alloca (sizeof (rx_side_effect) * (al + 1)));
+      rx_side_effect * bv = ((rx_side_effect *)
+			     _alloca (sizeof (rx_side_effect) * (bl + 1)));
+#else
+      rx_side_effect * av = ((rx_side_effect *)
+			     alloca (sizeof (rx_side_effect) * (al + 1)));
+      rx_side_effect * bv = ((rx_side_effect *)
+			     alloca (sizeof (rx_side_effect) * (bl + 1)));
+#endif
+      struct rx_se_list * ap = a;
+      struct rx_se_list * bp = b;
+      int ai, bi;
+      
+      for (ai = al - 1; ai >= 0; --ai)
+	{
+	  while ((long)ap->car < 0)
+	    ap = ap->cdr;
+	  av[ai] = ap->car;
+	  ap = ap->cdr;
+	}
+      av[al] = (rx_side_effect)-2;
+      for (bi = bl - 1; bi >= 0; --bi)
+	{
+	  while ((long)bp->car < 0)
+	    bp = bp->cdr;
+	  bv[bi] = bp->car;
+	  bp = bp->cdr;
+	}
+      bv[bl] = (rx_side_effect)-1;
+
+      {
+	int ret;
+	int x = 0;
+	while (av[x] == bv[x])
+	  ++x;
+ 	ret = (((unsigned *)(av[x]) < (unsigned *)(bv[x])) ? -1 : 1);
+	return ret;
+      }
+    }
+}
+
+
+
+
+
+/* re_compile_pattern is the GNU regular expression compiler: it
+   compiles PATTERN (of length SIZE) and puts the result in RXB.
+   Returns 0 if the pattern was valid, otherwise an error string.
+
+   Assumes the `allocated' (and perhaps `buffer') and `translate' fields
+   are set in RXB on entry.
+
+   We call rx_compile to do the actual compilation.  */
+
+#ifdef __STDC__
+__const__ char *
+re_compile_pattern (__const__ char *pattern,
+		    int length,
+		    struct re_pattern_buffer * rxb)
+#else
+__const__ char *
+re_compile_pattern (pattern, length, rxb)
+     __const__ char *pattern;
+     int length;
+     struct re_pattern_buffer * rxb;
+#endif
+{
+  reg_errcode_t ret;
+
+  /* GNU code is written to assume at least RE_NREGS registers will be set
+     (and at least one extra will be -1).  */
+  rxb->regs_allocated = REGS_UNALLOCATED;
+
+  /* And GNU code determines whether or not to get register information
+     by passing null for the REGS argument to re_match, etc., not by
+     setting no_sub.  */
+  rxb->no_sub = 0;
+
+  rxb->rx.local_cset_size = 256;
+
+  /* Match anchors at newline.  */
+  rxb->newline_anchor = 1;
+ 
+  rxb->re_nsub = 0;
+  rxb->start = 0;
+  rxb->se_params = 0;
+  rxb->rx.nodec = 0;
+  rxb->rx.epsnodec = 0;
+  rxb->rx.instruction_table = 0;
+  rxb->rx.nfa_states = 0;
+  rxb->rx.se_list_cmp = posix_se_list_order;
+  rxb->rx.start_set = 0;
+
+  ret = rx_compile (pattern, length, re_syntax_options, rxb);
+#ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+  _alloca (0);
+#else
+  alloca (0);
+#endif
+  return rx_error_msg[(int) ret];
+}
+
+
+
+#ifdef __STDC__
+int
+re_compile_fastmap (struct re_pattern_buffer * rxb)
+#else
+int
+re_compile_fastmap (rxb)
+     struct re_pattern_buffer * rxb;
+#endif
+{
+  rx_blow_up_fastmap (rxb);
+  return 0;
+}
+
+
+
+
+
+/* Entry points compatible with 4.2 BSD regex library.  We don't define
+   them if this is an Emacs or POSIX compilation.  */
+
+#if (!defined (emacs) && !defined (_POSIX_SOURCE)) || defined(USE_BSD_REGEX)
+
+/* BSD has one and only one pattern buffer.  */
+static struct re_pattern_buffer rx_comp_buf;
+
+#ifdef __STDC__
+char *
+re_comp (__const__ char *s)
+#else
+char *
+re_comp (s)
+    __const__ char *s;
+#endif
+{
+  reg_errcode_t ret;
+
+  if (!s || (*s == '\0'))
+    {
+      if (!rx_comp_buf.buffer)
+	return "No previous regular expression";
+      return 0;
+    }
+
+  if (!rx_comp_buf.fastmap)
+    {
+      rx_comp_buf.fastmap = (char *) malloc (1 << CHARBITS);
+      if (!rx_comp_buf.fastmap)
+	return "Memory exhausted";
+    }
+
+  /* Since `rx_exec' always passes NULL for the `regs' argument, we
+     don't need to initialize the pattern buffer fields which affect it.  */
+
+  /* Match anchors at newlines.  */
+  rx_comp_buf.newline_anchor = 1;
+
+  rx_comp_buf.re_nsub = 0;
+  rx_comp_buf.start = 0;
+  rx_comp_buf.se_params = 0;
+  rx_comp_buf.rx.nodec = 0;
+  rx_comp_buf.rx.epsnodec = 0;
+  rx_comp_buf.rx.instruction_table = 0;
+  rx_comp_buf.rx.nfa_states = 0;
+  rx_comp_buf.rx.start = 0;
+  rx_comp_buf.rx.se_list_cmp = posix_se_list_order;
+  rx_comp_buf.rx.local_cset_size = 256;
+
+  ret = rx_compile (s, strlen (s), re_syntax_options, &rx_comp_buf);
+#ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+  _alloca (0);
+#else
+  alloca (0);
+#endif
+
+  /* Yes, we're discarding `__const__' here.  */
+  return (char *) rx_error_msg[(int) ret];
+}
+
+
+#ifdef __STDC__
+int
+re_exec (__const__ char *s)
+#else
+int
+re_exec (s)
+    __const__ char *s;
+#endif
+{
+  __const__ int len = strlen (s);
+  return
+    0 <= re_search (&rx_comp_buf, s, len, 0, len, (struct re_registers *) 0);
+}
+#endif /* not emacs and not _POSIX_SOURCE */
+
+
+
+
+/* POSIX.2 functions.  Don't define these for Emacs.  */
+
+#if !defined(emacs)
+
+/* regcomp takes a regular expression as a string and compiles it.
+
+   PREG is a regex_t *.  We do not expect any fields to be initialized,
+   since POSIX says we shouldn't.  Thus, we set
+
+     `buffer' to the compiled pattern;
+     `used' to the length of the compiled pattern;
+     `syntax' to RE_SYNTAX_POSIX_EXTENDED if the
+       REG_EXTENDED bit in CFLAGS is set; otherwise, to
+       RE_SYNTAX_POSIX_BASIC;
+     `newline_anchor' to REG_NEWLINE being set in CFLAGS;
+     `fastmap' and `fastmap_accurate' to zero;
+     `re_nsub' to the number of subexpressions in PATTERN.
+
+   PATTERN is the address of the pattern string.
+
+   CFLAGS is a series of bits which affect compilation.
+
+     If REG_EXTENDED is set, we use POSIX extended syntax; otherwise, we
+     use POSIX basic syntax.
+
+     If REG_NEWLINE is set, then . and [^...] don't match newline.
+     Also, regexec will try a match beginning after every newline.
+
+     If REG_ICASE is set, then we considers upper- and lowercase
+     versions of letters to be equivalent when matching.
+
+     If REG_NOSUB is set, then when PREG is passed to regexec, that
+     routine will report only success or failure, and nothing about the
+     registers.
+
+   It returns 0 if it succeeds, nonzero if it doesn't.  (See regex.h for
+   the return codes and their meanings.)  */
+
+
+#ifdef __STDC__
+int
+regcomp (regex_t * preg, __const__ char * pattern, int cflags)
+#else
+int
+regcomp (preg, pattern, cflags)
+    regex_t * preg;
+    __const__ char * pattern;
+    int cflags;
+#endif
+{
+  reg_errcode_t ret;
+  unsigned syntax
+    = cflags & REG_EXTENDED ? RE_SYNTAX_POSIX_EXTENDED : RE_SYNTAX_POSIX_BASIC;
+
+  /* regex_compile will allocate the space for the compiled pattern.  */
+  preg->buffer = 0;
+  preg->allocated = 0;
+  preg->fastmap = malloc (256);
+  if (!preg->fastmap)
+    return REG_ESPACE;
+  preg->fastmap_accurate = 0;
+
+  if (cflags & REG_ICASE)
+    {
+      unsigned i;
+
+      preg->translate = (unsigned char *) malloc (256);
+      if (!preg->translate)
+        return (int) REG_ESPACE;
+
+      /* Map uppercase characters to corresponding lowercase ones.  */
+      for (i = 0; i < CHAR_SET_SIZE; i++)
+        preg->translate[i] = isupper (i) ? tolower (i) : i;
+    }
+  else
+    preg->translate = 0;
+
+  /* If REG_NEWLINE is set, newlines are treated differently.  */
+  if (cflags & REG_NEWLINE)
+    { /* REG_NEWLINE implies neither . nor [^...] match newline.  */
+      syntax &= ~RE_DOT_NEWLINE;
+      syntax |= RE_HAT_LISTS_NOT_NEWLINE;
+      /* It also changes the matching behavior.  */
+      preg->newline_anchor = 1;
+    }
+  else
+    preg->newline_anchor = 0;
+
+  preg->no_sub = !!(cflags & REG_NOSUB);
+
+  /* POSIX says a null character in the pattern terminates it, so we
+     can use strlen here in compiling the pattern.  */
+  preg->re_nsub = 0;
+  preg->start = 0;
+  preg->se_params = 0;
+  preg->syntax_parens = 0;
+  preg->rx.nodec = 0;
+  preg->rx.epsnodec = 0;
+  preg->rx.instruction_table = 0;
+  preg->rx.nfa_states = 0;
+  preg->rx.local_cset_size = 256;
+  preg->rx.start = 0;
+  preg->rx.se_list_cmp = posix_se_list_order;
+  preg->rx.start_set = 0;
+  ret = rx_compile (pattern, strlen (pattern), syntax, preg);
+
+#ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+  _alloca (0);
+#else
+  alloca (0);
+#endif
+
+  /* POSIX doesn't distinguish between an unmatched open-group and an
+     unmatched close-group: both are REG_EPAREN.  */
+  if (ret == REG_ERPAREN) ret = REG_EPAREN;
+
+  return (int) ret;
+}
+
+
+/* regexec searches for a given pattern, specified by PREG, in the
+   string STRING.
+
+   If NMATCH is zero or REG_NOSUB was set in the cflags argument to
+   `regcomp', we ignore PMATCH.  Otherwise, we assume PMATCH has at
+   least NMATCH elements, and we set them to the offsets of the
+   corresponding matched substrings.
+
+   EFLAGS specifies `execution flags' which affect matching: if
+   REG_NOTBOL is set, then ^ does not match at the beginning of the
+   string; if REG_NOTEOL is set, then $ does not match at the end.
+
+   We return 0 if we find a match and REG_NOMATCH if not.  */
+
+#ifdef __STDC__
+int
+regexec (__const__ regex_t *preg, __const__ char *string,
+	 size_t nmatch, regmatch_t pmatch[],
+	 int eflags)
+#else
+int
+regexec (preg, string, nmatch, pmatch, eflags)
+    __const__ regex_t *preg;
+    __const__ char *string;
+    size_t nmatch;
+    regmatch_t pmatch[];
+    int eflags;
+#endif
+{
+  int ret;
+  struct re_registers regs;
+  regex_t private_preg;
+  int len = strlen (string);
+  boolean want_reg_info = !preg->no_sub && nmatch > 0;
+
+  private_preg = *preg;
+
+  private_preg.not_bol = !!(eflags & REG_NOTBOL);
+  private_preg.not_eol = !!(eflags & REG_NOTEOL);
+
+  /* The user has told us exactly how many registers to return
+   * information about, via `nmatch'.  We have to pass that on to the
+   * matching routines.
+   */
+  private_preg.regs_allocated = REGS_FIXED;
+
+  if (want_reg_info)
+    {
+      regs.num_regs = nmatch;
+      regs.start =  (( regoff_t *) malloc ((nmatch) * sizeof ( regoff_t)));
+      regs.end =  (( regoff_t *) malloc ((nmatch) * sizeof ( regoff_t)));
+      if (regs.start == 0 || regs.end == 0)
+        return (int) REG_NOMATCH;
+    }
+
+  /* Perform the searching operation.  */
+  ret = re_search (&private_preg,
+		   string, len,
+                   /* start: */ 0,
+		   /* range: */ len,
+                   want_reg_info ? &regs : (struct re_registers *) 0);
+
+  /* Copy the register information to the POSIX structure.  */
+  if (want_reg_info)
+    {
+      if (ret >= 0)
+        {
+          unsigned r;
+
+          for (r = 0; r < nmatch; r++)
+            {
+              pmatch[r].rm_so = regs.start[r];
+              pmatch[r].rm_eo = regs.end[r];
+            }
+        }
+
+      /* If we needed the temporary register info, free the space now.  */
+      free (regs.start);
+      free (regs.end);
+    }
+
+  /* We want zero return to mean success, unlike `re_search'.  */
+  return ret >= 0 ? (int) REG_NOERROR : (int) REG_NOMATCH;
+}
+
+
+/* Returns a message corresponding to an error code, ERRCODE, returned
+   from either regcomp or regexec.   */
+
+#ifdef __STDC__
+size_t
+regerror (int errcode, __const__ regex_t *preg,
+	  char *errbuf, size_t errbuf_size)
+#else
+size_t
+regerror (errcode, preg, errbuf, errbuf_size)
+    int errcode;
+    __const__ regex_t *preg;
+    char *errbuf;
+    size_t errbuf_size;
+#endif
+{
+  __const__ char *msg
+    = rx_error_msg[errcode] == 0 ? "Success" : rx_error_msg[errcode];
+  size_t msg_size = strlen (msg) + 1; /* Includes the 0.  */
+
+  if (errbuf_size != 0)
+    {
+      if (msg_size > errbuf_size)
+        {
+          strncpy (errbuf, msg, errbuf_size - 1);
+          errbuf[errbuf_size - 1] = 0;
+        }
+      else
+        strcpy (errbuf, msg);
+    }
+
+  return msg_size;
+}
+
+
+/* Free dynamically allocated space used by PREG.  */
+
+#ifdef __STDC__
+void
+regfree (regex_t *preg)
+#else
+void
+regfree (preg)
+    regex_t *preg;
+#endif
+{
+  if (preg->buffer != 0)
+    free (preg->buffer);
+  preg->buffer = 0;
+  preg->allocated = 0;
+
+  if (preg->fastmap != 0)
+    free (preg->fastmap);
+  preg->fastmap = 0;
+  preg->fastmap_accurate = 0;
+
+  if (preg->translate != 0)
+    free (preg->translate);
+  preg->translate = 0;
+}
+
+#endif /* not emacs  */
+
+
+
+
+
Index: /tags/gsdl-21/gsdl/packages/mg/lib/rx.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/rx.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/rx.h	(revision 699)
@@ -0,0 +1,3755 @@
+#if !defined(RXH) || defined(RX_WANT_SE_DEFS)
+#define RXH
+
+/*	Copyright (C) 1992, 1993 Free Software Foundation, Inc.
+
+This file is part of the librx library.
+
+Librx is free software; you can redistribute it and/or modify it under
+the terms of the GNU Library General Public License as published by
+the Free Software Foundation; either version 2, or (at your option)
+any later version.
+
+Librx is distributed in the hope that it will be useful, but WITHOUT
+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU Library General Public
+License along with this software; see the file COPYING.LIB.  If not,
+write to the Free Software Foundation, 675 Mass Ave, Cambridge, MA
+02139, USA.  */
+/*  t. lord	Wed Sep 23 18:20:57 1992	*/
+
+
+
+
+
+
+
+
+
+#ifndef RX_WANT_SE_DEFS
+
+/* This page: Bitsets */
+
+#ifndef RX_subset
+typedef unsigned int RX_subset;
+#define RX_subset_bits	(32)
+#define RX_subset_mask	(RX_subset_bits - 1)
+#endif
+
+typedef RX_subset * rx_Bitset;
+
+#ifdef __STDC__
+typedef void (*rx_bitset_iterator) (void *, int member_index);
+#else
+typedef void (*rx_bitset_iterator) ();
+#endif
+
+#define rx_bitset_subset(N)  ((N) / RX_subset_bits)
+#define rx_bitset_subset_val(B,N)  ((B)[rx_bitset_subset(N)])
+#define RX_bitset_access(B,N,OP) \
+  ((B)[rx_bitset_subset(N)] OP rx_subset_singletons[(N) & RX_subset_mask])
+#define RX_bitset_member(B,N)   RX_bitset_access(B, N, &)
+#define RX_bitset_enjoin(B,N)   RX_bitset_access(B, N, |=)
+#define RX_bitset_remove(B,N)   RX_bitset_access(B, N, &= ~)
+#define RX_bitset_toggle(B,N)   RX_bitset_access(B, N, ^= )
+#define rx_bitset_numb_subsets(N) (((N) + RX_subset_bits - 1) / RX_subset_bits)
+#define rx_sizeof_bitset(N)	(rx_bitset_numb_subsets(N) * sizeof(RX_subset))
+
+
+
+
+/* This page: Splay trees. */
+
+#ifdef __STDC__
+typedef int (*rx_sp_comparer) (void * a, void * b);
+#else
+typedef int (*rx_sp_comparer) ();
+#endif
+
+struct rx_sp_node 
+{
+  void * key;
+  void * data;
+  struct rx_sp_node * kids[2];
+};
+
+#ifdef __STDC__
+typedef void (*rx_sp_key_data_freer) (struct rx_sp_node *);
+#else
+typedef void (*rx_sp_key_data_freer) ();
+#endif
+
+
+
+/* giant inflatable hash trees */
+
+struct rx_hash_item
+{
+  struct rx_hash_item * next_same_hash;
+  struct rx_hash * table;
+  unsigned long hash;
+  void * data;
+  void * binding;
+};
+
+struct rx_hash
+{
+  struct rx_hash * parent;
+  int refs;
+  struct rx_hash * children[13];
+  struct rx_hash_item * buckets [13];
+  int bucket_size [13];
+};
+
+struct rx_hash_rules;
+
+#ifdef __STDC__
+/* should return like == */
+typedef int (*rx_hash_eq)(void *, void *);
+typedef struct rx_hash * (*rx_alloc_hash)(struct rx_hash_rules *);
+typedef void (*rx_free_hash)(struct rx_hash *,
+			    struct rx_hash_rules *);
+typedef struct rx_hash_item * (*rx_alloc_hash_item)(struct rx_hash_rules *,
+						    void *);
+typedef void (*rx_free_hash_item)(struct rx_hash_item *,
+				 struct rx_hash_rules *);
+#else
+typedef int (*rx_hash_eq)();
+typedef struct rx_hash * (*rx_alloc_hash)();
+typedef void (*rx_free_hash)();
+typedef struct rx_hash_item * (*rx_alloc_hash_item)();
+typedef void (*rx_free_hash_item)();
+#endif
+
+struct rx_hash_rules
+{
+  rx_hash_eq eq;
+  rx_alloc_hash hash_alloc;
+  rx_free_hash free_hash;
+  rx_alloc_hash_item hash_item_alloc;
+  rx_free_hash_item free_hash_item;
+};
+
+
+
+/* Forward declarations */
+
+struct rx_cache;
+struct rx_superset;
+struct rx;
+struct rx_se_list;
+
+
+
+
+/* 
+ * GLOSSARY
+ *
+ * regexp
+ * regular expression
+ * expression
+ * pattern - a `regular' expression.  The expression
+ *       need not be formally regular -- it can contain
+ *       constructs that don't correspond to purely regular
+ *       expressions.
+ *
+ * buffer
+ * string - the string (or strings) being searched or matched.
+ *
+ * pattern buffer - a structure of type `struct re_pattern_buffer'
+ *       This in turn contains a `struct rx', which holds the
+ *       NFA compiled from a pattern, as well as some of the state
+ *       of a matcher using the pattern.
+ *
+ * NFA - nondeterministic finite automata.  Some people
+ *       use this term to a member of the class of 
+ *       regular automata (those corresponding to a regular
+ *       language).  However, in this code, the meaning is
+ *       more general.  The automata used by Rx are comperable
+ *       in power to what are usually called `push down automata'.
+ *
+ *       Two NFA are built by rx for every pattern.  One is built
+ *       by the compiler.  The other is built from the first, on
+ *       the fly, by the matcher.  The latter is called the `superstate
+ *       NFA' because its states correspond to sets of states from
+ *       the first NFA.  (Joe Keane gets credit for the name
+ *       `superstate NFA').
+ *
+ * NFA edges
+ * epsilon edges
+ * side-effect edges - The NFA compiled from a pattern can have three
+ *       kinds of edges.  Epsilon edges can be taken freely anytime
+ *       their source state is reached.  Character set edges can be
+ *       taken when their source state is reached and when the next 
+ *       character in the buffer is a member of the set.  Side effect
+ *       edges imply a transition that can only be taken after the
+ *       indicated side effect has been successfully accomplished.
+ *       Some examples of side effects are:
+ *
+ *		Storing the current match position to record the
+ *              location of a parentesized subexpression.
+ *
+ *              Advancing the matcher over N characters if they
+ *              match the N characters previously matched by a 
+ *              parentesized subexpression.
+ *
+ *       Both of those kinds of edges occur in the NFA generated
+ *       by the pattern:  \(.\)\1
+ *
+ *       Epsilon and side effect edges are similar.  Unfortunately,
+ *       some of the code uses the name `epsilon edge' to mean
+ *       both epsilon and side effect edges.  For example,  the
+ *       function has_non_idempotent_epsilon_path computes the existance
+ *       of a non-trivial path containing only a mix of epsilon and
+ *       side effect edges.  In that case `nonidempotent epsilon' is being
+ *       used to mean `side effect'.
+ */
+
+
+
+
+
+
+/* LOW LEVEL PATTERN BUFFERS */
+
+/* Suppose that from some NFA state, more than one path through
+ * side-effect edges is possible.  In what order should the paths
+ * be tried?  A function of type rx_se_list_order answers that
+ * question.  It compares two lists of side effects, and says
+ * which list comes first.
+ */
+ 
+#ifdef __STDC__
+typedef int (*rx_se_list_order) (struct rx *,
+				 struct rx_se_list *, 
+				 struct rx_se_list *);
+#else
+typedef int (*rx_se_list_order) ();
+#endif
+
+
+
+/* Struct RX holds a compiled regular expression - that is, an nfa
+ * ready to be converted on demand to a more efficient superstate nfa.
+ * This is for the low level interface.  The high-level interfaces enclose
+ * this in a `struct re_pattern_buffer'.  
+ */
+struct rx
+{
+  /* The compiler assigns a unique id to every pattern.
+   * Like sequence numbers in X, there is a subtle bug here
+   * if you use Rx in a system that runs for a long time.
+   * But, because of the way the caches work out, it is almost
+   * impossible to trigger the Rx version of this bug.
+   *
+   * The id is used to validate superstates found in a cache
+   * of superstates.  It isn't sufficient to let a superstate
+   * point back to the rx for which it was compiled -- the caller
+   * may be re-using a `struct rx' in which case the superstate
+   * is not really valid.  So instead, superstates are validated
+   * by checking the sequence number of the pattern for which
+   * they were built.
+   */
+  int rx_id;
+
+  /* This is memory mgt. state for superstates.  This may be 
+   * shared by more than one struct rx.
+   */
+  struct rx_cache * cache;
+
+  /* Every regex defines the size of its own character set. 
+   * A superstate has an array of this size, with each element
+   * a `struct rx_inx'.  So, don't make this number too large.
+   * In particular, don't make it 2^16.
+   */
+  int local_cset_size;
+
+  /* After the NFA is built, it is copied into a contiguous region
+   * of memory (mostly for compatability with GNU regex).
+   * Here is that region, and it's size:
+   */
+  void * buffer;
+  unsigned long allocated;
+
+  /* Clients of RX can ask for some extra storage in the space pointed
+   * to by BUFFER.  The field RESERVED is an input parameter to the
+   * compiler.  After compilation, this much space will be available 
+   * at (buffer + allocated - reserved)
+   */
+  unsigned long reserved;
+
+  /* --------- The remaining fields are for internal use only. --------- */
+  /* --------- But! they must be initialized to 0.	       --------- */
+
+  /* NODEC is the number of nodes in the NFA with non-epsilon
+   * transitions. 
+   */
+  int nodec;
+
+  /* EPSNODEC is the number of nodes with only epsilon transitions. */
+  int epsnodec;
+
+  /* The sum (NODEC + EPSNODEC) is the total number of states in the
+   * compiled NFA.
+   */
+
+  /* Lists of side effects as stored in the NFA are `hash consed'..meaning
+   * that lists with the same elements are ==.  During compilation, 
+   * this table facilitates hash-consing.
+   */
+  struct rx_hash se_list_memo;
+
+  /* Lists of NFA states are also hashed. 
+   */
+  struct rx_hash set_list_memo;
+
+
+
+
+  /* The compiler and matcher must build a number of instruction frames.
+   * The format of these frames is fixed (c.f. struct rx_inx).  The values
+   * of the instructions is not fixed.
+   *
+   * An enumerated type (enum rx_opcode) defines the set of instructions
+   * that the compiler or matcher might generate.  When filling an instruction
+   * frame, the INX field is found by indexing this instruction table
+   * with an opcode:
+   */
+  void ** instruction_table;
+
+  /* The list of all states in an NFA.
+   * During compilation, the NEXT field of NFA states links this list.
+   * After compilation, all the states are compacted into an array,
+   * ordered by state id numbers.  At that time, this points to the base 
+   * of that array.
+   */
+  struct rx_nfa_state *nfa_states;
+
+  /* Every nfa begins with one distinguished starting state:
+   */
+  struct rx_nfa_state *start;
+
+  /* This orders the search through super-nfa paths.
+   * See the comment near the typedef of rx_se_list_order.
+   */
+  rx_se_list_order se_list_cmp;
+
+  struct rx_superset * start_set;
+};
+
+
+
+
+
+/* SYNTAX TREES */
+
+/* Compilation is in stages.  
+ *
+ * In the first stage, a pattern specified by a string is 
+ * translated into a syntax tree.  Later stages will convert
+ * the syntax tree into an NFA optimized for conversion to a
+ * superstate-NFA.
+ *
+ * This page is about syntax trees.
+ */
+
+enum rexp_node_type
+{
+  r_cset,			/* Match from a character set. `a' or `[a-z]'*/
+  r_concat,			/* Concat two subexpressions.   `ab' */
+  r_alternate,			/* Choose one of two subexpressions. `a\|b' */
+  r_opt,			/* Optional subexpression. `a?' */
+  r_star,			/* Repeated subexpression. `a*' */
+
+
+  /* A 2phase-star is a variation on a repeated subexpression.
+   * In this case, there are two subexpressions.  The first, if matched,
+   * begins a repitition (otherwise, the whole expression is matches the
+   * empth string).  
+   * 
+   * After matching the first subexpression, a 2phase star either finishes,
+   * or matches the second subexpression.  If the second subexpression is
+   * matched, then the whole construct repeats.
+   *
+   * 2phase stars are used in two circumstances.  First, they
+   * are used as part of the implementation of POSIX intervals (counted
+   * repititions).  Second, they are used to implement proper star
+   * semantics when the repeated subexpression contains paths of
+   * only side effects.  See rx_compile for more information.
+   */
+  r_2phase_star,
+
+
+  /* c.f. "typedef void * rx_side_effect" */
+  r_side_effect,
+
+  /* This is an extension type:  It is for transient use in source->source
+   * transformations (implemented over syntax trees).
+   */
+  r_data
+};
+
+/* A side effect is a matcher-specific action associated with
+ * transitions in the NFA.  The details of side effects are up
+ * to the matcher.  To the compiler and superstate constructors
+ * side effects are opaque:
+ */
+
+typedef void * rx_side_effect;
+
+/* Nodes in a syntax tree are of this type:
+ */
+struct rexp_node
+{
+  enum rexp_node_type type;
+  union
+  {
+    rx_Bitset cset;
+    rx_side_effect side_effect;
+    struct
+      {
+	struct rexp_node *left;
+	struct rexp_node *right;
+      } pair;
+    void * data;
+  } params;
+};
+
+
+
+
+/* NFA
+ *
+ * A syntax tree is compiled into an NFA.  This page defines the structure
+ * of that NFA.
+ */
+
+struct rx_nfa_state
+{
+  /* These are kept in a list as the NFA is being built. */
+  struct rx_nfa_state *next;
+
+  /* After the NFA is built, states are given integer id's.
+   * States whose outgoing transitions are all either epsilon or 
+   * side effect edges are given ids less than 0.  Other states
+   * are given successive non-negative ids starting from 0.
+   */
+  int id;
+
+  /* The list of NFA edges that go from this state to some other. */
+  struct rx_nfa_edge *edges;
+
+  /* If you land in this state, then you implicitly land
+   * in all other states reachable by only epsilon translations.
+   * Call the set of maximal paths to such states the epsilon closure
+   * of this state.
+   *
+   * There may be other states that are reachable by a mixture of
+   * epsilon and side effect edges.  Consider the set of maximal paths
+   * of that sort from this state.  Call it the epsilon-side-effect
+   * closure of the state.
+   * 
+   * The epsilon closure of the state is a subset of the epsilon-side-
+   * effect closure.  It consists of all the paths that contain 
+   * no side effects -- only epsilon edges.
+   * 
+   * The paths in the epsilon-side-effect closure  can be partitioned
+   * into equivalance sets. Two paths are equivalant if they have the
+   * same set of side effects, in the same order.  The epsilon-closure
+   * is one of these equivalance sets.  Let's call these equivalance
+   * sets: observably equivalant path sets.  That name is chosen
+   * because equivalance of two paths means they cause the same side
+   * effects -- so they lead to the same subsequent observations other
+   * than that they may wind up in different target states.
+   *
+   * The superstate nfa, which is derived from this nfa, is based on
+   * the observation that all of the paths in an observably equivalant
+   * path set can be explored at the same time, provided that the
+   * matcher keeps track not of a single nfa state, but of a set of
+   * states.   In particular, after following all the paths in an
+   * observably equivalant set, you wind up at a set of target states.
+   * That set of target states corresponds to one state in the
+   * superstate NFA.
+   *
+   * Staticly, before matching begins, it is convenient to analyze the
+   * nfa.  Each state is labeled with a list of the observably
+   * equivalant path sets who's union covers all the
+   * epsilon-side-effect paths beginning in this state.  This list is
+   * called the possible futures of the state.
+   *
+   * A trivial example is this NFA:
+   *             s1
+   *         A  --->  B
+   *
+   *             s2  
+   *            --->  C
+   *
+   *             epsilon           s1
+   *            --------->  D   ------> E
+   * 
+   * 
+   * In this example, A has two possible futures.
+   * One invokes the side effect `s1' and contains two paths,
+   * one ending in state B, the other in state E.
+   * The other invokes the side effect `s2' and contains only
+   * one path, landing in state C.
+   */
+  struct rx_possible_future *futures;
+
+
+  /* There are exactly two distinguished states in every NFA: */
+  unsigned int is_final:1;
+  unsigned int is_start:1;
+
+  /* These are used during NFA construction... */
+  unsigned int eclosure_needed:1;
+  unsigned int mark:1;
+};
+
+
+/* An edge in an NFA is typed: */
+enum rx_nfa_etype
+{
+  /* A cset edge is labled with a set of characters one of which
+   * must be matched for the edge to be taken.
+   */
+  ne_cset,
+
+  /* An epsilon edge is taken whenever its starting state is
+   * reached. 
+   */
+  ne_epsilon,
+
+  /* A side effect edge is taken whenever its starting state is
+   * reached.  Side effects may cause the match to fail or the
+   * position of the matcher to advance.
+   */
+  ne_side_effect		/* A special kind of epsilon. */
+};
+
+struct rx_nfa_edge
+{
+  struct rx_nfa_edge *next;
+  enum rx_nfa_etype type;
+  struct rx_nfa_state *dest;
+  union
+  {
+    rx_Bitset cset;
+    rx_side_effect side_effect;
+  } params;
+};
+
+
+
+/* A possible future consists of a list of side effects
+ * and a set of destination states.  Below are their
+ * representations.  These structures are hash-consed which
+ * means that lists with the same elements share a representation
+ * (their addresses are ==).
+ */
+
+struct rx_nfa_state_set
+{
+  struct rx_nfa_state * car;
+  struct rx_nfa_state_set * cdr;
+};
+
+struct rx_se_list
+{
+  rx_side_effect car;
+  struct rx_se_list * cdr;
+};
+
+struct rx_possible_future
+{
+  struct rx_possible_future *next;
+  struct rx_se_list * effects;
+  struct rx_nfa_state_set * destset;
+};
+
+
+
+
+/* This begins the description of the superstate NFA.
+ *
+ * The superstate NFA corresponds to the NFA in these ways:
+ *
+ * Every superstate NFA states SUPER correspond to sets of NFA states,
+ * nfa_states(SUPER).
+ *
+ * Superstate edges correspond to NFA paths.
+ *
+ * The superstate has no epsilon transitions;
+ * every edge has a character label, and a (possibly empty) side
+ * effect label.   The side effect label corresponds to a list of
+ * side effects that occur in the NFA.  These parts are referred
+ * to as:   superedge_character(EDGE) and superedge_sides(EDGE).
+ *
+ * For a superstate edge EDGE starting in some superstate SUPER,
+ * the following is true (in pseudo-notation :-):
+ *
+ *       exists DEST in nfa_states s.t. 
+ *         exists nfaEDGE in nfa_edges s.t.
+ *                 origin (nfaEDGE) == DEST
+ *              && origin (nfaEDGE) is a member of nfa_states(SUPER)
+ *              && exists PF in possible_futures(dest(nfaEDGE)) s.t.
+ * 	                sides_of_possible_future (PF) == superedge_sides (EDGE)
+ *
+ * also:
+ *
+ *      let SUPER2 := superedge_destination(EDGE)
+ *          nfa_states(SUPER2)
+ *           == union of all nfa state sets S s.t.
+ *                          exists PF in possible_futures(dest(nfaEDGE)) s.t.
+ * 	                       sides_of_possible_future (PF) == superedge_sides (EDGE)
+ *                          && S == dests_of_possible_future (PF) }
+ *
+ * Or in english, every superstate is a set of nfa states.  A given
+ * character and a superstate implies many transitions in the NFA --
+ * those that begin with an edge labeled with that character from a
+ * state in the set corresponding to the superstate.
+ * 
+ * The destinations of those transitions each have a set of possible
+ * futures.  A possible future is a list of side effects and a set of
+ * destination NFA states.  Two sets of possible futures can be
+ * `merged' by combining all pairs of possible futures that have the
+ * same side effects.  A pair is combined by creating a new future
+ * with the same side effect but the union of the two destination sets.
+ * In this way, all the possible futures suggested by a superstate
+ * and a character can be merged into a set of possible futures where
+ * no two elements of the set have the same set of side effects.
+ *
+ * The destination of a possible future, being a set of NFA states, 
+ * corresponds to a supernfa state.  So, the merged set of possible
+ * futures we just created can serve as a set of edges in the
+ * supernfa.
+ *
+ * The representation of the superstate nfa and the nfa is critical.
+ * The nfa has to be compact, but has to facilitate the rapid
+ * computation of missing superstates.  The superstate nfa has to 
+ * be fast to interpret, lazilly constructed, and bounded in space.
+ *
+ * To facilitate interpretation, the superstate data structures are 
+ * peppered with `instruction frames'.  There is an instruction set
+ * defined below which matchers using the supernfa must be able to
+ * interpret.
+ *
+ * We'd like to make it possible but not mandatory to use code
+ * addresses to represent instructions (c.f. gcc's computed goto).
+ * Therefore, we define an enumerated type of opcodes, and when
+ * writing one of these instructions into a data structure, use
+ * the opcode as an index into a table of instruction values.
+ * 
+ * Here are the opcodes that occur in the superstate nfa:
+ */
+ 
+
+/* Every superstate contains a table of instruction frames indexed 
+ * by characters.  A normal `move' in a matcher is to fetch the next
+ * character and use it as an index into a superstates transition
+ * table.
+ *
+ * In the fasted case, only one edge follows from that character.
+ * In other cases there is more work to do.
+ * 
+ * The descriptions of the opcodes refer to data structures that are
+ * described further below. 
+ */
+
+enum rx_opcode
+{
+  /* 
+   * BACKTRACK_POINT is invoked when a character transition in 
+   * a superstate leads to more than one edge.  In that case,
+   * the edges have to be explored independently using a backtracking
+   * strategy.
+   *
+   * A BACKTRACK_POINT instruction is stored in a superstate's 
+   * transition table for some character when it is known that that
+   * character crosses more than one edge.  On encountering this
+   * instruction, the matcher saves enough state to backtrack to this
+   * point in the match later.
+   */
+  rx_backtrack_point = 0,	/* data is (struct transition_class *) */
+
+  /* 
+   * RX_DO_SIDE_EFFECTS evaluates the side effects of an epsilon path.
+   * There is one occurence of this instruction per rx_distinct_future.
+   * This instruction is skipped if a rx_distinct_future has no side effects.
+   */
+  rx_do_side_effects = rx_backtrack_point + 1,
+
+  /* data is (struct rx_distinct_future *) */
+
+  /* 
+   * RX_CACHE_MISS instructions are stored in rx_distinct_futures whose
+   * destination superstate has been reclaimed (or was never built).
+   * It recomputes the destination superstate.
+   * RX_CACHE_MISS is also stored in a superstate transition table before
+   * any of its edges have been built.
+   */
+  rx_cache_miss = rx_do_side_effects + 1,
+  /* data is (struct rx_distinct_future *) */
+
+  /* 
+   * RX_NEXT_CHAR is called to consume the next character and take the
+   * corresponding transition.  This is the only instruction that uses 
+   * the DATA field of the instruction frame instead of DATA_2.
+   * (see EXPLORE_FUTURE in regex.c).
+   */
+  rx_next_char = rx_cache_miss + 1, /* data is (struct superstate *) */
+
+  /* RX_BACKTRACK indicates that a transition fails.
+   */
+  rx_backtrack = rx_next_char + 1, /* no data */
+
+  /* 
+   * RX_ERROR_INX is stored only in places that should never be executed.
+   */
+  rx_error_inx = rx_backtrack + 1, /* Not supposed to occur. */
+
+  rx_num_instructions = rx_error_inx + 1
+};
+
+/* An id_instruction_table holds the values stored in instruction
+ * frames.  The table is indexed by the enums declared above.
+ */
+extern void * rx_id_instruction_table[rx_num_instructions];
+
+/* The heart of the matcher is a `word-code-interpreter' 
+ * (like a byte-code interpreter, except that instructions
+ * are a full word wide).
+ *
+ * Instructions are not stored in a vector of code, instead,
+ * they are scattered throughout the data structures built
+ * by the regexp compiler and the matcher.  One word-code instruction,
+ * together with the arguments to that instruction, constitute
+ * an instruction frame (struct rx_inx).
+ *
+ * This structure type is padded by hand to a power of 2 because
+ * in one of the dominant cases, we dispatch by indexing a table
+ * of instruction frames.  If that indexing can be accomplished
+ * by just a shift of the index, we're happy.
+ *
+ * Instructions take at most one argument, but there are two
+ * slots in an instruction frame that might hold that argument.
+ * These are called data and data_2.  The data slot is only
+ * used for one instruction (RX_NEXT_CHAR).  For all other 
+ * instructions, data should be set to 0.
+ *
+ * RX_NEXT_CHAR is the most important instruction by far.
+ * By reserving the data field for its exclusive use, 
+ * instruction dispatch is sped up in that case.  There is
+ * no need to fetch both the instruction and the data,
+ * only the data is needed.  In other words, a `cycle' begins
+ * by fetching the field data.  If that is non-0, then it must
+ * be the destination state of a next_char transition, so
+ * make that value the current state, advance the match position
+ * by one character, and start a new cycle.  On the other hand,
+ * if data is 0, fetch the instruction and do a more complicated
+ * dispatch on that.
+ */
+
+struct rx_inx 
+{
+  void * data;
+  void * data_2;
+  void * inx;
+  void * fnord;
+};
+
+#ifndef RX_TAIL_ARRAY
+#define RX_TAIL_ARRAY  1
+#endif
+
+/* A superstate corresponds to a set of nfa states.  Those sets are
+ * represented by STRUCT RX_SUPERSET.  The constructors
+ * guarantee that only one (shared) structure is created for a given set.
+ */
+struct rx_superset
+{
+  int refs;			/* This is a reference counted structure. */
+
+  /* We keep these sets in a cache because (in an unpredictable way),
+   * the same set is often created again and again.  But that is also
+   * problematic -- compatibility with POSIX and GNU regex requires
+   * that we not be able to tell when a program discards a particular
+   * NFA (thus invalidating the supersets created from it).
+   *
+   * But when a cache hit appears to occur, we will have in hand the
+   * nfa for which it may have happened.  That is why every nfa is given
+   * its own sequence number.  On a cache hit, the cache is validated
+   * by comparing the nfa sequence number to this field:
+   */
+  int id;
+
+  struct rx_nfa_state * car;	/* May or may not be a valid addr. */
+  struct rx_superset * cdr;
+
+  /* If the corresponding superstate exists: */
+  struct rx_superstate * superstate;
+
+
+  /* There is another bookkeeping problem.  It is expensive to 
+   * compute the starting nfa state set for an nfa.  So, once computed,
+   * it is cached in the `struct rx'.
+   *
+   * But, the state set can be flushed from the superstate cache.
+   * When that happens, we can't know if the corresponding `struct rx'
+   * is still alive or if it has been freed or re-used by the program.
+   * So, the cached pointer to this set in a struct rx might be invalid
+   * and we need a way to validate it.
+   *
+   * Fortunately, even if this set is flushed from the cache, it is
+   * not freed.  It just goes on the free-list of supersets.
+   * So we can still examine it.  
+   *
+   * So to validate a starting set memo, check to see if the
+   * starts_for field still points back to the struct rx in question,
+   * and if the ID matches the rx sequence number.
+   */
+  struct rx * starts_for;
+
+  /* This is used to link into a hash bucket so these objects can
+   * be `hash-consed'.
+   */
+  struct rx_hash_item hash_item;
+};
+
+#define rx_protect_superset(RX,CON) (++(CON)->refs)
+
+/* The terminology may be confusing (rename this structure?).
+ * Every character occurs in at most one rx_super_edge per super-state.
+ * But, that structure might have more than one option, indicating a point
+ * of non-determinism. 
+ *
+ * In other words, this structure holds a list of superstate edges
+ * sharing a common starting state and character label.  The edges
+ * are in the field OPTIONS.  All superstate edges sharing the same
+ * starting state and character are in this list.
+ */
+struct rx_super_edge
+{
+  struct rx_super_edge *next;
+  struct rx_inx rx_backtrack_frame;
+  int cset_size;
+  rx_Bitset cset;
+  struct rx_distinct_future *options;
+};
+
+/* A superstate is a set of nfa states (RX_SUPERSET) along
+ * with a transition table.  Superstates are built on demand and reclaimed
+ * without warning.  To protect a superstate from this ghastly fate,
+ * use LOCK_SUPERSTATE. 
+ */
+struct rx_superstate
+{
+  int rx_id;			/* c.f. the id field of rx_superset */
+  int locks;			/* protection from reclamation */
+
+  /* Within a superstate cache, all the superstates are kept in a big
+   * queue.  The tail of the queue is the state most likely to be
+   * reclaimed.  The *recyclable fields hold the queue position of 
+   * this state.
+   */
+  struct rx_superstate * next_recyclable;
+  struct rx_superstate * prev_recyclable;
+
+  /* The supernfa edges that exist in the cache and that have
+   * this state as their destination are kept in this list:
+   */
+  struct rx_distinct_future * transition_refs;
+
+  /* The list of nfa states corresponding to this superstate: */
+  struct rx_superset * contents;
+
+  /* The list of edges in the cache beginning from this state. */
+  struct rx_super_edge * edges;
+
+  /* A tail of the recyclable queue is marked as semifree.  A semifree
+   * state has no incoming next_char transitions -- any transition
+   * into a semifree state causes a complex dispatch with the side
+   * effect of rescuing the state from its semifree state.
+   *
+   * An alternative to this might be to make next_char more expensive,
+   * and to move a state to the head of the recyclable queue whenever
+   * it is entered.  That way, popular states would never be recycled.
+   *
+   * But unilaterally making next_char more expensive actually loses.
+   * So, incoming transitions are only made expensive for states near
+   * the tail of the recyclable queue.  The more cache contention
+   * there is, the more frequently a state will have to prove itself
+   * and be moved back to the front of the queue.  If there is less 
+   * contention, then popular states just aggregate in the front of 
+   * the queue and stay there.
+   */
+  int is_semifree;
+
+
+  /* This keeps track of the size of the transition table for this
+   * state.  There is a half-hearted attempt to support variable sized
+   * superstates.
+   */
+  int trans_size;
+
+  /* Indexed by characters... */
+  struct rx_inx transitions[RX_TAIL_ARRAY];
+};
+
+
+/* A list of distinct futures define the edges that leave from a 
+ * given superstate on a given character.  c.f. rx_super_edge.
+ */
+
+struct rx_distinct_future
+{
+  struct rx_distinct_future * next_same_super_edge[2];
+  struct rx_distinct_future * next_same_dest;
+  struct rx_distinct_future * prev_same_dest;
+  struct rx_superstate * present;	/* source state */
+  struct rx_superstate * future;	/* destination state */
+  struct rx_super_edge * edge;
+
+
+  /* The future_frame holds the instruction that should be executed
+   * after all the side effects are done, when it is time to complete
+   * the transition to the next state.
+   *
+   * Normally this is a next_char instruction, but it may be a
+   * cache_miss instruction as well, depending on whether or not
+   * the superstate is in the cache and semifree.
+   * 
+   * If this is the only future for a given superstate/char, and
+   * if there are no side effects to be performed, this frame is
+   * not used (directly) at all.  Instead, its contents are copied
+   * into the transition table of the starting state of this dist. future.
+   */
+  struct rx_inx future_frame;
+
+  struct rx_inx side_effects_frame;
+  struct rx_se_list * effects;
+};
+
+#define rx_lock_superstate(R,S)  ((S)->locks++)
+#define rx_unlock_superstate(R,S) (--(S)->locks)
+
+
+
+/* This page destined for rx.h */
+
+struct rx_blocklist
+{
+  struct rx_blocklist * next;
+  int bytes;
+};
+
+struct rx_freelist
+{
+  struct rx_freelist * next;
+};
+
+struct rx_cache;
+
+#ifdef __STDC__
+typedef void (*rx_morecore_fn)(struct rx_cache *);
+#else
+typedef void (*rx_morecore_fn)();
+#endif
+
+/* You use this to control the allocation of superstate data 
+ * during matching.  Most of it should be initialized to 0.
+ *
+ * A MORECORE function is necessary.  It should allocate
+ * a new block of memory or return 0.
+ * A default that uses malloc is called `rx_morecore'.
+ *
+ * The number of SUPERSTATES_ALLOWED indirectly limits how much memory
+ * the system will try to allocate.  The default is 128.  Batch style
+ * applications that are very regexp intensive should use as high a number
+ * as possible without thrashing.
+ * 
+ * The LOCAL_CSET_SIZE is the number of characters in a character set.
+ * It is therefore the number of entries in a superstate transition table.
+ * Generally, it should be 256.  If your character set has 16 bits, 
+ * it is better to translate your regexps into equivalent 8 bit patterns.
+ */
+
+struct rx_cache
+{
+  struct rx_hash_rules superset_hash_rules;
+
+  /* Objects are allocated by incrementing a pointer that 
+   * scans across rx_blocklists.
+   */
+  struct rx_blocklist * memory;
+  struct rx_blocklist * memory_pos;
+  int bytes_left;
+  char * memory_addr;
+  rx_morecore_fn morecore;
+
+  /* Freelists. */
+  struct rx_freelist * free_superstates;
+  struct rx_freelist * free_transition_classes;
+  struct rx_freelist * free_discernable_futures;
+  struct rx_freelist * free_supersets;
+  struct rx_freelist * free_hash;
+
+  /* Two sets of superstates -- those that are semifreed, and those
+   * that are being used.
+   */
+  struct rx_superstate * lru_superstate;
+  struct rx_superstate * semifree_superstate;
+
+  struct rx_superset * empty_superset;
+
+  int superstates;
+  int semifree_superstates;
+  int hits;
+  int misses;
+  int superstates_allowed;
+
+  int local_cset_size;
+  void ** instruction_table;
+
+  struct rx_hash superset_table;
+};
+
+
+
+
+/* The lowest-level search function supports arbitrarily fragmented
+ * strings and (optionally) suspendable/resumable searches.
+ *
+ * Callers have to provide a few hooks.
+ */
+
+#ifndef __GNUC__
+#ifdef __STDC__
+#define __const__ const
+#else
+#define __const__
+#endif
+#endif
+
+/* This holds a matcher position */
+struct rx_string_position
+{
+  __const__ unsigned char * pos;	/* The current pos. */
+  __const__ unsigned char * string; /* The current string burst. */
+  __const__ unsigned char * end;	/* First invalid position >= POS. */
+  int offset;			/* Integer address of the current burst. */
+  int size;			/* Current string's size. */
+  int search_direction;		/* 1 or -1 */
+  int search_end;		/* First position to not try. */
+};
+
+
+enum rx_get_burst_return
+{
+  rx_get_burst_continuation,
+  rx_get_burst_error,
+  rx_get_burst_ok,
+  rx_get_burst_no_more
+};
+
+
+/* A call to get burst should make POS valid.  It might be invalid
+ * if the STRING field doesn't point to a burst that actually
+ * contains POS.
+ *
+ * GET_BURST should take a clue from SEARCH_DIRECTION (1 or -1) as to
+ * whether or not to pad to the left.  Padding to the right is always
+ * appropriate, but need not go past the point indicated by STOP.
+ *
+ * If a continuation is returned, then the reentering call to
+ * a search function will retry the get_burst.
+ */
+
+#ifdef __STDC__
+typedef enum rx_get_burst_return
+  (*rx_get_burst_fn) (struct rx_string_position * pos,
+		      void * app_closure,
+		      int stop);
+					       
+#else
+typedef enum rx_get_burst_return (*rx_get_burst_fn) ();
+#endif
+
+
+enum rx_back_check_return
+{
+  rx_back_check_continuation,
+  rx_back_check_error,
+  rx_back_check_pass,
+  rx_back_check_fail
+};
+
+/* Back_check should advance the position it is passed 
+ * over rparen - lparen characters and return pass iff
+ * the characters starting at POS match those indexed
+ * by [LPAREN..RPAREN].
+ *
+ * If a continuation is returned, then the reentering call to
+ * a search function will retry the back_check.
+ */
+
+#ifdef __STDC__
+typedef enum rx_back_check_return
+  (*rx_back_check_fn) (struct rx_string_position * pos,
+		       int lparen,
+		       int rparen,
+		       unsigned char * translate,
+		       void * app_closure,
+		       int stop);
+					       
+#else
+typedef enum rx_back_check_return (*rx_back_check_fn) ();
+#endif
+
+
+
+
+/* A call to fetch_char should return the character at POS or POS + 1.
+ * Returning continuations here isn't supported.  OFFSET is either 0 or 1
+ * and indicates which characters is desired.
+ */
+
+#ifdef __STDC__
+typedef int (*rx_fetch_char_fn) (struct rx_string_position * pos,
+				 int offset,
+				 void * app_closure,
+				 int stop);
+#else
+typedef int (*rx_fetch_char_fn) ();
+#endif
+
+
+enum rx_search_return
+{
+  rx_search_continuation = -4,
+  rx_search_error = -3,
+  rx_search_soft_fail = -2,	/* failed by running out of string */
+  rx_search_fail = -1		/* failed only by reaching failure states */
+  /* return values >= 0 indicate the position of a successful match */
+};
+
+
+
+
+
+
+
+/* regex.h
+ * 
+ * The remaining declarations replace regex.h.
+ */
+
+/* This is an array of error messages corresponding to the error codes.
+ */
+extern __const__ char *re_error_msg[];
+
+/* If any error codes are removed, changed, or added, update the
+   `re_error_msg' table in regex.c.  */
+typedef enum
+{
+  REG_NOERROR = 0,	/* Success.  */
+  REG_NOMATCH,		/* Didn't find a match (for regexec).  */
+
+  /* POSIX regcomp return error codes.  (In the order listed in the
+     standard.)  */
+  REG_BADPAT,		/* Invalid pattern.  */
+  REG_ECOLLATE,		/* Not implemented.  */
+  REG_ECTYPE,		/* Invalid character class name.  */
+  REG_EESCAPE,		/* Trailing backslash.  */
+  REG_ESUBREG,		/* Invalid back reference.  */
+  REG_EBRACK,		/* Unmatched left bracket.  */
+  REG_EPAREN,		/* Parenthesis imbalance.  */ 
+  REG_EBRACE,		/* Unmatched \{.  */
+  REG_BADBR,		/* Invalid contents of \{\}.  */
+  REG_ERANGE,		/* Invalid range end.  */
+  REG_ESPACE,		/* Ran out of memory.  */
+  REG_BADRPT,		/* No preceding re for repetition op.  */
+
+  /* Error codes we've added.  */
+  REG_EEND,		/* Premature end.  */
+  REG_ESIZE,		/* Compiled pattern bigger than 2^16 bytes.  */
+  REG_ERPAREN		/* Unmatched ) or \); not returned from regcomp.  */
+} reg_errcode_t;
+
+/* The regex.c support, as a client of rx, defines a set of possible
+ * side effects that can be added to the edge lables of nfa edges.
+ * Here is the list of sidef effects in use.
+ */
+
+enum re_side_effects
+{
+#define RX_WANT_SE_DEFS 1
+#undef RX_DEF_SE
+#undef RX_DEF_CPLX_SE
+#define RX_DEF_SE(IDEM, NAME, VALUE)	      NAME VALUE,
+#define RX_DEF_CPLX_SE(IDEM, NAME, VALUE)     NAME VALUE,
+#include "rx.h"
+#undef RX_DEF_SE
+#undef RX_DEF_CPLX_SE
+#undef RX_WANT_SE_DEFS
+   re_floogle_flap = 65533
+};
+
+/* These hold paramaters for the kinds of side effects that are possible
+ * in the supported pattern languages.  These include things like the 
+ * numeric bounds of {} operators and the index of paren registers for 
+ * subexpression measurement or backreferencing.
+ */
+struct re_se_params
+{
+  enum re_side_effects se;
+  int op1;
+  int op2;
+};
+
+typedef unsigned reg_syntax_t;
+
+struct re_pattern_buffer
+{
+  struct rx rx;
+  reg_syntax_t syntax;		/* See below for syntax bit definitions. */
+
+  unsigned int no_sub:1;	/* If set, don't  return register offsets. */
+  unsigned int not_bol:1;	/* If set, the anchors ('^' and '$') don't */
+  unsigned int not_eol:1;	/*     match at the ends of the string.  */  
+  unsigned int newline_anchor:1;/* If true, an anchor at a newline matches.*/
+  unsigned int least_subs:1;	/* If set, and returning registers, return
+				 * as few values as possible.  Only 
+				 * backreferenced groups and group 0 (the whole
+				 * match) will be returned.
+				 */
+
+  /* If true, this says that the matcher should keep registers on its
+   * backtracking stack.  For many patterns, we can easily determine that
+   * this isn't necessary.
+   */
+  unsigned int match_regs_on_stack:1;
+  unsigned int search_regs_on_stack:1;
+
+  /* is_anchored and begbuf_only are filled in by rx_compile. */
+  unsigned int is_anchored:1;	/* Anchorded by ^? */
+  unsigned int begbuf_only:1;	/* Anchored to char position 0? */
+
+  
+  /* If REGS_UNALLOCATED, allocate space in the `regs' structure
+   * for `max (RE_NREGS, re_nsub + 1)' groups.
+   * If REGS_REALLOCATE, reallocate space if necessary.
+   * If REGS_FIXED, use what's there.  
+   */
+#define REGS_UNALLOCATED 0
+#define REGS_REALLOCATE 1
+#define REGS_FIXED 2
+  unsigned int regs_allocated:2;
+
+  
+  /* Either a translate table to apply to all characters before
+   * comparing them, or zero for no translation.  The translation
+   * is applied to a pattern when it is compiled and to a string
+   * when it is matched.
+   */
+  unsigned char * translate;
+
+  /* If this is a valid pointer, it tells rx not to store the extents of 
+   * certain subexpressions (those corresponding to non-zero entries).
+   * Passing 0x1 is the same as passing an array of all ones.  Passing 0x0
+   * is the same as passing an array of all zeros.
+   * The array should contain as many entries as their are subexps in the 
+   * regexp.
+   *
+   * For POSIX compatability, when using regcomp and regexec this field
+   * is zeroed and ignored.
+   */
+  char * syntax_parens;
+
+	/* Number of subexpressions found by the compiler.  */
+  size_t re_nsub;
+
+  void * buffer;		/* Malloced memory for the nfa. */
+  unsigned long allocated;	/* Size of that memory. */
+
+  /* Pointer to a fastmap, if any, otherwise zero.  re_search uses
+   * the fastmap, if there is one, to skip over impossible
+   * starting points for matches.  */
+  char *fastmap;
+
+  unsigned int fastmap_accurate:1; /* These three are internal. */
+  unsigned int can_match_empty:1;  
+  struct rx_nfa_state * start;	/* The nfa starting state. */
+
+  /* This is the list of iterator bounds for {lo,hi} constructs.
+   * The memory pointed to is part of the rx->buffer.
+   */
+  struct re_se_params *se_params;
+
+  /* This is a bitset representation of the fastmap.
+   * This is a true fastmap that already takes the translate
+   * table into account.
+   */
+  rx_Bitset fastset;
+};
+
+/* Type for byte offsets within the string.  POSIX mandates this.  */
+typedef int regoff_t;
+
+/* This is the structure we store register match data in.  See
+   regex.texinfo for a full description of what registers match.  */
+struct re_registers
+{
+  unsigned num_regs;
+  regoff_t *start;
+  regoff_t *end;
+};
+
+typedef struct re_pattern_buffer regex_t;
+
+/* POSIX specification for registers.  Aside from the different names than
+   `re_registers', POSIX uses an array of structures, instead of a
+   structure of arrays.  */
+typedef struct
+{
+  regoff_t rm_so;  /* Byte offset from string's start to substring's start.  */
+  regoff_t rm_eo;  /* Byte offset from string's start to substring's end.  */
+} regmatch_t;
+
+
+
+/* The following bits are used to determine the regexp syntax we
+   recognize.  The set/not-set meanings are chosen so that Emacs syntax
+   remains the value 0.  The bits are given in alphabetical order, and
+   the definitions shifted by one from the previous bit; thus, when we
+   add or remove a bit, only one other definition need change.  */
+
+/* If this bit is not set, then \ inside a bracket expression is literal.
+   If set, then such a \ quotes the following character.  */
+#define RE_BACKSLASH_ESCAPE_IN_LISTS (1)
+
+/* If this bit is not set, then + and ? are operators, and \+ and \? are
+     literals. 
+   If set, then \+ and \? are operators and + and ? are literals.  */
+#define RE_BK_PLUS_QM (RE_BACKSLASH_ESCAPE_IN_LISTS << 1)
+
+/* If this bit is set, then character classes are supported.  They are:
+     [:alpha:], [:upper:], [:lower:],  [:digit:], [:alnum:], [:xdigit:],
+     [:space:], [:print:], [:punct:], [:graph:], and [:cntrl:].
+   If not set, then character classes are not supported.  */
+#define RE_CHAR_CLASSES (RE_BK_PLUS_QM << 1)
+
+/* If this bit is set, then ^ and $ are always anchors (outside bracket
+     expressions, of course).
+   If this bit is not set, then it depends:
+        ^  is an anchor if it is at the beginning of a regular
+           expression or after an open-group or an alternation operator;
+        $  is an anchor if it is at the end of a regular expression, or
+           before a close-group or an alternation operator.  
+
+   This bit could be (re)combined with RE_CONTEXT_INDEP_OPS, because
+   POSIX draft 11.2 says that * etc. in leading positions is undefined.
+   We already implemented a previous draft which made those constructs
+   invalid, though, so we haven't changed the code back.  */
+#define RE_CONTEXT_INDEP_ANCHORS (RE_CHAR_CLASSES << 1)
+
+/* If this bit is set, then special characters are always special
+     regardless of where they are in the pattern.
+   If this bit is not set, then special characters are special only in
+     some contexts; otherwise they are ordinary.  Specifically, 
+     * + ? and intervals are only special when not after the beginning,
+     open-group, or alternation operator.  */
+#define RE_CONTEXT_INDEP_OPS (RE_CONTEXT_INDEP_ANCHORS << 1)
+
+/* If this bit is set, then *, +, ?, and { cannot be first in an re or
+     immediately after an alternation or begin-group operator.  */
+#define RE_CONTEXT_INVALID_OPS (RE_CONTEXT_INDEP_OPS << 1)
+
+/* If this bit is set, then . matches newline.
+   If not set, then it doesn't.  */
+#define RE_DOT_NEWLINE (RE_CONTEXT_INVALID_OPS << 1)
+
+/* If this bit is set, then . doesn't match NUL.
+   If not set, then it does.  */
+#define RE_DOT_NOT_NULL (RE_DOT_NEWLINE << 1)
+
+/* If this bit is set, nonmatching lists [^...] do not match newline.
+   If not set, they do.  */
+#define RE_HAT_LISTS_NOT_NEWLINE (RE_DOT_NOT_NULL << 1)
+
+/* If this bit is set, either \{...\} or {...} defines an
+     interval, depending on RE_NO_BK_BRACES. 
+   If not set, \{, \}, {, and } are literals.  */
+#define RE_INTERVALS (RE_HAT_LISTS_NOT_NEWLINE << 1)
+
+/* If this bit is set, +, ? and | aren't recognized as operators.
+   If not set, they are.  */
+#define RE_LIMITED_OPS (RE_INTERVALS << 1)
+
+/* If this bit is set, newline is an alternation operator.
+   If not set, newline is literal.  */
+#define RE_NEWLINE_ALT (RE_LIMITED_OPS << 1)
+
+/* If this bit is set, then `{...}' defines an interval, and \{ and \}
+     are literals.
+  If not set, then `\{...\}' defines an interval.  */
+#define RE_NO_BK_BRACES (RE_NEWLINE_ALT << 1)
+
+/* If this bit is set, (...) defines a group, and \( and \) are literals.
+   If not set, \(...\) defines a group, and ( and ) are literals.  */
+#define RE_NO_BK_PARENS (RE_NO_BK_BRACES << 1)
+
+/* If this bit is set, then \<digit> matches <digit>.
+   If not set, then \<digit> is a back-reference.  */
+#define RE_NO_BK_REFS (RE_NO_BK_PARENS << 1)
+
+/* If this bit is set, then | is an alternation operator, and \| is literal. 
+   If not set, then \| is an alternation operator, and | is literal.  */
+#define RE_NO_BK_VBAR (RE_NO_BK_REFS << 1)
+
+/* If this bit is set, then an ending range point collating higher
+     than the starting range point, as in [z-a], is invalid.
+   If not set, then when ending range point collates higher than the
+     starting range point, the range is ignored.  */
+#define RE_NO_EMPTY_RANGES (RE_NO_BK_VBAR << 1)
+
+/* If this bit is set, then an unmatched ) is ordinary.
+   If not set, then an unmatched ) is invalid.  */
+#define RE_UNMATCHED_RIGHT_PAREN_ORD (RE_NO_EMPTY_RANGES << 1)
+
+/* This global variable defines the particular regexp syntax to use (for
+   some interfaces).  When a regexp is compiled, the syntax used is
+   stored in the pattern buffer, so changing this does not affect
+   already-compiled regexps.  */
+extern reg_syntax_t re_syntax_options;
+
+
+/* Define combinations of the above bits for the standard possibilities.
+   (The [[[ comments delimit what gets put into the Texinfo file, so
+   don't delete them!)  */ 
+/* [[[begin syntaxes]]] */
+#define RE_SYNTAX_EMACS 0
+
+#define RE_SYNTAX_AWK							\
+  (RE_BACKSLASH_ESCAPE_IN_LISTS | RE_DOT_NOT_NULL			\
+   | RE_NO_BK_PARENS            | RE_NO_BK_REFS				\
+   | RE_NO_BK_VAR               | RE_NO_EMPTY_RANGES			\
+   | RE_UNMATCHED_RIGHT_PAREN_ORD)
+
+#define RE_SYNTAX_POSIX_AWK 						\
+  (RE_SYNTAX_POSIX_EXTENDED | RE_BACKSLASH_ESCAPE_IN_LISTS)
+
+#define RE_SYNTAX_GREP							\
+  (RE_BK_PLUS_QM              | RE_CHAR_CLASSES				\
+   | RE_HAT_LISTS_NOT_NEWLINE | RE_INTERVALS				\
+   | RE_NEWLINE_ALT)
+
+#define RE_SYNTAX_EGREP							\
+  (RE_CHAR_CLASSES        | RE_CONTEXT_INDEP_ANCHORS			\
+   | RE_CONTEXT_INDEP_OPS | RE_HAT_LISTS_NOT_NEWLINE			\
+   | RE_NEWLINE_ALT       | RE_NO_BK_PARENS				\
+   | RE_NO_BK_VBAR)
+
+#define RE_SYNTAX_POSIX_EGREP						\
+  (RE_SYNTAX_EGREP | RE_INTERVALS | RE_NO_BK_BRACES)
+
+#define RE_SYNTAX_SED RE_SYNTAX_POSIX_BASIC
+
+/* Syntax bits common to both basic and extended POSIX regex syntax.  */
+#define _RE_SYNTAX_POSIX_COMMON						\
+  (RE_CHAR_CLASSES | RE_DOT_NEWLINE      | RE_DOT_NOT_NULL		\
+   | RE_INTERVALS  | RE_NO_EMPTY_RANGES)
+
+#define RE_SYNTAX_POSIX_BASIC						\
+  (_RE_SYNTAX_POSIX_COMMON | RE_BK_PLUS_QM)
+
+/* Differs from ..._POSIX_BASIC only in that RE_BK_PLUS_QM becomes
+   RE_LIMITED_OPS, i.e., \? \+ \| are not recognized.  Actually, this
+   isn't minimal, since other operators, such as \`, aren't disabled.  */
+#define RE_SYNTAX_POSIX_MINIMAL_BASIC					\
+  (_RE_SYNTAX_POSIX_COMMON | RE_LIMITED_OPS)
+
+#define RE_SYNTAX_POSIX_EXTENDED					\
+  (_RE_SYNTAX_POSIX_COMMON | RE_CONTEXT_INDEP_ANCHORS			\
+   | RE_CONTEXT_INDEP_OPS  | RE_NO_BK_BRACES				\
+   | RE_NO_BK_PARENS       | RE_NO_BK_VBAR				\
+   | RE_UNMATCHED_RIGHT_PAREN_ORD)
+
+/* Differs from ..._POSIX_EXTENDED in that RE_CONTEXT_INVALID_OPS
+   replaces RE_CONTEXT_INDEP_OPS and RE_NO_BK_REFS is added.  */
+#define RE_SYNTAX_POSIX_MINIMAL_EXTENDED				\
+  (_RE_SYNTAX_POSIX_COMMON  | RE_CONTEXT_INDEP_ANCHORS			\
+   | RE_CONTEXT_INVALID_OPS | RE_NO_BK_BRACES				\
+   | RE_NO_BK_PARENS        | RE_NO_BK_REFS				\
+   | RE_NO_BK_VBAR	    | RE_UNMATCHED_RIGHT_PAREN_ORD)
+/* [[[end syntaxes]]] */
+
+/* Maximum number of duplicates an interval can allow.  Some systems
+   (erroneously) define this in other header files, but we want our
+   value, so remove any previous define.  */
+#ifdef RE_DUP_MAX
+#undef RE_DUP_MAX
+#endif
+#define RE_DUP_MAX ((1 << 15) - 1) 
+
+
+
+/* POSIX `cflags' bits (i.e., information for `regcomp').  */
+
+/* If this bit is set, then use extended regular expression syntax.
+   If not set, then use basic regular expression syntax.  */
+#define REG_EXTENDED 1
+
+/* If this bit is set, then ignore case when matching.
+   If not set, then case is significant.  */
+#define REG_ICASE (REG_EXTENDED << 1)
+ 
+/* If this bit is set, then anchors do not match at newline
+     characters in the string.
+   If not set, then anchors do match at newlines.  */
+#define REG_NEWLINE (REG_ICASE << 1)
+
+/* If this bit is set, then report only success or fail in regexec.
+   If not set, then returns differ between not matching and errors.  */
+#define REG_NOSUB (REG_NEWLINE << 1)
+
+
+/* POSIX `eflags' bits (i.e., information for regexec).  */
+
+/* If this bit is set, then the beginning-of-line operator doesn't match
+     the beginning of the string (presumably because it's not the
+     beginning of a line).
+   If not set, then the beginning-of-line operator does match the
+     beginning of the string.  */
+#define REG_NOTBOL 1
+
+/* Like REG_NOTBOL, except for the end-of-line.  */
+#define REG_NOTEOL (1 << 1)
+
+/* If `regs_allocated' is REGS_UNALLOCATED in the pattern buffer,
+ * `re_match_2' returns information about at least this many registers
+ * the first time a `regs' structure is passed. 
+ *
+ * Also, this is the greatest number of backreferenced subexpressions
+ * allowed in a pattern being matched without caller-supplied registers.
+ */
+#ifndef RE_NREGS
+#define RE_NREGS 30
+#endif
+
+extern int rx_cache_bound;
+extern char rx_version_string[];
+
+
+
+
+#ifdef RX_WANT_RX_DEFS
+
+/* This is decls to the interesting subsystems and lower layers
+ * of rx.  Everything which doesn't have a public counterpart in 
+ * regex.c is declared here.
+ */
+
+
+#ifdef __STDC__
+typedef void (*rx_hash_freefn) (struct rx_hash_item * it);
+#else /* ndef __STDC__ */
+typedef void (*rx_hash_freefn) ();
+#endif /* ndef __STDC__ */
+
+
+
+
+
+#ifdef __STDC__
+RX_DECL int rx_bitset_is_equal (int size, rx_Bitset a, rx_Bitset b);
+RX_DECL int rx_bitset_is_subset (int size, rx_Bitset a, rx_Bitset b);
+RX_DECL int rx_bitset_empty (int size, rx_Bitset set);
+RX_DECL void rx_bitset_null (int size, rx_Bitset b);
+RX_DECL void rx_bitset_universe (int size, rx_Bitset b);
+RX_DECL void rx_bitset_complement (int size, rx_Bitset b);
+RX_DECL void rx_bitset_assign (int size, rx_Bitset a, rx_Bitset b);
+RX_DECL void rx_bitset_union (int size, rx_Bitset a, rx_Bitset b);
+RX_DECL void rx_bitset_intersection (int size,
+				     rx_Bitset a, rx_Bitset b);
+RX_DECL void rx_bitset_difference (int size, rx_Bitset a, rx_Bitset b);
+RX_DECL void rx_bitset_revdifference (int size,
+				      rx_Bitset a, rx_Bitset b);
+RX_DECL void rx_bitset_xor (int size, rx_Bitset a, rx_Bitset b);
+RX_DECL unsigned long rx_bitset_hash (int size, rx_Bitset b);
+RX_DECL struct rx_hash_item * rx_hash_find (struct rx_hash * table,
+					    unsigned long hash,
+					    void * value,
+					    struct rx_hash_rules * rules);
+RX_DECL struct rx_hash_item * rx_hash_store (struct rx_hash * table,
+					     unsigned long hash,
+					     void * value,
+					     struct rx_hash_rules * rules);
+RX_DECL void rx_hash_free (struct rx_hash_item * it, struct rx_hash_rules * rules);
+RX_DECL void rx_free_hash_table (struct rx_hash * tab, rx_hash_freefn freefn,
+				 struct rx_hash_rules * rules);
+RX_DECL rx_Bitset rx_cset (struct rx *rx);
+RX_DECL rx_Bitset rx_copy_cset (struct rx *rx, rx_Bitset a);
+RX_DECL void rx_free_cset (struct rx * rx, rx_Bitset c);
+RX_DECL struct rexp_node * rexp_node (struct rx *rx,
+				      enum rexp_node_type type);
+RX_DECL struct rexp_node * rx_mk_r_cset (struct rx * rx,
+					 rx_Bitset b);
+RX_DECL struct rexp_node * rx_mk_r_concat (struct rx * rx,
+					   struct rexp_node * a,
+					   struct rexp_node * b);
+RX_DECL struct rexp_node * rx_mk_r_alternate (struct rx * rx,
+					      struct rexp_node * a,
+					      struct rexp_node * b);
+RX_DECL struct rexp_node * rx_mk_r_opt (struct rx * rx,
+					struct rexp_node * a);
+RX_DECL struct rexp_node * rx_mk_r_star (struct rx * rx,
+					 struct rexp_node * a);
+RX_DECL struct rexp_node * rx_mk_r_2phase_star (struct rx * rx,
+						struct rexp_node * a,
+						struct rexp_node * b);
+RX_DECL struct rexp_node * rx_mk_r_side_effect (struct rx * rx,
+						rx_side_effect a);
+RX_DECL struct rexp_node * rx_mk_r_data  (struct rx * rx,
+					  void * a);
+RX_DECL void rx_free_rexp (struct rx * rx, struct rexp_node * node);
+RX_DECL struct rexp_node * rx_copy_rexp (struct rx *rx,
+					 struct rexp_node *node);
+RX_DECL struct rx_nfa_state * rx_nfa_state (struct rx *rx);
+RX_DECL void rx_free_nfa_state (struct rx_nfa_state * n);
+RX_DECL struct rx_nfa_state * rx_id_to_nfa_state (struct rx * rx,
+						  int id);
+RX_DECL struct rx_nfa_edge * rx_nfa_edge (struct rx *rx,
+					  enum rx_nfa_etype type,
+					  struct rx_nfa_state *start,
+					  struct rx_nfa_state *dest);
+RX_DECL void rx_free_nfa_edge (struct rx_nfa_edge * e);
+RX_DECL void rx_free_nfa (struct rx *rx);
+RX_DECL int rx_build_nfa (struct rx *rx,
+			  struct rexp_node *rexp,
+			  struct rx_nfa_state **start,
+			  struct rx_nfa_state **end);
+RX_DECL void rx_name_nfa_states (struct rx *rx);
+RX_DECL int rx_eclose_nfa (struct rx *rx);
+RX_DECL void rx_delete_epsilon_transitions (struct rx *rx);
+RX_DECL int rx_compactify_nfa (struct rx *rx,
+			       void **mem, unsigned long *size);
+RX_DECL void rx_release_superset (struct rx *rx,
+				  struct rx_superset *set);
+RX_DECL struct rx_superset * rx_superset_cons (struct rx * rx,
+					       struct rx_nfa_state *car, struct rx_superset *cdr);
+RX_DECL struct rx_superset * rx_superstate_eclosure_union
+  (struct rx * rx, struct rx_superset *set, struct rx_nfa_state_set *ecl);
+RX_DECL struct rx_superstate * rx_superstate (struct rx *rx,
+					      struct rx_superset *set);
+RX_DECL struct rx_inx * rx_handle_cache_miss
+  (struct rx *rx, struct rx_superstate *super, unsigned char chr, void *data);
+RX_DECL reg_errcode_t rx_compile (__const__ char *pattern, int size,
+				  reg_syntax_t syntax,
+				  struct re_pattern_buffer * rxb);
+RX_DECL void rx_blow_up_fastmap (struct re_pattern_buffer * rxb);
+#else /* STDC */
+RX_DECL int rx_bitset_is_equal ();
+RX_DECL int rx_bitset_is_subset ();
+RX_DECL int rx_bitset_empty ();
+RX_DECL void rx_bitset_null ();
+RX_DECL void rx_bitset_universe ();
+RX_DECL void rx_bitset_complement ();
+RX_DECL void rx_bitset_assign ();
+RX_DECL void rx_bitset_union ();
+RX_DECL void rx_bitset_intersection ();
+RX_DECL void rx_bitset_difference ();
+RX_DECL void rx_bitset_revdifference ();
+RX_DECL void rx_bitset_xor ();
+RX_DECL unsigned long rx_bitset_hash ();
+RX_DECL struct rx_hash_item * rx_hash_find ();
+RX_DECL struct rx_hash_item * rx_hash_store ();
+RX_DECL void rx_hash_free ();
+RX_DECL void rx_free_hash_table ();
+RX_DECL rx_Bitset rx_cset ();
+RX_DECL rx_Bitset rx_copy_cset ();
+RX_DECL void rx_free_cset ();
+RX_DECL struct rexp_node * rexp_node ();
+RX_DECL struct rexp_node * rx_mk_r_cset ();
+RX_DECL struct rexp_node * rx_mk_r_concat ();
+RX_DECL struct rexp_node * rx_mk_r_alternate ();
+RX_DECL struct rexp_node * rx_mk_r_opt ();
+RX_DECL struct rexp_node * rx_mk_r_star ();
+RX_DECL struct rexp_node * rx_mk_r_2phase_star ();
+RX_DECL struct rexp_node * rx_mk_r_side_effect ();
+RX_DECL struct rexp_node * rx_mk_r_data  ();
+RX_DECL void rx_free_rexp ();
+RX_DECL struct rexp_node * rx_copy_rexp ();
+RX_DECL struct rx_nfa_state * rx_nfa_state ();
+RX_DECL void rx_free_nfa_state ();
+RX_DECL struct rx_nfa_state * rx_id_to_nfa_state ();
+RX_DECL struct rx_nfa_edge * rx_nfa_edge ();
+RX_DECL void rx_free_nfa_edge ();
+RX_DECL void rx_free_nfa ();
+RX_DECL int rx_build_nfa ();
+RX_DECL void rx_name_nfa_states ();
+RX_DECL int rx_eclose_nfa ();
+RX_DECL void rx_delete_epsilon_transitions ();
+RX_DECL int rx_compactify_nfa ();
+RX_DECL void rx_release_superset ();
+RX_DECL struct rx_superset * rx_superset_cons ();
+RX_DECL struct rx_superset * rx_superstate_eclosure_union ();
+RX_DECL struct rx_superstate * rx_superstate ();
+RX_DECL struct rx_inx * rx_handle_cache_miss ();
+RX_DECL reg_errcode_t rx_compile ();
+RX_DECL void rx_blow_up_fastmap ();
+#endif /* STDC */
+
+
+#endif /* RX_WANT_RX_DEFS */
+
+
+
+
+#ifdef __STDC__
+extern int re_search_2 (struct re_pattern_buffer *rxb,
+			__const__ char * string1, int size1,
+			__const__ char * string2, int size2,
+			int startpos, int range,
+			struct re_registers *regs,
+			int stop);
+extern int re_search (struct re_pattern_buffer * rxb, __const__ char *string,
+		      int size, int startpos, int range,
+		      struct re_registers *regs);
+extern int re_match_2 (struct re_pattern_buffer * rxb,
+		       __const__ char * string1, int size1,
+		       __const__ char * string2, int size2,
+		       int pos, struct re_registers *regs, int stop);
+extern int re_match (struct re_pattern_buffer * rxb,
+		     __const__ char * string,
+		     int size, int pos,
+		     struct re_registers *regs);
+extern reg_syntax_t re_set_syntax (reg_syntax_t syntax);
+extern void re_set_registers (struct re_pattern_buffer *bufp,
+			      struct re_registers *regs,
+			      unsigned num_regs,
+			      regoff_t * starts, regoff_t * ends);
+extern __const__ char * re_compile_pattern (__const__ char *pattern,
+					int length,
+					struct re_pattern_buffer * rxb);
+extern int re_compile_fastmap (struct re_pattern_buffer * rxb);
+extern char * re_comp (__const__ char *s);
+extern int re_exec (__const__ char *s);
+extern int regcomp (regex_t * preg, __const__ char * pattern, int cflags);
+extern int regexec (__const__ regex_t *preg, __const__ char *string,
+		    size_t nmatch, regmatch_t pmatch[],
+		    int eflags);
+extern size_t regerror (int errcode, __const__ regex_t *preg,
+			char *errbuf, size_t errbuf_size);
+extern void regfree (regex_t *preg);
+
+#else /* STDC */
+extern int re_search_2 ();
+extern int re_search ();
+extern int re_match_2 ();
+extern int re_match ();
+extern reg_syntax_t re_set_syntax ();
+extern void re_set_registers ();
+extern __const__ char * re_compile_pattern ();
+extern int re_compile_fastmap ();
+extern char * re_comp ();
+extern int re_exec ();
+extern int regcomp ();
+extern int regexec ();
+extern size_t regerror ();
+extern void regfree ();
+
+#endif /* STDC */
+
+
+
+
+#ifdef RX_WANT_RX_DEFS
+
+struct rx_counter_frame
+{
+  int tag;
+  int val;
+  struct rx_counter_frame * inherited_from; /* If this is a copy. */
+  struct rx_counter_frame * cdr;
+};
+
+struct rx_backtrack_frame
+{
+  char * counter_stack_sp;
+
+  /* A frame is used to save the matchers state when it crosses a 
+   * backtracking point.  The `stk_' fields correspond to variables
+   * in re_search_2 (just strip off thes `stk_').  They are documented
+   * tere.
+   */
+  struct rx_superstate * stk_super;
+  unsigned int stk_c;
+  struct rx_string_position stk_test_pos;
+  int stk_last_l;
+  int stk_last_r;
+  int stk_test_ret;
+
+  /* This is the list of options left to explore at the backtrack
+   * point for which this frame was created. 
+   */
+  struct rx_distinct_future * df;
+  struct rx_distinct_future * first_df;
+
+#ifdef RX_DEBUG
+   int stk_line_no;
+#endif
+};
+
+struct rx_stack_chunk
+{
+  struct rx_stack_chunk * next_chunk;
+  int bytes_left;
+  char * sp;
+};
+
+enum rx_outer_entry
+{
+  rx_outer_start,
+  rx_outer_fastmap,
+  rx_outer_test,
+  rx_outer_restore_pos
+};
+
+enum rx_fastmap_return
+{
+  rx_fastmap_continuation,
+  rx_fastmap_error,
+  rx_fastmap_ok,
+  rx_fastmap_fail
+};
+
+enum rx_fastmap_entry
+{
+  rx_fastmap_start,
+  rx_fastmap_string_break
+};
+
+enum rx_test_return
+{
+  rx_test_continuation,
+  rx_test_error,
+  rx_test_fail,
+  rx_test_ok
+};
+
+enum rx_test_internal_return
+{
+  rx_test_internal_error,
+  rx_test_found_first,
+  rx_test_line_finished
+};
+
+enum rx_test_match_entry
+{
+  rx_test_start,
+  rx_test_cache_hit_loop,
+  rx_test_backreference_check,
+  rx_test_backtrack_return
+};
+
+struct rx_search_state
+{
+  /* Two groups of registers are kept.  The group with the register state
+   * of the current test match, and the group that holds the state at the end
+   * of the best known match, if any.
+   *
+   * For some patterns, there may also be registers saved on the stack.
+   */
+  unsigned num_regs;		/* Includes an element for register zero. */
+  regoff_t * lparen;		/* scratch space for register returns */
+  regoff_t * rparen;
+  regoff_t * best_lpspace;	/* in case the user doesn't want these */
+  regoff_t * best_rpspace;	/* values, we still need space to store
+				 * them.  Normally, this memoryis unused
+				 * and the space pointed to by REGS is 
+				 * used instead.
+				 */
+  
+  int last_l;			/* Highest index of a valid lparen. */
+  int last_r;			/* It's dual. */
+  
+  int * best_lparen;		/* This contains the best known register */
+  int * best_rparen;		/* assignments. 
+				 * This may point to the same mem as
+				 * best_lpspace, or it might point to memory
+				 * passed by the caller.
+				 */
+  int best_last_l;		/* best_last_l:best_lparen::last_l:lparen */
+  int best_last_r;
+
+
+  unsigned char * translate;  
+
+  struct rx_string_position outer_pos;
+
+  struct rx_superstate * start_super;
+  int nfa_choice;
+  int first_found;		/* If true, return after finding any match. */
+  int ret_val;
+
+  /* For continuations... */
+  enum rx_outer_entry outer_search_resume_pt;
+  struct re_pattern_buffer * saved_rxb;
+  int saved_startpos;
+  int saved_range;
+  int saved_stop;
+  int saved_total_size;
+  rx_get_burst_fn saved_get_burst;
+  rx_back_check_fn saved_back_check;
+  struct re_registers * saved_regs;
+  
+  /**
+   ** state for fastmap
+   **/
+  char * fastmap;
+  int fastmap_chr;
+  int fastmap_val;
+
+  /* for continuations in the fastmap procedure: */
+  enum rx_fastmap_entry fastmap_resume_pt;
+
+  /**
+   ** state for test_match 
+   **/
+
+  /* The current superNFA position of the matcher. */
+  struct rx_superstate * super;
+  
+  /* The matcher interprets a series of instruction frames.
+   * This is the `instruction counter' for the interpretation.
+   */
+  struct rx_inx * ifr;
+  
+  /* We insert a ghost character in the string to prime
+   * the nfa.  test_pos.pos, test_pos.str_half, and test_pos.end_half
+   * keep track of the test-match position and string-half.
+   */
+  unsigned char c;
+  
+  /* Position within the string. */
+  struct rx_string_position test_pos;
+
+  struct rx_stack_chunk * counter_stack;
+  struct rx_stack_chunk * backtrack_stack;
+  int backtrack_frame_bytes;
+  int chunk_bytes;
+  struct rx_stack_chunk * free_chunks;
+
+  /* To return from this function, set test_ret and 
+   * `goto test_do_return'.
+   *
+   * Possible return values are:
+   *     1   --- end of string while the superNFA is still going
+   *     0   --- internal error (out of memory)
+   *	-1   --- search completed by reaching the superNFA fail state
+   *    -2   --- a match was found, maybe not the longest.
+   *
+   * When the search is complete (-1), best_last_r indicates whether
+   * a match was found.
+   *
+   * -2 is return only if search_state.first_found is non-zero.
+   *
+   * if search_state.first_found is non-zero, a return of -1 indicates no match,
+   * otherwise, best_last_r has to be checked.
+   */
+  int test_ret;
+
+  int could_have_continued;
+  
+#ifdef RX_DEBUG
+  int backtrack_depth;
+  /* There is a search tree with every node as set of deterministic
+   * transitions in the super nfa.  For every branch of a 
+   * backtrack point is an edge in the tree.
+   * This counts up a pre-order of nodes in that tree.
+   * It's saved on the search stack and printed when debugging. 
+   */
+  int line_no;
+  int lines_found;
+#endif
+
+
+  /* For continuations within the match tester */
+  enum rx_test_match_entry test_match_resume_pt;
+  struct rx_inx * saved_next_tr_table;
+  struct rx_inx * saved_this_tr_table;
+  int saved_reg;
+  struct rx_backtrack_frame * saved_bf;
+  
+};
+
+
+
+extern char rx_slowmap[];
+extern unsigned char rx_id_translation[];
+
+static __inline__ void
+init_fastmap (rxb, search_state)
+     struct re_pattern_buffer * rxb;
+     struct rx_search_state * search_state;
+{
+  search_state->fastmap = (rxb->fastmap
+			   ? (char *)rxb->fastmap
+			   : (char *)rx_slowmap);
+  /* Update the fastmap now if not correct already. 
+   * When the regexp was compiled, the fastmap was computed
+   * and stored in a bitset.  This expands the bitset into a
+   * character array containing 1s and 0s.
+   */
+  if ((search_state->fastmap == rxb->fastmap) && !rxb->fastmap_accurate)
+    rx_blow_up_fastmap (rxb);
+  search_state->fastmap_chr = -1;
+  search_state->fastmap_val = 0;
+  search_state->fastmap_resume_pt = rx_fastmap_start;
+}
+
+static __inline__ void
+uninit_fastmap (rxb, search_state)
+     struct re_pattern_buffer * rxb;
+     struct rx_search_state * search_state;
+{
+  /* Unset the fastmap sentinel */
+  if (search_state->fastmap_chr >= 0)
+    search_state->fastmap[search_state->fastmap_chr]
+      = search_state->fastmap_val;
+}
+
+static __inline__ int
+fastmap_search (rxb, stop, get_burst, app_closure, search_state)
+     struct re_pattern_buffer * rxb;
+     int stop;
+     rx_get_burst_fn get_burst;
+     void * app_closure;
+     struct rx_search_state * search_state;
+{
+  enum rx_fastmap_entry pc;
+
+  if (0)
+    {
+    return_continuation:
+      search_state->fastmap_resume_pt = pc;
+      return rx_fastmap_continuation;
+    }
+
+  pc = search_state->fastmap_resume_pt;
+
+  switch (pc)
+    {
+    default:
+      return rx_fastmap_error;
+    case rx_fastmap_start:
+    init_fastmap_sentinal:
+      /* For the sake of fast fastmapping, set a sentinal in the fastmap.
+       * This sentinal will trap the fastmap loop when it reaches the last
+       * valid character in a string half.
+       *
+       * This must be reset when the fastmap/search loop crosses a string 
+       * boundry, and before returning to the caller.  So sometimes,
+       * the fastmap loop is restarted with `continue', othertimes by
+       * `goto init_fastmap_sentinal'.
+       */
+      if (search_state->outer_pos.size)
+	{
+	  search_state->fastmap_chr = ((search_state->outer_pos.search_direction == 1)
+				       ? *(search_state->outer_pos.end - 1)
+				       : *search_state->outer_pos.string);
+	  search_state->fastmap_val
+	    = search_state->fastmap[search_state->fastmap_chr];
+	  search_state->fastmap[search_state->fastmap_chr] = 1;
+	}
+      else
+	{
+	  search_state->fastmap_chr = -1;
+	  search_state->fastmap_val = 0;
+	}
+      
+      if (search_state->outer_pos.pos >= search_state->outer_pos.end)
+	goto fastmap_hit_bound;
+      else
+	{
+	  if (search_state->outer_pos.search_direction == 1)
+	    {
+	      if (search_state->fastmap_val)
+		{
+		  for (;;)
+		    {
+		      while (!search_state->fastmap[*search_state->outer_pos.pos])
+			++search_state->outer_pos.pos;
+		      return rx_fastmap_ok;
+		    }
+		}
+	      else
+		{
+		  for (;;)
+		    {
+		      while (!search_state->fastmap[*search_state->outer_pos.pos])
+			++search_state->outer_pos.pos;
+		      if (*search_state->outer_pos.pos != search_state->fastmap_chr)
+			return rx_fastmap_ok;
+		      else 
+			{
+			  ++search_state->outer_pos.pos;
+			  if (search_state->outer_pos.pos == search_state->outer_pos.end)
+			    goto fastmap_hit_bound;
+			}
+		    }
+		}
+	    }
+	  else
+	    {
+	      __const__ unsigned char * bound;
+	      bound = search_state->outer_pos.string - 1;
+	      if (search_state->fastmap_val)
+		{
+		  for (;;)
+		    {
+		      while (!search_state->fastmap[*search_state->outer_pos.pos])
+			--search_state->outer_pos.pos;
+		      return rx_fastmap_ok;
+		    }
+		}
+	      else
+		{
+		  for (;;)
+		    {
+		      while (!search_state->fastmap[*search_state->outer_pos.pos])
+			--search_state->outer_pos.pos;
+		      if ((*search_state->outer_pos.pos != search_state->fastmap_chr) || search_state->fastmap_val)
+			return rx_fastmap_ok;
+		      else 
+			{
+			  --search_state->outer_pos.pos;
+			  if (search_state->outer_pos.pos == bound)
+			    goto fastmap_hit_bound;
+			}
+		    }
+		}
+	    }
+	}
+      
+    case rx_fastmap_string_break:
+    fastmap_hit_bound:
+      {
+	/* If we hit a bound, it may be time to fetch another burst
+	 * of string, or it may be time to return a continuation to 
+ 	 * the caller, or it might be time to fail.
+	 */
+
+	int burst_state;
+	burst_state = get_burst (&search_state->outer_pos, app_closure, stop);
+	switch (burst_state)
+	  {
+	  default:
+	  case rx_get_burst_error:
+	    return rx_fastmap_error;
+	  case rx_get_burst_continuation:
+	    {
+	      pc = rx_fastmap_string_break;
+	      goto return_continuation;
+	    }
+	  case rx_get_burst_ok:
+	    goto init_fastmap_sentinal;
+	  case rx_get_burst_no_more:
+	    /* ...not a string split, simply no more string. 
+	     *
+	     * When searching backward, running out of string
+	     * is reason to quit.
+	     *
+	     * When searching forward, we allow the possibility
+	     * of an (empty) match after the last character in the
+	     * virtual string.  So, fall through to the matcher
+	     */
+	    return (  (search_state->outer_pos.search_direction == 1)
+		    ? rx_fastmap_ok
+		    : rx_fastmap_fail);
+	  }
+      }
+    }
+
+}
+
+
+
+
+#ifdef emacs
+/* The `emacs' switch turns on certain matching commands
+ * that make sense only in Emacs. 
+ */
+#include "sysfuncs.h"
+#include "lisp.h"
+#include "buffer.h"
+#include "syntax.h"
+#endif /* emacs */
+
+/* Setting RX_MEMDBUG is useful if you have dbmalloc.  Maybe with similar
+ * packages too.
+ */
+#ifdef RX_MEMDBUG
+#include <malloc.h>
+#endif /* RX_RX_MEMDBUG */
+
+/* We used to test for `BSTRING' here, but only GCC and Emacs define
+ * `BSTRING', as far as I know, and neither of them use this code.  
+ */
+#if HAVE_STRING_H || STDC_HEADERS
+#include <string.h>
+
+#ifndef bcmp
+#define bcmp(s1, s2, n)	memcmp ((s1), (s2), (n))
+#endif
+
+#ifndef bcopy
+#define bcopy(s, d, n)	memcpy ((d), (s), (n))
+#endif
+
+#ifndef bzero
+#define bzero(s, n)	memset ((s), 0, (n))
+#endif
+
+#else /*  HAVE_STRING_H || STDC_HEADERS */
+#include <strings.h>
+#endif   /* not (HAVE_STRING_H || STDC_HEADERS) */
+
+#ifdef STDC_HEADERS
+#include <stdlib.h>
+#else /* not STDC_HEADERS */
+char *malloc ();
+char *realloc ();
+#endif /* not STDC_HEADERS */
+
+
+
+
+
+/* How many characters in the character set.  */
+#define CHAR_SET_SIZE (1 << CHARBITS)
+
+#ifndef emacs
+/* Define the syntax basics for \<, \>, etc.
+ * This must be nonzero for the wordchar and notwordchar pattern
+ * commands in re_match_2.
+ */
+#ifndef Sword 
+#define Sword 1
+#endif
+#define SYNTAX(c) re_syntax_table[c]
+#ifdef SYNTAX_TABLE
+extern char *re_syntax_table;
+#else
+RX_DECL char re_syntax_table[CHAR_SET_SIZE];
+#endif
+#endif /* not emacs */
+
+
+/* Test if at very beginning or at very end of the virtual concatenation
+ *  of `string1' and `string2'.  If only one string, it's `string2'.  
+ */
+
+#define AT_STRINGS_BEG() \
+  (   -1		 \
+   == ((search_state.test_pos.pos - search_state.test_pos.string) \
+       + search_state.test_pos.offset))
+
+#define AT_STRINGS_END() \
+  (   (total_size - 1)	 \
+   == ((search_state.test_pos.pos - search_state.test_pos.string) \
+       + search_state.test_pos.offset))
+
+
+/* Test if POS + 1 points to a character which is word-constituent.  We have
+ * two special cases to check for: if past the end of string1, look at
+ * the first character in string2; and if before the beginning of
+ * string2, look at the last character in string1.
+ *
+ * Assumes `string1' exists, so use in conjunction with AT_STRINGS_BEG ().  
+ */
+#define LETTER_P(POS,OFF)						\
+  (   SYNTAX (fetch_char(POS, OFF, app_closure, stop))			\
+   == Sword)
+
+/* Test if the character at D and the one after D differ with respect
+ * to being word-constituent.  
+ */
+#define AT_WORD_BOUNDARY(d)						\
+  (AT_STRINGS_BEG () || AT_STRINGS_END () || LETTER_P (d,0) != LETTER_P (d, 1))
+
+
+#ifdef RX_SUPPORT_CONTINUATIONS
+#define RX_STACK_ALLOC(BYTES) malloc(BYTES)
+#define RX_STACK_FREE(MEM) free(MEM)
+#else
+#define RX_STACK_ALLOC(BYTES) alloca(BYTES)
+#define RX_STACK_FREE(MEM) \
+      ((struct rx_stack_chunk *)MEM)->next_chunk = search_state.free_chunks; \
+      search_state.free_chunks = ((struct rx_stack_chunk *)MEM);
+
+#endif
+
+#define PUSH(CHUNK_VAR,BYTES)   \
+  if (!CHUNK_VAR || (CHUNK_VAR->bytes_left < (BYTES)))  \
+    {					\
+      struct rx_stack_chunk * new_chunk;	\
+      if (search_state.free_chunks)			\
+	{				\
+	  new_chunk = search_state.free_chunks;	\
+	  search_state.free_chunks = search_state.free_chunks->next_chunk; \
+	}				\
+      else				\
+	{				\
+	  new_chunk = (struct rx_stack_chunk *)RX_STACK_ALLOC(search_state.chunk_bytes); \
+	  if (!new_chunk)		\
+	    {				\
+	      search_state.ret_val = 0;		\
+	      goto test_do_return;	\
+	    }				\
+	}				\
+      new_chunk->sp = (char *)new_chunk + sizeof (struct rx_stack_chunk); \
+      new_chunk->bytes_left = (search_state.chunk_bytes \
+			       - (BYTES) \
+			       - sizeof (struct rx_stack_chunk)); \
+      new_chunk->next_chunk = CHUNK_VAR; \
+      CHUNK_VAR = new_chunk;		\
+    } \
+  else \
+    (CHUNK_VAR->sp += (BYTES)), (CHUNK_VAR->bytes_left -= (BYTES))
+
+#define POP(CHUNK_VAR,BYTES) \
+  if (CHUNK_VAR->sp == ((char *)CHUNK_VAR + sizeof(*CHUNK_VAR))) \
+    { \
+      struct rx_stack_chunk * new_chunk = CHUNK_VAR->next_chunk; \
+      RX_STACK_FREE(CHUNK_VAR); \
+      CHUNK_VAR = new_chunk; \
+    } \
+  else \
+    (CHUNK_VAR->sp -= BYTES), (CHUNK_VAR->bytes_left += BYTES)
+
+
+
+#define SRCH_TRANSLATE(C)  search_state.translate[(unsigned char) (C)]
+
+
+
+
+
+#ifdef __STDC__
+RX_DECL __inline__ int
+rx_search  (struct re_pattern_buffer * rxb,
+	    int startpos,
+	    int range,
+	    int stop,
+	    int total_size,
+	    rx_get_burst_fn get_burst,
+	    rx_back_check_fn back_check,
+	    rx_fetch_char_fn fetch_char,
+	    void * app_closure,
+	    struct re_registers * regs,
+	    struct rx_search_state * resume_state,
+	    struct rx_search_state * save_state)
+#else
+RX_DECL __inline__ int
+rx_search  (rxb, startpos, range, stop, total_size,
+	    get_burst, back_check, fetch_char,
+	    app_closure, regs, resume_state, save_state)
+     struct re_pattern_buffer * rxb;
+     int startpos;
+     int range;
+     int stop;
+     int total_size;
+     rx_get_burst_fn get_burst;
+     rx_back_check_fn back_check;
+     rx_fetch_char_fn fetch_char;
+     void * app_closure;
+     struct re_registers * regs;
+     struct rx_search_state * resume_state;
+     struct rx_search_state * save_state;
+#endif
+{
+  int pc;
+  int test_state;
+  struct rx_search_state search_state;
+
+  search_state.free_chunks = 0;
+  if (!resume_state)
+    pc = rx_outer_start;
+  else
+    {
+      search_state = *resume_state;
+      regs = search_state.saved_regs;
+      rxb = search_state.saved_rxb;
+      startpos = search_state.saved_startpos;
+      range = search_state.saved_range;
+      stop = search_state.saved_stop;
+      total_size = search_state.saved_total_size;
+      get_burst = search_state.saved_get_burst;
+      back_check = search_state.saved_back_check;
+      pc = search_state.outer_search_resume_pt;
+      if (0)
+	{
+	return_continuation:
+	  if (save_state)
+	    {
+	      *save_state = search_state;
+	      save_state->saved_regs = regs;
+	      save_state->saved_rxb = rxb;
+	      save_state->saved_startpos = startpos;
+	      save_state->saved_range = range;
+	      save_state->saved_stop = stop;
+	      save_state->saved_total_size = total_size;
+	      save_state->saved_get_burst = get_burst;
+	      save_state->saved_back_check = back_check;
+	      save_state->outer_search_resume_pt = pc;
+	    }
+	  return rx_search_continuation;
+	}
+    }
+
+  switch (pc)
+    {
+    case rx_outer_start:
+      search_state.ret_val = rx_search_fail;
+      (  search_state.lparen
+       = search_state.rparen
+       = search_state.best_lpspace
+       = search_state.best_rpspace
+       = 0);
+      
+      /* figure the number of registers we may need for use in backreferences.
+       * the number here includes an element for register zero.  
+       */
+      search_state.num_regs = rxb->re_nsub + 1;
+      
+      
+      /* check for out-of-range startpos.  */
+      if ((startpos < 0) || (startpos > total_size))
+	return rx_search_fail;
+      
+      /* fix up range if it might eventually take us outside the string. */
+      {
+	int endpos;
+	endpos = startpos + range;
+	if (endpos < -1)
+	  range = (-1 - startpos);
+	else if (endpos > (total_size + 1))
+	  range = total_size - startpos;
+      }
+      
+      /* if the search isn't to be a backwards one, don't waste time in a
+       * long search for a pattern that says it is anchored.
+       */
+      if (rxb->begbuf_only && (range > 0))
+	{
+	  if (startpos > 0)
+	    return rx_search_fail;
+	  else
+	    range = 1;
+	}
+      
+      /* decide whether to use internal or user-provided reg buffers. */
+      if (!regs || rxb->no_sub)
+	{
+	  search_state.best_lpspace =
+	    (regoff_t *)REGEX_ALLOCATE (search_state.num_regs * sizeof(regoff_t));
+	  search_state.best_rpspace =
+	    (regoff_t *)REGEX_ALLOCATE (search_state.num_regs * sizeof(regoff_t));
+	  search_state.best_lparen = search_state.best_lpspace;
+	  search_state.best_rparen = search_state.best_rpspace;
+	}
+      else
+	{	
+	  /* have the register data arrays been allocated?  */
+	  if (rxb->regs_allocated == REGS_UNALLOCATED)
+	    { /* no.  so allocate them with malloc.  we need one
+		 extra element beyond `search_state.num_regs' for the `-1' marker
+		 gnu code uses.  */
+	      regs->num_regs = MAX (RE_NREGS, rxb->re_nsub + 1);
+	      regs->start = ((regoff_t *)
+			     malloc (regs->num_regs * sizeof ( regoff_t)));
+	      regs->end = ((regoff_t *)
+			   malloc (regs->num_regs * sizeof ( regoff_t)));
+	      if (regs->start == 0 || regs->end == 0)
+		return rx_search_error;
+	      rxb->regs_allocated = REGS_REALLOCATE;
+	    }
+	  else if (rxb->regs_allocated == REGS_REALLOCATE)
+	    { /* yes.  if we need more elements than were already
+		 allocated, reallocate them.  if we need fewer, just
+		 leave it alone.  */
+	      if (regs->num_regs < search_state.num_regs + 1)
+		{
+		  regs->num_regs = search_state.num_regs + 1;
+		  regs->start = ((regoff_t *)
+				 realloc (regs->start,
+					  regs->num_regs * sizeof (regoff_t)));
+		  regs->end = ((regoff_t *)
+			       realloc (regs->end,
+					regs->num_regs * sizeof ( regoff_t)));
+		  if (regs->start == 0 || regs->end == 0)
+		    return rx_search_error;
+		}
+	    }
+	  else if (rxb->regs_allocated != REGS_FIXED)
+	    return rx_search_error;
+	  
+	  if (regs->num_regs < search_state.num_regs + 1)
+	    {
+	      search_state.best_lpspace =
+		((regoff_t *)
+		 REGEX_ALLOCATE (search_state.num_regs * sizeof(regoff_t)));
+	      search_state.best_rpspace =
+		((regoff_t *)
+		 REGEX_ALLOCATE (search_state.num_regs * sizeof(regoff_t)));
+	      search_state.best_lparen = search_state.best_lpspace;
+	      search_state.best_rparen = search_state.best_rpspace;
+	    }
+	  else
+	    {
+	      search_state.best_lparen = regs->start;
+	      search_state.best_rparen = regs->end;
+	    }
+	}
+      
+      search_state.lparen =
+	(regoff_t *) REGEX_ALLOCATE (search_state.num_regs * sizeof(regoff_t));
+      search_state.rparen =
+	(regoff_t *) REGEX_ALLOCATE (search_state.num_regs * sizeof(regoff_t)); 
+      
+      if (! (   search_state.best_rparen
+	     && search_state.best_lparen
+	     && search_state.lparen && search_state.rparen))
+	return rx_search_error;
+      
+      search_state.best_last_l = search_state.best_last_r = -1;
+      
+      search_state.translate = (rxb->translate
+				? rxb->translate
+				: rx_id_translation);
+      
+      
+      
+      /*
+       * two nfa's were compiled.  
+       * `0' is complete.
+       * `1' faster but gets registers wrong and ends too soon.
+       */
+      search_state.nfa_choice = (regs && !rxb->least_subs) ? '\0' : '\1';
+      
+      /* we have the option to look for the best match or the first
+       * one we can find.  if the user isn't asking for register information,
+       * we don't need to find the best match.
+       */
+      search_state.first_found = !regs;
+      
+      if (range >= 0)
+	{
+	  search_state.outer_pos.search_end = startpos + range;
+	  search_state.outer_pos.search_direction = 1;
+	}
+      else
+	{
+	  search_state.outer_pos.search_end = startpos + range;
+	  search_state.outer_pos.search_direction = -1;
+	}
+      
+      /* the vacuous search always turns up nothing. */
+      if ((search_state.outer_pos.search_direction == 1)
+	  ? (startpos > search_state.outer_pos.search_end)
+	  : (startpos < search_state.outer_pos.search_end))
+	return rx_search_fail;
+      
+      /* now we build the starting state of the supernfa. */
+      {
+	struct rx_superset * start_contents;
+	struct rx_nfa_state_set * start_nfa_set;
+	
+	/* we presume here that the nfa start state has only one
+	 * possible future with no side effects.  
+	 */
+	start_nfa_set = rxb->start->futures->destset;
+	if (   rxb->rx.start_set
+	    && (rxb->rx.start_set->starts_for == &rxb->rx))
+	  start_contents = rxb->rx.start_set;
+	else
+	  {
+	    start_contents =
+	      rx_superstate_eclosure_union (&rxb->rx,
+					    rx_superset_cons (&rxb->rx, 0, 0),
+					    start_nfa_set);
+	    
+	    if (!start_contents)
+	      return rx_search_fail;
+	    
+	    start_contents->starts_for = &rxb->rx;
+	    rxb->rx.start_set = start_contents;
+	  }
+	if (   start_contents->superstate
+	    && (start_contents->superstate->rx_id == rxb->rx.rx_id))
+	  {
+	    search_state.start_super = start_contents->superstate;
+	    rx_lock_superstate (&rxb->rx, search_state.start_super);
+	  }
+	else
+	  {
+	    rx_protect_superset (&rxb->rx, start_contents);
+	    
+	    search_state.start_super = rx_superstate (&rxb->rx, start_contents);
+	    if (!search_state.start_super)
+	      return rx_search_fail;
+	    rx_lock_superstate (&rxb->rx, search_state.start_super);
+	    rx_release_superset (&rxb->rx, start_contents);
+	  }
+      }
+      
+      
+      /* The outer_pos tracks the position within the strings
+       * as seen by loop that calls fastmap_search.
+       *
+       * The caller supplied get_burst function actually 
+       * gives us pointers to chars.
+       * 
+       * Communication with the get_burst function is through an
+       * rx_string_position structure.  Here, the structure for
+       * outer_pos is initialized.   It is set to point to the
+       * NULL string, at an offset of STARTPOS.  STARTPOS is out
+       * of range of the NULL string, so the first call to 
+       * getburst will patch up the rx_string_position to point
+       * to valid characters.
+       */
+
+      (  search_state.outer_pos.string
+       = search_state.outer_pos.end
+       = 0);
+
+      search_state.outer_pos.offset = 0;
+      search_state.outer_pos.size = 0;
+      search_state.outer_pos.pos = (unsigned char *)startpos;
+      init_fastmap (rxb, &search_state);
+
+      search_state.fastmap_resume_pt = rx_fastmap_start;
+    case rx_outer_fastmap:
+      /* do { */
+    pseudo_do:
+      {
+	{
+	  int fastmap_state;
+	  fastmap_state = fastmap_search (rxb, stop, get_burst, app_closure,
+					  &search_state);
+	  switch (fastmap_state)
+	    {
+	    case rx_fastmap_continuation:
+	      pc = rx_outer_fastmap;
+	      goto return_continuation;
+	    case rx_fastmap_fail:
+	      goto finish;
+	    case rx_fastmap_ok:
+	      break;
+	    }
+	}
+	
+	/* now the fastmap loop has brought us to a plausible 
+	 * starting point for a match.  so, it's time to run the
+	 * nfa and see if a match occured.
+	 */
+	startpos = (  search_state.outer_pos.pos
+		    - search_state.outer_pos.string
+		    + search_state.outer_pos.offset);
+/*|*/	if ((range > 0) && (startpos == search_state.outer_pos.search_end))
+/*|*/	  goto finish;
+      }
+
+      search_state.test_match_resume_pt = rx_test_start;
+      /* do interrupted for entry point... */
+    case rx_outer_test:
+      /* ...do continued */
+      {
+	goto test_match;
+      test_returns_to_search:
+	switch (test_state)
+	  {
+	  case rx_test_continuation:
+	    pc = rx_outer_test;
+	    goto return_continuation;
+	  case rx_test_error:
+	    search_state.ret_val = rx_search_error;
+	    goto finish;
+	  case rx_test_fail:
+	    break;
+	  case rx_test_ok:
+	    goto finish;
+	  }
+	search_state.outer_pos.pos += search_state.outer_pos.search_direction;
+	startpos += search_state.outer_pos.search_direction;
+#if 0
+/*|*/	if (search_state.test_pos.pos < search_state.test_pos.end)
+/*|*/	  break;
+#endif
+      }
+      /* do interrupted for entry point... */
+    case rx_outer_restore_pos:
+      {
+	int x;
+	x = get_burst (&search_state.outer_pos, app_closure, stop);
+	switch (x)
+	  {
+	  case rx_get_burst_continuation:
+	    pc = rx_outer_restore_pos;
+	    goto return_continuation;
+	  case rx_get_burst_error:
+	    search_state.ret_val = rx_search_error;
+	    goto finish;
+	  case rx_get_burst_no_more:
+	    if (rxb->can_match_empty)
+	      break;
+	    goto finish;
+	  case rx_get_burst_ok:
+	    break;
+	  }
+      } /* } while (...see below...) */
+
+      if ((search_state.outer_pos.search_direction == 1)
+	  ? (startpos < search_state.outer_pos.search_end)
+	  : (startpos > search_state.outer_pos.search_end))
+	goto pseudo_do;
+
+	
+    finish:
+      uninit_fastmap (rxb, &search_state);
+      if (search_state.start_super)
+	rx_unlock_superstate (&rxb->rx, search_state.start_super);
+      
+#ifdef regex_malloc
+      if (search_state.lparen) free (search_state.lparen);
+      if (search_state.rparen) free (search_state.rparen);
+      if (search_state.best_lpspace) free (search_state.best_lpspace);
+      if (search_state.best_rpspace) free (search_state.best_rpspace);
+#endif
+      return search_state.ret_val;
+    }
+
+
+ test_match:
+  {
+    enum rx_test_match_entry test_pc;
+    int inx;
+    test_pc = search_state.test_match_resume_pt;
+    if (test_pc == rx_test_start)
+      {
+#ifdef RX_DEBUG
+	search_state.backtrack_depth = 0;
+#endif
+	search_state.last_l = search_state.last_r = 0;
+	search_state.lparen[0] = startpos;
+	search_state.super = search_state.start_super;
+	search_state.c = search_state.nfa_choice;
+	search_state.test_pos.pos = search_state.outer_pos.pos - 1;    
+	search_state.test_pos.string = search_state.outer_pos.string;
+	search_state.test_pos.end = search_state.outer_pos.end;
+	search_state.test_pos.offset = search_state.outer_pos.offset;
+	search_state.test_pos.size = search_state.outer_pos.size;
+	search_state.test_pos.search_direction = 1;
+	search_state.counter_stack = 0;
+	search_state.backtrack_stack = 0;
+	search_state.backtrack_frame_bytes =
+	  (sizeof (struct rx_backtrack_frame)
+	   + (rxb->match_regs_on_stack
+	      ? sizeof (regoff_t) * (search_state.num_regs + 1) * 2
+	      : 0));
+	search_state.chunk_bytes = search_state.backtrack_frame_bytes * 64;
+	search_state.test_ret = rx_test_line_finished;
+	search_state.could_have_continued = 0;
+      }  
+    /* This is while (1)...except that the body of the loop is interrupted 
+     * by some alternative entry points.
+     */
+  pseudo_while_1:
+    switch (test_pc)
+      {
+      case rx_test_cache_hit_loop:
+	goto resume_continuation_1;
+      case rx_test_backreference_check:
+	goto resume_continuation_2;
+      case rx_test_backtrack_return:
+	goto resume_continuation_3;
+      case rx_test_start:
+#ifdef RX_DEBUG
+	/* There is a search tree with every node as set of deterministic
+	 * transitions in the super nfa.  For every branch of a 
+	 * backtrack point is an edge in the tree.
+	 * This counts up a pre-order of nodes in that tree.
+	 * It's saved on the search stack and printed when debugging. 
+	 */
+	search_state.line_no = 0;
+	search_state.lines_found = 0;
+#endif
+	
+      top_of_cycle:
+	/* A superstate is basicly a transition table, indexed by 
+	 * characters from the string being tested, and containing 
+	 * RX_INX (`instruction frame') structures.
+	 */
+	search_state.ifr = &search_state.super->transitions [search_state.c];
+	
+      recurse_test_match:
+	/* This is the point to which control is sent when the
+	 * test matcher `recurses'.  Before jumping here, some variables
+	 * need to be saved on the stack and the next instruction frame
+	 * has to be computed.
+	 */
+	
+      restart:
+	/* Some instructions don't advance the matcher, but just
+	 * carry out some side effects and fetch a new instruction.
+	 * To dispatch that new instruction, `goto restart'.
+	 */
+	
+	{
+	  struct rx_inx * next_tr_table;
+	  struct rx_inx * this_tr_table;
+	  /* The fastest route through the loop is when the instruction 
+	   * is RX_NEXT_CHAR.  This case is detected when SEARCH_STATE.IFR->DATA
+	   * is non-zero.  In that case, it points to the next
+	   * superstate. 
+	   *
+	   * This allows us to not bother fetching the bytecode.
+	   */
+	  next_tr_table = (struct rx_inx *)search_state.ifr->data;
+	  this_tr_table = search_state.super->transitions;
+	  while (next_tr_table)
+	    {
+#ifdef RX_DEBUG_0
+	      if (rx_debug_trace)
+		{
+		  struct rx_superset * setp;
+		  
+		  fprintf (stderr, "%d %d>> re_next_char @ %d (%d)",
+			   search_state.line_no,
+			   search_state.backtrack_depth,
+			   (search_state.test_pos.pos - search_state.test_pos.string
+			    + search_state.test_pos.offset), search_state.c);
+		  
+		  search_state.super =
+		    ((struct rx_superstate *)
+		     ((char *)this_tr_table
+		      - ((unsigned long)
+			 ((struct rx_superstate *)0)->transitions)));
+		  
+		  setp = search_state.super->contents;
+		  fprintf (stderr, "   superstet (rx=%d, &=%x: ",
+			   rxb->rx.rx_id, setp);
+		  while (setp)
+		    {
+		      fprintf (stderr, "%d ", setp->id);
+		      setp = setp->cdr;
+		    }
+		  fprintf (stderr, "\n");
+		}
+#endif
+	      this_tr_table = next_tr_table;
+	      ++search_state.test_pos.pos;
+	      if (search_state.test_pos.pos == search_state.test_pos.end)
+		{
+		  int burst_state;
+		try_burst_1:
+		  burst_state = get_burst (&search_state.test_pos,
+					   app_closure, stop);
+		  switch (burst_state)
+		    {
+		    case rx_get_burst_continuation:
+		      search_state.saved_this_tr_table = this_tr_table;
+		      search_state.saved_next_tr_table = next_tr_table;
+		      test_pc = rx_test_cache_hit_loop;
+		      goto test_return_continuation;
+		      
+		    resume_continuation_1:
+		      /* Continuation one jumps here to do its work: */
+		      search_state.saved_this_tr_table = this_tr_table;
+		      search_state.saved_next_tr_table = next_tr_table;
+		      goto try_burst_1;
+		      
+		    case rx_get_burst_ok:
+		      /* get_burst succeeded...keep going */
+		      break;
+		      
+		    case rx_get_burst_no_more:
+		      search_state.test_ret = rx_test_line_finished;
+		      search_state.could_have_continued = 1;
+		      goto test_do_return;
+		      
+		    case rx_get_burst_error:
+		      /* An error... */
+		      search_state.test_ret = rx_test_internal_error;
+		      goto test_do_return;
+		    }
+		}
+	      search_state.c = *search_state.test_pos.pos;
+	      search_state.ifr = this_tr_table + search_state.c;
+	      next_tr_table = (struct rx_inx *)search_state.ifr->data;
+	    } /* Fast loop through cached transition tables */
+	  
+	  /* Here when we ran out of cached next-char transitions. 
+	   * So, it will be necessary to do a more expensive
+	   * dispatch on the current instruction.  The superstate
+	   * pointer is allowed to become invalid during next-char
+	   * transitions -- now we must bring it up to date.
+	   */
+	  search_state.super =
+	    ((struct rx_superstate *)
+	     ((char *)this_tr_table
+	      - ((unsigned long)
+		 ((struct rx_superstate *)0)->transitions)));
+	}
+	
+	/* We've encountered an instruction other than next-char.
+	 * Dispatch that instruction:
+	 */
+	inx = (int)search_state.ifr->inx;
+#ifdef RX_DEBUG_0
+	if (rx_debug_trace)
+	  {
+	    struct rx_superset * setp = search_state.super->contents;
+	    
+	    fprintf (stderr, "%d %d>> %s @ %d (%d)", search_state.line_no,
+		     search_state.backtrack_depth,
+		     inx_names[inx],
+		     (search_state.test_pos.pos - search_state.test_pos.string
+		      + (test_pos.half == 0 ? 0 : size1)), search_state.c);
+	    
+	    fprintf (stderr, "   superstet (rx=%d, &=%x: ",
+		     rxb->rx.rx_id, setp);
+	    while (setp)
+	      {
+		fprintf (stderr, "%d ", setp->id);
+		setp = setp->cdr;
+	      }
+	    fprintf (stderr, "\n");
+	  }
+#endif
+	switch ((enum rx_opcode)inx)
+	  {
+	  case rx_do_side_effects:
+	    
+	    /*  RX_DO_SIDE_EFFECTS occurs when we cross epsilon 
+	     *  edges associated with parentheses, backreferencing, etc.
+	     */
+	    {
+	      struct rx_distinct_future * df =
+		(struct rx_distinct_future *)search_state.ifr->data_2;
+	      struct rx_se_list * el = df->effects;
+	      /* Side effects come in lists.  This walks down
+	       * a list, dispatching.
+	       */
+	      while (el)
+		{
+		  long effect;
+		  effect = (long)el->car;
+		  if (effect < 0)
+		    {
+#ifdef RX_DEBUG_0
+		      if (rx_debug_trace)
+			{
+			  struct rx_superset * setp = search_state.super->contents;
+			  
+			  fprintf (stderr, "....%d %d>> %s\n", search_state.line_no,
+				   search_state.backtrack_depth,
+				   efnames[-effect]);
+			}
+#endif
+		      switch ((enum re_side_effects) effect)
+
+			{
+			case re_se_pushback:
+			  search_state.ifr = &df->future_frame;
+			  if (!search_state.ifr->data)
+			    {
+			      struct rx_superstate * sup;
+			      sup = search_state.super;
+			      rx_lock_superstate (rx, sup);
+			      if (!rx_handle_cache_miss (&rxb->rx,
+							 search_state.super,
+							 search_state.c,
+							 (search_state.ifr
+							  ->data_2)))
+				{
+				  rx_unlock_superstate (rx, sup);
+				  search_state.test_ret = rx_test_internal_error;
+				  goto test_do_return;
+				}
+			      rx_unlock_superstate (rx, sup);
+			    }
+			  /* --search_state.test_pos.pos; */
+			  search_state.c = 't';
+			  search_state.super
+			    = ((struct rx_superstate *)
+			       ((char *)search_state.ifr->data
+				- (long)(((struct rx_superstate *)0)
+					 ->transitions)));
+			  goto top_of_cycle;
+			  break;
+			case re_se_push0:
+			  {
+			    struct rx_counter_frame * old_cf
+			      = (search_state.counter_stack
+				 ? ((struct rx_counter_frame *)
+				    search_state.counter_stack->sp)
+				 : 0);
+			    struct rx_counter_frame * cf;
+			    PUSH (search_state.counter_stack,
+				  sizeof (struct rx_counter_frame));
+			    cf = ((struct rx_counter_frame *)
+				  search_state.counter_stack->sp);
+			    cf->tag = re_se_iter;
+			    cf->val = 0;
+			    cf->inherited_from = 0;
+			    cf->cdr = old_cf;
+			    break;
+			  }
+			case re_se_fail:
+			  goto test_do_return;
+			case re_se_begbuf:
+			  if (!AT_STRINGS_BEG ())
+			    goto test_do_return;
+			  break;
+			case re_se_endbuf:
+			  if (!AT_STRINGS_END ())
+			    goto test_do_return;
+			  break;
+			case re_se_wordbeg:
+			  if (   LETTER_P (&search_state.test_pos, 1)
+			      && (   AT_STRINGS_BEG()
+				  || !LETTER_P (&search_state.test_pos, 0)))
+			    break;
+			  else
+			    goto test_do_return;
+			case re_se_wordend:
+			  if (   !AT_STRINGS_BEG ()
+			      && LETTER_P (&search_state.test_pos, 0)
+			      && (AT_STRINGS_END ()
+				  || !LETTER_P (&search_state.test_pos, 1)))
+			    break;
+			  else
+			    goto test_do_return;
+			case re_se_wordbound:
+			  if (AT_WORD_BOUNDARY (&search_state.test_pos))
+			    break;
+			  else
+			    goto test_do_return;
+			case re_se_notwordbound:
+			  if (!AT_WORD_BOUNDARY (&search_state.test_pos))
+			    break;
+			  else
+			    goto test_do_return;
+			case re_se_hat:
+			  if (AT_STRINGS_BEG ())
+			    {
+			      if (rxb->not_bol)
+				goto test_do_return;
+			      else
+				break;
+			    }
+			  else
+			    {
+			      char pos_c = *search_state.test_pos.pos;
+			      if (   (SRCH_TRANSLATE (pos_c)
+				      == SRCH_TRANSLATE('\n'))
+				  && rxb->newline_anchor)
+				break;
+			      else
+				goto test_do_return;
+			    }
+			case re_se_dollar:
+			  if (AT_STRINGS_END ())
+			    {
+			      if (rxb->not_eol)
+				goto test_do_return;
+			      else
+				break;
+			    }
+			  else
+			    {
+			      if (   (   SRCH_TRANSLATE (fetch_char
+						    (&search_state.test_pos, 1,
+						     app_closure, stop))
+				      == SRCH_TRANSLATE ('\n'))
+				  && rxb->newline_anchor)
+				break;
+			      else
+				goto test_do_return;
+			    }
+			  
+			case re_se_try:
+			  /* This is the first side effect in every
+			   * expression.
+			   *
+			   *  FOR NO GOOD REASON...get rid of it...
+			   */
+			  break;
+			  
+			case re_se_pushpos:
+			  {
+			    int urhere =
+			      ((int)(search_state.test_pos.pos
+				     - search_state.test_pos.string)
+			       + search_state.test_pos.offset);
+			    struct rx_counter_frame * old_cf
+			      = (search_state.counter_stack
+				 ? ((struct rx_counter_frame *)
+				    search_state.counter_stack->sp)
+				 : 0);
+			    struct rx_counter_frame * cf;
+			    PUSH(search_state.counter_stack,
+				 sizeof (struct rx_counter_frame));
+			    cf = ((struct rx_counter_frame *)
+				  search_state.counter_stack->sp);
+			    cf->tag = re_se_pushpos;
+			    cf->val = urhere;
+			    cf->inherited_from = 0;
+			    cf->cdr = old_cf;
+			    break;
+			  }
+			  
+			case re_se_chkpos:
+			  {
+			    int urhere =
+			      ((int)(search_state.test_pos.pos
+				     - search_state.test_pos.string)
+			       + search_state.test_pos.offset);
+			    struct rx_counter_frame * cf
+			      = ((struct rx_counter_frame *)
+				 search_state.counter_stack->sp);
+			    if (cf->val == urhere)
+			      goto test_do_return;
+			    cf->val = urhere;
+			    break;
+			  }
+			  break;
+			  
+			case re_se_poppos:
+			  POP(search_state.counter_stack,
+			      sizeof (struct rx_counter_frame));
+			  break;
+			  
+			  
+			case re_se_at_dot:
+			case re_se_syntax:
+			case re_se_not_syntax:
+#ifdef emacs
+			  /* 
+			   * this release lacks emacs support
+			   */
+#endif
+			  break;
+			case re_se_win:
+			case re_se_lparen:
+			case re_se_rparen:
+			case re_se_backref:
+			case re_se_iter:
+			case re_se_end_iter:
+			case re_se_tv:
+			case re_floogle_flap:
+			  search_state.ret_val = 0;
+			  goto test_do_return;
+			}
+		    }
+		  else
+		    {
+#ifdef RX_DEBUG_0
+		      if (rx_debug_trace)
+			fprintf (stderr, "....%d %d>> %s %d %d\n", search_state.line_no,
+				 search_state.backtrack_depth,
+				 efnames2[rxb->se_params [effect].se],
+				 rxb->se_params [effect].op1,
+				 rxb->se_params [effect].op2);
+#endif
+		      switch (rxb->se_params [effect].se)
+			{
+			case re_se_win:
+			  /* This side effect indicates that we've 
+			   * found a match, though not necessarily the 
+			   * best match.  This is a fancy assignment to 
+			   * register 0 unless the caller didn't 
+			   * care about registers.  In which case,
+			   * this stops the match.
+			   */
+			  {
+			    int urhere =
+			      ((int)(search_state.test_pos.pos
+				     - search_state.test_pos.string)
+			       + search_state.test_pos.offset);
+			    
+			    if (   (search_state.best_last_r < 0)
+				|| (urhere + 1 > search_state.best_rparen[0]))
+			      {
+				/* Record the best known and keep
+				 * looking.
+				 */
+				int x;
+				for (x = 0; x <= search_state.last_l; ++x)
+				  search_state.best_lparen[x] = search_state.lparen[x];
+				search_state.best_last_l = search_state.last_l;
+				for (x = 0; x <= search_state.last_r; ++x)
+				  search_state.best_rparen[x] = search_state.rparen[x];
+				search_state.best_rparen[0] = urhere + 1;
+				search_state.best_last_r = search_state.last_r;
+			      }
+			    /* If we're not reporting the match-length 
+			     * or other register info, we need look no
+			     * further.
+			     */
+			    if (search_state.first_found)
+			      {
+				search_state.test_ret = rx_test_found_first;
+				goto test_do_return;
+			      }
+			  }
+			  break;
+			case re_se_lparen:
+			  {
+			    int urhere =
+			      ((int)(search_state.test_pos.pos
+				     - search_state.test_pos.string)
+			       + search_state.test_pos.offset);
+			    
+			    int reg = rxb->se_params [effect].op1;
+#if 0
+			    if (reg > search_state.last_l)
+#endif
+			      {
+				search_state.lparen[reg] = urhere + 1;
+				/* In addition to making this assignment,
+				 * we now know that lower numbered regs
+				 * that haven't already been assigned,
+				 * won't be.  We make sure they're
+				 * filled with -1, so they can be
+				 * recognized as unassigned.
+				 */
+				if (search_state.last_l < reg)
+				  while (++search_state.last_l < reg)
+				    search_state.lparen[search_state.last_l] = -1;
+			      }
+			    break;
+			  }
+			  
+			case re_se_rparen:
+			  {
+			    int urhere =
+			      ((int)(search_state.test_pos.pos
+				     - search_state.test_pos.string)
+			       + search_state.test_pos.offset);
+			    int reg = rxb->se_params [effect].op1;
+			    search_state.rparen[reg] = urhere + 1;
+			    if (search_state.last_r < reg)
+			      {
+				while (++search_state.last_r < reg)
+				  search_state.rparen[search_state.last_r]
+				    = -1;
+			      }
+			    break;
+			  }
+			  
+			case re_se_backref:
+			  {
+			    int reg = rxb->se_params [effect].op1;
+			    if (   reg > search_state.last_r
+				|| search_state.rparen[reg] < 0)
+			      goto test_do_return;
+			    
+			    {
+			      int backref_status;
+			    check_backreference:
+			      backref_status
+				= back_check (&search_state.test_pos,
+					      search_state.lparen[reg],
+					      search_state.rparen[reg],
+					      search_state.translate,
+					      app_closure,
+					      stop);
+			      switch (backref_status)
+				{
+				case rx_back_check_continuation:
+				  search_state.saved_reg = reg;
+				  test_pc = rx_test_backreference_check;
+				  goto test_return_continuation;
+				resume_continuation_2:
+				  reg = search_state.saved_reg;
+				  goto check_backreference;
+				case rx_back_check_fail:
+				  /* Fail */
+				  goto test_do_return;
+				case rx_back_check_pass:
+				  /* pass --
+				   * test_pos now advanced to last
+				   * char matched by backref
+				   */
+				  break;
+				}
+			    }
+			    break;
+			  }
+			case re_se_iter:
+			  {
+			    struct rx_counter_frame * csp
+			      = ((struct rx_counter_frame *)
+				 search_state.counter_stack->sp);
+			    if (csp->val == rxb->se_params[effect].op2)
+			      goto test_do_return;
+			    else
+			      ++csp->val;
+			    break;
+			  }
+			case re_se_end_iter:
+			  {
+			    struct rx_counter_frame * csp
+			      = ((struct rx_counter_frame *)
+				 search_state.counter_stack->sp);
+			    if (csp->val < rxb->se_params[effect].op1)
+			      goto test_do_return;
+			    else
+			      {
+				struct rx_counter_frame * source = csp;
+				while (source->inherited_from)
+				  source = source->inherited_from;
+				if (!source || !source->cdr)
+				  {
+				    POP(search_state.counter_stack,
+					sizeof(struct rx_counter_frame));
+				  }
+				else
+				  {
+				    source = source->cdr;
+				    csp->val = source->val;
+				    csp->tag = source->tag;
+				    csp->cdr = 0;
+				    csp->inherited_from = source;
+				  }
+			      }
+			    break;
+			  }
+			case re_se_tv:
+			  /* is a noop */
+			  break;
+			case re_se_try:
+			case re_se_pushback:
+			case re_se_push0:
+			case re_se_pushpos:
+			case re_se_chkpos:
+			case re_se_poppos:
+			case re_se_at_dot:
+			case re_se_syntax:
+			case re_se_not_syntax:
+			case re_se_begbuf:
+			case re_se_hat:
+			case re_se_wordbeg:
+			case re_se_wordbound:
+			case re_se_notwordbound:
+			case re_se_wordend:
+			case re_se_endbuf:
+			case re_se_dollar:
+			case re_se_fail:
+			case re_floogle_flap:
+			  search_state.ret_val = 0;
+			  goto test_do_return;
+			}
+		    }
+		  el = el->cdr;
+		}
+	      /* Now the side effects are done,
+	       * so get the next instruction.
+	       * and move on.
+	       */
+	      search_state.ifr = &df->future_frame;
+	      goto restart;
+	    }
+	    
+	  case rx_backtrack_point:
+	    {
+	      /* A backtrack point indicates that we've reached a
+	       * non-determinism in the superstate NFA.  This is a
+	       * loop that exhaustively searches the possibilities.
+	       *
+	       * A backtracking strategy is used.  We keep track of what
+	       * registers are valid so we can erase side effects.
+	       *
+	       * First, make sure there is some stack space to hold 
+	       * our state.
+	       */
+	      
+	      struct rx_backtrack_frame * bf;
+	      
+	      PUSH(search_state.backtrack_stack,
+		   search_state.backtrack_frame_bytes);
+#ifdef RX_DEBUG_0
+	      ++search_state.backtrack_depth;
+#endif
+	      
+	      bf = ((struct rx_backtrack_frame *)
+		    search_state.backtrack_stack->sp);
+	      {
+		bf->stk_super = search_state.super;
+		/* We prevent the current superstate from being
+		 * deleted from the superstate cache.
+		 */
+		rx_lock_superstate (&rxb->rx, search_state.super);
+#ifdef RX_DEBUG_0
+		bf->stk_search_state.line_no = search_state.line_no;
+#endif
+		bf->stk_c = search_state.c;
+		bf->stk_test_pos = search_state.test_pos;
+		bf->stk_last_l = search_state.last_l;
+		bf->stk_last_r = search_state.last_r;
+		bf->df = ((struct rx_super_edge *)
+			  search_state.ifr->data_2)->options;
+		bf->first_df = bf->df;
+		bf->counter_stack_sp = (search_state.counter_stack
+					? search_state.counter_stack->sp
+					: 0);
+		bf->stk_test_ret = search_state.test_ret;
+		if (rxb->match_regs_on_stack)
+		  {
+		    int x;
+		    regoff_t * stk =
+		      (regoff_t *)((char *)bf + sizeof (*bf));
+		    for (x = 0; x <= search_state.last_l; ++x)
+		      stk[x] = search_state.lparen[x];
+		    stk += x;
+		    for (x = 0; x <= search_state.last_r; ++x)
+		      stk[x] = search_state.rparen[x];
+		  }
+	      }
+	      
+	      /* Here is a while loop whose body is mainly a function
+	       * call and some code to handle a return from that
+	       * function.
+	       *
+	       * From here on for the rest of `case backtrack_point' it
+	       * is unsafe to assume that the search_state copies of 
+	       * variables saved on the backtracking stack are valid
+	       * -- so read their values from the backtracking stack.
+	       *
+	       * This lets us use one generation fewer stack saves in
+	       * the call-graph of a search.
+	       */
+	      
+	    while_non_det_options:
+#ifdef RX_DEBUG_0
+	      ++search_state.lines_found;
+	      if (rx_debug_trace)
+		fprintf (stderr, "@@@ %d calls %d @@@\n",
+			 search_state.line_no, search_state.lines_found);
+	      
+	      search_state.line_no = search_state.lines_found;
+#endif
+	      
+	      if (bf->df->next_same_super_edge[0] == bf->first_df)
+		{
+		  /* This is a tail-call optimization -- we don't recurse
+		   * for the last of the possible futures.
+		   */
+		  search_state.ifr = (bf->df->effects
+				      ? &bf->df->side_effects_frame
+				      : &bf->df->future_frame);
+		  
+		  rx_unlock_superstate (&rxb->rx, search_state.super);
+		  POP(search_state.backtrack_stack,
+		      search_state.backtrack_frame_bytes);
+#ifdef RX_DEBUG
+		  --search_state.backtrack_depth;
+#endif
+		  goto restart;
+		}
+	      else
+		{
+		  if (search_state.counter_stack)
+		    {
+		      struct rx_counter_frame * old_cf
+			= ((struct rx_counter_frame *)search_state.counter_stack->sp);
+		      struct rx_counter_frame * cf;
+		      PUSH(search_state.counter_stack, sizeof (struct rx_counter_frame));
+		      cf = ((struct rx_counter_frame *)search_state.counter_stack->sp);
+		      cf->tag = old_cf->tag;
+		      cf->val = old_cf->val;
+		      cf->inherited_from = old_cf;
+		      cf->cdr = 0;
+		    }			
+		  /* `Call' this test-match block */
+		  search_state.ifr = (bf->df->effects
+				      ? &bf->df->side_effects_frame
+				      : &bf->df->future_frame);
+		  goto recurse_test_match;
+		}
+	      
+	      /* Returns in this block are accomplished by
+	       * goto test_do_return.  There are two cases.
+	       * If there is some search-stack left,
+	       * then it is a return from a `recursive' call.
+	       * If there is no search-stack left, then
+	       * we should return to the fastmap/search loop.
+	       */
+	      
+	    test_do_return:
+	      
+	      if (!search_state.backtrack_stack)
+		{
+#ifdef RX_DEBUG_0
+		  if (rx_debug_trace)
+		    fprintf (stderr, "!!! %d bails returning %d !!!\n",
+			     search_state.line_no, search_state.test_ret);
+#endif
+		  
+		  /* No more search-stack -- this test is done. */
+		  if (search_state.test_ret)
+		    goto return_from_test_match;
+		  else
+		    goto error_in_testing_match;
+		}
+	      
+	      /* Returning from a recursive call to 
+	       * the test match block:
+	       */
+	      
+	      bf = ((struct rx_backtrack_frame *)
+		    search_state.backtrack_stack->sp);
+#ifdef RX_DEBUG_0
+	      if (rx_debug_trace)
+		fprintf (stderr, "+++ %d returns %d (to %d)+++\n",
+			 search_state.line_no,
+			 search_state.test_ret,
+			 bf->stk_search_state.line_no);
+#endif
+	      
+	      while (search_state.counter_stack
+		     && (!bf->counter_stack_sp
+			 || (bf->counter_stack_sp
+			     != search_state.counter_stack->sp)))
+		{
+		  POP(search_state.counter_stack,
+		      sizeof (struct rx_counter_frame));
+		}
+	      
+	      if (search_state.test_ret == rx_test_error)
+		{
+		  POP (search_state.backtrack_stack,
+		       search_state.backtrack_frame_bytes);
+		  goto test_do_return;
+		}
+	      
+	      /* If a non-longest match was found and that is good 
+	       * enough, return immediately.
+	       */
+	      if (   (search_state.test_ret == rx_test_found_first)
+		  && search_state.first_found)
+		{
+		  rx_unlock_superstate (&rxb->rx, bf->stk_super);
+		  POP (search_state.backtrack_stack,
+		       search_state.backtrack_frame_bytes);
+		  goto test_do_return;
+		}
+	      
+	      search_state.test_ret = bf->stk_test_ret;
+	      search_state.last_l = bf->stk_last_l;
+	      search_state.last_r = bf->stk_last_r;
+	      bf->df = bf->df->next_same_super_edge[0];
+	      search_state.super = bf->stk_super;
+	      search_state.c = bf->stk_c;
+#ifdef RX_DEBUG_0
+	      search_state.line_no = bf->stk_search_state.line_no;
+#endif
+	      
+	      if (rxb->match_regs_on_stack)
+		{
+		  int x;
+		  regoff_t * stk =
+		    (regoff_t *)((char *)bf + sizeof (*bf));
+		  for (x = 0; x <= search_state.last_l; ++x)
+		    search_state.lparen[x] = stk[x];
+		  stk += x;
+		  for (x = 0; x <= search_state.last_r; ++x)
+		    search_state.rparen[x] = stk[x];
+		}
+	      
+	      {
+		int x;
+	      try_burst_2:
+		x = get_burst (&bf->stk_test_pos, app_closure, stop);
+		switch (x)
+		  {
+		  case rx_get_burst_continuation:
+		    search_state.saved_bf = bf;
+		    test_pc = rx_test_backtrack_return;
+		    goto test_return_continuation;
+		  resume_continuation_3:
+		    bf = search_state.saved_bf;
+		    goto try_burst_2;
+		  case rx_get_burst_no_more:
+		    /* Since we've been here before, it is some kind of
+		     * error that we can't return.
+		     */
+		  case rx_get_burst_error:
+		    search_state.test_ret = rx_test_internal_error;
+		    goto test_do_return;
+		  case rx_get_burst_ok:
+		    break;
+		  }
+	      }
+	      search_state.test_pos = bf->stk_test_pos;
+	      goto while_non_det_options;
+	    }
+	    
+	    
+	  case rx_cache_miss:
+	    /* Because the superstate NFA is lazily constructed,
+	     * and in fact may erode from underneath us, we sometimes
+	     * have to construct the next instruction from the hard way.
+	     * This invokes one step in the lazy-conversion.
+	     */
+	    search_state.ifr = rx_handle_cache_miss (&rxb->rx,
+						     search_state.super,
+						     search_state.c,
+						     search_state.ifr->data_2);
+	    if (!search_state.ifr)
+	      {
+		search_state.test_ret = rx_test_internal_error;
+		goto test_do_return;
+	      }
+	    goto restart;
+	    
+	  case rx_backtrack:
+	    /* RX_BACKTRACK means that we've reached the empty
+	     * superstate, indicating that match can't succeed
+	     * from this point.
+	     */
+	    goto test_do_return;
+	    
+	  case rx_next_char:
+	  case rx_error_inx:
+	  case rx_num_instructions:
+	    search_state.ret_val = 0;
+	    goto test_do_return;
+	  }
+	goto pseudo_while_1;
+      }
+    
+    /* Healthy exits from the test-match loop do a 
+     * `goto return_from_test_match'   On the other hand, 
+     * we might end up here.
+     */
+  error_in_testing_match:
+    test_state = rx_test_error;
+    goto test_returns_to_search;
+    
+    /***** fastmap/search loop body
+     *	      considering the results testing for a match
+     */
+    
+  return_from_test_match:
+    
+    if (search_state.best_last_l >= 0)
+      {
+	if (regs && (regs->start != search_state.best_lparen))
+	  {
+	    bcopy (search_state.best_lparen, regs->start,
+		   regs->num_regs * sizeof (int));
+	    bcopy (search_state.best_rparen, regs->end,
+		   regs->num_regs * sizeof (int));
+	  }
+	if (regs && !rxb->no_sub)
+	  {
+	    int q;
+	    int bound = (regs->num_regs > search_state.num_regs
+			 ? regs->num_regs
+			 : search_state.num_regs);
+	    regoff_t * s = regs->start;
+	    regoff_t * e = regs->end;
+	    for (q = search_state.best_last_l + 1;  q < bound; ++q)
+	      s[q] = e[q] = -1;
+	  }
+	search_state.ret_val = search_state.best_lparen[0];
+	test_state = rx_test_ok;
+	goto test_returns_to_search;
+      }
+    else
+      {
+	test_state = rx_test_fail;
+	goto test_returns_to_search;
+      }
+    
+  test_return_continuation:
+    search_state.test_match_resume_pt = test_pc;
+    test_state = rx_test_continuation;
+    goto test_returns_to_search;
+  }
+}
+
+
+
+#endif /* RX_WANT_RX_DEFS */
+
+
+
+#else /* RX_WANT_SE_DEFS */
+  /* Integers are used to represent side effects.
+   *
+   * Simple side effects are given negative integer names by these enums.
+   * 
+   * Non-negative names are reserved for complex effects.
+   *
+   * Complex effects are those that take arguments.  For example, 
+   * a register assignment associated with a group is complex because
+   * it requires an argument to tell which group is being matched.
+   * 
+   * The integer name of a complex effect is an index into rxb->se_params.
+   */
+ 
+  RX_DEF_SE(1, re_se_try, = -1)		/* Epsilon from start state */
+
+  RX_DEF_SE(0, re_se_pushback, = re_se_try - 1)
+  RX_DEF_SE(0, re_se_push0, = re_se_pushback -1)
+  RX_DEF_SE(0, re_se_pushpos, = re_se_push0 - 1)
+  RX_DEF_SE(0, re_se_chkpos, = re_se_pushpos -1)
+  RX_DEF_SE(0, re_se_poppos, = re_se_chkpos - 1)
+
+  RX_DEF_SE(1, re_se_at_dot, = re_se_poppos - 1)	/* Emacs only */
+  RX_DEF_SE(0, re_se_syntax, = re_se_at_dot - 1) /* Emacs only */
+  RX_DEF_SE(0, re_se_not_syntax, = re_se_syntax - 1) /* Emacs only */
+
+  RX_DEF_SE(1, re_se_begbuf, = re_se_not_syntax - 1) /* match beginning of buffer */
+  RX_DEF_SE(1, re_se_hat, = re_se_begbuf - 1) /* match beginning of line */
+
+  RX_DEF_SE(1, re_se_wordbeg, = re_se_hat - 1) 
+  RX_DEF_SE(1, re_se_wordbound, = re_se_wordbeg - 1)
+  RX_DEF_SE(1, re_se_notwordbound, = re_se_wordbound - 1)
+
+  RX_DEF_SE(1, re_se_wordend, = re_se_notwordbound - 1)
+  RX_DEF_SE(1, re_se_endbuf, = re_se_wordend - 1)
+
+  /* This fails except at the end of a line. 
+   * It deserves to go here since it is typicly one of the last steps 
+   * in a match.
+   */
+  RX_DEF_SE(1, re_se_dollar, = re_se_endbuf - 1)
+
+  /* Simple effects: */
+  RX_DEF_SE(1, re_se_fail, = re_se_dollar - 1)
+
+  /* Complex effects.  These are used in the 'se' field of 
+   * a struct re_se_params.  Indexes into the se array
+   * are stored as instructions on nfa edges.
+   */
+  RX_DEF_CPLX_SE(1, re_se_win, = 0)
+  RX_DEF_CPLX_SE(1, re_se_lparen, = re_se_win + 1)
+  RX_DEF_CPLX_SE(1, re_se_rparen, = re_se_lparen + 1)
+  RX_DEF_CPLX_SE(0, re_se_backref, = re_se_rparen + 1)
+  RX_DEF_CPLX_SE(0, re_se_iter, = re_se_backref + 1) 
+  RX_DEF_CPLX_SE(0, re_se_end_iter, = re_se_iter + 1)
+  RX_DEF_CPLX_SE(0, re_se_tv, = re_se_end_iter + 1)
+
+#endif
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/lib/simplefrenchstem.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/simplefrenchstem.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/simplefrenchstem.c	(revision 699)
@@ -0,0 +1,72 @@
+/**************************************************************************
+ *
+ * simplefrenchstem.c -- a simple french stemmer
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "simplefrenchstem.h"
+
+
+
+/* =========================================================================
+ * Function: simplefrenchstem
+ * Description: a simple french stemmer
+ * Input: a word string with the length in the first byte
+ * Output: the stemmed word
+ * ========================================================================= */
+
+void simplefrenchstem (unsigned char *word) {      
+  unsigned short out[256]; /* temp space */
+  unsigned short *wordstart; /* points to first letter of word */
+  int last;                  /* last points to the last character */
+
+  /* decode */
+  utf8_word_to_unicode (word, out, 255);
+  wordstart = out + 1;
+  last = out[0]-1;
+  
+
+  if (last > 4) {
+    if (wordstart[last]=='x') {
+      if (wordstart[last-1]=='u' && wordstart[last-2]=='a') {
+	wordstart[last-1]='l';
+      }
+      last--;
+
+    } else {
+      if (last>=0 && wordstart[last]=='s') last--;
+      if (last>=0 && wordstart[last]=='r') last--;
+      if (last>=0 && wordstart[last]=='e') last--;
+
+      /* letter with accent e + ' -- there are two possible encodings  */
+      if (last>=0 && wordstart[last]==0xe9) { 
+	last--;
+      } else if (last>=1 && wordstart[last-1]=='e' && wordstart[last]==0x301) {
+	last -= 2; 
+      }
+
+      if (last >= 1 && wordstart[last]==wordstart[last-1]) last--;
+    }  /* end else */
+
+    out[0] = (unsigned char)(last+1);
+  } /* end if (len > 4) */
+
+  /* re-code, make sure the result is not longer than the input */
+  unicode_to_utf8_word (out, word, word[0]+1);
+}
Index: /tags/gsdl-21/gsdl/packages/mg/lib/simplefrenchstem.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/simplefrenchstem.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/simplefrenchstem.h	(revision 699)
@@ -0,0 +1,50 @@
+/**************************************************************************
+ *
+ * simplefrenchstem.h -- a simple french stemmer
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:17:04  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/25 07:55:37  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/24 06:37:53  rjmcnab
+
+   Incorporated the french stemmer better.
+
+
+ */
+
+#ifndef SIMPLEFRENCHSTEM_H
+#define SIMPLEFRENCHSTEM_H
+
+void simplefrenchstem (unsigned char *word);
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/lib/sptree.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/sptree.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/sptree.c	(revision 699)
@@ -0,0 +1,391 @@
+/**************************************************************************
+ *
+ * sptree.c -- Splay tree code
+ * Copyright (C) 1994  Gary Eddy, Alistair Moffat and Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#include	"sysfuncs.h"
+#include	"sptree.h"
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:17:04  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:32:45  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/08/22  00:24:50  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#define TRACE(w)
+
+#define POOL_SIZE 1024
+
+
+typedef struct node_rec node;
+
+struct node_rec
+  {
+    node *left, *rght;
+    node *par;
+    char *datsun;
+  };
+
+struct pool
+  {
+    int length, pos;
+    struct pool *next;
+    node nodes[POOL_SIZE];
+  };
+
+static node *SP_splay ();
+
+#define	CNULL	((char *) NULL)
+#define	NNULL	((node *) NULL)
+#define	SNULL	((Splay_Tree *) NULL)
+
+
+/*
+ *    createset():
+ *              allocates memory for and initialises a set.
+ *
+ *      Accesses outside variables:     none
+ *
+ *      Return value...:                none
+ */
+
+Splay_Tree *
+SP_createset (int (*cmp) (void *, void *))
+{
+  Splay_Tree *S;
+
+  if (!(S = (Splay_Tree *) malloc (sizeof (Splay_Tree))))
+    return (SNULL);
+  S->cmp = cmp;
+  S->root = CNULL;
+  S->no_of_items = 0;
+  S->mem_in_use = sizeof (Splay_Tree);
+  S->pool = NULL;
+  S->pool_chunks = 1024;
+  return S;
+}				/* createset() */
+
+
+
+static node *
+SP_get_node (Splay_Tree * S)
+{
+  struct pool *p = S->pool;
+  if (!p || p->pos == p->length)
+    {
+      p = malloc (sizeof (struct pool));
+      if (!p)
+	return NULL;
+      S->mem_in_use += sizeof (struct pool);
+      p->length = POOL_SIZE;
+      p->pos = 0;
+      p->next = S->pool;
+      S->pool = p;
+    }
+  return &(p->nodes[p->pos++]);
+}
+
+
+
+/*
+ *    SP_freeset():
+ *              free the memory allocated to a splay tree. Note this Does not
+ *              free the data in the tree you must do this yourself.
+ *
+ *      Accesses outside variables:     none
+ *
+ *      Return value...:                none
+ */
+
+void 
+SP_freeset (Splay_Tree * S)
+{
+  while (S->pool)
+    {
+      struct pool *p = S->pool;
+      S->pool = p->next;
+      free (p);
+    }
+  free (S);
+}
+
+
+/*
+ *    insert():
+ *              inserts the item given into the set given in the order
+ *              specified by the comparison function in the set 
+ *              definition.
+ *
+ *      Accesses outside variables:     none
+ *
+ *      Return value...:                none
+ */
+
+
+void *
+SP_insert (void *item, Splay_Tree * S)
+{
+  int dir = 0;
+  register node *curr, *prev;
+  node *tmp;
+
+  prev = NNULL;
+  curr = (node *) S->root;
+  for (; curr;)
+    {
+      prev = curr;
+      dir = S->cmp (item, curr->datsun);
+      if (dir < 0)
+	curr = curr->left;
+      else
+	curr = curr->rght;
+    }
+  if (!(tmp = SP_get_node (S)))
+    return NULL;
+  tmp->datsun = item;
+
+  if (prev == NNULL)
+    S->root = (char *) tmp;
+  else if (dir < 0)
+    prev->left = tmp;
+  else
+    prev->rght = tmp;
+
+  tmp->par = prev;
+  tmp->left = NNULL;
+  tmp->rght = NNULL;
+  S->no_of_items++;
+  S->root = (char *) SP_splay (tmp);
+  return (tmp->datsun);
+}				/* insert() */
+
+/*
+ *    get_first():
+ *              returns a pointer to the first item in the set. If the set 
+ *              is empty a NULL pointer is returned.
+ *
+ *      Accesses outside variables:     none
+ */
+
+void *
+SP_get_first (Splay_Tree * S)
+{
+  node *curr, *prev;
+
+  for (curr = (node *) S->root, prev = NNULL; curr;
+       prev = curr, curr = curr->left)
+    ;				/* NULLBODY */
+
+  if (prev)
+    S->root = (char *) SP_splay (prev);
+  return (prev ? prev->datsun : CNULL);
+}				/* get_first() */
+
+/*
+ *    get_next():
+ *              returns the in-order successor of the currently fingered
+ *              item of the set. If no item is fingered or if there is 
+ *              no in-order successor a NULL pointer is returned.
+ *
+ *      Accesses outside variables:     none
+ */
+
+void *
+SP_get_next (Splay_Tree * S)
+{
+  node *curr, *prev;
+
+  curr = (node *) S->root;
+  if (!curr || !curr->rght)
+    return (CNULL);
+
+  for (prev = curr, curr = curr->rght; curr;
+       prev = curr, curr = curr->left)
+    ;				/* NULLBODY */
+
+  if (prev)
+    S->root = (char *) SP_splay (prev);
+  return (prev ? prev->datsun : CNULL);
+}				/* get_next() */
+
+/*
+ *    deletemin():
+ *              removes the first element in the set and returns a
+ *              pointer to it. If the set is empty a NULL pointer 
+ *              is returned.
+ *
+ *      Accesses outside variables:     none
+ */
+
+void *
+SP_deletemin (Splay_Tree * S)
+{
+  node *tmp;
+  char *dat;
+
+  dat = SP_get_first (S);
+  if (dat)
+    {
+      tmp = (node *) S->root;
+      S->root = (char *) tmp->rght;
+      if (S->root)
+	((node *) S->root)->par = NULL;
+    }
+  S->no_of_items--;
+  return dat;
+}				/* deletemin() */
+
+
+/*
+ *    member():
+ *              searches the given set for an item which matches (has
+ *              the same key) and returns a pointer to a copy of that item.
+ *              If no matching item is found a NULL pointer is 
+ *              returned.
+ *
+ *      Accesses outside variables:     none
+ */
+
+void *
+SP_member (void *dat, Splay_Tree * S)
+{
+  register node *curr;
+  register int dir;
+  static int calls = 0;
+
+  TRACE ("in member");
+  for (curr = (node *) S->root; curr;)
+    {
+      TRACE ("call to cmp");
+      dir = S->cmp (dat, curr->datsun);
+      TRACE ("out of cmp");
+      if (dir < 0)
+	curr = curr->left;
+      else if (dir > 0)
+	curr = curr->rght;
+      else
+	{
+	  /* splay occasionaly to speed up execution */
+	  if ((calls++ % 4) == 0 && curr)
+	    S->root = (char *) SP_splay (curr);
+	  TRACE ("out of member");
+	  return (curr->datsun);
+	}
+    }
+  TRACE ("out of member");
+  return (CNULL);
+}				/* member() */
+
+#define	ROTATEL(p,q) 					\
+	do{						\
+		p->par = q->par;			\
+		q->par = p;				\
+		q->rght = p->left;			\
+		if(q->rght)				\
+			q->rght->par = q;		\
+		p->left = q;				\
+		if(p->par)				\
+			if(q == p->par->left)		\
+				p->par->left = p;	\
+			else				\
+				p->par->rght = p;	\
+	}while(0)
+
+#define	ROTATER(p,q) 					\
+	do{						\
+		p->par = q->par;			\
+		q->par = p;				\
+		q->left = p->rght;			\
+		if(q->left)				\
+			q->left->par = q;		\
+		p->rght = q;				\
+		if(p->par)				\
+			if(q == p->par->left)		\
+				p->par->left = p;	\
+			else				\
+				p->par->rght = p;	\
+	}while(0)
+
+
+/*
+ *    splay():
+ *              i
+ *
+ *      Accesses outside variables:     i
+ *
+ *      Return value...:                i
+ */
+
+static node *
+SP_splay (p)
+     register node *p;
+{
+  register node *q, *g;
+
+  q = p->par;
+  /* splay the tree about p */
+  while (q != NNULL)
+    {
+      g = q->par;
+      if (p == q->left)
+	{
+	  if (g == NNULL)
+	    ROTATER (p, q);
+	  else if (q == g->left)
+	    {
+	      ROTATER (q, g);
+	      ROTATER (p, q);
+	    }
+	  else
+	    {
+	      ROTATER (p, q);
+	      ROTATEL (p, g);
+	    }
+	}
+      else
+	{
+	  if (g == NNULL)
+	    ROTATEL (p, q);
+	  else if (q == g->rght)
+	    {
+	      ROTATEL (q, g);
+	      ROTATEL (p, q);
+	    }
+	  else
+	    {
+	      ROTATEL (p, q);
+	      ROTATER (p, g);
+	    }
+	}
+      q = p->par;
+    }
+  return (p);
+}
Index: /tags/gsdl-21/gsdl/packages/mg/lib/sptree.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/sptree.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/sptree.h	(revision 699)
@@ -0,0 +1,47 @@
+/**************************************************************************
+ *
+ * sptree.h -- Splay tree code
+ * Copyright (C) 1994  Gary Eddy and Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef SPTREE_H
+#define SPTREE_H
+
+typedef struct set_rec Splay_Tree;
+
+struct set_rec
+  {
+    int no_of_items;
+    unsigned long mem_in_use;
+    int (*cmp) (void *, void *);
+    void *root;
+    void *pool;
+    int pool_chunks;
+  };
+
+Splay_Tree *SP_createset (int (*cmp) (void *, void *));
+void *SP_insert (void *item, Splay_Tree * S);
+void *SP_get_first (Splay_Tree * S);
+void *SP_get_next (Splay_Tree * S);
+void *SP_deletemin (Splay_Tree * S);
+void *SP_member (void *dat, Splay_Tree * S);
+void SP_freeset (Splay_Tree * S);
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/lib/strcasecmp.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/strcasecmp.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/strcasecmp.c	(revision 699)
@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 1987 Regents of the University of California.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms are permitted
+ * provided that this notice is preserved and that due credit is given
+ * to the University of California at Berkeley. The name of the University
+ * may not be used to endorse or promote products derived from this
+ * software without specific written prior permission. This software
+ * is provided ``as is'' without express or implied warranty.
+ */
+
+#if defined(LIBC_SCCS) && !defined(lint)
+static char sccsid[] = "@(#)strcasecmp.c	5.5 (Berkeley) 11/24/87";
+#endif /* LIBC_SCCS and not lint */
+
+/*
+ * This array is designed for mapping upper and lower case letter
+ * together for a case independent comparison.  The mappings are
+ * based upon ascii character sequences.
+ */
+static unsigned char charmap[] = {
+	'\000', '\001', '\002', '\003', '\004', '\005', '\006', '\007',
+	'\010', '\011', '\012', '\013', '\014', '\015', '\016', '\017',
+	'\020', '\021', '\022', '\023', '\024', '\025', '\026', '\027',
+	'\030', '\031', '\032', '\033', '\034', '\035', '\036', '\037',
+	'\040', '\041', '\042', '\043', '\044', '\045', '\046', '\047',
+	'\050', '\051', '\052', '\053', '\054', '\055', '\056', '\057',
+	'\060', '\061', '\062', '\063', '\064', '\065', '\066', '\067',
+	'\070', '\071', '\072', '\073', '\074', '\075', '\076', '\077',
+	'\100', '\141', '\142', '\143', '\144', '\145', '\146', '\147',
+	'\150', '\151', '\152', '\153', '\154', '\155', '\156', '\157',
+	'\160', '\161', '\162', '\163', '\164', '\165', '\166', '\167',
+	'\170', '\171', '\172', '\133', '\134', '\135', '\136', '\137',
+	'\140', '\141', '\142', '\143', '\144', '\145', '\146', '\147',
+	'\150', '\151', '\152', '\153', '\154', '\155', '\156', '\157',
+	'\160', '\161', '\162', '\163', '\164', '\165', '\166', '\167',
+	'\170', '\171', '\172', '\173', '\174', '\175', '\176', '\177',
+	'\200', '\201', '\202', '\203', '\204', '\205', '\206', '\207',
+	'\210', '\211', '\212', '\213', '\214', '\215', '\216', '\217',
+	'\220', '\221', '\222', '\223', '\224', '\225', '\226', '\227',
+	'\230', '\231', '\232', '\233', '\234', '\235', '\236', '\237',
+	'\240', '\241', '\242', '\243', '\244', '\245', '\246', '\247',
+	'\250', '\251', '\252', '\253', '\254', '\255', '\256', '\257',
+	'\260', '\261', '\262', '\263', '\264', '\265', '\266', '\267',
+	'\270', '\271', '\272', '\273', '\274', '\275', '\276', '\277',
+	'\300', '\341', '\342', '\343', '\344', '\345', '\346', '\347',
+	'\350', '\351', '\352', '\353', '\354', '\355', '\356', '\357',
+	'\360', '\361', '\362', '\363', '\364', '\365', '\366', '\367',
+	'\370', '\371', '\372', '\333', '\334', '\335', '\336', '\337',
+	'\340', '\341', '\342', '\343', '\344', '\345', '\346', '\347',
+	'\350', '\351', '\352', '\353', '\354', '\355', '\356', '\357',
+	'\360', '\361', '\362', '\363', '\364', '\365', '\366', '\367',
+	'\370', '\371', '\372', '\373', '\374', '\375', '\376', '\377',
+};
+
+int
+strcasecmp(s1, s2)
+    char *s1, *s2;
+{
+    register unsigned char u1, u2;
+
+    for (;;) {
+	u1 = (unsigned char) *s1++;
+	u2 = (unsigned char) *s2++;
+	if (charmap[u1] != charmap[u2]) {
+	    return charmap[u1] - charmap[u2];
+	}
+	if (u1 == '\0') {
+	    return 0;
+	}
+    }
+}
+
Index: /tags/gsdl-21/gsdl/packages/mg/lib/strstr.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/strstr.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/strstr.c	(revision 699)
@@ -0,0 +1,44 @@
+/* strstr.c -- return the offset of one string within another
+   Copyright (C) 1989, 1990 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+/* Written by Mike Rendell <michael@cs.mun.ca>.  */
+
+/* Return the starting address of string S2 in S1;
+   return 0 if it is not found. */
+
+char *
+strstr (s1, s2)
+     char *s1;
+     char *s2;
+{
+  int i;
+  char *p1;
+  char *p2;
+  char *s = s1;
+
+  for (p2 = s2, i = 0; *s; p2 = s2, i++, s++)
+    {
+      for (p1 = s; *p1 && *p2 && *p1 == *p2; p1++, p2++)
+	;
+      if (!*p2)
+	break;
+    }
+  if (!*p2)
+    return s1 + i;
+
+  return 0;
+}
Index: /tags/gsdl-21/gsdl/packages/mg/lib/timing.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/timing.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/timing.c	(revision 699)
@@ -0,0 +1,234 @@
+/**************************************************************************
+ *
+ * timing.c -- Program timing routines
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:17:05  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:32:49  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/08/22  00:24:53  tes
+   * Initial placement under CVS.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+#include "timing.h"
+
+
+/*
+ * Return the time in seconds since 00:00:00 GMT, Jan. 1, 1970 to the
+ * best precision possible
+ *
+ */
+double 
+RealTime (void)
+{
+  return ((double) time (NULL));
+}
+
+
+
+/*
+ * Return the amount of system and user CPU used by the process to date to 
+ * the best precision possible. If user is non-null then it is initialised to 
+ * the user time. If sys is non-null then it is initialised to the system time.
+ *
+ */
+double 
+CPUTime (double *user, double *sys)
+{
+
+#if defined(HAVE_TIMES)
+
+  struct tms buffer;
+  static double clk_tck = 0;
+  double u, s;
+
+  times (&buffer);
+
+  if (clk_tck == 0)
+    clk_tck = CLK_TCK;
+
+  u = (double) buffer.tms_utime / clk_tck;
+  s = (double) buffer.tms_stime / clk_tck;
+  if (user)
+    *user = u;
+  if (sys)
+    *sys = s;
+  return u + s;
+
+#elif defined(HAVE_GETRUSAGE)
+
+  struct rusage ruse;
+  getrusage (RUSAGE_SELF, &ruse);
+  if (user)
+    *user = (double) ruse.ru_utime.tv_sec +
+      (double) ruse.ru_utime.tv_usec / 1000000;
+  if (sys)
+    *sys = (double) ruse.ru_stime.tv_sec +
+      (double) ruse.ru_stime.tv_usec / 1000000;
+  return ((double) ruse.ru_utime.tv_sec + ruse.ru_stime.tv_sec +
+	((double) ruse.ru_utime.tv_usec + ruse.ru_stime.tv_usec) / 1000000);
+
+#else
+
+  /* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+  if (user) *user = 0.0;
+  if (sys) *sys = 0.0;
+  return 0.0;
+#else
+  -->Can NOT find a system time routine to use ! !!
+#endif  /* __WIN32__ */
+
+#endif
+
+}
+
+
+
+
+/*
+ * Get the Real and CPU time and store them in a ProgTime structure
+ */
+void 
+GetTime (ProgTime * StartTime)
+{
+  StartTime->RealTime = RealTime ();
+  StartTime->CPUTime = CPUTime (NULL, NULL);
+}
+
+
+
+
+/*
+ * Display the Real and CPU time elapsed since the StartTime anf FinishTime 
+ * structures were initialised. If FinishTime is NULL then FinishTime is
+ * Now.
+ */
+char *
+ElapsedTime (ProgTime * StartTime,
+	     ProgTime * FinishTime)
+{
+  static char buf[50];
+  double Real, CPU, hour, min, sec;
+  if (!FinishTime)
+    {
+      Real = RealTime () - StartTime->RealTime;
+      CPU = CPUTime (NULL, NULL) - StartTime->CPUTime;
+    }
+  else
+    {
+      Real = FinishTime->RealTime - StartTime->RealTime;
+      CPU = FinishTime->CPUTime - StartTime->CPUTime;
+    }
+  hour = floor (CPU / 3600);
+  min = floor ((CPU - hour * 3600) / 60);
+  sec = CPU - hour * 3600 - min * 60;
+  sprintf (buf, "%02.0f:%02.0f:%05.2f cpu, %02d:%02d:%02d elapsed.",
+	   hour, min, sec,
+	   ((int) ceil (Real)) / 3600,
+	   (((int) ceil (Real)) % 3600) / 60, ((int) ceil (Real)) % 60);
+  return (buf);
+}
+
+
+#define MILLION 1000000
+
+/* ===============================================================================
+ * Function: cputime_string
+ * Description:
+ *      Prints out given cpu time into a string buffer.
+ * Input: cpu-time in clock_t or timeval format
+ * Output: pointer to internal buffer 
+ * =============================================================================== */
+
+#ifdef HAVE_TIMES
+char *
+cputime_string (clock_t clk)
+{
+  static char buf[15];
+  double CPU, hour, min, sec;
+
+  CPU = (double) clk / (double) CLK_TCK;
+  hour = floor (CPU / 3600);
+  min = floor ((CPU - hour * 3600) / 60);
+  sec = CPU - hour * 3600 - min * 60;
+  sprintf (buf, "%02.0f:%02.0f:%05.2f", hour, min, sec);
+  return (buf);
+}
+#else
+char *
+cputime_string (struct timeval *t)
+{
+  static char buf[15];
+  double CPU, hour, min, sec;
+
+  CPU = (double) t->tv_sec + (double) t->tv_usec / MILLION;
+  hour = floor (CPU / 3600);
+  min = floor ((CPU - hour * 3600) / 60);
+  sec = CPU - hour * 3600 - min * 60;
+  sprintf (buf, "%02.0f:%02.0f:%05.2f", hour, min, sec);
+  return (buf);
+}
+#endif
+
+
+
+/* ===============================================================================
+ * Function: time_normalise
+ * Description:
+ *      If the micro-second component is negative or over a million,
+ *      then must take the over/under flow and add it to the second component.
+ * Usage:
+ *      Use this routine when you have just been doing straight addition and
+ *      subtraction on the micro-second component. For this could lead to a
+ *      negative number or over a million.
+ *
+ * =============================================================================== */
+
+#ifndef HAVE_TIMES
+
+#ifndef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+static 
+#endif
+void
+time_normalise (struct timeval *t)
+{
+  while (t->tv_usec < 0)
+    {
+      t->tv_usec += MILLION;
+      t->tv_sec--;
+    }
+  while (t->tv_usec > MILLION)
+    {
+      t->tv_usec -= MILLION;
+      t->tv_sec++;
+    }
+}
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/lib/timing.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/timing.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/timing.h	(revision 699)
@@ -0,0 +1,75 @@
+/**************************************************************************
+ *
+ * timing.h -- Program timing routines
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef H_TIMING
+#define H_TIMING
+
+
+#ifdef HAVE_TIMES
+# if TIME_WITH_SYS_TIME
+#  include <sys/times.h>
+#  include <time.h>
+#  ifndef CLK_TCK
+#   define CLK_TCK sysconf(_SC_CLK_TCK)
+#  endif
+# else
+#  if HAVE_SYS_TIME_H
+#   include <sys/time.h>
+#  else
+#   include <time.h>
+#  endif
+# endif
+#else
+/* [RPAP - Feb 97: WIN32 Port] */
+# ifdef __WIN32__
+   struct timeval { long tv_sec, tv_usec; };
+# else
+#  include <sys/time.h>
+#  include <sys/resource.h>
+# endif
+#endif
+
+
+typedef struct
+  {
+    double RealTime, CPUTime;
+  }
+ProgTime;
+
+double RealTime (void);
+
+double CPUTime (double *user, double *sys);
+
+void GetTime (ProgTime * StartTime);
+
+/* FinishTime may be NULL */
+char *ElapsedTime (ProgTime * StartTime, ProgTime * FinishTime);
+
+#ifdef HAVE_TIMES
+char *cputime_string (clock_t clk);
+#else
+void time_normalise (struct timeval *t);
+char *cputime_string (struct timeval *t);
+#endif
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/lib/unitool.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/unitool.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/unitool.c	(revision 699)
@@ -0,0 +1,1756 @@
+#include "unitool.h"
+#include "sysfuncs.h"
+
+
+/* unicode_cat_data is a combination of the letter, digit, and
+ * space information in a form that is quick to look up.
+ * The information about each byte is stored in two bits
+ *
+ * 00 - neither letter, digit or space
+ * 01 - letter
+ * 10 - space
+ * 11 - digit
+ *
+ * You can test for letter | space by looking at the right
+ * column
+ */
+unsigned long *unicode_cat_data=NULL;
+
+
+typedef struct unidecompose {
+  unsigned short ch;
+  unsigned short str[3];
+} unidecompose;
+
+typedef struct unirelate {
+  unsigned short ch1;
+  unsigned short ch2;
+} unirelate;
+
+typedef struct unirange {
+  unsigned short first;
+  unsigned short last;
+} unirange;
+
+
+
+/* This decomposition data only includes canonical decompositional
+ * characters.  Any unused bytes are set to 0x0. */
+
+#define NUM_DECOMPOSE_INFO 1269
+static const unidecompose decompose_info[NUM_DECOMPOSE_INFO] = {
+  {0xC0,0x41,0x300,0x0}, {0xC1,0x41,0x301,0x0}, {0xC2,0x41,0x302,0x0}, 
+  {0xC3,0x41,0x303,0x0}, {0xC4,0x41,0x308,0x0}, {0xC5,0x41,0x30A,0x0}, 
+  {0xC7,0x43,0x327,0x0}, {0xC8,0x45,0x300,0x0}, {0xC9,0x45,0x301,0x0}, 
+  {0xCA,0x45,0x302,0x0}, {0xCB,0x45,0x308,0x0}, {0xCC,0x49,0x300,0x0}, 
+  {0xCD,0x49,0x301,0x0}, {0xCE,0x49,0x302,0x0}, {0xCF,0x49,0x308,0x0}, 
+  {0xD1,0x4E,0x303,0x0}, {0xD2,0x4F,0x300,0x0}, {0xD3,0x4F,0x301,0x0}, 
+  {0xD4,0x4F,0x302,0x0}, {0xD5,0x4F,0x303,0x0}, {0xD6,0x4F,0x308,0x0}, 
+  {0xD9,0x55,0x300,0x0}, {0xDA,0x55,0x301,0x0}, {0xDB,0x55,0x302,0x0}, 
+  {0xDC,0x55,0x308,0x0}, {0xDD,0x59,0x301,0x0}, {0xE0,0x61,0x300,0x0}, 
+  {0xE1,0x61,0x301,0x0}, {0xE2,0x61,0x302,0x0}, {0xE3,0x61,0x303,0x0}, 
+  {0xE4,0x61,0x308,0x0}, {0xE5,0x61,0x30A,0x0}, {0xE7,0x63,0x327,0x0}, 
+  {0xE8,0x65,0x300,0x0}, {0xE9,0x65,0x301,0x0}, {0xEA,0x65,0x302,0x0}, 
+  {0xEB,0x65,0x308,0x0}, {0xEC,0x69,0x300,0x0}, {0xED,0x69,0x301,0x0}, 
+  {0xEE,0x69,0x302,0x0}, {0xEF,0x69,0x308,0x0}, {0xF1,0x6E,0x303,0x0}, 
+  {0xF2,0x6F,0x300,0x0}, {0xF3,0x6F,0x301,0x0}, {0xF4,0x6F,0x302,0x0}, 
+  {0xF5,0x6F,0x303,0x0}, {0xF6,0x6F,0x308,0x0}, {0xF9,0x75,0x300,0x0}, 
+  {0xFA,0x75,0x301,0x0}, {0xFB,0x75,0x302,0x0}, {0xFC,0x75,0x308,0x0}, 
+  {0xFD,0x79,0x301,0x0}, {0xFF,0x79,0x308,0x0}, {0x100,0x41,0x304,0x0}, 
+  {0x101,0x61,0x304,0x0}, {0x102,0x41,0x306,0x0}, {0x103,0x61,0x306,0x0}, 
+  {0x104,0x41,0x328,0x0}, {0x105,0x61,0x328,0x0}, {0x106,0x43,0x301,0x0}, 
+  {0x107,0x63,0x301,0x0}, {0x108,0x43,0x302,0x0}, {0x109,0x63,0x302,0x0}, 
+  {0x10A,0x43,0x307,0x0}, {0x10B,0x63,0x307,0x0}, {0x10C,0x43,0x30C,0x0}, 
+  {0x10D,0x63,0x30C,0x0}, {0x10E,0x44,0x30C,0x0}, {0x10F,0x64,0x30C,0x0}, 
+  {0x112,0x45,0x304,0x0}, {0x113,0x65,0x304,0x0}, {0x114,0x45,0x306,0x0}, 
+  {0x115,0x65,0x306,0x0}, {0x116,0x45,0x307,0x0}, {0x117,0x65,0x307,0x0}, 
+  {0x118,0x45,0x328,0x0}, {0x119,0x65,0x328,0x0}, {0x11A,0x45,0x30C,0x0}, 
+  {0x11B,0x65,0x30C,0x0}, {0x11C,0x47,0x302,0x0}, {0x11D,0x67,0x302,0x0}, 
+  {0x11E,0x47,0x306,0x0}, {0x11F,0x67,0x306,0x0}, {0x120,0x47,0x307,0x0}, 
+  {0x121,0x67,0x307,0x0}, {0x122,0x47,0x327,0x0}, {0x123,0x67,0x327,0x0}, 
+  {0x124,0x48,0x302,0x0}, {0x125,0x68,0x302,0x0}, {0x128,0x49,0x303,0x0}, 
+  {0x129,0x69,0x303,0x0}, {0x12A,0x49,0x304,0x0}, {0x12B,0x69,0x304,0x0}, 
+  {0x12C,0x49,0x306,0x0}, {0x12D,0x69,0x306,0x0}, {0x12E,0x49,0x328,0x0}, 
+  {0x12F,0x69,0x328,0x0}, {0x130,0x49,0x307,0x0}, {0x134,0x4A,0x302,0x0}, 
+  {0x135,0x6A,0x302,0x0}, {0x136,0x4B,0x327,0x0}, {0x137,0x6B,0x327,0x0}, 
+  {0x139,0x4C,0x301,0x0}, {0x13A,0x6C,0x301,0x0}, {0x13B,0x4C,0x327,0x0}, 
+  {0x13C,0x6C,0x327,0x0}, {0x13D,0x4C,0x30C,0x0}, {0x13E,0x6C,0x30C,0x0}, 
+  {0x143,0x4E,0x301,0x0}, {0x144,0x6E,0x301,0x0}, {0x145,0x4E,0x327,0x0}, 
+  {0x146,0x6E,0x327,0x0}, {0x147,0x4E,0x30C,0x0}, {0x148,0x6E,0x30C,0x0}, 
+  {0x14C,0x4F,0x304,0x0}, {0x14D,0x6F,0x304,0x0}, {0x14E,0x4F,0x306,0x0}, 
+  {0x14F,0x6F,0x306,0x0}, {0x150,0x4F,0x30B,0x0}, {0x151,0x6F,0x30B,0x0}, 
+  {0x154,0x52,0x301,0x0}, {0x155,0x72,0x301,0x0}, {0x156,0x52,0x327,0x0}, 
+  {0x157,0x72,0x327,0x0}, {0x158,0x52,0x30C,0x0}, {0x159,0x72,0x30C,0x0}, 
+  {0x15A,0x53,0x301,0x0}, {0x15B,0x73,0x301,0x0}, {0x15C,0x53,0x302,0x0}, 
+  {0x15D,0x73,0x302,0x0}, {0x15E,0x53,0x327,0x0}, {0x15F,0x73,0x327,0x0}, 
+  {0x160,0x53,0x30C,0x0}, {0x161,0x73,0x30C,0x0}, {0x162,0x54,0x327,0x0}, 
+  {0x163,0x74,0x327,0x0}, {0x164,0x54,0x30C,0x0}, {0x165,0x74,0x30C,0x0}, 
+  {0x168,0x55,0x303,0x0}, {0x169,0x75,0x303,0x0}, {0x16A,0x55,0x304,0x0}, 
+  {0x16B,0x75,0x304,0x0}, {0x16C,0x55,0x306,0x0}, {0x16D,0x75,0x306,0x0}, 
+  {0x16E,0x55,0x30A,0x0}, {0x16F,0x75,0x30A,0x0}, {0x170,0x55,0x30B,0x0}, 
+  {0x171,0x75,0x30B,0x0}, {0x172,0x55,0x328,0x0}, {0x173,0x75,0x328,0x0}, 
+  {0x174,0x57,0x302,0x0}, {0x175,0x77,0x302,0x0}, {0x176,0x59,0x302,0x0}, 
+  {0x177,0x79,0x302,0x0}, {0x178,0x59,0x308,0x0}, {0x179,0x5A,0x301,0x0}, 
+  {0x17A,0x7A,0x301,0x0}, {0x17B,0x5A,0x307,0x0}, {0x17C,0x7A,0x307,0x0}, 
+  {0x17D,0x5A,0x30C,0x0}, {0x17E,0x7A,0x30C,0x0}, {0x1A0,0x4F,0x31B,0x0}, 
+  {0x1A1,0x6F,0x31B,0x0}, {0x1AF,0x55,0x31B,0x0}, {0x1B0,0x75,0x31B,0x0}, 
+  {0x1CD,0x41,0x30C,0x0}, {0x1CE,0x61,0x30C,0x0}, {0x1CF,0x49,0x30C,0x0}, 
+  {0x1D0,0x69,0x30C,0x0}, {0x1D1,0x4F,0x30C,0x0}, {0x1D2,0x6F,0x30C,0x0}, 
+  {0x1D3,0x55,0x30C,0x0}, {0x1D4,0x75,0x30C,0x0}, {0x1D5,0xDC,0x304,0x0}, 
+  {0x1D6,0xFC,0x304,0x0}, {0x1D7,0xDC,0x301,0x0}, {0x1D8,0xFC,0x301,0x0}, 
+  {0x1D9,0xDC,0x30C,0x0}, {0x1DA,0xFC,0x30C,0x0}, {0x1DB,0xDC,0x300,0x0}, 
+  {0x1DC,0xFC,0x300,0x0}, {0x1DE,0xC4,0x304,0x0}, {0x1DF,0xE4,0x304,0x0}, 
+  {0x1E0,0x41,0x307,0x304}, {0x1E1,0x61,0x307,0x304}, {0x1E2,0xC6,0x304,0x0}, 
+  {0x1E3,0xE6,0x304,0x0}, {0x1E6,0x47,0x30C,0x0}, {0x1E7,0x67,0x30C,0x0}, 
+  {0x1E8,0x4B,0x30C,0x0}, {0x1E9,0x6B,0x30C,0x0}, {0x1EA,0x4F,0x328,0x0}, 
+  {0x1EB,0x6F,0x328,0x0}, {0x1EC,0x1EA,0x304,0x0}, {0x1ED,0x1EB,0x304,0x0}, 
+  {0x1EE,0x1B7,0x30C,0x0}, {0x1EF,0x292,0x30C,0x0}, {0x1F0,0x6A,0x30C,0x0}, 
+  {0x1F4,0x47,0x301,0x0}, {0x1F5,0x67,0x301,0x0}, {0x1FA,0xC5,0x301,0x0}, 
+  {0x1FB,0xE5,0x301,0x0}, {0x1FC,0xC6,0x301,0x0}, {0x1FD,0xE6,0x301,0x0}, 
+  {0x1FE,0xD8,0x301,0x0}, {0x1FF,0xF8,0x301,0x0}, {0x200,0x41,0x30F,0x0}, 
+  {0x201,0x61,0x30F,0x0}, {0x202,0x41,0x311,0x0}, {0x203,0x61,0x311,0x0}, 
+  {0x204,0x45,0x30F,0x0}, {0x205,0x65,0x30F,0x0}, {0x206,0x45,0x311,0x0}, 
+  {0x207,0x65,0x311,0x0}, {0x208,0x49,0x30F,0x0}, {0x209,0x69,0x30F,0x0}, 
+  {0x20A,0x49,0x311,0x0}, {0x20B,0x69,0x311,0x0}, {0x20C,0x4F,0x30F,0x0}, 
+  {0x20D,0x6F,0x30F,0x0}, {0x20E,0x4F,0x311,0x0}, {0x20F,0x6F,0x311,0x0}, 
+  {0x210,0x52,0x30F,0x0}, {0x211,0x72,0x30F,0x0}, {0x212,0x52,0x311,0x0}, 
+  {0x213,0x72,0x311,0x0}, {0x214,0x55,0x30F,0x0}, {0x215,0x75,0x30F,0x0}, 
+  {0x216,0x55,0x311,0x0}, {0x217,0x75,0x311,0x0}, {0x310,0x306,0x307,0x0}, 
+  {0x340,0x300,0x0,0x0}, {0x341,0x301,0x0,0x0}, {0x343,0x313,0x0,0x0}, 
+  {0x344,0x308,0x30D,0x0}, {0x374,0x2B9,0x0,0x0}, {0x37E,0x3B,0x0,0x0}, 
+  {0x385,0xA8,0x30D,0x0}, {0x386,0x391,0x30D,0x0}, {0x387,0xB7,0x0,0x0}, 
+  {0x388,0x395,0x30D,0x0}, {0x389,0x397,0x30D,0x0}, {0x38A,0x399,0x30D,0x0}, 
+  {0x38C,0x39F,0x30D,0x0}, {0x38E,0x3A5,0x30D,0x0}, {0x38F,0x3A9,0x30D,0x0}, 
+  {0x390,0x3B9,0x344,0x0}, {0x3AA,0x399,0x308,0x0}, {0x3AB,0x3A5,0x308,0x0}, 
+  {0x3AC,0x3B1,0x30D,0x0}, {0x3AD,0x3B5,0x30D,0x0}, {0x3AE,0x3B7,0x30D,0x0}, 
+  {0x3AF,0x3B9,0x30D,0x0}, {0x3B0,0x3C5,0x344,0x0}, {0x3CA,0x3B9,0x308,0x0}, 
+  {0x3CB,0x3C5,0x308,0x0}, {0x3CC,0x3BF,0x30D,0x0}, {0x3CD,0x3C5,0x30D,0x0}, 
+  {0x3CE,0x3C9,0x30D,0x0}, {0x3D3,0x3D2,0x30D,0x0}, {0x3D4,0x3D2,0x308,0x0}, 
+  {0x401,0x415,0x308,0x0}, {0x403,0x413,0x301,0x0}, {0x407,0x406,0x308,0x0}, 
+  {0x40C,0x41A,0x301,0x0}, {0x40E,0x423,0x306,0x0}, {0x419,0x418,0x306,0x0}, 
+  {0x439,0x438,0x306,0x0}, {0x451,0x435,0x308,0x0}, {0x453,0x433,0x301,0x0}, 
+  {0x457,0x456,0x308,0x0}, {0x45C,0x43A,0x301,0x0}, {0x45E,0x443,0x306,0x0}, 
+  {0x476,0x474,0x30F,0x0}, {0x477,0x475,0x30F,0x0}, {0x4C1,0x416,0x306,0x0}, 
+  {0x4C2,0x436,0x306,0x0}, {0x4D0,0x410,0x306,0x0}, {0x4D1,0x430,0x306,0x0}, 
+  {0x4D2,0x410,0x308,0x0}, {0x4D3,0x430,0x308,0x0}, {0x4D6,0x415,0x306,0x0}, 
+  {0x4D7,0x435,0x306,0x0}, {0x4DA,0x4D8,0x308,0x0}, {0x4DB,0x4D9,0x308,0x0}, 
+  {0x4DC,0x416,0x308,0x0}, {0x4DD,0x436,0x308,0x0}, {0x4DE,0x417,0x308,0x0}, 
+  {0x4DF,0x437,0x308,0x0}, {0x4E2,0x418,0x304,0x0}, {0x4E3,0x438,0x304,0x0}, 
+  {0x4E4,0x418,0x308,0x0}, {0x4E5,0x438,0x308,0x0}, {0x4E6,0x41E,0x308,0x0}, 
+  {0x4E7,0x43E,0x308,0x0}, {0x4EA,0x4E8,0x308,0x0}, {0x4EB,0x4E9,0x308,0x0}, 
+  {0x4EE,0x423,0x304,0x0}, {0x4EF,0x443,0x304,0x0}, {0x4F0,0x423,0x308,0x0}, 
+  {0x4F1,0x443,0x308,0x0}, {0x4F2,0x423,0x30B,0x0}, {0x4F3,0x443,0x30B,0x0}, 
+  {0x4F4,0x427,0x308,0x0}, {0x4F5,0x447,0x308,0x0}, {0x4F8,0x42B,0x308,0x0}, 
+  {0x4F9,0x44B,0x308,0x0}, {0x929,0x928,0x93C,0x0}, {0x931,0x930,0x93C,0x0}, 
+  {0x934,0x933,0x93C,0x0}, {0x958,0x915,0x93C,0x0}, {0x959,0x916,0x93C,0x0}, 
+  {0x95A,0x917,0x93C,0x0}, {0x95B,0x91C,0x93C,0x0}, {0x95C,0x921,0x93C,0x0}, 
+  {0x95D,0x922,0x93C,0x0}, {0x95E,0x92B,0x93C,0x0}, {0x95F,0x92F,0x93C,0x0}, 
+  {0x9B0,0x9AC,0x9BC,0x0}, {0x9CB,0x9C7,0x9BE,0x0}, {0x9CC,0x9C7,0x9D7,0x0}, 
+  {0x9DC,0x9A1,0x9BC,0x0}, {0x9DD,0x9A2,0x9BC,0x0}, {0x9DF,0x9AF,0x9BC,0x0}, 
+  {0xA59,0xA16,0xA3C,0x0}, {0xA5A,0xA17,0xA3C,0x0}, {0xA5B,0xA1C,0xA3C,0x0}, 
+  {0xA5C,0xA21,0xA3C,0x0}, {0xA5E,0xA2B,0xA3C,0x0}, {0xB48,0xB47,0xB56,0x0}, 
+  {0xB4B,0xB47,0xB3E,0x0}, {0xB4C,0xB47,0xB57,0x0}, {0xB5C,0xB21,0xB3C,0x0}, 
+  {0xB5D,0xB22,0xB3C,0x0}, {0xB5F,0xB2F,0xB3C,0x0}, {0xB94,0xB92,0xBD7,0x0}, 
+  {0xBCA,0xBC6,0xBBE,0x0}, {0xBCB,0xBC7,0xBBE,0x0}, {0xBCC,0xBC6,0xBD7,0x0}, 
+  {0xC48,0xC46,0xC56,0x0}, {0xCC0,0xCBF,0xCD5,0x0}, {0xCC7,0xCC6,0xCD5,0x0}, 
+  {0xCC8,0xCC6,0xCD6,0x0}, {0xCCA,0xCC6,0xCC2,0x0}, {0xCCB,0xCC6,0xCC2,0xCD5}, 
+  {0xD4A,0xD46,0xD3E,0x0}, {0xD4B,0xD47,0xD3E,0x0}, {0xD4C,0xD46,0xD57,0x0}, 
+  {0xE33,0xE4D,0xE32,0x0}, {0xEB3,0xECD,0xEB2,0x0}, {0xF43,0xF42,0xFB7,0x0}, 
+  {0xF4D,0xF4C,0xFB7,0x0}, {0xF52,0xF51,0xFB7,0x0}, {0xF57,0xF56,0xFB7,0x0}, 
+  {0xF5C,0xF5B,0xFB7,0x0}, {0xF69,0xF40,0xFB5,0x0}, {0xF73,0xF72,0xF71,0x0}, 
+  {0xF75,0xF71,0xF74,0x0}, {0xF76,0xFB2,0xF80,0x0}, {0xF77,0xF76,0xF71,0x0}, 
+  {0xF78,0xFB3,0xF80,0x0}, {0xF79,0xF78,0xF71,0x0}, {0xF81,0xF80,0xF71,0x0}, 
+  {0xF93,0xF92,0xFB7,0x0}, {0xF9D,0xF9C,0xFB7,0x0}, {0xFA2,0xFA1,0xFB7,0x0}, 
+  {0xFA7,0xFA6,0xFB7,0x0}, {0xFAC,0xFAB,0xFB7,0x0}, {0xFB9,0xF90,0xFB5,0x0}, 
+  {0x1E00,0x41,0x325,0x0}, {0x1E01,0x61,0x325,0x0}, {0x1E02,0x42,0x307,0x0}, 
+  {0x1E03,0x62,0x307,0x0}, {0x1E04,0x42,0x323,0x0}, {0x1E05,0x62,0x323,0x0}, 
+  {0x1E06,0x42,0x331,0x0}, {0x1E07,0x62,0x331,0x0}, {0x1E08,0xC7,0x301,0x0}, 
+  {0x1E09,0xE7,0x301,0x0}, {0x1E0A,0x44,0x307,0x0}, {0x1E0B,0x64,0x307,0x0}, 
+  {0x1E0C,0x44,0x323,0x0}, {0x1E0D,0x64,0x323,0x0}, {0x1E0E,0x44,0x331,0x0}, 
+  {0x1E0F,0x64,0x331,0x0}, {0x1E10,0x44,0x327,0x0}, {0x1E11,0x64,0x327,0x0}, 
+  {0x1E12,0x44,0x32D,0x0}, {0x1E13,0x64,0x32D,0x0}, {0x1E14,0x112,0x300,0x0}, 
+  {0x1E15,0x113,0x300,0x0}, {0x1E16,0x112,0x301,0x0}, {0x1E17,0x113,0x301,0x0}, 
+  {0x1E18,0x45,0x32D,0x0}, {0x1E19,0x65,0x32D,0x0}, {0x1E1A,0x45,0x330,0x0}, 
+  {0x1E1B,0x65,0x330,0x0}, {0x1E1C,0x114,0x327,0x0}, {0x1E1D,0x115,0x327,0x0}, 
+  {0x1E1E,0x46,0x307,0x0}, {0x1E1F,0x66,0x307,0x0}, {0x1E20,0x47,0x304,0x0}, 
+  {0x1E21,0x67,0x304,0x0}, {0x1E22,0x48,0x307,0x0}, {0x1E23,0x68,0x307,0x0}, 
+  {0x1E24,0x48,0x323,0x0}, {0x1E25,0x68,0x323,0x0}, {0x1E26,0x48,0x308,0x0}, 
+  {0x1E27,0x68,0x308,0x0}, {0x1E28,0x48,0x327,0x0}, {0x1E29,0x68,0x327,0x0}, 
+  {0x1E2A,0x48,0x32E,0x0}, {0x1E2B,0x68,0x32E,0x0}, {0x1E2C,0x49,0x330,0x0}, 
+  {0x1E2D,0x69,0x330,0x0}, {0x1E2E,0xCF,0x301,0x0}, {0x1E2F,0xEF,0x301,0x0}, 
+  {0x1E30,0x4B,0x301,0x0}, {0x1E31,0x6B,0x301,0x0}, {0x1E32,0x4B,0x323,0x0}, 
+  {0x1E33,0x6B,0x323,0x0}, {0x1E34,0x4B,0x331,0x0}, {0x1E35,0x6B,0x331,0x0}, 
+  {0x1E36,0x4C,0x323,0x0}, {0x1E37,0x6C,0x323,0x0}, {0x1E38,0x1E36,0x304,0x0}, 
+  {0x1E39,0x1E37,0x304,0x0}, {0x1E3A,0x4C,0x331,0x0}, {0x1E3B,0x6C,0x331,0x0}, 
+  {0x1E3C,0x4C,0x32D,0x0}, {0x1E3D,0x6C,0x32D,0x0}, {0x1E3E,0x4D,0x301,0x0}, 
+  {0x1E3F,0x6D,0x301,0x0}, {0x1E40,0x4D,0x307,0x0}, {0x1E41,0x6D,0x307,0x0}, 
+  {0x1E42,0x4D,0x323,0x0}, {0x1E43,0x6D,0x323,0x0}, {0x1E44,0x4E,0x307,0x0}, 
+  {0x1E45,0x6E,0x307,0x0}, {0x1E46,0x4E,0x323,0x0}, {0x1E47,0x6E,0x323,0x0}, 
+  {0x1E48,0x4E,0x331,0x0}, {0x1E49,0x6E,0x331,0x0}, {0x1E4A,0x4E,0x32D,0x0}, 
+  {0x1E4B,0x6E,0x32D,0x0}, {0x1E4C,0xD5,0x301,0x0}, {0x1E4D,0xF5,0x301,0x0}, 
+  {0x1E4E,0xD5,0x308,0x0}, {0x1E4F,0xF5,0x308,0x0}, {0x1E50,0x14C,0x300,0x0}, 
+  {0x1E51,0x14D,0x300,0x0}, {0x1E52,0x14C,0x301,0x0}, {0x1E53,0x14D,0x301,0x0}, 
+  {0x1E54,0x50,0x301,0x0}, {0x1E55,0x70,0x301,0x0}, {0x1E56,0x50,0x307,0x0}, 
+  {0x1E57,0x70,0x307,0x0}, {0x1E58,0x52,0x307,0x0}, {0x1E59,0x72,0x307,0x0}, 
+  {0x1E5A,0x52,0x323,0x0}, {0x1E5B,0x72,0x323,0x0}, {0x1E5C,0x1E5A,0x304,0x0}, 
+  {0x1E5D,0x1E5B,0x304,0x0}, {0x1E5E,0x52,0x331,0x0}, {0x1E5F,0x72,0x331,0x0}, 
+  {0x1E60,0x53,0x307,0x0}, {0x1E61,0x73,0x307,0x0}, {0x1E62,0x53,0x323,0x0}, 
+  {0x1E63,0x73,0x323,0x0}, {0x1E64,0x15A,0x307,0x0}, {0x1E65,0x15B,0x307,0x0}, 
+  {0x1E66,0x160,0x307,0x0}, {0x1E67,0x161,0x307,0x0}, {0x1E68,0x1E62,0x307,0x0}, 
+  {0x1E69,0x1E63,0x307,0x0}, {0x1E6A,0x54,0x307,0x0}, {0x1E6B,0x74,0x307,0x0}, 
+  {0x1E6C,0x54,0x323,0x0}, {0x1E6D,0x74,0x323,0x0}, {0x1E6E,0x54,0x331,0x0}, 
+  {0x1E6F,0x74,0x331,0x0}, {0x1E70,0x54,0x32D,0x0}, {0x1E71,0x74,0x32D,0x0}, 
+  {0x1E72,0x55,0x324,0x0}, {0x1E73,0x75,0x324,0x0}, {0x1E74,0x55,0x330,0x0}, 
+  {0x1E75,0x75,0x330,0x0}, {0x1E76,0x55,0x32D,0x0}, {0x1E77,0x75,0x32D,0x0}, 
+  {0x1E78,0x168,0x301,0x0}, {0x1E79,0x169,0x301,0x0}, {0x1E7A,0x16A,0x308,0x0}, 
+  {0x1E7B,0x16B,0x308,0x0}, {0x1E7C,0x56,0x303,0x0}, {0x1E7D,0x76,0x303,0x0}, 
+  {0x1E7E,0x56,0x323,0x0}, {0x1E7F,0x76,0x323,0x0}, {0x1E80,0x57,0x300,0x0}, 
+  {0x1E81,0x77,0x300,0x0}, {0x1E82,0x57,0x301,0x0}, {0x1E83,0x77,0x301,0x0}, 
+  {0x1E84,0x57,0x308,0x0}, {0x1E85,0x77,0x308,0x0}, {0x1E86,0x57,0x307,0x0}, 
+  {0x1E87,0x77,0x307,0x0}, {0x1E88,0x57,0x323,0x0}, {0x1E89,0x77,0x323,0x0}, 
+  {0x1E8A,0x58,0x307,0x0}, {0x1E8B,0x78,0x307,0x0}, {0x1E8C,0x58,0x308,0x0}, 
+  {0x1E8D,0x78,0x308,0x0}, {0x1E8E,0x59,0x307,0x0}, {0x1E8F,0x79,0x307,0x0}, 
+  {0x1E90,0x5A,0x302,0x0}, {0x1E91,0x7A,0x302,0x0}, {0x1E92,0x5A,0x323,0x0}, 
+  {0x1E93,0x7A,0x323,0x0}, {0x1E94,0x5A,0x331,0x0}, {0x1E95,0x7A,0x331,0x0}, 
+  {0x1E96,0x68,0x331,0x0}, {0x1E97,0x74,0x308,0x0}, {0x1E98,0x77,0x30A,0x0}, 
+  {0x1E99,0x79,0x30A,0x0}, {0x1E9B,0x17F,0x307,0x0}, {0x1EA0,0x41,0x323,0x0}, 
+  {0x1EA1,0x61,0x323,0x0}, {0x1EA2,0x41,0x309,0x0}, {0x1EA3,0x61,0x309,0x0}, 
+  {0x1EA4,0xC2,0x301,0x0}, {0x1EA5,0xE2,0x301,0x0}, {0x1EA6,0xC2,0x300,0x0}, 
+  {0x1EA7,0xE2,0x300,0x0}, {0x1EA8,0xC2,0x309,0x0}, {0x1EA9,0xE2,0x309,0x0}, 
+  {0x1EAA,0xC2,0x303,0x0}, {0x1EAB,0xE2,0x303,0x0}, {0x1EAC,0xC2,0x323,0x0}, 
+  {0x1EAD,0xE2,0x323,0x0}, {0x1EAE,0x102,0x301,0x0}, {0x1EAF,0x103,0x301,0x0}, 
+  {0x1EB0,0x102,0x300,0x0}, {0x1EB1,0x103,0x300,0x0}, {0x1EB2,0x102,0x309,0x0}, 
+  {0x1EB3,0x103,0x309,0x0}, {0x1EB4,0x102,0x303,0x0}, {0x1EB5,0x103,0x303,0x0}, 
+  {0x1EB6,0x102,0x323,0x0}, {0x1EB7,0x103,0x323,0x0}, {0x1EB8,0x45,0x323,0x0}, 
+  {0x1EB9,0x65,0x323,0x0}, {0x1EBA,0x45,0x309,0x0}, {0x1EBB,0x65,0x309,0x0}, 
+  {0x1EBC,0x45,0x303,0x0}, {0x1EBD,0x65,0x303,0x0}, {0x1EBE,0xCA,0x301,0x0}, 
+  {0x1EBF,0xEA,0x301,0x0}, {0x1EC0,0xCA,0x300,0x0}, {0x1EC1,0xEA,0x300,0x0}, 
+  {0x1EC2,0xCA,0x309,0x0}, {0x1EC3,0xEA,0x309,0x0}, {0x1EC4,0xCA,0x303,0x0}, 
+  {0x1EC5,0xEA,0x303,0x0}, {0x1EC6,0xCA,0x323,0x0}, {0x1EC7,0xEA,0x323,0x0}, 
+  {0x1EC8,0x49,0x309,0x0}, {0x1EC9,0x69,0x309,0x0}, {0x1ECA,0x49,0x323,0x0}, 
+  {0x1ECB,0x69,0x323,0x0}, {0x1ECC,0x4F,0x323,0x0}, {0x1ECD,0x6F,0x323,0x0}, 
+  {0x1ECE,0x4F,0x309,0x0}, {0x1ECF,0x6F,0x309,0x0}, {0x1ED0,0xD4,0x301,0x0}, 
+  {0x1ED1,0xF4,0x301,0x0}, {0x1ED2,0xD4,0x300,0x0}, {0x1ED3,0xF4,0x300,0x0}, 
+  {0x1ED4,0xD4,0x309,0x0}, {0x1ED5,0xF4,0x309,0x0}, {0x1ED6,0xD4,0x303,0x0}, 
+  {0x1ED7,0xF4,0x303,0x0}, {0x1ED8,0xD4,0x323,0x0}, {0x1ED9,0xF4,0x323,0x0}, 
+  {0x1EDA,0x1A0,0x301,0x0}, {0x1EDB,0x1A1,0x301,0x0}, {0x1EDC,0x1A0,0x300,0x0}, 
+  {0x1EDD,0x1A1,0x300,0x0}, {0x1EDE,0x1A0,0x309,0x0}, {0x1EDF,0x1A1,0x309,0x0}, 
+  {0x1EE0,0x1A0,0x303,0x0}, {0x1EE1,0x1A1,0x303,0x0}, {0x1EE2,0x1A0,0x323,0x0}, 
+  {0x1EE3,0x1A1,0x323,0x0}, {0x1EE4,0x55,0x323,0x0}, {0x1EE5,0x75,0x323,0x0}, 
+  {0x1EE6,0x55,0x309,0x0}, {0x1EE7,0x75,0x309,0x0}, {0x1EE8,0x1AF,0x301,0x0}, 
+  {0x1EE9,0x1B0,0x301,0x0}, {0x1EEA,0x1AF,0x300,0x0}, {0x1EEB,0x1B0,0x300,0x0}, 
+  {0x1EEC,0x1AF,0x309,0x0}, {0x1EED,0x1B0,0x309,0x0}, {0x1EEE,0x1AF,0x303,0x0}, 
+  {0x1EEF,0x1B0,0x303,0x0}, {0x1EF0,0x1AF,0x323,0x0}, {0x1EF1,0x1B0,0x323,0x0}, 
+  {0x1EF2,0x59,0x300,0x0}, {0x1EF3,0x79,0x300,0x0}, {0x1EF4,0x59,0x323,0x0}, 
+  {0x1EF5,0x79,0x323,0x0}, {0x1EF6,0x59,0x309,0x0}, {0x1EF7,0x79,0x309,0x0}, 
+  {0x1EF8,0x59,0x303,0x0}, {0x1EF9,0x79,0x303,0x0}, {0x1F00,0x3B1,0x313,0x0}, 
+  {0x1F01,0x3B1,0x314,0x0}, {0x1F02,0x1F00,0x300,0x0}, {0x1F03,0x1F01,0x300,0x0}, 
+  {0x1F04,0x1F00,0x301,0x0}, {0x1F05,0x1F01,0x301,0x0}, {0x1F06,0x1F00,0x342,0x0}, 
+  {0x1F07,0x1F01,0x342,0x0}, {0x1F08,0x391,0x313,0x0}, {0x1F09,0x391,0x314,0x0}, 
+  {0x1F0A,0x1F08,0x300,0x0}, {0x1F0B,0x1F09,0x300,0x0}, {0x1F0C,0x1F08,0x301,0x0}, 
+  {0x1F0D,0x1F09,0x301,0x0}, {0x1F0E,0x1F08,0x342,0x0}, {0x1F0F,0x1F09,0x342,0x0}, 
+  {0x1F10,0x3B5,0x313,0x0}, {0x1F11,0x3B5,0x314,0x0}, {0x1F12,0x1F10,0x300,0x0}, 
+  {0x1F13,0x1F11,0x300,0x0}, {0x1F14,0x1F10,0x301,0x0}, {0x1F15,0x1F11,0x301,0x0}, 
+  {0x1F18,0x395,0x313,0x0}, {0x1F19,0x395,0x314,0x0}, {0x1F1A,0x1F18,0x300,0x0}, 
+  {0x1F1B,0x1F19,0x300,0x0}, {0x1F1C,0x1F18,0x301,0x0}, {0x1F1D,0x1F19,0x301,0x0}, 
+  {0x1F20,0x3B7,0x313,0x0}, {0x1F21,0x3B7,0x314,0x0}, {0x1F22,0x1F20,0x300,0x0}, 
+  {0x1F23,0x1F21,0x300,0x0}, {0x1F24,0x1F20,0x301,0x0}, {0x1F25,0x1F21,0x301,0x0}, 
+  {0x1F26,0x1F20,0x342,0x0}, {0x1F27,0x1F21,0x342,0x0}, {0x1F28,0x397,0x313,0x0}, 
+  {0x1F29,0x397,0x314,0x0}, {0x1F2A,0x1F28,0x300,0x0}, {0x1F2B,0x1F29,0x300,0x0}, 
+  {0x1F2C,0x1F28,0x301,0x0}, {0x1F2D,0x1F29,0x301,0x0}, {0x1F2E,0x1F28,0x342,0x0}, 
+  {0x1F2F,0x1F29,0x342,0x0}, {0x1F30,0x3B9,0x313,0x0}, {0x1F31,0x3B9,0x314,0x0}, 
+  {0x1F32,0x1F30,0x300,0x0}, {0x1F33,0x1F31,0x300,0x0}, {0x1F34,0x1F30,0x301,0x0}, 
+  {0x1F35,0x1F31,0x301,0x0}, {0x1F36,0x1F30,0x342,0x0}, {0x1F37,0x1F31,0x342,0x0}, 
+  {0x1F38,0x399,0x313,0x0}, {0x1F39,0x399,0x314,0x0}, {0x1F3A,0x1F38,0x300,0x0}, 
+  {0x1F3B,0x1F39,0x300,0x0}, {0x1F3C,0x1F38,0x301,0x0}, {0x1F3D,0x1F39,0x301,0x0}, 
+  {0x1F3E,0x1F38,0x342,0x0}, {0x1F3F,0x1F39,0x342,0x0}, {0x1F40,0x3BF,0x313,0x0}, 
+  {0x1F41,0x3BF,0x314,0x0}, {0x1F42,0x1F40,0x300,0x0}, {0x1F43,0x1F41,0x300,0x0}, 
+  {0x1F44,0x1F40,0x301,0x0}, {0x1F45,0x1F41,0x301,0x0}, {0x1F48,0x39F,0x313,0x0}, 
+  {0x1F49,0x39F,0x314,0x0}, {0x1F4A,0x1F48,0x300,0x0}, {0x1F4B,0x1F49,0x300,0x0}, 
+  {0x1F4C,0x1F48,0x301,0x0}, {0x1F4D,0x1F49,0x301,0x0}, {0x1F50,0x3C5,0x313,0x0}, 
+  {0x1F51,0x3C5,0x314,0x0}, {0x1F52,0x1F50,0x300,0x0}, {0x1F53,0x1F51,0x300,0x0}, 
+  {0x1F54,0x1F50,0x301,0x0}, {0x1F55,0x1F51,0x301,0x0}, {0x1F56,0x1F50,0x342,0x0}, 
+  {0x1F57,0x1F51,0x342,0x0}, {0x1F59,0x3A5,0x314,0x0}, {0x1F5B,0x1F59,0x300,0x0}, 
+  {0x1F5D,0x1F59,0x301,0x0}, {0x1F5F,0x1F59,0x342,0x0}, {0x1F60,0x3C9,0x313,0x0}, 
+  {0x1F61,0x3C9,0x314,0x0}, {0x1F62,0x1F60,0x300,0x0}, {0x1F63,0x1F61,0x300,0x0}, 
+  {0x1F64,0x1F60,0x301,0x0}, {0x1F65,0x1F61,0x301,0x0}, {0x1F66,0x1F60,0x342,0x0}, 
+  {0x1F67,0x1F61,0x342,0x0}, {0x1F68,0x3A9,0x313,0x0}, {0x1F69,0x3A9,0x314,0x0}, 
+  {0x1F6A,0x1F68,0x300,0x0}, {0x1F6B,0x1F69,0x300,0x0}, {0x1F6C,0x1F68,0x301,0x0}, 
+  {0x1F6D,0x1F69,0x301,0x0}, {0x1F6E,0x1F68,0x342,0x0}, {0x1F6F,0x1F69,0x342,0x0}, 
+  {0x1F70,0x3B1,0x300,0x0}, {0x1F71,0x3B1,0x301,0x0}, {0x1F72,0x3B5,0x300,0x0}, 
+  {0x1F73,0x3B5,0x301,0x0}, {0x1F74,0x3B7,0x300,0x0}, {0x1F75,0x3B7,0x301,0x0}, 
+  {0x1F76,0x3B9,0x300,0x0}, {0x1F77,0x3B9,0x301,0x0}, {0x1F78,0x3BF,0x300,0x0}, 
+  {0x1F79,0x3BF,0x301,0x0}, {0x1F7A,0x3C5,0x300,0x0}, {0x1F7B,0x3C5,0x301,0x0}, 
+  {0x1F7C,0x3C9,0x300,0x0}, {0x1F7D,0x3C9,0x301,0x0}, {0x1F80,0x1F00,0x345,0x0}, 
+  {0x1F81,0x1F01,0x345,0x0}, {0x1F82,0x1F02,0x345,0x0}, {0x1F83,0x1F03,0x345,0x0}, 
+  {0x1F84,0x1F04,0x345,0x0}, {0x1F85,0x1F05,0x345,0x0}, {0x1F86,0x1F06,0x345,0x0}, 
+  {0x1F87,0x1F07,0x345,0x0}, {0x1F88,0x1F08,0x345,0x0}, {0x1F89,0x1F09,0x345,0x0}, 
+  {0x1F8A,0x1F0A,0x345,0x0}, {0x1F8B,0x1F0B,0x345,0x0}, {0x1F8C,0x1F0C,0x345,0x0}, 
+  {0x1F8D,0x1F0D,0x345,0x0}, {0x1F8E,0x1F0E,0x345,0x0}, {0x1F8F,0x1F0F,0x345,0x0}, 
+  {0x1F90,0x1F20,0x345,0x0}, {0x1F91,0x1F21,0x345,0x0}, {0x1F92,0x1F22,0x345,0x0}, 
+  {0x1F93,0x1F23,0x345,0x0}, {0x1F94,0x1F24,0x345,0x0}, {0x1F95,0x1F25,0x345,0x0}, 
+  {0x1F96,0x1F26,0x345,0x0}, {0x1F97,0x1F27,0x345,0x0}, {0x1F98,0x1F28,0x345,0x0}, 
+  {0x1F99,0x1F29,0x345,0x0}, {0x1F9A,0x1F2A,0x345,0x0}, {0x1F9B,0x1F2B,0x345,0x0}, 
+  {0x1F9C,0x1F2C,0x345,0x0}, {0x1F9D,0x1F2D,0x345,0x0}, {0x1F9E,0x1F2E,0x345,0x0}, 
+  {0x1F9F,0x1F2F,0x345,0x0}, {0x1FA0,0x1F60,0x345,0x0}, {0x1FA1,0x1F61,0x345,0x0}, 
+  {0x1FA2,0x1F62,0x345,0x0}, {0x1FA3,0x1F63,0x345,0x0}, {0x1FA4,0x1F64,0x345,0x0}, 
+  {0x1FA5,0x1F65,0x345,0x0}, {0x1FA6,0x1F66,0x345,0x0}, {0x1FA7,0x1F67,0x345,0x0}, 
+  {0x1FA8,0x1F68,0x345,0x0}, {0x1FA9,0x1F69,0x345,0x0}, {0x1FAA,0x1F6A,0x345,0x0}, 
+  {0x1FAB,0x1F6B,0x345,0x0}, {0x1FAC,0x1F6C,0x345,0x0}, {0x1FAD,0x1F6D,0x345,0x0}, 
+  {0x1FAE,0x1F6E,0x345,0x0}, {0x1FAF,0x1F6F,0x345,0x0}, {0x1FB0,0x3B1,0x306,0x0}, 
+  {0x1FB1,0x3B1,0x304,0x0}, {0x1FB2,0x1F70,0x345,0x0}, {0x1FB3,0x3B1,0x345,0x0}, 
+  {0x1FB4,0x1F71,0x345,0x0}, {0x1FB6,0x3B1,0x342,0x0}, {0x1FB7,0x1FB6,0x345,0x0}, 
+  {0x1FB8,0x391,0x306,0x0}, {0x1FB9,0x391,0x304,0x0}, {0x1FBA,0x391,0x300,0x0}, 
+  {0x1FBB,0x391,0x301,0x0}, {0x1FBC,0x391,0x345,0x0}, {0x1FBE,0x3B9,0x0,0x0}, 
+  {0x1FC1,0xA8,0x342,0x0}, {0x1FC2,0x1F74,0x345,0x0}, {0x1FC3,0x3B7,0x345,0x0}, 
+  {0x1FC4,0x1F75,0x345,0x0}, {0x1FC6,0x3B7,0x342,0x0}, {0x1FC7,0x1FC6,0x345,0x0}, 
+  {0x1FC8,0x395,0x300,0x0}, {0x1FC9,0x395,0x301,0x0}, {0x1FCA,0x397,0x300,0x0}, 
+  {0x1FCB,0x397,0x301,0x0}, {0x1FCC,0x397,0x345,0x0}, {0x1FCD,0x1FBF,0x300,0x0}, 
+  {0x1FCE,0x1FBF,0x301,0x0}, {0x1FCF,0x1FBF,0x342,0x0}, {0x1FD0,0x3B9,0x306,0x0}, 
+  {0x1FD1,0x3B9,0x304,0x0}, {0x1FD2,0x3CA,0x300,0x0}, {0x1FD3,0x3CA,0x301,0x0}, 
+  {0x1FD6,0x3B9,0x342,0x0}, {0x1FD7,0x3CA,0x342,0x0}, {0x1FD8,0x399,0x306,0x0}, 
+  {0x1FD9,0x399,0x304,0x0}, {0x1FDA,0x399,0x300,0x0}, {0x1FDB,0x399,0x301,0x0}, 
+  {0x1FDD,0x1FFE,0x300,0x0}, {0x1FDE,0x1FFE,0x301,0x0}, {0x1FDF,0x1FFE,0x342,0x0}, 
+  {0x1FE0,0x3C5,0x306,0x0}, {0x1FE1,0x3C5,0x304,0x0}, {0x1FE2,0x3CB,0x300,0x0}, 
+  {0x1FE3,0x3CB,0x301,0x0}, {0x1FE4,0x3C1,0x313,0x0}, {0x1FE5,0x3C1,0x314,0x0}, 
+  {0x1FE6,0x3C5,0x342,0x0}, {0x1FE7,0x3CB,0x342,0x0}, {0x1FE8,0x3A5,0x306,0x0}, 
+  {0x1FE9,0x3A5,0x304,0x0}, {0x1FEA,0x3A5,0x300,0x0}, {0x1FEB,0x3A5,0x301,0x0}, 
+  {0x1FEC,0x3A1,0x314,0x0}, {0x1FED,0xA8,0x300,0x0}, {0x1FEE,0xA8,0x301,0x0}, 
+  {0x1FEF,0x60,0x0,0x0}, {0x1FF2,0x1F7C,0x345,0x0}, {0x1FF3,0x3C9,0x345,0x0}, 
+  {0x1FF4,0x1F79,0x345,0x0}, {0x1FF6,0x3C9,0x342,0x0}, {0x1FF7,0x1FF6,0x345,0x0}, 
+  {0x1FF8,0x39F,0x300,0x0}, {0x1FF9,0x39F,0x301,0x0}, {0x1FFA,0x3A9,0x300,0x0}, 
+  {0x1FFB,0x3A9,0x301,0x0}, {0x1FFC,0x3A9,0x345,0x0}, {0x1FFD,0xB4,0x0,0x0}, 
+  {0x2000,0x2002,0x0,0x0}, {0x2001,0x2003,0x0,0x0}, {0x2126,0x3A9,0x0,0x0}, 
+  {0x212A,0x4B,0x0,0x0}, {0x212B,0xC5,0x0,0x0}, {0x2204,0x2203,0x338,0x0}, 
+  {0x2209,0x2208,0x338,0x0}, {0x220C,0x220B,0x338,0x0}, {0x2224,0x2223,0x338,0x0}, 
+  {0x2226,0x2225,0x338,0x0}, {0x2241,0x7E,0x338,0x0}, {0x2244,0x2243,0x338,0x0}, 
+  {0x2247,0x2245,0x338,0x0}, {0x2249,0x2248,0x338,0x0}, {0x2260,0x3D,0x338,0x0}, 
+  {0x2262,0x2261,0x338,0x0}, {0x226D,0x224D,0x338,0x0}, {0x226E,0x3C,0x338,0x0}, 
+  {0x226F,0x3E,0x338,0x0}, {0x2270,0x2264,0x338,0x0}, {0x2271,0x2265,0x338,0x0}, 
+  {0x2274,0x2272,0x338,0x0}, {0x2275,0x2273,0x338,0x0}, {0x2278,0x2276,0x338,0x0}, 
+  {0x2279,0x2277,0x338,0x0}, {0x2280,0x227A,0x338,0x0}, {0x2281,0x227B,0x338,0x0}, 
+  {0x2284,0x2282,0x338,0x0}, {0x2285,0x2283,0x338,0x0}, {0x2288,0x2286,0x338,0x0}, 
+  {0x2289,0x2287,0x338,0x0}, {0x22AC,0x22A2,0x338,0x0}, {0x22AD,0x22A8,0x338,0x0}, 
+  {0x22AE,0x22A9,0x338,0x0}, {0x22AF,0x22AB,0x338,0x0}, {0x22E0,0x227C,0x338,0x0}, 
+  {0x22E1,0x227D,0x338,0x0}, {0x22E2,0x2291,0x338,0x0}, {0x22E3,0x2292,0x338,0x0}, 
+  {0x22EA,0x22B2,0x338,0x0}, {0x22EB,0x22B3,0x338,0x0}, {0x22EC,0x22B4,0x338,0x0}, 
+  {0x22ED,0x22B5,0x338,0x0}, {0x2329,0x3008,0x0,0x0}, {0x232A,0x3009,0x0,0x0}, 
+  {0x304C,0x304B,0x3099,0x0}, {0x304E,0x304D,0x3099,0x0}, {0x3050,0x304F,0x3099,0x0}, 
+  {0x3052,0x3051,0x3099,0x0}, {0x3054,0x3053,0x3099,0x0}, {0x3056,0x3055,0x3099,0x0}, 
+  {0x3058,0x3057,0x3099,0x0}, {0x305A,0x3059,0x3099,0x0}, {0x305C,0x305B,0x3099,0x0}, 
+  {0x305E,0x305D,0x3099,0x0}, {0x3060,0x305F,0x3099,0x0}, {0x3062,0x3061,0x3099,0x0}, 
+  {0x3065,0x3064,0x3099,0x0}, {0x3067,0x3066,0x3099,0x0}, {0x3069,0x3068,0x3099,0x0}, 
+  {0x3070,0x306F,0x3099,0x0}, {0x3071,0x306F,0x309A,0x0}, {0x3073,0x3072,0x3099,0x0}, 
+  {0x3074,0x3072,0x309A,0x0}, {0x3076,0x3075,0x3099,0x0}, {0x3077,0x3075,0x309A,0x0}, 
+  {0x3079,0x3078,0x3099,0x0}, {0x307A,0x3078,0x309A,0x0}, {0x307C,0x307B,0x3099,0x0}, 
+  {0x307D,0x307B,0x309A,0x0}, {0x3094,0x3046,0x3099,0x0}, {0x309E,0x309D,0x3099,0x0}, 
+  {0x30AC,0x30AB,0x3099,0x0}, {0x30AE,0x30AD,0x3099,0x0}, {0x30B0,0x30AF,0x3099,0x0}, 
+  {0x30B2,0x30B1,0x3099,0x0}, {0x30B4,0x30B3,0x3099,0x0}, {0x30B6,0x30B5,0x3099,0x0}, 
+  {0x30B8,0x30B7,0x3099,0x0}, {0x30BA,0x30B9,0x3099,0x0}, {0x30BC,0x30BB,0x3099,0x0}, 
+  {0x30BE,0x30BD,0x3099,0x0}, {0x30C0,0x30BF,0x3099,0x0}, {0x30C2,0x30C1,0x3099,0x0}, 
+  {0x30C5,0x30C4,0x3099,0x0}, {0x30C7,0x30C6,0x3099,0x0}, {0x30C9,0x30C8,0x3099,0x0}, 
+  {0x30D0,0x30CF,0x3099,0x0}, {0x30D1,0x30CF,0x309A,0x0}, {0x30D3,0x30D2,0x3099,0x0}, 
+  {0x30D4,0x30D2,0x309A,0x0}, {0x30D6,0x30D5,0x3099,0x0}, {0x30D7,0x30D5,0x309A,0x0}, 
+  {0x30D9,0x30D8,0x3099,0x0}, {0x30DA,0x30D8,0x309A,0x0}, {0x30DC,0x30DB,0x3099,0x0}, 
+  {0x30DD,0x30DB,0x309A,0x0}, {0x30F4,0x30A6,0x3099,0x0}, {0x30F7,0x30EF,0x3099,0x0}, 
+  {0x30F8,0x30F0,0x3099,0x0}, {0x30F9,0x30F1,0x3099,0x0}, {0x30FA,0x30F2,0x3099,0x0}, 
+  {0x30FE,0x30FD,0x3099,0x0}, {0xF900,0x8C48,0x0,0x0}, {0xF901,0x66F4,0x0,0x0}, 
+  {0xF902,0x8ECA,0x0,0x0}, {0xF903,0x8CC8,0x0,0x0}, {0xF904,0x6ED1,0x0,0x0}, 
+  {0xF905,0x4E32,0x0,0x0}, {0xF906,0x53E5,0x0,0x0}, {0xF907,0x9F9C,0x0,0x0}, 
+  {0xF908,0x9F9C,0x0,0x0}, {0xF909,0x5951,0x0,0x0}, {0xF90A,0x91D1,0x0,0x0}, 
+  {0xF90B,0x5587,0x0,0x0}, {0xF90C,0x5948,0x0,0x0}, {0xF90D,0x61F6,0x0,0x0}, 
+  {0xF90E,0x7669,0x0,0x0}, {0xF90F,0x7F85,0x0,0x0}, {0xF910,0x863F,0x0,0x0}, 
+  {0xF911,0x87BA,0x0,0x0}, {0xF912,0x88F8,0x0,0x0}, {0xF913,0x908F,0x0,0x0}, 
+  {0xF914,0x6A02,0x0,0x0}, {0xF915,0x6D1B,0x0,0x0}, {0xF916,0x70D9,0x0,0x0}, 
+  {0xF917,0x73DE,0x0,0x0}, {0xF918,0x843D,0x0,0x0}, {0xF919,0x916A,0x0,0x0}, 
+  {0xF91A,0x99F1,0x0,0x0}, {0xF91B,0x4E82,0x0,0x0}, {0xF91C,0x5375,0x0,0x0}, 
+  {0xF91D,0x6B04,0x0,0x0}, {0xF91E,0x721B,0x0,0x0}, {0xF91F,0x862D,0x0,0x0}, 
+  {0xF920,0x9E1E,0x0,0x0}, {0xF921,0x5D50,0x0,0x0}, {0xF922,0x6FEB,0x0,0x0}, 
+  {0xF923,0x85CD,0x0,0x0}, {0xF924,0x8964,0x0,0x0}, {0xF925,0x62C9,0x0,0x0}, 
+  {0xF926,0x81D8,0x0,0x0}, {0xF927,0x881F,0x0,0x0}, {0xF928,0x5ECA,0x0,0x0}, 
+  {0xF929,0x6717,0x0,0x0}, {0xF92A,0x6D6A,0x0,0x0}, {0xF92B,0x72FC,0x0,0x0}, 
+  {0xF92C,0x90CE,0x0,0x0}, {0xF92D,0x4F86,0x0,0x0}, {0xF92E,0x51B7,0x0,0x0}, 
+  {0xF92F,0x52DE,0x0,0x0}, {0xF930,0x64C4,0x0,0x0}, {0xF931,0x6AD3,0x0,0x0}, 
+  {0xF932,0x7210,0x0,0x0}, {0xF933,0x76E7,0x0,0x0}, {0xF934,0x8001,0x0,0x0}, 
+  {0xF935,0x8606,0x0,0x0}, {0xF936,0x865C,0x0,0x0}, {0xF937,0x8DEF,0x0,0x0}, 
+  {0xF938,0x9732,0x0,0x0}, {0xF939,0x9B6F,0x0,0x0}, {0xF93A,0x9DFA,0x0,0x0}, 
+  {0xF93B,0x788C,0x0,0x0}, {0xF93C,0x797F,0x0,0x0}, {0xF93D,0x7DA0,0x0,0x0}, 
+  {0xF93E,0x83C9,0x0,0x0}, {0xF93F,0x9304,0x0,0x0}, {0xF940,0x9E7F,0x0,0x0}, 
+  {0xF941,0x8AD6,0x0,0x0}, {0xF942,0x58DF,0x0,0x0}, {0xF943,0x5F04,0x0,0x0}, 
+  {0xF944,0x7C60,0x0,0x0}, {0xF945,0x807E,0x0,0x0}, {0xF946,0x7262,0x0,0x0}, 
+  {0xF947,0x78CA,0x0,0x0}, {0xF948,0x8CC2,0x0,0x0}, {0xF949,0x96F7,0x0,0x0}, 
+  {0xF94A,0x58D8,0x0,0x0}, {0xF94B,0x5C62,0x0,0x0}, {0xF94C,0x6A13,0x0,0x0}, 
+  {0xF94D,0x6DDA,0x0,0x0}, {0xF94E,0x6F0F,0x0,0x0}, {0xF94F,0x7D2F,0x0,0x0}, 
+  {0xF950,0x7E37,0x0,0x0}, {0xF951,0x96FB,0x0,0x0}, {0xF952,0x52D2,0x0,0x0}, 
+  {0xF953,0x808B,0x0,0x0}, {0xF954,0x51DC,0x0,0x0}, {0xF955,0x51CC,0x0,0x0}, 
+  {0xF956,0x7A1C,0x0,0x0}, {0xF957,0x7DBE,0x0,0x0}, {0xF958,0x83F1,0x0,0x0}, 
+  {0xF959,0x9675,0x0,0x0}, {0xF95A,0x8B80,0x0,0x0}, {0xF95B,0x62CF,0x0,0x0}, 
+  {0xF95C,0x6A02,0x0,0x0}, {0xF95D,0x8AFE,0x0,0x0}, {0xF95E,0x4E39,0x0,0x0}, 
+  {0xF95F,0x5BE7,0x0,0x0}, {0xF960,0x6012,0x0,0x0}, {0xF961,0x7387,0x0,0x0}, 
+  {0xF962,0x7570,0x0,0x0}, {0xF963,0x5317,0x0,0x0}, {0xF964,0x78FB,0x0,0x0}, 
+  {0xF965,0x4FBF,0x0,0x0}, {0xF966,0x5FA9,0x0,0x0}, {0xF967,0x4E0D,0x0,0x0}, 
+  {0xF968,0x6CCC,0x0,0x0}, {0xF969,0x6578,0x0,0x0}, {0xF96A,0x7D22,0x0,0x0}, 
+  {0xF96B,0x53C3,0x0,0x0}, {0xF96C,0x585E,0x0,0x0}, {0xF96D,0x7701,0x0,0x0}, 
+  {0xF96E,0x8449,0x0,0x0}, {0xF96F,0x8AAA,0x0,0x0}, {0xF970,0x6BBA,0x0,0x0}, 
+  {0xF971,0x8FB0,0x0,0x0}, {0xF972,0x6C88,0x0,0x0}, {0xF973,0x62FE,0x0,0x0}, 
+  {0xF974,0x82E5,0x0,0x0}, {0xF975,0x63A0,0x0,0x0}, {0xF976,0x7565,0x0,0x0}, 
+  {0xF977,0x4EAE,0x0,0x0}, {0xF978,0x5169,0x0,0x0}, {0xF979,0x51C9,0x0,0x0}, 
+  {0xF97A,0x6881,0x0,0x0}, {0xF97B,0x7CE7,0x0,0x0}, {0xF97C,0x826F,0x0,0x0}, 
+  {0xF97D,0x8AD2,0x0,0x0}, {0xF97E,0x91CF,0x0,0x0}, {0xF97F,0x52F5,0x0,0x0}, 
+  {0xF980,0x5442,0x0,0x0}, {0xF981,0x5973,0x0,0x0}, {0xF982,0x5EEC,0x0,0x0}, 
+  {0xF983,0x65C5,0x0,0x0}, {0xF984,0x6FFE,0x0,0x0}, {0xF985,0x792A,0x0,0x0}, 
+  {0xF986,0x95AD,0x0,0x0}, {0xF987,0x9A6A,0x0,0x0}, {0xF988,0x9E97,0x0,0x0}, 
+  {0xF989,0x9ECE,0x0,0x0}, {0xF98A,0x529B,0x0,0x0}, {0xF98B,0x66C6,0x0,0x0}, 
+  {0xF98C,0x6B77,0x0,0x0}, {0xF98D,0x8F62,0x0,0x0}, {0xF98E,0x5E74,0x0,0x0}, 
+  {0xF98F,0x6190,0x0,0x0}, {0xF990,0x6200,0x0,0x0}, {0xF991,0x649A,0x0,0x0}, 
+  {0xF992,0x6F23,0x0,0x0}, {0xF993,0x7149,0x0,0x0}, {0xF994,0x7489,0x0,0x0}, 
+  {0xF995,0x79CA,0x0,0x0}, {0xF996,0x7DF4,0x0,0x0}, {0xF997,0x806F,0x0,0x0}, 
+  {0xF998,0x8F26,0x0,0x0}, {0xF999,0x84EE,0x0,0x0}, {0xF99A,0x9023,0x0,0x0}, 
+  {0xF99B,0x934A,0x0,0x0}, {0xF99C,0x5217,0x0,0x0}, {0xF99D,0x52A3,0x0,0x0}, 
+  {0xF99E,0x54BD,0x0,0x0}, {0xF99F,0x70C8,0x0,0x0}, {0xF9A0,0x88C2,0x0,0x0}, 
+  {0xF9A1,0x8AAA,0x0,0x0}, {0xF9A2,0x5EC9,0x0,0x0}, {0xF9A3,0x5FF5,0x0,0x0}, 
+  {0xF9A4,0x637B,0x0,0x0}, {0xF9A5,0x6BAE,0x0,0x0}, {0xF9A6,0x7C3E,0x0,0x0}, 
+  {0xF9A7,0x7375,0x0,0x0}, {0xF9A8,0x4EE4,0x0,0x0}, {0xF9A9,0x56F9,0x0,0x0}, 
+  {0xF9AA,0x5BE7,0x0,0x0}, {0xF9AB,0x5DBA,0x0,0x0}, {0xF9AC,0x601C,0x0,0x0}, 
+  {0xF9AD,0x73B2,0x0,0x0}, {0xF9AE,0x7469,0x0,0x0}, {0xF9AF,0x7F9A,0x0,0x0}, 
+  {0xF9B0,0x8046,0x0,0x0}, {0xF9B1,0x9234,0x0,0x0}, {0xF9B2,0x96F6,0x0,0x0}, 
+  {0xF9B3,0x9748,0x0,0x0}, {0xF9B4,0x9818,0x0,0x0}, {0xF9B5,0x4F8B,0x0,0x0}, 
+  {0xF9B6,0x79AE,0x0,0x0}, {0xF9B7,0x91B4,0x0,0x0}, {0xF9B8,0x96B8,0x0,0x0}, 
+  {0xF9B9,0x60E1,0x0,0x0}, {0xF9BA,0x4E86,0x0,0x0}, {0xF9BB,0x50DA,0x0,0x0}, 
+  {0xF9BC,0x5BEE,0x0,0x0}, {0xF9BD,0x5C3F,0x0,0x0}, {0xF9BE,0x6599,0x0,0x0}, 
+  {0xF9BF,0x6A02,0x0,0x0}, {0xF9C0,0x71CE,0x0,0x0}, {0xF9C1,0x7642,0x0,0x0}, 
+  {0xF9C2,0x84FC,0x0,0x0}, {0xF9C3,0x907C,0x0,0x0}, {0xF9C4,0x9F8D,0x0,0x0}, 
+  {0xF9C5,0x6688,0x0,0x0}, {0xF9C6,0x962E,0x0,0x0}, {0xF9C7,0x5289,0x0,0x0}, 
+  {0xF9C8,0x677B,0x0,0x0}, {0xF9C9,0x67F3,0x0,0x0}, {0xF9CA,0x6D41,0x0,0x0}, 
+  {0xF9CB,0x6E9C,0x0,0x0}, {0xF9CC,0x7409,0x0,0x0}, {0xF9CD,0x7559,0x0,0x0}, 
+  {0xF9CE,0x786B,0x0,0x0}, {0xF9CF,0x7D10,0x0,0x0}, {0xF9D0,0x985E,0x0,0x0}, 
+  {0xF9D1,0x516D,0x0,0x0}, {0xF9D2,0x622E,0x0,0x0}, {0xF9D3,0x9678,0x0,0x0}, 
+  {0xF9D4,0x502B,0x0,0x0}, {0xF9D5,0x5D19,0x0,0x0}, {0xF9D6,0x6DEA,0x0,0x0}, 
+  {0xF9D7,0x8F2A,0x0,0x0}, {0xF9D8,0x5F8B,0x0,0x0}, {0xF9D9,0x6144,0x0,0x0}, 
+  {0xF9DA,0x6817,0x0,0x0}, {0xF9DB,0x7387,0x0,0x0}, {0xF9DC,0x9686,0x0,0x0}, 
+  {0xF9DD,0x5229,0x0,0x0}, {0xF9DE,0x540F,0x0,0x0}, {0xF9DF,0x5C65,0x0,0x0}, 
+  {0xF9E0,0x6613,0x0,0x0}, {0xF9E1,0x674E,0x0,0x0}, {0xF9E2,0x68A8,0x0,0x0}, 
+  {0xF9E3,0x6CE5,0x0,0x0}, {0xF9E4,0x7406,0x0,0x0}, {0xF9E5,0x75E2,0x0,0x0}, 
+  {0xF9E6,0x7F79,0x0,0x0}, {0xF9E7,0x88CF,0x0,0x0}, {0xF9E8,0x88E1,0x0,0x0}, 
+  {0xF9E9,0x91CC,0x0,0x0}, {0xF9EA,0x96E2,0x0,0x0}, {0xF9EB,0x533F,0x0,0x0}, 
+  {0xF9EC,0x6EBA,0x0,0x0}, {0xF9ED,0x541D,0x0,0x0}, {0xF9EE,0x71D0,0x0,0x0}, 
+  {0xF9EF,0x7498,0x0,0x0}, {0xF9F0,0x85FA,0x0,0x0}, {0xF9F1,0x96A3,0x0,0x0}, 
+  {0xF9F2,0x9C57,0x0,0x0}, {0xF9F3,0x9E9F,0x0,0x0}, {0xF9F4,0x6797,0x0,0x0}, 
+  {0xF9F5,0x6DCB,0x0,0x0}, {0xF9F6,0x81E8,0x0,0x0}, {0xF9F7,0x7ACB,0x0,0x0}, 
+  {0xF9F8,0x7B20,0x0,0x0}, {0xF9F9,0x7C92,0x0,0x0}, {0xF9FA,0x72C0,0x0,0x0}, 
+  {0xF9FB,0x7099,0x0,0x0}, {0xF9FC,0x8B58,0x0,0x0}, {0xF9FD,0x4EC0,0x0,0x0}, 
+  {0xF9FE,0x8336,0x0,0x0}, {0xF9FF,0x523A,0x0,0x0}, {0xFA00,0x5207,0x0,0x0}, 
+  {0xFA01,0x5EA6,0x0,0x0}, {0xFA02,0x62D3,0x0,0x0}, {0xFA03,0x7CD6,0x0,0x0}, 
+  {0xFA04,0x5B85,0x0,0x0}, {0xFA05,0x6D1E,0x0,0x0}, {0xFA06,0x66B4,0x0,0x0}, 
+  {0xFA07,0x8F3B,0x0,0x0}, {0xFA08,0x884C,0x0,0x0}, {0xFA09,0x964D,0x0,0x0}, 
+  {0xFA0A,0x898B,0x0,0x0}, {0xFA0B,0x5ED3,0x0,0x0}, {0xFA0C,0x5140,0x0,0x0}, 
+  {0xFA0D,0x55C0,0x0,0x0}, {0xFA10,0x585A,0x0,0x0}, {0xFA12,0x6674,0x0,0x0}, 
+  {0xFA15,0x51DE,0x0,0x0}, {0xFA16,0x732A,0x0,0x0}, {0xFA17,0x76CA,0x0,0x0}, 
+  {0xFA18,0x793C,0x0,0x0}, {0xFA19,0x795E,0x0,0x0}, {0xFA1A,0x7965,0x0,0x0}, 
+  {0xFA1B,0x798F,0x0,0x0}, {0xFA1C,0x9756,0x0,0x0}, {0xFA1D,0x7CBE,0x0,0x0}, 
+  {0xFA1E,0x7FBD,0x0,0x0}, {0xFA20,0x8612,0x0,0x0}, {0xFA22,0x8AF8,0x0,0x0}, 
+  {0xFA25,0x9038,0x0,0x0}, {0xFA26,0x90FD,0x0,0x0}, {0xFA2A,0x98EF,0x0,0x0}, 
+  {0xFA2B,0x98FC,0x0,0x0}, {0xFA2C,0x9928,0x0,0x0}, {0xFA2D,0x9DB4,0x0,0x0}, 
+  {0xFB1F,0x5F2,0x5B7,0x0}, {0xFB2A,0x5E9,0x5C1,0x0}, {0xFB2B,0x5E9,0x5C2,0x0}, 
+  {0xFB2C,0x5E9,0x5BC,0x5C1}, {0xFB2D,0x5E9,0x5BC,0x5C2}, {0xFB2E,0x5D0,0x5B7,0x0}, 
+  {0xFB2F,0x5D0,0x5B8,0x0}, {0xFB30,0x5D0,0x5BC,0x0}, {0xFB31,0x5D1,0x5BC,0x0}, 
+  {0xFB32,0x5D2,0x5BC,0x0}, {0xFB33,0x5D3,0x5BC,0x0}, {0xFB34,0x5D4,0x5BC,0x0}, 
+  {0xFB35,0x5D5,0x5BC,0x0}, {0xFB36,0x5D6,0x5BC,0x0}, {0xFB38,0x5D8,0x5BC,0x0}, 
+  {0xFB39,0x5D9,0x5BC,0x0}, {0xFB3A,0x5DA,0x5BC,0x0}, {0xFB3B,0x5DB,0x5BC,0x0}, 
+  {0xFB3C,0x5DC,0x5BC,0x0}, {0xFB3E,0x5DE,0x5BC,0x0}, {0xFB40,0x5E0,0x5BC,0x0}, 
+  {0xFB41,0x5E1,0x5BC,0x0}, {0xFB43,0x5E3,0x5BC,0x0}, {0xFB44,0x5E4,0x5BC,0x0}, 
+  {0xFB46,0x5E6,0x5BC,0x0}, {0xFB47,0x5E7,0x5BC,0x0}, {0xFB48,0x5E8,0x5BC,0x0}, 
+  {0xFB49,0x5E9,0x5BC,0x0}, {0xFB4A,0x5EA,0x5BC,0x0}, {0xFB4B,0x5D5,0x5B9,0x0}, 
+  {0xFB4C,0x5D1,0x5BF,0x0}, {0xFB4D,0x5DB,0x5BF,0x0}, {0xFB4E,0x5E4,0x5BF,0x0} 
+};
+
+
+/* All upper-case translations are included here */
+
+#define NUM_TOUPPER_INFO 677
+static const unirelate toupper_info[NUM_TOUPPER_INFO] = {
+  {0x61,0x41}, {0x62,0x42}, {0x63,0x43}, {0x64,0x44}, {0x65,0x45}, 
+  {0x66,0x46}, {0x67,0x47}, {0x68,0x48}, {0x69,0x49}, {0x6A,0x4A}, 
+  {0x6B,0x4B}, {0x6C,0x4C}, {0x6D,0x4D}, {0x6E,0x4E}, {0x6F,0x4F}, 
+  {0x70,0x50}, {0x71,0x51}, {0x72,0x52}, {0x73,0x53}, {0x74,0x54}, 
+  {0x75,0x55}, {0x76,0x56}, {0x77,0x57}, {0x78,0x58}, {0x79,0x59}, 
+  {0x7A,0x5A}, {0xE0,0xC0}, {0xE1,0xC1}, {0xE2,0xC2}, {0xE3,0xC3}, 
+  {0xE4,0xC4}, {0xE5,0xC5}, {0xE6,0xC6}, {0xE7,0xC7}, {0xE8,0xC8}, 
+  {0xE9,0xC9}, {0xEA,0xCA}, {0xEB,0xCB}, {0xEC,0xCC}, {0xED,0xCD}, 
+  {0xEE,0xCE}, {0xEF,0xCF}, {0xF0,0xD0}, {0xF1,0xD1}, {0xF2,0xD2}, 
+  {0xF3,0xD3}, {0xF4,0xD4}, {0xF5,0xD5}, {0xF6,0xD6}, {0xF8,0xD8}, 
+  {0xF9,0xD9}, {0xFA,0xDA}, {0xFB,0xDB}, {0xFC,0xDC}, {0xFD,0xDD}, 
+  {0xFE,0xDE}, {0xFF,0x178}, {0x101,0x100}, {0x103,0x102}, {0x105,0x104}, 
+  {0x107,0x106}, {0x109,0x108}, {0x10B,0x10A}, {0x10D,0x10C}, {0x10F,0x10E}, 
+  {0x111,0x110}, {0x113,0x112}, {0x115,0x114}, {0x117,0x116}, {0x119,0x118}, 
+  {0x11B,0x11A}, {0x11D,0x11C}, {0x11F,0x11E}, {0x121,0x120}, {0x123,0x122}, 
+  {0x125,0x124}, {0x127,0x126}, {0x129,0x128}, {0x12B,0x12A}, {0x12D,0x12C}, 
+  {0x12F,0x12E}, {0x131,0x49}, {0x133,0x132}, {0x135,0x134}, {0x137,0x136}, 
+  {0x13A,0x139}, {0x13C,0x13B}, {0x13E,0x13D}, {0x140,0x13F}, {0x142,0x141}, 
+  {0x144,0x143}, {0x146,0x145}, {0x148,0x147}, {0x14B,0x14A}, {0x14D,0x14C}, 
+  {0x14F,0x14E}, {0x151,0x150}, {0x153,0x152}, {0x155,0x154}, {0x157,0x156}, 
+  {0x159,0x158}, {0x15B,0x15A}, {0x15D,0x15C}, {0x15F,0x15E}, {0x161,0x160}, 
+  {0x163,0x162}, {0x165,0x164}, {0x167,0x166}, {0x169,0x168}, {0x16B,0x16A}, 
+  {0x16D,0x16C}, {0x16F,0x16E}, {0x171,0x170}, {0x173,0x172}, {0x175,0x174}, 
+  {0x177,0x176}, {0x17A,0x179}, {0x17C,0x17B}, {0x17E,0x17D}, {0x17F,0x53}, 
+  {0x183,0x182}, {0x185,0x184}, {0x188,0x187}, {0x18C,0x18B}, {0x192,0x191}, 
+  {0x199,0x198}, {0x1A1,0x1A0}, {0x1A3,0x1A2}, {0x1A5,0x1A4}, {0x1A8,0x1A7}, 
+  {0x1AD,0x1AC}, {0x1B0,0x1AF}, {0x1B4,0x1B3}, {0x1B6,0x1B5}, {0x1B9,0x1B8}, 
+  {0x1BD,0x1BC}, {0x1C5,0x1C4}, {0x1C6,0x1C4}, {0x1C8,0x1C7}, {0x1C9,0x1C7}, 
+  {0x1CB,0x1CA}, {0x1CC,0x1CA}, {0x1CE,0x1CD}, {0x1D0,0x1CF}, {0x1D2,0x1D1}, 
+  {0x1D4,0x1D3}, {0x1D6,0x1D5}, {0x1D8,0x1D7}, {0x1DA,0x1D9}, {0x1DC,0x1DB}, 
+  {0x1DD,0x18E}, {0x1DF,0x1DE}, {0x1E1,0x1E0}, {0x1E3,0x1E2}, {0x1E5,0x1E4}, 
+  {0x1E7,0x1E6}, {0x1E9,0x1E8}, {0x1EB,0x1EA}, {0x1ED,0x1EC}, {0x1EF,0x1EE}, 
+  {0x1F2,0x1F1}, {0x1F3,0x1F1}, {0x1F5,0x1F4}, {0x1FB,0x1FA}, {0x1FD,0x1FC}, 
+  {0x1FF,0x1FE}, {0x201,0x200}, {0x203,0x202}, {0x205,0x204}, {0x207,0x206}, 
+  {0x209,0x208}, {0x20B,0x20A}, {0x20D,0x20C}, {0x20F,0x20E}, {0x211,0x210}, 
+  {0x213,0x212}, {0x215,0x214}, {0x217,0x216}, {0x253,0x181}, {0x254,0x186}, 
+  {0x256,0x189}, {0x257,0x18A}, {0x259,0x18F}, {0x25B,0x190}, {0x260,0x193}, 
+  {0x263,0x194}, {0x268,0x197}, {0x269,0x196}, {0x26F,0x19C}, {0x272,0x19D}, 
+  {0x275,0x19F}, {0x283,0x1A9}, {0x288,0x1AE}, {0x28A,0x1B1}, {0x28B,0x1B2}, 
+  {0x292,0x1B7}, {0x3AC,0x386}, {0x3AD,0x388}, {0x3AE,0x389}, {0x3AF,0x38A}, 
+  {0x3B1,0x391}, {0x3B2,0x392}, {0x3B3,0x393}, {0x3B4,0x394}, {0x3B5,0x395}, 
+  {0x3B6,0x396}, {0x3B7,0x397}, {0x3B8,0x398}, {0x3B9,0x399}, {0x3BA,0x39A}, 
+  {0x3BB,0x39B}, {0x3BC,0x39C}, {0x3BD,0x39D}, {0x3BE,0x39E}, {0x3BF,0x39F}, 
+  {0x3C0,0x3A0}, {0x3C1,0x3A1}, {0x3C2,0x3A3}, {0x3C3,0x3A3}, {0x3C4,0x3A4}, 
+  {0x3C5,0x3A5}, {0x3C6,0x3A6}, {0x3C7,0x3A7}, {0x3C8,0x3A8}, {0x3C9,0x3A9}, 
+  {0x3CA,0x3AA}, {0x3CB,0x3AB}, {0x3CC,0x38C}, {0x3CD,0x38E}, {0x3CE,0x38F}, 
+  {0x3D0,0x392}, {0x3D1,0x398}, {0x3D5,0x3A6}, {0x3D6,0x3A0}, {0x3E3,0x3E2}, 
+  {0x3E5,0x3E4}, {0x3E7,0x3E6}, {0x3E9,0x3E8}, {0x3EB,0x3EA}, {0x3ED,0x3EC}, 
+  {0x3EF,0x3EE}, {0x3F0,0x39A}, {0x3F1,0x3A1}, {0x3F2,0x3A3}, {0x430,0x410}, 
+  {0x431,0x411}, {0x432,0x412}, {0x433,0x413}, {0x434,0x414}, {0x435,0x415}, 
+  {0x436,0x416}, {0x437,0x417}, {0x438,0x418}, {0x439,0x419}, {0x43A,0x41A}, 
+  {0x43B,0x41B}, {0x43C,0x41C}, {0x43D,0x41D}, {0x43E,0x41E}, {0x43F,0x41F}, 
+  {0x440,0x420}, {0x441,0x421}, {0x442,0x422}, {0x443,0x423}, {0x444,0x424}, 
+  {0x445,0x425}, {0x446,0x426}, {0x447,0x427}, {0x448,0x428}, {0x449,0x429}, 
+  {0x44A,0x42A}, {0x44B,0x42B}, {0x44C,0x42C}, {0x44D,0x42D}, {0x44E,0x42E}, 
+  {0x44F,0x42F}, {0x451,0x401}, {0x452,0x402}, {0x453,0x403}, {0x454,0x404}, 
+  {0x455,0x405}, {0x456,0x406}, {0x457,0x407}, {0x458,0x408}, {0x459,0x409}, 
+  {0x45A,0x40A}, {0x45B,0x40B}, {0x45C,0x40C}, {0x45E,0x40E}, {0x45F,0x40F}, 
+  {0x461,0x460}, {0x463,0x462}, {0x465,0x464}, {0x467,0x466}, {0x469,0x468}, 
+  {0x46B,0x46A}, {0x46D,0x46C}, {0x46F,0x46E}, {0x471,0x470}, {0x473,0x472}, 
+  {0x475,0x474}, {0x477,0x476}, {0x479,0x478}, {0x47B,0x47A}, {0x47D,0x47C}, 
+  {0x47F,0x47E}, {0x481,0x480}, {0x491,0x490}, {0x493,0x492}, {0x495,0x494}, 
+  {0x497,0x496}, {0x499,0x498}, {0x49B,0x49A}, {0x49D,0x49C}, {0x49F,0x49E}, 
+  {0x4A1,0x4A0}, {0x4A3,0x4A2}, {0x4A5,0x4A4}, {0x4A7,0x4A6}, {0x4A9,0x4A8}, 
+  {0x4AB,0x4AA}, {0x4AD,0x4AC}, {0x4AF,0x4AE}, {0x4B1,0x4B0}, {0x4B3,0x4B2}, 
+  {0x4B5,0x4B4}, {0x4B7,0x4B6}, {0x4B9,0x4B8}, {0x4BB,0x4BA}, {0x4BD,0x4BC}, 
+  {0x4BF,0x4BE}, {0x4C2,0x4C1}, {0x4C4,0x4C3}, {0x4C8,0x4C7}, {0x4CC,0x4CB}, 
+  {0x4D1,0x4D0}, {0x4D3,0x4D2}, {0x4D5,0x4D4}, {0x4D7,0x4D6}, {0x4D9,0x4D8}, 
+  {0x4DB,0x4DA}, {0x4DD,0x4DC}, {0x4DF,0x4DE}, {0x4E1,0x4E0}, {0x4E3,0x4E2}, 
+  {0x4E5,0x4E4}, {0x4E7,0x4E6}, {0x4E9,0x4E8}, {0x4EB,0x4EA}, {0x4EF,0x4EE}, 
+  {0x4F1,0x4F0}, {0x4F3,0x4F2}, {0x4F5,0x4F4}, {0x4F9,0x4F8}, {0x561,0x531}, 
+  {0x562,0x532}, {0x563,0x533}, {0x564,0x534}, {0x565,0x535}, {0x566,0x536}, 
+  {0x567,0x537}, {0x568,0x538}, {0x569,0x539}, {0x56A,0x53A}, {0x56B,0x53B}, 
+  {0x56C,0x53C}, {0x56D,0x53D}, {0x56E,0x53E}, {0x56F,0x53F}, {0x570,0x540}, 
+  {0x571,0x541}, {0x572,0x542}, {0x573,0x543}, {0x574,0x544}, {0x575,0x545}, 
+  {0x576,0x546}, {0x577,0x547}, {0x578,0x548}, {0x579,0x549}, {0x57A,0x54A}, 
+  {0x57B,0x54B}, {0x57C,0x54C}, {0x57D,0x54D}, {0x57E,0x54E}, {0x57F,0x54F}, 
+  {0x580,0x550}, {0x581,0x551}, {0x582,0x552}, {0x583,0x553}, {0x584,0x554}, 
+  {0x585,0x555}, {0x586,0x556}, {0x1E01,0x1E00}, {0x1E03,0x1E02}, {0x1E05,0x1E04}, 
+  {0x1E07,0x1E06}, {0x1E09,0x1E08}, {0x1E0B,0x1E0A}, {0x1E0D,0x1E0C}, {0x1E0F,0x1E0E}, 
+  {0x1E11,0x1E10}, {0x1E13,0x1E12}, {0x1E15,0x1E14}, {0x1E17,0x1E16}, {0x1E19,0x1E18}, 
+  {0x1E1B,0x1E1A}, {0x1E1D,0x1E1C}, {0x1E1F,0x1E1E}, {0x1E21,0x1E20}, {0x1E23,0x1E22}, 
+  {0x1E25,0x1E24}, {0x1E27,0x1E26}, {0x1E29,0x1E28}, {0x1E2B,0x1E2A}, {0x1E2D,0x1E2C}, 
+  {0x1E2F,0x1E2E}, {0x1E31,0x1E30}, {0x1E33,0x1E32}, {0x1E35,0x1E34}, {0x1E37,0x1E36}, 
+  {0x1E39,0x1E38}, {0x1E3B,0x1E3A}, {0x1E3D,0x1E3C}, {0x1E3F,0x1E3E}, {0x1E41,0x1E40}, 
+  {0x1E43,0x1E42}, {0x1E45,0x1E44}, {0x1E47,0x1E46}, {0x1E49,0x1E48}, {0x1E4B,0x1E4A}, 
+  {0x1E4D,0x1E4C}, {0x1E4F,0x1E4E}, {0x1E51,0x1E50}, {0x1E53,0x1E52}, {0x1E55,0x1E54}, 
+  {0x1E57,0x1E56}, {0x1E59,0x1E58}, {0x1E5B,0x1E5A}, {0x1E5D,0x1E5C}, {0x1E5F,0x1E5E}, 
+  {0x1E61,0x1E60}, {0x1E63,0x1E62}, {0x1E65,0x1E64}, {0x1E67,0x1E66}, {0x1E69,0x1E68}, 
+  {0x1E6B,0x1E6A}, {0x1E6D,0x1E6C}, {0x1E6F,0x1E6E}, {0x1E71,0x1E70}, {0x1E73,0x1E72}, 
+  {0x1E75,0x1E74}, {0x1E77,0x1E76}, {0x1E79,0x1E78}, {0x1E7B,0x1E7A}, {0x1E7D,0x1E7C}, 
+  {0x1E7F,0x1E7E}, {0x1E81,0x1E80}, {0x1E83,0x1E82}, {0x1E85,0x1E84}, {0x1E87,0x1E86}, 
+  {0x1E89,0x1E88}, {0x1E8B,0x1E8A}, {0x1E8D,0x1E8C}, {0x1E8F,0x1E8E}, {0x1E91,0x1E90}, 
+  {0x1E93,0x1E92}, {0x1E95,0x1E94}, {0x1E9B,0x1E60}, {0x1EA1,0x1EA0}, {0x1EA3,0x1EA2}, 
+  {0x1EA5,0x1EA4}, {0x1EA7,0x1EA6}, {0x1EA9,0x1EA8}, {0x1EAB,0x1EAA}, {0x1EAD,0x1EAC}, 
+  {0x1EAF,0x1EAE}, {0x1EB1,0x1EB0}, {0x1EB3,0x1EB2}, {0x1EB5,0x1EB4}, {0x1EB7,0x1EB6}, 
+  {0x1EB9,0x1EB8}, {0x1EBB,0x1EBA}, {0x1EBD,0x1EBC}, {0x1EBF,0x1EBE}, {0x1EC1,0x1EC0}, 
+  {0x1EC3,0x1EC2}, {0x1EC5,0x1EC4}, {0x1EC7,0x1EC6}, {0x1EC9,0x1EC8}, {0x1ECB,0x1ECA}, 
+  {0x1ECD,0x1ECC}, {0x1ECF,0x1ECE}, {0x1ED1,0x1ED0}, {0x1ED3,0x1ED2}, {0x1ED5,0x1ED4}, 
+  {0x1ED7,0x1ED6}, {0x1ED9,0x1ED8}, {0x1EDB,0x1EDA}, {0x1EDD,0x1EDC}, {0x1EDF,0x1EDE}, 
+  {0x1EE1,0x1EE0}, {0x1EE3,0x1EE2}, {0x1EE5,0x1EE4}, {0x1EE7,0x1EE6}, {0x1EE9,0x1EE8}, 
+  {0x1EEB,0x1EEA}, {0x1EED,0x1EEC}, {0x1EEF,0x1EEE}, {0x1EF1,0x1EF0}, {0x1EF3,0x1EF2}, 
+  {0x1EF5,0x1EF4}, {0x1EF7,0x1EF6}, {0x1EF9,0x1EF8}, {0x1F00,0x1F08}, {0x1F01,0x1F09}, 
+  {0x1F02,0x1F0A}, {0x1F03,0x1F0B}, {0x1F04,0x1F0C}, {0x1F05,0x1F0D}, {0x1F06,0x1F0E}, 
+  {0x1F07,0x1F0F}, {0x1F10,0x1F18}, {0x1F11,0x1F19}, {0x1F12,0x1F1A}, {0x1F13,0x1F1B}, 
+  {0x1F14,0x1F1C}, {0x1F15,0x1F1D}, {0x1F20,0x1F28}, {0x1F21,0x1F29}, {0x1F22,0x1F2A}, 
+  {0x1F23,0x1F2B}, {0x1F24,0x1F2C}, {0x1F25,0x1F2D}, {0x1F26,0x1F2E}, {0x1F27,0x1F2F}, 
+  {0x1F30,0x1F38}, {0x1F31,0x1F39}, {0x1F32,0x1F3A}, {0x1F33,0x1F3B}, {0x1F34,0x1F3C}, 
+  {0x1F35,0x1F3D}, {0x1F36,0x1F3E}, {0x1F37,0x1F3F}, {0x1F40,0x1F48}, {0x1F41,0x1F49}, 
+  {0x1F42,0x1F4A}, {0x1F43,0x1F4B}, {0x1F44,0x1F4C}, {0x1F45,0x1F4D}, {0x1F51,0x1F59}, 
+  {0x1F53,0x1F5B}, {0x1F55,0x1F5D}, {0x1F57,0x1F5F}, {0x1F60,0x1F68}, {0x1F61,0x1F69}, 
+  {0x1F62,0x1F6A}, {0x1F63,0x1F6B}, {0x1F64,0x1F6C}, {0x1F65,0x1F6D}, {0x1F66,0x1F6E}, 
+  {0x1F67,0x1F6F}, {0x1F70,0x1FBA}, {0x1F71,0x1FBB}, {0x1F72,0x1FC8}, {0x1F73,0x1FC9}, 
+  {0x1F74,0x1FCA}, {0x1F75,0x1FCB}, {0x1F76,0x1FDA}, {0x1F77,0x1FDB}, {0x1F78,0x1FF8}, 
+  {0x1F79,0x1FF9}, {0x1F7A,0x1FEA}, {0x1F7B,0x1FEB}, {0x1F7C,0x1FFA}, {0x1F7D,0x1FFB}, 
+  {0x1F80,0x1F88}, {0x1F81,0x1F89}, {0x1F82,0x1F8A}, {0x1F83,0x1F8B}, {0x1F84,0x1F8C}, 
+  {0x1F85,0x1F8D}, {0x1F86,0x1F8E}, {0x1F87,0x1F8F}, {0x1F90,0x1F98}, {0x1F91,0x1F99}, 
+  {0x1F92,0x1F9A}, {0x1F93,0x1F9B}, {0x1F94,0x1F9C}, {0x1F95,0x1F9D}, {0x1F96,0x1F9E}, 
+  {0x1F97,0x1F9F}, {0x1FA0,0x1FA8}, {0x1FA1,0x1FA9}, {0x1FA2,0x1FAA}, {0x1FA3,0x1FAB}, 
+  {0x1FA4,0x1FAC}, {0x1FA5,0x1FAD}, {0x1FA6,0x1FAE}, {0x1FA7,0x1FAF}, {0x1FB0,0x1FB8}, 
+  {0x1FB1,0x1FB9}, {0x1FB3,0x1FBC}, {0x1FC3,0x1FCC}, {0x1FD0,0x1FD8}, {0x1FD1,0x1FD9}, 
+  {0x1FE0,0x1FE8}, {0x1FE1,0x1FE9}, {0x1FE5,0x1FEC}, {0x1FF3,0x1FFC}, {0x2170,0x2160}, 
+  {0x2171,0x2161}, {0x2172,0x2162}, {0x2173,0x2163}, {0x2174,0x2164}, {0x2175,0x2165}, 
+  {0x2176,0x2166}, {0x2177,0x2167}, {0x2178,0x2168}, {0x2179,0x2169}, {0x217A,0x216A}, 
+  {0x217B,0x216B}, {0x217C,0x216C}, {0x217D,0x216D}, {0x217E,0x216E}, {0x217F,0x216F}, 
+  {0x24D0,0x24B6}, {0x24D1,0x24B7}, {0x24D2,0x24B8}, {0x24D3,0x24B9}, {0x24D4,0x24BA}, 
+  {0x24D5,0x24BB}, {0x24D6,0x24BC}, {0x24D7,0x24BD}, {0x24D8,0x24BE}, {0x24D9,0x24BF}, 
+  {0x24DA,0x24C0}, {0x24DB,0x24C1}, {0x24DC,0x24C2}, {0x24DD,0x24C3}, {0x24DE,0x24C4}, 
+  {0x24DF,0x24C5}, {0x24E0,0x24C6}, {0x24E1,0x24C7}, {0x24E2,0x24C8}, {0x24E3,0x24C9}, 
+  {0x24E4,0x24CA}, {0x24E5,0x24CB}, {0x24E6,0x24CC}, {0x24E7,0x24CD}, {0x24E8,0x24CE}, 
+  {0x24E9,0x24CF}, {0xFF41,0xFF21}, {0xFF42,0xFF22}, {0xFF43,0xFF23}, {0xFF44,0xFF24}, 
+  {0xFF45,0xFF25}, {0xFF46,0xFF26}, {0xFF47,0xFF27}, {0xFF48,0xFF28}, {0xFF49,0xFF29}, 
+  {0xFF4A,0xFF2A}, {0xFF4B,0xFF2B}, {0xFF4C,0xFF2C}, {0xFF4D,0xFF2D}, {0xFF4E,0xFF2E}, 
+  {0xFF4F,0xFF2F}, {0xFF50,0xFF30}, {0xFF51,0xFF31}, {0xFF52,0xFF32}, {0xFF53,0xFF33}, 
+  {0xFF54,0xFF34}, {0xFF55,0xFF35}, {0xFF56,0xFF36}, {0xFF57,0xFF37}, {0xFF58,0xFF38}, 
+  {0xFF59,0xFF39}, {0xFF5A,0xFF3A} 
+};
+
+
+
+/* All lower-case translations are included here */
+
+#define NUM_TOLOWER_INFO 705
+static const unirelate tolower_info[NUM_TOLOWER_INFO] = {
+  {0x41,0x61}, {0x42,0x62}, {0x43,0x63}, {0x44,0x64}, {0x45,0x65}, 
+  {0x46,0x66}, {0x47,0x67}, {0x48,0x68}, {0x49,0x69}, {0x4A,0x6A}, 
+  {0x4B,0x6B}, {0x4C,0x6C}, {0x4D,0x6D}, {0x4E,0x6E}, {0x4F,0x6F}, 
+  {0x50,0x70}, {0x51,0x71}, {0x52,0x72}, {0x53,0x73}, {0x54,0x74}, 
+  {0x55,0x75}, {0x56,0x76}, {0x57,0x77}, {0x58,0x78}, {0x59,0x79}, 
+  {0x5A,0x7A}, {0xC0,0xE0}, {0xC1,0xE1}, {0xC2,0xE2}, {0xC3,0xE3}, 
+  {0xC4,0xE4}, {0xC5,0xE5}, {0xC6,0xE6}, {0xC7,0xE7}, {0xC8,0xE8}, 
+  {0xC9,0xE9}, {0xCA,0xEA}, {0xCB,0xEB}, {0xCC,0xEC}, {0xCD,0xED}, 
+  {0xCE,0xEE}, {0xCF,0xEF}, {0xD0,0xF0}, {0xD1,0xF1}, {0xD2,0xF2}, 
+  {0xD3,0xF3}, {0xD4,0xF4}, {0xD5,0xF5}, {0xD6,0xF6}, {0xD8,0xF8}, 
+  {0xD9,0xF9}, {0xDA,0xFA}, {0xDB,0xFB}, {0xDC,0xFC}, {0xDD,0xFD}, 
+  {0xDE,0xFE}, {0x100,0x101}, {0x102,0x103}, {0x104,0x105}, {0x106,0x107}, 
+  {0x108,0x109}, {0x10A,0x10B}, {0x10C,0x10D}, {0x10E,0x10F}, {0x110,0x111}, 
+  {0x112,0x113}, {0x114,0x115}, {0x116,0x117}, {0x118,0x119}, {0x11A,0x11B}, 
+  {0x11C,0x11D}, {0x11E,0x11F}, {0x120,0x121}, {0x122,0x123}, {0x124,0x125}, 
+  {0x126,0x127}, {0x128,0x129}, {0x12A,0x12B}, {0x12C,0x12D}, {0x12E,0x12F}, 
+  {0x130,0x69}, {0x132,0x133}, {0x134,0x135}, {0x136,0x137}, {0x139,0x13A}, 
+  {0x13B,0x13C}, {0x13D,0x13E}, {0x13F,0x140}, {0x141,0x142}, {0x143,0x144}, 
+  {0x145,0x146}, {0x147,0x148}, {0x14A,0x14B}, {0x14C,0x14D}, {0x14E,0x14F}, 
+  {0x150,0x151}, {0x152,0x153}, {0x154,0x155}, {0x156,0x157}, {0x158,0x159}, 
+  {0x15A,0x15B}, {0x15C,0x15D}, {0x15E,0x15F}, {0x160,0x161}, {0x162,0x163}, 
+  {0x164,0x165}, {0x166,0x167}, {0x168,0x169}, {0x16A,0x16B}, {0x16C,0x16D}, 
+  {0x16E,0x16F}, {0x170,0x171}, {0x172,0x173}, {0x174,0x175}, {0x176,0x177}, 
+  {0x178,0xFF}, {0x179,0x17A}, {0x17B,0x17C}, {0x17D,0x17E}, {0x181,0x253}, 
+  {0x182,0x183}, {0x184,0x185}, {0x186,0x254}, {0x187,0x188}, {0x189,0x256}, 
+  {0x18A,0x257}, {0x18B,0x18C}, {0x18E,0x1DD}, {0x18F,0x259}, {0x190,0x25B}, 
+  {0x191,0x192}, {0x193,0x260}, {0x194,0x263}, {0x196,0x269}, {0x197,0x268}, 
+  {0x198,0x199}, {0x19C,0x26F}, {0x19D,0x272}, {0x19F,0x275}, {0x1A0,0x1A1}, 
+  {0x1A2,0x1A3}, {0x1A4,0x1A5}, {0x1A7,0x1A8}, {0x1A9,0x283}, {0x1AC,0x1AD}, 
+  {0x1AE,0x288}, {0x1AF,0x1B0}, {0x1B1,0x28A}, {0x1B2,0x28B}, {0x1B3,0x1B4}, 
+  {0x1B5,0x1B6}, {0x1B7,0x292}, {0x1B8,0x1B9}, {0x1BC,0x1BD}, {0x1C4,0x1C6}, 
+  {0x1C5,0x1C6}, {0x1C7,0x1C9}, {0x1C8,0x1C9}, {0x1CA,0x1CC}, {0x1CB,0x1CC}, 
+  {0x1CD,0x1CE}, {0x1CF,0x1D0}, {0x1D1,0x1D2}, {0x1D3,0x1D4}, {0x1D5,0x1D6}, 
+  {0x1D7,0x1D8}, {0x1D9,0x1DA}, {0x1DB,0x1DC}, {0x1DE,0x1DF}, {0x1E0,0x1E1}, 
+  {0x1E2,0x1E3}, {0x1E4,0x1E5}, {0x1E6,0x1E7}, {0x1E8,0x1E9}, {0x1EA,0x1EB}, 
+  {0x1EC,0x1ED}, {0x1EE,0x1EF}, {0x1F1,0x1F3}, {0x1F2,0x1F3}, {0x1F4,0x1F5}, 
+  {0x1FA,0x1FB}, {0x1FC,0x1FD}, {0x1FE,0x1FF}, {0x200,0x201}, {0x202,0x203}, 
+  {0x204,0x205}, {0x206,0x207}, {0x208,0x209}, {0x20A,0x20B}, {0x20C,0x20D}, 
+  {0x20E,0x20F}, {0x210,0x211}, {0x212,0x213}, {0x214,0x215}, {0x216,0x217}, 
+  {0x386,0x3AC}, {0x388,0x3AD}, {0x389,0x3AE}, {0x38A,0x3AF}, {0x38C,0x3CC}, 
+  {0x38E,0x3CD}, {0x38F,0x3CE}, {0x391,0x3B1}, {0x392,0x3B2}, {0x393,0x3B3}, 
+  {0x394,0x3B4}, {0x395,0x3B5}, {0x396,0x3B6}, {0x397,0x3B7}, {0x398,0x3B8}, 
+  {0x399,0x3B9}, {0x39A,0x3BA}, {0x39B,0x3BB}, {0x39C,0x3BC}, {0x39D,0x3BD}, 
+  {0x39E,0x3BE}, {0x39F,0x3BF}, {0x3A0,0x3C0}, {0x3A1,0x3C1}, {0x3A3,0x3C3}, 
+  {0x3A4,0x3C4}, {0x3A5,0x3C5}, {0x3A6,0x3C6}, {0x3A7,0x3C7}, {0x3A8,0x3C8}, 
+  {0x3A9,0x3C9}, {0x3AA,0x3CA}, {0x3AB,0x3CB}, {0x3E2,0x3E3}, {0x3E4,0x3E5}, 
+  {0x3E6,0x3E7}, {0x3E8,0x3E9}, {0x3EA,0x3EB}, {0x3EC,0x3ED}, {0x3EE,0x3EF}, 
+  {0x401,0x451}, {0x402,0x452}, {0x403,0x453}, {0x404,0x454}, {0x405,0x455}, 
+  {0x406,0x456}, {0x407,0x457}, {0x408,0x458}, {0x409,0x459}, {0x40A,0x45A}, 
+  {0x40B,0x45B}, {0x40C,0x45C}, {0x40E,0x45E}, {0x40F,0x45F}, {0x410,0x430}, 
+  {0x411,0x431}, {0x412,0x432}, {0x413,0x433}, {0x414,0x434}, {0x415,0x435}, 
+  {0x416,0x436}, {0x417,0x437}, {0x418,0x438}, {0x419,0x439}, {0x41A,0x43A}, 
+  {0x41B,0x43B}, {0x41C,0x43C}, {0x41D,0x43D}, {0x41E,0x43E}, {0x41F,0x43F}, 
+  {0x420,0x440}, {0x421,0x441}, {0x422,0x442}, {0x423,0x443}, {0x424,0x444}, 
+  {0x425,0x445}, {0x426,0x446}, {0x427,0x447}, {0x428,0x448}, {0x429,0x449}, 
+  {0x42A,0x44A}, {0x42B,0x44B}, {0x42C,0x44C}, {0x42D,0x44D}, {0x42E,0x44E}, 
+  {0x42F,0x44F}, {0x460,0x461}, {0x462,0x463}, {0x464,0x465}, {0x466,0x467}, 
+  {0x468,0x469}, {0x46A,0x46B}, {0x46C,0x46D}, {0x46E,0x46F}, {0x470,0x471}, 
+  {0x472,0x473}, {0x474,0x475}, {0x476,0x477}, {0x478,0x479}, {0x47A,0x47B}, 
+  {0x47C,0x47D}, {0x47E,0x47F}, {0x480,0x481}, {0x490,0x491}, {0x492,0x493}, 
+  {0x494,0x495}, {0x496,0x497}, {0x498,0x499}, {0x49A,0x49B}, {0x49C,0x49D}, 
+  {0x49E,0x49F}, {0x4A0,0x4A1}, {0x4A2,0x4A3}, {0x4A4,0x4A5}, {0x4A6,0x4A7}, 
+  {0x4A8,0x4A9}, {0x4AA,0x4AB}, {0x4AC,0x4AD}, {0x4AE,0x4AF}, {0x4B0,0x4B1}, 
+  {0x4B2,0x4B3}, {0x4B4,0x4B5}, {0x4B6,0x4B7}, {0x4B8,0x4B9}, {0x4BA,0x4BB}, 
+  {0x4BC,0x4BD}, {0x4BE,0x4BF}, {0x4C1,0x4C2}, {0x4C3,0x4C4}, {0x4C7,0x4C8}, 
+  {0x4CB,0x4CC}, {0x4D0,0x4D1}, {0x4D2,0x4D3}, {0x4D4,0x4D5}, {0x4D6,0x4D7}, 
+  {0x4D8,0x4D9}, {0x4DA,0x4DB}, {0x4DC,0x4DD}, {0x4DE,0x4DF}, {0x4E0,0x4E1}, 
+  {0x4E2,0x4E3}, {0x4E4,0x4E5}, {0x4E6,0x4E7}, {0x4E8,0x4E9}, {0x4EA,0x4EB}, 
+  {0x4EE,0x4EF}, {0x4F0,0x4F1}, {0x4F2,0x4F3}, {0x4F4,0x4F5}, {0x4F8,0x4F9}, 
+  {0x531,0x561}, {0x532,0x562}, {0x533,0x563}, {0x534,0x564}, {0x535,0x565}, 
+  {0x536,0x566}, {0x537,0x567}, {0x538,0x568}, {0x539,0x569}, {0x53A,0x56A}, 
+  {0x53B,0x56B}, {0x53C,0x56C}, {0x53D,0x56D}, {0x53E,0x56E}, {0x53F,0x56F}, 
+  {0x540,0x570}, {0x541,0x571}, {0x542,0x572}, {0x543,0x573}, {0x544,0x574}, 
+  {0x545,0x575}, {0x546,0x576}, {0x547,0x577}, {0x548,0x578}, {0x549,0x579}, 
+  {0x54A,0x57A}, {0x54B,0x57B}, {0x54C,0x57C}, {0x54D,0x57D}, {0x54E,0x57E}, 
+  {0x54F,0x57F}, {0x550,0x580}, {0x551,0x581}, {0x552,0x582}, {0x553,0x583}, 
+  {0x554,0x584}, {0x555,0x585}, {0x556,0x586}, {0x10A0,0x10D0}, {0x10A1,0x10D1}, 
+  {0x10A2,0x10D2}, {0x10A3,0x10D3}, {0x10A4,0x10D4}, {0x10A5,0x10D5}, {0x10A6,0x10D6}, 
+  {0x10A7,0x10D7}, {0x10A8,0x10D8}, {0x10A9,0x10D9}, {0x10AA,0x10DA}, {0x10AB,0x10DB}, 
+  {0x10AC,0x10DC}, {0x10AD,0x10DD}, {0x10AE,0x10DE}, {0x10AF,0x10DF}, {0x10B0,0x10E0}, 
+  {0x10B1,0x10E1}, {0x10B2,0x10E2}, {0x10B3,0x10E3}, {0x10B4,0x10E4}, {0x10B5,0x10E5}, 
+  {0x10B6,0x10E6}, {0x10B7,0x10E7}, {0x10B8,0x10E8}, {0x10B9,0x10E9}, {0x10BA,0x10EA}, 
+  {0x10BB,0x10EB}, {0x10BC,0x10EC}, {0x10BD,0x10ED}, {0x10BE,0x10EE}, {0x10BF,0x10EF}, 
+  {0x10C0,0x10F0}, {0x10C1,0x10F1}, {0x10C2,0x10F2}, {0x10C3,0x10F3}, {0x10C4,0x10F4}, 
+  {0x10C5,0x10F5}, {0x1E00,0x1E01}, {0x1E02,0x1E03}, {0x1E04,0x1E05}, {0x1E06,0x1E07}, 
+  {0x1E08,0x1E09}, {0x1E0A,0x1E0B}, {0x1E0C,0x1E0D}, {0x1E0E,0x1E0F}, {0x1E10,0x1E11}, 
+  {0x1E12,0x1E13}, {0x1E14,0x1E15}, {0x1E16,0x1E17}, {0x1E18,0x1E19}, {0x1E1A,0x1E1B}, 
+  {0x1E1C,0x1E1D}, {0x1E1E,0x1E1F}, {0x1E20,0x1E21}, {0x1E22,0x1E23}, {0x1E24,0x1E25}, 
+  {0x1E26,0x1E27}, {0x1E28,0x1E29}, {0x1E2A,0x1E2B}, {0x1E2C,0x1E2D}, {0x1E2E,0x1E2F}, 
+  {0x1E30,0x1E31}, {0x1E32,0x1E33}, {0x1E34,0x1E35}, {0x1E36,0x1E37}, {0x1E38,0x1E39}, 
+  {0x1E3A,0x1E3B}, {0x1E3C,0x1E3D}, {0x1E3E,0x1E3F}, {0x1E40,0x1E41}, {0x1E42,0x1E43}, 
+  {0x1E44,0x1E45}, {0x1E46,0x1E47}, {0x1E48,0x1E49}, {0x1E4A,0x1E4B}, {0x1E4C,0x1E4D}, 
+  {0x1E4E,0x1E4F}, {0x1E50,0x1E51}, {0x1E52,0x1E53}, {0x1E54,0x1E55}, {0x1E56,0x1E57}, 
+  {0x1E58,0x1E59}, {0x1E5A,0x1E5B}, {0x1E5C,0x1E5D}, {0x1E5E,0x1E5F}, {0x1E60,0x1E61}, 
+  {0x1E62,0x1E63}, {0x1E64,0x1E65}, {0x1E66,0x1E67}, {0x1E68,0x1E69}, {0x1E6A,0x1E6B}, 
+  {0x1E6C,0x1E6D}, {0x1E6E,0x1E6F}, {0x1E70,0x1E71}, {0x1E72,0x1E73}, {0x1E74,0x1E75}, 
+  {0x1E76,0x1E77}, {0x1E78,0x1E79}, {0x1E7A,0x1E7B}, {0x1E7C,0x1E7D}, {0x1E7E,0x1E7F}, 
+  {0x1E80,0x1E81}, {0x1E82,0x1E83}, {0x1E84,0x1E85}, {0x1E86,0x1E87}, {0x1E88,0x1E89}, 
+  {0x1E8A,0x1E8B}, {0x1E8C,0x1E8D}, {0x1E8E,0x1E8F}, {0x1E90,0x1E91}, {0x1E92,0x1E93}, 
+  {0x1E94,0x1E95}, {0x1EA0,0x1EA1}, {0x1EA2,0x1EA3}, {0x1EA4,0x1EA5}, {0x1EA6,0x1EA7}, 
+  {0x1EA8,0x1EA9}, {0x1EAA,0x1EAB}, {0x1EAC,0x1EAD}, {0x1EAE,0x1EAF}, {0x1EB0,0x1EB1}, 
+  {0x1EB2,0x1EB3}, {0x1EB4,0x1EB5}, {0x1EB6,0x1EB7}, {0x1EB8,0x1EB9}, {0x1EBA,0x1EBB}, 
+  {0x1EBC,0x1EBD}, {0x1EBE,0x1EBF}, {0x1EC0,0x1EC1}, {0x1EC2,0x1EC3}, {0x1EC4,0x1EC5}, 
+  {0x1EC6,0x1EC7}, {0x1EC8,0x1EC9}, {0x1ECA,0x1ECB}, {0x1ECC,0x1ECD}, {0x1ECE,0x1ECF}, 
+  {0x1ED0,0x1ED1}, {0x1ED2,0x1ED3}, {0x1ED4,0x1ED5}, {0x1ED6,0x1ED7}, {0x1ED8,0x1ED9}, 
+  {0x1EDA,0x1EDB}, {0x1EDC,0x1EDD}, {0x1EDE,0x1EDF}, {0x1EE0,0x1EE1}, {0x1EE2,0x1EE3}, 
+  {0x1EE4,0x1EE5}, {0x1EE6,0x1EE7}, {0x1EE8,0x1EE9}, {0x1EEA,0x1EEB}, {0x1EEC,0x1EED}, 
+  {0x1EEE,0x1EEF}, {0x1EF0,0x1EF1}, {0x1EF2,0x1EF3}, {0x1EF4,0x1EF5}, {0x1EF6,0x1EF7}, 
+  {0x1EF8,0x1EF9}, {0x1F08,0x1F00}, {0x1F09,0x1F01}, {0x1F0A,0x1F02}, {0x1F0B,0x1F03}, 
+  {0x1F0C,0x1F04}, {0x1F0D,0x1F05}, {0x1F0E,0x1F06}, {0x1F0F,0x1F07}, {0x1F18,0x1F10}, 
+  {0x1F19,0x1F11}, {0x1F1A,0x1F12}, {0x1F1B,0x1F13}, {0x1F1C,0x1F14}, {0x1F1D,0x1F15}, 
+  {0x1F28,0x1F20}, {0x1F29,0x1F21}, {0x1F2A,0x1F22}, {0x1F2B,0x1F23}, {0x1F2C,0x1F24}, 
+  {0x1F2D,0x1F25}, {0x1F2E,0x1F26}, {0x1F2F,0x1F27}, {0x1F38,0x1F30}, {0x1F39,0x1F31}, 
+  {0x1F3A,0x1F32}, {0x1F3B,0x1F33}, {0x1F3C,0x1F34}, {0x1F3D,0x1F35}, {0x1F3E,0x1F36}, 
+  {0x1F3F,0x1F37}, {0x1F48,0x1F40}, {0x1F49,0x1F41}, {0x1F4A,0x1F42}, {0x1F4B,0x1F43}, 
+  {0x1F4C,0x1F44}, {0x1F4D,0x1F45}, {0x1F59,0x1F51}, {0x1F5B,0x1F53}, {0x1F5D,0x1F55}, 
+  {0x1F5F,0x1F57}, {0x1F68,0x1F60}, {0x1F69,0x1F61}, {0x1F6A,0x1F62}, {0x1F6B,0x1F63}, 
+  {0x1F6C,0x1F64}, {0x1F6D,0x1F65}, {0x1F6E,0x1F66}, {0x1F6F,0x1F67}, {0x1F88,0x1F80}, 
+  {0x1F89,0x1F81}, {0x1F8A,0x1F82}, {0x1F8B,0x1F83}, {0x1F8C,0x1F84}, {0x1F8D,0x1F85}, 
+  {0x1F8E,0x1F86}, {0x1F8F,0x1F87}, {0x1F98,0x1F90}, {0x1F99,0x1F91}, {0x1F9A,0x1F92}, 
+  {0x1F9B,0x1F93}, {0x1F9C,0x1F94}, {0x1F9D,0x1F95}, {0x1F9E,0x1F96}, {0x1F9F,0x1F97}, 
+  {0x1FA8,0x1FA0}, {0x1FA9,0x1FA1}, {0x1FAA,0x1FA2}, {0x1FAB,0x1FA3}, {0x1FAC,0x1FA4}, 
+  {0x1FAD,0x1FA5}, {0x1FAE,0x1FA6}, {0x1FAF,0x1FA7}, {0x1FB8,0x1FB0}, {0x1FB9,0x1FB1}, 
+  {0x1FBA,0x1F70}, {0x1FBB,0x1F71}, {0x1FBC,0x1FB3}, {0x1FC8,0x1F72}, {0x1FC9,0x1F73}, 
+  {0x1FCA,0x1F74}, {0x1FCB,0x1F75}, {0x1FCC,0x1FC3}, {0x1FD8,0x1FD0}, {0x1FD9,0x1FD1}, 
+  {0x1FDA,0x1F76}, {0x1FDB,0x1F77}, {0x1FE8,0x1FE0}, {0x1FE9,0x1FE1}, {0x1FEA,0x1F7A}, 
+  {0x1FEB,0x1F7B}, {0x1FEC,0x1FE5}, {0x1FF8,0x1F78}, {0x1FF9,0x1F79}, {0x1FFA,0x1F7C}, 
+  {0x1FFB,0x1F7D}, {0x1FFC,0x1FF3}, {0x2160,0x2170}, {0x2161,0x2171}, {0x2162,0x2172}, 
+  {0x2163,0x2173}, {0x2164,0x2174}, {0x2165,0x2175}, {0x2166,0x2176}, {0x2167,0x2177}, 
+  {0x2168,0x2178}, {0x2169,0x2179}, {0x216A,0x217A}, {0x216B,0x217B}, {0x216C,0x217C}, 
+  {0x216D,0x217D}, {0x216E,0x217E}, {0x216F,0x217F}, {0x24B6,0x24D0}, {0x24B7,0x24D1}, 
+  {0x24B8,0x24D2}, {0x24B9,0x24D3}, {0x24BA,0x24D4}, {0x24BB,0x24D5}, {0x24BC,0x24D6}, 
+  {0x24BD,0x24D7}, {0x24BE,0x24D8}, {0x24BF,0x24D9}, {0x24C0,0x24DA}, {0x24C1,0x24DB}, 
+  {0x24C2,0x24DC}, {0x24C3,0x24DD}, {0x24C4,0x24DE}, {0x24C5,0x24DF}, {0x24C6,0x24E0}, 
+  {0x24C7,0x24E1}, {0x24C8,0x24E2}, {0x24C9,0x24E3}, {0x24CA,0x24E4}, {0x24CB,0x24E5}, 
+  {0x24CC,0x24E6}, {0x24CD,0x24E7}, {0x24CE,0x24E8}, {0x24CF,0x24E9}, {0xFF21,0xFF41}, 
+  {0xFF22,0xFF42}, {0xFF23,0xFF43}, {0xFF24,0xFF44}, {0xFF25,0xFF45}, {0xFF26,0xFF46}, 
+  {0xFF27,0xFF47}, {0xFF28,0xFF48}, {0xFF29,0xFF49}, {0xFF2A,0xFF4A}, {0xFF2B,0xFF4B}, 
+  {0xFF2C,0xFF4C}, {0xFF2D,0xFF4D}, {0xFF2E,0xFF4E}, {0xFF2F,0xFF4F}, {0xFF30,0xFF50}, 
+  {0xFF31,0xFF51}, {0xFF32,0xFF52}, {0xFF33,0xFF53}, {0xFF34,0xFF54}, {0xFF35,0xFF55}, 
+  {0xFF36,0xFF56}, {0xFF37,0xFF57}, {0xFF38,0xFF58}, {0xFF39,0xFF59}, {0xFF3A,0xFF5A}
+};
+
+
+
+#define NUM_TOSIMPLIFIED_INFO 2145
+static const unirelate tosimplified_info[NUM_TOSIMPLIFIED_INFO] = {
+  {0x4E48,0x9EBD}, {0x4E82,0x4E71}, {0x4E9E,0x4E9A}, {0x4F47,0x4F2B}, {0x4F59,0x9980}, 
+  {0x4F86,0x6765}, {0x4F96,0x4ED1}, {0x4FE0,0x4FA0}, {0x5000,0x4F25}, {0x5006,0x4FE9}, 
+  {0x5009,0x4ED3}, {0x500B,0x4E2A}, {0x5011,0x4EEC}, {0x502B,0x4F26}, {0x5049,0x4F1F}, 
+  {0x5074,0x4FA7}, {0x5075,0x4FA6}, {0x5096,0x4F27}, {0x5098,0x4F1E}, {0x5099,0x5907}, 
+  {0x50AD,0x4F63}, {0x50B3,0x4F20}, {0x50B4,0x4F1B}, {0x50B5,0x503A}, {0x50B7,0x4F24}, 
+  {0x50BE,0x503E}, {0x50C2,0x507B}, {0x50C5,0x4EC5}, {0x50C9,0x4F65}, {0x50D1,0x4FA8}, 
+  {0x50D5,0x4EC6}, {0x50DE,0x4F2A}, {0x50E5,0x4FA5}, {0x50E8,0x507E}, {0x50F9,0x4EF7}, 
+  {0x5100,0x4EEA}, {0x5102,0x4FAC}, {0x5104,0x4EBF}, {0x5108,0x4FA9}, {0x5109,0x4FED}, 
+  {0x5110,0x50A7}, {0x5114,0x4FE6}, {0x5115,0x4FAA}, {0x511F,0x507F}, {0x512A,0x4F18}, 
+  {0x5132,0x50A8}, {0x5137,0x4FEA}, {0x513A,0x50A9}, {0x513B,0x50A5}, {0x513C,0x4FE8}, 
+  {0x5152,0x513F}, {0x5169,0x4E24}, {0x51C8,0x51C0}, {0x51CD,0x51BB}, {0x51F1,0x51EF}, 
+  {0x5244,0x522D}, {0x5247,0x5219}, {0x525B,0x521A}, {0x526E,0x5250}, {0x5274,0x5240}, 
+  {0x5275,0x521B}, {0x5283,0x5212}, {0x5287,0x5267}, {0x5289,0x5218}, {0x528A,0x523D}, 
+  {0x528C,0x523F}, {0x528D,0x5251}, {0x5291,0x5242}, {0x52C1,0x52B2}, {0x52D5,0x52A8}, 
+  {0x52D9,0x52A1}, {0x52DB,0x52CB}, {0x52DD,0x80DC}, {0x52DE,0x52B3}, {0x52E2,0x52BF}, 
+  {0x52F1,0x52A2}, {0x52F5,0x52B1}, {0x52F8,0x529D}, {0x532D,0x5326}, {0x532F,0x6C47}, 
+  {0x5331,0x532E}, {0x5340,0x533A}, {0x5354,0x534F}, {0x5399,0x538D}, {0x53A0,0x5395}, 
+  {0x53AD,0x538C}, {0x53B2,0x5389}, {0x53B4,0x53A3}, {0x53C3,0x53C2}, {0x53E2,0x4E1B}, 
+  {0x540E,0x5F8C}, {0x54BC,0x5459}, {0x54E1,0x5458}, {0x5504,0x5457}, {0x554F,0x95EE}, 
+  {0x5553,0x542F}, {0x555E,0x54D1}, {0x559A,0x5524}, {0x55AA,0x4E27}, {0x55AC,0x4E54}, 
+  {0x55AE,0x5355}, {0x55B2,0x54DF}, {0x55C6,0x545B}, {0x55C7,0x556C}, {0x55CE,0x5417}, 
+  {0x55DA,0x545C}, {0x55E9,0x5522}, {0x55F6,0x54D4}, {0x5606,0x53F9}, {0x560D,0x55BD}, 
+  {0x5614,0x5455}, {0x5616,0x5567}, {0x5617,0x5C1D}, {0x561C,0x551B}, {0x5629,0x54D7}, 
+  {0x562E,0x5520}, {0x562F,0x5578}, {0x5630,0x53FD}, {0x5635,0x54D3}, {0x5638,0x5452}, 
+  {0x565D,0x549D}, {0x5660,0x54D2}, {0x5665,0x54DD}, {0x5666,0x54D5}, {0x566F,0x55F3}, 
+  {0x5672,0x54D9}, {0x5674,0x55B7}, {0x5678,0x5428}, {0x5680,0x549B}, {0x5687,0x5413}, 
+  {0x568C,0x54DC}, {0x5695,0x565C}, {0x5699,0x556E}, {0x56A6,0x5456}, {0x56A8,0x5499}, 
+  {0x56B3,0x55BE}, {0x56B4,0x4E25}, {0x56B6,0x5624}, {0x56C0,0x556D}, {0x56C1,0x55EB}, 
+  {0x56C2,0x56A3}, {0x56C5,0x5181}, {0x56C8,0x5453}, {0x56D1,0x5631}, {0x5707,0x56F5}, 
+  {0x570B,0x56FD}, {0x570D,0x56F4}, {0x5712,0x56ED}, {0x5713,0x5706}, {0x5716,0x56FE}, 
+  {0x5718,0x56E2}, {0x57E1,0x57AD}, {0x57F7,0x6267}, {0x5805,0x575A}, {0x580A,0x57A9}, 
+  {0x5816,0x57B4}, {0x581D,0x57DA}, {0x582F,0x5C27}, {0x5831,0x62A5}, {0x5834,0x573A}, 
+  {0x584A,0x5757}, {0x584B,0x8314}, {0x584F,0x57B2}, {0x5852,0x57D8}, {0x5857,0x6D82}, 
+  {0x5862,0x575E}, {0x5864,0x57D9}, {0x5875,0x5C18}, {0x5879,0x5811}, {0x588A,0x57AB}, 
+  {0x589C,0x5760}, {0x58AE,0x5815}, {0x58B3,0x575F}, {0x58BB,0x5899}, {0x58BE,0x57A6}, 
+  {0x58C7,0x575B}, {0x58D3,0x538B}, {0x58D8,0x5792}, {0x58D9,0x5739}, {0x58DA,0x5786}, 
+  {0x58DE,0x574F}, {0x58DF,0x5784}, {0x58E0,0x5785}, {0x58E2,0x575C}, {0x58E9,0x575D}, 
+  {0x58EF,0x58EE}, {0x58FA,0x58F6}, {0x58FD,0x5BFF}, {0x5922,0x68A6}, {0x593E,0x5939}, 
+  {0x5950,0x5942}, {0x5969,0x5941}, {0x596A,0x593A}, {0x596C,0x5956}, {0x596E,0x594B}, 
+  {0x599D,0x5986}, {0x5A41,0x5A04}, {0x5A66,0x5987}, {0x5A6D,0x5A05}, {0x5AA7,0x5A32}, 
+  {0x5ABD,0x5988}, {0x5AD7,0x59AA}, {0x5AF5,0x59A9}, {0x5AFB,0x5A34}, {0x5B00,0x59AB}, 
+  {0x5B08,0x5A06}, {0x5B0B,0x5A75}, {0x5B0C,0x5A07}, {0x5B19,0x5AF1}, {0x5B21,0x5AD2}, 
+  {0x5B2A,0x5AD4}, {0x5B30,0x5A74}, {0x5B38,0x5A76}, {0x5B4C,0x5A08}, {0x5B6B,0x5B59}, 
+  {0x5B78,0x5B66}, {0x5B7F,0x5B6A}, {0x5BE2,0x5BDD}, {0x5BE6,0x5B9E}, {0x5BE7,0x5B81}, 
+  {0x5BE9,0x5BA1}, {0x5BEB,0x5199}, {0x5BEC,0x5BBD}, {0x5BF5,0x5BA0}, {0x5BF6,0x5B9D}, 
+  {0x5C07,0x5C06}, {0x5C08,0x4E13}, {0x5C0B,0x5BFB}, {0x5C0D,0x5BF9}, {0x5C0E,0x5BFC}, 
+  {0x5C37,0x5C34}, {0x5C62,0x5C61}, {0x5C64,0x5C42}, {0x5C68,0x5C66}, {0x5C6C,0x5C5E}, 
+  {0x5CA1,0x5188}, {0x5CF4,0x5C98}, {0x5CF6,0x5C9B}, {0x5CFD,0x5CE1}, {0x5D0D,0x5D03}, 
+  {0x5D17,0x5C97}, {0x5D22,0x5CE5}, {0x5D2C,0x5CBD}, {0x5D50,0x5C9A}, {0x5D81,0x5D5D}, 
+  {0x5D84,0x5D2D}, {0x5D87,0x5C96}, {0x5D97,0x5D02}, {0x5DA0,0x5CE4}, {0x5DA7,0x5CC4}, 
+  {0x5DB8,0x5D58}, {0x5DBA,0x5CAD}, {0x5DBC,0x5C7F}, {0x5DCB,0x5CBF}, {0x5DD2,0x5CE6}, 
+  {0x5DD4,0x5DC5}, {0x5DF0,0x5DEF}, {0x5E25,0x5E05}, {0x5E2B,0x5E08}, {0x5E33,0x5E10}, 
+  {0x5E36,0x5E26}, {0x5E40,0x5E27}, {0x5E43,0x5E0F}, {0x5E57,0x5E3C}, {0x5E58,0x5E3B}, 
+  {0x5E5F,0x5E1C}, {0x5E63,0x5E01}, {0x5E6B,0x5E2E}, {0x5E6C,0x5E31}, {0x5E79,0x5E72}, 
+  {0x5E7E,0x51E0}, {0x5EAB,0x5E93}, {0x5EDF,0x5E99}, {0x5EE0,0x5382}, {0x5EE1,0x5E91}, 
+  {0x5EE2,0x5E9F}, {0x5EE3,0x5E7F}, {0x5EEC,0x5E90}, {0x5EF3,0x5385}, {0x5F33,0x5F2A}, 
+  {0x5F35,0x5F20}, {0x5F48,0x5F39}, {0x5F4C,0x5F25}, {0x5F4E,0x5F2F}, {0x5F8C,0x540E}, 
+  {0x5F91,0x5F84}, {0x5F9E,0x4ECE}, {0x5FA0,0x5F95}, {0x5FA9,0x590D}, {0x5FB9,0x5F7B}, 
+  {0x60B5,0x6005}, {0x60B6,0x95F7}, {0x60E1,0x6076}, {0x60F1,0x607C}, {0x60F2,0x607D}, 
+  {0x60FB,0x607B}, {0x611B,0x7231}, {0x611C,0x60EC}, {0x6134,0x6006}, {0x6137,0x607A}, 
+  {0x613E,0x5FFE}, {0x614B,0x6001}, {0x6158,0x60E8}, {0x615A,0x60ED}, {0x615F,0x6078}, 
+  {0x6163,0x60EF}, {0x6164,0x60AB}, {0x616A,0x6004}, {0x616B,0x6002}, {0x616E,0x8651}, 
+  {0x6173,0x60AD}, {0x6176,0x5E86}, {0x6182,0x5FE7}, {0x618A,0x60EB}, {0x6190,0x601C}, 
+  {0x6191,0x51ED}, {0x6192,0x6126}, {0x619A,0x60EE}, {0x61A4,0x6124}, {0x61AB,0x60AF}, 
+  {0x61AE,0x6003}, {0x61B2,0x5BAA}, {0x61B6,0x5FC6}, {0x61C7,0x6073}, {0x61C9,0x5E94}, 
+  {0x61CC,0x603F}, {0x61DF,0x603C}, {0x61E3,0x61D1}, {0x61E8,0x6079}, {0x61F2,0x60E9}, 
+  {0x61F6,0x61D2}, {0x61F7,0x6000}, {0x61F8,0x60AC}, {0x61FA,0x5FCF}, {0x61FC,0x60E7}, 
+  {0x61FE,0x6151}, {0x6200,0x604B}, {0x6207,0x6206}, {0x6214,0x620B}, {0x6227,0x6217}, 
+  {0x6229,0x622C}, {0x6230,0x6218}, {0x6232,0x620F}, {0x633E,0x631F}, {0x6368,0x820D}, 
+  {0x636B,0x626A}, {0x6383,0x626B}, {0x6384,0x62A1}, {0x6399,0x6323}, {0x63C0,0x62E3}, 
+  {0x63DA,0x626C}, {0x63DB,0x6362}, {0x63EE,0x6325}, {0x640D,0x635F}, {0x6417,0x6363}, 
+  {0x6436,0x62A2}, {0x6451,0x63B4}, {0x645C,0x63BC}, {0x645F,0x6402}, {0x646F,0x631A}, 
+  {0x6473,0x62A0}, {0x6476,0x629F}, {0x647B,0x63BA}, {0x6488,0x635E}, {0x6493,0x6320}, 
+  {0x649F,0x6322}, {0x64A3,0x63B8}, {0x64A5,0x62E8}, {0x64AB,0x629A}, {0x64B2,0x6251}, 
+  {0x64B3,0x63FF}, {0x64BB,0x631E}, {0x64BE,0x631D}, {0x64BF,0x6361}, {0x64C1,0x62E5}, 
+  {0x64C4,0x63B3}, {0x64C7,0x62E9}, {0x64CA,0x51FB}, {0x64CB,0x6321}, {0x64D4,0x62C5}, 
+  {0x64DA,0x636E}, {0x64E0,0x6324}, {0x64EC,0x62DF}, {0x64EF,0x6448}, {0x64F0,0x62E7}, 
+  {0x64F1,0x6401}, {0x64F2,0x63B7}, {0x64F4,0x6269}, {0x64F7,0x64B7}, {0x64FA,0x6446}, 
+  {0x64FB,0x64DE}, {0x64FC,0x64B8}, {0x64FE,0x6270}, {0x6504,0x6445}, {0x6506,0x64B5}, 
+  {0x650F,0x62E2}, {0x6514,0x62E6}, {0x6516,0x6484}, {0x6519,0x6400}, {0x651B,0x64BA}, 
+  {0x651D,0x6444}, {0x6522,0x6512}, {0x6523,0x631B}, {0x6524,0x644A}, {0x652A,0x6405}, 
+  {0x652C,0x63FD}, {0x6557,0x8D25}, {0x6575,0x654C}, {0x6578,0x6570}, {0x6582,0x655B}, 
+  {0x6583,0x6BD9}, {0x6595,0x6593}, {0x65AC,0x65A9}, {0x65B7,0x65AD}, {0x6642,0x65F6}, 
+  {0x6649,0x664B}, {0x665D,0x663C}, {0x6688,0x6655}, {0x6689,0x6656}, {0x66A2,0x7545}, 
+  {0x66AB,0x6682}, {0x66C4,0x6654}, {0x66C7,0x6619}, {0x66C9,0x6653}, {0x66D6,0x66A7}, 
+  {0x66E0,0x65F7}, {0x66EC,0x6652}, {0x66F8,0x4E66}, {0x6703,0x4F1A}, {0x6727,0x80E7}, 
+  {0x6771,0x4E1C}, {0x6898,0x67A7}, {0x689D,0x6761}, {0x689F,0x67AD}, {0x68C4,0x5F03}, 
+  {0x68D6,0x67A8}, {0x68D7,0x67A3}, {0x68DF,0x680B}, {0x68E7,0x6808}, {0x68F2,0x6816}, 
+  {0x690F,0x6860}, {0x694A,0x6768}, {0x6953,0x67AB}, {0x6968,0x6862}, {0x696D,0x4E1A}, 
+  {0x6975,0x6781}, {0x69AA,0x6769}, {0x69AE,0x8363}, {0x69BF,0x6864}, {0x69CB,0x6784}, 
+  {0x69CD,0x67AA}, {0x69E7,0x6920}, {0x69F3,0x6868}, {0x6A01,0x6869}, {0x6A02,0x4E50}, 
+  {0x6A05,0x679E}, {0x6A13,0x697C}, {0x6A19,0x6807}, {0x6A1E,0x67A2}, {0x6A23,0x6837}, 
+  {0x6A38,0x6734}, {0x6A39,0x6811}, {0x6A3A,0x6866}, {0x6A48,0x6861}, {0x6A4B,0x6865}, 
+  {0x6A5F,0x673A}, {0x6A62,0x692D}, {0x6A89,0x67FD}, {0x6A94,0x6863}, {0x6A9C,0x6867}, 
+  {0x6AA2,0x68C0}, {0x6AA3,0x6A2F}, {0x6AB3,0x69DF}, {0x6AB8,0x67E0}, {0x6ABB,0x69DB}, 
+  {0x6AC3,0x67DC}, {0x6AD3,0x6A79}, {0x6ADA,0x6988}, {0x6ADB,0x6809}, {0x6ADD,0x691F}, 
+  {0x6ADE,0x6A7C}, {0x6ADF,0x680E}, {0x6AE7,0x69E0}, {0x6AE8,0x680C}, {0x6AEA,0x67A5}, 
+  {0x6AEC,0x6987}, {0x6AF3,0x680A}, {0x6AF8,0x6989}, {0x6AFB,0x6A31}, {0x6B04,0x680F}, 
+  {0x6B0A,0x6743}, {0x6B0F,0x6924}, {0x6B12,0x683E}, {0x6B16,0x6984}, {0x6B1E,0x68C2}, 
+  {0x6B3D,0x94A6}, {0x6B50,0x6B27}, {0x6B5F,0x6B24}, {0x6B61,0x6B22}, {0x6B72,0x5C81}, 
+  {0x6B77,0x5386}, {0x6B78,0x5F52}, {0x6B98,0x6B8B}, {0x6B9E,0x6B92}, {0x6BA4,0x6B87}, 
+  {0x6BAB,0x6B9A}, {0x6BAE,0x6B93}, {0x6BAF,0x6BA1}, {0x6BB2,0x6B7C}, {0x6BBA,0x6740}, 
+  {0x6BBB,0x58F3}, {0x6BC6,0x6BB4}, {0x6BFF,0x6BF5}, {0x6C08,0x6BE1}, {0x6C0C,0x6C07}, 
+  {0x6C23,0x6C14}, {0x6C2B,0x6C22}, {0x6C2C,0x6C29}, {0x6D79,0x6D43}, {0x6D87,0x6CFE}, 
+  {0x6DEA,0x6CA6}, {0x6DF5,0x6E0A}, {0x6DF6,0x6D9E}, {0x6DFA,0x6D45}, {0x6E19,0x6DA3}, 
+  {0x6E26,0x6DA1}, {0x6E2C,0x6D4B}, {0x6E3E,0x6D51}, {0x6E5E,0x6D48}, {0x6E6F,0x6C64}, 
+  {0x6E96,0x51C6}, {0x6E9D,0x6C9F}, {0x6EC4,0x6CA7}, {0x6EC5,0x706D}, {0x6ECC,0x6DA4}, 
+  {0x6ECE,0x8365}, {0x6EEC,0x6CAA}, {0x6EEF,0x6EDE}, {0x6EF2,0x6E17}, {0x6EF7,0x5364}, 
+  {0x6EF8,0x6D52}, {0x6EFF,0x6EE1}, {0x6F01,0x6E14}, {0x6F1A,0x6CA4}, {0x6F22,0x6C49}, 
+  {0x6F23,0x6D9F}, {0x6F2C,0x6E0D}, {0x6F32,0x6DA8}, {0x6F38,0x6E10}, {0x6F3F,0x6D46}, 
+  {0x6F41,0x988D}, {0x6F51,0x6CFC}, {0x6F54,0x6D01}, {0x6F59,0x6CA9}, {0x6F5B,0x6F5C}, 
+  {0x6F64,0x6DA6}, {0x6F6F,0x6D54}, {0x6F70,0x6E83}, {0x6F77,0x6ED7}, {0x6F7F,0x6DA0}, 
+  {0x6F80,0x6DA9}, {0x6F86,0x6D47}, {0x6F87,0x6D9D}, {0x6F97,0x6DA7}, {0x6FA0,0x6E11}, 
+  {0x6FA4,0x6CFD}, {0x6FA9,0x6CF6}, {0x6FAE,0x6D4D}, {0x6FB1,0x6DC0}, {0x6FC1,0x6D4A}, 
+  {0x6FC3,0x6D53}, {0x6FD5,0x6E7F}, {0x6FD8,0x6CDE}, {0x6FDF,0x6D4E}, {0x6FE4,0x6D9B}, 
+  {0x6FEB,0x6EE5}, {0x6FF0,0x6F4D}, {0x6FF1,0x6EE8}, {0x6FFA,0x6E85}, {0x6FFC,0x6CFA}, 
+  {0x6FFE,0x6EE4}, {0x7005,0x6EE2}, {0x7006,0x6E0E}, {0x7009,0x6CFB}, {0x700B,0x6E16}, 
+  {0x700F,0x6D4F}, {0x7015,0x6FD2}, {0x7018,0x6CF8}, {0x701D,0x6CA5}, {0x701F,0x6F47}, 
+  {0x7020,0x6F46}, {0x7027,0x6CF7}, {0x7028,0x6FD1}, {0x7032,0x6F4B}, {0x703E,0x6F9C}, 
+  {0x7043,0x6CA3}, {0x7044,0x6EE0}, {0x7051,0x6D12}, {0x7055,0x6F13}, {0x7058,0x6EE9}, 
+  {0x705D,0x704F}, {0x7063,0x6E7E}, {0x7064,0x6EE6}, {0x7067,0x6EDF}, {0x70CF,0x4E4C}, 
+  {0x70F4,0x70C3}, {0x7121,0x65E0}, {0x7149,0x70BC}, {0x7152,0x709C}, {0x7162,0x8315}, 
+  {0x7165,0x7115}, {0x7169,0x70E6}, {0x716C,0x7080}, {0x7192,0x8367}, {0x7197,0x709D}, 
+  {0x71B1,0x70ED}, {0x71BE,0x70BD}, {0x71C1,0x70E8}, {0x71C8,0x706F}, {0x71D2,0x70E7}, 
+  {0x71D9,0x70EB}, {0x71DC,0x7116}, {0x71DF,0x8425}, {0x71E6,0x707F}, {0x71ED,0x70DB}, 
+  {0x71F4,0x70E9}, {0x71FC,0x70EC}, {0x71FE,0x7118}, {0x720D,0x70C1}, {0x7210,0x7089}, 
+  {0x721B,0x70C2}, {0x722D,0x4E89}, {0x7232,0x4E3A}, {0x723A,0x7237}, {0x723E,0x5C14}, 
+  {0x7258,0x724D}, {0x727D,0x7275}, {0x7296,0x8366}, {0x72A2,0x728A}, {0x72A7,0x727A}, 
+  {0x72C0,0x72B6}, {0x72F9,0x72ED}, {0x72FD,0x72C8}, {0x7319,0x72F0}, {0x7336,0x72B9}, 
+  {0x733B,0x72F2}, {0x7341,0x72B8}, {0x7344,0x72F1}, {0x7345,0x72EE}, {0x7368,0x72EC}, 
+  {0x736A,0x72EF}, {0x736B,0x7303}, {0x7370,0x72DE}, {0x7372,0x83B7}, {0x7375,0x730E}, 
+  {0x7377,0x72B7}, {0x7378,0x517D}, {0x737A,0x736D}, {0x737B,0x732E}, {0x737C,0x7315}, 
+  {0x7380,0x7321}, {0x73FE,0x73B0}, {0x743F,0x73F2}, {0x744B,0x73AE}, {0x7463,0x7410}, 
+  {0x7469,0x83B9}, {0x746A,0x739B}, {0x7489,0x740F}, {0x74A3,0x7391}, {0x74A6,0x7477}, 
+  {0x74B0,0x73AF}, {0x74BD,0x73BA}, {0x74CA,0x743C}, {0x74CF,0x73D1}, {0x74D4,0x748E}, 
+  {0x74DA,0x74D2}, {0x750C,0x74EF}, {0x7523,0x4EA7}, {0x755D,0x4EA9}, {0x7562,0x6BD5}, 
+  {0x756B,0x753B}, {0x7576,0x5F53}, {0x7587,0x7574}, {0x758A,0x53E0}, {0x75D9,0x75C9}, 
+  {0x7602,0x75D6}, {0x760B,0x75AF}, {0x760D,0x75A1}, {0x7613,0x75EA}, {0x761E,0x7617}, 
+  {0x7621,0x75AE}, {0x7627,0x759F}, {0x763B,0x7618}, {0x7642,0x7597}, {0x7646,0x75E8}, 
+  {0x7647,0x75EB}, {0x7649,0x7605}, {0x7658,0x75A0}, {0x765F,0x762A}, {0x7662,0x75D2}, 
+  {0x7664,0x7596}, {0x7665,0x75C7}, {0x7667,0x75AC}, {0x7669,0x765E}, {0x766C,0x7663}, 
+  {0x766D,0x763F}, {0x766E,0x763E}, {0x7670,0x75C8}, {0x7671,0x762B}, {0x7672,0x766B}, 
+  {0x767C,0x53D1}, {0x769A,0x7691}, {0x76B8,0x76B2}, {0x76BA,0x76B1}, {0x76DE,0x76CF}, 
+  {0x76E1,0x5C3D}, {0x76E3,0x76D1}, {0x76E4,0x76D8}, {0x76E7,0x5362}, {0x775C,0x7741}, 
+  {0x775E,0x7750}, {0x7798,0x770D}, {0x779E,0x7792}, {0x77BC,0x7751}, {0x77DA,0x77A9}, 
+  {0x77EF,0x77EB}, {0x7864,0x7856}, {0x7868,0x7817}, {0x786F,0x781A}, {0x78A9,0x7855}, 
+  {0x78AD,0x7800}, {0x78B8,0x781C}, {0x78BA,0x786E}, {0x78BC,0x7801}, {0x78DA,0x7816}, 
+  {0x78E3,0x789C}, {0x78E7,0x789B}, {0x78EF,0x77F6}, {0x78FD,0x7857}, {0x790E,0x7840}, 
+  {0x7919,0x788D}, {0x7926,0x77FF}, {0x792A,0x783A}, {0x792B,0x783E}, {0x792C,0x77FE}, 
+  {0x7931,0x783B}, {0x798D,0x7978}, {0x798E,0x796F}, {0x79AA,0x7985}, {0x79AE,0x793C}, 
+  {0x79B0,0x7962}, {0x79B1,0x7977}, {0x7A2E,0x79CD}, {0x7A31,0x79F0}, {0x7A4C,0x7A23}, 
+  {0x7A4D,0x79EF}, {0x7A4E,0x9896}, {0x7A61,0x7A51}, {0x7A62,0x79FD}, {0x7A69,0x7A33}, 
+  {0x7AA9,0x7A9D}, {0x7AAA,0x6D3C}, {0x7AAE,0x7A77}, {0x7AB6,0x7AAD}, {0x7ABA,0x7AA5}, 
+  {0x7AC4,0x7A9C}, {0x7AC5,0x7A8D}, {0x7AC7,0x7AA6}, {0x7AC8,0x7076}, {0x7ACA,0x7A83}, 
+  {0x7AEA,0x7AD6}, {0x7AF6,0x7ADE}, {0x7B46,0x7B14}, {0x7B67,0x7B15}, {0x7B8B,0x7B3A}, 
+  {0x7B8F,0x7B5D}, {0x7BC0,0x8282}, {0x7BC4,0x8303}, {0x7BC9,0x7B51}, {0x7BCB,0x7BA7}, 
+  {0x7BE4,0x7B03}, {0x7BE9,0x7B5B}, {0x7BF3,0x7B5A}, {0x7C00,0x7BA6}, {0x7C0D,0x7BD3}, 
+  {0x7C1E,0x7BAA}, {0x7C21,0x7B80}, {0x7C23,0x7BD1}, {0x7C2B,0x7BAB}, {0x7C3D,0x7B7E}, 
+  {0x7C3E,0x5E18}, {0x7C43,0x7BEE}, {0x7C4C,0x7B79}, {0x7C5C,0x7BA8}, {0x7C5F,0x7C41}, 
+  {0x7C60,0x7B3C}, {0x7C69,0x7B3E}, {0x7C6A,0x7C16}, {0x7C6C,0x7BF1}, {0x7C6E,0x7BA9}, 
+  {0x7CDD,0x7CC1}, {0x7CDE,0x7CAA}, {0x7CE7,0x7CAE}, {0x7CF2,0x7C9D}, {0x7CF4,0x7C74}, 
+  {0x7CF6,0x7C9C}, {0x7CF9,0x7E9F}, {0x7CFE,0x7EA0}, {0x7D00,0x7EAA}, {0x7D02,0x7EA3}, 
+  {0x7D04,0x7EA6}, {0x7D05,0x7EA2}, {0x7D06,0x7EA1}, {0x7D07,0x7EA5}, {0x7D08,0x7EA8}, 
+  {0x7D09,0x7EAB}, {0x7D0B,0x7EB9}, {0x7D0D,0x7EB3}, {0x7D10,0x7EBD}, {0x7D13,0x7EBE}, 
+  {0x7D14,0x7EAF}, {0x7D15,0x7EB0}, {0x7D17,0x7EB1}, {0x7D19,0x7EB8}, {0x7D1A,0x7EA7}, 
+  {0x7D1B,0x7EB7}, {0x7D1C,0x7EAD}, {0x7D21,0x7EBA}, {0x7D30,0x7EC6}, {0x7D31,0x7EC2}, 
+  {0x7D32,0x7EC1}, {0x7D33,0x7EC5}, {0x7D39,0x7ECD}, {0x7D3A,0x7EC0}, {0x7D3C,0x7ECB}, 
+  {0x7D3F,0x7ED0}, {0x7D40,0x7ECC}, {0x7D42,0x7EC8}, {0x7D44,0x7EC4}, {0x7D46,0x7ECA}, 
+  {0x7D4E,0x7ED7}, {0x7D50,0x7ED3}, {0x7D5D,0x7ED4}, {0x7D5E,0x7EDE}, {0x7D61,0x7EDC}, 
+  {0x7D62,0x7EDA}, {0x7D66,0x7ED9}, {0x7D68,0x7ED2}, {0x7D71,0x7EDF}, {0x7D72,0x4E1D}, 
+  {0x7D73,0x7EDB}, {0x7D76,0x7EDD}, {0x7D79,0x7EE2}, {0x7D81,0x7ED1}, {0x7D83,0x7EE1}, 
+  {0x7D86,0x7EE0}, {0x7D88,0x7EE8}, {0x7D89,0x7EE3}, {0x7D8F,0x7EE5}, {0x7D93,0x7ECF}, 
+  {0x7D9C,0x7EFC}, {0x7D9E,0x7F0D}, {0x7DA2,0x7EF8}, {0x7DA3,0x7EFB}, {0x7DAB,0x7EBF}, 
+  {0x7DAC,0x7EF6}, {0x7DAD,0x7EF4}, {0x7DB0,0x7EFE}, {0x7DB1,0x7EB2}, {0x7DB2,0x7F51}, 
+  {0x7DB3,0x7EF7}, {0x7DB4,0x7F00}, {0x7DB8,0x7EB6}, {0x7DB9,0x7EFA}, {0x7DBA,0x7EEE}, 
+  {0x7DBB,0x7EFD}, {0x7DBD,0x7EF0}, {0x7DBE,0x7EEB}, {0x7DBF,0x7EF5}, {0x7DC4,0x7EF2}, 
+  {0x7DC7,0x7F01}, {0x7DCA,0x7D27}, {0x7DCB,0x7EEF}, {0x7DD1,0x7EFF}, {0x7DD2,0x7EEA}, 
+  {0x7DD4,0x7EF1}, {0x7DD7,0x7F03}, {0x7DD8,0x7F04}, {0x7DD9,0x7F02}, {0x7DDD,0x7F09}, 
+  {0x7DDE,0x7F0E}, {0x7DE0,0x7F14}, {0x7DE1,0x7F17}, {0x7DE3,0x7F18}, {0x7DE6,0x7F0C}, 
+  {0x7DE8,0x7F16}, {0x7DE9,0x7F13}, {0x7DEC,0x7F05}, {0x7DEF,0x7EAC}, {0x7DF1,0x7F11}, 
+  {0x7DF2,0x7F08}, {0x7DF4,0x7EC3}, {0x7DF6,0x7F0F}, {0x7DF9,0x7F07}, {0x7E08,0x8426}, 
+  {0x7E09,0x7F19}, {0x7E0A,0x7F22}, {0x7E0B,0x7F12}, {0x7E10,0x7EC9}, {0x7E11,0x7F23}, 
+  {0x7E1B,0x7F1A}, {0x7E1D,0x7F1C}, {0x7E1E,0x7F1F}, {0x7E1F,0x7F1B}, {0x7E23,0x53BF}, 
+  {0x7E27,0x7EE6}, {0x7E2B,0x7F1D}, {0x7E2D,0x7F21}, {0x7E2E,0x7F29}, {0x7E31,0x7EB5}, 
+  {0x7E32,0x7F27}, {0x7E35,0x7F26}, {0x7E36,0x7D77}, {0x7E37,0x7F15}, {0x7E39,0x7F25}, 
+  {0x7E3D,0x603B}, {0x7E3E,0x7EE9}, {0x7E45,0x7F2B}, {0x7E46,0x7F2A}, {0x7E52,0x7F2F}, 
+  {0x7E54,0x7EC7}, {0x7E55,0x7F2E}, {0x7E5A,0x7F2D}, {0x7E5E,0x7ED5}, {0x7E62,0x7F0B}, 
+  {0x7E69,0x7EF3}, {0x7E6A,0x7ED8}, {0x7E6D,0x8327}, {0x7E6E,0x7F30}, {0x7E6F,0x7F33}, 
+  {0x7E70,0x7F32}, {0x7E73,0x7F34}, {0x7E79,0x7ECE}, {0x7E7C,0x7EE7}, {0x7E7D,0x7F24}, 
+  {0x7E7E,0x7F31}, {0x7E88,0x7F2C}, {0x7E8A,0x7EA9}, {0x7E8C,0x7EED}, {0x7E8F,0x7F20}, 
+  {0x7E93,0x7F28}, {0x7E96,0x7EA4}, {0x7E98,0x7F35}, {0x7E9C,0x7F06}, {0x7F4C,0x7F42}, 
+  {0x7F70,0x7F5A}, {0x7F77,0x7F62}, {0x7F85,0x7F57}, {0x7F86,0x7F74}, {0x7F88,0x7F81}, 
+  {0x7FA5,0x7F9F}, {0x7FA9,0x4E49}, {0x7FD2,0x4E60}, {0x7FF9,0x7FD8}, {0x802C,0x8027}, 
+  {0x802E,0x8022}, {0x8056,0x5723}, {0x805E,0x95FB}, {0x806F,0x8054}, {0x8070,0x806A}, 
+  {0x8072,0x58F0}, {0x8073,0x8038}, {0x8075,0x8069}, {0x8076,0x8042}, {0x8077,0x804C}, 
+  {0x8079,0x804D}, {0x807D,0x542C}, {0x807E,0x804B}, {0x8085,0x8083}, {0x8105,0x80C1}, 
+  {0x811B,0x80EB}, {0x8139,0x80C0}, {0x814E,0x80BE}, {0x8156,0x80E8}, {0x8161,0x8136}, 
+  {0x8166,0x8111}, {0x816B,0x80BF}, {0x8178,0x80A0}, {0x819A,0x80A4}, {0x81A0,0x80F6}, 
+  {0x81A9,0x817B}, {0x81BD,0x80C6}, {0x81BE,0x810D}, {0x81BF,0x8113}, {0x81C9,0x8138}, 
+  {0x81CD,0x8110}, {0x81CF,0x8191}, {0x81D8,0x814A}, {0x81DA,0x80EA}, {0x81DF,0x810F}, 
+  {0x81E0,0x8114}, {0x81E8,0x4E34}, {0x81FA,0x53F0}, {0x8207,0x4E0E}, {0x8208,0x5174}, 
+  {0x8209,0x4E3E}, {0x820A,0x65E7}, {0x8259,0x8231}, {0x8264,0x8223}, {0x8266,0x8230}, 
+  {0x826B,0x823B}, {0x8271,0x8270}, {0x8277,0x8273}, {0x82BB,0x520D}, {0x82E7,0x82CE}, 
+  {0x838A,0x5E84}, {0x8396,0x830E}, {0x83A2,0x835A}, {0x83A7,0x82CB}, {0x83EF,0x534E}, 
+  {0x8407,0x82CC}, {0x840A,0x83B1}, {0x842C,0x4E07}, {0x8435,0x83B4}, {0x8449,0x53F6}, 
+  {0x8452,0x836D}, {0x8464,0x836E}, {0x8466,0x82C7}, {0x8477,0x8364}, {0x8493,0x83BC}, 
+  {0x8494,0x83B3}, {0x84BC,0x82CD}, {0x84C0,0x836A}, {0x84CB,0x76D6}, {0x84EE,0x83B2}, 
+  {0x84EF,0x82C1}, {0x84FD,0x835C}, {0x851E,0x848C}, {0x8523,0x848B}, {0x8526,0x8311}, 
+  {0x852D,0x836B}, {0x8541,0x8368}, {0x8546,0x8487}, {0x854E,0x835E}, {0x8552,0x836C}, 
+  {0x8553,0x82B8}, {0x8555,0x83B8}, {0x8558,0x835B}, {0x8562,0x8489}, {0x8569,0x8361}, 
+  {0x856A,0x829C}, {0x856D,0x8427}, {0x8577,0x84E3}, {0x8588,0x835F}, {0x858A,0x84DF}, 
+  {0x858C,0x8297}, {0x8594,0x8537}, {0x859F,0x83B6}, {0x85A6,0x8350}, {0x85A9,0x8428}, 
+  {0x85BA,0x8360}, {0x85CD,0x84DD}, {0x85CE,0x8369}, {0x85DD,0x827A}, {0x85E5,0x836F}, 
+  {0x85EA,0x85AE}, {0x85F4,0x8574}, {0x85F6,0x82C8}, {0x85F9,0x853C}, {0x85FA,0x853A}, 
+  {0x8604,0x8572}, {0x8606,0x82A6}, {0x8607,0x82CF}, {0x860B,0x82F9}, {0x861A,0x85D3}, 
+  {0x861E,0x8539}, {0x8622,0x830F}, {0x862D,0x5170}, {0x863A,0x84E0}, {0x863F,0x841D}, 
+  {0x8655,0x5904}, {0x865C,0x864F}, {0x865F,0x53F7}, {0x8667,0x4E8F}, {0x86FA,0x86F1}, 
+  {0x8706,0x86AC}, {0x8755,0x8680}, {0x8766,0x867E}, {0x8778,0x8717}, {0x8784,0x86F3}, 
+  {0x879E,0x8682}, {0x87A2,0x8424}, {0x87BB,0x877C}, {0x87C4,0x86F0}, {0x87C8,0x8748}, 
+  {0x87CE,0x87A8}, {0x87E3,0x866E}, {0x87EC,0x8749}, {0x87EF,0x86F2}, {0x87F2,0x866B}, 
+  {0x87F6,0x86CF}, {0x87FB,0x8681}, {0x8805,0x8747}, {0x8806,0x867F}, {0x8810,0x86F4}, 
+  {0x8811,0x877E}, {0x881F,0x8721}, {0x8823,0x86CE}, {0x8831,0x86CA}, {0x8836,0x8695}, 
+  {0x883B,0x86EE}, {0x8846,0x4F17}, {0x8853,0x672F}, {0x885B,0x536B}, {0x885D,0x51B2}, 
+  {0x8879,0x53EA}, {0x88CA,0x8885}, {0x88CF,0x91CC}, {0x88DC,0x8865}, {0x88DD,0x88C5}, 
+  {0x8932,0x88E4}, {0x8933,0x88E2}, {0x8938,0x891B}, {0x893B,0x4EB5}, {0x8947,0x88E5}, 
+  {0x8956,0x8884}, {0x895D,0x88E3}, {0x8960,0x88C6}, {0x8964,0x8934}, {0x896A,0x889C}, 
+  {0x896F,0x886C}, {0x8972,0x88AD}, {0x898B,0x89C1}, {0x898F,0x89C4}, {0x8993,0x89C5}, 
+  {0x8996,0x89C6}, {0x8998,0x89C7}, {0x89A1,0x89CB}, {0x89A6,0x89CE}, {0x89AA,0x4EB2}, 
+  {0x89AC,0x89CA}, {0x89AF,0x89CF}, {0x89B2,0x89D0}, {0x89B7,0x89D1}, {0x89BA,0x89C9}, 
+  {0x89BD,0x89C8}, {0x89BF,0x89CC}, {0x89C0,0x89C2}, {0x89F4,0x89DE}, {0x89F6,0x89EF}, 
+  {0x89F8,0x89E6}, {0x8A01,0x8BA0}, {0x8A02,0x8BA2}, {0x8A03,0x8BA3}, {0x8A08,0x8BA1}, 
+  {0x8A0A,0x8BAF}, {0x8A0C,0x8BA7}, {0x8A0E,0x8BA8}, {0x8A10,0x8BA6}, {0x8A13,0x8BAD}, 
+  {0x8A15,0x8BAA}, {0x8A16,0x8BAB}, {0x8A18,0x8BB0}, {0x8A1B,0x8BB9}, {0x8A1D,0x8BB6}, 
+  {0x8A1F,0x8BBC}, {0x8A23,0x8BC0}, {0x8A25,0x8BB7}, {0x8A2A,0x8BBF}, {0x8A2D,0x8BBE}, 
+  {0x8A31,0x8BB8}, {0x8A34,0x8BC9}, {0x8A36,0x8BC3}, {0x8A3A,0x8BCA}, {0x8A41,0x8BC2}, 
+  {0x8A46,0x8BCB}, {0x8A4E,0x8BB5}, {0x8A50,0x8BC8}, {0x8A52,0x8BD2}, {0x8A54,0x8BCF}, 
+  {0x8A55,0x8BC4}, {0x8A58,0x8BCE}, {0x8A5B,0x8BC5}, {0x8A5E,0x8BCD}, {0x8A61,0x8BE9}, 
+  {0x8A62,0x8BE2}, {0x8A63,0x8BE3}, {0x8A66,0x8BD5}, {0x8A69,0x8BD7}, {0x8A6B,0x8BE7}, 
+  {0x8A6C,0x8BDF}, {0x8A6D,0x8BE1}, {0x8A6E,0x8BE0}, {0x8A70,0x8BD8}, {0x8A71,0x8BDD}, 
+  {0x8A72,0x8BE5}, {0x8A73,0x8BE6}, {0x8A75,0x8BDC}, {0x8A7C,0x8BD9}, {0x8A7F,0x8BD6}, 
+  {0x8A84,0x8BD4}, {0x8A85,0x8BDB}, {0x8A86,0x8BD3}, {0x8A87,0x5938}, {0x8A8D,0x8BA4}, 
+  {0x8A91,0x8BF3}, {0x8A92,0x8BF6}, {0x8A95,0x8BDE}, {0x8A98,0x8BF1}, {0x8A9A,0x8BEE}, 
+  {0x8A9E,0x8BED}, {0x8AA0,0x8BDA}, {0x8AA1,0x8BEB}, {0x8AA3,0x8BEC}, {0x8AA4,0x8BEF}, 
+  {0x8AA5,0x8BF0}, {0x8AA6,0x8BF5}, {0x8AA8,0x8BF2}, {0x8AAC,0x8BF4}, {0x8AB0,0x8C01}, 
+  {0x8AB2,0x8BFE}, {0x8AB6,0x8C07}, {0x8AB9,0x8BFD}, {0x8ABC,0x8C0A}, {0x8ABF,0x8C03}, 
+  {0x8AC2,0x8C04}, {0x8AC4,0x8C06}, {0x8AC7,0x8C08}, {0x8AC9,0x8BFF}, {0x8ACB,0x8BF7}, 
+  {0x8ACD,0x8BE4}, {0x8ACF,0x8BF9}, {0x8AD1,0x8BFC}, {0x8AD2,0x8C05}, {0x8AD6,0x8BBA}, 
+  {0x8AD7,0x8C02}, {0x8ADB,0x8C00}, {0x8ADC,0x8C0D}, {0x8ADE,0x8C1D}, {0x8AE2,0x8BE8}, 
+  {0x8AE4,0x8C14}, {0x8AE6,0x8C1B}, {0x8AE7,0x8C10}, {0x8AEB,0x8C0F}, {0x8AED,0x8C15}, 
+  {0x8AEE,0x8C18}, {0x8AF1,0x8BB3}, {0x8AF3,0x8C19}, {0x8AF6,0x8C0C}, {0x8AF7,0x8BBD}, 
+  {0x8AF8,0x8BF8}, {0x8AFA,0x8C1A}, {0x8AFC,0x8C16}, {0x8AFE,0x8BFA}, {0x8B00,0x8C0B}, 
+  {0x8B01,0x8C12}, {0x8B02,0x8C13}, {0x8B04,0x8A8A}, {0x8B05,0x8BCC}, {0x8B0A,0x8C0E}, 
+  {0x8B0E,0x8C1C}, {0x8B10,0x8C27}, {0x8B14,0x8C11}, {0x8B16,0x8C21}, {0x8B17,0x8C24}, 
+  {0x8B19,0x8C26}, {0x8B1A,0x8C25}, {0x8B1B,0x8BB2}, {0x8B1D,0x8C22}, {0x8B21,0x8C23}, 
+  {0x8B28,0x8C1F}, {0x8B2B,0x8C2A}, {0x8B2C,0x8C2C}, {0x8B2D,0x8C2B}, {0x8B33,0x8BB4}, 
+  {0x8B39,0x8C28}, {0x8B3E,0x8C29}, {0x8B49,0x8BC1}, {0x8B4E,0x8C32}, {0x8B4F,0x8BA5}, 
+  {0x8B56,0x8C2E}, {0x8B58,0x8BC6}, {0x8B59,0x8C2F}, {0x8B5A,0x8C2D}, {0x8B5C,0x8C31}, 
+  {0x8B6B,0x8C35}, {0x8B6F,0x8BD1}, {0x8B70,0x8BAE}, {0x8B74,0x8C34}, {0x8B77,0x62A4}, 
+  {0x8B7D,0x8A89}, {0x8B80,0x8BFB}, {0x8B8A,0x53D8}, {0x8B8E,0x96E0}, {0x8B92,0x8C17}, 
+  {0x8B93,0x8BA9}, {0x8B95,0x8C30}, {0x8B96,0x8C36}, {0x8B9C,0x8C20}, {0x8B9E,0x8C33}, 
+  {0x8C48,0x5C82}, {0x8C50,0x4E30}, {0x8C9D,0x8D1D}, {0x8C9E,0x8D1E}, {0x8CA0,0x8D1F}, 
+  {0x8CA1,0x8D22}, {0x8CA2,0x8D21}, {0x8CA7,0x8D2B}, {0x8CA8,0x8D27}, {0x8CA9,0x8D29}, 
+  {0x8CAA,0x8D2A}, {0x8CAB,0x8D2F}, {0x8CAC,0x8D23}, {0x8CAF,0x8D2E}, {0x8CB0,0x8D33}, 
+  {0x8CB2,0x8D40}, {0x8CB3,0x8D30}, {0x8CB4,0x8D35}, {0x8CB6,0x8D2C}, {0x8CB7,0x4E70}, 
+  {0x8CB8,0x8D37}, {0x8CBA,0x8D36}, {0x8CBB,0x8D39}, {0x8CBC,0x8D34}, {0x8CBD,0x8D3B}, 
+  {0x8CBF,0x8D38}, {0x8CC0,0x8D3A}, {0x8CC1,0x8D32}, {0x8CC2,0x8D42}, {0x8CC3,0x8D41}, 
+  {0x8CC4,0x8D3F}, {0x8CC5,0x8D45}, {0x8CC7,0x8D44}, {0x8CC8,0x8D3E}, {0x8CCA,0x8D3C}, 
+  {0x8CD1,0x8D48}, {0x8CD2,0x8D4A}, {0x8CD3,0x5BBE}, {0x8CD5,0x8D47}, {0x8CDA,0x8D49}, 
+  {0x8CDC,0x8D50}, {0x8CDE,0x8D4F}, {0x8CE0,0x8D54}, {0x8CE1,0x8D53}, {0x8CE2,0x8D24}, 
+  {0x8CE3,0x5356}, {0x8CE4,0x8D31}, {0x8CE6,0x8D4B}, {0x8CE7,0x8D55}, {0x8CEA,0x8D28}, 
+  {0x8CEB,0x8D4D}, {0x8CEC,0x8D26}, {0x8CED,0x8D4C}, {0x8CF4,0x8D56}, {0x8CFA,0x8D5A}, 
+  {0x8CFB,0x8D59}, {0x8CFC,0x8D2D}, {0x8CFD,0x8D5B}, {0x8CFE,0x8D5C}, {0x8D04,0x8D3D}, 
+  {0x8D05,0x8D58}, {0x8D08,0x8D60}, {0x8D0A,0x8D5E}, {0x8D0B,0x8D5D}, {0x8D0D,0x8D61}, 
+  {0x8D0F,0x8D62}, {0x8D10,0x8D46}, {0x8D16,0x8D4E}, {0x8D1B,0x8D63}, {0x8D1C,0x8D43}, 
+  {0x8D95,0x8D76}, {0x8D99,0x8D75}, {0x8DA8,0x8D8B}, {0x8DB2,0x8DB1}, {0x8E10,0x8DF5}, 
+  {0x8E34,0x8E0A}, {0x8E4C,0x8DC4}, {0x8E55,0x8DF8}, {0x8E63,0x8E52}, {0x8E7A,0x8DF7}, 
+  {0x8E89,0x8DB8}, {0x8E8A,0x8E0C}, {0x8E8B,0x8DFB}, {0x8E8D,0x8DC3}, {0x8E91,0x8E2F}, 
+  {0x8E92,0x8DDE}, {0x8E93,0x8E2C}, {0x8E9A,0x8DF9}, {0x8EA1,0x8E51}, {0x8EA5,0x8E7F}, 
+  {0x8EA6,0x8E9C}, {0x8EAA,0x8E8F}, {0x8EC0,0x8EAF}, {0x8ECA,0x8F66}, {0x8ECB,0x8F67}, 
+  {0x8ECC,0x8F68}, {0x8ECD,0x519B}, {0x8ED2,0x8F69}, {0x8ED4,0x8F6B}, {0x8EDB,0x8F6D}, 
+  {0x8EDF,0x8F6F}, {0x8EE4,0x8F77}, {0x8EEB,0x8F78}, {0x8EF2,0x8F71}, {0x8EF8,0x8F74}, 
+  {0x8EF9,0x8F75}, {0x8EFA,0x8F7A}, {0x8EFB,0x8F72}, {0x8EFC,0x8F76}, {0x8EFE,0x8F7C}, 
+  {0x8F03,0x8F83}, {0x8F05,0x8F82}, {0x8F07,0x8F81}, {0x8F09,0x8F7D}, {0x8F0A,0x8F7E}, 
+  {0x8F12,0x8F84}, {0x8F14,0x8F85}, {0x8F15,0x8F7B}, {0x8F1B,0x8F86}, {0x8F1C,0x8F8E}, 
+  {0x8F1D,0x8F89}, {0x8F1E,0x8F8B}, {0x8F1F,0x8F8D}, {0x8F25,0x8F8A}, {0x8F26,0x8F87}, 
+  {0x8F29,0x8F88}, {0x8F2A,0x8F6E}, {0x8F2F,0x8F91}, {0x8F33,0x8F8F}, {0x8F38,0x8F93}, 
+  {0x8F3B,0x8F90}, {0x8F3E,0x8F97}, {0x8F3F,0x8206}, {0x8F42,0x6BC2}, {0x8F44,0x8F96}, 
+  {0x8F45,0x8F95}, {0x8F46,0x8F98}, {0x8F49,0x8F6C}, {0x8F4D,0x8F99}, {0x8F4E,0x8F7F}, 
+  {0x8F54,0x8F9A}, {0x8F5F,0x8F70}, {0x8F61,0x8F94}, {0x8F62,0x8F79}, {0x8F64,0x8F73}, 
+  {0x8FA6,0x529E}, {0x8FAD,0x8F9E}, {0x8FAE,0x8FAB}, {0x8FAF,0x8FA9}, {0x8FB2,0x519C}, 
+  {0x9015,0x8FF3}, {0x9019,0x8FD9}, {0x9023,0x8FDE}, {0x9032,0x8FDB}, {0x904B,0x8FD0}, 
+  {0x904E,0x8FC7}, {0x9054,0x8FBE}, {0x9055,0x8FDD}, {0x905C,0x900A}, {0x905E,0x9012}, 
+  {0x9060,0x8FDC}, {0x9069,0x9002}, {0x9072,0x8FDF}, {0x9077,0x8FC1}, {0x9078,0x9009}, 
+  {0x907A,0x9057}, {0x907C,0x8FBD}, {0x9081,0x8FC8}, {0x9084,0x8FD8}, {0x9087,0x8FE9}, 
+  {0x908A,0x8FB9}, {0x908F,0x903B}, {0x9090,0x9026}, {0x90DF,0x90CF}, {0x90F5,0x90AE}, 
+  {0x9106,0x90D3}, {0x9109,0x4E61}, {0x9112,0x90B9}, {0x9114,0x90AC}, {0x9116,0x90E7}, 
+  {0x9127,0x9093}, {0x912D,0x90D1}, {0x9130,0x90BB}, {0x9132,0x90F8}, {0x9134,0x90BA}, 
+  {0x9136,0x90D0}, {0x913A,0x909D}, {0x9148,0x90E6}, {0x9196,0x915D}, {0x919C,0x4E11}, 
+  {0x91AB,0x533B}, {0x91AC,0x9171}, {0x91C0,0x917F}, {0x91C1,0x8845}, {0x91C3,0x917E}, 
+  {0x91C5,0x917D}, {0x91CB,0x91CA}, {0x91D2,0x9485}, {0x91D3,0x9486}, {0x91D4,0x9487}, 
+  {0x91D5,0x948C}, {0x91D7,0x948A}, {0x91D8,0x9489}, {0x91D9,0x948B}, {0x91DD,0x9488}, 
+  {0x91E3,0x9493}, {0x91E4,0x9490}, {0x91E7,0x948F}, {0x91E9,0x9492}, {0x91F5,0x9497}, 
+  {0x91F7,0x948D}, {0x91F9,0x9495}, {0x91FA,0x948E}, {0x9200,0x94AF}, {0x9201,0x94AB}, 
+  {0x9204,0x94AD}, {0x9208,0x949A}, {0x9209,0x94A0}, {0x920D,0x949D}, {0x920E,0x94A9}, 
+  {0x9210,0x94A4}, {0x9211,0x94A3}, {0x9214,0x949E}, {0x9215,0x94AE}, {0x921E,0x94A7}, 
+  {0x9223,0x9499}, {0x9225,0x94AC}, {0x9226,0x949B}, {0x9227,0x94AA}, {0x922E,0x94CC}, 
+  {0x9230,0x94C8}, {0x9233,0x94B6}, {0x9234,0x94C3}, {0x9237,0x94B4}, {0x9238,0x94B9}, 
+  {0x9239,0x94CD}, {0x923A,0x94B0}, {0x923D,0x94B8}, {0x923E,0x94C0}, {0x923F,0x94BF}, 
+  {0x9240,0x94BE}, {0x9245,0x949C}, {0x9248,0x94CA}, {0x9249,0x94C9}, {0x924D,0x94CB}, 
+  {0x9251,0x94C2}, {0x9255,0x94B7}, {0x9257,0x94B3}, {0x925A,0x94C6}, {0x925B,0x94C5}, 
+  {0x925E,0x94BA}, {0x9262,0x94B5}, {0x9266,0x94B2}, {0x926C,0x94BC}, {0x926D,0x94BD}, 
+  {0x9278,0x94F0}, {0x927A,0x94D2}, {0x927B,0x94EC}, {0x927F,0x94EA}, {0x9280,0x94F6}, 
+  {0x9283,0x94F3}, {0x9285,0x94DC}, {0x9291,0x94E3}, {0x9293,0x94E8}, {0x9296,0x94E2}, 
+  {0x9298,0x94ED}, {0x929A,0x94EB}, {0x929C,0x8854}, {0x92A0,0x94D1}, {0x92A3,0x94F7}, 
+  {0x92A5,0x94F1}, {0x92A6,0x94DF}, {0x92A8,0x94F5}, {0x92A9,0x94E5}, {0x92AA,0x94D5}, 
+  {0x92AB,0x94EF}, {0x92AC,0x94D0}, {0x92B1,0x94DE}, {0x92B7,0x9500}, {0x92B9,0x9508}, 
+  {0x92BB,0x9511}, {0x92BC,0x9509}, {0x92C1,0x94DD}, {0x92C3,0x9512}, {0x92C5,0x950C}, 
+  {0x92C7,0x94A1}, {0x92CC,0x94E4}, {0x92CF,0x94D7}, {0x92D2,0x950B}, {0x92DD,0x950A}, 
+  {0x92DF,0x9513}, {0x92E3,0x94D8}, {0x92E4,0x9504}, {0x92E5,0x9503}, {0x92E6,0x9514}, 
+  {0x92E8,0x9507}, {0x92EA,0x94FA}, {0x92ED,0x9510}, {0x92EE,0x94D6}, {0x92EF,0x9506}, 
+  {0x92F0,0x9502}, {0x92F1,0x94FD}, {0x92F6,0x950D}, {0x92F8,0x952F}, {0x92FC,0x94A2}, 
+  {0x9301,0x951E}, {0x9306,0x9516}, {0x9307,0x952B}, {0x9308,0x9529}, {0x9310,0x9525}, 
+  {0x9312,0x9515}, {0x9315,0x951F}, {0x9318,0x9524}, {0x9319,0x9531}, {0x931A,0x94EE}, 
+  {0x931B,0x951B}, {0x931F,0x952C}, {0x9320,0x952D}, {0x9322,0x94B1}, {0x9326,0x9526}, 
+  {0x9328,0x951A}, {0x932B,0x9521}, {0x932E,0x9522}, {0x932F,0x9519}, {0x9332,0x5F55}, 
+  {0x9333,0x9530}, {0x9338,0x94FC}, {0x9340,0x951D}, {0x9341,0x9528}, {0x9343,0x952A}, 
+  {0x9346,0x9494}, {0x9347,0x9534}, {0x934B,0x9505}, {0x934D,0x9540}, {0x9354,0x9537}, 
+  {0x9358,0x94E1}, {0x935B,0x953B}, {0x9364,0x9538}, {0x9365,0x9532}, {0x9369,0x9518}, 
+  {0x936C,0x9539}, {0x9370,0x953E}, {0x9375,0x952E}, {0x9376,0x9536}, {0x937A,0x9517}, 
+  {0x937E,0x953A}, {0x9382,0x9541}, {0x9384,0x953F}, {0x9387,0x9545}, {0x938A,0x9551}, 
+  {0x9396,0x9501}, {0x9398,0x9549}, {0x93A2,0x94A8}, {0x93A3,0x84E5}, {0x93A6,0x954F}, 
+  {0x93A7,0x94E0}, {0x93A9,0x94E9}, {0x93AA,0x953C}, {0x93AC,0x9550}, {0x93AE,0x9547}, 
+  {0x93B0,0x9552}, {0x93B3,0x954D}, {0x93B5,0x9553}, {0x93B8,0x954C}, {0x93BF,0x954E}, 
+  {0x93C3,0x955E}, {0x93C7,0x955F}, {0x93C8,0x94FE}, {0x93CC,0x9546}, {0x93CD,0x9559}, 
+  {0x93D1,0x955D}, {0x93D7,0x94FF}, {0x93D8,0x9535}, {0x93DC,0x9557}, {0x93DD,0x9558}, 
+  {0x93DE,0x955B}, {0x93DF,0x94F2}, {0x93E1,0x955C}, {0x93E2,0x9556}, {0x93E4,0x9542}, 
+  {0x93E8,0x933E}, {0x93F5,0x94E7}, {0x93F7,0x9564}, {0x93F9,0x956A}, {0x9403,0x94D9}, 
+  {0x940B,0x94F4}, {0x9410,0x9563}, {0x9412,0x94F9}, {0x9413,0x9566}, {0x9414,0x9561}, 
+  {0x9418,0x949F}, {0x9419,0x956B}, {0x941D,0x9562}, {0x9420,0x9568}, {0x9426,0x950E}, 
+  {0x9427,0x950F}, {0x9428,0x9544}, {0x942E,0x9570}, {0x9432,0x956F}, {0x9433,0x956D}, 
+  {0x9435,0x94C1}, {0x9438,0x94CE}, {0x943A,0x94DB}, {0x943F,0x9571}, {0x9444,0x94F8}, 
+  {0x944A,0x956C}, {0x944C,0x9554}, {0x9452,0x9274}, {0x9454,0x9572}, {0x9460,0x94C4}, 
+  {0x9463,0x9573}, {0x9465,0x9565}, {0x946D,0x9567}, {0x9470,0x94A5}, {0x9472,0x9576}, 
+  {0x9477,0x954A}, {0x9479,0x9569}, {0x947C,0x9523}, {0x947D,0x94BB}, {0x947E,0x92AE}, 
+  {0x947F,0x51FF}, {0x9577,0x957F}, {0x9580,0x95E8}, {0x9582,0x95E9}, {0x9583,0x95EA}, 
+  {0x9586,0x95EB}, {0x9589,0x95ED}, {0x958B,0x5F00}, {0x958C,0x95F6}, {0x958E,0x95F3}, 
+  {0x958F,0x95F0}, {0x9591,0x95F2}, {0x9593,0x95F4}, {0x9594,0x95F5}, {0x9598,0x95F8}, 
+  {0x95A1,0x9602}, {0x95A3,0x9601}, {0x95A5,0x9600}, {0x95A8,0x95FA}, {0x95A9,0x95FD}, 
+  {0x95AB,0x9603}, {0x95AC,0x9606}, {0x95AD,0x95FE}, {0x95B2,0x9605}, {0x95B6,0x960A}, 
+  {0x95B9,0x9609}, {0x95BB,0x960E}, {0x95BC,0x960F}, {0x95BD,0x960D}, {0x95BE,0x9608}, 
+  {0x95BF,0x960C}, {0x95C3,0x9612}, {0x95C8,0x95F1}, {0x95CA,0x9614}, {0x95CB,0x9615}, 
+  {0x95CC,0x9611}, {0x95D0,0x9617}, {0x95D4,0x9616}, {0x95D5,0x9619}, {0x95D6,0x95EF}, 
+  {0x95DC,0x5173}, {0x95DE,0x961A}, {0x95E1,0x9610}, {0x95E5,0x95FC}, {0x9658,0x9649}, 
+  {0x965D,0x9655}, {0x9663,0x9635}, {0x9670,0x9634}, {0x9673,0x9648}, {0x9678,0x9646}, 
+  {0x967D,0x9633}, {0x9689,0x9667}, {0x968A,0x961F}, {0x968E,0x9636}, {0x9695,0x9668}, 
+  {0x969B,0x9645}, {0x96A8,0x968F}, {0x96AA,0x9669}, {0x96B1,0x9690}, {0x96B4,0x9647}, 
+  {0x96B8,0x96B6}, {0x96D6,0x867D}, {0x96D9,0x53CC}, {0x96DB,0x96CF}, {0x96DC,0x6742}, 
+  {0x96E2,0x79BB}, {0x96E3,0x96BE}, {0x96F2,0x4E91}, {0x96FB,0x7535}, {0x9727,0x96FE}, 
+  {0x973D,0x9701}, {0x9742,0x96F3}, {0x9744,0x972D}, {0x9748,0x7075}, {0x975A,0x9753}, 
+  {0x975C,0x9759}, {0x9768,0x9765}, {0x978F,0x5DE9}, {0x97BD,0x9792}, {0x97C3,0x9791}, 
+  {0x97C9,0x97AF}, {0x97CB,0x97E6}, {0x97CC,0x97E7}, {0x97D3,0x97E9}, {0x97D9,0x97EA}, 
+  {0x97DC,0x97EC}, {0x97DE,0x97EB}, {0x97FF,0x54CD}, {0x9801,0x9875}, {0x9802,0x9876}, 
+  {0x9803,0x9877}, {0x9805,0x9879}, {0x9806,0x987A}, {0x9807,0x9878}, {0x9808,0x987B}, 
+  {0x980A,0x987C}, {0x980C,0x9882}, {0x980E,0x9880}, {0x980F,0x9883}, {0x9810,0x9884}, 
+  {0x9811,0x987D}, {0x9812,0x9881}, {0x9813,0x987F}, {0x9817,0x9887}, {0x9818,0x9886}, 
+  {0x981C,0x988C}, {0x9821,0x9889}, {0x9824,0x9890}, {0x9826,0x988F}, {0x982D,0x5934}, 
+  {0x9830,0x988A}, {0x9837,0x9894}, {0x9838,0x9888}, {0x983B,0x9891}, {0x983D,0x9893}, 
+  {0x9846,0x9897}, {0x984C,0x9898}, {0x984D,0x989D}, {0x984E,0x989A}, {0x9853,0x989B}, 
+  {0x9854,0x989C}, {0x9858,0x613F}, {0x9859,0x98A1}, {0x985B,0x98A0}, {0x985E,0x7C7B}, 
+  {0x9862,0x989F}, {0x9865,0x98A2}, {0x9867,0x987E}, {0x986B,0x98A4}, {0x986C,0x98A5}, 
+  {0x986F,0x663E}, {0x9870,0x98A6}, {0x9871,0x9885}, {0x9873,0x989E}, {0x9874,0x98A7}, 
+  {0x98A8,0x98CE}, {0x98AE,0x98D1}, {0x98AF,0x98D2}, {0x98B6,0x98D3}, {0x98BC,0x98D5}, 
+  {0x98C4,0x98D8}, {0x98C6,0x98D9}, {0x98C8,0x98DA}, {0x98DB,0x98DE}, {0x98E0,0x9963}, 
+  {0x98E9,0x9968}, {0x98EA,0x996A}, {0x98EB,0x996B}, {0x98ED,0x996C}, {0x98EF,0x996D}, 
+  {0x98F2,0x996E}, {0x98F4,0x9974}, {0x98FC,0x9972}, {0x98FD,0x9971}, {0x98FE,0x9970}, 
+  {0x9903,0x997A}, {0x9905,0x997C}, {0x9909,0x9977}, {0x990A,0x517B}, {0x990C,0x9975}, 
+  {0x9911,0x997D}, {0x9912,0x9981}, {0x9913,0x997F}, {0x9918,0x4F59}, {0x991B,0x9984}, 
+  {0x991E,0x996F}, {0x9921,0x9985}, {0x9928,0x9986}, {0x9933,0x9967}, {0x9937,0x9987}, 
+  {0x993C,0x9969}, {0x993E,0x998F}, {0x993F,0x998A}, {0x9943,0x998D}, {0x9945,0x9992}, 
+  {0x9948,0x9990}, {0x9949,0x9991}, {0x994A,0x9993}, {0x994B,0x9988}, {0x994C,0x9994}, 
+  {0x9951,0x9965}, {0x9952,0x9976}, {0x9957,0x98E8}, {0x995C,0x990D}, {0x995E,0x998B}, 
+  {0x9962,0x9995}, {0x99AC,0x9A6C}, {0x99AD,0x9A6D}, {0x99AE,0x51AF}, {0x99B1,0x9A6E}, 
+  {0x99B3,0x9A70}, {0x99B4,0x9A6F}, {0x99C1,0x9A73}, {0x99D0,0x9A7B}, {0x99D1,0x9A7D}, 
+  {0x99D2,0x9A79}, {0x99D4,0x9A75}, {0x99D5,0x9A7E}, {0x99D8,0x9A80}, {0x99D9,0x9A78}, 
+  {0x99DB,0x9A76}, {0x99DD,0x9A7C}, {0x99DF,0x9A77}, {0x99E1,0x9A82}, {0x99E2,0x9A88}, 
+  {0x99ED,0x9A87}, {0x99F1,0x9A86}, {0x99FF,0x9A8F}, {0x9A01,0x9A8B}, {0x9A05,0x9A93}, 
+  {0x9A0D,0x9A92}, {0x9A0E,0x9A91}, {0x9A0F,0x9A90}, {0x9A16,0x9A9B}, {0x9A19,0x9A97}, 
+  {0x9A2B,0x9A9E}, {0x9A2D,0x9A98}, {0x9A2E,0x9A9D}, {0x9A30,0x817E}, {0x9A36,0x9A7A}, 
+  {0x9A37,0x9A9A}, {0x9A38,0x9A9F}, {0x9A3E,0x9AA1}, {0x9A40,0x84E6}, {0x9A41,0x9A9C}, 
+  {0x9A42,0x9A96}, {0x9A43,0x9AA0}, {0x9A44,0x9AA2}, {0x9A45,0x9A71}, {0x9A4A,0x9A85}, 
+  {0x9A4D,0x9A81}, {0x9A4F,0x9AA3}, {0x9A55,0x9A84}, {0x9A57,0x9A8C}, {0x9A5A,0x60CA}, 
+  {0x9A5B,0x9A7F}, {0x9A5F,0x9AA4}, {0x9A62,0x9A74}, {0x9A64,0x9AA7}, {0x9A65,0x9AA5}, 
+  {0x9A6A,0x9A8A}, {0x9AAF,0x80AE}, {0x9ACF,0x9AC5}, {0x9AD4,0x4F53}, {0x9AD5,0x9ACC}, 
+  {0x9AD6,0x9ACB}, {0x9B06,0x677E}, {0x9B22,0x9B13}, {0x9B25,0x6597}, {0x9B27,0x95F9}, 
+  {0x9B29,0x960B}, {0x9B2E,0x9604}, {0x9B31,0x90C1}, {0x9B4E,0x9B49}, {0x9B58,0x9B47}, 
+  {0x9B5A,0x9C7C}, {0x9B6F,0x9C81}, {0x9B74,0x9C82}, {0x9B77,0x9C7F}, {0x9B81,0x9C85}, 
+  {0x9B83,0x9C86}, {0x9B8E,0x9C87}, {0x9B90,0x9C90}, {0x9B91,0x9C8D}, {0x9B92,0x9C8B}, 
+  {0x9B9A,0x9C92}, {0x9B9D,0x9C9E}, {0x9B9E,0x9C95}, {0x9BAA,0x9C94}, {0x9BAB,0x9C9B}, 
+  {0x9BAD,0x9C91}, {0x9BAE,0x9C9C}, {0x9BC0,0x9CA7}, {0x9BC1,0x9CA0}, {0x9BC7,0x9CA9}, 
+  {0x9BC9,0x9CA4}, {0x9BCA,0x9CA8}, {0x9BD4,0x9CBB}, {0x9BD6,0x9CAD}, {0x9BDB,0x9CB7}, 
+  {0x9BDD,0x9CB4}, {0x9BE1,0x9CB1}, {0x9BE2,0x9CB5}, {0x9BE4,0x9CB2}, {0x9BE7,0x9CB3}, 
+  {0x9BE8,0x9CB8}, {0x9BEA,0x9CAE}, {0x9BEB,0x9CB0}, {0x9BF0,0x9CB6}, {0x9BF4,0x9CBA}, 
+  {0x9BFD,0x9CAB}, {0x9BFF,0x9CCA}, {0x9C08,0x9CBD}, {0x9C09,0x9CC7}, {0x9C0D,0x9CC5}, 
+  {0x9C10,0x9CC4}, {0x9C12,0x9CC6}, {0x9C13,0x9CC3}, {0x9C20,0x9CCB}, {0x9C23,0x9CA5}, 
+  {0x9C25,0x9CCF}, {0x9C28,0x9CCE}, {0x9C29,0x9CD0}, {0x9C2D,0x9CCD}, {0x9C31,0x9CA2}, 
+  {0x9C32,0x9CCC}, {0x9C33,0x9CD3}, {0x9C35,0x9CD8}, {0x9C37,0x9CA6}, {0x9C39,0x9CA3}, 
+  {0x9C3B,0x9CD7}, {0x9C3E,0x9CD4}, {0x9C45,0x9CD9}, {0x9C48,0x9CD5}, {0x9C49,0x9CD6}, 
+  {0x9C52,0x9CDF}, {0x9C54,0x9CDD}, {0x9C56,0x9CDC}, {0x9C57,0x9CDE}, {0x9C58,0x9C9F}, 
+  {0x9C5D,0x9CBC}, {0x9C5F,0x9C8E}, {0x9C67,0x9CE2}, {0x9C6D,0x9C9A}, {0x9C78,0x9C88}, 
+  {0x9C7A,0x9CA1}, {0x9CE5,0x9E1F}, {0x9CE9,0x9E20}, {0x9CEC,0x51EB}, {0x9CF3,0x51E4}, 
+  {0x9CF4,0x9E23}, {0x9CF6,0x9E22}, {0x9D06,0x9E29}, {0x9D07,0x9E28}, {0x9D09,0x9E26}, 
+  {0x9D15,0x9E35}, {0x9D1B,0x9E33}, {0x9D1D,0x9E32}, {0x9D1F,0x9E31}, {0x9D23,0x9E2A}, 
+  {0x9D26,0x9E2F}, {0x9D28,0x9E2D}, {0x9D2F,0x9E38}, {0x9D30,0x9E39}, {0x9D3B,0x9E3F}, 
+  {0x9D3F,0x9E3D}, {0x9D42,0x9E3A}, {0x9D51,0x9E43}, {0x9D52,0x9E46}, {0x9D53,0x9E41}, 
+  {0x9D5C,0x9E48}, {0x9D5D,0x9E45}, {0x9D60,0x9E44}, {0x9D61,0x9E49}, {0x9D6A,0x9E4C}, 
+  {0x9D6C,0x9E4F}, {0x9D6F,0x9E4E}, {0x9D72,0x9E4A}, {0x9D87,0x9E2B}, {0x9D89,0x9E51}, 
+  {0x9D93,0x9E4B}, {0x9D98,0x9E55}, {0x9D9A,0x9E57}, {0x9DA5,0x9E5B}, {0x9DA9,0x9E5C}, 
+  {0x9DAF,0x83BA}, {0x9DB4,0x9E64}, {0x9DBB,0x9E58}, {0x9DBC,0x9E63}, {0x9DC0,0x9E5A}, 
+  {0x9DC2,0x9E5E}, {0x9DC4,0x9E21}, {0x9DD3,0x9E67}, {0x9DD7,0x9E25}, {0x9DD9,0x9E37}, 
+  {0x9DDA,0x9E68}, {0x9DE5,0x9E36}, {0x9DE6,0x9E6A}, {0x9DEF,0x9E69}, {0x9DF2,0x9E6B}, 
+  {0x9DF3,0x9E47}, {0x9DF8,0x9E6C}, {0x9DF9,0x9E70}, {0x9DFA,0x9E6D}, {0x9E0C,0x9E71}, 
+  {0x9E15,0x9E2C}, {0x9E1A,0x9E66}, {0x9E1B,0x9E73}, {0x9E1D,0x9E42}, {0x9E1E,0x9E3E}, 
+  {0x9E7A,0x9E7E}, {0x9E7C,0x7877}, {0x9E7D,0x76D0}, {0x9E97,0x4E3D}, {0x9EA5,0x9EA6}, 
+  {0x9EA9,0x9EB8}, {0x9EBD,0x4E48}, {0x9ECC,0x9EC9}, {0x9EDE,0x70B9}, {0x9EE8,0x515A}, 
+  {0x9EF2,0x9EEA}, {0x9EF7,0x9EE9}, {0x9EFD,0x9EFE}, {0x9EFF,0x9F0B}, {0x9F09,0x9F0D}, 
+  {0x9F4A,0x9F50}, {0x9F4B,0x658B}, {0x9F4F,0x9F51}, {0x9F52,0x9F7F}, {0x9F54,0x9F80}, 
+  {0x9F59,0x9F85}, {0x9F5C,0x9F87}, {0x9F5F,0x9F83}, {0x9F60,0x9F86}, {0x9F61,0x9F84}, 
+  {0x9F66,0x9F88}, {0x9F6A,0x9F8A}, {0x9F6C,0x9F89}, {0x9F72,0x9F8B}, {0x9F77,0x9F8C}, 
+  {0x9F8D,0x9F99}, {0x9F90,0x5E9E}, {0x9F94,0x9F9A}, {0x9F95,0x9F9B}, {0x9F9C,0x9F9F}
+};
+
+
+
+/* the letters correspond to unicode characters which
+ * have Lu, Ll, Lt, Lm, Lo, Mn, Mc, or Me in their
+ * GENERAL CATEGORY */
+
+#define NUM_LETTER_INFO 274
+static const unirange letter_info[NUM_LETTER_INFO] = {
+  {0x41,0x5a}, {0x61,0x7a}, {0xaa,0xaa}, {0xb5,0xb5}, 
+  {0xba,0xba}, {0xc0,0xd6}, {0xd8,0xf6}, {0xf8,0x1f5}, 
+  {0x1fa,0x217}, {0x250,0x2a8}, {0x2b0,0x2b8}, {0x2bb,0x2c1}, 
+  {0x2d0,0x2d1}, {0x2e0,0x2e4}, {0x300,0x345}, {0x360,0x361}, 
+  {0x37a,0x37a}, {0x386,0x386}, {0x388,0x38a}, {0x38c,0x38c}, 
+  {0x38e,0x3a1}, {0x3a3,0x3ce}, {0x3d0,0x3d6}, {0x3da,0x3da}, 
+  {0x3dc,0x3dc}, {0x3de,0x3de}, {0x3e0,0x3e0}, {0x3e2,0x3f3}, 
+  {0x401,0x40c}, {0x40e,0x44f}, {0x451,0x45c}, {0x45e,0x481}, 
+  {0x483,0x486}, {0x490,0x4c4}, {0x4c7,0x4c8}, {0x4cb,0x4cc}, 
+  {0x4d0,0x4eb}, {0x4ee,0x4f5}, {0x4f8,0x4f9}, {0x531,0x556}, 
+  {0x559,0x559}, {0x561,0x587}, {0x591,0x5a1}, {0x5a3,0x5b9}, 
+  {0x5bb,0x5bd}, {0x5bf,0x5bf}, {0x5c1,0x5c2}, {0x5c4,0x5c4}, 
+  {0x5d0,0x5ea}, {0x5f0,0x5f2}, {0x621,0x63a}, {0x640,0x652}, 
+  {0x670,0x6b7}, {0x6ba,0x6be}, {0x6c0,0x6ce}, {0x6d0,0x6d3}, 
+  {0x6d5,0x6e8}, {0x6ea,0x6ed}, {0x901,0x903}, {0x905,0x939}, 
+  {0x93c,0x94d}, {0x951,0x954}, {0x958,0x963}, {0x981,0x983}, 
+  {0x985,0x98c}, {0x98f,0x990}, {0x993,0x9a8}, {0x9aa,0x9b0}, 
+  {0x9b2,0x9b2}, {0x9b6,0x9b9}, {0x9bc,0x9bc}, {0x9be,0x9c4}, 
+  {0x9c7,0x9c8}, {0x9cb,0x9cd}, {0x9d7,0x9d7}, {0x9dc,0x9dd}, 
+  {0x9df,0x9e3}, {0x9f0,0x9f1}, {0xa02,0xa02}, {0xa05,0xa0a}, 
+  {0xa0f,0xa10}, {0xa13,0xa28}, {0xa2a,0xa30}, {0xa32,0xa33}, 
+  {0xa35,0xa36}, {0xa38,0xa39}, {0xa3c,0xa3c}, {0xa3e,0xa42}, 
+  {0xa47,0xa48}, {0xa4b,0xa4d}, {0xa59,0xa5c}, {0xa5e,0xa5e}, 
+  {0xa70,0xa74}, {0xa81,0xa83}, {0xa85,0xa8b}, {0xa8d,0xa8d}, 
+  {0xa8f,0xa91}, {0xa93,0xaa8}, {0xaaa,0xab0}, {0xab2,0xab3}, 
+  {0xab5,0xab9}, {0xabc,0xac5}, {0xac7,0xac9}, {0xacb,0xacd}, 
+  {0xae0,0xae0}, {0xb01,0xb03}, {0xb05,0xb0c}, {0xb0f,0xb10}, 
+  {0xb13,0xb28}, {0xb2a,0xb30}, {0xb32,0xb33}, {0xb36,0xb39}, 
+  {0xb3c,0xb43}, {0xb47,0xb48}, {0xb4b,0xb4d}, {0xb56,0xb57}, 
+  {0xb5c,0xb5d}, {0xb5f,0xb61}, {0xb82,0xb83}, {0xb85,0xb8a}, 
+  {0xb8e,0xb90}, {0xb92,0xb95}, {0xb99,0xb9a}, {0xb9c,0xb9c}, 
+  {0xb9e,0xb9f}, {0xba3,0xba4}, {0xba8,0xbaa}, {0xbae,0xbb5}, 
+  {0xbb7,0xbb9}, {0xbbe,0xbc2}, {0xbc6,0xbc8}, {0xbca,0xbcd}, 
+  {0xbd7,0xbd7}, {0xc01,0xc03}, {0xc05,0xc0c}, {0xc0e,0xc10}, 
+  {0xc12,0xc28}, {0xc2a,0xc33}, {0xc35,0xc39}, {0xc3e,0xc44}, 
+  {0xc46,0xc48}, {0xc4a,0xc4d}, {0xc55,0xc56}, {0xc60,0xc61}, 
+  {0xc82,0xc83}, {0xc85,0xc8c}, {0xc8e,0xc90}, {0xc92,0xca8}, 
+  {0xcaa,0xcb3}, {0xcb5,0xcb9}, {0xcbe,0xcc4}, {0xcc6,0xcc8}, 
+  {0xcca,0xccd}, {0xcd5,0xcd6}, {0xcde,0xcde}, {0xce0,0xce1}, 
+  {0xd02,0xd03}, {0xd05,0xd0c}, {0xd0e,0xd10}, {0xd12,0xd28}, 
+  {0xd2a,0xd39}, {0xd3e,0xd43}, {0xd46,0xd48}, {0xd4a,0xd4d}, 
+  {0xd57,0xd57}, {0xd60,0xd61}, {0xe01,0xe2e}, {0xe30,0xe3a}, 
+  {0xe40,0xe4e}, {0xe81,0xe82}, {0xe84,0xe84}, {0xe87,0xe88}, 
+  {0xe8a,0xe8a}, {0xe8d,0xe8d}, {0xe94,0xe97}, {0xe99,0xe9f}, 
+  {0xea1,0xea3}, {0xea5,0xea5}, {0xea7,0xea7}, {0xeaa,0xeab}, 
+  {0xead,0xeae}, {0xeb0,0xeb9}, {0xebb,0xebd}, {0xec0,0xec4}, 
+  {0xec6,0xec6}, {0xec8,0xecd}, {0xedc,0xedd}, {0xf18,0xf19}, 
+  {0xf35,0xf35}, {0xf37,0xf37}, {0xf39,0xf39}, {0xf40,0xf47}, 
+  {0xf49,0xf69}, {0xf71,0xf84}, {0xf86,0xf87}, {0xf90,0xf95}, 
+  {0xf97,0xf97}, {0xf99,0xfad}, {0xfb1,0xfb7}, {0xfb9,0xfb9}, 
+  {0x10a0,0x10c5}, {0x10d0,0x10f6}, {0x1100,0x1159}, {0x115f,0x11a2}, 
+  {0x11a8,0x11f9}, {0x1e00,0x1e9b}, {0x1ea0,0x1ef9}, {0x1f00,0x1f15}, 
+  {0x1f18,0x1f1d}, {0x1f20,0x1f45}, {0x1f48,0x1f4d}, {0x1f50,0x1f57}, 
+  {0x1f59,0x1f59}, {0x1f5b,0x1f5b}, {0x1f5d,0x1f5d}, {0x1f5f,0x1f7d}, 
+  {0x1f80,0x1fb4}, {0x1fb6,0x1fbc}, {0x1fbe,0x1fbe}, {0x1fc2,0x1fc4}, 
+  {0x1fc6,0x1fcc}, {0x1fd0,0x1fd3}, {0x1fd6,0x1fdb}, {0x1fe0,0x1fec}, 
+  {0x1ff2,0x1ff4}, {0x1ff6,0x1ffc}, {0x207f,0x207f}, {0x20d0,0x20e1}, 
+  {0x2102,0x2102}, {0x2107,0x2107}, {0x210a,0x2113}, {0x2115,0x2115}, 
+  {0x2118,0x211d}, {0x2124,0x2124}, {0x2126,0x2126}, {0x2128,0x2128}, 
+  {0x212a,0x2131}, {0x2133,0x2138}, {0x3005,0x3005}, {0x302a,0x302f}, 
+  {0x3031,0x3035}, {0x3041,0x3094}, {0x3099,0x309e}, {0x30a1,0x30fa}, 
+  {0x30fc,0x30fe}, {0x3105,0x312c}, {0x3131,0x318e}, {0x4e00,0x9fa5}, 
+  {0xac00,0xd7a3}, {0xf900,0xfa2d}, {0xfb00,0xfb06}, {0xfb13,0xfb17}, 
+  {0xfb1e,0xfb28}, {0xfb2a,0xfb36}, {0xfb38,0xfb3c}, {0xfb3e,0xfb3e}, 
+  {0xfb40,0xfb41}, {0xfb43,0xfb44}, {0xfb46,0xfbb1}, {0xfbd3,0xfd3d}, 
+  {0xfd50,0xfd8f}, {0xfd92,0xfdc7}, {0xfdf0,0xfdfb}, {0xfe20,0xfe23}, 
+  {0xfe70,0xfe72}, {0xfe74,0xfe74}, {0xfe76,0xfefc}, {0xff21,0xff3a}, 
+  {0xff41,0xff5a}, {0xff66,0xffbe}, {0xffc2,0xffc7}, {0xffca,0xffcf}, 
+  {0xffd2,0xffd7}, {0xffda,0xffdc}
+};
+
+
+/* the digits correspond to unicode characters which
+ * have Nd in their GENERAL CATEGORY */
+
+#define NUM_DIGIT_INFO 16
+static const unirange digit_info[NUM_DIGIT_INFO] = {
+  {0x30,0x39}, {0x660,0x669}, {0x6f0,0x6f9}, {0x966,0x96f}, 
+  {0x9e6,0x9ef}, {0xa66,0xa6f}, {0xae6,0xaef}, {0xb66,0xb6f}, 
+  {0xbe7,0xbef}, {0xc66,0xc6f}, {0xce6,0xcef}, {0xd66,0xd6f}, 
+  {0xe50,0xe59}, {0xed0,0xed9}, {0xf20,0xf29}, {0xff10,0xff19}
+};
+
+
+/* the spaces correspond to unicode characters which
+ * have Zs, Zl, or Zp in their GENERAL CATEGORY */
+
+#define NUM_SPACE_INFO 5
+static const unirange space_info[NUM_SPACE_INFO] = {
+  {0x20,0x20}, {0xa0,0xa0}, {0x2000,0x200b}, {0x2028,0x2029}, 
+  {0x3000,0x3000}
+};
+
+
+
+/* parse_utf8_char parses the next utf-8 character, placing its
+ * unicode equivalent in *value. The length of the utf-8 character
+ * is returned */
+int parse_utf8_char(const unsigned char *here,
+		    const unsigned char *end,
+		    unsigned short *value) {
+  unsigned short c1;
+  unsigned short c2;
+  unsigned short c3;
+  
+  *value = 0; /* will be the value for illegal characters */
+  
+  if (here <= end) {
+    c1 = (unsigned short)(*here);
+    
+    if (c1 <= 0x7f) {
+      /* found a one-byte character */
+      *value = c1;
+      return 1;
+      
+    } else if (c1 >= 0xc0 && c1 <= 0xdf && here+1 <= end &&
+	       (c2=(unsigned short)(*(here+1))) >= 0x80) {
+      /* found a two-byte character */
+      *value = ((c1 & 0x1f) << 6) + (c2 & 0x3f);
+      return 2;
+      
+    } else if (c1 >= 0xe0 && c1 <= 0xef && here+2 <= end &&
+	       (c2=(unsigned int)(*(here+1))) >= 0x80 &&
+	       (c3=(unsigned int)(*(here+2))) >= 0x80) {
+      /* found a three-byte character */
+      *value = ((c1 & 0xf) << 12) + ((c2 & 0x3f) << 6) +
+	(c3 & 0x3f);
+      return 3;
+      
+    } else {
+      /* error in encoding, return the length as 1 */
+      return 1;
+    }
+  }
+
+  /* end of string */
+  return 0;
+}
+
+
+/* output_utf8_char encodes a unicode character as a UTF-8 character.
+ * The length of the encoding is returned. If the string was not
+ * long enough to encode the character 0 is returned. */
+int output_utf8_char (unsigned short value,
+		      unsigned char *here,
+		      unsigned char *end) {
+  int len = (int)(end - here + 1); /* end is the last character */
+  if (value < 0x80) {
+    /* will be encoded in one byte */
+    if (len >= 1) {
+      *here = (unsigned char)value;
+      return 1;
+    }
+
+  } else if (value < 0x800) {
+    /* will be encoded in two bytes */
+    if (len >= 2) {
+      *here++ = 0xc0 | (unsigned char)((value >> 6) & 0x1f);
+      *here = 0x80 | (unsigned char)(value & 0x3f);
+
+      return 2;
+    }
+
+  } else {
+    /* will be encoded in three bytes */
+    if (len >= 3) {
+      *here++ = 0xe0 | (unsigned char)((value >> 12) & 0xf);
+      *here++ = 0x80 | (unsigned char)((value >> 6) & 0x3f);
+      *here = 0x80 | (unsigned char)(value & 0x3f);
+      return 3;
+    }
+  }
+
+  return 0;
+}
+
+
+static int lookup_decompose_table (unsigned short value) {
+  int first = 0;
+  int last = NUM_DECOMPOSE_INFO;
+  int here;
+
+  while (first < last) {
+    here = (last-first)/2 + first;
+    if (value < decompose_info[here].ch) {
+      last = here;
+    } else if (value > decompose_info[here].ch) {
+      first = here+1;
+    } else {
+      return here;
+    }
+  }
+
+  return -1;
+}
+
+/* decompose_str will decompose a unicode string into its canonical
+ * equivalents. NULL is returned if the input array was not 
+ * large enough to contain the fully decomposed string (the array
+ * will be in a correct, but partially decomposed state). The input 
+ * must be null-terminated. */
+unsigned short *decompose_str (unsigned short *input,
+			       int max_output_len) {
+  int curpos = 0;
+  int curlen = unicode_strlen (input);
+  int tablepos = 0;
+  int leninc;
+  int i;
+
+  while (input[curpos] != 0) {
+    tablepos = lookup_decompose_table (input[curpos]);
+    if (tablepos >= 0) {
+      /* found a decomposition */
+      leninc = unicode_strnlen (decompose_info[tablepos].str, 3)-1;
+      if (curlen+leninc < max_output_len) {
+	/* enough space for this decomposition */
+
+	if (leninc > 1) {
+	  /* we have to shift the array along */
+	  for (i=curlen-1; i>=curpos; i--) {
+	    input[i+leninc] = input[i];
+	  }
+	  input[curlen+leninc] = 0;
+	  curlen += leninc;
+	}
+
+	/* decompose the character */
+	for (i=0; i<=leninc; i++) {
+	  input[curpos+i] = decompose_info[tablepos].str[i];
+	}
+
+	/* note: curpos is not incremented because the decomposed */
+	/*       characters might need decomposing */
+
+      } else {
+	/* no space left in string */
+	return NULL;
+      }
+
+    } else {
+      /* the character cannot be decomposed */
+      curpos++;
+    }
+  }
+
+  return input;
+}
+
+
+
+/* unicode_cat_data is a combination of the letter, digit, and
+ * space information in a form that is quick to look up.
+ * The information about each byte is stored in two bits
+ *
+ * 00 - neither letter, digit or space
+ * 01 - letter
+ * 10 - space
+ * 11 - digit
+ *
+ * You can test for letter | space by looking at the right
+ * column
+ */
+unsigned long *unicode_cat_data;
+
+void create_unicat_data () {
+  int i=0;
+  int min_start=0;
+  int min_end=0;
+  register unsigned long min_value=0; /* filling the whole byte */
+  
+  const unirange *letter_here = letter_info;
+  const unirange *letter_end = letter_info+NUM_LETTER_INFO;
+  const unirange *digit_here = digit_info;
+  const unirange *digit_end = digit_info+NUM_DIGIT_INFO;
+  const unirange *space_here = space_info;
+  const unirange *space_end = space_info+NUM_SPACE_INFO;
+
+  if (unicode_cat_data != NULL) return;
+  unicode_cat_data = malloc (0x10000/4);
+  if (unicode_cat_data == NULL) return;
+  
+  while (i <= 0xffff) {
+    /* decide what to fill with next */
+    min_start = i;
+    min_end = 0xffff;
+    min_value = 0; /* 00 */
+    
+    while (letter_here < letter_end && letter_here->last < i) letter_here++;
+    if (letter_here < letter_end && letter_here->last <= min_end) {
+      min_start = letter_here->first;
+      min_end = letter_here->last;
+      min_value = 0x55555555; /* 01 */
+    }
+    
+    while (digit_here < digit_end && digit_here->last < i) digit_here++;
+    if (digit_here < digit_end && digit_here->last <= min_end) {
+      min_start = digit_here->first;
+      min_end = digit_here->last;
+      min_value = 0xffffffff; /* 11 */
+    }
+    
+    while (space_here < space_end && space_here->last < i) space_here++;
+    if (space_here < space_end && space_here->last <= min_end) {
+      min_start = space_here->first;
+      min_end = space_here->last;
+      min_value = 0xaaaaaaaa; /* 10 */
+    }
+
+    if (min_start < i) min_start = i;
+    else if (min_start > i) {
+      min_end = min_start-1;
+      min_start = i;
+      min_value = 0; /* 00 */
+    }
+
+    /* fill the entries from min_start to min_end with min_value */
+    while (i <= min_end) {
+      /* special version for common case */
+      if (i % 16 == 0) {
+	unicode_cat_data[i/16] = min_value; /* doesn't matter if too long */
+	i += 16;
+	if (i > min_end) {
+	  i = min_end+1;
+	  break;
+	}
+	
+      } else {
+	register unsigned long mask = 3 << ((i % 16) * 2);
+	unicode_cat_data[i/16] = (unicode_cat_data[i/16] & (~mask)) |
+				  (min_value & mask);
+	i++;
+      }
+    }
+  }
+}
+
+/* does a binary search on a range table */
+static int in_unirange (const unirange *info, int num_info,
+			unsigned short value) {
+  int first = 0;
+  int last = num_info;
+  int here;
+
+  while (first < last) {
+    here = (last-first)/2 + first;
+    if (value < info[here].first) {
+      last = here;
+    } else if (value > info[here].last) {
+      first = here+1;
+    } else {
+      return 1;
+    }
+  }
+
+  return 0;
+}
+
+
+
+/* tests to see whether 'value' is a valid Unicode letter */
+int is_unicode_letter (unsigned short value) {
+  if (unicode_cat_data == NULL) create_unicat_data ();
+  if (unicode_cat_data == NULL) {
+    /* use old method */
+    return in_unirange(letter_info, NUM_LETTER_INFO, value);
+  }
+
+  return (((unicode_cat_data[value/16] >> ((value%16)*2)) & 0x3) == 0x1);
+}
+
+/* tests to see whether 'value' is a valid Unicode digit */
+int is_unicode_digit (unsigned short value) {
+  if (unicode_cat_data == NULL) create_unicat_data ();
+  if (unicode_cat_data == NULL) {
+    /* use old method */
+    return in_unirange(digit_info, NUM_DIGIT_INFO, value);
+  }
+
+  return (((unicode_cat_data[value/16] >> ((value%16)*2)) & 0x3) == 0x3);
+}
+
+/* tests to see whether 'value' is a valid Unicode letter or 
+ * digit */
+int is_unicode_letdig (unsigned short value) {
+  if (unicode_cat_data == NULL) create_unicat_data ();
+  if (unicode_cat_data == NULL) {
+    /* use old method */
+    return (in_unirange(letter_info, NUM_LETTER_INFO, value) ||
+	    in_unirange(digit_info, NUM_DIGIT_INFO, value));
+    
+  }
+
+  return ((unicode_cat_data[value/16] >> ((value%16)*2)) & 0x1);
+}
+
+/* tests to see whether 'value' is a valid space
+ * The test includes both "C" spaces and "Unicode" spaces, i.e. 
+ * form-feed, newline, carriage return, horizontal tab,
+ * vertical tab, and the Zs, Zl, and Zp Unicode categorizations */
+int is_unicode_space (unsigned short value) {
+  if (value < 0x80) return isspace (value);
+  if (unicode_cat_data == NULL) create_unicat_data ();
+  if (unicode_cat_data == NULL) {
+    /* use old method */
+    return in_unirange(space_info, NUM_SPACE_INFO, value);
+  }
+
+  return (((unicode_cat_data[value/16] >> ((value%16)*2)) & 0x3) == 0x2);
+}
+
+/* returns the length of the unicode string */
+int unicode_strlen (const unsigned short *str) {
+  int len = 0;
+
+  /* keep going till you find the null-terminator */
+  while (*str != 0) {
+    len++;
+    str++;
+  }
+
+  return len;
+}
+
+/* returns the length of the unicode string, up to a maximum */
+int unicode_strnlen (const unsigned short *str, int max_length) {
+  int len = 0;
+
+  /* keep going till you find the null-terminator */
+  while (*str != 0 && len < max_length ) {
+    len++;
+    str++;
+  }
+
+  return len;
+}
+
+
+/* Does a binary search on a relate table, returns -1 if it is not
+ * found. */
+static int search_unirelate (const unirelate *info, int num_info,
+			     unsigned short value) {
+  int first = 0;
+  int last = num_info;
+  int here;
+
+  while (first < last) {
+    here = (last-first)/2 + first;
+    if (value < info[here].ch1) {
+      last = here;
+    } else if (value > info[here].ch1) {
+      first = here+1;
+    } else {
+      return here;
+    }
+  }
+
+  return -1;
+}
+
+
+/* returns the upper-case equivalent of value */
+unsigned short unicode_toupper (unsigned short value) {
+  int related = search_unirelate (toupper_info, NUM_TOUPPER_INFO, value);
+  if (related >= 0) {
+    value = toupper_info[related].ch2;
+  }
+  return value;
+}
+
+/* returns the lower-case equivalent of value */
+unsigned short unicode_tolower (unsigned short value) {
+  int related = search_unirelate (tolower_info, NUM_TOLOWER_INFO, value);
+  if (related >= 0) {
+    value = tolower_info[related].ch2;
+  }
+  return value;
+}
+
+/* returns the simplified Chinese character equivalent of 
+ * another Chinese character */
+unsigned short unicode_tosimplified (unsigned short value) {
+  int related = search_unirelate (tosimplified_info, NUM_TOSIMPLIFIED_INFO, value);
+  if (related >= 0) {
+    value = tosimplified_info[related].ch2;
+  }
+  return value;
+}
+
+
+/* converts a utf-8 word (string with length stored in the first byte
+ * to a Unicode array. To handle all situations the output buffer should
+ * be 256 unsigned shorts long. The output will also have the length as
+ * the first entry. */
+unsigned short *utf8_word_to_unicode (const unsigned char *input,
+				      unsigned short *output,
+				      int max_output_length) {
+  int inlen = *input;
+  const unsigned char *inhere = input+1;
+  const unsigned char *inend = &inhere[inlen-1];
+  unsigned short c;
+  int clen;
+  
+  int outlen = 0;
+  unsigned short *outhere = output+1;
+  
+  while (inhere <= inend && inlen < max_output_length-1) {
+    /* decode the character */
+    clen = parse_utf8_char (inhere, inend, &c);
+    inhere += clen;
+
+    if (clen > 0) {
+      /* store the character */
+      *outhere = c;
+      outhere++;
+      outlen++;
+    }
+  }
+
+  output[0] = outlen;
+  return output;
+}
+
+
+/* converts a unicode word buffer (with the length stored in the
+ * entry) to a utf8 encoded word output (with the length stored in
+ * the first byte. Only 255 bytes (not characters) can be stored 
+ * in the output. */
+unsigned char *unicode_to_utf8_word (const unsigned short *input,
+				     unsigned char *output,
+				     int max_output_length) {
+  int inlen = *input;
+  const unsigned short *word = input + 1;
+  int in_i = 0;
+  int clen;
+
+  int outlen = 0;
+  unsigned char *outhere = output+1;
+  unsigned char *outend = output+max_output_length;
+
+  while (in_i < inlen) {
+    if ((clen=output_utf8_char(word[in_i], outhere, outend)) == 0) {
+      /* ran out of space to encode the character */
+      break;
+    }
+
+    outhere += clen;
+    outlen += clen;
+    in_i++;
+  }
+
+  output[0] = outlen;
+
+  return output;
+}
+
Index: /tags/gsdl-21/gsdl/packages/mg/lib/unitool.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/unitool.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/unitool.h	(revision 699)
@@ -0,0 +1,88 @@
+#ifndef UNITOOL_H
+#define UNITOOL_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+/* This module is based on Unicode 2.1 */
+
+
+/* parse_utf8_char parses the next utf-8 character, placing its
+ * unicode equivalent in *value. The length of the utf-8 character
+ * is returned. end is the address of the last character. */
+int parse_utf8_char (const unsigned char *here,
+		     const unsigned char *end,
+		     unsigned short *value);
+
+/* output_utf8_char encodes a unicode character as a UTF-8 character.
+ * The length of the encoding is returned. If the string was not
+ * long enough to encode the character 0 is returned. end is the
+ * address of the last character */
+int output_utf8_char (unsigned short value,
+		      unsigned char *here,
+		      unsigned char *end);
+
+/* decompose_str will decompose a unicode string into its canonical
+ * equivalents. NULL is returned if the input array was not 
+ * large enough to contain the fully decomposed string (the array
+ * will be in a correct, but partially decomposed state). The input 
+ * must be null-terminated. */
+unsigned short *decompose_str (unsigned short *input,
+			       int max_output_len);
+
+/* tests to see whether 'value' is a valid Unicode letter */
+int is_unicode_letter (unsigned short value);
+
+/* tests to see whether 'value' is a valid Unicode digit */
+int is_unicode_digit (unsigned short value);
+
+/* tests to see whether 'value' is a valid Unicode letter or 
+ * digit */
+int is_unicode_letdig (unsigned short value);
+
+/* tests to see whether 'value' is a valid space
+ * The test includes both "C" spaces and "Unicode" spaces, i.e. 
+ * form-feed, newline, carriage return, horizontal tab,
+ * vertical tab, and the Zs, Zl, and Zp Unicode categorizations */
+int is_unicode_space (unsigned short value);
+
+/* returns the length of the unicode string */
+int unicode_strlen (const unsigned short *str);
+
+/* returns the length of the unicode string, up to a maximum */
+int unicode_strnlen (const unsigned short *str, int max_length);
+
+/* returns the upper-case equivalent of value */
+unsigned short unicode_toupper (unsigned short value);
+
+/* returns the lower-case equivalent of value */
+unsigned short unicode_tolower (unsigned short value);
+
+/* returns the simplified Chinese character equivalent of 
+ * another Chinese character */
+unsigned short unicode_tosimplified (unsigned short value);
+
+
+/* converts a utf-8 word (string with length stored in the first byte
+ * to a Unicode array. To handle all situations the output buffer should
+ * be 256 unsigned shorts long. The output will also have the length as
+ * the first entry. */
+unsigned short *utf8_word_to_unicode (const unsigned char *input,
+				      unsigned short *output,
+				      int max_output_length);
+
+/* converts a unicode word buffer (with the length stored in the
+ * entry) to a utf8 encoded word output (with the length stored in
+ * the first byte. Only 255 bytes (not characters) can be stored 
+ * in the output. */
+unsigned char *unicode_to_utf8_word (const unsigned short *input,
+				     unsigned char *output,
+				     int max_output_length);
+
+#ifdef __cplusplus
+	   }
+#endif
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/lib/win32in.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/win32in.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/win32in.c	(revision 699)
@@ -0,0 +1,72 @@
+/* This module provides an equivalent of <netinet/in.h> on  */
+/* unix systems. */
+
+unsigned long htonl(unsigned long x) {
+#if defined (LITTLE_ENDIAN) || defined (_LITTLE_ENDIAN) || defined(__LITTLE_ENDIAN)
+	x = ((x >> 24) & 0x000000FF) | ((x >> 8) & 0x0000FF00) |
+		((x << 8) & 0x00FF0000) | ((x << 24) & 0xFF000000);
+#endif
+
+	return x;
+}
+
+unsigned long ntohl(unsigned long x) {
+#if defined (LITTLE_ENDIAN) || defined (_LITTLE_ENDIAN) || defined(__LITTLE_ENDIAN)
+	x = ((x >> 24) & 0x000000FF) | ((x >> 8) & 0x0000FF00) |
+		((x << 8) & 0x00FF0000) | ((x << 24) & 0xFF000000);
+#endif
+	return x;
+}
+
+unsigned short htons(unsigned short x) {
+#if defined (LITTLE_ENDIAN) || defined (_LITTLE_ENDIAN) || defined(__LITTLE_ENDIAN)
+	x = ((x >> 8) & 0x00FF) | ((x << 8) & 0xFF00);
+#endif
+	return x;
+}
+
+unsigned short ntohs(unsigned short x) {
+#if defined (LITTLE_ENDIAN) || defined (_LITTLE_ENDIAN) || defined(__LITTLE_ENDIAN)
+	x = ((x >> 8) & 0x00FF) | ((x << 8) & 0xFF00);
+#endif
+	return x;
+}
+
+/* just to test stuff
+#include <stdio.h>
+
+main () {
+	long x;
+	short sx;
+
+	x = 0x000000FF;
+	printf ("before %i, ",(int)x);
+	x = htonl (x);
+	printf ("after %i\n",(int)x);
+
+	x = 0x0000FF00;
+	printf ("before %i, ",(int)x);
+	x = htonl (x);
+	printf ("after %i\n",(int)x);
+
+	x = 0x00FF0000;
+	printf ("before %i, ",(int)x);
+	x = htonl (x);
+	printf ("after %i\n",(int)x);
+
+	x = 0xFF000000;
+	printf ("before %i, ",(int)x);
+	x = htonl (x);
+	printf ("after %i\n\n",(int)x);
+
+	sx = 0x00FF;
+	printf ("before %i, ",(int)sx);
+	sx = htons (sx);
+	printf ("after %i\n",(int)sx);
+
+	sx = 0xFF00;
+	printf ("before %i, ",(int)sx);
+	sx = htons (sx);
+	printf ("after %i\n",(int)sx);
+}
+*/
Index: /tags/gsdl-21/gsdl/packages/mg/lib/win32in.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/win32in.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/win32in.h	(revision 699)
@@ -0,0 +1,7 @@
+/* This module provides an equivalent of <netinet/in.h> on  */
+/* unix systems. */
+
+unsigned long htonl(unsigned long x);
+unsigned long ntohl(unsigned long x);
+unsigned short htons(unsigned short x);
+unsigned short ntohs(unsigned short x);
Index: /tags/gsdl-21/gsdl/packages/mg/lib/xmalloc.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/lib/xmalloc.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/lib/xmalloc.c	(revision 699)
@@ -0,0 +1,107 @@
+/* xmalloc.c -- malloc with out of memory checking
+   Copyright (C) 1990, 91, 92, 93, 94 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+#ifdef HAVE_CONFIG_H
+# ifdef __WIN32__  /* [RPAP - Feb 97: WIN32 Port] */
+#  include <win32cfg.h>
+# else
+#  include <sysfuncs.h>
+# endif
+#endif
+
+#if __STDC__
+#define VOID void
+#else
+#define VOID char
+#endif
+
+#include <sys/types.h>
+
+#if STDC_HEADERS
+#include <stdlib.h>
+#else
+VOID *malloc ();
+VOID *realloc ();
+void free ();
+#endif
+
+/* This is for other GNU distributions with internationalized messages.
+   The GNU C Library itself does not yet support such messages.  */
+#if HAVE_LIBINTL_H
+# include <libintl.h>
+#else
+# define gettext(msgid) (msgid)
+#endif
+
+#ifndef EXIT_FAILURE
+#define EXIT_FAILURE 1
+#endif
+
+/* Exit value when the requested amount of memory is not available.
+   The caller may set it to some other value.  */
+int xmalloc_exit_failure = EXIT_FAILURE;
+
+#if __STDC__ && (HAVE_VPRINTF || HAVE_DOPRNT)
+void error (int, int, const char *, ...);
+#else
+void error ();
+#endif
+
+static VOID *
+fixup_null_alloc (n)
+     size_t n;
+{
+  VOID *p;
+
+  p = 0;
+  if (n == 0)
+    p = malloc ((size_t) 1);
+  if (p == 0)
+    error (xmalloc_exit_failure, 0, gettext ("Memory exhausted"));
+  return p;
+}
+
+/* Allocate N bytes of memory dynamically, with error checking.  */
+
+VOID *
+xmalloc (n)
+     size_t n;
+{
+  VOID *p;
+
+  p = malloc (n);
+  if (p == 0)
+    p = fixup_null_alloc (n);
+  return p;
+}
+
+/* Change the size of an allocated block of memory P to N bytes,
+   with error checking.
+   If P is NULL, run xmalloc.  */
+
+VOID *
+xrealloc (p, n)
+     VOID *p;
+     size_t n;
+{
+  if (p == 0)
+    return xmalloc (n);
+  p = realloc (p, n);
+  if (p == 0)
+    p = fixup_null_alloc (n);
+  return p;
+}
Index: /tags/gsdl-21/gsdl/packages/mg/man/man1/mg_compression_dict.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/man/man1/mg_compression_dict.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/man/man1/mg_compression_dict.1	(revision 699)
@@ -0,0 +1,219 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_compression_dict 1 \*(Dt CITRI
+.SH NAME
+mg_compression_dict \- build a compression dictionary.
+.SH SYNOPSIS
+.B mg_compression_dict
+[
+.B \-h
+]
+[
+.BR \-C " |"
+.BR \-P " |"
+.B  \-S
+]
+.if n .ti +9n
+[
+.BR \-0 " |"
+.BR \-1 " |"
+.BR \-2 " |"
+.B  \-3
+]
+[
+.BR \-H " |"
+.BR \-B " |"
+.BR \-D " |"
+.BR \-Y " |"
+.B  \-M
+]
+.if n .ti +9n
+.if t .ti +.5i
+[
+.BI \-l " lookback"
+]
+[
+.BI \-k " mem"
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_compression_dict
+builds a compression dictionary based on the statistics gathered
+during the first pass over the text.  The options to the program are
+mainly concerned with limiting the amount of memory the dictionary
+will use and with how the text compressor will cope with any novel
+words found during the compression phase.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-C
+Build a complete dictionary from the statistics file.  If during the
+text compression phase a novel word is found, then the compressor will
+produce an error message and stop.
+.TP
+.B \-P
+Build a partial dictionary from the statistics file.  This dictionary
+assumes that the statistics file are based on the entire text.  The
+statistics of words not includes in the dictionary are used to
+calculate the escape probability.  If novel words are being coded
+character by character, then there may not be a Huffman code for every
+possible character.  This means that the compressor may fail if a novel
+word contains a novel character.
+.TP
+.B \-S
+Build a seed dictionary from the statistics file.  This dictionary
+assumes that the statistics file is based on only a portion of the
+text to be compressed.  The probability of a novel word is based on the
+number of words that have only occurred once.  If novel words are being
+coded character by character, then the Huffman codes for characters are
+based on the frequency of characters in the dictionary.
+.TP
+.B \-0
+All words from the statistics file are included in the built
+dictionary.
+.TP
+.B \-1
+Words are included in the dictionary until the dictionary reaches the
+desired size.  Words are selected for the dictionary based on the order
+they occurred in the source text.
+.TP
+.B \-2
+Words are included in the dictionary until the dictionary reaches the
+desired size.  The most frequent words are included in the dictionary
+first; where there is a tie for frequency, the shortest word is
+included first.
+.TP
+.B \-3
+Words are included in the dictionary until the dictionary reaches the
+desired size.  The most frequent words are included in the dictionary
+first; where there is a tie for frequency, the shortest word is
+included first.  Words are the shuffled back and forth between the
+`keep' and `discard' lists to find the `optimal' set of words that
+should be in the dictionary.
+.TP
+.B \-H
+This specifies that novel words will be coded character by character
+using Huffman codes.
+.TP
+.B \-B
+This specifies that an auxiliary dictionary will be built by the
+compressor.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using binary codes.  The binary code represents their occurrence
+position in the auxiliary dictionary.
+.TP
+.B \-D
+This specifies that an auxiliary dictionary will be built by the
+compressor.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using delta codes.  The delta code represents their occurrence position
+in the auxiliary dictionary.
+.TP
+.B \-Y
+This specifies that an auxiliary dictionary will be built by the
+compressor.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using a combination of gamma and binary codes.  The code represents
+their occurrence position in the auxiliary dictionary.  This generally
+produces better compression than
+.B \-B
+or
+.BR \-D .
+.TP
+.B \-M
+This specifies that an auxiliary dictionary will be built by the
+compressor.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using a combination of gamma and binary codes.  The code represents
+their occurrence position in the auxiliary dictionary.  This method is
+adaptive within documents, and generally produces better compression
+than
+.BR \-B ,
+.B \-D
+or
+.BR \-Y .
+.TP
+.BI \-l " lookback"
+The generated dictionary is designed to be front coded when it is
+loaded into memory.  Under normal circumstances, a front-coded
+dictionary would require scanning from the beginning in order to find
+any particular word.  However, every
+.I lookback
+words in the dictionary, the whole word is stored and a pointer to that
+word maintained.  E.g., if
+.I lookback
+is 4, then every fourth word is stored in its entirety.
+.TP
+.BI \-k " mem"
+This limits the amount of memory to use for the generated
+dictionary.  Words are selected for the dictionary based of the text
+statistics, and whether
+.BR \-0 , " \-1" , " \-2"
+or
+.B \-3
+is specified.  The memory is calculated assuming a lookback of 0,
+irrespective of what actual lookback is specified.  This means that if
+a non-zero lookback is given, the dictionary will actually occupy
+less space than specified by
+.BR \-k .
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.text.stats
+Statistics about the source text.
+.TP
+.B *.text.dict
+Compression dictionary for the source text.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /tags/gsdl-21/gsdl/packages/mg/man/man1/mg_decompress_text.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/man/man1/mg_decompress_text.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/man/man1/mg_decompress_text.1	(revision 699)
@@ -0,0 +1,78 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_decompress_text 1 \*(Dt CITRI
+.SH NAME
+mg_decompress_text \- decompresses the compressed text of a collection
+.SH SYNOPSIS
+.B mg_decompress_text
+[
+.B \-h
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_decompress_text
+decompresses the compressed text of a collection, inserts paragraph and
+document markers, writing the output to
+.IR stdout ".  "
+This program was created
+to be used as a type of
+.BR mg_get (1).
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 25
+.B *.text
+Compressed documents.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.text.dict.aux
+Auxiliary compression dictionary.
+.TP
+.B *.text.dict.fast
+Fast loading compression dictionary.
+.TP
+.B *.invf.paragraph
+Paragraph counts for documents.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_passes (1).
Index: /tags/gsdl-21/gsdl/packages/mg/man/man1/mg_fast_comp_dict.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/man/man1/mg_fast_comp_dict.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/man/man1/mg_fast_comp_dict.1	(revision 699)
@@ -0,0 +1,86 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_fast_comp_dict 1 \*(Dt CITRI
+.SH NAME
+mg_fast_comp_dict \- build a fast loading compression dictionary
+.SH SYNOPSIS
+.B mg_fast_comp_dict
+[
+.B \-h
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_fast_comp_dict
+builds a fast-loading compression dictionary from the standard
+compression dictionary.  The new dictionary has the property that
+.BR mgquery (1)
+can load it in very quickly, making the startup time for
+.BR mgquery (1)
+much shorter.  The new dictionary is larger on disk than the original,
+but they are no different when it comes to memory usage.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.text.dict
+Compression dictionary.
+.TP
+.B *.text.dict.fast
+Fast loading compression dictionary.
+.TP
+.B *.text.dict.aux
+Auxiliary compression dictionary.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /tags/gsdl-21/gsdl/packages/mg/man/man1/mg_get.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/man/man1/mg_get.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/man/man1/mg_get.1	(revision 699)
@@ -0,0 +1,154 @@
+.TH mg_get 1 "31 January 1994"
+.SH NAME
+mg_get \- output source texts for processing
+.SH SYNOPSIS
+ 
+.B mg_get
+.I collection-name
+.RB [ \-init | \-i | \-text\fP|\fB\-t\fP|\fB\-cleanup\fP|\fB\-c\fP]
+.SH DESCRIPTION
+.LP
+This program is the default one used by mgbuild to generate the source text for the MG system. Any program may be used to generate the source text for mgbuild as long as it confirms to the interface specified here. 
+.SH OPTIONS
+.LP
+The
+.I collection\-name
+must appear before any other option. Only the first option has any significance. If no option is specified
+.B \-text
+is assumed.
+.RS
+.TP
+.BR \-init " and " \-i
+The program is called once with this flag at the start of building a collection.
+.TP
+.BR \-text " and " \-t
+The program is called with this flag multiple times during the building of a collection. The program outputs (on stdout) the text of the collection. \*(lq\fBDocuments\fP\*(rq within the collection are separated with by ctrl-B's (ASCII code 2). \*(lq\fBParagraphs\fP\*(rq within the collection are separated with ctrl-C's (ASCII code 3). A collection need not contain paragraphs unless a level 3 inverted file is being generated (see 
+.B mgbuild 
+and 
+.BR mg_passes ). 
+.TP
+.BR \-cleanup " and " \-c
+The program is called once with this flag at the completion of building a collection.
+.SH ENVIRONMENT
+.TP
+.SB MGDATA
+If this environment variable exists then its value is used a the default 
+directory where the mg collection files are. If this variable does not exist
+then the directory \*(lq\fB.\fP\*(rq is used by default. The command line 
+option 
+.BI \-d " directory"
+overrides the directory in 
+.BR MGDATA .
+.TP
+.SB MG_GETRC
+This environment variable specifies where the file containing the users 
+mg source configurations is. If not set, mg_get uses a default of 
+.B \*\~/.mg_getrc.
+
+This file contains TAB delimited lines of the form
+.RS 
+.RS 
+.I CollectionName
+.I CollectionType
+.I files or directory
+.RE
+
+.I CollectionName
+is the name of the collection supplied to mg_get.
+
+.I CollectionType
+specifies how mg_get should process the
+named files and directories and are descibed below.
+
+.I files or directory
+is either a list of files separted by blanks or a single directory.
+Some of the
+.I CollectionTypes
+deal with files and others with just a single directory.
+Any files used ending with .gz or .Z are decompressed with gzip before
+processing. References to '~' expands to the users HOME directory.
+
+.I CollectionType
+is one of
+.RS
+.TP
+.BR PARA
+For text based documents. The list of files specified are treated as
+a series of paragraphs separated by blank lines. Each paragraph becomes
+a seperate document on the indexed collection.
+.TP
+.BR MAIL
+for mail files. The list of files specified are treated as UNIX mail
+files separated by lines starting with 'From'. Each mail message becomes
+a seperate document on the indexed collection. As a extra feature any
+embedded tarmail encoded contents (enclosed by a 'xbtoa Begin' and 'xbtoa End'
+pair are removed.
+.TP
+.BR DIR
+(and DIR2)
+For a single directory of files. Each file in the directory (and in any of
+it's subdirectories) are treated as a single document. With
+.I DIR
+the pathname of the file is prefixed to the contents of each file as an
+extra line while this is not done for
+.I DIR2
+collections.
+.TP
+.BR BIB
+for biblography files. The list of files specified are treated as
+a series of biblography files (eg BIBTEX or TROFF) separated by lines
+starting with '@'. Each reference  becomes a seperate document on the indexed
+collection.
+.TP
+.BR TXTIMG
+for integrated collections of text and images. The single directory should
+contain files that have the same prefix if they are related. For example,
+monaLisa.pgm might be a gray-level image, and monaLisa.txt would be a textual
+file describing the image. The suffixes recognised are:
+.BR   .txt
+for  ascii text
+.BR  .ptm
+for  scanned text stored as a bilevel image
+.BR  .pbm
+for a black and white image (typically a line drawing)
+.BR  .pgm
+for a gray-scale image
+.BR In addition, if no corresponding ascii text file is found for
+a .pbm or .pgm file, then one is created with suffix .tmp.txt,
+and it stores the name of the image file (in principle it could
+store the OCR of a .txt.pbm file). At present the .tmp.txt files
+are deleted by the '-cleanup' option.
+
+.SH "EXAMPLE"
+
+An example ~/.mg_getrc file might look like:
+   alice   PARA    /users/alice13a.txt.Z
+   mail95  MAIL    ~/MAIL/1995/*
+   doc     DIR     ~/documents
+   bibs    BIB     ~/etc/REFS/*.bib ~/etc/REFS/CC/*.bib
+   letters DIR     ~/LETTERS
+   davinci TXTIMG  ~/images/davinci
+
+.SH "SEE ALSO"
+.LP
+.BR mg_compression_dict (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_make_fast_dict (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1)
+
Index: /tags/gsdl-21/gsdl/packages/mg/man/man1/mg_get_merge.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/man/man1/mg_get_merge.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/man/man1/mg_get_merge.1	(revision 699)
@@ -0,0 +1,86 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_get_merge 1 \*(Dt CITRI
+.SH NAME
+mg_get_merge \- output source texts for processing
+.SH SYNOPSIS
+.B mg_get_merge
+.I collection-name
+.if n .ti +9n
+[
+.BR \-init " |"
+.BR \-i " |"
+.BR \-text " |"
+.BR \-t " |"
+.BR \-cleanup " |"
+.B  \-c
+]
+.SH DESCRIPTION
+This program is the default one used by
+.BR mgmerge (1)
+to generate the source text for new documents to appended to
+an
+.BR mg (1)
+system database.
+Its function is exactly the same as
+.BR mg_get (1)
+except that it generates the 
+.I new 
+documents to be added to an
+.BR mg (1)
+system database that has already been built using
+.BR mgbuild(1).
+Any program may be used to generate the source
+text for
+.BR mgmerge (1),
+as long as it confirms to the interface specified
+here.
+.SH OPTIONS
+The
+.I collection-name
+must appear before any other option.  Only the first option has any
+significance.  If no option is specified,
+.B \-text
+is assumed.
+.TP "\w'\fB\-cleanup\fP and \fB\-c\fP'u+2n"
+.BR \-init " and " \-i
+The program is called once with this flag at the start of merging a
+collection.
+.TP
+.BR \-text " and " \-t
+The program is called with this flag multiple times during the
+merging of a collection.  The program outputs (on
+.IR stdout )
+the text of the collection.
+.I Documents
+within the collection are separated by ctrl-B's (ASCII code 2).
+.TP
+.BR \-cleanup " and " \-c
+The program is called once with this flag at the completion of
+merging a collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_get (1),
+.BR mgbuild (1),
+.BR mgmerge (1),
+.BR mg_text_merge (1),
+.BR mg_invf_merge (1).
Index: /tags/gsdl-21/gsdl/packages/mg/man/man1/mg_hilite_words.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/man/man1/mg_hilite_words.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/man/man1/mg_hilite_words.1	(revision 699)
@@ -0,0 +1,95 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_hilite_words 1 \*(Dt CITRI
+.SH NAME
+mg_hilite_words \- display text and highlight particular words which it contains
+.SH SYNOPSIS
+.B mg_hilite_words
+.if n .ti +9n
+[
+.B --stem_method [0-3]
+]
+.if n .ti +9n
+[
+.B --stemmer [english|lovin|french|simplefrench]
+]
+.if n .ti +9n
+[
+.B --style [bold|underline|italic|emphasis|strong]
+]
+.if n .ti +9n
+[
+.B --pager [less|more|html|???]
+]
+.if n .ti +9n
+.I list-of-words-to-highlight
+
+.SH DESCRIPTION
+.B mg_hilite_words
+reads text from stdin and outputs it into a pager such as
+less. It's command arguments include a list of stemmed words
+which if extracted from the text will be highlighted on the output
+using a highlighting style.
+
+It has been extended to output the highlighted text using html
+straight to stdout.
+
+.SH OPTIONS
+Options may appear in any order, but the
+.IR list-of-words-to-highlight ,
+must be last.
+.TP
+.B --stem_method [0-3]
+This option allows one to set the stemming method for the extraction of
+the words from the incoming text.
+This should match the way the inverted file index was created.
+The default is 3, which is fold and stem.
+.TP
+.B --stemmer [english|lovin|french|simplefrench]
+This option allows you to select the stemmer to use.  The
+default is the English stemmer.
+.TP
+.B --style [bold|underline|italic|emphasis|strong]
+This option sets the highlighting style.
+.IR Bold 
+and 
+.IR underline
+are utilised by the pager or for html output.
+.IR Italic,
+.IR emphasis,
+and
+.IR strong
+can only be used for html output.
+.TP
+.B --pager [less|more|html|???]
+One can use the standard 
+.IR less
+or
+.IR more
+pages for displaying the highlighted text.
+Or one can generate highlighted html using the paging setting of
+.IR html.
+One can also specify a custom pager of their own choosing (i.e. ???).
+.TP
+.B list-of-words-to-highlight
+Each word is taken as a command line argument.
+Each word should already be stemmed and folded according to
+the stem option.
+.SH USAGE
+This program was written primarily to be called by 
+.IR mgquery(1)
+.
+When the user selects the mode of "hilite" then this program
+is called for displaying the query-result documents instead of
+just the pager.
+.SH "SEE ALSO"
+.na
+.BR mgquery (1).
+
+
Index: /tags/gsdl-21/gsdl/packages/mg/man/man1/mg_invf_dict.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/man/man1/mg_invf_dict.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/man/man1/mg_invf_dict.1	(revision 699)
@@ -0,0 +1,104 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_invf_dict 1 \*(Dt CITRI
+.SH NAME
+mg_invf_dict \- build an `on-disk' stemmed dictionary
+.SH SYNOPSIS
+.B mg_invf_dict
+[
+.B \-h
+]
+[
+.B \-F
+]
+[
+.BI \-b " num"
+]
+[
+.BI \-d " directory"
+]
+.if n .ti +9n
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_invf_dict
+builds an `on-disk' stemmed dictionary from the
+compressed stemmed dictionary that the
+.BR mgquery (1)
+program can access.  Each entry in the dictionary contains the disk
+pointer to the associated inverted file entry, and various other pieces
+of information about the stemmed word.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-b " num"
+The dictionary is stored in blocks on disk; this option is used to set
+the block size.  The default is 4-KB blocks.
+.TP
+.B \-F
+This option causes the disk blocks to be aligned on the appropriate
+boundaries.  For example, if 4-KB blocks are used, they will be aligned
+in the file on 4-KB boundaries.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.dict.build
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict
+The `on-disk' stemmed dictionary.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /tags/gsdl-21/gsdl/packages/mg/man/man1/mg_invf_dump.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/man/man1/mg_invf_dump.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/man/man1/mg_invf_dump.1	(revision 699)
@@ -0,0 +1,142 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_invf_dump 1 \*(Dt CITRI
+.SH NAME
+mg_invf_dump \- Dump out an inverted file in ASCII
+.SH SYNOPSIS
+.B mg_invf_dump
+[
+.B \-h
+]
+[
+.B \-b
+]
+[
+.B \-w
+]
+[
+.B \-t
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+This program dumps out an inverted file produced by the
+.BR mg (1)
+system as ASCII numbers.  This program could be used in conjunction
+with
+.BR mgdictlist (1)
+to write simple shell programs that work with the inverted files.  The
+output from the program looks something like this:
+.LP
+.RS
+.ft 3
+.nf
+6 337
+1
+ 1 1
+1
+ 5 1
+1
+ 2 1
+1
+ 1 1
+4
+ 1 3
+ 2 7
+ 3 2
+ 5 2
+1
+ 1 1
+ . . .
+.fi
+.ft
+.RE
+The first number (6) is the number of documents in the collection.  It
+is followed by the number of different terms in the collection.  For
+each term, the number of different documents it occurs in follows. The
+document numbers that the term occurs in are listed next.  If
+.B \-w
+is specified, then the number of times the term occurred in the
+document is printed alongside the document number.
+If
+.B \-t
+is specified, the term itself is also displayed in quotes.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-b
+This option will cause the output from the program to be in fixed-size
+binary numbers, rather than in ASCII.
+.TP
+.B \-w
+If the inverted file is a level-2 or level-3 inverted file, this causes
+the word counts per document to be output.
+.TP
+.B \-t
+This option causes each term to be displayed along with the number of
+documents it appears in. The term is printed in quotes,
+as in "the".
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf.dict
+The compressed stemmed dictionary.
+.TP
+.B *.invf.ORG
+The original inverted file saved by
+.BR mg_invf_rebuild .
+.TP
+.B *.invf
+The inverted file.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /tags/gsdl-21/gsdl/packages/mg/man/man1/mg_invf_merge.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/man/man1/mg_invf_merge.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/man/man1/mg_invf_merge.1	(revision 699)
@@ -0,0 +1,112 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_invf_merge 1 \*(Dt CITRI
+.SH NAME
+mg_invf_merge \- Merge two inverted files and stemmed dictionaries
+.SH SYNOPSIS
+.B mg_invf_merge
+[
+.B \-h
+]
+[
+.B \-s
+]
+[
+.B \-w
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+This program takes two
+.BR mg (1)
+database compressed text files and appends the second to
+the first.
+It is part of the
+.BR mgmerge (1)
+addition to the
+.BR mg (1)
+package.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-s
+This option will cause a slow merge to be performed on the inverted files,
+where each inverted file entry is decoded and recoded.
+The default is a fast merge. Accumulated fast merges slowly degrade
+compression performance on the resulting inverted file so
+a periodic slow merge is recommended.
+.TP
+.B \-w
+By default the database .weights file is updated with
+new weights for the new documents in
+.B mg_text_merge.
+The
+.B -w
+option turns this step off.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.old.invf.dict
+The old compressed stemmed dictionary.
+.TP
+.B *.new.invf.dict
+The new compressed stemmed dictionary.
+.TP
+.B *.invf.dict
+The resulting merged compressed stemmed dictionary.
+.TP
+.B *.old.invf.ORG
+The original inverted file saved by
+.BR mg_invf_rebuild .
+.TP
+.B *.old.invf
+The old inverted file.
+.TP
+.B *.new.invf
+The new inverted file.
+.TP
+.B *.invf
+The resulting merged inverted file.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mgmerge (1),
+.BR mg_get_merge (1),
+.BR mg_text_merge (1),
+.BR mg_invf_rebuild (1),
+.BR mg_invf_dict (1),
+.BR mg_weights_build (1),
+.BR mgquery (1).
Index: /tags/gsdl-21/gsdl/packages/mg/man/man1/mg_invf_rebuild.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/man/man1/mg_invf_rebuild.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/man/man1/mg_invf_rebuild.1	(revision 699)
@@ -0,0 +1,156 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_invf_rebuild 1 \*(Dt CITRI
+.SH NAME
+mg_invf_rebuild \- Rebuild an mg inverted file with or without skips.
+.SH SYNOPSIS
+.B mg_invf_rebuild
+.RB [ \-h ]
+.if n .ti +9n
+[
+.BR \-0 " |"
+.B \-1
+[
+.BI \-k " num"
+] |
+.B \-2
+[
+.BI \-s " num"
+]
+[
+.BI \-m " num"
+]
+]
+.if n .ti +9n
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_invf_rebuild
+builds a new inverted file, with or without skipping, from an old
+inverted file.  This file uses
+.I *.invf.ORG
+and
+.I *.invf.idx.ORG
+as the
+source from which it builds
+.I *.invf
+and
+.IR *.invf.idx .
+If
+.I *.invf.ORG
+or
+.I *.invf.idx.ORG
+do not exist, the program renames
+.I *.invf
+or
+.I *.invf.idx
+to
+.I *.invf.ORG
+or
+.I *.invf.idx.ORG
+as appropriate.  The old inverted file may contain skipping.  This
+means that it is possible to delete the
+.I *.ORG
+files
+after the new inverted file is built.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-m\fP \fInum\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-0
+This generates a non-skipped inverted file.  This option is normally
+only needed if the
+.I *.ORG
+files have been deleted.
+.TP
+.B \-1
+This generates a skipped inverted file.  The
+.BI \-k " num"
+argument specifies the number of pointers hopped over with each skip.
+.TP
+.B \-2
+This option generates a skipped inverted file.  The skipped inverted
+file is built so that it is `optimal' for ranking using a specific
+number of accumulators.  Each term in the inverted file has a
+different skip length.  The arguments
+.BR \-s " and " \-m
+control the sizes of the skips.
+.TP
+.BI \-k " num"
+This specifies the number of pointers that should be hopped over with
+each skip.  This option is only valid if
+.B \-1
+is specified.
+.TP
+.BI \-m " num"
+This specifies the intended number of accumulators that will be used
+when ranking queries are done on the collection.
+.TP
+.BI \-s " num"
+This specifies the minimum size for skips.  If the calculation of the
+optimal skip size results in a number smaller than
+.IR num ,
+the skip size is set to
+.IR num .
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.ORG
+Original inverted file.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.TP
+.B *.invf.idx.ORG
+The original index into the inverted file.
+.TP
+.B *.invf.dict.build
+Compressed stemmed dictionary.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /tags/gsdl-21/gsdl/packages/mg/man/man1/mg_passes.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/man/man1/mg_passes.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/man/man1/mg_passes.1	(revision 699)
@@ -0,0 +1,441 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_passes 1 \*(Dt CITRI
+.SH NAME
+mg_passes \- builds mg databases
+.SH SYNOPSIS
+.B mg_passes
+[
+.B \-h
+]
+[
+.B \-G
+]
+[
+.B \-S
+]
+[
+.B \-D
+]
+[
+.B \-W
+]
+.if n .ti +9n
+[
+.BR \-1 " |"
+.BR \-2 " |"
+.B  \-3
+]
+[
+.BI \-C " compstatpoint"
+]
+.if n .ti +9n
+[
+.BI \-n " tracename"
+]
+.if t .ti +.5i
+[
+.BI \-b " bufsize"
+]
+[
+.BI \-m " memlimit"
+]
+.if n .ti +9n
+[
+.BI \-c " numchunks"
+]
+[
+.BI \-a " stemmer"
+]
+[
+.BI \-s " stemmethod"
+]
+[
+.BI \-t " tracepos"
+]
+.if n .ti +9n
+[
+.B \-T1
+]
+[
+.B \-T2
+]
+.if t .ti +.5i
+[
+.B \-I1
+]
+[
+.B \-I2
+]
+.if n .ti +9n
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+[
+.I filename(s)
+]
+.SH DESCRIPTION
+.B mg_passes
+is the program that does most of the work when building
+.BR mg (1)
+database systems.  The input documents can come from either
+.I stdin
+or from a list of files on the command line.  Individual documents
+must be separated with control-B characters.  In general,
+.B mg_passes
+must be run twice to build a database, first with the
+.B \-T1
+and
+.B \-I1
+options, and second with the
+.B \-T2
+and
+.B \-I2
+options.  Several other programs must be run in order to get an
+.BR mg (1)
+database that is ready for the
+.BR mgquery (1)
+program.  The
+.SB EXAMPLE
+section below gives an example of how to build a complete
+.BR mg (1)
+database.
+.SH OPTIONS
+Options may appear in any order, but the
+.IR filename(s) ,
+if specified, must be last.
+.TP "\w'\fB\-C\fP \fIcompstatpoint\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-G
+Treat SGML tags as non-words when building the inverted file.  An SGML
+tag is anything between angle brackets, i.e., `<' and `>'.
+.TP
+.B \-S
+This option causes a special pass to be executed.  It is up to a user
+to modify
+.I mg.special.c
+in the source code to do something with the documents it is given.
+.TP
+.B \-D
+If
+.B mg_passes
+fails, then print the document that caused the failure to the trace
+file if tracing is active, or to
+.I stderr
+if it is not.
+.TP
+.B \-W
+This option enables the generation of the weights file when
+.B \-I2
+is specified.  It causes
+.B \-I2
+to use a little more memory and CPU.
+.TP
+.B \-1
+Produce a level-1 inverted file.  This option is only useful when
+specified with
+.BR "\-I1 ".
+A level-1 inverted file makes it possible for
+.BR mgquery (1)
+to do Boolean queries.  Ranked queries can still be done,
+although the quality of the ranking is abysmal.
+.TP
+.B \-2
+Produce a level-2 inverted file.  This option is only useful when
+specified with
+.BR "\-I1 ".
+This is the default when neither
+.BR \-1 ", " "\-2 " "or " \-3
+is specified.
+A level-2 inverted file makes it possible for
+.BR mgquery (1)
+to do Boolean queries and cosine-ranked queries.
+.TP
+.B \-3
+Produce a level-3 inverted file.  This option is only useful when
+specified with
+.BR "\-I1 ".
+This has been implemented to enable paragraph-level inversion.
+Paragraphs are delimited by control-C characters in the source text.
+.TP
+.BI \-C " compstatpoint"
+This option causes statistics on the compression performance to be
+output to a file called
+.IR *.compression.stats .
+.I compstatpoint
+specifies the interval between outputting each line of statistics.  The
+units of
+.I compstatpoint
+are kilobytes of source text.  E.g., if
+.I compstatpoint
+is 10, then a line is output to the file every 10 KB of input
+source.  Each line of the file consists of 4 numbers The first number
+is the amount of input text, in bytes, processed so far.  The second
+number is the amount of input text, in bytes, processed since the
+last line was output to the file.  The third number is the number of
+output bytes generated since the last line was output to the file, and
+the fourth number is the compression achieved since the last line was
+output, i.e., the third number divided by the second number.
+.TP
+.BI \-n " tracename"
+This specifies the filename to use for the trace log, if tracing is
+enabled using the
+.B \-t
+option.  If
+.BI \-n " tracename"
+is not given and tracing is enabled, a default trace filename will be
+used.
+.TP
+.BI \-s " stemmethod"
+This specifies the method to use to \*(lqstem\*(rq the words in the
+inverted file dictionary.  This is a bit mask specifying the
+operations to do on words as they are parsed out of the text, where
+bit number 0 is the low-order (rightmost) bit.  Bit 0 does case
+folding, and bit 1 does simple stemming, so the value 3 for
+.I stemmethod
+does both case folding and stemming.
+.TP
+.BI \-a " stemmer"
+This specifies the stemmer to use when stemming words.  This
+is a description of the language the stemmer is intended for
+or a description of the stemmer.  Valid options include:
+english, lovin, french, and simplefrench.
+.TP
+.BI \-b " bufsize"
+Specify the size of the document buffer in kilobytes.  If any document
+is larger than
+.IR bufsize ,
+the program will abort with an error message.  This should probably be
+replaced with some system which automatically increases the buffer
+size as required.  The default size is 3072 KB (3 MB).
+.TP
+.BI \-m " memlimit"
+Maximum amount of memory to use for the pass-2 file inversion in
+megabytes.  This option is only useful when used in conjunction with
+the option
+.BR \-I1 .
+The larger this value, the faster the pass-2 inversion will proceed.
+The default value is 5 MB.
+.TP
+.BI \-c " numchunks"
+The maximum number of inversion chunks to write to disk.  Each chunk
+will be approximately as large as
+.IR memlimit .
+This option is only useful when used in conjunction with the option
+.BR \-I2 .
+The larger this value, the faster the pass-2 inversion will proceed.
+The default value is 5 MB.
+.TP
+.BI \-t " tracepos"
+This option activates tracing.  A line will be generated in the
+trace file for every
+.I tracepos
+input bytes processed.  The default name for the trace file can be
+overridden using the
+.BI \-n " tracename"
+option.
+.TP
+.B \-T1
+Generate the
+.I *.text.stats
+file.
+.TP
+.B \-T2
+Generate the
+.IR *.text ,
+.IR *.text.idx ,
+and possibly the
+.I *.text.dict.aux
+files.  Using this option requires that the
+.I *.text.dict
+file be present.
+.TP
+.B \-I1
+Generate the
+.IR *.invf.dict ,
+.IR *.invf.chunk ,
+and
+.I *.invf.chunk.trans
+files.
+.TP
+.B \-I2
+Generate the
+.I *.invf
+and
+.I *.invf.idx
+files.  Using this option requires
+that the
+.IR *.invf.dict.hash ,
+.IR *.invf.chunk ,
+and
+.I *.invf.chunk.trans
+files
+be present.  The
+.I *.invf.dict.hash
+file is generated by
+.BR mg_perf_hash_build (1)
+from the
+.I *.invf.dict.build
+file.  If the
+.B \-W
+option is specified, the
+.I *.weight
+file will also be generated.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection is to be
+written.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection that will be
+created.
+.TP
+.I filename(s)
+This specifies the source text. If this is not specified, then the
+program expects the source text from
+.IR stdin .
+.SH EXAMPLE
+What follows is a UNIX
+.BR csh (1)
+script as an example of how to build an
+.BR mg (1)
+document collection.
+.LP
+.nf
+.DT
+.ft B
+.I #! /bin/csh
+.I
+# The first argument on the command line specifies the
+.I
+#   source of the text
+set source = ($1)
+.PP
+.I
+# The second argument is the name of the collection
+set text = ($2)
+.PP
+.I
+# Create *.text.stats,  *.invf.dict.build,
+.I
+#   *.invf.chunk and *.invf.chunks.trans
+${source} | mg_passes -T1 -I1 -m 1 -t 1 -f ${text}
+.PP
+.I
+# Create *.text.dict
+mg_compression_dict -f ${text}
+.PP
+.I
+# Create *.invf.dict.hash
+mg_perf_hash_build -f ${text}
+.PP
+.I
+# Create *.text,  *.text.idx,
+.I
+#   *.invf and *.invf.idx
+${source} | mg_passes -T2 -I2 -c 2 -t 1 -f ${text}
+.PP
+.I
+# Create *.text.idx.wgt and *.weight.approx
+mg_weights_build -f ${text} -b 8
+.PP
+.I
+# Create *.invf.dict
+mg_invf_dict -f ${text} -b 4096
+.PP
+.I
+# Create *.text.dict
+mg_fast_comp_dict -f ${text}
+.ft R
+.fi
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.chunk
+Inverted file chunk descriptor file.  When the inverted file is
+created it is created in chunks that use no more than a set amount of
+memory.  This file describes those chunks.
+.TP
+.B *.invf.chunk.trans
+Word-occurrence-order to lexical-order translation file.  The
+.B *.invf.chunk
+file is written in word-occurrence order but is required by
+.B \-I2
+to be in lexical order.
+.TP
+.B *.invf.dict.build
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict.hash
+Data for an order-preserving perfect hash function.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.B *.text
+Compressed documents.
+.TP
+.B *.text.stats
+Statistics about the text.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.trace
+The default trace file.
+.TP
+.B *.compression.stats
+Statistics about the compression of the text.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /tags/gsdl-21/gsdl/packages/mg/man/man1/mg_perf_hash_build.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/man/man1/mg_perf_hash_build.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/man/man1/mg_perf_hash_build.1	(revision 699)
@@ -0,0 +1,89 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_perf_hash_build 1 \*(Dt CITRI
+.SH NAME
+mg_perf_hash_build \- generate an order-preserving hash function for the stemmed dictionary
+.SH SYNOPSIS
+.B mg_perf_hash_build
+[
+.B \-h
+]
+[
+.BI \-r " rndseed"
+]
+[
+.BI \-d  " directory"
+]
+.if n .ti +9n
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_perf_hash_build
+generates an order-preserving hashing function from the compressed
+stemmed dictionary and writes it out to disk.
+.BR mg_passes (1)
+will make use of the hash function when it builds the inverted file.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-r " rndseed"
+This specifies the random seed to be used in the generation of the
+hash function; it must be an integer.  If this is not specified, the
+current time will be used as the random seed.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf.dict
+Stemmed dictionary.
+.TP
+.B *.invf.dict.hash
+Data for a order-preserving perfect hash function.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /tags/gsdl-21/gsdl/packages/mg/man/man1/mg_stem_idx.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/man/man1/mg_stem_idx.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/man/man1/mg_stem_idx.1	(revision 699)
@@ -0,0 +1,95 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_stem_idx 1 \*(Dt CITRI
+.SH NAME
+mg_stem_idx \- builds a stem index file
+.SH SYNOPSIS
+.B mg_stem_idx
+[
+.B \-h
+]
+[
+.B \-F
+]
+[
+.BI \-b " num"
+]
+[
+.BI \-d " directory"
+]
+.if n .ti +9n
+.B \-s 1|2|3
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_stem_idx
+generates a stem index file for a collection that is to be built with
+stem indexes.  This program should be called three times: once for each
+.B -s
+parameter.  It uses the stemmed dictionary to create the stem index which
+contains pointers into the stemmed dictionary.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-b " num"
+The dictionary is stored in blocks on disk; this option is used to set
+the block size.  The default is 4-KB blocks.
+.TP
+.B \-F
+This option causes the disk blocks to be aligned on the appropriate
+boundaries.  For example, if 4-KB blocks are used, they will be aligned
+in the file on 4-KB boundaries.
+.TP
+.B -s 1|2|3
+The stem method to apply for the stem index.
+.br
+1 = casefolded and non-stemmed
+.br
+2 = non-casefolded and stemmed
+.br
+3 = casefolded and stemmed
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 25
+.B *.invf.dict.blocked
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict.blocked.1
+Stem index with stem index method 1.
+.TP
+.B *.invf.dict.blocked.2
+Stem index with stem index method 2.
+.TP
+.B *.invf.dict.blocked.3
+Stem index with stem index method 3.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_invf_dict (1),
+.BR mgstemidxlist (1).
Index: /tags/gsdl-21/gsdl/packages/mg/man/man1/mg_text_estimate.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/man/man1/mg_text_estimate.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/man/man1/mg_text_estimate.1	(revision 699)
@@ -0,0 +1,112 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_text_estimate 1 \*(Dt CITRI
+.SH NAME
+mg_text_estimate \- Estimate the size of the compressed text based on the text statistics and a compression dictionary.
+.SH SYNOPSIS
+.B mg_text_estimate
+[
+.B \-h
+]
+[
+.BR \-H " |"
+.BR \-B " |"
+.BR \-D " |"
+.BR \-Y " |"
+.B \-M
+]
+.if n .ti +9n
+.I stats-dict
+.I compression-dict
+.SH DESCRIPTION
+This program estimates the size of the compressed text that would be
+generated if text with certain statistics were compressed with a
+particular dictionary.  This program has no real use in the
+.BR mg (1)
+system, except for experimenting with the
+.BR mg_compression_dict (1)
+program.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-m\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-H
+This specifies that novel words will be coded character by character
+using Huffman codes.
+.TP
+.B \-B
+This specifies that the size of an auxiliary dictionary will be
+estimated.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using binary codes.  The binary code represents their occurrence
+position in the auxiliary dictionary.
+.TP
+.B \-D
+This specifies that the size of an auxiliary dictionary will be
+estimated.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using delta codes.  The delta code represents their occurrence position
+in the auxiliary dictionary.
+.TP
+.B \-Y
+This specifies that the size of an auxiliary dictionary will be
+estimated.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using a combination of gamma and binary codes.  The code represents
+their occurrence position in the auxiliary dictionary.  This generally
+produces better compression than
+.B \-B
+or
+.BR \-D .
+.TP
+.B \-M
+This specifies that the size of an auxiliary dictionary will be
+estimated.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using a combination of gamma and binary codes.  The code represents
+their occurrence position in the auxiliary dictionary.  This method is
+adaptive within documents, and generally produces better compression
+than
+.BR \-B ,
+.B \-D
+or
+.BR \-Y .
+.SH FILES
+.TP 20
+.B *.text.stats
+Statistics about the text.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /tags/gsdl-21/gsdl/packages/mg/man/man1/mg_text_merge.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/man/man1/mg_text_merge.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/man/man1/mg_text_merge.1	(revision 699)
@@ -0,0 +1,76 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_text_merge 1 \*(Dt CITRI
+.SH NAME
+mg_text_merge \- Append one compressed text file to another
+.SH SYNOPSIS
+.B mg_text_merge
+[
+.B \-h
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+This program takes two
+.BR mg (1)
+database inverted files and stemmed
+dictionaries, named "old" and "new" respectively, and creates
+a merged stemmed dictionary and inverted file.
+It is part of the
+.BR mgmerge (1)
+addition to the
+.BR mg (1)
+package.
+The two compressed text files are
+.IR "name" .old.text
+and
+.IR "name" .new.text.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.old.text
+The old compressed text.
+.TP
+.B *.new.text
+The new compressed text that will be appended
+.TP
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mgmerge (1),
+.BR mg_get_merge (1),
+.BR mg_invf_merge (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mgquery (1).
Index: /tags/gsdl-21/gsdl/packages/mg/man/man1/mg_weights_build.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/man/man1/mg_weights_build.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/man/man1/mg_weights_build.1	(revision 699)
@@ -0,0 +1,112 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_weights_build 1 \*(Dt CITRI
+.SH NAME
+mg_weights_build \- builds the approximate weights file
+.SH SYNOPSIS
+.B mg_weights_build
+[
+.B \-h
+]
+[
+.B \-s
+]
+[
+.BI \-b " num"
+]
+[
+.BI \-d " directory"
+]
+.if n .ti +9n
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_weights_build
+is the program that generates the approximate weights file and the
+interleaved text index and exact weights file.  To do this, it uses
+the text index and either the exact weights file, or the inverted file
+and stemmed dictionary.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-s
+Save the weights generated from the
+.I *.invf
+and
+.I *.invf.dict
+file in a
+.I *.weight
+file.
+.TP
+.BI \-b " num"
+This specifies the number of bits of precision that should be used for
+the approximate weights.  If not specified, it defaults to 8.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection is.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.dict
+Compressed stemmed dictionary.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.text.idx.wgt
+Interleaved index into the compressed documents and document weights.
+.TP
+.B *.weight.approx
+Approximate document weights.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /tags/gsdl-21/gsdl/packages/mg/man/man1/mgbilevel.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/man/man1/mgbilevel.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/man/man1/mgbilevel.1	(revision 699)
@@ -0,0 +1,116 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgbilevel 1 \*(Dt CITRI
+.SH NAME
+mgbilevel \- compresses a bilevel image using a specified template
+.SH SYNOPSIS
+.B mgbilevel
+.B \-e
+[
+.B \-t
+.I \*(lqstring\*(rq
+]
+.I infile
+[
+.I \>compressed-outfile
+]
+.nf
+or
+.fi
+.B mgbilevel
+.B \-d
+.I compressed-infile
+[
+.I \>outfile
+]
+.SH DESCRIPTION
+.B mgbilevel
+performs (up to) a two-stage template-based compression.  The
+user can specify the template to be used in the compression process.  The
+default template size is a two-stage template 22/10.  The compressed file
+contains the template type in the stream.  The output files are written
+to
+.IR stdout .
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-t\fP \fI\*(lqstring\*(rq\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-e
+Compresses
+.I infile
+using a template specified, and writes to
+.IR compressed-outfile .
+.TP
+.B \-d
+Decompresses
+.I compressed-infile
+using the template contained in the compressed file.  Writes to
+.IR outfile .
+.TP
+\fB\-t \fP\fI\*(lqstring\*(rq\fP
+This option specifies which template combination is to
+be used in the compression.  Five different characters
+can occur in the string: `.' for a pixel not
+included in the template, `p' for a pixel included in the 1st
+level template, `2' for a pixel included in the 2nd level
+template (a subset of the 1st level), `*' denoting the position
+of the
+.I current
+pixel, and a `;' denoting the end of each template line.
+The default template is a two-level 22/10 template:
+.IP
+.RS
+.nf
+\&.ppppp.;
+pp222pp;
+p22222p;
+p22*.\|.\|.;
+.fi
+.RE
+.IP
+This would be specified on the command line as:
+.nf
+\&\fB\-T\fP ".ppppp.;pp222pp;p22222p;p22*.\|.\|.;"
+.fi
+.TP
+.I infile
+This is the input
+.I .pbm
+file to be compressed.
+.TP
+.I outfile
+This is the decompressed
+.I .pbm
+file.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /tags/gsdl-21/gsdl/packages/mg/man/man1/mgbuild.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/man/man1/mgbuild.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/man/man1/mgbuild.1	(revision 699)
@@ -0,0 +1,169 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgbuild 1 \*(Dt CITRI
+.SH NAME
+mgbuild \- build an mg system database
+.SH SYNOPSIS
+.B mgbuild
+[
+.B \-c
+]
+[
+.BI \-g " get"
+]
+[
+.BI \-s " source"
+]
+[
+.BI \-d " mgdata dir"
+]
+.I collection-name
+.SH DESCRIPTION
+.B mgbuild
+is a
+.B csh
+script that executes all the appropriate programs in the correct order
+to completely build an
+.BR mg (1)
+system database ready for queries to be made
+by
+.BR mgquery.
+This program makes use of the
+.BR mg_get (1)
+script to obtain the text of the collection.
+.SH OPTIONS
+Options can occur in any order, but the collection name must be last.
+.TP "\w'\fIcollection-name\fP'u+2n"
+.BI \-c
+This specifies whether the
+.I get
+program is \*(lqcomplex\*(rq.  If a
+.I get
+program is \*(lqcomplex\*(rq, then it requires initialisation and
+cleanup with the
+.B \-i
+and
+.B \-c
+options.
+.TP
+.BI \-g " get"
+This specifies the program to use for getting the source text for the
+build.  If no
+.B \-g
+option is given, the default program
+.BR mg_get (1)
+is used.
+.TP
+.BI \-s " source"
+The
+.B mgbuild
+program consists of two parts.  The first part initializes variables
+to default values.  The second part uses these variables to control
+how the
+.BR mg (1)
+database is built.  This option specifies a program to execute between
+the first and second parts.  The details of what the variables are, and
+how they may be changed, are in comments in the
+.B mgbuild
+program.
+.TP
+.BI \-d " mgdata dir"
+Setting this option allows one to override the MGDATA environment variable.
+.TP
+.I collection-name
+This is the collection name, as required by the
+.BR mg_get (1)
+program.  It serves both as a
+.I case
+statement selector, and as the name of a subdirectory that holds the
+indexing files.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.chunk
+Inverted file chunk descriptor file.  When the inverted file is
+created, it is written in chunks that use no more than a set amount of
+memory.  This file describes those chunks.
+.TP
+.B *.invf.chunk.trans
+Word-occurrence-order to lexical-order translation file.  The
+.B *.invf.chunk
+file is written in word-occurrence order but is required by
+.B \-N2
+to be in lexical order.
+.TP
+.B *.invf.dict
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict.blocked
+The `on-disk' stemmed dictionary.
+.TP
+.B *.invf.dict.hash
+Data for an order-preserving perfect hash function.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.B *.text
+Compressed documents.
+.TP
+.B *.text.stats
+Text statistics.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.text.idx.wgt
+Interleaved index into the compressed documents and document weights.
+.TP
+.B *.weight.approx
+Approximate document weights.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /tags/gsdl-21/gsdl/packages/mg/man/man1/mgbuild_4.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/man/man1/mgbuild_4.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/man/man1/mgbuild_4.1	(revision 699)
@@ -0,0 +1,53 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgbuild_4 1 \*(Dt CITRI
+.SH NAME
+mgbuild_4 \- build an mg system database using 4 passes instead of 2
+.SH SYNOPSIS
+.B mgbuild_4
+[
+.B \-c
+]
+[
+.BI \-g " get"
+]
+[
+.BI \-s " source"
+]
+[
+.BI \-d " mgdata dir"
+]
+.I collection-name
+.SH DESCRIPTION
+.B mgbuild_4
+is a
+.B csh
+script that executes all the appropriate programs in the correct order
+to completely build an
+.BR mg (1)
+system database ready for queries to be made
+by
+.BR mgquery.
+This program makes use of the
+.BR mg_get (1)
+script to obtain the text of the collection.
+In 
+.BR mgbuild (1)
+the build is done by two passes over the input text. The first
+pass creates the compressed text dictionary and inverted file dictionary,
+the second pass creates the compressed text and inverted file. What 
+.BR mgbuild_4
+does is use a separate pass for each function. This will be slower but will
+not require as much RAM at one time. Therefore, if your build is failing
+due to running short of memory, then 
+.BR mgbuild_4
+is worth a try.
+.SH "SEE ALSO"
+.na
+.BR mgbuild (1)
Index: /tags/gsdl-21/gsdl/packages/mg/man/man1/mgdictlist.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/man/man1/mgdictlist.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/man/man1/mgdictlist.1	(revision 699)
@@ -0,0 +1,72 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgdictlist 1 \*(Dt CITRI
+.SH NAME
+mgdictlist \- list the contents of a dictionary
+.SH SYNOPSIS
+.B mgdictlist
+[
+.B \-q
+]
+.I filename
+.SH DESCRIPTION
+.B mgdictlist
+lists the contents of either a compressed compression dictionary, a
+text statistics file, or a compressed stemmed dictionary.  Unlike all
+the other programs in the
+.BR mg (1)
+suite, it is necessary to specify the complete pathname of the
+dictionary to list.
+.SH OPTIONS
+The
+.I filename
+must be last.
+.TP "\w'\fIfilename\fP'u+2n"
+.B \-q
+Generate an abbreviated output (machine-readable rather than
+human-readable).
+.TP
+.I filename
+This is the name of the dictionary to list.
+.B mgdictlist
+automatically detects whether the file is a compressed compression
+dictionary, text statistics file, or a compressed stemmed dictionary.
+.SH FILES
+.TP 20
+.B *.text.dict
+Text statistics file.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.invf.build
+Compressed stemmed dictionary.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /tags/gsdl-21/gsdl/packages/mg/man/man1/mgfelics.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/man/man1/mgfelics.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/man/man1/mgfelics.1	(revision 699)
@@ -0,0 +1,103 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgfelics 1 \*(Dt CITRI
+.SH NAME
+mgfelics \- fast and efficient lossless image (de)compression
+.SH SYNOPSIS
+.B mgfelics
+.B \-e
+[
+.BI \-k " n"
+]
+.I infile
+[
+.I \>compressed-outfile
+]
+.br
+or
+.br
+.B mgfelics
+.B \-d
+.I compressed-infile
+[
+.I \>outfile
+]
+.SH DESCRIPTION
+.B mgfelics
+is a lossless compression process which can (de)compress portable
+bitmap files in
+.I pgm
+format.
+.B mgfelics
+writes to
+.IR stdout .
+.SH OPTIONS
+.TP "\w'\fB\-k\fP \fIn\fP'u+2n"
+.BI \-k " n"
+An integer parameter (< 32) which controls the compression speed and
+ratio.  The value of
+.I n
+is included in the output stream, and therefore is not needed in the
+decompression stage.  The default value of
+.I n
+is equal to the number of bits required to represent each grey level.
+.TP
+.B \-e
+Encodes the file
+.IR infile ,
+sending the output to
+.IR stdout .
+.TP
+.B \-d
+Decodes the file from
+.IR compressed-infile ,
+sending the output to
+.IR stdout .
+.SH NOTE
+.B mgfelics
+uses a combination of adjusted binary and Rice codes, the choice of
+which for any given pixel depends on the environment as determined by
+the intensity values of neighbouring pixels.  The result is a very
+fast and lossless algorithm for compressing grayscale images.  The
+optional parameter
+.BI \-k " n"
+corresponds to the maximum exponent of the same name in Rice encoding.
+Smaller values of
+.BI \-k " n"
+can result in significant improvement in compression times with a
+corresponding loss in compression ratio.  However the reduction in
+compression ratio with a small
+.BI \-k " n"
+parameter may be insignificant for many images.
+.SH REFERENCES
+Algorithm by Paul G. Howard and Jeffrey Scott Vitter
+of Brown University, RI, and Duke University, NC, respectively.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /tags/gsdl-21/gsdl/packages/mg/man/man1/mgmerge.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/man/man1/mgmerge.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/man/man1/mgmerge.1	(revision 699)
@@ -0,0 +1,175 @@
+`.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgmerge 1 \*(Dt CITRI
+.SH NAME
+mgmerge \- update an mg system database with new documents
+.SH SYNOPSIS
+.B mgmerge
+[
+.B \-c
+]
+[
+.BI \-g " get"
+]
+[
+.BI \-s " source"
+]
+[
+.B \-S
+]
+[
+.B \-w
+]
+.I collection-name
+.SH DESCRIPTION
+.B mgmerge
+is a
+.B csh
+script that executes all the appropriate programs in the correct order
+to completely merge a current
+.BR mg (1)
+system database ready with some new documents, saving the need for a complete
+database rebuild with
+.B mgbuild.
+It does this by building a second
+.BR mg (1)
+system database from the new documents and then merging this database
+with the old one.
+This program makes use of the
+.BR mg_get_merge (1)
+script to obtain the text of the collection.
+.B mgmerge
+cannot edit or delete documents already in a database; only new documents
+can be added.
+.SH OPTIONS
+Options can occur in any order, but the collection name must be last.
+.TP "\w'\fIcollection-name\fP'u+2n"
+.BI \-c
+This specifies whether the
+.I get
+program is \*(lqcomplex\*(rq.  If a
+.I get
+program is \*(lqcomplex\*(rq, then it requires initialisation and
+cleanup with the
+.B \-i
+and
+.B \-c
+options.
+.TP
+.BI \-g " get"
+This specifies the program to use for getting the source text for the
+build.  If no
+.B \-g
+option is given, the default program
+.BR mg_get_merge (1)
+is used.
+.TP
+.BI \-s " source"
+The
+.B mgmerge
+program consists of two parts.  The first part initializes variables
+to default values.  The second part uses these variables to control
+how the
+.BR mg (1)
+database is built.  This option specifies a program to execute between
+the first and second parts.  The details of what the variables are, and
+how they may be changed, are in comments in the
+.B mgmerge
+program.
+If this option is used, the program should ideally be the same one that
+was called by
+.BR mgbuild (1)
+to build the database since some parameters need to be consistent between
+.B mgbuild
+and
+.B mgmerge.
+.TP
+.B \-S
+This option will cause a slow merge to be performed on the inverted files,
+where each inverted file entry is decoded and recoded.
+The default is a fast merge. Accumulated fast merges slowly degrade
+compression performance on the resulting inverted file so
+a periodic slow merge is recommended.
+.TP
+.B \-w
+Adding new documents can have an effect on the weight of the previous ones.
+By default the weights for documents already in the collection are not 
+recomputed since the change in their values is usually small.
+This option forces new weights to be recomputed.
+Periodic use of this option, as for the "-S" option, is recomended,
+otherwise query rankings may become inaccurate.
+.TP
+.I collection-name
+This is the collection name, as required by the
+.BR mg_get_merge (1)
+program.  It serves both as a
+.I case
+statement selector, and as the name of a subdirectory that holds the
+indexing files.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+Note that a temporary directory under the
+.B MGDATA
+directory is used to perform the merge.
+The default name for this directory is
+.BR MERGE .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.dict
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict.blocked
+The `on-disk' stemmed dictionary.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.B *.text
+Compressed documents.
+.TP
+.B *.text.stats
+Text statistics.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.text.idx.wgt
+Interleaved index into the compressed documents and document weights.
+.TP
+.B *.weight.approx
+Approximate document weights.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mgbuild (1),
+.BR mg_get_merge (1),
+.BR mg_get (1),
+.BR mg_invf_merge (1),
+.BR mg_text_merge (1),
+.BR mg_query (1),
+.BR mg_weights_build (1).
Index: /tags/gsdl-21/gsdl/packages/mg/man/man1/mgquery.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/man/man1/mgquery.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/man/man1/mgquery.1	(revision 699)
@@ -0,0 +1,693 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgquery 1 \*(Dt CITRI
+.SH NAME
+mgquery \- query program for the mg system
+.SH SYNOPSIS
+.B mgquery
+[
+.B \-h
+]
+[
+.B \-D
+]
+[
+.BI \-f " name"
+]
+[
+.BI \-d " directory"
+]
+.if n .ti +9n
+[
+.I collection-name
+]
+.SH DESCRIPTION
+.B mgquery
+enables users to make Boolean or ranked queries from a data base
+generated by the
+.BR mg (1)
+system.  It accepts queries from
+.I stdin
+and sends the retrieved documents to
+.IR stdout .
+Information on the resource usage of
+.B mgquery
+as it processes queries can be obtained interactively.
+.SH OPTIONS
+Options may appear in any order, but the
+.IR collection-name ,
+if specified, must be last.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-D
+This option causes the entire text to be decompressed and sent to
+.IR stdout .
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection that will be
+used.  If a collection with the specified base
+.I name
+does not exist, an error message will be displayed and
+.B mgquery
+will exit.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.SH USAGE
+Prior to processing the command line arguments, the
+.B mgquery
+program attempts to read in a startup script called
+.IR ./.mgrc .
+If that fails, it attempts to read in the file
+.IR $HOME/.mgrc .
+The startup file can only contain commands\(emno queries are
+permitted in the
+.I .mgrc
+file.  Lines starting with \*(lq\fB#\fP\*(rq in the file are comments.
+The most common use for the
+.I .mgrc
+file is to personalise the initial values of the predefined parameters
+with
+.B .set
+commands.
+.LP
+The input to
+.B mgquery
+consists of a series of input lines.  The backslash
+character
+.RB (\*(lq \e \*(rq)
+is used at the end of lines to indicate
+that input continues on the next line.
+.LP
+Input lines on which the first character is a dot
+.RB (\*(lq . \*(rq)
+are commands to the
+.B mgquery
+program.  Input lines that do not start with a dot are queries.
+.LP
+A query consists of two parts.  One part is a Boolean or ranked query
+that identifies documents.  The second part is a post-processing
+pattern matching operation.  Any text between the first speech mark
+(\*(lq) and the last speech mark (\*(rq) is considered to be a
+post-processing pattern.
+.SH COMMANDS
+The
+.B mgquery
+program can accept the following commands.
+.TP 17
+.B .help
+Display several pages of help text.
+.TP
+.B .quit
+Quit the program.
+.TP
+.B .warranty
+Display the
+.BR mg (1)
+warranty.
+.TP
+.B .conditions
+Display the conditions of use and distribution of
+.BR mg (1).
+.TP
+.BI ".set " "name value"
+Set the parameter
+.I name
+to the specified
+.IR value .
+If the parameter is a Boolean
+.I value
+and the
+.I value
+is omitted, the parameter will be inverted (i.e., if it was
+.IR true ,
+then it will change to
+.IR false ;
+if it was
+.IR false ,
+then it will change to
+.IR true ).
+.TP
+.BI ".unset " name
+Delete the parameter
+.I name
+from the currently-defined parameters.
+.TP
+.B .reset
+Reset the parameters to the state that they had after the processing
+of the
+.B mgquery
+command line.
+.TP
+.B .display
+Display the values of all the currently-defined parameters.
+.TP
+.B .push
+Push the currently-defined parameters onto a stack.
+.TP
+.B .pop
+Pops a set of parameters off the stack, replacing the currently-defined
+ones.
+.TP
+.BI ".output " arg
+This is used to specify where to send the text of the documents.  Once
+the
+.B .output
+command is specified, all subsequent output will be sent to the place
+specified by
+.IR arg .
+If
+.I arg
+is not specified subsequent output will be directed to
+.IR stdout .
+.I Arg
+may be any of the following.
+.RS
+.TP 13
+.BI "> " filename
+Send output to the specified file.
+.TP
+.BI ">> " filename
+Append output to the specified file.
+.TP
+.BI "| " command
+Pipe the output to
+.IR command ,
+which is executed by
+.IR sh .
+.RE
+.TP
+.BI ".input " arg
+This is used to specify where input (queries and commands) comes
+from.  Once the
+.B .input
+command is specified all subsequent input will be come from the place
+specified by
+.IR arg .
+If
+.I arg
+is not specified subsequent input will come from
+.IR stdin .
+.RS
+.TP 13
+.BI "< " filename
+Get input from the specified file.
+.TP
+.BI "| " command
+The input comes from the standard output of
+.IR command ,
+which is executed by
+.IR sh .
+.RE
+.SH PARAMETERS
+The following parameters are predefined and have special
+significance.  Each parameter will be followed by its default
+value.  Parameters are initialised before the
+.I .mgrc
+file is read or the command line arguments are processed.
+.TP 17
+.BI accumulator_method " `array'"
+This parameter is used during ranking, and specifies how the
+weight for each document should be accumulated.  The following
+methods are available:
+.IR array ,
+.IR splay_tree ,
+.IR hash_table ,
+and
+.IR list .
+.TP
+.BI briefstats " `off'"
+This is a Boolean parameter that determines whether the
+totals for disk, memory and time usage statistics will be
+displayed at the end of each query.
+.IR Note :
+this takes precedence over the parameters
+.BR diskstats ,
+.BR memstats " and " timestats .
+This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI buffer " `1048576'"
+When the documents are being read in, they are read into a
+buffer of this size and then displayed from this buffer.  If
+the documents are larger than this buffer, the buffer is
+expanded automatically.  Having a large buffer gives a very
+slight performance improvement, because it allows the order of
+disk operations to be optimised.  The buffer size is measured
+in bytes.
+.TP
+.BI diskstats " `off'"
+This is a Boolean parameter that determines whether the disk
+usage statistics for the preceding query will be displayed
+after each query.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI doc_sepstr " `---------------------------------- %n\en\'"
+This specifies the string that will be used to separate
+documents when they are displayed for `Boolean' or `docnums'
+queries.  The standard C escape character sequences
+may be used to place special characters in the
+string.  For example, a newline would be `\en'.  To include a `%',
+use the sequence `%%'.  To include the
+.BR mg (1)
+document number, use the sequence `%n'.  The following escape character
+sequences are available
+.nf
+.ta 1.7iL
+.B Sequence	Meaning
+`\e\e'	backslash
+`\eb'	backspace
+`\ef'	formfeed
+`\en'	newline
+`\er'	carriage return
+`\et'	tab
+`\e"'	speech marks
+`\e''	quote mark
+`\ex\fIhh\fP'	ASCII code in hexadecimal
+`\ennn'	ASCII code in octal
+.fi
+.TP
+.BI expert " `false'"
+If this is
+.IR true ,
+then much of the dialogue output is suppressed.  This parameter may
+take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI hash_tbl_size " `1000'"
+One of the options during ranking queries is to use a hash
+table to accumulate the weights for each document.  The hash
+table is a simple chained type.  This parameter specifies the
+size of the hash table and may take any value between 8 and
+268435456 (2^28).
+.TP
+.BI heads_length " `50'"
+When the mode is
+.BR heads ,
+this specifies the number of characters that will be output for each
+document.
+.TP
+.BI maxdocs " `all'"
+The maximum number of documents to display in response to a
+query.  This parameter may take on a numeric value between 1
+and 429467295 (2^32 - 1) or the word
+.IR all .
+.TP
+.BI maxparas " `1000'"
+The maximum number of paragraphs to identify during a ranked
+query with paragraph indexing.  After the paragraphs have been
+identified, the paragraphs are converted into documents, and
+because some of the paragraphs may refer to the same documents
+the final number of answers may be less than
+.BR maxparas .
+The
+.B maxdocs
+parameter will then be applied.  This parameter may take on a numeric
+value between 1 and 429467295 (2^32 - 1).
+.TP
+.BI max_accumulators " `50000'"
+This parameter limits the number of different paragraph and
+document numbers to be accumulated during ranked queries when
+the parameter
+.B accumulator_method
+is set to
+.IR splay_tree ,
+.IR hash_table ,
+or
+.IR list .
+This parameter may take any value between 8 and 268435456 (2^28).
+.TP
+.BI max_terms " `all'"
+This parameter limits the number of terms that will actually
+be used during a ranked query.  If more terms than the number
+specified by
+.B max_terms
+are entered, then the extra terms will be discarded.  If
+.B sorted_terms
+is on, then the limiting will be done after the terms have been
+sorted.  This parameter may take any value between 1 and 429467295
+(2^32 - 1), or the word
+.IR all.
+.TP
+.BI memstats " `off'"
+This is a Boolean parameter that determines whether the memory
+usage statistics for the preceding query will be displayed
+after each query.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI mgdir " `.'"
+This is set to the directory where the
+.BR mg (1)
+data files may be found.  If
+the environment variable
+.B MGDATA
+exists, then this is instead initialised to the value of
+.BR MGDATA .
+The value of this parameter may be changed, either in the
+.I .mgrc
+file with a
+.BI ".set mgdir "directory
+command, or from the command line using the
+.BI \-d " directory"
+option.  Once the \*(lq\fB>\fP\*(rq prompt appears, changing this
+parameter will have no effect.
+.TP
+.BI mgname " `bible'"
+This is set to the name of the
+.BR mg (1)
+collection that is to be used for the session.  The value of this
+parameter may be changed, either in the
+.I .mgrc
+file with a
+.BI ".set mgname "name
+command, or from the command line using the
+.BI \-f " name"
+option.  Once the \*(lq\fB>\fP\*(rq prompt appears, changing this
+parameter will have no effect.
+.TP
+.BI mode " `text'"
+This specifies how documents should be displayed when they
+are retrieved.  It may take six different values:
+.IR text ,
+.IR hilite ,
+.IR docnums ,
+.IR heads ,
+.IR silent ,
+or
+.IR count .
+.I text
+displays the contents of the document.
+.I hilite
+displays the contents of the document and highlights any of the 
+stemmed query terms.
+.I docnums
+displays only the document numbers.
+.I heads
+is used to print out the head of each document.
+.I silent
+retrieves all the documents but displays nothing except how many
+documents were retrieved.  This mode is intended to be used in timing
+experiments.
+.I count
+does the minimum
+amount of work required to determine how many documents would
+be retrieved, but does not retrieve them.
+.TP
+.BI optimise_type " `1'"
+There are three types of boolean query optimisation (parse tree
+rearrangement). Type 0 leaves parse tree unaltered. Type 1 optimises
+for AND of terms and AND of OR of terms. Type 2 converts the tree
+into DNF (an experiment :-).
+.TP
+.BI pager " `more'"
+This is the name of the program that will be used to display
+the help and the retrieved documents.  If the environment
+variable
+.B PAGER
+is defined, then
+.B pager
+takes on that value.
+.TP
+.BI hilite_style " `bold'"
+This specifies the type of highlighting method.
+It may take one of two different values:
+.IR bold,
+or
+.IR underline.
+.TP
+.BI para_sepstr " `\en######## PARAGRAPH %n ########\en'"
+This specifies the string that will be used to separate paragraphs.
+The standard C escape character sequences may be used to place special
+characters in the string.  For example, a newline would be written
+as `\en'.  To include a `%', use the sequence `%%'.  To include the
+paragraph number within the document, use the sequence `%n'.
+.TP
+.BI para_start " `***** Weight = %w *****\en'"
+This specifies the string that will be used at the head of paragraphs
+for a paragraph-level index following a ranked query.  The standard
+C-language escape character sequences may be used to place special
+characters in the string.  For example, a newline would be written as
+`\en'.  To include a `%', use the sequence `%%'.  To include the
+paragraph weight, use the sequence `%w'.
+.TP
+.BI qfreq " `true'"
+This determine whether the ranked queries will take into account the
+number of times each query term is specified.  When this is
+.IR true ,
+the number of times a term appears in the query is used in the
+ranking.  When this is
+.IR false ,
+all query terms are assumed to occur only once.  This parameter may
+take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI query  " `Boolean'"
+This specifies the type of queries that are to be specified.
+It can take four different values:
+.IR Boolean ,
+.IR ranked ,
+.IR docnums " or " approx-ranked.
+.I Boolean
+is for Boolean queries.
+The
+.BR yacc (1)
+grammar for Boolean queries is as follows.
+.IP
+.nf
+        query   : or;
+.IP
+        or      : or '|' and
+                | and ;
+.IP
+        and     : and '&' not
+                | and not
+                | not ;
+.IP
+        not     : term
+                | '!' not ;
+.IP
+        term    : TERM
+                | '(' or ')' ;
+.fi
+.IP
+.IR ranked " and " approx-ranked
+are for queries ranked by the cosine measure.
+.I approx-ranked
+uses only the low-precision document lengths, and therefore only
+produces an approximation to full cosine ranking.
+.IP
+.nf
+        query   : TERM
+                | query TERM ;
+.fi
+.IP
+.I docnums
+allows the entry of document numbers.  Multiple numbers separated by
+spaces may be specified, or ranges separated by hyphens.
+.IP
+.nf
+        query   : range
+                | query range ;
+.IP
+        range   : num
+                | num '-' num ;
+.fi
+.TP
+.BI ranked_doc_sepstr " `-------------------------------- %n %w\en'"
+This specifies the string that will be used to separate documents when
+they are displayed for `ranked' or `approx-ranked' queries.  The
+standard C escape character sequences may be used to place special
+characters in the string.  For example, a newline would be written as
+`\en'.  To include a `%', use the sequence `%%'.  To include the
+.BR mg (1)
+document number, use the sequence `%n'.  To include the document
+weight, use the sequence `%w'.
+.TP
+.BI sizestats " `false'"
+If this is
+.IR true ,
+then various numbers are output at the end of each query indicating
+what went on during the query.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI skip_dump " `skips.%d'"
+If this parameter is set, then a file will be produced in the current
+directory during ranked queries on skipped inverted files when
+.B accumulator_method
+is set to
+.IR splay_tree ,
+.IR hash_table ,
+or
+.IR list .
+The name of the file is the value of this parameter.  A `%d' in the
+file name will be replaced with the process id of
+.BR mgquery .
+This file will contain information about the usage of skips during the
+query processing.  This option is expensive; use
+.B .unset skip_dump
+to obtain optimal performance.
+.TP
+.BI sorted_terms " `on'"
+This specifies whether or not the terms should be sorted into
+decreasing occurrence in documents so that the least-often occurring
+terms are processed first when ranked queries are being done.  When
+this is
+.IR true ,
+the terms are sorted.  When this is
+.IR false ,
+the terms are not sorted, and are instead processed in order of
+occurrence.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI stop_at_max_accum " `on'"
+This specifies what should happen when the maximum number of
+accumulators set by
+.B max_accumulators
+is reached.  When this is
+.IR true ,
+the processing of terms is stopped at the completion of the current
+term.  When this is
+.IR false ,
+processing continues but no new accumulators are created.  This
+parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI terminator " `'"
+This specifies the string that will be output after the last document
+from the previous query has been output.  The standard C escape
+character sequences may be used to place special characters in the
+string.  For example, a newline would be written as `\en'.  To include
+a `%', use the sequence `%%'.
+.TP
+.BI timestats " `false'"
+If this is
+.IR true ,
+then the time to process a query is displayed in both real time and
+CPU time.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI verbatim " `off'"
+This is a Boolean parameter that determines whether the program
+should attempt to do a regular-expression match on the retrieved
+text.  If verbatim is
+.I on
+and a post-processing string is specified with the query, then the
+post-processing string will be searched for in the documents just
+before they are displayed.  If the string is found, the document will
+be displayed; if not, the document will not be displayed.  If verbatim
+is
+.IR off ,
+the post-processing string will be considered a regular expression
+as in
+.BR egrep (1)
+or
+.BR vi (1).
+E.g., if verbatim is
+.I on,
+\*(lq\fBand.*the\fP\*(rq will look for the 8-character sequence
+\*(lq\fBand.*the\fP\*(rq.  If verbatim is
+.IR off ,
+\*(lq\fBand.*the\fP\*(rq will look for the sequence
+\*(lq\fBand\fP\*(rq followed somewhere later in the document by the
+sequence \*(lq\fBthe\fP\*(rq.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.I .mgrc
+.B mgquery
+startup file
+.TP
+.B help.mg
+Help file for
+.BR mgquery .
+The contents of this file is displayed with the
+.B .help
+command.
+.TP
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.dict
+The `on-disk' stemmed dictionary.
+.TP
+.B *.text
+Compressed documents.
+.TP
+.B *.text.dict
+Compression dictionary.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.text.idx.wgt
+Interleaved index into the compressed documents and document weights.
+.TP
+.B *.weight.approx
+Approximate document weights.
+.SH "SEE ALSO"
+.na
+.BR egrep (1),
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1),
+.BR vi (1),
+.BR yacc (1).
Index: /tags/gsdl-21/gsdl/packages/mg/man/man1/mgstat.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/man/man1/mgstat.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/man/man1/mgstat.1	(revision 699)
@@ -0,0 +1,131 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgstat 1 \*(Dt CITRI
+.SH NAME
+mgstat \- print out statistics about a document collection
+.SH SYNOPSIS
+.B mgstat
+[
+.B \-h
+]
+[
+.B \-E
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mgstat
+prints out various statistics about an existing
+.BR mg (1)
+document collection.  Depending on the size of the collection, sizes
+will be printed in either kilobytes or megabytes.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stdout .
+.TP
+.B \-E
+This option forces sizes to be printed in bytes rather than kilobytes
+or megabytes.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.text
+Compressed documents.
+.TP
+.B *.invf
+Inverted file.
+.TP
+.B *.text.idx.wgt
+Interleaved index into the compressed documents and document weights.
+.TP
+.B *.weight.approx
+Approximate document weights.
+.TP
+.B *.invf.dict.blocked
+Compressed stemmed dictionary and index into the inverted file merged
+into an inverted file.
+.TP
+.B *.text.dict.fast
+Fast loading compression dictionary.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.invf.dict
+Compressed stemmed dictionary.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.TP
+.B *.text.stats
+Statistics about the text.
+.TP
+.B *.text.dict.aux
+Auxiliary compression dictionary.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.B *.invf.chunk
+Maps stemmed terms from occurrence order to lexical order.
+.TP
+.B *.invf.chunk.trans
+Describes where the source text is broken up into chunks for the
+inversion pass.
+.TP
+.B *.invf.dict.hash
+A perfect hash function for the terms in the stemmed dictionary.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /tags/gsdl-21/gsdl/packages/mg/man/man1/mgstemidxlist.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/man/man1/mgstemidxlist.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/man/man1/mgstemidxlist.1	(revision 699)
@@ -0,0 +1,86 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgstemidxlist 1 \*(Dt CITRI
+.SH NAME
+mgstemidxlist \- dumps the stem index file to STDOUT in ASCII format
+.SH SYNOPSIS
+.B mgstemidxlist
+[
+.B \-h
+]
+[
+.BI \-d " directory"
+]
+.B \-s 1|2|3
+.BI \-f " name"
+.SH DESCRIPTION
+.B mgstemidxlist
+is a utility program which passes through the stem index and displays
+the information.  (This program was used for debugging but is included
+for completeness!)  The header information is printed out, then for each
+block in the file the block header is displayed.  For each word in the
+stem index, the output is formatted as:
+
+.RS
+.I number_of_references \*(lqword\*(rq
+.br
+.RS
+.I "->  number_of_cases block 3-in-4_block offset"
+.br
+.I "->  ..."
+.RE
+.RE
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B -s 1|2|3
+The stem index file with the associated stem method.
+.br
+1 = casefolded and non-stemmed
+.br
+2 = non-casefolded and stemmed
+.br
+3 = casefolded and stemmed
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 25
+.B *.invf.dict.blocked.1
+Stem index with stem index method 1.
+.TP
+.B *.invf.dict.blocked.2
+Stem index with stem index method 2.
+.TP
+.B *.invf.dict.blocked.3
+Stem index with stem index method 3.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_invf_dict (1),
+.BR mg_stem_idx (1).
Index: /tags/gsdl-21/gsdl/packages/mg/man/man1/mgtic.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/man/man1/mgtic.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/man/man1/mgtic.1	(revision 699)
@@ -0,0 +1,137 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgtic 1 \*(Dt CITRI
+.SH NAME
+mgtic \- textual image compression.
+.SH SYNOPSIS
+.B mgtic
+.B \-e
+[
+.B \-L
+]
+[
+.B \-Q
+]
+[
+.B \-X
+]
+.I libraryfile
+.I infile
+.if n .ti +9n
+[
+.I \>compressed\-outfile
+]
+.nf
+or
+.fi
+.B mgtic
+.B \-d
+[
+.B \-L
+]
+[
+.BI \-X " libraryfile"
+]
+.I compressed-infile
+.if n .ti +9n
+[
+.I \>outfile
+]
+.SH DESCRIPTION
+This is the main routine in the
+.B mgtic
+system.
+.B mgtic
+is used to
+processes a series of files, and by using the
+.I libraryfile
+produced by
+.BR mgticbuild (1),
+compresses input files with respect to the marks contained in the
+library.  On the CCITT test faxes, lossless mode achieved around 20:1
+compression, and lossy, around 40:1 compression.  On more textual
+images, the amount of compression is determined by the resolution of
+the original file, and the amount of noise in the image.  By default,
+.B mgtic
+will reconstruct a lossy version of the image.  This
+can be changed to lossless by using the
+.B \-L
+option.
+.SH OPTIONS
+Options may appear in any order, but must come before the input
+filenames.
+.TP "\w'\fB\-X\fP \fI[libraryfile]\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-e
+Encode
+.I infile
+with respect to the library.
+.TP
+.B \-d
+Decodes
+.IR compressed-infile
+with respect to the library.
+.TP
+.B \-l
+This specifies lossy mode, and is the default mode for (de)compression.
+Do not generate the residue.  If used in encoding mode, does not store
+the residue in the output.  If used in decoding mode, the residue (if
+it exists) is ignored.
+.TP
+.B \-L
+Use lossless mode for (de)compression, which generates the residue.
+If used in encoding mode, this stores the residue in the compressed
+output.  If used in decoding mode (assuming the residue is present),
+this allows the original file to be recovered exactly.
+.TP
+.BI \-R\  filename
+Stores the residue as an external file
+.I filename
+which could possibly be stored on another medium.  During
+decompression, if the residue file specified does not relate to the
+compressed file, the resulting decompressed image will be a mess.
+Note that only the width and height of the residue image is checked.
+.TP
+.BI \-Q
+Uses a quicker form of compression.  This option decreases the time
+spent looking for a good match during encoding.  On average this
+option decreases compression ratios, but increases compression speed.
+.TP
+.BI \-X " [libraryfile]"
+Use an external library file, instead of (de)compressing the library
+within the normal stream.  When decoding, the
+.B \-X
+option must be followed by the
+.IR libraryfile .
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /tags/gsdl-21/gsdl/packages/mg/man/man1/mgticbuild.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/man/man1/mgticbuild.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/man/man1/mgticbuild.1	(revision 699)
@@ -0,0 +1,61 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgticbuild 1 \*(Dt CITRI
+.SH NAME
+mgticbuild \- build a library of marks from training documents
+.SH SYNOPSIS
+.B mgticbuild
+[
+.B \-h
+]
+.I "libraryfile infile1 infile2 .\|.\|."
+.SH DESCRIPTION
+.B mgticbuild
+builds an uncompressed library of marks that have been
+extracted from a series of input files.  If the
+.I libraryfile
+does
+not exist, it is created, otherwise new marks are appended to the
+file.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fIlibraryfile\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.I libraryfile
+This specifies the name of the library file.
+.TP
+.I infile(s)
+The training files; the only currently-supported format is
+.IR pbm .
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /tags/gsdl-21/gsdl/packages/mg/man/man1/mgticdump.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/man/man1/mgticdump.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/man/man1/mgticdump.1	(revision 699)
@@ -0,0 +1,55 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgticdump 1 \*(Dt CITRI
+.SH NAME
+mgticdump \- displays the library in a user-friendly manner
+.SH SYNOPSIS
+.B mgticdump
+[
+.B \-h
+]
+.I libraryfile
+.SH DESCRIPTION
+.B mgticdump
+displays the marks in a library in a user-viewable form.  The output
+is to
+.I stdout
+and is in a plain ASCII form.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fIlibraryfile\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.I libraryfile
+This specifies the name of the library file.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /tags/gsdl-21/gsdl/packages/mg/man/man1/mgticprune.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/man/man1/mgticprune.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/man/man1/mgticprune.1	(revision 699)
@@ -0,0 +1,91 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgticprune 1 \*(Dt CITRI
+.SH NAME
+mgticprune \- prunes the library to remove duplicates
+.SH SYNOPSIS
+.B mgticprune
+[
+.BI \-a " n"
+]
+[
+.BI \-s " n"
+]
+[
+.B \-Q
+]
+.I libraryfile
+.if n .ti +9n
+[
+.I pruned-libraryfile
+]
+.SH DESCRIPTION
+.B mgticprune
+examines
+.I libraryfile
+and removes duplicate marks.  If
+.I pruned-libraryfile
+is not specified,
+.I libraryfile
+is overwritten with the marks in the pruned library.  The Combined
+Symbol Independent Strategy (CSIS) method is used to match marks.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fIpruned-libraryfile\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-a " n"
+Prune marks when their area is under or equal to the threshold,
+.IR n .
+The default value is\ 3.
+.TP
+.BI \-s " n"
+Prune marks when their width and height is under or equal to
+the threshold,
+.IR n .
+The default value is\ 0.
+.TP
+.BI \-Q
+Use a quicker method of pruning.  The number of iterations through the
+averaging and pruning processes is decreased.  This usually reduces
+time by a half, but compression ratios decrease.
+.TP
+.I libraryfile
+This specifies the name of the library file
+.TP
+.I pruned-libraryfile
+This specifies the name of the prune library, after duplicates and
+matches have been removed.  If this file is not specified, then
+.I libraryfile
+will be overwritten.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticstat (1).
Index: /tags/gsdl-21/gsdl/packages/mg/man/man1/mgticstat.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/man/man1/mgticstat.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/man/man1/mgticstat.1	(revision 699)
@@ -0,0 +1,57 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgticstat 1 \*(Dt CITRI
+.SH NAME
+mgticstat \- displays the statistics for the library
+.SH SYNOPSIS
+.B mgticstat
+[
+.B \-h
+]
+.I libraryfile
+.SH DESCRIPTION
+.B mgticstat
+displays statistics about the
+.I libraryfile
+file.
+The output of this program contains the average mark size, average
+centroid position, and the number of marks with an area less than or
+equal to a few powers of\ 2.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fIlibraryfile\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.I libraryfile
+This specifies the name of the library file.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1).
Index: /tags/gsdl-21/gsdl/packages/mg/man/man1/xmg.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/man/man1/xmg.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/man/man1/xmg.1	(revision 699)
@@ -0,0 +1,130 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH xmg 1 \*(Dt CITRI
+.SH NAME
+xmg \- An X interface to mgquery
+.SH SYNOPSIS
+.B xmg
+[
+.I collection-name
+]
+.SH DESCRIPTION
+.B xmg
+is an X interface to the
+.BR mg (1)
+information retrieval system.
+.PP
+Type a query in the ``Query'' box.  The escape key clears the query text.
+.PP
+The first line or so of each matching document is listed.
+.PP
+Click on any line to show the complete document.
+.PP
+Shift-click to show the document in a new window.
+.PP
+Press
+.I Next
+and
+.I Previous
+to cycle through the list of documents (shift works here too).
+.PP
+If you change the query, existing windows continue to refer to the
+original query.
+.PP
+You can select either Boolean or ranked queries:
+.RS
+.TP "\w'\fIBoolean queries:\fP'u+2n"
+.I "Boolean queries:"
+use ``&'' for AND, ``|'' for OR, ``!'' for NOT, and parentheses for
+grouping.
+.TP
+.I "Ranked queries:"
+Documents are listed in order of relevance.  A weight is shown
+alongside each.  Weights are normalized to 100 for the most relevant
+document.  Choose
+.I "Approximate Ranking"
+for greater retrieval speed.
+.RE
+.PP
+In document windows (other than the main window):
+.RS
+.br
+Typing `n' = next document;
+.br
+Typing `N' = next document in new window;
+.br
+Typing `p' = previous document;
+.br
+Typing `P' = previous document in new window;
+.br
+Typing `q' = close window.
+.RE
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.TP
+.SB Xdefaults
+To override the default font options, use RESOURCE_MANAGER properties
+like the following in your
+.I .Xdefaults
+file:
+.IP
+.RS
+.nf
+.I "xmg.textfont: fixed"
+.I "xmg.buttonfont: fixed"
+.fi
+.RE
+.IP
+To override the default query type of ranked, use
+.IP
+.RS
+.I "xmg.querytype: boolean"
+.RE
+.IP
+To override the default collection name (the first one listed by
+.BR ls (1)
+from your MGDATA environment variable), use
+.IP
+.RS
+.I "xmg.collection: mail93"
+.RE
+.SH "SEE ALSO"
+.na
+.BR ls (1),
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /tags/gsdl-21/gsdl/packages/mg/mkinstalldirs
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/mkinstalldirs	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/mkinstalldirs	(revision 699)
@@ -0,0 +1,32 @@
+#! /bin/sh
+# mkinstalldirs --- make directory hierarchy
+# Author: Noah Friedman <friedman@prep.ai.mit.edu>
+# Created: 1993-05-16
+# Last modified: 1994-03-25
+# Public domain
+
+errstatus=0
+
+for file in ${1+"$@"} ; do 
+   set fnord `echo ":$file" | sed -ne 's/^:\//#/;s/^://;s/\// /g;s/^#/\//;p'`
+   shift
+
+   pathcomp=
+   for d in ${1+"$@"} ; do
+     pathcomp="$pathcomp$d"
+     case "$pathcomp" in
+       -* ) pathcomp=./$pathcomp ;;
+     esac
+
+     if test ! -d "$pathcomp"; then
+        echo "mkdir $pathcomp" 1>&2
+        mkdir "$pathcomp" || errstatus=$?
+     fi
+
+     pathcomp="$pathcomp/"
+   done
+done
+
+exit $errstatus
+
+# mkinstalldirs ends here
Index: /tags/gsdl-21/gsdl/packages/mg/src/images/Makefile.in
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/images/Makefile.in	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/images/Makefile.in	(revision 699)
@@ -0,0 +1,216 @@
+###########################################################################
+#
+# Makefile.in -- Makefile for the MG system
+# Copyright (C) 1994  Neil Sharman
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)Makefile.in	1.10 22 Mar 1994
+#
+###########################################################################
+
+PACKAGE = @PACKAGE@
+VERSION = @VERSION@
+ 
+SHELL = /bin/sh
+VPATH = @srcdir@
+exec_prefix = @exec_prefix@
+prefix = @prefix@
+srcdir = @srcdir@
+subdir = src/images
+top_srcdir = @top_srcdir@
+transform = @program_transform_name@
+ 
+CC = @CC@
+CFLAGS = @CFLAGS@ 
+CPPFLAGS = @CPPFLAGS@
+DEFS = @DEFS@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+LDFLAGS = @LDFLAGS@
+LIBS = ../../lib/libmg.a @LIBS@
+TOUCH = echo timestamp >
+ 
+bindir = $(exec_prefix)/bin
+datadir = $(prefix)/share
+libexecdir = $(exec_prefix)/libexec
+localedir = $(datadir)/locale
+mandir = $(prefix)/man/man1
+ 
+COMPILE = $(CC) -c $(DEFS) $(INCLUDES) $(CPPFLAGS) $(CFLAGS)
+LINK = $(CC) $(LDFLAGS) -o $@
+
+.SUFFIXES:
+.SUFFIXES: .c .o
+.c.o:
+	$(COMPILE) $<
+ 
+
+ANSI2KNR = @ANSI2KNR@
+o = .@U@o
+ 
+.SUFFIXES: ._c ._o
+.c._c:
+	./ansi2knr $< > $*.tmp && mv $*.tmp $@
+._c._o:
+	@echo $(COMPILE) $<
+	@rm -f _$*.c
+	@ln $< _$*.c && $(COMPILE) _$*.c && mv _$*.o $@ && rm _$*.c
+.c._o:
+	./ansi2knr $< > $*.tmp && mv $*.tmp $*._c
+	@echo $(COMPILE) $*._c
+	@rm -f _$*.c
+	@ln $*._c _$*.c && $(COMPILE) _$*.c && mv _$*.o $@ && rm _$*.c
+ 
+INCLUDES = -I../../../.. -I../.. -I../../lib -I.
+
+
+SOURCES = \
+  arithcode.c     extractor.c     mgbilevel.c     mgticdump.c     utils.c \
+  bilevel.c       felics.c        mgfelics.c      mgticprune.c    mgticstat.c\
+  codeoffsets.c   marklist.c      mgtic.c         pbmtools.c      lstevent.c \
+  codesyms.c      match.c         mgticbuild.c    sortmarks.c     ppm.c  
+
+
+ 
+HEADERS = \
+  arithcode.h     codeoffsets.h   marklist.h      sortmarks.h \
+  basictypes.h    codesyms.h      match.h         utils.h     \
+  bilevel.h       extractor.h     pbmtools.h      model.h
+
+MISC          = Makefile.in 
+
+ALLFILES      =	$(SOURCES) $(HEADERS) $(MISC) $(MAN)
+
+DISTFILES     = $(ALLFILES) 
+
+EXEC          = mgfelics mgbilevel mgtic mgticbuild mgticprune mgticstat \
+                mgticdump
+
+MAN           = mgfelics.1 mgbilevel.1 mgtic.1 mgticbuild.1  mgticprune.1 \
+                mgticstat.1 mgticdump.1
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+
+all: $(EXEC)
+
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+FELICS_OBJS = felics$o mgfelics$o utils$o
+
+mgfelics: $(FELICS_OBJS)
+	$(LINK) $(FELICS_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+BILEVEL_OBJS = \
+	arithcode$o bilevel$o extractor$o marklist$o mgbilevel$o \
+	pbmtools$o utils$o
+            
+mgbilevel: $(BILEVEL_OBJS)
+	$(LINK) $(BILEVEL_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+TIC_OBJS = \
+	mgtic$o extractor$o marklist$o pbmtools$o utils$o match$o \
+	sortmarks$o codesyms$o ppm$o lstevent$o arithcode$o codeoffsets$o bilevel$o
+
+mgtic: $(TIC_OBJS) 
+	$(LINK) $(TIC_OBJS) $(LIBS) 
+
+TICBUILD_OBJS = mgticbuild$o extractor$o marklist$o pbmtools$o utils$o
+
+mgticbuild: $(TICBUILD_OBJS)
+	$(LINK) $(TICBUILD_OBJS) $(LIBS)
+ 
+TICPRUNE_OBJS = \
+	mgticprune$o extractor$o marklist$o pbmtools$o \
+	utils$o match$o
+
+mgticprune: $(TICPRUNE_OBJS)
+	$(LINK) $(TICPRUNE_OBJS) $(LIBS)
+ 
+TICSTAT_OBJS = mgticstat$o extractor$o marklist$o pbmtools$o utils$o
+
+mgticstat: $(TICSTAT_OBJS)
+	$(LINK) $(TICSTAT_OBJS) $(LIBS) 
+ 
+TICDUMP = mgticdump$o extractor$o marklist$o pbmtools$o utils$o
+
+mgticdump: $(TICDUMP)
+	$(LINK) $(TICDUMP) $(LIBS) 
+ 
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+ansi2knr: ansi2knr.o
+	$(LINK) ansi2knr.o $(LIBS)
+
+install: execinstall maninstall
+uninstall: execuninstall manuninstall
+
+execinstall: $(EXEC)
+	$(top_srcdir)/mkinstalldirs $(bindir); \
+	for name in $(EXEC); do \
+	  $(INSTALL_PROGRAM) $$name \
+	  $(bindir)/`echo $$name | sed '$(transform)'`; \
+	done
+
+execuninstall:
+	for name in $(EXEC); do \
+	 rm -f $(bindir)/`echo $$name | sed '$(transform)'`; \
+	done
+
+maninstall: $(MAN)
+	$(top_srcdir)/mkinstalldirs $(mandir); \
+	for f in $(MAN); do \
+	  $(INSTALL_DATA) $$f $(mandir)/$$f ;\
+	done
+
+manuninstall: $(MAN)
+	for f in $(MAN); do \
+	  rm -f $(mandir)/$$f ;\
+	done
+
+
+mostlyclean:
+	rm -f *$o _*.c _*.o *._c *._o core core.*
+ 
+clean: mostlyclean
+	rm -f $(EXEC)
+ 
+distclean: clean
+	rm -f ansi2knr
+	rm -f Makefile
+ 
+maintainer-clean: distclean
+	@echo "This command is intended only for maintainers to use;"
+	@echo "rebuilding the deleted files may require special tools."
+ 
+dist: $(DISTFILES)
+	for file in $(DISTFILES); do \
+	  ln $(srcdir)/$$file $(distdir) 2> /dev/null \
+	  || cp -p $(srcdir)/$$file $(distdir); \
+	done
+ 
+Makefile: Makefile.in ../../../../config.status
+	cd ../../../.. && CONFIG_FILES=$(subdir)/$@ CONFIG_HEADERS= ./config.status
+ 
+# Tell versions [3.59,3.63) of GNU make not to export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:
Index: /tags/gsdl-21/gsdl/packages/mg/src/images/WIN32.MAK
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/images/WIN32.MAK	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/images/WIN32.MAK	(revision 699)
@@ -0,0 +1,201 @@
+# Generated automatically from Makefile.in by configure.
+###########################################################################
+#
+# Makefile.in -- Makefile for the MG system
+# Copyright (C) 1994  Neil Sharman
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)Makefile.in	1.10 22 Mar 1994
+#
+###########################################################################
+
+#PACKAGE = mg
+#VERSION = 1.2
+ 
+#SHELL = /bin/sh
+#exec_prefix = ${prefix}
+#prefix = /home/rpeeters/mg/mg-1.3.2
+#srcdir = .
+#subdir = src/images
+#top_srcdir = ../..
+#transform = s,x,x,
+ 
+CC = cl
+#CFLAGS = -g -O -Wall 
+CPPFLAGS = 
+DEFS = -DHAVE_CONFIG_H -D__WIN32__ -D_LITTLE_ENDIAN
+#INSTALL = /usr/local/gnu/bin/install -c
+#INSTALL_DATA = ${INSTALL} -m 644
+#INSTALL_PROGRAM = ${INSTALL}
+LDFLAGS = 
+LIBS = ..\..\lib\libmg.lib 
+#TOUCH = echo timestamp >
+ 
+#bindir = $(exec_prefix)/bin
+#datadir = $(prefix)/share
+#libexecdir = $(exec_prefix)/libexec
+#localedir = $(datadir)/locale
+#mandir = $(prefix)/man/man1
+ 
+COMPILE = $(CC) -c $(DEFS) -I. -I..\.. -I..\..\lib
+LINK = $(CC) $(LDFLAGS)
+
+.SUFFIXES:
+.SUFFIXES: .c .obj
+.c.obj:
+	$(COMPILE) $<
+ 
+
+o = .obj
+e = .exe
+ 
+
+SOURCES = \
+  arithcode.c     extractor.c     mgbilevel.c     mgticdump.c     utils.c \
+  bilevel.c       felics.c        mgfelics.c      mgticprune.c    mgticstat.c\
+  codeoffsets.c   marklist.c      mgtic.c         pbmtools.c      lstevent.c \
+  codesyms.c      match.c         mgticbuild.c    sortmarks.c     ppm.c  
+
+
+ 
+HEADERS = \
+  arithcode.h     codeoffsets.h   marklist.h      sortmarks.h \
+  basictypes.h    codesyms.h      match.h         utils.h     \
+  bilevel.h       extractor.h     pbmtools.h      model.h
+
+MISC          = Makefile.in 
+
+ALLFILES      =	$(SOURCES) $(HEADERS) $(MISC) $(MAN)
+
+DISTFILES     = $(ALLFILES) 
+
+EXEC          = mgfelics$e mgbilevel$e mgtic$e mgticbuild$e mgticprune$e mgticstat$e \
+                mgticdump$e
+
+MAN           = mgfelics.1 mgbilevel.1 mgtic.1 mgticbuild.1  mgticprune.1 \
+                mgticstat.1 mgticdump.1
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+
+all: $(EXEC)
+
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+FELICS_OBJS = felics$o mgfelics$o utils$o
+
+mgfelics$e: $(FELICS_OBJS)
+	$(LINK) $(FELICS_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+BILEVEL_OBJS = \
+	arithcode$o bilevel$o extractor$o marklist$o mgbilevel$o \
+	pbmtools$o utils$o
+            
+mgbilevel$e: $(BILEVEL_OBJS)
+	$(LINK) $(BILEVEL_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+TIC_OBJS = \
+	mgtic$o extractor$o marklist$o pbmtools$o utils$o match$o \
+	sortmarks$o codesyms$o ppm$o lstevent$o arithcode$o codeoffsets$o bilevel$o
+
+mgtic$e: $(TIC_OBJS) 
+	$(LINK) $(TIC_OBJS) $(LIBS) 
+
+TICBUILD_OBJS = mgticbuild$o extractor$o marklist$o pbmtools$o utils$o
+
+mgticbuild$e: $(TICBUILD_OBJS)
+	$(LINK) $(TICBUILD_OBJS) $(LIBS)
+ 
+TICPRUNE_OBJS = \
+	mgticprune$o extractor$o marklist$o pbmtools$o \
+	utils$o match$o
+
+mgticprune$e: $(TICPRUNE_OBJS)
+	$(LINK) $(TICPRUNE_OBJS) $(LIBS)
+ 
+TICSTAT_OBJS = mgticstat$o extractor$o marklist$o pbmtools$o utils$o
+
+mgticstat$e: $(TICSTAT_OBJS)
+	$(LINK) $(TICSTAT_OBJS) $(LIBS) 
+ 
+TICDUMP = mgticdump$o extractor$o marklist$o pbmtools$o utils$o
+
+mgticdump$e: $(TICDUMP)
+	$(LINK) $(TICDUMP) $(LIBS) 
+ 
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+#ansi2knr: ansi2knr.o
+#	$(LINK) ansi2knr.o $(LIBS)
+#
+install: execinstall maninstall
+uninstall: execuninstall manuninstall
+
+#execinstall: $(EXEC)
+#	$(top_srcdir)/mkinstalldirs $(bindir); \
+#	for name in $(EXEC); do \
+#	  $(INSTALL_PROGRAM) $$name \
+#	  $(bindir)/`echo $$name | sed '$(transform)'`; \
+#	done
+
+#execuninstall:
+#	for name in $(EXEC); do \
+#	 rm -f $(bindir)/`echo $$name | sed '$(transform)'`; \
+#	done
+
+maninstall: $(MAN)
+	$(top_srcdir)/mkinstalldirs $(mandir); \
+	for f in $(MAN); do \
+	  $(INSTALL_DATA) $$f $(mandir)/$$f ;\
+	done
+
+manuninstall: $(MAN)
+	for f in $(MAN); do \
+	  rm -f $(mandir)/$$f ;\
+	done
+
+
+mostlyclean:
+	rm -f *$o _*.c _*.o *._c *._o core core.*
+ 
+clean: mostlyclean
+	rm -f $(EXEC)
+ 
+distclean: clean
+	rm -f ansi2knr
+	rm -f Makefile
+ 
+maintainer-clean: distclean
+	@echo "This command is intended only for maintainers to use;"
+	@echo "rebuilding the deleted files may require special tools."
+ 
+#dist: $(DISTFILES)
+#	for file in $(DISTFILES); do \
+#	  ln $(srcdir)/$$file $(distdir) 2> /dev/null \
+#	  || cp -p $(srcdir)/$$file $(distdir); \
+#	done
+ 
+#Makefile: Makefile.in ../../config.status
+#	cd ../.. && CONFIG_FILES=$(subdir)/$@ CONFIG_HEADERS= ./config.status
+ 
+# Tell versions [3.59,3.63) of GNU make not to export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+#.NOEXPORT:
Index: /tags/gsdl-21/gsdl/packages/mg/src/images/arithcode.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/images/arithcode.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/images/arithcode.c	(revision 699)
@@ -0,0 +1,471 @@
+/**************************************************************************
+ *
+ * arithcode.c -- Arithmetic coding
+ * Copyright (C) 1994  Alistair Moffat
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ *	Arithmetic encoding. Taken largely from Witten, Neal, Cleary,
+ *		CACM, 1987, p520.
+ *	Includes bitfiddling routines.
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "arithcode.h"
+
+
+FILE *arith_in, *arith_out;
+
+/*
+   $log$
+ */
+
+static char *RCSID = "$Id$";
+
+#undef BECAREFUL
+#define INTEGRATED
+
+
+codevalue S_low = 0, S_high = 0, S_value = 0;
+long S_bitstofollow = 0;
+int S_buffer = 0, S_bitstogo = 0;
+
+long cmpbytes = 0, rawbytes = 0;
+
+#ifdef INTEGRATED
+long CountOfBitsOut;
+#endif
+
+
+/*==================================*/
+
+#define BITPLUSFOLLOW(b)		\
+{	OUTPUTBIT((b));			\
+	while (bitstofollow)		\
+	{	OUTPUTBIT(!(b));	\
+		bitstofollow -= 1;	\
+	}				\
+}
+
+#ifdef INTEGRATED
+#define OUTPUTBIT(b)			\
+{	buffer >>= 1;			\
+	if (b) buffer |= 0x80;	 	\
+	bitstogo -= 1;			\
+	if (bitstogo==0)		\
+	{	putc(buffer, arith_out);	\
+		bitstogo = 8;		\
+		cmpbytes += 1;		\
+	}				\
+	CountOfBitsOut++;		\
+}
+#else
+#define OUTPUTBIT(b)			\
+{	buffer >>= 1;			\
+	if (b) buffer |= 0x80;	 	\
+	bitstogo -= 1;			\
+	if (bitstogo==0)		\
+	{	putc(buffer, arith_out);	\
+		bitstogo = 8;		\
+		cmpbytes += 1;		\
+	}				\
+}
+#endif
+
+#define ADDNEXTINPUTBIT(v)		\
+{	if (bitstogo==0)		\
+	{	buffer = getc(arith_in);	\
+		bitstogo = 8;		\
+	}				\
+	v = (v<<1)+(buffer&1);		\
+	buffer >>= 1;			\
+	bitstogo -=1;			\
+}
+
+/*==================================*/
+
+void
+arithmetic_encode (lbnd, hbnd, totl)
+     int lbnd, hbnd, totl;
+{
+  register codevalue low, high;
+
+#ifdef BECAREFUL
+  if (!(0 <= lbnd && lbnd < hbnd && hbnd <= totl && totl < maxfrequency))
+    {
+      fprintf (stderr, "(corrupt file) coding error: %d %d %d\n", lbnd, hbnd, totl);
+      abort ();
+    }
+#endif
+
+  {
+    register codevalue range;
+    range = S_high - S_low + 1;
+    high = S_low + range * hbnd / totl - 1;
+    low = S_low + range * lbnd / totl;
+  }
+
+  {
+    register codevalue H;
+    register long bitstofollow;
+    register int buffer, bitstogo;
+
+    bitstofollow = S_bitstofollow;
+    buffer = S_buffer;
+    bitstogo = S_bitstogo;
+    H = half;
+
+    for (;;)
+      {
+	if (high < H)
+	  {
+	    BITPLUSFOLLOW (0);
+	  }
+	else if (low >= H)
+	  {
+	    BITPLUSFOLLOW (1);
+	    low -= H;
+	    high -= H;
+	  }
+	else if (low >= firstqtr && high < thirdqtr)
+	  {
+	    bitstofollow++;
+	    low -= firstqtr;
+	    high -= firstqtr;
+	  }
+	else
+	  break;
+	low += low;
+	high += high;
+	high++;
+      }
+    S_bitstofollow = bitstofollow;
+    S_buffer = buffer;
+    S_bitstogo = bitstogo;
+    S_low = low;
+    S_high = high;
+  }
+}
+
+/*==================================*/
+
+/* Made into a macro 
+
+   codevalue
+   arithmetic_decode_target(totl)
+   register int totl;
+   {    return (((S_value-S_low+1)*totl-1) / (S_high -S_low+1));
+   } 
+ */
+
+/*==================================*/
+
+void
+arithmetic_decode (lbnd, hbnd, totl)
+     int lbnd, hbnd, totl;
+{
+  register codevalue low, high;
+
+#ifdef BECAREFUL
+  if (!(0 <= lbnd && lbnd < hbnd && hbnd <= totl && totl < maxfrequency))
+    {
+      fprintf (stderr, "(corrupt file) coding error: %d %d %d\n", lbnd, hbnd, totl);
+      abort ();
+    }
+#endif
+
+  {
+    register codevalue range;
+    range = S_high - S_low + 1;
+    high = S_low + range * hbnd / totl - 1;
+    low = S_low + range * lbnd / totl;
+  }
+
+  {
+    register codevalue value, H;
+    register int buffer, bitstogo;
+
+    buffer = S_buffer;
+    bitstogo = S_bitstogo;
+    H = half;
+    value = S_value;
+
+    for (;;)
+      {
+	if (high < H)
+	  {			/* nothing */
+	  }
+	else if (low >= H)
+	  {
+	    value -= H;
+	    low -= H;
+	    high -= H;
+	  }
+	else if (low >= firstqtr && high < thirdqtr)
+	  {
+	    value -= firstqtr;
+	    low -= firstqtr;
+	    high -= firstqtr;
+	  }
+	else
+	  break;
+	low += low;
+	high += high;
+	high += 1;
+	ADDNEXTINPUTBIT (value);
+      }
+    S_buffer = buffer;
+    S_bitstogo = bitstogo;
+    S_low = low;
+    S_high = high;
+    S_value = value;
+  }
+}
+
+/*==================================*/
+
+
+static void
+startoutputingbits ()
+{
+  S_buffer = 0;
+  S_bitstogo = 8;
+}
+
+static void
+startinputingbits ()
+{
+  S_buffer = 0;
+  S_bitstogo = 0;
+}
+
+static void
+doneoutputingbits ()
+{				/* write another byte if necessary */
+  if (S_bitstogo < 8)
+    {
+      putc (S_buffer >> S_bitstogo, arith_out);
+      cmpbytes += 1;
+    }
+}
+
+/*==================================*/
+
+static void
+startencoding ()
+{
+  S_low = 0;
+  S_high = topvalue;
+  S_bitstofollow = 0;
+}
+
+static void
+startdecoding ()
+{
+  register int i;
+  register int buffer, bitstogo;
+  S_low = 0;
+  S_high = topvalue;
+  S_value = 0;
+  bitstogo = S_bitstogo;
+  buffer = S_buffer;
+  for (i = 0; i < codevaluebits; i++)
+    ADDNEXTINPUTBIT (S_value);
+  S_bitstogo = bitstogo;
+  S_buffer = buffer;
+}
+
+static void
+doneencoding ()
+{
+  register long bitstofollow;
+  register int buffer, bitstogo;
+  bitstofollow = S_bitstofollow;
+  buffer = S_buffer;
+  bitstogo = S_bitstogo;
+  bitstofollow += 1;
+  if (S_low < firstqtr)
+    {
+      BITPLUSFOLLOW (0);
+    }
+  else
+    {
+      BITPLUSFOLLOW (1);
+    }
+  S_bitstofollow = bitstofollow;
+  S_buffer = buffer;
+  S_bitstogo = bitstogo;
+}
+
+
+
+void 
+EncodeGammaDist (int Off)
+/* Encode Off with probability distribution derived from the Elias gamma distribution. */
+{
+  int Len, i;
+
+  Off++;
+  for (Len = 31; Len >= 0 && !(Off & (1 << Len)); Len--);
+
+  for (i = Len; i >= 0; i--)
+    arithmetic_encode (0, 1, 2);
+  arithmetic_encode (1, 2, 2);
+
+  for (i = Len - 1; i >= 0; i--)
+    if (Off & (1 << i))
+      arithmetic_encode (1, 2, 2);
+    else
+      arithmetic_encode (0, 1, 2);
+}
+
+
+
+int 
+DecodeGammaDist ()
+/* Decode next Off using probability distribution derived from the Elias gamma distribution. Inverse of EncodeGammaDist. */
+{
+  int i = 1, Len = 0, bit;
+
+  do
+    {
+      bit = arithmetic_decode_target (2) < 1 ? 0 : 1;
+      if (bit)
+	arithmetic_decode (1, 2, 2);
+      else
+	{
+	  arithmetic_decode (0, 1, 2);
+	  Len++;
+	}
+    }
+  while (!bit);
+
+  Len--;
+  while (Len-- > 0)
+    {
+      bit = arithmetic_decode_target (2) < 1 ? 0 : 1;
+      if (bit)
+	arithmetic_decode (1, 2, 2);
+      else
+	arithmetic_decode (0, 1, 2);
+      i = (i << 1) | bit;
+    }
+  return (i - 1);
+}
+
+
+
+
+
+void 
+EncodeGammaSigned (int snum, int *pos, int *neg)
+{
+  if (snum >= 0)
+    arithmetic_encode (0, *pos, *pos + (*neg)), (*pos)++;
+  else
+    arithmetic_encode (*pos, *pos + (*neg), *pos + (*neg)), (*neg)++;
+  if (*pos + (*neg) >= maxfrequency)
+    {
+      *pos = (*pos + 1) / 2;
+      *neg = (*neg + 1) / 2;
+    }
+  EncodeGammaDist (abs (snum));
+}
+
+
+int 
+DecodeGammaSigned (int *pos, int *neg)
+{
+  int signedpos = 1;
+  if (arithmetic_decode_target (*pos + (*neg)) < *pos)
+    arithmetic_decode (0, *pos, *pos + (*neg)), (*pos)++;
+  else
+    arithmetic_decode (*pos, *pos + (*neg), *pos + (*neg)), (*neg)++, signedpos = 0;
+
+  if (*pos + (*neg) >= maxfrequency)
+    {
+      *pos = (*pos + 1) / 2;
+      *neg = (*neg + 1) / 2;
+    }
+
+  if (signedpos)
+    return DecodeGammaDist ();
+  else
+    return (-DecodeGammaDist ());
+}
+
+
+void 
+InitArithEncoding (void)
+{
+  S_low = 0, S_high = 0, S_value = 0;
+  S_bitstofollow = 0;
+  S_buffer = 0, S_bitstogo = 0;
+  cmpbytes = 0, rawbytes = 0;
+
+  startoutputingbits ();
+  startencoding ();
+}
+
+void 
+InitArithDecoding (void)
+{
+  S_low = 0, S_high = 0, S_value = 0;
+  S_bitstofollow = 0;
+  S_buffer = 0, S_bitstogo = 0;
+  cmpbytes = 0, rawbytes = 0;
+
+  startinputingbits ();
+  startdecoding ();
+}
+
+void 
+CloseDownArithEncoding (void)
+{
+  doneencoding ();
+  doneoutputingbits ();
+}
+
+void 
+CloseDownArithDecoding (void)
+{
+}
+
+
+
+
+
+/* encode that famous number, 42 uses 12 bits */
+void 
+EncodeChecksum ()
+{
+  EncodeGammaDist (42);
+}
+
+void 
+DecodeChecksum (char str[])
+{
+  if (DecodeGammaDist () != 42)
+    {
+      fprintf (stderr, "%s: checksum error, file corrupt.\n", str);
+      exit (1);
+    }
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/images/arithcode.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/images/arithcode.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/images/arithcode.h	(revision 699)
@@ -0,0 +1,78 @@
+/**************************************************************************
+ *
+ * arithcode.h -- Arithmetic coding
+ * Copyright (C) 1994  Alistair Moffat
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef _arithcode_h
+#define _arithcode_h
+
+extern FILE *arith_in, *arith_out;
+
+#define codevaluebits 16
+#if (codevaluebits<16)
+typedef unsigned short codevalue;
+#else
+typedef unsigned long codevalue;
+#endif
+
+#define topvalue ((codevalue)((1<<codevaluebits)-1))
+#define maxfrequency ((topvalue+1)/4 + 1)
+
+#define firstqtr (topvalue/4+1)
+#define half     (2*firstqtr)
+#define thirdqtr (3*firstqtr)
+
+#define	escape_event	U->totalcnt-U->notfound, U->totalcnt, U->totalcnt
+
+void arithmetic_encode ();
+codevalue arithmetic_decode_target ();
+
+#define	arithmetic_decode_target(totl)	\
+	(  ((S_value-S_low+1)*(totl)-1) / (S_high-S_low+1)  )
+
+void arithmetic_decode ();
+
+
+extern codevalue S_low, S_high, S_value;
+extern long S_bitstofollow;
+extern int S_buffer, S_bitstogo;
+
+extern long cmpbytes, rawbytes;
+
+extern long CountOfBitsOut;
+
+void EncodeGammaDist (int Off);
+int DecodeGammaDist ();
+
+void EncodeGammaSigned (int snum, int *pos, int *neg);
+int DecodeGammaSigned (int *pos, int *neg);
+
+
+void InitArithEncoding (void);
+void InitArithDecoding (void);
+
+void CloseDownArithEncoding (void);
+void CloseDownArithDecoding (void);
+
+void EncodeChecksum ();
+void DecodeChecksum (char str[]);
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/src/images/basictypes.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/images/basictypes.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/images/basictypes.h	(revision 699)
@@ -0,0 +1,34 @@
+/**************************************************************************
+ *
+ * basictypes.h -- base data types
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ * This file contains information relating to the base data types
+ *
+ **************************************************************************/
+
+#ifndef __basictypes_h
+#define __basictypes_h
+
+typedef unsigned long Pixel;
+
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/src/images/bilevel.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/images/bilevel.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/images/bilevel.c	(revision 699)
@@ -0,0 +1,747 @@
+/**************************************************************************
+ *
+ * bilevel.c -- Compress a bilevel bitmap
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ *  
+ *  
+ *  This is the code used to compress a bitmap.  The template is given in
+ *  the following format. 
+ *   '.'  = pixel not used
+ *   'p'  = pixel used in 1st order context (the superset)
+ *   '2'  = pixel used in 2nd order context (the subset)
+ *   ';'  = marker used to denote the end of a row in the template
+ *   '*'  = the current context pixel.
+ *  for instance, the common 22/10 template would be shown as:
+ *     ".ppppp.;"
+ *     "pp222pp;"
+ *     "p22222p;"
+ *     "p22*...;"
+ * 
+ **************************************************************************/
+
+
+#include "sysfuncs.h"
+
+#include "bilevel.h"
+#include "extractor.h"
+#include "pbmtools.h"
+#include "marklist.h"
+#include "arithcode.h"
+#include "utils.h"
+
+
+
+#define INITVALUE 2
+#define INCREMENT 4
+
+#define MAX_T_W 32
+
+/* note memory usage is 2*TEMPLATESIZE*sizeof(prob), which is currently 4.9 Mb */
+#define TEMPLATESIZE 612317
+
+
+#define HASH(x) ((x)%TEMPLATESIZE)
+
+
+typedef struct
+  {
+    unsigned short s, n;
+  }
+prob;				/* set and notset */
+typedef struct
+  {
+    short int x, y;
+  }
+coordinate;
+
+coordinate MM[MAX_T_W][MAX_T_W];	/* the actual (x,y) positions */
+unsigned int MMcount[MAX_T_W];	/* number of bits in this row of the template */
+unsigned int MMmask[MAX_T_W];	/* the "current" mask value */
+unsigned int MMbits[MAX_T_W];	/* the bitmask to mask out higher bits */
+unsigned int MMrows;		/* total number of rows */
+unsigned int MM2level[MAX_T_W];	/* bitmask to calculate the 2nd level mask */
+int twolevel, curr_row, surr;
+
+prob *P = NULL;			/* probability array, up to N bits */
+prob *P2 = NULL;
+
+void 
+bl_clearmodel ()
+{
+  int i;
+
+  P = (prob *) malloc (TEMPLATESIZE * sizeof (prob));
+  if (!P)
+    error_msg ("bl_clearmodel()", "not enough memory.", "");
+
+  P2 = (prob *) malloc (TEMPLATESIZE * sizeof (prob));
+  if (!P2)
+    error_msg ("bl_clearmodel()", "not enough memory.", "");
+
+  for (i = 0; i < TEMPLATESIZE; i++)
+    {
+      P[i].s = P[i].n = INITVALUE;
+      P2[i].s = P2[i].n = INITVALUE;
+    }
+}
+
+
+void 
+bl_freemodel ()
+{
+  free (P);
+  free (P2);
+}
+
+
+
+
+#define update_counts(p, pixel)                        \
+    pixel ? (p->s+=INCREMENT) : (p->n+=INCREMENT);     \
+                                                       \
+    if(p->s+p->n >= (unsigned)(maxfrequency)) {        \
+       p->s=(p->s+1)/(unsigned)2;                      \
+       p->n=(p->n+1)/(unsigned)2;                      \
+    }
+
+
+
+
+/* mask is the bigger one, mask2l is smaller one */
+static void 
+bl_encode_in_context (unsigned long mask, unsigned long mask2l, unsigned long pixel)
+{
+  register int twol, t;
+  prob *p = &P[mask];
+  prob *p2 = &P2[mask2l];
+
+  twol = twolevel && (p->s + p->n < (unsigned) (2 * INITVALUE + 5 * INCREMENT));
+
+  t = twol ? (p2->s + p2->n) : (p->s + p->n);
+
+  if (twol == 0)
+    {
+      if (pixel)
+	arithmetic_encode (0, p->s, t);
+      else
+	arithmetic_encode (p->s, t, t);
+      update_counts (p, pixel);
+    }
+  else
+    {
+      if (pixel)
+	arithmetic_encode (0, p2->s, t);
+      else
+	arithmetic_encode (p2->s, t, t);
+      update_counts (p2, pixel);
+      update_counts (p, pixel);
+    }
+}
+
+
+static void 
+bl_decode_in_context (unsigned long mask, unsigned long mask2l, unsigned long *val)
+{
+  register int twol, pixel, t;
+  prob *p = &P[mask];
+  prob *p2 = &P2[mask2l];
+
+
+  if (twolevel && (p->s + p->n < (unsigned) (2 * INITVALUE + 5 * INCREMENT)))
+    {
+      pixel = arithmetic_decode_target (p2->s + p2->n) < p2->s, twol = 1;
+    }
+  else
+    {
+      pixel = arithmetic_decode_target (p->s + p->n) < p->s, twol = 0;
+    }
+
+  t = twol ? (p2->s + p2->n) : (p->s + p->n);
+
+  if (twol == 0)
+    {
+      if (pixel)
+	arithmetic_decode (0, p->s, t);
+      else
+	arithmetic_decode (p->s, t, t);
+      update_counts (p, pixel);
+    }
+  else
+    {
+      if (pixel)
+	arithmetic_decode (0, p2->s, t);
+      else
+	arithmetic_decode (p2->s, t, t);
+      update_counts (p2, pixel);
+      update_counts (p, pixel);
+    }
+
+  *val = pixel;
+}
+
+static void 
+bl_string_to_array (char inputstr[], char parsestr[], int *w, int *h)
+{
+  int r, c;
+  int i, j, lastc, p, l;
+  int xc, yc;
+  int foundstatus;
+
+  surr = twolevel = 0;
+  for (i = 0; i < MAX_T_W; i++)
+    MMcount[i] = MM2level[i] = 0;
+  MMrows = 0;
+
+  for (i = 0; i < (int) strlen (inputstr); i++)
+    {
+      p = inputstr[i];
+      if ((p != '.') && (p != '2') && (p != '*') && (p != 'p') && (p != ';'))
+	{
+	  fprintf (stderr, "Template string can only contain [.p2*;], not \"%c\".\n", inputstr[i]);
+	  exit (1);
+	}
+    }
+
+  /* get the width and height, test rectangularity... */
+  parsestr[l = 0] = 0;
+  for (c = 0, r = 0, lastc = -1, p = 0; p < (int) strlen (inputstr); p++)
+    {
+      if ((inputstr[p] == ';'))
+	{
+	  r++;
+	  if (lastc != -1)
+	    {			/* this is the 2nd row at least */
+	      if (lastc != c)
+		{
+		  fprintf (stderr, "The rows have to be the same width. "
+			   "Row %d has width %d (previous row has width %d)\n", r, c, lastc);
+		  exit (1);
+		}
+	    }
+	  lastc = c;
+	  c = 0;
+	}
+      else
+	{
+	  c++;
+	  parsestr[l] = inputstr[p];
+	  parsestr[++l] = 0;
+	}
+    }
+
+  if (c != 0)
+    lastc = c;
+  *w = lastc;
+  *h = r;
+  if ((*w) <= 0)
+    {
+      fprintf (stderr, "Must have at least one column!\n");
+      exit (1);
+    }
+  if ((*h) <= 0)
+    {
+      fprintf (stderr, "Must have at least one row! Terminate each row with a ';'\n");
+      exit (1);
+    }
+  if ((*w > MAX_T_W) || (*h > MAX_T_W))
+    {
+      fprintf (stderr, "template size bigger than internal threshold of %d\n", MAX_T_W);
+      exit (1);
+    }
+
+  /* get the (x,y) position of the current (*) pixel */
+  for (xc = -1, yc = 0, i = 0; i < *w; i++)
+    for (j = 0; j < *h; j++)
+      if (parsestr[j * (*w) + i] == '*')
+	{
+	  if (xc >= 0)
+	    {
+	      fprintf (stderr, "There can only be ONE current ('*') pixel.\n");
+	      exit (1);
+	    }
+	  xc = i;
+	  yc = j;
+	  curr_row = j + 1;
+	}
+  if (xc < 0)
+    error_msg ("bl_string_to_array", "Error in mask, no current ('*') pixel.", "");
+  /* found the position (xc,yc) */
+
+
+  /* convert parsestr[] into a MM[][] structure */
+  for (j = 0; j < *h; j++)
+    {
+      foundstatus = 0;
+      for (i = 0; i < *w; i++)
+	{
+	  p = j * (*w) + i;
+	  if ((parsestr[p] == 'p') || (parsestr[p] == '2'))
+	    {
+	      if (!foundstatus)
+		foundstatus = 1;
+	      MM2level[j] = MM2level[j] * 2 | (parsestr[p] == '2');
+	      if (parsestr[p] == '2')
+		twolevel = 1;
+
+	      MM[j][MMcount[j]].x = i - xc;
+	      MM[j][MMcount[j]++].y = j - yc;
+	      if (((i - xc) >= 0) && ((j - yc) >= 0))
+		surr = 1;
+	    }
+	  else
+	    {
+	      if (foundstatus == 1)
+		foundstatus = 2;
+	    }
+
+	}
+      if (foundstatus)
+	MMrows++;
+      MMbits[j] = (1 << MMcount[j]) - 1;
+    }
+
+  /* reverse matrix, so [][0] is the closest to the right */
+  {
+    coordinate temp;
+    for (i = 0; i < MMrows; i++)
+      for (j = 0; j < MMcount[i] / (unsigned) 2; j++)
+	{
+	  temp = MM[i][MMcount[i] - 1 - j];
+	  MM[i][MMcount[i] - 1 - j] = MM[i][j];
+	  MM[i][j] = temp;
+	}
+  }
+}
+
+
+
+
+
+
+
+
+void 
+bl_writetemplate (char inputstr[])
+{
+  register int i, j;
+  int w, h;
+  char ind[255];
+  char *str;
+
+  str = (char *) malloc (strlen (inputstr) * sizeof (char) + 2);
+
+  bl_string_to_array (inputstr, str, &w, &h);
+
+  EncodeGammaDist (surr);
+  EncodeGammaDist (w);
+  EncodeGammaDist (h);
+  ind['.'] = 0;
+  ind['p'] = 1;
+  ind['2'] = 2;
+  ind['*'] = 3;
+  for (j = 0; j < h; j++)
+    for (i = 0; i < w; i++)
+      EncodeGammaDist (ind[(int) str[j * w + i]]);
+
+  free (str);
+}
+
+
+void 
+bl_readtemplate ()
+{
+  register int i, j;
+  int w, h, p;
+  char ind[4];
+  char *inputstr, *str;
+
+  surr = DecodeGammaDist (surr);
+  if (surr)
+    {
+      fprintf (stderr, "It's fine compressing an image using a 'surrounding' template, but I can't "
+	       "decompress it!\n");
+      exit (1);
+    }
+  w = DecodeGammaDist ();
+  h = DecodeGammaDist ();
+
+  inputstr = (char *) malloc ((w + 2) * h * sizeof (char) + 2);
+  ind[0] = '.';
+  ind[1] = 'p';
+  ind[2] = '2';
+  ind[3] = '*';
+  for (p = 0, j = 0; j < h; j++)
+    {
+      for (i = 0; i < w; i++)
+	{
+	  inputstr[p++] = ind[DecodeGammaDist ()];
+	}
+      inputstr[p++] = ';';
+      inputstr[p] = 0;
+    }
+
+  str = (char *) malloc (strlen (inputstr) * sizeof (char) + 2);
+  bl_string_to_array (inputstr, str, &w, &h);
+}
+
+
+
+
+
+/* Currently if there are more than 32 bits, the topmost 32 will be discarded, if the 
+   topbits are to be used in compression, define _MORETHAN32 */
+#undef _MORETHAN32
+
+void 
+bl_compress_mark (marktype d)
+{
+  register int r, c;
+  register int i, j;
+  register unsigned long mask, mask2level;
+  char *array[MAX_T_W];
+
+
+  for (i = 0; i < MAX_T_W; i++)
+    {
+      array[i] = (char *) calloc ((size_t) d.w + 2 * MAX_T_W + 1, (size_t) sizeof (char));
+      array[i] = array[i] + MAX_T_W;
+    }
+
+  EncodeGammaDist (d.w);
+  EncodeGammaDist (d.h);
+
+  for (r = 0; r < min (curr_row, d.h); r++)
+    for (c = 0; c < d.w; c++)
+      array[r % MAX_T_W][c] = pbm_getpixel (d.bitmap, c, r);
+
+  for (r = 0; r < d.h; r++)
+    {
+      if (V)
+	if (r && (r % 200 == 0))
+	  fprintf (stderr, ".");
+
+      if (r + curr_row < d.h)
+	for (c = 0; c < d.w; c++)
+	  array[(r + curr_row) % MAX_T_W][c] = pbm_getpixel (d.bitmap, c, r + curr_row);
+      else
+	for (c = 0; c < d.w; c++)
+	  array[(r + curr_row) % MAX_T_W][c] = 0;
+
+      for (c = 0; c < d.w; c++)
+	{
+	  for (mask = 0, mask2level = 0, i = 0; i < MMrows; i++)
+	    {
+	      if (c != 0)
+		{
+		  MMmask[i] = (MMmask[i] * 2);
+		  MMmask[i] |= array[(r + MM[i][0].y + MAX_T_W) % MAX_T_W][c + MM[i][0].x];
+		}
+	      else
+		{
+		  for (MMmask[i] = 0, j = MMcount[i] - 1; j >= 0; j--)
+		    MMmask[i] = 2 * MMmask[i] | array[(r + MM[i][j].y + MAX_T_W) % MAX_T_W][c + MM[i][j].x];
+		}
+	      MMmask[i] &= MMbits[i];
+#ifdef _MORETHAN32
+	      mask = ((mask * 8191) << MMcount[i]) | MMmask[i];
+	      mask2level = ((mask2level * 499) << MMcount[i]) | (MMmask[i] & MM2level[i]);
+#else
+	      mask = (mask << MMcount[i]) | MMmask[i];
+	      mask2level = (mask2level << MMcount[i]) | (MMmask[i] & MM2level[i]);
+#endif
+	    }
+
+	  mask = HASH (mask);
+	  mask2level = HASH (mask2level);
+
+	  bl_encode_in_context (mask, mask2level, pbm_getpixel (d.bitmap, c, r));
+	}
+    }
+  for (i = 0; i < MAX_T_W; i++)
+    free (array[i] - MAX_T_W);
+}
+
+
+
+void 
+bl_decompress_mark (marktype * d)
+{
+  register int r, c;
+  register int i, j;
+  unsigned long pix;
+  register unsigned long mask, mask2level;
+  char *array[MAX_T_W];
+
+
+  d->w = DecodeGammaDist ();
+  d->h = DecodeGammaDist ();
+
+  d->bitmap = pbm_allocarray (d->w, d->h);
+  for (i = 0; i < MAX_T_W; i++)
+    {
+      array[i] = (char *) calloc ((size_t) d->w + 2 * MAX_T_W + 1, (size_t) sizeof (char));
+      array[i] = array[i] + MAX_T_W;
+    }
+
+  for (r = 0; r < d->h; r++)
+    {
+      if (V)
+	if (r && (r % 200 == 0))
+	  fprintf (stderr, ".");
+      for (c = 0; c < d->w; c++)
+	array[r % MAX_T_W][c] = pbm_getpixel (d->bitmap, c, r);
+      for (c = 0; c < d->w; c++)
+	{
+	  for (mask = 0, mask2level = 0, i = 0; i < MMrows; i++)
+	    {
+	      if (c != 0)
+		{
+		  MMmask[i] = (MMmask[i] * 2);
+		  MMmask[i] |= array[(r + MM[i][0].y + MAX_T_W) % MAX_T_W][c + MM[i][0].x];
+		}
+	      else
+		{
+		  for (MMmask[i] = 0, j = MMcount[i] - 1; j >= 0; j--)	/* get all pixels */
+		    MMmask[i] = 2 * MMmask[i] | array[(r + MM[i][j].y + MAX_T_W) % MAX_T_W][c + MM[i][j].x];
+		}
+	      MMmask[i] &= MMbits[i];
+
+#ifdef _MORETHAN32
+	      mask = ((mask * 8191) << MMcount[i]) | MMmask[i];
+	      mask2level = ((mask2level * 499) << MMcount[i]) | (MMmask[i] & MM2level[i]);
+#else
+	      mask = (mask << MMcount[i]) | MMmask[i];
+	      mask2level = (mask2level << MMcount[i]) | (MMmask[i] & MM2level[i]);
+#endif
+	    }
+
+	  mask = HASH (mask);
+	  mask2level = HASH (mask2level);
+
+	  bl_decode_in_context (mask, mask2level, &pix);
+	  if (pix)
+	    {
+	      pbm_putpixel (d->bitmap, c, r, pix);
+	      array[r % MAX_T_W][c] = pix;
+	    }
+	}
+    }
+  for (i = 0; i < MAX_T_W; i++)
+    free (array[i] - MAX_T_W);
+}
+
+
+
+void 
+bl_compress (marktype d, char inputstr[])
+{
+  bl_clearmodel ();
+  bl_writetemplate (inputstr);
+  bl_compress_mark (d);
+  bl_freemodel ();
+}
+
+
+void 
+bl_decompress (marktype * d)
+{
+  bl_clearmodel ();
+  bl_readtemplate ();
+  bl_decompress_mark (d);
+  bl_freemodel ();
+}
+
+
+
+
+
+
+/* 
+   The bitmap in d2, can be examined clairvoyantly.  The ordering of the
+   params is therefore non-clairvoyant, and clairvoyant in that order.
+
+ */
+/*
+   coordinate M1[6]={        { 0,-2},
+   {-1,-1},{ 0,-1}, {1,-1},
+   {-2,0},  {-1, 0}};
+   };
+   coordinate M2[13]={       { 0,-2},
+   {-1,-1}, { 0,-1},   {1, -1},
+   {-2,0}, {-1, 0}, { 0, 0},   { 1, 0},  {2,0},
+   {-1, 1}, { 0, 1},   { 1, 1},
+   { 0, 2}
+ */
+
+coordinate M1[6] =
+{
+  {0, -2},
+  {-2, 0},
+  {1, -1},
+  {-1, -1},
+  {0, -1},
+  {-1, 0}};
+coordinate M2[13] =
+{
+  {0, -2},
+  {1, -1},
+  {-2, 0},
+  {2, 0},
+  {-1, 1},
+  {1, 1},
+  {0, 2},
+  {-1, -1},
+  {0, -1},
+  {1, 0},
+  {-1, 0},
+  {0, 1},
+  {0, 0}
+};
+#define M1S 6
+#define M1S_2level 3
+#define M2S 13
+#define M2S_2level 6
+
+
+void 
+bl_clair_compress (marktype d1, marktype d2)
+{
+  register int r, c;
+  register int i, j, n;
+  register unsigned long mask, mask2level = 0;
+  int cols, rows;
+  int gap;
+  Pixel **b1, **b2;
+
+
+  bl_clearmodel ();
+  twolevel = 1;
+
+  b1 = d1.bitmap;
+  b2 = d2.bitmap;
+  cols = d1.w;
+  rows = d1.h;
+
+  gap = max (50, rows / 10);
+
+  EncodeGammaDist (cols);
+  EncodeGammaDist (rows);
+
+  for (r = 0; r < rows; r++)
+    {
+      if (V)
+	if (!(r % gap))
+	  fprintf (stderr, ".");
+      for (c = 0; c < cols; c++)
+	{
+
+	  mask = 0;
+	  for (n = 0; n < M1S; n++)
+	    {
+	      i = c + M1[n].x;
+	      j = r + M1[n].y;
+	      mask = (2 * mask) | pbm_getpixel_trunc (b1, i, j, cols, rows);
+	    }
+
+	  mask2level = mask & ((1 << M1S_2level) - 1);
+
+	  for (n = 0; n < M2S; n++)
+	    {
+	      i = c + M2[n].x;
+	      j = r + M2[n].y;
+	      mask = (2 * mask) | pbm_getpixel_trunc (b2, i, j, cols, rows);
+	    }
+
+	  mask2level = (mask2level << M2S_2level) | (mask & ((1 << M2S_2level) - 1));
+
+	  bl_encode_in_context (mask, mask2level, pbm_getpixel (b1, c, r));
+
+	}
+    }
+  if (V)
+    fprintf (stderr, "\n");
+  bl_freemodel ();
+}
+
+
+void 
+bl_clair_decompress (marktype d1, marktype d2)
+{
+  register int r, c;
+  register int i, j, n;
+  register unsigned long mask, mask2level = 0;
+  unsigned long p;
+  int cols, rows;
+  int gap;
+  Pixel **b1, **b2;
+
+  bl_clearmodel ();
+  twolevel = 1;
+
+  b1 = d1.bitmap;
+  b2 = d2.bitmap;
+  cols = d1.w;
+  rows = d1.h;
+
+  gap = max (50, rows / 10);
+
+  i = DecodeGammaDist ();
+  j = DecodeGammaDist ();
+  if ((i != cols) || (j != rows))
+    error_msg ("bl_clair_decompress", "incorrect size of residue image", "");
+
+  if (V)
+    fprintf (stderr, "cols=%d rows=%d\n", cols, rows);
+
+  for (r = 0; r < rows; r++)
+    {
+      if (V)
+	if (!(r % gap))
+	  fprintf (stderr, ".");
+      for (c = 0; c < cols; c++)
+	{
+	  mask = 0;
+	  for (n = 0; n < M1S; n++)
+	    {
+	      i = c + M1[n].x;
+	      j = r + M1[n].y;
+	      mask = (2 * mask) | pbm_getpixel_trunc (b1, i, j, cols, rows);
+	    }
+	  mask2level = mask & ((1 << M1S_2level) - 1);
+	  for (n = 0; n < M2S; n++)
+	    {
+	      i = c + M2[n].x;
+	      j = r + M2[n].y;
+	      mask = (2 * mask) | pbm_getpixel_trunc (b2, i, j, cols, rows);
+	    }
+	  mask2level = (mask2level << M2S_2level) | (mask & ((1 << M2S_2level) - 1));
+
+
+	  bl_decode_in_context (mask, mask2level, &p);
+	  if (p)
+	    pbm_putpixel (b1, c, r, 1);
+	}
+    }
+  if (V)
+    fprintf (stderr, "\n");
+  bl_freemodel ();
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/images/bilevel.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/images/bilevel.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/images/bilevel.h	(revision 699)
@@ -0,0 +1,47 @@
+/**************************************************************************
+ *
+ * bilevel.h -- Compress a bilevel bitmap
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef __BILEVEL_H
+#define __BILEVEL_H
+
+#include "marklist.h"
+
+/* must be called before the start of de/compression */
+void bl_clearmodel ();
+void bl_freemodel ();
+
+/* writes the template to the stream, and sets up the de/compression params */
+void bl_writetemplate (char inputstr[]);
+void bl_readtemplate ();
+
+/* actually performs the de/compression */
+void bl_compress (marktype d, char str[]);
+void bl_compress_mark (marktype d);
+void bl_decompress (marktype * d);
+void bl_decompress_mark (marktype * d);
+
+/* clairvoyantly de/compress d1 with respect to clairvoyantly viewable d2 */
+void bl_clair_compress (marktype d1, marktype d2);
+void bl_clair_decompress (marktype d1, marktype d2);
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/src/images/codeoffsets.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/images/codeoffsets.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/images/codeoffsets.c	(revision 699)
@@ -0,0 +1,257 @@
+/**************************************************************************
+ *
+ * codeoffsets.c -- Functions which code the mark offsets
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "marklist.h"
+#include "arithcode.h"
+#include "utils.h"
+
+#define TOP_INC 2
+#define MID_INC 1
+#define BOT_INC 1
+
+
+void 
+EncodeOffsets (marklistptr symbol_list, int nsyms)
+{
+  int xpos = 1, xneg = 1, ypos = 1, yneg = 1;
+  marklistptr step;
+  int found;
+  int dox = 0, leveltop = 1, levelmid = 1, levelbot = 1;
+  elemlist xlist, ylist;	/* all previous offsets */
+  int p, f, t, xoff, yoff, symnum, count;
+  elemlist *xcond, *ycond;	/* previous offsets conditioned on symbol num */
+
+  xcond = (elemlist *) malloc (nsyms * sizeof (elemlist));
+  if (!xcond)
+    error_msg ("EncodeOffsets", "Out of memory...", "");
+  ycond = (elemlist *) malloc (nsyms * sizeof (elemlist));
+  if (!ycond)
+    error_msg ("EncodeOffsets", "Out of memory...", "");
+
+  for (t = 0; t < nsyms; t++)
+    {
+      xcond[t].head = ycond[t].head = NULL;
+      xcond[t].tot = ycond[t].tot = 0;
+    }
+  xlist.head = NULL;
+  xlist.tot = 0;
+  ylist.head = NULL;
+  ylist.tot = 0;
+  for (step = symbol_list, count = 0; step; step = step->next, count++)
+    {
+      xoff = step->data.xoffset;
+      yoff = step->data.yoffset;
+      symnum = step->data.symnum;
+      for (dox = 1; dox >= 0; dox--)
+	{
+	  /* encode the offsets, x first then y */
+
+	  if (dox)
+	    found = elem_find (&xcond[symnum], xoff, &p, &f, &t);
+	  else
+	    found = elem_find (&ycond[symnum], yoff, &p, &f, &t);
+
+	  if (found)
+	    {
+	      arithmetic_encode (0, leveltop, leveltop + levelmid + levelbot);
+	      leveltop += TOP_INC;
+	      arithmetic_encode (p, f, t);
+	    }
+	  else
+	    {
+	      if (dox)
+		found = elem_find (&xlist, xoff, &p, &f, &t);
+	      else
+		found = elem_find (&ylist, yoff, &p, &f, &t);
+
+	      if (found)
+		{
+		  arithmetic_encode (leveltop, leveltop + levelmid, leveltop + levelmid + levelbot);
+		  levelmid += MID_INC;
+		  arithmetic_encode (p, f, t);
+		}
+	      else
+		{
+		  arithmetic_encode (leveltop + levelmid, leveltop + levelmid + levelbot, leveltop + levelmid + levelbot);
+		  levelbot += BOT_INC;
+		  if (dox)
+		    EncodeGammaSigned (xoff, &xpos, &xneg);
+		  else
+		    EncodeGammaSigned (yoff, &ypos, &yneg);
+		}
+	    }
+	  if (dox)
+	    {
+	      elem_add (&xlist, xoff);
+	      elem_add (&xcond[symnum], xoff);
+	    }
+	  else
+	    {
+	      elem_add (&ylist, yoff);
+	      elem_add (&ycond[symnum], yoff);
+	    }
+
+	  if (leveltop + levelmid >= maxfrequency)
+	    {
+	      leveltop = (leveltop + 1) / 2;
+	      levelmid = (levelmid + 1) / 2;
+	      levelbot = (levelbot + 1) / 2;
+	    }
+	}
+    }
+  for (t = 0; t < nsyms; t++)
+    {
+      elem_free (&xcond[t]);
+      elem_free (&ycond[t]);
+    }
+  elem_free (&xlist);
+  elem_free (&ylist);
+  if (V)
+    fprintf (stderr, "offset models: top %d, mid %d, bot %d\n", (leveltop - 1) / TOP_INC, (levelmid - 1) / MID_INC, (levelbot - 1) / BOT_INC);
+}
+
+
+
+
+
+void 
+DecodeOffsets (marklistptr symbol_list, int nsyms)
+{
+  int xpos = 1, xneg = 1, ypos = 1, yneg = 1;
+  marklistptr step;
+  int i, target, dox;
+  int leveltop = 1, levelmid = 1, levelbot = 1;
+  elemlist xlist, ylist;
+  int p, f, t, xoff = 0, yoff = 0, symnum, count = 0;
+  elemlist *xcond, *ycond;
+
+  xcond = (elemlist *) malloc (nsyms * sizeof (elemlist));
+  if (!xcond)
+    error_msg ("EncodeOffsets", "Out of memory...", "");
+  ycond = (elemlist *) malloc (nsyms * sizeof (elemlist));
+  if (!ycond)
+    error_msg ("EncodeOffsets", "Out of memory...", "");
+
+  for (i = 0; i < nsyms; i++)
+    {
+      xcond[i].head = ycond[i].head = NULL;
+      xcond[i].tot = ycond[i].tot = 0;
+    }
+
+  xlist.head = NULL;
+  xlist.tot = 0;
+  ylist.head = NULL;
+  ylist.tot = 0;
+
+  for (step = symbol_list, count = 0; step; step = step->next, count++)
+    {
+      symnum = step->data.symnum;
+      for (dox = 1; dox >= 0; dox--)
+	{
+	  /* decode x offset first, then y */
+	  target = arithmetic_decode_target (leveltop + levelmid + levelbot);
+	  if (target < leveltop)
+	    {
+	      arithmetic_decode (0, leveltop, leveltop + levelmid + levelbot);
+	      leveltop += TOP_INC;
+
+	      if (dox)
+		i = arithmetic_decode_target (xcond[symnum].tot);
+	      else
+		i = arithmetic_decode_target (ycond[symnum].tot);
+
+	      if (dox)
+		{
+		  elem_getrange (&xcond[symnum], i, &xoff);
+		  elem_find (&xcond[symnum], xoff, &p, &f, &t);
+		}
+	      else
+		{
+		  elem_getrange (&ycond[symnum], i, &yoff);
+		  elem_find (&ycond[symnum], yoff, &p, &f, &t);
+		}
+	      arithmetic_decode (p, f, t);
+	    }
+	  else if (target < leveltop + levelmid)
+	    {
+	      arithmetic_decode (leveltop, leveltop + levelmid, leveltop + levelmid + levelbot);
+	      levelmid += MID_INC;
+
+	      if (dox)
+		i = arithmetic_decode_target (xlist.tot);
+	      else
+		i = arithmetic_decode_target (ylist.tot);
+
+	      if (dox)
+		{
+		  elem_getrange (&xlist, i, &xoff);
+		  elem_find (&xlist, xoff, &p, &f, &t);
+		}
+	      else
+		{
+		  elem_getrange (&ylist, i, &yoff);
+		  elem_find (&ylist, yoff, &p, &f, &t);
+		}
+	      arithmetic_decode (p, f, t);
+	    }
+	  else
+	    {
+	      arithmetic_decode (leveltop + levelmid, leveltop + levelmid + levelbot, leveltop + levelmid + levelbot);
+	      levelbot += BOT_INC;
+	      if (dox)
+		xoff = DecodeGammaSigned (&xpos, &xneg);
+	      else
+		yoff = DecodeGammaSigned (&ypos, &yneg);
+	    }
+
+	  if (dox)
+	    {
+	      step->data.xoffset = xoff;
+	      elem_add (&xlist, xoff);
+	      elem_add (&xcond[symnum], xoff);
+	    }
+	  else
+	    {
+	      step->data.yoffset = yoff;
+	      elem_add (&ylist, yoff);
+	      elem_add (&ycond[symnum], yoff);
+	    }
+
+	  if (leveltop + levelmid >= maxfrequency)
+	    {
+	      leveltop = (leveltop + 1) / 2;
+	      levelmid = (levelmid + 1) / 2;
+	      levelbot = (levelbot + 1) / 2;
+	    }
+	}
+    }
+  for (t = 0; t < nsyms; t++)
+    {
+      elem_free (&xcond[t]);
+      elem_free (&ycond[t]);
+    }
+  elem_free (&xlist);
+  elem_free (&ylist);
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/images/codeoffsets.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/images/codeoffsets.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/images/codeoffsets.h	(revision 699)
@@ -0,0 +1,32 @@
+/**************************************************************************
+ *
+ * codeoffsets.h -- Functions which code the mark offsets
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef __CODEOFFSETS_H
+#define __CODEOFFSETS_H
+
+#include "marklist.h"
+
+void EncodeOffsets (marklistptr MarkList, int num_symbols);
+void DecodeOffsets (marklistptr MarkList, int num_symbols);
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/src/images/codesyms.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/images/codesyms.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/images/codesyms.c	(revision 699)
@@ -0,0 +1,96 @@
+/**************************************************************************
+ *
+ * codesyms.c -- Functions which code the symbol sequence
+ * Copyright (C) 1994  Hugh Emberson
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "arithcode.h"
+#include "marklist.h"
+#include "pbmtools.h"
+#include "utils.h"
+#include "codesyms.h"
+
+int charsetsize = 256;
+
+void 
+EncodeSymbols (marklistptr MarkList, int NSyms)
+{
+  marklistptr step;
+  unsigned short Events[PPM_CHUNK];
+  int j = 0;
+
+  charsetsize = max (NSyms, 256);
+
+  step = MarkList;
+  while (step)
+    {
+      Events[j] = step->data.symnum;
+
+      assert (Events[j] > 0 || Events[j] < (unsigned) NSyms);
+
+      j++;
+
+      if (j == PPM_CHUNK)
+	{
+	  encodestring (Events, PPM_CHUNK);
+	  j = 0;
+	}
+      step = step->next;
+    };
+
+  if (j != 0)
+    encodestring (Events, j);
+
+  encodestring (Events, 0);
+}
+
+
+marklistptr 
+DecodeSymbols (int NSyms)
+{
+  marklistptr MarkList = NULL;
+  unsigned short Events[PPM_CHUNK];
+  int j, n;
+  marktype d;
+
+  charsetsize = max (NSyms, 256);
+
+  do
+    {
+      decodestring (Events, &n);
+
+      for (j = 0; j < n; j++)
+	{
+	  d.symnum = Events[j];
+	  marklist_add (&MarkList, d);
+	}
+    }
+  while (n != 0);
+
+  return MarkList;
+}
+
+void 
+InitPPM (void)
+{
+  kbytes = 896;
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/images/codesyms.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/images/codesyms.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/images/codesyms.h	(revision 699)
@@ -0,0 +1,44 @@
+/**************************************************************************
+ *
+ * codesyms.h -- Functions which code the symbol sequence
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+#ifndef _CodeSymbols_h_
+#define _CodeSymbols_h_
+
+#include "marklist.h"
+#include "arithcode.h"
+
+#ifndef FIXEDORDER
+#define FIXEDORDER	3
+#endif
+
+#define PPM_CHUNK	1024
+
+extern int kbytes;
+extern void decodestring ();
+extern void encodestring ();
+
+void EncodeSymbols (marklistptr MarkList, int NSyms);
+marklistptr DecodeSymbols (int NSyms);
+
+void InitPPM (void);
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/src/images/extractor.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/images/extractor.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/images/extractor.c	(revision 699)
@@ -0,0 +1,423 @@
+/**************************************************************************
+ *
+ * extractor.c -- Functions related to extracting marks
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "extractor.h"
+#include "pbmtools.h"
+#include "marklist.h"
+#include "utils.h"
+
+
+/* 
+   'splitjoined' defines whether the extractor modules tries to break
+   "seemingly" merged/joined marks.
+ */
+int splitjoined = 1;
+
+#define MAX_FILL_STACK_SIZE 10000
+
+/* RunRegionFill uses line conherence to implement a better fill
+   routine.  The stack has currently got 10000 elements, although the
+   biggest I have encountered used a maximum stack size of 1294.  usage:
+   RunRegionFill(bitmap,mark,i,j,xpos,ypos); xpos and ypos are the top
+   left corners of the mark.  */
+
+int maxst = 0, st = 0;
+struct
+  {
+    int x, y;
+  }
+a[MAX_FILL_STACK_SIZE];
+
+#define push(i,j) a[st].x=i;a[st].y=j;st++;if(st>MAX_FILL_STACK_SIZE){error_msg ("extractor/flood_fill","woooo! Fill stack overflow!","");}
+#define pop(i,j) st--;i=a[st].x;j=a[st].y
+
+void 
+RunRegionFill (Pixel ** bitmap, marktype * mark, int i, int j, int xl, int yt, int cols, int rows)
+{
+  int l, r, t, start_up, start_down;
+
+  push (i, j);
+  while (st > 0)
+    {
+      if (st > maxst)
+	maxst = st;
+
+      pop (l, j);
+      r = l;
+
+      while (pbm_getpixel (bitmap, l - 1, j))
+	l--;
+      while (pbm_getpixel (bitmap, r + 1, j))
+	r++;
+      start_up = start_down = 1;
+      for (t = min (r + 1, cols - 1); t >= max (l - 1, 0); t--)
+	{
+	  if ((t >= l) && (t <= r))
+	    {
+	      pbm_putpixel (bitmap, t, j, 0);
+
+	      if (mark)
+		pbm_putpixel (mark->bitmap, t - xl, j - yt, 1);
+	    }
+
+	  if ((j - 1 >= 0) && (pbm_getpixel (bitmap, t, j - 1)))
+	    {
+	      if (start_up)
+		{
+		  push (t, j - 1);
+		  start_up = 0;
+		}
+	    }
+	  else
+	    start_up = 1;
+	  if ((j + 1 < rows) && (pbm_getpixel (bitmap, t, j + 1)))
+	    {
+	      if (start_down)
+		{
+		  push (t, j + 1);
+		  start_down = 0;
+		}
+	    }
+	  else
+	    start_down = 1;
+	}
+    }
+}
+
+
+
+
+/*
+
+   ExtractNextMark starts at (tx,ty) and looks for a set (==1) pixel.
+   The mark is then boundary traced to give an outline {The process is a
+   4-pixel exterior check -- giving 8 way connectivity}, and then
+   RunRegionFill is used to remove the mark and set the pixels in the
+   mark.
+
+   usage: ExtractNextMark(bitmap,&marklist,&startx,&starty,int cols,int rows);
+   Defines:
+   gp - gets a pixel, but returns 0 for outside regions.
+   boundary_{x,y}comp - returns the {x,y} component of the direction.
+   dir 0=right, 1=down, 2=left, 3=up.
+ */
+
+#define gp(i,j) ( (((i)<0)||((j)<0)||((i)>=cols)||((j)>=rows))? 0 : pbm_getpixel(bitmap,(i),(j)))
+
+#define boundary_xcomp(x) (x==0?1:(x==2?-1 : 0))
+#define boundary_ycomp(x) (x==1?1:(x==3?-1 : 0))
+
+int 
+ExtractNextMark (Pixel ** bitmap, marklistptr * marklist, int *tx, int *ty, int cols, int rows)
+{
+  int i, j, xl, xr, yt, yb;
+  register int dir, tempdir;
+  marktype d;
+
+  xl = INT_MAX;
+  xr = (-1);
+  yt = INT_MAX;
+  yb = (-1);
+
+  i = *tx;
+  j = *ty;
+  if ((i < 0) || (j < 0))
+    error_msg ("ExtractNextMark()", "Stupid values for x,y in findnextmark()", "");
+  while (!pbm_getpixel (bitmap, i, j))
+    {
+      i++;
+      if (i >= cols)
+	{
+	  i = 0;
+	  j++;
+	  if (j >= rows)
+	    return 0;
+	}
+    }
+
+  *tx = i;
+  *ty = j;
+  dir = 0;
+  j = j - 1;			/* start above */
+  do
+    {
+      if (i > xr)
+	xr = i - 1;
+      if (i < xl)
+	xl = i + 1;
+      if (j < yt)
+	yt = j + 1;
+      if (j > yb)
+	yb = j - 1;
+      tempdir = (dir + 1) % 4;
+      if (gp (i + boundary_xcomp (tempdir), j + boundary_ycomp (tempdir)) == 0)
+	dir = tempdir;
+      else
+	{
+	  tempdir = dir;
+	  if (gp (i + boundary_xcomp (tempdir), j + boundary_ycomp (tempdir)) == 0)
+	    dir = tempdir;
+	  else
+	    {
+	      tempdir = (dir + 3) % 4;
+	      if (gp (i + boundary_xcomp (tempdir), j + boundary_ycomp (tempdir)) == 0)
+		dir = tempdir;
+	      else
+		dir = (dir + 2) % 4;
+	    }
+	}
+      i += boundary_xcomp (dir);
+      j += boundary_ycomp (dir);
+    }
+  while ((i != (*tx)) || (j != (*ty - 1)));
+
+  d.w = (xr - xl + 1);
+  d.h = (yb - yt + 1);
+  d.xpos = xl;
+  d.ypos = yt;
+  d.bitmap = pbm_allocarray (d.w, d.h);
+
+  RunRegionFill (bitmap, &d, *tx, *ty, xl, yt, cols, rows);
+
+  marktype_calc_features (&d, "?");
+  marklist_add (marklist, d);
+
+  *tx = *tx + 1;
+  return 1;
+}
+
+
+
+
+static void 
+normalise (float *p, int w)
+{
+  int x;
+  float mx;
+
+  mx = p[0];
+  for (x = 0; x < w; x++)
+    if (p[x] > mx)
+      mx = p[x];
+  if (mx != 0)
+    {
+      for (x = 0; x < w; x++)
+	p[x] = p[x] / mx;
+    }
+}
+
+
+
+#define MAXPROFILE 1000
+float profile[MAXPROFILE], p2[MAXPROFILE];
+
+
+static int 
+possibly_joined (marktype * d)
+{
+  float aspect;
+  int i, n, x, y;
+  int leftable, l2;
+
+
+  aspect = d->w / (float) d->h;
+
+  if ((aspect >= 1.1) || (d->w < MAXPROFILE))
+    {
+      /* examine it more closely */
+
+
+      /* set up profile[] with a vertical profile */
+      for (x = 0; x < d->w; x++)
+	{
+	  profile[x] = 0;
+	  for (y = 0; y < d->h; y++)
+	    if (pbm_getpixel (d->bitmap, x, y))
+	      profile[x]++;
+	}
+
+      /* average the profile from (up to) 3 either way */
+      for (x = 0; x < d->w; x++)
+	{
+	  n = 0;
+	  p2[x] = 0;
+	  for (i = x - 3; i <= x + 3; i++)
+	    if ((i >= 0) && (i < d->w))
+	      {
+		n++;
+		p2[x] += profile[x];
+	      }
+	  p2[x] = p2[x] / (float) n;
+	}
+      for (x = 0; x < d->w; x++)
+	profile[x] = p2[x];
+
+      /* normalise profile[] */
+      normalise (profile, d->w);
+
+      /* find a little local minimum, with profile <0.25, make p2[] either 0 or 1 */
+      for (x = 0; x < d->w; x++)
+	p2[x] = 0;
+      for (x = 3; x < d->w - 3; x++)
+	{
+	  if (((profile[x - 2] - profile[x]) > 0.00) &&
+	      ((profile[x + 2] - profile[x]) > 0.00) &&
+	      (profile[x] < .25))
+	    p2[x] = 1;
+	}
+
+      /* travel rightwards, checking aspect ratio, looking at p2 now */
+      leftable = l2 = 0;
+      for (x = 0; x < d->w; x++)
+	if (p2[x] == 1)
+	  {
+	    if ((fabs ((abs (x - leftable) / (float) d->h) - 0.7) > 0.3) &&
+		(fabs ((abs (x - l2) / (float) d->h) - 0.7) > 0.3))
+	      p2[x] = 0;
+	    if (p2[x] == 1)
+	      {
+		leftable = l2;
+		l2 = x;
+	      }
+	  }
+
+      /* travel inwards, checking widths of chars, looking at p2 */
+      /* if a char has an aspect ratio of <= ? kill it */
+      l2 = 0;
+      leftable = d->w - 1;
+      x = 0;
+      y = d->w - 1;
+      while (x <= y)
+	{
+	  if (p2[x])
+	    {
+	      if (abs (x - l2) < max (d->h * 0.6, 5))
+		p2[x] = 0;
+	      if (p2[x])
+		l2 = x;
+	    }
+	  if (p2[y])
+	    {
+	      if (abs (y - leftable) < max (d->h * 0.6, 5))
+		p2[y] = 0;
+	      if (p2[y])
+		leftable = y;
+	    }
+	  x++;
+	  y--;
+	}
+
+
+      for (x = 0; x < d->w; x++)
+	if (p2[x])
+	  return 1;
+    }
+  return 0;
+}
+
+
+static void 
+split_mark (marklistptr * list, marktype * joinedmark)
+{
+  marktype new, old;
+  int x, found = 0, left = 0;
+  int i, j;
+  int oldw, oldh;
+  int overwrite = 1;
+
+  old = marktype_copy (*joinedmark);
+  oldw = joinedmark->w;
+  oldh = joinedmark->h;
+  for (x = 0; x < old.w; x++)
+    if (p2[x])
+      {
+	found = x;
+	break;
+      }
+  do
+    {
+      /* split at (found) position */
+      new.w = found + 1 - left;
+      new.h = old.h;
+      new.xpos = old.xpos + left;
+      new.ypos = old.ypos;
+      new.bitmap = pbm_allocarray (new.w, new.h);
+
+      for (i = 0; i < new.w; i++)
+	for (j = 0; j < new.h; j++)
+	  pbm_putpixel (new.bitmap, i, j, pbm_getpixel (old.bitmap, left + i, j));
+
+      marktype_adj_bound (&new);
+
+      if (overwrite)
+	{
+	  overwrite = 0;
+	  pbm_freearray (&(joinedmark->bitmap), oldh);
+	  *joinedmark = new;
+	}
+      else
+	marklist_add (list, new);
+
+
+      p2[found] = 0;
+      left = found;
+      found = 0;
+      for (x = left; x < old.w; x++)
+	if (p2[x])
+	  {
+	    found = x;
+	    break;
+	  }
+      if (left == old.w - 1)
+	break;			/* if we've got to the right */
+      if (found == 0)
+	found = old.w - 1;
+    }
+  while (found);
+  pbm_freearray (&(old.bitmap), oldh);
+}
+
+
+
+
+
+void 
+ExtractAllMarks (Pixel ** bitmap, marklistptr * list, int cols, int rows)
+{
+  int startx, starty, count;
+  marklistptr step;
+
+  startx = starty = 0;
+  while (ExtractNextMark (bitmap, list, &startx, &starty, cols, rows))
+    ;
+
+  if (splitjoined)
+    for (step = (*list), count = 0; step; step = step->next, count++)
+      if (possibly_joined (&(step->data)))
+	split_mark (list, &(step->data));
+
+  for (step = (*list), count = 0; step; step = step->next, count++)
+    step->data.symnum = count;
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/images/extractor.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/images/extractor.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/images/extractor.h	(revision 699)
@@ -0,0 +1,38 @@
+/**************************************************************************
+ *
+ * extractor.h -- Functions related to extracting marks
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef _EXTRACTOR_H_
+#define _EXTRACTOR_H_
+
+#include "marklist.h"
+#include "pbmtools.h"
+
+extern int splitjoined;
+
+void RunRegionFill (Pixel ** bitmap, marktype * mark, int i, int j, int xl, int yt, int cols, int rows);
+
+int ExtractNextMark (Pixel ** bitmap, marklistptr * marklist, int *tx, int *ty, int cols, int rows);
+void ExtractAllMarks (Pixel ** bitmap, marklistptr * marklist, int cols, int rows);
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/src/images/felics.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/images/felics.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/images/felics.c	(revision 699)
@@ -0,0 +1,340 @@
+/**************************************************************************
+ *
+ * felics.c -- The main guts of the felics algorithm
+ * Copyright (C) 1994  Neil Sharman and Kerry Guise
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "bitio_m.h"
+#include "bitio_m_stdio.h"
+
+
+#undef putw
+#define putw(x)	{					\
+	short z;					\
+	byteptr = (unsigned char *)(x);			\
+	for (z = 0; z < sizeof(int); z++)		\
+		(void) putc((int) byteptr[z], fp_out);	\
+}
+#define INTSIZE sizeof(int) * 8
+#define BOOLEAN short
+#define TRUE 1
+#define FALSE 0
+
+#define IN_RANGE 0
+#define OUT_OF_RANGE 1
+#define BELOW_RANGE 0
+#define ABOVE_RANGE 1
+
+
+void 
+encode (unsigned int *line, int width, int height, int maxval, int maxk,
+	FILE * fp_in, FILE * fp_out, int (*get) (FILE *))
+{
+  int x, y;
+  unsigned long P, L, H, N1, N2, delta;
+  unsigned long cumul_param[256][INTSIZE];
+  unsigned max_cumul;
+
+  memset (cumul_param, '0', sizeof (cumul_param));
+
+  if (maxk)
+    max_cumul = maxk;
+  else
+    CEILLOG_2 (maxval, max_cumul);
+
+  ENCODE_START (fp_out)
+
+    for (y = 0; y < height; y++)
+    for (x = 0; x < width; x++)
+      {
+	if (y > 0)
+	  {
+	    if (x > 0)
+	      {			/* ******** */
+		N1 = line[x];	/* ****1*** */
+		N2 = line[x - 1];	/* ***2P*** */
+	      }			/* ******** */
+	    else
+	      {			/* |******* */
+		N1 = line[x];	/* |12***** */
+		N2 = line[x + 1];	/* |P***** */
+	      }			/* |******* */
+	  }
+	else
+	  {
+	    if (x > 1)
+	      {
+		N1 = line[x - 1];	/* -------- */
+		N2 = line[x - 2];	/* **12P*** */
+	      }			/* ******** */
+	    else if (x > 0)
+	      {
+		N1 = line[x - 1];	/* +------- */
+		N2 = N1;	/* |3P***** */
+	      }			/* |******* */
+	    else
+	      {
+		line[x] = get (fp_in);	/* +------- */
+		BINARY_ENCODE (line[x] + 1, maxval + 1);	/* |P****** */
+		continue;	/* |******* */
+	      }
+	  }
+
+
+	H = N1 > N2 ? N1 : N2;
+	L = N1 < N2 ? N1 : N2;
+	delta = H - L;
+
+	line[x] = P = get (fp_in);
+
+	if ((L <= P) && (P <= H))
+	  {
+	    /* IN-RANGE */
+	    unsigned long range, logofrange, thresh, numlong;
+	    int nbits;
+
+	    ENCODE_BIT (IN_RANGE);
+
+	    P -= L;
+
+	    range = delta + 1;
+
+	    if (range > 1)
+	      {
+		/* 2**i <= delta + 1 < 2**(i+1) ; find i ... */
+		CEILLOG_2 (range, logofrange);
+
+		/* number of n-bit values in the range */
+		thresh = (1 << logofrange) - range;
+
+		/* number of n+1-bit values in the range */
+		numlong = range - thresh;
+
+		/* rotate the n-bit codes into the centre */
+		P -= (numlong >> 1);
+		if ((long) P < 0)
+		  P += range;
+
+		if (P < thresh)
+		  nbits = logofrange - 1;
+		else
+		  {
+		    nbits = logofrange;
+		    P += thresh;
+		  }
+		while (--nbits >= 0)
+		  ENCODE_BIT ((P >> nbits) & 1);
+	      }
+	  }
+	else
+	  {
+	    /* OUT_OF_RANGE */
+	    unsigned long diff, parm, i, min;
+
+	    ENCODE_BIT (OUT_OF_RANGE);
+
+	    if (P < L)
+	      {
+		ENCODE_BIT (BELOW_RANGE);
+		diff = L - P - 1;
+	      }
+	    else
+	      {
+		ENCODE_BIT (ABOVE_RANGE);
+		diff = P - H - 1;
+	      }
+
+	    /* Now code 'diff' as a Golomb-Rice code */
+
+	    parm = 0;
+	    min = cumul_param[delta][0];
+	    for (i = 1; i < max_cumul; i++)
+	      if (cumul_param[delta][i] < min)
+		{
+		  min = cumul_param[delta][i];
+		  parm = i;
+		}
+
+	    UNARY_ENCODE ((diff >> parm) + 1);
+	    BINARY_ENCODE ((diff & ((1 << parm) - 1)) + 1, 1 << parm);
+
+	    for (i = 0; i < max_cumul; i++)
+	      cumul_param[delta][i] += (diff >> i) + 1 + i;
+	  }
+
+      }
+
+  ENCODE_DONE
+
+}
+
+
+
+
+
+
+void 
+decode (unsigned int *line, int width, int height, int maxval, int maxk,
+	FILE * fp_in, FILE * fp_out)
+{
+  int x, y;
+  unsigned int P, L, H, N1, N2, delta;
+  unsigned long cumul_param[256][INTSIZE];
+  unsigned max_cumul;
+
+  memset (cumul_param, '0', sizeof (cumul_param));
+
+  if (maxk)
+    max_cumul = maxk;
+  else
+    CEILLOG_2 (maxval, max_cumul);
+
+  DECODE_START (fp_in);
+
+  for (y = 0; y < height; y++)
+    for (x = 0; x < width; x++)
+      {
+	if (y > 0)
+	  {
+	    if (x > 0)
+	      {			/* ******** */
+		N1 = line[x];	/* ****1*** */
+		N2 = line[x - 1];	/* ***2P*** */
+	      }			/* ******** */
+	    else
+	      {			/* |******* */
+		N1 = line[x];	/* |12***** */
+		N2 = line[x + 1];	/* |P***** */
+	      }			/* |******* */
+	  }
+	else
+	  {
+	    if (x > 1)
+	      {
+		N1 = line[x - 1];	/* -------- */
+		N2 = line[x - 2];	/* **12P*** */
+	      }			/* ******** */
+	    else if (x > 0)
+	      {
+		N1 = line[x - 1];	/* +------- */
+		N2 = N1;	/* |3P***** */
+	      }			/* |******* */
+	    else
+	      {
+		BINARY_DECODE (line[x], maxval + 1);
+		line[x]--;
+		if (maxval < 256)
+		  putc (line[x], fp_out);	/* +------- */
+		else		/* |P****** */
+		  fprintf (fp_out, "%u ", line[x]);	/* |******* */
+		continue;
+	      }
+	  }
+
+
+	H = N1 > N2 ? N1 : N2;
+	L = N1 < N2 ? N1 : N2;
+	delta = H - L;
+
+	if (DECODE_BIT == IN_RANGE)
+	  {
+	    unsigned long i, range, logofrange, thresh, numlong;
+	    unsigned xx, l;
+
+	    range = delta + 1;
+
+	    if (range > 1)
+	      {
+		/* 2**i <= delta + 1 < 2**(i+1) ; find i ... */
+		CEILLOG_2 (range, logofrange);
+
+		/* number of n-bit values in the range */
+		thresh = (1 << logofrange) - range;
+		for (P = i = 0; i < logofrange - 1; i++)
+		  DECODE_ADD (P);
+		xx = P;
+		l = logofrange - 1;
+		if (P >= thresh)
+		  {
+		    DECODE_ADD (P);
+		    xx = P;
+		    l++;
+		    P -= thresh;
+		  }
+
+		/* number of n+1-bit values in the range */
+		numlong = range - thresh;
+
+		/* rotate the n-bit codes into the centre */
+		P += (numlong >> 1);
+		if ((long) P >= range)
+		  P -= range;
+
+		P += L;
+	      }
+	    else
+	      P = L;
+	  }
+	else
+	  {
+	    int out_of_range = DECODE_BIT;
+	    unsigned long diff, unary, binary, i, min, parm;
+
+	    /* Now code 'diff' as a Golomb-Rice code */
+
+	    parm = 0;
+	    min = cumul_param[delta][0];
+	    for (i = 1; i < max_cumul; i++)
+	      if (cumul_param[delta][i] < min)
+		{
+		  min = cumul_param[delta][i];
+		  parm = i;
+		}
+
+	    UNARY_DECODE (unary);
+	    diff = (unary - 1) << parm;
+	    BINARY_DECODE (binary, 1 << parm);
+	    diff |= binary - 1;
+
+	    for (i = 0; i < max_cumul; i++)
+	      cumul_param[delta][i] += (diff >> i) + 1 + i;
+
+	    if (out_of_range == BELOW_RANGE)
+	      {
+		P = L - diff - 1;
+	      }
+	    else
+	      {
+		P = diff + H + 1;
+	      }
+	  }
+
+	line[x] = P;
+
+	if (maxval < 256)
+	  (void) putc (P, fp_out);
+	else
+	  fprintf (fp_out, "%u ", P);
+      }
+
+  DECODE_DONE
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/images/lstevent.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/images/lstevent.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/images/lstevent.c	(revision 699)
@@ -0,0 +1,421 @@
+/**************************************************************************
+ *
+ * lstevent.c -- Routines to implement "events" using MTF lists.
+ * Copyright (C) 1987  Alistair Moffat
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "arithcode.h"
+#include "model.h"
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:17:26  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:33:37  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.2  1994/09/20  04:41:40  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+void
+write_method ()
+{
+  fprintf (stderr, "C");
+}
+
+
+/*===========================================*/
+
+/*
+ * [Rodney Brown - Sep/95]
+ * On IBM RS/6000 AIX version 3.2 the compiler defines NULL as (void *)0
+ * when in ANSI mode. Hence the (event) and (point) casts
+ */
+
+
+#define GETNEWNODE(e,node)		\
+do 					\
+{	node = p32(numnodes) ;		\
+	node->eventnum = (event)e ;	\
+	node->count = increment ;	\
+	node->foll.totalcnt = 0 ;	\
+	node->foll.notfound = 0 ;	\
+	node->foll.list = (point)NULL ;	\
+	numnodes += 1 ;			\
+	nfreenodes -= 1 ;		\
+}					\
+while (false)
+
+/*===========================================*/
+
+#define INSERTNODE(U,node)		\
+do					\
+{	node->next = U->list ;		\
+	U->list = p16(node) ;		\
+	U->totalcnt += increment;	\
+}					\
+while (false)
+
+/*===========================================*/
+
+int
+halvelist (p)
+     register eventnode *p;
+{
+  register int sum = 0;
+  while (p != ENULL)
+    {
+      sum += p->count = (p->count + 1) / 2;
+      p = p32 (p->next);
+    }
+  return (sum);
+}
+
+/*===========================================*/
+
+void
+halveallcounts (U)
+     register eventset *U;
+{
+  U->notfound = (U->notfound + 1) / 2;
+  U->totalcnt = U->notfound;
+  U->totalcnt += halvelist (p32 (U->list));
+}
+
+/*===========================================*/
+
+eventnode *
+encode_event_noex (U, e, success)
+     register eventset *U;
+     event e;
+     boolean *success;
+{
+  register eventnode *pnode, *node;
+  register int lbnd, totl;
+
+  if (U->list == (point)NULL)
+    {
+      *success = false;
+      U->notfound += increment;
+      U->totalcnt += increment;
+      GETNEWNODE (e, node);
+      INSERTNODE (U, node);
+      return (node);
+    }
+
+  lbnd = 0;
+  pnode = ENULL;
+  node = p32 (U->list);
+  while (node != ENULL)
+    {
+      if (e == node->eventnum)
+	break;
+      lbnd += node->count;
+      pnode = node;
+      node = p32 (node->next);
+    }
+
+  totl = U->totalcnt;
+
+  if (node == ENULL)
+    {				/* event not found in list */
+      if ((totl > U->notfound))
+	arithmetic_encode (totl - U->notfound, totl, totl);
+      U->notfound += increment;
+      U->totalcnt += increment;
+      *success = false;
+      GETNEWNODE (e, node);
+      INSERTNODE (U, node);
+    }
+  else
+    {				/* found in the list, so go ahead and code */
+      arithmetic_encode (lbnd, lbnd + node->count, totl);
+      node->count += increment;
+      U->totalcnt += increment;
+      *success = true;
+      /* and move to front */
+      if (pnode != ENULL)
+	{
+	  pnode->next = node->next;
+	  node->next = U->list;
+	  U->list = p16 (node);
+	}
+    }
+  if (U->totalcnt > maxtotalcnt)
+    halveallcounts (U);
+  return (node);
+}
+
+/*===========================================*/
+
+eventnode *
+decode_event_noex (U, success, e)
+     register eventset *U;
+     boolean *success;
+     event *e;
+{
+  register eventnode *node, *pnode;
+  register int tget, lbnd, totl;
+
+  if (U->list == (point)NULL)
+    {
+      *success = false;
+      U->totalcnt += increment;
+      U->notfound += increment;
+      GETNEWNODE (NULL, node);
+      INSERTNODE (U, node);
+      return (node);
+    }
+  totl = U->totalcnt;
+  tget = arithmetic_decode_target (totl);
+
+  if (tget >= totl - U->notfound)
+    {
+      if (totl > U->notfound)
+	arithmetic_decode (totl - U->notfound, totl, totl);
+      U->totalcnt += increment;
+      U->notfound += increment;
+      *success = false;
+      GETNEWNODE (NULL, node);
+      INSERTNODE (U, node);
+    }
+  else
+    {
+      pnode = ENULL;
+      node = p32 (U->list);
+      lbnd = 0;
+      for (;;)
+	{
+	  if (tget < (lbnd += node->count))
+	    break;
+	  pnode = node;
+	  node = p32 (node->next);
+	}
+
+      arithmetic_decode (lbnd - node->count, lbnd, totl);
+      node->count += increment;
+      U->totalcnt += increment;
+      *e = node->eventnum;
+      *success = true;
+      /* and move to front */
+      if (pnode != ENULL)
+	{
+	  pnode->next = node->next;
+	  node->next = U->list;
+	  U->list = p16 (node);
+	}
+    }
+  if (U->totalcnt > maxtotalcnt)
+    halveallcounts (U);
+  return (node);
+}
+
+/*===========================================*/
+
+eventnode *
+encode_event (U, e, success)
+     register eventset *U;
+     event e;
+     boolean *success;
+{
+  register eventnode *pnode, *node, *p;
+  register int lbnd, totl;
+
+  if (U->list == (point)NULL)
+    {
+      *success = false;
+      U->notfound += increment;
+      U->totalcnt += increment;
+      GETNEWNODE (e, node);
+      INSERTNODE (U, node);
+      return (node);
+    }
+
+  lbnd = 0;
+  pnode = ENULL;
+  node = p32 (U->list);
+  while (node != ENULL)
+    {
+      if (!excluded[node->eventnum])
+	{
+	  if (e == node->eventnum)
+	    break;
+	  lbnd += node->count;
+	}
+      pnode = node;
+      node = p32 (node->next);
+    }
+
+  if (node == ENULL)
+    {				/* event not found in list, and whole list has been scanned */
+      totl = lbnd + U->notfound;
+      if ((totl > U->notfound))
+	arithmetic_encode (totl - U->notfound, totl, totl);
+      U->notfound += increment;
+      U->totalcnt += increment;
+      *success = false;
+      GETNEWNODE (e, node);
+      INSERTNODE (U, node);
+    }
+  else
+    {				/* found in the list, so go ahead and code */
+      /* first, total up rest of list */
+      totl = lbnd + node->count;
+      p = p32 (node->next);
+      while (p != ENULL)
+	{
+	  if (!excluded[p->eventnum])
+	    totl += p->count;
+	  p = p32 (p->next);
+	}
+      totl += U->notfound;
+
+      arithmetic_encode (lbnd, lbnd + node->count, totl);
+      node->count += increment;
+      U->totalcnt += increment;
+      *success = true;
+      /* and move to front */
+      if (pnode != ENULL)
+	{
+	  pnode->next = node->next;
+	  node->next = U->list;
+	  U->list = p16 (node);
+	}
+    }
+  if (U->totalcnt > maxtotalcnt)
+    halveallcounts (U);
+  return (node);
+}
+
+/*===========================================*/
+
+eventnode *
+decode_event (U, success, e)
+     register eventset *U;
+     boolean *success;
+     event *e;
+{
+  register eventnode *node, *pnode;
+  register int tget, lbnd, totl;
+
+  if (U->list == (point)NULL)
+    {
+      *success = false;
+      U->totalcnt += increment;
+      U->notfound += increment;
+      GETNEWNODE (NULL, node);
+      INSERTNODE (U, node);
+      return (node);
+    }
+  totl = 0;
+  node = p32 (U->list);
+  while (node != ENULL)
+    {
+      if (!excluded[node->eventnum])
+	totl += node->count;
+      node = p32 (node->next);
+    }
+  totl += U->notfound;
+  tget = arithmetic_decode_target (totl);
+
+  if (tget >= totl - U->notfound)
+    {
+      if (totl > U->notfound)
+	arithmetic_decode (totl - U->notfound, totl, totl);
+      U->totalcnt += increment;
+      U->notfound += increment;
+      *success = false;
+      GETNEWNODE (NULL, node);
+      INSERTNODE (U, node);
+    }
+  else
+    {
+      pnode = ENULL;
+      node = p32 (U->list);
+      lbnd = 0;
+      for (;;)
+	{
+	  if (!excluded[node->eventnum])
+	    {
+	      lbnd += node->count;
+	      if (tget < lbnd)
+		break;
+	    }
+	  pnode = node;
+	  node = p32 (node->next);
+	}
+
+      arithmetic_decode (lbnd - node->count, lbnd, totl);
+      node->count += increment;
+      U->totalcnt += increment;
+      *e = node->eventnum;
+      *success = true;
+      /* and move to front */
+      if (pnode != ENULL)
+	{
+	  pnode->next = node->next;
+	  node->next = U->list;
+	  U->list = p16 (node);
+	}
+    }
+  if (U->totalcnt > maxtotalcnt)
+    halveallcounts (U);
+  return (node);
+}
+
+/*===========================================*/
+
+eventnode *
+newnode (e)
+     event e;
+{
+  register eventnode *node;
+  GETNEWNODE (e, node);
+  node->next = (point)NULL;
+  node->prev = (point)NULL;
+  return (node);
+}
+
+/*===========================================*/
+
+/* not required in this version */
+
+#if 0
+eventnode *
+addevent (U, e)
+     register eventset *U;
+     event e;
+{
+  register eventnode *node;
+  /* get an event node */
+  GETNEWNODE (e, node);
+  INSERTNODE (U, node);
+  if (U->totalcnt > maxtotalcnt)
+    halveallcounts (U);
+  return (node);
+}
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/src/images/marklist.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/images/marklist.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/images/marklist.c	(revision 699)
@@ -0,0 +1,827 @@
+/**************************************************************************
+ *
+ * marklist.c -- Functions managing the list of marks
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "marklist.h"
+#include "pbmtools.h"
+#include "utils.h"
+
+void 
+marktype_dump (marktype d)
+{
+  fprintf (stderr, "symnum: %d\n", d.symnum);
+  fprintf (stderr, "xpos: %d, ypos: %d\n", d.xpos, d.ypos);
+  fprintf (stderr, "w: %d, h: %d\n", d.w, d.h);
+  fprintf (stderr, "xcen: %d, ycen: %d\n", d.xcen, d.ycen);
+  write_library_mark (stderr, d);
+}
+
+
+/* copy, including make a copy of the bitmap */
+marktype 
+marktype_copy (marktype d)
+{
+  marktype d2;
+  int i, j;
+
+  d2 = d;
+  d2.bitmap = pbm_allocarray (d2.w, d2.h);
+  for (i = 0; i < d2.w; i++)
+    for (j = 0; j < d2.h; j++)
+      pbm_putpixel (d2.bitmap, i, j, pbm_getpixel (d.bitmap, i, j));
+  return d2;
+}
+
+
+float 
+marktype_density (marktype d)
+{
+  return d.set / (float) (d.w * d.h);
+}
+
+
+
+void 
+marktype_calc_centroid (marktype * b)
+{
+  int r, c;
+  int count = 0;
+  int xtot = 0, ytot = 0;
+
+  for (r = 0; r < b->h; r++)
+    for (c = 0; c < b->w; c++)
+      if (pbm_getpixel (b->bitmap, c, r))
+	{
+	  count++;
+	  xtot += c;
+	  ytot += r;
+	}
+  if (count)
+    {
+      xtot /= count;
+      ytot /= count;
+    }
+  b->xcen = xtot;
+  b->ycen = ytot;
+}
+
+
+
+static void 
+marktype_area (marktype * d)
+{
+  int x, y;
+  int set = 0;
+
+  for (y = 0; y < d->h; y++)
+    for (x = 0; x < d->w; x++)
+      if (pbm_getpixel (d->bitmap, x, y))
+	set++;
+  d->set = set;
+}
+
+
+static void 
+marktype_h_runs (marktype * d)
+{
+  int x, y;
+  int inrun = 0, num = 0;
+  int foundblack = 0;
+
+  for (y = 0; y < d->h; y++)
+    {
+      inrun = 0;
+      foundblack = 0;
+      for (x = 0; x < d->w; x++)
+	{
+	  if ((!foundblack) && (pbm_getpixel (d->bitmap, x, y)))
+	    foundblack = 1;
+	  if ((foundblack) && (pbm_getpixel (d->bitmap, x, y) == 0))
+	    inrun = 1;
+	  if ((inrun) && (pbm_getpixel (d->bitmap, x, y)))
+	    {
+	      num++;
+	      inrun = 0;
+	      foundblack = 0;
+	    }
+	}
+    }
+  d->h_runs = num;
+}
+
+static void 
+marktype_v_runs (marktype * d)
+{
+  int x, y;
+  int inrun = 0, num = 0;
+  int foundblack = 0;
+
+
+  for (x = 0; x < d->w; x++)
+    {
+      inrun = 0;
+      foundblack = 0;
+      for (y = 0; y < d->h; y++)
+	{
+	  if ((!foundblack) && (pbm_getpixel (d->bitmap, x, y)))
+	    foundblack = 1;
+	  if ((foundblack) && (pbm_getpixel (d->bitmap, x, y) == 0))
+	    inrun = 1;
+	  if ((inrun) && (pbm_getpixel (d->bitmap, x, y)))
+	    {
+	      num++;
+	      inrun = 0;
+	      foundblack = 0;
+	    }
+	}
+    }
+  d->v_runs = num;
+}
+
+
+/* assume we already have the centroid! */
+static void 
+marktype_prog_cent (marktype * d)
+{
+  int x, y;
+  int xtot = 0, ytot = 0;
+  int num = 0;
+
+  xtot = ytot = num = 0;
+  for (x = 0; x < d->xcen; x++)
+    for (y = 0; y < d->ycen; y++)
+      if (pbm_getpixel (d->bitmap, x, y))
+	{
+	  xtot += x;
+	  ytot += y;
+	  num++;
+	}
+  if (num == 0)
+    {
+      d->xcen1 = -1;
+      d->ycen1 = -1;
+    }
+  else
+    {
+      d->xcen1 = xtot / num;
+      d->ycen1 = ytot / num;
+    }
+
+
+  xtot = ytot = num = 0;
+  for (x = d->xcen; x < d->w; x++)
+    for (y = 0; y < d->ycen; y++)
+      if (pbm_getpixel (d->bitmap, x, y))
+	{
+	  xtot += x;
+	  ytot += y;
+	  num++;
+	}
+  if (num == 0)
+    {
+      d->xcen2 = -1;
+      d->ycen2 = -1;
+    }
+  else
+    {
+      d->xcen2 = xtot / num;
+      d->ycen2 = ytot / num;
+    }
+
+
+
+  xtot = ytot = num = 0;
+  for (x = 0; x < d->xcen; x++)
+    for (y = d->ycen; y < d->h; y++)
+      if (pbm_getpixel (d->bitmap, x, y))
+	{
+	  xtot += x;
+	  ytot += y;
+	  num++;
+	}
+
+
+  if (num == 0)
+    {
+      d->xcen3 = -1;
+      d->ycen3 = -1;
+    }
+  else
+    {
+      d->xcen3 = xtot / num;
+      d->ycen3 = ytot / num;
+    }
+
+  xtot = ytot = num = 0;
+  for (x = d->xcen; x < d->w; x++)
+    for (y = d->ycen; y < d->h; y++)
+      if (pbm_getpixel (d->bitmap, x, y))
+	{
+	  xtot += x;
+	  ytot += y;
+	  num++;
+	}
+  if (num == 0)
+    {
+      d->xcen4 = -1;
+      d->ycen4 = -1;
+    }
+  else
+    {
+      d->xcen4 = xtot / num;
+      d->ycen4 = ytot / num;
+    }
+}
+
+
+
+/* call this to get all features */
+void 
+marktype_calc_features (marktype * d, char str[])
+{
+  if (strlen (str))
+    strcpy (d->name, str);
+  marktype_calc_centroid (d);
+  marktype_area (d);
+  marktype_prog_cent (d);
+  marktype_h_runs (d);
+  marktype_v_runs (d);
+}
+
+
+
+
+
+void 
+marktype_adj_bound (marktype * m)
+{
+  int x, y, w, h;
+  int xpos, ypos;
+  int top = 0, left = 0, right = 0, bot = 0;
+  marktype temp;
+
+  w = m->w;
+  h = m->h;
+  xpos = m->xpos;
+  ypos = m->ypos;
+
+  for (y = 0; y < h; y++)
+    for (x = 0; x < w; x++)
+      if (pbm_getpixel (m->bitmap, x, y))
+	{
+	  top = y;
+	  x = w;
+	  y = h;
+	}
+
+  for (y = h - 1; y >= 0; y--)
+    for (x = 0; x < w; x++)
+      if (pbm_getpixel (m->bitmap, x, y))
+	{
+	  bot = y;
+	  x = w;
+	  y = -1;
+	}
+
+  for (x = 0; x < w; x++)
+    for (y = 0; y < h; y++)
+      if (pbm_getpixel (m->bitmap, x, y))
+	{
+	  left = x;
+	  x = w;
+	  y = h;
+	}
+
+  for (x = w - 1; x >= 0; x--)
+    for (y = 0; y < h; y++)
+      if (pbm_getpixel (m->bitmap, x, y))
+	{
+	  right = x;
+	  x = -1;
+	  y = h;
+	}
+
+  temp.w = right - left + 1;
+  temp.h = bot - top + 1;
+  temp.xpos = xpos + left;
+  temp.ypos = ypos + top;
+  temp.bitmap = pbm_allocarray (temp.w, temp.h);
+
+  for (x = 0; x < temp.w; x++)
+    for (y = 0; y < temp.h; y++)
+      pbm_putpixel (temp.bitmap, x, y, pbm_getpixel (m->bitmap, x + left, y + top));
+
+  pbm_freearray (&(m->bitmap), h);
+
+  marktype_calc_features (&temp, "?");
+  *m = temp;
+}
+
+
+
+
+
+
+/* returns a newly created mark reprenting the average of the marklist */
+void 
+marklist_average (marklistptr list, marktype * avgmark)
+{
+  int w = 0, h = 0;
+  int t = INT_MAX, b = 0, l = INT_MAX, r = 0, matched = 0;
+  marktype d, temp;
+  marklistptr step;
+  static int a[100][100];
+  int x, y;
+
+  if (list == NULL)
+    error_msg ("marklist_average", "Can't average 'nothing'!", "");
+
+  for (step = list; step; step = step->next)
+    {
+      matched++;
+      temp = step->data;
+      w = max (w, temp.w);
+      h = max (h, temp.h);
+      r = max (r, temp.xcen);
+      l = min (l, temp.xcen);
+      b = max (b, temp.ycen);
+      t = min (t, temp.ycen);
+    }
+  w = w + (r - l);
+  h = h + (b - t);
+  if ((w >= 100) || (h >= 100))
+    matched = 0;
+
+  if (matched > 1)
+    {
+      int x_off, y_off;
+      for (x = 0; x < w; x++)
+	for (y = 0; y < h; y++)
+	  a[x][y] = 0;
+
+      for (step = list; step; step = step->next)
+	{
+	  x_off = r - step->data.xcen;
+	  y_off = b - step->data.ycen;
+	  for (x = 0; x < step->data.w; x++)
+	    for (y = 0; y < step->data.h; y++)
+	      if (pbm_getpixel (step->data.bitmap, x, y))
+		a[x + x_off][y + y_off]++;
+	}
+
+      d.w = w;
+      d.h = h;
+      d.bitmap = pbm_allocarray (w, h);
+      for (x = 0; x < w; x++)
+	for (y = 0; y < h; y++)
+	  if (a[x][y] > max (1, (matched / 2)))		/* > gives better CR than >= */
+	    pbm_putpixel (d.bitmap, x, y, 1);
+      marktype_adj_bound (&d);
+      strcpy (d.name, "?");
+      marktype_calc_centroid (&d);
+    }
+  else
+    d = marktype_copy (list->data);
+
+  *avgmark = d;
+}
+
+
+
+
+
+
+void 
+marklist_setnull (marklistptr * list)
+{
+  *list = NULL;
+}
+
+
+marklistptr 
+marklist_add (marklistptr * list, marktype m)
+{
+  marklistptr p;
+
+  if (*list == NULL)
+    {
+      *list = (marklistptr) malloc (sizeof (marklisttype));
+      if ((*list) == NULL)
+	error_msg ("marklist_add", "OUT of Memory in marklist_add", "");
+      p = *list;
+    }
+  else
+    {
+      p = *list;
+      while (p->next != NULL)
+	p = p->next;
+      p->next = (marklistptr) malloc (sizeof (marklisttype));
+      if (p->next == NULL)
+	error_msg ("marklist_add", "OUT of Memory in marklist_add", "");
+      p = p->next;
+    }
+  p->data = m;
+  p->next = NULL;
+  return p;
+
+}
+
+
+
+marklistptr 
+marklist_addcopy (marklistptr * list, marktype m)
+{
+  marktype d;
+
+  d = marktype_copy (m);
+  return marklist_add (list, d);
+}
+
+
+
+int 
+marklist_length (marklistptr list)
+{
+  int count = 0;
+
+  while (list)
+    {
+      count++;
+      list = list->next;
+    }
+  return count;
+}
+
+
+
+
+
+void 
+marklist_dump (marklistptr list)
+{
+  int count = 0;
+  marktype d;
+
+  while (list)
+    {
+      d = list->data;
+      marktype_dump (d);
+
+      count++;
+      list = list->next;
+    }
+}
+
+
+
+
+/* pos starts from 0, to up marklist_length-1 */
+marklistptr 
+marklist_getat (marklistptr list, int pos, marktype * d)
+{
+  int count = 0;
+
+  while (list)
+    {
+      if (count == pos)
+	{
+	  *d = list->data;
+	  return list;
+	}
+      count++;
+      list = list->next;
+    }
+  fprintf (stderr, "marklist_getat(): access off ends of list: pos %d\n", pos);
+  return NULL;
+}
+
+
+
+void 
+marklist_free (marklistptr * list)
+{
+  marklistptr n;
+
+  while ((*list) != NULL)
+    {
+      n = (*list)->next;
+      pbm_freearray (&((*list)->data.bitmap), (*list)->data.h);
+      free ((*list));
+      (*list) = n;
+    }
+}
+
+
+
+int 
+marklist_removeat (marklistptr * list, int pos)
+{
+  marklistptr c, n, p = NULL;
+  int count = 0;
+
+  if ((pos == 0) && (*list))
+    {				/* if removing the 1st element */
+      *list = (*list)->next;
+      return 1;
+    }
+  else
+    {
+      c = *list;
+      while (c)
+	{
+	  if (count == pos)
+	    {			/* two cases, no previous, >=1 previous */
+	      if (p == NULL)	/* no previous */
+		{
+		  n = c->next;
+		  pbm_freearray (&c->data.bitmap, c->data.h);
+		  free (c);
+		  c = n;
+		  return 1;
+		}
+	      else
+		/* >=1 previous */
+		{
+		  n = c->next;
+		  pbm_freearray (&c->data.bitmap, c->data.h);
+		  free (c);
+		  p->next = n;
+		  return 1;
+		}
+	    }
+	  count++;
+	  p = c;
+	  c = c->next;
+	}
+    }
+  fprintf (stderr, "marklist_removeat(): access off ends of list: pos %d\n", pos);
+  return 0;			/* nothing was removed */
+}
+
+
+
+marklistptr 
+marklist_next (marklistptr list)
+{
+  return list->next;
+}
+
+
+/* only writes out ascii form */
+void 
+write_library_mark (FILE * fp, marktype d)
+{
+  int r, c, rows, cols;
+  int xtot = d.xcen, ytot = d.ycen;
+
+  rows = d.h;
+  cols = d.w;
+
+  fprintf (fp, "Mark: %d\n", d.symnum);
+  fprintf (fp, "Char: %s\n", d.name);
+  fprintf (fp, "Cols: %d Rows: %d\n", d.w, d.h);
+  for (r = 0; r < d.h; r++)
+    {
+      for (c = 0; c < d.w; c++)
+	if ((r == ytot) && (c == xtot))
+	  putc (pbm_getpixel (d.bitmap, c, r) ? 'C' : '!', fp);
+	else
+	  putc (pbm_getpixel (d.bitmap, c, r) ? 'X' : '.', fp);
+      putc ('\n', fp);
+    }
+}
+
+
+
+/* only reads in ascii library */
+int 
+read_library_mark (FILE * fp, marktype * d)
+{
+  int r, c, rows, cols, numread, ch1 = 'C';
+  char str1[100], str2[100];
+
+  numread = fscanf (fp, "%s %d\n", str1, &(d->symnum));
+  if ((numread != 2) || (strcmp (str1, "Mark:") != 0))
+    return 0;
+  fscanf (fp, "%s %s\n", str1, (d->name));
+  fscanf (fp, "%s %d %s %d\n", str1, &(d->w), str2, &(d->h));
+  cols = d->w;
+  rows = d->h;
+  d->bitmap = pbm_allocarray (cols, rows);
+  for (r = 0; r < rows; r++)
+    {
+      for (c = 0; c < cols; c++)
+	{
+	  ch1 = getc (fp);
+	  if ((ch1 == 'X') || (ch1 == 'C'))
+	    pbm_putpixel (d->bitmap, c, r, 1);
+	  else
+	    pbm_putpixel (d->bitmap, c, r, 0);
+	}
+      ch1 = getc (fp);
+    }
+  marktype_calc_features (d, "?");
+
+  return 1;
+}
+
+
+
+
+
+
+
+int 
+read_library (char libraryname[], marklistptr * library)
+{
+  int err, count;
+  FILE *lib;
+  marktype d;
+  marklistptr step;
+
+  lib = fopen (libraryname, "rb");
+  if (lib == NULL)
+    error_msg ("read_library()", "Trouble opening library file", "");
+  *library = NULL;
+  count = 0;
+  while (!isEOF (lib))
+    {
+      err = read_library_mark (lib, &d);
+      if (!err)
+	error_msg ("read_library()", "unknown format of the library file.", "");
+      d.symnum = count;
+      count++;
+      if ((*library) == NULL)
+	step = marklist_add (library, d);
+      else
+	step = marklist_add (&step, d);
+    }
+  count = marklist_length (*library);
+  fclose (lib);
+
+  return count;
+}
+
+
+
+
+
+
+/**********************************************************************
+
+'elem' routines for coding offsets
+
+the elem datatype contains the 3 params for arithmetic coding,
+stored as a no-duplicates order linked list.
+
+************************************************************************/
+
+
+
+
+void 
+elem_add (elemlist * list, int num)
+{
+  elemptr temp, nn;
+  int p, f;
+
+  list->tot++;
+  if (list->head == NULL)
+    {
+      list->head = (elemptr) malloc (sizeof (elem));
+      if ((list->head) == NULL)
+	error_msg ("", "ekk", "");
+      temp = list->head;
+      temp->next = NULL;
+      temp->num = num;
+      temp->pos = 0;
+      temp->freq = 1;
+    }
+  else
+    {				/* add to list */
+      temp = list->head;
+      p = 0;
+      f = 0;
+      while ((num > temp->num) && (temp->next != NULL))
+	{
+	  p = temp->pos;
+	  f = temp->freq;
+	  temp = temp->next;
+	}
+
+      if (temp->num == num)
+	temp->freq++;
+      else
+	{			/* insert it */
+	  nn = (elemptr) malloc (sizeof (elem));
+	  if (nn == NULL)
+	    error_msg ("", "ekk", "");
+
+	  if (num < temp->num)
+	    {
+	      nn->num = temp->num;
+	      nn->pos = temp->pos;
+	      nn->freq = temp->freq;
+	      nn->next = temp->next;
+	      temp->num = num;
+	      temp->pos = p + f;
+	      temp->freq = 1;
+	      temp->next = nn;
+	    }
+	  else
+	    {
+	      nn->next = temp->next;
+	      temp->next = nn;
+	      nn->pos = temp->pos + temp->freq;
+	      nn->num = num;
+	      nn->freq = 1;
+	      temp = nn;
+	    }
+	}
+    }
+  /* update frequencies */
+  temp = temp->next;
+  if (temp)
+    do
+      {
+	temp->pos++;
+	temp = temp->next;
+      }
+    while (temp);
+}
+
+
+void 
+elem_free (elemlist * list)
+{
+  elemptr n, p;
+
+  p = list->head;
+  while (p != NULL)
+    {
+      n = p->next;
+      free (p);
+      p = n;
+    }
+  list->head = NULL;
+}
+
+int 
+elem_find (elemlist * list, int n, int *p, int *f, int *t)
+{
+  elemptr temp;
+
+  temp = list->head;
+  while (temp)
+    {
+      if (temp->num == n)
+	{
+	  *p = temp->pos;
+	  *f = temp->freq + temp->pos;
+	  *t = list->tot;
+	  return 1;
+	}
+      temp = temp->next;
+    }
+  return 0;
+}
+
+
+int 
+elem_getrange (elemlist * list, int range, int *num)
+{
+  elemptr temp;
+
+  temp = list->head;
+  while (temp)
+    {
+      if ((range >= temp->pos) && (range < (temp->pos + temp->freq)))
+	{
+	  *num = temp->num;
+	  return 1;
+	}
+      temp = temp->next;
+    }
+  return 0;
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/images/marklist.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/images/marklist.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/images/marklist.h	(revision 699)
@@ -0,0 +1,114 @@
+/**************************************************************************
+ *
+ * marklist.h -- Functions managing the list of marks
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef __MARKLIST_H
+#define __MARKLIST_H
+
+#include "sysfuncs.h"
+#include "basictypes.h"
+
+typedef struct
+  {
+    int symnum;
+    int xpos, ypos;
+    int w, h;
+    int xcen, ycen;
+    int xoffset, yoffset;
+    char name[10];
+    Pixel **bitmap;
+
+    int xcen1, ycen1;
+    int xcen2, ycen2;
+    int xcen3, ycen3;
+    int xcen4, ycen4;
+    int set;
+    int h_runs;
+    int v_runs;
+  }
+marktype;
+
+
+typedef struct marklisttype *marklistptr;
+typedef struct marklisttype
+  {
+    marktype data;
+    marklistptr next;
+  }
+marklisttype;
+
+
+void marktype_dump (marktype d);
+marktype marktype_copy (marktype d);
+
+float marktype_density (marktype d);
+void marktype_calc_centroid (marktype * b);
+void marktype_calc_features (marktype * d, char str[]);
+void marktype_adj_bound (marktype * m);
+
+void marklist_average (marklistptr list, marktype * avgmark);
+void marklist_setnull (marklistptr * list);
+marklistptr marklist_add (marklistptr * list, marktype m);
+marklistptr marklist_addcopy (marklistptr * list, marktype m);
+int marklist_length (marklistptr list);
+void marklist_dump (marklistptr list);
+marklistptr marklist_getat (marklistptr list, int pos, marktype * d);
+void marklist_free (marklistptr * list);
+int marklist_removeat (marklistptr * list, int pos);
+marklistptr marklist_next (marklistptr list);
+
+void write_library_mark (FILE * fp, marktype d);
+int read_library_mark (FILE * fp, marktype * d);
+int read_library (char libraryname[], marklistptr * library);
+
+
+
+
+
+
+/* 
+   The 'elem' structure is an ordered linked list, with fields corresponding
+   directly to the arithmetic coding routines.  
+ */
+typedef struct elem *elemptr;
+typedef struct elem
+  {
+    int num;
+    int pos;
+    int freq;
+    elemptr next;
+  }
+elem;
+
+typedef struct elemlist
+  {
+    elemptr head;
+    int tot;
+  }
+elemlist;
+
+void elem_add (elemlist * list, int num);
+void elem_free (elemlist * list);
+int elem_find (elemlist * list, int n, int *p, int *f, int *t);
+int elem_getrange (elemlist * list, int range, int *num);
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/src/images/match.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/images/match.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/images/match.c	(revision 699)
@@ -0,0 +1,417 @@
+/**************************************************************************
+ *
+ * match.c -- Functions related to matching marks
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "marklist.h"
+#include "pbmtools.h"
+#include "utils.h"
+
+
+/* error mask goes from 0..8, single pixel errors are worth nothing */
+int 
+WXOR_match (marktype b1, marktype b2)
+{
+  int r, c, i, j;
+  int minr, minc;
+  int maxr, maxc;
+  int offx, offy;
+  int p1, p2;
+  int count = 0;
+  marktype d;
+
+  offx = b1.xcen - b2.xcen;
+  offy = b1.ycen - b2.ycen;
+
+  minc = min (0, offx);
+  minr = min (0, offy);
+
+  maxc = max (b1.w, b2.w + offx);
+  maxr = max (b1.h, b2.h + offy);
+
+  d.w = maxc - minc + 1;
+  d.h = maxr - minr + 1;
+  d.bitmap = pbm_allocarray (d.w, d.h);
+
+
+  for (r = minr; r < maxr; r++)
+    for (c = minc; c < maxc; c++)
+      {
+	if ((r >= 0) && (c >= 0) && (r < b1.h) && (c < b1.w))
+	  p1 = pbm_getpixel (b1.bitmap, c, r);
+	else
+	  p1 = 0;		/* 0 = back ground colour */
+	if ((r - offy >= 0) && (c - offx >= 0) && (r - offy < b2.h) && (c - offx < b2.w))
+	  p2 = pbm_getpixel (b2.bitmap, c - offx, r - offy);
+	else
+	  p2 = 0;		/* background colour */
+	pbm_putpixel (d.bitmap, c - minc, r - minr, p1 != p2);
+      }
+  for (r = minr; r < maxr; r++)
+    for (c = minc; c < maxc; c++)
+      if (pbm_getpixel (d.bitmap, c - minc, r - minr))
+	for (i = -1; i <= 1; i++)
+	  for (j = -1; j <= 1; j++)
+	    if ((r + j >= 0) && (c + i >= 0) && (r + j < d.h) && (c + i < d.w) && ((i != 0) || (j != 0)) && (pbm_getpixel (d.bitmap, c + i - minc, r + j - minr)))
+	      count++;
+  pbm_freearray (&d.bitmap, d.h);
+  return count;
+}
+
+
+
+/* define an array for most matches, if it is a massive mark, then WXOR is called */
+#define S_SIZE 500
+
+static unsigned char scratch[S_SIZE][S_SIZE], sc2[S_SIZE][S_SIZE];
+
+
+
+
+/* Weighted AND-NOT */
+
+int 
+WAN_match (marktype b1, marktype b2)
+{
+  int r, c, i, j;
+  int minr, minc;
+  int maxr, maxc;
+  int offx, offy;
+  int p1, p2;
+  int count = 0;
+
+  offx = b1.xcen - b2.xcen;
+  offy = b1.ycen - b2.ycen;
+
+  minc = min (0, offx);
+  minr = min (0, offy);
+
+  maxc = max (b1.w, b2.w + offx);
+  maxr = max (b1.h, b2.h + offy);
+
+  if (((maxc - minc) >= S_SIZE) || ((maxr - minr) >= S_SIZE))
+    return WXOR_match (b1, b2);
+
+  for (r = minr; r < maxr; r++)
+    for (c = minc; c < maxc; c++)
+      {
+	scratch[r - minr][c - minc] = sc2[r - minr][c - minc] = 0;
+	if ((r >= 0) && (c >= 0) && (r < b1.h) && (c < b1.w))
+	  p1 = pbm_getpixel (b1.bitmap, c, r);
+	else
+	  p1 = 0;		/* 0 == background colour */
+	if ((r - offy >= 0) && (c - offx >= 0) && (r - offy < b2.h) && (c - offx < b2.w))
+	  p2 = pbm_getpixel (b2.bitmap, c - offx, r - offy);
+	else
+	  p2 = 0;
+
+	if ((p1 == 0) && (p2 == 1))	/* p1 not set, p2 = set */
+	  scratch[r - minr][c - minc] = 1;
+	if ((p1 == 1) && (p2 == 0))	/* p1 set, p2 not set */
+	  sc2[r - minr][c - minc] = 1;
+      }
+
+  count = 0;
+  for (r = minr; r < maxr; r++)
+    for (c = minc; c < maxc; c++)
+      if (scratch[r - minr][c - minc])
+	for (i = -1; i <= 1; i++)
+	  for (j = -1; j <= 1; j++)
+	    if ((r + j >= minr) && (c + i >= minc) && (r + j < maxr) && (c + i < maxc) && ((i != 0) || (j != 0)) && scratch[r - minr + j][c - minc + i])
+	      count++;
+
+  for (r = minr; r < maxr; r++)
+    for (c = minc; c < maxc; c++)
+      if (sc2[r - minr][c - minc])
+	for (i = -1; i <= 1; i++)
+	  for (j = -1; j <= 1; j++)
+	    if ((r + j >= minr) && (c + i >= minc) && (r + j < maxr) && (c + i < maxc) && ((i != 0) || (j != 0)) && sc2[r - minr + j][c - minc + i])
+	      count++;
+
+  return count;
+}
+
+/* oc 8 must be in a clock wise order, with oc[0] a corner */
+struct
+{
+  int x, y;
+}
+oc[8] =
+{
+  {
+    -1, -1
+  }
+  ,
+  {
+    0, -1
+  }
+  ,
+  {
+    1, -1
+  }
+  ,
+  {
+    1, 0
+  }
+  ,
+  {
+    1, 1
+  }
+  ,
+  {
+    0, 1
+  }
+  ,
+  {
+    -1, 1
+  }
+  ,
+  {
+    -1, 0
+  }
+};
+
+
+
+static void 
+xormatchscratch (marktype * b1, marktype * b2, int minr, int maxr, int minc, int maxc, int offx, int offy)
+{
+  int r, c, rmm, roo, coo;
+  int p1, p2;
+  Pixel **bm1 = b1->bitmap, **bm2 = b2->bitmap;
+  int w1 = b1->w, h1 = b1->h, w2 = b2->w, h2 = b2->h;
+
+  for (r = minr; r < maxr; r++)
+    {
+      rmm = r - minr;
+      roo = r - offy;
+      p1 = 0;
+      for (c = minc; c < 0; c++)
+	{
+	  coo = c - offx;
+	  scratch[rmm][c - minc] = sc2[rmm][c - minc] = 0;
+	  if ((coo >= 0) && (roo >= 0) && (coo < w2) && (roo < h2))
+	    p2 = pbm_getpixel (bm2, coo, roo);
+	  else
+	    p2 = 0;		/* background */
+	  scratch[rmm][c - minc] = p1 != p2;
+	}
+      for (c = 0; c < maxc; c++)
+	{
+	  coo = c - offx;
+	  scratch[rmm][c - minc] = sc2[rmm][c - minc] = 0;
+	  if ((r >= 0) && (c < w1) && (r < h1))
+	    p1 = pbm_getpixel (bm1, c, r);
+	  else
+	    p1 = 0;		/* background */
+	  if ((coo >= 0) && (roo >= 0) && (coo < w2) && (roo < h2))
+	    p2 = pbm_getpixel (bm2, coo, roo);
+	  else
+	    p2 = 0;		/* background */
+	  scratch[rmm][c - minc] = p1 != p2;
+	}
+    }
+}
+
+
+
+int 
+CSIS_match (marktype * b1, marktype * b2)
+{
+  int r, c, i;
+  int minr, minc;
+  int maxr, maxc;
+  int offx, offy, fepcount;
+  int totscratch = 0;
+
+
+  offx = b1->xcen - b2->xcen;
+  offy = b1->ycen - b2->ycen;
+
+  minc = min (0, offx);
+  minr = min (0, offy);
+
+  maxc = max (b1->w, b2->w + offx);
+  maxr = max (b1->h, b2->h + offy);
+
+  if (((maxc - minc) >= S_SIZE) || ((maxr - minr) >= S_SIZE))
+    return WXOR_match (*b1, *b2);
+
+  /* set the scratch array to 1 if there is an XOR match */
+
+  xormatchscratch (b1, b2, minr, maxr, minc, maxc, offx, offy);
+  /* We how have the XOR map. Scratch contains the number of neighbours, 1=1 neighbour set...etc */
+
+  /* CSIS rule 1, if FEP (four error pels) then exit */
+  fepcount = 0;
+  for (r = minr; r < maxr - 1; r++)
+    for (c = minc; c < maxc - 1; c++)
+      if ((scratch[r - minr][c - minc]) && (scratch[r - minr + 1][c - minc]) &&
+	  (scratch[r - minr][c - minc + 1]) && (scratch[r - minr + 1][c - minc + 1]))
+	{
+	  fepcount++;
+	  if (fepcount == 1)
+	    return 10001;
+	}
+
+  /* foreach set pixel, update scratch with the number of pels around in five dir's. 
+     FPN (five-pel neighbour hood) */
+  for (r = minr; r < maxr; r++)
+    for (c = minc; c < maxc; c++)
+      if (scratch[r - minr][c - minc])
+	{
+	  sc2[r - minr][c - minc] = 1;	/* itself */
+	  for (i = 1; i < 8; i += 3)
+	    if (((r + oc[i].y) >= minr) && ((c + oc[i].x) >= minc) && ((r + oc[i].y) < maxr) && ((c + oc[i].x) < maxc))
+	      if (scratch[r + oc[i].y - minr][c + oc[i].x - minc])
+		sc2[r - minr][c - minc]++;
+	}
+  /* now scratch has the value of the FPN - 1=single error, 5=4 neighbouring errors etc... */
+
+  /* set scratch and sc2 to be the same */
+  for (r = minr; r < maxr; r++)
+    for (c = minc; c < maxc; c++)
+      totscratch += scratch[r - minr][c - minc] = sc2[r - minr][c - minc];
+
+  for (r = minr; r < maxr; r++)
+    for (c = minc; c < maxc; c++)
+      {
+	/* CSIS rule 2i: an error pel has 2 or more neighbours---this num is the same as PMS */
+	if (scratch[r - minr][c - minc] >= 3)
+	  if ((r - minr >= 1) && (c - minc >= 1) && (r - minr < (maxr - 1)) && (c - minc < (maxc - 1)))
+	    {
+	      int i, numsep = 0;
+	      /* find how many neighbours a set pixel has, only check the 4 edges-FPN */
+	      for (i = 1; i < 8; i += 2)
+		if (scratch[r + oc[i].y - minr][c + oc[i].x - minc])	/* if a scratch pixel is set */
+		  {
+		    if ((scratch[r + oc[(i + 2) % 8].y - minr][c + oc[(i + 2) % 8].x - minc] == 0) &&
+			(scratch[r + oc[(i + 6) % 8].y - minr][c + oc[(i + 6) % 8].x - minc] == 0)
+		      )
+		      numsep++;
+		  }
+
+	      if ((b1->w >= 12) || (b2->w >= 12) || (b1->h >= 12) || (b2->h >= 12))
+		{
+
+		  /* PMS rule 2ii: at least two of its neigh. error p's aren't connected */
+		  if (numsep >= 2)
+		    {
+		      int i, j, p, p2, same;
+
+		      /* PMS rule 2(iii), look for regions in the original images of 3x3 the same */
+		      if ((c >= 0) && (r >= 0) && (c < b1->w) && (r < b1->h))
+			{
+			  p = pbm_getpixel (b1->bitmap, c, r);
+			  same = 0;
+			  for (i = -1; i <= 1; i++)
+			    for (j = -1; j <= 1; j++)
+			      {
+				if ((c + i >= 0) && (r + j >= 0) && (c + i < b1->w) && (r + j < b1->h))
+				  p2 = pbm_getpixel (b1->bitmap, c + i, r + j);
+				else
+				  p2 = 0;
+				if (p2 == p)
+				  same++;
+			      }	/* i,j loop */
+			  /* PMS rule 2(iii) */
+			  if (same == 9)
+			    {
+			      return 10002;
+			    }
+			}	/* (*b1) in range */
+
+		      if ((c - offx >= 0) && (r - offy >= 0) && (c - offx < b2->w) && (r - offy < b2->h))
+			{
+			  p = pbm_getpixel (b2->bitmap, c - offx, r - offy);
+			  same = 0;
+			  for (i = -1; i <= 1; i++)
+			    for (j = -1; j <= 1; j++)
+			      {
+				if ((c + i - offx >= 0) && (r + j - offy >= 0) && (c + i - offx < b2->w) && (r + j - offy < b2->h))
+				  p2 = pbm_getpixel (b2->bitmap, c + i - offx, r + j - offy);
+				else
+				  p2 = 0;
+				if (p2 == p)
+				  same++;
+			      }	/* i,j loop */
+			  /* PMS rule 2(iii) */
+			  if (same == 9)
+			    {
+			      return 10003;
+			    }
+			}
+		    }
+		}
+	      else
+		{		/* do the CSIS rule */
+		  /* CSIS rule 2ii: at least two of its neigh. error p's aren't connected */
+		  if (numsep >= 2)
+		    {
+		      int i, p, p2, same;
+
+		      /* PMS rule 2(iii), look for regions in the original images of 3x3 the same */
+		      if ((c >= 0) && (r >= 0) && (c < b1->w) && (r < b1->h))
+			{
+			  p = pbm_getpixel (b1->bitmap, c, r);
+			  same = 1;
+			  for (i = 1; i < 8; i += 2)
+			    if ((c + oc[i].x >= 0) && (r + oc[i].y >= 0) && (c + oc[i].x < b1->w) && (r + oc[i].y < b1->h))
+			      p2 = pbm_getpixel (b1->bitmap, c + oc[i].x, r + oc[i].y);
+			    else
+			      p2 = 0;
+			  if (p2 == p)
+			    same++;
+
+			  /* CSIS rule 2(iii) */
+			  if (same == 5)
+			    {
+			      return 10002;
+			    }
+			}
+
+		      if ((c - offx >= 0) && (r - offy >= 0) && (c - offx < b2->w) && (r - offy < b2->h))
+			{
+			  p = pbm_getpixel (b2->bitmap, c - offx, r - offy);
+			  same = 1;
+			  for (i = 1; i < 8; i += 2)
+			    if ((c + oc[i].x - offx >= 0) && (r + oc[i].y - offy >= 0) && (c + oc[i].x - offx < b2->w) && (r + oc[i].y - offy < b2->h))
+			      p2 = pbm_getpixel (b2->bitmap, c + oc[i].x - offx, r + oc[i].y - offy);
+			    else
+			      p2 = 0;
+			  if (p2 == p)
+			    same++;
+
+			  /* CSIS rule 2(iii) */
+			  if (same == 5)
+			    {
+			      return 10003;
+			    }
+			}
+		    }
+		}
+	    }
+      }
+
+  return totscratch;
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/images/match.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/images/match.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/images/match.h	(revision 699)
@@ -0,0 +1,47 @@
+/**************************************************************************
+ *
+ * match.h -- Functions related to matching marks
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef __MATCH_H
+#define __MATCH_H
+
+#include "marklist.h"
+#include "pbmtools.h"
+#include "utils.h"
+
+
+/*
+   #define NOT_SCREENED(b1,b2) ((abs(b1.w-b2.w)<=4) && (abs(b1.h-b2.h)<=4))
+ */
+
+#define NOT_SCREENED(b1,b2) ((abs(b1.xcen-b2.xcen)+abs(b1.ycen-b2.ycen))<=3)
+
+#define MATCH_VAL 1000
+#define MATCH(b1,b2) (CSIS_match(b1,b2))
+
+int XOR_match (marktype b1, marktype b2);
+int WXOR_match (marktype b1, marktype b2);
+int WAN_match (marktype b1, marktype b2);
+int PMS_match (marktype b1, marktype b2);
+int CSIS_match (marktype * b1, marktype * b2);
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/src/images/mgbilevel.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/images/mgbilevel.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/images/mgbilevel.1	(revision 699)
@@ -0,0 +1,116 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgbilevel 1 \*(Dt CITRI
+.SH NAME
+mgbilevel \- compresses a bilevel image using a specified template
+.SH SYNOPSIS
+.B mgbilevel
+.B \-e
+[
+.B \-t
+.I \*(lqstring\*(rq
+]
+.I infile
+[
+.I \>compressed-outfile
+]
+.nf
+or
+.fi
+.B mgbilevel
+.B \-d
+.I compressed-infile
+[
+.I \>outfile
+]
+.SH DESCRIPTION
+.B mgbilevel
+performs (up to) a two-stage template-based compression.  The
+user can specify the template to be used in the compression process.  The
+default template size is a two-stage template 22/10.  The compressed file
+contains the template type in the stream.  The output files are written
+to
+.IR stdout .
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-t\fP \fI\*(lqstring\*(rq\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-e
+Compresses
+.I infile
+using a template specified, and writes to
+.IR compressed-outfile .
+.TP
+.B \-d
+Decompresses
+.I compressed-infile
+using the template contained in the compressed file.  Writes to
+.IR outfile .
+.TP
+\fB\-t \fP\fI\*(lqstring\*(rq\fP
+This option specifies which template combination is to
+be used in the compression.  Five different characters
+can occur in the string: `.' for a pixel not
+included in the template, `p' for a pixel included in the 1st
+level template, `2' for a pixel included in the 2nd level
+template (a subset of the 1st level), `*' denoting the position
+of the
+.I current
+pixel, and a `;' denoting the end of each template line.
+The default template is a two-level 22/10 template:
+.IP
+.RS
+.nf
+\&.ppppp.;
+pp222pp;
+p22222p;
+p22*.\|.\|.;
+.fi
+.RE
+.IP
+This would be specified on the command line as:
+.nf
+\&\fB\-T\fP ".ppppp.;pp222pp;p22222p;p22*.\|.\|.;"
+.fi
+.TP
+.I infile
+This is the input
+.I .pbm
+file to be compressed.
+.TP
+.I outfile
+This is the decompressed
+.I .pbm
+file.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /tags/gsdl-21/gsdl/packages/mg/src/images/mgbilevel.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/images/mgbilevel.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/images/mgbilevel.c	(revision 699)
@@ -0,0 +1,149 @@
+/**************************************************************************
+ *
+ * mgbilevel.c -- Program to compress/decompress bilevel images
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "utils.h"
+#include "pbmtools.h"
+#include "arithcode.h"
+#include "bilevel.h"
+
+
+
+static char default_template[] =
+{
+  ".ppppp.;"
+  "pp222pp;"
+  "p22222p;"
+  "p22*...;"
+};
+
+void 
+Encode_bitmap (marktype b, char template[])
+{
+  InitArithEncoding ();
+
+  bl_compress (b, template);
+
+  CloseDownArithEncoding ();
+}
+
+void 
+Decode_bitmap (marktype * b)
+{
+  InitArithDecoding ();
+
+  bl_decompress (b);
+
+  CloseDownArithDecoding ();
+}
+
+
+void 
+usage ()
+{
+  fprintf (stderr,
+	   "usage:\n"
+	   "       mgbilevel -e  [-t \"string\"]  infile      >compressed\n"
+	   "or\n"
+	   "       mgbilevel -d                 compressed  >outfile\n");
+  exit (1);
+}
+
+#define BUF_SIZE 65536
+
+void 
+main (int argc, char *args[])
+{
+  marktype d;
+  char *filename = NULL, *template = NULL;
+  Pixel **bitmap;
+  int encode = 0, decode = 0, i;
+  char buffer[BUF_SIZE];
+
+  if (argc < 2)
+    usage ();
+  template = default_template;
+  for (i = 1; i < argc; i++)
+    {
+      if (strcmp (args[i], "-e") == 0)
+	encode = 1;
+      else if (strcmp (args[i], "-d") == 0)
+	decode = 1;
+      else if (strcmp (args[i], "-h") == 0)
+	usage ();
+      else if (strcmp (args[i], "-t") == 0)
+	{
+	  template = args[i + 1];
+	  i++;
+	}
+      else if (args[i][0] == '-')
+	error_msg (args[0], "unknown switch:", args[i]);
+      else if (!filename)
+	filename = args[i];
+      else
+	error_msg (args[0], "only specify 1 filename", "");
+    }
+  if (encode == decode)
+    {
+      fprintf (stderr, "You must specify either encode XOR decode.\n");
+      exit (1);
+    }
+
+  if (!filename)
+    {
+      fprintf (stderr, "You must specify a filename.\n");
+      exit (1);
+    }
+
+  if (encode)
+    {
+      bitmap = pbm_readnamedfile (filename, &d.w, &d.h);
+      if (bitmap == NULL)
+	error_msg (args[0], "problem reading bitmap", "");
+
+      arith_out = stdout;
+      setbuffer (stdout, buffer, BUF_SIZE);
+      magic_write (stdout, MAGIC_BILEVEL);
+      if (bitmap)
+	{
+	  d.bitmap = bitmap;
+	  Encode_bitmap (d, template);
+	}
+    }
+  else if (decode)
+    {
+      FILE *tpin;
+      tpin = fopen (filename, "rb");
+      if (!tpin)
+	error_msg ("mgbilevel", "Can't open file:", filename);
+      arith_in = tpin;
+      setbuffer (tpin, buffer, BUF_SIZE);
+      magic_check (tpin, MAGIC_BILEVEL);
+      Decode_bitmap (&d);
+      fclose (tpin);
+
+      setbuffer (stdout, buffer, BUF_SIZE);
+      pbm_writefile (stdout, d.bitmap, d.w, d.h);
+    }
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/images/mgfelics.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/images/mgfelics.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/images/mgfelics.1	(revision 699)
@@ -0,0 +1,103 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgfelics 1 \*(Dt CITRI
+.SH NAME
+mgfelics \- fast and efficient lossless image (de)compression
+.SH SYNOPSIS
+.B mgfelics
+.B \-e
+[
+.BI \-k " n"
+]
+.I infile
+[
+.I \>compressed-outfile
+]
+.br
+or
+.br
+.B mgfelics
+.B \-d
+.I compressed-infile
+[
+.I \>outfile
+]
+.SH DESCRIPTION
+.B mgfelics
+is a lossless compression process which can (de)compress portable
+bitmap files in
+.I pgm
+format.
+.B mgfelics
+writes to
+.IR stdout .
+.SH OPTIONS
+.TP "\w'\fB\-k\fP \fIn\fP'u+2n"
+.BI \-k " n"
+An integer parameter (< 32) which controls the compression speed and
+ratio.  The value of
+.I n
+is included in the output stream, and therefore is not needed in the
+decompression stage.  The default value of
+.I n
+is equal to the number of bits required to represent each grey level.
+.TP
+.B \-e
+Encodes the file
+.IR infile ,
+sending the output to
+.IR stdout .
+.TP
+.B \-d
+Decodes the file from
+.IR compressed-infile ,
+sending the output to
+.IR stdout .
+.SH NOTE
+.B mgfelics
+uses a combination of adjusted binary and Rice codes, the choice of
+which for any given pixel depends on the environment as determined by
+the intensity values of neighbouring pixels.  The result is a very
+fast and lossless algorithm for compressing grayscale images.  The
+optional parameter
+.BI \-k " n"
+corresponds to the maximum exponent of the same name in Rice encoding.
+Smaller values of
+.BI \-k " n"
+can result in significant improvement in compression times with a
+corresponding loss in compression ratio.  However the reduction in
+compression ratio with a small
+.BI \-k " n"
+parameter may be insignificant for many images.
+.SH REFERENCES
+Algorithm by Paul G. Howard and Jeffrey Scott Vitter
+of Brown University, RI, and Duke University, NC, respectively.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /tags/gsdl-21/gsdl/packages/mg/src/images/mgfelics.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/images/mgfelics.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/images/mgfelics.c	(revision 699)
@@ -0,0 +1,155 @@
+/**************************************************************************
+ *
+ * mgfelics.c -- Program to compress/decompress grey scale images
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "utils.h"
+
+
+#define MAXKVALUE 32
+
+
+
+void encode (unsigned int *line, int width, int height, int maxval, int maxk,
+	     FILE * fp_in, FILE * fp_out, int (*get) (FILE *));
+
+void decode (unsigned int *line, int width, int height, int maxval, int maxk,
+	     FILE * fp_in, FILE * fp_out);
+
+
+void 
+usage (char *prog)
+{
+  fprintf (stderr, "usage:\n");
+  fprintf (stderr, "\t%s -e [-k n]   infile    [>compressed]\n"
+	   "or\n"
+	   "\t%s -d  compressed-infile   [>outfile]\n", prog, prog);
+  exit (1);
+}
+
+
+
+
+int 
+main (int argc, char **argv)
+{
+  FILE *fp_in = (stdin), *fp_out = (stdout);
+  int width, height, maxval;
+  u_long magic;
+  char *filename = NULL;
+  unsigned int *line;
+  int maxk = 0;
+  int i;
+  int encode_flag = 0, decode_flag = 0;
+
+  if (argc < 2)
+    usage (argv[0]);
+
+  for (i = 1; i < argc; i++)
+    {
+      if (!strcmp (argv[i], "-h"))
+	usage (argv[0]);
+      else if (!strcmp (argv[i], "-e"))
+	encode_flag = 1;
+      else if (!strcmp (argv[i], "-d"))
+	decode_flag = 1;
+      else if (!strcmp (argv[i], "-k"))
+	{
+	  maxk = atoi (argv[++i]);
+	}
+      else if (argv[i][0] == '-')
+	error_msg (argv[0], "unknown switch:", argv[i]);
+      else if (!filename)
+	filename = argv[i];
+      else
+	error_msg (argv[0], "too many filenames specified.", "");
+    }
+
+  if (encode_flag == decode_flag)
+    error_msg (argv[0], "please specify either encode XOR decode.", "");
+
+  if (!filename)
+    error_msg (argv[0], "please specify a filename.", "");
+
+  if ((encode_flag) && (maxk >= MAXKVALUE))
+    {				/* to fit into 32 bit integers */
+      fprintf (stderr, "%s: k value of %d is too large; it must be less than %d\n",
+	       argv[0], maxk, MAXKVALUE);
+      exit (1);
+    }
+
+
+  if ((fp_in = fopen (filename, "rb")) == NULL)
+    error_msg (argv[0], "unable to open file:", filename);
+
+  /* read file header to determine the type and dimensions */
+
+
+  magic = magic_read (fp_in);
+  if (magic != MAGIC_FELICS)
+    {
+      rewind (fp_in);
+      magic = getmagicno_short_pop (fp_in);
+    }
+
+  if ((decode_flag && (magic != MAGIC_FELICS)) ||
+      (encode_flag && ((magic != MAGIC_P2) && (magic != MAGIC_P5))))
+    {
+      fclose (fp_in);
+      error_msg (argv[0], encode_flag ? "only PGM files can be used." : "only felics compressed files can be used.", "");
+    }
+
+  width = gethint (fp_in);
+  height = gethint (fp_in);
+  maxval = gethint (fp_in);
+
+  if ((line = (unsigned int *) malloc (width * sizeof (int *))) == NULL)
+    {
+      fclose (fp_in);
+      error_msg (argv[0], "unable to allocate memory,", "");
+    }
+
+
+  if (encode_flag)
+    {
+      magic_write (fp_out, MAGIC_FELICS);
+      fprintf (fp_out, "%d\n%d\n%d\n%d\n", width, height, maxval, maxk);
+      encode (line, width, height, maxval, maxk, fp_in, fp_out,
+	      magic == MAGIC_P2 ? getint : fgetc);
+    }
+  else if (decode_flag)
+    {
+      if (maxk)
+	warn (argv[0], "k value not required for decompression, ignored.", "");
+
+      maxk = gethint (fp_in);
+      fprintf (fp_out, "P%d\n%d %d\n%d\n", maxval < 256 ? 5 : 2,
+	       width, height, maxval);
+      decode (line, width, height, maxval, maxk, fp_in, fp_out);
+    }
+
+
+  fclose (fp_in);
+  fclose (fp_out);
+
+  return 0;
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/images/mgtic.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/images/mgtic.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/images/mgtic.1	(revision 699)
@@ -0,0 +1,137 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgtic 1 \*(Dt CITRI
+.SH NAME
+mgtic \- textual image compression.
+.SH SYNOPSIS
+.B mgtic
+.B \-e
+[
+.B \-L
+]
+[
+.B \-Q
+]
+[
+.B \-X
+]
+.I libraryfile
+.I infile
+.if n .ti +9n
+[
+.I \>compressed\-outfile
+]
+.nf
+or
+.fi
+.B mgtic
+.B \-d
+[
+.B \-L
+]
+[
+.BI \-X " libraryfile"
+]
+.I compressed-infile
+.if n .ti +9n
+[
+.I \>outfile
+]
+.SH DESCRIPTION
+This is the main routine in the
+.B mgtic
+system.
+.B mgtic
+is used to
+processes a series of files, and by using the
+.I libraryfile
+produced by
+.BR mgticbuild (1),
+compresses input files with respect to the marks contained in the
+library.  On the CCITT test faxes, lossless mode achieved around 20:1
+compression, and lossy, around 40:1 compression.  On more textual
+images, the amount of compression is determined by the resolution of
+the original file, and the amount of noise in the image.  By default,
+.B mgtic
+will reconstruct a lossy version of the image.  This
+can be changed to lossless by using the
+.B \-L
+option.
+.SH OPTIONS
+Options may appear in any order, but must come before the input
+filenames.
+.TP "\w'\fB\-X\fP \fI[libraryfile]\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-e
+Encode
+.I infile
+with respect to the library.
+.TP
+.B \-d
+Decodes
+.IR compressed-infile
+with respect to the library.
+.TP
+.B \-l
+This specifies lossy mode, and is the default mode for (de)compression.
+Do not generate the residue.  If used in encoding mode, does not store
+the residue in the output.  If used in decoding mode, the residue (if
+it exists) is ignored.
+.TP
+.B \-L
+Use lossless mode for (de)compression, which generates the residue.
+If used in encoding mode, this stores the residue in the compressed
+output.  If used in decoding mode (assuming the residue is present),
+this allows the original file to be recovered exactly.
+.TP
+.BI \-R\  filename
+Stores the residue as an external file
+.I filename
+which could possibly be stored on another medium.  During
+decompression, if the residue file specified does not relate to the
+compressed file, the resulting decompressed image will be a mess.
+Note that only the width and height of the residue image is checked.
+.TP
+.BI \-Q
+Uses a quicker form of compression.  This option decreases the time
+spent looking for a good match during encoding.  On average this
+option decreases compression ratios, but increases compression speed.
+.TP
+.BI \-X " [libraryfile]"
+Use an external library file, instead of (de)compressing the library
+within the normal stream.  When decoding, the
+.B \-X
+option must be followed by the
+.IR libraryfile .
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /tags/gsdl-21/gsdl/packages/mg/src/images/mgtic.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/images/mgtic.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/images/mgtic.c	(revision 699)
@@ -0,0 +1,596 @@
+/**************************************************************************
+ *
+ * mgtic.c -- Program to compress/decompress textual images
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************
+ *
+ * This file is the basis of mgtic, which compresses the components of an
+ * image. 
+ * The compressed file contains the following sections:
+ *         mgtic magic_no
+ *         mgtic version
+ *         lossy/lossless flag 
+ *         internal/external library flag 
+ *         cols, rows, number of marks, library size
+ *         [library sequence]
+ *         symbol sequence
+ *         offset sequence
+ *         [residue bitmap]
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "marklist.h"
+#include "pbmtools.h"
+#include "extractor.h"
+#include "utils.h"
+#include "match.h"
+#include "sortmarks.h"
+#include "codesyms.h"
+#include "arithcode.h"
+#include "codeoffsets.h"
+#include "bilevel.h"
+
+static char library_template[] =
+{
+  ".ppppp.;"
+  "pp222pp;"
+  "p22222p;"
+  "p22*...;"
+};
+
+void 
+usage ()
+{
+  fprintf (stderr, "usage: \n"
+	"\tmgtic -e [-L] [-Q] [-X]  libraryfile   infile    [>compressed]\n"
+	   "or\n"
+	   "\tmgtic -d [-L] [-X libraryfile]  compressed  [>outfile]\n");
+  exit (1);
+}
+
+
+/* encode the list of marks 'list', w.r.t. the library 'library' */
+void 
+match_sequence (marklistptr list, marklistptr library, marklistptr * symbol_list, Pixel ** recon, int quick)
+{
+  int carryx, carryy, gap;
+  int lastx = 0, lasty = 0;
+  int i, j;
+  marktype d, d2;
+  marklistptr liststep, librarystep;
+  int score, found;
+  int r, c;
+  int p;
+
+  carryx = carryy = 0;
+  gap = max (1, marklist_length (list) / 10);
+
+  for (liststep = list, i = 0; liststep; liststep = liststep->next, i++)
+    {
+      if (V)
+	if (!(i % gap))
+	  fprintf (stderr, ".");
+      d = liststep->data;
+
+      found = -1;
+      score = INT_MAX;
+      for (librarystep = library, j = 0; librarystep; librarystep = librarystep->next, j++)
+	{
+	  d2 = librarystep->data;
+
+	  if (NOT_SCREENED (d2, d))
+	    if ((p = MATCH (&d, &d2)) < MATCH_VAL)
+	      {
+		if (p < score)
+		  {
+		    score = p;
+		    found = j;
+		    if (quick)
+		      break;
+		  }
+	      }
+	}
+      /*
+         if(found<0){
+         for(librarystep=library,j=0; librarystep; librarystep=librarystep->next,j++){
+         d2=librarystep->data;
+
+         if((p=MATCH(&d,&d2))<MATCH_VAL){
+         if(p<score){
+         score=p;found=j; if(quick) break;
+         }
+         }
+         }
+         }
+       */
+      if (found >= 0)
+	{
+	  liststep->data.symnum = found;
+	  marklist_getat (library, found, &d2);
+	  d2.symnum = found;
+
+	  d2.xoffset = d.xpos - lastx + carryx;		/* propagate changes to next mark */
+	  d2.yoffset = d.ypos - lasty + carryy;
+	  carryx = (d2.xcen - d.xcen);	/* offet centroids because thats where we match them! */
+	  carryy = (d2.ycen - d.ycen);
+	  d2.xoffset -= carryx;	/* make the changes to this mark */
+	  d2.yoffset -= carryy;
+	  lastx = d.xpos + d2.w;
+	  lasty = d.ypos;
+
+	  marklist_add (symbol_list, d2);
+
+	  /* add to the reconstructed image--this is the lossy version */
+	  for (r = 0; r < d2.h; r++)
+	    for (c = 0; c < d2.w; c++)
+	      if (pbm_getpixel (d2.bitmap, c, r))
+		pbm_putpixel_trunc (recon, d.xpos + c - carryx, d.ypos + r - carryy, 1, d2.w, d2.h);
+	}
+      else
+	{
+	  /* didn't match at all..hopefully noise */
+	  if ((liststep->data.w >= 10) || (liststep->data.w >= 10))
+	    {
+	      fprintf (stderr, "mgtic: warning, unable to match a char.\n");
+	      write_library_mark (stderr, liststep->data);
+	    }
+	  if (V)
+	    fprintf (stderr, "x");
+	  liststep->data.symnum = -1;
+	}
+    }
+  if (V)
+    fprintf (stderr, "\n");
+}
+
+
+
+
+void 
+main (int argc, char *args[])
+{
+  int i, n, r, c, ch;
+  FILE *inf;
+  marklistptr library = NULL, list = NULL, step = NULL;
+  marklistptr symbol_list = NULL;
+  marktype d, d2;
+  Pixel **bitmap, **recon, **bitmapcopy;
+  char *splitfilename = NULL;
+  int cols, rows, count, librarysize, lossy = 1;
+  int quick = 0;
+  int ticencode = 0, ticdecode = 0, external = 0;
+  char *libraryname = NULL, *infile = NULL;
+  char *s1 = NULL, *s2 = NULL;
+  char bufferin[BUFSIZ], bufferout[BUFSIZ];
+
+  if (argc < 2)
+    usage ();
+
+  while ((ch = getopt (argc, args, "edlLQhvXR:")) != -1)
+    switch (ch)
+      {
+      case 'e':
+	ticencode = 1;
+	break;
+      case 'd':
+	ticdecode = 1;
+	break;
+      case 'l':
+	lossy = 1;
+	break;
+      case 'L':
+	lossy = 0;
+	break;
+      case 'Q':
+	quick = 1;
+	break;
+      case 'v':
+	V = 1;
+	break;
+      case 'X':
+	external = 1;
+	break;
+      case 'R':
+	splitfilename = optarg;
+	break;
+      case 'h':
+      case '?':
+	usage ();
+      }
+
+  for (i = 1; i < argc; i++)
+    if (args[i][0] != '-')
+      if (strcmp (args[i - 1], "-R"))
+	{			/* ignore arg following -R, ie. -R <filename> */
+	  if (!s1)
+	    s1 = args[i];
+	  else if (!s2)
+	    s2 = args[i];
+	  else
+	    error_msg (args[0], "too many filenames", "");
+	}
+
+  if (ticencode == ticdecode)
+    error_msg (args[0], "please specify either encode XOR decode", "");
+
+  if (ticencode)
+    {
+	  /*****************
+	    ENCODING STAGE
+	  *****************/
+      long PrevBits = 0;
+      long BITS_header = 0, BITS_symbols = 0, BITS_offsets = 0, BITS_residue = 0,
+        BITS_footer = 0, BITS_library = 0;
+
+      libraryname = s1;
+      if (!libraryname)
+	error_msg (args[0], "please specify a library file", "");
+      infile = s2;
+      if (!infile)
+	error_msg (args[0], "please specify a file name", "");
+
+      inf = fopen (infile, "rb");
+      if (inf == NULL)
+	error_msg (args[0], "Trouble opening file:", infile);
+      setbuf (inf, bufferin);
+
+      count = librarysize = read_library (libraryname, &library);
+
+      if (V)
+	fprintf (stderr, "%s: processing...\n", args[0]);
+
+      if (pbm_isapbmfile (inf))
+	{
+	  if (V)
+	    fprintf (stderr, "reading file %s...\n", infile);
+	  bitmap = pbm_readfile (inf, &cols, &rows);
+	  fclose (inf);
+
+	  bitmapcopy = pbm_copy (bitmap, cols, rows);
+
+	  if (V)
+	    fprintf (stderr, "extracting...");
+	  ExtractAllMarks (bitmap, &list, cols, rows);
+	  if (V)
+	    fprintf (stderr, "(%d marks)\n", marklist_length (list));
+
+	  /* sort into reading order */
+	  if (V)
+	    fprintf (stderr, "sorting...\n");
+	  list = sortmarks (list);
+
+	  pbm_freearray (&bitmap, rows);	/* clear old version */
+	  bitmap = bitmapcopy;	/* point to the copy */
+	  recon = pbm_allocarray (cols, rows);
+
+	  if (V)
+	    fprintf (stderr, "matching...");
+
+	  match_sequence (list, library, &symbol_list, recon, quick);
+
+
+	  PrevBits = 0;
+	  setbuf (stdout, bufferout);
+	  /* start output */
+	  magic_write (stdout, MAGIC_TIC);
+	  InitArithEncoding ();
+
+	  EncodeGammaDist (1);	/* version 1 of the program */
+	  EncodeGammaDist (lossy);	/* lossy=1 = no residue */
+	  EncodeGammaDist (external);	/* 1== external file */
+
+	  count = marklist_length (symbol_list);
+	  if (V)
+	    fprintf (stderr, "encoding cols, rows, and number of symbols=%d\n", count);
+
+	  EncodeGammaDist (cols);
+	  EncodeGammaDist (rows);
+	  EncodeGammaDist (count);
+	  EncodeGammaDist (librarysize);
+
+	  BITS_header = CountOfBitsOut - PrevBits;
+	  PrevBits = CountOfBitsOut;
+
+
+	  /* output the library sequence */
+	  if (external == 0)
+	    {
+	      if (V)
+		fprintf (stderr, "encoding library\n");
+
+	      bl_clearmodel ();
+	      bl_writetemplate (library_template);
+	      for (step = library; step; step = step->next)
+		bl_compress_mark (step->data);
+	      bl_freemodel ();
+	      BITS_library = CountOfBitsOut - PrevBits;
+	      PrevBits = CountOfBitsOut;
+	    }
+
+	  /* output the symbol sequence */
+	  if (V)
+	    fprintf (stderr, "encoding symbol sequence\n");
+	  InitPPM ();
+	  EncodeSymbols (symbol_list, count);
+	  BITS_symbols = CountOfBitsOut - PrevBits;
+	  PrevBits = CountOfBitsOut;
+
+
+	  /* output the offset sequence */
+	  if (V)
+	    fprintf (stderr, "encoding offset sequence\n");
+	  EncodeOffsets (symbol_list, count);
+	  BITS_offsets = CountOfBitsOut - PrevBits;
+	  PrevBits = CountOfBitsOut;
+
+	  EncodeChecksum ();	/* code lossy checksum */
+
+	  /* calculate the residue...and compress it---if need be! */
+	  if (!lossy)
+	    {
+	      if (V)
+		fprintf (stderr, "encoding residue...\n");
+
+	      d.bitmap = bitmap;
+	      d.h = rows;
+	      d.w = cols;
+	      d2.bitmap = recon;
+	      d2.h = rows;
+	      d2.w = cols;
+
+	      if (splitfilename)
+		{
+		  FILE *temp;
+
+		  CloseDownArithEncoding ();
+		  fclose (stdout);
+		  /* no residue result */
+		  BITS_footer = CountOfBitsOut - PrevBits;
+
+		  temp = fopen (splitfilename, "wb");
+		  if (temp == NULL)
+		    error_msg (args[0], "Trouble creating file:", splitfilename);
+
+		  arith_out = temp;
+		  InitArithEncoding ();
+		  bl_clair_compress (d, d2);
+		  CloseDownArithEncoding ();
+		  BITS_residue = CountOfBitsOut;
+		  fclose (temp);
+		}
+	      else
+		{
+		  bl_clair_compress (d, d2);
+		  BITS_residue = CountOfBitsOut - PrevBits;
+		  PrevBits = CountOfBitsOut;
+		  EncodeChecksum ();	/* code lossless checksum */
+		  CloseDownArithEncoding ();
+		  BITS_footer = CountOfBitsOut - PrevBits;
+		}
+	    }
+	  else
+	    {
+	      if (V)
+		fprintf (stderr, "not encoding residue..lossy mode\n");
+	      CloseDownArithEncoding ();
+	      BITS_footer = CountOfBitsOut - PrevBits;
+	    }
+
+	  /* because we edit the values above */
+	  CountOfBitsOut = BITS_header + BITS_library + BITS_symbols + BITS_offsets + BITS_residue + BITS_footer;
+
+	  fprintf (stderr, "bits: header=%ld, library=%ld, "
+		   "symbols=%ld, offsets=%ld, residue=%ld, footer=%ld\n",
+		   BITS_header, BITS_library, BITS_symbols,
+		   BITS_offsets, BITS_residue, BITS_footer);
+	  fprintf (stderr, "total bits: %ld, ", CountOfBitsOut);
+	  fprintf (stderr, "Lossy CR: %4.2f", (cols * rows) / (float) (CountOfBitsOut - BITS_residue));
+	  if (external)
+	    fprintf (stderr, " (excluding external lib)");
+	  fprintf (stderr, ", Lossless CR: %4.2f\n", (!lossy) * (cols * rows) / (float) (CountOfBitsOut));
+	}
+      else
+	error_msg (args[0], "unknown format of bitmap--expecting PBM.", "");
+    }
+  else
+    {
+	  /*****************
+	    DECODING STAGE
+	  *****************/
+      int lastx, lasty;
+      librarysize = 0;
+
+      if (external)
+	{
+	  libraryname = s1;
+	  infile = s2;
+	  count = librarysize = read_library (libraryname, &library);
+	}
+      else
+	{
+	  infile = s1;
+	  if (infile && s2)
+	    error_msg (args[0], "too many filenames", "");
+	}
+
+      if (!freopen (infile, "rb", stdin))
+	error_msg (args[0], "Trouble opening file:", infile);
+
+
+      if (V)
+	fprintf (stderr, "decompressing...\n");
+
+      setbuf (stdin, bufferin);
+      magic_check (stdin, MAGIC_TIC);
+
+      InitArithDecoding ();
+
+      {
+	int version = DecodeGammaDist ();
+	if (version != 1)
+	  error_msg (args[0], "Need later version of decompressor.", "");
+      }
+
+      {
+	int templossy = DecodeGammaDist ();
+	if (!lossy)
+	  lossy = templossy;	/* can only choose if encoded file is lossless */
+	if (V)
+	  {
+	    if (lossy)
+	      fprintf (stderr, "lossy mode\n");
+	    else
+	      fprintf (stderr, "lossless mode\n");
+	  }
+      }
+
+      if (DecodeGammaDist ())
+	{			/* if compressed file doesn't contain library */
+	  if (!external)
+	    error_msg (args[0], "compressed file doesn't contain library, specify externally", "");
+	  external = 1;
+	}
+      else
+	{			/* if compressed file contains library */
+	  if (external)
+	    fprintf (stderr, "ignoring external library file\n");
+	  external = 0;
+	}
+
+      cols = DecodeGammaDist ();
+      rows = DecodeGammaDist ();
+      count = DecodeGammaDist ();
+
+      i = DecodeGammaDist ();	/* librarysize */
+      if (external)
+	{
+	  if (i > librarysize)
+	    error_msg (args[0], "external library file is too small!", "");
+	  else if (i < librarysize)
+	    fprintf (stderr, "%s: warning, expecting a different (smaller) library.\n", args[0]);
+	}
+      librarysize = i;
+
+      if (V)
+	fprintf (stderr, "cols %d, rows %d, num syms %d, library size %d\n", cols, rows, count, librarysize);
+
+      /* decode library */
+      if (external == 0)
+	{
+	  if (V)
+	    fprintf (stderr, "reading library\n");
+	  bl_clearmodel ();
+	  bl_readtemplate ();
+	  for (n = 0; n < librarysize; n++)
+	    {
+	      bl_decompress_mark (&d);
+	      d.symnum = n;
+	      if (library == NULL)
+		step = marklist_add (&library, d);
+	      else
+		step = marklist_add (&step, d);
+	    }
+	  bl_freemodel ();
+	  if (V)
+	    fprintf (stderr, "read %d marks from library\n", marklist_length (library));
+	}
+
+      recon = pbm_allocarray (cols, rows);
+
+      /* decode symbols */
+      InitPPM ();
+      if (V)
+	fprintf (stderr, "decompressing %d symbols\n", count);
+      symbol_list = DecodeSymbols (count);
+
+      /* decode offsets */
+      if (V)
+	fprintf (stderr, "reading offsets...\n");
+      DecodeOffsets (symbol_list, count);
+      lastx = lasty = 0;
+      for (step = symbol_list; step; step = step->next)
+	{
+	  lastx = lastx + step->data.xoffset;
+	  lasty = lasty + step->data.yoffset;
+
+	  marklist_getat (library, step->data.symnum, &d2);
+	  for (r = 0; r < d2.h; r++)
+	    for (c = 0; c < d2.w; c++)
+	      if (pbm_getpixel (d2.bitmap, c, r))
+		pbm_putpixel_trunc (recon, lastx + c, lasty + r, 1, cols, rows);	/* we don't care, already warned them! */
+	  lastx += d2.w;
+	}
+
+      DecodeChecksum (args[0]);
+
+      /* decode the residue */
+      if (!lossy)
+	{
+	  if (V)
+	    fprintf (stderr, "decoding residue...\n");
+
+	  bitmap = pbm_allocarray (cols, rows);
+	  d.bitmap = bitmap;
+	  d.w = d2.w = cols;
+	  d.h = d2.h = rows;
+
+	  d2.bitmap = recon;
+
+	  /* NOTE: the 2nd argument is clairvoyantly compressed */
+	  if (splitfilename)
+	    {
+	      FILE *temp;
+
+	      CloseDownArithDecoding ();
+	      fclose (stdin);
+
+	      temp = fopen (splitfilename, "rb");
+	      if (temp == NULL)
+		error_msg (args[0], "Trouble opening file:", splitfilename);
+
+	      arith_in = temp;
+	      InitArithDecoding ();
+	      bl_clair_decompress (d, d2);
+	      CloseDownArithDecoding ();
+	      fclose (temp);
+	    }
+	  else
+	    {
+	      bl_clair_decompress (d, d2);
+	      DecodeChecksum (args[0]);
+	      CloseDownArithDecoding ();
+	    }
+	  pbm_freearray (&recon, rows);
+	  recon = bitmap;	/* point to the bitmap */
+	}
+      else
+	{
+	  CloseDownArithDecoding ();
+	}
+
+      if (V)
+	fprintf (stderr, "writing pbm file...\n");
+      setbuf (stdout, bufferout);
+      pbm_writefile (stdout, recon, cols, rows);
+      pbm_freearray (&recon, rows);
+    }				/* end decoding */
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/images/mgticbuild.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/images/mgticbuild.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/images/mgticbuild.1	(revision 699)
@@ -0,0 +1,61 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgticbuild 1 \*(Dt CITRI
+.SH NAME
+mgticbuild \- build a library of marks from training documents
+.SH SYNOPSIS
+.B mgticbuild
+[
+.B \-h
+]
+.I "libraryfile infile1 infile2 .\|.\|."
+.SH DESCRIPTION
+.B mgticbuild
+builds an uncompressed library of marks that have been
+extracted from a series of input files.  If the
+.I libraryfile
+does
+not exist, it is created, otherwise new marks are appended to the
+file.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fIlibraryfile\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.I libraryfile
+This specifies the name of the library file.
+.TP
+.I infile(s)
+The training files; the only currently-supported format is
+.IR pbm .
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /tags/gsdl-21/gsdl/packages/mg/src/images/mgticbuild.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/images/mgticbuild.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/images/mgticbuild.c	(revision 699)
@@ -0,0 +1,145 @@
+/**************************************************************************
+ *
+ * mgticbuild.c -- Program to build the textual image library
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "marklist.h"
+#include "pbmtools.h"
+#include "extractor.h"
+#include "utils.h"
+
+
+void 
+usage ()
+{
+  fprintf (stderr, "usage: \n"
+	"\tmgticbuild   libraryfile   infile.pbm  [infile2 infile3 ...]\n");
+  exit (1);
+}
+
+void 
+main (int argc, char *args[])
+{
+  int i;
+  FILE *lib = NULL, *inf = NULL;
+  marklistptr list = NULL, step;
+  Pixel **bitmap;
+  int cols, rows;
+  int count, pos = 0;
+  char *libraryname = NULL;
+
+  if (argc < 2)
+    usage ();
+
+  for (i = 1; i < argc; i++)
+    {
+      if (!strcmp (args[i], "-v"))
+	V = 1;
+      else if (!strcmp (args[i], "-h"))
+	usage ();
+      else if (args[i][0] == '-')
+	error_msg (args[0], "unknown switch:", args[i]);
+      else if (!libraryname)
+	libraryname = args[i];
+      else
+	pos = i;
+    }
+
+  if (!libraryname)
+    error_msg (args[0], "please specify a library file", "");
+  if (pos < 2)
+    usage ();
+
+  count = 0;
+  for (i = pos; i < argc; i++)
+    if (args[i][0] != '-')
+      {
+	inf = fopen (args[i], "rb");
+	count = 0;
+	if (inf == NULL)
+	  fprintf (stderr, "Trouble opening file: %s\n", args[i]);
+	else
+	  {
+	    if (pbm_isapbmfile (inf))
+	      {
+		if (V)
+		  fprintf (stderr, "%s: processing...\n", args[0]);
+		if (V)
+		  fprintf (stderr, "reading file %s...", args[i]);
+		bitmap = pbm_readfile (inf, &cols, &rows);
+		if (V && (count == 0))
+		  fprintf (stderr, "building...");
+
+		list = NULL;
+
+		ExtractAllMarks (bitmap, &list, cols, rows);
+
+		if (V)
+		  fprintf (stderr, "writing...");
+
+		if ((!lib) && (marklist_length (list)))
+		  {
+		    lib = fopen (libraryname, "r");
+		    if (lib)
+		      {
+			fclose (lib);
+			lib = fopen (libraryname, "ab");
+			if (lib == NULL)
+			  error_msg (args[0], "trouble appending library file", "");
+			fprintf (stderr, "\nwarning: appending library file: %s\n", libraryname);
+		      }
+		    else
+		      {
+			lib = fopen (libraryname, "wb");
+			if (lib == NULL)
+			  error_msg (args[0], "trouble creating library file", "");
+		      }
+		  }		/* if !lib */
+
+		step = list;
+		while (step)
+		  {
+		    step->data.symnum = count;
+		    write_library_mark (lib, step->data);
+		    count++;
+		    step = step->next;
+		  }
+		pbm_freearray (&bitmap, rows);
+	      }
+	    else
+	      fprintf (stderr, "warning: %s is not a PBM file.\n", args[i]);
+	    fclose (inf);
+	  }
+	if (count)
+	  {
+	    if (V)
+	      fprintf (stderr, "%d marks written.\n", count);
+	  }
+	else
+	  fprintf (stderr, "No marks written!\n");
+      }
+
+  if (lib)
+    fclose (lib);
+  marklist_free (&list);
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/images/mgticdump.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/images/mgticdump.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/images/mgticdump.1	(revision 699)
@@ -0,0 +1,55 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgticdump 1 \*(Dt CITRI
+.SH NAME
+mgticdump \- displays the library in a user-friendly manner
+.SH SYNOPSIS
+.B mgticdump
+[
+.B \-h
+]
+.I libraryfile
+.SH DESCRIPTION
+.B mgticdump
+displays the marks in a library in a user-viewable form.  The output
+is to
+.I stdout
+and is in a plain ASCII form.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fIlibraryfile\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.I libraryfile
+This specifies the name of the library file.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /tags/gsdl-21/gsdl/packages/mg/src/images/mgticdump.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/images/mgticdump.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/images/mgticdump.c	(revision 699)
@@ -0,0 +1,86 @@
+/**************************************************************************
+ *
+ * mgticdump.c -- Program to dump out a library in a human readable form
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "marklist.h"
+#include "pbmtools.h"
+#include "extractor.h"
+#include "utils.h"
+
+
+void 
+usage ()
+{
+  fprintf (stderr, "usage:\n"
+	   "\tmgticdump   libraryfile\n");
+  exit (1);
+}
+
+
+
+void 
+main (int argc, char *args[])
+{
+  FILE *lib;
+  int count, i;
+  marktype d;
+  char *libraryname = NULL;
+  FILE *outf = (stdout);
+
+  if (argc < 2)
+    usage ();
+
+  for (i = 1; i < argc; i++)
+    {
+      if (!strcmp (args[i], "-h"))
+	usage ();
+      else if (args[i][0] == '-')
+	error_msg (args[0], "unknown switch:", args[i]);
+      else if (!libraryname)
+	libraryname = args[i];
+      else
+	error_msg (args[0], "too many filenames", "");
+    }
+
+  if (!libraryname)
+    error_msg (args[0], "please specify a library file", "");
+
+
+  lib = fopen (libraryname, "rb");
+  if (lib == NULL)
+    error_msg (args[0], "trouble opening library file:", libraryname);
+
+  count = 0;
+  while (!isEOF (lib))
+    {
+      read_library_mark (lib, &d);
+
+      write_library_mark (outf, d);
+      count++;
+
+      pbm_freearray (&d.bitmap, d.h);
+    }
+  fprintf (stderr, "count=%d", count);
+  fclose (lib);
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/images/mgticprune.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/images/mgticprune.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/images/mgticprune.1	(revision 699)
@@ -0,0 +1,91 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgticprune 1 \*(Dt CITRI
+.SH NAME
+mgticprune \- prunes the library to remove duplicates
+.SH SYNOPSIS
+.B mgticprune
+[
+.BI \-a " n"
+]
+[
+.BI \-s " n"
+]
+[
+.B \-Q
+]
+.I libraryfile
+.if n .ti +9n
+[
+.I pruned-libraryfile
+]
+.SH DESCRIPTION
+.B mgticprune
+examines
+.I libraryfile
+and removes duplicate marks.  If
+.I pruned-libraryfile
+is not specified,
+.I libraryfile
+is overwritten with the marks in the pruned library.  The Combined
+Symbol Independent Strategy (CSIS) method is used to match marks.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fIpruned-libraryfile\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-a " n"
+Prune marks when their area is under or equal to the threshold,
+.IR n .
+The default value is\ 3.
+.TP
+.BI \-s " n"
+Prune marks when their width and height is under or equal to
+the threshold,
+.IR n .
+The default value is\ 0.
+.TP
+.BI \-Q
+Use a quicker method of pruning.  The number of iterations through the
+averaging and pruning processes is decreased.  This usually reduces
+time by a half, but compression ratios decrease.
+.TP
+.I libraryfile
+This specifies the name of the library file
+.TP
+.I pruned-libraryfile
+This specifies the name of the prune library, after duplicates and
+matches have been removed.  If this file is not specified, then
+.I libraryfile
+will be overwritten.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticstat (1).
Index: /tags/gsdl-21/gsdl/packages/mg/src/images/mgticprune.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/images/mgticprune.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/images/mgticprune.c	(revision 699)
@@ -0,0 +1,393 @@
+/**************************************************************************
+ *
+ * mgticprune.c -- Program to prune a textual image library
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "marklist.h"
+#include "pbmtools.h"
+#include "extractor.h"
+#include "utils.h"
+#include "match.h"
+
+marklistptr *avglib = NULL;
+int avglibsize;
+int libsize = 0;
+
+
+void 
+usage ()
+{
+  fprintf (stderr, "usage:\n"
+    "\tmgticbuild [-a n]  [-s n]  [-Q]   libraryfile   [pruned-library]\n");
+  exit (1);
+}
+
+
+
+
+/* foreach list in the avglib[] array, an average mark is returned */
+marklistptr 
+average_library ()
+{
+  marklistptr librarylist;
+  int i;
+  marktype avgmark;
+
+  librarylist = NULL;
+  for (i = 0; i < libsize; i++)
+    {
+      if (avglib[i])
+	{
+	  marklist_average (avglib[i], &avgmark);
+	  avgmark.symnum = i;
+	  marklist_addcopy (&librarylist, avgmark);
+	}
+    }
+  return librarylist;
+}
+
+
+void 
+average_space_enlarge ()
+{
+  int j;
+
+  if (libsize >= avglibsize)
+    {
+      avglib = (marklistptr *) realloc (avglib, (avglibsize + 100) * sizeof (marklistptr));
+      if (!avglib)
+	error_msg ("mgticprune", "out of memory in reallocating..", "");
+      for (j = avglibsize; j < avglibsize + 100; j++)
+	avglib[j] = NULL;
+      avglibsize += 100;
+    }
+}
+
+
+/* for each mark in 'origlist', if it can't be matched with one in 
+   'librarylist', then add to the librarylist, and update the relevant
+   avglib[] position */
+void 
+greedy_add (marklistptr origlist, marklistptr * librarylist)
+{
+  marklistptr step, step_i;
+  int i, j, notfound;
+  marktype d_i;
+
+  for (i = 0, step_i = origlist; step_i; i++, step_i = step_i->next)
+    {
+      d_i = step_i->data;
+
+      notfound = 1;
+      for (j = 0, step = *librarylist; step; j++, step = step->next)
+	{
+	  if (NOT_SCREENED (d_i, step->data))
+	    if (MATCH (&d_i, &step->data) < MATCH_VAL)
+	      {
+		d_i.symnum = j;
+		marklist_addcopy (&avglib[j], d_i);
+		notfound = 0;
+		break;
+	      }
+	}
+
+      if (notfound)
+	{
+	  d_i.symnum = libsize;
+	  marklist_addcopy (librarylist, d_i);
+	  marklist_addcopy (&avglib[libsize], d_i);
+	  libsize++;
+	  if (libsize >= avglibsize)
+	    average_space_enlarge ();
+	}
+    }
+}
+
+
+
+/* for each mark in 'library', if another matches with it, remove
+   one copy from the library, and erase the relevant avglib[] */
+void 
+greedy_prune (marklistptr * library)
+{
+  int count;
+  int i, j;
+  marklistptr step;
+  marktype d;
+
+  count = marklist_length (*library);
+  /* an invariant, all marks <i have been kept in the pruned library */
+  for (i = 0; i < count; i++)
+    {
+      marklist_getat (*library, i, &d);
+
+      for (j = 0, step = *library; j < i; j++, step = step->next)
+	{
+	  if (NOT_SCREENED (d, step->data))
+	    if (MATCH (&d, &step->data) < MATCH_VAL)
+	      {
+		/* [i,d] has matched with library element [j,d2]. */
+		marklist_removeat (library, i);		/* because we kill it here */
+		marklist_free (&avglib[d.symnum]);
+		avglib[d.symnum] = NULL;
+		i--, count--;
+		break;
+	      }
+	}
+    }
+}
+
+
+void 
+prune_on_size (marklistptr * list, int sizethres)
+{
+  int count, i;
+  marktype d;
+
+  if (sizethres)
+    {
+      count = marklist_length (*list);
+      i = 0;
+      do
+	{
+	  marklist_getat (*list, i, &d);
+	  if ((d.w <= sizethres) && (d.h <= sizethres))
+	    {
+	      marklist_removeat (list, i);
+	      i--;
+	      count--;
+	    }
+	  i++;
+	}
+      while (i < count);
+    }
+}
+
+
+
+void 
+prune_on_area (marklistptr * list, int areathres)
+{
+  int count, i;
+  marktype d;
+
+  if (areathres)
+    {
+      count = marklist_length (*list);
+      i = 0;
+      do
+	{
+	  marklist_getat (*list, i, &d);
+	  if (d.set <= areathres)
+	    {
+	      marklist_removeat (list, i);
+	      i--;
+	      count--;
+	    }
+	  i++;
+	}
+      while (i < count);
+    }
+}
+
+
+void 
+averagedout_addagain (marklistptr origlist, marklistptr * librarylist)
+{
+  marklistptr step, stepi;
+  int notfound;
+  int i, j;
+
+  for (i = 0, step = origlist; step; i++, step = step->next)
+    {
+      notfound = 1;
+
+      for (j = 0, stepi = *librarylist; stepi; j++, stepi = stepi->next)
+	if (NOT_SCREENED (step->data, stepi->data))
+	  if (MATCH (&step->data, &stepi->data) < MATCH_VAL)
+	    {
+	      notfound = 0;
+	      break;
+	    }
+
+      if (notfound)
+	{
+	  marklist_addcopy (librarylist, step->data);
+	  marklist_addcopy (&avglib[libsize], step->data);
+	  libsize++;
+	  if (libsize >= avglibsize)
+	    average_space_enlarge ();
+	}
+    }
+}
+
+
+
+
+
+void 
+main (int argc, char *args[])
+{
+  int i;
+  FILE *lib;
+
+  marklistptr origlist = NULL, librarylist = NULL;
+  marklistptr step;
+  char tempn[1000], sysstr[1000];
+  int count;
+  char *libraryname = NULL, *prunename = NULL;
+  int sizethres = 3, areathres = 0;
+  int pass;
+  int num_passes = 3;
+  int overwrite = 0;
+
+  V = 0;
+
+  if (argc < 2)
+    usage ();
+
+  for (i = 1; i < argc; i++)
+    {
+      if (!strcmp (args[i], "-v"))
+	V = 1;
+      else if (!strcmp (args[i], "-h"))
+	usage ();
+      else if (!strcmp (args[i], "-Q"))
+	{
+	  num_passes = 0;
+	}
+      else if (!strcmp (args[i], "-s"))
+	{
+	  i++;
+	  if ((i < argc) && (isinteger (args[i])))
+	    sizethres = atoi (args[i]);
+	  else
+	    error_msg (args[0], "follow the -s option with a numeric arg >=0", "");
+	}
+      else if (!strcmp (args[i], "-a"))
+	{
+	  i++;
+	  if ((i < argc) && (isinteger (args[i])))
+	    areathres = atoi (args[i]);
+	  else
+	    error_msg (args[0], "follow the -a option with a numeric arg >=0", "");
+	}
+      else if (args[i][0] == '-')
+	error_msg (args[0], "unknown switch:", args[i]);
+      else if (!libraryname)
+	libraryname = args[i];
+      else if (!prunename)
+	prunename = args[i];
+      else
+	error_msg (args[0], "too many filenames", "");
+    }
+
+  if (!libraryname)
+    error_msg (args[0], "please specify a library file", "");
+
+  if (V)
+    fprintf (stderr, "%s: processing...\n", args[0]);
+
+  overwrite = 0;
+  if (!prunename)
+    {
+      prunename = libraryname;
+      if (V)
+	fprintf (stderr, "overwriting previous library file.\n");
+      overwrite = 1;
+    }
+
+  if (overwrite)
+    {
+      tmpnam (tempn);
+    }
+  else
+    strcpy (tempn, prunename);
+
+
+  avglibsize = count = read_library (libraryname, &origlist);
+  if (V)
+    fprintf (stderr, "%d marks...", count);
+
+  avglib = (marklistptr *) malloc (count * sizeof (marklistptr));
+  if (!avglib)
+    error_msg (args[0], "out of memory", "");
+  for (i = 0; i < avglibsize; i++)
+    avglib[i] = NULL;
+
+
+  prune_on_size (&origlist, sizethres);
+
+  prune_on_area (&origlist, areathres);
+  if (V)
+    fprintf (stderr, "after threshold pruning, %d marks\n", count = marklist_length (origlist));
+
+
+  if (V)
+    fprintf (stderr, "starting greedy library creation...\n");
+
+  greedy_add (origlist, &librarylist);
+
+
+  for (pass = 0; pass < num_passes; pass++)
+    {
+      marklist_free (&librarylist);
+      if (V)
+	fprintf (stderr, "averaging...\n");
+      librarylist = average_library ();
+      averagedout_addagain (origlist, &librarylist);
+      greedy_prune (&librarylist);
+      if (V)
+	fprintf (stderr, "after pass %d greedy prune %d marks\n", 1 + pass, marklist_length (librarylist));
+    }
+
+
+  lib = fopen (tempn, "wb");
+  if (lib == NULL)
+    error_msg (args[0], "trouble creating prune-library file.", "");
+
+
+  if (V)
+    fprintf (stderr, "writing to file: %s...", prunename);
+  for (step = librarylist, count = 0; step; step = step->next, count++)
+    {
+      step->data.symnum = count;
+      write_library_mark (lib, step->data);
+    }
+
+  if (V)
+    fprintf (stderr, "%d marks.\n", count);
+  fclose (lib);
+
+  if (overwrite)
+    {
+      sprintf (sysstr, "mv %s %s", tempn, prunename);
+      system (sysstr);		/* move temp to proper name */
+    }
+
+  for (i = 0; i < avglibsize; i++)
+    {
+      marklist_free (&avglib[i]);
+      free (avglib[i]);
+    }
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/images/mgticstat.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/images/mgticstat.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/images/mgticstat.1	(revision 699)
@@ -0,0 +1,57 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgticstat 1 \*(Dt CITRI
+.SH NAME
+mgticstat \- displays the statistics for the library
+.SH SYNOPSIS
+.B mgticstat
+[
+.B \-h
+]
+.I libraryfile
+.SH DESCRIPTION
+.B mgticstat
+displays statistics about the
+.I libraryfile
+file.
+The output of this program contains the average mark size, average
+centroid position, and the number of marks with an area less than or
+equal to a few powers of\ 2.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fIlibraryfile\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.I libraryfile
+This specifies the name of the library file.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1).
Index: /tags/gsdl-21/gsdl/packages/mg/src/images/mgticstat.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/images/mgticstat.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/images/mgticstat.c	(revision 699)
@@ -0,0 +1,130 @@
+/**************************************************************************
+ *
+ * mgticstat.c -- Program to print out stats about a textual image library
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "marklist.h"
+#include "pbmtools.h"
+#include "extractor.h"
+#include "utils.h"
+
+void 
+usage ()
+{
+  fprintf (stderr, "usage:\n"
+	   "\tmgticstat   libraryfile\n");
+  exit (1);
+}
+
+void 
+main (int argc, char *args[])
+{
+  FILE *lib, *outf = (stdout);
+  int count;
+  int small1, small2, small4, small9, small16, small25, small36, small49,
+    small64, small81, small100;
+  marktype d;
+  int avx, avy;
+  int hhh, www, i;
+  int avruns_h = 0, avruns_v = 0;
+  int av_area = 0;
+  char *libraryname = NULL;
+
+
+  if (argc < 2)
+    usage ();
+
+  for (i = 1; i < argc; i++)
+    {
+      if (!strcmp (args[i], "-v"))
+	V = 1;
+      else if (!strcmp (args[i], "-h"))
+	usage ();
+      else if (args[i][0] == '-')
+	error_msg (args[0], "unknown switch:", args[i]);
+      else if (!libraryname)
+	libraryname = args[i];
+      else
+	error_msg (args[0], "too many filenames", "");
+    }
+
+  if (!libraryname)
+    error_msg (args[0], "please specify a library file", "");
+
+
+  lib = fopen (libraryname, "r");
+  if (lib == NULL)
+    error_msg (args[0], "Trouble opening library file", "");
+
+  count = 0;
+  small1 = small2 = small4 = small9 = small16 = small25 = small36 = small49 = small64 = small81 = small100 = 0;
+  avx = avy = 0;
+  www = hhh = 0;
+  while (!isEOF (lib))
+    {
+      if (!read_library_mark (lib, &d))
+	error_msg (args[0], "error in library file", "");
+
+      av_area += d.set;
+      if (d.set <= 1)
+	small1++;
+      if (d.set <= 2)
+	small2++;
+      if (d.set <= 4)
+	small4++;
+      if (d.set <= 9)
+	small9++;
+      if (d.set <= 16)
+	small16++;
+      if (d.set <= 25)
+	small25++;
+      if (d.set <= 36)
+	small36++;
+      if (d.set <= 49)
+	small49++;
+      if (d.set <= 64)
+	small64++;
+      if (d.set <= 81)
+	small81++;
+      if (d.set <= 100)
+	small100++;
+      avx += d.xcen;
+      avy += d.ycen;
+      www += d.w;
+      hhh += d.h;
+      avruns_h += d.h_runs;
+      avruns_v += d.v_runs;
+      pbm_freearray (&d.bitmap, d.h);
+      count++;
+      if (count % 5000 == 0)
+	fprintf (outf, "%dk ", count / 1000);
+    }
+  fprintf (outf, "number of marks=%d\n", count);
+  fprintf (outf, "average area=%d\n", av_area / count);
+  fprintf (outf, "area <=1:%d, <=2:%d, <=4:%d, <=9:%d, <=16:%d, <=25:%d, <=36:%d, <=49:%d, <=64:%d, <=81:%d, <=100:%d\n", small1, small2, small4, small9, small16, small25, small36, small49, small64, small81, small100);
+
+  fprintf (outf, "average num runs, horizontal=%d, vertical=%d\n", avruns_h / count, avruns_v / count);
+  fprintf (outf, "average centroid position=(%d,%d)\n", avx / count, avy / count);
+  fprintf (outf, "average width=%d, average height=%d\n", www / count, hhh / count);
+  fclose (lib);
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/images/model.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/images/model.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/images/model.h	(revision 699)
@@ -0,0 +1,92 @@
+/**************************************************************************
+ *
+ * model.h -- Part pf PPM data compression
+ * Copyright (C) 1987  Alistair Moffat
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#define defaultkbytes	896
+#define minkbytes	56
+#define maxkbytes	((65536/1024)*sizeof(eventnode))
+
+#ifndef increment
+#define	increment	8
+#endif
+#ifndef halveat
+#define halveat		64
+#endif
+#define maxtotalcnt 	(increment*halveat)
+
+typedef int boolean;
+#define false           0
+#define true            1
+
+typedef short event;
+typedef unsigned short point;
+
+typedef struct
+  {
+    short totalcnt;
+    short notfound;
+    point list;			/* list storing the event records */
+  }
+eventset;
+
+typedef struct
+  {
+    event eventnum;
+    short count;
+    point next, prev;
+    eventset foll;
+  }
+eventnode, *eventptr;
+
+#define memsize		(kbytes*1024)
+#define maxnodes	(memsize/sizeof(eventnode))
+/* #define charsetsize     256 -- MAHE */
+#define nchars          (charsetsize+1)
+
+extern int charsetsize;		/* -- MAHE */
+
+extern eventptr E;
+extern long kbytes;
+extern point numnodes, nfreenodes;
+extern boolean excluded[];	/* -- MAHE */
+
+#define ENULL		(E)
+
+typedef unsigned u;
+#define p32(p)	(E+p)
+#define p16(p)	(p-E)
+
+/* in ppm.c */
+void write_model ();
+void encodestring ();
+void decodestring ();
+void encodefile ();
+void decodefile ();
+
+/* in lstevent.c */
+void write_method ();
+eventnode *encode_event_noex ();
+eventnode *decode_event_noex ();
+eventnode *encode_event ();
+eventnode *decode_event ();
+/* eventnode *addevent(); */
+eventnode *newnode ();
Index: /tags/gsdl-21/gsdl/packages/mg/src/images/pbmtools.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/images/pbmtools.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/images/pbmtools.c	(revision 699)
@@ -0,0 +1,288 @@
+/**************************************************************************
+ *
+ * pbmtools.c -- Routines for dealing with portable bitmap files (PBM)
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "marklist.h"
+#include "pbmtools.h"
+#include "extractor.h"
+#include "utils.h"
+
+
+
+void 
+pbm_putpixel_range (Pixel ** bitmap, int c, int r, int val, int cols, int rows)
+{
+  if (((c) >= 0) && ((c) < cols) && ((r) >= 0) && ((r) < rows))
+    pbm_putpixel (bitmap, c, r, val);
+  else
+    fprintf (stderr, "putpitxel (%d %d) outside (%d %d)\n", c, r, cols, rows);
+}
+
+int 
+pbm_getpixel_range (Pixel ** bitmap, int c, int r, int cols, int rows)
+{
+  if (((c) >= 0) && ((c) < cols) && ((r) >= 0) && ((r) < rows))
+    return pbm_getpixel (bitmap, c, r);
+  else
+    {
+      fprintf (stderr, "getpixel (%d %d) outside (%d %d)\n", c, r, cols, rows);
+      return 0;
+    }
+}
+
+
+
+
+
+/*
+   Returns 1 if *fp points to a PBM (magic is P1 or P4) file.
+   leaves the fp in it's original position.
+   usage: if(pbm_isapbmfile(fp)) process(fp);
+ */
+int 
+pbm_isapbmfile (FILE * fp)
+{
+  u_long magic;
+
+  magic = getmagicno_short (fp);
+  if ((magic == MAGIC_P4) || (magic == MAGIC_P1))
+    return 1;
+  else
+    return 0;
+}
+
+
+/*
+   Frees the memory used in the bitmap structure.
+   usage: pbm_freearray(&bitmap);
+   Sets bitmap to be NULL on return.
+ */
+void 
+pbm_freearray (Pixel *** bitmap, int rows)
+{
+  int r;
+  if (rows == 0)
+    rows++;
+  for (r = 0; r < rows; r++)
+    free ((*bitmap)[r]);
+  free (*bitmap);
+  (*bitmap) = NULL;
+}
+
+
+/*
+   This function allocates the bitmap.
+   usage: bitmap=pbm_allocarray(cols,rows);
+ */
+Pixel **
+pbm_allocarray (int cols, int rows)
+{
+  Pixel **p;
+  int i, newc = (cols + PBITS) / PBITS;
+
+  if (cols == 0)
+    cols++;
+  if (rows == 0)
+    rows++;
+  p = (Pixel **) malloc (rows * sizeof (Pixel *));
+  if (p == NULL)
+    error_msg ("pbm_allocarray", "out of memory", "");
+  for (i = 0; i < rows; i++)
+    {
+      p[i] = (Pixel *) calloc (newc, sizeof (Pixel));
+      if (p[i] == NULL)
+	error_msg ("pbm_allocarray", "out of memory", "");
+    }
+  return p;
+}
+
+
+
+Pixel **
+pbm_copy (Pixel ** bitmap, int cols, int rows)
+{
+  Pixel **p;
+  int newc = (cols + PBITS) / PBITS;
+  int x, y;
+
+  p = pbm_allocarray (cols, rows);
+  for (x = 0; x < newc; x++)
+    for (y = 0; y < rows; y++)
+      p[y][x] = bitmap[y][x];
+  return p;
+}
+
+
+
+
+/*
+   Reads in a pbm file and returns a pointer to the allocated bitmap, 
+   in the P4 format.
+   Assumes fp was opened and keeps it that way.
+   usage: bitmap=pbm_readfile(fp,&cols,&rows);
+ */
+Pixel **
+pbm_readfile (FILE * fp, int *cols, int *rows)
+{
+  Pixel **p = NULL;
+  char s[100];
+  unsigned char *buf;
+  int r, c, ii = 0;
+  int p1 = 0;
+  int arg;
+
+
+  do
+    {
+      readline (s, fp);
+    }
+  while (s[0] == '#');
+
+  if (strcmp (s, "P1") == 0)
+    p1 = 1;
+  else if (strcmp (s, "P4") == 0)
+    p1 = 0;
+  else
+    error_msg ("pbm_readfile", "only pbm files are supported", "");
+
+  do
+    {
+      readline (s, fp);
+    }
+  while (s[0] == '#');
+
+  arg = sscanf (s, "%d %d", cols, rows);
+  if (arg == 1)
+    {
+      readline (s, fp);
+      arg = sscanf (s, "%d", rows);
+      if (arg != 1)
+	error_msg ("pbm_readfile", "bizzare pbm format", "");
+    }
+  p = pbm_allocarray (*cols, *rows);
+  buf = (unsigned char *) malloc ((*cols + 2) * sizeof (unsigned char));
+  if (!buf)
+    error_msg ("pbm_readfile", "out of memory", "");
+
+  if (p1)
+    {
+      for (r = 0; r < *rows; r++)
+	for (c = 0; c < *cols; c++)
+	  {
+	    fscanf (fp, "%d", &ii);
+	    pbm_putpixel (p, c, r, ii);
+	  }
+    }
+  else
+    {
+      for (r = 0; r < *rows; r++)
+	{
+	  c = 0;
+	  fread (buf, 1, (*cols + 7) / 8, fp);
+	  for (c = 0; c < *cols; c++)
+	    {
+	      if (!(c % 8))
+		ii = buf[c / 8];
+	      if (ii < 0)
+		error_msg ("pbm_readfile", "file reading error", "");
+	      pbm_putpixel (p, c, r, (ii & 128) ? 1 : 0);
+	      ii <<= 1;
+	    }
+	}
+    }				/* p4 type */
+  free (buf);
+  return p;
+}
+
+
+/*
+   Writes a pbm file given a file pointer and bitmap. Assumes that
+   fp is already open, and keeps in that way.
+   usage: pbm_writefile(fp,bitmap,cols,rows);
+ */
+void 
+pbm_writefile (FILE * fp, Pixel ** bitmap, int cols, int rows)
+{
+  int r, c, ii = 0, j;
+  unsigned char *buf;
+
+  buf = (unsigned char *) malloc (((cols + 14) / 8) * sizeof (unsigned char));
+
+  fprintf (fp, "%s\n", "P4");
+  fprintf (fp, "%d %d\n", cols, rows);
+
+  for (r = 0; r < rows; r++)
+    {
+      for (c = 0; c < cols; c += 8)
+	{
+	  ii = 0;
+	  for (j = 0; j < 8; j++)
+	    if (c + j < cols)
+	      {
+		if (pbm_getpixel (bitmap, c + j, r))
+		  ii |= (128 >> j);
+	      }
+	  buf[c / 8] = ii;
+	}
+      fwrite (buf, 1, (cols + 7) / 8, fp);
+    }
+  free (buf);
+/*      putc('\n',fp);   if this line is included programs don't say things like 'short image' &c, but then again,
+   in most cases a 'cmp' between the original will fail because of the last byte...sigh, "standards". */
+}
+
+#define BUF_SIZE 131072
+char buffer[BUF_SIZE];
+
+void 
+pbm_writenamedfile (char fn[], Pixel ** bitmap, int cols, int rows)
+{
+  FILE *tempfp;
+
+  tempfp = fopen (fn, "wb");
+  if (tempfp == NULL)
+    error_msg ("pbm_writenamedfile", "can't open file:", fn);
+  setbuffer (tempfp, buffer, BUF_SIZE);
+  pbm_writefile (tempfp, bitmap, cols, rows);
+  fclose (tempfp);
+}
+
+
+
+Pixel **
+pbm_readnamedfile (char fn[], int *cols, int *rows)
+{
+  FILE *tempfp;
+  Pixel **temp;
+
+  tempfp = fopen (fn, "rb");
+  if (tempfp == NULL)
+    return NULL;
+  setbuffer (tempfp, buffer, BUF_SIZE);
+
+  temp = pbm_readfile (tempfp, cols, rows);
+
+  fclose (tempfp);
+  return temp;
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/images/pbmtools.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/images/pbmtools.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/images/pbmtools.h	(revision 699)
@@ -0,0 +1,86 @@
+/**************************************************************************
+ *
+ * pbmtools.h -- Routines for dealing with portable bitmap files (PBM)
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef __PBMTOOLS_H
+#define __PBMTOOLS_H
+
+#include "sysfuncs.h"
+#include "basictypes.h"
+
+#include "utils.h"
+#include "marklist.h"
+
+#define PBITS 32
+#define PSH 5
+
+
+/* The next two are ordered pixels col0=left, cols31=right */
+
+/*
+   #define pbm_getpixel(bitmap,c,r)  \
+   ((bitmap[r][(c)>>PSH] >> ((PBITS-1)-((c)&(PBITS-1))))&1)
+   #define pbm_putpixel(bitmap,c,r,val) \
+   if((val))  \
+   bitmap[r][(c)>>PSH] |=  (((unsigned)1<<(PBITS-1)) >> ((c)&(PBITS-1))); \
+   else \
+   bitmap[r][(c)>>PSH] &= ~(((unsigned)1<<(PBITS-1)) >> ((c)&(PBITS-1))) 
+ */
+
+
+
+
+/* The next two defines are ordered pixel cols0=right, cols31=left */
+
+#define pbm_getpixel(bitmap,c,r)                            \
+         ((bitmap[r][(c)>>PSH] >> ((c)&(PBITS-1)))&1)
+#define pbm_putpixel(bitmap,c,r,val)                        \
+         if((val))                                          \
+              bitmap[r][(c)>>PSH] |= (1<<((c)&(PBITS-1)));  \
+         else                                               \
+              bitmap[r][(c)>>PSH] &= ~(1<<((c)&(PBITS-1)))
+
+
+#define pbm_putpixel_trunc(bitmap,c,r,val,cols,rows) \
+       if(((c)>=0)&&((c)<cols)&&((r)>=0)&&((r)<rows)) \
+             pbm_putpixel(bitmap,c,r,val)
+
+#define pbm_getpixel_trunc(bitmap,c,r,cols,rows)       \
+       ((((c)>=0)&&((c)<cols)&&((r)>=0)&&((r)<rows)) ?  \
+             pbm_getpixel(bitmap,c,r) : 0)
+
+
+
+
+int pbm_isapbmfile (FILE * fp);
+void pbm_freearray (Pixel *** bitmap, int rows);
+Pixel **pbm_copy (Pixel ** bitmap, int cols, int rows);
+Pixel **pbm_allocarray (int cols, int rows);
+Pixel **pbm_readfile (FILE * fp, int *cols, int *rows);
+Pixel **pbm_readnamedfile (char fn[], int *cols, int *rows);
+void pbm_writefile (FILE * fp, Pixel ** bitmap, int cols, int rows);
+void pbm_writenamedfile (char fn[], Pixel ** bitmap, int cols, int rows);
+
+void pbm_putpixel_range (Pixel ** bitmap, int c, int r, int val, int cols, int rows);
+int pbm_getpixel_range (Pixel ** bitmap, int c, int r, int cols, int rows);
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/src/images/ppm.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/images/ppm.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/images/ppm.c	(revision 699)
@@ -0,0 +1,585 @@
+/**************************************************************************
+ *
+ * ppm.c -- PPM data compression
+ * Copyright (C) 1987  Alistair Moffat
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "arithcode.h"
+#include "model.h"
+
+
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:17:33  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:33:55  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.2  1994/09/20  04:42:02  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+/* FIXEDORDER sets the maximum context length that will be used.
+   Making FIXEDORDER large increases the compression if there is enough
+   space to store the model, but when space limit is reached increasing
+   FIXEDORDER will decrease compression because of model thrashing.
+   Third order (FIXEDORDER=3) is suitable for a few hundred Kb of model
+   space, FIXEDORDER=1 is suitable for a few tens of kB.
+   This version requires that FIXEDORDER be a preprocessor rather
+   than runtime variable to allow loops to be unrolled and make for faster
+   execution. Maximum in this implementation is 3 */
+
+#ifndef FIXEDORDER
+#define FIXEDORDER	3
+#endif
+
+/* global variables */
+eventptr E = NULL;		/* array of eventnodes */
+point numnodes = 0, nfreenodes = 0;
+long kbytes = defaultkbytes;
+/*boolean               excluded[nchars]={false};  - MAHE */
+boolean excluded[16384] =
+{false};			/* excluded characters */
+
+/* local variables */
+#define defaultchunksize        4096	/* compression chunks */
+
+#define EOSchar         (charsetsize)
+
+unsigned chunksize = defaultchunksize;
+eventptr s[FIXEDORDER + 1] =
+{NULL},				/* current contexts */
+  t[FIXEDORDER + 1] =
+{NULL};				/* new contexts */
+
+#if (FIXEDORDER==3)
+#define SHIFT_ST	s[1] = t[0]; s[2] = t[1] ; s[3] = t[2]
+#else
+#if (FIXEDORDER==2)
+#define SHIFT_ST	s[1] = t[0]; s[2] = t[1]
+#else
+#if (FIXEDORDER==1)
+#define SHIFT_ST	s[1] = t[0]
+#else
+#define SHIFT_ST
+#endif
+#endif
+#endif
+
+/* follow a character list and set the exclusions */
+#define SET_EX(p)				\
+do						\
+{	register eventptr q;			\
+	q = p32((p32(p))->next);		\
+	for (; q!=ENULL; q=p32(q->next))	\
+		excluded[q->eventnum] = true;	\
+}						\
+while (false)
+
+/* clear the exclusions */
+#define CLEAR_EX(p)				\
+do						\
+{	register eventptr q;			\
+	q = p32((p32(p))->next);		\
+	for (; q!=ENULL; q=p32(q->next))	\
+		excluded[q->eventnum] = false;	\
+}						\
+while (false)
+
+/*==================================*/
+
+void
+write_model ()
+{
+  fprintf (stderr, "ppm");
+  write_method ();
+#ifdef NOEXCLUSIONS
+  fprintf (stderr, "nx");
+#endif
+  fprintf (stderr, "-o%1d", FIXEDORDER);
+  fprintf (stderr, "-k%ld", kbytes);
+}
+
+/*==================================*/
+
+void
+encode_in_order (n, e)
+     int n;
+     event e;
+/* encode character e in order n */
+{
+  boolean encoded;
+  int i, c;
+  t[n] = encode_event_noex (&s[n]->foll, e, &encoded);
+  if (encoded)
+    c = n;
+  else
+    for (c = n - 1; c >= 0; c--)
+      {
+#ifdef NOEXCLUSIONS
+	t[c] = encode_event_noex (&s[c]->foll, e, &encoded);
+#else
+	SET_EX (s[c + 1]->foll.list);
+	t[c] = encode_event (&s[c]->foll, e, &encoded);
+#endif
+	if (encoded)
+	  break;
+      }
+  if (!encoded)
+    arithmetic_encode (e, e + 1, nchars);
+  for (i = c + 1; i <= n; i++)
+    if (i > 0)
+      t[i]->prev = p16 (t[i - 1]);
+    else
+      t[0]->prev = p16 (s[0]);
+  for (i = c - 1; i >= 0; i--)
+    t[i] = p32 (t[i + 1]->prev);
+#ifndef NOEXCLUSIONS
+  if (c != n)
+    CLEAR_EX (s[c + 1]->foll.list);
+#endif
+  SHIFT_ST;
+}
+
+/*==================================*/
+
+/* not required in this version */
+
+#if 0
+
+void
+decode_in_order (n, e)
+     int n;
+     event *e;
+{
+  boolean decoded;
+  int i, c;
+  t[n] = decode_event_noex (&s[n]->foll, &decoded, e);
+  if (decoded)
+    c = n;
+  else
+    for (c = n - 1; c >= 0; c--)
+      {
+#ifdef NOEXCLUSIONS
+	t[c] = decode_event_noex (&s[c]->foll, &decoded, e);
+#else
+	SET_EX (s[c + 1]->foll.list);
+	t[c] = decode_event (&s[c]->foll, &decoded, e);
+#endif
+	if (decoded)
+	  break;
+      }
+  if (!decoded)
+    {
+      *e = arithmetic_decode_target (nchars);
+      arithmetic_decode (*e, *e + 1, nchars);
+    }
+  for (i = c + 1; i <= n; i++)
+    {
+      t[i]->eventnum = *e;
+      if (i > 0)
+	t[i]->prev = p16 (t[i - 1]);
+      else
+	t[0]->prev = p16 (s[0]);
+    }
+  {
+    register int i;
+    for (i = c - 1; i >= 0; i--)
+      t[i] = p32 (t[i + 1]->prev);
+  }
+#ifndef NOEXCLUSIONS
+  if (c != n)
+    CLEAR_EX (s[c + 1]->foll.list);
+#endif
+  SHIFT_ST;
+}
+
+#endif
+
+/*==================================*/
+
+void
+start_model ()
+{
+  int i;
+  /* is this the first call? */
+  if (E == NULL)
+    E = (eventptr) malloc ((unsigned) kbytes * 1024);
+  /* start off with no information */
+  numnodes = 0;
+  nfreenodes = maxnodes;
+  for (i = 0; i <= FIXEDORDER; i++)
+    {
+      s[i] = newnode (NULL);
+      if (i > 0)
+	s[i]->prev = p16 (s[i - 1]);
+    }
+}
+
+/*==================================*/
+
+void
+encodestring (str, n)
+     unsigned short *str;
+     int n;
+/* encode all n characters of the supplied string str */
+{
+  event e;
+  boolean encoded;
+  register eventptr S, T;
+  int i;
+#ifndef NOEXCLUSIONS
+  register eventptr Sex;
+#endif
+
+  if (E == NULL)
+    start_model ();
+  /* set the initial context node */
+  S = s[FIXEDORDER];
+
+  for (i = 0; i < n; i++)
+    {
+      /* encode one character */
+      e = *str++;
+      /* this next monstrosity is logically equivalent to 
+         encode_in_order(FIXEDORDER,e)
+         the loop has been unrolled to make it all go faster */
+      for (;;)
+	{
+	  T = encode_event_noex (&S->foll, e, &encoded);
+	  if (encoded)
+	    break;
+	  t[FIXEDORDER] = T;
+
+#ifdef NOEXCLUSIONS
+#if (FIXEDORDER>2)
+	  S = p32 (S->prev);
+	  t[2] = encode_event_noex (&S->foll, e, &encoded);
+	  t[3]->prev = p16 (t[2]);
+	  if (encoded)
+	    break;
+#endif
+
+#if (FIXEDORDER>1)
+	  S = p32 (S->prev);
+	  t[1] = encode_event_noex (&S->foll, e, &encoded);
+	  t[2]->prev = p16 (t[1]);
+	  if (encoded)
+	    break;
+#endif
+
+#if (FIXEDORDER>0)
+	  S = p32 (S->prev);
+	  t[0] = encode_event_noex (&S->foll, e, &encoded);
+	  t[1]->prev = p16 (t[0]);
+	  if (encoded)
+	    break;
+#endif
+#else
+#if (FIXEDORDER>2)
+	  Sex = S;
+	  SET_EX (Sex->foll.list);
+	  S = p32 (S->prev);
+	  t[2] = encode_event (&S->foll, e, &encoded);
+	  t[3]->prev = p16 (t[2]);
+	  if (encoded)
+	    {
+	      CLEAR_EX (Sex->foll.list);
+	      break;
+	    }
+#endif
+
+#if (FIXEDORDER>1)
+	  Sex = S;
+	  SET_EX (Sex->foll.list);
+	  S = p32 (S->prev);
+	  t[1] = encode_event (&S->foll, e, &encoded);
+	  t[2]->prev = p16 (t[1]);
+	  if (encoded)
+	    {
+	      CLEAR_EX (Sex->foll.list);
+	      break;
+	    }
+#endif
+
+#if (FIXEDORDER>0)
+	  Sex = S;
+	  SET_EX (Sex->foll.list);
+	  S = p32 (S->prev);
+	  t[0] = encode_event (&S->foll, e, &encoded);
+	  t[1]->prev = p16 (t[0]);
+	  if (encoded)
+	    {
+	      CLEAR_EX (Sex->foll.list);
+	      break;
+	    }
+	  CLEAR_EX (Sex->foll.list);
+#endif
+#endif
+
+	  arithmetic_encode (e, e + 1, nchars);
+	  t[0]->prev = p16 (s[0]);
+	  break;
+	}
+      /* phew! Update the current context to the latest node */
+      S = p32 (T->prev);
+
+      /* is there a danger of running out of space? */
+      if (nfreenodes <= FIXEDORDER)
+	{
+	  start_model ();
+	  S = s[FIXEDORDER];
+	}
+    }
+
+  /* save the current contexts for the various order submodels */
+  s[FIXEDORDER] = S;
+  for (i = FIXEDORDER; i > 0; i--)
+    s[i - 1] = p32 (s[i]->prev);
+  /* and send the terminator character */
+  encode_in_order (FIXEDORDER, EOSchar);
+  /* and check the space again */
+  if (nfreenodes <= FIXEDORDER)
+    start_model ();
+}
+
+/*==================================*/
+
+void
+decodestring (str, n)
+     unsigned short *str;
+     int *n;
+{
+  event e;
+  boolean decoded;
+  register eventptr S, T;
+#ifndef NOEXCLUSIONS
+  register eventptr Sex;
+#endif
+
+  /* just mimic the corresponding encoding actions */
+  if (E == NULL)
+    start_model ();
+  S = s[FIXEDORDER];
+
+  *n = 0;
+  for (;;)
+    {				/* logically equivalent to "decode_in_order(FIXEDORDER,&e)" */
+
+      for (;;)
+	{
+	  T = decode_event_noex (&S->foll, &decoded, &e);
+	  if (decoded)
+	    break;
+	  t[FIXEDORDER] = T;
+
+#ifdef NOEXCLUSIONS
+#if (FIXEDORDER>2)
+	  S = p32 (S->prev);
+	  t[2] = decode_event_noex (&S->foll, &decoded, &e);
+	  t[3]->prev = p16 (t[2]);
+	  if (decoded)
+	    {
+	      t[3]->eventnum = e;
+	      break;
+	    }
+#endif
+
+#if (FIXEDORDER>1)
+	  S = p32 (S->prev);
+	  t[1] = decode_event_noex (&S->foll, &decoded, &e);
+	  t[2]->prev = p16 (t[1]);
+	  if (decoded)
+	    {
+#if (FIXEDORDER>2)
+	      t[3]->eventnum =
+#endif
+		t[2]->eventnum = e;
+	      break;
+	    }
+#endif
+
+#if (FIXEDORDER>0)
+	  S = p32 (S->prev);
+	  t[0] = decode_event_noex (&S->foll, &decoded, &e);
+	  t[1]->prev = p16 (t[0]);
+	  if (decoded)
+	    {
+#if (FIXEDORDER>2)
+	      t[3]->eventnum =
+#endif
+#if (FIXEDORDER>1)
+		t[2]->eventnum =
+#endif
+		t[1]->eventnum = e;
+	      break;
+	    }
+#endif
+#else
+#if (FIXEDORDER>2)
+	  Sex = S;
+	  SET_EX (Sex->foll.list);
+	  S = p32 (S->prev);
+	  t[2] = decode_event (&S->foll, &decoded, &e);
+	  t[3]->prev = p16 (t[2]);
+	  if (decoded)
+	    {
+	      t[3]->eventnum = e;
+	      CLEAR_EX (Sex->foll.list);
+	      break;
+	    }
+#endif
+
+#if (FIXEDORDER>1)
+	  Sex = S;
+	  SET_EX (Sex->foll.list);
+	  S = p32 (S->prev);
+	  t[1] = decode_event (&S->foll, &decoded, &e);
+	  t[2]->prev = p16 (t[1]);
+	  if (decoded)
+	    {
+#if (FIXEDORDER>2)
+	      t[3]->eventnum =
+#endif
+		t[2]->eventnum = e;
+	      CLEAR_EX (Sex->foll.list);
+	      break;
+	    }
+#endif
+
+#if (FIXEDORDER>0)
+	  Sex = S;
+	  SET_EX (Sex->foll.list);
+	  S = p32 (S->prev);
+	  t[0] = decode_event (&S->foll, &decoded, &e);
+	  t[1]->prev = p16 (t[0]);
+	  if (decoded)
+	    {
+#if (FIXEDORDER>2)
+	      t[3]->eventnum =
+#endif
+#if (FIXEDORDER>1)
+		t[2]->eventnum =
+#endif
+		t[1]->eventnum = e;
+	      CLEAR_EX (Sex->foll.list);
+	      break;
+	    }
+	  CLEAR_EX (Sex->foll.list);
+#endif
+#endif
+
+	  e = arithmetic_decode_target (nchars);
+	  arithmetic_decode (e, e + 1, nchars);
+#if (FIXEDORDER>2)
+	  t[3]->eventnum =
+#endif
+#if (FIXEDORDER>1)
+	    t[2]->eventnum =
+#endif
+#if (FIXEDORDER>0)
+	    t[1]->eventnum =
+#endif
+	    t[0]->eventnum = e;
+	  t[0]->prev = p16 (s[0]);
+	  break;
+	}
+      S = p32 (T->prev);
+
+      /* check the space requirements */
+      if (nfreenodes <= FIXEDORDER)
+	{
+	  start_model ();
+	  S = s[FIXEDORDER];
+	}
+      /* are we done?  or do we save the char */
+      if (e == EOSchar)
+	break;
+      *str++ = e;
+      (*n)++;
+    }
+  s[FIXEDORDER] = S;
+}
+
+
+#if 0				/* Writing my own version of this - MAHE */
+void
+encodefile ()
+/* encodes a file by breaking into a sequence of fixed length chunks */
+{
+  unsigned char *str;
+  int c, n = 0;
+
+  str = (unsigned char *) malloc ((unsigned) chunksize);
+  startoutputingbits ();
+  startencoding ();
+
+  /* to encode the file, break it into chunks and encode the
+     chunks one by one */
+  while ((c = getchar ()) != EOF)
+    {
+      str[n++] = c;
+      if (n == chunksize)
+	{
+	  encodestring (str, n);
+	  rawbytes += n;
+	  n = 0;
+	}
+    }
+  /* send final partial chunk */
+  if (n > 0)
+    {
+      encodestring (str, n);
+      rawbytes += n;
+    }
+  /* send EOF as an empty string */
+  encodestring (str, 0);
+  doneencoding ();
+  doneoutputingbits ();
+}
+
+void
+decodefile ()
+{
+  unsigned char *str;
+  int n, i;
+
+  str = (unsigned char *) malloc ((unsigned) chunksize);
+  startinputingbits ();
+  startdecoding ();
+  for (;;)
+    {
+      decodestring (str, &n);
+      if (n == 0)
+	break;
+      for (i = 0; i < n; i++)
+	putchar ((int) str[i]);
+      rawbytes += n;
+    }
+}
+
+#endif /* #if 0 */
Index: /tags/gsdl-21/gsdl/packages/mg/src/images/sortmarks.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/images/sortmarks.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/images/sortmarks.c	(revision 699)
@@ -0,0 +1,93 @@
+/**************************************************************************
+ *
+ * sortmarks.c -- Functions which sort the marks into a "readable sequence"
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "marklist.h"
+#include "sortmarks.h"
+
+#define GAP 4
+
+
+
+static int 
+CmpOnY (const void *e1, const void *e2)
+{
+  return (((marktype *) e1)->ycen * 0 + ((marktype *) e1)->ypos) -
+    (((marktype *) e2)->ycen * 0 + ((marktype *) e2)->ypos);
+}
+
+static int 
+CmpOnX (const void *e1, const void *e2)
+{
+  return (((marktype *) e1)->xcen * 0 + ((marktype *) e1)->xpos) -
+    (((marktype *) e2)->xcen * 0 + ((marktype *) e2)->xpos);
+}
+
+
+
+
+
+
+
+
+marklistptr 
+sortmarks (marklistptr listofmarks)
+{
+  marktype *table;
+  int count = 0;
+  int i, s, e;
+
+
+  table = calloc ((unsigned int) marklist_length (listofmarks), sizeof (marklisttype));
+
+  while (listofmarks)
+    {
+      table[count] = listofmarks->data;
+      marklist_removeat (&listofmarks, 0);
+      count++;
+    }
+
+  /* sort first on the y position of their centroids */
+  qsort ((void *) table, (unsigned int) (count), sizeof (*table), CmpOnY);
+
+  for (s = 0; s < count;)
+    {
+      int c;
+
+      for (e = s + 1, c = 0; (e < count) &&
+	   ((table[e].ypos + table[e].ycen * 0) - (table[e - 1].ypos + table[e - 1].ycen * 0) <= GAP); c++, e++);
+
+      if (e < count && c >= 1)
+	qsort (&table[s], (unsigned int) (e - s), sizeof (*table), CmpOnX);
+      s = e;
+    }
+
+  for (i = 0; i < count; i++)
+    {
+      table[i].symnum = i;
+      marklist_add (&listofmarks, table[i]);
+    }
+
+  return listofmarks;
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/images/sortmarks.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/images/sortmarks.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/images/sortmarks.h	(revision 699)
@@ -0,0 +1,30 @@
+/**************************************************************************
+ *
+ * sortmarks.h -- Functions which sort the marks into a "readable sequence"
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef _SortMarks_h_
+#define _SortMarks_h_
+
+marklistptr sortmarks (marklistptr listofmarks);
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/src/images/utils.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/images/utils.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/images/utils.c	(revision 699)
@@ -0,0 +1,270 @@
+/**************************************************************************
+ *
+ * utils.c -- Functions which are common utilities for the image programs
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "utils.h"
+
+
+int V = 0;
+
+
+int 
+isEOF (FILE * fp)
+{
+  int ch;
+
+  ch = getc (fp);
+  ungetc (ch, fp);
+  if (ch == EOF)
+    return 1;
+  return 0;
+}
+
+
+int 
+getmagicno_short (FILE * fp)
+{
+  int ch1, ch2;
+  ch1 = getc (fp);
+  if (ch1 == EOF)
+    {
+      ungetc (ch1, fp);
+      return 0;
+    }
+  ch2 = getc (fp);
+  if (ch2 == EOF)
+    {
+      ungetc (ch2, fp);
+      return 0;
+    }
+  ungetc (ch2, fp);
+  ungetc (ch1, fp);
+
+  return ((ch1 << 8) | ch2);
+}
+
+int 
+getmagicno_short_pop (FILE * fp)
+{
+  int ch1, ch2;
+  ch1 = getc (fp);
+  if (ch1 == EOF)
+    return 0;
+  ch2 = getc (fp);
+  if (ch2 == EOF)
+    return 0;
+
+  return ((ch1 << 8) | ch2);  
+}
+
+int 
+getmagicno_byte (FILE * fp)
+{
+  int ch1;
+  ch1 = getc (fp);
+  if (ch1 == EOF)
+    {
+      ungetc (ch1, fp);
+      return 0;
+    }
+  ungetc (ch1, fp);
+  return (ch1);
+}
+
+
+int 
+getmagicno_long (FILE * fp)
+{
+  int ch1, ch2, ch3, ch4;
+  ch1 = getc (fp);
+  if (ch1 == EOF)
+    {
+      ungetc (ch1, fp);
+      return 0;
+    }
+  ch2 = getc (fp);
+  if (ch2 == EOF)
+    {
+      ungetc (ch2, fp);
+      return 0;
+    }
+  ch3 = getc (fp);
+  if (ch3 == EOF)
+    {
+      ungetc (ch3, fp);
+      return 0;
+    }
+  ch4 = getc (fp);
+  if (ch4 == EOF)
+    {
+      ungetc (ch4, fp);
+      return 0;
+    }
+  ungetc (ch4, fp);
+  ungetc (ch3, fp);
+  ungetc (ch2, fp);
+  ungetc (ch1, fp);
+
+  return ((ch1 << 24) | (ch2 << 16) | (ch3 << 8) | (ch4));
+}
+
+
+
+
+void 
+error_msg (char *prog, char *message, char *extra)
+{
+  fprintf (stderr, "%s: %s %s\n", prog, message, extra);
+  exit (1);
+}
+
+void 
+warn (char *prog, char *message, char *extra)
+{
+  fprintf (stderr, "%s: %s %s\n", prog, message, extra);
+}
+
+
+
+void 
+readline (char str[], FILE * fp)
+{
+  int i = 0, ch;
+
+  while (((ch = fgetc (fp)) != '\n') && (!feof (fp)))
+    str[i++] = ch;
+  str[i] = '\0';
+}
+
+
+
+
+
+
+
+int 
+getint (FILE * fp)
+{
+  register char ch;
+  register unsigned int i = 0;
+
+  do
+    {
+      ch = getc (fp);
+      if (feof (fp))
+	return EOF;
+    }
+  while (ch < '0' || ch > '9');
+
+  do
+    {
+      i = i * 10 + ch - '0';
+      ch = getc (fp);
+      if (feof (fp))
+	return EOF;
+    }
+  while (ch >= '0' && ch <= '9');
+
+  return i;
+}
+
+/* get_header_int i.e. get an integer from a PNM header ; basically  
+   the same as
+   above except for comment checking ; above kept because of lower  
+   overhead */
+
+unsigned int 
+gethint (FILE * fp)
+{
+  register char ch, prev = '\0';
+  register unsigned int i = 0;
+
+  do
+    {
+      ch = getc (fp);
+      if ((ch == '#') && (prev == '\n'))
+	while (((ch = getc (fp)) != '\n') && !(feof (fp)));
+      prev = ch;
+      if (feof (fp))
+	return EOF;
+    }
+  while (ch < '0' || ch > '9');
+
+  do
+    {
+      i = i * 10 + ch - '0';
+      ch = getc (fp);
+      if (feof (fp))
+	return EOF;
+    }
+  while (ch >= '0' && ch <= '9');
+
+  return i;
+}
+
+
+
+
+void 
+magic_write (FILE * fp, u_long magic_num)
+{
+  HTONUL(magic_num);  /* [RPAP - Jan 97: Endian Ordering] */
+  if (fwrite (&magic_num, sizeof (magic_num), 1, fp) != 1)
+    error_msg ("magic num", "Couldn't write magic number.", "");
+}
+
+
+void 
+magic_check (FILE * fp, u_long magic_num)
+{
+  u_long magic;
+  if (fread (&magic, sizeof (magic), 1, fp) != 1 || NTOHUL(magic) != magic_num)  /* [RPAP - Jan 97: Endian Ordering] */
+    error_msg ("magic num", "Incorrect magic number.", "");
+}
+
+
+u_long 
+magic_read (FILE * fp)
+{
+  u_long magic;
+  if (fread (&magic, sizeof (magic), 1, fp) != 1)
+    error_msg ("magic num", "Couldn't read magic number.", "");
+  return NTOHUL(magic);  /* [RPAP - Jan 97: Endian Ordering] */
+}
+
+
+
+
+int 
+isinteger (char s[])
+{
+  int i = 0;
+
+  for (i = 0; s[i] != '\0'; i++)
+    if (!isdigit (s[i]))
+      return 0;
+  return 1;
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/images/utils.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/images/utils.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/images/utils.h	(revision 699)
@@ -0,0 +1,75 @@
+/**************************************************************************
+ *
+ * utils.h -- Functions which are common utilities for the image programs
+ * Copyright (C) 1994  Stuart Inglis
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef __UTILS_H
+#define __UTILS_H
+
+#include "sysfuncs.h"
+
+extern int V;
+
+#define DEBUG
+
+#define min(a,b) ((a)<(b) ? (a) : (b))
+#define max(a,b) ((a)>(b) ? (a) : (b))
+
+#define GEN_MAGIC(a,b,c,d) ((unsigned long)(((a)<<24) + ((b)<<16) + \
+                                            ((c)<<8) + (d)))
+ 
+/* --- MG magic file codes [TS:Aug/95] --- */
+#define MAGIC_FELICS            GEN_MAGIC('M','G','F','X')
+#define MAGIC_BILEVEL           GEN_MAGIC('M','G','B','I')
+#define MAGIC_TIC               GEN_MAGIC('M','G','T','C')
+#define IS_MAGIC(a) ((((u_long)(a)) & 0xffff0000) == MAGIC_XXXX)
+/* --------------------------------------- */
+
+#define MAGIC_P1      0x5031	/* P1   - pbm */
+#define MAGIC_P2      0x5032	/* P2   - pgm */
+#define MAGIC_P3      0x5033	/* P3   - ppm */
+#define MAGIC_P4      0x5034	/* P4   - rawbits pbm */
+#define MAGIC_P5      0x5035	/* P5   - rawbits pgm */
+#define MAGIC_P6      0x5036	/* P6   - rawbits ppm */
+
+int isEOF (FILE * fp);
+int getmagicno_byte (FILE * fp);
+int getmagicno_short (FILE * fp);
+int getmagicno_short_pop (FILE * fp);
+int getmagicno_long (FILE * fp);
+
+
+
+void error_msg (char *prog, char *message, char *extra);
+void warn (char *prog, char *message, char *extra);
+
+void readline (char str[], FILE * fp);
+
+void magic_write (FILE * fp, u_long magic_num);
+void magic_check (FILE * fp, u_long magic_num);
+u_long magic_read (FILE * fp);
+/* kerry's code */
+
+int getint (FILE * fp);
+unsigned int gethint (FILE * fp);
+int isinteger (char s[]);
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/src/scripts/Makefile.in
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/scripts/Makefile.in	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/scripts/Makefile.in	(revision 699)
@@ -0,0 +1,151 @@
+###########################################################################
+#
+# Makefile.in -- Makefile for the MG system
+# Copyright (C) 1994  Neil Sharman
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)Makefile.in	1.10 22 Mar 1994
+#
+###########################################################################
+
+PACKAGE = @PACKAGE@
+VERSION = @VERSION@
+ 
+SHELL = /bin/sh
+VPATH = @srcdir@
+exec_prefix = @exec_prefix@
+prefix = @prefix@
+srcdir = @srcdir@
+subdir = src/scripts
+top_srcdir = @top_srcdir@
+transform = @program_transform_name@
+ 
+DEFS = @DEFS@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+TOUCH = echo timestamp >
+ 
+bindir = $(exec_prefix)/bin
+datadir = $(prefix)/share
+libexecdir = $(exec_prefix)/libexec
+localedir = $(datadir)/locale
+mandir = $(prefix)/man/man1
+ 
+SHFILES       =	mgbuild.sh mgbuild_4.sh mg_get.sh xmg.sh  mg_get_merge.sh mgmerge.sh
+
+MISC          = Makefile.in 
+
+ALLFILES      =	$(SHFILES) $(MISC) $(MAN)
+
+DISTFILES     = $(ALLFILES) 
+
+EXEC          = mgbuild mg_get xmg mg_get_merge mgmerge mgbuild_4
+
+MAN           = mgbuild.1 mgbuild_4.1 mg_get.1 xmg.1 mg_get_merge.1 mgmerge.1
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+
+all:	$(EXEC)
+
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+mgbuild : mgbuild.sh
+	@rm -f $@
+	cat mgbuild.sh >$@
+	chmod +x $@
+
+mgbuild_4 : mgbuild_4.sh
+	@rm -f $@
+	cat mgbuild_4.sh >$@
+	chmod +x $@
+
+mg_get : mg_get.sh
+	@rm -f $@
+	cat mg_get.sh >$@
+	chmod +x $@
+
+mgmerge : mgmerge.sh
+	@rm -f $@
+	cat mgmerge.sh >$@
+	chmod +x $@
+
+mg_get_merge : mg_get_merge.sh
+	@rm -f $@
+	cat mg_get_merge.sh >$@
+	chmod +x $@
+
+xmg : xmg.sh
+	@rm -f $@
+	cat xmg.sh >$@
+	chmod +x $@
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+
+install: execinstall maninstall
+uninstall: execuninstall manuninstall
+
+execinstall: $(EXEC)
+	$(top_srcdir)/mkinstalldirs $(bindir); \
+	for name in $(EXEC); do \
+	  $(INSTALL_PROGRAM) $$name \
+	  $(bindir)/`echo $$name | sed '$(transform)'`; \
+	done
+
+execuninstall:
+	for name in $(EXEC); do \
+	 rm -f $(bindir)/`echo $$name | sed '$(transform)'`; \
+	done
+
+maninstall: $(MAN)
+	$(top_srcdir)/mkinstalldirs $(mandir); \
+	for f in $(MAN); do \
+	  $(INSTALL_DATA) $$f $(mandir)/$$f ;\
+	done
+
+manuninstall: $(MAN)
+	for f in $(MAN); do \
+	  rm -f $(mandir)/$$f ;\
+	done
+
+mostlyclean:
+ 
+clean: mostlyclean
+	rm -f $(EXEC)
+ 
+distclean: clean
+	rm -f ansi2knr
+	rm -f Makefile
+ 
+maintainer-clean: distclean
+	@echo "This command is intended only for maintainers to use;"
+	@echo "rebuilding the deleted files may require special tools."
+ 
+dist: $(DISTFILES)
+	for file in $(DISTFILES); do \
+	  ln $(srcdir)/$$file $(distdir) 2> /dev/null \
+	  || cp -p $(srcdir)/$$file $(distdir); \
+	done
+ 
+Makefile: Makefile.in ../../../../config.status
+	cd ../../../.. && CONFIG_FILES=$(subdir)/$@ CONFIG_HEADERS= ./config.status
+ 
+# Tell versions [3.59,3.63) of GNU make not to export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:
Index: /tags/gsdl-21/gsdl/packages/mg/src/scripts/mg_get.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/scripts/mg_get.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/scripts/mg_get.1	(revision 699)
@@ -0,0 +1,154 @@
+.TH mg_get 1 "31 January 1994"
+.SH NAME
+mg_get \- output source texts for processing
+.SH SYNOPSIS
+ 
+.B mg_get
+.I collection-name
+.RB [ \-init | \-i | \-text\fP|\fB\-t\fP|\fB\-cleanup\fP|\fB\-c\fP]
+.SH DESCRIPTION
+.LP
+This program is the default one used by mgbuild to generate the source text for the MG system. Any program may be used to generate the source text for mgbuild as long as it confirms to the interface specified here. 
+.SH OPTIONS
+.LP
+The
+.I collection\-name
+must appear before any other option. Only the first option has any significance. If no option is specified
+.B \-text
+is assumed.
+.RS
+.TP
+.BR \-init " and " \-i
+The program is called once with this flag at the start of building a collection.
+.TP
+.BR \-text " and " \-t
+The program is called with this flag multiple times during the building of a collection. The program outputs (on stdout) the text of the collection. \*(lq\fBDocuments\fP\*(rq within the collection are separated with by ctrl-B's (ASCII code 2). \*(lq\fBParagraphs\fP\*(rq within the collection are separated with ctrl-C's (ASCII code 3). A collection need not contain paragraphs unless a level 3 inverted file is being generated (see 
+.B mgbuild 
+and 
+.BR mg_passes ). 
+.TP
+.BR \-cleanup " and " \-c
+The program is called once with this flag at the completion of building a collection.
+.SH ENVIRONMENT
+.TP
+.SB MGDATA
+If this environment variable exists then its value is used a the default 
+directory where the mg collection files are. If this variable does not exist
+then the directory \*(lq\fB.\fP\*(rq is used by default. The command line 
+option 
+.BI \-d " directory"
+overrides the directory in 
+.BR MGDATA .
+.TP
+.SB MG_GETRC
+This environment variable specifies where the file containing the users 
+mg source configurations is. If not set, mg_get uses a default of 
+.B \*\~/.mg_getrc.
+
+This file contains TAB delimited lines of the form
+.RS 
+.RS 
+.I CollectionName
+.I CollectionType
+.I files or directory
+.RE
+
+.I CollectionName
+is the name of the collection supplied to mg_get.
+
+.I CollectionType
+specifies how mg_get should process the
+named files and directories and are descibed below.
+
+.I files or directory
+is either a list of files separted by blanks or a single directory.
+Some of the
+.I CollectionTypes
+deal with files and others with just a single directory.
+Any files used ending with .gz or .Z are decompressed with gzip before
+processing. References to '~' expands to the users HOME directory.
+
+.I CollectionType
+is one of
+.RS
+.TP
+.BR PARA
+For text based documents. The list of files specified are treated as
+a series of paragraphs separated by blank lines. Each paragraph becomes
+a seperate document on the indexed collection.
+.TP
+.BR MAIL
+for mail files. The list of files specified are treated as UNIX mail
+files separated by lines starting with 'From'. Each mail message becomes
+a seperate document on the indexed collection. As a extra feature any
+embedded tarmail encoded contents (enclosed by a 'xbtoa Begin' and 'xbtoa End'
+pair are removed.
+.TP
+.BR DIR
+(and DIR2)
+For a single directory of files. Each file in the directory (and in any of
+it's subdirectories) are treated as a single document. With
+.I DIR
+the pathname of the file is prefixed to the contents of each file as an
+extra line while this is not done for
+.I DIR2
+collections.
+.TP
+.BR BIB
+for biblography files. The list of files specified are treated as
+a series of biblography files (eg BIBTEX or TROFF) separated by lines
+starting with '@'. Each reference  becomes a seperate document on the indexed
+collection.
+.TP
+.BR TXTIMG
+for integrated collections of text and images. The single directory should
+contain files that have the same prefix if they are related. For example,
+monaLisa.pgm might be a gray-level image, and monaLisa.txt would be a textual
+file describing the image. The suffixes recognised are:
+.BR   .txt
+for  ascii text
+.BR  .ptm
+for  scanned text stored as a bilevel image
+.BR  .pbm
+for a black and white image (typically a line drawing)
+.BR  .pgm
+for a gray-scale image
+.BR In addition, if no corresponding ascii text file is found for
+a .pbm or .pgm file, then one is created with suffix .tmp.txt,
+and it stores the name of the image file (in principle it could
+store the OCR of a .txt.pbm file). At present the .tmp.txt files
+are deleted by the '-cleanup' option.
+
+.SH "EXAMPLE"
+
+An example ~/.mg_getrc file might look like:
+   alice   PARA    /users/alice13a.txt.Z
+   mail95  MAIL    ~/MAIL/1995/*
+   doc     DIR     ~/documents
+   bibs    BIB     ~/etc/REFS/*.bib ~/etc/REFS/CC/*.bib
+   letters DIR     ~/LETTERS
+   davinci TXTIMG  ~/images/davinci
+
+.SH "SEE ALSO"
+.LP
+.BR mg_compression_dict (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_make_fast_dict (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1)
+
Index: /tags/gsdl-21/gsdl/packages/mg/src/scripts/mg_get.sh
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/scripts/mg_get.sh	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/scripts/mg_get.sh	(revision 699)
@@ -0,0 +1,447 @@
+#! /bin/csh -f
+###########################################################################
+#
+# mg_get.sh -- script to get text for mgbuild
+# Copyright (C) 1994  Tim Bell
+# Changed to allow it to be driven from a file ~/.mg_getrc
+# by Bruce McKenzie, Oct 1994
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)mg_get.sh	1.11 25 Mar 1994
+#
+###########################################################################
+#
+# "mg_get" formats the text for mgbuild by collecting the appropriate
+#   documents, and outputing them with control-B after each document.
+#   The name of the text must be supplied as an argument.
+#   mg_get is called with -i (init) first, then with -t (text) each time the 
+#   text is needed, then with -c (cleanup).
+#   e.g. mg_get alice -t
+#
+# The file ~/.mg_get contains lines of the form:
+#     name	TYPE	files and directories
+# where name is the name supplied to mg_get, TYPE is one of
+# PARA, MAIL, DIR, DIR2, BIB or TXTIMG and says how mg_get should process the
+# named files and directories
+#
+###########################################################################
+
+if ($?MG_GETRC) then
+  set mg_getrc = $MG_GETRC
+else
+  set mg_getrc = ~/.mg_getrc
+endif
+
+# Added by [TS:May/95]
+# if we don't have an mg_getrc file in existence then create a default one
+if !(-e $mg_getrc) then 
+  echo "Can not find a .mg_getrc, so creating a default one: $mg_getrc"
+
+  if ($?MGSAMPLE) then
+    set SampleData = $MGSAMPLE
+  else
+    set SampleData = ./SampleData
+  endif
+
+  cat > $mg_getrc << END
+alice	PARA	$SampleData/alice13a.txt.Z
+davinci	TXTIMG	$SampleData/davinci
+mailfiles	MAIL	~/mbox ~/.sentmail
+allfiles	DIR	~/Mail
+mods	LINESEP	$SampleData/MODIFICATIONS
+bible	PLAIN	$SampleData/bible/genesis.txt.Z $SampleData/bible/revelation.txt.Z
+END
+endif
+
+if (! $?MGIMAGEVIEWER) then
+  setenv MGIMAGEVIEWER "xv -"
+endif
+
+set rm = /bin/rm
+
+switch ($#argv)
+  case 1:
+  set flag = '-text'
+  breaksw
+
+  case 2:
+  set flag = $2
+  switch ($flag)
+     case '-i':
+	set flag = '-init'
+	breaksw
+     case '-t':
+	set flag = '-text'
+	breaksw
+     case '-c':
+	set flag = '-cleanup'
+	breaksw
+     case '-init':
+     case '-text':
+     case '-cleanup':
+     breaksw
+     default:
+     echo 'Usage: mg_get <document> [-i | -t | -c]'
+     exit(1)
+     endsw
+  breaksw
+
+  default:
+  echo 'Usage: get <document> [-i | -t | -c]'
+  exit(1)
+  endsw
+
+set COLLECTION = $1
+set TYPE  = `grep \^$1 $mg_getrc | cut -f2`
+set FILES = `grep \^$1 $mg_getrc | cut -f3`
+
+set bindir = $0
+set bindir = $bindir:h
+
+switch ($TYPE)
+  case PLAIN:
+# Just outputs the given text, assumes ^Bs already inserted 
+  switch ($flag)
+     case '-init':
+     breaksw
+
+     case '-text':
+      foreach f ($FILES)
+	if ($f:e == 'Z') then
+	  set decoder=zcat
+	else if ($f:e == 'gz') then
+	  set decoder=gunzip
+	else
+	  set decoder=cat
+	endif
+        $decoder < $f
+      end
+     breaksw #-text
+
+     case '-cleanup':
+     breaksw
+  endsw #flag
+  breaksw
+
+  case LINESEP:
+# Assumes that the documents are separated by a line of =====
+  switch ($flag)
+     case '-init':
+     breaksw
+
+     case '-text':
+      foreach f ($FILES)
+	if ($f:e == 'Z') then
+	  set decoder=zcat
+	else if ($f:e == 'gz') then
+	  set decoder=gunzip
+	else
+	  set decoder=cat
+	endif
+        $decoder < $f |  sed -e '/\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*\*/s/.*//' < $f
+      end
+     breaksw #-text
+
+     case '-cleanup':
+     breaksw
+  endsw #flag
+  breaksw
+
+  case PARA:
+# Splits the file at blank lines, which corresponds to paragraphs.
+  switch ($flag)
+     case '-init':
+     breaksw
+
+     case '-text':
+      foreach f ($FILES)
+	if ($f:e == 'Z') then
+	  set decoder=zcat
+	else if ($f:e == 'gz') then
+	  set decoder=gunzip
+	else
+	  set decoder=cat
+	endif
+      $decoder <$f |awk '   /^ *$/ {if (b!=1) printf "";b=1} \
+		\!/^ *$/ {print;b=0} \
+		 END {printf ""}'
+       end #foreach
+     breaksw #-text
+
+     case '-cleanup':
+     breaksw
+  endsw #flag
+  breaksw
+
+  case BIB:
+# Takes a list of files that contain bibliographies, and splits them up 
+#   by putting ^B after each reference. Assumes that each reference
+#   begins with a line '^@'.
+
+  switch ($flag)
+    case '-init':
+    breaksw
+ 
+    case '-text':
+      foreach f ($FILES)
+	if ($f:e == 'Z') then
+	  set decoder=zcat
+	else if ($f:e == 'gz') then
+	  set decoder=gunzip
+	else
+	  set decoder=cat
+	endif
+      $decoder <$f |awk '/^@/&&NR!=1{printf ""} {print $0} END{print ""}'
+      end # foreach
+    breaksw #-text
+ 
+    case '-cleanup':
+    breaksw #-cleanup
+  endsw #flag
+  breaksw #BIB
+
+  case MAIL:
+# Takes a list of files that contain mail, and splits them up 
+#   by putting ^B after each message. Assumes that each message
+#   begins with a line '^From '.
+
+  switch ($flag)
+    case '-init':
+    breaksw
+ 
+    case '-text':
+      foreach f ($FILES)
+	if ($f:e == 'Z') then
+	  set decoder=zcat
+	else if ($f:e == 'gz') then
+	  set decoder=gunzip
+	else
+	  set decoder=cat
+	endif
+      $decoder <$f| awk '/xbtoa Begin/,/xbtoa End/ {next} /^From /&&NR!=1{printf ""} {print $0} END{print ""}'
+      end # foreach
+    breaksw #-text
+ 
+    case '-cleanup':
+    breaksw #-cleanup
+  endsw #flag
+  breaksw #MAIL
+
+  case DIR:
+# Recursively concatenates every file in every subdirectory of the given
+#   directory.
+
+  switch ($flag)
+    case '-init':
+    breaksw
+
+    case '-text'
+      find $FILES -type f -name '*.gz' -exec echo '<' {} '>' \;		\
+			 -exec gzcat {} \; -exec echo -n '' \;
+      find $FILES -type f -name '*.Z' -exec echo '<' {} '>' \;		\
+			 -exec zcat {} \; -exec echo -n '' \;
+#     find $FILES -type f -not -regex '.*\.\(gz\|Z\)' -exec echo '<' {} \
+#			'>' \; -exec cat {} \; -exec echo -n '' \;
+      find $FILES -type f \! \( -name '*.gz' -o -name '*.Z' \) \
+			-exec echo '<' {} '>' \; \
+			-exec cat {} \; -exec echo -n '' \;
+
+    breaksw
+
+    case '-cleanup':
+    breaksw #-cleanup
+  endsw #flag
+  breaksw #DIR
+
+
+
+  case DIR2:
+# Recursively concatenates every file in every subdirectory of the given
+#   directory. Does not include filename
+
+  switch ($flag)
+    case '-init':
+    breaksw
+
+    case '-text'
+      find $FILES -type f -name '*.gz'  		\
+			 -exec gzcat {} \; -exec echo -n '' \;
+      find $FILES -type f -name '*.Z' 	 		\
+			 -exec zcat {} \; -exec echo -n '' \;
+#     find $FILES -type f -not -regex '.*\.\(gz\|Z\)'	\
+#			-exec cat {} \; -exec echo -n '' \;
+      find $FILES -type f \! \( -name '*.gz' -o -name '*.Z' \) \
+			-exec cat {} \; -exec echo -n '' \;
+    breaksw
+
+    case '-cleanup':
+    breaksw #-cleanup
+  endsw #flag
+  breaksw #DIR2
+
+
+
+
+  case TXTIMG:
+# compress and index the collection of text and images
+# (this code is suitable for all sorts of integrated collections
+#  of text and images)
+# Files that are related have the same prefix. For example,
+#  monaLisa.pgm might be a gray-level image, and
+#  monaLisa.txt would be a textual file describing the image.
+# The suffixes recognised are:
+#    .txt	ascii text
+#    .ptm	scanned text stored as a bilevel image
+#    .pbm	a black and white image (typically a line drawing)
+#    .pgm	a gray-scale image
+# In addition, if no corresponding ascii text file is found for
+#  a .pbm or .pgm file, then one is created with suffix .tmp.txt,
+#  and it stores the name of the image file (in principle it could
+#  store the OCR of a .txt.pbm file). At present the .tmp.txt files
+#  are deleted by the '-cleanup' option.
+  
+#### the next two 'set' statements define which directory the files
+#    come from, and where they will be stored. This is the only
+#    part of this code that is specific to the 'davinci' collection
+  set sourceDir = $FILES
+  set targetDir = $MGDATA/$COLLECTION
+
+  switch ($flag)
+    case '-init':
+      mkdir $targetDir >&/dev/null	   # create the directory to store
+					   # compressed images in
+      $rm -rf $targetDir/* >&/dev/null     # in case it already existed
+    
+      # take care if no match to foreach statements
+      set nonomatch
+
+      # process all pbm (black and white) images of text
+      foreach f ($sourceDir/*.ptm)
+        if ($f == "$sourceDir/*.ptm") then
+                break
+        endif
+	set base = $f:t
+	
+        $bindir/mgticbuild $targetDir/$base:r.ticlib.$$ $f
+        $bindir/mgticprune $targetDir/$base:r.ticlib.$$
+
+	$bindir/mgtic -L -e $targetDir/$base:r.ticlib.$$ $f >$targetDir/$base:r.tic
+	$rm $targetDir/$base:r.ticlib.$$
+	set r = $f:r	#root name of file
+	if (! (-e $r.txt || -e $r.tmp)) then
+	# creates a file; could do OCR here to get its contents
+	    echo "No corresponding txt file for" $f "- creating" $r.tmp
+	    echo '#######' $f > $r.tmp
+	endif
+      end
+      # process all pgm (gray scale) images
+      foreach f ($sourceDir/*.pgm)
+        if ($f == "$sourceDir/*.pgm") then
+                break
+        endif
+	set base = $f:t
+	$bindir/mgfelics -e $f >$targetDir/$base:r.flx
+	set r = $f:r	#root name of file
+	if (! (-e $r.txt || -e $r.tmp)) then
+	# creates a file; could do image recognition here to create one(!)
+	    echo "No corresponding txt file for" $f "- creating" $r.tmp
+	    echo '#######' $f > $r.tmp
+	endif
+      end
+      # process all pbm (black and white) images
+      foreach f ($sourceDir/*.pbm)
+        if ($f == "$sourceDir/*.pbm") then
+                break
+        endif
+	set base = $f:t
+	$bindir/mgbilevel -e $f >$targetDir/$base:r.blv
+	set r = $f:r	#root name of file
+	if (! (-e $r.txt || -e $r.tmp)) then
+	# creates a file; could do drawing recognition here to create one(!)
+	    echo "No corresponding txt file for" $f "- creating" $r.tmp
+	    echo '####### No corresponding text file available for this image' >$r.tmp
+	    echo 'Original image file name was' $f >> $r.tmp
+	endif
+      end
+      unset nonomatch
+    breaksw #-init
+
+    case '-text'
+      # take care if no match to foreach statements
+      set nonomatch
+
+      #output each text file and tmp.txt, with names of other associated files
+
+      foreach f ($sourceDir/*.txt $sourceDir/*.tmp)
+        if ($f == "$sourceDir/*.txt") then
+           continue
+        endif
+        if ($f == "$sourceDir/*.tmp") then
+           continue
+        endif
+	set r = $f:r	#root name of file
+	set base = $r:t
+        set first_image = 1 # set so that I print out image header once
+                            # also signifies if header printed out at all
+
+	foreach d ($targetDir/$base.*)
+          if ($d == "$targetDir/$base.*") then
+                break
+          endif
+
+          if ($first_image) then
+            set first_image = 0
+            echo '::::::::::'  #separate the image information from the text
+            echo 'Image(s) available:'
+          endif
+
+	  switch ($d:e)	# work out decoding method from suffix
+            case 'tic':
+              echo 'MGDATA (mgtic -d ' $d '  | '$MGIMAGEVIEWER') '
+            breaksw
+            case 'flx':
+              echo 'MGDATA (mgfelics -d ' $d '  | '$MGIMAGEVIEWER')'
+            breaksw
+            case 'blv':
+              echo 'MGDATA (mgbilevel -d ' $d '  | '$MGIMAGEVIEWER')'
+	    breaksw
+	  endsw # $d:e (suffix)
+
+	end #foreach d (associated file)
+
+        if (! $first_image) then
+	  echo '::::::::::'  #separate the image information from the text
+        endif
+
+	cat $f  # output the text associated with the images
+	echo -n ''
+      end #foreach f (text file)
+      unset nonomatch
+    breaksw #-text
+
+    case '-cleanup':
+      #remove temporary text files
+      $rm $sourceDir/*.tmp
+    breaksw #-cleanup
+  endsw #flag
+  breaksw #IMGTXT
+
+
+  default:
+    echo 'Sorry, I do not know how to get' $1
+    exit 1	
+endsw
+exit 0
Index: /tags/gsdl-21/gsdl/packages/mg/src/scripts/mg_get_merge.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/scripts/mg_get_merge.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/scripts/mg_get_merge.1	(revision 699)
@@ -0,0 +1,86 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_get_merge 1 \*(Dt CITRI
+.SH NAME
+mg_get_merge \- output source texts for processing
+.SH SYNOPSIS
+.B mg_get_merge
+.I collection-name
+.if n .ti +9n
+[
+.BR \-init " |"
+.BR \-i " |"
+.BR \-text " |"
+.BR \-t " |"
+.BR \-cleanup " |"
+.B  \-c
+]
+.SH DESCRIPTION
+This program is the default one used by
+.BR mgmerge (1)
+to generate the source text for new documents to appended to
+an
+.BR mg (1)
+system database.
+Its function is exactly the same as
+.BR mg_get (1)
+except that it generates the 
+.I new 
+documents to be added to an
+.BR mg (1)
+system database that has already been built using
+.BR mgbuild(1).
+Any program may be used to generate the source
+text for
+.BR mgmerge (1),
+as long as it confirms to the interface specified
+here.
+.SH OPTIONS
+The
+.I collection-name
+must appear before any other option.  Only the first option has any
+significance.  If no option is specified,
+.B \-text
+is assumed.
+.TP "\w'\fB\-cleanup\fP and \fB\-c\fP'u+2n"
+.BR \-init " and " \-i
+The program is called once with this flag at the start of merging a
+collection.
+.TP
+.BR \-text " and " \-t
+The program is called with this flag multiple times during the
+merging of a collection.  The program outputs (on
+.IR stdout )
+the text of the collection.
+.I Documents
+within the collection are separated by ctrl-B's (ASCII code 2).
+.TP
+.BR \-cleanup " and " \-c
+The program is called once with this flag at the completion of
+merging a collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_get (1),
+.BR mgbuild (1),
+.BR mgmerge (1),
+.BR mg_text_merge (1),
+.BR mg_invf_merge (1).
Index: /tags/gsdl-21/gsdl/packages/mg/src/scripts/mg_get_merge.sh
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/scripts/mg_get_merge.sh	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/scripts/mg_get_merge.sh	(revision 699)
@@ -0,0 +1,313 @@
+#! /bin/csh -f
+###########################################################################
+#
+# mg_get_merge.sh -- script to get text for mgbuild
+# Copyright (C) 1994  Tim Bell, Shane Hudson (mods. for merging)
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#
+###########################################################################
+#
+# "mg_get_merge" formats the text for mgbuild by collecting the appropriate
+#   documents, and outputing them with control-B after each document.
+#   The name of the text must be supplied as an argument.
+#   mg_get is called with -i (init) first, then with -t (text) each time the 
+#   text is needed, then with -c (cleanup).
+#   e.g. mg_get alice -t
+#
+###########################################################################
+
+
+
+# The directory containing the data files is stored in the environment
+#   variable MGSAMPLE
+
+if ($?MGSAMPLE) then
+  set SampleData = $MGSAMPLE
+else
+  set SampleData = ./SampleData
+endif
+
+if (! $?MGIMAGEVIEWER) then
+  setenv MGIMAGEVIEWER "xv -"
+endif
+
+set rm = /bin/rm
+
+switch ($#argv)
+  case 1:
+  set flag = '-text'
+  breaksw
+
+  case 2:
+  set flag = $2
+  switch ($flag)
+     case '-i':
+	set flag = '-init'
+	breaksw
+     case '-t':
+	set flag = '-text'
+	breaksw
+     case '-c':
+	set flag = '-cleanup'
+	breaksw
+     case '-init':
+     case '-text':
+     case '-cleanup':
+     breaksw
+     default:
+     echo 'Usage: get <document> [-i | -t | -c]'
+     exit(1)
+     endsw
+  breaksw
+
+  default:
+  echo 'Usage: get <document> [-i | -t | -c]'
+  exit(1)
+  endsw
+
+set bindir = $0
+set bindir = $bindir:h
+
+switch ($1)
+  case alice:
+  switch ($flag)
+     case '-init':
+     breaksw
+
+     case '-text':
+	echo "This is a test document 1 2 3 4 5 6 7 8 9 "
+	echo ""
+	echo "A B C"
+     breaksw
+
+     case '-cleanup':
+     breaksw
+  endsw #flag
+  breaksw
+
+  case bible:
+  switch ($flag)
+    case '-init':
+    breaksw
+
+    case '-text':
+       # use uncompressed file if it exists
+       # revelation.txt already contains '^B' characters between verses
+       if (-e $SampleData/bible/revelation.txt) then
+          cat $SampleData/bible/revelation.txt
+       else
+          uncompress <$SampleData/bible/revelation.txt.Z
+    breaksw #-text
+
+    case '-cleanup':
+    breaksw #-cleanup
+
+  endsw #flag
+  breaksw #bible
+
+
+
+  case mailfiles:
+# Takes a list of files that contain mail, and splits them up 
+#   by putting ^B after each message. Assumes that each message
+#   begins with a line '^From '.
+# If messages are already in individual files, use "get allfiles"
+#   rather than this one.
+# The EXTRA documents added are in files in the directory mailextra
+#### The variable 'mailfiles' contains a list of files to be split up
+  set mailfiles = '~/mbox ~/.sentmail'
+  set mailextra = '~/extra/*'
+
+  switch ($flag)
+    case '-init':
+    breaksw
+
+    case '-text':
+      foreach f ($mailextra)
+	awk '/^From /&&NR!=1{print ""} {print $0} END{print ""}' $f
+      end # foreach
+    breaksw #-text
+
+    case '-cleanup':
+    breaksw #-cleanup
+
+  endsw #flag
+  breaksw #mailfiles
+
+
+
+  case allfiles:
+# Recursively concatenates every file in every subdirectory of the given
+#   directory.
+
+# The variable 'directory' on the next line contains the directory to be used
+  set directory = ~/Mail
+
+# The date of the file ".mgbuildtime" in 'directory' is the last
+# build or merge. At the start of the merge ("-init" argument) the
+# file ".mgmergetime" is touched. Only files newer than ".mgbuildtime"
+# but older than ".mgmergetime" are output by mg_get_merge
+
+  switch ($flag)
+    case '-init':
+        touch $directory/.mgmergetime
+    breaksw
+
+    case '-text'
+      find $directory -type f -newer $directory/.mgbuildtime \! -newer $directory/.mgmergetime \! -name '.*' -exec echo '<' {} '>' \; -exec cat {} \; -exec echo -n '' \;
+    breaksw
+
+    case '-cleanup':
+        touch $directory/.mgmergetime
+        touch $directory/.mgbuildtime
+    breaksw #-cleanup
+  endsw #flag
+  breaksw #allfiles
+
+
+
+
+  case davinci:
+# compress and index the DaVinci collection of text and images
+# (this code is suitable for all sorts of integrated collections
+#  of text and images)
+# It assumes that all files are in one directory ($sourceDir).
+# Files that are related have the same prefix. For example,
+#  monaLisa.img.pgm might be a gray-level image, and
+#  monaLisa.txt.txt would be a textual file describing the image.
+# The suffixes recognised are:
+#    .txt.txt	ascii text
+#    .txt.pbm	scanned text stored as a bilevel image
+#    .img.pbm	a black and white image (typically a line drawing)
+#    .img.pgm	a gray-scale image
+# In addition, if no corresponding ascii text file is found for
+#  a .pbm or .pgm file, then one is created with suffix .tmp.txt,
+#  and it stores the name of the image file (in principle it could
+#  store the OCR of a .txt.pbm file). At present the .tmp.txt files
+#  are deleted by the '-cleanup' option.
+  
+#### the next two 'set' statements define which directory the files
+#    come from, and where they will be stored. This is the only
+#    part of this code that is specific to the 'davinci' collection
+  set sourceDir = $SampleData/davinci
+  set targetDir = $MGDATA/davinci
+
+  switch ($flag)
+    case '-init':
+      mkdir $targetDir >&/dev/null	   #create the directory to store
+					   # compressed images in
+      $rm -rf $targetDir/* >&/dev/null #in case it already existed
+    
+      # process all pbm (black and white) images of text
+      foreach f ($sourceDir/*.ptm)
+	set base = $f:t
+	
+        $bindir/mgticbuild $targetDir/$base:r.ticlib.$$ $f
+        $bindir/mgticprune $targetDir/$base:r.ticlib.$$
+
+	$bindir/mgtic -L -e $targetDir/$base:r.ticlib.$$ $f >$targetDir/$base:r.tic
+	$rm $targetDir/$base:r.ticlib.$$
+	set r = $f:r	#root name of file
+	if (! (-e $r.txt || -e $r.tmp)) then
+	# creates a file; could do OCR here to get its contents
+	    echo "No corresponding txt file for" $f "- creating" $r.tmp
+	    echo '#######' $f > $r.tmp
+	endif
+      end
+      # process all pgm (gray scale) images
+      foreach f ($sourceDir/*.pgm)
+	set base = $f:t
+	$bindir/mgfelics -e $f >$targetDir/$base:r.flx
+	set r = $f:r	#root name of file
+	if (! (-e $r.txt || -e $r.tmp)) then
+	# creates a file; could do image recognition here to create one(!)
+	    echo "No corresponding txt file for" $f "- creating" $r.tmp
+	    echo '#######' $f > $r.tmp
+	endif
+      end
+      # process all pbm (black and white) images
+      foreach f ($sourceDir/*.pbm)
+	set base = $f:t
+	$bindir/mgbilevel -e $f >$targetDir/$base:r.blv
+	set r = $f:r	#root name of file
+	if (! (-e $r.txt || -e $r.tmp)) then
+	# creates a file; could do drawing recognition here to create one(!)
+	    echo "No corresponding txt file for" $f "- creating" $r.tmp
+	    echo '####### No corresponding text file available for this image' >$r.tmp
+	    echo 'Original image file name was' $f >> $r.tmp
+	endif
+      end
+    breaksw #-init
+
+    case '-text'
+      #output each text file and tmp.txt, with names of other associated files
+      foreach f ($sourceDir/*.txt $sourceDir/*.tmp)
+	set r = $f:r	#root name of file
+	set base = $r:t
+	set n = `/bin/ls $sourceDir/$base.* | wc -l`
+	if ($n != 1) then # more files than just the text one
+	 echo '::::::::::'  #separate the image information from the text
+	 echo 'Image(s) available:'
+	 foreach d ($targetDir/$base.*)
+	  switch ($d:e)	# work out decoding method from suffix
+            case 'tic':
+              echo 'MGDATA (mgtic -d ' $d '  | '$MGIMAGEVIEWER') '
+            breaksw
+            case 'flx':
+              echo 'MGDATA (mgfelics -d ' $d '  | '$MGIMAGEVIEWER')'
+            breaksw
+            case 'blv':
+              echo 'MGDATA (mgbilevel -d ' $d '  | '$MGIMAGEVIEWER')'
+	    breaksw
+	  endsw # $d:e (suffix)
+	 end #foreach d (associated file)
+	echo '::::::::::'  #separate the image information from the text
+	endif # if more than one file
+	cat $f  # output the text associated with the images
+	echo -n ''
+      end #foreach f (text file)
+    breaksw #-text
+
+    case '-cleanup':
+      #remove temporary text files
+      $rm $sourceDir/*.tmp
+    breaksw #-cleanup
+  endsw #flag
+  breaksw #davinci
+
+
+#---------------------------
+  case mods:
+# Splits the modifications file at the mod separator "--------"
+  switch ($flag)
+     case '-init':
+     breaksw
+ 
+     case '-text':
+       sed -e 's/^---.*//g' < ${SampleData}/MODIFICATIONS
+     breaksw
+ 
+     case '-cleanup':
+     breaksw
+  endsw #flag
+  breaksw #mods
+
+  default:
+    echo 'Sorry, I do not know how to get' $1
+    exit 1	
+endsw
+exit 0
Index: /tags/gsdl-21/gsdl/packages/mg/src/scripts/mgbuild.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/scripts/mgbuild.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/scripts/mgbuild.1	(revision 699)
@@ -0,0 +1,169 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgbuild 1 \*(Dt CITRI
+.SH NAME
+mgbuild \- build an mg system database
+.SH SYNOPSIS
+.B mgbuild
+[
+.B \-c
+]
+[
+.BI \-g " get"
+]
+[
+.BI \-s " source"
+]
+[
+.BI \-d " mgdata dir"
+]
+.I collection-name
+.SH DESCRIPTION
+.B mgbuild
+is a
+.B csh
+script that executes all the appropriate programs in the correct order
+to completely build an
+.BR mg (1)
+system database ready for queries to be made
+by
+.BR mgquery.
+This program makes use of the
+.BR mg_get (1)
+script to obtain the text of the collection.
+.SH OPTIONS
+Options can occur in any order, but the collection name must be last.
+.TP "\w'\fIcollection-name\fP'u+2n"
+.BI \-c
+This specifies whether the
+.I get
+program is \*(lqcomplex\*(rq.  If a
+.I get
+program is \*(lqcomplex\*(rq, then it requires initialisation and
+cleanup with the
+.B \-i
+and
+.B \-c
+options.
+.TP
+.BI \-g " get"
+This specifies the program to use for getting the source text for the
+build.  If no
+.B \-g
+option is given, the default program
+.BR mg_get (1)
+is used.
+.TP
+.BI \-s " source"
+The
+.B mgbuild
+program consists of two parts.  The first part initializes variables
+to default values.  The second part uses these variables to control
+how the
+.BR mg (1)
+database is built.  This option specifies a program to execute between
+the first and second parts.  The details of what the variables are, and
+how they may be changed, are in comments in the
+.B mgbuild
+program.
+.TP
+.BI \-d " mgdata dir"
+Setting this option allows one to override the MGDATA environment variable.
+.TP
+.I collection-name
+This is the collection name, as required by the
+.BR mg_get (1)
+program.  It serves both as a
+.I case
+statement selector, and as the name of a subdirectory that holds the
+indexing files.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.chunk
+Inverted file chunk descriptor file.  When the inverted file is
+created, it is written in chunks that use no more than a set amount of
+memory.  This file describes those chunks.
+.TP
+.B *.invf.chunk.trans
+Word-occurrence-order to lexical-order translation file.  The
+.B *.invf.chunk
+file is written in word-occurrence order but is required by
+.B \-N2
+to be in lexical order.
+.TP
+.B *.invf.dict
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict.blocked
+The `on-disk' stemmed dictionary.
+.TP
+.B *.invf.dict.hash
+Data for an order-preserving perfect hash function.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.B *.text
+Compressed documents.
+.TP
+.B *.text.stats
+Text statistics.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.text.idx.wgt
+Interleaved index into the compressed documents and document weights.
+.TP
+.B *.weight.approx
+Approximate document weights.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /tags/gsdl-21/gsdl/packages/mg/src/scripts/mgbuild.sh
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/scripts/mgbuild.sh	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/scripts/mgbuild.sh	(revision 699)
@@ -0,0 +1,339 @@
+#!/bin/csh -f
+###########################################################################
+#
+# mgbuild.sh -- Script used to build mg text collection.
+# Copyright (C) 1994  Neil Sharman
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)mgbuild.sh	1.9 16 Mar 1994
+#
+###########################################################################
+#  mg.build
+# Creates a Full Text Retrieval database
+#
+#  @(#)mgbuild.sh	1.9 16 Mar 1994
+#
+
+set complex = ""
+
+# Parse the command line arguments
+while ($#argv >= 1)
+  if ("$1" == "-s") then
+    shift
+    if ($#argv >= 1) then
+      set source = $1
+      shift
+    endif
+  else if ("$1" == "-d") then
+    shift
+    if ($#argv >= 1) then
+      set mgdata = $1
+      shift
+    endif
+  else if ("$1" == "-g") then
+    shift
+    if ($#argv >= 1) then
+      set get = $1
+      shift
+    endif
+  else if ("$1" == "-c") then
+    set complex = "-text"
+    shift
+  else
+    if ($?text == "0") then
+      set text = $1
+    endif
+    shift
+  endif
+    
+end
+
+if ($?text == "0") then
+  set prog = $0
+  echo "USAGE:"
+  echo "     "$prog:t" [-s config-script] [-g get-program] [-d mgdata dir] [-c] source"
+  echo ""
+  echo " The config-script is only needed if a non-standard build is required."
+  echo " The get-program defaults to mg_get if not specified."
+  exit 1	
+endif
+
+set bindir = $0
+set bindir = $bindir:h
+
+# if $pipe == 1 then pipe in the source text using $get and $text otherwise
+# read the source text directly from the file names specified in $input_files
+set pipe = 1
+
+if ($?get == "0") then
+  set complex = "-text"
+  set get = $bindir/mg_get
+endif
+
+if ($?mgdata) then
+  setenv MGDATA $mgdata
+endif
+
+if (-e $MGDATA/${text}.chunks) then
+  set input_files = `cat $MGDATA/${text}.chunks`
+endif
+
+# Set the stemming method
+#   Bit 0 = case folding
+#   Bit 1 = S stemmer
+set stem_method = 3
+
+
+# [RPAP - Jan 97: Stem Index Change]
+# If do_indexes == 1 then build collection with full indexes to blocked file
+# overriding stem_method (stem_method will be set to 0)
+# Otherwise don't build with indexes
+set do_indexes = 0
+
+
+# If do_text == 1 then do the text component of building a mg database. 
+# Otherwise don't do the text component.
+set do_text = 1
+
+
+# If do_invf == 1 then do the inversion component of building a mg database. 
+# Otherwise don't do the inversion component.
+set do_invf = 1
+
+
+# If do_pass1 == 1 then do pass1 of building a mg database. 
+# Otherwise don't do pass1.
+set do_pass1 = 1
+
+# If do_pass2 == 1 then do pass2 of building a mg database. 
+# Otherwise don't do pass2.
+set do_pass2 = 1
+
+
+# $invf_mem specifies the amount of memory to use for the pass2 inversion
+set invf_mem = 32
+
+
+# $num_chunks specifies the number of interium chunks of inverted file that
+# may be written to disc before a merge into the invf file is done
+set num_chunks = 3
+
+# $invf_level specifies the level of the inverted file that will be generated.
+set invf_level = 2
+
+
+# If $strip_sgml == 1 then sgml tags are stripped from the inversion phase.
+# Otherwise sgml tags are kepted.
+set strip_sgml = 1
+
+
+# $trace specifies the interval between trace entries in Mb.
+# If this is not set no trace entries will be generated.
+set trace = 10
+
+
+# $weight_bits specifies the number of bits of precision bo be given to the 
+# approximate weights.
+set weight_bits = 6
+
+
+# $mcd specifies the commandline arguments for the mg_compression_dict program
+#### Note: Set -S so novel words can be encoded, as this will happen if
+#### mgmerge is used.
+set mcd = -S
+
+
+# Source the parameter file to modify parameters of the build.
+if ($?source) then
+  source ${source}
+endif
+
+
+# [RPAP - Jan 97: Stem Index Change]
+# If do_indexes == 1 then set stem_method = 0
+if ("$do_indexes" == "1") then
+  set stem_method = 0
+endif
+
+
+# Generate the collection name.
+set coll_name = ${text}
+if ("$invf_level" == "3") then
+  set coll_name = ${text}-p
+endif
+
+# Generate the directory for the collection.
+if (-e $MGDATA/${coll_name}) then
+else
+  mkdir $MGDATA/${coll_name}
+endif
+
+# Generate the base name for the collection.
+set bname = ${coll_name}/${coll_name}
+
+
+#build up the command lines for pass 1 and 2
+set pass1 = (-f ${bname} -${invf_level} -m ${invf_mem} -s ${stem_method})
+set pass2 = (-f ${bname} -${invf_level} -c ${num_chunks})
+
+if ($strip_sgml) then
+  set pass1 = (${pass1} -G)
+  set pass2 = (${pass2} -G)
+endif
+
+if ($?trace) then
+  set pass1 = (${pass1} -t ${trace})
+  set pass2 = (${pass2} -t ${trace})
+endif
+
+if ($do_text) then
+  set pass1 = (${pass1} -T1)
+  set pass2 = (${pass2} -T2)
+endif
+
+if ($do_invf) then
+  set pass1 = (${pass1} -I1)
+  set pass2 = (${pass2} -I2)
+endif
+  
+if ($?trace_name) then
+  set pass1 = (${pass1} -n ${trace_name})
+  set pass2 = (${pass2} -n ${trace_name})
+endif
+  
+if ($?comp_stats) then
+  set pass2 = (${pass2} -C ${comp_stats})
+endif
+
+echo "-----------------------------------"
+echo "`uname -n`, `date`"
+echo "${text} --> ${bname}"
+echo "-----------------------------------"
+if ($pipe) then
+  if ("$complex" != "") then
+    echo "$get $text -init"
+    $get $text -init
+    if ("$status" != "0") exit 1 
+    echo "-----------------------------------"
+  endif  
+endif
+
+if (${do_pass1}) then
+  if ($pipe) then
+    if ($?pass1filter) then
+      echo "$get $text $complex | $pass1filter | mg_passes ${pass1}"
+      $get $text $complex| $pass1filter | $bindir/mg_passes ${pass1}
+      if ("$status" != "0") exit 1 
+    else
+      echo "$get $text $complex | mg_passes ${pass1}"
+      $get $text $complex| $bindir/mg_passes ${pass1}
+      if ("$status" != "0") exit 1 
+    endif
+  else
+    echo mg_passes ${pass1} ${input_files}
+    $bindir/mg_passes ${pass1} ${input_files}
+    if ("$status" != "0") exit 1 
+  endif
+  echo "-----------------------------------"
+
+  echo "mg_perf_hash_build -f ${bname}"
+  $bindir/mg_perf_hash_build -f ${bname}
+  if ("$status" != "0") exit 1 
+
+  echo "-----------------------------------"
+  echo "mg_compression_dict -f ${bname} ${mcd}"
+  $bindir/mg_compression_dict -f ${bname} ${mcd}
+  if ("$status" != "0") exit 1 
+  echo "-----------------------------------"
+endif
+
+if (${do_pass2}) then
+  if ($pipe) then
+    if ($?pass2filter) then
+      echo "$get $text $complex | $pass2filter | mg_passes ${pass2}"
+      $get $text $complex | $pass2filter | $bindir/mg_passes ${pass2}
+      if ("$status" != "0") exit 1 
+    else
+      echo "$get $text $complex | mg_passes ${pass2}"
+      $get $text $complex | $bindir/mg_passes ${pass2}
+      if ("$status" != "0") exit 1 
+    endif
+  else
+    echo mg_passes ${pass2} ${input_files}
+    $bindir/mg_passes ${pass2} ${input_files}
+    if ("$status" != "0") exit 1 
+  endif
+  echo "-----------------------------------"
+endif
+
+echo "mg_weights_build -f ${bname} -b ${weight_bits}"
+$bindir/mg_weights_build -f ${bname} -b ${weight_bits}
+if ("$status" != "0") exit 1 
+
+echo "-----------------------------------"
+
+echo "mg_invf_dict -f ${bname} -b 4096"
+$bindir/mg_invf_dict -f ${bname} -b 4096
+if ("$status" != "0") exit 1 
+
+echo "-----------------------------------"
+
+# [RPAP - Jan 97: Stem Index Change]
+if ("$do_indexes" == "1") then
+  echo "mg_stem_idx -f ${bname} -b 4096 -s1"
+  $bindir/mg_stem_idx -f ${bname} -b 4096 -s1
+  if ("$status" != "0") exit 1 
+
+  echo ""
+
+  echo "mg_stem_idx -f ${bname} -b 4096 -s2"
+  $bindir/mg_stem_idx -f ${bname} -b 4096 -s2
+  if ("$status" != "0") exit 1 
+
+  echo ""
+
+  echo "mg_stem_idx -f ${bname} -b 4096 -s3"
+  $bindir/mg_stem_idx -f ${bname} -b 4096 -s3
+  if ("$status" != "0") exit 1 
+
+  echo "-----------------------------------"
+endif
+
+echo "mgstat -f ${bname} -E"
+$bindir/mgstat -f ${bname} -E
+if ("$status" != "0") exit 1 
+
+if ($pipe) then
+  if ("$complex" != "") then
+    echo "-----------------------------------"
+    echo "$get $text -cleanup"
+    $get $text -cleanup
+    if ("$status" != "0") exit 1 
+  endif
+endif
+
+echo "-----------------------------------"
+echo "`uname -n`, `date`"
+echo "-----------------------------------"
+
+echo "-- The fast-loading compression dictionary has not been built."
+echo "-- If you wish to build it, execute the following command:"
+echo "-- "$bindir/mg_fast_comp_dict -f ${bname}
+
+echo "-----------------------------------"
+
+echo ""
+
Index: /tags/gsdl-21/gsdl/packages/mg/src/scripts/mgbuild_4.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/scripts/mgbuild_4.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/scripts/mgbuild_4.1	(revision 699)
@@ -0,0 +1,53 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgbuild_4 1 \*(Dt CITRI
+.SH NAME
+mgbuild_4 \- build an mg system database using 4 passes instead of 2
+.SH SYNOPSIS
+.B mgbuild_4
+[
+.B \-c
+]
+[
+.BI \-g " get"
+]
+[
+.BI \-s " source"
+]
+[
+.BI \-d " mgdata dir"
+]
+.I collection-name
+.SH DESCRIPTION
+.B mgbuild_4
+is a
+.B csh
+script that executes all the appropriate programs in the correct order
+to completely build an
+.BR mg (1)
+system database ready for queries to be made
+by
+.BR mgquery.
+This program makes use of the
+.BR mg_get (1)
+script to obtain the text of the collection.
+In 
+.BR mgbuild (1)
+the build is done by two passes over the input text. The first
+pass creates the compressed text dictionary and inverted file dictionary,
+the second pass creates the compressed text and inverted file. What 
+.BR mgbuild_4
+does is use a separate pass for each function. This will be slower but will
+not require as much RAM at one time. Therefore, if your build is failing
+due to running short of memory, then 
+.BR mgbuild_4
+is worth a try.
+.SH "SEE ALSO"
+.na
+.BR mgbuild (1)
Index: /tags/gsdl-21/gsdl/packages/mg/src/scripts/mgbuild_4.sh
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/scripts/mgbuild_4.sh	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/scripts/mgbuild_4.sh	(revision 699)
@@ -0,0 +1,435 @@
+#!/bin/csh -f
+###########################################################################
+#
+# mgbuild_4.sh -- Script used to build mg text collection in 4 passes.
+# Copyright (C) 1994  Neil Sharman; Modified by tes@kbs 1995
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#
+###########################################################################
+#
+# Does an mgbuild in 4 passes instead of 2 passes.
+# This is useful if one does not have as much RAM and a large collection.
+#
+
+set complex = ""
+
+# ************ parse the command line arguments ************
+
+while ($#argv >= 1)
+  if ("$1" == "-s" || "$1" == "--source") then
+    shift
+    if ($#argv >= 1) then
+      set source = $1
+      shift
+    endif
+  else if ("$1" == "-g" || "$1" == "--get") then
+    shift
+    if ($#argv >= 1) then
+      set get = $1
+      shift
+    endif
+  else if ("$1" == "-c" || "$1" == "--complex") then
+    set complex = "-text"
+    shift
+  else
+    if (! $?text ) then
+      set text = $1
+    endif
+    shift
+  endif
+    
+end
+
+if (! $?text ) then
+  set prog = $0
+  echo "USAGE:"
+  echo "     "$prog:t" [-s config-script] [-g get-program] [-c] source"
+  echo ""
+  echo " The config-script is only needed if a non-standard build is required."
+  echo " The get-program defaults to mg_get if not specified."
+  exit 1	
+endif
+
+set bindir = `which $0` # find which one we are executing
+set bindir = $bindir:h # use it as the bindir path
+
+# ************ set up shell variables ************
+# Override these by providing a script to source using the -s option
+
+# if $pipe == 1 then pipe in the source text using $get and $text otherwise
+# read the source text directly from the file names specified in $input_files
+set pipe = 1
+
+set get_options = ""
+
+
+if (! $?get ) then
+  set complex = "-text"
+  set get = $bindir/mg_get $get_options
+endif
+
+
+if (-e $MGDATA/${text}.chunks) then
+  set input_files = `cat $MGDATA/${text}.chunks`
+endif
+
+# Set the stemming method
+#   Bit 0 = case folding
+#   Bit 1 = S stemmer
+set stem_method = 3
+
+
+# [RPAP - Jan 97: Stem Index Change]
+# If do_indexes == 1 then build collection with full indexes to blocked file
+# overriding stem_method (stem_method will be set to 0)
+# Otherwise don't build with indexes
+set do_indexes = 0
+
+
+# whether to build weights or not
+set do_weights = 1
+
+# If do_pass1 == 1 then do pass1 of building a mg database. 
+# Otherwise don't do pass1.
+# Text 1st pass
+set do_pass1 = 1
+
+# If do_pass2 == 1 then do pass2 of building a mg database. 
+# Otherwise don't do pass2.
+# Text 2nd pass
+set do_pass2 = 1
+
+# If do_pass3 == 1 then do pass3 of building a mg database. 
+# Otherwise don't do pass3.
+# Invf 1st pass
+set do_pass3 = 1
+
+# If do_pass4 == 1 then do pass4 of building a mg database. 
+# Otherwise don't do pass4.
+# Invf 2nd pass
+set do_pass4 = 1
+
+# Buffer size used by mg_passes
+# Should be big enough to hold largest document
+# In Kilobytes
+set buf_size = 3072
+
+# $invf_mem specifies the amount of memory to use for the pass2 inversion
+set invf_mem = 5
+
+# $num_chunks specifies the number of interium chunks of inverted file that
+# may be written to disc before a merge into the invf file is done
+set num_chunks = 3
+
+# $invf_level specifies the level of the inverted file that will be generated.
+set invf_level = 2
+
+
+# If $strip_sgml == 1 then sgml tags are stripped from the inversion phase.
+# Otherwise sgml tags are kepted.
+set strip_sgml = 1
+
+
+# $trace specifies the interval between trace entries in Mb.
+# If this is not set no trace entries will be generated.
+set trace = 10
+
+
+# $weight_bits specifies the number of bits of precision bo be given to the 
+# approximate weights.
+set weight_bits = 6
+
+
+# $mcd specifies the commandline arguments for the mg_compression_dict program
+set mcd = -C
+
+
+# Source the parameter file to modify parameters of the build.
+if ($?source) then
+  source ${source}
+endif
+
+
+# [RPAP - Jan 97: Stem Index Change]
+# If do_indexes == 1 then set stem_method = 0
+if ("$do_indexes" == "1") then
+  set stem_method = 0
+endif
+
+
+# Generate the collection name.
+set coll_name = ${text}
+if ("$invf_level" == "3") then
+  set coll_name = ${text}-p
+endif
+
+# Generate the directory for the collection.
+if (-e $MGDATA/${coll_name}) then
+else
+  mkdir $MGDATA/${coll_name}
+endif
+
+# Generate the base name for the collection.
+set bname = ${coll_name}/${coll_name}
+
+# ************ set up pass parameters ************
+
+if ($do_pass1) then
+  set pass1_text = (-f ${bname} -T1)
+endif
+
+if ($do_pass2) then
+  set pass2_text = (-f ${bname} -T2)
+endif
+
+if ($do_pass3) then
+  set pass3_invf = (-f ${bname} -${invf_level} -m ${invf_mem} -s ${stem_method})
+  set pass3_invf = (${pass3_invf} -N1)
+endif
+
+if ($do_pass4) then
+  set pass4_invf = (-f ${bname} -${invf_level} -N2)
+endif
+
+if ($strip_sgml) then
+  set pass1_text = (${pass1_text} -G)
+  set pass2_text = (${pass2_text} -G)
+  set pass3_invf = (${pass3_invf} -G)
+  set pass4_invf = (${pass4_invf} -G)
+endif
+
+if ($?trace) then
+  set pass1_text = (${pass1_text} -t ${trace})
+  set pass2_text = (${pass2_text} -t ${trace})
+  set pass3_invf = (${pass3_invf} -t ${trace})
+  set pass4_invf = (${pass4_invf} -t ${trace})
+endif
+
+if ($?trace_name) then
+  set pass1_text = (${pass1_text} -n ${trace_name})
+  set pass2_text = (${pass2_text} -n ${trace_name})
+  set pass3_invf = (${pass3_invf} -n ${trace_name})
+  set pass4_invf = (${pass4_invf} -n ${trace_name})
+endif
+
+if ($buf_size) then
+  set pass1_text = (${pass1_text} -b ${buf_size})
+  set pass2_text = (${pass2_text} -b ${buf_size})
+  set pass3_invf = (${pass3_invf} -b ${buf_size})
+  set pass4_invf = (${pass4_invf} -b ${buf_size})
+endif
+  
+if ($?comp_stats) then
+  set pass2_text = (${pass2_text} -C ${comp_stats})
+endif
+
+echo "-----------------------------------"
+echo "`uname -n`, `date`"
+echo "${text} --> ${bname}"
+echo "-----------------------------------"
+
+# ************ init get ************
+
+if ($pipe) then
+  if ("$complex" != "") then
+    echo "$get $text -init"
+    $get $text -init
+    if ("$status" != "0") exit 1 
+    echo "-----------------------------------"
+  endif  
+endif
+
+# ************************************* T E X T **********************************
+
+# ************ pass 1_text ************
+
+if (${do_pass1}) then
+  if ($pipe) then
+    if ($?pass1filter) then
+      echo "$get $text $complex | $pass1filter | mg_passes ${pass1_text}"
+      $get $text $complex| $pass1filter | $bindir/mg_passes ${pass1_text}
+      if ("$status" != "0") exit 1 
+    else
+      echo "$get $text $complex | mg_passes ${pass1_text}"
+      $get $text $complex| $bindir/mg_passes ${pass1_text}
+      if ("$status" != "0") exit 1 
+    endif
+  else
+    echo mg_passes ${pass1_text} ${input_files}
+    $bindir/mg_passes ${pass1_text} ${input_files}
+    if ("$status" != "0") exit 1 
+  endif
+  echo "-----------------------------------"
+endif
+
+# ************ compression dict  ************
+if ($do_pass2) then 
+  echo "mg_compression_dict -f ${bname} ${mcd}"
+  $bindir/mg_compression_dict -f ${bname} ${mcd}
+  if ("$status" != "0") exit 1 
+  echo "-----------------------------------"
+endif
+
+# ************ pass 2_text ************
+if (${do_pass2}) then
+  if ($pipe) then
+    if ($?pass2filter) then
+      echo "$get $text $complex | $pass2filter | mg_passes ${pass2_text}"
+      $get $text $complex| $pass2filter | $bindir/mg_passes ${pass2_text}
+      if ("$status" != "0") exit 1 
+    else
+      echo "$get $text $complex | mg_passes ${pass2_text}"
+      $get $text $complex| $bindir/mg_passes ${pass2_text}
+      if ("$status" != "0") exit 1 
+    endif
+  else
+    echo mg_passes ${pass2_text} ${input_files}
+    $bindir/mg_passes ${pass2_text} ${input_files}
+    if ("$status" != "0") exit 1 
+  endif
+  echo "-----------------------------------"
+endif
+
+# ************************************* I N V F **********************************
+
+# ************ pass 3_invf ************
+if (${do_pass3}) then
+  if ($pipe) then
+    if ($?pass3filter) then
+      echo "$get $text $complex | $pass3filter | mg_passes ${pass3_invf}"
+      $get $text $complex| $pass3filter | $bindir/mg_passes ${pass3_invf}
+      if ("$status" != "0") exit 1 
+    else
+      echo "$get $text $complex | mg_passes ${pass3_invf}"
+      $get $text $complex| $bindir/mg_passes ${pass3_invf}
+      if ("$status" != "0") exit 1 
+    endif
+  else
+    echo mg_passes ${pass3_invf} ${input_files}
+    $bindir/mg_passes ${pass3_invf} ${input_files}
+    if ("$status" != "0") exit 1 
+  endif
+  echo "-----------------------------------"
+endif
+
+# ************ perfect hash  ************
+if ($do_pass4) then
+  echo "mg_perf_hash_build -f ${bname}"
+  $bindir/mg_perf_hash_build -f ${bname}
+  if ("$status" != "0") exit 1 
+
+  echo "-----------------------------------"
+endif
+
+# ************ pass 4_invf ************
+if (${do_pass4}) then
+  if ($pipe) then
+    if ($?pass4filter) then
+      echo "$get $text $complex | $pass4filter | mg_passes ${pass4_invf}"
+      $get $text $complex| $pass4filter | $bindir/mg_passes ${pass4_invf}
+      if ("$status" != "0") exit 1 
+    else
+      echo "$get $text $complex | mg_passes ${pass4_invf}"
+      $get $text $complex| $bindir/mg_passes ${pass4_invf}
+      if ("$status" != "0") exit 1 
+    endif
+  else
+    echo mg_passes ${pass4_invf} ${input_files}
+    $bindir/mg_passes ${pass4_invf} ${input_files}
+    if ("$status" != "0") exit 1 
+  endif
+  echo "-----------------------------------"
+endif
+
+
+# ************ build invf dictionary ************
+
+if ($do_pass4) then
+  echo "mg_invf_dict -f ${bname} -b 4096"
+  $bindir/mg_invf_dict -f ${bname} -b 4096
+  if ("$status" != "0") exit 1 
+  echo "-----------------------------------"
+endif
+
+# ******** build indexes to stem dict *********
+
+# [RPAP - Jan 97: Stem Index Change]
+if ($do_pass4 && "$do_indexes" == "1") then
+  echo "mg_stem_idx -f ${bname} -b 4096 -s1"
+  $bindir/mg_stem_idx -f ${bname} -b 4096 -s1
+  if ("$status" != "0") exit 1 
+
+  echo ""
+
+  echo "mg_stem_idx -f ${bname} -b 4096 -s2"
+  $bindir/mg_stem_idx -f ${bname} -b 4096 -s2
+  if ("$status" != "0") exit 1 
+
+  echo ""
+
+  echo "mg_stem_idx -f ${bname} -b 4096 -s3"
+  $bindir/mg_stem_idx -f ${bname} -b 4096 -s3
+  if ("$status" != "0") exit 1 
+
+  echo "-----------------------------------"
+endif
+
+# ************ build weights ************
+
+if ($do_pass4 && $do_weights) then
+  echo "mg_weights_build -f ${bname} -b ${weight_bits} "
+  $bindir/mg_weights_build -f ${bname} -b ${weight_bits}
+  if ("$status" != "0") exit 1 
+  echo "-----------------------------------"
+endif
+
+# *******************************************************************************
+
+
+# ************ print out statistics ************
+if ($do_pass1 && $do_pass2 && $do_pass3 && $do_pass4) then
+  echo "mgstat -f ${bname} -E"
+  $bindir/mgstat -f ${bname} -E
+  if ("$status" != "0") exit 1 
+endif
+
+
+# ************ cleanup get ************
+
+if ($pipe) then
+  if ("$complex" != "") then
+    echo "-----------------------------------"
+    echo "$get $text -cleanup"
+    $get $text -cleanup
+    if ("$status" != "0") exit 1 
+  endif
+endif
+
+echo "-----------------------------------"
+echo "`uname -n`, `date`"
+echo "-----------------------------------"
+
+if ($do_pass2) then
+  echo "-- The fast-loading compression dictionary has not been built."
+  echo "-- If you wish to build it, execute the following command:"
+  echo "-- "$bindir/mg_fast_comp_dict -f ${bname}
+
+  echo "-----------------------------------"
+endif
+
+echo ""
+
Index: /tags/gsdl-21/gsdl/packages/mg/src/scripts/mgmerge.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/scripts/mgmerge.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/scripts/mgmerge.1	(revision 699)
@@ -0,0 +1,175 @@
+`.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgmerge 1 \*(Dt CITRI
+.SH NAME
+mgmerge \- update an mg system database with new documents
+.SH SYNOPSIS
+.B mgmerge
+[
+.B \-c
+]
+[
+.BI \-g " get"
+]
+[
+.BI \-s " source"
+]
+[
+.B \-S
+]
+[
+.B \-w
+]
+.I collection-name
+.SH DESCRIPTION
+.B mgmerge
+is a
+.B csh
+script that executes all the appropriate programs in the correct order
+to completely merge a current
+.BR mg (1)
+system database ready with some new documents, saving the need for a complete
+database rebuild with
+.B mgbuild.
+It does this by building a second
+.BR mg (1)
+system database from the new documents and then merging this database
+with the old one.
+This program makes use of the
+.BR mg_get_merge (1)
+script to obtain the text of the collection.
+.B mgmerge
+cannot edit or delete documents already in a database; only new documents
+can be added.
+.SH OPTIONS
+Options can occur in any order, but the collection name must be last.
+.TP "\w'\fIcollection-name\fP'u+2n"
+.BI \-c
+This specifies whether the
+.I get
+program is \*(lqcomplex\*(rq.  If a
+.I get
+program is \*(lqcomplex\*(rq, then it requires initialisation and
+cleanup with the
+.B \-i
+and
+.B \-c
+options.
+.TP
+.BI \-g " get"
+This specifies the program to use for getting the source text for the
+build.  If no
+.B \-g
+option is given, the default program
+.BR mg_get_merge (1)
+is used.
+.TP
+.BI \-s " source"
+The
+.B mgmerge
+program consists of two parts.  The first part initializes variables
+to default values.  The second part uses these variables to control
+how the
+.BR mg (1)
+database is built.  This option specifies a program to execute between
+the first and second parts.  The details of what the variables are, and
+how they may be changed, are in comments in the
+.B mgmerge
+program.
+If this option is used, the program should ideally be the same one that
+was called by
+.BR mgbuild (1)
+to build the database since some parameters need to be consistent between
+.B mgbuild
+and
+.B mgmerge.
+.TP
+.B \-S
+This option will cause a slow merge to be performed on the inverted files,
+where each inverted file entry is decoded and recoded.
+The default is a fast merge. Accumulated fast merges slowly degrade
+compression performance on the resulting inverted file so
+a periodic slow merge is recommended.
+.TP
+.B \-w
+Adding new documents can have an effect on the weight of the previous ones.
+By default the weights for documents already in the collection are not 
+recomputed since the change in their values is usually small.
+This option forces new weights to be recomputed.
+Periodic use of this option, as for the "-S" option, is recomended,
+otherwise query rankings may become inaccurate.
+.TP
+.I collection-name
+This is the collection name, as required by the
+.BR mg_get_merge (1)
+program.  It serves both as a
+.I case
+statement selector, and as the name of a subdirectory that holds the
+indexing files.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+Note that a temporary directory under the
+.B MGDATA
+directory is used to perform the merge.
+The default name for this directory is
+.BR MERGE .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.dict
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict.blocked
+The `on-disk' stemmed dictionary.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.B *.text
+Compressed documents.
+.TP
+.B *.text.stats
+Text statistics.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.text.idx.wgt
+Interleaved index into the compressed documents and document weights.
+.TP
+.B *.weight.approx
+Approximate document weights.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mgbuild (1),
+.BR mg_get_merge (1),
+.BR mg_get (1),
+.BR mg_invf_merge (1),
+.BR mg_text_merge (1),
+.BR mg_query (1),
+.BR mg_weights_build (1).
Index: /tags/gsdl-21/gsdl/packages/mg/src/scripts/mgmerge.sh
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/scripts/mgmerge.sh	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/scripts/mgmerge.sh	(revision 699)
@@ -0,0 +1,417 @@
+#!/bin/csh -f
+###########################################################################
+#
+# mgmerge.sh -- Script used to build mg text collection.
+# Copyright (C) 1994  Neil Sharman, Shane Hudson (mods. for merging)
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#
+###########################################################################
+
+set complex = ""
+set slow_merge = ""
+set guess_weights = "-w "
+# [RPAP - Feb 97: Level 3 Merge]
+set text_pass = 1
+
+# Parse the command line arguments
+while ($#argv >= 1)
+  if ("$1" == "-s") then
+    shift
+    if ($#argv >= 1) then
+      set source = $1
+      shift
+    endif
+  else if ("$1" == "-g") then
+    shift
+    if ($#argv >= 1) then
+      set get = $1
+      shift
+    endif
+  else if ("$1" == "-c") then
+    set complex = "-text"
+    shift
+  else if ("$1" == "-S") then
+# set slow merge option in mg_invf_merge
+    set slow_merge = "-s "
+    shift
+  else if ("$1" == "-w") then
+# turn weights guessing off (rebuild weights file from scratch)
+    set guess_weights = ""
+    shift
+
+# [RPAP - Feb 97: Level 3 Merge]
+  else if ("$1" == "-T") then
+# turn text pass off
+    set text_pass = 0
+    shift
+
+  else
+    if ($?text == "0") then
+      set text = $1
+    endif
+    shift
+  endif
+    
+end
+
+if ($?text == "0") then
+  set prog = $0
+  echo "USAGE:"
+  echo "     "$prog:t" [-s config-script] [-g get-program] [-c] source"
+  echo ""
+  echo " The config-script is only needed if a non-standard build is required."
+  echo " The get-program defaults to mg_get if not specified."
+  exit 1	
+endif
+
+set bindir = $0
+set bindir = $bindir:h
+
+# if $pipe == 1 then pipe in the source text using $get and $text otherwise
+# read the source text directly from the file names specified in $input_files
+set pipe = 1
+
+if ($?get == "0") then
+  set complex = "-text"
+  set get = $bindir/mg_get_merge
+endif
+
+if (-e $MGDATA/${text}.chunks) then
+  set input_files = `cat $MGDATA/${text}.chunks`
+endif
+
+# Set the inversion method this may be either "I" or "N"
+# N is memory-efficient, I is faster for small collections
+set invf_method = N
+
+
+# Set the stemming method
+#   Bit 0 = case folding
+#   Bit 1 = S stemmer
+set stem_method = 3
+
+
+# [RPAP - Jan 97: Stem Index Change]
+# If do_indexes == 1 then build collection with full indexes to blocked file
+# overriding stem_method (stem_method will be set to 0)
+# Otherwise don't build with indexes
+set do_indexes = 0
+
+
+# $invf_mem specifies the amount of memory to use for the pass2 inversion
+# This only has an effect if $invf_method is "N".
+set invf_mem = 32
+
+
+# $num_chunks specifies the number of interium chunks of inverted file that
+# may be written to disc before a merge into the invf file is done
+# This only has an effect if $invf_method is "N".
+set num_chunks = 3
+
+# $invf_level specifies the level of the inverted file that will be generated.
+# Note: The value *MUST* be the same as was used to build the collection
+# that is being merged.
+# In the current mg_invf_merge implementation the maximum level is 2. 
+# Paragraph-level inversion (invf_level = 3) is NOT supported.
+set invf_level = 2
+
+
+# If $strip_sgml == 1 then sgml tags are stripped from the inversion phase.
+# Otherwise sgml tags are kepted.
+set strip_sgml = 1
+
+
+# $trace specifies the interval between trace entries in Mb.
+# If this is not set no trace entries will be generated.
+set trace = 10
+
+
+# $weight_bits specifies the number of bits of precision bo be given to the 
+# approximate weights.
+set weight_bits = 6
+
+
+# $mcd specifies the commandline arguments for the mg_compression_dict program
+set mcd = -S
+
+# $merge_name is the name of the subdirectory under ${MGDATA} where
+# the mergeing steps will be performed. There should NOT be an mg
+# collection with this name! The default is "MERGE"
+set merge = "MERGE"
+
+# Source the parameter file to modify parameters of the build.
+if ($?source) then
+  source ${source}
+endif
+
+
+# [RPAP - Jan 97: Stem Index Change]
+# If do_indexes == 1 then set stem_method = 0
+if ($do_indexes) then
+  set stem_method = 0
+endif
+
+
+# Generate the collection name.
+set coll_name = ${text}
+
+# Generate the directory where merging will take place
+if (-e $MGDATA/${merge}) then
+else
+  mkdir $MGDATA/${merge}
+endif
+
+# Generate the base name for the collection.
+# [RPAP - Feb 97: Level 3 Merge]
+set bname = ${coll_name}
+
+# generate the old, new and merge names in the $merge directory
+set oldname = ${merge}/${coll_name}.old
+set newname = ${merge}/${coll_name}.new
+set mergename = ${merge}/${coll_name}
+
+# build up the command lines for pass 1 and 2 to build a
+# dummy collection for the new documents
+set pass1 = (-f ${newname} -${invf_level} -m ${invf_mem} -s ${stem_method})
+set pass2 = (-f ${newname} -${invf_level} -c ${num_chunks})
+
+if ($strip_sgml) then
+  set pass1 = (${pass1} -G)
+  set pass2 = (${pass2} -G)
+endif
+
+if ($?trace) then
+  set pass1 = (${pass1} -t ${trace})
+  set pass2 = (${pass2} -t ${trace})
+endif
+
+# Note that a -T1 pass isnt done since the old .text.stats file is used
+# as the compression model for compressing the new documents.
+
+# [RPAP - Feb 97: Level 3 Merge]
+# Do not perform -T2 pass if text_pass == 0
+if ($text_pass) then
+  set pass2 = (${pass2} -T2)
+endif
+
+set pass1 = (${pass1} -${invf_method}1)
+set pass2 = (${pass2} -${invf_method}2)
+
+
+if ($?trace_name) then
+  set pass1 = (${pass1} -n ${trace_name})
+  set pass2 = (${pass2} -n ${trace_name})
+endif
+  
+if ($?comp_stats) then
+  set pass2 = (${pass2} -C ${comp_stats})
+endif
+
+########################################################################
+# Here is where mgmerge goes to work.
+
+echo "--------------------------------------------------------------"
+echo "`uname -n`, `date`"
+echo "MGMERGE collection: ${bname}"
+echo "--------------------------------------------------------------"
+echo "FIRST PHASE: build new collection from new documents"
+echo "--------------------------------------------------------------"
+
+# move and rename the text dictionary file
+# [RPAP - Feb 97: Level 3 Merge] - only if performing text pass
+if ($text_pass) then
+  mv $MGDATA/${bname}.text.dict $MGDATA/${newname}.text.dict
+endif
+
+if ($pipe) then
+  if ("$complex" != "") then
+    echo ">> $get $text -init"
+    $get $text -init
+    if ("$status" != "0") exit 1 
+    echo "-----------------------------------"
+  endif  
+endif
+
+if ($pipe) then
+  if ($?pass1filter) then
+    echo ">> $get $text $complex | $pass1filter | mg_passes ${pass1}"
+    $get $text $complex| $pass1filter | $bindir/mg_passes ${pass1}
+    if ("$status" != "0") exit 1 
+  else
+    echo ">> $get $text $complex | mg_passes ${pass1}"
+    $get $text $complex| $bindir/mg_passes ${pass1}
+    if ("$status" != "0") exit 1 
+  endif
+else
+  echo ">> mg_passes ${pass1} ${input_files}"
+  $bindir/mg_passes ${pass1} ${input_files}
+  if ("$status" != "0") exit 1 
+endif
+echo "-----------------------------------"
+
+echo "mg_perf_hash_build -f ${newname}"
+$bindir/mg_perf_hash_build -f ${newname}
+if ("$status" != "0") exit 1 
+echo "-----------------------------------"
+
+if ($pipe) then
+  if ($?pass2filter) then
+    echo ">> $get $text $complex | $pass2filter | mg_passes ${pass2}"
+    $get $text $complex | $pass2filter | $bindir/mg_passes ${pass2}
+    if ("$status" != "0") exit 1 
+  else
+    echo ">> $get $text $complex | mg_passes ${pass2}"
+    $get $text $complex | $bindir/mg_passes ${pass2}
+    if ("$status" != "0") exit 1 
+  endif
+else
+  echo ">> mg_passes ${pass2} ${input_files}"
+  $bindir/mg_passes ${pass2} ${input_files}
+  if ("$status" != "0") exit 1 
+endif
+echo "-----------------------------------"
+
+if ($pipe) then
+  if ("$complex" != "") then
+    echo "-----------------------------------"
+    echo ">> $get $text -cleanup"
+    $get $text -cleanup
+    if ("$status" != "0") exit 1 
+  endif
+endif
+
+echo "---------------------------------------"
+date
+echo "--------------------------------------------------------------"
+echo "SECOND PHASE: merge the two collections"
+echo "--------------------------------------------------------------"
+
+# move the appropriate files from the base directory to the merge dir
+echo "(moving files....)"
+# [RPAP - Feb 97: Level 3 Merge]
+# Move files if doing text pass
+if ($text_pass) then
+  mv $MGDATA/${bname}.text       $MGDATA/${oldname}.text
+  mv $MGDATA/${bname}.text.idx   $MGDATA/${oldname}.text.idx
+else
+# Need this file if building weights
+  cp -d $MGDATA/${bname}.text.idx   $MGDATA/${mergename}.text.idx
+endif
+mv $MGDATA/${bname}.invf       $MGDATA/${oldname}.invf
+mv $MGDATA/${bname}.invf.idx   $MGDATA/${oldname}.invf.idx
+mv $MGDATA/${bname}.invf.dict  $MGDATA/${oldname}.invf.dict
+# the old weight file may be used for the new file
+mv $MGDATA/${bname}.weight     $MGDATA/${mergename}.weight
+# EOC: the old invf.paragraph file may be used for the new file for Level 3 Invf
+if (-e $MGDATA/${bname}.invf.paragraph) then
+  mv $MGDATA/${bname}.invf.paragraph  $MGDATA/${mergename}.invf.paragraph
+endif
+echo "---------------------------------------"
+
+# [RPAP - Feb 97: Level 3 Merge]
+# Only merge text if doing text pass
+if ($text_pass) then
+  echo ">> mg_text_merge -f ${mergename}"
+  $bindir/mg_text_merge -f $mergename
+  echo "---------------------------------------"
+endif
+
+echo ">> mg_invf_merge ${slow_merge}${guess_weights} -f ${mergename}"
+$bindir/mg_invf_merge ${slow_merge}${guess_weights} -f ${mergename}
+echo "---------------------------------------"
+
+#remove .weight file, so a new one will be created by mg_weights_build
+if ("$guess_weights" == "") then
+  echo "rm ${mergename}.weight"
+  rm $MGDATA/${mergename}.weight
+endif
+
+# mv $MGDATA/${mergename}.weight $MGDATA/${mergename}.w2
+
+echo ">> mg_weights_build -f ${mergename} -b ${weight_bits}"
+$bindir/mg_weights_build -f ${mergename} -b ${weight_bits}
+if ("$status" != "0") exit 1 
+
+echo "---------------------------------------"
+
+echo ">> mg_invf_dict -f ${mergename} -b 4096"
+$bindir/mg_invf_dict -f ${mergename} -b 4096
+
+# [RPAP - Jan 97: Stem Index Change]
+if ($do_indexes) then
+  echo "-----------------------------------"
+
+  echo "mg_stem_idx -f ${mergename} -b 4096 -s1"
+  $bindir/mg_stem_idx -f ${mergename} -b 4096 -s1
+  if ("$status" != "0") exit 1 
+
+  echo ""
+
+  echo "mg_stem_idx -f ${mergename} -b 4096 -s2"
+  $bindir/mg_stem_idx -f ${mergename} -b 4096 -s2
+  if ("$status" != "0") exit 1 
+
+  echo ""
+
+  echo "mg_stem_idx -f ${mergename} -b 4096 -s3"
+  $bindir/mg_stem_idx -f ${mergename} -b 4096 -s3
+  if ("$status" != "0") exit 1 
+endif
+
+echo "--------------------------------------------------------------"
+
+# move files back to base dir
+# [RPAP - Feb 97: Level 3 Merge]
+# mv the text files if doing text pass
+if ($text_pass) then
+  mv $MGDATA/${oldname}.text         $MGDATA/${bname}.text
+  mv $MGDATA/${mergename}.text.idx   $MGDATA/${bname}.text.idx
+  mv $MGDATA/${newname}.text.dict    $MGDATA/${bname}.text.dict
+endif
+mv $MGDATA/${mergename}.invf       $MGDATA/${bname}.invf
+mv $MGDATA/${mergename}.invf.idx   $MGDATA/${bname}.invf.idx
+mv $MGDATA/${mergename}.invf.dict  $MGDATA/${bname}.invf.dict
+mv $MGDATA/${mergename}.invf.dict.blocked  $MGDATA/${bname}.invf.dict.blocked
+
+# [RPAP - Jan 97: Stem Index Change]
+if ($do_indexes) then
+  mv $MGDATA/${mergename}.invf.dict.blocked.1 $MGDATA/${bname}.invf.dict.blocked.1
+  mv $MGDATA/${mergename}.invf.dict.blocked.2 $MGDATA/${bname}.invf.dict.blocked.2
+  mv $MGDATA/${mergename}.invf.dict.blocked.3 $MGDATA/${bname}.invf.dict.blocked.3
+endif
+
+mv $MGDATA/${mergename}.weight     $MGDATA/${bname}.weight
+mv $MGDATA/${mergename}.weight.approx $MGDATA/${bname}.weight.approx
+mv $MGDATA/${mergename}.text.idx.wgt $MGDATA/${bname}.text.idx.wgt
+
+# EOC: mv .invf.paragraph file if exists
+if (-e $MGDATA/${mergename}.invf.paragraph) then
+  mv $MGDATA/${mergename}.invf.paragraph   $MGDATA/${bname}.invf.paragraph
+endif
+
+# remove old files in the MERGE directory
+rm -f $MGDATA/${merge}/*
+
+echo "mgstat -f ${bname} -E"
+$bindir/mgstat -f ${bname} -E
+if ("$status" != "0") exit 1 
+
+echo "--------------------------------------------------------------"
+echo "`uname -n`, `date`"
+echo "--------------------------------------------------------------"
+
+echo ""
Index: /tags/gsdl-21/gsdl/packages/mg/src/scripts/xmg.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/scripts/xmg.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/scripts/xmg.1	(revision 699)
@@ -0,0 +1,130 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH xmg 1 \*(Dt CITRI
+.SH NAME
+xmg \- An X interface to mgquery
+.SH SYNOPSIS
+.B xmg
+[
+.I collection-name
+]
+.SH DESCRIPTION
+.B xmg
+is an X interface to the
+.BR mg (1)
+information retrieval system.
+.PP
+Type a query in the ``Query'' box.  The escape key clears the query text.
+.PP
+The first line or so of each matching document is listed.
+.PP
+Click on any line to show the complete document.
+.PP
+Shift-click to show the document in a new window.
+.PP
+Press
+.I Next
+and
+.I Previous
+to cycle through the list of documents (shift works here too).
+.PP
+If you change the query, existing windows continue to refer to the
+original query.
+.PP
+You can select either Boolean or ranked queries:
+.RS
+.TP "\w'\fIBoolean queries:\fP'u+2n"
+.I "Boolean queries:"
+use ``&'' for AND, ``|'' for OR, ``!'' for NOT, and parentheses for
+grouping.
+.TP
+.I "Ranked queries:"
+Documents are listed in order of relevance.  A weight is shown
+alongside each.  Weights are normalized to 100 for the most relevant
+document.  Choose
+.I "Approximate Ranking"
+for greater retrieval speed.
+.RE
+.PP
+In document windows (other than the main window):
+.RS
+.br
+Typing `n' = next document;
+.br
+Typing `N' = next document in new window;
+.br
+Typing `p' = previous document;
+.br
+Typing `P' = previous document in new window;
+.br
+Typing `q' = close window.
+.RE
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.TP
+.SB Xdefaults
+To override the default font options, use RESOURCE_MANAGER properties
+like the following in your
+.I .Xdefaults
+file:
+.IP
+.RS
+.nf
+.I "xmg.textfont: fixed"
+.I "xmg.buttonfont: fixed"
+.fi
+.RE
+.IP
+To override the default query type of ranked, use
+.IP
+.RS
+.I "xmg.querytype: boolean"
+.RE
+.IP
+To override the default collection name (the first one listed by
+.BR ls (1)
+from your MGDATA environment variable), use
+.IP
+.RS
+.I "xmg.collection: mail93"
+.RE
+.SH "SEE ALSO"
+.na
+.BR ls (1),
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /tags/gsdl-21/gsdl/packages/mg/src/scripts/xmg.sh
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/scripts/xmg.sh	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/scripts/xmg.sh	(revision 699)
@@ -0,0 +1,1376 @@
+#!/usr/local/bin/wish -f
+# Edit the line above to point to your version of wish
+#
+###############################################################################
+#
+# xmg -- X interface to the mg information retrieval system
+# Copyright (C) 1994  Bruce McKenzie, Ian Witten and Craig Nevill-Manning
+#
+# This program is free software; you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free Software
+# Foundation; either version 2 of the License, or (at your option) any later
+# version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)xmg.sh	1.4 25 Mar 1994
+#
+###############################################################################
+#
+# "xmg" is an X interface to the mg information retrieval system.
+#
+# Type a query in the "Query" box.
+# The first line or so of each matching document is listed.
+# Click on any line to show the complete document.
+# Shift-click to show the document in a new window.
+# Press Next and Previous to cycle through the list of documents (Shift works
+#   here too). 
+# If you change the query, existing windows continue to refer to the original
+#   query. 
+# 
+# You can select either Boolean or Ranked queries.
+# Boolean queries: use "&" for AND, "|" for OR, "!" for NOT, and parentheses
+# for grouping 
+# Ranked queries:  - documents are listed in order of relevance; a weight is
+# 		     shown alongside each 
+#                  - weights are normalized to 100 for the most relevant
+#		     document 
+#                  - choose Approximate Ranking for greater retrieval speed. 
+# 
+# In document windows (other than the main window): 
+#		Typing 'n' = next document
+#		Typing 'N' = next document in new window
+#		Typing 'p' = previous document
+#		Typing 'P' = previous document in new window
+#		Typing 'q' = close window
+#
+###############################################################################
+
+# Date: 13 March 1994
+# Authors: Bruce McKenzie, Ian Witten and Craig Nevill-Manning
+
+###############################################################################
+# Customizing xmg
+###############################################################################
+
+# To override the default font options, use RESOURCE_MANAGER properties like
+# the following in your .Xdefaults file
+#
+#	xmg.textfont: fixed
+#	xmg.buttonfont: fixed
+#
+# To override the default query type of ranked, use
+#
+#	xmg.querytype: boolean
+#
+# To override the default collection name (the first one listed by /bin/ls from
+# your MGDATA environment variable), use
+#
+#	xmg.collection: mail93
+
+###############################################################################
+# Global variables
+###############################################################################
+
+#   genuine constants:
+#
+#	env		the environment variables
+#	Terminator	terminator for mgquery
+#	MaxWindowHeight	the maximum height permitted for a document window
+#	ListBoxLength	the number of items in the document list
+#	HeadLength	the length document head displayed in the list box
+#	ImageString	the string at the start of a document associated
+#			with an image
+#
+#   things that rarely get changed:
+#
+#	querytype	the current query type (boolean, ranked, approx-ranked)
+#	booleanquery	says whether the query is boolean or ranked
+#	approxquery	says whether ranking is to be approximate or not
+#	collection	the name of the document collection being used
+#	fileId		file identifer for a pipe to the mgquery process
+#       errorOccurred	flag to say that there has been an I/O error on the
+#			current collection
+#
+#   things that reflect items in the main panel
+#
+#	query		the current query
+#	hits		number of hits for the current query
+#	display		shell command to display the image associated with
+#			the last document retrieved
+#	scrolledto	document prefixes have been obtained for all documents
+#			up to this position in the document list
+#
+#   static variable needed in just one procedure
+#
+#	top_rank	weight associated with the first document retrieved
+#	windowname	unique id for each document window
+
+###############################################################################
+# High-level procedures
+###############################################################################
+
+proc SaveDocument {w} {
+  set file [FSBox "Save document to file:"]
+
+  if {$file != ""} {
+    if [file exists $file] {
+      toplevel .r
+      wm title .r "\n"
+      .r configure -borderwidth 2 -relief raised 
+      focus .r
+      label .r.message -text "The file already exists"
+      button .r.replace -text "Replace" -width 10 -height 2 \
+          -command "destroy .r; WriteDocument $file \"w\" $w" 
+      button .r.append -text "Append" -width 10 -height 2 \
+          -command "destroy .r; WriteDocument $file \"a\" $w"
+      button .r.cancel -text "Cancel" -command {destroy .r} -width 10 -height 2
+      pack .r.message -side top -pady 10 -padx 20
+      pack .r.replace .r.append .r.cancel -side left -pady 10 -padx 20
+
+    } else {
+       WriteDocument $file "w" $w
+    }
+  }
+}
+
+proc Dialog {text command} {
+      if {![winfo exists .r] } { toplevel .r }
+      wm title .r "\n"
+      .r configure -borderwidth 2 -relief raised 
+      focus .r
+      label .r.message -text $text
+      button .r.ok -text "OK" -width 10 -height 2 \
+          -command "destroy .r $command"
+      pack .r.message -side top -pady 10 -padx 20
+      pack .r.ok -side bottom -pady 10 -padx 20
+}
+
+proc WriteDocument {file mode w} {
+    if {[catch {set f [open $file $mode]}] == 0} {
+        if [winfo exists $w] {
+            puts $f [$w get 0.0 end]
+        } else {
+            Dialog "Unable to save text" ""
+        }
+        flush $f
+        close $f
+    } else {
+        Dialog "Unable to open file for writing" "; SaveDocument $w"
+    }
+}
+
+# Send a query to mg and enter the response in the document list
+#
+proc sendQuery {window} {
+    global query hits querytype errorOccurred scrolledto
+
+    .m.t.q.retrieved.label configure -foreground Gray
+    .m.t.q.retrieved.hits configure -foreground Gray
+
+    sendto ".set query \"$querytype\""
+    sendto ".set mode \"docnums\""
+    foreach f "sdocnums sdocs srank" { $window.$f delete 0 end }
+    disableButton .m.d.buttons.previous
+    disableButton .m.d.buttons.next
+    if {[string index $query 0] == "." || [string length $query] > 255} {
+        set doclist ""
+    } else { set doclist [sendandget $query] }
+
+    if {$errorOccurred} {
+        enterText .m.d.text [mgqueryFailure]
+        .m.t.q.retrieved.label configure -foreground Black
+        .m.t.q.retrieved.hits configure -foreground Black 
+        update
+        return
+    }
+    sendto ".set query \"DocNums\""
+    sendto ".set mode  \"text\""
+    .m.t.q.retrieved.label configure -foreground Black
+    .m.t.q.retrieved.hits configure -foreground Black 
+    set hits [llength $doclist]
+    set i 0
+    global top_rank
+
+    scan [lindex $doclist 0] "%d %f %d" d rank size
+    set top_rank $rank
+    if {$top_rank == 0} {set top_rank 1}
+
+    while {$i < $hits} {
+	scan [lindex $doclist $i] "%d %f %d" d rank size
+        if {$rank == "Inf"} {set rank 1}
+	if {$querytype == "boolean"} {
+	    $window.srank insert end ""
+	} else {
+	    $window.srank insert end \
+		[format "%3.0f" [expr $rank / $top_rank * 99 + 1]]
+	}
+	$window.sdocnums insert end [format " %d" $d]
+	incr i
+    }
+
+    set scrolledto 0
+    scroll $window 0
+    bind $window.sdocs <ButtonRelease-1> \
+	"showDocument $window.sdocs %y .m.d \"$doclist\" \"$query\""
+    bind $window.sdocs <Shift-ButtonRelease-1> \
+	"showDocument $window.sdocs %y New \"$doclist\" \"$query\""
+    enterText .m.d.text ""
+}
+
+# Scrolling routine for the document list
+#
+proc scroll {window n} {
+    global hits scrolledto ListBoxLength Terminator fileId ImageString
+
+    $window.sdocnums yview $n
+    $window.sdocs yview $n
+    $window.srank yview $n
+    set last $n
+    if {$last <= 0} {set last 0}
+    incr last $ListBoxLength
+    if {$last > $hits} {set last $hits}
+
+#    .m config -cursor {watch}
+#    update
+    sendto ".set mode \"heads\""
+    sendto ".set terminator \"\""
+    while {$scrolledto < $last} {
+	if {[scan [$window.sdocnums get $scrolledto] "%d" d] == 1} {
+            sendto $d
+            gets $fileId line
+            set i [string first " " $line]
+            if {$i != -1} {set line [string range $line [expr $i + 1] end]}
+            if {[string compare $ImageString [string range $line 0 9]] == 0} {
+                set line [getImageDocumentHead $d]
+            }
+	    $window.sdocs insert $scrolledto $line
+	}
+	incr scrolledto
+    }
+    sendto ".set terminator \"$Terminator\\n\""
+    sendto ".set mode text"
+#   .m config -cursor {}
+}
+
+proc getImageDocumentHead {d} {
+    global HeadLength ImageString fileId
+
+    sendto ".set heads_length 1000"
+    sendto $d
+    gets $fileId line
+
+    set line [string range $line 11 end]
+
+    set s [string first $ImageString $line]
+    if {$s != -1} {
+        incr s 11
+        set line [string range $line $s [expr $s + $HeadLength]]
+    }
+  
+    sendto ".set heads_length $HeadLength"
+
+    return "* $line"
+}
+
+# Show document from the document list
+#
+proc showDocument {window y newWindow doclist query} {
+    showThisDocument [$window nearest $y] $doclist $newWindow $query
+}
+
+proc adjacentDocument {index doclist window q button} {
+    global ListBoxLength
+
+    showThisDocument $index $doclist $window $q
+
+    if {$window == ".m.d"} {
+      set first [.m.t.q.r.sdocs nearest 0]
+      if {$index < $first || $index >= $first + $ListBoxLength} {
+          scroll .m.t.q.r [expr $index - $ListBoxLength / 2]
+      }
+      .m.t.q.r.sdocs select from $index
+    } 
+}
+
+
+# Show document with this index in the document list
+#
+proc showThisDocument {index doclist window q} {
+    global collection MaxWindowHeight display windowname
+
+    set document ""
+    scan [lindex $doclist $index] "%d %f" document rank
+    if {$document == ""} { return }
+    set textlist [sendandget $document]
+	
+    if {[string length $display] != 0} { exec csh -c $display >& /dev/null & }
+	
+# throw away the first line
+    set text [join [lrange $textlist 1 end] "\n"]
+
+# if necessary, create a new window for this document
+    if {$window == "New"} {
+        .m.t.q.r.sdocs select clear
+	set window .m.document$windowname
+        incr windowname
+        set existing ""
+        set raised 0
+        foreach w [lrange [winfo children .m] 2 end] {
+           if {[lindex [$w.docnum configure -text] 4] == $index && \
+               [lindex [$w.query configure -text] 4] == $q} {
+              if {$raised} {
+                 closeDocumentWindow $w
+              } else {
+                  raise $w
+		  getFocus $w
+                  set window $w
+                  set raised 1
+              }
+           }
+        }
+  
+        if {!$raised} {
+   	    toplevel $window
+            $window configure -borderwidth 2 -relief raised
+            wm protocol $window WM_TAKE_FOCUS "getFocus $window"
+	    getFocus $window
+	    makeDocumentPanel $window
+            bind $window q "$window.buttons.close invoke"
+            label $window.docnum -text $index
+            label $window.query -text $q
+       }
+    }
+
+# configure the window for this document
+    set rows [expr [llength $textlist] - 1]
+    if {$window != ".m.d"} {
+	wm title $window "$collection \"$q\""
+	wm iconname $window "$document"
+	$window.text configure -height $rows
+	if {$rows > $MaxWindowHeight} {
+	    $window.text configure -height $MaxWindowHeight
+	}
+    }
+
+# show the document text in the window
+    enterText $window.text $text
+
+# configure the window's Previous and Next buttons
+    if {$index == 0} {
+        disableButton $window.buttons.previous
+    } else {
+	set newindex [expr $index-1]
+	bindBoth $window previous $newindex $doclist $q
+        bind $window p [bind $window.buttons.previous <1>]
+        bind $window P [bind $window.buttons.previous <Shift-1>]
+    }
+    if {$index == [expr [llength $doclist]-1]} {
+	disableButton $window.buttons.next 
+    } else {
+	set newindex [expr $index+1]
+	bindBoth $window next $newindex $doclist $q
+        bind $window n [bind $window.buttons.next <1>]
+        bind $window N [bind $window.buttons.next <Shift-1>]
+    }
+
+    if {$window != ".m.d"} { $window.docnum configure -text $index }
+
+    if {[string length $display] != 0} { 
+        set on "{
+            if \[winfo exists $window\] \{
+                 $window.buttons.image configure \
+                   -foreground Black
+                 update\}
+        }"
+        set off "{
+            if \[winfo exists $window\] \{
+                 $window.buttons.image configure \
+                   -foreground [lindex [.m.d configure -background] 4]
+                 update\}
+        }"
+        set t 0
+        while {$t < 5} {
+            after [expr $t * 1000] eval $on
+            after [expr $t * 1000 + 500] eval $off
+            incr t
+        }   
+    }
+}
+
+# Bind both Button-1 and Shift-Button-1 commands
+#
+proc bindBoth {window button index doclist query} {
+    $window.buttons.$button configure -state normal
+    bind $window.buttons.$button <Button-1> \
+	"adjacentDocument $index \"$doclist\" $window \"$query\" $button"
+    bind $window.buttons.$button <Shift-Button-1> \
+	"adjacentDocument $index \"$doclist\" New \"$query\" $button"
+}
+
+# Change the type of query
+#
+proc changeQuerytype {name element op} {
+    global querytype booleanquery approxquery
+
+    if {$booleanquery} {
+	.m.t.b.type.approx configure -state disabled
+	.m.t.q.retrieved.weight configure \
+             -foreground [lindex [.m configure -background] 4]
+	set querytype boolean
+    } else {
+	.m.t.b.type.approx configure -state normal
+	.m.t.q.retrieved.weight configure -foreground Black
+	if {$approxquery} {
+	    set querytype "approx-ranked"
+	} else {
+	    set querytype "ranked"
+	}
+    }
+}
+
+proc enterText {window text} {
+    $window configure -state normal
+    $window delete 0.0 end
+    $window insert 0.0 $text
+    $window configure -state disabled
+}
+
+###############################################################################
+# IO and initialization
+###############################################################################
+
+# Send a message to mg and return the reply
+#
+proc sendandget {message} {
+    global fileId Terminator display errorOccurred ImageString
+    set display ""
+
+    .m config -cursor {watch}
+    update idletasks
+    set reply ""
+    sendto "$message\n"
+    if {$message != "" && !$errorOccurred} {
+        gets $fileId line
+    	while {[string compare $Terminator $line] != 0} {
+            lappend reply $line;
+            gets $fileId line
+        }
+
+        if {[string compare $ImageString [lindex $reply 1]] == 0 } {
+          if {[string compare "MGDATA" [string range [lindex $reply 3] 0 5]] == 0} {
+             set display [string range [lindex $reply 3] 7 end];
+          }
+          set reply [lrange $reply 4 end]
+        }
+    }
+
+    .m config -cursor {}
+    return $reply
+}
+    
+# Send a message to mg
+#
+proc sendto {message} {
+    global fileId errorOccurred
+
+    puts $fileId $message
+    set errorOccurred [catch {flush $fileId}]
+}
+
+# Reinitialise mgquery to work with a new document collection
+#
+proc reInitMG {c} {
+    global fileId
+    catch {close $fileId}
+    initMG $c
+}
+
+# Initialize mgquery to work with a specified text collection
+#
+proc initMG {c} {
+    global query hits fileId collection Terminator env errorOccurred
+    global HeadLength
+
+    set collection $c
+    set query ""
+    set hits 0
+    set errorOccurred 0
+
+    foreach f "sdocs srank" { .m.t.q.r.$f delete 0 end }
+    enterText .m.d.text ""
+    .m.t.b.collect.m configure -text $collection
+    foreach w [winfo children .m] {
+	if {[string first ".m.document" $w] == 0} { closeDocumentWindow $w }
+    }
+
+    disableButton .m.d.buttons.previous 
+    disableButton .m.d.buttons.next
+
+    if {$c != ""} {
+        set f $env(MGDATA)/$c
+
+        if {[file exists $f] && [file readable $f]} {
+    	    set fileId [open "| mgquery $collection" RDWR]
+	    sendto ".set expert \"true\""
+	    sendto ".set terminator \"$Terminator\\n\""
+	    sendto ".set pager \"cat\""
+            sendto ".set heads_length $HeadLength"
+	    return
+        }
+    }
+    bind .m.t.q.e.entry <Return> {null}
+    enterText .m.d.text [initializationFailure]
+    update
+}
+
+# Routine to obtain values from RESOURCE_MANAGER
+#
+proc query_resource {option class default} {
+	set val [option get . $option $class]
+	if {[string length $val] == 0} { return $default }
+	return $val
+}
+
+
+proc disableButton {b} {
+    bind $b <Button-1> {null}
+    bind $b <Shift-Button-1> {null}
+    $b configure -state disabled
+}
+
+proc null {} {}
+
+###############################################################################
+# Setting up windows
+###############################################################################
+
+proc removeFromWindowList {window} {
+    global stacking
+
+    set i 0
+    while {$i < [llength $stacking]} {
+      if {$window == [lindex $stacking $i]} {
+         set stacking [lreplace $stacking $i $i]
+      }
+      incr i
+    }
+}
+
+proc getFocus {window} {
+    global stacking
+
+    focus $window
+    removeFromWindowList $window
+    set stacking [linsert $stacking 0 $window]
+}
+
+proc closeDocumentWindow {w} {
+   global stacking
+   removeFromWindowList $w
+   set t [lindex $stacking 0]
+
+   if {$t != ""} {
+     focus $t
+     raise $t
+  }
+  destroy $w
+}
+
+# Make a "document" panel, including Previous, Next, and Close buttons
+#
+proc makeDocumentPanel {window} {
+    frame $window.buttons
+    button $window.buttons.previous -text "Previous" -width 10 -state disabled
+    button $window.buttons.next -text "Next" -width 10 -state disabled
+    button $window.buttons.close -text "Close" -width 10 \
+	-command "closeDocumentWindow $window"
+    button $window.buttons.save -text "Save" -width 10 \
+        -command "SaveDocument $window.text"
+
+    label $window.buttons.image -text "Fetching image..." \
+            -foreground [lindex [$window configure -background] 4]
+    pack $window.buttons.previous $window.buttons.next -side left
+    pack $window.buttons.image -side left -padx 10
+    pack $window.buttons.close $window.buttons.save -side right 
+    pack $window.buttons -side top -fill x 
+
+    text $window.text -borderwidth 2 -relief raised \
+             -setgrid true -width 80 -state disabled
+
+    $window.text configure -yscrollcommand "$window.sy set" 
+    scrollbar $window.sy -relief flat -command "$window.text yview"
+    pack $window.sy -side right -fill y 
+    pack $window.text -side top -expand yes -fill both
+}
+
+# Make the "buttons" panel, with the query type and collection name
+#
+proc mkButtons {window} {
+    global collection booleanquery approxquery env
+    frame $window.type
+    label $window.type.label -text "Query type"
+
+    radiobutton $window.type.boolean -text "Boolean" -variable booleanquery \
+	  -value 1 -relief flat
+
+    radiobutton $window.type.ranked -text "Ranked" -variable booleanquery \
+	  -value 0 -relief flat
+
+    checkbutton $window.type.approx -text "Approximate" -variable approxquery \
+	  -relief flat
+
+    trace variable booleanquery w changeQuerytype
+    trace variable approxquery w changeQuerytype
+
+    pack $window.type.label   -side top -anchor w
+    pack $window.type.boolean -side top -anchor w -padx 12
+    pack $window.type.ranked  -side top -anchor w -padx 12
+    pack $window.type.approx  -side top -anchor w -padx 36
+
+    pack $window.type -side top -anchor w -pady 12
+
+    frame $window.collect
+    label $window.collect.label -text "Collection: "
+    pack $window.collect.label -side left -anchor w
+
+    getViewer
+
+    set dirs [getMGDATA]
+    set collection ""
+    if {$dirs != ""} { set collection [lindex $dirs 0] }
+   
+    menubutton $window.collect.m -menu $window.collect.m.menu \
+	-relief sunken
+    menu $window.collect.m.menu
+    foreach c $dirs {
+        if {[file readable $env(MGDATA)/$c] && 
+            [file isdirectory $env(MGDATA)/$c]} {
+    	    $window.collect.m.menu add command -label $c \
+	        -command "reInitMG $c"
+        }
+    }
+    pack $window.collect.m -side left
+    pack $window.collect -side bottom -anchor w
+
+
+}
+
+# Check that the MGDATA environment variable is set, and is non-null
+#
+proc getMGDATA {} {
+    global env
+
+    set MGDATA 0
+    foreach i [array names env] {
+	if {$i == "MGDATA"} {
+	    set dirs [exec /bin/ls $env(MGDATA)]
+	    if {[info exists dirs]} { return $dirs }
+	}
+    }
+
+    return ""
+}
+
+proc getViewer {} {
+    global env
+
+    set viewer 0
+    foreach i [array names env] {
+	if {$i == "MGIMAGEVIEWER"} {set viewer $env(MGIMAGEVIEWER)}
+    }
+
+    if {$viewer == 0} {set env(MGIMAGEVIEWER) "xloadimage -quiet -fit stdin"}
+}
+
+# Make the "query" panel, with the query and list of retrieved documents
+#
+proc mkQuery {window} {
+    global hits ListBoxLength
+
+# at the top of the "query" panel is the "entry" frame
+    frame $window.e
+    label $window.e.label -text "Query"
+    entry $window.e.entry -relief sunk -width 47 -textvariable query
+    pack $window.e.label -side left
+    pack $window.e.entry -side right -expand yes -fill x
+    focus default $window.e.entry
+    focus $window.e.entry
+
+    bind $window.e.entry <Control-Key> {null}
+    bind $window.e.entry <Return> "sendQuery $window.r"
+    bind $window.e.entry <Escape> { set query "" }
+    bind $window.e.entry <2> {
+	%W insert insert [selection get STRING]
+	tk_entrySeeCaret %W
+    }
+
+# next is the "retrieved" frame
+    frame $window.retrieved
+    label $window.retrieved.weight -text "Weight"
+    label $window.retrieved.label -text "   Documents retrieved: "
+    label $window.retrieved.hits -width 4 -relief sunk -textvariable hits
+    set documentsRetrieved 0
+    pack $window.retrieved.weight $window.retrieved.label \
+        $window.retrieved.hits -side left
+
+# next is the "results" frame (note that the docnums listbox doesn't appear on
+# the screen)
+    frame $window.r
+    scrollbar $window.r.sscroll -relief sunken -command "scroll $window.r"
+#    bind $window.r.sscroll <ButtonRelease-1> "updateSdocs $window.r.sdocs $window.r.sdocnums"
+    listbox $window.r.sdocnums
+    listbox $window.r.sdocs -relief sunken \
+            -setgrid 1 -geometry 50x$ListBoxLength
+    listbox $window.r.srank -yscroll "$window.r.sscroll set" \
+            -setgrid 1 -geometry 4x$ListBoxLength
+  
+    bind $window.r.srank <Any-1> {null}
+    bind $window.r.srank <Any-2> {null}
+    bind $window.r.srank <Any-3> {null}
+
+    bind $window.r.srank <Any-B1-Motion> {null}
+    bind $window.r.srank <Any-B2-Motion> {null}
+    bind $window.r.srank <Any-B3-Motion> {null}
+
+    bind $window.r.sdocs <Any-2> {null}
+    bind $window.r.sdocs <Any-B2-Motion> {null}
+
+    pack $window.r.sscroll -side right -fill y
+    pack $window.r.srank -side left 
+    pack $window.r.sdocs -side left -fill x -expand yes
+
+# pack them up
+    pack $window.e -side top -anchor e -pady 10 -fill x -expand yes
+    pack $window.retrieved -anchor w
+    pack $window.r -side bottom -anchor e -pady 10 -fill x -expand yes
+}
+
+# Make a "help" window
+#
+proc mkHelp {} {
+    if [winfo exists .help] { destroy .help }
+    toplevel .help
+    wm title .help "X interface to the mg information retrieval system"
+    wm iconname .help "About xmg"
+    frame .help.buttons
+    button .help.buttons.close -text Close -command "destroy .help" -width 10
+    button .help.buttons.help -text Help -width 10
+    button .help.buttons.conditions -text Conditions -width 10
+    button .help.buttons.warranty -text Warranty -width 10
+    button .help.buttons.bugs -text Bugs -width 10
+
+    frame .help.t
+    text .help.t.text -relief raised -bd 2 -setgrid true -height 24 -width 95
+    .help.t.text configure -yscrollcommand ".help.t.sy set" 
+    scrollbar .help.t.sy -relief flat -command ".help.t.text yview"
+    
+    pack .help.buttons.close .help.buttons.help -side right -fill x
+    pack .help.buttons.conditions .help.buttons.warranty \
+         .help.buttons.bugs -side left
+    pack .help.buttons -side bottom -fill x
+
+    pack .help.t.text -side left -expand yes -fill both
+    pack .help.t.sy -side right -fill y 
+    pack .help.t -side top -fill both -expand yes
+    enterText .help.t.text [HelpText]
+    grayButton help
+}
+
+proc grayButton {button} {
+    bind .help.buttons.help <1> "enterText .help.t.text \[HelpText\]
+         grayButton help" 
+    .help.buttons.help configure -state normal
+
+    bind .help.buttons.bugs <1> "enterText .help.t.text \[Bugs\]
+         grayButton bugs" 
+    .help.buttons.bugs configure -state normal
+
+    bind .help.buttons.warranty <1> "enterText .help.t.text \[Warranty\]
+         grayButton warranty"
+    .help.buttons.warranty configure -state normal
+
+    bind .help.buttons.conditions <1> "enterText .help.t.text \[Conditions\]
+         grayButton conditions" 
+    .help.buttons.conditions configure -state normal
+    disableButton .help.buttons.$button
+}
+
+# The text for errors in the MG environment variable
+#
+
+
+proc bookDetails {} {
+  return { \
+
+ This X interface was written by Bruce McKenzie, Ian Witten and Craig Nevill-Manning.  
+ For information on mg, see
+             Witten, I.H.,  Moffat, A. and Bell, T.C. (1994) "Managing gigabytes: compressing
+             and indexing documents and images." Van Nostrand Reinhold, New York.
+             ISBN 0-442-01863-0, US $54.95; call 1 (800) 544-0550 to order.
+}
+}
+
+proc initializationFailure {} {
+    return [format "%s\n%s" {\
+In order to run XMG you need to have already created and indexed some document collections,
+ and placed them in a directory whose name is given by the environment variable MGDATA.
+ Your MGDATA variable is unset or does not specify a directory with files in; therefore XMG
+ cannot proceed.
+} [bookDetails]]
+}
+
+proc mgqueryFailure {} {
+    return [format "%s\n%s" {\
+In order to run XMG you need to have already created and indexed some document collections,
+ and placed them in a directory whose name is given by the environment variable MGDATA.
+ There is a problem with the collection that you have chosen. Please re-index the collection before
+ you use it again.
+} [bookDetails]]
+
+}
+
+# The text for the help message
+#
+proc HelpText {} {
+    return [format "%s\n%s" {\
+This is an X interface to the mg information retrieval system.
+
+ Type a query in the "Query" box.
+ The first line or so of each matching document is listed. 
+ Click on any line to show the complete document.
+ Shift-click to show the document in its own window
+ Press Next and Previous to cycle through the list of documents (Shift works here too).
+ In document windows (other than the main window), typing n or p shows the next and previous 
+ documents; N or P creates a new window for them. q closes the most recently created window.
+ If you change the query, existing windows continue to refer to the original query.
+ 
+ You can select either Boolean or Ranked queries.
+ Boolean queries: use & for AND, | for OR, ! for NOT, and parentheses for grouping
+ Ranked queries:  documents are listed in order of relevance; a weight is shown alongside each
+                                  weights are normalized to 100 for the most relevant document
+                                  choose Approximate Ranking for greater retrieval speed. 
+ } [bookDetails]]
+}
+
+proc reformat {text} {
+    regsub -all "\n *\n *" $text "BLEQUEWFI" text
+    regsub -all "\[ \t\n\r\]+" $text " " text
+    regsub -all "BLEQUEWFI" $text "\n\n" text
+    return $text
+}
+
+proc Conditions {} {
+    return [reformat [join [sendandget ".conditions\nTHISWILLNEVERMATCHANYTHING\n"] "\n"]]
+}
+
+proc Warranty {} {
+    return [reformat [join [sendandget ".warranty\nTHISWILLNEVERMATCHANYTHING\n"] "\n"]]
+}
+
+proc Bugs {} {
+    return [reformat {
+Known shortcomings of xmg include:
+
+    Slow picture viewing.  Pictures are decompressed in their entirety before a window is created 
+        for the picture.  This often results in a long delay with no feedback. Moreover, when 
+        an image's document is displayed repeatedly, multiple identical images are shown.
+
+    Sequential viewing.  There is no way of going to the next document in the collection to see the
+        context of documents that have been retrieved.  This would be trivial to implement but
+        conflicts with the interpretation of the "previous" and "next" buttons.
+    }]
+}
+
+###############################################################################
+# The main program
+###############################################################################
+
+scan [info tclversion] "%d.%d" major minor
+
+if {$major < 7} {
+    puts "xmg requires version 7.0 of the tcl interpreter. You are using version $major.$minor"
+    exit
+}
+
+# Default font settings
+set defaulttextfont   -adobe-times-medium-r-normal--14-140-75-75-p-74-iso8859-1
+set defaultbuttonfont -adobe-times-bold-r-normal--14-140-75-75-p-77-iso8859-1
+set textfont   [query_resource textfont Textfont $defaulttextfont]
+set buttonfont [query_resource buttonfont Buttonfont $defaultbuttonfont]
+
+set MaxWindowHeight 10
+set HeadLength 100
+set scrolledto 0
+set ListBoxLength 12
+set windowname 0
+set Terminator "RECORD_END"
+set fileId ""
+set query ""
+set ImageString "::::::::::"
+set stacking ".m.t"
+
+tk_listboxSingleSelect Listbox
+option add *Text.wrap word
+
+option add *Text.font $textfont
+option add *Listbox.font $textfont
+option add *Entry.font $textfont
+
+option add *Button.font $buttonfont
+option add *Menubutton.font $buttonfont
+option add *Radiobutton.font $buttonfont
+option add *Checkbutton.font $buttonfont
+option add *Label.font $buttonfont
+option add *Menu.font $buttonfont
+
+wm title . "Managing Gigabytes"
+wm iconname . "xmg"
+frame .m -borderwidth 5 -relief raised
+
+# top part of frame
+frame .m.t
+
+# on the left side of the top part is the "buttons" panel
+frame .m.t.b
+mkButtons .m.t.b
+
+# on the right side is the "query" panel
+frame .m.t.q
+mkQuery .m.t.q
+
+set q [query_resource querytype Querytype approx-ranked]
+set booleanquery [expr \"$q\" == \"boolean\"]
+set approxquery [expr \"$q\" == \"approx-ranked\"]
+# the above lines will trigger changeQuerytype
+
+pack .m.t.b -side left -anchor nw
+pack .m.t.q -side right -anchor ne -fill both -expand yes
+
+# at the bottom is the "document" panel
+frame .m.d
+makeDocumentPanel .m.d
+
+.m.d.buttons.close configure -text "Exit" \
+    -command "destroy ."
+button .m.d.buttons.help -text "Help" -width 10 \
+	-command "mkHelp"
+
+pack .m.d.buttons.help -side right
+pack .m.t -side top -fill x -expand yes
+pack .m.d -side bottom -fill both -expand yes
+
+pack .m -fill both -expand yes
+wm geometry . +300+85
+
+set collection [query_resource collection Collection $collection]
+if {$argc>0} { set collection $argv }
+
+enterText .m.d.text [HelpText]
+
+initMG $collection
+
+###############################################################################
+###############################################################################
+# The following file selection box procedure was written by Andrew Donkin
+# of the University of Waikato. 
+###############################################################################
+
+proc FSBox { {fsBoxMessage "Select file:"}} {
+# xf ignore me 5
+  global fsBox
+
+  set fsBox(name) ""
+  set fsBox(internalPath) [pwd]
+  set fsBox(pattern) "*"
+  set fsBox(all) 0
+
+  # build widget structure
+
+  # start build of toplevel
+  if {"[info commands XFDestroy]" != ""} {
+    catch {XFDestroy .fsBox}
+  } {
+    catch {destroy .fsBox}
+  }
+  toplevel .fsBox -borderwidth 0
+  wm geometry .fsBox 350x300 
+  wm title .fsBox {File select box}
+  wm maxsize .fsBox 1000 1000
+  wm minsize .fsBox 100 100
+  # end build of toplevel
+
+  label .fsBox.message1 -anchor c -text "$fsBoxMessage"
+  frame .fsBox.frame1  -borderwidth 0  -relief raised
+
+  button .fsBox.frame1.ok  -text "OK"  -command "
+      global fsBox
+      set fsBox(name) \[.fsBox.file.file get\]
+      set fsBox(path) \[.fsBox.path.path get\]
+
+      set fsBox(internalPath) \[.fsBox.path.path get\]
+
+      if {\"\[info commands XFDestroy\]\" != \"\"} {
+        catch {XFDestroy .fsBox}
+      } {
+        catch {destroy .fsBox}
+      }"
+
+  button .fsBox.frame1.rescan -text "Rescan" -command {
+      global fsBox
+      FSBoxFSShow [.fsBox.path.path get] [.fsBox.pattern.pattern get] $fsBox(all)
+  }
+
+  button .fsBox.frame1.cancel -text "Cancel" -command "
+      global fsBox
+      set fsBox(name) {}
+      set fsBox(path) {}
+
+      if {\"\[info commands XFDestroy\]\" != \"\"} {
+        catch {XFDestroy .fsBox}
+      } {
+        catch {destroy .fsBox}
+      }"
+
+  frame .fsBox.path -borderwidth 0 -relief raised
+  frame .fsBox.path.paths 
+  label .fsBox.path.paths.paths -borderwidth 0 -text "Pathname:"
+
+  entry .fsBox.path.path -relief sunken
+  .fsBox.path.path insert 0 $fsBox(internalPath)
+  frame .fsBox.pattern -borderwidth 0 -relief raised
+  frame .fsBox.pattern.patterns 
+
+  label .fsBox.pattern.patterns.patterns -borderwidth 0 -text "Selection pattern:"
+  entry .fsBox.pattern.pattern -relief sunken
+  .fsBox.pattern.pattern insert 0 $fsBox(pattern)
+  
+  frame .fsBox.files -borderwidth 0 -relief raised
+  scrollbar .fsBox.files.vscroll -relief raised -command ".fsBox.files.files yview"
+  scrollbar .fsBox.files.hscroll -orient horiz -relief raised -command ".fsBox.files.files xview" 
+  listbox .fsBox.files.files -exportselection false -relief raised -xscrollcommand \
+        ".fsBox.files.hscroll set" -yscrollcommand ".fsBox.files.vscroll set"
+  frame .fsBox.file -borderwidth 0 -relief raised
+  label .fsBox.file.labelfile -text "Filename:"
+  entry .fsBox.file.file -relief sunken
+  focus .fsBox.file.file
+
+  .fsBox.file.file delete 0 end
+  .fsBox.file.file insert 0 $fsBox(name)
+  
+  checkbutton .fsBox.pattern.all -offvalue 0 -onvalue 1 -text "Show all files" -variable fsBox(all) -command {
+      global fsBox
+      FSBoxFSShow [.fsBox.path.path get] [.fsBox.pattern.pattern get] $fsBox(all)
+    }
+
+  FSBoxFSShow $fsBox(internalPath) $fsBox(pattern) $fsBox(all)
+
+  # bindings
+  bind .fsBox.files.files <Double-Button-1> "FSBoxFSFileSelectDouble %W %y"
+  bind .fsBox.files.files <ButtonPress-1> "FSBoxFSFileSelect %W %y"
+  bind .fsBox.files.files <Button1-Motion> "FSBoxFSFileSelect %W %y"
+  bind .fsBox.files.files <Shift-Button1-Motion> "FSBoxFSFileSelect %W %y"
+  bind .fsBox.files.files <Shift-ButtonPress-1> "FSBoxFSFileSelect %W %y"
+
+  bind .fsBox.path.path <Return> {
+    global fsBox
+    FSBoxFSShow [.fsBox.path.path get] [.fsBox.pattern.pattern get] $fsBox(all)
+    .fsBox.file.file icursor end
+    focus .fsBox.file.file}
+  catch "bind .fsBox.path.path <Up> {}"
+  bind .fsBox.path.path <Down> {
+    .fsBox.file.file icursor end
+    focus .fsBox.file.file}
+
+  bind .fsBox.file.file <Return> "
+    global fsBox
+    set fsBox(name) \[.fsBox.file.file get\]
+    set fsBox(path) \[.fsBox.path.path get\]
+    set fsBox(internalPath) \[.fsBox.path.path get\]
+
+    if {\"\[info commands XFDestroy\]\" != \"\"} {
+      catch {XFDestroy .fsBox}
+    } {
+      catch {destroy .fsBox}
+    }"
+  bind .fsBox.file.file <Up> {.fsBox.path.path icursor end
+    focus .fsBox.path.path}
+  bind .fsBox.file.file <Down> {.fsBox.pattern.pattern icursor end
+    focus .fsBox.pattern.pattern}
+
+  bind .fsBox.pattern.pattern <Return> {
+    global fsBox
+    FSBoxFSShow [.fsBox.path.path get] [.fsBox.pattern.pattern get] $fsBox(all)}
+  bind .fsBox.pattern.pattern <Up> {
+    .fsBox.file.file icursor end
+    focus .fsBox.file.file}
+  catch "bind .fsBox.pattern.pattern <Down> {}"
+
+  # packing
+  pack .fsBox.files.vscroll -side left -fill y 
+  pack .fsBox.files.hscroll -side bottom -fill x
+  pack .fsBox.files.files -side left -fill both -expand yes
+
+  pack .fsBox.file.labelfile -side left
+  pack .fsBox.file.file -side left -fill  both -expand yes
+
+  pack .fsBox.frame1.ok -side left -fill both -expand yes
+  pack .fsBox.frame1.rescan -side left -fill both -expand yes
+  pack .fsBox.frame1.cancel -side left -fill both -expand yes
+
+  pack .fsBox.path.paths.paths -side left
+
+  pack .fsBox.pattern.patterns.patterns -side left
+
+  pack .fsBox.path.paths -side left
+  pack .fsBox.path.path -side left -fill both -expand yes
+
+  pack .fsBox.pattern.patterns -side left
+  pack .fsBox.pattern.all -side right -fill both
+  pack .fsBox.pattern.pattern -side left -fill both -expand yes
+
+  pack .fsBox.message1 -side top -fill both
+  pack .fsBox.frame1 -side bottom -fill both
+  pack .fsBox.pattern -side bottom -fill both
+  pack .fsBox.file -side bottom -fill both
+  pack .fsBox.path -side bottom -fill both
+  pack .fsBox.files -side left -fill both -expand yes
+
+  update idletask
+  grab .fsBox
+  tkwait window .fsBox
+
+  if {"[string trim $fsBox(path)]" != "" ||
+      "[string trim $fsBox(name)]" != ""} {
+    if {"[string trimleft [string trim $fsBox(name)] /]" == ""} {
+      return [string trimright [string trim $fsBox(path)] /]
+    } {
+      return [string trimright [string trim $fsBox(path)] /]/[string trimleft [string trim $fsBox(name)] /]
+    }
+  }
+}
+
+proc FSBoxBindSelectOne { fsBoxW fsBoxY} {
+# xf ignore me 6
+
+  set fsBoxNearest [$fsBoxW nearest $fsBoxY]
+  if {$fsBoxNearest >= 0} {
+    $fsBoxW select from $fsBoxNearest
+    $fsBoxW select to $fsBoxNearest
+  }
+}
+
+proc FSBoxFSFileSelect { fsBoxW fsBoxY} {
+# xf ignore me 6
+  global fsBox
+
+  FSBoxBindSelectOne $fsBoxW $fsBoxY
+  set fsBoxNearest [$fsBoxW nearest $fsBoxY]
+  if {$fsBoxNearest >= 0} {
+    set fsBoxTmpEntry [$fsBoxW get $fsBoxNearest]
+    if {"[string index $fsBoxTmpEntry           [expr [string length $fsBoxTmpEntry]-1]]" == "/" ||
+        "[string index $fsBoxTmpEntry           [expr [string length $fsBoxTmpEntry]-1]]" == "@"} {
+      set fsBoxFileName [string range $fsBoxTmpEntry 0             [expr [string length $fsBoxTmpEntry]-2]]
+      if {![IsADir [string trimright $fsBox(internalPath)/$fsBoxFileName @]] &&
+          ![IsASymlink [string trimright $fsBox(internalPath)/$fsBoxFileName @]]} {
+        set fsBoxFileName $fsBoxTmpEntry
+      }
+    } {
+      if {"[string index $fsBoxTmpEntry             [expr [string length $fsBoxTmpEntry]-1]]" == "*"} {
+        set fsBoxFileName [string range $fsBoxTmpEntry 0           [expr [string length $fsBoxTmpEntry]-2]]
+        if {![file executable $fsBox(internalPath)/$fsBoxFileName]} {
+          set fsBoxFileName $fsBoxTmpEntry
+        }
+      } {
+        set fsBoxFileName $fsBoxTmpEntry
+      }
+    }
+    if {![IsADir [string trimright $fsBox(internalPath)/$fsBoxFileName @]]} {
+      set fsBox(name) $fsBoxFileName
+      .fsBox.file.file delete 0 end
+      .fsBox.file.file insert 0 $fsBox(name)
+    }
+  }
+}
+
+proc FSBoxFSFileSelectDouble { fsBoxW fsBoxY} {
+# xf ignore me 6
+  global fsBox
+
+  FSBoxBindSelectOne $fsBoxW $fsBoxY
+  set fsBoxNearest [$fsBoxW nearest $fsBoxY]
+  if {$fsBoxNearest >= 0} {
+    set fsBoxTmpEntry [$fsBoxW get $fsBoxNearest]
+    if {"$fsBoxTmpEntry" == "../"} {
+      set fsBoxTmpEntry [string trimright [string trim $fsBox(internalPath)] "@/"]
+      if {"$fsBoxTmpEntry" == ""} {
+        return
+      }
+      FSBoxFSShow [file dirname $fsBoxTmpEntry]         [.fsBox.pattern.pattern get] $fsBox(all)
+      .fsBox.path.path delete 0 end
+      .fsBox.path.path insert 0 $fsBox(internalPath)
+    } {
+      if {"[string index $fsBoxTmpEntry [expr [string length $fsBoxTmpEntry]-1]]" == "/" ||
+          "[string index $fsBoxTmpEntry [expr [string length $fsBoxTmpEntry]-1]]" == "@"} {
+        set fsBoxFileName [string range $fsBoxTmpEntry 0               [expr [string length $fsBoxTmpEntry]-2]]
+        if {![IsADir [string trimright $fsBox(internalPath)/$fsBoxFileName @]] &&
+            ![IsASymlink [string trimright $fsBox(internalPath)/$fsBoxFileName @]]} {
+          set fsBoxFileName $fsBoxTmpEntry
+        }
+      } {
+        if {"[string index $fsBoxTmpEntry [expr [string length $fsBoxTmpEntry]-1]]" == "*"} {
+          set fsBoxFileName [string range $fsBoxTmpEntry 0                 [expr [string length $fsBoxTmpEntry]-2]]
+          if {![file executable $fsBox(internalPath)/$fsBoxFileName]} {
+            set fsBoxFileName $fsBoxTmpEntry
+          }
+        } {
+          set fsBoxFileName $fsBoxTmpEntry
+        }
+      }
+      if {[IsADir [string trimright $fsBox(internalPath)/$fsBoxFileName @]]} {
+        set fsBox(internalPath) "[string trimright $fsBox(internalPath) {/@}]/$fsBoxFileName"
+        FSBoxFSShow $fsBox(internalPath) [.fsBox.pattern.pattern get] $fsBox(all)
+        .fsBox.path.path delete 0 end
+        .fsBox.path.path insert 0 $fsBox(internalPath)
+      } {
+        set fsBox(name) $fsBoxFileName
+        set fsBox(path) $fsBox(internalPath)
+        if {"[info commands XFDestroy]" != ""} {
+          catch {XFDestroy .fsBox}
+        } {
+          catch {destroy .fsBox}
+        }
+      }
+    }
+  }
+}
+
+proc FSBoxFSShow { fsBoxPath fsBoxPattern fsBoxAll} {
+# xf ignore me 6
+  global fsBox
+
+  set fsBox(pattern) $fsBoxPattern
+
+  if {[file exists $fsBoxPath] && [file readable $fsBoxPath] &&
+      [IsADir $fsBoxPath]} {
+    set fsBox(internalPath) $fsBoxPath
+  } {
+    if {[file exists $fsBoxPath] && [file readable $fsBoxPath] &&
+        [IsAFile $fsBoxPath]} {
+      set fsBox(internalPath) [file dirname $fsBoxPath]
+      .fsBox.file.file delete 0 end
+      .fsBox.file.file insert 0 [file tail $fsBoxPath]
+      set fsBoxPath $fsBox(internalPath)
+    } {
+      while {"$fsBoxPath" != "" && "$fsBoxPath" != "/" &&
+             ![file isdirectory $fsBoxPath]} {
+        set fsBox(internalPath) [file dirname $fsBoxPath]
+         set fsBoxPath $fsBox(internalPath)
+      }
+    }
+  }
+  if {"$fsBoxPath" == ""} {
+    set fsBoxPath "/"
+    set fsBox(internalPath) "/"
+  }
+  .fsBox.path.path delete 0 end
+  .fsBox.path.path insert 0 $fsBox(internalPath)
+
+  if {[.fsBox.files.files size] > 0} {
+    .fsBox.files.files delete 0 end
+  }
+  if {$fsBoxAll} {
+    if {[catch "exec ls -F -a $fsBoxPath" fsBoxResult]} {
+      puts stderr "$fsBoxResult"
+    }
+  } {
+    if {[catch "exec ls -F $fsBoxPath" fsBoxResult]} {
+      puts stderr "$fsBoxResult"
+    }
+  }
+  set fsBoxElementList [lsort $fsBoxResult]
+
+  foreach fsBoxCounter [winfo children .fsBox.pattern.patterns.patterns] {
+    if {[string length [info commands XFDestroy]] > 0} {
+      catch {XFDestroy $fsBoxCounter}
+    } {
+      catch {destroy $fsBoxCounter}
+    }
+  }
+  menu .fsBox.pattern.patterns.patterns.menu
+
+  if {"$fsBoxPath" != "/"} {
+    .fsBox.files.files insert end "../"
+  }
+  foreach fsBoxCounter $fsBoxElementList {
+    if {[string match $fsBoxPattern $fsBoxCounter] ||
+        [IsADir [string trimright $fsBoxPath/$fsBoxCounter "/@"]]} {
+      if {"$fsBoxCounter" != "../" &&
+          "$fsBoxCounter" != "./"} {
+        .fsBox.files.files insert end $fsBoxCounter
+      }
+    }
+  }
+}
+
+proc IsADir { pathName} {
+# xf ignore me 5
+  if {[file isdirectory $pathName]} {
+    return 1
+  } {
+    catch "file type $pathName" fileType
+    if {"$fileType" == "link"} {
+      if {[catch "file readlink $pathName" linkName]} {
+        return 0
+      }
+      catch "file type $linkName" fileType
+      while {"$fileType" == "link"} {
+        if {[catch "file readlink $linkName" linkName]} {
+          return 0
+        }
+        catch "file type $linkName" fileType
+      }
+      return [file isdirectory $linkName]
+    }
+  }
+  return 0
+}
+
+proc IsAFile { fileName} {
+# xf ignore me 5
+  if {[file isfile $fileName]} {
+    return 1
+  } {
+    catch "file type $fileName" fileType
+    if {"$fileType" == "link"} {
+      if {[catch "file readlink $fileName" linkName]} {
+        return 0
+      }
+      catch "file type $linkName" fileType
+      while {"$fileType" == "link"} {
+        if {[catch "file readlink $linkName" linkName]} {
+          return 0
+        }
+        catch "file type $linkName" fileType
+      }
+      return [file isfile $linkName]
+    }
+  }
+  return 0
+}
+
+proc IsASymlink { fileName} {
+# xf ignore me 5
+  catch "file type $fileName" fileType
+  if {"$fileType" == "link"} {
+    return 1
+  }
+  return 0
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/MGPASS.C
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/MGPASS.C	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/MGPASS.C	(revision 699)
@@ -0,0 +1,502 @@
+/**************************************************************************
+ *
+ * mgpass.cpp -- Driver for the various passes -
+
+	V1 -  removed all the pipe processing and replaced with
+	      code to directly explore a directory of files.
+	V2 -  rebuilt to extract non text files from web browser
+	      'catch' file.  Also to display progress count
+			   GH/WJR
+
+ * Copyright (C) 1994  Neil Sharman, ..
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include <malloc.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <io.h>
+#include <fcntl.h>
+#include <string.h>
+
+#include "memlib.h"
+#include "messages.h"
+
+#include "mg_files.h"
+#include "mg.h"
+#include "build.h"
+#include "text.h"
+#include "stemmer.h"
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:17:39  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.5  1998/12/02 01:32:40  sjboddie
+   fixed 1 missing semi-colon..
+
+   Revision 1.4  1998/12/02 00:57:10  rjmcnab
+
+   Fixed to compile with the changes to the stemming.
+
+   Revision 1.3  1998/11/24 03:39:23  rjmcnab
+
+   Fixed a small error in the windows compilation
+
+   Revision 1.2  1998/11/24 01:29:39  rjmcnab
+
+   Fixed a few problems with the windows build
+
+   Revision 1.1  1998/11/17 09:34:12  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:57  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:52  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#define MAX_PASSES 5
+
+#define SPECIAL 1
+#define TEXT_PASS_1 2
+#define TEXT_PASS_2 4
+#define IVF_PASS_1 8
+#define IVF_PASS_2 16
+
+#define MIN_BUF 8192
+#define path_length 256
+
+unsigned long buf_size = 3 * 1024 * 1024;       /* 3Mb */
+unsigned long invf_buffer_size = 5 * 1024 * 1024;       /* 5Mb */
+unsigned long ChunkLimit = 0;
+char InvfLevel = 2;
+char SkipSGML = 0;
+char MakeWeights = 0;
+FILE *Comp_Stats = NULL;
+int comp_stat_point = 0;
+double bytes_processed = 0;
+int num_docs = 0;
+double bytes_received = 0;
+int stem_method = 0;
+int stemmer_num = 0; /* default to the Lovin stemmer */
+
+static char Passes = 0;
+static unsigned long trace = 0;
+static int Dump = 0;
+static char **files = NULL;
+static int num_files = 0;
+static char *trace_name = NULL;
+
+static char dirname[path_length], wildname[path_length];
+static int by_para = 0, recurse = 0, html_catch = 0;
+static char *buffer;
+char *line_start, *data_end, *base, *scan;
+
+typedef struct pass_data
+  {
+    char *name;
+    int (*init) (char *);
+    int (*process) (u_char *, int);
+    int (*done) (char *);
+  }
+pass_data;
+
+static pass_data PassData[MAX_PASSES] =
+{
+  {"special", init_special, process_special, done_special},
+  {"text.pass1", init_text_1, process_text_1, done_text_1},
+  {"text.pass2", init_text_2, process_text_2, done_text_2},
+  {"ivf.pass1", init_ivf_1, process_ivf_1, done_ivf_1},
+  {"ivf.pass2", init_ivf_2, process_ivf_2, done_ivf_2},
+};
+
+static char *usage_str = "\nUSAGE:\n"
+"  %s [-h] [-G] [-D] [-1|-2|-3] [-T1] [-T2] [-I1] [-I2] [-N1]\n"
+"  %*s [-N2] [-W] [-S] [-b buffer-size] [-d dictionary-directory]\n"
+"  %*s [-t trace-point Mb] [-m invf-memory] [-c chunk-limit]\n"
+"  %*s [-n trace-name] [-C comp-stat-size] [-s stem_method] -f doc-collection-name\n"
+"  %*s [source directory\\] [source file]\n";
+
+static void usage (char *err)
+{
+  if (err) Message (err);
+  fprintf (stderr, usage_str, msg_prefix, strlen (msg_prefix), "",
+    strlen (msg_prefix), "", strlen (msg_prefix), "");
+  exit (1);
+}
+
+void do_process(char *buffer, int num)
+{
+  int pass;
+  if (num == 0) Message ("Warning : Processing zero length document");
+  num_docs++;
+  bytes_processed += num;
+  for (pass = 0; pass < MAX_PASSES; pass++)
+    if (Passes & (1 << pass))
+	if (PassData[pass].process((u_char *)buffer, num) == COMPERROR)
+	   FatalError(1, "Error during file processing");
+}
+
+int refill(int in_fd)
+{
+  int num, bitleft;
+  bitleft = data_end - base;
+  memmove(buffer, base, bitleft);
+  if (buf_size - bitleft < MIN_BUF)
+    FatalError(1, "Paragraph too big for buffer");
+  num = read(in_fd, &buffer[bitleft], buf_size - bitleft);
+  line_start -= (base - buffer);
+  scan -= (base - buffer);
+  base = buffer;
+  data_end = buffer + bitleft + num;
+  if (num > 0) return 1; else return 0;
+}
+
+char *scanpara(int in_fd)
+{
+  int num, blank, in_blank, at_end;
+  num = read(in_fd,buffer,buf_size);
+  in_blank = 1;  base = buffer;  line_start = buffer;  scan = buffer;
+  data_end = buffer + num;  at_end = 0;
+  for (;;) {
+    blank = 1;  line_start = scan;                   // Get a line
+    while (scan < data_end  &&  *scan != '\n')
+      if (*scan++ > ' ') blank = 0;
+    if (scan >= data_end) {
+      if (refill(in_fd)) {
+	while (scan < data_end  &&  *scan != '\n')
+	  if (*scan++ > ' ') blank = 0;
+	if (scan < data_end) scan++;
+	}
+      else at_end = 1;
+      }
+    else scan++;
+    if (line_start < scan) {                         // If we have a line
+      if (in_blank) {
+	if (!blank) { in_blank = 0; base = line_start; }
+	}
+      else {
+	if (blank) {
+	  do_process(base, line_start - base);
+	  in_blank = 1;  base = line_start;
+	  }
+	}
+      }
+    if (at_end) break;
+    }
+  if (in_blank) base = scan;
+  if (scan + 2 <= buffer + buf_size) {
+    *scan++ = 26;  *scan++ = 10;
+    }
+  if (base < scan) do_process(base, scan - base);
+  return NULL;
+}
+
+char *scanfile(int in_fd)
+{
+  int num = read(in_fd,buffer,buf_size);  /*expect to read the whole file*/
+  if (num < 0) return "file locked";
+  if (num >= buf_size-1) return "file too long";
+  do_process(buffer, num);
+  return NULL;
+}
+
+void search(char *dname, char *fname)
+{
+  long dirtag;  struct _finddata_t dirinfo;  int in_fd;  char *res;
+  char search_name[path_length], found_name[path_length];
+
+  sprintf(search_name, "%s%s", dname, fname);    /*Scan files*/
+  dirtag = _findfirst(search_name, &dirinfo);
+  if (dirtag >= 0) {
+    do {
+      if ((dirinfo.attrib & (_A_SUBDIR | _A_HIDDEN | _A_SYSTEM)) == 0) {
+	sprintf(found_name,"%s%s",dname,dirinfo.name);
+	in_fd = open(found_name,O_RDONLY|O_BINARY);
+	if (in_fd >= 0) {
+	  if (by_para)
+	    res = scanpara(in_fd);
+	  else
+	    res = scanfile(in_fd);
+	  if (res != NULL) {
+	    Message("Error %s in processing file %s\n", res, found_name);
+	    exit(1);
+	    }
+	  close(in_fd);
+	  }
+	}
+      } while (_findnext(dirtag, &dirinfo) == 0);
+    _findclose(dirtag);
+    }
+
+  if (recurse == 0) return;
+
+  sprintf(search_name, "%s*.*", dname);          /*Look for subdirs*/
+  dirtag = _findfirst(search_name, &dirinfo);
+  if (dirtag < 0) return;
+  do {
+    if ( ((dirinfo.attrib & (_A_HIDDEN | _A_SYSTEM)) == 0)  &&
+	 ((dirinfo.attrib & _A_SUBDIR) != 0)  &&
+	 strcmp(dirinfo.name,".") != 0 &&
+	 strcmp(dirinfo.name,"..") != 0) {
+      sprintf(found_name,"%s%s",dname,dirinfo.name);
+      strcat(found_name,"\\");
+      search(found_name,fname);
+      }
+    } while (_findnext(dirtag, &dirinfo) == 0);
+  _findclose(dirtag);
+}
+
+static int toobig(int n)
+{
+  if (n > path_length) {
+    printf("Cannot handle urls > %d characters in length\n", path_length);
+    exit(1);
+    }
+  return 0;
+}
+
+void scan_catch(char *dname, char *fname)
+{
+  int in_fd, urllen, conlen, filesize;  char catch_name[path_length];
+  int filecount = 0;
+  enum { filekind_redirected, filekind_text, filekind_other } filekind;
+  sprintf(catch_name, "%s%s", dname, fname);
+  in_fd = open(catch_name,O_RDONLY|O_BINARY);
+  if (in_fd < 0)
+    FatalError(1, "Couldn't open catch file \"%s\"", catch_name);
+  for (;;) {
+    filecount++;  if (filecount%100 == 0) { printf("%d\r", filecount);  fflush(stdout); }
+    if (read(in_fd, &urllen, sizeof(int)) != sizeof(int) || toobig(urllen) ||
+	read(in_fd, catch_name, urllen) != urllen ||
+	read(in_fd, &conlen, sizeof(int)) != sizeof(int) || toobig(conlen) ||
+	read(in_fd, catch_name, conlen) != conlen) break;
+    if (conlen >= 1 && *catch_name == '@')
+      filekind = filekind_redirected;
+    else {
+      if (conlen >= 4 && strncmp(catch_name, "text", 4) == 0)
+	filekind = filekind_text;
+      else
+	filekind = filekind_other;
+      if (read(in_fd, &filesize, sizeof(int)) != sizeof(int))
+	FatalError(1, "File read failed for size field");
+      if (filesize > buf_size)
+	FatalError(1, "File too large (%d > %d)", filesize, buf_size);
+      if (read(in_fd, buffer, filesize) != filesize)
+	FatalError(1, "Failed to read file data");
+      }
+    if (filekind == filekind_text) do_process(buffer, filesize);
+    }
+  close(in_fd);
+}
+
+static void driver (FILE * Trace, char *file_name)
+{
+  int pass;
+
+  buffer = (char *)Xmalloc (buf_size);
+
+  for (pass = 0; pass < MAX_PASSES; pass++)
+    if (Passes & (1 << pass))
+      {
+	if (PassData[pass].init (file_name) == COMPERROR)
+	  FatalError (1, "Error during init of \"%s\"",PassData[pass].name);
+      }
+
+  if (html_catch == 0) search(dirname, wildname);
+		  else scan_catch(dirname, wildname);
+
+  for (pass = 0; pass < MAX_PASSES; pass++)
+    if (Passes & (1 << pass))
+      {
+	if (PassData[pass].done (file_name) == COMPERROR)
+	  FatalError (1, "Error during done of \"%s\"", PassData[pass].name);
+      }
+
+
+  free (buffer);
+}
+
+void main (int argc, char **argv)
+{
+  int ch;
+  char *filename = NULL;
+  FILE *Trace = NULL;
+
+  msg_prefix = argv[0];
+
+  opterr = 0;
+  while ((ch = getopt (argc, argv, "hC:WHGpSD123f:d:b:T:I:t:m:N:c:n:s:")) != -1)
+    {
+      switch (ch)
+	{
+	case 'H':
+	  html_catch = 1;
+	  break;
+	case 'G':
+	  SkipSGML = 1;
+	  break;
+	case 'p':
+	  by_para = 1;
+	  break;
+	case 'S':
+	  Passes |= SPECIAL;
+	  break;
+	case '1':
+	  InvfLevel = 1;
+	  break;
+	case '2':
+	  InvfLevel = 2;
+	  break;
+	case '3':
+	  InvfLevel = 3;
+	  break;
+	case 'f':
+	  filename = optarg;
+	  break;
+	case 'n':
+	  trace_name = optarg;
+	  break;
+	case 'D':
+	  Dump = 1;
+	  break;
+	case 'W':
+	  MakeWeights = 1;
+	  break;
+	case 'd':
+	  set_basepath (optarg);
+	  break;
+	case 's':
+	  stem_method = atoi (optarg) & STEMMER_MASK;
+	  break;
+	case 'b':
+	  buf_size = atoi (optarg) * 1024;
+	  break;
+	case 'C':
+	  comp_stat_point = atoi (optarg) * 1024;
+	  break;
+	case 'c':
+	  ChunkLimit = atoi (optarg);
+	  break;
+	case 'm':
+	  invf_buffer_size = (int) (atof (optarg) * 1024 * 1024);
+	  break;
+	case 'I':
+	case 'N': /* N kept for compatability */
+	  if (*optarg == '1')
+	    Passes |= IVF_PASS_1;
+	  else if (*optarg == '2')
+	    Passes |= IVF_PASS_2;
+	  else
+	    usage ("Invalid pass number");
+	  break;
+	case 'T':
+	  if (*optarg == '1')
+	    Passes |= TEXT_PASS_1;
+	  else if (*optarg == '2')
+	    Passes |= TEXT_PASS_2;
+	  else
+	    usage ("Invalid pass number");
+	  break;
+	case 't':
+	  trace = (unsigned long) (atof (optarg) * 1024 * 1024);
+	  break;
+	case 'h':
+	case '?':
+	  usage (NULL);
+	}
+    }
+
+  if (!filename || *filename == '\0')
+    FatalError (1, "A document collection name must be specified.");
+
+  if (buf_size < MIN_BUF)
+    FatalError (1, "The buffer size must exceed 1024 bytes.");
+
+  if ((Passes & (IVF_PASS_1 | IVF_PASS_2)) == (IVF_PASS_1 | IVF_PASS_2))
+    FatalError (1, "I1 and I2 cannot be done simultaneously.");
+
+  if ((Passes & (TEXT_PASS_1 | TEXT_PASS_2)) == (TEXT_PASS_1 | TEXT_PASS_2))
+    FatalError (1, "T1 and T2 cannot be done simultaneously.");
+
+  if (!Passes)
+    FatalError (1, "S, T1, T2, I1 or I2 must be specified.");
+
+  if (argc - optind == 1) {
+    strcpy(dirname,"");
+    strcpy(wildname,argv[optind]);
+    }
+  else if (argc - optind == 2) {
+    strcpy(dirname,argv[optind]);
+    strcpy(wildname,argv[optind+1]);
+    }
+  else FatalError(1, "Finder code requires directory and filespec.");
+
+  if (strrchr(wildname,'*') != NULL  ||  strrchr(wildname,'?') != NULL)
+    recurse = 1;
+
+  if (trace)
+    {
+      if (!trace_name)
+	trace_name = make_name (filename, TRACE_SUFFIX, NULL);
+      if (!(Trace = fopen (trace_name, "a")))
+	Message ("Unable to open \"%s\". No tracing will be done.", trace_name);
+      else
+	setbuf (Trace, NULL);
+    }
+  else
+    Trace = NULL;
+
+  if (comp_stat_point)
+    {
+      char *name = make_name (filename, COMPRESSION_STATS_SUFFIX, NULL);
+      if (!(Comp_Stats = fopen (name, "wb")))
+	Message ("Unable to open \"%s\". No comp. stats. will be generated.",
+		 name);
+    }
+
+
+  if (Trace)
+    {
+      int i;
+      fprintf (Trace, "\n\n\t\t-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\n\n");
+      for (i = 0; i < argc; i++)
+	fprintf (Trace, "%s ", argv[i]);
+      fprintf (Trace, "\n\n");
+    }
+
+  driver (Trace, filename);
+
+  if (Trace)
+    fclose (Trace);
+
+  if (Comp_Stats)
+    fclose (Comp_Stats);
+
+  exit (0);
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/Makefile.in
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/Makefile.in	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/Makefile.in	(revision 699)
@@ -0,0 +1,377 @@
+###########################################################################
+#
+# Makefile.in -- Makefile for the MG system
+# Copyright (C) 1994  Neil Sharman;  1995 Tim Shimmin
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)Makefile.in	1.10 22 Mar 1994
+#
+###########################################################################
+
+PACKAGE = @PACKAGE@
+VERSION = @VERSION@
+ 
+SHELL = /bin/sh
+VPATH = @srcdir@
+exec_prefix = @exec_prefix@
+prefix = @prefix@
+srcdir = @srcdir@
+subdir = src/text
+top_srcdir = @top_srcdir@
+transform = @program_transform_name@
+ 
+AR = ar
+CC = @CC@
+CFLAGS = @CFLAGS@
+CPPFLAGS = @CPPFLAGS@
+DEFS = @DEFS@
+RANLIB = @RANLIB@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+LDFLAGS = @LDFLAGS@
+LIBS = ../../lib/libmg.a @LIBS@
+TOUCH = echo timestamp >
+AWK = @AWK@
+
+##########################################
+# Uncomment for compiling for the nzdl project
+#
+CFLAGS = @CFLAGS@ -DNZDL -DQUIET -DSHORT_SUFFIX -DPARADOCNUM
+##########################################
+
+ 
+bindir = $(exec_prefix)/bin
+datadir = $(prefix)/share
+libexecdir = $(exec_prefix)/libexec
+localedir = $(datadir)/locale
+mandir = $(prefix)/man/man1
+ 
+COMPILE = $(CC) -c $(DEFS) $(TREC_MODE) $(READLINE) $(INCLUDES) $(CPPFLAGS) $(CFLAGS)
+LINK = $(CC) $(LDFLAGS) -o $@
+
+.SUFFIXES:
+.SUFFIXES: .c .o
+.c.o:
+	$(COMPILE) $<
+ 
+
+ANSI2KNR = @ANSI2KNR@
+o = .@U@o
+ 
+.SUFFIXES: ._c ._o
+.c._c:
+	./ansi2knr $< > $*.tmp && mv $*.tmp $@
+._c._o:
+	@echo $(COMPILE) $<
+	@rm -f _$*.c
+	@ln $< _$*.c && $(COMPILE) _$*.c && mv _$*.o $@ && rm _$*.c
+.c._o:
+	./ansi2knr $< > $*.tmp && mv $*.tmp $*._c
+	@echo $(COMPILE) $*._c
+	@rm -f _$*.c
+	@ln $*._c _$*.c && $(COMPILE) _$*.c && mv _$*.o $@ && rm _$*.c
+ 
+INCLUDES = -I../../../.. -I../.. -I../../lib -I.
+
+#############################################################################
+#
+# Uncomment (and possibly edit) the following lines if you wish
+# to use the GNU readline library for the input in mgquery
+# 
+#READLINE = -DGNU_READLINE -I/opt/gnu/include/readline -L/opt/gnu/lib
+#READLINE_LIBS = -lreadline -ltermcap
+#
+#
+#############################################################################
+
+#############################################################################
+#
+# When working with the trec collection this should be uncommented.
+#
+# TREC_MODE = -DTREC_MODE
+#
+#
+#############################################################################
+
+SOURCES = \
+  backend.c               locallib.c              mg_weights_build.c   \
+  bool_optimiser.c        mg.special.c            mgdictlist.c         \
+  bool_parser.c           mg_compression_dict.c   mgquery.c            \
+  bool_query.c            mg_errors.c             mgstat.c             \
+  bool_tester.c           mg_fast_comp_dict.c     mg_decompress_text.c \
+  bool_tree.c             mg_files.c              query.docnums.c      \
+  commands.c              mg_hilite_words.c       query.ranked.c       \
+  comp_dict.c             mg_invf_dict.c          read_line.c          \
+  environment.c           mg_invf_dump.c          stem_search.c        \
+  mg_invf_merge.c         stemmer.c               mg_stem_idx.c        \
+  mg_invf_rebuild.c       term_lists.c            mgstemidxlist.c      \
+  invf_get.c              mg_passes.c             text.pass1.c         \
+  ivf.pass1.c             mg_perf_hash_build.c    text.pass2.c         \
+  ivf.pass2.c             mg_text_estimate.c      text_get.c           \
+  lists.c                 mg_text_merge.c         weights.c            \
+  query_term_list.c       words.c
+
+HEADERS = \
+  backend.h          conditions.h       locallib.h         term_lists.h \
+  bool_optimiser.h   environment.h      mg.h               text.h       \
+  bool_parser.h      globals.h          mg_errors.h        text_get.h   \
+  bool_query.h       hash.h             mg_files.h         warranty.h   \
+  bool_tree.h        help.mg.h          mg_merge.h         weights.h    \
+  build.h            invf.h             read_line.h        words.h      \
+  commands.h         invf_get.h         stem_search.h                   \
+  comp_dict.h        lists.h            stemmer.h                       \
+  query_term_list.h
+
+
+MISC          = Makefile.in help.mg.src bool_parser.y warranty.src conditions.src
+
+ALLFILES      =	$(SOURCES) $(HEADERS) $(MISC) $(MAN)
+
+DISTFILES     = $(ALLFILES) 
+
+EXEC          = mgquery mg_weights_build mgstat mg_invf_dump mg_invf_dict \
+                mg_invf_rebuild mgdictlist mg_passes mg_perf_hash_build \
+                mg_compression_dict mg_text_estimate mg_fast_comp_dict \
+                mg_hilite_words mg_invf_merge mg_text_merge bool_tester \
+                mg_stem_idx mgstemidxlist mg_decompress_text
+
+MAN           = mgquery.1 mg_weights_build.1 mgstat.1 mg_invf_dump.1 mg_invf_dict.1 \
+                mg_invf_rebuild.1 mgdictlist.1 mg_passes.1 mg_perf_hash_build.1 \
+                mg_compression_dict.1 mg_text_estimate.1 mg_fast_comp_dict.1 \
+                mg_hilite_words.1 mg_invf_merge.1 mg_text_merge.1 \
+                mg_stem_idx.1 mgstemidxlist.1 mg_decompress_text.1 
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+
+all:	$(EXEC) libtextin.a
+
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+help.mg.h : help.mg.src
+	sed -e 's/\\/\\\\/g' -e 's/\"/\\\"/g' help.mg.src | $(AWK) '/^#/ { next; };{ if (x == 0) printf("static char *help_str[] = {\n\"%s\\n\"", $$0); else printf(",\n\"%s\\n\"", $$0); x = 1; }; END { printf("};\n"); }' - >help.mg.h
+
+warranty.h : warranty.src
+	sed  -e 's/\\/\\\\/g' -e 's/\"/\\\"/g' warranty.src | $(AWK) '/^#/ { next; };{ if (x == 0) printf("static char *warranty_str[] = {\n\"%s\\n\"", $$0); else printf(",\n\"%s\\n\"", $$0); x = 1; }; END { printf("};\n"); }' - >warranty.h
+
+conditions.h : conditions.src
+	sed -e 's/\\/\\\\/g' -e 's/\"/\\\"/g' conditions.src | $(AWK) '/^#/ { next; };{ if (x == 0) printf("static char *cond_str[] = {\n\"%s\\n\"", $$0); else printf(",\n\"%s\\n\"", $$0); x = 1; }; END { printf("};\n"); }' - >conditions.h
+
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+HILITE_OBJS = mg_hilite_words$o stemmer$o locallib$o words$o
+
+mg_hilite_words: $(HILITE_OBJS)
+	$(LINK) $(HILITE_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+bool_parser.c: bool_parser.y
+	$(YACC) $?
+	mv y.tab.c $@
+
+BOOL_OBJS = bool_tree$o bool_parser$o bool_optimiser$o bool_tester$o \
+            term_lists$o stemmer$o stem_search$o mg_errors$o query_term_list$o \
+	    words$o
+
+bool_tester: $(BOOL_OBJS)
+	$(LINK) $(BOOL_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+QUERY_OBJS = \
+	mgquery$o locallib$o lists$o \
+	query.ranked$o query.docnums$o stem_search$o \
+	environment$o commands$o weights$o text_get$o stemmer$o \
+	read_line$o mg_errors$o backend$o invf_get$o \
+	term_lists$o bool_tree$o bool_parser$o bool_optimiser$o bool_query$o \
+	query_term_list$o words$o
+
+mgquery: $(QUERY_OBJS)	
+	$(LINK) $(READLINE) $(QUERY_OBJS) $(READLINE_LIBS) $(LIBS)
+
+commands$o : help.mg.h warranty.h conditions.h
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+PASSES_OBJS = \
+	mg_passes$o text.pass1$o comp_dict$o stemmer$o \
+	text.pass2$o locallib$o \
+	ivf.pass1$o ivf.pass2$o mg.special$o mg_files$o \
+	words$o
+
+mg_passes: $(PASSES_OBJS)
+	$(LINK) $(PASSES_OBJS) $(LIBS) 
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+WEIGHTS_BUILD_OBJS = mg_weights_build$o mg_files$o
+
+mg_weights_build: $(WEIGHTS_BUILD_OBJS)
+	$(LINK) $(WEIGHTS_BUILD_OBJS) $(LIBS)
+
+COMP_DICT_OBJS = mg_compression_dict$o mg_files$o locallib$o
+
+mg_compression_dict: $(COMP_DICT_OBJS)
+	$(LINK) $(COMP_DICT_OBJS) $(LIBS)
+
+FAST_COMP_OBJS = mg_fast_comp_dict$o mg_files$o locallib$o
+
+mg_fast_comp_dict: $(FAST_COMP_OBJS)
+	$(LINK) $(FAST_COMP_OBJS) $(LIBS)
+
+TEXT_EST_OBJS = mg_text_estimate$o locallib$o comp_dict$o mg_files$o
+
+mg_text_estimate: $(TEXT_EST_OBJS)
+	$(LINK) $(TEXT_EST_OBJS) $(LIBS)
+
+STAT_OBJS = mgstat$o mg_files$o locallib$o
+
+mgstat:	$(STAT_OBJS)
+	$(LINK) $(STAT_OBJS) $(LIBS)
+
+INVF_DUMP_OBJS = mg_invf_dump$o locallib$o mg_files$o
+
+mg_invf_dump: $(INVF_DUMP_OBJS)
+	$(LINK) $(INVF_DUMP_OBJS) $(LIBS)
+
+INVF_DICT_OBJS = mg_invf_dict$o mg_files$o locallib$o
+
+mg_invf_dict: $(INVF_DICT_OBJS)
+	$(LINK) $(INVF_DICT_OBJS) $(LIBS)
+
+INVF_REB_OBJS = mg_invf_rebuild$o locallib$o
+
+mg_invf_rebuild: $(INVF_REB_OBJS)
+	$(LINK) $(INVF_REB_OBJS) $(LIBS)
+
+DICTLIST_OBJS = mgdictlist$o locallib$o
+
+mgdictlist: $(DICTLIST_OBJS)
+	$(LINK) $(DICTLIST_OBJS) $(LIBS)
+
+PERF_HASH_OBJS = mg_perf_hash_build$o mg_files$o
+
+mg_perf_hash_build: $(PERF_HASH_OBJS)
+	$(LINK) $(PERF_HASH_OBJS) $(LIBS)
+
+STEM_IDX_OBJS = \
+                mg_stem_idx$o mg_files$o stemmer$o locallib$o \
+                mg_errors$o term_lists$o
+
+mg_stem_idx: $(STEM_IDX_OBJS)
+	$(LINK) $(STEM_IDX_OBJS) $(LIBS)
+
+STEM_IDX_LIST_OBJS = mgstemidxlist$o mg_files$o
+
+mgstemidxlist: $(STEM_IDX_LIST_OBJS)
+	$(LINK) $(STEM_IDX_LIST_OBJS) $(LIBS)
+
+DECOMPRESS_TEXT_OBJS = \
+                mg_decompress_text.o query_term_list.o mg_files.o mg_errors.o \
+                text_get.o locallib.o backend.o stem_search.o \
+                term_lists.o stemmer.o weights.o invf_get.o lists.o
+
+mg_decompress_text: $(DECOMPRESS_TEXT_OBJS)
+	$(LINK) $(DECOMPRESS_TEXT_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+TEXT_MERGE_OBJS = mg_text_merge$o mg_files$o locallib$o
+
+mg_text_merge: $(TEXT_MERGE_OBJS)
+	$(LINK) $(TEXT_MERGE_OBJS) $(LIBS)
+
+INVF_MERGE_OBJS = mg_invf_merge$o mg_files$o locallib$o
+
+mg_invf_merge: $(INVF_MERGE_OBJS)
+	$(LINK) $(INVF_MERGE_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+ansi2knr: ansi2knr.o
+	$(LINK) ansi2knr.o $(LIBS)
+
+install: execinstall maninstall
+uninstall: execuninstall manuninstall
+
+execinstall: $(EXEC)
+	$(top_srcdir)/mkinstalldirs $(bindir); \
+	for name in $(EXEC); do \
+	  $(INSTALL_PROGRAM) $$name \
+	  $(bindir)/`echo $$name | sed '$(transform)'`; \
+	done
+
+execuninstall:
+	for name in $(EXEC); do \
+	 rm -f $(bindir)/`echo $$name | sed '$(transform)'`; \
+	done
+
+maninstall: $(MAN)
+	$(top_srcdir)/mkinstalldirs $(mandir); \
+	for f in $(MAN); do \
+	  $(INSTALL_DATA) $$f $(mandir)/$$f ;\
+	done
+
+manuninstall: $(MAN)
+	for f in $(MAN); do \
+	  rm -f $(mandir)/$$f ;\
+	done
+
+
+LIB_OBJS = \
+	query_term_list$o \
+	locallib$o lists$o \
+	query.ranked$o query.docnums$o stem_search$o \
+	environment$o commands$o weights$o text_get$o stemmer$o \
+	read_line$o mg_errors$o backend$o invf_get$o \
+	term_lists$o bool_tree$o bool_parser$o bool_optimiser$o \
+	bool_query$o words$o
+
+libtextin.a: $(LIB_OBJS)
+	rm -f libtextin.a
+	$(AR) cru libtextin.a $(LIB_OBJS)
+	$(RANLIB) libtextin.a
+
+
+mostlyclean:
+	rm -f *$o _*.c _*.o *._c *._o core core.*
+ 
+clean: mostlyclean
+	rm -f $(EXEC)
+ 
+distclean: clean
+	rm -f ansi2knr
+	rm -f Makefile
+ 
+maintainer-clean: distclean
+	@echo "This command is intended only for maintainers to use;"
+	@echo "rebuilding the deleted files may require special tools."
+ 
+dist: $(DISTFILES)
+	for file in $(DISTFILES); do \
+	  ln $(srcdir)/$$file $(distdir) 2> /dev/null \
+	  || cp -p $(srcdir)/$$file $(distdir); \
+	done
+ 
+Makefile: Makefile.in ../../../../config.status
+	cd ../../../.. && CONFIG_FILES=$(subdir)/$@ CONFIG_HEADERS= ./config.status
+ 
+# Tell versions [3.59,3.63) of GNU make not to export all variables.
+# Otherwise a system limit (for SysV at least) may be exc
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/WIN32.MAK
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/WIN32.MAK	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/WIN32.MAK	(revision 699)
@@ -0,0 +1,358 @@
+#PACKAGE = mg
+#VERSION = 1.3
+
+#SHELL = /bin/sh
+#exec_prefix = ${prefix}
+#prefix = /image/httpd/mg-1.2
+#srcdir = .
+#subdir = src/text
+#top_srcdir = ../..
+#transform = s,x,x,
+
+CC = cl
+#CC = cl /Z7  this gives symbol and line number debugging (set /DEBUG on link)
+#CFLAGS = -g -O
+CPPFLAGS =
+DEFS = -DNZDL -DQUIET -DSHORT_SUFFIX -DPARADOCNUM -DHAVE_CONFIG_H -D__WIN32__ -D_LITTLE_ENDIAN
+#INSTALL = /usr/local/gnu/bin/install -c
+#INSTALL_DATA = ${INSTALL} -m 644
+#INSTALL_PROGRAM = ${INSTALL}
+LDFLAGS =
+LIBS = ..\..\lib\libmg.lib
+#TOUCH = echo timestamp >
+#AWK = gawk
+
+#bindir = $(exec_prefix)/bin
+#datadir = $(prefix)/share
+#libexecdir = $(exec_prefix)/libexec
+#localedir = $(datadir)/locale
+#mandir = $(prefix)/man/man1
+
+COMPILE = $(CC) -c $(DEFS) -I. -I..\.. -I..\..\lib
+LINK = $(CC) $(LDFLAGS)
+
+.SUFFIXES:
+.SUFFIXES: .c .obj .cpp
+.c.obj:
+	$(COMPILE) $<
+.cpp.obj:
+	$(COMPILE) $<
+
+o = .obj
+e = .exe
+
+#############################################################################
+#
+# Uncomment (and possibly edit) the following lines if you wish
+# to use the GNU readline library for the input in mgquery
+#
+#READLINE = -DGNU_READLINE -I/opt/gnu/include/readline -L/opt/gnu/lib
+#READLINE_LIBS = -lreadline -ltermcap
+#
+#
+#############################################################################
+
+#############################################################################
+#
+# When working with the trec collection this should be uncommented.
+#
+# TREC_MODE = -DTREC_MODE
+#
+#
+#############################################################################
+
+SOURCES = \
+  backend.c               locallib.c              mg_weights_build.c   \
+  bool_optimiser.c        mg.special.c            mgdictlist.c         \
+  bool_parser.c           mg_compression_dict.c   mgquery.c            \
+  bool_query.c            mg_errors.c             mgstat.c             \
+  bool_tester.c           mg_fast_comp_dict.c     mg_decompress_text.c \
+  bool_tree.c             mg_files.c              query.docnums.c      \
+  commands.c              mg_hilite_words.c       query.ranked.c       \
+  comp_dict.c             mg_invf_dict.c          read_line.c          \
+  environment.c           mg_invf_dump.c          stem_search.c        \
+  mg_invf_merge.c         stemmer.c               mg_stem_idx.c        \
+  mg_invf_rebuild.c       term_lists.c            mgstemidxlist.c      \
+  invf_get.c              mg_passes.c             text.pass1.c         \
+  ivf.pass1.c             mg_perf_hash_build.c    text.pass2.c         \
+  ivf.pass2.c             mg_text_estimate.c      text_get.c           \
+  lists.c                 mg_text_merge.c         weights.c            \
+  mgpass.c                query_term_list.c       words.c
+
+HEADERS = \
+  backend.h          conditions.h       locallib.h         term_lists.h \
+  bool_optimiser.h   environment.h      mg.h               text.h       \
+  bool_parser.h      globals.h          mg_errors.h        text_get.h   \
+  bool_query.h       hash.h             mg_files.h         warranty.h   \
+  bool_tree.h        help.mg.h          mg_merge.h         weights.h    \
+  build.h            invf.h             read_line.h        words.h      \
+  commands.h         invf_get.h         stem_search.h                   \
+  comp_dict.h        lists.h            stemmer.h          query_term_list.h
+
+
+#MISC          = Makefile.in help.mg.src bool_parser.y warranty.src conditions.src
+
+ALLFILES      = $(SOURCES) $(HEADERS) $(MISC) $(MAN)
+
+DISTFILES     = $(ALLFILES)
+
+EXEC          = mgquery$e mg_weights_build$e mgstat$e \
+		mg_invf_dump$e     mg_invf_dict$e \
+		mg_invf_rebuild$e     mgdictlist$e \
+		mg_passes$e     mg_perf_hash_build$e \
+		mg_compression_dict$e     mg_text_estimate$e \
+		mg_fast_comp_dict$e \
+		mg_hilite_words$e     mg_invf_merge$e \
+		mg_text_merge$e     bool_tester$e  mgpass$e \
+		mg_stem_idx$e  mgstemidxlist$e \
+                mg_decompress_text$e
+
+MAN           = mgquery.1 mg_weights_build.1 mgstat.1 mg_invf_dump.1 mg_invf_dict.1 \
+		mg_invf_rebuild.1 mgdictlist.1 mg_passes.1 mg_perf_hash_build.1 \
+		mg_compression_dict.1 mg_text_estimate.1 mg_fast_comp_dict.1 \
+		mg_hilite_words.1 mg_invf_merge.1 mg_text_merge.1 \
+		mg_stem_idx.1 mgstemidxlist.1 mg_decompress_text.1
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+
+all:    $(EXEC) libtextin.lib
+
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+#help.mg.h : help.mg.src
+#        sed -e 's/\\/\\\\/g' -e 's/\"/\\\"/g' help.mg.src | $(AWK) '/^#/ { next; };{ if (x == 0) printf("static char *help_str[] = {\n\"%s\\n\"", $$0); else printf(",\n\"%s\\n\"", $$0); x = 1; }; END { printf("};\n"); }' - >help.mg.h
+
+#warranty.h : warranty.src
+#        sed  -e 's/\\/\\\\/g' -e 's/\"/\\\"/g' warranty.src | $(AWK) '/^#/ { next; };{ if (x == 0) printf("static char *warranty_str[] = {\n\"%s\\n\"", $$0); else printf(",\n\"%s\\n\"", $$0); x = 1; }; END { printf("};\n"); }' - >warranty.h
+
+#conditions.h : conditions.src
+#        sed -e 's/\\/\\\\/g' -e 's/\"/\\\"/g' conditions.src | $(AWK) '/^#/ { next; };{ if (x == 0) printf("static char *cond_str[] = {\n\"%s\\n\"", $$0); else printf(",\n\"%s\\n\"", $$0); x = 1; }; END { printf("};\n"); }' - >conditions.h
+
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+HILITE_OBJS = mg_hilite_words$o stemmer$o locallib$o words$o
+
+mg_hilite_words$e: $(HILITE_OBJS)
+	$(LINK) $(HILITE_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+#bool_parser.c: bool_parser.y
+#        $(YACC) $?
+#        mv y.tab.c $@
+#
+BOOL_OBJS = bool_tree$o bool_parser$o bool_optimiser$o bool_tester$o \
+	    term_lists$o stemmer$o stem_search$o mg_errors$o query_term_list$o \
+	    words$o
+
+bool_tester$e: $(BOOL_OBJS)
+	$(LINK) $(BOOL_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+QUERY_OBJS = \
+	mgquery$o locallib$o lists$o \
+	query.ranked$o query.docnums$o stem_search$o \
+	environment$o commands$o weights$o text_get$o stemmer$o \
+	read_line$o mg_errors$o backend$o invf_get$o \
+	term_lists$o bool_tree$o bool_parser$o bool_optimiser$o bool_query$o \
+	query_term_list$o words$o
+
+mgquery$e: $(QUERY_OBJS)
+	$(LINK) $(READLINE) $(QUERY_OBJS) $(READLINE_LIBS) $(LIBS)
+
+commands$o : help.mg.h warranty.h conditions.h
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+PASSES_OBJS = \
+	mg_passes$o text.pass1$o comp_dict$o stemmer$o \
+	text.pass2$o locallib$o \
+	ivf.pass1$o ivf.pass2$o mg.special$o mg_files$o \
+	words$o
+
+mg_passes$e: $(PASSES_OBJS)
+	$(LINK) $(PASSES_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+PASS_OBJS = \
+	words$o mgpass$o text.pass1$o comp_dict$o stemmer$o \
+	text.pass2$o locallib$o \
+	ivf.pass1$o ivf.pass2$o mg.special$o mg_files$o
+
+#mgpass$o:  mgpass.cpp
+#	$(COMPILE) mgpass.cpp
+
+mgpass$e: $(PASS_OBJS)
+	$(LINK) $(PASS_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+LINK32_FLAGS=kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib\
+ advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib\
+ odbccp32.lib
+
+BUILDER_OBJS = \
+	builder$o
+
+builder$e: $(BUILDER_OBJS)
+	$(LINK) $(BUILDER_OBJS) $(LIBS) $(LINK32_FLAGS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+WEIGHTS_BUILD_OBJS = mg_weights_build$o mg_files$o
+
+mg_weights_build$e: $(WEIGHTS_BUILD_OBJS)
+	$(LINK) $(WEIGHTS_BUILD_OBJS) $(LIBS)
+
+COMP_DICT_OBJS = mg_compression_dict$o mg_files$o locallib$o
+
+mg_compression_dict$e: $(COMP_DICT_OBJS)
+	$(LINK) $(COMP_DICT_OBJS) $(LIBS)
+
+FAST_COMP_OBJS = mg_fast_comp_dict$o mg_files$o locallib$o
+
+mg_fast_comp_dict$e: $(FAST_COMP_OBJS)
+	$(LINK) $(FAST_COMP_OBJS) $(LIBS)
+
+TEXT_EST_OBJS = mg_text_estimate$o locallib$o comp_dict$o mg_files$o
+
+mg_text_estimate$e: $(TEXT_EST_OBJS)
+	$(LINK) $(TEXT_EST_OBJS) $(LIBS)
+
+STAT_OBJS = mgstat$o mg_files$o locallib$o
+
+mgstat$e: $(STAT_OBJS)
+	$(LINK) $(STAT_OBJS) $(LIBS)
+
+INVF_DUMP_OBJS = mg_invf_dump$o locallib$o mg_files$o
+
+mg_invf_dump$e: $(INVF_DUMP_OBJS)
+	$(LINK) $(INVF_DUMP_OBJS) $(LIBS)
+
+INVF_DICT_OBJS = mg_invf_dict$o mg_files$o locallib$o
+
+mg_invf_dict$e: $(INVF_DICT_OBJS)
+	$(LINK) $(INVF_DICT_OBJS) $(LIBS)
+
+INVF_REB_OBJS = mg_invf_rebuild$o locallib$o
+
+mg_invf_rebuild$e: $(INVF_REB_OBJS)
+	$(LINK) $(INVF_REB_OBJS) $(LIBS)
+
+DICTLIST_OBJS = mgdictlist$o locallib$o
+
+mgdictlist$e: $(DICTLIST_OBJS)
+	$(LINK) $(DICTLIST_OBJS) $(LIBS)
+
+PERF_HASH_OBJS = mg_perf_hash_build$o mg_files$o
+
+mg_perf_hash_build$e: $(PERF_HASH_OBJS)
+	$(LINK) $(PERF_HASH_OBJS) $(LIBS)
+
+STEM_IDX_OBJS = mg_stem_idx$o mg_files$o stemmer$o locallib$o \
+                mg_errors$o term_lists$o
+
+mg_stem_idx$e: $(STEM_IDX_OBJS)
+	$(LINK) $(STEM_IDX_OBJS) $(LIBS)
+
+STEM_IDX_LIST_OBJS = mgstemidxlist$o mg_files$o
+
+mgstemidxlist$e: $(STEM_IDX_LIST_OBJS)
+	$(LINK) $(STEM_IDX_LIST_OBJS) $(LIBS)
+
+DECOMPRESS_TEXT_OBJS = mg_decompress_text$o query_term_list$o mg_files$o mg_errors$o \
+                       text_get$o locallib$o backend$o stem_search$o \
+                       term_lists$o stemmer$o weights$o invf_get$o lists$o
+
+mg_decompress_text$e: $(DECOMPRESS_TEXT_OBJS)
+	$(LINK) $(DECOMPRESS_TEXT_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+TEXT_MERGE_OBJS = mg_text_merge$o mg_files$o locallib$o
+
+mg_text_merge$e: $(TEXT_MERGE_OBJS)
+	$(LINK) $(TEXT_MERGE_OBJS) $(LIBS)
+
+INVF_MERGE_OBJS = mg_invf_merge$o mg_files$o locallib$o
+
+mg_invf_merge$e: $(INVF_MERGE_OBJS)
+	$(LINK) $(INVF_MERGE_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+#ansi2knr: ansi2knr.o
+#        $(LINK) ansi2knr.o $(LIBS)
+#
+install: execinstall maninstall
+uninstall: execuninstall manuninstall
+
+#execinstall: $(EXEC)
+#        $(top_srcdir)/mkinstalldirs $(bindir); \
+#        for name in $(EXEC); do \
+#          $(INSTALL_PROGRAM) $$name \
+#          $(bindir)/`echo $$name | sed '$(transform)'`; \
+#        done
+
+#execuninstall:
+#        for name in $(EXEC); do \
+#         rm -f $(bindir)/`echo $$name | sed '$(transform)'`; \
+#        done
+
+maninstall: $(MAN)
+	$(top_srcdir)/mkinstalldirs $(mandir); \
+	for f in $(MAN); do \
+	  $(INSTALL_DATA) $$f $(mandir)/$$f ;\
+	done
+
+manuninstall: $(MAN)
+	for f in $(MAN); do \
+	  rm -f $(mandir)/$$f ;\
+	done
+
+
+LIB_OBJS = \
+	query_term_list$o \
+	locallib$o lists$o \
+	query.ranked$o query.docnums$o stem_search$o \
+	environment$o commands$o weights$o text_get$o stemmer$o \
+	read_line$o mg_errors$o backend$o invf_get$o \
+	term_lists$o bool_tree$o bool_parser$o bool_optimiser$o \
+	bool_query$o words$o
+
+libtextin.lib: $(LIB_OBJS)
+	del libtextin.lib
+	lib /out:libtextin.lib $(LIB_OBJS)
+
+
+mostlyclean:
+	rm -f *$o _*.c _*.o *._c *._o core core.*
+ 
+clean: mostlyclean
+	rm -f $(EXEC)
+ 
+distclean: clean
+	rm -f ansi2knr
+	rm -f Makefile
+ 
+maintainer-clean: distclean
+	@echo "This command is intended only for maintainers to use;"
+	@echo "rebuilding the deleted files may require special tools."
+ 
+#dist: $(DISTFILES)
+#        for file in $(DISTFILES); do \
+#          ln $(srcdir)/$$file $(distdir) 2> /dev/null \
+#          || cp -p $(srcdir)/$$file $(distdir); \
+#        done
+ 
+#Makefile: Makefile.in ../../config.status
+#        cd ../.. && CONFIG_FILES=$(subdir)/$@ CONFIG_HEADERS= ./config.status
+ 
+# Tell versions [3.59,3.63) of GNU make not to export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+#.NOEXPORT:
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/backend.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/backend.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/backend.c	(revision 699)
@@ -0,0 +1,838 @@
+/**************************************************************************
+ *
+ * backend.c -- Underlying routines for mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "timing.h"
+#include "filestats.h"
+#include "sptree.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+
+#include "mg_files.h"
+#include "mg.h"
+#include "invf.h"
+#include "text.h"
+#include "lists.h"
+#include "backend.h"
+#include "stem_search.h"
+#include "invf_get.h"
+#include "text_get.h"
+#include "weights.h"
+#include "locallib.h"
+#include "mg_errors.h"
+
+
+static File *
+OpenFile (char *base, char *suffix, unsigned long magic, int *ok)
+{
+  char FileName[512];
+  File *F;
+  sprintf (FileName, "%s%s", base, suffix);
+  if (!(F = Fopen (FileName, "rb", 0)))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      mg_errno = MG_NOFILE;
+      MgErrorData (FileName);
+      if (ok)
+	*ok = 0;
+      return (NULL);
+    }
+  if (magic)
+    {
+      unsigned long m;
+      if (fread ((char *) &m, sizeof (m), 1, F->f) == 0)
+	{
+	  mg_errno = MG_READERR;
+	  MgErrorData (FileName);
+	  if (ok)
+	    *ok = 0;
+	  Fclose (F);
+	  return (NULL);
+	}
+      NTOHUL(m);  /* [RPAP - Jan 97: Endian Ordering] */
+      if (m != magic)
+	{
+	  mg_errno = MG_BADMAGIC;
+	  MgErrorData (FileName);
+	  if (ok)
+	    *ok = 0;
+	  Fclose (F);
+	  return (NULL);
+	}
+    }
+  return (F);
+}
+
+
+static int 
+open_all_files (query_data * qd)
+{
+  int ok = 1;
+
+  qd->File_text = OpenFile (qd->textpathname, TEXT_SUFFIX, /* [RJM 06/97: text filename] */
+			    MAGIC_TEXT, &ok);
+  qd->File_fast_comp_dict = OpenFile (qd->textpathname, /* [RJM 06/97: text filename] */
+				      TEXT_DICT_FAST_SUFFIX, MAGIC_FAST_DICT, NULL);
+  if (!qd->File_fast_comp_dict)
+    {
+      qd->File_comp_dict = OpenFile (qd->textpathname, /* [RJM 06/97: text filename] */
+				     TEXT_DICT_SUFFIX, MAGIC_DICT, &ok);
+      qd->File_aux_dict = OpenFile (qd->textpathname, /* [RJM 06/97: text filename] */
+				    TEXT_DICT_AUX_SUFFIX, MAGIC_AUX_DICT, NULL);
+    }
+  else
+    qd->File_comp_dict = qd->File_aux_dict = NULL;
+
+  qd->File_stem = OpenFile (qd->pathname, INVF_DICT_BLOCKED_SUFFIX,
+			    MAGIC_STEM, &ok);
+
+  /* [RPAP - Jan 97: Stem Index Change]
+     These will fail if collection not built with stem indexes */
+  qd->File_stem1 = OpenFile (qd->pathname, INVF_DICT_BLOCKED_1_SUFFIX,
+			     MAGIC_STEM_1, NULL);
+  qd->File_stem2 = OpenFile (qd->pathname, INVF_DICT_BLOCKED_2_SUFFIX,
+			     MAGIC_STEM_2, NULL);
+  qd->File_stem3 = OpenFile (qd->pathname, INVF_DICT_BLOCKED_3_SUFFIX,
+			     MAGIC_STEM_3, NULL);
+
+  qd->File_invf = OpenFile (qd->pathname, INVF_SUFFIX,
+			    MAGIC_INVF, &ok);
+
+  /* These will fail if a level 1 inverted file was created because there 
+     will be no document weights */
+  qd->File_text_idx_wgt = OpenFile (qd->pathname, TEXT_IDX_WGT_SUFFIX,
+				    MAGIC_TEXI_WGT, NULL);
+  qd->File_weight_approx = OpenFile (qd->pathname, APPROX_WEIGHTS_SUFFIX,
+				     MAGIC_WGHT_APPROX, NULL);
+  if (qd->File_text_idx_wgt == NULL && qd->File_weight_approx == NULL)
+    qd->File_text_idx = OpenFile (qd->textpathname, /* [RJM 06/97: text filename] */
+				  TEXT_IDX_SUFFIX, MAGIC_TEXI, NULL);
+  else
+    qd->File_text_idx = NULL;
+
+
+  if (!ok)
+    {
+      Fclose (qd->File_text);
+      if (qd->File_fast_comp_dict)
+	Fclose (qd->File_fast_comp_dict);
+      if (qd->File_comp_dict)
+	Fclose (qd->File_comp_dict);
+      Fclose (qd->File_stem);
+
+      /* [RPAP - Jan 97: Stem Index Change] */
+      if (qd->File_stem1)
+	Fclose (qd->File_stem1);
+      if (qd->File_stem2)
+	Fclose (qd->File_stem2);
+      if (qd->File_stem3)
+	Fclose (qd->File_stem3);
+
+      Fclose (qd->File_invf);
+      if (qd->File_text_idx_wgt)
+	Fclose (qd->File_text_idx_wgt);
+      if (qd->File_weight_approx)
+	Fclose (qd->File_weight_approx);
+      if (qd->File_text_idx)
+	Fclose (qd->File_text_idx);
+      return (-1);
+    }
+  return (0);
+
+}
+
+static void 
+close_all_files (query_data * qd)
+{
+  Fclose (qd->File_text);
+  if (qd->File_fast_comp_dict)
+    Fclose (qd->File_fast_comp_dict);
+  if (qd->File_aux_dict)
+    Fclose (qd->File_aux_dict);
+  if (qd->File_comp_dict)
+    Fclose (qd->File_comp_dict);
+  Fclose (qd->File_stem);
+
+  /* [RPAP - Jan 97: Stem Index Change] */
+  if (qd->File_stem1)
+    Fclose (qd->File_stem1);
+  if (qd->File_stem2)
+    Fclose (qd->File_stem2);
+  if (qd->File_stem3)
+    Fclose (qd->File_stem3);
+
+  Fclose (qd->File_invf);
+  if (qd->File_text_idx_wgt)
+    Fclose (qd->File_text_idx_wgt);
+  if (qd->File_weight_approx)
+    Fclose (qd->File_weight_approx);
+  if (qd->File_text_idx)
+    Fclose (qd->File_text_idx);
+}
+
+/* If textname is equal to null then name will be used instead */
+/* [RJM 06/97: text filename] */
+query_data *
+InitQuerySystem (char *dir, char *name, char *textname, InitQueryTimes * iqt)
+{
+  query_data *qd;
+  char *s;
+
+  if (textname == NULL) textname = name; /* [RJM 06/97: text filename] */
+
+  if (!(qd = Xmalloc (sizeof (query_data))))
+    {
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  bzero ((char *) qd, sizeof (*qd));
+
+  qd->mem_in_use = qd->max_mem_in_use = 0;
+
+  qd->doc_pos = qd->buf_in_use = 0;
+  qd->TextBufferLen = 0;
+  qd->DL = NULL;
+
+  /* [RPAP - Feb 97: Term Frequency] */
+  qd->TL = NULL;
+  qd->QTL = NULL;
+
+  qd->TextBuffer = NULL;
+
+  qd->tot_hops_taken = 0;
+  qd->tot_num_of_ptrs = 0;
+  qd->tot_num_of_accum = 0;
+  qd->tot_num_of_terms = 0;
+  qd->tot_num_of_ans = 0;
+  qd->tot_text_idx_lookups = 0;
+
+  qd->hops_taken = 0;
+  qd->num_of_ptrs = 0;
+  qd->num_of_accum = 0;
+  qd->num_of_terms = 0;
+  qd->num_of_ans = 0;
+  qd->text_idx_lookups = 0;
+
+  qd->pathname = NULL; /* RJM 06/97: text filename] */
+  qd->textpathname = NULL; /* RJM 06/97: text filename] */
+
+  s = strrchr (dir, '/');
+  if (s && *(s + 1) == '\0')
+    {
+      /* [RJM 06/97: text filename] */
+      if (!(qd->pathname = Xmalloc (strlen (dir) + strlen (name) + 1)) ||
+	  !(qd->textpathname = Xmalloc (strlen (dir) + strlen (textname) + 1)))
+	{
+	  mg_errno = MG_NOMEM;
+	  if (qd->pathname) Xfree (qd->pathname); /* [RJM 06/97: text filename] */
+	  Xfree (qd);
+	  return (NULL);
+	}
+      sprintf (qd->pathname, "%s%s", dir, name);
+      sprintf (qd->textpathname, "%s%s", dir, textname); /* [RJM 06/97: text filename] */
+    }
+
+  else
+    {
+      /* [RJM 06/97: text filename] */
+      if (!(qd->pathname = Xmalloc (strlen (dir) + strlen (name) + 2)) ||
+	  !(qd->textpathname = Xmalloc (strlen (dir) + strlen (textname) + 2)))
+	{
+	  mg_errno = MG_NOMEM;
+	  if (qd->pathname) Xfree (qd->pathname); /* [RJM 06/97: text filename] */
+	  Xfree (qd);
+	  return (NULL);
+	}
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+      if (dir == NULL || dir[0] == '\0') {
+	sprintf (qd->pathname, "%s", name);
+	sprintf (qd->textpathname, "%s", textname); /* [RJM 06/97: text filename] */
+      } else {
+	sprintf (qd->pathname, "%s%s", dir, name);
+	sprintf (qd->textpathname, "%s%s", dir, textname); /* [RJM 06/97: text filename] */
+      }
+#else
+      sprintf (qd->pathname, "%s/%s", dir, name);
+      sprintf (qd->textpathname, "%s/%s", dir, textname); /* [RJM 06/97: text filename] */
+#endif
+    }
+
+  if (open_all_files (qd) == -1)
+    {
+      Xfree (qd->pathname);
+      Xfree (qd->textpathname); /* [RJM 06/97: text filename] */
+      Xfree (qd);
+      return (NULL);
+    }
+
+  if (iqt)
+    GetTime (&iqt->Start);
+
+  /* Initialise the stemmed dictionary system */
+  if (!(qd->sd = ReadStemDictBlk (qd->File_stem)))
+    {
+      close_all_files (qd);
+      Xfree (qd->pathname);
+      Xfree (qd->textpathname); /* [RJM 06/97: text filename] */
+      Xfree (qd);
+      return (NULL);
+    }
+
+  /* [RPAP - Jan 97: Stem Index Change] */
+  if ((qd->sd->sdh.indexed & 7) && qd->File_stem1 && qd->File_stem2 && qd->File_stem3)
+    {
+      if (!(qd->sd->stem1 = ReadStemIdxBlk (qd->File_stem1)))
+	{
+	  FreeStemDict (qd->sd);
+	  close_all_files (qd);
+	  Xfree (qd->pathname);
+	  Xfree (qd->textpathname); /* [RJM 06/97: text filename] */
+	  Xfree (qd);
+	  return (NULL);
+	}
+      if (!(qd->sd->stem2 = ReadStemIdxBlk (qd->File_stem2)))
+	{
+	  FreeStemDict (qd->sd);
+	  close_all_files (qd);
+	  Xfree (qd->pathname);
+	  Xfree (qd->textpathname); /* [RJM 06/97: text filename] */
+	  Xfree (qd);
+	  return (NULL);
+	}
+      if (!(qd->sd->stem3 = ReadStemIdxBlk (qd->File_stem3)))
+	{
+	  FreeStemDict (qd->sd);
+	  close_all_files (qd);
+	  Xfree (qd->pathname);
+	  Xfree (qd->textpathname); /* [RJM 06/97: text filename] */
+	  Xfree (qd);
+	  return (NULL);
+	}
+      }
+  else if (qd->sd->sdh.indexed != 0)
+    {
+      FreeStemDict (qd->sd);
+      close_all_files (qd);
+      Xfree (qd->pathname);
+      Xfree (qd->textpathname); /* [RJM 06/97: text filename] */
+      Xfree (qd);
+      return (NULL);
+    }
+  else
+    {
+      if (qd->File_stem1)
+	Fclose (qd->File_stem1);
+      if (qd->File_stem2)
+	Fclose (qd->File_stem2);
+      if (qd->File_stem3)
+	Fclose (qd->File_stem3);
+      qd->File_stem1 = NULL;
+      qd->File_stem2 = NULL;
+      qd->File_stem3 = NULL;
+      qd->sd->stem1 = NULL;
+      qd->sd->stem2 = NULL;
+      qd->sd->stem3 = NULL;
+    }
+
+  if (iqt)
+    GetTime (&iqt->StemDict);
+  if (qd->File_weight_approx)
+    {
+      if (!(qd->awd = LoadDocWeights (qd->File_weight_approx,
+				      qd->sd->sdh.num_of_docs)))
+	{
+	  FreeStemDict (qd->sd);
+	  close_all_files (qd);
+	  Xfree (qd->pathname);
+	  Xfree (qd->textpathname); /* [RJM 06/97: text filename] */
+	  Xfree (qd);
+	  return (NULL);
+	}
+    }
+  else
+    qd->awd = NULL;
+
+
+  if (iqt)
+    GetTime (&iqt->ApproxWeights);
+
+  if (!(qd->cd = LoadCompDict (qd->File_comp_dict, qd->File_aux_dict,
+			       qd->File_fast_comp_dict)))
+    {
+      if (qd->awd)
+	FreeWeights (qd->awd);
+      FreeStemDict (qd->sd);
+      close_all_files (qd);
+      Xfree (qd->pathname);
+      Xfree (qd->textpathname); /* [RJM 06/97: text filename] */
+      Xfree (qd);
+      return (NULL);
+    }
+
+  if (iqt)
+    GetTime (&iqt->CompDict);
+
+  if (!(qd->id = InitInvfFile (qd->File_invf, qd->sd)))
+    {
+      FreeCompDict (qd->cd);
+      if (qd->awd)
+	FreeWeights (qd->awd);
+      FreeStemDict (qd->sd);
+      close_all_files (qd);
+      Xfree (qd->pathname);
+      Xfree (qd->textpathname); /* [RJM 06/97: text filename] */
+      Xfree (qd);
+      return (NULL);
+    }
+  if ((qd->File_text_idx_wgt == NULL || qd->File_weight_approx == NULL) &&
+      qd->id->ifh.InvfLevel >= 2)
+    {
+      FreeInvfData (qd->id);
+      FreeCompDict (qd->cd);
+      if (qd->awd)
+	FreeWeights (qd->awd);
+      FreeStemDict (qd->sd);
+      close_all_files (qd);
+      Xfree (qd->pathname);
+      Xfree (qd->textpathname); /* [RJM 06/97: text filename] */
+      Xfree (qd);
+      mg_errno = MG_INVERSION;
+      return (NULL);
+    }
+  if (iqt)
+    GetTime (&iqt->Invf);
+
+  if (!(qd->td = LoadTextData (qd->File_text, qd->File_text_idx_wgt,
+			       qd->File_text_idx)))
+    {
+      FreeInvfData (qd->id);
+      FreeCompDict (qd->cd);
+      if (qd->awd)
+	FreeWeights (qd->awd);
+      FreeStemDict (qd->sd);
+      close_all_files (qd);
+      Xfree (qd->pathname);
+      Xfree (qd->textpathname); /* [RJM 06/97: text filename] */
+      Xfree (qd);
+      return (NULL);
+    }
+
+/* [RPAP - Feb 97: NZDL Additions] */
+#if defined(PARADOCNUM) || defined(NZDL)
+
+/*
+
+This code is based on the TREC_MODE code below to read the .paragraph
+file to determine what document numbers correspond to what paragraphs.
+This code is more space efficient, reading in the .paragraph file
+into memory as an accumulate docnum array.  Eg.  the .paragraph may contain
+
+	[5 3 6 4 7 9 4]
+
+indicating the first document has 5 paragraphs, the next 3, etc.
+This will be stored in memory as
+
+	[0 5 8 14 18 25 34 38]
+
+so a binary search can be performed.  The first 0 is for convenience;
+it prevents testing boundary conditions.
+
+
+The TREC_MODE code does this differently; it stores the array
+
+	[1 1 1 1 1 2 2 2 3 3 3 3 3 3 ....]
+
+allowing directy paragraph to docnum conversion, at the expense
+of memory.
+
+*/ 
+  qd->paragraph = NULL;
+
+  if (qd->id->ifh.InvfLevel == 3)
+    {
+      unsigned long magic;
+      FILE *paragraph;
+      int i;
+      char paraFile[512];
+
+      sprintf(paraFile, "%s%s", qd->pathname, INVF_PARAGRAPH_SUFFIX);
+      paragraph = fopen(paraFile, "rb");
+      if (!paragraph)
+	FatalError(1, "Unable to open 'paraFile'.", paraFile);
+
+      fread((void *)&magic, sizeof(magic), 1, paragraph);
+      qd->paragraph = Xmalloc((qd->td->cth.num_of_docs+1)*sizeof(int));
+      qd->paragraph[0] = 0;
+      for (i = 1; i <= qd->td->cth.num_of_docs; i++)
+	{
+	  int count;
+
+	  if (fread((void *)&count, sizeof(count), 1, paragraph) != 1)
+	    FatalError(1, "Unexpected EOF while reading '%s'.", paraFile);
+	  NTOHSI(count);  /* [RPAP - Jan 97: Endian Ordering] */
+	  qd->paragraph[i] = qd->paragraph[i-1]+count;
+	}
+
+      fclose (paragraph); /* [RJM 07/98: Memory Leak] */
+    }
+  
+  
+#endif
+
+#ifdef TREC_MODE
+  {
+    extern char *trec_ids;
+    extern long *trec_paras;
+    int size;
+    char FileName[512];
+    FILE *f;
+    if (!strstr (qd->pathname, "trec"))
+      goto error;
+    sprintf (FileName, "%s%s", qd->pathname, ".DOCIDS");
+    if (!(f = fopen (FileName, "rb")))  /* [RPAP - Feb 97: WIN32 Port] */
+      {
+	Message ("Unable to open \"%s\"", FileName);
+	goto error;
+      }
+    fseek (f, 0, 2);
+    size = ftell (f);
+    fseek (f, 0, 0);
+    trec_ids = Xmalloc (size);
+    if (!trec_ids)
+      {
+	fclose (f);
+	goto error;
+      }
+    fread (trec_ids, 1, size, f);
+    fclose (f);
+    if (qd->id->ifh.InvfLevel == 3)
+      {
+	int i, d;
+	unsigned long magic;
+	trec_paras = Xmalloc (qd->sd->sdh.num_of_docs * sizeof (long));
+	if (!trec_paras)
+	  {
+	    Xfree (trec_ids);
+	    trec_ids = NULL;
+	    goto error;
+	  }
+	sprintf (FileName, "%s%s", qd->pathname, INVF_PARAGRAPH_SUFFIX);
+	if (!(f = fopen (FileName, "rb")))  /* [RPAP - Feb 97: WIN32 Port] */
+	  {
+	    Message ("Unable to open \"%s\"", FileName);
+	    goto error;
+	  }
+	if (fread ((char *) &magic, sizeof (magic), 1, f) != 1 ||
+	    NTOHUL(magic) != MAGIC_PARAGRAPH)  /* [RPAP - Jan 97: Endian Ordering] */
+	  {
+	    fclose (f);
+	    Message ("Bad magic number in \"%s\"", FileName);
+	    goto error;
+	  }
+
+	for (d = i = 0; i < qd->td->cth.num_of_docs; i++)
+	  {
+	    int count;
+	    if (fread ((char *) &count, sizeof (count), 1, f) != 1)
+	      {
+		fclose (f);
+		goto error;
+	      }
+	    NTOHSI(count);  /* [RPAP - Jan 97: Endian Ordering] */
+	    while (count--)
+	      trec_paras[d++] = i;
+	  }
+	fclose (f);
+      }
+    goto ok;
+  error:
+    if (trec_ids)
+      Xfree (trec_ids);
+    if (trec_paras)
+      Xfree (trec_paras);
+    trec_ids = NULL;
+    trec_paras = NULL;
+  ok:
+    ;
+  }
+#endif
+
+  if (iqt)
+    GetTime (&iqt->Text);
+
+  return (qd);
+}
+
+
+
+
+
+
+/*
+ * Change the amount of memory currently in use
+ *
+ */
+void 
+ChangeMemInUse (query_data * qd, long delta)
+{
+  qd->mem_in_use += delta;
+  if (qd->mem_in_use > qd->max_mem_in_use)
+    qd->max_mem_in_use = qd->mem_in_use;
+}
+
+
+void 
+FinishQuerySystem (query_data * qd)
+{
+/* [RJM 07/98: Memory Leak] */
+#if defined(PARADOCNUM) || defined(NZDL)
+  if (qd->paragraph != NULL) {
+    Xfree (qd->paragraph);
+    qd->paragraph = NULL;
+  }
+#endif
+
+  FreeTextData (qd->td);
+  FreeInvfData (qd->id);
+  FreeCompDict (qd->cd);
+  if (qd->awd)
+    FreeWeights (qd->awd);
+  FreeStemDict (qd->sd);
+  close_all_files (qd);
+  Xfree (qd->textpathname); /* [RJM 06/97: text filename] */
+  Xfree (qd->pathname);
+  FreeQueryDocs (qd);
+  if (qd->TL != NULL) FreeTermList(&qd->TL);         /* [RJM 07/98: Memory Leak] */
+  if (qd->QTL != NULL) FreeQueryTermList(&qd->QTL);  /* [RJM 07/98: Memory Leak] */
+  Xfree (qd);
+
+  /* other global stuff hanging around */
+  MgErrorDeinit ();
+}
+
+
+void 
+ResetFileStats (query_data * qd)
+{
+  ZeroFileStats (qd->File_text);
+  if (qd->File_comp_dict)
+    ZeroFileStats (qd->File_comp_dict);
+  if (qd->File_fast_comp_dict)
+    ZeroFileStats (qd->File_fast_comp_dict);
+  ZeroFileStats (qd->File_stem);
+
+  /* [RPAP - Jan 97: Stem Index Change] */
+  if (qd->File_stem1)
+    ZeroFileStats (qd->File_stem1);
+  if (qd->File_stem2)
+    ZeroFileStats (qd->File_stem2);
+  if (qd->File_stem3)
+    ZeroFileStats (qd->File_stem3);
+
+  ZeroFileStats (qd->File_invf);
+  if (qd->File_text_idx_wgt)
+    ZeroFileStats (qd->File_text_idx_wgt);
+  if (qd->File_weight_approx)
+    ZeroFileStats (qd->File_weight_approx);
+  if (qd->File_text_idx)
+    ZeroFileStats (qd->File_text_idx);
+}
+
+
+void 
+TransFileStats (query_data * qd)
+{
+  qd->File_text->Current = qd->File_text->Cumulative;
+  if (qd->File_comp_dict)
+    qd->File_comp_dict->Current = qd->File_comp_dict->Cumulative;
+  if (qd->File_fast_comp_dict)
+    qd->File_fast_comp_dict->Current = qd->File_fast_comp_dict->Cumulative;
+  qd->File_stem->Current = qd->File_stem->Cumulative;
+
+  /* [RPAP - Jan 97: Stem Index Change] */
+  if (qd->File_stem1)
+    qd->File_stem1->Current = qd->File_stem1->Cumulative;
+  if (qd->File_stem2)
+    qd->File_stem2->Current = qd->File_stem2->Cumulative;
+  if (qd->File_stem3)
+    qd->File_stem3->Current = qd->File_stem3->Cumulative;
+
+  qd->File_invf->Current = qd->File_invf->Cumulative;
+  if (qd->File_text_idx_wgt)
+    qd->File_text_idx_wgt->Current = qd->File_text_idx_wgt->Cumulative;
+  if (qd->File_weight_approx)
+    qd->File_weight_approx->Current = qd->File_weight_approx->Cumulative;
+  if (qd->File_text_idx)
+    qd->File_text_idx->Current = qd->File_text_idx->Cumulative;
+}
+
+
+void 
+FreeTextBuffer (query_data * qd)
+{
+  if (qd->TextBuffer)
+    {
+      Xfree (qd->TextBuffer);
+      ChangeMemInUse (qd, -qd->TextBufferLen);
+    }
+  qd->TextBuffer = NULL;
+  qd->TextBufferLen = 0;
+}
+
+void 
+FreeQueryDocs (query_data * qd)
+{
+  qd->doc_pos = 0;
+  qd->buf_in_use = 0;
+  if (qd->DL)
+    {
+      int i;
+      for (i = 0; i < qd->DL->num; i++)
+	if (qd->DL->DE[i].CompTextBuffer)
+	  {
+	    Xfree (qd->DL->DE[i].CompTextBuffer);
+	    qd->DL->DE[i].CompTextBuffer = NULL;
+	    ChangeMemInUse (qd, -qd->DL->DE[i].Len);
+	  }
+      Xfree (qd->DL);
+    }
+  qd->DL = NULL;
+  FreeTextBuffer (qd);
+}
+
+int 
+LoadCompressedText (query_data * qd, int max_mem)
+{
+  DocEntry *DE;
+  if (qd->DL == NULL || qd->doc_pos >= qd->DL->num)
+    return -1;
+
+  DE = &qd->DL->DE[qd->doc_pos];
+  if (!DE->CompTextBuffer)
+    {
+      int i;
+      DocEntry *de;
+      for (i = 0, de = qd->DL->DE; i < qd->DL->num; i++, de++)
+	if (de->CompTextBuffer)
+	  {
+	    Xfree (de->CompTextBuffer);
+	    de->CompTextBuffer = NULL;
+	    ChangeMemInUse (qd, -de->Len);
+	  }
+      if (LoadBuffers (qd, &qd->DL->DE[qd->doc_pos], max_mem,
+		       qd->DL->num - qd->doc_pos) == -1)
+	return -1;
+    }
+  return 0;
+}
+
+int 
+GetDocNum (query_data * qd)
+{
+  if (qd->DL == NULL || qd->doc_pos >= qd->DL->num)
+    return -1;
+  return qd->DL->DE[qd->doc_pos].DocNum;
+}
+
+DocEntry *
+GetDocChain (query_data * qd)
+{
+  if (qd->DL == NULL || qd->doc_pos >= qd->DL->num)
+    return NULL;
+  return &(qd->DL->DE[qd->doc_pos]);
+}
+
+float 
+GetDocWeight (query_data * qd)
+{
+  if (qd->DL == NULL || qd->doc_pos >= qd->DL->num)
+    return -1;
+  return qd->DL->DE[qd->doc_pos].Weight;
+}
+
+long 
+GetDocCompLength (query_data * qd)
+{
+  if (qd->DL == NULL || qd->doc_pos >= qd->DL->num)
+    return -1;
+  return qd->DL->DE[qd->doc_pos].Len;
+}
+
+
+u_char *
+GetDocText (query_data * qd, unsigned long *len)
+{
+  DocEntry *DE;
+  int ULen;
+  if (qd->DL == NULL || qd->doc_pos >= qd->DL->num)
+    return NULL;
+
+  DE = &qd->DL->DE[qd->doc_pos];
+
+  if (!DE->CompTextBuffer)
+    {
+      fprintf (stderr, "The compressed text buffer is NULL\n");
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  FreeTextBuffer (qd);
+
+  qd->TextBufferLen = (int) (qd->td->cth.ratio * 1.01 *
+			     DE->Len) + 100;
+  if (!(qd->TextBuffer = Xmalloc (qd->TextBufferLen)))
+    {
+      fprintf (stderr, "No memory for TextBuffer\n");
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  DecodeText (qd->cd, (u_char *) (DE->CompTextBuffer), DE->Len,
+	      (u_char *) (qd->TextBuffer), &ULen);
+  qd->TextBuffer[ULen] = '\0';
+
+  if (ULen >= qd->TextBufferLen)
+    {
+      fprintf (stderr, "%d >= %d\n", ULen, qd->TextBufferLen);
+      mg_errno = MG_BUFTOOSMALL;
+      return (NULL);
+    }
+
+  if (len)
+    *len = ULen;
+
+  return qd->TextBuffer;
+}
+
+int 
+NextDoc (query_data * qd)
+{
+  if (qd->DL == NULL || qd->doc_pos >= qd->DL->num)
+    return 0;
+  qd->doc_pos++;
+  return qd->doc_pos < qd->DL->num;
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/backend.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/backend.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/backend.h	(revision 699)
@@ -0,0 +1,263 @@
+/**************************************************************************
+ *
+ * backend.h -- Underlying routines and datastructures for mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef BACKEND_H
+#define BACKEND_H
+
+#include "sysfuncs.h"
+
+#include "timing.h"
+#include "lists.h"
+#include "term_lists.h"
+#include "query_term_list.h"  /* [RPAP - Feb 97: Term Frequency] */
+#include "mg.h"
+#include "invf.h"
+#include "text.h"
+
+
+typedef struct invf_data
+  {
+    File *InvfFile;
+    unsigned long N;
+    unsigned long Nstatic;	/* N parameter for decoding inverted file entries */
+    struct invf_file_header ifh;
+  }
+invf_data;
+
+typedef struct text_data
+  {
+    File *TextFile;
+    File *TextIdxFile;
+    File *TextIdxWgtFile;
+    long current_pos;
+    struct
+      {
+	unsigned long Start;
+	float Weight;
+      }
+     *idx_data;
+    compressed_text_header cth;
+  }
+text_data;
+
+
+typedef struct auxiliary_dict
+  {
+    aux_frags_header afh[2];
+    u_char *word_data[2];
+    u_char **words[2];
+    int blk_start[2][33], blk_end[2][33];	/* blk_start and blk_end are required
+						   for the hybrid methods */
+  }
+auxiliary_dict;
+
+
+typedef struct compression_dict
+  {
+    compression_dict_header cdh;
+    comp_frags_header *cfh[2];
+    unsigned long MemForCompDict;
+    u_char ***values[2];
+    u_char *escape[2];
+    huff_data *chars_huff[2];
+    u_long **chars_vals[2];
+    huff_data *lens_huff[2];
+    u_long **lens_vals[2];
+    auxiliary_dict *ad;
+    int fast_loaded;
+  }
+compression_dict;
+
+
+typedef struct stemmed_idx   /* [RPAP - Jan 97: Stem Index Change] */
+  {
+    File *stem_idx_file;
+    struct stem_idx_header sih;
+    u_char **index;
+    unsigned long *pos;
+    int active;
+    u_char *buffer;
+    unsigned long MemForStemIdx;
+  }
+stemmed_idx;
+
+
+typedef struct stemmed_dict
+  {
+    File *stem_file;
+    struct stem_dict_header sdh;
+    u_char **index;
+    unsigned long *pos;
+    int active;
+    u_char *buffer;
+    unsigned long MemForStemDict;
+
+    /* [RPAP - Jan 97: Stem Index Change] */
+    stemmed_idx *stem1;
+    stemmed_idx *stem2;
+    stemmed_idx *stem3;
+  }
+stemmed_dict;
+
+
+typedef struct approx_weights_data
+  {
+    double L;
+    double B;
+    unsigned long *DocWeights;
+    char bits;
+    float *table;
+    unsigned long mask;
+    unsigned long MemForWeights;
+    unsigned long num_of_docs;
+  }
+approx_weights_data;
+
+
+typedef struct RankedQueryInfo
+  {
+    int QueryFreqs;
+    int Exact;           /* use exact weights for ranking or not */
+    long MaxDocsToRetrieve;	/* may be -1 for all */
+    long MaxParasToRetrieve;
+    int Sort;        
+    char AccumMethod;		/* 'A' = array,  'S' = splay tree,  'H' = hash_table */
+    long MaxAccums;		/* may be -1 for all */
+    long MaxTerms;		/* may be -1 for all */
+    int StopAtMaxAccum; /* Stop at maximum accumulator or not */
+    long HashTblSize;
+    char *skip_dump;
+  }
+RankedQueryInfo;
+
+
+
+typedef struct BooleanQueryInfo
+  {
+    long MaxDocsToRetrieve;
+  }
+BooleanQueryInfo;
+
+
+/* [TS:24/Aug/94] - maximum number of characters in term string */
+#define MAXTERMSTRLEN 1023
+
+typedef struct query_data
+  {
+    stemmed_dict *sd;
+    compression_dict *cd;
+    approx_weights_data *awd;
+    invf_data *id;
+    text_data *td;
+#if defined(PARADOCNUM) || defined(NZDL)
+    int *paragraph;
+#endif
+    char *pathname;
+    char *textpathname; /* [RJM 06/97: text filename] */
+    File *File_text;
+    File *File_comp_dict;
+    File *File_aux_dict;
+    File *File_fast_comp_dict;
+    File *File_text_idx_wgt;
+    File *File_text_idx;
+    File *File_stem;
+
+    /* [RPAP - Jan 97: Stem Index Change] */
+    File *File_stem1;
+    File *File_stem2;
+    File *File_stem3;
+
+    File *File_invf;
+    File *File_weight_approx;
+    unsigned long mem_in_use, max_mem_in_use;
+    unsigned long num_of_ptrs, tot_num_of_ptrs;
+    unsigned long num_of_terms, tot_num_of_terms;
+    unsigned long num_of_accum, tot_num_of_accum;
+    unsigned long num_of_ans, tot_num_of_ans;
+    unsigned long hops_taken, tot_hops_taken;
+    unsigned long text_idx_lookups, tot_text_idx_lookups;
+    unsigned long max_buffers;
+    unsigned doc_pos;
+    unsigned buf_in_use;
+    DocList *DL;
+    TermList *TL;		/* [TS:Oct/94] - so term list for query can easily be accessed */
+    u_char *TextBuffer;
+    int TextBufferLen;
+    QueryTermList *QTL;    /* [RPAP - Feb 97: Term Frequency] */
+  }
+query_data;
+
+
+
+typedef struct InitQueryTimes
+  {
+    ProgTime Start;
+    ProgTime StemDict;
+    ProgTime ApproxWeights;
+    ProgTime CompDict;
+    ProgTime Invf;
+    ProgTime Text;
+  }
+InitQueryTimes;
+
+
+/* [RJM 06/97: text filename] */
+query_data *InitQuerySystem (char *dir, char *name, char *textname, InitQueryTimes * iqt);
+
+void ChangeMemInUse (query_data * qd, long delta);
+
+void FinishQuerySystem (query_data * qd);
+
+void ResetFileStats (query_data * qd);
+
+void TransFileStats (query_data * qd);
+
+void ChangeMemInUse (query_data * qd, long delta);
+
+void RankedQuery (query_data * qd, char *Query, RankedQueryInfo * rqi);
+
+void BooleanQuery (query_data * qd, char *Query, BooleanQueryInfo * bqi,
+		   int stem_method);
+
+void DocnumsQuery (query_data * qd, char *QueryLine);
+
+void FreeTextBuffer (query_data * qd);
+
+void FreeQueryDocs (query_data * qd);
+
+int LoadCompressedText (query_data * qd, int max_mem);
+
+int GetDocNum (query_data * qd);
+
+float GetDocWeight (query_data * qd);
+
+long GetDocCompLength (query_data * qd);
+
+u_char *GetDocText (query_data * qd, unsigned long *len);
+
+DocEntry *GetDocChain (query_data * qd);
+
+int NextDoc (query_data * qd);
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/bool_optimiser.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/bool_optimiser.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/bool_optimiser.c	(revision 699)
@@ -0,0 +1,1108 @@
+/**************************************************************************
+ *
+ * bool_optimiser -- optimise boolean parse tree
+ * Copyright (C) 1994,95  Tim Shimmin
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:17:42  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:34:20  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.4  1995/07/27  04:54:55  tes
+   * Committing optimiser before I add the code
+   * to find diff-nodes in any kind of tree.
+   *
+   * Revision 1.3  1994/10/20  00:02:55  tes
+   * Fixed bug with bool_optimiser:
+   *   - it couldn't handle "false | false"
+   *   - it couldn't handle "true & true"
+   * These would collapse into an empty list and cause an error.
+   *
+   * Revision 1.2  1994/10/18  06:11:02  tes
+   * The boolean optimiser seems to be modifying the parse tree
+   * like it is supposed to.
+   * Paragraph ranking now works without any text files if required to.
+   *
+   * Revision 1.1  1994/10/12  01:15:28  tes
+   * Found bugs in the existing boolean query optimiser.
+   * So decided to rewrite it.
+   * I accidentally deleted query.bool.y, but I have replaced it
+   * with bool_parser.y (which I have forgotten to add here ! ;-(
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "bool_tree.h"
+#include "term_lists.h"
+#include "messages.h"
+
+typedef int (*CompFunc) (const void *, const void *);
+typedef void (*TreeFunc) (bool_tree_node *);
+
+/* --- routines --- */
+static void PermeateNots (bool_tree_node * tree);
+static void DoubleNeg (bool_tree_node * not_tree);
+static void AndDeMorgan (bool_tree_node * not_tree);
+static void OrDeMorgan (bool_tree_node * not_tree);
+static int AndDistribute (bool_tree_node * tree);
+static void DNF_Distribute (bool_tree_node * tree);
+static void ReplaceWithChildren (bool_tree_node * tree);
+static void CollapseTree (bool_tree_node * tree);
+static void AndTrueFalse (bool_tree_node * and_tree);
+static void OrTrueFalse (bool_tree_node * or_tree);
+static void TF_Idempotent (bool_tree_node * tree);
+static void AndNotToDiff (bool_tree_node * and_tree, TreeFunc recurs_func);
+static void DiffDNF (bool_tree_node * or_node);
+static void DiffTree (bool_tree_node * tree);
+static void AndSort (bool_tree_node * and_tree, TermList * tl, CompFunc comp);
+static void AndOrSort (bool_tree_node * tree, TermList * tl);
+static void AndSortDNF (bool_tree_node * or_node, TermList * tl);
+static void MarkOrTerms (bool_tree_node * tree, TermList * tl);
+static int calc_sum_ft (bool_tree_node * term_tree, TermList * tl);
+static int AndOrSort_comp (const void *A, const void *B);
+static int AndSort_comp (const void *A, const void *B);
+
+/* =========================================================================
+ * Function: OptimiseBoolTree
+ * Description: 
+ *      For case 2:
+ *        Do three major steps:
+ *        (i) put into standard form
+ *            -> put into DNF (disjunctive normal form - or of ands)
+ *            Use DeMorgan's, Double-negative, Distributive rules
+ *        (ii) simplify
+ *             apply idempotency rules
+ *        (iii) ameliorate
+ *              convert &! to diff nodes, order terms by frequency,...
+ *     Could also do the matching idempotency laws i.e. ...
+ *     (A | A), (A | !A), (A & !A), (A & A), (A & (A | B)), (A | (A & B)) 
+ *     Job for future.... ;-) 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+void
+OptimiseBoolTree (bool_tree_node * parse_tree, TermList * tl, int type)
+{
+  switch (type)
+    {
+    case 1:
+      /* --- sorts for and of or-terms --- */
+      PermeateNots (parse_tree);
+      TF_Idempotent (parse_tree);
+      CollapseTree (parse_tree);
+      DiffTree (parse_tree);
+      MarkOrTerms (parse_tree, tl);
+      AndOrSort (parse_tree, tl);
+      break;
+    case 2:
+      /* --- put into DNF --- */
+      PermeateNots (parse_tree);
+      TF_Idempotent (parse_tree);
+      DNF_Distribute (parse_tree);
+      CollapseTree (parse_tree);
+      AndSortDNF (parse_tree, tl);
+      DiffDNF (parse_tree);
+      break;
+    default:
+      /* --- do nothing --- */
+      break;
+    }				/*switch */
+}				/*OptimiseBoolTree */
+
+/* =========================================================================
+ * Function: DoubleNeg
+ * Description: 
+ *      !(!(a) = a
+ *      Assumes binary tree.
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+static void
+DoubleNeg (bool_tree_node * not_tree)
+{
+  bool_tree_node *child = BOOL_CHILD (not_tree);
+  bool_tree_node *a = BOOL_CHILD (child);
+  bool_tree_node *sibling = BOOL_SIBLING (not_tree);
+
+  /* copy a into not tree */
+  bcopy (a, not_tree, sizeof (bool_tree_node));
+
+  BOOL_SIBLING (not_tree) = sibling;
+
+  Xfree (child);
+  Xfree (a);
+
+}
+
+/* =========================================================================
+ * Function: AndDeMorgan
+ * Description: 
+ *      DeMorgan's rule for 'not' of an 'and'  i.e. !(a & b) <=> (!a) | (!b)
+ *      Assumes Binary Tree
+ * Input: 
+ *      not of and tree
+ * Output: 
+ *      or of not trees
+ * ========================================================================= */
+
+static void
+AndDeMorgan (bool_tree_node * not_tree)
+{
+  bool_tree_node *child = BOOL_CHILD (not_tree);
+  bool_tree_node *a = BOOL_CHILD (child);
+  bool_tree_node *b = BOOL_SIBLING (a);
+  bool_tree_node *new_not = NULL;
+
+  /* change 'not' to 'or' */
+  BOOL_TAG (not_tree) = N_or;
+
+  /* change 'and' to 'not' */
+  BOOL_TAG (child) = N_not;
+
+  /* 'a' no longer has a sibling */
+  BOOL_SIBLING (a) = NULL;
+
+  /* 'a's old sibling is now a child of a new not */
+  new_not = CreateBoolTreeNode (N_not, b, NULL);
+  BOOL_SIBLING (child) = new_not;
+}
+
+/* =========================================================================
+ * Function: OrDeMorgan
+ * Description: 
+ *      DeMorgan's rule for 'not' of an 'or' i.e. !(a | b) <=> (!a) & (!b)
+ *      Assumes Binary Tree
+ * Input: 
+ *      not of and tree
+ * Output: 
+ *      or of not trees
+ * ========================================================================= */
+
+static void
+OrDeMorgan (bool_tree_node * not_tree)
+{
+  bool_tree_node *child = BOOL_CHILD (not_tree);
+  bool_tree_node *a = BOOL_CHILD (child);
+  bool_tree_node *b = BOOL_SIBLING (a);
+  bool_tree_node *new_not = NULL;
+
+  /* change 'not' to 'and' */
+  BOOL_TAG (not_tree) = N_and;
+
+  /* change 'or' to 'not' */
+  BOOL_TAG (child) = N_not;
+
+  /* 'a' no longer has a sibling */
+  BOOL_SIBLING (a) = NULL;
+
+  /* 'a's old sibling is now a child of a new not */
+  new_not = CreateBoolTreeNode (N_not, b, NULL);
+  BOOL_SIBLING (child) = new_not;
+}
+
+/* =========================================================================
+ * Function: PermeateNots
+ * Description: 
+ *      Use DeMorgan's and Double-negative
+ *      Assumes tree in binary form (i.e. No ands/ors collapsed)
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+PermeateNots (bool_tree_node * tree)
+{
+  if (!tree)
+    return;
+
+  if (BOOL_TAG (tree) == N_not)
+    {
+      switch (BOOL_TAG (BOOL_CHILD (tree)))
+	{
+	case N_not:
+	  DoubleNeg (tree);
+/** NOTE: I do not go on to do the children **/
+	  /* I permeate the nots on the tree here & leave it at that */
+	  /* In this case I do not know what type 'tree' will end up */
+	  PermeateNots (tree);
+	  return;
+	case N_and:
+	  AndDeMorgan (tree);
+	  break;
+	case N_or:
+	  OrDeMorgan (tree);
+	  break;
+	default:
+	  /* leave alone */
+	  break;
+	}
+    }
+
+  /* Permeate Nots for children */
+  if (BOOL_HAS_CHILD (tree))
+    {
+      PermeateNots (BOOL_CHILD (tree));
+      PermeateNots (BOOL_SIBLING (BOOL_CHILD (tree)));
+    }
+
+}
+
+/* =========================================================================
+ * Function: DNF_Distribute
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+
+#define ITER_LIMIT 500		/* if reach limit then probably in trouble */
+
+static void
+DNF_Distribute (bool_tree_node * tree)
+{
+  int i = 0;
+
+  /* multiple passes until no change */
+  /* not very smart :-( */
+  while (1)
+    {
+      if (AndDistribute (tree) == 0)
+	return;
+      if (i == ITER_LIMIT)
+	FatalError (1, "DNF infinite loop");
+      i++;
+    }				/*while */
+
+}
+
+/* =========================================================================
+ * Function: AndDistribute
+ * Description: 
+ *      (a | b) & A <=> (a & A) | (b & A)
+ * Input: 
+ *      binary tree of "AND" , "OR"s.
+ * Output: 
+ *      return 1 if changed the tree
+ *      return 0 if there was NO change (no distributive rule to apply)
+ * ========================================================================= */
+
+static int
+AndDistribute (bool_tree_node * tree)
+{
+  if (!tree)
+    return 0;
+
+  if (BOOL_TAG (tree) != N_and)
+    {
+      /* try the children if got some */
+      if (BOOL_TAG (tree) == N_or)
+	{
+	  bool_tree_node *left_child = BOOL_CHILD (tree);
+	  int left = AndDistribute (left_child);
+	  int right = AndDistribute (BOOL_SIBLING (left_child));
+	  return (left || right);
+	}
+
+      /* no children, then no change */
+      return 0;
+    }
+
+  /* AND case */
+  {
+    /* we have to check the children to see if can distribute */
+    bool_tree_node *left_child = BOOL_CHILD (tree);
+    bool_tree_node *right_child = BOOL_SIBLING (left_child);
+    int swap = 0;
+
+    /* If "or" on the right side then do a swap */
+    /* This is done for code simplicity */
+    if (BOOL_TAG (right_child) == N_or)
+      {
+	/* do a swap */
+	bool_tree_node *temp;
+	temp = right_child;
+	right_child = left_child;
+	left_child = temp;
+
+	BOOL_SIBLING (left_child) = right_child;
+	BOOL_SIBLING (right_child) = NULL;
+	BOOL_CHILD (tree) = left_child;
+
+	swap = 1;
+      }
+
+    /* do the actual distribution */
+    if (swap || BOOL_TAG (left_child) == N_or)
+      {
+	bool_tree_node *A = right_child;
+	bool_tree_node *A_copy = CopyBoolTree (A);
+	bool_tree_node *a = BOOL_CHILD (left_child);
+	bool_tree_node *b = BOOL_SIBLING (a);
+
+	/* then can distribute away ... */
+	BOOL_TAG (tree) = N_or;
+	BOOL_TAG (left_child) = N_and;
+	BOOL_SIBLING (left_child) = CreateBoolTreeNode (N_and, b, A);
+	BOOL_SIBLING (a) = A_copy;
+
+	AndDistribute (tree);
+	return 1;
+      }
+
+    /* if "and" of something other than "or" s then try the children */
+    {
+      int left = AndDistribute (left_child);
+      int right = AndDistribute (right_child);
+      return (left || right);
+    }
+  }				/* AND case */
+
+}				/*AndDistribute */
+
+/* =========================================================================
+ * Function: CollapseTree
+ * Description: 
+ *      Collapse "and" and "or"s.
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+CollapseTree (bool_tree_node * tree)
+{
+  if (!tree)
+    return;
+
+  if ((BOOL_TAG (tree) == N_and) || (BOOL_TAG (tree) == N_or))
+    {
+      bool_tree_node *child = BOOL_CHILD (tree);
+      while (child)
+	{
+	  while (BOOL_TAG (child) == BOOL_TAG (tree))
+	    {
+	      ReplaceWithChildren (child);
+	    }
+	  CollapseTree (child);
+	  child = BOOL_SIBLING (child);
+	}
+    }
+
+  /* assume "not"s permeated */
+  return;
+}
+
+/* =========================================================================
+ * Function: ReplaceWithChildren
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+ReplaceWithChildren (bool_tree_node * tree)
+{
+  bool_tree_node *next = BOOL_SIBLING (tree);
+  bool_tree_node *child = BOOL_CHILD (tree);
+  bool_tree_node *last = LastBoolSibling (child);
+
+  bcopy ((char *) child, (char *) tree, sizeof (bool_tree_node));
+  Xfree (child);
+  BOOL_SIBLING (last) = next;
+
+}
+
+/* =========================================================================
+ * Function: TF_Idempotent
+ * Description: 
+ *      Applies Idempotency laws which refer to True or False (hence "TF")
+ *      uses the true or false idempotent rules
+ * Input: 
+ *      operates on n-ary trees (not restricted to binary)
+ * Output: 
+ * ========================================================================= */
+
+static void
+TF_Idempotent (bool_tree_node * tree)
+{
+  bool_tree_node *child = NULL;
+
+  while (tree)
+    {
+      if (BOOL_HAS_CHILD (tree))
+	{
+	  child = BOOL_CHILD (tree);
+	  TF_Idempotent (child);
+	}
+
+      /* --- process node --- */
+      switch (BOOL_TAG (tree))
+	{
+	case N_not:
+	  child = BOOL_CHILD (tree);
+
+	  /* if true then falsify */
+	  if (BOOL_TAG (child) == N_all)
+	    {
+	      BOOL_TAG (tree) = N_none;
+	      BOOL_CHILD (tree) = NULL;
+	      Xfree (child);
+	      break;
+	    }
+
+	  /* if false then trueify */
+	  if (BOOL_TAG (child) == N_none)
+	    {
+	      BOOL_TAG (tree) = N_all;
+	      BOOL_CHILD (tree) = NULL;
+	      Xfree (child);
+	      break;
+	    }
+
+	  break;
+	case N_and:
+	  AndTrueFalse (tree);
+	  break;
+	case N_or:
+	  OrTrueFalse (tree);
+	  break;
+
+	default:
+	  /* do nothing */
+	  break;
+	}			/*switch */
+
+      tree = BOOL_SIBLING (tree);
+    }				/*while */
+}				/*TF_Idempotent */
+
+/* =========================================================================
+ * Function: AndTrueFalse
+ * Description: 
+ *      If any "false" then falsify whole tree 
+ *      If any "true" then delete them
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+AndTrueFalse (bool_tree_node * and_tree)
+{
+  bool_tree_node *child = BOOL_CHILD (and_tree);
+
+  bool_tree_node *prev = NULL;
+
+  while (child)
+    {
+      if (BOOL_TAG (child) == N_none)
+	{
+	  FreeBoolTree (&(BOOL_CHILD (and_tree)));
+	  BOOL_TAG (and_tree) = N_none;
+	  return;
+	}
+
+      if (BOOL_TAG (child) == N_all)
+	{
+	  bool_tree_node *sibling = BOOL_SIBLING (child);
+
+	  /* --- link over child --- */
+	  if (!prev)
+	    {
+	      BOOL_CHILD (and_tree) = sibling;
+	    }
+	  else
+	    {
+	      BOOL_SIBLING (prev) = sibling;
+	    }
+
+	  Xfree (child);
+	  child = sibling;
+	}
+      else
+	{
+	  prev = child;
+	  child = BOOL_SIBLING (child);
+	}
+    }				/*while */
+
+  child = BOOL_CHILD (and_tree);
+
+  /* if no children ... */
+  /* => all have been true */
+  if (!child)
+    {
+      BOOL_TAG (and_tree) = N_all;
+      return;
+    }
+
+  /* If only one child ... */
+  if (!BOOL_SIBLING (child))
+    {
+      /* replace the parent with child - move it up */
+      bool_tree_node *save_sibling = BOOL_SIBLING (and_tree);
+      bcopy ((char *) child, (char *) and_tree, sizeof (bool_tree_node));
+      BOOL_SIBLING (and_tree) = save_sibling;
+      Xfree (child);
+    }
+
+}				/*AndTrueFalse */
+
+/* =========================================================================
+ * Function: OrTrueFalse
+ * Description: 
+ *      If any "true" then trueify whole tree 
+ *      If any "false" then delete them 
+ *      ***** NOTE: This is a reflection of AndTrueFalse *****
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+OrTrueFalse (bool_tree_node * or_tree)
+{
+  bool_tree_node *child = BOOL_CHILD (or_tree);
+
+  bool_tree_node *prev = NULL;
+
+  while (child)
+    {
+      if (BOOL_TAG (child) == N_all)
+	{
+	  FreeBoolTree (&(BOOL_CHILD (or_tree)));
+	  BOOL_TAG (or_tree) = N_all;
+	  return;
+	}
+
+      if (BOOL_TAG (child) == N_none)
+	{
+	  bool_tree_node *sibling = BOOL_SIBLING (child);
+	  if (!prev)
+	    {
+	      BOOL_CHILD (or_tree) = sibling;
+	    }
+	  else
+	    {
+	      BOOL_SIBLING (prev) = sibling;
+	    }
+	  Xfree (child);
+	  child = sibling;
+
+	}
+      else
+	{
+	  prev = child;
+	  child = BOOL_SIBLING (child);
+	}
+    }				/*while */
+
+  child = BOOL_CHILD (or_tree);
+
+  /* if no children ... */
+  /* => all have been false */
+  if (!child)
+    {
+      BOOL_TAG (or_tree) = N_none;
+      return;
+    }
+
+  /* If only one child ... */
+  if (!BOOL_SIBLING (child))
+    {
+      /* replace the parent with child - move it up */
+      bool_tree_node *save_sibling = BOOL_SIBLING (or_tree);
+      bcopy ((char *) child, (char *) or_tree, sizeof (bool_tree_node));
+      BOOL_SIBLING (or_tree) = save_sibling;
+      Xfree (child);
+    }
+
+}				/*AndTrueFalse */
+
+/* =========================================================================
+ * Function: AndNotToDiff
+ * Description: 
+ *      Convert an "And" tree with possibly some negated terms into
+ *      a "And - neg-terms". 
+ * Input: 
+ *      Expects "And" of only literals - as tree should be in DNF form.
+ *      This means that no recursion is necessary.
+ * Output: 
+ * ========================================================================= */
+
+static void
+AndNotToDiff (bool_tree_node * and_tree, TreeFunc recurs_func)
+{
+
+  /* -------------------------------------------------- */
+  /* --- Divide the children up into positive and negative literals --- */
+  /* Preserves ordering of children */
+  bool_tree_node *pos = NULL;
+  bool_tree_node *neg = NULL;
+  bool_tree_node *last_pos = NULL;
+  bool_tree_node *last_neg = NULL;
+
+  bool_tree_node *child = BOOL_CHILD (and_tree);
+  while (child)
+    {
+      bool_tree_node *next = BOOL_SIBLING (child);
+      if (BOOL_TAG (child) == N_not)
+	{
+	  /* add to end of neg list */
+	  if (!neg)
+	    neg = child;
+	  else
+	    BOOL_SIBLING (last_neg) = child;
+	  last_neg = child;
+	}
+      else
+	{
+	  /* add to end of pos list */
+	  if (!pos)
+	    pos = child;
+	  else
+	    BOOL_SIBLING (last_pos) = child;
+	  last_pos = child;
+	}
+      child = next;
+    }				/*while */
+  if (last_pos)
+    BOOL_SIBLING (last_pos) = NULL;
+  if (last_neg)
+    BOOL_SIBLING (last_neg) = NULL;
+  /* -------------------------------------------------- */
+
+  if (!neg)			/* no negtives => no subtraction */
+    {
+      /* fix up parent's child link */
+      BOOL_CHILD (and_tree) = pos;
+      return;
+    }
+
+  if (!pos)			/* no positives = all negatives */
+    {
+      /* Choose one of the negs for the left-hand-side of the subtraction */
+      /* In particular, choose the first */
+      pos = neg;
+      neg = BOOL_SIBLING (neg);
+      BOOL_SIBLING (pos) = NULL;
+    }
+
+  /* Get rid of all '!' from not nodes */
+  /* Make a list out of the children */
+  {
+    bool_tree_node *n = neg;
+    bool_tree_node *next = NULL;
+
+    neg = BOOL_CHILD (neg);
+    while (n)
+      {
+	bool_tree_node *c = BOOL_CHILD (n);
+	next = BOOL_SIBLING (n);
+	Xfree (n);
+	if (next)
+	  BOOL_SIBLING (c) = BOOL_CHILD (next);
+	n = next;
+      }
+  }
+
+  /* --- restructure the tree with "and" and "diff" --- */
+  BOOL_TAG (and_tree) = N_diff;
+  if (!BOOL_SIBLING (pos))	/* i.e. only one pos */
+    {
+      /* (A & !B & !C & !D & ...) ==> A - (B,C,D,...) */
+
+      BOOL_CHILD (and_tree) = pos;
+      BOOL_SIBLING (pos) = neg;
+    }
+  else
+    {
+      /* (A & !C & B & !D & ...) ==> (A & B & ....) - (C, D, ...) */
+
+      bool_tree_node *n = CreateBoolNode (N_and);
+      BOOL_CHILD (and_tree) = n;
+      BOOL_CHILD (n) = pos;
+      BOOL_SIBLING (n) = neg;
+    }				/*if */
+
+  if (recurs_func)
+    {
+      recurs_func (BOOL_CHILD (and_tree));
+    }
+
+}				/*AndNotToDiff */
+
+/* =========================================================================
+ * Function: DiffTree
+ * Description: 
+ *      Applies "AndNotToNeg" to each conjunct.
+ *      Unlike DiffDNF, this routine will search throughout the tree for
+ *      "and"-nodes. It will call AndNotToNeg with this as its recursive 
+ *      routine.
+ * Date: 27/July/95
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+DiffTree (bool_tree_node * tree)
+{
+  bool_tree_node *t = tree;
+
+  while (t)
+    {
+      if (BOOL_TAG (t) == N_and)
+	{
+	  AndNotToDiff (t, DiffTree);
+	}
+      t = BOOL_SIBLING (t);
+    }				/*while */
+
+}				/*DiffTree */
+
+/* =========================================================================
+ * Function: DiffDNF
+ * Description: 
+ *      Applies "AndNotToNeg" to each conjunct.
+ * Input: 
+ *      A DNF tree
+ * Output: 
+ *      A tree possibly with some diffs in it.
+ * ========================================================================= */
+
+static void
+DiffDNF (bool_tree_node * DNF_tree)
+{
+
+  if (BOOL_TAG (DNF_tree) == N_or)
+    {
+      bool_tree_node *child = BOOL_CHILD (DNF_tree);
+
+      while (child)
+	{
+	  if (BOOL_TAG (child) == N_and)
+	    {
+	      AndNotToDiff (child, NULL);
+	    }
+	  child = BOOL_SIBLING (child);
+	}
+    }
+  else if (BOOL_TAG (DNF_tree) == N_and)
+    {
+      AndNotToDiff (DNF_tree, NULL);
+    }
+}
+
+/* =========================================================================
+ * Function: AndSort_comp 
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+/* Module variable, "term_list",  used as parameter for comparison function */
+/* The comparison function only allows certain fixed parameters */
+static TermList *term_list;
+
+static int
+AndSort_comp (const void *A, const void *B)
+{
+  const bool_tree_node *const *a = A;
+  const bool_tree_node *const *b = B;
+
+  if (BOOL_TAG (*a) == N_term && BOOL_TAG (*b) == N_term)
+    {
+      /* look up in term list */
+      int a_doc_count = GetNthWE (term_list, BOOL_TERM (*a))->doc_count;
+      int b_doc_count = GetNthWE (term_list, BOOL_TERM (*b))->doc_count;
+
+      return a_doc_count - b_doc_count;
+    }
+
+  if (BOOL_TAG (*a) == N_term && BOOL_TAG (*b) != N_term)
+    return -1;
+
+  if (BOOL_TAG (*a) != N_term && BOOL_TAG (*b) == N_term)
+    return 1;
+
+  return 0;
+}				/*AndSort_comp */
+
+/* =========================================================================
+ * Function: AndSort
+ * Description: 
+ *      Sort the list of nodes by increasing doc_count 
+ *      Using some Neil Sharman code - pretty straight forward.
+ *      Note: not-terms are sent to the end of the list
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+AndSort (bool_tree_node * and_tree, TermList * tl, CompFunc comp)
+{
+  bool_tree_node **list = NULL;
+  bool_tree_node *child = NULL;
+  int count = 0;
+  int i = 0;
+
+  if (BOOL_TAG (and_tree) != N_and)
+    return;
+
+  /* --- Convert linked list into an array for sorting --- */
+  child = BOOL_CHILD (and_tree);
+  count = 0;
+  while (child)
+    {
+      count++;
+      child = BOOL_SIBLING (child);
+    }
+
+  if (!(list = Xmalloc (sizeof (bool_tree_node *) * count)))
+    return;
+
+  child = BOOL_CHILD (and_tree);
+  for (i = 0; i < count; i++)
+    {
+      list[i] = child;
+      child = BOOL_SIBLING (child);
+    }
+  /* ----------------------------------------------------- */
+
+  term_list = tl;
+  qsort (list, count, sizeof (bool_tree_node *), comp);
+
+  /* --- Convert array back into linked list --- */
+  for (i = 0; i < count - 1; i++)
+    BOOL_SIBLING (list[i]) = list[i + 1];
+  BOOL_SIBLING (list[count - 1]) = NULL;
+  BOOL_CHILD (and_tree) = list[0];
+  Xfree (list);
+  /* ----------------------------------------------------- */
+
+}				/*AndSort */
+
+/* =========================================================================
+ * Function: AndSortDNF
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+
+static void
+AndSortDNF (bool_tree_node * DNF_tree, TermList * tl)
+{
+
+  if (BOOL_TAG (DNF_tree) == N_or)
+    {
+      bool_tree_node *child = BOOL_CHILD (DNF_tree);
+
+      while (child)
+	{
+	  if (BOOL_TAG (child) == N_and)
+	    {
+	      AndSort (child, tl, AndSort_comp);
+	    }
+	  child = BOOL_SIBLING (child);
+	}
+    }
+  else if (BOOL_TAG (DNF_tree) == N_and)
+    {
+      AndSort (DNF_tree, tl, AndSort_comp);
+    }
+}
+
+/* =========================================================================
+ * Function: MarkOrTerms
+ * Description: 
+ *      Mark or-nodes whose children are all terms.
+ *      Also store the sum of the ft's of these nodes.
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+MarkOrTerms (bool_tree_node * tree, TermList * tl)
+{
+  if (BOOL_TAG (tree) == N_or)
+    {
+      int found_non_term = 0;
+      bool_tree_node *child = BOOL_CHILD (tree);
+
+      /* --- check to see if got all terms --- */
+      while (child)
+	{
+	  if (BOOL_TAG (child) != N_term)
+	    {
+	      found_non_term = 1;
+	    }
+	  if (BOOL_HAS_CHILD (tree))
+	    {
+	      MarkOrTerms (child, tl);
+	    }
+	  child = BOOL_SIBLING (child);
+	}			/*while */
+
+      if (!found_non_term)
+	{
+	  /* Must have all terms for the OR node */
+	  /* So change its tag and calculate its sum */
+	  BOOL_TAG (tree) = N_or_terms;
+	  BOOL_SUM_FT (tree) = calc_sum_ft (tree, tl);
+	}			/*if */
+      /* ------------------------------------- */
+
+    }				/*if OR node */
+
+  else if (BOOL_HAS_CHILD (tree))
+    {
+      bool_tree_node *child = BOOL_CHILD (tree);
+      while (child)
+	{
+	  MarkOrTerms (child, tl);
+	  child = BOOL_SIBLING (child);
+	}
+    }
+
+}				/*MarkOrTerms */
+
+/* =========================================================================
+ * Function: calc_sum_ft
+ * Description: 
+ *      Sums up the ft's of all the terms 
+ * Input: 
+ *      Assumes that the tree only contains terms in it 
+ * Output: 
+ *      the ft sum
+ * ========================================================================= */
+static int
+calc_sum_ft (bool_tree_node * term_tree, TermList * tl)
+{
+  int sum_ft = 0;
+  bool_tree_node *child = BOOL_CHILD (term_tree);
+  while (child)
+    {
+      sum_ft += GetNthWE (tl, BOOL_TERM (child))->doc_count;
+      child = BOOL_SIBLING (child);
+    }
+  return sum_ft;
+}
+
+/* =========================================================================
+ * Function: AndOrSort
+ * Description: 
+ *      This is like an AndSort of just terms. 
+ *      The difference is that we are now also 
+ *      considering sorting with <or of terms> by using the sum of f_t's.
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+static void
+AndOrSort (bool_tree_node * tree, TermList * tl)
+{
+  if (BOOL_HAS_CHILD (tree))
+    {
+
+      if (BOOL_TAG (tree) == N_and)
+	{
+	  AndSort (tree, tl, AndOrSort_comp);
+	}
+
+      /* Check out any children down the tree */
+      /* Probably a waste of time */
+      {
+	bool_tree_node *child = BOOL_CHILD (tree);
+	while (child)
+	  {
+	    AndOrSort (child, tl);
+	    child = BOOL_SIBLING (child);
+	  }
+      }				/*do the tree */
+
+    }				/*if has children */
+}				/*AndOrSort */
+
+/* =========================================================================
+ * Function: AndOrSort_comp
+ * Description: 
+ *      Comparison function used to order "term"s and "or_terms"s.
+ *      It's purpose is to be used by a sort routine.
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+static int
+AndOrSort_comp (const void *A, const void *B)
+{
+  const bool_tree_node *const *a = A;
+  const bool_tree_node *const *b = B;
+
+  if (BOOL_TAG (*a) == N_term && BOOL_TAG (*b) == N_term)
+    {
+      /* look up in term list */
+      int a_doc_count = GetNthWE (term_list, BOOL_TERM (*a))->doc_count;
+      int b_doc_count = GetNthWE (term_list, BOOL_TERM (*b))->doc_count;
+
+      return a_doc_count - b_doc_count;
+    }
+
+  if (BOOL_TAG (*a) == N_term && BOOL_TAG (*b) == N_or_terms)
+    {
+      /* look up in term list */
+      int a_doc_count = GetNthWE (term_list, BOOL_TERM (*a))->doc_count;
+      int b_doc_count = BOOL_SUM_FT (*b);
+
+      return a_doc_count - b_doc_count;
+    }
+
+  if (BOOL_TAG (*a) == N_or_terms && BOOL_TAG (*b) == N_term)
+    {
+      /* look up in term list */
+      int a_doc_count = BOOL_SUM_FT (*a);
+      int b_doc_count = GetNthWE (term_list, BOOL_TERM (*b))->doc_count;
+
+      return a_doc_count - b_doc_count;
+    }
+
+  if (BOOL_TAG (*a) == N_or_terms && BOOL_TAG (*b) == N_or_terms)
+    {
+      /* look up in term list */
+      int a_doc_count = BOOL_SUM_FT (*a);
+      int b_doc_count = BOOL_SUM_FT (*b);
+
+      return a_doc_count - b_doc_count;
+    }
+
+  if ((BOOL_TAG (*a) == N_term || BOOL_TAG (*a) == N_or_terms) &&
+      (BOOL_TAG (*b) != N_term && BOOL_TAG (*b) != N_or_terms))
+    return -1;
+
+  if ((BOOL_TAG (*a) != N_term && BOOL_TAG (*a) != N_or_terms) &&
+      (BOOL_TAG (*b) == N_term || BOOL_TAG (*b) == N_or_terms))
+    return 1;
+
+  return 0;
+}				/*AndOrSort_comp */
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/bool_optimiser.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/bool_optimiser.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/bool_optimiser.h	(revision 699)
@@ -0,0 +1,62 @@
+/**************************************************************************
+ *
+ * bool_optimiser -- optimise boolean parse tree
+ * Copyright (C) 1994  Tim Shimmin
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:17:42  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:34:23  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.2  1995/03/14  05:15:21  tes
+   * Updated the boolean query optimiser to do different types of optimisation.
+   * A query environment variable "optimise_type" specifies which one is to be
+   * used. Type 1 is the new one which is faster than 2.
+   *
+   * Revision 1.1  1994/10/20  03:56:32  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/10/18  06:11:03  tes
+   * The boolean optimiser seems to be modifying the parse tree
+   * like it is supposed to.
+   * Paragraph ranking now works without any text files if required to.
+   *
+   * Revision 1.1  1994/10/12  01:15:29  tes
+   * Found bugs in the existing boolean query optimiser.
+   * So decided to rewrite it.
+   * I accidentally deleted query.bool.y, but I have replaced it
+   * with bool_parser.y (which I have forgotten to add here ! ;-(
+   *
+ */
+
+#ifndef BOOL_OPTIMISER_H
+#define BOOL_OPTIMISER_H
+
+#include "bool_tree.h"
+#include "term_lists.h"
+
+void OptimiseBoolTree (bool_tree_node * parse_tree, TermList * term_list, int type);
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/bool_parser.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/bool_parser.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/bool_parser.c	(revision 699)
@@ -0,0 +1,643 @@
+#ifndef lint
+static char yysccsid[] = "@(#)yaccpar	1.9 (Berkeley) 02/21/93";
+#endif
+#define YYBYACC 1
+#define YYMAJOR 1
+#define YYMINOR 9
+#define yyclearin (yychar=(-1))
+#define yyerrok (yyerrflag=0)
+#define YYRECOVERING (yyerrflag!=0)
+#define YYPREFIX "yy"
+#line 25 "bool_parser.y"
+ 
+#include "sysfuncs.h"
+
+#include "messages.h"
+ 
+#include "memlib.h"
+#include "words.h"
+#include "stemmer.h"
+#include "term_lists.h"
+#include "bool_tree.h"
+/* [RPAP - Jan 97: Stem Index Change] */
+#include "backend.h"     /* for stemmed_dict def */
+#include "stem_search.h"
+
+#include "query_term_list.h"  /* [RPAP - Feb 97: Term Frequency] */
+
+/* --- routines --- */
+static int query_lex();
+static int yyerror(char *);
+#define yylex() query_lex(&ch_buf, end_buf)
+ 
+/* --- module variables --- */
+static char *ch_buf; /* ptr to the character query line buffer */
+static char *end_buf; /* ptr to the last character of the line buffer */
+static bool_tree_node *tree_base = NULL;
+static TermList **term_list;
+static int stemmer_num;
+static int stem_method;
+/* [RPAP - Jan 97: Stem Index Change] */
+stemmed_dict *p__sd;
+static int indexed;
+/* [RPAP - Feb 97: Term Frequency] */
+static QueryTermList **query_term_list;
+static int word_num;
+static u_long count;
+static u_long doc_count;
+static u_long invf_ptr;
+static u_long invf_len;
+#line 65 "bool_parser.y"
+typedef union {
+  char *text;
+  bool_tree_node *node;
+} YYSTYPE;
+#line 55 "y.tab.c"
+#define TERM 257
+#define YYERRCODE 256
+short yylhs[] = {                                        -1,
+    0,    1,    1,    1,    1,    2,    2,    3,    3,    3,
+    4,    4,
+};
+short yylen[] = {                                         2,
+    1,    1,    3,    1,    1,    1,    2,    3,    2,    1,
+    3,    1,
+};
+short yydefred[] = {                                      0,
+    2,    0,    4,    5,    0,    0,    6,   10,    0,    0,
+    0,    7,    0,    9,    0,    3,    8,    0,
+};
+short yydgoto[] = {                                       6,
+    7,    8,    9,   10,
+};
+short yysindex[] = {                                    -32,
+    0,  -32,    0,    0,  -32,    0,    0,    0,  -33, -118,
+  -37,    0,  -32,    0,  -32,    0,    0,  -33,
+};
+short yyrindex[] = {                                      0,
+    0,    0,    0,    0,    0,    0,    0,    0,    2,   12,
+    0,    0,    0,    0,    0,    0,    0,    3,
+};
+short yygindex[] = {                                      0,
+    0,    6,   -2,   14,
+};
+#define YYTABLESIZE 225
+short yytable[] = {                                       5,
+    5,   12,   11,   16,   13,   15,    2,    2,    3,    3,
+   12,    1,   18,    0,   14,   11,    0,    0,   17,    0,
+    0,    0,    0,   14,    0,    0,    0,    0,    0,    0,
+    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
+    0,    0,   12,   11,    0,    0,    0,    0,    0,    0,
+    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
+    0,    4,    4,    0,    0,    0,    0,    0,    0,    0,
+    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
+    0,    0,    0,    0,    0,    0,   15,    0,    0,    0,
+    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
+    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
+    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
+    0,    0,    0,    0,    0,   12,   11,    0,    0,    0,
+    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
+    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
+    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
+    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
+    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
+    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
+    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
+    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
+    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
+    0,    0,    0,    1,    1,
+};
+short yycheck[] = {                                      33,
+   33,    0,    0,   41,   38,  124,   40,   40,   42,   42,
+    5,    0,   15,   -1,    9,    2,   -1,   -1,   13,   -1,
+   -1,   -1,   -1,   18,   -1,   -1,   -1,   -1,   -1,   -1,
+   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,
+   -1,   -1,   41,   41,   -1,   -1,   -1,   -1,   -1,   -1,
+   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,
+   -1,   95,   95,   -1,   -1,   -1,   -1,   -1,   -1,   -1,
+   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,
+   -1,   -1,   -1,   -1,   -1,   -1,  124,   -1,   -1,   -1,
+   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,
+   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,
+   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,
+   -1,   -1,   -1,   -1,   -1,  124,  124,   -1,   -1,   -1,
+   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,
+   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,
+   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,
+   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,
+   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,
+   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,
+   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,
+   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,
+   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,
+   -1,   -1,   -1,  257,  257,
+};
+#define YYFINAL 6
+#ifndef YYDEBUG
+#define YYDEBUG 0
+#endif
+#define YYMAXTOKEN 257
+#if YYDEBUG
+char *yyname[] = {
+"end-of-file",0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+"'!'",0,0,0,0,"'&'",0,"'('","')'","'*'",0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,"'_'",0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,"'|'",0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+"TERM",
+};
+char *yyrule[] = {
+"$accept : query",
+"query : or",
+"term : TERM",
+"term : '(' or ')'",
+"term : '*'",
+"term : '_'",
+"not : term",
+"not : '!' not",
+"and : and '&' not",
+"and : and not",
+"and : not",
+"or : or '|' and",
+"or : and",
+};
+#endif
+#ifdef YYSTACKSIZE
+#undef YYMAXDEPTH
+#define YYMAXDEPTH YYSTACKSIZE
+#else
+#ifdef YYMAXDEPTH
+#define YYSTACKSIZE YYMAXDEPTH
+#else
+#define YYSTACKSIZE 500
+#define YYMAXDEPTH 500
+#endif
+#endif
+int yydebug;
+int yynerrs;
+int yyerrflag;
+int yychar;
+short *yyssp;
+YYSTYPE *yyvsp;
+YYSTYPE yyval;
+YYSTYPE yylval;
+short yyss[YYSTACKSIZE];
+YYSTYPE yyvs[YYSTACKSIZE];
+#define yystacksize YYSTACKSIZE
+#line 99 "bool_parser.y"
+ 
+/* Bison on one mips machine defined "const" to be nothing but
+   then did not undef it */
+#ifdef const
+#undef const
+#endif
+
+/**************************************************************************/ 
+
+
+/* =========================================================================
+ * Function: query_lex
+ * Description:
+ *      Hand written lexical analyser for the parser.
+ * Input:
+ *      ptr = ptr to a ptr into character query-line buffer
+ *      end = ptr to last char in buffer
+ * Output:
+ *      yylval.text = the token's text
+ * Notes:
+ *      does NOT produce WILD tokens at the moment
+ * ========================================================================= */
+ 
+/* [RPAP - Jan 97: Stem Index Change]
+   state mode:
+      0 = Read next token
+      1 = Output word
+      2 = Output '|' or ')'
+ */
+static int query_lex(char **ptr, const char *end)
+{
+  char *buf_ptr = *ptr;
+  static int mode = 0;
+  static int termnum = 0;
+  static TermList *Terms = NULL;
+
+  if (mode == 0)
+    {
+      /* jump over whitespace */
+      while (isspace(*buf_ptr))
+	buf_ptr++;
+ 
+      if (inaword(buf_ptr, end))
+	{
+	  static char word[MAXSTEMLEN + 1]; /* [RJM 07/98: Memory Leak] */
+	  char *sWord = Xmalloc(MAXSTEMLEN + 1);
+	  int stem_to_apply, method_using = -1;
+
+	  PARSE_STEM_WORD(word, buf_ptr, end);
+
+	  /* Extract any parameters */
+	  stem_to_apply = stem_method;
+	  while (buf_ptr <= end)
+	    {
+	      int stem_param, param_type;
+	      char param[MAXPARAMLEN + 1];
+
+	      param_type = 0;
+	      PARSE_OPT_TERM_PARAM (param, param_type, buf_ptr, end);
+	      if (!param_type)
+		break;
+
+	      if (param_type == STEMPARAM)
+		{
+		  stem_param = atoi (param);
+		  if (errno != ERANGE && indexed && stem_param >= 0 && stem_param <= 3)
+		    method_using = stem_to_apply = stem_param;
+		}
+	    }
+
+	  bcopy ((char *) word, (char *) sWord, *word + 1);
+	  stemmer (stem_to_apply, stemmer_num, sWord);
+
+	  if (stem_to_apply == 0 || !indexed || p__sd == NULL)
+	    {
+	      /* [RPAP - Feb 97: Term Frequency] */
+	      word_num = FindWord (p__sd, sWord, &count, &doc_count, &invf_ptr, &invf_len);
+	      if (word_num == -1)
+		count = doc_count = invf_ptr = invf_len = 0;
+	      AddQueryTerm (query_term_list, (u_char *) word, count, method_using);
+
+	      yylval.text = word;
+	      *ptr = buf_ptr; /* fix up ptr */
+	      Xfree (sWord);
+	      return TERM;
+	    }
+	  else
+	    {
+	      *ptr = buf_ptr; /* fix up ptr */
+	      termnum = 0;
+	      ResetTermList (&Terms);
+	      if (FindWords (p__sd, (u_char *) sWord, stem_to_apply, &Terms) > 0)
+		{
+		  /* [RPAP - Feb 97: Term Frequency] */ 
+		  int i, freq = 0;
+		  for (i = 0; i < Terms->num; i++)
+		    freq += Terms->TE[i].WE.count;
+		  AddQueryTerm (query_term_list, word, freq, method_using);
+
+		  Xfree (sWord);
+		  mode = 1;
+		  return '(';
+		}
+	      else
+		{
+		  /* Word does not exists - include in tree anyway */
+		  Xfree (sWord);
+
+		  /* [RPAP - Feb 97: Term Frequency] */
+		  word_num = -1;
+		  count = doc_count = invf_ptr = invf_len = 0;
+		  AddQueryTerm (query_term_list, (u_char *) word, count, method_using);
+
+		  yylval.text = word;
+		  return TERM;
+		}
+	    }
+	}
+      else /* NON-WORD */
+	{
+	  if (*buf_ptr == '\0')
+	    {
+	      /* return null-char if it is one */
+	      *ptr = buf_ptr; /* fix up ptr */
+	      return 0;
+	    }
+	  else
+	    {
+	      /* return 1st char, and delete from buffer */
+	      char c = *buf_ptr++;
+	      *ptr = buf_ptr; /* fix up ptr */
+	      return c;
+	    }
+	}
+    }
+  else if (mode == 1)
+    {
+      yylval.text = Terms->TE[termnum].Word;
+      
+      /* [RPAP - Feb 97: Term Frequency] */
+      word_num = Terms->TE[termnum].WE.word_num;
+      count = Terms->TE[termnum].WE.count;
+      doc_count = Terms->TE[termnum].WE.doc_count;
+      invf_ptr = Terms->TE[termnum].WE.invf_ptr;
+      invf_len = Terms->TE[termnum].WE.invf_len;
+
+      termnum++;
+      mode = 2;
+      return TERM;
+    }
+  else  /* mode == 2 */
+    {
+      if (termnum >= Terms->num)
+	{
+	  mode = 0;
+	  return ')';
+	}
+      else
+	{
+	  mode = 1;
+	  return '|';
+	}
+    }
+}/*query_lex*/
+
+/* =========================================================================
+ * Function: yyerror
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */ 
+static int yyerror(char *s)
+{
+  Message("%s", s);
+  return(1);
+}
+
+ 
+/* =========================================================================
+ * Function: ParseBool
+ * Description:
+ *      Parse a boolean query string into a term-list and a boolean parse tree
+ * Input:
+ *      query_line = query line string
+ *      query_len = query line length
+ *      the_stem_method = stem method id used for stemming
+ * Output:
+ *      the_term_list = the list of terms
+ *      res = parser result code
+ * ========================================================================= */
+ 
+bool_tree_node *
+ParseBool(char *query_line, int query_len,
+          TermList **the_term_list, int the_stemmer_num, int the_stem_method, 
+	  int *res,
+	  stemmed_dict * the_sd, int is_indexed,   /* [RPAP - Jan 97: Stem Index Change] */
+	  QueryTermList **the_query_term_list)  /* [RPAP - Feb 97: Term Frequency] */
+{
+  /* global variables to be accessed by bison/yacc created parser */
+  term_list = the_term_list;
+  stemmer_num = the_stemmer_num;
+  stem_method = the_stem_method;
+  ch_buf = query_line;
+  end_buf = query_line + query_len;
+  p__sd = the_sd;   /* [RPAP - Jan 97: Stem Index Change] */
+  indexed = is_indexed;  /* [RPAP - Jan 97: Stem Index Change] */
+  query_term_list = the_query_term_list; /* [RPAP - Feb 97: Term Frequency] */
+
+  FreeBoolTree(&(tree_base));
+ 
+  ResetTermList(term_list);
+  ResetQueryTermList(query_term_list);  /* [RPAP - Feb 97: Term Frequency] */
+
+  *res = yyparse();
+ 
+  return tree_base;
+}
+ 
+
+#line 407 "y.tab.c"
+#define YYABORT goto yyabort
+#define YYREJECT goto yyabort
+#define YYACCEPT goto yyaccept
+#define YYERROR goto yyerrlab
+int
+yyparse()
+{
+    register int yym, yyn, yystate;
+#if YYDEBUG
+    register char *yys;
+    extern char *getenv();
+
+    if (yys = getenv("YYDEBUG"))
+    {
+        yyn = *yys;
+        if (yyn >= '0' && yyn <= '9')
+            yydebug = yyn - '0';
+    }
+#endif
+
+    yynerrs = 0;
+    yyerrflag = 0;
+    yychar = (-1);
+
+    yyssp = yyss;
+    yyvsp = yyvs;
+    *yyssp = yystate = 0;
+
+yyloop:
+    if (yyn = yydefred[yystate]) goto yyreduce;
+    if (yychar < 0)
+    {
+        if ((yychar = yylex()) < 0) yychar = 0;
+#if YYDEBUG
+        if (yydebug)
+        {
+            yys = 0;
+            if (yychar <= YYMAXTOKEN) yys = yyname[yychar];
+            if (!yys) yys = "illegal-symbol";
+            printf("%sdebug: state %d, reading %d (%s)\n",
+                    YYPREFIX, yystate, yychar, yys);
+        }
+#endif
+    }
+    if ((yyn = yysindex[yystate]) && (yyn += yychar) >= 0 &&
+            yyn <= YYTABLESIZE && yycheck[yyn] == yychar)
+    {
+#if YYDEBUG
+        if (yydebug)
+            printf("%sdebug: state %d, shifting to state %d\n",
+                    YYPREFIX, yystate, yytable[yyn]);
+#endif
+        if (yyssp >= yyss + yystacksize - 1)
+        {
+            goto yyoverflow;
+        }
+        *++yyssp = yystate = yytable[yyn];
+        *++yyvsp = yylval;
+        yychar = (-1);
+        if (yyerrflag > 0)  --yyerrflag;
+        goto yyloop;
+    }
+    if ((yyn = yyrindex[yystate]) && (yyn += yychar) >= 0 &&
+            yyn <= YYTABLESIZE && yycheck[yyn] == yychar)
+    {
+        yyn = yytable[yyn];
+        goto yyreduce;
+    }
+    if (yyerrflag) goto yyinrecovery;
+#ifdef lint
+    goto yynewerror;
+#endif
+yynewerror:
+    yyerror("syntax error");
+#ifdef lint
+    goto yyerrlab;
+#endif
+yyerrlab:
+    ++yynerrs;
+yyinrecovery:
+    if (yyerrflag < 3)
+    {
+        yyerrflag = 3;
+        for (;;)
+        {
+            if ((yyn = yysindex[*yyssp]) && (yyn += YYERRCODE) >= 0 &&
+                    yyn <= YYTABLESIZE && yycheck[yyn] == YYERRCODE)
+            {
+#if YYDEBUG
+                if (yydebug)
+                    printf("%sdebug: state %d, error recovery shifting\
+ to state %d\n", YYPREFIX, *yyssp, yytable[yyn]);
+#endif
+                if (yyssp >= yyss + yystacksize - 1)
+                {
+                    goto yyoverflow;
+                }
+                *++yyssp = yystate = yytable[yyn];
+                *++yyvsp = yylval;
+                goto yyloop;
+            }
+            else
+            {
+#if YYDEBUG
+                if (yydebug)
+                    printf("%sdebug: error recovery discarding state %d\n",
+                            YYPREFIX, *yyssp);
+#endif
+                if (yyssp <= yyss) goto yyabort;
+                --yyssp;
+                --yyvsp;
+            }
+        }
+    }
+    else
+    {
+        if (yychar == 0) goto yyabort;
+#if YYDEBUG
+        if (yydebug)
+        {
+            yys = 0;
+            if (yychar <= YYMAXTOKEN) yys = yyname[yychar];
+            if (!yys) yys = "illegal-symbol";
+            printf("%sdebug: state %d, error recovery discards token %d (%s)\n",
+                    YYPREFIX, yystate, yychar, yys);
+        }
+#endif
+        yychar = (-1);
+        goto yyloop;
+    }
+yyreduce:
+#if YYDEBUG
+    if (yydebug)
+        printf("%sdebug: state %d, reducing by rule %d (%s)\n",
+                YYPREFIX, yystate, yyn, yyrule[yyn]);
+#endif
+    yym = yylen[yyn];
+    yyval = yyvsp[1-yym];
+    switch (yyn)
+    {
+case 1:
+#line 75 "bool_parser.y"
+{ tree_base = yyvsp[0].node;}
+break;
+case 2:
+#line 79 "bool_parser.y"
+{ yyval.node = CreateBoolTermNode(term_list, yyvsp[0].text, 1, word_num, count, doc_count, invf_ptr, invf_len, stemmer_num); }
+break;
+case 3:
+#line 80 "bool_parser.y"
+{ yyval.node = yyvsp[-1].node; }
+break;
+case 4:
+#line 81 "bool_parser.y"
+{ yyval.node = CreateBoolTreeNode(N_all, NULL, NULL); }
+break;
+case 5:
+#line 82 "bool_parser.y"
+{ yyval.node = CreateBoolTreeNode(N_none, NULL, NULL); }
+break;
+case 7:
+#line 86 "bool_parser.y"
+{ yyval.node = CreateBoolTreeNode(N_not, yyvsp[0].node, NULL); }
+break;
+case 8:
+#line 89 "bool_parser.y"
+{ yyval.node = CreateBoolTreeNode(N_and, yyvsp[-2].node, yyvsp[0].node); }
+break;
+case 9:
+#line 90 "bool_parser.y"
+{ yyval.node = CreateBoolTreeNode(N_and, yyvsp[-1].node, yyvsp[0].node); }
+break;
+case 11:
+#line 94 "bool_parser.y"
+{ yyval.node = CreateBoolTreeNode(N_or, yyvsp[-2].node, yyvsp[0].node); }
+break;
+#line 584 "y.tab.c"
+    }
+    yyssp -= yym;
+    yystate = *yyssp;
+    yyvsp -= yym;
+    yym = yylhs[yyn];
+    if (yystate == 0 && yym == 0)
+    {
+#if YYDEBUG
+        if (yydebug)
+            printf("%sdebug: after reduction, shifting from state 0 to\
+ state %d\n", YYPREFIX, YYFINAL);
+#endif
+        yystate = YYFINAL;
+        *++yyssp = YYFINAL;
+        *++yyvsp = yyval;
+        if (yychar < 0)
+        {
+            if ((yychar = yylex()) < 0) yychar = 0;
+#if YYDEBUG
+            if (yydebug)
+            {
+                yys = 0;
+                if (yychar <= YYMAXTOKEN) yys = yyname[yychar];
+                if (!yys) yys = "illegal-symbol";
+                printf("%sdebug: state %d, reading %d (%s)\n",
+                        YYPREFIX, YYFINAL, yychar, yys);
+            }
+#endif
+        }
+        if (yychar == 0) goto yyaccept;
+        goto yyloop;
+    }
+    if ((yyn = yygindex[yym]) && (yyn += yystate) >= 0 &&
+            yyn <= YYTABLESIZE && yycheck[yyn] == yystate)
+        yystate = yytable[yyn];
+    else
+        yystate = yydgoto[yym];
+#if YYDEBUG
+    if (yydebug)
+        printf("%sdebug: after reduction, shifting from state %d \
+to state %d\n", YYPREFIX, *yyssp, yystate);
+#endif
+    if (yyssp >= yyss + yystacksize - 1)
+    {
+        goto yyoverflow;
+    }
+    *++yyssp = yystate;
+    *++yyvsp = yyval;
+    goto yyloop;
+yyoverflow:
+    yyerror("yacc stack overflow");
+yyabort:
+    return (1);
+yyaccept:
+    return (0);
+}
+
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/bool_parser.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/bool_parser.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/bool_parser.h	(revision 699)
@@ -0,0 +1,67 @@
+/**************************************************************************
+ *
+ * filename -- description
+ * Copyright (C) 1994  Authors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:17:43  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:40  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:34:26  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/10/20  03:56:33  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.1  1994/10/12  01:15:31  tes
+   * Found bugs in the existing boolean query optimiser.
+   * So decided to rewrite it.
+   * I accidentally deleted query.bool.y, but I have replaced it
+   * with bool_parser.y (which I have forgotten to add here ! ;-(
+   *
+ */
+
+#ifndef BOOL_PARSER_H
+#define BOOL_PARSER_H
+
+#include "backend.h"
+#include "term_lists.h"
+#include "invf.h"
+
+bool_tree_node *ParseBool (char *query_line, int query_len,
+			   TermList ** the_term_list, int the_stemmer_num, 
+			   int the_stem_method, int *res,
+			   stemmed_dict * the_sd, int is_indexed,  /* [RPAP - Jan 97: Stem Index Change] */
+			   QueryTermList ** the_query_term_list);  /* [RPAP - Feb 97: Term Frequency] */
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/bool_parser.y
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/bool_parser.y	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/bool_parser.y	(revision 699)
@@ -0,0 +1,315 @@
+/**************************************************************************
+ *
+ * bool_parser - boolean query parser
+ * Copyright (C) 1994  Neil Sharman & Tim Shimmin
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *
+ **************************************************************************/
+ 
+/**************************************************************************/
+%{
+ 
+#include "sysfuncs.h"
+
+#include "messages.h"
+ 
+#include "memlib.h"
+#include "words.h"
+#include "stemmer.h"
+#include "term_lists.h"
+#include "bool_tree.h"
+/* [RPAP - Jan 97: Stem Index Change] */
+#include "backend.h"     /* for stemmed_dict def */
+#include "stem_search.h"
+
+#include "query_term_list.h"  /* [RPAP - Feb 97: Term Frequency] */
+
+/* --- routines --- */
+static int query_lex();
+static int yyerror(char *);
+#define yylex() query_lex(&ch_buf, end_buf)
+ 
+/* --- module variables --- */
+static char *ch_buf; /* ptr to the character query line buffer */
+static char *end_buf; /* ptr to the last character of the line buffer */
+static bool_tree_node *tree_base = NULL;
+static TermList **term_list;
+static int stem_method;
+/* [RPAP - Jan 97: Stem Index Change] */
+stemmed_dict *p__sd;
+static int indexed;
+/* [RPAP - Feb 97: Term Frequency] */
+static QueryTermList **query_term_list;
+static int word_num;
+static u_long count;
+static u_long doc_count;
+static u_long invf_ptr;
+static u_long invf_len;
+%}
+ 
+ 
+%union {
+  char *text;
+  bool_tree_node *node;
+}
+ 
+%token <text> TERM
+%type <node> query term not and or
+ 
+%%
+ 
+query: or  { tree_base = $1;}
+;
+ 
+ 
+term:     TERM  { $$ = CreateBoolTermNode(term_list, $1, 1, word_num, count, doc_count, invf_ptr, invf_len); }
+        | '(' or ')' { $$ = $2; }
+        | '*' { $$ = CreateBoolTreeNode(N_all, NULL, NULL); }
+        | '_' { $$ = CreateBoolTreeNode(N_none, NULL, NULL); }
+;
+ 
+not:      term
+        | '!' not { $$ = CreateBoolTreeNode(N_not, $2, NULL); }
+;
+ 
+and:      and '&' not { $$ = CreateBoolTreeNode(N_and, $1, $3); }
+        | and not { $$ = CreateBoolTreeNode(N_and, $1, $2); }
+        | not
+;
+ 
+or:       or '|' and { $$ = CreateBoolTreeNode(N_or, $1, $3); }
+        | and
+;
+ 
+%%
+ 
+/* Bison on one mips machine defined "const" to be nothing but
+   then did not undef it */
+#ifdef const
+#undef const
+#endif
+
+/**************************************************************************/ 
+
+
+/* =========================================================================
+ * Function: query_lex
+ * Description:
+ *      Hand written lexical analyser for the parser.
+ * Input:
+ *      ptr = ptr to a ptr into character query-line buffer
+ *      end = ptr to last char in buffer
+ * Output:
+ *      yylval.text = the token's text
+ * Notes:
+ *      does NOT produce WILD tokens at the moment
+ * ========================================================================= */
+ 
+/* [RPAP - Jan 97: Stem Index Change]
+   state mode:
+      0 = Read next token
+      1 = Output word
+      2 = Output '|' or ')'
+ */
+static int query_lex(char **ptr, const char *end)
+{
+  char *buf_ptr = *ptr;
+  static int mode = 0;
+  static int termnum = 0;
+  static TermList *Terms = NULL;
+
+  if (mode == 0)
+    {
+      /* jump over whitespace */
+      while (isspace(*buf_ptr))
+	buf_ptr++;
+ 
+      if (INAWORD(*buf_ptr))
+	{
+	  static char word[MAXSTEMLEN + 1]; /* [RJM 07/98: Memory Leak] */
+	  char *sWord = Xmalloc(MAXSTEMLEN + 1);
+	  int stem_to_apply, method_using = -1;
+
+	  PARSE_STEM_WORD(word, buf_ptr, end);
+
+	  /* Extract any parameters */
+	  stem_to_apply = stem_method;
+	  while (buf_ptr <= end)
+	    {
+	      int stem_param, param_type;
+	      char param[MAXPARAMLEN + 1];
+
+	      param_type = 0;
+	      PARSE_OPT_TERM_PARAM (param, param_type, buf_ptr, end);
+	      if (!param_type)
+		break;
+
+	      if (param_type == STEMPARAM)
+		{
+		  stem_param = atoi (param);
+		  if (errno != ERANGE && indexed && stem_param >= 0 && stem_param <= 3)
+		    method_using = stem_to_apply = stem_param;
+		}
+	    }
+
+	  bcopy ((char *) word, (char *) sWord, *word + 1);
+	  stemmer (stem_to_apply, sWord);
+
+	  if (stem_to_apply == 0 || !indexed || p__sd == NULL)
+	    {
+	      /* [RPAP - Feb 97: Term Frequency] */
+	      word_num = FindWord (p__sd, sWord, &count, &doc_count, &invf_ptr, &invf_len);
+	      if (word_num == -1)
+		count = doc_count = invf_ptr = invf_len = 0;
+	      AddQueryTerm (query_term_list, (u_char *) word, count, method_using);
+
+	      yylval.text = word;
+	      *ptr = buf_ptr; /* fix up ptr */
+	      Xfree (sWord);
+	      return TERM;
+	    }
+	  else
+	    {
+	      *ptr = buf_ptr; /* fix up ptr */
+	      termnum = 0;
+	      ResetTermList (&Terms);
+	      if (FindWords (p__sd, (u_char *) sWord, stem_to_apply, &Terms) > 0)
+		{
+		  /* [RPAP - Feb 97: Term Frequency] */ 
+		  int i, freq = 0;
+		  for (i = 0; i < Terms->num; i++)
+		    freq += Terms->TE[i].WE.count;
+		  AddQueryTerm (query_term_list, word, freq, method_using);
+
+		  Xfree (sWord);
+		  mode = 1;
+		  return '(';
+		}
+	      else
+		{
+		  /* Word does not exists - include in tree anyway */
+		  Xfree (sWord);
+
+		  /* [RPAP - Feb 97: Term Frequency] */
+		  word_num = -1;
+		  count = doc_count = invf_ptr = invf_len = 0;
+		  AddQueryTerm (query_term_list, (u_char *) word, count, method_using);
+
+		  yylval.text = word;
+		  return TERM;
+		}
+	    }
+	}
+      else /* NON-WORD */
+	{
+	  if (*buf_ptr == '\0')
+	    {
+	      /* return null-char if it is one */
+	      *ptr = buf_ptr; /* fix up ptr */
+	      return 0;
+	    }
+	  else
+	    {
+	      /* return 1st char, and delete from buffer */
+	      char c = *buf_ptr++;
+	      *ptr = buf_ptr; /* fix up ptr */
+	      return c;
+	    }
+	}
+    }
+  else if (mode == 1)
+    {
+      yylval.text = Terms->TE[termnum].Word;
+      
+      /* [RPAP - Feb 97: Term Frequency] */
+      word_num = Terms->TE[termnum].WE.word_num;
+      count = Terms->TE[termnum].WE.count;
+      doc_count = Terms->TE[termnum].WE.doc_count;
+      invf_ptr = Terms->TE[termnum].WE.invf_ptr;
+      invf_len = Terms->TE[termnum].WE.invf_len;
+
+      termnum++;
+      mode = 2;
+      return TERM;
+    }
+  else  /* mode == 2 */
+    {
+      if (termnum >= Terms->num)
+	{
+	  mode = 0;
+	  return ')';
+	}
+      else
+	{
+	  mode = 1;
+	  return '|';
+	}
+    }
+}/*query_lex*/
+
+/* =========================================================================
+ * Function: yyerror
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */ 
+static int yyerror(char *s)
+{
+  Message("%s", s);
+  return(1);
+}
+
+ 
+/* =========================================================================
+ * Function: ParseBool
+ * Description:
+ *      Parse a boolean query string into a term-list and a boolean parse tree
+ * Input:
+ *      query_line = query line string
+ *      query_len = query line length
+ *      the_stem_method = stem method id used for stemming
+ * Output:
+ *      the_term_list = the list of terms
+ *      res = parser result code
+ * ========================================================================= */
+ 
+bool_tree_node *
+ParseBool(char *query_line, int query_len,
+          TermList **the_term_list, int the_stem_method, int *res,
+	  stemmed_dict * the_sd, int is_indexed,   /* [RPAP - Jan 97: Stem Index Change] */
+	  QueryTermList **the_query_term_list)  /* [RPAP - Feb 97: Term Frequency] */
+{
+  /* global variables to be accessed by bison/yacc created parser */
+  term_list = the_term_list;
+  stem_method = the_stem_method;
+  ch_buf = query_line;
+  end_buf = query_line + query_len;
+  p__sd = the_sd;   /* [RPAP - Jan 97: Stem Index Change] */
+  indexed = is_indexed;  /* [RPAP - Jan 97: Stem Index Change] */
+  query_term_list = the_query_term_list; /* [RPAP - Feb 97: Term Frequency] */
+
+  FreeBoolTree(&(tree_base));
+ 
+  ResetTermList(term_list);
+  ResetQueryTermList(query_term_list);  /* [RPAP - Feb 97: Term Frequency] */
+
+  *res = yyparse();
+ 
+  return tree_base;
+}
+ 
+
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/bool_query.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/bool_query.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/bool_query.c	(revision 699)
@@ -0,0 +1,353 @@
+/**************************************************************************
+ *
+ * bool_query.c -- Boolean query evaluation
+ * Copyright (C) 1994  Neil Sharman & Tim Shimmin
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "filestats.h"
+#include "timing.h"
+#include "sptree.h"
+
+
+#include "mg.h"
+#include "text.h"
+#include "invf.h"
+#include "lists.h"
+#include "term_lists.h"
+#include "backend.h"
+#include "words.h"
+#include "stemmer.h"
+#include "stem_search.h"
+#include "invf_get.h"
+#include "text_get.h"
+#include "read_line.h"
+#include "bool_tree.h"
+#include "bool_parser.h"
+#include "bool_optimiser.h"
+#include "environment.h"
+
+
+/**********************
+  [RPAP - Feb 97: Term Frequency]
+
+  Shifted to bool_parser.y
+
+  *********************/
+
+/* =========================================================================
+ * Function: ReadInTermInfo
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+/**************************
+static void 
+ReadInTermInfo (query_data * qd)
+{
+  int i;
+  TermList *tl = qd->TL;
+
+  for (i = 0; i < tl->num; i++)
+    {
+      TermEntry *te = &(tl->TE[i]);
+      WordEntry *we = &(te->WE);
+
+       we->word_num = FindWord (qd->sd, te->Word,
+			       &we->count, &we->doc_count,
+			       &we->invf_ptr, &we->invf_len);
+      if (we->word_num == -1)
+	we->count = we->doc_count = 0;
+    }
+}
+
+***********************/
+
+
+
+/* =========================================================================
+ * Function: FindNoneTerms 
+ * Description: 
+ *      Set any terms which are not in dictionary to N_none (i.e. false)
+ *      Assumes a prior call to ReadInTermInfo
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void 
+FindNoneTerms (query_data * qd, bool_tree_node * tree)
+{
+  if (BOOL_HAS_CHILD (tree))
+    {
+      bool_tree_node *child = BOOL_CHILD (tree);
+      while (child)
+	{
+	  FindNoneTerms (qd, child);
+	  child = BOOL_SIBLING (child);
+	}
+    }
+  else if (BOOL_TAG (tree) == N_term)
+    {
+      int doc_count = qd->TL->TE[BOOL_TERM (tree)].WE.doc_count;
+      if (!doc_count)
+	BOOL_TAG (tree) = N_none;
+    }
+
+}				/*FindNoneTerms */
+
+/* =========================================================================
+ * Function: ReduceList
+ * Description: 
+ *      Go thru doc-set and get rid of all docs which are unmarked
+ *      The motivation for this routine is in the case of CNF queries.
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+void
+ReduceList (DocList * cset)
+{
+  DocEntry *src;
+  DocEntry *dest;
+
+  src = dest = cset->DE;
+  for (src = dest = cset->DE; src < cset->DE + cset->num; src++)
+    {
+      if (src->or_included)
+	{
+	  src->or_included = 0;
+	  *dest++ = *src;
+	}
+    }
+  cset->num = dest - cset->DE;
+
+}				/*ReduceList */
+
+/* =========================================================================
+ * Function: BooleanGet
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+static DocList *
+BooleanGet (query_data * qd, bool_tree_node * n,
+	    BooleanQueryInfo * bqi)
+{
+  if (n)
+    {
+      DocList *res = NULL;
+      switch (BOOL_TAG (n))
+	{
+	case N_all:
+	  Message ("Unexpected \"all\" node in the parse tree.");
+	  res = MakeDocList (0);
+	  break;
+	case N_not:
+	  Message ("Unexpected \"not\" node in the parse tree.");
+	  res = MakeDocList (0);
+	  break;
+	case N_none:
+	  {
+	    res = MakeDocList (0);
+	    break;
+	  }
+	case N_diff:
+	  {
+	    res = BooleanGet (qd, BOOL_CHILD (n), bqi);
+	    n = BOOL_SIBLING (BOOL_CHILD (n));
+	    while (n)
+	      {
+		bool_tree_node *next = BOOL_SIBLING (n);
+		if (BOOL_TAG (n) == N_term)
+		  {
+		    WordEntry *we = GetNthWE (qd->TL, BOOL_TERM (n));
+		    res = GetDocsOp (qd, we, op_diff_terms, res);
+		  }
+		else
+		  res = DiffLists (res, BooleanGet (qd, n, bqi));
+		n = next;
+	      }
+	    break;
+	  }
+	case N_and:
+	  {
+	    res = BooleanGet (qd, BOOL_CHILD (n), bqi);		/* initial fill of c-set */
+	    n = BOOL_SIBLING (BOOL_CHILD (n));
+	    while (n)
+	      {
+		bool_tree_node *next = BOOL_SIBLING (n);
+		switch (BOOL_TAG (n))
+		  {
+		  case N_term:
+		    {
+		      WordEntry *we = GetNthWE (qd->TL, BOOL_TERM (n));
+		      res = GetDocsOp (qd, we, op_and_terms, res);
+		    }
+		    break;
+		  case N_or_terms:
+		    {
+		      bool_tree_node *child = BOOL_CHILD (n);
+		      /* note: all children are terms - predetermined */
+		      while (child)
+			{
+			  WordEntry *we = GetNthWE (qd->TL, BOOL_TERM (child));
+			  res = GetDocsOp (qd, we, op_and_or_terms, res);
+			  child = BOOL_SIBLING (child);
+			}
+		      ReduceList (res);
+		    }
+		    break;
+		  default:
+		    res = IntersectLists (res, BooleanGet (qd, n, bqi));
+		  }		/*switch */
+		n = next;
+	      }
+	    break;
+	  }
+	case N_or:
+	case N_or_terms:
+	  {
+	    res = BooleanGet (qd, BOOL_CHILD (n), bqi);
+	    n = BOOL_SIBLING (BOOL_CHILD (n));
+	    while (n)
+	      {
+		bool_tree_node *next = BOOL_SIBLING (n);
+		res = MergeLists (res, BooleanGet (qd, n, bqi));
+		n = next;
+	      }
+	    break;
+	  }
+	case N_term:
+	  {
+	    res = GetDocsOp (qd, GetNthWE (qd->TL, BOOL_TERM (n)), op_term, NULL);
+	    break;
+	  }
+	}
+      return (res ? res : MakeDocList (0));
+    }
+  else
+    return MakeDocList (0);
+}
+
+/* =========================================================================
+ * Function: AdjustParaDocs
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+static int 
+DE_comp (void *a, void *b)
+{
+  return ((DocEntry *) a)->SeekPos - ((DocEntry *) b)->SeekPos;
+}
+
+static void
+AdjustParaDocs (query_data * qd, DocList * Docs)
+{
+  DocEntry *DE = Docs->DE;
+  int s, d;
+  Splay_Tree *SP;
+
+
+  /* Make the doc-list contain only references to unique
+   * real documents. So can get rid of cases where there
+   * are multiple paragraphs for a real document.
+   */
+
+  /* Note: [TS: 29/Sep/94] */
+  /* reads in seek-pos for docnums so can compare docs */
+  /* seek-pos used as real doc identifier */
+  /* whereas doc-num entries refer to paragraphs -- I think */
+  for (s = 0; s < Docs->num; s++)
+    FetchDocStart (qd, DE[s].DocNum, &DE[s].SeekPos, &DE[s].Len);
+
+  Message ("Original Number = %d\n", Docs->num);
+  SP = SP_createset (DE_comp);
+  d = 0;
+  for (s = 0; s < Docs->num; s++)
+    if (!SP_member (&DE[s], SP))
+      {
+	DE[d] = DE[s];
+	SP_insert (&DE[d], SP);
+	d++;
+      }
+  SP_freeset (SP);
+  Docs->num = d;
+  Message ("Final Number = %d\n", Docs->num);
+}
+
+/* =========================================================================
+ * Function: BooleanQuery 
+ * Description: 
+ *      The main control function for the boolean query.
+ * Input: 
+ * Output: 
+ *      qd->DL
+ * ========================================================================= */
+
+void 
+BooleanQuery (query_data * qd, char *Query, BooleanQueryInfo * bqi,
+	      int stem_method)
+{
+  bool_tree_node *tree;
+  DocList *Docs;
+  int res = 0;
+
+  tree = ParseBool (Query, MAXLINEBUFFERLEN, &(qd->TL), 
+		    qd->sd->sdh.stemmer_num, stem_method, &res,
+		    qd->sd, qd->sd->sdh.indexed,         /* [RPAP - Jan 97: Stem Index Change] */
+		    &(qd->QTL));                         /* [RPAP - Feb 97: Term Frequency] */
+
+  /* [TS:Aug/94] no longer tries to continue after parse failure */
+  if (res != 0)			/* failed the parse */
+    {
+      qd->DL = NULL;		/* have empty doc-list */
+      return;
+    }
+
+  /* [RPAP - Feb 97: Term Frequency] */
+  /*  ReadInTermInfo (qd); */
+
+  FindNoneTerms (qd, tree);
+
+  OptimiseBoolTree (tree, qd->TL, atoi (GetEnv ("optimise_type")));
+
+  qd->hops_taken = qd->num_of_ptrs = 0;
+  qd->num_of_terms = 0;
+
+  Docs = BooleanGet (qd, tree, bqi);
+
+/* [RPAP - Feb 97: NZDL Specifics] */
+#if !defined(PARABOOLEANMATCHES) && !defined(NZDL)
+  if (qd->id->ifh.InvfLevel == 3)
+    {
+      AdjustParaDocs (qd, Docs);
+    }
+#endif
+
+  if (bqi->MaxDocsToRetrieve != -1 && Docs->num > bqi->MaxDocsToRetrieve)
+    Docs->num = bqi->MaxDocsToRetrieve;
+
+  FreeQueryDocs (qd);
+
+  qd->DL = Docs;
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/bool_query.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/bool_query.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/bool_query.h	(revision 699)
@@ -0,0 +1,52 @@
+/**************************************************************************
+ *
+ * bool_query.h -- boolean query processing
+ * Copyright (C) 1994  Authors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:17:45  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:34:28  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/10/20  03:56:36  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.1  1994/10/12  01:15:33  tes
+   * Found bugs in the existing boolean query optimiser.
+   * So decided to rewrite it.
+   * I accidentally deleted query.bool.y, but I have replaced it
+   * with bool_parser.y (which I have forgotten to add here ! ;-(
+   *
+ */
+
+#ifndef BOOL_QUERY_H
+#define BOOL_QUERY_H
+
+#include "backend.h"
+
+void BooleanQuery (query_data * qd, char *Query, BooleanQueryInfo * bqi,
+		   int stem_method);
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/bool_tester.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/bool_tester.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/bool_tester.c	(revision 699)
@@ -0,0 +1,171 @@
+/**************************************************************************
+ *
+ * bool_tester -- used to test bool_tree, bool_parser, bool_optimiser
+ * Copyright (C) 1994  Authors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:17:45  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:41  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:34:29  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.2  1995/03/14  05:15:23  tes
+   * Updated the boolean query optimiser to do different types of optimisation.
+   * A query environment variable "optimise_type" specifies which one is to be
+   * used. Type 1 is the new one which is faster than 2.
+   *
+   * Revision 1.2  1994/10/18  06:11:05  tes
+   * The boolean optimiser seems to be modifying the parse tree
+   * like it is supposed to.
+   * Paragraph ranking now works without any text files if required to.
+   *
+   * Revision 1.1  1994/10/12  01:15:33  tes
+   * Found bugs in the existing boolean query optimiser.
+   * So decided to rewrite it.
+   * I accidentally deleted query.bool.y, but I have replaced it
+   * with bool_parser.y (which I have forgotten to add here ! ;-(
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+#include "local_strings.h"
+#include "term_lists.h"
+#include "bool_tree.h"
+#include "bool_parser.h"
+#include "bool_optimiser.h"
+#include "query_term_list.h"  /* [RPAP - Feb 97: Term Frequency] */
+
+#define MAX_LINE_LEN  255
+#define STEMMER_NUM 0 /* Lovin's stemmer */
+#define STEM_METHOD 3
+
+static FILE *file_in = stdin;
+static FILE *file_out = stdout;
+static char line[MAX_LINE_LEN + 1];
+
+/* --- prototypes --- */
+static char *prompt (char *str);
+
+
+/* =========================================================================
+ * Function: main
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+int main (int argc, char *argv[])
+{
+  bool_tree_node *tree = NULL;
+  TermList *term_list = NULL;
+  QueryTermList *query_term_list = NULL;  /* [RPAP - Feb 97: Term Frequency] */
+  int opt_type = 0;
+
+  while (1)
+    {
+      int res = 0;
+      int len = 0;
+
+      /* get a line */
+      if (!prompt ("Please enter boolean expression..."))
+	break;
+
+      len = strlen (line) - 1;	/* -1 => ignore the \n */
+
+      tree = ParseBool (line, len, &term_list, STEMMER_NUM, STEM_METHOD, &res,
+			NULL, 0,             /* [RPAP - Jan 97: Stem Index Change] */
+			&query_term_list);   /* [RPAP - Feb 97: Term Frequency] */
+
+      {
+	int done = 0;
+	while (!done)
+	  {
+	    if (!prompt ("Which type of optimisation 1 or 2 ?"))
+	      break;
+	    opt_type = atoi (line);
+	    done = (opt_type == 1 || opt_type == 2);
+	  }
+      }
+
+      if (!prompt ("Do you want to assign doc_counts to terms ?"))
+	break;
+
+      if (toupper (line[0]) == 'Y')
+	{
+	  /* cycle thru terms asking for doc_counts */
+	  int i;
+	  for (i = 0; i < term_list->num; i++)
+	    {
+	      char str[80];
+	      TermEntry *te = &(term_list->TE[i]);
+
+	      sprintf (str, "Please enter doc count for term '%s'",
+		       str255_to_string (te->Word, NULL));
+	      prompt (str);
+	      te->WE.doc_count = atoi (line);
+	      printf ("doc_count = %ld\n", te->WE.doc_count);
+	    }			/*for */
+	}			/*if */
+
+
+      if (res == 0)
+	{
+	  fprintf (file_out, "\n***Parsed Expression***\n");
+	  PrintBoolTree (tree, file_out);
+	  fputc ('\n', file_out);
+	  OptimiseBoolTree (tree, term_list, opt_type);
+	  fprintf (file_out, "\n***Optimised Expression ***\n");
+	  PrintBoolTree (tree, file_out);
+	  fputc ('\n', file_out);
+	}
+
+    }
+  return 0;
+}
+
+/* =========================================================================
+ * Function: 
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static char *
+prompt (char *str)
+{
+  fprintf (file_out, "\n%s\n", str);
+  return fgets (line, MAX_LINE_LEN, file_in);
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/bool_tree.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/bool_tree.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/bool_tree.c	(revision 699)
@@ -0,0 +1,292 @@
+/**************************************************************************
+ *
+ * bool_tree.c -- Boolean parse tree ADT
+ * Copyright (C) 1994  Tim Shimmin
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *       @(#)query.bool.y	1.9 16 Mar 1994
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "bool_tree.h"
+
+static char *RCSID = "$Id$";
+
+
+/* =========================================================================
+ * Function: CreateBoolNode
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+bool_tree_node *
+CreateBoolNode (N_Tag tag)
+{
+  bool_tree_node *n;
+  if (!(n = Xmalloc (sizeof (bool_tree_node))))
+    FatalError (1, "Can not create bool_tree_node for boolean query");
+  n->tag = tag;
+  BOOL_SIBLING (n) = NULL;
+  return n;
+}
+
+/* =========================================================================
+ * Function: 
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+bool_tree_node *
+CreateBoolTermNode (TermList ** tl, char *text, int Count, int word_num,
+		    u_long count, u_long doc_count, u_long invf_ptr, u_long invf_len,  /* [RPAP - Feb 97: Term Frequency] */
+		    int stemmer_num) 
+{
+  bool_tree_node *n = NULL;
+
+  /* allocate bool_tree_node */
+  n = CreateBoolNode (N_term);
+
+  BOOL_TERM (n) = AddTerm (tl, (u_char *) text, Count, word_num, count, doc_count, 
+			   invf_ptr, invf_len, stemmer_num);  /* [RPAP - Feb 97: Term Frequency] */
+
+  return n;
+}
+
+
+/* =========================================================================
+ * Function: CreateTreeNode
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+bool_tree_node *
+CreateBoolTreeNode (N_Tag tag, bool_tree_node * left, bool_tree_node * right)
+{
+  bool_tree_node *n = CreateBoolNode (tag);
+
+  BOOL_CHILD (n) = left;
+  if (left)
+    BOOL_SIBLING (left) = right;
+  if (right)
+    BOOL_SIBLING (right) = NULL;
+
+  return (n);
+}
+
+/* =========================================================================
+ * Function: 
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+int
+EqualBoolNodes (bool_tree_node * a, bool_tree_node * b)
+{
+  if (BOOL_TAG (a) == N_term && BOOL_TAG (a) == N_term)
+    return BOOL_TERM (a) == BOOL_TERM (b);
+  return 0;
+}
+
+/* =========================================================================
+ * Function: FreeBoolTree
+ * Description: 
+ *      Notes: does not fix up any parent or sibling links around given tree
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void 
+FreeTree (bool_tree_node * tree)
+{
+  bool_tree_node *sibling;
+
+  while (tree)
+    {
+      if (BOOL_HAS_CHILD (tree))
+	FreeTree (BOOL_CHILD (tree));
+      sibling = BOOL_SIBLING (tree);
+      Xfree (tree);
+      tree = sibling;
+    }
+}
+
+void 
+FreeBoolTree (bool_tree_node ** the_tree)
+{
+  bool_tree_node *tree = *the_tree;
+
+  if (!tree)
+    return;
+
+  FreeTree (tree);
+  *the_tree = NULL;
+/*********************************/
+  /* THIS IS POTENTIALLY DANGEROUS */
+  /* FIX UP LATER */
+  /* Need to correct the links for parent/siblings */
+/*********************************/
+
+}
+
+/* =========================================================================
+ * Function: PrintBoolTree
+ * Description: 
+ *      Print out a text version of the tree to the file
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+PrintBinaryOp (bool_tree_node * tree, FILE * file, char op)
+{
+  bool_tree_node *child = NULL;
+  assert (BOOL_HAS_CHILD (tree));
+
+  child = BOOL_CHILD (tree);
+  while (child)
+    {
+      PrintBoolTree (child, file);
+      child = BOOL_SIBLING (child);
+      if (child)
+	fprintf (file, " %c ", op);
+    }
+}
+
+static void
+PrintUnaryOp (bool_tree_node * tree, FILE * file, char op)
+{
+  bool_tree_node *child = NULL;
+
+  fprintf (file, " %c ", op);
+  assert (BOOL_HAS_CHILD (tree));
+  child = BOOL_CHILD (tree);
+  PrintBoolTree (child, file);
+}
+
+void
+PrintBoolTree (bool_tree_node * tree, FILE * file)
+{
+  if (!tree)
+    return;
+
+  fprintf (file, "(");
+  switch (BOOL_TAG (tree))
+    {
+    case N_term:
+      fprintf (file, "term %d", BOOL_TERM (tree));
+      break;
+    case N_and:
+      PrintBinaryOp (tree, file, '&');
+      break;
+    case N_or:
+      PrintBinaryOp (tree, file, '|');
+      break;
+    case N_or_terms:
+      PrintBinaryOp (tree, file, '+');
+      break;
+    case N_diff:
+      PrintBinaryOp (tree, file, '-');
+      break;
+    case N_not:
+      PrintUnaryOp (tree, file, '!');
+      break;
+    case N_all:
+      fprintf (file, " TRUE ");
+      break;
+    case N_none:
+      fprintf (file, " FALSE ");
+      break;
+    default:
+      break;
+    }
+
+  fprintf (file, ")");
+}
+
+
+/* =========================================================================
+ * Function: CopyBoolTree
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+bool_tree_node *
+CopyBoolTree (bool_tree_node * tree)
+{
+  bool_tree_node *sibling = NULL;
+  bool_tree_node *tree_copy = NULL;
+  bool_tree_node *prev = NULL;
+
+  if (!tree)
+    return NULL;
+
+  sibling = tree;
+  prev = NULL;
+  while (sibling)
+    {
+      /* copy sibling */
+      bool_tree_node *sibling_copy = (bool_tree_node *) malloc (sizeof (bool_tree_node));
+      if (!sibling_copy)
+	FatalError (1, "No memory for bool tree copy");
+      bcopy ((char *) sibling, (char *) sibling_copy, sizeof (bool_tree_node));
+
+      if (BOOL_HAS_CHILD (sibling))
+	{
+	  BOOL_CHILD (sibling_copy) = CopyBoolTree (BOOL_CHILD (sibling));
+	}
+
+      /* do the link up of copies */
+      if (prev)
+	BOOL_SIBLING (prev) = sibling_copy;
+      else
+	tree_copy = sibling_copy;
+      prev = sibling_copy;
+
+      sibling = BOOL_SIBLING (sibling);
+    }
+  return tree_copy;
+}
+
+
+/* =========================================================================
+ * Function: LastBoolSibling
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+bool_tree_node *
+LastBoolSibling (bool_tree_node * tree)
+{
+
+  if (!tree)
+    return NULL;
+
+  do
+    {
+      tree = BOOL_SIBLING (tree);
+    }
+  while (BOOL_SIBLING (tree));
+
+  return tree;
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/bool_tree.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/bool_tree.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/bool_tree.h	(revision 699)
@@ -0,0 +1,110 @@
+/**************************************************************************
+ *
+ * bool_tree.h -- description
+ * Copyright (C) 1994  Authors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:17:46  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:42  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:34:31  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.2  1995/03/14  05:15:25  tes
+   * Updated the boolean query optimiser to do different types of optimisation.
+   * A query environment variable "optimise_type" specifies which one is to be
+   * used. Type 1 is the new one which is faster than 2.
+   *
+   * Revision 1.2  1994/10/18  06:11:07  tes
+   * The boolean optimiser seems to be modifying the parse tree
+   * like it is supposed to.
+   * Paragraph ranking now works without any text files if required to.
+   *
+   * Revision 1.1  1994/10/12  01:15:35  tes
+   * Found bugs in the existing boolean query optimiser.
+   * So decided to rewrite it.
+   * I accidentally deleted query.bool.y, but I have replaced it
+   * with bool_parser.y (which I have forgotten to add here ! ;-(
+   *
+ */
+
+#ifndef BOOL_TREE_H
+#define BOOL_TREE_H
+
+#include "sysfuncs.h"
+#include "term_lists.h"
+
+typedef enum
+  {
+    N_term, N_and, N_or, N_diff, N_none, N_all,
+    N_not, N_or_terms
+  }
+N_Tag;
+
+typedef struct bool_tree_node
+  {
+    N_Tag tag;
+    struct bool_tree_node *sibling;
+    union
+      {
+	struct bool_tree_node *child;
+	int term_id;
+      }
+    fields;
+    int sum_ft;			/* for or_terms nodes */
+  }
+bool_tree_node;
+
+
+#define BOOL_HAS_CHILD(n) (\
+			(n)->tag==N_and || \
+			(n)->tag==N_or || (n)->tag==N_or_terms || \
+			(n)->tag==N_not || (n)->tag==N_diff)
+#define BOOL_TAG(n) ((n)->tag)
+#define BOOL_SIBLING(n) ((n)->sibling)
+#define BOOL_TERM(n) ((n)->fields.term_id)
+#define BOOL_CHILD(n) ((n)->fields.child)
+#define BOOL_SUM_FT(n) ((n)->sum_ft)
+
+bool_tree_node *CreateBoolNode (N_Tag tag);
+bool_tree_node *CopyBoolTree (bool_tree_node * tree);
+bool_tree_node *CreateBoolTermNode (TermList ** tl, char *text, int Count, int word_num,
+				    u_long count, u_long doc_count, u_long invf_ptr, 
+				    u_long invf_len, int stemmer_num);
+bool_tree_node *CreateBoolTreeNode (N_Tag tag,
+			     bool_tree_node * left, bool_tree_node * right);
+void FreeBoolTree (bool_tree_node ** tree);
+void PrintBoolTree (bool_tree_node * tree, FILE * file);
+bool_tree_node *LastBoolSibling (bool_tree_node * tree);
+
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/build.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/build.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/build.h	(revision 699)
@@ -0,0 +1,170 @@
+/**************************************************************************
+ *
+ * build.h -- Global information for the passes of mg_passes
+ * Copyright (C) 1994  Neil Sharman, Alistair Moffat and Lachlan Andrew
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef H_BUILD
+#define H_BUILD
+
+
+#define	TERMPARAGRAPH	'\003'
+
+
+int init_special (char *file_name);
+int init_text_1 (char *file_name);
+int init_text_2 (char *file_name);
+int init_invf_1 (char *file_name);
+int init_invf_2 (char *file_name);
+int init_ivf_1 (char *file_name);
+int init_ivf_2 (char *file_name);
+/*
+ * file_name  IN      The name of the dictionary file
+ * 
+ * returns      COMPALLOK for all ok,
+ *              COMPERROR for any error.  e.g. cannot read file
+ */
+
+
+
+
+
+int process_special (u_char * s_in, int l_in);
+int process_text_1 (u_char * s_in, int l_in);
+int process_text_2 (u_char * s_in, int l_in);
+int process_invf_1 (u_char * s_in, int l_in);
+int process_invf_2 (u_char * s_in, int l_in);
+int process_ivf_1 (u_char * s_in, int l_in);
+int process_ivf_2 (u_char * s_in, int l_in);
+/*
+ * s_in               IN      The binary string to be compressed
+ * l_in         IN      The number of characters in s_in
+ *
+ * returns      COMPALLOK for all ok,
+ *              COMPERROR for any error.  e.g. cannot read file
+ *
+ * The calling routine is responsible for ensuring that s_out is long 
+ * enough.
+ */
+
+
+
+
+
+int done_special (char *filename);
+int done_text_1 (char *filename);
+int done_text_2 (char *filename);
+int done_invf_1 (char *filename);
+int done_invf_2 (char *filename);
+int done_ivf_1 (char *filename);
+int done_ivf_2 (char *filename);
+/*
+ * returns    COMPALLOK for all ok,
+ *              COMPERROR for any error.  e.g. cannot write file
+ */
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+extern char InvfLevel;
+/*
+ * This will determine the level of the inverted file it can take on the
+ * values 1, 2, or 3.
+ *
+ * Level 1: The inverted file contains only document numbers making it possible
+ *          to do only boolean queries.
+ * 
+ * Level 2: The inverted file also contains word counts per document making it
+ *          possible to do cosine ranked queries.
+ *
+ * Level 3: The inverted file contains word positions.
+ */
+
+
+extern unsigned long buf_size;
+/*
+ * The size of the document input buffer.
+ */
+
+extern unsigned long ChunkLimit;
+/*
+ * The maximum number of chunks that can be written to disk.
+ */
+
+extern unsigned long invf_buffer_size;
+/*
+ * The amount of memory to allocate to the invertion buffer.
+ */
+
+extern char SkipSGML;
+/*
+ * 1 if SGML tags are to be considered non-words when building the 
+ * inverted file.
+ */
+
+extern char MakeWeights;
+/*
+ * 1 if the weights file should be generated.
+ */
+
+extern FILE *Comp_Stats;
+/*
+ * Contains a file pointer to the file where compression stats should be sent
+ */
+
+extern int comp_stat_point;
+/*
+ * Generate a compression stat entry every comp_stat_point bytes 
+ */
+
+extern double bytes_processed; /* [RJM 07/97: 4G limit] */
+/*
+ * The number of bytes processed. NOTE: This excludes document separators.
+ */
+
+extern double bytes_received; /* [RJM 07/97: 4G limit] */
+/*
+ * The number of bytes processed. NOTE: This includes document separators.
+ */
+
+
+extern int stemmer_num;
+/*
+ * The stemmer to use for stemming words for the inverted file.
+ * see stemmer.h
+ */
+
+extern int stem_method;
+/*
+ * The method to use for stemming words for the inverted file.
+ * see stemmer.h
+ */
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/commands.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/commands.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/commands.c	(revision 699)
@@ -0,0 +1,582 @@
+/**************************************************************************
+ *
+ * commands.c -- mgquery command processing functions
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:17:47  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:34:32  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.2  1994/09/20  04:41:22  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+#include "memlib.h"
+#include "messages.h"
+
+#include "environment.h"
+#include "mg.h"
+#include "locallib.h"
+#include "backend.h"		/* for qd */
+#include "term_lists.h"		/* for MAXTERMSTRLEN */
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+# define OPENPIPE        _popen
+# define CLOSEPIPE       _pclose
+#else
+# define OPENPIPE        popen
+# define CLOSEPIPE       pclose
+#endif
+
+extern char *PathName;
+extern FILE *OutFile, *InFile;
+extern int OutPipe, InPipe;
+extern int Quitting;
+
+char *CommandsErrorStr = NULL;
+
+/* [TS: June/95] */
+/* current qd for use by any commands */
+static query_data *the_qd = NULL;
+
+
+#include "help.mg.h"
+#include "warranty.h"
+#include "conditions.h"
+
+static void 
+CmdQuit (void)
+{
+  Quitting = 1;
+}
+
+static void 
+CmdPush (void)
+{
+  PushEnv ();
+}
+
+
+static void 
+CmdPop (void)
+{
+  PopEnv ();
+  if (EnvStackHeight () == 1)
+    PushEnv ();
+}
+
+static void 
+CmdHelp (void)
+{
+  FILE *more;
+  int i;
+
+  if (!(more = OPENPIPE (GetDefEnv ("pager", "more"), "w")))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      fprintf (stderr, "ERROR : Unable to run more\n");
+      return;
+    }
+  for (i = 0; i < sizeof (help_str) / sizeof (help_str[0]); i++)
+    fputs (help_str[i], more);
+
+  CLOSEPIPE (more);  /* [RPAP - Feb 97: WIN32 Port] */
+}
+
+
+static void 
+CmdWarranty (void)
+{
+  FILE *more;
+  int i;
+
+  if (!(more = OPENPIPE (GetDefEnv ("pager", "more"), "w")))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      fprintf (stderr, "ERROR : Unable to run more\n");
+      return;
+    }
+  for (i = 0; i < sizeof (warranty_str) / sizeof (warranty_str[0]); i++)
+    fputs (warranty_str[i], more);
+
+  CLOSEPIPE (more);  /* [RPAP - Feb 97: WIN32 Port] */
+}
+
+
+static void 
+CmdConditions (void)
+{
+  FILE *more;
+  int i;
+
+  if (!(more = OPENPIPE (GetDefEnv ("pager", "more"), "w")))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      fprintf (stderr, "ERROR : Unable to run more\n");
+      return;
+    }
+  for (i = 0; i < sizeof (cond_str) / sizeof (cond_str[0]); i++)
+    fputs (cond_str[i], more);
+
+  CLOSEPIPE (more);
+}
+
+static void 
+CmdReset (void)
+{
+  while (EnvStackHeight () > 1)
+    PopEnv ();
+  PushEnv ();
+}
+
+static void 
+CmdDisplay (void)
+{
+  int i;
+  size_t l = 0;
+  char *name;
+  for (i = 0; (name = GetEnvName (i)) != NULL; i++)
+    if (strlen (name) > l)
+      l = strlen (name);
+
+  for (i = 0; (name = GetEnvName (i)) != NULL; i++)
+    printf ("%-*s = \"%s\"\n", (int) l, name, GetEnv (name));
+}
+
+static void 
+CmdQueryTerms (void)
+{
+  static char terms_str[MAXTERMSTRLEN + 1];
+
+  if (the_qd && the_qd->TL)
+    {
+      ConvertTermsToString (the_qd->TL, terms_str);
+      printf ("%s\n", terms_str);
+      fflush (stdout);
+    }
+
+}
+
+
+static void 
+CmdSet (char *name, char *value)
+{
+  if (!*name)
+    {
+      CommandsErrorStr = "Blank names are not permitted";
+      return;
+    }
+  switch (SetEnv (name, value, NULL))
+    {
+    case -1:
+      CommandsErrorStr = "Out of memory\n";
+      break;
+    case -2:
+      CommandsErrorStr = ConstraintErrorStr;
+      break;
+    }
+}
+
+static void 
+CmdUnset (char *name)
+{
+  if (!*name)
+    {
+      CommandsErrorStr = "Blank names are not permitted";
+      return;
+    }
+  if (UnsetEnv (name, 0) == -1)
+    CommandsErrorStr = "Specified name does not exist";
+}
+
+
+
+
+
+
+/*
+ * Do the .input command 
+ */
+static void 
+CmdInput (char *param1, char *param2)
+{
+  enum
+  {
+    READ, APPEND, PIPEFROM
+  }
+  mode;
+  FILE *NewInput;
+  char *buf, *s;
+
+  /* join the two parameters together */
+  buf = Xmalloc (strlen (param1) + strlen (param2) + 1);
+  strcpy (buf, param1);
+  strcat (buf, param2);
+  s = buf;
+
+
+  while (*s == ' ' || *s == '\t')
+    s++;
+
+  if (*s == '<')
+    {
+      s++;
+      mode = READ;
+    }
+  else if (*s == '|')
+    {
+      mode = PIPEFROM;
+      s++;
+    }
+  else
+    {
+      if (InPipe)
+	CLOSEPIPE (InFile);  /* [RPAP - Feb 97: WIN32 Port] */
+      else if (InFile != stdin)
+	fclose (InFile);
+      InPipe = 0;
+      InFile = stdin;
+      Xfree (buf);
+      return;
+    }
+
+  while (*s == ' ' || *s == '\t')
+    s++;
+
+  if (*s)
+    {
+      if (mode == READ)
+	NewInput = fopen (s, "r");
+      else
+	NewInput = OPENPIPE (s, "r");  /* [RPAP - Feb 97: WIN32 Port] */
+
+      if (NewInput)
+	{
+	  if (InPipe)
+	    CLOSEPIPE (InFile);  /* [RPAP - Feb 97: WIN32 Port] */
+	  else if (InFile != stdin)
+	    fclose (InFile);
+	  InPipe = (mode == PIPEFROM);
+	  InFile = NewInput;
+	}
+      else
+	CommandsErrorStr = "Bad input file/pipe";
+    }
+  else
+    {
+      if (InPipe)
+	CLOSEPIPE (InFile);  /* [RPAP - Feb 97: WIN32 Port] */
+      else if (InFile != stdin)
+	fclose (InFile);
+      InPipe = 0;
+      InFile = stdin;
+    }
+  Xfree (buf);
+}
+
+
+
+static void 
+CmdOutput (char *param1, char *param2)
+{
+  enum
+  {
+    WRITE, APPEND, PIPETO
+  }
+  mode;
+  char *buf, *s;
+  FILE *NewOutput;
+
+  /* join the two parameters together */
+  buf = Xmalloc (strlen (param1) + strlen (param2) + 1);
+  strcpy (buf, param1);
+  strcat (buf, param2);
+  s = buf;
+
+  while (*s == ' ' || *s == '\t')
+    s++;
+
+  if (*s == '>')
+    {
+      s++;
+      mode = APPEND;
+      if (*s == '>')
+	s++;
+      else
+	mode = WRITE;
+    }
+  else if (*s == '|')
+    {
+      mode = PIPETO;
+      s++;
+    }
+  else
+    {
+      if (OutPipe)
+	CLOSEPIPE (OutFile);  /* [RPAP - Feb 97: WIN32 Port] */
+      else if (OutFile != stdout)
+	fclose (OutFile);
+      OutPipe = 0;
+      OutFile = stdout;
+      Xfree (buf);
+      return;
+    }
+
+  while (*s == ' ' || *s == '\t')
+    s++;
+
+  if (*s)
+    {
+      if (mode == PIPETO)
+	NewOutput = OPENPIPE (s, "w");  /* [RPAP - Feb 97: WIN32 Port] */
+      else
+	NewOutput = fopen (s, mode == WRITE ? "w" : "a");
+      if (NewOutput)
+	{
+	  if (OutPipe)
+	    CLOSEPIPE (OutFile);  /* [RPAP - Feb 97: WIN32 Port] */
+	  else if (OutFile != stdout)
+	    fclose (OutFile);
+	  OutPipe = (mode == PIPETO);
+	  OutFile = NewOutput;
+	}
+      else
+	CommandsErrorStr = "Bad output file/pipe";
+    }
+  else
+    {
+      if (OutPipe)
+	CLOSEPIPE (OutFile);  /* [RPAP - Feb 97: WIN32 Port] */
+      else if (OutFile != stdout)
+	fclose (OutFile);
+      OutPipe = 0;
+      OutFile = stdout;
+    }
+  Xfree (buf);
+}
+
+
+
+static char *
+ParseArg (char *line, char *buffer)
+{
+  if (*line == '\"')
+    {
+      line++;
+      while (*line != '\"' && *line != '\0')
+	{
+	  if (*line == '\\')
+	    *buffer++ = *line++;
+	  if (*line != '\0')
+	    *buffer++ = *line++;
+	}
+      if (*line == '\"')
+	line++;
+    }
+  else
+    while (*line != ' ' && *line != '\t' && *line != '\0')
+      {
+	if (*line == '\\')
+	  line++;
+	if (*line != '\0')
+	  *buffer++ = *line++;
+      }
+  *buffer = '\0';
+  while (*line == ' ' || *line == '\t')
+    line++;
+  return (line);
+}
+
+
+
+char *
+ProcessCommands (char *line, query_data * qd)
+{
+  struct
+  {
+    char *Command;
+    void (*action) (void);
+    void (*action1) (char *);
+    void (*action2) (char *, char *);
+  }
+   *dc, dot_command[] =
+  {
+    {
+      "set", NULL, NULL, CmdSet
+    }
+    ,
+    {
+      "input", NULL, NULL, CmdInput
+    }
+    ,
+    {
+      "output", NULL, NULL, CmdOutput
+    }
+    ,
+    {
+      "unset", NULL, CmdUnset, NULL
+    }
+    ,
+    {
+      "help", CmdHelp, NULL, NULL
+    }
+    ,
+    {
+      "reset", CmdReset, NULL, NULL
+    }
+    ,
+    {
+      "display", CmdDisplay, NULL, NULL
+    }
+    ,
+    {
+      "push", CmdPush, NULL, NULL
+    }
+    ,
+    {
+      "quit", CmdQuit, NULL, NULL
+    }
+    ,
+    {
+      "pop", CmdPop, NULL, NULL
+    }
+    ,
+    {
+      "warranty", CmdWarranty, NULL, NULL
+    }
+    ,
+    {
+      "conditions", CmdConditions, NULL, NULL
+    }
+    ,
+    {
+      "queryterms", CmdQueryTerms, NULL, NULL
+    }
+  };
+
+
+  char command[512];
+  char param1[512], param2[512];
+  CommandsErrorStr = NULL;
+  the_qd = qd;			/* set globally to be accessed by command functions if needed */
+  while (*line == ' ' || *line == '\t')
+    line++;
+  while (*line == '.')
+    {
+      int i;
+      line++;
+      for (i = 0; isalpha (*line) && i < 512; i++, line++)
+	command[i] = tolower (*line);
+      while (*line == ' ' || *line == '\t')
+	line++;
+      if (i != 512)
+	{
+	  command[i] = '\0';
+	  for (i = 0, dc = dot_command; i < NUMOF (dot_command); i++, dc++)
+	    if (!strcmp (dc->Command, command))
+	      {
+		if (dc->action)
+		  dc->action ();
+		else if (dc->action1)
+		  {
+		    line = ParseArg (line, param1);
+		    dc->action1 (param1);
+		    break;
+		  }
+		else if (dc->action2)
+		  {
+		    line = ParseArg (line, param1);
+		    line = ParseArg (line, param2);
+		    dc->action2 (param1, param2);
+		    break;
+		  }
+		if (CommandsErrorStr)
+		  *line = '\0';
+		break;
+	      }
+	  if (i == NUMOF (dot_command))
+	    i = 100;
+	}
+      if (i == 100)
+	{
+	  CommandsErrorStr = "Illegal command";
+	  *line = '\0';
+	}
+      while (*line == ' ' || *line == '\t')
+	line++;
+    }
+  return (line);
+}
+
+
+
+
+
+void 
+read_mgrc_file (void)
+{
+  char line[1000];
+  char FileName[256];
+  int LineNum = 0;
+  FILE *mgrc;
+
+  strcpy (FileName, ".mgrc");
+  mgrc = fopen (FileName, "r");
+
+  if (!mgrc)
+    {
+      char *home_path = getenv ("HOME");
+      if (home_path)
+	{
+	  sprintf (FileName, "%s/.mgrc", home_path);
+	  mgrc = fopen (FileName, "r");
+	}
+    }
+
+  if (!mgrc)
+    return;
+
+  while (fgets (line, sizeof (line), mgrc))
+    {
+      char *s, linebuf[1000];
+      LineNum++;
+      if ((s = strchr (line, '\n')) != NULL)
+	*s = '\0';
+      strcpy (linebuf, line);
+      if ((s = strchr (line, '#')) != NULL)
+	*s = '\0';
+      if (!*line)
+	continue;
+      ProcessCommands (line, NULL);
+      if (CommandsErrorStr)
+	Message ("%s\n%s : ERROR in line %d of %s\n%s : %s\n",
+		 linebuf,
+		 msg_prefix, LineNum, FileName,
+		 msg_prefix, CommandsErrorStr);
+    }
+  fclose (mgrc);
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/commands.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/commands.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/commands.h	(revision 699)
@@ -0,0 +1,33 @@
+/**************************************************************************
+ *
+ * commands.h -- mgquery command processing functions
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef COMMANDS_H
+#define COMMANDS_H
+
+#include "backend.h"
+
+extern char *CommandsErrorStr;
+char *ProcessCommands (char *line, query_data * qd);
+void read_mgrc_file (void);
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/comp_dict.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/comp_dict.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/comp_dict.c	(revision 699)
@@ -0,0 +1,255 @@
+/**************************************************************************
+ *
+ * comp_dict.c -- Functions for loading the compression dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "huffman.h"
+#include "local_strings.h"
+#include "memlib.h"
+#include "messages.h"
+
+#include "mg.h"
+#include "hash.h"
+#include "text.h"
+#include "comp_dict.h"
+#include "locallib.h"
+#include "mg_files.h"
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:17:48  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:34:34  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:41  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:24  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+compression_dict_header cdh;
+compressed_text_header cth;
+comp_frags_header cfh[2];
+
+dict_hash_table *ht[2];
+
+huff_data char_huff[2];
+huff_data lens_huff[2];
+u_long *char_codes[2], *lens_codes[2];
+u_long Words_disk = 0;
+u_long Chars_disk = 0;
+
+
+static dict_hash_table *
+ReadInWords (FILE * dict, comp_frags_header * cfh,
+	     int esc)
+{
+  int i;
+  u_char *allwords, *prev = NULL;
+  dict_hash_table *ht;
+  u_char **words;
+  u_long ht_size;
+
+  ht_size = prime (cfh->hd.num_codes * HASH_RATIO);
+  if (!(ht = Xmalloc (sizeof (dict_hash_table) +
+		      (ht_size - 1) * sizeof (ht->table[0]))))
+    {
+      Message ("no memory for hash_table\n");
+      return NULL;
+    }
+
+  ht->size = ht_size;
+  ht->hd = &cfh->hd;
+
+  if (!(ht->codes = Generate_Huffman_Codes (&cfh->hd, NULL)))
+    {
+      Message ("no memory for huffman codes\n");
+      return NULL;
+    }
+
+  if (!(ht->words = Xmalloc (sizeof (u_char *) * cfh->hd.num_codes)))
+    {
+      Message ("no memory for word pointers\n");
+      return NULL;
+    }
+  words = ht->words;
+
+  bzero ((char *) ht->table, ht_size * sizeof (ht->table[0]));
+
+  if (!(allwords = Xmalloc (sizeof (u_char) * cfh->uncompressed_size)))
+    {
+      Message ("no memory for words\n");
+      return NULL;
+    }
+
+  for (i = 0; i < cfh->hd.num_codes; i++, words++)
+    {
+      register int val, copy;
+      val = fgetc (dict);
+      copy = (val >> 4) & 0xf;
+      val &= 0xf;
+
+      *words = allwords;
+
+      memcpy (allwords + 1, prev + 1, copy);
+      fread (allwords + copy + 1, sizeof (u_char), val, dict);
+      *allwords = val + copy;
+
+      Words_disk += val + 1;
+
+      /* insert into the hash table */
+      if (i < cfh->hd.num_codes - esc)
+	{
+	  register u_char **wptr;
+	  register int tsize = ht->size;
+	  register unsigned long hashval, step;
+
+	  HASH (hashval, step, allwords, tsize);
+
+	  wptr = ht->table[hashval];
+	  for (; wptr;)
+	    {
+	      hashval += step;
+	      if (hashval >= tsize)
+		hashval -= tsize;
+	      wptr = ht->table[hashval];
+	    }
+	  ht->table[hashval] = words;
+	}
+      prev = allwords;
+      allwords += *allwords + 1;
+    }
+  return ht;
+}
+
+
+
+int 
+LoadCompressionDictionary (char *dict_file_name)
+{
+  FILE *dict;
+  int which;
+  if (!(dict = open_named_file (dict_file_name, "rb", MAGIC_DICT, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return COMPERROR;
+
+  Words_disk = sizeof (u_long);
+
+  if (Read_cdh (dict, &cdh, NULL, &Words_disk) == -1)
+    goto error;
+
+  for (which = 0; which < 2; which++)
+    switch (cdh.dict_type)
+      {
+      case MG_COMPLETE_DICTIONARY:
+	{
+	  if (Read_cfh (dict, &cfh[which], NULL, &Words_disk) == -1)
+	    goto error;
+
+	  if (!(ht[which] = ReadInWords (dict, &cfh[which], 0)))
+	    goto error;
+
+	}
+	break;
+      case MG_PARTIAL_DICTIONARY:
+	{
+	  if (cdh.num_words[which])
+	    {
+	      if (Read_cfh (dict, &cfh[which], NULL, &Words_disk) == -1)
+		goto error;
+	      if (!(ht[which] = ReadInWords (dict, &cfh[which], 1)))
+		goto error;
+	    }
+	  else
+	    ht[which] = NULL;
+	  if (Read_Huffman_Data (dict, &char_huff[which], NULL,
+				 &Chars_disk) == -1)
+	    goto error;
+	  if (!(char_codes[which] =
+		Generate_Huffman_Codes (&char_huff[which], NULL)))
+	    goto error;
+	  if (Read_Huffman_Data (dict, &lens_huff[which], NULL,
+				 &Chars_disk) == -1)
+	    goto error;
+	  if (!(lens_codes[which] =
+		Generate_Huffman_Codes (&lens_huff[which], NULL)))
+	    goto error;
+	}
+	break;
+      case MG_SEED_DICTIONARY:
+	{
+	  if (cdh.num_words[which])
+	    {
+	      if (Read_cfh (dict, &cfh[which], NULL, &Words_disk) == -1)
+		goto error;
+	      if (!(ht[which] = ReadInWords (dict, &cfh[which], 1)))
+		goto error;
+	    }
+	  else
+	    ht[which] = NULL;
+	  switch (cdh.novel_method)
+	    {
+	    case MG_NOVEL_HUFFMAN_CHARS:
+	      if (Read_Huffman_Data (dict, &char_huff[which], NULL,
+				     &Chars_disk) == -1)
+		goto error;
+	      if (!(char_codes[which] =
+		    Generate_Huffman_Codes (&char_huff[which], NULL)))
+		goto error;
+	      if (Read_Huffman_Data (dict, &lens_huff[which], NULL,
+				     &Chars_disk) == -1)
+		goto error;
+	      if (!(lens_codes[which] =
+		    Generate_Huffman_Codes (&lens_huff[which], NULL)))
+		goto error;
+	      break;
+	    case MG_NOVEL_BINARY:
+	      break;
+	    case MG_NOVEL_DELTA:
+	      break;
+	    case MG_NOVEL_HYBRID:
+	      break;
+	    case MG_NOVEL_HYBRID_MTF:
+	      break;
+	    default:
+	      FatalError (1, "Bad novel method");
+	    }
+	}
+	break;
+      default:
+	FatalError (1, "Bad dictionary kind\n");
+      }
+
+  return (COMPALLOK);
+
+
+error:
+  fclose (dict);
+  return (COMPERROR);
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/comp_dict.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/comp_dict.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/comp_dict.h	(revision 699)
@@ -0,0 +1,59 @@
+/**************************************************************************
+ *
+ * comp_dict.h -- Functions for loading the compression dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+
+
+#define HASH_RATIO 2
+
+
+
+
+typedef struct dict_hash_table
+  {
+    u_long size;
+    huff_data *hd;
+    u_long *codes;
+    u_char **words;
+    u_char **table[1];
+  }
+dict_hash_table;
+
+
+
+
+extern compression_dict_header cdh;
+extern compressed_text_header cth;
+extern comp_frags_header cfh[2];
+
+extern dict_hash_table *ht[2];
+
+extern huff_data char_huff[2];
+extern huff_data lens_huff[2];
+extern u_long *char_codes[2], *lens_codes[2];
+extern u_long Words_disk;
+extern u_long Chars_disk;
+
+
+
+int LoadCompressionDictionary (char *dict_file_name);
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/conditions.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/conditions.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/conditions.h	(revision 699)
@@ -0,0 +1,226 @@
+static char *cond_str[] = {
+"		    GNU GENERAL PUBLIC LICENSE\n",
+"   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION\n",
+"\n",
+"  0. This License applies to any program or other work which contains\n",
+"a notice placed by the copyright holder saying it may be distributed\n",
+"under the terms of this General Public License.  The \"Program\", below,\n",
+"refers to any such program or work, and a \"work based on the Program\"\n",
+"means either the Program or any derivative work under copyright law:\n",
+"that is to say, a work containing the Program or a portion of it,\n",
+"either verbatim or with modifications and/or translated into another\n",
+"language.  (Hereinafter, translation is included without limitation in\n",
+"the term \"modification\".)  Each licensee is addressed as \"you\".\n",
+"\n",
+"Activities other than copying, distribution and modification are not\n",
+"covered by this License; they are outside its scope.  The act of\n",
+"running the Program is not restricted, and the output from the Program\n",
+"is covered only if its contents constitute a work based on the\n",
+"Program (independent of having been made by running the Program).\n",
+"Whether that is true depends on what the Program does.\n",
+"\n",
+"  1. You may copy and distribute verbatim copies of the Program's\n",
+"source code as you receive it, in any medium, provided that you\n",
+"conspicuously and appropriately publish on each copy an appropriate\n",
+"copyright notice and disclaimer of warranty; keep intact all the\n",
+"notices that refer to this License and to the absence of any warranty;\n",
+"and give any other recipients of the Program a copy of this License\n",
+"along with the Program.\n",
+"\n",
+"You may charge a fee for the physical act of transferring a copy, and\n",
+"you may at your option offer warranty protection in exchange for a fee.\n",
+"\n",
+"  2. You may modify your copy or copies of the Program or any portion\n",
+"of it, thus forming a work based on the Program, and copy and\n",
+"distribute such modifications or work under the terms of Section 1\n",
+"above, provided that you also meet all of these conditions:\n",
+"\n",
+"    a) You must cause the modified files to carry prominent notices\n",
+"    stating that you changed the files and the date of any change.\n",
+"\n",
+"    b) You must cause any work that you distribute or publish, that in\n",
+"    whole or in part contains or is derived from the Program or any\n",
+"    part thereof, to be licensed as a whole at no charge to all third\n",
+"    parties under the terms of this License.\n",
+"\n",
+"    c) If the modified program normally reads commands interactively\n",
+"    when run, you must cause it, when started running for such\n",
+"    interactive use in the most ordinary way, to print or display an\n",
+"    announcement including an appropriate copyright notice and a\n",
+"    notice that there is no warranty (or else, saying that you provide\n",
+"    a warranty) and that users may redistribute the program under\n",
+"    these conditions, and telling the user how to view a copy of this\n",
+"    License.  (Exception: if the Program itself is interactive but\n",
+"    does not normally print such an announcement, your work based on\n",
+"    the Program is not required to print an announcement.)\n",
+"
+\n",
+"These requirements apply to the modified work as a whole.  If\n",
+"identifiable sections of that work are not derived from the Program,\n",
+"and can be reasonably considered independent and separate works in\n",
+"themselves, then this License, and its terms, do not apply to those\n",
+"sections when you distribute them as separate works.  But when you\n",
+"distribute the same sections as part of a whole which is a work based\n",
+"on the Program, the distribution of the whole must be on the terms of\n",
+"this License, whose permissions for other licensees extend to the\n",
+"entire whole, and thus to each and every part regardless of who wrote it.\n",
+"\n",
+"Thus, it is not the intent of this section to claim rights or contest\n",
+"your rights to work written entirely by you; rather, the intent is to\n",
+"exercise the right to control the distribution of derivative or\n",
+"collective works based on the Program.\n",
+"\n",
+"In addition, mere aggregation of another work not based on the Program\n",
+"with the Program (or with a work based on the Program) on a volume of\n",
+"a storage or distribution medium does not bring the other work under\n",
+"the scope of this License.\n",
+"\n",
+"  3. You may copy and distribute the Program (or a work based on it,\n",
+"under Section 2) in object code or executable form under the terms of\n",
+"Sections 1 and 2 above provided that you also do one of the following:\n",
+"\n",
+"    a) Accompany it with the complete corresponding machine-readable\n",
+"    source code, which must be distributed under the terms of Sections\n",
+"    1 and 2 above on a medium customarily used for software interchange; or,\n",
+"\n",
+"    b) Accompany it with a written offer, valid for at least three\n",
+"    years, to give any third party, for a charge no more than your\n",
+"    cost of physically performing source distribution, a complete\n",
+"    machine-readable copy of the corresponding source code, to be\n",
+"    distributed under the terms of Sections 1 and 2 above on a medium\n",
+"    customarily used for software interchange; or,\n",
+"\n",
+"    c) Accompany it with the information you received as to the offer\n",
+"    to distribute corresponding source code.  (This alternative is\n",
+"    allowed only for noncommercial distribution and only if you\n",
+"    received the program in object code or executable form with such\n",
+"    an offer, in accord with Subsection b above.)\n",
+"\n",
+"The source code for a work means the preferred form of the work for\n",
+"making modifications to it.  For an executable work, complete source\n",
+"code means all the source code for all modules it contains, plus any\n",
+"associated interface definition files, plus the scripts used to\n",
+"control compilation and installation of the executable.  However, as a\n",
+"special exception, the source code distributed need not include\n",
+"anything that is normally distributed (in either source or binary\n",
+"form) with the major components (compiler, kernel, and so on) of the\n",
+"operating system on which the executable runs, unless that component\n",
+"itself accompanies the executable.\n",
+"\n",
+"If distribution of executable or object code is made by offering\n",
+"access to copy from a designated place, then offering equivalent\n",
+"access to copy the source code from the same place counts as\n",
+"distribution of the source code, even though third parties are not\n",
+"compelled to copy the source along with the object code.\n",
+"
+\n",
+"  4. You may not copy, modify, sublicense, or distribute the Program\n",
+"except as expressly provided under this License.  Any attempt\n",
+"otherwise to copy, modify, sublicense or distribute the Program is\n",
+"void, and will automatically terminate your rights under this License.\n",
+"However, parties who have received copies, or rights, from you under\n",
+"this License will not have their licenses terminated so long as such\n",
+"parties remain in full compliance.\n",
+"\n",
+"  5. You are not required to accept this License, since you have not\n",
+"signed it.  However, nothing else grants you permission to modify or\n",
+"distribute the Program or its derivative works.  These actions are\n",
+"prohibited by law if you do not accept this License.  Therefore, by\n",
+"modifying or distributing the Program (or any work based on the\n",
+"Program), you indicate your acceptance of this License to do so, and\n",
+"all its terms and conditions for copying, distributing or modifying\n",
+"the Program or works based on it.\n",
+"\n",
+"  6. Each time you redistribute the Program (or any work based on the\n",
+"Program), the recipient automatically receives a license from the\n",
+"original licensor to copy, distribute or modify the Program subject to\n",
+"these terms and conditions.  You may not impose any further\n",
+"restrictions on the recipients' exercise of the rights granted herein.\n",
+"You are not responsible for enforcing compliance by third parties to\n",
+"this License.\n",
+"\n",
+"  7. If, as a consequence of a court judgment or allegation of patent\n",
+"infringement or for any other reason (not limited to patent issues),\n",
+"conditions are imposed on you (whether by court order, agreement or\n",
+"otherwise) that contradict the conditions of this License, they do not\n",
+"excuse you from the conditions of this License.  If you cannot\n",
+"distribute so as to satisfy simultaneously your obligations under this\n",
+"License and any other pertinent obligations, then as a consequence you\n",
+"may not distribute the Program at all.  For example, if a patent\n",
+"license would not permit royalty-free redistribution of the Program by\n",
+"all those who receive copies directly or indirectly through you, then\n",
+"the only way you could satisfy both it and this License would be to\n",
+"refrain entirely from distribution of the Program.\n",
+"\n",
+"If any portion of this section is held invalid or unenforceable under\n",
+"any particular circumstance, the balance of the section is intended to\n",
+"apply and the section as a whole is intended to apply in other\n",
+"circumstances.\n",
+"\n",
+"It is not the purpose of this section to induce you to infringe any\n",
+"patents or other property right claims or to contest validity of any\n",
+"such claims; this section has the sole purpose of protecting the\n",
+"integrity of the free software distribution system, which is\n",
+"implemented by public license practices.  Many people have made\n",
+"generous contributions to the wide range of software distributed\n",
+"through that system in reliance on consistent application of that\n",
+"system; it is up to the author/donor to decide if he or she is willing\n",
+"to distribute software through any other system and a licensee cannot\n",
+"impose that choice.\n",
+"\n",
+"This section is intended to make thoroughly clear what is believed to\n",
+"be a consequence of the rest of this License.\n",
+"
+\n",
+"  8. If the distribution and/or use of the Program is restricted in\n",
+"certain countries either by patents or by copyrighted interfaces, the\n",
+"original copyright holder who places the Program under this License\n",
+"may add an explicit geographical distribution limitation excluding\n",
+"those countries, so that distribution is permitted only in or among\n",
+"countries not thus excluded.  In such case, this License incorporates\n",
+"the limitation as if written in the body of this License.\n",
+"\n",
+"  9. The Free Software Foundation may publish revised and/or new versions\n",
+"of the General Public License from time to time.  Such new versions will\n",
+"be similar in spirit to the present version, but may differ in detail to\n",
+"address new problems or concerns.\n",
+"\n",
+"Each version is given a distinguishing version number.  If the Program\n",
+"specifies a version number of this License which applies to it and \"any\n",
+"later version\", you have the option of following the terms and conditions\n",
+"either of that version or of any later version published by the Free\n",
+"Software Foundation.  If the Program does not specify a version number of\n",
+"this License, you may choose any version ever published by the Free Software\n",
+"Foundation.\n",
+"\n",
+"  10. If you wish to incorporate parts of the Program into other free\n",
+"programs whose distribution conditions are different, write to the author\n",
+"to ask for permission.  For software which is copyrighted by the Free\n",
+"Software Foundation, write to the Free Software Foundation; we sometimes\n",
+"make exceptions for this.  Our decision will be guided by the two goals\n",
+"of preserving the free status of all derivatives of our free software and\n",
+"of promoting the sharing and reuse of software generally.\n",
+"\n",
+"			    NO WARRANTY\n",
+"\n",
+"  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY\n",
+"FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN\n",
+"OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES\n",
+"PROVIDE THE PROGRAM \"AS IS\" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED\n",
+"OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF\n",
+"MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS\n",
+"TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE\n",
+"PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,\n",
+"REPAIR OR CORRECTION.\n",
+"\n",
+"  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING\n",
+"WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR\n",
+"REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,\n",
+"INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING\n",
+"OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED\n",
+"TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY\n",
+"YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER\n",
+"PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE\n",
+"POSSIBILITY OF SUCH DAMAGES.\n",
+"\n",
+"		     END OF TERMS AND CONDITIONS\n"};
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/conditions.src
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/conditions.src	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/conditions.src	(revision 699)
@@ -0,0 +1,225 @@
+		    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+			    NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+		     END OF TERMS AND CONDITIONS
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/environment.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/environment.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/environment.c	(revision 699)
@@ -0,0 +1,678 @@
+/**************************************************************************
+ *
+ * environment.c -- mgquery environment functions
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:17:50  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:34:37  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.5  1995/03/14  05:15:26  tes
+   * Updated the boolean query optimiser to do different types of optimisation.
+   * A query environment variable "optimise_type" specifies which one is to be
+   * used. Type 1 is the new one which is faster than 2.
+   *
+   * Revision 1.4  1994/11/25  03:47:43  tes
+   * Committing files before adding the merge stuff.
+   *
+   * Revision 1.3  1994/10/20  03:56:43  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:25  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "locallib.h"
+#include "local_strings.h"
+#include "messages.h"
+
+#include "environment.h"
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+# define STRCASECMP      stricmp
+#else
+# define STRCASECMP      strcasecmp
+#endif
+
+typedef struct
+  {
+    char *Name, *Data;
+    char *(*Constraint) (char *Old, char *New);
+  }
+EEntry;
+
+typedef struct Env
+  {
+    EEntry *Environment;
+    int NumEnv;
+    struct Env *Next;
+  }
+Env;
+
+
+char *ConstraintErrorStr;
+
+static Env Base =
+{NULL, 0, NULL};
+
+
+/*
+ * Add name and data to the environment (data may be NULL)
+ * All names are converted to lower case
+ * returns 0 on success, -1 on memory failure or -2 on constraint failure
+ */
+int 
+SetEnv (char *name, char *data, char *(*Constraint) (char *, char *))
+{
+  int i;
+  EEntry *newenv, *env = Base.Environment;
+
+  if (data && *data == '\0')
+    data = NULL;
+
+  /* search for name in the environment */
+  for (i = 0; i < Base.NumEnv && STRCASECMP (name, env->Name); i++, env++);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  /* if not found the increase the environment size */
+  if (i >= Base.NumEnv)
+    {
+      if (Constraint)
+	{
+	  data = Constraint (NULL, data);
+	  if (!data)
+	    return (-2);
+	}
+      /* allocate memory for the new environment */
+      newenv = Base.Environment ?
+	Xrealloc (Base.Environment, sizeof (EEntry) * (Base.NumEnv + 1)) :
+	Xmalloc (sizeof (EEntry) * (Base.NumEnv + 1));
+      if (!newenv)
+	return (-1);
+      Base.Environment = newenv;
+      env = &newenv[Base.NumEnv];
+      env->Data = NULL;
+      env->Name = NULL;
+      env->Constraint = Constraint;
+      env->Name = Xstrdup (name);
+      if (!env->Name)
+	return (-1);
+      Base.NumEnv++;
+      for (i = 0; env->Name[i]; i++)
+	env->Name[i] = tolower (env->Name[i]);
+    }
+  if (env->Constraint)
+    {
+      data = env->Constraint (env->Data, data);
+      if (!data)
+	return (-2);
+    }
+  if (env->Data)
+    Xfree (env->Data);
+  env->Data = NULL;
+  if (data && *data != '\0')
+    {
+      env->Data = Xstrdup (data);
+      if (!env->Data)
+	return (-1);
+    }
+  return (0);
+}
+
+
+
+/*
+ * Returns the data associated with name. If Data = NULL then this routine
+ * returns a pointer to "". If the specified name does not exist then this 
+ * routine returns NULL.
+ */
+char *
+GetEnv (char *name)
+{
+  int i;
+  EEntry *env = Base.Environment;
+  /* search for name in the environment */
+  for (i = 0; i < Base.NumEnv && STRCASECMP (name, env->Name); i++, env++);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  if (i >= Base.NumEnv)
+    return (NULL);
+
+  return (env->Data ? env->Data : "");
+}
+
+/*
+ * Returns the data associated with name. If Data = NULL then this routine
+ * returns a pointer to "". If the specified name does not exist then this 
+ * routine returns default.
+ */
+char *
+GetDefEnv (char *name, char *def)
+{
+  char *data = GetEnv (name);
+  return (data ? data : def);
+}
+
+
+
+/*
+ * This function pushes the environment on to a stack and duplicated it in 
+ * the current stack.
+ *
+ */
+int 
+PushEnv (void)
+{
+  Env *env = Xmalloc (sizeof (Env));
+  int i;
+  *env = Base;
+  Base.Next = env;
+  Base.Environment = NULL;
+  Base.NumEnv = 0;
+  for (i = 0; i < env->NumEnv; i++)
+    if (SetEnv (env->Environment[i].Name, env->Environment[i].Data,
+		env->Environment[i].Constraint))
+      return (-1);
+  return (0);
+}
+
+int 
+PopEnv (void)
+{
+  Env *env;
+  int i;
+  if (!Base.Next)
+    return (-1);
+  for (i = 0; i < Base.NumEnv; i++)
+    {
+      Xfree (Base.Environment[i].Name);
+      if (Base.Environment[i].Data)
+	Xfree (Base.Environment[i].Data);
+    }
+  if (Base.Environment)
+    Xfree (Base.Environment);
+  env = Base.Next;
+  Base = *env;
+  Xfree (env);
+  return (0);
+}
+
+
+/*
+ * Count the number of environments on the stack, including the current one.
+ *
+ */
+int 
+EnvStackHeight (void)
+{
+  int i = 1;
+  Env *env = &Base;
+  while (env->Next)
+    {
+      i++;
+      env = env->Next;
+    }
+  return (i);
+}
+
+/*
+ * Delete environment variable name. If name does not exist it returns -1
+ * otherwise it returns 0
+ */
+int 
+UnsetEnv (char *name, int Force)
+{
+  int i;
+  EEntry *env = Base.Environment;
+  /* search for name in the environment */
+  for (i = 0; i < Base.NumEnv && STRCASECMP (name, env->Name); i++, env++);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  if (i >= Base.NumEnv ||
+      (Base.Environment[i].Constraint && !Force))
+    return (-1);
+  Xfree (Base.Environment[i].Name);
+  if (Base.Environment[i].Data)
+    Xfree (Base.Environment[i].Data);
+  for (i++; i < Base.NumEnv; i++)
+    Base.Environment[i - 1] = Base.Environment[i];
+  Base.NumEnv--;
+  if (Base.NumEnv == 0)
+    {
+      Xfree (Base.Environment);
+      Base.Environment = NULL;
+    }
+  return (0);
+}
+
+
+
+
+
+
+
+
+/*
+ * Returns the name of environment variable number i or NULL if i is greater 
+ * than or equal to the number of environment variables. Names start from
+ * zero. 
+ */
+char *
+GetEnvName (int i)
+{
+  return (i < Base.NumEnv ? Base.Environment[i].Name : NULL);
+}
+
+static char *BooleanStrs[] =
+{"false", "true", "no", "yes", "off", "on"};
+
+static char *
+BooleanCons (char *Old, char *New)
+{
+  int i;
+  int old = -1, new = -1;
+  if (Old)
+    for (i = 0; i < sizeof (BooleanStrs) / sizeof (char *); i++)
+      if (!STRCASECMP (Old, BooleanStrs[i]))  /* [RPAP - Feb 97: WIN32 Port] */
+	{
+	  old = i;
+	  break;
+	}
+  if (New)
+    for (i = 0; i < sizeof (BooleanStrs) / sizeof (char *); i++)
+      if (!STRCASECMP (New, BooleanStrs[i]))  /* [RPAP - Feb 97: WIN32 Port] */
+	{
+	  new = i;
+	  break;
+	}
+  if (new >= 0)
+    return (New);
+  if (!New)
+    return (BooleanStrs[Old ? old ^ 1 : 0]);
+  ConstraintErrorStr = "Invalid argument [true|false|yes|no|on|off] required";
+  return (NULL);
+}
+
+/*
+ * returns 0 or 1  for a boolean string
+ * or default on a error
+ */
+int 
+BooleanEnv (char *data, int def)
+{
+  int i;
+  if (!data)
+    return (def);
+  for (i = 0; i < sizeof (BooleanStrs) / sizeof (char *); i++)
+    if (!STRCASECMP (data, BooleanStrs[i]))  /* [RPAP - Feb 97: WIN32 Port] */
+      return (i & 1);
+  return (def);
+}
+
+/*
+ * returns the value for a integer string
+ * or default on a error
+ */
+long 
+IntEnv (char *data, long def)
+{
+  long val;
+  char *ptr;
+  if (!data)
+    return (def);
+  val = strtol (data, &ptr, 10);
+  return *ptr ? def : val;
+}
+
+static char *
+NumberCmp (char *num, int min, int max)
+{
+  char *err;
+  int val;
+  static char Err[100];
+  ConstraintErrorStr = "Not a valid number";
+  if (!num)
+    return (NULL);
+  val = strtol (num, &err, 10);
+  if (*err)
+    return (NULL);
+  sprintf (Err, "Not in legal range [%d <= num <= %d]", min, max);
+  ConstraintErrorStr = Err;
+  if (val < min || val > max)
+    return (NULL);
+  return (num);
+}
+
+/*ARGSUSED */
+static char *
+MaxDocsCons (char *Old, char *New)
+{
+  if (!STRCASECMP (New, "all"))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (New);
+  return (NumberCmp (New, 1, ((unsigned) (~0)) >> 1));
+}
+
+/*ARGSUSED */
+static char *
+MaxTermsCons (char *Old, char *New)
+{
+  if (!STRCASECMP (New, "all"))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (New);
+  return (NumberCmp (New, 1, ((unsigned) (~0)) >> 1));
+}
+
+/* ARGSUSED */
+static char *
+BufferCons (char *Old, char *New)
+{
+  return (NumberCmp (New, 0, 16 * 1024 * 1024));
+}
+
+/* ARGSUSED */
+static char *
+MaxNodesCons (char *Old, char *New)
+{
+  if (!STRCASECMP (New, "all"))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (New);
+  return (NumberCmp (New, 8, 256 * 1024 * 1024));
+}
+
+/* ARGSUSED */
+static char *
+MaxHashCons (char *Old, char *New)
+{
+  return (NumberCmp (New, 8, 256 * 1024 * 1024));
+}
+
+/* ARGSUSED */
+static char *
+MaxParasCons (char *Old, char *New)
+{
+  return (NumberCmp (New, 1, 256 * 1024 * 1024));
+}
+
+/* ARGSUSED */
+static char *
+MaxHeadsCons (char *Old, char *New)
+{
+  return (NumberCmp (New, 1, 1000));
+}
+
+/* ARGSUSED */
+char *
+OptimiseCons (char *Old, char *New)
+{
+  return (NumberCmp (New, 0, 2));
+}
+
+
+
+/*
+ * Makes sure that New is a valid query type
+ *
+ */
+/* ARGSUSED */
+static char *
+QueryCons (char *Old, char *New)
+{
+  static char *QueryStrs[] =
+  {
+    "boolean", "ranked",
+    "docnums", "approx-ranked"};
+  int i;
+  int new = -1;
+  if (New)
+    for (i = 0; i < sizeof (QueryStrs) / sizeof (char *); i++)
+      if (!STRCASECMP (New, QueryStrs[i]))  /* [RPAP - Feb 97: WIN32 Port] */
+	{
+	  new = i;
+	  break;
+	}
+  if (new >= 0)
+    return (New);
+  ConstraintErrorStr = "Invalid argument [boolean|ranked|docnums|approx-ranked] required";
+  return (NULL);
+}
+
+/*
+ * Makes sure that New is a valid accum type
+ *
+ */
+/* ARGSUSED */
+static char *
+AccumCons (char *Old, char *New)
+{
+  static char *AccumStrs[] =
+  {
+    "array", "splay_tree", "hash_table", "list"};
+  int i;
+  int new = -1;
+  if (New)
+    for (i = 0; i < sizeof (AccumStrs) / sizeof (char *); i++)
+      if (!STRCASECMP (New, AccumStrs[i]))  /* [RPAP - Feb 97: WIN32 Port] */
+	{
+	  new = i;
+	  break;
+	}
+  if (new >= 0)
+    return (New);
+  ConstraintErrorStr = "Invalid argument "
+    "[array|splay_tree|hash_table|list] required";
+  return (NULL);
+}
+
+
+
+/*
+ * Makes sure that New is a valid output type type
+ *
+ */
+/* ARGSUSED */
+static char *
+OutputTypeCons (char *Old, char *New)
+{
+  static char *OutputTypeStrs[] =
+  {"text", "silent", "docnums",
+   "count", "heads", "hilite"
+#ifdef TREC_MODE
+   ,"extras_for_trec"
+#endif
+  };
+
+  int i;
+  int new = -1;
+  if (New)
+    for (i = 0; i < sizeof (OutputTypeStrs) / sizeof (char *); i++)
+      if (!STRCASECMP (New, OutputTypeStrs[i]))  /* [RPAP - Feb 97: WIN32 Port] */
+	{
+	  new = i;
+	  break;
+	}
+  if (new >= 0)
+    return (New);
+  ConstraintErrorStr = "Invalid argument [text|silent|docnums|heads|count|hilite"
+#ifdef TREC_MODE
+    "|extras_for_trec"
+#endif
+    "] required";
+  return (NULL);
+}
+
+/*
+ * Makes sure that New is a valid query type
+ *
+ */
+/* ARGSUSED */
+char *
+HiliteStyleCons (char *Old, char *New)
+{
+  static char *StyleStrs[] =
+  {"bold", "underline"};
+  int i;
+  int new = -1;
+  if (New)
+    for (i = 0; i < sizeof (StyleStrs) / sizeof (char *); i++)
+      if (!STRCASECMP (New, StyleStrs[i]))  /* [RPAP - Feb 97: WIN32 Port] */
+	{
+	  new = i;
+	  break;
+	}
+  if (new >= 0)
+    return (New);
+  ConstraintErrorStr = "Invalid argument [bold|underline] required";
+  return (NULL);
+}
+
+/* =========================================================================
+ * Function: get_output_type
+ * Description:
+ *      Map a mode string onto a mode char e.g. OUTPUT_????
+ * Input:
+ * Output:
+ *      The mode char which is a subrange value.
+ * ========================================================================= */
+
+char
+get_output_type (void)
+{
+  char *env_str = GetDefEnv ("mode", "text");
+  char ch1 = toupper (env_str[0]);
+  char ch2 = toupper (env_str[1]);
+
+  switch (ch1)
+    {
+    case 'T':
+      return OUTPUT_TEXT;
+    case 'S':
+      return OUTPUT_SILENT;
+    case 'E':
+      return OUTPUT_EXTRAS;
+    case 'C':
+      return OUTPUT_COUNT;
+    case 'D':
+      return OUTPUT_DOCNUMS;
+    case 'H':
+      if (ch2 == 'E')
+	return OUTPUT_HEADERS;
+      else
+	return OUTPUT_HILITE;
+    default:
+      FatalError (1, "Problem in output type switch");
+    }
+
+  return '\0';			/*shouldn't reach here -- keep compiler happy */
+}
+
+/* =========================================================================
+ * Function: get_query_type
+ * Description:
+ *      Map a query type string onto a char.
+ * Input:
+ * Output:
+ *      Query type char.
+ * ========================================================================= */
+
+char
+get_query_type (void)
+{
+  char *env_str = GetDefEnv ("query", "boolean");
+  char ch1 = toupper (env_str[0]);
+
+  switch (ch1)
+    {
+    case 'R':
+      return QUERY_RANKED;
+    case 'A':
+      return QUERY_APPROX;
+    case 'B':
+      return QUERY_BOOLEAN;
+    case 'D':
+      return QUERY_DOCNUMS;
+    default:
+      FatalError (1, "Problem in query type switch");
+    }
+
+  return '\0';			/*shouldn't reach here */
+}
+
+
+
+/*
+ * This initialises certain environment variables
+ *
+ */
+void 
+InitEnv (void)
+{
+  SetEnv ("hilite_style", "underline", HiliteStyleCons);	/*[TS:Sep/94] */
+  SetEnv ("briefstats", "off", BooleanCons);
+  SetEnv ("diskstats", "off", BooleanCons);
+  SetEnv ("expert", "false", BooleanCons);
+  SetEnv ("mgdir", getenv ("MGDATA") ? getenv ("MGDATA") : ".", NULL);
+  SetEnv ("mgname", "", NULL);
+  SetEnv ("maxdocs", "all", MaxDocsCons);
+  SetEnv ("memstats", "off", BooleanCons);
+  SetEnv ("mode", "text", OutputTypeCons);
+  SetEnv ("pager", getenv ("PAGER") ? getenv ("PAGER") : "more", NULL);
+  SetEnv ("qfreq", "true", BooleanCons);
+  SetEnv ("query", "boolean", QueryCons);
+  SetEnv ("sizestats", "off", BooleanCons);
+  SetEnv ("timestats", "off", BooleanCons);
+  SetEnv ("verbatim", "off", BooleanCons);
+  SetEnv ("sorted_terms", "on", BooleanCons);
+  SetEnv ("accumulator_method", "array", AccumCons);
+  SetEnv ("stop_at_max_accum", "off", BooleanCons);
+  SetEnv ("buffer", "1048576", BufferCons);
+  SetEnv ("max_accumulators", "50000", MaxNodesCons);
+  SetEnv ("max_terms", "all", MaxTermsCons);
+  SetEnv ("maxparas", "1000", MaxParasCons);
+  SetEnv ("hash_tbl_size", "1000", MaxHashCons);
+  SetEnv ("skip_dump", "skips.%d", NULL);
+  SetEnv ("ranked_doc_sepstr", "---------------------------------- %n %w\\n", NULL);
+  SetEnv ("doc_sepstr", "---------------------------------- %n\\n", NULL);
+  SetEnv ("para_sepstr", "\\n######## PARAGRAPH %n ########\\n", NULL);
+  SetEnv ("para_start", "***** Weight = %w *****\\n", NULL);
+  SetEnv ("terminator", "", NULL);
+  SetEnv ("heads_length", "50", MaxHeadsCons);
+  SetEnv ("optimise_type", "1", OptimiseCons);	/*[TS:Mar/95] */
+  SetEnv ("casefold", "on", BooleanCons);  /* [RPAP - Jan 97: Stem Index Change] */
+  SetEnv ("stem", "on", BooleanCons);  /* [RPAP - Jan 97: Stem Index Change] */
+  SetEnv ("term_freq", "off", BooleanCons);  /* [RPAP - Feb 97: Term Frequency] */
+}
+
+
+void 
+UninitEnv (void)
+{
+  char *name;
+  while (PopEnv () == 0);
+  while ((name = GetEnvName (0)) != 0)
+    UnsetEnv (name, 1);
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/environment.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/environment.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/environment.h	(revision 699)
@@ -0,0 +1,61 @@
+/**************************************************************************
+ *
+ * environment.h -- mgquery environment functions
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef ENVIRONMENT_H
+#define ENVIRONMENT_H
+
+#define OUTPUT_TEXT     'T'
+#define OUTPUT_HEADERS  'H'
+#define OUTPUT_SILENT   'S'
+#define OUTPUT_EXTRAS   'E'
+#define OUTPUT_HILITE   'I'
+#define OUTPUT_COUNT    'C'
+#define OUTPUT_DOCNUMS  'D'
+
+#define QUERY_RANKED    'R'
+#define QUERY_APPROX    'A'
+#define QUERY_BOOLEAN   'B'
+#define QUERY_DOCNUMS   'D'
+
+
+char get_output_type (void);
+char get_query_type (void);
+
+extern char *ConstraintErrorStr;
+int SetEnv (char *name, char *data, char *(*Constraint) (char *, char *));
+char *GetEnv (char *name);
+char *GetDefEnv (char *name, char *def);
+int UnsetEnv (char *name, int Force);
+char *de_escape_string (char *s);
+int PushEnv (void);
+int PopEnv (void);
+int EnvStackHeight (void);
+char *GetEnvName (int i);
+int BooleanEnv (char *data, int def);
+long IntEnv (char *data, long def);
+
+void InitEnv (void);
+void UninitEnv (void);
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/globals.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/globals.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/globals.h	(revision 699)
@@ -0,0 +1,32 @@
+/**************************************************************************
+ *
+ * globals.h -- Some global stuff for mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef H_GLOBALS
+#define H_GLOBALS
+
+
+extern FILE *OutFile;
+extern FILE *InFile;
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/hash.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/hash.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/hash.h	(revision 699)
@@ -0,0 +1,44 @@
+/**************************************************************************
+ *
+ * hash.h -- Hash function
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef H_HASH
+#define H_HASH
+
+
+
+#define HASH(h, step, wd, tablesize)			\
+	do {						\
+		register u_char *p=wd;			\
+		register long len = *p++;		\
+		register unsigned long hval;		\
+		if (len>25) len = 25;			\
+		for (hval=0; len; len--)		\
+		  hval = (hval << 5) - hval + *p++;	\
+		step = hval % (tablesize - 2) + 1;	\
+		h = hval % tablesize;			\
+	} while (0)
+
+
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/help.mg.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/help.mg.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/help.mg.h	(revision 699)
@@ -0,0 +1,320 @@
+static char *help_str[] = {
+"\n",
+"			  HELP for mgquery\n",
+"		   ================================\n",
+"\n",
+"The text is a summary of the information in the \"mgquery\" manual pages.\n",
+"\n",
+"The input to 'mgquery' consists of a series of input lines. The backslash\n",
+"character (\"\\\") is used at the end of lines to indicate that input \n",
+"continues on the next line.\n",
+"\n",
+"Input lines on which the first character is a dot (\".\") are commands to\n",
+"the mgquery program. Input lines that do not start with a dot are queries.\n",
+"\n",
+"A query consists of two parts. One part is a boolean or ranked query that\n",
+"identifies documents. The second part is a post-processing pattern matching\n",
+"operation. Any text between the first speech mark (\") and the last speech\n",
+"mark is considered to the a post-processing pattern. \n",
+"\n",
+"\n",
+"The following command are available :-\n",
+"\n",
+"    .help	     - displays this text.\n",
+"    .quit            - quits the program.\n",
+"    .set name value  - sets parameter \"name\" to \"value\" . If the parameter\n",
+"                       is a boolean parameter and value is omitted the\n",
+"	    	       parameter will be inverted (i.e. if it is true it\n",
+"		       will change to false, if it is false it will change\n",
+"		       to true).\n",
+"    .unset name      - deletes parameter \"name\"\n",
+"    .reset           - sets all the parameters to their initial state.\n",
+"    .display         - displays the values of all the current parameters.\n",
+"    .push            - pushes the current parameters on to a stack.\n",
+"    .pop             - destroys the current parameters and pops a new set\n",
+"                       of parameters off the stack.\n",
+"    .output arg      - This is used to specify where to send the documents.\n",
+"                       Arg may one be of the following:\n",
+"		       > filename  : Send output to the specified file.\n",
+"		       >> filename : Append output to the specified file.\n",
+"                       | command   : The output is piped into command, \n",
+"                                     which is executed by sh.\n",
+"    .input arg       - This is used to specify where input comes from.\n",
+"                       Arg may one be of the following:\n",
+"		       < filename  : Get the input from the specified\n",
+"                                     file.\n",
+"                       | command   : The input comes from the standard\n",
+"                                     output of command, which is executed\n",
+"                                     by sh.\n",
+"\n",
+"On startup the mgquery program reads from the file .mgrc a sequence \n",
+"of commands (NOTE: The .mgrc file may not contain any queries). mgquery \n",
+"first looks for .mgrc in the current directory and then in the users home \n",
+"directory. Lines starting with a '#' in the .mgrc file are considered to\n",
+"be comments and are ignored.\n",
+"\n",
+"The following parameters (used in the .set and .unset commands) are \n",
+"predefined and have special significance :-\n",
+"\n",
+"accumulator_method = `array'\n",
+"	This parameter is used during ranking, and specifies how the\n",
+"	weight for each document should be accumulated. The following\n",
+"	methods are available `array', `splay_tree', `hash_table', and\n",
+"	`list'.\n",
+"\n",
+"briefstats = `off'\n",
+"	This is a boolean parameter that determines whether the the\n",
+"	totals for disk, memory and time usage statistics will be \n",
+"	displayed. at the end of each query. \n",
+"	NOTE: this takes precedence over the parameters \"diskstats\",\n",
+"	\"memstats\" and \"timestats\". This parameter may take the values \n",
+"	`yes', `no', `true', `false', `on' or `off'.\n",
+"\n",
+"buffer = `1048576'\n",
+"	When the documents are being read in they are read into a \n",
+"	buffer of this size and then displayed from this buffer. If \n",
+"        the documents are larger than this buffer the buffer is\n",
+"	expanded automatically. Having a large buffer gives a very\n",
+"	slight performance improvement because it allows the order of \n",
+"	disk operations to be optimised. The buffer size is measured\n",
+"	in bytes.\n",
+"\n",
+"diskstats = `off'\n",
+"	This is a boolean parameter that determines whether the disk\n",
+"	usage statistics for the preceding query will be displayed\n",
+"	after each query. This parameter may take the values `yes',\n",
+"	`no', `true', `false', `on' or `off'.\n",
+"\n",
+"doc_sepstr = `---------------------------------- %n\\n'\n",
+"	This specifies the string that will be used to separate \n",
+"	documents when they are displayed for `boolean' or `docnums'\n",
+"	queries. The standard C escape character sequences (see the \n",
+"	man page) may be used to place special characters in the \n",
+"	string. For example, a newline would the `\\n'. To include a `%'\n",
+"	use the sequence `%%'. To include the MG document number use\n",
+"	the sequence `%n'.\n",
+"\n",
+"expert = `false'\n",
+"	If this is true then a lot of the waffle that the program \n",
+"	spits out is suppressed. This parameter may take the values\n",
+"	`yes', `no', `true', `false', `on' or `off'.\n",
+"\n",
+"hash_tbl_size = `1000'\n",
+"	One of the options during ranking queries is to use a hash \n",
+"	table to accumulate the weights for each document. The hash \n",
+"	table is a simple chained type. This parameter specifies the \n",
+"	size of the hash table and may take any value between 8 and\n",
+"	268435456. \n",
+"\n",
+"heads_length = `50'\n",
+"	When the mode is `heads' this specifies the number of \n",
+"	characters that	will be output for each document.\n",
+"\n",
+"maxdocs = `all'\n",
+"	The maximum number of documents to display in response to a\n",
+"	query. This parameter may take on a numeric value between 1 \n",
+"	and 429467295 or the word `all'.\n",
+"\n",
+"maxparas = `1000'\n",
+"	The maximum number of paragraphs to identify during a ranked\n",
+"	query with paragraph indexing. After the paragraphs have been\n",
+"	identified the paragraphs are converted into documents, and \n",
+"	because some of the paragraphs may refer to the same documents \n",
+"	the final number of answers may be less that maxparas. The \n",
+"	maxdocs parameter will then be applied. This parameter may \n",
+"	take on a numeric value between 1 and 429467295.\n",
+"\n",
+"max_accumulators = `50000'\n",
+"	This parameter limits the number of different paragraph/\n",
+"	document numbers to be accumulated during ranked queries when \n",
+"	the parameter `accumulator_method` is set to `splay_tree',\n",
+"	`hash_table', or `list'.  This parameter may take any value \n",
+"	between 8 and 268435456.\n",
+"\n",
+"max_terms = `all'\n",
+"	This parameter limits the number of terms that will actually\n",
+"	be used during a ranked query. If more terms than the number\n",
+"	specified by max_terms are entered, then the extra terms will\n",
+"	be discarded. If `sorted_terms' is on then the limiting will \n",
+"	be done after the terms have been sorted. This parameter may\n",
+"	take any value between 1 and 429467295 or the word `all'.\n",
+"\n",
+"memstats = `off'\n",
+"	This is a boolean parameter that determines whether the memory \n",
+"	usage statistics for the preceding query will be displayed\n",
+"	after each query. This parameter may take the values `yes', \n",
+"	`no', `true', `false', `on' or `off'.\n",
+"\n",
+"mgdir = `.'\n",
+"	This specifies the directory where the MG files may be found.\n",
+"	If the environment variable `MGDATA' is set then `mgdir' is\n",
+"	initialised to the value in `MGDATA'.\n",
+"\n",
+"mgname = `'\n",
+"	This specifies the name of the MG database to process.\n",
+"\n",
+"mode = `text'\n",
+"	This specifies how documents should be displayed when they\n",
+"	are retrieved it may take four different values `text', \n",
+"	`docnums', `silent', `heads' or `count'. `text' displays \n",
+"	the contents of the document. `docnums' displays only the\n",
+"	document numbers. `Silent' retrieves all the documents but\n",
+"	displays nothing except how many documents were retrieved.\n",
+"	This mode is intended to be used in timing experiments. \n",
+"	`Heads` is used to print out the head of each document.\n",
+"	`Count' does the minimum amount of work required to determine\n",
+"	how many documents would be retrieved, but does not retrieve\n",
+"	them.\n",
+"\n",
+"pager = `more'\n",
+"	This is the name of the program that will be used to display\n",
+"	the help and the retrieved documents. If the environment \n",
+"	variable \"PAGER\" is defined then `pager' takes on that value.\n",
+"\n",
+"para_sepstr = `\\n######## PARAGRAPH %n ########\\n'\n",
+"	This specifies the string that will be used to separate \n",
+"	paragraphs. The standard C escape character sequences (see the \n",
+"	man page) may be used to place special characters in the \n",
+"	string. For example, a newline would the `\\n'. To include a `%'\n",
+"	use the sequence `%%'. To include the paragraph number within\n",
+"	the document use the sequence `%n'.\n",
+"\n",
+"para_start = `***** Weight = %w *****\\n'\n",
+"	This specifies the string that will be used at the head of \n",
+"	paragraphs for a paraghaph level index following a ranked query.\n",
+"	The standard C escape character sequences (see the man page)\n",
+"	may be used to place special characters in the string. For \n",
+"	example, a newline would the `\\n'. To include a `%' use the\n",
+"	sequence `%%'. To include the paragraph weight use the \n",
+"	sequence `%w'.\n",
+"\n",
+"qfreq = `true'\n",
+"	This determine whether the ranked queries will take into \n",
+"	account the number of times each query term is specified.\n",
+"	When this is `true' the number of times a term appears in\n",
+"	the query is used in the ranking. When this is `false' all \n",
+"	query term are assumed to occur only once. This parameter\n",
+"	may take the values `yes', `no', `true', `false', `on' or\n",
+"	`off'.\n",
+"\n",
+"query = `boolean'\n",
+"	This specifies the type of queries that are to be specified.\n",
+"	It can take four different values `boolean', `ranked', \n",
+"	`docnums' or `approx-ranked'. \n",
+"\n",
+"	`boolean' is for boolean queries. \n",
+"		The yacc grammar for boolean queries is as follows :-\n",
+"\n",
+"		query   : or;\n",
+" \n",
+"		or      : or '|' and\n",
+"		        | and ;	\n",
+" \n",
+"		and     : and '&' not\n",
+"        		| and not\n",
+"        		| not ;\n",
+"\n",
+"		not     : term\n",
+"		        | '!' not ;\n",
+" \n",
+"		term    : TERM\n",
+"		        | '(' or ')' ;\n",
+" \n",
+"\n",
+"\n",
+"\n",
+"	`ranked' and `approx-ranked' are for queries ranked by the\n",
+"		cosine measure. `approx-ranked' uses only the low\n",
+"		precision document lengths, and therefore only\n",
+"		produces an approximation to full cosine ranking.\n",
+"\n",
+"		query   : TERM\n",
+"      			| query TERM ;\n",
+"	\n",
+"	`docnums' allows the entry of document numbers. Multiple \n",
+"		numbers separated by spaces may be specified\n",
+"		or ranges separated by hyphens.\n",
+"\n",
+"\n",
+"		query   : range\n",
+"			| query range ;\n",
+"\n",
+"\n",
+"		range   : num\n",
+"			| num '-' num ;\n",
+"\n",
+"\n",
+"ranked_doc_sepstr = `---------------------------------- %n %w\\n'\n",
+"	This specifies the string that will be used to separate \n",
+"	documents when they are displayed for `ranked' or \n",
+"	`approx-ranked'	queries. The standard C escape character \n",
+"	sequences (see the man page) may be used to place special\n",
+"	characters in the string. For example, a newline would the\n",
+"	`\\n'. To include a `%' use the sequence `%%'. To include the\n",
+"	MG document number use the sequence `%n'. To include the\n",
+"	document weight use the sequence `%w'.\n",
+"\n",
+"sizestats = `false'\n",
+"	If this is true then various numbers are output at the end\n",
+"	of each query indicating what went on during the query. \n",
+"	This parameter may take the values `yes', `no', `true', \n",
+"	`false', `on' or `off'.\n",
+"\n",
+"skip_dump = `skips.%d'\n",
+"	If this parameter is set then during ranked queries on skipped\n",
+"	inverted files when `accumulator_method' is set to `splay_tree',\n",
+"	`hash_table', or `list` a file will be produced in the current\n",
+"	directory. The name of the file is the value of this parameter,\n",
+"	a `%d' in the file name will be replaced with the process id of\n",
+"	mgquery. This file will contain information about the usage of\n",
+"	skips during the query processing. This option is expensive; \n",
+"	use `.unset skip_dump' to obtain optimal performance.\n",
+"\n",
+"sorted_terms = `on'\n",
+"	This specifies whether of not the terms should be sorted into\n",
+"	decreasing occurrence in documents so that the least often\n",
+"	occurring terms are processed first when ranked queries are\n",
+"	being done. When this is true the terms are sorted. When this\n",
+"	is false the terms are not sorted and are instead processed in\n",
+"	order of occurrence. This parameter may take the values `yes',\n",
+"	`no', `true', `false', `on' or `off'.\n",
+"\n",
+"\n",
+"stop_at_max_accum = `on'\n",
+"	This specifies what should happen when the maximum number of\n",
+"	accumulators set by `max_accumulators' is reached. When this\n",
+"	is true the the processing of terms is stopped at the completion\n",
+"	of the current term. When this is false processing continues but\n",
+"	no new accumulators are created. This parameter may take the \n",
+"	values `yes', `no', `true', `false', `on' or `off'.\n",
+"\n",
+"terminator = `'\n",
+"	This specifies the string that will be output after the	last\n",
+"	document from the previous query has been output. The standard\n",
+"	C escape character sequences (see the man page) may be used to\n",
+"	place special characters in the string. For example, a newline\n",
+"	would the `\\n'. To include a `%' use the sequence `%%'.\n",
+"	\n",
+"\n",
+"timestats = `false'\n",
+"	If this is true then the time to process a query is displayed\n",
+"	in both real time and CPU time. This parameter may take the\n",
+"	values `yes', `no', `true', `false', `on' or `off'.\n",
+"\n",
+"verbatim = `off'\n",
+"	This is a boolean parameter that determines whether the program\n",
+"	should attempt to do a regular expression match on the retrieved\n",
+"	text. If verbatim is `on' and a post-processing strng is specified\n",
+"	with the query then the post-processing string will be searched for\n",
+"	in the documents just before they are displayed. If the string is\n",
+"	found the document will be displayed, if not the document will not\n",
+"	be displayed. If verbatim is `off' the post-processing string will\n",
+"	be considered a regular expression like in `vi' or `egrep'.\n",
+"	E.G. If verbatim is `on', \"and.*the\" will look for the 8 character\n",
+"	     sequence \"and.*the\". If verbatim is `off', \"and.*the\" will\n",
+"	     look for the sequence \"and\" followed somewhere later in the\n",
+"	     document by the sequence \"the\".\n",
+"	This parameter may take the values `yes', `no', `true', `false',\n",
+"	`on' or `off'.\n",
+"\n",
+"\n"};
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/help.mg.src
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/help.mg.src	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/help.mg.src	(revision 699)
@@ -0,0 +1,347 @@
+###########################################################################
+#
+# help.mg.src -- Source for the help command
+# Copyright (C) 1994  Neil Sharman
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)help.mg.src	1.8 21 Mar 1994
+#
+###########################################################################
+#
+# The help file for mgquery.  
+# Lines starting with '#' are treated as comments and are discarded when
+# the 'help.mg.h' file is produced
+#
+###########################################################################
+
+			  HELP for mgquery
+		   ================================
+
+The text is a summary of the information in the "mgquery" manual pages.
+
+The input to 'mgquery' consists of a series of input lines. The backslash
+character ("\") is used at the end of lines to indicate that input 
+continues on the next line.
+
+Input lines on which the first character is a dot (".") are commands to
+the mgquery program. Input lines that do not start with a dot are queries.
+
+A query consists of two parts. One part is a boolean or ranked query that
+identifies documents. The second part is a post-processing pattern matching
+operation. Any text between the first speech mark (") and the last speech
+mark is considered to the a post-processing pattern. 
+
+
+The following command are available :-
+
+    .help	     - displays this text.
+    .quit            - quits the program.
+    .set name value  - sets parameter "name" to "value" . If the parameter
+                       is a boolean parameter and value is omitted the
+	    	       parameter will be inverted (i.e. if it is true it
+		       will change to false, if it is false it will change
+		       to true).
+    .unset name      - deletes parameter "name"
+    .reset           - sets all the parameters to their initial state.
+    .display         - displays the values of all the current parameters.
+    .push            - pushes the current parameters on to a stack.
+    .pop             - destroys the current parameters and pops a new set
+                       of parameters off the stack.
+    .output arg      - This is used to specify where to send the documents.
+                       Arg may one be of the following:
+		       > filename  : Send output to the specified file.
+		       >> filename : Append output to the specified file.
+                       | command   : The output is piped into command, 
+                                     which is executed by sh.
+    .input arg       - This is used to specify where input comes from.
+                       Arg may one be of the following:
+		       < filename  : Get the input from the specified
+                                     file.
+                       | command   : The input comes from the standard
+                                     output of command, which is executed
+                                     by sh.
+
+On startup the mgquery program reads from the file .mgrc a sequence 
+of commands (NOTE: The .mgrc file may not contain any queries). mgquery 
+first looks for .mgrc in the current directory and then in the users home 
+directory. Lines starting with a '#' in the .mgrc file are considered to
+be comments and are ignored.
+
+The following parameters (used in the .set and .unset commands) are 
+predefined and have special significance :-
+
+accumulator_method = `array'
+	This parameter is used during ranking, and specifies how the
+	weight for each document should be accumulated. The following
+	methods are available `array', `splay_tree', `hash_table', and
+	`list'.
+
+briefstats = `off'
+	This is a boolean parameter that determines whether the the
+	totals for disk, memory and time usage statistics will be 
+	displayed. at the end of each query. 
+	NOTE: this takes precedence over the parameters "diskstats",
+	"memstats" and "timestats". This parameter may take the values 
+	`yes', `no', `true', `false', `on' or `off'.
+
+buffer = `1048576'
+	When the documents are being read in they are read into a 
+	buffer of this size and then displayed from this buffer. If 
+        the documents are larger than this buffer the buffer is
+	expanded automatically. Having a large buffer gives a very
+	slight performance improvement because it allows the order of 
+	disk operations to be optimised. The buffer size is measured
+	in bytes.
+
+diskstats = `off'
+	This is a boolean parameter that determines whether the disk
+	usage statistics for the preceding query will be displayed
+	after each query. This parameter may take the values `yes',
+	`no', `true', `false', `on' or `off'.
+
+doc_sepstr = `---------------------------------- %n\n'
+	This specifies the string that will be used to separate 
+	documents when they are displayed for `boolean' or `docnums'
+	queries. The standard C escape character sequences (see the 
+	man page) may be used to place special characters in the 
+	string. For example, a newline would the `\n'. To include a `%'
+	use the sequence `%%'. To include the MG document number use
+	the sequence `%n'.
+
+expert = `false'
+	If this is true then a lot of the waffle that the program 
+	spits out is suppressed. This parameter may take the values
+	`yes', `no', `true', `false', `on' or `off'.
+
+hash_tbl_size = `1000'
+	One of the options during ranking queries is to use a hash 
+	table to accumulate the weights for each document. The hash 
+	table is a simple chained type. This parameter specifies the 
+	size of the hash table and may take any value between 8 and
+	268435456. 
+
+heads_length = `50'
+	When the mode is `heads' this specifies the number of 
+	characters that	will be output for each document.
+
+maxdocs = `all'
+	The maximum number of documents to display in response to a
+	query. This parameter may take on a numeric value between 1 
+	and 429467295 or the word `all'.
+
+maxparas = `1000'
+	The maximum number of paragraphs to identify during a ranked
+	query with paragraph indexing. After the paragraphs have been
+	identified the paragraphs are converted into documents, and 
+	because some of the paragraphs may refer to the same documents 
+	the final number of answers may be less that maxparas. The 
+	maxdocs parameter will then be applied. This parameter may 
+	take on a numeric value between 1 and 429467295.
+
+max_accumulators = `50000'
+	This parameter limits the number of different paragraph/
+	document numbers to be accumulated during ranked queries when 
+	the parameter `accumulator_method` is set to `splay_tree',
+	`hash_table', or `list'.  This parameter may take any value 
+	between 8 and 268435456.
+
+max_terms = `all'
+	This parameter limits the number of terms that will actually
+	be used during a ranked query. If more terms than the number
+	specified by max_terms are entered, then the extra terms will
+	be discarded. If `sorted_terms' is on then the limiting will 
+	be done after the terms have been sorted. This parameter may
+	take any value between 1 and 429467295 or the word `all'.
+
+memstats = `off'
+	This is a boolean parameter that determines whether the memory 
+	usage statistics for the preceding query will be displayed
+	after each query. This parameter may take the values `yes', 
+	`no', `true', `false', `on' or `off'.
+
+mgdir = `.'
+	This specifies the directory where the MG files may be found.
+	If the environment variable `MGDATA' is set then `mgdir' is
+	initialised to the value in `MGDATA'.
+
+mgname = `'
+	This specifies the name of the MG database to process.
+
+mode = `text'
+	This specifies how documents should be displayed when they
+	are retrieved it may take four different values `text', 
+	`docnums', `silent', `heads' or `count'. `text' displays 
+	the contents of the document. `docnums' displays only the
+	document numbers. `Silent' retrieves all the documents but
+	displays nothing except how many documents were retrieved.
+	This mode is intended to be used in timing experiments. 
+	`Heads` is used to print out the head of each document.
+	`Count' does the minimum amount of work required to determine
+	how many documents would be retrieved, but does not retrieve
+	them.
+
+pager = `more'
+	This is the name of the program that will be used to display
+	the help and the retrieved documents. If the environment 
+	variable "PAGER" is defined then `pager' takes on that value.
+
+para_sepstr = `\n######## PARAGRAPH %n ########\n'
+	This specifies the string that will be used to separate 
+	paragraphs. The standard C escape character sequences (see the 
+	man page) may be used to place special characters in the 
+	string. For example, a newline would the `\n'. To include a `%'
+	use the sequence `%%'. To include the paragraph number within
+	the document use the sequence `%n'.
+
+para_start = `***** Weight = %w *****\n'
+	This specifies the string that will be used at the head of 
+	paragraphs for a paraghaph level index following a ranked query.
+	The standard C escape character sequences (see the man page)
+	may be used to place special characters in the string. For 
+	example, a newline would the `\n'. To include a `%' use the
+	sequence `%%'. To include the paragraph weight use the 
+	sequence `%w'.
+
+qfreq = `true'
+	This determine whether the ranked queries will take into 
+	account the number of times each query term is specified.
+	When this is `true' the number of times a term appears in
+	the query is used in the ranking. When this is `false' all 
+	query term are assumed to occur only once. This parameter
+	may take the values `yes', `no', `true', `false', `on' or
+	`off'.
+
+query = `boolean'
+	This specifies the type of queries that are to be specified.
+	It can take four different values `boolean', `ranked', 
+	`docnums' or `approx-ranked'. 
+
+	`boolean' is for boolean queries. 
+		The yacc grammar for boolean queries is as follows :-
+
+		query   : or;
+ 
+		or      : or '|' and
+		        | and ;	
+ 
+		and     : and '&' not
+        		| and not
+        		| not ;
+
+		not     : term
+		        | '!' not ;
+ 
+		term    : TERM
+		        | '(' or ')' ;
+ 
+
+
+
+	`ranked' and `approx-ranked' are for queries ranked by the
+		cosine measure. `approx-ranked' uses only the low
+		precision document lengths, and therefore only
+		produces an approximation to full cosine ranking.
+
+		query   : TERM
+      			| query TERM ;
+	
+	`docnums' allows the entry of document numbers. Multiple 
+		numbers separated by spaces may be specified
+		or ranges separated by hyphens.
+
+
+		query   : range
+			| query range ;
+
+
+		range   : num
+			| num '-' num ;
+
+
+ranked_doc_sepstr = `---------------------------------- %n %w\n'
+	This specifies the string that will be used to separate 
+	documents when they are displayed for `ranked' or 
+	`approx-ranked'	queries. The standard C escape character 
+	sequences (see the man page) may be used to place special
+	characters in the string. For example, a newline would the
+	`\n'. To include a `%' use the sequence `%%'. To include the
+	MG document number use the sequence `%n'. To include the
+	document weight use the sequence `%w'.
+
+sizestats = `false'
+	If this is true then various numbers are output at the end
+	of each query indicating what went on during the query. 
+	This parameter may take the values `yes', `no', `true', 
+	`false', `on' or `off'.
+
+skip_dump = `skips.%d'
+	If this parameter is set then during ranked queries on skipped
+	inverted files when `accumulator_method' is set to `splay_tree',
+	`hash_table', or `list` a file will be produced in the current
+	directory. The name of the file is the value of this parameter,
+	a `%d' in the file name will be replaced with the process id of
+	mgquery. This file will contain information about the usage of
+	skips during the query processing. This option is expensive; 
+	use `.unset skip_dump' to obtain optimal performance.
+
+sorted_terms = `on'
+	This specifies whether of not the terms should be sorted into
+	decreasing occurrence in documents so that the least often
+	occurring terms are processed first when ranked queries are
+	being done. When this is true the terms are sorted. When this
+	is false the terms are not sorted and are instead processed in
+	order of occurrence. This parameter may take the values `yes',
+	`no', `true', `false', `on' or `off'.
+
+
+stop_at_max_accum = `on'
+	This specifies what should happen when the maximum number of
+	accumulators set by `max_accumulators' is reached. When this
+	is true the the processing of terms is stopped at the completion
+	of the current term. When this is false processing continues but
+	no new accumulators are created. This parameter may take the 
+	values `yes', `no', `true', `false', `on' or `off'.
+
+terminator = `'
+	This specifies the string that will be output after the	last
+	document from the previous query has been output. The standard
+	C escape character sequences (see the man page) may be used to
+	place special characters in the string. For example, a newline
+	would the `\n'. To include a `%' use the sequence `%%'.
+	
+
+timestats = `false'
+	If this is true then the time to process a query is displayed
+	in both real time and CPU time. This parameter may take the
+	values `yes', `no', `true', `false', `on' or `off'.
+
+verbatim = `off'
+	This is a boolean parameter that determines whether the program
+	should attempt to do a regular expression match on the retrieved
+	text. If verbatim is `on' and a post-processing strng is specified
+	with the query then the post-processing string will be searched for
+	in the documents just before they are displayed. If the string is
+	found the document will be displayed, if not the document will not
+	be displayed. If verbatim is `off' the post-processing string will
+	be considered a regular expression like in `vi' or `egrep'.
+	E.G. If verbatim is `on', "and.*the" will look for the 8 character
+	     sequence "and.*the". If verbatim is `off', "and.*the" will
+	     look for the sequence "and" followed somewhere later in the
+	     document by the sequence "the".
+	This parameter may take the values `yes', `no', `true', `false',
+	`on' or `off'.
+
+
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/invf.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/invf.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/invf.h	(revision 699)
@@ -0,0 +1,79 @@
+/**************************************************************************
+ *
+ * invf.h -- Data structures for inverted files
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+
+#ifndef H_INVF
+#define H_INVF
+
+/* NOTE: This does not include the magic number */
+struct invf_dict_header
+  {
+    unsigned long lookback;
+    unsigned long dict_size;
+    unsigned long total_bytes;
+    unsigned long index_string_bytes;
+    double input_bytes; /* [RJM 07/97: 4G limit] */
+    unsigned long num_of_docs;
+    unsigned long static_num_of_docs;
+    unsigned long num_of_words;
+    unsigned long stemmer_num;
+    unsigned long stem_method;
+  };
+
+struct stem_dict_header
+  {
+    unsigned long lookback;
+    unsigned long block_size;
+    unsigned long num_blocks;
+    unsigned long blocks_start;
+    unsigned long index_chars;
+    unsigned long num_of_docs;
+    unsigned long static_num_of_docs;
+    unsigned long num_of_words;
+    unsigned long stemmer_num;
+    unsigned long stem_method;
+    unsigned long indexed;       /* [RPAP - Jan 97: Stem Index Change] */
+  };
+
+struct invf_file_header
+  {
+    unsigned long no_of_words;
+    unsigned long no_of_ptrs;
+    unsigned long skip_mode;
+    unsigned long params[16];
+    unsigned long InvfLevel;
+  };
+
+/* [RPAP - Jan 97: Stem Index Change] */
+struct stem_idx_header
+  {
+    unsigned long lookback;
+    unsigned long block_size;
+    unsigned long num_blocks;
+    unsigned long blocks_start;
+    unsigned long index_chars;
+    unsigned long num_of_words;
+  };
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/invf_get.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/invf_get.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/invf_get.c	(revision 699)
@@ -0,0 +1,1666 @@
+/**************************************************************************
+ *
+ * invf_get.c -- Inverted file reading routines
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "filestats.h"
+#include "timing.h"
+#include "bitio_gen.h"
+#include "bitio_m_mem.h"
+#include "bitio_m.h"
+#include "sptree.h"
+#include "messages.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg.h"
+#include "invf.h"
+#include "text.h"
+#include "lists.h"
+#include "backend.h"
+#include "invf_get.h"
+#include "mg_errors.h"
+
+#define SKIPPING
+#define SHORT_CUT
+
+struct pool
+  {
+    int length, pos;
+    struct pool *next;
+    Invf_Doc_Entry ide[1];
+  };
+
+static int skip_header = 0;
+
+static char skip_header_text0[] =
+"#######################################################################\n"
+"#\n"
+"# 1\tWord number.\n"
+"# 2\tNumber of time the word occurs in the collection.\n"
+"# 3\tNumber of documents the word occurs in.\n"
+"# 4\tThe word.\n"
+"# 5\tNumber of docnum/word_count entries decoded.\n"
+"# 6\tNumber of hits.\n"
+"# 7\tNumber of entries in the splay tree or hash table.\n"
+"# 8\tAmount added to the accumulators while while processing this word.\n"
+"#\n"
+"#######################################################################\n";
+
+static char skip_header_text1[] =
+"#######################################################################\n"
+"#\n"
+"# 1\tWord number.\n"
+"# 2\tNumber of time the word occurs in the collection.\n"
+"# 3\tNumber of documents the word occurs in.\n"
+"# 4\tThe word.\n"
+"# 5\tNumber of skips taken in evaluating the inverted file entry.\n"
+"# 6\tNumber of docnum/word_count entries decoded.\n"
+"# 7\tNumber of hits.\n"
+"# 8\tNumber of entries in the splay tree or hash table.\n"
+"# 9\tAmount added to the accumulators while while processing this word.\n"
+"#\n"
+"#######################################################################\n";
+
+
+static char skip_header_text2[] =
+"#######################################################################\n"
+"#\n"
+"#  1\tWord number.\n"
+"#  2\tNumber of time the word occurs in the collection.\n"
+"#  3\tNumber of documents the word occurs in.\n"
+"#  4\tThe word.\n"
+"#  5\tSkip size.\n"
+"#  6\tNumber of skips taken in evaluating the inverted file entry.\n"
+"#  7\tNumber of docnum/word_count entries decoded.\n"
+"#  8\tNumber of hits.\n"
+"#  9\tNumber of entries in the splay tree or hash table.\n"
+"# 10\tAmount added to the accumulators while while processing this word.\n"
+"#\n"
+"#######################################################################\n";
+
+static char *skip_header_text[] =
+{skip_header_text0,
+ skip_header_text1,
+ skip_header_text2};
+
+
+
+
+
+
+
+
+/**************************************************************************
+ *
+ * Code for the hash table.
+ *
+ */
+
+static Hash_Table *
+HT_create (query_data * qd, unsigned max_entries,
+	   unsigned max_size)
+{
+  unsigned es, hts;
+  Hash_Table *HT;
+  if (max_entries == 0)
+    max_entries = 8192;
+  es = sizeof (Hash_Table) + sizeof (Invf_Doc_EntryH) * (max_entries - 1);
+  hts = sizeof (Invf_Doc_EntryH *) * max_size;
+
+  HT = Xmalloc (es);
+  if (!HT)
+    return NULL;
+
+  HT->HashTable = Xmalloc (hts);
+  if (!(HT->HashTable))
+    {
+      Xfree (HT);
+      return NULL;
+    }
+
+  ChangeMemInUse (qd, es + hts);
+  bzero ((char *) (HT->HashTable), hts);
+  HT->max = max_entries;
+  HT->num = 0;
+  HT->Suplimentary_Entries = NULL;
+  HT->Suplimentary_Size = 0;
+  HT->Suplimentary_Num = 0;
+  HT->Head = NULL;
+  HT->size = max_size;
+  return HT;
+}
+
+
+static void 
+HT_Sup_create (query_data * qd, Hash_Table * HT, unsigned max_size)
+{
+  int es = sizeof (Invf_Doc_EntryH) * max_size;
+  HT->Suplimentary_Entries = Xmalloc (es);
+  if (!HT->Suplimentary_Entries)
+    return;
+  HT->Suplimentary_Size = max_size;
+  HT->Suplimentary_Num = 0;
+  ChangeMemInUse (qd, es);
+}
+
+static void 
+HT_Sup_Add (Hash_Table * HT, unsigned long DN, float Sum)
+{
+  Invf_Doc_EntryH *IDEH;
+  IDEH = &HT->Suplimentary_Entries[HT->Suplimentary_Num++];
+  IDEH->next = NULL;
+  IDEH->IDE.DocNum = DN;
+  IDEH->IDE.Sum = Sum;
+}
+
+Invf_Doc_Entry *
+HT_find (Hash_Table * HT, unsigned long DN)
+{
+  Invf_Doc_EntryH *IDEH;
+  if (!HT->HashTable)
+    FatalError (1, "Accessing the hash_table through HT_find after HT_sort");
+  IDEH = HT->HashTable[DN % HT->size];
+  while (IDEH)
+    {
+      if (IDEH->IDE.DocNum == DN)
+	return &(IDEH->IDE);
+      IDEH = IDEH->next;
+    }
+  return NULL;
+}
+
+
+static int 
+HT_insert (Hash_Table * HT,
+	   unsigned long DN, float Sum)
+{
+  Invf_Doc_EntryH **IDEH;
+  if (HT->num == HT->max)
+    return 0;
+  IDEH = &(HT->HashTable[DN % HT->size]);
+  while (*IDEH)
+    IDEH = &((*IDEH)->next);
+  *IDEH = &(HT->IDEH[HT->num++]);
+  (*IDEH)->next = NULL;
+  (*IDEH)->IDE.DocNum = DN;
+  (*IDEH)->IDE.Sum = Sum;
+  return 1;
+}
+
+
+void 
+HT_free (query_data * qd, Hash_Table * HT)
+{
+  unsigned es;
+  es = sizeof (Hash_Table) + sizeof (Invf_Doc_EntryH) * HT->max +
+    sizeof (Invf_Doc_EntryH *) * HT->size;
+  if (HT->HashTable)
+    Xfree (HT->HashTable);
+  if (HT->Suplimentary_Entries)
+    {
+      Xfree (HT->Suplimentary_Entries);
+      es += sizeof (Invf_Doc_EntryH) * HT->Suplimentary_Size;
+    }
+  Xfree (HT);
+  ChangeMemInUse (qd, -es);
+}
+
+
+static int 
+IDEH_comp (const void *a, const void *b)
+{
+  const Invf_Doc_EntryH *A = a;
+  const Invf_Doc_EntryH *B = b;
+  return A->IDE.DocNum - B->IDE.DocNum;
+}
+
+static void 
+HT_sort (query_data * qd, Hash_Table * HT)
+{
+  int i;
+  Invf_Doc_EntryH *l, *r, **b;
+  unsigned es = sizeof (Invf_Doc_EntryH *) * HT->size;
+  if (HT->HashTable)
+    Xfree (HT->HashTable);
+  HT->size = 0;
+  HT->HashTable = NULL;
+  ChangeMemInUse (qd, -es);
+
+  qsort (HT->IDEH, HT->num, sizeof (Invf_Doc_EntryH), IDEH_comp);
+
+  for (i = 0; i < HT->num - 1; i++)
+    HT->IDEH[i].next = &HT->IDEH[i + 1];
+  HT->IDEH[i].next = NULL;
+  l = HT->IDEH;
+
+  if (HT->Suplimentary_Entries)
+    {
+      for (i = 0; i < HT->Suplimentary_Num - 1; i++)
+	HT->Suplimentary_Entries[i].next = &HT->Suplimentary_Entries[i + 1];
+      HT->Suplimentary_Entries[i].next = NULL;
+    }
+  r = HT->Suplimentary_Entries;
+
+  b = &HT->Head;
+
+  while (l || r)
+    {
+      if (r && (!l || (l->IDE.DocNum > r->IDE.DocNum)))
+	{
+	  *b = r;
+	  b = &r->next;
+	  r = *b;
+	}
+      else
+	{
+	  *b = l;
+	  b = &l->next;
+	  l = *b;
+	}
+    }
+  *b = NULL;
+}
+
+
+
+
+
+
+/**************************************************************************
+ *
+ * Code for the hash table.
+ *
+ */
+
+static List_Table *
+LT_create (query_data * qd, unsigned max_entries)
+{
+  unsigned es;
+  List_Table *LT;
+  if (max_entries == 0)
+    max_entries = 8192;
+  es = sizeof (List_Table) + sizeof (Invf_Doc_Entry) * (max_entries - 1);
+
+  LT = Xmalloc (es);
+  if (!LT)
+    return NULL;
+
+  ChangeMemInUse (qd, es);
+  LT->max = max_entries;
+  LT->num = 0;
+  return LT;
+}
+
+
+
+Invf_Doc_Entry *
+LT_find (List_Table * LT, unsigned long DN)
+{
+  int l = 0;
+  int r = LT->num - 1;
+  int m, c;
+  while (l <= r)
+    {
+      m = (l + r) / 2;
+      c = DN - LT->IDE[m].DocNum;
+      if (c < 0)
+	r = m - 1;
+      else if (c > 0)
+	l = m + 1;
+      else
+	return &LT->IDE[m];
+    }
+  return NULL;
+}
+
+static List_Table *
+LT_add (query_data * qd, List_Table * LT,
+	unsigned long DN, float Sum)
+{
+  Invf_Doc_Entry *ide;
+
+  /* increase the size of the list if there is not enough */
+  /* room for this entry */
+  if (LT->num == LT->max)
+    {
+      int old, new, max_entries;
+      old = sizeof (List_Table) + sizeof (Invf_Doc_Entry) * LT->max;
+      max_entries = LT->max + (LT->max >> 1);
+      new = sizeof (List_Table) + sizeof (Invf_Doc_Entry) * max_entries;
+
+      LT = Xrealloc (LT, new);
+      if (!LT)
+	return NULL;
+
+      ChangeMemInUse (qd, new - old);
+      LT->max = max_entries;
+    }
+
+  /* add the entry to the end of the list */
+  ide = &LT->IDE[LT->num];
+  ide->DocNum = DN;
+  ide->Sum = Sum;
+  LT->num++;
+
+  return LT;
+}
+
+
+void 
+LT_free (query_data * qd, List_Table * LT)
+{
+  unsigned es;
+  es = sizeof (List_Table) + sizeof (Invf_Doc_Entry) * LT->max;
+  Xfree (LT);
+  ChangeMemInUse (qd, -es);
+}
+
+
+static int 
+IDE_comp (const void *a, const void *b)
+{
+  const Invf_Doc_Entry *A = a;
+  const Invf_Doc_Entry *B = b;
+  return A->DocNum - B->DocNum;
+}
+
+static void 
+LT_sort (List_Table * LT)
+{
+  qsort (LT->IDE, LT->num, sizeof (Invf_Doc_Entry), IDE_comp);
+}
+
+
+/* [RJM 07/97: Ranked Required Terms] */
+/* accumulators with a sum less than or equal to 0.0 are to be deleted */
+static void 
+LT_pack (List_Table * LT)
+{
+  int s, d = 0;
+  for (s = 0; s < LT->num; s++) {
+      if (LT->IDE[s].Sum <= 0.0) {
+	/* ignore this accumulator */
+
+      } else if (d > 0 && LT->IDE[s].DocNum == LT->IDE[d-1].DocNum) {
+	LT->IDE[d-1].Sum += LT->IDE[s].Sum;
+
+      } else {
+	if (d != s) LT->IDE[d] = LT->IDE[s];
+	d++;
+      }
+  }
+
+  if (LT->num) LT->num = d;
+}
+
+
+/**************************************************************************/
+
+
+
+invf_data *
+InitInvfFile (File * InvfFile, stemmed_dict * sd)
+{
+  invf_data *id;
+  if (!(id = Xmalloc (sizeof (invf_data))))
+    {
+      mg_errno = MG_NOMEM;
+      return NULL;
+    }
+
+  id->InvfFile = InvfFile;
+
+  Fread (&id->ifh, sizeof (id->ifh), 1, InvfFile);
+  /* [RPAP - Jan 97: Endian Ordering] */
+  {
+    int i;
+    
+    NTOHUL(id->ifh.no_of_words);
+    NTOHUL(id->ifh.no_of_ptrs);
+    NTOHUL(id->ifh.skip_mode);
+    for (i = 0; i < 16; i++)
+      NTOHUL(id->ifh.params[i]);
+    NTOHUL(id->ifh.InvfLevel);
+  }
+
+  id->N = sd->sdh.num_of_docs;
+  id->Nstatic = sd->sdh.static_num_of_docs;
+
+  return (id);
+}
+
+
+void 
+FreeInvfData (invf_data * id)
+{
+  Xfree (id);
+}
+
+/****************************************************************************/
+
+
+static void 
+CalcBlks (query_data * qd, WordEntry * we, int *blk,
+	  int *dn_blk, int *len_blk, int *k, int *p)
+{
+  *p = we->doc_count;
+
+  *blk = BIO_Bblock_Init (qd->id->Nstatic, *p);
+  switch (qd->id->ifh.skip_mode)
+    {
+    case 1:
+      {
+	unsigned long len;
+	if (*p <= qd->id->ifh.params[0])
+	  *k = 0;
+	else
+	  {
+	    *k = qd->id->ifh.params[0];
+	    len = BIO_Bblock_Bound (qd->id->N, *p);
+	    if (qd->id->ifh.InvfLevel >= 2)
+	      len += we->count;
+	    *dn_blk = BIO_Bblock_Init (qd->sd->sdh.num_of_docs, (*p + *k - 1) / *k);
+	    *len_blk = BIO_Bblock_Init (len, (*p + *k - 1) / *k);
+	  }
+	break;
+      }
+    case 2:
+      {
+	unsigned long len;
+	if (*p <= qd->id->ifh.params[1])
+	  *k = 0;
+	else
+	  {
+	    *k = (int) (2 * sqrt (((double) (*p)) / qd->id->ifh.params[0]));
+	    if (*k <= qd->id->ifh.params[1])
+	      *k = qd->id->ifh.params[1];
+	    len = BIO_Bblock_Bound (qd->id->N, *p);
+	    if (qd->id->ifh.InvfLevel >= 2)
+	      len += we->count;
+	    *dn_blk = BIO_Bblock_Init (qd->sd->sdh.num_of_docs,
+				       (*p + *k - 1) / *k);
+	    *len_blk = BIO_Bblock_Init (len, (*p + *k - 1) / *k);
+	  }
+	break;
+      }
+    default:
+      *k = 0;
+    }
+}
+
+
+/* =========================================================================
+ * Function: GetDocsOp
+ * Description: 
+ *      if op == op_term then 
+ *        look up term in invf and decode all its entries/documents
+ *      if op == op_and_terms then 
+ *        go thru L, testing if it exists in invf using skips  
+ *        if it does exist then overwrite the entry nearer the start of the list
+ *      if op == op_and_or_terms then
+ *        go thru L, testing if it exists in invf using skips  
+ *        BUT only mark ones which are found - keep list intact
+ *      if op == op_diff_terms then     >>>> NOT tested <<<<
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+DocList *
+GetDocsOp (query_data * qd, WordEntry * we, Op_types op, DocList * L)
+{
+  u_char *buffer;
+  DocEntry *src = NULL, *dest = NULL;
+  int i;			/* index counter into inverted list */
+  unsigned long pos;
+  int next_start, next_doc_num, next_i;
+  int block_counter;		/* within counter in block */
+  int dn_blk = 0, len_blk = 0;
+  int skip_size, have_skipping;
+  int blk;			/* bblock control parameter */
+  int ft;			/* The number of documents the word occurs in */
+  unsigned long CurrDoc = 0;	/* NOTE: Document numbers start at 1 */
+  float Weight;
+
+  /* ... AND empty */
+  if (op == op_and_terms && (!L || !L->num))
+    {
+      if (L)
+	Xfree (L);
+      return MakeDocList (0);
+    }
+
+  if (op == op_term)
+    L = MakeDocList (we->doc_count);
+
+  CalcBlks (qd, we, &blk, &dn_blk, &len_blk, &skip_size, &ft);
+  have_skipping = (skip_size != 0);
+  if (!(buffer = Xmalloc (we->invf_len)))
+    return MakeDocList (0);
+
+  ChangeMemInUse (qd, we->invf_len);
+
+  qd->num_of_terms++;
+
+  /* --- Read an inverted file entry for the given word --- */
+  Fseek (qd->File_invf, we->invf_ptr, 0);
+  Fread (buffer, sizeof (char), we->invf_len, qd->File_invf);
+
+  /* Read from the buffer */
+  DECODE_START (buffer, we->invf_len);
+
+  Weight = qd->sd->sdh.num_of_words / (100 * log2 ((float) we->count + 1));
+
+  if (op != op_term)
+    dest = src = L->DE;
+  i = pos = 0;
+  next_i = next_start = next_doc_num = 0;
+  block_counter = skip_size - 1;
+
+  /* Go thru the elements of an inverted list for a term */
+  /* May not go sequentially if skipping is done */
+  while (i < ft && ((op == op_term) || src - L->DE < L->num))
+    {
+      unsigned long diff, Temp;
+
+      if (have_skipping)
+	{
+
+	  /* --- Skip to next block --- */
+	  if (op != op_term && i + skip_size < ft &&
+	      src->DocNum > next_doc_num && next_doc_num >= 0)
+	    {
+	    ShortCut:
+	      i = next_i;
+	      DECODE_SEEK (next_start);
+	      pos = next_start;
+	      CurrDoc = next_doc_num;
+	      qd->hops_taken++;
+	      block_counter = skip_size - 1;
+	    }
+
+	  /* --- Decode the skip block header --- */
+	  if (block_counter == skip_size - 1)
+	    {
+	      if (i + skip_size < ft)
+		{
+		  /* --- Decode the skip block header --- */
+		  int doc_diff, pos_diff;
+		  BBLOCK_DECODE_L (doc_diff, dn_blk, pos);
+		  next_doc_num += doc_diff;
+		  next_i += skip_size;
+		  BBLOCK_DECODE_L (pos_diff, len_blk, pos);
+		  next_start = pos + pos_diff;
+		}
+	      else
+		next_doc_num = -1;
+	    }
+
+	  /* --- Do a skip if c-set doc is higher than block header doc --- */
+	  if (op != op_term && i + skip_size < ft &&
+	      src->DocNum > next_doc_num && next_doc_num >= 0)
+	    goto ShortCut;
+
+
+	}			/*if skipping */
+
+      /* --- increment CurrDoc --- */
+      if (have_skipping && block_counter == 0 && i != ft - 1)
+	CurrDoc = next_doc_num;
+      else
+	{
+	  BBLOCK_DECODE_L (diff, blk, pos);
+	  CurrDoc += diff;
+	}
+
+      /* --- If have f_dt's then eat them up --- */
+      if (qd->id->ifh.InvfLevel >= 2)
+	GAMMA_DECODE_L (Temp, pos);	/* discard word count */
+
+      /* --- Increment counters --- */
+      qd->num_of_ptrs++;
+      i++;
+      block_counter--;
+      if (block_counter < 0)	/* ??? Why would this happen ? */
+	block_counter = skip_size - 1;
+
+      switch (op)
+	{
+	case op_term:
+	  /* --- Add the doc from invf onto the c-set list --- */
+	  L->DE[L->num].DocNum = CurrDoc;
+	  L->DE[L->num].Weight = Weight;
+	  L->DE[L->num].SeekPos = 0;
+	  L->DE[L->num].Len = 0;
+	  L->DE[L->num].CompTextBuffer = NULL;
+	  L->DE[L->num].Next = NULL;
+	  L->DE[L->num].or_included = 0;
+	  L->num++;
+	  break;
+
+	case op_and_terms:
+	case op_and_or_terms:
+	  /* --- see if have a match and save it --- */
+
+	  while (src < (L->DE + L->num) && CurrDoc > src->DocNum)
+	    {
+	      src++;
+	    }
+	  /* Unless at end of list, assert(CurrDoc <= src->DocNum) */
+
+	  /* --- Accept the doc as a candidate --- */
+	  if (src < (L->DE + L->num) && CurrDoc == src->DocNum)
+	    {
+	      if (op == op_and_terms)
+		{
+		  *dest++ = *src;
+		}
+	      else
+		{
+		  src->or_included = 1;
+		}
+	      src++;		/* Move onto next one */
+	    }
+
+	  /* If the next one to try has already been tried then move on again */
+	  if (op == op_and_or_terms)
+	    {
+	      while (src < (L->DE + L->num) && src->or_included)
+		src++;
+	    }
+
+	  break;
+
+	case op_diff_terms:
+	  while (src < (L->DE + L->num) && src->DocNum < CurrDoc)
+	    *dest++ = *src++;
+	  if (src < (L->DE + L->num) && src->DocNum == CurrDoc)
+	    src++;
+	  break;
+
+	default:
+	  break;
+	}			/*switch */
+
+    }				/*while traversing invf */
+
+  if (op == op_and_terms || op == op_diff_terms)
+    {
+      if (op == op_diff_terms)
+	while (src - L->DE < L->num)
+	  *dest++ = *src++;
+
+      /* update size of c-set */
+      L->num = dest - L->DE;
+    }
+  DECODE_DONE
+
+    Xfree (buffer);
+  ChangeMemInUse (qd, -we->invf_len);
+
+  return (L);
+}
+
+
+
+
+
+
+
+float *
+CosineDecode (query_data * qd, TermList * Terms, RankedQueryInfo * rqi)
+{
+  float *AccumulatedWeights;
+  int j, num_docs, num_terms;
+  int seen_must_match = 0; /* [RJM 07/97: Ranked Required Terms] says whether to 'and' queries or not */
+
+  num_docs = qd->sd->sdh.num_of_docs;
+  /* Create the array for the documents */
+  if (!(AccumulatedWeights = Xmalloc (num_docs * sizeof (float))))
+      return (NULL);
+
+  bzero ((char *) AccumulatedWeights, sizeof (float) * num_docs);
+  ChangeMemInUse (qd, sizeof (float) * num_docs);
+
+  if (rqi->MaxTerms == -1)
+    num_terms = Terms->num;
+  else
+    num_terms = rqi->MaxTerms < Terms->num ? rqi->MaxTerms : Terms->num;
+
+
+  /* For each word in the list . . . */
+  for (j = 0; j < num_terms; j++)
+    {
+      u_char *buffer;
+      int i, kd;
+      unsigned long next_mjr_dn = 0;
+      unsigned long CurrDocNum = 0;	/* NOTE: Document numbers start at 1 */
+      unsigned long LastDocNum = 0; /* [RJM 07/97: Ranked Required Terms] */
+      unsigned long TempDocNum = 0; /* [RJM 07/97: Ranked Required Terms] */
+      double Wqt, WordLog;
+      int dn_blk = 0, len_blk = 0, k;
+      int blk;			/* bblock control parameter */
+      int p;			/* The number of documents the word occurs in */
+      int require_match = Terms->TE[j].require_match; /* [RJM 07/97: Ranked Required Terms] */
+      WordEntry *we = &Terms->TE[j].WE;
+      CalcBlks (qd, we, &blk, &dn_blk, &len_blk, &k, &p);
+      if (!(buffer = Xmalloc (we->invf_len)))
+	break;
+
+      ChangeMemInUse (qd, we->invf_len);
+
+      Fseek (qd->File_invf, we->invf_ptr, 0);
+      Fread (buffer, sizeof (char), we->invf_len, qd->File_invf);
+
+      /* Read from the buffer */
+      DECODE_START (buffer, we->invf_len);
+
+      WordLog = log ((double) num_docs / (double) we->max_doc_count);  /* [RPAP - Jan 97: Stem Index Change] */
+
+      Wqt = (rqi->QueryFreqs ? Terms->TE[j].Count : 1) * WordLog;
+
+      qd->num_of_terms++;
+
+      kd = 0;
+      for (i = 0; i < p; i++, kd++)
+	{
+	  int Count;		/* The number of times the word occurs in a document */
+	  double Wdt;
+	  unsigned long diff;
+	  if (kd == k)
+	    {
+	      kd = 0;
+	    }
+	  if (k && kd == 0)
+	    if (i + k < p)
+	      {
+		int temp;
+		BBLOCK_DECODE (next_mjr_dn, dn_blk);
+		next_mjr_dn += CurrDocNum;
+		BBLOCK_DECODE (temp, len_blk);
+	      }
+	  if (k && kd == k - 1 && i != p - 1)
+	    CurrDocNum = next_mjr_dn;
+	  else
+	    {
+	      BBLOCK_DECODE (diff, blk);
+	      CurrDocNum += diff;
+	    }
+	  if (qd->id->ifh.InvfLevel >= 2)
+	    GAMMA_DECODE (Count);
+	  else
+	    Count = (double) we->count / p;
+	  Wdt = Count * WordLog;
+
+	  /* [RJM 07/97: Ranked Required Terms] */
+	  /* zero the weights of intermediate documents if this term is required to match */
+	  if (require_match == 1) {
+	    for (TempDocNum = LastDocNum+1; TempDocNum < CurrDocNum; TempDocNum++)
+	      AccumulatedWeights[TempDocNum - 1] = 0.0;
+	  }
+
+	  /* [RJM 07/97: Ranked Required Terms] */
+	  /* if a term has been required to match before, the weight we are */
+	  /* adding to must be non-zero */
+	  if (!seen_must_match || (AccumulatedWeights[CurrDocNum - 1] > 0.0)) {
+	      AccumulatedWeights[CurrDocNum - 1] += Wqt * Wdt;
+	  }
+
+	  qd->num_of_ptrs++;
+	  LastDocNum = CurrDocNum; /* [RJM 07/97: Ranked Required Terms] */
+	}
+
+      DECODE_DONE
+
+	Xfree (buffer);
+      ChangeMemInUse (qd, -we->invf_len);
+
+      /* [RJM 07/97: Ranked Required Terms] */
+      /* zero the remaining weights if this term is required to match */
+      if (require_match == 1) {
+	for (TempDocNum = LastDocNum+1; TempDocNum <= num_docs; TempDocNum++)
+	  AccumulatedWeights[TempDocNum - 1] = 0.0;
+	seen_must_match = 1;
+      }
+    }
+  return AccumulatedWeights;
+}
+
+
+
+static int 
+doc_comp (void *a, void *b)
+{
+  return ((Invf_Doc_Entry *) a)->DocNum - ((Invf_Doc_Entry *) b)->DocNum;
+}
+
+
+static Invf_Doc_Entry *
+get_ide (query_data * qd, Invf_Doc_Entry_Pool * pool)
+{
+  struct pool *p = pool->pool;
+  if (!p || p->pos == p->length)
+    {
+      register unsigned size = sizeof (struct pool) +
+      (pool->pool_chunks - 1) * sizeof (Invf_Doc_Entry);
+      p = malloc (size);
+      if (!p)
+	return NULL;
+      ChangeMemInUse (qd, size);
+      p->length = pool->pool_chunks;
+      p->pos = 0;
+      p->next = pool->pool;
+      pool->pool = p;
+    }
+  return &(p->ide[p->pos++]);
+}
+
+
+void 
+free_ide_pool (query_data * qd, Invf_Doc_Entry_Pool * pool)
+{
+  while (pool->pool)
+    {
+      struct pool *p = pool->pool;
+      pool->pool = p->next;
+      ChangeMemInUse (qd, -(sizeof (struct pool) + (p->length - 1) *
+			    sizeof (Invf_Doc_Entry)));
+      free (p);
+    }
+  pool->pool = NULL;
+}
+
+
+FILE *
+open_skip (char *sd)
+{
+  char buf[256];
+  if (sd == NULL || *sd == '\0')
+    return NULL;
+  sprintf (buf, sd, getpid ());
+  return fopen (buf, "a");
+}
+
+
+
+Splay_Tree *
+CosineDecodeSplay (query_data * qd, TermList * Terms,
+		   RankedQueryInfo * rqi, Invf_Doc_Entry_Pool * pool)
+{
+  Splay_Tree *Doc_Set;
+  int j, num_docs, num_terms, MoreAccum;
+  int Anding = 0;
+  FILE *sk = open_skip (rqi->skip_dump);
+  u_char indent = 0;
+  if (sk)
+    {
+      if (!skip_header)
+	{
+	  skip_header = 1;
+	  fprintf (sk, "%s", skip_header_text[qd->id->ifh.skip_mode]);
+	  fprintf (sk, "\nSkipping method %ld\n", qd->id->ifh.skip_mode);
+	  switch (qd->id->ifh.skip_mode)
+	    {
+	    case 1:
+	      fprintf (sk, "Skipping is every %ld docnums\n",
+		       qd->id->ifh.params[0]);
+	      break;
+	    case 2:
+	      fprintf (sk, "Max nodes = %ld\nNo skips smaller or equal to %ld\n",
+		       qd->id->ifh.params[0], qd->id->ifh.params[1]);
+	      break;
+	    }
+	}
+      fprintf (sk, "\n\t\t\t-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\n\n");
+      for (j = 0; j < Terms->num; j++)
+	if (Terms->TE[j].Word[0] > indent)
+	  indent = Terms->TE[j].Word[0];
+    }
+
+  num_docs = qd->sd->sdh.num_of_docs;
+  /* Create the array for the documents */
+  Doc_Set = SP_createset (doc_comp);
+
+  pool->pool = NULL;
+  pool->pool_chunks = 1024;
+
+  if (rqi->MaxTerms == -1)
+    num_terms = Terms->num;
+  else
+    num_terms = rqi->MaxTerms < Terms->num ? rqi->MaxTerms : Terms->num;
+
+  MoreAccum = 1;
+
+  /* For each word in the list . . . */
+  for (j = 0; j < num_terms; j++)
+    {
+      int Abort = 0;
+      u_char *buffer;
+      int pos, i;
+      unsigned long CurrDocNum = 0;	/* NOTE: Document numbers start at 1 */
+      double Wqt, WordLog;
+      int dn_blk = 0, len_blk = 0, k, kd;
+      int blk;			/* bblock control parameter */
+      int p;			/* The number of documents the word occurs in */
+      WordEntry *we = &Terms->TE[j].WE;
+      Invf_Doc_Entry *next = NULL;
+      int next_i, next_doc_num, next_start;
+      int skips_taken = 0, ptrs_dec = 0;
+      double total_added = 0.0;
+      int hits = 0;
+      if (sk)
+	{
+	  fprintf (sk, "%3d : %8ld %6ld \"%.*s\"%*s", j, we->count,
+		 we->doc_count, Terms->TE[j].Word[0], Terms->TE[j].Word + 1,
+		   indent - Terms->TE[j].Word[0], "");
+	}
+
+      CalcBlks (qd, we, &blk, &dn_blk, &len_blk, &k, &p);
+      if (!(buffer = Xmalloc (we->invf_len)))
+	break;
+
+      ChangeMemInUse (qd, we->invf_len);
+
+      if (sk && qd->id->ifh.skip_mode == 2)
+	fprintf (sk, "%4d", k);
+
+      Fseek (qd->File_invf, we->invf_ptr, 0);
+      Fread (buffer, sizeof (char), we->invf_len, qd->File_invf);
+
+      /* Read from the buffer */
+      DECODE_START (buffer, we->invf_len);
+
+      WordLog = log ((double) num_docs / (double) we->max_doc_count);  /* [RPAP - Jan 97: Stem Index Change] */
+
+      Wqt = (rqi->QueryFreqs ? Terms->TE[j].Count : 1) * WordLog;
+
+      qd->num_of_terms++;
+
+      if (rqi->MaxAccums != -1 && Doc_Set->no_of_items >= rqi->MaxAccums)
+	{
+	  MoreAccum = 0;
+	  Anding = 1;
+	  next = SP_get_first (Doc_Set);
+	}
+      i = pos = 0;
+      next_i = next_start = next_doc_num = 0;
+      kd = k - 1;
+      while (i < p && (!Anding || next))
+	{
+	  int Count;		/* The number of times the occurs in a document */
+	  double Wdt;
+	  unsigned long diff;
+	  Invf_Doc_Entry ent, *mem;
+	  if (k)
+	    {
+	      if (Anding && i + k < p && next->DocNum > next_doc_num &&
+		  next_doc_num >= 0)
+		{
+		  i = next_i;
+		  DECODE_SEEK (next_start);
+		  pos = next_start;
+		  CurrDocNum = next_doc_num;
+		  qd->hops_taken++;
+		  skips_taken++;
+		  kd = k - 1;
+		}
+	      if (kd == k - 1)
+		if (i + k < p)
+		  {
+		    int doc_diff, pos_diff;
+		    BBLOCK_DECODE_L (doc_diff, dn_blk, pos);
+		    next_doc_num += doc_diff;
+		    next_i += k;
+		    BBLOCK_DECODE_L (pos_diff, len_blk, pos);
+		    next_start = pos + pos_diff;
+		  }
+		else
+		  next_doc_num = -1;
+	      if (Anding && i + k < p && next->DocNum > next_doc_num &&
+		  next_doc_num >= 0)
+		continue;
+	    }
+	  if (k && kd == 0 && i != p - 1)
+	    CurrDocNum = next_doc_num;
+	  else
+	    {
+	      BBLOCK_DECODE_L (diff, blk, pos);
+	      CurrDocNum += diff;
+	    }
+	  qd->num_of_ptrs++;
+	  ptrs_dec++;
+	  if (qd->id->ifh.InvfLevel >= 2)
+	    GAMMA_DECODE_L (Count, pos);
+	  else
+	    Count = (double) we->count / p;
+	  i++;
+	  kd--;
+	  if (kd < 0)
+	    kd = k - 1;
+	  Wdt = Count * WordLog;
+
+
+	  if (Anding)
+	    {
+	      while (next && next->DocNum < CurrDocNum - 1)
+		next = SP_get_next (Doc_Set);
+	      if (next && next->DocNum == CurrDocNum - 1)
+		{
+		  next->Sum += Wqt * Wdt;
+		  total_added += Wqt * Wdt;
+		  hits++;
+		}
+	      if (next && next->DocNum <= CurrDocNum - 1)
+		next = SP_get_next (Doc_Set);
+	      goto NextDocNum;
+	    }
+	  ent.DocNum = CurrDocNum - 1;
+	  if ((mem = SP_member (&ent, Doc_Set)) == NULL)
+	    {
+	      hits++;
+	      if (rqi->MaxAccums != -1 &&
+		  Doc_Set->no_of_items >= rqi->MaxAccums &&
+		  !MoreAccum)
+		goto NextDocNum;
+	      if ((mem = get_ide (qd, pool)) == NULL)
+		return NULL;
+	      ChangeMemInUse (qd, sizeof (*mem));
+	      mem->DocNum = CurrDocNum - 1;
+	      mem->Sum = 0;
+	      ChangeMemInUse (qd, -Doc_Set->mem_in_use);
+	      if (SP_insert (mem, Doc_Set) == NULL)
+		{
+		  Message ("Unable to allocate memory for a splay node");
+		  Abort = 1;
+		  goto FastAbort;
+		}
+	      ChangeMemInUse (qd, Doc_Set->mem_in_use);
+	      if (rqi->MaxAccums != -1 &&
+		  Doc_Set->no_of_items >= rqi->MaxAccums &&
+		  rqi->StopAtMaxAccum)
+		Abort = 1;
+	    }
+
+	  mem->Sum += Wqt * Wdt;
+	  total_added += Wqt * Wdt;
+
+	NextDocNum:
+	  ;
+	}
+
+    FastAbort:
+
+      DECODE_DONE
+
+	Xfree (buffer);
+      ChangeMemInUse (qd, -we->invf_len);
+      if (sk)
+	{
+	  if (qd->id->ifh.skip_mode != 0)
+	    fprintf (sk, " %6d", skips_taken);
+	  fprintf (sk, " %6d %6d %6d%12.2f\n",
+		   ptrs_dec, hits, Doc_Set->no_of_items, total_added);
+	}
+      if (Abort)
+	break;
+    }
+  if (sk)
+    fclose (sk);
+  return Doc_Set;
+}
+
+
+
+
+Hash_Table *
+CosineDecodeHash (query_data * qd, TermList * Terms,
+		  RankedQueryInfo * rqi)
+{
+  int j, num_docs, num_terms, MoreAccum;
+  Hash_Table *HT;
+  int Anding = 0;
+  int Sorted = 0;
+  FILE *sk = open_skip (rqi->skip_dump);
+  u_char indent = 0;
+  if (sk)
+    {
+      if (!skip_header)
+	{
+	  skip_header = 1;
+	  fprintf (sk, "%s", skip_header_text[qd->id->ifh.skip_mode]);
+	  fprintf (sk, "\nSkipping method %ld\n", qd->id->ifh.skip_mode);
+	  switch (qd->id->ifh.skip_mode)
+	    {
+	    case 1:
+	      fprintf (sk, "Skipping is every %ld docnums\n",
+		       qd->id->ifh.params[0]);
+	      break;
+	    case 2:
+	      fprintf (sk, "Max nodes = %ld\nNo skips smaller or equal to %ld\n",
+		       qd->id->ifh.params[0], qd->id->ifh.params[1]);
+	      break;
+	    }
+	}
+      fprintf (sk, "\n\t\t\t-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\n\n");
+      for (j = 0; j < Terms->num; j++)
+	if (Terms->TE[j].Word[0] > indent)
+	  indent = Terms->TE[j].Word[0];
+    }
+
+  num_docs = qd->sd->sdh.num_of_docs;
+  /* Create the array for the documents */
+  HT = HT_create (qd, rqi->MaxAccums, rqi->HashTblSize);
+  if (!HT)
+    return NULL;
+
+  if (rqi->MaxTerms == -1)
+    num_terms = Terms->num;
+  else
+    num_terms = rqi->MaxTerms < Terms->num ? rqi->MaxTerms : Terms->num;
+
+  MoreAccum = 1;
+
+  /* For each word in the list . . . */
+  for (j = 0; j < num_terms; j++)
+    {
+      int Abort = 0;
+      u_char *buffer;
+      int pos;
+      int i;
+      unsigned long CurrDocNum = 0;	/* NOTE: Document numbers start at 1 */
+      double Wqt, WordLog;
+      int dn_blk = 0, len_blk = 0, k, kd;
+      int blk;			/* bblock control parameter */
+      int p;			/* The number of documents the word occurs in */
+      WordEntry *we = &Terms->TE[j].WE;
+      Invf_Doc_EntryH *next = NULL;
+      int next_i, next_doc_num, next_start;
+      int skips_taken = 0, ptrs_dec = 0;
+      double total_added = 0.0;
+      int hits = 0;
+      if (sk)
+	{
+	  fprintf (sk, "%3d : %8ld %6ld \"%.*s\"%*s", j, we->count,
+		 we->doc_count, Terms->TE[j].Word[0], Terms->TE[j].Word + 1,
+		   indent - Terms->TE[j].Word[0], "");
+	}
+
+      CalcBlks (qd, we, &blk, &dn_blk, &len_blk, &k, &p);
+      if (!(buffer = Xmalloc (we->invf_len)))
+	break;
+
+      ChangeMemInUse (qd, we->invf_len);
+
+      if (sk && qd->id->ifh.skip_mode == 2)
+	fprintf (sk, "%4d", k);
+
+      Fseek (qd->File_invf, we->invf_ptr, 0);
+      Fread (buffer, sizeof (char), we->invf_len, qd->File_invf);
+
+      /* Read from the buffer */
+      DECODE_START (buffer, we->invf_len);
+
+      WordLog = log ((double) num_docs / (double) we->max_doc_count);  /* [RPAP - Jan 97: Stem Index Change] */
+
+      Wqt = (rqi->QueryFreqs ? Terms->TE[j].Count : 1) * WordLog;
+
+      qd->num_of_terms++;
+
+      if (rqi->MaxAccums != -1 && HT->num >= rqi->MaxAccums)
+	{
+	  MoreAccum = 0;
+	  Anding = 1;
+	  if (!Sorted)
+	    {
+	      HT_sort (qd, HT);
+	      Sorted = 1;
+	    }
+	  next = HT->Head;
+	}
+      i = pos = 0;
+      next_i = next_start = next_doc_num = 0;
+      kd = k - 1;
+      while (i < p && (!Anding || next))
+	{
+	  int Count;		/* The number of times the occurs in a document */
+	  double Wdt;
+	  unsigned long diff;
+	  Invf_Doc_Entry *mem;
+	  if (k)
+	    {
+	      if (Anding && i + k < p && next->IDE.DocNum > next_doc_num &&
+		  next_doc_num >= 0)
+		{
+#ifdef SHORT_CUT
+		ShortCut:
+#endif
+		  i = next_i;
+		  DECODE_SEEK (next_start);
+		  pos = next_start;
+		  CurrDocNum = next_doc_num;
+		  qd->hops_taken++;
+		  skips_taken++;
+		  kd = k - 1;
+		}
+	      if (kd == k - 1)
+		{
+		  if (i + k < p)
+		    {
+		      int doc_diff, pos_diff;
+		      BBLOCK_DECODE_L (doc_diff, dn_blk, pos);
+		      next_doc_num += doc_diff;
+		      next_i += k;
+		      BBLOCK_DECODE_L (pos_diff, len_blk, pos);
+		      next_start = pos + pos_diff;
+		    }
+		  else
+		    next_doc_num = -1;
+		}
+#ifdef SHORT_CUT
+	      if (Anding && i + k < p && next->IDE.DocNum > next_doc_num &&
+		  next_doc_num >= 0)
+		goto ShortCut;
+#else
+	      if (Anding && i + k < p && next->IDE.DocNum > next_doc_num &&
+		  next_doc_num >= 0)
+		continue;
+#endif
+	    }
+	  if (k && kd == 0 && i != p - 1)
+	    {
+	      CurrDocNum = next_doc_num;
+	    }
+	  else
+	    {
+	      BBLOCK_DECODE_L (diff, blk, pos);
+	      CurrDocNum += diff;
+	    }
+	  qd->num_of_ptrs++;
+	  ptrs_dec++;
+	  if (qd->id->ifh.InvfLevel >= 2)
+	    GAMMA_DECODE_L (Count, pos);
+	  else
+	    Count = (double) we->count / p;
+	  i++;
+	  kd--;
+	  if (kd < 0)
+	    kd = k - 1;
+
+
+	  if (Anding)
+	    {
+	      while (next && next->IDE.DocNum < CurrDocNum - 1)
+		next = next->next;
+	      if (next && next->IDE.DocNum == CurrDocNum - 1)
+		{
+		  Wdt = Count * WordLog;
+		  next->IDE.Sum += Wqt * Wdt;
+		  total_added += Wqt * Wdt;
+		  hits++;
+		}
+	      if (next && next->IDE.DocNum <= CurrDocNum - 1)
+		next = next->next;
+	      goto NextDocNum;
+	    }
+	  if ((mem = HT_find (HT, CurrDocNum - 1)) == NULL)
+	    {
+	      if (rqi->MaxAccums != -1 && HT->num >= rqi->MaxAccums &&
+		  !MoreAccum)
+		goto NextDocNum;
+	      Wdt = Count * WordLog;
+	      if (!HT_insert (HT, CurrDocNum - 1, Wqt * Wdt))
+		{
+		  if (!HT->Suplimentary_Entries)
+		    HT_Sup_create (qd, HT, we->doc_count - i + 1);
+		  HT_Sup_Add (HT, CurrDocNum - 1, Wqt * Wdt);
+		}
+	      if (rqi->MaxAccums != -1 && HT->num >= rqi->MaxAccums &&
+		  rqi->StopAtMaxAccum)
+		Abort = 1;
+	    }
+	  else
+	    {
+	      Wdt = Count * WordLog;
+	      mem->Sum += Wqt * Wdt;
+	      total_added += Wqt * Wdt;
+	      hits++;
+	    }
+
+	NextDocNum:
+	  ;
+	}
+
+      DECODE_DONE
+
+	Xfree (buffer);
+      ChangeMemInUse (qd, -we->invf_len);
+      if (sk)
+	{
+	  if (qd->id->ifh.skip_mode != 0)
+	    fprintf (sk, " %6d", skips_taken);
+	  fprintf (sk, " %6d %6d %6d%12.2f\n",
+	       ptrs_dec, hits, HT->num + HT->Suplimentary_Num, total_added);
+	}
+      if (Abort)
+	break;
+    }
+  if (sk)
+    fclose (sk);
+  if (!HT->Head)
+    HT_sort (qd, HT);
+  return HT;
+}
+
+
+
+List_Table *
+CosineDecodeList (query_data * qd, TermList * Terms,
+		  RankedQueryInfo * rqi)
+{
+  int j, num_docs, num_terms, MoreAccum;
+  List_Table *LT;
+  int Anding = 0;
+  int Sorted = 0;
+  int has_grown = 0; /* [RJM 07/97: Ranked Required Match] whether elements have been added lately */
+  FILE *sk = open_skip (rqi->skip_dump);
+  u_char indent = 0;
+  if (sk)
+    {
+      if (!skip_header)
+	{
+	  skip_header = 1;
+	  fprintf (sk, "%s", skip_header_text[qd->id->ifh.skip_mode]);
+	  fprintf (sk, "\nSkipping method %ld\n", qd->id->ifh.skip_mode);
+	  switch (qd->id->ifh.skip_mode)
+	    {
+	    case 1:
+	      fprintf (sk, "Skipping is every %ld docnums\n",
+		       qd->id->ifh.params[0]);
+	      break;
+	    case 2:
+	      fprintf (sk, "Max nodes = %ld\nNo skips smaller or equal to %ld\n",
+		       qd->id->ifh.params[0], qd->id->ifh.params[1]);
+	      break;
+	    }
+	}
+      fprintf (sk, "\n\t\t\t-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\n\n");
+      for (j = 0; j < Terms->num; j++)
+	if (Terms->TE[j].Word[0] > indent)
+	  indent = Terms->TE[j].Word[0];
+    }
+
+  num_docs = qd->sd->sdh.num_of_docs;
+
+  /* Create the list for the documents */
+  LT = LT_create (qd, rqi->MaxAccums);
+  if (!LT)
+    return NULL;
+
+  if (rqi->MaxTerms == -1)
+    num_terms = Terms->num;
+  else
+    num_terms = rqi->MaxTerms < Terms->num ? rqi->MaxTerms : Terms->num;
+
+  MoreAccum = 1;
+
+  /* For each word in the list . . . */
+  for (j = 0; j < num_terms; j++)
+    {
+      int Abort = 0;
+      u_char *buffer;
+      int pos;
+      int i, n = 0;
+      unsigned long CurrDocNum = 0;	/* NOTE: Document numbers start at 1 */
+      unsigned long LastDocNum = 0;	/* [RJM 07/97: Ranked Required Match] */
+      int this_item = 0; /* [RJM 07/97: Ranked Required Match] points to the next possible LT to delete */
+      double Wqt, WordLog;
+      int dn_blk = 0, len_blk = 0, k, kd;
+      int blk;			/* bblock control parameter */
+      int p;			/* The number of documents the word occurs in */
+      int require_match = Terms->TE[j].require_match; /* [RJM 07/97: Ranked Required Match] */
+      WordEntry *we = &Terms->TE[j].WE;
+      Invf_Doc_Entry *next = NULL;
+      int next_i, next_doc_num, next_start;
+      int skips_taken = 0, ptrs_dec = 0;
+      double total_added = 0.0;
+      int hits = 0;
+
+      if (sk)
+	{
+	  fprintf (sk, "%3d : %8ld %6ld \"%.*s\"%*s", j, we->count,
+		 we->doc_count, Terms->TE[j].Word[0], Terms->TE[j].Word + 1,
+		   indent - Terms->TE[j].Word[0], "");
+	}
+
+      CalcBlks (qd, we, &blk, &dn_blk, &len_blk, &k, &p);
+      if (!(buffer = Xmalloc (we->invf_len)))
+	break;
+
+      ChangeMemInUse (qd, we->invf_len);
+
+      if (sk && qd->id->ifh.skip_mode == 2)
+	fprintf (sk, "%4d", k);
+
+      Fseek (qd->File_invf, we->invf_ptr, 0);
+      Fread (buffer, sizeof (char), we->invf_len, qd->File_invf);
+
+      /* Read from the buffer */
+      DECODE_START (buffer, we->invf_len);
+
+      WordLog = log ((double) num_docs / (double) we->max_doc_count);  /* [RPAP - Jan 97: Stem Index Change] */
+
+      Wqt = (rqi->QueryFreqs ? Terms->TE[j].Count : 1) * WordLog;
+
+      qd->num_of_terms++;
+
+      if (rqi->MaxAccums != -1 && LT->num >= rqi->MaxAccums)
+	MoreAccum = 0;
+
+      if ((rqi->MaxAccums != -1 && LT->num >= rqi->MaxAccums) || Anding)
+	{
+	  Anding = 1;
+	  if (!Sorted || has_grown)
+	    {
+	      LT_sort (LT);
+	      LT_pack (LT);
+	      Sorted = 1;
+	      has_grown = 0;
+	    }
+	  next = LT->IDE;
+	  n = 0;
+	}
+
+      /* [RJM 07/97: Ranked Required Terms] */
+      /* make sure that the document list is set up correctly for */
+      /* requiring a term to match */
+      if (require_match == 1) {
+	if (!Sorted || has_grown) {
+	  LT_sort (LT);
+	  LT_pack (LT);
+	  Sorted = 1;
+	  has_grown = 0;
+	}
+	/* this_item is already set to zero */
+      }
+
+      i = pos = 0;
+      next_i = next_start = next_doc_num = 0;
+      kd = k - 1;
+      while (i < p && (!Anding || n < LT->num))
+	{
+	  int Count;		/* The number of times the occurs in a document */
+	  double Wdt;
+	  unsigned long diff;
+	  Invf_Doc_Entry *mem;
+
+	  /* deal with skipping */
+	  if (k)
+	    {
+	      /* see if we should take a skip */
+	      if (Anding && i + k < p && next->DocNum > next_doc_num &&
+		  next_doc_num >= 0)
+		{
+		  i = next_i;
+		  DECODE_SEEK (next_start);
+		  pos = next_start;
+		  CurrDocNum = next_doc_num;
+		  qd->hops_taken++;
+		  skips_taken++;
+		  kd = k - 1;
+		}
+	      if (kd == k - 1)
+		if (i + k < p)
+		  {
+		    int doc_diff, pos_diff;
+		    BBLOCK_DECODE_L (doc_diff, dn_blk, pos);
+		    next_doc_num += doc_diff;
+		    next_i += k;
+		    BBLOCK_DECODE_L (pos_diff, len_blk, pos);
+		    next_start = pos + pos_diff;
+		  }
+		else
+		  next_doc_num = -1;
+
+	      /* should we take another skip? */
+	      if (Anding && i + k < p && next->DocNum > next_doc_num &&
+		  next_doc_num >= 0)
+		continue;
+	    }
+	  if (k && kd == 0 && i != p - 1)
+	    CurrDocNum = next_doc_num;
+	  else
+	    {
+	      BBLOCK_DECODE_L (diff, blk, pos);
+	      CurrDocNum += diff;
+	    }
+	  qd->num_of_ptrs++;
+	  ptrs_dec++;
+	  if (qd->id->ifh.InvfLevel >= 2)
+	    GAMMA_DECODE_L (Count, pos);
+	  else
+	    Count = (double) we->count / p;
+	  i++;
+	  kd--;
+	  if (kd < 0)
+	    kd = k - 1;
+	  Wdt = Count * WordLog;
+
+	  /* [RJM 07/97: Ranked Required Terms] */
+	  /* if this is a required match we must remove all accumulators */
+	  /* of intermediate documents */
+	  if (require_match == 1) {
+	    while (this_item < LT->num && LT->IDE[this_item].DocNum < CurrDocNum - 1) {
+	      if (LT->IDE[this_item].DocNum > LastDocNum - 1)
+		LT->IDE[this_item].Sum = -1000.0; /* mark for deletion */
+	      this_item++;
+	    }
+	  }
+
+	  if (Anding)
+	    {
+	      while (n < LT->num && next->DocNum < CurrDocNum - 1)
+		{
+		  next++;
+		  n++;
+		}
+	      if (n < LT->num && next->DocNum == CurrDocNum - 1)
+		{
+		  next->Sum += Wqt * Wdt;
+		  total_added += Wqt * Wdt;
+		  hits++;
+		}
+	      if (n < LT->num && next->DocNum <= CurrDocNum - 1)
+		{
+		  next++;
+		  n++;
+		}
+	      goto NextDocNum;
+	    }
+	  if (!Anding)
+	    {
+	      if (rqi->MaxAccums != -1 && LT->num >= rqi->MaxAccums &&
+		  !MoreAccum)
+		goto NextDocNum;
+	      LT = LT_add (qd, LT, CurrDocNum - 1, Wqt * Wdt);
+	      if (!LT)
+		{
+		  Abort = 1;
+		  goto FastAbort;
+		}
+
+	      /* the list of accumulators just got bigger */
+	      has_grown = 1;
+
+	      if (rqi->MaxAccums != -1 && LT->num >= rqi->MaxAccums &&
+		  rqi->StopAtMaxAccum)
+		Abort = 1;
+
+
+	    }
+	  else if ((mem = LT_find (LT, CurrDocNum - 1)))
+	    {
+	      mem->Sum += Wqt * Wdt;
+	      total_added += Wqt * Wdt;
+	      hits++;
+	    }
+
+	NextDocNum:
+
+	  LastDocNum = CurrDocNum;
+	}
+
+    FastAbort:
+
+      DECODE_DONE
+
+	Xfree (buffer);
+      ChangeMemInUse (qd, -we->invf_len);
+      if (sk)
+	{
+	  if (qd->id->ifh.skip_mode != 0)
+	    fprintf (sk, " %6d", skips_taken);
+	  fprintf (sk, " %6d %6d %6d%12.2f\n",
+		   ptrs_dec, hits, LT->num, total_added);
+	}
+      if (Abort)
+	break;
+
+      /* [RJM 07/97: Ranked Required Terms] */
+      /* remove the remaining accumulators if this term is required to match */
+      if (require_match == 1) {
+	while (this_item < LT->num) {
+	  if (LT->IDE[this_item].DocNum > LastDocNum - 1)
+	    LT->IDE[this_item].Sum = -1000.0; /* mark for deletion */
+	  this_item++;
+	}
+	Anding = 1; /* no more documents should be added */
+
+	/* Get rid of any documents which have been deleted */
+	LT_pack (LT);
+      }
+    }
+
+  if (sk)
+    fclose (sk);
+  return LT;
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/invf_get.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/invf_get.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/invf_get.h	(revision 699)
@@ -0,0 +1,103 @@
+/**************************************************************************
+ *
+ * invf_get.h -- Inverted file reading routines
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef H_INVF_GET
+#define H_INVF_GET
+
+typedef struct Invf_Doc_Entry
+  {
+    int DocNum;
+    float Sum;
+  }
+Invf_Doc_Entry;
+
+typedef struct Invf_Doc_Entry_Pool
+  {
+    void *pool;
+    unsigned pool_chunks;
+  }
+Invf_Doc_Entry_Pool;
+
+typedef struct Invf_Doc_EntryH
+  {
+    Invf_Doc_Entry IDE;
+    struct Invf_Doc_EntryH *next;
+  }
+Invf_Doc_EntryH;
+
+typedef struct Hash_Table
+  {
+    Invf_Doc_EntryH *Suplimentary_Entries;
+    int Suplimentary_Size;
+    int Suplimentary_Num;
+    unsigned max;
+    unsigned num;
+    unsigned size;
+    Invf_Doc_EntryH **HashTable;
+    Invf_Doc_EntryH *Head;
+    Invf_Doc_EntryH IDEH[1];
+  }
+Hash_Table;
+
+typedef struct List_Table
+  {
+    unsigned max;
+    unsigned num;
+    Invf_Doc_Entry IDE[1];
+  }
+List_Table;
+
+typedef enum
+  {
+    op_term, op_and_terms, op_and_or_terms, op_diff_terms
+  }
+Op_types;
+
+/*Invf_Doc_Entry *HT_find(Hash_Table *HT, unsigned long DN); */
+void HT_free (query_data * qd, Hash_Table * HT);
+void LT_free (query_data * qd, List_Table * LT);
+
+invf_data *InitInvfFile (File * InvfFile, stemmed_dict * sd);
+void FreeInvfData (invf_data * id);
+
+DocList *GetDocs (query_data * qd, WordEntry * we, BooleanQueryInfo * bqi);
+
+float *CosineDecode (query_data * qd, TermList * Terms, RankedQueryInfo * rqi);
+
+void free_ide_pool (query_data * qd, Invf_Doc_Entry_Pool * pool);
+
+Splay_Tree *CosineDecodeSplay (query_data * qd, TermList * Terms,
+			 RankedQueryInfo * rqi, Invf_Doc_Entry_Pool * pool);
+
+Hash_Table *CosineDecodeHash (query_data * qd, TermList * Terms,
+			      RankedQueryInfo * rqi);
+List_Table *CosineDecodeList (query_data * qd, TermList * Terms,
+			      RankedQueryInfo * rqi);
+
+
+DocList *GetDocsOp (query_data * qd, WordEntry * we, Op_types op, DocList * L);
+
+
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/ivf.pass1.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/ivf.pass1.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/ivf.pass1.c	(revision 699)
@@ -0,0 +1,796 @@
+/**************************************************************************
+ *
+ * ivf.pass1.c -- Memory efficient pass 1 inversion
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "bitio_m.h"
+#include "bitio_m_stdio.h"
+#include "bitio_gen.h"
+#include "local_strings.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "invf.h"
+#include "mg.h"
+#include "build.h"
+#include "locallib.h"
+#include "words.h"
+#include "stemmer.h"
+#include "hash.h"
+
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:17:54  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.3  1998/12/17 09:12:50  rjmcnab
+
+   Altered mg to process utf-8 encoded Unicode. The main changes
+   are in the parsing of the input, the casefolding, and the stemming.
+
+   Revision 1.2  1998/11/25 07:55:43  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:34:44  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.6  1995/01/16  03:57:05  tes
+   * Fixed bug for index_string_bytes which was calculated
+   * by adding the suffix lengths except for every lookback (block)
+   * number of words. This was incorrect as invf.dict is fully
+   * front coded - perhaps it was partially coded in the past.
+   * This would only affect a statistic - hence the reason why the
+   * bug could exist ;-)
+   *
+   * Revision 1.5  1994/11/29  00:31:59  tes
+   * Committing the new merged files and changes.
+   *
+   * Revision 1.3  1994/10/20  03:56:48  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:34  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#define LOGLOOKBACK	2
+#define POOL_SIZE 1024*1024
+#define HASH_POOL_SIZE 8192
+#define INITIAL_HASH_SIZE 7927
+
+#define INIT_CHECK_FRAC 0.10
+#define CHECK_FRAC 0.75
+#define CHECK_CLOSE 0.999
+#define PARA_DIV 1.5
+#define NORM_DIV 1.0
+
+
+typedef struct hash_rec
+  {
+    unsigned long fcnt;		/* fragment frequency */
+    unsigned long lfcnt;	/* local fragment frequency */
+    unsigned long fnum;		/* last fragment to use stem */
+    unsigned long wcnt;		/* stem frequency */
+    unsigned long lwcnt;	/* local stem frequency */
+    u_char *word;
+  }
+hash_rec;
+
+
+
+
+
+
+
+
+static unsigned long words_read = 0, words_diff = 0, bytes_diff = 0;
+static unsigned long outputbytes = 0;
+static double inputbytes = 0; /* [RJM 07/97: 4G limit] */
+static unsigned long MaxMemInUse = 0;
+static unsigned long MemInUse = 0;
+static unsigned long ChunksWritten = 0;
+
+
+static FILE *ic;		/* The invf block file */
+static stdio_bitio_state sbs;
+
+static hash_rec **HashTable;
+static unsigned long HashSize;
+static unsigned long HashUsed;
+static u_char *Pool;
+static int PoolLeft;
+
+static hash_rec *hr_pool;
+static int hr_PoolLeft;
+
+static hash_rec **first_occr;
+static int max_first_occr;
+
+static unsigned long L1_bits = 0, L1_ohead = 0;
+static unsigned long L2_bits = 0, L2_ohead = 0;
+static unsigned long L3_bits = 0, L3_ohead = 0;
+static unsigned long callnum = 0, lcallnum = 0, wordnum = 0, lwordnum = 0;
+static unsigned long ptrcnt = 0;
+static unsigned long checknum;
+static long max_mem = 0;
+
+
+static void 
+ChangeMem (int Change)
+{
+  MemInUse += Change;
+  if (MemInUse > MaxMemInUse)
+    MaxMemInUse = MemInUse;
+}
+
+
+
+int 
+init_ivf_1 (char *file_name)
+{
+  if (!(ic = create_file (file_name, INVF_CHUNK_SUFFIX, "wb", MAGIC_CHUNK,
+			  MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (COMPERROR);
+  fwrite ("    ", sizeof (u_long), 1, ic);	/* Space for the maxmem */
+  ENCODE_START (ic)
+    ENCODE_PAUSE (sbs)
+
+    if (!(Pool = Xmalloc (POOL_SIZE)))
+    {
+      Message ("Unable to allocate memory for pool");
+      return (COMPERROR);
+    }
+  PoolLeft = POOL_SIZE;
+  ChangeMem (POOL_SIZE);
+
+  if (!(hr_pool = Xmalloc (HASH_POOL_SIZE * sizeof (hash_rec))))
+    {
+      Message ("Unable to allocate memory for pool");
+      return (COMPERROR);
+    }
+  hr_PoolLeft = HASH_POOL_SIZE;
+  ChangeMem (HASH_POOL_SIZE * sizeof (hash_rec));
+
+  HashSize = INITIAL_HASH_SIZE;
+  HashUsed = 0;
+  if (!(HashTable = Xmalloc (sizeof (hash_rec *) * HashSize)))
+    {
+      Message ("Unable to allocate memory for table");
+      return (COMPERROR);
+    }
+  ChangeMem (sizeof (hash_rec *) * HashSize);
+  bzero ((char *) HashTable, sizeof (hash_rec *) * HashSize);
+  max_first_occr = 8192;
+  if (!(first_occr = Xmalloc (sizeof (hash_rec *) * max_first_occr)))
+    {
+      Message ("Unable to allocate memory for first_occur");
+      return (COMPERROR);
+    }
+  ChangeMem (sizeof (hash_rec *) * max_first_occr);
+
+  checknum = (invf_buffer_size * INIT_CHECK_FRAC) /
+    (InvfLevel == 3 ? PARA_DIV : NORM_DIV);
+
+  return (COMPALLOK);
+}
+
+
+
+static unsigned long 
+mem_reqd (void)
+{
+  register int i;
+  register unsigned long total = 0;
+/*  register unsigned long N = InvfLevel == 3 ? lwordnum : lcallnum; */
+  register unsigned long N = lcallnum;
+  for (i = 0; i < HashUsed; i++)
+    {
+      register hash_rec *ent = first_occr[i];
+/*      register unsigned long p = InvfLevel == 3 ? ent->lwcnt : ent->lfcnt; */
+      register unsigned long p = ent->lfcnt;
+      if (p)
+	total += BIO_Bblock_Bound (N, p);
+      if (InvfLevel >= 2)
+	total += ent->lwcnt;
+    }
+
+  total = (total + 7) >> 3;
+  return total;
+}
+
+static unsigned long 
+max_mem_reqd (void)
+{
+  register int i;
+  register unsigned long total = 0;
+/*  register unsigned long N = InvfLevel == 3 ? wordnum : callnum; */
+  register unsigned long N = callnum;
+  for (i = 0; i < HashUsed; i++)
+    {
+      register hash_rec *ent = first_occr[i];
+/*      register unsigned long p = InvfLevel == 3 ? ent->wcnt : ent->fcnt; */
+      register unsigned long p = ent->fcnt;
+      if (p)
+	total += BIO_Bblock_Bound (N, p);
+      if (InvfLevel >= 2)
+	total += ent->wcnt;
+    }
+
+  total = (total + 7) >> 3;
+  return total;
+}
+
+
+static void 
+dump_dict (unsigned long mem)
+{
+  int i;
+
+  ChunksWritten++;
+
+  ENCODE_CONTINUE (sbs)
+
+    GAMMA_ENCODE (lcallnum + 1);
+  if (mem > max_mem)
+    max_mem = mem;
+  GAMMA_ENCODE (mem + 1);
+  lwordnum = lcallnum = 0;
+  GAMMA_ENCODE (HashUsed + 1);
+  for (i = 0; i < HashUsed; i++)
+    {
+      hash_rec *ent = first_occr[i];
+      GAMMA_ENCODE (ent->lwcnt + 1);
+      if (ent->lwcnt >= 2)
+	GAMMA_ENCODE (ent->lfcnt);
+      ent->lwcnt = ent->lfcnt = 0;
+    }
+  ptrcnt = 0;
+  ENCODE_PAUSE (sbs)
+}
+
+
+
+static int 
+process_doc (u_char * s_in, int l_in)
+{
+  u_char *end = s_in + l_in - 1;
+
+  callnum++;
+  lcallnum++;
+  inputbytes += l_in;
+
+  if (!inaword (s_in, end))
+    if (SkipSGML)
+      PARSE_NON_STEM_WORD_OR_SGML_TAG (s_in, end);
+    else
+      PARSE_NON_STEM_WORD (s_in, end);
+  /*
+   ** Alternately parse off words and non-words from the input
+   ** stream beginning with a non-word. Each token is then
+   ** inserted into the set if it does not exist or has it's
+   ** frequency count incremented if it does.
+   */
+  while (s_in <= end)
+    {
+      u_char Word[MAXSTEMLEN + 1];
+
+      PARSE_STEM_WORD (Word, s_in, end);
+      
+      stemmer (stem_method, stemmer_num, Word);
+      if (SkipSGML)
+	PARSE_NON_STEM_WORD_OR_SGML_TAG (s_in, end);
+      else
+	PARSE_NON_STEM_WORD (s_in, end);
+
+      words_read++;
+      wordnum++;
+      lwordnum++;
+      /* Search the hash table for Word */
+      {
+	register unsigned long hashval, step;
+	register int hsize = HashSize;
+	HASH (hashval, step, Word, hsize);
+	for (;;)
+	  {
+	    register u_char *s1;
+	    register u_char *s2;
+	    register int len;
+	    register hash_rec *ent;
+	    ent = HashTable[hashval];
+	    if (!ent)
+	      {
+		int len = *Word + 1;
+		if (!hr_PoolLeft)
+		  {
+		    if (!(hr_pool = Xmalloc (HASH_POOL_SIZE *
+					     sizeof (hash_rec))))
+		      {
+			Message ("Unable to allocate memory for pool");
+			return (COMPERROR);
+		      }
+		    hr_PoolLeft = HASH_POOL_SIZE;
+		    ChangeMem (HASH_POOL_SIZE * sizeof (hash_rec));
+		  }
+		ent = hr_pool++;
+		hr_PoolLeft--;
+		if (len > PoolLeft)
+		  {
+		    if (!(Pool = Xmalloc (POOL_SIZE)))
+		      {
+			Message ("Unable to allocate memory for pool");
+			return (COMPERROR);
+		      }
+		    PoolLeft = POOL_SIZE;
+		    ChangeMem (POOL_SIZE);
+		  }
+		ent->wcnt = 1;
+		ent->fcnt = 1;
+		ent->lwcnt = 1;
+		ent->lfcnt = 1;
+		ent->fnum = callnum;
+		ent->word = Pool;
+		memcpy (Pool, Word, len);
+		Pool += len;
+		PoolLeft -= len;
+		if (HashUsed == max_first_occr - 1)
+		  {
+		    ChangeMem (-sizeof (hash_rec *) * max_first_occr);
+		    max_first_occr *= 2;
+		    if (!(first_occr = Xrealloc (first_occr, sizeof (hash_rec *) *
+						 max_first_occr)))
+		      {
+			Message ("Unable to allocate memory for first_occr");
+			return (COMPERROR);
+		      }
+		    ChangeMem (sizeof (hash_rec *) * max_first_occr);
+		  }
+		first_occr[HashUsed] = ent;
+		HashUsed++;
+		HashTable[hashval] = ent;
+		bytes_diff += Word[0];
+		break;
+	      }
+
+	    /* Compare the words */
+	    s1 = Word;
+	    s2 = ent->word;
+	    len = *s1 + 1;
+	    for (; len; len--)
+	      if (*s1++ != *s2++)
+		break;
+
+	    if (len)
+	      {
+		hashval = (hashval + step);
+		if (hashval >= hsize)
+		  hashval -= hsize;
+	      }
+	    else
+	      {
+		ent->wcnt++;
+		ent->lwcnt++;
+		if (callnum > ent->fnum)
+		  {
+		    ptrcnt++;
+		    ent->fcnt++;
+		    ent->lfcnt++;
+		    ent->fnum = callnum;
+		  }
+		break;
+	      }
+	  }
+      }
+
+      if (HashUsed >= HashSize >> 1)
+	{
+	  hash_rec **ht;
+	  unsigned long size;
+	  unsigned long i;
+	  size = prime (HashSize * 2);
+	  if (!(ht = Xmalloc (sizeof (hash_rec *) * size)))
+	    {
+	      Message ("Unable to allocate memory for table");
+	      return (COMPERROR);
+	    }
+	  bzero ((char *) ht, sizeof (hash_rec *) * size);
+	  ChangeMem (sizeof (hash_rec *) * size);
+
+	  for (i = 0; i < HashSize; i++)
+	    if (HashTable[i])
+	      {
+		register u_char *wptr;
+		hash_rec *ent;
+		register unsigned long hashval, step;
+
+		wptr = HashTable[i]->word;
+		HASH (hashval, step, wptr, size);
+		ent = ht[hashval];
+		while (ent)
+		  {
+		    hashval += step;
+		    if (hashval >= size)
+		      hashval -= size;
+		    ent = ht[hashval];
+		  }
+		ht[hashval] = HashTable[i];
+	      }
+	  Xfree (HashTable);
+	  ChangeMem (-sizeof (hash_rec *) * HashSize);
+	  HashTable = ht;
+	  HashSize = size;
+	}
+
+    }
+
+  if (ptrcnt >= checknum)
+    {
+      unsigned long mem;
+      /*fprintf(stderr, "Checking at %u . . . ", ptrcnt); */
+      mem = mem_reqd ();
+      if (mem >= invf_buffer_size * CHECK_CLOSE)
+	{
+	  /*fprintf(stderr, "Got a match\n"); */
+	  dump_dict (mem);
+	  checknum = (invf_buffer_size * INIT_CHECK_FRAC) /
+	    (InvfLevel == 3 ? PARA_DIV : NORM_DIV);
+	}
+      else
+	{
+	  checknum = checknum * ((CHECK_FRAC * (invf_buffer_size - mem)) / mem) +
+	    checknum;
+	  if (checknum <= ptrcnt)
+	    checknum = ptrcnt + 1;
+	  /*fprintf(stderr, "Next check at %u\n", checknum); */
+	}
+    }
+  return (COMPALLOK);
+
+}				/* encode */
+
+
+int 
+process_ivf_1 (u_char * s_in, int l_in)
+{
+  if (InvfLevel <= 2)
+    return process_doc (s_in, l_in);
+  else
+    {
+      int pos = 0;
+      u_char *start = s_in;
+      while (pos < l_in)
+	{
+	  if (s_in[pos] == TERMPARAGRAPH)
+	    {
+	      int len = pos + s_in + 1 - start;
+	      if (process_doc (start, len) != COMPALLOK)
+		return (COMPERROR);
+	      start = s_in + pos + 1;
+	    }
+	  pos++;
+	}
+      if (start < s_in + pos)
+	return process_doc (start, pos + s_in - start);
+    }
+  return COMPALLOK;
+}
+
+static int 
+PackHashTable (void)
+{
+  int s, d;
+  for (s = d = 0; s < HashSize; s++)
+    if (HashTable[s])
+      HashTable[d++] = HashTable[s];
+  ChangeMem (-sizeof (hash_rec *) * HashSize);
+  ChangeMem (sizeof (hash_rec *) * HashUsed);
+  if (!(HashTable = Xrealloc (HashTable, sizeof (hash_rec *) * HashUsed)))
+    {
+      Message ("Out of memory");
+      return COMPERROR;
+    }
+  HashSize = HashUsed;
+  return COMPALLOK;
+}
+
+
+
+
+
+static int 
+ent_comp (const void *A, const void *B)
+{
+  u_char *s1 = (*((hash_rec **) A))->word;
+  u_char *s2 = (*((hash_rec **) B))->word;
+
+  return (casecompare (s1, s2)); /* [RPAP - Jan 97: Stem Index Change] */
+}				/* stem_comp */
+
+
+
+
+
+/*
+ * void count_text()
+ *
+ * The maths used in this function is described in the paper "Coding for 
+ * Compression in Full-Text Retrieval Systems"
+ *
+ */
+static void 
+count_text ()
+{
+  int i;
+  for (i = 0; i < HashUsed; i++)
+    {
+      hash_rec *wrd = HashTable[i];
+      /* estimate size of a level 1 inverted file */
+      L1_bits += (int) (0.99 + wrd->fcnt * (1.6 + log2 (1.0 * callnum / wrd->fcnt)));
+      L1_ohead += BIO_Gamma_Length (wrd->fcnt);
+
+      L2_bits += BIO_Unary_Length (wrd->wcnt);
+      L2_ohead += BIO_Gamma_Length (wrd->wcnt - wrd->fcnt + 1);
+
+      L3_bits += (int) (0.99 + wrd->wcnt *
+		  (1.6 + log2 (1.0 * words_read / (wrd->wcnt + callnum))));
+      L3_ohead += 0;
+    }
+  L3_bits = (L3_bits + L2_bits + L1_bits + 7) / 8;
+  L3_ohead = (L3_ohead + L2_ohead + L1_ohead + 7) / 8;
+  L2_bits = (L2_bits + L1_bits + 7) / 8;
+  L2_ohead = (L2_ohead + L1_ohead + 7) / 8;
+  L1_bits = (L1_bits + 7) / 8;
+  L1_ohead = (L1_ohead + 7) / 8;
+}				/* count_text */
+
+
+
+/*
+ *    write_stem_file():
+ *              writes out the stemmed dictionary file
+ *              in the following format 
+ *                      lookback value (int)
+ *                      totalbytes value (int)
+ *                      indexstringbytes (int)
+ *                      for each word 
+ *                        wordlen (4 bits)
+ *                        prefix match (4 bits)
+ *                        word (wordlen bytes)
+ *                        word frequency (int)
+ *                        word count (int)
+ *
+ *      Accesses outside variables:     
+ *
+ *      Return value...:                
+ */
+
+static void 
+write_stem_file (char *file_name)
+{
+  long j;
+  struct invf_dict_header idh;
+  long lookback = (1 << LOGLOOKBACK);	/* ???? */
+  long totalbytes = 0;		/* The sum of the length of all words, including 
+				   the length byte */
+  long indexstringbytes = 0;	/* The amount of space required to store the 
+				   words in the diction, this takes into account 
+				   the prefixes */
+  u_char *lastword = NULL;	/* A pointer to the last word processed */
+  FILE *sp;
+
+
+  /* Calculate the size of certain things */
+  for (j = 0; j < HashSize; j++)
+    {
+      u_char *word = HashTable[j]->word;
+      indexstringbytes += word[0] + 2;
+      totalbytes += word[0] + 1;
+      if (lastword)
+	indexstringbytes -= prefixlen (lastword, word);
+      lastword = word;
+    }
+
+  lastword = NULL;
+
+  if (!(sp = create_file (file_name, INVF_DICT_SUFFIX, "wb", MAGIC_STEM_BUILD,
+			  MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return;
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL2(lookback, idh.lookback);
+  HTONUL2(HashSize, idh.dict_size);
+  HTONUL2(totalbytes, idh.total_bytes);
+  HTONUL2(indexstringbytes, idh.index_string_bytes);
+  HTOND2(inputbytes, idh.input_bytes); /* [RJM 07/97: 4G limit] */
+  HTONUL2(callnum, idh.num_of_docs);
+  HTONUL2(callnum, idh.static_num_of_docs);
+  HTONUL2(words_read, idh.num_of_words);
+  HTONUL2(stemmer_num, idh.stemmer_num);
+  HTONUL2(stem_method, idh.stem_method);
+
+  fwrite ((char *) &idh, sizeof (idh), 1, sp);
+  outputbytes += sizeof (idh);
+
+  for (j = 0; j < HashSize; j++)
+    {
+      int i;
+      unsigned long wcnt, fcnt;  /* [RPAP - Jan 97: Endian Ordering] */
+      hash_rec *ent = HashTable[j];
+      if (lastword != NULL)
+	/* look for prefix match with prev string */
+	i = prefixlen (lastword, ent->word);
+      else
+	i = 0;
+      fputc (i, sp);
+      fputc (ent->word[0] - i, sp);
+      fwrite ((char *) ent->word + i + 1, sizeof (u_char), ent->word[0] - i, sp);
+      outputbytes += ent->word[0] - i + 1;
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      HTONUL2(ent->fcnt, fcnt);
+      fwrite ((char *) &(fcnt), sizeof (fcnt), 1, sp);
+
+      outputbytes += sizeof (ent->fcnt);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      HTONUL2(ent->wcnt, wcnt);
+      fwrite ((char *) &(wcnt), sizeof (wcnt), 1, sp);
+
+      outputbytes += sizeof (ent->wcnt);
+      lastword = ent->word;
+    }
+
+  fclose (sp);
+}				/* write_stem_file() */
+
+
+
+/*
+ *    write_codes():
+ *              calls functions to assign and write out codes and
+ *              then prints out stats about execution results.
+ *
+ *      Accesses outside variables:     z
+ *
+ *      Return value...:                z
+ */
+
+static void 
+write_codes (char *file_name)
+{
+  unsigned long dicts = 0;
+  outputbytes = 0;
+  write_stem_file (file_name);
+  dicts = outputbytes;
+
+#ifndef SILENT
+  Message ("Chunks written      : %d\n",
+	   ChunksWritten);
+  if (InvfLevel == 3)
+    Message ("Paragraphs          : %8d\n", callnum);
+  Message ("Peak memory usage   : %10.1f Mb\n",
+	   (double) MaxMemInUse / 1024.0 / 1024.0);
+  Message ("Stems size          : %10.1f kB  %5.2f%%\n",
+	   (double) dicts / 1024, (double) 100 * dicts / inputbytes);
+  Message ("Lev 1 inverted file : %10.1f kB  %5.2f%%\n",
+	   (double) (L1_bits + L1_ohead) / 1024,
+	   (double) 100 * (L1_bits + L1_ohead) / inputbytes);
+  Message ("Lev 2 inverted file : %10.1f kB  %5.2f%%\n",
+	   (double) (L2_bits + L2_ohead) / 1024,
+	   (double) 100 * (L2_bits + L2_ohead) / inputbytes);
+  Message ("Lev 3 inverted file : %10.1f kB  %5.2f%%\n",
+	   (double) (L3_bits + L3_ohead) / 1024,
+	   (double) 100 * (L3_bits + L3_ohead) / inputbytes);
+  Message ("Record addresses    : %10.1f kB  %5.2f%%\n",
+	   (double) words_diff * 4 / 1024, (double) 100 * words_diff * 4 / inputbytes);
+#endif
+}				/* write_codes() */
+
+
+
+void 
+write_num_file (char *file_name)
+{
+
+  int i;
+  FILE *f;
+
+  for (i = 0; i < HashSize; i++)
+    HashTable[i]->fnum = i;
+
+  if (!(f = create_file (file_name, INVF_CHUNK_TRANS_SUFFIX, "wb",
+			 MAGIC_CHUNK_TRANS, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return;
+
+#if 1
+  ENCODE_START (f)
+
+    for (i = 0; i < HashSize; i++)
+    BINARY_ENCODE (first_occr[i]->fnum + 1, HashSize + 1);
+
+  ENCODE_DONE;
+#else
+  for (i = 0; i < HashSize; i++)
+    {
+      /* [RPAP - Jan 97: Endian Ordering] */
+      unsigned long fnum;
+      HTONUL2(first_occur[i]->fnum, fnum);
+      fwrite ((char *) &fnum, sizeof (unsigned long), 1, f);
+    }
+#endif
+
+  fclose (f);
+}
+
+
+
+
+int 
+done_ivf_1 (char *FileName)
+{
+  char *temp_str = msg_prefix;
+  msg_prefix = "ivf.pass1";
+#ifndef SILENT
+  Message ("Mem reqd for 1 chunk: %8u bytes\n",
+	   max_mem_reqd ());
+#endif
+  if (lcallnum)
+    dump_dict (mem_reqd ());
+
+  if (PackHashTable () == COMPERROR)
+    return COMPERROR;
+
+  qsort (HashTable, HashUsed, sizeof (hash_rec *), ent_comp);
+
+  count_text ();
+  write_codes (FileName);
+
+  ENCODE_CONTINUE (sbs)
+    GAMMA_ENCODE (1);
+  ENCODE_DONE
+    fseek (ic, sizeof (long), 0);
+  
+  HTONSL(max_mem);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite (&max_mem, sizeof (max_mem), 1, ic);
+  NTOHSL(max_mem);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  fclose (ic);
+
+  write_num_file (FileName);
+
+  msg_prefix = temp_str;
+
+  return (COMPALLOK);
+}				/* done_encode */
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/ivf.pass2.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/ivf.pass2.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/ivf.pass2.c	(revision 699)
@@ -0,0 +1,1096 @@
+/**************************************************************************
+ *
+ * ivf.pass2.c -- Memory efficient pass 2 inversion
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:17:54  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.3  1998/12/17 09:12:51  rjmcnab
+
+   Altered mg to process utf-8 encoded Unicode. The main changes
+   are in the parsing of the input, the casefolding, and the stemming.
+
+   Revision 1.2  1998/11/25 07:55:43  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:34:45  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:49  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:35  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "local_strings.h"
+#include "sysfuncs.h"
+#include "memlib.h"
+#include "messages.h"
+#include "stemmer.h"
+#include "perf_hash.h"
+#include "bitio_m.h"
+#include "bitio_m_mems.h"
+#include "bitio_gen.h"
+#include "bitio_random.h"
+#include "bitio_stdio.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "invf.h"
+#include "locallib.h"
+#include "mg.h"
+#include "build.h"
+#include "words.h"
+#include "hash.h"
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+#include <io.h>
+#endif
+
+#ifndef RND_BUF_SIZE
+#define RND_BUF_SIZE 8*1024
+/*#define RND_BUF_SIZE 128 */
+#endif
+
+#define print_fsize(file)\
+do\
+  {\
+    struct stat file_state;\
+    fstat(fileno(invf_out), &file_state);\
+    Message("len(invf) = %ld", file_state.st_size);\
+  }while(0)
+
+typedef struct word_rec
+  {
+    unsigned long ptr;
+    unsigned long last;
+  }
+word_rec;
+
+typedef struct invf_state_rec
+  {
+    unsigned long Disk_Ptr;
+    unsigned long Disk_Last;
+    unsigned long Disk_B;
+  }
+invf_state_rec;
+
+typedef struct chunk
+  {
+    unsigned long start_doc;
+    unsigned long params_pos;
+    unsigned long disk_pos;
+    unsigned long N;
+  }
+chunk;
+
+
+static FILE *dict;		/* Stemmed dictionary file */
+static FILE *hash;		/* Stemmed dictionary hash file */
+static FILE *invf;		/* Inverted file */
+static FILE *invf_in;		/* Inverted file */
+static FILE *invf_out;		/* Inverted file */
+static FILE *invf_idx;		/* Inverted index file */
+static FILE *count;		/* Count file */
+static FILE *count_trans;	/* Count translation file */
+static FILE *invf_state;	/* Inverted file State */
+static FILE *chunk_state;	/* Chunk state */
+static FILE *chunks;		/* Chunk state */
+static FILE *invf_para = NULL;	/* Paragraph counts file */
+static FILE *weights = NULL;	/* Weights file */
+
+static stdio_bitio_state sbs;
+static random_bitio_state crbs;
+static chunk *chunk_data = NULL;
+static random_bitio_state rbs, rbsp;
+
+static int docs_left = 0, next_docs_left = 0;
+static unsigned long N;
+
+static word_rec *WordRecs;
+static u_char *lg_bs;
+static float *idf = NULL;
+
+static char *MemoryBuffer = NULL;
+static unsigned long MemBufSize;
+static unsigned long BufToUse;
+static struct invf_dict_header idh;
+
+static perf_hash_data *phd;
+
+static unsigned long *word_list = NULL;
+static unsigned long wl_size = 0;
+
+static unsigned long dict_size;
+static unsigned long no_of_ptrs = 0;
+static unsigned long chunks_read = 0;
+static unsigned long Disk_pos = 0;
+static unsigned long callnum = 0;
+static unsigned long wordnum = 0;
+
+static unsigned long totalIbytes = 0;
+static unsigned long totalDbytes = 0;
+static unsigned long totalHbytes = 0;
+
+static unsigned long MemInUse = 0;
+static unsigned long MaxMemInUse = 0;
+static unsigned long max_buffer_len;
+
+void 
+ChangeMemInUse (int mem)
+{
+  MemInUse += mem;
+  if (MemInUse > MaxMemInUse)
+    MaxMemInUse = MemInUse;
+}
+
+
+
+
+static int 
+open_files (char *file_name)
+{
+  char FName[200];
+
+  if (!(dict = open_file (file_name, INVF_DICT_SUFFIX, "rb",
+			  MAGIC_STEM_BUILD, MG_CONTINUE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (COMPERROR);
+
+  if (!(hash = open_file (file_name, INVF_DICT_HASH_SUFFIX, "rb",
+			  MAGIC_HASH, MG_CONTINUE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (COMPERROR);
+
+  if (!(count = open_file (file_name, INVF_CHUNK_SUFFIX, "rb",
+			   MAGIC_CHUNK, MG_CONTINUE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (COMPERROR);
+  fread (&max_buffer_len, sizeof (max_buffer_len), 1, count);
+  NTOHUL(max_buffer_len);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  BIO_Stdio_Decode_Start (count, &sbs);
+  next_docs_left = BIO_Stdio_Gamma_Decode (&sbs, NULL) - 1;
+
+  if (!(count_trans = open_file (file_name, INVF_CHUNK_TRANS_SUFFIX, "rb",
+				 MAGIC_CHUNK_TRANS, MG_CONTINUE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (COMPERROR);
+
+  if (!(invf = create_file (file_name, INVF_SUFFIX, "w+b",
+			    MAGIC_INVF, MG_CONTINUE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (COMPERROR);
+  fflush (invf);
+  if (!(invf_in = open_file (file_name, INVF_SUFFIX, "rb",
+			     MAGIC_INVF, MG_CONTINUE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (COMPERROR);
+  if (!(invf_out = create_file (file_name, INVF_SUFFIX, "wb",
+				MAGIC_INVF, MG_CONTINUE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (COMPERROR);
+  BIO_Random_Start (invf, RND_BUF_SIZE, &rbs);
+  BIO_Random_Start (invf, RND_BUF_SIZE, &rbsp);
+  ChangeMemInUse (RND_BUF_SIZE * 2);
+
+  if (!(invf_idx = create_file (file_name, INVF_IDX_SUFFIX, "wb",
+				MAGIC_INVI, MG_CONTINUE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (COMPERROR);
+
+  if (InvfLevel == 3)
+    if (!(invf_para = create_file (file_name, INVF_PARAGRAPH_SUFFIX, "wb",
+				   MAGIC_PARAGRAPH, MG_CONTINUE)))  /* [RPAP - Feb 97: WIN32 Port] */
+      return (COMPERROR);
+
+  sprintf (FName, FILE_NAME_FORMAT ".%ld", get_basepath (), file_name,
+	   ".invf.state", (long) getpid ());  /* [RPAP - Feb 97: WIN32 Port] */
+  if (!(invf_state = fopen (FName, "w+b")))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      Message ("Unable to create \"%s\"", FName);
+      return (COMPERROR);
+    }
+  unlink (FName);
+
+  sprintf (FName, FILE_NAME_FORMAT ".%ld", get_basepath (), file_name,
+	   ".chunk.state", (long) getpid ());  /* [RPAP - Feb 97: WIN32 Port] */
+  if (!(chunk_state = fopen (FName, "w+b")))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      Message ("Unable to create \"%s\"", FName);
+      return (COMPERROR);
+    }
+  unlink (FName);
+  BIO_Random_Start (chunk_state, RND_BUF_SIZE, &crbs);
+  ChangeMemInUse (RND_BUF_SIZE);
+
+  sprintf (FName, FILE_NAME_FORMAT ".%ld", get_basepath (), file_name,
+	   ".chunks", (long) getpid ());  /* [RPAP - Feb 97: WIN32 Port] */
+  if (!(chunks = fopen (FName, "w+b")))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      Message ("Unable to create \"%s\"", FName);
+      return (COMPERROR);
+    }
+  unlink (FName);
+
+  return (COMPALLOK);
+}
+
+
+
+
+
+#define ISR_CACHE 1024
+#define ISR_ENTRY_SIZE (sizeof(unsigned long)*2 + sizeof(unsigned long))
+
+invf_state_rec *
+in_cache (int pos)
+{
+  static char isr_data[ISR_CACHE * ISR_ENTRY_SIZE];
+  static invf_state_rec isr;
+  static int isr_base = 0, isr_num = -1, isr_pos = -1;
+  if (isr_pos >= 0)
+    bcopy ((char *) &isr, &isr_data[isr_pos * ISR_ENTRY_SIZE], ISR_ENTRY_SIZE);
+  if (pos < isr_base || pos >= isr_base + isr_num)
+    {
+      if (isr_num >= 0)
+	{
+	  fseek (invf_state, isr_base * ISR_ENTRY_SIZE, 0);
+	  fwrite (isr_data, 1, ISR_ENTRY_SIZE * isr_num, invf_state);
+	}
+      isr_base = pos;
+      fseek (invf_state, isr_base * ISR_ENTRY_SIZE, 0);
+      fread (isr_data, 1, ISR_ENTRY_SIZE * ISR_CACHE, invf_state);
+      isr_num = ISR_CACHE;
+    }
+  isr_pos = pos - isr_base;
+  bcopy (&isr_data[isr_pos * ISR_ENTRY_SIZE], (char *) &isr, ISR_ENTRY_SIZE);
+  return &isr;
+}
+
+
+
+
+
+unsigned long 
+occur_to_lexical (long occ)
+{
+  static long pos = -1;
+  static random_bitio_state rbs;
+  static int val = 0;
+  if (pos == -1)
+    {
+      BIO_Random_Start (count_trans, RND_BUF_SIZE, &rbs);
+      pos = 0x7fffffff;
+    }
+  if (occ < pos)
+    {
+      if (occ == -1)
+	{
+	  BIO_Random_Done (&rbs);
+	  return 0;
+	}
+      BIO_Random_Seek (32, &rbs);
+      pos = 0;
+    }
+  while (pos <= occ)
+    {
+      val = BIO_Random_Binary_Decode (dict_size + 1, &rbs, NULL) - 1;
+      pos++;
+    }
+  return (val);
+}
+
+
+void 
+add_chunk_state (unsigned long pos, unsigned long start_doc,
+		 unsigned long N)
+{
+  chunk_data[chunks_read].params_pos = pos;
+  chunk_data[chunks_read].start_doc = start_doc;
+  chunk_data[chunks_read].N = N;
+  chunks_read++;
+}
+
+
+int 
+init_ivf_2 (char *file_name)
+{
+  u_char prev[MAXSTEMLEN + 1];
+  int i;
+  unsigned long totalIbits;
+  double logN = 0.0;
+
+  if (open_files (file_name) == COMPERROR)
+    return COMPERROR;
+
+
+  /* Read in the stemmed dictionary file header */
+  fread ((char *) &idh, sizeof (idh), 1, dict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(idh.lookback);
+  NTOHUL(idh.dict_size);
+  NTOHUL(idh.total_bytes);
+  NTOHUL(idh.index_string_bytes);
+  NTOHD(idh.input_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(idh.num_of_docs);
+  NTOHUL(idh.static_num_of_docs);
+  NTOHUL(idh.num_of_words);
+  NTOHUL(idh.stemmer_num);
+  NTOHUL(idh.stem_method);
+
+  dict_size = idh.dict_size;
+
+  N = idh.num_of_docs;
+
+  if (!(phd = read_perf_hash_data (hash)))
+    {
+      Message ("Unable to read in hash data");
+      return COMPERROR;
+    }
+  totalHbytes = sizeof (perf_hash_data) + sizeof (u_char) * 256 +
+    sizeof (int) * (phd->MAX_N + 1) + sizeof (int *) * 3 * phd->MAX_CH +
+    sizeof (long) * phd->MAX_CH * phd->MAX_L;
+
+  if (!(WordRecs = Xmalloc (sizeof (word_rec) * idh.dict_size)))
+    {
+      Message ("No memory for word entries");
+      return COMPERROR;
+    }
+  totalDbytes += sizeof (word_rec) * idh.dict_size;
+
+  /* separate storage for the log(b) values, one byte each */
+  if (!(lg_bs = Xmalloc (sizeof (u_char) * idh.dict_size)))
+    {
+      Message ("No memory for lg b's");
+      return COMPERROR;
+    }
+  totalDbytes += sizeof (u_char) * idh.dict_size;
+
+  if (MakeWeights)
+    {
+      /* separate storage for the idf values, one single each */
+      if (!(idf = Xmalloc (sizeof (float) * idh.dict_size)))
+	{
+	  Message ("No memory for idf's");
+	  return COMPERROR;
+	}
+      totalDbytes += sizeof (float) * idh.dict_size;
+
+      if (!(weights = create_file (file_name, WEIGHTS_SUFFIX, "wb",
+				 MAGIC_WGHT, MG_CONTINUE))) { /* [RPAP - Feb 97: WIN32 Port] */
+	Message ("Couldn't open weights file for writing");
+	return (COMPERROR);
+      }
+    }
+  else
+    {
+      unlink (make_name (file_name, WEIGHTS_SUFFIX, NULL));
+    }
+
+  chunk_data = Xmalloc (sizeof (chunk) * (ChunkLimit + 2));
+  totalDbytes += sizeof (chunk) * (ChunkLimit + 2);
+
+  totalIbits = 32;		/* The magic number */
+  totalIbits += 8 * 100;	/* A 100 byte gap */
+
+  if (MakeWeights)
+    {
+      wl_size = 1024;
+      if (!(word_list = Xmalloc (sizeof (*word_list) * wl_size)))
+	{
+	  Message ("No memory for word_list");
+	  return COMPERROR;
+	}
+
+      logN = log ((double) N);
+    }
+
+  for (i = 0; i < idh.dict_size; i++)
+    {
+      invf_state_rec *isr;
+      register unsigned long copy, suff, p;
+      unsigned long fcnt, wcnt;
+
+      copy = fgetc (dict);
+      suff = fgetc (dict);
+      *prev = copy + suff;
+      fread (prev + copy + 1, sizeof (u_char), suff, dict);
+
+      fread ((char *) &fcnt, sizeof (fcnt), 1, dict);
+      fread ((char *) &wcnt, sizeof (wcnt), 1, dict);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fcnt);
+      NTOHUL(wcnt);
+
+      WordRecs[i].last = 0;
+      WordRecs[i].ptr = 0;
+
+      p = fcnt;
+
+      if (MakeWeights)
+	idf[i] = logN - log ((double) fcnt);
+
+
+      isr = in_cache (i);
+
+      isr->Disk_Last = 0;
+      isr->Disk_Ptr = totalIbits;
+
+      isr->Disk_B = BIO_Bblock_Init (N, p);
+
+      totalIbits += BIO_Bblock_Bound_b (N, p, isr->Disk_B);
+
+      if (InvfLevel >= 2)
+	totalIbits += BIO_Gamma_Bound (wcnt, fcnt);
+
+      totalIbits = (totalIbits + 7ul) & 0xfffffff8ul;
+
+    }
+
+
+  /* now convert to bytes, and actually get the space */
+  totalIbytes = (totalIbits + 7ul) >> 3ul;
+
+
+  return (COMPALLOK);
+
+}
+
+
+
+
+
+static void 
+LoadCounts (void)
+{
+  unsigned long numwords, i, last_total;
+  static unsigned long local_N = 0;
+  unsigned long totalIbits, crbs_pos;
+  word_rec *wr;
+  unsigned long *counts;
+
+  if (MemoryBuffer == NULL)
+    {
+      MemBufSize = sizeof (unsigned long) * dict_size;
+      if (max_buffer_len > MemBufSize)
+	MemBufSize = max_buffer_len;
+      if (!(MemoryBuffer = Xmalloc (MemBufSize)))
+	FatalError (1, "Unable to allocate memory for buffer");
+      ChangeMemInUse (MemBufSize);
+    }
+
+  counts = (unsigned long *) MemoryBuffer;
+  bzero ((char *) counts, sizeof (unsigned long) * dict_size);
+
+  docs_left = next_docs_left;
+  if (!docs_left)
+    FatalError (1, "The number of docs in the current chunk is 0");
+
+  BufToUse = BIO_Stdio_Gamma_Decode (&sbs, NULL) - 1;
+
+  numwords = BIO_Stdio_Gamma_Decode (&sbs, NULL) - 1;
+
+  local_N = docs_left;
+
+
+
+  for (wr = WordRecs, i = 0; i < dict_size; i++, wr++)
+    wr->ptr = 0;
+
+  bzero ((char *) lg_bs, dict_size);
+
+  for (i = 0; i < numwords; i++)
+    {
+      unsigned long word_num, wcnt, fcnt, p;
+      word_num = occur_to_lexical (i);
+
+      wr = &WordRecs[word_num];
+
+      wcnt = BIO_Stdio_Gamma_Decode (&sbs, NULL) - 1;
+      if (wcnt >= 2)
+	fcnt = BIO_Stdio_Gamma_Decode (&sbs, NULL);
+      else
+	fcnt = wcnt;
+
+      p = fcnt;
+
+      if (wcnt)
+	{
+	  register unsigned long length;
+	  counts[word_num] = p;
+	  length = BIO_Bblock_Bound (local_N, p);
+	  if (InvfLevel >= 2)
+	    length += wcnt;
+	  wr->ptr = length;
+	  lg_bs[word_num] = floorlog_2 (BIO_Bblock_Init_W (local_N, p));
+	}
+
+    }
+
+  crbs_pos = BIO_Random_Tell (&crbs);
+
+  totalIbits = 0;
+  last_total = 0;
+  for (wr = WordRecs, i = 0; i < dict_size; i++, wr++)
+    {
+      register unsigned long length;
+      length = wr->ptr;
+      wr->last = callnum;
+      BIO_Random_Gamma_Encode (counts[i] + 1, &crbs, NULL);
+      if (counts[i])
+	{
+	  if (i)
+	    BIO_Random_Delta_Encode (totalIbits - last_total + 1, &crbs, NULL);
+	  else
+	    BIO_Random_Delta_Encode (1, &crbs, NULL);
+
+	  last_total = totalIbits;
+	}
+      wr->ptr = totalIbits;
+      totalIbits += length;
+    }
+  add_chunk_state (crbs_pos, callnum, local_N);
+
+  if ((totalIbits + 7ul) >> 3ul > BufToUse)
+    FatalError (1, "Pointers exceed buffer size");
+
+  next_docs_left = BIO_Stdio_Gamma_Decode (&sbs, NULL) - 1;
+}
+
+
+
+
+static void 
+DumpChunk (void)
+{
+  chunk_data[chunks_read - 1].disk_pos = Disk_pos << 3;
+  fseek (chunks, Disk_pos, 0);
+  fwrite (MemoryBuffer, sizeof (char), BufToUse, chunks);
+  Disk_pos += BufToUse;
+}
+
+
+
+
+static void 
+DiskMerge (void)
+{
+  random_bitio_state *rbsi;
+  random_bitio_state *chks = NULL;
+  unsigned long *chunk_ptrs;
+  int i;
+
+  BIO_Random_Flush (&crbs);
+
+  chunk_ptrs = Xmalloc (chunks_read * sizeof (unsigned long));
+  ChangeMemInUse (chunks_read * sizeof (unsigned long));
+  bzero ((char *) chunk_ptrs, chunks_read * sizeof (unsigned long));
+
+  rbsi = Xmalloc (chunks_read * sizeof (random_bitio_state));
+  ChangeMemInUse (chunks_read * sizeof (random_bitio_state));
+  for (i = 0; i < chunks_read; i++)
+    {
+      rbsi[i] = crbs;
+      rbsi[i].Buf = Xmalloc (rbsi[i].len);
+      ChangeMemInUse (rbsi[i].len);
+      bcopy ((char *) (crbs.Buf), (char *) (rbsi[i].Buf), rbsi[i].len);
+      BIO_Random_Seek (chunk_data[i].params_pos, &rbsi[i]);
+    }
+
+  if (chunks_read > 1)
+    {
+      int j;
+      chks = Xmalloc ((chunks_read - 1) * sizeof (random_bitio_state));
+      ChangeMemInUse ((chunks_read - 1) * sizeof (random_bitio_state));
+      BIO_Random_Start (chunks, RND_BUF_SIZE, &chks[0]);
+      ChangeMemInUse (RND_BUF_SIZE);
+      for (j = 1; j < chunks_read - 1; j++)
+	{
+	  chks[j] = chks[0];
+	  chks[j].Buf = Xmalloc (chks[0].len);
+	  ChangeMemInUse (chks[0].len);
+	  bcopy ((char *) (chks[0].Buf), (char *) (chks[j].Buf), chks[0].len);
+	}
+    }
+  for (i = 0; i < dict_size; i++)
+    {
+      int j;
+      invf_state_rec *isr = in_cache (i);
+      register int B;
+
+      BIO_Random_Seek (isr->Disk_Ptr, &rbs);	/* Position in invf file */
+
+      B = isr->Disk_B;
+
+      for (j = 0; j < chunks_read; j++)
+	{
+	  int p;
+	  p = BIO_Random_Gamma_Decode (&rbsi[j], NULL) - 1;
+
+	  if (p)
+	    {
+	      int ptr, b;
+	      chunk_ptrs[j] += BIO_Random_Delta_Decode (&rbsi[j], NULL) - 1;
+	      ptr = chunk_ptrs[j];
+	      b = 1 << floorlog_2 (BIO_Bblock_Init_W (chunk_data[j].N, p));
+
+	      if (j == chunks_read - 1)
+		{
+		  int k, CurrDoc;
+		  DECODE_START ((u_char *) MemoryBuffer, ptr)
+		    CurrDoc = isr->Disk_Last;
+		  for (k = 0; k < p; k++)
+		    {
+		      register unsigned long x, tf;
+		      BBLOCK_DECODE (x, b);
+		      if (k == 0)
+			x = x + chunk_data[j].start_doc - isr->Disk_Last;
+		      CurrDoc += x;
+		      BIO_Random_Bblock_Encode (x, B, &rbs, NULL);
+		      if (InvfLevel >= 2)
+			{
+			  UNARY_DECODE (tf);
+			  BIO_Random_Gamma_Encode (tf, &rbs, NULL);
+			}
+		    }
+		  DECODE_DONE
+		    isr->Disk_Last = CurrDoc;
+		}
+	      else
+		{
+		  int k, CurrDoc;
+		  random_bitio_state *Chks = chks + j;
+		  BIO_Random_Seek (chunk_data[j].disk_pos + ptr, Chks);
+		  CurrDoc = isr->Disk_Last;
+		  for (k = 0; k < p; k++)
+		    {
+		      register unsigned long x, tf;
+		      x = BIO_Random_Bblock_Decode (b, Chks, NULL);
+		      if (k == 0)
+			x = x + chunk_data[j].start_doc - isr->Disk_Last;
+		      CurrDoc += x;
+		      BIO_Random_Bblock_Encode (x, B, &rbs, NULL);
+		      if (InvfLevel >= 2)
+			{
+			  tf = BIO_Random_Unary_Decode (Chks, NULL);
+			  BIO_Random_Gamma_Encode (tf, &rbs, NULL);
+			}
+		    }
+		  isr->Disk_Last = CurrDoc;
+		}
+	    }
+	}
+
+      isr->Disk_Ptr = BIO_Random_Tell (&rbs);
+
+    }
+  if (chunks_read > 1)
+    {
+      int j;
+      for (j = 0; j < chunks_read - 1; j++)
+	{
+	  Xfree (chks[j].Buf);
+	  ChangeMemInUse (-chks[j].len);
+	}
+      Xfree (chks);
+      ChangeMemInUse (-(chunks_read - 1) * sizeof (random_bitio_state));
+    }
+
+  for (i = 0; i < chunks_read; i++)
+    {
+      Xfree (rbsi[i].Buf);
+      ChangeMemInUse (-rbsi[i].len);
+    }
+  Xfree (rbsi);
+  ChangeMemInUse (-chunks_read * sizeof (random_bitio_state));
+  chunks_read = 0;
+  Xfree (chunk_ptrs);
+  ChangeMemInUse (-chunks_read * sizeof (unsigned long));
+  Disk_pos = 0;
+  BIO_Random_Seek (0, &crbs);
+}
+
+static void 
+MergeIn (void)
+{
+  static int disk_chunks = 0;
+  static header = 0;
+  if (!header)
+    {
+      fprintf (stderr, "ivf.pass2 : ");
+      header = 1;
+    }
+  if (disk_chunks == ChunkLimit || next_docs_left == 0)
+    {
+      fprintf (stderr, "M");
+      DiskMerge ();
+      disk_chunks = 0;
+    }
+  else
+    {
+      fprintf (stderr, "-");
+      DumpChunk ();
+      disk_chunks++;
+    }
+  if (next_docs_left == 0)
+    fprintf (stderr, "\n");
+}
+
+
+static int 
+wl_comp (const void *a, const void *b)
+{
+  return *((int *) a) - *((int *) b);
+}
+
+static int 
+process_doc (u_char * s_in, int l_in)
+{
+  int res;
+  u_char *end = s_in + l_in - 1;
+  unsigned long tocode;
+  unsigned long wl_pos = 0;
+
+  if (!docs_left)
+    LoadCounts ();
+
+  callnum++;
+
+  if (!inaword (s_in, end))
+    if (SkipSGML)
+      PARSE_NON_STEM_WORD_OR_SGML_TAG (s_in, end);
+    else
+      PARSE_NON_STEM_WORD (s_in, end);
+
+  while (s_in <= end)
+    {
+      u_char Word[MAXSTEMLEN + 1];
+
+      PARSE_STEM_WORD (Word, s_in, end);
+      stemmer (idh.stem_method, idh.stemmer_num, Word);
+      if (SkipSGML)
+	PARSE_NON_STEM_WORD_OR_SGML_TAG (s_in, end);
+      else
+	PARSE_NON_STEM_WORD (s_in, end);
+
+      if (*Word == 0)
+	continue;
+
+      res = perf_hash (phd, Word);
+
+      {
+	word_rec *arr = &WordRecs[res];
+	int b = 1 << lg_bs[res];
+	wordnum++;
+
+	tocode = callnum;
+
+	ENCODE_START ((u_char *) MemoryBuffer, arr->ptr)
+
+	  if (tocode > arr->last)
+	  {
+	    register int x;
+	    x = tocode - arr->last - 1;
+	    BBLOCK_ENCODE (x + 1, b);
+	    if (InvfLevel >= 2)
+	      ENCODE_BIT (1);
+	    no_of_ptrs++;
+	    arr->last = tocode;
+	  }
+	else if (InvfLevel >= 2)
+	  {
+	    __pos--;
+	    ENCODE_BIT (0);
+	    ENCODE_BIT (1);
+	  }
+	arr->ptr = __pos;
+	ENCODE_DONE
+      }
+
+      if (MakeWeights)
+	{
+	  if (wl_pos >= wl_size)
+	    {
+	      wl_size += (wl_size >> 1);
+	      word_list = Xrealloc (word_list, sizeof (*word_list) * wl_size);
+	    }
+	  word_list[wl_pos++] = res;
+	}
+    }
+  if (MakeWeights)
+    {
+      float doc_weight = 0.0;
+      if (wl_pos)
+	{
+	  unsigned long *wl = word_list;
+	  unsigned long i, count, val;
+	  qsort (wl, wl_pos, sizeof (*wl), wl_comp);
+	  count = 1;
+	  val = *wl++;
+	  for (i = 1; i <= wl_pos; i++, wl++)
+	    if (i == wl_pos || val != *wl)
+	      {
+		double weight = count * idf[val];
+		doc_weight += weight * weight;
+		count = 1;
+		val = *wl;
+	      }
+	    else
+	      count++;
+	}
+      HTONF(doc_weight);  /* [RPAP - Jan 97: Endian Ordering] */
+      fwrite ((char *) &doc_weight, sizeof (doc_weight), 1, weights);
+    }
+  docs_left--;
+  if (!docs_left)
+    MergeIn ();
+
+  return COMPALLOK;
+}
+
+int 
+process_ivf_2 (u_char * s_in, int l_in)
+{
+  if (InvfLevel <= 2)
+    return process_doc (s_in, l_in);
+  else
+    {
+      int count = 0;
+      int pos = 0;
+      u_char *start = s_in;
+      while (pos < l_in)
+	{
+	  if (s_in[pos] == TERMPARAGRAPH)
+	    {
+	      int len = pos + s_in + 1 - start;
+	      if (process_doc (start, len) != COMPALLOK)
+		return (COMPERROR);
+	      start = s_in + pos + 1;
+	      count++;
+	    }
+	  pos++;
+	}
+      if (start < s_in + pos)
+	{
+	  if (process_doc (start, pos + s_in - start) != COMPALLOK)
+	    return (COMPERROR);
+	  count++;
+	}
+      HTONSI(count);  /* [RPAP - Jan 97: Endian Ordering] */
+      fwrite ((char *) &count, sizeof (count), 1, invf_para);
+    }
+  return COMPALLOK;
+}
+
+
+
+
+
+static void 
+stats (unsigned long len)
+{
+#ifndef SILENT
+  fseek (count, 0, 2);
+  fseek (count_trans, 0, 2);
+  fseek (invf_state, 0, 2);
+  fseek (invf, 0, 0);
+  fseek (invf, 0, 2);
+  fseek (chunks, 0, 2);
+  fseek (chunk_state, 0, 2);
+  Message ("File sizes\n");
+  Message ("  Chunk desc    : %10u bytes\n", ftell (count));
+  Message ("  Chunk trans   : %10u bytes\n", ftell (count_trans));
+  Message ("  Chunks        : %10u bytes\n", ftell (chunks));
+  Message ("  Chunk state   : %10u bytes\n", ftell (chunk_state));
+  Message ("  Invf state    : %10u bytes\n", ftell (invf_state));
+  Message ("  Peak invf     : %10u bytes\n", len);
+  Message ("  Final invf    : %10u bytes\n", ftell (invf));
+  Message ("Peak disk usage : %10.2f %%\n",
+	   (double) (ftell (count) + ftell (count_trans) +
+		     ftell (invf_state) + ftell (chunks) +
+		     ftell (chunk_state) + len) / ftell (invf) * 100.0);
+#endif
+}
+
+
+/* ARGSUSED */
+int 
+done_ivf_2 (char *FileName)
+{
+  long i;
+  unsigned long totalIbits;
+  unsigned long invf_len;
+  unsigned long bytes_output;
+  struct invf_file_header ifh;
+
+  if (weights)
+    fclose (weights);
+  if (invf_para)
+    fclose (invf_para);
+
+  free_perf_hash (phd);
+
+  free (MemoryBuffer);
+  ChangeMemInUse (-MemBufSize);
+
+  BIO_Random_Done (&rbs);
+  BIO_Random_Done (&rbsp);
+  fflush (invf);
+
+  fseek (invf, 0, 2);
+  invf_len = ftell (invf);
+
+  fseek (invf_out, sizeof (long), 0);
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL2(dict_size, ifh.no_of_words);
+  HTONUL2(no_of_ptrs, ifh.no_of_ptrs);
+  ifh.skip_mode = 0;
+  bzero ((char *) ifh.params, sizeof (ifh.params));
+  HTONUL2(InvfLevel, ifh.InvfLevel);
+  fwrite ((char *) &ifh, sizeof (ifh), 1, invf_out);
+
+  bytes_output = ftell (invf_out);
+
+  totalIbits = 32;		/* The magic number */
+  totalIbits += 8 * 100;	/* A 100 byte gap */
+
+  /* find the right place in the file to start reading p values */
+  fseek (dict, sizeof (unsigned long) + sizeof (struct invf_dict_header), 0);
+  for (i = 0; i < dict_size; i++)
+    {
+      invf_state_rec *isr;
+      unsigned long fcnt, wcnt, s, e;
+      register unsigned long p;
+      u_char dummy1, dummy2[MAXSTEMLEN + 1];
+
+      /* output location to the invf_idx  */
+      HTONUL(bytes_output);  /* [RPAP - Jan 97: Endian Ordering] */
+      fwrite ((char *) &bytes_output, sizeof (bytes_output), 1, invf_idx);
+      NTOHUL(bytes_output);  /* [RPAP - Jan 97: Endian Ordering] */
+
+      /* read an entry for a word, just to get p value */
+      dummy1 = fgetc (dict);
+      dummy1 = fgetc (dict);
+      fread (dummy2, sizeof (u_char), dummy1, dict);
+      fread ((char *) &fcnt, sizeof (fcnt), 1, dict);
+      fread ((char *) &wcnt, sizeof (wcnt), 1, dict);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fcnt);
+      NTOHUL(wcnt);
+
+      p = fcnt;
+
+      isr = in_cache (i);
+
+      e = (isr->Disk_Ptr + 7ul) >> 3ul;
+      s = totalIbits >> 3;
+
+      fseek (invf_in, s, 0);
+      while (s < e)
+	{
+	  u_char c = getc (invf_in);
+	  if (s == e - 1)
+	    {
+	      u_char ands[8] =
+	      {0xff, 0x80, 0xc0, 0xe0, 0xf0, 0xf8, 0xfc, 0xfe};
+	      c &= ands[isr->Disk_Ptr & 7ul];
+	    }
+	  putc (c, invf_out);
+	  bytes_output++;
+	  s++;
+	}
+
+      totalIbits += BIO_Bblock_Bound_b (N, p, isr->Disk_B);
+      if (InvfLevel >= 2)
+	totalIbits += BIO_Gamma_Bound (wcnt, fcnt);
+      totalIbits = (totalIbits + 7ul) & 0xfffffff8ul;
+
+    }
+
+  fclose (invf_in);
+
+  /* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+  if (!(_chsize (_fileno (invf_out), bytes_output)))
+    Message ("Could not truncate invf.");
+#else
+  ftruncate (fileno (invf_out), bytes_output);
+#endif
+
+  fclose (invf_out);
+
+  HTONUL(bytes_output);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite ((char *) &bytes_output, sizeof (bytes_output), 1, invf_idx);
+  NTOHUL(bytes_output);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  fclose (invf_idx);
+
+#ifndef SILENT
+  {
+    char *temp_str = msg_prefix;
+    unsigned long total;
+    msg_prefix = "ivf.pass2";
+    stats (invf_len);
+    Message ("Pass two data structures      : %6.3f Mbyte\n",
+	     (double) totalDbytes / 1024 / 1024);
+    total = totalDbytes;
+    Message ("Pass two hash structure(s)    : %6.3f Mbyte\n",
+	     (double) totalHbytes / 1024 / 1024);
+    total += totalHbytes;
+    Message ("Peak extra memory in use      : %6.3f Mbyte\n",
+	     (double) MaxMemInUse / 1024 / 1024);
+    total += MaxMemInUse;
+    Message ("Peak total memory in use      : %6.3f Mbyte\n",
+	     (double) total / 1024 / 1024);
+    msg_prefix = temp_str;
+  }
+#endif
+
+  Xfree (WordRecs);
+  Xfree (lg_bs);
+
+  /* Free the memory allocated for the BIO_Random */
+  occur_to_lexical (-1);
+
+  BIO_Random_Done (&crbs);
+
+  fclose (invf);
+  fclose (dict);
+  fclose (hash);
+  fclose (count);
+  fclose (count_trans);
+  fclose (chunk_state);
+  fclose (chunks);
+  fclose (invf_state);
+  return (COMPALLOK);
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/lists.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/lists.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/lists.c	(revision 699)
@@ -0,0 +1,214 @@
+/**************************************************************************
+ *
+ * lists.c -- List processing functions for boolean queries
+ * Copyright (C) 1994  Neil Sharman, Alistair Moffat and Lachlan Andrew
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+
+#include "lists.h"
+#include "local_strings.h"
+#include "locallib.h"
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:18:01  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1999/07/01 09:30:48  rjmcnab
+   Changes for better match reporting.
+
+   Revision 1.1  1998/11/17 09:34:47  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:50  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:37  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+
+
+
+/*
+ * Make a DocList large enough to hold num entries 
+ */
+DocList *
+MakeDocList (int num)
+{
+  DocList *d;
+  if (!(d = Xmalloc (sizeof (DocList) + sizeof (DocEntry) * (num - 1))))
+    return (NULL);
+  d->num = 0;
+  d->total_retrieved = 0;
+  d->is_approx = 0;
+  return (d);
+}
+
+
+/*
+ * Change the size of a DocList so that it can hold num entries 
+ */
+DocList *
+ResizeDocList (DocList * d, int num)
+{
+  if (!(d = Xrealloc (d, sizeof (DocList) + sizeof (DocEntry) * (num - 1))))
+    return (NULL);
+  return (d);
+}
+
+
+/* IntersectLists () */
+/* Returns the intersection of List1 and List2 by deleting from List1 all */
+/* entries not in List2.  All of List2 is freed, and the nodes of List1 not */
+/* included in the resultant list are freed. */
+/* On exit the intersection of the lists is returned, or NULL on empty */
+/* intersection. */
+/* Assumes that both lists are ordered on increasing Value. */
+DocList *
+IntersectLists (DocList * List1, DocList * List2)
+{
+  DocEntry *s1, *d1, *s2;
+  if (!List2 || !List1)
+    {
+      if (List1)
+	Xfree (List1);
+      if (List2)
+	Xfree (List2);
+      return (MakeDocList (0));
+    }
+  d1 = s1 = List1->DE;
+  s2 = List2->DE;
+  while (s1 - List1->DE < List1->num && s2 - List2->DE < List2->num)
+    {
+      if (s1->DocNum == s2->DocNum)
+	*d1++ = *s1++;
+      if (s1 - List1->DE < List1->num)
+	while (s2 - List2->DE < List2->num && s2->DocNum < s1->DocNum)
+	  s2++;
+      if (s2 - List2->DE < List2->num)
+	while (s1 - List1->DE < List1->num && s1->DocNum < s2->DocNum)
+	  s1++;
+    }
+  List1->num = d1 - List1->DE;
+  Xfree (List2);
+  return (List1);
+}
+
+
+/* DiffLists () */
+/* Returns the difference of List1 and List2 by deleting from List1 all */
+/* entries in List2.  All of List2 is freed, and the nodes of List1 not */
+/* included in the resultant list are freed. */
+/* On exit the differrence of the lists is returned, or NULL on empty */
+/* intersection. */
+/* Assumes that both lists are ordered on increasing Value. */
+DocList *
+DiffLists (DocList * List1, DocList * List2)
+{
+  DocEntry *s1, *d1, *s2;
+  if (!List1)
+    {
+      if (List2)
+	Xfree (List2);
+      return (MakeDocList (0));
+    }
+  if (!List2)
+    return (List1);
+  d1 = s1 = List1->DE;
+  s2 = List2->DE;
+  while (s1 - List1->DE < List1->num)
+    {
+      if (s2 - List2->DE == List2->num)
+	*d1++ = *s1++;
+      else
+	{
+	  if (s1->DocNum == s2->DocNum)
+	    {
+	      s1++;
+	      s2++;
+	    }
+	  else if (s1->DocNum < s2->DocNum)
+	    *d1++ = *s1++;
+	  else
+	    s2++;
+	}
+    }
+  List1->num = d1 - List1->DE;
+  Xfree (List2);
+  return (List1);
+}
+
+/* MergeLists () */
+/* Finds the union of two lists and returns the head of the new list formed. */
+/* Assumes that both lists are ordered on increasing Value. */
+DocList *
+MergeLists (DocList * List1, DocList * List2)
+{
+  DocList *NewList;
+  DocEntry *d, *s1, *s2;
+  if (!List1)
+    {
+      if (List2)
+	return (List2);
+      else
+	return (MakeDocList (0));
+    }
+  if (!List2)
+    return (List1);
+  if (!(NewList = MakeDocList (List1->num + List2->num)))
+    {
+      Xfree (List1);
+      Xfree (List2);
+      return (NULL);
+    }
+
+  s1 = List1->DE;
+  s2 = List2->DE;
+  d = NewList->DE;
+  while (s1 - List1->DE < List1->num || s2 - List2->DE < List2->num)
+    {
+      if (s1 - List1->DE >= List1->num)
+	*d++ = *s2++;
+      else if (s2 - List2->DE >= List2->num)
+	*d++ = *s1++;
+      else if (s1->DocNum == s2->DocNum)
+	{
+	  *d = *s1++;
+	  (d++)->Weight += (s2++)->Weight;
+	}
+      else if (s1->DocNum < s2->DocNum)
+	*d++ = *s1++;
+      else
+	*d++ = *s2++;
+    }
+  NewList->num = d - NewList->DE;
+  Xfree (List1);
+  Xfree (List2);
+  return (NewList);
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/lists.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/lists.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/lists.h	(revision 699)
@@ -0,0 +1,59 @@
+/**************************************************************************
+ *
+ * lists.h -- List processing functions for boolean queries
+ * Copyright (C) 1994  Neil Sharman, Alistair Moffat and Lachlan Andrew
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef H_LISTS
+#define H_LISTS
+
+
+
+typedef struct DocEntry
+  {
+    float Weight;
+    int DocNum;
+    unsigned long SeekPos;	/* position in the text file in bytes */
+    unsigned long Len;		/* length of the document in bytes */
+    char *CompTextBuffer;
+    struct DocEntry *Next;
+    short or_included;		/*[TS:Mar/95] whether included in an AND of ORed-terms */
+  }
+DocEntry;
+
+typedef struct DocList
+  {
+    int num;
+    int total_retrieved; /* valid if > 0 */
+    int is_approx;       /* whether total_retrieved is approximate */
+    DocEntry DE[1];
+  }
+DocList;
+
+DocList *MakeDocList (int num);
+DocList *ResizeDocList (DocList * d, int num);
+DocList *IntersectLists (DocList * List1, DocList * List2);
+DocList *DiffLists (DocList * List1, DocList * List2);
+DocList *MergeLists (DocList * List1, DocList * List2);
+
+
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/locallib.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/locallib.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/locallib.c	(revision 699)
@@ -0,0 +1,304 @@
+/**************************************************************************
+ *
+ * locallib.c -- Misc functions
+ * Copyright (C) 1994  Gary Eddy, Alistair Moffat and Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "bitio_gen.h"
+#include "huffman.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "locallib.h"
+#include "text.h"
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:18:02  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:34:49  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:51  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:38  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+int 
+vecentropy (int *A, int n)
+{
+  int total = 0, i;
+  double bits = 0.0;
+
+  for (i = 0; i < n; i++)
+    total += A[i];
+  for (i = 0; i < n; i++)
+    if (A[i])
+      bits += A[i] * log2 (1.0 * total / A[i]);
+  return ((int) (bits + 0.9999));
+
+}
+
+static void 
+siftdown (int *A, int i, int n)
+{
+  int ms, t;
+
+  while (2 * i + 1 <= n - 1)
+    {
+      if (2 * i + 1 == n - 1 || A[2 * i + 1] < A[2 * i + 2])
+	ms = 2 * i + 1;
+      else
+	ms = 2 * i + 2;
+      if (A[i] > A[ms])
+	{
+	  t = A[i];
+	  A[i] = A[ms];
+	  A[ms] = t;
+	}
+      i = ms;
+    }
+}
+
+static void 
+buildheap (int *A, int n)
+{
+  int i;
+  for (i = n / 2 - 1; i >= 0; i--)
+    siftdown (A, i, n);
+}
+
+int 
+huffcodebits (unsigned long *A, int n)
+{
+  int i, j, tot = 0, bits = 0, v1, v2, *B;
+
+  B = (int *) Xmalloc (n * sizeof (int));
+  j = 0;
+  for (i = 0; i < n; i++)
+    {
+      if (A[i])
+	{
+	  tot += (B[j++] = A[i]);
+	}
+    }
+  n = j;
+
+  if (n == 0 || tot == 0)
+    return (0);
+
+  buildheap (B, n);
+  while (n > 1)
+    {
+      v1 = B[0];
+      B[0] = B[n - 1];
+      n--;
+      siftdown (B, 0, n);
+      v2 = B[0];
+      B[0] = v1 + v2;
+      siftdown (B, 0, n);
+      bits += v1 + v2;
+    }
+  Xfree (B);
+  return (bits);
+}
+
+
+int 
+modelbits (unsigned long *A, int n)
+{
+  int i, bits = 0, last, N = 0;
+
+  last = -1;
+  for (i = 0; i < n; i++)
+    {
+      if (A[i])
+	{
+	  bits += BIO_Gamma_Length (i - last) + BIO_Gamma_Length (A[i]);
+	  last = i;
+	  N++;
+	}
+    }
+  if (N == 0)
+    return (0);
+  bits += BIO_Gamma_Length (N);
+  return (bits);
+}
+
+
+
+
+
+/* Find the next prime number larger than p */
+int 
+prime (int p)
+{
+  int limit;
+  p += (p & 1) + 1;
+  limit = (int) sqrt ((double) p) + 1;
+  do
+    {
+      int j;
+      while (limit * limit < p)
+	limit++;
+      for (j = 3; j <= limit && p % j; j += 2)
+	;			/* NULLBODY */
+      if (j > limit)
+	break;
+      p += 2;
+    }
+  while (1);
+  return (p);
+}
+
+
+
+int 
+Read_cdh (FILE * f, compression_dict_header * cdh, u_long * mem,
+	  u_long * disk)
+{
+  if (disk)
+    (*disk) += sizeof (*cdh);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  if (fread (cdh, sizeof (*cdh), 1, f) == 1)
+    {
+      int i;
+      NTOHUL(cdh->dict_type);
+      NTOHUL(cdh->novel_method);
+      for (i = 0; i < TEXT_PARAMS; i++)
+	NTOHUL(cdh->params[i]);
+      NTOHUL(cdh->num_words[0]);
+      NTOHUL(cdh->num_words[1]);
+      NTOHUL(cdh->num_word_chars[0]);
+      NTOHUL(cdh->num_word_chars[1]);
+      NTOHUL(cdh->lookback);
+      return 0;
+    }
+  else
+    return -1;
+}
+
+
+int 
+F_Read_cdh (File * f, compression_dict_header * cdh, u_long * mem,
+	    u_long * disk)
+{
+  if (disk)
+    (*disk) += sizeof (*cdh);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  if (Fread (cdh, sizeof (*cdh), 1, f) == 1)
+    {
+      int i;
+      NTOHUL(cdh->dict_type);
+      NTOHUL(cdh->novel_method);
+      for (i = 0; i < TEXT_PARAMS; i++)
+	NTOHUL(cdh->params[i]);
+      NTOHUL(cdh->num_words[0]);
+      NTOHUL(cdh->num_words[1]);
+      NTOHUL(cdh->num_word_chars[0]);
+      NTOHUL(cdh->num_word_chars[1]);
+      NTOHUL(cdh->lookback);
+      return 0;
+    }
+  else
+    return -1;
+}
+
+
+int 
+Read_cfh (FILE * f, comp_frags_header * cfh, u_long * mem, u_long * disk)
+{
+  int i;  /* [RPAP - Jan 97: Endian Ordering] */
+
+  if (Read_Huffman_Data (f, &cfh->hd, mem, disk) == -1)
+    return -1;
+  if (fread (&cfh->uncompressed_size,
+	     sizeof (cfh->uncompressed_size), 1, f) != 1)
+    return -1;
+  NTOHUL(cfh->uncompressed_size);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  if (disk)
+    (*disk) += sizeof (cfh->uncompressed_size);
+
+  bzero ((char *) cfh->huff_words_size, sizeof (cfh->huff_words_size));
+
+  if (fread (cfh->huff_words_size + cfh->hd.mincodelen,
+	     sizeof (*cfh->huff_words_size),
+	     cfh->hd.maxcodelen - cfh->hd.mincodelen + 1, f) !=
+      cfh->hd.maxcodelen - cfh->hd.mincodelen + 1)
+    return -1;
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = cfh->hd.mincodelen; i < cfh->hd.maxcodelen + 1; i++)
+    NTOHUL(cfh->huff_words_size[i]);
+
+  if (disk)
+    (*disk) += sizeof (*cfh->huff_words_size) *
+      (cfh->hd.maxcodelen - cfh->hd.mincodelen + 1);
+  return 0;
+}
+
+
+
+int 
+F_Read_cfh (File * f, comp_frags_header * cfh, u_long * mem, u_long * disk)
+{
+  int i;  /* [RPAP - Jan 97: Endian Ordering] */
+
+  if (F_Read_Huffman_Data (f, &cfh->hd, mem, disk) == -1)
+    return -1;
+  if (Fread (&cfh->uncompressed_size,
+	     sizeof (cfh->uncompressed_size), 1, f) != 1)
+    return -1;
+  NTOHUL(cfh->uncompressed_size);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  if (disk)
+    (*disk) += sizeof (cfh->uncompressed_size);
+
+  bzero ((char *) cfh->huff_words_size, sizeof (cfh->huff_words_size));
+
+  if (Fread (cfh->huff_words_size + cfh->hd.mincodelen,
+	     sizeof (*cfh->huff_words_size),
+	     cfh->hd.maxcodelen - cfh->hd.mincodelen + 1, f) !=
+      cfh->hd.maxcodelen - cfh->hd.mincodelen + 1)
+    return -1;
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = cfh->hd.mincodelen; i < cfh->hd.maxcodelen + 1; i++)
+    NTOHUL(cfh->huff_words_size[i]);
+
+  if (disk)
+    (*disk) += sizeof (*cfh->huff_words_size) *
+      (cfh->hd.maxcodelen - cfh->hd.mincodelen + 1);
+
+  return 0;
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/locallib.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/locallib.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/locallib.h	(revision 699)
@@ -0,0 +1,52 @@
+/**************************************************************************
+ *
+ * locallib.h -- Misc functions
+ * Copyright (C) 1994  Gary Eddy, Alistair Moffat and Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef LOCALLIB_H
+#define LOCALLIB_H
+
+#include "sysfuncs.h"
+
+#include "text.h"
+
+#define NUMOF(a) (sizeof(a)/sizeof((a)[0]))
+
+int vecentropy (int *A, int n);
+
+int huffcodebits (unsigned long *A, int n);
+
+int modelbits (unsigned long *A, int n);
+
+int prime (int p);
+
+
+int Read_cdh (FILE * f, compression_dict_header * cdh, u_long * mem, u_long * disk);
+
+int Read_cfh (FILE * f, comp_frags_header * cfh, u_long * mem, u_long * disk);
+
+int F_Read_cdh (File * f, compression_dict_header * cdh, u_long * mem,
+		u_long * disk);
+
+int F_Read_cfh (File * f, comp_frags_header * cfh, u_long * mem, u_long * disk);
+
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/mg.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/mg.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/mg.h	(revision 699)
@@ -0,0 +1,42 @@
+/**************************************************************************
+ *
+ * mg.h -- General header file
+ * Copyright (C) 1994  Neil Sharman, Alistair Moffat and Lachlan Andrew
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef H_MG
+#define H_MG
+
+
+#define COMPALLOK	(0)
+#define COMPERROR	(-1)
+
+#define MAXSTRING       3000000
+#define BUFFERSIZE      (MAXSTRING+30000)
+
+
+
+
+#define	MAXLN	65536
+#define WORDS           1
+#define NONWORDS        0
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/mg.special.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/mg.special.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/mg.special.c	(revision 699)
@@ -0,0 +1,115 @@
+/**************************************************************************
+ *
+ * mg.special.c -- Special pass for mg_passes
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "mg.h"
+#include "build.h"
+
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:18:03  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:34:51  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:53  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:44  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+FILE *fp;
+
+
+int 
+init_special (char *FileName)
+{
+  char FName[200];
+  sprintf (FName, "%s%s", FileName, ".docnums");
+  if (!(fp = fopen (FName, "w")))
+    return (COMPERROR);
+  return (COMPALLOK);
+}
+
+
+
+
+int 
+process_special (u_char * s_in, int l_in)
+{
+  static int un = 1;
+  char *s_done = (char *) (s_in + l_in);
+  for (; (char *) s_in != s_done; s_in++)
+    if (*s_in == '<')
+      {
+	if (strncmp ((char *) s_in, "<DOCNO>", 7) == 0)
+	  {
+	    char *s_end = (char *) (s_in + 1);
+	    for (; s_end != s_done; s_end++)
+	      if (*s_end == '<')
+		if (strncmp (s_end, "</DOCNO>", 8) == 0)
+		  break;
+	    if (s_end == s_done)
+	      {
+		fprintf (fp, "NO DOC ID %d\n", un++);
+		return COMPALLOK;
+	      }
+	    s_in += 7;
+	    while (s_end != (char *) s_in &&
+		   (*(s_end - 1) == ' ' || *(s_end - 1) == '\n'))
+	      s_end--;
+	    while ((char *) s_in != s_end &&
+		   (*s_in == ' ' || *s_in == '\n'))
+	      s_in++;
+	    for (; (char *) s_in != s_end; s_in++)
+	      putc (*s_in, fp);
+	    putc ('\n', fp);
+	    return (COMPALLOK);
+	  }
+      }
+  fprintf (fp, "NO DOC ID %d\n", un++);
+  return (COMPALLOK);
+}
+
+
+
+
+
+
+
+
+int 
+done_special (char *FileName)
+{
+  fclose (fp);
+  return COMPALLOK;
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/mg_compression_dict.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/mg_compression_dict.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/mg_compression_dict.1	(revision 699)
@@ -0,0 +1,219 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_compression_dict 1 \*(Dt CITRI
+.SH NAME
+mg_compression_dict \- build a compression dictionary.
+.SH SYNOPSIS
+.B mg_compression_dict
+[
+.B \-h
+]
+[
+.BR \-C " |"
+.BR \-P " |"
+.B  \-S
+]
+.if n .ti +9n
+[
+.BR \-0 " |"
+.BR \-1 " |"
+.BR \-2 " |"
+.B  \-3
+]
+[
+.BR \-H " |"
+.BR \-B " |"
+.BR \-D " |"
+.BR \-Y " |"
+.B  \-M
+]
+.if n .ti +9n
+.if t .ti +.5i
+[
+.BI \-l " lookback"
+]
+[
+.BI \-k " mem"
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_compression_dict
+builds a compression dictionary based on the statistics gathered
+during the first pass over the text.  The options to the program are
+mainly concerned with limiting the amount of memory the dictionary
+will use and with how the text compressor will cope with any novel
+words found during the compression phase.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-C
+Build a complete dictionary from the statistics file.  If during the
+text compression phase a novel word is found, then the compressor will
+produce an error message and stop.
+.TP
+.B \-P
+Build a partial dictionary from the statistics file.  This dictionary
+assumes that the statistics file are based on the entire text.  The
+statistics of words not includes in the dictionary are used to
+calculate the escape probability.  If novel words are being coded
+character by character, then there may not be a Huffman code for every
+possible character.  This means that the compressor may fail if a novel
+word contains a novel character.
+.TP
+.B \-S
+Build a seed dictionary from the statistics file.  This dictionary
+assumes that the statistics file is based on only a portion of the
+text to be compressed.  The probability of a novel word is based on the
+number of words that have only occurred once.  If novel words are being
+coded character by character, then the Huffman codes for characters are
+based on the frequency of characters in the dictionary.
+.TP
+.B \-0
+All words from the statistics file are included in the built
+dictionary.
+.TP
+.B \-1
+Words are included in the dictionary until the dictionary reaches the
+desired size.  Words are selected for the dictionary based on the order
+they occurred in the source text.
+.TP
+.B \-2
+Words are included in the dictionary until the dictionary reaches the
+desired size.  The most frequent words are included in the dictionary
+first; where there is a tie for frequency, the shortest word is
+included first.
+.TP
+.B \-3
+Words are included in the dictionary until the dictionary reaches the
+desired size.  The most frequent words are included in the dictionary
+first; where there is a tie for frequency, the shortest word is
+included first.  Words are the shuffled back and forth between the
+`keep' and `discard' lists to find the `optimal' set of words that
+should be in the dictionary.
+.TP
+.B \-H
+This specifies that novel words will be coded character by character
+using Huffman codes.
+.TP
+.B \-B
+This specifies that an auxiliary dictionary will be built by the
+compressor.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using binary codes.  The binary code represents their occurrence
+position in the auxiliary dictionary.
+.TP
+.B \-D
+This specifies that an auxiliary dictionary will be built by the
+compressor.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using delta codes.  The delta code represents their occurrence position
+in the auxiliary dictionary.
+.TP
+.B \-Y
+This specifies that an auxiliary dictionary will be built by the
+compressor.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using a combination of gamma and binary codes.  The code represents
+their occurrence position in the auxiliary dictionary.  This generally
+produces better compression than
+.B \-B
+or
+.BR \-D .
+.TP
+.B \-M
+This specifies that an auxiliary dictionary will be built by the
+compressor.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using a combination of gamma and binary codes.  The code represents
+their occurrence position in the auxiliary dictionary.  This method is
+adaptive within documents, and generally produces better compression
+than
+.BR \-B ,
+.B \-D
+or
+.BR \-Y .
+.TP
+.BI \-l " lookback"
+The generated dictionary is designed to be front coded when it is
+loaded into memory.  Under normal circumstances, a front-coded
+dictionary would require scanning from the beginning in order to find
+any particular word.  However, every
+.I lookback
+words in the dictionary, the whole word is stored and a pointer to that
+word maintained.  E.g., if
+.I lookback
+is 4, then every fourth word is stored in its entirety.
+.TP
+.BI \-k " mem"
+This limits the amount of memory to use for the generated
+dictionary.  Words are selected for the dictionary based of the text
+statistics, and whether
+.BR \-0 , " \-1" , " \-2"
+or
+.B \-3
+is specified.  The memory is calculated assuming a lookback of 0,
+irrespective of what actual lookback is specified.  This means that if
+a non-zero lookback is given, the dictionary will actually occupy
+less space than specified by
+.BR \-k .
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.text.stats
+Statistics about the source text.
+.TP
+.B *.text.dict
+Compression dictionary for the source text.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/mg_compression_dict.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/mg_compression_dict.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/mg_compression_dict.c	(revision 699)
@@ -0,0 +1,1081 @@
+/**************************************************************************
+ *
+ * mg_compression_dict.c -- Routines for reading the compression dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "local_strings.h"
+#include "bitio_gen.h"
+#include "bitio_m.h"
+#include "bitio_m_stdio.h"
+#include "timing.h"
+#include "mgheap.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "locallib.h"
+#include "invf.h"
+#include "text.h"
+#include "words.h"
+#include "mg.h"
+
+#define MAXBITS (sizeof(unsigned long) * 8)
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:18:04  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.4  1999/01/15 03:05:51  rjmcnab
+
+   Renamed lib/heap to be lib/mgheap (it was causing some problems with
+   some versions of the STL libraries which contained a heap.h)
+
+   Revision 1.3  1998/12/17 09:12:52  rjmcnab
+
+   Altered mg to process utf-8 encoded Unicode. The main changes
+   are in the parsing of the input, the casefolding, and the stemming.
+
+   Revision 1.2  1998/11/25 07:55:44  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:34:52  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:54  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:45  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+/* #define DEBUG1 */
+
+/* #define DEBUG */
+
+#define is_power_of_two(a) ((a) != 0 && (((a) & ((a)-1)) == 0))
+
+#define MAX_RECALCULATIONS 100
+
+typedef struct WordData
+  {
+    u_long freq, occur_num;
+    float saving;
+    float char_bit_cost;
+    u_long num_trans;
+    u_char *word;
+  }
+WordData;
+
+static WordData *Words[2];
+static u_long Num[2];
+static u_long chars[2];
+
+#define KIND(p) (((p) >= Words[0] && (p) < Words[0] + Num[0]) ? 0 : 1)
+#define IsWord(p) (((p) >= Words[1] && (p) < Words[1] + Num[1]) ? 1 : 0)
+#define IsNonWord(p) (((p) >= Words[0] && (p) < Words[0] + Num[0]) ? 1 : 0)
+
+
+typedef struct DictData
+  {
+    WordData **wd;
+    u_long num_wds;
+    u_long chars;
+  }
+DictData;
+
+typedef DictData DictInfo[2];
+
+static DictInfo keep, discard, all;
+static compression_stats_header csh;
+
+static char *file_name = "";
+static u_long novel_method = MG_NOVEL_HUFFMAN_CHARS;
+
+
+static void ReadInWords (char *);
+static void Select_on (int k, heap_comp hc);
+static void Method3 (int k);
+static void Select_all (void);
+static u_long WriteOutWords (char *, u_long, int);
+static int DecFreqIncWL (void *a, void *b);
+static int OccuranceOrder (void *a, void *b);
+
+
+
+int main (int argc, char **argv)
+{
+  ProgTime StartTime;
+  int ch;
+  char type = 'C';
+  char mode = '0';
+  int lookback = 2;
+  double k = 0;
+  u_long mem_reqd;
+  opterr = 0;
+  msg_prefix = argv[0];
+  while ((ch = getopt (argc, argv, "0123CPSf:d:l:hk:HBDYM")) != -1)
+    switch (ch)
+      {
+      case 'H':
+	novel_method = MG_NOVEL_HUFFMAN_CHARS;
+	break;
+      case 'B':
+	novel_method = MG_NOVEL_BINARY;
+	break;
+      case 'D':
+	novel_method = MG_NOVEL_DELTA;
+	break;
+      case 'Y':
+	novel_method = MG_NOVEL_HYBRID;
+	break;
+      case 'M':
+	novel_method = MG_NOVEL_HYBRID_MTF;
+	break;
+      case 'f':		/* input file */
+	file_name = optarg;
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 'C':
+      case 'P':
+      case 'S':
+	type = ch;
+	break;
+      case '0':
+      case '1':
+      case '2':
+      case '3':
+	mode = ch;
+	break;
+      case 'l':
+	lookback = atoi (optarg);
+	if (!is_power_of_two (lookback))
+	  FatalError (1, "The lookback value must be a power of 2");
+	lookback = floorlog_2 (lookback);
+	break;
+      case 'k':
+	k = atof (optarg) * 1024;
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-l lookback] [-f input_file]"
+	"[-d data directory] [-h] [-0|-1|-2|-3] [-C|-P|-S] [-k mem (Kb)]\n",
+		 argv[0]);
+	exit (1);
+      }
+  GetTime (&StartTime);
+
+  ReadInWords (file_name);
+
+  if (type == 'C')
+    {
+      Select_all ();
+      mem_reqd = WriteOutWords (file_name, MG_COMPLETE_DICTIONARY, lookback);
+    }
+  else
+    {
+      switch (mode)
+	{
+	case '0':
+	  Select_all ();
+	  break;
+	case '1':
+	  Message ("Dictionary limit of %.2f Kb", k / 1024);
+	  Select_on ((int) k, OccuranceOrder);
+	  break;
+	case '2':
+	  Message ("Dictionary limit of %.2f Kb", k / 1024);
+	  Select_on ((int) k, DecFreqIncWL);
+	  break;
+	case '3':
+	  Message ("Dictionary limit of %.2f Kb", k / 1024);
+	  Method3 ((int) k);
+	  break;
+	}
+      if (type == 'P')
+	{
+	  mem_reqd = WriteOutWords (file_name, MG_PARTIAL_DICTIONARY, lookback);
+	}
+      else
+	{
+	  mem_reqd = WriteOutWords (file_name, MG_SEED_DICTIONARY, lookback);
+	}
+    }
+
+  Message ("Num words           : %8u -> %8u\n", Num[1], keep[1].num_wds);
+  Message ("Num non-words       : %8u -> %8u\n", Num[0], keep[0].num_wds);
+  Message ("Chars of words      : %8u -> %8u\n", chars[1], keep[1].chars);
+  Message ("Chars of non-words  : %8u -> %8u\n", chars[0], keep[0].chars);
+  Message ("Mem usage           : %8u -> %8u\n",
+	   (Num[0] + Num[1]) * sizeof (char *) + chars[0] + chars[1],
+	     (keep[0].num_wds + keep[1].num_wds) * sizeof (char *) +
+	   keep[0].chars + keep[1].chars);
+  Message ("Actual mem required : %8u\n", mem_reqd);
+
+  Message ("%s", ElapsedTime (&StartTime, NULL));
+  return 0;
+}
+
+
+
+
+static void 
+ReadInWords (char *filename)
+{
+  FILE *f;
+  unsigned long i;
+  f = open_file (filename, TEXT_STATS_DICT_SUFFIX, "rb",
+		 MAGIC_STATS_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  fread (&csh, sizeof (csh), 1, f);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(csh.num_docs);
+  NTOHUL(csh.num_bytes);
+
+  for (i = 0; i < 2; i++)
+    {
+      frags_stats_header fsh;
+      char *buf;
+      WordData *wd;
+      int j;
+      chars[i] = 0;
+
+      fread (&fsh, sizeof (fsh), 1, f);
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fsh.num_frags);
+      NTOHUL(fsh.mem_for_frags);
+
+      Num[i] = all[i].num_wds = fsh.num_frags;
+
+      /* The +1 on the following line is to leave room for the esc code. */
+      all[i].wd = Xmalloc (sizeof (WordData *) * Num[i] + 1);
+
+      buf = Xmalloc (fsh.mem_for_frags);
+      wd = Words[i] = Xmalloc (sizeof (WordData) * Num[i]);
+      for (j = 0; j < Num[i]; j++, wd++)
+	{
+	  int len;
+	  fread (&wd->freq, sizeof (wd->freq), 1, f);
+	  NTOHUL(wd->freq);  /* [RPAP - Jan 97: Endian Ordering] */
+	  fread (&wd->occur_num, sizeof (wd->occur_num), 1, f);
+	  NTOHUL(wd->occur_num);  /* [RPAP - Jan 97: Endian Ordering] */
+	  len = fgetc (f);
+	  wd->word = (u_char *) buf;
+	  *buf++ = len;
+	  fread (buf, len, 1, f);
+	  buf += len;
+	  all[i].wd[j] = wd;
+	}
+      chars[i] = fsh.mem_for_frags - fsh.num_frags;
+    }
+  fclose (f);
+}
+
+
+static void 
+Alloc_keep_discard (void)
+{
+  keep[0].num_wds = 0;
+  keep[0].wd = Xmalloc ((Num[0] + 1) * sizeof (WordData *));
+  keep[1].num_wds = 0;
+  keep[1].wd = Xmalloc ((Num[1] + 1) * sizeof (WordData *));
+  discard[0].num_wds = 0;
+  discard[0].wd = Xmalloc ((Num[0] + 1) * sizeof (WordData *));
+  discard[1].num_wds = 0;
+  discard[1].wd = Xmalloc ((Num[1] + 1) * sizeof (WordData *));
+}
+
+
+static int 
+sort_comp (const void *a, const void *b)
+{
+  WordData *aa = *(WordData **) a;
+  WordData *bb = *(WordData **) b;
+  return casecompare (aa->word, bb->word);  /* [RPAP - Jan 97: Stem Index Change] */
+}
+
+
+
+static void 
+SortAndCount_DictData (DictData * dd)
+{
+  int i;
+  WordData **wd;
+  qsort (dd->wd, dd->num_wds, sizeof (WordData *), sort_comp);
+  dd->chars = 0;
+  wd = dd->wd;
+  for (i = 0; i < dd->num_wds; i++, wd++)
+    dd->chars += (*wd)->word[0];
+}
+
+
+static void 
+Select_all (void)
+{
+  int i;
+  Alloc_keep_discard ();
+  keep[0].num_wds = Num[0];
+  for (i = 0; i < Num[0]; i++)
+    keep[0].wd[i] = Words[0] + i;
+  keep[1].num_wds = Num[1];
+  for (i = 0; i < Num[1]; i++)
+    keep[1].wd[i] = Words[1] + i;
+  SortAndCount_DictData (&keep[0]);
+  SortAndCount_DictData (&keep[1]);
+}
+
+
+
+
+static int 
+DecFreqIncWL (void *a, void *b)
+{
+  WordData *aa = *(WordData **) a;
+  WordData *bb = *(WordData **) b;
+  if (aa->freq > bb->freq)
+    return -1;
+  else if (aa->freq < bb->freq)
+    return 1;
+  else
+    return bb->word[0] - aa->word[0];
+}
+
+
+static int 
+OccuranceOrder (void *a, void *b)
+{
+  WordData *aa = *(WordData **) a;
+  WordData *bb = *(WordData **) b;
+  if (aa->occur_num > bb->occur_num)
+    return 1;
+  else if (aa->occur_num < bb->occur_num)
+    return -1;
+  else
+    return 0;
+}
+
+
+static void 
+Select_on (int k, heap_comp hc)
+{
+  int i, num, mem;
+  WordData **wd;
+
+  Alloc_keep_discard ();
+
+  num = Num[0] + Num[1];
+  wd = Xmalloc (num * sizeof (WordData *));
+  for (i = 0; i < Num[0]; i++)
+    wd[i] = Words[0] + i;
+  for (i = 0; i < Num[1]; i++)
+    wd[i + Num[0]] = Words[1] + i;
+
+  heap_build (wd, sizeof (*wd), num, hc);
+
+  mem = 0;
+  while (k > mem && num)
+    {
+      int idx;
+      WordData *word = wd[0];
+#ifdef DEBUG
+      fprintf (stderr, "%4d:%4d:%8d :%8d :%8d : \"%s\"\n",
+	       keep[0].num_wds, keep[1].num_wds,
+	       mem, word->freq, word->occur_num, word2str (word->word));
+#endif
+      mem += sizeof (u_char *) + word->word[0];
+      heap_deletehead (wd, sizeof (*wd), &num, hc);
+      idx = KIND (word);
+      keep[idx].wd[keep[idx].num_wds++] = word;
+    }
+
+  for (i = 0; i < num; i++)
+    {
+      WordData *word = wd[i];
+      int idx = KIND (word);
+      discard[idx].wd[discard[idx].num_wds++] = word;
+    }
+  SortAndCount_DictData (&keep[0]);
+  SortAndCount_DictData (&keep[1]);
+  SortAndCount_DictData (&discard[0]);
+  SortAndCount_DictData (&discard[1]);
+
+  assert (keep[0].num_wds + discard[0].num_wds == Num[0]);
+  assert (keep[1].num_wds + discard[1].num_wds == Num[1]);
+
+}
+
+
+
+static int 
+BigSaving (void *a, void *b)
+{
+  WordData *aa = *(WordData **) a;
+  WordData *bb = *(WordData **) b;
+  return (aa->saving > bb->saving) ? -1 : (aa->saving < bb->saving);
+}
+
+static int 
+SmallSaving (void *a, void *b)
+{
+  WordData *aa = *(WordData **) a;
+  WordData *bb = *(WordData **) b;
+  return (aa->saving < bb->saving) ? -1 : (aa->saving > bb->saving);
+}
+
+
+static void 
+CalcCharCounts (WordData ** wd, int num,
+		char *char_lens[2], char *len_lens[2],
+		u_long escape[2])
+{
+  long char_freqs[2][256];
+  long len_freqs[2][16];
+  huff_data hd;
+  int i;
+  escape[0] = 0;
+  escape[1] = 0;
+  bzero ((char *) char_freqs, sizeof (char_freqs));
+  bzero ((char *) len_freqs, sizeof (len_freqs));
+  for (i = 0; i < num; i++, wd++)
+    {
+      u_long freq = (*wd)->freq;
+      u_char *buf = (*wd)->word;
+      int len = *buf++;
+      int idx = KIND (*wd);
+      len_freqs[idx][len] += freq;
+      escape[idx] += freq;
+      for (; len; len--, buf++)
+	char_freqs[idx][(u_long) (*buf)] += freq;
+    }
+  Generate_Huffman_Data (256, char_freqs[0], &hd, NULL);
+  char_lens[0] = hd.clens;
+  Generate_Huffman_Data (256, char_freqs[1], &hd, NULL);
+  char_lens[1] = hd.clens;
+
+  Generate_Huffman_Data (16, len_freqs[0], &hd, NULL);
+  len_lens[0] = hd.clens;
+  Generate_Huffman_Data (16, len_freqs[1], &hd, NULL);
+  len_lens[1] = hd.clens;
+}
+
+
+
+
+
+
+
+
+
+void 
+CalcBitCost (WordData ** discard_word, int discard_num,
+	     WordData ** keep_word, int keep_num, double freqs_trans[2],
+	     u_long escape[2], int num_trans)
+{
+  char *char_lens[2];
+  char *len_lens[2];
+  double esc[2];
+  int j;
+  CalcCharCounts (discard_word, discard_num, char_lens, len_lens, escape);
+  esc[0] = -log2 (escape[0] / (freqs_trans[0] + escape[0]));
+  esc[1] = -log2 (escape[1] / (freqs_trans[1] + escape[1]));
+  for (j = 0; j < discard_num; j++, discard_word++)
+    {
+      float cbc, wbc;
+      u_char *buf = (*discard_word)->word;
+      int len = *buf++;
+      u_long freq = (*discard_word)->freq;
+      int idx = KIND (*discard_word);
+
+      cbc = len_lens[idx][len];
+      for (; len; len--, buf++)
+	cbc += char_lens[idx][(u_long) (*buf)];
+
+      (*discard_word)->char_bit_cost = (cbc + esc[idx]) * freq;
+
+      wbc = -log2 (freq / (freqs_trans[idx] + escape[idx])) * freq;
+
+      (*discard_word)->saving = ((*discard_word)->char_bit_cost - wbc) /
+	(sizeof (char *) + (*discard_word)->word[0]);
+
+      (*discard_word)->num_trans = num_trans;
+    }
+  for (j = 0; j < keep_num; j++, keep_word++)
+    {
+      float cbc, wbc;
+      u_char *buf = (*keep_word)->word;
+      int len = *buf++;
+      u_long freq = (*keep_word)->freq;
+      int idx = KIND (*keep_word);
+
+      cbc = len_lens[idx][len];
+      for (; len; len--, buf++)
+	cbc += char_lens[idx][(u_long) (*buf)];
+
+      (*keep_word)->char_bit_cost = (cbc + esc[idx]) * freq;
+
+      wbc = -log2 (freq / (freqs_trans[idx] + escape[idx])) * freq;
+
+      (*keep_word)->saving = ((*keep_word)->char_bit_cost - wbc) /
+	(sizeof (char *) + (*keep_word)->word[0]);
+
+      (*keep_word)->num_trans = num_trans;
+    }
+  Xfree (char_lens[0]);
+  Xfree (char_lens[1]);
+  Xfree (len_lens[0]);
+  Xfree (len_lens[1]);
+}
+
+
+
+static void 
+Method3 (int k)
+{
+  int i, keep_num, discard_num, mem, num_trans, recalc_reqd;
+  int keep_to_discard = 0;
+  int discard_to_keep = 0;
+  int recalcs = 0;
+  double freqs_trans[2], total;
+  u_long escape[2];
+  WordData **keep_heap, **discard_heap;
+
+  freqs_trans[0] = freqs_trans[1] = 0;
+  num_trans = 0;
+
+  discard_num = Num[0] + Num[1];
+  discard_heap = Xmalloc (discard_num * sizeof (WordData *));
+
+  keep_num = 0;
+  keep_heap = Xmalloc (discard_num * sizeof (WordData *));
+
+
+  for (i = 0; i < Num[0]; i++)
+    discard_heap[i] = Words[0] + i;
+  for (i = 0; i < Num[1]; i++)
+    discard_heap[i + Num[0]] = Words[1] + i;
+
+  heap_build (discard_heap, sizeof (*discard_heap), discard_num, DecFreqIncWL);
+
+  mem = 0;
+  while (k > mem && discard_num)
+    {
+      WordData *word = discard_heap[0];
+      mem += sizeof (u_char *) + word->word[0];
+      heap_deletehead (discard_heap, sizeof (word), &discard_num, DecFreqIncWL);
+      keep_heap[keep_num++] = word;
+      freqs_trans[KIND (word)] += word->freq;
+      num_trans++;
+    }
+
+  CalcBitCost (discard_heap, discard_num, keep_heap, keep_num,
+	       freqs_trans, escape, num_trans);
+  heap_build (discard_heap, sizeof (*discard_heap), discard_num, BigSaving);
+  heap_build (keep_heap, sizeof (*keep_heap), keep_num, SmallSaving);
+
+  total = 0;
+  recalc_reqd = 0;
+  while (keep_num && discard_num)
+    {
+      if ((keep_num && keep_heap[0]->num_trans < num_trans) ||
+	  (discard_num && discard_heap[0]->num_trans < num_trans))
+	{
+	  if (keep_num && keep_heap[0]->num_trans < num_trans)
+	    {
+	      WordData *word = keep_heap[0];
+	      int idx = KIND (word);
+	      float wbc;
+#ifdef DEBUG1
+	      fprintf (stderr, "KEEP \"%s\" %.2f ->", word2str (word->word),
+		       word->saving);
+#endif
+	      wbc = -log2 (word->freq / (freqs_trans[idx] + escape[idx])) *
+		word->freq;
+	      word->saving = (word->char_bit_cost - wbc) /
+		(sizeof (char *) + word->word[0]);
+#ifdef DEBUG1
+	      fprintf (stderr, " %.2f\n", word->saving);
+#endif
+	      word->num_trans = num_trans;
+	      heap_changedhead (keep_heap, sizeof (word), keep_num, SmallSaving);
+	    }
+
+	  if (discard_num && discard_heap[0]->num_trans < num_trans)
+	    {
+	      WordData *word = discard_heap[0];
+	      int idx = KIND (word);
+	      float wbc;
+#ifdef DEBUG1
+	      fprintf (stderr, "DISCARD \"%s\" %.2f ->", word2str (word->word),
+		       word->saving);
+#endif
+	      wbc = -log2 (word->freq / (freqs_trans[idx] + escape[idx])) *
+		word->freq;
+	      word->saving = (word->char_bit_cost - wbc) /
+		(sizeof (char *) + word->word[0]);
+#ifdef DEBUG1
+	      fprintf (stderr, " %.2f\n", word->saving);
+#endif
+	      word->num_trans = num_trans;
+	      heap_changedhead (discard_heap, sizeof (word),
+				discard_num, BigSaving);
+	    }
+	}
+      else if (keep_heap[0]->saving < discard_heap[0]->saving)
+	{
+	  assert (keep_num && discard_num);
+	  if (keep_num && mem + sizeof (char *) + discard_heap[0]->word[0] > k)
+	    {
+	      /* Transfer the word at the top of the keep heap to the
+	         discard heap. */
+	      WordData *word = keep_heap[0];
+	      int idx = KIND (word);
+	      heap_deletehead (keep_heap, sizeof (word), &keep_num, SmallSaving);
+	      discard_heap[discard_num] = word;
+	      heap_additem (discard_heap, sizeof (word), &discard_num,
+			    BigSaving);
+	      freqs_trans[idx] -= word->freq;
+	      mem -= sizeof (u_char *) + word->word[0];
+	      num_trans++;
+	      total += word->saving;
+	      keep_to_discard++;
+#ifdef DEBUG
+	      fprintf (stderr,
+		       "KEEP -> DISCARD %8d :%8d :%8.0f :%8.0f : \"%s\"\n",
+		       mem, word->freq, word->char_bit_cost,
+		       word->saving, word2str (word->word));
+#endif
+	    }
+	  else
+	    {
+	      /* Transfer the word at the top of the discard heap to the
+	         keep heap. */
+	      WordData *word = discard_heap[0];
+	      int idx = KIND (word);
+	      heap_deletehead (discard_heap, sizeof (word),
+			       &discard_num, BigSaving);
+	      keep_heap[keep_num] = word;
+	      heap_additem (keep_heap, sizeof (word), &keep_num, SmallSaving);
+	      freqs_trans[idx] += word->freq;
+	      mem += sizeof (u_char *) + word->word[0];
+	      num_trans++;
+	      total += word->saving;
+	      discard_to_keep++;
+#ifdef DEBUG
+	      fprintf (stderr,
+		       "DISCARD -> KEEP %8d :%8d :%8.0f :%8.0f : \"%s\"\n",
+		       mem, word->freq, word->char_bit_cost,
+		       word->saving, word2str (word->word));
+#endif
+	    }
+
+	  recalc_reqd = 1;
+
+	}
+      else
+	{
+	  if (recalc_reqd == 0)
+	    break;
+#ifdef DEBUG1
+	  fprintf (stderr, "--------------\n");
+#endif
+	  if (recalcs == MAX_RECALCULATIONS)
+	    break;
+	  CalcBitCost (discard_heap, discard_num, keep_heap, keep_num,
+		       freqs_trans, escape, num_trans);
+	  heap_build (discard_heap, sizeof (*discard_heap),
+		      discard_num, BigSaving);
+	  heap_build (keep_heap, sizeof (keep_heap), keep_num, SmallSaving);
+	  recalc_reqd = 0;
+	  recalcs++;
+	}
+    }
+
+  Alloc_keep_discard ();
+
+  for (i = 0; i < discard_num; i++)
+    {
+      WordData *word = discard_heap[i];
+      int idx = KIND (word);
+      assert (IsWord (word) || IsNonWord (word));
+      discard[idx].wd[discard[idx].num_wds++] = word;
+    }
+  for (i = 0; i < keep_num; i++)
+    {
+      WordData *word = keep_heap[i];
+      int idx = KIND (word);
+      assert (IsWord (word) || IsNonWord (word));
+      keep[idx].wd[keep[idx].num_wds++] = word;
+    }
+  SortAndCount_DictData (&keep[0]);
+  SortAndCount_DictData (&keep[1]);
+  SortAndCount_DictData (&discard[0]);
+  SortAndCount_DictData (&discard[1]);
+
+  assert (keep[0].num_wds + discard[0].num_wds == Num[0]);
+  assert (keep[1].num_wds + discard[1].num_wds == Num[1]);
+  Message ("Keep -> Discard        : %8d", keep_to_discard);
+  Message ("Discard -> Keep        : %8d", discard_to_keep);
+  Message ("Huffman Recalculations : %8d", recalcs);
+  if (recalcs == MAX_RECALCULATIONS)
+    Message ("WARNING: The number of recalculations == %d", MAX_RECALCULATIONS);
+}
+
+
+
+
+
+
+
+
+
+
+/****************************************************************************
+ *****									*****
+ *****			Dictionary saving code				*****
+ *****									*****
+ ****************************************************************************/
+
+
+
+static void 
+Write_cdh (FILE * f, compression_dict_header * cdh)
+{
+  /* [RPAP - Jan 97: Endian Ordering] */
+  int i;
+  compression_dict_header tmp = *cdh;
+  HTONUL(tmp.dict_type);
+  HTONUL(tmp.novel_method);
+  for (i = 0; i < TEXT_PARAMS; i++)
+    HTONUL(tmp.params[i]);
+  HTONUL(tmp.num_words[0]);
+  HTONUL(tmp.num_words[1]);
+  HTONUL(tmp.num_word_chars[0]);
+  HTONUL(tmp.num_word_chars[1]);
+  HTONUL(tmp.lookback);
+
+  fwrite (&tmp, sizeof (tmp), 1, f);
+}
+
+
+static void 
+Write_words (FILE * f, DictData * dd)
+{
+  int i;
+  u_char *curr, *prev = NULL;
+  for (i = 0; i < dd->num_wds; i++)
+    {
+      int len;
+      curr = dd->wd[i]->word;
+      if (prev)
+	/* look for prefix match with prev string */
+	len = prefixlen (prev, curr);
+      else
+	len = 0;
+      fputc ((len << 4) + (curr[0] - len), f);
+      fwrite (curr + len + 1, sizeof (u_char), curr[0] - len, f);
+      prev = curr;
+    }
+
+}
+
+
+static int 
+Uncompressed_size (DictData * dd)
+{
+  int i, us;
+  for (us = i = 0; i < dd->num_wds; i++)
+    us += dd->wd[i]->word[0];
+  return us;
+}
+
+
+static u_long 
+Write_data (FILE * f, DictData * dd, int lookback)
+{
+  u_long mem_reqd;
+  huff_data *hd;
+  int i; 
+  u_long us = dd->num_wds;
+  long *freqs;
+  u_long huff_words_size[MAX_HUFFCODE_LEN + 1];
+  u_long lencounts[MAX_HUFFCODE_LEN + 1];
+  u_char *lastword[MAX_HUFFCODE_LEN + 1];
+
+  if (!(freqs = Xmalloc ((dd->num_wds) * sizeof (*freqs))))
+    FatalError (1, "Unable to allocate memory for freqs");
+
+  for (i = 0; i < dd->num_wds; i++)
+    {
+      freqs[i] = dd->wd[i]->freq;
+      us += dd->wd[i]->word[0];
+    }
+
+  if (!(hd = Generate_Huffman_Data (dd->num_wds, freqs, NULL, NULL)))
+    FatalError (1, "Unable to allocate memory for huffman data");
+
+  Xfree (freqs), freqs = NULL;
+
+  if (Write_Huffman_Data (f, hd) == -1)
+    FatalError (1, "Unable to write huffman data");
+
+  HTONUL(us);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite (&us, sizeof (us), 1, f);
+  NTOHUL(us);  /* [RPAP - Jan 97: Endian Ordering] */
+
+
+/* Calculate the amount of memory that will be required to store the text for
+   each different huffman code len. Every 1<<lookback words for each different
+   codelen length will not be prefixed by previous strings. */
+
+
+  bzero ((char *) &huff_words_size, sizeof (huff_words_size));
+  bzero ((char *) &lencounts, sizeof (lencounts));
+
+  mem_reqd = 0;
+
+  for (i = 0; i < dd->num_wds; i++)
+    {
+      int codelen = hd->clens[i];
+      u_char *word = dd->wd[i]->word;
+
+      if (!codelen)
+	FatalError (1, "The length of a code for a word was zero");
+
+      huff_words_size[codelen] += word[0] + 1;
+      mem_reqd += word[0] + (lookback != 0);
+#if 0
+      if ((lencounts[codelen] & ((1 << lookback) - 1)) == 0)
+	lastword[codelen] = word;
+      else
+	huff_words_size[codelen] -= prefixlen (lastword[codelen], word);
+#else
+      if ((lencounts[codelen] & ((1 << lookback) - 1)) != 0)
+	{
+	  int save = prefixlen (lastword[codelen], word);
+	  mem_reqd -= save;
+	  huff_words_size[codelen] -= save;
+	}
+      else
+	{
+	  mem_reqd += sizeof (u_char *);
+	}
+      lastword[codelen] = word;
+#endif
+      lencounts[codelen]++;
+    }
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = hd->mincodelen; i < hd->maxcodelen + 1; i++)
+    HTONUL(huff_words_size[i]);
+
+  fwrite (huff_words_size + hd->mincodelen, sizeof (*huff_words_size),
+	  hd->maxcodelen - hd->mincodelen + 1, f);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = hd->mincodelen; i < hd->maxcodelen + 1; i++)
+    NTOHUL(huff_words_size[i]);
+
+  Write_words (f, dd);
+
+  Xfree (hd->clens);
+  Xfree (hd);
+
+  return mem_reqd;
+}
+
+
+
+static void 
+Write_charfreqs (FILE * f, DictData * dd, int words,
+		 int zero_freq_permitted)
+{
+  int j;
+  long freqs[256];
+  WordData **wd = dd->wd;
+  huff_data *hd;
+
+  bzero ((char *) freqs, sizeof (freqs));
+
+  for (j = 0; j < dd->num_wds; j++, wd++)
+    {
+      u_char *buf = (*wd)->word;
+      int len = *buf++;
+      for (; len; len--, buf++)
+	freqs[(u_long) (*buf)] += (*wd)->freq;
+    }
+
+  if (!zero_freq_permitted)
+    for (j = 0; j < 256; j++)
+      if (!freqs[j] && PESINAWORD (j) == words)
+	freqs[j] = 1;
+
+  if (!(hd = Generate_Huffman_Data (256, freqs, NULL, NULL)))
+    FatalError (1, "Unable to allocate memory for huffman data");
+
+  if (Write_Huffman_Data (f, hd) == -1)
+    FatalError (1, "Unable to write huffman data");
+
+  Xfree (hd->clens);
+  Xfree (hd);
+}
+
+
+
+
+static void 
+Write_wordlens (FILE * f, DictData * dd, int zero_freq_permitted)
+{
+  int j;
+  long freqs[16];
+  WordData **wd = dd->wd;
+  huff_data *hd;
+
+  bzero ((char *) freqs, sizeof (freqs));
+
+  for (j = 0; j < dd->num_wds; j++, wd++)
+    freqs[(*wd)->word[0]] += (*wd)->freq;
+
+  if (!zero_freq_permitted)
+    for (j = 0; j < 16; j++)
+      if (!freqs[j])
+	freqs[j] = 1;
+
+  if (!(hd = Generate_Huffman_Data (16, freqs, NULL, NULL)))
+    FatalError (1, "Unable to allocate memory for huffman data");
+
+  if (Write_Huffman_Data (f, hd) == -1)
+    FatalError (1, "Unable to write huffman data");
+
+
+  Xfree (hd->clens);
+  Xfree (hd);
+}
+
+
+
+static u_long 
+WriteOutWords (char *file_name, u_long type, int lookback)
+{
+  FILE *f;
+  int i;
+  u_long mem_reqd = 0;
+
+  compression_dict_header cdh;
+  f = create_file (file_name, TEXT_DICT_SUFFIX, "w+b",
+		   MAGIC_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  bzero((char *) &cdh, sizeof(cdh));
+
+  cdh.dict_type = type;
+  cdh.novel_method = (type == MG_SEED_DICTIONARY) ? novel_method :
+    MG_NOVEL_HUFFMAN_CHARS;
+
+  cdh.num_words[1] = keep[1].num_wds;
+  cdh.num_words[0] = keep[0].num_wds;
+  cdh.num_word_chars[1] = Uncompressed_size (&keep[1]);
+  cdh.num_word_chars[0] = Uncompressed_size (&keep[0]);
+  cdh.lookback = lookback;
+
+  Write_cdh (f, &cdh);
+
+  for (i = 0; i < 2; i++)
+    switch (type)
+      {
+      case MG_COMPLETE_DICTIONARY:
+	{
+	  mem_reqd += Write_data (f, &keep[i], lookback);
+	}
+	break;
+      case MG_PARTIAL_DICTIONARY:
+	{
+	  if (keep[i].num_wds)
+	    {
+	      int j;
+	      WordData esc;
+	      esc.freq = 0;
+	      esc.word = (u_char *) "";
+	      keep[i].wd[keep[i].num_wds++] = &esc;
+	      for (j = 0; j < discard[i].num_wds; j++)
+		esc.freq += discard[i].wd[j]->freq;
+	      if (!esc.freq)
+		esc.freq++;
+	      mem_reqd += Write_data (f, &keep[i], lookback);
+	    }
+	  Write_charfreqs (f, &discard[i], i, 1);
+	  Write_wordlens (f, &discard[i], 1);
+	}
+	break;
+      case MG_SEED_DICTIONARY:
+	{
+	  if (keep[i].num_wds)
+	    {
+	      int j;
+	      WordData esc;
+	      esc.freq = 0;
+	      esc.word = (u_char *) "";
+	      keep[i].wd[keep[i].num_wds++] = &esc;
+	      for (j = 0; j < all[i].num_wds; j++)
+		if (all[i].wd[j]->freq == 1)
+		  esc.freq++;
+	      if (!esc.freq)
+		esc.freq++;
+	      mem_reqd += Write_data (f, &keep[i], lookback);
+	    }
+	  switch (novel_method)
+	    {
+	    case MG_NOVEL_HUFFMAN_CHARS:
+	      Write_charfreqs (f, &all[i], i, 0);
+	      Write_wordlens (f, &all[i], 0);
+	      break;
+	    case MG_NOVEL_BINARY:
+	      break;
+	    case MG_NOVEL_DELTA:
+	      break;
+	    case MG_NOVEL_HYBRID:
+	      break;
+	    case MG_NOVEL_HYBRID_MTF:
+	      break;
+	    default:
+	      FatalError (1, "Bad novel method");
+	    }
+	}
+	break;
+      }
+  fclose (f);
+  return mem_reqd;
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/mg_decompress_text.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/mg_decompress_text.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/mg_decompress_text.1	(revision 699)
@@ -0,0 +1,78 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_decompress_text 1 \*(Dt CITRI
+.SH NAME
+mg_decompress_text \- decompresses the compressed text of a collection
+.SH SYNOPSIS
+.B mg_decompress_text
+[
+.B \-h
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_decompress_text
+decompresses the compressed text of a collection, inserts paragraph and
+document markers, writing the output to
+.IR stdout ".  "
+This program was created
+to be used as a type of
+.BR mg_get (1).
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 25
+.B *.text
+Compressed documents.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.text.dict.aux
+Auxiliary compression dictionary.
+.TP
+.B *.text.dict.fast
+Fast loading compression dictionary.
+.TP
+.B *.invf.paragraph
+Paragraph counts for documents.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_passes (1).
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/mg_decompress_text.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/mg_decompress_text.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/mg_decompress_text.c	(revision 699)
@@ -0,0 +1,167 @@
+#include "sysfuncs.h"
+
+#include "filestats.h"
+#include "netorder.h"
+#include "memlib.h"
+#include "messages.h"
+#include "huffman.h"
+#include "bitio_m_mem.h"
+#include "bitio_m.h"
+#include "bitio_stdio.h"
+#include "huffman_stdio.h"
+
+#include "mg_files.h"
+#include "mg.h"
+#include "backend.h"  /* compressed_dict struct */
+#include "text_get.h"
+#include "words.h"
+
+#define EOD         '\002'
+#define EOP         '\003'
+
+/* These are defined in mgquery, and they need to be here for backend.o */
+/* Messy I know but this was meant to be a quick hack :-) */
+#if defined(PARADOCNUM) || defined(NZDL)
+int *Paragraph = NULL;
+int Documents = 0;
+#endif
+
+int
+main (int argc, char **argv)
+{
+  FILE *text, *text_idx, *para;
+  File *text_comp_dict = NULL, *text_aux_dict = NULL, *text_fast_comp_dict = NULL;
+
+  int ch;
+  unsigned long start, finish, len;
+  u_long total_chars = 0;
+  u_char *c_buffer = 0;
+  int buf_len = -1;
+  u_char *uc_buffer = 0;
+  int ULen;
+  compression_dict *cd = NULL;
+  compressed_text_header cth;
+  u_long doc;
+  int count;
+  char *filename = "";
+  char path[512];
+  opterr = 0;
+  msg_prefix = argv[0];
+
+  while ((ch = getopt (argc, argv, "f:d:h")) != -1)
+    switch (ch)
+      {
+      case 'f':		/* input file */
+	filename = optarg;
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-h] [-d directory] -f name\n",
+		 argv[0]);
+	exit (1);
+      }
+
+  /* Open files */
+  text = open_file (filename, TEXT_SUFFIX, "rb", MAGIC_TEXT, MG_ABORT);
+  fread ((char *)&cth, sizeof (cth), 1, text);
+
+  text_idx = open_file (filename, TEXT_IDX_SUFFIX, "rb", MAGIC_TEXI, MG_ABORT);
+  fread ((char *)&cth, sizeof (cth), 1, text_idx);
+
+  NTOHUL(cth.num_of_docs);
+  NTOHD(cth.num_of_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(cth.num_of_words);
+  NTOHUL(cth.length_of_longest_doc);
+  NTOHD(cth.ratio);
+
+  para = open_file (filename, INVF_PARAGRAPH_SUFFIX, "rb", MAGIC_PARAGRAPH, MG_CONTINUE);
+
+  sprintf (path, FILE_NAME_FORMAT, get_basepath(), filename, TEXT_DICT_FAST_SUFFIX);  /* [RPAP - Feb 97: WIN32 Port] */
+  if (!(text_fast_comp_dict = Fopen (path, "rb", MAGIC_FAST_DICT)))
+    {
+      sprintf (path, FILE_NAME_FORMAT, get_basepath(), filename, TEXT_DICT_SUFFIX);  /* [RPAP - Feb 97: WIN32 Port] */
+      text_comp_dict = Fopen (path, "rb", MAGIC_DICT);
+      if (!text_comp_dict)
+	FatalError (1, "Could not open compressed dictionary");
+      sprintf (path, FILE_NAME_FORMAT, get_basepath(), filename, TEXT_DICT_AUX_SUFFIX);  /* [RPAP - Feb 97: WIN32 Port] */
+      text_aux_dict = Fopen (path, "rb", MAGIC_AUX_DICT);
+    }
+
+  cd = LoadCompDict (text_comp_dict, text_aux_dict, text_fast_comp_dict);
+
+
+  /* Uncompress text */
+
+  doc = 1;
+  fread ((char *)&start, sizeof (start), 1, text_idx);
+  if (para)
+    fread ((char *)&count, sizeof (count), 1, para);
+  else
+    count = 1;
+
+  for (;;)
+    {
+      fread ((char *) &finish, sizeof (finish), 1, text_idx);
+      len = finish - start;
+      
+      if ((int) len > buf_len)
+	{
+	  if (c_buffer)
+	    {
+	      Xfree (c_buffer);
+	      Xfree (uc_buffer);
+	    }
+	  if (!(c_buffer = Xmalloc (len)))
+	    FatalError (1, "Cannot allocate memory for compressed buffer");
+	  if (!(uc_buffer = Xmalloc ((int) (cth.ratio * 1.01 * len) + 100)))
+	    FatalError (1, "Cannot allocate memory for uncompressed buffer");
+	  buf_len = len;
+	}
+
+      fread (c_buffer, 1, len, text);
+      DecodeText (cd, c_buffer, len, uc_buffer, &ULen);
+      fwrite (uc_buffer, ULen, sizeof (u_char), stdout);
+      fflush (stdout);
+
+      total_chars += ULen;
+
+      if (++doc > cth.num_of_docs)
+	{
+	  if (para)
+	    fputc (EOP, stdout);
+	  fputc (EOD, stdout);
+	  fflush (stdout);
+	  break;
+	}
+
+      if (!--count)
+	{
+	  /* End of document */
+	  fputc (EOD, stdout);
+	  if (para)
+	    fread ((char *)&count, sizeof (count), 1, para);
+	  else
+	    count = 1;
+	}
+      else
+	fputc (EOP, stdout);
+      fflush (stdout);
+      start = finish;
+    }
+
+  /* Close files */
+
+  fclose (text);
+  fclose (text_idx);
+  if (para)
+    fclose (para);
+
+#if 0
+  printf ("\n#Total chars output = %lu\n", total_chars);
+#endif
+
+  return 0;
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/mg_errors.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/mg_errors.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/mg_errors.c	(revision 699)
@@ -0,0 +1,73 @@
+/**************************************************************************
+ *
+ * mg_errors.c -- Error related stuff for mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "memlib.h"
+
+#include "mg_errors.h"
+
+
+
+int mg_errno = MG_NOERROR;
+
+char *mg_errorstrs[] =
+{
+  "No error",
+  "Out of memory",
+  "File \"%s\" not found",
+  "Bad magic number in \"%s\"",
+  "Error reading \"%s\"",
+  "MG_BUFTOOSMALL",
+  "Files required for level 2 and 3 inversion are missing"};
+
+
+/* [RJM 07/98: Memory Leak] -- rest of file */
+
+static char null_data[] = "";
+char *mg_error_data = null_data;
+
+
+void MgErrorData (char *s)
+{
+  /* free the current error string, unless it is the null string */
+  if ((mg_error_data != NULL) && (mg_error_data != null_data)) {
+    Xfree (mg_error_data);
+    mg_error_data = null_data;
+  }
+
+  /* make a copy of the string */
+  mg_error_data = Xstrdup (s);
+
+  /* if the alloc failed set the error to the null string */
+  if (!mg_error_data) mg_error_data = null_data;
+}
+
+
+void MgErrorDeinit (void)
+{
+  /* free the current error string, unless it is the null string */
+  if ((mg_error_data != NULL) && (mg_error_data != null_data)) {
+    Xfree (mg_error_data);
+    mg_error_data = null_data;
+  }
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/mg_errors.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/mg_errors.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/mg_errors.h	(revision 699)
@@ -0,0 +1,39 @@
+/**************************************************************************
+ *
+ * mg_errors.h -- Error related stuff for mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+extern int mg_errno;
+extern char *mg_error_data;
+extern char *mg_errorstrs[];
+
+
+#define MG_NOERROR 0
+#define MG_NOMEM 1
+#define MG_NOFILE 2
+#define MG_BADMAGIC 3
+#define MG_READERR 4
+#define MG_BUFTOOSMALL 5
+#define MG_INVERSION 6
+
+void MgErrorData (char *s);
+void MgErrorDeinit (void);
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/mg_fast_comp_dict.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/mg_fast_comp_dict.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/mg_fast_comp_dict.1	(revision 699)
@@ -0,0 +1,86 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_fast_comp_dict 1 \*(Dt CITRI
+.SH NAME
+mg_fast_comp_dict \- build a fast loading compression dictionary
+.SH SYNOPSIS
+.B mg_fast_comp_dict
+[
+.B \-h
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_fast_comp_dict
+builds a fast-loading compression dictionary from the standard
+compression dictionary.  The new dictionary has the property that
+.BR mgquery (1)
+can load it in very quickly, making the startup time for
+.BR mgquery (1)
+much shorter.  The new dictionary is larger on disk than the original,
+but they are no different when it comes to memory usage.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.text.dict
+Compression dictionary.
+.TP
+.B *.text.dict.fast
+Fast loading compression dictionary.
+.TP
+.B *.text.dict.aux
+Auxiliary compression dictionary.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/mg_fast_comp_dict.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/mg_fast_comp_dict.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/mg_fast_comp_dict.c	(revision 699)
@@ -0,0 +1,798 @@
+/**************************************************************************
+ *
+ * mg_fast_comp_dict.c -- Program to generate a fast compression dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:18:06  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:44  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:34:57  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:55  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:47  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+#include "filestats.h"
+#include "huffman.h"
+#include "timing.h"
+#include "messages.h"
+#include "memlib.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "local_strings.h"
+#include "mg.h"
+#include "text.h"
+#include "invf.h"
+#include "lists.h"
+#include "backend.h"
+#include "mg_files.h"
+#include "locallib.h"
+#include "words.h"
+
+
+
+#define ALIGN_SIZE(p,t) (((p) + (sizeof(t)-1)) & (~(sizeof(t)-1)))
+
+#define WORDNO(p, base) ((((char*)(p))-((char*)(base)))/sizeof(u_char*))
+#define FIXUP(p) (fixup[WORDNO(p,buffer)/8] |= (1<<(WORDNO(p, buffer) & 7)))
+
+#define IS_FIXUP(p) ((fixup[WORDNO(p,buffer)/8] & (1<<(WORDNO(p, buffer) & 7))) != 0)
+
+#define FIXUP_VALS(vals) do {						\
+	int i;								\
+	for (i=0; i < MAX_HUFFCODE_LEN+1; i++)				\
+	  FIXUP(&vals[i]);						\
+      } while(0)
+
+
+
+static u_long mem_for_comp_dict (char *filename);
+static void load_comp_dict (char *filename);
+static void save_fast_dict (char *filename);
+static void unfixup_buffer (void);
+
+
+static void *buffer;
+static void *cur;
+static u_char *fixup;
+static u_long mem, fixup_mem;
+
+int main (int argc, char **argv)
+{
+  ProgTime StartTime;
+  int ch;
+  char *filename = "";
+  opterr = 0;
+  msg_prefix = argv[0];
+  while ((ch = getopt (argc, argv, "f:d:h")) != -1)
+    switch (ch)
+      {
+      case 'f':		/* input file */
+	filename = optarg;
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-f input_file] [-d data directory] [-h]\n",
+		 argv[0]);
+	exit (1);
+      }
+  GetTime (&StartTime);
+
+  mem = mem_for_comp_dict (filename);
+
+  fixup_mem = (ALIGN_SIZE (mem, u_char *) / sizeof (u_char *) + 7) / 8;
+
+  cur = buffer = Xmalloc (mem);
+  bzero (buffer, mem);
+  fixup = Xmalloc (fixup_mem);
+  bzero (fixup, fixup_mem);
+
+  Message ("Estimated memory for fast_dict %u", mem);
+  Message ("Estimated memory for fixups %u", fixup_mem);
+
+  load_comp_dict (filename);
+
+  Message ("Actual memory for fast_dict %u", (char *) cur - (char *) buffer);
+
+  if ((u_long) cur > (u_long) buffer + mem)
+    FatalError (1, "The buffer was not big enough for the dictionary");
+
+  {
+    /* [RPAP - Jan 97: Endian Ordering] */
+    compression_dict *cd = (compression_dict *) buffer;
+    int i, which;
+
+    /* cfh */
+    for (which = 0; which <= 1; which++)
+      {
+	int j;
+	
+	HTONSI(cd->cfh[which]->hd.num_codes);
+	HTONSI(cd->cfh[which]->hd.mincodelen);
+	HTONSI(cd->cfh[which]->hd.maxcodelen);
+	for (j = 0; j < MAX_HUFFCODE_LEN + 1; j++)
+	  {
+	    HTONSI(cd->cfh[which]->hd.lencount[j]);
+	    HTONUL(cd->cfh[which]->hd.min_code[j]);
+	  }
+	HTONUL(cd->cfh[which]->uncompressed_size);
+	for (j = 0; j < MAX_HUFFCODE_LEN + 1; j++)
+	  HTONUL(cd->cfh[which]->huff_words_size[j]);
+      }
+    HTONUL(cd->MemForCompDict);
+    /* ad */
+    if (cd->cdh.novel_method == MG_NOVEL_BINARY ||
+	cd->cdh.novel_method == MG_NOVEL_DELTA ||
+	cd->cdh.novel_method == MG_NOVEL_HYBRID ||
+	cd->cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+      for (which = 0; which <= 1; which++)
+	{
+	  int j;
+	  
+	  HTONUL(cd->ad->afh[which].num_frags);
+	  HTONUL(cd->ad->afh[which].mem_for_frags);
+	  for (j = 0; j < 33; j++)
+	    {
+	      HTONSI(cd->ad->blk_start[which][j]);
+	      HTONSI(cd->ad->blk_end[which][j]);
+	    }
+	}
+    /* cdh */
+    HTONUL(cd->cdh.dict_type);
+    HTONUL(cd->cdh.novel_method);
+    for (i = 0; i < TEXT_PARAMS; i++)
+      HTONUL(cd->cdh.params[which]);
+    HTONUL(cd->cdh.num_words[0]);
+    HTONUL(cd->cdh.num_words[1]);
+    HTONUL(cd->cdh.num_word_chars[0]);
+    HTONUL(cd->cdh.num_word_chars[1]);
+    HTONUL(cd->cdh.lookback);
+    HTONSI(cd->fast_loaded);
+  }
+
+  unfixup_buffer ();
+
+  save_fast_dict (filename);
+
+  Message ("%s", ElapsedTime (&StartTime, NULL));
+  return 0;
+}
+
+
+
+static void 
+unfixup_buffer ()
+{
+  u_long *p;
+  for (p = buffer; (u_long) p < (u_long) cur; p++)
+    {
+      if (IS_FIXUP (p))
+	*p = *p - (u_long) buffer;
+    }
+}
+
+
+
+
+static u_long 
+mem_for_aux_dict (compression_dict_header * cdh, char *filename)
+{
+  int i;
+  u_long mem = sizeof (auxiliary_dict);
+  FILE *aux;
+
+  aux = open_file (filename, TEXT_DICT_AUX_SUFFIX, "rb",
+		   MAGIC_AUX_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  for (i = 0; i <= 1; i++)
+    {
+      aux_frags_header afh;
+      fread (&afh, sizeof (afh), 1, aux);
+      NTOHUL(afh.num_frags);  /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(afh.mem_for_frags);  /* [RPAP - Jan 97: Endian Ordering] */
+      mem += afh.num_frags * sizeof (u_char *);
+      mem = ALIGN_SIZE (mem + afh.mem_for_frags, u_char *);
+      fseek (aux, afh.mem_for_frags, SEEK_CUR);
+    }
+  fclose (aux);
+
+  return mem;
+}
+
+
+
+static u_long 
+mem_for_words (FILE * dict, compression_dict_header * cdh,
+	       comp_frags_header * cfh)
+{
+  u_long mem = 0;
+  u_long i, lookback;
+  int ptrs_reqd = 0;
+  int mem_reqd = 0;
+
+  lookback = cdh->lookback;
+
+  for (i = cfh->hd.mincodelen; i <= cfh->hd.maxcodelen; i++)
+    {
+      ptrs_reqd += (cfh->hd.lencount[i] + ((1 << lookback) - 1)) >> lookback;
+      mem_reqd += cfh->huff_words_size[i];
+    }
+
+  mem += ptrs_reqd * sizeof (u_char *);
+  mem += (MAX_HUFFCODE_LEN + 1) * sizeof (u_char **);
+  mem += mem_reqd;
+
+  for (i = 0; i < cfh->hd.num_codes; i++)
+    {
+      register int val;
+      for (val = getc (dict) & 0xf; val; val--)
+	getc (dict);
+    }
+  return ALIGN_SIZE (mem, u_char *);
+}
+
+
+
+
+static u_long 
+mem_for_comp_dict (char *filename)
+{
+  u_long mem = sizeof (compression_dict);
+  compression_dict_header cdh;
+  comp_frags_header cfh;
+  FILE *dict;
+  int which;
+  int i;  /* [RPAP - Jan 97: Endian Ordering] */
+
+  dict = open_file (filename, TEXT_DICT_SUFFIX, "rb",
+		    MAGIC_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  fread (&cdh, sizeof (cdh), 1, dict);
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(cdh.dict_type);
+  NTOHUL(cdh.novel_method);
+  for (i = 0; i < TEXT_PARAMS; i++)
+    NTOHUL(cdh.params[i]);
+  NTOHUL(cdh.num_words[0]);
+  NTOHUL(cdh.num_words[1]);
+  NTOHUL(cdh.num_word_chars[0]);
+  NTOHUL(cdh.num_word_chars[1]);
+  NTOHUL(cdh.lookback);
+
+  for (which = 0; which < 2; which++)
+    switch (cdh.dict_type)
+      {
+      case MG_COMPLETE_DICTIONARY:
+	{
+	  mem += sizeof (comp_frags_header);
+	  Read_cfh (dict, &cfh, NULL, NULL);
+
+	  /* Don't need to count the space for the clens of the huffman data */
+
+	  mem += mem_for_words (dict, &cdh, &cfh);
+	  if (cfh.hd.clens)
+	    Xfree (cfh.hd.clens);
+	}
+	break;
+      case MG_PARTIAL_DICTIONARY:
+	{
+	  huff_data hd;
+	  if (cdh.num_words[which])
+	    {
+	      mem += sizeof (comp_frags_header);
+	      Read_cfh (dict, &cfh, NULL, NULL);
+
+	      /* Don't need to count the space for the clens of the
+	         huffman data */
+
+	      mem += mem_for_words (dict, &cdh, &cfh);
+	      if (cfh.hd.clens)
+		Xfree (cfh.hd.clens);
+
+	    }
+
+	  mem += sizeof (hd);
+	  Read_Huffman_Data (dict, &hd, NULL, NULL);
+	  if (hd.clens)
+	    Xfree (hd.clens);
+	  mem += hd.num_codes * sizeof (unsigned long);
+	  mem += (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *);
+
+	  mem += sizeof (hd);
+	  Read_Huffman_Data (dict, &hd, NULL, NULL);
+	  if (hd.clens)
+	    Xfree (hd.clens);
+	  mem += hd.num_codes * sizeof (unsigned long);
+	  mem += (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *);
+	}
+	break;
+      case MG_SEED_DICTIONARY:
+	{
+	  huff_data hd;
+	  if (cdh.num_words[which])
+	    {
+	      mem += sizeof (comp_frags_header);
+	      Read_cfh (dict, &cfh, NULL, NULL);
+
+	      /* Don't need to count the space for the clens of the
+	         huffman data */
+
+	      mem += mem_for_words (dict, &cdh, &cfh);
+	      if (cfh.hd.clens)
+		Xfree (cfh.hd.clens);
+
+	    }
+	  switch (cdh.novel_method)
+	    {
+	    case MG_NOVEL_HUFFMAN_CHARS:
+	      mem += sizeof (hd);
+	      Read_Huffman_Data (dict, &hd, NULL, NULL);
+	      if (hd.clens)
+		Xfree (hd.clens);
+	      mem += hd.num_codes * sizeof (unsigned long);
+	      mem += (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *);
+
+	      mem += sizeof (hd);
+	      Read_Huffman_Data (dict, &hd, NULL, NULL);
+	      if (hd.clens)
+		Xfree (hd.clens);
+	      mem += hd.num_codes * sizeof (unsigned long);
+	      mem += (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *);
+	      break;
+	    case MG_NOVEL_BINARY:
+	      break;
+	    case MG_NOVEL_DELTA:
+	      break;
+	    case MG_NOVEL_HYBRID:
+	      break;
+	    case MG_NOVEL_HYBRID_MTF:
+	      break;
+	    }
+	  break;
+	}
+      }
+  fclose (dict);
+
+  if (cdh.novel_method == MG_NOVEL_BINARY ||
+      cdh.novel_method == MG_NOVEL_DELTA ||
+      cdh.novel_method == MG_NOVEL_HYBRID ||
+      cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+    mem += mem_for_aux_dict (&cdh, filename);
+
+  return ALIGN_SIZE (mem, u_char *);
+}
+
+
+
+void *
+getmem (u_long size, int align)
+{
+  void *res;
+  cur = (void *) (((u_long) cur + (align - 1)) & (~(align - 1)));
+  res = cur;
+  cur = (char *) cur + size;
+  if ((long) cur > (long) buffer + mem)
+    FatalError (1, "The buffer was not big enough for the dictionary");
+  return res;
+}
+
+
+
+
+
+
+
+
+
+
+
+static auxiliary_dict *
+LoadAuxDict (compression_dict_header * cdh,
+	     char *filename)
+{
+  auxiliary_dict *ad;
+  int i;
+  FILE *aux;
+
+  aux = open_file (filename, TEXT_DICT_AUX_SUFFIX, "rb",
+		   MAGIC_AUX_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  ad = getmem (sizeof (auxiliary_dict), sizeof (u_char *));
+
+  for (i = 0; i <= 1; i++)
+    {
+      int j;
+      u_char *pos;
+
+      fread (&ad->afh[i], sizeof (aux_frags_header), 1, aux);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(ad->afh[i].num_frags);
+      NTOHUL(ad->afh[i].mem_for_frags);
+
+      ad->word_data[i] = getmem (ad->afh[i].mem_for_frags, 1);
+      FIXUP (&ad->word_data[i]);
+
+      ad->words[i] = getmem (ad->afh[i].num_frags * sizeof (u_char *),
+			     sizeof (u_char *));
+      FIXUP (&ad->words[i]);
+
+      fread (ad->word_data[i], ad->afh[i].mem_for_frags, sizeof (u_char), aux);
+
+      pos = ad->word_data[i];
+      for (j = 0; j < ad->afh[i].num_frags; j++)
+	{
+	  ad->words[i][j] = pos;
+	  FIXUP (&ad->words[i][j]);
+	  pos += *pos + 1;
+	}
+      if (cdh->novel_method == MG_NOVEL_HYBRID ||
+	  cdh->novel_method == MG_NOVEL_HYBRID_MTF)
+	{
+	  int num;
+	  num = 1;
+	  ad->blk_start[i][0] = 0;
+	  ad->blk_end[i][0] = cdh->num_words[i] - 1;
+	  while (num < 33)
+	    {
+	      ad->blk_start[i][num] = ad->blk_end[i][num - 1] + 1;
+	      ad->blk_end[i][num] = ad->blk_start[i][num] +
+		(ad->blk_end[i][num - 1] - ad->blk_start[i][num - 1]) * 2;
+	      num++;
+	    }
+	}
+    }
+  fclose (aux);
+  return (ad);
+}
+
+
+
+
+
+static u_char ***
+ReadInWords (FILE * dict, compression_dict * cd,
+	     comp_frags_header * cfh, u_char ** escape)
+{
+  int i, lookback;
+  int ptrs_reqd = 0;
+  int mem_reqd = 0;
+  int num_set[MAX_HUFFCODE_LEN + 1];
+  u_char *next_word[MAX_HUFFCODE_LEN + 1];
+  u_char **vals;
+  u_char ***values;
+  u_char word[MAXWORDLEN + 1];
+  u_char last_word[MAX_HUFFCODE_LEN + 1][MAXWORDLEN + 1];
+
+  lookback = cd->cdh.lookback;
+
+  for (i = cfh->hd.mincodelen; i <= cfh->hd.maxcodelen; i++)
+    {
+      ptrs_reqd += (cfh->hd.lencount[i] + ((1 << lookback) - 1)) >> lookback;
+      mem_reqd += cfh->huff_words_size[i];
+    }
+
+  vals = getmem (ptrs_reqd * sizeof (*vals), sizeof (u_char *));
+
+  values = getmem ((MAX_HUFFCODE_LEN + 1) * sizeof (u_char **), sizeof (u_char **));
+
+  next_word[0] = getmem (mem_reqd, sizeof (char));
+
+  cd->MemForCompDict += ptrs_reqd * sizeof (*vals) +
+    (MAX_HUFFCODE_LEN + 1) * sizeof (u_char **) +
+    mem_reqd;
+
+  values[0] = vals;
+  FIXUP (&values[0]);
+  values[0][0] = next_word[0];
+  FIXUP (&values[0][0]);
+  for (i = 1; i <= cfh->hd.maxcodelen; i++)
+    {
+      int next_start = (values[i - 1] - vals) +
+      ((cfh->hd.lencount[i - 1] + ((1 << lookback) - 1)) >> lookback);
+      values[i] = &vals[next_start];
+      FIXUP (&values[i]);
+      next_word[i] = next_word[i - 1] + cfh->huff_words_size[i - 1];
+      values[i][0] = next_word[i];
+      FIXUP (&values[i][0]);
+    }
+
+  bzero ((char *) num_set, sizeof (num_set));
+
+  for (i = 0; i < cfh->hd.num_codes; i++)
+    {
+      register int val, copy;
+      register int len = cfh->hd.clens[i];
+      val = getc (dict);
+      copy = (val >> 4) & 0xf;
+      val &= 0xf;
+
+      fread (word + copy + 1, sizeof (u_char), val, dict);
+      *word = val + copy;
+
+      if ((num_set[len] & ((1 << lookback) - 1)) == 0)
+	{
+	  values[len][num_set[len] >> lookback] = next_word[len];
+	  FIXUP (&values[len][num_set[len] >> lookback]);
+	  memcpy (next_word[len], word, *word + 1);
+	  if (escape && i == cfh->hd.num_codes - 1)
+	    {
+	      *escape = next_word[len];
+	      FIXUP (escape);
+	    }
+	  next_word[len] += *word + 1;
+	}
+      else
+	{
+	  copy = prefixlen (last_word[len], word);
+	  memcpy (next_word[len] + 1, word + copy + 1, *word - copy);
+	  *next_word[len] = (copy << 4) + (*word - copy);
+	  if (escape && i == cfh->hd.num_codes - 1)
+	    {
+	      *escape = next_word[len];
+	      FIXUP (escape);
+	    }
+	  next_word[len] += (*word - copy) + 1;
+	}
+      memcpy (last_word[len], word, *word + 1);
+      num_set[len]++;
+    }
+  if (cfh->hd.clens)
+    Xfree (cfh->hd.clens);
+  cfh->hd.clens = NULL;
+  return values;
+}
+
+
+static unsigned long **
+Generate_Fast_Huffman_Vals (huff_data * data, u_long * mem)
+{
+  int i;
+  unsigned long *fcode[MAX_HUFFCODE_LEN + 1];
+  unsigned long **values;
+  unsigned long *vals;
+
+  if (!data)
+    return (NULL);
+  vals = getmem (data->num_codes * sizeof (*vals), sizeof (long *));
+  values = getmem ((MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *),
+		   sizeof (long *));
+
+  bzero ((char *) values, (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *));
+
+  if (mem)
+    *mem += data->num_codes * sizeof (*vals) +
+      (MAX_HUFFCODE_LEN + 1) * sizeof (unsigned long *);
+
+  fcode[0] = values[0] = &vals[0];
+  FIXUP (&values[0]);
+  for (i = 1; i <= data->maxcodelen; i++)
+    {
+      fcode[i] = values[i] = &vals[(values[i - 1] - vals) + data->lencount[i - 1]];
+      FIXUP (&values[i]);
+    }
+
+  for (i = 0; i < data->num_codes; i++)
+    if (data->clens[i])
+      *fcode[(int) (data->clens[i])]++ = i;
+  return (values);
+}
+
+
+
+static void 
+load_comp_dict (char *filename)
+{
+  FILE *dict;
+  int which;
+  compression_dict *cd;
+
+  cd = getmem (sizeof (compression_dict), sizeof (u_char *));
+  cd->fast_loaded = 1;
+
+  dict = open_file (filename, TEXT_DICT_SUFFIX, "rb",
+		    MAGIC_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  Read_cdh (dict, &cd->cdh, NULL, NULL);
+
+  for (which = 0; which < 2; which++)
+    switch (cd->cdh.dict_type)
+      {
+      case MG_COMPLETE_DICTIONARY:
+	{
+	  cd->cfh[which] = getmem (sizeof (*cd->cfh[which]), sizeof (u_char *));
+	  cd->MemForCompDict += sizeof (*cd->cfh[which]);
+	  Read_cfh (dict, cd->cfh[which], &cd->MemForCompDict, NULL);
+
+	  cd->values[which] = ReadInWords (dict, cd, cd->cfh[which], NULL);
+	  FIXUP (&cd->cfh[which]);
+	  FIXUP (&cd->values[which]);
+	  cd->escape[which] = NULL;
+	}
+	break;
+      case MG_PARTIAL_DICTIONARY:
+	{
+	  huff_data *hd;
+	  u_long **vals;
+	  if (cd->cdh.num_words[which])
+	    {
+	      cd->cfh[which] = getmem (sizeof (*cd->cfh[which]), sizeof (u_char *));
+	      cd->MemForCompDict += sizeof (*cd->cfh[which]);
+	      Read_cfh (dict, cd->cfh[which], &cd->MemForCompDict, NULL);
+	      cd->values[which] = ReadInWords (dict, cd, cd->cfh[which],
+					       &cd->escape[which]);
+	      FIXUP (&cd->cfh[which]);
+	      FIXUP (&cd->values[which]);
+	      FIXUP (&cd->escape[which]);
+	    }
+
+	  hd = getmem (sizeof (huff_data), sizeof (char *));
+	  cd->MemForCompDict += sizeof (huff_data);
+	  Read_Huffman_Data (dict, hd, &cd->MemForCompDict, NULL);
+	  vals = Generate_Fast_Huffman_Vals (hd, &cd->MemForCompDict);
+	  cd->chars_huff[which] = hd;
+	  FIXUP (&cd->chars_huff[which]);
+	  cd->chars_vals[which] = vals;
+	  FIXUP (&cd->chars_vals[which]);
+	  if (hd->clens)
+	    Xfree (hd->clens);
+	  hd->clens = NULL;
+
+	  hd = getmem (sizeof (huff_data), sizeof (char *));
+	  cd->MemForCompDict += sizeof (huff_data);
+	  Read_Huffman_Data (dict, hd, &cd->MemForCompDict, NULL);
+	  vals = Generate_Fast_Huffman_Vals (hd, &cd->MemForCompDict);
+	  cd->lens_huff[which] = hd;
+	  FIXUP (&cd->lens_huff[which]);
+	  cd->lens_vals[which] = vals;
+	  FIXUP (&cd->lens_vals[which]);
+	  if (hd->clens)
+	    Xfree (hd->clens);
+	  hd->clens = NULL;
+	}
+	break;
+      case MG_SEED_DICTIONARY:
+	{
+	  huff_data *hd;
+	  u_long **vals;
+	  if (cd->cdh.num_words[which])
+	    {
+	      cd->cfh[which] = getmem (sizeof (*cd->cfh[which]), sizeof (u_char *));
+	      cd->MemForCompDict += sizeof (*cd->cfh[which]);
+	      Read_cfh (dict, cd->cfh[which], &cd->MemForCompDict, NULL);
+	      cd->values[which] = ReadInWords (dict, cd, cd->cfh[which],
+					       &cd->escape[which]);
+	      FIXUP (&cd->cfh[which]);
+	      FIXUP (&cd->values[which]);
+	      FIXUP (&cd->escape[which]);
+	    }
+	  switch (cd->cdh.novel_method)
+	    {
+	    case MG_NOVEL_HUFFMAN_CHARS:
+	      hd = getmem (sizeof (huff_data), sizeof (char *));
+	      cd->MemForCompDict += sizeof (huff_data);
+	      Read_Huffman_Data (dict, hd, &cd->MemForCompDict, NULL);
+	      vals = Generate_Fast_Huffman_Vals (hd, &cd->MemForCompDict);
+	      cd->chars_huff[which] = hd;
+	      FIXUP (&cd->chars_huff[which]);
+	      cd->chars_vals[which] = vals;
+	      FIXUP (&cd->chars_vals[which]);
+	      if (hd->clens)
+		Xfree (hd->clens);
+	      hd->clens = NULL;
+
+	      hd = getmem (sizeof (huff_data), sizeof (char *));
+	      cd->MemForCompDict += sizeof (huff_data);
+	      Read_Huffman_Data (dict, hd, &cd->MemForCompDict, NULL);
+	      vals = Generate_Fast_Huffman_Vals (hd, &cd->MemForCompDict);
+	      cd->lens_huff[which] = hd;
+	      FIXUP (&cd->lens_huff[which]);
+	      cd->lens_vals[which] = vals;
+	      FIXUP (&cd->lens_vals[which]);
+	      if (hd->clens)
+		Xfree (hd->clens);
+	      hd->clens = NULL;
+	      break;
+	    case MG_NOVEL_BINARY:
+	      break;
+	    case MG_NOVEL_DELTA:
+	      break;
+	    case MG_NOVEL_HYBRID:
+	      break;
+	    case MG_NOVEL_HYBRID_MTF:
+	      break;
+	    }
+	  break;
+	}
+      }
+  fclose (dict);
+
+
+  if (cd->cdh.novel_method == MG_NOVEL_BINARY ||
+      cd->cdh.novel_method == MG_NOVEL_DELTA ||
+      cd->cdh.novel_method == MG_NOVEL_HYBRID ||
+      cd->cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+    {
+      cd->ad = LoadAuxDict (&cd->cdh, filename);
+      FIXUP (&cd->ad);
+    }
+}
+
+
+static void 
+save_fast_dict (char *filename)
+{
+  FILE *fdict;
+
+  fdict = create_file (filename, TEXT_DICT_FAST_SUFFIX, "wb",
+		       MAGIC_FAST_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  {
+    u_long *p;
+    for (p = buffer; (u_long) p < (u_long) cur; p++)
+      {
+	if (IS_FIXUP (p))
+	  HTONUL(*p);
+      }
+  }
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(mem);
+  HTONUL(fixup_mem);
+
+  fwrite (&mem, sizeof (mem), 1, fdict);
+  fwrite (&fixup_mem, sizeof (fixup_mem), 1, fdict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(mem);
+  NTOHUL(fixup_mem);
+
+  fwrite (buffer, sizeof (u_char), mem, fdict);
+  fwrite (fixup, sizeof (u_char), fixup_mem, fdict);
+
+  fclose (fdict);
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/mg_files.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/mg_files.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/mg_files.c	(revision 699)
@@ -0,0 +1,261 @@
+/**************************************************************************
+ *
+ * mg_files.c -- Routines for handling files for the auxillary programs
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "memlib.h"
+#include "messages.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+
+
+/* This must contain a valid path without a trailing slash */
+static char *basepath = NULL;
+
+
+
+/* This sets the base path for all file operations */
+void 
+set_basepath (const char *bp)
+{
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+  basepath = "";
+#else
+  char *s;
+  /* Free the memory for the base path if it has already been allocated */
+  if (basepath)
+    {
+      Xfree (basepath);
+      basepath = NULL;
+    }
+
+  if (!bp || *bp == '\0')
+    bp = ".";
+
+  s = strrchr (bp, '/');
+  if (s && *(s + 1) == '\0')
+    {
+      basepath = Xmalloc (strlen (bp) + 2);
+      if (!basepath)
+	return;
+      strcpy (basepath, bp);
+      strcat (basepath, ".");
+    }
+  else
+    basepath = Xstrdup (bp);
+#endif
+}
+
+
+
+/* return the currently defined basepath */
+char *
+get_basepath (void)
+{
+  if (!basepath)
+    set_basepath (getenv ("MGDATA"));
+  return basepath;
+}
+
+
+
+/* This generates the name of a file. It places the name in the buffer
+   specified or if that is NULL it uses a static buffer. */
+char *
+make_name (const char *name, const char *suffix, char *buffer)
+{
+  static char path[512];
+  if (!buffer)
+    buffer = path;
+  if (!basepath)
+    set_basepath (getenv ("MGDATA"));
+  sprintf (buffer, FILE_NAME_FORMAT, basepath, name, suffix);  /* [RPAP - Feb 97: WIN32 Port] */
+  return buffer;
+}
+
+
+
+/* This will open the specified file and check its magic number.
+   Mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *
+open_named_file (const char *name, const char *mode,
+		 u_long magic_num, int err_mode)
+{
+  unsigned long magic;
+  FILE *f = NULL;
+  char *err;
+  f = fopen (name, mode);
+
+  if (!f)
+    {
+      err = "Unable to open \"%s\"";
+      goto error;
+    }
+
+  if (magic_num)
+    {
+      if (fread (&magic, sizeof (magic), 1, f) != 1)
+	{
+	  err = "No magic number \"%s\"";
+	  goto error;
+	}
+
+      NTOHUL(magic);  /* [RPAP - Jan 97: Endian Ordering] */
+
+      if (!IS_MAGIC (magic))
+	{
+	  err = "No MG magic number \"%s\"";
+	  goto error;
+	}
+
+      if (magic != magic_num)
+	{
+	  err = "Wrong MG magic number \"%s\"";
+	  goto error;
+	}
+    }
+  return f;
+
+error:
+  if (f)
+    fclose (f);
+  if (err_mode == MG_ABORT)
+    FatalError (1, err, name);
+  if (err_mode == MG_MESSAGE)
+    Message (err, name);
+  return NULL;
+}
+
+
+
+/* This will open the specified file and check its magic number.
+   Mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *
+open_file (const char *name, const char *suffix, const char *mode,
+	   u_long magic_num, int err_mode)
+{
+  char path[512];
+  if (!basepath)
+    set_basepath (getenv ("MGDATA"));
+  sprintf (path, FILE_NAME_FORMAT, basepath, name, suffix);  /* [RPAP - Feb 97: WIN32 Port] */
+  return open_named_file (path, mode, magic_num, err_mode);
+}
+
+
+
+
+
+
+/* This will create the specified file and set its magic number.
+
+   Mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *
+create_named_file (const char *name, const char *mode,
+		   u_long magic_num, int err_mode)
+{
+  FILE *f = NULL;
+  char *err;
+  f = fopen (name, mode);
+
+  if (!f)
+    {
+      err = "Unable to open \"%s\"";
+      goto error;
+    }
+
+  if (magic_num)
+    HTONUL(magic_num);  /* [RPAP - Jan 97: Endian Ordering] */
+    if (fwrite (&magic_num, sizeof (magic_num), 1, f) != 1)
+      {
+	err = "Couldn't write magic number \"%s\"";
+	goto error;
+      }
+
+  return f;
+
+error:
+  if (f)
+    fclose (f);
+  if (err_mode == MG_ABORT)
+    FatalError (1, err, name);
+  if (err_mode == MG_MESSAGE)
+    Message (err, name);
+  return NULL;
+}
+
+
+
+
+
+
+
+/* This will create the specified file and set its magic number.
+
+   Mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *
+create_file (const char *name, const char *suffix, const char *mode,
+	     u_long magic_num, int err_mode)
+{
+  char path[512];
+  if (!basepath)
+    set_basepath (getenv ("MGDATA"));
+  sprintf (path, FILE_NAME_FORMAT, basepath, name, suffix);  /* [RPAP - Feb 97: WIN32 Port] */
+  return create_named_file (path, mode, magic_num, err_mode);
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/mg_files.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/mg_files.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/mg_files.h	(revision 699)
@@ -0,0 +1,329 @@
+/**************************************************************************
+ *
+ * mg_files.h -- Routines for handling files for the auxillary programs
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef MG_FILES_H
+#define MG_FILES_H
+
+#include "sysfuncs.h"
+
+/* Magic numbers for the different types of files */
+
+#define GEN_MAGIC(a,b,c,d) ((unsigned long)(((a)<<24) + ((b)<<16) + \
+					    ((c)<<8) + (d)))
+
+#define MAGIC_XXXX		GEN_MAGIC('M','G', 0 , 0)
+#define MAGIC_STATS_DICT	GEN_MAGIC('M','G','S','D')
+#define MAGIC_AUX_DICT		GEN_MAGIC('M','G','A','D')
+#define MAGIC_FAST_DICT		GEN_MAGIC('M','G','F','D')
+#define MAGIC_DICT		GEN_MAGIC('M','G','D', 0 )
+#define MAGIC_STEM_BUILD	GEN_MAGIC('M','G','S', 0 )
+#define MAGIC_HASH		GEN_MAGIC('M','G','H', 0 )
+#define MAGIC_STEM		GEN_MAGIC('M','G','s', 0 )
+#define MAGIC_CHUNK		GEN_MAGIC('M','G','C', 0 )
+#define MAGIC_CHUNK_TRANS	GEN_MAGIC('M','G','c', 0 )
+#define MAGIC_TEXT		GEN_MAGIC('M','G','T', 0 )
+#define MAGIC_TEXI		GEN_MAGIC('M','G','t', 0 )
+#define MAGIC_TEXI_WGT		GEN_MAGIC('M','G','t','W')
+#define MAGIC_INVF		GEN_MAGIC('M','G','I', 0 )
+#define MAGIC_INVI		GEN_MAGIC('M','G','i', 0 )
+#define MAGIC_WGHT		GEN_MAGIC('M','G','W', 0 )
+#define MAGIC_WGHT_APPROX	GEN_MAGIC('M','G','w', 0 )
+#define MAGIC_PARAGRAPH		GEN_MAGIC('M','G','P', 0 )
+/* [RPAP - Jan 97: Stem Index Change] */
+#define MAGIC_STEM_1            GEN_MAGIC('M','G','s','1')
+#define MAGIC_STEM_2            GEN_MAGIC('M','G','s','2')
+#define MAGIC_STEM_3            GEN_MAGIC('M','G','s','3')
+
+#define IS_MAGIC(a) ((((u_long)(a)) & 0xffff0000) == MAGIC_XXXX)
+
+
+/* err_mode values for open_file and create_file */
+#define MG_ABORT 0
+#define MG_MESSAGE 1
+#define MG_CONTINUE 2
+
+
+
+
+
+/* File suffixes */
+
+
+/* The compression dictionary built by txt.pass1 */
+#ifdef SHORT_SUFFIX
+# define TEXT_STATS_DICT_SUFFIX	 ".tsd"
+#else
+# define TEXT_STATS_DICT_SUFFIX	 ".text.stats"
+#endif
+
+/* The compression dictionary built by text.pass1 and comp_dict.process */
+#ifdef SHORT_SUFFIX
+# define TEXT_DICT_SUFFIX	 ".td"
+#else
+# define TEXT_DICT_SUFFIX	 ".text.dict"
+#endif
+
+/* The compression dictionary built by mg_make_fast_dict */
+#ifdef SHORT_SUFFIX
+# define TEXT_DICT_FAST_SUFFIX	 ".tdf"
+#else
+# define TEXT_DICT_FAST_SUFFIX	 ".text.dict.fast"
+#endif
+
+/* The auxilary dictionary built by text.pass2 */
+#ifdef SHORT_SUFFIX
+# define TEXT_DICT_AUX_SUFFIX	 ".tda"
+#else
+# define TEXT_DICT_AUX_SUFFIX	 ".text.dict.aux"
+#endif
+
+/* The compressed text build by text.pass2 */
+#ifdef SHORT_SUFFIX
+# define TEXT_SUFFIX		 ".t"
+#else
+# define TEXT_SUFFIX		 ".text"
+#endif
+
+/* The combined compressed text index and document weight file */
+#ifdef SHORT_SUFFIX
+# define TEXT_IDX_WGT_SUFFIX	 ".tiw"
+#else
+# define TEXT_IDX_WGT_SUFFIX	 ".text.idx.wgt"
+#endif
+
+/* The compressed text index file */
+#ifdef SHORT_SUFFIX
+# define TEXT_IDX_SUFFIX	 	".ti"
+#else
+# define TEXT_IDX_SUFFIX	 	".text.idx"
+#endif
+
+/* The dictionary of stemmed words build by invf.pass1 and ivf.pass1 */
+#ifdef SHORT_SUFFIX
+# define INVF_DICT_SUFFIX         ".id"
+#else
+# define INVF_DICT_SUFFIX         ".invf.dict"
+#endif
+
+/* The dictionary of stemmed words build by stem.process */
+#ifdef SHORT_SUFFIX
+# define INVF_DICT_BLOCKED_SUFFIX ".idb"
+#else
+# define INVF_DICT_BLOCKED_SUFFIX ".invf.dict.blocked"
+#endif
+
+/* The exact document weights file build by make.weights, invf.pass2,
+   or ivf.pass2 */
+#ifdef SHORT_SUFFIX
+# define WEIGHTS_SUFFIX           ".w"
+#else
+# define WEIGHTS_SUFFIX           ".weight"
+#endif
+
+/* The approximate weights file built by make.weights */
+#ifdef SHORT_SUFFIX
+# define APPROX_WEIGHTS_SUFFIX    ".wa"
+#else
+# define APPROX_WEIGHTS_SUFFIX    ".weight.approx"
+#endif
+
+/* The inverted file build by invf.pass2 or ivf.pass2 */
+#ifdef SHORT_SUFFIX
+# define INVF_SUFFIX              ".i"
+#else
+# define INVF_SUFFIX              ".invf"
+#endif
+
+/* The inverted file index build by invf.pass2 or ivf.pass2 */
+#ifdef SHORT_SUFFIX
+# define INVF_IDX_SUFFIX          ".ii"
+#else
+# define INVF_IDX_SUFFIX          ".invf.idx"
+#endif
+
+/* The inverted file chunk descriptor built by ivf.pass1 */
+#ifdef SHORT_SUFFIX
+# define INVF_CHUNK_SUFFIX        ".ic"
+#else
+# define INVF_CHUNK_SUFFIX        ".invf.chunk"
+#endif
+
+/* The word index translation file built by ivf.pass1 */
+#ifdef SHORT_SUFFIX
+# define INVF_CHUNK_TRANS_SUFFIX  ".ict"
+#else
+# define INVF_CHUNK_TRANS_SUFFIX  ".invf.chunk.trans"
+#endif
+
+/* The hashed stemmed dictionary built by make.perf_hash */
+#ifdef SHORT_SUFFIX
+# define INVF_DICT_HASH_SUFFIX    ".idh"
+#else
+# define INVF_DICT_HASH_SUFFIX    ".invf.dict.hash"
+#endif
+
+/* The paragraph descriptior file built by invf.pass1 or ivf.pass1 */
+#ifdef SHORT_SUFFIX
+# define INVF_PARAGRAPH_SUFFIX    ".ip"
+#else
+# define INVF_PARAGRAPH_SUFFIX    ".invf.paragraph"
+#endif
+
+/* The trace file build by mg.builder. */
+#ifdef SHORT_SUFFIX
+# define TRACE_SUFFIX             ".trc"
+#else
+# define TRACE_SUFFIX             ".trace"
+#endif
+
+/* The compression stats file build by mg.builder. */
+#ifdef SHORT_SUFFIX
+# define COMPRESSION_STATS_SUFFIX ".cs"
+#else
+# define COMPRESSION_STATS_SUFFIX ".compression.stats"
+#endif
+
+/* [RPAP - Jan 97: Stem Index Change] */
+/* The casefolded index into the stemmed dictionary */
+#ifdef SHORT_SUFFIX
+# define INVF_DICT_BLOCKED_1_SUFFIX ".ib1"
+#else
+# define INVF_DICT_BLOCKED_1_SUFFIX ".invf.dict.blocked.1"
+#endif
+
+/* [RPAP - Jan 97: Stem Index Change] */
+/* The stemmed index into the stemmed dictionary */
+#ifdef SHORT_SUFFIX
+# define INVF_DICT_BLOCKED_2_SUFFIX ".ib2"
+#else
+# define INVF_DICT_BLOCKED_2_SUFFIX ".invf.dict.blocked.2"
+#endif
+
+/* [RPAP - Jan 97: Stem Index Change] */
+/* The casefolded and stemmed  index into the stemmed dictionary */
+#ifdef SHORT_SUFFIX
+# define INVF_DICT_BLOCKED_3_SUFFIX ".ib3"
+#else
+# define INVF_DICT_BLOCKED_3_SUFFIX ".invf.dict.blocked.3"
+#endif
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+#define FILE_NAME_FORMAT "%s%s%s"
+#else
+#define FILE_NAME_FORMAT "%s/%s%s"
+#endif
+
+
+/* This sets the base path for all file operations */
+void set_basepath (const char *bp);
+
+
+/* return the currently defined basepath */
+char *get_basepath (void);
+
+
+
+
+/* This generates the name of a file. It places the name in the buffer
+   specified or if that is NULL it uses a static buffer. */
+char *make_name (const char *name, const char *suffix, char *buffer);
+
+
+
+
+
+
+
+/* This will open the specified file and check its magic number.
+   Mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *open_named_file (const char *name, const char *mode,
+		       u_long magic_num, int err_mode);
+
+
+
+
+/* This will open the specified file and check its magic number.
+
+   err_mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *open_file (const char *name, const char *suffix, const char *mode,
+		 u_long magic_num, int err_mode);
+
+
+
+
+
+/* This will create the specified file and set its magic number.
+
+   Mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *create_named_file (const char *name, const char *mode,
+			 u_long magic_num, int err_mode);
+
+
+
+/* This will create the specified file and set its magic number.
+
+   err_mode may take on the following values
+   MG_ABORT    : causes an error message to be generated and the
+   program aborted if there is an error.
+   MG_MESSAGE  : causes a message to be generated and a NULL value to
+   be returned if there is an error.
+   MG_CONTINUE : causes a NULL value to be returned if there is an error.
+
+   On success if returns the FILE *. On failure it will return a NULL value
+   and possibly generate an error message, or it will exit the program with
+   an error message.   */
+FILE *create_file (const char *name, const char *suffix, const char *mode,
+		   u_long magic_num, int err_mode);
+
+
+
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/mg_hilite_words.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/mg_hilite_words.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/mg_hilite_words.1	(revision 699)
@@ -0,0 +1,95 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_hilite_words 1 \*(Dt CITRI
+.SH NAME
+mg_hilite_words \- display text and highlight particular words which it contains
+.SH SYNOPSIS
+.B mg_hilite_words
+.if n .ti +9n
+[
+.B --stem_method [0-3]
+]
+.if n .ti +9n
+[
+.B --stemmer [english|lovin|french|simplefrench]
+]
+.if n .ti +9n
+[
+.B --style [bold|underline|italic|emphasis|strong]
+]
+.if n .ti +9n
+[
+.B --pager [less|more|html|???]
+]
+.if n .ti +9n
+.I list-of-words-to-highlight
+
+.SH DESCRIPTION
+.B mg_hilite_words
+reads text from stdin and outputs it into a pager such as
+less. It's command arguments include a list of stemmed words
+which if extracted from the text will be highlighted on the output
+using a highlighting style.
+
+It has been extended to output the highlighted text using html
+straight to stdout.
+
+.SH OPTIONS
+Options may appear in any order, but the
+.IR list-of-words-to-highlight ,
+must be last.
+.TP
+.B --stem_method [0-3]
+This option allows one to set the stemming method for the extraction of
+the words from the incoming text.
+This should match the way the inverted file index was created.
+The default is 3, which is fold and stem.
+.TP
+.B --stemmer [english|lovin|french|simplefrench]
+This option allows you to select the stemmer to use.  The
+default is the English stemmer.
+.TP
+.B --style [bold|underline|italic|emphasis|strong]
+This option sets the highlighting style.
+.IR Bold 
+and 
+.IR underline
+are utilised by the pager or for html output.
+.IR Italic,
+.IR emphasis,
+and
+.IR strong
+can only be used for html output.
+.TP
+.B --pager [less|more|html|???]
+One can use the standard 
+.IR less
+or
+.IR more
+pages for displaying the highlighted text.
+Or one can generate highlighted html using the paging setting of
+.IR html.
+One can also specify a custom pager of their own choosing (i.e. ???).
+.TP
+.B list-of-words-to-highlight
+Each word is taken as a command line argument.
+Each word should already be stemmed and folded according to
+the stem option.
+.SH USAGE
+This program was written primarily to be called by 
+.IR mgquery(1)
+.
+When the user selects the mode of "hilite" then this program
+is called for displaying the query-result documents instead of
+just the pager.
+.SH "SEE ALSO"
+.na
+.BR mgquery (1).
+
+
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/mg_hilite_words.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/mg_hilite_words.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/mg_hilite_words.c	(revision 699)
@@ -0,0 +1,693 @@
+/**************************************************************************
+ *
+ * mg_hilite_words -- display text and highlight particular words which 
+ *                    it contains
+ * Copyright (C) 1994 Tim Shimmin
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+#include "getopt.h"
+#include "messages.h"
+#include "local_strings.h"
+#include "stemmer.h"
+#include "words.h"
+
+
+/*
+ * Description
+ * -----------
+ *   Hilite_words reads text from stdin and outputs it into a pager such as
+ *   less. It's command arguments include a list of stemmed words 
+ *   which if extracted from the text will be highlighted on the output.
+ *
+ * Implementation
+ * --------------
+ *   Extracting of words - using PARSE_STEM_WORD & stemmer
+ *   
+ *   Highlighting of words - for standard pager:
+ *                             using back-space character code
+ *                             e.g. bolded a = "a\ba", underlined a = "_\ba"
+ *                         - for pager==html:
+ *                             use some of the standard html character 
+ *                             formatting tags
+ *
+ *   Storage of words - using hashtable (set) of size equalling a constant
+ *                      times the number of words
+ * 
+ * Usage
+ * -----
+ *   mg_hilite_words --stem_method [0-3] 
+ *                   --style [bold|underline|italic|emphasis|strong]
+ *                   --pager [less|more|html|???]
+ *                   --terminator [terminator-string]
+ *                   list-of-words-to-highlight
+ */
+
+/*
+ * Modifications:
+ *
+ * 21/Apr/95: To handle outputting html tags to stdout
+ *
+ */
+
+/* --- constants --- */
+
+/* highlighting styles */
+#define HILITE_MAX      5	/* the number of styles */
+#define BOLD 		0
+#define UNDERLINE	1
+#define ITALIC     	2
+#define EMPHASIS     	3
+#define STRONG     	4
+
+/* maximum length of line buffer */
+#define MAX_LINE_BUFFER 200
+
+/* set pager to this to get html hilited text to stdout */
+#define HTML_OUT "html"
+
+/* output types */
+#define PAGER 0
+#define HTML  1
+
+/* --- types --- */
+
+typedef u_char *Word;
+
+/* --- globals --- */
+
+/* keep in synch with constants */
+static char *hilite_names[] =
+{"bold", "underline", "italic", "emphasis", "strong"};
+static char *hilite_tags[] =
+{"B", "", "I", "EM", "STRONG"};
+static short hilite_style = BOLD;
+static char *pager = "less";
+static int stemmer_num = 0;	/* Lovin's stemmer */
+static int stem_method = 3;	/* fold & stem */
+static char **word_list;
+static int num_of_words = 0;
+static int output_type = PAGER;
+static char *terminator = NULL;
+
+/* --- prototypes --- */
+
+static Word copy_c_word (char *w);
+static void process_words (char **words, int num_of_words);
+static int get_line (u_char * line, int n, FILE * stream);
+static void process_text (FILE * input_file, FILE * output_file);
+static void copy_word (u_char * w1, u_char * w2);
+static void process_buffer (u_char * s_in, int len, FILE * output_file);
+static void output_word (u_char * s_start, u_char * s_finish, FILE * output_file);
+static void output_hilite_word (u_char * s_start, u_char * s_finish,
+				FILE * output_file);
+static void process_args (int argc, char *argv[]);
+static void print_line (u_char * line, int n, FILE * stream);
+
+
+/******************************** word set ************************************/
+
+#include "hash.h"
+#include "locallib.h"
+
+#define SIZE_FACTOR 2
+
+typedef Word *WordSet;
+static WordSet set_of_words = NULL;
+static int hash_size = 0;
+
+/* prototypes - set routines */
+static void set_create (int num_of_words);
+static void set_add (Word word);
+static int set_member (Word word);
+static void set_print (void);
+
+/* =========================================================================
+ * Function: set_print
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+set_print (void)
+{
+  int i = 0;
+
+  for (i = 0; i < hash_size; i++)
+    {
+      Word word = set_of_words[i];
+
+      if (word)
+	{
+	  int len = *word++;
+
+	  fprintf (stderr, "[%d] = ", i);
+	  while (len--)
+	    fputc (*word++, stderr);
+	  fputc ('\n', stderr);
+	}
+    }
+
+}
+
+
+/* =========================================================================
+ * Function: set_create
+ * Description: 
+ *      Allocate memory for the set
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+set_create (int num_of_words)
+{
+  WordSet set;
+
+  hash_size = prime (num_of_words * SIZE_FACTOR);
+  set = (WordSet) malloc (hash_size * sizeof (Word));
+  if (!set)
+    FatalError (1, "Runout of memory for word hashtable");
+  bzero ((char *) set, hash_size * sizeof (Word));  /* [RPAP - Feb 97: WIN32 Port] */
+
+  set_of_words = set;
+
+}
+
+/* =========================================================================
+ * Function: set_add
+ * Description: 
+ *      Add a string element to the set.
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+set_add (Word word)
+{
+  int hash_val;
+  int hash_step;
+
+  HASH (hash_val, hash_step, word, hash_size);
+
+  /* loop around in case of collisions and need to step */
+  while (1)
+    {
+
+      Word entry = set_of_words[hash_val];
+
+      /* if doesn't exist then */
+      if (!entry)
+	{
+	  set_of_words[hash_val] = word;
+	  break;
+	}
+
+      /* if we have a matching word */
+      if (compare (entry, word) == 0)
+	break;
+
+      /* if collides with a different word */
+      hash_val = (hash_val + hash_step) % hash_size;
+
+    }
+}
+
+
+/* =========================================================================
+ * Function: set_member
+ * Description: 
+ *      Tests whether a string is a member of the set
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static int
+set_member (Word word)
+{
+  int hash_val;
+  int hash_step;
+
+  HASH (hash_val, hash_step, word, hash_size);
+
+  /* loop around in case of collisions and need to step */
+  while (1)
+    {
+
+      Word entry = set_of_words[hash_val];
+
+      /* if doesn't exist then */
+      if (!entry)
+	return 0;
+
+      /* if we have a matching word */
+      if (compare (entry, word) == 0)
+	return 1;
+
+      /* if collides with a different word */
+      hash_val = (hash_val + hash_step) % hash_size;
+
+    }
+}
+
+/******************************** end of word set ******************************/
+
+
+
+/* =========================================================================
+ * Function: copy_c_word
+ * Description: 
+ *      Allocate enough memory and copy word over
+ * Input: 
+ *      w = null terminated string (c_word)
+ * Output: 
+ *      word with length in 1st byte
+ * ========================================================================= */
+
+static Word
+copy_c_word (char *w)
+{
+  int len = strlen (w);
+  Word w_copy = (Word) malloc (len + 1);
+  Word w_ptr = NULL;
+  int j = 0;
+
+  if (!w_copy)
+    FatalError (1, "Not enough memory to copy a word");
+
+  w_copy[0] = len;
+  w_ptr = w_copy + 1;
+  for (j = 0; j < len; j++)
+    *w_ptr++ = *w++;
+
+  return w_copy;
+}
+
+/* =========================================================================
+ * Function: process_words
+ * Description: 
+ *      Go through the stemmed words and add to word set
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+process_words (char **words, int num_of_words)
+{
+  int i = 0;
+
+  set_create (num_of_words);
+
+  for (i = 0; i < num_of_words; i++)
+    {
+      Word word = copy_c_word (words[i]);
+      set_add (word);
+    }
+
+}
+
+/* =========================================================================
+ * Function: get_line
+ * Description: 
+ *      Equivalent of fgets for u_char*.
+ *      But returns length of read-in line.
+ *      Expects to see a '\n' before an EOF
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static int
+get_line (u_char * line, int n, FILE * stream)
+{
+  int i = 0;
+  int ch = '\0';
+
+  while (1)
+    {
+      if (i == n)
+	return i;
+
+      ch = fgetc (stream);
+
+      if (ch == EOF)
+	{
+	  if (!feof (stream))
+	    FatalError (1, "Error on reading a line from stdin");
+	  return EOF;
+	}
+
+      if (ch == '\n')
+	return i;
+
+      *line++ = ch;
+      i++;
+    }
+
+}
+
+/* =========================================================================
+ * Function: print_line
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+print_line (u_char * line, int n, FILE * stream)
+{
+
+  while (n--)
+    {
+      fputc (*line++, stream);
+    }
+  fputc ('\n', stream);
+
+}
+
+/* =========================================================================
+ * Function: process_text
+ * Description: 
+ *      Go through the text from input_file and highlight to output_file 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+process_text (FILE * input_file, FILE * output_file)
+{
+  static u_char line_buffer[MAX_LINE_BUFFER];
+
+
+  while (1)
+    {
+      int len = get_line (line_buffer, MAX_LINE_BUFFER, input_file);
+
+      if (len == EOF)
+	break;
+      process_buffer (line_buffer, len, output_file);
+
+    }
+}
+
+/* =========================================================================
+ * Function: copy_word
+ * Description: 
+ *      Copies w2 into w1. Assumes both have storage allocated.
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+copy_word (u_char * w1, u_char * w2)
+{
+  int i;
+  int len = w2[0];
+
+  for (i = 0; i <= len; i++)
+    *w1++ = *w2++;
+
+}
+
+/* =========================================================================
+ * Function: process_buffer
+ * Description: 
+ *      Parse & stem words of line buffer
+ *      Based on the usage of PARSEing in other mg files.
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+process_buffer (u_char * s_in, int len, FILE * output_file)
+{
+  u_char *end = s_in + len - 1;
+  u_char *s_start = NULL;
+
+  if (!inaword (s_in, end))
+    {
+      s_start = s_in;
+      PARSE_NON_STEM_WORD (s_in, end);
+      output_word (s_start, s_in - 1, output_file);
+    }
+
+  while (s_in <= end)
+    {
+      u_char word[MAXSTEMLEN + 1];
+
+      s_start = s_in;
+      PARSE_STEM_WORD (word, s_in, end);
+
+      stemmer (stem_method, stemmer_num, word);
+
+      if (set_member (word))	/* output with highlighting */
+	{
+	  output_hilite_word (s_start, s_start + word[0] - 1, output_file);
+	  s_start += word[0];	/* step over hilited output */
+	}
+      output_word (s_start, s_in - 1, output_file);
+
+      s_start = s_in;
+      PARSE_NON_STEM_WORD (s_in, end);
+      output_word (s_start, s_in - 1, output_file);
+
+    }				/*while */
+
+  fputc ('\n', output_file);
+  fflush (output_file);
+
+}				/*process_buffer */
+
+/* =========================================================================
+ * Function: output_word
+ * Description: 
+ *      Output a word which lies from s_start to s_finish in buffer
+ * Input: 
+ *      s_start = ptr to 1st char
+ *      s_finish = ptr to last char
+ * Output: 
+ * ========================================================================= */
+
+static void
+output_word (u_char * s_start, u_char * s_finish, FILE * output_file)
+{
+  while (s_start <= s_finish)
+    {
+      fputc (*s_start++, output_file);
+    }
+}
+
+
+/* =========================================================================
+ * Function: output_hilite_word
+ * Description: 
+ *      Highlight a word (with length in 1st byte)
+ *      Pager highlighting:
+ *        Highlighting is either by bolding or underlining using
+ *        the method used by UNIX utilities More(1) and Less(1)
+ *      HTML highlighting:
+ *        use the appropriate start and end tags around the word
+ * ========================================================================= */
+
+static void
+output_hilite_word (u_char * s_start, u_char * s_finish, FILE * output_file)
+{
+
+  if (output_type == HTML)
+    {
+      char *hilite_tag = hilite_tags[hilite_style];
+
+      /* print start tag */
+      fprintf (output_file, "<%s>", hilite_tag);
+
+      output_word (s_start, s_finish, output_file);
+
+      /* print end tag */
+      fprintf (output_file, "</%s>", hilite_tag);
+    }
+
+  else
+    /* PAGER */
+    {
+      /* use backspaces around each letter */
+      while (s_start <= s_finish)
+	{
+	  switch (hilite_style)
+	    {
+	    case BOLD:
+	      fputc (*s_start, output_file);
+	      fputc ('\b', output_file);
+	      fputc (*s_start, output_file);
+	      break;
+	    case UNDERLINE:
+	      fputc ('_', output_file);
+	      fputc ('\b', output_file);
+	      fputc (*s_start, output_file);
+	      break;
+	    default:
+	      fputc (*s_start, output_file);
+	    }
+	  s_start++;
+	}			/*while */
+    }
+}
+
+/* =========================================================================
+ * Function: process_args
+ * Description: 
+ *      sets the global variables:
+ *              hilite_style, pager, num_of_words, word_list
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+struct option long_opts[] =
+{
+  {"style", required_argument, 0, 's'},
+  {"terminator", required_argument, 0, 't'},
+  {"pager", required_argument, 0, 'p'},
+  {"stem_method", required_argument, 0, 'm'},
+  {"stemmer", required_argument, 0, 'a'},
+  {0, 0, 0, 0}
+};
+
+static void
+process_args (int argc, char *argv[])
+{
+  int ch;
+
+
+  opterr = 0;
+  while ((ch = getopt_long (argc, argv, "s:p:t:m:a:", long_opts, (int *) 0)) != -1)
+    {
+      switch (ch)
+	{
+	case 's':
+	  {
+	    int i;
+	    for (i = 0; i < HILITE_MAX; i++)
+	      if (strcmp (optarg, hilite_names[i]) == 0)
+		break;
+
+	    if (i < HILITE_MAX)
+	      hilite_style = i;
+	  }
+	  break;
+	case 'a':
+	  stemmer_num = stemmernumber (optarg);
+	  break;
+	case 't':
+	  terminator = optarg;
+	  break;
+	case 'm':
+	  stem_method = atoi (optarg);
+	  break;
+	case 'p':
+	  pager = optarg;
+	  break;
+	default:
+	  FatalError (1, "Usage: \n"
+		      "mg_hilite_words --stem_method [0-3]\n"
+	  "                --stemmer [english|lovin|french|simplefrench]\n"
+	  "                --style [bold|underline|italic|emphasis|strong]\n"
+		      "                --pager [less|more|html|???]\n");
+	}
+    }
+
+  num_of_words = argc - optind;
+
+  word_list = &argv[optind];
+
+  /* fix up output type */
+  if (strcmp (pager, HTML_OUT) == 0)
+    output_type = HTML;
+  else
+    output_type = PAGER;
+
+}
+
+/* =========================================================================
+ * Function: main
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+int
+main (int argc, char *argv[])
+{
+  FILE *output = NULL;
+
+  process_args (argc, argv);
+
+  /* set output file */
+  if (output_type == PAGER)
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+    output = _popen (pager, "w");
+#else
+    output = popen (pager, "w");
+#endif
+  else
+    output = stdout;
+
+  if (!output)
+    FatalError (1, "Unable to run \"%s\"\n", pager);
+
+  if (num_of_words < 1)
+    {
+      int ch;
+
+      /* just echo the input */
+      /* better not to call this program at all ;-) */
+      while ((ch = fgetc (stdin)) != EOF)
+	{
+	  fputc (ch, output);
+	}
+    }
+  else
+    {
+      /* set up hash table for words */
+      process_words (word_list, num_of_words);
+
+
+      /* Go thru lines of text from stdin and 
+       * output words with hilite info if
+       * words parse into existence in the hash table 
+       */
+      process_text (stdin, output);
+    }
+
+  if (terminator)
+    fprintf (output, "%s\n", terminator);
+
+  if (output != stdout)
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+    _pclose (output);
+#else
+    pclose (output);
+#endif
+
+  return 0;
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/mg_invf_dict.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/mg_invf_dict.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/mg_invf_dict.1	(revision 699)
@@ -0,0 +1,104 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_invf_dict 1 \*(Dt CITRI
+.SH NAME
+mg_invf_dict \- build an `on-disk' stemmed dictionary
+.SH SYNOPSIS
+.B mg_invf_dict
+[
+.B \-h
+]
+[
+.B \-F
+]
+[
+.BI \-b " num"
+]
+[
+.BI \-d " directory"
+]
+.if n .ti +9n
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_invf_dict
+builds an `on-disk' stemmed dictionary from the
+compressed stemmed dictionary that the
+.BR mgquery (1)
+program can access.  Each entry in the dictionary contains the disk
+pointer to the associated inverted file entry, and various other pieces
+of information about the stemmed word.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-b " num"
+The dictionary is stored in blocks on disk; this option is used to set
+the block size.  The default is 4-KB blocks.
+.TP
+.B \-F
+This option causes the disk blocks to be aligned on the appropriate
+boundaries.  For example, if 4-KB blocks are used, they will be aligned
+in the file on 4-KB boundaries.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.dict.build
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict
+The `on-disk' stemmed dictionary.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/mg_invf_dict.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/mg_invf_dict.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/mg_invf_dict.c	(revision 699)
@@ -0,0 +1,344 @@
+/**************************************************************************
+ *
+ * mg_invf_dict.c -- Program to build the blocked stemmed dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "invf.h"
+#include "locallib.h"
+#include "words.h"
+#include "mg.h"
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:18:08  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:45  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:03  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.4  1994/11/29  00:32:00  tes
+   * Committing the new merged files and changes.
+   *
+   * Revision 1.3  1994/10/20  03:56:56  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:49  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+int block_size = 1024 * 4;
+
+int force = 0;
+
+static void process_files (char *filename);
+
+int main (int argc, char **argv)
+{
+  char *file_name = "";
+  int ch;
+  msg_prefix = argv[0];
+  opterr = 0;
+  msg_prefix = argv[0];
+  while ((ch = getopt (argc, argv, "f:d:b:Fh")) != -1)
+    switch (ch)
+      {
+      case 'f':		/* input file */
+	file_name = optarg;
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 'b':
+	block_size = atoi (optarg);
+	break;
+      case 'F':
+	force = 1;
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-f input_file]"
+		 "[-d data directory] [-b num] [-F] [-h]\n", argv[0]);
+	exit (1);
+      }
+
+  process_files (file_name);
+  return 0;
+}
+
+
+
+
+static void 
+process_files (char *filename)
+{
+  FILE *id, *idb, *tmp, *ii;
+  unsigned long i, pos, num, First_word, invf_ptr, invf_len;
+  unsigned long last_ptr = 0;
+  char *FName;
+  struct invf_dict_header idh;
+  struct stem_dict_header sdh;
+  u_char prev[MAXSTEMLEN + 1];
+  u_char *buffer;
+  unsigned short *pointers;
+  int buf_in_use;
+  unsigned short ptrs_in_use, word_num;
+
+  id = open_file (filename, INVF_DICT_SUFFIX, "rb", MAGIC_STEM_BUILD, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  ii = open_file (filename, INVF_IDX_SUFFIX, "rb", MAGIC_INVI, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  idb = create_file (filename, INVF_DICT_BLOCKED_SUFFIX, "w+b", MAGIC_STEM,
+		     MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  FName = make_name (filename, ".tmp", NULL);
+  if (!(tmp = fopen (FName, "w+b")))  /* [RPAP - Feb 97: WIN32 Port] */
+    FatalError (1, "Unable to open \"%s\".\n", FName);
+
+  /* Delete the file now */
+  unlink (FName);
+
+  fread (&idh, sizeof (idh), 1, id);
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(idh.lookback);
+  NTOHUL(idh.dict_size);
+  NTOHUL(idh.total_bytes);
+  NTOHUL(idh.index_string_bytes);
+  NTOHD(idh.input_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(idh.num_of_docs);
+  NTOHUL(idh.static_num_of_docs);
+  NTOHUL(idh.num_of_words);
+  NTOHUL(idh.stemmer_num);
+  NTOHUL(idh.stem_method);
+
+  sdh.lookback = idh.lookback;
+  sdh.block_size = block_size;
+  sdh.num_blocks = 0;
+  sdh.index_chars = 0;
+  sdh.blocks_start = 0;
+  sdh.num_of_docs = idh.num_of_docs;
+  sdh.static_num_of_docs = idh.static_num_of_docs;
+  sdh.num_of_words = idh.num_of_words;
+  sdh.stemmer_num = idh.stemmer_num;
+  sdh.stem_method = idh.stem_method;
+  sdh.indexed = 0;  /* [RPAP - Jan 97: Stem Index Change] */
+
+  fwrite (&sdh, sizeof (sdh), 1, idb);
+
+  if (!(buffer = Xmalloc (block_size + 512)))
+    FatalError (1, "Unable to allocate memory for \"buffer\"\n");
+  if (!(pointers = Xmalloc (block_size + 512)))
+    FatalError (1, "Unable to allocate memory for \"buffer\"\n");
+
+  buf_in_use = 0;
+  pos = 0;
+  word_num = 0;
+  ptrs_in_use = 0;
+  First_word = 0;
+  for (i = 0; i < idh.dict_size; i++)
+    {
+      register unsigned long extra, copy, suff;
+      unsigned long wcnt, fcnt;
+
+      /* build a new word on top of prev */
+      copy = getc (id);
+      suff = getc (id);
+      *prev = copy + suff;
+      fread (prev + copy + 1, sizeof (u_char), suff, id);
+
+      /* read other data, but no need to store it */
+      fread (&fcnt, sizeof (fcnt), 1, id);
+      fread (&wcnt, sizeof (wcnt), 1, id);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fcnt);
+      NTOHUL(wcnt);
+
+      /* read in the inverted file position */
+      fread (&invf_ptr, sizeof (invf_ptr), 1, ii);
+      NTOHUL(invf_ptr);  /* [RPAP - Jan 97: Endian Ordering] */
+      if (word_num % idh.lookback == 0)
+	extra = copy + sizeof (*pointers);
+      else
+	extra = 0;
+      if ((ptrs_in_use + 1) * sizeof (*pointers) + sizeof (ptrs_in_use) + extra +
+	  buf_in_use + suff + 1 + sizeof (fcnt) + sizeof (wcnt) +
+	  sizeof (First_word) + sizeof (invf_ptr) + sizeof (invf_len) > block_size)
+	{
+	  int chunk;
+	  invf_len = invf_ptr - last_ptr;
+
+	  /* [RPAP - Jan 97: Endian Ordering] */
+	  HTONUL(First_word);
+	  HTONUL(invf_len);
+	  HTONUS(word_num);
+
+	  fwrite (&First_word, sizeof (First_word), 1, tmp);
+	  fwrite (&invf_len, sizeof (invf_len), 1, tmp);
+	  fwrite (&word_num, sizeof (word_num), 1, tmp);
+	  fwrite (pointers, sizeof (*pointers), ptrs_in_use, tmp);
+	  fwrite (buffer, sizeof (u_char), buf_in_use, tmp);
+	  bzero ((char *) buffer, block_size);
+	  chunk = buf_in_use + ptrs_in_use * sizeof (*pointers) +
+	    sizeof (ptrs_in_use) + sizeof (First_word) + sizeof (invf_len);
+	  if (force && chunk < block_size)
+	    {
+	      fwrite (buffer, sizeof (u_char), block_size - chunk, tmp);
+	      chunk = block_size;
+	    }
+
+	  pos += chunk;
+
+	  buf_in_use = 0;
+	  word_num = 0;
+	  ptrs_in_use = 0;
+	  sdh.num_blocks++;
+	}
+
+      if (word_num % idh.lookback == 0)
+	{
+	  HTONUS2(buf_in_use, pointers[ptrs_in_use++]);  /* [RPAP - Jan 97: Endian Ordering] */
+	  suff += copy;
+	  copy = 0;
+	}
+      buffer[buf_in_use++] = copy;
+      buffer[buf_in_use++] = suff;
+      bcopy ((char *) (prev + copy + 1), (char *) (buffer + buf_in_use), suff);
+      buf_in_use += suff;
+      HTONUL(fcnt);  /* [RPAP - Jan 97: Endian Ordering] */
+      bcopy ((char *) &fcnt, (char *) (buffer + buf_in_use), sizeof (fcnt));
+      buf_in_use += sizeof (fcnt);
+      HTONUL(wcnt);  /* [RPAP - Jan 97: Endian Ordering] */
+      bcopy ((char *) &wcnt, (char *) (buffer + buf_in_use), sizeof (wcnt));
+      buf_in_use += sizeof (wcnt);
+      last_ptr = invf_ptr;
+      HTONUL(invf_ptr);  /* [RPAP - Jan 97: Endian Ordering] */
+      bcopy ((char *) &invf_ptr, (char *) (buffer + buf_in_use), sizeof (invf_ptr));
+      NTOHUL(invf_ptr);  /* [RPAP - Jan 97: Endian Ordering] */
+      buf_in_use += sizeof (invf_ptr);
+      if (buf_in_use + ptrs_in_use * sizeof (*pointers) +
+	  sizeof (ptrs_in_use) > block_size)
+	FatalError (1, "Fatal Internal Error # 34876234\n");
+      if (word_num == 0)
+	{
+	  fwrite (prev, sizeof (u_char), *prev + 1, idb);
+	  HTONUL(pos);  /* [RPAP - Jan 97: Endian Ordering] */
+	  fwrite (&pos, sizeof (pos), 1, idb);
+	  NTOHUL(pos);  /* [RPAP - Jan 97: Endian Ordering] */
+	  sdh.index_chars += *prev + 1;
+	  First_word = i;
+	}
+      word_num++;
+    }
+  if (buf_in_use)
+    {
+      int chunk;
+      fread (&invf_ptr, sizeof (invf_ptr), 1, ii);
+      NTOHUL(invf_ptr);  /* [RPAP - Jan 97: Endian Ordering] */
+      invf_len = invf_ptr - last_ptr;
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      HTONUL(First_word);
+      HTONUL(invf_len);
+      HTONUS(word_num);
+
+      fwrite (&First_word, sizeof (First_word), 1, tmp);
+      fwrite (&invf_len, sizeof (invf_len), 1, tmp);
+      fwrite (&word_num, sizeof (word_num), 1, tmp);
+      fwrite (pointers, sizeof (*pointers), ptrs_in_use, tmp);
+      fwrite (buffer, sizeof (u_char), buf_in_use, tmp);
+      bzero ((char *) buffer, block_size);
+      chunk = buf_in_use + ptrs_in_use * sizeof (*pointers) +
+	sizeof (ptrs_in_use) + sizeof (First_word) + sizeof (invf_len);
+      if (force && chunk < block_size)
+	{
+	  fwrite (buffer, sizeof (u_char), block_size - chunk, tmp);
+	  chunk = block_size;
+	}
+
+      sdh.num_blocks++;
+    }
+  fclose (id);
+  fclose (ii);
+
+  rewind (tmp);
+  sdh.blocks_start = sdh.index_chars + sizeof (u_long) + sizeof (sdh) +
+    sdh.num_blocks * sizeof (pos);
+  if (force)
+    {
+      int amount;
+      amount = sdh.blocks_start % block_size;
+      if (amount != 0)
+	{
+	  bzero ((char *) buffer, block_size);
+	  fwrite (buffer, sizeof (u_char), block_size - amount, idb);
+	  sdh.blocks_start += block_size - amount;
+	}
+    }
+
+  while ((num = fread (buffer, sizeof (u_char), block_size, tmp)) != 0)
+    fwrite (buffer, sizeof (u_char), num, idb);
+  fclose (tmp);
+
+  /* skip over the magic number */
+  fseek (idb, sizeof (u_long), 0);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(sdh.lookback);
+  HTONUL(sdh.block_size);
+  HTONUL(sdh.num_blocks);
+  HTONUL(sdh.blocks_start);
+  HTONUL(sdh.index_chars);
+  HTONUL(sdh.num_of_docs);
+  HTONUL(sdh.static_num_of_docs);
+  HTONUL(sdh.num_of_words);
+  HTONUL(sdh.stemmer_num);
+  HTONUL(sdh.stem_method);
+  HTONUL(sdh.indexed);
+
+  fwrite (&sdh, sizeof (sdh), 1, idb);
+  fclose (idb);
+
+
+  Message ("Block size = %d\n", block_size);
+  Message ("Number of blocks written = %d\n", NTOHUL(sdh.num_blocks));
+
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/mg_invf_dump.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/mg_invf_dump.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/mg_invf_dump.1	(revision 699)
@@ -0,0 +1,142 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_invf_dump 1 \*(Dt CITRI
+.SH NAME
+mg_invf_dump \- Dump out an inverted file in ASCII
+.SH SYNOPSIS
+.B mg_invf_dump
+[
+.B \-h
+]
+[
+.B \-b
+]
+[
+.B \-w
+]
+[
+.B \-t
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+This program dumps out an inverted file produced by the
+.BR mg (1)
+system as ASCII numbers.  This program could be used in conjunction
+with
+.BR mgdictlist (1)
+to write simple shell programs that work with the inverted files.  The
+output from the program looks something like this:
+.LP
+.RS
+.ft 3
+.nf
+6 337
+1
+ 1 1
+1
+ 5 1
+1
+ 2 1
+1
+ 1 1
+4
+ 1 3
+ 2 7
+ 3 2
+ 5 2
+1
+ 1 1
+ . . .
+.fi
+.ft
+.RE
+The first number (6) is the number of documents in the collection.  It
+is followed by the number of different terms in the collection.  For
+each term, the number of different documents it occurs in follows. The
+document numbers that the term occurs in are listed next.  If
+.B \-w
+is specified, then the number of times the term occurred in the
+document is printed alongside the document number.
+If
+.B \-t
+is specified, the term itself is also displayed in quotes.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-b
+This option will cause the output from the program to be in fixed-size
+binary numbers, rather than in ASCII.
+.TP
+.B \-w
+If the inverted file is a level-2 or level-3 inverted file, this causes
+the word counts per document to be output.
+.TP
+.B \-t
+This option causes each term to be displayed along with the number of
+documents it appears in. The term is printed in quotes,
+as in "the".
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf.dict
+The compressed stemmed dictionary.
+.TP
+.B *.invf.ORG
+The original inverted file saved by
+.BR mg_invf_rebuild .
+.TP
+.B *.invf
+The inverted file.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/mg_invf_dump.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/mg_invf_dump.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/mg_invf_dump.c	(revision 699)
@@ -0,0 +1,222 @@
+/**************************************************************************
+ *
+ * mg_invf_dump.c -- Program to dump uot an inverted fil
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "messages.h"
+#include "timing.h"
+#include "bitio_m.h"
+#include "bitio_m_stdio.h"
+#include "bitio_gen.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "locallib.h"
+#include "words.h"
+#include "invf.h"
+
+extern unsigned long S_btg;
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:18:09  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:46  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:05  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/11/29  00:32:01  tes
+   * Committing the new merged files and changes.
+   *
+   * Revision 1.2  1994/09/20  04:41:50  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+
+static void process_files (char *filename);
+
+int binary = 0;
+int word_counts = 0;
+int term_dump = 0;
+
+
+int main (int argc, char **argv)
+{
+  ProgTime start;
+  char *dir_name, *file_name = "";
+  int ch;
+  msg_prefix = argv[0];
+  dir_name = getenv ("MGDATA");
+  opterr = 0;
+  msg_prefix = argv[0];
+  while ((ch = getopt (argc, argv, "hbwtf:d:")) != -1)
+    switch (ch)
+      {
+      case 'f':		/* input file */
+	file_name = optarg;
+	break;
+      case 'd':
+        set_basepath(optarg);
+	break;
+      case 'b':
+	binary = 1;
+	break;
+      case 'w':
+	word_counts = 1;
+	break;
+      case 't':
+	term_dump = 1;
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-h] [-b] [-w] [-t] [-f input_file]"
+		 "[-d data directory]\n", argv[0]);
+	exit (1);
+      }
+  GetTime (&start);
+  process_files (file_name);
+  Message ("%s\n", ElapsedTime (&start, NULL));
+  return 0;
+}
+
+static void 
+process_files (char *name)
+{
+  unsigned long N, Nstatic, k;
+  FILE *dict;
+  FILE *invf;
+  struct invf_dict_header idh;
+  struct invf_file_header ifh;
+  int i;  /* [RPAP - Jan 97: Endian Ordering] */
+
+  dict = open_file (name, INVF_DICT_SUFFIX, "rb", MAGIC_STEM_BUILD, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  fread ((char *) &idh, sizeof (idh), 1, dict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(idh.lookback);
+  NTOHUL(idh.dict_size);
+  NTOHUL(idh.total_bytes);
+  NTOHUL(idh.index_string_bytes);
+  NTOHD(idh.input_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(idh.num_of_docs);
+  NTOHUL(idh.static_num_of_docs);
+  NTOHUL(idh.num_of_words);
+  NTOHUL(idh.stemmer_num);
+  NTOHUL(idh.stem_method);
+
+  if (!(invf = open_file (name, INVF_SUFFIX ".ORG", "rb", MAGIC_INVF, MG_CONTINUE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    invf = open_file (name, INVF_SUFFIX, "rb", MAGIC_INVF, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  fread ((char *) &ifh, sizeof (ifh), 1, invf);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(ifh.no_of_words);
+  NTOHUL(ifh.no_of_ptrs);
+  NTOHUL(ifh.skip_mode);
+  for (i = 0; i < 16; i++)
+    NTOHUL(ifh.params[i]);
+  NTOHUL(ifh.InvfLevel);
+
+  if (ifh.skip_mode != 0)
+    FatalError (1, "The invf file contains skips. Unable to dump.");
+
+  DECODE_START (invf)
+    N = idh.num_of_docs;
+  Nstatic = idh.static_num_of_docs;
+  if (binary)
+    {
+      fwrite ((char *) &N, sizeof (N), 1, stdout);
+      fwrite ((char *) &ifh.no_of_words, sizeof (ifh.no_of_words), 1, stdout);
+    }
+  else
+    printf ("%ld %ld\n", N, ifh.no_of_words);
+  for (k = 0; k < ifh.no_of_words; k++)
+    {
+      int i, blk, doc;
+      register unsigned long suff, prefix;
+      unsigned long fcnt, wcnt;
+      char term[MAXSTEMLEN + 1];
+
+      prefix = fgetc (dict);
+      suff = fgetc (dict);
+      fread (&term[prefix], sizeof (char), suff, dict);
+      fread ((char *) &fcnt, sizeof (fcnt), 1, dict);
+      fread ((char *) &wcnt, sizeof (wcnt), 1, dict);
+      term[prefix + suff] = '\0';
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fcnt);
+      NTOHUL(wcnt);
+
+      if (binary)
+	fwrite ((char *) &fcnt, sizeof (fcnt), 1, stdout);
+      else
+	{
+	  if (term_dump)
+	    printf ("%ld    \"%s\"\n", fcnt, term);
+	  else
+	    printf ("%ld\n", fcnt);
+	}
+      blk = BIO_Bblock_Init (Nstatic, fcnt);
+      for (doc = i = 0; i < fcnt; i++)
+	{
+	  int num;
+	  BBLOCK_DECODE (num, blk);
+	  doc += num;
+	  if (binary)
+	    fwrite ((char *) &doc, sizeof (doc), 1, stdout);
+	  else
+	    printf (" %d", doc);
+	  if (ifh.InvfLevel >= 2)
+	    {
+	      int count;
+	      GAMMA_DECODE (count);
+	      if (word_counts)
+		if (binary)
+		  fwrite ((char *) &count, sizeof (count), 1, stdout);
+		else
+		  printf (" %d", count);
+	    }
+	  if (!binary)
+	    putchar ('\n');
+	}
+      while (__btg)
+	DECODE_BIT;
+    }
+  DECODE_DONE
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/mg_invf_merge.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/mg_invf_merge.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/mg_invf_merge.1	(revision 699)
@@ -0,0 +1,112 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_invf_merge 1 \*(Dt CITRI
+.SH NAME
+mg_invf_merge \- Merge two inverted files and stemmed dictionaries
+.SH SYNOPSIS
+.B mg_invf_merge
+[
+.B \-h
+]
+[
+.B \-s
+]
+[
+.B \-w
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+This program takes two
+.BR mg (1)
+database compressed text files and appends the second to
+the first.
+It is part of the
+.BR mgmerge (1)
+addition to the
+.BR mg (1)
+package.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-s
+This option will cause a slow merge to be performed on the inverted files,
+where each inverted file entry is decoded and recoded.
+The default is a fast merge. Accumulated fast merges slowly degrade
+compression performance on the resulting inverted file so
+a periodic slow merge is recommended.
+.TP
+.B \-w
+By default the database .weights file is updated with
+new weights for the new documents in
+.B mg_text_merge.
+The
+.B -w
+option turns this step off.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.old.invf.dict
+The old compressed stemmed dictionary.
+.TP
+.B *.new.invf.dict
+The new compressed stemmed dictionary.
+.TP
+.B *.invf.dict
+The resulting merged compressed stemmed dictionary.
+.TP
+.B *.old.invf.ORG
+The original inverted file saved by
+.BR mg_invf_rebuild .
+.TP
+.B *.old.invf
+The old inverted file.
+.TP
+.B *.new.invf
+The new inverted file.
+.TP
+.B *.invf
+The resulting merged inverted file.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mgmerge (1),
+.BR mg_get_merge (1),
+.BR mg_text_merge (1),
+.BR mg_invf_rebuild (1),
+.BR mg_invf_dict (1),
+.BR mg_weights_build (1),
+.BR mgquery (1).
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/mg_invf_merge.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/mg_invf_merge.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/mg_invf_merge.c	(revision 699)
@@ -0,0 +1,852 @@
+/**************************************************************************
+ *
+ * mg_invf_merge.c -- description
+ * Copyright (C) 1995  Shane Hudson (shane@cosc.canterbury.ac.nz)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * Last edited:  31 March, 1995 
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "locallib.h"
+#include "local_strings.h"
+#include "messages.h"
+#include "timing.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "bitio_m.h"
+#include "bitio_gen.h"
+#include "bitio_stdio.h"
+
+#include "mg.h"
+#include "mg_merge.h"
+#include "mg_files.h"
+#include "invf.h"
+#include "words.h"
+
+typedef char FileName[256];
+
+
+/*
+ *  The merge_info struct contains all the data for an inverted file and
+ *  lexicon. We declare 3 merge_info variables, m[OLD] m[NEW] and m[MERGE].
+ */
+typedef struct merge_info_type
+  {
+    FileName fname;
+    FILE *invf;
+    FILE *dict;
+    FILE *idx;
+    FILE *para;  /* [RPAP - Feb 97: Level 3 Merge] */
+    struct invf_dict_header idh;
+    struct invf_file_header ifh;
+    u_long nDocs;
+
+    /* Lexicon processing variables */
+    u_long fcnt, wcnt;
+    int suff, pref;
+    unsigned char term[MAXSTEMLEN + 1];
+    u_long term_count;
+    int done;
+
+    /* Variables used for merging */
+    struct stdio_bitio_state sbs;	/* for bit-based I/O on inverted files */
+    u_long nTerms;		/* = number of invf entries */
+    int *fcntlist;		/* store fcnt values read in from the lexicons */
+
+  }
+merge_info;
+
+
+/***************************************************
+ **** GLOBALS ****
+ ***************************************************/
+
+merge_info m[3];		/*  m[OLD], m[NEW] and m[MERGE] */
+u_long magicsize;		/* how big is magic number? currently always 4 */
+u_long Nstatic;			/* static # of documents in file */
+
+float *DocWeightBuffer;
+int weightOption = 0;		/* true if weights file is to be appended */
+
+/*** Globals for processing the lexicons -- see readTerm() and writeTerm() 
+ ***/
+ProgTime start;			/* for printing elapsed time */
+unsigned char prevTerm[MAXSTEMLEN + 1];
+
+/*** Globals for merging -- see processEntry() 
+ ***/
+char *mergedata;		/* stores value OLD/NEW/MERGE for each term */
+int fastMerge = 1;		/* True if fast merge selected. Default = true */
+
+u_long oldIdxValue;		/* stores last read value from m[OLD].idx
+				   Used to calculate entry lengths */
+u_long bytes_output;		/* for writing .invf.idx file */
+u_long oldOnlyBits = 0;		/* entries in old only */
+u_long oldMergeBits = 0;	/* entries in old that are merged with 
+				   a new entry */
+
+/**************************************************************************/
+
+
+/*=======================================================================
+ * init_merge_invf(): open files, initialise globals, etc
+ *=======================================================================*/
+int 
+init_merge_invf ()
+{
+  prevTerm[0] = prevTerm[1] = 0;
+
+    /***
+     *  open .dict files
+     ***/
+  m[OLD].dict = open_file (m[OLD].fname, INVF_DICT_SUFFIX, "rb",
+			   MAGIC_STEM_BUILD, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+  magicsize = ftell (m[OLD].dict);
+  fread (&(m[OLD].idh), sizeof (m[OLD].idh), 1, m[OLD].dict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(m[OLD].idh.lookback);
+  NTOHUL(m[OLD].idh.dict_size);
+  NTOHUL(m[OLD].idh.total_bytes);
+  NTOHUL(m[OLD].idh.index_string_bytes);
+  NTOHD(m[OLD].idh.input_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(m[OLD].idh.num_of_docs);
+  NTOHUL(m[OLD].idh.static_num_of_docs);
+  NTOHUL(m[OLD].idh.num_of_words);
+  NTOHUL(m[OLD].idh.stemmer_num);
+  NTOHUL(m[OLD].idh.stem_method);
+
+  m[NEW].dict = open_file (m[NEW].fname, INVF_DICT_SUFFIX, "rb",
+			   MAGIC_STEM_BUILD, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+  fread (&(m[NEW].idh), sizeof (m[NEW].idh), 1, m[NEW].dict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(m[NEW].idh.lookback);
+  NTOHUL(m[NEW].idh.dict_size);
+  NTOHUL(m[NEW].idh.total_bytes);
+  NTOHUL(m[NEW].idh.index_string_bytes);
+  NTOHD(m[NEW].idh.input_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(m[NEW].idh.num_of_docs);
+  NTOHUL(m[NEW].idh.static_num_of_docs);
+  NTOHUL(m[NEW].idh.num_of_words);
+  NTOHUL(m[NEW].idh.stemmer_num);
+  NTOHUL(m[NEW].idh.stem_method);
+
+  m[MERGE].dict = create_file (m[MERGE].fname, INVF_DICT_SUFFIX, "wb",
+			       MAGIC_STEM_BUILD, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  /* write space for header */
+  fwrite ((char *) &(m[OLD].idh), sizeof (m[OLD].idh), 1, m[MERGE].dict);
+
+  m[OLD].nDocs = m[OLD].idh.num_of_docs;
+  m[NEW].nDocs = m[NEW].idh.num_of_docs;
+  Nstatic = m[OLD].idh.static_num_of_docs;
+  m[OLD].nTerms = m[OLD].idh.dict_size;
+  m[NEW].nTerms = m[NEW].idh.dict_size;
+  m[MERGE].nDocs = m[OLD].nDocs + m[NEW].nDocs;
+  m[MERGE].nTerms = 0;
+
+  /* Set up weight buffer for weights of new documents */
+  if (!(DocWeightBuffer = Xmalloc (m[NEW].nDocs * sizeof (*DocWeightBuffer))))
+    {
+      Message ("Insufficient memory\n");
+      return COMPERROR;
+    }
+  bzero ((char *) DocWeightBuffer, m[NEW].nDocs * sizeof (*DocWeightBuffer));
+
+    /***
+     *  open .invf files
+     *  Try ".ORG" extension first for m[OLD].invf since an inverted file
+     *  with skips may have been created.
+     ****/
+  if (!(m[OLD].invf = open_file (m[OLD].fname, INVF_SUFFIX ".ORG", "rb",
+				 MAGIC_INVF, MG_CONTINUE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    m[OLD].invf = open_file (m[OLD].fname, INVF_SUFFIX, "rb",
+			     MAGIC_INVF, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+  fread (&(m[OLD].ifh), sizeof (m[OLD].ifh), 1, m[OLD].invf);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(m[OLD].ifh.no_of_words);
+  NTOHUL(m[OLD].ifh.no_of_ptrs);
+  NTOHUL(m[OLD].ifh.skip_mode);
+  {
+    int i;
+    for (i = 0; i < 16; i++)
+      NTOHUL(m[OLD].ifh.params[i]);
+  }
+  NTOHUL(m[OLD].ifh.InvfLevel);
+
+  m[NEW].invf = open_file (m[NEW].fname, INVF_SUFFIX, "rb",
+			   MAGIC_INVF, MG_ABORT);
+  fread (&(m[NEW].ifh), sizeof (m[NEW].ifh), 1, m[NEW].invf);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(m[NEW].ifh.no_of_words);
+  NTOHUL(m[NEW].ifh.no_of_ptrs);
+  NTOHUL(m[NEW].ifh.skip_mode);
+  {
+    int i;
+    for (i = 0; i < 16; i++)
+      NTOHUL(m[NEW].ifh.params[i]);
+  }
+  NTOHUL(m[NEW].ifh.InvfLevel);
+
+  if (m[OLD].ifh.skip_mode != 0)
+    FatalError (1, "The old invf file contains skips. Unable to merge.");
+
+  if (m[OLD].ifh.InvfLevel != m[NEW].ifh.InvfLevel)
+    FatalError (1, "The two invf files have different inversion levels!");
+
+  /********** [RPAP - Feb 97: Level 3 Merge]
+
+  if ((m[OLD].ifh.InvfLevel > 2) || (m[NEW].ifh.InvfLevel > 2))
+    FatalError (1, "mgmerge cannot handle level 3 inverted files!");
+
+    *************/
+
+  m[MERGE].invf = create_file (m[MERGE].fname, INVF_SUFFIX, "wb",
+			       MAGIC_INVF, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  /* write space for header in inverted file */
+  m[MERGE].ifh.no_of_words = m[MERGE].ifh.no_of_ptrs = 0;
+  m[MERGE].ifh.skip_mode = 0;
+  m[MERGE].ifh.InvfLevel = 2;
+  {
+    int i;
+    for (i = 0; i < 16; i++)
+      m[MERGE].ifh.params[i] = 0;
+  }
+  fwrite ((char *) &(m[MERGE].ifh), sizeof (m[MERGE].ifh), 1, m[MERGE].invf);
+
+    /***
+     *  open .invf.idx files
+     ***/
+  m[OLD].idx = open_file (m[OLD].fname, INVF_IDX_SUFFIX, "rb",
+			  MAGIC_INVI, MG_ABORT);
+  fread (&oldIdxValue, sizeof (u_long), 1, m[OLD].idx);
+  NTOHUL(oldIdxValue);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  m[NEW].idx = open_file (m[NEW].fname, INVF_IDX_SUFFIX, "rb",
+			  MAGIC_INVI, MG_ABORT);
+
+  m[MERGE].idx = create_file (m[MERGE].fname, INVF_IDX_SUFFIX, "wb",
+			      MAGIC_INVI, MG_ABORT);
+
+
+  /* [RPAP - Feb 97: Level 3 Merge] */
+
+    /***
+     *  open .invf.parargraph files
+     ***/
+  if (m[OLD].ifh.InvfLevel == 3)
+    {
+      m[OLD].para = NULL;
+      m[NEW].para = open_file (m[NEW].fname, INVF_PARAGRAPH_SUFFIX, "rb",
+			       MAGIC_PARAGRAPH, MG_ABORT);
+      m[MERGE].para = open_file (m[MERGE].fname, INVF_PARAGRAPH_SUFFIX, "rb+",
+				 MAGIC_PARAGRAPH, MG_ABORT);
+    }
+  else
+    {
+      m[OLD].para = NULL;
+      m[NEW].para = NULL;
+      m[MERGE].para = NULL;
+    }
+
+  return OK;
+}
+
+
+/*=======================================================================
+ * procEntry_slow(): merge an entry, slow method
+ *=======================================================================*/
+void 
+procEntry_slow (int tnum, int type, u_long oldN, u_long newN, u_long mergeN)
+/* 
+ *  Process an invf entry. 
+ *  Slow Merge. (Always decode/re-encode).
+ *
+ *  tnum is the entry number, type = OLD or NEW or MERGE.
+ *  oldN, newN and mergeN are the N parameters for Bblock coding.
+ *  They are passed so other procEntry_fast can call this one
+ *  when a decode/recode is needed to save repeating code.
+ */
+{
+  int doc = 0;
+  int fcntOLD = m[OLD].fcntlist[tnum];
+  int fcntNEW = m[NEW].fcntlist[tnum];
+  int inblk, outblk;
+  u_long inbits = 0, outbits = 0;
+
+  /* write .invf.idx pointer */
+  HTONUL(bytes_output);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite (&bytes_output, sizeof (u_long), 1, m[MERGE].idx);
+  NTOHUL(bytes_output);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  outblk = BIO_Bblock_Init (mergeN, (fcntOLD + fcntNEW));
+
+/*** OLD entry -> merge entry ***/
+  if (type != NEW)
+    {
+      int i;
+      inblk = BIO_Bblock_Init (oldN, fcntOLD);
+      for (i = 0; i < fcntOLD; i++)
+	{
+	  int num;
+	  num = BIO_Stdio_Bblock_Decode (inblk, &(m[OLD].sbs), &inbits);
+	  doc += num;
+	  BIO_Stdio_Bblock_Encode (num, outblk, &(m[MERGE].sbs), &outbits);
+	  if (m[OLD].ifh.InvfLevel >= 2)	/* frequencies */
+	    {
+	      num = BIO_Stdio_Gamma_Decode (&(m[OLD].sbs), &inbits);
+	      BIO_Stdio_Gamma_Encode (num, &(m[MERGE].sbs), &outbits);
+	    }
+	}
+      if (type == OLD)
+	oldOnlyBits += (inbits + m[OLD].sbs.Btg);
+      else
+	oldMergeBits += (inbits + m[OLD].sbs.Btg);
+    }
+
+/*** NEW entry -> merge entry ***/
+  if (type != OLD)
+    {
+      int i;
+      int offset;
+      double logN = log ((double) m[MERGE].nDocs);
+      double idf = logN - log ((double) (fcntOLD + fcntNEW));
+
+      inblk = BIO_Bblock_Init (newN, fcntNEW);
+      offset = (m[OLD].nDocs - doc);	/* amount to be added to 1ST pointer */
+      for (i = 0; i < fcntNEW; i++)
+	{
+	  register int num;
+	  register double weight;
+	  num = BIO_Stdio_Bblock_Decode (inblk, &(m[NEW].sbs), &inbits);
+	  if (i == 0)
+	    num += offset;
+	  doc += num;
+	  BIO_Stdio_Bblock_Encode (num, outblk, &(m[MERGE].sbs), &outbits);
+	  if (m[OLD].ifh.InvfLevel >= 2)	/* frequencies */
+	    {
+	      num = BIO_Stdio_Gamma_Decode (&(m[NEW].sbs), &inbits);
+	      BIO_Stdio_Gamma_Encode (num, &(m[MERGE].sbs), &outbits);
+	      weight = num * idf;
+	      DocWeightBuffer[doc - m[OLD].nDocs - 1] += weight * weight;
+	    }
+	}
+    }
+/*** Now the padding bits ***/
+  while (m[OLD].sbs.Btg)
+    BIO_Stdio_Decode_Bit (&(m[OLD].sbs));
+  while (m[NEW].sbs.Btg)
+    BIO_Stdio_Decode_Bit (&(m[NEW].sbs));
+  while (m[MERGE].sbs.Btg != 8)
+    {
+      BIO_Stdio_Encode_Bit (0, &(m[MERGE].sbs));
+      outbits++;
+    }
+  bytes_output += (outbits >> 3);
+  return;
+}
+
+
+/*=======================================================================
+ * procEntry_fast(): merge an entry, WITHOUT decoding if it's in the
+ *                   old inverted file only
+ *=======================================================================*/
+void 
+procEntry_fast (int tnum, int type)
+/*
+ *  Merge an invf entry. 
+ *  Faster Method : Copy (don't re-code) entries for terms only in IFold.
+ *  The N parameter is Nstatic, from m[OLD].idh.static_num_of_docs.
+ *  If entry type isnt OLD, just call procEntry_slow()
+ */
+{
+  u_long newIdxValue, len = 0;
+
+  /* Calculate IFold entry length in bytes if not NEW type */
+  if (type != NEW)		/* read in an index number from m[OLD].idx */
+    {
+      fread (&newIdxValue, sizeof (u_long), 1, m[OLD].idx);
+      NTOHUL(newIdxValue);  /* [RPAP - Jan 97: Endian Ordering] */
+      len = newIdxValue - oldIdxValue;
+      oldIdxValue = newIdxValue;
+    }
+
+  if (type != OLD)		/* must decode/recode */
+    procEntry_slow (tnum, type, Nstatic, m[NEW].nDocs, Nstatic);
+  else
+    {
+      /* write .invf.idx pointer */
+      HTONUL(bytes_output);  /* [RPAP - Jan 97: Endian Ordering] */
+      fwrite (&bytes_output, sizeof (u_long), 1, m[MERGE].idx);
+      NTOHUL(bytes_output);  /* [RPAP - Jan 97: Endian Ordering] */
+
+      /* copy entry */
+      {
+	int i;
+	char c;
+	for (i = 0; i < len; i++)
+	  {
+	    c = fgetc (m[OLD].invf);
+	    fputc (c, m[MERGE].invf);
+	  }
+      }
+      bytes_output += len;
+      oldOnlyBits += (len * 8);
+    }
+  return;
+}
+
+
+/*=======================================================================
+ * processEntry(): merge an entry. Call either procEntry_slow() or
+ *                 procEntry_fast.
+ *=======================================================================*/
+void 
+processEntry (int tnum, int type)
+/* 
+ *  Read an entry from IFold and/or IFnew, and write to IFmerge.
+ *  tnum is the term number, type is OLD or NEW or MERGE
+ */
+{
+  if (fastMerge)
+    procEntry_fast (tnum, type);
+  else
+    procEntry_slow (tnum, type, Nstatic, m[NEW].nDocs, m[MERGE].nDocs);
+  return;
+}
+
+
+/*=======================================================================
+ * readTerm()
+ *=======================================================================*/
+void 
+readTerm (int x)
+/*
+ *  Read the next term from the appropriate ".invf.dict" file.
+ *  x should be either OLD or NEW.
+ *  Sets m[x].done to true when that lexicon has been completely read.
+ */
+{
+  int i;
+
+  if (m[x].term_count >= m[x].nTerms)
+    {
+      m[x].term[0] = 0;
+      m[x].done = 1;
+      return;
+    }
+  m[x].pref = fgetc (m[x].dict);
+  m[x].suff = fgetc (m[x].dict);
+  m[x].term[0] = m[x].pref + m[x].suff;
+
+  for (i = 0; i < m[x].suff; i++)
+    {
+      m[x].term[m[x].pref + i + 1] = fgetc (m[x].dict);
+    }
+  m[x].term_count = m[x].term_count + 1;
+
+  fread ((char *) &(m[x].fcnt), sizeof (u_long), 1, m[x].dict);
+  fread ((char *) &(m[x].wcnt), sizeof (u_long), 1, m[x].dict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(m[x].fcnt);
+  NTOHUL(m[x].wcnt);
+
+  return;
+}
+
+
+/*=======================================================================
+ * writeTerm()
+ *=======================================================================*/
+void 
+writeTerm (int x)
+/*
+ *  Write the current term to m[MERGE].dict file.
+ *  x is OLD, NEW or MERGE.
+ */
+{
+  unsigned char i, prefix, suffix;
+  if (x == MERGE)
+    {
+      m[MERGE].fcnt = m[OLD].fcnt + m[NEW].fcnt;
+      m[MERGE].wcnt = m[OLD].wcnt + m[NEW].wcnt;
+      m[MERGE].pref = m[OLD].pref;
+      m[MERGE].suff = m[OLD].suff;
+      memcpy (m[MERGE].term, m[OLD].term, m[OLD].term[0] + 1);
+    }
+  if (x == OLD)
+    {
+      m[MERGE].fcnt = m[OLD].fcnt;
+      m[MERGE].wcnt = m[OLD].wcnt;
+    }
+  if (x == NEW)
+    {
+      m[MERGE].fcnt = m[NEW].fcnt;
+      m[MERGE].wcnt = m[NEW].wcnt;
+    }
+  prefix = prefixlen (prevTerm, m[x].term);
+  suffix = m[x].term[0] - prefix;
+  fputc (prefix, m[MERGE].dict);	/* prefix length */
+  fputc (suffix, m[MERGE].dict);	/* suffix length */
+  for (i = 0; i < suffix; i++)
+    fputc (m[x].term[prefix + i + 1], m[MERGE].dict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(m[MERGE].fcnt);
+  HTONUL(m[MERGE].wcnt);
+
+  fwrite ((char *) &(m[MERGE].fcnt), sizeof (m[MERGE].fcnt), 1, m[MERGE].dict);
+  fwrite ((char *) &(m[MERGE].wcnt), sizeof (m[MERGE].wcnt), 1, m[MERGE].dict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(m[MERGE].fcnt);
+  NTOHUL(m[MERGE].wcnt);
+
+  memcpy (prevTerm, m[x].term, prefix + suffix + 1);
+  m[MERGE].idh.total_bytes += m[x].term[0] + 1;
+  m[MERGE].idh.index_string_bytes += m[x].term[0] + 2 - prefix;
+  fflush (m[MERGE].dict);
+}
+
+
+/*=======================================================================
+ * process_merge_invf(): The main loop
+ *=======================================================================*/
+int 
+process_merge_invf (void)
+/*
+ *  The main function to merge the lexicons and inverted files.
+ */
+{
+/*** Some extra Initialisation stuff first ***/
+
+  m[OLD].term[0] = m[NEW].term[0] = '\0';
+  m[OLD].term_count = m[NEW].term_count = 0;
+  m[OLD].done = m[NEW].done = 0;
+  m[MERGE].idh.total_bytes = m[MERGE].idh.index_string_bytes = 0;
+
+/*** malloc arrays: mergedata, m[OLD].fcntlist, m[NEW].fcntlist ***/
+  mergedata = malloc ((m[OLD].nTerms + m[NEW].nTerms) * sizeof (char));
+  if (mergedata == 0)
+    {
+      fprintf (stderr, "MALLOC error!\n");
+      exit (1);
+    }
+  {
+    int i;
+    for (i = 0; i < 2; i++)
+      {
+	m[i].fcntlist = malloc ((m[OLD].nTerms + m[NEW].nTerms) * sizeof (int));
+	if (m[i].fcntlist == 0)
+	  {
+	    fprintf (stderr, "MALLOC error!\n");
+	    exit (1);
+	  }
+      }
+  }
+
+/*========================*/
+/***    LEXICON PASS    ***/
+/*========================*/
+
+/*** read first terms ***/
+  readTerm (OLD);
+  readTerm (NEW);
+
+  m[MERGE].nTerms = 0;
+  while (m[OLD].done == 0 || m[NEW].done == 0)
+    {
+      int i;
+      if (m[OLD].done)
+	i = 1; /* NEW will always be greater */
+      else if (m[NEW].done)
+	i = -1; /* OLD will always be greater */
+      else
+	i = casecompare (m[OLD].term, m[NEW].term);
+
+      if (i < 0)
+	{			/* term in OLD only */
+	  mergedata[m[MERGE].nTerms] = (char) OLD;
+	  m[OLD].fcntlist[m[MERGE].nTerms] = m[OLD].fcnt;
+	  m[NEW].fcntlist[m[MERGE].nTerms] = 0;
+	  m[MERGE].nTerms++;
+	  writeTerm (OLD);
+	  readTerm (OLD);
+	}
+      if (i == 0)
+	{			/* term in both lexions */
+	  mergedata[m[MERGE].nTerms] = (char) MERGE;
+	  m[OLD].fcntlist[m[MERGE].nTerms] = m[OLD].fcnt;
+	  m[NEW].fcntlist[m[MERGE].nTerms] = m[NEW].fcnt;
+	  m[MERGE].nTerms++;
+	  writeTerm (MERGE);
+	  readTerm (OLD);
+	  readTerm (NEW);
+	}
+      if (i > 0)
+	{			/* term in NEW only */
+	  mergedata[m[MERGE].nTerms] = (char) NEW;
+	  m[NEW].fcntlist[m[MERGE].nTerms] = m[NEW].fcnt;
+	  m[OLD].fcntlist[m[MERGE].nTerms] = 0;
+	  m[MERGE].nTerms++;
+	  writeTerm (NEW);
+	  readTerm (NEW);
+	}
+    }/*while*/
+  Message ("%s\n", ElapsedTime (&start, NULL));
+  /* print some results about terms */
+  {
+    fprintf (stderr, "   Terms: OLD = %ld,  NEW = %ld,  MERGE = %ld\n",
+	     m[OLD].nTerms, m[NEW].nTerms, m[MERGE].nTerms);
+    fprintf (stderr, "   In OLD only: %ld,  In NEW only: %ld,  BOTH: %ld\n",
+	     (m[MERGE].nTerms - m[NEW].nTerms),
+	     (m[MERGE].nTerms - m[OLD].nTerms),
+	     (m[OLD].nTerms + m[NEW].nTerms - m[MERGE].nTerms));
+  }
+
+
+/*========================*/
+/***      INVF PASS     ***/
+/*========================*/
+
+  bytes_output = ftell (m[MERGE].invf);
+  BIO_Stdio_Decode_Start (m[OLD].invf, &m[OLD].sbs);
+  BIO_Stdio_Decode_Start (m[NEW].invf, &m[NEW].sbs);
+  BIO_Stdio_Encode_Start (m[MERGE].invf, &m[MERGE].sbs);
+  {
+    int i;
+    for (i = 0; i < m[MERGE].nTerms; i++)
+      {
+	processEntry (i, mergedata[i]);
+      }
+    /* write final .invf.idx pointer */
+    HTONUL(bytes_output);  /* [RPAP - Jan 97: Endian Ordering] */
+    fwrite (&bytes_output, sizeof (u_long), 1, m[MERGE].idx);
+    NTOHUL(bytes_output);  /* [RPAP - Jan 97: Endian Ordering] */
+
+    fprintf (stderr, "Old invf: old only bits: %ld, merged bits: %ld\n",
+	     oldOnlyBits, oldMergeBits);
+  }
+
+  return OK;
+}
+
+
+/*=======================================================================
+ * done_merge_invf(): write headers, close files, append weights to
+ *                     weights file
+ *=======================================================================*/
+int 
+done_merge_invf (void)
+{
+/*** write new lexicon header ***/
+  m[MERGE].idh.dict_size = m[MERGE].nTerms;
+  m[MERGE].idh.lookback = m[OLD].idh.lookback;
+  m[MERGE].idh.input_bytes = m[OLD].idh.input_bytes + m[NEW].idh.input_bytes;
+  m[MERGE].idh.total_bytes = m[OLD].idh.total_bytes + m[NEW].idh.total_bytes;
+  m[MERGE].idh.num_of_docs = m[MERGE].nDocs;
+  m[MERGE].idh.num_of_words = m[OLD].idh.num_of_words + m[NEW].idh.num_of_words;
+  m[MERGE].idh.stemmer_num = m[OLD].idh.stemmer_num;
+  m[MERGE].idh.stem_method = m[OLD].idh.stem_method;
+
+  /* if fastMerge, static num of docs stays the same! */
+  if (fastMerge)
+    m[MERGE].idh.static_num_of_docs = m[OLD].idh.static_num_of_docs;
+  else
+    m[MERGE].idh.static_num_of_docs = m[MERGE].nDocs;
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(m[MERGE].idh.lookback);
+  HTONUL(m[MERGE].idh.dict_size);
+  HTONUL(m[MERGE].idh.total_bytes);
+  HTONUL(m[MERGE].idh.index_string_bytes);
+  HTOND(m[MERGE].idh.input_bytes); /* [RJM 07/97: 4G limit] */
+  HTONUL(m[MERGE].idh.num_of_docs);
+  HTONUL(m[MERGE].idh.static_num_of_docs);
+  HTONUL(m[MERGE].idh.num_of_words);
+  HTONUL(m[MERGE].idh.stemmer_num);
+  HTONUL(m[MERGE].idh.stem_method);
+
+  fseek (m[MERGE].dict, magicsize, 0);
+  fwrite (&(m[MERGE].idh), sizeof (struct invf_dict_header), 1, m[MERGE].dict);
+
+/*** write new inverted file header ***/
+  m[MERGE].ifh.no_of_words = m[MERGE].nTerms;
+  m[MERGE].ifh.no_of_ptrs = m[OLD].ifh.no_of_ptrs + m[NEW].ifh.no_of_ptrs;
+  m[MERGE].ifh.skip_mode = m[OLD].ifh.skip_mode;
+  m[MERGE].ifh.InvfLevel = m[OLD].ifh.InvfLevel;
+
+  /* ifh.params[16] -- I have NO IDEA what these do!!!!! */
+  /* only set if a inverted file is in skipped format, I think */
+  {
+    int i;
+    for (i = 0; i < 16; i++)
+      m[MERGE].ifh.params[i] = m[OLD].ifh.params[i];
+  }
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(m[MERGE].ifh.no_of_words);
+  HTONUL(m[MERGE].ifh.no_of_ptrs);
+  HTONUL(m[MERGE].ifh.skip_mode);
+  {
+    int i;
+    for (i = 0; i < 16; i++)
+      HTONUL(m[MERGE].ifh.params[i]);
+  }
+  HTONUL(m[MERGE].ifh.InvfLevel);
+
+  fseek (m[MERGE].invf, magicsize, 0);	/* go to start of header */
+  fwrite (&(m[MERGE].ifh), sizeof (struct invf_file_header), 1, m[MERGE].invf);
+
+/*** Write new document weights --- leave old weights alone ***/
+  if ((weightOption) && (m[OLD].ifh.InvfLevel >= 2))
+    {
+      FILE *weightfile = open_file (m[MERGE].fname, WEIGHTS_SUFFIX, "r+b",
+				    MAGIC_WGHT, MG_CONTINUE);  /* [RPAP - Feb 97: WIN32 Port] */
+      if (weightfile)
+	{
+	  u_long i;  /* [RPAP - Jan 97: Endian Ordering] */
+
+	  fprintf (stderr, "writing new weights: Nnew = %ld\n", m[NEW].nDocs);
+	  fseek (weightfile, 0, 2);	/* end of file */
+
+	  /* [RPAP - Jan 97: Endian Ordering] */
+	  for (i = 0; i < m[NEW].nDocs; i++)
+	    HTONF(DocWeightBuffer[i]);
+
+	  fwrite ((char *) DocWeightBuffer, sizeof (float), m[NEW].nDocs,
+		  weightfile);
+	}
+      fclose (weightfile);
+    }
+
+
+  /* [RPAP - Feb 97: Level 3 Merge] */
+  if (m[OLD].ifh.InvfLevel == 3)
+    {
+      /* Update paragraph file */
+      int buffer_size = 1000;
+      u_char *buffer;
+      int num;
+
+      if (!(buffer = (u_char *) Xmalloc (buffer_size)))
+	FatalError (1, "Could not allocate memory for paragraph buffer");
+
+      fseek (m[MERGE].para, 0, SEEK_END);
+      while ((num = fread ((char *) buffer, sizeof (*buffer), buffer_size, m[NEW].para)) != 0)
+	fwrite ((char *) buffer, sizeof (*buffer), num, m[MERGE].para);
+
+      Xfree (buffer);
+    }
+
+/*** close files ***/
+  fclose (m[MERGE].invf);
+  fclose (m[OLD].invf);
+  fclose (m[NEW].invf);
+  fclose (m[MERGE].dict);
+  fclose (m[OLD].dict);
+  fclose (m[NEW].dict);
+  fclose (m[MERGE].idx);
+  fclose (m[OLD].idx);
+  fclose (m[NEW].idx);
+
+  /* [RPAP - Feb 97: Level 3 Merge] */
+  if (m[OLD].ifh.InvfLevel == 3)
+    {
+      fclose (m[NEW].para);
+      fclose (m[MERGE].para);
+    }
+
+  return OK;
+}
+
+
+/*=======================================================================
+ * usage()
+ *=======================================================================*/
+void 
+usage (char *progname)
+{
+  fprintf (stderr, "usage:  %s [-s] [-w] [-d directory] -f mg_collection\n",
+	   progname);
+  fprintf (stderr, "  example: %s -s -w -f MERGE/mailfiles\n", progname);
+  exit (1);
+}
+
+
+/*=======================================================================
+ * main()
+ *=======================================================================*/
+int 
+main (int argc, char *argv[])
+{
+  int ch;
+  char *progname;
+  progname = argv[0];
+  m[MERGE].fname[0] = '\0';
+
+  /* message and timing information */
+  msg_prefix = argv[0];
+
+  while ((ch = getopt (argc, argv, "f:d:swh")) != -1)
+    switch (ch)
+      {
+      case 'f':
+	strcpy (m[MERGE].fname, optarg);
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 's':
+	fastMerge = 0;
+	break;
+      case 'w':
+	weightOption = 1;
+	break;
+      case 'h':
+      case '?':
+      default:
+	usage (progname);
+      }
+
+  if (m[MERGE].fname[0] == '\0')
+    usage (progname);
+  strcpy (m[OLD].fname, m[MERGE].fname);
+  strcat (m[OLD].fname, ".old");
+  strcpy (m[NEW].fname, m[MERGE].fname);
+  strcat (m[NEW].fname, ".new");
+
+  GetTime (&start);
+  init_merge_invf ();
+  process_merge_invf ();
+  done_merge_invf ();
+  Message ("%s\n", ElapsedTime (&start, NULL));
+  exit (0);
+}
+
+/*************************************************************************/
+/*   EOF: mg_invf_merge_new.c                                            */
+/*************************************************************************/
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/mg_invf_rebuild.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/mg_invf_rebuild.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/mg_invf_rebuild.1	(revision 699)
@@ -0,0 +1,156 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_invf_rebuild 1 \*(Dt CITRI
+.SH NAME
+mg_invf_rebuild \- Rebuild an mg inverted file with or without skips.
+.SH SYNOPSIS
+.B mg_invf_rebuild
+.RB [ \-h ]
+.if n .ti +9n
+[
+.BR \-0 " |"
+.B \-1
+[
+.BI \-k " num"
+] |
+.B \-2
+[
+.BI \-s " num"
+]
+[
+.BI \-m " num"
+]
+]
+.if n .ti +9n
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_invf_rebuild
+builds a new inverted file, with or without skipping, from an old
+inverted file.  This file uses
+.I *.invf.ORG
+and
+.I *.invf.idx.ORG
+as the
+source from which it builds
+.I *.invf
+and
+.IR *.invf.idx .
+If
+.I *.invf.ORG
+or
+.I *.invf.idx.ORG
+do not exist, the program renames
+.I *.invf
+or
+.I *.invf.idx
+to
+.I *.invf.ORG
+or
+.I *.invf.idx.ORG
+as appropriate.  The old inverted file may contain skipping.  This
+means that it is possible to delete the
+.I *.ORG
+files
+after the new inverted file is built.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-m\fP \fInum\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-0
+This generates a non-skipped inverted file.  This option is normally
+only needed if the
+.I *.ORG
+files have been deleted.
+.TP
+.B \-1
+This generates a skipped inverted file.  The
+.BI \-k " num"
+argument specifies the number of pointers hopped over with each skip.
+.TP
+.B \-2
+This option generates a skipped inverted file.  The skipped inverted
+file is built so that it is `optimal' for ranking using a specific
+number of accumulators.  Each term in the inverted file has a
+different skip length.  The arguments
+.BR \-s " and " \-m
+control the sizes of the skips.
+.TP
+.BI \-k " num"
+This specifies the number of pointers that should be hopped over with
+each skip.  This option is only valid if
+.B \-1
+is specified.
+.TP
+.BI \-m " num"
+This specifies the intended number of accumulators that will be used
+when ranking queries are done on the collection.
+.TP
+.BI \-s " num"
+This specifies the minimum size for skips.  If the calculation of the
+optimal skip size results in a number smaller than
+.IR num ,
+the skip size is set to
+.IR num .
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.ORG
+Original inverted file.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.TP
+.B *.invf.idx.ORG
+The original index into the inverted file.
+.TP
+.B *.invf.dict.build
+Compressed stemmed dictionary.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/mg_invf_rebuild.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/mg_invf_rebuild.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/mg_invf_rebuild.c	(revision 699)
@@ -0,0 +1,564 @@
+/**************************************************************************
+ *
+ * mg_invf_rebuild.c -- Program to rebuild an inverted file with skipping
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "timing.h"
+#include "bitio_m.h"
+#include "bitio_m_stdio.h"
+#include "bitio_gen.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "invf.h"
+#include "locallib.h"
+#include "words.h"
+#include "mg.h"
+
+#define LEN 40
+#define WRD 1
+
+typedef struct invf_info
+  {
+    unsigned long doc_num, count, bits_so_far, count_bits;
+  }
+invf_info;
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:18:11  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:46  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:10  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.4  1994/11/29  00:32:03  tes
+   * Committing the new merged files and changes.
+   *
+   * Revision 1.3  1994/10/20  03:56:56  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:51  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+static char pathname[256];
+
+static void process_files (char *filename);
+
+static int k = -1;
+static int mode = 0;
+static int max_nodes = -1;
+static int mins = -1;
+
+void 
+usage (char *pgname)
+{
+  fprintf (stderr, "usage: %s [-f input_file]"
+       "[-d data directory] -0|-1|-2 [-k num] [-m num] [-s num]\n", pgname);
+  printf ("Mode 0: (the default)\n");
+  printf ("\tk, m, and s have no meaning and if specified produce an error\n");
+  printf ("Mode 1:\n");
+  printf ("\tm, and s have no meaning and if specified produce an error\n");
+  printf ("\tk The size of skips\n");
+  printf ("Mode 2:\n");
+  printf ("\tk has no meaning and if specified will produce an error\n");
+  printf ("\tm is the the number of accumulators that will be used for the\n");
+  printf ("\t  ranking. The program builds an inverted file that is \n");
+  printf ("\t  \"optimal\" for that number of accumulators.\n");
+  printf ("\ts is the minimum size for skips.\n");
+  exit (1);
+}
+
+
+
+int main (int argc, char **argv)
+{
+  ProgTime start;
+  char *dir_name, *file_name = "";
+  int ch;
+  msg_prefix = argv[0];
+  dir_name = getenv ("MGDATA");
+  strcpy (pathname, dir_name ? dir_name : ".");
+  opterr = 0;
+  msg_prefix = argv[0];
+  while ((ch = getopt (argc, argv, "012hf:d:k:b:s:m:")) != -1)
+    switch (ch)
+      {
+      case '0':
+	mode = 0;
+	break;
+      case '1':
+	mode = 1;
+	break;
+      case '2':
+	mode = 2;
+	break;
+      case 'f':		/* input file */
+	file_name = optarg;
+	break;
+      case 'd':
+	strcpy (pathname, optarg);
+	break;
+      case 'k':
+	k = atoi (optarg);
+	break;
+      case 'm':
+	max_nodes = atoi (optarg);
+	break;
+      case 's':
+	mins = atoi (optarg);
+	if (mins < 1)
+	  FatalError (1, "The number for the -m option must be greater"
+		      " than or equal to 1");
+	break;
+      case 'h':
+      case '?':
+	usage (argv[0]);
+      }
+
+  if ((mode == 0 && (k != -1 || max_nodes != -1 || mins != -1)) ||
+      (mode == 1 && (max_nodes != -1 || mins != -1)) ||
+      (mode == 2 && (k != -1)))
+    {
+      Message ("Illegal parameters for mode");
+      usage (argv[0]);
+    }
+
+  if (mode == 1 && k == -1)
+    {
+      k = 8;
+      Message ("k is required for mode 1: defaulting k to %d", k);
+    }
+  if (mode == 2 && max_nodes == -1)
+    {
+      max_nodes = 1024;
+      Message ("m is required for mode 2: defaulting m to %d", max_nodes);
+    }
+  if (mode == 2 && mins == -1)
+    {
+      mins = 1;
+      Message ("s is required for mode 2: defaulting s to %d", mins);
+    }
+
+
+  GetTime (&start);
+  process_files (file_name);
+  Message ("%s\n", ElapsedTime (&start, NULL));
+  Message ("**** Don\'t forget to rebuild the stemmed dictionary with mg_invf_dict. ****\n");
+  Message ("**** If the collection was built with stem indexes don\'t forget to     ****\n");
+  Message ("**** rebuild them with mg_stem_idx.                                     ****\n");
+
+  return 0;
+}
+
+
+
+
+
+static void 
+process_files (char *filename)
+{
+  FILE *in, *out, *idx, *odx, *dict;
+  unsigned long magic, outmode, N, in_k, out_k;
+  unsigned long bits_out, bytes_out, i, j;
+  stdio_bitio_state out_buf, in_buf;
+  struct invf_dict_header idh;
+  struct invf_file_header ifh_in, ifh_out;
+  char FName[256];
+
+  outmode = mode;
+
+  /* open .invf.ORG, rename .invf if have to */
+  sprintf (FName, FILE_NAME_FORMAT ".ORG", pathname, filename, INVF_SUFFIX);  /* [RPAP - Feb 97: WIN32 Port] */
+  if (!(in = fopen (FName, "rb")))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      char fname[256];
+      sprintf (fname, FILE_NAME_FORMAT, pathname, filename, INVF_SUFFIX);  /* [RPAP - Feb 97: WIN32 Port] */
+      rename (fname, FName);
+      if (!(in = fopen (FName, "rb")))  /* [RPAP - Feb 97: WIN32 Port] */
+	FatalError (1, "Unable to open \"%s\".\n", FName);
+    }
+  else
+    {
+      char fname[256];
+      sprintf (fname, FILE_NAME_FORMAT, pathname, filename, INVF_SUFFIX);  /* [RPAP - Feb 97: WIN32 Port] */
+      unlink (fname);
+    }
+  Message ("Opening \"%s\"\n", FName);
+
+  /* check the magic number for .invf.ORG */
+  if (fread (&magic, sizeof (magic), 1, in) != 1 || NTOHUL(magic) != MAGIC_INVF)  /* [RPAP - Jan 97: Endian Ordering] */
+    FatalError (1, "Bad magic number in \"%s\".\n", FName);
+
+
+  /* open .invf.idx.ORG, rename .invf.idx if have to */
+  sprintf (FName, FILE_NAME_FORMAT ".ORG", pathname, filename, INVF_IDX_SUFFIX);  /* [RPAP - Feb 97: WIN32 Port] */
+  if (!(idx = fopen (FName, "rb")))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      char fname[256];
+      sprintf (fname, FILE_NAME_FORMAT, pathname, filename, INVF_IDX_SUFFIX);
+      rename (fname, FName);
+      if (!(idx = fopen (FName, "rb")))  /* [RPAP - Feb 97: WIN32 Port] */
+	FatalError (1, "Unable to open \"%s\".\n", FName);
+    }
+  else
+    {
+      char fname[256];
+      sprintf (fname, FILE_NAME_FORMAT, pathname, filename, INVF_IDX_SUFFIX);
+      unlink (fname);
+    }
+  Message ("Opening \"%s\"\n", FName);
+
+  /* check the magic number for .invf.idx.ORG */
+  if (fread (&magic, sizeof (magic), 1, idx) != 1 || NTOHUL(magic) != MAGIC_INVI)  /* [RPAP - Jan 97: Endian Ordering] */
+    FatalError (1, "Bad magic number in \"%s\".\n", FName);
+
+  sprintf (FName, FILE_NAME_FORMAT, pathname, filename, INVF_SUFFIX);  /* [RPAP - Feb 97: WIN32 Port] */
+  Message ("Creating \"%s\"\n", FName);
+  if (!(out = fopen (FName, "wb")))  /* [RPAP - Feb 97: WIN32 Port] */
+    FatalError (1, "Unable to open \"%s\".\n", FName);
+
+  sprintf (FName, FILE_NAME_FORMAT, pathname, filename, INVF_IDX_SUFFIX);  /* [RPAP - Feb 97: WIN32 Port] */
+  Message ("Creating \"%s\"\n", FName);
+  if (!(odx = fopen (FName, "wb")))  /* [RPAP - Feb 97: WIN32 Port] */
+    FatalError (1, "Unable to open \"%s\".\n", FName);
+
+  sprintf (FName, FILE_NAME_FORMAT, pathname, filename, INVF_DICT_SUFFIX);  /* [RPAP - Feb 97: WIN32 Port] */
+  Message ("Opening \"%s\"\n", FName);
+  if (!(dict = fopen (FName, "rb")))  /* [RPAP - Feb 97: WIN32 Port] */
+    FatalError (1, "Unable to open \"%s\".\n", FName);
+
+  if (fread (&magic, sizeof (magic), 1, dict) != 1 || NTOHUL(magic) != MAGIC_STEM_BUILD)  /* [RPAP - Jan 97: Endian Ordering] */
+    FatalError (1, "Bad magic number in \"%s\".\n", FName);
+
+  fread ((char *) &idh, sizeof (idh), 1, dict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(idh.lookback);
+  NTOHUL(idh.dict_size);
+  NTOHUL(idh.total_bytes);
+  NTOHUL(idh.index_string_bytes);
+  NTOHD(idh.input_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(idh.num_of_docs);
+  NTOHUL(idh.static_num_of_docs);
+  NTOHUL(idh.num_of_words);
+  NTOHUL(idh.stemmer_num);
+  NTOHUL(idh.stem_method);
+
+  HTONUL2(MAGIC_INVF, magic);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite ((char *) &magic, sizeof (magic), 1, out);
+
+  fread ((char *) &ifh_in, sizeof (ifh_in), 1, in);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(ifh_in.no_of_words);
+  NTOHUL(ifh_in.no_of_ptrs);
+  NTOHUL(ifh_in.skip_mode);
+  for (i = 0; i < 16; i++)
+    NTOHUL(ifh_in.params[i]);
+  NTOHUL(ifh_in.InvfLevel);
+
+  ifh_out = ifh_in;
+  ifh_out.skip_mode = outmode;
+  bzero ((char *) ifh_out.params, sizeof (ifh_out.params));
+  switch (outmode)
+    {
+    case 0:
+      break;
+    case 1:
+      ifh_out.params[0] = k;
+      break;
+    case 2:
+      ifh_out.params[0] = max_nodes;
+      ifh_out.params[1] = mins;
+      break;
+    }
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(ifh_out.no_of_words);
+  HTONUL(ifh_out.no_of_ptrs);
+  HTONUL(ifh_out.skip_mode);
+  for (i = 0; i < 16; i++)
+    HTONUL(ifh_out.params[i]);
+  HTONUL(ifh_out.InvfLevel);
+
+  fwrite ((char *) &ifh_out, sizeof (ifh_out), 1, out);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(ifh_out.no_of_words);
+  NTOHUL(ifh_out.no_of_ptrs);
+  NTOHUL(ifh_out.skip_mode);
+  for (i = 0; i < 16; i++)
+    NTOHUL(ifh_out.params[i]);
+  NTOHUL(ifh_out.InvfLevel);
+
+  Message ("The file is a level %d inverted file.\n", ifh_in.InvfLevel);
+
+  bits_out = ftell (out) * 8;
+
+
+  HTONUL2(MAGIC_INVI, magic);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite ((char *) &magic, sizeof (magic), 1, odx);
+
+  DECODE_START (in)
+    DECODE_PAUSE (in_buf)
+
+    ENCODE_START (out)
+    ENCODE_PAUSE (out_buf)
+
+    N = idh.num_of_docs;
+
+  for (i = 0; i < ifh_in.no_of_words; i++)
+    {
+      unsigned long blk, p;
+      unsigned long odn_blk = 0, olen_blk = 0;
+      unsigned long idn_blk = 0, ilen_blk = 0;
+      register unsigned long suff;
+      unsigned long fcnt, wcnt, doc_num, bits_so_far, last_major;
+      unsigned long next_mjr_dn, kd;
+      char dummy2[MAXSTEMLEN + 1];
+      invf_info *ii;
+
+      fgetc (dict);
+      suff = fgetc (dict);
+      fread (dummy2, sizeof (u_char), suff, dict);
+      fread ((char *) &fcnt, sizeof (fcnt), 1, dict);
+      fread ((char *) &wcnt, sizeof (wcnt), 1, dict);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fcnt);
+      NTOHUL(wcnt);
+
+      HTONUL2(bits_out >> 3, bytes_out);  /* [RPAP - Jan 97: Endian Ordering] */
+      fwrite ((char *) &bytes_out, sizeof (bytes_out), 1, odx);
+      NTOHUL(bytes_out);  /* [RPAP - Jan 97: Endian Ordering] */
+
+      p = fcnt;
+      blk = BIO_Bblock_Init (idh.static_num_of_docs, p);
+      switch (outmode)
+	{
+	case 1:
+	  {
+	    unsigned long len;
+	    if (p <= ifh_out.params[0])
+	      out_k = 0;
+	    else
+	      {
+		out_k = ifh_out.params[0];
+		len = BIO_Bblock_Bound (N, p);
+		if (ifh_in.InvfLevel >= 2)
+		  len += wcnt;
+		odn_blk = BIO_Bblock_Init (idh.num_of_docs, (p + out_k - 1) / out_k);
+		olen_blk = BIO_Bblock_Init (len, (p + out_k - 1) / out_k);
+	      }
+	    break;
+	  }
+	case 2:
+	  {
+	    unsigned long len;
+	    if (p <= mins)
+	      out_k = 0;
+	    else
+	      {
+		out_k = (int) (2 * sqrt ((double) p / max_nodes));
+		if (out_k <= mins)
+		  out_k = mins;
+		len = BIO_Bblock_Bound (N, p);
+		if (ifh_in.InvfLevel >= 2)
+		  len += wcnt;
+		odn_blk = BIO_Bblock_Init (idh.num_of_docs,
+					   (p + out_k - 1) / out_k);
+		olen_blk = BIO_Bblock_Init (len, (p + out_k - 1) / out_k);
+	      }
+	    break;
+	  }
+	default:
+	  out_k = 0;
+	}
+
+      switch (ifh_in.skip_mode)
+	{
+	case 1:
+	  {
+	    unsigned long len;
+	    if (p <= ifh_in.params[0])
+	      in_k = 0;
+	    else
+	      {
+		in_k = ifh_in.params[0];
+		len = BIO_Bblock_Bound (N, p);
+		if (ifh_in.InvfLevel >= 2)
+		  len += wcnt;
+		idn_blk = BIO_Bblock_Init (idh.num_of_docs, (p + in_k - 1) / in_k);
+		ilen_blk = BIO_Bblock_Init (len, (p + in_k - 1) / in_k);
+	      }
+	    break;
+	  }
+	case 2:
+	  {
+	    unsigned long len;
+	    if (p <= ifh_in.params[1])
+	      {
+		in_k = 0;
+	      }
+	    else
+	      {
+		in_k = (int) (2 * sqrt ((double) p / ifh_in.params[0]));
+		if (in_k <= ifh_in.params[1])
+		  in_k = ifh_in.params[1];
+		len = BIO_Bblock_Bound (N, p);
+		if (ifh_in.InvfLevel >= 2)
+		  len += wcnt;
+		idn_blk = BIO_Bblock_Init (idh.num_of_docs,
+					   (p + in_k - 1) / in_k);
+		ilen_blk = BIO_Bblock_Init (len, (p + in_k - 1) / in_k);
+	      }
+	    break;
+	  }
+	default:
+	  in_k = 0;
+	}
+
+      if (!(ii = Xmalloc (sizeof (invf_info) * p)))
+	FatalError (1, "Unable to allocate memory for \"ii\"\n");
+
+      doc_num = bits_so_far = 0;
+      next_mjr_dn = 0;
+      kd = 0;
+      DECODE_CONTINUE (in_buf)
+	for (j = 0; j < p; j++, kd++)
+	{
+	  unsigned long doc_diff, count = 0;
+	  if (kd == in_k)
+	    kd = 0;
+	  if (in_k && kd == 0 && j + in_k < p)
+	    {
+	      int temp;
+	      BBLOCK_DECODE (next_mjr_dn, idn_blk);
+	      next_mjr_dn += doc_num;
+	      BBLOCK_DECODE (temp, ilen_blk);
+	    }
+	  ii[j].bits_so_far = bits_so_far;
+	  if (in_k && kd == in_k - 1 && j != p - 1)
+	    {
+	      int count;
+	      BBLOCK_LENGTH (next_mjr_dn - doc_num, blk, count);
+	      bits_so_far += count;
+	      doc_num = next_mjr_dn;
+	    }
+	  else
+	    {
+	      BBLOCK_DECODE_L (doc_diff, blk, bits_so_far);
+	      doc_num += doc_diff;
+	    }
+	  ii[j].doc_num = doc_num;
+	  if (ifh_in.InvfLevel >= 2)
+	    {
+	      int count_bits = 0;
+	      GAMMA_DECODE_L (count, count_bits);
+	      ii[j].count_bits = count_bits;
+	      bits_so_far += count_bits;
+	      ii[j].count = count;
+	    }
+	}
+
+      /* read till a byte boundary */
+      while (__btg)
+	{
+	  DECODE_BIT;
+	  bits_so_far++;
+	}
+
+      DECODE_PAUSE (in_buf)
+
+	doc_num = bits_so_far = 0;
+      last_major = 0;
+      kd = 0;
+      ENCODE_CONTINUE (out_buf)
+	for (j = 0; j < p; j++, kd++)
+	{
+	  if (kd == out_k)
+	    kd = 0;
+	  if (out_k && kd == 0)
+	    {
+	      if (j + out_k < p)
+		{
+		  int num = ii[j + out_k - 1].doc_num - last_major;
+		  BBLOCK_ENCODE_L (num, odn_blk, bits_out);
+		  last_major = ii[j + out_k - 1].doc_num;
+
+		  num = ii[j + out_k - 1].bits_so_far + ii[j + out_k - 1].count_bits -
+		    bits_so_far;
+		  BBLOCK_ENCODE_L (num, olen_blk, bits_out);
+		  bits_so_far = ii[j + out_k].bits_so_far;
+		}
+	    }
+	  if (!(out_k && kd == out_k - 1 && j != p - 1))
+	    BBLOCK_ENCODE_L (ii[j].doc_num - doc_num, blk, bits_out);
+	  doc_num = ii[j].doc_num;
+	  if (ifh_in.InvfLevel >= 2)
+	    GAMMA_ENCODE_L (ii[j].count, bits_out);
+	}
+
+      /* write till a byte boundary */
+      while (__btg != 8)
+	{
+	  ENCODE_BIT (0);
+	  bits_out++;
+	}
+      ENCODE_PAUSE (out_buf)
+
+	Xfree (ii);
+
+    }
+  ENCODE_CONTINUE (out_buf)
+    ENCODE_DONE
+
+    HTONUL2(bits_out >> 3, bytes_out);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite ((char *) &bytes_out, sizeof (bytes_out), 1, odx);
+  NTOHUL(bytes_out);
+
+  fclose (idx);
+  fclose (odx);
+  fclose (in);
+  fclose (out);
+  fclose (dict);
+
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/mg_merge.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/mg_merge.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/mg_merge.h	(revision 699)
@@ -0,0 +1,37 @@
+/**************************************************************************
+ *
+ * mgmerge.h --- mgmerge #defines, etc
+ * Copyright (C) 1995  Shane Hudson (shane@cosc.canterbury.ac.nz)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ * Last edtited: November 12 1994
+ *
+ **************************************************************************/
+
+#ifndef MGMERGE_H
+#define MGMERGE_H
+
+typedef unsigned char byte;
+
+#define OLD 0
+#define NEW 1
+#define MERGE 2
+
+#define OK 0
+#define ERROR_MGMERGE 1
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/mg_passes.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/mg_passes.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/mg_passes.1	(revision 699)
@@ -0,0 +1,441 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_passes 1 \*(Dt CITRI
+.SH NAME
+mg_passes \- builds mg databases
+.SH SYNOPSIS
+.B mg_passes
+[
+.B \-h
+]
+[
+.B \-G
+]
+[
+.B \-S
+]
+[
+.B \-D
+]
+[
+.B \-W
+]
+.if n .ti +9n
+[
+.BR \-1 " |"
+.BR \-2 " |"
+.B  \-3
+]
+[
+.BI \-C " compstatpoint"
+]
+.if n .ti +9n
+[
+.BI \-n " tracename"
+]
+.if t .ti +.5i
+[
+.BI \-b " bufsize"
+]
+[
+.BI \-m " memlimit"
+]
+.if n .ti +9n
+[
+.BI \-c " numchunks"
+]
+[
+.BI \-a " stemmer"
+]
+[
+.BI \-s " stemmethod"
+]
+[
+.BI \-t " tracepos"
+]
+.if n .ti +9n
+[
+.B \-T1
+]
+[
+.B \-T2
+]
+.if t .ti +.5i
+[
+.B \-I1
+]
+[
+.B \-I2
+]
+.if n .ti +9n
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+[
+.I filename(s)
+]
+.SH DESCRIPTION
+.B mg_passes
+is the program that does most of the work when building
+.BR mg (1)
+database systems.  The input documents can come from either
+.I stdin
+or from a list of files on the command line.  Individual documents
+must be separated with control-B characters.  In general,
+.B mg_passes
+must be run twice to build a database, first with the
+.B \-T1
+and
+.B \-I1
+options, and second with the
+.B \-T2
+and
+.B \-I2
+options.  Several other programs must be run in order to get an
+.BR mg (1)
+database that is ready for the
+.BR mgquery (1)
+program.  The
+.SB EXAMPLE
+section below gives an example of how to build a complete
+.BR mg (1)
+database.
+.SH OPTIONS
+Options may appear in any order, but the
+.IR filename(s) ,
+if specified, must be last.
+.TP "\w'\fB\-C\fP \fIcompstatpoint\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-G
+Treat SGML tags as non-words when building the inverted file.  An SGML
+tag is anything between angle brackets, i.e., `<' and `>'.
+.TP
+.B \-S
+This option causes a special pass to be executed.  It is up to a user
+to modify
+.I mg.special.c
+in the source code to do something with the documents it is given.
+.TP
+.B \-D
+If
+.B mg_passes
+fails, then print the document that caused the failure to the trace
+file if tracing is active, or to
+.I stderr
+if it is not.
+.TP
+.B \-W
+This option enables the generation of the weights file when
+.B \-I2
+is specified.  It causes
+.B \-I2
+to use a little more memory and CPU.
+.TP
+.B \-1
+Produce a level-1 inverted file.  This option is only useful when
+specified with
+.BR "\-I1 ".
+A level-1 inverted file makes it possible for
+.BR mgquery (1)
+to do Boolean queries.  Ranked queries can still be done,
+although the quality of the ranking is abysmal.
+.TP
+.B \-2
+Produce a level-2 inverted file.  This option is only useful when
+specified with
+.BR "\-I1 ".
+This is the default when neither
+.BR \-1 ", " "\-2 " "or " \-3
+is specified.
+A level-2 inverted file makes it possible for
+.BR mgquery (1)
+to do Boolean queries and cosine-ranked queries.
+.TP
+.B \-3
+Produce a level-3 inverted file.  This option is only useful when
+specified with
+.BR "\-I1 ".
+This has been implemented to enable paragraph-level inversion.
+Paragraphs are delimited by control-C characters in the source text.
+.TP
+.BI \-C " compstatpoint"
+This option causes statistics on the compression performance to be
+output to a file called
+.IR *.compression.stats .
+.I compstatpoint
+specifies the interval between outputting each line of statistics.  The
+units of
+.I compstatpoint
+are kilobytes of source text.  E.g., if
+.I compstatpoint
+is 10, then a line is output to the file every 10 KB of input
+source.  Each line of the file consists of 4 numbers The first number
+is the amount of input text, in bytes, processed so far.  The second
+number is the amount of input text, in bytes, processed since the
+last line was output to the file.  The third number is the number of
+output bytes generated since the last line was output to the file, and
+the fourth number is the compression achieved since the last line was
+output, i.e., the third number divided by the second number.
+.TP
+.BI \-n " tracename"
+This specifies the filename to use for the trace log, if tracing is
+enabled using the
+.B \-t
+option.  If
+.BI \-n " tracename"
+is not given and tracing is enabled, a default trace filename will be
+used.
+.TP
+.BI \-s " stemmethod"
+This specifies the method to use to \*(lqstem\*(rq the words in the
+inverted file dictionary.  This is a bit mask specifying the
+operations to do on words as they are parsed out of the text, where
+bit number 0 is the low-order (rightmost) bit.  Bit 0 does case
+folding, and bit 1 does simple stemming, so the value 3 for
+.I stemmethod
+does both case folding and stemming.
+.TP
+.BI \-a " stemmer"
+This specifies the stemmer to use when stemming words.  This
+is a description of the language the stemmer is intended for
+or a description of the stemmer.  Valid options include:
+english, lovin, french, and simplefrench.
+.TP
+.BI \-b " bufsize"
+Specify the size of the document buffer in kilobytes.  If any document
+is larger than
+.IR bufsize ,
+the program will abort with an error message.  This should probably be
+replaced with some system which automatically increases the buffer
+size as required.  The default size is 3072 KB (3 MB).
+.TP
+.BI \-m " memlimit"
+Maximum amount of memory to use for the pass-2 file inversion in
+megabytes.  This option is only useful when used in conjunction with
+the option
+.BR \-I1 .
+The larger this value, the faster the pass-2 inversion will proceed.
+The default value is 5 MB.
+.TP
+.BI \-c " numchunks"
+The maximum number of inversion chunks to write to disk.  Each chunk
+will be approximately as large as
+.IR memlimit .
+This option is only useful when used in conjunction with the option
+.BR \-I2 .
+The larger this value, the faster the pass-2 inversion will proceed.
+The default value is 5 MB.
+.TP
+.BI \-t " tracepos"
+This option activates tracing.  A line will be generated in the
+trace file for every
+.I tracepos
+input bytes processed.  The default name for the trace file can be
+overridden using the
+.BI \-n " tracename"
+option.
+.TP
+.B \-T1
+Generate the
+.I *.text.stats
+file.
+.TP
+.B \-T2
+Generate the
+.IR *.text ,
+.IR *.text.idx ,
+and possibly the
+.I *.text.dict.aux
+files.  Using this option requires that the
+.I *.text.dict
+file be present.
+.TP
+.B \-I1
+Generate the
+.IR *.invf.dict ,
+.IR *.invf.chunk ,
+and
+.I *.invf.chunk.trans
+files.
+.TP
+.B \-I2
+Generate the
+.I *.invf
+and
+.I *.invf.idx
+files.  Using this option requires
+that the
+.IR *.invf.dict.hash ,
+.IR *.invf.chunk ,
+and
+.I *.invf.chunk.trans
+files
+be present.  The
+.I *.invf.dict.hash
+file is generated by
+.BR mg_perf_hash_build (1)
+from the
+.I *.invf.dict.build
+file.  If the
+.B \-W
+option is specified, the
+.I *.weight
+file will also be generated.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection is to be
+written.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection that will be
+created.
+.TP
+.I filename(s)
+This specifies the source text. If this is not specified, then the
+program expects the source text from
+.IR stdin .
+.SH EXAMPLE
+What follows is a UNIX
+.BR csh (1)
+script as an example of how to build an
+.BR mg (1)
+document collection.
+.LP
+.nf
+.DT
+.ft B
+.I #! /bin/csh
+.I
+# The first argument on the command line specifies the
+.I
+#   source of the text
+set source = ($1)
+.PP
+.I
+# The second argument is the name of the collection
+set text = ($2)
+.PP
+.I
+# Create *.text.stats,  *.invf.dict.build,
+.I
+#   *.invf.chunk and *.invf.chunks.trans
+${source} | mg_passes -T1 -I1 -m 1 -t 1 -f ${text}
+.PP
+.I
+# Create *.text.dict
+mg_compression_dict -f ${text}
+.PP
+.I
+# Create *.invf.dict.hash
+mg_perf_hash_build -f ${text}
+.PP
+.I
+# Create *.text,  *.text.idx,
+.I
+#   *.invf and *.invf.idx
+${source} | mg_passes -T2 -I2 -c 2 -t 1 -f ${text}
+.PP
+.I
+# Create *.text.idx.wgt and *.weight.approx
+mg_weights_build -f ${text} -b 8
+.PP
+.I
+# Create *.invf.dict
+mg_invf_dict -f ${text} -b 4096
+.PP
+.I
+# Create *.text.dict
+mg_fast_comp_dict -f ${text}
+.ft R
+.fi
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.chunk
+Inverted file chunk descriptor file.  When the inverted file is
+created it is created in chunks that use no more than a set amount of
+memory.  This file describes those chunks.
+.TP
+.B *.invf.chunk.trans
+Word-occurrence-order to lexical-order translation file.  The
+.B *.invf.chunk
+file is written in word-occurrence order but is required by
+.B \-I2
+to be in lexical order.
+.TP
+.B *.invf.dict.build
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict.hash
+Data for an order-preserving perfect hash function.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.B *.text
+Compressed documents.
+.TP
+.B *.text.stats
+Statistics about the text.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.trace
+The default trace file.
+.TP
+.B *.compression.stats
+Statistics about the compression of the text.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/mg_passes.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/mg_passes.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/mg_passes.c	(revision 699)
@@ -0,0 +1,625 @@
+/**************************************************************************
+ *
+ * mg_passes.c -- Driver for the various passes
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#ifdef HAVE_MALLINFO
+# include <malloc.h> 
+#endif
+
+#include "memlib.h"
+#include "messages.h"
+#include "timing.h"
+
+#include "mg_files.h"
+#include "mg.h"
+#include "build.h"
+#include "text.h"
+#include "stemmer.h"
+
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:18:12  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.3  1998/12/17 09:12:53  rjmcnab
+
+   Altered mg to process utf-8 encoded Unicode. The main changes
+   are in the parsing of the input, the casefolding, and the stemming.
+
+   Revision 1.2  1998/11/25 07:55:47  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:13  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:57  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:52  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#define MAX_PASSES 5
+
+#define SPECIAL 1
+#define TEXT_PASS_1 2
+#define TEXT_PASS_2 4
+#define IVF_PASS_1 8
+#define IVF_PASS_2 16
+
+#define MIN_BUF 8192
+#define	TERMRECORD	'\002'
+
+unsigned long buf_size = 3 * 1024 * 1024;	/* 3Mb */
+unsigned long invf_buffer_size = 5 * 1024 * 1024;	/* 5Mb */
+unsigned long ChunkLimit = 0;
+char InvfLevel = 2;
+char SkipSGML = 0;
+char MakeWeights = 0;
+FILE *Comp_Stats = NULL;
+int comp_stat_point = 0;
+double bytes_processed = 0; /* [RJM 07/97: 4G limit] */
+double bytes_received = 0; /* [RJM 07/97: 4G limit] */
+int stemmer_num = 0; /* default to the lovin stemmer */
+int stem_method = 0;
+
+static char Passes = 0;
+static unsigned long trace = 0;
+static int Dump = 0;
+static char **files = NULL;
+static int num_files = 0;
+static char *trace_name = NULL;
+
+
+typedef struct pass_data
+  {
+    char *name;
+    int (*init) (char *);
+    int (*process) (u_char *, int);
+    int (*done) (char *);
+#ifdef HAVE_TIMES
+    clock_t init_time;
+    clock_t process_time;
+    clock_t done_time;
+#else
+    struct timeval init_time;
+    struct timeval process_time;
+    struct timeval done_time;
+#endif
+  }
+pass_data;
+
+#ifdef HAVE_TIMES
+#define NULL_TIMES 0, 0, 0
+#else
+#define NULL_TIMES {0, 0}, {0, 0}, {0, 0}
+#endif
+
+static pass_data PassData[MAX_PASSES] =
+{
+  {"special", init_special, process_special, done_special, NULL_TIMES},
+  {"text.pass1", init_text_1, process_text_1, done_text_1, NULL_TIMES},
+  {"text.pass2", init_text_2, process_text_2, done_text_2, NULL_TIMES},
+  {"ivf.pass1", init_ivf_1, process_ivf_1, done_ivf_1, NULL_TIMES},
+  {"ivf.pass2", init_ivf_2, process_ivf_2, done_ivf_2, NULL_TIMES},
+};
+
+static char *usage_str = "\nUSAGE:\n"
+"  %s [-h] [-G] [-D] [-1|-2|-3] [-T1] [-T2] [-I1] [-I2] [-N1]\n"
+"  %*s [-N2] [-W] [-S] [-b buffer-size] [-d dictionary-directory]\n"
+"  %*s [-t trace-point Mb] [-m invf-memory] [-c chunk-limit]\n"
+"  %*s [-n trace-name] [-C comp-stat-size] [-s stem_method]\n"
+"  %*s [-a stemmer] -f doc-collection-name\n";
+
+
+
+
+
+
+
+
+
+static void 
+usage (char *err)
+{
+  if (err)
+    Message (err);
+  fprintf (stderr, usage_str, msg_prefix, strlen (msg_prefix), "",
+	   strlen (msg_prefix), "", strlen (msg_prefix), "");
+  exit (1);
+}
+
+
+
+
+#if 0
+static char *
+str_comma (unsigned long u)
+{
+  static char buf[20];
+  unsigned long a, b, c, d;
+  a = u / 1000000000;
+  u -= a * 1000000000;
+  b = u / 1000000;
+  u -= b * 1000000;
+  c = u / 1000;
+  u -= c * 1000;
+  d = u;
+
+  if (a)
+    sprintf (buf, "%u,%03u,%03u,%03u", a, b, c, d);
+  else if (b)
+    sprintf (buf, "%u,%03u,%03u", b, c, d);
+  else if (c)
+    sprintf (buf, "%u,%03u", c, d);
+  else
+    sprintf (buf, "%u", d);
+  return (buf);
+}
+#endif
+
+
+
+
+int 
+open_next_file (int in_fd)
+{
+  if (in_fd > 0)
+    close (in_fd);
+  if (num_files == 0)
+    return (-1);
+  if ((in_fd = open (files[0], O_RDONLY)) == -1)
+    FatalError (1, "Cannot open %s", files[0]);
+  files++;
+  num_files--;
+  return (in_fd);
+}
+
+
+static void 
+driver (int in_fd, FILE * Trace, char *file_name)
+{
+  int pass, num = 1;
+
+  char *buffer = Xmalloc (buf_size);
+  unsigned long num_docs = 0;
+  unsigned long block_bytes = 0;
+  register int buf_left = buf_size;
+  register char *look_pos = buffer;
+  register char *end_pos = buffer;
+
+  ProgTime StartTime, InitTime, ProcTime, DoneTime;
+
+  GetTime (&StartTime);
+
+  for (pass = 0; pass < MAX_PASSES; pass++)
+    if (Passes & (1 << pass))
+      {
+	pass_data *pd = &PassData[pass];
+#ifdef HAVE_TIMES
+	struct tms tims;
+	times (&tims);
+	pd->init_time -= tims.tms_utime + tims.tms_stime;
+#elif defined(HAVE_GETRUSAGE)  /* [RPAP - Feb 97: WIN32 Port] */
+	struct rusage ru;
+
+	getrusage (RUSAGE_SELF, &ru);
+	pd->init_time.tv_sec -= ru.ru_utime.tv_sec + ru.ru_stime.tv_sec;
+	pd->init_time.tv_usec -= ru.ru_utime.tv_usec + ru.ru_stime.tv_usec;
+#endif
+	if (pd->init (file_name) == COMPERROR)
+	  FatalError (1, "Error during init of \"%s\"", pd->name);
+
+#ifdef HAVE_TIMES
+	times (&tims);
+	pd->init_time += tims.tms_utime + tims.tms_stime;
+#elif defined(HAVE_GETRUSAGE)  /* [RPAP - Feb 97: WIN32 Port] */
+	getrusage (RUSAGE_SELF, &ru);
+	pd->init_time.tv_sec += ru.ru_utime.tv_sec + ru.ru_stime.tv_sec;
+	pd->init_time.tv_usec += ru.ru_utime.tv_usec + ru.ru_stime.tv_usec;
+	time_normalise (&pd->init_time);
+#endif
+      }
+
+  GetTime (&InitTime);
+  for (;;)
+    {
+      int len = 0;
+      char *base = look_pos;
+
+      while (look_pos != end_pos && *look_pos != TERMRECORD)
+	look_pos++;
+
+      while (look_pos == end_pos)
+	{
+	  if (buf_left < MIN_BUF)
+	    {
+	      bcopy (base, buffer, end_pos - base);
+	      look_pos = buffer + (end_pos - base);
+	      buf_left = buf_size - (end_pos - base);
+	      end_pos = look_pos;
+	      base = buffer;
+	    }
+	  if (buf_left)
+	    {
+    	      num = read (in_fd, end_pos, buf_left);
+	      if (num < 0) num = 0; /* RJM - quick hack :-) */
+	      if (num == 0)
+		if ((in_fd = open_next_file (in_fd)) != -1)
+		  num = read (in_fd, end_pos, buf_left);
+	      bytes_received += num;
+	      buf_left -= num;
+	      end_pos += num;
+	    }
+	  while (look_pos < end_pos && *look_pos != TERMRECORD)
+	    look_pos++;
+	  if (buf_left == 0 && base == buffer && look_pos == end_pos)
+	    {
+	      Message ("Unable to find document terminator (i.e ^B)"
+		       " in the document");
+	      FatalError (1, "The document is in excess of %d chars long",
+			  look_pos - base);
+	    }
+	  if (!num)
+	    break;
+	}
+      len = look_pos++ - base;
+
+      if (!num && base == end_pos)
+	break;
+
+      bytes_processed += len;
+
+#ifndef QUIET
+      if (!len)
+	Message ("Warning : Processing zero length document");
+#endif
+
+      for (pass = 0; pass < MAX_PASSES; pass++)
+	if (Passes & (1 << pass))
+	  {
+	    register pass_data *pd = &PassData[pass];
+
+#ifdef HAVE_TIMES
+	    struct tms tims;
+	    times (&tims);
+	    pd->process_time -= tims.tms_utime + tims.tms_stime;
+#elif defined(HAVE_GETRUSAGE)  /* [RPAP - Feb 97: WIN32 Port] */
+	    struct rusage ru;
+	    register struct timeval *tv = &pd->process_time;
+
+	    getrusage (RUSAGE_SELF, &ru);
+	    tv->tv_sec -= ru.ru_utime.tv_sec + ru.ru_stime.tv_sec;
+	    tv->tv_usec -= ru.ru_utime.tv_usec + ru.ru_stime.tv_usec;
+#endif
+
+	    if (pd->process ((u_char *) base, len) == COMPERROR)
+	      {
+		Message ("Error during processing of \"%s\"", pd->name);
+		if (Dump || Trace)
+		  {
+		    int i;
+		    FILE *f = Trace ? Trace : stderr;
+		    fprintf (f, "-=- * -=- * -=- * -=- * -=- * -=- * -=-\n");
+		    for (i = 0; i < len; i++)
+		      {
+			char ch = base[i];
+			if (ch == '\1' || ch == '\2')
+			  ch = '\n';
+			putc (ch, f);
+		      }
+		    fprintf (f, "-=- * -=- * -=- * -=- * -=- * -=- * -=-\n");
+		  }
+		if (Trace)
+		  fprintf (Trace, "%10.0f bytes |%7lu docs | %s\n",
+			   bytes_processed, num_docs,
+			   ElapsedTime (&StartTime, NULL));
+		exit (1);
+	      }
+
+#ifdef HAVE_TIMES
+	    times (&tims);
+	    pd->process_time += tims.tms_utime + tims.tms_stime;
+#elif defined(HAVE_GETRUSAGE)  /* [RPAP - Feb 97: WIN32 Port] */
+	    getrusage (RUSAGE_SELF, &ru);
+	    tv->tv_sec += ru.ru_utime.tv_sec + ru.ru_stime.tv_sec;
+	    tv->tv_usec += ru.ru_utime.tv_usec + ru.ru_stime.tv_usec;
+#endif
+	  }
+      num_docs++;
+      if (Trace)
+	{
+	  block_bytes += (look_pos - base);
+	  if (block_bytes >= trace)
+	    {
+#ifdef HAVE_MALLINFO
+	      struct mallinfo mi;
+	      mi = mallinfo ();
+	      block_bytes -= trace;
+	      fprintf (Trace, "%10.0f bytes |%7lu docs |%7.3f Mb | %s\n",
+		       bytes_processed, num_docs, mi.arena / 1024.0 / 1024.0,
+		       ElapsedTime (&StartTime, NULL));
+#else
+	      block_bytes -= trace;
+	      fprintf (Trace, "%10.0f bytes |%7lu docs | %s\n",
+		       bytes_processed, num_docs,
+		       ElapsedTime (&StartTime, NULL));
+#endif
+	    }
+	}
+      if (!num && look_pos - 1 == end_pos)
+	break;
+    }
+
+#ifndef HAVE_TIMES
+  for (pass = 0; pass < MAX_PASSES; pass++)
+    if (Passes & (1 << pass))
+      time_normalise (&PassData[pass].process_time);
+#endif
+
+  GetTime (&ProcTime);
+
+  for (pass = 0; pass < MAX_PASSES; pass++)
+    if (Passes & (1 << pass))
+      {
+	pass_data *pd = &PassData[pass];
+#ifdef HAVE_TIMES
+	struct tms tims;
+	times (&tims);
+	pd->done_time -= tims.tms_utime + tims.tms_stime;
+#elif defined(HAVE_GETRUSAGE)  /* [RPAP - Feb 97: WIN32 Port] */
+	struct rusage ru;
+
+	getrusage (RUSAGE_SELF, &ru);
+	pd->done_time.tv_sec -= ru.ru_utime.tv_sec + ru.ru_stime.tv_sec;
+	pd->done_time.tv_usec -= ru.ru_utime.tv_usec + ru.ru_stime.tv_usec;
+#endif
+	if (pd->done (file_name) == COMPERROR)
+	  FatalError (1, "Error during done of \"%s\"", pd->name);
+
+#ifdef HAVE_TIMES
+	times (&tims);
+	pd->done_time += tims.tms_utime + tims.tms_stime;
+#elif defined(HAVE_GETRUSAGE)  /* [RPAP - Feb 97: WIN32 Port] */
+	getrusage (RUSAGE_SELF, &ru);
+	pd->done_time.tv_sec += ru.ru_utime.tv_sec + ru.ru_stime.tv_sec;
+	pd->done_time.tv_usec += ru.ru_utime.tv_usec + ru.ru_stime.tv_usec;
+	time_normalise (&pd->done_time);
+#endif
+      }
+  if (Trace)
+    {
+#ifdef HAVE_MALLINFO
+      struct mallinfo mi;
+      mi = mallinfo ();
+      fprintf (Trace, "%10.0f bytes |%7lu docs |%7.3f Mb | %s\n",
+	       bytes_processed, num_docs, mi.arena / 1024.0 / 1024.0,
+	       ElapsedTime (&StartTime, NULL));
+#else
+      fprintf (Trace, "%10.0f bytes |%7lu docs | %s\n",
+	       bytes_processed, num_docs,
+	       ElapsedTime (&StartTime, NULL));
+#endif
+    }
+
+  GetTime (&DoneTime);
+
+  Message ("");
+  Message ("%10s      :    init         process        done", "");
+  for (pass = 0; pass < MAX_PASSES; pass++)
+    if (Passes & (1 << pass))
+      {
+	pass_data *pd = &PassData[pass];
+	char it[15], pt[15], dt[15];
+#ifdef HAVE_TIMES
+	strcpy (it, cputime_string (pd->init_time));
+	strcpy (pt, cputime_string (pd->process_time));
+	strcpy (dt, cputime_string (pd->done_time));
+#else
+	strcpy (it, cputime_string (&pd->init_time));
+	strcpy (pt, cputime_string (&pd->process_time));
+	strcpy (dt, cputime_string (&pd->done_time));
+#endif
+	Message ("%-10s      : %s   %s   %s", pd->name, it, pt, dt);
+      }
+  Message ("");
+  Message ("Init time       : %s", ElapsedTime (&StartTime, &InitTime));
+  Message ("Process time    : %s", ElapsedTime (&InitTime, &ProcTime));
+  Message ("Done time       : %s", ElapsedTime (&ProcTime, &DoneTime));
+  Message ("Total time      : %s", ElapsedTime (&StartTime, &DoneTime));
+  Message ("Documents       : %u", num_docs);
+  Message ("Bytes received  : %.0f", bytes_received);
+  Message ("Bytes processed : %.0f", bytes_processed);
+  Message ("Process Rate    : %.1f kB per cpu second",
+   (double) bytes_processed / (ProcTime.CPUTime - InitTime.CPUTime) / 1024);
+  free (buffer);
+}
+
+
+
+int main (int argc, char **argv)
+{
+  int ch, in_fd;
+  char *filename = NULL;
+  FILE *Trace = NULL;
+
+  msg_prefix = argv[0];
+
+  opterr = 0;
+  while ((ch = getopt (argc, argv, "hC:WGSD123f:d:b:T:I:t:m:N:c:n:s:a:")) != -1)
+    {
+      switch (ch)
+	{
+	case 'G':
+	  SkipSGML = 1;
+	  break;
+	case 'S':
+	  Passes |= SPECIAL;
+	  break;
+	case '1':
+	  InvfLevel = 1;
+	  break;
+	case '2':
+	  InvfLevel = 2;
+	  break;
+	case '3':
+	  InvfLevel = 3;
+	  break;
+	case 'f':
+	  filename = optarg;
+	  break;
+	case 'n':
+	  trace_name = optarg;
+	  break;
+	case 'D':
+	  Dump = 1;
+	  break;
+	case 'W':
+	  MakeWeights = 1;
+	  break;
+	case 'd':
+	  set_basepath (optarg);
+	  break;
+	case 'a':
+	  stemmer_num = stemmernumber (optarg);
+	  break;
+	case 's':
+	  stem_method = atoi (optarg) & STEMMER_MASK;
+	  break;
+	case 'b':
+	  buf_size = atoi (optarg) * 1024;
+	  break;
+	case 'C':
+	  comp_stat_point = atoi (optarg) * 1024;
+	  break;
+	case 'c':
+	  ChunkLimit = atoi (optarg);
+	  break;
+	case 'm':
+	  invf_buffer_size = (int) (atof (optarg) * 1024 * 1024);
+	  break;
+	case 'I':
+	case 'N': /* N kept for compatability */
+	  if (*optarg == '1')
+	    Passes |= IVF_PASS_1;
+	  else if (*optarg == '2')
+	    Passes |= IVF_PASS_2;
+	  else
+	    usage ("Invalid pass number");
+	  break;
+	case 'T':
+	  if (*optarg == '1')
+	    Passes |= TEXT_PASS_1;
+	  else if (*optarg == '2')
+	    Passes |= TEXT_PASS_2;
+	  else
+	    usage ("Invalid pass number");
+	  break;
+	case 't':
+	  trace = (unsigned long) (atof (optarg) * 1024 * 1024);
+	  break;
+	case 'h':
+	case '?':
+	  usage (NULL);
+	}
+    }
+
+  if (!filename || *filename == '\0')
+    FatalError (1, "A document collection name must be specified.");
+
+  if (buf_size < MIN_BUF)
+    FatalError (1, "The buffer size must exceed 1024 bytes.");
+
+  if ((Passes & (IVF_PASS_1 | IVF_PASS_2)) == (IVF_PASS_1 | IVF_PASS_2))
+    FatalError (1, "I1 and I2 cannot be done simultaneously.");
+
+  if ((Passes & (TEXT_PASS_1 | TEXT_PASS_2)) == (TEXT_PASS_1 | TEXT_PASS_2))
+    FatalError (1, "T1 and T2 cannot be done simultaneously.");
+
+  if (!Passes)
+    FatalError (1, "S, T1, T2, I1 or I2 must be specified.");
+
+  if (optind < argc)
+    {
+      if ((in_fd = open (argv[optind], O_RDONLY)) == -1)
+	FatalError (1, "Cannot open %s", argv[optind]);
+      files = &argv[optind + 1];
+      num_files = argc - (optind + 1);
+    }
+  else
+    in_fd = 0;			/* stdin */
+
+
+  if (trace)
+    {
+      if (!trace_name)
+	trace_name = make_name (filename, TRACE_SUFFIX, NULL);
+      if (!(Trace = fopen (trace_name, "a")))
+	Message ("Unable to open \"%s\". No tracing will be done.", trace_name);
+      else
+	setbuf (Trace, NULL);
+    }
+  else
+    Trace = NULL;
+
+  if (comp_stat_point)
+    {
+      char *name = make_name (filename, COMPRESSION_STATS_SUFFIX, NULL);
+      if (!(Comp_Stats = fopen (name, "wb")))  /* [RPAP - Feb 97: WIN32 Port] */
+	Message ("Unable to open \"%s\". No comp. stats. will be generated.",
+		 name);
+    }
+
+
+  if (Trace)
+    {
+      int i;
+      fprintf (Trace, "\n\n\t\t-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\n\n");
+      for (i = 0; i < argc; i++)
+	fprintf (Trace, "%s ", argv[i]);
+      fprintf (Trace, "\n\n");
+    }
+
+  driver (in_fd, Trace, filename);
+
+  if (Trace)
+    fclose (Trace);
+
+  if (Comp_Stats)
+    fclose (Comp_Stats);
+
+  return 0;
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/mg_perf_hash_build.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/mg_perf_hash_build.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/mg_perf_hash_build.1	(revision 699)
@@ -0,0 +1,89 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_perf_hash_build 1 \*(Dt CITRI
+.SH NAME
+mg_perf_hash_build \- generate an order-preserving hash function for the stemmed dictionary
+.SH SYNOPSIS
+.B mg_perf_hash_build
+[
+.B \-h
+]
+[
+.BI \-r " rndseed"
+]
+[
+.BI \-d  " directory"
+]
+.if n .ti +9n
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_perf_hash_build
+generates an order-preserving hashing function from the compressed
+stemmed dictionary and writes it out to disk.
+.BR mg_passes (1)
+will make use of the hash function when it builds the inverted file.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-r " rndseed"
+This specifies the random seed to be used in the generation of the
+hash function; it must be an integer.  If this is not specified, the
+current time will be used as the random seed.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf.dict
+Stemmed dictionary.
+.TP
+.B *.invf.dict.hash
+Data for a order-preserving perfect hash function.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/mg_perf_hash_build.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/mg_perf_hash_build.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/mg_perf_hash_build.c	(revision 699)
@@ -0,0 +1,183 @@
+/**************************************************************************
+ *
+ * mg_perf_hash_build.c -- Program to build a perfect hash function
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "memlib.h"
+#include "messages.h"
+#include "timing.h"
+#include "local_strings.h"
+#include "perf_hash.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "invf.h"
+#include "locallib.h"
+#include "words.h"
+#include "mg.h"
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:18:13  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:47  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:15  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:58  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:53  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+
+#define POOL_SIZE 1024*1024
+
+static void process_files (char *filename);
+
+int r = -1;
+
+int main (int argc, char **argv)
+{
+  ProgTime start;
+  char *file_name = "";
+  int ch;
+  msg_prefix = argv[0];
+  opterr = 0;
+  while ((ch = getopt (argc, argv, "f:d:r:h")) != -1)
+    switch (ch)
+      {
+      case 'f':		/* input file */
+	file_name = optarg;
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 'r':
+	r = atoi (optarg);
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-f input_file]"
+		 "[-d data directory] [-r random seed] [-h]\n", argv[0]);
+	exit (1);
+      }
+
+  GetTime (&start);
+  process_files (file_name);
+  Message ("%s\n", ElapsedTime (&start, NULL));
+  return 0;
+}
+
+
+
+
+
+static void 
+process_files (char *filename)
+{
+  FILE *dict, *hash;
+  unsigned long i;
+  u_char prev[MAXSTEMLEN + 1];
+  struct invf_dict_header idh;
+  perf_hash_data *phd;
+  u_char *pool;
+  int pool_left;
+  u_char **starts;
+
+
+  dict = open_file (filename, INVF_DICT_SUFFIX, "rb",
+		    MAGIC_STEM_BUILD, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  fread ((char *) &idh, sizeof (idh), 1, dict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(idh.lookback);
+  NTOHUL(idh.dict_size);
+  NTOHUL(idh.total_bytes);
+  NTOHUL(idh.index_string_bytes);
+  NTOHD(idh.input_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(idh.num_of_docs);
+  NTOHUL(idh.static_num_of_docs);
+  NTOHUL(idh.num_of_words);
+  NTOHUL(idh.stemmer_num);
+  NTOHUL(idh.stem_method);
+
+  hash = create_file (filename, INVF_DICT_HASH_SUFFIX, "wb",
+		      MAGIC_HASH, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  if (!(pool = Xmalloc (POOL_SIZE)))
+    FatalError (1, "Out of memory");
+  pool_left = POOL_SIZE;
+
+  if (!(starts = Xmalloc (sizeof (u_char *) * idh.dict_size)))
+    FatalError (1, "Out of memory");
+
+  for (i = 0; i < idh.dict_size; i++)
+    {
+      register unsigned long copy, suff, l;
+      unsigned long wcnt, fcnt;
+
+      /* build a new word on top of prev */
+      copy = getc (dict);
+      suff = getc (dict);
+      *prev = copy + suff;
+      fread (prev + copy + 1, sizeof (u_char), suff, dict);
+
+      /* read other data, but no need to store it */
+      fread (&fcnt, sizeof (fcnt), 1, dict);
+      fread (&wcnt, sizeof (wcnt), 1, dict);
+
+      l = *prev + 1;
+      if (pool_left < l)
+	{
+	  pool = Xmalloc (POOL_SIZE);
+	  pool_left = POOL_SIZE;
+	}
+      starts[i] = pool;
+      bcopy ((char *) prev, (char *) pool, l);
+      pool += l;
+      pool_left -= l;
+    }
+  if (!(phd = gen_hash_func (idh.dict_size, starts, r)))
+    FatalError (1, "Unable to generate hash function");
+  if (write_perf_hash_data (hash, phd) == -1)
+    FatalError (1, "Unable to write hash function");
+  fclose (dict);
+  fclose (hash);
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/mg_stem_idx.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/mg_stem_idx.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/mg_stem_idx.1	(revision 699)
@@ -0,0 +1,95 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_stem_idx 1 \*(Dt CITRI
+.SH NAME
+mg_stem_idx \- builds a stem index file
+.SH SYNOPSIS
+.B mg_stem_idx
+[
+.B \-h
+]
+[
+.B \-F
+]
+[
+.BI \-b " num"
+]
+[
+.BI \-d " directory"
+]
+.if n .ti +9n
+.B \-s 1|2|3
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_stem_idx
+generates a stem index file for a collection that is to be built with
+stem indexes.  This program should be called three times: once for each
+.B -s
+parameter.  It uses the stemmed dictionary to create the stem index which
+contains pointers into the stemmed dictionary.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-b " num"
+The dictionary is stored in blocks on disk; this option is used to set
+the block size.  The default is 4-KB blocks.
+.TP
+.B \-F
+This option causes the disk blocks to be aligned on the appropriate
+boundaries.  For example, if 4-KB blocks are used, they will be aligned
+in the file on 4-KB boundaries.
+.TP
+.B -s 1|2|3
+The stem method to apply for the stem index.
+.br
+1 = casefolded and non-stemmed
+.br
+2 = non-casefolded and stemmed
+.br
+3 = casefolded and stemmed
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 25
+.B *.invf.dict.blocked
+Compressed stemmed dictionary.
+.TP
+.B *.invf.dict.blocked.1
+Stem index with stem index method 1.
+.TP
+.B *.invf.dict.blocked.2
+Stem index with stem index method 2.
+.TP
+.B *.invf.dict.blocked.3
+Stem index with stem index method 3.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_invf_dict (1),
+.BR mgstemidxlist (1).
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/mg_stem_idx.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/mg_stem_idx.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/mg_stem_idx.c	(revision 699)
@@ -0,0 +1,892 @@
+/**************************************************************************
+ *
+ * mg_stem_idx.c -- Memory efficient stem index builder
+ * Copyright (C) 1997  Ross Peeters
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "filestats.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "invf.h"
+#include "mg.h"
+#include "locallib.h"
+#include "backend.h"     /* For struct stemmed_dict */
+#include "words.h"
+#include "stemmer.h"
+#include "hash.h"
+#include "local_strings.h"
+
+typedef struct PosEntry
+{
+  unsigned int num_cases;
+  unsigned int blk;
+  unsigned short blk_index;
+  unsigned short offset;
+}
+PosEntry;
+
+
+typedef struct PosList
+{
+  unsigned int list_size;
+  unsigned int num_entries;
+  PosEntry PE[1];
+}
+PosList;
+
+
+typedef struct idx_hash_rec
+{
+  u_char *word;
+  PosList *PL;
+}
+idx_hash_rec;
+
+
+#define POOL_SIZE 1024*1024
+#define HASH_POOL_SIZE 8192
+#define INITIAL_HASH_SIZE 7927
+
+
+static unsigned long MaxMemInUse = 0;
+static unsigned long MemInUse = 0;
+
+static idx_hash_rec **IdxHashTable;
+static unsigned long IdxHashSize;
+static unsigned long IdxHashUsed;
+static u_char *IdxPool;
+static int IdxPoolLeft;
+
+static idx_hash_rec *idx_hr_pool;
+static int idx_hr_PoolLeft;
+
+static idx_hash_rec **idx_first_occr;
+static int idx_max_first_occr;
+
+int block_size = 1024 * 4;
+
+int force = 0;
+
+static long lookback = 4;
+
+static void 
+ChangeMem (int Change)
+{
+  MemInUse += Change;
+  if (MemInUse > MaxMemInUse)
+    MaxMemInUse = MemInUse;
+}
+
+
+/* =========================================================================
+ * Function: MakePosList
+ * Description:
+ * Input:
+ * Output:
+ * ========================================================================= */
+PosList *
+MakePosList (int n)
+{
+  PosList *pl;
+  int list_size = (n == 0 ? 1 : n);	/* always allocate at least one node */
+
+  pl = Xmalloc (sizeof (PosList) + (list_size - 1) * sizeof (PosEntry));
+  if (!pl)
+    FatalError (1, "Unable to allocate term list");
+  ChangeMem (sizeof (PosList) + (list_size - 1) * sizeof (PosEntry));
+
+  pl->num_entries = n;
+  pl->list_size = list_size;
+
+  return pl;
+}
+
+/* =========================================================================
+ * Function: ResizePosList
+ * Description:
+ * Input:
+ * Output:
+ * ========================================================================= */
+
+#define GROWTH_FACTOR 2
+#define MIN_SIZE 2
+
+static void
+ResizePosList (PosList ** pos_list)
+{
+  PosList *pl = *pos_list;
+
+  ChangeMem (-(sizeof (PosList) + (pl->list_size - 1) * sizeof (PosEntry)));
+  if (pl->num_entries > pl->list_size)
+    {
+      if (pl->list_size)
+	pl->list_size *= GROWTH_FACTOR;
+      else
+	pl->list_size = MIN_SIZE;
+    }
+  pl = Xrealloc (pl, sizeof (PosList) + (pl->list_size - 1) * sizeof (PosEntry));
+
+  if (!pl)
+    FatalError (1, "Unable to resize pos list");
+  ChangeMem (sizeof (PosList) + (pl->list_size - 1) * sizeof (PosEntry));
+
+  *pos_list = pl;
+}
+
+
+/* =========================================================================
+ * Function: FreePosList
+ * Description:
+ * Input:
+ * Output:
+ * ========================================================================= */
+
+void
+FreePosList (PosList ** the_pl)
+{
+  PosList *pl = *the_pl;
+
+  ChangeMem (-(sizeof(PosList) + sizeof (PosEntry) * (pl->list_size - 1)));
+  Xfree (pl);
+
+  *the_pl = NULL;
+}
+
+
+/* =========================================================================
+ * Function: ResetPosList
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+void
+ResetPosList (PosList ** pl)
+{
+  if (*pl)
+    FreePosList (pl);
+  *pl = MakePosList (0);
+}
+
+/* =========================================================================
+ * Function: AddPosEntry
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+int
+AddPosEntry (PosList ** pos_list, PosEntry * pe)
+{
+  PosList *pl = *pos_list;
+
+  pl->num_entries++;
+  ResizePosList (pos_list);
+  pl = *pos_list;
+
+  /* copy the structure contents */
+  bcopy ((char *) pe, (char *) &(pl->PE[pl->num_entries - 1]), sizeof (PosEntry));
+
+  return pl->num_entries - 1;
+}
+
+
+/* Modified from stem_search.c */
+stemmed_dict *
+ReadStemDictBlk (File * stem_file)
+{
+  unsigned long i;
+  stemmed_dict *sd;
+  u_char *buffer;
+
+  if (!(sd = Xmalloc (sizeof (stemmed_dict))))
+    FatalError (1, "Could not allocate memory for stemmed dict");
+
+
+  sd->stem_file = stem_file;
+  sd->MemForStemDict = 0;
+
+  Fread (&sd->sdh, sizeof (sd->sdh), 1, stem_file);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(sd->sdh.lookback);
+  NTOHUL(sd->sdh.block_size);
+  NTOHUL(sd->sdh.num_blocks);
+  NTOHUL(sd->sdh.blocks_start);
+  NTOHUL(sd->sdh.index_chars);
+  NTOHUL(sd->sdh.num_of_docs);
+  NTOHUL(sd->sdh.static_num_of_docs);
+  NTOHUL(sd->sdh.num_of_words);
+  NTOHUL(sd->sdh.stemmer_num);
+  NTOHUL(sd->sdh.stem_method);
+  NTOHUL(sd->sdh.indexed);
+
+  if (!(buffer = Xmalloc (sd->sdh.index_chars)))
+    {
+      Xfree (sd);
+      FatalError (1, "Could not allocate memory for stemmed dict");
+      return (NULL);
+    };
+  sd->MemForStemDict += sd->sdh.index_chars;
+
+  if (!(sd->index = Xmalloc (sd->sdh.num_blocks * sizeof (*sd->index))))
+    {
+      Xfree (sd);
+      Xfree (buffer);
+      FatalError (1, "Could not allocate memory for stemmed dict");
+      return (NULL);
+    };
+  sd->MemForStemDict += sd->sdh.num_blocks * sizeof (*sd->index);
+
+  if (!(sd->pos = Xmalloc (sd->sdh.num_blocks * sizeof (*sd->pos))))
+    {
+      Xfree (sd);
+      Xfree (buffer);
+      Xfree (sd->index);
+      FatalError (1, "Could not allocate memory for stemmed dict");
+      return (NULL);
+    };
+  sd->MemForStemDict += sd->sdh.num_blocks * sizeof (*sd->pos);
+
+  if (!(sd->buffer = Xmalloc (sd->sdh.block_size * sizeof (*sd->buffer))))
+    {
+      Xfree (sd);
+      Xfree (buffer);
+      Xfree (sd->index);
+      Xfree (sd->buffer);
+      FatalError (1, "Could not allocate memory for stemmed dict");
+      return (NULL);
+    };
+  sd->MemForStemDict += sd->sdh.block_size * sizeof (*sd->buffer);
+
+  sd->active = -1;
+
+  for (i = 0; i < sd->sdh.num_blocks; i++)
+    {
+      register u_char len;
+      sd->index[i] = buffer;
+      len = Getc (stem_file);
+      *buffer++ = len;
+      Fread (buffer, sizeof (u_char), len, stem_file);
+      buffer += len;
+      Fread (&sd->pos[i], sizeof (*sd->pos), 1, stem_file);
+      NTOHUL(sd->pos[i]);  /* [RPAP - Jan 97: Endian Ordering] */
+    }
+
+  return sd;
+}
+
+
+void
+init_process ()
+{
+  /* Allocate memory for idx hash table */
+
+  if (!(IdxPool = Xmalloc (POOL_SIZE)))
+    FatalError (1, "Unable to allocate memory for idx pool");
+  IdxPoolLeft = POOL_SIZE;
+  ChangeMem (POOL_SIZE);
+
+  if (!(idx_hr_pool = Xmalloc (HASH_POOL_SIZE * sizeof (idx_hash_rec))))
+    FatalError (1, "Unable to allocate memory for idx pool");
+  idx_hr_PoolLeft = HASH_POOL_SIZE;
+  ChangeMem (HASH_POOL_SIZE * sizeof (idx_hash_rec));
+
+  IdxHashSize = INITIAL_HASH_SIZE;
+  IdxHashUsed = 0;
+  if (!(IdxHashTable = Xmalloc (sizeof (idx_hash_rec *) * IdxHashSize)))
+    FatalError (1, "Unable to allocate memory for idx table");
+  ChangeMem (sizeof (idx_hash_rec *) * IdxHashSize);
+  bzero ((char *) IdxHashTable, sizeof (idx_hash_rec *) * IdxHashSize);
+  idx_max_first_occr = 8192;
+  if (!(idx_first_occr = Xmalloc (sizeof (idx_hash_rec *) * idx_max_first_occr)))
+    FatalError (1, "Unable to allocate memory for idx_first_occur");
+  ChangeMem (sizeof (idx_hash_rec *) * idx_max_first_occr);
+}
+
+
+void
+PackIdxHashTable (void)
+{
+  int s, d;
+  for (s = d = 0; s < IdxHashSize; s++)
+    if (IdxHashTable[s])
+      IdxHashTable[d++] = IdxHashTable[s];
+  ChangeMem (-sizeof (idx_hash_rec *) * IdxHashSize);
+  ChangeMem (sizeof (idx_hash_rec *) * IdxHashUsed);
+  if (!(IdxHashTable = Xrealloc (IdxHashTable, sizeof (idx_hash_rec *) * IdxHashUsed)))
+    FatalError (1, "Out of memory");
+  IdxHashSize = IdxHashUsed;
+}
+
+
+void
+process_stem_dict (stemmed_dict * sd, int stem_method)
+{
+  int block;
+  short blk_index = -1;
+  int wordnum = -1;
+  PosEntry *prevPE = NULL;
+  idx_hash_rec *prevIdx = NULL;
+  u_char word[MAXSTEMLEN + 1];
+  u_char prev[MAXSTEMLEN + 1];
+
+  /* For each block in stem dict... */
+  for (block = 0; block < sd->sdh.num_blocks; block++)
+    {
+      register unsigned int res;
+      int num_indexes;
+      unsigned long *first_word, *last_invf_len;
+      unsigned short *num_words;
+      u_char *base;
+      unsigned short *index;
+
+      /* Read block into buffer */
+      Fseek (sd->stem_file, sd->pos[block] + sd->sdh.blocks_start, 0);
+      Fread (sd->buffer, sd->sdh.block_size, sizeof (u_char), sd->stem_file);
+      sd->active = sd->pos[block];
+
+      /* Move through block header */
+      first_word = (unsigned long *) (sd->buffer);
+      NTOHUL(*first_word);  /* [RPAP - Jan 97: Endian Ordering] */
+      last_invf_len = (unsigned long *) (first_word + 1);
+      NTOHUL(*last_invf_len);  /* [RPAP - Jan 97: Endian Ordering] */
+      num_words = (unsigned short *) (last_invf_len + 1);
+      NTOHUS(*num_words);  /* [RPAP - Jan 97: Endian Ordering] */
+      index = num_words + 1;
+      num_indexes = ((*num_words - 1) / sd->sdh.lookback) + 1;
+
+      {
+	/* [RPAP - Jan 97: Endian Ordering] */
+	int i;
+	for (i = 0; i < num_indexes; i++)
+	  NTOHUS(index[i]);
+      }
+
+      base = (u_char *) (index + num_indexes);
+      blk_index = -1;
+
+      /* For each word in block... */
+      for (res = 0; res < *num_words; res++)
+	{
+	  unsigned copy, suff;
+	  register idx_hash_rec *idx_ent = 0;
+
+	  /* Update blk_index */
+	  if (!(res % sd->sdh.lookback))
+	    blk_index++;
+
+	  copy = *base++;
+	  suff = *base++;
+	  bcopy ((char *) base, (char *) (prev + copy + 1), suff);
+	  base += suff;
+	  *prev = copy + suff;
+
+	  /* Skip irrelevant word info */
+	  base += sizeof (u_long);
+	  base += sizeof (u_long);
+	  base += sizeof (u_long);
+
+	  /* Stem word */
+	  bcopy ((char *) prev, (char *) word, *prev + 1);
+	  stemmer (stem_method, sd->sdh.stemmer_num, word);
+
+	  /* Check if word follows straight on from previous word */
+	  if (prevIdx && !compare (word, prevIdx->word))
+	    prevPE->num_cases++;
+
+	  else
+	    {
+	      /* Search the idx hash table for word */
+	      register unsigned long hashval, step;
+	      register int hsize = IdxHashSize;
+	      HASH (hashval, step, word, hsize);
+	      for (;;)
+		{
+		  register u_char *s1;
+		  register u_char *s2;
+		  register int len;
+		  idx_ent = IdxHashTable[hashval];
+		  if (!idx_ent)
+		    {
+		      /* Create a next entry in the hash table */
+		      int len = *word + 1;
+		      if (!idx_hr_PoolLeft)
+			{
+			  if (!(idx_hr_pool = Xmalloc (HASH_POOL_SIZE *
+						       sizeof (idx_hash_rec))))
+			    FatalError (1, "Unable to allocate memory for pool");
+			  idx_hr_PoolLeft = HASH_POOL_SIZE;
+			  ChangeMem (HASH_POOL_SIZE * sizeof (idx_hash_rec));
+			}
+		      idx_ent = idx_hr_pool++;
+		      idx_hr_PoolLeft--;
+		      if (len > IdxPoolLeft)
+			{
+			  if (!(IdxPool = Xmalloc (POOL_SIZE)))
+			    FatalError (1, "Unable to allocate memory for pool");
+			  IdxPoolLeft = POOL_SIZE;
+			  ChangeMem (POOL_SIZE);
+			}
+		      wordnum++;
+
+		      idx_ent->word = IdxPool;
+		      idx_ent->PL = MakePosList (0);
+		      {
+			PosEntry PE;
+			PE.num_cases = 1;
+			PE.blk = block;
+			PE.blk_index = (unsigned short) blk_index;
+			PE.offset = res % sd->sdh.lookback;
+			AddPosEntry (&(idx_ent->PL), &PE);
+		      }
+		      prevIdx = idx_ent;
+		      prevPE = &(idx_ent->PL->PE[idx_ent->PL->num_entries - 1]);
+
+		      bcopy ((char *) word, (char *) IdxPool, len);
+		      IdxPool += len;
+		      IdxPoolLeft -= len;
+		      if (IdxHashUsed == idx_max_first_occr - 1)
+			{
+			  ChangeMem (-sizeof (idx_hash_rec *) * idx_max_first_occr);
+			  idx_max_first_occr *= 2;
+			  if (!(idx_first_occr = Xrealloc (idx_first_occr, sizeof (idx_hash_rec *) *
+							   idx_max_first_occr)))
+			    FatalError (1, "Unable to allocate memory for idx_first_occr");
+			  ChangeMem (sizeof (idx_hash_rec *) * idx_max_first_occr);
+			}
+		      idx_first_occr[IdxHashUsed] = idx_ent;
+		      IdxHashUsed++;
+		      IdxHashTable[hashval] = idx_ent;
+		      break;
+		    }
+		  
+		  /* Compare the words */
+		  s1 = word;
+		  s2 = idx_ent->word;
+		  len = *s1 + 1;
+		  for (; len; len--)
+		    if (*s1++ != *s2++)
+		      break;
+		  
+		  if (len)
+		    {
+		      /* Entry is not the right one - move to next hash index */
+		      hashval = (hashval + step);
+		      if (hashval >= hsize)
+			hashval -= hsize;
+		    }
+		  else
+		    {
+		      /* Entry is correct - added PosEntry to word */
+		      PosEntry PE;
+		      PE.num_cases = 1;
+		      PE.blk = block;
+		      PE.blk_index = (unsigned short) blk_index;
+		      PE.offset = res % sd->sdh.lookback;
+		      AddPosEntry (&(idx_ent->PL), &PE);
+		      prevIdx = idx_ent;
+		      prevPE = &(idx_ent->PL->PE[idx_ent->PL->num_entries - 1]);
+		      break;
+		    }
+		}      
+	    }
+
+	  if (IdxHashUsed >= IdxHashSize >> 1)
+	    {
+	      idx_hash_rec **ht;
+	      unsigned long size;
+	      unsigned long i;
+	      size = prime (IdxHashSize * 2);
+	      if (!(ht = Xmalloc (sizeof (idx_hash_rec *) * size)))
+		FatalError (1, "Unable to allocate memory for idx table");
+	      bzero ((char *) ht, sizeof (idx_hash_rec *) * size);
+	      ChangeMem (sizeof (idx_hash_rec *) * size);
+
+	      for (i = 0; i < IdxHashSize; i++)
+		if (IdxHashTable[i])
+		  {
+		    register u_char *wptr;
+		    idx_hash_rec *ent;
+		    register unsigned long hashval, step;
+		    
+		    wptr = IdxHashTable[i]->word;
+		    HASH (hashval, step, wptr, size);
+		    ent = ht[hashval];
+		    while (ent)
+		      {
+			hashval += step;
+			if (hashval >= size)
+			  hashval -= size;
+			ent = ht[hashval];
+		      }
+		    ht[hashval] = IdxHashTable[i];
+		  }
+	      Xfree (IdxHashTable);
+	      ChangeMem (-sizeof (idx_hash_rec *) * IdxHashSize);
+	      IdxHashTable = ht;
+	      IdxHashSize = size;
+	    }
+	  
+	} /* end for each word */
+
+    } /* end for each block */
+
+}
+ 
+static int 
+idx_comp (const void *A, const void *B)
+{
+  u_char *s1 = (*((idx_hash_rec **) A))->word;
+  u_char *s2 = (*((idx_hash_rec **) B))->word;
+  return (casecompare (s1, s2));
+}
+
+
+void
+save_idx (char * filename, int stem_method)
+{
+  char *FName;
+  unsigned long i, j, pos, First_word, num;
+  struct stem_idx_header sih;
+  u_char *buffer, *last_word = NULL;
+  unsigned short *pointers;
+  int buf_in_use;
+  unsigned short ptrs_in_use, word_num;
+  FILE *idbi = NULL, *tmp = NULL;
+
+  FName = make_name (filename, ".tmp", NULL);
+  if (!(tmp = fopen (FName, "w+b")))
+    FatalError (1, "Unable to open \"%s\".\n", FName);
+
+  /* Delete the file now */
+  unlink (FName);
+
+  /* Create appropriate stem index file */
+  switch (stem_method)
+    {
+    case (1):
+      {
+	idbi = create_file (filename, INVF_DICT_BLOCKED_1_SUFFIX, "wb", MAGIC_STEM_1,
+			    MG_ABORT);
+	break;
+      }
+
+    case (2):
+      {
+	idbi = create_file (filename, INVF_DICT_BLOCKED_2_SUFFIX, "wb", MAGIC_STEM_2,
+			    MG_ABORT);
+	break;
+      }
+    case (3):
+      {
+	idbi = create_file (filename, INVF_DICT_BLOCKED_3_SUFFIX, "wb", MAGIC_STEM_3,
+			    MG_ABORT);
+	break;
+      }
+    }
+
+  if (!idbi)
+    FatalError (1, "Could NOT create .invf.blocked.%d file", stem_method);
+
+  PackIdxHashTable();
+  qsort (IdxHashTable, IdxHashUsed, sizeof (idx_hash_rec *), idx_comp);
+
+  sih.lookback = lookback;
+  sih.block_size = block_size;
+  sih.num_blocks = 0;
+  sih.blocks_start = 0;
+  sih.index_chars = 0;
+  sih.num_of_words = IdxHashUsed;
+
+  fwrite ((char *) &sih, sizeof (sih), 1, idbi);
+
+  if (!(buffer = Xmalloc (block_size + 512)))
+    FatalError (1, "Unable to allocate memory for \"buffer\"\n");
+  if (!(pointers = Xmalloc (block_size + 512)))
+    FatalError (1, "Unable to allocate memory for \"buffer\"\n");
+
+  buf_in_use = 0;
+  pos = 0;
+  word_num = 0;
+  ptrs_in_use = 0;
+  First_word = 0;
+
+  /* For each word in the hashtable... */
+  for (i = 0; i < IdxHashUsed; i++)
+    {
+      register unsigned long extra, copy, suff;
+      register struct idx_hash_rec *ent = IdxHashTable[i];
+
+      /* build a new word on top of prev */
+      if (last_word != NULL)
+	copy = prefixlen (last_word, ent->word);
+      else
+	copy = 0;
+      suff = *(ent->word) - copy;
+      last_word = ent->word;
+
+      if (word_num % sih.lookback == 0)
+	/* Will need copy chars to add + a pointer in index */
+	extra = copy + sizeof (*pointers);
+      else
+	extra = 0;
+      if ((ptrs_in_use + 1) * sizeof (*pointers) + sizeof (ptrs_in_use) + extra +
+	  buf_in_use + sizeof (First_word) + suff + 1 + sizeof (ent->PL->num_entries) + 
+	  ent->PL->num_entries * sizeof (PosEntry) > block_size)
+	{
+	  /* Dump buffer to tmp file */
+	  int chunk;
+	  HTONUL(First_word);  /* [RPAP - Jan 97: Endian Ordering] */
+	  HTONUS(word_num);  /* [RPAP - Jan 97: Endian Ordering] */
+	  fwrite (&First_word, sizeof (First_word), 1, tmp);
+	  fwrite (&word_num, sizeof (word_num), 1, tmp);
+	  fwrite (pointers, sizeof (*pointers), ptrs_in_use, tmp);
+	  fwrite (buffer, sizeof (u_char), buf_in_use, tmp);
+	  bzero ((char *) buffer, block_size);
+	  chunk = buf_in_use + ptrs_in_use * sizeof (*pointers) +
+	    sizeof (ptrs_in_use) + sizeof (First_word);
+	  if (force && chunk < block_size)
+	    {
+	      fwrite (buffer, sizeof (u_char), block_size - chunk, tmp);
+	      chunk = block_size;
+	    }
+
+	  pos += chunk;
+
+	  buf_in_use = 0;
+	  word_num = 0;
+	  ptrs_in_use = 0;
+	  sih.num_blocks++;
+
+	  /* Check that entry will fit into new block */
+	  if (sizeof (*pointers) + sizeof (ptrs_in_use) + extra + sizeof (First_word) +
+	      suff + 1 + sizeof (ent->PL->num_entries) + 
+	      ent->PL->num_entries * sizeof (PosEntry) > block_size)
+	    FatalError (1, "Block size to small");
+
+	}
+
+      if (word_num % sih.lookback == 0)
+	{
+	  HTONUS2(buf_in_use, pointers[ptrs_in_use++]);  /* [RPAP - Jan 97: Endian Ordering] */
+	  suff += copy;
+	  copy = 0;
+	}
+
+      /* Output Word information */
+      buffer[buf_in_use++] = copy;
+      buffer[buf_in_use++] = suff;
+      bcopy ((char *) (ent->word + copy + 1), (char *) (buffer + buf_in_use), suff);
+      buf_in_use += suff;
+      HTONUI(ent->PL->num_entries);  /* [RPAP - Jan 97: Endian Ordering] */
+      bcopy ((char *) &(ent->PL->num_entries), (char *) (buffer + buf_in_use), sizeof (ent->PL->num_entries));
+      NTOHUI(ent->PL->num_entries);  /* [RPAP - Jan 97: Endian Ordering] */
+      buf_in_use += sizeof (ent->PL->num_entries);
+
+      for (j = 0; j < ent->PL->num_entries; j++)
+	{
+	  register PosEntry *pe = &(ent->PL->PE[j]);
+	  HTONUI(pe->num_cases);  /* [RPAP - Jan 97: Endian Ordering] */
+	  bcopy ((char *) &(pe->num_cases), (char *) (buffer + buf_in_use), sizeof (pe->num_cases));
+	  buf_in_use += sizeof (pe->num_cases);
+	  HTONUI(pe->blk);  /* [RPAP - Jan 97: Endian Ordering] */
+	  bcopy ((char *) &(pe->blk), (char *) (buffer + buf_in_use), sizeof (pe->blk));
+	  buf_in_use += sizeof (pe->blk);
+	  HTONUS(pe->blk_index);  /* [RPAP - Jan 97: Endian Ordering] */
+	  bcopy ((char *) &(pe->blk_index), (char *) (buffer + buf_in_use), sizeof (pe->blk_index));
+	  buf_in_use += sizeof (pe->blk_index);
+	  HTONUS(pe->offset);  /* [RPAP - Jan 97: Endian Ordering] */
+	  bcopy ((char *) &(pe->offset), (char *) (buffer + buf_in_use), sizeof (pe->offset));
+	  buf_in_use += sizeof (pe->offset);
+	}
+
+      if (buf_in_use + ptrs_in_use * sizeof (*pointers) +
+	  sizeof (ptrs_in_use) > block_size)
+	FatalError (1, "Fatal Internal Error # 64209258\n");
+
+      if (word_num == 0)
+	{
+	  /* Write word to main index */
+	  fwrite (ent->word, sizeof (u_char), *(ent->word) + 1, idbi);
+	  HTONUL(pos);  /* [RPAP - Jan 97: Endian Ordering] */
+	  fwrite (&pos, sizeof (pos), 1, idbi);
+	  NTOHUL(pos);  /* [RPAP - Jan 97: Endian Ordering] */
+	  sih.index_chars += *(ent->word) + 1;
+	  First_word = i;
+	}
+      word_num++;
+    } /* end for each word */
+
+  if (buf_in_use)
+    {
+      /* Write last buffer to tmp file */
+      int chunk;
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      HTONUL(First_word);
+      HTONUS(word_num);
+
+      fwrite (&First_word, sizeof (First_word), 1, tmp);
+      fwrite (&word_num, sizeof (word_num), 1, tmp);
+      fwrite (pointers, sizeof (*pointers), ptrs_in_use, tmp);
+      fwrite (buffer, sizeof (u_char), buf_in_use, tmp);
+      bzero ((char *) buffer, block_size);
+      chunk = buf_in_use + ptrs_in_use * sizeof (*pointers) +
+	sizeof (ptrs_in_use) + sizeof (First_word);
+      if (force && chunk < block_size)
+	{
+	  fwrite (buffer, sizeof (u_char), block_size - chunk, tmp);
+	  chunk = block_size;
+	}
+
+      sih.num_blocks++;
+    }
+
+  rewind (tmp);
+  sih.blocks_start = sih.index_chars + sizeof (u_long) + sizeof (sih) +
+    sih.num_blocks * sizeof (pos);
+  if (force)
+    {
+      int amount;
+      amount = sih.blocks_start % block_size;
+      if (amount != 0)
+	{
+	  bzero ((char *) buffer, block_size);
+	  fwrite (buffer, sizeof (u_char), block_size - amount, idbi);
+	  sih.blocks_start += block_size - amount;
+	}
+    }
+
+  while ((num = fread (buffer, sizeof (u_char), block_size, tmp)) != 0)
+    fwrite (buffer, sizeof (u_char), num, idbi);
+  fclose (tmp);
+
+  /* skip over the magic number */
+  fseek (idbi, sizeof (u_long), 0);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(sih.lookback);
+  HTONUL(sih.block_size);
+  HTONUL(sih.num_blocks);
+  HTONUL(sih.blocks_start);
+  HTONUL(sih.index_chars);
+  HTONUL(sih.num_of_words);
+
+  fwrite (&sih, sizeof (sih), 1, idbi);
+  fclose (idbi);
+
+#ifndef SILENT
+  Message ("Stem %d:\n", stem_method);
+  Message ("     Block size   : %10d\n", block_size);
+  Message ("     Num_blocks   : %10d\n", NTOHUL(sih.num_blocks));  /* [RPAP - Jan 97: Endian Ordering] */
+  Message ("     Max mem used : %10.1f Mb\n", (double) MaxMemInUse / 1024.0 / 1024.0);
+  Message ("     Num_of_words : %10d\n", NTOHUL(sih.num_of_words));  /* [RPAP - Jan 97: Endian Ordering] */
+#endif
+}
+
+
+void
+UpdateStemDict (char * filename, int stem_method)
+{
+  FILE *idb;
+  struct stem_dict_header sdh;
+
+  if (!(idb = open_file (filename, INVF_DICT_BLOCKED_SUFFIX, "r+b",
+			 MAGIC_STEM, MG_CONTINUE)))
+    FatalError (1, "Could not update stemmed dict");
+
+  fread ((char *) &sdh, sizeof (sdh), 1, idb);
+  NTOHUL(sdh.indexed);  /* [RPAP - Jan 97: Endian Ordering] */
+  sdh.indexed |= 1 << (stem_method - 1);
+  HTONUL(sdh.indexed);
+  fseek (idb, sizeof (u_long), 0);
+  fwrite ((char *) &sdh, sizeof (sdh), 1, idb);
+  fclose (idb);
+}
+
+
+
+/* Main */
+int main (int argc, char **argv)
+{
+  File *idb;    /* File to .invf.dict.blocked */
+  char *filename = "";
+  stemmed_dict *sd;   /* Stemmed dictionary */
+  int ch;
+  char path[512];
+  int stem_method = 0;
+
+  msg_prefix = argv[0];
+  opterr = 0;
+  while ((ch = getopt (argc, argv, "f:d:b:hFs:")) != -1)
+    switch (ch)
+      {
+      case 'f':		/* input file */
+	filename = optarg;
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 'b':
+	block_size = atoi (optarg);
+	break;
+      case 'F':
+	force = 1;
+	break;
+      case 's':
+	stem_method = atoi (optarg);
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-d directory] "
+		 "[-b num] [-F] [-h] -s 1|2|3 -f name\n", argv[0]);
+	exit (1);
+      }
+
+  if (stem_method < 1 || stem_method > 3)
+    FatalError (1, "Stem method must be 1, 2 or 3");
+
+  /* Open required stem dict file */
+  sprintf (path, FILE_NAME_FORMAT, get_basepath (), filename, INVF_DICT_BLOCKED_SUFFIX);
+  if (!(idb = Fopen (path, "rb", MAGIC_STEM)))
+    FatalError (1, "Unable to open \"%s\"", path);
+
+  /* Read in idb header and index to blocks */
+  if (!(sd = ReadStemDictBlk (idb)))
+    FatalError (1, "Could not read stemmed dictionary");
+
+  /* Process stemmed dictionary */
+  init_process ();
+  process_stem_dict (sd, stem_method);
+  save_idx (filename, stem_method);
+ 
+  /* Close stemmed dict */
+  Fclose (idb);
+
+  /* Update stemmed dict */
+  UpdateStemDict (filename, stem_method);
+
+  return 0;
+}
+
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/mg_text_estimate.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/mg_text_estimate.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/mg_text_estimate.1	(revision 699)
@@ -0,0 +1,112 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_text_estimate 1 \*(Dt CITRI
+.SH NAME
+mg_text_estimate \- Estimate the size of the compressed text based on the text statistics and a compression dictionary.
+.SH SYNOPSIS
+.B mg_text_estimate
+[
+.B \-h
+]
+[
+.BR \-H " |"
+.BR \-B " |"
+.BR \-D " |"
+.BR \-Y " |"
+.B \-M
+]
+.if n .ti +9n
+.I stats-dict
+.I compression-dict
+.SH DESCRIPTION
+This program estimates the size of the compressed text that would be
+generated if text with certain statistics were compressed with a
+particular dictionary.  This program has no real use in the
+.BR mg (1)
+system, except for experimenting with the
+.BR mg_compression_dict (1)
+program.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-m\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-H
+This specifies that novel words will be coded character by character
+using Huffman codes.
+.TP
+.B \-B
+This specifies that the size of an auxiliary dictionary will be
+estimated.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using binary codes.  The binary code represents their occurrence
+position in the auxiliary dictionary.
+.TP
+.B \-D
+This specifies that the size of an auxiliary dictionary will be
+estimated.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using delta codes.  The delta code represents their occurrence position
+in the auxiliary dictionary.
+.TP
+.B \-Y
+This specifies that the size of an auxiliary dictionary will be
+estimated.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using a combination of gamma and binary codes.  The code represents
+their occurrence position in the auxiliary dictionary.  This generally
+produces better compression than
+.B \-B
+or
+.BR \-D .
+.TP
+.B \-M
+This specifies that the size of an auxiliary dictionary will be
+estimated.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using a combination of gamma and binary codes.  The code represents
+their occurrence position in the auxiliary dictionary.  This method is
+adaptive within documents, and generally produces better compression
+than
+.BR \-B ,
+.B \-D
+or
+.BR \-Y .
+.SH FILES
+.TP 20
+.B *.text.stats
+Statistics about the text.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/mg_text_estimate.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/mg_text_estimate.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/mg_text_estimate.c	(revision 699)
@@ -0,0 +1,581 @@
+/**************************************************************************
+ *
+ * mg_text_estimate.c -- Program to estimate the compressed text size
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "local_strings.h"
+#include "bitio_gen.h"
+#include "bitio_m.h"
+#include "bitio_m_stdio.h"
+#include "timing.h"
+#include "mgheap.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "locallib.h"
+#include "invf.h"
+#include "text.h"
+#include "words.h"
+#include "mg.h"
+#include "comp_dict.h"
+#include "hash.h"
+
+
+
+
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:18:14  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.4  1999/01/15 03:05:51  rjmcnab
+
+   Renamed lib/heap to be lib/mgheap (it was causing some problems with
+   some versions of the STL libraries which contained a heap.h)
+
+   Revision 1.3  1998/12/17 09:12:53  rjmcnab
+
+   Altered mg to process utf-8 encoded Unicode. The main changes
+   are in the parsing of the input, the casefolding, and the stemming.
+
+   Revision 1.2  1998/11/25 07:55:48  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:19  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:59  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:54  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+typedef struct stats_t
+  {
+    u_long freq, occur;
+    u_long len;
+  }
+stats_t;
+
+static u_long huff_count = 0;
+static u_long escape_count = 0;
+
+
+char mode = 'H';
+
+/* [RJM 07/97: 4G limit] */
+static double ReadInWordsStandard (char *name, double *num_bytes);
+static double ReadInWordsSpecial (char *name, double *num_bytes,
+				  double *num_extra_bytes,
+				  u_long *aux_compressed);
+
+
+
+
+int main (int argc, char **argv)
+{
+  char *stats_dict = NULL, *comp_dict = NULL;
+  ProgTime StartTime;
+  double num_bytes, num_extra_bytes = 0; /* [RJM 07/97: 4G limit] */
+  u_long aux_compressed;
+  int ch;
+  double bytes;
+  opterr = 0;
+  msg_prefix = argv[0];
+  while ((ch = getopt (argc, argv, "BMDYHh")) != -1)
+    switch (ch)
+      {
+      case 'B':
+	Message ("The compressed text size for this method cannot be\n"
+		 "estimated exactly, so an upper bound will be calculated.");
+	mode = ch;
+	break;
+      case 'M':
+	Message ("The compressed text size for this method cannot be\n"
+		 "estimated exactly, so an upper bound will be calculated.");
+	mode = 'Y';
+	break;
+      case 'D':
+      case 'Y':
+      case 'H':
+	mode = ch;
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-h] [B|M|D|Y|H] stats_dict comp_dict\n",
+		 argv[0]);
+	exit (1);
+      }
+
+  if (optind < argc)
+    stats_dict = argv[optind++];
+  else
+    stats_dict = NULL;
+  if (optind < argc)
+    comp_dict = argv[optind++];
+  else
+    {
+      fprintf (stderr, "usage: %s [-h] [B|M|D|Y|H] stats_dict comp_dict\n",
+	       argv[0]);
+      exit (1);
+    }
+
+
+
+  GetTime (&StartTime);
+
+  if (LoadCompressionDictionary (comp_dict) == COMPERROR)
+    exit (1);
+
+  if (mode == 'H')
+    bytes = ReadInWordsStandard (stats_dict, &num_bytes);
+  else
+    bytes = ReadInWordsSpecial (stats_dict, &num_bytes, &num_extra_bytes,
+				&aux_compressed);
+
+
+  Message ("Compressed Text %.2f Mb ( %.0f bytes   %0.3f %%)",
+	   bytes / 1024.0 / 1024, bytes, bytes * 100 / num_bytes);
+  Message ("Words portion of the dictionary %.2f Mb ( %.0f bytes   %0.3f %%)",
+	   Words_disk / 1024.0 / 1024, Words_disk * 1.0,
+	   (Words_disk * 100.0) / num_bytes);
+  if (mode == 'H')
+    Message ("Huffman info for chars in dictionary %.2f Mb ( %.0f bytes   %0.3f %%)",
+	     Chars_disk / 1024.0 / 1024, Chars_disk * 1.0,
+	     (Chars_disk * 100.0) / num_bytes);
+  else
+    {
+      Message ("Aux dictionary (Uncompressed) %.2f Mb ( %.0f bytes   %0.3f %%)",
+	       num_extra_bytes / 1024.0 / 1024, num_extra_bytes * 1.0,
+	       (num_extra_bytes * 100.0) / num_bytes);
+      Message ("Aux dictionary (Compressed)   %.2f Mb ( %.0f bytes   %0.3f %%)",
+	       aux_compressed / 1024.0 / 1024, aux_compressed * 1.0,
+	       (aux_compressed * 100.0) / num_bytes);
+    }
+  Message ("Words and non-words in the text %u", huff_count + escape_count);
+  Message ("Words and non-words coded by escapes %u ( %0.3f %%)", escape_count,
+	   escape_count * 100.0 / huff_count);
+  Message ("%s", ElapsedTime (&StartTime, NULL));
+  return 0;
+}
+
+
+
+
+
+static double 
+ReadInWordsStandard (char *name, double *num_bytes)
+{
+  compression_stats_header csh;
+  FILE *f;
+  u_long which;
+  double bytes, bits = 0;
+
+  f = open_named_file (name, "rb", MAGIC_STATS_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  fread (&csh, sizeof (csh), 1, f);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(csh.num_docs);
+  NTOHD(csh.num_bytes); /* [RJM 07/97: 4G limit] */
+
+  for (which = 0; which < 2; which++)
+    {
+      frags_stats_header fsh;
+      int j;
+      fread (&fsh, sizeof (fsh), 1, f);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fsh.num_frags);
+      NTOHUL(fsh.mem_for_frags);
+
+      for (j = 0; j < fsh.num_frags; j++)
+	{
+	  int len, freq, res, occur_num;
+	  u_char Word[16];
+	  fread (&freq, sizeof (freq), 1, f);
+	  fread (&occur_num, sizeof (occur_num), 1, f);
+	  len = fgetc (f);
+
+	  /* [RPAP - Jan 97: Endian Ordering] */
+	  NTOHSI(freq);
+	  NTOHSI(occur_num);
+
+	  Word[0] = len;
+	  fread (Word + 1, len, 1, f);
+
+	  /* Search the hash table for Word */
+	  if (ht[which])
+	    {
+	      register unsigned long hashval, step;
+	      register int tsize = ht[which]->size;
+	      register u_char **wptr;
+	      HASH (hashval, step, Word, tsize);
+	      for (;;)
+		{
+		  register u_char *s1;
+		  register u_char *s2;
+		  register int len;
+		  wptr = ht[which]->table[hashval];
+		  if (wptr == NULL)
+		    {
+		      res = COMPERROR;
+		      break;
+		    }
+
+		  /* Compare the words */
+		  s1 = Word;
+		  s2 = *wptr;
+		  len = *s1 + 1;
+		  for (; len; len--)
+		    if (*s1++ != *s2++)
+		      break;
+
+		  if (len)
+		    {
+		      hashval += step;
+		      if (hashval >= tsize)
+			hashval -= tsize;
+		    }
+		  else
+		    {
+		      res = ht[which]->table[hashval] - ht[which]->words;
+		      break;
+		    }
+		}
+	    }
+	  else
+	    res = COMPERROR;
+
+	  /* Check that the word was found in the dictionary */
+	  if (res == COMPERROR)
+	    {
+	      escape_count += freq;
+	      if (cdh.dict_type == MG_COMPLETE_DICTIONARY)
+		Message ("Unknown word \"%.*s\"\n", *Word, Word + 1);
+	      if (cdh.dict_type == MG_PARTIAL_DICTIONARY)
+		{
+		  unsigned i;
+		  if (ht[which])
+		    {
+		      res = ht[which]->hd->num_codes - 1;
+		      bits += ht[which]->hd->clens[res] * freq;
+		    }
+		  bits += lens_huff[which].clens[Word[0]] * freq;
+		  for (i = 0; i < Word[0]; i++)
+		    bits += char_huff[which].clens[Word[i + 1]] * freq;
+		}
+	      if (cdh.dict_type == MG_SEED_DICTIONARY)
+		{
+		  unsigned i;
+		  if (ht[which])
+		    {
+		      res = ht[which]->hd->num_codes - 1;
+		      bits += ht[which]->hd->clens[res] * freq;
+		    }
+		  bits += lens_huff[which].clens[Word[0]] * freq;
+		  for (i = 0; i < Word[0]; i++)
+		    bits += char_huff[which].clens[Word[i + 1]] * freq;
+		}
+	    }
+	  else
+	    {
+	      bits += ht[which]->hd->clens[res] * freq;
+	      huff_count += freq;
+	    }
+	}
+    }
+  fclose (f);
+
+
+  /*
+   * The 5.5 bits added on per document consist of
+   *   1 bit for the word/non-word flag at the start of the document.
+   *   1 bit on average for document termination.
+   *   3.5 bits on average for padding to a byte boundary.
+   */
+  bits += 5.5 * csh.num_docs;
+
+  bytes = bits / 8 + sizeof (compressed_text_header) + sizeof (u_long);
+
+  *num_bytes = csh.num_bytes;
+
+  return bytes;
+}
+
+
+static int 
+stats_comp (const void *a, const void *b)
+{
+  return ((stats_t *) a)->occur - ((stats_t *) b)->occur;
+}
+
+
+
+
+
+static double 
+ReadInWordsSpecial (char *name, double *num_bytes,
+		    double *num_extra_bytes,
+		    u_long *aux_compressed)
+{
+  compression_stats_header csh;
+  FILE *f;
+  u_long magic, which;
+  double bytes, bits = 0;
+  double size[2];
+
+  if (!(f = fopen (name, "rb")))  /* [RPAP - Feb 97: WIN32 Port] */
+    FatalError (1, "Unable to open file \"%s\"\n", name);
+
+  if (fread (&magic, sizeof (magic), 1, f) != 1)
+    FatalError (1, "Unable to read magic number");
+
+  NTOHUL(magic);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  if (magic != MAGIC_STATS_DICT)
+    FatalError (1, "Bad magic number");
+
+  fread (&csh, sizeof (csh), 1, f);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(csh.num_docs);
+  NTOHD(csh.num_bytes); /* [RJM 07/97: 4G limit] */
+
+  for (which = 0; which < 2; which++)
+    {
+      stats_t *stats;
+      frags_stats_header fsh;
+      int j, num = 0;
+      long lens[16], chars[256];
+
+      bzero ((char *) lens, sizeof (lens));
+      bzero ((char *) chars, sizeof (chars));
+
+      fread (&fsh, sizeof (fsh), 1, f);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fsh.num_frags);
+      NTOHUL(fsh.mem_for_frags);
+
+      stats = Xmalloc (fsh.num_frags * sizeof (*stats));
+      if (!stats)
+	FatalError (1, "Unable to allocate memory for stats");
+      for (j = 0; j < fsh.num_frags; j++)
+	{
+	  int len, res;
+	  u_char Word[16];
+	  fread (&stats[num].freq, sizeof (stats[num].freq), 1, f);
+	  fread (&stats[num].occur, sizeof (stats[num].occur), 1, f);
+	  len = fgetc (f);
+
+	  /* [RPAP - Jan 97: Endian Ordering] */
+	  NTOHSI(stats[num].freq);
+	  NTOHSI(stats[num].occur);
+
+	  stats[num].len = len + 1;
+
+	  Word[0] = len;
+	  fread (Word + 1, len, 1, f);
+
+	  /* Search the hash table for Word */
+	  if (ht[which])
+	    {
+	      register unsigned long hashval, step;
+	      register int tsize = ht[which]->size;
+	      register u_char **wptr;
+	      HASH (hashval, step, Word, tsize);
+	      for (;;)
+		{
+		  register u_char *s1;
+		  register u_char *s2;
+		  register int len;
+		  wptr = ht[which]->table[hashval];
+		  if (wptr == NULL)
+		    {
+		      res = COMPERROR;
+		      break;
+		    }
+
+		  /* Compare the words */
+		  s1 = Word;
+		  s2 = *wptr;
+		  len = *s1 + 1;
+		  for (; len; len--)
+		    if (*s1++ != *s2++)
+		      break;
+
+		  if (len)
+		    {
+		      hashval += step;
+		      if (hashval >= tsize)
+			hashval -= tsize;
+		    }
+		  else
+		    {
+		      res = ht[which]->table[hashval] - ht[which]->words;
+		      break;
+		    }
+		}
+	    }
+	  else
+	    res = COMPERROR;
+
+	  assert (stats[num].freq > 0);
+
+	  /* Check that the word was found in the dictionary */
+	  if (res == COMPERROR)
+	    {
+	      if (cdh.dict_type == MG_COMPLETE_DICTIONARY)
+		Message ("Unknown word \"%.*s\"\n", *Word, Word + 1);
+	      if ((cdh.dict_type == MG_PARTIAL_DICTIONARY ||
+		   cdh.dict_type == MG_SEED_DICTIONARY) &&
+		  ht[which])
+		{
+		  int k;
+		  res = ht[which]->hd->num_codes - 1;
+		  bits += ht[which]->hd->clens[res] * stats[num].freq;
+		  lens[Word[0]]++;
+		  for (k = 0; k < (int) Word[0]; k++)
+		    chars[Word[k + 1]]++;
+		}
+	      num++;
+	    }
+	  else
+	    {
+	      bits += ht[which]->hd->clens[res] * stats[num].freq;
+	    }
+	}
+
+      {
+	int j, num_buckets = 0;
+	int start[10], end[10];
+	long flens[16], fchars[256];
+
+	for (j = 0; j < 256; j++)
+	  if (!chars[j] && PESINAWORD (j) == which)
+	    fchars[j] = 1;
+	  else
+	    fchars[j] = chars[j];
+	for (j = 0; j < 16; j++)
+	  if (!lens[j])
+	    flens[j] = 1;
+	  else
+	    flens[j] = lens[j];
+
+	size[which] = Calculate_Huffman_Size (16, flens, lens) +
+	  Calculate_Huffman_Size (256, fchars, chars);
+
+
+	qsort (stats, num, sizeof (*stats), stats_comp);
+
+
+	if (mode == 'Y')
+	  {
+	    int k;
+	    num_buckets = 1;
+	    start[0] = 0;
+	    end[0] = cdh.num_words[which] - 1;
+	    Message ("There are %d novel %swords", num, which ? "" : "non-");
+	    while (end[num_buckets - 1] < num)
+	      {
+		start[num_buckets] = end[num_buckets - 1] + 1;
+		end[num_buckets] = start[num_buckets] +
+		  (end[num_buckets - 1] - start[num_buckets - 1]) * 2;
+		num_buckets++;
+	      }
+	    for (k = 0; k < num_buckets; k++)
+	      Message ("Bucket %d   %d <= x <= %d\n", k + 1, start[k], end[k]);
+	  }
+
+	for (j = 0; j < num; j++)
+	  {
+	    (*num_extra_bytes) += stats[j].len;
+	    switch (mode)
+	      {
+	      case 'D':
+		bits += BIO_Delta_Length (j + 1) * stats[j].freq;
+		break;
+	      case 'B':
+		bits += BIO_Binary_Length (j + 1, num) * stats[j].freq;
+		break;
+	      case 'Y':
+		{
+		  int k = 0;
+		  while (j > end[k])
+		    k++;
+		  if (!(j - start[k] + 1 >= 1 &&
+			j - start[k] + 1 <= end[k] - start[k] + 1))
+		    {
+		      Message ("k = %d, j = %d", k, j);
+		      assert (j - start[k] + 1 >= 1 &&
+			      j - start[k] + 1 <= end[k] - start[k] + 1);
+		    }
+		  else
+		    {
+		      bits += (BIO_Gamma_Length (k + 1) +
+			       BIO_Binary_Length (j - start[k] + 1,
+						  end[k] - start[k] + 1)) *
+			stats[j].freq;
+		    }
+		}
+	      }
+	  }
+	if (mode == 'B' && num)
+	  bits += BIO_Delta_Length (num) * csh.num_docs;
+      }
+      Xfree (stats);
+    }
+  fclose (f);
+
+  *aux_compressed = (size[0] + size[1]) / 8;
+
+  /*
+   * The 5.5 bits added on per document consist of
+   *   1 bit for the word/non-word flag at the start of the document.
+   *   1 bit on average for document termination.
+   *   3.5 bits on average for padding to a byte boundary.
+   */
+  bits += 5.5 * csh.num_docs;
+
+  bytes = bits / 8 + sizeof (compressed_text_header) + sizeof (u_long);
+
+  *num_bytes = csh.num_bytes;
+
+  return bytes;
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/mg_text_merge.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/mg_text_merge.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/mg_text_merge.1	(revision 699)
@@ -0,0 +1,76 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_text_merge 1 \*(Dt CITRI
+.SH NAME
+mg_text_merge \- Append one compressed text file to another
+.SH SYNOPSIS
+.B mg_text_merge
+[
+.B \-h
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+This program takes two
+.BR mg (1)
+database inverted files and stemmed
+dictionaries, named "old" and "new" respectively, and creates
+a merged stemmed dictionary and inverted file.
+It is part of the
+.BR mgmerge (1)
+addition to the
+.BR mg (1)
+package.
+The two compressed text files are
+.IR "name" .old.text
+and
+.IR "name" .new.text.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.old.text
+The old compressed text.
+.TP
+.B *.new.text
+The new compressed text that will be appended
+.TP
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mgmerge (1),
+.BR mg_get_merge (1),
+.BR mg_invf_merge (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mgquery (1).
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/mg_text_merge.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/mg_text_merge.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/mg_text_merge.c	(revision 699)
@@ -0,0 +1,249 @@
+/**************************************************************************
+ *
+ * mg_text_merge.c --- merge  *.text, *.text.idx files
+ *                     part of the mgmerge utility
+ * Copyright (C) 1995 Shane Hudson (shane@cosc.canterbury.ac.nz)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ * Last edited:  November 11 1994
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "messages.h"
+#include "timing.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "locallib.h"
+#include "mg.h"
+#include "mg_merge.h"
+#include "mg_files.h"
+#include "text.h"
+
+
+/**** GLOBALS ****/
+FILE *text[3], *idx[3];
+
+typedef char FileName[256];
+FileName old_name, new_name, merge_name;
+
+long magicsize;			/* == where the header in a file begins */
+
+compressed_text_header cth[3];
+
+/*=======================================================================
+ * init_merge_text(): open files, set up global variables, etc
+ *=======================================================================*/
+int 
+init_merge_text ()
+{
+
+  /* open .text files */
+  text[OLD] = open_file (old_name, TEXT_SUFFIX, "r+b",
+			 MAGIC_TEXT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+  magicsize = ftell (text[OLD]);
+  fread (&cth[OLD], sizeof (cth[OLD]), 1, text[OLD]);
+
+  text[NEW] = open_file (new_name, TEXT_SUFFIX, "rb+",
+			 MAGIC_TEXT, MG_ABORT);
+  fread (&cth[NEW], sizeof (cth[NEW]), 1, text[NEW]);
+
+  /* open .text.idx files */
+  idx[OLD] = open_file (old_name, TEXT_IDX_SUFFIX, "rb+",
+			MAGIC_TEXI, MG_ABORT);
+  fread (&cth[OLD], sizeof (cth[OLD]), 1, idx[OLD]);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(cth[OLD].num_of_docs);
+  NTOHD(cth[OLD].num_of_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(cth[OLD].num_of_words);
+  NTOHUL(cth[OLD].length_of_longest_doc);
+  NTOHD(cth[OLD].ratio);
+
+  idx[NEW] = open_file (new_name, TEXT_IDX_SUFFIX, "rb+",
+			MAGIC_TEXI, MG_ABORT);
+  fread (&cth[NEW], sizeof (cth[NEW]), 1, idx[NEW]);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(cth[NEW].num_of_docs);
+  NTOHD(cth[NEW].num_of_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(cth[NEW].num_of_words);
+  NTOHUL(cth[NEW].length_of_longest_doc);
+  NTOHD(cth[NEW].ratio);
+
+  idx[MERGE] = create_file (merge_name, TEXT_IDX_SUFFIX, "wb",
+			    MAGIC_TEXI, MG_ABORT);
+  return OK;
+}
+
+
+/*=======================================================================
+ * process_merge_text(): merge the files
+ *=======================================================================*/
+int 
+process_merge_text (void)
+{
+  int i;
+  u_long data, offset;
+  byte c;
+
+  /* update and write merged header to .text and .text.idx files */
+  /* they have the exact same header */
+  cth[MERGE].num_of_docs = cth[OLD].num_of_docs
+    + cth[NEW].num_of_docs;
+  cth[MERGE].num_of_bytes = cth[OLD].num_of_bytes
+    + cth[NEW].num_of_bytes;
+  cth[MERGE].num_of_words = cth[OLD].num_of_words;
+  cth[MERGE].length_of_longest_doc =
+    (cth[OLD].length_of_longest_doc > cth[NEW].length_of_longest_doc
+     ? cth[OLD].length_of_longest_doc
+     : cth[NEW].length_of_longest_doc);
+  cth[MERGE].ratio = ((cth[OLD].num_of_bytes * cth[OLD].ratio) +
+		      (cth[NEW].num_of_bytes * cth[NEW].ratio))
+    / cth[MERGE].num_of_bytes;
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(cth[MERGE].num_of_docs);
+  HTOND(cth[MERGE].num_of_bytes); /* [RJM 07/97: 4G limit] */
+  HTONUL(cth[MERGE].num_of_words);
+  HTONUL(cth[MERGE].length_of_longest_doc);
+  HTOND(cth[MERGE].ratio);
+
+  fwrite (&cth[MERGE], sizeof (cth[MERGE]), 1, idx[MERGE]);
+  fseek (text[OLD], magicsize, 0);
+  fwrite (&cth[MERGE], sizeof (cth[MERGE]), 1, text[OLD]);
+
+  /*
+   *  Update *.text.idx:  need to know where each new doc starts
+   *  in the appended .text file
+   */
+  for (i = 0; i < cth[OLD].num_of_docs; i++)
+    {
+      fread (&data, sizeof (u_long), 1, idx[OLD]);
+      fwrite (&data, sizeof (u_long), 1, idx[MERGE]);
+    }
+
+  /* offset is the amount to add to each entry from idx[NEW] */
+  fread (&offset, sizeof (u_long), 1, idx[OLD]);
+  NTOHUL(offset);  /* [RPAP - Jan 97: Endian Ordering] */
+  offset -= (4 + sizeof (cth[OLD]));	/* 4 for the magic number */
+
+  for (i = 0; i < cth[NEW].num_of_docs; i++)
+    {
+      fread (&data, sizeof (u_long), 1, idx[NEW]);
+      NTOHUL(data);  /* [RPAP - Jan 97: Endian Ordering] */
+      data += offset;
+      HTONUL(data);  /* [RPAP - Jan 97: Endian Ordering] */
+      fwrite (&data, sizeof (u_long), 1, idx[MERGE]);
+    }
+  /* write last u_long in idx[MERGE] (= length of file) */
+  fread (&data, sizeof (u_long), 1, idx[NEW]);
+  NTOHUL(data);  /* [RPAP - Jan 97: Endian Ordering] */
+  data += offset;
+  HTONUL(data);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite (&data, sizeof (u_long), 1, idx[MERGE]);
+
+/******* update .text *******/
+  /* simply cat's the files together, except for the headers 
+   * and magic numbers, of course
+   */
+  fseek (text[OLD], 0L, 2);
+  while (!feof (text[NEW]))
+    {
+      fread (&c, sizeof (c), 1, text[NEW]);
+      if (!feof (text[NEW]))
+	fwrite (&c, sizeof (c), 1, text[OLD]);
+    }
+
+  return OK;
+}
+
+
+
+/*=======================================================================
+ * done_merge_text(): close files.
+ *=======================================================================*/
+int 
+done_merge_text (void)
+{
+  fclose (idx[MERGE]);
+  fclose (text[OLD]);
+  fclose (idx[OLD]);
+  fclose (text[NEW]);
+  fclose (idx[NEW]);
+
+  fprintf (stderr, "mg_text_merge: %ld documents added to %s\n",
+	   cth[NEW].num_of_docs, merge_name);
+
+  return OK;
+}
+
+
+/*=======================================================================
+ * usage()
+ *=======================================================================*/
+void 
+usage (char *progname)
+{
+  fprintf (stderr, "usage:  %s -f collection_name\n", progname);
+  exit (1);
+}
+
+
+/*=======================================================================
+ * main()
+ *=======================================================================*/
+int main (int argc, char *argv[])
+{
+  char *progname;
+  ProgTime start;
+  int ch;			/* for command line processing */
+
+  progname = argv[0];
+  msg_prefix = argv[0];
+  merge_name[0] = '\0';
+
+  while ((ch = getopt (argc, argv, "f:d:h")) != -1)
+    switch (ch)
+      {
+      case 'f':
+	strcpy (merge_name, optarg);
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 'h':
+      case '?':
+      default:
+	usage (progname);
+      }
+
+  if (merge_name[0] == '\0')
+    usage (progname);
+  strcpy (old_name, merge_name);
+  strcat (old_name, ".old");
+  strcpy (new_name, merge_name);
+  strcat (new_name, ".new");
+
+  GetTime (&start);
+  init_merge_text ();
+  process_merge_text ();
+  done_merge_text ();
+  Message ("%s\n", ElapsedTime (&start, NULL));
+  return 0;
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/mg_weights_build.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/mg_weights_build.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/mg_weights_build.1	(revision 699)
@@ -0,0 +1,112 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_weights_build 1 \*(Dt CITRI
+.SH NAME
+mg_weights_build \- builds the approximate weights file
+.SH SYNOPSIS
+.B mg_weights_build
+[
+.B \-h
+]
+[
+.B \-s
+]
+[
+.BI \-b " num"
+]
+[
+.BI \-d " directory"
+]
+.if n .ti +9n
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_weights_build
+is the program that generates the approximate weights file and the
+interleaved text index and exact weights file.  To do this, it uses
+the text index and either the exact weights file, or the inverted file
+and stemmed dictionary.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-s
+Save the weights generated from the
+.I *.invf
+and
+.I *.invf.dict
+file in a
+.I *.weight
+file.
+.TP
+.BI \-b " num"
+This specifies the number of bits of precision that should be used for
+the approximate weights.  If not specified, it defaults to 8.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection is.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.dict
+Compressed stemmed dictionary.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.text.idx.wgt
+Interleaved index into the compressed documents and document weights.
+.TP
+.B *.weight.approx
+Approximate document weights.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/mg_weights_build.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/mg_weights_build.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/mg_weights_build.c	(revision 699)
@@ -0,0 +1,508 @@
+/**************************************************************************
+ *
+ * mg_weights_build.c -- Program to build the document weights file
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "memlib.h"
+#include "messages.h"
+#include "local_strings.h"
+#include "bitio_gen.h"
+#include "bitio_m.h"
+#include "bitio_m_stdio.h"
+#include "timing.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "locallib.h"
+#include "invf.h"
+#include "text.h"
+#include "words.h"
+
+#define MAXBITS (sizeof(unsigned long) * 8)
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:18:16  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:49  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:22  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.4  1994/11/29  00:32:05  tes
+   * Committing the new merged files and changes.
+   *
+   * Revision 1.3  1994/10/20  03:57:00  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:55  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+unsigned char bits = 8;
+static char *file_name = "";
+static char *text_file_name = "";
+static unsigned long NumPara = 0;
+static unsigned long StaticNumOfDocs = 0;
+
+unsigned long get_NumPara (void);
+unsigned long get_StaticNumOfDocs (void);
+void GenerateWeights (void);
+void Make_weight_approx (void);
+void Make_text_idx_wgt (void);
+
+
+int main (int argc, char **argv)
+{
+  ProgTime StartTime;
+  int ch;
+  opterr = 0;
+  msg_prefix = argv[0];
+  while ((ch = getopt (argc, argv, "f:t:d:b:sh")) != -1) /* [RJM 10/98 - Text Filename] */
+    switch (ch)
+      {
+      case 'f':		/* input file */
+	file_name = optarg;
+	if (strlen(text_file_name) == 0) text_file_name = optarg;
+	break;
+      /* [RJM 10/98 - Text Filename] */
+      case 't':		/* text input file */
+	text_file_name = optarg;
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 'b':
+	bits = atoi (optarg);
+	if (bits > 32)
+	  {
+	    fprintf (stderr, "b may only take values 0-32\n");
+	    exit (1);
+	  }
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-f input_file]"
+		 "[-d data directory] [-b bits] [-s] [-h]\n", argv[0]);
+	exit (1);
+      }
+  GetTime (&StartTime);
+
+  GenerateWeights ();
+
+  Make_weight_approx ();
+
+  Make_text_idx_wgt ();
+
+  Message ("%s", ElapsedTime (&StartTime, NULL));
+
+  return 0;
+}
+
+
+
+
+unsigned long 
+get_NumPara (void)
+{
+  struct invf_dict_header idh;
+  FILE *invf_dict;
+  if (NumPara)
+    return (NumPara);
+  invf_dict = open_file (file_name, INVF_DICT_SUFFIX, "rb", MAGIC_STEM_BUILD,
+			 MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+  fread ((char *) &idh, sizeof (idh), 1, invf_dict);
+  fclose (invf_dict);
+  NTOHUL2(idh.num_of_docs, NumPara);  /* [RPAP - Jan 97: Endian Ordering] */
+  return NumPara;
+}
+
+
+
+unsigned long 
+get_StaticNumOfDocs (void)
+/* the static number of documents is the N parameter used to
+ * decode document gaps in the inverted file encoded using
+ * the Bblock method.
+ */
+{
+  struct invf_dict_header idh;
+  FILE *invf_dict;
+  if (StaticNumOfDocs)
+    return (StaticNumOfDocs);
+  invf_dict = open_file (file_name, INVF_DICT_SUFFIX, "rb", MAGIC_STEM_BUILD,
+			 MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+  fread ((char *) &idh, sizeof (idh), 1, invf_dict);
+  fclose (invf_dict);
+  NTOHUL2(idh.static_num_of_docs, StaticNumOfDocs);  /* [RPAP - Jan 97: Endian Ordering] */
+  return StaticNumOfDocs;
+}
+
+
+
+void GenerateWeights (void) {
+  FILE *dict, *invf, *f, *idx;
+  struct invf_dict_header idh;
+  struct invf_file_header ifh;
+  int i;
+  double logN;
+  float *DocWeights;
+
+  /* make sure the globals NumPara and StaticNumOfDocs are loaded */
+  get_NumPara ();
+  get_StaticNumOfDocs ();
+
+  /* check to see if the weights file has already been built */
+  if ((f = open_file (file_name, WEIGHTS_SUFFIX, "rb", MAGIC_WGHT,
+		      MG_CONTINUE)) != NULL) {
+      fclose (f);
+      return;
+  }
+  Message ("The file \"%s.weight\" does not exist.", file_name);
+  Message ("Building the weight data from the file \"%s.invf\".", file_name);
+
+  logN = log ((double) NumPara);
+
+  /* allocate memory for the weights */
+  if (!(DocWeights = Xmalloc (sizeof (float) * (NumPara + 1))))
+      FatalError (1, "No memory for doc weights");
+  bzero ((char *) DocWeights, sizeof (float) * (NumPara + 1));
+
+  /* open the .invf.dict file and read in its header */
+  dict = open_file (file_name, INVF_DICT_SUFFIX, "rb", MAGIC_STEM_BUILD,
+		    MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+  fread ((char *) &idh, sizeof (idh), 1, dict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(idh.lookback);
+  NTOHUL(idh.dict_size);
+  NTOHUL(idh.total_bytes);
+  NTOHUL(idh.index_string_bytes);
+  NTOHD(idh.input_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(idh.num_of_docs);
+  NTOHUL(idh.static_num_of_docs);
+  NTOHUL(idh.num_of_words);
+  NTOHUL(idh.stemmer_num);
+  NTOHUL(idh.stem_method);
+
+  /* open .invf.idx */
+  idx = open_file (file_name, INVF_IDX_SUFFIX, "rb", MAGIC_INVI,
+		   MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  /* open .invf and read in its header */
+  invf = open_file (file_name, INVF_SUFFIX, "rb", MAGIC_INVF,
+		    MG_ABORT);
+  fread ((char *) &ifh, sizeof (ifh), 1, invf);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(ifh.no_of_words);
+  NTOHUL(ifh.no_of_ptrs);
+  NTOHUL(ifh.skip_mode);
+  for (i = 0; i <= 15; i++)
+    NTOHUL(ifh.params[i]);
+  NTOHUL(ifh.InvfLevel);
+
+  /* make sure the inverted file does not contain skips and is not level 1 */
+  if (ifh.skip_mode != 0)
+    FatalError (0, "Can\'t make weights file from a skipped inverted file.");
+  if (ifh.InvfLevel == 1)
+    FatalError (0, "Can\'t make weights file from level 1 inverted file.");
+
+  DECODE_START (invf)
+
+    /* process each word adding its contributions to the document weights */
+    for (i = 0; i < ifh.no_of_words; i++)
+    {
+      u_char dummy1, dummy2[MAXSTEMLEN + 1];
+      unsigned long fcnt, wcnt, blk, CurrDoc, p, j;
+      float idf;
+
+      /* give a little feedback every 4096 words */
+      if ((i & 0xfff) == 0)
+	fprintf (stderr, ".");
+
+      /* read an entry for a word, just to get p value */
+      dummy1 = fgetc (dict);
+      dummy1 = fgetc (dict);
+      fread (dummy2, sizeof (u_char), dummy1, dict);
+      fread ((char *) &fcnt, sizeof (fcnt), 1, dict);
+      fread ((char *) &wcnt, sizeof (wcnt), 1, dict);
+
+      dummy2[dummy1] = '\0';
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fcnt);
+      NTOHUL(wcnt);
+
+      p = fcnt;
+
+      idf = logN - log ((double) fcnt);
+      blk = BIO_Bblock_Init (StaticNumOfDocs, p);
+      CurrDoc = 0;
+
+      /* check the inverted file index entry for this word */
+      {
+	unsigned long loc;
+	fread ((char *) &loc, sizeof (loc), 1, idx);
+	NTOHUL(loc);  /* [RPAP - Jan 97: Endian Ordering] */
+	if (ftell (invf) != loc)
+	  {
+	    FatalError (1, "Word %d  %d != %d", i, ftell (invf), loc);
+	  }
+      }
+
+      for (j = 0; j < p; j++)
+	{
+	  unsigned long x, tf;
+	  BBLOCK_DECODE (x, blk);
+	  CurrDoc += x;
+
+	  if (CurrDoc > idh.num_of_docs) {
+	    FatalError (1, "CurrDoc = %d, number of documents = %d", 
+			CurrDoc, idh.num_of_docs);
+	  }	  
+
+	  if (ifh.InvfLevel >= 2)
+	    {
+	      double weight;
+	      GAMMA_DECODE (tf);
+	      weight = tf * idf;
+	      DocWeights[CurrDoc - 1] += weight * weight;
+	    }
+	}
+      
+      while (__btg)
+	DECODE_BIT;
+    }
+
+  DECODE_DONE
+
+  fclose (dict);
+  fclose (invf);
+  fprintf (stderr, "\n");
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = 0; i < NumPara; i++)
+    HTONF(DocWeights[i]);
+
+  f = create_file (file_name, WEIGHTS_SUFFIX, "wb", MAGIC_WGHT,
+		   MG_ABORT);
+
+  fwrite ((char *) DocWeights, sizeof (float), NumPara, f);
+  fclose (f);
+  Xfree (DocWeights);
+}
+
+
+
+
+
+
+
+
+
+
+
+void 
+Make_weight_approx (void)
+{
+  int i, pos, max;
+  unsigned long buf;
+  double U, L, B;
+  FILE *approx, *exact;
+
+  exact = open_file (file_name, WEIGHTS_SUFFIX, "rb", MAGIC_WGHT,
+		     MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  /* calculate U and L */
+  L = 1e300;
+  U = 0;
+  for (i = 0; i < NumPara; i++)
+    {
+      float wgt;
+      fread ((char *) &wgt, sizeof (wgt), 1, exact);
+      NTOHF(wgt);  /* [RPAP - Jan 97: Endian Ordering] */
+      wgt = sqrt (wgt);
+      if (wgt > U)
+	U = wgt;
+      if (wgt > 0 && wgt < L)
+	L = wgt;
+
+    }
+  fseek (exact, sizeof (u_long), SEEK_SET);
+
+  B = pow (U / L, pow (2.0, -(double) bits));
+
+  fprintf (stderr, "L = %f\n", L);
+  fprintf (stderr, "U = %f\n", U);
+  fprintf (stderr, "B = %f\n", B);
+
+
+
+  approx = create_file (file_name, APPROX_WEIGHTS_SUFFIX, "wb",
+			MAGIC_WGHT_APPROX, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  fwrite ((char *) &bits, sizeof (bits), 1, approx);
+  HTOND(L);  /* [RPAP - Jan 97: Endian Ordering] */
+  HTOND(B);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite ((char *) &L, sizeof (L), 1, approx);
+  fwrite ((char *) &B, sizeof (B), 1, approx);
+  NTOHD(L);  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHD(B);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  max = bits == 32 ? 0xffffffff : (1 << bits) - 1;
+  for (buf = pos = i = 0; i < NumPara; i++)
+    {
+      unsigned long fx;
+      float wgt;
+      fread ((char *) &wgt, sizeof (wgt), 1, exact);
+      NTOHF(wgt);  /* [RPAP - Jan 97: Endian Ordering] */
+      wgt = sqrt (wgt);
+      if (wgt == 0)
+	{
+	  wgt = L;
+#ifndef QUIET
+	  Message ("Warning: Document %d had a weight of 0.", i);
+#endif
+	}
+      fx = (int) floor (log (wgt / L) / log (B));
+
+      if (fx > max)
+	fx = max;
+
+      buf |= (fx << pos);
+      pos += bits;
+
+      if (pos >= MAXBITS)
+	{
+	  HTONUL(buf);
+	  fwrite ((char *) &buf, sizeof (buf), 1, approx);
+	  buf = fx >> (bits - (pos - MAXBITS));
+	  pos = pos - MAXBITS;
+	}
+    }
+  if (pos > 0)
+    {
+      /* [RPAP - Jan 97: Endian Ordering] */
+      HTONUL(buf);
+      fwrite ((char *) &buf, sizeof (buf), 1, approx);
+    }
+
+  fclose (approx);
+  fclose (exact);
+}
+
+
+
+
+
+void 
+Make_text_idx_wgt (void)
+{
+  compressed_text_header cth;
+  int i;
+  FILE *idx_wgt, *idx, *para, *exact;
+
+  idx_wgt = create_file (file_name, TEXT_IDX_WGT_SUFFIX, "wb", MAGIC_TEXI_WGT,
+			 MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  /* [RJM 10/98 - Text Filename] */
+  idx = open_file (text_file_name, TEXT_IDX_SUFFIX, "rb", MAGIC_TEXI,
+		   MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+  if (fread (&cth, sizeof (cth), 1, idx) != 1)
+    FatalError (1, "Unable to read header of index file");
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(cth.num_of_docs);
+  NTOHD(cth.num_of_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(cth.num_of_words);
+  NTOHUL(cth.length_of_longest_doc);
+  NTOHD(cth.ratio);
+
+  exact = open_file (file_name, WEIGHTS_SUFFIX, "rb", MAGIC_WGHT,
+		     MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  get_NumPara ();
+  if (cth.num_of_docs != NumPara)
+    {
+      Message ("The number of documents does not equal "
+	       "the number of paragraphs.");
+      Message ("Using the \"%s.invf.paragraph\" file\n", file_name);
+      para = open_file (file_name, INVF_PARAGRAPH_SUFFIX, "rb", MAGIC_PARAGRAPH,
+			MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+    }
+  else
+    para = NULL;
+
+  {
+    struct
+      {
+	unsigned long Start;
+	float Weight;
+      }
+    data;
+    for (i = 0; i < cth.num_of_docs; i++)
+      {
+	int count;
+	fread ((char *) &data.Start, sizeof (unsigned long), 1, idx);
+	if (para && i < cth.num_of_docs)
+	  {
+	    /* [RPAP - Jan 97: Endian Ordering] */
+	    fread ((char *) &count, sizeof (count), 1, para);
+	    NTOHSI(count);
+	  }
+	else
+	  count = 1;
+	while (count--)
+	  {
+	    fread ((char *) &data.Weight, sizeof (float), 1, exact);
+	    NTOHF(data.Weight);  /* [RPAP - Jan 97: Endian Ordering] */
+	    data.Weight = sqrt (data.Weight);
+	    HTONF(data.Weight);  /* [RPAP - Jan 97: Endian Ordering] */
+	    fwrite ((char *) &data, sizeof (data), 1, idx_wgt);
+	  }
+      }
+    /* Write out the extra entry for the idx file */
+    fread ((char *) &data.Start, sizeof (unsigned long), 1, idx);
+    data.Weight = 0;
+    fwrite((char*)&data, sizeof(data), 1, idx_wgt);
+  }
+
+  fclose (idx_wgt);
+  fclose (idx);
+  fclose (exact);
+  if (para)
+    fclose (para);
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/mgdictlist.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/mgdictlist.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/mgdictlist.1	(revision 699)
@@ -0,0 +1,72 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgdictlist 1 \*(Dt CITRI
+.SH NAME
+mgdictlist \- list the contents of a dictionary
+.SH SYNOPSIS
+.B mgdictlist
+[
+.B \-q
+]
+.I filename
+.SH DESCRIPTION
+.B mgdictlist
+lists the contents of either a compressed compression dictionary, a
+text statistics file, or a compressed stemmed dictionary.  Unlike all
+the other programs in the
+.BR mg (1)
+suite, it is necessary to specify the complete pathname of the
+dictionary to list.
+.SH OPTIONS
+The
+.I filename
+must be last.
+.TP "\w'\fIfilename\fP'u+2n"
+.B \-q
+Generate an abbreviated output (machine-readable rather than
+human-readable).
+.TP
+.I filename
+This is the name of the dictionary to list.
+.B mgdictlist
+automatically detects whether the file is a compressed compression
+dictionary, text statistics file, or a compressed stemmed dictionary.
+.SH FILES
+.TP 20
+.B *.text.dict
+Text statistics file.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.invf.build
+Compressed stemmed dictionary.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/mgdictlist.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/mgdictlist.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/mgdictlist.c	(revision 699)
@@ -0,0 +1,405 @@
+/**************************************************************************
+ *
+ * mgdictlist.c -- Program to list a dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "messages.h"
+#include "memlib.h"
+#include "local_strings.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "text.h"
+#include "invf.h"
+#include "locallib.h"
+#include "words.h"
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:18:17  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:49  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:24  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.4  1994/11/29  00:32:07  tes
+   * Committing the new merged files and changes.
+   *
+   * Revision 1.3  1994/10/20  03:57:01  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:56  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+int quick = 0;
+int no_of_words[2];
+u_long maxcodelen[2];
+
+char *dictname = "";
+
+
+
+
+void 
+DumpStemDict (FILE * f)
+{
+  struct invf_dict_header idh;
+  int i;
+  u_char prev[MAXSTEMLEN + 1];
+
+  fread (&idh, sizeof (idh), 1, f);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(idh.lookback);
+  NTOHUL(idh.dict_size);
+  NTOHUL(idh.total_bytes);
+  NTOHUL(idh.index_string_bytes);
+  NTOHD(idh.input_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(idh.num_of_docs);
+  NTOHUL(idh.static_num_of_docs);
+  NTOHUL(idh.num_of_words);
+  NTOHUL(idh.stemmer_num);
+  NTOHUL(idh.stem_method);
+
+  if (quick)
+    printf ("%ld\n", idh.dict_size);
+  else
+    {
+      printf ("# lookback           = %lu\n", idh.lookback);
+      printf ("# dict size          = %lu\n", idh.dict_size);
+      printf ("# total bytes        = %lu\n", idh.total_bytes);
+      printf ("# index string bytes = %lu\n", idh.index_string_bytes);
+      printf ("# input bytes        = %.0f\n", idh.input_bytes); /* [RJM 07/97: 4G limit] */
+      printf ("# num of docs        = %lu\n", idh.num_of_docs);
+      printf ("# static num of docs = %lu\n", idh.static_num_of_docs);
+      printf ("# num of words       = %lu\n", idh.num_of_words);
+      printf ("#\n");
+    }
+
+  for (i = 0; i < idh.dict_size; i++)
+    {
+      register unsigned long copy, suff;
+      unsigned long wcnt, fcnt;
+
+      /* build a new word on top of prev */
+      copy = getc (f);
+      suff = getc (f);
+      *prev = copy + suff;
+      fread (prev + copy + 1, sizeof (u_char), suff, f);
+
+      /* read other data, but no need to store it */
+      fread (&fcnt, sizeof (fcnt), 1, f);
+      fread (&wcnt, sizeof (wcnt), 1, f);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fcnt);
+      NTOHUL(wcnt);
+
+      if (!quick)
+	{
+	  printf ("%d: %8ld ", i, wcnt);
+	  printf ("/ %5ld ", fcnt);
+	  printf ("%2d %2ld\t\"", *prev, copy);
+	}
+      printf ("%s", word2str (prev));
+      if (quick)
+	printf (" %ld %ld\n", wcnt, fcnt);
+      else
+	{
+	  putchar ('"');
+	  putchar ('\n');
+	}
+    }
+}
+
+
+
+
+void 
+ReadInWords (FILE * f)
+{
+  comp_frags_header cfh;
+  u_long *codes;
+  u_char prev[MAXSTEMLEN + 1];
+  int i;
+
+  if (Read_cfh (f, &cfh, NULL, NULL) == -1)
+    FatalError (1, "Unable to read in the dictionary");
+
+  printf ("#\n");
+  printf ("#   max code len       = %u\n", cfh.hd.maxcodelen);
+  printf ("#   total bytes        = %lu\n", cfh.uncompressed_size);
+  printf ("#\n");
+
+  if (!(codes = Generate_Huffman_Codes (&cfh.hd, NULL)))
+    FatalError (1, "no memory for huffman codes\n");
+
+  for (i = 0; i < cfh.hd.num_codes; i++)
+    {
+      register int val, copy, j, k;
+      char code[33];
+      val = fgetc (f);
+      copy = (val >> 4) & 0xf;
+      val &= 0xf;
+
+      fread (prev + copy + 1, sizeof (u_char), val, f);
+      *prev = val + copy;
+
+      for (k = 0, j = cfh.hd.clens[i] - 1; j >= 0; j--, k++)
+	code[k] = '0' + ((codes[i] >> j) & 1);
+      code[k] = '\0';
+
+      printf ("%d: %2d : %*s : \"%s\"\n", i, cfh.hd.clens[i],
+	      cfh.hd.maxcodelen, code, word2str (prev));
+    }
+  Xfree (codes);
+  Xfree (cfh.hd.clens);
+}
+
+
+void 
+ReadCharHuffman (FILE * f, char *title)
+{
+  int i;
+  huff_data hd;
+  u_long *codes;
+
+  if (Read_Huffman_Data (f, &hd, NULL, NULL) == -1)
+    FatalError (1, "Unable to read huffman data");
+
+  if (!(codes = Generate_Huffman_Codes (&hd, NULL)))
+    FatalError (1, "no memory for huffman codes\n");
+
+  printf ("#\n# %s\n#\n", title);
+  for (i = 0; i < hd.num_codes; i++)
+    if (hd.clens[i])
+      {
+	int j, k;
+	char code[33];
+	for (k = 0, j = hd.clens[i] - 1; j >= 0; j--, k++)
+	  code[k] = '0' + ((codes[i] >> j) & 1);
+	code[k] = '\0';
+	printf ("%2d : %*s : \"%s\"\n", hd.clens[i],
+		hd.maxcodelen, code, char2str (i));
+      }
+  Xfree (codes);
+  Xfree (hd.clens);
+}
+
+
+void 
+ReadLenHuffman (FILE * f, char *title)
+{
+  int i;
+  huff_data hd;
+  u_long *codes;
+
+  if (Read_Huffman_Data (f, &hd, NULL, NULL) == -1)
+    FatalError (1, "Unable to read huffman data");
+
+  if (!(codes = Generate_Huffman_Codes (&hd, NULL)))
+    FatalError (1, "no memory for huffman codes\n");
+
+  printf ("#\n# %s\n#\n", title);
+  for (i = 0; i < hd.num_codes; i++)
+    if (hd.clens[i])
+      {
+	int j, k;
+	char code[33];
+	for (k = 0, j = hd.clens[i] - 1; j >= 0; j--, k++)
+	  code[k] = '0' + ((codes[i] >> j) & 1);
+	code[k] = '\0';
+	printf ("%2d : %*s : %d\n", hd.clens[i],
+		hd.maxcodelen, code, i);
+      }
+  Xfree (codes);
+  Xfree (hd.clens);
+}
+
+
+
+
+
+void 
+DumpTextDict (FILE * f)
+{
+  struct compression_dict_header cdh;
+  int which;
+
+  if (Read_cdh (f, &cdh, NULL, NULL) == -1)
+    FatalError (1, "Unable to read dictionary header");
+  switch (cdh.dict_type)
+    {
+    case MG_COMPLETE_DICTIONARY:
+      printf ("# COMPLETE DICTIONARY\n");
+      break;
+    case MG_PARTIAL_DICTIONARY:
+      printf ("# PARTIAL DICTIONARY\n");
+      break;
+    case MG_SEED_DICTIONARY:
+      printf ("# SEED DICTIONARY\n");
+      break;
+    }
+  printf ("# num words          = %lu\n", cdh.num_words[1]);
+  printf ("# num word chars     = %lu\n", cdh.num_word_chars[1]);
+  printf ("# num non-words      = %lu\n", cdh.num_words[0]);
+  printf ("# num non-word chars = %lu\n", cdh.num_word_chars[0]);
+  printf ("# lookback           = %lu\n", cdh.lookback);
+
+  for (which = 0; which < 2; which++)
+    switch (cdh.dict_type)
+      {
+      case MG_COMPLETE_DICTIONARY:
+	{
+	  ReadInWords (f);
+	}
+	break;
+      case MG_PARTIAL_DICTIONARY:
+	{
+	  if (cdh.num_words[which])
+	    ReadInWords (f);
+
+	  ReadCharHuffman (f, "Characters");
+	  ReadLenHuffman (f, "Lengths");
+	}
+	break;
+      case MG_SEED_DICTIONARY:
+	{
+	  if (cdh.num_words[which])
+	    ReadInWords (f);
+
+	  ReadCharHuffman (f, "Characters");
+	  ReadLenHuffman (f, "Lengths");
+	}
+	break;
+      }
+}
+
+
+
+
+void 
+DumpStatsDict (FILE * f)
+{
+  int i;
+  compression_stats_header csh;
+
+  fread (&csh, sizeof (csh), 1, f);
+
+  for (i = 0; i < 2; i++)
+    {
+      int j;
+      frags_stats_header fsh;
+
+      fread (&fsh, sizeof (fsh), 1, f);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fsh.num_frags);
+      NTOHUL(fsh.mem_for_frags);
+
+      if (!quick)
+	printf ("#\n# num %9s      = %lu\n#\n", i ? "words" : "non-words",
+		fsh.num_frags);
+
+      for (j = 0; j < fsh.num_frags; j++)
+	{
+	  u_char Word[16];
+	  u_long freq, occur_num;
+
+	  fread (&freq, sizeof (freq), 1, f);
+	  fread (&occur_num, sizeof (occur_num), 1, f);
+
+	  /* [RPAP - Jan 97: Endian Ordering] */
+	  NTOHUL(freq);
+	  NTOHUL(occur_num);
+
+	  Word[0] = fgetc (f);
+	  fread (Word + 1, Word[0], 1, f);
+	  printf ("%d: %7ld : %7ld : \"%s\"\n", j, freq,
+		  occur_num, word2str (Word));
+	}
+    }
+}
+
+
+int main (int argc, char **argv)
+{
+  FILE *fp;
+  unsigned long magic = 0;
+
+  if (argc < 2)
+    FatalError (1, "A file name must be specified");
+  dictname = argv[1];
+  if (strcmp (dictname, "-q") == 0)
+    {
+      quick = 1;
+      if (argc < 3)
+	FatalError (1, "A file name must be specified");
+      dictname = argv[2];
+    }
+  if (!(fp = fopen (dictname, "rb")))  /* [RPAP - Feb 97: WIN32 Port] */
+    FatalError (1, "Unable to open \"%s\"", dictname);
+
+  fread (&magic, sizeof (magic), 1, fp);
+
+  NTOHUL(magic);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  switch (magic)
+    {
+    case MAGIC_STEM_BUILD:
+      if (!quick)
+	printf ("# Contents of STEM file \"%s\"\n#\n", dictname);
+      DumpStemDict (fp);
+      break;
+    case MAGIC_DICT:
+      if (!quick)
+	printf ("# Contents of DICT file \"%s\"\n#\n", dictname);
+      DumpTextDict (fp);
+      break;
+    case MAGIC_STATS_DICT:
+      if (!quick)
+	printf ("# Contents of STATS file \"%s\"\n#\n", dictname);
+      DumpStatsDict (fp);
+      break;
+    default:
+      FatalError (1, "Bad magic number. \"%s\" cannot be dumped", dictname);
+    }
+  fclose (fp);
+  return 0;
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/mgquery.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/mgquery.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/mgquery.1	(revision 699)
@@ -0,0 +1,693 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgquery 1 \*(Dt CITRI
+.SH NAME
+mgquery \- query program for the mg system
+.SH SYNOPSIS
+.B mgquery
+[
+.B \-h
+]
+[
+.B \-D
+]
+[
+.BI \-f " name"
+]
+[
+.BI \-d " directory"
+]
+.if n .ti +9n
+[
+.I collection-name
+]
+.SH DESCRIPTION
+.B mgquery
+enables users to make Boolean or ranked queries from a data base
+generated by the
+.BR mg (1)
+system.  It accepts queries from
+.I stdin
+and sends the retrieved documents to
+.IR stdout .
+Information on the resource usage of
+.B mgquery
+as it processes queries can be obtained interactively.
+.SH OPTIONS
+Options may appear in any order, but the
+.IR collection-name ,
+if specified, must be last.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-D
+This option causes the entire text to be decompressed and sent to
+.IR stdout .
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection that will be
+used.  If a collection with the specified base
+.I name
+does not exist, an error message will be displayed and
+.B mgquery
+will exit.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.SH USAGE
+Prior to processing the command line arguments, the
+.B mgquery
+program attempts to read in a startup script called
+.IR ./.mgrc .
+If that fails, it attempts to read in the file
+.IR $HOME/.mgrc .
+The startup file can only contain commands\(emno queries are
+permitted in the
+.I .mgrc
+file.  Lines starting with \*(lq\fB#\fP\*(rq in the file are comments.
+The most common use for the
+.I .mgrc
+file is to personalise the initial values of the predefined parameters
+with
+.B .set
+commands.
+.LP
+The input to
+.B mgquery
+consists of a series of input lines.  The backslash
+character
+.RB (\*(lq \e \*(rq)
+is used at the end of lines to indicate
+that input continues on the next line.
+.LP
+Input lines on which the first character is a dot
+.RB (\*(lq . \*(rq)
+are commands to the
+.B mgquery
+program.  Input lines that do not start with a dot are queries.
+.LP
+A query consists of two parts.  One part is a Boolean or ranked query
+that identifies documents.  The second part is a post-processing
+pattern matching operation.  Any text between the first speech mark
+(\*(lq) and the last speech mark (\*(rq) is considered to be a
+post-processing pattern.
+.SH COMMANDS
+The
+.B mgquery
+program can accept the following commands.
+.TP 17
+.B .help
+Display several pages of help text.
+.TP
+.B .quit
+Quit the program.
+.TP
+.B .warranty
+Display the
+.BR mg (1)
+warranty.
+.TP
+.B .conditions
+Display the conditions of use and distribution of
+.BR mg (1).
+.TP
+.BI ".set " "name value"
+Set the parameter
+.I name
+to the specified
+.IR value .
+If the parameter is a Boolean
+.I value
+and the
+.I value
+is omitted, the parameter will be inverted (i.e., if it was
+.IR true ,
+then it will change to
+.IR false ;
+if it was
+.IR false ,
+then it will change to
+.IR true ).
+.TP
+.BI ".unset " name
+Delete the parameter
+.I name
+from the currently-defined parameters.
+.TP
+.B .reset
+Reset the parameters to the state that they had after the processing
+of the
+.B mgquery
+command line.
+.TP
+.B .display
+Display the values of all the currently-defined parameters.
+.TP
+.B .push
+Push the currently-defined parameters onto a stack.
+.TP
+.B .pop
+Pops a set of parameters off the stack, replacing the currently-defined
+ones.
+.TP
+.BI ".output " arg
+This is used to specify where to send the text of the documents.  Once
+the
+.B .output
+command is specified, all subsequent output will be sent to the place
+specified by
+.IR arg .
+If
+.I arg
+is not specified subsequent output will be directed to
+.IR stdout .
+.I Arg
+may be any of the following.
+.RS
+.TP 13
+.BI "> " filename
+Send output to the specified file.
+.TP
+.BI ">> " filename
+Append output to the specified file.
+.TP
+.BI "| " command
+Pipe the output to
+.IR command ,
+which is executed by
+.IR sh .
+.RE
+.TP
+.BI ".input " arg
+This is used to specify where input (queries and commands) comes
+from.  Once the
+.B .input
+command is specified all subsequent input will be come from the place
+specified by
+.IR arg .
+If
+.I arg
+is not specified subsequent input will come from
+.IR stdin .
+.RS
+.TP 13
+.BI "< " filename
+Get input from the specified file.
+.TP
+.BI "| " command
+The input comes from the standard output of
+.IR command ,
+which is executed by
+.IR sh .
+.RE
+.SH PARAMETERS
+The following parameters are predefined and have special
+significance.  Each parameter will be followed by its default
+value.  Parameters are initialised before the
+.I .mgrc
+file is read or the command line arguments are processed.
+.TP 17
+.BI accumulator_method " `array'"
+This parameter is used during ranking, and specifies how the
+weight for each document should be accumulated.  The following
+methods are available:
+.IR array ,
+.IR splay_tree ,
+.IR hash_table ,
+and
+.IR list .
+.TP
+.BI briefstats " `off'"
+This is a Boolean parameter that determines whether the
+totals for disk, memory and time usage statistics will be
+displayed at the end of each query.
+.IR Note :
+this takes precedence over the parameters
+.BR diskstats ,
+.BR memstats " and " timestats .
+This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI buffer " `1048576'"
+When the documents are being read in, they are read into a
+buffer of this size and then displayed from this buffer.  If
+the documents are larger than this buffer, the buffer is
+expanded automatically.  Having a large buffer gives a very
+slight performance improvement, because it allows the order of
+disk operations to be optimised.  The buffer size is measured
+in bytes.
+.TP
+.BI diskstats " `off'"
+This is a Boolean parameter that determines whether the disk
+usage statistics for the preceding query will be displayed
+after each query.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI doc_sepstr " `---------------------------------- %n\en\'"
+This specifies the string that will be used to separate
+documents when they are displayed for `Boolean' or `docnums'
+queries.  The standard C escape character sequences
+may be used to place special characters in the
+string.  For example, a newline would be `\en'.  To include a `%',
+use the sequence `%%'.  To include the
+.BR mg (1)
+document number, use the sequence `%n'.  The following escape character
+sequences are available
+.nf
+.ta 1.7iL
+.B Sequence	Meaning
+`\e\e'	backslash
+`\eb'	backspace
+`\ef'	formfeed
+`\en'	newline
+`\er'	carriage return
+`\et'	tab
+`\e"'	speech marks
+`\e''	quote mark
+`\ex\fIhh\fP'	ASCII code in hexadecimal
+`\ennn'	ASCII code in octal
+.fi
+.TP
+.BI expert " `false'"
+If this is
+.IR true ,
+then much of the dialogue output is suppressed.  This parameter may
+take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI hash_tbl_size " `1000'"
+One of the options during ranking queries is to use a hash
+table to accumulate the weights for each document.  The hash
+table is a simple chained type.  This parameter specifies the
+size of the hash table and may take any value between 8 and
+268435456 (2^28).
+.TP
+.BI heads_length " `50'"
+When the mode is
+.BR heads ,
+this specifies the number of characters that will be output for each
+document.
+.TP
+.BI maxdocs " `all'"
+The maximum number of documents to display in response to a
+query.  This parameter may take on a numeric value between 1
+and 429467295 (2^32 - 1) or the word
+.IR all .
+.TP
+.BI maxparas " `1000'"
+The maximum number of paragraphs to identify during a ranked
+query with paragraph indexing.  After the paragraphs have been
+identified, the paragraphs are converted into documents, and
+because some of the paragraphs may refer to the same documents
+the final number of answers may be less than
+.BR maxparas .
+The
+.B maxdocs
+parameter will then be applied.  This parameter may take on a numeric
+value between 1 and 429467295 (2^32 - 1).
+.TP
+.BI max_accumulators " `50000'"
+This parameter limits the number of different paragraph and
+document numbers to be accumulated during ranked queries when
+the parameter
+.B accumulator_method
+is set to
+.IR splay_tree ,
+.IR hash_table ,
+or
+.IR list .
+This parameter may take any value between 8 and 268435456 (2^28).
+.TP
+.BI max_terms " `all'"
+This parameter limits the number of terms that will actually
+be used during a ranked query.  If more terms than the number
+specified by
+.B max_terms
+are entered, then the extra terms will be discarded.  If
+.B sorted_terms
+is on, then the limiting will be done after the terms have been
+sorted.  This parameter may take any value between 1 and 429467295
+(2^32 - 1), or the word
+.IR all.
+.TP
+.BI memstats " `off'"
+This is a Boolean parameter that determines whether the memory
+usage statistics for the preceding query will be displayed
+after each query.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI mgdir " `.'"
+This is set to the directory where the
+.BR mg (1)
+data files may be found.  If
+the environment variable
+.B MGDATA
+exists, then this is instead initialised to the value of
+.BR MGDATA .
+The value of this parameter may be changed, either in the
+.I .mgrc
+file with a
+.BI ".set mgdir "directory
+command, or from the command line using the
+.BI \-d " directory"
+option.  Once the \*(lq\fB>\fP\*(rq prompt appears, changing this
+parameter will have no effect.
+.TP
+.BI mgname " `bible'"
+This is set to the name of the
+.BR mg (1)
+collection that is to be used for the session.  The value of this
+parameter may be changed, either in the
+.I .mgrc
+file with a
+.BI ".set mgname "name
+command, or from the command line using the
+.BI \-f " name"
+option.  Once the \*(lq\fB>\fP\*(rq prompt appears, changing this
+parameter will have no effect.
+.TP
+.BI mode " `text'"
+This specifies how documents should be displayed when they
+are retrieved.  It may take six different values:
+.IR text ,
+.IR hilite ,
+.IR docnums ,
+.IR heads ,
+.IR silent ,
+or
+.IR count .
+.I text
+displays the contents of the document.
+.I hilite
+displays the contents of the document and highlights any of the 
+stemmed query terms.
+.I docnums
+displays only the document numbers.
+.I heads
+is used to print out the head of each document.
+.I silent
+retrieves all the documents but displays nothing except how many
+documents were retrieved.  This mode is intended to be used in timing
+experiments.
+.I count
+does the minimum
+amount of work required to determine how many documents would
+be retrieved, but does not retrieve them.
+.TP
+.BI optimise_type " `1'"
+There are three types of boolean query optimisation (parse tree
+rearrangement). Type 0 leaves parse tree unaltered. Type 1 optimises
+for AND of terms and AND of OR of terms. Type 2 converts the tree
+into DNF (an experiment :-).
+.TP
+.BI pager " `more'"
+This is the name of the program that will be used to display
+the help and the retrieved documents.  If the environment
+variable
+.B PAGER
+is defined, then
+.B pager
+takes on that value.
+.TP
+.BI hilite_style " `bold'"
+This specifies the type of highlighting method.
+It may take one of two different values:
+.IR bold,
+or
+.IR underline.
+.TP
+.BI para_sepstr " `\en######## PARAGRAPH %n ########\en'"
+This specifies the string that will be used to separate paragraphs.
+The standard C escape character sequences may be used to place special
+characters in the string.  For example, a newline would be written
+as `\en'.  To include a `%', use the sequence `%%'.  To include the
+paragraph number within the document, use the sequence `%n'.
+.TP
+.BI para_start " `***** Weight = %w *****\en'"
+This specifies the string that will be used at the head of paragraphs
+for a paragraph-level index following a ranked query.  The standard
+C-language escape character sequences may be used to place special
+characters in the string.  For example, a newline would be written as
+`\en'.  To include a `%', use the sequence `%%'.  To include the
+paragraph weight, use the sequence `%w'.
+.TP
+.BI qfreq " `true'"
+This determine whether the ranked queries will take into account the
+number of times each query term is specified.  When this is
+.IR true ,
+the number of times a term appears in the query is used in the
+ranking.  When this is
+.IR false ,
+all query terms are assumed to occur only once.  This parameter may
+take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI query  " `Boolean'"
+This specifies the type of queries that are to be specified.
+It can take four different values:
+.IR Boolean ,
+.IR ranked ,
+.IR docnums " or " approx-ranked.
+.I Boolean
+is for Boolean queries.
+The
+.BR yacc (1)
+grammar for Boolean queries is as follows.
+.IP
+.nf
+        query   : or;
+.IP
+        or      : or '|' and
+                | and ;
+.IP
+        and     : and '&' not
+                | and not
+                | not ;
+.IP
+        not     : term
+                | '!' not ;
+.IP
+        term    : TERM
+                | '(' or ')' ;
+.fi
+.IP
+.IR ranked " and " approx-ranked
+are for queries ranked by the cosine measure.
+.I approx-ranked
+uses only the low-precision document lengths, and therefore only
+produces an approximation to full cosine ranking.
+.IP
+.nf
+        query   : TERM
+                | query TERM ;
+.fi
+.IP
+.I docnums
+allows the entry of document numbers.  Multiple numbers separated by
+spaces may be specified, or ranges separated by hyphens.
+.IP
+.nf
+        query   : range
+                | query range ;
+.IP
+        range   : num
+                | num '-' num ;
+.fi
+.TP
+.BI ranked_doc_sepstr " `-------------------------------- %n %w\en'"
+This specifies the string that will be used to separate documents when
+they are displayed for `ranked' or `approx-ranked' queries.  The
+standard C escape character sequences may be used to place special
+characters in the string.  For example, a newline would be written as
+`\en'.  To include a `%', use the sequence `%%'.  To include the
+.BR mg (1)
+document number, use the sequence `%n'.  To include the document
+weight, use the sequence `%w'.
+.TP
+.BI sizestats " `false'"
+If this is
+.IR true ,
+then various numbers are output at the end of each query indicating
+what went on during the query.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI skip_dump " `skips.%d'"
+If this parameter is set, then a file will be produced in the current
+directory during ranked queries on skipped inverted files when
+.B accumulator_method
+is set to
+.IR splay_tree ,
+.IR hash_table ,
+or
+.IR list .
+The name of the file is the value of this parameter.  A `%d' in the
+file name will be replaced with the process id of
+.BR mgquery .
+This file will contain information about the usage of skips during the
+query processing.  This option is expensive; use
+.B .unset skip_dump
+to obtain optimal performance.
+.TP
+.BI sorted_terms " `on'"
+This specifies whether or not the terms should be sorted into
+decreasing occurrence in documents so that the least-often occurring
+terms are processed first when ranked queries are being done.  When
+this is
+.IR true ,
+the terms are sorted.  When this is
+.IR false ,
+the terms are not sorted, and are instead processed in order of
+occurrence.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI stop_at_max_accum " `on'"
+This specifies what should happen when the maximum number of
+accumulators set by
+.B max_accumulators
+is reached.  When this is
+.IR true ,
+the processing of terms is stopped at the completion of the current
+term.  When this is
+.IR false ,
+processing continues but no new accumulators are created.  This
+parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI terminator " `'"
+This specifies the string that will be output after the last document
+from the previous query has been output.  The standard C escape
+character sequences may be used to place special characters in the
+string.  For example, a newline would be written as `\en'.  To include
+a `%', use the sequence `%%'.
+.TP
+.BI timestats " `false'"
+If this is
+.IR true ,
+then the time to process a query is displayed in both real time and
+CPU time.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI verbatim " `off'"
+This is a Boolean parameter that determines whether the program
+should attempt to do a regular-expression match on the retrieved
+text.  If verbatim is
+.I on
+and a post-processing string is specified with the query, then the
+post-processing string will be searched for in the documents just
+before they are displayed.  If the string is found, the document will
+be displayed; if not, the document will not be displayed.  If verbatim
+is
+.IR off ,
+the post-processing string will be considered a regular expression
+as in
+.BR egrep (1)
+or
+.BR vi (1).
+E.g., if verbatim is
+.I on,
+\*(lq\fBand.*the\fP\*(rq will look for the 8-character sequence
+\*(lq\fBand.*the\fP\*(rq.  If verbatim is
+.IR off ,
+\*(lq\fBand.*the\fP\*(rq will look for the sequence
+\*(lq\fBand\fP\*(rq followed somewhere later in the document by the
+sequence \*(lq\fBthe\fP\*(rq.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.I .mgrc
+.B mgquery
+startup file
+.TP
+.B help.mg
+Help file for
+.BR mgquery .
+The contents of this file is displayed with the
+.B .help
+command.
+.TP
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.dict
+The `on-disk' stemmed dictionary.
+.TP
+.B *.text
+Compressed documents.
+.TP
+.B *.text.dict
+Compression dictionary.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.text.idx.wgt
+Interleaved index into the compressed documents and document weights.
+.TP
+.B *.weight.approx
+Approximate document weights.
+.SH "SEE ALSO"
+.na
+.BR egrep (1),
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1),
+.BR vi (1),
+.BR yacc (1).
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/mgquery.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/mgquery.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/mgquery.c	(revision 699)
@@ -0,0 +1,1495 @@
+/**************************************************************************
+ *
+ * mgquery.c -- The   M G Q U E R Y   program
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:18:18  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.3  1999/01/08 00:33:46  rjmcnab
+
+   Enabled mg and the library software to read in more than one index
+   at a time.
+
+   Revision 1.2  1998/11/25 07:55:49  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:29  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:57:02  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:58  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+#if defined(HAVE_SYS_PROCFS_H) && defined(HAVE_PR_BRKSIZE) && \
+    (__STDC__ == 0)
+/* STDC test was included to allow cc -Xc on SunOS 5 to work */
+#define USE_PROCESS_MEM
+#endif
+
+#ifdef USE_PROCESS_MEM
+# include <sys/procfs.h>
+#endif
+
+#ifdef HAVE_GETRUSAGE
+# ifdef HAVE_SYS_TIME_H
+#  include <sys/time.h>
+# endif
+# include <sys/resource.h>
+#endif
+
+#ifndef HAVE_GETPAGESIZE
+# include "getpagesize.h"
+#endif
+ 
+#if WITH_REGEX
+# include <regex.h>
+#else
+# include <rx.h>
+#endif
+
+
+#include <stdarg.h>
+#include <signal.h>
+
+#include "messages.h"
+#include "timing.h"
+#include "memlib.h"
+#include "local_strings.h"  /* [RPAP - Feb 97: Term Frequency] */
+
+#include "filestats.h"
+#include "invf.h"
+#include "text.h"
+#include "mg.h"
+#include "lists.h"
+#include "backend.h"
+#include "environment.h"
+#include "globals.h"
+#include "read_line.h"
+#include "mg_errors.h"
+#include "commands.h"
+#include "text_get.h"
+#include "term_lists.h"
+#include "query_term_list.h"
+
+
+FILE *OutFile = NULL, *InFile = NULL;
+int OutPipe = 0, InPipe = 0;
+int Quitting = 0;
+
+/* [RPAP - Feb 97: NZDL Additions] */
+#if defined(PARADOCNUM) ||  defined(NZDL)
+int GetDocNumFromParaNum(query_data *qd, int paranum) {
+  int Documents = qd->td->cth.num_of_docs;
+  int *Paragraph = qd->paragraph;
+  int low = 1, high = Documents;
+  int mid = (low+high)/2;
+
+  while ((mid = (low+high)/2) >=1 && mid <= Documents)
+    {
+      if (paranum > Paragraph[mid])
+	low = mid+1;
+      else if (paranum <= Paragraph[mid-1])
+	high = mid-1;
+      else
+	return mid;
+    }
+  FatalError(1, "Bad paragraph number.\n");
+}
+#endif
+
+#ifdef TREC_MODE
+char *trec_ids = NULL;
+long *trec_paras = NULL;
+#endif
+
+static volatile int PagerRunning = 0;
+static volatile int Ctrl_C = 0;
+
+
+/*****************************************************************************/
+
+typedef enum
+  {
+    S_Time, S_Mem, S_Size, S_File
+  }
+S_Type;
+
+static struct Stat
+  {
+    S_Type typ;
+    char *name;
+    char *text;
+  }
+ *Stats = NULL;
+static int NumStats = 0;
+
+static void 
+Clear_Stats (void)
+{
+  if (Stats)
+    {
+      int i;
+      for (i = 0; i < NumStats; i++)
+	{
+	  if (Stats[i].name)
+	    Xfree (Stats[i].name);
+	  if (Stats[i].text)
+	    Xfree (Stats[i].text);
+	}
+      Xfree (Stats);
+      Stats = NULL;
+      NumStats = 0;
+    }
+}
+
+static void 
+Add_Stats (S_Type typ, char *name, char *fmt,...)
+{
+  char buf[1024];
+  va_list args;
+  va_start (args, fmt);
+  vsprintf (buf, fmt, args);
+  if (Stats)
+    Stats = Xrealloc (Stats, (++NumStats) * sizeof (*Stats));
+  else
+    Stats = Xmalloc ((++NumStats) * sizeof (*Stats));
+  Stats[NumStats - 1].typ = typ;
+  Stats[NumStats - 1].name = Xstrdup (name);
+  Stats[NumStats - 1].text = Xstrdup (buf);
+}
+
+static void 
+Display_Stats (FILE * f)
+{
+  static char *sep = "-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="
+  "-=-=-=-=-=-=-=-=-=-=-";
+  char *names[] =
+  {"Time:  ", "Memory:", "Sizes: ", "Disk:  ", "       "};
+  int i, last_typ = -1;
+  size_t len = 0;
+  if (NumStats == 0)
+    return;
+  fprintf (f, "%s\n", sep);
+  for (i = 0; i < NumStats; i++)
+    if (strlen (Stats[i].name) > len)
+      len = strlen (Stats[i].name);
+  for (i = 0; i < NumStats; i++)
+    {
+      int typ = 4;
+      if (Stats[i].typ != last_typ)
+	typ = last_typ = Stats[i].typ;
+      fprintf (f, "%s %-*s %s\n", names[typ], (int) len, Stats[i].name, Stats[i].text);
+    }
+  fprintf (f, "%s\n", sep);
+}
+
+/*****************************************************************************/
+
+
+static void 
+QueryTimeStats (ProgTime * Start, ProgTime * invf, ProgTime * text)
+{
+  if (!BooleanEnv (GetEnv ("briefstats"), 0))
+    {
+      Add_Stats (S_Time, "invf", ElapsedTime (Start, invf));
+      Add_Stats (S_Time, "text", ElapsedTime (invf, text));
+    }
+  Add_Stats (S_Time, "total", ElapsedTime (Start, text));
+}
+
+static void 
+StartUpTimeStats (InitQueryTimes * iqt)
+{
+  if (!BooleanEnv (GetEnv ("briefstats"), 0))
+    {
+      Add_Stats (S_Time, "dict [stem]", ElapsedTime (&iqt->Start,
+						     &iqt->StemDict));
+      Add_Stats (S_Time, "weights", ElapsedTime (&iqt->StemDict,
+						 &iqt->ApproxWeights));
+      Add_Stats (S_Time, "dict [text]", ElapsedTime (&iqt->ApproxWeights,
+						     &iqt->CompDict));
+      Add_Stats (S_Time, "Inverted", ElapsedTime (&iqt->CompDict,
+						  &iqt->Invf));
+      Add_Stats (S_Time, "Compressed", ElapsedTime (&iqt->Invf,
+						    &iqt->Text));
+    }
+  Add_Stats (S_Time, "total", ElapsedTime (&iqt->Start, &iqt->Text));
+}
+
+
+
+
+#ifdef USE_PROCESS_MEM
+static u_long 
+process_mem (void)
+{
+  prstatus_t pr;
+  static int fd = -1;
+  if (fd == -1)
+    {
+      char buf[128];
+      sprintf (buf, "/proc/%ld", (long) getpid ());
+      fd = open (buf, O_RDONLY);
+    }
+  if (fd == -1 || ioctl (fd, PIOCSTATUS, &pr) == -1)
+    return 0;
+  return pr.pr_brksize;
+}
+#endif
+
+
+
+
+static void 
+MemStats (query_data * qd)
+{
+  if (!BooleanEnv (GetEnv ("briefstats"), 0))
+    {
+#ifdef HAVE_GETRUSAGE
+      struct rusage rusage;
+      getrusage (RUSAGE_SELF, &rusage);
+
+      Add_Stats (S_Mem, "process mem", "%7.3f Mb",
+	    (double) (rusage.ru_maxrss * getpagesize () / 1024.0 / 1024.0));
+#endif
+#ifdef USE_PROCESS_MEM
+      Add_Stats (S_Mem, "process mem", "%7.3f Mb",
+		 (double) (process_mem () / 1024.0 / 1024.0));
+#endif
+      Add_Stats (S_Mem, "dict [stem]", "%7.1f kB",
+		 (double) qd->sd->MemForStemDict / 1024);
+      Add_Stats (S_Mem, "dict [text]", "%7.1f kB",
+		 (double) qd->cd->MemForCompDict / 1024);
+      if (qd->awd)
+	Add_Stats (S_Mem, "weights", "%7.1f kB",
+		   (double) qd->awd->MemForWeights / 1024);
+    }
+  if (qd->awd)
+    Add_Stats (S_Mem, "total [peak]", "%7.1f kB",
+	       (double) (qd->max_mem_in_use + qd->sd->MemForStemDict +
+		   qd->cd->MemForCompDict + qd->awd->MemForWeights) / 1024);
+  else
+    Add_Stats (S_Mem, "total [peak]", "%7.1f kB",
+	       (double) (qd->max_mem_in_use + qd->sd->MemForStemDict +
+			 qd->cd->MemForCompDict) / 1024);
+
+}
+
+
+
+static void 
+SizeStats (query_data * qd)
+{
+  Add_Stats (S_Size, "skips", "%7d", qd->hops_taken);
+  Add_Stats (S_Size, "pointers", "%7d", qd->num_of_ptrs);
+  Add_Stats (S_Size, "accumulators", "%7d", qd->num_of_accum);
+  Add_Stats (S_Size, "terms", "%7d", qd->num_of_terms);
+  Add_Stats (S_Size, "answers", "%7d", qd->num_of_ans);
+  Add_Stats (S_Size, "index lookups", "%7d", qd->text_idx_lookups);
+}
+
+static void 
+TotalSizeStats (query_data * qd)
+{
+  Add_Stats (S_Size, "skips", "%7d", qd->tot_hops_taken);
+  Add_Stats (S_Size, "pointers", "%7d", qd->tot_num_of_ptrs);
+  Add_Stats (S_Size, "accumulators", "%7d", qd->tot_num_of_accum);
+  Add_Stats (S_Size, "terms", "%7d", qd->tot_num_of_terms);
+  Add_Stats (S_Size, "answers", "%7d", qd->tot_num_of_ans);
+  Add_Stats (S_Size, "index lookups", "%7d", qd->tot_text_idx_lookups);
+}
+
+
+static void 
+StatFile (File * F)
+{
+  static unsigned long NumBytes = 0, NumSeeks = 0, NumReads = 0;
+  if (F)
+    if ((int) F != -1)
+      {
+	if (!BooleanEnv (GetEnv ("briefstats"), 0))
+	  Add_Stats (S_File, F->name, "%7.1f kB (%3d seeks, %7d reads)",
+		   (double) F->Current.NumBytes / 1024, F->Current.NumSeeks,
+		     F->Current.NumReads);
+	NumBytes += F->Current.NumBytes;
+	NumSeeks += F->Current.NumSeeks;
+	NumReads += F->Current.NumReads;
+      }
+    else
+      {
+	Add_Stats (S_File, "total", "%7.1f kB (%3d seeks, %7d reads)",
+		   (double) NumBytes / 1024, NumSeeks, NumReads);
+	NumSeeks = NumReads = NumBytes = 0;
+      }
+
+}
+
+
+static void 
+File_Stats (query_data * qd)
+{
+  StatFile (qd->File_comp_dict);
+  StatFile (qd->File_fast_comp_dict);
+  StatFile (qd->File_text_idx_wgt);
+  StatFile (qd->File_text);
+  StatFile (qd->File_stem);
+
+  /* [RPAP - Jan 97: Stem Index Change] */
+  if (qd->sd->sdh.indexed)
+    {
+      StatFile (qd->File_stem1);
+      StatFile (qd->File_stem2);
+      StatFile (qd->File_stem3);
+    }
+
+  StatFile (qd->File_invf);
+  StatFile (qd->File_weight_approx);
+  StatFile (qd->File_text_idx);
+  StatFile ((File *) (-1));
+}
+
+
+char *
+get_query (query_data * qd)
+{
+  char *line, *LinePtr;
+  WritePrompt ();
+  do
+    {
+      do
+	{
+	  line = GetMultiLine ();
+	  if (line == NULL)
+	    {
+	      if (stdin == InFile)
+		return (NULL);	/* EOF */
+	      if (InPipe)
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+		_pclose (InFile);
+#else
+		pclose (InFile);
+#endif
+	      else
+		fclose (InFile);
+	      InPipe = 0;
+	      InFile = stdin;
+	    }
+	}
+      while (line == NULL);
+      LinePtr = ProcessCommands (line, qd);
+      if (CommandsErrorStr)
+	fprintf (stderr, "%s\n", CommandsErrorStr);
+    }
+  while (*LinePtr == '\0' && !Quitting);
+  return (LinePtr);
+}
+
+
+/* This is executed when a SIGPIPE is detected
+   i.e. If some one quits out of the PAGER, this is executed */
+#ifdef HAVE_SIGCONTEXT
+static RETSIGTYPE 
+SIGPIPE_handler (int sig, int code,
+		 struct sigcontext *scp, char *addr)
+#else
+static RETSIGTYPE 
+SIGPIPE_handler (int sig)
+#endif
+{
+  /* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+  signal (sig, SIG_IGN);
+#else
+  signal (SIGPIPE, SIG_IGN);
+#endif
+  PagerRunning = 0;
+}
+
+/* This is executed when a SIGINT (i.e. CTRL-C) is detected */
+#ifdef  HAVE_SIGCONTEXT
+static RETSIGTYPE 
+SIGINT_handler (int sig, int code,
+		struct sigcontext *scp, char *addr)
+#else
+static RETSIGTYPE 
+SIGINT_handler (int sig)
+#endif
+{
+  Ctrl_C = 1;
+}
+
+
+
+static char *post_proc = NULL;
+
+
+
+void 
+GetPostProc (char *line)
+{
+  char *start, *finish;
+  if (post_proc)
+    {
+      Xfree (post_proc);
+      post_proc = NULL;
+    }
+  start = strchr (line, '\"');
+  finish = strrchr (line, '\"');
+  if (start != finish)
+    {
+      /* found a pattern */
+      *finish = '\0';
+      post_proc = Xstrdup (start + 1);
+      strcpy (start, finish + 1);
+      if (BooleanEnv (GetEnv ("verbatim"), 1) == 0)
+	{
+	  char *s;
+	  s = re_comp (post_proc);
+	  if (!s)
+	    {
+	      Xfree (post_proc);
+	      post_proc = NULL;
+	    }
+	}
+    }
+  else if (start != NULL)
+    {
+      /* found a single speech mark. Delete It. */
+      strcpy (start, start + 1);
+    }
+}
+
+int 
+PostProc (char *UDoc, int verbatim)
+{
+  if (!post_proc)
+    return 1;
+
+  if (verbatim) {
+    return (strstr (UDoc, post_proc) != NULL);
+  }
+  return re_exec ((char *) UDoc);
+}
+
+
+
+static DocEntry *
+in_chain (int para, int ip, DocEntry * dc)
+{
+  while (dc)
+    {
+      if (dc->DocNum - ip == para)
+	return dc;
+      dc = dc->Next;
+    }
+  return NULL;
+}
+
+/* num should be greater than or equal to 1 */
+int 
+RawDocOutput (query_data * qd, u_long num, FILE * Output)
+{
+  static last_pos = 0;
+  static u_char *c_buffer = 0;
+  static int buf_len = -1;
+  static u_char *uc_buffer = 0;
+  u_long pos, len;
+  int ULen;
+
+  FetchDocStart (qd, num, &pos, &len);
+
+  if ((int) len > buf_len)
+    {
+      if (c_buffer)
+	{
+	  Xfree (c_buffer);
+	  Xfree (uc_buffer);
+	}
+      if (!(c_buffer = Xmalloc (len)))
+	return -1;
+      if (!(uc_buffer = Xmalloc ((int) (qd->td->cth.ratio * 1.01 *
+					len) + 100)))
+	return -1;
+      buf_len = len;
+    }
+  if (last_pos != pos)
+    Fseek (qd->td->TextFile, pos, 0);
+  Fread (c_buffer, 1, len, qd->td->TextFile);
+  last_pos = pos + len;
+  DecodeText (qd->cd, c_buffer, len, uc_buffer, &ULen);
+  fwrite (uc_buffer, ULen, sizeof (u_char), Output);
+  return 0;
+}
+
+
+void 
+StringOut (FILE * Output, char *string,
+	   int intvalid, unsigned long intval,
+	   int floatvalid, double floatval)
+{
+  char *s;
+  for (s = string; *s; s++)
+    if (*s == '%' &&
+	(*(s + 1) == 'n' || *(s + 1) == 'w' || *(s + 1) == '%'))
+      {
+	s++;
+	switch (*s)
+	  {
+	  case 'n':
+	    if (intvalid)
+	      fprintf (Output, "%lu", intval);
+	    else
+	      fprintf (Output, "%%n");
+	    break;
+	  case 'w':
+	    if (floatvalid)
+	      fprintf (Output, "%f", floatval);
+	    else
+	      fprintf (Output, "%%w");
+	    break;
+	  case '%':
+	    fputc ('%', Output);
+	  }
+      }
+    else
+      fputc (*s, Output);
+}
+
+
+void 
+HeaderOut (FILE * Output, u_char * UDoc, unsigned long ULen, int heads_length)
+{
+  int i, space = 1, num = 0;
+  for (i = 0; i < ULen && num < heads_length; i++)
+    {
+      char c = UDoc[i];
+      if (c == '\02')
+	break;
+
+      if (isspace (c) || c == '\01' || c == '\03')
+	{
+	  if (!space)
+	    {
+	      fputc (' ', Output);
+	      num++;
+	    }
+	  space = 1;
+	}
+      else
+	{
+	  space = 0;
+	  fputc (c, Output);
+	  num++;
+	}
+    }
+}
+
+/* [RPAP - Feb 97: NZDL Additions] */
+#if defined(PARADOCNUM) || defined(NZDL)
+void PrintDocNum(FILE *output, char query_type,
+		 int docnum, int indexnum, float weight)
+{
+  if (query_type == 'R' || query_type == 'A')
+    fprintf(output, "%7d.%-7d  %6.4f\n", docnum, indexnum, weight);
+  else
+    fprintf(output, "%7d.%-7d\n", docnum, indexnum);
+}
+#endif
+
+static int 
+ProcessDocs (query_data * qd, int num, int verbatim,
+	     char OutputType, FILE * Output)
+{
+  int max_buf = 0;
+  int DocCount = 0;
+  char *doc_sepstr = NULL;
+  char *para_sepstr = NULL;
+  char *para_start = NULL;
+  int heads_length = atoi (GetDefEnv ("heads_length", "50"));
+  char QueryType = get_query_type ();
+  int need_text = (OutputType == OUTPUT_TEXT || OutputType == OUTPUT_HILITE ||
+		   OutputType == OUTPUT_HEADERS || OutputType == OUTPUT_SILENT || 
+		   post_proc); /* [RJM June 1997 -- fixing post retrieval scan] */
+
+  if (OutputType == OUTPUT_TEXT || OutputType == OUTPUT_HILITE)
+    {
+      if (QueryType == QUERY_APPROX || QueryType == QUERY_RANKED)
+	{
+	  doc_sepstr = de_escape_string (
+				    Xstrdup (GetDefEnv ("ranked_doc_sepstr",
+			   "---------------------------------- %n %w\\n")));
+	}
+      else
+	{
+	  doc_sepstr = de_escape_string (
+					  Xstrdup (GetDefEnv ("doc_sepstr",
+			      "---------------------------------- %n\\n")));
+	}
+      para_sepstr = de_escape_string (
+				       Xstrdup (GetDefEnv ("para_sepstr",
+				  "\\n######## PARAGRAPH %n ########\\n")));
+
+      para_start = de_escape_string (
+				      Xstrdup (GetDefEnv ("para_start",
+					    "***** Weight = %w *****\\n")));
+    }
+
+  if (need_text)
+    {
+      max_buf = atoi (GetDefEnv ("buffer", "1048576"));
+    }
+
+  do
+    {
+      u_char *UDoc = NULL;
+      unsigned long ULen;
+
+      if (need_text)
+	{
+	  /* load the compressed text */
+	  if (LoadCompressedText (qd, max_buf))
+	    {
+	      Message ("Unable to load compressed text.");
+	      FatalError (1, "This is probably due to lack of memory.");
+	    }
+
+	  /* uncompress the loaded text */
+	  UDoc = GetDocText (qd, &ULen);
+	  if (UDoc == NULL)
+	    FatalError (1, "UDoc is unexpectedly NULL");
+	}
+
+      if (!UDoc || PostProc ((char *) UDoc, verbatim))
+	{
+	  switch (OutputType)
+	    {
+	    case OUTPUT_COUNT:
+	    case OUTPUT_SILENT:
+	      break;
+	    case OUTPUT_DOCNUMS:	/* This prints out the docnums string */
+	      if (PagerRunning)
+		{
+
+/* [RPAP - Feb 97: NZDL Additions] */
+#if defined(PARADOCNUM) || defined(NZDL)
+		  int doc_num = GetDocNum(qd);
+
+		  if (qd->paragraph)
+		    {
+		      if (qd->id->ifh.InvfLevel == 3 &&
+			  (QueryType == 'R' || QueryType == 'A'))
+			{
+			  /* Print weights for each paragraph in document */
+			  
+
+			  int true_doc_num = GetDocNumFromParaNum(qd, doc_num);
+
+			  /* Get number of paragraphs in this document */
+
+			  int num_paragraphs = 
+			    qd->paragraph[true_doc_num]-qd->paragraph[true_doc_num-1];
+
+			  int init_para = FetchInitialParagraph(qd->td,
+								doc_num);
+			  DocEntry *de, *doc_chain = GetDocChain(qd);
+			  int i;
+
+			  for (i = 0; i < num_paragraphs; i++)
+			    {
+			      if ((de = in_chain(i, init_para, doc_chain)))
+				PrintDocNum(Output, QueryType,
+					    true_doc_num, init_para+i,
+					    de->Weight);
+			    }
+			}
+		      else
+			PrintDocNum(Output, QueryType,
+				    GetDocNumFromParaNum(qd, GetDocNum(qd)),
+				    GetDocNum(qd),
+				    GetDocWeight(qd));
+		    }
+		  else
+		    {
+		      PrintDocNum(Output, QueryType,
+				  doc_num, doc_num, GetDocWeight(qd));
+		    }
+#else
+		fprintf (Output, "%7d   %6.4f   %7lu\n", GetDocNum (qd),
+			 GetDocWeight (qd), GetDocCompLength (qd));
+#endif
+		}
+	      break;
+	    case OUTPUT_HEADERS:	/* This prints out the headers of the documents */
+	      if (PagerRunning)
+		fprintf (Output, "%d ", GetDocNum (qd));
+	      HeaderOut (Output, UDoc, ULen, heads_length);
+	      if (PagerRunning)
+		fputc ('\n', Output);
+	      break;
+#if TREC_MODE
+	    case OUTPUT_EXTRAS:	/* This prints out the docnums string */
+	      if (PagerRunning && trec_ids)
+		{
+		  long DN, PN = GetDocNum (qd) - 1;
+		  if (trec_paras)
+		    DN = trec_paras[PN];
+		  else
+		    DN = PN;
+		  fprintf (Output, "%-14.14s  %8ld  %10.5f\n",
+			   &trec_ids[DN * 14], PN + 1, GetDocWeight (qd));
+		}
+	      break;
+#endif
+	    case OUTPUT_TEXT:
+	    case OUTPUT_HILITE:
+	      {
+		int j, para = -1, curr_para = 0;
+		int init_para = -1;
+		DocEntry *de, *doc_chain = NULL;
+		register char ch = ' ';
+		register char lch = '\n';
+
+/* [RPAP - Feb 97: NZDL Additions] */
+#if defined(PARADOCNUM) || defined(NZDL)
+		if (qd->id->ifh.InvfLevel == 3)
+		  {
+		    init_para = FetchInitialParagraph(qd->td, GetDocNum(qd));
+		    
+		    StringOut(Output, para_sepstr, 
+			      1, init_para+curr_para,
+			      0, 0);
+
+		  }
+		else
+		  StringOut(Output, doc_sepstr, 
+			    1, GetDocNum(qd),
+			    QueryType == 'A' || QueryType == 'R',
+			    GetDocWeight(qd));
+
+#else
+		int p_on = 0;
+
+		if (PagerRunning)
+		  {
+		    StringOut (Output, doc_sepstr,
+			       1, GetDocNum (qd),
+			       QueryType == 'A' || QueryType == 'R',
+			       GetDocWeight (qd));
+		  }
+		if (qd->id->ifh.InvfLevel == 3)
+		  {
+		    init_para = FetchInitialParagraph (qd->td, GetDocNum (qd));
+		    doc_chain = GetDocChain (qd);
+		    para = GetDocNum (qd) - init_para;
+
+		    StringOut (Output, para_sepstr,
+			       1, curr_para + 1,
+			       0, 0);
+
+		    if ((de = in_chain (0, init_para, doc_chain)))
+		      StringOut (Output, para_start,
+				 0, 0,
+				 1, de->Weight);
+
+		    if (doc_chain->DocNum - init_para == 0)
+		      p_on = 1;
+		  }
+#endif
+		for (j = 0; j < ULen; j++)
+		  {
+		    ch = UDoc[j];
+		    switch (ch)
+		      {
+		      case '\02':
+			break;
+		      case '\01':
+			ch = '\n';
+		      case '\03':
+/* [RPAP - Feb 97: NZDL Additions] */
+#if defined(PARADOCNUM) || defined(NZDL)
+			/* print paragraph numbers only if this is
+			   a level 3 index */
+			if (qd->id->ifh.InvfLevel == 3)
+			  {
+			    curr_para++;
+			    StringOut(Output, para_sepstr, 
+				      1, init_para+curr_para,
+				      0, 0);
+			  }
+#else
+			p_on = 0;
+			curr_para++;
+			StringOut (Output, para_sepstr,
+				   1, curr_para + 1,
+				   0, 0);
+			lch = *(strchr (para_sepstr, '\0') - 1);
+			if ((de = in_chain (curr_para, init_para, doc_chain)))
+			  StringOut (Output, para_start,
+				     0, 0,
+				     1, de->Weight);
+			if (doc_chain &&
+			    doc_chain->DocNum - init_para == curr_para)
+			  p_on = 1;
+#endif
+			break;
+		      default:
+			{
+			  if (PagerRunning)
+			    {
+			      fputc (ch, Output);
+/* [RPAP - Feb 97: NZDL Additions] */
+#if !defined(PARADOCNUM) && !defined(NZDL)
+			      if (p_on && isprint (ch))
+				{
+				  fputc ('\b', Output);
+				  fputc ('_', Output);
+				}
+#endif
+			    }
+
+			  lch = ch;
+			}
+		      }
+		  }
+		if (PagerRunning && lch != '\n')
+		  fputc ('\n', Output);
+/* [RPAP - Feb 97: NZDL Additions] */
+#if !defined(PARADOCNUM) && !defined(NZDL)
+		p_on = 0;
+#endif
+	      }
+	    }
+	  if (PagerRunning)
+	    fflush (Output);
+
+	  DocCount++; /* moved within if statement [RJM June 1997 -- fixing post retrieval scan] */
+	}
+    }
+  while (NextDoc (qd) && PagerRunning && (!Ctrl_C));
+
+  if (need_text)
+    {
+      FreeTextBuffer (qd);
+    }
+
+  if (OutputType == OUTPUT_TEXT || OutputType == OUTPUT_HILITE)
+    {
+      Xfree (doc_sepstr);
+      Xfree (para_sepstr);
+      Xfree (para_start);
+    }
+
+  return (DocCount);
+}
+
+
+void 
+output_terminator (FILE * out)
+{
+  char *terminator = Xstrdup (GetDefEnv ("terminator", ""));
+  de_escape_string (terminator);
+  fputs (terminator, out);
+  Xfree (terminator);
+}
+
+
+
+
+/* MoreDocs () */
+/* Displays all documents in list  DocList. */
+/* Documents are fetched, then decompressed and displayed according to the */
+/* format implied in  FormString(). */
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+# define HILITE_PAGER "mg_hilite_words.exe"
+#else
+# define HILITE_PAGER "mg_hilite_words"
+#endif
+
+#define MAX_HILITE_PAGER_STR 80	/* for command & its options */
+
+static void 
+MoreDocs (query_data * qd, char *Query, char OutputType)
+{
+  static char terms_str[MAXTERMSTRLEN + 1];
+  int DocCount = 0;		/* number of actual matches */
+  FILE *Output = NULL;
+  int using_pipe = 0;
+  char *pager = NULL;
+
+  Ctrl_C = 0;
+
+  qd->num_of_ans = qd->DL->num;
+
+  /* [RPAP - Feb 97: WIN32 Port] */
+#ifndef __WIN32__
+  signal (SIGPIPE, SIGPIPE_handler);
+#endif
+  signal (SIGINT, SIGINT_handler);
+
+  PagerRunning = 1;
+  if (isatty (fileno (OutFile)) && GetEnv ("pager") &&
+      OutputType != OUTPUT_HILITE &&
+      OutputType != OUTPUT_SILENT && OutputType != OUTPUT_COUNT)
+    {
+      pager = GetEnv ("pager");
+    }
+  else if (isatty (fileno (OutFile)) && OutputType == OUTPUT_HILITE)
+    {
+      /* concat the pager and its word argument strings */
+      ConvertTermsToString (qd->TL, terms_str);
+      pager = Xmalloc (MAX_HILITE_PAGER_STR + strlen (terms_str) + 1);
+      if (!pager)
+	{
+	  fprintf (stderr, "Unable to allocate memory for highlighting\n");
+	  return;
+	}
+      sprintf (pager, "%s --style=%s --pager=%s --stem_method=%ld --stemmer=%ld %s",
+	       HILITE_PAGER,
+	       GetEnv ("hilite_style"),
+	       GetEnv ("pager"),
+	       qd->sd->sdh.stem_method,
+	       qd->sd->sdh.stemmer_num,
+	       terms_str);
+
+    }
+  else
+    {
+      Output = OutFile;
+    }
+
+/* [RPAP - Feb 97: NZDL Additions] */
+#if defined(OUTPUTSTEMMEDWORDS) || defined(NZDL)
+  if (!isatty(fileno(OutFile)) && get_query_type() != QUERY_DOCNUMS)
+    {
+      ConvertTermsToString(qd->TL, terms_str);
+      fprintf(Output, "%s\n", terms_str);
+    }
+#endif
+  if (pager)
+    {
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+      Output = _popen (pager, "w");
+#else
+      Output = popen (pager, "w");
+#endif
+      using_pipe = (Output != NULL);
+      if (!using_pipe)
+	{
+	  fprintf (stderr, "Unable to run \"%s\"\n", pager);
+	  return;
+	}
+    }
+
+
+  if (qd->DL->num > 0)
+    {
+      if (OutputType == OUTPUT_COUNT && !post_proc)
+	DocCount = qd->DL->num;
+      else {
+	DocCount = ProcessDocs (qd, qd->DL->num,
+				BooleanEnv (GetEnv ("verbatim"), 1),
+				OutputType, Output);
+      }
+    }
+
+  if (PagerRunning)
+    {
+      output_terminator (Output);
+      fflush (Output);
+    }
+
+  if (OutputType == OUTPUT_HILITE && pager)
+    free (pager);		/* as needed to malloc to create the pager string */
+
+  if (using_pipe)
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+    _pclose (Output);
+#else
+    pclose (Output);
+#endif
+
+  if (qd->DL->num == 0)
+    fprintf (stderr, "No entries correspond to that query.\n");
+  else
+    {
+      if (OutputType == OUTPUT_COUNT)
+	fprintf (stderr, "%d documents match.\n", DocCount);
+      else
+	fprintf (stderr, "%d documents retrieved.\n", DocCount);
+    }
+
+  signal (SIGINT, SIG_DFL);
+}
+
+
+void 
+start_up_stats (query_data * qd, InitQueryTimes iqt)
+{
+  Clear_Stats ();
+  if (BooleanEnv (GetEnv ("timestats"), 0) ||
+      BooleanEnv (GetEnv ("briefstats"), 0))
+    StartUpTimeStats (&iqt);
+
+  if (BooleanEnv (GetEnv ("diskstats"), 0) ||
+      BooleanEnv (GetEnv ("briefstats"), 0))
+    File_Stats (qd);
+
+  if (BooleanEnv (GetEnv ("memstats"), 0) ||
+      BooleanEnv (GetEnv ("briefstats"), 0))
+    MemStats (qd);
+
+}
+
+
+void 
+shut_down_stats (query_data * qd, ProgTime * start,
+		 ProgTime * invf, ProgTime * text)
+{
+  Clear_Stats ();
+  if (BooleanEnv (GetEnv ("timestats"), 0) ||
+      BooleanEnv (GetEnv ("briefstats"), 0))
+    QueryTimeStats (start, invf, text);
+
+  if (BooleanEnv (GetEnv ("diskstats"), 0) ||
+      BooleanEnv (GetEnv ("briefstats"), 0))
+    {
+      TransFileStats (qd);
+      File_Stats (qd);
+    }
+
+  if (BooleanEnv (GetEnv ("sizestats"), 0))
+    TotalSizeStats (qd);
+}
+
+
+
+char *wordfreqword2str (u_char * s)
+{
+  static char buf[1024];
+  int i, len = (int) *s++;
+
+  for (i = 0; i < len; i++)
+    {
+      buf[i] = (char)s[i];
+    }
+  buf[len] = '\0';
+
+  return buf;
+}
+
+
+/* [RPAP - Feb 97: Term Frequency] */
+/*********************************
+ * PrintQueryTermFreq
+ *
+ * Prints the query terms and their respective frequencies within the collection
+ *********************************/
+void
+PrintQueryTermFreqs (QueryTermList *qtl)
+{
+  int i;
+
+  /* Print the number of terms */
+  fprintf (OutFile, "%d\n", qtl->num);
+
+  /* Print the terms and their respective frequency within the collection */
+  for (i = 0; i < qtl->num; i++)
+    if (qtl->QTE[i].stem_method == -1)
+      /* Using default stem method - don't print stem method beside term */
+      fprintf (OutFile, "%s %d\n", wordfreqword2str (qtl->QTE[i].Term), qtl->QTE[i].Count);
+    else
+      /* Term was forced with a stem, print stem method with term */
+      fprintf (OutFile, "%s#%d %d\n", wordfreqword2str (qtl->QTE[i].Term), qtl->QTE[i].stem_method, qtl->QTE[i].Count);
+}
+
+
+void 
+query (void)
+{
+  ProgTime TotalStartTime, TotalInvfTime, TotalTextTime;
+  InitQueryTimes iqt;
+  query_data *qd;
+
+  TotalStartTime.RealTime = TotalStartTime.CPUTime = 0;
+  TotalInvfTime.RealTime = TotalInvfTime.CPUTime = 0;
+  TotalTextTime.RealTime = TotalTextTime.CPUTime = 0;
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+  qd = InitQuerySystem (GetDefEnv ("mgdir", ".\\"),
+                        GetDefEnv ("mgname", ""),
+			GetDefEnv ("textname", NULL), /* [RJM 06/97: text filename] */
+			&iqt);
+#else
+  qd = InitQuerySystem (GetDefEnv ("mgdir", "./"),
+                        GetDefEnv ("mgname", ""),
+			GetDefEnv ("textname", NULL), /* [RJM 06/97: text filename] */
+                        &iqt);
+#endif
+
+  if (!qd)
+    FatalError (1, mg_errorstrs[mg_errno], mg_error_data);
+  start_up_stats (qd, iqt);
+
+
+  while (1)
+    {
+      ProgTime StartTime, InvfTime, TextTime;
+      char QueryType;
+      char OutputType;
+      char *line;
+      ResetFileStats (qd);
+      qd->max_mem_in_use = qd->mem_in_use = 0;
+
+      qd->tot_hops_taken += qd->hops_taken;
+      qd->tot_num_of_ptrs += qd->num_of_ptrs;
+      qd->tot_num_of_accum += qd->num_of_accum;
+      qd->tot_num_of_terms += qd->num_of_terms;
+      qd->tot_num_of_ans += qd->num_of_ans;
+      qd->tot_text_idx_lookups += qd->text_idx_lookups;
+      qd->hops_taken = qd->num_of_ptrs = 0;
+      qd->num_of_accum = qd->num_of_ans = qd->num_of_terms = 0;
+      qd->text_idx_lookups = 0;
+
+      Display_Stats (stderr);
+      Clear_Stats ();
+      line = get_query (qd);
+      if (!line || Quitting)
+	break;
+
+      GetPostProc (line);
+
+      GetTime (&StartTime);
+
+      FreeQueryDocs (qd);
+
+      QueryType = get_query_type ();
+      OutputType = get_output_type ();
+      /* No point in hiliting words on a docnum query */
+      if (OutputType == OUTPUT_HILITE && QueryType == QUERY_DOCNUMS)
+	OutputType = OUTPUT_TEXT;
+
+      switch (QueryType)
+	{
+	case QUERY_BOOLEAN:
+	  {
+	    char *maxdocs;
+	    BooleanQueryInfo bqi;
+	    maxdocs = GetDefEnv ("maxdocs", "all");
+	    bqi.MaxDocsToRetrieve = strcmp (maxdocs, "all") ? atoi (maxdocs) : -1;
+	    /* [RPAP - Jan 97: Stem Index Change] */
+	    if (qd->sd->sdh.indexed)
+	      BooleanQuery (qd, line, &bqi, (BooleanEnv (GetEnv ("casefold"), 0) |
+					     (BooleanEnv (GetEnv ("stem"), 0) << 1)));
+	    else
+	      BooleanQuery (qd, line, &bqi, qd->sd->sdh.stem_method);
+
+	    break;
+	  }
+	case QUERY_APPROX:
+	case QUERY_RANKED:
+	  {
+	    char *maxdocs;
+	    char *maxterms;
+	    char *maxaccum;
+	    RankedQueryInfo rqi;
+	    maxdocs = GetDefEnv ("maxdocs", "all");
+	    maxterms = GetDefEnv ("max_terms", "all");
+	    maxaccum = GetDefEnv ("max_accumulators", "all");
+	    rqi.Sort = BooleanEnv (GetEnv ("sorted_terms"), 0);
+	    rqi.QueryFreqs = BooleanEnv (GetEnv ("qfreq"), 1);
+	    rqi.Exact = QueryType == QUERY_RANKED;
+	    rqi.MaxDocsToRetrieve = strcmp (maxdocs, "all") ? atoi (maxdocs) : -1;
+	    rqi.MaxTerms = strcmp (maxterms, "all") ? atoi (maxterms) : -1;
+	    rqi.MaxParasToRetrieve = rqi.MaxDocsToRetrieve;
+	    if (qd->id->ifh.InvfLevel == 3 && GetEnv ("maxparas"))
+	      rqi.MaxParasToRetrieve = atoi (GetEnv ("maxparas"));
+	    rqi.AccumMethod = toupper (*GetDefEnv ("accumulator_method", "A"));
+	    rqi.MaxAccums = strcmp (maxaccum, "all") ? atoi (maxaccum) : -1;
+	    rqi.HashTblSize = IntEnv (GetEnv ("hash_tbl_size"), 1000);
+	    rqi.StopAtMaxAccum = BooleanEnv (GetEnv ("stop_at_max_accum"), 0);
+	    rqi.skip_dump = GetEnv ("skip_dump");
+	    RankedQuery (qd, line, &rqi);
+	    break;
+	  }
+	case QUERY_DOCNUMS:
+	  {
+	    DocnumsQuery (qd, line);
+	    break;
+	  }
+	}
+
+      GetTime (&InvfTime);
+
+      /* [RPAP - Feb 97: Term Frequency] */
+      if (qd->QTL && BooleanEnv (GetEnv ("term_freq"), 0))
+	PrintQueryTermFreqs (qd->QTL);
+
+      if (qd->DL)
+	MoreDocs (qd, line, OutputType);
+
+      GetTime (&TextTime);
+
+      if (BooleanEnv (GetEnv ("timestats"), 0) ||
+	  BooleanEnv (GetEnv ("briefstats"), 0))
+	QueryTimeStats (&StartTime, &InvfTime, &TextTime);
+
+      if (BooleanEnv (GetEnv ("diskstats"), 0) ||
+	  BooleanEnv (GetEnv ("briefstats"), 0))
+	File_Stats (qd);
+
+      if (BooleanEnv (GetEnv ("memstats"), 0) ||
+	  BooleanEnv (GetEnv ("briefstats"), 0))
+	MemStats (qd);
+
+      if (BooleanEnv (GetEnv ("sizestats"), 0))
+	SizeStats (qd);
+
+      TotalInvfTime.RealTime += InvfTime.RealTime - StartTime.RealTime;
+      TotalInvfTime.CPUTime += InvfTime.CPUTime - StartTime.CPUTime;
+      TotalTextTime.RealTime += TextTime.RealTime - StartTime.RealTime;
+      TotalTextTime.CPUTime += TextTime.CPUTime - StartTime.CPUTime;
+    }
+
+  if (isatty (fileno (InFile)) && !Quitting)
+    fprintf (stderr, "\n");
+
+  shut_down_stats (qd, &TotalStartTime, &TotalInvfTime, &TotalTextTime);
+
+  Display_Stats (stderr);
+
+}
+
+
+void 
+search_for_collection (char *name)
+{
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+  char *dir = GetDefEnv ("mgdir", ".\\");
+#else
+  char *dir = GetDefEnv ("mgdir", "./");
+#endif
+  char buffer[512];
+  struct stat stat_buf;
+  if (strrchr (dir, '/') && *(strrchr (dir, '/') + 1) != '\0')
+    {
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+      sprintf (buffer, "%s", dir);
+#else
+      sprintf (buffer, "%s/", dir);
+#endif
+      SetEnv ("mgdir", buffer, NULL);
+      dir = GetEnv ("mgdir");
+    }
+
+  sprintf (buffer, "%s.text", name);
+  if (stat (buffer, &stat_buf) != -1)
+    {
+      if ((stat_buf.st_mode & S_IFREG) != 0)
+	{
+	  /* The name is a directory */
+	  SetEnv ("mgname", name, NULL);
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+          SetEnv ("mgdir", ".\\", NULL);
+#else
+	  SetEnv ("mgdir", "./", NULL);
+#endif
+	  return;
+	}
+    }
+
+  sprintf (buffer, "%s%s", dir, name);
+  if (stat (buffer, &stat_buf) != -1)
+    {
+      if ((stat_buf.st_mode & S_IFDIR) != 0)
+	{
+	  /* The name is a directory */
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+          sprintf (buffer, "%s%s", name, name);
+#else
+	  sprintf (buffer, "%s/%s", name, name);
+#endif
+	  SetEnv ("mgname", buffer, NULL);
+	  return;
+	}
+    }
+
+  /* Look in the current directory last */
+  if (stat (name, &stat_buf) != -1)
+    {
+      if ((stat_buf.st_mode & S_IFDIR) != 0)
+	{
+	  /* The name is a directory */
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+          sprintf (buffer, "%s%s", name, name);
+          SetEnv ("mgdir", ".\\", NULL);
+#else
+	  sprintf (buffer, "%s/%s", name, name);
+          SetEnv ("mgdir", "./", NULL);
+#endif
+	  SetEnv ("mgname", buffer, NULL);
+	  return;
+	}
+    }
+
+  SetEnv ("mgname", name, NULL);
+}
+
+/* main () */
+/* Initialises global variables based on command line switches, and opens */
+/* files.  Then calls  query ()  to perform the querying. */
+int main (int argc, char **argv)
+{
+  ProgTime StartTime;
+  int decomp = 0;
+  int ch;
+
+  msg_prefix = argv[0];
+  GetTime (&StartTime);
+
+  /* Initialise the environment with default values */
+
+  InitEnv ();
+
+  read_mgrc_file ();
+
+  OutFile = stdout;
+  InFile = stdin;
+
+  opterr = 0;
+  /* [RJM 06/97: text filename] */
+  while ((ch = getopt (argc, argv, "Df:d:t:h")) != -1) {
+    switch (ch) {
+      case 'f':
+	SetEnv ("mgname", optarg, NULL);
+	break;
+      case 'd':
+	SetEnv ("mgdir", optarg, NULL);
+	break;
+      case 't': /* [RJM 06/97: text filename] */
+	SetEnv ("textname", optarg, NULL);
+	break;
+      case 'D':
+	decomp = 1;
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-D] [-f base name of collection] "
+		 "[-t base name of files for text] " /* [RJM 06/97: text filename] */
+		 "[-d data directory] [collection]\n", argv[0]);
+	exit (1);
+    }      
+  }
+
+  PushEnv ();
+
+  if (decomp == 0)
+    {
+
+      Init_ReadLine ();
+
+      /* write a first prompt, let the user start thinking */
+      if (!BooleanEnv (GetEnv ("expert"), 0) && isatty (fileno (InFile)))
+	{
+	  fprintf (stderr, "\n\n\t     FULL TEXT RETRIEVAL QUERY PROGRAM\n");
+	  fprintf (stderr, "%24s%s\n\n", "", *"21 Mar 1994" == '%' ? __DATE__ : "21 Mar 1994");
+	  fprintf (stderr, "\n");
+	  fprintf (stderr, "  mgquery version " VERSION ", Copyright (C) 1994 Neil Sharman\n");
+	  fprintf (stderr, "  mgquery comes with ABSOLUTELY NO WARRANTY; for details type `.warranty'\n");
+	  fprintf (stderr, "  This is free software, and you are welcome to redistribute it\n");
+	  fprintf (stderr, "  under certain conditions; type `.conditions' for details.\n");
+	  fprintf (stderr, "\n");
+	}
+    }
+  if (optind < argc)
+    search_for_collection (argv[optind]);
+
+  if (decomp == 0)
+    {
+      query ();
+    }
+  else
+    {
+      int i;
+      InitQueryTimes iqt;
+      query_data *qd;
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+      qd = InitQuerySystem (GetDefEnv ("mgdir", ".\\"),
+                            GetDefEnv ("mgname", ""),
+                            GetDefEnv ("textname", NULL), /* [RJM 06/97: text filename] */
+                            &iqt);
+#else
+      qd = InitQuerySystem (GetDefEnv ("mgdir", "./"),
+			    GetDefEnv ("mgname", ""),
+                            GetDefEnv ("textname", NULL), /* [RJM 06/97: text filename] */
+			    &iqt);
+#endif
+      if (!qd)
+	FatalError (1, mg_errorstrs[mg_errno], mg_error_data);
+
+
+      start_up_stats (qd, iqt);
+
+      Display_Stats (stderr);
+      for (i = 0; i < qd->td->cth.num_of_docs; i++)
+	{
+	  RawDocOutput (qd, i + 1, stdout);
+	  putc ('\2', stdout);
+	}
+      Message ("%s", ElapsedTime (&StartTime, NULL));
+
+      FinishQuerySystem (qd);
+    }
+
+  UninitEnv ();
+  return 0;
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/mgstat.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/mgstat.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/mgstat.1	(revision 699)
@@ -0,0 +1,131 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgstat 1 \*(Dt CITRI
+.SH NAME
+mgstat \- print out statistics about a document collection
+.SH SYNOPSIS
+.B mgstat
+[
+.B \-h
+]
+[
+.B \-E
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mgstat
+prints out various statistics about an existing
+.BR mg (1)
+document collection.  Depending on the size of the collection, sizes
+will be printed in either kilobytes or megabytes.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stdout .
+.TP
+.B \-E
+This option forces sizes to be printed in bytes rather than kilobytes
+or megabytes.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.text
+Compressed documents.
+.TP
+.B *.invf
+Inverted file.
+.TP
+.B *.text.idx.wgt
+Interleaved index into the compressed documents and document weights.
+.TP
+.B *.weight.approx
+Approximate document weights.
+.TP
+.B *.invf.dict.blocked
+Compressed stemmed dictionary and index into the inverted file merged
+into an inverted file.
+.TP
+.B *.text.dict.fast
+Fast loading compression dictionary.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.invf.dict
+Compressed stemmed dictionary.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.TP
+.B *.text.stats
+Statistics about the text.
+.TP
+.B *.text.dict.aux
+Auxiliary compression dictionary.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.B *.invf.chunk
+Maps stemmed terms from occurrence order to lexical order.
+.TP
+.B *.invf.chunk.trans
+Describes where the source text is broken up into chunks for the
+inversion pass.
+.TP
+.B *.invf.dict.hash
+A perfect hash function for the terms in the stemmed dictionary.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/mgstat.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/mgstat.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/mgstat.c	(revision 699)
@@ -0,0 +1,399 @@
+/**************************************************************************
+ *
+ * mgstat.c -- Program to generate statistics on a text collection
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "locallib.h"
+#include "mg.h"
+#include "words.h"
+#include "invf.h"
+#include "text.h"
+
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:18:19  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:50  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:31  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.2  1994/09/20  04:41:59  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+char *search_for_collection (char *name);
+int process_file (char *, char *, int);
+void ProcessDict (char *);
+void ProcessStem (char *);
+long ProcessStemBlk (char *name);  /* [RPAP - Jan 97: Stem Index Change] */
+
+static double inputbytes = 0.0; /* [RJM 07/97: 4G limit] */
+static unsigned long total = 0;
+
+
+
+
+
+int main (int argc, char **argv)
+{
+  unsigned long sub_total;
+  int fast;
+  char *file_name = "";
+  int ch;
+  int exact = 0;
+  long indexed = 0; /* [RPAP - Jan 97: Stem Index Change] */
+  opterr = 0;
+  while ((ch = getopt (argc, argv, "Ehf:d:")) != -1)
+    switch (ch)
+      {
+      case 'E':
+	exact = 1;
+	break;
+      case 'f':		/* input file */
+	file_name = optarg;
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-f input_file]"
+		 "[-d data directory] [-h]\n", argv[0]);
+	exit (1);
+      }
+
+  if (optind < argc)
+    file_name = search_for_collection (argv[optind]);
+
+  ProcessDict (file_name);
+  ProcessStem (file_name);
+  indexed = ProcessStemBlk (file_name); /* [RPAP - Jan 97: Stem Index Change] */
+
+  {
+    char Name[256];
+    char *s;
+    sprintf (Name, FILE_NAME_FORMAT, get_basepath (), file_name, "");
+    s = strrchr (Name, '/');
+    if (s)
+      *s = '\0';
+    printf ("\nThe collection is in \"%s\"\n", Name);
+  }
+
+
+  printf ("\n\t\tFiles required by mgquery\n");
+  process_file (file_name, TEXT_SUFFIX, exact);
+  process_file (file_name, INVF_SUFFIX, exact);
+  process_file (file_name, TEXT_IDX_WGT_SUFFIX, exact);
+  process_file (file_name, APPROX_WEIGHTS_SUFFIX, exact);
+  process_file (file_name, INVF_DICT_BLOCKED_SUFFIX, exact);
+
+  /* [RPAP - Jan 97: Stem Index Change] */
+  if (indexed)
+    {
+      process_file (file_name, INVF_DICT_BLOCKED_1_SUFFIX, exact);
+      process_file (file_name, INVF_DICT_BLOCKED_2_SUFFIX, exact);
+      process_file (file_name, INVF_DICT_BLOCKED_3_SUFFIX, exact);
+    }
+
+  fast = process_file (file_name, TEXT_DICT_FAST_SUFFIX, exact);
+  if (!fast)
+    {
+      process_file (file_name, TEXT_DICT_SUFFIX, exact);
+      process_file (file_name, TEXT_DICT_AUX_SUFFIX, exact);
+    }
+
+
+  process_file (NULL, "SUB TOTAL", exact);
+  sub_total = total;
+  total = 0;
+
+  printf ("\n\t\tFiles NOT required by mgquery\n");
+  if (fast)
+    {
+      process_file (file_name, TEXT_DICT_SUFFIX, exact);
+      process_file (file_name, TEXT_DICT_AUX_SUFFIX, exact);
+    }
+  process_file (file_name, INVF_DICT_SUFFIX, exact);
+  process_file (file_name, INVF_IDX_SUFFIX, exact);
+  process_file (file_name, TEXT_STATS_DICT_SUFFIX, exact);
+  process_file (file_name, TEXT_IDX_SUFFIX, exact);
+  process_file (file_name, WEIGHTS_SUFFIX, exact);
+  process_file (file_name, INVF_CHUNK_SUFFIX, exact);
+  process_file (file_name, INVF_CHUNK_TRANS_SUFFIX, exact);
+  process_file (file_name, INVF_DICT_HASH_SUFFIX, exact);
+  process_file (file_name, INVF_PARAGRAPH_SUFFIX, exact);
+  process_file (NULL, "SUB TOTAL", exact);
+  total += sub_total;
+  printf ("\n");
+  process_file (NULL, "TOTAL", exact);
+
+  return 0;
+}
+
+
+
+char *
+search_for_collection (char *name)
+{
+  char *dir = get_basepath ();
+  static char buffer[512];
+  struct stat stat_buf;
+
+  /* Look in the current directory first */
+  if (stat (name, &stat_buf) != -1)
+    {
+      if (S_ISDIR(stat_buf.st_mode))
+	{
+	  /* The name is a directory */
+	  sprintf (buffer, "%s/%s", name, name);
+	  set_basepath (".");
+	  return buffer;
+	}
+    }
+
+  sprintf (buffer, "%s.text", name);
+  if (stat (buffer, &stat_buf) != -1)
+    {
+      if (S_ISREG(stat_buf.st_mode))
+	{
+	  /* The name is a directory */
+	  set_basepath (".");
+	  return name;
+	}
+    }
+  sprintf (buffer, "%s/%s", dir, name);
+  if (stat (buffer, &stat_buf) != -1)
+    {
+      if (S_ISDIR(stat_buf.st_mode))
+	{
+	  /* The name is a directory */
+	  sprintf (buffer, "%s/%s", name, name);
+	  return buffer;
+	}
+    }
+  return name;
+}
+
+
+
+
+
+
+void 
+ProcessDict (char *name)
+{
+  FILE *f;
+  compression_dict_header cdh;
+  int have_cdh = 0;
+  compressed_text_header cth;
+  int have_cth = 0;
+
+  if ((f = open_file (name, TEXT_DICT_SUFFIX, "rb", MAGIC_DICT, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      Read_cdh (f, &cdh, NULL, NULL);
+      fclose (f);
+      have_cdh = 1;
+    }
+
+  if ((f = open_file (name, TEXT_SUFFIX, "rb", MAGIC_TEXT, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      fread ((char *) &cth, sizeof (cth), 1, f);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(cth.num_of_docs);
+      NTOHD(cth.num_of_bytes); /* [RJM 07/97: 4G limit] */
+      NTOHUL(cth.num_of_words);
+      NTOHUL(cth.length_of_longest_doc);
+      NTOHD(cth.ratio);
+
+      fclose (f);
+      have_cth = 1;
+    }
+
+  if (have_cth)
+    {
+      inputbytes = cth.num_of_bytes;
+      printf ("Input bytes                        : %10.0f, %8.2f Mbyte\n",
+	      cth.num_of_bytes, cth.num_of_bytes / 1024.0 / 1024.0); /* [RJM 07/97: 4G limit] */
+      printf ("Documents                          : %10lu\n", cth.num_of_docs);
+      printf ("Words in collection [dict]         : %10lu\n", cth.num_of_words);
+      printf ("Longest doc in collection [dict]   : %10lu characters\n",
+	      cth.length_of_longest_doc);
+      printf ("Maximum ratio                      : %10.2f\n", cth.ratio);
+    }
+
+  if (have_cdh)
+    {
+      printf ("Words in dict                      : %10lu\n", cdh.num_words[1]);
+      printf ("Non-words in dict                  : %10lu\n", cdh.num_words[0]);
+      printf ("Total chars of distinct words      : %10lu\n", cdh.num_word_chars[1]);
+      printf ("Total chars of distinct non-words  : %10lu\n", cdh.num_word_chars[0]);
+    }
+
+}
+
+
+
+
+
+
+
+
+void 
+ProcessStem (char *name)
+{
+  FILE *f;
+  struct invf_dict_header idh;
+
+  if (!(f = open_file (name, INVF_DICT_SUFFIX, "rb",
+		       MAGIC_STEM_BUILD, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return;
+  fread ((char *) &idh, sizeof (idh), 1, f);
+  printf ("Words in collection [stem]         : %10ld\n", NTOHUL(idh.num_of_words));  /* [RPAP - Jan 97: Endian Ordering] */
+  printf ("Words in stem                      : %10ld\n", NTOHUL(idh.dict_size));  /* [RPAP - Jan 97: Endian Ordering] */
+  printf ("Indexed fragments                  : %10ld\n", NTOHUL(idh.num_of_docs));  /* [RPAP - Jan 97: Endian Ordering] */
+  printf ("Total chars of stem words          : %10ld\n", NTOHUL(idh.total_bytes));  /* [RPAP - Jan 97: Endian Ordering] */
+  fclose (f);
+}
+
+
+
+
+
+/* [RPAP - Jan 97: Stem Index Change] */
+long 
+ProcessStemBlk (char *name)
+{
+  FILE *f;
+  struct stem_dict_header sdh;
+
+  if (!(f = open_file (name, INVF_DICT_BLOCKED_SUFFIX, "rb",
+		       MAGIC_STEM, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return 0;
+  fread ((char *) &sdh, sizeof (sdh), 1, f);
+  fclose (f);
+  return NTOHUL(sdh.indexed);  /* [RPAP - Jan 97: Endian Ordering] */
+}
+
+
+
+
+int 
+process_file (char *name, char *ext, int exact)
+{
+  static double scale = 0;
+  static char *units;
+  struct stat buf;
+  if (scale == 0)
+    {
+      /* So can get output in Mb or Kb, */
+      /* will divide by a scale of 1024(Kb) or 1024*1024(Mb) */
+      /* Note: if inputbytes==0, then use 1024 as default */
+      scale = inputbytes > 10 * 1000 * 1000 ? 1024 * 1024 : 1024;
+      units = scale == 1024 ? "Kb" : "Mb";
+    }
+  if (name)
+    {
+      char Name[256];
+      sprintf (Name, FILE_NAME_FORMAT, get_basepath (), name, ext);
+      if (!stat (Name, &buf))
+	{
+	  char fname[256];
+	  char *nam = strrchr (name, '/');
+	  nam = nam ? nam + 1 : name;
+	  sprintf (fname, "%s%s", nam, ext);
+
+	  if (inputbytes == 0.0)
+	    {
+	      if (exact)
+		printf ("%s%*s: %10ld bytes   %7.3f%%\n", fname,
+			35 - (int) strlen (fname), "",
+			buf.st_size,
+			100.0 * buf.st_size / inputbytes);
+	      else
+		printf ("%s%*s: %8.2f %s   %7.3f%%\n", fname,
+			35 - (int) strlen (fname), "",
+			buf.st_size / scale,
+			units,
+			100.0 * buf.st_size / inputbytes);
+	    }
+	  else
+	    {
+	      if (exact)
+		printf ("%s%*s: %10ld bytes\n", fname,
+			35 - (int) strlen (fname), "",
+			buf.st_size);
+	      else
+		printf ("%s%*s: %8.2f %s\n", fname,
+			35 - (int) strlen (fname), "",
+			buf.st_size / scale,
+			units);
+	    }
+	  total += buf.st_size;
+	  return 1;
+	}
+      else
+	return 0;
+    }
+  else
+    {
+      if (inputbytes == 0.0)
+	{
+	  if (exact)
+	    printf ("%-34s : %10ld bytes   %7.3f%%\n", ext,
+		    total,
+		    100.0 * total / inputbytes);
+	  else
+	    printf ("%-34s : %8.2f %s   %7.3f%%\n", ext,
+		    total / scale, units,
+		    100.0 * total / inputbytes);
+	}
+      else
+	{
+	  if (exact)
+	    printf ("%-34s : %10ld bytes\n", ext, total);
+	  else
+	    printf ("%-34s : %8.2f %s\n", ext,
+		    total / scale, units);
+	}
+    }
+  return 1;
+
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/mgstemidxlist.1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/mgstemidxlist.1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/mgstemidxlist.1	(revision 699)
@@ -0,0 +1,86 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgstemidxlist 1 \*(Dt CITRI
+.SH NAME
+mgstemidxlist \- dumps the stem index file to STDOUT in ASCII format
+.SH SYNOPSIS
+.B mgstemidxlist
+[
+.B \-h
+]
+[
+.BI \-d " directory"
+]
+.B \-s 1|2|3
+.BI \-f " name"
+.SH DESCRIPTION
+.B mgstemidxlist
+is a utility program which passes through the stem index and displays
+the information.  (This program was used for debugging but is included
+for completeness!)  The header information is printed out, then for each
+block in the file the block header is displayed.  For each word in the
+stem index, the output is formatted as:
+
+.RS
+.I number_of_references \*(lqword\*(rq
+.br
+.RS
+.I "->  number_of_cases block 3-in-4_block offset"
+.br
+.I "->  ..."
+.RE
+.RE
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B -s 1|2|3
+The stem index file with the associated stem method.
+.br
+1 = casefolded and non-stemmed
+.br
+2 = non-casefolded and stemmed
+.br
+3 = casefolded and stemmed
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 25
+.B *.invf.dict.blocked.1
+Stem index with stem index method 1.
+.TP
+.B *.invf.dict.blocked.2
+Stem index with stem index method 2.
+.TP
+.B *.invf.dict.blocked.3
+Stem index with stem index method 3.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_invf_dict (1),
+.BR mg_stem_idx (1).
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/mgstemidxlist.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/mgstemidxlist.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/mgstemidxlist.c	(revision 699)
@@ -0,0 +1,245 @@
+/**************************************************************************
+ *
+ * mgstemidxlist.c -- Text dumper for the stem indexes
+ * Copyright (C) 1997  Ross Peeters
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "messages.h"
+#include "memlib.h"
+#include "local_strings.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "mg.h"
+#include "invf.h"
+#include "words.h"
+#include "backend.h"
+
+void
+read_3_in_4 (FILE * idbi)
+{
+  unsigned long i;
+  stemmed_idx *si;
+  u_char *buffer;
+  int block = 0;
+
+  if (!(si = Xmalloc (sizeof (stemmed_idx))))
+    {
+      return;
+    }
+
+  si->MemForStemIdx = 0;
+
+  fread (&(si->sih), sizeof (si->sih), 1, idbi);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(si->sih.lookback);
+  NTOHUL(si->sih.block_size);
+  NTOHUL(si->sih.num_blocks);
+  NTOHUL(si->sih.blocks_start);
+  NTOHUL(si->sih.index_chars);
+  NTOHUL(si->sih.num_of_words);
+
+  if (!(buffer = Xmalloc (si->sih.index_chars)))
+    {
+      Xfree (si);
+      return;
+    };
+  si->MemForStemIdx += si->sih.index_chars;
+
+  if (!(si->index = Xmalloc (si->sih.num_blocks * sizeof (*si->index))))
+    {
+      Xfree (si);
+      Xfree (buffer);
+      return;
+    };
+  si->MemForStemIdx += si->sih.num_blocks * sizeof (*si->index);
+
+  if (!(si->pos = Xmalloc (si->sih.num_blocks * sizeof (*si->pos))))
+    {
+      Xfree (si->index);
+      Xfree (si);
+      Xfree (buffer);
+      return;
+    };
+  si->MemForStemIdx += si->sih.num_blocks * sizeof (*si->pos);
+
+  if (!(si->buffer = Xmalloc (si->sih.block_size * sizeof (*si->buffer))))
+    {
+      Xfree (buffer);
+      Xfree (si->index);
+      Xfree (si->buffer);
+      Xfree (si);
+      return;
+    };
+  si->MemForStemIdx += si->sih.block_size * sizeof (*si->buffer);
+
+  si->active = -1;
+
+  for (i = 0; i < si->sih.num_blocks; i++)
+    {
+      register u_char len;
+      si->index[i] = buffer;
+      len = fgetc (idbi);
+      *buffer++ = len;
+      fread (buffer, sizeof (u_char), len, idbi);
+      buffer += len;
+      fread (&si->pos[i], sizeof (*si->pos), 1, idbi);
+      NTOHUL(si->pos[i]);  /* [RPAP - Jan 97: Endian Ordering] */
+    }
+
+  printf ("# lookback     = %lu\n", si->sih.lookback);
+  printf ("# block_size   = %lu\n", si->sih.block_size);
+  printf ("# num_blocks   = %lu\n", si->sih.num_blocks);
+  printf ("# blocks_start = %lu\n", si->sih.blocks_start);
+  printf ("# index_chars  = %lu\n", si->sih.index_chars);
+  printf ("# num_of_words = %lu\n", si->sih.num_of_words);
+
+  block = 0;
+  while (block < si->sih.num_blocks)
+    {
+      unsigned long *first_word;
+      unsigned short *num_words;
+      unsigned short *index;
+      long res;
+      u_char *base;
+      int num_indexes;
+
+      /* Read in next block */
+      fseek (idbi, si->pos[block] + si->sih.blocks_start, 0);
+      fread (si->buffer, si->sih.block_size, sizeof (u_char), idbi);
+      si->active = si->pos[block];
+
+      first_word = (unsigned long *) (si->buffer);
+      NTOHUL(*first_word);  /* [RPAP - Jan 97: Endian Ordering] */
+      num_words = (unsigned short *) (first_word + 1);
+      NTOHUS(*num_words);  /* [RPAP - Jan 97: Endian Ordering] */
+      index = num_words + 1;
+      num_indexes = ((*num_words - 1) / si->sih.lookback) + 1;
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (i = 0; i < num_indexes; i++)
+	NTOHUS(index[i]);
+
+      base = (u_char *) (index + num_indexes);
+      base += index[0];
+
+      printf ("\n# block      = %d\n", block);
+      printf ("# first_word = %lu\n", *first_word);
+      printf ("# num_words  = %u\n", *num_words);
+
+      res = 0;
+      while (res < *num_words)
+	{
+	  unsigned copy, suff;
+	  u_char prev[MAXSTEMLEN + 1];
+	  unsigned int num_entries, num_cases, blk;
+	  unsigned short blk_index, offset;
+
+	  /* Read word entry */
+	  copy = *base++;
+	  suff = *base++;
+	  bcopy ((char *) base, (char *) (prev + copy + 1), suff);
+	  *prev = copy + suff;
+	  base += suff;
+	  bcopy ((char *) base, (char *) &num_entries, sizeof (num_entries));
+	  base += sizeof (num_entries);
+	  NTOHUI(num_entries);  /* [RPAP - Jan 97: Endian Ordering] */
+	  printf ("%u \"%s\"\n", num_entries, word2str (prev));
+
+	  /* For all the PosEntries for the word... */
+	  for (i = 0; i < num_entries; i++)
+	    {
+	      bcopy ((char *) base, (char *) &num_cases, sizeof (num_cases));
+	      NTOHUI(num_cases);  /* [RPAP - Jan 97: Endian Ordering] */
+	      base += sizeof (num_cases);
+	      bcopy ((char *) base, (char *) &blk, sizeof (blk));
+	      NTOHUI(blk);  /* [RPAP - Jan 97: Endian Ordering] */
+	      base += sizeof (blk);
+	      bcopy ((char *) base, (char *) &blk_index, sizeof (blk_index));
+	      NTOHUS(blk_index);  /* [RPAP - Jan 97: Endian Ordering] */
+	      base += sizeof (blk_index);
+	      bcopy ((char *) base, (char *) &offset, sizeof (offset));
+	      NTOHUS(offset);  /* [RPAP - Jan 97: Endian Ordering] */
+	      base += sizeof (offset);
+
+	      printf ("    -> %4u %4u %4u %4u\n", num_cases, blk, blk_index, offset);
+	    }
+	  res++;
+	}
+      block++;
+    }
+  fclose (idbi);
+}
+
+int
+main (int argc, char **argv)
+{
+  FILE *idbi;
+  char *filename = "";
+  int ch;
+  int stem_method = 0;
+
+  msg_prefix = argv[0];
+  opterr = 0;
+  while ((ch = getopt (argc, argv, "f:d:hs:")) != -1)
+    switch (ch)
+      {
+      case 'f':		/* input file */
+	filename = optarg;
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 's':
+	stem_method = atoi (optarg);
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-d data directory] [-h] -s 1|2|3 -f name\n", argv[0]);
+	exit (1);
+      }
+
+  /* Open required files */
+  switch (stem_method)
+    {
+    case (1):
+      idbi = open_file (filename, INVF_DICT_BLOCKED_1_SUFFIX, "rb", MAGIC_STEM_1,
+			MG_ABORT);
+      break;
+    case (2):
+      idbi = open_file (filename, INVF_DICT_BLOCKED_2_SUFFIX, "rb", MAGIC_STEM_2,
+			MG_ABORT);
+      break;
+    case (3):
+      idbi = open_file (filename, INVF_DICT_BLOCKED_3_SUFFIX, "rb", MAGIC_STEM_3,
+			MG_ABORT);
+      break;
+    default:
+      FatalError (1, "Stem method must be 1, 2 or 3\n");
+    }
+
+  if (!idbi)
+    FatalError (1, "Could NOT open file");
+
+  read_3_in_4 (idbi);
+
+  return 0;
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/query.docnums.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/query.docnums.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/query.docnums.c	(revision 699)
@@ -0,0 +1,100 @@
+/**************************************************************************
+ *
+ * query.docnums.c -- Docnums query evaluation
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "filestats.h"
+#include "timing.h"
+
+#include "mg.h"
+#include "invf.h"
+#include "text.h"
+#include "lists.h"
+#include "backend.h"
+
+
+static DocList *
+AddToDocList (DocList * Docs, int res)
+{
+  int j;
+  /* Search the list for the word */
+  for (j = 0; j < Docs->num; j++)
+    if (Docs->DE[j].DocNum == res)
+      break;
+
+  /* Increment the weight if the word is in the list */
+  if (j < Docs->num)
+    Docs->DE[j].Weight++;
+  else
+    {
+      /* Create a new entry in the list for the new document */
+      Docs = ResizeDocList (Docs, Docs->num + 1);
+      Docs->DE[Docs->num].DocNum = res;
+      Docs->DE[Docs->num].Weight = 1;
+      Docs->DE[Docs->num].SeekPos = 0;
+      Docs->DE[Docs->num].Len = 0;
+      Docs->DE[Docs->num].CompTextBuffer = NULL;
+      Docs->DE[Docs->num].Next = NULL;
+      Docs->num++;
+    }
+  return (Docs);
+}
+
+void 
+DocnumsQuery (query_data * qd, char *QueryLine)
+{
+  DocList *Docs = MakeDocList (0);
+  unsigned long start = 0, finish;
+  int range = 0; /* is there a range or not ? */
+
+  /* find the start of the first word */
+  while (*QueryLine)
+    {
+      char *end;
+      finish = strtol (QueryLine, &end, 10);
+      if (end != (char *) QueryLine && finish >= 1 &&
+	  finish <= qd->sd->sdh.num_of_docs)
+	{
+	  unsigned long i;
+	  if (!range)
+	    start = finish;
+	  else
+	    range = 0;
+	  for (i = start; i <= finish; i++)
+	    Docs = AddToDocList (Docs, i);
+	  while (*end == ' ' || *end == '\t' || *end == '\n')
+	    end++;
+	  if (*end == '-')
+	    {
+	      start = finish;
+	      range = 1;
+	      end++;
+	    }
+	}
+      else
+	break;
+      QueryLine = end;
+    }
+  FreeQueryDocs (qd);
+
+  qd->DL = Docs;
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/query.ranked.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/query.ranked.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/query.ranked.c	(revision 699)
@@ -0,0 +1,991 @@
+/**************************************************************************
+ *
+ * query.ranked.c -- Ranked query evaluation
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "filestats.h"
+#include "messages.h"
+#include "timing.h"
+#include "sptree.h"
+
+#include "mg.h"
+#include "invf.h"
+#include "text.h"
+#include "lists.h"
+#include "backend.h"
+#include "stem_search.h"
+#include "weights.h"
+#include "text_get.h"
+#include "invf_get.h"
+#include "words.h"
+#include "stemmer.h"
+#include "locallib.h"
+#include "environment.h"
+#include "term_lists.h"
+#include "local_strings.h"
+#include "query_term_list.h"  /* [RPAP - Feb 97: Term Frequency] */
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:18:20  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:50  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:34  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.4  1994/11/25  03:47:46  tes
+   * Committing files before adding the merge stuff.
+   *
+   * Revision 1.3  1994/10/20  03:57:03  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:42:04  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+/*************************************************************************/
+
+typedef struct HeapEntry
+  {
+    float Weight;
+    float *OrgWeight;
+    int DocNum;
+    u_long SeekPos;		/* position in the text file in bytes */
+    u_long Len;			/* length of the document in bytes */
+  }
+HeapEntry;
+
+typedef int (*HeapComp) (HeapEntry *, HeapEntry *);
+
+typedef struct Heap
+  {
+    int NumItems;
+    int MaxSize;
+    HeapComp HC;
+    HeapEntry HE[1];
+  }
+Heap;
+
+
+Heap *
+Heap_Make (int size, HeapComp hc)
+{
+  Heap *H;
+  H = Xmalloc (sizeof (Heap) + (size - 1) * sizeof (HeapEntry));
+  if (!H)
+    return NULL;
+  H->NumItems = 0;
+  H->MaxSize = size;
+  H->HC = hc;
+  return H;
+}
+
+int 
+Heap_Size (Heap * H)
+{
+  return sizeof (Heap) + H->MaxSize * sizeof (HeapEntry);
+}
+
+HeapEntry *
+Heap_GetHead (Heap * H)
+{
+  if (H && H->NumItems)
+    return &H->HE[0];
+  else
+    return NULL;
+}
+
+
+
+void 
+Heap_Heapify (Heap * H, int i)
+{
+  register int curr, child;
+  curr = i;
+  child = curr * 2;
+  while (child <= H->NumItems)
+    {
+      if (child < H->NumItems && H->HC (&H->HE[child], &H->HE[child - 1]) > 0)
+	child++;
+      if (H->HC (&H->HE[curr - 1], &H->HE[child - 1]) < 0)
+	{
+	  HeapEntry temp = H->HE[child - 1];
+	  H->HE[child - 1] = H->HE[curr - 1];
+	  H->HE[curr - 1] = temp;
+	  curr = child;
+	  child = 2 * child;
+	}
+      else
+	break;
+    }
+}
+
+
+void 
+Heap_Build (Heap * H)
+{
+  register int i;
+  for (i = H->NumItems / 2; i > 0; i--)
+    Heap_Heapify (H, i);
+}
+
+
+void 
+Heap_Sort (Heap * H)
+{
+  register int i;
+  for (i = H->NumItems; i > 1; i--)
+    {
+      HeapEntry temp = H->HE[0];
+      H->HE[0] = H->HE[i - 1];
+      H->HE[i - 1] = temp;
+      H->NumItems--;
+      Heap_Heapify (H, 1);
+    }
+}
+
+
+void 
+Heap_DeleteHead (Heap * H)
+{
+  H->HE[0] = H->HE[--H->NumItems];
+  Heap_Heapify (H, 1);
+}
+
+int 
+Heap_Lesser (HeapEntry * a, HeapEntry * b)
+{
+  return (a->Weight > b->Weight ? -1 :
+	  (a->Weight == b->Weight ? 0 : 1));
+}
+
+
+int 
+Heap_Greater (HeapEntry * a, HeapEntry * b)
+{
+  return (a->Weight > b->Weight ? 1 :
+	  (a->Weight == b->Weight ? 0 : -1));
+}
+
+int 
+Make_Exact_Root (query_data * qd, Heap * H)
+{
+  int num = 0;
+  HeapEntry *he = H->HE;
+  while (he->SeekPos == 0)
+    {
+      he->Weight = he->Weight *
+	GetLowerApproxDocWeight (qd->awd, he->DocNum - 1) /
+	FetchDocStart (qd, he->DocNum, &he->SeekPos, &he->Len);
+      Heap_Heapify (H, 1);
+      num++;
+    }
+  return num;
+}
+
+void 
+Heap_Dump (Heap * H, int num)
+{
+  int i, l, lines, r;
+  if (num > H->NumItems)
+    num = H->NumItems;
+  lines = (num + 3) / 4;
+  for (l = 0; l < lines; l++)
+    for (r = 0; r < 4; r++)
+      {
+	i = lines * r + l;
+	if (i < num)
+	  fprintf (stderr, "[%2d] %7.4f", i, H->HE[i].Weight);
+	fprintf (stderr, r == 3 ? "\n" : "        ");
+      }
+  fprintf (stderr, "\n");
+}
+
+/*************************************************************************/
+
+
+
+
+
+
+int 
+doc_count_comp (const void *A, const void *B)
+{
+  const TermEntry *a = A;
+  const TermEntry *b = B;
+  return (a->WE.doc_count - b->WE.doc_count);
+}
+
+/* =========================================================================
+ * Function: ParseRankedQuery
+ * Description:
+ *      Takes a string query line and extracts the terms in the query
+ *      which exist in the stemmed dictionary.
+ *      Optionally sorts the terms into ascending order by doc. number.
+ * Input:
+ *      stemmed dictionary, query line, sort-flag
+ * Output:
+ *      A list of terms.
+ * ========================================================================= */
+
+/* [RPAP - Jan 97: Stem Index Change] */
+static TermList *
+ParseRankedQuery (stemmed_dict * sd, char *QueryLine, int Sort, int indexed,
+		  QueryTermList **query_term_list)  /* [RPAP - Feb 97: Term Frequency] */
+{
+  u_char Word[MAXSTEMLEN + 1];
+  u_char sWord[MAXSTEMLEN + 1];
+  u_char *end, *s_in;
+  int default_stem_method = 0;
+  TermList *Terms = MakeTermList(0);
+
+  s_in = (u_char *) QueryLine;
+  end = s_in + strlen ((char *) s_in) - 1;
+  *query_term_list = MakeQueryTermList(0);  /* [RPAP - Feb 97: Term Frequency] */
+
+  if (indexed)
+    default_stem_method = BooleanEnv (GetEnv ("casefold"), 0) | (BooleanEnv (GetEnv ("stem"), 0) << 1);
+  else
+    default_stem_method = sd->sdh.stem_method;
+
+  while (s_in <= end)
+    {
+      int j;
+      long num_entries, word_num;
+      unsigned long count, doc_count, invf_ptr, invf_len;
+      int weight_to_apply, stem_to_apply;
+      int method_using = -1;
+
+      /* 0=optional, 1=mustmatch */
+      int require_match = 0; /* [RJM 07/97: Ranked Required Terms] */
+
+      /* Skip over the non word separator taking note of any parameters */
+      PARSE_RANKED_NON_STEM_WORD (require_match, s_in, end); /* [RJM 07/97: Ranked Required Terms] */
+      if (s_in > end) break;
+
+      /* Get a word and stem it */
+      PARSE_STEM_WORD (Word, s_in, end);
+
+      /* Extract any parameters */
+      weight_to_apply = 1;
+      stem_to_apply = default_stem_method;
+      while (s_in <= end)
+	{
+	  int stem_param, weight_param, param_type;
+	  char param[MAXPARAMLEN + 1];
+
+	  param_type = 0;
+	  PARSE_OPT_TERM_PARAM (param, param_type, s_in, end);
+	  if (!param_type)
+	    break;
+
+	  switch (param_type)
+	    {
+	    case (WEIGHTPARAM):
+	      weight_param = atoi (param);
+	      if (errno != ERANGE && weight_param > 0)
+		weight_to_apply = weight_param;
+	      break;
+
+	    case (STEMPARAM):
+	      stem_param = atoi (param);
+	      if (errno != ERANGE && indexed && stem_param >= 0 && stem_param <= 3)
+		method_using = stem_to_apply = stem_param;
+	      break;
+	    }
+	}
+
+      bcopy ((char *) Word, (char *) sWord, *Word + 1);
+      stemmer (stem_to_apply, sd->sdh.stemmer_num, sWord);
+
+      if (!indexed || stem_to_apply == 0)
+	{
+	  /* Look for the word in the already identified terms */
+	  for (j = 0; j < Terms->num; j++)
+	    if (compare (Terms->TE[j].Word, Word) == 0)
+	      break;
+
+	  /* Increment the weight if the word is in the list */
+	  /* Update the require match attribute */
+	  if (j < Terms->num)
+	    {
+	      Terms->TE[j].Count = ((Terms->TE[j].Count + weight_to_apply > INT_MAX) ?
+				    INT_MAX : (Terms->TE[j].Count + weight_to_apply));
+	      Terms->TE[j].require_match = require_match; /* [RJM 07/97: Ranked Require match] */
+	      AddQueryTerm (query_term_list, Word, Terms->TE[j].WE.count, method_using);  /* [RPAP - Feb 97: Term Frequency] */
+	    }
+	  else
+	    {
+	      /* Look for it in the stemmed dictionary */
+	      if ((word_num = FindWord (sd, sWord, &count, &doc_count,
+					&invf_ptr, &invf_len)) != -1)
+		{
+		  /* Search the list for the word */
+		  for (j = 0; j < Terms->num; j++)
+		    if (Terms->TE[j].WE.word_num == word_num)
+		      break;
+
+		  /* Increment the weight if the word is in the list */
+		  if (j < Terms->num)
+		    {
+		      Terms->TE[j].Count = ((Terms->TE[j].Count + weight_to_apply > INT_MAX) ?
+					    INT_MAX : (Terms->TE[j].Count + weight_to_apply));
+		      Terms->TE[j].require_match = require_match; /* [RJM 07/97: Ranked Require match] */
+		      AddQueryTerm (query_term_list, Word, Terms->TE[j].WE.count, method_using);  /* [RPAP - Feb 97: Term Frequency] */
+		    }
+		  else
+		    {
+		      /* Create a new entry in the list for the new word */
+		      TermEntry te;
+
+		      te.WE.word_num = word_num;
+		      te.WE.count = count;
+		      te.WE.doc_count = doc_count;
+		      te.WE.max_doc_count = doc_count;
+		      te.WE.invf_ptr = invf_ptr;
+		      te.WE.invf_len = invf_len;
+		      te.Count = weight_to_apply;
+		      te.Word = copy_string (Word);
+		      if (!te.Word)
+			FatalError (1, "Could NOT create memory to add term");
+		      te.Stem = NULL;
+		      te.require_match = require_match;
+
+		      AddTermEntry (&Terms, &te);
+
+		      /* [RPAP - Feb 97: Term Frequency] */
+		      AddQueryTerm (query_term_list, Word, count, method_using);
+		    }
+		}
+	      /* [RPAP - Feb 97: Term Frequency] */
+	      else
+		AddQueryTerm (query_term_list, Word, 0, method_using);
+	    }
+	}
+      else
+	{
+	  int total_count = 0;  /* [RPAP - Feb 97: Term Frequency] */
+	  TermList *tempList = MakeTermList (0);
+	  if ((num_entries = FindWords (sd, sWord, stem_to_apply, &tempList)) > 0)
+	    {
+	      int i;
+	      u_long max_doc_count = 0;
+
+	      /* get the maximum doc count */
+	      for (i = 0; i < tempList->num; i++)
+		{
+		  if (tempList->TE[i].WE.doc_count > max_doc_count)
+		    max_doc_count = tempList->TE[i].WE.doc_count;
+		  total_count += tempList->TE[i].WE.count;  /* [RPAP - Feb 97: Term Frequency] */
+		}
+
+	      for (i = 0; i < tempList->num; i++)
+		{
+		  /* Look for the word(s) in the already identified terms */
+		  for (j = 0; j < Terms->num; j++) 
+		    {
+		      if (compare (Terms->TE[j].Word, tempList->TE[i].Word) == 0)
+			{
+			  /* found the word */
+			  /* Modify weight */
+			  Terms->TE[j].Count = ((Terms->TE[j].Count + weight_to_apply > INT_MAX) ?
+						INT_MAX : (Terms->TE[j].Count + weight_to_apply));
+			  if (Terms->TE[j].WE.max_doc_count < max_doc_count)
+			    Terms->TE[j].WE.max_doc_count = max_doc_count;
+			  break;
+			}
+		    }
+
+		  if (j == Terms->num)
+		    {
+		      /* word was not found */
+		      tempList->TE[i].WE.max_doc_count = max_doc_count;
+		      tempList->TE[i].Count = weight_to_apply;
+
+		      /* We cannot require a term to match if it is expanded */
+		      /* into multiple terms :-( */
+		      tempList->TE[i].require_match = 0; /* [RJM 07/97: Ranked Required Terms] */
+
+		      AddTermEntry (&Terms, &tempList->TE[i]);
+		    }
+		}
+	    }
+	  /* [RPAP - Feb 97: Term Frequency] */
+	  AddQueryTerm (query_term_list, Word, total_count, method_using);
+
+	  if (tempList != NULL) Xfree(tempList); /* [RJM 07/98: Memory Leak] */
+	} /* end indexed */
+    } /* end while */
+
+  if (Sort)
+    /* Sort the terms in ascending order by doc_count */
+    qsort (Terms->TE, Terms->num, sizeof (TermEntry), doc_count_comp);
+  return (Terms);
+}
+
+
+
+
+static int 
+DE_comp (void *a, void *b)
+{
+  return ((DocEntry *) a)->SeekPos - ((DocEntry *) b)->SeekPos;
+}
+
+
+
+
+
+/*
+ * This function is given a list of term numbers and it returns a list
+ * of document numbers based on the cosine document weighting system.
+ * This puts the entries in an array.
+ * inverted file.
+ * If MaxDocs == -1 then it means all 
+ */
+static DocList *
+CosineGet (query_data * qd, TermList * Terms, RankedQueryInfo * rqi) {
+  DocList *Docs;
+  float *AccumulatedWeights = NULL;
+  Splay_Tree *ST = NULL;
+  Splay_Tree *Para_ST = NULL;
+  Hash_Table *HT = NULL;
+  List_Table *LT = NULL;
+  Heap *H;
+  HeapEntry *he;
+  register float *fptr = NULL;
+  register Invf_Doc_Entry *ide = NULL;
+  register Invf_Doc_EntryH *ideh = NULL;
+  int BackEnd, NumExact, MaxExact, NumParas;
+  int MaxDocs = 0, MaxParas = 0;
+  int i;
+  Invf_Doc_Entry_Pool ide_pool;
+  ide_pool.pool = NULL;
+
+  qd->hops_taken = qd->num_of_ptrs = qd->num_of_accum = 0;
+
+  switch (rqi->AccumMethod)
+    {
+    case 'S':
+      ST = CosineDecodeSplay (qd, Terms, rqi, &ide_pool);
+      if (!ST)
+	return NULL;
+      break;
+    case 'A':
+      AccumulatedWeights = CosineDecode (qd, Terms, rqi);
+      if (!AccumulatedWeights)
+	return NULL;
+      break;
+    case 'H':
+      HT = CosineDecodeHash (qd, Terms, rqi);
+      if (!HT)
+	return NULL;
+      break;
+    case 'L':
+      LT = CosineDecodeList (qd, Terms, rqi);
+      if (!LT)
+	return NULL;
+      break;
+    }
+
+#if 0
+  if (rqi->UseSplayTree)
+    {
+
+      AccumulatedWeights = CosineDecode (qd, Terms, rqi);
+      fptr = AccumulatedWeights;
+      ide = SP_get_first (ST);
+      for (i = 0; i < qd->sd->sdh.num_of_docs; i++)
+	{
+	  if (AccumulatedWeights[i] != 0)
+	    {
+	      if (i != ide->DocNum)
+		fprintf (stderr, "Sum mismatch for %d %f %d %f\n", i + 1,
+			 AccumulatedWeights[i], ide->DocNum + 1, ide->Sum);
+	      ide = SP_get_next (ST);
+	    }
+	}
+    }
+#endif
+
+  switch (rqi->AccumMethod)
+    {
+    case 'S':
+      MaxParas = ST->no_of_items;
+      break;
+    case 'A':
+      {				/* count the number of non-zero document weights */
+	register int i = qd->sd->sdh.num_of_docs;
+	register float *d;
+	MaxParas = 0;
+	for (d = AccumulatedWeights; i; i--, d++)
+	  if (*d)
+	    MaxParas++;
+      }
+      break;
+    case 'H':
+      MaxParas = HT->num + HT->Suplimentary_Num;
+      break;
+    case 'L':
+      MaxParas = LT->num;
+      break;
+    }
+
+  if (rqi->MaxParasToRetrieve != -1 && MaxParas > rqi->MaxParasToRetrieve)
+    MaxParas = rqi->MaxParasToRetrieve;
+  MaxDocs = MaxParas;
+
+  /* Allocate memory for the heap */
+  Docs = MakeDocList (MaxDocs);
+  ChangeMemInUse (qd, sizeof (DocEntry) * MaxDocs);
+
+  H = Heap_Make (MaxDocs, Heap_Lesser);
+
+
+  /* Get the sums from the array divide the sums by the 
+     document weights which we retrieve from the ".idx.wgt" file and put
+     the resulting data into a heap */
+
+
+  he = H->HE;
+  H->NumItems = MaxDocs;
+  switch (rqi->AccumMethod)
+    {
+    case 'S':
+      {
+	ide = SP_get_first (ST);
+	for (i = 0; i < H->NumItems; i++, ide = SP_get_next (ST), he++)
+	  {
+	    he->DocNum = ide->DocNum + 1;
+	    he->OrgWeight = &ide->Sum;
+	    qd->num_of_accum++;
+	  }
+      }
+      break;
+    case 'A':
+      {
+	fptr = AccumulatedWeights;
+	for (i = 0; i < H->NumItems; i++, fptr++, he++)
+	  {
+	    he->DocNum = i + 1;
+	    he->OrgWeight = fptr;
+	    if (*fptr)
+	      qd->num_of_accum++;
+	  }
+      }
+      break;
+    case 'H':
+      {
+	ideh = HT->Head;
+	for (i = 0; i < H->NumItems; i++, ideh = ideh->next, he++)
+	  {
+	    he->DocNum = ideh->IDE.DocNum + 1;
+	    he->OrgWeight = &ideh->IDE.Sum;
+	    qd->num_of_accum++;
+	  }
+      }
+      break;
+    case 'L':
+      {
+	ide = LT->IDE;
+	for (i = 0; i < H->NumItems; i++, ide++, he++)
+	  {
+	    he->DocNum = ide->DocNum + 1;
+	    he->OrgWeight = &ide->Sum;
+	    qd->num_of_accum++;
+	  }
+      }
+      break;
+    }
+
+  he = H->HE;
+  for (i = 0; i < H->NumItems; i++, he++)
+    {
+      *he->OrgWeight /= GetLowerApproxDocWeight (qd->awd, he->DocNum - 1);
+      he->Weight = *he->OrgWeight;
+      *he->OrgWeight = 0;
+      he->SeekPos = he->Len = 0;
+    }
+
+  Heap_Build (H);
+
+  he = H->HE;
+  switch (rqi->AccumMethod)
+    {
+    case 'S':
+      {
+	for (i = MaxDocs; i < ST->no_of_items; i++, ide = SP_get_next (ST))
+	  {
+	    ide->Sum /= GetLowerApproxDocWeight (qd->awd, ide->DocNum);
+	    qd->num_of_accum++;
+	    if (ide->Sum <= he->Weight)
+	      continue;
+	    *he->OrgWeight = he->Weight;
+	    he->DocNum = ide->DocNum + 1;
+	    he->Weight = ide->Sum;
+	    he->OrgWeight = &ide->Sum;
+	    *he->OrgWeight = 0;
+	    Heap_Heapify (H, 1);
+	  }
+      }
+      break;
+    case 'A':
+      {
+	for (i = MaxDocs; i < qd->sd->sdh.num_of_docs; i++, fptr++)
+	  {
+	    if (!*fptr)
+	      continue;
+	    qd->num_of_accum++;
+	    *fptr /= GetLowerApproxDocWeight (qd->awd, i);
+	    if (*fptr <= he->Weight)
+	      continue;
+	    *he->OrgWeight = he->Weight;
+	    he->DocNum = i + 1;
+	    he->Weight = *fptr;
+	    he->OrgWeight = fptr;
+	    *he->OrgWeight = 0;
+	    Heap_Heapify (H, 1);
+	  }
+      }
+      break;
+    case 'H':
+      {
+	for (; ideh; ideh = ideh->next)
+	  {
+	    qd->num_of_accum++;
+	    ideh->IDE.Sum /=
+	      GetLowerApproxDocWeight (qd->awd, ideh->IDE.DocNum);
+	    if (ideh->IDE.Sum <= he->Weight)
+	      continue;
+	    *he->OrgWeight = he->Weight;
+	    he->DocNum = ideh->IDE.DocNum + 1;
+	    he->Weight = ideh->IDE.Sum;
+	    he->OrgWeight = &ideh->IDE.Sum;
+	    *he->OrgWeight = 0;
+	    Heap_Heapify (H, 1);
+	  }
+      }
+      break;
+    case 'L':
+      {
+	for (i = MaxDocs; i < LT->num; i++, ide++)
+	  {
+	    qd->num_of_accum++;
+	    ide->Sum /=
+	      GetLowerApproxDocWeight (qd->awd, ide->DocNum);
+	    if (ide->Sum <= he->Weight)
+	      continue;
+	    *he->OrgWeight = he->Weight;
+	    he->DocNum = ide->DocNum + 1;
+	    he->Weight = ide->Sum;
+	    he->OrgWeight = &ide->Sum;
+	    *he->OrgWeight = 0;
+	    Heap_Heapify (H, 1);
+	  }
+      }
+      break;
+    }
+
+
+  if (rqi->Exact && qd->id->ifh.InvfLevel != 3)
+    {
+      HeapEntry *he = H->HE;
+
+      for (i = 0; i < H->NumItems; i++, he++)
+	{
+	  he->Weight = he->Weight *
+	    GetLowerApproxDocWeight (qd->awd, he->DocNum - 1) /
+	    FetchDocStart (qd, he->DocNum, &he->SeekPos, &he->Len);
+	}
+
+      Heap_Build (H);
+
+      he = H->HE;
+
+      switch (rqi->AccumMethod)
+	{
+	case 'S':
+	  {
+	    ide = SP_get_first (ST);
+	    for (i = 0; i < ST->no_of_items; i++, ide = SP_get_next (ST))
+	      {
+		u_long SeekPos, Len;
+		float Weight;
+		if (!ide->Sum)
+		  continue;
+		if (ide->Sum <= he->Weight)
+		  continue;
+		Weight = ide->Sum *
+		  GetLowerApproxDocWeight (qd->awd, ide->DocNum) /
+		  FetchDocStart (qd, ide->DocNum + 1, &SeekPos, &Len);
+		if (Weight <= he->Weight)
+		  continue;
+		he->DocNum = ide->DocNum + 1;
+		he->OrgWeight = &ide->Sum;
+		he->Weight = Weight;
+		he->SeekPos = SeekPos;
+		he->Len = Len;
+		ide->Sum = 0;
+		Heap_Heapify (H, 1);
+	      }
+	  }
+	  break;
+
+	  /* up to here */
+
+	case 'A':
+	  {
+	    fptr = AccumulatedWeights;
+	    for (i = 0; i < qd->sd->sdh.num_of_docs; i++, fptr++)
+	      {
+		u_long SeekPos, Len;
+		float Weight;
+		if (!*fptr)
+		  continue;
+		if (*fptr <= he->Weight)
+		  continue;
+		Weight = *fptr *
+		  GetLowerApproxDocWeight (qd->awd, i) /
+		  FetchDocStart (qd, i + 1, &SeekPos, &Len);
+		if (Weight <= he->Weight)
+		  continue;
+		he->DocNum = i + 1;
+		he->OrgWeight = fptr;
+		he->Weight = Weight;
+		he->SeekPos = SeekPos;
+		he->Len = Len;
+		*fptr = 0;
+		Heap_Heapify (H, 1);
+	      }
+	  }
+	  break;
+	case 'H':
+	  {
+	    ideh = HT->Head;
+	    for (ideh = HT->Head; ideh; ideh = ideh->next)
+	      {
+		u_long SeekPos, Len;
+		float Weight;
+		if (!ideh->IDE.Sum)
+		  continue;
+		if (ideh->IDE.Sum <= he->Weight)
+		  continue;
+		Weight = ideh->IDE.Sum *
+		  GetLowerApproxDocWeight (qd->awd, ideh->IDE.DocNum) /
+		  FetchDocStart (qd, ideh->IDE.DocNum + 1, &SeekPos, &Len);
+		if (Weight <= he->Weight)
+		  continue;
+		he->DocNum = ideh->IDE.DocNum + 1;
+		he->OrgWeight = &ideh->IDE.Sum;
+		he->Weight = Weight;
+		he->SeekPos = SeekPos;
+		he->Len = Len;
+		ideh->IDE.Sum = 0;
+		Heap_Heapify (H, 1);
+	      }
+	  }
+	  break;
+	case 'L':
+	  {
+	    ide = LT->IDE;
+	    for (i = 0; i < LT->num; i++, ide++)
+	      {
+		u_long SeekPos, Len;
+		float Weight;
+		if (!ide->Sum)
+		  continue;
+		if (ide->Sum <= he->Weight)
+		  continue;
+		Weight = ide->Sum *
+		  GetLowerApproxDocWeight (qd->awd, ide->DocNum) /
+		  FetchDocStart (qd, ide->DocNum + 1, &SeekPos, &Len);
+		if (Weight <= he->Weight)
+		  continue;
+		he->DocNum = ide->DocNum + 1;
+		he->OrgWeight = &ide->Sum;
+		he->Weight = Weight;
+		he->SeekPos = SeekPos;
+		he->Len = Len;
+		ide->Sum = 0;
+		Heap_Heapify (H, 1);
+	      }
+	  }
+	  break;
+	}
+    }
+
+
+
+  H->HC = Heap_Greater;
+  Heap_Build (H);
+
+
+  MaxDocs = H->NumItems;
+  if (rqi->MaxDocsToRetrieve != -1 && MaxDocs > rqi->MaxDocsToRetrieve)
+    MaxDocs = rqi->MaxDocsToRetrieve;
+
+  /* Alarm */
+
+  he = H->HE;
+  BackEnd = H->NumItems - 1;
+  NumExact = 0;
+  MaxExact = H->NumItems;
+  NumParas = 0;
+  Para_ST = SP_createset (DE_comp);
+  while (H->NumItems && Docs->num < MaxDocs)
+    {
+      DocEntry DocEnt;
+      DocEntry *mem;
+
+      if (rqi->Exact)
+	{
+	  if (H->HE[0].SeekPos == 0)
+	    NumExact += Make_Exact_Root (qd, H);
+	}
+      else
+	FetchDocStart (qd, he->DocNum, &he->SeekPos, &he->Len);
+
+      NumParas++;
+
+      DocEnt.DocNum = he->DocNum;
+      DocEnt.Weight = he->Weight;
+      DocEnt.Len = he->Len;
+      DocEnt.SeekPos = he->SeekPos;
+      DocEnt.CompTextBuffer = NULL;
+      DocEnt.Next = NULL;
+
+      Heap_DeleteHead (H);
+
+      if (!(mem = SP_member (&DocEnt, Para_ST)))
+	{
+	  Docs->DE[Docs->num] = DocEnt;
+	  SP_insert (&Docs->DE[Docs->num], Para_ST);
+	  Docs->num++;
+	}
+      else
+	{
+	  DocEnt.Next = mem->Next;
+	  Docs->DE[BackEnd] = DocEnt;
+	  mem->Next = &Docs->DE[BackEnd--];
+	}
+    }
+  SP_freeset (Para_ST);
+
+  if (qd->id->ifh.InvfLevel == 3)
+    {
+      Message ("%d Paragraphs were required to get %d documents",
+	       NumParas, Docs->num);
+      if (NumExact == MaxExact)
+	{
+	  Message ("The exact weights of all %d paragraphs had to be calculated", MaxExact);
+	  Message ("to obtain %d paragraphs. This may mean that the the documents", NumParas);
+	  Message ("returned do not necessarly represent an exact cosine ranking.");
+	  Message ("This problem may be corrected by increasing \'maxparas\'.");
+	}
+    }
+#if 0
+  {
+    int i;
+    FILE *f = fopen ("top.paras", "w");
+    fprintf (f, "=========================\nTop Paragraphs\n");
+    for (i = 0; i < Docs->num; i++)
+      {
+	DocEntry *d;
+	fprintf (f, "<%d(%f)>  ", Heap[i].DocNum, Heap[i].Weight);
+	for (d = Heap[i].Next; d; d = d->Next)
+	  fprintf (f, "%d(%f)  ", d->DocNum, d->Weight);
+	fprintf (f, "\n");
+      }
+    fprintf (f, "=========================\n");
+    fclose (f);
+  }
+#endif
+
+  if (AccumulatedWeights)
+    {
+      Xfree (AccumulatedWeights);
+      ChangeMemInUse (qd, -sizeof (float) * qd->sd->sdh.num_of_docs);
+    }
+  if (ST)
+    {
+      int mem = ST->mem_in_use;
+      SP_freeset (ST);
+      ChangeMemInUse (qd, -mem);
+      free_ide_pool (qd, &ide_pool);
+    }
+  if (HT)
+    HT_free (qd, HT);
+
+  if (LT)
+    LT_free (qd, LT);
+
+  if (H) Xfree (H); /* [RJM 07/98: Memory Leak] */
+
+  return (Docs);
+}
+
+
+
+
+/* if MaxDocs == -1 it means all */
+void 
+RankedQuery (query_data *qd, char *Query, RankedQueryInfo *rqi)
+{
+  DocList *dl;
+
+  if (qd->TL)
+    FreeTermList (&(qd->TL));
+
+  /* [RPAP - Feb 97: Term Frequency] */
+  if (qd->QTL)
+    FreeQueryTermList (&(qd->QTL));
+
+  qd->TL = ParseRankedQuery (qd->sd, Query, rqi->Sort, qd->sd->sdh.indexed,  /* [RPAP - Jan 97: Stem Index Change] */
+			     &(qd->QTL));  /* [RPAP - Feb 97: Term Frequency] */
+
+  /*  PrintTermList (qd->TL, stderr); */
+
+  dl = CosineGet (qd, qd->TL, rqi);
+
+  if (!dl)
+    FatalError (1, "Out of memory\n");
+
+  FreeQueryDocs (qd);
+
+  qd->DL = dl;
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/query_term_list.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/query_term_list.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/query_term_list.c	(revision 699)
@@ -0,0 +1,225 @@
+/**************************************************************************
+ *
+ * filename -- description
+ * Copyright (C) 1994  Authors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:18:21  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:35:36  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/10/20  03:57:07  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+ */
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "local_strings.h"
+#include "query_term_list.h"
+#include "term_lists.h"  /* for MAXTERMSTRLEN */
+#include "messages.h"
+
+
+/* =========================================================================
+ * Function: MakeQueryTermList
+ * Description:
+ * Input:
+ * Output:
+ * ========================================================================= */
+QueryTermList *
+MakeQueryTermList (int n)
+{
+  QueryTermList *t;
+  int list_size = (n == 0 ? 1 : n);	/* always allocate at least one node */
+
+  t = Xmalloc (sizeof (QueryTermList) + (list_size - 1) * sizeof (QueryTermEntry));
+  if (!t)
+    FatalError (1, "Unable to allocate query term list");
+
+  t->num = n;
+  t->list_size = list_size;
+
+  return t;
+}
+
+/* =========================================================================
+ * Function: ResizeQueryTermList
+ * Description:
+ * Input:
+ * Output:
+ * ========================================================================= */
+
+#define GROWTH_FACTOR 2
+#define MIN_SIZE 2
+
+static void
+ResizeQueryTermList (QueryTermList ** query_list)
+{
+  QueryTermList *qtl = *query_list;
+
+  if (qtl->num > qtl->list_size)
+    {
+      if (qtl->list_size)
+	qtl->list_size *= GROWTH_FACTOR;
+      else
+	qtl->list_size = MIN_SIZE;
+    }
+  qtl = Xrealloc (qtl, sizeof (QueryTermList) + (qtl->list_size - 1) * sizeof (QueryTermEntry));
+
+  if (!qtl)
+    FatalError (1, "Unable to resize query term list");
+
+  *query_list = qtl;
+}
+
+/* =========================================================================
+ * Function: ConvertQueryTermsToString
+ * Description:
+ *      Convert term list into null-terminated string
+ * Input:
+ *      query_term_list = query list
+ * Output:
+ *      str = term string
+ * ========================================================================= */
+
+void
+ConvertQueryTermsToString (QueryTermList * query_term_list, char *str)
+{
+  int i = 0;
+  int total_len = 0;
+
+  /* terms_str should be preallocated */
+  if (!str)
+    return;
+
+  for (i = 0; i < query_term_list->num; i++)
+    {
+      unsigned char *word = query_term_list->QTE[i].Term;
+      int len = word[0];
+      total_len += len + 1;	/* +1 for space */
+      if (query_term_list->QTE[i].stem_method >= 0)
+	total_len += 2;         /* '#' and the stem number */
+      if (total_len > MAXTERMSTRLEN)
+	break;
+      strncpy (str, (char *) word + 1, len);
+      str += len;
+      if (query_term_list->QTE[i].stem_method >= 0)
+	{
+	  *str++ = '#';
+	  *str++ = '0' + query_term_list->QTE[i].stem_method;
+	}
+      if (i != (query_term_list->num) - 1)
+	{
+	  *str = ' ';
+	  str++;		/* add space gap */
+	}
+
+    }
+  *str = '\0';
+}
+
+/* =========================================================================
+ * Function: ResetTermList
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+void
+ResetQueryTermList (QueryTermList ** qtl)
+{
+  if (*qtl)
+    FreeQueryTermList (qtl);
+  *qtl = MakeQueryTermList (0);
+}
+
+/* =========================================================================
+ * Function: AddQueryTermEntry
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+int
+AddQueryTermEntry (QueryTermList ** query_term_list, QueryTermEntry * qte)
+{
+  QueryTermList *qtl = *query_term_list;
+
+  qtl->num++;
+  ResizeQueryTermList (query_term_list);
+  qtl = *query_term_list;
+
+  /* copy the structure contents */
+  bcopy ((char *) qte, (char *) &(qtl->QTE[qtl->num - 1]), sizeof (QueryTermEntry));
+
+  return qtl->num - 1;
+}
+
+
+
+/* =========================================================================
+ * Function: AddQueryTerm
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+int
+AddQueryTerm (QueryTermList ** query_term_list, u_char * Term, int Count, int stem_method)
+{
+  /* Create a new entry in the list for the new word */
+  QueryTermEntry qte;
+
+  qte.Count = Count;
+  qte.stem_method = stem_method;
+  qte.Term = copy_string (Term);
+  if (!qte.Term)
+    FatalError (1, "Could NOT create memory to add term");
+
+  return AddQueryTermEntry (query_term_list, &qte);
+}
+
+/* =========================================================================
+ * Function: FreeQueryTermList
+ * Description:
+ * Input:
+ * Output:
+ * ========================================================================= */
+
+void
+FreeQueryTermList (QueryTermList ** the_qtl)
+{
+  int j;
+  QueryTermList *qtl = *the_qtl;
+
+  for (j = 0; j < qtl->num; j++)
+    if (qtl->QTE[j].Term)
+      Xfree (qtl->QTE[j].Term);
+  Xfree (qtl);
+
+  *the_qtl = NULL;
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/query_term_list.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/query_term_list.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/query_term_list.h	(revision 699)
@@ -0,0 +1,67 @@
+/**************************************************************************
+ *
+ * term_lists.h -- description
+ * Copyright (C) 1994  Authors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:18:21  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:35:37  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/10/20  03:57:08  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+ */
+
+#ifndef QUERY_TERM_LIST_H
+#define QUERY_TERM_LIST_H
+
+#include "sysfuncs.h"
+
+typedef struct QueryTermEntry
+  {
+    int Count;			/* The number of times the word occurs in the query */
+    u_char *Term;		/* The query term */
+    int stem_method;            /* The steming method to apply to the term : -1 is default */
+  }
+QueryTermEntry;
+
+typedef struct QueryTermList
+  {
+    int list_size;
+    int num;
+    QueryTermEntry QTE[1];
+  }
+QueryTermList;
+
+/* --- prototypes --- */
+void ConvertQueryTermsToString (QueryTermList * query_term_list, char *str);
+int AddQueryTerm (QueryTermList ** query_term_list, u_char * Term, int Count, int stem_method);
+void ResetQueryTermList (QueryTermList ** tl);
+void FreeQueryTermList (QueryTermList ** the_qtl);
+QueryTermList *MakeQueryTermList (int n);
+
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/read_line.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/read_line.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/read_line.c	(revision 699)
@@ -0,0 +1,217 @@
+/**************************************************************************
+ *
+ * read_line.c -- Input line reading routines for mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "memlib.h"
+
+#ifdef GNU_READLINE
+#include "readline.h"
+#include "chardefs.h"
+#include "history.h"
+#endif
+
+
+#include "globals.h"
+#include "environment.h"
+#include "read_line.h"
+
+
+
+#ifdef GNU_READLINE
+int rl_bind_key (int, int (*)());
+
+
+void 
+Init_ReadLine (void)
+{
+  /* Make TAB just insert itself rather than do a file name completion */
+  rl_bind_key (TAB, rl_insert);
+}
+#else
+void 
+Init_ReadLine (void)
+{
+}
+#endif
+
+
+
+
+/* WritePrompt() */
+/* Write out a prompt if user is a TTY */
+void 
+WritePrompt (void)
+{
+  if (isatty (fileno (InFile)))
+    {
+      if (!BooleanEnv (GetEnv ("expert"), 0))
+	fprintf (stderr, "Enter a command or query (.quit to terminate, .help for assistance).\n");
+    }
+}
+
+#ifdef GNU_READLINE
+static void memory_error_and_abort ();
+
+char *
+xmalloc (bytes)
+     int bytes;
+{
+  char *temp = (char *) Xmalloc (bytes);
+
+  if (!temp)
+    memory_error_and_abort ();
+  return (temp);
+}
+
+char *
+xrealloc (pointer, bytes)
+     char *pointer;
+     int bytes;
+{
+  char *temp;
+
+  if (!pointer)
+    temp = (char *) xmalloc (bytes);
+  else
+    temp = (char *) Xrealloc (pointer, bytes);
+
+  if (!temp)
+    memory_error_and_abort ();
+
+  return (temp);
+}
+
+static void
+memory_error_and_abort ()
+{
+  fprintf (stderr, "history: Out of virtual memory!\n");
+  abort ();
+}
+#endif
+
+
+#ifndef GNU_READLINE
+
+static FILE *rl_instream = stdin;
+static FILE *rl_outstream = stdout;
+
+static char *
+readline (char *pmt)
+{
+  static char buf[MAXLINEBUFFERLEN + 1];
+  char *s;
+
+  fprintf (rl_outstream, "%s", pmt);
+  s = fgets (buf, sizeof (buf), rl_instream);
+  if (s)
+    {
+      char *s1 = strrchr (s, '\n');
+      if (s1 && *(s1 + 1) == '\0')
+	*s1 = '\0';
+    }
+  return s ? Xstrdup (s) : NULL;
+}
+#endif
+
+/*
+ * This routine returns a pointer to the users entered line 
+ *
+ */
+static char *
+GetLine (char *pmt)
+{
+  static char *the_line = NULL;
+  if (the_line)
+    Xfree (the_line);
+  the_line = NULL;
+  rl_instream = InFile;
+  if (isatty (fileno (InFile)))
+    {
+      fputc ('\r', stderr);
+      if (!isatty (fileno (OutFile)))
+	rl_outstream = stderr;
+      else
+	rl_outstream = OutFile;
+      the_line = readline (pmt);
+    }
+  else
+    {
+      if (isatty (fileno (OutFile)))
+	{
+	  the_line = readline (pmt);
+	  fprintf (stderr, "%s\n", the_line ? the_line : "");
+	}
+      else
+	{
+	  the_line = readline ("");
+	  if (the_line)
+	    fprintf (stderr, "%s%s\n", pmt, the_line);
+	}
+    }
+#ifdef GNU_READLINE
+  if (the_line && *the_line)
+    add_history (the_line);
+#endif
+  return (the_line);
+}
+
+
+
+
+
+
+char *
+GetMultiLine (void)
+{
+  static char *line = NULL;
+  char *s;
+  if (line)
+    Xfree (line);
+  line = NULL;
+  if (!(s = GetLine ("> ")))
+    return (NULL);
+  if (!(line = Xstrdup (s)))
+    {
+      fprintf (stderr, "Unable to allocate memory for the line\n");
+      abort ();
+    }
+
+  while ((s = strrchr (line, '\\')) && *(s + 1) == '\0')
+    {
+      char *new;
+      *strrchr (line, '\\') = '\0';
+      if (!(s = GetLine ("? ")))
+	return (NULL);
+      if (!(new = Xmalloc (strlen (line) + strlen (s) + 2)))
+	{
+	  fprintf (stderr, "Unable to allocate memory for the line\n");
+	  abort ();
+	}
+      strcpy (new, line);
+      strcat (new, "\n");
+      strcat (new, s);
+      Xfree (line);
+      line = new;
+    }
+  return (line);
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/read_line.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/read_line.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/read_line.h	(revision 699)
@@ -0,0 +1,34 @@
+/**************************************************************************
+ *
+ * read_line.h -- Input line reading routines for mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef READ_LINE_H
+#define READ_LINE_H
+
+#define MAXLINEBUFFERLEN  65535
+
+void Init_ReadLine (void);
+void WritePrompt (void);
+char *GetMultiLine (void);
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/stem_search.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/stem_search.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/stem_search.c	(revision 699)
@@ -0,0 +1,772 @@
+/**************************************************************************
+ *
+ * stem_search.c -- Functions for searching the blocked stemmed dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "filestats.h"
+#include "timing.h"
+#include "local_strings.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg.h"
+#include "invf.h"
+#include "text.h"
+#include "lists.h"
+#include "backend.h"
+#include "words.h"
+#include "locallib.h"
+#include "stem_search.h"
+#include "mg_errors.h"
+#include "term_lists.h"
+#include "stemmer.h"
+
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:18:22  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.3  1999/07/02 00:18:55  rjmcnab
+   Changed so FindWords could be used in new ways.
+
+   Revision 1.2  1998/11/25 07:55:51  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:39  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:57:04  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:42:08  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+stemmed_dict *
+ReadStemDictBlk (File * stem_file)
+{
+  unsigned long i;
+  stemmed_dict *sd;
+  u_char *buffer;
+
+  if (!(sd = Xmalloc (sizeof (stemmed_dict))))
+    {
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  sd->stem_file = stem_file;
+  sd->MemForStemDict = 0;
+
+  Fread (&sd->sdh, sizeof (sd->sdh), 1, stem_file);
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(sd->sdh.lookback);
+  NTOHUL(sd->sdh.block_size);
+  NTOHUL(sd->sdh.num_blocks);
+  NTOHUL(sd->sdh.blocks_start);
+  NTOHUL(sd->sdh.index_chars);
+  NTOHUL(sd->sdh.num_of_docs);
+  NTOHUL(sd->sdh.static_num_of_docs);
+  NTOHUL(sd->sdh.num_of_words);
+  NTOHUL(sd->sdh.stem_method);
+  NTOHUL(sd->sdh.indexed);
+
+  if (!(buffer = Xmalloc (sd->sdh.index_chars)))
+    {
+      Xfree (sd);
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    };
+  sd->MemForStemDict += sd->sdh.index_chars;
+
+  if (!(sd->index = Xmalloc (sd->sdh.num_blocks * sizeof (*sd->index))))
+    {
+      Xfree (sd);
+      Xfree (buffer);
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    };
+  sd->MemForStemDict += sd->sdh.num_blocks * sizeof (*sd->index);
+
+  if (!(sd->pos = Xmalloc (sd->sdh.num_blocks * sizeof (*sd->pos))))
+    {
+      Xfree (sd);
+      Xfree (buffer);
+      Xfree (sd->index);
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    };
+  sd->MemForStemDict += sd->sdh.num_blocks * sizeof (*sd->pos);
+
+  if (!(sd->buffer = Xmalloc (sd->sdh.block_size * sizeof (*sd->buffer))))
+    {
+      Xfree (sd);
+      Xfree (buffer);
+      Xfree (sd->index);
+      Xfree (sd->buffer);
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    };
+  sd->MemForStemDict += sd->sdh.block_size * sizeof (*sd->buffer);
+
+  sd->active = -1;
+
+  for (i = 0; i < sd->sdh.num_blocks; i++)
+    {
+      register u_char len;
+      sd->index[i] = buffer;
+      len = Getc (stem_file);
+      *buffer++ = len;
+      Fread (buffer, sizeof (u_char), len, stem_file);
+      buffer += len;
+      Fread (&sd->pos[i], sizeof (*sd->pos), 1, stem_file);
+      NTOHUL(sd->pos[i]); /* [RPAP - Jan 97: Endian Ordering] */
+    }
+
+  mg_errno = MG_NOERROR;
+
+  /*  fprintf (stderr, "mem for stem dict = %i\n", sd->MemForStemDict); */
+
+  return sd;
+}
+
+
+/* [RPAP - Jan 97: Stem Index Change] */
+stemmed_idx *
+ReadStemIdxBlk (File * stem_idx_file)
+{
+  unsigned long i;
+  stemmed_idx *si;
+  u_char *buffer;
+
+  if (!(si = Xmalloc (sizeof (stemmed_idx))))
+    {
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  si->stem_idx_file = stem_idx_file;
+  si->MemForStemIdx = 0;
+
+  Fread (&si->sih, sizeof (si->sih), 1, stem_idx_file);
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(si->sih.lookback);
+  NTOHUL(si->sih.block_size);
+  NTOHUL(si->sih.num_blocks);
+  NTOHUL(si->sih.blocks_start);
+  NTOHUL(si->sih.index_chars);
+  NTOHUL(si->sih.num_of_words);
+
+  if (!(buffer = Xmalloc (si->sih.index_chars)))
+    {
+      Xfree (si);
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    };
+  si->MemForStemIdx += si->sih.index_chars;
+
+  if (!(si->index = Xmalloc (si->sih.num_blocks * sizeof (*si->index))))
+    {
+      Xfree (si);
+      Xfree (buffer);
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    };
+  si->MemForStemIdx += si->sih.num_blocks * sizeof (*si->index);
+
+  if (!(si->pos = Xmalloc (si->sih.num_blocks * sizeof (*si->pos))))
+    {
+      Xfree (si->index);
+      Xfree (si);
+      Xfree (buffer);
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    };
+  si->MemForStemIdx += si->sih.num_blocks * sizeof (*si->pos);
+
+  if (!(si->buffer = Xmalloc (si->sih.block_size * sizeof (*si->buffer))))
+    {
+      Xfree (buffer);
+      Xfree (si->index);
+      Xfree (si->buffer);
+      Xfree (si);
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    };
+  si->MemForStemIdx += si->sih.block_size * sizeof (*si->buffer);
+
+  si->active = -1;
+
+  for (i = 0; i < si->sih.num_blocks; i++)
+    {
+      register u_char len;
+      si->index[i] = buffer;
+      len = Getc (stem_idx_file);
+      *buffer++ = len;
+      Fread (buffer, sizeof (u_char), len, stem_idx_file);
+      buffer += len;
+      Fread (&si->pos[i], sizeof (*si->pos), 1, stem_idx_file);
+      NTOHUL(si->pos[i]);  /* [RPAP - Jan 97: Endian Ordering] */
+    }
+  mg_errno = MG_NOERROR;
+
+  /*  fprintf (stderr, "mem for stem idx = %i\n", si->MemForStemIdx); */
+
+  return si;
+}
+
+
+/* [RPAP - Jan 97: Stem Index Change] */
+/* word should be appropriately stemed */
+static int
+GetIdxBlock (stemmed_idx * si, u_char * word)
+{
+  register int lo = 0, hi = si->sih.num_blocks - 1;
+  register int mid = 0, c = 0;
+
+  while (lo <= hi)
+    {
+      mid = (lo + hi) / 2;
+      c = casecompare (word, si->index[mid]);
+      if (c < 0)
+	hi = mid - 1;
+      else if (c > 0)
+	lo = mid + 1;
+      else
+	return mid;
+    }
+  return hi < 0 ? 0 : (c < 0 ? mid - 1 : mid);
+}
+
+
+static int 
+GetBlock (stemmed_dict * sd, u_char * Word)
+{
+  register int lo = 0, hi = sd->sdh.num_blocks - 1;
+  register int mid = 0, c = 0;
+  while (lo <= hi)
+    {
+      mid = (lo + hi) / 2;
+      c = casecompare (Word, sd->index[mid]);  /* [RPAP - Jan 97: Stem Index Change] */
+      if (c < 0)
+	hi = mid - 1;
+      else if (c > 0)
+	lo = mid + 1;
+      else
+	return mid;
+    }
+  return hi < 0 ? 0 : (c < 0 ? mid - 1 : mid);
+}
+
+
+/*
+ * This function looks up a word in the stemmed dictionary, it returns -1
+ * if the word cound not be found, and 0 if it successfully finds the word.
+ * If count is non-null the ulong it is pointing to is set to the number of 
+ * occurances of the stemmed word in the collection. i.e wcnt.
+ * If doc_count is non-null the ulong it is pointing to is set to the number
+ * of documents that the word occurs in. i.e fcnt
+ * If invf_ptr is non-null the ulong it is pointing to is set to the position
+ * of the inverted file where the entry for this word start.
+ */
+int 
+FindWord (stemmed_dict * sd, u_char * Word, unsigned long *count,
+	  unsigned long *doc_count, unsigned long *invf_ptr,
+	  unsigned long *invf_len)
+{
+  register int lo, hi, mid, c;
+  register unsigned int res;
+  int block, num_indexes;
+  unsigned long *first_word, *last_invf_len;
+  unsigned short *num_words;
+  u_char *base;
+  unsigned short *index;
+  u_char prev[MAXSTEMLEN + 1];
+
+  block = GetBlock (sd, Word);
+  /* [RPAP - Jan 97: Endian Ordering] */
+  if (sd->active != sd->pos[block])
+    {
+      int i;
+
+      Fseek (sd->stem_file, sd->pos[block] + sd->sdh.blocks_start, 0);
+      Fread (sd->buffer, sd->sdh.block_size, sizeof (u_char), sd->stem_file);
+      sd->active = sd->pos[block];
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      first_word = (unsigned long *) (sd->buffer);
+      NTOHUL(*first_word);
+      last_invf_len = (unsigned long *) (first_word + 1);
+      NTOHUL(*last_invf_len);
+      num_words = (unsigned short *) (last_invf_len + 1);
+      NTOHUS(*num_words);
+      index = num_words + 1;
+      num_indexes = ((*num_words - 1) / sd->sdh.lookback) + 1;
+
+      for (i = 0; i < num_indexes; i++)
+	NTOHUS(index[i]);
+    }
+  else
+    {
+      first_word = (unsigned long *) (sd->buffer);
+      last_invf_len = (unsigned long *) (first_word + 1);
+      num_words = (unsigned short *) (last_invf_len + 1);
+      index = num_words + 1;
+      num_indexes = ((*num_words - 1) / sd->sdh.lookback) + 1;
+    }
+  base = (u_char *) (index + num_indexes);
+
+  lo = 0;
+  hi = num_indexes - 1;
+  while (lo <= hi)
+    {
+      mid = (lo + hi) / 2;
+      c = casecompare (Word, base + index[mid] + 1);  /* [RPAP - Jan 97: Stem Index Change] */
+      if (c < 0)
+	hi = mid - 1;
+      else if (c > 0)
+	lo = mid + 1;
+      else
+	{
+	  hi = mid;
+	  break;
+	}
+    }
+  if (hi < 0)
+    hi = 0;
+
+  res = hi * sd->sdh.lookback;
+  base += index[hi];
+
+  for (;;)
+    {
+      unsigned copy, suff;
+      unsigned long invfp;
+      if (res >= *num_words)
+	return (-1);
+      copy = *base++;
+      suff = *base++;
+      bcopy ((char *) base, (char *) (prev + copy + 1), suff);
+      base += suff;
+      *prev = copy + suff;
+
+      c = casecompare (Word, prev);   /* [RPAP - Jan 97: Stem Index Change] */
+      if (c < 0)
+	return (-1);
+
+      if (c == 0 && doc_count)
+	{
+	  bcopy ((char *) base, (char *) doc_count, sizeof (*doc_count));
+	  NTOHUL(*doc_count);  /* [RPAP - Jan 97: Endian Ordering] */
+	}
+      base += sizeof (*doc_count);
+
+      if (c == 0 && count)
+	{
+	  bcopy ((char *) base, (char *) count, sizeof (*count));
+	  NTOHUL(*count);  /* [RPAP - Jan 97: Endian Ordering] */
+	}
+      base += sizeof (*count);
+
+      if (c == 0 && invf_ptr)
+	{
+	  bcopy ((char *) base, (char *) &invfp, sizeof (invf_ptr));
+	  NTOHUL(invfp);  /* [RPAP - Jan 97: Endian Ordering] */
+	  *invf_ptr = invfp;
+	}
+      base += sizeof (*invf_ptr);
+
+      if (c == 0)
+	{
+	  /* Calculate invf_len is necessary */
+	  unsigned long next_invfp;
+	  if (!invf_len)
+	    return (*first_word + res);
+
+	  /* If the current word is the last word of the block the get the 
+	     length from last_invf_len */
+	  if (res == *num_words - 1)
+	    {
+	      *invf_len = *last_invf_len;
+	      return (*first_word + res);
+	    }
+
+	  /* Skip over most of the next word to get to the invf_ptr */
+	  base++;
+	  suff = *base++;
+	  base += suff + sizeof (unsigned long) * 2;
+	  bcopy ((char *) base, (char *) &next_invfp, sizeof (next_invfp));
+	  NTOHUL(next_invfp);  /* [RPAP - Jan 97: Endian Ordering] */
+	  *invf_len = next_invfp - invfp;
+	  return (*first_word + res);
+	}
+      res++;
+    }
+}
+
+
+/* [RPAP - Jan 97: Stem Index Change] */
+int 
+FindWords (stemmed_dict * sd, u_char * sWord, int stem_method, TermList ** tl)
+{
+  register unsigned int res;
+  unsigned int idx_res;
+  unsigned copy, suff;
+  int j, k;
+
+  int block, num_indexes;
+  unsigned long *first_word, *last_invf_len;
+  unsigned short *num_words;
+  u_char *base;
+  unsigned short *index;
+  u_char prev[MAXSTEMLEN + 1];
+
+  int idx_block, idx_num_indexes;
+  unsigned long *idx_first_word;
+  unsigned short *idx_num_words;
+  u_char *idx_base;
+  unsigned short *idx_index;
+  u_char idx_prev[MAXSTEMLEN + 1];
+
+  unsigned int num_entries, num_cases;
+  unsigned short blk_index, offset;
+  stemmed_idx * si = NULL;
+
+  /* handle stem_method 0 seperately */
+  if (stem_method == 0) {
+    TermEntry te;
+    if ((te.WE.word_num = FindWord (sd, sWord, &te.WE.count, &te.WE.doc_count,
+				   &te.WE.invf_ptr, &te.WE.invf_len)) != -1) {
+      te.WE.max_doc_count = te.WE.doc_count;
+
+      te.Count = 1;
+      te.Word = copy_string (sWord);
+      if (!te.Word)
+	FatalError (1, "Could NOT create memory to add term");
+      te.Stem = copy_string (sWord);
+      if (!te.Stem)
+	FatalError (1, "Could NOT create memory to add term");
+      /*      te.query_mask = NULL;*/
+      
+      AddTermEntry (tl, &te);
+      return (*tl)->num;
+      
+    } else {
+      /* didn't match */
+      return 0;
+    }
+  }
+  
+  if (stem_method == 1)
+    si = sd->stem1;
+  else if (stem_method == 2)
+    si = sd->stem2;
+  else
+    si = sd->stem3;
+
+  /* Locate block */
+  idx_block = GetIdxBlock (si, sWord);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  if (si->active != si->pos[idx_block])
+    {
+      Fseek (si->stem_idx_file, si->pos[idx_block] + si->sih.blocks_start, 0);
+      Fread (si->buffer, si->sih.block_size, sizeof (u_char), si->stem_idx_file);
+      si->active = si->pos[idx_block];
+
+      idx_first_word = (unsigned long *) (si->buffer);
+      NTOHUL(*idx_first_word);  /* [RPAP - Jan 97: Endian Ordering] */
+      idx_num_words = (unsigned short *) (idx_first_word + 1);
+      NTOHUS(*idx_num_words);  /* [RPAP - Jan 97: Endian Ordering] */
+      idx_index = idx_num_words + 1;
+      idx_num_indexes = ((*idx_num_words - 1) / si->sih.lookback) + 1;
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (j = 0; j < idx_num_indexes; j++)
+	NTOHUS(idx_index[j]);
+    }
+  else
+    {
+      idx_first_word = (unsigned long *) (si->buffer);
+      idx_num_words = (unsigned short *) (idx_first_word + 1);
+      idx_index = idx_num_words + 1;
+      idx_num_indexes = ((*idx_num_words - 1) / si->sih.lookback) + 1;
+    }
+  idx_base = (u_char *) (idx_index + idx_num_indexes);
+  
+  {
+    /* Locate 3-in-4 block */
+    register int lo, hi, mid, c;
+    lo = 0;
+    hi = idx_num_indexes - 1;
+    while (lo <= hi)
+      {
+	mid = (lo + hi) / 2;
+	c = casecompare (sWord, idx_base + idx_index[mid] + 1);
+	if (c < 0)
+	  hi = mid - 1;
+	else if (c > 0)
+	  lo = mid + 1;
+	else
+	  {
+	    hi = mid;
+	    break;
+	  }
+      }
+    if (hi < 0)
+      hi = 0;
+
+    idx_res = hi * si->sih.lookback;
+    idx_base += idx_index[hi];
+  }
+
+  /* Locate actual word entry */
+  for (;;)
+    {
+      int c;
+      if (idx_res >= *idx_num_words)
+	return (-1);
+      copy = *idx_base++;
+      suff = *idx_base++;
+      bcopy ((char *) idx_base, (char *) (idx_prev + copy + 1), suff);
+      idx_base += suff;
+      *idx_prev = copy + suff;
+
+      c = casecompare (sWord, idx_prev);
+      if (c < 0)
+	return (-1);
+     
+      bcopy ((char *) idx_base, (char *) &num_entries, sizeof (num_entries));
+      NTOHUI(num_entries);  /* [RPAP - Jan 97: Endian Ordering] */
+      idx_base += sizeof (num_entries);
+      
+      if (c > 0)
+	idx_base += num_entries * (sizeof (num_cases) + sizeof (block) + 
+				   sizeof (blk_index) + sizeof (offset));
+
+      else
+	break;
+
+      idx_res++;
+    }
+
+  for (k = 0; k < num_entries; k++)
+    {
+      unsigned copy, suff;
+      unsigned long invfp;
+      /* Read next stem index pos */
+      bcopy ((char *) idx_base, (char *) &num_cases, sizeof (num_cases));
+      NTOHUI(num_cases);  /* [RPAP - Jan 97: Endian Ordering] */
+      idx_base += sizeof (num_cases);
+      bcopy ((char *) idx_base, (char *) &block, sizeof (block));
+      NTOHUI(block);  /* [RPAP - Jan 97: Endian Ordering] */
+      idx_base += sizeof (block);
+      bcopy ((char *) idx_base, (char *) &blk_index, sizeof (blk_index));
+      NTOHUS(blk_index);  /* [RPAP - Jan 97: Endian Ordering] */
+      idx_base += sizeof (blk_index);
+      bcopy ((char *) idx_base, (char *) &offset, sizeof (offset));
+      NTOHUS(offset);  /* [RPAP - Jan 97: Endian Ordering] */
+      idx_base += sizeof (offset);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      if (sd->active != sd->pos[block])
+	{
+	  Fseek (sd->stem_file, sd->pos[block] + sd->sdh.blocks_start, 0);
+	  Fread (sd->buffer, sd->sdh.block_size, sizeof (u_char), sd->stem_file);
+	  sd->active = sd->pos[block];
+
+	  first_word = (unsigned long *) (sd->buffer);
+	  NTOHUL(*first_word);  /* [RPAP - Jan 97: Endian Ordering] */
+	  last_invf_len = (unsigned long *) (first_word + 1);
+	  NTOHUL(*last_invf_len);  /* [RPAP - Jan 97: Endian Ordering] */
+	  num_words = (unsigned short *) (last_invf_len + 1);
+	  NTOHUS(*num_words);  /* [RPAP - Jan 97: Endian Ordering] */
+	  index = num_words + 1;
+	  num_indexes = ((*num_words - 1) / sd->sdh.lookback) + 1;
+
+	  /* [RPAP - Jan 97: Endian Ordering] */
+	  for (j = 0; j < num_indexes; j++)
+	    NTOHUS(index[j]);
+	}
+      else
+	{
+	  first_word = (unsigned long *) (sd->buffer);
+	  last_invf_len = (unsigned long *) (first_word + 1);
+	  num_words = (unsigned short *) (last_invf_len + 1);
+	  index = num_words + 1;
+	  num_indexes = ((*num_words - 1) / sd->sdh.lookback) + 1;
+	}
+      base = (u_char *) (index + num_indexes);
+      
+      res = blk_index * sd->sdh.lookback;
+      base += index[blk_index];
+      
+      for (j = 0; j < offset; j++)
+	{
+	  copy = *base++;
+	  suff = *base++;
+	  bcopy ((char *) base, (char *) (prev + copy + 1), suff);
+	  base += suff;
+	  *prev = copy + suff;
+	  base += sizeof (unsigned long);   /* skip doc_count */
+	  base += sizeof (unsigned long);   /* skip count */
+	  base += sizeof (unsigned long);   /* skip invf_ptr */
+	  res++;
+	}
+
+      for (j = 0; j < num_cases; j++)
+	{
+	  TermEntry te;
+	  
+	  if (res >= *num_words)
+	    return (-1);
+	  copy = *base++;
+	  suff = *base++;
+	  bcopy ((char *) base, (char *) (prev + copy + 1), suff);
+	  base += suff;
+	  *prev = copy + suff;
+	      
+	  te.Word = copy_string (prev);
+	  if (!te.Word)
+	    FatalError (1, "Could NOT create memory to add term");
+	  te.Stem = copy_string (prev);
+	  if (!te.Stem)
+	    FatalError (1, "Could NOT create memory to add term");
+	  stemmer (2, sd->sdh.stemmer_num, te.Stem);
+
+	  te.Count = 1;
+	  te.WE.word_num = *first_word + res;
+	  bcopy ((char *) base, (char *) &te.WE.doc_count, sizeof (te.WE.doc_count));
+	  NTOHUL(te.WE.doc_count);  /* [RPAP - Jan 97: Endian Ordering] */
+	  te.WE.max_doc_count = te.WE.doc_count;
+	  base += sizeof (te.WE.doc_count);
+	      
+	  bcopy ((char *) base, (char *) &te.WE.count, sizeof (te.WE.count));
+	  NTOHUL(te.WE.count);
+	  base += sizeof (te.WE.count);
+	      
+	  bcopy ((char *) base, (char *) &invfp, sizeof (te.WE.invf_ptr));
+	  NTOHUL(invfp);  /* [RPAP - Jan 97: Endian Ordering] */
+	  te.WE.invf_ptr = invfp;
+	  base += sizeof (te.WE.invf_ptr);
+		
+	  /* If the current word is the last word of the block the get the 
+	     length from last_invf_len */
+	  if (res == *num_words - 1)
+	    te.WE.invf_len = *last_invf_len;
+	  else
+	    {
+	      unsigned long next_invfp;
+	      u_char *oldbase = base;
+		  
+	      /* Skip over most of the next word to get to the invf_ptr */
+	      base++;
+	      suff = *base++;
+	      base += suff + sizeof (unsigned long) * 2;
+	      bcopy ((char *) base, (char *) &next_invfp, sizeof (next_invfp));
+	      NTOHUL(next_invfp);  /* [RPAP - Jan 97: Endian Ordering] */
+	      te.WE.invf_len = next_invfp - invfp;
+	      base = oldbase;
+	    }
+	      
+	  /* Add term entry to term list */
+	  /*	  te.query_mask = NULL;*/
+	  AddTermEntry (tl, &te);
+	      
+	  if (res == *num_words - 1 && j + 1 < num_cases)
+	    {
+	      int ii;
+	      /* Read in next block */
+	      block++;
+	      Fseek (sd->stem_file, sd->pos[block] + sd->sdh.blocks_start, 0);
+	      Fread (sd->buffer, sd->sdh.block_size, sizeof (u_char), sd->stem_file);
+	      sd->active = sd->pos[block];
+
+	      first_word = (unsigned long *) (sd->buffer);
+	      NTOHUL(*first_word);  /* [RPAP - Jan 97: Endian Ordering] */
+	      last_invf_len = (unsigned long *) (first_word + 1);
+	      NTOHUL(*last_invf_len);  /* [RPAP - Jan 97: Endian Ordering] */
+	      num_words = (unsigned short *) (last_invf_len + 1);
+	      NTOHUS(*num_words);  /* [RPAP - Jan 97: Endian Ordering] */
+	      index = num_words + 1;
+	      num_indexes = ((*num_words - 1) / sd->sdh.lookback) + 1;
+
+	      /* [RPAP - Jan 97: Endian Ordering] */
+	      for (ii = 0; ii < num_indexes; ii++)
+		NTOHUS(index[ii]);
+
+	      base = (u_char *) (index + num_indexes);
+	      base += index[0];
+	      res = 0;
+	      blk_index = 0;
+	    }
+	  else
+	    res++;
+	} /* end for num_cases */
+    } /* end for num_entries */
+  return (*tl)->num;
+}
+
+
+void 
+FreeStemDict (stemmed_dict * sd)
+{
+  /* [RPAP - Jan 97: Stem Index Change] */
+  if (sd->stem1)
+    FreeStemIdx (sd->stem1);
+  if (sd->stem2)
+    FreeStemIdx (sd->stem2);
+  if (sd->stem3)
+    FreeStemIdx (sd->stem3);
+
+  Xfree (sd->index[0]);
+  Xfree (sd->index);
+  Xfree (sd->buffer);
+  Xfree (sd->pos);
+  Xfree (sd);
+}
+
+/* [RPAP - Jan 97: Stem Index Change] */
+void
+FreeStemIdx (stemmed_idx * si)
+{
+  Xfree (si->index[0]);
+  Xfree (si->index);
+  Xfree (si->buffer);
+  Xfree (si->pos);
+  Xfree (si);
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/stem_search.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/stem_search.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/stem_search.h	(revision 699)
@@ -0,0 +1,43 @@
+/**************************************************************************
+ *
+ * stem_search.h -- Functions for searching the blocked stemmed dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef H_STEM_SEARCH
+#define H_STEM_SEARCH
+
+stemmed_dict *ReadStemDictBlk (File * stem_file);
+
+stemmed_idx *ReadStemIdxBlk (File * stem_idx_file);
+
+int FindWord (stemmed_dict * sd, u_char * Word, unsigned long *count,
+	      unsigned long *doc_count, unsigned long *invf_ptr,
+	      unsigned long *invf_len);
+
+/* [RPAP - Jan 97: Stem Index Change] */
+int FindWords (stemmed_dict * sd, u_char * sWord, int stem_method, TermList ** tl);
+
+void FreeStemDict (stemmed_dict * sd);
+
+/* [RPAP - Jan 97: Stem Index Change] */
+void FreeStemIdx (stemmed_idx * si);
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/stemmer.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/stemmer.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/stemmer.c	(revision 699)
@@ -0,0 +1,142 @@
+/**************************************************************************
+ *
+ * stemmer.c -- The stemmer/case folder
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "stemmer.h"
+
+#include "lovinstem.h"
+#include "simplefrenchstem.h"
+#include "unitool.h"
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:18:23  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.3  1998/12/17 09:12:54  rjmcnab
+
+   Altered mg to process utf-8 encoded Unicode. The main changes
+   are in the parsing of the input, the casefolding, and the stemming.
+
+   Revision 1.2  1998/11/25 07:55:51  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:42  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:57:05  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:42:10  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+#define LOVINSTEMMER        0
+#define SIMPLEFRENCHSTEMMER 1
+
+
+/* decode the utf-8 encoded unicode, casefold and then recode 
+ * making sure the final length doesn't exceed the original
+ * length */
+static void unicode_casefold (u_char *word) {
+  unsigned short out[256]; /* temp space */
+  int i;
+  int len;
+
+  /* decode */
+  utf8_word_to_unicode (word, out, 255);
+  len = out[0];
+
+  /* casefold and simplify-fold */
+  for (i=0; i<len; i++) {
+    out[i+1] = unicode_tosimplified(unicode_tolower(out[i+1]));
+  }
+
+  /* re-code */
+  unicode_to_utf8_word (out, word, word[0]+1);
+}
+
+
+int stemmernumber (u_char *stemmerdescription) {
+  u_char descript[MAX_STEM_DESCRIPTION_LEN];
+  int i;
+
+  /* copy and case-fold the description */
+  for (i=0; (stemmerdescription[i] != '\0') && 
+	 (i < MAX_STEM_DESCRIPTION_LEN-1); i++)
+    descript[i] = tolower (stemmerdescription[i]);
+  descript[i] = '\0';
+
+  /* map the description to its number */
+
+  if ((strcmp (descript, "0") == 0) ||
+      (strcmp (descript, "english") == 0) ||
+      (strcmp (descript, "lovin") == 0))
+    return LOVINSTEMMER;
+
+  if ((strcmp (descript, "1") == 1) ||
+      (strcmp (descript, "french") == 0) ||
+      (strcmp (descript, "simplefrench") == 0))
+    return SIMPLEFRENCHSTEMMER;
+
+  return -1;
+}
+
+
+
+/*
+ * Method 0 - Do not stem or case fold.
+ * Method 1 - Case fold.
+ * Method 2 - Stem.
+ * Method 3 - Case fold and stem.
+ *
+ * The stemmer number should be obtained using
+ * the stemmernumber function above.
+ */
+void 
+stemmer (int method, int stemmer, u_char *word) {
+  if (method & 1) {
+    unicode_casefold (word);
+  }
+
+  if (method & 2) {
+    switch (stemmer) {
+    case LOVINSTEMMER: lovinstem (word);
+      break;
+    case SIMPLEFRENCHSTEMMER: simplefrenchstem (word);
+      break;
+    }
+  }
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/stemmer.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/stemmer.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/stemmer.h	(revision 699)
@@ -0,0 +1,64 @@
+/**************************************************************************
+ *
+ * stemmer.h -- The stemmer/case folder
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef STEMMER_H
+#define STEMMER_H
+
+#include "sysfuncs.h"
+
+#define STEMMER_MASK 3
+#define MAX_STEM_DESCRIPTION_LEN 16
+
+/* stemmernumber will return the stemmer for
+ * a description of the stemmer. Stemmer descriptions
+ * are not case sensitive. Valid descriptions are:
+ *
+ *   'English'
+ *   'Lovin'
+ *   'French'
+ *   'SimpleFrench'
+ *
+ * More than one description might result in the same
+ * stemmer number (for example, for stemming 'English'
+ * we currently use the 'Lovin' stemmer).
+ *
+ * stemmerdescription is a normal C, null-terminated,
+ * string.
+ *
+ * stemmernumber will return -1 if it doesn't know the
+ * stemmer description.
+ */
+int stemmernumber (u_char *stemmerdescription);
+
+/*
+ * Method 0 - Do not stem or case fold.
+ * Method 1 - Case fold.
+ * Method 2 - Stem.
+ * Method 3 - Case fold and stem.
+ *
+ * The stemmer number should be obtained using function
+ * stemmernumber above.
+ */
+void stemmer (int method, int stemmer, u_char * word);
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/term_lists.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/term_lists.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/term_lists.c	(revision 699)
@@ -0,0 +1,331 @@
+/**************************************************************************
+ *
+ * filename -- description
+ * Copyright (C) 1994  Authors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:18:24  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:52  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:43  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/10/20  03:57:07  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "local_strings.h"
+#include "term_lists.h"
+#include "messages.h"
+#include "stemmer.h"
+
+TermList *query_term_list = NULL;
+
+/* =========================================================================
+ * Function: MakeTermList
+ * Description:
+ * Input:
+ * Output:
+ * ========================================================================= */
+TermList *
+MakeTermList (int n)
+{
+  TermList *t;
+  int list_size = (n == 0 ? 1 : n);	/* always allocate at least one node */
+
+  t = Xmalloc (sizeof (TermList) + (list_size - 1) * sizeof (TermEntry));
+  if (!t)
+    FatalError (1, "Unable to allocate term list");
+
+  t->num = n;
+  t->list_size = list_size;
+
+  return t;
+}
+
+/* =========================================================================
+ * Function: ResizeTermList
+ * Description:
+ * Input:
+ * Output:
+ * ========================================================================= */
+
+#define GROWTH_FACTOR 2
+#define MIN_SIZE 2
+
+static void
+ResizeTermList (TermList ** term_list)
+{
+  TermList *tl = *term_list;
+
+  if (tl->num > tl->list_size)
+    {
+      if (tl->list_size)
+	tl->list_size *= GROWTH_FACTOR;
+      else
+	tl->list_size = MIN_SIZE;
+    }
+  tl = Xrealloc (tl, sizeof (TermList) + (tl->list_size - 1) * sizeof (TermEntry));
+
+  if (!tl)
+    FatalError (1, "Unable to resize term list");
+
+  *term_list = tl;
+}
+
+/* =========================================================================
+ * Function: ConvertTermsToString
+ * Description:
+ *      Convert term list into null-terminated string
+ * Input:
+ *      query_term_list = term list
+ * Output:
+ *      str = term string
+ * ========================================================================= */
+
+void
+ConvertTermsToString (TermList * query_term_list, char *str)
+{
+  int i = 0;
+  int total_len = 0;
+
+  /* terms_str should be preallocated */
+  if (!str)
+    return;
+
+  for (i = 0; i < query_term_list->num; i++)
+    {
+      unsigned char *word = query_term_list->TE[i].Word;
+      int len = word[0];
+      total_len += len + 1;	/* +1 for space */
+      if (total_len > MAXTERMSTRLEN)
+	break;
+      strncpy (str, (char *) word + 1, len);
+      str += len;
+      if (i != (query_term_list->num) - 1)
+	{
+	  *str = ' ';
+	  str++;		/* add space gap */
+	}
+
+    }
+  *str = '\0';
+}
+
+/* =========================================================================
+ * Function: ResetTermList
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+void
+ResetTermList (TermList ** tl)
+{
+  if (*tl)
+    FreeTermList (tl);
+  *tl = MakeTermList (0);
+}
+
+/* =========================================================================
+ * Function: AddTermEntry
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+int
+AddTermEntry (TermList ** query_term_list, TermEntry * te)
+{
+  TermList *tl = *query_term_list;
+
+  tl->num++;
+  ResizeTermList (query_term_list);
+  tl = *query_term_list;
+
+  /* copy the structure contents */
+  bcopy ((char *) te, (char *) &(tl->TE[tl->num - 1]), sizeof (TermEntry));
+
+  return tl->num - 1;
+}
+
+
+
+/* =========================================================================
+ * Function: AddTerm
+ * Description: Used in boolean parser - see bool_tree   [RPAP - Feb 97: Term Frequency] 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+int
+AddTerm (TermList ** query_term_list, u_char * Word, int Count, int word_num,
+	 u_long count, u_long doc_count, u_long invf_ptr, u_long invf_len, int stemmer_num)  /* [RPAP - Feb 97: Term Frequency] */
+{
+  int j;
+  TermList *tl = *query_term_list;
+
+  /* Look for the word in the already identified terms */
+  for (j = 0; j < tl->num; j++)
+    {
+      TermEntry *te = &(tl->TE[j]);
+      if (compare (te->Word, Word) == 0)
+	{
+	  te->Count++;
+	  return j;
+	}
+    }
+
+
+  {
+    /* Create a new entry in the list for the new word */
+    TermEntry te;
+
+    /* [RPAP - Feb 97: Term Frequency] */
+    te.WE.word_num = word_num;
+    te.WE.count = count;
+    te.WE.doc_count = doc_count;
+    te.WE.max_doc_count = doc_count;   /* [RPAP - Jan 97: Stem Index Change] */
+    te.WE.invf_ptr = invf_ptr;
+    te.WE.invf_len = invf_len;
+    te.Count = Count;
+    te.Word = copy_string (Word);
+    if (!te.Word)
+      FatalError (1, "Could NOT create memory to add term");
+
+    /* [RPAP - Jan 97: Stem Index Change] */
+    te.Stem = copy_string (Word);
+    if (!te.Stem)
+      FatalError (1, "Could NOT create memory to add term");
+    stemmer (2, stemmer_num, te.Stem);
+
+    te.require_match = 0;  /* [RJM - 07/97: Ranked Required Terms] */
+
+    return AddTermEntry (query_term_list, &te);
+  }
+
+}
+
+/* =========================================================================
+ * Function: FreeTermList
+ * Description:
+ * Input:
+ * Output:
+ * ========================================================================= */
+
+void
+FreeTermList (TermList ** the_tl)
+{
+  int j;
+  TermList *tl = *the_tl;
+
+  for (j = 0; j < tl->num; j++)
+    {
+      if (tl->TE[j].Word)
+	Xfree (tl->TE[j].Word);
+      /* [RPAP - Jan 97: Stem Index Change] */
+      if (tl->TE[j].Stem)
+	Xfree (tl->TE[j].Stem);
+    }
+  Xfree (tl);
+
+  *the_tl = NULL;
+}
+
+/* =========================================================================
+ * Function: PrintWordEntry
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+void
+PrintWordEntry (WordEntry * we, FILE * file)
+{
+  fprintf (file, "we->word_num = %d\n", we->word_num);
+  fprintf (file, "we->count = %ld\n", we->count);
+  fprintf (file, "we->doc_count = %ld\n", we->doc_count);
+  fprintf (file, "we->max_doc_count = %ld\n", we->max_doc_count);
+  fprintf (file, "we->invf_ptr = %ld\n", we->invf_ptr);
+  fprintf (file, "we->invf_len = %ld\n", we->invf_len);
+}
+
+/* =========================================================================
+ * Function: PrintTermEntry
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+void
+PrintTermEntry (TermEntry * te, FILE * file)
+{
+
+  fprintf (file, "Term Entry\n");
+  fprintf (file, "te->Count = %d\n", te->Count);
+  fprintf (file, "te->Word = %s\n", str255_to_string (te->Word, NULL));
+  if (te->Stem != NULL)
+    fprintf (file, "te->Stem = %s\n", str255_to_string (te->Stem, NULL));  /* [RPAP - Jan 97: Stem Index Change] */
+  fprintf (file, "te->require_match = %i\n", te->require_match);  /* [RJM 07/97: Ranked Required Terms] */
+  PrintWordEntry (&(te->WE), file);
+
+}
+
+/* =========================================================================
+ * Function: PrintTermList
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+void
+PrintTermList (TermList * tl, FILE * file)
+{
+  int i;
+
+  fprintf (file, "Term List\n");
+  fprintf (file, "tl->list_size = %d\n", tl->list_size);
+  fprintf (file, "tl->num = %d\n", tl->num);
+
+  for (i = 0; i < tl->num; i++)
+    {
+      fprintf (file, "[%d]\n", i);
+      PrintTermEntry (&(tl->TE[i]), file);
+    }
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/term_lists.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/term_lists.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/term_lists.h	(revision 699)
@@ -0,0 +1,102 @@
+/**************************************************************************
+ *
+ * term_lists.h -- description
+ * Copyright (C) 1994  Authors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:18:24  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:52  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:44  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/10/20  03:57:08  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+ */
+
+#ifndef TERM_LISTS_H
+#define TERM_LISTS_H
+
+#include "sysfuncs.h"
+
+#define MAXTERMSTRLEN 1023	/* maximum number of characters in term string */
+
+typedef struct WordEntry
+  {
+    int word_num;		/* Unique number for each different word */
+    u_long count;		/* Number of times the word occurs in the text */
+    u_long doc_count;		/* Number of documents that contain the word */
+    u_long max_doc_count;       /* [RPAP - Jan 97: Stem Index Change] */
+    u_long invf_ptr;		/* This is a byte position of the  
+				   inverted file entry corresponding to the word */
+    u_long invf_len;		/* This is the length of the inverted 
+				   file entry in bytes */
+  }
+WordEntry;
+
+typedef struct TermEntry
+  {
+    WordEntry WE;
+    int Count;			/* The number of times the word occurs in the query */
+    u_char *Word;		/* The word. */
+    u_char *Stem;               /* [RPAP - Jan 97: Stem Index Change] - The stemmed non-cf version of Word */
+    int require_match;          /* [RJM 07/97: Ranked Required Terms] 0=optional match, 1=must match */
+  }
+TermEntry;
+
+typedef struct TermList
+  {
+    int list_size;
+    int num;
+    TermEntry TE[1];
+  }
+TermList;
+
+#define GetNthWE(term_list, n) (&((term_list)->TE[(n)].WE))
+
+/* --- prototypes --- */
+void ConvertTermsToString (TermList * query_term_list, char *str);
+int AddTermEntry (TermList ** query_term_list, TermEntry * te);
+int AddTerm (TermList ** query_term_list, u_char * Word, int Count, int word_num,
+	     u_long count, u_long doc_count, u_long invf_ptr, u_long invf_len, /* [RPAP - Feb 97: Term Frequency] */
+	     int stemmer_num);  
+void ResetTermList (TermList ** tl);
+void FreeTermList (TermList ** the_tl);
+void PrintWordEntry (WordEntry * we, FILE * file);
+void PrintTermEntry (TermEntry * te, FILE * file);
+void PrintTermList (TermList * tl, FILE * file);
+TermList *MakeTermList (int n);
+
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/text.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/text.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/text.h	(revision 699)
@@ -0,0 +1,176 @@
+/**************************************************************************
+ *
+ * text.h -- Header file for compression related stuff
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+
+#ifndef H_TEXT
+#define H_TEXT
+
+#include "huffman.h"
+
+
+
+
+
+/*****************************************************************************
+ *
+ * There are several different methods of compressing the text of the
+ * database. The following defines list the different methods of text
+ * compression
+ *
+ */
+
+
+/* The dictionary contains all the fragments that occur in the collection
+   i.e. escapes are not possible */
+#define MG_COMPLETE_DICTIONARY 0
+
+
+/* Certain words have been deleted from the dictionary. The words deleted
+   have been used to create the frequency huffman codes of the characters. 
+   This dictionary has an escape code and may be used to compress novel
+   words. This dictionary may fail if there is a novel character. */
+#define MG_PARTIAL_DICTIONARY 1
+
+
+/* This dictionary has an escape so that novel words and non-words can be 
+   coded. The method for coding the novel words and non-words is determined
+   by a dictionary parameter. */
+#define MG_SEED_DICTIONARY 2
+
+
+
+
+
+
+
+
+
+/*****************************************************************************
+ *
+ * With a seed dictionary there are several methods for coding the novel
+ * words and non-words the following defined values specify the different
+ * methods of coding.
+ *
+ */
+
+
+/* Code novel words and non-words character by character using huffman codes.
+   The huffman codes for the word and non-word lengths and characters are 
+   generated from the distribution of lengths and characters in the
+   dictionary. */
+#define MG_NOVEL_HUFFMAN_CHARS 0
+
+
+/* This method codes novel words using binary codes. The novel words are stored
+   in a auxillary dictionary which is built by pass two. */
+#define MG_NOVEL_BINARY 1
+
+
+/* This method codes novel words using delta codes. The novel words are stored
+   in a auxillary dictionary which is built by pass two. */
+#define MG_NOVEL_DELTA 2
+
+
+/* This method codes novel words using hybrid version of delta. The novel
+   words are stored in a auxillary dictionary which is built by pass two. */
+#define MG_NOVEL_HYBRID 3
+
+
+/* This method codes novel words using hybrid version of delta and a MTF
+   operation. The novel words are stored in a auxillary dictionary which
+   is built by pass two. */
+#define MG_NOVEL_HYBRID_MTF 4
+
+
+
+
+
+
+/* This specified an amount of extra space allocated in the
+   compression_dict_header for adding new parameters. As new
+   parameters are added this should be decreased. */
+#define TEXT_PARAMS 15
+
+
+
+typedef struct compression_dict_header
+  {
+    u_long dict_type;
+    u_long novel_method;
+    u_long params[TEXT_PARAMS];
+    u_long num_words[2];
+    u_long num_word_chars[2];
+    u_long lookback;
+  }
+compression_dict_header;
+
+
+typedef struct comp_frags_header
+  {
+    huff_data hd;
+    u_long uncompressed_size;
+    u_long huff_words_size[MAX_HUFFCODE_LEN + 1];
+  }
+comp_frags_header;
+
+
+
+typedef struct compressed_text_header
+  {
+    u_long num_of_docs;
+    u_long num_of_words;	/* number of words in collection */
+    double num_of_bytes; /* [RJM 07/97: 4G limit] */
+    double ratio;
+    u_long length_of_longest_doc;	/* (characters) */
+    u_long dummy;    /* added to make the structure the same on different architectures */
+  }
+compressed_text_header;
+
+
+typedef struct compression_stats_header
+  {
+    u_long num_docs;
+    u_long dummy;   /* added to make the structure the same on different architectures */
+    double num_bytes; /* [RJM 07/97: 4G limit] */
+  }
+compression_stats_header;
+
+
+typedef struct frags_stats_header
+  {
+    u_long num_frags;
+    u_long mem_for_frags;
+  }
+frags_stats_header;
+
+
+typedef struct aux_frags_header
+  {
+    u_long num_frags;
+    u_long mem_for_frags;
+  }
+aux_frags_header;
+
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/text.pass1.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/text.pass1.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/text.pass1.c	(revision 699)
@@ -0,0 +1,427 @@
+/**************************************************************************
+ *
+ * text.pass1.c -- Text compression (Pass 1)
+ * Copyright (C) 1994  Neil Sharman, Gary Eddy and Alistair Moffat
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "huffman.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+
+#include "mg_files.h"
+#include "mg.h"
+#include "build.h"
+#include "locallib.h"
+#include "words.h"
+#include "text.h"
+#include "hash.h"
+#include "local_strings.h"
+
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:18:25  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/12/17 09:12:54  rjmcnab
+
+   Altered mg to process utf-8 encoded Unicode. The main changes
+   are in the parsing of the input, the casefolding, and the stemming.
+
+   Revision 1.1  1998/11/17 09:35:46  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.4  1994/11/25  03:47:47  tes
+   * Committing files before adding the merge stuff.
+   *
+   * Revision 1.3  1994/10/20  03:57:09  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:42:13  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+#define POOL_SIZE 1024*1024
+#define INITIAL_HASH_SIZE 7927
+
+
+
+
+
+
+
+typedef struct hash_rec
+  {
+    unsigned long wcnt;		/* word frequency */
+    unsigned long occurance_num;
+    u_char *word;
+  }
+hash_rec;
+
+typedef struct dict_data
+  {
+    hash_rec *HashTable;
+    unsigned long HashSize;
+    unsigned long HashUsed;
+    unsigned long wordnum;
+    unsigned long words_read;
+    unsigned long bytes_diff;
+    huff_data hd;
+  }
+dict_data;
+
+
+
+static unsigned long LongestDoc = 0;
+static unsigned long occurance_num = 0;
+static dict_data DictData[2];
+
+static u_char *Pool;
+static int PoolLeft;
+static double inputbytes = 0; /* [RJM 07/97: 4G limit] */
+static unsigned long MaxMemInUse = 0;
+static unsigned long MemInUse = 0;
+static compression_stats_header csh =
+{0, 0.0}; /* [RJM 07/97: 4G limit] */
+
+
+static void 
+ChangeMem (int Change)
+{
+  MemInUse += Change;
+  if (MemInUse > MaxMemInUse)
+    MaxMemInUse = MemInUse;
+}
+
+
+
+
+int 
+init_text_1 (char *FileName)
+{
+  int which;
+
+  if (!(Pool = Xmalloc (POOL_SIZE)))
+    {
+      Message ("Unable to allocate memory for pool");
+      return (COMPERROR);
+    }
+  PoolLeft = POOL_SIZE;
+  ChangeMem (POOL_SIZE);
+
+  for (which = 1; which >= 0; which--)
+    {
+      u_char *word;
+      hash_rec *ent;
+      dict_data *dd = &DictData[which];
+
+      dd->wordnum = 0;
+      dd->words_read = 0;
+      dd->bytes_diff = 0;
+      dd->HashSize = INITIAL_HASH_SIZE;
+      dd->HashUsed = 0;
+
+      if (!(dd->HashTable = Xmalloc (sizeof (hash_rec) * dd->HashSize)))
+	{
+	  Message ("Unable to allocate memory for table");
+	  return (COMPERROR);
+	}
+      ChangeMem (sizeof (hash_rec) * dd->HashSize);
+      bzero ((char *) (dd->HashTable), sizeof (hash_rec) * dd->HashSize);
+
+      word = Pool;
+      *Pool++ = '\0';
+      PoolLeft--;
+      {
+	register u_char *wptr;
+	register int hsize = dd->HashSize;
+	register unsigned long hashval, step;
+
+	HASH (hashval, step, word, hsize);
+	wptr = (dd->HashTable + hashval)->word;
+	while (wptr)
+	  {
+	    hashval += step;
+	    if (hashval >= hsize)
+	      hashval -= hsize;
+	    wptr = (dd->HashTable + hashval)->word;
+	  }
+	ent = dd->HashTable + hashval;
+      }
+      ent->wcnt = 1;
+      ent->word = word;
+      dd->HashUsed = 1;
+    }
+  return (COMPALLOK);
+}
+
+
+
+
+int 
+process_text_1 (u_char * s_in, int l_in)
+{
+  int which;
+  u_char *end = s_in + l_in - 1;
+
+  if (l_in > LongestDoc)
+    LongestDoc = l_in;
+
+  csh.num_docs++;
+  csh.num_bytes += l_in;
+
+  which = inaword (s_in, end);
+  /*
+   ** Alternately parse off words and non-words from the input
+   ** stream beginning with a non-word. Each token is then
+   ** inserted into the set if it does not exist or has it's
+   ** frequency count incremented if it does.
+   */
+  for (; s_in <= end; which = !which)
+    {
+      u_char Word[MAXWORDLEN + 1];
+      dict_data *dd = &DictData[which];
+
+      /* First parse a word or non-word out of the string */
+      if (which)
+	PARSE_WORD (Word, s_in, end);
+      else
+	PARSE_NON_WORD (Word, s_in, end);
+
+      dd->wordnum++;
+      inputbytes += *Word;
+      dd->words_read++;
+
+      /* Search the hash table for Word */
+      {
+	register unsigned long hashval, step;
+	register int hsize = dd->HashSize;
+	HASH (hashval, step, Word, hsize);
+	for (;;)
+	  {
+	    register u_char *s1;
+	    register u_char *s2;
+	    register int len;
+	    register hash_rec *ent;
+	    ent = dd->HashTable + hashval;
+	    if (!ent->word)
+	      {
+		int len = *Word + 1;
+		if (len > PoolLeft)
+		  {
+		    if (!(Pool = Xmalloc (POOL_SIZE)))
+		      {
+			Message ("Unable to allocate memory for pool");
+			return (COMPERROR);
+		      }
+		    PoolLeft = POOL_SIZE;
+		    ChangeMem (POOL_SIZE);
+		  }
+		ent->occurance_num = occurance_num++;
+		ent->wcnt = 1;
+		ent->word = Pool;
+		memcpy (Pool, Word, len);
+		Pool += len;
+		PoolLeft -= len;
+		dd->HashUsed++;
+		dd->bytes_diff += Word[0];
+		break;
+	      }
+
+	    /* Compare the words */
+	    s1 = Word;
+	    s2 = ent->word;
+	    len = *s1 + 1;
+	    for (; len; len--)
+	      if (*s1++ != *s2++)
+		break;
+
+	    if (len)
+	      {
+		hashval = (hashval + step);
+		if (hashval >= hsize)
+		  hashval -= hsize;
+	      }
+	    else
+	      {
+		ent->wcnt++;
+		break;
+	      }
+	  }
+      }
+
+
+      if (dd->HashUsed >= dd->HashSize >> 1)
+	{
+	  hash_rec *ht;
+	  unsigned long size;
+	  unsigned long i;
+	  size = prime (dd->HashSize * 2);
+	  if (!(ht = Xmalloc (sizeof (hash_rec) * size)))
+	    {
+	      Message ("Unable to allocate memory for table");
+	      return (COMPERROR);
+	    }
+	  ChangeMem (sizeof (hash_rec) * size);
+	  bzero ((char *) ht, sizeof (hash_rec) * size);
+
+	  for (i = 0; i < dd->HashSize; i++)
+	    if (dd->HashTable[i].word)
+	      {
+		register u_char *wptr;
+		register unsigned long hashval, step;
+
+		wptr = dd->HashTable[i].word;
+		HASH (hashval, step, wptr, size);
+		wptr = (ht + hashval)->word;
+		while (wptr)
+		  {
+		    hashval += step;
+		    if (hashval >= size)
+		      hashval -= size;
+		    wptr = (ht + hashval)->word;
+		  }
+		ht[hashval] = dd->HashTable[i];
+	      }
+	  Xfree (dd->HashTable);
+	  ChangeMem (-sizeof (hash_rec) * dd->HashSize);
+	  dd->HashTable = ht;
+	  dd->HashSize = size;
+
+
+	}
+    }
+  return (COMPALLOK);
+}				/* encode */
+
+
+
+static int 
+PackHashTable (dict_data * dd)
+{
+  int s, d;
+  for (s = d = 0; s < dd->HashSize; s++)
+    if (dd->HashTable[s].word)
+      dd->HashTable[d++] = dd->HashTable[s];
+  ChangeMem (-sizeof (hash_rec) * dd->HashSize);
+  ChangeMem (sizeof (hash_rec) * dd->HashUsed);
+  if (!(dd->HashTable = Xrealloc (dd->HashTable,
+				  sizeof (hash_rec) * dd->HashUsed)))
+    {
+      Message ("Out of memory");
+      return COMPERROR;
+    }
+  dd->HashSize = dd->HashUsed;
+  return COMPALLOK;
+}
+
+
+
+
+
+static int 
+ent_comp (const void *s1, const void *s2)
+{
+  return casecompare (((hash_rec *) s1)->word, ((hash_rec *) s2)->word);
+}
+
+
+
+static void 
+WriteHashTable (FILE * fp, dict_data * dd)
+{
+  frags_stats_header fsh;
+  u_long j = 0;
+  u_char *curr;
+
+  if (PackHashTable (dd) == COMPERROR)
+    return;
+
+  qsort (dd->HashTable, dd->HashUsed, sizeof (hash_rec), ent_comp);
+
+  fsh.num_frags = dd->HashSize;
+  fsh.mem_for_frags = dd->HashSize;
+  for (j = 0; j < dd->HashSize; j++)
+    fsh.mem_for_frags += dd->HashTable[j].word[0];
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(fsh.num_frags);
+  HTONUL(fsh.mem_for_frags);
+
+  fwrite (&fsh, sizeof (fsh), 1, fp);
+
+  for (j = 0; j < dd->HashSize; j++)
+    {
+      curr = dd->HashTable[j].word;
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      HTONUL(dd->HashTable[j].wcnt);
+      HTONUL(dd->HashTable[j].occurance_num);
+
+      fwrite (&dd->HashTable[j].wcnt, sizeof (dd->HashTable[j].wcnt), 1, fp);
+      fwrite (&dd->HashTable[j].occurance_num,
+	      sizeof (dd->HashTable[j].occurance_num), 1, fp);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(dd->HashTable[j].wcnt);
+      NTOHUL(dd->HashTable[j].occurance_num);
+
+      fwrite (curr, sizeof (u_char), curr[0] + 1, fp);
+    }
+}
+
+
+int 
+done_text_1 (char *file_name)
+{
+  char *temp_str;
+  FILE *fp;
+
+  if (!(fp = create_file (file_name, TEXT_STATS_DICT_SUFFIX, "wb",
+			  MAGIC_STATS_DICT, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return COMPERROR;
+
+  temp_str = msg_prefix;
+  msg_prefix = "text.pass1";
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(csh.num_docs);
+  HTOND(csh.num_bytes); /* [RJM 07/97: 4G limit] */
+
+  fwrite (&csh, sizeof (csh), 1, fp);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(csh.num_docs);
+  NTOHD(csh.num_bytes); /* [RJM 07/97: 4G limit] */
+
+  WriteHashTable (fp, &DictData[0]);
+  WriteHashTable (fp, &DictData[1]);
+  msg_prefix = temp_str;
+  return COMPALLOK;
+}				/* done_encode */
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/text.pass2.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/text.pass2.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/text.pass2.c	(revision 699)
@@ -0,0 +1,776 @@
+/**************************************************************************
+ *
+ * text.pass2.c -- Text compression (Pass 2)
+ * Copyright (C) 1994  Neil Sharman, Gary Eddy and Alistair Moffat
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "local_strings.h"
+#include "bitio_m_mem.h"
+#include "bitio_m.h"
+#include "huffman.h"
+#include "bitio_stdio.h"
+#include "huffman_stdio.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg.h"
+#include "mg_files.h"
+#include "build.h"
+#include "words.h"
+#include "text.h"
+#include "hash.h"
+#include "locallib.h"
+#include "comp_dict.h"
+
+
+
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:18:25  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/12/17 09:12:54  rjmcnab
+
+   Altered mg to process utf-8 encoded Unicode. The main changes
+   are in the parsing of the input, the casefolding, and the stemming.
+
+   Revision 1.1  1998/11/17 09:35:47  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:57:10  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:42:14  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#define POOL_SIZE 1024*256
+
+typedef struct char_pool
+  {
+    struct char_pool *next;
+    u_long left;
+    u_char *ptr;
+    u_char pool[POOL_SIZE];
+  }
+char_pool;
+
+typedef struct novel_hash_rec
+  {
+    u_long ordinal_num;
+    u_char *word;
+  }
+novel_hash_rec;
+
+
+#define INITIAL_HASH_SIZE 7927
+#define MAX_SWAPS 10000
+
+typedef struct novel_hash_table
+  {
+    novel_hash_rec *HashTable;
+    u_long HashSize, HashUsed;
+    char_pool *first_pool;
+    char_pool *pool;
+    u_long next_num, binary_start;
+    novel_hash_rec **code_to_nhr;
+  }
+novel_hash_table;
+
+
+static FILE *text, *text_idx;
+
+static u_char *comp_buffer;
+
+static u_long text_length;
+
+/* [RJM 07/97: 4G limit] */
+static double stats_in_tot_bytes = 0.0;
+static double stats_in_bytes = 0.0;
+static double stats_out_bytes = 0.0;
+
+
+static novel_hash_table nht[2];
+
+static u_long prefix_len = 0;
+
+int blk_start[2][33], blk_end[2][33];
+
+
+static char_pool *
+new_pool (char_pool * pool)
+{
+  char_pool *p = Xmalloc (sizeof (char_pool));
+  if (!p)
+    FatalError (1, "Unable to allocate memory for pool");
+  if (pool)
+    pool->next = p;
+  p->next = NULL;
+  p->left = POOL_SIZE;
+  p->ptr = p->pool;
+  return p;
+}
+
+
+
+int 
+init_text_2 (char *file_name)
+{
+  char path[512];
+  int i;
+
+  if (LoadCompressionDictionary (make_name (file_name, TEXT_DICT_SUFFIX,
+					    path)) == COMPERROR)
+    return COMPERROR;
+
+  if (!(text = create_file (file_name, TEXT_SUFFIX, "w+b",
+			    MAGIC_TEXT, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return COMPERROR;
+
+  bzero ((char *) &cth, sizeof (cth));
+
+  if (fwrite (&cth, sizeof (cth), 1, text) != 1)
+    return COMPERROR;
+
+  text_length = sizeof (u_long) + sizeof (cth);
+
+  if (!(text_idx = create_file (file_name, TEXT_IDX_SUFFIX, "w+b",
+				MAGIC_TEXI, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return COMPERROR;
+
+  if (fwrite (&cth, sizeof (cth), 1, text_idx) != 1)
+    return COMPERROR;
+
+  if (!(comp_buffer = Xmalloc (sizeof (u_char) * buf_size)))
+    {
+      Message ("No memory for compression buffer");
+      return (COMPERROR);
+    }
+
+#if 0
+  MaxMemInUse += sizeof (u_char) * buf_size;
+#endif
+
+  if (cdh.novel_method != MG_NOVEL_HUFFMAN_CHARS)
+    for (i = 0; i <= 1; i++)
+      {
+	nht[i].HashSize = INITIAL_HASH_SIZE;
+	nht[i].HashTable = Xmalloc (sizeof (novel_hash_rec) * nht[i].HashSize);
+	bzero ((char *) nht[i].HashTable,
+	       sizeof (novel_hash_rec) * nht[i].HashSize);
+	nht[i].HashUsed = 0;
+	nht[i].HashSize = INITIAL_HASH_SIZE;
+	nht[i].pool = nht[i].first_pool = new_pool (NULL);
+	nht[i].next_num = 1;
+	nht[i].binary_start = 1;
+	if (cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+	  nht[i].code_to_nhr = Xmalloc (sizeof (novel_hash_rec *) *
+					((nht[i].HashSize >> 1) + 2));
+	else
+	  nht[i].code_to_nhr = NULL;
+	if (cdh.novel_method == MG_NOVEL_HYBRID ||
+	    cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+	  {
+	    int num;
+	    num = 1;
+	    blk_start[i][0] = 0;
+	    blk_end[i][0] = cdh.num_words[i] - 1;
+	    while (num < 33)
+	      {
+		blk_start[i][num] = blk_end[i][num - 1] + 1;
+		blk_end[i][num] = blk_start[i][num] +
+		  (blk_end[i][num - 1] - blk_start[i][num - 1]) * 2;
+		num++;
+	      }
+	  }
+      }
+
+  return (COMPALLOK);
+}
+
+
+
+int 
+ic (void *a, void *b)
+{
+  return *((int *) a) - *((int *) b);
+}
+
+
+
+/* #define DOCDUMP 477 */
+
+int 
+process_text_2 (u_char * s_in, int l_in)
+{
+  int which, byte_length;
+  u_char *end = s_in + l_in - 1;
+  int novels_used[2];
+  int swaps[2][MAX_SWAPS];
+
+  which = inaword (s_in, end);
+
+  ENCODE_START (comp_buffer, buf_size)
+
+    ENCODE_BIT (which);
+
+  if (cdh.novel_method == MG_NOVEL_BINARY)
+    {
+      DELTA_ENCODE_L (nht[0].binary_start, prefix_len);
+      DELTA_ENCODE_L (nht[1].binary_start, prefix_len);
+    }
+
+  novels_used[0] = novels_used[1] = 0;
+
+#ifdef DOCDUMP
+  if (cth.num_of_docs == DOCDUMP)
+    {
+      printf ("---------------------------------------------------\n");
+      printf ("which = %d\n", which);
+    }
+#endif
+
+  for (; s_in <= end; which = !which)
+    {
+      u_char Word[MAXWORDLEN + 1];
+      int res;
+
+      if (which)
+	cth.num_of_words++;
+
+      /* First parse a word or non-word out of the string */
+      if (which)
+	PARSE_WORD (Word, s_in, end);
+      else
+	PARSE_NON_WORD (Word, s_in, end);
+
+#ifdef DOCDUMP
+      if (cth.num_of_docs == DOCDUMP)
+	{
+	  printf ("%sword : \"%.*s\"", which ? "    " : "non-", Word[0], Word + 1);
+	}
+#endif
+
+      /* Search the hash table for Word */
+      if (ht[which])
+	{
+	  register unsigned long hashval, step;
+	  register int tsize = ht[which]->size;
+	  register u_char **wptr;
+	  HASH (hashval, step, Word, tsize);
+	  for (;;)
+	    {
+	      register u_char *s1;
+	      register u_char *s2;
+	      register int len;
+	      wptr = ht[which]->table[hashval];
+	      if (wptr == NULL)
+		{
+		  res = COMPERROR;
+		  break;
+		}
+
+	      /* Compare the words */
+	      s1 = Word;
+	      s2 = *wptr;
+	      len = *s1 + 1;
+	      for (; len; len--)
+		if (*s1++ != *s2++)
+		  break;
+
+	      if (len)
+		{
+		  hashval += step;
+		  if (hashval >= tsize)
+		    hashval -= tsize;
+		}
+	      else
+		{
+		  res = ht[which]->table[hashval] - ht[which]->words;
+		  break;
+		}
+	    }
+	}
+      else
+	res = COMPERROR;
+      /* Check that the word was found in the dictionary */
+      if (res == COMPERROR)
+	{
+	  if (cdh.dict_type == MG_COMPLETE_DICTIONARY)
+	    {
+	      Message ("Unknown word \"%.*s\"\n", *Word, Word + 1);
+	      return (COMPERROR);
+	    }
+	  if (cdh.dict_type == MG_PARTIAL_DICTIONARY)
+	    {
+	      u_long i;
+	      if (ht[which])
+		{
+		  res = ht[which]->hd->num_codes - 1;
+		  HUFF_ENCODE (res, ht[which]->codes, ht[which]->hd->clens);
+		}
+	      HUFF_ENCODE (Word[0], lens_codes[which], lens_huff[which].clens);
+	      for (i = 0; i < Word[0]; i++)
+		HUFF_ENCODE (Word[i + 1], char_codes[which],
+			     char_huff[which].clens);
+	    }
+	  if (cdh.dict_type == MG_SEED_DICTIONARY)
+	    {
+	      if (ht[which])
+		{
+		  res = ht[which]->hd->num_codes - 1;
+		  HUFF_ENCODE (res, ht[which]->codes, ht[which]->hd->clens);
+		}
+	      switch (cdh.novel_method)
+		{
+		case MG_NOVEL_HUFFMAN_CHARS:
+		  {
+		    u_long i;
+		    HUFF_ENCODE (Word[0], lens_codes[which],
+				 lens_huff[which].clens);
+		    for (i = 0; i < Word[0]; i++)
+		      HUFF_ENCODE (Word[i + 1], char_codes[which],
+				   char_huff[which].clens);
+		  }
+		  break;
+		case MG_NOVEL_BINARY:
+		case MG_NOVEL_DELTA:
+		case MG_NOVEL_HYBRID:
+		case MG_NOVEL_HYBRID_MTF:
+		  {
+		    register unsigned long hashval, step;
+		    register novel_hash_table *h = &nht[which];
+		    register int hsize = h->HashSize;
+		    register novel_hash_rec *ent;
+		    HASH (hashval, step, Word, hsize);
+		    for (;;)
+		      {
+			register u_char *s1, *s2;
+			register int len;
+			ent = h->HashTable + hashval;
+			if (!ent->word)
+			  {
+			    int len = *Word + 1;
+			    if (len > h->pool->left)
+			      h->pool = new_pool (h->pool);
+			    ent->word = h->pool->ptr;
+			    ent->ordinal_num = h->next_num++;
+			    if (cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+			      h->code_to_nhr[ent->ordinal_num - 1] = ent;
+			    memcpy (h->pool->ptr, Word, len);
+			    h->pool->ptr += len;
+			    h->pool->left -= len;
+			    h->HashUsed++;
+			    break;
+			  }
+			/* Compare the words */
+			s1 = Word;
+			s2 = ent->word;
+			len = *s1 + 1;
+			for (; len; len--)
+			  if (*s1++ != *s2++)
+			    break;
+
+			if (!len)
+			  break;
+
+			hashval = (hashval + step);
+			if (hashval >= hsize)
+			  hashval -= hsize;
+		      }
+
+		    switch (cdh.novel_method)
+		      {
+		      case MG_NOVEL_BINARY:
+			{
+			  BINARY_ENCODE (ent->ordinal_num, h->binary_start);
+			  if (ent->ordinal_num == h->binary_start)
+			    h->binary_start++;
+			}
+			break;
+		      case MG_NOVEL_DELTA:
+			{
+			  DELTA_ENCODE (ent->ordinal_num);
+			}
+			break;
+		      case MG_NOVEL_HYBRID:
+			{
+			  int k = 0;
+			  int j = ent->ordinal_num - 1;
+			  while (j > blk_end[which][k])
+			    k++;
+			  assert (j - blk_start[which][k] + 1 >= 1 &&
+				  j - blk_start[which][k] + 1 <=
+			       blk_end[which][k] - blk_start[which][k] + 1);
+
+			  GAMMA_ENCODE (k + 1);
+			  BINARY_ENCODE (j - blk_start[which][k] + 1,
+					 blk_end[which][k] -
+					 blk_start[which][k] + 1);
+			}
+			break;
+		      case MG_NOVEL_HYBRID_MTF:
+			{
+			  int k = 0;
+			  int j = ent->ordinal_num - 1;
+			  while (j > blk_end[which][k])
+			    k++;
+			  assert (j - blk_start[which][k] + 1 >= 1 &&
+				  j - blk_start[which][k] + 1 <=
+			       blk_end[which][k] - blk_start[which][k] + 1);
+			  GAMMA_ENCODE (k + 1);
+			  BINARY_ENCODE (j - blk_start[which][k] + 1,
+					 blk_end[which][k] -
+					 blk_start[which][k] + 1);
+
+			  if (ent->ordinal_num - 1 >= novels_used[which])
+			    {
+			      int a = novels_used[which];
+			      int b = ent->ordinal_num - 1;
+			      novel_hash_rec *temp;
+
+
+/*                            fprintf(stderr, "a = %d , b = %d\n", a, b);
+ */
+			      temp = h->code_to_nhr[a];
+			      h->code_to_nhr[a] = h->code_to_nhr[b];
+			      h->code_to_nhr[b] = temp;
+			      h->code_to_nhr[a]->ordinal_num = a + 1;
+			      h->code_to_nhr[b]->ordinal_num = b + 1;
+			      if (novels_used[which] == MAX_SWAPS)
+				FatalError (1, "Not enough mem for swapping");
+			      swaps[which][novels_used[which]] = b;
+			      novels_used[which]++;
+			    }
+			}
+			break;
+		      }
+		    if (h->HashUsed >= h->HashSize >> 1)
+		      {
+			novel_hash_rec *ht;
+			unsigned long size;
+			unsigned long i;
+			size = prime (h->HashSize * 2);
+			if (cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+			  {
+			    Xfree (h->code_to_nhr);
+			    h->code_to_nhr = Xmalloc (sizeof (novel_hash_rec *) *
+						      ((size >> 1) + 2));
+			  }
+			if (!(ht = Xmalloc (sizeof (novel_hash_rec) * size)))
+			  {
+			    Message ("Unable to allocate memory for table");
+			    return (COMPERROR);
+			  }
+			bzero ((char *) ht, sizeof (novel_hash_rec) * size);
+
+			for (i = 0; i < h->HashSize; i++)
+			  if (h->HashTable[i].word)
+			    {
+			      register u_char *wptr;
+			      register unsigned long hashval, step;
+
+			      wptr = h->HashTable[i].word;
+			      HASH (hashval, step, wptr, size);
+			      wptr = (ht + hashval)->word;
+			      while (wptr)
+				{
+				  hashval += step;
+				  if (hashval >= size)
+				    hashval -= size;
+				  wptr = (ht + hashval)->word;
+				}
+			      ht[hashval] = h->HashTable[i];
+			      if (cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+				h->code_to_nhr[ht[hashval].ordinal_num - 1] =
+				  &ht[hashval];
+			    }
+			Xfree (h->HashTable);
+			h->HashTable = ht;
+			h->HashSize = size;
+		      }
+		  }
+		  break;
+		}
+	    }
+	}
+      else
+	{
+	  HUFF_ENCODE (res, ht[which]->codes, ht[which]->hd->clens);
+#ifdef DOCDUMP
+	  if (cth.num_of_docs == DOCDUMP)
+	    {
+	      printf ("   %d %d\n", ht[which]->hd->clens[res],
+		      ht[which]->codes[res]);
+	    }
+#endif
+	}
+    }
+
+
+  /* Add a 1 bit onto the end of the buffer the remaining bits in the last
+     byte will all be zero */
+
+  ENCODE_BIT (1);
+
+  ENCODE_FLUSH;
+
+  byte_length = __pos - __base;
+  if (!__remaining)
+    {
+      Message ("The end of the buffer was probably overrun");
+      return COMPERROR;
+    }
+
+  ENCODE_DONE
+
+#ifdef DOCDUMP
+    if (cth.num_of_docs == DOCDUMP)
+    {
+      printf ("unused bits = %d\n", bits_unused);
+    }
+#endif
+
+  HTONUL(text_length);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite (&text_length, sizeof (text_length), 1, text_idx);
+  NTOHUL(text_length);  /* [RPAP - Jan 97: Endian Ordering] */
+  text_length += byte_length;
+
+#ifdef DOCDUMP
+  if (cth.num_of_docs == DOCDUMP)
+    {
+      int i;
+      for (i = 0; i < byte_length; i++)
+	printf ("%02x ", comp_buffer[i]);
+      printf ("\n");
+    }
+#endif
+
+  if (cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+    for (which = 0; which <= 1; which++)
+      for (novels_used[which]--; novels_used[which] >= 0; novels_used[which]--)
+	{
+	  int a = novels_used[which];
+	  int b = swaps[which][novels_used[which]];
+	  novel_hash_rec *temp;
+	  temp = nht[which].code_to_nhr[a];
+	  nht[which].code_to_nhr[a] = nht[which].code_to_nhr[b];
+	  nht[which].code_to_nhr[b] = temp;
+	  nht[which].code_to_nhr[a]->ordinal_num = a + 1;
+	  nht[which].code_to_nhr[b]->ordinal_num = b + 1;
+	}
+
+
+  fwrite (comp_buffer, sizeof (*comp_buffer), byte_length, text);
+
+  if ((double) l_in / (double) byte_length > cth.ratio)
+    cth.ratio = (double) l_in / (double) byte_length;
+
+  cth.num_of_docs++;
+  if (l_in > cth.length_of_longest_doc)
+    cth.length_of_longest_doc = l_in;
+
+  cth.num_of_bytes += l_in;
+
+  if (Comp_Stats)
+    {
+      stats_in_tot_bytes += l_in;
+      stats_in_bytes += l_in;
+      stats_out_bytes += byte_length;
+      if (stats_in_bytes >= comp_stat_point)
+	{
+	  fprintf (Comp_Stats, "%10.0f %10.0f %10.0f %f\n", stats_in_tot_bytes,
+		   stats_in_bytes, stats_out_bytes,
+		   (double) stats_out_bytes / (double) stats_in_bytes);
+	  stats_in_bytes = 0.0;
+	  stats_out_bytes = 0.0;
+	}
+    }
+
+  return COMPALLOK;
+}
+
+
+
+
+
+
+int 
+write_aux_dict (char *FileName)
+{
+  int i;
+  FILE *aux;
+  if (!(aux = create_file (FileName, TEXT_DICT_AUX_SUFFIX, "wb",
+			   MAGIC_AUX_DICT, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return COMPERROR;
+
+  for (i = 0; i <= 1; i++)
+    {
+      aux_frags_header afh;
+      char_pool *cp;
+
+      afh.num_frags = nht[i].HashUsed;
+      afh.mem_for_frags = 0;
+      for (cp = nht[i].first_pool; cp; cp = cp->next)
+	afh.mem_for_frags += POOL_SIZE - cp->left;
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      HTONUL(afh.num_frags);
+      HTONUL(afh.mem_for_frags);
+
+      fwrite (&afh, sizeof (afh), 1, aux);
+
+      for (cp = nht[i].first_pool; cp; cp = cp->next)
+	fwrite (cp->pool, POOL_SIZE - cp->left, sizeof (u_char), aux);
+    }
+  fclose (aux);
+  return COMPALLOK;
+}
+
+
+void 
+estimate_compressed_aux_dict (void)
+{
+  int i;
+  u_long aux_compressed = 0, total_uncomp = 0;
+  for (i = 0; i <= 1; i++)
+    {
+      int j;
+      long chars[256], fchars[256];
+      long lens[16], flens[16];
+      char_pool *cp;
+      bzero ((char *) chars, sizeof (chars));
+      bzero ((char *) lens, sizeof (lens));
+      for (cp = nht[i].first_pool; cp; cp = cp->next)
+	{
+	  u_char *buf = cp->pool;
+	  while (buf != cp->ptr)
+	    {
+	      int len = *buf++;
+	      lens[len]++;
+	      total_uncomp += len + 4;
+	      for (; len; len--)
+		chars[*buf++]++;
+	    }
+	}
+      for (j = 0; j < 256; j++)
+	if (!chars[j] && PESINAWORD (j) == i)
+	  fchars[j] = 1;
+	else
+	  fchars[j] = chars[j];
+      for (j = 0; j < 16; j++)
+	if (!lens[j])
+	  flens[j] = 1;
+	else
+	  flens[j] = lens[j];
+
+      aux_compressed += (Calculate_Huffman_Size (16, flens, lens) +
+			 Calculate_Huffman_Size (256, fchars, chars)) / 8;
+
+    }
+
+  Message ("Aux dictionary (Uncompressed) %.2f Mb ( %u bytes   %0.3f %%)",
+	   total_uncomp / 1024.0 / 1024, total_uncomp,
+	   (total_uncomp * 100.0) / bytes_processed);
+  Message ("Aux dictionary (Compressed)   %.2f Mb ( %.0f bytes   %0.3f %%)",
+	   aux_compressed / 1024.0 / 1024, aux_compressed * 1.0,
+	   (aux_compressed * 100.0) / bytes_processed);
+}
+
+
+
+
+
+
+int 
+done_text_2 (char *FileName)
+{
+  if (Comp_Stats)
+    fprintf (Comp_Stats, "%10.0f %10.0f %10.0f %f\n", stats_in_tot_bytes,
+	     stats_in_bytes, stats_out_bytes,
+	     (double) stats_out_bytes / (double) stats_in_bytes);
+
+  HTONUL(text_length);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite (&text_length, sizeof (text_length), 1, text_idx);
+  NTOHUL(text_length);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(cth.num_of_docs);
+  HTOND(cth.num_of_bytes); /* [RJM 07/97: 4G limit] */
+  HTONUL(cth.num_of_words);
+  HTONUL(cth.length_of_longest_doc);
+  HTOND(cth.ratio);
+
+  if (fseek (text_idx, sizeof (u_long), SEEK_SET) == -1 ||
+      fwrite (&cth, sizeof (cth), 1, text_idx) != 1)
+    return COMPERROR;
+  fclose (text_idx);
+
+  if (fseek (text, sizeof (u_long), SEEK_SET) == -1 ||
+      fwrite (&cth, sizeof (cth), 1, text) != 1)
+    return COMPERROR;
+  fclose (text);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(cth.num_of_docs);
+  NTOHD(cth.num_of_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(cth.num_of_words);
+  NTOHUL(cth.length_of_longest_doc);
+  NTOHD(cth.ratio);
+  
+
+  Message ("Compressed Text %.2f Mb ( %u bytes   %0.3f %%)",
+	   text_length / 1024.0 / 1024.0, text_length,
+	   (text_length * 100.0) / bytes_processed);
+  Message ("Words portion of the dictionary %.2f Mb ( %.0f bytes   %0.3f %%)",
+	   Words_disk / 1024.0 / 1024, Words_disk * 1.0,
+	   (Words_disk * 100.0) / bytes_processed);
+
+  if (cdh.dict_type != MG_COMPLETE_DICTIONARY &&
+      (cdh.novel_method == MG_NOVEL_BINARY ||
+       cdh.novel_method == MG_NOVEL_DELTA ||
+       cdh.novel_method == MG_NOVEL_HYBRID ||
+       cdh.novel_method == MG_NOVEL_HYBRID_MTF))
+    {
+      if (write_aux_dict (FileName) == COMPERROR)
+	return COMPERROR;
+      estimate_compressed_aux_dict ();
+    }
+  else
+    {
+      if (cdh.dict_type != MG_COMPLETE_DICTIONARY)
+	Message ("Huffman info for chars in dictionary %.2f Mb"
+		 " ( %u bytes   %0.3f %%)",
+		 Chars_disk / 1024.0 / 1024, Chars_disk,
+		 (Chars_disk * 100.0) / bytes_processed);
+      unlink (make_name (FileName, TEXT_DICT_AUX_SUFFIX, NULL));
+    }
+
+  return (COMPALLOK);
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/text_get.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/text_get.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/text_get.c	(revision 699)
@@ -0,0 +1,1087 @@
+/**************************************************************************
+ *
+ * text_get.c -- Function for reading documents from the compressed text
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "filestats.h"
+#include "timing.h"
+#include "messages.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "huffman.h"
+#include "bitio_m_mem.h"
+#include "bitio_m.h"
+#include "bitio_stdio.h"
+#include "huffman_stdio.h"
+
+#include "mg.h"
+#include "invf.h"
+#include "text.h"
+#include "lists.h"
+#include "backend.h"
+#include "text_get.h"
+#include "locallib.h"
+#include "words.h"
+#include "mg_errors.h"
+#include "local_strings.h"
+
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:18:26  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:35:48  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:57:11  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:42:15  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+
+
+
+/* FetchDocStart () 
+ * Reads into DocEnt the starting position of the document in the *.text file
+ * Where the first document is document number 1
+ * It returns the true weight of the document.
+ */
+
+
+
+
+static double 
+FetchDocStartLev1 (text_data * td, u_long DN,
+		   u_long * seek_pos, u_long * len)
+{
+  unsigned long data[2];
+  /* [TS:Sep/94] Fixed up the seek call to give the correct offset */
+  Fseek (td->TextIdxFile,
+	 sizeof (unsigned long) * (DN - 1) +	/* the doc offsets */
+	 sizeof (unsigned long) +	/* the magic number */
+	 sizeof (compressed_text_header),	/* the header */
+	 0);
+  Fread ((char *) &data, sizeof (data), 1, td->TextIdxFile);
+  
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(data[0]);
+  NTOHUL(data[1]);
+
+  *seek_pos = data[0];
+  *len = data[1] - data[0];
+  return (1.0);
+}
+
+#define MG_PAGE_SIZE 2048
+
+static int 
+LoadIdx (text_data * td, unsigned long DN)
+{
+  if (!td->idx_data)
+    {
+      td->idx_data = Xmalloc (sizeof (*(td->idx_data)) * MG_PAGE_SIZE);
+      if (!td->idx_data)
+	FatalError (1, "Out of memory in FDSL2");
+    }
+  if (td->current_pos == -1 || DN >= td->current_pos + MG_PAGE_SIZE - 1 ||
+      DN < td->current_pos)
+    {
+      int i, num;  /* [RPAP - Jan 97: Endian Ordering] */
+
+      long rn = (long) DN - (MG_PAGE_SIZE >> 1);
+      if (rn < 1)
+	rn = 1;
+      Fseek (td->TextIdxWgtFile, (sizeof (unsigned long) + sizeof (float)) *
+	       (rn - 1) + sizeof (unsigned long), 0);
+      num = Fread ((char *) td->idx_data, sizeof (*(td->idx_data)), MG_PAGE_SIZE, /* [RPAP - Jan 97: Endian Ordering] */
+		   td->TextIdxWgtFile);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (i = 0; i < num; i++)
+	{
+	  NTOHUL(td->idx_data[i].Start);
+	  NTOHF(td->idx_data[i].Weight);
+	}
+
+      td->current_pos = rn;
+    }
+  return DN - td->current_pos;
+}
+
+static double 
+FDSL2 (text_data * td, unsigned long DN, unsigned long *Pos)
+{
+  unsigned long pos = LoadIdx (td, DN);
+  *Pos = td->idx_data[pos].Start;
+  return (td->idx_data[pos].Weight);
+}
+
+
+static double 
+FetchDocStartLev2 (text_data * td, u_long DN,
+		   u_long * seek_pos, u_long * len)
+{
+  double Weight;
+  unsigned long s1, s2;
+  Weight = FDSL2 (td, DN, &s1);
+  do
+    {
+      DN++;
+      FDSL2 (td, DN, &s2);
+    }
+  while (s2 == s1);
+  *seek_pos = s1;
+  *len = s2 - s1;
+  return (Weight);
+}
+
+
+
+
+double 
+FetchDocStart (query_data * qd, u_long DN, u_long * seek_pos, u_long * len)
+{
+  qd->text_idx_lookups++;
+  if (qd->td->TextIdxWgtFile)
+    return FetchDocStartLev2 (qd->td, DN, seek_pos, len);
+  else
+    return FetchDocStartLev1 (qd->td, DN, seek_pos, len);
+}
+
+unsigned long 
+FetchInitialParagraph (text_data * td, unsigned long ParaNum)
+{
+  if (td->TextIdxWgtFile)
+    {
+      unsigned long pos;
+      unsigned long start;
+      int PN = ParaNum - 1;
+      pos = LoadIdx (td, ParaNum);
+      start = td->idx_data[pos].Start;
+      while (PN > 0)
+	{
+	  pos = LoadIdx (td, PN);
+	  if (td->idx_data[pos].Start != start)
+	    return PN + 1;
+	  PN--;
+	}
+      return PN + 1;
+    }
+  else
+    return ParaNum;
+}
+
+
+
+/* FetchCompressed () 
+ * Reads into buffer DocBuff the compressed form of document DocNum. 
+ * Where the first document is document number 1
+ */
+int 
+FetchCompressed (query_data * qd, char **DocBuff, DocEntry * DocEnt)
+{
+  if (!DocEnt->SeekPos)
+    FetchDocStart (qd, DocEnt->DocNum, &DocEnt->SeekPos, &DocEnt->Len);
+  if (!(*DocBuff = Xmalloc (DocEnt->Len)))
+    return (-1);
+
+  if (Fseek (qd->td->TextFile, DocEnt->SeekPos, 0) == -1)
+    FatalError (1, "Error when seeking into text file");
+#if 0
+  printf ("Loading compressed text %d %d\n", DocEnt->SeekPos, DocEnt->Len);
+#endif
+  if (Fread (*DocBuff, 1, DocEnt->Len, qd->td->TextFile) != DocEnt->Len)
+    FatalError (1, "Error when reading data");
+
+  return (DocEnt->Len);
+
+}
+
+
+text_data *
+LoadTextData (File * text, File * text_idx_wgt, File * text_idx)
+{
+  text_data *td;
+
+  if (!(td = Xmalloc (sizeof (text_data))))
+    {
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  td->TextFile = text;
+  td->TextIdxWgtFile = text_idx_wgt;
+  td->TextIdxFile = text_idx;
+  td->current_pos = -1;
+  td->idx_data = NULL;
+  Fread (&td->cth, sizeof (td->cth), 1, text);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(td->cth.num_of_docs);
+  NTOHD(td->cth.num_of_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(td->cth.num_of_words);
+  NTOHUL(td->cth.length_of_longest_doc);
+  NTOHD(td->cth.ratio);
+
+  return (td);
+}
+
+
+void 
+FreeTextData (text_data * td)
+{
+  if (td)
+    {
+      if (td->idx_data)
+	Xfree (td->idx_data);
+      Xfree (td);
+    }
+}
+
+
+static int 
+pts_comp (const void *A, const void *B)
+{
+  const DocEntry *const *a = A;
+  const DocEntry *const *b = B;
+  return (*a)->DocNum - (*b)->DocNum;
+}
+
+
+
+
+int 
+GetPosLens (query_data * qd, DocEntry * Docs, int num)
+{
+  DocEntry **pts;
+  int i, j;
+  if (!(pts = Xmalloc (num * sizeof (DocEntry *))))
+    {
+      mg_errno = MG_NOMEM;
+      return (-1);
+    }
+  for (i = j = 0; i < num; i++, Docs++)
+    if (!Docs->SeekPos)
+      pts[j++] = Docs;
+
+  if (j)
+    {
+      qsort (pts, j, sizeof (DocEntry *), pts_comp);
+      for (i = 0; i < j; i++)
+	FetchDocStart (qd, pts[i]->DocNum, &pts[i]->SeekPos, &pts[i]->Len);
+    }
+
+  Xfree (pts);
+  return (0);
+}
+
+
+
+
+
+int 
+LoadBuffers (query_data * qd, DocEntry * Docs, int max_mem, int num)
+{
+  DocEntry **pts;
+  int i, j;
+  int mem;
+
+  if (!num)
+    return (0);
+  if (!(pts = Xmalloc (num * sizeof (DocEntry *))))
+    {
+      mg_errno = MG_NOMEM;
+      return (-1);
+    }
+
+  mem = i = 0;
+  do
+    {
+      pts[i] = Docs;
+      mem += Docs->Len;
+      i++;
+      Docs++;
+    }
+  while (i < num && mem < max_mem);
+  if (i > 1)
+    qsort (pts, i, sizeof (DocEntry *), pts_comp);
+  for (j = 0; j < i; j++)
+    {
+      if (FetchCompressed (qd, &pts[j]->CompTextBuffer, pts[j]) == -1)
+	return (-1);
+      ChangeMemInUse (qd, pts[j]->Len);
+    }
+
+  Xfree (pts);
+
+  return (i);
+}
+
+
+
+
+
+void 
+FreeBuffers (query_data * qd, DocEntry * Docs, int num)
+{
+  int i;
+  for (i = 0; i < num; i++, Docs++)
+    if (Docs->CompTextBuffer)
+      {
+	Xfree (Docs->CompTextBuffer);
+	Docs->CompTextBuffer = NULL;
+	ChangeMemInUse (qd, -Docs->Len);
+      }
+}
+
+
+
+/****************************************************************************/
+
+static void 
+FreeAuxDict (auxiliary_dict * ad)
+{
+  if (!ad)
+    return;
+  if (ad->word_data[0])
+    Xfree (ad->word_data[0]);
+  if (ad->word_data[1])
+    Xfree (ad->word_data[1]);
+  if (ad->words[0])
+    Xfree (ad->words[0]);
+  if (ad->words[1])
+    Xfree (ad->words[1]);
+  Xfree (ad);
+}
+
+static auxiliary_dict *
+LoadAuxDict (compression_dict * cd, File * text_aux_dict)
+{
+  auxiliary_dict *ad;
+  int i;
+
+  if (!(ad = Xmalloc (sizeof (auxiliary_dict))))
+    {
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  bzero ((char *) ad, sizeof (*ad));
+
+  for (i = 0; i <= 1; i++)
+    {
+      int j;
+      u_char *pos;
+
+      Fread (&ad->afh[i], sizeof (aux_frags_header), 1, text_aux_dict);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(ad->afh[i].num_frags);
+      NTOHUL(ad->afh[i].mem_for_frags);
+
+      if (!(ad->word_data[i] = Xmalloc (ad->afh[i].mem_for_frags)))
+	{
+	  mg_errno = MG_NOMEM;
+	  FreeAuxDict (ad);
+	  return (NULL);
+	}
+      if (!(ad->words[i] = Xmalloc (ad->afh[i].num_frags * sizeof (u_char *))))
+	{
+	  mg_errno = MG_NOMEM;
+	  FreeAuxDict (ad);
+	  return (NULL);
+	}
+
+      Fread (ad->word_data[i], ad->afh[i].mem_for_frags, sizeof (u_char),
+	     text_aux_dict);
+
+      pos = ad->word_data[i];
+      for (j = 0; j < ad->afh[i].num_frags; j++)
+	{
+	  ad->words[i][j] = pos;
+	  pos += *pos + 1;
+	}
+      if (cd->cdh.novel_method == MG_NOVEL_HYBRID ||
+	  cd->cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+	{
+	  int num;
+	  num = 1;
+	  ad->blk_start[i][0] = 0;
+	  ad->blk_end[i][0] = cd->cdh.num_words[i] - 1;
+	  while (num < 33)
+	    {
+	      ad->blk_start[i][num] = ad->blk_end[i][num - 1] + 1;
+	      ad->blk_end[i][num] = ad->blk_start[i][num] +
+		(ad->blk_end[i][num - 1] - ad->blk_start[i][num - 1]) * 2;
+	      num++;
+	    }
+	}
+    }
+  return (ad);
+}
+
+
+
+
+
+
+static u_char ***
+ReadInWords (File * dict, compression_dict * cd,
+	     comp_frags_header * cfh, u_char ** escape)
+{
+  int i, lookback;
+  int ptrs_reqd = 0;
+  int mem_reqd = 0;
+  int num_set[MAX_HUFFCODE_LEN + 1];
+  u_char *next_word[MAX_HUFFCODE_LEN + 1];
+  u_char **vals;
+  u_char ***values;
+  u_char word[MAXWORDLEN + 1];
+  u_char last_word[MAX_HUFFCODE_LEN + 1][MAXWORDLEN + 1];
+
+  lookback = cd->cdh.lookback;
+
+  for (i = cfh->hd.mincodelen; i <= cfh->hd.maxcodelen; i++)
+    {
+      ptrs_reqd += (cfh->hd.lencount[i] + ((1 << lookback) - 1)) >> lookback;
+      mem_reqd += cfh->huff_words_size[i];
+    }
+
+  if (!(vals = Xmalloc (ptrs_reqd * sizeof (*vals))))
+    return (NULL);
+
+  if (!(values = Xmalloc ((MAX_HUFFCODE_LEN + 1) * sizeof (u_char **))))
+    return (NULL);
+
+  if (!(next_word[0] = Xmalloc (mem_reqd))) return (NULL);
+
+  cd->MemForCompDict += ptrs_reqd * sizeof (*vals) +
+    (MAX_HUFFCODE_LEN + 1) * sizeof (u_char **) +
+    mem_reqd;
+
+  values[0] = vals;
+  values[0][0] = next_word[0];
+  for (i = 1; i <= cfh->hd.maxcodelen; i++)
+    {
+      int next_start = (values[i - 1] - vals) +
+      ((cfh->hd.lencount[i - 1] + ((1 << lookback) - 1)) >> lookback);
+      values[i] = &vals[next_start];
+      next_word[i] = next_word[i - 1] + cfh->huff_words_size[i - 1];
+      values[i][0] = next_word[i];
+    }
+
+  bzero ((char *) num_set, sizeof (num_set));
+
+  for (i = 0; i < cfh->hd.num_codes; i++)
+    {
+      register int val, copy;
+      register int len = cfh->hd.clens[i];
+      val = Getc (dict);
+      copy = (val >> 4) & 0xf;
+      val &= 0xf;
+
+      Fread (word + copy + 1, sizeof (u_char), val, dict);
+      *word = val + copy;
+
+      if ((num_set[len] & ((1 << lookback) - 1)) == 0)
+	{
+	  values[len][num_set[len] >> lookback] = next_word[len];
+	  memcpy (next_word[len], word, *word + 1);
+	  if (escape && i == cfh->hd.num_codes - 1)
+	    *escape = next_word[len];
+	  next_word[len] += *word + 1;
+	}
+      else
+	{
+	  copy = prefixlen (last_word[len], word);
+	  memcpy (next_word[len] + 1, word + copy + 1, *word - copy);
+	  *next_word[len] = (copy << 4) + (*word - copy);
+	  if (escape && i == cfh->hd.num_codes - 1)
+	    *escape = next_word[len];
+	  next_word[len] += (*word - copy) + 1;
+	}
+      memcpy (last_word[len], word, *word + 1);
+      num_set[len]++;
+    }
+  if (cfh->hd.clens)
+    Xfree (cfh->hd.clens);
+  cfh->hd.clens = NULL;
+  return values;
+}
+
+
+static compression_dict *
+Load_Comp_Dict (File * dict, File * aux_dict)
+{
+  int which;
+  compression_dict *cd;
+
+  if (!(cd = Xmalloc (sizeof (compression_dict))))
+    {
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  bzero ((char *) cd, sizeof (compression_dict));
+
+  cd->MemForCompDict = sizeof (compression_dict);
+
+  if (F_Read_cdh (dict, &cd->cdh, &cd->MemForCompDict, NULL) == -1)
+    return NULL;
+
+  for (which = 0; which < 2; which++)
+    switch (cd->cdh.dict_type)
+      {
+      case MG_COMPLETE_DICTIONARY:
+	{
+	  if (!(cd->cfh[which] = Xmalloc (sizeof (*cd->cfh[which]))))
+	    return NULL;
+	  cd->MemForCompDict += sizeof (*cd->cfh[which]);
+	  if (F_Read_cfh (dict, cd->cfh[which], &cd->MemForCompDict, NULL) == -1)
+	    return NULL;
+
+	  if (!(cd->values[which] = ReadInWords (dict, cd, cd->cfh[which],
+						 NULL)))
+	    return NULL;
+	  cd->escape[which] = NULL;
+
+	}
+	break;
+      case MG_PARTIAL_DICTIONARY:
+	{
+	  huff_data *hd;
+	  u_long **vals;
+	  if (cd->cdh.num_words[which])
+	    {
+	      if (!(cd->cfh[which] = Xmalloc (sizeof (*cd->cfh[which]))))
+		return NULL;
+	      cd->MemForCompDict += sizeof (*cd->cfh[which]);
+	      if (F_Read_cfh (dict, cd->cfh[which], &cd->MemForCompDict, NULL) == -1)
+		return NULL;
+
+	      if (!(cd->values[which] = ReadInWords (dict, cd, cd->cfh[which],
+						     &cd->escape[which])))
+		return NULL;
+	    }
+	  if (!(hd = Xmalloc (sizeof (huff_data))))
+	    return NULL;
+	  cd->MemForCompDict += sizeof (huff_data);
+	  if (F_Read_Huffman_Data (dict, hd, &cd->MemForCompDict, NULL) == -1)
+	    return NULL;
+	  if (!(vals = Generate_Huffman_Vals (hd, &cd->MemForCompDict)))
+	    return NULL;
+	  if (hd->clens)
+	    Xfree (hd->clens);
+	  hd->clens = NULL;
+	  cd->chars_huff[which] = hd;
+	  cd->chars_vals[which] = vals;
+	  if (!(hd = Xmalloc (sizeof (huff_data))))
+	    return NULL;
+	  cd->MemForCompDict += sizeof (huff_data);
+	  if (F_Read_Huffman_Data (dict, hd, &cd->MemForCompDict, NULL) == -1)
+	    return NULL;
+	  if (!(vals = Generate_Huffman_Vals (hd, &cd->MemForCompDict)))
+	    return NULL;
+	  cd->lens_huff[which] = hd;
+	  cd->lens_vals[which] = vals;
+	  if (hd->clens)
+	    Xfree (hd->clens);
+	  hd->clens = NULL;
+	}
+	break;
+      case MG_SEED_DICTIONARY:
+	{
+	  huff_data *hd;
+	  u_long **vals;
+	  if (cd->cdh.num_words[which])
+	    {
+	      if (!(cd->cfh[which] = Xmalloc (sizeof (*cd->cfh[which]))))
+		return NULL;
+	      cd->MemForCompDict += sizeof (*cd->cfh[which]);
+	      if (F_Read_cfh (dict, cd->cfh[which], &cd->MemForCompDict, NULL) == -1)
+		return NULL;
+
+	      if (!(cd->values[which] = ReadInWords (dict, cd, cd->cfh[which],
+						     &cd->escape[which])))
+		return NULL;
+	    }
+	  switch (cd->cdh.novel_method)
+	    {
+	    case MG_NOVEL_HUFFMAN_CHARS:
+	      if (!(hd = Xmalloc (sizeof (huff_data))))
+		return NULL;
+	      cd->chars_huff[which] = hd;
+	      cd->MemForCompDict += sizeof (huff_data);
+	      if (F_Read_Huffman_Data (dict, hd, &cd->MemForCompDict,
+				       NULL) == -1)
+		return NULL;
+	      if (!(vals = Generate_Huffman_Vals (hd, &cd->MemForCompDict)))
+		return NULL;
+	      cd->chars_vals[which] = vals;
+	      if (hd->clens)
+		Xfree (hd->clens);
+	      hd->clens = NULL;
+	      if (!(hd = Xmalloc (sizeof (huff_data))))
+		return NULL;
+	      cd->MemForCompDict += sizeof (huff_data);
+	      cd->lens_huff[which] = hd;
+	      if (F_Read_Huffman_Data (dict, hd, &cd->MemForCompDict
+				       ,NULL) == -1)
+		return NULL;
+	      if (!(vals = Generate_Huffman_Vals (hd, &cd->MemForCompDict)))
+		return NULL;
+	      cd->lens_vals[which] = vals;
+	      if (hd->clens)
+		Xfree (hd->clens);
+	      hd->clens = NULL;
+	      break;
+	    case MG_NOVEL_BINARY:
+	      break;
+	    case MG_NOVEL_DELTA:
+	      break;
+	    case MG_NOVEL_HYBRID:
+	      break;
+	    case MG_NOVEL_HYBRID_MTF:
+	      break;
+	    }
+	  break;
+	}
+      }
+
+  if (cd->cdh.novel_method == MG_NOVEL_BINARY ||
+      cd->cdh.novel_method == MG_NOVEL_DELTA ||
+      cd->cdh.novel_method == MG_NOVEL_HYBRID ||
+      cd->cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+    {
+      if (!aux_dict)
+	{
+	  mg_errno = MG_NOFILE;
+	  FreeCompDict (cd);
+	  return (NULL);
+	}
+
+      if (!(cd->ad = LoadAuxDict (cd, aux_dict)))
+	{
+	  FreeCompDict (cd);
+	  return (NULL);
+	}
+    }
+
+
+  mg_errno = MG_NOERROR;
+
+  cd->fast_loaded = 0;
+  return (cd);
+}
+
+#define WORDNO(p, base) ((((char*)(p))-((char*)(base)))/sizeof(u_char*))
+
+#define IS_FIXUP(p) ((fixup[WORDNO(p,cd)/8] & (1<<(WORDNO(p, cd) & 7))) != 0)
+
+
+static compression_dict *
+Load_Fast_Comp_Dict (File * text_fast_comp_dict)
+{
+  compression_dict *cd;
+  u_long *p, *end;
+  u_char *fixup;
+  u_long mem;
+  u_long fixup_mem;
+  int i;  /* [RPAP - Jan 97: Endian Ordering] */
+
+  Fread (&mem, sizeof (mem), 1, text_fast_comp_dict);
+  NTOHUL(mem);  /* [RPAP - Jan 97: Endian Ordering] */
+  Fread (&fixup_mem, sizeof (fixup_mem), 1, text_fast_comp_dict);
+  NTOHUL(fixup_mem);  /* [RPAP - Jan 97: Endian Ordering] */
+  if (!(cd = Xmalloc (mem)))
+    {
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  end = (u_long *) (((u_char *) cd) + mem);
+  Fread (cd, sizeof (u_char), mem, text_fast_comp_dict);
+
+  if (!(fixup = Xmalloc (fixup_mem)))
+    {
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  Fread (fixup, fixup_mem, sizeof (u_char), text_fast_comp_dict);
+
+  for (p = (u_long *) cd; (u_long) p < (u_long) end; p++)
+    if (IS_FIXUP (p))
+      {
+	NTOHUL(*p);  /* [RPAP - Jan 97: Endian Ordering] */
+	*p = *p + (u_long) cd;
+      }
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  /* cdh */
+  NTOHUL(cd->cdh.dict_type);
+  NTOHUL(cd->cdh.novel_method);
+  for (i = 0; i < TEXT_PARAMS; i++)
+    NTOHUL(cd->cdh.params[i]);
+  NTOHUL(cd->cdh.num_words[0]);
+  NTOHUL(cd->cdh.num_words[1]);
+  NTOHUL(cd->cdh.num_word_chars[0]);
+  NTOHUL(cd->cdh.num_word_chars[1]);
+  NTOHUL(cd->cdh.lookback);
+  /* cfh */
+  for (i = 0; i <= 1; i++)
+    {
+      int j;
+
+      NTOHSI(cd->cfh[i]->hd.num_codes);
+      NTOHSI(cd->cfh[i]->hd.mincodelen);
+      NTOHSI(cd->cfh[i]->hd.maxcodelen);
+      for (j = 0; j < MAX_HUFFCODE_LEN + 1; j++)
+	{
+	  NTOHSI(cd->cfh[i]->hd.lencount[j]);
+	  NTOHUL(cd->cfh[i]->hd.min_code[j]);
+	}
+      NTOHUL(cd->cfh[i]->uncompressed_size);
+      for (j = 0; j < MAX_HUFFCODE_LEN + 1; j++)
+	NTOHUL(cd->cfh[i]->huff_words_size[j]);
+    }
+  NTOHUL(cd->MemForCompDict);
+  /* ad */
+  if (cd->cdh.novel_method == MG_NOVEL_BINARY ||
+      cd->cdh.novel_method == MG_NOVEL_DELTA ||
+      cd->cdh.novel_method == MG_NOVEL_HYBRID ||
+      cd->cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+    for (i = 0; i <= 1; i++)
+      {
+	int j;
+	
+	NTOHUL(cd->ad->afh[i].num_frags);
+	NTOHUL(cd->ad->afh[i].mem_for_frags);
+	for (j = 0; j < 33; j++)
+	  {
+	    NTOHSI(cd->ad->blk_start[i][j]);
+	    NTOHSI(cd->ad->blk_end[i][j]);
+	  }
+      }
+  NTOHSI(cd->fast_loaded);
+
+  Xfree (fixup);
+  return (cd);
+}
+
+
+
+
+compression_dict *
+LoadCompDict (File * text_comp_dict,
+	      File * text_aux_dict,
+	      File * text_fast_comp_dict)
+{
+  return text_fast_comp_dict ?
+    Load_Fast_Comp_Dict (text_fast_comp_dict) :
+    Load_Comp_Dict (text_comp_dict, text_aux_dict);
+}
+
+
+
+
+void 
+FreeCompDict (compression_dict * cd)
+{
+  int which;
+  if (cd->fast_loaded)
+    {
+      Xfree (cd);
+      return;
+    }
+  for (which = 0; which < 2; which++)
+    {
+      if (cd->cfh[which])
+	Xfree (cd->cfh[which]);
+      if (cd->chars_huff[which])
+	Xfree (cd->chars_huff[which]);
+      if (cd->lens_huff[which])
+	Xfree (cd->lens_huff[which]);
+      if (cd->values[which])
+	{
+	  Xfree (cd->values[which][0][0]); /* [RJM 07/98: Memory Leak] */
+	  Xfree (cd->values[which][0]);
+	  Xfree (cd->values[which]);
+	}
+      if (cd->chars_vals[which])
+	{
+	  Xfree (cd->chars_vals[which][0]);
+	  Xfree (cd->chars_vals[which]);
+	}
+      if (cd->lens_vals[which])
+	{
+	  Xfree (cd->lens_vals[which][0]);
+	  Xfree (cd->lens_vals[which]);
+	}
+    }
+  if (cd->ad)
+    FreeAuxDict (cd->ad);
+  Xfree (cd);
+}
+
+
+
+
+
+#define MY_HUFF_DECODE(len, code, mcodes)				\
+  do {									\
+    register unsigned long *__min_code = (mcodes);			\
+    register unsigned long *__mclen = __min_code;			\
+    register unsigned long __code = 0;					\
+    do									\
+      {									\
+        DECODE_ADD(__code);						\
+      }									\
+    while (__code < *++__mclen);					\
+    (len) = __mclen - __min_code;					\
+    (code) = __code - *__mclen;						\
+  } while(0);
+
+
+/*#define DUMPDOC */
+
+#define MAX_SWAPS 10000
+
+int 
+DecodeText (compression_dict * cd,
+	    u_char * s_in, int l_in, u_char * s_out, int *l_out)
+{
+  auxiliary_dict *ad = cd->ad;
+  int which;
+  u_long num_bits, bits;
+  u_char *ptr = s_out;
+  static int num = 0;
+  u_long binary_start[2];
+  int novels_used[2];
+  int swaps[2][MAX_SWAPS];
+  novels_used[0] = novels_used[1] = 0;
+
+  {
+    unsigned char bf = s_in[l_in - 1];
+    num_bits = 1;
+    while ((bf & 1) != 1)
+      {
+	num_bits++;
+	bf >>= 1;
+      }
+    num_bits = l_in * 8 - num_bits;
+  }
+
+  DECODE_START (s_in, l_in)
+
+    which = DECODE_BIT;
+  bits = 1;
+
+  if (cd->cdh.novel_method == MG_NOVEL_BINARY)
+    {
+      DELTA_DECODE_L (binary_start[0], bits);
+      DELTA_DECODE_L (binary_start[1], bits);
+    }
+
+
+  while (bits < num_bits)
+    {
+      register unsigned code, len;
+      register int r;
+      register u_char *t, *b = NULL;
+      u_char word[MAXWORDLEN + 1];
+
+#ifdef DUMPDOC
+      printf ("\n%d %d  ", bits, num_bits);
+#endif
+      if (cd->cfh[which])
+	{
+	  MY_HUFF_DECODE (len, code, cd->cfh[which]->hd.min_code);
+	  bits += len;
+
+	  r = code & ((1 << cd->cdh.lookback) - 1);
+	  t = cd->values[which][len][code >> cd->cdh.lookback];
+
+	  /* step through from base pointer */
+	  b = word + 1;
+	  while (r--)
+	    {
+	      register int copy = *t >> 4;
+	      memcpy (word + copy + 1, t + 1, *t & 0xf);
+	      word[0] = copy + (*t & 0xf);
+	      t += ((*t) & 0xf) + 1;
+	    }
+	}
+      else
+	t = NULL;
+      if (t == cd->escape[which])
+	{
+	  switch (cd->cdh.novel_method)
+	    {
+	    case MG_NOVEL_HUFFMAN_CHARS:
+	      {
+		int len, i;
+		int c;
+		HUFF_DECODE_L (len, cd->lens_huff[which]->min_code,
+			       cd->lens_vals[which], bits);
+		for (i = 0; i < len; i++)
+		  {
+		    HUFF_DECODE_L (c, cd->chars_huff[which]->min_code,
+				   cd->chars_vals[which], bits);
+		    *ptr++ = c;
+		  }
+	      }
+	      break;
+	    case MG_NOVEL_BINARY:
+	    case MG_NOVEL_DELTA:
+	    case MG_NOVEL_HYBRID:
+	    case MG_NOVEL_HYBRID_MTF:
+	      {
+		int idx = 0, len;
+		u_char *base;
+		switch (cd->cdh.novel_method)
+		  {
+		  case MG_NOVEL_BINARY:
+		    {
+		      BINARY_DECODE_L (idx, binary_start[which], bits);
+		      if (idx == binary_start[which])
+			binary_start[which]++;
+		      idx--;
+		    }
+		    break;
+		  case MG_NOVEL_DELTA:
+		    {
+		      DELTA_DECODE_L (idx, bits);
+		      idx--;
+		    }
+		    break;
+		  case MG_NOVEL_HYBRID:
+		    {
+		      int k;
+		      GAMMA_DECODE_L (k, bits);
+		      k--;
+		      BINARY_DECODE_L (idx,
+				       ad->blk_end[which][k] -
+				       ad->blk_start[which][k] + 1, bits);
+		      idx += ad->blk_start[which][k] - 1;
+		    }
+		    break;
+		  case MG_NOVEL_HYBRID_MTF:
+		    {
+		      int k;
+		      GAMMA_DECODE_L (k, bits);
+		      k--;
+		      BINARY_DECODE_L (idx,
+				       ad->blk_end[which][k] -
+				       ad->blk_start[which][k] + 1, bits);
+		      idx += ad->blk_start[which][k] - 1;
+		      if (idx >= novels_used[which])
+			{
+			  u_char *temp;
+			  temp = ad->words[which][idx];
+			  ad->words[which][idx] =
+			    ad->words[which][novels_used[which]];
+			  ad->words[which][novels_used[which]] = temp;
+			  swaps[which][novels_used[which]] = idx;
+			  idx = novels_used[which]++;
+			}
+		    }
+		    break;
+		  }
+		base = ad->words[which][idx];
+		len = *base++;
+#ifdef DUMPDOC
+		printf ("[[");
+#endif
+		for (; len; len--)
+		  {
+		    *ptr++ = *base++;
+#ifdef DUMPDOC
+		    putchar (*(base - 1));
+#endif
+		  }
+#ifdef DUMPDOC
+		printf ("]]");
+#endif
+	      }
+	      break;
+	    }
+	}
+      else
+	{
+	  /* copy over the matching prefix */
+	  r = (*t >> 4);
+	  while (r--)
+#ifndef DUMPDOC
+	    *ptr++ = *b++;
+#else
+	    {
+	      *ptr = *b++;
+	      putchar (*ptr);
+	      ptr++;
+	    }
+#endif
+
+	  /* and the stored suffix */
+	  r = ((*t) & 0xf);
+	  while (r--)
+#ifndef DUMPDOC
+	    *ptr++ = *++t;
+#else
+	    {
+	      *ptr = *++t;
+	      putchar (*ptr);
+	      ptr++;
+	    }
+#endif
+	}
+      which = !which;
+    }
+
+  DECODE_DONE
+
+    * l_out = ptr - s_out;
+  num += *l_out + 1;
+
+  if (cd->cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+    for (which = 0; which <= 1; which++)
+      for (novels_used[which]--; novels_used[which] >= 0; novels_used[which]--)
+	{
+	  int a = novels_used[which];
+	  int b = swaps[which][novels_used[which]];
+	  u_char *temp;
+	  temp = ad->words[which][a];
+	  ad->words[which][a] = ad->words[which][b];
+	  ad->words[which][b] = temp;
+	}
+  return (COMPALLOK);
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/text_get.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/text_get.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/text_get.h	(revision 699)
@@ -0,0 +1,67 @@
+/**************************************************************************
+ *
+ * text_get.h -- Function for reading documents from the compressed text
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef H_TEXT_GET
+#define H_TEXT_GET
+
+
+
+/* FetchDocStart () 
+ * Reads into DocEnt the starting position of the document in the *.text file
+ * Where the first document is document number 1
+ * It returns the true weight of the document.
+ */
+double FetchDocStart (query_data * qd, u_long DN, u_long * seek_pos, u_long * len);
+
+unsigned long FetchInitialParagraph (text_data * td, unsigned long ParaNum);
+
+
+/* FetchCompressed () 
+ * Reads into buffer DocBuff the compressed form of document DocNum. 
+ * Where the first document is document number 1
+ */
+int FetchCompressed (query_data * qd, char **DocBuff, DocEntry * DocEnt);
+
+text_data *LoadTextData (File * text, File * text_idx_wgt, File * text_idx);
+
+void FreeTextData (text_data * td);
+
+int GetPosLens (query_data * qd, DocEntry * Docs, int num);
+
+int LoadBuffers (query_data * qd, DocEntry * Docs, int max_mem, int num);
+
+void FreeBuffers (query_data * qd, DocEntry * Docs, int num);
+
+
+
+compression_dict *LoadCompDict (File * text_comp_dict,
+				File * text_aux_dict,
+				File * text_fast_comp_dict);
+
+void FreeCompDict (compression_dict * cd);
+
+
+int DecodeText (compression_dict * cd,
+		u_char * s_in, int l_in, u_char * s_out, int *l_out);
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/warranty.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/warranty.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/warranty.h	(revision 699)
@@ -0,0 +1,24 @@
+static char *warranty_str[] = {
+"			    NO WARRANTY\n",
+"\n",
+"  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY\n",
+"FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN\n",
+"OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES\n",
+"PROVIDE THE PROGRAM \"AS IS\" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED\n",
+"OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF\n",
+"MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS\n",
+"TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE\n",
+"PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,\n",
+"REPAIR OR CORRECTION.\n",
+"\n",
+"  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING\n",
+"WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR\n",
+"REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,\n",
+"INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING\n",
+"OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED\n",
+"TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY\n",
+"YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER\n",
+"PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE\n",
+"POSSIBILITY OF SUCH DAMAGES.\n",
+"\n",
+"		     END OF TERMS AND CONDITIONS\n"};
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/warranty.src
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/warranty.src	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/warranty.src	(revision 699)
@@ -0,0 +1,23 @@
+			    NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+		     END OF TERMS AND CONDITIONS
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/weights.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/weights.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/weights.c	(revision 699)
@@ -0,0 +1,158 @@
+/**************************************************************************
+ *
+ * weights.c -- Functions for reading the weights file in mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "filestats.h"
+#include "memlib.h"
+#include "messages.h"
+#include "timing.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg.h"
+#include "invf.h"
+#include "text.h"
+#include "lists.h"
+#include "backend.h"
+#include "weights.h"
+#include "locallib.h"
+#include "mg_errors.h"
+
+#define MAXBITS (sizeof(unsigned long) * 8)
+
+/*
+   $Log$
+   Revision 1.1  1999/08/10 21:18:27  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:35:51  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.2  1994/09/20  04:42:18  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+
+approx_weights_data *
+LoadDocWeights (File * weight_file,
+		unsigned long num_of_docs)
+{
+  approx_weights_data *awd;
+  int num;
+
+  if (!(awd = Xmalloc (sizeof (*awd))))
+    {
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  Fseek (weight_file, sizeof (long), 0);
+  Fread (&awd->bits, sizeof (awd->bits), 1, weight_file);
+  Fread (&awd->L, sizeof (awd->L), 1, weight_file);
+  NTOHD(awd->L);  /* [RPAP - Jan 97: Endian Ordering] */
+  Fread (&awd->B, sizeof (awd->B), 1, weight_file);
+  NTOHD(awd->B);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  awd->mask = awd->bits == 32 ? 0xffffffff : (1 << awd->bits) - 1;
+
+  num = (num_of_docs * awd->bits + 31) / 32;
+  if (!(awd->DocWeights = Xmalloc (sizeof (unsigned long) * num)))
+    {
+      Xfree (awd);
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  Fread (awd->DocWeights, sizeof (unsigned long), num, weight_file);
+  {
+    /* [RPAP - Jan 97: Endian Ordering] */
+    unsigned long i;
+    for (i = 0; i < num; i++)
+      NTOHUL(awd->DocWeights[i]);
+  }
+
+  awd->MemForWeights = num * sizeof (unsigned long);
+  awd->num_of_docs = num_of_docs;
+
+  mg_errno = MG_NOERROR;
+
+  if (awd->bits <= 12)
+    {
+      int i, size = (1 << awd->bits);
+      if (!(awd->table = (float *) Xmalloc (size * sizeof (float))))
+	  return (awd);
+      awd->table[0] = awd->L;
+      for (i = 1; i < size; i++)
+	awd->table[i] = awd->table[i - 1] * awd->B;
+    }
+  else
+    awd->table = NULL;
+  return awd;
+}
+
+
+
+
+
+
+/*
+ * the first document in the collection has DocNum = 0
+ */
+float 
+GetLowerApproxDocWeight (approx_weights_data * awd, register int DocNum)
+{
+  register unsigned long c, Pos;
+  register unsigned long *dw;
+  if (awd == NULL)
+    return 1.0;
+#if 0
+  if (DocNum < 0 || DocNum >= awd->num_of_docs)
+    FatalError (1, "Something is wrong in \"GetDocWeight\" DocNum = %d\n",
+		DocNum);
+#endif
+  Pos = DocNum * awd->bits;
+  dw = &(awd->DocWeights[Pos / MAXBITS]);
+  Pos &= (MAXBITS - 1);
+  c = *dw >> Pos;
+  if (Pos + awd->bits > MAXBITS)
+    c |= *(dw + 1) << (MAXBITS - Pos);
+  c &= awd->mask;
+  if (awd->table)
+    return (awd->table[c]);
+  else
+    return (awd->L * pow (awd->B, (double) c));
+
+}
+
+void 
+FreeWeights (approx_weights_data * awd)
+{
+  Xfree (awd->DocWeights);
+  if (awd->table)
+    Xfree (awd->table);
+  Xfree (awd);
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/weights.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/weights.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/weights.h	(revision 699)
@@ -0,0 +1,34 @@
+/**************************************************************************
+ *
+ * weights.h -- Functions for reading the weights file in mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef H_WEIGHTS
+#define H_WEIGHTS
+
+approx_weights_data *LoadDocWeights (File * weight_file,
+				     unsigned long num_of_docs);
+
+float GetLowerApproxDocWeight (approx_weights_data * awd, register int DocNum);
+
+void FreeWeights (approx_weights_data * awd);
+
+#endif
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/words.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/words.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/words.c	(revision 699)
@@ -0,0 +1,11 @@
+#include "words.h"
+
+
+/* Takes the place of the old INAWORD macro. It determines
+   whether a given place in a UTF-8 encoded Unicode string
+   is part of a word. */
+int inaword (const u_char *here, const u_char *end) {
+  unsigned short c;
+  if (parse_utf8_char(here, end, &c) > 0) return is_unicode_letdig (c);
+  return 0;
+}
Index: /tags/gsdl-21/gsdl/packages/mg/src/text/words.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/src/text/words.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/src/text/words.h	(revision 699)
@@ -0,0 +1,387 @@
+/**************************************************************************
+ *
+ * words.h -- Macros for parsing out words from the source text
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "unitool.h"
+
+
+/*
+ * This has been cleaned up by Tim Shimmin. 
+ */
+
+/*
+ * ---NOTE---
+ *
+ * "WORD" refers to a word in the compressed text.
+ * "STEM" or "STEM_WORD" refers to a word for indexing on
+ *
+ */
+
+#define MAXWORDLEN	15
+	/* Maximum length in bytes of any word or non-word. Note that
+	   variations to MAXWORDLEN may have dramatic effects on the rest
+	   of the program, as the length and the prefix match are packed
+	   together into a four bit nibble, and there is not check that
+	   this is possible, i.e., leave MAXWORDLEN alone... */
+
+#define MAXSTEMLEN	255
+	/* Maximum length in bytes of any stem. Note that
+	   variations to MAXSTEMLEN may have dramatic effects on the rest
+	   of the program, , i.e., leave MAXSTEMLEN alone... */
+
+#define MAXNUMERIC	4
+	/* Maximum number of numeric characters permitted in a word.
+	   This avoids long sequences of numbers creating just one
+	   word occurrence for each number. At most 10,000 all numeric
+	   words will be permitted. */
+
+/* [RPAP - Jan 97: Stem Index Change] */
+#define MAXPARAMLEN     20
+        /* Maximum number of bytes to read for a parameter value for a
+	   term in a query. */
+#define WEIGHTPARAM     '/'
+#define STEMPARAM       '#'
+
+/* [RJM 07/97: Ranked Required Terms] */
+#define MUSTMATCHPARAM     '+'
+
+
+#define PESINAWORD(c)      (isalnum(c) || ((c) >= 0x80 && (c) <= 0xff))
+        /* The definition of what characters are permitted in a word.
+	   This macro is pessimistic, you cannot tell from a particular
+	   byte above 0x80 whether it is a character or not. This function
+	   is needed by various functions relating to huffman coding
+	   where frequency counts need to be primed, it should not be
+	   used in parsing the UTF-8 encoded input. */
+
+int inaword (const u_char *here, const u_char *end);
+        /* Takes the place of the old INAWORD macro. It determines
+	   whether a given place in a UTF-8 encoded Unicode string
+	   is part of a word. */
+
+
+/* =========================================================================
+ * Macro: PARSE_WORD
+ * Description: 
+ *      Extract a word out for compressing text
+ * Input: 
+ *      s_in = string start in buffer
+ *      end = string end in buffer
+ * Output: 
+ *      Word = extracted word with length in 1st byte
+ *      s_in = ptr to next character in buffer yet to be processed
+ * ========================================================================= */
+#define PARSE_WORD(Word, s_in, end)                                \
+  do {                                                             \
+    register int charlength = 0;                                   \
+    register u_char *wptr = (Word)+1;                              \
+    register int length = 0;                                       \
+    register int numeric = 0;                                      \
+    unsigned short c;                                              \
+                                                                   \
+    charlength = parse_utf8_char((s_in),(end),&c);                 \
+                                                                   \
+    while (length+charlength <= MAXWORDLEN && charlength > 0 &&    \
+	   (is_unicode_letter(c) || (is_unicode_digit(c) &&        \
+				     ++numeric <= MAXNUMERIC))) {  \
+      while (charlength-- > 0) {                                   \
+        *wptr++ = *(s_in)++; length++;                             \
+      }                                                            \
+      charlength = parse_utf8_char((s_in),(end),&c);               \
+    }                                                              \
+    *(Word) = length;                                              \
+  }while(0)
+
+  /*  
+#define PARSE_WORD(Word, s_in, end)                                  \
+  do {                                                               \
+	  register u_char  *wptr = (Word)+1;                         \
+	  register int    length = 0;                                \
+	  register int    c = *(s_in);                               \
+	  register int	numeric = 0;                                 \
+                                                                     \
+	  while( length < MAXWORDLEN && INAWORD(c) && (s_in)<=(end)) \
+	    {                                                        \
+	      if ((numeric += INNUMBER(c)) > MAXNUMERIC)             \
+		break;                                               \
+	      *wptr++ = c;                                           \
+	      length++;                                              \
+	      c = *++(s_in);                                         \
+	    }                                                        \
+	  *(Word) = length;                                          \
+  }while(0)
+  */
+
+
+/* =========================================================================
+ * Macro: PARSE_NON_WORD
+ * Description: 
+ *      Extract a non-word out for storing compressed text
+ * Input: as above
+ * Output: as above
+ * ========================================================================= */
+#define PARSE_NON_WORD(Word, s_in, end)                            \
+  do {                                                             \
+    register int charlength = 0;                                   \
+    register u_char *wptr = (Word)+1;                              \
+    register int length = 0;                                       \
+    unsigned short c;                                              \
+                                                                   \
+    charlength = parse_utf8_char((s_in),(end),&c);                 \
+                                                                   \
+    while (length+charlength <= MAXWORDLEN && charlength > 0 &&    \
+	   !is_unicode_letdig(c)) {                                \
+      while (charlength-- > 0) {                                   \
+        *wptr++ = *(s_in)++; length++;                             \
+      }                                                            \
+      charlength = parse_utf8_char((s_in),(end),&c);               \
+    }                                                              \
+    *(Word) = length;                                              \
+  }while(0)
+
+    /*
+#define PARSE_NON_WORD(Word, s_in, end)                            \
+  do {                                                             \
+	  register u_char  *wptr = (Word)+1;                       \
+	  register int    length = 0;                              \
+	  register int    c = *(s_in);                             \
+                                                                   \
+	  while( length < MAXWORDLEN && !INAWORD(c) && (s_in)<=(end) ) \
+	    {                                                      \
+	      *wptr++ = c;                                         \
+	      length++;                                            \
+	      c = *++(s_in);                                       \
+	    }                                                      \
+	  *(Word) = length;                                        \
+  }while(0)
+    */
+
+
+/* =========================================================================
+ * Macro: PARSE_STEM_WORD 
+ * Description: 
+ *      Extracts out Word.      
+ * Input: 
+ *      s_in points to 1st letter in buffer to test
+ *      end points to last letter in buffer
+ * Output: 
+ *      s_in is modified to move to next word
+ *      Returns Word filled in with length in 1st byte.
+ * ========================================================================= */
+#define PARSE_STEM_WORD(Word, s_in, end)                           \
+  do {                                                             \
+    register int charlength = 0;                                   \
+    register u_char *wptr = (Word)+1;                              \
+    register int length = 0;                                       \
+    register int numeric = 0;                                      \
+    unsigned short c;                                              \
+                                                                   \
+    charlength = parse_utf8_char((s_in),(end),&c);                 \
+                                                                   \
+    while (length+charlength <= MAXSTEMLEN && charlength > 0 &&    \
+	   (is_unicode_letter(c) || (is_unicode_digit(c) &&        \
+				     ++numeric <= MAXNUMERIC))) {  \
+      while (charlength-- > 0) {                                   \
+        *wptr++ = *(s_in)++; length++;                             \
+      }                                                            \
+      charlength = parse_utf8_char((s_in),(end),&c);               \
+    }                                                              \
+    *(Word) = length;                                              \
+  }while(0)
+
+    /*
+#define PARSE_STEM_WORD(Word, s_in, end)                      \
+  do                                                          \
+    {                                                         \
+      register u_char  *wptr = (Word)+1;                      \
+      register int    length = 0;                             \
+      register int    c = *(s_in);                            \
+      register int    numeric = 0;                            \
+                                                              \
+      while ( length < MAXSTEMLEN && INAWORD(c) && (s_in)<=(end)) \
+        {                                                     \
+ 	  if ((numeric += INNUMBER(c)) > MAXNUMERIC)          \
+	    break;                                            \
+	  *wptr++ = c;                                        \
+	  length++;                                           \
+	  c = *++(s_in);                                      \
+	}                                                     \
+      *(Word) = length;                                       \
+    }while(0)
+    */
+
+
+/* =========================================================================
+ * Macro: PARSE_NON_STEM_WORD 
+ * Description: 
+ *      Eat up non-word. Do not store non-word.
+ *      It is not needed in index only in text !
+ *      
+ * Input: as above but no Word needed
+ * Output: as above
+ * ========================================================================= */
+#define PARSE_NON_STEM_WORD(s_in, end)                             \
+  do {                                                             \
+    register int charlength = 0;                                   \
+    unsigned short c;                                              \
+                                                                   \
+    charlength = parse_utf8_char((s_in),(end),&c);                 \
+                                                                   \
+    while (charlength > 0 && !is_unicode_letdig(c)) {              \
+      (s_in) += charlength;                                        \
+      charlength = parse_utf8_char((s_in),(end),&c);               \
+    }                                                              \
+  }while(0)
+
+    /*
+#define PARSE_NON_STEM_WORD(s_in, end)           \
+  do                                             \
+    {                                            \
+      while (!INAWORD(*(s_in)) && (s_in)<=(end)) \
+	(s_in)++;                                \
+    }while(0)
+    */
+
+
+/* =========================================================================
+ * Macro: PARSE_NON_STEM_WORD_OR_SGML_TAG 
+ * Description: 
+ *      Like PARSE_NON_STEM_WORD but also eats up SGML tags
+ * Input: as above
+ * Output: as above
+ * ========================================================================= */
+#define PARSE_NON_STEM_WORD_OR_SGML_TAG(s_in, end)                 \
+  do {                                                             \
+    register int charlength = 0;                                   \
+    unsigned short c;                                              \
+                                                                   \
+    charlength = parse_utf8_char((s_in),(end),&c);                 \
+                                                                   \
+    while (charlength > 0 && !is_unicode_letdig(c)) {              \
+      if (c == '<') {                                              \
+	while (charlength > 0 && c != '>') {                       \
+	  (s_in) += charlength;                                    \
+	  charlength = parse_utf8_char((s_in),(end),&c);           \
+	}                                                          \
+      }                                                            \
+      (s_in) += charlength;                                        \
+      charlength = parse_utf8_char((s_in),(end),&c);               \
+    }                                                              \
+  }while(0)
+
+    /*
+#define PARSE_NON_STEM_WORD_OR_SGML_TAG(s_in, end) \
+  do                                               \
+    {                                              \
+      register int    c = *(s_in);                 \
+                                                   \
+      while (!INAWORD(c) && (s_in)<=(end))         \
+        {                                          \
+	  if (c == '<')                            \
+            {                                      \
+	      while (c != '>' && (s_in)<=(end))    \
+		c = *++(s_in);                     \
+            }                                      \
+	  if ((s_in)<=(end))                       \
+	    c = *++(s_in);                         \
+	}                                          \
+    }while(0)
+    */
+
+
+/* =========================================================================
+ * Macro: PARSE_OPT_TERM_PARAM     [RPAP - Jan 97: Stem Index Change]
+ * Description: 
+ *      Extracts out optional paramater for query term.
+ *      Needed only in parsing the query line !
+ *
+ *      Note: that this function has not been converted to use UTF-8
+ *            as it should still work as it is (only uses ascii
+ *            characters)
+ *      
+ * Input: as above but no Word needed
+ * Output: as above
+ * ========================================================================= */
+#define PARSE_OPT_TERM_PARAM(Param, type, s_in, end)                       \
+  do                                                                       \
+    {                                                                      \
+	  register u_char  *wptr = (Param);                                \
+	  register int    length = 0;                                      \
+	  register int    c = *(s_in);                                     \
+                                                                           \
+          if (c == WEIGHTPARAM || c == STEMPARAM)                          \
+	    {                                                              \
+	      type = c;                                                    \
+	      c = *++(s_in);                                               \
+	      while( length < MAXPARAMLEN && isdigit(c) && (s_in)<=(end))  \
+		{                                                          \
+	           *wptr++ = c;                                            \
+	           length++;                                               \
+	           c = *++(s_in);                                          \
+	        }                                                          \
+	      *wptr = '\0';                                                \
+              for (; isdigit(c) && (s_in)<=(end); c = *++(s_in))           \
+                ;                                                          \
+            }							           \
+    }while(0)
+
+/* =========================================================================
+ * Macro: PARSE_RANKED_NON_STEM_WORD    [RJM 07/97: Ranked Required Terms]
+ * Description: 
+ *      Eat up non-word. Do not store non-word.
+ *      If come across a match requirement store it in require_match
+ *      It is not needed in index only in text !
+ *      
+ * Input: as above
+ * Output: the requirement mode for the next term. -1=must not match,
+ *      0=optional match, 1=must match
+ * ========================================================================= */
+#define PARSE_RANKED_NON_STEM_WORD(require_match, s_in, end)       \
+  do {                                                             \
+    register int charlength = 0;                                   \
+    unsigned short c;                                              \
+    (require_match) = 0;                                           \
+                                                                   \
+    charlength = parse_utf8_char((s_in),(end),&c);                 \
+                                                                   \
+    while (charlength > 0 && !is_unicode_letdig(c)) {              \
+      if (c == MUSTMATCHPARAM) (require_match) = 1;                \
+      (s_in) += charlength;                                        \
+      charlength = parse_utf8_char((s_in),(end),&c);               \
+    }                                                              \
+  }while(0)
+
+    /*
+#define PARSE_RANKED_NON_STEM_WORD(require_match, s_in, end)         \
+  do {                                            \
+    (require_match) = 0;                          \
+    while (!INAWORD(*(s_in)) && (s_in)<=(end)) {  \
+      if (*(s_in) == MUSTMATCHPARAM) {            \
+        (require_match) = 1;                      \
+      }                                           \
+      (s_in)++;                                   \
+    }                                             \
+  } while (0)
+    */
+
Index: /tags/gsdl-21/gsdl/packages/mg/stamp-h.in
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/stamp-h.in	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/stamp-h.in	(revision 699)
@@ -0,0 +1,1 @@
+timestamp
Index: /tags/gsdl-21/gsdl/packages/mg/stemmerinterface.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/stemmerinterface.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/stemmerinterface.c	(revision 699)
@@ -0,0 +1,39 @@
+#include "lib/lovinstem.h"
+#include <stdio.h>
+#include <ctype.h>
+
+int main () {
+  char word[256];
+  char c;
+  int len;
+
+  c = fgetc (stdin);
+  while (!feof (stdin)) {
+    /* output non words */
+    while (!feof (stdin) && !isalnum (c)) {
+      fputc (c, stdout);
+      c = fgetc (stdin);
+    }
+
+    /* get the word */
+    len = 0;
+    while (!feof (stdin) && isalnum (c) && len < 255) {
+      len++;
+      word[len] = c;
+      c = fgetc (stdin);
+    }
+    word[0] = len;
+
+    /* if we found a word stem and output it */
+    if (len > 0) {
+      lovinstem (word);
+      len = 0;
+      while (len < word[0]) {
+        len++;
+        fputc(word[len], stdout);
+      }
+    }
+  }
+}
+
+
Index: /tags/gsdl-21/gsdl/packages/mg/sysfuncs.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/sysfuncs.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/sysfuncs.h	(revision 699)
@@ -0,0 +1,352 @@
+/* System dependent definitions for GNU tar.
+   Copyright (C) 1994 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+/*
+ * [TS:Aug/95]
+ * Based on code from system.h file in GNU's tar package 
+ *
+ */
+
+#ifndef SYSFUNCS_H
+#define SYSFUNCS_H
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef HAVE_CONFIG_H
+# ifdef __WIN32__
+#  include <WIN32cfg.h>
+# else
+#  include <config.h>
+# endif
+#endif
+
+/* Declare alloca.  AIX requires this to be the first thing in the file.  */
+
+#ifdef __GNUC__
+#define alloca __builtin_alloca
+#else
+#if HAVE_ALLOCA_H
+#include <alloca.h>
+#else
+#ifdef _AIX
+#pragma alloca
+#else
+#ifndef alloca
+char *alloca ();
+#endif
+#endif
+#endif
+#endif
+
+#include <sys/types.h>
+
+/* ------------------------------------------------- */
+/* [TS:Aug/95]
+ * Some extra include files that mg needs
+ */
+#include <stdlib.h>
+#include <math.h>
+
+#ifndef log2
+# ifndef M_LN2
+#  define M_LN2 0.69314718055994530942
+# endif
+# define log2(x) (log(x)/M_LN2)
+#endif
+
+
+#ifndef HAVE_SETBUFFER
+# define setbuffer(f, b, s)  setvbuf(f, b, _IOFBF, s)
+#endif
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#if defined(__MSDOS__)  && !defined(__WIN32__)
+# define SHORT_SUFFIX 1
+#endif
+
+
+
+#include <assert.h>
+
+/* ------------------------------------------------- */
+
+/* Declare a generic pointer type.  */
+#if __STDC__ || defined(__TURBOC__)
+#define voidstar void *
+#else
+#define voidstar char *
+#endif
+
+/* Declare ISASCII.  */
+
+#include <ctype.h>
+
+#if STDC_HEADERS
+#define ISASCII(Char) 1
+#else
+#ifdef isascii
+#define ISASCII(Char) isascii (Char)
+#else
+#if HAVE_ISASCII
+#define ISASCII(Char) isascii (Char)
+#else
+#define ISASCII(Char) 1
+#endif
+#endif
+#endif
+
+
+/* ------------------------------------------------- */
+/* [TS:Aug/95] 
+ * The following is copied from GNU Autoconf.Info (1.1)   
+ * It sets up the approriate string and memory functions.
+ */
+
+#if STDC_HEADERS || HAVE_STRING_H
+#include <string.h>
+   /* An ANSI string.h and pre-ANSI memory.h might conflict */
+#if !STDC_HEADERS && HAVE_MEMORY_H
+#include <memory.h>
+#endif /* not STDC_HEADERS and HAVE_MEMORY_H */
+#ifndef index
+#define index strchr
+#endif
+#ifndef rindex
+#define rindex strrchr
+#endif
+#ifndef bcopy
+#define bcopy(s, d, n) memcpy((d), (s), (n))
+#endif
+#ifndef bcmp
+#define bcmp(s1, s2, n) memcmp((s1), (s2), (n))
+#endif
+#ifndef bzero
+#define bzero(s, n) memset((s), 0, (n))
+#endif
+#else /* not STDC_HEADERS and not HAVE_STRING_H */
+#include <strings.h>
+   /* memory.h and strings.h conflict on some systems */
+#endif
+/* ------------------------------------------------- */
+
+
+/* Declare errno.  */
+
+#include <errno.h>
+#ifndef errno
+extern int errno;
+#endif
+
+/* Declare open parameters.  */
+
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#else
+#include <sys/file.h>
+#endif
+
+#ifndef	O_BINARY
+#define O_BINARY 0
+#endif
+#ifndef O_CREAT
+#define O_CREAT 0
+#endif
+#ifndef	O_NDELAY
+#define O_NDELAY 0
+#endif
+#ifndef	O_RDONLY
+#define O_RDONLY 0
+#endif
+#ifndef O_RDWR
+#define O_RDWR 2
+#endif
+
+/* Declare file status routines and bits.  */
+
+#include <sys/stat.h>
+
+#ifdef STAT_MACROS_BROKEN
+#undef S_ISBLK
+#undef S_ISCHR
+#undef S_ISDIR
+#undef S_ISFIFO
+#undef S_ISLNK
+#undef S_ISMPB
+#undef S_ISMPC
+#undef S_ISNWK
+#undef S_ISREG
+#undef S_ISSOCK
+#endif
+
+/* On MSDOS, there are missing things from <sys/stat.h>.  */
+#ifdef __MSDOS__
+#define S_ISUID 0
+#define S_ISGID 0
+#define S_ISVTX 0
+#endif
+
+#ifndef S_ISREG			/* POSIX.1 stat stuff missing */
+#define mode_t unsigned short
+#endif
+#if !defined(S_ISBLK) && defined(S_IFBLK)
+#define S_ISBLK(Mode) (((Mode) & S_IFMT) == S_IFBLK)
+#endif
+#if !defined(S_ISCHR) && defined(S_IFCHR)
+#define S_ISCHR(Mode) (((Mode) & S_IFMT) == S_IFCHR)
+#endif
+#if !defined(S_ISDIR) && defined(S_IFDIR)
+#define S_ISDIR(Mode) (((Mode) & S_IFMT) == S_IFDIR)
+#endif
+#if !defined(S_ISREG) && defined(S_IFREG)
+#define S_ISREG(Mode) (((Mode) & S_IFMT) == S_IFREG)
+#endif
+#if !defined(S_ISFIFO) && defined(S_IFIFO)
+#define S_ISFIFO(Mode) (((Mode) & S_IFMT) == S_IFIFO)
+#endif
+#if !defined(S_ISLNK) && defined(S_IFLNK)
+#define S_ISLNK(Mode) (((Mode) & S_IFMT) == S_IFLNK)
+#endif
+#if !defined(S_ISSOCK) && defined(S_IFSOCK)
+#define S_ISSOCK(Mode) (((Mode) & S_IFMT) == S_IFSOCK)
+#endif
+#if !defined(S_ISMPB) && defined(S_IFMPB)	/* V7 */
+#define S_ISMPB(Mode) (((Mode) & S_IFMT) == S_IFMPB)
+#define S_ISMPC(Mode) (((Mode) & S_IFMT) == S_IFMPC)
+#endif
+#if !defined(S_ISNWK) && defined(S_IFNWK)	/* HP/UX */
+#define S_ISNWK(Mode) (((Mode) & S_IFMT) == S_IFNWK)
+#endif
+
+#if !defined(S_ISCTG) && defined(S_IFCTG)	/* contiguous file */
+#define S_ISCTG(Mode) (((Mode) & S_IFMT) == S_IFCTG)
+#endif
+#if !defined(S_ISVTX)
+#define S_ISVTX 0001000
+#endif
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#if !defined(_POSIX_SOURCE) && !defined(__WIN32__)
+#include <sys/param.h>
+#endif
+
+/* Include <unistd.h> before any preprocessor test of _POSIX_VERSION.  */
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+
+/* Declare standard functions.  */
+
+#ifdef STDC_HEADERS
+#include <stdlib.h>
+#else
+voidstar malloc ();
+voidstar realloc ();
+char *getenv ();
+#endif
+
+#include <stdio.h>
+
+#ifndef _POSIX_VERSION
+#ifdef __MSDOS__
+#include <io.h>
+#else
+off_t lseek ();
+#endif
+#endif
+
+#include <pathmax.h>
+
+/* Until getoptold function is declared in getopt.h, we need this here for
+   struct option.  */
+#include <getopt.h>
+
+#ifdef WITH_DMALLOC
+#undef HAVE_VALLOC
+#define DMALLOC_FUNC_CHECK
+#include <dmalloc.h>
+#endif
+
+/* Prototypes for external functions.  */
+
+#ifndef __P
+#if PROTOTYPES
+#define __P(Args) Args
+#else
+#define __P(Args) ()
+#endif
+#endif
+
+#if HAVE_LOCALE_H
+#include <locale.h>
+#endif
+#if !HAVE_SETLOCALE
+#define setlocale(Category, Locale)
+#endif
+
+#if ENABLE_NLS
+#include <libintl.h>
+#define _(Text) gettext (Text)
+#else
+#define textdomain(Domain)
+#define _(Text) Text
+#endif
+
+/* Library modules.  */
+
+#ifdef HAVE_VPRINTF
+void error __P ((int, int, const char *,...));
+#else
+void error ();
+#endif
+
+#ifndef HAVE_STRSTR
+char *strstr __P ((const char *, const char *));
+#endif
+
+/* [RB/TS:Oct/95] commented out. Used by lib/gmalloc.c - but it defines it anyway ???
+ *#ifndef HAVE_VALLOC
+ *#define valloc(Size) malloc (Size)
+ *#endif
+ */
+
+voidstar xmalloc __P ((size_t));
+voidstar xrealloc __P ((voidstar, size_t));
+char *xstrdup __P ((const char *));
+
+/* [TS:Aug/95] 
+ * These were required by files which referred 
+ * to these as function pointers. 
+ * e.g. huffman.c - fread,  mgfelics.c - fgetc
+ */
+#ifndef HAVE_FREAD_DECL
+extern size_t   fread __P ((void *, size_t, size_t, FILE *));
+#endif
+#ifndef HAVE_FGETC_DECL
+extern int      fgetc __P ((FILE *));
+#endif
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifndef u_long
+# define u_long unsigned long
+#endif
+#ifndef u_char
+# define u_char unsigned char
+#endif
+
+
+#endif /* SYSFUNCS_H */
Index: /tags/gsdl-21/gsdl/packages/mg/test/Makefile
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/test/Makefile	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/test/Makefile	(revision 699)
@@ -0,0 +1,10 @@
+DISTFILES = Makefile q1 q2 q3 r1 r2 r3 mgrc1 mgrc2 mgrc3 testmg \
+            alice.sum simple_sum.c alice.len
+
+all: simple_sum
+
+dist:
+	ln $(DISTFILES) $(distdir)
+
+simple_sum: simple_sum.c
+	cc -o $@ $?
Index: /tags/gsdl-21/gsdl/packages/mg/test/alice.len
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/test/alice.len	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/test/alice.len	(revision 699)
@@ -0,0 +1,2 @@
+./mgdata/alice/alice.invf.dict.hash: 13256
+./mgdata/alice/alice.text.dict: 19321
Index: /tags/gsdl-21/gsdl/packages/mg/test/alice.sum
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/test/alice.sum	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/test/alice.sum	(revision 699)
@@ -0,0 +1,8 @@
+./mgdata/alice/alice.invf: 2618428 19209
+./mgdata/alice/alice.invf.chunk: 189460 2134
+./mgdata/alice/alice.invf.chunk.trans: 331998 2585
+./mgdata/alice/alice.invf.dict.blocked: 1030626 33214
+./mgdata/alice/alice.invf.idx: 309022 7572
+./mgdata/alice/alice.text: 5075885 40099
+./mgdata/alice/alice.text.idx: 185535 3380
+./mgdata/alice/alice.text.stats: 2432380 49146
Index: /tags/gsdl-21/gsdl/packages/mg/test/mgrc1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/test/mgrc1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/test/mgrc1	(revision 699)
@@ -0,0 +1,2 @@
+.set mode text
+.set query boolean
Index: /tags/gsdl-21/gsdl/packages/mg/test/mgrc2
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/test/mgrc2	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/test/mgrc2	(revision 699)
@@ -0,0 +1,2 @@
+.set mode text
+.set query ranked
Index: /tags/gsdl-21/gsdl/packages/mg/test/mgrc3
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/test/mgrc3	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/test/mgrc3	(revision 699)
@@ -0,0 +1,2 @@
+.set mode docnums
+.set query boolean
Index: /tags/gsdl-21/gsdl/packages/mg/test/q1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/test/q1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/test/q1	(revision 699)
@@ -0,0 +1,1 @@
+flamingo
Index: /tags/gsdl-21/gsdl/packages/mg/test/q2
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/test/q2	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/test/q2	(revision 699)
@@ -0,0 +1,1 @@
+flamingo
Index: /tags/gsdl-21/gsdl/packages/mg/test/q3
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/test/q3	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/test/q3	(revision 699)
@@ -0,0 +1,2 @@
+Rabbit & !(Alice | she)
+laser
Index: /tags/gsdl-21/gsdl/packages/mg/test/r1
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/test/r1	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/test/r1	(revision 699)
@@ -0,0 +1,46 @@
+> flamingo
+flamingo
+---------------------------------- 482
+  The chief difficulty Alice found at first was in managing her 
+flamingo:  she succeeded in getting its body tucked away, 
+comfortably enough, under her arm, with its legs hanging down, 
+but generally, just as she had got its neck nicely straightened 
+out, and was going to give the hedgehog a blow with its head, it 
+WOULD twist itself round and look up in her face, with such a 
+puzzled expression that she could not help bursting out laughing:  
+and when she had got its head down, and was going to begin again, 
+it was very provoking to find that the hedgehog had unrolled 
+itself, and was in the act of crawling away:  besides all this, 
+there was generally a ridge or furrow in the way wherever she 
+wanted to send the hedgehog to, and, as the doubled-up soldiers 
+were always getting up and walking off to other parts of the 
+ground, Alice soon came to the conclusion that it was a very 
+difficult game indeed. 
+---------------------------------- 487
+  Alice waited till the eyes appeared, and then nodded.  `It's no 
+use speaking to it,' she thought, `till its ears have come, or at 
+least one of them.'  In another minute the whole head appeared, 
+and then Alice put down her flamingo, and began an account of the 
+game, feeling very glad she had someone to listen to her.  The 
+Cat seemed to think that there was enough of it now in sight, and 
+no more of it appeared. 
+---------------------------------- 502
+  The hedgehog was engaged in a fight with another hedgehog, 
+which seemed to Alice an excellent opportunity for croqueting one 
+of them with the other:  the only difficulty was, that her 
+flamingo was gone across to the other side of the garden, where 
+Alice could see it trying in a helpless sort of way to fly up 
+into a tree. 
+---------------------------------- 503
+  By the time she had caught the flamingo and brought it back, 
+the fight was over, and both the hedgehogs were out of sight:  
+`but it doesn't matter much,' thought Alice, `as all the arches 
+are gone from the side of the ground.'  So she tucked it away 
+under her arm, that it might not escape again, and went back for 
+a little more conversation with her friend. 
+---------------------------------- 526
+  `I dare say you're wondering why I don't put my arm round your 
+waist,' the Duchess said after a pause:  `the reason is, that I'm 
+doubtful about the temper of your flamingo.  Shall I try the 
+experiment?' 
+5 documents retrieved.
Index: /tags/gsdl-21/gsdl/packages/mg/test/r2
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/test/r2	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/test/r2	(revision 699)
@@ -0,0 +1,46 @@
+> flamingo
+flamingo
+---------------------------------- 526 1.322623
+  `I dare say you're wondering why I don't put my arm round your 
+waist,' the Duchess said after a pause:  `the reason is, that I'm 
+doubtful about the temper of your flamingo.  Shall I try the 
+experiment?' 
+---------------------------------- 503 1.013693
+  By the time she had caught the flamingo and brought it back, 
+the fight was over, and both the hedgehogs were out of sight:  
+`but it doesn't matter much,' thought Alice, `as all the arches 
+are gone from the side of the ground.'  So she tucked it away 
+under her arm, that it might not escape again, and went back for 
+a little more conversation with her friend. 
+---------------------------------- 502 1.009519
+  The hedgehog was engaged in a fight with another hedgehog, 
+which seemed to Alice an excellent opportunity for croqueting one 
+of them with the other:  the only difficulty was, that her 
+flamingo was gone across to the other side of the garden, where 
+Alice could see it trying in a helpless sort of way to fly up 
+into a tree. 
+---------------------------------- 487 0.882496
+  Alice waited till the eyes appeared, and then nodded.  `It's no 
+use speaking to it,' she thought, `till its ears have come, or at 
+least one of them.'  In another minute the whole head appeared, 
+and then Alice put down her flamingo, and began an account of the 
+game, feeling very glad she had someone to listen to her.  The 
+Cat seemed to think that there was enough of it now in sight, and 
+no more of it appeared. 
+---------------------------------- 482 0.529432
+  The chief difficulty Alice found at first was in managing her 
+flamingo:  she succeeded in getting its body tucked away, 
+comfortably enough, under her arm, with its legs hanging down, 
+but generally, just as she had got its neck nicely straightened 
+out, and was going to give the hedgehog a blow with its head, it 
+WOULD twist itself round and look up in her face, with such a 
+puzzled expression that she could not help bursting out laughing:  
+and when she had got its head down, and was going to begin again, 
+it was very provoking to find that the hedgehog had unrolled 
+itself, and was in the act of crawling away:  besides all this, 
+there was generally a ridge or furrow in the way wherever she 
+wanted to send the hedgehog to, and, as the doubled-up soldiers 
+were always getting up and walking off to other parts of the 
+ground, Alice soon came to the conclusion that it was a very 
+difficult game indeed. 
+5 documents retrieved.
Index: /tags/gsdl-21/gsdl/packages/mg/test/r3
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/test/r3	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/test/r3	(revision 699)
@@ -0,0 +1,21 @@
+> Rabbit & !(Alice | she)
+Rabbit Alice she
+     14.14     
+    126.126    
+    143.143    
+    478.478    
+    701.701    
+    704.704    
+    705.705    
+    755.755    
+    774.774    
+    786.786    
+    788.788    
+    791.791    
+    793.793    
+    801.801    
+    803.803    
+15 documents retrieved.
+> laser
+laser
+No entries correspond to that query.
Index: /tags/gsdl-21/gsdl/packages/mg/test/simple_sum.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/mg/test/simple_sum.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/mg/test/simple_sum.c	(revision 699)
@@ -0,0 +1,100 @@
+/**************************************************************************
+ *
+ * sum.c -- byte summing program
+ * Copyright (C) 1995  Tim Shimmin
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+$Log$
+Revision 1.1  1999/08/10 21:18:33  sjboddie
+renamed mg-1.3d directory mg
+
+Revision 1.1  1998/11/17 09:36:00  rjmcnab
+*** empty log message ***
+
+*/
+
+/*
+ * If given -l option then just prints the length.
+ * Normally prints the sum and the length for the file.
+ */
+
+static int length_only = 0;
+
+#include <stdio.h>
+
+void 
+output_sum(fname, input_file, output_file)
+char *fname;
+FILE *input_file;
+FILE *output_file;
+{
+  int ch = '\0';
+  unsigned long sum = 0;
+  unsigned long num_bytes = 0;
+
+  while((ch = getc(input_file))!=EOF){
+    sum += ch;
+    num_bytes++;
+  }
+
+  if (length_only)
+    fprintf(output_file, "%s: %ld\n", fname, num_bytes);
+  else
+    fprintf(output_file, "%s: %ld %ld\n", fname, sum, num_bytes);
+}
+
+void
+main(argc, argv)
+int argc;
+char *argv[];
+{
+  int i = 0;
+
+  /* use stdin */
+  if (argc == 1){
+    output_sum("-", stdin, stdout);
+    exit(0);
+  }
+
+  /* use args as file names for input */
+  for (i=1;i<argc; i++){
+    char *fname = argv[i];
+    FILE *input_file = NULL;
+    
+    if (!length_only && (argv[i][0] == '-') && (argv[i][1] =='l') ) 
+      { 
+        length_only = 1; 
+        continue;
+      }
+
+    input_file = fopen(fname, "r");
+    if (!input_file){
+      fprintf(stderr,"Could not open %s\n", fname);
+      continue;
+    }
+    else{
+      output_sum(fname, input_file, stdout);
+      fclose(input_file);
+    }
+  }/*for*/
+  exit(0);
+
+}
Index: /tags/gsdl-21/gsdl/packages/wingdbm/Makefile.in
===================================================================
--- /tags/gsdl-21/gsdl/packages/wingdbm/Makefile.in	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/wingdbm/Makefile.in	(revision 699)
@@ -0,0 +1,84 @@
+
+srcdir = c:\home\rjmcnab\humanity\gdbm
+VPATH = $(srcdir)
+
+AR = lib
+CC = cl
+#CFLAGS = -g -O
+CPPFLAGS =
+DEFS = -DHAVE_CONFIG_H -D__WIN32__ -D_LITTLE_ENDIAN
+LDFLAGS =
+
+o = .obj
+
+.SUFFIXES:
+.SUFFIXES: .c .$o
+
+.c.$o:
+	$(CC) -c $(CPPFLAGS) $(DEFS) -I. -I$(srcdir) $(CFLAGS) $<
+
+
+PROGS = gdbm.lib testgdbm.exe
+
+GDBM_CF = gdbmopen.c gdbmdelete.c gdbmfetch.c  gdbmstore.c gdbmclose.c \
+	gdbmreorg.c gdbmseq.c gdbmsync.c gdbmerrno.c gdbmexists.c gdbmsetopt.c \
+	bucket.c falloc.c findkey.c global.c hash.c update.c \
+	version.c
+
+TEST_CF = testgdbm.c
+
+GDBM_OF = gdbmopen.$o gdbmdelete.$o gdbmfetch.$o  gdbmstore.$o gdbmclose.$o \
+	gdbmreorg.$o gdbmseq.$o gdbmsync.$o gdbmerrno.$o gdbmexists.$o gdbmsetopt.$o \
+	bucket.$o falloc.$o findkey.$o global.$o hash.$o update.$o \
+	version.$o
+     
+SRCS = $(GDBM_CF) $(TEST_CF)
+
+OBJS = $(GDBM_OF)
+
+HDRS = gdbmdefs.h extern.h gdbmerrno.h systems.h gdbmconst.h \
+	proto.h
+
+all: gdbm.lib
+
+progs: $(PROGS)
+
+gdbm.lib: $(OBJS) gdbm.h
+	del gdbm.lib
+	$(AR) /out:gdbm.lib $(OBJS) 
+
+testgdbm.exe: testgdbm.$o gdbm.lib
+	$(CC) $(LDFLAGS) /Fe"testgdbm.exe" testgdbm.$o gdbm.lib
+
+
+clean:
+	rm -f $(PROGS) *.$o core junk*
+
+
+# gdbm files
+gdbmclose.$o:	autoconf.h gdbmdefs.h 
+gdbmdelete.$o:	autoconf.h gdbmdefs.h gdbmerrno.h 
+gdbmfetch.$o:	autoconf.h gdbmdefs.h gdbmerrno.h 
+gdbmopen.$o:	autoconf.h gdbmdefs.h gdbmerrno.h 
+gdbmreorg.$o:	autoconf.h gdbmdefs.h gdbmerrno.h extern.h
+gdbmseq.$o:	autoconf.h gdbmdefs.h 
+gdbmstore.$o:	autoconf.h gdbmdefs.h gdbmerrno.h 
+
+# gdbm support files
+bucket.$o:	autoconf.h gdbmdefs.h
+falloc.$o:	autoconf.h gdbmdefs.h
+findkey.$o:	autoconf.h gdbmdefs.h
+global.$o:	autoconf.h gdbmdefs.h gdbmerrno.h 
+hash.$o:		autoconf.h gdbmdefs.h
+update.$o:	autoconf.h gdbmdefs.h
+version.$o:
+extern.h:
+gdbmdefs.h:	gdbmconst.h systems.h
+	touch gdbmdefs.h
+
+# other programs
+testgdbm.$o:	autoconf.h gdbmdefs.h extern.h gdbmerrno.h systems.h
+testdbm.$o:	autoconf.h
+testndbm.$o:	autoconf.h ndbm.h
+tdbm.$o:		autoconf.h
+conv2gdbm.$o:	autoconf.h gdbm.h
Index: /tags/gsdl-21/gsdl/packages/wingdbm/alloca.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/wingdbm/alloca.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/wingdbm/alloca.c	(revision 699)
@@ -0,0 +1,465 @@
+/* alloca.c -- allocate automatically reclaimed memory
+   (Mostly) portable public-domain implementation -- D A Gwyn
+
+   This implementation of the PWB library alloca function,
+   which is used to allocate space off the run-time stack so
+   that it is automatically reclaimed upon procedure exit,
+   was inspired by discussions with J. Q. Johnson of Cornell.
+   J.Otto Tennant <jot@cray.com> contributed the Cray support.
+
+   There are some preprocessor constants that can
+   be defined when compiling for your specific system, for
+   improved efficiency; however, the defaults should be okay.
+
+   The general concept of this implementation is to keep
+   track of all alloca-allocated blocks, and reclaim any
+   that are found to be deeper in the stack than the current
+   invocation.  This heuristic does not reclaim storage as
+   soon as it becomes invalid, but it will do so eventually.
+
+   As a special case, alloca(0) reclaims storage without
+   allocating any.  It is a good idea to use alloca(0) in
+   your main control loop, etc. to force garbage collection.  */
+
+/* include system configuration before all else. */
+#include "autoconf.h"
+
+#if HAVE_STDLIB_H
+#include <stdlib.h>
+#endif
+
+/* If compiling with GCC, this file's not needed.  */
+#ifndef alloca
+
+#ifdef emacs
+#ifdef static
+/* actually, only want this if static is defined as ""
+   -- this is for usg, in which emacs must undefine static
+   in order to make unexec workable
+   */
+#ifndef STACK_DIRECTION
+you
+lose
+-- must know STACK_DIRECTION at compile-time
+#endif /* STACK_DIRECTION undefined */
+#endif /* static */
+#endif /* emacs */
+
+/* If your stack is a linked list of frames, you have to
+   provide an "address metric" ADDRESS_FUNCTION macro.  */
+
+#ifdef CRAY
+long i00afunc ();
+#define ADDRESS_FUNCTION(arg) (char *) i00afunc (&(arg))
+#else
+#define ADDRESS_FUNCTION(arg) &(arg)
+#endif
+
+#if __STDC__
+typedef void *pointer;
+#else
+typedef char *pointer;
+#endif
+
+#ifndef NULL
+#define	NULL	0
+#endif
+
+/* Define STACK_DIRECTION if you know the direction of stack
+   growth for your system; otherwise it will be automatically
+   deduced at run-time.
+
+   STACK_DIRECTION > 0 => grows toward higher addresses
+   STACK_DIRECTION < 0 => grows toward lower addresses
+   STACK_DIRECTION = 0 => direction of growth unknown  */
+
+#ifndef STACK_DIRECTION
+#define	STACK_DIRECTION	0	/* Direction unknown.  */
+#endif
+
+#if STACK_DIRECTION != 0
+
+#define	STACK_DIR	STACK_DIRECTION	/* Known at compile-time.  */
+
+#else /* STACK_DIRECTION == 0; need run-time code.  */
+
+static int stack_dir;		/* 1 or -1 once known.  */
+#define	STACK_DIR	stack_dir
+
+static void
+find_stack_direction ()
+{
+  static char *addr = NULL;	/* Address of first `dummy', once known.  */
+  auto char dummy;		/* To get stack address.  */
+
+  if (addr == NULL)
+    {				/* Initial entry.  */
+      addr = ADDRESS_FUNCTION (dummy);
+
+      find_stack_direction ();	/* Recurse once.  */
+    }
+  else
+    {
+      /* Second entry.  */
+      if (ADDRESS_FUNCTION (dummy) > addr)
+	stack_dir = 1;		/* Stack grew upward.  */
+      else
+	stack_dir = -1;		/* Stack grew downward.  */
+    }
+}
+
+#endif /* STACK_DIRECTION == 0 */
+
+/* An "alloca header" is used to:
+   (a) chain together all alloca'ed blocks;
+   (b) keep track of stack depth.
+
+   It is very important that sizeof(header) agree with malloc
+   alignment chunk size.  The following default should work okay.  */
+
+#ifndef	ALIGN_SIZE
+#define	ALIGN_SIZE	sizeof(double)
+#endif
+
+typedef union hdr
+{
+  char align[ALIGN_SIZE];	/* To force sizeof(header).  */
+  struct
+    {
+      union hdr *next;		/* For chaining headers.  */
+      char *deep;		/* For stack depth measure.  */
+    } h;
+} header;
+
+static header *last_alloca_header = NULL;	/* -> last alloca header.  */
+
+/* Return a pointer to at least SIZE bytes of storage,
+   which will be automatically reclaimed upon exit from
+   the procedure that called alloca.  Originally, this space
+   was supposed to be taken from the current stack frame of the
+   caller, but that method cannot be made to work for some
+   implementations of C, for example under Gould's UTX/32.  */
+
+pointer
+alloca (size)
+     unsigned size;
+{
+  auto char probe;		/* Probes stack depth: */
+  register char *depth = ADDRESS_FUNCTION (probe);
+
+#if STACK_DIRECTION == 0
+  if (STACK_DIR == 0)		/* Unknown growth direction.  */
+    find_stack_direction ();
+#endif
+
+  /* Reclaim garbage, defined as all alloca'd storage that
+     was allocated from deeper in the stack than currently. */
+
+  {
+    register header *hp;	/* Traverses linked list.  */
+
+    for (hp = last_alloca_header; hp != NULL;)
+      if ((STACK_DIR > 0 && hp->h.deep > depth)
+	  || (STACK_DIR < 0 && hp->h.deep < depth))
+	{
+	  register header *np = hp->h.next;
+
+	  free ((pointer) hp);	/* Collect garbage.  */
+
+	  hp = np;		/* -> next header.  */
+	}
+      else
+	break;			/* Rest are not deeper.  */
+
+    last_alloca_header = hp;	/* -> last valid storage.  */
+  }
+
+  if (size == 0)
+    return NULL;		/* No allocation required.  */
+
+  /* Allocate combined header + user data storage.  */
+
+  {
+    register pointer new = malloc (sizeof (header) + size);
+    /* Address of header.  */
+
+    ((header *) new)->h.next = last_alloca_header;
+    ((header *) new)->h.deep = depth;
+
+    last_alloca_header = (header *) new;
+
+    /* User storage begins just after header.  */
+
+    return (pointer) ((char *) new + sizeof (header));
+  }
+}
+
+#ifdef CRAY
+
+#ifdef DEBUG_I00AFUNC
+#include <stdio.h>
+#endif
+
+#ifndef CRAY_STACK
+#define CRAY_STACK
+#ifndef CRAY2
+/* Stack structures for CRAY-1, CRAY X-MP, and CRAY Y-MP */
+struct stack_control_header
+  {
+    long shgrow:32;		/* Number of times stack has grown.  */
+    long shaseg:32;		/* Size of increments to stack.  */
+    long shhwm:32;		/* High water mark of stack.  */
+    long shsize:32;		/* Current size of stack (all segments).  */
+  };
+
+/* The stack segment linkage control information occurs at
+   the high-address end of a stack segment.  (The stack
+   grows from low addresses to high addresses.)  The initial
+   part of the stack segment linkage control information is
+   0200 (octal) words.  This provides for register storage
+   for the routine which overflows the stack.  */
+
+struct stack_segment_linkage
+  {
+    long ss[0200];		/* 0200 overflow words.  */
+    long sssize:32;		/* Number of words in this segment.  */
+    long ssbase:32;		/* Offset to stack base.  */
+    long:32;
+    long sspseg:32;		/* Offset to linkage control of previous
+				   segment of stack.  */
+    long:32;
+    long sstcpt:32;		/* Pointer to task common address block.  */
+    long sscsnm;		/* Private control structure number for
+				   microtasking.  */
+    long ssusr1;		/* Reserved for user.  */
+    long ssusr2;		/* Reserved for user.  */
+    long sstpid;		/* Process ID for pid based multi-tasking.  */
+    long ssgvup;		/* Pointer to multitasking thread giveup.  */
+    long sscray[7];		/* Reserved for Cray Research.  */
+    long ssa0;
+    long ssa1;
+    long ssa2;
+    long ssa3;
+    long ssa4;
+    long ssa5;
+    long ssa6;
+    long ssa7;
+    long sss0;
+    long sss1;
+    long sss2;
+    long sss3;
+    long sss4;
+    long sss5;
+    long sss6;
+    long sss7;
+  };
+
+#else /* CRAY2 */
+/* The following structure defines the vector of words
+   returned by the STKSTAT library routine.  */
+struct stk_stat
+  {
+    long now;			/* Current total stack size.  */
+    long maxc;			/* Amount of contiguous space which would
+				   be required to satisfy the maximum
+				   stack demand to date.  */
+    long high_water;		/* Stack high-water mark.  */
+    long overflows;		/* Number of stack overflow ($STKOFEN) calls.  */
+    long hits;			/* Number of internal buffer hits.  */
+    long extends;		/* Number of block extensions.  */
+    long stko_mallocs;		/* Block allocations by $STKOFEN.  */
+    long underflows;		/* Number of stack underflow calls ($STKRETN).  */
+    long stko_free;		/* Number of deallocations by $STKRETN.  */
+    long stkm_free;		/* Number of deallocations by $STKMRET.  */
+    long segments;		/* Current number of stack segments.  */
+    long maxs;			/* Maximum number of stack segments so far.  */
+    long pad_size;		/* Stack pad size.  */
+    long current_address;	/* Current stack segment address.  */
+    long current_size;		/* Current stack segment size.  This
+				   number is actually corrupted by STKSTAT to
+				   include the fifteen word trailer area.  */
+    long initial_address;	/* Address of initial segment.  */
+    long initial_size;		/* Size of initial segment.  */
+  };
+
+/* The following structure describes the data structure which trails
+   any stack segment.  I think that the description in 'asdef' is
+   out of date.  I only describe the parts that I am sure about.  */
+
+struct stk_trailer
+  {
+    long this_address;		/* Address of this block.  */
+    long this_size;		/* Size of this block (does not include
+				   this trailer).  */
+    long unknown2;
+    long unknown3;
+    long link;			/* Address of trailer block of previous
+				   segment.  */
+    long unknown5;
+    long unknown6;
+    long unknown7;
+    long unknown8;
+    long unknown9;
+    long unknown10;
+    long unknown11;
+    long unknown12;
+    long unknown13;
+    long unknown14;
+  };
+
+#endif /* CRAY2 */
+#endif /* not CRAY_STACK */
+
+#ifdef CRAY2
+/* Determine a "stack measure" for an arbitrary ADDRESS.
+   I doubt that "lint" will like this much. */
+
+static long
+i00afunc (long *address)
+{
+  struct stk_stat status;
+  struct stk_trailer *trailer;
+  long *block, size;
+  long result = 0;
+
+  /* We want to iterate through all of the segments.  The first
+     step is to get the stack status structure.  We could do this
+     more quickly and more directly, perhaps, by referencing the
+     $LM00 common block, but I know that this works.  */
+
+  STKSTAT (&status);
+
+  /* Set up the iteration.  */
+
+  trailer = (struct stk_trailer *) (status.current_address
+				    + status.current_size
+				    - 15);
+
+  /* There must be at least one stack segment.  Therefore it is
+     a fatal error if "trailer" is null.  */
+
+  if (trailer == 0)
+    abort ();
+
+  /* Discard segments that do not contain our argument address.  */
+
+  while (trailer != 0)
+    {
+      block = (long *) trailer->this_address;
+      size = trailer->this_size;
+      if (block == 0 || size == 0)
+	abort ();
+      trailer = (struct stk_trailer *) trailer->link;
+      if ((block <= address) && (address < (block + size)))
+	break;
+    }
+
+  /* Set the result to the offset in this segment and add the sizes
+     of all predecessor segments.  */
+
+  result = address - block;
+
+  if (trailer == 0)
+    {
+      return result;
+    }
+
+  do
+    {
+      if (trailer->this_size <= 0)
+	abort ();
+      result += trailer->this_size;
+      trailer = (struct stk_trailer *) trailer->link;
+    }
+  while (trailer != 0);
+
+  /* We are done.  Note that if you present a bogus address (one
+     not in any segment), you will get a different number back, formed
+     from subtracting the address of the first block.  This is probably
+     not what you want.  */
+
+  return (result);
+}
+
+#else /* not CRAY2 */
+/* Stack address function for a CRAY-1, CRAY X-MP, or CRAY Y-MP.
+   Determine the number of the cell within the stack,
+   given the address of the cell.  The purpose of this
+   routine is to linearize, in some sense, stack addresses
+   for alloca.  */
+
+static long
+i00afunc (long address)
+{
+  long stkl = 0;
+
+  long size, pseg, this_segment, stack;
+  long result = 0;
+
+  struct stack_segment_linkage *ssptr;
+
+  /* Register B67 contains the address of the end of the
+     current stack segment.  If you (as a subprogram) store
+     your registers on the stack and find that you are past
+     the contents of B67, you have overflowed the segment.
+
+     B67 also points to the stack segment linkage control
+     area, which is what we are really interested in.  */
+
+  stkl = CRAY_STACKSEG_END ();
+  ssptr = (struct stack_segment_linkage *) stkl;
+
+  /* If one subtracts 'size' from the end of the segment,
+     one has the address of the first word of the segment.
+
+     If this is not the first segment, 'pseg' will be
+     nonzero.  */
+
+  pseg = ssptr->sspseg;
+  size = ssptr->sssize;
+
+  this_segment = stkl - size;
+
+  /* It is possible that calling this routine itself caused
+     a stack overflow.  Discard stack segments which do not
+     contain the target address.  */
+
+  while (!(this_segment <= address && address <= stkl))
+    {
+#ifdef DEBUG_I00AFUNC
+      fprintf (stderr, "%011o %011o %011o\n", this_segment, address, stkl);
+#endif
+      if (pseg == 0)
+	break;
+      stkl = stkl - pseg;
+      ssptr = (struct stack_segment_linkage *) stkl;
+      size = ssptr->sssize;
+      pseg = ssptr->sspseg;
+      this_segment = stkl - size;
+    }
+
+  result = address - this_segment;
+
+  /* If you subtract pseg from the current end of the stack,
+     you get the address of the previous stack segment's end.
+     This seems a little convoluted to me, but I'll bet you save
+     a cycle somewhere.  */
+
+  while (pseg != 0)
+    {
+#ifdef DEBUG_I00AFUNC
+      fprintf (stderr, "%011o %011o\n", pseg, size);
+#endif
+      stkl = stkl - pseg;
+      ssptr = (struct stack_segment_linkage *) stkl;
+      size = ssptr->sssize;
+      pseg = ssptr->sspseg;
+      result += size;
+    }
+  return (result);
+}
+
+#endif /* not CRAY2 */
+#endif /* CRAY */
+
+#endif /* no alloca */
Index: /tags/gsdl-21/gsdl/packages/wingdbm/autoconf.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/wingdbm/autoconf.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/wingdbm/autoconf.h	(revision 699)
@@ -0,0 +1,85 @@
+/* autoconf.h.in.  Generated automatically from configure.in by autoheader.  */
+
+#define MSDOS
+
+/* Define if using alloca.c.  */
+#undef C_ALLOCA
+
+/* Define to empty if the keyword does not work.  */
+#undef const
+
+/* Define to one of _getb67, GETB67, getb67 for Cray-2 and Cray-YMP systems.
+   This function is required for alloca.c support on those systems.  */
+#undef CRAY_STACKSEG_END
+
+/* Define if you have alloca.h and it should be used (not Ultrix).  */
+#undef HAVE_ALLOCA_H
+
+/* Define if your struct stat has st_blksize.  */
+#undef HAVE_ST_BLKSIZE
+
+/* Define if int is 16 bits instead of 32.  */
+#undef INT_16_BITS
+
+/* Define if long int is 64 bits.  */
+#undef LONG_64_BITS
+
+/* Define to `long' if <sys/types.h> doesn't define.  */
+#undef off_t
+
+/* If using the C implementation of alloca, define if you know the
+   direction of stack growth for your system; otherwise it will be
+   automatically deduced at run-time.
+	STACK_DIRECTION > 0 => grows toward higher addresses
+	STACK_DIRECTION < 0 => grows toward lower addresses
+	STACK_DIRECTION = 0 => direction of growth unknown
+ */
+#undef STACK_DIRECTION
+
+/* Define if your processor stores words with the most significant
+   byte first (like Motorola and SPARC, unlike Intel and VAX).  */
+#undef WORDS_BIGENDIAN
+
+/* Define if you have bcopy.  */
+#undef HAVE_BCOPY
+
+/* Define if you have flock.  */
+#undef HAVE_FLOCK
+
+/* Define if you have fsync.  */
+#undef HAVE_FSYNC
+
+/* Define if you have ftruncate.  */
+#undef HAVE_FTRUNCATE
+
+/* Define if you have rename.  */
+#define HAVE_RENAME 1
+
+/* Define if you have the <fcntl.h> header file.  */
+#define HAVE_FCNTL_H 1
+
+/* Define if you have the <memory.h> header file.  */
+#define HAVE_MEMORY_H 1
+
+/* Define if you have the <stdlib.h> header file.  */
+#define HAVE_STDLIB_H 1
+
+/* Define if you have the <string.h> header file.  */
+#define HAVE_STRING_H 1
+
+/* Define if you have the <sys/file.h> header file.  */
+#undef HAVE_SYS_FILE_H
+
+/* Define if you have the <sys/types.h> header file.  */
+#define HAVE_SYS_TYPES_H 1
+
+/* Define if you have the <unistd.h> header file.  */
+#undef HAVE_UNISTD_H
+
+/* Define if you have the dbm library (-ldbm).  */
+#undef HAVE_LIBDBM
+
+/* Define if you have the ndbm library (-lndbm).  */
+#undef HAVE_LIBNDBM
+
+
Index: /tags/gsdl-21/gsdl/packages/wingdbm/bucket.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/wingdbm/bucket.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/wingdbm/bucket.c	(revision 699)
@@ -0,0 +1,336 @@
+/* bucket.c - The routines for playing with hash buckets. */
+
+/*  This file is part of GDBM, the GNU data base manager, by Philip A. Nelson.
+    Copyright (C) 1990, 1991, 1993  Free Software Foundation, Inc.
+
+    GDBM is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2, or (at your option)
+    any later version.
+
+    GDBM is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with GDBM; see the file COPYING.  If not, write to
+    the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+
+    You may contact the author by:
+       e-mail:  phil@cs.wwu.edu
+      us-mail:  Philip A. Nelson
+                Computer Science Department
+                Western Washington University
+                Bellingham, WA 98226
+       
+*************************************************************************/
+
+
+/* include system configuration before all else. */
+#include "autoconf.h"
+#include "gdbmdefs.h"
+
+
+/* Initializing a new hash buckets sets all bucket entries to -1 hash value. */
+void
+_gdbm_new_bucket (dbf, bucket, bits)
+     gdbm_file_info *dbf;
+     hash_bucket *bucket;
+     int bits;
+{
+  int index;
+  
+  /* Initialize the avail block. */
+  bucket->av_count = 0;
+
+  /* Set the information fields first. */
+  bucket->bucket_bits = bits;
+  bucket->count = 0;
+  
+  /* Initialize all bucket elements. */
+  for (index = 0; index < dbf->header->bucket_elems; index++)
+    bucket->h_table[index].hash_value = -1;
+}
+
+
+
+/* Find a bucket for DBF that is pointed to by the bucket directory from
+   location DIR_INDEX.   The bucket cache is first checked to see if it
+   is already in memory.  If not, a bucket may be tossed to read the new
+   bucket.  In any case, the requested bucket is make the "current" bucket
+   and dbf->bucket points to the correct bucket. */
+
+void
+_gdbm_get_bucket (dbf, dir_index)
+     gdbm_file_info *dbf;
+     word_t dir_index;
+{
+  off_t bucket_adr;	/* The address of the correct hash bucket.  */
+  int   num_bytes;	/* The number of bytes read. */
+  off_t	file_pos;	/* The return address for lseek. */
+  register int index;	/* Loop index. */
+
+  /* Initial set up. */
+  dbf->bucket_dir = dir_index;
+  bucket_adr = dbf->dir [dir_index];
+  
+  if (dbf->bucket_cache == NULL)
+    {
+      if(_gdbm_init_cache(dbf, DEFAULT_CACHESIZE) == -1)
+        _gdbm_fatal(dbf, "couldn't init cache");
+    }
+
+  /* Is that one is not already current, we must find it. */
+  if (dbf->cache_entry->ca_adr != bucket_adr)
+    {
+      /* Look in the cache. */
+      for (index = 0; index < dbf->cache_size; index++)
+        {
+	  if (dbf->bucket_cache[index].ca_adr == bucket_adr)
+	    {
+	      dbf->bucket = dbf->bucket_cache[index].ca_bucket;
+	      dbf->cache_entry = &dbf->bucket_cache[index];
+	      return;
+	    }
+        }
+
+      /* It is not in the cache, read it from the disk. */
+      dbf->last_read = (dbf->last_read + 1) % dbf->cache_size;
+      if (dbf->bucket_cache[dbf->last_read].ca_changed)
+	_gdbm_write_bucket (dbf, &dbf->bucket_cache[dbf->last_read]);
+      dbf->bucket_cache[dbf->last_read].ca_adr = bucket_adr;
+      dbf->bucket = dbf->bucket_cache[dbf->last_read].ca_bucket;
+      dbf->cache_entry = &dbf->bucket_cache[dbf->last_read];
+      dbf->cache_entry->ca_data.elem_loc = -1;
+      dbf->cache_entry->ca_changed = FALSE;
+
+      /* Read the bucket. */
+      file_pos = lseek (dbf->desc, bucket_adr, L_SET);
+      if (file_pos != bucket_adr)
+	_gdbm_fatal (dbf, "lseek error");
+
+      num_bytes = read (dbf->desc, dbf->bucket, dbf->header->bucket_size);
+      if (num_bytes != dbf->header->bucket_size)
+	_gdbm_fatal (dbf, "read error");
+    }
+
+  return;
+}
+
+
+/* Split the current bucket.  This includes moving all items in the bucket to
+   a new bucket.  This doesn't require any disk reads because all hash values
+   are stored in the buckets.  Splitting the current bucket may require
+   doubling the size of the hash directory.  */
+void
+_gdbm_split_bucket (dbf, next_insert)
+     gdbm_file_info *dbf;
+     word_t next_insert;
+{
+  hash_bucket *bucket[2]; 	/* Pointers to the new buckets. */
+
+  int          new_bits;	/* The number of bits for the new buckets. */
+  int	       cache_0;		/* Location in the cache for the buckets. */
+  int	       cache_1;
+  off_t        adr_0;		/* File address of the new bucket 0. */
+  off_t        adr_1;		/* File address of the new bucket 1. */
+  avail_elem   old_bucket;	/* Avail Struct for the old bucket. */
+
+  off_t        dir_start0;	/* Used in updating the directory. */
+  off_t        dir_start1;
+  off_t        dir_end;
+
+  off_t       *new_dir;		/* Pointer to the new directory. */
+  off_t        dir_adr; 	/* Address of the new directory. */
+  int          dir_size;	/* Size of the new directory. */
+  off_t        old_adr[31]; 	/* Address of the old directories. */
+  int          old_size[31]; 	/* Size of the old directories. */
+  int	       old_count;	/* Number of old directories. */
+
+  int          index;		/* Used in array indexing. */
+  int          index1;		/* Used in array indexing. */
+  int          elem_loc;	/* Location in new bucket to put element. */
+  bucket_element *old_el;	/* Pointer into the old bucket. */
+  int	       select;		/* Used to index bucket during movement. */
+
+
+  /* No directories are yet old. */
+  old_count = 0;
+
+  if (dbf->bucket_cache == NULL)
+    {
+      if(_gdbm_init_cache(dbf, DEFAULT_CACHESIZE) == -1)
+        _gdbm_fatal(dbf, "couldn't init cache");
+    }
+ 
+  while (dbf->bucket->count == dbf->header->bucket_elems)
+    {
+      /* Initialize the "new" buckets in the cache. */
+      do
+	{
+	  dbf->last_read = (dbf->last_read + 1) % dbf->cache_size;
+	  cache_0 = dbf->last_read;
+	}      
+      while (dbf->bucket_cache[cache_0].ca_bucket == dbf->bucket);
+      bucket[0] = dbf->bucket_cache[cache_0].ca_bucket;
+      if (dbf->bucket_cache[cache_0].ca_changed)
+	_gdbm_write_bucket (dbf, &dbf->bucket_cache[cache_0]);
+      do
+	{
+	  dbf->last_read = (dbf->last_read + 1) % dbf->cache_size;
+	  cache_1 = dbf->last_read;
+	}      
+      while (dbf->bucket_cache[cache_1].ca_bucket == dbf->bucket);
+      bucket[1] = dbf->bucket_cache[cache_1].ca_bucket;
+      if (dbf->bucket_cache[cache_1].ca_changed)
+	_gdbm_write_bucket (dbf, &dbf->bucket_cache[cache_1]);
+      new_bits = dbf->bucket->bucket_bits+1;
+      _gdbm_new_bucket (dbf, bucket[0], new_bits);
+      _gdbm_new_bucket (dbf, bucket[1], new_bits);
+      adr_0 = _gdbm_alloc (dbf, dbf->header->bucket_size); 
+      dbf->bucket_cache[cache_0].ca_adr = adr_0;
+      adr_1 = _gdbm_alloc (dbf, dbf->header->bucket_size);
+      dbf->bucket_cache[cache_1].ca_adr = adr_1;
+
+      
+      
+      /* Double the directory size if necessary. */
+      if (dbf->header->dir_bits == dbf->bucket->bucket_bits)
+	{
+	  dir_size = dbf->header->dir_size * 2;
+	  dir_adr  = _gdbm_alloc (dbf, dir_size);
+	  new_dir  = (off_t *) malloc (dir_size);
+	  if (new_dir == NULL) _gdbm_fatal (dbf, "malloc error");
+	  for (index = 0;
+	  	(unsigned int)index < dbf->header->dir_size/sizeof (off_t); index++)
+	    {
+	      new_dir[2*index]   = dbf->dir[index];
+	      new_dir[2*index+1] = dbf->dir[index];
+	    }
+	  
+	  /* Update header. */
+	  old_adr[old_count] = dbf->header->dir;
+	  dbf->header->dir = dir_adr;
+	  old_size[old_count] = dbf->header->dir_size;
+	  dbf->header->dir_size = dir_size;
+	  dbf->header->dir_bits = new_bits;
+	  old_count++;
+	  
+	  /* Now update dbf.  */
+	  dbf->header_changed = TRUE;
+	  dbf->bucket_dir *= 2;
+	  free (dbf->dir);
+	  dbf->dir = new_dir;
+	}
+
+      /* Copy all elements in dbf->bucket into the new buckets. */
+      for (index = 0; index < dbf->header->bucket_elems; index++)
+	{
+	  old_el = & (dbf->bucket->h_table[index]);
+	  select = (old_el->hash_value >> (31-new_bits)) & 1;
+	  elem_loc = old_el->hash_value % dbf->header->bucket_elems;
+	  while (bucket[select]->h_table[elem_loc].hash_value != -1)
+	    elem_loc = (elem_loc + 1) % dbf->header->bucket_elems;
+	  bucket[select]->h_table[elem_loc] = *old_el;
+	  bucket[select]->count += 1;
+	}
+      
+      /* Allocate avail space for the bucket[1]. */
+      bucket[1]->bucket_avail[0].av_adr
+	= _gdbm_alloc (dbf, dbf->header->block_size);
+      bucket[1]->bucket_avail[0].av_size = dbf->header->block_size;
+      bucket[1]->av_count = 1;
+      
+      /* Copy the avail elements in dbf->bucket to bucket[0]. */
+      bucket[0]->av_count = dbf->bucket->av_count;
+      index = 0;
+      index1 = 0;
+      if (bucket[0]->av_count == BUCKET_AVAIL)
+	{
+	  /* The avail is full, move the first one to bucket[1]. */
+	  _gdbm_put_av_elem (dbf->bucket->bucket_avail[0],
+			     bucket[1]->bucket_avail,
+			     &bucket[1]->av_count);
+	  index = 1;
+	  bucket[0]->av_count --;
+	}
+      for (; index < dbf->bucket->av_count; index++)
+	{
+	  bucket[0]->bucket_avail[index1++] = dbf->bucket->bucket_avail[index];
+	}
+      
+      /* Update the directory.  We have new file addresses for both buckets. */
+      dir_start1 = (dbf->bucket_dir >> (dbf->header->dir_bits - new_bits)) | 1;
+      dir_end = (dir_start1 + 1) << (dbf->header->dir_bits - new_bits);
+      dir_start1 = dir_start1 << (dbf->header->dir_bits - new_bits);
+      dir_start0 = dir_start1 - (dir_end - dir_start1);
+      for (index = dir_start0; index < dir_start1; index++)
+	dbf->dir[index] = adr_0;
+      for (index = dir_start1; index < dir_end; index++)
+	dbf->dir[index] = adr_1;
+      
+      
+      /* Set changed flags. */
+      dbf->bucket_cache[cache_0].ca_changed = TRUE;
+      dbf->bucket_cache[cache_1].ca_changed = TRUE;
+      dbf->bucket_changed = TRUE;
+      dbf->directory_changed = TRUE;
+      dbf->second_changed = TRUE;
+      
+      /* Update the cache! */
+      dbf->bucket_dir = next_insert >> (31-dbf->header->dir_bits);
+      
+      /* Invalidate old cache entry. */
+      old_bucket.av_adr  = dbf->cache_entry->ca_adr;
+      old_bucket.av_size = dbf->header->bucket_size;
+      dbf->cache_entry->ca_adr = 0;
+      dbf->cache_entry->ca_changed = FALSE;
+      
+      /* Set dbf->bucket to the proper bucket. */
+      if (dbf->dir[dbf->bucket_dir] == adr_0)
+	{
+	  dbf->bucket = bucket[0];
+	  dbf->cache_entry = &dbf->bucket_cache[cache_0];
+	  _gdbm_put_av_elem (old_bucket,
+			     bucket[1]->bucket_avail,
+			     &bucket[1]->av_count);
+	}
+      else
+	{
+	  dbf->bucket = bucket[1];
+	  dbf->cache_entry = &dbf->bucket_cache[cache_1];
+	  _gdbm_put_av_elem (old_bucket,
+			     bucket[0]->bucket_avail,
+			     &bucket[0]->av_count);
+	}
+      
+    }
+
+  /* Get rid of old directories. */
+  for (index = 0; index < old_count; index++)
+    _gdbm_free (dbf, old_adr[index], old_size[index]);
+}
+
+
+/* The only place where a bucket is written.  CA_ENTRY is the
+   cache entry containing the bucket to be written. */
+
+void
+_gdbm_write_bucket (dbf, ca_entry)
+     gdbm_file_info *dbf;
+     cache_elem *ca_entry;
+{
+  int  num_bytes;	/* The return value for write. */
+  off_t file_pos;	/* The return value for lseek. */
+  
+  file_pos = lseek (dbf->desc, ca_entry->ca_adr, L_SET);
+  if (file_pos != ca_entry->ca_adr)
+    _gdbm_fatal (dbf, "lseek error");
+  num_bytes = write (dbf->desc, ca_entry->ca_bucket, dbf->header->bucket_size);
+  if (num_bytes != dbf->header->bucket_size)
+    _gdbm_fatal (dbf, "write error");
+  ca_entry->ca_changed = FALSE;
+  ca_entry->ca_data.hash_val = -1;
+  ca_entry->ca_data.elem_loc = -1;
+}
Index: /tags/gsdl-21/gsdl/packages/wingdbm/close.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/wingdbm/close.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/wingdbm/close.c	(revision 699)
@@ -0,0 +1,55 @@
+/* close.c - Close the "original" style database. */
+
+/*  This file is part of GDBM, the GNU data base manager, by Philip A. Nelson.
+    Copyright (C) 1993  Free Software Foundation, Inc.
+
+    GDBM is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2, or (at your option)
+    any later version.
+
+    GDBM is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with GDBM; see the file COPYING.  If not, write to
+    the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+
+    You may contact the original author by:
+       e-mail:  phil@cs.wwu.edu
+      us-mail:  Philip A. Nelson
+                Computer Science Department
+                Western Washington University
+                Bellingham, WA 98226
+       
+    The author of this file is:
+       e-mail:  downsj@csos.orst.edu
+
+*************************************************************************/
+
+
+/* include system configuration before all else. */
+#include "autoconf.h"
+
+#include "gdbmdefs.h"
+#include "extern.h"
+
+
+/* it's unclear whether dbmclose() is *always* a void function in old
+   C libraries. we use int, here. */
+
+int
+dbmclose()
+{
+  if (_gdbm_file != NULL)
+    {
+      gdbm_close (_gdbm_file);
+      _gdbm_file = NULL;
+      if (_gdbm_memory.dptr != NULL) free(_gdbm_memory.dptr);
+      _gdbm_memory.dptr = NULL;
+      _gdbm_memory.dsize = 0;
+    }
+  return (0);
+}
Index: /tags/gsdl-21/gsdl/packages/wingdbm/extern.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/wingdbm/extern.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/wingdbm/extern.h	(revision 699)
@@ -0,0 +1,35 @@
+/* extern.h - The collection of external definitions needed. */
+
+/*  This file is part of GDBM, the GNU data base manager, by Philip A. Nelson.
+    Copyright (C) 1990, 1991, 1993  Free Software Foundation, Inc.
+
+    GDBM is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2, or (at your option)
+    any later version.
+
+    GDBM is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with GDBM; see the file COPYING.  If not, write to
+    the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+
+    You may contact the author by:
+       e-mail:  phil@cs.wwu.edu
+      us-mail:  Philip A. Nelson
+                Computer Science Department
+                Western Washington University
+                Bellingham, WA 98226
+       
+*************************************************************************/
+
+
+/* The global variables used for the "original" interface. */
+extern gdbm_file_info  *_gdbm_file;
+
+/* Memory for return data for the "original" interface. */
+extern datum _gdbm_memory;
+extern char *_gdbm_fetch_val;
Index: /tags/gsdl-21/gsdl/packages/wingdbm/falloc.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/wingdbm/falloc.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/wingdbm/falloc.c	(revision 699)
@@ -0,0 +1,422 @@
+/* falloc.c - The file space management routines for dbm. */
+
+/*  This file is part of GDBM, the GNU data base manager, by Philip A. Nelson.
+    Copyright (C) 1990, 1991, 1993, 1994  Free Software Foundation, Inc.
+
+    GDBM is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2, or (at your option)
+    any later version.
+
+    GDBM is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with GDBM; see the file COPYING.  If not, write to
+    the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+
+    You may contact the author by:
+       e-mail:  phil@cs.wwu.edu
+      us-mail:  Philip A. Nelson
+                Computer Science Department
+                Western Washington University
+                Bellingham, WA 98226
+       
+*************************************************************************/
+
+
+/* AIX demands this be the very first thing in the file. */
+#if !defined(__GNUC__) && defined(_AIX)
+ #pragma alloca
+#endif
+
+/* include system configuration before all else. */
+#include "autoconf.h"
+
+#include "gdbmdefs.h"
+
+
+/* The forward definitions for this file.  See the functions for
+   the definition of the function. */
+
+static avail_elem get_elem _ARGS((int, avail_elem [], int *));
+static avail_elem get_block _ARGS((int, gdbm_file_info *));
+static void push_avail_block _ARGS((gdbm_file_info *));
+static void pop_avail_block _ARGS((gdbm_file_info *));
+static void adjust_bucket_avail _ARGS((gdbm_file_info *));
+
+/* Allocate space in the file DBF for a block NUM_BYTES in length.  Return
+   the file address of the start of the block.  
+
+   Each hash bucket has a fixed size avail table.  We first check this
+   avail table to satisfy the request for space.  In most cases we can
+   and this causes changes to be only in the current hash bucket.
+   Allocation is done on a first fit basis from the entries.  If a
+   request can not be satisfied from the current hash bucket, then it is
+   satisfied from the file header avail block.  If nothing is there that
+   has enough space, another block at the end of the file is allocated
+   and the unused portion is returned to the avail block.  This routine
+   "guarantees" that an allocation does not cross a block boundary unless
+   the size is larger than a single block.  The avail structure is
+   changed by this routine if a change is needed.  If an error occurs,
+   the value of 0 will be returned.  */
+
+off_t
+_gdbm_alloc (dbf, num_bytes)
+     gdbm_file_info *dbf;
+     int num_bytes;
+{
+  off_t file_adr;		/* The address of the block. */
+  avail_elem av_el;		/* For temporary use. */
+
+  /* The current bucket is the first place to look for space. */
+  av_el = get_elem (num_bytes, dbf->bucket->bucket_avail,
+		    &dbf->bucket->av_count);
+
+  /* If we did not find some space, we have more work to do. */
+  if (av_el.av_size == 0)
+    {
+      /* Is the header avail block empty and there is something on the stack. */
+      if ((dbf->header->avail.count == 0)
+          && (dbf->header->avail.next_block != 0))
+        pop_avail_block (dbf);
+
+      /* check the header avail table next */
+      av_el = get_elem (num_bytes, dbf->header->avail.av_table,
+      			&dbf->header->avail.count);
+      if (av_el.av_size == 0)
+        /* Get another full block from end of file. */
+        av_el = get_block (num_bytes, dbf);
+
+      dbf->header_changed = TRUE;
+    }
+
+  /* We now have the place from which we will allocate the new space. */
+  file_adr = av_el.av_adr;
+
+  /* Put the unused space back in the avail block. */
+  av_el.av_adr += num_bytes;
+  av_el.av_size -= num_bytes;
+  _gdbm_free (dbf, av_el.av_adr, av_el.av_size);
+
+  /* Return the address. */
+  return file_adr;
+  
+}
+
+
+
+/* Free space of size NUM_BYTES in the file DBF at file address FILE_ADR.  Make
+   it avaliable for reuse through _gdbm_alloc.  This routine changes the
+   avail structure.  The value TRUE is returned if there were errors.  If no
+   errors occured, the value FALSE is returned. */
+
+void
+_gdbm_free (dbf, file_adr, num_bytes)
+     gdbm_file_info *dbf;
+     off_t file_adr;
+     int num_bytes;
+{
+  avail_elem temp;
+
+  /* Is it too small to worry about? */
+  if (num_bytes <= IGNORE_SIZE)
+    return;
+
+  /* Initialize the avail element. */
+  temp.av_size = num_bytes;
+  temp.av_adr = file_adr;
+
+  /* Is the freed space large or small? */
+  if (num_bytes >= dbf->header->block_size)
+    {
+      if (dbf->header->avail.count == dbf->header->avail.size)
+	{
+	  push_avail_block (dbf);
+	}
+      _gdbm_put_av_elem (temp, dbf->header->avail.av_table,
+			 &dbf->header->avail.count);
+      dbf->header_changed = TRUE;
+    }
+  else
+    {
+      /* Try to put into the current bucket. */
+      if (dbf->bucket->av_count < BUCKET_AVAIL)
+	_gdbm_put_av_elem (temp, dbf->bucket->bucket_avail,
+			   &dbf->bucket->av_count);
+      else
+	{
+	  if (dbf->header->avail.count == dbf->header->avail.size)
+	    {
+	      push_avail_block (dbf);
+	    }
+	  _gdbm_put_av_elem (temp, dbf->header->avail.av_table,
+			     &dbf->header->avail.count);
+	  dbf->header_changed = TRUE;
+	}
+    }
+
+  if (dbf->header_changed)
+    adjust_bucket_avail (dbf);
+
+  /* All work is done. */
+  return;
+}
+
+
+
+/* The following are all utility routines needed by the previous two. */
+
+
+/* Gets the avail block at the top of the stack and loads it into the
+   active avail block.  It does a "free" for itself! */
+
+static void
+pop_avail_block (dbf)
+     gdbm_file_info *dbf;
+{
+  int  num_bytes;		/* For use with the read system call. */
+  off_t file_pos;		/* For use with the lseek system call. */
+  avail_elem temp;
+
+  /* Set up variables. */
+  temp.av_adr = dbf->header->avail.next_block;
+  temp.av_size = ( ( (dbf->header->avail.size * sizeof (avail_elem)) >> 1)
+		  + sizeof (avail_block));
+
+  /* Read the block. */
+  file_pos = lseek (dbf->desc, temp.av_adr, L_SET);
+  if (file_pos != temp.av_adr)  _gdbm_fatal (dbf, "lseek error");
+  num_bytes = read (dbf->desc, &dbf->header->avail, temp.av_size);
+  if (num_bytes != temp.av_size) _gdbm_fatal (dbf, "read error");
+
+  /* We changed the header. */
+  dbf->header_changed = TRUE;
+
+  /* Free the previous avail block. */
+  _gdbm_put_av_elem (temp, dbf->header->avail.av_table,
+		     &dbf->header->avail.count);
+}
+
+
+/* Splits the header avail block and pushes half onto the avail stack. */
+
+static void
+push_avail_block (dbf)
+     gdbm_file_info *dbf;
+{
+  int  num_bytes;
+  int  av_size;
+  off_t av_adr;
+  int  index;
+  off_t file_pos;
+  avail_block *temp;
+  avail_elem  new_loc;
+ 
+
+  /* Caclulate the size of the split block. */
+  av_size = ( (dbf->header->avail.size * sizeof (avail_elem)) >> 1)
+            + sizeof (avail_block);
+
+  /* Get address in file for new av_size bytes. */
+  new_loc = get_elem (av_size, dbf->header->avail.av_table,
+		      &dbf->header->avail.count);
+  if (new_loc.av_size == 0)
+    new_loc = get_block (av_size, dbf);
+  av_adr = new_loc.av_adr;
+
+
+  /* Split the header block. */
+  temp = (avail_block *) alloca (av_size);
+#ifdef MSDOS
+  if (temp == (avail_block *) 0)
+    {
+      fprintf (stderr, "Error: alloca() failed in gdbm (%s).\n", __FILE__);
+      exit (-2);
+    }
+#endif /* MSDOS */  /* Set the size to be correct AFTER the pop_avail_block. */
+  temp->size = dbf->header->avail.size;
+  temp->count = 0;
+  temp->next_block = dbf->header->avail.next_block;
+  dbf->header->avail.next_block = av_adr;
+  for (index = 1; index < dbf->header->avail.count; index++)
+    if ( (index & 0x1) == 1)	/* Index is odd. */
+      temp->av_table[temp->count++] = dbf->header->avail.av_table[index];
+    else
+      dbf->header->avail.av_table[index>>1]
+	= dbf->header->avail.av_table[index];
+
+  /* Update the header avail count to previous size divided by 2. */
+  dbf->header->avail.count >>= 1;
+
+  /* Free the unneeded space. */
+  new_loc.av_adr += av_size;
+  new_loc.av_size -= av_size;
+  _gdbm_free (dbf, new_loc.av_adr, new_loc.av_size);
+
+  /* Update the disk. */
+  file_pos = lseek (dbf->desc, av_adr, L_SET);
+  if (file_pos != av_adr) _gdbm_fatal (dbf, "lseek error");
+  num_bytes = write (dbf->desc, temp, av_size);
+  if (num_bytes != av_size) _gdbm_fatal (dbf, "write error");
+
+}
+
+
+
+/* Get_elem returns an element in the AV_TABLE block which is
+   larger than SIZE.  AV_COUNT is the number of elements in the
+   AV_TABLE.  If an item is found, it extracts it from the AV_TABLE
+   and moves the other elements up to fill the space. If no block is 
+   found larger than SIZE, get_elem returns a size of zero.  This
+   routine does no I/O. */
+
+static avail_elem
+get_elem (size, av_table, av_count)
+     int size;
+     avail_elem av_table[];
+     int *av_count;
+{
+  int index;			/* For searching through the avail block. */
+  avail_elem val;		/* The default return value. */
+
+  /* Initialize default return value. */
+  val.av_adr = 0;
+  val.av_size = 0;
+
+  /* Search for element.  List is sorted by size. */
+  index = 0;
+  while (index < *av_count && av_table[index].av_size < size)
+    {
+      index++;
+    }
+
+  /* Did we find one of the right size? */
+  if (index >= *av_count)
+    return val;
+
+  /* Ok, save that element and move all others up one. */
+  val = av_table[index];
+  *av_count -= 1;
+  while (index < *av_count)
+    {
+      av_table[index] = av_table[index+1];
+      index++;
+    }
+
+  return val;
+}
+
+
+/* This routine inserts a single NEW_EL into the AV_TABLE block in
+   sorted order. This routine does no I/O. */
+
+int
+_gdbm_put_av_elem (new_el, av_table, av_count)
+     avail_elem new_el;
+     avail_elem av_table[];
+     int *av_count;
+{
+  int index;			/* For searching through the avail block. */
+  int index1;
+
+  /* Is it too small to deal with? */
+  if (new_el.av_size <= IGNORE_SIZE)
+    return FALSE; 
+
+  /* Search for place to put element.  List is sorted by size. */
+  index = 0;
+  while (index < *av_count && av_table[index].av_size < new_el.av_size)
+    {
+      index++;
+    }
+
+  /* Move all others up one. */
+  index1 = *av_count-1;
+  while (index1 >= index)
+    {
+      av_table[index1+1] = av_table[index1];
+      index1--;
+    }
+  
+  /* Add the new element. */
+  av_table[index] = new_el;
+
+  /* Increment the number of elements. */
+  *av_count += 1;  
+
+  return TRUE;
+}
+
+
+
+
+
+/* Get_block "allocates" new file space and the end of the file.  This is
+   done in integral block sizes.  (This helps insure that data smaller than
+   one block size is in a single block.)  Enough blocks are allocated to
+   make sure the number of bytes allocated in the blocks is larger than SIZE.
+   DBF contains the file header that needs updating.  This routine does
+   no I/O.  */
+
+static avail_elem
+get_block (size, dbf)
+     int size;
+     gdbm_file_info *dbf;
+{
+  avail_elem val;
+
+  /* Need at least one block. */
+  val.av_adr  = dbf->header->next_block;
+  val.av_size = dbf->header->block_size;
+
+  /* Get enough blocks to fit the need. */
+  while (val.av_size < size)
+    val.av_size += dbf->header->block_size;
+
+  /* Update the header and return. */
+  dbf->header->next_block += val.av_size;
+
+  /* We changed the header. */
+  dbf->header_changed = TRUE;
+
+  return val;
+  
+}
+
+
+/*  When the header already needs writing, we can make sure the current
+    bucket has its avail block as close to 1/2 full as possible. */
+static void
+adjust_bucket_avail (dbf)
+     gdbm_file_info *dbf;
+{
+  int third = BUCKET_AVAIL / 3;
+  avail_elem av_el;
+
+  /* Can we add more entries to the bucket? */
+  if (dbf->bucket->av_count < third)
+    {
+      if (dbf->header->avail.count > 0)
+	{
+	  dbf->header->avail.count -= 1;
+	  av_el = dbf->header->avail.av_table[dbf->header->avail.count];
+	  _gdbm_put_av_elem (av_el, dbf->bucket->bucket_avail,
+			     &dbf->bucket->av_count);
+	  dbf->bucket_changed = TRUE;
+	}
+      return;
+    }
+
+  /* Is there too much in the bucket? */
+  while (dbf->bucket->av_count > BUCKET_AVAIL-third
+	 && dbf->header->avail.count < dbf->header->avail.size)
+    {
+      av_el = get_elem (0, dbf->bucket->bucket_avail, &dbf->bucket->av_count);
+      _gdbm_put_av_elem (av_el, dbf->header->avail.av_table,
+			 &dbf->header->avail.count);
+      dbf->bucket_changed = TRUE;
+    }
+}
Index: /tags/gsdl-21/gsdl/packages/wingdbm/findkey.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/wingdbm/findkey.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/wingdbm/findkey.c	(revision 699)
@@ -0,0 +1,160 @@
+/* findkey.c - The routine that finds a key entry in the file. */
+
+/*  This file is part of GDBM, the GNU data base manager, by Philip A. Nelson.
+    Copyright (C) 1990, 1991, 1993  Free Software Foundation, Inc.
+
+    GDBM is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2, or (at your option)
+    any later version.
+
+    GDBM is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with GDBM; see the file COPYING.  If not, write to
+    the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+
+    You may contact the author by:
+       e-mail:  phil@cs.wwu.edu
+      us-mail:  Philip A. Nelson
+                Computer Science Department
+                Western Washington University
+                Bellingham, WA 98226
+       
+*************************************************************************/
+
+
+/* include system configuration before all else. */
+#include "autoconf.h"
+
+#include "gdbmdefs.h"
+
+
+/* Read the data found in bucket entry ELEM_LOC in file DBF and
+   return a pointer to it.  Also, cache the read value. */
+
+char *
+_gdbm_read_entry (dbf, elem_loc)
+     gdbm_file_info *dbf;
+     int elem_loc;
+{
+  int num_bytes;		/* For seeking and reading. */
+  int key_size;
+  int data_size;
+  off_t file_pos;
+  data_cache_elem *data_ca;
+
+  /* Is it already in the cache? */
+  if (dbf->cache_entry->ca_data.elem_loc == elem_loc)
+    return dbf->cache_entry->ca_data.dptr;
+
+  /* Set sizes and pointers. */
+  key_size = dbf->bucket->h_table[elem_loc].key_size;
+  data_size = dbf->bucket->h_table[elem_loc].data_size;
+  data_ca = &dbf->cache_entry->ca_data;
+  
+  /* Set up the cache. */
+  if (data_ca->dptr != NULL) free (data_ca->dptr);
+  data_ca->key_size = key_size;
+  data_ca->data_size = data_size;
+  data_ca->elem_loc = elem_loc;
+  data_ca->hash_val = dbf->bucket->h_table[elem_loc].hash_value;
+  if (key_size+data_size == 0)
+    data_ca->dptr = (char *) malloc (1);
+  else
+    data_ca->dptr = (char *) malloc (key_size+data_size);
+  if (data_ca->dptr == NULL) _gdbm_fatal (dbf, "malloc error");
+
+
+  /* Read into the cache. */
+  file_pos = lseek (dbf->desc,
+		    dbf->bucket->h_table[elem_loc].data_pointer, L_SET);
+  if (file_pos != dbf->bucket->h_table[elem_loc].data_pointer)
+    _gdbm_fatal (dbf, "lseek error");
+  num_bytes = read (dbf->desc, data_ca->dptr, key_size+data_size);
+  if (num_bytes != key_size+data_size) _gdbm_fatal (dbf, "read error");
+  
+  return data_ca->dptr;
+}
+
+
+
+/* Find the KEY in the file and get ready to read the associated data.  The
+   return value is the location in the current hash bucket of the KEY's
+   entry.  If it is found, a pointer to the data and the key are returned
+   in DPTR.  If it is not found, the value -1 is returned.  Since find
+   key computes the hash value of key, that value */
+int
+_gdbm_findkey (dbf, key, dptr, new_hash_val)
+     gdbm_file_info *dbf;
+     datum key;
+     char **dptr;
+     word_t *new_hash_val;		/* The new hash value. */
+{
+  word_t bucket_hash_val;	/* The hash value from the bucket. */
+  char  *file_key;		/* The complete key as stored in the file. */
+  int    elem_loc;		/* The location in the bucket. */
+  int    home_loc;		/* The home location in the bucket. */
+  int    key_size;		/* Size of the key on the file.  */
+
+  /* Compute hash value and load proper bucket.  */
+  *new_hash_val = _gdbm_hash (key);
+  _gdbm_get_bucket (dbf, *new_hash_val>> (31-dbf->header->dir_bits));
+
+  /* Is the element the last one found for this bucket? */
+  if (dbf->cache_entry->ca_data.elem_loc != -1 
+      && *new_hash_val == dbf->cache_entry->ca_data.hash_val
+      && dbf->cache_entry->ca_data.key_size == key.dsize
+      && dbf->cache_entry->ca_data.dptr != NULL
+      && bcmp (dbf->cache_entry->ca_data.dptr, key.dptr, key.dsize) == 0)
+    {
+      /* This is it. Return the cache pointer. */
+      *dptr = dbf->cache_entry->ca_data.dptr+key.dsize;
+      return dbf->cache_entry->ca_data.elem_loc;
+    }
+      
+  /* It is not the cached value, search for element in the bucket. */
+  elem_loc = *new_hash_val % dbf->header->bucket_elems;
+  home_loc = elem_loc;
+  bucket_hash_val = dbf->bucket->h_table[elem_loc].hash_value;
+  while (bucket_hash_val != -1)
+    {
+      key_size = dbf->bucket->h_table[elem_loc].key_size;
+      if (bucket_hash_val != *new_hash_val
+	 || key_size != key.dsize
+	 || bcmp (dbf->bucket->h_table[elem_loc].key_start, key.dptr,
+			(SMALL < key_size ? SMALL : key_size)) != 0) 
+	{
+	  /* Current elem_loc is not the item, go to next item. */
+	  elem_loc = (elem_loc + 1) % dbf->header->bucket_elems;
+	  if (elem_loc == home_loc) return -1;
+	  bucket_hash_val = dbf->bucket->h_table[elem_loc].hash_value;
+	}
+      else
+	{
+	  /* This may be the one we want.
+	     The only way to tell is to read it. */
+	  file_key = _gdbm_read_entry (dbf, elem_loc);
+	  if (bcmp (file_key, key.dptr, key_size) == 0)
+	    {
+	      /* This is the item. */
+	      *dptr = file_key+key.dsize;
+	      return elem_loc;
+	    }
+	  else
+	    {
+	      /* Not the item, try the next one.  Return if not found. */
+	      elem_loc = (elem_loc + 1) % dbf->header->bucket_elems;
+	      if (elem_loc == home_loc) return -1;
+	      bucket_hash_val = dbf->bucket->h_table[elem_loc].hash_value;
+	    }
+	}
+    }
+
+  /* If we get here, we never found the key. */
+  return -1;
+
+}
Index: /tags/gsdl-21/gsdl/packages/wingdbm/gdbm.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/wingdbm/gdbm.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/wingdbm/gdbm.h	(revision 699)
@@ -0,0 +1,343 @@
+/* gdbm.h  -  The include file for dbm users.  */
+
+/*  This file is part of GDBM, the GNU data base manager, by Philip A. Nelson.
+    Copyright (C) 1990, 1991, 1993  Free Software Foundation, Inc.
+
+    GDBM is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2, or (at your option)
+    any later version.
+
+    GDBM is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with GDBM; see the file COPYING.  If not, write to
+    the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+
+    You may contact the author by:
+       e-mail:  phil@cs.wwu.edu
+      us-mail:  Philip A. Nelson
+                Computer Science Department
+                Western Washington University
+                Bellingham, WA 98226
+       
+*************************************************************************/
+
+
+/* Protection for multiple includes. */
+#ifndef _GDBM_H_
+#define _GDBM_H_
+
+/* Parameters to gdbm_open for READERS, WRITERS, and WRITERS who
+   can create the database. */
+#define  GDBM_READER  0		/* A reader. */
+#define  GDBM_WRITER  1		/* A writer. */
+#define  GDBM_WRCREAT 2		/* A writer.  Create the db if needed. */
+#define  GDBM_NEWDB   3		/* A writer.  Always create a new db. */
+#define  GDBM_FAST    16	/* Write fast! => No fsyncs. */
+
+/* Parameters to gdbm_store for simple insertion or replacement in the
+   case that the key is already in the database. */
+#define  GDBM_INSERT  0		/* Never replace old data with new. */
+#define  GDBM_REPLACE 1		/* Always replace old data with new. */
+
+/* Parameters to gdbm_setopt, specifing the type of operation to perform. */
+#define  GDBM_CACHESIZE 1       /* Set the cache size. */
+#define  GDBM_FASTMODE  2       /* Toggle fast mode. */
+
+/* The data and key structure.  This structure is defined for compatibility. */
+typedef struct {
+	char *dptr;
+	int   dsize;
+      } datum;
+
+
+/* The available file space is stored in an "avail" table.  The one with
+   most activity is contained in the file header. (See below.)  When that
+   one filles up, it is split in half and half is pushed on an "avail
+   stack."  When the active avail table is empty and the "avail stack" is
+   not empty, the top of the stack is popped into the active avail table. */
+
+/* The following structure is the element of the avaliable table.  */
+typedef struct {
+  	int   av_size;		/* The size of the available block. */
+	off_t  av_adr;		/* The file address of the available block. */
+      } avail_elem;
+
+/* This is the actual table. The in-memory images of the avail blocks are
+   allocated by malloc using a calculated size.  */
+typedef struct {
+	int   size;		/* The number of avail elements in the table.*/
+	int   count;		/* The number of entries in the table. */
+	off_t next_block;	/* The file address of the next avail block. */
+	avail_elem av_table[1]; /* The table.  Make it look like an array.  */
+      } avail_block;
+
+/* The dbm file header keeps track of the current location of the hash
+   directory and the free space in the file.  */
+
+typedef struct {
+	word_t header_magic;  /* 0x13579ace to make sure the header is good. */
+	int   block_size;    /* The  optimal i/o blocksize from stat. */
+	off_t dir;	     /* File address of hash directory table. */
+	int   dir_size;	     /* Size in bytes of the table.  */
+	int   dir_bits;	     /* The number of address bits used in the table.*/
+	int   bucket_size;   /* Size in bytes of a hash bucket struct. */
+	int   bucket_elems;  /* Number of elements in a hash bucket. */
+	off_t next_block;    /* The next unallocated block address. */
+	avail_block avail;   /* This must be last because of the psuedo
+				array in avail.  This avail grows to fill
+				the entire block. */
+      }  gdbm_file_header;
+
+
+/* The dbm hash bucket element contains the full 31 bit hash value, the
+   "pointer" to the key and data (stored together) with their sizes.  It also
+   has a small part of the actual key value.  It is used to verify the first
+   part of the key has the correct value without having to read the actual
+   key. */
+
+typedef struct {
+	word_t hash_value;	/* The complete 31 bit value. */
+	char  key_start[SMALL];	/* Up to the first SMALL bytes of the key.  */
+	off_t data_pointer;	/* The file address of the key record. The
+				   data record directly follows the key.  */
+	int   key_size;		/* Size of key data in the file. */
+	int   data_size;	/* Size of associated data in the file. */
+      } bucket_element;
+
+
+/* A bucket is a small hash table.  This one consists of a number of
+   bucket elements plus some bookkeeping fields.  The number of elements
+   depends on the optimum blocksize for the storage device and on a
+   parameter given at file creation time.  This bucket takes one block.
+   When one of these tables gets full, it is split into two hash buckets.
+   The contents are split between them by the use of the first few bits
+   of the 31 bit hash function.  The location in a bucket is the hash
+   value modulo the size of the bucket.  The in-memory images of the
+   buckets are allocated by malloc using a calculated size depending of
+   the file system buffer size.  To speed up write, each bucket will have
+   BUCKET_AVAIL avail elements with the bucket. */
+
+typedef struct {
+        int   av_count;		   /* The number of bucket_avail entries. */
+        avail_elem bucket_avail[BUCKET_AVAIL];  /* Distributed avail. */
+	int   bucket_bits;	   /* The number of bits used to get here. */
+	int   count;		   /* The number of element buckets full. */
+	bucket_element h_table[1]; /* The table.  Make it look like an array.*/
+      } hash_bucket;
+
+/* We want to keep from reading buckets as much as possible.  The following is
+   to implement a bucket cache.  When full, buckets will be dropped in a
+   least recently read from disk order.  */
+
+/* To speed up fetching and "sequential" access, we need to implement a
+   data cache for key/data pairs read from the file.  To find a key, we
+   must exactly match the key from the file.  To reduce overhead, the
+   data will be read at the same time.  Both key and data will be stored
+   in a data cache.  Each bucket cached will have a one element data
+   cache.  */
+
+typedef struct {
+        word_t hash_val;
+	int   data_size;
+	int   key_size;
+	char *dptr;
+	int   elem_loc;
+      }  data_cache_elem;
+
+typedef struct {
+        hash_bucket *   ca_bucket;
+	off_t           ca_adr;
+	char		ca_changed;   /* Data in the bucket changed. */
+	data_cache_elem ca_data;
+      } cache_elem;
+
+
+
+/* This final structure contains all main memory based information for
+   a gdbm file.  This allows multiple gdbm files to be opened at the same
+   time by one program. */
+
+typedef struct {
+	/* Global variables and pointers to dynamic variables used by gdbm.  */
+
+  	/* The file name. */
+	char *name;
+
+	/* The reader/writer status. */
+	int read_write;
+
+	/* Fast_write is set to 1 if no fsyncs are to be done. */
+	int fast_write;
+
+	/* The fatal error handling routine. */
+	void (*fatal_err) ();
+
+	/* The gdbm file descriptor which is set in gdbm_open.  */
+	int  desc;
+
+	/* The file header holds information about the database. */
+	gdbm_file_header *header;
+
+	/* The hash table directory from extendible hashing.  See Fagin et al, 
+	   ACM Trans on Database Systems, Vol 4, No 3. Sept 1979, 315-344 */
+	off_t *dir;
+
+	/* The bucket cache. */
+	cache_elem *bucket_cache;
+	int cache_size;
+	int last_read;
+
+	/* Points to the current hash bucket in the cache. */
+	hash_bucket *bucket;
+
+	/* The directory entry used to get the current hash bucket. */
+	word_t bucket_dir;
+
+	/* Pointer to the current bucket's cache entry. */
+	cache_elem *cache_entry;
+
+
+	/* Bookkeeping of things that need to be written back at the
+	   end of an update. */
+	char  header_changed;
+	char  directory_changed;
+	char  bucket_changed;
+	char  second_changed;
+	
+      } gdbm_file_info;
+
+/* The file information header. This is good enough for most applications. */
+#define GDBM_FILE gdbm_file_info *
+/* typedef struct {int dummy[10];} *GDBM_FILE;*/
+
+/* Determine if the C(++) compiler requires complete function prototype  */
+#if  __STDC__ || defined(__cplusplus) || defined(c_plusplus)
+#define GDBM_Proto(x) x
+#else
+#define GDBM_Proto(x) ()
+#endif /* NeedFunctionPrototypes */
+
+/* External variable, the gdbm build release string. */
+extern char *gdbm_version;	
+
+
+/* GDBM C++ support */
+#if defined(__cplusplus) || defined(c_plusplus)
+extern "C" {
+#endif
+
+/* These are the routines! */
+
+extern GDBM_FILE gdbm_open GDBM_Proto((
+     char *file,
+     int  block_size,
+     int  flags,
+     int  mode,
+     void (*fatal_func)(),
+	 int need_filelock
+));
+
+extern void gdbm_close GDBM_Proto((
+     GDBM_FILE dbf
+));
+
+extern int gdbm_store GDBM_Proto((
+     GDBM_FILE dbf,
+     datum key,
+     datum content,
+     int flags
+));
+
+extern datum gdbm_fetch GDBM_Proto((
+     GDBM_FILE dbf,
+     datum key
+));
+
+extern int gdbm_delete GDBM_Proto((
+     GDBM_FILE dbf,
+     datum key
+));
+
+extern datum gdbm_firstkey GDBM_Proto((
+     GDBM_FILE dbf
+));
+
+extern datum gdbm_nextkey GDBM_Proto((
+     GDBM_FILE dbf,
+     datum key
+));
+
+extern int gdbm_reorganize GDBM_Proto((
+     GDBM_FILE dbf
+));
+
+extern void gdbm_sync GDBM_Proto((
+     GDBM_FILE dbf
+));
+
+extern int gdbm_exists GDBM_Proto((
+     GDBM_FILE dbf,
+     datum key
+));
+
+extern int gdbm_setopt GDBM_Proto((
+     GDBM_FILE dbf,
+     int optflag,
+     int *optval,
+     int optlen
+));
+
+#if defined(__cplusplus) || defined(c_plusplus)
+}
+#endif
+
+/* gdbm sends back the following error codes in the variable gdbm_errno. */
+typedef enum {	GDBM_NO_ERROR,
+		GDBM_MALLOC_ERROR,
+		GDBM_BLOCK_SIZE_ERROR,
+		GDBM_FILE_OPEN_ERROR,
+		GDBM_FILE_WRITE_ERROR,
+		GDBM_FILE_SEEK_ERROR,
+		GDBM_FILE_READ_ERROR,
+		GDBM_BAD_MAGIC_NUMBER,
+		GDBM_EMPTY_DATABASE,
+		GDBM_CANT_BE_READER,
+	        GDBM_CANT_BE_WRITER,
+		GDBM_READER_CANT_DELETE,
+		GDBM_READER_CANT_STORE,
+		GDBM_READER_CANT_REORGANIZE,
+		GDBM_UNKNOWN_UPDATE,
+		GDBM_ITEM_NOT_FOUND,
+		GDBM_REORGANIZE_FAILED,
+		GDBM_CANNOT_REPLACE,
+		GDBM_ILLEGAL_DATA,
+		GDBM_OPT_ALREADY_SET,
+		GDBM_OPT_ILLEGAL}
+	gdbm_error;
+
+extern gdbm_error gdbm_errno;
+
+
+/* extra prototypes */
+
+/* GDBM C++ support */
+#if defined(__cplusplus) || defined(c_plusplus)
+extern "C" {
+#endif
+
+extern const char *gdbm_strerror GDBM_Proto((
+     gdbm_error error
+));
+
+#if defined(__cplusplus) || defined(c_plusplus)
+}
+#endif
+
+#endif
+
+
Index: /tags/gsdl-21/gsdl/packages/wingdbm/gdbmclose.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/wingdbm/gdbmclose.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/wingdbm/gdbmclose.c	(revision 699)
@@ -0,0 +1,73 @@
+/* gdbmclose.c - Close a previously opened dbm file. */
+
+/*  This file is part of GDBM, the GNU data base manager, by Philip A. Nelson.
+    Copyright (C) 1990, 1991, 1993  Free Software Foundation, Inc.
+
+    GDBM is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2, or (at your option)
+    any later version.
+
+    GDBM is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with GDBM; see the file COPYING.  If not, write to
+    the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+
+    You may contact the author by:
+       e-mail:  phil@cs.wwu.edu
+      us-mail:  Philip A. Nelson
+                Computer Science Department
+                Western Washington University
+                Bellingham, WA 98226
+       
+*************************************************************************/
+
+
+/* AIX demands this be the very first thing in the file. */
+#if !defined(__GNUC__) && defined(_AIX)
+ #pragma alloca
+#endif
+
+/* include system configuration before all else. */
+#include "autoconf.h"
+
+#include "gdbmdefs.h"
+
+/* Close the dbm file and free all memory associated with the file DBF.
+   Before freeing members of DBF, check and make sure that they were
+   allocated.  */
+
+void
+gdbm_close (dbf)
+     gdbm_file_info *dbf;
+{
+  register int index;	/* For freeing the bucket cache. */
+
+  /* Make sure the database is all on disk. */
+  if (dbf->read_write == GDBM_WRITER)
+    fsync (dbf->desc);
+
+  /* Close the file and free all malloced memory. */
+  if (dbf->need_filelock) {
+	  UNLOCK_FILE(dbf);
+  }
+  close (dbf->desc);
+  free (dbf->name);
+  if (dbf->dir != NULL) free (dbf->dir);
+
+  if (dbf->bucket_cache != NULL) {
+    for (index = 0; index < dbf->cache_size; index++) {
+      if (dbf->bucket_cache[index].ca_bucket != NULL)
+	free (dbf->bucket_cache[index].ca_bucket);
+      if (dbf->bucket_cache[index].ca_data.dptr != NULL)
+	free (dbf->bucket_cache[index].ca_data.dptr);
+    }
+    free (dbf->bucket_cache);
+  }
+  if ( dbf->header != NULL ) free (dbf->header);
+  free (dbf);
+}
Index: /tags/gsdl-21/gsdl/packages/wingdbm/gdbmconst.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/wingdbm/gdbmconst.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/wingdbm/gdbmconst.h	(revision 699)
@@ -0,0 +1,60 @@
+/* gdbmconst.h - The constants defined for use in gdbm. */
+
+/*  This file is part of GDBM, the GNU data base manager, by Philip A. Nelson.
+    Copyright (C) 1990, 1991, 1993  Free Software Foundation, Inc.
+
+    GDBM is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2, or (at your option)
+    any later version.
+
+    GDBM is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with GDBM; see the file COPYING.  If not, write to
+    the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+
+    You may contact the author by:
+       e-mail:  phil@cs.wwu.edu
+      us-mail:  Philip A. Nelson
+                Computer Science Department
+                Western Washington University
+                Bellingham, WA 98226
+       
+*************************************************************************/
+
+/* Start with the constant definitions.  */
+#define  TRUE    1
+#define  FALSE   0
+
+/* Parameters to gdbm_open. */
+#define  GDBM_READER  0		/* READERS only. */
+#define  GDBM_WRITER  1		/* READERS and WRITERS.  Can not create. */
+#define  GDBM_WRCREAT 2		/* If not found, create the db. */
+#define  GDBM_NEWDB   3		/* ALWAYS create a new db.  (WRITER) */
+#define  GDBM_FAST    16	/* Write fast! => No fsyncs. */
+
+/* Parameters to gdbm_store for simple insertion or replacement in the
+   case a key to store is already in the database. */
+#define  GDBM_INSERT  0		/* Do not overwrite data in the database. */
+#define  GDBM_REPLACE 1		/* Replace the old value with the new value. */
+
+/* Parameters to gdbm_setopt, specifing the type of operation to perform. */
+#define	 GDBM_CACHESIZE	1	/* Set the cache size. */
+#define  GDBM_FASTMODE	2	/* Turn on or off fast mode. */
+
+/* In freeing blocks, we will ignore any blocks smaller (and equal) to
+   IGNORE_SIZE number of bytes. */
+#define IGNORE_SIZE 4
+
+/* The number of key bytes kept in a hash bucket. */
+#define SMALL    4
+
+/* The number of bucket_avail entries in a hash bucket. */
+#define BUCKET_AVAIL 6
+
+/* The size of the bucket cache. */
+#define DEFAULT_CACHESIZE  100
Index: /tags/gsdl-21/gsdl/packages/wingdbm/gdbmdefs.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/wingdbm/gdbmdefs.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/wingdbm/gdbmdefs.h	(revision 699)
@@ -0,0 +1,200 @@
+/* gdbmdefs.h - The include file for dbm.  Defines structure and constants. */
+
+/*  This file is part of GDBM, the GNU data base manager, by Philip A. Nelson.
+    Copyright (C) 1990, 1991, 1993  Free Software Foundation, Inc.
+
+    GDBM is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2, or (at your option)
+    any later version.
+
+    GDBM is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with GDBM; see the file COPYING.  If not, write to
+    the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+
+    You may contact the author by:
+       e-mail:  phil@cs.wwu.edu
+      us-mail:  Philip A. Nelson
+                Computer Science Department
+                Western Washington University
+                Bellingham, WA 98226
+*************************************************************************/
+
+#include "systems.h"
+#include "gdbmconst.h"
+
+/* The type definitions are next.  */
+
+/* The data and key structure.  This structure is defined for compatibility. */
+
+typedef struct {
+	char *dptr;
+	int   dsize;
+      } datum;
+
+
+/* The available file space is stored in an "avail" table.  The one with
+   most activity is contained in the file header. (See below.)  When that
+   one filles up, it is split in half and half is pushed on an "avail
+   stack."  When the active avail table is empty and the "avail stack" is
+   not empty, the top of the stack is popped into the active avail table. */
+
+/* The following structure is the element of the avaliable table.  */
+typedef struct {
+  	int   av_size;		/* The size of the available block. */
+	off_t  av_adr;		/* The file address of the available block. */
+      } avail_elem;
+
+/* This is the actual table. The in-memory images of the avail blocks are
+   allocated by malloc using a calculated size.  */
+typedef struct {
+	int   size;		/* The number of avail elements in the table.*/
+	int   count;		/* The number of entries in the table. */
+	off_t next_block;	/* The file address of the next avail block. */
+	avail_elem av_table[1]; /* The table.  Make it look like an array.  */
+      } avail_block;
+
+/* The dbm file header keeps track of the current location of the hash
+   directory and the free space in the file.  */
+
+typedef struct {
+	word_t header_magic;  /* 0x13579ace to make sure the header is good. */
+	int   block_size;    /* The  optimal i/o blocksize from stat. */
+	off_t dir;	     /* File address of hash directory table. */
+	int   dir_size;	     /* Size in bytes of the table.  */
+	int   dir_bits;	     /* The number of address bits used in the table.*/
+	int   bucket_size;   /* Size in bytes of a hash bucket struct. */
+	int   bucket_elems;  /* Number of elements in a hash bucket. */
+	off_t next_block;    /* The next unallocated block address. */
+	avail_block avail;   /* This must be last because of the psuedo
+				array in avail.  This avail grows to fill
+				the entire block. */
+      }  gdbm_file_header;
+
+
+/* The dbm hash bucket element contains the full 31 bit hash value, the
+   "pointer" to the key and data (stored together) with their sizes.  It also
+   has a small part of the actual key value.  It is used to verify the first
+   part of the key has the correct value without having to read the actual
+   key. */
+
+typedef struct {
+	word_t hash_value;	/* The complete 31 bit value. */
+	char  key_start[SMALL];	/* Up to the first SMALL bytes of the key.  */
+	off_t data_pointer;	/* The file address of the key record. The
+				   data record directly follows the key.  */
+	int   key_size;		/* Size of key data in the file. */
+	int   data_size;	/* Size of associated data in the file. */
+      } bucket_element;
+
+
+/* A bucket is a small hash table.  This one consists of a number of
+   bucket elements plus some bookkeeping fields.  The number of elements
+   depends on the optimum blocksize for the storage device and on a
+   parameter given at file creation time.  This bucket takes one block.
+   When one of these tables gets full, it is split into two hash buckets.
+   The contents are split between them by the use of the first few bits
+   of the 31 bit hash function.  The location in a bucket is the hash
+   value modulo the size of the bucket.  The in-memory images of the
+   buckets are allocated by malloc using a calculated size depending of
+   the file system buffer size.  To speed up write, each bucket will have
+   BUCKET_AVAIL avail elements with the bucket. */
+
+typedef struct {
+        int   av_count;		   /* The number of bucket_avail entries. */
+        avail_elem bucket_avail[BUCKET_AVAIL];  /* Distributed avail. */
+	int   bucket_bits;	   /* The number of bits used to get here. */
+	int   count;		   /* The number of element buckets full. */
+	bucket_element h_table[1]; /* The table.  Make it look like an array.*/
+      } hash_bucket;
+
+/* We want to keep from reading buckets as much as possible.  The following is
+   to implement a bucket cache.  When full, buckets will be dropped in a
+   least recently read from disk order.  */
+
+/* To speed up fetching and "sequential" access, we need to implement a
+   data cache for key/data pairs read from the file.  To find a key, we
+   must exactly match the key from the file.  To reduce overhead, the
+   data will be read at the same time.  Both key and data will be stored
+   in a data cache.  Each bucket cached will have a one element data
+   cache.  */
+
+typedef struct {
+        word_t hash_val;
+	int   data_size;
+	int   key_size;
+	char *dptr;
+	int   elem_loc;
+      }  data_cache_elem;
+
+typedef struct {
+        hash_bucket *   ca_bucket;
+	off_t           ca_adr;
+	char		ca_changed;   /* Data in the bucket changed. */
+	data_cache_elem ca_data;
+      } cache_elem;
+
+
+
+/* This final structure contains all main memory based information for
+   a gdbm file.  This allows multiple gdbm files to be opened at the same
+   time by one program. */
+
+typedef struct {
+	/* Global variables and pointers to dynamic variables used by gdbm.  */
+
+  	/* The file name. */
+	char *name;
+
+	/* The reader/writer status. */
+	int read_write;
+
+	/* Fast_write is set to 1 if no fsyncs are to be done. */
+	int fast_write;
+
+	/* The fatal error handling routine. */
+	void (*fatal_err) ();
+
+    int need_filelock;
+
+	/* The gdbm file descriptor which is set in gdbm_open.  */
+	int  desc;
+
+	/* The file header holds information about the database. */
+	gdbm_file_header *header;
+
+	/* The hash table directory from extendible hashing.  See Fagin et al, 
+	   ACM Trans on Database Systems, Vol 4, No 3. Sept 1979, 315-344 */
+	off_t *dir;
+
+	/* The bucket cache. */
+	cache_elem *bucket_cache;
+	int cache_size;
+	int last_read;
+
+	/* Points to the current hash bucket in the cache. */
+	hash_bucket *bucket;
+
+	/* The directory entry used to get the current hash bucket. */
+	word_t bucket_dir;
+
+	/* Pointer to the current bucket's cache entry. */
+	cache_elem *cache_entry;
+
+
+	/* Bookkeeping of things that need to be written back at the
+	   end of an update. */
+	char  header_changed;
+	char  directory_changed;
+	char  bucket_changed;
+	char  second_changed;
+	
+      } gdbm_file_info;
+
+/* Now define all the routines in use. */
+#include "proto.h"
Index: /tags/gsdl-21/gsdl/packages/wingdbm/gdbmdelete.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/wingdbm/gdbmdelete.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/wingdbm/gdbmdelete.c	(revision 699)
@@ -0,0 +1,119 @@
+/* gdbmdelete.c - Remove the key and its associated data from the database. */
+
+/*  This file is part of GDBM, the GNU data base manager, by Philip A. Nelson.
+    Copyright (C) 1990, 1991, 1993  Free Software Foundation, Inc.
+
+    GDBM is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2, or (at your option)
+    any later version.
+
+    GDBM is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with GDBM; see the file COPYING.  If not, write to
+    the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+
+    You may contact the author by:
+       e-mail:  phil@cs.wwu.edu
+      us-mail:  Philip A. Nelson
+                Computer Science Department
+                Western Washington University
+                Bellingham, WA 98226
+       
+*************************************************************************/
+
+
+/* include system configuration before all else. */
+#include "autoconf.h"
+
+#include "gdbmdefs.h"
+#include "gdbmerrno.h"
+
+/* Remove the KEYed item and the KEY from the database DBF.  The file on disk
+   is updated to reflect the structure of the new database before returning
+   from this procedure.  */
+
+int
+gdbm_delete (dbf, key)
+     gdbm_file_info *dbf;
+     datum key;
+{
+  int elem_loc;		/* The location in the current hash bucket. */
+  int last_loc;		/* Last location emptied by the delete.  */
+  int home;		/* Home position of an item. */
+  bucket_element elem;  /* The element to be deleted. */
+  char *find_data;	/* Return pointer from findkey. */
+  word_t hash_val;	/* Returned by findkey. */
+  off_t free_adr;       /* Temporary stroage for address and size. */
+  int   free_size;
+
+  /* First check to make sure this guy is a writer. */
+  if (dbf->read_write != GDBM_WRITER)
+    {
+      gdbm_errno = GDBM_READER_CANT_DELETE;
+      return -1;
+    }
+
+  /* Initialize the gdbm_errno variable. */
+  gdbm_errno = GDBM_NO_ERROR;
+
+  /* Find the item. */
+  elem_loc = _gdbm_findkey (dbf, key, &find_data, &hash_val);
+  if (elem_loc == -1)
+    {
+      gdbm_errno = GDBM_ITEM_NOT_FOUND;
+      return -1;
+    }
+
+  /* Save the element.  */
+  elem = dbf->bucket->h_table[elem_loc];
+
+  /* Delete the element.  */
+  dbf->bucket->h_table[elem_loc].hash_value = -1;
+  dbf->bucket->count -= 1;
+
+  /* Move other elements to guarantee that they can be found. */
+  last_loc = elem_loc;
+  elem_loc = (elem_loc + 1) % dbf->header->bucket_elems;
+  while (elem_loc != last_loc
+	 && dbf->bucket->h_table[elem_loc].hash_value != -1)
+    {
+      home = dbf->bucket->h_table[elem_loc].hash_value
+	     % dbf->header->bucket_elems;
+      if ( (last_loc < elem_loc && (home <= last_loc || home > elem_loc))
+	  || (last_loc > elem_loc && home <= last_loc && home > elem_loc))
+	
+	{
+	  dbf->bucket->h_table[last_loc] = dbf->bucket->h_table[elem_loc];
+	  dbf->bucket->h_table[elem_loc].hash_value = -1;
+	  last_loc = elem_loc;
+	}
+      elem_loc = (elem_loc + 1) % dbf->header->bucket_elems;
+    }
+
+  /* Free the file space. */
+  free_adr = elem.data_pointer;
+  free_size = elem.key_size + elem.data_size;
+  _gdbm_free (dbf, free_adr, free_size);
+
+  /* Set the flags. */
+  dbf->bucket_changed = TRUE;
+
+  /* Clear out the data cache for the current bucket. */
+  if (dbf->cache_entry->ca_data.dptr != NULL)
+    {
+      free (dbf->cache_entry->ca_data.dptr);
+      dbf->cache_entry->ca_data.dptr = NULL;
+    }
+  dbf->cache_entry->ca_data.hash_val = -1;
+  dbf->cache_entry->ca_data.key_size = 0;
+  dbf->cache_entry->ca_data.elem_loc = -1;
+
+  /* Do the writes. */
+  _gdbm_end_update (dbf);
+  return 0;  
+}
Index: /tags/gsdl-21/gsdl/packages/wingdbm/gdbmerrno.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/wingdbm/gdbmerrno.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/wingdbm/gdbmerrno.c	(revision 699)
@@ -0,0 +1,62 @@
+/* gdbmerrno.c - convert gdbm errors into english. */
+
+/*  This file is part of GDBM, the GNU data base manager, by Philip A. Nelson.
+    Copyright (C) 1993  Free Software Foundation, Inc.
+
+    GDBM is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2, or (at your option)
+    any later version.
+
+    GDBM is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with GDBM; see the file COPYING.  If not, write to
+    the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+
+    You may contact the original author by:
+       e-mail:  phil@cs.wwu.edu
+      us-mail:  Philip A. Nelson
+                Computer Science Department
+                Western Washington University
+                Bellingham, WA 98226
+       
+    The author of this file is:
+       e-mail:  downsj@csos.orst.edu
+
+*************************************************************************/
+
+
+/* include system configuration before all else. */
+#include "autoconf.h"
+
+#include "gdbmerrno.h"
+
+/* this is not static so that applications may access the array if they
+   like. it must be in the same order as the error codes! */
+
+const char * const gdbm_errlist[] = {
+  "No error", "Malloc error", "Block size error", "File open error",
+  "File write error", "File seek error", "File read error",
+  "Bad magic number", "Empty database", "Can't be reader", "Can't be writer",
+  "Reader can't delete", "Reader can't store", "Reader can't reorganize",
+  "Unknown update", "Item not found", "Reorganize failed", "Cannot replace",
+  "Illegal data", "Option already set", "Illegal option"
+};
+
+const char *
+gdbm_strerror(error)
+    gdbm_error error;
+{
+  if(((int)error < 0) || ((int)error > 18))
+    {
+      return("Unkown error");
+    }
+  else
+    {
+      return(gdbm_errlist[(int)error]);
+    }
+}
Index: /tags/gsdl-21/gsdl/packages/wingdbm/gdbmerrno.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/wingdbm/gdbmerrno.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/wingdbm/gdbmerrno.h	(revision 699)
@@ -0,0 +1,54 @@
+/* gdbmerrno.h - The enumeration type describing all the dbm errors. */
+
+/*  This file is part of GDBM, the GNU data base manager, by Philip A. Nelson.
+    Copyright (C) 1990, 1991, 1993  Free Software Foundation, Inc.
+
+    GDBM is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2, or (at your option)
+    any later version.
+
+    GDBM is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with GDBM; see the file COPYING.  If not, write to
+    the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+
+    You may contact the author by:
+       e-mail:  phil@wwu.edu
+      us-mail:  Philip A. Nelson
+                Computer Science Department
+                Western Washington University
+                Bellingham, WA 98226
+       
+*************************************************************************/
+
+
+/* gdbm sends back the following error codes in the variable gdbm_errno. */
+typedef enum {	GDBM_NO_ERROR,
+		GDBM_MALLOC_ERROR,
+		GDBM_BLOCK_SIZE_ERROR,
+		GDBM_FILE_OPEN_ERROR,
+		GDBM_FILE_WRITE_ERROR,
+		GDBM_FILE_SEEK_ERROR,
+		GDBM_FILE_READ_ERROR,
+		GDBM_BAD_MAGIC_NUMBER,
+		GDBM_EMPTY_DATABASE,
+		GDBM_CANT_BE_READER,
+	        GDBM_CANT_BE_WRITER,
+		GDBM_READER_CANT_DELETE,
+		GDBM_READER_CANT_STORE,
+		GDBM_READER_CANT_REORGANIZE,
+		GDBM_UNKNOWN_UPDATE,
+		GDBM_ITEM_NOT_FOUND,
+		GDBM_REORGANIZE_FAILED,
+		GDBM_CANNOT_REPLACE,
+		GDBM_ILLEGAL_DATA,
+		GDBM_OPT_ALREADY_SET,
+		GDBM_OPT_ILLEGAL}
+	gdbm_error;
+
+extern gdbm_error gdbm_errno;
Index: /tags/gsdl-21/gsdl/packages/wingdbm/gdbmexists.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/wingdbm/gdbmexists.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/wingdbm/gdbmexists.c	(revision 699)
@@ -0,0 +1,51 @@
+/* gdbmexists.c - Check to see if a key exists */
+
+/*  This file is part of GDBM, the GNU data base manager, by Philip A. Nelson.
+    Copyright (C) 1993  Free Software Foundation, Inc.
+
+    GDBM is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 1, or (at your option)
+    any later version.
+
+    GDBM is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with GDBM; see the file COPYING.  If not, write to
+    the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+
+    You may contact the original author by:
+       e-mail:  phil@cs.wwu.edu
+      us-mail:  Philip A. Nelson
+                Computer Science Department
+                Western Washington University
+                Bellingham, WA 98226
+       
+    The author of this file is:
+       e-mail:  downsj@csos.orst.edu
+
+*************************************************************************/
+
+
+/* include system configuration before all else. */
+#include "autoconf.h"
+
+#include "gdbmdefs.h"
+#include "gdbmerrno.h"
+
+/* this is nothing more than a wrapper around _gdbm_findkey(). the
+   point? it doesn't alloate any memory. */
+
+int
+gdbm_exists (dbf, key)
+    gdbm_file_info *dbf;
+    datum key;
+{
+char *find_data;		/* dummy */
+word_t hash_val;		/* dummy */
+
+  return (_gdbm_findkey (dbf, key, &find_data, &hash_val) >= 0);
+}
Index: /tags/gsdl-21/gsdl/packages/wingdbm/gdbmfetch.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/wingdbm/gdbmfetch.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/wingdbm/gdbmfetch.c	(revision 699)
@@ -0,0 +1,81 @@
+/* gdbmfetch.c - Find a key and return the associated data.  */
+
+/*  This file is part of GDBM, the GNU data base manager, by Philip A. Nelson.
+    Copyright (C) 1990, 1991, 1993  Free Software Foundation, Inc.
+
+    GDBM is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2, or (at your option)
+    any later version.
+
+    GDBM is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with GDBM; see the file COPYING.  If not, write to
+    the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+
+    You may contact the author by:
+       e-mail:  phil@cs.wwu.edu
+      us-mail:  Philip A. Nelson
+                Computer Science Department
+                Western Washington University
+                Bellingham, WA 98226
+       
+*************************************************************************/
+
+
+/* AIX demands this be the very first thing in the file. */
+#if !defined(__GNUC__) && defined(_AIX)
+ #pragma alloca
+#endif
+
+/* include system configuration before all else. */
+#include "autoconf.h"
+
+#include "gdbmdefs.h"
+#include "gdbmerrno.h"
+
+/* Look up a given KEY and return the information associated with that KEY.
+   The pointer in the structure that is  returned is a pointer to dynamically
+   allocated memory block.  */
+
+datum
+gdbm_fetch (dbf, key)
+     gdbm_file_info *dbf;
+     datum key;
+{
+  datum  return_val;		/* The return value. */
+  int    elem_loc;		/* The location in the bucket. */
+  char  *find_data;		/* Returned from find_key. */
+  word_t   hash_val;		/* Returned from find_key. */
+
+  /* Set the default return value. */
+  return_val.dptr  = NULL;
+  return_val.dsize = 0;
+
+  /* Initialize the gdbm_errno variable. */
+  gdbm_errno = GDBM_NO_ERROR;
+
+  /* Find the key and return a pointer to the data. */
+  elem_loc = _gdbm_findkey (dbf, key, &find_data, &hash_val);
+
+  /* Copy the data if the key was found.  */
+  if (elem_loc >= 0)
+    {
+      /* This is the item.  Return the associated data. */
+      return_val.dsize = dbf->bucket->h_table[elem_loc].data_size;
+      if (return_val.dsize == 0)
+	return_val.dptr = (char *) malloc (1);
+      else
+	return_val.dptr = (char *) malloc (return_val.dsize);
+      if (return_val.dptr == NULL) _gdbm_fatal (dbf, "malloc error");
+      bcopy (find_data, return_val.dptr, return_val.dsize);
+    }
+
+  /* Check for an error and return. */
+  if (return_val.dptr == NULL) gdbm_errno = GDBM_ITEM_NOT_FOUND;
+  return return_val;
+}
Index: /tags/gsdl-21/gsdl/packages/wingdbm/gdbmopen.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/wingdbm/gdbmopen.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/wingdbm/gdbmopen.c	(revision 699)
@@ -0,0 +1,453 @@
+/* gdbmopen.c - Open the dbm file and initialize data structures for use. */
+
+/*  This file is part of GDBM, the GNU data base manager, by Philip A. Nelson.
+    Copyright (C) 1990, 1991, 1993  Free Software Foundation, Inc.
+
+    GDBM is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2, or (at your option)
+    any later version.
+
+    GDBM is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with GDBM; see the file COPYING.  If not, write to
+    the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+
+    You may contact the author by:
+       e-mail:  phil@cs.wwu.edu
+      us-mail:  Philip A. Nelson
+                Computer Science Department
+                Western Washington University
+                Bellingham, WA 98226
+       
+*************************************************************************/
+
+
+/* AIX demands this be the very first thing in the file. */
+#if !defined(__GNUC__) && defined(_AIX)
+ #pragma alloca
+#endif
+
+/* include system configuration before all else. */
+#include "autoconf.h"
+
+#include "gdbmdefs.h"
+#include "gdbmerrno.h"
+
+/* Initialize dbm system.  FILE is a pointer to the file name.  If the file
+   has a size of zero bytes, a file initialization procedure is performed,
+   setting up the initial structure in the file.  BLOCK_SIZE is used during
+   initialization to determine the size of various constructs.  If the value
+   is less than 512, the file system blocksize is used, otherwise the value
+   of BLOCK_SIZE is used.  BLOCK_SIZE is ignored if the file has previously
+   initialized.  If FLAGS is set to GDBM_READ the user wants to just
+   read the database and any call to dbm_store or dbm_delete will fail. Many
+   readers can access the database at the same time.  If FLAGS is set to
+   GDBM_WRITE, the user wants both read and write access to the database and
+   requires exclusive access.  If FLAGS is GDBM_WRCREAT, the user wants
+   both read and write access to the database and if the database does not
+   exist, create a new one.  If FLAGS is GDBM_NEWDB, the user want a
+   new database created, regardless of whether one existed, and wants read
+   and write access to the new database.  Any error detected will cause a 
+   return value of null and an approprate value will be in gdbm_errno.  If
+   no errors occur, a pointer to the "gdbm file descriptor" will be
+   returned. */
+   
+
+gdbm_file_info *
+gdbm_open (file, block_size, flags, mode, fatal_func, need_filelock)
+     char *file;
+     int  block_size;
+     int  flags;
+     int  mode;
+     void (*fatal_func) ();
+	 int need_filelock;
+{
+  gdbm_file_info *dbf;		/* The record to return. */
+  struct stat file_stat;	/* Space for the stat information. */
+  int         len;		/* Length of the file name. */
+  int         num_bytes;	/* Used in reading and writing. */
+  off_t       file_pos;		/* Used with seeks. */
+  int	      lock_val;         /* Returned by the flock call. */
+  int	      file_block_size;	/* Block size to use for a new file. */
+  int 	      index;		/* Used as a loop index. */
+  char        need_trunc;	/* Used with GDBM_NEWDB and locking to avoid
+				   truncating a file from under a reader. */
+
+  /* Initialize the gdbm_errno variable. */
+  gdbm_errno = GDBM_NO_ERROR;
+
+  /* Allocate new info structure. */
+  dbf = (gdbm_file_info *) malloc (sizeof (gdbm_file_info));
+  if (dbf == NULL)
+    {
+      gdbm_errno = GDBM_MALLOC_ERROR;
+      return NULL;
+    }
+
+  /* Initialize some fields for known values.  This is done so gdbm_close
+     will work if called before allocating some structures. */
+  dbf->need_filelock = need_filelock;
+  dbf->dir  = NULL;
+  dbf->bucket = NULL;
+  dbf->header = NULL;
+  dbf->bucket_cache = NULL;
+  dbf->cache_size = 0;
+  
+  /* Save name of file. */
+  len = strlen (file);
+  dbf->name = (char *) malloc (len + 1);
+  if (dbf->name == NULL)
+    {
+      free (dbf);
+      gdbm_errno = GDBM_MALLOC_ERROR;
+      return NULL;
+    }
+  strcpy (dbf->name, file);
+
+  /* Initialize the fatal error routine. */
+  dbf->fatal_err = fatal_func;
+
+  /* Check for fast writers. */
+  if (flags & GDBM_FAST)
+    {
+      dbf->fast_write = TRUE;
+      flags -= GDBM_FAST;
+    }
+  else
+    {
+      dbf->fast_write = FALSE;
+    }
+  
+  /* Open the file. */
+  need_trunc = FALSE;
+  if (flags == GDBM_READER)
+    {
+      dbf->desc = open (dbf->name, O_RDONLY|O_BINARY, 0);
+    }
+  else if (flags == GDBM_WRITER)
+    {
+      dbf->desc = open (dbf->name, O_RDWR|O_BINARY, 0);
+    }
+  else if (flags == GDBM_NEWDB)
+    {
+      dbf->desc = open (dbf->name, O_RDWR|O_CREAT|O_BINARY, mode);
+      flags = GDBM_WRITER;
+      need_trunc = TRUE;
+    }
+  else
+    {
+      dbf->desc = open (dbf->name, O_RDWR|O_CREAT|O_BINARY, mode);
+      flags = GDBM_WRITER;
+    }
+  if (dbf->desc < 0)
+    {
+      free (dbf->name);
+      free (dbf);
+      gdbm_errno = GDBM_FILE_OPEN_ERROR;
+      return NULL;
+    }
+
+  /* Get the status of the file. */
+  fstat (dbf->desc, &file_stat);
+
+  /* Lock the file in the approprate way. */
+  if (flags == GDBM_READER)
+    {
+      if (file_stat.st_size == 0)
+	{
+	  close (dbf->desc);
+	  free (dbf->name);
+	  free (dbf);
+	  gdbm_errno = GDBM_EMPTY_DATABASE;
+	  return NULL;
+	}
+      /* Sets lock_val to 0 for success.  See systems.h. */
+	  if (dbf->need_filelock) {
+	      READLOCK_FILE(dbf);
+	  } else {
+		  lock_val = 0;
+	  }
+    }
+  else
+    {
+      /* Sets lock_val to 0 for success.  See systems.h. */
+	  if (dbf->need_filelock) {
+		WRITELOCK_FILE(dbf);
+	  } else {
+		  lock_val = 0;
+	  }
+    }
+  if (lock_val != 0)
+    {
+      close (dbf->desc);
+      free (dbf->name);
+      free (dbf);
+      if (flags == GDBM_READER)
+	gdbm_errno = GDBM_CANT_BE_READER;
+      else
+	gdbm_errno = GDBM_CANT_BE_WRITER;
+      return NULL;
+    }
+
+  /* Record the kind of user. */
+  dbf->read_write = flags;
+
+  /* If we do have a write lock and it was a GDBM_NEWDB, it is 
+     now time to truncate the file. */
+  if (need_trunc && file_stat.st_size != 0)
+    {
+      TRUNCATE (dbf);
+      fstat (dbf->desc, &file_stat);
+    }
+
+  /* Decide if this is a new file or an old file. */
+  if (file_stat.st_size == 0)
+    {
+
+      /* This is a new file.  Create an empty database.  */
+
+      /* Start with the blocksize. */
+      if (block_size < 512)
+	file_block_size = STATBLKSIZE;
+      else
+	file_block_size = block_size;
+
+      /* Get space for the file header. */
+      dbf->header = (gdbm_file_header *) malloc (file_block_size);
+      if (dbf->header == NULL)
+	{
+	  gdbm_close (dbf);
+	  gdbm_errno = GDBM_MALLOC_ERROR;
+	  return NULL;
+	}
+
+      /* Set the magic number and the block_size. */
+      dbf->header->header_magic = 0x13579ace;
+      dbf->header->block_size = file_block_size;
+     
+      /* Create the initial hash table directory.  */
+      dbf->header->dir_size = 8 * sizeof (off_t);
+      dbf->header->dir_bits = 3;
+      while (dbf->header->dir_size < dbf->header->block_size)
+	{
+	  dbf->header->dir_size <<= 1;
+	  dbf->header->dir_bits += 1;
+	}
+
+      /* Check for correct block_size. */
+      if (dbf->header->dir_size != dbf->header->block_size)
+	{
+	  gdbm_close (dbf);
+	  gdbm_errno = GDBM_BLOCK_SIZE_ERROR;
+	  return NULL;
+	}
+
+      /* Allocate the space for the directory. */
+      dbf->dir = (off_t *) malloc (dbf->header->dir_size);
+      if (dbf->dir == NULL)
+	{
+	  gdbm_close (dbf);
+	  gdbm_errno = GDBM_MALLOC_ERROR;
+	  return NULL;
+	}
+      dbf->header->dir = dbf->header->block_size;
+
+      /* Create the first and only hash bucket. */
+      dbf->header->bucket_elems =
+	(dbf->header->block_size - sizeof (hash_bucket))
+	/ sizeof (bucket_element) + 1;
+      dbf->header->bucket_size  = dbf->header->block_size;
+#if !defined(sgi)
+      dbf->bucket = (hash_bucket *) (alloca (dbf->header->bucket_size));
+#else	/* sgi */
+      /* The SGI C compiler doesn't accept the previous form. */
+      {
+        hash_bucket *ptr;
+	ptr = (hash_bucket *) (alloca (dbf->header->bucket_size));
+	dbf->bucket = ptr;
+      }
+#endif	/* sgi */
+      if (dbf->bucket == NULL)
+	{
+	  gdbm_close (dbf);
+	  gdbm_errno = GDBM_MALLOC_ERROR;
+	  return NULL;
+	}
+      _gdbm_new_bucket (dbf, dbf->bucket, 0);
+      dbf->bucket->av_count = 1;
+      dbf->bucket->bucket_avail[0].av_adr = 3*dbf->header->block_size;
+      dbf->bucket->bucket_avail[0].av_size = dbf->header->block_size;
+
+      /* Set table entries to point to hash buckets. */
+      for (index = 0; index < dbf->header->dir_size / sizeof (off_t); index++)
+	dbf->dir[index] = 2*dbf->header->block_size;
+
+      /* Initialize the active avail block. */
+      dbf->header->avail.size
+	= ( (dbf->header->block_size - sizeof (gdbm_file_header))
+	 / sizeof (avail_elem)) + 1;
+      dbf->header->avail.count = 0;
+      dbf->header->avail.next_block = 0;
+      dbf->header->next_block  = 4*dbf->header->block_size;
+
+      /* Write initial configuration to the file. */
+      /* Block 0 is the file header and active avail block. */
+      num_bytes = write (dbf->desc, dbf->header, dbf->header->block_size);
+      if (num_bytes != dbf->header->block_size)
+	{
+	  gdbm_close (dbf);
+	  gdbm_errno = GDBM_FILE_WRITE_ERROR;
+	  return NULL;
+	}
+
+      /* Block 1 is the initial bucket directory. */
+      num_bytes = write (dbf->desc, dbf->dir, dbf->header->dir_size);
+      if (num_bytes != dbf->header->dir_size)
+	{
+	  gdbm_close (dbf);
+	  gdbm_errno = GDBM_FILE_WRITE_ERROR;
+	  return NULL;
+	}
+
+      /* Block 2 is the only bucket. */
+      num_bytes = write (dbf->desc, dbf->bucket, dbf->header->bucket_size);
+      if (num_bytes != dbf->header->bucket_size)
+	{
+	  gdbm_close (dbf);
+	  gdbm_errno = GDBM_FILE_WRITE_ERROR;
+	  return NULL;
+	}
+
+      /* Wait for initial configuration to be written to disk. */
+      fsync (dbf->desc);
+
+    }
+  else
+    {
+      /* This is an old database.  Read in the information from the file
+	 header and initialize the hash directory. */
+
+      gdbm_file_header partial_header;  /* For the first part of it. */
+
+      /* Read the partial file header. */
+      num_bytes = read (dbf->desc, &partial_header, sizeof (gdbm_file_header));
+      if (num_bytes != sizeof (gdbm_file_header))
+	{
+	  gdbm_close (dbf);
+	  gdbm_errno = GDBM_FILE_READ_ERROR;
+	  return NULL;
+	}
+
+      /* Is the magic number good? */
+      if (partial_header.header_magic != 0x13579ace)
+	{
+	  gdbm_close (dbf);
+	  gdbm_errno = GDBM_BAD_MAGIC_NUMBER;
+	  return NULL;
+	}
+
+      /* It is a good database, read the entire header. */
+      dbf->header = (gdbm_file_header *) malloc (partial_header.block_size);
+      if (dbf->header == NULL)
+	{
+	  gdbm_close (dbf);
+	  gdbm_errno = GDBM_MALLOC_ERROR;
+	  return NULL;
+	}
+      bcopy (&partial_header, dbf->header, sizeof (gdbm_file_header));
+      num_bytes = read (dbf->desc, &dbf->header->avail.av_table[1],
+			dbf->header->block_size-sizeof (gdbm_file_header));
+      if (num_bytes != dbf->header->block_size-sizeof (gdbm_file_header))
+	{
+	  gdbm_close (dbf);
+	  gdbm_errno = GDBM_FILE_READ_ERROR;
+	  return NULL;
+	}
+	
+      /* Allocate space for the hash table directory.  */
+      dbf->dir = (off_t *) malloc (dbf->header->dir_size);
+      if (dbf->dir == NULL)
+	{
+	  gdbm_close (dbf);
+	  gdbm_errno = GDBM_MALLOC_ERROR;
+	  return NULL;
+	}
+
+      /* Read the hash table directory. */
+      file_pos = lseek (dbf->desc, dbf->header->dir, L_SET);
+      if (file_pos != dbf->header->dir)
+	{
+	  gdbm_close (dbf);
+	  gdbm_errno = GDBM_FILE_SEEK_ERROR;
+	  return NULL;
+	}
+
+      num_bytes = read (dbf->desc, dbf->dir, dbf->header->dir_size);
+      if (num_bytes != dbf->header->dir_size)
+	{
+	  gdbm_close (dbf);
+	  gdbm_errno = GDBM_FILE_READ_ERROR;
+	  return NULL;
+	}
+
+    }
+      
+  /* Finish initializing dbf. */
+  dbf->last_read = -1;
+  dbf->bucket = NULL;
+  dbf->bucket_dir = 0;
+  dbf->cache_entry = NULL;
+  dbf->header_changed = FALSE;
+  dbf->directory_changed = FALSE;
+  dbf->bucket_changed = FALSE;
+  dbf->second_changed = FALSE;
+  
+  /* Everything is fine, return the pointer to the file
+     information structure.  */
+  return dbf;
+
+}
+
+/* initialize the bucket cache. */
+int
+_gdbm_init_cache(dbf, size)
+    gdbm_file_info *dbf;
+    int size;
+{
+register int index;
+
+  if (dbf->bucket_cache == NULL)
+    {
+      dbf->bucket_cache = (cache_elem *) malloc(sizeof(cache_elem) * size);
+      if(dbf->bucket_cache == NULL)
+        {
+          gdbm_errno = GDBM_MALLOC_ERROR;
+          return(-1);
+        }
+      dbf->cache_size = size;
+
+      for(index = 0; index < size; index++)
+        {
+          (dbf->bucket_cache[index]).ca_bucket
+            = (hash_bucket *) malloc (dbf->header->bucket_size);
+          if ((dbf->bucket_cache[index]).ca_bucket == NULL)
+	    {
+              gdbm_errno = GDBM_MALLOC_ERROR;
+	      return(-1);
+            }
+          (dbf->bucket_cache[index]).ca_adr = 0;
+          (dbf->bucket_cache[index]).ca_changed = FALSE;
+          (dbf->bucket_cache[index]).ca_data.hash_val = -1;
+          (dbf->bucket_cache[index]).ca_data.elem_loc = -1;
+          (dbf->bucket_cache[index]).ca_data.dptr = NULL;
+        }
+      dbf->bucket = dbf->bucket_cache[0].ca_bucket;
+      dbf->cache_entry = &dbf->bucket_cache[0];
+    }
+  return(0);
+}
Index: /tags/gsdl-21/gsdl/packages/wingdbm/gdbmreorg.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/wingdbm/gdbmreorg.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/wingdbm/gdbmreorg.c	(revision 699)
@@ -0,0 +1,224 @@
+/* gdbmreorg.c - Reorganize the database file. */
+
+/*  This file is part of GDBM, the GNU data base manager, by Philip A. Nelson.
+    Copyright (C) 1990, 1991, 1993  Free Software Foundation, Inc.
+
+    GDBM is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2, or (at your option)
+    any later version.
+
+    GDBM is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with GDBM; see the file COPYING.  If not, write to
+    the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+
+    You may contact the author by:
+       e-mail:  phil@cs.wwu.edu
+      us-mail:  Philip A. Nelson
+                Computer Science Department
+                Western Washington University
+                Bellingham, WA 98226
+       
+************************************************************************/
+
+
+/* AIX demands this be the very first thing in the file. */
+#if !defined(__GNUC__) && defined(_AIX)
+ #pragma alloca
+#endif
+
+/* include system configuration before all else. */
+#include "autoconf.h"
+
+#include "gdbmdefs.h"
+#include "gdbmerrno.h"
+#include "extern.h"
+
+#if !HAVE_RENAME
+
+/* Rename takes OLD_NAME and renames it as NEW_NAME.  If it can not rename
+   the file a non-zero value is returned.  OLD_NAME is guaranteed to
+   remain if it can't be renamed. It assumes NEW_NAME always exists (due
+   to being used in gdbm). */
+
+static int
+rename (old_name, new_name)
+     char* old_name;
+     char* new_name;
+{
+  if (unlink (new_name) != 0)   
+    return -1;
+
+  if (link (old_name, new_name) != 0)
+    return -1;
+  
+  unlink (old_name);
+  return 0;
+
+}
+#endif
+
+
+
+/* Reorganize the database.  This requires creating a new file and inserting
+   all the elements in the old file DBF into the new file.  The new file
+   is then renamed to the same name as the old file and DBF is updated to
+   contain all the correct information about the new file.  If an error
+   is detected, the return value is negative.  The value zero is returned
+   after a successful reorganization. */
+
+int
+gdbm_reorganize (dbf)
+     gdbm_file_info *dbf;
+
+{
+  gdbm_file_info *new_dbf;		/* The new file. */
+  char *new_name;			/* A temporary name. */
+  int  len;				/* Used in new_name construction. */
+  datum key, nextkey, data;		/* For the sequential sweep. */
+  struct stat fileinfo;			/* Information about the file. */
+  int  index;				/* Use in moving the bucket cache. */
+
+
+  /* Readers can not reorganize! */
+  if (dbf->read_write == GDBM_READER)
+    {
+      gdbm_errno = GDBM_READER_CANT_REORGANIZE;
+      return -1;
+    }
+  
+  /* Initialize the gdbm_errno variable. */
+  gdbm_errno = GDBM_NO_ERROR;
+
+  /* Construct new name for temporary file. */
+  len = strlen (dbf->name);
+  new_name = (char *) alloca (len + 3);
+  if (new_name == NULL)
+    {
+      gdbm_errno = GDBM_MALLOC_ERROR;
+      return -1;
+    }
+  strcpy (&new_name[0], dbf->name);
+  new_name[len+2] = 0;
+  new_name[len+1] = '#';
+  while ( (len > 0) && new_name[len-1] != '/')
+    {
+      new_name[len] = new_name[len-1];
+      len -= 1;
+    }
+  new_name[len] = '#';
+
+  /* Get the mode for the old file and open the new database.
+     The "fast" mode is used because the reorganization will fail
+     unless we create a complete copy of the database. */
+  fstat (dbf->desc, &fileinfo);
+  new_dbf = gdbm_open (new_name, dbf->header->block_size,
+		       GDBM_WRCREAT | GDBM_FAST,
+		       fileinfo.st_mode, dbf->fatal_err);
+
+  if (new_dbf == NULL)
+    {
+      gdbm_errno = GDBM_REORGANIZE_FAILED;
+      return -1;
+    }
+
+  
+  /* For each item in the old database, add an entry in the new. */
+  key = gdbm_firstkey (dbf);
+
+  while (key.dptr != NULL)
+    {
+      data = gdbm_fetch (dbf, key);
+      if (data.dptr != NULL)
+ 	{
+	  /* Add the data to the new file. */
+	  if (gdbm_store (new_dbf, key, data, GDBM_INSERT) != 0)
+	    {
+	      gdbm_close (new_dbf);
+	      gdbm_errno = GDBM_REORGANIZE_FAILED;
+	      unlink (new_name);
+	      return -1;
+	    }
+ 	}
+      else
+ 	{
+	  /* ERROR! Abort and don't finish reorganize. */
+	  gdbm_close (new_dbf);
+	  gdbm_errno = GDBM_REORGANIZE_FAILED;
+	  unlink (new_name);
+	  return -1;
+ 	}
+      nextkey = gdbm_nextkey (dbf, key);
+      free (key.dptr);
+      free (data.dptr);
+      key = nextkey;
+    }
+
+  /* Write everything. */
+  _gdbm_end_update (new_dbf);
+  gdbm_sync (new_dbf);
+
+
+  /* Move the new file to old name. */
+
+#ifdef MSDOS
+  if (close (new_dbf->desc)
+      || unlink (dbf->name)
+      || rename (new_name, dbf->name)
+      || (new_dbf->desc = open (dbf->name, O_RDWR|O_BINARY)) < 0)
+#else /* not MSDOS */
+  if (rename (new_name, dbf->name) != 0)
+#endif /* not MSDOS */
+   {
+      gdbm_errno = GDBM_REORGANIZE_FAILED;
+      gdbm_close (new_dbf);
+      return -1;
+    }
+
+  /* Fix up DBF to have the correct information for the new file. */
+  if (dbf->need_filelock) {
+	  UNLOCK_FILE(dbf);
+  }
+  close (dbf->desc);
+  free (dbf->header);
+  free (dbf->dir);
+
+  if (dbf->bucket_cache != NULL) {
+    for (index = 0; index < dbf->cache_size; index++) {
+      if (dbf->bucket_cache[index].ca_bucket != NULL)
+	free (dbf->bucket_cache[index].ca_bucket);
+      if (dbf->bucket_cache[index].ca_data.dptr != NULL)
+	free (dbf->bucket_cache[index].ca_data.dptr);
+    }
+    free (dbf->bucket_cache);
+  }
+
+  dbf->desc           = new_dbf->desc;
+  dbf->header         = new_dbf->header;
+  dbf->dir            = new_dbf->dir;
+  dbf->bucket         = new_dbf->bucket;
+  dbf->bucket_dir     = new_dbf->bucket_dir;
+  dbf->last_read      = new_dbf->last_read;
+  dbf->bucket_cache   = new_dbf->bucket_cache;
+  dbf->cache_size     = new_dbf->cache_size;
+  dbf->header_changed    = new_dbf->header_changed;
+  dbf->directory_changed = new_dbf->directory_changed;
+  dbf->bucket_changed    = new_dbf->bucket_changed;
+  dbf->second_changed    = new_dbf->second_changed;
+      
+  free (new_dbf);
+
+  /* Make sure the new database is all on disk. */
+  fsync (dbf->desc);
+
+  /* Force the right stuff for a correct bucket cache. */
+  dbf->cache_entry    = &dbf->bucket_cache[0];
+  _gdbm_get_bucket (dbf, 0);
+
+  return 0;
+}
Index: /tags/gsdl-21/gsdl/packages/wingdbm/gdbmseq.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/wingdbm/gdbmseq.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/wingdbm/gdbmseq.c	(revision 699)
@@ -0,0 +1,146 @@
+/* gdbmseq.c - Routines to visit all keys.  Not in sorted order. */
+
+/*  This file is part of GDBM, the GNU data base manager, by Philip A. Nelson.
+    Copyright (C) 1990, 1991, 1993  Free Software Foundation, Inc.
+
+    GDBM is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2, or (at your option)
+    any later version.
+
+    GDBM is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with GDBM; see the file COPYING.  If not, write to
+    the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+
+    You may contact the author by:
+       e-mail:  phil@cs.wwu.edu
+      us-mail:  Philip A. Nelson
+                Computer Science Department
+                Western Washington University
+                Bellingham, WA 98226
+       
+*************************************************************************/
+
+
+/* include system configuration before all else. */
+#include "autoconf.h"
+
+#include "gdbmdefs.h"
+#include "gdbmerrno.h"
+
+/* Special extern for this file. */
+extern char *_gdbm_read_entry _ARGS((gdbm_file_info *, int));
+
+
+/* Find and read the next entry in the hash structure for DBF starting
+   at ELEM_LOC of the current bucket and using RETURN_VAL as the place to
+   put the data that is found. */
+
+static void
+get_next_key (dbf, elem_loc, return_val)
+     gdbm_file_info *dbf;
+     int elem_loc;
+     datum *return_val;
+{
+  int   found;			/* Have we found the next key. */
+  char  *find_data;		/* Data pointer returned by find_key. */
+
+  /* Find the next key. */
+  found = FALSE;
+  while (!found)
+    {
+      /* Advance to the next location in the bucket. */
+      elem_loc++;
+      if (elem_loc == dbf->header->bucket_elems)
+	{
+	  /* We have finished the current bucket, get the next bucket.  */
+	  elem_loc = 0;
+
+	  /* Find the next bucket.  It is possible several entries in
+	     the bucket directory point to the same bucket. */
+	  while (dbf->bucket_dir < dbf->header->dir_size / sizeof (off_t)
+		 && dbf->cache_entry->ca_adr == dbf->dir[dbf->bucket_dir])
+	    dbf->bucket_dir++;
+
+	  /* Check to see if there was a next bucket. */
+	  if (dbf->bucket_dir < dbf->header->dir_size / sizeof (off_t))
+	    _gdbm_get_bucket (dbf, dbf->bucket_dir);	      
+	  else
+	    /* No next key, just return. */
+	    return ;
+	}
+      found = dbf->bucket->h_table[elem_loc].hash_value != -1;
+    }
+  
+  /* Found the next key, read it into return_val. */
+  find_data = _gdbm_read_entry (dbf, elem_loc);
+  return_val->dsize = dbf->bucket->h_table[elem_loc].key_size;
+  if (return_val->dsize == 0)
+    return_val->dptr = (char *) malloc (1);
+  else
+    return_val->dptr = (char *) malloc (return_val->dsize);
+  if (return_val->dptr == NULL) _gdbm_fatal (dbf, "malloc error");
+  bcopy (find_data, return_val->dptr, return_val->dsize);
+}
+
+
+/* Start the visit of all keys in the database.  This produces something in
+   hash order, not in any sorted order.  */
+
+datum
+gdbm_firstkey (dbf)
+     gdbm_file_info *dbf;
+{
+  datum return_val;		/* To return the first key. */
+
+  /* Set the default return value for not finding a first entry. */
+  return_val.dptr = NULL;
+
+  /* Initialize the gdbm_errno variable. */
+  gdbm_errno = GDBM_NO_ERROR;
+
+  /* Get the first bucket.  */
+  _gdbm_get_bucket (dbf, 0);
+
+  /* Look for first entry. */
+  get_next_key (dbf, -1, &return_val);
+
+  return return_val;
+}
+
+
+/* Continue visiting all keys.  The next key following KEY is returned. */
+
+datum
+gdbm_nextkey (dbf, key)
+     gdbm_file_info *dbf;
+     datum key;
+{
+  datum  return_val;		/* The return value. */
+  int    elem_loc;		/* The location in the bucket. */
+  char  *find_data;		/* Data pointer returned by _gdbm_findkey. */
+  word_t hash_val;		/* Returned by _gdbm_findkey. */
+
+  /* Initialize the gdbm_errno variable. */
+  gdbm_errno = GDBM_NO_ERROR;
+
+  /* Set the default return value for no next entry. */
+  return_val.dptr = NULL;
+
+  /* Do we have a valid key? */
+  if (key.dptr == NULL) return return_val;
+
+  /* Find the key.  */
+  elem_loc = _gdbm_findkey (dbf, key, &find_data, &hash_val);
+  if (elem_loc == -1) return return_val;
+
+  /* Find the next key. */  
+  get_next_key (dbf, elem_loc, &return_val);
+
+  return return_val;
+}
Index: /tags/gsdl-21/gsdl/packages/wingdbm/gdbmsetopt.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/wingdbm/gdbmsetopt.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/wingdbm/gdbmsetopt.c	(revision 699)
@@ -0,0 +1,78 @@
+/* gdbmsetopt.c - set options pertaining to a GDBM descriptor. */
+
+/*  This file is part of GDBM, the GNU data base manager, by Philip A. Nelson.
+    Copyright (C) 1993, 1994  Free Software Foundation, Inc.
+
+    GDBM is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2, or (at your option)
+    any later version.
+
+    GDBM is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with GDBM; see the file COPYING.  If not, write to
+    the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+
+    You may contact the original author by:
+       e-mail:  phil@cs.wwu.edu
+      us-mail:  Philip A. Nelson
+                Computer Science Department
+                Western Washington University
+                Bellingham, WA 98226
+       
+    The author of this file is:
+       e-mail:  downsj@csos.orst.edu
+
+*************************************************************************/
+
+
+/* include system configuration before all else. */
+#include "autoconf.h"
+
+#include "gdbmdefs.h"
+#include "gdbmerrno.h"
+
+/* operate on an already open descriptor. */
+
+/* ARGSUSED */
+int
+gdbm_setopt(dbf, optflag, optval, optlen)
+    gdbm_file_info *dbf;	/* descriptor to operate on. */
+    int optflag;		/* option to set. */
+    int *optval;		/* pointer to option value. */
+    int optlen;			/* size of optval. */
+{
+  switch(optflag)
+    {
+      case GDBM_CACHESIZE:
+        /* here, optval will point to the new size of the cache. */
+        if (dbf->bucket_cache != NULL)
+          {
+            gdbm_errno = GDBM_OPT_ALREADY_SET;
+            return(-1);
+          }
+
+        return(_gdbm_init_cache(dbf, ((*optval) > 9) ? (*optval) : 10));
+
+      case GDBM_FASTMODE:
+      	/* here, optval will point to either true or false. */
+	if ((*optval != TRUE) && (*optval != FALSE))
+	  {
+	    gdbm_errno = GDBM_OPT_ILLEGAL;
+	    return(-1);
+	  }
+
+	dbf->fast_write = *optval;
+	break;
+
+      default:
+        gdbm_errno = GDBM_OPT_ILLEGAL;
+        return(-1);
+    }
+
+  return(0);
+}
Index: /tags/gsdl-21/gsdl/packages/wingdbm/gdbmstore.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/wingdbm/gdbmstore.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/wingdbm/gdbmstore.c	(revision 699)
@@ -0,0 +1,167 @@
+/* gdbmstore.c - Add a new key/data pair to the database. */
+
+/*  This file is part of GDBM, the GNU data base manager, by Philip A. Nelson.
+    Copyright (C) 1990, 1991, 1993  Free Software Foundation, Inc.
+
+    GDBM is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2, or (at your option)
+    any later version.
+
+    GDBM is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with GDBM; see the file COPYING.  If not, write to
+    the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+
+    You may contact the author by:
+       e-mail:  phil@cs.wwu.edu
+      us-mail:  Philip A. Nelson
+                Computer Science Department
+                Western Washington University
+                Bellingham, WA 98226
+       
+*************************************************************************/
+
+
+/* AIX demands this be the very first thing in the file. */
+#if !defined(__GNUC__) && defined(_AIX)
+ #pragma alloca
+#endif
+
+/* include system configuration before all else. */
+#include "autoconf.h"
+
+#include "gdbmdefs.h"
+#include "gdbmerrno.h"
+
+
+/* Add a new element to the database.  CONTENT is keyed by KEY.  The
+   file on disk is updated to reflect the structure of the new database
+   before returning from this procedure.  The FLAGS define the action to
+   take when the KEY is already in the database.  The value GDBM_REPLACE
+   asks that the old data be replaced by the new CONTENT.  The value
+   GDBM_INSERT asks that an error be returned and no action taken.  A
+   return value of 0 means no errors.  A return value of -1 means that
+   the item was not stored in the data base because the caller was not an
+   official writer. A return value of 0 means that the item was not stored
+   because the argument FLAGS was GDBM_INSERT and the KEY was already in
+   the database. */
+
+int
+gdbm_store (dbf, key, content, flags)
+     gdbm_file_info *dbf;
+     datum key;
+     datum content;
+     int flags;
+{
+  word_t new_hash_val;		/* The new hash value. */
+  int  elem_loc;		/* The location in hash bucket. */
+  off_t file_adr;		/* The address of new space in the file.  */
+  off_t file_pos;		/* The position after a lseek. */
+  int  num_bytes;		/* Used for error detection. */
+  off_t free_adr;		/* For keeping track of a freed section. */
+  int  free_size;
+
+  /* char *write_data; */	/* To write both key and data in 1 call. */
+  /* char *src;	*/		/* Used to prepare write_data. */
+  /* char *dst;	*/		/* Used to prepare write_data. */
+  /* int   cnt;	*/		/* Counter for loops to fill write_data. */
+  int   new_size;		/* Used in allocating space. */
+  char *temp;			/* Used in _gdbm_findkey call. */
+
+
+  /* First check to make sure this guy is a writer. */
+  if (dbf->read_write != GDBM_WRITER)
+    {
+      gdbm_errno = GDBM_READER_CANT_STORE;
+      return -1;
+    }
+
+  /* Check for illegal data values.  A NULL dptr field is illegal because
+     NULL dptr returned by a lookup procedure indicates an error. */
+  if ((key.dptr == NULL) || (content.dptr == NULL))
+    {
+      gdbm_errno = GDBM_ILLEGAL_DATA;
+      return -1;
+    }
+
+  /* Initialize the gdbm_errno variable. */
+  gdbm_errno = GDBM_NO_ERROR;
+
+  /* Look for the key in the file.
+     A side effect loads the correct bucket and calculates the hash value. */
+  elem_loc = _gdbm_findkey (dbf, key, &temp, &new_hash_val);
+
+
+  /* Did we find the item? */
+  if (elem_loc != -1)
+    {
+      if (flags == GDBM_REPLACE)
+	{
+	  /* Just replace the data. */
+	  free_adr = dbf->bucket->h_table[elem_loc].data_pointer;
+	  free_size = dbf->bucket->h_table[elem_loc].key_size
+	              + dbf->bucket->h_table[elem_loc].data_size;
+	  _gdbm_free (dbf, free_adr, free_size);
+	}
+      else
+	{
+	  gdbm_errno = GDBM_CANNOT_REPLACE;
+	  return 1;
+	}
+    }
+
+
+  /* Get the file address for the new space.
+     (Current bucket's free space is first place to look.) */
+  new_size = key.dsize+content.dsize;
+  file_adr = _gdbm_alloc (dbf, new_size);
+
+  /* If this is a new entry in the bucket, we need to do special things. */
+  if (elem_loc == -1)
+    {
+      if (dbf->bucket->count == dbf->header->bucket_elems)
+	{
+	  /* Split the current bucket. */
+	  _gdbm_split_bucket (dbf, new_hash_val);
+	}
+      
+      /* Find space to insert into bucket and set elem_loc to that place. */
+      elem_loc = new_hash_val % dbf->header->bucket_elems;
+      while (dbf->bucket->h_table[elem_loc].hash_value != -1)
+	{  elem_loc = (elem_loc + 1) % dbf->header->bucket_elems; }
+
+      /* We now have another element in the bucket.  Add the new information.*/
+      dbf->bucket->count += 1;
+      dbf->bucket->h_table[elem_loc].hash_value = new_hash_val;
+      bcopy (key.dptr, dbf->bucket->h_table[elem_loc].key_start,
+	     (SMALL < key.dsize ? SMALL : key.dsize));
+    }
+
+
+  /* Update current bucket data pointer and sizes. */
+  dbf->bucket->h_table[elem_loc].data_pointer = file_adr;
+  dbf->bucket->h_table[elem_loc].key_size = key.dsize;
+  dbf->bucket->h_table[elem_loc].data_size = content.dsize;
+
+  /* Write the data to the file. */
+  file_pos = lseek (dbf->desc, file_adr, L_SET);
+  if (file_pos != file_adr) _gdbm_fatal (dbf, "lseek error");
+  num_bytes = write (dbf->desc, key.dptr, key.dsize);
+  if (num_bytes != key.dsize) _gdbm_fatal (dbf, "write error");
+  num_bytes = write (dbf->desc, content.dptr, content.dsize);
+  if (num_bytes != content.dsize) _gdbm_fatal (dbf, "write error");
+
+  /* Current bucket has changed. */
+  dbf->cache_entry->ca_changed = TRUE;
+  dbf->bucket_changed = TRUE;
+
+  /* Write everything that is needed to the disk. */
+  _gdbm_end_update (dbf);
+  return 0;
+  
+}
Index: /tags/gsdl-21/gsdl/packages/wingdbm/gdbmsync.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/wingdbm/gdbmsync.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/wingdbm/gdbmsync.c	(revision 699)
@@ -0,0 +1,49 @@
+/* gdbmsync.c - Sync the disk with the in memory state. */
+
+/*  This file is part of GDBM, the GNU data base manager, by Philip A. Nelson.
+    Copyright (C) 1990, 1991, 1993  Free Software Foundation, Inc.
+
+    GDBM is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2, or (at your option)
+    any later version.
+
+    GDBM is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with GDBM; see the file COPYING.  If not, write to
+    the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+
+    You may contact the author by:
+       e-mail:  phil@cs.wwu.edu
+      us-mail:  Philip A. Nelson
+                Computer Science Department
+                Western Washington University
+                Bellingham, WA 98226
+       
+*************************************************************************/
+
+
+/* include system configuration before all else. */
+#include "autoconf.h"
+
+#include "gdbmdefs.h"
+#include "gdbmerrno.h"
+
+/* Make sure the database is all on disk. */
+
+void
+gdbm_sync (dbf)
+     gdbm_file_info *dbf;
+{
+
+  /* Initialize the gdbm_errno variable. */
+  gdbm_errno = GDBM_NO_ERROR;
+
+  /* Do the sync on the file. */
+  fsync (dbf->desc);
+
+}
Index: /tags/gsdl-21/gsdl/packages/wingdbm/getopt.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/wingdbm/getopt.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/wingdbm/getopt.c	(revision 699)
@@ -0,0 +1,735 @@
+/* Getopt for GNU.
+   NOTE: getopt is now part of the C library, so if you don't know what
+   "Keep this file name-space clean" means, talk to roland@gnu.ai.mit.edu
+   before changing it!
+
+   Copyright (C) 1987, 88, 89, 90, 91, 92, 1993
+   	Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the
+   Free Software Foundation; either version 2, or (at your option) any
+   later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+
+/* NOTE!!!  AIX requires this to be the first thing in the file.
+   Do not put ANYTHING before it!  */
+#if !defined (__GNUC__) && defined (_AIX)
+ #pragma alloca
+#endif
+
+#include "autoconf.h"
+
+#ifdef __GNUC__
+#define alloca __builtin_alloca
+#else /* not __GNUC__ */
+#if defined (HAVE_ALLOCA_H) || (defined(sparc) && (defined(sun) || (!defined(USG) && !defined(SVR4) && !defined(__svr4__))))
+#include <alloca.h>
+#else
+#ifndef _AIX
+char *alloca ();
+#endif
+#endif /* alloca.h */
+#endif /* not __GNUC__ */
+
+#if !__STDC__ && !defined(const) && IN_GCC
+#define const
+#endif
+
+/* This tells Alpha OSF/1 not to define a getopt prototype in <stdio.h>.  */
+#ifndef _NO_PROTO
+#define _NO_PROTO
+#endif
+
+#include <string.h>
+#include <stdio.h>
+
+/* Comment out all this code if we are using the GNU C Library, and are not
+   actually compiling the library itself.  This code is part of the GNU C
+   Library, but also included in many other GNU distributions.  Compiling
+   and linking in this code is a waste when using the GNU C library
+   (especially if it is a shared library).  Rather than having every GNU
+   program understand `configure --with-gnu-libc' and omit the object files,
+   it is simpler to just do this in the source for each such file.  */
+
+#if defined (_LIBC) || !defined (__GNU_LIBRARY__)
+
+
+/* This needs to come after some library #include
+   to get __GNU_LIBRARY__ defined.  */
+#ifdef	__GNU_LIBRARY__
+#undef	alloca
+/* Don't include stdlib.h for non-GNU C libraries because some of them
+   contain conflicting prototypes for getopt.  */
+#include <stdlib.h>
+#else	/* Not GNU C library.  */
+#define	__alloca	alloca
+#endif	/* GNU C library.  */
+
+/* If GETOPT_COMPAT is defined, `+' as well as `--' can introduce a
+   long-named option.  Because this is not POSIX.2 compliant, it is
+   being phased out.  */
+/* #define GETOPT_COMPAT */
+
+/* This version of `getopt' appears to the caller like standard Unix `getopt'
+   but it behaves differently for the user, since it allows the user
+   to intersperse the options with the other arguments.
+
+   As `getopt' works, it permutes the elements of ARGV so that,
+   when it is done, all the options precede everything else.  Thus
+   all application programs are extended to handle flexible argument order.
+
+   Setting the environment variable POSIXLY_CORRECT disables permutation.
+   Then the behavior is completely standard.
+
+   GNU application programs can use a third alternative mode in which
+   they can distinguish the relative order of options and other arguments.  */
+
+#include "getopt.h"
+
+/* For communication from `getopt' to the caller.
+   When `getopt' finds an option that takes an argument,
+   the argument value is returned here.
+   Also, when `ordering' is RETURN_IN_ORDER,
+   each non-option ARGV-element is returned here.  */
+
+char *optarg = 0;
+
+/* Index in ARGV of the next element to be scanned.
+   This is used for communication to and from the caller
+   and for communication between successive calls to `getopt'.
+
+   On entry to `getopt', zero means this is the first call; initialize.
+
+   When `getopt' returns EOF, this is the index of the first of the
+   non-option elements that the caller should itself scan.
+
+   Otherwise, `optind' communicates from one call to the next
+   how much of ARGV has been scanned so far.  */
+
+/* XXX 1003.2 says this must be 1 before any call.  */
+int optind = 0;
+
+/* The next char to be scanned in the option-element
+   in which the last option character we returned was found.
+   This allows us to pick up the scan where we left off.
+
+   If this is zero, or a null string, it means resume the scan
+   by advancing to the next ARGV-element.  */
+
+static char *nextchar;
+
+/* Callers store zero here to inhibit the error message
+   for unrecognized options.  */
+
+int opterr = 1;
+
+/* Set to an option character which was unrecognized.
+   This must be initialized on some systems to avoid linking in the
+   system's own getopt implementation.  */
+
+int optopt = '?';
+
+/* Describe how to deal with options that follow non-option ARGV-elements.
+
+   If the caller did not specify anything,
+   the default is REQUIRE_ORDER if the environment variable
+   POSIXLY_CORRECT is defined, PERMUTE otherwise.
+
+   REQUIRE_ORDER means don't recognize them as options;
+   stop option processing when the first non-option is seen.
+   This is what Unix does.
+   This mode of operation is selected by either setting the environment
+   variable POSIXLY_CORRECT, or using `+' as the first character
+   of the list of option characters.
+
+   PERMUTE is the default.  We permute the contents of ARGV as we scan,
+   so that eventually all the non-options are at the end.  This allows options
+   to be given in any order, even with programs that were not written to
+   expect this.
+
+   RETURN_IN_ORDER is an option available to programs that were written
+   to expect options and other ARGV-elements in any order and that care about
+   the ordering of the two.  We describe each non-option ARGV-element
+   as if it were the argument of an option with character code 1.
+   Using `-' as the first character of the list of option characters
+   selects this mode of operation.
+
+   The special argument `--' forces an end of option-scanning regardless
+   of the value of `ordering'.  In the case of RETURN_IN_ORDER, only
+   `--' can cause `getopt' to return EOF with `optind' != ARGC.  */
+
+static enum
+{
+  REQUIRE_ORDER, PERMUTE, RETURN_IN_ORDER
+} ordering;
+
+
+#ifdef	__GNU_LIBRARY__
+/* We want to avoid inclusion of string.h with non-GNU libraries
+   because there are many ways it can cause trouble.
+   On some systems, it contains special magic macros that don't work
+   in GCC.  */
+#include <string.h>
+#define	my_index	strchr
+#define	my_bcopy(src, dst, n)	memcpy ((dst), (src), (n))
+#else
+
+/* Avoid depending on library functions or files
+   whose names are inconsistent.  */
+
+char *getenv ();
+
+static char *
+my_index (str, chr)
+     const char *str;
+     int chr;
+{
+  while (*str)
+    {
+      if (*str == chr)
+	return (char *) str;
+      str++;
+    }
+  return 0;
+}
+
+static void
+my_bcopy (from, to, size)
+     const char *from;
+     char *to;
+     int size;
+{
+  int i;
+  for (i = 0; i < size; i++)
+    to[i] = from[i];
+}
+#endif				/* GNU C library.  */
+
+
+/* Handle permutation of arguments.  */
+
+/* Describe the part of ARGV that contains non-options that have
+   been skipped.  `first_nonopt' is the index in ARGV of the first of them;
+   `last_nonopt' is the index after the last of them.  */
+
+static int first_nonopt;
+static int last_nonopt;
+
+/* Exchange two adjacent subsequences of ARGV.
+   One subsequence is elements [first_nonopt,last_nonopt)
+   which contains all the non-options that have been skipped so far.
+   The other is elements [last_nonopt,optind), which contains all
+   the options processed since those non-options were skipped.
+
+   `first_nonopt' and `last_nonopt' are relocated so that they describe
+   the new indices of the non-options in ARGV after they are moved.  */
+
+static void
+exchange (argv)
+     char **argv;
+{
+  int nonopts_size = (last_nonopt - first_nonopt) * sizeof (char *);
+  char **temp = (char **) __alloca (nonopts_size);
+
+  /* Interchange the two blocks of data in ARGV.  */
+
+  my_bcopy ((char *) &argv[first_nonopt], (char *) temp, nonopts_size);
+  my_bcopy ((char *) &argv[last_nonopt], (char *) &argv[first_nonopt],
+	    (optind - last_nonopt) * sizeof (char *));
+  my_bcopy ((char *) temp,
+	    (char *) &argv[first_nonopt + optind - last_nonopt],
+	    nonopts_size);
+
+  /* Update records for the slots the non-options now occupy.  */
+
+  first_nonopt += (optind - last_nonopt);
+  last_nonopt = optind;
+}
+
+
+/* Scan elements of ARGV (whose length is ARGC) for option characters
+   given in OPTSTRING.
+
+   If an element of ARGV starts with '-', and is not exactly "-" or "--",
+   then it is an option element.  The characters of this element
+   (aside from the initial '-') are option characters.  If `getopt'
+   is called repeatedly, it returns successively each of the option characters
+   from each of the option elements.
+
+   If `getopt' finds another option character, it returns that character,
+   updating `optind' and `nextchar' so that the next call to `getopt' can
+   resume the scan with the following option character or ARGV-element.
+
+   If there are no more option characters, `getopt' returns `EOF'.
+   Then `optind' is the index in ARGV of the first ARGV-element
+   that is not an option.  (The ARGV-elements have been permuted
+   so that those that are not options now come last.)
+
+   OPTSTRING is a string containing the legitimate option characters.
+   If an option character is seen that is not listed in OPTSTRING,
+   return '?' after printing an error message.  If you set `opterr' to
+   zero, the error message is suppressed but we still return '?'.
+
+   If a char in OPTSTRING is followed by a colon, that means it wants an arg,
+   so the following text in the same ARGV-element, or the text of the following
+   ARGV-element, is returned in `optarg'.  Two colons mean an option that
+   wants an optional arg; if there is text in the current ARGV-element,
+   it is returned in `optarg', otherwise `optarg' is set to zero.
+
+   If OPTSTRING starts with `-' or `+', it requests different methods of
+   handling the non-option ARGV-elements.
+   See the comments about RETURN_IN_ORDER and REQUIRE_ORDER, above.
+
+   Long-named options begin with `--' instead of `-'.
+   Their names may be abbreviated as long as the abbreviation is unique
+   or is an exact match for some defined option.  If they have an
+   argument, it follows the option name in the same ARGV-element, separated
+   from the option name by a `=', or else the in next ARGV-element.
+   When `getopt' finds a long-named option, it returns 0 if that option's
+   `flag' field is nonzero, the value of the option's `val' field
+   if the `flag' field is zero.
+
+   The elements of ARGV aren't really const, because we permute them.
+   But we pretend they're const in the prototype to be compatible
+   with other systems.
+
+   LONGOPTS is a vector of `struct option' terminated by an
+   element containing a name which is zero.
+
+   LONGIND returns the index in LONGOPT of the long-named option found.
+   It is only valid when a long-named option has been found by the most
+   recent call.
+
+   If LONG_ONLY is nonzero, '-' as well as '--' can introduce
+   long-named options.  */
+
+int
+_getopt_internal (argc, argv, optstring, longopts, longind, long_only)
+     int argc;
+     char *const *argv;
+     const char *optstring;
+     const struct option *longopts;
+     int *longind;
+     int long_only;
+{
+  int option_index;
+
+  optarg = 0;
+
+  /* Initialize the internal data when the first call is made.
+     Start processing options with ARGV-element 1 (since ARGV-element 0
+     is the program name); the sequence of previously skipped
+     non-option ARGV-elements is empty.  */
+
+  if (optind == 0)
+    {
+      first_nonopt = last_nonopt = optind = 1;
+
+      nextchar = NULL;
+
+      /* Determine how to handle the ordering of options and nonoptions.  */
+
+      if (optstring[0] == '-')
+	{
+	  ordering = RETURN_IN_ORDER;
+	  ++optstring;
+	}
+      else if (optstring[0] == '+')
+	{
+	  ordering = REQUIRE_ORDER;
+	  ++optstring;
+	}
+      else if (getenv ("POSIXLY_CORRECT") != NULL)
+	ordering = REQUIRE_ORDER;
+      else
+	ordering = PERMUTE;
+    }
+
+  if (nextchar == NULL || *nextchar == '\0')
+    {
+      if (ordering == PERMUTE)
+	{
+	  /* If we have just processed some options following some non-options,
+	     exchange them so that the options come first.  */
+
+	  if (first_nonopt != last_nonopt && last_nonopt != optind)
+	    exchange ((char **) argv);
+	  else if (last_nonopt != optind)
+	    first_nonopt = optind;
+
+	  /* Now skip any additional non-options
+	     and extend the range of non-options previously skipped.  */
+
+	  while (optind < argc
+		 && (argv[optind][0] != '-' || argv[optind][1] == '\0')
+#ifdef GETOPT_COMPAT
+		 && (longopts == NULL
+		     || argv[optind][0] != '+' || argv[optind][1] == '\0')
+#endif				/* GETOPT_COMPAT */
+		 )
+	    optind++;
+	  last_nonopt = optind;
+	}
+
+      /* Special ARGV-element `--' means premature end of options.
+	 Skip it like a null option,
+	 then exchange with previous non-options as if it were an option,
+	 then skip everything else like a non-option.  */
+
+      if (optind != argc && !strcmp (argv[optind], "--"))
+	{
+	  optind++;
+
+	  if (first_nonopt != last_nonopt && last_nonopt != optind)
+	    exchange ((char **) argv);
+	  else if (first_nonopt == last_nonopt)
+	    first_nonopt = optind;
+	  last_nonopt = argc;
+
+	  optind = argc;
+	}
+
+      /* If we have done all the ARGV-elements, stop the scan
+	 and back over any non-options that we skipped and permuted.  */
+
+      if (optind == argc)
+	{
+	  /* Set the next-arg-index to point at the non-options
+	     that we previously skipped, so the caller will digest them.  */
+	  if (first_nonopt != last_nonopt)
+	    optind = first_nonopt;
+	  return EOF;
+	}
+
+      /* If we have come to a non-option and did not permute it,
+	 either stop the scan or describe it to the caller and pass it by.  */
+
+      if ((argv[optind][0] != '-' || argv[optind][1] == '\0')
+#ifdef GETOPT_COMPAT
+	  && (longopts == NULL
+	      || argv[optind][0] != '+' || argv[optind][1] == '\0')
+#endif				/* GETOPT_COMPAT */
+	  )
+	{
+	  if (ordering == REQUIRE_ORDER)
+	    return EOF;
+	  optarg = argv[optind++];
+	  return 1;
+	}
+
+      /* We have found another option-ARGV-element.
+	 Start decoding its characters.  */
+
+      nextchar = (argv[optind] + 1
+		  + (longopts != NULL && argv[optind][1] == '-'));
+    }
+
+  if (longopts != NULL
+      && ((argv[optind][0] == '-'
+	   && (argv[optind][1] == '-' || long_only))
+#ifdef GETOPT_COMPAT
+	  || argv[optind][0] == '+'
+#endif				/* GETOPT_COMPAT */
+	  ))
+    {
+      const struct option *p;
+      char *s = nextchar;
+      int exact = 0;
+      int ambig = 0;
+      const struct option *pfound = NULL;
+      int indfound;
+
+      while (*s && *s != '=')
+	s++;
+
+      /* Test all options for either exact match or abbreviated matches.  */
+      for (p = longopts, option_index = 0; p->name;
+	   p++, option_index++)
+	if (!strncmp (p->name, nextchar, s - nextchar))
+	  {
+	    if ((unsigned int)(s - nextchar) == strlen (p->name))
+		{ 
+		/* Exact match found.  */
+		pfound = p;
+		indfound = option_index;
+		exact = 1;
+		break;
+	      }
+	    else if (pfound == NULL)
+	      {
+		/* First nonexact match found.  */
+		pfound = p;
+		indfound = option_index;
+	      }
+	    else
+	      /* Second nonexact match found.  */
+	      ambig = 1;
+	  }
+
+      if (ambig && !exact)
+	{
+	  if (opterr)
+	    fprintf (stderr, "%s: option `%s' is ambiguous\n",
+		     argv[0], argv[optind]);
+	  nextchar += strlen (nextchar);
+	  optind++;
+	  return '?';
+	}
+
+      if (pfound != NULL)
+	{
+	  option_index = indfound;
+	  optind++;
+	  if (*s)
+	    {
+	      /* Don't test has_arg with >, because some C compilers don't
+		 allow it to be used on enums.  */
+	      if (pfound->has_arg)
+		optarg = s + 1;
+	      else
+		{
+		  if (opterr)
+		    {
+		      if (argv[optind - 1][1] == '-')
+			/* --option */
+			fprintf (stderr,
+				 "%s: option `--%s' doesn't allow an argument\n",
+				 argv[0], pfound->name);
+		      else
+			/* +option or -option */
+			fprintf (stderr,
+			     "%s: option `%c%s' doesn't allow an argument\n",
+			     argv[0], argv[optind - 1][0], pfound->name);
+		    }
+		  nextchar += strlen (nextchar);
+		  return '?';
+		}
+	    }
+	  else if (pfound->has_arg == 1)
+	    {
+	      if (optind < argc)
+		optarg = argv[optind++];
+	      else
+		{
+		  if (opterr)
+		    fprintf (stderr, "%s: option `%s' requires an argument\n",
+			     argv[0], argv[optind - 1]);
+		  nextchar += strlen (nextchar);
+		  return optstring[0] == ':' ? ':' : '?';
+		}
+	    }
+	  nextchar += strlen (nextchar);
+	  if (longind != NULL)
+	    *longind = option_index;
+	  if (pfound->flag)
+	    {
+	      *(pfound->flag) = pfound->val;
+	      return 0;
+	    }
+	  return pfound->val;
+	}
+      /* Can't find it as a long option.  If this is not getopt_long_only,
+	 or the option starts with '--' or is not a valid short
+	 option, then it's an error.
+	 Otherwise interpret it as a short option.  */
+      if (!long_only || argv[optind][1] == '-'
+#ifdef GETOPT_COMPAT
+	  || argv[optind][0] == '+'
+#endif				/* GETOPT_COMPAT */
+	  || my_index (optstring, *nextchar) == NULL)
+	{
+	  if (opterr)
+	    {
+	      if (argv[optind][1] == '-')
+		/* --option */
+		fprintf (stderr, "%s: unrecognized option `--%s'\n",
+			 argv[0], nextchar);
+	      else
+		/* +option or -option */
+		fprintf (stderr, "%s: unrecognized option `%c%s'\n",
+			 argv[0], argv[optind][0], nextchar);
+	    }
+	  nextchar = (char *) "";
+	  optind++;
+	  return '?';
+	}
+    }
+
+  /* Look at and handle the next option-character.  */
+
+  {
+    char c = *nextchar++;
+    char *temp = my_index (optstring, c);
+
+    /* Increment `optind' when we start to process its last character.  */
+    if (*nextchar == '\0')
+      ++optind;
+
+    if (temp == NULL || c == ':')
+      {
+	if (opterr)
+	  {
+#if 0
+	    if (c < 040 || c >= 0177)
+	      fprintf (stderr, "%s: unrecognized option, character code 0%o\n",
+		       argv[0], c);
+	    else
+	      fprintf (stderr, "%s: unrecognized option `-%c'\n", argv[0], c);
+#else
+	    /* 1003.2 specifies the format of this message.  */
+	    fprintf (stderr, "%s: illegal option -- %c\n", argv[0], c);
+#endif
+	  }
+	optopt = c;
+	return '?';
+      }
+    if (temp[1] == ':')
+      {
+	if (temp[2] == ':')
+	  {
+	    /* This is an option that accepts an argument optionally.  */
+	    if (*nextchar != '\0')
+	      {
+		optarg = nextchar;
+		optind++;
+	      }
+	    else
+	      optarg = 0;
+	    nextchar = NULL;
+	  }
+	else
+	  {
+	    /* This is an option that requires an argument.  */
+	    if (*nextchar != '\0')
+	      {
+		optarg = nextchar;
+		/* If we end this ARGV-element by taking the rest as an arg,
+		   we must advance to the next element now.  */
+		optind++;
+	      }
+	    else if (optind == argc)
+	      {
+		if (opterr)
+		  {
+#if 0
+		    fprintf (stderr, "%s: option `-%c' requires an argument\n",
+			     argv[0], c);
+#else
+		    /* 1003.2 specifies the format of this message.  */
+		    fprintf (stderr, "%s: option requires an argument -- %c\n",
+			     argv[0], c);
+#endif
+		  }
+		optopt = c;
+		if (optstring[0] == ':')
+		  c = ':';
+		else
+		  c = '?';
+	      }
+	    else
+	      /* We already incremented `optind' once;
+		 increment it again when taking next ARGV-elt as argument.  */
+	      optarg = argv[optind++];
+	    nextchar = NULL;
+	  }
+      }
+    return c;
+  }
+}
+
+int
+getopt (argc, argv, optstring)
+     int argc;
+     char *const *argv;
+     const char *optstring;
+{
+  return _getopt_internal (argc, argv, optstring,
+			   (const struct option *) 0,
+			   (int *) 0,
+			   0);
+}
+
+#endif	/* _LIBC or not __GNU_LIBRARY__.  */
+
+
+#ifdef TEST
+
+/* Compile with -DTEST to make an executable for use in testing
+   the above definition of `getopt'.  */
+
+int
+main (argc, argv)
+     int argc;
+     char **argv;
+{
+  int c;
+  int digit_optind = 0;
+
+  while (1)
+    {
+      int this_option_optind = optind ? optind : 1;
+
+      c = getopt (argc, argv, "abc:d:0123456789");
+      if (c == EOF)
+	break;
+
+      switch (c)
+	{
+	case '0':
+	case '1':
+	case '2':
+	case '3':
+	case '4':
+	case '5':
+	case '6':
+	case '7':
+	case '8':
+	case '9':
+	  if (digit_optind != 0 && digit_optind != this_option_optind)
+	    printf ("digits occur in two different argv-elements.\n");
+	  digit_optind = this_option_optind;
+	  printf ("option %c\n", c);
+	  break;
+
+	case 'a':
+	  printf ("option a\n");
+	  break;
+
+	case 'b':
+	  printf ("option b\n");
+	  break;
+
+	case 'c':
+	  printf ("option c with value `%s'\n", optarg);
+	  break;
+
+	case '?':
+	  break;
+
+	default:
+	  printf ("?? getopt returned character code 0%o ??\n", c);
+	}
+    }
+
+  if (optind < argc)
+    {
+      printf ("non-option ARGV-elements: ");
+      while (optind < argc)
+	printf ("%s ", argv[optind++]);
+      printf ("\n");
+    }
+
+  exit (0);
+}
+
+#endif /* TEST */
Index: /tags/gsdl-21/gsdl/packages/wingdbm/getopt.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/wingdbm/getopt.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/wingdbm/getopt.h	(revision 699)
@@ -0,0 +1,129 @@
+/* Declarations for getopt.
+   Copyright (C) 1989, 1990, 1991, 1992, 1993 Free Software Foundation, Inc.
+
+   This program is free software; you can redistribute it and/or modify it
+   under the terms of the GNU General Public License as published by the
+   Free Software Foundation; either version 2, or (at your option) any
+   later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.  */
+
+#ifndef _GETOPT_H
+#define _GETOPT_H 1
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+/* For communication from `getopt' to the caller.
+   When `getopt' finds an option that takes an argument,
+   the argument value is returned here.
+   Also, when `ordering' is RETURN_IN_ORDER,
+   each non-option ARGV-element is returned here.  */
+
+extern char *optarg;
+
+/* Index in ARGV of the next element to be scanned.
+   This is used for communication to and from the caller
+   and for communication between successive calls to `getopt'.
+
+   On entry to `getopt', zero means this is the first call; initialize.
+
+   When `getopt' returns EOF, this is the index of the first of the
+   non-option elements that the caller should itself scan.
+
+   Otherwise, `optind' communicates from one call to the next
+   how much of ARGV has been scanned so far.  */
+
+extern int optind;
+
+/* Callers store zero here to inhibit the error message `getopt' prints
+   for unrecognized options.  */
+
+extern int opterr;
+
+/* Set to an option character which was unrecognized.  */
+
+extern int optopt;
+
+/* Describe the long-named options requested by the application.
+   The LONG_OPTIONS argument to getopt_long or getopt_long_only is a vector
+   of `struct option' terminated by an element containing a name which is
+   zero.
+
+   The field `has_arg' is:
+   no_argument		(or 0) if the option does not take an argument,
+   required_argument	(or 1) if the option requires an argument,
+   optional_argument 	(or 2) if the option takes an optional argument.
+
+   If the field `flag' is not NULL, it points to a variable that is set
+   to the value given in the field `val' when the option is found, but
+   left unchanged if the option is not found.
+
+   To have a long-named option do something other than set an `int' to
+   a compiled-in constant, such as set a value from `optarg', set the
+   option's `flag' field to zero and its `val' field to a nonzero
+   value (the equivalent single-letter option character, if there is
+   one).  For long options that have a zero `flag' field, `getopt'
+   returns the contents of the `val' field.  */
+
+struct option
+{
+#if	__STDC__
+  const char *name;
+#else
+  char *name;
+#endif
+  /* has_arg can't be an enum because some compilers complain about
+     type mismatches in all the code that assumes it is an int.  */
+  int has_arg;
+  int *flag;
+  int val;
+};
+
+/* Names for the values of the `has_arg' field of `struct option'.  */
+
+#define	no_argument		0
+#define required_argument	1
+#define optional_argument	2
+
+#if __STDC__
+#if defined(__GNU_LIBRARY__)
+/* Many other libraries have conflicting prototypes for getopt, with
+   differences in the consts, in stdlib.h.  To avoid compilation
+   errors, only prototype getopt for the GNU C library.  */
+extern int getopt (int argc, char *const *argv, const char *shortopts);
+#else /* not __GNU_LIBRARY__ */
+extern int getopt ();
+#endif /* not __GNU_LIBRARY__ */
+extern int getopt_long (int argc, char *const *argv, const char *shortopts,
+		        const struct option *longopts, int *longind);
+extern int getopt_long_only (int argc, char *const *argv,
+			     const char *shortopts,
+		             const struct option *longopts, int *longind);
+
+/* Internal only.  Users should not call this directly.  */
+extern int _getopt_internal (int argc, char *const *argv,
+			     const char *shortopts,
+		             const struct option *longopts, int *longind,
+			     int long_only);
+#else /* not __STDC__ */
+extern int getopt ();
+extern int getopt_long ();
+extern int getopt_long_only ();
+
+extern int _getopt_internal ();
+#endif /* not __STDC__ */
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif /* _GETOPT_H */
Index: /tags/gsdl-21/gsdl/packages/wingdbm/global.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/wingdbm/global.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/wingdbm/global.c	(revision 699)
@@ -0,0 +1,78 @@
+/* global.c - The external variables needed for "original" interface and
+   error messages. */
+
+/*  This file is part of GDBM, the GNU data base manager, by Philip A. Nelson.
+    Copyright (C) 1990, 1991, 1993  Free Software Foundation, Inc.
+
+    GDBM is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2, or (at your option)
+    any later version.
+
+    GDBM is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with GDBM; see the file COPYING.  If not, write to
+    the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+
+    You may contact the author by:
+       e-mail:  phil@cs.wwu.edu
+      us-mail:  Philip A. Nelson
+                Computer Science Department
+                Western Washington University
+                Bellingham, WA 98226
+       
+*************************************************************************/
+
+
+/* include system configuration before all else. */
+#include "autoconf.h"
+
+#include "gdbmdefs.h"
+#include "gdbmerrno.h"
+
+#include <io.h>
+#include <windows.h>
+
+/* The global variables used for the "original" interface. */
+gdbm_file_info  *_gdbm_file = NULL;
+
+/* Memory for return data for the "original" interface. */
+datum _gdbm_memory = {NULL, 0};	/* Used by firstkey and nextkey. */
+char *_gdbm_fetch_val = NULL;	/* Used by fetch. */
+
+/* The dbm error number is placed in the variable GDBM_ERRNO. */
+gdbm_error gdbm_errno = GDBM_NO_ERROR;
+
+
+/* a simple locking function for windows that uses file descriptors */
+int my_locking(int fh, int lmode, long lockoffset, long nbytes) {
+	int result = -1;
+	int trys = 10;
+
+	lseek (fh, lockoffset, L_SET);
+
+	while (trys > 0 && result != 0) {
+		if (lmode == MY_UNLOCK) {
+			if (UnlockFile((HANDLE)_get_osfhandle(fh),lockoffset,0,nbytes,0))
+				result = 0;
+
+		} else if (lmode == MY_READLOCK) {
+/*			if (LockFileEx((HANDLE)_get_osfhandle(fh),0,0,nbytes,0,0))*/
+			if (LockFile((HANDLE)_get_osfhandle(fh),lockoffset,0,nbytes,0))
+				result = 0;
+
+		} else if (lmode == MY_WRITELOCK) {
+			if (LockFile((HANDLE)_get_osfhandle(fh),lockoffset,0,nbytes,0))
+				result = 0;
+		}
+
+		Sleep(100L);
+		trys--;
+	}
+
+	return result;
+}
Index: /tags/gsdl-21/gsdl/packages/wingdbm/hash.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/wingdbm/hash.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/wingdbm/hash.c	(revision 699)
@@ -0,0 +1,62 @@
+/* hash.c - The gdbm hash function. */
+
+/*  This file is part of GDBM, the GNU data base manager, by Philip A. Nelson.
+    Copyright (C) 1990, 1991, 1993  Free Software Foundation, Inc.
+
+    GDBM is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2, or (at your option)
+    any later version.
+
+    GDBM is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with GDBM; see the file COPYING.  If not, write to
+    the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+
+    You may contact the author by:
+       e-mail:  phil@cs.wwu.edu
+      us-mail:  Philip A. Nelson
+                Computer Science Department
+                Western Washington University
+                Bellingham, WA 98226
+       
+*************************************************************************/
+
+
+/* include system configuration before all else. */
+#include "autoconf.h"
+
+#include "gdbmdefs.h"
+
+
+/* This hash function computes a 31 bit value.  The value is used to index
+   the hash directory using the top n bits.  It is also used in a hash bucket
+   to find the home position of the element by taking the value modulo the
+   bucket hash table size. */
+
+word_t
+_gdbm_hash (key)
+     datum key;
+{
+#if LONG_64_BITS || !INT_16_BITS
+  unsigned int value;	/* Used to compute the hash value.  */
+#else
+  unsigned long value;	/* Used to compute the hash value.  */
+#endif
+  int   index;		/* Used to cycle through random values. */
+
+
+  /* Set the initial value from key. */
+  value = 0x238F13AF * key.dsize;
+  for (index = 0; index < key.dsize; index++)
+    value = (value + (key.dptr[index] << (index*5 % 24))) & 0x7FFFFFFF;
+
+  value = (1103515243 * value + 12345) & 0x7FFFFFFF;  
+
+  /* Return the value. */
+  return((word_t) value);
+}
Index: /tags/gsdl-21/gsdl/packages/wingdbm/proto.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/wingdbm/proto.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/wingdbm/proto.h	(revision 699)
@@ -0,0 +1,87 @@
+/* proto.h - The prototypes for the dbm routines. */
+
+/*  This file is part of GDBM, the GNU data base manager, by Philip A. Nelson.
+    Copyright (C) 1990, 1991, 1993  Free Software Foundation, Inc.
+
+    GDBM is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2, or (at your option)
+    any later version.
+
+    GDBM is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with GDBM; see the file COPYING.  If not, write to
+    the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+
+    You may contact the author by:
+       e-mail:  phil@cs.wwu.edu
+      us-mail:  Philip A. Nelson
+                Computer Science Department
+                Western Washington University
+                Bellingham, WA 98226
+
+*************************************************************************/
+
+
+/* Define our _PROTOTYPE macro. */
+
+#ifdef _ARGS
+#undef _ARGS
+#endif
+
+#ifdef __STDC__
+#define _ARGS(args) args
+#else
+#define _ARGS(args) ()
+#endif
+
+
+/* From bucket.c */
+void _gdbm_new_bucket    _ARGS((gdbm_file_info *dbf, hash_bucket *bucket,
+				int bits ));
+void _gdbm_get_bucket    _ARGS((gdbm_file_info *dbf, word_t dir_index ));
+void _gdbm_split_bucket  _ARGS((gdbm_file_info *dbf, word_t next_insert ));
+void _gdbm_write_bucket  _ARGS((gdbm_file_info *dbf, cache_elem *ca_entry ));
+
+/* From falloc.c */
+off_t _gdbm_alloc       _ARGS((gdbm_file_info *dbf, int num_bytes ));
+void _gdbm_free         _ARGS((gdbm_file_info *dbf, off_t file_adr,
+			       int num_bytes ));
+int  _gdbm_put_av_elem  _ARGS((avail_elem new_el, avail_elem av_table [],
+			       int *av_count ));
+
+/* From findkey.c */
+char *_gdbm_read_entry  _ARGS((gdbm_file_info *dbf, int elem_loc));
+int _gdbm_findkey       _ARGS((gdbm_file_info *dbf, datum key,  char **dptr,
+			       word_t *new_hash_val));
+
+/* From hash.c */
+word_t _gdbm_hash _ARGS((datum key ));
+
+/* From update.c */
+void _gdbm_end_update   _ARGS((gdbm_file_info *dbf ));
+int _gdbm_fatal         _ARGS((gdbm_file_info *dbf, char *val ));
+
+/* From gdbmopen.c */
+int _gdbm_init_cache	_ARGS((gdbm_file_info *dbf, int size));
+
+/* The user callable routines.... */
+void  gdbm_close	  _ARGS((gdbm_file_info *dbf ));
+int   gdbm_delete	  _ARGS((gdbm_file_info *dbf, datum key ));
+datum gdbm_fetch	  _ARGS((gdbm_file_info *dbf, datum key ));
+gdbm_file_info *gdbm_open _ARGS((char *file, int block_size,
+				 int flags, int mode,
+				 void (*fatal_func) (void), int need_filelock));
+int   gdbm_reorganize	  _ARGS((gdbm_file_info *dbf ));
+datum gdbm_firstkey       _ARGS((gdbm_file_info *dbf ));
+datum gdbm_nextkey        _ARGS((gdbm_file_info *dbf, datum key ));
+int   gdbm_store          _ARGS((gdbm_file_info *dbf, datum key,
+				 datum content, int flags ));
+int   gdbm_exists	  _ARGS((gdbm_file_info *dbf, datum key));
+void  gdbm_sync		  _ARGS((gdbm_file_info *dbf));
+int   gdbm_setopt	  _ARGS((gdbm_file_info *dbf, int optflag,
+				 int *optval, int optlen));
Index: /tags/gsdl-21/gsdl/packages/wingdbm/systems.h
===================================================================
--- /tags/gsdl-21/gsdl/packages/wingdbm/systems.h	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/wingdbm/systems.h	(revision 699)
@@ -0,0 +1,195 @@
+/* systems.h - Most of the system dependant code and defines are here. */
+
+/*  This file is part of GDBM, the GNU data base manager, by Philip A. Nelson.
+    Copyright (C) 1990, 1991, 1993  Free Software Foundation, Inc.
+
+    GDBM is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2, or (at your option)
+    any later version.
+
+    GDBM is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with GDBM; see the file COPYING.  If not, write to
+    the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+
+    You may contact the author by:
+       e-mail:  phil@wwu.edu
+      us-mail:  Philip A. Nelson
+                Computer Science Department
+                Western Washington University
+                Bellingham, WA 98226
+       
+*************************************************************************/
+
+
+#ifdef __GNUC__
+#define alloca	__builtin_alloca
+#else	/* not __GNUC__ */
+#ifdef HAVE_ALLOCA_H
+#include <alloca.h>
+#endif	/* not HAVE_ALLOCA_H */
+#endif 	/* not __GNUC__ */
+
+/* Include all system headers first. */
+#if HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+
+#include <stdio.h>
+#if HAVE_SYS_FILE_H
+#include <sys/file.h>
+#endif
+
+#include <sys/stat.h>
+#if HAVE_STDLIB_H
+#include <stdlib.h>
+#endif
+
+#if HAVE_STRING_H
+#include <string.h>
+#else
+#include <strings.h>
+#endif
+
+#if HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#if HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+
+#ifndef SEEK_SET
+#define SEEK_SET        0
+#endif
+
+#ifndef L_SET
+#define L_SET SEEK_SET
+#endif
+
+/* Do we have flock?  (BSD...) */
+
+#if HAVE_FLOCK
+
+#ifndef LOCK_SH
+#define LOCK_SH	1
+#endif
+
+#ifndef LOCK_EX
+#define LOCK_EX	2
+#endif
+
+#ifndef LOCK_NB
+#define LOCK_NB 4
+#endif
+
+#ifndef LOCK_UN
+#define LOCK_UN 8
+#endif
+
+#define UNLOCK_FILE(dbf) flock (dbf->desc, LOCK_UN)
+#define READLOCK_FILE(dbf) lock_val = flock (dbf->desc, LOCK_SH + LOCK_NB)
+#define WRITELOCK_FILE(dbf) lock_val = flock (dbf->desc, LOCK_EX + LOCK_NB)
+
+#else
+
+#ifdef MSDOS
+#define MY_UNLOCK    0
+#define MY_READLOCK  1
+#define MY_WRITELOCK 2
+int my_locking(int fh, int lmode, long lockoffset, long nbytes);
+
+#define UNLOCK_FILE(dbf)    my_locking(dbf->desc,MY_UNLOCK,0,10)
+#define READLOCK_FILE(dbf)  lock_val=my_locking(dbf->desc,MY_READLOCK,0,10)
+#define WRITELOCK_FILE(dbf)	lock_val=my_locking(dbf->desc,MY_WRITELOCK,0,10)
+
+#else
+/* Assume it is done like System V. */
+
+#define UNLOCK_FILE(dbf) \
+	{					\
+	  struct flock flock;			\
+	  flock.l_type = F_UNLCK;		\
+	  flock.l_whence = SEEK_SET;		\
+	  flock.l_start = flock.l_len = 0L;	\
+	  fcntl (dbf->desc, F_SETLK, &flock);	\
+	}
+#define READLOCK_FILE(dbf) \
+	{					\
+	  struct flock flock;			\
+	  flock.l_type = F_RDLCK;		\
+	  flock.l_whence = SEEK_SET;			\
+	  flock.l_start = flock.l_len = 0L;	\
+	  lock_val = fcntl (dbf->desc, F_SETLK, &flock);	\
+	}
+#define WRITELOCK_FILE(dbf) \
+	{					\
+	  struct flock flock;			\
+	  flock.l_type = F_WRLCK;		\
+	  flock.l_whence = SEEK_SET;			\
+	  flock.l_start = flock.l_len = 0L;	\
+	  lock_val = fcntl (dbf->desc, F_SETLK, &flock);	\
+	}
+#endif
+#endif
+
+/* Do we have bcopy?  */
+#if !HAVE_BCOPY
+#if HAVE_MEMORY_H
+#include <memory.h>
+#endif
+#define bcmp(d1, d2, n)	memcmp(d1, d2, n)
+#define bcopy(d1, d2, n) memcpy(d2, d1, n)
+#endif
+
+/* Do we have fsync? */
+#if !HAVE_FSYNC
+#ifdef MSDOS
+#define fsync(f)
+#else
+#define fsync(f) {sync(); sync();}
+#endif
+#endif
+
+/* Default block size.  Some systems do not have blocksize in their
+   stat record. This code uses the BSD blocksize from stat. */
+
+#if HAVE_ST_BLKSIZE
+#define STATBLKSIZE file_stat.st_blksize
+#else
+#define STATBLKSIZE 1024
+#endif
+
+/* Do we have ftruncate? */
+#if HAVE_FTRUNCATE
+#define TRUNCATE(dbf) ftruncate (dbf->desc, 0)
+#else
+#define TRUNCATE(dbf) close( open (dbf->name, O_RDWR|O_TRUNC|O_BINARY, mode));
+#endif
+
+/* Do we have 32bit or 64bit longs? */
+#if LONG_64_BITS || !INT_16_BITS
+typedef int word_t;
+#else
+typedef long word_t;
+#endif
+
+
+#include <io.h>
+
+#ifdef __GNUC__
+#define alloca __builtin_alloca
+#else /* not __GNUC__ */
+#if defined (HAVE_ALLOCA_H) || (defined(sparc) && (defined(sun) || (!defined(USG) && !defined(SVR4) && !defined(__svr4__))))
+#include <alloca.h>
+#else
+#ifndef _AIX
+char *alloca ();
+#endif
+#endif /* alloca.h */
+#endif /* not __GNUC__ */
Index: /tags/gsdl-21/gsdl/packages/wingdbm/testgdbm.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/wingdbm/testgdbm.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/wingdbm/testgdbm.c	(revision 699)
@@ -0,0 +1,546 @@
+/* testgdbm.c - Driver program to test the database routines and to
+   help debug gdbm.  Uses inside information to show "system" information */
+
+/*  This file is part of GDBM, the GNU data base manager, by Philip A. Nelson.
+    Copyright (C) 1990, 1991, 1993  Free Software Foundation, Inc.
+
+    GDBM is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2, or (at your option)
+    any later version.
+
+    GDBM is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with GDBM; see the file COPYING.  If not, write to
+    the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+
+    You may contact the author by:
+       e-mail:  phil@cs.wwu.edu
+      us-mail:  Philip A. Nelson
+                Computer Science Department
+                Western Washington University
+                Bellingham, WA 98226
+       
+*************************************************************************/
+
+
+/* AIX demands this be the very first thing in the file. */
+#if !defined(__GNUC__) && defined(_AIX)
+ #pragma alloca
+#endif
+
+/* include system configuration before all else. */
+#include "autoconf.h"
+
+#include "gdbmdefs.h"
+#include "gdbmerrno.h"
+#include "extern.h"
+
+#include "getopt.h"
+
+extern const char * gdbm_version;
+
+extern const char *gdbm_strerror _ARGS((gdbm_error));
+
+gdbm_file_info *gdbm_file;
+
+/* Debug procedure to print the contents of the current hash bucket. */
+void
+print_bucket (bucket, mesg)
+     hash_bucket *bucket;
+     char *mesg;
+{
+  int  index;
+
+  printf ("******* %s **********\n\nbits = %d\ncount= %d\nHash Table:\n",
+	 mesg, bucket->bucket_bits, bucket->count);
+  printf ("     #    hash value     key size    data size     data adr  home\n");
+  for (index = 0; index < gdbm_file->header->bucket_elems; index++)
+    printf ("  %4d  %12x  %11d  %11d  %11d %5d\n", index,
+	   bucket->h_table[index].hash_value,
+	   bucket->h_table[index].key_size,
+	   bucket->h_table[index].data_size,
+	   bucket->h_table[index].data_pointer,
+	   bucket->h_table[index].hash_value % gdbm_file->header->bucket_elems);
+
+  printf ("\nAvail count = %1d\n", bucket->av_count);
+  printf ("Avail  adr     size\n");
+  for (index = 0; index < bucket->av_count; index++)
+    printf ("%9d%9d\n", bucket->bucket_avail[index].av_adr,
+	                bucket->bucket_avail[index].av_size);
+}
+
+
+void
+_gdbm_print_avail_list (dbf)
+     gdbm_file_info *dbf;
+{
+  int temp;
+  int size;
+  avail_block *av_stk;
+ 
+  /* Print the the header avail block.  */
+  printf ("\nheader block\nsize  = %d\ncount = %d\n",
+	  dbf->header->avail.size, dbf->header->avail.count);
+  for (temp = 0; temp < dbf->header->avail.count; temp++)
+    {
+      printf ("  %15d   %10d \n", dbf->header->avail.av_table[temp].av_size,
+	      dbf->header->avail.av_table[temp].av_adr);
+    }
+
+  /* Initialize the variables for a pass throught the avail stack. */
+  temp = dbf->header->avail.next_block;
+  size = ( ( (dbf->header->avail.size * sizeof (avail_elem)) >> 1)
+	  + sizeof (avail_block));
+  av_stk = (avail_block *) alloca (size);
+
+  /* Print the stack. */
+  while (FALSE)
+    {
+      lseek (dbf->desc, temp, L_SET);
+      read  (dbf->desc, av_stk, size);
+
+      /* Print the block! */
+      printf ("\nblock = %d\nsize  = %d\ncount = %d\n", temp,
+	      av_stk->size, av_stk->count);
+      for (temp = 0; temp < av_stk->count; temp++)
+	{
+	  printf ("  %15d   %10d \n", av_stk->av_table[temp].av_size,
+	    av_stk->av_table[temp].av_adr);
+	}
+      temp = av_stk->next_block;
+    }
+}
+
+void
+_gdbm_print_bucket_cache (dbf)
+     gdbm_file_info *dbf;
+{
+  register int index;
+  char changed;
+ 
+  if (dbf->bucket_cache != NULL) {
+    printf(
+      "Bucket Cache (size %d):\n  Index:  Address  Changed  Data_Hash \n",
+      dbf->cache_size);
+    for (index=0; index < dbf->cache_size; index++) {
+      changed = dbf->bucket_cache[index].ca_changed;
+      printf ("  %5d:  %7d  %7s  %x\n",
+	      index,
+	      dbf->bucket_cache[index].ca_adr,
+	      (changed ? "True" : "False"),
+	      dbf->bucket_cache[index].ca_data.hash_val);
+    }
+  } else
+    printf("Bucket cache has not been initialized.\n");
+}
+
+void
+usage (s)
+     char *s;
+{
+  printf(
+      "Usage: %s [-r or -nf] [-b block-size] [-c cache-size] [-g gdbm-file]\n",
+      s);
+  exit (2);
+}
+
+void
+print_keydatum (kd)
+  datum kd;
+{
+  char data[10240];
+  int num = kd.dsize;
+
+  if (num > 10230) num = 10230;
+  strncpy (data, kd.dptr, num);
+  data[num]='\0';
+  printf ("key is  ->\"%s\"\n", data);
+}
+
+void
+print_valuedatum (kd)
+  datum kd;
+{
+  char data[10240];
+  int num = kd.dsize;
+
+  if (num > 10230) num = 10230;
+  strncpy (data, kd.dptr, num);
+  data[num]='\0';
+  printf ("data is  ->\"%s\"\n", data);
+}
+
+
+/* The test program allows one to call all the routines plus the hash function.
+   The commands are single letter commands.  The user is prompted for all other
+   information.  See the help command (?) for a list of all commands. */
+
+int
+main (argc, argv)
+     int argc;
+     char *argv[];
+
+{
+
+  char  cmd_ch;
+
+  datum key_data;
+  datum data_data;
+  datum return_data;
+
+  char key_line[500];
+  char data_line[1000];
+
+  char done = FALSE;
+  int  opt;
+  char reader = FALSE;
+  char newdb = FALSE;
+  int  fast  = 0;
+
+  int  cache_size = DEFAULT_CACHESIZE;
+  int  block_size = 0;
+
+  char *file_name = NULL;
+
+
+  /* Argument checking. */
+  opterr = 0;
+  while ((opt = getopt (argc, argv, "frnc:b:g:")) != -1)
+    switch (opt) {
+    case 'f':  fast = GDBM_FAST;
+               if (reader) usage (argv[0]);
+               break;
+    case 'r':  reader = TRUE;
+               if (newdb) usage (argv[0]);
+               break;
+    case 'n':  newdb = TRUE;
+               if (reader) usage (argv[0]);
+               break;
+    case 'c':  cache_size = atoi(optarg);
+    	       break;
+    case 'b':  block_size = atoi(optarg);
+               break;
+    case 'g':  file_name = optarg;
+               break;
+    default:  usage(argv[0]);
+    }
+
+  if(file_name == NULL) 
+    file_name = "junk.gdbm";
+
+  /* Initialize variables. */
+  key_data.dptr = NULL;
+  data_data.dptr = data_line;
+
+  if (reader)
+    {
+      gdbm_file = gdbm_open (file_name, block_size, GDBM_READER, 00664, NULL);
+    }
+  else if (newdb)
+    {
+      gdbm_file =
+    	gdbm_open (file_name, block_size, GDBM_NEWDB | fast, 00664, NULL);
+    }
+  else
+    {
+      gdbm_file =
+    	gdbm_open (file_name, block_size, GDBM_WRCREAT | fast, 00664, NULL);
+    }
+  if (gdbm_file == NULL)
+    {
+      printf("gdbm_open failed, %s\n", gdbm_strerror(gdbm_errno));
+      exit (2);
+    }
+
+  if (gdbm_setopt(gdbm_file, GDBM_CACHESIZE, &cache_size, sizeof(int)) == -1)
+    {
+      printf("gdbm_setopt failed, %s\n", gdbm_strerror(gdbm_errno));
+      exit(2);
+    }
+
+  /* Welcome message. */
+  printf ("\nWelcome to the gdbm test program.  Type ? for help.\n\n");
+  
+  while (!done)
+    {
+      printf ("com -> ");
+      cmd_ch = getchar ();
+      if (cmd_ch != '\n')
+	{
+	  char temp;
+	  do
+	      temp = getchar ();
+	  while (temp != '\n' && temp != EOF);
+	}
+      if (cmd_ch == EOF) cmd_ch = 'q';
+      switch (cmd_ch)
+	{
+	
+	/* Standard cases found in all test{dbm,ndbm,gdbm} programs. */
+	case '\n':
+	  printf ("\n");
+	  break;
+
+	case 'c':
+	  {
+	    int temp;
+	    temp = 0;
+	    if (key_data.dptr != NULL) free (key_data.dptr);
+	    return_data = gdbm_firstkey (gdbm_file);
+	    while (return_data.dptr != NULL)
+	      {
+		temp++;
+		key_data = return_data;
+		return_data = gdbm_nextkey (gdbm_file, key_data);
+		free (key_data.dptr);
+	      }
+	    printf ("There are %d items in the database.\n\n", temp);
+	    key_data.dptr = NULL;
+	  }
+	  break;
+
+	case 'd':
+	  if (key_data.dptr != NULL) free (key_data.dptr);
+	  printf ("key -> ");
+	  gets (key_line);
+	  key_data.dptr = key_line;
+	  key_data.dsize = strlen (key_line);
+	  /*	  key_data.dsize = strlen (key_line)+1;*/
+	  if (gdbm_delete (gdbm_file, key_data) != 0)
+	    printf ("Item not found or deleted\n");
+	  printf ("\n");
+	  key_data.dptr = NULL;
+	  break;
+
+	case 'f':
+	  if (key_data.dptr != NULL) free (key_data.dptr);
+	  printf ("key -> ");
+	  gets (key_line);
+	  key_data.dptr = key_line;
+	  key_data.dsize = strlen (key_line);
+	  /*	  key_data.dsize = strlen (key_line)+1;*/
+	  return_data = gdbm_fetch (gdbm_file, key_data);
+	  if (return_data.dptr != NULL)
+	    {
+	      print_valuedatum(return_data);
+	      /*	      printf ("data is ->%s\n\n", return_data.dptr); */
+	      free (return_data.dptr);
+	    }
+	  else
+	    printf ("No such item found.\n\n");
+	  key_data.dptr = NULL;
+	  break;
+
+	case 'n':
+	  if (key_data.dptr != NULL) free (key_data.dptr);
+	  printf ("key -> ");
+	  gets (key_line);
+	  key_data.dptr = key_line;
+	  key_data.dsize = strlen (key_line);
+	  /*	  key_data.dsize = strlen (key_line)+1;*/
+	  return_data = gdbm_nextkey (gdbm_file, key_data);
+	  if (return_data.dptr != NULL)
+	    {
+	      key_data = return_data;
+	      print_keydatum(key_data);
+	      /*	      printf ("key is  ->%s\n", key_data.dptr); */
+	      return_data = gdbm_fetch (gdbm_file, key_data);
+	      print_valuedatum(return_data);
+	      /*	      printf ("data is ->%s\n\n", return_data.dptr); */
+	      free (return_data.dptr);
+	    }
+	  else
+	    {
+	      printf ("No such item found.\n\n");
+	      key_data.dptr = NULL;
+	    }
+	  break;
+
+	case 'q':
+	  done = TRUE;
+	  break;
+
+	case 's':
+	  if (key_data.dptr != NULL) free (key_data.dptr);
+	  printf ("key -> ");
+	  gets (key_line);
+	  key_data.dptr = key_line;
+	  key_data.dsize = strlen (key_line);
+	  /*	  key_data.dsize = strlen (key_line)+1;*/
+	  printf ("data -> ");
+	  gets (data_line);
+	  data_data.dsize = strlen (data_line);
+	  /* data_data.dsize = strlen (data_line)+1;*/
+	  if (gdbm_store (gdbm_file, key_data, data_data, GDBM_REPLACE) != 0)
+	    printf ("Item not inserted. \n");
+	  printf ("\n");
+	  key_data.dptr = NULL;
+	  break;
+
+	case '1':
+	  if (key_data.dptr != NULL) free (key_data.dptr);
+	  key_data = gdbm_firstkey (gdbm_file);
+	  if (key_data.dptr != NULL)
+	    {
+	      print_keydatum(key_data);
+	      /*	      printf ("key is  ->%s\n", key_data.dptr); */
+	      return_data = gdbm_fetch (gdbm_file, key_data);
+	      print_valuedatum(return_data);
+	      /*	      printf ("data is ->%s\n\n", return_data.dptr); */
+	      free (return_data.dptr);
+	    }
+	  else
+	    printf ("No such item found.\n\n");
+	  break;
+
+	case '2':
+	  return_data = gdbm_nextkey (gdbm_file, key_data);
+	  if (return_data.dptr != NULL)
+	    {
+	      free (key_data.dptr);
+	      key_data = return_data;
+	      print_keydatum(key_data);
+	      /*	      printf ("key is  ->%s\n", key_data.dptr); */
+	      return_data = gdbm_fetch (gdbm_file, key_data);
+	      print_valuedatum(return_data);
+	      /*	      printf ("data is ->%s\n\n", return_data.dptr); */
+	      free (return_data.dptr);
+	    }
+	  else
+	    printf ("No such item found.\n\n");
+	  break;
+
+
+	/* Special cases for the testgdbm program. */
+	case 'r':
+	  {
+	    if (gdbm_reorganize (gdbm_file))
+	      printf ("Reorganization failed. \n\n");
+	    else
+	      printf ("Reorganization succeeded. \n\n");
+	  }
+	  break;
+
+	case 'A':
+	  _gdbm_print_avail_list (gdbm_file);
+	  printf ("\n");
+	  break;
+
+	case 'B':
+	  {
+	    int temp;
+	    char number[80];
+
+	    printf ("bucket? ");
+	    gets (number);
+	    sscanf (number,"%d",&temp);
+
+	    if (temp >= gdbm_file->header->dir_size /4)
+	      {
+		printf ("Not a bucket. \n\n");
+		break;
+	      }
+	    _gdbm_get_bucket (gdbm_file, temp);
+	  }
+	  printf ("Your bucket is now ");
+
+	case 'C':
+	  print_bucket (gdbm_file->bucket, "Current bucket");
+	  printf ("\n current directory entry = %d.\n", gdbm_file->bucket_dir);
+	  printf (" current bucket address  = %d.\n\n",
+		  gdbm_file->cache_entry->ca_adr);
+	  break;
+
+	case 'D':
+	  printf ("Hash table directory.\n");
+	  printf ("  Size =  %d.  Bits = %d. \n\n",gdbm_file->header->dir_size,
+		  gdbm_file->header->dir_bits);
+	  {
+	    int temp;
+
+	    for (temp = 0; temp < gdbm_file->header->dir_size / 4; temp++)
+	      {
+		printf ("  %10d:  %12d\n", temp, gdbm_file->dir[temp]);
+		if ( (temp+1) % 20 == 0 && isatty (0))
+		  {
+		    printf ("*** CR to continue: ");
+		    while (getchar () != '\n') /* Do nothing. */;
+		  }
+	      }
+	  }
+	  printf ("\n");
+	  break;
+
+	case 'F':
+	  {
+	    printf ("\nFile Header: \n\n");
+	    printf ("  table        = %d\n", gdbm_file->header->dir);
+	    printf ("  table size   = %d\n", gdbm_file->header->dir_size);
+	    printf ("  table bits   = %d\n", gdbm_file->header->dir_bits);
+	    printf ("  block size   = %d\n", gdbm_file->header->block_size);
+	    printf ("  bucket elems = %d\n", gdbm_file->header->bucket_elems);
+	    printf ("  bucket size  = %d\n", gdbm_file->header->bucket_size);
+	    printf ("  header magic = %x\n", gdbm_file->header->header_magic);
+	    printf ("  next block   = %d\n", gdbm_file->header->next_block);
+	    printf ("  avail size   = %d\n", gdbm_file->header->avail.size);
+	    printf ("  avail count  = %d\n", gdbm_file->header->avail.count);
+	    printf ("  avail nx blk = %d\n", gdbm_file->header->avail.next_block);
+	    printf ("\n");
+	  }
+	  break;
+
+        case 'H':
+	  if (key_data.dptr != NULL) free (key_data.dptr);
+	  printf ("key -> ");
+	  gets (key_line);
+	  key_data.dptr = key_line;
+	  key_data.dsize = strlen (key_line)+1;
+	  printf ("hash value = %x. \n\n", _gdbm_hash (key_data));
+	  key_data.dptr = NULL;
+	  break;
+
+	case 'K':
+	  _gdbm_print_bucket_cache (gdbm_file);
+	  break;
+
+	case 'V':
+	  printf ("%s\n\n", gdbm_version);
+	  break;
+
+	case '?':
+	  printf ("c - count (number of entries)\n");
+	  printf ("d - delete\n");
+	  printf ("f - fetch\n");
+	  printf ("n - nextkey\n");
+	  printf ("q - quit\n");
+	  printf ("s - store\n");
+	  printf ("1 - firstkey\n");
+	  printf ("2 - nextkey on last key (from n, 1 or 2)\n\n");
+
+	  printf ("r - reorganize\n");
+	  printf ("A - print avail list\n");
+	  printf ("B - get and print current bucket n\n");
+	  printf ("C - print current bucket\n");
+	  printf ("D - print hash directory\n");
+	  printf ("F - print file header\n");
+	  printf ("H - hash value of key\n");
+	  printf ("K - print the bucket cache\n");
+	  printf ("V - print version of gdbm\n");
+	  break;
+
+	default:
+	  printf ("What? \n\n");
+	  break;
+
+	}
+    }
+
+  /* Quit normally. */
+  exit (0);
+
+}
Index: /tags/gsdl-21/gsdl/packages/wingdbm/update.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/wingdbm/update.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/wingdbm/update.c	(revision 699)
@@ -0,0 +1,126 @@
+/* update.c - The routines for updating the file to a consistent state. */
+
+/*  This file is part of GDBM, the GNU data base manager, by Philip A. Nelson.
+    Copyright (C) 1990, 1991, 1993  Free Software Foundation, Inc.
+
+    GDBM is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2, or (at your option)
+    any later version.
+
+    GDBM is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with GDBM; see the file COPYING.  If not, write to
+    the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+
+    You may contact the author by:
+       e-mail:  phil@cs.wwu.edu
+      us-mail:  Philip A. Nelson
+                Computer Science Department
+                Western Washington University
+                Bellingham, WA 98226
+       
+*************************************************************************/
+
+
+/* include system configuration before all else. */
+#include "autoconf.h"
+
+#include "gdbmdefs.h"
+
+static void write_header _ARGS((gdbm_file_info *));
+
+/* This procedure writes the header back to the file described by DBF. */
+
+static void
+write_header (dbf)
+     gdbm_file_info *dbf;
+{
+  int  num_bytes;	/* Return value for write. */
+  off_t file_pos;	/* Return value for lseek. */
+
+  file_pos = lseek (dbf->desc, 0L, L_SET);
+  if (file_pos != 0) _gdbm_fatal (dbf, "lseek error");
+  num_bytes = write (dbf->desc, dbf->header, dbf->header->block_size);
+  if (num_bytes != dbf->header->block_size)
+    _gdbm_fatal (dbf, "write error");
+
+  /* Wait for all output to be done. */
+  if (!dbf->fast_write)
+    fsync (dbf->desc);
+}
+
+
+/* After all changes have been made in memory, we now write them
+   all to disk. */
+void
+_gdbm_end_update (dbf)
+     gdbm_file_info *dbf;
+{
+  int  num_bytes;	/* Return value for write. */
+  off_t file_pos;	/* Return value for lseek. */
+  
+  
+  /* Write the current bucket. */
+  if (dbf->bucket_changed && (dbf->cache_entry != NULL))
+    {
+      _gdbm_write_bucket (dbf, dbf->cache_entry);
+      dbf->bucket_changed = FALSE;
+    }
+
+  /* Write the other changed buckets if there are any. */
+  if (dbf->second_changed)
+    {
+      if(dbf->bucket_cache != NULL)
+        {
+          register int index;
+
+          for (index = 0; index < dbf->cache_size; index++)
+	    {
+	      if (dbf->bucket_cache[index].ca_changed)
+	        _gdbm_write_bucket (dbf, &dbf->bucket_cache[index]);
+            }
+        }
+      dbf->second_changed = FALSE;
+    }
+  
+  /* Write the directory. */
+  if (dbf->directory_changed)
+    {
+      file_pos = lseek (dbf->desc, dbf->header->dir, L_SET);
+      if (file_pos != dbf->header->dir) _gdbm_fatal (dbf, "lseek error");
+      num_bytes = write (dbf->desc, dbf->dir, dbf->header->dir_size);
+      if (num_bytes != dbf->header->dir_size)
+	_gdbm_fatal (dbf, "write error");
+      dbf->directory_changed = FALSE;
+      if (!dbf->header_changed && !dbf->fast_write)
+	fsync (dbf->desc);
+    }
+
+  /* Final write of the header. */
+  if (dbf->header_changed)
+    {
+      write_header (dbf);
+      dbf->header_changed = FALSE;
+    }
+}
+
+
+/* If a fatal error is detected, come here and exit. VAL tells which fatal
+   error occured. */
+int
+_gdbm_fatal (dbf, val)
+     gdbm_file_info *dbf;
+     char *val;
+{
+  if (dbf->fatal_err != NULL)
+    (*dbf->fatal_err) (val);
+  else
+    fprintf (stderr, "gdbm fatal: %s.\n", val);
+  exit (-1);
+  /* NOTREACHED */
+}
Index: /tags/gsdl-21/gsdl/packages/wingdbm/version.c
===================================================================
--- /tags/gsdl-21/gsdl/packages/wingdbm/version.c	(revision 699)
+++ /tags/gsdl-21/gsdl/packages/wingdbm/version.c	(revision 699)
@@ -0,0 +1,33 @@
+/* version.c - This is file contains the version number for gdbm source. */
+
+/*  This file is part of GDBM, the GNU data base manager, by Philip A. Nelson.
+    Copyright (C) 1990, 1991, 1993  Free Software Foundation, Inc.
+
+    GDBM is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2, or (at your option)
+    any later version.
+
+    GDBM is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with GDBM; see the file COPYING.  If not, write to
+    the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+
+    You may contact the author by:
+       e-mail:  phil@cs.wwu.edu
+      us-mail:  Philip A. Nelson
+                Computer Science Department
+                Western Washington University
+                Bellingham, WA 98226
+       
+*************************************************************************/
+
+#include "autoconf.h"
+
+/* Keep a string with the version number in it! */
+const char * gdbm_version =
+	"This is GDBM version 1.7.3, as of May 19, 1994.";
Index: /tags/gsdl-21/gsdl/perllib/arcinfo.pm
===================================================================
--- /tags/gsdl-21/gsdl/perllib/arcinfo.pm	(revision 699)
+++ /tags/gsdl-21/gsdl/perllib/arcinfo.pm	(revision 699)
@@ -0,0 +1,163 @@
+###########################################################################
+#
+# arcinfo.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+
+# This module stores information about the archives. At the moment
+# this information just consists of the file name (relative to the
+# directory the archives information file is in) and its OID.
+
+# This module assumes there is a one to one correspondance between
+# a file in the archives directory and an OID.
+
+
+package arcinfo;
+
+sub new {
+    my ($class) = @_;
+    my $self = {'info'=>{},
+		'order'=>[]};
+
+    return bless $self, $class;
+}
+
+sub load_info {
+    my $self = shift (@_);
+    my ($filename) = @_;
+
+    $self->{'info'} = {};
+
+    if (-e $filename) {
+	open (INFILE, $filename) || 
+	    die "arcinfo::load_info couldn't read $filename\n";
+
+	my ($line, @line);
+	while (defined ($line = <INFILE>)) {
+	    $line =~ s/\cM|\cJ//g; # remove end-of-line characters
+	    @line = split ("\t", $line); # filename, 
+	    if (scalar(@line) >= 2) {
+		$self->add_info (@line);
+	    }
+	}
+
+	close (INFILE);
+    }
+}
+
+sub save_info {
+    my $self = shift (@_);
+    my ($filename) = @_;
+
+    my ($OID, $info);
+
+    open (OUTFILE, ">$filename") || 
+	die "arcinfo::save_info couldn't write $filename\n";
+
+    foreach $info (@{$self->get_OID_list()}) {
+	if (defined $info) {
+	    print OUTFILE join("\t", @$info), "\n";
+	}
+    }
+    
+    close (OUTFILE);
+}
+
+sub delete_info {
+    my $self = shift (@_);
+    my ($OID) = @_;
+
+    if (defined $self->{'info'}->{$OID}) {
+	delete $self->{'info'}->{$OID};
+	
+	my $i = 0;
+	while ($i < scalar (@{$self->{'order'}})) {
+	    if ($self->{'order'}->[$i] eq $OID) {
+		splice (@{$self->{'order'}}, $i, 1);
+		last;
+	    }
+	    
+	    $i ++;
+	}
+    }
+}
+
+sub add_info {
+    my $self = shift (@_);
+    my ($OID, $doc_file) = @_;
+
+    $self->delete_info ($OID);
+    $self->{'info'}->{$OID} = [$doc_file];
+    push (@{$self->{'order'}}, $OID);
+}
+
+# returns a list of the form [[OID, doc_file], ...]
+sub get_OID_list {
+    my $self = shift (@_);
+
+    my ($OID);
+    my @list = ();
+
+    foreach $OID (@{$self->{'order'}}) {
+	push (@list, [$OID, $self->{'info'}->{$OID}->[0]]);
+    }
+
+    return \@list;
+}
+
+# returns a list of the form [[doc_file, OID], ...]
+sub get_file_list {
+    my $self = shift (@_);
+
+    my ($OID);
+    my @list = ();
+
+    foreach $OID (@{$self->{'order'}}) {
+	push (@list, [$self->{'info'}->{$OID}->[0], $OID]);
+    }
+
+    return \@list;
+}
+
+
+# returns a list of the form [doc_file]
+sub get_info {
+    my $self = shift (@_);
+    my ($OID) = @_;
+
+    if (defined $self->{'info'}->{$OID}) {
+	return $self->{'info'}->{$OID};
+    }
+
+    return undef;
+}
+
+
+# returns the number of documents so far
+sub size {
+    my $self = shift (@_);
+    return (scalar(@{$self->{'order'}}));
+}
+
+1;
+
Index: /tags/gsdl-21/gsdl/perllib/cfgread.pm
===================================================================
--- /tags/gsdl-21/gsdl/perllib/cfgread.pm	(revision 699)
+++ /tags/gsdl-21/gsdl/perllib/cfgread.pm	(revision 699)
@@ -0,0 +1,130 @@
+###########################################################################
+#
+# cfgread.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# reads in configuration files
+
+package cfgread;
+
+
+sub read_cfg_line {
+    my ($handle) = @_;
+    my $line = "";
+    my @line = ();
+    my $linecontinues = 0;
+
+    while (defined($line = <$handle>)) {
+	$line =~ s/^\#.*$//;   # remove comments
+	$line =~ s/\cM|\cJ//g; # remove end-of-line characters
+	$line =~ s/^\s+//;     # remove initial white space
+	$linecontinues = $line =~ s/\\$//;
+	while ($line =~ s/\s*(\"[^\"]*\"|\'[^\']*\'|\S+)\s*//) {
+	    if (defined $1) {
+		# remove any enclosing quotes
+		my $entry = $1;
+		$entry =~ s/^([\"\'])(.*)\1$/$2/;
+
+		# substitute an environment variables
+		$entry =~ s/\$(\w+)/$ENV{$1}/g;
+		$entry =~ s/\$\{(\w+)\}/$ENV{$1}/g;
+
+		push (@line, $entry);
+	    } else {
+		push (@line, "");
+	    }
+	}
+
+	if (scalar(@line) > 0 && !$linecontinues) {
+#	    print STDERR "line: \"" . join ("\" \"", @line) . "\"\n";
+	    return \@line;
+	}
+    }
+
+    return undef;
+}
+
+sub write_cfg_line {
+    my ($handle, $line) = @_;
+    print $handle join ("\t", @$line), "\n";
+}
+
+
+# stringexp and arrayexp should be something like
+#   '^(this|that)$'
+sub read_cfg_file {
+    my ($filename, $stringexp, $arrayexp, $hashexp) = @_;
+    my ($line);
+    my $data = {};
+
+    if (open (COLCFG, $filename)) {
+	while (defined ($line = &read_cfg_line('COLCFG'))) {
+	    if (scalar(@$line) >= 2) {
+		my $key = shift (@$line);
+		if (defined $stringexp && $key =~ /$stringexp/) {
+		    $data->{$key} = shift (@$line);
+
+		} elsif (defined $arrayexp && $key =~ /$arrayexp/) {
+		    push (@{$data->{$key}}, @$line);
+
+		} elsif (defined $hashexp && $key =~ /$hashexp/) {
+		    my $k = shift @$line;
+		    my $v = shift @$line;
+		    $data->{$key}->{$k} = $v;
+		}
+	    }
+	}
+	close (COLCFG);
+    } else {
+	print STDERR "cfgread::read_cfg_file couldn't read the cfg file $filename\n";
+    }
+
+    return $data;
+}
+
+
+# stringexp and arrayexp should be something like
+#   '^(this|that)$'
+sub write_cfg_file {
+    my ($filename, $data, $stringexp, $arrayexp, $hashexp) = @_;
+
+    if (open (COLCFG, ">$filename")) {
+	foreach $key (sort(keys(%$data))) {
+	    if ($key =~ /$stringexp/) {
+		&write_cfg_line ('COLCFG', [$key, $data->{$key}]);
+	    } elsif ($key =~ /$arrayexp/) {
+		&write_cfg_line ('COLCFG', [$key, @{$data->{$key}}]);
+	    } elsif ($key =~ /$hashexp/) {
+		foreach $k (keys (%{$data->{$key}})) {
+		    &write_cfg_line ('COLCFG', [$key, $k, $data->{$key}->{$k}]);
+		}
+	    }
+	}
+	close (COLCFG);
+    } else {
+	print STDERR "cfgread::write_cfg_file couldn't write the cfg file $filename\n";
+    }
+}
+
+
+1;
Index: /tags/gsdl-21/gsdl/perllib/classify.pm
===================================================================
--- /tags/gsdl-21/gsdl/perllib/classify.pm	(revision 699)
+++ /tags/gsdl-21/gsdl/perllib/classify.pm	(revision 699)
@@ -0,0 +1,187 @@
+###########################################################################
+#
+# classify.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# functions to handle classifiers
+
+package classify;
+
+require util;
+
+
+$next_classify_num = 1;
+
+# load_classifier will load one classifier. $classinfo is an
+# array containing information about the classifier to load. The
+# first element of the array is the name of the classifier, the
+# rest of the elements are classifier specific.
+sub load_classifier {
+    my ($classinfo) = @_;
+
+    # get the classifier name
+    my $classname = shift(@$classinfo);
+    return undef unless defined $classname;
+    
+    # find the classifier
+    my $colclassname = &util::filename_cat($ENV{'GSDLCOLLECTDIR'},"perllib/classify", 
+					   "${classname}.pm");
+    my $mainclassname = &util::filename_cat($ENV{'GSDLHOME'},"perllib/classify", 
+					    "${classname}.pm");
+    if (-e $colclassname) { require $colclassname; }
+    elsif (-e $mainclassname) { require $mainclassname; }
+    else { die "ERROR - couldn't find classifier \"$classname\"\n"; }
+
+    # create the classify object
+    my ($classobj);
+    map { $_ = "\"$_\""; } @$classinfo;
+    my $options = join (",", @$classinfo);
+    eval ("\$classobj = new $classname($options)");
+    die "$@" if $@;
+
+    return $classobj;
+}
+
+# init_classifiers resets all the classifiers and readys them to process
+# the documents.
+sub init_classifiers {
+    my ($classifiers) = @_;
+    
+    foreach $classobj (@$classifiers) {
+	$classobj->init();
+    }
+}
+
+# classify_doc lets each of the classifiers classify a document
+sub classify_doc {
+    my ($classifiers, $doc_obj) = @_;
+    
+    foreach $classobj (@$classifiers) {
+	$classobj->classify($doc_obj);
+    }
+}
+
+# output_classify_info outputs all the info needed for the classification
+# to the gdbm
+sub output_classify_info {
+    my ($classifiers, $handle, $allclassifications) = @_;
+#    $handle = "main::STDOUT";
+
+    # create a classification containing all the info
+    my $classifyinfo = {'classifyOID'=>'browse',
+			'contains'=>[]};
+
+    # get each of the classifications
+    foreach $classobj (@$classifiers) {
+	my $tempinfo = $classobj->get_classify_info();
+	$tempinfo->{'classifyOID'} = "CL$next_classify_num";
+	$next_classify_num++;
+	push (@{$classifyinfo->{'contains'}}, $tempinfo);
+    }
+
+    &print_classify_info ($handle, $classifyinfo, "", $allclassifications);
+}
+
+sub print_classify_info {
+    my ($handle, $classifyinfo, $OID, $allclassifications) = @_;
+    $OID =~ s/^\.+//; # just for good luck
+
+    # book information is printed elsewhere
+    return if (defined ($classifyinfo->{'OID'}));
+ 
+    # don't want empty classifications
+    if ($allclassifications || &check_contents ($classifyinfo) > 0) {
+   
+	$OID = $classifyinfo->{'classifyOID'} if defined ($classifyinfo->{'classifyOID'});
+	
+	my $outputtext = "[$OID]\n";
+	$outputtext .= "<doctype>classify\n";
+	$outputtext .= "<hastxt>0\n";
+	$outputtext .= "<childtype>$classifyinfo->{'childtype'}\n" 
+	    if defined $classifyinfo->{'childtype'};
+	$outputtext .= "<Title>$classifyinfo->{'Title'}\n" 
+	    if defined $classifyinfo->{'Title'};
+	$outputtext .= "<numleafdocs>$classifyinfo->{'numleafdocs'}\n" 
+	    if defined $classifyinfo->{'numleafdocs'};
+	$outputtext .= "<thistype>$classifyinfo->{'thistype'}\n" 
+	    if defined $classifyinfo->{'thistype'};
+
+	
+	$outputtext .= "<contains>";
+	
+	my $next_subOID = 1;
+	my $first = 1;
+	foreach $tempinfo (@{$classifyinfo->{'contains'}}) {
+	    # empty contents were made undefined by clean_contents()
+	    next unless defined $tempinfo;
+	    
+	    $outputtext .= ";" unless $first;
+	    $first = 0;
+	    
+	    if (defined ($tempinfo->{'classifyOID'})) {
+		$outputtext .= $tempinfo->{'classifyOID'};
+		&print_classify_info ($handle, $tempinfo, $tempinfo->{'classifyOID'},
+				      $allclassifications);
+	    } elsif (defined ($tempinfo->{'OID'})) {
+		$outputtext .= $tempinfo->{'OID'};
+		# note: we don't want to print the contents of the books
+	    } else {
+		$outputtext .= "\".$next_subOID";
+		&print_classify_info ($handle, $tempinfo, "$OID.$next_subOID",
+				      $allclassifications);
+		$next_subOID++;
+	    }
+	}
+	$outputtext .= "\n";
+	$outputtext .= '-' x 70 . "\n";
+	
+	print $handle $outputtext;
+    }
+}
+
+sub check_contents {
+    my ($classifyinfo) = @_;
+    my $num_leaf_docs = 0;
+    my $sub_num_leaf_docs = 0;
+
+    return $classifyinfo->{'numleafdocs'} if (defined $classifyinfo->{'numleafdocs'});
+
+    foreach $content (@{$classifyinfo->{'contains'}}) {
+	if (defined $content->{'OID'}) {
+	    # found a book
+	    $num_leaf_docs ++;
+	} elsif (($sub_num_leaf_docs = &check_contents ($content)) > 0) {
+	    # there's a book somewhere below
+	    $num_leaf_docs += $sub_num_leaf_docs;
+	} else {
+	    # section contains no books so we want to remove 
+	    # it from its parents contents
+	    $content = undef;
+	}
+    }
+
+    $classifyinfo->{'numleafdocs'} = $num_leaf_docs;
+    return $num_leaf_docs;
+}
+
+1;
Index: /tags/gsdl-21/gsdl/perllib/classify/AZList.pm
===================================================================
--- /tags/gsdl-21/gsdl/perllib/classify/AZList.pm	(revision 699)
+++ /tags/gsdl-21/gsdl/perllib/classify/AZList.pm	(revision 699)
@@ -0,0 +1,237 @@
+###########################################################################
+#
+# AZList.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# classifier plugin for sorting alphabetically
+# options are:
+# metadata=Metaname -- all documents with Metaname metadata
+#                      will be included in list, list will be sorted
+#                      by this element.
+# title=Title       -- (optional) the title field for this classification.
+#                      if not included title field will be Metaname. 
+
+package AZList;
+
+use sorttools;
+
+sub new {
+    my ($class, @options) = @_;
+
+    my ($metaname, $title);
+
+    foreach $option (@options) {
+	if ($option =~ /^metadata=(.*)$/i) {
+	    $metaname = $1;
+	} elsif ($option =~ /^title=(.*)$/i) {
+	    $title = $1;
+	}
+    }
+
+    die "AZList used with no metadata name to classify by\n"
+	unless defined $metaname;
+    $title = $metaname unless defined $title;
+
+    return bless {
+	'list'=>{},
+	'metaname' => $metaname,
+	'title' => $title
+	}, $class;
+}
+
+sub init {
+    my $self = shift (@_);
+
+    $self->{'list'} = {};
+}
+
+sub classify {
+    my $self = shift (@_);
+    my ($doc_obj) = @_;
+
+    my $doc_OID = $doc_obj->get_OID();
+    my $metavalue = $doc_obj->get_metadata_element ($doc_obj->get_top_section(),
+						    $self->{'metaname'});
+
+    # if this document doesn't contain the metadata element we're
+    # sorting by we won't include it in this classification
+    if (defined $metavalue && $metavalue ne "") {
+	if ($self->{'metaname'} eq 'Creator') {
+	    &sorttools::format_string_name_english (\$metavalue);
+	} else {
+	    &sorttools::format_string_english (\$metavalue);
+	}
+	if (defined $self->{'list'}->{$doc_OID}) {
+	    print STDERR "WARNING: AZList::classify called multiple times for $doc_OID\n";
+	} 
+	$self->{'list'}->{$doc_OID} = $metavalue;
+    }
+}
+
+sub get_classify_info {
+    my $self = shift (@_);
+
+    my @classlist = sort {$self->{'list'}->{$a} cmp $self->{'list'}->{$b};} keys %{$self->{'list'}};
+
+    return $self->splitlist (\@classlist);
+}
+
+sub get_entry {
+    my $self = shift (@_);
+    my ($title, $childtype, $thistype) = @_;
+    
+    # organise into classification structure
+    my %classifyinfo = ('childtype'=>$childtype,
+			'Title'=>$title,
+			'contains'=>[]);
+    $classifyinfo{'thistype'} = $thistype 
+	if defined $thistype && $thistype =~ /\w/;
+
+    return \%classifyinfo;
+}
+
+# splitlist takes an ordered list of classifications (@$classlistref) and splits it
+# up into alphabetical sub-sections.
+sub splitlist {
+    my $self = shift (@_);
+    my ($classlistref) = @_;
+    my $classhash = {};
+
+    # top level
+    my $childtype = "HList";
+    if (scalar (@$classlistref) <= 20) {$childtype = "VList";}
+    my $classifyinfo = $self->get_entry ($self->{'metaname'}, $childtype, "Invisible");
+
+    # don't need to do any splitting if there are less than 20 classifications
+    if ((scalar @$classlistref) <= 20) {
+	foreach $subOID (@$classlistref) {
+	    push (@{$classifyinfo->{'contains'}}, {'OID'=>$subOID});
+	}
+	return $classifyinfo;
+    }
+	
+    # first split up the list into separate A-Z and 0-9 classifications
+    foreach $classification (@$classlistref) {
+	my $title = $self->{'list'}->{$classification};
+	$title =~ s/^(.).*$/$1/;
+	$title =~ tr/[a-z]/[A-Z]/;
+	if ($title =~ /^[0-9]$/) {$title = '0-9';}
+	elsif ($title !~ /^[A-Z]$/) {
+	    print STDERR "AZList: WARNING $classification has badly formatted title ($title)\n";
+	}
+	$classhash->{$title} = [] unless defined $classhash->{$title};
+	push (@{$classhash->{$title}}, $classification);
+    }
+    $classhash = $self->compactlist ($classhash);
+
+    my @tmparr = ();
+    foreach $subsection (sort keys (%$classhash)) {
+	push (@tmparr, $subsection);
+    }
+    
+    # if there's a 0-9 section it will have been sorted to the beginning
+    # but we want it at the end
+    if ($tmparr[0] eq '0-9') {
+	shift @tmparr;
+	push (@tmparr, '0-9');
+    }
+
+    foreach $subclass (@tmparr) {
+	my $tempclassify = $self->get_entry($subclass, "VList");
+	foreach $subsubOID (@{$classhash->{$subclass}}) {
+	    push (@{$tempclassify->{'contains'}}, {'OID'=>$subsubOID});
+	}
+	push (@{$classifyinfo->{'contains'}}, $tempclassify);
+    }
+
+    return $classifyinfo;
+}
+
+sub compactlist {
+    my $self = shift (@_);
+    my ($classhashref) = @_;
+    my $compactedhash = {};
+    my @currentOIDs = ();
+    my $currentfirstletter = "";
+    my $currentlastletter = "";
+    my $lastkey = "";
+
+    # minimum and maximum documents to be displayed per page.
+    # the actual maximum will be max + (min-1).
+    # the smallest sub-section is a single letter at present
+    # so in this case there may be many times max documents
+    # displayed on a page.
+    my $min = 10; 
+    my $max = 30;
+
+    foreach $subsection (sort keys %$classhashref) {
+	if ($subsection eq '0-9') {
+	    @{$compactedhash->{$subsection}} = @{$classhashref->{$subsection}};
+	    next;
+	}
+	$currentfirstletter = $subsection if $currentfirstletter eq "";
+	if ((scalar (@currentOIDs) < $min) ||
+	    ((scalar (@currentOIDs) + scalar (@{$classhashref->{$subsection}})) <= $max)) {
+	    push (@currentOIDs, @{$classhashref->{$subsection}});
+	    $currentlastletter = $subsection;
+	} else {
+
+	    if ($currentfirstletter eq $currentlastletter) {
+		@{$compactedhash->{$currentfirstletter}} = @currentOIDs;
+		$lastkey = $currentfirstletter;
+	    } else {
+		@{$compactedhash->{"$currentfirstletter-$currentlastletter"}} = @currentOIDs;
+		$lastkey = "$currentfirstletter-$currentlastletter";
+	    } 
+	    if (scalar (@{$classhashref->{$subsection}}) >= $max) {
+		$compactedhash->{$subsection} = $classhashref->{$subsection};
+		@currentOIDs = ();
+		$currentfirstletter = "";
+		$lastkey = $subsection;
+	    } else {
+		@currentOIDs = @{$classhashref->{$subsection}};
+		$currentfirstletter = $subsection;
+		$currentlastletter = $subsection;
+	    }
+	}
+    }
+
+    # add final OIDs to last sub-classification if there aren't many otherwise
+    # add final sub-classification
+    if (scalar (@currentOIDs) < $min) {
+	my ($newkey) = $lastkey =~ /^(.)/;
+	@currentOIDs = (@{$compactedhash->{$lastkey}}, @currentOIDs);
+	delete $compactedhash->{$lastkey};
+	@{$compactedhash->{"$newkey-$currentlastletter"}} = @currentOIDs;	
+    } else {
+	if ($currentfirstletter eq $currentlastletter) {
+	    @{$compactedhash->{$currentfirstletter}} = @currentOIDs;
+	} else {
+	    @{$compactedhash->{"$currentfirstletter-$currentlastletter"}} = @currentOIDs;
+	} 
+    }
+
+    return $compactedhash;
+}
+
+1;
Index: /tags/gsdl-21/gsdl/perllib/classify/AZSectionList.pm
===================================================================
--- /tags/gsdl-21/gsdl/perllib/classify/AZSectionList.pm	(revision 699)
+++ /tags/gsdl-21/gsdl/perllib/classify/AZSectionList.pm	(revision 699)
@@ -0,0 +1,253 @@
+###########################################################################
+#
+# AZSectionList.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# classifier plugin for sorting sections alphabetically
+
+# this is very similar to AZList except it sorts by
+# section level metadata (excluding the top level)
+# instead of just top level metadata
+
+# options are:
+# metadata=Metadata
+
+# the only change is to the classify() subroutine which 
+# must now iterate through each section, adding each
+# to the classification
+
+package AZSectionList;
+
+use sorttools;
+
+sub new {
+    my ($class, @options) = @_;
+
+    my ($metaname);
+    foreach $option (@options) {
+	if ($option =~ /^metadata=(.*)$/i) {
+	    $metaname = $1;
+	}
+    }
+
+    if (!defined $metaname) {
+	die "AZSectionList used with no metadata name to classify by\n";
+    }
+    
+    return bless {
+	'list'=>{},
+	'metaname' => $metaname
+	}, $class;
+}
+
+sub init {
+    my $self = shift (@_);
+
+    $self->{'list'} = {};
+}
+
+sub classify {
+    my $self = shift (@_);
+    my ($doc_obj) = @_;
+
+    my $doc_OID = $doc_obj->get_OID();
+    my $thissection = $doc_obj->get_next_section ($doc_obj->get_top_section());
+
+    while (defined $thissection) {
+	$self->classify_section ($thissection, $doc_obj);
+	$thissection = $doc_obj->get_next_section ($thissection);
+    }
+}
+
+sub classify_section {
+    my $self = shift (@_);
+    my ($section, $doc_obj) = @_;
+
+    my $doc_OID = $doc_obj->get_OID();
+    my $metavalue = $doc_obj->get_metadata_element ($section, $self->{'metaname'});
+
+    # if this section doesn't contain the metadata element we're
+    # sorting by we won't include it in this classification
+    if (defined $metavalue && $metavalue ne "") {
+	if ($self->{'metaname'} eq 'Creator') {
+	    &sorttools::format_string_name_english (\$metavalue);
+	} else {
+	    &sorttools::format_string_english (\$metavalue);
+	}
+	if (defined $self->{'list'}->{"$doc_OID.$section"}) {
+	    print STDERR "WARNING: AZSectionList::classify called multiple times " .
+		"for $doc_OID.$section\n";
+	} 
+	$self->{'list'}->{"$doc_OID.$section"} = $metavalue;
+    }
+}
+
+sub get_classify_info {
+    my $self = shift (@_);
+
+    my @classlist = sort {$self->{'list'}->{$a} cmp $self->{'list'}->{$b};}
+    keys %{$self->{'list'}};
+
+    return $self->splitlist (\@classlist);
+}
+
+sub get_entry {
+    my $self = shift (@_);
+    my ($title, $childtype, $thistype) = @_;
+    
+    # organise into classification structure
+    my %classifyinfo = ('childtype'=>$childtype,
+			'Title'=>$title,
+			'contains'=>[]);
+    $classifyinfo{'thistype'} = $thistype 
+	if defined $thistype && $thistype =~ /\w/;
+
+    return \%classifyinfo;
+}
+
+# splitlist takes an ordered list of classifications (@$classlistref) and splits it
+# up into alphabetical sub-sections.
+sub splitlist {
+    my $self = shift (@_);
+    my ($classlistref) = @_;
+    my $classhash = {};
+
+    # top level
+    my $childtype = "HList";
+    if (scalar (@$classlistref) <= 20) {$childtype = "VList";}
+    my $classifyinfo = $self->get_entry ($self->{'metaname'}, $childtype, "Invisible");
+
+    # don't need to do any splitting if there are less than 20 classifications
+    if ((scalar @$classlistref) <= 20) {
+	foreach $subOID (@$classlistref) {
+	    push (@{$classifyinfo->{'contains'}}, {'OID'=>$subOID});
+	}
+	return $classifyinfo;
+    }
+	
+    # first split up the list into separate A-Z and 0-9 classifications
+    foreach $classification (@$classlistref) {
+	my $title = $self->{'list'}->{$classification};
+	$title =~ s/^(.).*$/$1/;
+	$title =~ tr/[a-z]/[A-Z]/;
+	if ($title =~ /^[0-9]$/) {$title = '0-9';}
+	elsif ($title !~ /^[A-Z]$/) {
+	    print STDERR "AZSectionList: WARNING $classification has badly " .
+		"formatted title ($title)\n";
+	}
+	$classhash->{$title} = [] unless defined $classhash->{$title};
+	push (@{$classhash->{$title}}, $classification);
+    }
+    $classhash = $self->compactlist ($classhash);
+
+    my @tmparr = ();
+    foreach $subsection (sort keys (%$classhash)) {
+	push (@tmparr, $subsection);
+    }
+    
+    # if there's a 0-9 section it will have been sorted to the beginning
+    # but we want it at the end
+    if ($tmparr[0] eq '0-9') {
+	shift @tmparr;
+	push (@tmparr, '0-9');
+    }
+
+    foreach $subclass (@tmparr) {
+	my $tempclassify = $self->get_entry($subclass, "VList");
+	foreach $subsubOID (@{$classhash->{$subclass}}) {
+	    push (@{$tempclassify->{'contains'}}, {'OID'=>$subsubOID});
+	}
+	push (@{$classifyinfo->{'contains'}}, $tempclassify);
+    }
+
+    return $classifyinfo;
+}
+
+sub compactlist {
+    my $self = shift (@_);
+    my ($classhashref) = @_;
+    my $compactedhash = {};
+    my @currentOIDs = ();
+    my $currentfirstletter = "";
+    my $currentlastletter = "";
+    my $lastkey = "";
+
+    # minimum and maximum documents to be displayed per page.
+    # the actual maximum will be max + (min-1).
+    # the smallest sub-section is a single letter at present
+    # so in this case there may be many times max documents
+    # displayed on a page.
+    my $min = 10; 
+    my $max = 30;
+
+    foreach $subsection (sort keys %$classhashref) {
+	if ($subsection eq '0-9') {
+	    @{$compactedhash->{$subsection}} = @{$classhashref->{$subsection}};
+	    next;
+	}
+	$currentfirstletter = $subsection if $currentfirstletter eq "";
+	if ((scalar (@currentOIDs) < $min) ||
+	    ((scalar (@currentOIDs) + scalar (@{$classhashref->{$subsection}})) <= $max)) {
+	    push (@currentOIDs, @{$classhashref->{$subsection}});
+	    $currentlastletter = $subsection;
+	} else {
+
+	    if ($currentfirstletter eq $currentlastletter) {
+		@{$compactedhash->{$currentfirstletter}} = @currentOIDs;
+		$lastkey = $currentfirstletter;
+	    } else {
+		@{$compactedhash->{"$currentfirstletter-$currentlastletter"}} = @currentOIDs;
+		$lastkey = "$currentfirstletter-$currentlastletter";
+	    } 
+	    if (scalar (@{$classhashref->{$subsection}}) >= $max) {
+		$compactedhash->{$subsection} = $classhashref->{$subsection};
+		@currentOIDs = ();
+		$currentfirstletter = "";
+		$lastkey = $subsection;
+	    } else {
+		@currentOIDs = @{$classhashref->{$subsection}};
+		$currentfirstletter = $subsection;
+		$currentlastletter = $subsection;
+	    }
+	}
+    }
+
+    # add final OIDs to last sub-classification if there aren't many otherwise
+    # add final sub-classification
+    if (scalar (@currentOIDs) < $min) {
+	my ($newkey) = $lastkey =~ /^(.)/;
+	@currentOIDs = (@{$compactedhash->{$lastkey}}, @currentOIDs);
+	delete $compactedhash->{$lastkey};
+	@{$compactedhash->{"$newkey-$currentlastletter"}} = @currentOIDs;	
+    } else {
+	if ($currentfirstletter eq $currentlastletter) {
+	    @{$compactedhash->{$currentfirstletter}} = @currentOIDs;
+	} else {
+	    @{$compactedhash->{"$currentfirstletter-$currentlastletter"}} = @currentOIDs;
+	} 
+    }
+
+    return $compactedhash;
+}
+
+1;
Index: /tags/gsdl-21/gsdl/perllib/classify/DateList.pm
===================================================================
--- /tags/gsdl-21/gsdl/perllib/classify/DateList.pm	(revision 699)
+++ /tags/gsdl-21/gsdl/perllib/classify/DateList.pm	(revision 699)
@@ -0,0 +1,197 @@
+###########################################################################
+#
+# DateList.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# classifier plugin for sorting by date
+
+# no options - always sorts by 'Date' metadata
+
+# date is assumed to be in the form yyyymmdd
+
+# at present dates are split by year - this should change
+
+package DateList;
+
+use sorttools;
+
+sub new {
+    my ($class, @options) = @_;
+
+    return bless {
+	'list'=>{},
+	}, $class;
+}
+
+sub init {
+    my $self = shift (@_);
+
+    $self->{'list'} = {};
+}
+
+sub classify {
+    my $self = shift (@_);
+    my ($doc_obj) = @_;
+
+    my $doc_OID = $doc_obj->get_OID();
+    my $date = $doc_obj->get_metadata_element ($doc_obj->get_top_section(), 'Date');
+
+    # if this document doesn't contain Date element we won't 
+    # include it in this classification
+    if (defined $date && $date =~ /\d/) {
+	if (defined $self->{'list'}->{$doc_OID}) {
+	    print STDERR "WARNING: DateList::classify called multiple times for $doc_OID\n";
+	} 
+	$self->{'list'}->{$doc_OID} = $date;
+    }
+}
+
+
+sub get_classify_info {
+    my $self = shift (@_);
+
+    my @classlist = sort {$self->{'list'}->{$a} cmp $self->{'list'}->{$b};} keys %{$self->{'list'}};
+
+    return $self->splitlist (\@classlist);
+}
+
+
+sub get_entry {
+    my $self = shift (@_);
+    my ($title, $childtype, $thistype) = @_;
+    
+    # organise into classification structure
+    my %classifyinfo = ('childtype'=>$childtype,
+			'Title'=>$title,
+			'contains'=>[]);
+    $classifyinfo{'thistype'} = $thistype 
+	if defined $thistype && $thistype =~ /\w/;
+
+    return \%classifyinfo;
+}
+
+# splitlist takes an ordered list of classifications (@$classlistref) and splits it
+# up into sub-sections by date
+sub splitlist {
+    my $self = shift (@_);
+    my ($classlistref) = @_;
+    my $classhash = {};
+
+    # top level
+    my $childtype = "HList";
+    if (scalar (@$classlistref) <= 20) {$childtype = "DateList";}
+    my $classifyinfo = $self->get_entry ("Date", $childtype, "Invisible");
+
+    # don't need to do any splitting if there are less than 20 classifications
+    if ((scalar @$classlistref) <= 20) {
+	foreach $subOID (@$classlistref) {
+	    push (@{$classifyinfo->{'contains'}}, {'OID'=>$subOID});
+	}
+	return $classifyinfo;
+    }
+
+    # first split up the list into separate year classifications
+    foreach $classification (@$classlistref) {
+	my $date = $self->{'list'}->{$classification};
+	$date =~ s/^(\d\d\d\d).*$/$1/;
+	$classhash->{$date} = [] unless defined $classhash->{$date};
+	push (@{$classhash->{$date}}, $classification);
+    }
+
+    $classhash = $self->compactlist ($classhash);
+
+    foreach $subclass (sort keys %$classhash) {
+	my $tempclassify = $self->get_entry($subclass, "DateList");
+	foreach $subsubOID (@{$classhash->{$subclass}}) {
+	    push (@{$tempclassify->{'contains'}}, {'OID'=>$subsubOID});
+	}
+	push (@{$classifyinfo->{'contains'}}, $tempclassify);
+    }
+
+    return $classifyinfo;
+}
+
+sub compactlist {
+    my $self = shift (@_);
+    my ($classhashref) = @_;
+    my $compactedhash = {};
+    my @currentOIDs = ();
+    my $currentfirstletter = "";
+    my $currentlastletter = "";
+    my $lastkey = "";
+
+    # minimum and maximum documents to be displayed per page.
+    # the actual maximum will be max + (min-1).
+    # the smallest sub-section is a single letter at present
+    # so in this case there may be many times max documents
+    # displayed on a page.
+    my $min = 10; 
+    my $max = 30;
+
+    foreach $subsection (sort keys %$classhashref) {
+	$currentfirstletter = $subsection if $currentfirstletter eq "";
+	if ((scalar (@currentOIDs) < $min) ||
+	    ((scalar (@currentOIDs) + scalar (@{$classhashref->{$subsection}})) <= $max)) {
+	    push (@currentOIDs, @{$classhashref->{$subsection}});
+	    $currentlastletter = $subsection;
+	} else {
+
+	    if ($currentfirstletter eq $currentlastletter) {
+		@{$compactedhash->{$currentfirstletter}} = @currentOIDs;
+		$lastkey = $currentfirstletter;
+	    } else {
+		@{$compactedhash->{"$currentfirstletter-$currentlastletter"}} = @currentOIDs;
+		$lastkey = "$currentfirstletter-$currentlastletter";
+	    } 
+	    if (scalar (@{$classhashref->{$subsection}}) >= $max) {
+		$compactedhash->{$subsection} = $classhashref->{$subsection};
+		@currentOIDs = ();
+		$currentfirstletter = "";
+		$lastkey = $subsection;
+	    } else {
+		@currentOIDs = @{$classhashref->{$subsection}};
+		$currentfirstletter = $subsection;
+		$currentlastletter = $subsection;
+	    }
+	}
+    }
+
+    # add final OIDs to last sub-classification if there aren't many otherwise
+    # add final sub-classification
+    if (scalar (@currentOIDs) < $min) {
+	my ($newkey) = $lastkey =~ /^(\d\d\d\d)/;
+	@currentOIDs = (@{$compactedhash->{$lastkey}}, @currentOIDs);
+	delete $compactedhash->{$lastkey};
+	@{$compactedhash->{"$newkey-$currentlastletter"}} = @currentOIDs;	
+    } else {
+	if ($currentfirstletter eq $currentlastletter) {
+	    @{$compactedhash->{$currentfirstletter}} = @currentOIDs;
+	} else {
+	    @{$compactedhash->{"$currentfirstletter-$currentlastletter"}} = @currentOIDs;
+	} 
+    }
+
+    return $compactedhash;
+}
+
+1;
Index: /tags/gsdl-21/gsdl/perllib/classify/HTML.pm
===================================================================
--- /tags/gsdl-21/gsdl/perllib/classify/HTML.pm	(revision 699)
+++ /tags/gsdl-21/gsdl/perllib/classify/HTML.pm	(revision 699)
@@ -0,0 +1,83 @@
+###########################################################################
+#
+# HTML.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# html classifier plugin - creates an empty classification
+# that's simply a link to a web page
+# options are:
+# title=Title       -- (optional) the title field for this classification.
+#                      if not included title field 'Browse'
+# url=url           -- the url of the web page to link to
+
+package HTML;
+
+sub new {
+    my ($class, @options) = @_;
+
+    my ($title, $url);
+
+    foreach $option (@options) {
+	if ($option =~ /^url=(.*)$/i) {
+	    $url = $1;
+	} elsif ($option =~ /^title=(.*)$/i) {
+	    $title = $1;
+	}
+    }
+
+    die "Error: HTML classifier contains no 'url' option\n" 
+	unless defined $url;
+
+    $title = 'Browse' unless defined $title;
+
+    return bless {
+	'url' => $url,
+	'title' => $title
+	}, $class;
+}
+
+sub init {
+    my $self = shift (@_);
+}
+
+sub classify {
+    my $self = shift (@_);
+    my ($doc_obj) = @_;
+
+    # we don't do anything for individual documents
+}
+
+sub get_classify_info {
+    my $self = shift (@_);
+
+    my %classifyinfo = ('classifytype'=>'HTML',
+			'Title'=>$self->{'title'},
+			'contains'=>[]);
+
+    push (@{$classifyinfo{'contains'}}, {'OID'=>$self->{'url'}});
+
+    return \%classifyinfo;
+}
+
+
+1;
Index: /tags/gsdl-21/gsdl/perllib/classify/Hierarchy.pm
===================================================================
--- /tags/gsdl-21/gsdl/perllib/classify/Hierarchy.pm	(revision 699)
+++ /tags/gsdl-21/gsdl/perllib/classify/Hierarchy.pm	(revision 699)
@@ -0,0 +1,218 @@
+###########################################################################
+#
+# Hierarchy.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# classifier plugin for generating hierarchical classifications
+
+# options for this classifier are:
+# hfile=file.txt    - classification file
+# metadata=Metaname - metadata field to test against file.txt
+# sort=Meta         - this option is optional (genious;-). by default this 
+#                     classifier will sort documents within each section 
+#                     alphabetically by Title. sort=nosort prevents sorting 
+#                     (i.e. documents will end up in build order), sort=Meta 
+#                     will sort each field alphabetically by Meta (Meta may 
+#                     also be 'Filename' to sort by the original filename).
+# title=Title       - another optional field. this is what will end up in the
+#                     Title field for this classification. if not present it
+#                     defaults to Metaname
+
+package Hierarchy;
+
+use util;
+use cfgread;
+use sorttools;
+
+sub new {
+    my ($class, @options) = @_;
+
+    my $sortname = "Title";
+    my ($hfile, $metadata, $title);
+
+    foreach $option (@options) {
+	if ($option =~ /^sort=(.*)$/i) {
+	    $sortname = $1;
+	    $sortname = undef if $sortname =~ /^nosort$/i;
+	} elsif ($option =~ /^hfile=(.*)$/i) {
+	    $hfile = $1;
+	} elsif ($option =~ /^metadata=(.*)$/i) {
+	    $metadata = $1;
+	} elsif ($option =~ /^title=(.*)$/i) {
+	    $title = $1;
+	}
+    }
+
+    if (!defined ($hfile) || !defined ($metadata)) {
+	die "Error in options passed to Hierarchy classification\n" .
+	    "hfile and metadata options aren't optional\n\n" .
+		"Options:\n" .
+		    "hfile -- the classification structure file\n" .
+			"metadata -- metadata field to test against hfile\n" .
+			    "sort -- metadata field to sort by\n" .
+				"title -- title of classification\n\n";
+    }
+
+    $title = $metadata unless defined $title;
+    
+    my $subjectfile = &util::filename_cat($ENV{'GSDLCOLLECTDIR'},"etc", $hfile);
+    if (!-e $subjectfile) {
+	my $collfile = $subjectfile;
+	$subjectfile = &util::filename_cat($ENV{'GSDLHOME'},"etc", $hfile);
+	if (!-e $subjectfile) {
+	    die "Hierarchy Error: Can't locate subject file $hfile\n" .
+		"This file should be in $collfile or $subjectfile\n";
+	}
+    }
+
+    return bless {
+	'descriptorlist'=>{}, # first field in subject file
+	'locatorlist'=>{},    # second field in subject file
+	'subjectfile' => $subjectfile,
+	'metaname' => $metadata,
+	'sortname' => $sortname,
+	'title' => $title
+	}, $class;
+}
+
+sub init {
+    my $self = shift (@_);
+
+    # read in the subject file
+    my $list = &cfgread::read_cfg_file ($self->{'subjectfile'}, undef, '^[^#]?\w');
+    
+    # $list is a hash that is indexed by the descriptor. The contents of this
+    # hash is a list of two items. The first item is the OID and the second item
+    # is the title
+    foreach $descriptor (keys (%$list)) {
+	$self->{'descriptorlist'}->{$descriptor} = $list->{$descriptor}->[0];
+	unless (defined $self->{'locatorlist'}->{$list->{$descriptor}->[0]}) {
+	    $self->{'locatorlist'}->{$list->{$descriptor}->[0]}->{'title'} = $list->{$descriptor}->[1];
+	    $self->{'locatorlist'}->{$list->{$descriptor}->[0]}->{'contents'} = [];
+	}
+    }
+}
+
+sub classify {
+    my $self = shift (@_);
+    my ($doc_obj) = @_;
+
+    my $doc_OID = $doc_obj->get_OID();
+
+    my $metadata = $doc_obj->get_metadata ($doc_obj->get_top_section(),
+					   $self->{'metaname'});
+
+    my $sortmeta = "";
+    if (defined $self->{'sortname'}) {
+	if ($self->{'sortname'} =~ /^filename$/i) {
+	    $sortmeta = $doc_obj->get_source_filename();
+	} else {
+	    $sortmeta = $doc_obj->get_metadata_element($doc_obj->get_top_section(),
+						       $self->{'sortname'});
+	    if (defined $sortmeta) {
+		if ($self->{'sortname'} eq "Creator") {
+		    &sorttools::format_string_name_english (\$sortmeta);
+		} else {
+		    &sorttools::format_string_english (\$sortmeta);
+		}
+	    }
+	}
+	$sortmeta = "" unless defined $sortmeta;
+    }
+
+    foreach $metaelement (@$metadata) {
+	if ((defined $self->{'descriptorlist'}->{$metaelement}) &&
+	    (defined $self->{'locatorlist'}->{$self->{'descriptorlist'}->{$metaelement}})) {
+
+	    push (@{$self->{'locatorlist'}->{$self->{'descriptorlist'}->{$metaelement}}->{'contents'}}, 
+		  [$doc_OID, $sortmeta]);
+	}
+    }
+}
+
+sub get_classify_info {
+    my $self = shift (@_);
+    
+    my $list = $self->{'locatorlist'};
+
+    my $classifyinfo = $self->get_entry ($self->{'title'}, "VList", "Invisible");
+    foreach $OID (keys (%$list)) {
+	my $tempinfo = $self->get_OID_entry ($OID, $classifyinfo, $list->{$OID}->{'title'}, "VList");
+
+	if (defined $self->{'sortname'}) {
+	    foreach $subOID (sort {$a->[1] cmp $b->[1];} @{$list->{$OID}->{'contents'}}) {
+		push (@{$tempinfo->{'contains'}}, {'OID'=>$subOID->[0]});
+	    }
+	} else {
+	    foreach $subOID (@{$list->{$OID}->{'contents'}}) {
+		push (@{$tempinfo->{'contains'}}, {'OID'=>$subOID->[0]});
+	    }
+	}
+    }
+    
+    return $classifyinfo;
+}
+
+sub get_OID_entry {
+    my $self = shift (@_);
+    my ($OID, $classifyinfo, $title, $classifytype) = @_;
+
+    $OID = "" unless defined $OID;
+    $OID =~ s/^\.+//;
+    
+    my ($headOID, $tailOID) = $OID =~ /^(\d+)(.*)$/;
+    $tailOID = "" unless defined $tailOID;
+
+
+    if (!defined $headOID) {
+	$classifyinfo->{'Title'} = $title;
+	$classifyinfo->{'classifytype'} = $classifytype;
+	return $classifyinfo;
+    }
+
+    $classifyinfo->{'contains'} = [] unless defined $classifyinfo->{'contains'};
+    
+    # add entries to the contains list until we have one for headOID
+    while (scalar(@{$classifyinfo->{'contains'}}) < $headOID) {
+	push (@{$classifyinfo->{'contains'}}, $self->get_entry("", $classifytype));
+    }
+
+    return $self->get_OID_entry ($tailOID, $classifyinfo->{'contains'}->[($headOID-1)], $title, $classifytype);
+}
+
+sub get_entry {
+    my $self = shift (@_);
+    my ($title, $childtype, $thistype) = @_;
+    
+    # organise into classification structure
+    my %classifyinfo = ('childtype'=>$childtype,
+			'Title'=>$title,
+			'contains'=>[]);
+    $classifyinfo{'thistype'} = $thistype 
+	if defined $thistype && $thistype =~ /\w/;
+
+    return \%classifyinfo;
+}
+
+
+1;
Index: /tags/gsdl-21/gsdl/perllib/classify/List.pm
===================================================================
--- /tags/gsdl-21/gsdl/perllib/classify/List.pm	(revision 699)
+++ /tags/gsdl-21/gsdl/perllib/classify/List.pm	(revision 699)
@@ -0,0 +1,170 @@
+###########################################################################
+#
+# List.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# simple list classifier plugin 
+# options are:
+# metadata=Metaname -- (optional) all documents with Metaname metadata
+#                      will be included in list. if not included all documents
+#                      will be included in list.
+# sort=Meta         -- (optional) sort documents in list alphabetically by
+#                      Meta. by default it will sort by Metaname, if neither
+#                      are set documents will be in build (random) order.
+#                      Meta may be Filename to sort by original filename or
+#                      nosort to force not to sort
+# title=Title       -- (optional) the title field for this classification.
+#                      if not included title field will be Metaname. 
+#                      if metadata is also not included title will be 'List'
+
+package List;
+
+use sorttools;
+
+sub new {
+    my ($class, @options) = @_;
+
+    my $list = [];
+    my ($metaname, $title, $sortname);
+
+    foreach $option (@options) {
+	if ($option =~ /^metadata=(.*)$/i) {
+	    $metaname = $1;
+	    $list = {};
+	} elsif ($option =~ /^title=(.*)$/i) {
+	    $title = $1;
+	} elsif ($option =~ /^sort=(.*)$/i) {
+	    $sortname = $1;
+	}
+    }
+
+    if (!defined $title) {
+	if (defined $metaname) {
+	    $title = $metaname;
+	} else {
+	    $title = 'List';
+	}
+    }
+
+    if (defined $sortname && $sortname =~ /^nosort$/i) {
+	$sortname = undef;
+    } elsif (!defined $sortname && defined $metaname) {
+	$sortname = $metaname;
+    }
+
+    return bless {
+	'list'=>$list,
+	'metaname' => $metaname,
+	'title' => $title,
+	'sortname' => $sortname
+	}, $class;
+}
+
+sub init {
+    my $self = shift (@_);
+
+    if (defined $self->{'sortname'}) {
+	$self->{'list'} = {};
+    } else {
+	$self->{'list'} = [];
+    }
+}
+
+sub classify {
+    my $self = shift (@_);
+    my ($doc_obj) = @_;
+
+    my $doc_OID = $doc_obj->get_OID();
+
+    my $sortmeta = "";
+    if (defined $self->{'sortname'}) {
+	if ($self->{'sortname'} =~ /^filename$/i) {
+	    $sortmeta = $doc_obj->get_source_filename();
+	} else {
+	    $sortmeta = $doc_obj->get_metadata_element($doc_obj->get_top_section(),
+						       $self->{'sortname'});
+	    if (defined $sortmeta) {
+		if ($self->{'sortname'} eq "Creator") {
+		    &sorttools::format_string_name_english (\$sortmeta);
+		} else {
+		    &sorttools::format_string_english (\$sortmeta);
+		}
+	    }
+	}
+	$sortmeta = "" unless defined $sortmeta;
+
+	if (defined $self->{'metaname'}) {
+	    my $metavalue = $doc_obj->get_metadata_element ($doc_obj->get_top_section(),
+							    $self->{'metaname'});
+	    if (defined $metavalue) {
+		if (defined $self->{'list'}->{$doc_OID}) {
+		    print STDERR "WARNING: List::classify called multiple times for $doc_OID\n";
+		} 
+		$self->{'list'}->{$doc_OID} = $sortmeta;
+	    }
+	} else {
+	    if (defined $self->{'list'}->{$doc_OID}) {
+		print STDERR "WARNING: List::classify called multiple times for $doc_OID\n";
+	    } 
+	    $self->{'list'}->{$doc_OID} = $sortmeta;
+	}
+    } else {
+	if (defined $self->{'metaname'}) {
+	    my $metavalue = $doc_obj->get_metadata_element ($doc_obj->get_top_section(),
+							    $self->{'metaname'});
+	    if (defined $metavalue) {
+		push (@{$self->{'list'}}, $doc_OID);
+	    }
+	} else {
+	    push (@{$self->{'list'}}, $doc_OID);
+	}	    	    
+    }
+}
+
+sub get_classify_info {
+    my $self = shift (@_);
+
+    my @list = ();
+    if (defined $self->{'sortname'}) {
+	if (keys %{$self->{'list'}}) {
+	    @list = sort {$self->{'list'}->{$a} 
+			  cmp $self->{'list'}->{$b};} keys %{$self->{'list'}};
+	}
+    } else {
+	@list = @{$self->{'list'}};
+    }
+
+    # organise into classification structure
+    my %classifyinfo = ('thistype'=>'Invisible',
+			'childtype'=>'VList',
+			'Title'=>$self->{'title'},
+			'contains'=>[]);
+    foreach $OID (@list) {
+	push (@{$classifyinfo{'contains'}}, {'OID'=>$OID});
+    }
+
+    return \%classifyinfo;
+}
+
+
+1;
Index: /tags/gsdl-21/gsdl/perllib/classify/SectionList.pm
===================================================================
--- /tags/gsdl-21/gsdl/perllib/classify/SectionList.pm	(revision 699)
+++ /tags/gsdl-21/gsdl/perllib/classify/SectionList.pm	(revision 699)
@@ -0,0 +1,186 @@
+###########################################################################
+#
+# SectionList.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# Same as List classifier but includes all sections of document
+# (excluding top level) rather than just top level document 
+# itself
+# options are:
+# metadata=Metaname -- (optional) all documents with Metaname metadata
+#                      will be included in list. if not included all documents
+#                      will be included in list.
+# sort=Meta         -- (optional) sort documents in list alphabetically by
+#                      Meta. by default it will sort by Metaname, if neither
+#                      are set documents will be in build (random) order.
+#                      Meta may be Filename to sort by original filename or
+#                      nosort to force not to sort
+# title=Title       -- (optional) the title field for this classification.
+#                      if not included title field will be Metaname. 
+#                      if metadata is also not included title will be 'List'
+
+package SectionList;
+
+use sorttools;
+
+sub new {
+    my ($class, @options) = @_;
+
+    my $list = [];
+    my ($metaname, $title, $sortname);
+
+    foreach $option (@options) {
+	if ($option =~ /^metadata=(.*)$/i) {
+	    $metaname = $1;
+	    $list = {};
+	} elsif ($option =~ /^title=(.*)$/i) {
+	    $title = $1;
+	} elsif ($option =~ /^sort=(.*)$/i) {
+	    $sortname = $1;
+	}
+    }
+
+    if (!defined $title) {
+	if (defined $metaname) {
+	    $title = $metaname;
+	} else {
+	    $title = 'List';
+	}
+    }
+
+    if (defined $sortname && $sortname =~ /^nosort$/i) {
+	$sortname = undef;
+    } elsif (!defined $sortname && defined $metaname) {
+	$sortname = $metaname;
+    }
+
+    return bless {
+	'list'=>$list,
+	'metaname' => $metaname,
+	'title' => $title,
+	'sortname' => $sortname
+	}, $class;
+}
+
+sub init {
+    my $self = shift (@_);
+
+    if (defined $self->{'sortname'}) {
+	$self->{'list'} = {};
+    } else {
+	$self->{'list'} = [];
+    }
+}
+
+sub classify {
+    my $self = shift (@_);
+    my ($doc_obj) = @_;
+
+    my $thissection = $doc_obj->get_next_section ($doc_obj->get_top_section());
+
+    my $sortmeta = "";
+    if (defined $self->{'sortname'}) {
+	if ($self->{'sortname'} =~ /^filename$/i) {
+	    $sortmeta = $doc_obj->get_source_filename();
+	} else {
+	    $sortmeta = $doc_obj->get_metadata_element($doc_obj->get_top_section(),
+						       $self->{'sortname'});
+	    if ($self->{'sortname'} eq "Creator") {
+		&sorttools::format_string_name_english (\$sortmeta);
+	    } else {
+		&sorttools::format_string_english (\$sortmeta);
+	    }
+	}
+	$sortmeta = "" unless defined $sortmeta;
+    }
+
+    while (defined $thissection) {
+	$self->classify_section($thissection, $doc_obj, $sortmeta);
+	$thissection = $doc_obj->get_next_section ($thissection);
+    }
+}
+
+sub classify_section {
+    my $self = shift (@_);
+    my ($section, $doc_obj, $sortmeta) = @_;
+
+    my $doc_OID = $doc_obj->get_OID();
+    $sortmeta = "" unless defined $sortmeta;
+
+    if (defined $self->{'sortname'}) {
+	if (defined $self->{'metaname'}) {
+
+	    my $metavalue = $doc_obj->get_metadata_element ($section, $self->{'metaname'});
+	    if (defined $metavalue) {
+		if (defined $self->{'list'}->{"$doc_OID.$section"}) {
+		    print STDERR "WARNING: SectionList::classify called multiple times for $doc_OID.$section\n";
+		} 
+		$self->{'list'}->{"$doc_OID.$section"} = $sortmeta;
+	    }
+	} else {
+	    if (defined $self->{'list'}->{"$doc_OID.$section"}) {
+		print STDERR "WARNING: SectionList::classify called multiple times for $doc_OID.$section\n";
+	    } 
+	    $self->{'list'}->{"$doc_OID.$section"} = $sortmeta;
+	}
+
+    } else {
+	if (defined $self->{'metaname'}) {
+
+	    my $metavalue = $doc_obj->get_metadata_element ($section, $self->{'metaname'});
+	    if (defined $metavalue) {
+		push (@{$self->{'list'}}, "$doc_OID.$section");
+	    }
+	} else {
+	    push (@{$self->{'list'}}, "$doc_OID.$section");
+	}
+    }
+}
+
+sub get_classify_info {
+    my $self = shift (@_);
+
+    my @list = ();
+    if (defined $self->{'sortname'}) {
+	if (keys %{$self->{'list'}}) {
+	    @list = sort {$self->{'list'}->{$a} 
+			  cmp $self->{'list'}->{$b};} keys %{$self->{'list'}};
+	}
+    } else {
+	@list = @{$self->{'list'}};
+    }
+
+    # organise into classification structure
+    my %classifyinfo = ('thistype'=>'Invisible',
+			'childtype'=>'VList',
+			'Title'=>$self->{'title'},
+			'contains'=>[]);
+    foreach $OID (@list) {
+	push (@{$classifyinfo{'contains'}}, {'OID'=>$OID});
+    }
+
+    return \%classifyinfo;
+}
+
+
+1;
Index: /tags/gsdl-21/gsdl/perllib/cnseg.pm
===================================================================
--- /tags/gsdl-21/gsdl/perllib/cnseg.pm	(revision 699)
+++ /tags/gsdl-21/gsdl/perllib/cnseg.pm	(revision 699)
@@ -0,0 +1,74 @@
+###########################################################################
+#
+# cnseg.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+
+# this package segments a chinese UTF-8 encoded Unicode
+# string into words.
+
+package cnseg;
+
+use unicode;
+
+
+# 'segment' takes a UTF-8 encoded Unicode Chinese-language
+# string and places U-200B between words -- the ZERO
+# WIDTH SPACE. Each line is treated as a separate 
+# paragraph, so lines in one paragraph should
+# be joined before using this method (normally a single
+# word might span more than one line within a paragraph).
+#
+# 'segment' is currently written in Perl, however, I (Rodger)
+# plan to use C++ (via pipes) once a more complex (and useful!)
+# algorithm is being used. Currently, each Chinese character
+# is treated as a seperate word.
+
+sub segment {
+    my ($in) = @_;
+    my ($c);
+
+    my $uniin = &unicode::utf82unicode($in);
+    my $out = [];
+
+    my $space = 1; # start doesn't need a space
+    foreach $c (@$uniin) {
+	if (($c >= 0x4e00 && $c <= 0x9fa5) ||
+	    ($c >= 0xf900 && $c <= 0xfa2d)) {
+	    # Chinese character
+	    push (@$out, 0x200b) unless $space;
+	    push (@$out, $c);
+	    push (@$out, 0x200b);
+	    $space = 1;
+
+	} else {
+	    # non-Chinese character
+	    push (@$out, $c);
+	    $space = 0;
+	}
+    }
+
+    return &unicode::unicode2utf8($out);
+}
+
+1;
Index: /tags/gsdl-21/gsdl/perllib/colcfg.pm
===================================================================
--- /tags/gsdl-21/gsdl/perllib/colcfg.pm	(revision 699)
+++ /tags/gsdl-21/gsdl/perllib/colcfg.pm	(revision 699)
@@ -0,0 +1,94 @@
+###########################################################################
+#
+# colcfg.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# reads in a collection configuration file
+
+package colcfg;
+
+use cfgread;
+
+
+# the collection configuration file data is stored in the form
+#
+# {'creator'}->string
+# {'maintainer'}->array of strings
+# {'public'}->string
+# {'beta'}->string
+# {'key'}->string
+# {'languages'}->array of strings
+# {'subcollection'}->hash of key-value pairs
+# {'indexsubcollections'}->array of strings
+# {'indexes'}->array of strings
+# {'defaultindex'}->string
+# {'plugins'}->array of strings
+
+sub read_collect_cfg {
+    my ($filename) = @_;
+
+    return &cfgread::read_cfg_file ($filename, 
+		    '^(creator|public|beta|key|defaultindex|importdir|' .
+				    'archivedir|cachedir|builddir|textindex|removeold)$',
+		    '^(maintainer|languages|indexsubcollections|indexes|plugins|dontbuild)$',
+		    '^(subcollection|collectionmeta)$');
+}
+
+sub write_collect_cfg {
+    my ($filename, $data) = @_;
+
+    &cfgread::write_cfg_file($filename, $data,
+		    '^(creator|public|beta|key|defaultindex|importdir|' .
+				    'archivedir|cachedir|builddir|textindex|removeold)$',
+		    '^(maintainer|languages|indexsubcollections|indexes|plugins|dontbuild)$',
+		    '^(subcollection|collectionmeta)$');
+}
+
+
+# the build configuration file data is stored in the form
+#
+# {'builddate'}->string
+# {'metadata'}->array of strings
+# {'languages'}->array of strings
+# {'numdocs'}->string
+# {'numwords'}->string
+# {'numbytes'}->string
+
+sub read_build_cfg {
+    my ($filename) = @_;
+
+    return &cfgread::read_cfg_file ($filename, 
+		    '^(builddate|numdocs|numwords|numbytes)$', 
+		    '^(metadata|languages)$');
+}
+
+sub write_build_cfg {
+    my ($filename, $data) = @_;
+
+    &cfgread::write_cfg_file($filename, $data,
+		    '^(builddate|numdocs|numwords|numbytes)$', 
+		    '^(metadata|languages)$');
+}
+
+
+1;
Index: /tags/gsdl-21/gsdl/perllib/doc.pm
===================================================================
--- /tags/gsdl-21/gsdl/perllib/doc.pm	(revision 699)
+++ /tags/gsdl-21/gsdl/perllib/doc.pm	(revision 699)
@@ -0,0 +1,769 @@
+###########################################################################
+#
+# doc.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# class to hold documents
+
+package doc;
+
+use unicode;
+use util;
+use html;
+
+# the document type may be indexed_doc, nonindexed_doc, or
+# classification
+
+sub new {
+    my $class = shift (@_);
+    my ($source_filename, $doc_type) = @_;
+    
+    my $self = bless {'associated_files'=>[],
+		      'subsection_order'=>[],
+		      'next_subsection'=>1,
+		      'subsections'=>{},
+		      'metadata'=>[],
+		      'text'=>""}, $class;
+
+
+#    $self->set_source_filename ($source_filename) if defined $source_filename;
+    push (@{$self->{'metadata'}}, ["gsdlsourcefilename", &unicode::ascii2utf8($source_filename)]) 
+	if defined $source_filename;
+#    $self->set_doc_type ($doc_type) if defined $doc_type;
+    push (@{$self->{'metadata'}}, ["gsdldoctype", &unicode::ascii2utf8($doc_type)]) 
+	if defined $doc_type;
+
+    return $self;
+}
+
+sub set_source_filename {
+    my $self = shift (@_);
+    my ($source_filename) = @_;
+
+    $self->set_metadata_element ($self->get_top_section(), 
+				 "gsdlsourcefilename", 
+				 $source_filename);
+}
+
+# returns the source_filename as it was provided
+sub get_source_filename {
+    my $self = shift (@_);
+
+    return $self->get_metadata_element ($self->get_top_section(), "gsdlsourcefilename");
+}
+
+sub set_doc_type {
+    my $self = shift (@_);
+    my ($doc_type) = @_;
+
+    $self->set_metadata_element ($self->get_top_section(), 
+				 "gsdldoctype", 
+				 $doc_type);
+}
+
+# returns the source_filename as it was provided
+# the default of "indexed_doc" is used if no document
+# type was provided
+sub get_doc_type {
+    my $self = shift (@_);
+
+    my $doc_type = $self->get_metadata_element ($self->get_top_section(), "gsdldoctype");
+    return $doc_type if (defined $doc_type);
+    return "indexed_doc";
+}
+
+sub _escape_text {
+    my ($text) = @_;
+
+    # special characters in the gml encoding
+    $text =~ s/&/&amp;/g; # this has to be first...
+    $text =~ s/</&lt;/g;
+    $text =~ s/>/&gt;/g;
+    $text =~ s/\"/&quot;/g;
+
+    return $text;
+}
+
+sub output_section {
+    my $self = shift (@_);
+    my ($handle, $section, $suppress_subject_info) = @_;
+    $suppress_subject_info = 0 unless defined $suppress_subject_info;
+    my ($data, $subsection);
+    
+    my $section_ptr = $self->_lookup_section ($section);
+    my ($section_num) = $section =~ /(\d+)$/;
+    
+    return unless defined $section_ptr;
+
+    # output the section header (including the section number
+    # and metadata)
+    print $handle "<gsdlsection";
+    print $handle " gsdlnum=\"$section_num\"" if defined $section_num;
+    foreach $data (@{$section_ptr->{'metadata'}}) {
+	print $handle " $data->[0]=\"" . &_escape_text($data->[1]) . "\"" 
+	    unless $suppress_subject_info && $data->[0] eq "Subject";
+    }
+    print $handle ">";
+
+    # output the text
+    print $handle &_escape_text($section_ptr->{'text'});
+
+    # output all the subsections
+    foreach $subsection (@{$section_ptr->{'subsection_order'}}) {
+	$self->output_section($handle, "$section.$subsection", $suppress_subject_info);
+    }
+    
+    # output the closing tag
+    print $handle "</gsdlsection>\n";
+}
+
+# look up the reference to the a particular section
+sub _lookup_section {
+    my $self = shift (@_);
+    my ($section) = @_;
+
+    my ($num);
+    my $sectionref = $self;
+
+    while (defined $section && $section ne "") {
+	($num, $section) = $section =~ /^\.?(\d+)(.*)$/;
+	$num =~ s/^0+(\d)/$1/; # remove leading 0s
+	$section = "" unless defined $section;
+	
+	if (defined $num && defined $sectionref->{'subsections'}->{$num}) {
+	    $sectionref = $sectionref->{'subsections'}->{$num};
+	} else {
+	    return undef;
+	}
+    }
+    
+    return $sectionref;
+}
+
+sub _calc_OID {
+    my $self = shift (@_);
+    my ($filename) = @_;
+
+    my $osexe = &util::get_os_exe();
+
+    my $hashfile_exe = &util::filename_cat($ENV{'GSDLHOME'},"bin",
+					   $ENV{'GSDLOS'},"hashfile$osexe");
+    my $result = "NULL";
+
+    if (-e "$hashfile_exe") {
+	$result = `$hashfile_exe \"$filename\"`;
+	($result) = $result =~ /:\s*([0-9a-f]+)/i;
+
+    } else {
+	print STDERR "doc::_calc_OID $hashfile_exe could not be found\n";
+    }
+
+    return "HASH$result";
+}
+
+# methods dealing with OID, not groups of them.
+
+# if $OID is not provided one is calculated from hashing the
+# current contents of the document
+# An OID are actually stored as metadata of the document
+sub set_OID {
+    my $self = shift (@_);
+    my ($OID) = @_;
+
+    # if an OID wasn't provided then feed this document to 
+    # hashfile.exe
+    if (!defined $OID) {
+	$OID = "NULL";
+	my $tmp_filename = &util::get_tmp_filename();
+	if (!open (OUTFILE, ">$tmp_filename")) {
+	    print STDERR "doc::set_OID could not write to $tmp_filename\n";
+	} else {
+	    $self->output_section('OUTFILE', $self->get_top_section(), 1);
+	    close (OUTFILE);
+
+	    $OID = $self->_calc_OID ($tmp_filename);
+	    &util::rm ($tmp_filename);
+	}
+    }
+
+    $self->set_metadata_element ($self->get_top_section(), "Identifier", $OID);
+}
+
+# returns the OID for this document
+sub get_OID {
+    my $self = shift (@_);
+    my $OID = $self->get_metadata_element ($self->get_top_section(), "Identifier");
+    return $OID if (defined $OID);
+    return "NULL";
+}
+
+sub delete_OID {
+    my $self = shift (@_);
+    
+    $self->set_metadata_element ($self->get_top_section(), "Identifier", "NULL");
+}
+
+
+# methods for manipulating section names
+
+# returns the name of the top-most section (the top
+# level of the document
+sub get_top_section {
+    my $self = shift (@_);
+    
+    return "";
+}
+
+# returns a section
+sub get_parent_section {
+    my $self = shift (@_);
+    my ($section) = @_;
+
+    $section =~ s/(^|\.)\d+$//;
+
+    return $section;
+}
+
+# returns the first child section (or the end child 
+# if there isn't any)
+sub get_begin_child {
+    my $self = shift (@_);
+    my ($section) = @_;
+
+    my $section_ptr = $self->_lookup_section($section);
+    return "" unless defined $section_ptr;
+
+    if (defined $section_ptr->{'subsection_order'}->[0]) {
+	return "$section.$section_ptr->{'subsection_order'}->[0]";
+    }
+
+    return $self->get_end_child ($section);
+}
+
+# returns the next child of a parent section
+sub get_next_child {
+    my $self = shift (@_);
+    my ($section) = @_;
+    
+    my $parent_section = $self->get_parent_section($section);
+    my $parent_section_ptr = $self->_lookup_section($parent_section);
+    return undef unless defined $parent_section_ptr;
+
+    my ($section_num) = $section =~ /(\d+)$/;
+    return undef unless defined $section_num;
+
+    my $i = 0;
+    my $section_order = $parent_section_ptr->{'subsection_order'};
+    while ($i < scalar(@$section_order)) {
+	last if $section_order->[$i] eq $section_num;
+	$i++;
+    }
+
+    $i++; # the next child
+    if ($i < scalar(@$section_order)) {
+	return $section_order->[$i] if $parent_section eq "";
+	return "$parent_section.$section_order->[$i]";
+    }
+
+    # no more sections in this level
+    return undef;
+}
+
+# returns a reference to a list of children
+sub get_children {
+    my $self = shift (@_);
+    my ($section) = @_;
+
+    my $section_ptr = $self->_lookup_section($section);
+    return [] unless defined $section_ptr;
+
+    my @children = @{$section_ptr->{'subsection_order'}};
+
+    map {$_ = "$section.$_"; $_ =~ s/^\.+//;} @children;
+    return \@children;
+}
+
+# returns the child section one past the last one (which
+# is coded as "0")
+sub get_end_child {
+    my $self = shift (@_);
+    my ($section) = @_;
+
+    return $section . ".0" unless $section eq "";
+    return "0";
+}
+
+# returns the next section in book order
+sub get_next_section {
+    my $self = shift (@_);
+    my ($section) = @_;
+
+    return undef unless defined $section;
+
+    my $section_ptr = $self->_lookup_section($section);
+    return undef unless defined $section_ptr;
+
+    # first try to find first child
+    if (defined $section_ptr->{'subsection_order'}->[0]) {
+	return $section_ptr->{'subsection_order'}->[0] if ($section eq "");
+	return "$section.$section_ptr->{'subsection_order'}->[0]";
+    }
+
+    do {
+	# try to find sibling
+	my $next_child = $self->get_next_child ($section);
+	return $next_child if (defined $next_child);
+
+	# move up one level
+	$section = $self->get_parent_section ($section);
+    } while $section =~ /\d/;
+
+    return undef;
+}
+
+sub is_leaf_section {
+    my $self = shift (@_);
+    my ($section) = @_;
+
+    my $section_ptr = $self->_lookup_section($section);
+    return 1 unless defined $section_ptr;
+
+    return (scalar (@{$section_ptr->{'subsection_order'}}) == 0);
+}
+
+# methods for dealing with sections
+
+# returns the name of the inserted section
+sub insert_section {
+    my $self = shift (@_);
+    my ($before_section) = @_;
+
+    # get the child to insert before and its parent section
+    my $parent_section = "";
+    my $before_child = "0";
+    my @before_section = split (/\./, $before_section);
+    if (scalar(@before_section) > 0) {
+	$before_child = pop (@before_section);
+	$parent_section = join (".", @before_section);
+    }
+
+    my $parent_section_ptr = $self->_lookup_section($parent_section);
+    if (!defined $parent_section_ptr) {
+	print STDERR "doc::insert_section couldn't find parent section " .
+	    "$parent_section\n";
+	return;
+    }
+
+    # get the next section number
+    my $section_num = $parent_section_ptr->{'next_subsection'}++;
+
+    my $i = 0;
+    while ($i < scalar(@{$parent_section_ptr->{'subsection_order'}}) &&
+	   $parent_section_ptr->{'subsection_order'}->[$i] ne $before_child) {
+	$i++;
+    }
+    
+    # insert the section number into the order list
+    splice (@{$parent_section_ptr->{'subsection_order'}}, $i, 0, $section_num);
+
+    # add this section to the parent section
+    my $section_ptr = {'subsection_order'=>[],
+		       'next_subsection'=>1,
+		       'subsections'=>{},
+		       'metadata'=>[],
+		       'text'=>""};
+    $parent_section_ptr->{'subsections'}->{$section_num} = $section_ptr;
+
+    # work out the full section number
+    my $section = $parent_section;
+    $section .= "." unless $section eq "";
+    $section .= $section_num;
+    
+    return $section;
+}
+
+# creates a pre-named section
+sub create_named_section {
+    my $self = shift (@_);
+    my ($mastersection) = @_;
+
+    my ($num);
+    my $section = $mastersection;
+    my $sectionref = $self;
+
+    while ($section ne "") {
+	($num, $section) = $section =~ /^\.?(\d+)(.*)$/;
+	$num =~ s/^0+(\d)/$1/; # remove leading 0s
+	$section = "" unless defined $section;
+	
+	if (defined $num) {
+	    if (!defined $sectionref->{'subsections'}->{$num}) {
+		push (@{$sectionref->{'subsection_order'}}, $num);
+		$sectionref->{'subsections'}->{$num} = {'subsection_order'=>[],
+							'next_subsection'=>1,
+							'subsections'=>{},
+							'metadata'=>[],
+							'text'=>""};
+		if ($num >= $sectionref->{'next_subsection'}) {
+		    $sectionref->{'next_subsection'} = $num + 1;
+		}
+	    }
+	    $sectionref = $sectionref->{'subsections'}->{$num};
+
+	} else {
+	    print STDERR "doc::create_named_section couldn't create section " .
+		"$mastersection\n";
+	    last;
+	}
+    }
+}
+
+# returns a reference to a list of subsections
+sub list_subsections {
+    my $self = shift (@_);
+    my ($section) = @_;
+
+    my $section_ptr = $self->_lookup_section ($section);
+    if (!defined $section_ptr) {
+	print STDERR "doc::list_subsections couldn't find section $section\n";
+	return [];
+    }
+
+    return [@{$section_ptr->{'subsection_order'}}];
+}
+
+sub delete_section {
+    my $self = shift (@_);
+    my ($section) = @_;
+
+#    my $section_ptr = {'subsection_order'=>[],
+#		       'next_subsection'=>1,
+#		       'subsections'=>{},
+#		       'metadata'=>[],
+#		       'text'=>""};
+
+    # if this is the top section reset everything
+    if ($section eq "") {
+	$self->{'subsection_order'} = [];
+	$self->{'subsections'} = {};
+	$self->{'metadata'} = [];
+	$self->{'text'} = "";
+	return;
+    }
+
+    # find the parent of the section to delete
+    my $parent_section = "";
+    my $child = "0";
+    my @section = split (".", $section);
+    if (scalar(@section) > 0) {
+	$child = pop (@section);
+	$parent_section = join (".", @section);
+    }
+
+    my $parent_section_ptr = $self->_lookup_section($parent_section);
+    if (!defined $parent_section_ptr) {
+	print STDERR "doc::delete_section couldn't find parent section " .
+	    "$parent_section\n";
+	return;
+    }
+
+    # remove this section from the subsection_order list
+    my $i = 0;
+    while ($i < scalar (@{$parent_section_ptr->{'subsection_order'}})) {
+	if ($parent_section_ptr->{'subsection_order'}->[$i] eq $child) {
+	    splice (@{$parent_section_ptr->{'subsection_order'}}, $i, 1);
+	    last;
+	}
+	$i++;
+    }
+
+    # remove this section from the subsection hash
+    if (defined ($parent_section_ptr->{'subsections'}->{$child})) {
+	undef $parent_section_ptr->{'subsections'}->{$child};
+    }
+}
+
+
+# methods for dealing with metadata
+
+# set_metadata_element and get_metadata_element are for metadata
+# which should only have one value. add_meta_data and get_metadata
+# are for metadata which can have more than one value.
+
+# set_metadata_element assumes the value is in (extended) ascii form.
+# For text which hash been already converted to the UTF-8 format use
+# set_utf8_metadata_element.
+sub set_metadata_element {
+    my $self = shift (@_);
+    my ($section, $field, $value) = @_;
+
+    $self->set_utf8_metadata_element ($section, $field, 
+				      &unicode::ascii2utf8($value));
+}
+
+# set_utf8_metadata_element assumes the text has already been
+# converted to the UTF-8 encoding.
+sub set_utf8_metadata_element {
+    my $self = shift (@_);
+    my ($section, $field, $value) = @_;
+
+    $self->delete_metadata ($section, $field);
+    $self->add_utf8_metadata ($section, $field, $value);
+}
+
+# returns the first metadata value which matches field
+sub get_metadata_element {
+    my $self = shift (@_);
+    my ($section, $field) = @_;
+    my ($data);
+
+    my $section_ptr = $self->_lookup_section($section);
+    if (!defined $section_ptr) {
+	print STDERR "doc::get_metadata_element couldn't find section " .
+	    "$section\n";
+	return;
+    }
+
+    foreach $data (@{$section_ptr->{'metadata'}}) {
+	return $data->[1] if (scalar(@$data) >= 2 && $data->[0] eq $field);
+    }
+	
+    return undef; # was not found
+}
+
+# add_metadata assumes the text is in (extended) ascii form. For
+# text which hash been already converted to the UTF-8 format use
+# add_utf8_metadata.
+sub add_metadata {
+    my $self = shift (@_);
+    my ($section, $field, $value) = @_;
+
+    $self->add_utf8_metadata ($section, $field,
+			      &unicode::ascii2utf8($value));
+}
+
+# add_utf8_metadata assumes the text has already been converted
+# to the UTF-8 encoding.
+sub add_utf8_metadata {
+    my $self = shift (@_);
+    my ($section, $field, $value) = @_;
+
+    my $section_ptr = $self->_lookup_section($section);
+    if (!defined $section_ptr) {
+	print STDERR "doc::add_utf8_metadata couldn't find section " .
+	    "$section\n";
+	return;
+    }
+    
+    push (@{$section_ptr->{'metadata'}}, [$field, $value]);
+}
+
+# returns a list of the form [value1, value2, ...]
+sub get_metadata {
+    my $self = shift (@_);
+    my ($section, $field) = @_;
+    my ($data);
+
+    my $section_ptr = $self->_lookup_section($section);
+    if (!defined $section_ptr) {
+	print STDERR "doc::get_metadata couldn't find section " .
+	    "$section\n";
+	return;
+    }
+
+    my @metadata = ();
+    foreach $data (@{$section_ptr->{'metadata'}}) {
+	push (@metadata, $data->[1]) if ($data->[0] eq $field);
+    }
+	
+    return \@metadata;
+}
+
+# returns a list of the form [[field,value],[field,value],...]
+sub get_all_metadata {
+    my $self = shift (@_);
+    my ($section) = @_;
+
+    my $section_ptr = $self->_lookup_section($section);
+    if (!defined $section_ptr) {
+	print STDERR "doc::get_all_metadata couldn't find section " .
+	    "$section\n";
+	return;
+    }
+    
+    return $section_ptr->{'metadata'};
+}
+
+# $value is optional
+sub delete_metadata {
+    my $self = shift (@_);
+    my ($section, $field, $value) = @_;
+
+    my $section_ptr = $self->_lookup_section($section);
+    if (!defined $section_ptr) {
+	print STDERR "doc::delete_metadata couldn't find section " .
+	    "$section\n";
+	return;
+    }
+
+    my $i = 0;
+    while ($i < scalar (@{$section_ptr->{'metadata'}})) {
+	if (($section_ptr->{'metadata'}->[$i]->[0] eq $field) &&
+	    (!defined $value || $section_ptr->{'metadata'}->[$i]->[1] eq $value)) {
+	    splice (@{$section_ptr->{'metadata'}}, $i, 1);
+	} else {
+	    $i++;
+	}
+    }
+}
+
+sub delete_all_metadata {
+    my $self = shift (@_);
+    my ($section) = @_;
+
+    my $section_ptr = $self->_lookup_section($section);
+    if (!defined $section_ptr) {
+	print STDERR "doc::delete_all_metadata couldn't find section " .
+	    "$section\n";
+	return;
+    }
+    
+    $section_ptr->{'metadata'} = [];
+}
+
+
+# methods for dealing with associated files
+
+sub associate_file {
+    my $self = shift (@_);
+    my ($real_filename, $assoc_filename, $mime_type) = @_;
+    $mime_type = &html::guess_mime_type ($real_filename) unless defined $mime_type;
+
+    # remove all associated files with the same name
+    $self->delete_assoc_file ($assoc_filename);
+
+    push (@{$self->{'associated_files'}}, 
+	  [$real_filename, $assoc_filename, $mime_type]);
+}
+
+# returns a list of associated files in the form
+#   [[real_filename, assoc_filename, mimetype], ...]
+sub get_assoc_files {
+    my $self = shift (@_);
+
+    return $self->{'associated_files'};
+}
+
+sub delete_assoc_file {
+    my $self = shift (@_);
+    my ($assoc_filename) = @_;
+
+    my $i=0;
+    while ($i < scalar (@{$self->{'associated_files'}})) {
+	if ($self->{'associated_files'}->[$i]->[1] eq $assoc_filename) {
+	    splice (@{$self->{'associated_files'}}, $i, 1);
+	} else {
+	    $i++;
+	}
+    }
+}
+
+
+# methods for dealing with text
+
+# add_text assumes the text is in (extended) ascii form. For
+# text which has been already converted to the UTF-8 format
+# use add_utf8_text.
+sub add_text {
+    my $self = shift (@_);
+    my ($section, $text) = @_;
+
+    # convert the text to UTF-8 encoded unicode characters
+    # and add the text
+    $self->add_utf8_text($section, &unicode::ascii2utf8($text));
+}
+
+
+# add_utf8_text assumes the text to be added has already
+# been converted to the UTF-8 encoding. For ascii text use
+# add_text
+sub add_utf8_text {
+    my $self = shift (@_);
+    my ($section, $text) = @_;
+
+    my $section_ptr = $self->_lookup_section($section);
+    if (!defined $section_ptr) {
+	print STDERR "doc::add_utf8_text couldn't find section " .
+	    "$section\n";
+	return;
+    }
+
+    $section_ptr->{'text'} .= $text;
+}
+
+# returns the text for a section
+sub get_text {
+    my $self = shift (@_);
+    my ($section) = @_;
+
+    my $section_ptr = $self->_lookup_section($section);
+    if (!defined $section_ptr) {
+	print STDERR "doc::get_text couldn't find section " .
+	    "$section\n";
+	return "";
+    }
+
+    return $section_ptr->{'text'};
+}
+
+# returns the (utf-8 encoded) length of the text for a section
+sub get_text_length {
+    my $self = shift (@_);
+    my ($section) = @_;
+
+    my $section_ptr = $self->_lookup_section($section);
+    if (!defined $section_ptr) {
+	print STDERR "doc::get_text_length couldn't find section " .
+	    "$section\n";
+	return 0;
+    }
+
+    return length ($section_ptr->{'text'});
+}
+
+sub delete_text {
+    my $self = shift (@_);
+    my ($section) = @_;
+
+    my $section_ptr = $self->_lookup_section($section);
+    if (!defined $section_ptr) {
+	print STDERR "doc::delete_text couldn't find section " .
+	    "$section\n";
+	return;
+    }
+
+    $section_ptr->{'text'} = "";
+}
+
+
+1;
Index: /tags/gsdl-21/gsdl/perllib/docproc.pm
===================================================================
--- /tags/gsdl-21/gsdl/perllib/docproc.pm	(revision 699)
+++ /tags/gsdl-21/gsdl/perllib/docproc.pm	(revision 699)
@@ -0,0 +1,46 @@
+###########################################################################
+#
+# docproc.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# document processors are used by the document reader plugins
+# to do some processing on some documents
+
+package docproc;
+
+sub new {
+    my ($class) = @_;
+    my $self = {};
+
+    return bless $self, $class;
+}
+
+sub process {
+    my $self = shift (@_);
+    my ($doc_obj) = @_;
+
+    die "docproc::process function must be implemented in sub classes\n";
+}
+
+1;
+
Index: /tags/gsdl-21/gsdl/perllib/docsave.pm
===================================================================
--- /tags/gsdl-21/gsdl/perllib/docsave.pm	(revision 699)
+++ /tags/gsdl-21/gsdl/perllib/docsave.pm	(revision 699)
@@ -0,0 +1,141 @@
+###########################################################################
+#
+# docsave.pm
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# This document processor saves a document in the 
+# archives directory of a collection
+
+
+package docsave;
+
+use arcinfo;
+use docproc;
+use util;
+
+
+sub BEGIN {
+    @ISA = ('docproc');
+}
+
+sub new {
+    my ($class, $collection, $archive_info, $verbosity, $gzip) = @_;
+    my $self = new docproc ();
+
+    $self->{'collection'} = $collection;
+    $self->{'archive_info'} = $archive_info;
+    $self->{'verbosity'} = $verbosity;
+    $self->{'gzip'} = $gzip;
+
+    # set a default for the archive directory
+    $self->{'archive_dir'} = "$ENV{'GSDLHOME'}/collect/$self->{'collection'}/archives";
+
+    return bless $self, $class;
+}
+
+sub setarchivedir {
+    my $self = shift (@_);
+    my ($archive_dir) = @_;
+
+    $self->{'archive_dir'} = $archive_dir;
+}
+
+sub process {
+    my $self = shift (@_);
+    my ($doc_obj) = @_;
+
+    my $archive_dir = $self->{'archive_dir'};
+    my $OID = $doc_obj->get_OID();
+    $OID = "NULL" unless defined $OID;
+
+    # get the document's directory.
+    my $doc_info = $self->{'archive_info'}->get_info($OID);
+    my $doc_dir = "";
+    if (defined $doc_info && scalar(@$doc_info) >= 1) {
+	# this OID already has an assigned directory, use the
+	# same one.
+	$doc_dir = $doc_info->[0];
+	$doc_dir =~ s/\/?doc\.gml(\.gz)?$//;
+
+    } else {
+	# have to get a new document directory
+	my $doc_dir_rest = $OID;
+	my $doc_dir_num = 0;
+	do {
+	    $doc_dir .= "/" if $doc_dir_num > 0;
+	    if ($doc_dir_rest =~ s/^(.{1,8})//) {
+		$doc_dir .= $1;
+		$doc_dir_num++;
+	    }
+	} while ($doc_dir_rest ne "" && 
+		 ((-d &util::filename_cat ($archive_dir, "$doc_dir.dir")) || 
+		  ($self->{'archive_info'}->size() >= 1024 && $doc_dir_num < 2)));
+	$doc_dir .= ".dir";
+    }
+
+    &util::mk_all_dir ("$archive_dir/$doc_dir");
+
+    # copy all the associated files, add this information as metadata
+    # to the document
+    my @assoc_files = ();
+    foreach $assoc_file (@{$doc_obj->get_assoc_files()}) {
+	if (-e $assoc_file->[0]) {
+	    my $afile = &util::filename_cat($archive_dir, $doc_dir, $assoc_file->[1]);
+	    &util::cp ($assoc_file->[0], $afile);
+	    $doc_obj->add_metadata ($doc_obj->get_top_section(), 
+				    "gsdlassocfile", 
+				    "$assoc_file->[1]:$assoc_file->[2]");
+	} else {
+	    print STDERR "docsave::process couldn't copy the associated file " .
+		"$assoc_file->[0] to $afile\n";
+	}
+    }
+
+    # save this document
+    my $doc_file = &util::filename_cat ($archive_dir, $doc_dir, "doc.gml");
+    my $short_doc_file = &util::filename_cat ($doc_dir, "doc.gml");
+
+    if (!open (OUTDOC, ">$doc_file")) {
+	print STDERR "docsave::process could not write to file $docfile\n";
+	return;
+    }
+    $doc_obj->output_section('docsave::OUTDOC', $doc_obj->get_top_section());
+    close OUTDOC;
+
+    if ($self->{'gzip'}) {
+	`gzip $doc_file`;
+	$doc_file .= ".gz";
+	$short_doc_file .= ".gz";
+	if (!-e $doc_file) {
+	    print STDERR "error while gzipping: $doc_file doesn't exist\n";
+	    return;
+	}
+    }
+
+    # store reference in the archive_info
+    $self->{'archive_info'}->add_info($OID, $short_doc_file);
+    
+}
+
+
+1;
Index: /tags/gsdl-21/gsdl/perllib/gb.pm
===================================================================
--- /tags/gsdl-21/gsdl/perllib/gb.pm	(revision 699)
+++ /tags/gsdl-21/gsdl/perllib/gb.pm	(revision 699)
@@ -0,0 +1,281 @@
+###########################################################################
+#
+# gb.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+#! /usr/local/bin/perl5 -w
+
+# useful methods for dealing with the GuoBiao (GB) encoding
+
+package gb;
+
+use util;
+
+# %encodings is of the form:
+#
+# encodings{encodingname-encodingname}->blocktranslation
+# blocktranslation->[[0-255],[256-511], ..., [65280-65535]]
+#
+# Any of the top translation blocks can point to an undefined
+# value. This data structure aims to allow fast translation and 
+# efficient storage.
+
+# @array256 is used for initialisation, there must be
+# a better way...
+@array256 = (0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0);
+
+
+%translations = ();
+
+
+# gb2unicode uses the GBK translation table
+sub gb2unicode {
+    generalgb2unicode ("GBK", @_);
+}
+
+# returns a pointer to unicode array
+sub generalgb2unicode {
+    my ($encoding, $intext) = @_;
+
+    if (!&loadmapencoding ("$encoding-unicode")) {
+	print STDERR "gb: ERROR - could not load encoding $encoding\n";
+	return [];
+    }
+    
+    my @outtext = ();
+    my $encodename = "$encoding-unicode";
+    my $len = length($intext);
+    my ($c1, $c2);
+    my $i = 0;
+
+    while ($i < $len) {
+	if (($c1 = ord(substr($intext, $i, 1))) >= 0x81) {
+	    if ($i+1 < $len) {
+		# found a GB character, put black square if cannot translate
+		$c2 = ord(substr($intext, $i+1, 1));
+		my $c = &transchar ($encodename, ($c1 << 8) | $c2);
+		$c = 0x25a1 if $c == 0;
+		push (@outtext, $c);
+		$i += 2;
+		
+	    } else {
+		# error
+		print STDERR "gb: ERROR missing second half of GB character\n";
+		$i++;
+	    }
+	    
+	} else {
+	    # normal ascii character
+	    push (@outtext, $c1) if ($c1 < 0x80);
+	    $i++;
+	}
+    }
+    
+    return \@outtext;
+}
+
+
+# unicode2gb uses the GBK translation table
+sub unicode2gb {
+    generalunicode2gb ("GBK", @_);
+}
+
+# returns a gb string. Note that this will not translate
+# traditional forms of characters to their simplified forms
+# Any traditional characters in the unicode will not be translated
+sub generalunicode2gb {
+    my ($encoding, $intext) = @_;
+
+    # load the encoding (if it is not already loaded)
+    if (!&loadmapencoding ("unicode-$encoding")) {
+	print STDERR "gb: ERROR - could not load encoding $encoding\n";
+	return "";
+    }
+    
+    # translate the string
+    my ($num, $gbc);
+    my $outtext = "";
+    my $encodename = "unicode-$encoding";
+    foreach $num (@$intext) {
+	if ($num < 0x80) {
+	    # normal ascii character
+	    $outtext .= chr ($num);
+	} else {
+	    # gb character
+	    my $gbc = &transchar ($encodename, $num);
+	    $gbc = 0xa1f5 if $gbc == 0;
+	    $outtext .= chr ($gbc >> 8);
+	    $outtext .= chr ($gbc & 0xff);
+	}
+    }
+
+    return $outtext;
+}
+
+
+# It returns 1 if successful, 0 if unsuccessful
+sub loadmapencoding {
+    my ($encoding) = @_;
+    
+    # check to see if the encoding has already been loaded
+    return 1 if (defined $translations{$encoding});
+
+    my $filename = $encoding;
+    $filename =~ s/\-//;
+    $filename =~ s/unicode/u/i;
+    $filename =~ tr/A-Z/a-z/;
+    $filename .= ".ump"; # unicode map file
+
+    return 0 unless open (MAPFILE, "$ENV{'GSDLHOME'}/unicode/$filename");
+
+    $translations{$encoding} = [@array256];
+    my $block = $translations{$encoding};
+
+    my ($in,$i,$j);
+    while (read(MAPFILE, $in, 1) == 1) {
+	$i = unpack ("C", $in);
+	$block->[$i] = [@array256];
+	for ($j=0; $j<256 && read(MAPFILE, $in, 2)==2; $j++) {
+	    my ($n1, $n2) = unpack ("CC", $in);
+	    $block->[$i]->[$j] = ($n1*256) + $n2;
+	}
+    }
+
+    close (MAPFILE);
+}
+
+
+
+# loadencoding expects the mapfile to contain (at least) two
+# tab-seperated fields. The first field is the mapped value
+# and the second field is the unicode value.
+#
+# It returns 1 if successful, 0 if unsuccessful
+sub loadencoding {
+    my ($encoding, $mapfile) = @_;
+    
+    my $to = "$encoding-unicode";
+    my $from = "unicode-$encoding";
+
+    # check to see if the encoding has already been loaded
+    if (defined $translations{$to} && defined $translations{$from}) {
+	return 1;
+    }
+
+    return 0 unless open (MAPFILE, $mapfile);
+
+    my ($line, @line);
+    $translations{$to} = [@array256];
+    $translations{$from} = [@array256];
+    while (defined ($line = <MAPFILE>)) {
+	# remove comments
+	$line =~ s/\#.*$//;
+	next unless $line =~ /\S/;
+
+	# split the line into fields and do a few
+	# simple sanity checks
+	@line = split (/\t/, $line);
+	next unless (scalar(@line) >= 2 &&
+		     $line[0] =~ /^0x/ &&
+		     $line[1] =~ /^0x/);
+
+	my $gbc = hex($line[0]);
+	my $unic = hex($line[1]);
+
+	# gbk has already had the necessary most significant
+	# bits set (it is used in the encoding
+	$gbc = $gbc | 0x8080 unless ($encoding =~ /gbk/i);
+
+	&addchartrans ($translations{$to}, $gbc, $unic);
+	&addchartrans ($translations{$from}, $unic, $gbc);
+    }
+
+    close (MAPFILE);
+
+    return 1;
+}
+
+sub showfilled {
+    my ($block) = @_;
+    my $count = 0;
+    foreach $el (@$block) {
+	if (ref ($el) eq "ARRAY") {
+	    print "1";
+	    $count++;
+	} else {
+	    print "0";
+	}
+    }
+    print "\n";
+    print "total: $count\n";
+}
+
+
+# addchartrans adds one character translation to a translation block.
+# It also simplifies the translation block if possible.
+sub addchartrans {
+    my ($block, $from, $to) = @_;
+    my $i = 0;
+
+    my $high = ($from / 256) % 256;
+    my $low = $from % 256;
+
+    if (ref ($block->[$high]) ne "ARRAY") {
+	$block->[$high] = [@array256];
+    }
+    $block->[$high]->[$low] = $to;
+}
+
+
+sub transchar {
+    my ($encoding, $from) = @_;
+    my $high = ($from / 256) % 256;
+    my $low = $from % 256;
+
+    return 0 unless defined $translations{$encoding};
+
+    my $block = $translations{$encoding};
+
+    if (ref ($block->[$high]) ne "ARRAY") {
+	return 0;
+    }
+    return $block->[$high]->[$low];
+}
+
+
+1;
Index: /tags/gsdl-21/gsdl/perllib/html.pm
===================================================================
--- /tags/gsdl-21/gsdl/perllib/html.pm	(revision 699)
+++ /tags/gsdl-21/gsdl/perllib/html.pm	(revision 699)
@@ -0,0 +1,273 @@
+###########################################################################
+#
+# html.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package html;
+
+# htmlsafe(TEXT)
+# 
+# Converts SGML meta characters in TEXT to entity references.
+# 
+sub htmlsafe
+{
+    $_[0] =~ s/&/&amp;/og;
+    $_[0] =~ s/</&lt;/og;
+    $_[0] =~ s/>/&gt;/og;
+}
+
+# urlsafe(TEXT)
+# 
+# Converts characters not allowed in a URL to their hex representation.
+# 
+sub urlsafe
+{
+    $_[0] =~ s/[\x09\x20\x22\x3c\x3e\x5b\x5c\x5d\x5e\x60\x7b\x7c\x7d\x7e\?\=\&\+_]/sprintf("%%%2x", ord($&))/ge;
+}
+
+# dmsafe
+#
+# Converts characters which could make display.pl or html crack to their entity references.
+# Don't use this on dm macros as they will no longer be recognisable by dm (which
+# by the way, is the whole idea).
+# -- Rodger 12/1/98
+sub dmsafe {
+    my ($s) = $_[0];
+    $s =~ s/&/&amp;/og; # for html
+    $s =~ s/</&lt;/og;  # for html
+    $s =~ s/>/&gt;/og;  # for html
+    $s =~ s/\_/&#095;/og; # for dm (we have a convention of starting macros with _
+    $s =~ s/\"/&quot;/og; # for html (don't want to be interpreted as a quote)
+    $s =~ s/\{/&#123;/og; # for dm blocks
+    $s =~ s/\}/&#125;/og; # for dm blocks
+    return $s;
+}
+
+
+# named entry to the standard html font
+%charnetosf = ("Agrave"=> "192",  "Aacute"=> "193",  "Acirc" => "194",  "Atilde"=> "195",
+	       "Auml"  => "196",  "Aring" => "197",  "AElig" => "198",  "Ccedil"=> "199",
+	       "Egrave"=> "200",  "Eacute"=> "201",  "Ecirc" => "202",  "Euml"  => "203",
+	       "Igrave"=> "204",  "Iacute"=> "205",  "Icirc" => "206",  "Iuml"  => "207",
+	       "ETH"   => "208",  "Ntilde"=> "209",  "Ograve"=> "210",  "Oacute"=> "211",
+	       "Ocirc" => "212",  "Otilde"=> "213",  "Ouml"  => "214",  
+	       "Oslash"=> "216",  "Ugrave"=> "217",  "Uacute"=> "218",  "Ucirc" => "219",
+	       "Uuml"  => "220",  "Yacute"=> "221",  "THORN" => "222",  "szlig" => "223",
+	       "agrave"=> "224",  "aacute"=> "225",  "acirc" => "226",  "atilde"=> "227",
+	       "auml"  => "228",  "aring" => "229",  "aelig" => "230",  "ccedil"=> "231",
+	       "egrave"=> "232",  "eacute"=> "233",  "ecirc" => "234",  "euml"  => "235",
+	       "igrave"=> "236",  "iacute"=> "237",  "icirc" => "238",  "iuml"  => "239",
+	       "eth"   => "240",  "ntidle"=> "241",  "ograve"=> "242",  "oacute"=> "243",
+	       "ocirc" => "244",  "otilde"=> "245",  "ouml"  => "246",  
+	       "oslash"=> "248",  "ugrave"=> "249",  "uacute"=> "250",  "ucirc" => "251",
+	       "uuml"  => "252",  "yacute"=> "253",  "thorn" => "254",  "yuml"  => "255");
+
+%symnetosf = ("quot"  => "34",   "amp"   => "38",   "lt"    => "60",   "gt"    => "62",
+	      "nbsp"  => "160",  "iexcl" => "161",  "cent"  => "162",  "pound" => "163",
+	      "curren"=> "164",  "yen"   => "165",  "brvbar"=> "166",  "sect"  => "167",
+	      "uml"   => "168",  "copy"  => "169",  "ordf"  => "170",  "laquo" => "171",
+	      "not"   => "172",  "shy"   => "173",  "reg"   => "174",  "macr"  => "175",
+	      "deg"   => "176",  "plusmn"=> "177",  "sup2"  => "178",  "sup3"  => "179",
+	      "acute" => "180",  "micro" => "181",  "para"  => "182",  "middot"=> "183",
+	      "cedil" => "184",  "sup1"  => "185",  "ordm"  => "186",  "raquo" => "187",
+	      "frac14"=> "188",  "frac12"=> "189",  "frac34"=> "190",  "iquest"=> "191",
+	      "times" => "215",  "divide"=> "247");
+
+
+
+# standard font to plain text
+%sftotxt = ("32"  => " ",  "33"  => "!",  "34"  => "\"",  "35"  => "\#",  "36"  => "\$",
+	    "37"  => "\%", "38"  => "&",  "39"  => "'",   "40"  => "(",   "41"  => ")",
+	    "42"  => "*",  "43"  => "+",  "44"  => ",",   "45"  => "-",   "46"  => ".",
+	    "47"  => "/",  "48"  => "0",  "49"  => "1",   "50"  => "2",   "51"  => "3",
+	    "52"  => "4",  "53"  => "5",  "54"  => "6",   "55"  => "7",   "56"  => "8",
+	    "57"  => "9",  "58"  => ":",  "59"  => ";",   "60"  => "<",   "61"  => "=",
+	    "62"  => ">",  "63"  => "?",  "64"  => "\@",  "65"  => "A",   "66"  => "B",
+	    "57"  => "9",  "58"  => ":",  "59"  => ";",   "61"  => "=",
+	    "63"  => "?",  "64"  => "\@", "65"  => "A",  "66"  => "B",
+	    "67"  => "C",  "68"  => "D",  "69"  => "E",   "70"  => "F",   "71"  => "G",
+	    "72"  => "H",  "73"  => "I",  "74"  => "J",   "75"  => "K",   "76"  => "L",
+	    "77"  => "M",  "78"  => "N",  "79"  => "O",   "80"  => "P",   "81"  => "Q",
+	    "82"  => "R",  "83"  => "S",  "84"  => "T",   "85"  => "U",   "86"  => "V",
+	    "87"  => "W",  "88"  => "X",  "89"  => "Y",   "90"  => "Z",   "91"  => "[",
+	    "92"  => "\\", "93"  => "]",  "94"  => "^",   "95"  => "_",   "96"  => "`",
+	    "97"  => "a",  "98"  => "b",  "99"  => "c",   "100" => "d",   "101" => "e",
+	    "102" => "f",  "103" => "g",  "104" => "h",   "105" => "i",   "106" => "j",
+	    "107" => "k",  "108" => "l",  "109" => "m",   "110" => "n",   "111" => "o",
+	    "112" => "p",  "113" => "q",  "114" => "r",   "115" => "s",   "116" => "t",
+	    "117" => "u",  "118" => "v",  "119" => "w",   "120" => "x",   "121" => "y",
+	    "122" => "z",  "123" => "{",  "124" => "|",   "125" => "}",   "126" => "~",
+	    "130" => ",",  "131" => "f",  "132" => "\"",   "133" => "...", "139" => "<",
+	    "140" => "OE", "145" => "'",  "146" => "'",   "147" => "\"",   "148" => "\"",
+	    "149" => "o",  "150" => "--", "151" => "-",   "152" => "~",   "153" => "TM",
+	    "155" => ">",  "156" => "oe", "159" => "Y",   "160" => " ",   "178" => "2",
+	    "179" => "3",  "185" => "1",  "188" => "1/4", "189" => "1/2", "190" => "3/4",
+	    "192" => "A",  "193" => "A",  "194" => "A",   "195" => "A",   "196" => "A",
+	    "197" => "A",  "198" => "AE", "199" => "C",   "200" => "E",   "201" => "E",
+	    "202" => "E",  "203" => "E",  "204" => "I",   "205" => "I",   "206" => "I",
+	    "207" => "I",  "208" => "D",  "209" => "N",   "210" => "O",   "211" => "O",
+	    "212" => "O",  "213" => "O",  "214" => "O",   "215" => "*",   "216" => "O",
+	    "217" => "U",  "218" => "U",  "219" => "U",   "220" => "U",   "221" => "Y",
+	    "223" => "ss", "224" => "a",  "225" => "a",   "226" => "a",   "227" => "a",
+	    "228" => "a",  "229" => "a",  "230" => "ae",  "231" => "c",   "232" => "e",
+	    "233" => "e",  "234" => "e",  "235" => "e",   "236" => "i",   "237" => "i",
+	    "238" => "i",  "239" => "i",  "241" => "n",   "242" => "o",   "243" => "o",
+	    "244" => "o",  "245" => "o",  "246" => "o",   "247" => "/",   "248" => "o",
+	    "249" => "u",  "250" => "u",  "251" => "u",   "252" => "u",   "253" => "y",
+	    "255" => "y",  "8218" => ",");
+
+
+# returns the character in the standard html font. It assumes that the
+# & and ; have been stripped off the string.
+sub getcharequiv {
+    my ($entity, $convertsymbols) = @_;
+
+    # replace &#8218 with comma (",")
+    $entity =~ s/8218/044/;
+
+    # a numeric entity
+    if ($entity =~ /^\#0*(\d+)/) {
+	return pack("c", $1);
+    }
+    
+    # a named character entity
+    if (defined $charnetosf{$entity}) {
+	return pack("c", $charnetosf{$entity});
+    }
+
+    # a named symbol entity
+    if ($convertsymbols && defined $symnetosf{$entity}) {
+	return pack("c", $symnetosf{$entity});
+    }
+
+    return "&$entity;"; # unknown character
+}
+
+# convert character entities from named equivalents to html font
+sub convertcharentities {
+    # args: the text that you want to convert
+
+    $_[0] =~ s/&([^;]+);/&getcharequiv($1,0)/ge;
+}
+
+# convert any entities from named equivalents to html font
+sub convertallentities {
+    # args: the text that you want to convert
+
+    $_[0] =~ s/&([^;]+);/&getcharequiv($1,1)/ge;
+}
+
+sub html2txt {
+    # args: the text that you want converted to ascii, 
+    # and whether to strip out sgml tags
+
+    # strip out sgml tags if needed
+    $_[0] =~ s/<[^>]>//g if $_[1];
+
+    # convert the char entities to the standard html font
+    &convertcharentities($_[0]); 
+    
+    # convert the html character set to a plain ascii character set
+    my $pos = 0;
+    while ($pos < length($_[0])) {
+	my $charnum = ord(substr($_[0], $pos, 1));
+	if ($charnum >= 32) { # only convert characters above #32
+	    my $replacechars = " ";
+	    $replacechars = $sftotxt{$charnum} if defined $sftotxt{$charnum};
+	    substr($_[0], $pos, 1) = $replacechars;
+	    $pos += length ($replacechars);
+
+	} else {
+	    $pos ++;
+	}
+    }
+}  
+
+
+sub guess_mime_type {
+    my ($filename) = @_;
+
+    my ($fileext) = $filename =~ /\.(\w+)$/;
+    return "unknown" unless defined $fileext;
+
+    my %mime_type = ("ai"=>"application/postscript", "aif"=>"audio/x-aiff", 
+		     "aifc"=>"audio/x-aiff", "aiff"=>"audio/x-aiff", 
+		     "au"=>"audio/basic", "avi"=>"video/x-msvideo",
+		     "bcpio"=>"application/x-bcpio", "bin"=>"application/octet-stream", 
+		     "cdf"=>"application/x-netcdf", "class"=>"application/octet-stream", 
+		     "cpio"=>"application/x-cpio", "cpt"=>"application/mac-compactpro",
+		     "csh"=>"application/x-csh", "dcr"=>"application/x-director", 
+		     "dir"=>"application/x-director", "dms"=>"application/octet-stream", 
+		     "doc"=>"application/msword", "dvi"=>"application/x-dvi",
+		     "dxr"=>"application/x-director", "eps"=>"application/postscript", 
+		     "etx"=>"text/x-setext",
+		     "exe"=>"application/octet-stream", "gif"=>"image/gif",
+		     "gtar"=>"application/x-gtar", "hdf"=>"application/x-hdf",
+		     "hqx"=>"application/mac-binhex40", "htm"=>"text/html",
+		     "html"=>"text/html", "ice"=>"x-conference/x-cooltalk",
+		     "ief"=>"image/ief", "jpe"=>"image/jpeg",
+		     "jpeg"=>"image/jpeg", "jpg"=>"image/jpeg",
+		     "kar"=>"audio/midi", "latex"=>"application/x-latex",
+		     "lha"=>"application/octet-stream", "lzh"=>"application/octet-stream",
+		     "man"=>"application/x-troff-man", "mcf"=>"image/vasa",
+		     "me"=>"application/x-troff-me", "mid"=>"audio/midi",
+		     "midi"=>"audio/midi", "mif"=>"application/x-mif",
+		     "mov"=>"video/quicktime", "movie"=>"video/x-sgi-movie",
+		     "mp2"=>"audio/mpeg", "mpe"=>"video/mpeg",
+		     "mpeg"=>"video/mpeg", "mpg"=>"video/mpeg",
+		     "mpga"=>"audio/mpeg", "ms"=>"application/x-troff-ms",
+		     "nc"=>"application/x-netcdf", "oda"=>"application/oda",
+		     "pbm"=>"image/x-portable-bitmap", "pdb"=>"chemical/x-pdb",
+		     "pdf"=>"application/pdf", "pgm"=>"image/x-portable-graymap",
+		     "png"=>"image/png", "pnm"=>"image/x-portable-anymap",
+		     "ppm"=>"image/x-portable-pixmap", "ppt"=>"application/powerpoint",
+		     "ps"=>"application/postscript", "qt"=>"video/quicktime",
+		     "ra"=>"audio/x-realaudio", "ram"=>"audio/x-pn-realaudio",
+		     "ras"=>"image/x-cmu-raster", "rgb"=>"image/x-rgb",
+		     "roff"=>"application/x-troff", "rpm"=>"audio/x-pn-realaudio-plugin",
+		     "rtf"=>"application/rtf", "rtx"=>"text/richtext",
+		     "sgm"=>"text/x-sgml", "sgml"=>"text/x-sgml",
+		     "sh"=>"application/x-sh", "shar"=>"application/x-shar",
+		     "sit"=>"application/x-stuffit", "skd"=>"application/x-koan",
+		     "skm"=>"application/x-koan", "skp"=>"application/x-koan",
+		     "skt"=>"application/x-koan", "snd"=>"audio/basic",
+		     "src"=>"application/x-wais-source", "sv4cpio"=>"application/x-sv4cpio",
+		     "sv4crc"=>"application/x-sv4crc", "t"=>"application/x-troff",
+		     "tar"=>"application/x-tar", "tcl"=>"application/x-tcl",
+		     "tex"=>"application/x-tex", "texi"=>"application/x-texinfo",
+		     "texinfo"=>"application/x-texinfo", "tif"=>"image/tiff",
+		     "tiff"=>"image/tiff", "tr"=>"application/x-troff",
+		     "tsv"=>"text/tab-separated-values", "txt"=>"text/plain",
+		     "ustar"=>"application/x-ustar", "vcd"=>"application/x-cdlink",
+		     "vrml"=>"x-world/x-vrml", "wav"=>"audio/x-wav",
+		     "wrl"=>"x-world/x-vrml", "xbm"=>"image/x-xbitmap",
+		     "xpm"=>"image/x-xpixmap", "xwd"=>"image/x-xwindowdump",
+		     "xyz"=>"chemical/x-pdb", "zip"=>"application/zip");
+
+    return $mime_type{$fileext} if (defined $mime_type{$fileext});
+
+    return "unknown";
+}
+
+
+1;
Index: /tags/gsdl-21/gsdl/perllib/lang.pm
===================================================================
--- /tags/gsdl-21/gsdl/perllib/lang.pm	(revision 699)
+++ /tags/gsdl-21/gsdl/perllib/lang.pm	(revision 699)
@@ -0,0 +1,201 @@
+###########################################################################
+#
+# lang.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# this module translates language descriptions to and from
+# the ISO 639 standard
+
+package lang;
+
+@trans = (["ab", "Abkhazian"],      ["om", "Afan Oromo"],
+	  ["om", "Oromo"],          ["aa", "Afar"],
+	  ["af", "Afrikaans"],      ["sq", "Albanian"],
+	  ["am", "Amharic"],        ["ar", "Arabic"],
+	  ["hy", "Armenian"],       ["as", "Assamese"],
+	  ["ay", "Aymara"],         ["az", "Azerbaijani"],
+
+	  ["ba", "Bashkir"],        ["eu", "Basque"],
+	  ["bn", "Bengali"],        ["bn", "Bangla"],
+	  ["dz", "Bhutani"],        ["bh", "Bihari"],
+	  ["bi", "Bislama"],        ["br", "Breton"],
+	  ["bg", "Bulgarian"],      ["my", "Burmese"],
+	  ["be", "Byelorussian"],
+
+	  ["km", "Cambodian"],      ["ca", "Catalan"],
+	  ["zh", "Chinese"],        ["co", "Corsican"],
+	  ["hr", "Croatian"],       ["cs", "Czech"],
+
+	  ["da", "Danish"],         ["nl", "Dutch"],
+
+	  ["en", "English"],        ["eo", "Esperanto"],
+	  ["et", "Estonian"],       ["fo", "Faroese"],
+	  ["fj", "Fijian"],         ["fi", "Finnish"],
+	  ["fr", "French"],         ["fy", "Frisian"],
+
+	  ["gl", "Galician"],       ["ka", "Georgian"],
+	  ["de", "German"],         ["el", "Greek"],
+	  ["kl", "Greenlandic"],    ["gn", "Guarani"],
+	  ["gu", "Gujarati"],
+
+	  ["ha", "Hausa"],          ["iw", "Hebrew"],
+	  ["hi", "Hindi"],          ["hu", "Hungarian"],
+
+	  ["is", "Icelandic"],      ["in", "Indonesian"],
+	  ["ia", "Interlingua"],    ["ie", "Interlingue"],
+	  ["ik", "Inupiak"],        ["ga", "Irish"],
+	  ["it", "Italian"],
+
+	  ["ja", "Japanese"],       ["jv", "Javanese"],
+
+	  ["kn", "Kannada"],        ["ks", "Kashmiri"],
+	  ["kk", "Kazakh"],         ["rw", "Kinyarwanda"],
+	  ["ky", "Kirghiz"],        ["rn", "Kirundi"],
+	  ["ko", "Korean"],         ["ku", "Kurdish"],
+
+	  ["lo", "Laothian"],       ["la", "Latin"],
+	  ["lv", "Latvian"],        ["lv", "Lettish"],
+	  ["ln", "Lingala"],        ["lt", "Lithuanian"],
+
+	  ["mk", "Macedonian"],     ["mg", "Malagasy"],
+	  ["ms", "Malay"],          ["ml", "Malayalam"],
+	  ["mt", "Maltese"],        ["mi", "Maori"],
+	  ["mr", "Marathi"],        ["mo", "Moldavian"],
+	  ["mn", "Mongolian"],
+
+	  ["na", "Nauru"],          ["ne", "Nepali"],
+	  ["no", "Norwegian"],
+
+	  ["oc", "Occitan"],        ["or", "Oriya"],
+
+	  ["ps", "Pashto"],         ["ps", "Pushto"],
+	  ["fa", "Persian"],        ["pl", "Polish"],
+	  ["pt", "Portuguese"],     ["pa", "Punjabi"],
+
+	  ["qu", "Quechua"],
+
+	  ["rm", "Rhaeto-Romance"], ["ro", "Romanian"],
+	  ["ru", "Russian"],
+
+	  ["sm", "Samoan"],         ["sg", "Sangho"],
+	  ["sa", "Sanskrit"],       ["gd", "Scots Gaelic"],
+	  ["sr", "Serbian"],        ["sh", "Serbo-Croatian"],
+	  ["st", "Sesotho"],        ["tn", "Setswana"],
+	  ["sn", "Shona"],          ["sd", "Sindhi"],
+	  ["si", "Singhalese"],     ["ss", "Siswati"],
+	  ["sk", "Slovak"],         ["sl", "Slovenian"],
+	  ["so", "Somali"],         ["es", "Spanish"],
+	  ["su", "Sundanese"],      ["sw", "Swahili"],
+	  ["sv", "Swedish"],
+
+	  ["tl", "Tagalog"],        ["tg", "Tajik"],
+	  ["ta", "Tamil"],          ["tt", "Tatar"],
+	  ["te", "Telugu"],         ["th", "Thai"],
+	  ["bo", "Tibetan"],        ["ti", "Tigrinya"],
+	  ["to", "Tonga"],          ["ts", "Tsonga"],
+	  ["tr", "Turkish"],        ["tk", "Turkmen"],
+	  ["tw", "Twi"],
+
+	  ["uk", "Ukrainian"],      ["ur", "Urdu"],
+	  ["uz", "Uzbek"],
+
+	  ["vi", "Vietnamese"],     ["vo", "Volapuk"],
+
+	  ["cy", "Welsh"],          ["wo", "Wolof"],
+
+	  ["xh", "Xhosa"],
+
+	  ["ji", "Yiddish"],        ["yo", "Yoruba"],
+
+	  ["zu", "Zulu"]);
+
+
+sub one_english_to_iso639 {
+    my ($english) = @_;
+    my $iso639 = "";
+
+    foreach $lang (@trans) {
+	my $code = $lang->[0];
+	my $enname = $lang->[1];
+	if ($english =~ /\s*($code|$enname)\s*$/i) {
+	    # found the language
+	    $iso639 = $code;
+	    last;
+
+	}
+    }
+
+    return $iso639;
+}
+
+
+sub english_to_iso639 {
+    my ($englishlist) = @_;
+    my ($english);
+
+    my @englishlist = split (",", $englishlist);
+    my @iso639list = ();
+
+    foreach $english (@englishlist) {
+	push (@iso639list, &one_english_to_iso639($english));
+    }
+
+    return join (",", @iso639list);
+}
+
+
+sub one_iso639_to_english {
+    my ($iso639) = @_;
+    my $english = "";
+
+    foreach $lang (@trans) {
+	my $code = $lang->[0];
+	my $enname = $lang->[1];
+	if ($iso639 =~ /\s*($code|$enname)\s*$/i) {
+	    # found the language
+	    $english = $enname;
+	    last;
+
+	}
+    }
+
+    return $enname;
+}
+
+
+sub iso639_to_english {
+    my ($iso639list) = @_;
+    my ($iso639);
+
+    my @iso639list = split (",", $iso639list);
+    my @englishlist = ();
+
+    foreach $iso639 (@iso639list) {
+	push (@englishlist, &one_iso639_to_english($iso639));
+    }
+
+    return join (",", @englishlist);
+}
+
+
+1;
Index: /tags/gsdl-21/gsdl/perllib/mgbuilder.pm
===================================================================
--- /tags/gsdl-21/gsdl/perllib/mgbuilder.pm	(revision 699)
+++ /tags/gsdl-21/gsdl/perllib/mgbuilder.pm	(revision 699)
@@ -0,0 +1,641 @@
+###########################################################################
+#
+# mgbuilder.pm -- MGBuilder object
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package mgbuilder;
+
+use classify;
+use cfgread;
+use colcfg;
+use plugin;
+use util;
+
+$maxdocsize = 12000;
+
+%wanted_index_files = ('td'=>1,
+		       't'=>1,
+		       'idb'=>1,
+		       'ib1'=>1,
+		       'ib2'=>1,
+		       'ib3'=>1,
+		       'i'=>1,
+		       'ip'=>1,
+		       'tiw'=>1,
+		       'wa'=>1);
+
+
+sub new {
+    my ($class, $collection, $source_dir, $build_dir, 
+	$verbosity, $maxdocs, $allclassifications) = @_;
+
+    # create an mgbuilder object
+    my $self = bless {'collection'=>$collection,
+		      'source_dir'=>$source_dir,
+		      'build_dir'=>$build_dir,
+		      'verbosity'=>$verbosity,
+		      'maxdocs'=>$maxdocs,
+		      'allclassifications'=>$allclassifications,
+		      'notbuilt'=>[]    # indexes not built
+		      }, $class;
+
+
+    # read in the collection configuration file
+    my $colcfgname = "$ENV{'GSDLCOLLECTDIR'}/etc/collect.cfg";
+    if (!-e $colcfgname) {
+	die "mgbuilder::new - couldn't find collect.cfg for collection $collection\n";
+    }
+    $self->{'collect_cfg'} = &colcfg::read_collect_cfg ($colcfgname);
+
+    # sort out subcollection indexes
+    if (defined $self->{'collect_cfg'}->{'indexsubcollections'}) {
+	my $indexes = $self->{'collect_cfg'}->{'indexes'};
+	$self->{'collect_cfg'}->{'indexes'} = [];
+	foreach $subcollection (@{$self->{'collect_cfg'}->{'indexsubcollections'}}) {
+	    foreach $index (@$indexes) {
+		push (@{$self->{'collect_cfg'}->{'indexes'}}, "$index:$subcollection");
+	    }
+	}
+    }
+
+    # sort out language subindexes
+    if (defined $self->{'collect_cfg'}->{'languages'}) {
+	my $indexes = $self->{'collect_cfg'}->{'indexes'};
+	$self->{'collect_cfg'}->{'indexes'} = [];
+	foreach $language (@{$self->{'collect_cfg'}->{'languages'}}) {
+	    foreach $index (@$indexes) {
+		push (@{$self->{'collect_cfg'}->{'indexes'}}, "$index:$language");
+	    }
+	}
+    }
+
+    # get the list of plugins for this collection
+    my @plugins = (); # some good choice of plugins .... ????
+    if (defined $self->{'collect_cfg'}->{'plugins'}) {
+	@plugins = @{$self->{'collect_cfg'}->{'plugins'}};
+    }
+    
+
+    # load all the plugins
+    $self->{'pluginfo'} = &plugin::load_plugins ($collection, \@plugins);
+    if (scalar(@{$self->{'pluginfo'}}) == 0) {
+	print STDERR "No plugins were loaded.\n";
+	die "\n";
+    }
+
+    # load all the classifiers
+    $self->{'classifiers'} = [];
+    if (open (COLCFG, $colcfgname)) {
+	while (defined ($line = &cfgread::read_cfg_line('mgbuilder::COLCFG'))) {
+	    if (scalar(@$line) >= 2) {
+		my $key = shift (@$line);
+		if ($key eq "classify") {
+		    my $classinfo = &classify::load_classifier($line);
+		    push (@{$self->{'classifiers'}}, $classinfo) 
+			if defined $classinfo;
+		}
+	    }
+	}
+	close (COLCFG);
+    } else {
+	print STDERR "mgbuilder::new couldn't read the cfg file $colcfgname\n";
+	print STDERR "               no classifiers were loaded\n";
+    }
+
+    # load up the document processor for building
+    # if a buildproc class has been created for this collection, use it
+    # otherwise, use the mg buildproc
+    my ($buildprocdir, $buildproctype);
+    if (-e "$ENV{'GSDLCOLLECTDIR'}/perllib/${collection}buildproc.pm") {
+	$buildprocdir = "$ENV{'GSDLCOLLECTDIR'}/perllib";
+	$buildproctype = "${collection}buildproc";
+    } else {
+	$buildprocdir = "$ENV{'GSDLHOME'}/perllib";
+	$buildproctype = "mgbuildproc";
+    }
+    require "$buildprocdir/$buildproctype.pm";
+
+    eval("\$self->{'buildproc'} = new $buildproctype(\$collection, " .
+	 "\$source_dir, \$build_dir, \$verbosity)");
+    die "$@" if $@;
+
+
+    return $self;
+}
+
+sub init {
+    my $self = shift (@_);
+
+    # remove any old builds
+    &util::rm_r($self->{'build_dir'});
+    &util::mk_all_dir($self->{'build_dir'});
+        
+    # make the text directory
+    my $textdir = "$self->{'build_dir'}/text";
+    &util::mk_all_dir($textdir);
+}
+
+sub compress_text {
+    my $self = shift (@_);
+    my ($textindex) = @_;
+    my $exedir = "$ENV{'GSDLHOME'}/bin/$ENV{'GSDLOS'}";
+    my $exe = &util::get_os_exe ();
+    my $mg_passes_exe = &util::filename_cat($exedir, "mg_passes$exe");
+    my $mg_compression_dict_exe = &util::filename_cat($exedir, "mg_compression_dict$exe");
+
+    &util::mk_all_dir (&util::filename_cat($self->{'build_dir'}, "text"));
+    my $basefilename = "text/$self->{'collection'}";
+    my $fulltextprefix = &util::filename_cat ($self->{'build_dir'}, $basefilename);
+
+    my $osextra = "";
+    if ($ENV{'GSDLOS'} =~ /^windows$/i) {
+	$fulltextprefix =~ s/\//\\/g;
+    } else {
+	$osextra = " -d /";
+    }
+
+    print STDERR "\n*** creating the compressed text\n" if ($self->{'verbosity'} >= 1);
+
+    # set up the document processor
+    $self->{'buildproc'}->set_output_handle ('mgbuilder::PIPEOUT');
+    $self->{'buildproc'}->set_mode ('text');
+    $self->{'buildproc'}->set_index ($textindex);
+    $self->{'buildproc'}->set_indexing_text (0);
+    
+    # collect the statistics for the text
+    # -b $maxdocsize sets the maximum document size to be 12 meg
+    print STDERR "\n    collecting text statistics\n"  if ($self->{'verbosity'} >= 1);
+    if (!-e "$mg_passes_exe" || !open (PIPEOUT, 
+            "| $mg_passes_exe -f $fulltextprefix -b $maxdocsize -T1 $osextra")) {
+	die "mgbuilder::compress_text - couldn't run $mg_passes_exe\n";
+    }
+    $self->{'buildproc'}->reset();
+    &plugin::read ($self->{'pluginfo'}, $self->{'source_dir'}, 
+		   "", {}, $self->{'buildproc'}, $self->{'maxdocs'});
+    close (PIPEOUT);
+
+    # create the compression dictionary
+    # the compression dictionary is built by assuming the stats are from a seed
+    # dictionary (-S), if a novel word is encountered it is spelled out (-H),
+    # and the resulting dictionary must be less than 5 meg with the most frequent
+    # words being put into the dictionary first (-2 -k 5120)
+    print STDERR "\n    creating the compression dictionary\n"  if ($self->{'verbosity'} >= 1);
+    if (!-e "$mg_compression_dict_exe") {
+	die "mgbuilder::compress_text - couldn't run $mg_compression_dict_exe\n";
+    }
+    system ("$mg_compression_dict_exe -f $fulltextprefix -S -H -2 -k 5120 $osextra");
+
+    # compress the text
+    # -b $maxdocsize sets the maximum document size to be 12 meg
+    print STDERR "\n    compressing the text\n"  if ($self->{'verbosity'} >= 1);
+    if (!-e "$mg_passes_exe" || !open (PIPEOUT, 
+            "| $mg_passes_exe -f $fulltextprefix -b $maxdocsize -T2 $osextra")) {
+	die "mgbuilder::compress_text - couldn't run $mg_passes_exe\n";
+    }
+    $self->{'buildproc'}->reset();
+    &plugin::read ($self->{'pluginfo'}, $self->{'source_dir'}, 
+		   "", {}, $self->{'buildproc'}, $self->{'maxdocs'});
+    close (PIPEOUT);
+}
+
+sub want_built {
+    my $self = shift (@_);
+    my ($index) = @_;
+
+    if (defined ($self->{'collect_cfg'}->{'dontbuild'})) {
+	foreach $checkstr (@{$self->{'collect_cfg'}->{'dontbuild'}}) {
+	    if ($index =~ /^$checkstr$/) {
+		push (@{$self->{'notbuilt'}}, $self->{'index_mapping'}->{$index});
+		return 0;
+	    }
+	}
+    }
+
+    return 1;
+}
+
+sub build_indexes {
+    my $self = shift (@_);
+    my $indexes = $self->{'collect_cfg'}->{'indexes'};
+
+    # create the mapping between the index descriptions 
+    # and their directory names
+    $self->{'index_mapping'} = $self->create_index_mapping ($indexes);
+
+    # build each of the indexes
+    foreach $index (@$indexes) {
+	if ($self->want_built($index)) {
+	    print STDERR "\n*** building index $index in subdirectory " . 
+		"$self->{'index_mapping'}->{$index}\n" if ($self->{'verbosity'} >= 1);
+	    $self->build_index($index);
+	} else {
+	    print STDERR "\n*** ignoring index $index\n" if ($self->{'verbosity'} >= 1);
+	}
+    }
+}
+
+# creates directory names for each of the index descriptions
+sub create_index_mapping {
+    my $self = shift (@_);
+    my ($indexes) = @_;
+
+    my %mapping = ();
+    $mapping{'indexmaporder'} = [];
+    $mapping{'subcollectionmaporder'} = [];
+    $mapping{'languagemaporder'} = [];
+    
+    # dirnames is used to check for collisions. Start this off
+    # with the manditory directory names
+    my %dirnames = ('text'=>'text',
+		    'extra'=>'extra');
+    my %pnames = ('index' => '', 'subcollection' => '', 'languages' => '');
+
+    foreach $index (@$indexes) {
+	my ($level, $gran, $subcollection, $languages) = split (":", $index);
+
+	# the directory name starts with the first character of the index level
+	my ($pindex) = $level =~ /^(.)/;
+
+	# next comes a processed version of the index
+	$pindex .= $self->process_field ($gran); 
+	$pindex = lc ($pindex);
+
+	# next comes a processed version of the subcollection if there is one.
+	my $psub = $self->process_field ($subcollection);
+	$psub = lc ($psub);
+
+	# next comes a processed version of the language if there is one.
+	my $plang = $self->process_field ($languages);
+	$plang = lc ($plang);
+
+	my $dirname = $pindex . $psub . $plang;
+
+	# check to be sure all index names are unique
+	while (defined ($dirnames{$dirname})) {
+	    $dirname = $self->make_unique (\%pnames, $index, \$pindex, \$psub, \$plang);
+	}
+
+	# store the mapping orders as well as the maps
+	if (!defined $mapping{'indexmap'}{"$level:$gran"}) {
+	    $mapping{'indexmap'}{"$level:$gran"} = $pindex;
+	    push (@{$mapping{'indexmaporder'}}, "$level:$gran");
+	}
+	if ($psub =~ /\w/ && !defined ($mapping{'subcollectionmap'}{$subcollection})) {
+	    $mapping{'subcollectionmap'}{$subcollection} = $psub;
+	    push (@{$mapping{'subcollectionmaporder'}}, $subcollection);
+	}
+	if ($plang =~ /\w/ && !defined ($mapping{'languagemap'}{$languages})) {
+	    $mapping{'languagemap'}{$languages} = $plang;
+	    push (@{$mapping{'languagemaporder'}}, $language);
+	}
+	$mapping{$index} = $dirname;
+	$dirnames{$dirname} = $index;
+	$pnames{'index'}{$pindex} = "$level:$gran";
+	$pnames{'subcollection'}{$psub} = $subcollection;
+	$pnames{'languages'}{$plang} = $languages;
+    }
+
+    return \%mapping;
+}
+
+# returns a processed version of a field.
+# if the field has only one component the processed
+# version will contain the first character and next consonant
+# of that componant - otherwise it will contain the first 
+# character of the first two components 
+sub process_field {
+    my $self = shift (@_);
+    my ($field) = @_;
+ 
+    return "" unless (defined ($field) && $field =~ /\w/);
+
+    my @components = split /,/, $field;
+    if (scalar @components >= 2) {
+	splice (@components, 2);
+	map {s/^(.).*$/$1/;} @components;
+	return join("", @components);
+    } else {
+	my ($a, $b) = $field =~ /^(.).*?([bcdfghjklmnpqrstvwxyz])/i;
+	($a, $b) = $field =~ /^(.)(.)/ unless defined $a && defined $b;
+	return "$a$b";
+    }
+}
+
+sub make_unique {
+    my $self = shift (@_);
+    my ($namehash, $index, $indexref, $subref, $langref) = @_;
+    my ($level, $gran, $subcollection, $languages) = split (":", $index);
+
+    if ($namehash->{'index'}->{$$indexref} ne "$level:$gran") {
+	$self->get_next_version ($indexref);
+    } elsif ($namehash->{'subcollection'}->{$$subref} ne $subcollection) {
+	$self->get_next_version ($subref);
+    } elsif ($namehash->{'languages'}->{$$langref} ne $languages) {
+	$self->get_next_version ($langref);
+    }
+    return "$$indexref$$subref$$langref";
+}	
+
+sub get_next_version {
+    my $self = shift (@_);
+    my ($nameref) = @_;
+
+    if ($$nameref =~ /(\d\d)$/) {
+	my $num = $1; $num ++;
+	$$nameref =~ s/\d\d$/$num/;
+    } elsif ($$nameref =~ /(\d)$/) {
+	my $num = $1;
+	if ($num == 9) {$$nameref =~ s/\d\d$/10/;}
+	else {$num ++; $$nameref =~ s/\d$/$num/;}
+    } else {
+	$$nameref =~ s/.$/0/;
+    }
+}
+
+sub build_index {
+    my $self = shift (@_);
+    my ($index) = @_;
+
+    # get the full index directory path and make sure it exists
+    my $indexdir = $self->{'index_mapping'}->{$index};
+    &util::mk_all_dir (&util::filename_cat($self->{'build_dir'}, $indexdir));
+    my $fullindexprefix = &util::filename_cat ($self->{'build_dir'}, $indexdir, 
+					       $self->{'collection'});
+    my $fulltextprefix = &util::filename_cat ($self->{'build_dir'}, "text", 
+					       $self->{'collection'});
+
+    # get any os specific stuff
+    my $exedir = "$ENV{'GSDLHOME'}/bin/$ENV{'GSDLOS'}";
+    my $exe = &util::get_os_exe ();
+    my $mg_passes_exe = &util::filename_cat($exedir, "mg_passes$exe");
+    my $mg_perf_hash_build_exe = 
+	&util::filename_cat($exedir, "mg_perf_hash_build$exe");
+    my $mg_weights_build_exe = 
+	&util::filename_cat ($exedir, "mg_weights_build$exe");
+    my $mg_invf_dict_exe = 
+	&util::filename_cat ($exedir, "mg_invf_dict$exe");
+    my $mg_stem_idx_exe =
+	&util::filename_cat ($exedir, "mg_stem_idx$exe");
+
+    my $osextra = "";
+    if ($ENV{'GSDLOS'} =~ /^windows$/i) {
+	$fullindexprefix =~ s/\//\\/g;
+    } else {
+	$osextra = " -d /";
+    }
+
+    # get the index level from the index description
+    # the index will be level 2 unless we are building a
+    # paragraph level index
+    my $index_level = 2;
+    $index_level = 3 if $index =~ /^paragraph/i;
+
+    # get the index expression if this index belongs
+    # to a subcollection
+    my $indexexparr = [];
+    my ($level, $fields, $subcollection) = split (":", $index);
+    my @subcollections = ();
+    @subcollections = split /,/, $subcollection if (defined $subcollection);
+
+    foreach $subcollection (@subcollections) {
+	if (defined ($self->{'collect_cfg'}->{'subcollection'}->{$subcollection})) {
+	    push (@$indexexparr, $self->{'collect_cfg'}->{'subcollection'}->{$subcollection});
+	} 
+    }
+    
+    # add expressions for languages if this index belongs to
+    # a language subcollection
+    foreach $language (@{$self->{'collect_cfg'}->{'languages'}}) {
+	if ($language =~ s/^\!//) {
+	    push (@$indexexparr, "!Language/$language/");
+	} else {
+	    push (@$indexexparr, "Language/$language/");
+	}
+    }
+    
+    # set up the document processor
+    $self->{'buildproc'}->set_output_handle ('mgbuilder::PIPEOUT');
+    $self->{'buildproc'}->set_mode ('text');
+    $self->{'buildproc'}->set_index ($index, $indexexparr);
+    $self->{'buildproc'}->set_indexing_text (1);
+
+
+    # Build index dictionary. Uses verbatim stem method
+    print STDERR "\n    creating index dictionary\n"  if ($self->{'verbosity'} >= 1);
+    if (!-e "$mg_passes_exe" || !open (PIPEOUT, 
+	    "| $mg_passes_exe -f $fullindexprefix -b $maxdocsize " .
+	    "-$index_level -m 32 -s 0 -G -t 10 -N1 $osextra")) {
+	die "mgbuilder::build_index - couldn't run $mg_passes_exe\n";
+    }
+    $self->{'buildproc'}->reset();
+    &plugin::read ($self->{'pluginfo'}, $self->{'source_dir'}, 
+		   "", {}, $self->{'buildproc'}, $self->{'maxdocs'});
+    close (PIPEOUT);
+
+    # create the perfect hash function
+    if (!-e "$mg_perf_hash_build_exe") {
+	die "mgbuilder::build_index - couldn't run $mg_perf_hash_build_exe\n";
+    }
+    system ("$mg_perf_hash_build_exe -f $fullindexprefix $osextra");
+
+    # invert the text
+    print STDERR "\n    inverting the text\n"  if ($self->{'verbosity'} >= 1);
+    if (!-e "$mg_passes_exe" || !open (PIPEOUT, 
+	    "| $mg_passes_exe -f $fullindexprefix -b $maxdocsize " .
+	    "-$index_level -c 3 -G -t 10 -N2 $osextra")) {
+	die "mgbuilder::build_index - couldn't run $mg_passes_exe\n";
+    }
+    $self->{'buildproc'}->reset();
+    &plugin::read ($self->{'pluginfo'}, $self->{'source_dir'}, 
+		   "", {}, $self->{'buildproc'}, $self->{'maxdocs'});
+    close (PIPEOUT);
+
+    # create the weights file
+    print STDERR "\n    create the weights file\n"  if ($self->{'verbosity'} >= 1);
+    if (!-e "$mg_weights_build_exe") {
+	die "mgbuilder::build_index - couldn't run $mg_weights_build_exe\n";
+    }
+    system ("$mg_weights_build_exe -f $fullindexprefix -t $fulltextprefix $osextra");
+
+    # create 'on-disk' stemmed dictionary
+    print STDERR "\n    creating 'on-disk' stemmed dictionary\n"  if ($self->{'verbosity'} >= 1);
+    if (!-e "$mg_invf_dict_exe") {
+	die "mgbuilder::build_index - couldn't run $mg_invf_dict_exe\n";
+    }
+    system ("$mg_invf_dict_exe -f $fullindexprefix $osextra");
+
+
+    # creates stem index files for the various stemming methods
+    print STDERR "\n    creating stem indexes\n"  if ($self->{'verbosity'} >= 1);
+    if (!-e "$mg_stem_idx_exe") {
+	die "mgbuilder::build_index - couldn't run $mg_stem_idx_exe\n";
+    }
+    system ("$mg_stem_idx_exe -b 4096 -s1 -f $fullindexprefix $osextra");
+    system ("$mg_stem_idx_exe -b 4096 -s2 -f $fullindexprefix $osextra");
+    system ("$mg_stem_idx_exe -b 4096 -s3 -f $fullindexprefix $osextra");
+
+    
+    # remove unwanted files
+    my $tmpdir = &util::filename_cat ($self->{'build_dir'}, $indexdir);
+    opendir (DIR, $tmpdir) || die
+	"mgbuilder::build_index - couldn't read directory $tmpdir\n";
+    foreach $file (readdir(DIR)) {
+	next if $file =~ /^\./;
+	my ($suffix) = $file =~ /\.([^\.]+)$/;
+	if (defined $suffix && !defined $wanted_index_files{$suffix}) {
+	    # delete it!
+#	    print STDERR "deleting $file\n";
+	    &util::rm (&util::filename_cat ($tmpdir, $file));
+	}
+    }
+    closedir (DIR);
+}
+
+sub make_infodatabase {
+    my $self = shift (@_);
+    my $textdir = &util::filename_cat($self->{'build_dir'}, "text");
+    &util::mk_all_dir ($textdir);
+
+    # get db name
+    my $dbext = ".bdb";
+    $dbext = ".ldb" if &util::is_little_endian();
+    my $fulldbname = &util::filename_cat ($textdir, "$self->{'collection'}$dbext");
+    $fulldbname =~ s/\//\\/g if ($ENV{'GSDLOS'} =~ /^windows$/i);
+
+    my $exedir = "$ENV{'GSDLHOME'}/bin/$ENV{'GSDLOS'}";
+    my $exe = &util::get_os_exe ();
+    my $txt2db_exe = &util::filename_cat($exedir, "txt2db$exe");
+
+    print STDERR "\n*** creating the info database\n" if ($self->{'verbosity'} >= 1);
+
+    # init all the classifiers
+    &classify::init_classifiers ($self->{'classifiers'});
+
+    # set up the document processor
+    $self->{'buildproc'}->set_output_handle ('mgbuilder::PIPEOUT');
+    $self->{'buildproc'}->set_mode ('infodb');
+    $self->{'buildproc'}->set_classifiers ($self->{'classifiers'});
+    $self->{'buildproc'}->set_indexing_text (0);
+    
+    # create the infodatabase
+    if (!-e "$txt2db_exe" || !open (PIPEOUT, 
+            "| $txt2db_exe $fulldbname")) {
+	die "mgbuilder::make_infodatabase - couldn't run $txt2db_exe\n";
+    }
+    $self->{'buildproc'}->reset();
+
+    if (defined $self->{'collect_cfg'}->{'collectionmeta'}) {
+
+	if (!defined $self->{'index_mapping'}) {
+	    $self->{'index_mapping'} = 
+		$self->create_index_mapping ($self->{'collect_cfg'}->{'indexes'});
+	}
+
+	print PIPEOUT "[collection]\n";
+
+	foreach $cmeta (keys (%{$self->{'collect_cfg'}->{'collectionmeta'}})) {
+	    if ($cmeta =~ s/^\.//) {
+		if (defined $self->{'index_mapping'}->{$cmeta}) {
+		    print PIPEOUT "<$self->{'index_mapping'}->{$cmeta}>" .
+			$self->{'collect_cfg'}->{'collectionmeta'}->{".$cmeta"} . "\n";
+		} else {
+		    print STDERR "mgbuilder: warning bad collectionmeta option '$cmeta' - ignored\n";
+		}
+	    } else {
+		print PIPEOUT "<$cmeta>$self->{'collect_cfg'}->{'collectionmeta'}->{$cmeta}\n";
+	    }
+	}
+	print PIPEOUT "\n" . ('-' x 70) . "\n";
+
+    }
+    
+
+    &plugin::read ($self->{'pluginfo'}, $self->{'source_dir'}, 
+		   "", {}, $self->{'buildproc'}, $self->{'maxdocs'});
+
+    # output classification information
+    &classify::output_classify_info ($self->{'classifiers'}, 'mgbuilder::PIPEOUT',
+				     $self->{'allclassifications'});
+
+    close (PIPEOUT);
+}
+
+sub collect_specific {
+    my $self = shift (@_);
+}
+
+sub make_auxiliary_files {
+    my $self = shift (@_);
+    my ($index);
+    my %build_cfg = ();
+
+    print STDERR "\n*** creating auxiliary files \n" if ($self->{'verbosity'} >= 1);
+
+    # get the text directory
+    &util::mk_all_dir ($self->{'build_dir'});
+
+    # store the build date
+    $build_cfg->{'builddate'} = time;
+
+    # store the number of documents and number of bytes
+    $build_cfg->{'numdocs'} = $self->{'buildproc'}->get_num_docs();
+    $build_cfg->{'numbytes'} = $self->{'buildproc'}->get_num_bytes();
+
+    # store the mapping between the index names and the directory names
+    my @indexmap = ();
+    foreach $index (@{$self->{'index_mapping'}->{'indexmaporder'}}) {
+	push (@indexmap, "$index\-\>$self->{'index_mapping'}->{'indexmap'}->{$index}");
+    }
+    $build_cfg->{'indexmap'} = \@indexmap;
+
+    my @subcollectionmap = ();
+    foreach $subcollection (@{$self->{'index_mapping'}->{'subcollectionmaporder'}}) {
+	push (@subcollectionmap, "$subcollection\-\>" .
+	      $self->{'index_mapping'}->{'subcollectionmap'}->{$subcollection});
+    }
+    $build_cfg->{'subcollectionmap'} = \@subcollectionmap if scalar (@subcollectionmap);
+
+    my @languagemap = ();
+    foreach $language (@{$self->{'index_mapping'}->{'languagemaporder'}}) {
+	push (@languagemap, "$language\-\>" .
+	      $self->{'index_mapping'}->{'languagemap'}->{$language});
+    }
+    $build_cfg->{'languagemap'} = \@languagemap if scalar (@languagemap);
+
+    $build_cfg->{'notbuilt'} = $self->{'notbuilt'};
+
+    # write out the build information
+    &cfgread::write_cfg_file("$self->{'build_dir'}/build.cfg", $build_cfg,
+			     '^(builddate|numdocs|numbytes)$', 
+                             '^(indexmap|subcollectionmap|languagemap|notbuilt)$');
+
+}
+
+sub deinit {
+    my $self = shift (@_);
+}
+
+
+1;
+
+
Index: /tags/gsdl-21/gsdl/perllib/mgbuildproc.pm
===================================================================
--- /tags/gsdl-21/gsdl/perllib/mgbuildproc.pm	(revision 699)
+++ /tags/gsdl-21/gsdl/perllib/mgbuildproc.pm	(revision 699)
@@ -0,0 +1,390 @@
+###########################################################################
+#
+# mgbuildproc.pm -- 
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# This document processor outputs a document
+# for mg to process
+
+
+package mgbuildproc;
+
+use classify;
+use doc;
+use docproc;
+use util;
+
+
+BEGIN {
+    @ISA = ('docproc');
+}
+
+
+sub new {
+    my ($class, $collection, $source_dir, $build_dir, $verbosity) = @_;
+    my $self = new docproc ();
+
+    $self->{'collection'} = $collection;
+    $self->{'source_dir'} = $source_dir;
+    $self->{'build_dir'} = $build_dir;
+    $self->{'verbosity'} = $verbosity;
+    $self->{'classifiers'} = [];
+    $self->{'mode'} = "text";
+    $self->{'index'} = "section:text";
+    $self->{'indexexparr'} = [];
+    $self->{'output_handle'} = "STDOUT";
+    $self->{'num_docs'} = 0;
+    $self->{'num_sections'} = 0;
+    $self->{'num_bytes'} = 0;
+
+    $self->{'indexing_text'} = 0;
+
+    return bless $self, $class;
+}
+
+sub reset {
+    my $self = shift (@_);
+    
+    $self->{'num_docs'} = 0;
+    $self->{'num_sections'} = 0;
+    $self->{'num_bytes'} = 0;
+}
+
+sub get_num_docs {
+    my $self = shift (@_);
+
+    return $self->{'num_docs'};
+}
+
+sub get_num_sections {
+    my $self = shift (@_);
+
+    return $self->{'num_sections'};
+}
+
+sub get_num_bytes {
+    my $self = shift (@_);
+
+    return $self->{'num_bytes'};
+}
+
+sub set_output_handle {
+    my $self = shift (@_);
+    my ($handle) = @_;
+
+    $self->{'output_handle'} = $handle;
+}
+
+sub set_mode {
+    my $self = shift (@_);
+    my ($mode) = @_;
+
+    $self->{'mode'} = $mode;
+}
+
+sub set_index {
+    my $self = shift (@_);
+    my ($index, $indexexparr) = @_;
+
+    $self->{'index'} = $index;
+    $self->{'indexexparr'} = $indexexparr if defined $indexexparr;
+}
+
+sub set_classifiers {
+    my $self = shift (@_);
+    my ($classifiers) = @_;
+
+    $self->{'classifiers'} = $classifiers;
+}
+
+sub set_indexing_text {
+    my $self = shift (@_);
+    my ($indexing_text) = @_;
+
+    $self->{'indexing_text'} = $indexing_text;
+}
+
+sub process {
+    my $self = shift (@_);
+    my $method = $self->{'mode'};
+
+    $self->$method(@_);
+}
+
+# use 'Paged' if document has no more than 2 levels
+# and each section at second level has a number for
+# Title metadata
+sub get_document_type {
+    my $self = shift (@_);
+    my ($doc_obj) = @_;
+
+    my $thistype = "VList";
+    my $childtype = "VList";
+    my $title;
+    my @tmp = ();
+    
+    my $section = $doc_obj->get_top_section ();
+    my $first = 1;
+    while (defined $section) {
+	@tmp = split /\./, $section;
+	if (scalar(@tmp) > 1) {
+	    return ($thistype, $childtype);
+	}
+	if (!$first) {
+	    $title = $doc_obj->get_metadata_element ($section, "Title");
+	    if (defined $title && $title !~ /^\d+$/) {
+		return ($thistype, $childtype);
+	    }
+	}
+	$first = 0;
+	$section = $doc_obj->get_next_section($section);
+    }
+    if ($doc_obj->get_text_length ($doc_obj->get_top_section())) {
+	$thistype = "Paged";
+    } else {
+	$thistype = "Invisible";
+    }
+    $childtype = "Paged";
+    return ($thistype, $childtype);
+}
+
+sub infodb {
+    my $self = shift (@_);
+    my ($doc_obj, $filename) = @_;
+    my $handle = $self->{'output_handle'};
+#    $handle = "main::STDOUT";
+
+    my $doctype = $doc_obj->get_doc_type();
+
+    # only output this document if it is one to be indexed
+    return if ($doctype ne "indexed_doc");
+
+    # this is another document
+    $self->{'num_docs'} += 1 unless ($doctype eq "classification");
+
+    # is this a paged or a hierarchical document
+    my ($thistype, $childtype) = $self->get_document_type ($doc_obj);
+
+    my $section = $doc_obj->get_top_section ();
+    my $doc_OID = $doc_obj->get_OID();
+    my $first = 1;
+    while (defined $section) {
+	# update a few statistics
+	$self->{'num_bytes'} += $doc_obj->get_text_length ($section);
+	$self->{'num_sections'} += 1 unless ($doctype eq "classification");
+
+	# output the section name
+	if ($section eq "") { print $handle "[$doc_OID]\n"; }
+	else { print $handle "[$doc_OID.$section]\n"; }
+
+	# output the fact that this document is a document
+	print $handle "<doctype>doc\n";
+
+	# output whether this node contains text
+	if ($doc_obj->get_text_length($section) > 0) {
+	    print $handle "<hastxt>1\n";
+	} else {
+	    print $handle "<hastxt>0\n";
+	}
+
+	# output all the section metadata
+	my $metadata = $doc_obj->get_all_metadata ($section);
+	foreach $pair (@$metadata) {
+	    my ($field, $value) = (@$pair);
+
+	    if ($field ne "Identifier" && $field !~ /^gsdl/ && 
+		defined $value && $value ne "") {
+		# escape problematic stuff
+		$value =~ s/\\/\\\\/g;
+		$value =~ s/\n/\\n/g;
+		$value =~ s/\r/\\r/g;
+		
+		print $handle "<$field>$value\n";
+	    }
+	}
+
+	# output archivedir if at top level
+	if ($section eq $doc_obj->get_top_section()) {
+	    my ($archivedir) = $filename =~ /^(.*?)(?:\/|\\)[^\/\\]*$/;
+	    $archivedir = "" unless defined $archivedir;
+	    $archivedir =~ s/^(\/|\\)*//;
+	    $archivedir =~ s/(\/|\\)*$//;
+	    print $handle "<archivedir>$archivedir\n";
+	}
+
+	# output document display type
+	if ($first) {
+	    print $handle "<thistype>$thistype\n";
+	}
+
+	# output a list of children
+	my $children = $doc_obj->get_children ($section);
+	if (scalar(@$children) > 0) {
+	    print $handle "<childtype>$childtype\n";
+	    print $handle "<contains>";
+	    my $firstchild = 1;
+	    foreach $child (@$children) {
+		print $handle ";" unless $firstchild;
+		$firstchild = 0;
+		if ($child =~ /^.*?\.(\d+)$/) {
+		    print $handle "\".$1";
+		} else {
+		    print $handle "\".$child";
+		}
+#		if ($child eq "") { print $handle "$doc_OID"; }
+#		elsif ($section eq "") { print $handle "$doc_OID.$child"; }
+#		else { print $handle "$doc_OID.$section.$child"; }
+	    }
+	    print $handle "\n";
+	}
+
+	# output the matching document number
+	print $handle "<docnum>$self->{'num_sections'}\n";
+
+	print $handle '-' x 70, "\n";
+
+	
+	# output a database entry for the document number
+	print $handle "[$self->{'num_sections'}]\n";
+	if ($section eq "") { print $handle "<section>$doc_OID\n"; }
+	else { print $handle "<section>$doc_OID.$section\n"; }
+	print $handle '-' x 70, "\n";
+
+	$first = 0;
+	$section = $doc_obj->get_next_section($section);
+    }
+
+    # classify this document
+    &classify::classify_doc ($self->{'classifiers'}, $doc_obj);
+
+}
+
+sub find_paragraphs {
+    $_[1] =~ s/(<p\b)/\cC$1/gi;
+}
+
+sub filter_text {
+    # $self->filter_text ($field, $new_text);
+    # don't want to do anything for this version, however,
+    # in a particular collection you might want to override
+    # this method to post-process certain fields depending on
+    # the field, or whether we are outputting it for indexing
+}
+
+sub text {
+    my $self = shift (@_);
+    my ($doc_obj) = @_;
+    my $handle = $self->{'output_handle'};
+    my $indexed_doc = 1;
+
+    # only output this document if it is one to be indexed
+    return if ($doc_obj->get_doc_type() ne "indexed_doc");
+
+    # see if this document belongs to this subcollection
+    foreach $indexexp (@{$self->{'indexexparr'}}) {
+	$indexed_doc = 0;
+	my ($field, $exp, $options) = split /\//, $indexexp;
+	if (defined ($field) && defined ($exp)) {
+	    my ($bool) = $field =~ /^(.)/;
+	    $field =~ s/^.// if $bool eq '!';
+	    if ($field =~ /^filename$/i) {
+		$field = $doc_obj->get_source_filename();
+	    } else {
+		$field = $doc_obj->get_metadata_element($doc_obj->get_top_section(), $field);
+	    }
+	    next unless defined $field;
+	    if ($bool eq '!') {
+		if ($options =~ /^i$/i) {
+		    if ($field !~ /$exp/i) {$indexed_doc = 1; last;}
+		} else {
+		    if ($field !~ /$exp/) {$indexed_doc = 1; last;}
+		}
+	    } else {
+		if ($options =~ /^i$/i) {
+		    if ($field =~ /$exp/i) {$indexed_doc = 1; last;}
+		} else {
+		    if ($field =~ /$exp/) {$indexed_doc = 1; last;}
+		}
+	    }
+	}
+    }
+
+    # this is another document
+    $self->{'num_docs'} += 1;
+
+    # get the parameters for the output
+    my ($level, $fields) = split (/:/, $self->{'index'});
+    $fields =~ s/\ball\b/Title,Creator,text/;
+    $fields =~ s/\btopall\b/topTitle,topCreator,toptext/;
+
+    my $doc_section = 0; # just for this document
+    my $text = "";
+    my $text_extra = "";
+
+    # get the text for this document
+    my $section = $doc_obj->get_top_section();
+    while (defined $section) {
+	# update a few statistics
+	$doc_section++;
+	$self->{'num_sections'} += 1;
+
+	if ($indexed_doc) {
+	    $self->{'num_bytes'} += $doc_obj->get_text_length ($section);
+	    foreach $field (split (/,/, $fields)) {
+		# only deal with this field if it doesn't start with top or
+		# this is the first section
+		my $real_field = $field;
+		if (!($real_field =~ s/^top//) || ($doc_section == 1)) {
+		    my $new_text = "";
+		    if ($real_field eq "text") {
+			$new_text = $doc_obj->get_text ($section);
+			$new_text =~ s/[\cB\cC]//g;
+			$self->find_paragraphs($new_text);
+			
+		    } else {
+			$new_text = join ("\cC", @{$doc_obj->get_metadata ($section, $real_field)});
+		    }
+		    
+		    # filter the text
+		    $self->filter_text ($field, $new_text);
+
+		    if ($self->{'indexing_text'} && 
+			$new_text =~ /[\(\)\{\}]/) {
+		    }
+
+		    $text .= "$new_text\cC";
+		}
+	    }
+	}
+	
+	if ($level eq "document") { $text_extra .= "\cB"; }
+	else { $text .= "\cB"; }
+	
+	$section = $doc_obj->get_next_section($section);
+    }
+
+    print $handle "$text$text_extra";
+}
+
+1;
+
Index: /tags/gsdl-21/gsdl/perllib/multiread.pm
===================================================================
--- /tags/gsdl-21/gsdl/perllib/multiread.pm	(revision 699)
+++ /tags/gsdl-21/gsdl/perllib/multiread.pm	(revision 699)
@@ -0,0 +1,252 @@
+###########################################################################
+#
+# multiread.pm --
+#
+# Copyright (C) 1999 DigiLib Systems Limited, NZ
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# the multiread object will read in a number of encodings,
+# the results are always returned in the utf-8 format
+
+# encodings currently supported are
+#
+# utf8     - either utf8 or unicode (automatically detected)
+# unicode  - just unicode (doesn't currently do endian detection)
+# gb       - GB
+# extended - extended ascii
+
+
+package multiread;
+
+use unicode;
+use gb;
+
+sub new {
+    my ($class) = @_;
+
+    my $self = {'handle'    => "",
+		'first'     => 1,
+		'encoding'  => "utf8",
+		'bigendian' => 1};
+
+    return bless $self, $class;
+}
+
+# set_handle expects the file to be already open but
+# not read yet
+sub set_handle {
+    my $self = shift (@_);
+    ($self->{'handle'}) = @_;
+    $self->{'first'} = 1;
+    $self->{'encoding'} = "utf8";
+    $self->{'bigendian'} = 1;
+}
+
+# set_encoding should be called after set_handle
+sub set_encoding {
+    my $self = shift (@_);
+    ($self->{'encoding'}) = @_;
+}
+
+sub get_encoding {
+    my $self = shift (@_);
+    return $self->{'encoding'};
+}
+
+# undef will be returned if the eof has been reached
+# the result will always be returned in utf-8
+# if automatic detection between utf8 and unicode is desired
+# then the encoding should be initially set to utf8
+sub read_char {
+    my $self = shift (@_);
+
+    # make sure we have a file handle
+    return undef if ($self->{'handle'} eq "");
+    my $handle = $self->{'handle'};
+
+    if ($self->{'encoding'} eq "utf8") {
+	# utf-8 text, how many characters we get depends 
+	# on what we find
+	my $c1 = "";
+	my $c2 = "";
+	my $c3 = "";
+
+	while (!eof ($handle)) {
+	    $c1 = ord (getc ($handle));
+	    if ($self->{'first'}) {
+		$self->{'first'} = 0;
+
+		if ($c1 == 0xfe || $c1 == 0xff) {
+		    $c2 = ord (getc ($handle)) if (!eof ($handle));
+		    
+		    # if unicode fall through to the unicode reading code
+		    if ($c1 == 0xff && $c2 == 0xfe) {
+			$self->{'encoding'} = "unicode";
+			$self->{'bigendian'} = 0;
+			if ($ENV{'GSDLOS'} =~ /windows/i) {
+			    binmode ($handle); # silly windows
+			}
+			last;
+
+		    } elsif ($c1 == 0xfe && $c2 == 0xff) {
+			$self->{'encoding'} = "unicode";
+			$self->{'bigendian'} = 1;
+			if ($ENV{'GSDLOS'} =~ /windows/i) {
+			    binmode ($handle); # silly windows
+			}
+			last;
+		    }
+
+		    # an error, but we might be able to recover
+		    # from it
+		    $c1 = $c2;
+		}
+	    }
+
+	    if ($c1 <= 0x7f) {
+		# one byte character
+		return chr ($c1);
+
+	    } elsif ($c1 >= 0xc0 && $c1 <= 0xdf) {
+		# two byte character
+		$c2 = getc ($handle) if (!eof ($handle));
+		return chr ($c1) . $c2;
+
+	    } elsif ($c1 >= 0xe0 && $c1 <= 0xef) {
+		# three byte character
+		$c2 = getc ($handle) if (!eof ($handle));
+		$c3 = getc ($handle) if (!eof ($handle));
+		return chr ($c1) . $c2 . $c3;
+	    }
+
+	    # if we get here there was an error in the file, we should
+	    # be able to recover from it however, maybe the file is in
+	    # another encoding
+	}
+
+	return undef if (eof ($handle));
+    }
+
+    if ($self->{'encoding'} eq "unicode") {
+	# unicode text, get the next two characters
+	return undef if (eof ($handle));
+	my $c1 = ord (getc ($handle));
+	return undef if (eof ($handle));
+	my $c2 = ord (getc ($handle));
+
+	return &unicode::unicode2utf8 ([(($self->{'bigendian'}) ? ($c1*256+$c2) : ($c2*256+$c1))]);
+    }
+
+    if ($self->{'encoding'} eq "gb") {
+	# GB or GBK
+	return undef if (eof ($handle));
+	my $c1 = getc ($handle);
+	if (ord ($c1) >= 0x81) {
+	    # double byte character
+	    return undef if (eof ($handle));
+	    my $c2 = getc ($handle);
+	    return &unicode::unicode2utf8 (&gb::gb2unicode ($c1.$c2));
+	    
+	} else {
+	    # single byte character
+	    return &unicode::ascii2utf8 ($c1);
+	}
+    }
+
+    if ($self->{'encoding'} eq "extended") {
+	# extended ascii
+	return undef if (eof ($handle));
+	return &unicode::ascii2utf8 (getc ($handle));
+    }
+
+    # unknown encoding
+    return undef;
+}
+
+
+# undef will be returned if the eof has been reached
+# the result will always be returned in utf-8
+sub read_line {
+    my $self = shift (@_);
+
+    # make sure we have a file handle
+    return undef if ($self->{'handle'} eq "");
+
+    my $handle = $self->{'handle'};
+
+    if ($self->{'first'} && $self->{'encoding'} eq "utf8") {
+	# special case for the first line of utf8 text to detect whether
+	# the file is in utf8 or unicode
+	my $out = "";
+	my $thisc = "";
+	while (defined ($thisc = $self->read_char())) {
+	    $out .= $thisc;
+	    last if ($thisc eq "\n");
+	}
+	
+	return $out if (length ($out) > 0);
+	return undef;
+    }
+
+
+    if ($self->{'encoding'} eq "utf8") {
+	# utf-8 line
+	return <$handle>;
+    }
+
+    if ($self->{'encoding'} eq "unicode") {
+	# unicode line
+	my $c = "";
+	my ($c1, $c2) = ("", "");
+	my $out = "";
+	while (read ($handle, $c, 2) == 2) {
+	    $c1 = ord (substr ($c, 0, 1));
+	    $c2 = ord (substr ($c, 1, 1));
+	    $c = &unicode::unicode2utf8([(($self->{'bigendian'}) ? ($c1*256+$c2) : ($c2*256+$c1))]);
+	    $out .= $c;
+	    last if ($c eq "\n");
+	}
+
+	return $out if (length ($out) > 0);
+	return undef;
+    }
+
+    if ($self->{'encoding'} eq "gb") {
+	# GB or GBK
+	my $line = "";
+	if (defined ($line = <$handle>)) {
+	    return &unicode::unicode2utf8 (&gb::gb2unicode ($line));
+	}
+	return undef;
+    }
+    
+    if ($self->{'encoding'} eq "extended") {
+	# extended ascii
+	my $line = "";
+	if (defined ($line = <$handle>)) {
+	    return &unicode::ascii2utf8 ($line);
+	}
+	return undef;
+    }
+
+    # unknown encoding
+    return undef;
+}
+
+
+1;
Index: /tags/gsdl-21/gsdl/perllib/muread.pm
===================================================================
--- /tags/gsdl-21/gsdl/perllib/muread.pm	(revision 699)
+++ /tags/gsdl-21/gsdl/perllib/muread.pm	(revision 699)
@@ -0,0 +1,151 @@
+###########################################################################
+#
+# muread.pm -- read a marked-up file
+#
+# Copyright (C) 1999 DigiLib Systems Limited, NZ
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+
+package muread;
+
+use unicode;
+use multiread;
+
+sub new {
+    my ($class) = @_;
+
+    my $self = {'filename'=>"",
+		'encoding'=>"",
+		'handle'=>"",
+		'reader'=>"",
+		'buffer'=>""};
+
+    return bless $self, $class;
+}
+
+# returns a new tag with a tag name and any options
+sub parse_tag {
+    my $self = shift (@_);
+    my ($orgtagtext) = @_;
+    my $tagtext = $orgtagtext;
+    my $newtag = {};
+    my $misformed = 0;
+
+#    print STDERR "parsing \"$tagtext\"\n";
+
+    # get tag name (if there is one)
+    if ($tagtext =~ /^(\w+)/) {
+	$newtag->{'_tagname'} = $1;
+	$tagtext =~ s/^(\w+)//;
+    } else {
+	print STDERR "muread::parse_tag error - no tag name found\n";
+    }
+
+    # get the tag arguments
+    while ($tagtext =~ /\S/) {
+	$tagtext =~ s/^\s+//s;
+	if ($tagtext =~ /^(\w+)\s*=\s*\"([^\"]*)\"/s) {
+	    $newtag->{$1} = (defined $2) ? $2 : "";
+	    $tagtext =~ s/^\w+\s*=\s*\"[^\"]*\"//s;
+
+	} else {
+	    if (!$misformed) {
+		print STDERR "muread::parse_tag error - miss-formed tag <$orgtagtext>\n";
+		$misformed = 1;
+	    }
+	    $tagtext =~ s/^\S+//s;
+	}
+    }
+
+    return $newtag;
+}
+
+sub read_tag_content {
+    my $self = shift (@_);
+    my ($tag) = @_;
+
+    # all tags contain a _tagname except the tag for the document
+
+    my $line = "";
+    while (1) {
+	# deal with preceeding text
+	if ($self->{'buffer'} =~ /^([^<]+)</s) {
+	    # add preceeding text
+	    $tag->{'_contains'} = [] unless defined $tag->{'_contains'};
+	    push (@{$tag->{'_contains'}}, {'_text'=>$1});
+
+	    $self->{'buffer'} =~ s/^[^<]+</</s;
+	}
+
+	if ($self->{'buffer'} =~ /^<([^>\/]+)>/s) {
+	    # add info from this tag
+	    my $tagtext = $1;
+	    my $newtag = $self->parse_tag ($tagtext);
+	    push (@{$tag->{'_contains'}}, $newtag);
+	    $self->{'buffer'} =~ s/^<[^>\/]+>//s;
+	    
+	    # deal with the contents of this tag
+	    $self->read_tag_content ($newtag);
+
+	} elsif ($self->{'buffer'} =~ /^<\/([^>\/]+)>/s) {
+	    my $tagname = $1;
+	    $self->{'buffer'} =~ s/^<\/[^>\/]+>//s;
+
+	    # check that this tag is the right tag
+	    if (!defined $tag->{'_tagname'} || $tag->{'_tagname'} ne $tagname) {
+		print STDERR "muread::read_tag_content error - mismatched tag </$tagname>, " .
+		    "expected </$tag->{'_tagname'}>\n";
+	    } else {
+		return;
+	    }
+	} elsif (defined ($line = $self->{'reader'}->read_line())) {
+	    $self->{'buffer'} .= $line;
+	} else {
+	    if ($self->{'buffer'} =~ /\S/) {
+		print STDERR "muread::read_tag_content error - can't parse text \"$self->{'buffer'}\"\n";
+	    }
+	    last;
+	}
+    }
+    
+    if (defined $tag->{'_tagname'}) {
+	print STDERR "muread::read_tag_content error - eof reached before closing " .
+	    "tag \"$tag->{'_tagname'}\" found\n";
+    }
+}
+
+sub read_file {
+    my $self = shift (@_);
+    ($self->{'handle'}, $self->{'filename'}, $self->{'encoding'}) = @_;
+    $self->{'encoding'} = "utf8" unless defined $self->{'encoding'};
+
+    my $doc = {};
+
+    # get reader set up
+    $self->{'reader'} = new multiread ();
+    $self->{'reader'}->set_handle ($self->{'handle'});
+    $self->{'reader'}->set_encoding ($self->{'encoding'});
+
+    # read in the file
+    $self->read_tag_content ($doc);
+
+    $self->{'handle'} = "";
+    return $doc;
+}
+
+1;
Index: /tags/gsdl-21/gsdl/perllib/parsargv.pm
===================================================================
--- /tags/gsdl-21/gsdl/perllib/parsargv.pm	(revision 699)
+++ /tags/gsdl-21/gsdl/perllib/parsargv.pm	(revision 699)
@@ -0,0 +1,206 @@
+###########################################################################
+#
+# parseargv.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package parsargv;
+# 
+# parse(ARGVREF, [SPEC, VARREF] ...)
+# 
+# Parse command line arguments.
+# 
+# ARGVREF is an array reference, usually to @ARGV.  The remaining
+# arguments are paired (SPEC, VARREF).  SPEC is a specification string
+# for a particular argument; VARREF is a variable reference that will
+# receive the argument.
+# 
+# SPEC is in one of the following forms:
+# 
+# ARG/REGEX/DEFAULT ARG is the name of command line argument.  REGEX is
+#     a regular expression that gives legal values for the argument.
+#     DEFAULT is the default value assigned to VARREF if the option does
+#     not appear on the command line.  Example
+# 
+# 
+# ARG/REGEX ARG and REGEX are as above.  Since no default is given, ARG
+#     must appear on the command line; if it doesn't, parse() returns 0.
+# 
+# ARG ARG is as above.  ARG is a boolean option.  VARREF is assigned 0 if ARG
+#     is not on the command line; 1 otherwise.
+# 
+# SPEC may start with a punctuation character, in which case this
+# character will be used instead of '/' as a delimiter.  Useful when '/'
+# is needed in the REGEX part.
+# 
+# VARREF is a reference to a scalar or an array.  If VARREF is an array
+# reference, then multiple command line options are allowed an append.  Example:
+# 
+# 	Command line: -day mon -day fri
+# 
+# 	parse(\@ARGV, "day/(mon|tue|wed|thu|fri)", \@days)
+# 
+# 	days => ('mon', 'fri')
+# 
+# Returns 0 if there was an error, nonzero otherwise.
+# 
+sub parse
+{
+    my $arglist = shift;
+    my ($spec, $var);
+    my %option;
+
+    while (($spec, $var) = splice(@_, 0, 2))
+    {
+	die "Variable for $spec is not a valid type."
+	    unless ref($var) eq 'SCALAR' || ref($var) eq 'ARRAY';
+
+	my $delimiter;
+	if ($spec !~ /^\w/)
+	{
+	    $delimiter = substr($spec, 0, 1);
+	    $spec = substr($spec, 1);
+	}
+	else
+	{
+	    $delimiter = '/';
+	}
+	my ($name, $regex, $default) = split(/$delimiter/, $spec, 3);
+	
+	if ($name)
+	{
+	    if ($default && $default !~ /$regex/)
+	    {
+		die "Default value for $name doesn't match regex ($spec).";
+	    }
+	    $option{$name} = {'name' => $name,
+			      'regex' => $regex,
+			      'default' => $default,
+			      'varref' => $var,
+			      'set' => 0};
+	}
+	else
+	{
+	    die "Invalid argument ($spec) for parsargv.";
+	}
+    }
+
+    my @argv;
+    my $arg;
+    my $parse_options = 1;
+    my $errors = 0;
+
+    while ($arg = shift(@$arglist))
+    {
+	if ($parse_options && $arg eq '--')
+	{
+	    $parse_options = 0;
+	    next;
+	}
+
+	if ($parse_options && $arg =~ /^-+\w/)
+	{
+	    $arg =~ s/^-+//;
+
+	    if (defined $option{$arg})
+	    {
+		&process_arg($option{$arg}, $arglist, \$errors);
+	    }
+	    else
+	    {
+		print STDERR "$arg is not a valid option.\n";
+		$errors++;
+	    }
+	}
+	else
+	{
+	    push(@argv, $arg);
+	}
+    }
+    @$arglist = @argv;
+
+    foreach $arg (keys %option)
+    {
+	if ($option{$arg}->{'set'} == 0)
+	{
+	    if (defined $option{$arg}->{'default'})
+	    {
+		&set_var($option{$arg}, $option{$arg}->{'default'});
+	    }
+	    elsif (!$option{$arg}->{'regex'})
+	    {
+		&set_var($option{$arg}, 0)
+	    }
+	    elsif (ref($option{$arg}->{'varref'}) ne 'ARRAY')
+	    {
+		print STDERR "Missing command line argument -$arg.\n";
+		$errors++;
+	    }
+	}
+    }
+    return $errors == 0;
+}
+
+sub process_arg
+{
+    my ($option, $arglist, $errors) = @_;
+
+    if ($option->{'regex'} && @$arglist > 0 && $arglist->[0] !~ /^-+\w/)
+    {
+	if ($arglist->[0] =~ /$option->{'regex'}/)
+	{
+	    &set_var($option, shift(@$arglist));
+	}
+	else
+	{
+	    print STDERR "Bad value for -$option->{'name'} argument.\n";
+	    $$errors++;
+	}
+    }
+    elsif (!$option->{'regex'})
+    {
+	&set_var($option, 1);
+    }
+    else
+    {
+	print STDERR "No value given for -$option->{'name'}.\n";
+	$$errors++;
+    }
+}
+
+sub set_var
+{
+    my ($option, $value) = @_;
+    my $type = ref($option->{'varref'});
+
+    if ($type eq 'SCALAR')
+    {
+	${$option->{'varref'}} = $value;
+    }
+    elsif ($type eq 'ARRAY')
+    {
+	push(@{$option->{'varref'}}, $value);
+    }
+    $option->{'set'} = 1;
+}
+
+1;
Index: /tags/gsdl-21/gsdl/perllib/plugin.pm
===================================================================
--- /tags/gsdl-21/gsdl/perllib/plugin.pm	(revision 699)
+++ /tags/gsdl-21/gsdl/perllib/plugin.pm	(revision 699)
@@ -0,0 +1,77 @@
+###########################################################################
+#
+# plugin.pm -- functions to handle using plugins
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package plugin;
+
+require util;
+
+sub load_plugins {
+    my ($collection, $plugin_list) = @_;
+    my @plugin_objects = ();
+
+    foreach $plugin (@$plugin_list) {
+	# find the plugin
+	my $colplugname = &util::filename_cat($ENV{'GSDLCOLLECTDIR'},"perllib/plugins", 
+					      "${plugin}.pm");
+	my $mainplugname = &util::filename_cat($ENV{'GSDLHOME'},"perllib/plugins", 
+					       "${plugin}.pm");
+	if (-e $colplugname) { require $colplugname; }
+	elsif (-e $mainplugname) { require $mainplugname; }
+	else { die "ERROR - couldn't find plugin $plugin\n"; }
+
+	# create a plugin object
+	my ($plugobj);
+	eval ("\$plugobj = new $plugin()");
+	die "$@" if $@;
+
+	# add this object to the list
+	push (@plugin_objects, $plugobj);
+    }
+
+    return \@plugin_objects;
+}
+
+sub read {
+    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs) = @_;
+
+    my $rv = 0;
+
+    # pass this file by each of the plugins in turn until one
+    # is found which will process it
+    foreach $plugobj (@$pluginfo) {
+	$rv = $plugobj->read($pluginfo, $base_dir, $file, 
+			     $metadata, $processor, $maxdocs);
+	return $rv if defined $rv;
+    }
+    
+    if (defined $processor->{'verbosity'} && $processor->{'verbosity'} >= 2) {
+	print STDERR "WARNING - no plugin could process " . 
+	    &util::filename_cat($base_dir,$file) . "\n";
+    }
+    return 0;
+}
+
+
+1;
Index: /tags/gsdl-21/gsdl/perllib/plugins/ArcPlug.pm
===================================================================
--- /tags/gsdl-21/gsdl/perllib/plugins/ArcPlug.pm	(revision 699)
+++ /tags/gsdl-21/gsdl/perllib/plugins/ArcPlug.pm	(revision 699)
@@ -0,0 +1,95 @@
+###########################################################################
+#
+# ArcPlug.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# plugin which recurses through an archives.inf file
+# (i.e. the file generated in the archives directory
+# when an import is done), processing each file it finds 
+
+package ArcPlug;
+
+use util;
+use BasPlug;
+use plugin;
+use arcinfo;
+
+BEGIN {
+    @ISA = ('BasPlug');
+}
+
+sub new {
+    my ($class) = @_;
+    my $self = new BasPlug ();
+
+    return bless $self, $class;
+}
+
+# return 1 if this class might recurse using $pluginfo
+sub is_recursive {
+    my $self = shift (@_);
+
+    return 1;
+}
+
+# return number of files processed, undef if can't process
+# Note that $base_dir might be "" and that $file might 
+# include directories
+sub read {
+    my $self = shift (@_);
+    ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs) = @_;
+
+    my $count = 0;
+
+    # see if this has a archives information file within it
+    $archive_info_filename = &util::filename_cat($base_dir,$file,"archives.inf");
+
+    if (-e $archive_info_filename) {
+
+	# found an archives.inf file
+	print STDERR "ArcPlug: processing $archive_info_filename\n";
+
+	# read in the archives information file
+	my $archive_info = new arcinfo ();
+	$archive_info->load_info ($archive_info_filename);
+	
+	my $file_list = $archive_info->get_file_list();
+
+	# process each file
+	foreach $subfile (@$file_list) {
+	    last if (defined $maxdocs && $maxdocs =~ /\d/ && $count >= $maxdocs);
+
+	    my $tmp = &util::filename_cat ($file, $subfile->[0]);
+	    next if $tmp eq $file;
+	    # note: metadata is not carried on to the next level
+	    $count += &plugin::read ($pluginfo, $base_dir, $tmp, {}, $processor, $maxdocs);
+	}
+
+	return $count;
+    }
+
+    # wasn't an archives directory, someone else will have to process it
+    return undef;
+}
+
+1;
Index: /tags/gsdl-21/gsdl/perllib/plugins/BasPlug.pm
===================================================================
--- /tags/gsdl-21/gsdl/perllib/plugins/BasPlug.pm	(revision 699)
+++ /tags/gsdl-21/gsdl/perllib/plugins/BasPlug.pm	(revision 699)
@@ -0,0 +1,55 @@
+###########################################################################
+#
+# BasPlug.pm -- base class for all the import plugins
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package BasPlug;
+
+
+sub new {
+    my ($class) = @_;
+
+    return bless {}, $class;
+}
+
+# return 1 if this class might recurse using $pluginfo
+sub is_recursive {
+    my $self = shift (@_);
+
+    die "BasPlug::is_recursive function must be implemented in sub classes\n";
+}
+
+# return number of files processed, undef if can't process
+# Note that $base_dir might be "" and that $file might 
+# include directories
+sub read {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs) = @_;
+
+    die "BasPlug::read function must be implemented in sub classes\n";
+
+    return undef; # will never get here
+}
+
+
+1;
Index: /tags/gsdl-21/gsdl/perllib/plugins/EMAILPlug.pm
===================================================================
--- /tags/gsdl-21/gsdl/perllib/plugins/EMAILPlug.pm	(revision 699)
+++ /tags/gsdl-21/gsdl/perllib/plugins/EMAILPlug.pm	(revision 699)
@@ -0,0 +1,268 @@
+###########################################################################
+#
+# EMAILPlug.pm - a plugin for parsing email files
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+
+#
+# EMAILPlug reads an email file (*.email)
+#
+# Version 1.1   1999 Sep 20  by Gordon Paynter (gwp@cs.waikato.ac.nz)
+#                            loosely based on the original HTMLPlug code
+#
+# Document text:
+#   The document text consists of all the text occuring after the first 
+#   blank line in this document.
+#
+# Metadata:
+#   $Subject      Subject: header
+#   $To           To: header
+#   $From         From: header - this will be stored as Creator
+#   $DateText     Date: header
+#   $Date         Date: header in GSDL format (eg: 19990924)
+#   $OtherHeaders All the other headers
+#   $NewText      The unquoted text in this message
+#
+
+
+package EMAILPlug;
+
+use BasPlug;
+use sorttools;
+use util;
+
+
+# EMAILPlug is a sub-class of BasPlug.
+
+sub BEGIN {
+    @ISA = ('BasPlug');
+}
+
+
+# Create a new EMAILPlug object with which to parse a file.
+# This is done by creating a new BasPlug and usig bless to 
+# turn it into an EMAILPlug.
+
+sub new {
+    my ($class) = @_;
+    $self = new BasPlug ();
+
+    return bless $self, $class;
+}
+
+
+# Is the EMAILPlug recursive?  No.
+
+sub is_recursive {
+    my $self = shift (@_);
+
+    return 0; # this is not a recursive plugin
+}
+
+
+#
+# read
+#
+# read attempts to read a file and store its contents in a 
+# new document object.
+#
+# Returns: number of files processed or undef if can't process
+# This plugin only processes one file at a time.
+#
+# Note: $base_dir might be "" and $file might include directories,
+# but that doesn't affect EMAILPlug
+#
+
+sub read {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $metadata, $processor) = @_;
+
+    # Make sure file exists and is an email file
+    my $filename = &util::filename_cat($base_dir, $file);
+    return undef unless ($filename =~ /\.email$/i && (-e $filename));
+
+    print STDERR "EMAILPlug: processing $filename\n" if $processor->{'verbosity'};
+
+    # create a new document object
+    my $doc_obj = new doc ($file, "indexed_doc");
+    open (FILE, $filename) || die "EMAILPlug::read - can't open $filename\n";
+    my $cursection = $doc_obj->get_top_section();
+
+    # Metadata fields 
+    my $Subject = "";
+    my $To = "";
+    my $From = ""; 
+    my $DateText = "";
+    my $Date = "";
+    my $OtherHeaders = "";
+    my $NewText = "";
+    my $text = "";
+    my $line = "";
+
+    my $headers_read = 0;
+    
+    # Read and process each line in te email file.
+    # Each file consists of a set of header lines, then a blank line, 
+    # then the body of the email.
+    while (<FILE>) {
+	
+	$line = $_;
+	
+	# Remove carriage returns from the line.
+	# We will later replace single cariage returns with <BR> tags
+	# and double carriage returns with <P> tags.
+	$line =~ s/\n/ /g;
+
+        if ($headers_read) {
+	    # The headers have been read, so add this line to the body text
+	    $text .= "$line\n";
+            # If the line isn't quoted, add it to the NewText metadata
+	    if ($line =~ /^[^>|]/) {
+		$NewText .= "$line\n";
+	    }
+	    
+	} elsif ($line =~ /^\s*$/) {
+	    # An empty line signals the end of the headers.
+	    $headers_read = 1;
+
+	} else {
+	    # Read a line of header information and add it to the metadata
+	    $line .= "\n";
+	    if ($line =~ /^From:/) {
+		$line =~ s/^From:\s*//;
+		$From .= $line;
+	    } elsif ($line =~ /^To:/) {
+		$line =~ s/^To:\s*//;
+		$To .= $line;
+	    } elsif ($line =~ /^Date:/) {
+		$line =~ s/^Date:\s*//;
+		$DateText .= $line;
+		if ($Date !~ /\d+/) {
+		    # Convert the date text to internal date format
+		    my ($day, $month, $year) = $line =~ /(\d?\d)\s([A-Z][a-z][a-z])\s(\d\d\d\d)/;
+		    $Date = &sorttools::format_date($day, $month, $year);
+		}
+	    } elsif ($line =~ /^Subject:/) {
+		$line =~ s/^Subject:\s*//;
+		$Subject .= $line;
+	    } else {
+		$OtherHeaders .= $line;
+	    }
+	}	
+    }
+
+    # Add Subject metadata
+    $Subject = &text_into_html($Subject);
+    $Subject = "No Subject" unless ($Subject =~ /\w/);
+    $doc_obj->add_metadata ($cursection, "Subject", $Subject);
+    
+    # Add Sender
+    $From = &text_into_html($From);
+    $From = "No Sender" unless ($From =~ /\w/);
+    $doc_obj->add_metadata ($cursection, "Creator", $From);
+
+    # Add Recipient
+    $To = &text_into_html($To);
+    $To = "No Recipient" unless ($To =~ /\w/);
+    $doc_obj->add_metadata ($cursection, "To", $To);
+
+    # Add Date Text
+    $DateText =~ &text_into_html($Date);
+    $doc_obj->add_metadata ($cursection, "DateText", $DateText) if ($DateText =~ /\w/);
+
+    # Add Date 
+    $Date =~ &text_into_html($Date);
+    $doc_obj->add_metadata ($cursection, "Date", $Date) if ($Date =~ /\w/);
+
+    # Add Other Headers
+    $OtherHeaders = &text_into_html($OtherHeaders);
+    $doc_obj->add_metadata ($cursection, "OtherHeaders", $OtherHeaders) if ($OtherHeaders =~ /\w/);
+    
+    # Add New Text
+    $NewText = &text_into_html($NewText);
+    $doc_obj->add_metadata ($cursection, "NewText", $NewText) if ($NewText =~ /\w/);
+
+    # Add text
+    $text =~ s/<BR>\s*<BR>/<P>/g;
+    $text = &text_into_html($text);
+    $doc_obj->add_text ($cursection, $text) if ($text =~ /\w/);
+	
+    # Add the OID - that is, the big HASH value used as a unique ID
+    $doc_obj->set_OID ();
+
+    # Process the document
+    $processor->process($doc_obj);
+
+    return 1; # processed the file
+}
+
+
+1;
+
+
+
+#
+# Convert a text string into HTML.
+#
+# The HTML is going to be inserted into a GML file, so 
+# we have to be careful not to use symbols like ">",
+# which ocurs frequently in email messages (and use
+# &gt instead.
+#
+# This function also turns links and email addresses into hyperlinks,
+# and replaces carriage returns with <BR> tags (and multiple carriage
+# returns with <P> tags).
+#
+
+sub text_into_html {
+    my ($text) = @_;
+
+
+    # Convert problem charaters into HTML symbols
+    $text =~ s/&/&amp;/g;
+    $text =~ s/</&lt;/g;
+    $text =~ s/>/&gt;/g;
+    $text =~ s/\"/&quot;/g;
+
+    # convert email addresses and URLs into links
+    $text =~ s/([\w\d\.\-]+@[\w\d\.\-]+)/<a href=\"mailto:$1\">$1<\/a>/g;
+    $text =~ s/(http:\/\/[\w\d\.\-]+[\/\w\d\.\-]*)/<a href=\"$1">$1<\/a>/g;
+
+    # Clean up whitespace and convert \n charaters to <BR> or <P>
+    $text =~ s/ +/ /g;
+    $text =~ s/\s*$//; 
+    $text =~ s/^\s*//; 
+    $text =~ s/\n/\n<BR>/g;
+    $text =~ s/<BR>\s*<BR>/<P>/g;
+
+    return $text;
+}
+
+    
+
+
+
+
+
+
Index: /tags/gsdl-21/gsdl/perllib/plugins/FOXPlug.pm
===================================================================
--- /tags/gsdl-21/gsdl/perllib/plugins/FOXPlug.pm	(revision 699)
+++ /tags/gsdl-21/gsdl/perllib/plugins/FOXPlug.pm	(revision 699)
@@ -0,0 +1,257 @@
+###########################################################################
+#
+# FOXPlug.pm
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# plugin to process a Foxbase dbt file. This plugin provides the basic
+# functionality to read in the dbt and dbf files and process each record.
+# This general plugin should be overridden for a particular database to process
+# the appropriate fields in the file.
+
+package FOXPlug;
+
+use BasPlug;
+use util;
+use doc;
+use unicode;
+use cnseg;
+use gb;
+
+
+sub BEGIN {
+    @ISA = ('BasPlug');
+}
+
+sub new {
+    my ($class) = @_;
+    $self = new BasPlug ();
+
+    return bless $self, $class;
+}
+
+sub is_recursive {
+    my $self = shift (@_);
+
+    return 0; # this is not a recursive plugin
+}
+
+
+# return number of files processed, undef if can't process
+# Note that $base_dir might be "" and that $file might 
+# include directories
+sub read {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $metadata, $processor) = @_;
+    my $fullname = &util::filename_cat ($base_dir, $file);
+
+    # dbt files are processed at the same time as dbf files
+    return 0 if ($fullname =~ /\.dbt$/i);
+
+    # see if this is a foxbase database
+    return undef unless (-f $fullname && $fullname =~ /\.dbf$/i);
+
+    my ($parent_dir) = $fullname =~ /^(.*)\/[^\/]+\.dbf$/i;
+
+    # open the file
+    if (!open (FOXBASEIN, $fullname)) {
+	print STDERR "FOXPlug::read - couldn't read $fullname\n";
+	return undef;
+    }
+    
+    print STDERR "FOXPlug: processing $file\n";
+
+    # read in the database header
+    my ($temp, %dbf);
+    
+    # read in information about dbt file
+    if (read (FOXBASEIN, $temp, 32) < 32) {
+	print STDERR "FOXPlug::read - eof while reading database header";
+	close (FOXBASEIN);
+	return undef;
+    }
+    
+    # unpack the header
+    ($dbf{'hasdbt'},
+     $dbf{'modyear'}, $dbf{'modmonth'}, $dbf{'modday'},
+     $dbf{'numrecords'}, $dbf{'headerlen'}, 
+     $dbf{'recordlen'}) = unpack ("CCCCVvv", $temp);
+    
+    # process hasdbt
+    if ($dbf{'hasdbt'} == 131) {
+	$dbf{'hasdbt'} = 1;
+    } elsif ($dbf{'hasdbt'} == 3) {
+	$dbf{'hasdbt'} = 0;
+    } else {
+	print STDERR "FOXPlug:read $fullname doesn't seem to be a Foxbase file\n";
+	return undef;
+    }
+
+    # read in the field description
+    $dbf{'numfields'} = 0;
+    $dbf{'fieldinfo'} = [];
+    while (read (FOXBASEIN, $temp, 1) > 0) {
+	last if ($temp eq "\x0d");
+	last if (read (FOXBASEIN, $temp, 31, 1) < 31);
+
+	my %field = ();
+	$field{'name'} = $self->extracttext($temp, 11);
+	($field{'type'}, $field{'pos'}, $field{'len'}, $field{'dp'}) 
+	    = unpack ("x11a1VCC", $temp);
+
+	push (@{$dbf{'fieldinfo'}}, \%field);
+
+	$dbf{'numfields'} ++;
+    }
+
+    # open the dbt file if we need to
+    $dbtfullname = $fullname;
+    if ($fullname =~ /f$/) {
+	$dbtfullname =~ s/f$/t/;
+    } else {
+	$dbtfullname =~ s/F$/T/;
+    }
+    if ($dbf{'hasdbt'} && !open (DBTIN, $dbtfullname)) {
+	print STDERR "FOXPlug::read - couldn't read $dbtfullname\n";
+	close (FOXBASEIN);
+	return undef;
+    }
+
+    # read in and process each record in the database
+    my $numrecords = 0;
+    while (($numrecords < $dbf{'numrecords'}) && 
+	   (read (FOXBASEIN, $temp, $dbf{'recordlen'}) == $dbf{'recordlen'})) {
+
+	# create a new record
+	my $record = [];
+	
+	foreach $field (@{$dbf{'fieldinfo'}}) {
+	    my $fieldvalue = "";
+	    
+	    if ($field->{'type'} eq "M" && $dbf{'hasdbt'}) {
+		# a memo field, look up this field in the dbt file
+		my $seekpos = substr ($temp, $field->{'pos'}, $field->{'len'});
+
+		$seekpos =~ s/^\s*//;
+		$seekpos = 0 unless $seekpos =~ /^\d+$/;
+		
+		$seekpos = $seekpos * 512;
+
+		if ($seekpos == 0) {
+		    # there is no memo field
+
+		} elsif (seek (DBTIN, $seekpos, 0)) {
+		    while (read (DBTIN, $fieldvalue, 512, length($fieldvalue)) > 0) {
+			last if ($fieldvalue =~ /\cZ/);
+		    }
+
+		    # remove everything after the control-Z
+		    substr($fieldvalue, index($fieldvalue, "\cZ")) = "";
+
+		} else {
+		    print STDERR "\nERROR - seek (to $seekpos) failed\n";
+		}
+
+	    } else {
+		# a normal field
+		$fieldvalue = substr ($temp, $field->{'pos'}, $field->{'len'});
+	    }
+
+	    push (@$record, {%$field, 'value'=>$fieldvalue});
+	}
+
+	# process this record
+	$self->process_record ($pluginfo, $base_dir, $file, $metadata, $processor, 
+			       $numrecords, $record);
+	
+	# finished another record...
+	$numrecords++;
+    }
+
+    # close the dbt file if we need to
+    if ($dbf{'hasdbt'}) {
+	close (DBTIN);
+    }
+
+    # close the dbf file
+    close (FOXBASEIN);
+
+    # finished processing
+    return 1;
+}
+
+
+# will extract a string from some larger string, making it
+# conform to a number of constraints
+sub extracttext {
+    my $self = shift (@_);
+    my ($text, $maxlen, $offset, $stopstr) = @_;
+    $offset = 0 unless defined $offset;
+    $stopstr = "\x00" unless defined $stopstr;
+    
+    # decide where the string finishes
+    my $end = index ($text, $stopstr, $offset);
+    $end = length ($text) if $end < 0;
+    $end = $offset+$maxlen if (defined $maxlen) && ($end-$offset > $maxlen);
+    
+    return "" if ($end <= $offset);
+    return substr ($text, $offset, $end-$offset);
+}
+
+
+# process_record should be overriden for a particular type
+# of database. This default version outputs an html document
+# containing all the fields in the record as a table.
+# It also assumes that the text is in utf-8.
+sub process_record {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $metadata, $processor, $numrecords, $record) = @_;
+
+    # create a new document
+    my $doc_obj = new doc ($file, "indexed_doc");
+    my $section = $doc_obj->get_top_section();
+
+    # start of document
+    $doc_obj->add_utf8_text($section, "<table>\n");
+
+    # add each field
+    foreach $field (@$record) {
+	if (defined ($field->{'name'}) && defined ($field->{'value'})) {
+	    $doc_obj->add_utf8_text($section, "  <tr>\n");
+	    $doc_obj->add_utf8_text($section, "    <td>$field->{'name'}</td>\n");
+	    $doc_obj->add_utf8_text($section, "    <td>$field->{'value'}</td>\n");
+	    $doc_obj->add_utf8_text($section, "  </tr>\n");
+	}
+    }
+
+    # end of document
+    $doc_obj->add_utf8_text($section, "</table>\n");
+
+    # add an object id
+    $doc_obj->set_OID();
+
+    # process the document
+    $processor->process($doc_obj);
+}
+
+
+1;
Index: /tags/gsdl-21/gsdl/perllib/plugins/GBBPlug.pm
===================================================================
--- /tags/gsdl-21/gsdl/perllib/plugins/GBBPlug.pm	(revision 699)
+++ /tags/gsdl-21/gsdl/perllib/plugins/GBBPlug.pm	(revision 699)
@@ -0,0 +1,246 @@
+###########################################################################
+#
+# GBBPlug.pm -- plugin for processing gb encoded html books
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# creates multi-level document from document containing 
+# <<TOC>> level tags. Metadata for each section is taken from any
+# other tags on the same line as the <<TOC>>. e.g. <<Title>>xxxx<</Title>>
+# sets Title metadata.
+
+# Everything else between TOC tags is expected to be simple html
+# and is treated accordingly.
+
+# Imput files are expected to be GB encoded and have .gbb file extension
+
+# An jpeg image with the same filename as the .gbb file will be associated
+# as cover.jpg if it exists
+
+package GBBPlug;
+
+use BasPlug;
+use sorttools;
+use util;
+use unicode;
+use cnseg;
+use gb;
+
+
+sub BEGIN {
+    @ISA = ('BasPlug');
+}
+
+sub new {
+    my ($class) = @_;
+    $self = new BasPlug ();
+
+    return bless $self, $class;
+}
+
+sub is_recursive {
+    my $self = shift (@_);
+
+    return 0; # this is not a recursive plugin
+}
+
+
+# return number of files processed, undef if can't process
+# Note that $base_dir might be "" and that $file might 
+# include directories
+sub read {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $metadata, $processor) = @_;
+
+    my $filename = &util::filename_cat($base_dir, $file);
+    my $absdir = $filename;
+    $absdir =~ s/[^\/\\]*$//;
+
+    return undef unless ($filename =~ /\.gbb$/i && (-e $filename));
+
+    print STDERR "GBBPlug: processing $filename\n" if $processor->{'verbosity'};
+
+    # create a new document
+    my $doc_obj = new doc ($file, "indexed_doc");
+    open (FILE, $filename) || die "GBBPlug::read - ERROR: can't open $filename\n";
+
+    my $cursection = $doc_obj->get_top_section();
+
+    # copy the book cover if it exists
+    my ($bookcover) = $filename =~ /^(.*?)\.gbb$/i;
+    $doc_obj->associate_file("$bookcover.jpg", "cover.jpg", "image/jpeg") 
+	if -e "$bookcover.jpg";
+
+    my $text = "";
+    my $line = "";
+    my $title = "";
+    while (defined ($line = <FILE>)) {
+	$text .= $line;
+    }
+
+    # convert to unicode
+    $text = &unicode::unicode2utf8(&gb::gb2unicode($text));
+    # segment the Chinese words
+    $text = &cnseg::segment($text);
+
+    # we'll use the worthless alarm thingy to temporarily replace 
+    # '\n' so we'd better check it doesn't occur naturally
+    if ($text =~ /\a/) {
+	print STDERR "GBBPlug::read - 'WARNING '\a' character occurs in text!!\n";
+    }
+
+    # need to add <classifytype> to each book as we'll be using 'Hierarchy'
+    # for HB collections rather than the default ('Book')
+    $doc_obj->add_metadata ($cursection, 'classifytype', 'Hierarchy');
+
+    # remove line breaks
+    $text =~ s/\n/\a/g;
+
+    # remove any leading rubbish
+    $text =~ s/^.*?(<<TOC)/$1/i;
+    
+    my $curtoclevel = 1;
+    my $firstsection = 1;
+    while (length ($text) > 0) {
+	$text =~ s/^<<TOC(\d+)>>([^\a]*)\a(.*?)(<<TOC|\Z)/$4/i;
+	my $toclevel = $1;
+	my $metadata = $2;
+	my $sectiontext = $3;
+	
+	# close any sections below the current level and
+	# create a new section (special case for the firstsection)
+	while (($curtoclevel > $toclevel) ||
+	       (!$firstsection && $curtoclevel == $toclevel)) {
+	    $cursection = $doc_obj->get_parent_section ($cursection);
+	    $curtoclevel--;
+	}
+	if ($curtoclevel+1 < $toclevel) {
+	    print STDERR "WARNING - jump in toc levels in $htmlfile " . 
+		"from $curtoclevel to $toclevel\n";
+	}
+	while ($curtoclevel < $toclevel) {
+	    $curtoclevel++;
+	    $cursection = 
+		$doc_obj->insert_section($doc_obj->get_end_child($cursection));
+	}
+
+	# sort out metadata
+	while ($metadata =~ s/^.*?<<([^>]*)>>(.*?)<<[^>]*>>//) {
+	    my $metakey = $1;
+	    my $metavalue = $2;
+
+	    if ($metavalue ne "" && $metakey ne "") {
+		# make sure key fits in with gsdl naming scheme 
+		$metakey =~ tr/[A-Z]/[a-z]/;
+		$metakey = ucfirst ($metakey);
+		$doc_obj->add_utf8_metadata ($cursection, $metakey, $metavalue);
+	    }
+	}
+
+
+	# remove header rubbish
+	$sectiontext =~ s/^.*?<body[^>]*>//i;
+
+	# and any other unwanted tags
+	$sectiontext =~ s/<(\/p|\/html|\/body)>//g;
+
+	# fix up the image links
+	$sectiontext =~ s/(<img[^>]*?src\s*=\s*\"?)([^\">]+)(\"?[^>]*>)/
+	    &replace_image_links($absdir, $doc_obj, $1, $2, $3)/ige;
+
+	# put line breaks back in
+	$sectiontext =~ s/\a/\n/g;
+
+	# add the text
+	$doc_obj->add_utf8_text($cursection, $sectiontext);
+
+	$firstsection = 0;
+    }
+
+    # add OID
+    $doc_obj->set_OID ();
+
+    # process the document
+    $processor->process($doc_obj);
+
+    return 1; # processed the file
+}
+
+sub replace_image_links {
+
+    my ($dir, $doc_obj, $front, $link, $back) = @_;
+
+    my ($filename, $error);
+    my $foundimage = 0;
+    
+    $link =~ s/\/\///;
+    my ($imagetype) = $link =~ /([^\.]*)$/;
+    $imagetype =~ tr/[A-Z]/[a-z]/;
+    if ($imagetype eq "jpg") {$imagetype = "jpeg";}
+    if ($imagetype !~ /^(jpg|gif|png)$/) {
+	print STDERR "GBHPlug: Warning - unknown image type ($imagetype)\n";
+    }
+    my ($imagefile) = $link =~ /([^\/]*)$/;
+    my ($imagepath) = $link =~ /^[^\/]*(.*)$/;
+
+    if (defined $imagepath && $imagepath =~ /\w/) {
+	# relative link
+	$filename = &util::filename_cat ($dir, $imagepath);
+	if (-e $filename) {
+	    $doc_obj->associate_file ($filename, $imagefile, "image/$imagetype");
+	    $foundimage = 1;
+	} else {
+	    $error = "GBHPlug: Warning - couldn't find image file $imagefile in either $filename or";
+	}
+    }
+
+    if (!$foundimage) {
+	$filename = &util::filename_cat ($dir, $imagefile);
+	if (-e $filename) {
+	    $doc_obj->associate_file ($filename, $imagefile, "image/$imagetype");    
+	    $foundimage = 1;
+	} elsif (defined $error) {
+	    print STDERR "$error $filename\n";
+	} else {
+	    print STDERR "GBHPlug: Warning - couldn't find image file $imagefile in $filename\n";
+	}
+    }
+
+    if ($foundimage) {
+	return "${front}_httpcollection_/archives/_thisOID_/${imagefile}${back}";
+    } else {
+	return "";
+    }
+}
+
+1;
+
+
+
+
+
+
+
+
+
+
+
Index: /tags/gsdl-21/gsdl/perllib/plugins/GBHPlug.pm
===================================================================
--- /tags/gsdl-21/gsdl/perllib/plugins/GBHPlug.pm	(revision 699)
+++ /tags/gsdl-21/gsdl/perllib/plugins/GBHPlug.pm	(revision 699)
@@ -0,0 +1,230 @@
+###########################################################################
+#
+# GBHPlug.pm -- plugin for processing gb encoded html
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# creates simple single-level document from .htm or .html files 
+# (case-insensitive match on filenames). Adds Title metadata 
+# taken from <title> tags if found otherwise first 100 characters
+# outside of tags.
+# will also attempt to include images which it will search for in
+# the same directory as the document itself (it will also search 
+# directories relative to that directory).
+
+# this plugin currently does nothing with href links so relative links 
+# may become broken.
+
+
+package GBHPlug;
+
+use BasPlug;
+use sorttools;
+use util;
+use unicode;
+use cnseg;
+use gb;
+
+
+sub BEGIN {
+    @ISA = ('BasPlug');
+}
+
+sub new {
+    my ($class) = @_;
+    $self = new BasPlug ();
+
+    return bless $self, $class;
+}
+
+sub is_recursive {
+    my $self = shift (@_);
+
+    return 0; # this is not a recursive plugin
+}
+
+
+# return number of files processed, undef if can't process
+# Note that $base_dir might be "" and that $file might 
+# include directories
+sub read {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $metadata, $processor) = @_;
+
+    my $filename = &util::filename_cat($base_dir, $file);
+    my $absdir = $filename;
+    $absdir =~ s/[^\/\\]*$//;
+
+    return undef unless ($filename =~ /\.(html?(\.gz)?)$/i && (-e $filename));
+
+    my $gz = 0;
+    if (defined $2) {
+	$gz = $2;
+	$gz = 1 if ($gz =~ /\.gz/i);
+    }
+
+    print STDERR "GBHPlug: processing $filename\n" if $processor->{'verbosity'};
+
+    # create a new document
+    my $doc_obj = new doc ($file, "indexed_doc");
+
+    if ($gz) {
+	open (FILE, "zcat $filename |") || die "GBHPlug::read - ERROR: zcat can't open $filename\n";
+    } else {
+	open (FILE, $filename) || die "GBHPlug::read - ERROR: can't open $filename\n";
+    }
+    my $cursection = $doc_obj->get_top_section();
+
+    my $text = "";
+    my $line = "";
+    my $title = "";
+    while (defined ($line = <FILE>)) {
+	$text .= $line;
+    }
+
+    # convert to unicode
+    $text = &unicode::unicode2utf8(&gb::gb2unicode($text));
+    # segment the Chinese words
+    $text = &cnseg::segment($text);
+
+    # we'll use the worthless alarm thingy to temporarily replace 
+    # '\n' so we'd better check it doesn't occur naturally
+    if ($text =~ /\a/) {
+	print STDERR "GBHPlug::read - 'WARNING '\a' character occurs in text!!\n";
+    }
+
+    # remove line breaks
+    $text =~ s/\n/\a/g;
+
+    # see if there's a <title> tag
+    if ($text =~ /<title[^>]*>([^<]*)<\/title[^>]*>/i) {
+	if (defined $1) {
+	    $title = $1;
+	}
+    }
+    # if no title use first 20 characters
+    if ($title eq "") {
+	my $tmptext = $text;
+	$tmptext =~ s/<[^>]*>//g;
+	$title = substr ($tmptext, 0, 20);
+    }
+
+    if ($title ne "") {
+	$doc_obj->add_utf8_metadata ($cursection, "Title", $title);
+    }
+
+    # remove header rubbish
+    $text =~ s/^.*?<body[^>]*>//i;
+
+    # and any other unwanted tags
+    $text =~ s/<(\/p|\/html|\/body)>//g;
+
+    # fix up the image links
+    $text =~ s/(<img[^>]*?src\s*=\s*\"?)([^\">]+)(\"?[^>]*>)/
+	&replace_image_links($absdir, $doc_obj, $1, $2, $3)/ige;
+
+    # put line breaks back in
+    $text =~ s/\a/\n/g;
+
+    # add the text
+    $doc_obj->add_utf8_text($cursection, $text);
+
+    # assume that any metadata passed to this plugin is already utf8
+    foreach $field (keys(%$metadata)) {
+	# $metadata->{$field} may be an array reference
+	if (ref ($metadata->{$field}) eq "ARRAY") {
+	    map { 
+		$doc_obj->add_utf8_metadata ($cursection, $field, $_); 
+	    } @{$metadata->{$field}};
+	} else {
+	    $doc_obj->add_utf8_metadata ($cursection, $field, $metadata->{$field}); 
+	}
+    }
+
+    # add OID
+    $doc_obj->set_OID ();
+
+    # process the document
+    $processor->process($doc_obj);
+
+    return 1; # processed the file
+}
+
+sub replace_image_links {
+
+    my ($dir, $doc_obj, $front, $link, $back) = @_;
+
+    my ($filename, $error);
+    my $foundimage = 0;
+    
+    $link =~ s/\/\///;
+    my ($imagetype) = $link =~ /([^\.]*)$/;
+    $imagetype =~ tr/[A-Z]/[a-z]/;
+    if ($imagetype eq "jpg") {$imagetype = "jpeg";}
+    if ($imagetype !~ /^(jpg|gif|png)$/) {
+	print STDERR "GBHPlug: Warning - unknown image type ($imagetype)\n";
+    }
+    my ($imagefile) = $link =~ /([^\/]*)$/;
+    my ($imagepath) = $link =~ /^[^\/]*(.*)$/;
+
+    if (defined $imagepath && $imagepath =~ /\w/) {
+	# relative link
+	$filename = &util::filename_cat ($dir, $imagepath);
+	if (-e $filename) {
+	    $doc_obj->associate_file ($filename, $imagefile, "image/$imagetype");
+	    $foundimage = 1;
+	} else {
+	    $error = "GBHPlug: Warning - couldn't find image file $imagefile in either $filename or";
+	}
+    }
+
+    if (!$foundimage) {
+	$filename = &util::filename_cat ($dir, $imagefile);
+	if (-e $filename) {
+	    $doc_obj->associate_file ($filename, $imagefile, "image/$imagetype");    
+	    $foundimage = 1;
+	} elsif (defined $error) {
+	    print STDERR "$error $filename\n";
+	} else {
+	    print STDERR "GBHPlug: Warning - couldn't find image file $imagefile in $filename\n";
+	}
+    }
+
+    if ($foundimage) {
+	return "${front}_httpcollection_/archives/_thisOID_/${imagefile}${back}";
+    } else {
+	return "";
+    }
+}
+
+1;
+
+
+
+
+
+
+
+
+
+
+
Index: /tags/gsdl-21/gsdl/perllib/plugins/GBTPlug.pm
===================================================================
--- /tags/gsdl-21/gsdl/perllib/plugins/GBTPlug.pm	(revision 699)
+++ /tags/gsdl-21/gsdl/perllib/plugins/GBTPlug.pm	(revision 699)
@@ -0,0 +1,137 @@
+###########################################################################
+#
+# GBTPlug.pm -- plugin for processing GB encoded text
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# creates simple single-level document from .txt or .text files 
+# (case-insensitive match on filenames). Adds Title metadata 
+# of filename.
+
+package GBTPlug;
+
+use BasPlug;
+use sorttools;
+use unicode;
+use cnseg;
+use gb;
+
+
+sub BEGIN {
+    @ISA = ('BasPlug');
+}
+
+sub new {
+    my ($class) = @_;
+    $self = new BasPlug ();
+
+    return bless $self, $class;
+}
+
+sub is_recursive {
+    my $self = shift (@_);
+
+    return 0; # this is not a recursive plugin
+}
+
+
+# return number of files processed, undef if can't process
+# Note that $base_dir might be "" and that $file might 
+# include directories
+sub read {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $metadata, $processor) = @_;
+
+    my $filename = &util::filename_cat($base_dir, $file);
+
+    return undef unless ($filename =~ /\.(te?xt(\.gz)?)$/i && (-e $filename));
+
+    my $gz = 0;
+    if (defined $2) {
+	$gz = $2;
+	$gz = 1 if ($gz =~ /\.gz/i);
+    }
+
+    print STDERR "GBTPlug: processing $filename\n" if $processor->{'verbosity'};
+
+    # create a new document
+    my $doc_obj = new doc ($file, "indexed_doc");
+
+    if ($gz) {
+	open (FILE, "zcat $filename |") || die "TEXTPlug::read - zcat can't open $filename\n";
+    } else {
+	open (FILE, $filename) || die "TEXTPlug::read - can't open $filename\n";
+    }
+    my $cursection = $doc_obj->get_top_section();
+
+    my $text = "";
+    my $line = "";
+    while (defined ($line = <FILE>)) {
+	$text .= $line;
+    }
+    
+    # convert to unicode
+    $text = &unicode::unicode2utf8(&gb::gb2unicode($text));
+
+    # segment the Chinese words
+    $text = &cnseg::segment($text);
+
+    $doc_obj->add_utf8_text($cursection, "<pre>$text</pre>");
+
+    # add Title metadata (filename);
+    my $title = $file;
+    $title =~ s/^[\/\\]+//;
+    $doc_obj->add_metadata($cursection, "Title", $title);
+
+    # assume that any metadata passed to this plugin is already utf8
+    foreach $field (keys(%$metadata)) {
+	# $metadata->{$field} may be an array reference
+	if (ref ($metadata->{$field}) eq "ARRAY") {
+	    map { 
+		$doc_obj->add_utf8_metadata ($cursection, $field, $_); 
+	    } @{$metadata->{$field}};
+	} else {
+	    $doc_obj->add_utf8_metadata ($cursection, $field, $metadata->{$field}); 
+	}
+    }
+
+    # add OID
+    $doc_obj->set_OID ();
+
+    # process the document
+    $processor->process($doc_obj);
+
+    return 1; # processed the file
+}
+
+1;
+
+
+
+
+
+
+
+
+
+
+
Index: /tags/gsdl-21/gsdl/perllib/plugins/GMLPlug.pm
===================================================================
--- /tags/gsdl-21/gsdl/perllib/plugins/GMLPlug.pm	(revision 699)
+++ /tags/gsdl-21/gsdl/perllib/plugins/GMLPlug.pm	(revision 699)
@@ -0,0 +1,184 @@
+###########################################################################
+#
+# GMLPlug.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# plugin which processes a GML format document
+
+package GMLPlug;
+
+use html;
+use BasPlug;
+use util;
+use doc;
+
+
+sub BEGIN {
+    @ISA = ('BasPlug');
+}
+
+sub new {
+    my ($class) = @_;
+    $self = new BasPlug ();
+
+    return bless $self, $class;
+}
+
+sub is_recursive {
+    my $self = shift (@_);
+
+    return 0; # this is not a recursive plugin
+}
+
+sub _unescape_text {
+    my ($text) = @_;
+
+    # special characters in the gml encoding
+    $text =~ s/&lt;/</g;
+    $text =~ s/&gt;/>/g;
+    $text =~ s/&quot;/\"/g;
+    $text =~ s/&amp;/&/g; # this has to be last...
+
+    return $text;
+}
+
+# return number of files processed, undef if can't process
+# Note that $base_dir might be "" and that $file might 
+# include directories
+sub read {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $metadata, $processor) = @_;
+    my $fullname = &util::filename_cat ($base_dir, $file);
+
+    # see if this is a gml book
+    return undef unless (-f $fullname && $fullname =~ /\.gml(\.gz)?$/i);
+
+    my ($parent_dir, $gz) = $fullname =~ /^(.*)\/[^\/]+.gml(\.gz)?$/i;
+
+    if (defined $gz && $gz =~ /\.gz/i) {
+	$gz = 1;
+    } else {
+	$gz = 0;
+    }
+
+    # create a new document
+    my $doc_obj = new doc ();
+    my $section = $doc_obj->get_top_section();
+
+    print STDERR "GMLPlug: processing $file\n";
+
+    # read in the document
+    my $gml = "";
+    my $line = "";
+
+    if ($gz) {
+	if (!open (INFILE, "zcat $fullname |")) {
+	    print STDERR "GMLPlug::read - zcat couldn't read $fullname\n";
+	    return undef;
+	}
+    } else {
+	if (!open (INFILE, $fullname)) {
+	    print STDERR "GMLPlug::read - couldn't read $fullname\n";
+	    return undef;
+	}
+    }
+
+    while (defined ($line = <INFILE>)) {
+	$gml .= $line;
+    }
+    close (INFILE);
+
+    # process the document
+    my $firstsection = 1;
+    while ($gml =~ /\S/) {
+	if ($gml =~ s/^\s*<gsdlsection([^>]*)>(.*?)(<\/?gsdlsection)/$3/is) {
+	    my $tags = $1;
+	    $tags = "" unless defined $tags;
+	    my $text = &_unescape_text($2);
+
+	    # create the section (unless this is the first section)
+	    if (!$firstsection) {
+		$tags =~ s/gsdlnum\s*=\s*\"?(\d+)\"?//i;
+		if (defined $1) {
+		    $section .= ".$1";
+		    $doc_obj->create_named_section($section);
+		} else {
+		    $section = $doc_obj->insert_section($doc_obj->get_end_child($section));
+		}
+	    }
+	    $firstsection = 0;
+
+	    # add the tags
+	    while ((defined $tags) && ($tags =~ s/^\s*(\w+)\s*=\s*\"([^\"]*)\"//)) {
+		$doc_obj->add_utf8_metadata($section, $1 , &_unescape_text($2)) 
+		    if (defined $1 and defined $2);
+	    }
+
+	    # add the text
+	    $doc_obj->add_utf8_text($section, $text) 
+		if ((defined $text) && ($text ne ""));
+
+	} elsif ($gml =~ s/^\s*<\/gsdlsection>//) {
+	    $section = $doc_obj->get_parent_section ($section);
+
+	} else {
+	    print STDERR "GMLPlug::read - error in file $fullname\n";
+	    print STDERR "text: \"$gml\"\n";
+	    last;
+	}
+    }
+
+    # add the associated files
+    $assoc_files = $doc_obj->get_metadata($doc_obj->get_top_section(), "gsdlassocfile");
+    my ($assoc_file_info);
+    foreach $assoc_file_info (@$assoc_files) {
+	my ($assoc_file, $mime_type) = split (":", $assoc_file_info);
+	$doc_obj->associate_file(&util::filename_cat($parent_dir, $assoc_file),
+				 $assoc_file, $mime_type);
+				 
+    }
+    $doc_obj->delete_metadata($doc_obj->get_top_section(), "gsdlassocfile");
+
+    # add metadata
+    foreach $field (keys(%$metadata)) {
+	# $metadata->{$field} may be an array reference
+	if (ref ($metadata->{$field}) eq "ARRAY") {
+	    map { 
+		$doc_obj->add_metadata ($doc_obj->get_top_section(), $field, $_); 
+	    } @{$metadata->{$field}};
+	} else {
+	    $doc_obj->add_metadata ($doc_obj->get_top_section(), $field, $metadata->{$field}); 
+	}
+    }
+
+
+    # assume the document has an OID
+
+    # process the document
+    $processor->process($doc_obj, $file);
+        
+    return 1; # processed the file
+}
+
+
+1;
Index: /tags/gsdl-21/gsdl/perllib/plugins/HBPlug.pm
===================================================================
--- /tags/gsdl-21/gsdl/perllib/plugins/HBPlug.pm	(revision 699)
+++ /tags/gsdl-21/gsdl/perllib/plugins/HBPlug.pm	(revision 699)
@@ -0,0 +1,278 @@
+###########################################################################
+#
+# HBPlug.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# plugin which process an HTML book directory
+
+package HBPlug;
+
+use plugin;
+use html;
+use BasPlug;
+use util;
+use lang;
+use doc;
+use cfgread;
+
+
+sub BEGIN {
+    @ISA = ('BasPlug');
+}
+
+sub new {
+    my ($class) = @_;
+    $self = new BasPlug ();
+
+    return bless $self, $class;
+}
+
+sub is_recursive {
+    my $self = shift (@_);
+
+    return 0; # this is not a recursive plugin
+}
+
+sub HB_read_html_file {
+    my $self = shift (@_);
+    my ($htmlfile, $text) = @_;
+
+    # load in the file
+    if (!open (FILE, $htmlfile)) {
+	print STDERR "ERROR - could not open $htmlfile\n";
+	return;
+    }
+
+    my $foundbody = 0;
+    $self->HB_gettext (\$foundbody, $text, FILE);
+    close FILE;
+
+    # just in case there was no <body> tag
+    if (!$foundbody) {
+	$foundbody = 1;
+	open (FILE, $htmlfile) || return;
+	$self->HB_gettext (\$foundbody, $text, FILE);	
+	close FILE;
+    }
+}
+
+sub HB_gettext {
+    my $self = shift (@_);
+    my ($foundbody, $text, $handle) = @_;
+
+    my $line = "";
+    while (defined ($line = <$handle>)) {
+	# look for body tag
+	if (!$$foundbody) {
+	    if ($line =~ s/^.*<body[^>]*>//i) {
+		$$foundbody = 1;
+	    } else {
+		next;
+	    }
+	}
+	
+	# check for symbol fonts
+	if ($line =~ /<font [^>]*?face\s*=\s*\"?(\w+)\"?/i) {
+	    my $font = $1;
+	    print STDERR "HBPlug::HB_gettext - warning removed font $font\n" 
+		if ($font !~ /^arial$/i);
+	}
+
+	$line =~ s/<\/p>//ig;   # remove </p> tags
+	$line =~ s/<\/?(body|html|font)\b[^>]*>//ig; # remove any unwanted tags
+
+	# convert any alphanumeric character entities to their extended
+	# ascii equivalent for indexing purposes
+	&html::convertcharentities ($line);
+
+	$$text .= $line;
+    }
+    $$text =~ s/\s+/ /g; # remove \n's
+}
+
+sub HB_clean_section {
+    my $self = shift (@_);
+    my ($section) = @_;
+
+    # remove tags without a starting tag from the section
+    my ($tag, $tagstart);
+    while ($section =~ /<\/([^>]{1,10})>/) {
+	$tag = $1;
+	$tagstart = index($section, "<$tag");
+	last if (($tagstart >= 0) && ($tagstart < index($section, "<\/$tag")));
+	$section =~ s/<\/$tag>//;
+    }
+    
+    # remove extra paragraph tags
+    while ($section =~ s/<p\b[^>]*>\s*<p\b/<p/ig) {}
+    
+    # remove extra stuff at the end of the section
+    while ($section =~ s/(<u>|<i>|<b>|<p\b[^>]*>|&nbsp;|\s)$//i) {}
+    
+    # add a newline at the beginning of each paragraph
+    $section =~ s/(.)\s*<p\b/$1\n\n<p/gi;
+    
+    # add a newline every 80 characters at a word boundary
+    # Note: this regular expression puts a line feed before
+    # the last word in each section, even when it is not
+    # needed.
+    $section =~ s/(.{1,80})\s/$1\n/g;
+    
+    # fix up the image links
+    $section =~ s/<img[^>]*?src=\"?([^\">]+)\"?[^>]*>/
+	<center><img src=\"_httpcollection_\/archives\/_thisOID_\/$1\"><\/center><br>/ig;
+    $section =~ s/&lt;&lt;I&gt;&gt;\s*([^\.]+\.(png|jpg|gif))/
+	<center><img src=\"_httpcollection_\/archives\/_thisOID_\/$1\"><\/center><br>/ig;
+
+    return $section;
+}
+
+
+sub shorten {
+    my $self = shift (@_);
+    my ($text) = @_;
+
+    return "\"$text\"" if (length($text) < 100);
+
+    return "\"" . substr ($text, 0, 50) . "\" ... \"" . 
+	substr ($text, length($text)-50) . "\"";
+}
+
+
+# return number of files processed, undef if can't process
+# Note that $base_dir might be "" and that $file might 
+# include directories
+sub read {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $metadata, $processor) = @_;
+
+    # get the html filename and see if this is an HTML Book...
+    my $jobnumber = $file;
+    if ($file =~ /[\\\/]/) {
+	($jobnumber) = $file =~ /[\\\/]([^\\\/]+)$/;
+    }
+    return undef unless defined $jobnumber;
+    my $htmlfile = &util::filename_cat($base_dir, $file, "$jobnumber.htm");
+    return undef unless -e $htmlfile;
+
+    print STDERR "HBPlug: processing $file\n";
+
+    # read in the file and do basic html cleaning (removing header etc)
+    my $html = "";
+    $self->HB_read_html_file ($htmlfile, \$html);
+
+    # create a new document
+    my $doc_obj = new doc ($file, "indexed_doc");
+
+    # copy the book cover if it exists
+    my $bookcover = &util::filename_cat($base_dir, $file, "$jobnumber.jpg");
+    $doc_obj->associate_file($bookcover, "cover.jpg", "image/jpeg") 
+	if -e $bookcover;
+
+    my $cursection = $doc_obj->get_top_section();
+
+    # add metadata for top level of document
+    foreach $field (keys(%$metadata)) {
+	# $metadata->{$field} may be an array reference
+	if (ref ($metadata->{$field}) eq "ARRAY") {
+	    map { 
+		$doc_obj->add_metadata ($cursection, $field, $_); 
+	    } @{$metadata->{$field}};
+	} else {
+	    $doc_obj->add_metadata ($cursection, $field, $metadata->{$field}); 
+	}
+    }
+    # need to add <classifytype> to each book as we'll be using 'Hierarchy'
+    # for HB collections rather than the default ('Book')
+    $doc_obj->add_metadata ($cursection, 'classifytype', 'Hierarchy');
+
+
+    # process the file one section at a time
+    my $curtoclevel = 1;
+    my $firstsection = 1;
+    while (length ($html) > 0) {
+	if ($html =~ s/^.*?(?:<p\b[^>]*>)?((<b>|<i>|<u>|\s)*)&lt;&lt;TOC(\d+)&gt;&gt;\s*(.*?)<p\b/<p/i) {
+	    my $toclevel = $3;
+	    my $title = $4;
+	    my $sectiontext = "";
+	    if ($html =~ s/^(.*?)((?:<p\b[^>]*>)?((<b>|<i>|<u>|\s)*)&lt;&lt;TOC\d+&gt;&gt;)/$2/i) {
+		$sectiontext = $1;
+	    } else {
+		$sectiontext = $html;
+		$html = "";
+	    }
+
+	    # remove tags and extra spaces from the title
+	    $title =~ s/<\/?[^>]+>//g;
+	    $title =~ s/^\s+|\s+$//g;
+
+	    # close any sections below the current level and
+	    # create a new section (special case for the firstsection)
+	    while (($curtoclevel > $toclevel) ||
+		   (!$firstsection && $curtoclevel == $toclevel)) {
+		$cursection = $doc_obj->get_parent_section ($cursection);
+		$curtoclevel--;
+	    }
+	    if ($curtoclevel+1 < $toclevel) {
+		print STDERR "WARNING - jump in toc levels in $htmlfile " . 
+		    "from $curtoclevel to $toclevel\n";
+	    }
+	    while ($curtoclevel < $toclevel) {
+		$curtoclevel++;
+		$cursection = 
+		    $doc_obj->insert_section($doc_obj->get_end_child($cursection));
+	    }
+
+	    # add the metadata to this section
+	    $doc_obj->add_metadata ($cursection, "Title", $title);
+
+	    # clean up the section html
+	    $sectiontext = $self->HB_clean_section($sectiontext);
+
+	    # associate any files
+	    map { $doc_obj->associate_file(&util::filename_cat ($base_dir, $file, $1), $1)
+		      if /_httpcollection_\/archives\/_thisOID_\/([^\"]+)\"/; 0; }
+	         split (/(_httpcollection_\/archives\/_thisOID_\/[^\"]+\")/, $sectiontext);
+
+	    # add the text for this section
+	    $doc_obj->add_text ($cursection, $sectiontext);
+	    
+	} else {
+	    print STDERR "WARNING - leftover text\n" , $self->shorten($html), 
+	    "\nin $htmlfile\n";
+	    last;
+	}
+	$firstsection = 0;
+    }
+
+    # add a OID
+    $doc_obj->set_OID ();
+
+    # process the document
+    $processor->process($doc_obj, &util::filename_cat($file, "$jobnumber.htm"));
+        
+    return 1; # processed the file
+}
+
+
+1;
Index: /tags/gsdl-21/gsdl/perllib/plugins/HTMLPlug.pm
===================================================================
--- /tags/gsdl-21/gsdl/perllib/plugins/HTMLPlug.pm	(revision 699)
+++ /tags/gsdl-21/gsdl/perllib/plugins/HTMLPlug.pm	(revision 699)
@@ -0,0 +1,221 @@
+###########################################################################
+#
+# HTMLPlug.pm -- basic html plugin
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# creates simple single-level document from .htm or .html files 
+# (case-insensitive match on filenames). Adds Title metadata 
+# taken from <title> tags if found otherwise first 100 characters
+# outside of tags.
+# will also attempt to include images which it will search for in
+# the same directory as the document itself (it will also search 
+# directories relative to that directory).
+
+# this plugin currently does nothing with href links so relative links 
+# may become broken.
+
+
+package HTMLPlug;
+
+use BasPlug;
+use sorttools;
+use util;
+
+sub BEGIN {
+    @ISA = ('BasPlug');
+}
+
+sub new {
+    my ($class) = @_;
+    $self = new BasPlug ();
+
+    return bless $self, $class;
+}
+
+sub is_recursive {
+    my $self = shift (@_);
+
+    return 0; # this is not a recursive plugin
+}
+
+
+# return number of files processed, undef if can't process
+# Note that $base_dir might be "" and that $file might 
+# include directories
+sub read {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $metadata, $processor) = @_;
+
+    my $filename = &util::filename_cat($base_dir, $file);
+    my $absdir = $filename;
+    $absdir =~ s/[^\/\\]*$//;
+
+    return undef unless ($filename =~ /\.(html?(\.gz)?)$/i && (-e $filename));
+
+    my $gz = 0;
+    if (defined $2) {
+	$gz = $2;
+	$gz = 1 if ($gz =~ /\.gz/i);
+    }
+
+    print STDERR "HTMLPlug: processing $filename\n" if $processor->{'verbosity'};
+
+    # create a new document
+    my $doc_obj = new doc ($file, "indexed_doc");
+
+    if ($gz) {
+	open (FILE, "zcat $filename |") || die "HTMLPlug::read - zcat can't open $filename\n";
+    } else {
+	open (FILE, $filename) || die "HTMLPlug::read - can't open $filename\n";
+    }
+    my $cursection = $doc_obj->get_top_section();
+
+    my $text = "";
+    my $line = "";
+    my $title = "";
+    while (defined ($line = <FILE>)) {
+	$text .= $line;
+    }
+
+    # we'll use the worthless alarm thingy to temporarily replace 
+    # '\n' so we'd better check it doesn't occur naturally
+    if ($text =~ /\a/) {
+	print STDERR "HTMLPlug::read - 'WARNING '\a' character occurs in text!!\n";
+    }
+
+    # remove line breaks
+    $text =~ s/\n/\a/g;
+
+    # see if there's a <title> tag
+    my $foundtitle = 0;
+    if ($text =~ /<title[^>]*>([^<]*)<\/title[^>]*>/i) {
+	if (defined $1) {
+	    my $title = $1;
+	    if ($title =~ /\w/) {
+		$doc_obj->add_metadata ($cursection, "Title", $title);
+		$foundtitle = 1;
+	    }
+	}
+    }
+    # if no title use first 100 characters
+    if (!$foundtitle) {
+	my $tmptext = $text;
+	$tmptext =~ s/<[^>]*>//g;
+	my $title = substr ($tmptext, 0, 100);
+	$doc_obj->add_metadata ($cursection, "Title", $title);
+    }
+    
+    # remove header rubbish
+    $text =~ s/^.*?<body[^>]*>//i;
+
+    # and any other unwanted tags
+    $text =~ s/<(\/p|\/html|\/body)>//g;
+
+    # fix up the image links
+    $text =~ s/(<img[^>]*?src\s*=\s*\"?)([^\">]+)(\"?[^>]*>)/
+	&replace_image_links($absdir, $doc_obj, $1, $2, $3)/ige;
+
+    # put line breaks back in
+    $text =~ s/\a/\n/g;
+
+    $doc_obj->add_text ($cursection, $text);
+
+    foreach $field (keys(%$metadata)) {
+	# $metadata->{$field} may be an array reference
+	if (ref ($metadata->{$field}) eq "ARRAY") {
+	    map { 
+		$doc_obj->add_metadata ($cursection, $field, $_); 
+	    } @{$metadata->{$field}};
+	} else {
+	    $doc_obj->add_metadata ($cursection, $field, $metadata->{$field}); 
+	}
+    }
+
+    # add OID
+    $doc_obj->set_OID ();
+
+    # process the document
+    $processor->process($doc_obj);
+
+    return 1; # processed the file
+}
+
+sub replace_image_links {
+
+    my ($dir, $doc_obj, $front, $link, $back) = @_;
+
+    my ($filename, $error);
+    my $foundimage = 0;
+    
+    $link =~ s/\/\///;
+    my ($imagetype) = $link =~ /([^\.]*)$/;
+    $imagetype =~ tr/[A-Z]/[a-z]/;
+    if ($imagetype eq "jpg") {$imagetype = "jpeg";}
+    if ($imagetype !~ /^(jpg|gif|png)$/) {
+	print STDERR "HTMLPlug: Warning - unknown image type ($imagetype)\n";
+    }
+    my ($imagefile) = $link =~ /([^\/]*)$/;
+    my ($imagepath) = $link =~ /^[^\/]*(.*)$/;
+
+    if (defined $imagepath && $imagepath =~ /\w/) {
+	# relative link
+	$filename = &util::filename_cat ($dir, $imagepath);
+	if (-e $filename) {
+	    $doc_obj->associate_file ($filename, $imagefile, "image/$imagetype");
+	    $foundimage = 1;
+	} else {
+	    $error = "HTMLPlug: Warning - couldn't find image file $imagefile in either $filename or";
+	}
+    }
+
+    if (!$foundimage) {
+	$filename = &util::filename_cat ($dir, $imagefile);
+	if (-e $filename) {
+	    $doc_obj->associate_file ($filename, $imagefile, "image/$imagetype");    
+	    $foundimage = 1;
+	} elsif (defined $error) {
+	    print STDERR "$error $filename\n";
+	} else {
+	    print STDERR "HTMLPlug: Warning - couldn't find image file $imagefile in $filename\n";
+	}
+    }
+
+    if ($foundimage) {
+	return "${front}_httpcollection_/archives/_thisOID_/${imagefile}${back}";
+    } else {
+	return "";
+    }
+}
+
+1;
+
+
+
+
+
+
+
+
+
+
+
Index: /tags/gsdl-21/gsdl/perllib/plugins/IndexPlug.pm
===================================================================
--- /tags/gsdl-21/gsdl/perllib/plugins/IndexPlug.pm	(revision 699)
+++ /tags/gsdl-21/gsdl/perllib/plugins/IndexPlug.pm	(revision 699)
@@ -0,0 +1,138 @@
+###########################################################################
+#
+# IndexPlug.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# This recursive plugin processes an index.txt file. 
+# The index.txt file should contain the list of files to be
+# included in the collection followed by any extra metadata to
+# be associated with each file.
+
+# The index.txt file should be formatted as follows:
+# The first line may be a key (beginning with key:)
+# to name the metadata fields
+# (e.g. key: Subject Organization Date) 
+# The following lines will contain a filename followed 
+# by the value that metadata entry is to be set to.
+# (e.g. 'irma/iw097e 3.2 unesco 1993' will associate the
+# metadata Subject=3.2, Organization=unesco, and Date=1993
+# with the file irma/iw097e if the above key line was used)
+
+# Note that if any of the metadata fields use the Hierarchy
+# classifier plugin then the value they're set to should
+# correspond to the first field (the descriptor) in the
+# appropriate classification file.
+
+# Metadata values may be named separately using a tag 
+# (e.g. <Subject>3.2) and this will override any name 
+# given to them by the key line.
+# If there's no key line any unnamed metadata value will be
+# named 'Subject'.
+
+package IndexPlug;
+
+use plugin;
+use BasPlug;
+use lang;
+use doc;
+use util;
+use cfgread;
+
+sub BEGIN {
+    @ISA = ('BasPlug');
+}
+
+sub new {
+    my ($class) = @_;
+    $self = new BasPlug ();
+
+    return bless $self, $class;
+}
+
+# return 1 if this class might recurse using $pluginfo
+sub is_recursive {
+    my $self = shift (@_);
+    
+    return 1;
+}
+
+
+# return number of files processed, undef if can't process
+# Note that $base_dir might be "" and that $file might 
+# include directories
+sub read {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs) = @_;
+
+    my $indexfile = &util::filename_cat($base_dir, $file, "index.txt");
+    if (!-f $indexfile) {
+	# not a directory containing an index file
+	return undef;
+    }
+
+    # found an index.txt file
+    print STDERR "IndexPlug: processing $indexfile\n";
+
+    # read in the index.txt
+    my $list = &cfgread::read_cfg_file ($indexfile, undef, '^[^#]\w');
+    my @fields = ();
+    # see if there's a 'key:' line
+    if (defined $list->{'key:'}) {
+	@fields = @{$list->{'key:'}};
+    }
+
+    # process each document
+    my $count = 0;
+    foreach $docfile (keys (%$list)) {
+	last if (defined $maxdocs && $maxdocs =~ /\d/ && $count >= $maxdocs);
+	$metadata = {}; # at present we can do this as metadata
+	                # will always be empty when it arrives
+	                # at this plugin - this might cause 
+	                # problems if things change though
+
+	# note that $list->{$docfile} is an array reference
+	if ($docfile !~ /key:/i) {
+	    for ($i = 0; $i < scalar (@{$list->{$docfile}}); $i ++) {
+		if ($list->{$docfile}->[$i] =~ /^<([^>]+)>(.+)$/) {
+		    unless (defined ($metadata->{$1})) {
+			$metadata->{$1} = [];
+		    } 
+		    push (@{$metadata->{$1}}, $2);
+		} elsif (scalar @fields >= $i) {
+		    unless (defined ($metadata->{$fields[$i]})) {
+			$metadata->{$fields[$i]} = [];
+		    } 
+		    push (@{$metadata->{$fields[$i]}}, $list->{$docfile}->[$i]);
+		} else {
+		    $metadata->{'Subject'} = $list->{$docfile};
+		}
+	    }
+	    $count += &plugin::read ($pluginfo, $base_dir, $docfile, $metadata, $processor, $maxdocs);
+	}
+    }
+
+    return $count; # was processed
+}
+
+
+1;
Index: /tags/gsdl-21/gsdl/perllib/plugins/RecPlug.pm
===================================================================
--- /tags/gsdl-21/gsdl/perllib/plugins/RecPlug.pm	(revision 699)
+++ /tags/gsdl-21/gsdl/perllib/plugins/RecPlug.pm	(revision 699)
@@ -0,0 +1,107 @@
+###########################################################################
+#
+# RecPlug.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# plugin which recurses through directories processing
+# each file it finds
+
+package RecPlug;
+
+use BasPlug;
+use plugin;
+use util;
+
+
+BEGIN {
+    @ISA = ('BasPlug');
+}
+
+sub new {
+    my ($class) = @_;
+    my $self = new BasPlug ();
+
+    $self->{'exclude_tail_dirs'} = []; # empty by default
+
+    return bless $self, $class;
+}
+
+# return 1 if this class might recurse using $pluginfo
+sub is_recursive {
+    my $self = shift (@_);
+
+    return 1;
+}
+
+
+# return number of files processed, undef if can't process
+# Note that $base_dir might be "" and that $file might 
+# include directories
+sub read {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $metadata, $processor, $maxdocs) = @_;
+
+    foreach $etd ( @{$self->{'exclude_tail_dirs'}} )
+    {
+	return 0 if ($file =~ m/$etd/);
+    }	
+
+    my (@dir, $subfile);
+
+    my $count = 0;
+
+    # see if this is a directory
+    $dirname = &util::filename_cat ($base_dir, $file);
+    if (-d $dirname) {
+
+	# read all the files in the directory
+        if (!opendir (DIR, $dirname))
+        {
+            print STDERR "RecPlug: WARNING - couldn't read directory $dirname\n";
+            return;
+        }
+
+	@dir = readdir (DIR);
+	closedir (DIR);
+
+	print STDERR "RecPlug: getting directory $dirname\n";
+
+	# process each file
+	foreach $subfile (@dir) {
+	    last if (defined $maxdocs && $maxdocs =~ /\d/ && $count >= $maxdocs);
+
+	    if ($subfile !~ /^\.\.?$/) {
+		# note: metadata is not carried on to the next level
+		$count += &plugin::read ($pluginfo, $base_dir, &util::filename_cat($file, $subfile),
+					 {}, $processor, $maxdocs);
+	    }
+	}
+	return $count;
+    }
+
+    # wasn't a directory, someone else will have to process it
+    return undef;
+}
+
+
+1;
Index: /tags/gsdl-21/gsdl/perllib/plugins/TEXTPlug.pm
===================================================================
--- /tags/gsdl-21/gsdl/perllib/plugins/TEXTPlug.pm	(revision 699)
+++ /tags/gsdl-21/gsdl/perllib/plugins/TEXTPlug.pm	(revision 699)
@@ -0,0 +1,134 @@
+###########################################################################
+#
+# TEXTPlug.pm -- simple text plugin
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# creates simple single-level document from .txt or .text files 
+# (case-insensitive match on filenames). Adds Title metadata 
+# of first 100 characters found.
+
+package TEXTPlug;
+
+use BasPlug;
+use sorttools;
+
+sub BEGIN {
+    @ISA = ('BasPlug');
+}
+
+sub new {
+    my ($class) = @_;
+    $self = new BasPlug ();
+
+    return bless $self, $class;
+}
+
+sub is_recursive {
+    my $self = shift (@_);
+
+    return 0; # this is not a recursive plugin
+}
+
+
+# return number of files processed, undef if can't process
+# Note that $base_dir might be "" and that $file might 
+# include directories
+sub read {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $metadata, $processor) = @_;
+
+    my $filename = &util::filename_cat($base_dir, $file);
+
+    return undef unless ($filename =~ /\.(te?xt(\.gz)?)$/i && (-e $filename));
+
+    my $gz = 0;
+    if (defined $2) {
+	$gz = $2;
+	$gz = 1 if ($gz =~ /\.gz/i);
+    }
+
+    print STDERR "TEXTPlug: processing $filename\n" if $processor->{'verbosity'};
+
+    # create a new document
+    my $doc_obj = new doc ($file, "indexed_doc");
+
+    if ($gz) {
+	open (FILE, "zcat $filename |") || die "TEXTPlug::read - zcat can't open $filename\n";
+    } else {
+	open (FILE, $filename) || die "TEXTPlug::read - can't open $filename\n";
+    }
+    my $cursection = $doc_obj->get_top_section();
+
+    my $text = "";
+    my $line = "";
+    my $first = 1;
+    while (defined ($line = <FILE>)) {
+	# use first line as title (or first 100 characters if it's long)
+	if ($first && length($line) > 5) {
+	    my $title = "";
+	    if (length($line) > 100) {
+		$title = substr ($line, 0, 100);
+	    } else {
+		$title = $line;
+	    }
+	    $doc_obj->add_metadata ($cursection, "Title", $title);
+	    $first = 0;
+	}
+	$text .= $line;
+    }
+    
+    $doc_obj->add_text ($cursection, "<pre>\n$text\n</pre>");
+
+
+    foreach $field (keys(%$metadata)) {
+	# $metadata->{$field} may be an array reference
+	if (ref ($metadata->{$field}) eq "ARRAY") {
+	    map { 
+		$doc_obj->add_metadata ($cursection, $field, $_); 
+	    } @{$metadata->{$field}};
+	} else {
+	    $doc_obj->add_metadata ($cursection, $field, $metadata->{$field}); 
+	}
+    }
+
+    # add OID
+    $doc_obj->set_OID ();
+
+    # process the document
+    $processor->process($doc_obj);
+
+    return 1; # processed the file
+}
+
+1;
+
+
+
+
+
+
+
+
+
+
+
Index: /tags/gsdl-21/gsdl/perllib/remproc.pm
===================================================================
--- /tags/gsdl-21/gsdl/perllib/remproc.pm	(revision 699)
+++ /tags/gsdl-21/gsdl/perllib/remproc.pm	(revision 699)
@@ -0,0 +1,59 @@
+###########################################################################
+#
+# remproc.pm -- document processor used to cache all documents in memory
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package remproc;
+
+use classify;
+use doc;
+use docproc;
+
+BEGIN {
+    @ISA = ('docproc');
+}
+
+
+sub new {
+    my ($class) = @_;
+    my $self = {'docs'=>[]};
+
+    return bless $self, $class;
+}
+
+sub process {
+    my $self = shift (@_);
+    my ($doc_obj, $filename) = @_;
+
+    push (@{$self->{'docs'}}, {'doc_obj'=>$doc_obj,
+			       'filename'=>$filename});
+}
+
+sub get_doc_list {
+    my $self = shift (@_);
+
+    return $self->{'docs'};
+}
+
+1;
+
Index: /tags/gsdl-21/gsdl/perllib/sorttools.pm
===================================================================
--- /tags/gsdl-21/gsdl/perllib/sorttools.pm	(revision 699)
+++ /tags/gsdl-21/gsdl/perllib/sorttools.pm	(revision 699)
@@ -0,0 +1,120 @@
+###########################################################################
+#
+# sorttools.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# various subroutines to format strings
+# for sorting
+
+package sorttools;
+
+
+# format an english string for sorting
+# i.e. convert to lowercase, remove the, a or an
+# from beginning of string etc.
+sub format_string_english {
+    my ($stringref) = @_;
+
+    $$stringref =~ tr/A-Z/a-z/;
+    $$stringref =~ s/[^a-z0-9 ]//g;
+    $$stringref =~ s/^\s*(the|a|an)\b//;
+    $$stringref =~ s/^\s+//;
+}
+
+# format an english name for sorting
+# i.e. convert to lowercase, put surname before
+# first names etc.    
+sub format_string_name_english {
+    my ($stringref) = @_;
+
+    $$stringref =~ tr/A-Z/a-z/;
+    $$stringref =~ s/[^a-z0-9 ]//g;
+    $$stringref =~ s/\s+/ /g;
+    $$stringref =~ s/^\s+//;
+    my @names = split / /, $$stringref;
+    my $surname = pop @names;
+    while (scalar @names && $surname =~ /^(jnr|snr)$/i) {
+	$surname = pop @names;
+    }
+    $$stringref = $surname . " " . $$stringref;
+}
+
+# takes arguments of day, month, year and converts to
+# date of form yyyymmdd. month may be full (e.g. "January"),
+# abbreviated (e.g. "Jan"), or a number (1-12). Years like "86" 
+# will be assumed to be "1986".
+sub format_date {
+    my ($day, $month, $year) = @_;
+
+    my %months = ('january' => '01', 'jan' => '01', 'february' => '02', 'feb' => '02',
+		  'march' => '03', 'mar' => '03', 'april' => '04', 'apr' => '04',
+		  'may' => '05', 'june' => '06', 'jun' => '06', 'july' => '07', 
+		  'jul' => '07', 'august' => '08', 'aug' => '08', 'september' => '09', 
+		  'sep' => '09', 'october' => '10', 'oct' => '10', 'november' => '11', 
+		  'nov' => '11', 'december' => '12', 'dec' => '12');
+
+    $month =~ tr/A-Z/a-z/;
+    
+    if ($day < 1) { 
+	print STDERR "sorttools::format_date WARNING day $day out of range\n";
+	$day = "01";
+    } elsif ($day > 31) {
+	print STDERR "sorttools::format_date WARNING day $day out of range\n";
+	$day = "31";
+    }
+
+    $day = "0$day" if (length($day) == 1);
+
+    if ($month =~ /^\d\d?$/) {
+	if ($month < 1) {
+	    print STDERR "sorttools::format_date WARNING month $month out of range\n";
+	    $month = "01";
+	} elsif ($month > 12) {
+	    print STDERR "sorttools::format_date WARNING month $month out of range\n";
+	    $month = "12";
+	}
+	if ($month =~ /^\d$/) {
+	    $month = "0" . $month;
+	}
+    } elsif (!defined $months{$month}) {
+	print STDERR "sorttools::format_date WARNING month $month out of range\n";
+	$month = "01";
+    } else {
+	$month = $months{$month};
+    }
+    
+    if ($year !~ /^\d\d\d\d$/) {
+	if ($year !~ /^\d\d$/) {
+	    print STDERR "sorttools::format_date WARNING year $year assumed to be 19$year\n";
+	    $year = "19" . $year;
+	} else {
+	    print STDERR "sorttools::format_date WARNING year $year out of range - reset to 1900\n";
+	    $year = "1900";
+	}
+    }
+
+    return "$year$month$day";
+}
+
+
+1;
Index: /tags/gsdl-21/gsdl/perllib/unicode.pm
===================================================================
--- /tags/gsdl-21/gsdl/perllib/unicode.pm	(revision 699)
+++ /tags/gsdl-21/gsdl/perllib/unicode.pm	(revision 699)
@@ -0,0 +1,197 @@
+###########################################################################
+#
+# unicode.pm --
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# useful functions for dealing with Unicode
+
+# Unicode strings are stored as arrays of scalars as perl 
+# lacks characters are 8-bit (currently)
+
+package unicode;
+
+
+
+# ascii2unicode takes a (extended) ascii string and
+# returns a unicode array.
+sub ascii2unicode {
+    my ($in) = @_;
+    my $out = [];
+
+    my $i = 0;
+    my $len = length($in);
+    while ($i < $len) {
+	push (@$out, ord(substr ($in, $i, 1)));	
+	$i++;
+    }
+
+    return $out;
+}
+
+
+# ascii2utf8 takes a (extended) ascii string and
+# returns a UTF-8 encoded string. This is just
+# a faster version of "&unicode2utf8(&ascii2unicode($str));"
+sub ascii2utf8 {
+    my ($in) = @_;
+    my $out = "";
+
+    my ($c);
+    my $i = 0;
+    my $len = length($in);
+    while ($i < $len) {
+	$c = ord (substr ($in, $i, 1));
+	if ($c < 0x80) {
+	    # ascii character
+	    $out .= chr ($c);
+
+	} else {
+	    # extended ascii character
+	    $out .= chr (0xc0 + (($c >> 6) & 0x1f));
+	    $out .= chr (0x80 + ($c & 0x3f));
+	}
+	$i++;
+    }
+
+    return $out;
+}
+
+
+# unicode2utf8 takes a unicode array as input and encodes it
+# using utf-8
+sub unicode2utf8 {
+    my ($in) = @_;
+    my $out = "";
+    
+    foreach $num (@$in) {
+	if ($num < 0x80) {
+	    $out .= chr ($num);
+
+	} elsif ($num < 0x800) {
+	    $out .= chr (0xc0 + (($num >> 6) & 0x1f));
+	    $out .= chr (0x80 + ($num & 0x3f));
+
+	} elsif ($num < 0xFFFF) {
+	    $out .= chr (0xe0 + (($num >> 12) & 0xf));
+	    $out .= chr (0x80 + (($num >> 6) & 0x3f));
+	    $out .= chr (0x80 + ($num & 0x3f));
+
+	} else {
+	    # error, don't encode anything
+	    die;
+	}
+    }
+
+    return $out;
+}
+
+
+# utf82unicode takes a utf-8 string and produces a unicode
+# array
+sub utf82unicode {
+    my ($in) = @_;
+    my $out = [];
+
+    my $i = 0;
+    my ($c1, $c2, $c3);
+    $len = length($in);
+    while ($i < $len) {
+	if (($c1 = ord(substr ($in, $i, 1))) < 0x80) {
+	    # normal ascii character
+	    push (@$out, $c1);
+
+	} elsif ($c1 < 0xc0) {
+	    # error, was expecting the first byte of an
+	    # encoded character. Do nothing.
+
+	} elsif ($c1 < 0xe0 && $i+1 < $len) {
+	    # an encoded character with two bytes
+	    $c2 = ord (substr ($in, $i+1, 1));
+	    if ($c2 >= 0x80 && $c2 < 0xc0) {
+		# everything looks ok
+		push (@$out, ((($c1 & 0x1f) << 6) +
+		      ($c2 & 0x3f)));
+		$i++; # gobbled an extra byte
+	    }
+
+	} elsif ($c1 < 0xf0 && $i+2 < $len) {
+	    # an encoded character with three bytes
+	    $c2 = ord (substr ($in, $i+1, 1));
+	    $c3 = ord (substr ($in, $i+2, 1));
+	    if ($c2 >= 0x80 && $c2 < 0xc0 &&
+		$c3 >= 0x80 && $c3 < 0xc0) {
+		# everything looks ok
+		push (@$out, ((($c1 & 0xf) << 12) +
+		      (($c2 & 0x3f) << 6) +
+		      ($c3 & 0x3f)));
+
+		$i += 2; # gobbled an extra two bytes
+	    }
+
+	} else {
+	    # error, only decode Unicode characters not full UCS.
+	    # Do nothing.
+	}
+
+	$i++;
+    }
+
+    return $out;
+}
+
+
+# unicode2ucs2 takes a unicode array and produces a UCS-2
+# unicode string (every two bytes forms a unicode character)
+sub unicode2ucs2 {
+    my ($in) = @_;
+    my $out = "";
+
+    foreach $num (@$in) {
+	$out .= chr (($num & 0xff00) >> 8);
+	$out .= chr ($num & 0xff);
+    }
+
+    return $out;
+}
+
+
+# ucs22unicode takes a UCS-2 string and produces a unicode array
+sub ucs22unicode {
+    my ($in) = @_;
+    my $out = [];
+
+    my $i = 0;
+    my $len = length ($in);
+    while ($i+1 < $len) {
+	push (@$out, ord (substr($in, $i, 1)) << 8 +
+	      ord (substr($in, $i+1, 1)));
+
+	$i ++;
+    }
+
+    return $out;
+}
+
+
+1;
+
Index: /tags/gsdl-21/gsdl/perllib/util.pm
===================================================================
--- /tags/gsdl-21/gsdl/perllib/util.pm	(revision 699)
+++ /tags/gsdl-21/gsdl/perllib/util.pm	(revision 699)
@@ -0,0 +1,472 @@
+###########################################################################
+#
+# util.pm -- various useful utilities
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package util;
+
+use File::Copy;
+use File::Basename;
+
+
+# removes files (but not directories)
+sub rm {
+    my (@files) = @_;
+    my @filefiles = ();
+
+    # make sure the files we want to delete exist 
+    # and are regular files
+    foreach $file (@files) {
+	if (!-e $file) {
+	    print STDERR "util::rm $file does not exist\n";
+	} elsif (!-f $file) {
+	    print STDERR "util::rm $file is not a regular file\n";
+	} else {
+	    push (@filefiles, $file);
+	}
+    }
+    
+    # remove the files
+    my $numremoved = unlink @filefiles;
+
+    # check to make sure all of them were removed
+    if ($numremoved != scalar(@filefiles)) {
+	print STDERR "util::rm Not all files were removed\n";
+    }
+}
+
+
+# recursive removal
+sub rm_r {
+    my (@files) = @_;
+
+    # recursively remove the files
+    foreach $file (@files) {
+	$file =~ s/[\/\\]+$//; # remove trailing slashes
+	
+	if (!-e $file) {
+	    print STDERR "util::rm_r $file does not exist\n";
+
+	} elsif (-d $file) {
+	    # get the contents of this directory
+	    if (!opendir (INDIR, $file)) {
+		print STDERR "util::rm_r could not open directory $file\n";
+	    } else {
+		my @filedir = grep (!/^\.\.?$/, readdir (INDIR));
+		closedir (INDIR);
+
+		# remove all the files in this directory
+		&rm_r (map {$_="$file/$_";} @filedir);
+
+		# remove this directory
+		if (!rmdir $file) {
+		    print STDERR "util::rm_r couldn't remove directory $file\n";
+		}
+	    }
+
+	} else {
+	    # remove this file
+	    &rm ($file);
+	}
+    }
+}
+
+
+# copies a file or a group of files
+sub cp {
+    my $dest = pop (@_);
+    my (@srcfiles) = @_;
+
+    # remove trailing slashes from source and destination files
+    $dest =~ s/[\\\/]+$//;
+    map {$_ =~ s/[\\\/]+$//;} @srcfiles;
+
+    # a few sanity checks
+    if (scalar (@srcfiles) == 0) {
+	print STDERR "util::cp no destination directory given\n";
+	return;
+    } elsif ((scalar (@srcfiles) > 1) && (!-d $dest)) {
+	print STDERR "util::cp if multiple source files are given the ".
+	    "destination must be a directory\n";
+	return;
+    }
+
+    # copy the files
+    foreach $file (@srcfiles) {
+	my $tempdest = $dest;
+	if (-d $tempdest) {
+	    my ($filename) = $file =~ /([^\\\/]+)$/;
+	    $tempdest .= "/$filename";
+	}
+	if (!-e $file) {
+	    print STDERR "util::cp $file does not exist\n";
+	} elsif (!-f $file) {
+	    print STDERR "util::cp $file is not a plain file\n";
+	} else {
+	    &File::Copy::copy ($file, $tempdest);
+	}
+    }
+}
+
+
+# recursively copies a file or group of files
+# syntax: cp_r (sourcefiles, destination file or directory)
+sub cp_r {
+    my $dest = pop (@_);
+    my (@srcfiles) = @_;
+
+    # remove trailing slashes from source and destination files
+    $dest =~ s/[\\\/]+$//;
+    map {$_ =~ s/[\\\/]+$//;} @srcfiles;
+
+    # a few sanity checks
+    if (scalar (@srcfiles) == 0) {
+	print STDERR "util::cp no destination directory given\n";
+	return;
+    } elsif ((scalar (@srcfiles) > 1) && (!-d $dest)) {
+	print STDERR "util::cp if multiple source files are given the ".
+	    "destination must be a directory\n";
+	return;
+    }
+    
+    # copy the files
+    foreach $file (@srcfiles) {
+	# copy the file to within dest if dest is a directory
+	# exception: if there is only one source file and that
+	# source file is a directory
+	my $tempdest = $dest;
+	if (-d $tempdest && !(scalar(@srcfiles) == 1 && -d $file)) {
+	    my ($filename) = $file =~ /([^\\\/]+)$/;
+	    $tempdest .= "/$filename";
+	}
+
+	if (!-e $file) {
+	    print STDERR "util::cp $file does not exist\n";
+
+	} elsif (-d $file) {
+	    # make a new directory (if needed)
+	    mkdir ($tempdest, 0775) unless -e $tempdest;
+	    
+	    # get the contents of this directory
+	    if (!opendir (INDIR, $file)) {
+		print STDERR "util::cp_r could not open directory $file\n";
+	    } else {
+		my @filedir = grep (!/^\.\.?$/, readdir (INDIR));
+		closedir (INDIR);
+
+		# copy all the files in this directory
+		&cp_r (map {$_="$file/$_";} @filedir, $tempdest);
+	    }
+
+	} else {
+	    &cp($file, $tempdest);
+	}
+    }
+}
+
+
+sub mk_all_dir {
+    my ($dir) = @_;
+
+    # use / for the directory separator, remove duplicate and
+    # trailing slashes
+    $dir=~s/[\\\/]+/\//g; 
+    $dir=~s/[\\\/]+$//;
+
+    # make sure the cache directory exists
+    my $dirsofar = "";
+    my $first = 1;
+    foreach $dirname (split ("/", $dir)) {
+	$dirsofar .= "/" unless $first;
+	$first = 0;
+
+	$dirsofar .= $dirname;
+
+	next if $dirname =~ /^(|[a-z]:)$/i;
+	if (!-e $dirsofar && !mkdir ($dirsofar, 0775)) {
+	    print STDERR "util::mk_all_dir could not create directory $dirsofar\n";
+	    return;
+	}
+    }
+}
+
+# make hard link to file if supported by OS, otherwise copy the file
+sub hard_link {
+    my ($src,$dest) = @_;
+
+    # remove trailing slashes from source and destination files
+    $src =~ s/[\\\/]+$//;
+    $dest =~ s/[\\\/]+$//;
+
+    # a few sanity checks
+    if (!-e $src) {
+	print STDERR "util::hard_link source file $src does not exist\n";
+	return;
+    }
+    elsif (-d $src) {
+	print STDERR "util::hard_link source $src is a directory\n";
+	return;
+    }
+
+    my $dest_dir = &File::Basename::dirname($dest);
+    mk_all_dir($dest_dir) if (!-e $dest_dir);
+
+    if (!link($src,$dest))
+    {
+	print STDERR "util::hard_link: unable to create hard link. ";
+	print STDERR " Attempting to copy file: $src -> $dest\n";
+	&File::Copy::copy ($src, $dest);
+    }
+    
+}
+
+
+
+
+# updates a copy of a directory in some other part of the filesystem
+# verbosity settings are: 0=low, 1=normal, 2=high
+# both $fromdir and $todir should be absolute paths
+sub cachedir {
+    my ($fromdir, $todir, $verbosity) = @_;
+    $verbosity = 1 unless defined $verbosity;
+
+    # use / for the directory separator, remove duplicate and
+    # trailing slashes
+    $fromdir=~s/[\\\/]+/\//g; 
+    $fromdir=~s/[\\\/]+$//;
+    $todir=~s/[\\\/]+/\//g; 
+    $todir=~s/[\\\/]+$//;
+
+    &mk_all_dir ($todir);
+
+    # get the directories in ascending order
+    if (!opendir (FROMDIR, $fromdir)) {
+	print STDERR "util::cachedir could not read directory $fromdir\n";
+	return;
+    }
+    my @fromdir = grep (!/^\.\.?$/, sort(readdir (FROMDIR)));
+    closedir (FROMDIR);
+
+    if (!opendir (TODIR, $todir)) {
+	print STDERR "util::cacedir could not read directory $todir\n";
+	return;
+    }
+    my @todir = grep (!/^\.\.?$/, sort(readdir (TODIR)));
+    closedir (TODIR);
+
+    my $fromi = 0;
+    my $toi = 0;
+		    
+    while ($fromi < scalar(@fromdir) || $toi < scalar(@todir)) {
+#	print "fromi: $fromi toi: $toi\n";
+
+	# see if we should delete a file/directory
+	# this should happen if the file/directory
+	# is not in the from list or if its a different
+	# size, or has an older timestamp
+	if ($toi < scalar(@todir)) {
+	    if (($fromi >= scalar(@fromdir)) ||
+		($todir[$toi] lt $fromdir[$fromi] || 
+		 ($todir[$toi] eq $fromdir[$fromi] && 
+		  &differentfiles("$fromdir/$fromdir[$fromi]","$todir/$todir[$toi]",
+				  $verbosity)))) {
+
+		# the files are different
+		&rm_r("$todir/$todir[$toi]");
+		splice(@todir, $toi, 1); # $toi stays the same
+
+	    } elsif ($todir[$toi] eq $fromdir[$fromi]) {
+		# the files are the same
+		# if it is a directory, check its contents
+		if (-d "$todir/$todir[$toi]") {
+		    &cachedir ("$fromdir/$fromdir[$fromi]",
+			       "$todir/$todir[$toi]", $verbosity);
+		}
+
+		$toi++;
+		$fromi++;
+		next;
+	    }
+	}
+  
+	# see if we should insert a file/directory
+	# we should insert a file/directory if there
+	# is no tofiles left or if the tofile does not exist
+	if ($fromi < scalar(@fromdir) && ($toi >= scalar(@todir) || 
+					  $todir[$toi] gt $fromdir[$fromi])) {
+	    &cp_r ("$fromdir/$fromdir[$fromi]", "$todir/$fromdir[$fromi]");
+	    splice (@todir, $toi, 0, $fromdir[$fromi]);
+
+	    $toi++;
+	    $fromi++;
+	}
+    }
+}
+
+# this function returns -1 if either file is not found
+# assumes that $file1 and $file2 are absolute file names or
+# in the current directory
+# $file2 is allowed to be newer than $file1
+sub differentfiles {
+    my ($file1, $file2, $verbosity) = @_;
+    $verbosity = 1 unless defined $verbosity;
+
+    $file1 =~ s/\/+$//;
+    $file2 =~ s/\/+$//;
+    
+    my ($file1name) = $file1 =~ /\/([^\/]*)$/;
+    my ($file2name) = $file2 =~ /\/([^\/]*)$/;
+
+    return -1 unless (-e $file1 && -e $file2);
+    if ($file1name ne $file2name) {
+	print STDERR "filenames are not the same\n" if ($verbosity >= 2);
+	return 1;
+    }
+
+    @file1stat = stat ($file1);
+    @file2stat = stat ($file2);
+
+    if (-d $file1) {
+	if (! -d $file2) {
+	    print STDERR "one file is a directory\n" if ($verbosity >= 2);
+	    return 1;
+	}
+	return 0;
+    }
+
+    # both must be regular files
+    unless (-f $file1 && -f $file2) {
+	print STDERR "one file is not a regular file\n" if ($verbosity >= 2);
+	return 1;
+    }
+
+    # the size of the files must be the same
+    if ($file1stat[7] != $file2stat[7]) {
+	print STDERR "different sized files\n" if ($verbosity >= 2);
+	return 1;
+    }
+
+    # the second file cannot be older than the first
+    if ($file1stat[9] > $file2stat[9]) {
+	print STDERR "file is older\n" if ($verbosity >= 2);
+	return 1;
+    }
+
+    return 0;
+}
+
+
+sub get_tmp_filename {
+    my $tmpdir = "$ENV{'GSDLHOME'}/tmp";
+    &mk_all_dir ($tmpdir) unless -e $tmpdir;
+
+    my $count = 1000;
+    my $rand = int(rand $count);
+    while (-e "$tmpdir/F$rand") {
+	$rand = int(rand $count);
+	$count++;
+    }
+
+    return "$tmpdir/F$rand";
+}
+
+
+sub filename_cat {
+    my (@filenames) = @_;
+    my $filename = join("/", @filenames);
+
+    # remove duplicate slashes and remove the last slash
+    if ($ENV{'GSDLOS'} =~ /^windows$/i) {
+	$filename =~ s/[\\\/]+/\\/g;
+    } else {
+	$filename =~ s/[\\\/]+/\//g;
+    }
+    $filename =~ s/[\\\/]$//;
+
+    return $filename;
+}
+
+sub get_os_dirsep {
+
+    if ($ENV{'GSDLOS'} =~ /^windows$/i) {
+	return "\\\\";
+    } else {
+	return "\\\/";
+    }
+}
+
+sub get_re_dirsep {
+
+    return "\\\\|\\\/";
+}
+
+
+# if this is running on windows we want binaries to end in
+# .exe, otherwise they don't have to end in any extension
+sub get_os_exe {
+    return ".exe" if $ENV{'GSDLOS'} =~ /^windows$/i;
+    return "";
+}
+
+
+# test to see whether this is a big or little endian machine
+sub is_little_endian {
+    return (ord(substr(pack("s",1), 0, 1)) == 1);
+}
+
+
+# will return the collection name if successful, "" otherwise
+sub use_collection {
+    my ($collection) = @_;
+
+    # get and check the collection
+    if (!defined($collection) || $collection eq "") {
+	if (defined $ENV{'GSDLCOLLECTION'}) {
+	    $collection = $ENV{'GSDLCOLLECTION'};
+	} else {
+	    print STDERR "No collection specified\n";
+	    return "";
+	}
+    }
+    
+    if ($collection eq "modelcol") {
+	print STDERR "You can't use modelcol.\n";
+	return "";
+    }
+
+    # make sure the environment variables GSDLCOLLECTION and GSDLCOLLECTDIR
+    # are defined
+    $ENV{'GSDLCOLLECTION'} = $collection unless defined $ENV{'GSDLCOLLECTION'};
+    $ENV{'GSDLCOLLECTDIR'} = "$ENV{'GSDLHOME'}/collect/$collection";
+
+    # make sure this collection exists
+    if (!-e $ENV{'GSDLCOLLECTDIR'}) {
+	print STDERR "Invalid collection ($collection).\n";
+	return "";
+    }
+
+    # everything is ready to go
+    return $collection;
+}
+
+1;
Index: /tags/gsdl-21/gsdl/setup.bash
===================================================================
--- /tags/gsdl-21/gsdl/setup.bash	(revision 699)
+++ /tags/gsdl-21/gsdl/setup.bash	(revision 699)
@@ -0,0 +1,3 @@
+export GSDLHOME=`pwd`
+export GSDLOS=`uname -s | tr A-Z a-z`
+export PATH=$PATH:$GSDLHOME/bin/script:$GSDLHOME/bin/$GSDLOS
Index: /tags/gsdl-21/gsdl/setup.csh
===================================================================
--- /tags/gsdl-21/gsdl/setup.csh	(revision 699)
+++ /tags/gsdl-21/gsdl/setup.csh	(revision 699)
@@ -0,0 +1,3 @@
+setenv GSDLHOME `pwd`
+setenv GSDLOS `uname -s | tr A-Z a-z`
+setenv PATH $PATH\:$GSDLHOME/bin/script\:$GSDLHOME/bin/$GSDLOS
Index: /tags/gsdl-21/gsdl/src/colservr/Makefile.in
===================================================================
--- /tags/gsdl-21/gsdl/src/colservr/Makefile.in	(revision 699)
+++ /tags/gsdl-21/gsdl/src/colservr/Makefile.in	(revision 699)
@@ -0,0 +1,83 @@
+###########################################################################
+#
+# Makefile -- src/colservr
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999  The New Zealand Digital Library Project
+#
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+
+AR = ar
+CC = @CC@
+CCFLAGS = @CFLAGS@ -Wall -Wunused -pedantic -W -Woverloaded-virtual
+DEFS = -DNZDL -DQUIET -DSHORT_SUFFIX -DPARADOCNUM -DGSDLSERVER -D_LITTLE_ENDIAN @DEFS@
+
+USE_FASTCGI = @USE_FASTCGI@
+ifneq ($(USE_FASTCGI), 1)
+INCLUDES = -I../../lib -I../recpt -I../../packages/mg \
+	   -I../../packages/mg/lib -I../../packages/mg/src/text
+else
+INCLUDES = -I../../lib -I../recpt -I../../packages/mg \
+	   -I../../packages/mg/lib -I../../packages/mg/src/text \
+           -I../../packages/fcgi/include
+endif
+
+RANLIB = @RANLIB@
+LDFLAGS = @LDFLAGS@
+LIBS = -lgdbm -lstdc++ -lm -lcrypt
+
+GSDLOS = @gsdlos@
+
+
+COMPILE = $(CC) -c $(CCFLAGS) $(DEFS) $(INCLUDES)
+LINK = $(CC) $(LDFLAGS) -o $@
+
+.SUFFIXES:
+.SUFFIXES: .c .o
+.c.o:
+	$(COMPILE) $<
+
+.SUFFIXES: .cpp .o
+.cpp.o:
+	$(COMPILE) $< 
+
+HEADERS = mgq.h mgsearch.h querycache.h queryinfo.h collectserver.h \
+          colservrconfig.h maptools.h filter.h source.h browsefilter.h \
+          queryfilter.h mggdbmsource.h phrasesearch.h phrasequeryfilter.h
+
+SOURCES = mgq.c mgsearch.cpp querycache.cpp queryinfo.cpp collectserver.cpp \
+          colservrconfig.cpp maptools.cpp filter.cpp source.cpp \
+          browsefilter.cpp queryfilter.cpp mggdbmsource.cpp phrasesearch.cpp \
+          phrasequeryfilter.cpp
+
+OBJECTS = mgq.o mgsearch.o querycache.o queryinfo.o collectserver.o \
+          colservrconfig.o maptools.o filter.o source.o browsefilter.o \
+          queryfilter.o mggdbmsource.o phrasesearch.o phrasequeryfilter.o
+
+all : $(OBJECTS)
+
+clean:
+	rm -f $(OBJECTS)
+
+install:
+
+depend:
+	makedepend -Y -- $(DEFS) $(INCLUDES) $(CCFLAGS) -- $(SOURCES)
Index: /tags/gsdl-21/gsdl/src/colservr/browsefilter.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/colservr/browsefilter.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/colservr/browsefilter.cpp	(revision 699)
@@ -0,0 +1,232 @@
+/**********************************************************************
+ *
+ * browsefilter.cpp -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.7  1999/09/07 04:57:20  sjboddie
+   added gpl notice
+
+   Revision 1.6  1999/08/31 22:33:21  rjmcnab
+   small update.
+
+   Revision 1.5  1999/07/16 03:42:23  sjboddie
+   changed isApprox
+
+   Revision 1.4  1999/05/10 03:43:47  sjboddie
+   lots of changes to lots of files - getting document action going
+
+   Revision 1.3  1999/04/30 02:00:45  sjboddie
+   lots of stuff to do with getting documentaction working
+
+   Revision 1.2  1999/04/19 23:56:05  rjmcnab
+   Finished the gdbm metadata stuff
+
+   Revision 1.1  1999/04/06 22:22:08  rjmcnab
+   Initial revision.
+
+ */
+
+
+#include "browsefilter.h"
+#include "fileutil.h"
+
+
+browsefilterclass::browsefilterclass () {
+  gdbmptr = NULL;
+
+  // -- onePerQuery StartResults   integer
+  FilterOption_t filtopt;
+  filtopt.name = "StartResults";
+  filtopt.type = FilterOption_t::integert;
+  filtopt.repeatable = FilterOption_t::onePerQuery;
+  filtopt.defaultValue = "1";
+  filtopt.validValues.push_back("1");
+  filtopt.validValues.push_back("10000");
+  filterOptions["StartResults"] = filtopt;
+
+  // -- onePerQuery EndResults     integer
+  filtopt.clear();
+  filtopt.name = "EndResults";
+  filtopt.type = FilterOption_t::integert;
+  filtopt.repeatable = FilterOption_t::onePerQuery;
+  filtopt.defaultValue = "10000";
+  filtopt.validValues.push_back("1");
+  filtopt.validValues.push_back("10000");
+  filterOptions["EndResults"] = filtopt;
+
+  // -- onePerQuery ParentNode     string ("" will return the browsing available)
+  filtopt.clear();
+  filtopt.name = "ParentNode";
+  filtopt.type = FilterOption_t::stringt;
+  filtopt.repeatable = FilterOption_t::onePerQuery;
+  filtopt.defaultValue = "";
+  filterOptions["ParentNode"] = filtopt;
+}
+
+browsefilterclass::~browsefilterclass () {
+}
+
+bool browsefilterclass::init (ostream &logout) {
+  outconvertclass text_t2ascii;
+
+  if (!filterclass::init(logout)) return false;
+
+  // get the filename for the database and make sure it exists
+  gdbm_filename = filename_cat(collectdir,"index","text",collection);
+
+#ifdef _LITTLE_ENDIAN
+  gdbm_filename += ".ldb";
+#else
+  gdbm_filename += ".bdb";
+#endif
+  if (!file_exists(gdbm_filename)) {
+    logout << text_t2ascii
+	   << "error: gdbm database \""
+	   << gdbm_filename << "\" does not exist\n\n";
+    return false;
+  }
+
+  return true;
+}
+
+void browsefilterclass::filter (const FilterRequest_t &request,
+				FilterResponse_t &response,
+				comerror_t &err, ostream &logout) {
+  int numDocs = 0;
+  outconvertclass text_t2ascii;
+
+  response.clear ();
+  err = noError;
+  if (gdbmptr == NULL) {
+    // most likely a configuration problem
+    logout << text_t2ascii 
+	   << "configuration error: browsefilter contains a null gdbmclass\n\n";
+    err = configurationError;
+    return;
+  }
+
+  // open the database
+  gdbmptr->setlogout(&logout);
+  if (!gdbmptr->opendatabase (gdbm_filename, GDBM_READER, 100, false)) {
+    // most likely a system problem (we have already checked that the
+    // gdbm database exists)
+    logout << text_t2ascii
+	   << "system problem: open on gdbm database \""
+	   << gdbm_filename << "\" failed\n\n";
+    err = systemProblem;
+    return;
+  }
+
+  // get the browse parameters
+  int startresults = filterOptions["StartResults"].defaultValue.getint();
+  int endresults = filterOptions["EndResults"].defaultValue.getint();
+  text_t parentnode = filterOptions["ParentNode"].defaultValue;
+  OptionValue_tarray::const_iterator options_here = request.filterOptions.begin();
+  OptionValue_tarray::const_iterator options_end = request.filterOptions.end();
+  while (options_here != options_end) {
+    if ((*options_here).name == "StartResults") 
+      startresults = (*options_here).value.getint();
+    else if ((*options_here).name == "EndResults")
+      endresults = (*options_here).value.getint();
+    else if ((*options_here).name == "ParentNode")
+      parentnode = (*options_here).value;
+    else {
+      logout << text_t2ascii
+	     << "warning: unknown browsefilter option \""
+	     << (*options_here).name
+	     << "\" ignored.\n\n";
+    }
+
+    options_here++;
+  }
+
+  infodbclass info;
+
+  // translate any ".fc", ".pr" etc. stuff in the parentnode
+  parentnode = gdbmptr->translate_OID (parentnode, info);
+
+  // adjust topmost browsing node
+  if (parentnode.empty()) parentnode = "browse";
+
+  // get the node
+  if ((request.filterResultOptions & FROID) ||
+      (request.filterResultOptions & FRmetadata)) {
+    if (!gdbmptr->getinfo(parentnode, info)) {
+      // didn't find the node
+      logout << text_t2ascii
+	     << "warning: lookup for node \"" << parentnode 
+	     << "\" failed for browsefilter.\n\n";
+    } else {
+      // found the node
+
+      // replace " with the parent node name and split the contains string
+      // into the result set
+      text_tarray resultset;
+      text_t tmptext;
+      text_t &contains = info["contains"];
+      text_t::iterator contains_here = contains.begin();
+      text_t::iterator contains_end = contains.end();
+      while (contains_here != contains_end) {
+	if (*contains_here == '"') tmptext += parentnode;
+	else if (*contains_here == ';') {
+	  if (!tmptext.empty()) resultset.push_back (tmptext);
+	  tmptext.clear();
+	} else tmptext.push_back(*contains_here);
+	
+	contains_here++;
+      }
+      // insert the last result in the set
+      if (!tmptext.empty()) resultset.push_back (tmptext);
+      
+      // do an intersection with the input set
+      if (!request.docSet.empty()) {
+	intersect (resultset, request.docSet);
+      }
+
+      // create the response
+      numDocs = resultset.size();
+      int resultnum = 1;
+      ResultDocInfo_t resultdoc;
+      text_tarray::iterator result_here = resultset.begin();
+      text_tarray::iterator result_end = resultset.end();
+
+      while (result_here != result_end) {
+	if (resultnum > endresults) break;
+	if (resultnum >= startresults) {
+	  resultdoc.OID = (*result_here);
+	  response.docInfo.push_back(resultdoc);
+	}
+
+	resultnum++;
+	result_here++;
+      }
+    }
+  }
+
+  response.numDocs = numDocs;
+  response.isApprox = Exact;
+}
Index: /tags/gsdl-21/gsdl/src/colservr/browsefilter.h
===================================================================
--- /tags/gsdl-21/gsdl/src/colservr/browsefilter.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/colservr/browsefilter.h	(revision 699)
@@ -0,0 +1,61 @@
+/**********************************************************************
+ *
+ * browsefilter.h -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef BROWSEFILTER_H
+#define BROWSEFILTER_H
+
+
+#include "gsdlconf.h"
+#include "text_t.h"
+#include "comtypes.h"
+#include "filter.h"
+#include "infodbclass.h"
+
+
+class browsefilterclass : public filterclass {
+protected:
+  text_t gdbm_filename;
+  gdbmclass *gdbmptr;
+
+public:
+  browsefilterclass ();
+  virtual ~browsefilterclass ();
+
+  // the gdbmptr remains the responsability of the calling code and
+  // should be destroyed after this browsefilter is destroyed
+  void set_gdbmptr (gdbmclass *thegdbmptr) {gdbmptr = thegdbmptr;}
+
+  bool init (ostream &logout);
+  text_t get_filter_name () {return "BrowseFilter";}
+  void filter (const FilterRequest_t &request,
+	       FilterResponse_t &response,
+	       comerror_t &err, ostream &logout);
+};
+
+
+#endif
Index: /tags/gsdl-21/gsdl/src/colservr/collectserver.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/colservr/collectserver.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/colservr/collectserver.cpp	(revision 699)
@@ -0,0 +1,357 @@
+
+/**********************************************************************
+ *
+ * collectserver.cpp -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.16  1999/10/10 08:20:35  sjboddie
+   - metadata now returns map rather than array
+   - redesigned browsing support (although it's not finished so
+   won't currently work ;-)
+
+   Revision 1.15  1999/09/07 04:57:20  sjboddie
+   added gpl notice
+
+   Revision 1.14  1999/08/31 22:34:55  rjmcnab
+   Changes to get compiling on AIX.
+
+   Revision 1.13  1999/08/25 04:50:00  sjboddie
+   changed FilterRequest_t::docSet into an array
+
+   Revision 1.12  1999/08/20 01:05:41  sjboddie
+   fixed a bug (or created another one)
+
+   Revision 1.11  1999/08/03 03:32:53  sjboddie
+   added ability to set receptionist from configuration files
+
+   Revision 1.10  1999/07/08 03:58:44  sjboddie
+   format stuff
+
+   Revision 1.9  1999/06/16 02:00:34  sjboddie
+   Few changes to get getParent filter option to return metadata of
+   parents as well as current OID
+
+   Revision 1.8  1999/05/10 03:43:47  sjboddie
+   lots of changes to lots of files - getting document action going
+
+   Revision 1.7  1999/04/30 02:00:45  sjboddie
+   lots of stuff to do with getting documentaction working
+
+   Revision 1.6  1999/04/06 22:20:29  rjmcnab
+   Got browsefilter working.
+
+   Revision 1.5  1999/03/31 23:44:44  rjmcnab
+   Altered the protocol so that the metadata is part of the filter.
+
+   Revision 1.4  1999/03/09 20:58:50  rjmcnab
+   Added dummy filter and metadata results.
+
+   Revision 1.3  1999/03/08 05:07:42  rjmcnab
+   Made some alterations to fit with the changes to the comtypes. Added the
+   "filteroptdefault" configuration option to alter default filter options.
+
+   Revision 1.2  1999/03/03 23:28:29  sjboddie
+
+   Provided stub functions for the protocol
+
+   Revision 1.1  1999/02/21 22:32:56  rjmcnab
+
+   Initial revision.
+
+ */
+
+
+#include "collectserver.h"
+#include "infodbclass.h"
+#include "OIDtools.h"
+#include <assert.h>
+
+
+collectserver::collectserver () {
+  configinfo.collection = "null";
+}
+
+collectserver::~collectserver () {
+}
+
+// configure should be called for each line in the
+// configuration files to configure the collection server and everything
+// it contains. The configuration should take place just before initialisation.
+void collectserver::configure (const text_t &key, const text_tarray &cfgline) {
+  if (cfgline.size() >= 1) {
+    const text_t &value = cfgline[0];
+    if (key == "gsdlhome") configinfo.gsdlhome = value;
+    else if (key == "collection") {
+      configinfo.collection = value;
+      collectinfo.shortInfo.name = value;
+    } else if (key == "collectdir") configinfo.collectdir = value;
+    else if (key == "host") collectinfo.shortInfo.host = value;
+    else if (key == "port") collectinfo.shortInfo.port = value.getint();
+    else if (key == "public") {
+      if (value == "true") collectinfo.isPublic = true;
+      else collectinfo.isPublic = false;
+    } else if (key == "beta") {
+      if (value == "true") collectinfo.isBeta = true;
+      else collectinfo.isBeta = false;
+    } else if (key == "builddate") collectinfo.buildDate = value.getint();
+    else if (key == "languages") collectinfo.languages = cfgline;
+    else if (key == "numdocs") collectinfo.numDocs = value.getint();
+    else if (key == "numwords") collectinfo.numWords = value.getint();
+    else if (key == "numbytes") collectinfo.numBytes = value.getint();
+    else if (key == "format" && cfgline.size() == 2)
+      collectinfo.format[cfgline[0]] = cfgline[1];
+    else if (key == "receptionist") collectinfo.receptionist = value;
+  }
+
+  // configure the filters
+  filtermapclass::iterator filter_here = filters.begin();
+  filtermapclass::iterator filter_end = filters.end();
+  while (filter_here != filter_end) {
+    assert ((*filter_here).second.f != NULL);
+    if ((*filter_here).second.f != NULL) 
+      (*filter_here).second.f->configure(key, cfgline);
+
+    filter_here++;
+  }
+
+  // configure the sources
+  sourcelistclass::iterator source_here = sources.begin();
+  sourcelistclass::iterator source_end = sources.end();
+  while (source_here != source_end) {
+    assert ((*source_here).s != NULL);
+    if ((*source_here).s != NULL)
+      (*source_here).s->configure(key, cfgline);
+    
+    source_here++;
+  }
+}
+
+void collectserver::configure (const text_t &key, const text_t &value) {
+  text_tarray cfgline;
+  cfgline.push_back (value);
+  configure(key, cfgline);
+}
+
+
+bool collectserver::init (ostream &logout) {
+  // init the filters
+  filtermapclass::iterator filter_here = filters.begin();
+  filtermapclass::iterator filter_end = filters.end();
+  while (filter_here != filter_end) {
+    assert ((*filter_here).second.f != NULL);
+    if (((*filter_here).second.f != NULL) &&
+	!(*filter_here).second.f->init(logout)) return false;
+    
+    filter_here++;
+  }
+
+  // init the sources
+  sourcelistclass::iterator source_here = sources.begin();
+  sourcelistclass::iterator source_end = sources.end();
+  while (source_here != source_end) {
+    assert ((*source_here).s != NULL);
+    if (((*source_here).s != NULL) &&
+	!(*source_here).s->init(logout)) return false;
+    
+    source_here++;
+  }
+
+  return true;
+}
+
+
+void collectserver::get_collectinfo (ColInfoResponse_t &reponse,
+		      comerror_t &err, ostream &/*logout*/) {
+  reponse = collectinfo; 
+  err = noError;
+}
+
+void collectserver::get_filterinfo (InfoFiltersResponse_t &response,
+				    comerror_t &err, ostream &/*logout*/) {
+  response.clear ();
+
+  // get a list of filter names
+  filtermapclass::iterator filter_here = filters.begin();
+  filtermapclass::iterator filter_end = filters.end();
+  while (filter_here != filter_end) {
+    response.filterNames.insert ((*filter_here).first);
+    filter_here++;
+  }
+
+  err = noError;
+}
+
+void collectserver::get_filteroptions (const InfoFilterOptionsRequest_t &request,
+				       InfoFilterOptionsResponse_t &response, 
+				       comerror_t &err, ostream &logout) {
+  outconvertclass text_t2ascii;
+
+  filterclass *thisfilter = filters.getfilter(request.filterName);
+  if (thisfilter != NULL) {
+    thisfilter->get_filteroptions (response, err, logout);
+  } else {
+    response.clear ();
+    err = protocolError;
+    logout << text_t2ascii << "Protocol Error: filter options requested for non-existent\n"
+	   << "filter \"" << request.filterName << "\".\n\n";
+  }
+}
+
+void collectserver::filter (FilterRequest_t &request,
+			    FilterResponse_t &response,
+			    comerror_t &err, ostream &logout) {
+  outconvertclass text_t2ascii;
+
+  // translate any ".fc", ".pr" etc. stuff in the docSet
+  text_t translatedOID;
+  text_tarray translatedOIDs;
+  text_tarray::iterator doc_here = request.docSet.begin();
+  text_tarray::iterator doc_end = request.docSet.end();
+  while (doc_here != doc_end) {
+    if (needs_translating (*doc_here)) {
+      sourcelistclass::iterator source_here = sources.begin();
+      sourcelistclass::iterator source_end = sources.end();
+      while (source_here != source_end) {
+	assert ((*source_here).s != NULL);
+	if (((*source_here).s != NULL) &&
+	    ((*source_here).s->translate_OID (*doc_here, translatedOID, err, logout))) {
+	  if (err != noError) return;
+	  break;
+	}
+	source_here++;
+      }
+      translatedOIDs.push_back (translatedOID);
+    } else {
+      translatedOIDs.push_back (*doc_here);
+    }
+    doc_here ++;
+  }
+  request.docSet = translatedOIDs;
+
+  response.clear();
+
+  filterclass *thisfilter = filters.getfilter(request.filterName);
+  if (thisfilter != NULL) {
+    // filter the data
+    thisfilter->filter (request, response, err, logout);
+
+    // fill in the metadata for each of the OIDs (if it is requested)
+    if (request.filterResultOptions & FRmetadata) {
+      bool processed = false;
+      ResultDocInfo_tarray::iterator resultdoc_here = response.docInfo.begin();
+      ResultDocInfo_tarray::iterator resultdoc_end = response.docInfo.end();
+      while (resultdoc_here != resultdoc_end) {
+	// try each of the sources in turn
+	sourcelistclass::iterator source_here = sources.begin();
+	sourcelistclass::iterator source_end = sources.end();
+	while (source_here != source_end) {
+	  assert ((*source_here).s != NULL);
+	  if (((*source_here).s != NULL) &&
+	      ((*source_here).s->get_metadata(request.requestParams, request.refParams,
+					      request.getParents, request.fields, 
+					      (*resultdoc_here).OID, (*resultdoc_here).metadata, 
+					      err, logout))) {
+	    if (err != noError) return;
+	    processed = true;
+	    break;
+	  }
+	  
+	  source_here++;
+	}
+	if (!processed) {
+	  err = protocolError;
+	  return;
+	}
+	resultdoc_here++;
+      }
+    }
+    
+  } else {
+    response.clear ();
+    err = protocolError;
+    logout << text_t2ascii << "Protocol Error: filter options requested for non-existent\n"
+	   << "filter \"" << request.filterName << "\".\n\n";
+  }
+  
+  err = noError;
+}
+
+void collectserver::get_document (const DocumentRequest_t &request,
+				  DocumentResponse_t &response,
+				  comerror_t &err, ostream &logout) {
+
+  sourcelistclass::iterator source_here = sources.begin();
+  sourcelistclass::iterator source_end = sources.end();
+  while (source_here != source_end) {
+    assert ((*source_here).s != NULL);
+    if (((*source_here).s != NULL) &&
+	((*source_here).s->get_document (request.OID, response.doc, err, logout))) {
+      if (err != noError) return;
+      break;
+    }
+    source_here++;
+  }
+}
+
+
+bool operator==(const collectserverptr &x, const collectserverptr &y) {
+  return (x.c == y.c);
+}
+
+bool operator<(const collectserverptr &x, const collectserverptr &y) {
+  return (x.c < y.c);
+}
+
+
+// thecollectserver remains the property of the calling code but
+// should not be deleted until it is removed from this list.
+void collectservermapclass::addcollectserver (collectserver *thecollectserver) {
+  // can't add a null collection server
+  assert (thecollectserver != NULL);
+  if (thecollectserver == NULL) return;
+  
+  // can't add an collection server with no collection name
+  assert (!(thecollectserver->get_collection_name()).empty());
+  if ((thecollectserver->get_collection_name()).empty()) return;
+
+  collectserverptr cptr;
+  cptr.c = thecollectserver;
+  collectserverptrs[thecollectserver->get_collection_name()] = cptr;
+}
+
+// getcollectserver will return NULL if the collectserver could not be found
+collectserver *collectservermapclass::getcollectserver (const text_t &collection) {
+  // can't find a collection with no name
+  assert (!collection.empty());
+  if (collection.empty()) return NULL;
+
+  iterator here = collectserverptrs.find (collection);
+  if (here == collectserverptrs.end()) return NULL;
+  
+  return (*here).second.c;
+}
+
Index: /tags/gsdl-21/gsdl/src/colservr/collectserver.h
===================================================================
--- /tags/gsdl-21/gsdl/src/colservr/collectserver.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/colservr/collectserver.h	(revision 699)
@@ -0,0 +1,183 @@
+/**********************************************************************
+ *
+ * collectserver.h -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef COLLECTSERVER_H
+#define COLLECTSERVER_H
+
+#include "gsdlconf.h"
+#include "text_t.h"
+#include "comtypes.h"
+#include "filter.h"
+#include "source.h"
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\iostream>
+#elif defined(GSDL_USE_IOS_H)
+#  include <iostream.h>
+#else
+#  include <iostream>
+#endif
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\map>
+#elif defined(GSDL_USE_STL_H)
+#  include <map.h>
+#else
+#  include <map>
+#endif
+
+
+struct colservrconf {
+  text_t gsdlhome;
+  text_t collection;
+  text_t collectdir;
+};
+
+
+class collectserver {
+protected:
+  colservrconf configinfo;
+  ColInfoResponse_t collectinfo;
+
+  filtermapclass filters;
+  sourcelistclass sources;
+
+public:
+  collectserver ();
+  virtual ~collectserver ();
+
+  // add_filter makes another filter available to the collection server
+  // the filter remains the property of the calling code and that
+  // code should destroy the filter after the collection server has been
+  // destroyed.
+  void add_filter (filterclass *thefilter) {filters.addfilter(thefilter);}
+  filtermapclass *get_filtermap_ptr () {return &filters;}
+
+  // add_source makes another source available to the collection server
+  // the source remains the property of the calling code and that
+  // code should destroy the source after the collection server has been
+  // destroyed.
+  void add_source (sourceclass *thesource) {sources.addsource(thesource);}
+  sourcelistclass *get_sourcelist_ptr () {return &sources;}
+
+  // configure should be called for each line in the
+  // configuration files to configure the collection server and everything
+  // it contains. The configuration should take place just before initialisation.
+  virtual void configure (const text_t &key, const text_tarray &cfgline);
+  void configure (const text_t &key, const text_t &value);
+  const colservrconf &get_configinfo () {return configinfo;}
+  text_t get_collection_name () {return configinfo.collection;}
+
+  // init should be called after the collection name has been set and
+  // all initialisation is done. init returns true if the initialisation
+  // was successful, if false is returned then no other functions
+  // should be called (without producing meaningless output).
+  virtual bool init (ostream &logout);
+
+  void get_collectinfo (ColInfoResponse_t &reponse,
+			comerror_t &err, ostream &logout);
+
+  virtual void get_filterinfo (InfoFiltersResponse_t &response,
+			       comerror_t &err, ostream &logout);
+
+  virtual void get_filteroptions (const InfoFilterOptionsRequest_t &request,
+				  InfoFilterOptionsResponse_t &response, 
+				  comerror_t &err, ostream &logout);
+
+  virtual void filter (FilterRequest_t &request,
+		       FilterResponse_t &response,
+		       comerror_t &err, ostream &logout);
+
+
+  virtual void get_document (const DocumentRequest_t &request,
+			     DocumentResponse_t &response,
+			     comerror_t &err, ostream &logout);
+
+};
+
+
+// The collectserverptr function does not 'own' the collectserver. The
+// collectserver should be deleted by the code which created it.
+class collectserverptr {
+public:
+  collectserver *c;
+
+  collectserverptr () {c=NULL;}
+  ~collectserverptr () {}
+};
+
+bool operator==(const collectserverptr &x, const collectserverptr &y);
+bool operator<(const collectserverptr &x, const collectserverptr &y);
+
+typedef map<text_t, collectserverptr, lttext_t> collectserverptrmap;
+
+// contains a list of collectservers indexed by their name
+class collectservermapclass {
+protected:
+  collectserverptrmap collectserverptrs;
+
+public:
+  // type support for collectserverptrmap
+  typedef collectserverptrmap::iterator iterator;
+  typedef collectserverptrmap::const_iterator const_iterator;
+  typedef collectserverptrmap::reference reference;
+  typedef collectserverptrmap::const_reference const_reference;
+  typedef collectserverptrmap::size_type size_type;
+
+  typedef collectserverptrmap::difference_type difference_type;
+  typedef collectserverptrmap::const_reverse_iterator const_reverse_iterator;
+  typedef collectserverptrmap::reverse_iterator reverse_iterator;
+  
+  // basic container support
+  iterator begin () {return collectserverptrs.begin();}
+  const_iterator begin () const {return collectserverptrs.begin();}
+  iterator end () {return collectserverptrs.end();}
+  const_iterator end () const {return collectserverptrs.end();}
+
+  void erase(iterator pos) {collectserverptrs.erase(pos);}
+  void erase(iterator first, iterator last) {collectserverptrs.erase(first, last);}
+  collectservermapclass &operator=(const collectservermapclass &x) 
+    {collectserverptrs=x.collectserverptrs;return *this;}
+
+  bool empty () const {return collectserverptrs.empty();}
+  size_type size() const {return collectserverptrs.size();}
+
+
+  // added functionality
+  void clear () {collectserverptrs.erase(collectserverptrs.begin(),collectserverptrs.end());}
+
+  // thecollectserver remains the property of the calling code but
+  // should not be deleted until it is removed from this list.
+  void addcollectserver (collectserver *thecollectserver);
+
+  // getcollectserver will return NULL if the collectserver could not be found
+  collectserver *getcollectserver (const text_t &collection);
+};
+
+
+#endif
Index: /tags/gsdl-21/gsdl/src/colservr/colservrconfig.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/colservr/colservrconfig.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/colservr/colservrconfig.cpp	(revision 699)
@@ -0,0 +1,129 @@
+/**********************************************************************
+ *
+ * colservrconfig.cpp -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.2  1999/09/07 04:57:20  sjboddie
+   added gpl notice
+
+   Revision 1.1  1999/02/21 22:32:55  rjmcnab
+
+   Initial revision.
+
+ */
+
+
+#include "colservrconfig.h"
+#include "fileutil.h"
+#include "cfgread.h"
+
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace/std/iostream>
+#  include <ospace/std/fstream>
+#elif defined(GSDL_USE_IOS_H)
+#  include <iostream.h>
+#  include <fstream.h>
+#else
+#  include <iostream>
+#  include <fstream>
+#endif
+
+// collect_cfg_read reads collect.cfg returning true on success.
+bool collect_cfg_read (collectserver &cserver, const text_t &gsdlhome,
+		       const text_t &collection) {
+  text_t filename = filename_cat (gsdlhome, "collect");
+  filename = filename_cat (filename, collection);
+  filename = filename_cat (filename, "etc");
+  filename = filename_cat (filename, "collect.cfg");
+  if (!file_exists (filename)) {
+    filename = filename_cat (gsdlhome, "etc");
+    filename = filename_cat (filename, "collect.cfg");
+    if (!file_exists (filename)) return false;
+  }
+
+  // read in the collection configuration file
+  text_t key;
+  text_tarray cfgline;
+  char *cstr = filename.getcstr();
+  ifstream confin (cstr);
+  delete cstr;
+
+  if (confin) {
+    while (read_cfg_line(confin, cfgline) >= 0) {
+      if (cfgline.size () >= 2) {
+	key = cfgline[0];
+	cfgline.erase(cfgline.begin());
+
+	// configure the collection server
+	cserver.configure (key, cfgline);
+      }
+    }
+    confin.close ();
+    return true;
+  }
+  return false;
+}
+
+
+// build_cfg_read reads build.cfg and returns true on success.
+bool build_cfg_read (collectserver &cserver, const text_t &gsdlhome,
+		     const text_t &collection) {
+  text_t filename = filename_cat (gsdlhome, "collect");
+  filename = filename_cat (filename, collection);
+  filename = filename_cat (filename, "index");
+
+  filename = filename_cat (filename, "build.cfg");
+  if (!file_exists (filename)) {
+    filename = filename_cat (gsdlhome, "index");
+    filename = filename_cat (filename, "build.cfg");
+    if (!file_exists (filename)) return false;
+  }
+
+  // read in the build configuration file
+  text_t key;
+  text_tarray cfgline;
+  char *cstr = filename.getcstr();
+  ifstream confin (cstr);
+  delete cstr;
+
+  if (confin) {
+    while (read_cfg_line(confin, cfgline) >= 0) {
+      if (cfgline.size () >= 2) {
+	key = cfgline[0];
+	cfgline.erase(cfgline.begin());
+
+	// configure the collection server
+	cserver.configure (key, cfgline);
+      }
+    }
+    confin.close ();
+    return true;
+  }
+  return false;
+}
+
Index: /tags/gsdl-21/gsdl/src/colservr/colservrconfig.h
===================================================================
--- /tags/gsdl-21/gsdl/src/colservr/colservrconfig.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/colservr/colservrconfig.h	(revision 699)
@@ -0,0 +1,47 @@
+/**********************************************************************
+ *
+ * colservrconfig.h -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef COLSERVRCONFIG_H
+#define COLSERVRCONFIG_H
+
+
+#include "gsdlconf.h"
+#include "text_t.h"
+#include "collectserver.h"
+
+
+// collect_cfg_read reads collect.cfg returning true on success.
+bool collect_cfg_read (collectserver &cserver, const text_t &gsdlhome,
+		       const text_t &collection);
+
+// build_cfg_read reads build.cfg and returns true on success.
+bool build_cfg_read (collectserver &cserver, const text_t &gsdlhome,
+		     const text_t &collection);
+
+
+#endif
Index: /tags/gsdl-21/gsdl/src/colservr/filter.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/colservr/filter.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/colservr/filter.cpp	(revision 699)
@@ -0,0 +1,342 @@
+/**********************************************************************
+ *
+ * filter.cpp -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.9  1999/09/07 04:57:21  sjboddie
+   added gpl notice
+
+   Revision 1.8  1999/08/31 22:37:34  rjmcnab
+   A few changes to get compiling on AIX.
+
+   Revision 1.7  1999/08/25 04:49:33  sjboddie
+   changed FilterRequest_t::docSet into an array
+
+   Revision 1.6  1999/07/16 03:42:23  sjboddie
+   changed isApprox
+
+   Revision 1.5  1999/04/30 02:00:46  sjboddie
+   lots of stuff to do with getting documentaction working
+
+   Revision 1.4  1999/04/12 03:45:02  rjmcnab
+   Finished the query filter.
+
+   Revision 1.3  1999/04/06 22:20:30  rjmcnab
+   Got browsefilter working.
+
+   Revision 1.2  1999/03/31 23:44:46  rjmcnab
+   Altered the protocol so that the metadata is part of the filter.
+
+   Revision 1.1  1999/03/30 05:10:06  rjmcnab
+   Initial revision.
+
+ */
+
+
+#include "filter.h"
+#include "fileutil.h"
+#include <assert.h>
+
+
+// default constructor does nothing
+filterclass::filterclass () {
+}
+
+// default destructor does nothing
+filterclass::~filterclass () {
+}
+
+// configure should be called once for each configuration line
+// default configures the default filter options
+void filterclass::configure (const text_t &key, const text_tarray &cfgline) {
+  if (cfgline.size() >= 1) {
+    const text_t &value = cfgline[0];
+
+    if (key == "collection") collection = value;
+    else if (key == "collectdir") collectdir = value;
+    else if (key == "gsdlhome") gsdlhome = value;
+    else if ((key == "filteroptdefault") && (cfgline.size() == 2)) {
+      // see if this filter has an option with this name
+      FilterOption_tmap::iterator thisfilteroption = 
+	filterOptions.find(cfgline[0]);
+      if (thisfilteroption != filterOptions.end())
+	(*thisfilteroption).second.defaultValue = cfgline[1];
+    }
+  }
+}
+
+// init should be called after all the configuration is done but
+// before any other methods are called
+// default checks all the filter option defaults
+bool filterclass::init (ostream &/*logout*/) {
+  // check all the filter defaults
+  FilterOption_tmap::iterator filteroption_here = filterOptions.begin();
+  FilterOption_tmap::iterator filteroption_end = filterOptions.end();
+  while (filteroption_here != filteroption_end) {
+    (*filteroption_here).second.check_defaultValue ();
+    
+    filteroption_here++;
+  }
+
+  // get the collection directory name
+  if (collectdir.empty()) {
+    collectdir = filename_cat (gsdlhome, "collect", collection);
+  }
+  
+  return true;
+}
+
+// returns the name of this filter
+// default returns "NullFilter"
+text_t filterclass::get_filter_name () {
+  return "NullFilter";
+}
+
+// returns the current filter options
+void filterclass::get_filteroptions (InfoFilterOptionsResponse_t &response,
+				     comerror_t &err, ostream &/*logout*/) {
+  response.clear();
+  response.filterOptions = filterOptions;
+  err = noError;
+}
+
+// default returns nothing
+void filterclass::filter (const FilterRequest_t &request,
+			  FilterResponse_t &response,
+			  comerror_t &err, ostream &/*logout*/) {
+  ResultDocInfo_t resultdoc;
+
+  response.clear();
+
+  if ((request.filterResultOptions & FROID) ||
+      (request.filterResultOptions & FRmetadata)) {
+    // copy the OIDs from the request to the response
+    text_tarray::const_iterator here = request.docSet.begin();
+    text_tarray::const_iterator end = request.docSet.end();
+    while (here != end) {
+      resultdoc.OID = (*here);
+      response.docInfo.push_back(resultdoc);
+
+      here++;
+    }
+  }
+
+  response.numDocs = response.docInfo.size();
+  response.isApprox = Exact;
+  err = noError;
+}
+
+
+bool operator==(const filterptr &x, const filterptr &y) {
+  return (x.f == y.f);
+}
+
+bool operator<(const filterptr &x, const filterptr &y) {
+  return (x.f < y.f);
+}
+
+
+// thefilter remains the property of the calling code but
+// should not be deleted until it is removed from this list.
+void filtermapclass::addfilter (filterclass *thefilter) {
+  // can't add a null filter
+  assert (thefilter != NULL);
+  if (thefilter == NULL) return;
+  
+  // can't add an filter with no name
+  assert (!(thefilter->get_filter_name()).empty());
+  if ((thefilter->get_filter_name()).empty()) return;
+
+  filterptr fptr;
+  fptr.f = thefilter;
+  filterptrs[thefilter->get_filter_name()] = fptr;
+}
+
+// getfilter will return NULL if the filter could not be found
+filterclass *filtermapclass::getfilter (const text_t &key) {
+  // can't find an filter with no name
+  assert (!key.empty());
+  if (key.empty()) return NULL;
+
+  iterator here = filterptrs.find (key);
+  if (here == filterptrs.end()) return NULL;
+  
+  return (*here).second.f;
+}
+
+
+
+
+// some useful functions for dealing with document sets
+
+// returns -1 if t1 is a child of t2
+// returns 0  if t1 and t2 are not parent-child related
+// returns 1  if t1 is a parent of t2
+int child_compare (const text_t &t1, const text_t &t2) {
+  text_t::const_iterator t1_here = t1.begin();
+  text_t::const_iterator t1_end = t1.end();
+  text_t::const_iterator t2_here = t2.begin();
+  text_t::const_iterator t2_end = t2.end();
+
+  while ((t1_here != t1_end) && (t2_here != t2_end)) {
+    if (*t1_here != *t2_here) return 0; // unrelated
+    t1_here++;
+    t2_here++;
+  }
+
+  if ((t1_here == t1_end) && (t2_here == t2_end)) return 0; // equal
+  if (t1_here != t1_end) {
+    if (*t1_here == '.') return -1; // t1 is child
+    else return 0; // unrelated
+  }
+
+  if (t2_here != t2_end) {
+    if (*t2_here == '.') return 1; // t2 is child
+    else return 0; // unrelated
+  }
+
+  return 0; // shouldn't get here...
+}
+
+// intersect places the result in set1
+void intersect (text_tset &set1, const text_tset &set2) {
+  text_tset resultset;
+  int childcomp = 0;
+
+  text_tset::const_iterator set1_here = set1.begin();
+  text_tset::const_iterator set1_end = set1.end();
+  text_tset::const_iterator set2_here = set2.begin();
+  text_tset::const_iterator set2_end = set2.end();
+  while ((set1_here != set1_end) && (set2_here != set2_end)) {
+    if (*set1_here == *set2_here) {
+      // equal
+      resultset.insert (*set1_here);
+      set1_here++;
+      set2_here++;
+
+    } else if ((childcomp=child_compare(*set1_here, *set2_here)) != 0) {
+      if (childcomp < 0) {
+	// set1_here is child
+	resultset.insert (*set1_here);
+	set1_here++;
+      } else {
+	// set2_here is child
+	resultset.insert (*set2_here);
+	set2_here++;
+      }
+
+    } else if (*set1_here < *set2_here) {
+      // set1 is less
+      set1_here++;
+
+    } else {
+      // set2 is less
+      set2_here++;
+    }
+  }
+
+  set1 = resultset;
+}
+
+void intersect (text_tarray &set1, const text_tset &set2) {
+  text_tarray resultset;
+
+  text_tarray::const_iterator set1_here = set1.begin();
+  text_tarray::const_iterator set1_end = set1.end();
+
+  while (set1_here != set1_end) {
+    if (in_set (set2, *set1_here))
+      resultset.push_back (*set1_here);
+    set1_here ++;
+  }
+  set1 = resultset;
+}
+
+void intersect (text_tarray &set1, const text_tarray &set2) {
+  text_tarray resultset;
+
+  text_tarray::const_iterator set1_here = set1.begin();
+  text_tarray::const_iterator set1_end = set1.end();
+
+  while (set1_here != set1_end) {
+    if (in_set (set2, *set1_here))
+      resultset.push_back (*set1_here);
+    set1_here ++;
+  }
+  set1 = resultset;
+}
+
+// tests to see if el is in set
+bool in_set (const text_tset &set1, const text_t &el) {
+  text_t::const_iterator here = el.begin();
+  text_t::const_iterator end = el.begin();
+  text_t tryel, tryel_add;
+  bool first = true;
+
+  // the element is in the set if any of its parents are
+  // in the set
+  do {
+    // get next possible element to try
+    here = getdelimitstr (here, end, '.', tryel_add);
+    if (!first) tryel += ".";
+    first = false;
+    tryel += tryel_add;
+
+    // see if this element is in the set
+    if (set1.find(tryel) != set1.end()) return true;
+  } while (here != end);
+
+  return false;
+}
+
+bool in_set (const text_tarray &set1, const text_t &el) {
+  text_t::const_iterator here = el.begin();
+  text_t::const_iterator end = el.begin();
+  text_t tryel, tryel_add;
+  bool first = true;
+
+  // the element is in the set if any of its parents are
+  // in the set
+  do {
+    // get next possible element to try
+    here = getdelimitstr (here, end, '.', tryel_add);
+    if (!first) tryel += ".";
+    first = false;
+    tryel += tryel_add;
+
+    // see if this element is in the set
+    text_tarray::const_iterator here = set1.begin();
+    text_tarray::const_iterator end = set1.end();
+    while (here != end) {
+      if (*here == tryel) break;
+      here ++;
+    }
+    if (here != end) return true;
+  } while (here != end);
+
+  return false;
+}
Index: /tags/gsdl-21/gsdl/src/colservr/filter.h
===================================================================
--- /tags/gsdl-21/gsdl/src/colservr/filter.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/colservr/filter.h	(revision 699)
@@ -0,0 +1,142 @@
+/**********************************************************************
+ *
+ * filter.h -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef FILTER_H
+#define FILTER_H
+
+#include "gsdlconf.h"
+#include "text_t.h"
+#include "comtypes.h"
+
+
+class filterclass {
+protected:
+  text_t gsdlhome;
+  text_t collection;
+  text_t collectdir;
+
+  FilterOption_tmap filterOptions;
+
+public:
+  filterclass ();
+  virtual ~filterclass ();
+
+  // configure should be called once for each configuration line
+  // default configures the default filter options
+  virtual void configure (const text_t &key, const text_tarray &cfgline);
+
+  // init should be called after all the configuration is done but
+  // before any other methods are called
+  // default checks all the filter option defaults
+  virtual bool init (ostream &logout);
+
+  // returns the name of this filter
+  virtual text_t get_filter_name ();
+
+  // returns the current filter options
+  virtual void get_filteroptions (InfoFilterOptionsResponse_t &response,
+				  comerror_t &err, ostream &logout);
+
+  virtual void filter (const FilterRequest_t &request,
+		       FilterResponse_t &response,
+		       comerror_t &err, ostream &logout);
+};
+
+
+// The filterptr class does not 'own' the filter. The 
+// filter should be deleted by the code which created it.
+class filterptr {
+public:
+  filterclass *f;
+
+  filterptr () {f=NULL;}
+  ~filterptr () {}
+};
+
+bool operator==(const filterptr &x, const filterptr &y);
+bool operator<(const filterptr &x, const filterptr &y);
+
+
+typedef map<text_t, filterptr, lttext_t> filterptrmap;
+
+// contains a list of filters indexed by their name
+class filtermapclass {
+protected:
+  filterptrmap filterptrs;
+
+public:
+  // type support for filterptrmap
+  typedef filterptrmap::iterator iterator;
+  typedef filterptrmap::const_iterator const_iterator;
+  typedef filterptrmap::reference reference;
+  typedef filterptrmap::const_reference const_reference;
+  typedef filterptrmap::size_type size_type;
+
+  typedef filterptrmap::difference_type difference_type;
+  typedef filterptrmap::const_reverse_iterator const_reverse_iterator;
+  typedef filterptrmap::reverse_iterator reverse_iterator;
+  
+  // basic container support
+  iterator begin () {return filterptrs.begin();}
+  const_iterator begin () const {return filterptrs.begin();}
+  iterator end () {return filterptrs.end();}
+  const_iterator end () const {return filterptrs.end();}
+
+  void erase(iterator pos) {filterptrs.erase(pos);}
+  void erase(iterator first, iterator last) {filterptrs.erase(first, last);}
+  filtermapclass &operator=(const filtermapclass &x) {filterptrs=x.filterptrs;return *this;}
+
+  bool empty () const {return filterptrs.empty();}
+  size_type size() const {return filterptrs.size();}
+
+
+  // added functionality
+  void clear () {filterptrs.erase(filterptrs.begin(),filterptrs.end());}
+
+  // thefilter remains the property of the calling code but
+  // should not be deleted until it is removed from this list.
+  void addfilter (filterclass *thefilter);
+
+  // getfilter will return NULL if the filter could not be found
+  filterclass *getfilter (const text_t &key);
+};
+
+
+
+// some useful functions for dealing with document sets
+
+// intersect places the result in set1
+void intersect (text_tset &set1, const text_tset &set2);
+void intersect (text_tarray &set1, const text_tset &set2);
+void intersect (text_tarray &set1, const text_tarray &set2);
+
+// tests to see if el is in set
+bool in_set (const text_tset &set1, const text_t &el);
+bool in_set (const text_tarray &set1, const text_t &el);
+
+#endif
Index: /tags/gsdl-21/gsdl/src/colservr/maptools.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/colservr/maptools.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/colservr/maptools.cpp	(revision 699)
@@ -0,0 +1,196 @@
+/**********************************************************************
+ *
+ * maptools.cpp -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.3  1999/09/07 04:57:22  sjboddie
+   added gpl notice
+
+   Revision 1.2  1999/08/31 22:39:28  rjmcnab
+   Changes for AIX.
+
+   Revision 1.1  1999/03/09 20:53:13  rjmcnab
+   Moved from src/recpt.
+
+   Revision 1.1  1999/03/08 19:45:53  rjmcnab
+   Initial revision.
+
+ */
+
+
+#include "maptools.h"
+
+
+
+void stringmap::clear () {
+  fromorder.erase (fromorder.begin(), fromorder.end());
+  mapfrom.erase (mapfrom.begin(), mapfrom.end());
+}
+
+void stringmap::importmap (const text_tarray &maparray) {
+  clear ();
+  
+  text_tarray::const_iterator here = maparray.begin();
+  text_tarray::const_iterator end = maparray.end();
+  text_t from;
+  text_t to;
+
+  while (here != end) {
+    splitmapentry (*here, from, to);
+    fromorder.push_back (from);
+    mapfrom[from] = to;
+    here++;
+  }
+}
+
+void stringmap::exportmap (text_tarray &maparray) const {
+  maparray.erase (maparray.begin(), maparray.end());
+
+  text_tarray::const_iterator here = fromorder.begin();
+  text_tarray::const_iterator end = fromorder.end();
+  text_tmap::const_iterator toptr;
+  text_t tempmap;
+
+  while (here != end) {
+    toptr = mapfrom.find (*here);
+    if (toptr != mapfrom.end()) {
+      joinmapentry ((*toptr).first, (*toptr).second, tempmap);
+      maparray.push_back (tempmap);
+    }
+
+    here++;
+  }
+}
+
+void stringmap::getfromarray (text_tarray &fromarray) const {
+  fromarray = fromorder;
+}
+
+void stringmap::gettoarray (text_tarray &toarray) const {
+  toarray.erase(toarray.begin(), toarray.end());
+
+  text_tarray::const_iterator here = fromorder.begin();
+  text_tarray::const_iterator end = fromorder.end();
+  text_tmap::const_iterator toptr;
+
+  while (here != end) {
+    toptr = mapfrom.find (*here);
+    if (toptr != mapfrom.end()) {
+      toarray.push_back((*toptr).second);
+    }
+    here++;
+  }
+}
+
+bool stringmap::fromexists (const text_t &from) const {
+  return (mapfrom.find (from) != mapfrom.end ());
+}
+
+bool stringmap::toexists (const text_t &to) const {
+  text_tarray::const_iterator here = fromorder.begin();
+  text_tarray::const_iterator end = fromorder.end();
+  text_tmap::const_iterator toptr;
+
+  while (here != end) {
+    toptr = mapfrom.find (*here);
+    if (toptr != mapfrom.end() && (*toptr).second == to) return true;
+    here++;
+  }
+  
+  // it was not found
+  return false;
+}
+
+bool stringmap::from2to (const text_t &from, text_t &to) const {
+  text_tmap::const_iterator toptr = mapfrom.find (from);
+  if (toptr == mapfrom.end()) {
+    to.clear();
+    return false;
+  }
+
+  to = (*toptr).second;
+  return true;
+}
+
+bool stringmap::to2from (const text_t &to, text_t &from) const {
+  text_tarray::const_iterator here = fromorder.begin();
+  text_tarray::const_iterator end = fromorder.end();
+  text_tmap::const_iterator toptr;
+
+  while (here != end) {
+    toptr = mapfrom.find (*here);
+    if (toptr != mapfrom.end() && (*toptr).second == to) {
+      from = (*toptr).first;
+      return true;
+    }
+    here++;
+  }
+  
+  // it was not found
+  from.clear();
+  return false;
+}
+
+
+
+void splitmapentry (const text_t &mapentry, text_t &from, text_t &to) {
+  from.clear ();
+  to.clear();
+
+  text_t::const_iterator here = mapentry.begin();
+  text_t::const_iterator end = mapentry.end();
+
+  // get the "from" part of the map
+  while (here != end) {
+    if (*here == '-') {
+      here++;
+      if (here != end && *here == '>') {
+	here++;
+	break; // found "->"
+      }
+
+      // didn't find "->"
+      from.push_back('-');
+
+    } else {
+      from.push_back(*here);
+      here++;
+    }
+  }
+
+  // get the "to" part of the map
+  while (here != end) {
+    to.push_back(*here);
+    here++;
+  }
+}
+
+void joinmapentry (const text_t &from, const text_t &to, text_t &mapentry) {
+  mapentry = from;
+  mapentry += "->";
+  mapentry += to;
+}
Index: /tags/gsdl-21/gsdl/src/colservr/maptools.h
===================================================================
--- /tags/gsdl-21/gsdl/src/colservr/maptools.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/colservr/maptools.h	(revision 699)
@@ -0,0 +1,67 @@
+/**********************************************************************
+ *
+ * maptools.h -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef MAPTOOLS_H
+#define MAPTOOLS_H
+
+
+#include "gsdlconf.h"
+#include "text_t.h"
+
+// stringmap is a mapping which remembers the original order 
+// of the map, but is very fast to look up.
+class stringmap {
+protected:
+  text_tarray fromorder;
+  text_tmap mapfrom;
+
+public:
+  void clear ();
+
+  void importmap (const text_tarray &maparray);
+  void exportmap (text_tarray &maparray) const;
+
+  void getfromarray (text_tarray &fromarray) const;
+  void gettoarray (text_tarray &toarray) const;
+  
+  bool fromexists (const text_t &from) const;
+  bool toexists (const text_t &to) const;
+
+  // from2to and to2from return false if the key (the "from"
+  // field or the "to" field) cannot be found
+  bool from2to (const text_t &from, text_t &to) const;
+  bool to2from (const text_t &to, text_t &from) const;
+};
+
+
+// a few useful functions used by stringmap
+void splitmapentry (const text_t &mapentry, text_t &from, text_t &to);
+void joinmapentry (const text_t &from, const text_t &to, text_t &mapentry);
+
+
+#endif
Index: /tags/gsdl-21/gsdl/src/colservr/mggdbmsource.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/colservr/mggdbmsource.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/colservr/mggdbmsource.cpp	(revision 699)
@@ -0,0 +1,385 @@
+/**********************************************************************
+ *
+ * mggdbmsource.cpp -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.16  1999/10/10 08:20:36  sjboddie
+   - metadata now returns map rather than array
+   - redesigned browsing support (although it's not finished so
+   won't currently work ;-)
+
+   Revision 1.15  1999/09/07 04:57:22  sjboddie
+   added gpl notice
+
+   Revision 1.14  1999/08/31 22:40:44  rjmcnab
+   A couple of gdbm changes.
+
+   Revision 1.13  1999/08/13 04:20:27  sjboddie
+   added ability to get return all available metadata when 'fields' is empty
+
+   Revision 1.12  1999/07/07 06:17:47  rjmcnab
+   broke search_index into index+subcollection+language
+   within mgsearch
+
+   Revision 1.11  1999/07/01 03:49:54  rjmcnab
+   fixed a small warning.
+
+   Revision 1.10  1999/06/29 23:06:07  sjboddie
+   Fixed up default index for get_document
+
+   Revision 1.9  1999/06/16 02:00:34  sjboddie
+   Few changes to get getParent filter option to return metadata of
+   parents as well as current OID
+
+   Revision 1.8  1999/05/10 03:43:48  sjboddie
+   lots of changes to lots of files - getting document action going
+
+   Revision 1.7  1999/04/30 02:00:47  sjboddie
+   lots of stuff to do with getting documentaction working
+
+   Revision 1.6  1999/04/21 22:40:44  sjboddie
+   made another change to the one I just committed. if requested metadata doesn't
+   exist it now puts an empty string in the response array so we don't always
+   have to test that a value exists before using it.
+
+   Revision 1.5  1999/04/21 05:23:46  sjboddie
+
+   changed the way metadata is returned
+
+   Revision 1.4  1999/04/19 23:56:07  rjmcnab
+   Finished the gdbm metadata stuff
+
+   Revision 1.3  1999/04/12 10:30:33  rjmcnab
+   Made a little more progress.
+
+   Revision 1.2  1999/04/12 05:21:51  rjmcnab
+   Started on a mg and gdbm source.
+
+   Revision 1.1  1999/04/12 03:40:40  rjmcnab
+   Initial revision.
+
+ */
+
+
+#include "mggdbmsource.h"
+#include "fileutil.h"
+#include "OIDtools.h"
+
+
+mggdbmsourceclass::mggdbmsourceclass () {
+  gdbmptr = NULL;
+  mgsearchptr = NULL;
+}
+
+mggdbmsourceclass::~mggdbmsourceclass () {
+}
+
+void mggdbmsourceclass::configure (const text_t &key, const text_tarray &cfgline) {
+  if (cfgline.size() >= 1) {
+    const text_t &value = cfgline[0];
+
+    if (key == "collection") collection = value;
+    else if (key == "collectdir") collectdir = value;
+    else if (key == "gsdlhome") gsdlhome = value;
+  }
+
+  if (key == "indexmap") {
+    indexmap.importmap (cfgline);
+
+  } else if (key == "defaultindex") {
+    indexmap.from2to (cfgline[0], defaultindex);
+
+  } else if (key == "subcollectionmap") {
+    subcollectionmap.importmap (cfgline);
+
+  } else if (key == "defaultsubcollection") {
+    subcollectionmap.from2to (cfgline[0], defaultsubcollection);
+
+  } else if (key == "languagemap") {
+    languagemap.importmap (cfgline);
+
+  } else if (key == "defaultlanguage") 
+    languagemap.from2to (cfgline[0], defaultlanguage);
+}
+
+bool mggdbmsourceclass::init (ostream &logout) {
+  outconvertclass text_t2ascii;
+  
+  if (!sourceclass::init (logout)) return false;
+  
+  // get the collection directory name
+  if (collectdir.empty()) {
+    collectdir = filename_cat (gsdlhome, "collect", collection);
+  }
+  
+  // get the filename for the database and make sure it exists
+  gdbm_filename = filename_cat(collectdir,"index","text",collection);
+
+#ifdef _LITTLE_ENDIAN
+  gdbm_filename += ".ldb";
+#else
+  gdbm_filename += ".bdb";
+#endif
+  if (!file_exists(gdbm_filename)) {
+    logout << text_t2ascii
+	   << "error: gdbm database \""
+	   << gdbm_filename << "\" does not exist\n\n";
+    return false;
+  }
+  
+  return true;
+}
+
+bool mggdbmsourceclass::translate_OID (const text_t &OIDin, text_t &OIDout, 
+				       comerror_t &err, ostream &logout) {
+
+  outconvertclass text_t2ascii;
+
+  err = noError;
+  if (gdbmptr == NULL) {
+    // most likely a configuration problem
+    logout << text_t2ascii 
+	   << "configuration error: mggdbmsource contains a null gdbmclass\n\n";
+    err = configurationError;
+    return true;
+  }
+
+  // open the database
+  gdbmptr->setlogout(&logout);
+  if (!gdbmptr->opendatabase (gdbm_filename, GDBM_READER, 100, false)) {
+    // most likely a system problem (we have already checked that the
+    // gdbm database exists)
+    logout << text_t2ascii
+	   << "system problem: open on gdbm database \""
+	   << gdbm_filename << "\" failed\n\n";
+    err = systemProblem;
+    return true;
+  }
+
+  infodbclass info;
+  OIDout = gdbmptr->translate_OID (OIDin, info);
+  return true;
+}
+
+bool mggdbmsourceclass::get_metadata (const text_t &/*requestParams*/, const text_t &/*refParams*/, 
+				      bool getParents, const text_tset &fields, 
+				      const text_t &OID, MetadataInfo_tmap &metadata, 
+				      comerror_t &err, ostream &logout) {
+  outconvertclass text_t2ascii;
+
+  metadata.erase(metadata.begin(), metadata.end());
+
+  err = noError;
+  if (gdbmptr == NULL) {
+    // most likely a configuration problem
+    logout << text_t2ascii 
+	   << "configuration error: mggdbmsource contains a null gdbmclass\n\n";
+    err = configurationError;
+    return true;
+  }
+
+  // open the database
+  gdbmptr->setlogout(&logout);
+  if (!gdbmptr->opendatabase (gdbm_filename, GDBM_READER, 100, false)) {
+    // most likely a system problem (we have already checked that the
+    // gdbm database exists)
+    logout << text_t2ascii
+	   << "system problem: open on gdbm database \""
+	   << gdbm_filename << "\" failed\n\n";
+    err = systemProblem;
+    return true;
+  }
+
+  // get the metadata - if getParents is set we need to get 
+  // info for all parents of OID as well as OID
+  vector<infodbclass> info_array;
+  text_tarray OIDs;
+  if (getParents) get_parents_array (OID, OIDs);
+  OIDs.push_back (OID);
+
+  text_tarray::const_iterator this_OID = OIDs.begin();
+  text_tarray::const_iterator end_OID = OIDs.end();
+
+  while (this_OID != end_OID) {
+    infodbclass info;
+    if (!gdbmptr->getinfo(*this_OID, info)) return false;
+
+    // adjust the metadata
+    text_t &contains = info["contains"];
+    if (contains.empty()) info["haschildren"] = 0;
+    else info["haschildren"] = 1;
+    contains.clear();
+
+    info_array.push_back(info);
+    this_OID ++;
+  }
+
+  // if fields set is empty we want to get all available metadata
+  text_tset tfields = fields;
+  if (tfields.empty() && !info_array.empty()) {
+    infodbclass::iterator t_info = info_array[0].begin(); 
+    infodbclass::iterator e_info = info_array[0].end(); 
+    while (t_info != e_info) {
+      if ((*t_info).first != "contains")
+	tfields.insert ((*t_info).first);
+      t_info ++;
+    }
+    tfields.insert ("hasnext");
+    tfields.insert ("hasprevious");
+  }
+    
+  // collect together the metadata
+  bool donenextprevtest = false;
+  bool hasnext=false, hasprevious=false;
+  MetadataInfo_t this_metadata;
+  text_tarray *pos_metadata;
+  text_tset::const_iterator fields_here = tfields.begin();
+  text_tset::const_iterator fields_end = tfields.end();
+  while (fields_here != fields_end) {
+    this_metadata.clear();
+    this_metadata.isRef = false;
+
+    vector<infodbclass>::reverse_iterator this_info = info_array.rbegin(); 
+    vector<infodbclass>::reverse_iterator end_info = info_array.rend(); 
+    MetadataInfo_t *tmetaptr = &this_metadata;
+    while (this_info != end_info) {
+
+      pos_metadata = (*this_info).getmultinfo(*fields_here);
+    
+      if ((*fields_here == "hasnext" || *fields_here == "hasprevious")) {
+	
+	// collect metadata
+	if (!donenextprevtest) {
+	  donenextprevtest = true;
+	  
+	  // cache parent contents array
+	  text_t thisparent = get_parent (OID);
+	  if (thisparent != parentOID) {
+	    parentOID = thisparent;
+	    parentcontents.erase(parentcontents.begin(), parentcontents.end());
+	    if (gdbmptr->getinfo(parentOID, parentinfo)) {
+	      text_t &parentinfocontains = parentinfo["contains"];
+	      if (!parentinfocontains.empty())
+		splitchar (parentinfocontains.begin(), parentinfocontains.end(), 
+			   ';', parentcontents);
+	    }
+	  }
+	  
+	  // do tests
+	  text_tarray::const_iterator parentcontents_here = parentcontents.begin();
+	  text_tarray::const_iterator parentcontents_end = parentcontents.end();
+	  text_t shrunk_OID = OID;
+	  shrink_parent (shrunk_OID);
+	  while (parentcontents_here != parentcontents_end) {
+	    if (*parentcontents_here == shrunk_OID) {
+	      if (parentcontents_here == parentcontents.begin()) hasprevious = false;
+	      else hasprevious = true;
+	      
+	      parentcontents_here++;
+	      
+	      if (parentcontents_here == parentcontents.end()) hasnext = false;
+	      else hasnext = true;
+	      
+	      break;
+	    }  
+	    
+	    parentcontents_here ++;
+	  }
+	}  
+	
+	// fill in metadata
+	if ((*fields_here == "hasnext" && hasnext) ||
+	    (*fields_here == "hasprevious" && hasprevious))
+	  tmetaptr->values.push_back("1");
+	else
+	  tmetaptr->values.push_back("0");
+	
+      }
+      else if (pos_metadata != NULL && *fields_here != "contains")
+	tmetaptr->values = *pos_metadata;
+      else
+	tmetaptr->values.push_back("");
+
+      this_info ++;
+      if (this_info != end_info) {
+	tmetaptr->parent = new MetadataInfo_t();
+	tmetaptr = tmetaptr->parent;
+      }
+    }
+    metadata[*fields_here] = this_metadata;
+    
+    fields_here++;
+  }
+
+  return true;
+}
+
+
+bool mggdbmsourceclass::get_document (const text_t &OID, text_t &doc, 
+				      comerror_t &err, ostream &logout) {
+
+  outconvertclass text_t2ascii;
+
+  err = noError;
+  if (gdbmptr == NULL) {
+    // most likely a configuration problem
+    logout << text_t2ascii 
+	   << "configuration error: mggdbmsource contains a null gdbmclass\n\n";
+    err = configurationError;
+    return true;
+  }
+
+  // open the database
+  gdbmptr->setlogout(&logout);
+  if (!gdbmptr->opendatabase (gdbm_filename, GDBM_READER, 100, false)) {
+    // most likely a system problem (we have already checked that the
+    // gdbm database exists)
+    logout << text_t2ascii
+	   << "system problem: open on gdbm database \""
+	   << gdbm_filename << "\" failed\n\n";
+    err = systemProblem;
+    return true;
+  }
+
+  text_t tOID = OID;
+  if (needs_translating (OID)) 
+    translate_OID (OID, tOID, err, logout);
+  infodbclass info;
+  if (!gdbmptr->getinfo(tOID, info)) return false;
+  
+  if (info["hastxt"].getint() == 1) {
+    int docnum = info["docnum"].getint();
+    
+    // set the collection directory
+    mgsearchptr->setcollectdir (collectdir);
+    
+    // get the text
+    mgsearchptr->docTargetDocument(defaultindex, defaultsubcollection,
+				   defaultlanguage, collection, docnum, doc);
+  }
+  return true;
+}
Index: /tags/gsdl-21/gsdl/src/colservr/mggdbmsource.h
===================================================================
--- /tags/gsdl-21/gsdl/src/colservr/mggdbmsource.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/colservr/mggdbmsource.h	(revision 699)
@@ -0,0 +1,89 @@
+/**********************************************************************
+ *
+ * mggdbmsource.h -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef MGGDBMSOURCE_H
+#define MGGDBMSOURCE_H
+
+#include "gsdlconf.h"
+#include "text_t.h"
+#include "comtypes.h"
+#include "infodbclass.h"
+#include "maptools.h"
+#include "mgsearch.h"
+#include "source.h"
+
+
+class mggdbmsourceclass : public sourceclass {
+protected:
+  text_t gsdlhome;
+  text_t collection;
+  text_t collectdir;
+
+  // these maps are only needed to convert the default indexes
+  stringmap indexmap;
+  stringmap subcollectionmap;
+  stringmap languagemap;
+
+  text_t defaultindex;
+  text_t defaultsubcollection;
+  text_t defaultlanguage;
+
+  text_t parentOID;
+  infodbclass parentinfo;
+  text_tarray parentcontents;
+
+  text_t gdbm_filename;
+  gdbmclass *gdbmptr;
+  
+  mgsearchclass *mgsearchptr;
+
+public:
+  mggdbmsourceclass ();
+  virtual ~mggdbmsourceclass ();
+
+  // the gdbmptr remains the responsability of the calling code
+  void set_gdbmptr (gdbmclass *thegdbmptr) {gdbmptr=thegdbmptr;}
+
+  // the mgsearchptr remains the responsability of the calling code
+  void set_mgsearchptr (mgsearchclass *themgsearchptr) {mgsearchptr=themgsearchptr;}
+
+  void configure (const text_t &key, const text_tarray &cfgline);
+  bool init (ostream &logout);
+  bool translate_OID (const text_t &OIDin, text_t &OIDout, 
+		      comerror_t &err, ostream &logout);
+  bool get_metadata (const text_t &requestParams, const text_t &refParams, 
+		     bool getParents, const text_tset &fields, 
+		     const text_t &OID, MetadataInfo_tmap &metadata, 
+		     comerror_t &err, ostream &logout);
+  bool get_document (const text_t &OID, text_t &doc, 
+		     comerror_t &err, ostream &logout);
+};
+
+
+
+#endif
Index: /tags/gsdl-21/gsdl/src/colservr/mgq.c
===================================================================
--- /tags/gsdl-21/gsdl/src/colservr/mgq.c	(revision 699)
+++ /tags/gsdl-21/gsdl/src/colservr/mgq.c	(revision 699)
@@ -0,0 +1,773 @@
+/**********************************************************************
+ *
+ * mgq.c -- cut-dowm version of mgquery
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.8  1999/09/09 04:12:22  sjboddie
+   added GPL header
+
+   Revision 1.7  1999/07/01 09:29:18  rjmcnab
+   Changes for better reporting of number documents which match a query. Changes
+   should still work as before with older versions of mg.
+
+   Revision 1.6  1999/07/01 03:52:05  rjmcnab
+   Added a function to get the equivalent terms of a query term. I also
+   fixed a small bug that was causing massive slowdown :-^
+
+   Revision 1.5  1999/06/30 04:04:11  rjmcnab
+   made stemming functions available from mgsearch and made the stems
+   for the query terms available in queryinfo
+
+   Revision 1.4  1999/06/28 08:56:29  rjmcnab
+   A bit of hacking to remove the restriction that the index to get
+   a document must be a level 2 index. Now both level 2 and level 3
+   indexes can be used to get the text of a document.
+
+   Revision 1.3  1999/01/19 01:38:16  rjmcnab
+
+   Made the source more portable.
+
+   Revision 1.2  1999/01/12 01:51:02  rjmcnab
+
+   Standard header.
+
+   Revision 1.1  1999/01/08 09:02:22  rjmcnab
+
+   Moved from src/library.
+
+ */
+
+
+#include "mgq.h"
+
+
+#include <stdio.h>
+#include <string.h>
+/* #include <io.h> */
+#include <fcntl.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+  
+#include "sysfuncs.h"
+  
+#include "messages.h"
+#include "memlib.h"
+  
+#include "invf.h"
+#include "text.h"
+#include "lists.h"
+#include "backend.h"
+#include "environment.h"
+#include "globals.h"
+#include "mg_errors.h"
+#include "commands.h"
+#include "text_get.h"
+#include "term_lists.h"
+#include "local_strings.h"
+
+#include "words.h"
+#include "stemmer.h"
+#include "stem_search.h"
+  
+#ifdef __cplusplus
+}
+#endif
+
+
+#include "mgq.h"
+
+/* get a reasonable database cache size */
+#ifndef MAXNUMDATABASEINFO
+#  ifdef GSDLSERVER
+#    define MAXNUMDATABASEINFO 10
+#  else
+#    define MAXNUMDATABASEINFO 2
+#  endif
+#endif
+
+#define MAXCOLLECTIONLEN  16
+#define MAXMGDIRLEN       256
+#define MAXGENSUFFIXLEN   256
+#define MAXTEXTSUFFIXLEN  256
+
+typedef struct DatabaseInfo {
+  int accessnum; /* -1 = invalid record */
+  char collection[MAXCOLLECTIONLEN];
+  char mgdir[MAXMGDIRLEN];
+  char gensuffix[MAXGENSUFFIXLEN];
+  char textsuffix[MAXTEXTSUFFIXLEN];
+  query_data *qd;
+} DatabaseInfo;
+
+
+/* globals needed by some vague part of mg... */
+FILE *OutFile = NULL, *InFile = NULL;
+int OutPipe = 0, InPipe = 0;
+int Quitting = 0;
+
+/* globals needed to handle loading of databases */
+static int cur_cachenum = -1;
+
+/* globals needed by the database cache */
+static DatabaseInfo dbcache[MAXNUMDATABASEINFO];
+static int cache_nextaccessnum = 0;
+static int cache_numloaded = 0;
+
+
+
+#if defined(PARADOCNUM) ||  defined(NZDL)
+static int GetDocNumFromParaNum(query_data *qd, int paranum) {
+  int Documents = qd->td->cth.num_of_docs;
+  int *Paragraph = qd->paragraph;
+  int low = 1, high = Documents;
+  int mid = (low+high)/2;
+
+  while ((mid = (low+high)/2) >=1 && mid <= Documents)
+    {
+      if (paranum > Paragraph[mid])
+        low = mid+1;
+      else if (paranum <= Paragraph[mid-1])
+        high = mid-1;
+      else
+        return mid;
+    }
+  FatalError(1, "Bad paragraph number.\n");
+  return 0;
+}
+
+static int GetParaNumFromDocNum(query_data *qd, int docnum) {
+  int Documents = qd->td->cth.num_of_docs;
+  int *Paragraph = qd->paragraph;
+
+  if (docnum > 0 && docnum <= Documents)
+    return Paragraph[docnum-1]+1;
+  return 0;
+}
+#endif
+
+
+/*****************************************************************************/
+
+static void MGQError(char *emsg)
+{
+  fprintf(stderr,"Fatal error: %s\n", emsg);
+  exit(1);
+}
+
+static int ProcessDocs (query_data * qd, int skip, int howmany, 
+			enum result_kinds kind, 
+			int (*sender)(char *,int,int,float,void *), void *ptr) {
+  int max_buf = 0, output_failure = 0;
+  int DocCount = 0;
+  int need_text = (kind == result_docs);
+
+  /* skip the requested number of documents */
+  while (skip > 0) {
+    if (!NextDoc(qd)) return 0;
+    skip--;
+  }
+
+  /* find out the maximum size for the text buffer */
+  if (need_text) max_buf = atoi (GetDefEnv ("buffer", "1048576"));
+
+  /* process each document */
+  do {
+    u_char *UDoc = NULL;
+    unsigned long ULen=0;
+
+#if defined(PARADOCNUM) ||  defined(NZDL)
+    /* adjust the document number for paragraph level result_docs */
+    /* this is a bit of a hack ... */
+    if (kind==result_docs && qd->id->ifh.InvfLevel == 3 &&
+	qd->DL != NULL && (int)qd->doc_pos < (int)qd->DL->num)
+      qd->DL->DE[qd->doc_pos].DocNum = GetParaNumFromDocNum(qd, qd->DL->DE[qd->doc_pos].DocNum);
+#endif
+    
+    if (need_text) {
+      /* load the compressed text */
+      if (LoadCompressedText (qd, max_buf))
+	MGQError("Unable to load compressed text(memory?).");
+
+      /* uncompress the loaded text */
+      UDoc = GetDocText (qd, &ULen);
+      if (UDoc == NULL) MGQError("UDoc is unexpectedly NULL");
+    }
+      
+    if (UDoc != NULL  ||  kind == result_docnums) {
+      int docnum = GetDocNum(qd);
+#if defined(PARADOCNUM) ||  defined(NZDL)
+      if (qd->id->ifh.InvfLevel == 3) docnum = GetDocNumFromParaNum(qd, docnum);
+#endif
+      switch (kind) {
+      case result_docnums:
+	if (sender != NULL)
+	  output_failure = (*sender)("",0,docnum,GetDocWeight(qd),ptr);
+	break;
+      case result_docs:
+	if (sender != NULL)
+	  output_failure = (*sender)((char *)UDoc,ULen,docnum,GetDocWeight(qd),ptr);
+	break;
+      default:
+	break;
+      }
+    }
+    DocCount++;
+      
+  } while (NextDoc (qd) && output_failure == 0 && --howmany > 0);
+  
+  /*  if (need_text) FreeTextBuffer (qd);*/
+  
+  return (DocCount);
+}
+
+
+static void send_query_term_freqs(QueryTermList *qtl,
+				  int (*sender)(char *,int,int,float,void *), void *ptr)
+{
+  int i = 0;
+  for (i = 0; i < qtl->num; i++)
+    if (sender != NULL) {
+      /*      word = word2str(qtl->QTE[i].Term); 
+	      (* sender)(word, strlen(word), qtl->QTE[i].Count, (float)0.0, ptr); */
+      (* sender)((char *)(qtl->QTE[i].Term+1), qtl->QTE[i].Term[0], 
+		 qtl->QTE[i].Count, (float)0.0, ptr);
+    }
+}
+
+
+static void send_terms (TermList * qtl, 
+			int (*sender)(char *,int,int,float,void *), void *ptr)
+{
+  int i = 0;
+  if (sender == NULL) return;
+  for (i = 0; i < qtl->num; i++)
+    {
+      /*      word = word2str(qtl->TE[i].Word);
+	      (* sender)(word, strlen(word), qtl->TE[i].Count, (float)0.0, ptr);*/
+      (* sender)((char *)(qtl->TE[i].Word+1), qtl->TE[i].Word[0], 
+		 qtl->TE[i].Count, (float)0.0, ptr);
+    }
+}
+
+
+/* MoreDocs () */
+/* Displays all documents in list  DocList. */
+/* Documents are fetched, then decompressed and displayed according to the */
+/* format implied in  FormString(). */
+
+static void
+MoreDocs (query_data * qd, enum result_kinds kind, 
+	  int skip, int howmany,
+	  int (*sender)(char *,int,int,float,void *), void *ptr)
+{
+  qd->num_of_ans = qd->DL->num;
+  switch (kind) {
+  case result_docs:
+  case result_docnums:
+    if (qd->num_of_ans > 0)
+      ProcessDocs (qd, skip, howmany, kind, sender, ptr);
+    break;
+  case result_termfreqs:
+    send_query_term_freqs(qd->QTL, sender, ptr);
+    break;
+  case result_terms:
+    send_terms(qd->TL, sender, ptr);
+    break;
+  }
+}
+
+
+
+
+
+
+/******************************************
+ * functions to handle the database cache *
+ ******************************************/
+
+/* init_dbcache should be called at the start of each */
+/* function which deals with the database cache */
+static void init_dbcache (void) {
+  static int dbcacheinited = 0;
+  int i = 0;
+
+  if (dbcacheinited) return;
+
+  cache_numloaded = 0;
+
+  for (i=0; i<MAXNUMDATABASEINFO; i++) {
+    dbcache[i].accessnum = -1;
+    dbcache[i].collection[0] = '\0';
+    dbcache[i].mgdir[0] = '\0';
+    dbcache[i].gensuffix[0] = '\0';
+    dbcache[i].textsuffix[0] = '\0';
+    dbcache[i].qd = NULL;
+  }
+
+  dbcacheinited = 1;
+}
+
+/* returns the next cache access number and increments it */
+static int get_next_accessnum (void) {
+  return cache_nextaccessnum++;
+}
+
+/* get_free_dbcache returns the cache number which */
+/* was used the longest time ago */
+/* init_dbcache should be called before this function */
+static int get_free_dbcache (void) {
+  int i = 0;
+  int minaccessnum = cache_nextaccessnum; /* the current max */
+  int minpos = 0;
+
+  for (i=0; i<MAXNUMDATABASEINFO; i++) {
+    if (dbcache[i].accessnum < minaccessnum) {
+      minaccessnum = dbcache[i].accessnum;
+      minpos = i;
+    }
+  }
+  
+  return minpos;
+}
+
+/* search_collect will search for an index which */
+/* belongs to a certain collection It returns -1 if none could be found. */
+/* init_dbcache should be called before this function */
+static int search_collect (char *collection) {
+  int i = 0;
+
+  for (i=0; i<MAXNUMDATABASEINFO; i++) {
+    if ((dbcache[i].accessnum >= 0) &&
+	(dbcache[i].qd != NULL) &&
+	(strcmp (collection, dbcache[i].collection) == 0)
+	/* && (dbcache[i].qd->id->ifh.InvfLevel == 2)*/
+	) {
+      dbcache[i].accessnum = get_next_accessnum ();
+      return i;
+    }
+  }
+
+  return -1;
+}
+
+/* search_gensuffix will search for an index which */
+/* has a certain gensuffix. It returns -1 if none could be found. */
+/* init_dbcache should be called before this function */
+static int search_gensuffix (char *gensuffix) {
+  int i = 0;
+
+  for (i=0; i<MAXNUMDATABASEINFO; i++) {
+    if ((dbcache[i].accessnum >= 0) &&
+	(dbcache[i].qd != NULL) &&
+	(strcmp (gensuffix, dbcache[i].gensuffix) == 0)) {
+      dbcache[i].accessnum = get_next_accessnum ();
+      return i;
+    }
+  }
+
+  return -1;
+}
+
+/* unload_database will unload a certain entry within */
+/* the database cache, clearing it for furture use. */
+static void unload_database (int i) {
+  /* check to see if it contains anything */
+  if (dbcache[i].accessnum < 0 || dbcache[i].qd == NULL) 
+    return;
+
+  /* unload all the query information  */
+  FinishQuerySystem(dbcache[i].qd);
+  
+  /* reset all the db info */
+  dbcache[i].accessnum = -1;
+  dbcache[i].collection[0] = '\0';
+  dbcache[i].mgdir[0] = '\0';
+  dbcache[i].gensuffix[0] = '\0';
+  dbcache[i].textsuffix[0] = '\0';
+  dbcache[i].qd = NULL;
+
+  cache_numloaded--;
+  if (cache_numloaded < 0) cache_numloaded = 0;
+}
+
+/* cache_database will store the information about */
+/* a database in the database cache. */
+static void cache_database (int i, char *collection, char *mgdir, char *gensuffix,
+			    char *textsuffix, query_data *qd) {
+  /* make sure this entry has been unloaded first */
+  if (dbcache[i].accessnum >= 0 && dbcache[i].qd != NULL) 
+    unload_database (i);
+
+  /* store the db info */
+  dbcache[i].accessnum = get_next_accessnum ();
+  strcpy (dbcache[i].collection, collection);
+  strcpy (dbcache[i].mgdir, mgdir);
+  strcpy (dbcache[i].gensuffix, gensuffix);
+  strcpy (dbcache[i].textsuffix, textsuffix);
+  dbcache[i].qd = qd;
+
+  cache_numloaded++;
+}
+
+static void make_current (int i) {
+  /* see if it is the current index */
+  if (i == cur_cachenum) return;
+ 
+  /* unload the old index */
+  if (cur_cachenum >= 0) UninitEnv ();
+  cur_cachenum = -1;
+
+  /* make sure the new one is ok */
+  if (i < 0 || dbcache[i].accessnum < 0 || dbcache[i].qd == NULL)
+    return;
+
+  /* load the new one */
+
+  /* Initialise the environment with default values */
+  InitEnv ();
+  
+  SetEnv("mgdir",dbcache[i].mgdir,NULL);
+  SetEnv("mgname",dbcache[i].gensuffix,NULL);
+  SetEnv("textname",dbcache[i].textsuffix,NULL);
+  
+  PushEnv ();
+  
+  cur_cachenum = i;
+}
+
+
+
+/********************
+ * public functions *
+ ********************/
+
+int mgq_ask(char *line)
+{
+  query_data *qd = (query_data *)NULL;
+  char QueryType = QUERY_BOOLEAN;
+  char OutputType = QUERY_DOCNUMS;
+  char *LinePtr = (char *)NULL;
+
+  if (cur_cachenum == -1) return 0;
+  qd = dbcache[cur_cachenum].qd;
+  if (qd == NULL) return 0;
+  
+  ResetFileStats (qd);
+  qd->max_mem_in_use = qd->mem_in_use = 0;
+  qd->tot_hops_taken += qd->hops_taken;
+  qd->tot_num_of_ptrs += qd->num_of_ptrs;
+  qd->tot_num_of_accum += qd->num_of_accum;
+  qd->tot_num_of_terms += qd->num_of_terms;
+  qd->tot_num_of_ans += qd->num_of_ans;
+  qd->tot_text_idx_lookups += qd->text_idx_lookups;
+  qd->hops_taken = qd->num_of_ptrs = 0;
+  qd->num_of_accum = qd->num_of_ans = qd->num_of_terms = 0;
+  qd->text_idx_lookups = 0;
+  
+  LinePtr = ProcessCommands (line, qd);
+  if (CommandsErrorStr) {
+    fprintf (stderr, "%s\n", CommandsErrorStr);
+    return 0;
+  }
+  if (*LinePtr == '\0') return 1;
+  
+  FreeQueryDocs (qd);
+  
+  QueryType = get_query_type ();
+  OutputType = get_output_type ();
+  /* No point in hiliting words on a docnum query */
+  if (OutputType == OUTPUT_HILITE && QueryType == QUERY_DOCNUMS)
+    OutputType = OUTPUT_TEXT;
+  
+  switch (QueryType)
+    {
+    case QUERY_BOOLEAN:
+      {
+	char *maxdocs = (char *)NULL;
+	BooleanQueryInfo bqi;
+	maxdocs = GetDefEnv ("maxdocs", "all");
+	bqi.MaxDocsToRetrieve = strcmp (maxdocs, "all") ? atoi (maxdocs) : -1;
+        if (qd->sd->sdh.indexed)
+	  BooleanQuery (qd, line, &bqi, (BooleanEnv (GetEnv ("casefold"), 0) |
+					 (BooleanEnv (GetEnv ("stem"), 0) << 1)));
+	else
+	  BooleanQuery (qd, line, &bqi, qd->sd->sdh.stem_method);
+	/*	if (qd->sd->sdh.indexed) BooleanQuery (qd, line, &bqi, 3);
+	else BooleanQuery (qd, line, &bqi, qd->sd->sdh.stem_method); */
+	break;
+      }
+    case QUERY_APPROX:
+    case QUERY_RANKED:
+      {
+	char *maxdocs = (char *)NULL;
+	char *maxterms = (char *)NULL;
+	char *maxaccum = (char *)NULL;
+	RankedQueryInfo rqi;
+	maxdocs = GetDefEnv ("maxdocs", "all");
+	maxterms = GetDefEnv ("max_terms", "all");
+	maxaccum = GetDefEnv ("max_accumulators", "all");
+	rqi.Sort = BooleanEnv (GetEnv ("sorted_terms"), 0);
+	rqi.QueryFreqs = BooleanEnv (GetEnv ("qfreq"), 1);
+	rqi.Exact = QueryType == QUERY_RANKED;
+	rqi.MaxDocsToRetrieve = strcmp (maxdocs, "all") ? atoi (maxdocs) : -1;
+	rqi.MaxTerms = strcmp (maxterms, "all") ? atoi (maxterms) : -1;
+	rqi.MaxParasToRetrieve = rqi.MaxDocsToRetrieve;
+	if (qd->id->ifh.InvfLevel == 3 && GetEnv ("maxparas"))
+	  rqi.MaxParasToRetrieve = atoi (GetEnv ("maxparas"));
+	rqi.AccumMethod = toupper (*GetDefEnv ("accumulator_method", "A"));
+	rqi.MaxAccums = strcmp (maxaccum, "all") ? atoi (maxaccum) : -1;
+	rqi.HashTblSize = IntEnv (GetEnv ("hash_tbl_size"), 1000);
+	rqi.StopAtMaxAccum = BooleanEnv (GetEnv ("stop_at_max_accum"), 0);
+	rqi.skip_dump = GetEnv ("skip_dump");
+	RankedQuery (qd, line, &rqi);
+	break;
+      }
+    case QUERY_DOCNUMS:
+      {
+	DocnumsQuery (qd, line);
+	break;
+      }
+    }
+
+  return 1;
+}
+
+int mgq_numdocs(void)
+{
+  query_data *qd = NULL;
+  if (cur_cachenum == -1) return 0;
+  qd = dbcache[cur_cachenum].qd;
+  if (qd == NULL) return 0;
+
+  if (qd->DL) return qd->DL->num;
+  else return 0;
+}
+
+int mgq_numterms(void)
+{
+  query_data *qd = NULL;
+  if (cur_cachenum == -1) return 0;
+  qd = dbcache[cur_cachenum].qd;
+  if (qd == NULL) return 0;
+
+  if (qd->QTL) return qd->QTL->num;
+  else return 0;
+}
+
+int mgq_results(enum result_kinds kind,int skip,int howmany, int (*sender)(char *, int, int, float, void *), void *ptr)
+{
+  query_data *qd = NULL;
+  if (cur_cachenum == -1) return 0;
+  qd = dbcache[cur_cachenum].qd;
+  if (qd == NULL) return 0;
+
+  if (qd->DL) {
+    qd->doc_pos = 0;
+    MoreDocs(qd, kind, skip, howmany, sender, ptr);
+  }
+  return 0;
+}
+
+/* get all the terms that match wordstem using the current stemmer and */
+/* stemming method. The callback is the same style used by mgq_results */
+int mgq_equivterms (unsigned char *wordstem, int (*sender)(char *, int, int, float, void *),
+		    void *ptr) {
+  int stem_method = 0;
+  query_data *qd = NULL;
+  TermList *equivterms = NULL;  /* used for equivalent terms */
+
+  if (cur_cachenum == -1) return 0;
+  qd = dbcache[cur_cachenum].qd;
+  if (qd == NULL || wordstem == NULL || sender == NULL) return 0;
+
+  if (qd->sd->sdh.indexed) {
+    stem_method = BooleanEnv(GetEnv("casefold"),0) | (BooleanEnv(GetEnv("stem"),0) << 1);
+  } else {
+    stem_method = qd->sd->sdh.stem_method;
+  }
+
+  /* make the term list */
+  equivterms = MakeTermList (0);
+  
+  /* expand out this word */
+  if (FindWords (qd->sd, wordstem, stem_method, &equivterms) > 0) {
+    int i;
+    for (i=0; i < equivterms->num; i++) {
+      (* sender)((char *)(equivterms->TE[i].Word+1), equivterms->TE[i].Word[0], 
+		 equivterms->TE[i].Count, (float)0.0, ptr);
+    }
+  }
+
+  /* free the term list */
+  if (equivterms != NULL) FreeTermList (&equivterms);
+
+  return 0;
+}
+
+/* gets the total number of documents retrieved. If this is not available */
+/* it will set total_retrieved to 0 (even when it obviously isn't) */
+int mgq_docsretrieved (int *total_retrieved, int *is_approx) {
+  query_data *qd = NULL;
+
+  if (cur_cachenum == -1) return 0;
+  qd = dbcache[cur_cachenum].qd;
+  if (qd == NULL || total_retrieved == NULL || is_approx == NULL) return 0;
+
+  /* set default values */
+  *total_retrieved = 0;
+  *is_approx = 0;
+
+  if (qd->DL == NULL) return 0;
+
+  *total_retrieved = qd->DL->total_retrieved;
+  *is_approx = qd->DL->is_approx;
+
+  return 0;
+}
+
+
+/* use mgq_getmaxstemlen to determine the length of the word stems to pass */
+/* to mgq_stemword */
+int mgq_getmaxstemlen () {
+  return MAXSTEMLEN;
+}
+
+/* note: the stemming method and the stemmer come from the last query */
+/* "word" should be at least maxstemlen+1 long and it is a string that */
+/* starts with the string length */
+void mgq_stemword (unsigned char *word) {
+  int stem_method = 0;
+  query_data *qd = NULL;
+
+  if (cur_cachenum == -1) return;
+  qd = dbcache[cur_cachenum].qd;
+  if (qd == NULL || word == NULL) return;
+
+  if (qd->sd->sdh.indexed) {
+    stem_method = BooleanEnv(GetEnv("casefold"),0) | (BooleanEnv(GetEnv("stem"),0) << 1);
+  } else {
+    stem_method = qd->sd->sdh.stem_method;
+  }
+
+  stemmer (stem_method, qd->sd->sdh.stemmer_num, word);
+}
+
+
+
+int is_dbcache_full (void) {
+  init_dbcache ();
+  if (cache_numloaded >= MAXNUMDATABASEINFO) return 1;
+  return 0;
+}
+
+int load_database (char *collection, char *mgdir, 
+		   char *gensuffix, char *textsuffix) {
+  int i = 0;
+  query_data *qd = NULL;
+  /*  FILE *deb = NULL; */
+  init_dbcache ();
+
+  /* print out some debug information */
+/*  deb = fopen ("/home/rjmcnab/gsdl/etc/deb.txt", "a");
+  fprintf (deb, "\ncache_nextaccessnum: %i\n", cache_nextaccessnum);
+  fprintf (deb, "cache_numloaded: %i\n", cache_numloaded);
+  fprintf (deb, "cur_cachenum: %i\n", cur_cachenum);
+  fprintf (deb, "MAXNUMDATABASEINFO: %i\n\n", MAXNUMDATABASEINFO);
+  for (i=0; i<MAXNUMDATABASEINFO; i++) {
+    fprintf (deb, "Entry %i\n", i);
+    fprintf (deb, "  accessnum: %i\n", dbcache[i].accessnum);
+    fprintf (deb, "  collection: %s\n", dbcache[i].collection);
+    fprintf (deb, "  mgdir: %s\n", dbcache[i].mgdir);
+    fprintf (deb, "  gensuffix: %s\n", dbcache[i].gensuffix);
+    fprintf (deb, "  textsuffix: %s\n", dbcache[i].textsuffix);
+    fprintf (deb, "  qd: %x\n", (int)(dbcache[i].qd));
+  }
+  fclose (deb); */
+
+  /* search for the index */
+  i = search_gensuffix (gensuffix);
+  if (i >= 0) {
+    make_current (i);
+    return 1;
+  }
+
+  /* if there was a current database then the */
+  /* environment needs uninitialising */
+  make_current (-1);
+
+  /* get a free cache number */
+  i = get_free_dbcache ();
+  unload_database (i);
+
+  /* load the index */
+  qd = InitQuerySystem (mgdir, gensuffix, textsuffix, NULL);
+  if (qd == NULL) return 0;
+
+  /* cache this index */
+  cache_database (i, collection, mgdir, gensuffix, textsuffix, qd);
+
+  /* make this index current */
+  make_current (i);
+
+  return 1;
+}
+
+/* load_text_database tries to make an index of the */
+/* specified collection current */
+int load_text_database (char *collection) {
+  int i = 0;
+  init_dbcache ();
+
+  /* search for the index */
+  i = search_collect (collection);
+
+  /* return if none were found */
+  if (i < 0) return 0;
+  
+  /* make this index current */
+  make_current (i);
+  return 1;
+}
+
+void close_all_databases (void) {
+  int i = 0;
+  init_dbcache ();
+  
+  /* unload all active databases */
+  for (i=0; i<MAXNUMDATABASEINFO; i++) {
+    unload_database (i);
+  }
+
+  /* if there was a current database then the */
+  /* environment needs uninitialising */
+  make_current (-1);
+}
+
+
+
Index: /tags/gsdl-21/gsdl/src/colservr/mgq.h
===================================================================
--- /tags/gsdl-21/gsdl/src/colservr/mgq.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/colservr/mgq.h	(revision 699)
@@ -0,0 +1,80 @@
+/**********************************************************************
+ *
+ * mgq.h -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef MGQ_H
+#define MGQ_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+  
+enum result_kinds {
+  result_docs,      /* Return the documents found in last search */
+  result_docnums,   /* Return document id numbers and weights */
+  result_termfreqs, /* Return terms and frequencies */
+  result_terms      /* Return matching query terms */
+};
+
+int mgq_ask(char *line);
+int mgq_results(enum result_kinds kind, int skip, int howmany,
+		int (*sender)(char *, int, int, float, void *), void *ptr);
+int mgq_numdocs(void);
+int mgq_numterms(void);
+
+/* get all the terms that match wordstem using the current stemmer and */
+/* stemming method. The callback is the same style used by mgq_results */
+int mgq_equivterms (unsigned char *wordstem, int (*sender)(char *, int, int, float, void *),
+		    void *ptr);
+
+/* gets the total number of documents retrieved. If this is not available */
+/* it will set total_retrieved to 0 (even when it obviously isn't) */
+int mgq_docsretrieved (int *total_retrieved, int *is_approx);
+  
+/* use mgq_getmaxstemlen to determine the length of the word stems to pass */
+/* to mgq_stemword */
+int mgq_getmaxstemlen ();
+
+/* note: the stemming method and the stemmer come from the last query */
+/* "word" should be at least maxstemlen+1 long and it is a string that */
+/* starts with the string length */
+void mgq_stemword (unsigned char *word);
+
+  
+int is_dbcache_full (void);
+int load_database (char *collection, char *mgdir, char *gensuffix, char *textsuffix);
+int load_text_database (char *collection);
+void close_all_databases (void);
+
+
+#ifdef __cplusplus
+}
+#endif
+
+
+#endif
Index: /tags/gsdl-21/gsdl/src/colservr/mgsearch.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/colservr/mgsearch.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/colservr/mgsearch.cpp	(revision 699)
@@ -0,0 +1,600 @@
+/**********************************************************************
+ *
+ * mgsearch.cpp -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.22  1999/09/24 02:41:21  rjmcnab
+   change to use has_unicode_letdig in text_t
+
+   Revision 1.21  1999/09/21 21:41:41  sjboddie
+   fixed an error in what I committed last
+
+   Revision 1.20  1999/09/21 11:59:26  sjboddie
+   added Maxdocs queryfilter option (which may be -1 for 'all)
+
+   Revision 1.19  1999/09/07 22:52:52  rjmcnab
+   Seems to be an error in mg for retrieving documents using a paragraph
+   based index for some cases. Just added a work around (loads the default
+   index every time).
+
+   Revision 1.18  1999/09/07 04:57:22  sjboddie
+   added gpl notice
+
+   Revision 1.17  1999/08/31 22:42:41  rjmcnab
+   A couple of minor things.
+
+   Revision 1.16  1999/08/25 04:51:06  sjboddie
+   small change to allow for searching using boolean operators
+
+   Revision 1.15  1999/07/16 08:35:03  rjmcnab
+   Fixed a weird bug to do with a faulty case statement.
+
+   Revision 1.14  1999/07/16 03:42:22  sjboddie
+   changed isApprox
+
+   Revision 1.13  1999/07/16 00:12:46  sjboddie
+   removed all the old post-processing stuff
+
+   Revision 1.12  1999/07/07 06:17:47  rjmcnab
+   broke search_index into index+subcollection+language
+   within mgsearch
+
+   Revision 1.11  1999/07/05 21:06:43  rjmcnab
+   Disabled quoted strings.
+
+   Revision 1.10  1999/07/01 09:29:19  rjmcnab
+   Changes for better reporting of number documents which match a query. Changes
+   should still work as before with older versions of mg.
+
+   Revision 1.9  1999/07/01 03:54:48  rjmcnab
+   Added code to plug in the equivalent terms of each of the query terms.
+   Also added a function to get a raw utf8 encoded mg document (for speeding
+   up a phrase matching function)
+
+   Revision 1.8  1999/06/30 04:04:12  rjmcnab
+   made stemming functions available from mgsearch and made the stems
+   for the query terms available in queryinfo
+
+   Revision 1.7  1999/06/27 22:07:27  sjboddie
+   got rid of all the old functions for dealing with dir indexes
+
+   Revision 1.6  1999/06/09 00:41:32  sjboddie
+   phrase searching now uses case-folding if it's turned on
+
+   Revision 1.5  1999/02/21 22:31:35  rjmcnab
+
+   Removed locateinfo.
+
+   Revision 1.4  1999/02/03 01:13:27  sjboddie
+
+   Got interface to handle subcollections and language subcollections -
+   committed changes made to some of the collections
+
+   Revision 1.3  1999/01/19 01:38:17  rjmcnab
+
+   Made the source more portable.
+
+   Revision 1.2  1999/01/12 01:51:02  rjmcnab
+
+   Standard header.
+
+   Revision 1.1  1999/01/08 09:02:16  rjmcnab
+
+   Moved from src/library.
+
+ */
+
+
+#include "gsdlconf.h"
+#include "mgsearch.h"
+#include "fileutil.h"
+
+#include <string.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <ctype.h>
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\iostream>
+#elif defined(GSDL_USE_IOS_H)
+#  include <iostream.h>
+#else
+#  include <iostream>
+#endif
+
+#if defined(__WIN32__)
+// gdbm stuff
+#  include "autoconf.h"
+#  include "systems.h"
+#  include "gdbmconst.h"
+#  include "gdbm.h"
+#else
+#  include <gdbm.h>
+#endif
+
+  
+#include <assert.h>
+
+#include "mgq.h"
+// #include "locateinfo.h"
+#include "gsdlunicode.h"
+#include "unitool.h"
+
+
+/////////////
+// globals //
+/////////////
+
+static char *tempdoc = NULL;
+static int templen = 0;
+
+
+//////////////////////
+// useful functions //
+//////////////////////
+
+
+// input and output are in utf8
+text_t mgsearch_stemword (const text_t &word) {
+  // allocate working stem space
+  int maxstemlen = mgq_getmaxstemlen ();
+  unsigned char *word_stem = new unsigned char [maxstemlen + 2];
+  if (word_stem == NULL) return "";
+
+  // copy word to word_stem
+  int len = 0;
+  text_t::const_iterator here = word.begin();
+  text_t::const_iterator end = word.end();
+  while (len < maxstemlen && here != end) {
+    word_stem[len+1] = (unsigned char)(*here);
+    len++; here++;
+  }
+  word_stem[len+1] = '\0';
+  word_stem[0] = len;
+
+  mgq_stemword (word_stem);
+
+  // copy word_stem back to tempstr
+  text_t tempstr;
+  tempstr.setcarr((char *)(&word_stem[1]), word_stem[0]);
+
+  delete [] word_stem;
+  
+  return tempstr;
+}
+
+
+
+////////////////////////
+// callback functions //
+////////////////////////
+
+// This routine is called for each document found in a search
+// it assumes that cache_num is set up correctly to point to
+// a suitable result cache
+int ourquerycallback(char * /*UDoc*/, int /*ULen*/, int DocNum, 
+		     float Weight, void *info) {
+
+  
+  queryresultsclass *queryresults = (queryresultsclass * )info;
+
+  // append this entry to the document results
+  docresultclass docresult;
+  docresult.docnum = DocNum;
+  docresult.num_query_terms_matched = (int)(Weight/100.0); // will always be 0 on some versions of mg...
+  docresult.docweight = Weight - docresult.num_query_terms_matched*100;
+  
+  queryresults->docs.docset[DocNum] = docresult;
+  queryresults->docs.docorder.push_back(DocNum);
+  
+  return 0;
+}
+
+int termequivcallback(char *Word, int ULen,  int /*Freq*/, 
+		      float /*Weight*/,  void *info) {
+  text_tset *equivterms = (text_tset *)info;
+  if (equivterms == NULL) return 0;
+
+  text_t thisterm;
+  thisterm.setcarr(Word, ULen);
+
+  equivterms->insert(thisterm);
+  
+  return 0;
+}
+
+
+void mgsearch_equivterms (const text_t &word, text_tset &equivterms) {
+  // allocate working stem space
+  int maxstemlen = mgq_getmaxstemlen ();
+  unsigned char *word_stem = new unsigned char [maxstemlen + 2];
+  if (word_stem == NULL) return;
+
+  // copy word to word_stem
+  int len = 0;
+  text_t::const_iterator here = word.begin();
+  text_t::const_iterator end = word.end();
+  while (len < maxstemlen && here != end) {
+    word_stem[len+1] = (unsigned char)(*here);
+    len++; here++;
+  }
+  word_stem[len+1] = '\0';
+  word_stem[0] = len;
+
+  // get the equivalent terms
+  mgq_equivterms (word_stem, termequivcallback, (void *)(&equivterms));
+  
+  delete [] word_stem;
+
+  return;
+}
+
+  text_tset utf8equivterms; // kept as utf8 string for fast matching
+
+
+// This callback is called once for each term in the query
+int termfreqcallback(char *Word, int ULen,  int Freq, 
+		     float /*Weight*/,  void *info) {
+  queryresultsclass *queryresults = (queryresultsclass *)info;
+  if (queryresults == NULL) return 0;
+
+  text_t term;
+  term.setcarr(Word, ULen);
+  termfreqclass termfreq;
+
+  termfreq.termstr = to_uni(term);
+  text_t utf8termstem = mgsearch_stemword (term);
+  termfreq.termstemstr = to_uni (utf8termstem);
+
+  mgsearch_equivterms (utf8termstem, termfreq.utf8equivterms);
+  
+  termfreq.termfreq = Freq;
+  queryresults->orgterms.push_back(termfreq);
+  
+  return 0;
+}
+
+// this callback is called once for each variation of each term
+int termvariantscallback(char *Word, int ULen, int /*Freq*/,
+			 float /*Weight*/, void *info) {
+
+  text_t term;
+  term.setcarr(Word, ULen);
+  queryresultsclass *queryresults = (queryresultsclass *)info;
+  queryresults->termvariants.insert(to_uni(term));
+
+  return 0;
+}
+
+// This callback is for getting document text
+int doctextcallback(char *Doc, int ULen,  int /*Freq*/, 
+		    float /*Weight*/,  void * /*info*/) {
+  tempdoc = Doc;
+  templen = ULen;
+  
+  return 0;
+}
+
+
+static text_t getindexsuffix (const text_t &collection, 
+			      const text_t &index) {
+
+  text_t indexsuffix = "index";  
+  indexsuffix = filename_cat (indexsuffix, index);
+  indexsuffix = filename_cat (indexsuffix, collection);
+  return indexsuffix;
+}
+
+
+
+
+////////////////////
+// mgsearch class //
+////////////////////
+
+mgsearchclass::mgsearchclass ()
+{
+  cache = new querycache (RESULTCACHESIZE);
+}
+
+mgsearchclass::~mgsearchclass () 
+{
+  if (cache != NULL) 
+    {
+      delete cache;
+      cache = NULL;
+    }
+}
+
+
+void mgsearchclass::setcollectdir (const text_t &thecollectdir)
+{
+  collectdir = thecollectdir;
+}
+
+// you only need to use this function before doing any stemming
+// casefolding and stemming will be set if values for them are
+// provided (0 or 1).
+// makeindexcurrent returns true if it was able to load the database
+bool mgsearchclass::makeindexcurrent (const text_t &index,
+				      const text_t &subcollection,
+				      const text_t &language,
+				      const text_t &collection,
+				      int casefolding,
+				      int stemming) {
+  bool databaseloaded = true;
+
+  // get the names of the collection, index and text suffixes
+  char *ccollection = collection.getcstr();
+  assert (ccollection != NULL);
+  char *idxsuffix = (getindexsuffix (collection, (index+subcollection+language))).getcstr();
+  assert (idxsuffix != NULL);
+  char *txtsuffix = (getindexsuffix (collection, "text")).getcstr();
+  assert (txtsuffix != NULL);
+
+#ifdef __WIN32__
+  char *ccollectdir = (collectdir+"\\").getcstr(); assert (ccollectdir != NULL);
+#else
+  char *ccollectdir = collectdir.getcstr(); assert (ccollectdir != NULL);
+#endif
+
+  if (load_database(ccollection, ccollectdir, idxsuffix, txtsuffix)) {
+    if (casefolding == 0) mgq_ask(".set casefold off");
+    else if (casefolding > 0) mgq_ask(".set casefold on");
+    if (stemming == 0) mgq_ask(".set stem off");
+    else if (stemming > 0) mgq_ask(".set stem on");
+    
+  } else databaseloaded = false;
+
+  // free up the c strings
+  delete ccollection;
+  delete idxsuffix;
+  delete txtsuffix;
+  delete ccollectdir;
+
+  return databaseloaded;
+}
+
+
+// stem word uses the values set in the last call to makeindexcurrent
+// to stem the word. It is assumed that word is in unicode
+text_t mgsearchclass::stemword (const text_t &word) {
+  return to_uni (mgsearch_stemword (to_utf8 (word)));
+}
+
+text_t mgsearchclass::stemword (text_t::const_iterator here, text_t::const_iterator end) {
+  return to_uni (mgsearch_stemword (to_utf8 (here, end)));
+}
+
+
+bool mgsearchclass::search(const queryparamclass &queryparams, 
+			   queryresultsclass &queryresults) {
+  assert (cache != NULL);
+
+  queryresults.clear();
+
+  // first check the cache
+  if (cache->find(queryparams, queryresults)) return true;
+
+  // make sure there is a query to be processed
+  if (!has_unicode_letdig(queryparams.querystring)) return true;
+
+  if (makeindexcurrent (queryparams.index, queryparams.subcollection,
+			queryparams.language, queryparams.collection)) {
+    setsearchmode (queryparams);
+    submitquery (queryparams);
+    getresults (queryparams, queryresults);
+    return true;
+  }
+
+  return false;
+}
+
+
+void mgsearchclass::setsearchmode (const queryparamclass &queryparams)
+{
+  mgq_ask(".set expert true");
+  mgq_ask(".set sorted_terms true");
+  mgq_ask(".set accumulator_method list");
+  mgq_ask(".set max_accumulators 500000");
+  mgq_ask(".set maxparas 500000");
+  mgq_ask(".set verbatim true");
+  mgq_ask(".unset skip_dump");
+  mgq_ask(".set mode docnums");
+
+  switch (queryparams.search_type) 
+    {
+    case 0: mgq_ask(".set query boolean");  break;
+    case 1:  mgq_ask(".set query ranked"); break;
+    }
+  switch (queryparams.casefolding) 
+    {
+    case 1: mgq_ask(".set casefold on");  break;
+    case 0: mgq_ask(".set casefold off"); break;
+    }
+  switch (queryparams.stemming) 
+    {
+    case 1: mgq_ask(".set stem on");  break;
+    case 0: mgq_ask(".set stem off"); break;
+    }
+  mgq_ask(".set heads_length 150");
+  
+  if (queryparams.maxdocs == -1) {
+    mgq_ask(".set maxdocs all");
+  } else {
+    char maxdocstr[32];
+    sprintf(maxdocstr, ".set maxdocs %i", queryparams.maxdocs);
+    mgq_ask(maxdocstr);
+  }
+}
+
+
+void mgsearchclass::submitquery (const queryparamclass &queryparams)
+{
+  // sort out the query string
+  text_t ttquerystring = queryparams.querystring;
+  filterquery (ttquerystring);
+  char *querystring = to_utf8(ttquerystring).getcstr();
+  
+  // submit the query
+  mgq_ask(querystring);
+
+  delete querystring;
+}
+
+
+void mgsearchclass::getresults (const queryparamclass &queryparams,
+				queryresultsclass &queryresults) {
+
+  int howmany = queryparams.maxdocs;
+  if (howmany == -1) howmany = MAXNUMDOCS;
+  mgq_results(result_docnums, 0, howmany, 
+	      ourquerycallback, (void *)(&queryresults));
+  
+  // get the term frequencies
+  mgq_results(result_termfreqs, 0, MAXNUMTERMS, 
+	      termfreqcallback, (void *)(&queryresults));
+  queryresults.sortuniqqueryterms();
+
+  // get term variants 
+  mgq_results(result_terms, 0, MAXNUMTERMS,
+              termvariantscallback, (void *)(&queryresults));
+
+  // get the number of documents retrieved
+  int total_retrieved = 0, is_approx = 0;
+  mgq_docsretrieved (&total_retrieved, &is_approx);
+
+  if (total_retrieved == 0) {
+    // not available (or really was zero)
+    queryresults.docs_matched = queryresults.docs.docset.size();
+    if ((queryparams.maxdocs == -1) ||
+	(queryresults.docs_matched < queryparams.maxdocs))
+      queryresults.is_approx = Exact;
+    else
+      queryresults.is_approx = MoreThan;
+  } else {
+    queryresults.docs_matched = total_retrieved;
+    if (is_approx) queryresults.is_approx = Approximate;
+    else queryresults.is_approx = Exact;
+  }
+}
+
+void mgsearchclass::filterquery (text_t &ttquerystring) {
+  text_t::iterator ithere = ttquerystring.begin ();
+  text_t::iterator itend = ttquerystring.end ();
+  
+  // remove all non alphanumeric characters (except
+  // boolean operators
+  while (ithere != itend) {
+    if ((!is_unicode_letdig(*ithere)) && (*ithere != '!') &&
+	(*ithere != '&') && (*ithere != '|') && (*ithere != '(') &&
+	(*ithere != ')')) (*ithere) = ' ';
+    ithere++;
+  }
+}
+
+
+// the document text for 'docnum' is placed in 'output'
+// docTargetDocument returns 'true' if it was able to
+// try to get a document
+// collection is needed to see if an index from the 
+// collection is loaded. If no index has been loaded
+// defaultindex is needed to load one
+bool mgsearchclass::docTargetDocument(const text_t &defaultindex,
+				      const text_t &defaultsubcollection,
+				      const text_t &defaultlanguage,
+				      const text_t &collection,
+				      int docnum,
+				      text_t &output) {
+  output.clear();
+
+  // get the mg version of the document
+  char *mgdoc = NULL;
+  int doclen = 0;
+  if (!mgdocument (defaultindex, defaultsubcollection, defaultlanguage,
+		   collection, docnum, mgdoc, doclen)) return false;
+  if (mgdoc == NULL) return false;
+
+  // replace all control-Cs with spaces
+  char *mgdoc_here = mgdoc;
+  char *mgdoc_end = mgdoc + doclen;
+  while (mgdoc_here < mgdoc_end) {
+    if (*mgdoc_here == '\x3') *mgdoc_here = ' ';
+    mgdoc_here++;
+  }
+
+  // convert this document to unicode
+  utf8inconvertclass inconvert;
+  convertclass::status_t status;
+  inconvert.reset ();
+  inconvert.setinput (mgdoc, doclen);
+  inconvert.convert (output, status);
+
+  return true;
+}
+
+
+bool mgsearchclass::mgdocument (const text_t &defaultindex, 
+				const text_t &defaultsubcollection,
+				const text_t &defaultlanguage,
+				const text_t &collection,
+				int docnum,
+				char *&UDoc, int &ULen) {
+  int databaseloaded = 0;
+
+  UDoc = NULL; ULen = 0;
+  
+  // see if we can make an appropriate database current
+//    char *ccollection = collection.getcstr();
+//    assert (ccollection != NULL);
+//    databaseloaded = load_text_database (ccollection);
+//    delete ccollection;
+  
+  // try and load the database
+//    if (!databaseloaded)
+  databaseloaded = makeindexcurrent (defaultindex, defaultsubcollection,
+				     defaultlanguage, collection);
+  
+  if (databaseloaded) {
+    // retrieve the document from mg
+    char docstr[32];
+    sprintf(docstr, "%i", docnum);
+    
+    mgq_ask(".set mode text");
+    mgq_ask(".set query docnums");
+    mgq_ask(docstr);
+
+    tempdoc = NULL;
+    templen = 0;
+    mgq_results (result_docs, 0, 1, doctextcallback, (void *)NULL);
+    UDoc = tempdoc;
+    ULen = templen;
+  }
+
+  return (bool)databaseloaded;
+}
+
Index: /tags/gsdl-21/gsdl/src/colservr/mgsearch.h
===================================================================
--- /tags/gsdl-21/gsdl/src/colservr/mgsearch.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/colservr/mgsearch.h	(revision 699)
@@ -0,0 +1,102 @@
+/**********************************************************************
+ *
+ * mgsearch.h -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef MGSEARCH_H
+#define MGSEARCH_H
+
+
+#include "text_t.h"
+#include "querycache.h"
+
+#define RESULTCACHESIZE 10
+#define MAXNUMDOCS      1000000
+#define MAXNUMTERMS     100
+
+class mgsearchclass {
+public:
+  mgsearchclass ();
+  virtual ~mgsearchclass ();
+
+  // the index directory must be set before any searching
+  // is done
+  void setcollectdir (const text_t &thecollectdir);
+
+  // you only need to use this function before doing any stemming
+  // casefolding and stemming will be set if values for them are
+  // provided (0 or 1).
+  // makeindexcurrent returns true if it was able to load the database
+  bool makeindexcurrent (const text_t &index, const text_t &subcollection,
+			 const text_t &language, const text_t &collection,
+			 int casefolding = -1, int stemming = -1);
+
+  // stem word uses the values set in the last call to makeindexcurrent
+  // to stem the word. It is assumed that word is in unicode
+  text_t stemword (const text_t &word);
+  text_t stemword (text_t::const_iterator here, text_t::const_iterator end);
+  
+  // the search results are returned in queryresults
+  // search returns 'true' if it was able to do a search
+  bool search(const queryparamclass &queryparams, 
+	      queryresultsclass &queryresults);
+  
+  // the document text for 'docnum' is placed in 'output'
+  // docTargetDocument returns 'true' if it was able to
+  // try to get a document
+  // collection is needed to see if an index from the 
+  // collection is loaded. If no index has been loaded
+  // defaultindex is needed to load one
+  bool docTargetDocument(const text_t &defaultindex,
+			 const text_t &defaultsubcollection,
+			 const text_t &defaultlanguage,
+			 const text_t &collection,
+			 int docnum,
+			 text_t &output);
+
+  // same as docTargetDocument except the document is returned
+  // in its raw form. UDoc remains the property of mg and may
+  // be destroyed on the next call to mg.
+  bool mgdocument (const text_t &defaultindex,
+		   const text_t &defaultsubcollection,
+		   const text_t &defaultlanguage,
+		   const text_t &collection,
+		   int docnum,
+		   char *&UDoc, int &ULen);
+
+protected:
+  querycache *cache;
+  text_t collectdir; // the collection directory
+
+  void setsearchmode (const queryparamclass &queryparams);
+  void submitquery (const queryparamclass &queryparams);
+  void getresults (const queryparamclass &queryparams, queryresultsclass &queryresults);
+
+  virtual void filterquery (text_t &ttquerystring);
+};
+
+
+#endif
Index: /tags/gsdl-21/gsdl/src/colservr/phrasequeryfilter.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/colservr/phrasequeryfilter.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/colservr/phrasequeryfilter.cpp	(revision 699)
@@ -0,0 +1,104 @@
+/**********************************************************************
+ *
+ * phrasequeryfilter.cpp -- 
+ * Copyright (C) 1999  DigiLib Systems Limited, New Zealand
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.4  1999/09/07 04:57:23  sjboddie
+   added gpl notice
+
+   Revision 1.3  1999/08/31 22:44:38  rjmcnab
+   Fixed small bug.
+
+   Revision 1.2  1999/07/07 06:19:45  rjmcnab
+   Added ability to combine two or more independant queries.
+
+   Revision 1.1  1999/07/01 04:01:47  rjmcnab
+   Initial revision.
+
+
+ */
+
+
+#include "phrasequeryfilter.h"
+#include "queryinfo.h"
+
+#include "phrasesearch.h"
+
+
+
+// do aditional query processing
+void phrasequeryfilterclass::post_process (const queryparamclass &queryparams,
+					   queryresultsclass &queryresults) {
+  bool results_shrunk = false;
+
+  // post-process the results if needed
+  if (queryresults.orgterms.size() > 1 && !queryresults.docs.docset.empty()) {
+    // get the long version of the index
+    text_t longindex;
+    indexmap.to2from (queryparams.index, longindex);
+      
+    // process each of the matched documents
+    docresultmap::iterator docs_here = queryresults.docs.docset.begin();
+    docresultmap::iterator docs_end = queryresults.docs.docset.end();
+    while (docs_here != docs_end) {
+      if ((*docs_here).second.num_query_terms_matched == queryresults.orgterms.size() &&
+	  OID_phrase_search (*mgsearchptr, *gdbmptr, queryparams.index,
+			     queryparams.subcollection, queryparams.language, 
+			     longindex, queryparams.collection, queryresults.orgterms,
+			     (*docs_here).second.docnum)) {
+	(*docs_here).second.num_phrase_match++;
+	docs_here++;
+
+      } else {
+	docresultmap::iterator this_doc_here = docs_here;
+	docs_here++;
+	if (queryparams.match_mode == 1) {
+	  // need to delete this element, there was no exact match
+	  queryresults.docs.docset.erase(this_doc_here);
+	  results_shrunk = true;
+	}
+      }
+    }
+  }
+
+  if (results_shrunk) queryresults.docs_matched = queryresults.docs.docset.size();
+}
+
+
+void phrasequeryfilterclass::sort_doc_results (const FilterRequest_t &/*request*/,
+					 docresultsclass &docs) {
+  resultsorderer_t resultsorderer;
+  resultsorderer.compare_phrase_match = true;
+  resultsorderer.compare_terms_match = true;
+  resultsorderer.compare_doc_weight = true;
+  resultsorderer.docset = &(docs.docset);
+
+  // first get a list of document numbers
+  docs.docnum_order();
+
+  sort (docs.docorder.begin(), docs.docorder.end(), resultsorderer);
+}
Index: /tags/gsdl-21/gsdl/src/colservr/phrasequeryfilter.h
===================================================================
--- /tags/gsdl-21/gsdl/src/colservr/phrasequeryfilter.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/colservr/phrasequeryfilter.h	(revision 699)
@@ -0,0 +1,45 @@
+/**********************************************************************
+ *
+ * phrasequeryfilter.h -- 
+ * Copyright (C) 1999  DigiLib Systems Limited, New Zealand
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef PHRASEQUERYFILTER_H
+#define PHRASEQUERYFILTER_H
+
+#include "queryfilter.h"
+
+
+class phrasequeryfilterclass : public queryfilterclass {
+protected:
+  void post_process (const queryparamclass &queryparams,
+		     queryresultsclass &queryresults);
+  void sort_doc_results (const FilterRequest_t &request,
+			 docresultsclass &docs);
+public:
+};
+
+
+#endif
Index: /tags/gsdl-21/gsdl/src/colservr/phrasesearch.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/colservr/phrasesearch.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/colservr/phrasesearch.cpp	(revision 699)
@@ -0,0 +1,217 @@
+/**********************************************************************
+ *
+ * phrasesearch.cpp -- tools to search for a phrase in a larger text
+ * Copyright (C) 1999  DigiLib Systems Limited
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.6  1999/09/07 04:57:23  sjboddie
+   added gpl notice
+
+   Revision 1.5  1999/08/31 22:45:12  rjmcnab
+   fixed small problem
+
+   Revision 1.4  1999/07/16 00:15:48  sjboddie
+   changed to use termfreqclassarray type
+
+   Revision 1.3  1999/07/07 06:19:45  rjmcnab
+   Added ability to combine two or more independant queries.
+
+   Revision 1.2  1999/07/01 09:25:54  rjmcnab
+   fixed bug :-^
+
+   Revision 1.1  1999/07/01 04:01:46  rjmcnab
+   Initial revision.
+
+ */
+
+
+#include "phrasesearch.h"
+#include "gsdlunicode.h"
+
+inline unsigned char *parse_nonword_word (unsigned char *here, unsigned char *end,
+					  text_t &word) {
+  int c_len = 0;
+  unsigned short c = 0;
+
+  word.clear();
+
+  // parse non word
+  while (here <= end) {
+    c_len = parse_utf8_char (here, end, &c);
+    if (c == '(') {
+      // found a note, look for '}'
+      while (here <= end && c != ')') {
+	c_len = parse_utf8_char (here, end, &c);
+	here += c_len;
+      }
+    }
+    if (c == '{') {
+      // found a composite character, look for '}'
+      while (here <= end && c != '}') {
+	c_len = parse_utf8_char (here, end, &c);
+	here += c_len;
+      }
+    }
+    if (is_unicode_letdig(c)) {
+      while (c_len > 0) {
+	// this is in a word
+	word.push_back(*here);
+	here++; c_len--;
+      }
+      break;
+    }
+    here += c_len;
+  }
+
+  // parse word
+  while (here <= end) {
+    c_len = parse_utf8_char (here, end, &c);
+    if (!is_unicode_letdig(c)) {
+      here += c_len;  // it is ok to skip a nonword character
+      break;
+    }
+    while (c_len > 0) {
+      word.push_back(*here);
+      here++; c_len--;
+    }
+  }
+
+  return here;
+}
+
+
+bool doc_phrase_search (unsigned char *doc, int doclen,
+			const termfreqclassarray &phrase) {
+  // note: this uses the most braindead search routine :-)
+  // however its not so bad as there shouldn't be many partial
+  // matches
+
+  // a null phrase matches anything
+  if (phrase.empty()) return true;
+
+  // if there is nothing then there can't be a match
+  if (doc == NULL || doclen == 0) return false;
+  
+  text_t doc_word;
+  doc_word.reserve (16);
+
+  bool first = true;
+
+  unsigned char *doc_here = doc;
+  unsigned char *doc_herefirstword = doc;
+  unsigned char *doc_end = doc+doclen-1; // unitool conventions :-/
+
+  while (doc_here <= doc_end) {
+    first = true;
+    
+    // there will be at least one member of phrase (see above)
+    termfreqclassarray::const_iterator phrase_here = phrase.begin();
+    termfreqclassarray::const_iterator phrase_end = phrase.end();
+    do {
+      // get the next non-word ... and ignore it, then get the next word
+      doc_here = parse_nonword_word (doc_here, doc_end, doc_word);
+      if (first) {doc_herefirstword = doc_here; first = false;}
+
+      // break if this word is not the next in the phrase
+      if ((*phrase_here).utf8equivterms.find (doc_word) ==
+	  (*phrase_here).utf8equivterms.end()) break;
+
+      phrase_here++;
+    } while (doc_here <= doc_end && phrase_here != phrase_end);
+
+    // see if we found a phrase
+    if (phrase_here == phrase_end) return true;
+    
+    doc_here = doc_herefirstword; // set the counter back
+  }
+
+  return false;
+}
+
+
+// looks for the stemmed phrase in the metadata or text associated with
+// an OID. This function has not been coded with all situations in mind
+bool OID_phrase_search (mgsearchclass &mgsearch,
+			gdbmclass &gdbm,
+			const text_t &index,
+			const text_t &subcollection,
+			const text_t &language,
+			const text_t &longindex,
+			const text_t &collection,
+			const termfreqclassarray &phrase,
+			int docnum) {
+  // disect the long index to find out where the text should come from
+  text_t level, gran;
+  text_t::const_iterator longindex_here = longindex.begin();
+  text_t::const_iterator longindex_end = longindex.end();
+  longindex_here = getdelimitstr (longindex_here, longindex_end, ':', level);
+  longindex_here = getdelimitstr (longindex_here, longindex_end, ':', gran);
+
+  if (gran.empty()) return false;
+  
+  if (gran == "text") {
+    char *doc = NULL;
+    int doclen = 0;
+  
+    // get text from mg.
+    if (!mgsearch.mgdocument (index, subcollection, language, collection,
+			      docnum, doc, doclen)) return false;
+    return doc_phrase_search ((unsigned char *)doc, doclen, phrase);
+  } 
+
+  // get OID
+  char *metadata = NULL;
+  text_t::size_type metadata_len = 0;
+  infodbclass docnum_info;
+  infodbclass OID_info;
+  
+  if (!gdbm.getinfo (docnum, docnum_info)) return false;
+  text_t &OID = docnum_info["section"];
+  if (OID.empty()) return false;
+  
+  // get field
+  if (!gdbm.getinfo (OID, OID_info)) return false;
+
+  bool result = false;
+  text_tarray *tarr_ptr = OID_info.getmultinfo (gran);
+  if (tarr_ptr != NULL ) {
+    text_tarray::const_iterator subvalue_here = (*tarr_ptr).begin();
+    text_tarray::const_iterator subvalue_end = (*tarr_ptr).end();
+    while (subvalue_here != subvalue_end) {
+      if (subvalue_here != NULL) {
+	metadata = (to_utf8(*subvalue_here)).getcarr(metadata_len);
+	result = doc_phrase_search ((unsigned char *)metadata, metadata_len, phrase);
+	delete [] metadata;
+
+	if (result) return true;
+      }
+      
+      subvalue_here++;
+    }
+  }
+
+  return result;
+}
Index: /tags/gsdl-21/gsdl/src/colservr/phrasesearch.h
===================================================================
--- /tags/gsdl-21/gsdl/src/colservr/phrasesearch.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/colservr/phrasesearch.h	(revision 699)
@@ -0,0 +1,52 @@
+/**********************************************************************
+ *
+ * phrasesearch.h -- tools to search for a phrase in a larger text
+ * Copyright (C) 1999  DigiLib Systems Limited
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef PHRASESEARCH_H
+#define PHRASESEARCH_H
+
+
+#include "text_t.h"
+#include "unitool.h"
+#include "mgsearch.h"
+#include "infodbclass.h"
+
+
+// looks for the phrase in the metadata or text associated with
+// a document number. This function has not been coded with all
+// situations in mind and expects mgsearch and gdbm set up ready to go
+bool OID_phrase_search (mgsearchclass &mgsearch,
+			gdbmclass &gdbm,
+			const text_t &index,
+			const text_t &subcollection,
+			const text_t &language,
+			const text_t &longindex,
+			const text_t &collection,
+			const termfreqclassarray &phrase,
+			int docnum);
+
+#endif
Index: /tags/gsdl-21/gsdl/src/colservr/querycache.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/colservr/querycache.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/colservr/querycache.cpp	(revision 699)
@@ -0,0 +1,127 @@
+/**********************************************************************
+ *
+ * querycache.cpp -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.4  1999/09/07 04:57:24  sjboddie
+   added gpl notice
+
+   Revision 1.3  1999/01/19 01:38:17  rjmcnab
+
+   Made the source more portable.
+
+   Revision 1.2  1999/01/12 01:51:02  rjmcnab
+
+   Standard header.
+
+   Revision 1.1  1999/01/08 09:02:18  rjmcnab
+
+   Moved from src/library.
+
+ */
+
+
+#include "querycache.h"
+
+
+
+
+resultcacheel::resultcacheel () 
+{
+  accessnum = -1;
+}
+
+
+
+querycache::querycache (int themaxcachesize)
+{
+  if (themaxcachesize < 1) themaxcachesize = 1;
+  
+  resultcache = new resultcacheel[themaxcachesize];
+
+  maxcachesize = themaxcachesize;
+  nextaccessnum = 1;
+}
+
+querycache::~querycache ()
+{
+  delete [] resultcache;
+}
+
+// returns true if the query was found in the cache
+// if the query was found then queryresults contains
+// the results
+bool querycache::find (const queryparamclass &queryparams, 
+		       queryresultsclass &queryresults)
+{
+  int i;
+
+  for (i=0; i < maxcachesize; i++)
+    {
+      if (resultcache[i].queryparameters == queryparams)
+	{
+	  queryresults = resultcache[i].queryresults;
+	  resultcache[i].accessnum = getnextaccessnum ();
+	  return true;
+	}
+    }
+
+  return false;
+}
+
+
+void querycache::cache (const queryparamclass &queryparams,
+	  const queryresultsclass &queryresults)
+{
+  int i = getfreecachenum();
+  resultcache[i].queryparameters = queryparams;
+  resultcache[i].queryresults = queryresults;
+  resultcache[i].accessnum = getnextaccessnum ();
+}
+
+int querycache::getnextaccessnum ()
+{
+  return nextaccessnum++;
+}
+
+int querycache::getfreecachenum ()
+{
+  int i;
+  int minaccessnum = 0;
+  int minaccessi = 0;
+  
+  for (i=0; i < maxcachesize; i++)
+    {
+      if (resultcache[i].accessnum < minaccessnum) {
+	minaccessnum = resultcache[i].accessnum;
+	minaccessi = i;
+      }
+    }
+
+  return minaccessi;
+}
+
Index: /tags/gsdl-21/gsdl/src/colservr/querycache.h
===================================================================
--- /tags/gsdl-21/gsdl/src/colservr/querycache.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/colservr/querycache.h	(revision 699)
@@ -0,0 +1,72 @@
+/**********************************************************************
+ *
+ * querycache.h -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef QUERYCACHE_H
+#define QUERYCACHE_H
+
+
+#include "text_t.h"
+#include "queryinfo.h"
+
+class resultcacheel
+{
+public:
+  resultcacheel ();
+
+  int accessnum;
+  queryparamclass queryparameters;
+  queryresultsclass queryresults;
+};
+
+
+class querycache {
+public:
+  querycache (int themaxcachesize);
+  ~querycache ();
+
+  // returns true if the query was found in the cache
+  // if the query was found then queryresults contains
+  // the results
+  bool find (const queryparamclass &queryparams, 
+	     queryresultsclass &queryresults);
+
+  void cache (const queryparamclass &queryparams,
+	    const queryresultsclass &queryresults);
+
+protected:
+  resultcacheel *resultcache;
+
+  int maxcachesize;
+  int nextaccessnum;
+  
+  int getnextaccessnum ();
+  int getfreecachenum ();
+};
+
+
+#endif
Index: /tags/gsdl-21/gsdl/src/colservr/queryfilter.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/colservr/queryfilter.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/colservr/queryfilter.cpp	(revision 699)
@@ -0,0 +1,764 @@
+/**********************************************************************
+ *
+ * queryfilter.cpp -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.18  1999/09/22 03:43:18  sjboddie
+   Endresults queryfilter option may now take '-1' for 'all'
+
+   Revision 1.17  1999/09/21 12:01:07  sjboddie
+   added Maxdocs queryfilter option (which may be -1 for 'all')
+
+   Revision 1.16  1999/09/07 04:57:24  sjboddie
+   added gpl notice
+
+   Revision 1.15  1999/08/31 22:47:09  rjmcnab
+   Added matchmode option for some and all.
+
+   Revision 1.14  1999/07/16 03:42:21  sjboddie
+   changed isApprox
+
+   Revision 1.13  1999/07/16 00:17:06  sjboddie
+   got using phrasesearch for post-processing
+
+   Revision 1.12  1999/07/09 02:19:43  rjmcnab
+   Fixed a couple of compiler conflicts
+
+   Revision 1.11  1999/07/08 20:49:44  rjmcnab
+   Added result_num to the ResultDocInto_t structure.
+
+   Revision 1.10  1999/07/07 06:19:46  rjmcnab
+   Added ability to combine two or more independant queries.
+
+   Revision 1.9  1999/07/01 09:29:20  rjmcnab
+   Changes for better reporting of number documents which match a query. Changes
+   should still work as before with older versions of mg.
+
+   Revision 1.8  1999/07/01 03:59:54  rjmcnab
+   reduced MAXDOCS to 200 (more reasonable ???). I also added a virtual
+   method for post-processing the query.
+
+   Revision 1.7  1999/06/30 04:04:13  rjmcnab
+   made stemming functions available from mgsearch and made the stems
+   for the query terms available in queryinfo
+
+   Revision 1.6  1999/06/29 22:06:23  rjmcnab
+   Added a couple of fields to queryinfo to handle a special version
+   of mg.
+
+   Revision 1.5  1999/06/27 22:08:48  sjboddie
+   now check for defaultindex, defaultsubcollection, and defaultlanguage
+   entries in config files
+
+   Revision 1.4  1999/06/16 02:03:25  sjboddie
+   fixed bug in isApprox and set MAXDOCS to always be 500
+
+   Revision 1.3  1999/04/19 23:56:09  rjmcnab
+   Finished the gdbm metadata stuff
+
+   Revision 1.2  1999/04/12 03:45:03  rjmcnab
+   Finished the query filter.
+
+   Revision 1.1  1999/04/06 22:22:09  rjmcnab
+   Initial revision.
+
+ */
+
+
+#include "queryfilter.h"
+#include "fileutil.h"
+#include "queryinfo.h"
+#include "phrasesearch.h"
+#include <assert.h>
+
+
+// some useful functions
+
+// translate will return true if successful
+static bool translate (gdbmclass *gdbmptr, int docnum, text_t &trans_OID) {
+  infodbclass info;
+
+  trans_OID.clear();
+
+  // get the info
+  if (gdbmptr == NULL) return false;
+  if (!gdbmptr->getinfo(docnum, info)) return false;
+
+  // translate
+  if (info["section"].empty()) return false;
+
+  trans_OID = info["section"];
+  return true;
+}
+
+
+// whether document results are needed
+static bool need_matching_docs (int filterResultOptions) {
+  return ((filterResultOptions & FROID) || (filterResultOptions & FRranking) ||
+	  (filterResultOptions & FRmetadata));
+}
+
+// whether term information is needed
+static bool need_term_info (int filterResultOptions) {
+  return ((filterResultOptions & FRtermFreq) || (filterResultOptions & FRmatchTerms));
+}
+
+///////////////////////////////
+// methods for resultsorderer_t
+///////////////////////////////
+
+resultsorderer_t::resultsorderer_t() {
+  clear ();
+}
+
+void resultsorderer_t::clear() {
+  compare_phrase_match = false;
+  compare_terms_match = false;
+  compare_doc_weight = true;
+
+  docset = NULL;
+}
+
+bool resultsorderer_t::operator()(const int &t1, const int &t2) const {
+  if (docset == NULL) return t1>t2;
+
+  docresultmap::iterator t1_here = docset->find(t1);
+  docresultmap::iterator t2_here = docset->find(t2);
+  docresultmap::iterator end = docset->end();
+
+  // sort all the document numbers not in the document set to
+  // the end of the list
+  if (t1_here == end) {
+    if (t2_here == end) return t1>t2;
+    else return true;
+  } else if (t2_here == end) return false;
+  
+  if (compare_phrase_match) {
+    if ((*t1_here).second.num_phrase_match > (*t2_here).second.num_phrase_match) return true;
+    if ((*t1_here).second.num_phrase_match < (*t2_here).second.num_phrase_match) return false;
+  }
+
+  if (compare_terms_match) {
+    if ((*t1_here).second.num_query_terms_matched > (*t2_here).second.num_query_terms_matched) return true;
+    if ((*t1_here).second.num_query_terms_matched < (*t2_here).second.num_query_terms_matched) return false;
+  }
+
+  if (compare_doc_weight) {
+    if ((*t1_here).second.docweight > (*t2_here).second.docweight) return true;
+    if ((*t1_here).second.docweight < (*t2_here).second.docweight) return false;
+  }
+
+  return t1>t2;
+}
+
+
+
+
+/////////////////////////////////
+// functions for queryfilterclass
+/////////////////////////////////
+
+// loads up phrases data structure with any phrases (that's the quoted bits)
+// occuring in the querystring
+void queryfilterclass::get_phrase_terms (const text_t &querystring, 
+					 const termfreqclassarray &orgterms,
+					 vector<termfreqclassarray> &phrases) {
+
+  text_t::const_iterator here = querystring.begin();
+  text_t::const_iterator end = querystring.end();
+
+  termfreqclassarray tmpterms;
+
+  int termcount = 0;
+  bool foundquote = false;
+  bool foundbreak = false;
+  bool start = true;
+  while (here != end) {
+    if (*here == '\"') {
+      if (foundquote) {
+	if (!foundbreak && !start) {
+	  tmpterms.push_back (orgterms[termcount]);
+	  termcount ++;
+	}
+	if (tmpterms.size() > 1) {
+	  phrases.push_back (tmpterms);
+	  tmpterms.erase (tmpterms.begin(), tmpterms.end());
+	}
+	foundquote = false;
+	foundbreak = true;
+      } else foundquote = true;
+    } else if (!is_unicode_letdig(*here)) {
+      // found a break between terms
+      if (!foundbreak && !start) {
+	if (foundquote)
+	  tmpterms.push_back (orgterms[termcount]);
+	termcount ++;
+      }
+      foundbreak = true;
+    } else {
+      start = false;
+      foundbreak = false;
+    }      
+    here++;
+  }
+}
+
+// do aditional query processing
+void queryfilterclass::post_process (const queryparamclass &queryparams, 
+				     queryresultsclass &queryresults) {
+
+  // post-process the results if needed
+  if (queryresults.orgterms.size() > 1 && !queryresults.docs.docset.empty()) {
+
+    // get the terms between quotes (if any)
+    vector<termfreqclassarray> phrases;
+    get_phrase_terms (queryparams.querystring, queryresults.orgterms, phrases);
+
+    if (phrases.size() > 0) {
+
+      // get the long version of the index
+      text_t longindex;
+      indexmap.to2from (queryparams.index, longindex);
+      
+      vector<termfreqclassarray>::const_iterator this_phrase = phrases.begin();
+      vector<termfreqclassarray>::const_iterator end_phrase = phrases.end();
+
+      while (this_phrase != end_phrase) {
+
+	// process each of the matched documents
+	docresultmap::iterator docs_here = queryresults.docs.docset.begin();
+	docresultmap::iterator docs_end = queryresults.docs.docset.end();
+	while (docs_here != docs_end) {
+	  if (OID_phrase_search (*mgsearchptr, *gdbmptr, queryparams.index,
+				 queryparams.subcollection, queryparams.language, 
+				 longindex, queryparams.collection, *this_phrase,
+				 (*docs_here).second.docnum)) {
+	    (*docs_here).second.num_phrase_match++;
+	  }
+	
+	  docs_here++;
+	}
+	this_phrase++;
+      }
+    }
+  }
+}
+
+// get the query parameters
+void queryfilterclass::parse_query_params (const FilterRequest_t &request,
+					   vector<queryparamclass> &query_params,
+					   int &startresults,
+					   int &endresults,
+					   ostream &logout) {
+  outconvertclass text_t2ascii;
+
+  // set defaults for the return parameters
+  query_params.erase(query_params.begin(), query_params.end());
+  startresults = filterOptions["StartResults"].defaultValue.getint();
+  endresults = filterOptions["EndResults"].defaultValue.getint();
+
+  // set defaults for query parameters
+  queryparamclass query;
+  query.combinequery = "or"; // first one must be "or"
+  query.collection = collection;
+  query.index = filterOptions["Index"].defaultValue;
+  query.subcollection = filterOptions["Subcollection"].defaultValue;
+  query.language = filterOptions["Language"].defaultValue;
+  query.querystring.clear();
+  query.search_type = (filterOptions["QueryType"].defaultValue == "ranked");
+  query.match_mode = (filterOptions["MatchMode"].defaultValue == "all");
+  query.casefolding = (filterOptions["Casefold"].defaultValue == "true");
+  query.stemming = (filterOptions["Stem"].defaultValue == "true");
+  query.maxdocs = filterOptions["Maxdocs"].defaultValue.getint();
+
+  OptionValue_tarray::const_iterator options_here = request.filterOptions.begin();
+  OptionValue_tarray::const_iterator options_end = request.filterOptions.end();
+  while (options_here != options_end) {
+    if ((*options_here).name == "CombineQuery") {
+      // add this query
+      
+      // "all", needed when combining queries where the document results are needed
+      if (need_matching_docs (request.filterResultOptions)) query.maxdocs = -1; 
+      query_params.push_back (query);
+
+      // start on next query
+      query.clear();
+      query.combinequery = (*options_here).value;
+
+      // set defaults for query parameters
+      query.collection = collection;
+      query.index = filterOptions["Index"].defaultValue;
+      query.subcollection = filterOptions["Subcollection"].defaultValue;
+      query.language = filterOptions["Language"].defaultValue;
+      query.querystring.clear();
+      query.search_type = (filterOptions["QueryType"].defaultValue == "ranked");
+      query.match_mode = (filterOptions["MatchMode"].defaultValue == "all");
+      query.casefolding = (filterOptions["Casefold"].defaultValue == "true");
+      query.stemming = (filterOptions["Stem"].defaultValue == "true");
+      
+      // "all", needed when combining queries where the document results are needed
+      if (need_matching_docs (request.filterResultOptions)) query.maxdocs = -1; 
+      else query.maxdocs = filterOptions["Maxdocs"].defaultValue.getint();
+      
+    } else if ((*options_here).name == "StartResults") {
+      startresults = (*options_here).value.getint();
+    } else if ((*options_here).name == "EndResults") {
+      endresults = (*options_here).value.getint();
+    } else if ((*options_here).name == "QueryType") {
+      query.search_type = ((*options_here).value == "ranked");
+    } else if ((*options_here).name == "MatchMode") {
+      query.match_mode = ((*options_here).value == "all");
+      if (query.match_mode == 1) query.maxdocs = -1;
+    } else if ((*options_here).name == "Term") {
+      query.querystring = (*options_here).value;
+    } else if ((*options_here).name == "Casefold") {
+      query.casefolding = ((*options_here).value == "true");
+    } else if ((*options_here).name == "Stem") {
+      query.stemming = ((*options_here).value == "true");
+    } else if ((*options_here).name == "Index") {
+      query.index = (*options_here).value;
+    } else if ((*options_here).name == "Subcollection") {
+      query.subcollection = (*options_here).value;
+    } else if ((*options_here).name == "Language") {
+      query.language = (*options_here).value;
+    } else if ((*options_here).name == "Maxdocs") {
+      query.maxdocs = (*options_here).value.getint();
+    } else {
+      logout << text_t2ascii
+	     << "warning: unknown queryfilter option \""
+	     << (*options_here).name
+	     << "\" ignored.\n\n";
+    }
+
+    options_here++;
+  }
+
+  // add the last query
+  query_params.push_back (query);
+}
+
+
+
+// do query that might involve multiple sub queries
+// mgsearchptr and gdbmptr are assumed to be valid
+void queryfilterclass::do_multi_query (const FilterRequest_t &request,
+				       const vector<queryparamclass> &query_params,
+				       queryresultsclass &multiresults,
+				       comerror_t &err, ostream &logout) {
+  outconvertclass text_t2ascii;
+
+  err = noError;
+  mgsearchptr->setcollectdir (collectdir);
+  multiresults.clear();
+  
+  vector<queryparamclass>::const_iterator query_here = query_params.begin();
+  vector<queryparamclass>::const_iterator query_end = query_params.end();
+  while (query_here != query_end) {
+    queryresultsclass thisqueryresults;
+    
+    if (!mgsearchptr->search(*query_here, thisqueryresults)) {
+      // most likely a system problem
+      logout << text_t2ascii
+	     << "system problem: could not do search with mg for index \""
+	     << (*query_here).index << (*query_here).subcollection
+	     << (*query_here).language << "\".\n\n";
+      err = systemProblem;
+      return;
+    }
+
+    // combine the results
+    if (need_matching_docs (request.filterResultOptions)) {
+      // post-process the results if needed
+      if (!thisqueryresults.postprocessed && thisqueryresults.orgterms.size() > 1 &&
+	  !thisqueryresults.docs.docset.empty()) {
+	post_process (*query_here, thisqueryresults);
+	thisqueryresults.postprocessed = true;
+	multiresults.postprocessed = true;
+      }
+      
+      if (query_params.size() == 1) {
+	multiresults.docs = thisqueryresults.docs; // just one set of results
+	multiresults.docs_matched = thisqueryresults.docs_matched;
+	multiresults.is_approx = thisqueryresults.is_approx;
+	
+      } else {
+	if ((*query_here).combinequery == "and") {
+	  multiresults.docs.combine_and (thisqueryresults.docs);
+	} else if ((*query_here).combinequery == "or") {
+	  multiresults.docs.combine_or (thisqueryresults.docs);
+	} else if ((*query_here).combinequery == "not") {
+	  multiresults.docs.combine_not (thisqueryresults.docs);
+	}
+	multiresults.docs_matched = multiresults.docs.docset.size();
+	multiresults.is_approx = Exact;
+      }
+    }
+
+    // combine the term information 
+    if (need_term_info (request.filterResultOptions)) {
+      // append the terms
+      multiresults.orgterms.insert(multiresults.orgterms.end(),
+				   thisqueryresults.orgterms.begin(),
+				   thisqueryresults.orgterms.end());
+
+      // add the term variants
+      text_tset::iterator termvar_here = thisqueryresults.termvariants.begin();
+      text_tset::iterator termvar_end = thisqueryresults.termvariants.end();
+      while (termvar_here != termvar_end) {
+	multiresults.termvariants.insert(*termvar_here);
+	termvar_here++;
+      }
+    }
+   
+    query_here++;
+  }
+
+  // sort and unique the query terms
+  multiresults.sortuniqqueryterms ();
+}
+
+
+void queryfilterclass::sort_doc_results (const FilterRequest_t &/*request*/,
+					 docresultsclass &docs) {
+  resultsorderer_t resultsorderer;
+  resultsorderer.compare_phrase_match = true;
+  resultsorderer.docset = &(docs.docset);
+
+  // first get a list of document numbers
+  docs.docnum_order();
+
+  sort (docs.docorder.begin(), docs.docorder.end(), resultsorderer);
+}
+
+
+
+queryfilterclass::queryfilterclass () {
+  gdbmptr = NULL;
+  mgsearchptr = NULL;
+
+  FilterOption_t filtopt;
+  filtopt.name = "CombineQuery";
+  filtopt.type = FilterOption_t::enumeratedt;
+  filtopt.repeatable = FilterOption_t::onePerQuery;
+  filtopt.defaultValue = "and";
+  filtopt.validValues.push_back("and");
+  filtopt.validValues.push_back("or");
+  filtopt.validValues.push_back("not");
+  filterOptions["CombineQuery"] = filtopt;
+
+  // -- onePerQuery StartResults   integer
+  filtopt.clear();
+  filtopt.name = "StartResults";
+  filtopt.type = FilterOption_t::integert;
+  filtopt.repeatable = FilterOption_t::onePerQuery;
+  filtopt.defaultValue = "1";
+  filtopt.validValues.push_back("1");
+  filtopt.validValues.push_back("1000");
+  filterOptions["StartResults"] = filtopt;
+
+  // -- onePerQuery EndResults     integer
+  filtopt.clear();
+  filtopt.name = "EndResults";
+  filtopt.type = FilterOption_t::integert;
+  filtopt.repeatable = FilterOption_t::onePerQuery;
+  filtopt.defaultValue = "10";
+  filtopt.validValues.push_back("-1");
+  filtopt.validValues.push_back("1000");
+  filterOptions["EndResults"] = filtopt;
+
+  // -- onePerQuery QueryType      enumerated (boolean, ranked)
+  filtopt.clear();
+  filtopt.name = "QueryType";
+  filtopt.type = FilterOption_t::enumeratedt;
+  filtopt.repeatable = FilterOption_t::onePerQuery;
+  filtopt.defaultValue = "ranked";
+  filtopt.validValues.push_back("boolean");
+  filtopt.validValues.push_back("ranked");
+  filterOptions["QueryType"] = filtopt;
+
+  // -- onePerQuery MatchMode      enumerated (some, all)
+  filtopt.clear();
+  filtopt.name = "MatchMode";
+  filtopt.type = FilterOption_t::enumeratedt;
+  filtopt.repeatable = FilterOption_t::onePerQuery;
+  filtopt.defaultValue = "some";
+  filtopt.validValues.push_back("some");
+  filtopt.validValues.push_back("all");
+  filterOptions["MatchMode"] = filtopt;
+
+  // -- onePerTerm  Term           string ???
+  filtopt.clear();
+  filtopt.name = "Term";
+  filtopt.type = FilterOption_t::stringt;
+  filtopt.repeatable = FilterOption_t::onePerTerm;
+  filtopt.defaultValue = "";
+  filterOptions["Term"] = filtopt;
+
+  // -- onePerTerm  Casefold       boolean
+  filtopt.clear();
+  filtopt.name = "Casefold";
+  filtopt.type = FilterOption_t::booleant;
+  filtopt.repeatable = FilterOption_t::onePerTerm;
+  filtopt.defaultValue = "true";
+  filtopt.validValues.push_back("false");
+  filtopt.validValues.push_back("true");
+  filterOptions["Casefold"] = filtopt;
+
+  // -- onePerTerm  Stem           boolean
+  filtopt.clear();
+  filtopt.name = "Stem";
+  filtopt.type = FilterOption_t::booleant;
+  filtopt.repeatable = FilterOption_t::onePerTerm;
+  filtopt.defaultValue = "false";
+  filtopt.validValues.push_back("false");
+  filtopt.validValues.push_back("true");
+  filterOptions["Stem"] = filtopt;
+
+  // -- onePerTerm  Index          enumerated
+  filtopt.clear();
+  filtopt.name = "Index";
+  filtopt.type = FilterOption_t::enumeratedt;
+  filtopt.repeatable = FilterOption_t::onePerTerm;
+  filtopt.defaultValue = "";
+  filterOptions["Index"] = filtopt;
+
+  // -- onePerTerm  Subcollection  enumerated
+  filtopt.clear();
+  filtopt.name = "Subcollection";
+  filtopt.type = FilterOption_t::enumeratedt;
+  filtopt.repeatable = FilterOption_t::onePerTerm;
+  filtopt.defaultValue = "";
+  filterOptions["Subcollection"] = filtopt;
+
+  // -- onePerTerm  Language  enumerated
+  filtopt.clear();
+  filtopt.name = "Language";
+  filtopt.type = FilterOption_t::enumeratedt;
+  filtopt.repeatable = FilterOption_t::onePerTerm;
+  filtopt.defaultValue = "";
+  filterOptions["Language"] = filtopt;
+
+  // -- onePerQuery  Maxdocs  integer
+  filtopt.clear();
+  filtopt.name = "Maxdocs";
+  filtopt.type = FilterOption_t::integert;
+  filtopt.repeatable = FilterOption_t::onePerQuery;
+  filtopt.defaultValue = "200";
+  filtopt.validValues.push_back("-1");
+  filtopt.validValues.push_back("1000");
+  filterOptions["Maxdocs"] = filtopt;
+}
+
+queryfilterclass::~queryfilterclass () {
+}
+
+void queryfilterclass::configure (const text_t &key, const text_tarray &cfgline) {
+  filterclass::configure (key, cfgline);
+
+  if (key == "indexmap") {
+    indexmap.importmap (cfgline);
+    
+    // update the list of indexes in the filter information
+    text_tarray options;
+    indexmap.gettoarray (options);
+    filterOptions["Index"].validValues = options;
+
+  } else if (key == "defaultindex") {
+    indexmap.from2to (cfgline[0], filterOptions["Index"].defaultValue);
+
+  } else if (key == "subcollectionmap") {
+    subcollectionmap.importmap (cfgline);
+
+    // update the list of subcollections in the filter information
+    text_tarray options;
+    subcollectionmap.gettoarray (options);
+    filterOptions["Subcollection"].validValues = options;
+
+  } else if (key == "defaultsubcollection") {
+    subcollectionmap.from2to (cfgline[0], filterOptions["Subcollection"].defaultValue);
+
+  } else if (key == "languagemap") {
+    languagemap.importmap (cfgline);
+
+    // update the list of languages in the filter information
+    text_tarray options;
+    languagemap.gettoarray (options);
+    filterOptions["Language"].validValues = options;
+
+  } else if (key == "defaultlanguage") 
+    languagemap.from2to (cfgline[0], filterOptions["Language"].defaultValue);
+}
+
+bool queryfilterclass::init (ostream &logout) {
+  outconvertclass text_t2ascii;
+
+  if (!filterclass::init(logout)) return false;
+
+  // get the filename for the database and make sure it exists
+  gdbm_filename = filename_cat(collectdir,"index","text",collection);
+
+#ifdef _LITTLE_ENDIAN
+  gdbm_filename += ".ldb";
+#else
+  gdbm_filename += ".bdb";
+#endif
+  if (!file_exists(gdbm_filename)) {
+    logout << text_t2ascii
+	   << "error: gdbm database \""
+	   << gdbm_filename << "\" does not exist\n\n";
+    return false;
+  }
+
+  return true;
+}
+
+void queryfilterclass::filter (const FilterRequest_t &request,
+			       FilterResponse_t &response,
+			       comerror_t &err, ostream &logout) {
+  outconvertclass text_t2ascii;
+
+  response.clear ();
+  err = noError;
+  if (gdbmptr == NULL) {
+    // most likely a configuration problem
+    logout << text_t2ascii 
+	   << "configuration error: queryfilter contains a null gdbmclass\n\n";
+    err = configurationError;
+    return;
+  }
+  if (mgsearchptr == NULL) {
+    // most likely a configuration problem
+    logout << text_t2ascii 
+	   << "configuration error: queryfilter contains a null mgsearchclass\n\n";
+    err = configurationError;
+    return;
+  }
+
+  // open the database
+  gdbmptr->setlogout(&logout);
+  if (!gdbmptr->opendatabase (gdbm_filename, GDBM_READER, 100, false)) {
+    // most likely a system problem (we have already checked that the
+    // gdbm database exists)
+    logout << text_t2ascii
+	   << "system problem: open on gdbm database \""
+	   << gdbm_filename << "\" failed\n\n";
+    err = systemProblem;
+    return;
+  }
+
+  // get the query parameters
+  int startresults = filterOptions["StartResults"].defaultValue.getint();
+  int endresults = filterOptions["EndResults"].defaultValue.getint();
+  vector<queryparamclass> queryfilterparams;
+  parse_query_params (request, queryfilterparams, startresults, endresults, logout);  
+  
+  // do query
+  queryresultsclass queryresults;
+  do_multi_query (request, queryfilterparams, queryresults, err, logout);
+  if (err != noError) return;
+  
+  // assemble document results
+  if (need_matching_docs (request.filterResultOptions)) {
+    // sort the query results
+    sort_doc_results (request, queryresults.docs);
+    
+    int resultnum = 1;
+    ResultDocInfo_t resultdoc;
+    text_t trans_OID;
+    vector<int>::iterator docorder_here = queryresults.docs.docorder.begin();
+    vector<int>::iterator docorder_end = queryresults.docs.docorder.end();
+
+    if (endresults == -1) endresults = MAXNUMDOCS;
+    while (docorder_here != docorder_end) {
+      if (resultnum > endresults) break;
+      
+      // translate the document number
+      if (!translate(gdbmptr, *docorder_here, trans_OID)) {
+	logout << text_t2ascii
+	       << "warning: could not translate mg document number \""
+	       << *docorder_here << "\"to OID.\n\n";
+	
+      } else {
+	docresultmap::iterator docset_here = queryresults.docs.docset.find (*docorder_here);
+	
+	// see if there is a result for this number,
+	// if it is in the request set (or the request set is empty)
+	if (docset_here != queryresults.docs.docset.end() &&
+	    (request.docSet.empty() || in_set(request.docSet, trans_OID))) {
+	  if (resultnum >= startresults) {
+	    // add this document
+	    resultdoc.OID = trans_OID;
+	    resultdoc.result_num = resultnum;
+	    resultdoc.ranking = (int)((*docset_here).second.docweight * 10000.0 + 0.5);
+
+	    // these next two are not available on all versions of mg
+	    resultdoc.num_terms_matched = (*docset_here).second.num_query_terms_matched; 
+	    resultdoc.num_phrase_match = (*docset_here).second.num_phrase_match;
+	    
+	    response.docInfo.push_back (resultdoc);
+	  }
+	  
+	  resultnum++;
+	}
+      }
+      
+      docorder_here++;
+    }
+  }
+
+  // assemble the term results
+  if (need_term_info(request.filterResultOptions)) {
+    // note: the terms have already been sorted and uniqued
+
+    TermInfo_t terminfo;
+    bool terms_first = true;
+    termfreqclassarray::iterator terms_here = queryresults.terms.begin();
+    termfreqclassarray::iterator terms_end = queryresults.terms.end();
+
+    while (terms_here != terms_end) {
+      terminfo.clear();
+      terminfo.term = (*terms_here).termstr;
+      terminfo.freq = (*terms_here).termfreq;
+      if (terms_first) {
+	text_tset::iterator termvariants_here = queryresults.termvariants.begin();
+	text_tset::iterator termvariants_end = queryresults.termvariants.end();
+	while (termvariants_here != termvariants_end) {
+	  terminfo.matchTerms.push_back (*termvariants_here);
+	  termvariants_here++;
+	}
+      }
+      terms_first = false;
+
+      response.termInfo.push_back (terminfo);
+
+      terms_here++;
+    }
+  }
+
+  response.numDocs = queryresults.docs_matched;
+  response.isApprox = queryresults.is_approx;
+}
Index: /tags/gsdl-21/gsdl/src/colservr/queryfilter.h
===================================================================
--- /tags/gsdl-21/gsdl/src/colservr/queryfilter.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/colservr/queryfilter.h	(revision 699)
@@ -0,0 +1,115 @@
+/**********************************************************************
+ *
+ * queryfilter.h -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef QUERYFILTER_H
+#define QUERYFILTER_H
+
+
+#include "gsdlconf.h"
+#include "text_t.h"
+#include "comtypes.h"
+#include "filter.h"
+#include "infodbclass.h"
+#include "maptools.h"
+#include "mgsearch.h"
+#include "queryinfo.h"
+
+
+// resultsorderer_t is used to sort the query results
+struct resultsorderer_t {
+  bool compare_phrase_match;
+  bool compare_terms_match;
+  bool compare_doc_weight;
+  docresultmap *docset;
+
+  resultsorderer_t();
+  void clear();
+  bool operator()(const int &t1, const int &t2) const;
+};
+
+
+
+class queryfilterclass : public filterclass {
+protected:
+
+  stringmap indexmap;
+  stringmap subcollectionmap;
+  stringmap languagemap;
+
+  text_t gdbm_filename;
+  gdbmclass *gdbmptr;
+  
+  mgsearchclass *mgsearchptr;
+
+
+  void get_phrase_terms (const text_t &querystring, 
+			 const termfreqclassarray &orgterms,
+			 vector<termfreqclassarray> &phrases);
+
+  // do aditional query processing
+  virtual void post_process (const queryparamclass &queryparams,
+			     queryresultsclass &queryresults);
+
+  // get the query parameters
+  void parse_query_params (const FilterRequest_t &request,
+			   vector<queryparamclass> &query_params,
+			   int &startresults,
+			   int &endresults,
+			   ostream &logout);
+
+  // do query that might involve multiple sub queries
+  // mgsearchptr and gdbmptr are assumed to be valid
+  void do_multi_query (const FilterRequest_t &request,
+		       const vector<queryparamclass> &query_params,
+		       queryresultsclass &multiresults,
+		       comerror_t &err, ostream &logout);
+
+  virtual void sort_doc_results (const FilterRequest_t &request,
+				 docresultsclass &docs);
+
+  
+public:
+  queryfilterclass ();
+  virtual ~queryfilterclass ();
+
+  // the gdbmptr remains the responsability of the calling code
+  void set_gdbmptr (gdbmclass *thegdbmptr) {gdbmptr=thegdbmptr;}
+
+  // the mgsearchptr remains the responsability of the calling code
+  void set_mgsearchptr (mgsearchclass *themgsearchptr) {mgsearchptr=themgsearchptr;}
+
+  void configure (const text_t &key, const text_tarray &cfgline);
+  bool init (ostream &logout);
+  text_t get_filter_name () {return "QueryFilter";}
+  void filter (const FilterRequest_t &request,
+	       FilterResponse_t &response,
+	       comerror_t &err, ostream &logout);
+};
+
+
+#endif
Index: /tags/gsdl-21/gsdl/src/colservr/queryinfo.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/colservr/queryinfo.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/colservr/queryinfo.cpp	(revision 699)
@@ -0,0 +1,443 @@
+/**********************************************************************
+ *
+ * queryinfo.cpp -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.12  1999/09/07 04:57:24  sjboddie
+   added gpl notice
+
+   Revision 1.11  1999/08/31 22:47:57  rjmcnab
+   Added MatchMode.
+
+   Revision 1.10  1999/07/16 03:42:23  sjboddie
+   changed isApprox
+
+   Revision 1.9  1999/07/16 00:14:01  sjboddie
+   added termfreqclassarray type
+
+   Revision 1.8  1999/07/09 02:19:44  rjmcnab
+   Fixed a couple of compiler conflicts
+
+   Revision 1.7  1999/07/07 06:19:47  rjmcnab
+   Added ability to combine two or more independant queries.
+
+   Revision 1.6  1999/07/01 09:29:21  rjmcnab
+   Changes for better reporting of number documents which match a query. Changes
+   should still work as before with older versions of mg.
+
+   Revision 1.5  1999/07/01 03:56:17  rjmcnab
+   Added a set of utf8 encoded equivalent terms of a query term. I also
+   added a flag for handling post-processing of the query.
+
+   Revision 1.4  1999/06/30 04:04:13  rjmcnab
+   made stemming functions available from mgsearch and made the stems
+   for the query terms available in queryinfo
+
+   Revision 1.3  1999/06/29 22:06:23  rjmcnab
+   Added a couple of fields to queryinfo to handle a special version
+   of mg.
+
+   Revision 1.2  1999/01/12 01:51:02  rjmcnab
+
+   Standard header.
+
+   Revision 1.1  1999/01/08 09:02:18  rjmcnab
+
+   Moved from src/library.
+
+ */
+
+
+#include "queryinfo.h"
+
+
+// query parameters
+
+queryparamclass::queryparamclass () {
+  clear ();
+}
+
+void queryparamclass::clear () {
+  combinequery.clear();
+  collection.clear();
+  index.clear();
+  subcollection.clear();
+  language.clear();
+  querystring.clear();
+  search_type = 0; // 0 = boolean, 1 = ranked
+  match_mode = 0; // 0 = some, 1 = all
+  casefolding = 0;
+  stemming = 0;
+  maxdocs = -1;    // all
+}
+
+
+queryparamclass &queryparamclass::operator=(const queryparamclass &q) {
+  combinequery = q.combinequery;
+  collection = q.collection;
+  index = q.index;
+  subcollection = q.subcollection;
+  language = q.language;
+  querystring = q.querystring;
+  search_type = q.search_type;
+  match_mode = q.match_mode;
+  casefolding = q.casefolding;
+  stemming = q.stemming;
+  maxdocs = q.maxdocs;
+
+  return *this;
+}
+
+
+bool operator==(const queryparamclass &x, const queryparamclass &y) {
+  return ((x.combinequery == y.combinequery) &&
+	  (x.collection == y.collection) &&
+	  (x.index == y.index) &&
+	  (x.subcollection == y.subcollection) &&
+	  (x.language == y.language) &&
+	  (x.querystring == y.querystring) &&
+	  (x.search_type == y.search_type) &&
+	  (x.match_mode == y.match_mode) &&
+	  (x.casefolding == y.casefolding) &&
+	  (x.stemming == y.stemming) &&
+	  (x.maxdocs == y.maxdocs));
+}
+
+bool operator!=(const queryparamclass &x, const queryparamclass &y) {
+  return !(x == y);
+}
+
+
+ostream &operator<< (ostream &outs, queryparamclass &q) {
+  outconvertclass text_t2ascii;
+
+  outs << "*** queryparamclass\n";
+  outs << text_t2ascii << " combinequery = \"" << q.combinequery << "\"\n";
+  outs << text_t2ascii << " collection = \"" << q.collection << "\"\n";
+  outs << text_t2ascii << " index = \"" << q.index << "\"\n";
+  outs << text_t2ascii << " subcollection = \"" << q.subcollection << "\"\n";
+  outs << text_t2ascii << " language = \"" << q.language << "\"\n";
+  outs << text_t2ascii << " querystring = \"" << q.querystring << "\"\n";
+  outs << " search_type = \"" << q.search_type << "\"\n";
+  outs << " match_mode = \"" << q.match_mode << "\"\n";
+  outs << " casefolding = \"" << q.casefolding << "\"\n";
+  outs << " stemming = \"" << q.stemming << "\"\n";
+  outs << " maxdocs = \"" << q.maxdocs << "\"\n";
+  outs << "\n";
+
+  return outs;
+}
+
+
+
+
+// term frequencies
+
+termfreqclass::termfreqclass () {
+  clear();
+}
+
+void termfreqclass::clear() {
+  termstr.clear();
+  termstemstr.clear();
+  utf8equivterms.erase(utf8equivterms.begin(), utf8equivterms.end());
+  termfreq = 0;
+}
+
+termfreqclass &termfreqclass::operator=(const termfreqclass &t)  {
+  termstr = t.termstr;
+  termstemstr = t.termstemstr;
+  utf8equivterms = t.utf8equivterms;
+  termfreq = t.termfreq;
+
+  return *this;
+}
+	
+bool operator==(const termfreqclass &x, const termfreqclass &y) {
+  return ((x.termstr == y.termstr) &&
+	  (x.termstemstr == y.termstemstr) &&
+	  (x.termfreq == y.termfreq));
+}
+
+bool operator!=(const termfreqclass &x, const termfreqclass &y) {
+  return !(x == y);
+}
+
+// ordered by termfreq and then by termstr
+bool operator<(const termfreqclass &x, const termfreqclass &y) {
+  return ((x.termfreq < y.termfreq) ||
+	  ((x.termfreq == y.termfreq) && (x.termstemstr < y.termstemstr)) ||
+	  ((x.termfreq == y.termfreq) && (x.termstemstr == y.termstemstr) && (x.termstr < y.termstr)));
+}
+
+bool operator>(const termfreqclass &x, const termfreqclass &y) {
+  return ((x.termfreq > y.termfreq) ||
+	  ((x.termfreq == y.termfreq) && (x.termstemstr > y.termstemstr)) ||
+	  ((x.termfreq == y.termfreq) && (x.termstemstr == y.termstemstr) && (x.termstr > y.termstr)));
+}
+
+// stream output for debugging purposes
+ostream &operator<< (ostream &outs, termfreqclass &t) {
+  outconvertclass text_t2ascii;
+
+  outs << text_t2ascii << " t:\"" << t.termstr << "\"";
+  outs << text_t2ascii << " s:\"" << t.termstemstr << "\"";
+  outs << " f:" << t.termfreq << "\n";
+
+  return outs;
+}
+
+
+
+// one query result
+
+docresultclass::docresultclass() {
+  clear ();
+}
+
+void docresultclass::clear () {
+  docnum=-1;
+  docweight=0.0;
+  num_query_terms_matched=0;
+  num_phrase_match=0;
+}
+
+// merges two result classes relating to a single docnum
+docresultclass &docresultclass::combine(const docresultclass &d) {
+  docweight += d.docweight; // budget!
+  num_query_terms_matched += d.num_query_terms_matched;
+  num_phrase_match += d.num_phrase_match;
+
+  return *this;
+}
+
+docresultclass &docresultclass::operator=(const docresultclass &d) {
+  docnum = d.docnum;
+  docweight = d.docweight;
+  num_query_terms_matched = d.num_query_terms_matched;
+  num_phrase_match = d.num_phrase_match;
+
+  return *this;
+}
+
+
+bool operator==(const docresultclass &x, const docresultclass &y) {
+  return ((x.docnum == y.docnum) && (x.docweight == y.docweight) &&
+	  (x.num_query_terms_matched == y.num_query_terms_matched) &&
+	  (x.num_phrase_match == y.num_phrase_match));
+}
+
+bool operator<(const docresultclass &x, const docresultclass &y) {
+  return ((x.docnum < y.docnum) ||
+	  ((x.docnum == y.docnum) &&
+	   ((x.docweight < y.docweight) ||
+	    ((x.docweight == y.docweight) &&
+	     ((x.num_query_terms_matched < y.num_query_terms_matched) ||
+	      ((x.num_query_terms_matched == y.num_query_terms_matched) &&
+	       ((x.num_phrase_match < y.num_phrase_match))))))));
+}
+
+
+// stream output for debugging purposes
+ostream &operator<< (ostream &outs, docresultclass &a) {
+  outs << " d:" << a.docnum << " w:" << a.docweight << "\n";
+  return outs;
+}
+
+
+
+// many document results
+
+docresultsclass::docresultsclass () {
+  clear ();
+}
+
+void docresultsclass::clear () {
+  docset.erase(docset.begin(), docset.end());
+  docorder.erase(docorder.begin(), docorder.end());
+}
+
+void docresultsclass::docnum_order() {
+  docorder.erase(docorder.begin(), docorder.end());
+  
+  docresultmap::iterator here = docset.begin();
+  docresultmap::iterator end = docset.end();
+  while (here != end) {
+    docorder.push_back ((*here).first);
+    here++;
+  }
+}
+
+void docresultsclass::combine_and (const docresultsclass &d) {
+  docorder.erase(docorder.begin(), docorder.end());
+
+  // put the resulting set in tempresults
+  docresultmap tempresults;
+  
+  docresultmap::const_iterator d_here = d.docset.begin();
+  docresultmap::const_iterator d_end = d.docset.end();
+  docresultmap::iterator found = docset.end();
+  while (d_here != d_end) {
+    found = docset.find((*d_here).first);
+    if (found != docset.end()) {
+      (*found).second.combine ((*d_here).second);
+      tempresults[(*found).first] = (*found).second;
+    }
+    d_here++;
+  }
+
+  // then copy it back to docset
+  docset = tempresults;
+}
+
+void docresultsclass::combine_or (const docresultsclass &d) {
+  docorder.erase(docorder.begin(), docorder.end());
+    
+  docresultmap::const_iterator d_here = d.docset.begin();
+  docresultmap::const_iterator d_end = d.docset.end();
+  docresultmap::iterator found = docset.end();
+  while (d_here != d_end) {
+    found = docset.find((*d_here).first);
+    if (found != docset.end()) {
+      (*found).second.combine ((*d_here).second);
+    } else {
+      docset[(*d_here).first] = (*d_here).second;
+    }
+    d_here++;
+  }
+}
+
+void docresultsclass::combine_not (const docresultsclass &d) {
+  docorder.erase(docorder.begin(), docorder.end());
+
+ docresultmap::const_iterator d_here = d.docset.begin();
+ docresultmap::const_iterator d_end = d.docset.end();
+ docresultmap::iterator found = docset.end();
+ while (d_here != d_end) {
+   found = docset.find((*d_here).first);
+   if (found != docset.end()) docset.erase (found);
+   d_here++;
+ }
+}
+
+docresultsclass &docresultsclass::operator=(const docresultsclass &d) {
+  docset = d.docset;
+  docorder = d.docorder;
+
+  return *this;
+}
+
+
+
+
+// query results
+
+void queryresultsclass::clear () {
+  docs_matched = 0;
+  is_approx = Exact;
+
+  postprocessed = false;
+
+  docs.clear();
+  orgterms.erase(orgterms.begin(),orgterms.end());
+  terms.erase(terms.begin(),terms.end());
+}
+
+queryresultsclass &queryresultsclass::operator=(const queryresultsclass &q) {
+  docs_matched = q.docs_matched;
+  is_approx = q.is_approx;
+
+  postprocessed = q.postprocessed;
+  
+  docs = q.docs;
+  terms = q.terms;
+  termvariants = q.termvariants;
+
+  return *this;
+}
+
+void queryresultsclass::sortuniqqueryterms() {
+  termfreqclassarray tempterms = orgterms;
+  text_tset seenterms;
+  terms.erase(terms.begin(), terms.end());
+
+  // sort the terms to get the frequencies in ascending order
+  sort (tempterms.begin(), tempterms.end());
+
+  // insert first occurance of each term (maximum)
+  termfreqclassarray::reverse_iterator here = tempterms.rbegin();
+  termfreqclassarray::reverse_iterator end = tempterms.rend();
+  while (here != end) {
+    if (seenterms.find((*here).termstr) == seenterms.end()) {
+      // the termstemstr and utf8equivterms might be different for
+      // different occurances of the term
+      (*here).termstemstr.clear();
+      (*here).utf8equivterms.erase((*here).utf8equivterms.begin(),
+				   (*here).utf8equivterms.end());
+      terms.push_back(*here);
+      seenterms.insert((*here).termstr);
+    }
+    here++;
+  }
+
+  // now re-sort in ascending order
+  sort (terms.begin(), terms.end());
+}
+
+
+// stream output for debugging purposes
+ostream &operator<< (ostream &outs, queryresultsclass &q) {
+  outs << "*** queryresultsclass\n";
+  outs << "docs\n";
+
+  docresultmap::iterator docshere = q.docs.docset.begin();
+  docresultmap::iterator docsend = q.docs.docset.end();
+  while (docshere != docsend) {
+    outs << (*docshere).second;
+    docshere++;
+  }
+
+  outs << "orgterms\n";
+  termfreqclassarray::iterator orgtermshere = q.orgterms.begin();
+  termfreqclassarray::iterator orgtermsend = q.orgterms.end();
+  while (orgtermshere != orgtermsend) {
+    outs << (*orgtermshere);
+    orgtermshere++;
+  }
+
+  outs << "terms\n";
+  termfreqclassarray::iterator termshere = q.terms.begin();
+  termfreqclassarray::iterator termsend = q.terms.end();
+  while (termshere != termsend) {
+    outs << (*termshere);
+    termshere++;
+  }
+
+  outs << "\n";
+
+  return outs;
+}
Index: /tags/gsdl-21/gsdl/src/colservr/queryinfo.h
===================================================================
--- /tags/gsdl-21/gsdl/src/colservr/queryinfo.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/colservr/queryinfo.h	(revision 699)
@@ -0,0 +1,189 @@
+/**********************************************************************
+ *
+ * queryinfo.h -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef QUERYINFO_H
+#define QUERYINFO_H
+
+
+#include "gsdlconf.h"
+#include "text_t.h"
+#include "comtypes.h"
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\vector>
+#  include <ospace\std\algorithm>
+#elif defined(GSDL_USE_STL_H)
+#  include <vector.h>
+#  if defined(GSDL_USE_ALGO_H)
+#    include <algo.h>
+#  else
+#    include <algorithm.h>
+#  endif
+#else
+#  include <vector>
+#  include <algorithm>
+#endif
+
+// query parameters
+
+struct queryparamclass {
+  text_t combinequery;
+  text_t collection;
+
+  // search_index = index+subcollection+language
+  text_t index;
+  text_t subcollection;
+  text_t language;
+  
+  text_t querystring;
+  int search_type; // 0 = boolean, 1 = ranked
+  int match_mode; // 0 = some, 1 = all
+  int casefolding;
+  int stemming;
+  int maxdocs;
+
+  queryparamclass ();
+  void clear ();
+  queryparamclass &operator=(const queryparamclass &q);
+};
+
+bool operator==(const queryparamclass &x, const queryparamclass &y);
+bool operator!=(const queryparamclass &x, const queryparamclass &y);
+
+// stream output for debugging purposes
+ostream &operator<< (ostream &outs, queryparamclass &q);
+
+
+
+// term frequencies
+
+struct termfreqclass {
+  text_t termstr;
+  text_t termstemstr;
+  text_tset utf8equivterms; // kept as utf8 string for fast matching
+  unsigned int termfreq;
+
+  termfreqclass ();
+  void clear();
+  termfreqclass &operator=(const termfreqclass &t);
+};
+
+typedef vector<termfreqclass> termfreqclassarray;
+
+bool operator==(const termfreqclass &x, const termfreqclass &y);
+bool operator!=(const termfreqclass &x, const termfreqclass &y);
+bool operator<(const termfreqclass &x, const termfreqclass &y);
+bool operator>(const termfreqclass &x, const termfreqclass &y);
+
+// stream output for debugging purposes
+ostream &operator<< (ostream &outs, termfreqclass &q);
+
+
+
+// one query result
+
+struct docresultclass {
+  int docnum;
+  float docweight;
+  unsigned int num_query_terms_matched;  // not available on all versions of mg
+  int num_phrase_match;                  // not available on all versions of mg
+
+  docresultclass();
+  ~docresultclass() {}
+  void clear ();
+  
+  // merges two result classes relating to a single docnum
+  docresultclass &combine(const docresultclass &d);
+
+  docresultclass &operator=(const docresultclass &d);
+};
+
+bool operator==(const docresultclass &x, const docresultclass &y);
+bool operator<(const docresultclass &x, const docresultclass &y);
+
+
+// stream output for debugging purposes
+ostream &operator<< (ostream &outs, docresultclass &a);
+
+
+struct ltint {
+  bool operator()(const int &t1, const int &t2) const
+  { return t1 < t2; }
+};
+
+typedef map<int, docresultclass, ltint> docresultmap;
+
+
+
+// many document results
+
+struct docresultsclass {
+  docresultmap docset;
+  vector<int> docorder;
+
+  docresultsclass ();
+  void clear ();
+  void docnum_order();
+
+  void combine_and (const docresultsclass &d);
+  void combine_or (const docresultsclass &d);
+  void combine_not (const docresultsclass &d);
+  
+  docresultsclass &operator=(const docresultsclass &d);
+};
+
+
+
+
+// query results
+
+struct queryresultsclass {
+  queryresultsclass () {clear();}
+
+  int docs_matched; // not available on all versions of mg
+  isapprox is_approx;
+  //  bool is_approx;   // not available on all versions of mg
+
+  bool postprocessed; // whether this record has been post-processed
+  
+  docresultsclass docs;
+  termfreqclassarray orgterms; // terms before they are sorted and uniqued
+  termfreqclassarray terms;
+  text_tset termvariants;
+  
+  void clear ();
+  queryresultsclass &operator=(const queryresultsclass &q);
+  
+  void sortuniqqueryterms();
+};
+
+// stream output for debugging purposes
+ostream &operator<< (ostream &outs, queryresultsclass &q);
+
+
+#endif
Index: /tags/gsdl-21/gsdl/src/colservr/source.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/colservr/source.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/colservr/source.cpp	(revision 699)
@@ -0,0 +1,134 @@
+/**********************************************************************
+ *
+ * source.cpp -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.8  1999/10/10 08:20:36  sjboddie
+   - metadata now returns map rather than array
+   - redesigned browsing support (although it's not finished so
+   won't currently work ;-)
+
+   Revision 1.7  1999/09/07 04:57:25  sjboddie
+   added gpl notice
+
+   Revision 1.6  1999/08/31 22:49:01  rjmcnab
+   Changes for AIX.
+
+   Revision 1.5  1999/06/16 02:00:33  sjboddie
+   Few changes to get getParent filter option to return metadata of
+   parents as well as current OID
+
+   Revision 1.4  1999/05/10 03:43:49  sjboddie
+   lots of changes to lots of files - getting document action going
+
+   Revision 1.3  1999/04/30 02:00:48  sjboddie
+   lots of stuff to do with getting documentaction working
+
+   Revision 1.2  1999/04/06 22:20:32  rjmcnab
+   Got browsefilter working.
+
+   Revision 1.1  1999/03/31 23:43:40  rjmcnab
+   Initial revision
+
+ */
+
+
+#include "source.h"
+#include <assert.h>
+
+
+// default constructor does nothing
+sourceclass::sourceclass () {
+}
+
+// default destructor does nothing
+sourceclass::~sourceclass () {
+}
+
+// configure should be called once for each configuration line
+// default does nothing
+void sourceclass::configure (const text_t &/*key*/, const text_tarray &/*cfgline*/) {
+}
+
+// init should be called after all the configuration is done but
+// before any other methods are called
+// default does nothing
+bool sourceclass::init (ostream &/*logout*/) {
+  return true;
+}
+
+
+// translate_OID translates OIDs using ".pr", ."fc" etc.
+bool sourceclass::translate_OID (const text_t &/*OIDin*/, text_t &/*OIDout*/, 
+				 comerror_t &err, ostream &/*logout*/) {
+  err = noError;
+
+  return false;
+}
+
+
+// get_metadata fills out the metadata if possible, if it is not responsable
+// for the given OID then it will return false.
+bool sourceclass::get_metadata (const text_t &/*requestParams*/, const text_t &/*refParams*/, 
+				bool /*getParents*/, const text_tset &/*fields*/, 
+				const text_t &/*OID*/, MetadataInfo_tmap &metadata, 
+				comerror_t &err, ostream &/*logout*/) {
+  metadata.erase(metadata.begin(), metadata.end());
+  err = noError;
+
+  return false;
+}
+
+bool sourceclass::get_document (const text_t &/*OID*/, text_t &/*doc*/, 
+				comerror_t &err, ostream &/*logout*/) {
+  err = noError;
+
+  return false;
+}
+
+
+bool operator==(const sourceptr &x, const sourceptr &y) {
+  return (x.s == y.s);
+}
+
+bool operator<(const sourceptr &x, const sourceptr &y) {
+  return (x.s < y.s);
+}
+
+
+// thesource remains the property of the calling code but
+// should not be deleted until it is removed from this list.
+void sourcelistclass::addsource (sourceclass *thesource) {
+  // can't add a source that doesn't exist
+  assert (thesource != NULL);
+  if (thesource == NULL) return;
+
+  sourceptr sp;
+  sp.s = thesource;
+
+  sourceptrs.push_back(sp);
+}
Index: /tags/gsdl-21/gsdl/src/colservr/source.h
===================================================================
--- /tags/gsdl-21/gsdl/src/colservr/source.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/colservr/source.h	(revision 699)
@@ -0,0 +1,122 @@
+/**********************************************************************
+ *
+ * source.h -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef SOURCE_H
+#define SOURCE_H
+
+#include "gsdlconf.h"
+#include "text_t.h"
+#include "comtypes.h"
+
+
+class sourceclass {
+public:
+  sourceclass ();
+  virtual ~sourceclass ();
+
+  // configure should be called once for each configuration line
+  virtual void configure (const text_t &key, const text_tarray &cfgline);
+
+  // init should be called after all the configuration is done but
+  // before any other methods are called
+  virtual bool init (ostream &logout);
+
+  // translate_OID translates OIDs using ".pr", ."fc" etc.
+  virtual bool translate_OID (const text_t &OIDin, text_t &OIDout, 
+			      comerror_t &err, ostream &logout);
+
+  // get_metadata fills out the metadata if possible, if it is not responsable
+  // for the given OID then it will return false.
+  virtual bool get_metadata (const text_t &requestParams, const text_t &refParams, 
+			     bool getParents, const text_tset &fields,
+			     const text_t &OID, MetadataInfo_tmap &metadata, 
+			     comerror_t &err, ostream &logout);
+
+  virtual bool get_document (const text_t &OID, text_t &doc, 
+			     comerror_t &err, ostream &logout);
+};
+
+
+// The sourceptr class does not 'own' the source. The 
+// source should be deleted by the code which created it.
+class sourceptr {
+public:
+  sourceclass *s;
+
+  sourceptr () {s=NULL;}
+  ~sourceptr () {}
+};
+
+bool operator==(const sourceptr &x, const sourceptr &y);
+bool operator<(const sourceptr &x, const sourceptr &y);
+
+typedef vector<sourceptr> sourceptrlist;
+
+// contains a list of sources
+class sourcelistclass {
+protected:
+  sourceptrlist sourceptrs;
+
+public:
+  // type support for sourcelistclass
+  typedef sourceptrlist::iterator iterator;
+  typedef sourceptrlist::const_iterator const_iterator;
+  typedef sourceptrlist::reference reference;
+  typedef sourceptrlist::const_reference const_reference;
+  typedef sourceptrlist::size_type size_type;
+
+  typedef sourceptrlist::difference_type difference_type;
+  typedef sourceptrlist::const_reverse_iterator const_reverse_iterator;
+  typedef sourceptrlist::reverse_iterator reverse_iterator;
+  
+  // basic container support
+  iterator begin () {return sourceptrs.begin();}
+  const_iterator begin () const {return sourceptrs.begin();}
+  iterator end () {return sourceptrs.end();}
+  const_iterator end () const {return sourceptrs.end();}
+
+  void erase(iterator pos) {sourceptrs.erase(pos);}
+  void erase(iterator first, iterator last) {sourceptrs.erase(first, last);}
+  sourcelistclass &operator=(const sourcelistclass &x) 
+    {sourceptrs=x.sourceptrs;return *this;}
+
+  bool empty () const {return sourceptrs.empty();}
+  size_type size() const {return sourceptrs.size();}
+
+
+  // added functionality
+  void clear () {sourceptrs.erase(sourceptrs.begin(),sourceptrs.end());}
+
+  // thesource remains the property of the calling code but
+  // should not be deleted until it is removed from this list.
+  void addsource (sourceclass *thesource);
+};
+
+
+
+#endif
Index: /tags/gsdl-21/gsdl/src/db2txt/Makefile.in
===================================================================
--- /tags/gsdl-21/gsdl/src/db2txt/Makefile.in	(revision 699)
+++ /tags/gsdl-21/gsdl/src/db2txt/Makefile.in	(revision 699)
@@ -0,0 +1,80 @@
+###########################################################################
+#
+# Makefile -- Makefile for db2txt
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 Rodger McNab, Stefan Boddie
+#
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)Makefile.in	1.3 16 Mar 1994
+#
+###########################################################################
+
+ 
+AR = ar
+CC = @CC@
+CCFLAGS = -Wall @CFLAGS@ 
+DEFS = @DEFS@
+RANLIB = @RANLIB@
+INCLUDES = -I../../lib
+LDFLAGS =
+DPLIBS = 
+LIBS = -lstdc++ -lgdbm
+GSDLOS = @gsdlos@
+
+ 
+COMPILE = $(CC) -c $(DEFS) $(INCLUDES) $(CCFLAGS)
+LINK = $(CC) $(LDFLAGS) -o $@
+ 
+.SUFFIXES:
+.SUFFIXES: .c .o
+.c.o:
+	$(COMPILE) $<
+
+.SUFFIXES: .cpp .o
+.cpp.o:
+	$(COMPILE) $< 
+ 
+
+HEADERS = 
+
+SOURCES = db2txt.cpp
+
+OBJECTS = db2txt.o
+
+EXEC    = db2txt
+ 
+all : $(EXEC)
+
+install: $(EXEC)
+	install -m 755 -s $(EXEC) ../../bin/$(GSDLOS)
+
+clean:
+	rm -f $(OBJECTS) $(EXEC)
+
+depend:
+	makedepend -Y -- $(DEFS) $(INCLUDES) $(CCFLAGS) -- $(SOURCES)
+
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+DB2TXT_OBJS = db2txt.o
+
+db2txt: $(DB2TXT_OBJS) $(DPLIBS)
+	$(LINK) $(DB2TXT_OBJS) $(LIBS) 
Index: /tags/gsdl-21/gsdl/src/db2txt/db2txt.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/db2txt/db2txt.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/db2txt/db2txt.cpp	(revision 699)
@@ -0,0 +1,87 @@
+/**********************************************************************
+ *
+ * db2txt.cpp -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+
+#ifdef __WIN32__
+#include "autoconf.h"
+#include "systems.h"
+#include "gdbmconst.h"
+#include "gdbm.h"
+
+#else
+#include <gdbm.h>
+#endif
+
+#include "gsdlconf.h"
+#include <stdlib.h>
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#include <ospace\std\iostream>
+#elif defined(GSDL_USE_IOS_H)
+#include <iostream.h>
+#else
+#include <iostream>
+#endif
+
+void print_usage (char *program_name) {
+  cerr << "usage: " << program_name << " database-name\n\n";
+}
+
+
+int main (int argc, char *argv[]) {
+  int block_size = 0, i;
+  GDBM_FILE dbf;
+  datum key, value;
+      
+  // sanity check
+  if (argc != 2) {
+    print_usage (argv[0]);
+    exit (0);
+  }
+  
+  // open the database
+  dbf = gdbm_open  (argv[1], block_size, GDBM_READER, 00664, NULL);
+  if (dbf == NULL) {
+    cerr << "couldn't create " << argv[1] << "\n";
+    exit (0);
+  }
+
+  key = gdbm_firstkey (dbf);
+  while (key.dptr != NULL) {
+    cout << "[";
+    for (i = 0; i < key.dsize; i++)
+      cout << key.dptr[i];
+    cout << "]\n";
+    value = gdbm_fetch (dbf, key);
+    for (i = 0; i < value.dsize; i++)
+      cout << value.dptr[i];
+    cout << "\n----------------------------------------------------------------------\n";
+    key = gdbm_nextkey (dbf, key);
+    delete value.dptr;
+  }
+  
+  gdbm_close (dbf);
+  return 0;
+}
Index: /tags/gsdl-21/gsdl/src/hashfile/Makefile.in
===================================================================
--- /tags/gsdl-21/gsdl/src/hashfile/Makefile.in	(revision 699)
+++ /tags/gsdl-21/gsdl/src/hashfile/Makefile.in	(revision 699)
@@ -0,0 +1,79 @@
+###########################################################################
+#
+# Makefile -- Makefile for hashfile
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 Rodger McNab, Stefan Boddie
+#
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)Makefile.in	1.3 16 Mar 1994
+#
+###########################################################################
+
+ 
+AR = ar
+CC = @CC@
+CCFLAGS = -Wall @CFLAGS@
+DEFS = @DEFS@
+RANLIB = @RANLIB@
+INCLUDES = -I../../lib
+LDFLAGS = 
+LIBS = ../../lib/gsdllib.a
+GSDLOS = @gsdlos@
+
+ 
+COMPILE = $(CC) -c $(DEFS) $(INCLUDES) $(CCFLAGS)
+LINK = $(CC) $(LDFLAGS) -o $@
+ 
+.SUFFIXES:
+.SUFFIXES: .c .o
+.c.o:
+	$(COMPILE) $<
+
+.SUFFIXES: .cpp .o
+.cpp.o:
+	$(COMPILE) $< 
+ 
+
+HEADERS = 
+
+SOURCES = hashfile.cpp
+
+OBJECTS = hashfile.o
+
+EXEC    = hashfile
+ 
+all : $(EXEC)
+
+install: $(EXEC)
+	install -m 755 -s $(EXEC) ../../bin/$(GSDLOS)
+
+clean:
+	rm -f $(OBJECTS) $(EXEC)
+
+depend:
+	makedepend -Y -- $(DEFS) $(INCLUDES) $(CCFLAGS) -- $(SOURCES)
+
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+HASHFILE_OBJS = hashfile.o
+
+hashfile: $(HASHFILE_OBJS) $(LIBS)
+	$(LINK) $(HASHFILE_OBJS) $(LIBS) 
Index: /tags/gsdl-21/gsdl/src/hashfile/hashfile.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/hashfile/hashfile.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/hashfile/hashfile.cpp	(revision 699)
@@ -0,0 +1,217 @@
+/**********************************************************************
+ *
+ * hashfile.cpp -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+#include <stdio.h>
+
+#define MAXNUMLEN  100   // bytes
+
+
+// The number class is used to store very long unsigned integers.
+// The least significant of each number is stored first. Numbers can 
+// be no longer than MAXNUMLEN bytes long. A number with length=0
+// is considered to be equal to 0. There should be no extra zeros
+// at the front of the number.
+
+struct number {
+  unsigned int len;
+  unsigned char num[MAXNUMLEN];
+
+  number () {len = 0;}
+};
+
+
+// evaluates a = a + b
+// a and b can be the same number
+void my_inc (number &a, number &b) {
+  unsigned int carry = 0;
+  unsigned int num = 0;
+  unsigned int i = 0;
+
+  while ((carry || (i < a.len) || (i < b.len)) && (i < MAXNUMLEN)) {
+    num = carry;
+    if (i < a.len) num += a.num[i];
+    if (i < b.len) num += b.num[i];
+
+    if (num >= 256) {
+      num -= 256; // assumes that each number in the list < 256
+      carry = 1;
+    } else {
+      carry = 0;
+    }
+    a.num[i] = (unsigned char)num;
+
+    i++;
+  }
+
+  a.len = i;
+}
+
+
+// evaluates a = a - b if a >= b
+// returns true if the subtraction was possible
+int my_ifpos_dec (number &a, number &b) {
+  int carry = 0;
+  int num = 0;
+  int i = 0;
+  unsigned int len = 0; // the last non-zero digit
+
+  // make sure a >= b
+  if (b.len > a.len) return 0;
+  if (b.len == a.len) {
+    i = a.len - 1;
+    while (i >= 0) {
+      if (a.num[i] > b.num[i]) break;
+      if (a.num[i] < b.num[i]) return 0;
+      i--;
+    }
+  }
+
+  unsigned int j = 0;
+  len = 0;
+  while ((j < a.len) || (j < b.len)) {
+    num = -carry;
+    if (j < a.len) num += a.num[j];
+    if (j < b.len) num -= b.num[j];
+
+    if (num < 0) {
+      num += 256; // assumes each number in the list < 256
+      carry = 1;
+    } else {
+      carry = 0;
+    }
+
+    a.num[j] = (unsigned char) num;
+    
+    j++;
+    if (num != 0) len = j;
+  }
+
+  a.len = len;
+
+  if (carry) {
+    // panic, this should NOT happen
+    fprintf (stderr, "panic: error in my_ifpos_dec\n");
+  }
+
+  return 1;
+}
+
+
+char *my_convert_num (number &a) {
+  char *result = new char[a.len+2];
+  int i,len;
+  char convert[16] = {'0','1','2','3','4','5','6','7',
+		      '8','9','a','b','c','d','e','f'};
+
+  // zero might be a special case
+  if (a.len == 0) {
+    result[0] = '0';
+    result[1] = '\0';
+    return result;
+  }
+
+  for (i=a.len-1, len=0; i >= 0; i--) {
+    result[len++] = convert[a.num[i]/16];
+    result[len++] = convert[a.num[i]%16];
+  }
+  result[len] = '\0';
+
+  return result;
+}
+
+
+char *hashfile (char *filename) {
+  FILE *infile = (FILE *)NULL;
+  int i;
+
+  // calculate the 8 multiples of the prime number to use
+  // in the long division
+  number primepow[8];
+  number pow;
+  number remainder;
+  unsigned char c;
+
+  // our prime number (618970019642690137449562111)
+  pow.num[0] = 255;  pow.num[1] = 255;
+  pow.num[2] = 255;  pow.num[3] = 255;
+  pow.num[4] = 255;  pow.num[5] = 255;
+  pow.num[6] = 255;  pow.num[7] = 255;
+  pow.num[8] = 255;  pow.num[9] = 255;
+  pow.num[10] = 255; pow.num[11] = 1;
+  pow.len = 12;
+
+  for (i=0; i<8; i++) {
+    primepow[i] = pow;
+    my_inc (pow, pow);
+  }
+
+  infile = fopen (filename, "rb");
+  if (infile == NULL) {
+    return (char *)NULL;
+  }
+  
+  remainder.len = 0;
+  c = (unsigned char)fgetc(infile);
+  while (!feof (infile)) {
+    // remainder = remainder * 256 + c
+    if (remainder.len == MAXNUMLEN-1) {
+      fprintf (stderr, "ERROR - number overflow\n");
+      return (char *)NULL;
+    }
+    for (i=remainder.len; i>0; i--) {
+      remainder.num[i] = remainder.num[i-1];
+    }
+    remainder.num[0] = c;
+	if (remainder.len > 0 || c != 0) remainder.len = remainder.len+1;
+    
+    for (i=7; i>=0; i--) {
+      my_ifpos_dec (remainder, primepow[i]);
+    }
+
+    c = (unsigned char)fgetc(infile);
+  }
+
+  fclose (infile);
+
+  return my_convert_num (remainder);
+}
+
+
+int main (int argc, char *argv[]) {
+  int i;
+  char *result;
+
+  for (i = 1; i < argc; i++) {
+    result = hashfile (argv[i]);
+    if (result == NULL) {
+      printf ("ERROR - could not process %s\n", argv[i]);
+    } else {
+      printf ("%s: %s\n", argv[i], result);
+      delete [] result;
+    }
+  }
+
+  return 0;
+}
Index: /tags/gsdl-21/gsdl/src/mgpp/INSTALL
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/INSTALL	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/INSTALL	(revision 699)
@@ -0,0 +1,176 @@
+Basic Installation
+==================
+
+   These are generic installation instructions.
+
+   The `configure' shell script attempts to guess correct values for
+various system-dependent variables used during compilation.  It uses
+those values to create a `Makefile' in each directory of the package.
+It may also create one or more `.h' files containing system-dependent
+definitions.  Finally, it creates a shell script `config.status' that
+you can run in the future to recreate the current configuration, a file
+`config.cache' that saves the results of its tests to speed up
+reconfiguring, and a file `config.log' containing compiler output
+(useful mainly for debugging `configure').
+
+   If you need to do unusual things to compile the package, please try
+to figure out how `configure' could check whether to do them, and mail
+diffs or instructions to the address given in the `README' so they can
+be considered for the next release.  If at some point `config.cache'
+contains results you don't want to keep, you may remove or edit it.
+
+   The file `configure.in' is used to create `configure' by a program
+called `autoconf'.  You only need `configure.in' if you want to change
+it or regenerate `configure' using a newer version of `autoconf'.
+
+The simplest way to compile this package is:
+
+  1. `cd' to the directory containing the package's source code and type
+     `./configure' to configure the package for your system.  If you're
+     using `csh' on an old version of System V, you might need to type
+     `sh ./configure' instead to prevent `csh' from trying to execute
+     `configure' itself.
+
+     Running `configure' takes awhile.  While running, it prints some
+     messages telling which features it is checking for.
+
+  2. Type `make' to compile the package.
+
+  3. Optionally, type `make check' to run any self-tests that come with
+     the package.
+
+  4. Type `make install' to install the programs and any data files and
+     documentation.
+
+  5. You can remove the program binaries and object files from the
+     source code directory by typing `make clean'.  To also remove the
+     files that `configure' created (so you can compile the package for
+     a different kind of computer), type `make distclean'.  There is
+     also a `make maintainer-clean' target, but that is intended mainly
+     for the package's developers.  If you use it, you may have to get
+     all sorts of other programs in order to regenerate files that came
+     with the distribution.
+
+Compilers and Options
+=====================
+
+   Some systems require unusual options for compilation or linking that
+the `configure' script does not know about.  You can give `configure'
+initial values for variables by setting them in the environment.  Using
+a Bourne-compatible shell, you can do that on the command line like
+this:
+     CC=c89 CFLAGS=-O2 LIBS=-lposix ./configure
+
+Or on systems that have the `env' program, you can do it like this:
+     env CPPFLAGS=-I/usr/local/include LDFLAGS=-s ./configure
+
+Compiling For Multiple Architectures
+====================================
+
+   You can compile the package for more than one kind of computer at the
+same time, by placing the object files for each architecture in their
+own directory.  To do this, you must use a version of `make' that
+supports the `VPATH' variable, such as GNU `make'.  `cd' to the
+directory where you want the object files and executables to go and run
+the `configure' script.  `configure' automatically checks for the
+source code in the directory that `configure' is in and in `..'.
+
+   If you have to use a `make' that does not supports the `VPATH'
+variable, you have to compile the package for one architecture at a time
+in the source code directory.  After you have installed the package for
+one architecture, use `make distclean' before reconfiguring for another
+architecture.
+
+Installation Names
+==================
+
+   By default, `make install' will install the package's files in
+`/usr/local/bin', `/usr/local/man', etc.  You can specify an
+installation prefix other than `/usr/local' by giving `configure' the
+option `--prefix=PATH'.
+
+   You can specify separate installation prefixes for
+architecture-specific files and architecture-independent files.  If you
+give `configure' the option `--exec-prefix=PATH', the package will use
+PATH as the prefix for installing programs and libraries.
+Documentation and other data files will still use the regular prefix.
+
+   If the package supports it, you can cause programs to be installed
+with an extra prefix or suffix on their names by giving `configure' the
+option `--program-prefix=PREFIX' or `--program-suffix=SUFFIX'.
+
+Optional Features
+=================
+
+   Some packages pay attention to `--enable-FEATURE' options to
+`configure', where FEATURE indicates an optional part of the package.
+They may also pay attention to `--with-PACKAGE' options, where PACKAGE
+is something like `gnu-as' or `x' (for the X Window System).  The
+`README' should mention any `--enable-' and `--with-' options that the
+package recognizes.
+
+   For packages that use the X Window System, `configure' can usually
+find the X include and library files automatically, but if it doesn't,
+you can use the `configure' options `--x-includes=DIR' and
+`--x-libraries=DIR' to specify their locations.
+
+Specifying the System Type
+==========================
+
+   There may be some features `configure' can not figure out
+automatically, but needs to determine by the type of host the package
+will run on.  Usually `configure' can figure that out, but if it prints
+a message saying it can not guess the host type, give it the
+`--host=TYPE' option.  TYPE can either be a short name for the system
+type, such as `sun4', or a canonical name with three fields:
+     CPU-COMPANY-SYSTEM
+
+See the file `config.sub' for the possible values of each field.  If
+`config.sub' isn't included in this package, then this package doesn't
+need to know the host type.
+
+   If you are building compiler tools for cross-compiling, you can also
+use the `--target=TYPE' option to select the type of system they will
+produce code for and the `--build=TYPE' option to select the type of
+system on which you are compiling the package.
+
+Sharing Defaults
+================
+
+   If you want to set default values for `configure' scripts to share,
+you can create a site shell script called `config.site' that gives
+default values for variables like `CC', `cache_file', and `prefix'.
+`configure' looks for `PREFIX/share/config.site' if it exists, then
+`PREFIX/etc/config.site' if it exists.  Or, you can set the
+`CONFIG_SITE' environment variable to the location of the site script.
+A warning: not all `configure' scripts look for a site script.
+
+Operation Controls
+==================
+
+   `configure' recognizes the following options to control how it
+operates.
+
+`--cache-file=FILE'
+     Use and save the results of the tests in FILE instead of
+     `./config.cache'.  Set FILE to `/dev/null' to disable caching, for
+     debugging `configure'.
+
+`--help'
+     Print a summary of the options to `configure', and exit.
+
+`--quiet'
+`--silent'
+`-q'
+     Do not print messages saying which checks are being made.
+
+`--srcdir=DIR'
+     Look for the package's source code in directory DIR.  Usually
+     `configure' can determine that directory automatically.
+
+`--version'
+     Print the version of Autoconf used to generate the `configure'
+     script, and exit.
+
+`configure' also accepts some other, not widely useful, options.
+
Index: /tags/gsdl-21/gsdl/src/mgpp/INSTALL.mg
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/INSTALL.mg	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/INSTALL.mg	(revision 699)
@@ -0,0 +1,38 @@
+                     BUILDING AND INSTALLATION OF MG
+                     ===============================
+ 
+This file explains how to build the mg system.
+ 
+(1) Follow the general GNU install instructions in the INSTALL file.
+    NOTE: this procedure has changed from version mg-1.1 which did
+    not use a GNU configure script.
+    e.g.
+      cd ~/mg-1.2
+      ./configure --prefix=`pwd`
+      make
+      make check
+      make install
+    Report portability problems to tes@kbs.citri.edu.au .
+ 
+(2) You will need to set the environment variable MGDATA which 
+    indicates where the mg data files will be created. You will also
+    need to create the directory you specify.
+ 
+    e.g.
+        setenv MGDATA ~/mgdata
+	mkdir ~/mgdata
+
+(4) You may also need to set the environment variable MGSAMPLE which
+    is used by mg_get and indicates where the sample data included
+    with this package is located.
+    If this is not specified it defaults to ``./SampleData''
+
+(4) Included with mg is an X11 interface to mgquery called xmg. 
+    Xmg is a ``wish'' script which uses the Tcl/Tk packages available
+    by anonymous ftp from allspice.berkeley.edu [128.32.150.27].
+ 
+(5) Another environment variable that you may wish to set is
+    MGIMAGEVIEWER, this variable sets the image viewer to be used
+    to display images. The image viewer must take the image from stdin.
+    If this is not specified it defaults to ``xv -''.
+ 
Index: /tags/gsdl-21/gsdl/src/mgpp/MODIFICATIONS
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/MODIFICATIONS	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/MODIFICATIONS	(revision 699)
@@ -0,0 +1,1023 @@
+TITLE
+  Parsing of Long Words
+APPLICATION
+ mg-1, mg-2
+TYPE
+  bug
+REPORT
+  tim@cosc.canterbury.ac.nz - May 11th 1994
+FIX
+  Tim Shimmin (no longer emailable) - August 9th 1994
+CLAIM
+  Mg didn't handle long words properly; it crashed.
+PROBLEM
+  Invf passes calls PARSE_LONG_WORD [words.h] which uses a limit of
+  MAXLONGWORD on iterating thru the string and storing into 
+  a word. MAXLONGWORD = 8192.
+  However, mg strings generally store the length in the first
+  byte limiting them to 255 characters. The word which was passed
+  to PARSE_LONG_WORD was an allocated string of MAXSTEMLEN = 255,
+  which is as large as we should get anyway. Thus when accessing
+  a larger word than 255 chars, PARSE_LONG_WORD would allow it
+  (less than 8192) and would try storing beyond the array limit.
+SOLUTION
+  The author can't remember why PARSE_LONG_WORD was used and what
+  the significance of MAXLONGWORD = 8192 is.
+  So PARSE_LONG_WORD has been changed to PARSE_STEM_WORD which
+  uses MAXSTEMLEN as its limit.
+FILES
+ *   words.h
+ *   invf.pass1.c
+ *   invf.pass2.c
+ *   ivf.pass1.c
+ *   ivf.pass2.c
+ *   query.ranked.c
+*************************************************************
+TITLE
+  Use of Lovins stemmer
+APPLICATION
+  mg-1
+TYPE
+  improve
+REPORT
+  local - 1994
+FIX
+  Linh Huynh (no longer emailable) - 1994
+CLAIM
+  Stemming was done naively.
+PROBLEM
+  Only a few types of words and their endings
+  were considered.
+SOLUTION
+  Replacement with a more elaborate "known" stemmer by Lovins.
+  The algorithm is described in:
+    J.B. Lovins, "Development of a Stemming Algorithm",
+    Mechanical Translation and Computational Linguistics, Vol 11,1968.
+FILES
+ *   stem.c
+ *   stem.h
+*************************************************************
+TITLE
+  Different term parsing
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  Tim Shimmin (no longer emailable) - 23 Aug 1994
+FIX
+  Tim Shimmin (no longer emailable) - 23 Aug 1994
+CLAIM
+  Boolean queries did not extract words/terms using the
+  same method as is done at inverted-file creation and
+  as is used for rank query parsing.
+PROBLEM
+  The hand-written lex. analyser, query_lex, which is called by
+  the boolean query parser was not calling a common
+  word-extraction routine as used by the rest of mg. 
+  This would be ok if the code did the same things - but they didn't.
+  Query_lex, for instance, did NOT place any limit on the
+  number of digits in a term.
+  Of even more concern, it would allow arbitrary sized words
+  although it used Pascal style strings which store the length
+  in the first byte and can therefore only be 255 characters in length.
+SOLUTION
+  Query_lex in "query.bool.y", was modified to call the routine
+  PARSE_STEM_WORD which is also used by text-inversion routines and
+  ranking query routines.
+  Now all terms are extracted by the same routine.
+  To do this, the end of the line buffer had to be noted as
+  PARSE_STEM_WORD requires a pointer to the end - which is the
+  safe thing to do (don't want to run over the end).
+  This meant I had to find the length of the query line buffer.
+  This was allocated in the file "read_line.c" by the routine,
+  "readline". Its size was the literal number 1024. 
+  This was changed to a constant and placed in "read_line.h".
+  The definition for PARSE_STEM_WORD can be found in "words.h".
+FILES
+ *   query.bool.y
+ *   query.bool.c (by bison)
+ *   read_line.c
+ *   read_line.h
+*************************************************************
+TITLE
+  Highlighting of query terms
+APPLICATION
+  mg-1
+TYPE
+  extend
+REPORT
+  Tim Shimmin (no longer emailable) - Aug 94
+FIX
+  Tim Shimmin (no longer emailable) - Sep 94
+CLAIM
+  Difficult to feel happy that the query-result returned is
+  satisfying the query - need to look hard to find the queried words.
+  Need to show words in results using some highlighting method.
+PROBLEM
+  No highlighting of query terms in results.
+SOLUTION
+  Mgquery was previously outputting the decompressed text to a pager
+  such as "less(1)" or "more(1)".
+  (Except when redirected or piped elsewhere :)
+  So what was needed was some sort of highlight pager that instead of
+  displaying the text would also use some means for highlighting the
+  stemmed query words. 
+  Two common forms of highlighting were chosen: underline and bolding.
+  These are supported by "less(1)" and possibly by "more(1)" by
+  using the backspace character.
+  A highlight pager will also need to know which words need to be
+  highlighted. Therefore, the code was modified to build up a
+  string of the stemmed query words for passing to the highlight pager.
+  Design Options:
+  ---------------
+    * Could do text filtering in mgquery before passing out to pager.
+      Instead I pipe to a separate process, the "hilite_words" pager,
+      which filters and pipes into less/more.
+    * Could do different highlighting or a combination.
+    * Could use a different structure for storing the query words other
+      than the hash-table I used.
+FILES
+ *   Makefile - to include hilite_words target
+ *   mg_hilite_words.c
+ *   mgquery.c
+ *   mgquery.1
+ *   query.bool.y
+ *   query.ranked.c
+ *   environment.c
+ *   environment.h
+ *   backend.h
+*************************************************************
+TITLE
+  Mg_compression_dict did premature free
+APPLICATION
+  mg-1
+TYPE
+  bug 
+REPORT
+  R.Sosic@cit.gu.edu.au - 23 Sep 94
+FIX
+  R.Sosic@cit.gu.edu.au - 23 Sep 94
+CLAIM
+  mg_compression_dict dumped core in 
+	file: mg_compression_dict.c
+	function: Write_data
+	line: int codelen = hd->clens[i];
+PROBLEM
+  Huffman data, hd, was freed *before* it was accessed again. 
+SOLUTION
+  The freeing of hd has been moved to after all accesses
+  (just before returning).
+FILES
+ *   mg_compression_dict.c
+*************************************************************
+TITLE
+  Boolean tree optimising rewrite
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  sosic@kurango.cit.gu.edu.au - 23 Sep 94
+FIX
+  Tim Shimmin (no longer emailable) - Oct 94
+CLAIM
+  "I am still getting core dump in "and" queries in mgquery, 
+   where the first word does not exist, but the second one does."
+PROBLEM
+  Having freed a particular node, it tried to refree it and 
+  access one of its fields.
+
+  I.e. code-fragment...
+
+       FreeNode(curr); /* where curr = CHILD(base) for 1st term in list */
+       FreeNodes(next);
+       FreeNodes(CHILD(base)); 
+              /* but CHILD(base) has already been freed above */
+              /* if the node was the first one in the list */
+
+SOLUTION
+  A number of things in the code seemed a bit dubious to me.
+  So I have rewritten the boolean optimising stage and abstracted out
+  the various stages - each file starts with "bool".
+  Boolean query optimising seems to be a tricky problem. 
+  It is not clear that putting an expression into a certain form will
+  actually simplify it and whether simplification means faster querying.
+  I have converted a given boolean expression into DNF 
+  (Disjunctive Normal Form).  "And not" nodes, which are readily apparent 
+  in DNF, are converted to "diff" nodes.  I have only applied the idempotency 
+  laws involving TRUE and FALSE, and not the ones requiring matching of 
+  expressions - it is a potentially more complicated problem.
+  The optimiser has been tested by playing with "bool_tester", and if you are
+  having a crash or problem in a boolean query it would be worth testing the
+  query on the "bool_tester." The token "*" stands for TRUE (or all documents) 
+  and the token "_" stands for FALSE (or no documents). This should show the 
+  expression before and after optimisation in an ascii tree bracketting format.
+FILES
+ *   bool_tree.c
+ *   bool_parser.y
+ *   bool_optimiser.c
+ *   bool_query.c 
+ *   bool_tester.c
+ *   term_lists.c
+*************************************************************
+TITLE
+  Mgtic pixel placement
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  Bruce McKenzie - bruce@cosc.canterbury.ac.nz (21st Oct 1994)
+FIX
+  singlis@cs.waikato.ac.nz
+CLAIM
+  mgtic crashed on certain files.
+PROBLEM
+  Placing pixels outside of bitmap.
+SOLUTION
+  Changed the putpixel routine to truncate at borders of the image.
+FILES
+ *   mgtic.c
+*************************************************************
+TITLE
+  Improved boolean tree optimising
+APPLICATION
+  mg-1
+TYPE
+  improve
+REPORT
+  Tim Shimmin (no longer emailable) - 12/Dec/94
+FIX
+  Tim Shimmin (no longer emailable) - 21/Dec/94, 14/Mar/95
+CLAIM
+  Optimising by conversion to DNF is not necessarily such
+  a good idea - can actually slow things down.
+PROBLEM
+  The distributive law used in converting to DNF
+  duplicates expressions.
+SOLUTION
+  Introduce a query environment variable, optimise_type = 0 | 1 | 2.
+  Type 0 does nothing to the parse tree. 
+  Type 2 does the DNF conversion.
+  Type 1 is the new default and does the following...
+    Do simple tree rearrangement like flattening. 
+    Optimise for CNF queries.
+FILES
+ *   bool_query.c, .h
+ *   bool_optimiser.c
+ *   environment.c
+ *   invf_get.c
+ *   bool_tree.c, .h
+ *   bool_tester.c
+ *   lists.h
+*************************************************************
+TITLE
+  Similarity variants
+APPLICATION
+  mg-2
+TYPE
+  extend
+REPORT
+  jz@cs.rmit.edu.edu.au/alistair@cs.mu.oz.au - June 1994
+FIX
+  Tim Shimmin (no longer emailable) - July 1994 .. Feb 1995
+CLAIM
+  Can only use one type of similarity measure - the
+  standard cosine measure.
+PROBLEM
+  See CITRI/TR-95-3 for more details.
+  The standard measure can be broken up into 7 components.
+  The 7 components are  
+  Each one of these components has a number of alternatives.
+  The overall measure, S_qd, can also be altered.
+  Thus the particular similarity measure used can be specified
+  by an 8 dimensional vector.
+  What is desired is to be able to specify to mgquery an option
+  and a 8-digit string representing this vector (assuming that
+  any one alternative can have at most 9 (not using zero) variants).
+SOLUTION
+  The programs which had to be modified were:
+  (i) mgquery,
+  (ii) mg_weights_build.
+  The other mg programs in existence store the text, indexing info,
+  and the basic statistics such as N, n, ft, fdt.  
+  Other programs which had to be created were:
+  (i) mg_fmd_build,
+  (ii) mg_wt_build.
+  Mg_fmd_build will create the file to store the f_md statistic, 
+  where f_md is the largest (maximum) f_dt of any term in document, d.
+  Mg_wt_build will create the file to store the w_t primitive.
+  It only creates this for the w_t variants 6-9 which would require
+  extra passes of invf at query time if they were not stored here.
+  For details on similarity changes for mgquery and mg_weights_build,
+  please see the other modification entries.  
+FILES
+ *   mg_fmd_build.c
+ *   mg_wt_build.c
+ *   build_lib.c, build_lib.h
+*************************************************************
+TITLE
+  Similarity variants for mgquery
+APPLICATION
+  mg-2
+TYPE
+  extend
+REPORT
+  jz@cs.rmit.edu.edu.au/alistair@cs.mu.oz.au - June 1994
+FIX
+  Tim Shimmin (no longer emailable) - July 1994 .. Feb 1995
+CLAIM
+  "mgquery" needs to be altered to allow modification of
+  the similarity measure. 
+PROBLEM
+  See CITRI/TR-95-3 for more details.
+SOLUTION
+
+  Most of the similarity measures, Sqd, are of the 
+  form:   Aqd
+         -----
+          Bqd
+  where Bqd is an expression involving Wd and possibly Wq,
+  where Aqd is a sum over the common document/query terms
+  of w_qt and w_dt.
+  Building of Aqd
+  ===============
+  The calculation of Aqd is done in the file build_Aqd.c .
+  The functions for doing this used to be in the file invf_get.c .
+  Build_Aqd.c contains 4 different functions for building Aqd, each
+  of them building a different data structure:
+    (i) Array, (ii) Splay Tree, (iii) Hash Table, (iv) List Table.
+  Each of these routines seems to have been construction by modifications
+  to duplicated code. This is often the easiest way to construct variants
+  but is quite difficult to maintain consistency. 
+  As the aim of the exercise was to try out different sim. measures for
+  retrieval effectiveness, I only modified the code that constructed
+  an array. This routine was called "CosineDecode"; I changed it to
+  "build_Aqd_Array." This change reflects the fact that we are only 
+  calculating Aqd and this need not be used for the Cosine measure.
+  The other routines: "CosineDecodeSplay," "CosineDecodeHash," and
+  "CosineDecodeList" have been left unaltered - they need to be updated
+  in the future which would be best be done by abstracting out common code.
+  By the stage of building Aqd, the query terms have been looked up in
+  the inverted file dictionary and put into a list.
+  This list of common terms is traversed to lookup the corresponding
+  invf entries. Before the invf entry is processed, all query and term
+  relevant statistics and primitive quantities are calculated.
+  For example, fqt, ft, wt, rqt, wqt, Wq-partial-sum.
+  To save unnecessary calculations, there is a test for each value
+  to see whether it is needed e.g. "if (sim->variant_needs & NEEDS_wt) ... ".  
+  Aside: Variant Needs
+  --------------------
+  The idea behind the "variant_needs" field is to be able to have all
+  the code in the one place for each possible variant and this code would
+  get the information at the correct time/place only when it is needed.
+  The overhead is a "bit-and" and "test" for each component.
+  The important concern though, is that the "variant_needs" must be
+  accurate i.e. it should be carefully maintained.
+  Each possible need is stored as a bit position in a constant/macro
+  of the form "NEEDS_component" e.g. NEEDS_wt, NEEDS_nt 
+  More recently (Jan/95), I have found that the _need_ing of a component
+  may be relevant to a particular purpose, that is, it may be needed in
+  one situation and not another. This was the case for Wd. 
+  For rdt#6, Wd was needed, however, it might not be needed for Bqd, the
+  denominator of Sqd. So I changed from NEEDS_Wd to NEEDS_Wd_for_Sqd and
+  NEEDS_Wd_for_rdt. 
+  --------------------
+  To handle the different calculations of the variants, I wrote macros
+  based around a switch statement. All these macros are stored in the file
+  "similarity.h" . The point of doing this is to centralise most of the changes
+  and cut down on the number of files which have to be altered if a new
+  way of calculating a primitive is to be added.  
+  This is achieved by having a data record called, "Similarity_variant" whose
+  fields includes all the statistics and the similarity primitives.
+  So the standard procedure is to see if something is needed and if so, then
+  extract it from an mg file or calculate it using a macro - most of the
+  input and output to the macro is done via the "Similarity_variant" structure.
+  As well as calculating Aqd, it is also a convenient place to calculate Wq.
+  Previously, for Cosine measure, Wq was not needed because each Aqd was directly
+  divided by Wq and thus would not change the ordering. However, there are
+  some Sqd's which divide by a sum involving Wq and thus need Wq. 
+  Calculating Sqd and Ordering Documents
+  ====================================== 
+  The file query.ranked.c contains the code for calculating Sqd using the
+  approximate Wd in order to do the ranking of the documents.
+  The mg-1 file was cleaned up and modified slightly.
+  All the heap data structures and routines were taken out and placed in 
+  their own file, heap_weights.c/.h .
+  The major components/steps in the ranking process were abstracted into
+  macros and functions:
+    calc_MaxParas
+    insert_heap_entry
+    insert_greater_heap_entry
+    approx_guided_insert
+    fill_initial_heap
+    add_heap_remainders
+    change_heap_to_exact
+    add_remainder_exact_weights
+    Make_Exact_Root
+    build_doc_list
+  Aside: Zeroing of an Aqd Element
+  -------------------------
+  One interesting change concerned the use of zeroing out an Aqd element
+  so as to mark it as being used in the heap.
+  In the heap, the approx. Sqd is stored based on Aqd/Wd-approx.
+  Later when Aqd is required, it can be extracted from the approx. Sqd
+  by muliplication of Wd-approx i.e. Aqd = Sqd-approx * Wd-approx.
+  This, however, is not always possible for the various Sqd variants.
+  So, instead of zeroing out Aqd, I decided just to make it negative.
+  -------------------------
+  In some Sqd cases, Wd and Wd-approx. are not required. In which case,
+  step 3 which calls on "change_heap_to_exact", "Heap_Build" and
+  "add_remainder_exact_weights" is not required.
+ 
+FILES
+ *    build_Aqd.c
+ *    query.ranked.c
+ *    similarity.c/.h (in libmg)
+ *    heap_weights.c/.h
+ *    backend.c/.h
+*************************************************************
+TITLE
+  Similarity variants for mg_weights_build
+APPLICATION
+  mg-2
+TYPE
+  extend
+REPORT
+  jz@cs.rmit.edu.edu.au/alistair@cs.mu.oz.au - June 1994
+FIX
+  Tim Shimmin (no longer emailable) - July 1994 .. Feb 1995
+CLAIM
+  "mg_weights_build" needs to be altered to allow modification of
+  the similarity measure.
+PROBLEM
+  See CITRI/TR-95-3 for more details.
+SOLUTION
+  The weight files which are generated for a particular similarity
+  measure have their names extended by a suffix.
+  In the case of Wd#1 no weights are generated.
+  In the case of the standard cosine weights, Wd#2, a 3 letter suffix
+  is used to represent .
+  In the case of the other Wd variants, a one letter suffix is used
+  to represent which Wd variant it is.
+  In each case, the variant input (e.g. -q 22222222) should be the whole
+  similarity variant string and the relevant fields will be extracted out.
+  This is done for consistency in code and interface. 
+  The code is fairly similar to the original.
+  A dependency check has been added so that the dates of files and the
+  type of needed files is verified before building.
+  The dependencies include, invf dictionary, invf index, invf, fmd and wt files.
+  The major change here, is the possible use of fmd and/or wt files.
+  Later when I was having to write mg_fmd_build and mg_wt_build,
+  I decided to abstract out some macros, namely:
+    Get_ft, Get_ft_Ft, loop_invf_entry,
+  which were put into "build_lib.c" .
+  "loop_invf_entry" takes a function/macro name as a parameter and applies
+  it to the sim record (with field fdt set), current doc number and modifies
+  the return value. 
+FILES
+ *   mg_weights_build.c
+ *   build_lib.c
+*************************************************************
+TITLE
+  Mgstat with non-existent files
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  beebe@math.utah.edu - 16 May 1994
+FIX
+  Tim Shimmin (no longer emailable) - 10 Aug 1994
+CLAIM
+  NaNs and Infinites would be printed out by mgstat
+  if unable to open .text or .text.dict file.
+PROBLEM
+  The NaNs etc. were output in the column stating
+  the percentage size of the file compared with the
+  number of input bytes of the source text data.
+  If it couldn't read the .text file with its
+  header describing the number of source text bytes, then
+  in working out the percentage it would divide by zero.
+  Also due to some bad control flow, it wouldn't attempt to
+  open the .text file if it failed when opening 
+  the .text.dict file.
+SOLUTION
+  Only printout the percentage if we can read the header
+  from the .text file.
+  Read in text header irrespective of text dictionary file.
+FILES
+ *   mgstat.c
+*************************************************************
+TITLE
+  Boolean tree optimisations
+APPLICATION
+  mg-2
+TYPE
+  extend
+REPORT
+  (i) Tim Shimmin (no longer emailable) - 28/Sep/94
+  (ii) Tim Shimmin (no longer emailable) - 12/Dec/94
+FIX
+  (i) Tim Shimmin (no longer emailable) - 18/Oct/94
+  (ii) Tim Shimmin (no longer emailable) - 21/Dec/94
+CLAIM
+  The initial prompt for investigating the optimisation of
+  boolean queries is noted in the mg-1 mod14.txt. 
+  The code for optimising seemed to have a number of faults.
+PROBLEM
+  Boolean optimisation was unreliable.
+SOLUTION
+  Initially (in case (i) above, see mg-1/mod14.txt), I rewrote 
+  all the boolean tree and optimising code. I converted the boolean
+  expression into DNF. I did this after reading some notes about 
+  the steps involved in optimisation and they suggested standardising
+  in some normal form. I thought that DNF would be appropriate so that
+  all the terms are converted to be part of "and" expressions and be
+  evaluated quickly using skipping.
+  This, however, can suffer quite badly if the distributive law is
+  applied to often and the query expands in size. If there was
+  some sort of cacheing of invf entries, then it might not be so
+  bad otherwise there is quite an overhead on reading the same
+  invf entry more than once.
+
+  As it happens, CNF queries are reasonably common, where the user
+  queries with a conjunction of disjunctions of similes:
+    e.g. (car | automobile | vehicle) & (fast | quick | speedy)
+  This sort of CNF query expands a hell of a lot !
+  So after speaking with Justin who wanted to benchmark Atlas with Mg on
+  these sort of queries, I looked up the MG book for other ideas.
+
+  The method that I implemented was the following:
+
+  -----------------------------------------------------------------
+  Steps of tree modifications:
+    Gets literals by pushing the nots down, detecting T/F at leaves
+    and collapsing the tree by detecting  'and' of 'and's and 'or' of
+    'or's.
+    Next it looks at the or nodes and if all the children are terms
+    then it marks the or-node as such.
+    Finally, the or-term-nodes are sorted by using the sum of their
+    ft's for comparison.
+  Steps at query evaluation:
+    If it comes across an 'and' of 'or-terms' then the evaluation is
+    done noting the distributive law.
+    I.e. a & (b | c | d) = (a & b) | (a & c) | (a & d)
+    Assuming 'a' is the c-set of documents.
+    All of 'a' is tested against 'b' and matching ones are marked.
+    Next, all the unmarked members of 'a' are tested against 'c'.
+    Likewise for 'd'.
+    Now all the marked members of our c-set are kept.
+    When we do the testing, we can use the skipping in the invf entries.
+  -----------------------------------------------------------------
+
+
+  After doing this, I added the choice of which type of optimisation
+  the user wanted by adding query-environment-variable, "optimise_type".
+  Type 0 = no parse tree modification. 
+  Type 1 = Or-term recognition and CNF query evaluation optimisation. 
+  Type 2 = Put into DNF form. [generally not recommended]
+FILES
+ *   bool_tree.c
+ *   bool_parser.y
+ *   bool_optimiser.c
+ *   bool_query.c 
+ *   bool_tester.c
+ *   term_lists.c
+ *   query_env.c
+ *   invf_get.c [GetDocsOp]
+*************************************************************
+TITLE
+  nonexistent HOME bug  
+APPLICATION
+  mg-1, mg-2
+TYPE
+  bug 
+REPORT
+  cgn@totara.cs.waikato.ac.nz - 2/May/95
+FIX
+  Tim Shimmin (no longer emailable) - 2/May/95
+CLAIM
+"The big problem was that mgquery crashes when the HOME environment
+ variable is not set, which is the case when it is run by the www server."
+ [...] "I expect it happens when looking for $HOME/.mgrc."
+PROBLEM
+  The result of getenv("HOME")" was used directly in
+  a sprintf call. If the environment variable HOME
+  was not in existence then null would be used.
+  In some C libraries sprintf will convert the 0
+  string into the string "(null)" on others it will core dump.
+  (For example, Solaris seems to core dump, sunos 4 seems ok).
+SOLUTION
+  The result from getenv("HOME")" is tested before
+  being used.
+FILES
+ *   commands.c
+*************************************************************
+TITLE
+  mgquery collection name preference  
+APPLICATION
+  mg-1, mg-2
+TYPE
+  improve
+REPORT
+  cgn@totara.cs.waikato.ac.nz - 2/May/95
+FIX
+  Tim Shimmin (no longer emailable) - 4/May/95
+CLAIM
+ Surely something must override mquery's preference for ./bib. 
+ If MGDATA is set correctly, I think it should prefer that collection, 
+ and -d should definitely override it. 
+ I could always say -d . if I really wanted ./bib.
+PROBLEM
+Currently the priority is:
+1.  Check if ./name is a directory,
+     If so then use it as the collection directory.
+2.  Check if ./name.text is a file,
+     If so then use ./ as the collection directory.
+3.  Check if mgdir/name is a directory,
+     If so then use mgdir/name as the collection directory.
+4.  Otherwise,
+     Use mgdir/name as the database file prefix.
+     This would be the case if one used "-f alice/alice".
+     However, one would then not specify a final name argument
+     and we'd never get here. Go figure ???
+SOLUTION
+Moved step 3 to the top instead.
+FILES
+ *   mgquery.c [search_for_collection()]
+*************************************************************
+TITLE
+  Printout of query terms
+APPLICATION
+  mg-1, mg-2
+TYPE
+  extend
+REPORT
+  Tim Shimmin (no longer emailable) - April 95
+FIX
+  Tim Shimmin (no longer emailable) - April 95
+CLAIM
+  No easy way to find out the parsed and stemmed words
+  used in the query. Would like to know these words
+  so I can call a separate highlighting program to 
+  highlight these words.
+PROBLEM
+  No facility available.
+SOLUTION
+  A ".queryterms" mgquery command was added which lists
+  out the parsed/stemmed queryterms of the last query.
+FILES
+ *    commands.c (added CmdQueryTerms)
+*************************************************************
+TITLE
+  mg_getrc  
+APPLICATION
+  mg-1, mg-2
+TYPE
+  extend
+REPORT
+  bruce@cosc.canterbury.ac.nz - 2/May/95
+FIX
+   - 
+CLAIM
+  Repeated code had to be written for different named
+  gets but really the same type of parsing required.
+  E.g. one might want to use a standard method for inserting
+  ^Bs between paragraphs for different books. One doesn't
+  want to write duplicate code for each different named book,
+  rather note that each book should be filtered "book" style.
+PROBLEM
+  There was no way of abstracting out types of filters from
+  the name of an instance of a collection.
+SOLUTION
+  Allow information to be given with <name, type, files>.
+  This extra info can be provided in a mg_getrc file.
+  See man page for mg_get for details.
+FILES
+ *    mg_get.sh
+*************************************************************
+TITLE
+  TREC DocNo file   
+APPLICATION
+  mg-2
+TYPE
+  improve
+REPORT
+  Tim Shimmin (no longer emailable) - 26/May/95
+FIX
+  Tim Shimmin (no longer emailable) - 26/May/95
+CLAIM
+  MG has problems dealing with trec docnos for trec disk 3.
+PROBLEM
+  Trec DocNos file didn't have a wide enough field
+  to handle disk 3. 
+SOLUTION
+  Allow different width fields for file.
+  It is still fixed width but a number in the header
+  says how wide the field is.
+FILES
+ *   passes/mg.special.c
+ *   query/mgquery.c
+*************************************************************
+TITLE
+  Boolean optimiser #1 with `!'  
+APPLICATION
+  mg-1, mg-2
+TYPE
+  bug 
+REPORT
+  triffid@iconz.co.nz - 20/7/95
+FIX
+  Tim Shimmin (no longer emailable) - 27/7/95
+CLAIM
+  Complained about not-nodes. 
+  e.g. complained about "croquet & !hedgehog"
+PROBLEM
+  Boolean optimiser type#1 didn't convert
+  "and not"s into diff nodes.
+SOLUTION
+  Added code to convert '&!' to '-'.
+FILES
+ *   mg/bool_optimiser.c [mg-1]
+ *   query/bool_optimiser.c [mg-2]
+*************************************************************
+TITLE
+   Autoconfiguring mg-1  
+APPLICATION
+  mg-1
+TYPE
+  improve
+REPORT
+  many people - 94/95
+FIX
+  Tim Shimmin (no longer emailable) - Aug/95
+CLAIM
+  Portability is limited by setting up c-macros just for particular
+  machines and operating systems.
+  People had to make changes for HP, Next, Linux, Dec Alpha, ...
+PROBLEM
+  Porting was only targetting at the machines that the author had
+  access to.
+SOLUTION
+  Use GNU's autoconfigure program.
+  This allows checking of the systems features/characteristics.
+  It also allows some checking for specific machines/OS - although
+  I have not utilised this option.
+  I used GNU's tar-1.11.8 as an example to base my changes on.
+  I also used autoscan to generate the initial "configure.in".
+  The "Makefile.in"s were done very similarly to GNU tar's.
+  The "config.h" and "sysfuncs.h" files were scrapped and 
+  rewritten. The new "config.h" file is generated by the configure
+  script - it contains all the #define's for the system features.
+  The "sysfuncs.h" file wraps up a number of system headers. 
+  For example, some systems use , while some use ;
+  which one is included is decided in "sysfuncs.h".
+  I have also used GNU tar's use of ansi2knr in its Makefiles.
+  This should hopefully allow the package to work on a system with
+  only a K&R C compiler.
+  However, there are probably problems with what I have done.
+  I am concerned about <stdarg.h> for example.
+  I also noticed that "ansi2knr" require function definitions as
+  the GNU coding style recommends ie. with function name the first
+  string on the line. This prompted me to run all the package's code
+  thru GNU's indent.
+  Setting up the configure changes is difficult. It really seems
+  necessary to try the package out on many target machines so one
+  can know what is necessary.
+  A simple check target for the main Makefile has been written.
+  It is used to see if the installation is working - it does
+  not test much of the functionality of mg.
+  It does cmp's on data files and diff's on query/result files.
+FILES
+  Most of the files in the distribution.
+*************************************************************
+TITLE
+  Consistent use of stderr
+APPLICATION
+  mg-1
+TYPE
+  improve
+REPORT
+  beebe@math.utah.edu - 16 May 1994
+FIX
+  Tim Shimmin (no longer emailable) - 11 August 1994
+CLAIM
+  Inconsistent use of stdout/stderr in usage messages.
+PROBLEM
+  Sometimes used "printf" and sometimes used "fprintf(stderr"
+  in usage messages.
+SOLUTION
+  All should now use "fprintf(stderr" in usage messages.
+FILES
+ *   mg_compression_dict.c
+ *   mg_compression_dict.1
+ *   mg_fast_comp_dict.c
+ *   mg_fast_comp_dict.1
+ *   mg_invf_dict.c
+ *   mg_invf_dict.1
+ *   mg_invf_dump.c
+ *   mg_invf_dump.1
+ *   mg_invf_rebuild.c
+ *   mg_invf_rebuild.1
+ *   mg_perf_hash_build.c
+ *   mg_perf_hash_build.1
+ *   mg_text_estimate.c
+ *   mg_text_estimate.1
+ *   mg_weights_build.c
+ *   mg_weights_build.1
+*************************************************************
+TITLE
+  xmg bug
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  cgn@cpsc.ucalgary.ca - 22 April 1994
+FIX
+  cgn@cpsc.ucalgary.ca - 22 April 1994
+CLAIM
+  "Serious problem in xmg, which I fear occurs whenever a query
+   doesn't return anything."
+PROBLEM
+  ??
+SOLUTION
+  [xmg.sh 201] set rank 0
+FILES
+ *   xmg.sh
+*************************************************************
+TITLE
+  Unnecessary loading of text
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  Tim Shimmin (no longer emailable) - ?? August 1994
+FIX
+  Tim Shimmin (no longer emailable) - 12 August 1994
+CLAIM
+  Mg was loading and uncompressing text when the
+  query did not require the text.
+PROBLEM
+  There was no test for the query mode
+  before loading and uncompressing the text.
+SOLUTION
+  Only load/uncompress text if query mode
+  is for text, headers or silent(for timing).
+FILES
+ *   mgquery.c
+*************************************************************
+TITLE
+  Man page errors
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  beebe@math.utah.edu - 16 May 1994
+FIX
+  beebe@math.utah.edu - 16 May 1994
+CLAIM
+  Man page errors.
+PROBLEM
+  See below.
+SOLUTION
+ "The mg_make_fast_dict.1 file has been renamed mg_fast_comp_dict.1,
+  and all mg_make_fast_dict strings changed to mg_fast_comp_dict in all
+  man pages.
+  A large number of errors of spelling, typography, spacing, fonts,
+  grammar, omitted words, slang, punctuation, missing man page
+  cross-references, and man-page style have been corrected."
+FILES
+ *    mg_compression_dict.1
+ *    mg_fast_comp_dict.1
+ *    mg_get.1
+ *    mg_invf_dict.1
+ *    mg_invf_dump.1
+ *    mg_invf_rebuild.1
+ *    mg_passes.1
+ *    mg_perf_hash_build.1
+ *    mg_text_estimate.1
+ *    mg_weights_build.1
+ *    mgbilevel.1
+ *    mgbuild.1
+ *    mgdictlist.1
+ *    mgfelics.1
+ *    mgquery.1
+ *    mgstat.1
+ *    mgtic.1
+ *    mgticbuild.1
+ *    mgticdump.1
+ *    mgticprune.1
+ *    mgticstat.1
+ *    xmg.1
+*************************************************************
+TITLE
+  Man page overview
+APPLICATION
+  mg-1
+TYPE
+  extend
+REPORT
+  beebe@math.utah.edu - 
+FIX
+  Tim Shimmin (no longer emailable) - 17 August 1994
+CLAIM
+  "Write new mg.1 file to give a brief overview of mg, with samples
+   of how to use it.  Otherwise, users are likely to be completely
+   overwhelmed by the number of programs (about 20) which might need to
+   be used, when in reality, only 2 or 3 are likely to be run by end
+   users."
+SOLUTION
+  It was thought that mg.1, written by Nelson Beebe, was very useful
+  but a bit too comprehensive for an introduction.
+  Therefore, two man files, mgintro.1 and mgintro++.1 were written
+  with the basic stuff in mgintro.1 and slightly more advanced stuff
+  in mgintro++.1 . 
+FILES
+ *   mg.1
+ *   mgintro.1
+ *   mgintro++.1 
+*************************************************************
+TITLE
+  Parse errors not bus errors
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  tim@cosc.canterbury.ac.nz - 2 Jun 94
+FIX
+  Tim Shimmin (no longer emailable) - 19 Aug 94
+CLAIM
+  "These two queries 
+   (which I typed in before I knew what I was doing!!)
+      > The Queen of Hearts, she made some tarts
+      > "The Queen of Hearts" and "she made some tarts"
+      produced the following result:
+      mgquery : parse error
+      Bus error
+  "
+PROBLEM
+  What is expected to happen under boolean querying:
+    Query1:
+    > The Queen of Hearts, she made some tarts
+    will produce a parse error due to the comma which
+    is not a valid TERM.
+    Query2:
+    > "The Queen of Hearts" and "she made some tarts"
+    will store a post-processing string
+    of ''The Queen of Hearts" and "she made some tarts'' and
+    will have a main boolean query of the empty string.
+    This is because the postprocessing string takes in
+    everything between the first quote and the last one.
+    An empty string is illegal for the boolean grammar and
+    hence a parse error.
+  The problem stems from the fact that the processing of
+  the parse tree is carried out, even though we have a 
+  parse error. In the case of using an empty string to build
+  a parse tree, it is likely to leave the parse tree undefined.
+SOLUTION
+  As soon as we find out that there is a parse-error,
+  we abandon any processing of the parse tree.
+FILES
+ *   query.bool.y
+ *   query.bool.c (generated from query.bool.y)
+*************************************************************
+TITLE
+  Perfect hashing on small vocab
+APPLICATION
+  mg-1
+TYPE
+  bug
+REPORT
+  Tim Shimmin (no longer emailable) - July 1994
+FIX
+  alistair@cs.mu.oz.au - July 1994
+CLAIM
+  Mg could not handle small collections in the case
+  where there was only a small number of unique words.
+  The perfect hash function would report an error.
+PROBLEM
+  Rounding of the arithmetic during the calculation of the
+  parameters of the perfect hash function was resulting in a
+  combination of values such that the probability of a hash
+  function being found was very small. This led to the limit
+  on the generation loop being exceeded, and eventual
+  failure.
+SOLUTION
+  By using ceiling rather than floor when converting from a
+  floating point value to an integer parameter, the arithmetic
+  is now correct for all lexicon sizes, and the probability of
+  each iteration successfully generating a hash function is
+  sufficiently great that with _very_ high probability the
+  execution loop counter will not be exceeded unless there
+  genuinely is no hash function (for example, if the lexicon
+  contains two words the same there cannot be  a hash
+  function).
+FILES
+ *   perf_hash.c
+*************************************************************
Index: /tags/gsdl-21/gsdl/src/mgpp/README
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/README	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/README	(revision 699)
@@ -0,0 +1,178 @@
+                 MG INFORMATION RETRIEVAL SYSTEM
+                 ===============================
+
+The MG system is a suite of programs for compressing and indexing text
+and images. Most of the functionality implemented in the suite is as
+described in the book ``Managing Gigabytes: Compressing and Indexing
+Documents and Images'', second edition, I.H. Witten, A. Moffat, and
+T.C. Bell, Morgan Kaufmann, San Francisco, 1999, ISBN 1-55860-570-3;
+US $54.95.  See also the web page <http://www.cs.mu.oz.au/mg/>.
+
+These features include:
+
+-- text compression using a Huffman-coded semi-static word-based scheme
+-- two-level context-based compression of bi-level images
+-- FELICS lossless compression of gray-scale images
+-- combined lossy/lossless compression for textual images
+-- indexing algorithms for large volumes of text in limited main memory
+-- index compression
+-- a retrieval system that processes Boolean and ranked queries
+-- an X windows interface to the retrieval system
+
+As one example, a collection of 2 Gb of text (1,700,000 documents) can
+be indexed (on a 266 MHz Pentium II) in about one hour and compressed
+in a further one hour to make a database that in total occupies less
+than 800 Mb, or 40% of the original size.  This includes a full index
+to every word and number in the original text.  Boolean queries such as
+``managing AND gigabytes'' run in a few tenths of a second on the same
+hardware, and ranked queries of 30--50 terms are evaluated in 1--3
+seconds.
+
+Details of these methods and further performance results appear in the
+MG book.
+
+The MG system comes with ABSOLUTELY NO WARRANTY; for details see the
+file COPYING.
+
+Instructions on how to build and install mg are in the files INSTALL.mg
+and INSTALL.
+
+
+** CHANGES FROM BOOK
+
+For copyright reasons the stemmer used in this distribution of MG is not
+the same as the one illustrated in Figure 3.9 on page 146 of the MG
+book.  (However, the numbers generated by the command ``mgstat alice''
+will match those numbers in Figure A.1 on page 454, unlike in the first
+edition of the MG book.)  Another stemmer was initially written as a
+simple stopgap for version 1.0.  That stemmer has now been replaced by a
+stemmer based on the Lovin's stemming algorithm.
+
+The output format of ``mgstat'' has changed slightly since Figure A.1
+(page 454) was prepared. The same information is displayed but formatted
+differently.
+
+Some of the on-disk data structures have changed slightly, to accomodate
+larger databases.  This is reflected in some increased file sizes -- in
+most cases, the increase is just 8 bytes.  This also means that databases
+built with older versions of mg are not compatible with this version.
+
+
+** MG VERSIONS
+
+The current version is mg-1.2.1, August 1999. The changes from earlier 
+versions are listed in the file MODIFICATIONS. This can be accessed
+with mg by building a database using ``mgbuild mods'' and can also be
+accessed from the mg web page (see below). 
+
+The mg-1.2.1 extensions include:
+
+-- Fixes to compile under Linux (various distributions).
+-- Fixes to avoid some 32-bit integer overflows when building large
+   (> 2Gb) databases.
+
+The mg-1.2 extensions include:
+
+-- Source modifications for use of GNU's autoconf.
+
+The mg-1.1 extensions include:
+
+-- A new highlighting mode.
+   The output mode ``hilite'' will highlight the query terms in the
+   retrieved text documents. The variable ``hilite_style'' can be set
+   to ``bold'' or ``underline''. It works best with the pager
+   ``less''.  A .mgrc to use would include:
+	.set pager less
+	.set mode hilite
+	.set hilite_style bold
+
+-- A web site containing manual pages and documentation is at
+   <http://www.mds.rmit.edu.au/mg/>
+   One of these pages ``about_mg.html'' is included in this
+   distribution.
+
+-- A revised mg_get script which uses a .mg_getrc file to map
+   specific collection names to filter types.  (Modifications by Bruce
+   McKenzie). See mg_get.1 for more details.
+
+-- Code to perform merging of existing databases. This code
+   was created by Shane Hudson and is documented in the mgmerge.README
+   file found in the docs subdirectory.  This code was written by
+   Shane Hudson (Canterbury).
+
+-- Revised man pages, including some new entries (thanks to Nelson
+   Beebe). See mg.1, mgintro.1, mgintro++.1.
+
+-- A real (rather than toy) stemmer.
+
+** PORTABILITY
+
+Please refer to "README.port".
+
+** CREDITS
+ 
+The MG development is largely the result of research collaboration
+between:
+
+        Tim C. Bell            <tim@cosc.canterbury.ac.nz>
+        Ian Witten             <ihw@cs.waikato.ac.nz>
+        Alistair Moffat        <alistair@cs.mu.oz.au>
+        Justin Zobel           <jz@cs.rmit.oz.au>
+
+The bulk of the programming work has been carried out by:
+
+        Stuart Inglis	       (Waikato)
+        Craig Nevill-Manning   (Waikato)
+        Neil Sharman           (Melbourne and RMIT)
+        Tim Shimmin            (RMIT)
+
+In addition to these, the following people have contributed to the
+development of the MG software:
+
+        Lachlan Andrew         (RMIT)
+        Tim A.H. Bell          (Melbourne)
+	Owen de Kretser        (Melbourne)
+        Gary Eddy              (Melbourne)
+        Hugh Emberson          (Canterbury)
+        Kerry Guise            (Waikato)
+	Shane Hudson           (Canterbury)
+        Linh Huynh             (Melbourne and RMIT)
+        Bohdan S. Majewski     (Queensland)
+        Bruce McKenzie         (Canterbury)
+	William Weber	       (RMIT)
+
+In addition to these, the following people have submitted bug reports
+and suggestions/fixes:
+
+        Rex Barzee
+        Nelson Beebe
+        Tim A.H. Bell
+        Tim C. Bell
+	Rodney Brown
+        Rok Sosic
+        Carl Staelin
+
+Development of the MG system was supported by the Australian Research
+Council; the Universities of Melbourne, Waikato, Canterbury, and
+Calgary; RMIT; and the Collaborative Information Technology Research
+Institute (Melbourne).
+
+** BUG REPORTS
+
+Send bug reports to <alistair@cs.mu.oz.au>.  But do please be aware
+that there is little likelihood of any immediate response apart from a
+"thank you for letting me know", as we have no funded support for MG,
+and any software development is voluntary on the part of my students.
+What I do guarantee is that your mail will be retained against the
+eventuality that one day someone does give us $50,000 for further
+software development. And if you have $50,000, and thought MG was
+wonderful, well, think of us...
+
+
+** FURTHER READING
+
+The bibliography of the MG book lists a wide range of relevant papers.
+Other recent work relevant to the project is listed at
+<http://www.cs.mu.oz.au/~alistair/abstracts/> and at
+<http://www.cs.rmit.edu.au/~jz/Papers.html>.  The NZDL project home
+page is at <http://www.nzdl.org>.
Index: /tags/gsdl-21/gsdl/src/mgpp/README.port
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/README.port	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/README.port	(revision 699)
@@ -0,0 +1,125 @@
+P O R T A B I L I T Y
+=====================
+
+The main development of MG is done under Solaris on a Sun sparc.
+However, we would also like MG to be reasonably portable.
+Therefore, MG-1.2 now uses a configure stript generated by GNU's autoconf.
+Below, is a list of the machines that the s/ware has been ported to. 
+Most only need the configure script and some require manual modifications 
+mentioned in the "Problems" field.
+
+------------------------------------------------------------------------- 
+** Systems which have successfully been built.
+
+Tester:		tes@kbs.citri.edu.au
+Date:
+Machine:	Sun
+System:		SunOS 4.1.2 8 sun4c
+Compiler:	gcc version 2.3.3
+Problems:	none
+
+Tester:		tes@kbs.citri.edu.au
+Date:
+Machine:	Sun
+System:		SunOS 4.1.2_DB 14 sun4c
+Compiler:	gcc version 2.3.3
+Problems:	none
+
+Tester:		tes@kbs.citri.edu.au
+Date:
+Machine:	Sun
+System:		SunOS 4.1.3_U1 1 sun4c
+Compiler:	gcc version 2.5.7
+Problems:	none
+
+Tester:		tes@kbs.citri.edu.au
+Date:
+Machine:	Sun
+System:		SunOS 5.3 Generic_Patch sun4m sparc
+Compiler:	gcc version 2.4.3
+Problems:	none
+
+Tester:		tes@kbs.citri.edu.au
+Date:
+Machine:	Silicon Graphics
+System:		IRIX 5.2 02282014 IP7
+Compiler:	gcc version 2.6.3
+Problems:	none
+
+Tester:		tes@kbs.citri.edu.au
+Date:
+Machine:	Sun
+System:		SunOS 5.4 Generic_101945-27 sun4c sparc
+Compiler:	cc: SC3.0.1 13 Jul 1994
+Problems:	none
+
+Tester:		RBrown@cmutual.com.au
+Date:
+Machine:
+System:		SCO 3.2v4.1
+Compiler:	gcc 2.6.1
+Problems:	requires RANLIB=true, use of tcsh
+
+Tester:		RBrown@cmutual.com.au
+Date:
+Machine:	HP
+System:		HP-UX A.09.04 D 9000/837 415303291 64-user license
+Compiler:	HP92453-01 A.09.73 HP C Compiler
+Problems:
+
+Tester:		RBrown@cmutual.com.au
+Date:
+Machine:	Data General
+System:		dgux 5.4R3.00 generic AViiON mc88110 
+Compiler:	gcc 2.4.5 (dg supplied)
+Problems:	none
+
+Tester:		RBrown@cmutual.com.au
+Date:
+Machine:	Pyramid MIPS R3000 box
+System:		
+Compiler:	cc (vendor supplied)
+Problems:	none
+
+Tester:		RBrown@cmutual.com.au
+Date:
+Machine:	IBM RS6000
+System:		AIX 3.2
+Compiler:	AIX XL C Compiler/6000
+Problems:	none
+
+Tester:		RBrown@cmutual.com.au
+Date:
+Machine:	Linux 1.2.9
+System:		
+Compiler:	gcc 2.6.3
+Problems:	none
+
+-----------------------------------------------------------------------
+** Systems which have NOT successfully been built.
+
+Tester:		RBrown@cmutual.com.au, 
+		Lloyd.Parkes@vuw.ac.nz
+Date:		Oct, Nov, 1995
+Machine:	Dec Alpha OSF/1
+System:
+Compiler:	cc
+Problems: 	Many - see below.
+
+* Problems with Dec Alpha
+  - 64 bit length longs as opposed to 32 bit longs.
+  - Lloyd.Parkes@vuw.ac.nz says that there is some bit encoding problems 
+    but the decoding seems to work.
+  - RBrown@cmutual.com.au says that it will compile but will core dump
+    when  building the test database.
+  - tes.kbs.citri.edu.au says that he needs to look into the repercussions
+    of 64 bit longs, however, he does not have access to such a machine.
+		
+-----------------------------------------------------------------------
+** Other systems. 
+
+Systems other than the above may have installation problems.
+If this is the case, please send me (tes@kbs.citri.edu.au) a log of the output
+and I will modify the configuration files in an effort to fix the problems.
+
+-----------------------------------------------------------------------
Index: /tags/gsdl-21/gsdl/src/mgpp/about_mg.html
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/about_mg.html	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/about_mg.html	(revision 699)
@@ -0,0 +1,75 @@
+<HTML>
+<HEAD>
+<TITLE>About MG</TITLE>
+</HEAD>
+<BODY>
+<H1>About MG</H1>
+ 
+<HR SIZE=4>
+
+MG is a public domain
+indexing and retrieval system for text, images, and textual images.
+MG is covered by a 
+<a href="http://deimos.kbs.citri.edu.au/mg/intro/copying.html">GNU 
+public licence</a>.
+<P>
+MG is available for ftp <a href="ftp://munnari.oz.au/pub/mg">here</a>
+
+<H2>Developers</H2>
+
+The MG system is an embodiment of ideas developed primarily by:
+<ul>
+<li> <a href="http://www.cosc.canterbury.ac.nz/~tim">Tim C. Bell</a>,
+	University of Canterbury;
+<li> <a href="http://www.cs.mu.oz.au/~alistair">Alistair Moffat</a>,
+	University of Melbourne;
+<li> <a href="http://www.cs.waikato.ac.nz/~ihw">Ian Witten</a>,
+	University of Waikato; and
+<li> <a href="http://www.cs.rmit.edu.au/~jz">Justin Zobel</a>,
+	RMIT.
+</ul>
+
+<H2>Programmers</H2>
+
+The bulk of the programming work has been undertaken by:
+<ul>
+<li><A HREF="http://www.cs.waikato.ac.nz/~singlis/">Stuart Inglis</A>,
+       University of Waikato;
+<li><A HREF="http://www.cs.waikato.ac.nz/~cgn/">Craig Nevill-Manning</A>,
+       University of Waikato;
+<li> Neil Sharman, University of Melbourne; and
+<li> Tim Shimmin, Collaborative Information Technology Research Institute (CITRI).
+</ul>
+A full list of contributors appears
+<A HREF="http://deimos.kbs.citri.edu.au/mg/intro/mgintro.html#xtocid2063010">
+here</a>.
+
+<H2>Institutional Support</H2>
+
+The MG project has been supported by the following organisations:
+<ul>
+<li>	The Australian Research Council;
+<li>	<a HREF="http://www.citri.edu.au/">CITRI</a>, Melbourne;
+<li>	The <a HREF="http://www.cpsc.ucalgary.ca/">University
+		of Calgary</a>, Canada;
+<li>	The <a HREF="http://www.cosc.canterbury.ac.nz/">University
+		of Canterbury</a>, New Zealand;
+<li>	The <a HREF="http://www.cs.mu.oz.au/">University
+		of Melbourne</a>, Australia;
+<li>	<a HREF="http://www.cs.rmit.edu.au/">RMIT</a>, Australia; and
+<li>	The <a HREF="http://www.cs.waikato.ac.nz/">University
+		of Waikato</a>, New Zealand.
+</ul>
+
+<H2>Backpointer</H2>
+
+If you didn't just come from there, the MG home page is accessible
+<a href="http://www.kbs.citri.edu.au/mg/">here</a>.
+
+<HR SIZE=4>
+<ADDRESS>alistair@cs.mu.oz.au</ADDRESS><BR>
+ 
+</BODY>
+</HTML>
+
+
Index: /tags/gsdl-21/gsdl/src/mgpp/test/Makefile
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/test/Makefile	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/test/Makefile	(revision 699)
@@ -0,0 +1,22 @@
+# Generated automatically from Makefile.in by configure.
+CC = gcc
+
+DISTFILES = Makefile.in q1 q2 q3 r1 r2 r3 mgrc1 mgrc2 mgrc3 testmg \
+            alice.sum.l alice.len.l \
+            alice.sum.ll alice.len.ll \
+	    simple_sum.c usinglonglong.c \
+
+all: simple_sum usinglonglong
+
+dist:
+	ln $(DISTFILES) $(distdir)
+
+clean:
+	rm -f simple_sum usinglonglong *.cmp *.diff
+	rm -rf mgdata
+
+simple_sum: simple_sum.c
+	$(CC) -o $@ $?
+
+usinglonglong: usinglonglong.c
+	$(CC) -I../lib -I.. -o $@ $?
Index: /tags/gsdl-21/gsdl/src/mgpp/test/Makefile.in
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/test/Makefile.in	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/test/Makefile.in	(revision 699)
@@ -0,0 +1,21 @@
+CC = @CC@
+
+DISTFILES = Makefile.in q1 q2 q3 r1 r2 r3 mgrc1 mgrc2 mgrc3 testmg \
+            alice.sum.l alice.len.l \
+            alice.sum.ll alice.len.ll \
+	    simple_sum.c usinglonglong.c \
+
+all: simple_sum usinglonglong
+
+dist:
+	ln $(DISTFILES) $(distdir)
+
+clean:
+	rm -f simple_sum usinglonglong *.cmp *.diff
+	rm -rf mgdata
+
+simple_sum: simple_sum.c
+	$(CC) -o $@ $?
+
+usinglonglong: usinglonglong.c
+	$(CC) -I../lib -I.. -o $@ $?
Index: /tags/gsdl-21/gsdl/src/mgpp/test/alice.len.bak
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/test/alice.len.bak	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/test/alice.len.bak	(revision 699)
@@ -0,0 +1,2 @@
+./mgdata/alice/alice.invf.dict.hash: 13256 
+./mgdata/alice/alice.text.dict: 17462
Index: /tags/gsdl-21/gsdl/src/mgpp/test/alice.len.cmp
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/test/alice.len.cmp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/test/alice.len.cmp	(revision 699)
@@ -0,0 +1,2 @@
+./mgdata/alice/alice.invf.dict.hash: 13256
+./mgdata/alice/alice.text.dict: 19329
Index: /tags/gsdl-21/gsdl/src/mgpp/test/alice.len.l
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/test/alice.len.l	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/test/alice.len.l	(revision 699)
@@ -0,0 +1,2 @@
+./mgdata/alice/alice.invf.dict.hash: 13256
+./mgdata/alice/alice.text.dict: 17462
Index: /tags/gsdl-21/gsdl/src/mgpp/test/alice.len.ll
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/test/alice.len.ll	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/test/alice.len.ll	(revision 699)
@@ -0,0 +1,2 @@
+./mgdata/alice/alice.invf.dict.hash: 13256
+./mgdata/alice/alice.text.dict: 19329
Index: /tags/gsdl-21/gsdl/src/mgpp/test/alice.sum.cmp
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/test/alice.sum.cmp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/test/alice.sum.cmp	(revision 699)
@@ -0,0 +1,9 @@
+./mgdata/alice/alice.invf: 2618428 19209
+./mgdata/alice/alice.invf.chunk: 189460 2134
+./mgdata/alice/alice.invf.chunk.trans: 331998 2585
+./mgdata/alice/alice.invf.dict: 517220 23283
+./mgdata/alice/alice.invf.dict.blocked: 1030630 33218
+./mgdata/alice/alice.invf.idx: 309022 7572
+./mgdata/alice/alice.text: 5075885 40099
+./mgdata/alice/alice.text.idx: 185535 3380
+./mgdata/alice/alice.text.stats: 2432380 49146
Index: /tags/gsdl-21/gsdl/src/mgpp/test/alice.sum.l
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/test/alice.sum.l	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/test/alice.sum.l	(revision 699)
@@ -0,0 +1,9 @@
+./mgdata/alice/alice.invf: 2618428 19209
+./mgdata/alice/alice.invf.chunk: 189460 2134
+./mgdata/alice/alice.invf.chunk.trans: 331998 2585
+./mgdata/alice/alice.invf.dict: 516730 23275
+./mgdata/alice/alice.invf.dict.blocked: 1030622 33210
+./mgdata/alice/alice.invf.idx: 309022 7572
+./mgdata/alice/alice.text: 5078095 40098
+./mgdata/alice/alice.text.idx: 185050 3380
+./mgdata/alice/alice.text.stats: 2432186 49138
Index: /tags/gsdl-21/gsdl/src/mgpp/test/alice.sum.ll
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/test/alice.sum.ll	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/test/alice.sum.ll	(revision 699)
@@ -0,0 +1,9 @@
+./mgdata/alice/alice.invf: 2618428 19209
+./mgdata/alice/alice.invf.chunk: 189460 2134
+./mgdata/alice/alice.invf.chunk.trans: 331998 2585
+./mgdata/alice/alice.invf.dict: 517220 23283
+./mgdata/alice/alice.invf.dict.blocked: 1030630 33218
+./mgdata/alice/alice.invf.idx: 309022 7572
+./mgdata/alice/alice.text: 5075885 40099
+./mgdata/alice/alice.text.idx: 185535 3380
+./mgdata/alice/alice.text.stats: 2432380 49146
Index: /tags/gsdl-21/gsdl/src/mgpp/test/build.log
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/test/build.log	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/test/build.log	(revision 699)
@@ -0,0 +1,144 @@
+-----------------------------------
+towai, Mon Oct 11 12:01:53 NZDT 1999
+alice --> alice/alice
+-----------------------------------
+../src/text/mg_get alice -init
+Can not find a .mg_getrc, so creating a default one: /home/cs025/.mg_getrc
+-----------------------------------
+../src/text/mg_get alice -text | mg_passes -f alice/alice -2 -m 32 -s 3 -G -t 10 -T1 -I1
+ivf.pass1 : Mem reqd for 1 chunk:    19095 bytes
+ivf.pass1 : Chunks written      : 1
+ivf.pass1 : Peak memory usage   :        1.2 Mb
+ivf.pass1 : Stems size          :       20.9 kB  14.11%
+ivf.pass1 : Lev 1 inverted file :       16.2 kB  10.96%
+ivf.pass1 : Lev 2 inverted file :       20.0 kB  13.52%
+ivf.pass1 : Lev 3 inverted file :       41.2 kB  27.82%
+ivf.pass1 : Record addresses    :        0.0 kB   0.00%
+mg_passes : 
+mg_passes :                 :    init         process        done
+mg_passes : text.pass1      : 00:00:00.00   00:00:00.17   00:00:00.05
+mg_passes : ivf.pass1       : 00:00:00.00   00:00:00.26   00:00:00.04
+mg_passes : 
+mg_passes : Init time       : 00:00:00.00 cpu, 00:00:01 elapsed.
+mg_passes : Process time    : 00:00:00.46 cpu, 00:00:00 elapsed.
+mg_passes : Done time       : 00:00:00.09 cpu, 00:00:01 elapsed.
+mg_passes : Total time      : 00:00:00.55 cpu, 00:00:02 elapsed.
+mg_passes : Documents       : 835
+mg_passes : Bytes received  : 152435
+mg_passes : Bytes processed : 151600
+mg_passes : Process Rate    : 321.8 kB per cpu second
+-----------------------------------
+mg_perf_hash_build -f alice/alice
+mg_perf_hash_build : 00:00:00.03 cpu, 00:00:00 elapsed.
+-----------------------------------
+mg_compression_dict -f alice/alice -S
+mg_compression_dict : Num words           :     3035 ->     3036
+mg_compression_dict : Num non-words       :      260 ->      261
+mg_compression_dict : Chars of words      :    17929 ->    17929
+mg_compression_dict : Chars of non-words  :     1526 ->     1526
+mg_compression_dict : Mem usage           :    32635 ->    32643
+mg_compression_dict : Actual mem required :    21873
+mg_compression_dict : 00:00:00.03 cpu, 00:00:00 elapsed.
+-----------------------------------
+../src/text/mg_get alice -text | mg_passes -f alice/alice -2 -c 3 -G -t 10 -T2 -I2
+ivf.pass2 : M
+mg_passes : Compressed Text 0.04 Mb ( 40099 bytes   26.451 %)
+mg_passes : Words portion of the dictionary 0.02 Mb ( 18293 bytes   12.067 %)
+mg_passes : Huffman info for chars in dictionary 0.00 Mb ( 1036 bytes   0.683 %)
+ivf.pass2 : File sizes
+ivf.pass2 :   Chunk desc    :       2134 bytes
+ivf.pass2 :   Chunk trans   :       2585 bytes
+ivf.pass2 :   Chunks        :          0 bytes
+ivf.pass2 :   Chunk state   :       8192 bytes
+ivf.pass2 :   Invf state    :      40960 bytes
+ivf.pass2 :   Peak invf     :      24576 bytes
+ivf.pass2 :   Final invf    :      19209 bytes
+ivf.pass2 : Peak disk usage :     408.39 %
+ivf.pass2 : Pass two data structures      :  0.016 Mbyte
+ivf.pass2 : Pass two hash structure(s)    :  0.011 Mbyte
+ivf.pass2 : Peak extra memory in use      :  0.049 Mbyte
+ivf.pass2 : Peak total memory in use      :  0.077 Mbyte
+mg_passes : 
+mg_passes :                 :    init         process        done
+mg_passes : text.pass2      : 00:00:00.00   00:00:00.16   00:00:00.00
+mg_passes : ivf.pass2       : 00:00:00.01   00:00:00.49   00:00:00.02
+mg_passes : 
+mg_passes : Init time       : 00:00:00.01 cpu, 00:00:01 elapsed.
+mg_passes : Process time    : 00:00:00.67 cpu, 00:00:01 elapsed.
+mg_passes : Done time       : 00:00:00.02 cpu, 00:00:01 elapsed.
+mg_passes : Total time      : 00:00:00.70 cpu, 00:00:03 elapsed.
+mg_passes : Documents       : 835
+mg_passes : Bytes received  : 152435
+mg_passes : Bytes processed : 151600
+mg_passes : Process Rate    : 221.0 kB per cpu second
+-----------------------------------
+mg_weights_build -f alice/alice -b 6
+mg_weights_build : The file "alice/alice.weight" does not exist.
+mg_weights_build : Building the weight data from the file "alice/alice.invf".
+.
+L = 2.906835
+U = 69.816032
+B = 1.050923
+mg_weights_build : Warning: Document 33 had a weight of 0.
+mg_weights_build : Warning: Document 34 had a weight of 0.
+mg_weights_build : Warning: Document 35 had a weight of 0.
+mg_weights_build : Warning: Document 43 had a weight of 0.
+mg_weights_build : Warning: Document 44 had a weight of 0.
+mg_weights_build : Warning: Document 45 had a weight of 0.
+mg_weights_build : Warning: Document 223 had a weight of 0.
+mg_weights_build : Warning: Document 224 had a weight of 0.
+mg_weights_build : Warning: Document 225 had a weight of 0.
+mg_weights_build : 00:00:00.05 cpu, 00:00:00 elapsed.
+-----------------------------------
+mg_invf_dict -f alice/alice -b 4096
+mg_invf_dict : Block size = 4096
+mg_invf_dict : Number of blocks written = 9
+-----------------------------------
+mgstat -f alice/alice -E
+Input bytes                        : 4684448949174534144,     0.14 Mbyte
+Documents                          :        835
+Words in collection [dict]         :      27618
+Longest doc in collection [dict]   :       1260 characters
+Maximum ratio                      :       6.22
+Words in dict                      :       3035
+Non-words in dict                  :        260
+Total chars of distinct words      :      17929
+Total chars of distinct non-words  :       1526
+Words in collection [stem]         :      27504
+Words in stem                      :       1891
+Indexed fragments                  :        835
+Total chars of stem words          :      11072
+
+The collection is in "../test/mgdata/alice"
+
+		Files required by mgquery
+alice.text                         :      40099 bytes
+alice.invf                         :      19209 bytes
+alice.text.idx.wgt                 :       6692 bytes
+alice.weight.approx                :        649 bytes
+alice.invf.dict.blocked            :      33218 bytes
+alice.text.dict                    :      19329 bytes
+SUB TOTAL                          :     119196 bytes
+
+		Files NOT required by mgquery
+alice.invf.dict                    :      23283 bytes
+alice.invf.idx                     :       7572 bytes
+alice.text.stats                   :      49146 bytes
+alice.text.idx                     :       3380 bytes
+alice.weight                       :       3344 bytes
+alice.invf.chunk                   :       2134 bytes
+alice.invf.chunk.trans             :       2585 bytes
+alice.invf.dict.hash               :      13256 bytes
+SUB TOTAL                          :     104700 bytes
+
+TOTAL                              :     223896 bytes
+-----------------------------------
+../src/text/mg_get alice -cleanup
+-----------------------------------
+towai, Mon Oct 11 12:01:59 NZDT 1999
+-----------------------------------
+-- The fast-loading compression dictionary has not been built.
+-- If you wish to build it, execute the following command:
+-- ../src/text/mg_fast_comp_dict -f alice/alice
+-----------------------------------
+
Index: /tags/gsdl-21/gsdl/src/mgpp/test/mgrc1
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/test/mgrc1	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/test/mgrc1	(revision 699)
@@ -0,0 +1,2 @@
+.set mode text
+.set query boolean
Index: /tags/gsdl-21/gsdl/src/mgpp/test/mgrc2
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/test/mgrc2	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/test/mgrc2	(revision 699)
@@ -0,0 +1,2 @@
+.set mode text
+.set query ranked
Index: /tags/gsdl-21/gsdl/src/mgpp/test/mgrc3
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/test/mgrc3	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/test/mgrc3	(revision 699)
@@ -0,0 +1,2 @@
+.set mode docnums
+.set query boolean
Index: /tags/gsdl-21/gsdl/src/mgpp/test/q1
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/test/q1	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/test/q1	(revision 699)
@@ -0,0 +1,1 @@
+flamingo
Index: /tags/gsdl-21/gsdl/src/mgpp/test/q2
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/test/q2	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/test/q2	(revision 699)
@@ -0,0 +1,1 @@
+flamingo
Index: /tags/gsdl-21/gsdl/src/mgpp/test/q3
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/test/q3	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/test/q3	(revision 699)
@@ -0,0 +1,2 @@
+Rabbit & !(Alice | she)
+laser
Index: /tags/gsdl-21/gsdl/src/mgpp/test/r1
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/test/r1	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/test/r1	(revision 699)
@@ -0,0 +1,45 @@
+> flamingo
+---------------------------------- 482
+  The chief difficulty Alice found at first was in managing her 
+flamingo:  she succeeded in getting its body tucked away, 
+comfortably enough, under her arm, with its legs hanging down, 
+but generally, just as she had got its neck nicely straightened 
+out, and was going to give the hedgehog a blow with its head, it 
+WOULD twist itself round and look up in her face, with such a 
+puzzled expression that she could not help bursting out laughing:  
+and when she had got its head down, and was going to begin again, 
+it was very provoking to find that the hedgehog had unrolled 
+itself, and was in the act of crawling away:  besides all this, 
+there was generally a ridge or furrow in the way wherever she 
+wanted to send the hedgehog to, and, as the doubled-up soldiers 
+were always getting up and walking off to other parts of the 
+ground, Alice soon came to the conclusion that it was a very 
+difficult game indeed. 
+---------------------------------- 487
+  Alice waited till the eyes appeared, and then nodded.  `It's no 
+use speaking to it,' she thought, `till its ears have come, or at 
+least one of them.'  In another minute the whole head appeared, 
+and then Alice put down her flamingo, and began an account of the 
+game, feeling very glad she had someone to listen to her.  The 
+Cat seemed to think that there was enough of it now in sight, and 
+no more of it appeared. 
+---------------------------------- 502
+  The hedgehog was engaged in a fight with another hedgehog, 
+which seemed to Alice an excellent opportunity for croqueting one 
+of them with the other:  the only difficulty was, that her 
+flamingo was gone across to the other side of the garden, where 
+Alice could see it trying in a helpless sort of way to fly up 
+into a tree. 
+---------------------------------- 503
+  By the time she had caught the flamingo and brought it back, 
+the fight was over, and both the hedgehogs were out of sight:  
+`but it doesn't matter much,' thought Alice, `as all the arches 
+are gone from the side of the ground.'  So she tucked it away 
+under her arm, that it might not escape again, and went back for 
+a little more conversation with her friend. 
+---------------------------------- 526
+  `I dare say you're wondering why I don't put my arm round your 
+waist,' the Duchess said after a pause:  `the reason is, that I'm 
+doubtful about the temper of your flamingo.  Shall I try the 
+experiment?' 
+5 documents retrieved.
Index: /tags/gsdl-21/gsdl/src/mgpp/test/r1.cmp
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/test/r1.cmp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/test/r1.cmp	(revision 699)
@@ -0,0 +1,45 @@
+> flamingo
+---------------------------------- 482
+  The chief difficulty Alice found at first was in managing her 
+flamingo:  she succeeded in getting its body tucked away, 
+comfortably enough, under her arm, with its legs hanging down, 
+but generally, just as she had got its neck nicely straightened 
+out, and was going to give the hedgehog a blow with its head, it 
+WOULD twist itself round and look up in her face, with such a 
+puzzled expression that she could not help bursting out laughing:  
+and when she had got its head down, and was going to begin again, 
+it was very provoking to find that the hedgehog had unrolled 
+itself, and was in the act of crawling away:  besides all this, 
+there was generally a ridge or furrow in the way wherever she 
+wanted to send the hedgehog to, and, as the doubled-up soldiers 
+were always getting up and walking off to other parts of the 
+ground, Alice soon came to the conclusion that it was a very 
+difficult game indeed. 
+---------------------------------- 487
+  Alice waited till the eyes appeared, and then nodded.  `It's no 
+use speaking to it,' she thought, `till its ears have come, or at 
+least one of them.'  In another minute the whole head appeared, 
+and then Alice put down her flamingo, and began an account of the 
+game, feeling very glad she had someone to listen to her.  The 
+Cat seemed to think that there was enough of it now in sight, and 
+no more of it appeared. 
+---------------------------------- 502
+  The hedgehog was engaged in a fight with another hedgehog, 
+which seemed to Alice an excellent opportunity for croqueting one 
+of them with the other:  the only difficulty was, that her 
+flamingo was gone across to the other side of the garden, where 
+Alice could see it trying in a helpless sort of way to fly up 
+into a tree. 
+---------------------------------- 503
+  By the time she had caught the flamingo and brought it back, 
+the fight was over, and both the hedgehogs were out of sight:  
+`but it doesn't matter much,' thought Alice, `as all the arches 
+are gone from the side of the ground.'  So she tucked it away 
+under her arm, that it might not escape again, and went back for 
+a little more conversation with her friend. 
+---------------------------------- 526
+  `I dare say you're wondering why I don't put my arm round your 
+waist,' the Duchess said after a pause:  `the reason is, that I'm 
+doubtful about the temper of your flamingo.  Shall I try the 
+experiment?' 
+5 documents retrieved.
Index: /tags/gsdl-21/gsdl/src/mgpp/test/r2
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/test/r2	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/test/r2	(revision 699)
@@ -0,0 +1,45 @@
+> flamingo
+---------------------------------- 526 1.322623
+  `I dare say you're wondering why I don't put my arm round your 
+waist,' the Duchess said after a pause:  `the reason is, that I'm 
+doubtful about the temper of your flamingo.  Shall I try the 
+experiment?' 
+---------------------------------- 503 1.013693
+  By the time she had caught the flamingo and brought it back, 
+the fight was over, and both the hedgehogs were out of sight:  
+`but it doesn't matter much,' thought Alice, `as all the arches 
+are gone from the side of the ground.'  So she tucked it away 
+under her arm, that it might not escape again, and went back for 
+a little more conversation with her friend. 
+---------------------------------- 502 1.009519
+  The hedgehog was engaged in a fight with another hedgehog, 
+which seemed to Alice an excellent opportunity for croqueting one 
+of them with the other:  the only difficulty was, that her 
+flamingo was gone across to the other side of the garden, where 
+Alice could see it trying in a helpless sort of way to fly up 
+into a tree. 
+---------------------------------- 487 0.882496
+  Alice waited till the eyes appeared, and then nodded.  `It's no 
+use speaking to it,' she thought, `till its ears have come, or at 
+least one of them.'  In another minute the whole head appeared, 
+and then Alice put down her flamingo, and began an account of the 
+game, feeling very glad she had someone to listen to her.  The 
+Cat seemed to think that there was enough of it now in sight, and 
+no more of it appeared. 
+---------------------------------- 482 0.529432
+  The chief difficulty Alice found at first was in managing her 
+flamingo:  she succeeded in getting its body tucked away, 
+comfortably enough, under her arm, with its legs hanging down, 
+but generally, just as she had got its neck nicely straightened 
+out, and was going to give the hedgehog a blow with its head, it 
+WOULD twist itself round and look up in her face, with such a 
+puzzled expression that she could not help bursting out laughing:  
+and when she had got its head down, and was going to begin again, 
+it was very provoking to find that the hedgehog had unrolled 
+itself, and was in the act of crawling away:  besides all this, 
+there was generally a ridge or furrow in the way wherever she 
+wanted to send the hedgehog to, and, as the doubled-up soldiers 
+were always getting up and walking off to other parts of the 
+ground, Alice soon came to the conclusion that it was a very 
+difficult game indeed. 
+5 documents retrieved.
Index: /tags/gsdl-21/gsdl/src/mgpp/test/r2.cmp
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/test/r2.cmp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/test/r2.cmp	(revision 699)
@@ -0,0 +1,45 @@
+> flamingo
+---------------------------------- 526 1.322623
+  `I dare say you're wondering why I don't put my arm round your 
+waist,' the Duchess said after a pause:  `the reason is, that I'm 
+doubtful about the temper of your flamingo.  Shall I try the 
+experiment?' 
+---------------------------------- 503 1.013693
+  By the time she had caught the flamingo and brought it back, 
+the fight was over, and both the hedgehogs were out of sight:  
+`but it doesn't matter much,' thought Alice, `as all the arches 
+are gone from the side of the ground.'  So she tucked it away 
+under her arm, that it might not escape again, and went back for 
+a little more conversation with her friend. 
+---------------------------------- 502 1.009519
+  The hedgehog was engaged in a fight with another hedgehog, 
+which seemed to Alice an excellent opportunity for croqueting one 
+of them with the other:  the only difficulty was, that her 
+flamingo was gone across to the other side of the garden, where 
+Alice could see it trying in a helpless sort of way to fly up 
+into a tree. 
+---------------------------------- 487 0.882496
+  Alice waited till the eyes appeared, and then nodded.  `It's no 
+use speaking to it,' she thought, `till its ears have come, or at 
+least one of them.'  In another minute the whole head appeared, 
+and then Alice put down her flamingo, and began an account of the 
+game, feeling very glad she had someone to listen to her.  The 
+Cat seemed to think that there was enough of it now in sight, and 
+no more of it appeared. 
+---------------------------------- 482 0.529432
+  The chief difficulty Alice found at first was in managing her 
+flamingo:  she succeeded in getting its body tucked away, 
+comfortably enough, under her arm, with its legs hanging down, 
+but generally, just as she had got its neck nicely straightened 
+out, and was going to give the hedgehog a blow with its head, it 
+WOULD twist itself round and look up in her face, with such a 
+puzzled expression that she could not help bursting out laughing:  
+and when she had got its head down, and was going to begin again, 
+it was very provoking to find that the hedgehog had unrolled 
+itself, and was in the act of crawling away:  besides all this, 
+there was generally a ridge or furrow in the way wherever she 
+wanted to send the hedgehog to, and, as the doubled-up soldiers 
+were always getting up and walking off to other parts of the 
+ground, Alice soon came to the conclusion that it was a very 
+difficult game indeed. 
+5 documents retrieved.
Index: /tags/gsdl-21/gsdl/src/mgpp/test/r3
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/test/r3	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/test/r3	(revision 699)
@@ -0,0 +1,19 @@
+> Rabbit & !(Alice | she)
+      14   48.0174         0
+     126   48.0174         0
+     143   48.0174         0
+     478   48.0174         0
+     701   48.0174         0
+     704   48.0174         0
+     705   48.0174         0
+     755   48.0174         0
+     774   48.0174         0
+     786   48.0174         0
+     788   48.0174         0
+     791   48.0174         0
+     793   48.0174         0
+     801   48.0174         0
+     803   48.0174         0
+15 documents retrieved.
+> laser
+No entries correspond to that query.
Index: /tags/gsdl-21/gsdl/src/mgpp/test/r3.cmp
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/test/r3.cmp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/test/r3.cmp	(revision 699)
@@ -0,0 +1,19 @@
+> Rabbit & !(Alice | she)
+      14   48.0174         0
+     126   48.0174         0
+     143   48.0174         0
+     478   48.0174         0
+     701   48.0174         0
+     704   48.0174         0
+     705   48.0174         0
+     755   48.0174         0
+     774   48.0174         0
+     786   48.0174         0
+     788   48.0174         0
+     791   48.0174         0
+     793   48.0174         0
+     801   48.0174         0
+     803   48.0174         0
+15 documents retrieved.
+> laser
+No entries correspond to that query.
Index: /tags/gsdl-21/gsdl/src/mgpp/test/simple_sum.c
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/test/simple_sum.c	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/test/simple_sum.c	(revision 699)
@@ -0,0 +1,103 @@
+/**************************************************************************
+ *
+ * sum.c -- byte summing program
+ * Copyright (C) 1995  Tim Shimmin
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+$Log$
+Revision 1.1  1999/10/11 02:56:32  cs025
+Base install of MG-PP
+
+Revision 1.1  1999/08/10 21:18:33  sjboddie
+renamed mg-1.3d directory mg
+
+Revision 1.1  1998/11/17 09:36:00  rjmcnab
+*** empty log message ***
+
+*/
+
+/*
+ * If given -l option then just prints the length.
+ * Normally prints the sum and the length for the file.
+ */
+
+static int length_only = 0;
+
+#include <stdio.h>
+
+void 
+output_sum(fname, input_file, output_file)
+char *fname;
+FILE *input_file;
+FILE *output_file;
+{
+  int ch = '\0';
+  unsigned long sum = 0;
+  unsigned long num_bytes = 0;
+
+  while((ch = getc(input_file))!=EOF){
+    sum += ch;
+    num_bytes++;
+  }
+
+  if (length_only)
+    fprintf(output_file, "%s: %ld\n", fname, num_bytes);
+  else
+    fprintf(output_file, "%s: %ld %ld\n", fname, sum, num_bytes);
+}
+
+void
+main(argc, argv)
+int argc;
+char *argv[];
+{
+  int i = 0;
+
+  /* use stdin */
+  if (argc == 1){
+    output_sum("-", stdin, stdout);
+    exit(0);
+  }
+
+  /* use args as file names for input */
+  for (i=1;i<argc; i++){
+    char *fname = argv[i];
+    FILE *input_file = NULL;
+    
+    if (!length_only && (argv[i][0] == '-') && (argv[i][1] =='l') ) 
+      { 
+        length_only = 1; 
+        continue;
+      }
+
+    input_file = fopen(fname, "r");
+    if (!input_file){
+      fprintf(stderr,"Could not open %s\n", fname);
+      continue;
+    }
+    else{
+      output_sum(fname, input_file, stdout);
+      fclose(input_file);
+    }
+  }/*for*/
+  exit(0);
+
+}
Index: /tags/gsdl-21/gsdl/src/mgpp/test/testmg
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/test/testmg	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/test/testmg	(revision 699)
@@ -0,0 +1,176 @@
+#!/bin/csh -f 
+###########################################################################
+#
+# mg_get.sh -- script to do rudimentary tests on mg
+# Copyright (C) 1995  Tim Shimmin (August)
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#
+###########################################################################
+
+#---------------------------------------------------------
+# Tests out the MG full-text retrieval system
+# ie. the files in src/text
+#
+# Builds up the alice database and compares its using
+# the output of "simple_sum" on a prebuilt alice database.
+# Since some file formats will vary on machines, these files
+# will only have their lengths compared.
+#
+# Does some queries on the database and compares the
+# results with precomputed ones.
+#
+# This is a very rudimentary test.
+# Not much functionality is exercised
+#---------------------------------------------------------
+
+# Modified 1999-08-05 Tim A. H. Bell <bhat@cs.mu.oz.au>
+# Cope with either using "long long" or not, which causes differences in
+# the lengths of various files.
+# Restore old ~/.mg_getrc file
+
+ 
+# setup directories
+set top_srcdir = $1
+set testdir = $top_srcdir/test
+set bindir = $top_srcdir/src/text # uses this dir as we haven't probably installed yet
+
+# do everything in the testdir directory
+cd $testdir
+
+# *** Test out build ***
+
+set collection = alice
+
+# set env variables
+setenv MGDATA "$testdir/mgdata"
+setenv MGSAMPLE "$top_srcdir/SampleData"
+
+# create dirs
+mkdir $MGDATA 
+
+# backup any past .mg_getrc file
+if (-e ~/.mg_getrc) then
+  mv ~/.mg_getrc ~/.mg_getrc.bak
+  set mggetrcbackedup = 1
+else
+  set mggetrcbackedup = 0
+endif
+
+# copy mgbuild into bindir directory so it uses the other binaries
+# need all binaries in same directory for mgbuild
+cp $top_srcdir/src/scripts/mgbuild $bindir
+cp $top_srcdir/src/scripts/mg_get $bindir
+
+# do the build
+echo "Building test database..."
+$bindir/mgbuild $collection >&! $testdir/build.log
+
+if ($status != 0) then
+  echo "FAIL: mgbuild $collection did not successfully complete"
+  set buildfailed = 1
+else
+  set buildfailed = 0
+endif
+
+# remove them now
+rm $bindir/mgbuild
+rm $bindir/mg_get
+
+# restore any past .mg_getrc file
+if ($mggetrcbackedup == 1) then
+  mv ~/.mg_getrc.bak ~/.mg_getrc
+endif
+
+if ($buildfailed == 1) then
+  exit 1
+endif
+
+@ num_pass = 0
+@ num_fail = 0
+
+# See whether "long long" is being used
+./usinglonglong
+if ($status == 0) then
+    set llsuffix = ll
+else
+    set llsuffix = l
+endif
+
+# *** Test out the build ***
+set sum_suffixes = \
+  (invf invf.chunk invf.chunk.trans invf.dict invf.dict.blocked \
+   invf.idx text text.idx text.stats)
+
+set length_suffixes = \
+  (invf.dict.hash text.dict)
+
+# create the data sum file
+cat < /dev/null >! ./$collection.sum.cmp # create empty file
+foreach suffix ($sum_suffixes)
+  set file = "./mgdata/$collection/$collection.$suffix"
+  ./simple_sum $file >> ./$collection.sum.cmp
+end
+
+# create the data length file
+cat < /dev/null >! ./$collection.len.cmp # create empty file
+foreach suffix ($length_suffixes)
+  set file = "./mgdata/$collection/$collection.$suffix"
+  ./simple_sum -l $file >> ./$collection.len.cmp
+end
+
+diff ./$collection.sum.$llsuffix ./$collection.sum.cmp > sum.diff
+if ($status == 0) then
+  echo "PASS: Build Sum comparison matched ok"
+  @ num_pass++
+else
+  echo "FAIL: Build Sum comparison did not match"
+  echo "Please inspect sum.diff to see which files are different" 
+  @ num_fail++
+endif
+
+diff ./$collection.len.$llsuffix ./$collection.len.cmp > len.diff
+if ($status == 0) then
+  echo "PASS: Build Length comparison matched ok"
+  @ num_pass++
+else
+  echo "FAIL: Build Length comparison did not match"
+  echo "Please inspect len.diff to see which files are different" 
+  @ num_fail++
+endif
+
+
+# *** Test out querying ***
+echo "Running queries..."
+@ num_queries = 3
+@ query_count = 1
+
+while ($query_count <= $num_queries)
+  cp ./mgrc$query_count ./.mgrc
+  $bindir/mgquery $collection < ./q$query_count >&! \
+    ./r${query_count}.cmp
+  diff ./r${query_count}.cmp ./r$query_count >&! ./diff.log
+  if ($status == 0) then
+    echo "PASS: query $query_count matched ok"
+    @ num_pass++
+  else
+    echo "FAIL: query $query_count did not match"
+    @ num_fail++
+  endif
+  @ query_count++
+end
+
+echo "Number of failures = $num_fail"
Index: /tags/gsdl-21/gsdl/src/mgpp/test/usinglonglong.c
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/test/usinglonglong.c	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/test/usinglonglong.c	(revision 699)
@@ -0,0 +1,13 @@
+#include <stdlib.h>
+#include "longlong.h"
+
+/* return successfully if using "long long" */
+
+int main(int argc, char *argv[]) 
+{
+#ifdef USE_LONG_LONG
+  exit(0);
+#else
+  exit(1);
+#endif
+}
Index: /tags/gsdl-21/gsdl/src/mgpp/text/DocEntry.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/DocEntry.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/DocEntry.cpp	(revision 699)
@@ -0,0 +1,70 @@
+/**************************************************************************
+ *
+ * DocEntry.cpp -- Underlying routines for mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "DocEntry.h"
+
+/**
+ * Block: Query Document handling;
+ *        Provides basic loading/unloading facilities for individual
+ *          documents, usually for presentation purposes (searches
+ *          are performed on inverted files
+ */
+
+long DocEntry_FreeTextBuffer(DocEntry *de)
+{
+  if (de->CompTextBuffer)
+  {
+    Xfree (de->CompTextBuffer);
+    de->CompTextBuffer = NULL;
+    return de->Len;
+  }
+  return 0;
+}
+
+/*
+ * Function: DE_comp
+ * Description: The void * parameters are deliberate; this function is used
+ *              with the SP code, and the more abstract handle is required
+ */
+int DocEntry_compare(void *a, void *b)
+{
+  return ((DocEntry *) a)->SeekPos - ((DocEntry *) b)->SeekPos;
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
Index: /tags/gsdl-21/gsdl/src/mgpp/text/DocEntry.h
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/DocEntry.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/DocEntry.h	(revision 699)
@@ -0,0 +1,43 @@
+#ifndef DocEntry_H
+#define DocEntry_H
+
+#include "sysfuncs.h"
+
+#include "lists.h"
+
+long DocEntry_FreeTextBuffer(DocEntry *de);
+
+#ifdef __cplusplus
+inline DocEntry *DocEntry_getNext(DocEntry *de)
+{ return de->Next;
+}
+
+inline unsigned long DocEntry_length(DocEntry *de)
+{ return de->Len;
+}
+
+inline void DocEntry_setNext(DocEntry *de, DocEntry *next)
+{ de->Next = next;
+}
+
+inline int DocEntry_getOrIncluded(DocEntry *de)
+{ return de->or_included;
+}
+
+inline void DocEntry_setOrIncluded(DocEntry *de, int to)
+{ de->or_included = to;
+}
+
+inline int DocEntry_TextLoaded(DocEntry *de)
+{ return (de->CompTextBuffer != NULL);
+}
+
+inline char *DocEntry_TextBuffer(DocEntry *de)
+{ return de->CompTextBuffer;
+}
+
+#endif
+
+int DocEntry_compare(void *a, void *b);
+
+#endif
Index: /tags/gsdl-21/gsdl/src/mgpp/text/MGPASS.C
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/MGPASS.C	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/MGPASS.C	(revision 699)
@@ -0,0 +1,505 @@
+/**************************************************************************
+ *
+ * mgpass.cpp -- Driver for the various passes -
+
+	V1 -  removed all the pipe processing and replaced with
+	      code to directly explore a directory of files.
+	V2 -  rebuilt to extract non text files from web browser
+	      'catch' file.  Also to display progress count
+			   GH/WJR
+
+ * Copyright (C) 1994  Neil Sharman, ..
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include <malloc.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <io.h>
+#include <fcntl.h>
+#include <string.h>
+
+#include "memlib.h"
+#include "messages.h"
+
+#include "mg_files.h"
+#include "mg.h"
+#include "build.h"
+#include "text.h"
+#include "stemmer.h"
+
+/*
+   $Log$
+   Revision 1.1  1999/10/11 02:56:55  cs025
+   Base install of MG-PP
+
+   Revision 1.1  1999/08/10 21:17:39  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.5  1998/12/02 01:32:40  sjboddie
+   fixed 1 missing semi-colon..
+
+   Revision 1.4  1998/12/02 00:57:10  rjmcnab
+
+   Fixed to compile with the changes to the stemming.
+
+   Revision 1.3  1998/11/24 03:39:23  rjmcnab
+
+   Fixed a small error in the windows compilation
+
+   Revision 1.2  1998/11/24 01:29:39  rjmcnab
+
+   Fixed a few problems with the windows build
+
+   Revision 1.1  1998/11/17 09:34:12  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:57  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:52  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#define MAX_PASSES 5
+
+#define SPECIAL 1
+#define TEXT_PASS_1 2
+#define TEXT_PASS_2 4
+#define IVF_PASS_1 8
+#define IVF_PASS_2 16
+
+#define MIN_BUF 8192
+#define path_length 256
+
+unsigned long buf_size = 3 * 1024 * 1024;       /* 3Mb */
+unsigned long invf_buffer_size = 5 * 1024 * 1024;       /* 5Mb */
+unsigned long ChunkLimit = 0;
+char InvfLevel = 2;
+char SkipSGML = 0;
+char MakeWeights = 0;
+FILE *Comp_Stats = NULL;
+int comp_stat_point = 0;
+double bytes_processed = 0;
+int num_docs = 0;
+double bytes_received = 0;
+int stem_method = 0;
+int stemmer_num = 0; /* default to the Lovin stemmer */
+
+static char Passes = 0;
+static unsigned long trace = 0;
+static int Dump = 0;
+static char **files = NULL;
+static int num_files = 0;
+static char *trace_name = NULL;
+
+static char dirname[path_length], wildname[path_length];
+static int by_para = 0, recurse = 0, html_catch = 0;
+static char *buffer;
+char *line_start, *data_end, *base, *scan;
+
+typedef struct pass_data
+  {
+    char *name;
+    int (*init) (char *);
+    int (*process) (u_char *, int);
+    int (*done) (char *);
+  }
+pass_data;
+
+static pass_data PassData[MAX_PASSES] =
+{
+  {"special", init_special, process_special, done_special},
+  {"text.pass1", init_text_1, process_text_1, done_text_1},
+  {"text.pass2", init_text_2, process_text_2, done_text_2},
+  {"ivf.pass1", init_ivf_1, process_ivf_1, done_ivf_1},
+  {"ivf.pass2", init_ivf_2, process_ivf_2, done_ivf_2},
+};
+
+static char *usage_str = "\nUSAGE:\n"
+"  %s [-h] [-G] [-D] [-1|-2|-3] [-T1] [-T2] [-I1] [-I2] [-N1]\n"
+"  %*s [-N2] [-W] [-S] [-b buffer-size] [-d dictionary-directory]\n"
+"  %*s [-t trace-point Mb] [-m invf-memory] [-c chunk-limit]\n"
+"  %*s [-n trace-name] [-C comp-stat-size] [-s stem_method] -f doc-collection-name\n"
+"  %*s [source directory\\] [source file]\n";
+
+static void usage (char *err)
+{
+  if (err) Message (err);
+  fprintf (stderr, usage_str, msg_prefix, strlen (msg_prefix), "",
+    strlen (msg_prefix), "", strlen (msg_prefix), "");
+  exit (1);
+}
+
+void do_process(char *buffer, int num)
+{
+  int pass;
+  if (num == 0) Message ("Warning : Processing zero length document");
+  num_docs++;
+  bytes_processed += num;
+  for (pass = 0; pass < MAX_PASSES; pass++)
+    if (Passes & (1 << pass))
+	if (PassData[pass].process((u_char *)buffer, num) == COMPERROR)
+	   FatalError(1, "Error during file processing");
+}
+
+int refill(int in_fd)
+{
+  int num, bitleft;
+  bitleft = data_end - base;
+  memmove(buffer, base, bitleft);
+  if (buf_size - bitleft < MIN_BUF)
+    FatalError(1, "Paragraph too big for buffer");
+  num = read(in_fd, &buffer[bitleft], buf_size - bitleft);
+  line_start -= (base - buffer);
+  scan -= (base - buffer);
+  base = buffer;
+  data_end = buffer + bitleft + num;
+  if (num > 0) return 1; else return 0;
+}
+
+char *scanpara(int in_fd)
+{
+  int num, blank, in_blank, at_end;
+  num = read(in_fd,buffer,buf_size);
+  in_blank = 1;  base = buffer;  line_start = buffer;  scan = buffer;
+  data_end = buffer + num;  at_end = 0;
+  for (;;) {
+    blank = 1;  line_start = scan;                   // Get a line
+    while (scan < data_end  &&  *scan != '\n')
+      if (*scan++ > ' ') blank = 0;
+    if (scan >= data_end) {
+      if (refill(in_fd)) {
+	while (scan < data_end  &&  *scan != '\n')
+	  if (*scan++ > ' ') blank = 0;
+	if (scan < data_end) scan++;
+	}
+      else at_end = 1;
+      }
+    else scan++;
+    if (line_start < scan) {                         // If we have a line
+      if (in_blank) {
+	if (!blank) { in_blank = 0; base = line_start; }
+	}
+      else {
+	if (blank) {
+	  do_process(base, line_start - base);
+	  in_blank = 1;  base = line_start;
+	  }
+	}
+      }
+    if (at_end) break;
+    }
+  if (in_blank) base = scan;
+  if (scan + 2 <= buffer + buf_size) {
+    *scan++ = 26;  *scan++ = 10;
+    }
+  if (base < scan) do_process(base, scan - base);
+  return NULL;
+}
+
+char *scanfile(int in_fd)
+{
+  int num = read(in_fd,buffer,buf_size);  /*expect to read the whole file*/
+  if (num < 0) return "file locked";
+  if (num >= buf_size-1) return "file too long";
+  do_process(buffer, num);
+  return NULL;
+}
+
+void search(char *dname, char *fname)
+{
+  long dirtag;  struct _finddata_t dirinfo;  int in_fd;  char *res;
+  char search_name[path_length], found_name[path_length];
+
+  sprintf(search_name, "%s%s", dname, fname);    /*Scan files*/
+  dirtag = _findfirst(search_name, &dirinfo);
+  if (dirtag >= 0) {
+    do {
+      if ((dirinfo.attrib & (_A_SUBDIR | _A_HIDDEN | _A_SYSTEM)) == 0) {
+	sprintf(found_name,"%s%s",dname,dirinfo.name);
+	in_fd = open(found_name,O_RDONLY|O_BINARY);
+	if (in_fd >= 0) {
+	  if (by_para)
+	    res = scanpara(in_fd);
+	  else
+	    res = scanfile(in_fd);
+	  if (res != NULL) {
+	    Message("Error %s in processing file %s\n", res, found_name);
+	    exit(1);
+	    }
+	  close(in_fd);
+	  }
+	}
+      } while (_findnext(dirtag, &dirinfo) == 0);
+    _findclose(dirtag);
+    }
+
+  if (recurse == 0) return;
+
+  sprintf(search_name, "%s*.*", dname);          /*Look for subdirs*/
+  dirtag = _findfirst(search_name, &dirinfo);
+  if (dirtag < 0) return;
+  do {
+    if ( ((dirinfo.attrib & (_A_HIDDEN | _A_SYSTEM)) == 0)  &&
+	 ((dirinfo.attrib & _A_SUBDIR) != 0)  &&
+	 strcmp(dirinfo.name,".") != 0 &&
+	 strcmp(dirinfo.name,"..") != 0) {
+      sprintf(found_name,"%s%s",dname,dirinfo.name);
+      strcat(found_name,"\\");
+      search(found_name,fname);
+      }
+    } while (_findnext(dirtag, &dirinfo) == 0);
+  _findclose(dirtag);
+}
+
+static int toobig(int n)
+{
+  if (n > path_length) {
+    printf("Cannot handle urls > %d characters in length\n", path_length);
+    exit(1);
+    }
+  return 0;
+}
+
+void scan_catch(char *dname, char *fname)
+{
+  int in_fd, urllen, conlen, filesize;  char catch_name[path_length];
+  int filecount = 0;
+  enum { filekind_redirected, filekind_text, filekind_other } filekind;
+  sprintf(catch_name, "%s%s", dname, fname);
+  in_fd = open(catch_name,O_RDONLY|O_BINARY);
+  if (in_fd < 0)
+    FatalError(1, "Couldn't open catch file \"%s\"", catch_name);
+  for (;;) {
+    filecount++;  if (filecount%100 == 0) { printf("%d\r", filecount);  fflush(stdout); }
+    if (read(in_fd, &urllen, sizeof(int)) != sizeof(int) || toobig(urllen) ||
+	read(in_fd, catch_name, urllen) != urllen ||
+	read(in_fd, &conlen, sizeof(int)) != sizeof(int) || toobig(conlen) ||
+	read(in_fd, catch_name, conlen) != conlen) break;
+    if (conlen >= 1 && *catch_name == '@')
+      filekind = filekind_redirected;
+    else {
+      if (conlen >= 4 && strncmp(catch_name, "text", 4) == 0)
+	filekind = filekind_text;
+      else
+	filekind = filekind_other;
+      if (read(in_fd, &filesize, sizeof(int)) != sizeof(int))
+	FatalError(1, "File read failed for size field");
+      if (filesize > buf_size)
+	FatalError(1, "File too large (%d > %d)", filesize, buf_size);
+      if (read(in_fd, buffer, filesize) != filesize)
+	FatalError(1, "Failed to read file data");
+      }
+    if (filekind == filekind_text) do_process(buffer, filesize);
+    }
+  close(in_fd);
+}
+
+static void driver (FILE * Trace, char *file_name)
+{
+  int pass;
+
+  buffer = (char *)Xmalloc (buf_size);
+
+  for (pass = 0; pass < MAX_PASSES; pass++)
+    if (Passes & (1 << pass))
+      {
+	if (PassData[pass].init (file_name) == COMPERROR)
+	  FatalError (1, "Error during init of \"%s\"",PassData[pass].name);
+      }
+
+  if (html_catch == 0) search(dirname, wildname);
+		  else scan_catch(dirname, wildname);
+
+  for (pass = 0; pass < MAX_PASSES; pass++)
+    if (Passes & (1 << pass))
+      {
+	if (PassData[pass].done (file_name) == COMPERROR)
+	  FatalError (1, "Error during done of \"%s\"", PassData[pass].name);
+      }
+
+
+  free (buffer);
+}
+
+void main (int argc, char **argv)
+{
+  int ch;
+  char *filename = NULL;
+  FILE *Trace = NULL;
+
+  msg_prefix = argv[0];
+
+  opterr = 0;
+  while ((ch = getopt (argc, argv, "hC:WHGpSD123f:d:b:T:I:t:m:N:c:n:s:")) != -1)
+    {
+      switch (ch)
+	{
+	case 'H':
+	  html_catch = 1;
+	  break;
+	case 'G':
+	  SkipSGML = 1;
+	  break;
+	case 'p':
+	  by_para = 1;
+	  break;
+	case 'S':
+	  Passes |= SPECIAL;
+	  break;
+	case '1':
+	  InvfLevel = 1;
+	  break;
+	case '2':
+	  InvfLevel = 2;
+	  break;
+	case '3':
+	  InvfLevel = 3;
+	  break;
+	case 'f':
+	  filename = optarg;
+	  break;
+	case 'n':
+	  trace_name = optarg;
+	  break;
+	case 'D':
+	  Dump = 1;
+	  break;
+	case 'W':
+	  MakeWeights = 1;
+	  break;
+	case 'd':
+	  set_basepath (optarg);
+	  break;
+	case 's':
+	  stem_method = atoi (optarg) & STEMMER_MASK;
+	  break;
+	case 'b':
+	  buf_size = atoi (optarg) * 1024;
+	  break;
+	case 'C':
+	  comp_stat_point = atoi (optarg) * 1024;
+	  break;
+	case 'c':
+	  ChunkLimit = atoi (optarg);
+	  break;
+	case 'm':
+	  invf_buffer_size = (int) (atof (optarg) * 1024 * 1024);
+	  break;
+	case 'I':
+	case 'N': /* N kept for compatability */
+	  if (*optarg == '1')
+	    Passes |= IVF_PASS_1;
+	  else if (*optarg == '2')
+	    Passes |= IVF_PASS_2;
+	  else
+	    usage ("Invalid pass number");
+	  break;
+	case 'T':
+	  if (*optarg == '1')
+	    Passes |= TEXT_PASS_1;
+	  else if (*optarg == '2')
+	    Passes |= TEXT_PASS_2;
+	  else
+	    usage ("Invalid pass number");
+	  break;
+	case 't':
+	  trace = (unsigned long) (atof (optarg) * 1024 * 1024);
+	  break;
+	case 'h':
+	case '?':
+	  usage (NULL);
+	}
+    }
+
+  if (!filename || *filename == '\0')
+    FatalError (1, "A document collection name must be specified.");
+
+  if (buf_size < MIN_BUF)
+    FatalError (1, "The buffer size must exceed 1024 bytes.");
+
+  if ((Passes & (IVF_PASS_1 | IVF_PASS_2)) == (IVF_PASS_1 | IVF_PASS_2))
+    FatalError (1, "I1 and I2 cannot be done simultaneously.");
+
+  if ((Passes & (TEXT_PASS_1 | TEXT_PASS_2)) == (TEXT_PASS_1 | TEXT_PASS_2))
+    FatalError (1, "T1 and T2 cannot be done simultaneously.");
+
+  if (!Passes)
+    FatalError (1, "S, T1, T2, I1 or I2 must be specified.");
+
+  if (argc - optind == 1) {
+    strcpy(dirname,"");
+    strcpy(wildname,argv[optind]);
+    }
+  else if (argc - optind == 2) {
+    strcpy(dirname,argv[optind]);
+    strcpy(wildname,argv[optind+1]);
+    }
+  else FatalError(1, "Finder code requires directory and filespec.");
+
+  if (strrchr(wildname,'*') != NULL  ||  strrchr(wildname,'?') != NULL)
+    recurse = 1;
+
+  if (trace)
+    {
+      if (!trace_name)
+	trace_name = make_name (filename, TRACE_SUFFIX, NULL);
+      if (!(Trace = fopen (trace_name, "a")))
+	Message ("Unable to open \"%s\". No tracing will be done.", trace_name);
+      else
+	setbuf (Trace, NULL);
+    }
+  else
+    Trace = NULL;
+
+  if (comp_stat_point)
+    {
+      char *name = make_name (filename, COMPRESSION_STATS_SUFFIX, NULL);
+      if (!(Comp_Stats = fopen (name, "wb")))
+	Message ("Unable to open \"%s\". No comp. stats. will be generated.",
+		 name);
+    }
+
+
+  if (Trace)
+    {
+      int i;
+      fprintf (Trace, "\n\n\t\t-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\n\n");
+      for (i = 0; i < argc; i++)
+	fprintf (Trace, "%s ", argv[i]);
+      fprintf (Trace, "\n\n");
+    }
+
+  driver (Trace, filename);
+
+  if (Trace)
+    fclose (Trace);
+
+  if (Comp_Stats)
+    fclose (Comp_Stats);
+
+  exit (0);
+}
Index: /tags/gsdl-21/gsdl/src/mgpp/text/Makefile
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/Makefile	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/Makefile	(revision 699)
@@ -0,0 +1,423 @@
+# Generated automatically from Makefile.in by configure.
+###########################################################################
+#
+# Makefile.in -- Makefile for the MG system
+# Copyright (C) 1994  Neil Sharman;  1995 Tim Shimmin
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)Makefile.in	1.10 22 Mar 1994
+#
+###########################################################################
+
+PACKAGE = mgpp
+VERSION = 1.3
+ 
+SHELL = /bin/sh
+exec_prefix = ${prefix}
+prefix = /usr/local
+srcdir = .
+subdir = src/text
+top_srcdir = ..
+transform = s,x,x,
+ 
+AR = ar
+CC = gcc
+CPP = c++
+ifneq ($(NZDL),1)
+CFLAGS = -g -O2 -ansi
+CPPFLAGS =  -ansi
+else
+CFLAGS = -g -O2 -ansi -DNZDL -DQUIET -DSHORT_SUFFIX -DPARADOCNUM
+CPPFLAGS =  -ansi -DNZDL -DQUIET -DSHORT_SUFFIX -DPARADOCNUM
+endif
+DEFS = -DHAVE_CONFIG_H
+RANLIB = ranlib
+INSTALL = /usr/bin/ginstall -c
+INSTALL_DATA = ${INSTALL} -m 644
+INSTALL_PROGRAM = ${INSTALL}
+LDFLAGS = 
+LIBS = ../lib/libmg.a -lm 
+TOUCH = echo timestamp >
+AWK = gawk
+
+##########################################
+# George Buchanan - August 1999
+# 
+# The below section is now irrelevant - 
+# see the ifndef on CFLAGS above; simply
+# compiling with make NZDL=1 will auto-
+# matically ensure that the NZDL compile
+# mode is used
+#
+##########################################
+#
+# Uncomment for compiling for the nzdl project
+#
+#CFLAGS = -g -O2 -DNZDL -DQUIET -DSHORT_SUFFIX -DPARADOCNUM
+##########################################
+
+ 
+bindir = $(exec_prefix)/bin
+datadir = $(prefix)/share
+libexecdir = $(exec_prefix)/libexec
+localedir = $(datadir)/locale
+mandir = $(prefix)/man/man1
+ 
+COMPILE = $(CC) -c $(DEFS) $(TREC_MODE) $(READLINE) $(INCLUDES) $(CPPFLAGS) $(CFLAGS)
+CPPCOMPILE = $(CPP) -c $(DEFS) $(TREC_MODE) $(READLINE) $(INCLUDES) $(CPPFLAGS) $(CFLAGS)
+LINK = $(CC) $(LDFLAGS) -o $@
+
+.SUFFIXES:
+.SUFFIXES: .c .o
+.c.o:
+	$(COMPILE) $<
+
+.SUFFIXES: .cpp .o
+.cpp.o:
+	$(CPPCOMPILE) -D__cplusplus $<
+ 
+
+ANSI2KNR = 
+o = .o
+ 
+.SUFFIXES: ._c ._o
+.c._c:
+	./ansi2knr $< > $*.tmp && mv $*.tmp $@
+._c._o:
+	@echo $(COMPILE) $<
+	@rm -f _$*.c
+	@ln $< _$*.c && $(COMPILE) _$*.c && mv _$*.o $@ && rm _$*.c
+.c._o:
+	./ansi2knr $< > $*.tmp && mv $*.tmp $*._c
+	@echo $(COMPILE) $*._c
+	@rm -f _$*.c
+	@ln $*._c _$*.c && $(COMPILE) _$*.c && mv _$*.o $@ && rm _$*.c
+ 
+INCLUDES = -I../ -I../lib -I.
+
+#############################################################################
+#
+# Uncomment (and possibly edit) the following lines if you wish
+# to use the GNU readline library for the input in mgquery
+# 
+#READLINE = -DGNU_READLINE -I/opt/gnu/include/readline -L/opt/gnu/lib
+#READLINE_LIBS = -lreadline -ltermcap
+#
+#
+#############################################################################
+
+#############################################################################
+#
+# When working with the trec collection this should be uncommented.
+#
+# TREC_MODE = -DTREC_MODE
+#
+#
+#############################################################################
+
+SOURCES = \
+  backend.cpp             DocEntry.cpp \
+  locallib.cpp            mg_weights_build.cpp   \
+  bool_optimiser.cpp      mg.special.cpp          mgdictlist.cpp         \
+  bool_parser.cpp         mg_compression_dict.cpp mgquery.cpp            \
+  bool_query.cpp          mg_errors.cpp           mgstat.cpp             \
+  bool_tester.cpp         mg_fast_comp_dict.cpp   mg_decompress_text.cpp \
+  bool_tree.cpp           mg_files.cpp            query.docnums.cpp      \
+  commands.cpp            mg_hilite_words.cpp     query.ranked.cpp       \
+  comp_dict.cpp           mg_invf_dict.cpp        read_line.cpp          \
+  environment.cpp         mg_invf_dump.cpp        stem_search.cpp        \
+  mg_invf_merge.cpp       stemmer.cpp             mg_stem_idx.cpp        \
+  mg_invf_rebuild.cpp     term_lists.cpp          mgstemidxlist.cpp      \
+  invf_get.cpp            mg_passes.cpp           text.pass1.cpp         \
+  ivf.pass1.cpp           mg_perf_hash_build.cpp  text.pass2.cpp         \
+  ivf.pass2.cpp           mg_text_estimate.cpp    text_get.cpp           \
+  lists.cpp               mg_text_merge.cpp       weights.cpp            \
+  query_term_list.cpp     words.cpp               StemBlock.cpp          \
+  StemIdx.cpp             invf.cpp
+
+HEADERS = \
+  backend.h          DocEntry.h \
+  conditions.h       locallib.h         term_lists.h \
+  bool_optimiser.h   environment.h      mg.h               text.h       \
+  bool_parser.h      globals.h          mg_errors.h        text_get.h   \
+  bool_query.h       hash.h             mg_files.h         warranty.h   \
+  bool_tree.h        help.mg.h          mg_merge.h         weights.h    \
+  build.h            invf.h             read_line.h        words.h      \
+  commands.h         invf_get.h         stem_search.h      StemIdx.h    \
+  comp_dict.h        lists.h            stemmer.h          StemBlock.h  \
+  query_term_list.h  WordData.h
+
+
+MISC          = Makefile.in help.mg.src bool_parser.ypp warranty.src conditions.src
+
+ALLFILES      =	$(SOURCES) $(HEADERS) $(MISC) $(MAN)
+
+DISTFILES     = $(ALLFILES) 
+
+EXEC          = mgquery gbquery mg_weights_build mgstat mg_invf_dump mg_invf_dict \
+                mg_invf_rebuild mgdictlist mg_passes mg_perf_hash_build \
+                mg_compression_dict mg_text_estimate mg_fast_comp_dict \
+                mg_hilite_words mg_invf_merge mg_text_merge bool_tester \
+                mg_stem_idx mgstemidxlist mg_decompress_text
+
+MAN           = mgquery.1 mg_weights_build.1 mgstat.1 mg_invf_dump.1 mg_invf_dict.1 \
+                mg_invf_rebuild.1 mgdictlist.1 mg_passes.1 mg_perf_hash_build.1 \
+                mg_compression_dict.1 mg_text_estimate.1 mg_fast_comp_dict.1 \
+                mg_hilite_words.1 mg_invf_merge.1 mg_text_merge.1 \
+                mg_stem_idx.1 mgstemidxlist.1 mg_decompress_text.1 
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+
+all:	$(EXEC) libtextin.a
+
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+help.mg.h : help.mg.src
+	sed -e 's/\\/\\\\/g' -e 's/\"/\\\"/g' help.mg.src | $(AWK) '/^#/ { next; };{ if (x == 0) printf("static char *help_str[] = {\n\"%s\\n\"", $$0); else printf(",\n\"%s\\n\"", $$0); x = 1; }; END { printf("};\n"); }' - >help.mg.h
+
+warranty.h : warranty.src
+	sed  -e 's/\\/\\\\/g' -e 's/\"/\\\"/g' warranty.src | $(AWK) '/^#/ { next; };{ if (x == 0) printf("static char *warranty_str[] = {\n\"%s\\n\"", $$0); else printf(",\n\"%s\\n\"", $$0); x = 1; }; END { printf("};\n"); }' - >warranty.h
+
+conditions.h : conditions.src
+	sed -e 's/\\/\\\\/g' -e 's/\"/\\\"/g' conditions.src | $(AWK) '/^#/ { next; };{ if (x == 0) printf("static char *cond_str[] = {\n\"%s\\n\"", $$0); else printf(",\n\"%s\\n\"", $$0); x = 1; }; END { printf("};\n"); }' - >conditions.h
+
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+HILITE_OBJS = mg_hilite_words$o stemmer$o locallib$o words$o
+
+mg_hilite_words: $(HILITE_OBJS)
+	$(LINK) $(HILITE_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+bool_parser.cpp: bool_parser.ypp
+	$(YACC) $?
+	mv y.tab.c $@
+
+BOOL_OBJS = bool_tree$o bool_parser$o bool_optimiser$o bool_tester$o \
+            term_lists$o stemmer$o stem_search$o StemBlock$o mg_errors$o \
+	    query_term_list$o words$o StemIdx$o
+
+bool_tester: $(BOOL_OBJS)
+	$(LINK) $(BOOL_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+QUERY_OBJS = \
+	mgquery$o locallib$o lists$o \
+	query.ranked$o query.docnums$o stem_search$o StemBlock$o \
+	environment$o commands$o weights$o text_get$o stemmer$o \
+	read_line$o mg_errors$o backend$o DocEntry$o invf_get$o \
+	term_lists$o bool_tree$o bool_parser$o bool_optimiser$o bool_query$o \
+	query_term_list$o words$o StemIdx$o
+
+mgquery: $(QUERY_OBJS)	
+	$(LINK) $(READLINE) $(QUERY_OBJS) $(READLINE_LIBS) $(LIBS)
+
+commands$o : help.mg.h warranty.h conditions.h
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+GBQUERY_OBJS = \
+	grbIndex$o locallib$o lists$o \
+	query.ranked$o query.docnums$o stem_search$o StemBlock$o \
+	environment$o commands$o weights$o text_get$o stemmer$o \
+	read_line$o mg_errors$o backend$o DocEntry$o invf_get$o \
+	term_lists$o bool_tree$o bool_parser$o bool_optimiser$o bool_query$o \
+	query_term_list$o words$o StemIdx$o
+
+gbquery: $(GBQUERY_OBJS)	
+	$(LINK) $(READLINE) $(GBQUERY_OBJS) $(READLINE_LIBS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+PASSES_OBJS = \
+	mg_passes$o text.pass1$o comp_dict$o stemmer$o \
+	text.pass2$o locallib$o \
+	ivf.pass1$o ivf.pass2$o mg.special$o mg_files$o \
+	words$o invf$o
+
+mg_passes: $(PASSES_OBJS)
+	$(LINK) $(PASSES_OBJS) $(LIBS) 
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+WEIGHTS_BUILD_OBJS = mg_weights_build$o mg_files$o invf$o
+
+mg_weights_build: $(WEIGHTS_BUILD_OBJS)
+	$(LINK) $(WEIGHTS_BUILD_OBJS) $(LIBS)
+
+COMP_DICT_OBJS = mg_compression_dict$o mg_files$o locallib$o
+
+mg_compression_dict: $(COMP_DICT_OBJS)
+	$(LINK) $(COMP_DICT_OBJS) $(LIBS)
+
+FAST_COMP_OBJS = mg_fast_comp_dict$o mg_files$o locallib$o
+
+mg_fast_comp_dict: $(FAST_COMP_OBJS)
+	$(LINK) $(FAST_COMP_OBJS) $(LIBS)
+
+TEXT_EST_OBJS = mg_text_estimate$o locallib$o comp_dict$o mg_files$o
+
+mg_text_estimate: $(TEXT_EST_OBJS)
+	$(LINK) $(TEXT_EST_OBJS) $(LIBS)
+
+STAT_OBJS = mgstat$o mg_files$o locallib$o
+
+mgstat:	$(STAT_OBJS)
+	$(LINK) $(STAT_OBJS) $(LIBS)
+
+INVF_DUMP_OBJS = mg_invf_dump$o locallib$o mg_files$o invf$o
+
+mg_invf_dump: $(INVF_DUMP_OBJS)
+	$(LINK) $(INVF_DUMP_OBJS) $(LIBS)
+
+INVF_DICT_OBJS = mg_invf_dict$o mg_files$o locallib$o invf$o
+
+mg_invf_dict: $(INVF_DICT_OBJS)
+	$(LINK) $(INVF_DICT_OBJS) $(LIBS)
+
+INVF_REB_OBJS = mg_invf_rebuild$o locallib$o invf$o
+
+mg_invf_rebuild: $(INVF_REB_OBJS)
+	$(LINK) $(INVF_REB_OBJS) $(LIBS)
+
+DICTLIST_OBJS = mgdictlist$o locallib$o invf$o
+
+mgdictlist: $(DICTLIST_OBJS)
+	$(LINK) $(DICTLIST_OBJS) $(LIBS)
+
+PERF_HASH_OBJS = mg_perf_hash_build$o mg_files$o invf$o
+
+mg_perf_hash_build: $(PERF_HASH_OBJS)
+	$(LINK) $(PERF_HASH_OBJS) $(LIBS)
+
+STEM_IDX_OBJS = \
+                mg_stem_idx$o mg_files$o stemmer$o locallib$o \
+                mg_errors$o term_lists$o StemBlock$o
+
+mg_stem_idx: $(STEM_IDX_OBJS)
+	$(LINK) $(STEM_IDX_OBJS) $(LIBS)
+
+STEM_IDX_LIST_OBJS = mgstemidxlist$o mg_files$o StemIdx$o
+
+mgstemidxlist: $(STEM_IDX_LIST_OBJS)
+	$(LINK) $(STEM_IDX_LIST_OBJS) $(LIBS)
+
+DECOMPRESS_TEXT_OBJS = \
+                mg_decompress_text.o query_term_list.o mg_files.o mg_errors.o \
+                text_get.o locallib.o backend.o DocEntry.o stem_search.o \
+                StemBlock.o term_lists.o stemmer.o weights.o invf_get.o lists.o \
+		StemIdx.o
+
+mg_decompress_text: $(DECOMPRESS_TEXT_OBJS)
+	$(LINK) $(DECOMPRESS_TEXT_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+TEXT_MERGE_OBJS = mg_text_merge$o mg_files$o locallib$o
+
+mg_text_merge: $(TEXT_MERGE_OBJS)
+	$(LINK) $(TEXT_MERGE_OBJS) $(LIBS)
+
+INVF_MERGE_OBJS = mg_invf_merge$o mg_files$o locallib$o invf$o
+
+mg_invf_merge: $(INVF_MERGE_OBJS)
+	$(LINK) $(INVF_MERGE_OBJS) $(LIBS)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+ansi2knr: ansi2knr.o
+	$(LINK) ansi2knr.o $(LIBS)
+
+install: execinstall maninstall
+uninstall: execuninstall manuninstall
+
+execinstall: $(EXEC)
+	$(top_srcdir)/mkinstalldirs $(bindir); \
+	for name in $(EXEC); do \
+	  $(INSTALL_PROGRAM) $$name \
+	  $(bindir)/`echo $$name | sed '$(transform)'`; \
+	done
+
+execuninstall:
+	for name in $(EXEC); do \
+	 rm -f $(bindir)/`echo $$name | sed '$(transform)'`; \
+	done
+
+maninstall: $(MAN)
+	$(top_srcdir)/mkinstalldirs $(mandir); \
+	for f in $(MAN); do \
+	  $(INSTALL_DATA) $$f $(mandir)/$$f ;\
+	done
+
+manuninstall: $(MAN)
+	for f in $(MAN); do \
+	  rm -f $(mandir)/$$f ;\
+	done
+
+
+LIB_OBJS = \
+	query_term_list$o \
+	locallib$o lists$o \
+	query.ranked$o query.docnums$o stem_search$o StemBlock$o \
+	environment$o commands$o weights$o text_get$o stemmer$o \
+	read_line$o mg_errors$o backend$o DocEntry$o invf_get$o \
+	term_lists$o bool_tree$o bool_parser$o bool_optimiser$o \
+	bool_query$o words$o StemIdx$o
+
+libtextin.a: $(LIB_OBJS)
+	rm -f libtextin.a
+	$(AR) cru libtextin.a $(LIB_OBJS)
+	$(RANLIB) libtextin.a
+
+
+mostlyclean:
+	rm -f *$o _*.c _*.o *._c *._o core core.*
+ 
+clean: mostlyclean
+	rm -f $(EXEC)
+ 
+distclean: clean
+	rm -f ansi2knr
+	rm -f Makefile
+ 
+maintainer-clean: distclean
+	@echo "This command is intended only for maintainers to use;"
+	@echo "rebuilding the deleted files may require special tools."
+ 
+dist: $(DISTFILES)
+	for file in $(DISTFILES); do \
+	  ln $(srcdir)/$$file $(distdir) 2> /dev/null \
+	  || cp -p $(srcdir)/$$file $(distdir); \
+	done
+ 
+movclean:
+	for file in $(SOURCES); do \
+          test = $$file; \
+          echo $("test.c"##*.c); \
+        done 
+ 
+Makefile: Makefile.in ../config.status
+	cd .. && CONFIG_FILES=$(subdir)/$@ CONFIG_HEADERS= ./config.status
+ 
+# Tell versions [3.59,3.63) of GNU make not to export all variables.
+# Otherwise a system limit (for SysV at least) may be exc
+
+
Index: /tags/gsdl-21/gsdl/src/mgpp/text/StemBlock.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/StemBlock.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/StemBlock.cpp	(revision 699)
@@ -0,0 +1,172 @@
+#include "sysfuncs.h"
+#include "netorder.h"
+#include "messages.h"
+#include "memlib.h"
+#include "local_strings.h"
+
+#include "backend.h"
+#include "StemBlock.h"
+#include "mg_errors.h"
+
+/**
+ * GRB: The abort macro defined below is used to provide relatively
+ *      graceful failure when an error occurs - it calls the Recovery
+ *      label defined after the standard return from the fn to give
+ *      cascading error handling (reducing repeated code).  The number
+ *      given in any given abort() "call" decides where in the error 
+ *      handling cascade the recovery starts 
+ */
+
+#define abort(x) abortvar=x;goto Recovery;
+
+void StemBlock_ReadWordString(u_char **base, u_char *prev)
+{
+  mgString::readString(base, prev);
+  /*
+  unsigned copy, suff;
+
+  copy  = **base;
+  *base = *base + 1;
+  suff  = **base;
+  *base = *base + 1;
+
+  bcopy ((void *) *base, (void *) (prev + copy + 1), suff);
+  *prev = copy + suff;
+  *base += suff;
+  */
+}
+
+stemmed_dict *
+ReadStemDictBlk (File * stem_file, StemBlock_errMethod method)
+{
+  unsigned long i;
+  stemmed_dict *sd;
+  u_char *buffer;
+  int abortvar;
+
+  if (!(sd = (stemmed_dict *) Xmalloc (sizeof (stemmed_dict))))
+    {
+      abort(0);
+    }
+
+  sd->stem_file = stem_file;
+  sd->MemForStemDict = 0;
+
+  Fread (&sd->sdh, sizeof (sd->sdh), 1, stem_file);
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(sd->sdh.lookback);
+  NTOHUL(sd->sdh.block_size);
+  NTOHUL(sd->sdh.num_blocks);
+  NTOHUL(sd->sdh.blocks_start);
+  NTOHUL(sd->sdh.index_chars);
+  NTOHUL(sd->sdh.num_of_docs);
+  NTOHUL(sd->sdh.static_num_of_docs);
+  NTOHUL(sd->sdh.num_of_words);
+  NTOHUL(sd->sdh.stem_method);
+  NTOHUL(sd->sdh.indexed);
+
+  if (!(buffer = (u_char *) Xmalloc (sd->sdh.index_chars)))
+    {
+      abort(1);
+    }
+  sd->MemForStemDict += sd->sdh.index_chars;
+
+  if (!(sd->index = (u_char **) Xmalloc (sd->sdh.num_blocks * sizeof (*sd->index))))
+    {
+      abort(2);
+    }
+  sd->MemForStemDict += sd->sdh.num_blocks * sizeof (*sd->index);
+
+  if (!(sd->pos = (long unsigned int *) Xmalloc (sd->sdh.num_blocks * sizeof (*sd->pos))))
+    {
+      abort(3);
+    }
+  sd->MemForStemDict += sd->sdh.num_blocks * sizeof (*sd->pos);
+
+  if (!(sd->buffer = (u_char *) Xmalloc (sd->sdh.block_size * sizeof (*sd->buffer))))
+    {
+      abort(4);
+    }
+  sd->MemForStemDict += sd->sdh.block_size * sizeof (*sd->buffer);
+
+  sd->active = -1;
+
+  for (i = 0; i < sd->sdh.num_blocks; i++)
+    {
+      register u_char len;
+      sd->index[i] = buffer;
+      len = Getc (stem_file);
+      *buffer++ = len;
+      Fread (buffer, sizeof (u_char), len, stem_file);
+      buffer += len;
+      Fread (&sd->pos[i], sizeof (*sd->pos), 1, stem_file);
+      NTOHUL(sd->pos[i]); /* [RPAP - Jan 97: Endian Ordering] */
+    }
+
+  mg_errno = MG_NOERROR;
+
+  /*  fprintf (stderr, "mem for stem dict = %i\n", sd->MemForStemDict); */
+
+  return sd;
+
+ Recovery:
+  switch (abortvar)
+    {
+      case 4:
+        Xfree (sd->buffer);
+      case 3:
+        Xfree (sd->index);
+      case 2:
+        Xfree (buffer);
+      case 1:
+        Xfree (sd);
+      break;
+
+    }
+
+  switch (method)
+    {
+      case _MGErr:
+      {
+	mg_errno = MG_NOMEM;
+      }
+      break;
+
+      case _FatalErr:
+      {
+	FatalError(1, "Could not allocate memory for stemmed dict");
+      }
+      break;
+    }
+  return NULL;
+}
+
+
+void StemBlock_read(stemmed_dict *sd,
+		    int block,
+		    unsigned long **first_word,
+		    unsigned long **last_invf_len,
+		    unsigned short **num_words,
+		    unsigned short **index,
+		    int *num_indexes)
+{
+      int i;
+
+      Fseek (sd->stem_file, sd->pos[block] + sd->sdh.blocks_start, 0);
+      Fread (sd->buffer, sd->sdh.block_size, sizeof (u_char), sd->stem_file);
+      sd->active = sd->pos[block];
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      *first_word = (unsigned long *) (sd->buffer);
+      NTOHUL(**first_word);
+      *last_invf_len = (unsigned long *) ((*first_word) + 1);
+      NTOHUL(**last_invf_len);
+      *num_words = (unsigned short *) ((*last_invf_len) + 1);
+      NTOHUS(**num_words);
+      *index = (*num_words) + 1;
+      *num_indexes = (((**num_words) - 1) / sd->sdh.lookback) + 1;
+
+      for (i = 0; i < *num_indexes; i++)
+	NTOHUS((*index)[i]);
+}
+
Index: /tags/gsdl-21/gsdl/src/mgpp/text/StemBlock.h
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/StemBlock.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/StemBlock.h	(revision 699)
@@ -0,0 +1,14 @@
+#ifndef H_STEMBLOCK
+
+typedef enum
+{ _FatalErr, _MGErr
+} StemBlock_errMethod;
+
+stemmed_dict *ReadStemDictBlk (File * stem_file, StemBlock_errMethod method);
+
+void StemBlock_ReadWordString(u_char **base, u_char *prev);
+
+void StemBlock_read(stemmed_dict *sd, int block, unsigned long **first_word, unsigned long **last_invf_len,
+		    unsigned short **num_words, unsigned short **index, int *num_indexes);
+#define H_STEMBLOCK
+#endif
Index: /tags/gsdl-21/gsdl/src/mgpp/text/StemIdx.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/StemIdx.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/StemIdx.cpp	(revision 699)
@@ -0,0 +1,155 @@
+#include "sysfuncs.h"
+
+#include "messages.h"
+#include "memlib.h"
+#include "local_strings.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "mg.h"
+#include "invf.h"
+#include "words.h"
+#include "backend.h"
+#include "StemIdx.h"
+
+#define abort(x) abortvar=x;goto Recovery;
+
+stemmed_idx *StemIdx_header_read(void *f, int notefile, abstractReader reader, abstractGetc creader)
+{
+  stemmed_idx *si;
+  unsigned long i;
+  u_char *buffer;
+  int abortvar;
+
+  if (!(si = (stemmed_idx *) Xmalloc (sizeof (stemmed_idx))))
+    {
+      abort(0);
+    }
+
+  si->MemForStemIdx = 0;
+  if (notefile)
+    {
+      si->stem_idx_file = (File *) f;
+    }
+
+  reader (&(si->sih), sizeof (si->sih), 1, f);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(si->sih.lookback);
+  NTOHUL(si->sih.block_size);
+  NTOHUL(si->sih.num_blocks);
+  NTOHUL(si->sih.blocks_start);
+  NTOHUL(si->sih.index_chars);
+  NTOHUL(si->sih.num_of_words);
+
+  if (!(buffer = (u_char *) Xmalloc (si->sih.index_chars)))
+    {
+      abort(1);
+    }
+  si->MemForStemIdx += si->sih.index_chars;
+
+  if (!(si->index = (u_char **) Xmalloc (si->sih.num_blocks * sizeof (*si->index))))
+    {
+      abort(2);
+    }
+  si->MemForStemIdx += si->sih.num_blocks * sizeof (*si->index);
+
+  if (!(si->pos = (long unsigned int *) Xmalloc (si->sih.num_blocks * sizeof (*si->pos))))
+    {
+      abort(3);
+    }
+  si->MemForStemIdx += si->sih.num_blocks * sizeof (*si->pos);
+
+  if (!(si->buffer = (u_char *) Xmalloc (si->sih.block_size * sizeof (*si->buffer))))
+    {
+      abort(4);
+    }
+  si->MemForStemIdx += si->sih.block_size * sizeof (*si->buffer);
+
+  si->active = -1;
+
+  for (i = 0; i < si->sih.num_blocks; i++)
+    {
+      register u_char len;
+      si->index[i] = buffer;
+      len = creader (f);
+      *buffer++ = len;
+      reader (buffer, sizeof (u_char), len, f);
+      buffer += len;
+      reader (&si->pos[i], sizeof (*si->pos), 1, f);
+      NTOHUL(si->pos[i]);  /* [RPAP - Jan 97: Endian Ordering] */
+    }
+  return si;
+
+ Recovery:
+  switch (abortvar)
+    {
+      case 4:
+	Xfree(si->pos);
+      case 3:
+	Xfree(si->index);
+      case 2:
+	Xfree(buffer);
+      case 1:
+        Xfree(si);
+      break;
+    }
+  return NULL;
+}
+
+void StemIdx_block_readNext(void *file, stemmed_idx *si, int block,
+			    unsigned long **first_word, unsigned short **num_words,
+			    unsigned short **index, int *num_indexes, abstractSeeker seeker,
+			    abstractReader reader)
+{ 
+  int i;
+
+  seeker (file, si->pos[block] + si->sih.blocks_start, 0);
+  reader (si->buffer, si->sih.block_size, sizeof (u_char), file);
+  si->active = si->pos[block];
+
+  *first_word = (unsigned long *) (si->buffer);
+  NTOHUL(**first_word);  /* [RPAP - Jan 97: Endian Ordering] */
+  *num_words = (unsigned short *) ((*first_word) + 1);
+  NTOHUS(**num_words);  /* [RPAP - Jan 97: Endian Ordering] */
+  *index = (*num_words) + 1;
+  *num_indexes = (((**num_words) - 1) / si->sih.lookback) + 1;
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  for (i = 0; i < *num_indexes; i++)
+    NTOHUS((*index)[i]);
+}
+
+void StemIdx_ReadWordString(u_char **base, u_char *prev)
+{ 
+  mgString::readString(base, prev);
+  /*
+  unsigned copy, suff;
+
+  copy = **base;
+  *base = *base + 1;
+  suff = **base;
+  *base = *base + 1;
+
+  bcopy ((char *) *base, (char *) (prev + copy + 1), suff);
+  *prev = copy + suff;
+  *base += suff;
+  */
+}
+
+void StemIdx_ReadPosEntry(u_char **base, unsigned int *num_cases, unsigned int *blk, 
+			  unsigned short *blk_index, unsigned short *offset)
+{
+  bcopy ((char *) *base, (char *) num_cases, sizeof (*num_cases));
+  NTOHUI(*num_cases);  /* [RPAP - Jan 97: Endian Ordering] */
+  *base += sizeof (*num_cases);
+  bcopy ((char *) *base, (char *) blk, sizeof (*blk));
+  NTOHUI(*blk);  /* [RPAP - Jan 97: Endian Ordering] */
+  *base += sizeof (*blk);
+  bcopy ((char *) *base, (char *) blk_index, sizeof (*blk_index));
+  NTOHUS(*blk_index);  /* [RPAP - Jan 97: Endian Ordering] */
+  *base += sizeof (*blk_index);
+  bcopy ((char *) *base, (char *) offset, sizeof (*offset));
+  NTOHUS(*offset);  /* [RPAP - Jan 97: Endian Ordering] */
+  *base += sizeof (*offset);
+}
Index: /tags/gsdl-21/gsdl/src/mgpp/text/StemIdx.h
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/StemIdx.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/StemIdx.h	(revision 699)
@@ -0,0 +1,14 @@
+#ifndef H_STEMIDX
+#define H_STEMIDX 1
+typedef size_t (* abstractReader) (void *ptr, size_t size, size_t nitems, void *file);
+typedef char (* abstractGetc) (void *file);
+typedef int (* abstractSeeker) (void *file, long offset, int ptrname);
+
+stemmed_idx *StemIdx_header_read(void *f, int notefile, abstractReader reader, abstractGetc creader);
+void StemIdx_block_readNext(void *f, stemmed_idx *s, int block, unsigned long **first_word, 
+			    unsigned short **num_words,  unsigned short **index, int *num_indexes, 
+			    abstractSeeker seeker, abstractReader reader);
+void StemIdx_ReadWordString(u_char **base, u_char *prev);
+void StemIdx_ReadPosEntry(u_char **base, unsigned int *num_cases, unsigned int *blk, 
+			  unsigned short *blk_index, unsigned short *offset);
+#endif
Index: /tags/gsdl-21/gsdl/src/mgpp/text/backend.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/backend.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/backend.cpp	(revision 699)
@@ -0,0 +1,938 @@
+/**************************************************************************
+ *
+ * backend.c -- Underlying routines for mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "timing.h"
+#include "filestats.h"
+#include "sptree.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+
+#include "mg_files.h"
+#include "mg.h"
+#include "invf.h"
+#include "text.h"
+#include "lists.h"
+#include "backend.h"
+#include "stem_search.h"
+#include "StemBlock.h"
+#include "invf_get.h"
+#include "text_get.h"
+#include "weights.h"
+#include "locallib.h"
+#include "mg_errors.h"
+#include "DocEntry.h"
+
+static File *
+OpenFile (char *base, char *suffix, unsigned long magic, int *ok)
+{
+  char FileName[512];
+  File *F;
+  sprintf (FileName, "%s%s", base, suffix);
+  if (!(F = Fopen (FileName, "rb", 0)))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      mg_errno = MG_NOFILE;
+      MgErrorData (FileName);
+      if (ok)
+	*ok = 0;
+      return (NULL);
+    }
+  if (magic)
+    {
+      unsigned long m;
+      if (fread ((char *) &m, sizeof (m), 1, F->f) == 0)
+	{
+	  mg_errno = MG_READERR;
+	  MgErrorData (FileName);
+	  if (ok)
+	    *ok = 0;
+	  Fclose (F);
+	  return (NULL);
+	}
+      NTOHUL(m);  /* [RPAP - Jan 97: Endian Ordering] */
+      if (m != magic)
+	{
+	  mg_errno = MG_BADMAGIC;
+	  MgErrorData (FileName);
+	  if (ok)
+	    *ok = 0;
+	  Fclose (F);
+	  return (NULL);
+	}
+    }
+  return (F);
+}
+
+/* --
+   -- GRB: Open text files only; based on open_all_files in MG1.2.1; 
+   --      should only be called after open_index_files
+   --
+*/
+static int 
+open_text_files (query_data * qd)
+{
+  int ok = 1;
+
+  qd->File_text = OpenFile (qd->textpathname, TEXT_SUFFIX, /* [RJM 06/97: text filename] */
+			    MAGIC_TEXT, &ok);
+  qd->File_fast_comp_dict = OpenFile (qd->textpathname, /* [RJM 06/97: text filename] */
+				      TEXT_DICT_FAST_SUFFIX, MAGIC_FAST_DICT, NULL);
+  if (!qd->File_fast_comp_dict)
+    {
+      qd->File_comp_dict = OpenFile (qd->textpathname, /* [RJM 06/97: text filename] */
+				     TEXT_DICT_SUFFIX, MAGIC_DICT, &ok);
+      qd->File_aux_dict = OpenFile (qd->textpathname, /* [RJM 06/97: text filename] */
+				    TEXT_DICT_AUX_SUFFIX, MAGIC_AUX_DICT, NULL);
+    }
+  else
+    qd->File_comp_dict = qd->File_aux_dict = NULL;
+
+  /* This will fail if a level 1 inverted file was created because there 
+     will be no document weights */
+  qd->File_text_idx_wgt = OpenFile (qd->pathname, TEXT_IDX_WGT_SUFFIX,
+				    MAGIC_TEXI_WGT, NULL);
+  if (qd->File_text_idx_wgt == NULL && qd->File_weight_approx == NULL)
+    qd->File_text_idx = OpenFile (qd->textpathname, /* [RJM 06/97: text filename] */
+				  TEXT_IDX_SUFFIX, MAGIC_TEXI, NULL);
+  else
+    qd->File_text_idx = NULL;
+
+  /**
+   * GRB: Additional check; if we didn't get idx_wgt and we are on
+   *        a level 2 or later inversion fileset, then error.
+   *      As the id structure may not have been initialised, only
+   *        error if it has been; we'll assume that if it isn't
+   *        initialised yet then something bespoke is happening and
+   *        the programmer should deal with it; 
+   *      open_all_files() calls this fn after open_index_files,
+   *        in which case id is NULL if when open_all_files is called
+   *        inside __InitQuerySystem (ie. when MG is just run as normal).
+   *      Programmers doing their own thing should prepare their
+   *        code to cope with the 2-stage initialisation of level 2
+   *        or higher indexes.  The idx_wgt index is only used in
+   *        retrieving text; it has no effect on indexed searches.  
+   */
+  if (qd->id != NULL &&
+      ((qd->allfiles == 1 && qd->File_text_idx_wgt == NULL) || 
+       qd->File_weight_approx == NULL) &&
+      qd->id->ifh.InvfLevel >= 2)
+    { ok = 0;
+    }
+
+
+  if (!ok)
+    { if (qd->File_text)
+        Fclose (qd->File_text);
+      if (qd->File_fast_comp_dict)
+	Fclose (qd->File_fast_comp_dict);
+      if (qd->File_aux_dict)
+	Fclose (qd->File_aux_dict);
+      if (qd->File_comp_dict)
+	Fclose (qd->File_comp_dict);
+
+      if (qd->File_text_idx_wgt)
+	Fclose (qd->File_text_idx_wgt);
+      if (qd->File_text_idx)
+	Fclose (qd->File_text_idx);
+      return (-1);
+    }
+  return (0);
+
+}
+
+/* --
+   -- GRB: Open index files only; based on open_all_files in MG1.2.1
+   --
+*/
+static int open_index_files(query_data *qd)
+{
+  int ok = 1;
+
+  qd->File_text           = NULL;
+  qd->File_fast_comp_dict = NULL;
+  qd->File_comp_dict      = NULL;
+  qd->File_aux_dict       = NULL;
+
+  qd->File_stem = OpenFile (qd->pathname, INVF_DICT_BLOCKED_SUFFIX,
+			    MAGIC_STEM, &ok);
+
+  /* [RPAP - Jan 97: Stem Index Change]
+     These will fail if collection not built with stem indexes */
+  qd->File_stem1 = OpenFile (qd->pathname, INVF_DICT_BLOCKED_1_SUFFIX,
+			     MAGIC_STEM_1, NULL);
+  qd->File_stem2 = OpenFile (qd->pathname, INVF_DICT_BLOCKED_2_SUFFIX,
+			     MAGIC_STEM_2, NULL);
+  qd->File_stem3 = OpenFile (qd->pathname, INVF_DICT_BLOCKED_3_SUFFIX,
+			     MAGIC_STEM_3, NULL);
+
+  qd->File_invf = OpenFile (qd->pathname, INVF_SUFFIX,
+			    MAGIC_INVF, &ok);
+
+  /* Ths will fail if a level 1 inverted file was created because there 
+     will be no document weights */
+  qd->File_weight_approx = OpenFile (qd->pathname, APPROX_WEIGHTS_SUFFIX,
+				     MAGIC_WGHT_APPROX, NULL);
+
+  qd->File_text_idx_wgt = NULL;
+  qd->File_text_idx = NULL;
+
+  if (!ok)
+    { Fclose (qd->File_stem);
+
+      /* [RPAP - Jan 97: Stem Index Change] */
+      if (qd->File_stem1)
+	Fclose (qd->File_stem1);
+      if (qd->File_stem2)
+	Fclose (qd->File_stem2);
+      if (qd->File_stem3)
+	Fclose (qd->File_stem3);
+
+      Fclose (qd->File_invf);
+      if (qd->File_weight_approx)
+	Fclose (qd->File_weight_approx);
+      return (-1);
+    }
+  return (0);
+
+}
+
+static int 
+open_all_files (query_data * qd)
+{
+  int ok = 1;
+
+  if (open_index_files(qd) != 0)
+    {
+      return -1;
+    }
+  if (open_text_files(qd) != 0)
+    {
+      return -1;
+    }
+  return 0;
+}
+
+static void 
+close_all_files (query_data * qd)
+{ if (qd->File_text)
+    Fclose (qd->File_text);
+  if (qd->File_fast_comp_dict)
+    Fclose (qd->File_fast_comp_dict);
+  if (qd->File_aux_dict)
+    Fclose (qd->File_aux_dict);
+  if (qd->File_comp_dict)
+    Fclose (qd->File_comp_dict);
+  Fclose (qd->File_stem);
+
+  /* [RPAP - Jan 97: Stem Index Change] */
+  if (qd->File_stem1)
+    Fclose (qd->File_stem1);
+  if (qd->File_stem2)
+    Fclose (qd->File_stem2);
+  if (qd->File_stem3)
+    Fclose (qd->File_stem3);
+
+  Fclose (qd->File_invf);
+  if (qd->File_text_idx_wgt)
+    Fclose (qd->File_text_idx_wgt);
+  if (qd->File_weight_approx)
+    Fclose (qd->File_weight_approx);
+  if (qd->File_text_idx)
+    Fclose (qd->File_text_idx);
+}
+
+/* If textname is equal to null then name will be used instead */
+/* [RJM 06/97: text filename] */
+
+/**
+ * The following define and the variable abortvar in the following function are designed
+ * to provide an exception-like recovery system for when an error occurs.
+ */
+#define abort(x) abortvar=x;goto Recovery;
+
+query_data *
+__InitQuerySystem (char *dir, char *name, char *textname, InitQueryTimes * iqt, int allfiles)
+{
+  query_data *qd;
+  char *s;
+  int abortvar;
+
+  if (textname == NULL) textname = name; /* [RJM 06/97: text filename] */
+
+  if (!(qd = (query_data *) Xmalloc (sizeof (query_data))))
+    {
+      abort(0);
+    }
+
+  bzero ((char *) qd, sizeof (*qd));
+
+  /* -- GRB: 13/09/99: note if all files were requested */
+  qd->allfiles = allfiles;
+
+  qd->mem_in_use = qd->max_mem_in_use = 0;
+
+  qd->doc_pos = qd->buf_in_use = 0;
+  qd->TextBufferLen = 0;
+  qd->DL = NULL;
+
+  /* [RPAP - Feb 97: Term Frequency] */
+  qd->TL = NULL;
+  qd->QTL = NULL;
+
+  qd->TextBuffer = NULL;
+
+  qd->tot_hops_taken = 0;
+  qd->tot_num_of_ptrs = 0;
+  qd->tot_num_of_accum = 0;
+  qd->tot_num_of_terms = 0;
+  qd->tot_num_of_ans = 0;
+  qd->tot_text_idx_lookups = 0;
+
+  qd->hops_taken = 0;
+  qd->num_of_ptrs = 0;
+  qd->num_of_accum = 0;
+  qd->num_of_terms = 0;
+  qd->num_of_ans = 0;
+  qd->text_idx_lookups = 0;
+
+  qd->pathname = NULL; /* RJM 06/97: text filename] */
+  qd->textpathname = NULL; /* RJM 06/97: text filename] */
+
+  s = strrchr (dir, '/');
+  if (s && *(s + 1) == '\0')
+    {
+      /* [RJM 06/97: text filename] */
+      if (!(qd->pathname = (char *) Xmalloc (strlen (dir) + strlen (name) + 1)) ||
+	  !(qd->textpathname = (char *) Xmalloc (strlen (dir) + strlen (textname) + 1)))
+	{
+	  abort(1);
+	}
+      sprintf (qd->pathname, "%s%s", dir, name);
+      sprintf (qd->textpathname, "%s%s", dir, textname); /* [RJM 06/97: text filename] */
+    }
+
+  else
+    {
+      /* [RJM 06/97: text filename] */
+      if (!(qd->pathname = (char *) Xmalloc (strlen (dir) + strlen (name) + 2)) ||
+	  !(qd->textpathname = (char *) Xmalloc (strlen (dir) + strlen (textname) + 2)))
+	{
+	  abort(1);
+	}
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+      if (dir == NULL || dir[0] == '\0') {
+	sprintf (qd->pathname, "%s", name);
+	sprintf (qd->textpathname, "%s", textname); /* [RJM 06/97: text filename] */
+      } else {
+	sprintf (qd->pathname, "%s%s", dir, name);
+	sprintf (qd->textpathname, "%s%s", dir, textname); /* [RJM 06/97: text filename] */
+      }
+#else
+      sprintf (qd->pathname, "%s/%s", dir, name);
+      sprintf (qd->textpathname, "%s/%s", dir, textname); /* [RJM 06/97: text filename] */
+#endif
+    }
+
+  if (((allfiles == 1) && (open_all_files (qd) == -1)) ||
+      ((allfiles == 0) && (open_index_files (qd) == -1)))
+    {
+      abort(2);
+      return (NULL);
+    }
+
+  if (iqt)
+    GetTime (&iqt->Start);
+
+  /* Initialise the stemmed dictionary system */
+  if (!(qd->sd = ReadStemDictBlk (qd->File_stem, _MGErr)))
+    {
+      abort(3);
+    }
+
+  /* [RPAP - Jan 97: Stem Index Change] */
+  if ((qd->sd->sdh.indexed & 7) && qd->File_stem1 && qd->File_stem2 && qd->File_stem3)
+    {
+      if (!(qd->sd->stem1 = ReadStemIdxBlk (qd->File_stem1)))
+	{
+	  abort(4);
+	}
+      if (!(qd->sd->stem2 = ReadStemIdxBlk (qd->File_stem2)))
+	{
+	  abort(4);
+	}
+      if (!(qd->sd->stem3 = ReadStemIdxBlk (qd->File_stem3)))
+	{
+	  abort(4);
+	}
+      }
+  else if (qd->sd->sdh.indexed != 0)
+    {
+      abort(4);
+    }
+  else
+    {
+      if (qd->File_stem1)
+	Fclose (qd->File_stem1);
+      if (qd->File_stem2)
+	Fclose (qd->File_stem2);
+      if (qd->File_stem3)
+	Fclose (qd->File_stem3);
+      qd->File_stem1 = NULL;
+      qd->File_stem2 = NULL;
+      qd->File_stem3 = NULL;
+      qd->sd->stem1 = NULL;
+      qd->sd->stem2 = NULL;
+      qd->sd->stem3 = NULL;
+    }
+
+  if (iqt)
+    GetTime (&iqt->StemDict);
+  if (qd->File_weight_approx)
+    {
+      if (!(qd->awd = LoadDocWeights (qd->File_weight_approx,
+				      qd->sd->sdh.num_of_docs)))
+	{
+	  abort(4);
+	}
+    }
+  else
+    qd->awd = NULL;
+
+
+  if (iqt)
+    GetTime (&iqt->ApproxWeights);
+
+  if ((allfiles == 1) &&
+      !(qd->cd = LoadCompDict (qd->File_comp_dict, qd->File_aux_dict,
+			       qd->File_fast_comp_dict)))
+    {
+      abort(5);
+    }
+  else if (allfiles == 0)
+    {
+      qd->cd = NULL;
+    }
+
+  if (iqt)
+    GetTime (&iqt->CompDict);
+
+  if (!(qd->id = InitInvfFile (qd->File_invf, qd->sd)))
+    {
+      abort(6);
+    }
+  if (((allfiles == 1 && qd->File_text_idx_wgt == NULL) || 
+       qd->File_weight_approx == NULL) &&
+      qd->id->ifh.InvfLevel >= 2)
+    {
+      abort(7);
+    }
+  if (iqt)
+    GetTime (&iqt->Invf);
+
+  if ((allfiles == 1) &&
+      !(qd->td = LoadTextData (qd->File_text, qd->File_text_idx_wgt,
+			       qd->File_text_idx)))
+    {
+      abort(8);
+    }
+  else if (allfiles == 0)
+    {
+      qd->td = NULL;
+    }
+
+/* [RPAP - Feb 97: NZDL Additions] */
+#if defined(PARADOCNUM) || defined(NZDL)
+
+/*
+
+This code is based on the TREC_MODE code below to read the .paragraph
+file to determine what document numbers correspond to what paragraphs.
+This code is more space efficient, reading in the .paragraph file
+into memory as an accumulate docnum array.  Eg.  the .paragraph may contain
+
+	[5 3 6 4 7 9 4]
+
+indicating the first document has 5 paragraphs, the next 3, etc.
+This will be stored in memory as
+
+	[0 5 8 14 18 25 34 38]
+
+so a binary search can be performed.  The first 0 is for convenience;
+it prevents testing boundary conditions.
+
+
+The TREC_MODE code does this differently; it stores the array
+
+	[1 1 1 1 1 2 2 2 3 3 3 3 3 3 ....]
+
+allowing directy paragraph to docnum conversion, at the expense
+of memory.
+
+*/ 
+  qd->paragraph = NULL;
+
+  if ((allfiles == 1) && (qd->id->ifh.InvfLevel == 3))
+    {
+      unsigned long magic;
+      FILE *paragraph;
+      int i;
+      char paraFile[512];
+
+      sprintf(paraFile, "%s%s", qd->pathname, INVF_PARAGRAPH_SUFFIX);
+      paragraph = fopen(paraFile, "rb");
+      if (!paragraph)
+	FatalError(1, "Unable to open 'paraFile'.", paraFile);
+
+      fread((void *)&magic, sizeof(magic), 1, paragraph);
+      qd->paragraph = Xmalloc((qd->td->cth.num_of_docs+1)*sizeof(int));
+      qd->paragraph[0] = 0;
+      for (i = 1; i <= qd->td->cth.num_of_docs; i++)
+	{
+	  int count;
+
+	  if (fread((void *)&count, sizeof(count), 1, paragraph) != 1)
+	    FatalError(1, "Unexpected EOF while reading '%s'.", paraFile);
+	  NTOHSI(count);  /* [RPAP - Jan 97: Endian Ordering] */
+	  qd->paragraph[i] = qd->paragraph[i-1]+count;
+	}
+
+      fclose (paragraph); /* [RJM 07/98: Memory Leak] */
+    }
+  
+  
+#endif
+
+#ifdef TREC_MODE
+  {
+    extern char *trec_ids;
+    extern long *trec_paras;
+    int size;
+    char FileName[512];
+    FILE *f;
+    if (!strstr (qd->pathname, "trec"))
+      goto error;
+    sprintf (FileName, "%s%s", qd->pathname, ".DOCIDS");
+    if (!(f = fopen (FileName, "rb")))  /* [RPAP - Feb 97: WIN32 Port] */
+      {
+	Message ("Unable to open \"%s\"", FileName);
+	goto error;
+      }
+    fseek (f, 0, 2);
+    size = ftell (f);
+    fseek (f, 0, 0);
+    trec_ids = Xmalloc (size);
+    if (!trec_ids)
+      {
+	fclose (f);
+	goto error;
+      }
+    fread (trec_ids, 1, size, f);
+    fclose (f);
+    if ((allfiles == 1) && (qd->id->ifh.InvfLevel == 3))
+      {
+	int i, d;
+	unsigned long magic;
+	trec_paras = Xmalloc (qd->sd->sdh.num_of_docs * sizeof (long));
+	if (!trec_paras)
+	  {
+	    Xfree (trec_ids);
+	    trec_ids = NULL;
+	    goto error;
+	  }
+	sprintf (FileName, "%s%s", qd->pathname, INVF_PARAGRAPH_SUFFIX);
+	if (!(f = fopen (FileName, "rb")))  /* [RPAP - Feb 97: WIN32 Port] */
+	  {
+	    Message ("Unable to open \"%s\"", FileName);
+	    goto error;
+	  }
+	if (fread ((char *) &magic, sizeof (magic), 1, f) != 1 ||
+	    NTOHUL(magic) != MAGIC_PARAGRAPH)  /* [RPAP - Jan 97: Endian Ordering] */
+	  {
+	    fclose (f);
+	    Message ("Bad magic number in \"%s\"", FileName);
+	    goto error;
+	  }
+
+	for (d = i = 0; i < qd->td->cth.num_of_docs; i++)
+	  {
+	    int count;
+	    if (fread ((char *) &count, sizeof (count), 1, f) != 1)
+	      {
+		fclose (f);
+		goto error;
+	      }
+	    NTOHSI(count);  /* [RPAP - Jan 97: Endian Ordering] */
+	    while (count--)
+	      trec_paras[d++] = i;
+	  }
+	fclose (f);
+      }
+    goto ok;
+  error:
+    if (trec_ids)
+      Xfree (trec_ids);
+    if (trec_paras)
+      Xfree (trec_paras);
+    trec_ids = NULL;
+    trec_paras = NULL;
+  ok:
+    ;
+  }
+#endif
+
+  if (iqt)
+    GetTime (&iqt->Text);
+
+  return (qd);
+
+ Recovery:
+  switch (abortvar)
+    {
+    case 8: // NB: 8 indicates a failure after the loading of the inverted file
+    case 7: // NB: 7 indicates a failure during the loading of the inverted file
+      FreeInvfData (qd->id);
+    case 6:
+      if (qd->cd)
+	FreeCompDict (qd->cd);
+    case 5:
+      if (qd->awd)
+	{
+	  FreeWeights (qd->awd);
+	}
+    case 4:
+      FreeStemDict (qd->sd);
+    case 3:
+      close_all_files (qd);
+    case 2:
+      Xfree (qd->textpathname); /* [RJM 06/97: text filename] */
+    case 1:
+      if (qd->pathname) 
+	Xfree (qd->pathname); /* [RJM 06/97: text filename] */
+      Xfree (qd);
+      break;
+    }
+
+  /* Set mg error status for particular failures; NB not every failure results in an
+   * mg error value being set.
+   */
+  if (abortvar < 2)
+    mg_errno = MG_NOMEM;
+  else if (abortvar == 7)
+    mg_errno = MG_INVERSION;
+  return (NULL);
+}
+
+
+
+query_data *
+InitQuerySystem (char *dir, char *name, char *textname, InitQueryTimes * iqt)
+{ return __InitQuerySystem(dir, name, textname, iqt, 1);
+}
+
+// This function initialises the query system without loading the text dictionary etc.
+query_data *
+InitQuerySystemNT (char *dir, char *name, char *textname, InitQueryTimes * iqt)
+{ return __InitQuerySystem(dir, name, textname, iqt, 0);
+}
+
+/*
+ * Change the amount of memory currently in use
+ *
+ */
+void 
+ChangeMemInUse (query_data * qd, long delta)
+{
+  qd->mem_in_use += delta;
+  if (qd->mem_in_use > qd->max_mem_in_use)
+    qd->max_mem_in_use = qd->mem_in_use;
+}
+
+
+void 
+FinishQuerySystem (query_data * qd)
+{
+/* [RJM 07/98: Memory Leak] */
+#if defined(PARADOCNUM) || defined(NZDL)
+  if (qd->paragraph != NULL) {
+    Xfree (qd->paragraph);
+    qd->paragraph = NULL;
+  }
+#endif
+
+  if (qd->td != NULL)
+    {
+      FreeTextData (qd->td);
+    }
+  FreeInvfData (qd->id);
+  if (qd->cd)
+    FreeCompDict (qd->cd);
+
+  if (qd->awd)
+    FreeWeights (qd->awd);
+  FreeStemDict (qd->sd);
+  close_all_files (qd);
+  Xfree (qd->textpathname); /* [RJM 06/97: text filename] */
+  Xfree (qd->pathname);
+  QueryData_FreeQueryDocs (qd);
+  if (qd->TL != NULL) TermList_destroy(&qd->TL);         /* [RJM 07/98: Memory Leak] */
+  if (qd->QTL != NULL) QueryTermList_free(&qd->QTL);  /* [RJM 07/98: Memory Leak] */
+  Xfree (qd);
+
+  /* other global stuff hanging around */
+  MgErrorDeinit ();
+}
+
+
+void 
+ResetFileStats (query_data * qd)
+{
+  if (qd->File_text)
+    ZeroFileStats (qd->File_text);
+  if (qd->File_comp_dict)
+    ZeroFileStats (qd->File_comp_dict);
+  if (qd->File_fast_comp_dict)
+    ZeroFileStats (qd->File_fast_comp_dict);
+  ZeroFileStats (qd->File_stem);
+
+  /* [RPAP - Jan 97: Stem Index Change] */
+  if (qd->File_stem1)
+    ZeroFileStats (qd->File_stem1);
+  if (qd->File_stem2)
+    ZeroFileStats (qd->File_stem2);
+  if (qd->File_stem3)
+    ZeroFileStats (qd->File_stem3);
+
+  ZeroFileStats (qd->File_invf);
+  if (qd->File_text_idx_wgt)
+    ZeroFileStats (qd->File_text_idx_wgt);
+  if (qd->File_weight_approx)
+    ZeroFileStats (qd->File_weight_approx);
+  if (qd->File_text_idx)
+    ZeroFileStats (qd->File_text_idx);
+}
+
+
+void 
+TransFileStats (query_data * qd)
+{
+  qd->File_text->Current = qd->File_text->Cumulative;
+  if (qd->File_comp_dict)
+    qd->File_comp_dict->Current = qd->File_comp_dict->Cumulative;
+  if (qd->File_fast_comp_dict)
+    qd->File_fast_comp_dict->Current = qd->File_fast_comp_dict->Cumulative;
+  qd->File_stem->Current = qd->File_stem->Cumulative;
+
+  /* [RPAP - Jan 97: Stem Index Change] */
+  if (qd->File_stem1)
+    qd->File_stem1->Current = qd->File_stem1->Cumulative;
+  if (qd->File_stem2)
+    qd->File_stem2->Current = qd->File_stem2->Cumulative;
+  if (qd->File_stem3)
+    qd->File_stem3->Current = qd->File_stem3->Cumulative;
+
+  qd->File_invf->Current = qd->File_invf->Cumulative;
+  if (qd->File_text_idx_wgt)
+    qd->File_text_idx_wgt->Current = qd->File_text_idx_wgt->Cumulative;
+  if (qd->File_weight_approx)
+    qd->File_weight_approx->Current = qd->File_weight_approx->Cumulative;
+  if (qd->File_text_idx)
+    qd->File_text_idx->Current = qd->File_text_idx->Cumulative;
+}
+
+
+/**
+ * Block: Query Document handling;
+ *        Provides basic loading/unloading facilities for individual
+ *          documents, usually for presentation purposes (searches
+ *          are performed on inverted files
+ */
+
+void 
+QueryData_FreeTextBuffer (query_data * qd)
+{
+  if (qd->TextBuffer)
+    {
+      Xfree (qd->TextBuffer);
+      ChangeMemInUse (qd, -qd->TextBufferLen);
+    }
+  qd->TextBuffer = NULL;
+  qd->TextBufferLen = 0;
+}
+
+void 
+QueryData_FreeQueryDocs (query_data * qd)
+{
+  qd->doc_pos = 0;
+  qd->buf_in_use = 0;
+  if (qd->DL)
+    {
+      //int i;
+      DocList_FreeTextBuffers(qd->DL, qd);
+      /*
+      for (i = 0; i < qd->DL->size(); i++)
+	DocEntry_FreeTextBuffer(&qd->DL->DE[i], qd);
+        
+	if (qd->DL->DE[i].CompTextBuffer)
+	  {
+	    Xfree (qd->DL->DE[i].CompTextBuffer);
+	    qd->DL->DE[i].CompTextBuffer = NULL;
+	    ChangeMemInUse (qd, -qd->DL->DE[i].Len);
+	  }
+      */
+      DocList_destroy(qd->DL);
+    }
+  qd->DL = NULL;
+  QueryData_FreeTextBuffer (qd);
+}
+
+int 
+LoadCompressedText (query_data * qd, int max_mem)
+{
+  DocEntry *DE;
+
+  if (qd->DL == NULL || qd->doc_pos >= qd->DL->size())
+    return -1;
+
+  /**
+   *  GRB: added test for td (text data) per addition of loading
+   *       indexes only; td may now be NULL; this returns a
+   *       special value to  indicated that the text was not
+   *       loaded; mgquery etc now test on that value
+   *  17/09/99
+   */
+  if (qd->td == NULL)
+    return -2;
+
+  DE = qd->DL->member(qd->doc_pos);
+
+  if (!DocEntry_TextLoaded(DE))
+    {
+      //int i;
+      //DocEntry *de;
+
+      DocList_FreeTextBuffers(qd->DL, qd);
+      /*
+      for (i = 0, de = qd->DL->DE; i < qd->DL->size(); i++, de++)
+	DocEntry_FreeTextBuffer(de);
+      */
+
+       if (LoadBuffers (qd, qd->DL->member(qd->doc_pos), max_mem,
+		       qd->DL->size() - qd->doc_pos) == -1)
+	return -1;
+    }
+  return 0;
+}
+
+
+int 
+GetDocNum (query_data * qd)
+{
+  if (qd->DL == NULL || qd->doc_pos >= qd->DL->size())
+    return -1;
+  return qd->DL->member(qd->doc_pos)->docNum();
+}
+
+DocEntry *
+GetDocChain (query_data * qd)
+{
+  if (qd->DL == NULL || qd->doc_pos >= qd->DL->size())
+    return NULL;
+  return qd->DL->member(qd->doc_pos);
+}
+
+float 
+GetDocWeight (query_data * qd)
+{
+  if (qd->DL == NULL || qd->doc_pos >= qd->DL->size())
+    return -1;
+  return qd->DL->member(qd->doc_pos)->docWeight();
+}
+
+long 
+GetDocCompLength (query_data * qd)
+{
+  if (qd->DL == NULL || qd->doc_pos >= qd->DL->size())
+    return -1;
+  return qd->DL->DE[qd->doc_pos].Len;
+}
+
+
+u_char *
+GetDocText (query_data * qd, unsigned long *len)
+{
+  DocEntry *DE;
+  int ULen;
+  if (qd->DL == NULL || qd->doc_pos >= qd->DL->size())
+    return NULL;
+
+  DE = qd->DL->member(qd->doc_pos);
+
+  if (!DocEntry_TextLoaded(DE))
+    {
+      fprintf (stderr, "The compressed text buffer is NULL\n");
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  QueryData_FreeTextBuffer (qd);
+
+  qd->TextBufferLen = (int) (qd->td->cth.ratio * 1.01 *
+			     DocEntry_length(DE)) + 100;
+  if (!(qd->TextBuffer = (unsigned char *) Xmalloc (qd->TextBufferLen)))
+    {
+      fprintf (stderr, "No memory for TextBuffer\n");
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  DecodeText (qd->cd, (u_char *) DocEntry_TextBuffer(DE), DE->Len,
+	      (u_char *) (qd->TextBuffer), &ULen);
+  qd->TextBuffer[ULen] = '\0';
+
+  if (ULen >= qd->TextBufferLen)
+    {
+      fprintf (stderr, "%d >= %d\n", ULen, qd->TextBufferLen);
+      mg_errno = MG_BUFTOOSMALL;
+      return (NULL);
+    }
+
+  if (len)
+    *len = ULen;
+
+  return qd->TextBuffer;
+}
+
+int 
+NextDoc (query_data * qd)
+{
+  if (qd->DL == NULL || qd->doc_pos >= qd->DL->size())
+    return 0;
+  qd->doc_pos++;
+  return qd->doc_pos < qd->DL->size();
+}
+
+
Index: /tags/gsdl-21/gsdl/src/mgpp/text/backend.h
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/backend.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/backend.h	(revision 699)
@@ -0,0 +1,289 @@
+/**************************************************************************
+ *
+ * backend.h -- Underlying routines and datastructures for mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef BACKEND_H
+#define BACKEND_H
+
+#include "sysfuncs.h"
+
+#include "timing.h"
+#include "lists.h"
+#include "term_lists.h"
+#include "query_term_list.h"  /* [RPAP - Feb 97: Term Frequency] */
+#include "mg.h"
+#include "invf.h"
+#include "text.h"
+
+typedef struct invf_data
+  {
+    File *InvfFile;
+    unsigned long N;
+    unsigned long Nstatic;	/* N parameter for decoding inverted file entries */
+    struct invf_file_header ifh;
+  }
+invf_data;
+
+typedef struct __idx_data
+{
+  unsigned long Start;
+  float Weight;
+} __idx_data;
+     
+
+typedef struct text_data
+  {
+    File *TextFile;
+    File *TextIdxFile;
+    File *TextIdxWgtFile;
+    long current_pos;
+    __idx_data *idx_data;
+    compressed_text_header cth;
+  }
+text_data;
+
+
+typedef struct auxiliary_dict
+  {
+    aux_frags_header afh[2];
+    u_char *word_data[2];
+    u_char **words[2];
+    int blk_start[2][33], blk_end[2][33];	/* blk_start and blk_end are required
+						   for the hybrid methods */
+  }
+auxiliary_dict;
+
+
+typedef struct compression_dict
+  {
+    compression_dict_header cdh;
+    comp_frags_header *cfh[2];
+    unsigned long MemForCompDict;
+    u_char ***values[2];
+    u_char *escape[2];
+    huff_data *chars_huff[2];
+    u_long **chars_vals[2];
+    huff_data *lens_huff[2];
+    u_long **lens_vals[2];
+    auxiliary_dict *ad;
+    int fast_loaded;
+  }
+compression_dict;
+
+
+typedef struct stemmed_idx   /* [RPAP - Jan 97: Stem Index Change] */
+  {
+    File *stem_idx_file;
+    struct stem_idx_header sih;
+    u_char **index;
+    unsigned long *pos;
+    int active;
+    u_char *buffer;
+    unsigned long MemForStemIdx;
+  }
+stemmed_idx;
+
+
+typedef struct stemmed_dict
+  {
+    File *stem_file;
+    struct stem_dict_header sdh;
+    u_char **index;
+    unsigned long *pos;
+    int active;
+    u_char *buffer;
+    unsigned long MemForStemDict;
+
+    /* [RPAP - Jan 97: Stem Index Change] */
+    stemmed_idx *stem1;
+    stemmed_idx *stem2;
+    stemmed_idx *stem3;
+  }
+stemmed_dict;
+
+
+typedef struct approx_weights_data
+  {
+    double L;
+    double B;
+    unsigned long *DocWeights;
+    char bits;
+    float *table;
+    unsigned long mask;
+    unsigned long MemForWeights;
+    unsigned long num_of_docs;
+  }
+approx_weights_data;
+
+typedef struct TextQueryInfo
+{
+  int stem;
+  int options;
+
+  int setWordActions(int _options, int _stem)
+  {
+    stem = _stem;
+    options = _options;
+  }
+
+  int getWordSpecial()
+  {
+    return stem | (options << 1);
+  }
+}
+TextQueryInfo;
+
+typedef struct RankedQueryInfo : public TextQueryInfo
+  {
+    int QueryFreqs;
+    int Exact;           /* use exact weights for ranking or not */
+    long MaxDocsToRetrieve;	/* may be -1 for all */
+    long MaxParasToRetrieve;
+    int Sort;        
+    char AccumMethod;		/* 'A' = array,  'S' = splay tree,  'H' = hash_table */
+    long MaxAccums;		/* may be -1 for all */
+    long MaxTerms;		/* may be -1 for all */
+    int StopAtMaxAccum; /* Stop at maximum accumulator or not */
+    long HashTblSize;
+    char *skip_dump;
+  }
+RankedQueryInfo;
+
+
+
+typedef struct BooleanQueryInfo : public TextQueryInfo
+  {
+    long MaxDocsToRetrieve;
+    int  OptimiseType;
+  }
+BooleanQueryInfo;
+
+
+/* [TS:24/Aug/94] - maximum number of characters in term string */
+#define MAXTERMSTRLEN 1023
+
+typedef struct query_data
+  {
+    stemmed_dict *sd;
+    compression_dict *cd;
+    approx_weights_data *awd;
+    invf_data *id;
+    text_data *td;
+#if defined(PARADOCNUM) || defined(NZDL)
+    int *paragraph;
+#endif
+    char *pathname;
+    char *textpathname; /* [RJM 06/97: text filename] */
+    File *File_text;
+    File *File_comp_dict;
+    File *File_aux_dict;
+    File *File_fast_comp_dict;
+    File *File_text_idx_wgt;
+    File *File_text_idx;
+    File *File_stem;
+
+    /* [RPAP - Jan 97: Stem Index Change] */
+    File *File_stem1;
+    File *File_stem2;
+    File *File_stem3;
+
+    File *File_invf;
+    File *File_weight_approx;
+    unsigned long mem_in_use, max_mem_in_use;
+    unsigned long num_of_ptrs, tot_num_of_ptrs;
+    unsigned long num_of_terms, tot_num_of_terms;
+    unsigned long num_of_accum, tot_num_of_accum;
+    unsigned long num_of_ans, tot_num_of_ans;
+    unsigned long hops_taken, tot_hops_taken;
+    unsigned long text_idx_lookups, tot_text_idx_lookups;
+    unsigned long max_buffers;
+    unsigned doc_pos;
+    unsigned buf_in_use;
+    DocList *DL;
+    TermList *TL;		/* [TS:Oct/94] - so term list for query can easily be accessed */
+    u_char *TextBuffer;
+    int TextBufferLen;
+    QueryTermList *QTL;    /* [RPAP - Feb 97: Term Frequency] */
+    int allfiles;          /* -- GRB : 14/09/99 - checks for all files having been opened */
+  }
+query_data;
+
+
+
+typedef struct InitQueryTimes
+  {
+    ProgTime Start;
+    ProgTime StemDict;
+    ProgTime ApproxWeights;
+    ProgTime CompDict;
+    ProgTime Invf;
+    ProgTime Text;
+  }
+InitQueryTimes;
+
+
+/* [RJM 06/97: text filename] */
+query_data *InitQuerySystem (char *dir, char *name, char *textname, InitQueryTimes * iqt);
+query_data *InitQuerySystemNT (char *dir, char *name, char *textname, InitQueryTimes * iqt);
+
+void ChangeMemInUse (query_data * qd, long delta);
+
+void FinishQuerySystem (query_data * qd);
+
+void ResetFileStats (query_data * qd);
+
+void TransFileStats (query_data * qd);
+
+void ChangeMemInUse (query_data * qd, long delta);
+
+void RankedQuery (query_data * qd, char *Query, RankedQueryInfo * rqi);
+
+void BooleanQuery (query_data * qd, char *Query, BooleanQueryInfo * bqi);
+
+void DocnumsQuery (query_data * qd, char *QueryLine);
+
+void QueryData_FreeTextBuffer (query_data * qd);
+
+void QueryData_FreeQueryDocs (query_data * qd);
+
+int LoadCompressedText (query_data * qd, int max_mem);
+
+void     DocList_FreeTextBuffers(DocList *List, query_data *qd);
+
+int GetDocNum (query_data * qd);
+
+float GetDocWeight (query_data * qd);
+
+long GetDocCompLength (query_data * qd);
+
+u_char *GetDocText (query_data * qd, unsigned long *len);
+
+DocEntry *GetDocChain (query_data * qd);
+
+int NextDoc (query_data * qd);
+
+#endif
+
+
+
+
Index: /tags/gsdl-21/gsdl/src/mgpp/text/bool_optimiser.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/bool_optimiser.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/bool_optimiser.cpp	(revision 699)
@@ -0,0 +1,1111 @@
+/**************************************************************************
+ *
+ * bool_optimiser -- optimise boolean parse tree
+ * Copyright (C) 1994,95  Tim Shimmin
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/10/11 02:57:01  cs025
+   Base install of MG-PP
+
+   Revision 1.1  1999/08/10 21:17:42  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:34:20  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.4  1995/07/27  04:54:55  tes
+   * Committing optimiser before I add the code
+   * to find diff-nodes in any kind of tree.
+   *
+   * Revision 1.3  1994/10/20  00:02:55  tes
+   * Fixed bug with bool_optimiser:
+   *   - it couldn't handle "false | false"
+   *   - it couldn't handle "true & true"
+   * These would collapse into an empty list and cause an error.
+   *
+   * Revision 1.2  1994/10/18  06:11:02  tes
+   * The boolean optimiser seems to be modifying the parse tree
+   * like it is supposed to.
+   * Paragraph ranking now works without any text files if required to.
+   *
+   * Revision 1.1  1994/10/12  01:15:28  tes
+   * Found bugs in the existing boolean query optimiser.
+   * So decided to rewrite it.
+   * I accidentally deleted query.bool.y, but I have replaced it
+   * with bool_parser.y (which I have forgotten to add here ! ;-(
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "bool_tree.h"
+#include "term_lists.h"
+#include "messages.h"
+
+typedef int (*CompFunc) (const void *, const void *);
+typedef void (*TreeFunc) (bool_tree_node *);
+
+/* --- routines --- */
+static void PermeateNots (bool_tree_node * tree);
+static void DoubleNeg (bool_tree_node * not_tree);
+static void AndDeMorgan (bool_tree_node * not_tree);
+static void OrDeMorgan (bool_tree_node * not_tree);
+static int AndDistribute (bool_tree_node * tree);
+static void DNF_Distribute (bool_tree_node * tree);
+static void ReplaceWithChildren (bool_tree_node * tree);
+static void CollapseTree (bool_tree_node * tree);
+static void AndTrueFalse (bool_tree_node * and_tree);
+static void OrTrueFalse (bool_tree_node * or_tree);
+static void TF_Idempotent (bool_tree_node * tree);
+static void AndNotToDiff (bool_tree_node * and_tree, TreeFunc recurs_func);
+static void DiffDNF (bool_tree_node * or_node);
+static void DiffTree (bool_tree_node * tree);
+static void AndSort (bool_tree_node * and_tree, TermList * tl, CompFunc comp);
+static void AndOrSort (bool_tree_node * tree, TermList * tl);
+static void AndSortDNF (bool_tree_node * or_node, TermList * tl);
+static void MarkOrTerms (bool_tree_node * tree, TermList * tl);
+static int calc_sum_ft (bool_tree_node * term_tree, TermList * tl);
+static int AndOrSort_comp (const void *A, const void *B);
+static int AndSort_comp (const void *A, const void *B);
+
+/* =========================================================================
+ * Function: OptimiseBoolTree
+ * Description: 
+ *      For case 2:
+ *        Do three major steps:
+ *        (i) put into standard form
+ *            -> put into DNF (disjunctive normal form - or of ands)
+ *            Use DeMorgan's, Double-negative, Distributive rules
+ *        (ii) simplify
+ *             apply idempotency rules
+ *        (iii) ameliorate
+ *              convert &! to diff nodes, order terms by frequency,...
+ *     Could also do the matching idempotency laws i.e. ...
+ *     (A | A), (A | !A), (A & !A), (A & A), (A & (A | B)), (A | (A & B)) 
+ *     Job for future.... ;-) 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+void
+OptimiseBoolTree (bool_tree_node * parse_tree, TermList * tl, int type)
+{
+  switch (type)
+    {
+    case 1:
+      /* --- sorts for and of or-terms --- */
+      PermeateNots (parse_tree);
+      TF_Idempotent (parse_tree);
+      CollapseTree (parse_tree);
+      DiffTree (parse_tree);
+      MarkOrTerms (parse_tree, tl);
+      AndOrSort (parse_tree, tl);
+      break;
+    case 2:
+      /* --- put into DNF --- */
+      PermeateNots (parse_tree);
+      TF_Idempotent (parse_tree);
+      DNF_Distribute (parse_tree);
+      CollapseTree (parse_tree);
+      AndSortDNF (parse_tree, tl);
+      DiffDNF (parse_tree);
+      break;
+    default:
+      /* --- do nothing --- */
+      break;
+    }				/*switch */
+}				/*OptimiseBoolTree */
+
+/* =========================================================================
+ * Function: DoubleNeg
+ * Description: 
+ *      !(!(a) = a
+ *      Assumes binary tree.
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+static void
+DoubleNeg (bool_tree_node * not_tree)
+{
+  bool_tree_node *child = BOOL_CHILD (not_tree);
+  bool_tree_node *a = BOOL_CHILD (child);
+  bool_tree_node *sibling = BOOL_SIBLING (not_tree);
+
+  /* copy a into not tree */
+  bcopy ((void *) a, (void *) not_tree, sizeof (bool_tree_node));
+
+  BOOL_SIBLING (not_tree) = sibling;
+
+  Xfree (child);
+  Xfree (a);
+
+}
+
+/* =========================================================================
+ * Function: AndDeMorgan
+ * Description: 
+ *      DeMorgan's rule for 'not' of an 'and'  i.e. !(a & b) <=> (!a) | (!b)
+ *      Assumes Binary Tree
+ * Input: 
+ *      not of and tree
+ * Output: 
+ *      or of not trees
+ * ========================================================================= */
+
+static void
+AndDeMorgan (bool_tree_node * not_tree)
+{
+  bool_tree_node *child = BOOL_CHILD (not_tree);
+  bool_tree_node *a = BOOL_CHILD (child);
+  bool_tree_node *b = BOOL_SIBLING (a);
+  bool_tree_node *new_not = NULL;
+
+  /* change 'not' to 'or' */
+  BOOL_TAG (not_tree) = N_or;
+
+  /* change 'and' to 'not' */
+  BOOL_TAG (child) = N_not;
+
+  /* 'a' no longer has a sibling */
+  BOOL_SIBLING (a) = NULL;
+
+  /* 'a's old sibling is now a child of a new not */
+  new_not = CreateBoolTreeNode (N_not, b, NULL);
+  BOOL_SIBLING (child) = new_not;
+}
+
+/* =========================================================================
+ * Function: OrDeMorgan
+ * Description: 
+ *      DeMorgan's rule for 'not' of an 'or' i.e. !(a | b) <=> (!a) & (!b)
+ *      Assumes Binary Tree
+ * Input: 
+ *      not of and tree
+ * Output: 
+ *      or of not trees
+ * ========================================================================= */
+
+static void
+OrDeMorgan (bool_tree_node * not_tree)
+{
+  bool_tree_node *child = BOOL_CHILD (not_tree);
+  bool_tree_node *a = BOOL_CHILD (child);
+  bool_tree_node *b = BOOL_SIBLING (a);
+  bool_tree_node *new_not = NULL;
+
+  /* change 'not' to 'and' */
+  BOOL_TAG (not_tree) = N_and;
+
+  /* change 'or' to 'not' */
+  BOOL_TAG (child) = N_not;
+
+  /* 'a' no longer has a sibling */
+  BOOL_SIBLING (a) = NULL;
+
+  /* 'a's old sibling is now a child of a new not */
+  new_not = CreateBoolTreeNode (N_not, b, NULL);
+  BOOL_SIBLING (child) = new_not;
+}
+
+/* =========================================================================
+ * Function: PermeateNots
+ * Description: 
+ *      Use DeMorgan's and Double-negative
+ *      Assumes tree in binary form (i.e. No ands/ors collapsed)
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+PermeateNots (bool_tree_node * tree)
+{
+  if (!tree)
+    return;
+
+  if (BOOL_TAG (tree) == N_not)
+    {
+      switch (BOOL_TAG (BOOL_CHILD (tree)))
+	{
+	case N_not:
+	  DoubleNeg (tree);
+/** NOTE: I do not go on to do the children **/
+	  /* I permeate the nots on the tree here & leave it at that */
+	  /* In this case I do not know what type 'tree' will end up */
+	  PermeateNots (tree);
+	  return;
+	case N_and:
+	  AndDeMorgan (tree);
+	  break;
+	case N_or:
+	  OrDeMorgan (tree);
+	  break;
+	default:
+	  /* leave alone */
+	  break;
+	}
+    }
+
+  /* Permeate Nots for children */
+  if (BOOL_HAS_CHILD (tree))
+    {
+      PermeateNots (BOOL_CHILD (tree));
+      PermeateNots (BOOL_SIBLING (BOOL_CHILD (tree)));
+    }
+
+}
+
+/* =========================================================================
+ * Function: DNF_Distribute
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+
+#define ITER_LIMIT 500		/* if reach limit then probably in trouble */
+
+static void
+DNF_Distribute (bool_tree_node * tree)
+{
+  int i = 0;
+
+  /* multiple passes until no change */
+  /* not very smart :-( */
+  while (1)
+    {
+      if (AndDistribute (tree) == 0)
+	return;
+      if (i == ITER_LIMIT)
+	FatalError (1, "DNF infinite loop");
+      i++;
+    }				/*while */
+
+}
+
+/* =========================================================================
+ * Function: AndDistribute
+ * Description: 
+ *      (a | b) & A <=> (a & A) | (b & A)
+ * Input: 
+ *      binary tree of "AND" , "OR"s.
+ * Output: 
+ *      return 1 if changed the tree
+ *      return 0 if there was NO change (no distributive rule to apply)
+ * ========================================================================= */
+
+static int
+AndDistribute (bool_tree_node * tree)
+{
+  if (!tree)
+    return 0;
+
+  if (BOOL_TAG (tree) != N_and)
+    {
+      /* try the children if got some */
+      if (BOOL_TAG (tree) == N_or)
+	{
+	  bool_tree_node *left_child = BOOL_CHILD (tree);
+	  int left = AndDistribute (left_child);
+	  int right = AndDistribute (BOOL_SIBLING (left_child));
+	  return (left || right);
+	}
+
+      /* no children, then no change */
+      return 0;
+    }
+
+  /* AND case */
+  {
+    /* we have to check the children to see if can distribute */
+    bool_tree_node *left_child = BOOL_CHILD (tree);
+    bool_tree_node *right_child = BOOL_SIBLING (left_child);
+    int swap = 0;
+
+    /* If "or" on the right side then do a swap */
+    /* This is done for code simplicity */
+    if (BOOL_TAG (right_child) == N_or)
+      {
+	/* do a swap */
+	bool_tree_node *temp;
+	temp = right_child;
+	right_child = left_child;
+	left_child = temp;
+
+	BOOL_SIBLING (left_child) = right_child;
+	BOOL_SIBLING (right_child) = NULL;
+	BOOL_CHILD (tree) = left_child;
+
+	swap = 1;
+      }
+
+    /* do the actual distribution */
+    if (swap || BOOL_TAG (left_child) == N_or)
+      {
+	bool_tree_node *A = right_child;
+	bool_tree_node *A_copy = CopyBoolTree (A);
+	bool_tree_node *a = BOOL_CHILD (left_child);
+	bool_tree_node *b = BOOL_SIBLING (a);
+
+	/* then can distribute away ... */
+	BOOL_TAG (tree) = N_or;
+	BOOL_TAG (left_child) = N_and;
+	BOOL_SIBLING (left_child) = CreateBoolTreeNode (N_and, b, A);
+	BOOL_SIBLING (a) = A_copy;
+
+	AndDistribute (tree);
+	return 1;
+      }
+
+    /* if "and" of something other than "or" s then try the children */
+    {
+      int left = AndDistribute (left_child);
+      int right = AndDistribute (right_child);
+      return (left || right);
+    }
+  }				/* AND case */
+
+}				/*AndDistribute */
+
+/* =========================================================================
+ * Function: CollapseTree
+ * Description: 
+ *      Collapse "and" and "or"s.
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+CollapseTree (bool_tree_node * tree)
+{
+  if (!tree)
+    return;
+
+  if ((BOOL_TAG (tree) == N_and) || (BOOL_TAG (tree) == N_or))
+    {
+      bool_tree_node *child = BOOL_CHILD (tree);
+      while (child)
+	{
+	  while (BOOL_TAG (child) == BOOL_TAG (tree))
+	    {
+	      ReplaceWithChildren (child);
+	    }
+	  CollapseTree (child);
+	  child = BOOL_SIBLING (child);
+	}
+    }
+
+  /* assume "not"s permeated */
+  return;
+}
+
+/* =========================================================================
+ * Function: ReplaceWithChildren
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+ReplaceWithChildren (bool_tree_node * tree)
+{
+  bool_tree_node *next = BOOL_SIBLING (tree);
+  bool_tree_node *child = BOOL_CHILD (tree);
+  bool_tree_node *last = LastBoolSibling (child);
+
+  bcopy ((void *) child, (void *) tree, sizeof (bool_tree_node));
+  Xfree (child);
+  BOOL_SIBLING (last) = next;
+
+}
+
+/* =========================================================================
+ * Function: TF_Idempotent
+ * Description: 
+ *      Applies Idempotency laws which refer to True or False (hence "TF")
+ *      uses the true or false idempotent rules
+ * Input: 
+ *      operates on n-ary trees (not restricted to binary)
+ * Output: 
+ * ========================================================================= */
+
+static void
+TF_Idempotent (bool_tree_node * tree)
+{
+  bool_tree_node *child = NULL;
+
+  while (tree)
+    {
+      if (BOOL_HAS_CHILD (tree))
+	{
+	  child = BOOL_CHILD (tree);
+	  TF_Idempotent (child);
+	}
+
+      /* --- process node --- */
+      switch (BOOL_TAG (tree))
+	{
+	case N_not:
+	  child = BOOL_CHILD (tree);
+
+	  /* if true then falsify */
+	  if (BOOL_TAG (child) == N_all)
+	    {
+	      BOOL_TAG (tree) = N_none;
+	      BOOL_CHILD (tree) = NULL;
+	      Xfree (child);
+	      break;
+	    }
+
+	  /* if false then trueify */
+	  if (BOOL_TAG (child) == N_none)
+	    {
+	      BOOL_TAG (tree) = N_all;
+	      BOOL_CHILD (tree) = NULL;
+	      Xfree (child);
+	      break;
+	    }
+
+	  break;
+	case N_and:
+	  AndTrueFalse (tree);
+	  break;
+	case N_or:
+	  OrTrueFalse (tree);
+	  break;
+
+	default:
+	  /* do nothing */
+	  break;
+	}			/*switch */
+
+      tree = BOOL_SIBLING (tree);
+    }				/*while */
+}				/*TF_Idempotent */
+
+/* =========================================================================
+ * Function: AndTrueFalse
+ * Description: 
+ *      If any "false" then falsify whole tree 
+ *      If any "true" then delete them
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+AndTrueFalse (bool_tree_node * and_tree)
+{
+  bool_tree_node *child = BOOL_CHILD (and_tree);
+
+  bool_tree_node *prev = NULL;
+
+  while (child)
+    {
+      if (BOOL_TAG (child) == N_none)
+	{
+	  FreeBoolTree (&(BOOL_CHILD (and_tree)));
+	  BOOL_TAG (and_tree) = N_none;
+	  return;
+	}
+
+      if (BOOL_TAG (child) == N_all)
+	{
+	  bool_tree_node *sibling = BOOL_SIBLING (child);
+
+	  /* --- link over child --- */
+	  if (!prev)
+	    {
+	      BOOL_CHILD (and_tree) = sibling;
+	    }
+	  else
+	    {
+	      BOOL_SIBLING (prev) = sibling;
+	    }
+
+	  Xfree (child);
+	  child = sibling;
+	}
+      else
+	{
+	  prev = child;
+	  child = BOOL_SIBLING (child);
+	}
+    }				/*while */
+
+  child = BOOL_CHILD (and_tree);
+
+  /* if no children ... */
+  /* => all have been true */
+  if (!child)
+    {
+      BOOL_TAG (and_tree) = N_all;
+      return;
+    }
+
+  /* If only one child ... */
+  if (!BOOL_SIBLING (child))
+    {
+      /* replace the parent with child - move it up */
+      bool_tree_node *save_sibling = BOOL_SIBLING (and_tree);
+      bcopy ((void *) child, (void *) and_tree, sizeof (bool_tree_node));
+      BOOL_SIBLING (and_tree) = save_sibling;
+      Xfree (child);
+    }
+
+}				/*AndTrueFalse */
+
+/* =========================================================================
+ * Function: OrTrueFalse
+ * Description: 
+ *      If any "true" then trueify whole tree 
+ *      If any "false" then delete them 
+ *      ***** NOTE: This is a reflection of AndTrueFalse *****
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+OrTrueFalse (bool_tree_node * or_tree)
+{
+  bool_tree_node *child = BOOL_CHILD (or_tree);
+
+  bool_tree_node *prev = NULL;
+
+  while (child)
+    {
+      if (BOOL_TAG (child) == N_all)
+	{
+	  FreeBoolTree (&(BOOL_CHILD (or_tree)));
+	  BOOL_TAG (or_tree) = N_all;
+	  return;
+	}
+
+      if (BOOL_TAG (child) == N_none)
+	{
+	  bool_tree_node *sibling = BOOL_SIBLING (child);
+	  if (!prev)
+	    {
+	      BOOL_CHILD (or_tree) = sibling;
+	    }
+	  else
+	    {
+	      BOOL_SIBLING (prev) = sibling;
+	    }
+	  Xfree (child);
+	  child = sibling;
+
+	}
+      else
+	{
+	  prev = child;
+	  child = BOOL_SIBLING (child);
+	}
+    }				/*while */
+
+  child = BOOL_CHILD (or_tree);
+
+  /* if no children ... */
+  /* => all have been false */
+  if (!child)
+    {
+      BOOL_TAG (or_tree) = N_none;
+      return;
+    }
+
+  /* If only one child ... */
+  if (!BOOL_SIBLING (child))
+    {
+      /* replace the parent with child - move it up */
+      bool_tree_node *save_sibling = BOOL_SIBLING (or_tree);
+      bcopy ((void *) child, (void *) or_tree, sizeof (bool_tree_node));
+      BOOL_SIBLING (or_tree) = save_sibling;
+      Xfree (child);
+    }
+
+}				/*AndTrueFalse */
+
+/* =========================================================================
+ * Function: AndNotToDiff
+ * Description: 
+ *      Convert an "And" tree with possibly some negated terms into
+ *      a "And - neg-terms". 
+ * Input: 
+ *      Expects "And" of only literals - as tree should be in DNF form.
+ *      This means that no recursion is necessary.
+ * Output: 
+ * ========================================================================= */
+
+static void
+AndNotToDiff (bool_tree_node * and_tree, TreeFunc recurs_func)
+{
+
+  /* -------------------------------------------------- */
+  /* --- Divide the children up into positive and negative literals --- */
+  /* Preserves ordering of children */
+  bool_tree_node *pos = NULL;
+  bool_tree_node *neg = NULL;
+  bool_tree_node *last_pos = NULL;
+  bool_tree_node *last_neg = NULL;
+
+  bool_tree_node *child = BOOL_CHILD (and_tree);
+  while (child)
+    {
+      bool_tree_node *next = BOOL_SIBLING (child);
+      if (BOOL_TAG (child) == N_not)
+	{
+	  /* add to end of neg list */
+	  if (!neg)
+	    neg = child;
+	  else
+	    BOOL_SIBLING (last_neg) = child;
+	  last_neg = child;
+	}
+      else
+	{
+	  /* add to end of pos list */
+	  if (!pos)
+	    pos = child;
+	  else
+	    BOOL_SIBLING (last_pos) = child;
+	  last_pos = child;
+	}
+      child = next;
+    }				/*while */
+  if (last_pos)
+    BOOL_SIBLING (last_pos) = NULL;
+  if (last_neg)
+    BOOL_SIBLING (last_neg) = NULL;
+  /* -------------------------------------------------- */
+
+  if (!neg)			/* no negtives => no subtraction */
+    {
+      /* fix up parent's child link */
+      BOOL_CHILD (and_tree) = pos;
+      return;
+    }
+
+  if (!pos)			/* no positives = all negatives */
+    {
+      /* Choose one of the negs for the left-hand-side of the subtraction */
+      /* In particular, choose the first */
+      pos = neg;
+      neg = BOOL_SIBLING (neg);
+      BOOL_SIBLING (pos) = NULL;
+    }
+
+  /* Get rid of all '!' from not nodes */
+  /* Make a list out of the children */
+  {
+    bool_tree_node *n = neg;
+    bool_tree_node *next = NULL;
+
+    neg = BOOL_CHILD (neg);
+    while (n)
+      {
+	bool_tree_node *c = BOOL_CHILD (n);
+	next = BOOL_SIBLING (n);
+	Xfree (n);
+	if (next)
+	  BOOL_SIBLING (c) = BOOL_CHILD (next);
+	n = next;
+      }
+  }
+
+  /* --- restructure the tree with "and" and "diff" --- */
+  BOOL_TAG (and_tree) = N_diff;
+  if (!BOOL_SIBLING (pos))	/* i.e. only one pos */
+    {
+      /* (A & !B & !C & !D & ...) ==> A - (B,C,D,...) */
+
+      BOOL_CHILD (and_tree) = pos;
+      BOOL_SIBLING (pos) = neg;
+    }
+  else
+    {
+      /* (A & !C & B & !D & ...) ==> (A & B & ....) - (C, D, ...) */
+
+      bool_tree_node *n = CreateBoolNode (N_and);
+      BOOL_CHILD (and_tree) = n;
+      BOOL_CHILD (n) = pos;
+      BOOL_SIBLING (n) = neg;
+    }				/*if */
+
+  if (recurs_func)
+    {
+      recurs_func (BOOL_CHILD (and_tree));
+    }
+
+}				/*AndNotToDiff */
+
+/* =========================================================================
+ * Function: DiffTree
+ * Description: 
+ *      Applies "AndNotToNeg" to each conjunct.
+ *      Unlike DiffDNF, this routine will search throughout the tree for
+ *      "and"-nodes. It will call AndNotToNeg with this as its recursive 
+ *      routine.
+ * Date: 27/July/95
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+DiffTree (bool_tree_node * tree)
+{
+  bool_tree_node *t = tree;
+
+  while (t)
+    {
+      if (BOOL_TAG (t) == N_and)
+	{
+	  AndNotToDiff (t, DiffTree);
+	}
+      t = BOOL_SIBLING (t);
+    }				/*while */
+
+}				/*DiffTree */
+
+/* =========================================================================
+ * Function: DiffDNF
+ * Description: 
+ *      Applies "AndNotToNeg" to each conjunct.
+ * Input: 
+ *      A DNF tree
+ * Output: 
+ *      A tree possibly with some diffs in it.
+ * ========================================================================= */
+
+static void
+DiffDNF (bool_tree_node * DNF_tree)
+{
+
+  if (BOOL_TAG (DNF_tree) == N_or)
+    {
+      bool_tree_node *child = BOOL_CHILD (DNF_tree);
+
+      while (child)
+	{
+	  if (BOOL_TAG (child) == N_and)
+	    {
+	      AndNotToDiff (child, NULL);
+	    }
+	  child = BOOL_SIBLING (child);
+	}
+    }
+  else if (BOOL_TAG (DNF_tree) == N_and)
+    {
+      AndNotToDiff (DNF_tree, NULL);
+    }
+}
+
+/* =========================================================================
+ * Function: AndSort_comp 
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+/* Module variable, "term_list",  used as parameter for comparison function */
+/* The comparison function only allows certain fixed parameters */
+static TermList *term_list;
+
+static int
+AndSort_comp (const void *A, const void *B)
+{
+  const bool_tree_node *const *a = (bool_tree_node **) A;
+  const bool_tree_node *const *b = (bool_tree_node **) B;
+
+  if (BOOL_TAG (*a) == N_term && BOOL_TAG (*b) == N_term)
+    {
+      /* look up in term list */
+      int a_doc_count = GetNthWE (term_list, BOOL_TERM (*a))->doc_count;
+      int b_doc_count = GetNthWE (term_list, BOOL_TERM (*b))->doc_count;
+
+      return a_doc_count - b_doc_count;
+    }
+
+  if (BOOL_TAG (*a) == N_term && BOOL_TAG (*b) != N_term)
+    return -1;
+
+  if (BOOL_TAG (*a) != N_term && BOOL_TAG (*b) == N_term)
+    return 1;
+
+  return 0;
+}				/*AndSort_comp */
+
+/* =========================================================================
+ * Function: AndSort
+ * Description: 
+ *      Sort the list of nodes by increasing doc_count 
+ *      Using some Neil Sharman code - pretty straight forward.
+ *      Note: not-terms are sent to the end of the list
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+AndSort (bool_tree_node * and_tree, TermList * tl, CompFunc comp)
+{
+  bool_tree_node **list = NULL;
+  bool_tree_node *child = NULL;
+  int count = 0;
+  int i = 0;
+
+  if (BOOL_TAG (and_tree) != N_and)
+    return;
+
+  /* --- Convert linked list into an array for sorting --- */
+  child = BOOL_CHILD (and_tree);
+  count = 0;
+  while (child)
+    {
+      count++;
+      child = BOOL_SIBLING (child);
+    }
+
+  if (!(list = (bool_tree_node **) Xmalloc (sizeof (bool_tree_node *) * count)))
+    return;
+
+  child = BOOL_CHILD (and_tree);
+  for (i = 0; i < count; i++)
+    {
+      list[i] = child;
+      child = BOOL_SIBLING (child);
+    }
+  /* ----------------------------------------------------- */
+
+  term_list = tl;
+  qsort (list, count, sizeof (bool_tree_node *), comp);
+
+  /* --- Convert array back into linked list --- */
+  for (i = 0; i < count - 1; i++)
+    BOOL_SIBLING (list[i]) = list[i + 1];
+  BOOL_SIBLING (list[count - 1]) = NULL;
+  BOOL_CHILD (and_tree) = list[0];
+  Xfree (list);
+  /* ----------------------------------------------------- */
+
+}				/*AndSort */
+
+/* =========================================================================
+ * Function: AndSortDNF
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+
+static void
+AndSortDNF (bool_tree_node * DNF_tree, TermList * tl)
+{
+
+  if (BOOL_TAG (DNF_tree) == N_or)
+    {
+      bool_tree_node *child = BOOL_CHILD (DNF_tree);
+
+      while (child)
+	{
+	  if (BOOL_TAG (child) == N_and)
+	    {
+	      AndSort (child, tl, AndSort_comp);
+	    }
+	  child = BOOL_SIBLING (child);
+	}
+    }
+  else if (BOOL_TAG (DNF_tree) == N_and)
+    {
+      AndSort (DNF_tree, tl, AndSort_comp);
+    }
+}
+
+/* =========================================================================
+ * Function: MarkOrTerms
+ * Description: 
+ *      Mark or-nodes whose children are all terms.
+ *      Also store the sum of the ft's of these nodes.
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+MarkOrTerms (bool_tree_node * tree, TermList * tl)
+{
+  if (BOOL_TAG (tree) == N_or)
+    {
+      int found_non_term = 0;
+      bool_tree_node *child = BOOL_CHILD (tree);
+
+      /* --- check to see if got all terms --- */
+      while (child)
+	{
+	  if (BOOL_TAG (child) != N_term)
+	    {
+	      found_non_term = 1;
+	    }
+	  if (BOOL_HAS_CHILD (tree))
+	    {
+	      MarkOrTerms (child, tl);
+	    }
+	  child = BOOL_SIBLING (child);
+	}			/*while */
+
+      if (!found_non_term)
+	{
+	  /* Must have all terms for the OR node */
+	  /* So change its tag and calculate its sum */
+	  BOOL_TAG (tree) = N_or_terms;
+	  BOOL_SUM_FT (tree) = calc_sum_ft (tree, tl);
+	}			/*if */
+      /* ------------------------------------- */
+
+    }				/*if OR node */
+
+  else if (BOOL_HAS_CHILD (tree))
+    {
+      bool_tree_node *child = BOOL_CHILD (tree);
+      while (child)
+	{
+	  MarkOrTerms (child, tl);
+	  child = BOOL_SIBLING (child);
+	}
+    }
+
+}				/*MarkOrTerms */
+
+/* =========================================================================
+ * Function: calc_sum_ft
+ * Description: 
+ *      Sums up the ft's of all the terms 
+ * Input: 
+ *      Assumes that the tree only contains terms in it 
+ * Output: 
+ *      the ft sum
+ * ========================================================================= */
+static int
+calc_sum_ft (bool_tree_node * term_tree, TermList * tl)
+{
+  int sum_ft = 0;
+  bool_tree_node *child = BOOL_CHILD (term_tree);
+  while (child)
+    {
+      sum_ft += GetNthWE (tl, BOOL_TERM (child))->doc_count;
+      child = BOOL_SIBLING (child);
+    }
+  return sum_ft;
+}
+
+/* =========================================================================
+ * Function: AndOrSort
+ * Description: 
+ *      This is like an AndSort of just terms. 
+ *      The difference is that we are now also 
+ *      considering sorting with <or of terms> by using the sum of f_t's.
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+static void
+AndOrSort (bool_tree_node * tree, TermList * tl)
+{
+  if (BOOL_HAS_CHILD (tree))
+    {
+
+      if (BOOL_TAG (tree) == N_and)
+	{
+	  AndSort (tree, tl, AndOrSort_comp);
+	}
+
+      /* Check out any children down the tree */
+      /* Probably a waste of time */
+      {
+	bool_tree_node *child = BOOL_CHILD (tree);
+	while (child)
+	  {
+	    AndOrSort (child, tl);
+	    child = BOOL_SIBLING (child);
+	  }
+      }				/*do the tree */
+
+    }				/*if has children */
+}				/*AndOrSort */
+
+/* =========================================================================
+ * Function: AndOrSort_comp
+ * Description: 
+ *      Comparison function used to order "term"s and "or_terms"s.
+ *      It's purpose is to be used by a sort routine.
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+static int
+AndOrSort_comp (const void *A, const void *B)
+{
+  const bool_tree_node *const *a = (bool_tree_node **) A;
+  const bool_tree_node *const *b = (bool_tree_node **) B;
+
+  if (BOOL_TAG (*a) == N_term && BOOL_TAG (*b) == N_term)
+    {
+      /* look up in term list */
+      int a_doc_count = GetNthWE (term_list, BOOL_TERM (*a))->doc_count;
+      int b_doc_count = GetNthWE (term_list, BOOL_TERM (*b))->doc_count;
+
+      return a_doc_count - b_doc_count;
+    }
+
+  if (BOOL_TAG (*a) == N_term && BOOL_TAG (*b) == N_or_terms)
+    {
+      /* look up in term list */
+      int a_doc_count = GetNthWE (term_list, BOOL_TERM (*a))->doc_count;
+      int b_doc_count = BOOL_SUM_FT (*b);
+
+      return a_doc_count - b_doc_count;
+    }
+
+  if (BOOL_TAG (*a) == N_or_terms && BOOL_TAG (*b) == N_term)
+    {
+      /* look up in term list */
+      int a_doc_count = BOOL_SUM_FT (*a);
+      int b_doc_count = GetNthWE (term_list, BOOL_TERM (*b))->doc_count;
+
+      return a_doc_count - b_doc_count;
+    }
+
+  if (BOOL_TAG (*a) == N_or_terms && BOOL_TAG (*b) == N_or_terms)
+    {
+      /* look up in term list */
+      int a_doc_count = BOOL_SUM_FT (*a);
+      int b_doc_count = BOOL_SUM_FT (*b);
+
+      return a_doc_count - b_doc_count;
+    }
+
+  if ((BOOL_TAG (*a) == N_term || BOOL_TAG (*a) == N_or_terms) &&
+      (BOOL_TAG (*b) != N_term && BOOL_TAG (*b) != N_or_terms))
+    return -1;
+
+  if ((BOOL_TAG (*a) != N_term && BOOL_TAG (*a) != N_or_terms) &&
+      (BOOL_TAG (*b) == N_term || BOOL_TAG (*b) == N_or_terms))
+    return 1;
+
+  return 0;
+}				/*AndOrSort_comp */
Index: /tags/gsdl-21/gsdl/src/mgpp/text/bool_optimiser.h
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/bool_optimiser.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/bool_optimiser.h	(revision 699)
@@ -0,0 +1,65 @@
+/**************************************************************************
+ *
+ * bool_optimiser -- optimise boolean parse tree
+ * Copyright (C) 1994  Tim Shimmin
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/10/11 02:57:03  cs025
+   Base install of MG-PP
+
+   Revision 1.1  1999/08/10 21:17:42  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:34:23  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.2  1995/03/14  05:15:21  tes
+   * Updated the boolean query optimiser to do different types of optimisation.
+   * A query environment variable "optimise_type" specifies which one is to be
+   * used. Type 1 is the new one which is faster than 2.
+   *
+   * Revision 1.1  1994/10/20  03:56:32  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/10/18  06:11:03  tes
+   * The boolean optimiser seems to be modifying the parse tree
+   * like it is supposed to.
+   * Paragraph ranking now works without any text files if required to.
+   *
+   * Revision 1.1  1994/10/12  01:15:29  tes
+   * Found bugs in the existing boolean query optimiser.
+   * So decided to rewrite it.
+   * I accidentally deleted query.bool.y, but I have replaced it
+   * with bool_parser.y (which I have forgotten to add here ! ;-(
+   *
+ */
+
+#ifndef BOOL_OPTIMISER_H
+#define BOOL_OPTIMISER_H
+
+#include "bool_tree.h"
+#include "term_lists.h"
+
+void OptimiseBoolTree (bool_tree_node * parse_tree, TermList * term_list, int type);
+
+#endif
Index: /tags/gsdl-21/gsdl/src/mgpp/text/bool_parser.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/bool_parser.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/bool_parser.cpp	(revision 699)
@@ -0,0 +1,639 @@
+#ifndef lint
+static char yysccsid[] = "@(#)yaccpar	1.8 (Berkeley) 01/20/90";
+#endif
+#define YYBYACC 1
+#line 25 "bool_parser.ypp"
+ 
+#include "sysfuncs.h"
+
+#include "messages.h"
+ 
+#include "memlib.h"
+#include "words.h"
+#include "stemmer.h"
+#include "term_lists.h"
+#include "bool_tree.h"
+/* [RPAP - Jan 97: Stem Index Change] */
+#include "backend.h"     /* for stemmed_dict def */
+#include "stem_search.h"
+
+#include "query_term_list.h"  /* [RPAP - Feb 97: Term Frequency] */
+
+/* --- routines --- */
+static int query_lex(char **, const char *);
+static int yyerror(char *);
+int yyparse();
+#define yylex() query_lex(&ch_buf, end_buf)
+ 
+/* --- module variables --- */
+static char *ch_buf; /* ptr to the character query line buffer */
+static char *end_buf; /* ptr to the last character of the line buffer */
+static bool_tree_node *tree_base = NULL;
+static TermList **term_list;
+static int stem_method;
+/* [RPAP - Jan 97: Stem Index Change] */
+stemmed_dict *p__sd;
+static int indexed;
+/* [RPAP - Feb 97: Term Frequency] */
+static QueryTermList **query_term_list;
+static int word_num;
+static u_long count;
+static u_long doc_count;
+static u_long invf_ptr;
+static u_long invf_len;
+static int stemmer_num; /* George Buchanan - Aug 99: Stemmer id */
+#line 67 "bool_parser.ypp"
+typedef union {
+  char *text;
+  bool_tree_node *node;
+} YYSTYPE;
+#line 51 "y.tab.c"
+#define TERM 257
+#define YYERRCODE 256
+short yylhs[] = {                                        -1,
+    0,    1,    1,    1,    1,    2,    2,    3,    3,    3,
+    4,    4,
+};
+short yylen[] = {                                         2,
+    1,    1,    3,    1,    1,    1,    2,    3,    2,    1,
+    3,    1,
+};
+short yydefred[] = {                                      0,
+    2,    0,    4,    5,    0,    0,    6,   10,    0,    0,
+    0,    7,    0,    9,    0,    3,    8,    0,
+};
+short yydgoto[] = {                                       6,
+    7,    8,    9,   10,
+};
+short yysindex[] = {                                    -32,
+    0,  -32,    0,    0,  -32,    0,    0,    0,  -33, -118,
+  -37,    0,  -32,    0,  -32,    0,    0,  -33,
+};
+short yyrindex[] = {                                      0,
+    0,    0,    0,    0,    0,    0,    0,    0,    2,   12,
+    0,    0,    0,    0,    0,    0,    0,    3,
+};
+short yygindex[] = {                                      0,
+    0,    6,   -2,   14,
+};
+#define YYTABLESIZE 225
+short yytable[] = {                                       5,
+    5,   12,   11,   16,   13,   15,    2,    2,    3,    3,
+   12,    1,   18,    0,   14,   11,    0,    0,   17,    0,
+    0,    0,    0,   14,    0,    0,    0,    0,    0,    0,
+    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
+    0,    0,   12,   11,    0,    0,    0,    0,    0,    0,
+    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
+    0,    4,    4,    0,    0,    0,    0,    0,    0,    0,
+    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
+    0,    0,    0,    0,    0,    0,   15,    0,    0,    0,
+    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
+    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
+    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
+    0,    0,    0,    0,    0,   12,   11,    0,    0,    0,
+    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
+    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
+    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
+    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
+    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
+    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
+    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
+    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
+    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
+    0,    0,    0,    1,    1,
+};
+short yycheck[] = {                                      33,
+   33,    0,    0,   41,   38,  124,   40,   40,   42,   42,
+    5,    0,   15,   -1,    9,    2,   -1,   -1,   13,   -1,
+   -1,   -1,   -1,   18,   -1,   -1,   -1,   -1,   -1,   -1,
+   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,
+   -1,   -1,   41,   41,   -1,   -1,   -1,   -1,   -1,   -1,
+   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,
+   -1,   95,   95,   -1,   -1,   -1,   -1,   -1,   -1,   -1,
+   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,
+   -1,   -1,   -1,   -1,   -1,   -1,  124,   -1,   -1,   -1,
+   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,
+   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,
+   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,
+   -1,   -1,   -1,   -1,   -1,  124,  124,   -1,   -1,   -1,
+   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,
+   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,
+   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,
+   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,
+   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,
+   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,
+   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,
+   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,
+   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,
+   -1,   -1,   -1,  257,  257,
+};
+#define YYFINAL 6
+#ifndef YYDEBUG
+#define YYDEBUG 0
+#endif
+#define YYMAXTOKEN 257
+#if YYDEBUG
+char *yyname[] = {
+"end-of-file",0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+"'!'",0,0,0,0,"'&'",0,"'('","')'","'*'",0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,"'_'",0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,"'|'",0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+"TERM",
+};
+char *yyrule[] = {
+"$accept : query",
+"query : or",
+"term : TERM",
+"term : '(' or ')'",
+"term : '*'",
+"term : '_'",
+"not : term",
+"not : '!' not",
+"and : and '&' not",
+"and : and not",
+"and : not",
+"or : or '|' and",
+"or : and",
+};
+#endif
+#define yyclearin (yychar=(-1))
+#define yyerrok (yyerrflag=0)
+#ifdef YYSTACKSIZE
+#ifndef YYMAXDEPTH
+#define YYMAXDEPTH YYSTACKSIZE
+#endif
+#else
+#ifdef YYMAXDEPTH
+#define YYSTACKSIZE YYMAXDEPTH
+#else
+#define YYSTACKSIZE 500
+#define YYMAXDEPTH 500
+#endif
+#endif
+int yydebug;
+int yynerrs;
+int yyerrflag;
+int yychar;
+short *yyssp;
+YYSTYPE *yyvsp;
+YYSTYPE yyval;
+YYSTYPE yylval;
+short yyss[YYSTACKSIZE];
+YYSTYPE yyvs[YYSTACKSIZE];
+#define yystacksize YYSTACKSIZE
+#line 101 "bool_parser.ypp"
+ 
+/* Bison on one mips machine defined "const" to be nothing but
+   then did not undef it */
+#ifdef const
+#undef const
+#endif
+
+/**************************************************************************/ 
+
+
+/* =========================================================================
+ * Function: query_lex
+ * Description:
+ *      Hand written lexical analyser for the parser.
+ * Input:
+ *      ptr = ptr to a ptr into character query-line buffer
+ *      end = ptr to last char in buffer
+ * Output:
+ *      yylval.text = the token's text
+ * Notes:
+ *      does NOT produce WILD tokens at the moment
+ * ========================================================================= */
+ 
+/* [RPAP - Jan 97: Stem Index Change]
+   state mode:
+      0 = Read next token
+      1 = Output word
+      2 = Output '|' or ')'
+ */
+static int query_lex(char **ptr, const char *end)
+{
+  char *buf_ptr = *ptr;
+  static int mode = 0;
+  static int termnum = 0;
+  static TermList *Terms = NULL;
+
+  if (mode == 0)
+    {
+      /* jump over whitespace */
+      while (isspace(*buf_ptr))
+	buf_ptr++;
+ 
+      if (inaword((unsigned char *) buf_ptr, (unsigned char *) end))
+	{
+	  static char word[MAXSTEMLEN + 1]; /* [RJM 07/98: Memory Leak] */
+	  char *sWord = (char *) Xmalloc(MAXSTEMLEN + 1);
+	  int stem_to_apply, method_using = -1;
+
+	  PARSE_STEM_WORD((unsigned char *) word, (unsigned char *) buf_ptr, 
+                          (unsigned char *) end);
+
+	  /* Extract any parameters */
+	  stem_to_apply = stem_method;
+	  while (buf_ptr <= end)
+	    {
+	      int stem_param, param_type;
+	      char param[MAXPARAMLEN + 1];
+
+	      param_type = 0;
+	      PARSE_OPT_TERM_PARAM (param, param_type, buf_ptr, end);
+	      if (!param_type)
+		break;
+
+	      if (param_type == STEMPARAM)
+		{
+		  stem_param = atoi (param);
+		  if (errno != ERANGE && indexed && stem_param >= 0 && stem_param <= 3)
+		    method_using = stem_to_apply = stem_param;
+		}
+	    }
+
+	  bcopy ((char *) word, (char *) sWord, *word + 1);
+	  stemmer (stem_to_apply, stemmer_num, (unsigned char *) sWord);
+
+	  if (stem_to_apply == 0 || !indexed || p__sd == NULL)
+	    {
+	      /* [RPAP - Feb 97: Term Frequency] */
+	      word_num = FindWord (p__sd, (unsigned char *) sWord, &count, &doc_count, &invf_ptr, &invf_len);
+	      if (word_num == -1)
+		count = doc_count = invf_ptr = invf_len = 0;
+	      QueryTermList_AddQueryTerm (query_term_list, (u_char *) word, count, method_using);
+
+	      yylval.text = word;
+	      *ptr = buf_ptr; /* fix up ptr */
+	      Xfree (sWord);
+	      return TERM;
+	    }
+	  else
+	    {
+	      *ptr = buf_ptr; /* fix up ptr */
+	      termnum = 0;
+	      TermList_reset (&Terms);
+	      if (FindWords (p__sd, (u_char *) sWord, stem_to_apply, &Terms) > 0)
+		{
+		  /* [RPAP - Feb 97: Term Frequency] */ 
+		  int i, freq = 0;
+		  for (i = 0; i < Terms->size(); i++)
+		    freq += Terms->TE[i].WE.count;
+		  QueryTermList_AddQueryTerm (query_term_list, (unsigned char *) word, freq, method_using);
+
+		  Xfree (sWord);
+		  mode = 1;
+		  return '(';
+		}
+	      else
+		{
+		  /* Word does not exists - include in tree anyway */
+		  Xfree (sWord);
+
+		  /* [RPAP - Feb 97: Term Frequency] */
+		  word_num = -1;
+		  count = doc_count = invf_ptr = invf_len = 0;
+		  QueryTermList_AddQueryTerm (query_term_list, (u_char *) word, count, method_using);
+
+		  yylval.text = word;
+		  return TERM;
+		}
+	    }
+	}
+      else /* NON-WORD */
+	{
+	  if (*buf_ptr == '\0')
+	    {
+	      /* return null-char if it is one */
+	      *ptr = buf_ptr; /* fix up ptr */
+	      return 0;
+	    }
+	  else
+	    {
+	      /* return 1st char, and delete from buffer */
+	      char c = *buf_ptr++;
+	      *ptr = buf_ptr; /* fix up ptr */
+	      return c;
+	    }
+	}
+    }
+  else if (mode == 1)
+    {
+      yylval.text = (char *) Terms->TE[termnum].Word;
+      
+      /* [RPAP - Feb 97: Term Frequency] */
+      word_num = Terms->TE[termnum].WE.word_num;
+      count = Terms->TE[termnum].WE.count;
+      doc_count = Terms->TE[termnum].WE.doc_count;
+      invf_ptr = Terms->TE[termnum].WE.invf_ptr;
+      invf_len = Terms->TE[termnum].WE.invf_len;
+
+      termnum++;
+      mode = 2;
+      return TERM;
+    }
+  else  /* mode == 2 */
+    {
+      if (termnum >= Terms->size())
+	{
+	  mode = 0;
+	  return ')';
+	}
+      else
+	{
+	  mode = 1;
+	  return '|';
+	}
+    }
+}/*query_lex*/
+
+/* =========================================================================
+ * Function: yyerror
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */ 
+static int yyerror(char *s)
+{
+  Message("%s", s);
+  return(1);
+}
+
+ 
+/* =========================================================================
+ * Function: ParseBool
+ * Description:
+ *      Parse a boolean query string into a term-list and a boolean parse tree
+ * Input:
+ *      query_line = query line string
+ *      query_len = query line length
+ *      the_stem_method = stem method id used for stemming
+ * Output:
+ *      the_term_list = the list of terms
+ *      res = parser result code
+ * ========================================================================= */
+ 
+extern "C" bool_tree_node *
+ParseBool(char *query_line, int query_len,
+          TermList **the_term_list, int the_stem_method, int the_stemmer_num, int *res,
+	  stemmed_dict * the_sd, int is_indexed,   /* [RPAP - Jan 97: Stem Index Change] */
+	  QueryTermList **the_query_term_list)  /* [RPAP - Feb 97: Term Frequency] */
+{ /* global variables to be accessed by bison/yacc created parser */
+  term_list = the_term_list;
+  stem_method = the_stem_method;
+  stemmer_num = the_stemmer_num;
+  ch_buf = query_line;
+  end_buf = query_line + query_len;
+  p__sd = the_sd;   /* [RPAP - Jan 97: Stem Index Change] */
+  indexed = is_indexed;  /* [RPAP - Jan 97: Stem Index Change] */
+  query_term_list = the_query_term_list; /* [RPAP - Feb 97: Term Frequency] */
+
+  FreeBoolTree(&(tree_base));
+ 
+  TermList_reset(term_list);
+
+  QueryTermList_reset(query_term_list);  /* [RPAP - Feb 97: Term Frequency] */
+
+  *res = yyparse();
+ 
+  return tree_base;
+}
+ 
+
+#line 408 "y.tab.c"
+#define YYABORT goto yyabort
+#define YYACCEPT goto yyaccept
+#define YYERROR goto yyerrlab
+int
+yyparse()
+{
+    register int yym, yyn, yystate;
+#if YYDEBUG
+    register char *yys;
+    extern char *getenv();
+
+    if (yys = getenv("YYDEBUG"))
+    {
+        yyn = *yys;
+        if (yyn >= '0' && yyn <= '9')
+            yydebug = yyn - '0';
+    }
+#endif
+
+    yynerrs = 0;
+    yyerrflag = 0;
+    yychar = (-1);
+
+    yyssp = yyss;
+    yyvsp = yyvs;
+    *yyssp = yystate = 0;
+
+yyloop:
+    if (yyn = yydefred[yystate]) goto yyreduce;
+    if (yychar < 0)
+    {
+        if ((yychar = yylex()) < 0) yychar = 0;
+#if YYDEBUG
+        if (yydebug)
+        {
+            yys = 0;
+            if (yychar <= YYMAXTOKEN) yys = yyname[yychar];
+            if (!yys) yys = "illegal-symbol";
+            printf("yydebug: state %d, reading %d (%s)\n", yystate,
+                    yychar, yys);
+        }
+#endif
+    }
+    if ((yyn = yysindex[yystate]) && (yyn += yychar) >= 0 &&
+            yyn <= YYTABLESIZE && yycheck[yyn] == yychar)
+    {
+#if YYDEBUG
+        if (yydebug)
+            printf("yydebug: state %d, shifting to state %d\n",
+                    yystate, yytable[yyn]);
+#endif
+        if (yyssp >= yyss + yystacksize - 1)
+        {
+            goto yyoverflow;
+        }
+        *++yyssp = yystate = yytable[yyn];
+        *++yyvsp = yylval;
+        yychar = (-1);
+        if (yyerrflag > 0)  --yyerrflag;
+        goto yyloop;
+    }
+    if ((yyn = yyrindex[yystate]) && (yyn += yychar) >= 0 &&
+            yyn <= YYTABLESIZE && yycheck[yyn] == yychar)
+    {
+        yyn = yytable[yyn];
+        goto yyreduce;
+    }
+    if (yyerrflag) goto yyinrecovery;
+#ifdef lint
+    goto yynewerror;
+#endif
+yynewerror:
+    yyerror("syntax error");
+#ifdef lint
+    goto yyerrlab;
+#endif
+yyerrlab:
+    ++yynerrs;
+yyinrecovery:
+    if (yyerrflag < 3)
+    {
+        yyerrflag = 3;
+        for (;;)
+        {
+            if ((yyn = yysindex[*yyssp]) && (yyn += YYERRCODE) >= 0 &&
+                    yyn <= YYTABLESIZE && yycheck[yyn] == YYERRCODE)
+            {
+#if YYDEBUG
+                if (yydebug)
+                    printf("yydebug: state %d, error recovery shifting\
+ to state %d\n", *yyssp, yytable[yyn]);
+#endif
+                if (yyssp >= yyss + yystacksize - 1)
+                {
+                    goto yyoverflow;
+                }
+                *++yyssp = yystate = yytable[yyn];
+                *++yyvsp = yylval;
+                goto yyloop;
+            }
+            else
+            {
+#if YYDEBUG
+                if (yydebug)
+                    printf("yydebug: error recovery discarding state %d\n",
+                            *yyssp);
+#endif
+                if (yyssp <= yyss) goto yyabort;
+                --yyssp;
+                --yyvsp;
+            }
+        }
+    }
+    else
+    {
+        if (yychar == 0) goto yyabort;
+#if YYDEBUG
+        if (yydebug)
+        {
+            yys = 0;
+            if (yychar <= YYMAXTOKEN) yys = yyname[yychar];
+            if (!yys) yys = "illegal-symbol";
+            printf("yydebug: state %d, error recovery discards token %d (%s)\n",
+                    yystate, yychar, yys);
+        }
+#endif
+        yychar = (-1);
+        goto yyloop;
+    }
+yyreduce:
+#if YYDEBUG
+    if (yydebug)
+        printf("yydebug: state %d, reducing by rule %d (%s)\n",
+                yystate, yyn, yyrule[yyn]);
+#endif
+    yym = yylen[yyn];
+    yyval = yyvsp[1-yym];
+    switch (yyn)
+    {
+case 1:
+#line 77 "bool_parser.ypp"
+{ tree_base = yyvsp[0].node;}
+break;
+case 2:
+#line 81 "bool_parser.ypp"
+{ yyval.node = CreateBoolTermNode(term_list, yyvsp[0].text, 1, word_num, count, doc_count, invf_ptr, invf_len, stemmer_num); }
+break;
+case 3:
+#line 82 "bool_parser.ypp"
+{ yyval.node = yyvsp[-1].node; }
+break;
+case 4:
+#line 83 "bool_parser.ypp"
+{ yyval.node = CreateBoolTreeNode(N_all, NULL, NULL); }
+break;
+case 5:
+#line 84 "bool_parser.ypp"
+{ yyval.node = CreateBoolTreeNode(N_none, NULL, NULL); }
+break;
+case 7:
+#line 88 "bool_parser.ypp"
+{ yyval.node = CreateBoolTreeNode(N_not, yyvsp[0].node, NULL); }
+break;
+case 8:
+#line 91 "bool_parser.ypp"
+{ yyval.node = CreateBoolTreeNode(N_and, yyvsp[-2].node, yyvsp[0].node); }
+break;
+case 9:
+#line 92 "bool_parser.ypp"
+{ yyval.node = CreateBoolTreeNode(N_and, yyvsp[-1].node, yyvsp[0].node); }
+break;
+case 11:
+#line 96 "bool_parser.ypp"
+{ yyval.node = CreateBoolTreeNode(N_or, yyvsp[-2].node, yyvsp[0].node); }
+break;
+#line 584 "y.tab.c"
+    }
+    yyssp -= yym;
+    yystate = *yyssp;
+    yyvsp -= yym;
+    yym = yylhs[yyn];
+    if (yystate == 0 && yym == 0)
+    {
+#if YYDEBUG
+        if (yydebug)
+            printf("yydebug: after reduction, shifting from state 0 to\
+ state %d\n", YYFINAL);
+#endif
+        yystate = YYFINAL;
+        *++yyssp = YYFINAL;
+        *++yyvsp = yyval;
+        if (yychar < 0)
+        {
+            if ((yychar = yylex()) < 0) yychar = 0;
+#if YYDEBUG
+            if (yydebug)
+            {
+                yys = 0;
+                if (yychar <= YYMAXTOKEN) yys = yyname[yychar];
+                if (!yys) yys = "illegal-symbol";
+                printf("yydebug: state %d, reading %d (%s)\n",
+                        YYFINAL, yychar, yys);
+            }
+#endif
+        }
+        if (yychar == 0) goto yyaccept;
+        goto yyloop;
+    }
+    if ((yyn = yygindex[yym]) && (yyn += yystate) >= 0 &&
+            yyn <= YYTABLESIZE && yycheck[yyn] == yystate)
+        yystate = yytable[yyn];
+    else
+        yystate = yydgoto[yym];
+#if YYDEBUG
+    if (yydebug)
+        printf("yydebug: after reduction, shifting from state %d \
+to state %d\n", *yyssp, yystate);
+#endif
+    if (yyssp >= yyss + yystacksize - 1)
+    {
+        goto yyoverflow;
+    }
+    *++yyssp = yystate;
+    *++yyvsp = yyval;
+    goto yyloop;
+yyoverflow:
+    yyerror("yacc stack overflow");
+yyabort:
+    return (1);
+yyaccept:
+    return (0);
+}
Index: /tags/gsdl-21/gsdl/src/mgpp/text/bool_parser.h
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/bool_parser.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/bool_parser.h	(revision 699)
@@ -0,0 +1,70 @@
+/**************************************************************************
+ *
+ * filename -- description
+ * Copyright (C) 1994  Authors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/10/11 02:57:04  cs025
+   Base install of MG-PP
+
+   Revision 1.1  1999/08/10 21:17:43  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:40  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:34:26  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/10/20  03:56:33  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.1  1994/10/12  01:15:31  tes
+   * Found bugs in the existing boolean query optimiser.
+   * So decided to rewrite it.
+   * I accidentally deleted query.bool.y, but I have replaced it
+   * with bool_parser.y (which I have forgotten to add here ! ;-(
+   *
+ */
+
+#ifndef BOOL_PARSER_H
+#define BOOL_PARSER_H
+
+#include "backend.h"
+#include "term_lists.h"
+#include "invf.h"
+
+extern "C" bool_tree_node *ParseBool (char *query_line, int query_len,
+			   TermList ** the_term_list, int the_stem_method, 
+                           int the_stemmer_num,  int *res,
+			   stemmed_dict * the_sd, int is_indexed,  /* [RPAP - Jan 97: Stem Index Change] */
+			   QueryTermList ** the_query_term_list);  /* [RPAP - Feb 97: Term Frequency] */
+
+#endif
Index: /tags/gsdl-21/gsdl/src/mgpp/text/bool_parser.ypp
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/bool_parser.ypp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/bool_parser.ypp	(revision 699)
@@ -0,0 +1,319 @@
+/**************************************************************************
+ *
+ * bool_parser - boolean query parser
+ * Copyright (C) 1994  Neil Sharman & Tim Shimmin
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *
+ **************************************************************************/
+ 
+/**************************************************************************/
+%{
+ 
+#include "sysfuncs.h"
+
+#include "messages.h"
+ 
+#include "memlib.h"
+#include "words.h"
+#include "stemmer.h"
+#include "term_lists.h"
+#include "bool_tree.h"
+/* [RPAP - Jan 97: Stem Index Change] */
+#include "backend.h"     /* for stemmed_dict def */
+#include "stem_search.h"
+
+#include "query_term_list.h"  /* [RPAP - Feb 97: Term Frequency] */
+
+/* --- routines --- */
+static int query_lex(char **, const char *);
+static int yyerror(char *);
+int yyparse();
+#define yylex() query_lex(&ch_buf, end_buf)
+ 
+/* --- module variables --- */
+static char *ch_buf; /* ptr to the character query line buffer */
+static char *end_buf; /* ptr to the last character of the line buffer */
+static bool_tree_node *tree_base = NULL;
+static TermList **term_list;
+static int stem_method;
+/* [RPAP - Jan 97: Stem Index Change] */
+stemmed_dict *p__sd;
+static int indexed;
+/* [RPAP - Feb 97: Term Frequency] */
+static QueryTermList **query_term_list;
+static int word_num;
+static u_long count;
+static u_long doc_count;
+static u_long invf_ptr;
+static u_long invf_len;
+static int stemmer_num; /* George Buchanan - Aug 99: Stemmer id */
+%}
+ 
+ 
+%union {
+  char *text;
+  bool_tree_node *node;
+}
+ 
+%token <text> TERM
+%type <node> query term not and or
+ 
+%%
+ 
+query: or  { tree_base = $1;}
+;
+ 
+ 
+term:     TERM  { $$ = CreateBoolTermNode(term_list, $1, 1, word_num, count, doc_count, invf_ptr, invf_len, stemmer_num); }
+        | '(' or ')' { $$ = $2; }
+        | '*' { $$ = CreateBoolTreeNode(N_all, NULL, NULL); }
+        | '_' { $$ = CreateBoolTreeNode(N_none, NULL, NULL); }
+;
+ 
+not:      term
+        | '!' not { $$ = CreateBoolTreeNode(N_not, $2, NULL); }
+;
+ 
+and:      and '&' not { $$ = CreateBoolTreeNode(N_and, $1, $3); }
+        | and not { $$ = CreateBoolTreeNode(N_and, $1, $2); }
+        | not
+;
+ 
+or:       or '|' and { $$ = CreateBoolTreeNode(N_or, $1, $3); }
+        | and
+;
+ 
+%%
+ 
+/* Bison on one mips machine defined "const" to be nothing but
+   then did not undef it */
+#ifdef const
+#undef const
+#endif
+
+/**************************************************************************/ 
+
+
+/* =========================================================================
+ * Function: query_lex
+ * Description:
+ *      Hand written lexical analyser for the parser.
+ * Input:
+ *      ptr = ptr to a ptr into character query-line buffer
+ *      end = ptr to last char in buffer
+ * Output:
+ *      yylval.text = the token's text
+ * Notes:
+ *      does NOT produce WILD tokens at the moment
+ * ========================================================================= */
+ 
+/* [RPAP - Jan 97: Stem Index Change]
+   state mode:
+      0 = Read next token
+      1 = Output word
+      2 = Output '|' or ')'
+ */
+static int query_lex(char **ptr, const char *end)
+{
+  char *buf_ptr = *ptr;
+  static int mode = 0;
+  static int termnum = 0;
+  static TermList *Terms = NULL;
+
+  if (mode == 0)
+    {
+      /* jump over whitespace */
+      while (isspace(*buf_ptr))
+	buf_ptr++;
+ 
+      if (inaword((unsigned char *) buf_ptr, (unsigned char *) end))
+	{
+	  static char word[MAXSTEMLEN + 1]; /* [RJM 07/98: Memory Leak] */
+	  char *sWord = (char *) Xmalloc(MAXSTEMLEN + 1);
+	  int stem_to_apply, method_using = -1;
+
+	  PARSE_STEM_WORD((unsigned char *) word, (unsigned char *) buf_ptr, 
+                          (unsigned char *) end);
+
+	  /* Extract any parameters */
+	  stem_to_apply = stem_method;
+	  while (buf_ptr <= end)
+	    {
+	      int stem_param, param_type;
+	      char param[MAXPARAMLEN + 1];
+
+	      param_type = 0;
+	      PARSE_OPT_TERM_PARAM (param, param_type, buf_ptr, end);
+	      if (!param_type)
+		break;
+
+	      if (param_type == STEMPARAM)
+		{
+		  stem_param = atoi (param);
+		  if (errno != ERANGE && indexed && stem_param >= 0 && stem_param <= 3)
+		    method_using = stem_to_apply = stem_param;
+		}
+	    }
+
+	  bcopy ((char *) word, (char *) sWord, *word + 1);
+	  stemmer (stem_to_apply, stemmer_num, (unsigned char *) sWord);
+
+	  if (stem_to_apply == 0 || !indexed || p__sd == NULL)
+	    {
+	      /* [RPAP - Feb 97: Term Frequency] */
+	      word_num = FindWord (p__sd, (unsigned char *) sWord, &count, &doc_count, &invf_ptr, &invf_len);
+	      if (word_num == -1)
+		count = doc_count = invf_ptr = invf_len = 0;
+	      QueryTermList_AddQueryTerm (query_term_list, (u_char *) word, count, method_using);
+
+	      yylval.text = word;
+	      *ptr = buf_ptr; /* fix up ptr */
+	      Xfree (sWord);
+	      return TERM;
+	    }
+	  else
+	    {
+	      *ptr = buf_ptr; /* fix up ptr */
+	      termnum = 0;
+	      TermList_reset (&Terms);
+	      if (FindWords (p__sd, (u_char *) sWord, stem_to_apply, &Terms) > 0)
+		{
+		  /* [RPAP - Feb 97: Term Frequency] */ 
+		  int i, freq = 0;
+		  for (i = 0; i < Terms->num; i++)
+		    freq += Terms->TE[i].WE.count;
+		  QueryTermList_AddQueryTerm (query_term_list, (unsigned char *) word, freq, method_using);
+
+		  Xfree (sWord);
+		  mode = 1;
+		  return '(';
+		}
+	      else
+		{
+		  /* Word does not exists - include in tree anyway */
+		  Xfree (sWord);
+
+		  /* [RPAP - Feb 97: Term Frequency] */
+		  word_num = -1;
+		  count = doc_count = invf_ptr = invf_len = 0;
+		  QueryTermList_AddQueryTerm (query_term_list, (u_char *) word, count, method_using);
+
+		  yylval.text = word;
+		  return TERM;
+		}
+	    }
+	}
+      else /* NON-WORD */
+	{
+	  if (*buf_ptr == '\0')
+	    {
+	      /* return null-char if it is one */
+	      *ptr = buf_ptr; /* fix up ptr */
+	      return 0;
+	    }
+	  else
+	    {
+	      /* return 1st char, and delete from buffer */
+	      char c = *buf_ptr++;
+	      *ptr = buf_ptr; /* fix up ptr */
+	      return c;
+	    }
+	}
+    }
+  else if (mode == 1)
+    {
+      yylval.text = (char *) Terms->TE[termnum].Word;
+      
+      /* [RPAP - Feb 97: Term Frequency] */
+      word_num = Terms->TE[termnum].WE.word_num;
+      count = Terms->TE[termnum].WE.count;
+      doc_count = Terms->TE[termnum].WE.doc_count;
+      invf_ptr = Terms->TE[termnum].WE.invf_ptr;
+      invf_len = Terms->TE[termnum].WE.invf_len;
+
+      termnum++;
+      mode = 2;
+      return TERM;
+    }
+  else  /* mode == 2 */
+    {
+      if (termnum >= Terms->num)
+	{
+	  mode = 0;
+	  return ')';
+	}
+      else
+	{
+	  mode = 1;
+	  return '|';
+	}
+    }
+}/*query_lex*/
+
+/* =========================================================================
+ * Function: yyerror
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */ 
+static int yyerror(char *s)
+{
+  Message("%s", s);
+  return(1);
+}
+
+ 
+/* =========================================================================
+ * Function: ParseBool
+ * Description:
+ *      Parse a boolean query string into a term-list and a boolean parse tree
+ * Input:
+ *      query_line = query line string
+ *      query_len = query line length
+ *      the_stem_method = stem method id used for stemming
+ * Output:
+ *      the_term_list = the list of terms
+ *      res = parser result code
+ * ========================================================================= */
+ 
+extern "C" bool_tree_node *
+ParseBool(char *query_line, int query_len,
+          TermList **the_term_list, int the_stem_method, int the_stemmer_num, int *res,
+	  stemmed_dict * the_sd, int is_indexed,   /* [RPAP - Jan 97: Stem Index Change] */
+	  QueryTermList **the_query_term_list)  /* [RPAP - Feb 97: Term Frequency] */
+{ /* global variables to be accessed by bison/yacc created parser */
+  term_list = the_term_list;
+  stem_method = the_stem_method;
+  stemmer_num = the_stemmer_num;
+  ch_buf = query_line;
+  end_buf = query_line + query_len;
+  p__sd = the_sd;   /* [RPAP - Jan 97: Stem Index Change] */
+  indexed = is_indexed;  /* [RPAP - Jan 97: Stem Index Change] */
+  query_term_list = the_query_term_list; /* [RPAP - Feb 97: Term Frequency] */
+
+  FreeBoolTree(&(tree_base));
+ 
+  TermList_reset(term_list);
+
+  QueryTermList_reset(query_term_list);  /* [RPAP - Feb 97: Term Frequency] */
+
+  *res = yyparse();
+ 
+  return tree_base;
+}
+ 
+
Index: /tags/gsdl-21/gsdl/src/mgpp/text/bool_query.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/bool_query.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/bool_query.cpp	(revision 699)
@@ -0,0 +1,328 @@
+/**************************************************************************
+ *
+ * bool_query.c -- Boolean query evaluation
+ * Copyright (C) 1994  Neil Sharman & Tim Shimmin
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "filestats.h"
+#include "timing.h"
+#include "sptree.h"
+
+
+#include "mg.h"
+#include "text.h"
+#include "invf.h"
+#include "lists.h"
+#include "term_lists.h"
+#include "backend.h"
+#include "DocEntry.h"
+#include "words.h"
+#include "stemmer.h"
+#include "stem_search.h"
+#include "invf_get.h"
+#include "text_get.h"
+#include "read_line.h"
+#include "bool_tree.h"
+#include "bool_parser.h"
+#include "bool_optimiser.h"
+#include "environment.h"
+
+
+/**********************
+  [RPAP - Feb 97: Term Frequency]
+
+  Shifted to bool_parser.y
+
+  *********************/
+
+/* =========================================================================
+ * Function: ReadInTermInfo
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+/**************************
+static void 
+ReadInTermInfo (query_data * qd)
+{
+  int i;
+  TermList *tl = qd->TL;
+
+  for (i = 0; i < tl->num; i++)
+    {
+      TermEntry *te = &(tl->TE[i]);
+      WordEntry *we = &(te->WE);
+
+       we->word_num = FindWord (qd->sd, te->Word,
+			       &we->count, &we->doc_count,
+			       &we->invf_ptr, &we->invf_len);
+      if (we->word_num == -1)
+	we->count = we->doc_count = 0;
+    }
+}
+
+***********************/
+
+
+
+/* =========================================================================
+ * Function: FindNoneTerms 
+ * Description: 
+ *      Set any terms which are not in dictionary to N_none (i.e. false)
+ *      Assumes a prior call to ReadInTermInfo
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void 
+FindNoneTerms (query_data * qd, bool_tree_node * tree)
+{
+  if (BOOL_HAS_CHILD (tree))
+    {
+      bool_tree_node *child = BOOL_CHILD (tree);
+      while (child)
+	{
+	  FindNoneTerms (qd, child);
+	  child = BOOL_SIBLING (child);
+	}
+    }
+  else if (BOOL_TAG (tree) == N_term)
+    {
+      int doc_count = GetNthWE(qd->TL, BOOL_TERM (tree))->doc_count;
+      if (!doc_count)
+	BOOL_TAG (tree) = N_none;
+    }
+
+}				/*FindNoneTerms */
+
+
+/* =========================================================================
+ * Function: BooleanGet
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+static DocList *
+BooleanGet (query_data * qd, bool_tree_node * n,
+	    BooleanQueryInfo * bqi)
+{
+  if (n)
+    {
+      DocList *res = NULL;
+      switch (BOOL_TAG (n))
+	{
+	case N_all:
+	  Message ("Unexpected \"all\" node in the parse tree.");
+	  res = DocList_create (0);
+	  break;
+	case N_not:
+	  Message ("Unexpected \"not\" node in the parse tree.");
+	  res = DocList_create (0);
+	  break;
+	case N_none:
+	  {
+	    res = DocList_create (0);
+	    break;
+	  }
+	case N_diff:
+	  {
+	    res = BooleanGet (qd, BOOL_CHILD (n), bqi);
+	    n = BOOL_SIBLING (BOOL_CHILD (n));
+	    while (n)
+	      {
+		bool_tree_node *next = BOOL_SIBLING (n);
+		if (BOOL_TAG (n) == N_term)
+		  {
+		    WordEntry *we = GetNthWE (qd->TL, BOOL_TERM (n));
+		    res = GetDocsOp (qd, we, op_diff_terms, res);
+		  }
+		else
+		  res = DocList_diff (res, BooleanGet (qd, n, bqi));
+		n = next;
+	      }
+	    break;
+	  }
+	case N_and:
+	  {
+	    res = BooleanGet (qd, BOOL_CHILD (n), bqi);		/* initial fill of c-set */
+	    n = BOOL_SIBLING (BOOL_CHILD (n));
+	    while (n)
+	      {
+		bool_tree_node *next = BOOL_SIBLING (n);
+		switch (BOOL_TAG (n))
+		  {
+		  case N_term:
+		    {
+		      WordEntry *we = GetNthWE (qd->TL, BOOL_TERM (n));
+		      res = GetDocsOp (qd, we, op_and_terms, res);
+		    }
+		    break;
+		  case N_or_terms:
+		    {
+		      bool_tree_node *child = BOOL_CHILD (n);
+		      /* note: all children are terms - predetermined */
+		      while (child)
+			{
+			  WordEntry *we = GetNthWE (qd->TL, BOOL_TERM (child));
+			  res = GetDocsOp (qd, we, op_and_or_terms, res);
+			  child = BOOL_SIBLING (child);
+			}
+		      DocList_reduceOrIncluded (res);
+		    }
+		    break;
+		  default:
+		    res = DocList_intersect (res, BooleanGet (qd, n, bqi));
+		  }		/*switch */
+		n = next;
+	      }
+	    break;
+	  }
+	case N_or:
+	case N_or_terms:
+	  {
+	    res = BooleanGet (qd, BOOL_CHILD (n), bqi);
+	    n = BOOL_SIBLING (BOOL_CHILD (n));
+	    while (n)
+	      {
+		bool_tree_node *next = BOOL_SIBLING (n);
+		res = DocList_merge (res, BooleanGet (qd, n, bqi));
+		n = next;
+	      }
+	    break;
+	  }
+	case N_term:
+	  {
+	    res = GetDocsOp (qd, GetNthWE (qd->TL, BOOL_TERM (n)), op_term, NULL);
+	    break;
+	  }
+	}
+      return (res ? res : DocList_create (0));
+    }
+  else
+    return DocList_create (0);
+}
+
+/* =========================================================================
+ * Function: ReduceList_UniqueDocs (was bool_query.AdjustParaDocs)
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+void
+ReduceList_UniqueDocs (query_data * qd, DocList * Docs)
+{
+  int s, d;
+  Splay_Tree *SP;
+
+
+  /* Make the doc-list contain only references to unique
+   * real documents. So can get rid of cases where there
+   * are multiple paragraphs for a real document.
+   */
+
+  /* Note: [TS: 29/Sep/94] */
+  /* reads in seek-pos for docnums so can compare docs */
+  /* seek-pos used as real doc identifier */
+  /* whereas doc-num entries refer to paragraphs -- I think */
+
+  /**
+   * Note: GRB 10/Sep/99
+   * DocNum entries *DO* refer to paragraphs; see note in
+   * backend.InitQuerySystem which articulates this more
+   * fully
+   */
+  for (s = 0; s < Docs->size(); s++)
+    Docs->member(s)->FetchStart (qd);
+
+  Message ("Original Number = %d\n", Docs->size());
+  SP = SP_createset (DocEntry_compare);
+  d = 0;
+  for (s = 0; s < Docs->size(); s++)
+    if (!SP_member (Docs->member(s), SP))
+      { DocList_moveMember(Docs, d, s);
+	SP_insert (Docs->member(d), SP);
+	d++;
+      }
+  SP_freeset (SP);
+  DocList_setsize(Docs, d);
+  Message ("Final Number = %d\n", Docs->size());
+}
+
+/* =========================================================================
+ * Function: BooleanQuery 
+ * Description: 
+ *      The main control function for the boolean query.
+ * Input: 
+ * Output: 
+ *      qd->DL
+ * ========================================================================= */
+
+void 
+BooleanQuery (query_data * qd, char *Query, BooleanQueryInfo * bqi)
+{
+  bool_tree_node *tree;
+  DocList *Docs;
+  int res = 0;
+
+  tree = ParseBool (Query, MAXLINEBUFFERLEN, &(qd->TL), 
+		    bqi->getWordSpecial(), qd->sd->sdh.stemmer_num, &res,
+		    qd->sd, qd->sd->sdh.indexed,         /* [RPAP - Jan 97: Stem Index Change] */
+		    &(qd->QTL));                         /* [RPAP - Feb 97: Term Frequency] */
+
+  /* [TS:Aug/94] no longer tries to continue after parse failure */
+  if (res != 0)			/* failed the parse */
+    {
+      qd->DL = NULL;		/* have empty doc-list */
+      return;
+    }
+
+  /* [RPAP - Feb 97: Term Frequency] */
+  /*  ReadInTermInfo (qd); */
+
+  FindNoneTerms (qd, tree);
+
+  OptimiseBoolTree (tree, qd->TL, bqi->OptimiseType);
+
+  qd->hops_taken = qd->num_of_ptrs = 0;
+  qd->num_of_terms = 0;
+
+  Docs = BooleanGet (qd, tree, bqi);
+
+/* [RPAP - Feb 97: NZDL Specifics] */
+#if !defined(PARABOOLEANMATCHES) && !defined(NZDL)
+  if (qd->id->ifh.InvfLevel == 3)
+    {
+      ReduceList_UniqueDocs (qd, Docs);
+    }
+#endif
+
+  if (bqi->MaxDocsToRetrieve != -1 && 
+      Docs->size() > bqi->MaxDocsToRetrieve)
+    DocList_setsize(Docs, bqi->MaxDocsToRetrieve);
+
+  QueryData_FreeQueryDocs (qd);
+
+  qd->DL = Docs;
+}
+
+
Index: /tags/gsdl-21/gsdl/src/mgpp/text/bool_query.h
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/bool_query.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/bool_query.h	(revision 699)
@@ -0,0 +1,58 @@
+/**************************************************************************
+ *
+ * bool_query.h -- boolean query processing
+ * Copyright (C) 1994  Authors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.2  1999/10/12 04:14:41  cs025
+   BooleanQuery interface had been changed to eradicate the use of
+   GetEnv; header altered accordingly.
+
+   Revision 1.1  1999/10/11 02:57:05  cs025
+   Base install of MG-PP
+
+   Revision 1.1  1999/08/10 21:17:45  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:34:28  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/10/20  03:56:36  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.1  1994/10/12  01:15:33  tes
+   * Found bugs in the existing boolean query optimiser.
+   * So decided to rewrite it.
+   * I accidentally deleted query.bool.y, but I have replaced it
+   * with bool_parser.y (which I have forgotten to add here ! ;-(
+   *
+ */
+
+#ifndef BOOL_QUERY_H
+#define BOOL_QUERY_H
+
+#include "backend.h"
+
+void BooleanQuery (query_data * qd, char *Query, BooleanQueryInfo * bqi);
+
+#endif
Index: /tags/gsdl-21/gsdl/src/mgpp/text/bool_tester.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/bool_tester.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/bool_tester.cpp	(revision 699)
@@ -0,0 +1,174 @@
+/**************************************************************************
+ *
+ * bool_tester -- used to test bool_tree, bool_parser, bool_optimiser
+ * Copyright (C) 1994  Authors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/10/11 02:57:06  cs025
+   Base install of MG-PP
+
+   Revision 1.1  1999/08/10 21:17:45  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:41  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:34:29  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.2  1995/03/14  05:15:23  tes
+   * Updated the boolean query optimiser to do different types of optimisation.
+   * A query environment variable "optimise_type" specifies which one is to be
+   * used. Type 1 is the new one which is faster than 2.
+   *
+   * Revision 1.2  1994/10/18  06:11:05  tes
+   * The boolean optimiser seems to be modifying the parse tree
+   * like it is supposed to.
+   * Paragraph ranking now works without any text files if required to.
+   *
+   * Revision 1.1  1994/10/12  01:15:33  tes
+   * Found bugs in the existing boolean query optimiser.
+   * So decided to rewrite it.
+   * I accidentally deleted query.bool.y, but I have replaced it
+   * with bool_parser.y (which I have forgotten to add here ! ;-(
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+#include "local_strings.h"
+#include "term_lists.h"
+#include "bool_tree.h"
+#include "bool_parser.h"
+#include "bool_optimiser.h"
+#include "query_term_list.h"  /* [RPAP - Feb 97: Term Frequency] */
+
+#define MAX_LINE_LEN  255
+#define STEMMER_NUM 0 /* Lovin's stemmer */
+#define STEM_METHOD 3
+
+static FILE *file_in = stdin;
+static FILE *file_out = stdout;
+static char line[MAX_LINE_LEN + 1];
+
+/* --- prototypes --- */
+static char *prompt (char *str);
+
+
+/* =========================================================================
+ * Function: main
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+int main (int argc, char *argv[])
+{
+  bool_tree_node *tree = NULL;
+  TermList *term_list = NULL;
+  QueryTermList *query_term_list = NULL;  /* [RPAP - Feb 97: Term Frequency] */
+  int opt_type = 0;
+
+  while (1)
+    {
+      int res = 0;
+      int len = 0;
+
+      /* get a line */
+      if (!prompt ("Please enter boolean expression..."))
+	break;
+
+      len = strlen (line) - 1;	/* -1 => ignore the \n */
+
+      tree = ParseBool (line, len, &term_list, STEMMER_NUM, STEM_METHOD, &res,
+			NULL, 0,             /* [RPAP - Jan 97: Stem Index Change] */
+			&query_term_list);   /* [RPAP - Feb 97: Term Frequency] */
+
+      {
+	int done = 0;
+	while (!done)
+	  {
+	    if (!prompt ("Which type of optimisation 1 or 2 ?"))
+	      break;
+	    opt_type = atoi (line);
+	    done = (opt_type == 1 || opt_type == 2);
+	  }
+      }
+
+      if (!prompt ("Do you want to assign doc_counts to terms ?"))
+	break;
+
+      if (toupper (line[0]) == 'Y')
+	{
+	  /* cycle thru terms asking for doc_counts */
+	  int i;
+	  for (i = 0; i < term_list->num; i++)
+	    {
+	      char str[80];
+	      TermEntry *te = &(term_list->TE[i]);
+
+	      sprintf (str, "Please enter doc count for term '%s'",
+		       str255_to_string (te->Word, NULL));
+	      prompt (str);
+	      te->WE.doc_count = atoi (line);
+	      printf ("doc_count = %ld\n", te->WE.doc_count);
+	    }			/*for */
+	}			/*if */
+
+
+      if (res == 0)
+	{
+	  fprintf (file_out, "\n***Parsed Expression***\n");
+	  PrintBoolTree (tree, file_out);
+	  fputc ('\n', file_out);
+	  OptimiseBoolTree (tree, term_list, opt_type);
+	  fprintf (file_out, "\n***Optimised Expression ***\n");
+	  PrintBoolTree (tree, file_out);
+	  fputc ('\n', file_out);
+	}
+
+    }
+  return 0;
+}
+
+/* =========================================================================
+ * Function: 
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static char *
+prompt (char *str)
+{
+  fprintf (file_out, "\n%s\n", str);
+  return fgets (line, MAX_LINE_LEN, file_in);
+}
Index: /tags/gsdl-21/gsdl/src/mgpp/text/bool_tree.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/bool_tree.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/bool_tree.cpp	(revision 699)
@@ -0,0 +1,292 @@
+/**************************************************************************
+ *
+ * bool_tree.c -- Boolean parse tree ADT
+ * Copyright (C) 1994  Tim Shimmin
+ * 
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *       @(#)query.bool.y	1.9 16 Mar 1994
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "bool_tree.h"
+
+static char *RCSID = "$Id$";
+
+
+/* =========================================================================
+ * Function: CreateBoolNode
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+bool_tree_node *
+CreateBoolNode (N_Tag tag)
+{
+  bool_tree_node *n;
+  if (!(n = (bool_tree_node *) Xmalloc (sizeof (bool_tree_node))))
+    FatalError (1, "Can not create bool_tree_node for boolean query");
+  n->tag = tag;
+  BOOL_SIBLING (n) = NULL;
+  return n;
+}
+
+/* =========================================================================
+ * Function: 
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+bool_tree_node *
+CreateBoolTermNode (TermList ** tl, char *text, int Count, int word_num,
+		    u_long count, u_long doc_count, u_long invf_ptr, u_long invf_len,  /* [RPAP - Feb 97: Term Frequency] */
+		    int stemmer_num) 
+{
+  bool_tree_node *n = NULL;
+
+  /* allocate bool_tree_node */
+  n = CreateBoolNode (N_term);
+
+  BOOL_TERM (n) = TermList_AddTerm (*tl, (u_char *) text, Count, word_num, count, doc_count, 
+			   invf_ptr, invf_len, stemmer_num);  /* [RPAP - Feb 97: Term Frequency] */
+
+  return n;
+}
+
+
+/* =========================================================================
+ * Function: CreateTreeNode
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+bool_tree_node *
+CreateBoolTreeNode (N_Tag tag, bool_tree_node * left, bool_tree_node * right)
+{
+  bool_tree_node *n = CreateBoolNode (tag);
+
+  BOOL_CHILD (n) = left;
+  if (left)
+    BOOL_SIBLING (left) = right;
+  if (right)
+    BOOL_SIBLING (right) = NULL;
+
+  return (n);
+}
+
+/* =========================================================================
+ * Function: 
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+int
+EqualBoolNodes (bool_tree_node * a, bool_tree_node * b)
+{
+  if (BOOL_TAG (a) == N_term && BOOL_TAG (a) == N_term)
+    return BOOL_TERM (a) == BOOL_TERM (b);
+  return 0;
+}
+
+/* =========================================================================
+ * Function: FreeBoolTree
+ * Description: 
+ *      Notes: does not fix up any parent or sibling links around given tree
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void 
+FreeTree (bool_tree_node * tree)
+{
+  bool_tree_node *sibling;
+
+  while (tree)
+    {
+      if (BOOL_HAS_CHILD (tree))
+	FreeTree (BOOL_CHILD (tree));
+      sibling = BOOL_SIBLING (tree);
+      Xfree (tree);
+      tree = sibling;
+    }
+}
+
+void 
+FreeBoolTree (bool_tree_node ** the_tree)
+{
+  bool_tree_node *tree = *the_tree;
+
+  if (!tree)
+    return;
+
+  FreeTree (tree);
+  *the_tree = NULL;
+/*********************************/
+  /* THIS IS POTENTIALLY DANGEROUS */
+  /* FIX UP LATER */
+  /* Need to correct the links for parent/siblings */
+/*********************************/
+
+}
+
+/* =========================================================================
+ * Function: PrintBoolTree
+ * Description: 
+ *      Print out a text version of the tree to the file
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+PrintBinaryOp (bool_tree_node * tree, FILE * file, char op)
+{
+  bool_tree_node *child = NULL;
+  assert (BOOL_HAS_CHILD (tree));
+
+  child = BOOL_CHILD (tree);
+  while (child)
+    {
+      PrintBoolTree (child, file);
+      child = BOOL_SIBLING (child);
+      if (child)
+	fprintf (file, " %c ", op);
+    }
+}
+
+static void
+PrintUnaryOp (bool_tree_node * tree, FILE * file, char op)
+{
+  bool_tree_node *child = NULL;
+
+  fprintf (file, " %c ", op);
+  assert (BOOL_HAS_CHILD (tree));
+  child = BOOL_CHILD (tree);
+  PrintBoolTree (child, file);
+}
+
+void
+PrintBoolTree (bool_tree_node * tree, FILE * file)
+{
+  if (!tree)
+    return;
+
+  fprintf (file, "(");
+  switch (BOOL_TAG (tree))
+    {
+    case N_term:
+      fprintf (file, "term %d", BOOL_TERM (tree));
+      break;
+    case N_and:
+      PrintBinaryOp (tree, file, '&');
+      break;
+    case N_or:
+      PrintBinaryOp (tree, file, '|');
+      break;
+    case N_or_terms:
+      PrintBinaryOp (tree, file, '+');
+      break;
+    case N_diff:
+      PrintBinaryOp (tree, file, '-');
+      break;
+    case N_not:
+      PrintUnaryOp (tree, file, '!');
+      break;
+    case N_all:
+      fprintf (file, " TRUE ");
+      break;
+    case N_none:
+      fprintf (file, " FALSE ");
+      break;
+    default:
+      break;
+    }
+
+  fprintf (file, ")");
+}
+
+
+/* =========================================================================
+ * Function: CopyBoolTree
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+bool_tree_node *
+CopyBoolTree (bool_tree_node * tree)
+{
+  bool_tree_node *sibling = NULL;
+  bool_tree_node *tree_copy = NULL;
+  bool_tree_node *prev = NULL;
+
+  if (!tree)
+    return NULL;
+
+  sibling = tree;
+  prev = NULL;
+  while (sibling)
+    {
+      /* copy sibling */
+      bool_tree_node *sibling_copy = (bool_tree_node *) malloc (sizeof (bool_tree_node));
+      if (!sibling_copy)
+	FatalError (1, "No memory for bool tree copy");
+      bcopy ((char *) sibling, (char *) sibling_copy, sizeof (bool_tree_node));
+
+      if (BOOL_HAS_CHILD (sibling))
+	{
+	  BOOL_CHILD (sibling_copy) = CopyBoolTree (BOOL_CHILD (sibling));
+	}
+
+      /* do the link up of copies */
+      if (prev)
+	BOOL_SIBLING (prev) = sibling_copy;
+      else
+	tree_copy = sibling_copy;
+      prev = sibling_copy;
+
+      sibling = BOOL_SIBLING (sibling);
+    }
+  return tree_copy;
+}
+
+
+/* =========================================================================
+ * Function: LastBoolSibling
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+bool_tree_node *
+LastBoolSibling (bool_tree_node * tree)
+{
+
+  if (!tree)
+    return NULL;
+
+  do
+    {
+      tree = BOOL_SIBLING (tree);
+    }
+  while (BOOL_SIBLING (tree));
+
+  return tree;
+}
Index: /tags/gsdl-21/gsdl/src/mgpp/text/bool_tree.h
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/bool_tree.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/bool_tree.h	(revision 699)
@@ -0,0 +1,122 @@
+/**************************************************************************
+ *
+ * bool_tree.h -- description
+ * Copyright (C) 1994  Authors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/10/11 02:57:07  cs025
+   Base install of MG-PP
+
+   Revision 1.1  1999/08/10 21:17:46  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:42  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:34:31  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.2  1995/03/14  05:15:25  tes
+   * Updated the boolean query optimiser to do different types of optimisation.
+   * A query environment variable "optimise_type" specifies which one is to be
+   * used. Type 1 is the new one which is faster than 2.
+   *
+   * Revision 1.2  1994/10/18  06:11:07  tes
+   * The boolean optimiser seems to be modifying the parse tree
+   * like it is supposed to.
+   * Paragraph ranking now works without any text files if required to.
+   *
+   * Revision 1.1  1994/10/12  01:15:35  tes
+   * Found bugs in the existing boolean query optimiser.
+   * So decided to rewrite it.
+   * I accidentally deleted query.bool.y, but I have replaced it
+   * with bool_parser.y (which I have forgotten to add here ! ;-(
+   *
+ */
+
+#ifndef BOOL_TREE_H
+#define BOOL_TREE_H
+
+#include "sysfuncs.h"
+#include "term_lists.h"
+
+typedef enum
+  {
+    N_term, N_and, N_or, N_diff, N_none, N_all,
+    N_not, N_or_terms
+  }
+N_Tag;
+
+typedef struct bool_tree_node
+  {
+    N_Tag tag;
+    struct bool_tree_node *sibling;
+    union
+      {
+	struct bool_tree_node *child;
+	int term_id;
+      }
+    fields;
+    int sum_ft;			/* for or_terms nodes */
+  }
+bool_tree_node;
+
+
+#define BOOL_HAS_CHILD(n) (\
+			(n)->tag==N_and || \
+			(n)->tag==N_or || (n)->tag==N_or_terms || \
+			(n)->tag==N_not || (n)->tag==N_diff)
+#define BOOL_TAG(n) ((n)->tag)
+#define BOOL_SIBLING(n) ((n)->sibling)
+#define BOOL_TERM(n) ((n)->fields.term_id)
+#define BOOL_CHILD(n) ((n)->fields.child)
+#define BOOL_SUM_FT(n) ((n)->sum_ft)
+
+bool_tree_node *CreateBoolNode (N_Tag tag);
+bool_tree_node *CopyBoolTree (bool_tree_node * tree);
+#ifdef __cplusplus
+extern "C"
+#endif
+bool_tree_node *CreateBoolTermNode (TermList ** tl, char *text, int Count, int word_num,
+				    u_long count, u_long doc_count, u_long invf_ptr, 
+				    u_long invf_len, int stemmer_num);
+#ifdef __cplusplus
+extern "C"
+#endif
+bool_tree_node *CreateBoolTreeNode (N_Tag tag,
+			     bool_tree_node * left, bool_tree_node * right);
+#ifdef __cplusplus
+extern "C"
+#endif
+void FreeBoolTree (bool_tree_node ** tree);
+void PrintBoolTree (bool_tree_node * tree, FILE * file);
+bool_tree_node *LastBoolSibling (bool_tree_node * tree);
+
+
+#endif
Index: /tags/gsdl-21/gsdl/src/mgpp/text/commands.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/commands.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/commands.cpp	(revision 699)
@@ -0,0 +1,589 @@
+/**************************************************************************
+ *
+ * commands.c -- mgquery command processing functions
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/10/11 02:57:08  cs025
+   Base install of MG-PP
+
+   Revision 1.1  1999/08/10 21:17:47  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:34:32  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.2  1994/09/20  04:41:22  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include <stdio.h>
+
+#include "non_ansi.h"
+
+#include "sysfuncs.h"
+#include "memlib.h"
+#include "messages.h"
+
+#include "environment.h"
+#include "mg.h"
+#include "locallib.h"
+#include "backend.h"		/* for qd */
+#include "term_lists.h"		/* for MAXTERMSTRLEN */
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+# define OPENPIPE        _popen
+# define CLOSEPIPE       _pclose
+#else
+# define OPENPIPE        popen
+# define CLOSEPIPE       pclose
+#endif
+
+extern char *PathName;
+extern FILE *OutFile, *InFile;
+extern int OutPipe, InPipe;
+extern int Quitting;
+
+char *CommandsErrorStr = NULL;
+
+/* [TS: June/95] */
+/* current qd for use by any commands */
+static query_data *the_qd = NULL;
+
+
+#include "help.mg.h"
+#include "warranty.h"
+#include "conditions.h"
+
+static void 
+CmdQuit (void)
+{
+  Quitting = 1;
+}
+
+static void 
+CmdPush (void)
+{
+  PushEnv ();
+}
+
+
+static void 
+CmdPop (void)
+{
+  PopEnv ();
+  if (EnvStackHeight () == 1)
+    PushEnv ();
+}
+
+static void 
+CmdHelp (void)
+{
+  FILE *more;
+  int i;
+
+  if (!(more = OPENPIPE (GetDefEnv ("pager", "more"), "w")))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      fprintf (stderr, "ERROR : Unable to run more\n");
+      return;
+    }
+  for (i = 0; i < sizeof (help_str) / sizeof (help_str[0]); i++)
+    fputs (help_str[i], more);
+
+  CLOSEPIPE (more);  /* [RPAP - Feb 97: WIN32 Port] */
+}
+
+
+static void 
+CmdWarranty (void)
+{
+  FILE *more;
+  int i;
+
+  if (!(more = OPENPIPE (GetDefEnv ("pager", "more"), "w")))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      fprintf (stderr, "ERROR : Unable to run more\n");
+      return;
+    }
+  for (i = 0; i < sizeof (warranty_str) / sizeof (warranty_str[0]); i++)
+    fputs (warranty_str[i], more);
+
+  CLOSEPIPE (more);  /* [RPAP - Feb 97: WIN32 Port] */
+}
+
+
+static void 
+CmdConditions (void)
+{
+  FILE *more;
+  int i;
+
+  if (!(more = OPENPIPE (GetDefEnv ("pager", "more"), "w")))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      fprintf (stderr, "ERROR : Unable to run more\n");
+      return;
+    }
+  for (i = 0; i < sizeof (cond_str) / sizeof (cond_str[0]); i++)
+    fputs (cond_str[i], more);
+
+  CLOSEPIPE (more);
+}
+
+static void 
+CmdReset (void)
+{
+  while (EnvStackHeight () > 1)
+    PopEnv ();
+  PushEnv ();
+}
+
+static void 
+CmdDisplay (void)
+{
+  int i;
+  size_t l = 0;
+  char *name;
+  for (i = 0; (name = GetEnvName (i)) != NULL; i++)
+    if (strlen (name) > l)
+      l = strlen (name);
+
+  for (i = 0; (name = GetEnvName (i)) != NULL; i++)
+    printf ("%-*s = \"%s\"\n", (int) l, name, GetEnv (name));
+}
+
+static void 
+CmdQueryTerms (void)
+{
+  static char terms_str[MAXTERMSTRLEN + 1];
+
+  if (the_qd && the_qd->TL)
+    {
+      TermList_toString (the_qd->TL, terms_str);
+      printf ("%s\n", terms_str);
+      fflush (stdout);
+    }
+
+}
+
+
+static void 
+CmdSet (char *name, char *value)
+{
+  if (!*name)
+    {
+      CommandsErrorStr = "Blank names are not permitted";
+      return;
+    }
+  switch (SetEnv (name, value, NULL))
+    {
+    case -1:
+      CommandsErrorStr = "Out of memory\n";
+      break;
+    case -2:
+      CommandsErrorStr = ConstraintErrorStr;
+      break;
+    }
+}
+
+static void 
+CmdUnset (char *name)
+{
+  if (!*name)
+    {
+      CommandsErrorStr = "Blank names are not permitted";
+      return;
+    }
+  if (UnsetEnv (name, 0) == -1)
+    CommandsErrorStr = "Specified name does not exist";
+}
+
+
+
+
+
+
+/*
+ * Do the .input command 
+ */
+static void 
+CmdInput (char *param1, char *param2)
+{
+  enum
+  {
+    READ, APPEND, PIPEFROM
+  }
+  mode;
+  FILE *NewInput;
+  char *buf, *s;
+
+  /* join the two parameters together */
+  buf = (char *) Xmalloc (strlen (param1) + strlen (param2) + 1);
+  strcpy (buf, param1);
+  strcat (buf, param2);
+  s = buf;
+
+
+  while (*s == ' ' || *s == '\t')
+    s++;
+
+  if (*s == '<')
+    {
+      s++;
+      mode = READ;
+    }
+  else if (*s == '|')
+    {
+      mode = PIPEFROM;
+      s++;
+    }
+  else
+    {
+      if (InPipe)
+	CLOSEPIPE (InFile);  /* [RPAP - Feb 97: WIN32 Port] */
+      else if (InFile != stdin)
+	fclose (InFile);
+      InPipe = 0;
+      InFile = stdin;
+      Xfree (buf);
+      return;
+    }
+
+  while (*s == ' ' || *s == '\t')
+    s++;
+
+  if (*s)
+    {
+      if (mode == READ)
+	NewInput = fopen (s, "r");
+      else
+	NewInput = OPENPIPE (s, "r");  /* [RPAP - Feb 97: WIN32 Port] */
+
+      if (NewInput)
+	{
+	  if (InPipe)
+	    CLOSEPIPE (InFile);  /* [RPAP - Feb 97: WIN32 Port] */
+	  else if (InFile != stdin)
+	    fclose (InFile);
+	  InPipe = (mode == PIPEFROM);
+	  InFile = NewInput;
+	}
+      else
+	CommandsErrorStr = "Bad input file/pipe";
+    }
+  else
+    {
+      if (InPipe)
+	CLOSEPIPE (InFile);  /* [RPAP - Feb 97: WIN32 Port] */
+      else if (InFile != stdin)
+	fclose (InFile);
+      InPipe = 0;
+      InFile = stdin;
+    }
+  Xfree (buf);
+}
+
+
+
+static void 
+CmdOutput (char *param1, char *param2)
+{
+  enum
+  {
+    WRITE, APPEND, PIPETO
+  }
+  mode;
+  char *buf, *s;
+  FILE *NewOutput;
+
+  /* join the two parameters together */
+  buf = (char *) Xmalloc (strlen (param1) + strlen (param2) + 1);
+  strcpy (buf, param1);
+  strcat (buf, param2);
+  s = buf;
+
+  while (*s == ' ' || *s == '\t')
+    s++;
+
+  if (*s == '>')
+    {
+      s++;
+      mode = APPEND;
+      if (*s == '>')
+	s++;
+      else
+	mode = WRITE;
+    }
+  else if (*s == '|')
+    {
+      mode = PIPETO;
+      s++;
+    }
+  else
+    {
+      if (OutPipe)
+	CLOSEPIPE (OutFile);  /* [RPAP - Feb 97: WIN32 Port] */
+      else if (OutFile != stdout)
+	fclose (OutFile);
+      OutPipe = 0;
+      OutFile = stdout;
+      Xfree (buf);
+      return;
+    }
+
+  while (*s == ' ' || *s == '\t')
+    s++;
+
+  if (*s)
+    {
+      if (mode == PIPETO)
+	NewOutput = OPENPIPE (s, "w");  /* [RPAP - Feb 97: WIN32 Port] */
+      else
+	NewOutput = fopen (s, mode == WRITE ? "w" : "a");
+      if (NewOutput)
+	{
+	  if (OutPipe)
+	    CLOSEPIPE (OutFile);  /* [RPAP - Feb 97: WIN32 Port] */
+	  else if (OutFile != stdout)
+	    fclose (OutFile);
+	  OutPipe = (mode == PIPETO);
+	  OutFile = NewOutput;
+	}
+      else
+	CommandsErrorStr = "Bad output file/pipe";
+    }
+  else
+    {
+      if (OutPipe)
+	CLOSEPIPE (OutFile);  /* [RPAP - Feb 97: WIN32 Port] */
+      else if (OutFile != stdout)
+	fclose (OutFile);
+      OutPipe = 0;
+      OutFile = stdout;
+    }
+  Xfree (buf);
+}
+
+
+
+static char *
+ParseArg (char *line, char *buffer)
+{
+  if (*line == '\"')
+    {
+      line++;
+      while (*line != '\"' && *line != '\0')
+	{
+	  if (*line == '\\')
+	    *buffer++ = *line++;
+	  if (*line != '\0')
+	    *buffer++ = *line++;
+	}
+      if (*line == '\"')
+	line++;
+    }
+  else
+    while (*line != ' ' && *line != '\t' && *line != '\0')
+      {
+	if (*line == '\\')
+	  line++;
+	if (*line != '\0')
+	  *buffer++ = *line++;
+      }
+  *buffer = '\0';
+  while (*line == ' ' || *line == '\t')
+    line++;
+  return (line);
+}
+
+
+
+char *
+ProcessCommands (char *line, query_data * qd)
+{
+  struct
+  {
+    char *Command;
+    void (*action) (void);
+    void (*action1) (char *);
+    void (*action2) (char *, char *);
+  }
+   *dc, dot_command[] =
+  {
+    {
+      "set", NULL, NULL, CmdSet
+    }
+    ,
+    {
+      "input", NULL, NULL, CmdInput
+    }
+    ,
+    {
+      "output", NULL, NULL, CmdOutput
+    }
+    ,
+    {
+      "unset", NULL, CmdUnset, NULL
+    }
+    ,
+    {
+      "help", CmdHelp, NULL, NULL
+    }
+    ,
+    {
+      "reset", CmdReset, NULL, NULL
+    }
+    ,
+    {
+      "display", CmdDisplay, NULL, NULL
+    }
+    ,
+    {
+      "push", CmdPush, NULL, NULL
+    }
+    ,
+    {
+      "quit", CmdQuit, NULL, NULL
+    }
+    ,
+    {
+      "pop", CmdPop, NULL, NULL
+    }
+    ,
+    {
+      "warranty", CmdWarranty, NULL, NULL
+    }
+    ,
+    {
+      "conditions", CmdConditions, NULL, NULL
+    }
+    ,
+    {
+      "queryterms", CmdQueryTerms, NULL, NULL
+    }
+  };
+
+
+  char command[512];
+  char param1[512], param2[512];
+  CommandsErrorStr = NULL;
+  the_qd = qd;			/* set globally to be accessed by command functions if needed */
+  while (*line == ' ' || *line == '\t')
+    line++;
+  while (*line == '.')
+    {
+      int i;
+      line++;
+      for (i = 0; isalpha (*line) && i < 512; i++, line++)
+	command[i] = tolower (*line);
+      while (*line == ' ' || *line == '\t')
+	line++;
+      if (i != 512)
+	{
+	  command[i] = '\0';
+	  for (i = 0, dc = dot_command; i < NUMOF (dot_command); i++, dc++)
+	    if (!strcmp (dc->Command, command))
+	      {
+		if (dc->action)
+		  dc->action ();
+		else if (dc->action1)
+		  {
+		    line = ParseArg (line, param1);
+		    dc->action1 (param1);
+		    break;
+		  }
+		else if (dc->action2)
+		  {
+		    line = ParseArg (line, param1);
+		    line = ParseArg (line, param2);
+		    dc->action2 (param1, param2);
+		    break;
+		  }
+		if (CommandsErrorStr)
+		  *line = '\0';
+		break;
+	      }
+	  if (i == NUMOF (dot_command))
+	    i = 100;
+	}
+      if (i == 100)
+	{
+	  CommandsErrorStr = "Illegal command";
+	  *line = '\0';
+	}
+      while (*line == ' ' || *line == '\t')
+	line++;
+    }
+  return (line);
+}
+
+
+
+
+
+void 
+read_mgrc_file (void)
+{
+  char line[1000];
+  char FileName[256];
+  int LineNum = 0;
+  FILE *mgrc;
+
+  strcpy (FileName, ".mgrc");
+  mgrc = fopen (FileName, "r");
+
+  if (!mgrc)
+    {
+      char *home_path = getenv ("HOME");
+      if (home_path)
+	{
+	  sprintf (FileName, "%s/.mgrc", home_path);
+	  mgrc = fopen (FileName, "r");
+	}
+    }
+
+  if (!mgrc)
+    return;
+
+  while (fgets (line, sizeof (line), mgrc))
+    {
+      char *s, linebuf[1000];
+      LineNum++;
+      if ((s = strchr (line, '\n')) != NULL)
+	*s = '\0';
+      strcpy (linebuf, line);
+      if ((s = strchr (line, '#')) != NULL)
+	*s = '\0';
+      if (!*line)
+	continue;
+      ProcessCommands (line, NULL);
+      if (CommandsErrorStr)
+	Message ("%s\n%s : ERROR in line %d of %s\n%s : %s\n",
+		 linebuf,
+		 msg_prefix, LineNum, FileName,
+		 msg_prefix, CommandsErrorStr);
+    }
+  fclose (mgrc);
+}
Index: /tags/gsdl-21/gsdl/src/mgpp/text/commands.h
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/commands.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/commands.h	(revision 699)
@@ -0,0 +1,33 @@
+/**************************************************************************
+ *
+ * commands.h -- mgquery command processing functions
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef COMMANDS_H
+#define COMMANDS_H
+
+#include "backend.h"
+
+extern char *CommandsErrorStr;
+char *ProcessCommands (char *line, query_data * qd);
+void read_mgrc_file (void);
+
+#endif
Index: /tags/gsdl-21/gsdl/src/mgpp/text/conditions.h
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/conditions.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/conditions.h	(revision 699)
@@ -0,0 +1,226 @@
+static char *cond_str[] = {
+"		    GNU GENERAL PUBLIC LICENSE\n",
+"   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION\n",
+"\n",
+"  0. This License applies to any program or other work which contains\n",
+"a notice placed by the copyright holder saying it may be distributed\n",
+"under the terms of this General Public License.  The \"Program\", below,\n",
+"refers to any such program or work, and a \"work based on the Program\"\n",
+"means either the Program or any derivative work under copyright law:\n",
+"that is to say, a work containing the Program or a portion of it,\n",
+"either verbatim or with modifications and/or translated into another\n",
+"language.  (Hereinafter, translation is included without limitation in\n",
+"the term \"modification\".)  Each licensee is addressed as \"you\".\n",
+"\n",
+"Activities other than copying, distribution and modification are not\n",
+"covered by this License; they are outside its scope.  The act of\n",
+"running the Program is not restricted, and the output from the Program\n",
+"is covered only if its contents constitute a work based on the\n",
+"Program (independent of having been made by running the Program).\n",
+"Whether that is true depends on what the Program does.\n",
+"\n",
+"  1. You may copy and distribute verbatim copies of the Program's\n",
+"source code as you receive it, in any medium, provided that you\n",
+"conspicuously and appropriately publish on each copy an appropriate\n",
+"copyright notice and disclaimer of warranty; keep intact all the\n",
+"notices that refer to this License and to the absence of any warranty;\n",
+"and give any other recipients of the Program a copy of this License\n",
+"along with the Program.\n",
+"\n",
+"You may charge a fee for the physical act of transferring a copy, and\n",
+"you may at your option offer warranty protection in exchange for a fee.\n",
+"\n",
+"  2. You may modify your copy or copies of the Program or any portion\n",
+"of it, thus forming a work based on the Program, and copy and\n",
+"distribute such modifications or work under the terms of Section 1\n",
+"above, provided that you also meet all of these conditions:\n",
+"\n",
+"    a) You must cause the modified files to carry prominent notices\n",
+"    stating that you changed the files and the date of any change.\n",
+"\n",
+"    b) You must cause any work that you distribute or publish, that in\n",
+"    whole or in part contains or is derived from the Program or any\n",
+"    part thereof, to be licensed as a whole at no charge to all third\n",
+"    parties under the terms of this License.\n",
+"\n",
+"    c) If the modified program normally reads commands interactively\n",
+"    when run, you must cause it, when started running for such\n",
+"    interactive use in the most ordinary way, to print or display an\n",
+"    announcement including an appropriate copyright notice and a\n",
+"    notice that there is no warranty (or else, saying that you provide\n",
+"    a warranty) and that users may redistribute the program under\n",
+"    these conditions, and telling the user how to view a copy of this\n",
+"    License.  (Exception: if the Program itself is interactive but\n",
+"    does not normally print such an announcement, your work based on\n",
+"    the Program is not required to print an announcement.)\n",
+"
+\n",
+"These requirements apply to the modified work as a whole.  If\n",
+"identifiable sections of that work are not derived from the Program,\n",
+"and can be reasonably considered independent and separate works in\n",
+"themselves, then this License, and its terms, do not apply to those\n",
+"sections when you distribute them as separate works.  But when you\n",
+"distribute the same sections as part of a whole which is a work based\n",
+"on the Program, the distribution of the whole must be on the terms of\n",
+"this License, whose permissions for other licensees extend to the\n",
+"entire whole, and thus to each and every part regardless of who wrote it.\n",
+"\n",
+"Thus, it is not the intent of this section to claim rights or contest\n",
+"your rights to work written entirely by you; rather, the intent is to\n",
+"exercise the right to control the distribution of derivative or\n",
+"collective works based on the Program.\n",
+"\n",
+"In addition, mere aggregation of another work not based on the Program\n",
+"with the Program (or with a work based on the Program) on a volume of\n",
+"a storage or distribution medium does not bring the other work under\n",
+"the scope of this License.\n",
+"\n",
+"  3. You may copy and distribute the Program (or a work based on it,\n",
+"under Section 2) in object code or executable form under the terms of\n",
+"Sections 1 and 2 above provided that you also do one of the following:\n",
+"\n",
+"    a) Accompany it with the complete corresponding machine-readable\n",
+"    source code, which must be distributed under the terms of Sections\n",
+"    1 and 2 above on a medium customarily used for software interchange; or,\n",
+"\n",
+"    b) Accompany it with a written offer, valid for at least three\n",
+"    years, to give any third party, for a charge no more than your\n",
+"    cost of physically performing source distribution, a complete\n",
+"    machine-readable copy of the corresponding source code, to be\n",
+"    distributed under the terms of Sections 1 and 2 above on a medium\n",
+"    customarily used for software interchange; or,\n",
+"\n",
+"    c) Accompany it with the information you received as to the offer\n",
+"    to distribute corresponding source code.  (This alternative is\n",
+"    allowed only for noncommercial distribution and only if you\n",
+"    received the program in object code or executable form with such\n",
+"    an offer, in accord with Subsection b above.)\n",
+"\n",
+"The source code for a work means the preferred form of the work for\n",
+"making modifications to it.  For an executable work, complete source\n",
+"code means all the source code for all modules it contains, plus any\n",
+"associated interface definition files, plus the scripts used to\n",
+"control compilation and installation of the executable.  However, as a\n",
+"special exception, the source code distributed need not include\n",
+"anything that is normally distributed (in either source or binary\n",
+"form) with the major components (compiler, kernel, and so on) of the\n",
+"operating system on which the executable runs, unless that component\n",
+"itself accompanies the executable.\n",
+"\n",
+"If distribution of executable or object code is made by offering\n",
+"access to copy from a designated place, then offering equivalent\n",
+"access to copy the source code from the same place counts as\n",
+"distribution of the source code, even though third parties are not\n",
+"compelled to copy the source along with the object code.\n",
+"
+\n",
+"  4. You may not copy, modify, sublicense, or distribute the Program\n",
+"except as expressly provided under this License.  Any attempt\n",
+"otherwise to copy, modify, sublicense or distribute the Program is\n",
+"void, and will automatically terminate your rights under this License.\n",
+"However, parties who have received copies, or rights, from you under\n",
+"this License will not have their licenses terminated so long as such\n",
+"parties remain in full compliance.\n",
+"\n",
+"  5. You are not required to accept this License, since you have not\n",
+"signed it.  However, nothing else grants you permission to modify or\n",
+"distribute the Program or its derivative works.  These actions are\n",
+"prohibited by law if you do not accept this License.  Therefore, by\n",
+"modifying or distributing the Program (or any work based on the\n",
+"Program), you indicate your acceptance of this License to do so, and\n",
+"all its terms and conditions for copying, distributing or modifying\n",
+"the Program or works based on it.\n",
+"\n",
+"  6. Each time you redistribute the Program (or any work based on the\n",
+"Program), the recipient automatically receives a license from the\n",
+"original licensor to copy, distribute or modify the Program subject to\n",
+"these terms and conditions.  You may not impose any further\n",
+"restrictions on the recipients' exercise of the rights granted herein.\n",
+"You are not responsible for enforcing compliance by third parties to\n",
+"this License.\n",
+"\n",
+"  7. If, as a consequence of a court judgment or allegation of patent\n",
+"infringement or for any other reason (not limited to patent issues),\n",
+"conditions are imposed on you (whether by court order, agreement or\n",
+"otherwise) that contradict the conditions of this License, they do not\n",
+"excuse you from the conditions of this License.  If you cannot\n",
+"distribute so as to satisfy simultaneously your obligations under this\n",
+"License and any other pertinent obligations, then as a consequence you\n",
+"may not distribute the Program at all.  For example, if a patent\n",
+"license would not permit royalty-free redistribution of the Program by\n",
+"all those who receive copies directly or indirectly through you, then\n",
+"the only way you could satisfy both it and this License would be to\n",
+"refrain entirely from distribution of the Program.\n",
+"\n",
+"If any portion of this section is held invalid or unenforceable under\n",
+"any particular circumstance, the balance of the section is intended to\n",
+"apply and the section as a whole is intended to apply in other\n",
+"circumstances.\n",
+"\n",
+"It is not the purpose of this section to induce you to infringe any\n",
+"patents or other property right claims or to contest validity of any\n",
+"such claims; this section has the sole purpose of protecting the\n",
+"integrity of the free software distribution system, which is\n",
+"implemented by public license practices.  Many people have made\n",
+"generous contributions to the wide range of software distributed\n",
+"through that system in reliance on consistent application of that\n",
+"system; it is up to the author/donor to decide if he or she is willing\n",
+"to distribute software through any other system and a licensee cannot\n",
+"impose that choice.\n",
+"\n",
+"This section is intended to make thoroughly clear what is believed to\n",
+"be a consequence of the rest of this License.\n",
+"
+\n",
+"  8. If the distribution and/or use of the Program is restricted in\n",
+"certain countries either by patents or by copyrighted interfaces, the\n",
+"original copyright holder who places the Program under this License\n",
+"may add an explicit geographical distribution limitation excluding\n",
+"those countries, so that distribution is permitted only in or among\n",
+"countries not thus excluded.  In such case, this License incorporates\n",
+"the limitation as if written in the body of this License.\n",
+"\n",
+"  9. The Free Software Foundation may publish revised and/or new versions\n",
+"of the General Public License from time to time.  Such new versions will\n",
+"be similar in spirit to the present version, but may differ in detail to\n",
+"address new problems or concerns.\n",
+"\n",
+"Each version is given a distinguishing version number.  If the Program\n",
+"specifies a version number of this License which applies to it and \"any\n",
+"later version\", you have the option of following the terms and conditions\n",
+"either of that version or of any later version published by the Free\n",
+"Software Foundation.  If the Program does not specify a version number of\n",
+"this License, you may choose any version ever published by the Free Software\n",
+"Foundation.\n",
+"\n",
+"  10. If you wish to incorporate parts of the Program into other free\n",
+"programs whose distribution conditions are different, write to the author\n",
+"to ask for permission.  For software which is copyrighted by the Free\n",
+"Software Foundation, write to the Free Software Foundation; we sometimes\n",
+"make exceptions for this.  Our decision will be guided by the two goals\n",
+"of preserving the free status of all derivatives of our free software and\n",
+"of promoting the sharing and reuse of software generally.\n",
+"\n",
+"			    NO WARRANTY\n",
+"\n",
+"  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY\n",
+"FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN\n",
+"OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES\n",
+"PROVIDE THE PROGRAM \"AS IS\" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED\n",
+"OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF\n",
+"MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS\n",
+"TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE\n",
+"PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,\n",
+"REPAIR OR CORRECTION.\n",
+"\n",
+"  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING\n",
+"WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR\n",
+"REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,\n",
+"INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING\n",
+"OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED\n",
+"TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY\n",
+"YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER\n",
+"PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE\n",
+"POSSIBILITY OF SUCH DAMAGES.\n",
+"\n",
+"		     END OF TERMS AND CONDITIONS\n"};
Index: /tags/gsdl-21/gsdl/src/mgpp/text/conditions.src
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/conditions.src	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/conditions.src	(revision 699)
@@ -0,0 +1,225 @@
+		    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+			    NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+		     END OF TERMS AND CONDITIONS
Index: /tags/gsdl-21/gsdl/src/mgpp/text/diff.txt
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/diff.txt	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/diff.txt	(revision 699)
@@ -0,0 +1,61 @@
+bool_query.c:  OptimiseBoolTree (tree, qd->TL, atoi (GetEnv ("optimise_type")));
+Binary file bool_query.o matches
+commands.c:  for (i = 0; (name = GetEnvName (i)) != NULL; i++)
+commands.c:  for (i = 0; (name = GetEnvName (i)) != NULL; i++)
+commands.c:    printf ("%-*s = \"%s\"\n", (int) l, name, GetEnv (name));
+Binary file commands.o matches
+environment.c:GetEnv (char *name)
+environment.c:  char *data = GetEnv (name);
+environment.c:GetEnvName (int i)
+environment.c:  while ((name = GetEnvName (0)) != 0)
+environment.h:char *GetEnv (char *name);
+environment.h:char *GetEnvName (int i);
+Binary file environment.o matches
+Binary file libtextin.a matches
+mgquery.c:  if (!BooleanEnv (GetEnv ("briefstats"), 0))
+mgquery.c:  if (!BooleanEnv (GetEnv ("briefstats"), 0))
+mgquery.c:  if (!BooleanEnv (GetEnv ("briefstats"), 0))
+mgquery.c:	if (!BooleanEnv (GetEnv ("briefstats"), 0))
+mgquery.c:      if (BooleanEnv (GetEnv ("verbatim"), 1) == 0)
+mgquery.c:  if (isatty (fileno (OutFile)) && GetEnv ("pager") &&
+mgquery.c:      pager = GetEnv ("pager");
+mgquery.c:	       GetEnv ("hilite_style"),
+mgquery.c:	       GetEnv ("pager"),
+mgquery.c:				BooleanEnv (GetEnv ("verbatim"), 1),
+mgquery.c:  if (BooleanEnv (GetEnv ("timestats"), 0) ||
+mgquery.c:      BooleanEnv (GetEnv ("briefstats"), 0))
+mgquery.c:  if (BooleanEnv (GetEnv ("diskstats"), 0) ||
+mgquery.c:      BooleanEnv (GetEnv ("briefstats"), 0))
+mgquery.c:  if (BooleanEnv (GetEnv ("memstats"), 0) ||
+mgquery.c:      BooleanEnv (GetEnv ("briefstats"), 0))
+mgquery.c:  if (BooleanEnv (GetEnv ("timestats"), 0) ||
+mgquery.c:      BooleanEnv (GetEnv ("briefstats"), 0))
+mgquery.c:  if (BooleanEnv (GetEnv ("diskstats"), 0) ||
+mgquery.c:      BooleanEnv (GetEnv ("briefstats"), 0))
+mgquery.c:  if (BooleanEnv (GetEnv ("sizestats"), 0))
+mgquery.c:	      BooleanQuery (qd, line, &bqi, (BooleanEnv (GetEnv ("casefold"), 0) |
+mgquery.c:					     (BooleanEnv (GetEnv ("stem"), 0) << 1)));
+mgquery.c:	    rqi.Sort = BooleanEnv (GetEnv ("sorted_terms"), 0);
+mgquery.c:	    rqi.QueryFreqs = BooleanEnv (GetEnv ("qfreq"), 1);
+mgquery.c:	    if (qd->id->ifh.InvfLevel == 3 && GetEnv ("maxparas"))
+mgquery.c:	      rqi.MaxParasToRetrieve = atoi (GetEnv ("maxparas"));
+mgquery.c:	    rqi.HashTblSize = IntEnv (GetEnv ("hash_tbl_size"), 1000);
+mgquery.c:	    rqi.StopAtMaxAccum = BooleanEnv (GetEnv ("stop_at_max_accum"), 0);
+mgquery.c:	    rqi.skip_dump = GetEnv ("skip_dump");
+mgquery.c:      if (qd->QTL && BooleanEnv (GetEnv ("term_freq"), 0))
+mgquery.c:      if (BooleanEnv (GetEnv ("timestats"), 0) ||
+mgquery.c:	  BooleanEnv (GetEnv ("briefstats"), 0))
+mgquery.c:      if (BooleanEnv (GetEnv ("diskstats"), 0) ||
+mgquery.c:	  BooleanEnv (GetEnv ("briefstats"), 0))
+mgquery.c:      if (BooleanEnv (GetEnv ("memstats"), 0) ||
+mgquery.c:	  BooleanEnv (GetEnv ("briefstats"), 0))
+mgquery.c:      if (BooleanEnv (GetEnv ("sizestats"), 0))
+mgquery.c:      dir = GetEnv ("mgdir");
+mgquery.c:      if (!BooleanEnv (GetEnv ("expert"), 0) && isatty (fileno (InFile)))
+Binary file mgquery.o matches
+query.ranked.cpp:    default_stem_method = BooleanEnv (GetEnv ("casefold"), 0) | (BooleanEnv (GetEnv ("stem"), 0) << 1);
+query.ranked.cpp~:    default_stem_method = BooleanEnv (GetEnv ("casefold"), 0) | (BooleanEnv (GetEnv ("stem"), 0) << 1);
+Binary file query.ranked.o matches
+read_line.c:      if (!BooleanEnv (GetEnv ("expert"), 0))
+Binary file read_line.o matches
+Binary file sta04357 matches
Index: /tags/gsdl-21/gsdl/src/mgpp/text/environment.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/environment.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/environment.cpp	(revision 699)
@@ -0,0 +1,683 @@
+/**************************************************************************
+ *
+ * environment.c -- mgquery environment functions
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/10/11 02:57:13  cs025
+   Base install of MG-PP
+
+   Revision 1.1  1999/08/10 21:17:50  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:34:37  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.5  1995/03/14  05:15:26  tes
+   * Updated the boolean query optimiser to do different types of optimisation.
+   * A query environment variable "optimise_type" specifies which one is to be
+   * used. Type 1 is the new one which is faster than 2.
+   *
+   * Revision 1.4  1994/11/25  03:47:43  tes
+   * Committing files before adding the merge stuff.
+   *
+   * Revision 1.3  1994/10/20  03:56:43  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:25  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+#include "non_ansi.h"
+
+#include "memlib.h"
+#include "locallib.h"
+#include "local_strings.h"
+#include "messages.h"
+
+#include "environment.h"
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+# define STRCASECMP      stricmp
+#else
+# define STRCASECMP      strcasecmp
+#endif
+
+typedef struct
+  {
+    char *Name, *Data;
+    char *(*Constraint) (char *Old, char *New);
+  }
+EEntry;
+
+typedef struct Env
+  {
+    EEntry *Environment;
+    int NumEnv;
+    struct Env *Next;
+  }
+Env;
+
+
+char *ConstraintErrorStr;
+
+static Env Base =
+{NULL, 0, NULL};
+
+
+/*
+ * Add name and data to the environment (data may be NULL)
+ * All names are converted to lower case
+ * returns 0 on success, -1 on memory failure or -2 on constraint failure
+ */
+int 
+SetEnv (char *name, char *data, char *(*Constraint) (char *, char *))
+{
+  int i;
+  EEntry *newenv, *env = Base.Environment;
+
+  if (data && *data == '\0')
+    data = NULL;
+
+  /* search for name in the environment */
+  for (i = 0; i < Base.NumEnv && STRCASECMP (name, env->Name); i++, env++);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  /* if not found the increase the environment size */
+  if (i >= Base.NumEnv)
+    {
+      if (Constraint)
+	{
+	  data = Constraint (NULL, data);
+	  if (!data)
+	    return (-2);
+	}
+      /* allocate memory for the new environment */
+      newenv = Base.Environment ?
+	(EEntry *) Xrealloc (Base.Environment, sizeof (EEntry) * (Base.NumEnv + 1)) :
+	(EEntry *) Xmalloc (sizeof (EEntry) * (Base.NumEnv + 1));
+      if (!newenv)
+	return (-1);
+      Base.Environment = newenv;
+      env = &newenv[Base.NumEnv];
+      env->Data = NULL;
+      env->Name = NULL;
+      env->Constraint = Constraint;
+      env->Name = Xstrdup (name);
+      if (!env->Name)
+	return (-1);
+      Base.NumEnv++;
+      for (i = 0; env->Name[i]; i++)
+	env->Name[i] = tolower (env->Name[i]);
+    }
+  if (env->Constraint)
+    {
+      data = env->Constraint (env->Data, data);
+      if (!data)
+	return (-2);
+    }
+  if (env->Data)
+    Xfree (env->Data);
+  env->Data = NULL;
+  if (data && *data != '\0')
+    {
+      env->Data = Xstrdup (data);
+      if (!env->Data)
+	return (-1);
+    }
+  return (0);
+}
+
+
+
+/*
+ * Returns the data associated with name. If Data = NULL then this routine
+ * returns a pointer to "". If the specified name does not exist then this 
+ * routine returns NULL.
+ */
+char *
+GetEnv (char *name)
+{
+  int i;
+  EEntry *env = Base.Environment;
+  /* search for name in the environment */
+  for (i = 0; i < Base.NumEnv && STRCASECMP (name, env->Name); i++, env++);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  if (i >= Base.NumEnv)
+    return (NULL);
+
+  return (char *) (env->Data ? env->Data : "");
+}
+
+/*
+ * Returns the data associated with name. If Data = NULL then this routine
+ * returns a pointer to "". If the specified name does not exist then this 
+ * routine returns default.
+ */
+char *
+GetDefEnv (char *name, char *def)
+{
+  char *data = GetEnv (name);
+  return (data ? data : def);
+}
+
+
+
+/*
+ * This function pushes the environment on to a stack and duplicated it in 
+ * the current stack.
+ *
+ */
+int 
+PushEnv (void)
+{
+  Env *env = (Env *) Xmalloc (sizeof (Env));
+  int i;
+  *env = Base;
+  Base.Next = env;
+  Base.Environment = NULL;
+  Base.NumEnv = 0;
+  for (i = 0; i < env->NumEnv; i++)
+    if (SetEnv (env->Environment[i].Name, env->Environment[i].Data,
+		env->Environment[i].Constraint))
+      return (-1);
+  return (0);
+}
+
+int 
+PopEnv (void)
+{
+  Env *env;
+  int i;
+  if (!Base.Next)
+    return (-1);
+  for (i = 0; i < Base.NumEnv; i++)
+    {
+      Xfree (Base.Environment[i].Name);
+      if (Base.Environment[i].Data)
+	Xfree (Base.Environment[i].Data);
+    }
+  if (Base.Environment)
+    Xfree (Base.Environment);
+  env = Base.Next;
+  Base = *env;
+  Xfree (env);
+  return (0);
+}
+
+
+/*
+ * Count the number of environments on the stack, including the current one.
+ *
+ */
+int 
+EnvStackHeight (void)
+{
+  int i = 1;
+  Env *env = &Base;
+  while (env->Next)
+    {
+      i++;
+      env = env->Next;
+    }
+  return (i);
+}
+
+/*
+ * Delete environment variable name. If name does not exist it returns -1
+ * otherwise it returns 0
+ */
+int 
+UnsetEnv (char *name, int Force)
+{
+  int i;
+  EEntry *env = Base.Environment;
+  /* search for name in the environment */
+  for (i = 0; i < Base.NumEnv && STRCASECMP (name, env->Name); i++, env++);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  if (i >= Base.NumEnv ||
+      (Base.Environment[i].Constraint && !Force))
+    return (-1);
+  Xfree (Base.Environment[i].Name);
+  if (Base.Environment[i].Data)
+    Xfree (Base.Environment[i].Data);
+  for (i++; i < Base.NumEnv; i++)
+    Base.Environment[i - 1] = Base.Environment[i];
+  Base.NumEnv--;
+  if (Base.NumEnv == 0)
+    {
+      Xfree (Base.Environment);
+      Base.Environment = NULL;
+    }
+  return (0);
+}
+
+
+
+
+
+
+
+
+/*
+ * Returns the name of environment variable number i or NULL if i is greater 
+ * than or equal to the number of environment variables. Names start from
+ * zero. 
+ */
+char *
+GetEnvName (int i)
+{
+  return (i < Base.NumEnv ? (char *) Base.Environment[i].Name : (char *) NULL);
+}
+
+static char *BooleanStrs[] =
+{"false", "true", "no", "yes", "off", "on"};
+
+static char *
+BooleanCons (char *Old, char *New)
+{
+  int i;
+  int old = -1, isnew = -1;
+  if (Old)
+    for (i = 0; i < sizeof (BooleanStrs) / sizeof (char *); i++)
+      if (!STRCASECMP (Old, BooleanStrs[i]))  /* [RPAP - Feb 97: WIN32 Port] */
+	{
+	  old = i;
+	  break;
+	}
+  if (New)
+    for (i = 0; i < sizeof (BooleanStrs) / sizeof (char *); i++)
+      if (!STRCASECMP (New, BooleanStrs[i]))  /* [RPAP - Feb 97: WIN32 Port] */
+	{
+	  isnew = i;
+	  break;
+	}
+  if (isnew >= 0)
+    return (New);
+  if (!New)
+    return (BooleanStrs[Old ? old ^ 1 : 0]);
+  ConstraintErrorStr = "Invalid argument [true|false|yes|no|on|off] required";
+  return (NULL);
+}
+
+/*
+ * returns 0 or 1  for a boolean string
+ * or default on a error
+ */
+int 
+BooleanEnv (char *data, int def)
+{
+  int i;
+  if (!data)
+    return (def);
+  for (i = 0; i < sizeof (BooleanStrs) / sizeof (char *); i++)
+    if (!STRCASECMP (data, BooleanStrs[i]))  /* [RPAP - Feb 97: WIN32 Port] */
+      return (i & 1);
+  return (def);
+}
+
+/*
+ * returns the value for a integer string
+ * or default on a error
+ */
+long 
+IntEnv (char *data, long def)
+{
+  long val;
+  char *ptr;
+  if (!data)
+    return (def);
+  val = strtol (data, &ptr, 10);
+  return *ptr ? def : val;
+}
+
+static char *
+NumberCmp (char *num, int min, int max)
+{
+  char *err;
+  int val;
+  static char Err[100];
+  ConstraintErrorStr = "Not a valid number";
+  if (!num)
+    return (NULL);
+  val = strtol (num, &err, 10);
+  if (*err)
+    return (NULL);
+  sprintf (Err, "Not in legal range [%d <= num <= %d]", min, max);
+  ConstraintErrorStr = Err;
+  if (val < min || val > max)
+    return (NULL);
+  return (num);
+}
+
+/*ARGSUSED */
+static char *
+MaxDocsCons (char *Old, char *New)
+{
+  if (!STRCASECMP (New, "all"))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (New);
+  return (NumberCmp (New, 1, ((unsigned) (~0)) >> 1));
+}
+
+/*ARGSUSED */
+static char *
+MaxTermsCons (char *Old, char *New)
+{
+  if (!STRCASECMP (New, "all"))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (New);
+  return (NumberCmp (New, 1, ((unsigned) (~0)) >> 1));
+}
+
+/* ARGSUSED */
+static char *
+BufferCons (char *Old, char *New)
+{
+  return (NumberCmp (New, 0, 16 * 1024 * 1024));
+}
+
+/* ARGSUSED */
+static char *
+MaxNodesCons (char *Old, char *New)
+{
+  if (!STRCASECMP (New, "all"))  /* [RPAP - Feb 97: WIN32 Port] */
+    return (New);
+  return (NumberCmp (New, 8, 256 * 1024 * 1024));
+}
+
+/* ARGSUSED */
+static char *
+MaxHashCons (char *Old, char *New)
+{
+  return (NumberCmp (New, 8, 256 * 1024 * 1024));
+}
+
+/* ARGSUSED */
+static char *
+MaxParasCons (char *Old, char *New)
+{
+  return (NumberCmp (New, 1, 256 * 1024 * 1024));
+}
+
+/* ARGSUSED */
+static char *
+MaxHeadsCons (char *Old, char *New)
+{
+  return (NumberCmp (New, 1, 1000));
+}
+
+/* ARGSUSED */
+char *
+OptimiseCons (char *Old, char *New)
+{
+  return (NumberCmp (New, 0, 2));
+}
+
+
+
+/*
+ * Makes sure that New is a valid query type
+ *
+ */
+/* ARGSUSED */
+static char *
+QueryCons (char *Old, char *New)
+{
+  static char *QueryStrs[] =
+  {
+    "boolean", "ranked",
+    "docnums", "approx-ranked"};
+  int i;
+  int isnew = -1;
+  if (New)
+    for (i = 0; i < sizeof (QueryStrs) / sizeof (char *); i++)
+      if (!STRCASECMP (New, QueryStrs[i]))  /* [RPAP - Feb 97: WIN32 Port] */
+	{
+	  isnew = i;
+	  break;
+	}
+  if (isnew >= 0)
+    return (New);
+  ConstraintErrorStr = "Invalid argument [boolean|ranked|docnums|approx-ranked] required";
+  return (NULL);
+}
+
+/*
+ * Makes sure that New is a valid accum type
+ *
+ */
+/* ARGSUSED */
+static char *
+AccumCons (char *Old, char *New)
+{
+  static char *AccumStrs[] =
+  {
+    "array", "splay_tree", "hash_table", "list"};
+  int i;
+  int isnew = -1;
+  if (New)
+    for (i = 0; i < sizeof (AccumStrs) / sizeof (char *); i++)
+      if (!STRCASECMP (New, AccumStrs[i]))  /* [RPAP - Feb 97: WIN32 Port] */
+	{
+	  isnew = i;
+	  break;
+	}
+  if (isnew >= 0)
+    return (New);
+  ConstraintErrorStr = "Invalid argument "
+    "[array|splay_tree|hash_table|list] required";
+  return (NULL);
+}
+
+
+
+/*
+ * Makes sure that New is a valid output type type
+ *
+ */
+/* ARGSUSED */
+static char *
+OutputTypeCons (char *Old, char *New)
+{
+  static char *OutputTypeStrs[] =
+  {"text", "silent", "docnums",
+   "count", "heads", "hilite"
+#ifdef TREC_MODE
+   ,"extras_for_trec"
+#endif
+  };
+
+  int i;
+  int isnew = -1;
+  if (New)
+    for (i = 0; i < sizeof (OutputTypeStrs) / sizeof (char *); i++)
+      if (!STRCASECMP (New, OutputTypeStrs[i]))  /* [RPAP - Feb 97: WIN32 Port] */
+	{
+	  isnew = i;
+	  break;
+	}
+  if (isnew >= 0)
+    return (New);
+  ConstraintErrorStr = "Invalid argument [text|silent|docnums|heads|count|hilite"
+#ifdef TREC_MODE
+    "|extras_for_trec"
+#endif
+    "] required";
+  return (NULL);
+}
+
+/*
+ * Makes sure that New is a valid query type
+ *
+ */
+/* ARGSUSED */
+char *
+HiliteStyleCons (char *Old, char *New)
+{
+  static char *StyleStrs[] =
+  {"bold", "underline"};
+  int i;
+  int isnew = -1;
+  if (New)
+    for (i = 0; i < sizeof (StyleStrs) / sizeof (char *); i++)
+      if (!STRCASECMP (New, StyleStrs[i]))  /* [RPAP - Feb 97: WIN32 Port] */
+	{
+	  isnew = i;
+	  break;
+	}
+  if (isnew >= 0)
+    return (New);
+  ConstraintErrorStr = "Invalid argument [bold|underline] required";
+  return (NULL);
+}
+
+/* =========================================================================
+ * Function: get_output_type
+ * Description:
+ *      Map a mode string onto a mode char e.g. OUTPUT_????
+ * Input:
+ * Output:
+ *      The mode char which is a subrange value.
+ * ========================================================================= */
+
+char
+get_output_type (void)
+{
+  char *env_str = GetDefEnv ("mode", "text");
+  char ch1 = toupper (env_str[0]);
+  char ch2 = toupper (env_str[1]);
+
+  switch (ch1)
+    {
+    case 'T':
+      return OUTPUT_TEXT;
+    case 'S':
+      return OUTPUT_SILENT;
+    case 'E':
+      return OUTPUT_EXTRAS;
+    case 'C':
+      return OUTPUT_COUNT;
+    case 'D':
+      return OUTPUT_DOCNUMS;
+    case 'H':
+      if (ch2 == 'E')
+	return OUTPUT_HEADERS;
+      else
+	return OUTPUT_HILITE;
+    default:
+      FatalError (1, "Problem in output type switch");
+    }
+
+  return '\0';			/*shouldn't reach here -- keep compiler happy */
+}
+
+/* =========================================================================
+ * Function: get_query_type
+ * Description:
+ *      Map a query type string onto a char.
+ * Input:
+ * Output:
+ *      Query type char.
+ * ========================================================================= */
+
+char
+get_query_type (void)
+{
+  char *env_str = GetDefEnv ("query", "boolean");
+  char ch1 = toupper (env_str[0]);
+
+  switch (ch1)
+    {
+    case 'R':
+      return QUERY_RANKED;
+    case 'A':
+      return QUERY_APPROX;
+    case 'B':
+      return QUERY_BOOLEAN;
+    case 'D':
+      return QUERY_DOCNUMS;
+    default:
+      FatalError (1, "Problem in query type switch");
+    }
+
+  return '\0';			/*shouldn't reach here */
+}
+
+
+
+/*
+ * This initialises certain environment variables
+ *
+ */
+void 
+InitEnv (void)
+{
+  SetEnv ("hilite_style", "underline", HiliteStyleCons);	/*[TS:Sep/94] */
+  SetEnv ("briefstats", "off", BooleanCons);
+  SetEnv ("diskstats", "off", BooleanCons);
+  SetEnv ("expert", "false", BooleanCons);
+  SetEnv ("mgdir", (char *) (getenv ("MGDATA") ? getenv ("MGDATA") : "."), NULL);
+  SetEnv ("mgname", "", NULL);
+  SetEnv ("maxdocs", "all", MaxDocsCons);
+  SetEnv ("memstats", "off", BooleanCons);
+  SetEnv ("mode", "text", OutputTypeCons);
+  SetEnv ("pager", (char *) (getenv ("PAGER") ? getenv ("PAGER") : "more"), NULL);
+  SetEnv ("qfreq", "true", BooleanCons);
+  SetEnv ("query", "boolean", QueryCons);
+  SetEnv ("sizestats", "off", BooleanCons);
+  SetEnv ("timestats", "off", BooleanCons);
+  SetEnv ("verbatim", "off", BooleanCons);
+  SetEnv ("sorted_terms", "on", BooleanCons);
+  SetEnv ("accumulator_method", "array", AccumCons);
+  SetEnv ("stop_at_max_accum", "off", BooleanCons);
+  SetEnv ("buffer", "1048576", BufferCons);
+  SetEnv ("max_accumulators", "50000", MaxNodesCons);
+  SetEnv ("max_terms", "all", MaxTermsCons);
+  SetEnv ("maxparas", "1000", MaxParasCons);
+  SetEnv ("hash_tbl_size", "1000", MaxHashCons);
+  SetEnv ("skip_dump", "skips.%d", NULL);
+  SetEnv ("ranked_doc_sepstr", "---------------------------------- %n %w\\n", NULL);
+  SetEnv ("doc_sepstr", "---------------------------------- %n\\n", NULL);
+  SetEnv ("para_sepstr", "\\n######## PARAGRAPH %n ########\\n", NULL);
+  SetEnv ("para_start", "***** Weight = %w *****\\n", NULL);
+  SetEnv ("terminator", "", NULL);
+  SetEnv ("heads_length", "50", MaxHeadsCons);
+  SetEnv ("optimise_type", "1", OptimiseCons);	/*[TS:Mar/95] */
+  SetEnv ("casefold", "on", BooleanCons);  /* [RPAP - Jan 97: Stem Index Change] */
+  SetEnv ("stem", "on", BooleanCons);  /* [RPAP - Jan 97: Stem Index Change] */
+  SetEnv ("term_freq", "off", BooleanCons);  /* [RPAP - Feb 97: Term Frequency] */
+}
+
+
+void 
+UninitEnv (void)
+{
+  char *name;
+  while (PopEnv () == 0);
+  while ((name = GetEnvName (0)) != 0)
+    UnsetEnv (name, 1);
+}
Index: /tags/gsdl-21/gsdl/src/mgpp/text/environment.h
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/environment.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/environment.h	(revision 699)
@@ -0,0 +1,61 @@
+/**************************************************************************
+ *
+ * environment.h -- mgquery environment functions
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef ENVIRONMENT_H
+#define ENVIRONMENT_H
+
+#define OUTPUT_TEXT     'T'
+#define OUTPUT_HEADERS  'H'
+#define OUTPUT_SILENT   'S'
+#define OUTPUT_EXTRAS   'E'
+#define OUTPUT_HILITE   'I'
+#define OUTPUT_COUNT    'C'
+#define OUTPUT_DOCNUMS  'D'
+
+#define QUERY_RANKED    'R'
+#define QUERY_APPROX    'A'
+#define QUERY_BOOLEAN   'B'
+#define QUERY_DOCNUMS   'D'
+
+
+char get_output_type (void);
+char get_query_type (void);
+
+extern char *ConstraintErrorStr;
+int SetEnv (char *name, char *data, char *(*Constraint) (char *, char *));
+char *GetEnv (char *name);
+char *GetDefEnv (char *name, char *def);
+int UnsetEnv (char *name, int Force);
+char *de_escape_string (char *s);
+int PushEnv (void);
+int PopEnv (void);
+int EnvStackHeight (void);
+char *GetEnvName (int i);
+int BooleanEnv (char *data, int def);
+long IntEnv (char *data, long def);
+
+void InitEnv (void);
+void UninitEnv (void);
+
+#endif
Index: /tags/gsdl-21/gsdl/src/mgpp/text/globals.h
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/globals.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/globals.h	(revision 699)
@@ -0,0 +1,32 @@
+/**************************************************************************
+ *
+ * globals.h -- Some global stuff for mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef H_GLOBALS
+#define H_GLOBALS
+
+
+extern FILE *OutFile;
+extern FILE *InFile;
+
+#endif
Index: /tags/gsdl-21/gsdl/src/mgpp/text/grbIndex.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/grbIndex.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/grbIndex.cpp	(revision 699)
@@ -0,0 +1,1531 @@
+/**************************************************************************
+ *
+ * mgquery.c -- The   M G Q U E R Y   program
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.3  1999/10/12 04:13:11  cs025
+   Changes made to eradicate use of GetEnv elsewhere.
+
+   Revision 1.2  1999/10/11 22:01:40  cs025
+   Got rid of regular expression calls.
+
+   Revision 1.1  1999/10/11 02:57:15  cs025
+   Base install of MG-PP
+
+   Revision 1.1  1999/08/10 21:18:18  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.3  1999/01/08 00:33:46  rjmcnab
+
+   Enabled mg and the library software to read in more than one index
+   at a time.
+
+   Revision 1.2  1998/11/25 07:55:49  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:29  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:57:02  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:58  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+#if defined(HAVE_SYS_PROCFS_H) && defined(HAVE_PR_BRKSIZE) && \
+    (__STDC__ == 0)
+/* STDC test was included to allow cc -Xc on SunOS 5 to work */
+#define USE_PROCESS_MEM
+#endif
+
+#ifdef USE_PROCESS_MEM
+# include <sys/procfs.h>
+#endif
+
+#ifdef HAVE_GETRUSAGE
+# ifdef HAVE_SYS_TIME_H
+#  include <sys/time.h>
+# endif
+# include <sys/resource.h>
+#endif
+
+#ifndef HAVE_GETPAGESIZE
+# include "getpagesize.h"
+#endif
+ 
+extern "C" {
+#if WITH_REGEX
+# include <regex.h>
+#else
+# include <rx.h>
+#endif
+#include <stdio.h>
+}
+
+#include "non_ansi.h"
+
+#include <stdarg.h>
+#include <signal.h>
+#include <unistd.h>
+
+#include "messages.h"
+#include "timing.h"
+#include "memlib.h"
+#include "local_strings.h"  /* [RPAP - Feb 97: Term Frequency] */
+
+#include "filestats.h"
+#include "invf.h"
+#include "text.h"
+#include "mg.h"
+#include "lists.h"
+#include "backend.h"
+#include "DocEntry.h"
+#include "environment.h"
+#include "globals.h"
+#include "read_line.h"
+#include "mg_errors.h"
+#include "commands.h"
+#include "text_get.h"
+#include "term_lists.h"
+#include "query_term_list.h"
+
+FILE *OutFile = NULL, *InFile = NULL;
+int OutPipe = 0, InPipe = 0;
+int Quitting = 0;
+
+/* [RPAP - Feb 97: NZDL Additions] */
+#if defined(PARADOCNUM) ||  defined(NZDL)
+int GetDocNumFromParaNum(query_data *qd, int paranum) {
+  int Documents = qd->td->cth.num_of_docs;
+  int *Paragraph = qd->paragraph;
+  int low = 1, high = Documents;
+  int mid = (low+high)/2;
+
+  while ((mid = (low+high)/2) >=1 && mid <= Documents)
+    {
+      if (paranum > Paragraph[mid])
+	low = mid+1;
+      else if (paranum <= Paragraph[mid-1])
+	high = mid-1;
+      else
+	return mid;
+    }
+  FatalError(1, "Bad paragraph number.\n");
+}
+#endif
+
+#ifdef TREC_MODE
+char *trec_ids = NULL;
+long *trec_paras = NULL;
+#endif
+
+static volatile int PagerRunning = 0;
+static volatile int Ctrl_C = 0;
+
+
+/*****************************************************************************/
+
+typedef enum
+  {
+    S_Time, S_Mem, S_Size, S_File
+  }
+S_Type;
+
+static struct Stat
+  {
+    S_Type typ;
+    char *name;
+    char *text;
+  }
+ *Stats = NULL;
+static int NumStats = 0;
+
+static void 
+Clear_Stats (void)
+{
+  if (Stats)
+    {
+      int i;
+      for (i = 0; i < NumStats; i++)
+	{
+	  if (Stats[i].name)
+	    Xfree (Stats[i].name);
+	  if (Stats[i].text)
+	    Xfree (Stats[i].text);
+	}
+      Xfree (Stats);
+      Stats = NULL;
+      NumStats = 0;
+    }
+}
+
+static void 
+Add_Stats (S_Type typ, char *name, char *fmt,...)
+{
+  char buf[1024];
+  va_list args;
+  va_start (args, fmt);
+  vsprintf (buf, fmt, args);
+  if (Stats)
+    Stats = (struct Stat *) Xrealloc (Stats, (++NumStats) * sizeof (*Stats));
+  else
+    Stats = (struct Stat *) Xmalloc ((++NumStats) * sizeof (*Stats));
+  Stats[NumStats - 1].typ = typ;
+  Stats[NumStats - 1].name = Xstrdup (name);
+  Stats[NumStats - 1].text = Xstrdup (buf);
+}
+
+static void 
+Display_Stats (FILE * f)
+{
+  static char *sep = "-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="
+  "-=-=-=-=-=-=-=-=-=-=-";
+  char *names[] =
+  {"Time:  ", "Memory:", "Sizes: ", "Disk:  ", "       "};
+  int i, last_typ = -1;
+  size_t len = 0;
+  if (NumStats == 0)
+    return;
+  fprintf (f, "%s\n", sep);
+  for (i = 0; i < NumStats; i++)
+    if (strlen (Stats[i].name) > len)
+      len = strlen (Stats[i].name);
+  for (i = 0; i < NumStats; i++)
+    {
+      int typ = 4;
+      if (Stats[i].typ != last_typ)
+	typ = last_typ = Stats[i].typ;
+      fprintf (f, "%s %-*s %s\n", names[typ], (int) len, Stats[i].name, Stats[i].text);
+    }
+  fprintf (f, "%s\n", sep);
+}
+
+/*****************************************************************************/
+
+
+static void 
+QueryTimeStats (ProgTime * Start, ProgTime * invf, ProgTime * text)
+{
+  if (!BooleanEnv (GetEnv ("briefstats"), 0))
+    {
+      Add_Stats (S_Time, "invf", ElapsedTime (Start, invf));
+      Add_Stats (S_Time, "text", ElapsedTime (invf, text));
+    }
+  Add_Stats (S_Time, "total", ElapsedTime (Start, text));
+}
+
+static void 
+StartUpTimeStats (InitQueryTimes * iqt)
+{
+  if (!BooleanEnv (GetEnv ("briefstats"), 0))
+    {
+      Add_Stats (S_Time, "dict [stem]", ElapsedTime (&iqt->Start,
+						     &iqt->StemDict));
+      Add_Stats (S_Time, "weights", ElapsedTime (&iqt->StemDict,
+						 &iqt->ApproxWeights));
+      Add_Stats (S_Time, "dict [text]", ElapsedTime (&iqt->ApproxWeights,
+						     &iqt->CompDict));
+      Add_Stats (S_Time, "Inverted", ElapsedTime (&iqt->CompDict,
+						  &iqt->Invf));
+      Add_Stats (S_Time, "Compressed", ElapsedTime (&iqt->Invf,
+						    &iqt->Text));
+    }
+  Add_Stats (S_Time, "total", ElapsedTime (&iqt->Start, &iqt->Text));
+}
+
+
+
+
+#ifdef USE_PROCESS_MEM
+static u_long 
+process_mem (void)
+{
+  prstatus_t pr;
+  static int fd = -1;
+  if (fd == -1)
+    {
+      char buf[128];
+      sprintf (buf, "/proc/%ld", (long) getpid ());
+      fd = open (buf, O_RDONLY);
+    }
+  if (fd == -1 || ioctl (fd, PIOCSTATUS, &pr) == -1)
+    return 0;
+  return pr.pr_brksize;
+}
+#endif
+
+
+
+
+static void 
+MemStats (query_data * qd)
+{
+  if (!BooleanEnv (GetEnv ("briefstats"), 0))
+    {
+#ifdef HAVE_GETRUSAGE
+      struct rusage rusage;
+      getrusage (RUSAGE_SELF, &rusage);
+
+      Add_Stats (S_Mem, "process mem", "%7.3f Mb",
+	    (double) (rusage.ru_maxrss * getpagesize () / 1024.0 / 1024.0));
+#endif
+#ifdef USE_PROCESS_MEM
+      Add_Stats (S_Mem, "process mem", "%7.3f Mb",
+		 (double) (process_mem () / 1024.0 / 1024.0));
+#endif
+      Add_Stats (S_Mem, "dict [stem]", "%7.1f kB",
+		 (double) qd->sd->MemForStemDict / 1024);
+      Add_Stats (S_Mem, "dict [text]", "%7.1f kB",
+		 (double) qd->cd->MemForCompDict / 1024);
+      if (qd->awd)
+	Add_Stats (S_Mem, "weights", "%7.1f kB",
+		   (double) qd->awd->MemForWeights / 1024);
+    }
+  if (qd->awd)
+    Add_Stats (S_Mem, "total [peak]", "%7.1f kB",
+	       (double) (qd->max_mem_in_use + qd->sd->MemForStemDict +
+		   qd->cd->MemForCompDict + qd->awd->MemForWeights) / 1024);
+  else
+    Add_Stats (S_Mem, "total [peak]", "%7.1f kB",
+	       (double) (qd->max_mem_in_use + qd->sd->MemForStemDict +
+			 qd->cd->MemForCompDict) / 1024);
+
+}
+
+
+
+static void 
+SizeStats (query_data * qd)
+{
+  Add_Stats (S_Size, "skips", "%7d", qd->hops_taken);
+  Add_Stats (S_Size, "pointers", "%7d", qd->num_of_ptrs);
+  Add_Stats (S_Size, "accumulators", "%7d", qd->num_of_accum);
+  Add_Stats (S_Size, "terms", "%7d", qd->num_of_terms);
+  Add_Stats (S_Size, "answers", "%7d", qd->num_of_ans);
+  Add_Stats (S_Size, "index lookups", "%7d", qd->text_idx_lookups);
+}
+
+static void 
+TotalSizeStats (query_data * qd)
+{
+  Add_Stats (S_Size, "skips", "%7d", qd->tot_hops_taken);
+  Add_Stats (S_Size, "pointers", "%7d", qd->tot_num_of_ptrs);
+  Add_Stats (S_Size, "accumulators", "%7d", qd->tot_num_of_accum);
+  Add_Stats (S_Size, "terms", "%7d", qd->tot_num_of_terms);
+  Add_Stats (S_Size, "answers", "%7d", qd->tot_num_of_ans);
+  Add_Stats (S_Size, "index lookups", "%7d", qd->tot_text_idx_lookups);
+}
+
+
+static void 
+StatFile (File * F)
+{
+  static unsigned long NumBytes = 0, NumSeeks = 0, NumReads = 0;
+  if (F)
+    if ((int) F != -1)
+      {
+	if (!BooleanEnv (GetEnv ("briefstats"), 0))
+	  Add_Stats (S_File, F->name, "%7.1f kB (%3d seeks, %7d reads)",
+		   (double) F->Current.NumBytes / 1024, F->Current.NumSeeks,
+		     F->Current.NumReads);
+	NumBytes += F->Current.NumBytes;
+	NumSeeks += F->Current.NumSeeks;
+	NumReads += F->Current.NumReads;
+      }
+    else
+      {
+	Add_Stats (S_File, "total", "%7.1f kB (%3d seeks, %7d reads)",
+		   (double) NumBytes / 1024, NumSeeks, NumReads);
+	NumSeeks = NumReads = NumBytes = 0;
+      }
+
+}
+
+
+static void 
+File_Stats (query_data * qd)
+{
+  StatFile (qd->File_comp_dict);
+  StatFile (qd->File_fast_comp_dict);
+  StatFile (qd->File_text_idx_wgt);
+  StatFile (qd->File_text);
+  StatFile (qd->File_stem);
+
+  /* [RPAP - Jan 97: Stem Index Change] */
+  if (qd->sd->sdh.indexed)
+    {
+      StatFile (qd->File_stem1);
+      StatFile (qd->File_stem2);
+      StatFile (qd->File_stem3);
+    }
+
+  StatFile (qd->File_invf);
+  StatFile (qd->File_weight_approx);
+  StatFile (qd->File_text_idx);
+  StatFile ((File *) (-1));
+}
+
+
+char *
+get_query (query_data * qd)
+{
+  char *line, *LinePtr;
+  WritePrompt ();
+  do
+    {
+      do
+	{
+	  line = GetMultiLine ();
+	  if (line == NULL)
+	    {
+	      if (stdin == InFile)
+		return (NULL);	/* EOF */
+	      if (InPipe)
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+		_pclose (InFile);
+#else
+		pclose (InFile);
+#endif
+	      else
+		fclose (InFile);
+	      InPipe = 0;
+	      InFile = stdin;
+	    }
+	}
+      while (line == NULL);
+      LinePtr = ProcessCommands (line, qd);
+      if (CommandsErrorStr)
+	fprintf (stderr, "%s\n", CommandsErrorStr);
+    }
+  while (*LinePtr == '\0' && !Quitting);
+  return (LinePtr);
+}
+
+
+/* This is executed when a SIGPIPE is detected
+   i.e. If some one quits out of the PAGER, this is executed */
+#ifdef HAVE_SIGCONTEXT
+static RETSIGTYPE 
+SIGPIPE_handler (int sig, int code,
+		 struct sigcontext *scp, char *addr)
+#else
+static RETSIGTYPE 
+SIGPIPE_handler (int sig)
+#endif
+{
+  /* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+  signal (sig, SIG_IGN);
+#else
+  signal (SIGPIPE, SIG_IGN);
+#endif
+  PagerRunning = 0;
+}
+
+/* This is executed when a SIGINT (i.e. CTRL-C) is detected */
+#ifdef  HAVE_SIGCONTEXT
+static RETSIGTYPE 
+SIGINT_handler (int sig, int code,
+		struct sigcontext *scp, char *addr)
+#else
+static RETSIGTYPE 
+SIGINT_handler (int sig)
+#endif
+{
+  Ctrl_C = 1;
+}
+
+
+
+static char *post_proc = NULL;
+
+
+
+void 
+GetPostProc (char *line)
+{
+  char *start, *finish;
+  if (post_proc)
+    {
+      Xfree (post_proc);
+      post_proc = NULL;
+    }
+  start = strchr (line, '\"');
+  finish = strrchr (line, '\"');
+  if (start != finish)
+    {
+      /* found a pattern */
+      *finish = '\0';
+      post_proc = Xstrdup (start + 1);
+      strcpy (start, finish + 1);
+    }
+  else if (start != NULL)
+    {
+      /* found a single speech mark. Delete It. */
+      strcpy (start, start + 1);
+    }
+}
+
+int 
+PostProc (char *UDoc, int verbatim)
+{
+  if (!post_proc)
+    return 1;
+
+  return (strstr (UDoc, post_proc) != NULL);
+}
+
+
+
+static DocEntry *
+in_chain (int para, int ip, DocEntry * dc)
+{
+  while (dc)
+    {
+      if (dc->DocNum - ip == para)
+	return dc;
+      dc = dc->Next;
+    }
+  return NULL;
+}
+
+/* num should be greater than or equal to 1 */
+int 
+RawDocOutput (query_data * qd, u_long num, FILE * Output)
+{
+  static int last_pos = 0;
+  static u_char *c_buffer = 0;
+  static int buf_len = -1;
+  static u_char *uc_buffer = 0;
+  u_long pos, len;
+  int ULen;
+
+  FetchDocStart (qd, num, &pos, &len);
+
+  if ((int) len > buf_len)
+    {
+      if (c_buffer)
+	{
+	  Xfree (c_buffer);
+	  Xfree (uc_buffer);
+	}
+      if (!(c_buffer = (u_char *) Xmalloc (len)))
+	return -1;
+      if (!(uc_buffer = (u_char *) Xmalloc ((int) (qd->td->cth.ratio * 1.01 *
+					len) + 100)))
+	return -1;
+      buf_len = len;
+    }
+  if (last_pos != pos)
+    Fseek (qd->td->TextFile, pos, 0);
+  Fread (c_buffer, 1, len, qd->td->TextFile);
+  last_pos = pos + len;
+  DecodeText (qd->cd, c_buffer, len, uc_buffer, &ULen);
+  fwrite (uc_buffer, ULen, sizeof (u_char), Output);
+  return 0;
+}
+
+
+void 
+StringOut (FILE * Output, char *string,
+	   int intvalid, unsigned long intval,
+	   int floatvalid, double floatval)
+{
+  char *s;
+  for (s = string; *s; s++)
+    if (*s == '%' &&
+	(*(s + 1) == 'n' || *(s + 1) == 'w' || *(s + 1) == '%'))
+      {
+	s++;
+	switch (*s)
+	  {
+	  case 'n':
+	    if (intvalid)
+	      fprintf (Output, "%lu", intval);
+	    else
+	      fprintf (Output, "%%n");
+	    break;
+	  case 'w':
+	    if (floatvalid)
+	      fprintf (Output, "%f", floatval);
+	    else
+	      fprintf (Output, "%%w");
+	    break;
+	  case '%':
+	    fputc ('%', Output);
+	  }
+      }
+    else
+      fputc (*s, Output);
+}
+
+
+void 
+HeaderOut (FILE * Output, u_char * UDoc, unsigned long ULen, int heads_length)
+{
+  int i, space = 1, num = 0;
+  for (i = 0; i < ULen && num < heads_length; i++)
+    {
+      char c = UDoc[i];
+      if (c == '\02')
+	break;
+
+      if (isspace (c) || c == '\01' || c == '\03')
+	{
+	  if (!space)
+	    {
+	      fputc (' ', Output);
+	      num++;
+	    }
+	  space = 1;
+	}
+      else
+	{
+	  space = 0;
+	  fputc (c, Output);
+	  num++;
+	}
+    }
+}
+
+/* [RPAP - Feb 97: NZDL Additions] */
+#if defined(PARADOCNUM) || defined(NZDL)
+void PrintDocNum(FILE *output, char query_type,
+		 int docnum, int indexnum, float weight)
+{
+  if (query_type == 'R' || query_type == 'A')
+    fprintf(output, "%7d.%-7d  %6.4f\n", docnum, indexnum, weight);
+  else
+    fprintf(output, "%7d.%-7d\n", docnum, indexnum);
+}
+#endif
+
+/* --
+   -- George Buchanan - August 1999
+   --
+
+      ProcessDocs prints out the results of a query to the output
+      stream, respecting the configuration choices for display
+      format etc.
+*/
+static int 
+ProcessDocs (query_data * qd, int num, int verbatim,
+	     char OutputType, FILE * Output)
+{
+  int max_buf = 0;
+  int DocCount = 0;
+  char *doc_sepstr = NULL;
+  char *para_sepstr = NULL;
+  char *para_start = NULL;
+  int heads_length = atoi (GetDefEnv ("heads_length", "50"));
+  char QueryType = get_query_type ();
+  int need_text = (OutputType == OUTPUT_TEXT || OutputType == OUTPUT_HILITE ||
+		   OutputType == OUTPUT_HEADERS || OutputType == OUTPUT_SILENT || 
+		   post_proc); /* [RJM June 1997 -- fixing post retrieval scan] */
+
+  if (OutputType == OUTPUT_TEXT || OutputType == OUTPUT_HILITE)
+    {
+      if (QueryType == QUERY_APPROX || QueryType == QUERY_RANKED)
+	{
+	  doc_sepstr = de_escape_string (
+				    Xstrdup (GetDefEnv ("ranked_doc_sepstr",
+			   "---------------------------------- %n %w\\n")));
+	}
+      else
+	{
+	  doc_sepstr = de_escape_string (
+					  Xstrdup (GetDefEnv ("doc_sepstr",
+			      "---------------------------------- %n\\n")));
+	}
+      para_sepstr = de_escape_string (
+				       Xstrdup (GetDefEnv ("para_sepstr",
+				  "\\n######## PARAGRAPH %n ########\\n")));
+
+      para_start = de_escape_string (
+				      Xstrdup (GetDefEnv ("para_start",
+					    "***** Weight = %w *****\\n")));
+    }
+
+  if (need_text)
+    {
+      max_buf = atoi (GetDefEnv ("buffer", "1048576"));
+    }
+
+  do
+    {
+      u_char *UDoc = NULL;
+      unsigned long ULen;
+
+      if (need_text)
+	{
+	  /* load the compressed text */
+          int _status;
+
+	  _status = LoadCompressedText (qd, max_buf);
+	  if (_status == -2)
+	    { /**
+	       * GRB: fallback wihen text requested, but text was not
+               *      loaded
+	       * 17/09/99
+	       */
+	      Message("Compressed text not loaded - falling back to Document numbers.");
+	      UDoc = NULL;
+	      OutputType = OUTPUT_DOCNUMS;
+	      need_text  = 0;
+	    }
+	  else if (_status != 0)
+	    {
+	      Message ("Unable to load compressed text.");
+	      FatalError (1, "This is probably due to lack of memory.");
+	    }
+	  else
+	    {
+	      /* uncompress the loaded text */
+	      UDoc = GetDocText (qd, &ULen);
+	      if (UDoc == NULL)
+		{
+		  FatalError (1, "UDoc is unexpectedly NULL");
+		}
+	    }
+	}
+
+      if (!UDoc || PostProc ((char *) UDoc, verbatim))
+	{
+	  switch (OutputType)
+	    {
+	    case OUTPUT_COUNT:
+	    case OUTPUT_SILENT:
+	      break;
+	    case OUTPUT_DOCNUMS:	/* This prints out the docnums string */
+	      if (PagerRunning)
+		{
+
+/* [RPAP - Feb 97: NZDL Additions] */
+#if defined(PARADOCNUM) || defined(NZDL)
+		  int doc_num = GetDocNum(qd);
+
+		  if (qd->paragraph)
+		    {
+		      if (qd->id->ifh.InvfLevel == 3 &&
+			  (QueryType == 'R' || QueryType == 'A'))
+			{
+			  /* Print weights for each paragraph in document */
+			  
+
+			  int true_doc_num = GetDocNumFromParaNum(qd, doc_num);
+
+			  /* Get number of paragraphs in this document */
+
+			  int num_paragraphs = 
+			    qd->paragraph[true_doc_num]-qd->paragraph[true_doc_num-1];
+
+			  int init_para = FetchInitialParagraph(qd->td,
+								doc_num);
+			  DocEntry *de, *doc_chain = GetDocChain(qd);
+			  int i;
+
+			  for (i = 0; i < num_paragraphs; i++)
+			    {
+			      if ((de = in_chain(i, init_para, doc_chain)))
+				PrintDocNum(Output, QueryType,
+					    true_doc_num, init_para+i,
+					    de->Weight);
+			    }
+			}
+		      else
+			PrintDocNum(Output, QueryType,
+				    GetDocNumFromParaNum(qd, GetDocNum(qd)),
+				    GetDocNum(qd),
+				    GetDocWeight(qd));
+		    }
+		  else
+		    {
+		      PrintDocNum(Output, QueryType,
+				  doc_num, doc_num, GetDocWeight(qd));
+		    }
+#else
+		fprintf (Output, "%8d   %6.4f   %7lu\n", GetDocNum (qd),
+			 GetDocWeight (qd), GetDocCompLength (qd));
+#endif
+		}
+	      break;
+	    case OUTPUT_HEADERS:	/* This prints out the headers of the documents */
+	      if (PagerRunning)
+		fprintf (Output, "%d ", GetDocNum (qd));
+	      HeaderOut (Output, UDoc, ULen, heads_length);
+	      if (PagerRunning)
+		fputc ('\n', Output);
+	      break;
+#if TREC_MODE
+	    case OUTPUT_EXTRAS:	/* This prints out the docnums string */
+	      if (PagerRunning && trec_ids)
+		{
+		  long DN, PN = GetDocNum (qd) - 1;
+		  if (trec_paras)
+		    DN = trec_paras[PN];
+		  else
+		    DN = PN;
+		  fprintf (Output, "%-14.14s  %8ld  %10.5f\n",
+			   &trec_ids[DN * 14], PN + 1, GetDocWeight (qd));
+		}
+	      break;
+#endif
+	    case OUTPUT_TEXT:
+	    case OUTPUT_HILITE:
+	      {
+		int j, para = -1, curr_para = 0;
+		int init_para = -1;
+		DocEntry *de, *doc_chain = NULL;
+		register char ch = ' ';
+		register char lch = '\n';
+
+/* [RPAP - Feb 97: NZDL Additions] */
+#if defined(PARADOCNUM) || defined(NZDL)
+		if (qd->id->ifh.InvfLevel == 3)
+		  {
+		    init_para = FetchInitialParagraph(qd->td, GetDocNum(qd));
+		    
+		    StringOut(Output, para_sepstr, 
+			      1, init_para+curr_para,
+			      0, 0);
+
+		  }
+		else
+		  StringOut(Output, doc_sepstr, 
+			    1, GetDocNum(qd),
+			    QueryType == 'A' || QueryType == 'R',
+			    GetDocWeight(qd));
+
+#else
+		int p_on = 0;
+
+		if (PagerRunning)
+		  {
+		    StringOut (Output, doc_sepstr,
+			       1, GetDocNum (qd),
+			       QueryType == 'A' || QueryType == 'R',
+			       GetDocWeight (qd));
+		  }
+		if (qd->id->ifh.InvfLevel == 3)
+		  {
+		    init_para = FetchInitialParagraph (qd->td, GetDocNum (qd));
+		    doc_chain = GetDocChain (qd);
+		    para = GetDocNum (qd) - init_para;
+
+		    StringOut (Output, para_sepstr,
+			       1, curr_para + 1,
+			       0, 0);
+
+		    if ((de = in_chain (0, init_para, doc_chain)))
+		      StringOut (Output, para_start,
+				 0, 0,
+				 1, de->Weight);
+
+		    if (doc_chain->DocNum - init_para == 0)
+		      p_on = 1;
+		  }
+#endif
+		for (j = 0; j < ULen; j++)
+		  {
+		    ch = UDoc[j];
+		    switch (ch)
+		      {
+		      case '\02':
+			break;
+		      case '\01':
+			ch = '\n';
+		      case '\03':
+/* [RPAP - Feb 97: NZDL Additions] */
+#if defined(PARADOCNUM) || defined(NZDL)
+			/* print paragraph numbers only if this is
+			   a level 3 index */
+			if (qd->id->ifh.InvfLevel == 3)
+			  {
+			    curr_para++;
+			    StringOut(Output, para_sepstr, 
+				      1, init_para+curr_para,
+				      0, 0);
+			  }
+#else
+			p_on = 0;
+			curr_para++;
+			StringOut (Output, para_sepstr,
+				   1, curr_para + 1,
+				   0, 0);
+			lch = *(strchr (para_sepstr, '\0') - 1);
+			if ((de = in_chain (curr_para, init_para, doc_chain)))
+			  StringOut (Output, para_start,
+				     0, 0,
+				     1, de->Weight);
+			if (doc_chain &&
+			    doc_chain->DocNum - init_para == curr_para)
+			  p_on = 1;
+#endif
+			break;
+		      default:
+			{
+			  if (PagerRunning)
+			    {
+			      fputc (ch, Output);
+/* [RPAP - Feb 97: NZDL Additions] */
+#if !defined(PARADOCNUM) && !defined(NZDL)
+			      if (p_on && isprint (ch))
+				{
+				  fputc ('\b', Output);
+				  fputc ('_', Output);
+				}
+#endif
+			    }
+
+			  lch = ch;
+			}
+		      }
+		  }
+		if (PagerRunning && lch != '\n')
+		  fputc ('\n', Output);
+/* [RPAP - Feb 97: NZDL Additions] */
+#if !defined(PARADOCNUM) && !defined(NZDL)
+		p_on = 0;
+#endif
+	      }
+	    }
+	  if (PagerRunning)
+	    fflush (Output);
+
+	  DocCount++; /* moved within if statement [RJM June 1997 -- fixing post retrieval scan] */
+	}
+    }
+  while (NextDoc (qd) && PagerRunning && (!Ctrl_C));
+
+  if (need_text)
+    {
+      QueryData_FreeTextBuffer (qd);
+    }
+
+  if (OutputType == OUTPUT_TEXT || OutputType == OUTPUT_HILITE)
+    {
+      Xfree (doc_sepstr);
+      Xfree (para_sepstr);
+      Xfree (para_start);
+    }
+
+  return (DocCount);
+}
+
+
+void 
+output_terminator (FILE * out)
+{
+  char *terminator = Xstrdup (GetDefEnv ("terminator", ""));
+  de_escape_string (terminator);
+  fputs (terminator, out);
+  Xfree (terminator);
+}
+
+
+
+
+/* MoreDocs () */
+/* Displays all documents in list  DocList. */
+/* Documents are fetched, then decompressed and displayed according to the */
+/* format implied in  FormString(). */
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+# define HILITE_PAGER "mg_hilite_words.exe"
+#else
+# define HILITE_PAGER "mg_hilite_words"
+#endif
+
+#define MAX_HILITE_PAGER_STR 80	/* for command & its options */
+
+static void 
+MoreDocs (query_data * qd, char *Query, char OutputType)
+{
+  static char terms_str[MAXTERMSTRLEN + 1];
+  int DocCount = 0;		/* number of actual matches */
+  FILE *Output = NULL;
+  int using_pipe = 0;
+  char *pager = NULL;
+
+  Ctrl_C = 0;
+
+  qd->num_of_ans = qd->DL->size();
+
+  /* [RPAP - Feb 97: WIN32 Port] */
+#ifndef __WIN32__
+  signal (SIGPIPE, SIGPIPE_handler);
+#endif
+  signal (SIGINT, SIGINT_handler);
+
+  PagerRunning = 1;
+  if (isatty (fileno (OutFile)) && GetEnv ("pager") &&
+      OutputType != OUTPUT_HILITE &&
+      OutputType != OUTPUT_SILENT && OutputType != OUTPUT_COUNT)
+    {
+      pager = GetEnv ("pager");
+    }
+  else if (isatty (fileno (OutFile)) && OutputType == OUTPUT_HILITE)
+    {
+      /* concat the pager and its word argument strings */
+      TermList_toString (qd->TL, terms_str);
+      pager = (char *) Xmalloc (MAX_HILITE_PAGER_STR + strlen (terms_str) + 1);
+      if (!pager)
+	{
+	  fprintf (stderr, "Unable to allocate memory for highlighting\n");
+	  return;
+	}
+      sprintf (pager, "%s --style=%s --pager=%s --stem_method=%ld --stemmer=%ld %s",
+	       HILITE_PAGER,
+	       GetEnv ("hilite_style"),
+	       GetEnv ("pager"),
+	       qd->sd->sdh.stem_method,
+	       qd->sd->sdh.stemmer_num,
+	       terms_str);
+
+    }
+  else
+    {
+      Output = OutFile;
+    }
+
+/* [RPAP - Feb 97: NZDL Additions] */
+#if defined(OUTPUTSTEMMEDWORDS) || defined(NZDL)
+  if (!isatty(fileno(OutFile)) && get_query_type() != QUERY_DOCNUMS)
+    {
+      TermList_toString(qd->TL, terms_str);
+      fprintf(Output, "%s\n", terms_str);
+    }
+#endif
+  if (pager)
+    {
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+      Output = _popen (pager, "w");
+#else
+      Output = (FILE *) popen (pager, "w");
+#endif
+      using_pipe = (Output != NULL);
+      if (!using_pipe)
+	{
+	  fprintf (stderr, "Unable to run \"%s\"\n", pager);
+	  return;
+	}
+    }
+
+
+  if (qd->DL->size() > 0)
+    {
+      if (OutputType == OUTPUT_COUNT && !post_proc)
+	DocCount = qd->DL->size();
+      else {
+	DocCount = ProcessDocs (qd, qd->DL->size(),
+				BooleanEnv (GetEnv ("verbatim"), 1),
+				OutputType, Output);
+      }
+    }
+
+  if (PagerRunning)
+    {
+      output_terminator (Output);
+      fflush (Output);
+    }
+
+  if (OutputType == OUTPUT_HILITE && pager)
+    free (pager);		/* as needed to malloc to create the pager string */
+
+  if (using_pipe)
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+    _pclose (Output);
+#else
+    pclose (Output);
+#endif
+
+  if (qd->DL->size() == 0)
+    fprintf (stderr, "No entries correspond to that query.\n");
+  else
+    {
+      if (OutputType == OUTPUT_COUNT)
+	fprintf (stderr, "%d documents match.\n", DocCount);
+      else
+	fprintf (stderr, "%d documents retrieved.\n", DocCount);
+    }
+
+  signal (SIGINT, SIG_DFL);
+}
+
+
+void 
+start_up_stats (query_data * qd, InitQueryTimes iqt)
+{
+  Clear_Stats ();
+  if (BooleanEnv (GetEnv ("timestats"), 0) ||
+      BooleanEnv (GetEnv ("briefstats"), 0))
+    StartUpTimeStats (&iqt);
+
+  if (BooleanEnv (GetEnv ("diskstats"), 0) ||
+      BooleanEnv (GetEnv ("briefstats"), 0))
+    File_Stats (qd);
+
+  if (BooleanEnv (GetEnv ("memstats"), 0) ||
+      BooleanEnv (GetEnv ("briefstats"), 0))
+    MemStats (qd);
+
+}
+
+
+void 
+shut_down_stats (query_data * qd, ProgTime * start,
+		 ProgTime * invf, ProgTime * text)
+{
+  Clear_Stats ();
+  if (BooleanEnv (GetEnv ("timestats"), 0) ||
+      BooleanEnv (GetEnv ("briefstats"), 0))
+    QueryTimeStats (start, invf, text);
+
+  if (BooleanEnv (GetEnv ("diskstats"), 0) ||
+      BooleanEnv (GetEnv ("briefstats"), 0))
+    {
+      TransFileStats (qd);
+      File_Stats (qd);
+    }
+
+  if (BooleanEnv (GetEnv ("sizestats"), 0))
+    TotalSizeStats (qd);
+}
+
+
+
+char *wordfreqword2str (u_char * s)
+{
+  static char buf[1024];
+  int i, len = (int) *s++;
+
+  for (i = 0; i < len; i++)
+    {
+      buf[i] = (char)s[i];
+    }
+  buf[len] = '\0';
+
+  return buf;
+}
+
+
+/* [RPAP - Feb 97: Term Frequency] */
+/*********************************
+ * PrintQueryTermFreq
+ *
+ * Prints the query terms and their respective frequencies within the collection
+ *********************************/
+void
+PrintQueryTermFreqs (QueryTermList *qtl)
+{
+  int i;
+
+  /* Print the number of terms */
+  fprintf (OutFile, "%d\n", qtl->num);
+
+  /* Print the terms and their respective frequency within the collection */
+  for (i = 0; i < qtl->num; i++)
+    if (qtl->QTE[i].stem_method == -1)
+      /* Using default stem method - don't print stem method beside term */
+      fprintf (OutFile, "%s %d\n", wordfreqword2str (qtl->QTE[i].Term), qtl->QTE[i].Count);
+    else
+      /* Term was forced with a stem, print stem method with term */
+      fprintf (OutFile, "%s#%d %d\n", wordfreqword2str (qtl->QTE[i].Term), qtl->QTE[i].stem_method, qtl->QTE[i].Count);
+}
+
+
+void 
+query (void)
+{
+  ProgTime TotalStartTime, TotalInvfTime, TotalTextTime;
+  InitQueryTimes iqt;
+  query_data *qd;
+
+  TotalStartTime.RealTime = TotalStartTime.CPUTime = 0;
+  TotalInvfTime.RealTime = TotalInvfTime.CPUTime = 0;
+  TotalTextTime.RealTime = TotalTextTime.CPUTime = 0;
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+  qd = InitQuerySystemNT (GetDefEnv ("mgdir", ".\\"),
+                        GetDefEnv ("mgname", ""),
+			GetDefEnv ("textname", NULL), /* [RJM 06/97: text filename] */
+			&iqt);
+#else
+  qd = InitQuerySystemNT (GetDefEnv ("mgdir", "./"),
+                        GetDefEnv ("mgname", ""),
+			GetDefEnv ("textname", NULL), /* [RJM 06/97: text filename] */
+                        &iqt);
+#endif
+  
+  if (!qd)
+    FatalError (1, mg_errorstrs[mg_errno], mg_error_data);
+
+  start_up_stats (qd, iqt);
+
+  /*
+  FinishQuerySystem(qd);
+
+  return;
+  */
+  while (1)
+    {
+      ProgTime StartTime, InvfTime, TextTime;
+      char QueryType;
+      char OutputType;
+      char *line;
+      ResetFileStats (qd);
+      qd->max_mem_in_use = qd->mem_in_use = 0;
+
+      qd->tot_hops_taken += qd->hops_taken;
+      qd->tot_num_of_ptrs += qd->num_of_ptrs;
+      qd->tot_num_of_accum += qd->num_of_accum;
+      qd->tot_num_of_terms += qd->num_of_terms;
+      qd->tot_num_of_ans += qd->num_of_ans;
+      qd->tot_text_idx_lookups += qd->text_idx_lookups;
+      qd->hops_taken = qd->num_of_ptrs = 0;
+      qd->num_of_accum = qd->num_of_ans = qd->num_of_terms = 0;
+      qd->text_idx_lookups = 0;
+
+      Display_Stats (stderr);
+      Clear_Stats ();
+      line = get_query (qd);
+      if (!line || Quitting)
+	break;
+
+      GetPostProc (line);
+
+      GetTime (&StartTime);
+
+      QueryData_FreeQueryDocs (qd);
+
+      QueryType = get_query_type ();
+      OutputType = get_output_type ();
+      /* No point in hiliting words on a docnum query */
+      if (OutputType == OUTPUT_HILITE && QueryType == QUERY_DOCNUMS)
+	OutputType = OUTPUT_TEXT;
+
+      switch (QueryType)
+	{
+	case QUERY_BOOLEAN:
+	  {
+	    char *maxdocs;
+	    BooleanQueryInfo bqi;
+
+	    maxdocs = GetDefEnv ("maxdocs", "all");
+	    bqi.MaxDocsToRetrieve = strcmp (maxdocs, "all") ? atoi (maxdocs) : -1;
+
+	    /* [RPAP - Jan 97: Stem Index Change] */
+	    if (qd->sd->sdh.indexed)
+	      bqi.setWordActions(BooleanEnv(GetEnv("casefold"), 0),
+				 BooleanEnv(GetEnv("stem"), 0));
+	    else
+	      bqi.setWordActions(0, qd->sd->sdh.stem_method);
+
+	    bqi.OptimiseType = atoi (GetEnv ("optimise_type"));
+
+	    BooleanQuery (qd, line, &bqi);
+	    break;
+	  }
+	case QUERY_APPROX:
+	case QUERY_RANKED:
+	  {
+	    char *maxdocs;
+	    char *maxterms;
+	    char *maxaccum;
+	    RankedQueryInfo rqi;
+	    maxdocs = GetDefEnv ("maxdocs", "all");
+	    maxterms = GetDefEnv ("max_terms", "all");
+	    maxaccum = GetDefEnv ("max_accumulators", "all");
+	    rqi.Sort = BooleanEnv (GetEnv ("sorted_terms"), 0);
+	    rqi.QueryFreqs = BooleanEnv (GetEnv ("qfreq"), 1);
+	    rqi.Exact = QueryType == QUERY_RANKED;
+	    rqi.MaxDocsToRetrieve = strcmp (maxdocs, "all") ? atoi (maxdocs) : -1;
+	    rqi.MaxTerms = strcmp (maxterms, "all") ? atoi (maxterms) : -1;
+	    rqi.MaxParasToRetrieve = rqi.MaxDocsToRetrieve;
+	    if (qd->id->ifh.InvfLevel == 3 && GetEnv ("maxparas"))
+	      rqi.MaxParasToRetrieve = atoi (GetEnv ("maxparas"));
+	    rqi.AccumMethod = toupper (*GetDefEnv ("accumulator_method", "A"));
+	    rqi.MaxAccums = strcmp (maxaccum, "all") ? atoi (maxaccum) : -1;
+	    rqi.HashTblSize = IntEnv (GetEnv ("hash_tbl_size"), 1000);
+	    rqi.StopAtMaxAccum = BooleanEnv (GetEnv ("stop_at_max_accum"), 0);
+	    rqi.skip_dump = GetEnv ("skip_dump");
+	    RankedQuery (qd, line, &rqi);
+	    break;
+	  }
+	case QUERY_DOCNUMS:
+	  {
+	    DocnumsQuery (qd, line);
+	    break;
+	  }
+	}
+
+      GetTime (&InvfTime);
+
+      /* [RPAP - Feb 97: Term Frequency] */
+      if (qd->QTL && BooleanEnv (GetEnv ("term_freq"), 0))
+	PrintQueryTermFreqs (qd->QTL);
+
+      if (qd->DL)
+	MoreDocs (qd, line, OutputType);
+
+      GetTime (&TextTime);
+
+      if (BooleanEnv (GetEnv ("timestats"), 0) ||
+	  BooleanEnv (GetEnv ("briefstats"), 0))
+	QueryTimeStats (&StartTime, &InvfTime, &TextTime);
+
+      if (BooleanEnv (GetEnv ("diskstats"), 0) ||
+	  BooleanEnv (GetEnv ("briefstats"), 0))
+	File_Stats (qd);
+
+      if (BooleanEnv (GetEnv ("memstats"), 0) ||
+	  BooleanEnv (GetEnv ("briefstats"), 0))
+	MemStats (qd);
+
+      if (BooleanEnv (GetEnv ("sizestats"), 0))
+	SizeStats (qd);
+
+      TotalInvfTime.RealTime += InvfTime.RealTime - StartTime.RealTime;
+      TotalInvfTime.CPUTime += InvfTime.CPUTime - StartTime.CPUTime;
+      TotalTextTime.RealTime += TextTime.RealTime - StartTime.RealTime;
+      TotalTextTime.CPUTime += TextTime.CPUTime - StartTime.CPUTime;
+    }
+
+  if (isatty (fileno (InFile)) && !Quitting)
+    fprintf (stderr, "\n");
+
+  shut_down_stats (qd, &TotalStartTime, &TotalInvfTime, &TotalTextTime);
+
+  Display_Stats (stderr);
+
+}
+
+
+void 
+search_for_collection (char *name)
+{
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+  char *dir = GetDefEnv ("mgdir", ".\\");
+#else
+  char *dir = GetDefEnv ("mgdir", "./");
+#endif
+  char buffer[512];
+  struct stat stat_buf;
+  if (strrchr (dir, '/') && *(strrchr (dir, '/') + 1) != '\0')
+    {
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+      sprintf (buffer, "%s", dir);
+#else
+      sprintf (buffer, "%s/", dir);
+#endif
+      SetEnv ("mgdir", buffer, NULL);
+      dir = GetEnv ("mgdir");
+    }
+
+  sprintf (buffer, "%s.text", name);
+  if (stat (buffer, &stat_buf) != -1)
+    {
+      if (S_ISREG(stat_buf.st_mode))
+	{
+	  /* The name is a regular file */
+	  SetEnv ("mgname", name, NULL);
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+          SetEnv ("mgdir", ".\\", NULL);
+#else
+	  SetEnv ("mgdir", "./", NULL);
+#endif
+	  return;
+	}
+    }
+
+  sprintf (buffer, "%s%s", dir, name);
+  if (stat (buffer, &stat_buf) != -1)
+    {
+      if (S_ISDIR(stat_buf.st_mode))
+	{
+	  /* The name is a directory */
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+          sprintf (buffer, "%s%s", name, name);
+#else
+	  sprintf (buffer, "%s/%s", name, name);
+#endif
+	  SetEnv ("mgname", buffer, NULL);
+	  return;
+	}
+    }
+
+  /* Look in the current directory last */
+  if (stat (name, &stat_buf) != -1)
+    {
+      if (S_ISDIR(stat_buf.st_mode))
+	{
+	  /* The name is a directory */
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+          sprintf (buffer, "%s%s", name, name);
+          SetEnv ("mgdir", ".\\", NULL);
+#else
+	  sprintf (buffer, "%s/%s", name, name);
+          SetEnv ("mgdir", "./", NULL);
+#endif
+	  SetEnv ("mgname", buffer, NULL);
+	  return;
+	}
+    }
+
+  SetEnv ("mgname", name, NULL);
+}
+
+/* main () */
+/* Initialises global variables based on command line switches, and opens */
+/* files.  Then calls  query ()  to perform the querying. */
+int main (int argc, char **argv)
+{
+  ProgTime StartTime;
+  int decomp = 0;
+  int ch;
+
+  msg_prefix = argv[0];
+  GetTime (&StartTime);
+
+  /* Initialise the environment with default values */
+
+  InitEnv ();
+
+  read_mgrc_file ();
+
+  OutFile = stdout;
+  InFile = stdin;
+
+  opterr = 0;
+  /* [RJM 06/97: text filename] */
+  while ((ch = getopt (argc, argv, "Df:d:t:h")) != -1) {
+    switch (ch) {
+      case 'f':
+	SetEnv ("mgname", optarg, NULL);
+	break;
+      case 'd':
+	SetEnv ("mgdir", optarg, NULL);
+	break;
+      case 't': /* [RJM 06/97: text filename] */
+	SetEnv ("textname", optarg, NULL);
+	break;
+      case 'D':
+	decomp = 1;
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-D] [-f base name of collection] "
+		 "[-t base name of files for text] " /* [RJM 06/97: text filename] */
+		 "[-d data directory] [collection]\n", argv[0]);
+	exit (1);
+    }      
+  }
+
+  PushEnv ();
+
+  if (decomp == 0)
+    {
+
+      Init_ReadLine ();
+
+      /* write a first prompt, let the user start thinking */
+      if (!BooleanEnv (GetEnv ("expert"), 0) && isatty (fileno (InFile)))
+	{
+	  fprintf (stderr, "\n\n\t     FULL TEXT RETRIEVAL QUERY PROGRAM\n");
+	  fprintf (stderr, "%24s%s\n\n", "", *"21 Mar 1994" == '%' ? __DATE__ : "21 Mar 1994");
+	  fprintf (stderr, "\n");
+	  fprintf (stderr, "  mgquery version " VERSION ", Copyright (C) 1994 Neil Sharman\n");
+	  fprintf (stderr, "  mgquery comes with ABSOLUTELY NO WARRANTY; for details type `.warranty'\n");
+	  fprintf (stderr, "  This is free software, and you are welcome to redistribute it\n");
+	  fprintf (stderr, "  under certain conditions; type `.conditions' for details.\n");
+	  fprintf (stderr, "\n");
+	}
+    }
+  if (optind < argc)
+    search_for_collection (argv[optind]);
+
+  if (decomp == 0)
+    { 
+      query ();
+    }
+  else
+    { int i;
+      InitQueryTimes iqt;
+      query_data *qd;
+      
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+      qd = InitQuerySystemNT (GetDefEnv ("mgdir", ".\\"),
+                            GetDefEnv ("mgname", ""),
+                            GetDefEnv ("textname", NULL), /* [RJM 06/97: text filename] */
+                            &iqt);
+#else
+      qd = InitQuerySystemNT (GetDefEnv ("mgdir", "./"),
+			    GetDefEnv ("mgname", ""),
+                            GetDefEnv ("textname", NULL), /* [RJM 06/97: text filename] */
+			    &iqt);
+#endif
+      if (!qd)
+	FatalError (1, mg_errorstrs[mg_errno], mg_error_data);
+
+
+      start_up_stats (qd, iqt);
+
+      Display_Stats (stderr);
+      for (i = 0; i < qd->td->cth.num_of_docs; i++)
+	{
+	  RawDocOutput (qd, i + 1, stdout);
+	  putc ('\2', stdout);
+	}
+      Message ("%s", ElapsedTime (&StartTime, NULL));
+
+      FinishQuerySystem (qd);
+    }
+
+  UninitEnv ();
+  return 0;
+}
Index: /tags/gsdl-21/gsdl/src/mgpp/text/help.mg.h
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/help.mg.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/help.mg.h	(revision 699)
@@ -0,0 +1,320 @@
+static char *help_str[] = {
+"\n",
+"			  HELP for mgquery\n",
+"		   ================================\n",
+"\n",
+"The text is a summary of the information in the \"mgquery\" manual pages.\n",
+"\n",
+"The input to 'mgquery' consists of a series of input lines. The backslash\n",
+"character (\"\\\") is used at the end of lines to indicate that input \n",
+"continues on the next line.\n",
+"\n",
+"Input lines on which the first character is a dot (\".\") are commands to\n",
+"the mgquery program. Input lines that do not start with a dot are queries.\n",
+"\n",
+"A query consists of two parts. One part is a boolean or ranked query that\n",
+"identifies documents. The second part is a post-processing pattern matching\n",
+"operation. Any text between the first speech mark (\") and the last speech\n",
+"mark is considered to the a post-processing pattern. \n",
+"\n",
+"\n",
+"The following command are available :-\n",
+"\n",
+"    .help	     - displays this text.\n",
+"    .quit            - quits the program.\n",
+"    .set name value  - sets parameter \"name\" to \"value\" . If the parameter\n",
+"                       is a boolean parameter and value is omitted the\n",
+"	    	       parameter will be inverted (i.e. if it is true it\n",
+"		       will change to false, if it is false it will change\n",
+"		       to true).\n",
+"    .unset name      - deletes parameter \"name\"\n",
+"    .reset           - sets all the parameters to their initial state.\n",
+"    .display         - displays the values of all the current parameters.\n",
+"    .push            - pushes the current parameters on to a stack.\n",
+"    .pop             - destroys the current parameters and pops a new set\n",
+"                       of parameters off the stack.\n",
+"    .output arg      - This is used to specify where to send the documents.\n",
+"                       Arg may one be of the following:\n",
+"		       > filename  : Send output to the specified file.\n",
+"		       >> filename : Append output to the specified file.\n",
+"                       | command   : The output is piped into command, \n",
+"                                     which is executed by sh.\n",
+"    .input arg       - This is used to specify where input comes from.\n",
+"                       Arg may one be of the following:\n",
+"		       < filename  : Get the input from the specified\n",
+"                                     file.\n",
+"                       | command   : The input comes from the standard\n",
+"                                     output of command, which is executed\n",
+"                                     by sh.\n",
+"\n",
+"On startup the mgquery program reads from the file .mgrc a sequence \n",
+"of commands (NOTE: The .mgrc file may not contain any queries). mgquery \n",
+"first looks for .mgrc in the current directory and then in the users home \n",
+"directory. Lines starting with a '#' in the .mgrc file are considered to\n",
+"be comments and are ignored.\n",
+"\n",
+"The following parameters (used in the .set and .unset commands) are \n",
+"predefined and have special significance :-\n",
+"\n",
+"accumulator_method = `array'\n",
+"	This parameter is used during ranking, and specifies how the\n",
+"	weight for each document should be accumulated. The following\n",
+"	methods are available `array', `splay_tree', `hash_table', and\n",
+"	`list'.\n",
+"\n",
+"briefstats = `off'\n",
+"	This is a boolean parameter that determines whether the the\n",
+"	totals for disk, memory and time usage statistics will be \n",
+"	displayed. at the end of each query. \n",
+"	NOTE: this takes precedence over the parameters \"diskstats\",\n",
+"	\"memstats\" and \"timestats\". This parameter may take the values \n",
+"	`yes', `no', `true', `false', `on' or `off'.\n",
+"\n",
+"buffer = `1048576'\n",
+"	When the documents are being read in they are read into a \n",
+"	buffer of this size and then displayed from this buffer. If \n",
+"        the documents are larger than this buffer the buffer is\n",
+"	expanded automatically. Having a large buffer gives a very\n",
+"	slight performance improvement because it allows the order of \n",
+"	disk operations to be optimised. The buffer size is measured\n",
+"	in bytes.\n",
+"\n",
+"diskstats = `off'\n",
+"	This is a boolean parameter that determines whether the disk\n",
+"	usage statistics for the preceding query will be displayed\n",
+"	after each query. This parameter may take the values `yes',\n",
+"	`no', `true', `false', `on' or `off'.\n",
+"\n",
+"doc_sepstr = `---------------------------------- %n\\n'\n",
+"	This specifies the string that will be used to separate \n",
+"	documents when they are displayed for `boolean' or `docnums'\n",
+"	queries. The standard C escape character sequences (see the \n",
+"	man page) may be used to place special characters in the \n",
+"	string. For example, a newline would the `\\n'. To include a `%'\n",
+"	use the sequence `%%'. To include the MG document number use\n",
+"	the sequence `%n'.\n",
+"\n",
+"expert = `false'\n",
+"	If this is true then a lot of the waffle that the program \n",
+"	spits out is suppressed. This parameter may take the values\n",
+"	`yes', `no', `true', `false', `on' or `off'.\n",
+"\n",
+"hash_tbl_size = `1000'\n",
+"	One of the options during ranking queries is to use a hash \n",
+"	table to accumulate the weights for each document. The hash \n",
+"	table is a simple chained type. This parameter specifies the \n",
+"	size of the hash table and may take any value between 8 and\n",
+"	268435456. \n",
+"\n",
+"heads_length = `50'\n",
+"	When the mode is `heads' this specifies the number of \n",
+"	characters that	will be output for each document.\n",
+"\n",
+"maxdocs = `all'\n",
+"	The maximum number of documents to display in response to a\n",
+"	query. This parameter may take on a numeric value between 1 \n",
+"	and 429467295 or the word `all'.\n",
+"\n",
+"maxparas = `1000'\n",
+"	The maximum number of paragraphs to identify during a ranked\n",
+"	query with paragraph indexing. After the paragraphs have been\n",
+"	identified the paragraphs are converted into documents, and \n",
+"	because some of the paragraphs may refer to the same documents \n",
+"	the final number of answers may be less that maxparas. The \n",
+"	maxdocs parameter will then be applied. This parameter may \n",
+"	take on a numeric value between 1 and 429467295.\n",
+"\n",
+"max_accumulators = `50000'\n",
+"	This parameter limits the number of different paragraph/\n",
+"	document numbers to be accumulated during ranked queries when \n",
+"	the parameter `accumulator_method` is set to `splay_tree',\n",
+"	`hash_table', or `list'.  This parameter may take any value \n",
+"	between 8 and 268435456.\n",
+"\n",
+"max_terms = `all'\n",
+"	This parameter limits the number of terms that will actually\n",
+"	be used during a ranked query. If more terms than the number\n",
+"	specified by max_terms are entered, then the extra terms will\n",
+"	be discarded. If `sorted_terms' is on then the limiting will \n",
+"	be done after the terms have been sorted. This parameter may\n",
+"	take any value between 1 and 429467295 or the word `all'.\n",
+"\n",
+"memstats = `off'\n",
+"	This is a boolean parameter that determines whether the memory \n",
+"	usage statistics for the preceding query will be displayed\n",
+"	after each query. This parameter may take the values `yes', \n",
+"	`no', `true', `false', `on' or `off'.\n",
+"\n",
+"mgdir = `.'\n",
+"	This specifies the directory where the MG files may be found.\n",
+"	If the environment variable `MGDATA' is set then `mgdir' is\n",
+"	initialised to the value in `MGDATA'.\n",
+"\n",
+"mgname = `'\n",
+"	This specifies the name of the MG database to process.\n",
+"\n",
+"mode = `text'\n",
+"	This specifies how documents should be displayed when they\n",
+"	are retrieved it may take four different values `text', \n",
+"	`docnums', `silent', `heads' or `count'. `text' displays \n",
+"	the contents of the document. `docnums' displays only the\n",
+"	document numbers. `Silent' retrieves all the documents but\n",
+"	displays nothing except how many documents were retrieved.\n",
+"	This mode is intended to be used in timing experiments. \n",
+"	`Heads` is used to print out the head of each document.\n",
+"	`Count' does the minimum amount of work required to determine\n",
+"	how many documents would be retrieved, but does not retrieve\n",
+"	them.\n",
+"\n",
+"pager = `more'\n",
+"	This is the name of the program that will be used to display\n",
+"	the help and the retrieved documents. If the environment \n",
+"	variable \"PAGER\" is defined then `pager' takes on that value.\n",
+"\n",
+"para_sepstr = `\\n######## PARAGRAPH %n ########\\n'\n",
+"	This specifies the string that will be used to separate \n",
+"	paragraphs. The standard C escape character sequences (see the \n",
+"	man page) may be used to place special characters in the \n",
+"	string. For example, a newline would the `\\n'. To include a `%'\n",
+"	use the sequence `%%'. To include the paragraph number within\n",
+"	the document use the sequence `%n'.\n",
+"\n",
+"para_start = `***** Weight = %w *****\\n'\n",
+"	This specifies the string that will be used at the head of \n",
+"	paragraphs for a paraghaph level index following a ranked query.\n",
+"	The standard C escape character sequences (see the man page)\n",
+"	may be used to place special characters in the string. For \n",
+"	example, a newline would the `\\n'. To include a `%' use the\n",
+"	sequence `%%'. To include the paragraph weight use the \n",
+"	sequence `%w'.\n",
+"\n",
+"qfreq = `true'\n",
+"	This determine whether the ranked queries will take into \n",
+"	account the number of times each query term is specified.\n",
+"	When this is `true' the number of times a term appears in\n",
+"	the query is used in the ranking. When this is `false' all \n",
+"	query term are assumed to occur only once. This parameter\n",
+"	may take the values `yes', `no', `true', `false', `on' or\n",
+"	`off'.\n",
+"\n",
+"query = `boolean'\n",
+"	This specifies the type of queries that are to be specified.\n",
+"	It can take four different values `boolean', `ranked', \n",
+"	`docnums' or `approx-ranked'. \n",
+"\n",
+"	`boolean' is for boolean queries. \n",
+"		The yacc grammar for boolean queries is as follows :-\n",
+"\n",
+"		query   : or;\n",
+" \n",
+"		or      : or '|' and\n",
+"		        | and ;	\n",
+" \n",
+"		and     : and '&' not\n",
+"        		| and not\n",
+"        		| not ;\n",
+"\n",
+"		not     : term\n",
+"		        | '!' not ;\n",
+" \n",
+"		term    : TERM\n",
+"		        | '(' or ')' ;\n",
+" \n",
+"\n",
+"\n",
+"\n",
+"	`ranked' and `approx-ranked' are for queries ranked by the\n",
+"		cosine measure. `approx-ranked' uses only the low\n",
+"		precision document lengths, and therefore only\n",
+"		produces an approximation to full cosine ranking.\n",
+"\n",
+"		query   : TERM\n",
+"      			| query TERM ;\n",
+"	\n",
+"	`docnums' allows the entry of document numbers. Multiple \n",
+"		numbers separated by spaces may be specified\n",
+"		or ranges separated by hyphens.\n",
+"\n",
+"\n",
+"		query   : range\n",
+"			| query range ;\n",
+"\n",
+"\n",
+"		range   : num\n",
+"			| num '-' num ;\n",
+"\n",
+"\n",
+"ranked_doc_sepstr = `---------------------------------- %n %w\\n'\n",
+"	This specifies the string that will be used to separate \n",
+"	documents when they are displayed for `ranked' or \n",
+"	`approx-ranked'	queries. The standard C escape character \n",
+"	sequences (see the man page) may be used to place special\n",
+"	characters in the string. For example, a newline would the\n",
+"	`\\n'. To include a `%' use the sequence `%%'. To include the\n",
+"	MG document number use the sequence `%n'. To include the\n",
+"	document weight use the sequence `%w'.\n",
+"\n",
+"sizestats = `false'\n",
+"	If this is true then various numbers are output at the end\n",
+"	of each query indicating what went on during the query. \n",
+"	This parameter may take the values `yes', `no', `true', \n",
+"	`false', `on' or `off'.\n",
+"\n",
+"skip_dump = `skips.%d'\n",
+"	If this parameter is set then during ranked queries on skipped\n",
+"	inverted files when `accumulator_method' is set to `splay_tree',\n",
+"	`hash_table', or `list` a file will be produced in the current\n",
+"	directory. The name of the file is the value of this parameter,\n",
+"	a `%d' in the file name will be replaced with the process id of\n",
+"	mgquery. This file will contain information about the usage of\n",
+"	skips during the query processing. This option is expensive; \n",
+"	use `.unset skip_dump' to obtain optimal performance.\n",
+"\n",
+"sorted_terms = `on'\n",
+"	This specifies whether of not the terms should be sorted into\n",
+"	decreasing occurrence in documents so that the least often\n",
+"	occurring terms are processed first when ranked queries are\n",
+"	being done. When this is true the terms are sorted. When this\n",
+"	is false the terms are not sorted and are instead processed in\n",
+"	order of occurrence. This parameter may take the values `yes',\n",
+"	`no', `true', `false', `on' or `off'.\n",
+"\n",
+"\n",
+"stop_at_max_accum = `on'\n",
+"	This specifies what should happen when the maximum number of\n",
+"	accumulators set by `max_accumulators' is reached. When this\n",
+"	is true the the processing of terms is stopped at the completion\n",
+"	of the current term. When this is false processing continues but\n",
+"	no new accumulators are created. This parameter may take the \n",
+"	values `yes', `no', `true', `false', `on' or `off'.\n",
+"\n",
+"terminator = `'\n",
+"	This specifies the string that will be output after the	last\n",
+"	document from the previous query has been output. The standard\n",
+"	C escape character sequences (see the man page) may be used to\n",
+"	place special characters in the string. For example, a newline\n",
+"	would the `\\n'. To include a `%' use the sequence `%%'.\n",
+"	\n",
+"\n",
+"timestats = `false'\n",
+"	If this is true then the time to process a query is displayed\n",
+"	in both real time and CPU time. This parameter may take the\n",
+"	values `yes', `no', `true', `false', `on' or `off'.\n",
+"\n",
+"verbatim = `off'\n",
+"	This is a boolean parameter that determines whether the program\n",
+"	should attempt to do a regular expression match on the retrieved\n",
+"	text. If verbatim is `on' and a post-processing strng is specified\n",
+"	with the query then the post-processing string will be searched for\n",
+"	in the documents just before they are displayed. If the string is\n",
+"	found the document will be displayed, if not the document will not\n",
+"	be displayed. If verbatim is `off' the post-processing string will\n",
+"	be considered a regular expression like in `vi' or `egrep'.\n",
+"	E.G. If verbatim is `on', \"and.*the\" will look for the 8 character\n",
+"	     sequence \"and.*the\". If verbatim is `off', \"and.*the\" will\n",
+"	     look for the sequence \"and\" followed somewhere later in the\n",
+"	     document by the sequence \"the\".\n",
+"	This parameter may take the values `yes', `no', `true', `false',\n",
+"	`on' or `off'.\n",
+"\n",
+"\n"};
Index: /tags/gsdl-21/gsdl/src/mgpp/text/help.mg.src
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/help.mg.src	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/help.mg.src	(revision 699)
@@ -0,0 +1,347 @@
+###########################################################################
+#
+# help.mg.src -- Source for the help command
+# Copyright (C) 1994  Neil Sharman
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)help.mg.src	1.8 21 Mar 1994
+#
+###########################################################################
+#
+# The help file for mgquery.  
+# Lines starting with '#' are treated as comments and are discarded when
+# the 'help.mg.h' file is produced
+#
+###########################################################################
+
+			  HELP for mgquery
+		   ================================
+
+The text is a summary of the information in the "mgquery" manual pages.
+
+The input to 'mgquery' consists of a series of input lines. The backslash
+character ("\") is used at the end of lines to indicate that input 
+continues on the next line.
+
+Input lines on which the first character is a dot (".") are commands to
+the mgquery program. Input lines that do not start with a dot are queries.
+
+A query consists of two parts. One part is a boolean or ranked query that
+identifies documents. The second part is a post-processing pattern matching
+operation. Any text between the first speech mark (") and the last speech
+mark is considered to the a post-processing pattern. 
+
+
+The following command are available :-
+
+    .help	     - displays this text.
+    .quit            - quits the program.
+    .set name value  - sets parameter "name" to "value" . If the parameter
+                       is a boolean parameter and value is omitted the
+	    	       parameter will be inverted (i.e. if it is true it
+		       will change to false, if it is false it will change
+		       to true).
+    .unset name      - deletes parameter "name"
+    .reset           - sets all the parameters to their initial state.
+    .display         - displays the values of all the current parameters.
+    .push            - pushes the current parameters on to a stack.
+    .pop             - destroys the current parameters and pops a new set
+                       of parameters off the stack.
+    .output arg      - This is used to specify where to send the documents.
+                       Arg may one be of the following:
+		       > filename  : Send output to the specified file.
+		       >> filename : Append output to the specified file.
+                       | command   : The output is piped into command, 
+                                     which is executed by sh.
+    .input arg       - This is used to specify where input comes from.
+                       Arg may one be of the following:
+		       < filename  : Get the input from the specified
+                                     file.
+                       | command   : The input comes from the standard
+                                     output of command, which is executed
+                                     by sh.
+
+On startup the mgquery program reads from the file .mgrc a sequence 
+of commands (NOTE: The .mgrc file may not contain any queries). mgquery 
+first looks for .mgrc in the current directory and then in the users home 
+directory. Lines starting with a '#' in the .mgrc file are considered to
+be comments and are ignored.
+
+The following parameters (used in the .set and .unset commands) are 
+predefined and have special significance :-
+
+accumulator_method = `array'
+	This parameter is used during ranking, and specifies how the
+	weight for each document should be accumulated. The following
+	methods are available `array', `splay_tree', `hash_table', and
+	`list'.
+
+briefstats = `off'
+	This is a boolean parameter that determines whether the the
+	totals for disk, memory and time usage statistics will be 
+	displayed. at the end of each query. 
+	NOTE: this takes precedence over the parameters "diskstats",
+	"memstats" and "timestats". This parameter may take the values 
+	`yes', `no', `true', `false', `on' or `off'.
+
+buffer = `1048576'
+	When the documents are being read in they are read into a 
+	buffer of this size and then displayed from this buffer. If 
+        the documents are larger than this buffer the buffer is
+	expanded automatically. Having a large buffer gives a very
+	slight performance improvement because it allows the order of 
+	disk operations to be optimised. The buffer size is measured
+	in bytes.
+
+diskstats = `off'
+	This is a boolean parameter that determines whether the disk
+	usage statistics for the preceding query will be displayed
+	after each query. This parameter may take the values `yes',
+	`no', `true', `false', `on' or `off'.
+
+doc_sepstr = `---------------------------------- %n\n'
+	This specifies the string that will be used to separate 
+	documents when they are displayed for `boolean' or `docnums'
+	queries. The standard C escape character sequences (see the 
+	man page) may be used to place special characters in the 
+	string. For example, a newline would the `\n'. To include a `%'
+	use the sequence `%%'. To include the MG document number use
+	the sequence `%n'.
+
+expert = `false'
+	If this is true then a lot of the waffle that the program 
+	spits out is suppressed. This parameter may take the values
+	`yes', `no', `true', `false', `on' or `off'.
+
+hash_tbl_size = `1000'
+	One of the options during ranking queries is to use a hash 
+	table to accumulate the weights for each document. The hash 
+	table is a simple chained type. This parameter specifies the 
+	size of the hash table and may take any value between 8 and
+	268435456. 
+
+heads_length = `50'
+	When the mode is `heads' this specifies the number of 
+	characters that	will be output for each document.
+
+maxdocs = `all'
+	The maximum number of documents to display in response to a
+	query. This parameter may take on a numeric value between 1 
+	and 429467295 or the word `all'.
+
+maxparas = `1000'
+	The maximum number of paragraphs to identify during a ranked
+	query with paragraph indexing. After the paragraphs have been
+	identified the paragraphs are converted into documents, and 
+	because some of the paragraphs may refer to the same documents 
+	the final number of answers may be less that maxparas. The 
+	maxdocs parameter will then be applied. This parameter may 
+	take on a numeric value between 1 and 429467295.
+
+max_accumulators = `50000'
+	This parameter limits the number of different paragraph/
+	document numbers to be accumulated during ranked queries when 
+	the parameter `accumulator_method` is set to `splay_tree',
+	`hash_table', or `list'.  This parameter may take any value 
+	between 8 and 268435456.
+
+max_terms = `all'
+	This parameter limits the number of terms that will actually
+	be used during a ranked query. If more terms than the number
+	specified by max_terms are entered, then the extra terms will
+	be discarded. If `sorted_terms' is on then the limiting will 
+	be done after the terms have been sorted. This parameter may
+	take any value between 1 and 429467295 or the word `all'.
+
+memstats = `off'
+	This is a boolean parameter that determines whether the memory 
+	usage statistics for the preceding query will be displayed
+	after each query. This parameter may take the values `yes', 
+	`no', `true', `false', `on' or `off'.
+
+mgdir = `.'
+	This specifies the directory where the MG files may be found.
+	If the environment variable `MGDATA' is set then `mgdir' is
+	initialised to the value in `MGDATA'.
+
+mgname = `'
+	This specifies the name of the MG database to process.
+
+mode = `text'
+	This specifies how documents should be displayed when they
+	are retrieved it may take four different values `text', 
+	`docnums', `silent', `heads' or `count'. `text' displays 
+	the contents of the document. `docnums' displays only the
+	document numbers. `Silent' retrieves all the documents but
+	displays nothing except how many documents were retrieved.
+	This mode is intended to be used in timing experiments. 
+	`Heads` is used to print out the head of each document.
+	`Count' does the minimum amount of work required to determine
+	how many documents would be retrieved, but does not retrieve
+	them.
+
+pager = `more'
+	This is the name of the program that will be used to display
+	the help and the retrieved documents. If the environment 
+	variable "PAGER" is defined then `pager' takes on that value.
+
+para_sepstr = `\n######## PARAGRAPH %n ########\n'
+	This specifies the string that will be used to separate 
+	paragraphs. The standard C escape character sequences (see the 
+	man page) may be used to place special characters in the 
+	string. For example, a newline would the `\n'. To include a `%'
+	use the sequence `%%'. To include the paragraph number within
+	the document use the sequence `%n'.
+
+para_start = `***** Weight = %w *****\n'
+	This specifies the string that will be used at the head of 
+	paragraphs for a paraghaph level index following a ranked query.
+	The standard C escape character sequences (see the man page)
+	may be used to place special characters in the string. For 
+	example, a newline would the `\n'. To include a `%' use the
+	sequence `%%'. To include the paragraph weight use the 
+	sequence `%w'.
+
+qfreq = `true'
+	This determine whether the ranked queries will take into 
+	account the number of times each query term is specified.
+	When this is `true' the number of times a term appears in
+	the query is used in the ranking. When this is `false' all 
+	query term are assumed to occur only once. This parameter
+	may take the values `yes', `no', `true', `false', `on' or
+	`off'.
+
+query = `boolean'
+	This specifies the type of queries that are to be specified.
+	It can take four different values `boolean', `ranked', 
+	`docnums' or `approx-ranked'. 
+
+	`boolean' is for boolean queries. 
+		The yacc grammar for boolean queries is as follows :-
+
+		query   : or;
+ 
+		or      : or '|' and
+		        | and ;	
+ 
+		and     : and '&' not
+        		| and not
+        		| not ;
+
+		not     : term
+		        | '!' not ;
+ 
+		term    : TERM
+		        | '(' or ')' ;
+ 
+
+
+
+	`ranked' and `approx-ranked' are for queries ranked by the
+		cosine measure. `approx-ranked' uses only the low
+		precision document lengths, and therefore only
+		produces an approximation to full cosine ranking.
+
+		query   : TERM
+      			| query TERM ;
+	
+	`docnums' allows the entry of document numbers. Multiple 
+		numbers separated by spaces may be specified
+		or ranges separated by hyphens.
+
+
+		query   : range
+			| query range ;
+
+
+		range   : num
+			| num '-' num ;
+
+
+ranked_doc_sepstr = `---------------------------------- %n %w\n'
+	This specifies the string that will be used to separate 
+	documents when they are displayed for `ranked' or 
+	`approx-ranked'	queries. The standard C escape character 
+	sequences (see the man page) may be used to place special
+	characters in the string. For example, a newline would the
+	`\n'. To include a `%' use the sequence `%%'. To include the
+	MG document number use the sequence `%n'. To include the
+	document weight use the sequence `%w'.
+
+sizestats = `false'
+	If this is true then various numbers are output at the end
+	of each query indicating what went on during the query. 
+	This parameter may take the values `yes', `no', `true', 
+	`false', `on' or `off'.
+
+skip_dump = `skips.%d'
+	If this parameter is set then during ranked queries on skipped
+	inverted files when `accumulator_method' is set to `splay_tree',
+	`hash_table', or `list` a file will be produced in the current
+	directory. The name of the file is the value of this parameter,
+	a `%d' in the file name will be replaced with the process id of
+	mgquery. This file will contain information about the usage of
+	skips during the query processing. This option is expensive; 
+	use `.unset skip_dump' to obtain optimal performance.
+
+sorted_terms = `on'
+	This specifies whether of not the terms should be sorted into
+	decreasing occurrence in documents so that the least often
+	occurring terms are processed first when ranked queries are
+	being done. When this is true the terms are sorted. When this
+	is false the terms are not sorted and are instead processed in
+	order of occurrence. This parameter may take the values `yes',
+	`no', `true', `false', `on' or `off'.
+
+
+stop_at_max_accum = `on'
+	This specifies what should happen when the maximum number of
+	accumulators set by `max_accumulators' is reached. When this
+	is true the the processing of terms is stopped at the completion
+	of the current term. When this is false processing continues but
+	no new accumulators are created. This parameter may take the 
+	values `yes', `no', `true', `false', `on' or `off'.
+
+terminator = `'
+	This specifies the string that will be output after the	last
+	document from the previous query has been output. The standard
+	C escape character sequences (see the man page) may be used to
+	place special characters in the string. For example, a newline
+	would the `\n'. To include a `%' use the sequence `%%'.
+	
+
+timestats = `false'
+	If this is true then the time to process a query is displayed
+	in both real time and CPU time. This parameter may take the
+	values `yes', `no', `true', `false', `on' or `off'.
+
+verbatim = `off'
+	This is a boolean parameter that determines whether the program
+	should attempt to do a regular expression match on the retrieved
+	text. If verbatim is `on' and a post-processing strng is specified
+	with the query then the post-processing string will be searched for
+	in the documents just before they are displayed. If the string is
+	found the document will be displayed, if not the document will not
+	be displayed. If verbatim is `off' the post-processing string will
+	be considered a regular expression like in `vi' or `egrep'.
+	E.G. If verbatim is `on', "and.*the" will look for the 8 character
+	     sequence "and.*the". If verbatim is `off', "and.*the" will
+	     look for the sequence "and" followed somewhere later in the
+	     document by the sequence "the".
+	This parameter may take the values `yes', `no', `true', `false',
+	`on' or `off'.
+
+
Index: /tags/gsdl-21/gsdl/src/mgpp/text/invf_get.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/invf_get.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/invf_get.cpp	(revision 699)
@@ -0,0 +1,1640 @@
+/**************************************************************************
+ *
+ * invf_get.c -- Inverted file reading routines
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "filestats.h"
+#include "timing.h"
+#include "bitio_gen.h"
+#include "bitio_m_abstract.h"
+#include "bitio_m_mem.h"
+#include "bitio_m.h"
+#include "sptree.h"
+#include "messages.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg.h"
+#include "invf.h"
+#include "text.h"
+#include "lists.h"
+#include "backend.h"
+#include "invf_get.h"
+#include "mg_errors.h"
+#include "DocEntry.h"
+
+#define SKIPPING
+#define SHORT_CUT 1
+
+struct pool
+  {
+    int length, pos;
+    struct pool *next;
+    Invf_Doc_Entry ide[1];
+  };
+
+static int skip_header = 0;
+
+static char skip_header_text0[] =
+"#######################################################################\n"
+"#\n"
+"# 1\tWord number.\n"
+"# 2\tNumber of time the word occurs in the collection.\n"
+"# 3\tNumber of documents the word occurs in.\n"
+"# 4\tThe word.\n"
+"# 5\tNumber of docnum/word_count entries decoded.\n"
+"# 6\tNumber of hits.\n"
+"# 7\tNumber of entries in the splay tree or hash table.\n"
+"# 8\tAmount added to the accumulators while while processing this word.\n"
+"#\n"
+"#######################################################################\n";
+
+static char skip_header_text1[] =
+"#######################################################################\n"
+"#\n"
+"# 1\tWord number.\n"
+"# 2\tNumber of time the word occurs in the collection.\n"
+"# 3\tNumber of documents the word occurs in.\n"
+"# 4\tThe word.\n"
+"# 5\tNumber of skips taken in evaluating the inverted file entry.\n"
+"# 6\tNumber of docnum/word_count entries decoded.\n"
+"# 7\tNumber of hits.\n"
+"# 8\tNumber of entries in the splay tree or hash table.\n"
+"# 9\tAmount added to the accumulators while while processing this word.\n"
+"#\n"
+"#######################################################################\n";
+
+
+static char skip_header_text2[] =
+"#######################################################################\n"
+"#\n"
+"#  1\tWord number.\n"
+"#  2\tNumber of time the word occurs in the collection.\n"
+"#  3\tNumber of documents the word occurs in.\n"
+"#  4\tThe word.\n"
+"#  5\tSkip size.\n"
+"#  6\tNumber of skips taken in evaluating the inverted file entry.\n"
+"#  7\tNumber of docnum/word_count entries decoded.\n"
+"#  8\tNumber of hits.\n"
+"#  9\tNumber of entries in the splay tree or hash table.\n"
+"# 10\tAmount added to the accumulators while while processing this word.\n"
+"#\n"
+"#######################################################################\n";
+
+static char *skip_header_text[] =
+{skip_header_text0,
+ skip_header_text1,
+ skip_header_text2};
+
+
+#define COSINE_STARTTERM \
+if (cs.startTerm() != 0)                                     \
+  break;                                                     \
+                                                             \
+DECODE_START (cs.buffer, cs.we->invf_len);                   \
+                                                             \
+cs.WordLog = log ((double) cs.num_docs / (double) cs.we->max_doc_count);  \
+                                                             \
+cs.Wqt = (rqi->QueryFreqs ? Terms->TE[cs.term].Count : 1) * cs.WordLog;
+
+
+/**************************************************************************
+ *
+ * Code for the hash table.
+ *
+ */
+
+static Hash_Table *
+HT_create (query_data * qd, unsigned max_entries,
+	   unsigned max_size)
+{
+  unsigned es, hts;
+  Hash_Table *HT;
+  if (max_entries == 0)
+    max_entries = 8192;
+  es = sizeof (Hash_Table) + sizeof (Invf_Doc_EntryH) * (max_entries - 1);
+  hts = sizeof (Invf_Doc_EntryH *) * max_size;
+
+  HT = (Hash_Table *) Xmalloc (es);
+  if (!HT)
+    return NULL;
+
+  HT->HashTable = (Invf_Doc_EntryH **) Xmalloc (hts);
+  if (!(HT->HashTable))
+    {
+      Xfree (HT);
+      return NULL;
+    }
+
+  ChangeMemInUse (qd, es + hts);
+  bzero ((char *) (HT->HashTable), hts);
+  HT->max = max_entries;
+  HT->num = 0;
+  HT->Suplimentary_Entries = NULL;
+  HT->Suplimentary_Size = 0;
+  HT->Suplimentary_Num = 0;
+  HT->Head = NULL;
+  HT->size = max_size;
+  return HT;
+}
+
+
+static void 
+HT_Sup_create (query_data * qd, Hash_Table * HT, unsigned max_size)
+{
+  int es = sizeof (Invf_Doc_EntryH) * max_size;
+  HT->Suplimentary_Entries = (Invf_Doc_EntryH *) Xmalloc (es);
+  if (!HT->Suplimentary_Entries)
+    return;
+  HT->Suplimentary_Size = max_size;
+  HT->Suplimentary_Num = 0;
+  ChangeMemInUse (qd, es);
+}
+
+static void 
+HT_Sup_Add (Hash_Table * HT, unsigned long DN, float Sum)
+{
+  Invf_Doc_EntryH *IDEH;
+  IDEH = &HT->Suplimentary_Entries[HT->Suplimentary_Num++];
+  IDEH->next = NULL;
+  IDEH->DocNum = DN;
+  IDEH->Weight = Sum;
+}
+
+abstractDocEntry *
+HT_find (Hash_Table * HT, unsigned long DN)
+{
+  Invf_Doc_EntryH *IDEH;
+  if (!HT->HashTable)
+    FatalError (1, "Accessing the hash_table through HT_find after HT_sort");
+  IDEH = HT->HashTable[DN % HT->size];
+  while (IDEH)
+    {
+      if (IDEH->DocNum == DN)
+	return IDEH;
+      IDEH = IDEH->next;
+    }
+  return NULL;
+}
+
+
+static int 
+HT_insert (Hash_Table * HT,
+	   unsigned long DN, float Sum)
+{
+  Invf_Doc_EntryH **IDEH;
+  if (HT->num == HT->max)
+    return 0;
+  IDEH = &(HT->HashTable[DN % HT->size]);
+  while (*IDEH)
+    IDEH = &((*IDEH)->next);
+  *IDEH = &(HT->IDEH[HT->num++]);
+  (*IDEH)->next = NULL;
+  (*IDEH)->DocNum = DN;
+  (*IDEH)->Weight = Sum;
+  return 1;
+}
+
+
+void 
+HT_free (query_data * qd, Hash_Table * HT)
+{
+  unsigned es;
+  es = sizeof (Hash_Table) + sizeof (Invf_Doc_EntryH) * HT->max +
+    sizeof (Invf_Doc_EntryH *) * HT->size;
+  if (HT->HashTable)
+    Xfree (HT->HashTable);
+  if (HT->Suplimentary_Entries)
+    {
+      Xfree (HT->Suplimentary_Entries);
+      es += sizeof (Invf_Doc_EntryH) * HT->Suplimentary_Size;
+    }
+  Xfree (HT);
+  ChangeMemInUse (qd, -es);
+}
+
+
+static int 
+IDEH_comp (const void *a, const void *b)
+{
+  const Invf_Doc_EntryH *A = (Invf_Doc_EntryH *) a;
+  const Invf_Doc_EntryH *B = (Invf_Doc_EntryH *) b;
+  return A->DocNum - B->DocNum;
+}
+
+static void 
+HT_sort (query_data * qd, Hash_Table * HT)
+{
+  int i;
+  Invf_Doc_EntryH *l, *r, **b;
+  unsigned es = sizeof (Invf_Doc_EntryH *) * HT->size;
+  if (HT->HashTable)
+    Xfree (HT->HashTable);
+  HT->size = 0;
+  HT->HashTable = NULL;
+  ChangeMemInUse (qd, -es);
+
+  qsort (HT->IDEH, HT->num, sizeof (Invf_Doc_EntryH), IDEH_comp);
+
+  for (i = 0; i < HT->num - 1; i++)
+    HT->IDEH[i].next = &HT->IDEH[i + 1];
+  HT->IDEH[i].next = NULL;
+  l = HT->IDEH;
+
+  if (HT->Suplimentary_Entries)
+    {
+      for (i = 0; i < HT->Suplimentary_Num - 1; i++)
+	HT->Suplimentary_Entries[i].next = &HT->Suplimentary_Entries[i + 1];
+      HT->Suplimentary_Entries[i].next = NULL;
+    }
+  r = HT->Suplimentary_Entries;
+
+  b = &HT->Head;
+
+  while (l || r)
+    {
+      if (r && (!l || (l->DocNum > r->DocNum)))
+	{
+	  *b = r;
+	  b = &r->next;
+	  r = *b;
+	}
+      else
+	{
+	  *b = l;
+	  b = &l->next;
+	  l = *b;
+	}
+    }
+  *b = NULL;
+}
+
+
+
+
+
+
+/**************************************************************************
+ *
+ * Code for the hash table.
+ *
+ */
+
+static List_Table *
+LT_create (query_data * qd, unsigned max_entries)
+{
+  unsigned es;
+  List_Table *LT;
+  if (max_entries == 0)
+    max_entries = 8192;
+  es = sizeof (List_Table) + sizeof (Invf_Doc_Entry) * (max_entries - 1);
+
+  LT = (List_Table *) Xmalloc (es);
+  if (!LT)
+    return NULL;
+
+  ChangeMemInUse (qd, es);
+  LT->max = max_entries;
+  LT->num = 0;
+  return LT;
+}
+
+
+
+Invf_Doc_Entry *
+LT_find (List_Table * LT, unsigned long DN)
+{
+  int l = 0;
+  int r = LT->num - 1;
+  int m, c;
+  while (l <= r)
+    {
+      m = (l + r) / 2;
+      c = DN - LT->IDE[m].DocNum;
+      if (c < 0)
+	r = m - 1;
+      else if (c > 0)
+	l = m + 1;
+      else
+	return &LT->IDE[m];
+    }
+  return NULL;
+}
+
+static List_Table *
+LT_add (query_data * qd, List_Table * LT,
+	unsigned long DN, float Sum)
+{
+  Invf_Doc_Entry *ide;
+
+  /* increase the size of the list if there is not enough */
+  /* room for this entry */
+  if (LT->num == LT->max)
+    {
+      int old, enew, max_entries;
+      old = sizeof (List_Table) + sizeof (Invf_Doc_Entry) * LT->max;
+      max_entries = LT->max + (LT->max >> 1);
+      enew = sizeof (List_Table) + sizeof (Invf_Doc_Entry) * max_entries;
+
+      LT = (List_Table *)Xrealloc (LT, enew);
+      if (!LT)
+	return NULL;
+
+      ChangeMemInUse (qd, enew - old);
+      LT->max = max_entries;
+    }
+
+  /* add the entry to the end of the list */
+  ide = &LT->IDE[LT->num];
+  ide->DocNum = DN;
+  ide->Weight = Sum;
+  LT->num++;
+
+  return LT;
+}
+
+
+void 
+LT_free (query_data * qd, List_Table * LT)
+{
+  unsigned es;
+  es = sizeof (List_Table) + sizeof (Invf_Doc_Entry) * LT->max;
+  Xfree (LT);
+  ChangeMemInUse (qd, -es);
+}
+
+
+static int 
+IDE_comp (const void *a, const void *b)
+{
+  const Invf_Doc_Entry *A = (Invf_Doc_Entry *) a;
+  const Invf_Doc_Entry *B = (Invf_Doc_Entry *) b;
+  return A->DocNum - B->DocNum;
+}
+
+static void 
+LT_sort (List_Table * LT)
+{
+  qsort (LT->IDE, LT->num, sizeof (Invf_Doc_Entry), IDE_comp);
+}
+
+
+/* [RJM 07/97: Ranked Required Terms] */
+/* accumulators with a sum less than or equal to 0.0 are to be deleted */
+static void 
+LT_pack (List_Table * LT)
+{
+  int s, d = 0;
+  for (s = 0; s < LT->num; s++) {
+      if (LT->IDE[s].Weight <= 0.0) {
+	/* ignore this accumulator */
+
+      } else if (d > 0 && LT->IDE[s].DocNum == LT->IDE[d-1].DocNum) {
+	LT->IDE[d-1].Weight += LT->IDE[s].Weight;
+
+      } else {
+	if (d != s) LT->IDE[d] = LT->IDE[s];
+	d++;
+      }
+  }
+
+  if (LT->num) LT->num = d;
+}
+
+
+/**************************************************************************/
+
+
+
+invf_data *
+InitInvfFile (File * InvfFile, stemmed_dict * sd)
+{
+  invf_data *id;
+  if (!(id = (invf_data *) Xmalloc (sizeof (invf_data))))
+    {
+      mg_errno = MG_NOMEM;
+      return NULL;
+    }
+
+  id->InvfFile = InvfFile;
+
+  Fread (&id->ifh, sizeof (id->ifh), 1, InvfFile);
+  /* [RPAP - Jan 97: Endian Ordering] */
+  {
+    int i;
+    
+    NTOHUL(id->ifh.no_of_words);
+    NTOHUL(id->ifh.no_of_ptrs);
+    NTOHUL(id->ifh.skip_mode);
+    for (i = 0; i < 16; i++)
+      NTOHUL(id->ifh.params[i]);
+    NTOHUL(id->ifh.InvfLevel);
+  }
+
+  id->N = sd->sdh.num_of_docs;
+  id->Nstatic = sd->sdh.static_num_of_docs;
+
+  return (id);
+}
+
+
+void 
+FreeInvfData (invf_data * id)
+{
+  Xfree (id);
+}
+
+/****************************************************************************/
+
+
+static void 
+CalcBlks (query_data * qd, WordEntry * we, int *blk,
+	  int *dn_blk, int *len_blk, int *k, int *p)
+{
+  *p = we->doc_count;
+
+  *blk = BIO_Bblock_Init (qd->id->Nstatic, *p);
+  switch (qd->id->ifh.skip_mode)
+    {
+    case 1:
+      {
+	unsigned long len;
+	if (*p <= qd->id->ifh.params[0])
+	  *k = 0;
+	else
+	  {
+	    *k = qd->id->ifh.params[0];
+	    len = BIO_Bblock_Bound (qd->id->N, *p);
+	    if (qd->id->ifh.InvfLevel >= 2)
+	      len += we->count;
+	    *dn_blk = BIO_Bblock_Init (qd->sd->sdh.num_of_docs, (*p + *k - 1) / *k);
+	    *len_blk = BIO_Bblock_Init (len, (*p + *k - 1) / *k);
+	  }
+	break;
+      }
+    case 2:
+      {
+	unsigned long len;
+	if (*p <= qd->id->ifh.params[1])
+	  *k = 0;
+	else
+	  {
+	    *k = (int) (2 * sqrt (((double) (*p)) / qd->id->ifh.params[0]));
+	    if (*k <= qd->id->ifh.params[1])
+	      *k = qd->id->ifh.params[1];
+	    len = BIO_Bblock_Bound (qd->id->N, *p);
+	    if (qd->id->ifh.InvfLevel >= 2)
+	      len += we->count;
+	    *dn_blk = BIO_Bblock_Init (qd->sd->sdh.num_of_docs,
+				       (*p + *k - 1) / *k);
+	    *len_blk = BIO_Bblock_Init (len, (*p + *k - 1) / *k);
+	  }
+	break;
+      }
+    default:
+      *k = 0;
+    }
+}
+
+
+struct cosineSearch
+{
+  int         term;
+  int         num_docs;
+  int         num_terms;
+  int         MoreAccum;
+  int         Anding;
+  FILE *      sk;
+  u_char      indent;
+  query_data *qd;
+  TermList *  Terms;
+  RankedQueryInfo *rqi;
+
+  // variables for inside term loop;
+  int Abort;
+  u_char *buffer;
+  unsigned long pos; 
+  int i;                        /* Pointer into inverted list */
+  unsigned long CurrDocNum;	/* NOTE: Document numbers start at 1 */
+  double Wqt, WordLog;
+  int dn_blk; 
+  int len_blk;
+  int skip_size, block_counter;
+  int blk;			/* bblock control parameter */
+  int doc_freq;			/* The number of documents the word occurs in */
+  WordEntry *we;
+  int next_i, next_doc_num, next_start;
+  int skips_taken, ptrs_dec;
+  double total_added;
+  int hits;
+
+  // variables inside term document loop
+  int Count;                      
+  double Wdt;
+  unsigned long diff;
+  abstractDocEntry *mem;
+
+  inline void noteHit(float *local)
+  {
+    *local = *local + (Wqt * Wdt);
+    total_added += Wqt * Wdt;
+    hits ++;
+  }
+
+  inline void postSkip(bitio_buffer *decodeBuffer)
+  {
+    if (skip_size != 0 && block_counter == 0 && i != doc_freq - 1)
+      {
+	CurrDocNum = next_doc_num;
+      }
+    else
+      {
+	diff = decodeBuffer->bblock_decode(blk, &pos);
+	CurrDocNum += diff;
+      }
+
+    if (qd->id->ifh.InvfLevel >= 2)
+      Count = decodeBuffer->gamma_decode(&pos);
+    else
+      Count = (int) ((double) we->count / doc_freq);
+    qd->num_of_ptrs++;
+    ptrs_dec++;
+    i++;
+    block_counter--;
+    if (block_counter < 0)
+      block_counter = skip_size - 1;
+  }
+
+  inline int skip(bitio_buffer *decodeBuffer, int nextDoc, int allowShortcut)
+  {
+    if (skip_size != 0)
+      {
+	if (Anding && i + skip_size < doc_freq && nextDoc > next_doc_num &&
+	    next_doc_num >= 0)
+	  {
+	  ShortCut:
+	    i = next_i;
+	    decodeBuffer->seek(next_start);
+	    pos = next_start;
+	    CurrDocNum = next_doc_num;
+	    qd->hops_taken++;
+	    block_counter = skip_size - 1;
+	    skips_taken ++;
+	  }
+	if (block_counter == skip_size - 1)
+	  {
+	    if (i + skip_size < doc_freq)
+	      {
+		int doc_diff, pos_diff;
+		doc_diff = decodeBuffer->bblock_decode(dn_blk, &pos);
+		next_doc_num += doc_diff;
+		next_i += skip_size;
+		pos_diff = decodeBuffer->bblock_decode(len_blk, &pos);
+		next_start = pos + pos_diff;
+	      }
+	    else
+	      {
+		next_doc_num = -1;
+	      }
+	  }
+
+	if (Anding && i + skip_size < doc_freq && nextDoc > next_doc_num &&
+	    next_doc_num >= 0)
+	  {
+	    if (allowShortcut)
+	      goto ShortCut;
+	    else
+	      return 1;
+	  }
+      }
+    return 0;
+  }
+
+  inline void prepareSkip()
+  {
+    i             = 0;
+    pos           = 0;
+    next_i        = 0;
+    next_start    = 0;
+    next_doc_num  = 0;
+    block_counter = skip_size - 1;
+  }
+
+  inline int startTerm()
+  {
+    CalcBlks (qd, we, &blk, &dn_blk, &len_blk, &skip_size, &doc_freq);
+
+    /* -- allocate a buffer big enough to hold the inverted */
+    if (!(buffer = (u_char *) Xmalloc (we->invf_len)))
+      return 1;
+       
+    ChangeMemInUse (qd, we->invf_len);
+    if (sk && qd->id->ifh.skip_mode == 2)
+      fprintf (sk, "%4d", skip_size);
+
+    /* --- Read an inverted file entry for the given word --- */
+    Fseek (qd->File_invf, we->invf_ptr, 0);
+    Fread (buffer, sizeof (char), we->invf_len, qd->File_invf);
+
+    qd->num_of_terms ++;
+    return 0;
+  }
+
+  inline void initTerm(WordEntry *_we)
+  {
+    Abort       = 0;
+    CurrDocNum  = 0;  // Note: Document numbers start at 1
+    dn_blk      = 0;
+    len_blk     = 0;
+    skips_taken = 0;
+    ptrs_dec    = 0;
+    total_added = 0.0;
+    hits        = 0;
+    we          = _we;
+  }
+
+  inline void closeSearch()
+  { if (sk)
+      fclose (sk);
+  }
+
+  /**
+   * GRB: Just write a single entry detailing the current position to the
+   *      skip dump file
+   */
+  inline void writeSkipLine(WordEntry *we)
+  {
+    if (sk)
+      {
+	fprintf (sk, "%3d : %8ld %6ld \"%.*s\"%*s", term, we->count,
+		 we->doc_count, Terms->entry(term)->wordLength(), Terms->entry(term)->word(),
+		   indent - Terms->entry(term)->wordLength(), "");
+      }
+  }
+
+  inline void initSearch()
+  {
+    num_docs = qd->sd->sdh.num_of_docs;
+    if (rqi->MaxTerms == -1)
+      num_terms = Terms->size();
+    else
+      num_terms = rqi->MaxTerms < Terms->size() ? rqi->MaxTerms : Terms->size();
+    MoreAccum = 1; // NB: this line isn't used in CosineDecode/GetDocsOp
+  }
+
+  inline void writeSkipHeader()
+  { 
+    if (!skip_header)
+      {
+	skip_header = 1;
+	fprintf (sk, "%s", skip_header_text[qd->id->ifh.skip_mode]);
+	fprintf (sk, "\nSkipping method %ld\n", qd->id->ifh.skip_mode);
+	switch (qd->id->ifh.skip_mode)
+	  {
+          case 1:
+            fprintf (sk, "Skipping is every %ld docnums\n",
+		     qd->id->ifh.params[0]);
+	    break;
+	  case 2:
+	    fprintf (sk, "Max nodes = %ld\nNo skips smaller or equal to %ld\n",
+	             qd->id->ifh.params[0], qd->id->ifh.params[1]);
+	    break;
+	  }
+      }
+    fprintf (sk, "\n\t\t\t-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-\n\n");
+    for (term = 0; term < Terms->size(); term++)
+      if (Terms->entry(term)->wordLength() > indent)
+        indent = Terms->entry(term)->wordLength();
+  }
+
+  void init(query_data *query, TermList *terms, RankedQueryInfo *qi)
+  {
+    /* -- note generic call parameters for later use
+    */
+    Terms = terms;
+    qd    = query;
+    rqi   = qi;
+
+    /* -- initialise some cosineSearch variables */
+    indent = 0;
+    Anding = 0;
+
+    /* -- If we've been given a skip dump file to write to, then open
+          it
+    */
+    if (rqi != NULL &&
+        rqi->skip_dump != NULL && 
+	rqi->skip_dump[0] != '\0')
+      {
+	char buf[256];
+	sprintf (buf, rqi->skip_dump, getpid());
+	sk = fopen (buf, "a");
+      }
+    else
+      {
+	sk = NULL;
+      }
+
+    /* -- if we've got an open skip dump file, then write headers
+          to it now
+    */
+    if (sk != NULL)
+      writeSkipHeader();      
+  }
+};
+
+
+/* =========================================================================
+ * Function: GetDocsOp
+ * Description: 
+ *      if op == op_term then 
+ *        look up term in invf and decode all its entries/documents
+ *      if op == op_and_terms then 
+ *        go thru L, testing if it exists in invf using skips  
+ *        if it does exist then overwrite the entry nearer the start of the list
+ *      if op == op_and_or_terms then
+ *        go thru L, testing if it exists in invf using skips  
+ *        BUT only mark ones which are found - keep list intact
+ *      if op == op_diff_terms then     >>>> NOT tested <<<<
+ * Input: 
+ * Output: 
+ * ========================================================================= 
+ *
+ * GRB: This operates on one term only (pointed to by we); hence Term 
+ *      initialisation operations only are performed
+ */
+
+DocList *
+GetDocsOp (query_data * qd, WordEntry * we, Op_types op, DocList * L)
+{
+  cosineSearch cs;
+
+  /* GRB: cheats to bluff in cosineSearch; it isn't initialised 
+          properly - so don't use functions too glibly!
+   */
+
+  DocEntry *src = NULL, *dest = NULL;
+  int have_skipping;
+  float Weight;
+
+  cs.init(qd, NULL, NULL);    // GRB: in effect this just sets the skip dump file to null;
+  cs.initTerm(we);
+
+  /* ... AND empty */
+  if (op == op_and_terms && DocList_isEmpty(L))
+    {
+      if (L)
+	Xfree (L);
+      return DocList_create (0);
+    }
+
+  if (op == op_term)
+    L = DocList_create (cs.we->doc_count);
+
+  if (cs.startTerm() != 0)
+    return DocList_create (0);
+
+  /* Read from the buffer */
+  DECODE_START (cs.buffer, cs.we->invf_len);
+
+  Weight = qd->sd->sdh.num_of_words / (100 * log2 ((float) cs.we->count + 1));
+
+  have_skipping = (cs.skip_size != 0);
+
+  if (op != op_term)
+    dest = src = L->member(0);
+
+  cs.prepareSkip();
+
+  /* Go thru the elements of an inverted list for a term */
+  /* May not go sequentially if skipping is done */
+  while (cs.i < cs.doc_freq && ((op == op_term) || DocList_memberValid(L, src)))
+    {
+      cs.Anding = (op != op_term); // GRB: I doubt that the op value varies!!!
+
+      (void) cs.skip(&buffer, (src != NULL ? src->docNum() : -1), 1);
+
+      cs.postSkip(&buffer);
+
+      switch (op)
+	{
+	case op_term:
+	  /* --- Add the doc from invf onto the c-set list --- */
+	  L->addMember(cs.CurrDocNum, Weight);
+	break;
+
+	case op_and_terms:
+	case op_and_or_terms:
+	  /* --- see if have a match and save it --- */
+
+	  while (DocList_memberValid(L, src) && cs.CurrDocNum > src->docNum())
+	    { 
+	      src = DocList_nextMember(L, src);
+	    }
+	  /* Unless at end of list, assert(cs.CurrDocNum <= src->docNum()) */
+
+	  /* --- Accept the doc as a candidate --- */
+	  if (DocList_memberValid(L, src) && cs.CurrDocNum == src->docNum())
+	    {
+	      if (op == op_and_terms)
+		{
+		  dest->copy(src); // *dest = *src;
+		  dest = DocList_nextMember(L, dest);
+		}
+	      else
+		{
+		  DocEntry_setOrIncluded(src, 1);
+		}
+	      src = DocList_nextMember(L, src);	 /* Move onto next one */
+	    }
+
+	  /* If the next one to try has already been tried then move on again */
+	  if (op == op_and_or_terms)
+	    {
+	      while (DocList_memberValid(L, src) && DocEntry_getOrIncluded(src))
+		src = DocList_nextMember(L, src);
+	    }
+
+	  break;
+
+	case op_diff_terms:
+	  while (DocList_memberValid(L, src) && src->docNum() < cs.CurrDocNum)
+	    {
+	      dest->copy(src); // *dest = *src;
+	      dest = DocList_nextMember(L, dest);
+	      src  = DocList_nextMember(L, src);
+	    }
+	  if (DocList_memberValid(L, src) && src->docNum() == cs.CurrDocNum)
+	    src = DocList_nextMember(L, src);
+	  break;
+
+	default:
+	  break;
+	}			/*switch */
+
+    }				/*while traversing invf */
+
+  if (op == op_and_terms || op == op_diff_terms)
+    {
+      if (op == op_diff_terms)
+	{
+	  while (DocList_memberValid(L, src))
+	    {	    
+	      dest->copy(src); // *dest = *src;
+	      dest = DocList_nextMember(L, dest);
+	      src  = DocList_nextMember(L, src);
+	    }
+	  }
+
+      /* update size of c-set */
+      DocList_setsize(L, dest - L->member(0));
+    }
+  DECODE_DONE
+
+    Xfree (cs.buffer);
+  ChangeMemInUse (qd, -cs.we->invf_len);
+
+  return (L);
+}
+
+class AccumMethod
+{
+public:
+  int doShortCuts;
+  virtual int  initMethod(cosineSearch *cs, void *privatedata) = 0;
+  virtual void initTerm() = 0;
+  virtual void startTerm() = 0;
+  virtual int  executeTerm() = 0;
+  virtual void endTerm() = 0;
+  virtual void endMethod() = 0;
+  virtual int  carryOn() = 0;
+  virtual int  endTermValue() = 0;
+  virtual int  nextDoc() = 0;
+  virtual void *endValue() = 0;
+
+  void *cosineDecode(query_data *qd, TermList *Terms, RankedQueryInfo *rqi, void *privatedata);
+};
+
+void *AccumMethod::cosineDecode(query_data * qd, TermList * Terms,
+		                RankedQueryInfo * rqi, void *privatedata)
+{
+  cosineSearch cs;
+
+  cs.init(qd, Terms, rqi);
+  if (this->initMethod(&cs, privatedata) > 0)
+    return NULL;
+
+  cs.initSearch();
+
+  /* For each word in the list . . . */
+  for (cs.term = 0; cs.term < cs.num_terms; cs.term++)
+    {
+      cs.initTerm(&Terms->TE[cs.term].WE);
+      this->initTerm();
+      cs.writeSkipLine(cs.we);
+
+      COSINE_STARTTERM
+      this->startTerm();
+
+      cs.prepareSkip();
+      while (cs.i < cs.doc_freq && this->carryOn())
+	{
+	  cs.skip(&buffer, this->nextDoc(), this->doShortCuts);
+	  cs.postSkip(&buffer);
+
+	  cs.Wdt = cs.Count * cs.WordLog;
+
+	  switch (this->executeTerm())
+	    {
+            case 1:
+              goto FastAbort;
+	      break;
+
+	    case 2:
+	      return NULL;
+	      break;
+	    }
+	}
+
+    FastAbort:
+      DECODE_DONE
+
+      Xfree (cs.buffer);
+      ChangeMemInUse (qd, -cs.we->invf_len);
+      if (cs.sk)
+	{
+	  if (qd->id->ifh.skip_mode != 0)
+	    fprintf (cs.sk, " %6d", cs.skips_taken);
+	  fprintf (cs.sk, " %6d %6d %6d%12.2f\n",
+		   cs.ptrs_dec, cs.hits, this->endTermValue(), cs.total_added);
+	}
+      if (cs.Abort)
+	break;
+      this->endTerm();
+    }
+
+  cs.closeSearch();
+  this->endMethod();
+
+  return this->endValue();
+}
+
+float *
+CosineDecode (query_data * qd, TermList * Terms, RankedQueryInfo * rqi)
+{
+  cosineSearch cs;
+
+  float *AccumulatedWeights;
+  int seen_must_match = 0; /* [RJM 07/97: Ranked Required Terms] says whether to 'and' queries or not */
+
+  if (rqi->skip_dump != NULL)
+    rqi->skip_dump[0] = '\0';
+  cs.init(qd, Terms, rqi);
+
+  cs.initSearch();
+
+  /* Create the array for the documents */
+  if (!(AccumulatedWeights = (float *) Xmalloc (cs.num_docs * sizeof (float))))
+      return (NULL);
+  bzero ((char *) AccumulatedWeights, sizeof (float) * cs.num_docs);
+  ChangeMemInUse (qd, sizeof (float) * cs.num_docs);
+
+  /* For each word in the list . . . */
+  for (cs.term = 0; cs.term < cs.num_terms; cs.term++)
+    {
+      unsigned long LastDocNum = 0; /* [RJM 07/97: Ranked Required Terms] */
+      unsigned long TempDocNum = 0; /* [RJM 07/97: Ranked Required Terms] */
+      unsigned long next_mjr_dn = 0;
+      int require_match = Terms->TE[cs.term].require_match; /* [RJM 07/97: Ranked Required Terms] */
+
+      cs.initTerm(&Terms->TE[cs.term].WE);
+
+      COSINE_STARTTERM
+
+      cs.block_counter = 0;
+      for (cs.i = 0; cs.i < cs.doc_freq; cs.i++, cs.block_counter++)
+	{
+	  if (cs.block_counter == cs.skip_size)
+	    {
+	      cs.block_counter = 0;
+	    }
+	  if (cs.skip_size && cs.block_counter == 0)
+	    if (cs.i + cs.skip_size < cs.doc_freq)
+	      {
+		int temp;
+		next_mjr_dn = buffer.bblock_decode(cs.dn_blk, NULL);
+		next_mjr_dn += cs.CurrDocNum;
+		temp = buffer.bblock_decode(cs.len_blk, NULL);
+	      }
+	  if (cs.skip_size && cs.block_counter == cs.skip_size - 1 && cs.i != cs.doc_freq - 1)
+	    cs.CurrDocNum = next_mjr_dn;
+	  else
+	    {
+	      cs.diff = buffer.bblock_decode(cs.blk, NULL);
+	      cs.CurrDocNum += cs.diff;
+	    }
+
+	  if (qd->id->ifh.InvfLevel >= 2)
+	    cs.Count = buffer.gamma_decode (NULL);
+	  else
+	    cs.Count = (int) ((double) cs.we->count / cs.doc_freq);
+	  cs.Wdt = cs.Count * cs.WordLog;
+
+	  /* [RJM 07/97: Ranked Required Terms] */
+	  /* zero the weights of intermediate documents if this term is required to match */
+	  if (require_match == 1) {
+	    for (TempDocNum = LastDocNum+1; TempDocNum < cs.CurrDocNum; TempDocNum++)
+	      AccumulatedWeights[TempDocNum - 1] = 0.0;
+	  }
+
+	  /* [RJM 07/97: Ranked Required Terms] */
+	  /* if a term has been required to match before, the weight we are */
+	  /* adding to must be non-zero */
+	  if (!seen_must_match || (AccumulatedWeights[cs.CurrDocNum - 1] > 0.0)) {
+	      AccumulatedWeights[cs.CurrDocNum - 1] += cs.Wqt * cs.Wdt;
+	  }
+
+	  qd->num_of_ptrs++;
+	  LastDocNum = cs.CurrDocNum; /* [RJM 07/97: Ranked Required Terms] */
+	}
+
+      DECODE_DONE
+
+	Xfree (cs.buffer);
+      ChangeMemInUse (qd, -cs.we->invf_len);
+
+      /* [RJM 07/97: Ranked Required Terms] */
+      /* zero the remaining weights if this term is required to match */
+      if (require_match == 1) {
+	for (TempDocNum = LastDocNum+1; TempDocNum <= cs.num_docs; TempDocNum++)
+	  AccumulatedWeights[TempDocNum - 1] = 0.0;
+	seen_must_match = 1;
+      }
+    }
+  return AccumulatedWeights;
+}
+
+
+
+static int 
+doc_comp (void *a, void *b)
+{
+  return ((Invf_Doc_Entry *) a)->DocNum - ((Invf_Doc_Entry *) b)->DocNum;
+}
+
+inline Invf_Doc_Entry *doc_fetch(int docNum, Splay_Tree *sptree)
+{ Invf_Doc_Entry  entry;
+  Invf_Doc_Entry *reply;
+
+  entry.DocNum = docNum;
+  reply = (Invf_Doc_Entry *) SP_member (&entry, sptree);
+  return reply;
+}
+
+static Invf_Doc_Entry *
+get_ide (query_data * qd, Invf_Doc_Entry_Pool * pool)
+{
+  struct pool *p = (struct pool *) pool->pool;
+  if (!p || p->pos == p->length)
+    {
+      register unsigned size = sizeof (struct pool) +
+      (pool->pool_chunks - 1) * sizeof (Invf_Doc_Entry);
+      p = (struct pool *) malloc (size);
+      if (!p)
+	return (Invf_Doc_Entry *) NULL;
+      ChangeMemInUse (qd, size);
+      p->length = pool->pool_chunks;
+      p->pos = 0;
+      p->next = (struct pool *) pool->pool;
+      pool->pool = p;
+    }
+  return &(p->ide[p->pos++]);
+}
+
+
+void 
+free_ide_pool (query_data * qd, Invf_Doc_Entry_Pool * pool)
+{
+  while (pool->pool)
+    {
+      struct pool *p = (struct pool *) pool->pool;
+      pool->pool = p->next;
+      ChangeMemInUse (qd, -(sizeof (struct pool) + (p->length - 1) *
+			    sizeof (Invf_Doc_Entry)));
+      free (p);
+    }
+  pool->pool = NULL;
+}
+
+FILE *
+open_skip (char *sd)
+{
+  char buf[256];
+  if (sd == NULL || *sd == '\0')
+    return NULL;
+  sprintf (buf, sd, getpid ());
+  return fopen (buf, "a");
+}
+
+class SplayMethod : public AccumMethod
+{ 
+  // Search values
+  cosineSearch *cs;
+  Splay_Tree *Doc_Set;
+  Invf_Doc_Entry_Pool *pool;
+
+  // Term values
+  Invf_Doc_Entry *next;
+
+  public:
+  SplayMethod();
+  int  initMethod(cosineSearch *cs, void *privatedata);
+  void initTerm();
+  void startTerm();
+  int  executeTerm();
+  void endTerm();
+  void endMethod();
+  int  carryOn();
+  int  endTermValue();
+  int  nextDoc();
+  void *endValue();
+};
+
+SplayMethod::SplayMethod()
+{
+  doShortCuts = 0;
+}
+
+int SplayMethod::initMethod(cosineSearch *_cs, void *privatedata)
+{
+  this->cs          = _cs;
+  this->Doc_Set     = SP_createset(doc_comp);
+  pool              = (Invf_Doc_Entry_Pool *) privatedata;
+  pool->pool        = NULL;
+  pool->pool_chunks = 1024;
+  return 0;
+}
+
+void SplayMethod::initTerm()
+{
+  next = NULL;
+}
+
+void SplayMethod::startTerm()
+{
+  if (this->cs->rqi->MaxAccums != -1 && 
+      this->Doc_Set->no_of_items >= this->cs->rqi->MaxAccums)
+  {
+    cs->MoreAccum = 0;
+    cs->Anding = 1;
+    this->next = (Invf_Doc_Entry *) SP_get_first (this->Doc_Set);
+  }
+}
+
+int SplayMethod::executeTerm()
+{
+  if (cs->Anding)
+    {	
+      while (next && next->DocNum < cs->CurrDocNum - 1)	
+	next = (Invf_Doc_Entry *) SP_get_next (Doc_Set);
+      if (next && next->DocNum == cs->CurrDocNum - 1)
+	{
+	  cs->noteHit(&next->Weight);
+	}
+      if (next && next->DocNum <= cs->CurrDocNum - 1)
+	next = (Invf_Doc_Entry *) SP_get_next (Doc_Set);
+      goto NextDocNum;
+    }
+  if ((cs->mem = doc_fetch(cs->CurrDocNum - 1, Doc_Set)) == NULL)
+    {
+      cs->hits++;
+      if (cs->rqi->MaxAccums != -1 &&
+	  Doc_Set->no_of_items >=cs-> rqi->MaxAccums &&
+	  !cs->MoreAccum)
+	goto NextDocNum;
+      if ((cs->mem = get_ide (cs->qd, pool)) == NULL)
+	return 2;
+      ChangeMemInUse (cs->qd, sizeof (*cs->mem));
+      cs->mem->DocNum = cs->CurrDocNum - 1;
+      cs->mem->Weight = 0;
+
+      ChangeMemInUse (cs->qd, -Doc_Set->mem_in_use);
+      if (SP_insert (cs->mem, Doc_Set) == NULL)
+	{
+	  Message ("Unable to allocate memory for a splay node");
+	  cs->Abort = 1;
+	  return 1; // abort return
+	}
+      ChangeMemInUse (cs->qd, Doc_Set->mem_in_use);
+      if (cs->rqi->MaxAccums != -1 &&
+	  Doc_Set->no_of_items >= cs->rqi->MaxAccums &&
+	  cs->rqi->StopAtMaxAccum)
+	cs->Abort = 1;
+    }
+
+  cs->mem->Weight += cs->Wqt * cs->Wdt;
+  cs->total_added += cs->Wqt * cs->Wdt;
+
+ NextDocNum:
+  return 0;    
+}
+
+int SplayMethod::carryOn()
+{ return (!cs->Anding || next);
+}
+
+void SplayMethod::endTerm()
+{
+  // do nothing
+}
+
+int SplayMethod::endTermValue()
+{
+  return Doc_Set->no_of_items;
+}
+
+void SplayMethod::endMethod()
+{
+  // do nothing
+}
+
+int SplayMethod::nextDoc()
+{
+  return (next != NULL ? next->DocNum : -1);
+}
+
+void * SplayMethod::endValue()
+{ return (void *) Doc_Set;
+}
+
+Splay_Tree *
+CosineDecodeSplay (query_data * qd, TermList * Terms,
+		   RankedQueryInfo * rqi, Invf_Doc_Entry_Pool * pool)
+{
+  SplayMethod  sm;
+
+  return (Splay_Tree *) sm.cosineDecode(qd, Terms, rqi, pool);
+}
+
+
+class HashMethod : public AccumMethod
+{ 
+  // Search values
+  cosineSearch *cs;
+  Hash_Table *HT;
+  int Sorted;
+
+  // Term values
+  Invf_Doc_EntryH *next;
+
+  public:
+  HashMethod();
+  int  initMethod(cosineSearch *cs, void *privatedata);
+  void initTerm();
+  void startTerm();
+  int  executeTerm();
+  void endTerm();
+  void endMethod();
+  int  carryOn();
+  int  endTermValue();
+  int  nextDoc();
+  void *endValue();
+};
+
+HashMethod::HashMethod()
+{
+  doShortCuts = SHORT_CUT;
+}
+
+int HashMethod::initMethod(cosineSearch *_cs, void *privatedata)
+{
+  this->cs          = _cs;
+  /* Create the array for the documents */
+  Sorted = 0;
+  HT = HT_create (cs->qd, cs->rqi->MaxAccums, cs->rqi->HashTblSize);
+  if (!HT)
+    return 1;
+  return 0;  
+}
+
+void HashMethod::initTerm()
+{
+  next = NULL;
+}
+
+void HashMethod::startTerm()
+{
+  if (cs->rqi->MaxAccums != -1 && HT->num >= cs->rqi->MaxAccums)
+    {
+      cs->MoreAccum = 0;
+      cs->Anding = 1;
+      if (!Sorted)
+	{
+	  HT_sort (cs->qd, HT);
+	  Sorted = 1;
+	}
+      next = HT->Head;
+    }
+}
+
+int HashMethod::executeTerm()
+{
+  if (cs->Anding)
+    {
+      while (next && next->DocNum < cs->CurrDocNum - 1)
+	next = next->next;
+      if (next && next->DocNum == cs->CurrDocNum - 1)
+	{
+	  cs->noteHit(&next->Weight);
+	}
+      if (next && next->DocNum <= cs->CurrDocNum - 1)
+	next = next->next;
+      goto NextDocNum;
+    }
+  if ((cs->mem = HT_find (HT, cs->CurrDocNum - 1)) == NULL)
+    {
+      if (cs->rqi->MaxAccums != -1 && HT->num >= cs->rqi->MaxAccums &&
+	  !cs->MoreAccum)
+	goto NextDocNum;
+      if (!HT_insert (HT, cs->CurrDocNum - 1, cs->Wqt * cs->Wdt))
+	{
+	  if (!HT->Suplimentary_Entries)
+	    HT_Sup_create (cs->qd, HT, cs->we->doc_count - cs->i + 1);
+	  HT_Sup_Add (HT, cs->CurrDocNum - 1, cs->Wqt * cs->Wdt);
+	}
+      if (cs->rqi->MaxAccums != -1 && HT->num >= cs->rqi->MaxAccums &&
+	  cs->rqi->StopAtMaxAccum)
+	cs->Abort = 1;
+    }
+  else
+    {
+      cs->noteHit(&cs->mem->Weight);
+    }
+  
+ NextDocNum:
+  return 0;
+}
+
+int HashMethod::carryOn()
+{ return (!cs->Anding || next);
+}
+
+void HashMethod::endTerm()
+{
+  // do nothing
+}
+
+int HashMethod::endTermValue()
+{
+  return HT->num + HT->Suplimentary_Num;
+}
+
+void HashMethod::endMethod()
+{
+  if (!HT->Head)
+    HT_sort (cs->qd, HT);
+}
+
+int HashMethod::nextDoc()
+{
+  return (next != NULL ? next->DocNum : -1);
+}
+
+void *HashMethod::endValue()
+{
+  return (void *) HT;
+}
+
+
+Hash_Table *
+CosineDecodeHash (query_data * qd, TermList * Terms,
+		  RankedQueryInfo * rqi)
+{
+  HashMethod   hm;
+  
+  return (Hash_Table *) hm.cosineDecode(qd, Terms, rqi, NULL);
+}
+
+
+class ListMethod : public AccumMethod
+{ 
+  // Search values
+  cosineSearch *cs;
+  List_Table *LT;
+  int Sorted;
+  int has_grown; /* [RJM 07/97: Ranked Required Match] whether elements have been added lately */
+
+  // Term values
+  int this_item; /* [RJM 07/97: Ranked Required Match] points to the next possible LT to delete */
+  unsigned long LastDocNum;	/* [RJM 07/97: Ranked Required Match] */
+  int require_match; /* [RJM 07/97: Ranked Required Match] */
+  Invf_Doc_Entry *next;
+  int n;
+
+  public:
+  ListMethod();
+  int  initMethod(cosineSearch *cs, void *privatedata);
+  void initTerm();
+  void startTerm();
+  int  executeTerm();
+  void endTerm();
+  void endMethod();
+  int  carryOn();
+  int  endTermValue();
+  int  nextDoc();
+  void *endValue();
+};
+
+ListMethod::ListMethod()
+{
+  // do nothing
+  doShortCuts = 0;
+}
+
+int ListMethod::initMethod(cosineSearch *_cs, void *privatedata)
+{
+  this->cs  = _cs;
+  Sorted    = 0;
+  has_grown = 0;
+
+  /* Create the list for the documents */
+  LT = LT_create (cs->qd, cs->rqi->MaxAccums);
+  if (!LT)
+    return 1;
+
+  return 0;  
+}
+
+void ListMethod::initTerm()
+{
+  next          = NULL;
+  n             = 0;
+  LastDocNum    = 0;
+  this_item     = 0;
+  require_match = cs->Terms->TE[cs->term].require_match;
+}
+
+void ListMethod::startTerm()
+{
+  if (cs->rqi->MaxAccums != -1 && LT->num >= cs->rqi->MaxAccums)
+    cs->MoreAccum = 0;
+
+  if ((cs->rqi->MaxAccums != -1 && LT->num >= cs->rqi->MaxAccums) || 
+      cs->Anding)
+    {
+      cs->Anding = 1;
+      if (!Sorted || has_grown)
+	{
+	  LT_sort (LT);
+	  LT_pack (LT);
+	  Sorted = 1;
+	  has_grown = 0;
+	}
+      next = LT->IDE;
+      n = 0;
+    }
+
+  /* [RJM 07/97: Ranked Required Terms] */
+  /* make sure that the document list is set up correctly for */
+  /* requiring a term to match */
+  if (require_match == 1) {
+    if (!Sorted || has_grown) {
+      LT_sort (LT);
+      LT_pack (LT);
+      Sorted = 1;
+      has_grown = 0;
+    }
+    /* this_item is already set to zero */
+  }
+}
+
+int ListMethod::executeTerm()
+{
+  /* [RJM 07/97: Ranked Required Terms] */
+  /* if this is a required match we must remove all accumulators */
+  /* of intermediate documents */
+  if (require_match == 1) {
+    while (this_item < LT->num && LT->IDE[this_item].DocNum < cs->CurrDocNum - 1) {
+      if (LT->IDE[this_item].DocNum > LastDocNum - 1)
+	LT->IDE[this_item].Weight = -1000.0; /* mark for deletion */
+      this_item++;
+    }
+  }
+  
+  if (cs->Anding)
+    {
+      while (n < LT->num && next->DocNum < cs->CurrDocNum - 1)
+	{
+	  next++;
+	  n++;
+	}
+      if (n < LT->num && next->DocNum == cs->CurrDocNum - 1)
+	{
+	  cs->noteHit(&next->Weight);
+	}
+      if (n < LT->num && next->DocNum <= cs->CurrDocNum - 1)
+	{
+	  next++;
+	  n++;
+	}
+      goto NextDocNum;
+    }
+  if (!cs->Anding)
+    {
+      if (cs->rqi->MaxAccums != -1 && LT->num >= cs->rqi->MaxAccums &&
+	  !cs->MoreAccum)
+	goto NextDocNum;
+      LT = LT_add (cs->qd, LT, cs->CurrDocNum - 1, cs->Wqt * cs->Wdt);
+      if (!LT)
+	{
+	  cs->Abort = 1;
+	  return 1;
+	}
+
+      /* the list of accumulators just got bigger */
+      has_grown = 1;
+
+      if (cs->rqi->MaxAccums != -1 && LT->num >= cs->rqi->MaxAccums &&
+	  cs->rqi->StopAtMaxAccum)
+	cs->Abort = 1;
+    }
+  else if ((cs->mem = LT_find (LT, cs->CurrDocNum - 1)))
+    {
+      cs->noteHit(&cs->mem->Weight);
+    }
+
+ NextDocNum:
+  LastDocNum = cs->CurrDocNum;
+  return 0;
+}
+
+int ListMethod::carryOn()
+{ return (!cs->Anding || n < LT->num);
+}
+
+void ListMethod::endTerm()
+{
+  /* [RJM 07/97: Ranked Required Terms] */
+  /* remove the remaining accumulators if this term is required to match */
+  if (require_match == 1) {
+    while (this_item < LT->num) {
+      if (LT->IDE[this_item].DocNum > LastDocNum - 1)
+	LT->IDE[this_item].Weight = -1000.0; /* mark for deletion */
+      this_item++;
+    }
+    cs->Anding = 1; /* no more documents should be added */
+
+    /* Get rid of any documents which have been deleted */
+    LT_pack (LT);
+  }
+}
+
+int ListMethod::endTermValue()
+{
+  return LT->num;
+}
+
+void ListMethod::endMethod()
+{
+  // do nothing
+}
+
+int ListMethod::nextDoc()
+{
+  return (next != NULL ? next->DocNum : -1);
+}
+
+void *ListMethod::endValue()
+{
+  return (void *) LT;
+}
+
+
+List_Table *
+CosineDecodeList (query_data * qd, TermList * Terms,
+		  RankedQueryInfo * rqi)
+{
+  ListMethod   lm;
+
+  return (List_Table *) lm.cosineDecode(qd, Terms, rqi, NULL);
+}
Index: /tags/gsdl-21/gsdl/src/mgpp/text/invf_get.h
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/invf_get.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/invf_get.h	(revision 699)
@@ -0,0 +1,99 @@
+/**************************************************************************
+ *
+ * invf_get.h -- Inverted file reading routines
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef H_INVF_GET
+#define H_INVF_GET
+
+class Invf_Doc_Entry : public abstractDocEntry
+  {
+  };
+
+typedef struct Invf_Doc_Entry_Pool
+  {
+    void *pool;
+    unsigned pool_chunks;
+  }
+Invf_Doc_Entry_Pool;
+
+class Invf_Doc_EntryH : public abstractDocEntry
+  {
+  public:
+    struct Invf_Doc_EntryH *next;
+  };
+
+typedef struct Hash_Table
+  {
+    Invf_Doc_EntryH *Suplimentary_Entries;
+    int Suplimentary_Size;
+    int Suplimentary_Num;
+    unsigned max;
+    unsigned num;
+    unsigned size;
+    Invf_Doc_EntryH **HashTable;
+    Invf_Doc_EntryH *Head;
+    Invf_Doc_EntryH IDEH[1];
+  }
+Hash_Table;
+
+typedef struct List_Table
+  {
+    unsigned max;
+    unsigned num;
+    Invf_Doc_Entry IDE[1];
+  }
+List_Table;
+
+typedef enum
+  {
+    op_term, op_and_terms, op_and_or_terms, op_diff_terms
+  }
+Op_types;
+
+/*Invf_Doc_Entry *HT_find(Hash_Table *HT, unsigned long DN); */
+void HT_free (query_data * qd, Hash_Table * HT);
+void LT_free (query_data * qd, List_Table * LT);
+
+invf_data *InitInvfFile (File * InvfFile, stemmed_dict * sd);
+void FreeInvfData (invf_data * id);
+
+DocList *GetDocs (query_data * qd, WordEntry * we, BooleanQueryInfo * bqi);
+
+float *CosineDecode (query_data * qd, TermList * Terms, RankedQueryInfo * rqi);
+
+void free_ide_pool (query_data * qd, Invf_Doc_Entry_Pool * pool);
+
+Splay_Tree *CosineDecodeSplay (query_data * qd, TermList * Terms,
+			 RankedQueryInfo * rqi, Invf_Doc_Entry_Pool * pool);
+
+Hash_Table *CosineDecodeHash (query_data * qd, TermList * Terms,
+			      RankedQueryInfo * rqi);
+List_Table *CosineDecodeList (query_data * qd, TermList * Terms,
+			      RankedQueryInfo * rqi);
+
+
+DocList *GetDocsOp (query_data * qd, WordEntry * we, Op_types op, DocList * L);
+
+
+
+#endif
Index: /tags/gsdl-21/gsdl/src/mgpp/text/lists.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/lists.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/lists.cpp	(revision 699)
@@ -0,0 +1,315 @@
+/**************************************************************************
+ *
+ * lists.c -- List processing functions for boolean queries
+ * Copyright (C) 1994  Neil Sharman, Alistair Moffat and Lachlan Andrew
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+
+#include "backend.h"
+#include "lists.h"
+#include "local_strings.h"
+#include "locallib.h"
+#include "DocEntry.h"
+
+/*
+   $Log$
+   Revision 1.1  1999/10/11 02:57:42  cs025
+   Base install of MG-PP
+
+   Revision 1.1  1999/08/10 21:18:01  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1999/07/01 09:30:48  rjmcnab
+   Changes for better match reporting.
+
+   Revision 1.1  1998/11/17 09:34:47  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:50  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:37  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+
+
+
+/*
+ * Make a DocList large enough to hold num entries 
+ */
+DocList *
+DocList_create (int num)
+{ int      list_size;
+  DocList *d;
+
+  list_size = (num > 0 ? num : 1);
+  if (!(d = (DocList *) Xmalloc (sizeof (DocList))))
+    return NULL;
+  if (!(d->DE = (DocEntry *) Xmalloc (sizeof (DocEntry) * list_size)))
+    return (NULL);
+  d->num = 0;
+  d->total_retrieved = 0;
+  d->is_approx = 0;
+  d->list_size = list_size;
+  return (d);
+}
+
+/*
+ * Change the size of a DocList so that it can hold num entries 
+ */
+DocList *
+DocList_resize (DocList * d, int num)
+{
+  if (num > d->list_size)
+    { 
+      if (d->list_size < 100)
+	{
+	  d->list_size *= 2;
+	}
+      else
+	{
+	  d->list_size += 50;
+	}
+      
+      if (!(d->DE = (DocEntry *) Xrealloc (d->DE, sizeof (DocEntry) * (d->list_size))))
+        return (NULL);
+    }
+  return (d);
+}
+
+void DocList_destroy(DocList *d)
+{ 
+  if (d != NULL)
+    {
+      if (d->DE != NULL)
+	{
+	  Xfree(d->DE);
+	  Xfree(d);
+	}
+    }
+}
+
+
+/* IntersectLists () */
+/* Returns the intersection of List1 and List2 by deleting from List1 all */
+/* entries not in List2.  All of List2 is freed, and the nodes of List1 not */
+/* included in the resultant list are freed. */
+/* On exit the intersection of the lists is returned, or NULL on empty */
+/* intersection. */
+/* Assumes that both lists are ordered on increasing Value. */
+DocList *
+DocList_intersect (DocList * List1, DocList * List2)
+{
+  DocEntry *s1, *d1, *s2;
+  
+  if (!List2 || !List1)
+    {
+      if (List1)
+	DocList_destroy(List1);
+      if (List2)
+	DocList_destroy(List2);
+      return (DocList_create (0));
+    }
+  d1 = s1 = List1->DE;
+  s2 = List2->DE;
+  while (s1 - List1->DE < List1->num && s2 - List2->DE < List2->num)
+    {
+      if (s1->DocNum == s2->DocNum)
+	*d1++ = *s1++;
+      if (s1 - List1->DE < List1->num)
+	while (s2 - List2->DE < List2->num && s2->DocNum < s1->DocNum)
+	  s2++;
+      if (s2 - List2->DE < List2->num)
+	while (s1 - List1->DE < List1->num && s1->DocNum < s2->DocNum)
+	  s1++;
+    }
+  List1->num = d1 - List1->DE;
+  DocList_destroy(List2);
+  return (List1);
+}
+
+
+/* DiffLists () */
+/* Returns the difference of List1 and List2 by deleting from List1 all */
+/* entries in List2.  All of List2 is freed, and the nodes of List1 not */
+/* included in the resultant list are freed. */
+/* On exit the differrence of the lists is returned, or NULL on empty */
+/* intersection. */
+/* Assumes that both lists are ordered on increasing Value. */
+DocList *
+DocList_diff (DocList * List1, DocList * List2)
+{
+  DocEntry *s1, *d1, *s2;
+  if (!List1)
+    {
+      if (List2)
+	DocList_destroy(List2);
+      return (DocList_create (0));
+    }
+  if (!List2)
+    return (List1);
+  d1 = s1 = List1->DE;
+  s2 = List2->DE;
+  while (s1 - List1->DE < List1->num)
+    {
+      if (s2 - List2->DE == List2->num)
+	*d1++ = *s1++;
+      else
+	{
+	  if (s1->DocNum == s2->DocNum)
+	    {
+	      s1++;
+	      s2++;
+	    }
+	  else if (s1->DocNum < s2->DocNum)
+	    *d1++ = *s1++;
+	  else
+	    s2++;
+	}
+    }
+  List1->num = d1 - List1->DE;
+  DocList_destroy(List2);
+  return (List1);
+}
+
+/* MergeLists () */
+/* Finds the union of two lists and returns the head of the new list formed. */
+/* Assumes that both lists are ordered on increasing Value. */
+DocList *
+DocList_merge (DocList * List1, DocList * List2)
+{
+  DocList *NewList;
+  DocEntry *d, *s1, *s2;
+  if (!List1)
+    {
+      if (List2)
+	return (List2);
+      else
+	return (DocList_create (0));
+    }
+  if (!List2)
+    return (List1);
+  if (!(NewList = DocList_create (List1->num + List2->num)))
+    {
+      DocList_destroy(List1);
+      DocList_destroy(List2);
+      return (NULL);
+    }
+
+  s1 = List1->DE;
+  s2 = List2->DE;
+  d = NewList->DE;
+  while (s1 - List1->DE < List1->num || s2 - List2->DE < List2->num)
+    {
+      if (s1 - List1->DE >= List1->num)
+	*d++ = *s2++;
+      else if (s2 - List2->DE >= List2->num)
+	*d++ = *s1++;
+      else if (s1->DocNum == s2->DocNum)
+	{
+	  *d = *s1++;
+	  (d++)->Weight += (s2++)->Weight;
+	}
+      else if (s1->DocNum < s2->DocNum)
+	*d++ = *s1++;
+      else
+	*d++ = *s2++;
+    }
+  NewList->num = d - NewList->DE;
+  DocList_destroy(List1);
+  DocList_destroy(List2);
+  return (NewList);
+}
+
+/* =========================================================================
+ * Function: ReduceList
+ * Description: 
+ *      Go thru doc-set and get rid of all docs which are unmarked
+ *      The motivation for this routine is in the case of CNF queries.
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+void
+DocList_reduceOrIncluded (DocList * cset)
+{
+  DocEntry *src;
+  DocEntry *dest;
+
+  src = dest = cset->DE;
+  for (src = dest = cset->DE; src < cset->DE + cset->num; src++)
+    {
+      if (src->or_included)
+	{
+	  src->or_included = 0;
+	  *dest++ = *src;
+	}
+    }
+  cset->num = dest - cset->DE;
+
+}				/*ReduceList_OrIncluded */
+
+DocList *
+DocList_addDoc (DocList * Docs, int res)
+{
+  int j;
+  /* Search the list for the word */
+  for (j = 0; j < Docs->num; j++)
+    if (Docs->DE[j].DocNum == res)
+      break;
+
+  /* Increment the weight if the word is in the list */
+  if (j < Docs->num)
+    Docs->DE[j].Weight++;
+  else
+    {
+      /* Create a new entry in the list for the new document */
+      Docs = DocList_resize (Docs, Docs->num + 1);
+      Docs->DE[Docs->num].DocNum = res;
+      Docs->DE[Docs->num].Weight = 1;
+      Docs->DE[Docs->num].SeekPos = 0;
+      Docs->DE[Docs->num].Len = 0;
+      Docs->DE[Docs->num].CompTextBuffer = NULL;
+      Docs->DE[Docs->num].Next = NULL;
+      Docs->num++;
+    }
+  return (Docs);
+}
+
+void DocList_FreeTextBuffers(DocList *DocList, query_data *qd)
+{ int  i;
+  long l;
+
+  for (i = 0; i < DocList->size(); i ++)
+   {
+     l = DocEntry_FreeTextBuffer(&DocList->DE[i]);
+     ChangeMemInUse (qd, -l);
+   }
+}
+
Index: /tags/gsdl-21/gsdl/src/mgpp/text/lists.h
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/lists.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/lists.h	(revision 699)
@@ -0,0 +1,154 @@
+/**************************************************************************
+ *
+ * lists.h -- List processing functions for boolean queries
+ * Copyright (C) 1994  Neil Sharman, Alistair Moffat and Lachlan Andrew
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef H_LISTS
+#define H_LISTS
+
+typedef struct query_data;
+extern double FetchDocStart(query_data *, u_long, u_long *, u_long *);
+
+class abstractDocEntry
+{
+ public:
+  int   DocNum;
+  float Weight;
+
+  // public:
+  inline int docNum()
+    {
+      return DocNum;
+    }
+
+  inline float docWeight()
+    {
+      return Weight;
+    }
+};
+
+class loadableDocEntry : public abstractDocEntry
+{
+ public:
+  u_long SeekPos;	/* position in the text file in bytes */
+  u_long Len;		/* length of the document in bytes */
+
+  inline double FetchStart(query_data *qd)
+    {
+      return FetchDocStart(qd, DocNum, &SeekPos, &Len);
+    }
+};
+
+class DocEntry : public loadableDocEntry
+  {
+  public:
+    char *CompTextBuffer;
+    DocEntry *Next;
+    short or_included;		/*[TS:Mar/95] whether included in an AND of ORed-terms */
+
+    inline void init(int doc, float weight)
+      { 
+	DocNum         = doc;
+	Weight         = weight;
+	SeekPos        = 0;
+	Len            = 0;
+	CompTextBuffer = NULL;
+	Next           = NULL;
+	or_included    = 0;
+      }
+
+    inline void copy(DocEntry *src)
+      { *this = *src;
+      }
+  };
+
+typedef struct DocList
+  {
+    int num;
+    int total_retrieved; /* valid if > 0 */
+    int is_approx;       /* whether total_retrieved is approximate */
+    int list_size;
+    DocEntry *DE;
+
+    inline int size()
+    { return num;
+    }
+
+    inline DocEntry *member(int at)
+    { return &DE[at];
+    }
+
+    inline DocEntry *lastMember()
+    { return member(num - 1);
+    }
+
+    inline void addMember(int doc, float weight)
+    { member(num)->init(doc, weight);
+      num ++;
+    }
+
+    inline void addMember(DocEntry *entry)
+    { DE[num] = *entry;
+      num ++;
+    }
+
+    inline void setMember(int at, DocEntry *entry)
+    { DE[at] = *entry;
+    }
+  }
+DocList;
+
+DocList *DocList_create (int num);
+DocList *DocList_resize (DocList * d, int num);
+void     DocList_destroy (DocList *d);
+DocList *DocList_intersect (DocList * List1, DocList * List2);
+DocList *DocList_diff (DocList * List1, DocList * List2);
+DocList *DocList_merge (DocList * List1, DocList * List2);
+DocList *DocList_addDoc(DocList *List, int res);
+void     DocList_reduceOrIncluded(DocList *List);
+
+#ifdef __cplusplus
+
+inline int DocList_isEmpty(DocList *List)
+{ return (List == NULL || List->num == 0);
+}
+
+inline void DocList_setsize(DocList *List, int n)
+{ List->num = n;
+}
+
+inline DocEntry *DocList_nextMember(DocList *List, DocEntry *entry)
+{ entry ++;
+  return entry; 
+}
+
+inline int DocList_memberValid(DocList *List, DocEntry *entry)
+{ return (entry < (List->DE + List->num));
+}
+
+
+inline void DocList_moveMember(DocList *List, int to, int from)
+{ List->DE[to] = List->DE[from];
+}
+
+#endif
+#endif
Index: /tags/gsdl-21/gsdl/src/mgpp/text/mg_decompress_text.1
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/mg_decompress_text.1	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/mg_decompress_text.1	(revision 699)
@@ -0,0 +1,78 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_decompress_text 1 \*(Dt CITRI
+.SH NAME
+mg_decompress_text \- decompresses the compressed text of a collection
+.SH SYNOPSIS
+.B mg_decompress_text
+[
+.B \-h
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_decompress_text
+decompresses the compressed text of a collection, inserts paragraph and
+document markers, writing the output to
+.IR stdout ".  "
+This program was created
+to be used as a type of
+.BR mg_get (1).
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 25
+.B *.text
+Compressed documents.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.text.dict.aux
+Auxiliary compression dictionary.
+.TP
+.B *.text.dict.fast
+Fast loading compression dictionary.
+.TP
+.B *.invf.paragraph
+Paragraph counts for documents.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_passes (1).
Index: /tags/gsdl-21/gsdl/src/mgpp/text/mg_hilite_words.1
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/mg_hilite_words.1	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/mg_hilite_words.1	(revision 699)
@@ -0,0 +1,95 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_hilite_words 1 \*(Dt CITRI
+.SH NAME
+mg_hilite_words \- display text and highlight particular words which it contains
+.SH SYNOPSIS
+.B mg_hilite_words
+.if n .ti +9n
+[
+.B --stem_method [0-3]
+]
+.if n .ti +9n
+[
+.B --stemmer [english|lovin|french|simplefrench]
+]
+.if n .ti +9n
+[
+.B --style [bold|underline|italic|emphasis|strong]
+]
+.if n .ti +9n
+[
+.B --pager [less|more|html|???]
+]
+.if n .ti +9n
+.I list-of-words-to-highlight
+
+.SH DESCRIPTION
+.B mg_hilite_words
+reads text from stdin and outputs it into a pager such as
+less. It's command arguments include a list of stemmed words
+which if extracted from the text will be highlighted on the output
+using a highlighting style.
+
+It has been extended to output the highlighted text using html
+straight to stdout.
+
+.SH OPTIONS
+Options may appear in any order, but the
+.IR list-of-words-to-highlight ,
+must be last.
+.TP
+.B --stem_method [0-3]
+This option allows one to set the stemming method for the extraction of
+the words from the incoming text.
+This should match the way the inverted file index was created.
+The default is 3, which is fold and stem.
+.TP
+.B --stemmer [english|lovin|french|simplefrench]
+This option allows you to select the stemmer to use.  The
+default is the English stemmer.
+.TP
+.B --style [bold|underline|italic|emphasis|strong]
+This option sets the highlighting style.
+.IR Bold 
+and 
+.IR underline
+are utilised by the pager or for html output.
+.IR Italic,
+.IR emphasis,
+and
+.IR strong
+can only be used for html output.
+.TP
+.B --pager [less|more|html|???]
+One can use the standard 
+.IR less
+or
+.IR more
+pages for displaying the highlighted text.
+Or one can generate highlighted html using the paging setting of
+.IR html.
+One can also specify a custom pager of their own choosing (i.e. ???).
+.TP
+.B list-of-words-to-highlight
+Each word is taken as a command line argument.
+Each word should already be stemmed and folded according to
+the stem option.
+.SH USAGE
+This program was written primarily to be called by 
+.IR mgquery(1)
+.
+When the user selects the mode of "hilite" then this program
+is called for displaying the query-result documents instead of
+just the pager.
+.SH "SEE ALSO"
+.na
+.BR mgquery (1).
+
+
Index: /tags/gsdl-21/gsdl/src/mgpp/text/mg_hilite_words.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/mg_hilite_words.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/mg_hilite_words.cpp	(revision 699)
@@ -0,0 +1,697 @@
+/**************************************************************************
+ *
+ * mg_hilite_words -- display text and highlight particular words which 
+ *                    it contains
+ * Copyright (C) 1994 Tim Shimmin
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+static char *RCSID = "$Id$";
+
+#include <stdio.h>
+
+#include "non_ansi.h"
+
+#include "sysfuncs.h"
+
+#include "getopt.h"
+#include "messages.h"
+#include "local_strings.h"
+#include "stemmer.h"
+#include "words.h"
+
+
+/*
+ * Description
+ * -----------
+ *   Hilite_words reads text from stdin and outputs it into a pager such as
+ *   less. It's command arguments include a list of stemmed words 
+ *   which if extracted from the text will be highlighted on the output.
+ *
+ * Implementation
+ * --------------
+ *   Extracting of words - using PARSE_STEM_WORD & stemmer
+ *   
+ *   Highlighting of words - for standard pager:
+ *                             using back-space character code
+ *                             e.g. bolded a = "a\ba", underlined a = "_\ba"
+ *                         - for pager==html:
+ *                             use some of the standard html character 
+ *                             formatting tags
+ *
+ *   Storage of words - using hashtable (set) of size equalling a constant
+ *                      times the number of words
+ * 
+ * Usage
+ * -----
+ *   mg_hilite_words --stem_method [0-3] 
+ *                   --style [bold|underline|italic|emphasis|strong]
+ *                   --pager [less|more|html|???]
+ *                   --terminator [terminator-string]
+ *                   list-of-words-to-highlight
+ */
+
+/*
+ * Modifications:
+ *
+ * 21/Apr/95: To handle outputting html tags to stdout
+ *
+ */
+
+/* --- constants --- */
+
+/* highlighting styles */
+#define HILITE_MAX      5	/* the number of styles */
+#define BOLD 		0
+#define UNDERLINE	1
+#define ITALIC     	2
+#define EMPHASIS     	3
+#define STRONG     	4
+
+/* maximum length of line buffer */
+#define MAX_LINE_BUFFER 200
+
+/* set pager to this to get html hilited text to stdout */
+#define HTML_OUT "html"
+
+/* output types */
+#define PAGER 0
+#define HTML  1
+
+/* --- types --- */
+
+typedef u_char *Word;
+
+/* --- globals --- */
+
+/* keep in synch with constants */
+static char *hilite_names[] =
+{"bold", "underline", "italic", "emphasis", "strong"};
+static char *hilite_tags[] =
+{"B", "", "I", "EM", "STRONG"};
+static short hilite_style = BOLD;
+static char *pager = "less";
+static int stemmer_num = 0;	/* Lovin's stemmer */
+static int stem_method = 3;	/* fold & stem */
+static char **word_list;
+static int num_of_words = 0;
+static int output_type = PAGER;
+static char *terminator = NULL;
+
+/* --- prototypes --- */
+
+static Word copy_c_word (char *w);
+static void process_words (char **words, int num_of_words);
+static int get_line (u_char * line, int n, FILE * stream);
+static void process_text (FILE * input_file, FILE * output_file);
+static void copy_word (u_char * w1, u_char * w2);
+static void process_buffer (u_char * s_in, int len, FILE * output_file);
+static void output_word (u_char * s_start, u_char * s_finish, FILE * output_file);
+static void output_hilite_word (u_char * s_start, u_char * s_finish,
+				FILE * output_file);
+static void process_args (int argc, char *argv[]);
+static void print_line (u_char * line, int n, FILE * stream);
+
+
+/******************************** word set ************************************/
+
+#include "hash.h"
+#include "locallib.h"
+
+#define SIZE_FACTOR 2
+
+typedef Word *WordSet;
+static WordSet set_of_words = NULL;
+static int hash_size = 0;
+
+/* prototypes - set routines */
+static void set_create (int num_of_words);
+static void set_add (Word word);
+static int set_member (Word word);
+static void set_print (void);
+
+/* =========================================================================
+ * Function: set_print
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+set_print (void)
+{
+  int i = 0;
+
+  for (i = 0; i < hash_size; i++)
+    {
+      Word word = set_of_words[i];
+
+      if (word)
+	{
+	  int len = *word++;
+
+	  fprintf (stderr, "[%d] = ", i);
+	  while (len--)
+	    fputc (*word++, stderr);
+	  fputc ('\n', stderr);
+	}
+    }
+
+}
+
+
+/* =========================================================================
+ * Function: set_create
+ * Description: 
+ *      Allocate memory for the set
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+set_create (int num_of_words)
+{
+  WordSet set;
+
+  hash_size = prime (num_of_words * SIZE_FACTOR);
+  set = (WordSet) malloc (hash_size * sizeof (Word));
+  if (!set)
+    FatalError (1, "Runout of memory for word hashtable");
+  bzero ((char *) set, hash_size * sizeof (Word));  /* [RPAP - Feb 97: WIN32 Port] */
+
+  set_of_words = set;
+
+}
+
+/* =========================================================================
+ * Function: set_add
+ * Description: 
+ *      Add a string element to the set.
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+set_add (Word word)
+{
+  int hash_val;
+  int hash_step;
+
+  HASH (hash_val, hash_step, word, hash_size);
+
+  /* loop around in case of collisions and need to step */
+  while (1)
+    {
+
+      Word entry = set_of_words[hash_val];
+
+      /* if doesn't exist then */
+      if (!entry)
+	{
+	  set_of_words[hash_val] = word;
+	  break;
+	}
+
+      /* if we have a matching word */
+      if (compare (entry, word) == 0)
+	break;
+
+      /* if collides with a different word */
+      hash_val = (hash_val + hash_step) % hash_size;
+
+    }
+}
+
+
+/* =========================================================================
+ * Function: set_member
+ * Description: 
+ *      Tests whether a string is a member of the set
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static int
+set_member (Word word)
+{
+  int hash_val;
+  int hash_step;
+
+  HASH (hash_val, hash_step, word, hash_size);
+
+  /* loop around in case of collisions and need to step */
+  while (1)
+    {
+
+      Word entry = set_of_words[hash_val];
+
+      /* if doesn't exist then */
+      if (!entry)
+	return 0;
+
+      /* if we have a matching word */
+      if (compare (entry, word) == 0)
+	return 1;
+
+      /* if collides with a different word */
+      hash_val = (hash_val + hash_step) % hash_size;
+
+    }
+}
+
+/******************************** end of word set ******************************/
+
+
+
+/* =========================================================================
+ * Function: copy_c_word
+ * Description: 
+ *      Allocate enough memory and copy word over
+ * Input: 
+ *      w = null terminated string (c_word)
+ * Output: 
+ *      word with length in 1st byte
+ * ========================================================================= */
+
+static Word
+copy_c_word (char *w)
+{
+  int len = strlen (w);
+  Word w_copy = (Word) malloc (len + 1);
+  Word w_ptr = NULL;
+  int j = 0;
+
+  if (!w_copy)
+    FatalError (1, "Not enough memory to copy a word");
+
+  w_copy[0] = len;
+  w_ptr = w_copy + 1;
+  for (j = 0; j < len; j++)
+    *w_ptr++ = *w++;
+
+  return w_copy;
+}
+
+/* =========================================================================
+ * Function: process_words
+ * Description: 
+ *      Go through the stemmed words and add to word set
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+process_words (char **words, int num_of_words)
+{
+  int i = 0;
+
+  set_create (num_of_words);
+
+  for (i = 0; i < num_of_words; i++)
+    {
+      Word word = copy_c_word (words[i]);
+      set_add (word);
+    }
+
+}
+
+/* =========================================================================
+ * Function: get_line
+ * Description: 
+ *      Equivalent of fgets for u_char*.
+ *      But returns length of read-in line.
+ *      Expects to see a '\n' before an EOF
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static int
+get_line (u_char * line, int n, FILE * stream)
+{
+  int i = 0;
+  int ch = '\0';
+
+  while (1)
+    {
+      if (i == n)
+	return i;
+
+      ch = fgetc (stream);
+
+      if (ch == EOF)
+	{
+	  if (!feof (stream))
+	    FatalError (1, "Error on reading a line from stdin");
+	  return EOF;
+	}
+
+      if (ch == '\n')
+	return i;
+
+      *line++ = ch;
+      i++;
+    }
+
+}
+
+/* =========================================================================
+ * Function: print_line
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+print_line (u_char * line, int n, FILE * stream)
+{
+
+  while (n--)
+    {
+      fputc (*line++, stream);
+    }
+  fputc ('\n', stream);
+
+}
+
+/* =========================================================================
+ * Function: process_text
+ * Description: 
+ *      Go through the text from input_file and highlight to output_file 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+process_text (FILE * input_file, FILE * output_file)
+{
+  static u_char line_buffer[MAX_LINE_BUFFER];
+
+
+  while (1)
+    {
+      int len = get_line (line_buffer, MAX_LINE_BUFFER, input_file);
+
+      if (len == EOF)
+	break;
+      process_buffer (line_buffer, len, output_file);
+
+    }
+}
+
+/* =========================================================================
+ * Function: copy_word
+ * Description: 
+ *      Copies w2 into w1. Assumes both have storage allocated.
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+copy_word (u_char * w1, u_char * w2)
+{
+  int i;
+  int len = w2[0];
+
+  for (i = 0; i <= len; i++)
+    *w1++ = *w2++;
+
+}
+
+/* =========================================================================
+ * Function: process_buffer
+ * Description: 
+ *      Parse & stem words of line buffer
+ *      Based on the usage of PARSEing in other mg files.
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+static void
+process_buffer (u_char * s_in, int len, FILE * output_file)
+{
+  u_char *end = s_in + len - 1;
+  u_char *s_start = NULL;
+
+  if (!inaword (s_in, end))
+    {
+      s_start = s_in;
+      PARSE_NON_STEM_WORD (s_in, end);
+      output_word (s_start, s_in - 1, output_file);
+    }
+
+  while (s_in <= end)
+    {
+      u_char word[MAXSTEMLEN + 1];
+
+      s_start = s_in;
+      PARSE_STEM_WORD (word, s_in, end);
+
+      stemmer (stem_method, stemmer_num, word);
+
+      if (set_member (word))	/* output with highlighting */
+	{
+	  output_hilite_word (s_start, s_start + word[0] - 1, output_file);
+	  s_start += word[0];	/* step over hilited output */
+	}
+      output_word (s_start, s_in - 1, output_file);
+
+      s_start = s_in;
+      PARSE_NON_STEM_WORD (s_in, end);
+      output_word (s_start, s_in - 1, output_file);
+
+    }				/*while */
+
+  fputc ('\n', output_file);
+  fflush (output_file);
+
+}				/*process_buffer */
+
+/* =========================================================================
+ * Function: output_word
+ * Description: 
+ *      Output a word which lies from s_start to s_finish in buffer
+ * Input: 
+ *      s_start = ptr to 1st char
+ *      s_finish = ptr to last char
+ * Output: 
+ * ========================================================================= */
+
+static void
+output_word (u_char * s_start, u_char * s_finish, FILE * output_file)
+{
+  while (s_start <= s_finish)
+    {
+      fputc (*s_start++, output_file);
+    }
+}
+
+
+/* =========================================================================
+ * Function: output_hilite_word
+ * Description: 
+ *      Highlight a word (with length in 1st byte)
+ *      Pager highlighting:
+ *        Highlighting is either by bolding or underlining using
+ *        the method used by UNIX utilities More(1) and Less(1)
+ *      HTML highlighting:
+ *        use the appropriate start and end tags around the word
+ * ========================================================================= */
+
+static void
+output_hilite_word (u_char * s_start, u_char * s_finish, FILE * output_file)
+{
+
+  if (output_type == HTML)
+    {
+      char *hilite_tag = hilite_tags[hilite_style];
+
+      /* print start tag */
+      fprintf (output_file, "<%s>", hilite_tag);
+
+      output_word (s_start, s_finish, output_file);
+
+      /* print end tag */
+      fprintf (output_file, "</%s>", hilite_tag);
+    }
+
+  else
+    /* PAGER */
+    {
+      /* use backspaces around each letter */
+      while (s_start <= s_finish)
+	{
+	  switch (hilite_style)
+	    {
+	    case BOLD:
+	      fputc (*s_start, output_file);
+	      fputc ('\b', output_file);
+	      fputc (*s_start, output_file);
+	      break;
+	    case UNDERLINE:
+	      fputc ('_', output_file);
+	      fputc ('\b', output_file);
+	      fputc (*s_start, output_file);
+	      break;
+	    default:
+	      fputc (*s_start, output_file);
+	    }
+	  s_start++;
+	}			/*while */
+    }
+}
+
+/* =========================================================================
+ * Function: process_args
+ * Description: 
+ *      sets the global variables:
+ *              hilite_style, pager, num_of_words, word_list
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+struct option long_opts[] =
+{
+  {"style", required_argument, 0, 's'},
+  {"terminator", required_argument, 0, 't'},
+  {"pager", required_argument, 0, 'p'},
+  {"stem_method", required_argument, 0, 'm'},
+  {"stemmer", required_argument, 0, 'a'},
+  {0, 0, 0, 0}
+};
+
+static void
+process_args (int argc, char *argv[])
+{
+  int ch;
+
+
+  opterr = 0;
+  while ((ch = getopt_long (argc, argv, "s:p:t:m:a:", long_opts, (int *) 0)) != -1)
+    {
+      switch (ch)
+	{
+	case 's':
+	  {
+	    int i;
+	    for (i = 0; i < HILITE_MAX; i++)
+	      if (strcmp (optarg, hilite_names[i]) == 0)
+		break;
+
+	    if (i < HILITE_MAX)
+	      hilite_style = i;
+	  }
+	  break;
+	case 'a':
+	  stemmer_num = stemmernumber ((unsigned char *) optarg);
+	  break;
+	case 't':
+	  terminator = optarg;
+	  break;
+	case 'm':
+	  stem_method = atoi (optarg);
+	  break;
+	case 'p':
+	  pager = optarg;
+	  break;
+	default:
+	  FatalError (1, "Usage: \n"
+		      "mg_hilite_words --stem_method [0-3]\n"
+	  "                --stemmer [english|lovin|french|simplefrench]\n"
+	  "                --style [bold|underline|italic|emphasis|strong]\n"
+		      "                --pager [less|more|html|???]\n");
+	}
+    }
+
+  num_of_words = argc - optind;
+
+  word_list = &argv[optind];
+
+  /* fix up output type */
+  if (strcmp (pager, HTML_OUT) == 0)
+    output_type = HTML;
+  else
+    output_type = PAGER;
+
+}
+
+/* =========================================================================
+ * Function: main
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+int
+main (int argc, char *argv[])
+{
+  FILE *output = NULL;
+
+  process_args (argc, argv);
+
+  /* set output file */
+  if (output_type == PAGER)
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+    output = _popen (pager, "w");
+#else
+    output = popen (pager, "w");
+#endif
+  else
+    output = stdout;
+
+  if (!output)
+    FatalError (1, "Unable to run \"%s\"\n", pager);
+
+  if (num_of_words < 1)
+    {
+      int ch;
+
+      /* just echo the input */
+      /* better not to call this program at all ;-) */
+      while ((ch = fgetc (stdin)) != EOF)
+	{
+	  fputc (ch, output);
+	}
+    }
+  else
+    {
+      /* set up hash table for words */
+      process_words (word_list, num_of_words);
+
+
+      /* Go thru lines of text from stdin and 
+       * output words with hilite info if
+       * words parse into existence in the hash table 
+       */
+      process_text (stdin, output);
+    }
+
+  if (terminator)
+    fprintf (output, "%s\n", terminator);
+
+  if (output != stdout)
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+    _pclose (output);
+#else
+    pclose (output);
+#endif
+
+  exit(0);
+}
Index: /tags/gsdl-21/gsdl/src/mgpp/text/mg_invf_dump.1
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/mg_invf_dump.1	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/mg_invf_dump.1	(revision 699)
@@ -0,0 +1,142 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_invf_dump 1 \*(Dt CITRI
+.SH NAME
+mg_invf_dump \- Dump out an inverted file in ASCII
+.SH SYNOPSIS
+.B mg_invf_dump
+[
+.B \-h
+]
+[
+.B \-b
+]
+[
+.B \-w
+]
+[
+.B \-t
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+This program dumps out an inverted file produced by the
+.BR mg (1)
+system as ASCII numbers.  This program could be used in conjunction
+with
+.BR mgdictlist (1)
+to write simple shell programs that work with the inverted files.  The
+output from the program looks something like this:
+.LP
+.RS
+.ft 3
+.nf
+6 337
+1
+ 1 1
+1
+ 5 1
+1
+ 2 1
+1
+ 1 1
+4
+ 1 3
+ 2 7
+ 3 2
+ 5 2
+1
+ 1 1
+ . . .
+.fi
+.ft
+.RE
+The first number (6) is the number of documents in the collection.  It
+is followed by the number of different terms in the collection.  For
+each term, the number of different documents it occurs in follows. The
+document numbers that the term occurs in are listed next.  If
+.B \-w
+is specified, then the number of times the term occurred in the
+document is printed alongside the document number.
+If
+.B \-t
+is specified, the term itself is also displayed in quotes.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-b
+This option will cause the output from the program to be in fixed-size
+binary numbers, rather than in ASCII.
+.TP
+.B \-w
+If the inverted file is a level-2 or level-3 inverted file, this causes
+the word counts per document to be output.
+.TP
+.B \-t
+This option causes each term to be displayed along with the number of
+documents it appears in. The term is printed in quotes,
+as in "the".
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf.dict
+The compressed stemmed dictionary.
+.TP
+.B *.invf.ORG
+The original inverted file saved by
+.BR mg_invf_rebuild .
+.TP
+.B *.invf
+The inverted file.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /tags/gsdl-21/gsdl/src/mgpp/text/mg_invf_merge.1
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/mg_invf_merge.1	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/mg_invf_merge.1	(revision 699)
@@ -0,0 +1,112 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_invf_merge 1 \*(Dt CITRI
+.SH NAME
+mg_invf_merge \- Merge two inverted files and stemmed dictionaries
+.SH SYNOPSIS
+.B mg_invf_merge
+[
+.B \-h
+]
+[
+.B \-s
+]
+[
+.B \-w
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+This program takes two
+.BR mg (1)
+database compressed text files and appends the second to
+the first.
+It is part of the
+.BR mgmerge (1)
+addition to the
+.BR mg (1)
+package.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-s
+This option will cause a slow merge to be performed on the inverted files,
+where each inverted file entry is decoded and recoded.
+The default is a fast merge. Accumulated fast merges slowly degrade
+compression performance on the resulting inverted file so
+a periodic slow merge is recommended.
+.TP
+.B \-w
+By default the database .weights file is updated with
+new weights for the new documents in
+.B mg_text_merge.
+The
+.B -w
+option turns this step off.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.old.invf.dict
+The old compressed stemmed dictionary.
+.TP
+.B *.new.invf.dict
+The new compressed stemmed dictionary.
+.TP
+.B *.invf.dict
+The resulting merged compressed stemmed dictionary.
+.TP
+.B *.old.invf.ORG
+The original inverted file saved by
+.BR mg_invf_rebuild .
+.TP
+.B *.old.invf
+The old inverted file.
+.TP
+.B *.new.invf
+The new inverted file.
+.TP
+.B *.invf
+The resulting merged inverted file.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mgmerge (1),
+.BR mg_get_merge (1),
+.BR mg_text_merge (1),
+.BR mg_invf_rebuild (1),
+.BR mg_invf_dict (1),
+.BR mg_weights_build (1),
+.BR mgquery (1).
Index: /tags/gsdl-21/gsdl/src/mgpp/text/mg_invf_merge.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/mg_invf_merge.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/mg_invf_merge.cpp	(revision 699)
@@ -0,0 +1,857 @@
+/**************************************************************************
+ *
+ * mg_invf_merge.c -- description
+ * Copyright (C) 1995  Shane Hudson (shane@cosc.canterbury.ac.nz)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * Last edited:  31 March, 1995 
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "locallib.h"
+#include "local_strings.h"
+#include "messages.h"
+#include "timing.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "bitio_m.h"
+#include "bitio_gen.h"
+#include "bitio_stdio.h"
+
+#include "mg.h"
+#include "mg_merge.h"
+#include "mg_files.h"
+#include "invf.h"
+#include "words.h"
+
+typedef char FileName[256];
+
+
+/*
+ *  The merge_info struct contains all the data for an inverted file and
+ *  lexicon. We declare 3 merge_info variables, m[OLD] m[NEW] and m[MERGE].
+ */
+typedef struct merge_info_type
+  {
+    FileName fname;
+    FILE *invf;
+    FILE *dict;
+    FILE *idx;
+    FILE *para;  /* [RPAP - Feb 97: Level 3 Merge] */
+    struct invf_dict_header idh;
+    struct invf_file_header ifh;
+    u_long nDocs;
+
+    /* Lexicon processing variables */
+    u_long fcnt, wcnt;
+    int suff, pref;
+    unsigned char term[MAXSTEMLEN + 1];
+    u_long term_count;
+    int done;
+
+    /* Variables used for merging */
+    struct stdio_bitio_state sbs;	/* for bit-based I/O on inverted files */
+    u_long nTerms;		/* = number of invf entries */
+    int *fcntlist;		/* store fcnt values read in from the lexicons */
+
+  }
+merge_info;
+
+
+/***************************************************
+ **** GLOBALS ****
+ ***************************************************/
+
+merge_info m[3];		/*  m[OLD], m[NEW] and m[MERGE] */
+u_long magicsize;		/* how big is magic number? currently always 4 */
+u_long Nstatic;			/* static # of documents in file */
+
+float *DocWeightBuffer;
+int weightOption = 0;		/* true if weights file is to be appended */
+
+/*** Globals for processing the lexicons -- see readTerm() and writeTerm() 
+ ***/
+ProgTime start;			/* for printing elapsed time */
+unsigned char prevTerm[MAXSTEMLEN + 1];
+
+/*** Globals for merging -- see processEntry() 
+ ***/
+char *mergedata;		/* stores value OLD/NEW/MERGE for each term */
+int fastMerge = 1;		/* True if fast merge selected. Default = true */
+
+u_long oldIdxValue;		/* stores last read value from m[OLD].idx
+				   Used to calculate entry lengths */
+u_long bytes_output;		/* for writing .invf.idx file */
+u_long oldOnlyBits = 0;		/* entries in old only */
+u_long oldMergeBits = 0;	/* entries in old that are merged with 
+				   a new entry */
+
+/**************************************************************************/
+
+
+/*=======================================================================
+ * init_merge_invf(): open files, initialise globals, etc
+ *=======================================================================*/
+int 
+init_merge_invf ()
+{
+  prevTerm[0] = prevTerm[1] = 0;
+
+    /***
+     *  open .dict files
+     ***/
+  m[OLD].dict = open_file (m[OLD].fname, INVF_DICT_SUFFIX, "rb",
+			   MAGIC_STEM_BUILD, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+  magicsize = ftell (m[OLD].dict);
+  Invf_Header_read(&(m[OLD].idh), m[OLD].dict);
+  /*
+  fread (&(m[OLD].idh), sizeof (m[OLD].idh), 1, m[OLD].dict);
+
+  // [RPAP - Jan 97: Endian Ordering]
+  NTOHUL(m[OLD].idh.lookback);
+  NTOHUL(m[OLD].idh.dict_size);
+  NTOHUL(m[OLD].idh.total_bytes);
+  NTOHUL(m[OLD].idh.index_string_bytes);
+  NTOHD(m[OLD].idh.input_bytes); // [RJM 07/97: 4G limit]
+  NTOHUL(m[OLD].idh.num_of_docs);
+  NTOHUL(m[OLD].idh.static_num_of_docs);
+  NTOHUL(m[OLD].idh.num_of_words);
+  NTOHUL(m[OLD].idh.stemmer_num);
+  NTOHUL(m[OLD].idh.stem_method);
+  */
+
+  m[NEW].dict = open_file (m[NEW].fname, INVF_DICT_SUFFIX, "rb",
+			   MAGIC_STEM_BUILD, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+  Invf_Header_read(&(m[NEW].idh), m[NEW].dict);
+  /*		 
+  fread (&(m[NEW].idh), sizeof (m[NEW].idh), 1, m[NEW].dict);
+
+  // [RPAP - Jan 97: Endian Ordering]
+  NTOHUL(m[NEW].idh.lookback);
+  NTOHUL(m[NEW].idh.dict_size);
+  NTOHUL(m[NEW].idh.total_bytes);
+  NTOHUL(m[NEW].idh.index_string_bytes);
+  NTOHD(m[NEW].idh.input_bytes); // [RJM 07/97: 4G limit]
+  NTOHUL(m[NEW].idh.num_of_docs);
+  NTOHUL(m[NEW].idh.static_num_of_docs);
+  NTOHUL(m[NEW].idh.num_of_words);
+  NTOHUL(m[NEW].idh.stemmer_num);
+  NTOHUL(m[NEW].idh.stem_method);
+  */
+  m[MERGE].dict = create_file (m[MERGE].fname, INVF_DICT_SUFFIX, "wb",
+			       MAGIC_STEM_BUILD, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  /* write space for header */
+  fwrite ((char *) &(m[OLD].idh), sizeof (m[OLD].idh), 1, m[MERGE].dict);
+
+  m[OLD].nDocs = m[OLD].idh.num_of_docs;
+  m[NEW].nDocs = m[NEW].idh.num_of_docs;
+  Nstatic = m[OLD].idh.static_num_of_docs;
+  m[OLD].nTerms = m[OLD].idh.dict_size;
+  m[NEW].nTerms = m[NEW].idh.dict_size;
+  m[MERGE].nDocs = m[OLD].nDocs + m[NEW].nDocs;
+  m[MERGE].nTerms = 0;
+
+  /* Set up weight buffer for weights of new documents */
+  if (!(DocWeightBuffer = (float *) Xmalloc (m[NEW].nDocs * sizeof (*DocWeightBuffer))))
+    {
+      Message ("Insufficient memory\n");
+      return COMPERROR;
+    }
+  bzero ((char *) DocWeightBuffer, m[NEW].nDocs * sizeof (*DocWeightBuffer));
+
+    /***
+     *  open .invf files
+     *  Try ".ORG" extension first for m[OLD].invf since an inverted file
+     *  with skips may have been created.
+     ****/
+  if (!(m[OLD].invf = open_file (m[OLD].fname, INVF_SUFFIX ".ORG", "rb",
+				 MAGIC_INVF, MG_CONTINUE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    m[OLD].invf = open_file (m[OLD].fname, INVF_SUFFIX, "rb",
+			     MAGIC_INVF, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+  fread (&(m[OLD].ifh), sizeof (m[OLD].ifh), 1, m[OLD].invf);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(m[OLD].ifh.no_of_words);
+  NTOHUL(m[OLD].ifh.no_of_ptrs);
+  NTOHUL(m[OLD].ifh.skip_mode);
+  {
+    int i;
+    for (i = 0; i < 16; i++)
+      NTOHUL(m[OLD].ifh.params[i]);
+  }
+  NTOHUL(m[OLD].ifh.InvfLevel);
+
+  m[NEW].invf = open_file (m[NEW].fname, INVF_SUFFIX, "rb",
+			   MAGIC_INVF, MG_ABORT);
+  fread (&(m[NEW].ifh), sizeof (m[NEW].ifh), 1, m[NEW].invf);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(m[NEW].ifh.no_of_words);
+  NTOHUL(m[NEW].ifh.no_of_ptrs);
+  NTOHUL(m[NEW].ifh.skip_mode);
+  {
+    int i;
+    for (i = 0; i < 16; i++)
+      NTOHUL(m[NEW].ifh.params[i]);
+  }
+  NTOHUL(m[NEW].ifh.InvfLevel);
+
+  if (m[OLD].ifh.skip_mode != 0)
+    FatalError (1, "The old invf file contains skips. Unable to merge.");
+
+  if (m[OLD].ifh.InvfLevel != m[NEW].ifh.InvfLevel)
+    FatalError (1, "The two invf files have different inversion levels!");
+
+  /********** [RPAP - Feb 97: Level 3 Merge]
+
+  if ((m[OLD].ifh.InvfLevel > 2) || (m[NEW].ifh.InvfLevel > 2))
+    FatalError (1, "mgmerge cannot handle level 3 inverted files!");
+
+    *************/
+
+  m[MERGE].invf = create_file (m[MERGE].fname, INVF_SUFFIX, "wb",
+			       MAGIC_INVF, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  /* write space for header in inverted file */
+  m[MERGE].ifh.no_of_words = m[MERGE].ifh.no_of_ptrs = 0;
+  m[MERGE].ifh.skip_mode = 0;
+  m[MERGE].ifh.InvfLevel = 2;
+  {
+    int i;
+    for (i = 0; i < 16; i++)
+      m[MERGE].ifh.params[i] = 0;
+  }
+  fwrite ((char *) &(m[MERGE].ifh), sizeof (m[MERGE].ifh), 1, m[MERGE].invf);
+
+    /***
+     *  open .invf.idx files
+     ***/
+  m[OLD].idx = open_file (m[OLD].fname, INVF_IDX_SUFFIX, "rb",
+			  MAGIC_INVI, MG_ABORT);
+  fread (&oldIdxValue, sizeof (u_long), 1, m[OLD].idx);
+  NTOHUL(oldIdxValue);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  m[NEW].idx = open_file (m[NEW].fname, INVF_IDX_SUFFIX, "rb",
+			  MAGIC_INVI, MG_ABORT);
+
+  m[MERGE].idx = create_file (m[MERGE].fname, INVF_IDX_SUFFIX, "wb",
+			      MAGIC_INVI, MG_ABORT);
+
+
+  /* [RPAP - Feb 97: Level 3 Merge] */
+
+    /***
+     *  open .invf.parargraph files
+     ***/
+  if (m[OLD].ifh.InvfLevel == 3)
+    {
+      m[OLD].para = NULL;
+      m[NEW].para = open_file (m[NEW].fname, INVF_PARAGRAPH_SUFFIX, "rb",
+			       MAGIC_PARAGRAPH, MG_ABORT);
+      m[MERGE].para = open_file (m[MERGE].fname, INVF_PARAGRAPH_SUFFIX, "rb+",
+				 MAGIC_PARAGRAPH, MG_ABORT);
+    }
+  else
+    {
+      m[OLD].para = NULL;
+      m[NEW].para = NULL;
+      m[MERGE].para = NULL;
+    }
+
+  return OK;
+}
+
+
+/*=======================================================================
+ * procEntry_slow(): merge an entry, slow method
+ *=======================================================================*/
+void 
+procEntry_slow (int tnum, int type, u_long oldN, u_long newN, u_long mergeN)
+/* 
+ *  Process an invf entry. 
+ *  Slow Merge. (Always decode/re-encode).
+ *
+ *  tnum is the entry number, type = OLD or NEW or MERGE.
+ *  oldN, newN and mergeN are the N parameters for Bblock coding.
+ *  They are passed so other procEntry_fast can call this one
+ *  when a decode/recode is needed to save repeating code.
+ */
+{
+  int doc = 0;
+  int fcntOLD = m[OLD].fcntlist[tnum];
+  int fcntNEW = m[NEW].fcntlist[tnum];
+  int inblk, outblk;
+  u_long inbits = 0, outbits = 0;
+
+  /* write .invf.idx pointer */
+  HTONUL(bytes_output);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite (&bytes_output, sizeof (u_long), 1, m[MERGE].idx);
+  NTOHUL(bytes_output);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  outblk = BIO_Bblock_Init (mergeN, (fcntOLD + fcntNEW));
+
+/*** OLD entry -> merge entry ***/
+  if (type != NEW)
+    {
+      int i;
+      inblk = BIO_Bblock_Init (oldN, fcntOLD);
+      for (i = 0; i < fcntOLD; i++)
+	{
+	  int num;
+	  num = BIO_Stdio_Bblock_Decode (inblk, &(m[OLD].sbs), &inbits);
+	  doc += num;
+	  BIO_Stdio_Bblock_Encode (num, outblk, &(m[MERGE].sbs), &outbits);
+	  if (m[OLD].ifh.InvfLevel >= 2)	/* frequencies */
+	    {
+	      num = BIO_Stdio_Gamma_Decode (&(m[OLD].sbs), &inbits);
+	      BIO_Stdio_Gamma_Encode (num, &(m[MERGE].sbs), &outbits);
+	    }
+	}
+      if (type == OLD)
+	oldOnlyBits += (inbits + m[OLD].sbs.Btg);
+      else
+	oldMergeBits += (inbits + m[OLD].sbs.Btg);
+    }
+
+/*** NEW entry -> merge entry ***/
+  if (type != OLD)
+    {
+      int i;
+      int offset;
+      double logN = log ((double) m[MERGE].nDocs);
+      double idf = logN - log ((double) (fcntOLD + fcntNEW));
+
+      inblk = BIO_Bblock_Init (newN, fcntNEW);
+      offset = (m[OLD].nDocs - doc);	/* amount to be added to 1ST pointer */
+      for (i = 0; i < fcntNEW; i++)
+	{
+	  register int num;
+	  register double weight;
+	  num = BIO_Stdio_Bblock_Decode (inblk, &(m[NEW].sbs), &inbits);
+	  if (i == 0)
+	    num += offset;
+	  doc += num;
+	  BIO_Stdio_Bblock_Encode (num, outblk, &(m[MERGE].sbs), &outbits);
+	  if (m[OLD].ifh.InvfLevel >= 2)	/* frequencies */
+	    {
+	      num = BIO_Stdio_Gamma_Decode (&(m[NEW].sbs), &inbits);
+	      BIO_Stdio_Gamma_Encode (num, &(m[MERGE].sbs), &outbits);
+	      weight = num * idf;
+	      DocWeightBuffer[doc - m[OLD].nDocs - 1] += weight * weight;
+	    }
+	}
+    }
+/*** Now the padding bits ***/
+  while (m[OLD].sbs.Btg)
+    BIO_Stdio_Decode_Bit (&(m[OLD].sbs));
+  while (m[NEW].sbs.Btg)
+    BIO_Stdio_Decode_Bit (&(m[NEW].sbs));
+  while (m[MERGE].sbs.Btg != 8)
+    {
+      BIO_Stdio_Encode_Bit (0, &(m[MERGE].sbs));
+      outbits++;
+    }
+  bytes_output += (outbits >> 3);
+  return;
+}
+
+
+/*=======================================================================
+ * procEntry_fast(): merge an entry, WITHOUT decoding if it's in the
+ *                   old inverted file only
+ *=======================================================================*/
+void 
+procEntry_fast (int tnum, int type)
+/*
+ *  Merge an invf entry. 
+ *  Faster Method : Copy (don't re-code) entries for terms only in IFold.
+ *  The N parameter is Nstatic, from m[OLD].idh.static_num_of_docs.
+ *  If entry type isnt OLD, just call procEntry_slow()
+ */
+{
+  u_long newIdxValue, len = 0;
+
+  /* Calculate IFold entry length in bytes if not NEW type */
+  if (type != NEW)		/* read in an index number from m[OLD].idx */
+    {
+      fread (&newIdxValue, sizeof (u_long), 1, m[OLD].idx);
+      NTOHUL(newIdxValue);  /* [RPAP - Jan 97: Endian Ordering] */
+      len = newIdxValue - oldIdxValue;
+      oldIdxValue = newIdxValue;
+    }
+
+  if (type != OLD)		/* must decode/recode */
+    procEntry_slow (tnum, type, Nstatic, m[NEW].nDocs, Nstatic);
+  else
+    {
+      /* write .invf.idx pointer */
+      HTONUL(bytes_output);  /* [RPAP - Jan 97: Endian Ordering] */
+      fwrite (&bytes_output, sizeof (u_long), 1, m[MERGE].idx);
+      NTOHUL(bytes_output);  /* [RPAP - Jan 97: Endian Ordering] */
+
+      /* copy entry */
+      {
+	int i;
+	char c;
+	for (i = 0; i < len; i++)
+	  {
+	    c = fgetc (m[OLD].invf);
+	    fputc (c, m[MERGE].invf);
+	  }
+      }
+      bytes_output += len;
+      oldOnlyBits += (len * 8);
+    }
+  return;
+}
+
+
+/*=======================================================================
+ * processEntry(): merge an entry. Call either procEntry_slow() or
+ *                 procEntry_fast.
+ *=======================================================================*/
+void 
+processEntry (int tnum, int type)
+/* 
+ *  Read an entry from IFold and/or IFnew, and write to IFmerge.
+ *  tnum is the term number, type is OLD or NEW or MERGE
+ */
+{
+  if (fastMerge)
+    procEntry_fast (tnum, type);
+  else
+    procEntry_slow (tnum, type, Nstatic, m[NEW].nDocs, m[MERGE].nDocs);
+  return;
+}
+
+
+/*=======================================================================
+ * readTerm()
+ *=======================================================================*/
+void 
+readTerm (int x)
+/*
+ *  Read the next term from the appropriate ".invf.dict" file.
+ *  x should be either OLD or NEW.
+ *  Sets m[x].done to true when that lexicon has been completely read.
+ */
+{
+  int i;
+
+  if (m[x].term_count >= m[x].nTerms)
+    {
+      m[x].term[0] = 0;
+      m[x].done = 1;
+      return;
+    }
+  m[x].pref = fgetc (m[x].dict);
+  m[x].suff = fgetc (m[x].dict);
+  m[x].term[0] = m[x].pref + m[x].suff;
+
+  for (i = 0; i < m[x].suff; i++)
+    {
+      m[x].term[m[x].pref + i + 1] = fgetc (m[x].dict);
+    }
+  m[x].term_count = m[x].term_count + 1;
+
+  fread ((char *) &(m[x].fcnt), sizeof (u_long), 1, m[x].dict);
+  fread ((char *) &(m[x].wcnt), sizeof (u_long), 1, m[x].dict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(m[x].fcnt);
+  NTOHUL(m[x].wcnt);
+
+  return;
+}
+
+
+/*=======================================================================
+ * writeTerm()
+ *=======================================================================*/
+void 
+writeTerm (int x)
+/*
+ *  Write the current term to m[MERGE].dict file.
+ *  x is OLD, NEW or MERGE.
+ */
+{
+  unsigned char i, prefix, suffix;
+  if (x == MERGE)
+    {
+      m[MERGE].fcnt = m[OLD].fcnt + m[NEW].fcnt;
+      m[MERGE].wcnt = m[OLD].wcnt + m[NEW].wcnt;
+      m[MERGE].pref = m[OLD].pref;
+      m[MERGE].suff = m[OLD].suff;
+      memcpy (m[MERGE].term, m[OLD].term, m[OLD].term[0] + 1);
+    }
+  if (x == OLD)
+    {
+      m[MERGE].fcnt = m[OLD].fcnt;
+      m[MERGE].wcnt = m[OLD].wcnt;
+    }
+  if (x == NEW)
+    {
+      m[MERGE].fcnt = m[NEW].fcnt;
+      m[MERGE].wcnt = m[NEW].wcnt;
+    }
+  prefix = prefixlen (prevTerm, m[x].term);
+  suffix = m[x].term[0] - prefix;
+  fputc (prefix, m[MERGE].dict);	/* prefix length */
+  fputc (suffix, m[MERGE].dict);	/* suffix length */
+  for (i = 0; i < suffix; i++)
+    fputc (m[x].term[prefix + i + 1], m[MERGE].dict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(m[MERGE].fcnt);
+  HTONUL(m[MERGE].wcnt);
+
+  fwrite ((char *) &(m[MERGE].fcnt), sizeof (m[MERGE].fcnt), 1, m[MERGE].dict);
+  fwrite ((char *) &(m[MERGE].wcnt), sizeof (m[MERGE].wcnt), 1, m[MERGE].dict);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(m[MERGE].fcnt);
+  NTOHUL(m[MERGE].wcnt);
+
+  memcpy (prevTerm, m[x].term, prefix + suffix + 1);
+  m[MERGE].idh.total_bytes += m[x].term[0] + 1;
+  m[MERGE].idh.index_string_bytes += m[x].term[0] + 2 - prefix;
+  fflush (m[MERGE].dict);
+}
+
+
+/*=======================================================================
+ * process_merge_invf(): The main loop
+ *=======================================================================*/
+int 
+process_merge_invf (void)
+/*
+ *  The main function to merge the lexicons and inverted files.
+ */
+{
+/*** Some extra Initialisation stuff first ***/
+
+  m[OLD].term[0] = m[NEW].term[0] = '\0';
+  m[OLD].term_count = m[NEW].term_count = 0;
+  m[OLD].done = m[NEW].done = 0;
+  m[MERGE].idh.total_bytes = m[MERGE].idh.index_string_bytes = 0;
+
+/*** malloc arrays: mergedata, m[OLD].fcntlist, m[NEW].fcntlist ***/
+  mergedata = (char *) malloc ((m[OLD].nTerms + m[NEW].nTerms) * sizeof (char));
+  if (mergedata == 0)
+    {
+      fprintf (stderr, "MALLOC error!\n");
+      exit (1);
+    }
+  {
+    int i;
+    for (i = 0; i < 2; i++)
+      {
+	m[i].fcntlist = (int *) malloc ((m[OLD].nTerms + m[NEW].nTerms) * sizeof (int));
+	if (m[i].fcntlist == 0)
+	  {
+	    fprintf (stderr, "MALLOC error!\n");
+	    exit (1);
+	  }
+      }
+  }
+
+/*========================*/
+/***    LEXICON PASS    ***/
+/*========================*/
+
+/*** read first terms ***/
+  readTerm (OLD);
+  readTerm (NEW);
+
+  m[MERGE].nTerms = 0;
+  while (m[OLD].done == 0 || m[NEW].done == 0)
+    {
+      int i;
+      if (m[OLD].done)
+	i = 1; /* NEW will always be greater */
+      else if (m[NEW].done)
+	i = -1; /* OLD will always be greater */
+      else
+	i = casecompare (m[OLD].term, m[NEW].term);
+
+      if (i < 0)
+	{			/* term in OLD only */
+	  mergedata[m[MERGE].nTerms] = (char) OLD;
+	  m[OLD].fcntlist[m[MERGE].nTerms] = m[OLD].fcnt;
+	  m[NEW].fcntlist[m[MERGE].nTerms] = 0;
+	  m[MERGE].nTerms++;
+	  writeTerm (OLD);
+	  readTerm (OLD);
+	}
+      if (i == 0)
+	{			/* term in both lexions */
+	  mergedata[m[MERGE].nTerms] = (char) MERGE;
+	  m[OLD].fcntlist[m[MERGE].nTerms] = m[OLD].fcnt;
+	  m[NEW].fcntlist[m[MERGE].nTerms] = m[NEW].fcnt;
+	  m[MERGE].nTerms++;
+	  writeTerm (MERGE);
+	  readTerm (OLD);
+	  readTerm (NEW);
+	}
+      if (i > 0)
+	{			/* term in NEW only */
+	  mergedata[m[MERGE].nTerms] = (char) NEW;
+	  m[NEW].fcntlist[m[MERGE].nTerms] = m[NEW].fcnt;
+	  m[OLD].fcntlist[m[MERGE].nTerms] = 0;
+	  m[MERGE].nTerms++;
+	  writeTerm (NEW);
+	  readTerm (NEW);
+	}
+    }/*while*/
+  Message ("%s\n", ElapsedTime (&start, NULL));
+  /* print some results about terms */
+  {
+    fprintf (stderr, "   Terms: OLD = %ld,  NEW = %ld,  MERGE = %ld\n",
+	     m[OLD].nTerms, m[NEW].nTerms, m[MERGE].nTerms);
+    fprintf (stderr, "   In OLD only: %ld,  In NEW only: %ld,  BOTH: %ld\n",
+	     (m[MERGE].nTerms - m[NEW].nTerms),
+	     (m[MERGE].nTerms - m[OLD].nTerms),
+	     (m[OLD].nTerms + m[NEW].nTerms - m[MERGE].nTerms));
+  }
+
+
+/*========================*/
+/***      INVF PASS     ***/
+/*========================*/
+
+  bytes_output = ftell (m[MERGE].invf);
+  BIO_Stdio_Decode_Start (m[OLD].invf, &m[OLD].sbs);
+  BIO_Stdio_Decode_Start (m[NEW].invf, &m[NEW].sbs);
+  BIO_Stdio_Encode_Start (m[MERGE].invf, &m[MERGE].sbs);
+  {
+    int i;
+    for (i = 0; i < m[MERGE].nTerms; i++)
+      {
+	processEntry (i, mergedata[i]);
+      }
+    /* write final .invf.idx pointer */
+    HTONUL(bytes_output);  /* [RPAP - Jan 97: Endian Ordering] */
+    fwrite (&bytes_output, sizeof (u_long), 1, m[MERGE].idx);
+    NTOHUL(bytes_output);  /* [RPAP - Jan 97: Endian Ordering] */
+
+    fprintf (stderr, "Old invf: old only bits: %ld, merged bits: %ld\n",
+	     oldOnlyBits, oldMergeBits);
+  }
+
+  return OK;
+}
+
+
+/*=======================================================================
+ * done_merge_invf(): write headers, close files, append weights to
+ *                     weights file
+ *=======================================================================*/
+int 
+done_merge_invf (void)
+{
+/*** write new lexicon header ***/
+  m[MERGE].idh.dict_size = m[MERGE].nTerms;
+  m[MERGE].idh.lookback = m[OLD].idh.lookback;
+  m[MERGE].idh.input_bytes = m[OLD].idh.input_bytes + m[NEW].idh.input_bytes;
+  m[MERGE].idh.total_bytes = m[OLD].idh.total_bytes + m[NEW].idh.total_bytes;
+  m[MERGE].idh.num_of_docs = m[MERGE].nDocs;
+  m[MERGE].idh.num_of_words = m[OLD].idh.num_of_words + m[NEW].idh.num_of_words;
+  m[MERGE].idh.stemmer_num = m[OLD].idh.stemmer_num;
+  m[MERGE].idh.stem_method = m[OLD].idh.stem_method;
+
+  /* if fastMerge, static num of docs stays the same! */
+  if (fastMerge)
+    m[MERGE].idh.static_num_of_docs = m[OLD].idh.static_num_of_docs;
+  else
+    m[MERGE].idh.static_num_of_docs = m[MERGE].nDocs;
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(m[MERGE].idh.lookback);
+  HTONUL(m[MERGE].idh.dict_size);
+  HTONUL(m[MERGE].idh.total_bytes);
+  HTONUL(m[MERGE].idh.index_string_bytes);
+  HTOND(m[MERGE].idh.input_bytes); /* [RJM 07/97: 4G limit] */
+  HTONUL(m[MERGE].idh.num_of_docs);
+  HTONUL(m[MERGE].idh.static_num_of_docs);
+  HTONUL(m[MERGE].idh.num_of_words);
+  HTONUL(m[MERGE].idh.stemmer_num);
+  HTONUL(m[MERGE].idh.stem_method);
+
+  fseek (m[MERGE].dict, magicsize, 0);
+  fwrite (&(m[MERGE].idh), sizeof (struct invf_dict_header), 1, m[MERGE].dict);
+
+/*** write new inverted file header ***/
+  m[MERGE].ifh.no_of_words = m[MERGE].nTerms;
+  m[MERGE].ifh.no_of_ptrs = m[OLD].ifh.no_of_ptrs + m[NEW].ifh.no_of_ptrs;
+  m[MERGE].ifh.skip_mode = m[OLD].ifh.skip_mode;
+  m[MERGE].ifh.InvfLevel = m[OLD].ifh.InvfLevel;
+
+  /* ifh.params[16] -- I have NO IDEA what these do!!!!! */
+  /* only set if a inverted file is in skipped format, I think */
+  {
+    int i;
+    for (i = 0; i < 16; i++)
+      m[MERGE].ifh.params[i] = m[OLD].ifh.params[i];
+  }
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(m[MERGE].ifh.no_of_words);
+  HTONUL(m[MERGE].ifh.no_of_ptrs);
+  HTONUL(m[MERGE].ifh.skip_mode);
+  {
+    int i;
+    for (i = 0; i < 16; i++)
+      HTONUL(m[MERGE].ifh.params[i]);
+  }
+  HTONUL(m[MERGE].ifh.InvfLevel);
+
+  fseek (m[MERGE].invf, magicsize, 0);	/* go to start of header */
+  fwrite (&(m[MERGE].ifh), sizeof (struct invf_file_header), 1, m[MERGE].invf);
+
+/*** Write new document weights --- leave old weights alone ***/
+  if ((weightOption) && (m[OLD].ifh.InvfLevel >= 2))
+    {
+      FILE *weightfile = open_file (m[MERGE].fname, WEIGHTS_SUFFIX, "r+b",
+				    MAGIC_WGHT, MG_CONTINUE);  /* [RPAP - Feb 97: WIN32 Port] */
+      if (weightfile)
+	{
+	  u_long i;  /* [RPAP - Jan 97: Endian Ordering] */
+
+	  fprintf (stderr, "writing new weights: Nnew = %ld\n", m[NEW].nDocs);
+	  fseek (weightfile, 0, 2);	/* end of file */
+
+	  /* [RPAP - Jan 97: Endian Ordering] */
+	  for (i = 0; i < m[NEW].nDocs; i++)
+	    HTONF(DocWeightBuffer[i]);
+
+	  fwrite ((char *) DocWeightBuffer, sizeof (float), m[NEW].nDocs,
+		  weightfile);
+	}
+      fclose (weightfile);
+    }
+
+
+  /* [RPAP - Feb 97: Level 3 Merge] */
+  if (m[OLD].ifh.InvfLevel == 3)
+    {
+      /* Update paragraph file */
+      int buffer_size = 1000;
+      u_char *buffer;
+      int num;
+
+      if (!(buffer = (u_char *) Xmalloc (buffer_size)))
+	FatalError (1, "Could not allocate memory for paragraph buffer");
+
+      fseek (m[MERGE].para, 0, SEEK_END);
+      while ((num = fread ((char *) buffer, sizeof (*buffer), buffer_size, m[NEW].para)) != 0)
+	fwrite ((char *) buffer, sizeof (*buffer), num, m[MERGE].para);
+
+      Xfree (buffer);
+    }
+
+/*** close files ***/
+  fclose (m[MERGE].invf);
+  fclose (m[OLD].invf);
+  fclose (m[NEW].invf);
+  fclose (m[MERGE].dict);
+  fclose (m[OLD].dict);
+  fclose (m[NEW].dict);
+  fclose (m[MERGE].idx);
+  fclose (m[OLD].idx);
+  fclose (m[NEW].idx);
+
+  /* [RPAP - Feb 97: Level 3 Merge] */
+  if (m[OLD].ifh.InvfLevel == 3)
+    {
+      fclose (m[NEW].para);
+      fclose (m[MERGE].para);
+    }
+
+  return OK;
+}
+
+
+/*=======================================================================
+ * usage()
+ *=======================================================================*/
+void 
+usage (char *progname)
+{
+  fprintf (stderr, "usage:  %s [-s] [-w] [-d directory] -f mg_collection\n",
+	   progname);
+  fprintf (stderr, "  example: %s -s -w -f MERGE/mailfiles\n", progname);
+  exit (1);
+}
+
+
+/*=======================================================================
+ * main()
+ *=======================================================================*/
+int 
+main (int argc, char *argv[])
+{
+  int ch;
+  char *progname;
+  progname = argv[0];
+  m[MERGE].fname[0] = '\0';
+
+  /* message and timing information */
+  msg_prefix = argv[0];
+
+  while ((ch = getopt (argc, argv, "f:d:swh")) != -1)
+    switch (ch)
+      {
+      case 'f':
+	strcpy (m[MERGE].fname, optarg);
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 's':
+	fastMerge = 0;
+	break;
+      case 'w':
+	weightOption = 1;
+	break;
+      case 'h':
+      case '?':
+      default:
+	usage (progname);
+      }
+
+  if (m[MERGE].fname[0] == '\0')
+    usage (progname);
+  strcpy (m[OLD].fname, m[MERGE].fname);
+  strcat (m[OLD].fname, ".old");
+  strcpy (m[NEW].fname, m[MERGE].fname);
+  strcat (m[NEW].fname, ".new");
+
+  GetTime (&start);
+  init_merge_invf ();
+  process_merge_invf ();
+  done_merge_invf ();
+  Message ("%s\n", ElapsedTime (&start, NULL));
+  exit (0);
+}
+
+/*************************************************************************/
+/*   EOF: mg_invf_merge_new.c                                            */
+/*************************************************************************/
Index: /tags/gsdl-21/gsdl/src/mgpp/text/mg_invf_rebuild.1
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/mg_invf_rebuild.1	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/mg_invf_rebuild.1	(revision 699)
@@ -0,0 +1,156 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_invf_rebuild 1 \*(Dt CITRI
+.SH NAME
+mg_invf_rebuild \- Rebuild an mg inverted file with or without skips.
+.SH SYNOPSIS
+.B mg_invf_rebuild
+.RB [ \-h ]
+.if n .ti +9n
+[
+.BR \-0 " |"
+.B \-1
+[
+.BI \-k " num"
+] |
+.B \-2
+[
+.BI \-s " num"
+]
+[
+.BI \-m " num"
+]
+]
+.if n .ti +9n
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mg_invf_rebuild
+builds a new inverted file, with or without skipping, from an old
+inverted file.  This file uses
+.I *.invf.ORG
+and
+.I *.invf.idx.ORG
+as the
+source from which it builds
+.I *.invf
+and
+.IR *.invf.idx .
+If
+.I *.invf.ORG
+or
+.I *.invf.idx.ORG
+do not exist, the program renames
+.I *.invf
+or
+.I *.invf.idx
+to
+.I *.invf.ORG
+or
+.I *.invf.idx.ORG
+as appropriate.  The old inverted file may contain skipping.  This
+means that it is possible to delete the
+.I *.ORG
+files
+after the new inverted file is built.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-m\fP \fInum\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-0
+This generates a non-skipped inverted file.  This option is normally
+only needed if the
+.I *.ORG
+files have been deleted.
+.TP
+.B \-1
+This generates a skipped inverted file.  The
+.BI \-k " num"
+argument specifies the number of pointers hopped over with each skip.
+.TP
+.B \-2
+This option generates a skipped inverted file.  The skipped inverted
+file is built so that it is `optimal' for ranking using a specific
+number of accumulators.  Each term in the inverted file has a
+different skip length.  The arguments
+.BR \-s " and " \-m
+control the sizes of the skips.
+.TP
+.BI \-k " num"
+This specifies the number of pointers that should be hopped over with
+each skip.  This option is only valid if
+.B \-1
+is specified.
+.TP
+.BI \-m " num"
+This specifies the intended number of accumulators that will be used
+when ranking queries are done on the collection.
+.TP
+.BI \-s " num"
+This specifies the minimum size for skips.  If the calculation of the
+optimal skip size results in a number smaller than
+.IR num ,
+the skip size is set to
+.IR num .
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.ORG
+Original inverted file.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.TP
+.B *.invf.idx.ORG
+The original index into the inverted file.
+.TP
+.B *.invf.dict.build
+Compressed stemmed dictionary.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /tags/gsdl-21/gsdl/src/mgpp/text/mg_invf_rebuild.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/mg_invf_rebuild.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/mg_invf_rebuild.cpp	(revision 699)
@@ -0,0 +1,569 @@
+/**************************************************************************
+ *
+ * mg_invf_rebuild.c -- Program to rebuild an inverted file with skipping
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "timing.h"
+#include "bitio_m.h"
+#include "bitio_m_stdio.h"
+#include "bitio_gen.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "invf.h"
+#include "locallib.h"
+#include "words.h"
+#include "mg.h"
+#include "WordData.h"
+
+#define LEN 40
+#define WRD 1
+
+typedef struct invf_info
+  {
+    unsigned long doc_num, count, bits_so_far, count_bits;
+  }
+invf_info;
+
+/*
+   $Log$
+   Revision 1.1  1999/10/11 02:57:58  cs025
+   Base install of MG-PP
+
+   Revision 1.1  1999/08/10 21:18:11  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:46  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:10  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.4  1994/11/29  00:32:03  tes
+   * Committing the new merged files and changes.
+   *
+   * Revision 1.3  1994/10/20  03:56:56  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:51  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+static char pathname[256];
+
+static void process_files (char *filename);
+
+static int k = -1;
+static int mode = 0;
+static int max_nodes = -1;
+static int mins = -1;
+
+void 
+usage (char *pgname)
+{
+  fprintf (stderr, "usage: %s [-f input_file]"
+       "[-d data directory] -0|-1|-2 [-k num] [-m num] [-s num]\n", pgname);
+  printf ("Mode 0: (the default)\n");
+  printf ("\tk, m, and s have no meaning and if specified produce an error\n");
+  printf ("Mode 1:\n");
+  printf ("\tm, and s have no meaning and if specified produce an error\n");
+  printf ("\tk The size of skips\n");
+  printf ("Mode 2:\n");
+  printf ("\tk has no meaning and if specified will produce an error\n");
+  printf ("\tm is the the number of accumulators that will be used for the\n");
+  printf ("\t  ranking. The program builds an inverted file that is \n");
+  printf ("\t  \"optimal\" for that number of accumulators.\n");
+  printf ("\ts is the minimum size for skips.\n");
+  exit (1);
+}
+
+
+
+int main (int argc, char **argv)
+{
+  ProgTime start;
+  char *dir_name, *file_name = "";
+  int ch;
+  msg_prefix = argv[0];
+  dir_name = getenv ("MGDATA");
+  strcpy (pathname, dir_name ? dir_name : ".");
+  opterr = 0;
+  msg_prefix = argv[0];
+  while ((ch = getopt (argc, argv, "012hf:d:k:b:s:m:")) != -1)
+    switch (ch)
+      {
+      case '0':
+	mode = 0;
+	break;
+      case '1':
+	mode = 1;
+	break;
+      case '2':
+	mode = 2;
+	break;
+      case 'f':		/* input file */
+	file_name = optarg;
+	break;
+      case 'd':
+	strcpy (pathname, optarg);
+	break;
+      case 'k':
+	k = atoi (optarg);
+	break;
+      case 'm':
+	max_nodes = atoi (optarg);
+	break;
+      case 's':
+	mins = atoi (optarg);
+	if (mins < 1)
+	  FatalError (1, "The number for the -m option must be greater"
+		      " than or equal to 1");
+	break;
+      case 'h':
+      case '?':
+	usage (argv[0]);
+      }
+
+  if ((mode == 0 && (k != -1 || max_nodes != -1 || mins != -1)) ||
+      (mode == 1 && (max_nodes != -1 || mins != -1)) ||
+      (mode == 2 && (k != -1)))
+    {
+      Message ("Illegal parameters for mode");
+      usage (argv[0]);
+    }
+
+  if (mode == 1 && k == -1)
+    {
+      k = 8;
+      Message ("k is required for mode 1: defaulting k to %d", k);
+    }
+  if (mode == 2 && max_nodes == -1)
+    {
+      max_nodes = 1024;
+      Message ("m is required for mode 2: defaulting m to %d", max_nodes);
+    }
+  if (mode == 2 && mins == -1)
+    {
+      mins = 1;
+      Message ("s is required for mode 2: defaulting s to %d", mins);
+    }
+
+
+  GetTime (&start);
+  process_files (file_name);
+  Message ("%s\n", ElapsedTime (&start, NULL));
+  Message ("**** Don\'t forget to rebuild the stemmed dictionary with mg_invf_dict. ****\n");
+  Message ("**** If the collection was built with stem indexes don\'t forget to     ****\n");
+  Message ("**** rebuild them with mg_stem_idx.                                     ****\n");
+
+  return 0;
+}
+
+
+
+
+
+static void 
+process_files (char *filename)
+{
+  FILE *in, *out, *idx, *odx, *dict;
+  unsigned long magic, outmode, N, in_k, out_k;
+  unsigned long bits_out, bytes_out, i, j;
+  stdio_bitio_state out_buf, in_buf;
+  struct invf_dict_header idh;
+  struct invf_file_header ifh_in, ifh_out;
+  char FName[256];
+
+  outmode = mode;
+
+  /* open .invf.ORG, rename .invf if have to */
+  sprintf (FName, FILE_NAME_FORMAT ".ORG", pathname, filename, INVF_SUFFIX);  /* [RPAP - Feb 97: WIN32 Port] */
+  if (!(in = fopen (FName, "rb")))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      char fname[256];
+      sprintf (fname, FILE_NAME_FORMAT, pathname, filename, INVF_SUFFIX);  /* [RPAP - Feb 97: WIN32 Port] */
+      rename (fname, FName);
+      if (!(in = fopen (FName, "rb")))  /* [RPAP - Feb 97: WIN32 Port] */
+	FatalError (1, "Unable to open \"%s\".\n", FName);
+    }
+  else
+    {
+      char fname[256];
+      sprintf (fname, FILE_NAME_FORMAT, pathname, filename, INVF_SUFFIX);  /* [RPAP - Feb 97: WIN32 Port] */
+      unlink (fname);
+    }
+  Message ("Opening \"%s\"\n", FName);
+
+  /* check the magic number for .invf.ORG */
+  if (fread (&magic, sizeof (magic), 1, in) != 1 || NTOHUL(magic) != MAGIC_INVF)  /* [RPAP - Jan 97: Endian Ordering] */
+    FatalError (1, "Bad magic number in \"%s\".\n", FName);
+
+
+  /* open .invf.idx.ORG, rename .invf.idx if have to */
+  sprintf (FName, FILE_NAME_FORMAT ".ORG", pathname, filename, INVF_IDX_SUFFIX);  /* [RPAP - Feb 97: WIN32 Port] */
+  if (!(idx = fopen (FName, "rb")))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      char fname[256];
+      sprintf (fname, FILE_NAME_FORMAT, pathname, filename, INVF_IDX_SUFFIX);
+      rename (fname, FName);
+      if (!(idx = fopen (FName, "rb")))  /* [RPAP - Feb 97: WIN32 Port] */
+	FatalError (1, "Unable to open \"%s\".\n", FName);
+    }
+  else
+    {
+      char fname[256];
+      sprintf (fname, FILE_NAME_FORMAT, pathname, filename, INVF_IDX_SUFFIX);
+      unlink (fname);
+    }
+  Message ("Opening \"%s\"\n", FName);
+
+  /* check the magic number for .invf.idx.ORG */
+  if (fread (&magic, sizeof (magic), 1, idx) != 1 || NTOHUL(magic) != MAGIC_INVI)  /* [RPAP - Jan 97: Endian Ordering] */
+    FatalError (1, "Bad magic number in \"%s\".\n", FName);
+
+  sprintf (FName, FILE_NAME_FORMAT, pathname, filename, INVF_SUFFIX);  /* [RPAP - Feb 97: WIN32 Port] */
+  Message ("Creating \"%s\"\n", FName);
+  if (!(out = fopen (FName, "wb")))  /* [RPAP - Feb 97: WIN32 Port] */
+    FatalError (1, "Unable to open \"%s\".\n", FName);
+
+  sprintf (FName, FILE_NAME_FORMAT, pathname, filename, INVF_IDX_SUFFIX);  /* [RPAP - Feb 97: WIN32 Port] */
+  Message ("Creating \"%s\"\n", FName);
+  if (!(odx = fopen (FName, "wb")))  /* [RPAP - Feb 97: WIN32 Port] */
+    FatalError (1, "Unable to open \"%s\".\n", FName);
+
+  sprintf (FName, FILE_NAME_FORMAT, pathname, filename, INVF_DICT_SUFFIX);  /* [RPAP - Feb 97: WIN32 Port] */
+  Message ("Opening \"%s\"\n", FName);
+  if (!(dict = fopen (FName, "rb")))  /* [RPAP - Feb 97: WIN32 Port] */
+    FatalError (1, "Unable to open \"%s\".\n", FName);
+
+  if (fread (&magic, sizeof (magic), 1, dict) != 1 || NTOHUL(magic) != MAGIC_STEM_BUILD)  /* [RPAP - Jan 97: Endian Ordering] */
+    FatalError (1, "Bad magic number in \"%s\".\n", FName);
+
+  Invf_Header_read(&idh, dict);
+  /*
+  fread ((char *) &idh, sizeof (idh), 1, dict);
+
+  // [RPAP - Jan 97: Endian Ordering] 
+  NTOHUL(idh.lookback);
+  NTOHUL(idh.dict_size);
+  NTOHUL(idh.total_bytes);
+  NTOHUL(idh.index_string_bytes);
+  NTOHD(idh.input_bytes); // [RJM 07/97: 4G limit] 
+  NTOHUL(idh.num_of_docs);
+  NTOHUL(idh.static_num_of_docs);
+  NTOHUL(idh.num_of_words);
+  NTOHUL(idh.stemmer_num);
+  NTOHUL(idh.stem_method);
+  */
+
+  HTONUL2(MAGIC_INVF, magic);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite ((char *) &magic, sizeof (magic), 1, out);
+
+  fread ((char *) &ifh_in, sizeof (ifh_in), 1, in);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(ifh_in.no_of_words);
+  NTOHUL(ifh_in.no_of_ptrs);
+  NTOHUL(ifh_in.skip_mode);
+  for (i = 0; i < 16; i++)
+    NTOHUL(ifh_in.params[i]);
+  NTOHUL(ifh_in.InvfLevel);
+
+  ifh_out = ifh_in;
+  ifh_out.skip_mode = outmode;
+  bzero ((char *) ifh_out.params, sizeof (ifh_out.params));
+  switch (outmode)
+    {
+    case 0:
+      break;
+    case 1:
+      ifh_out.params[0] = k;
+      break;
+    case 2:
+      ifh_out.params[0] = max_nodes;
+      ifh_out.params[1] = mins;
+      break;
+    }
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(ifh_out.no_of_words);
+  HTONUL(ifh_out.no_of_ptrs);
+  HTONUL(ifh_out.skip_mode);
+  for (i = 0; i < 16; i++)
+    HTONUL(ifh_out.params[i]);
+  HTONUL(ifh_out.InvfLevel);
+
+  fwrite ((char *) &ifh_out, sizeof (ifh_out), 1, out);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(ifh_out.no_of_words);
+  NTOHUL(ifh_out.no_of_ptrs);
+  NTOHUL(ifh_out.skip_mode);
+  for (i = 0; i < 16; i++)
+    NTOHUL(ifh_out.params[i]);
+  NTOHUL(ifh_out.InvfLevel);
+
+  Message ("The file is a level %d inverted file.\n", ifh_in.InvfLevel);
+
+  bits_out = ftell (out) * 8;
+
+
+  HTONUL2(MAGIC_INVI, magic);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite ((char *) &magic, sizeof (magic), 1, odx);
+
+  DECODE_START (in)
+    DECODE_PAUSE (in_buf)
+
+    ENCODE_START (out)
+    ENCODE_PAUSE (out_buf)
+
+    N = idh.num_of_docs;
+
+  for (i = 0; i < ifh_in.no_of_words; i++)
+    {
+      unsigned long blk, p;
+      unsigned long odn_blk = 0, olen_blk = 0;
+      unsigned long idn_blk = 0, ilen_blk = 0;
+      register unsigned long suff;      
+      unsigned long fcnt, wcnt, doc_num, bits_so_far, last_major;
+      WordData word;
+      unsigned long next_mjr_dn, kd;
+      char dummy2[MAXSTEMLEN + 1];
+      invf_info *ii;
+
+      fgetc (dict);
+      suff = fgetc (dict);
+      fread (dummy2, sizeof (u_char), suff, dict);
+      word.read(dict);
+
+      HTONUL2(bits_out >> 3, bytes_out);  /* [RPAP - Jan 97: Endian Ordering] */
+      fwrite ((char *) &bytes_out, sizeof (bytes_out), 1, odx);
+      NTOHUL(bytes_out);  /* [RPAP - Jan 97: Endian Ordering] */
+
+      p = word.documents();
+      blk = BIO_Bblock_Init (idh.static_num_of_docs, p);
+      switch (outmode)
+	{
+	case 1:
+	  {
+	    unsigned long len;
+	    if (p <= ifh_out.params[0])
+	      out_k = 0;
+	    else
+	      {
+		out_k = ifh_out.params[0];
+		len = BIO_Bblock_Bound (N, p);
+		if (ifh_in.InvfLevel >= 2)
+		  len += word.words();
+		odn_blk = BIO_Bblock_Init (idh.num_of_docs, (p + out_k - 1) / out_k);
+		olen_blk = BIO_Bblock_Init (len, (p + out_k - 1) / out_k);
+	      }
+	    break;
+	  }
+	case 2:
+	  {
+	    unsigned long len;
+	    if (p <= mins)
+	      out_k = 0;
+	    else
+	      {
+		out_k = (int) (2 * sqrt ((double) p / max_nodes));
+		if (out_k <= mins)
+		  out_k = mins;
+		len = BIO_Bblock_Bound (N, p);
+		if (ifh_in.InvfLevel >= 2)
+		  len += word.words();
+		odn_blk = BIO_Bblock_Init (idh.num_of_docs,
+					   (p + out_k - 1) / out_k);
+		olen_blk = BIO_Bblock_Init (len, (p + out_k - 1) / out_k);
+	      }
+	    break;
+	  }
+	default:
+	  out_k = 0;
+	}
+
+      switch (ifh_in.skip_mode)
+	{
+	case 1:
+	  {
+	    unsigned long len;
+	    if (p <= ifh_in.params[0])
+	      in_k = 0;
+	    else
+	      {
+		in_k = ifh_in.params[0];
+		len = BIO_Bblock_Bound (N, p);
+		if (ifh_in.InvfLevel >= 2)
+		  len += word.words();
+		idn_blk = BIO_Bblock_Init (idh.num_of_docs, (p + in_k - 1) / in_k);
+		ilen_blk = BIO_Bblock_Init (len, (p + in_k - 1) / in_k);
+	      }
+	    break;
+	  }
+	case 2:
+	  {
+	    unsigned long len;
+	    if (p <= ifh_in.params[1])
+	      {
+		in_k = 0;
+	      }
+	    else
+	      {
+		in_k = (int) (2 * sqrt ((double) p / ifh_in.params[0]));
+		if (in_k <= ifh_in.params[1])
+		  in_k = ifh_in.params[1];
+		len = BIO_Bblock_Bound (N, p);
+		if (ifh_in.InvfLevel >= 2)
+		  len += word.words();
+		idn_blk = BIO_Bblock_Init (idh.num_of_docs,
+					   (p + in_k - 1) / in_k);
+		ilen_blk = BIO_Bblock_Init (len, (p + in_k - 1) / in_k);
+	      }
+	    break;
+	  }
+	default:
+	  in_k = 0;
+	}
+
+      if (!(ii = (invf_info *) Xmalloc (sizeof (invf_info) * p)))
+	FatalError (1, "Unable to allocate memory for \"ii\"\n");
+
+      doc_num = bits_so_far = 0;
+      next_mjr_dn = 0;
+      kd = 0;
+      DECODE_CONTINUE (in_buf)
+	for (j = 0; j < p; j++, kd++)
+	{
+	  unsigned long doc_diff, count = 0;
+	  if (kd == in_k)
+	    kd = 0;
+	  if (in_k && kd == 0 && j + in_k < p)
+	    {
+	      int temp;
+	      next_mjr_dn = buffer.bblock_decode(idn_blk, NULL);
+	      next_mjr_dn += doc_num;
+	      temp = buffer.bblock_decode(ilen_blk, NULL);
+	    }
+	  ii[j].bits_so_far = bits_so_far;
+	  if (in_k && kd == in_k - 1 && j != p - 1)
+	    {
+	      int count;
+	      count = bitio_buffer::bblock_length(next_mjr_dn - doc_num, blk);
+	      bits_so_far += count;
+	      doc_num = next_mjr_dn;
+	    }
+	  else
+	    {
+	      doc_diff = buffer.bblock_decode(blk, &bits_so_far);
+	      doc_num += doc_diff;
+	    }
+	  ii[j].doc_num = doc_num;
+	  if (ifh_in.InvfLevel >= 2)
+	    {
+	      unsigned long count_bits = 0;
+	      count = buffer.gamma_decode (&count_bits);
+	      ii[j].count_bits = count_bits;
+	      bits_so_far += count_bits;
+	      ii[j].count = count;
+	    }
+	}
+
+      /* read till a byte boundary */
+      while (buffer.bitOffset())
+      //      while (__btg)
+	{
+	  buffer.bit();
+	  bits_so_far++;
+	}
+
+      DECODE_PAUSE (in_buf)
+
+	doc_num = bits_so_far = 0;
+      last_major = 0;
+      kd = 0;
+      ENCODE_CONTINUE (out_buf)
+	for (j = 0; j < p; j++, kd++)
+	{
+	  if (kd == out_k)
+	    kd = 0;
+	  if (out_k && kd == 0)
+	    {
+	      if (j + out_k < p)
+		{
+		  int num = ii[j + out_k - 1].doc_num - last_major;
+		  buffer.bblock_encode (num, odn_blk, &bits_out);
+		  last_major = ii[j + out_k - 1].doc_num;
+
+		  num = ii[j + out_k - 1].bits_so_far + ii[j + out_k - 1].count_bits -
+		    bits_so_far;
+		  buffer.bblock_encode (num, olen_blk, &bits_out);
+		  bits_so_far = ii[j + out_k].bits_so_far;
+		}
+	    }
+	  if (!(out_k && kd == out_k - 1 && j != p - 1))
+	    buffer.bblock_encode (ii[j].doc_num - doc_num, blk, &bits_out);
+	  doc_num = ii[j].doc_num;
+	  if (ifh_in.InvfLevel >= 2)
+	    buffer.gamma_encode (ii[j].count, &bits_out);
+	}
+
+      /* write till a byte boundary */
+      //while (buffer.bitOffset() != 8)
+        while (buffer.bitOffset() != 8)
+	{
+	  buffer.encodeBit (0);
+	  bits_out++;
+	}
+      ENCODE_PAUSE (out_buf)
+
+	Xfree (ii);
+
+    }
+  ENCODE_CONTINUE (out_buf)
+    ENCODE_DONE
+
+    HTONUL2(bits_out >> 3, bytes_out);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite ((char *) &bytes_out, sizeof (bytes_out), 1, odx);
+  NTOHUL(bytes_out);
+
+  fclose (idx);
+  fclose (odx);
+  fclose (in);
+  fclose (out);
+  fclose (dict);
+
+}
Index: /tags/gsdl-21/gsdl/src/mgpp/text/mg_merge.h
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/mg_merge.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/mg_merge.h	(revision 699)
@@ -0,0 +1,37 @@
+/**************************************************************************
+ *
+ * mgmerge.h --- mgmerge #defines, etc
+ * Copyright (C) 1995  Shane Hudson (shane@cosc.canterbury.ac.nz)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ * Last edtited: November 12 1994
+ *
+ **************************************************************************/
+
+#ifndef MGMERGE_H
+#define MGMERGE_H
+
+typedef unsigned char byte;
+
+#define OLD 0
+#define NEW 1
+#define MERGE 2
+
+#define OK 0
+#define ERROR_MGMERGE 1
+
+#endif
Index: /tags/gsdl-21/gsdl/src/mgpp/text/mg_text_estimate.1
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/mg_text_estimate.1	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/mg_text_estimate.1	(revision 699)
@@ -0,0 +1,112 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_text_estimate 1 \*(Dt CITRI
+.SH NAME
+mg_text_estimate \- Estimate the size of the compressed text based on the text statistics and a compression dictionary.
+.SH SYNOPSIS
+.B mg_text_estimate
+[
+.B \-h
+]
+[
+.BR \-H " |"
+.BR \-B " |"
+.BR \-D " |"
+.BR \-Y " |"
+.B \-M
+]
+.if n .ti +9n
+.I stats-dict
+.I compression-dict
+.SH DESCRIPTION
+This program estimates the size of the compressed text that would be
+generated if text with certain statistics were compressed with a
+particular dictionary.  This program has no real use in the
+.BR mg (1)
+system, except for experimenting with the
+.BR mg_compression_dict (1)
+program.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-m\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-H
+This specifies that novel words will be coded character by character
+using Huffman codes.
+.TP
+.B \-B
+This specifies that the size of an auxiliary dictionary will be
+estimated.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using binary codes.  The binary code represents their occurrence
+position in the auxiliary dictionary.
+.TP
+.B \-D
+This specifies that the size of an auxiliary dictionary will be
+estimated.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using delta codes.  The delta code represents their occurrence position
+in the auxiliary dictionary.
+.TP
+.B \-Y
+This specifies that the size of an auxiliary dictionary will be
+estimated.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using a combination of gamma and binary codes.  The code represents
+their occurrence position in the auxiliary dictionary.  This generally
+produces better compression than
+.B \-B
+or
+.BR \-D .
+.TP
+.B \-M
+This specifies that the size of an auxiliary dictionary will be
+estimated.  Each novel word found will be placed at the end of the
+auxiliary dictionary.  Novel words will be coded in the compressed text
+using a combination of gamma and binary codes.  The code represents
+their occurrence position in the auxiliary dictionary.  This method is
+adaptive within documents, and generally produces better compression
+than
+.BR \-B ,
+.B \-D
+or
+.BR \-Y .
+.SH FILES
+.TP 20
+.B *.text.stats
+Statistics about the text.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /tags/gsdl-21/gsdl/src/mgpp/text/mg_text_estimate.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/mg_text_estimate.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/mg_text_estimate.cpp	(revision 699)
@@ -0,0 +1,573 @@
+/**************************************************************************
+ *
+ * mg_text_estimate.c -- Program to estimate the compressed text size
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "local_strings.h"
+#include "bitio_gen.h"
+#include "bitio_m.h"
+#include "bitio_m_stdio.h"
+#include "timing.h"
+#include "mgheap.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "locallib.h"
+#include "invf.h"
+#include "text.h"
+#include "words.h"
+#include "mg.h"
+#include "comp_dict.h"
+#include "hash.h"
+
+
+
+
+
+/*
+   $Log$
+   Revision 1.1  1999/10/11 02:58:04  cs025
+   Base install of MG-PP
+
+   Revision 1.1  1999/08/10 21:18:14  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.4  1999/01/15 03:05:51  rjmcnab
+
+   Renamed lib/heap to be lib/mgheap (it was causing some problems with
+   some versions of the STL libraries which contained a heap.h)
+
+   Revision 1.3  1998/12/17 09:12:53  rjmcnab
+
+   Altered mg to process utf-8 encoded Unicode. The main changes
+   are in the parsing of the input, the casefolding, and the stemming.
+
+   Revision 1.2  1998/11/25 07:55:48  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:19  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:56:59  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:54  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+typedef struct stats_t
+  {
+    u_long freq, occur;
+    u_long len;
+  }
+stats_t;
+
+static u_long huff_count = 0;
+static u_long escape_count = 0;
+
+
+char mode = 'H';
+
+/* [RJM 07/97: 4G limit] */
+static double ReadInWordsStandard (char *name, double *num_bytes);
+static double ReadInWordsSpecial (char *name, double *num_bytes,
+				  double *num_extra_bytes,
+				  u_long *aux_compressed);
+
+
+
+
+int main (int argc, char **argv)
+{
+  char *stats_dict = NULL, *comp_dict = NULL;
+  ProgTime StartTime;
+  double num_bytes, num_extra_bytes = 0; /* [RJM 07/97: 4G limit] */
+  u_long aux_compressed;
+  int ch;
+  double bytes;
+  opterr = 0;
+  msg_prefix = argv[0];
+  while ((ch = getopt (argc, argv, "BMDYHh")) != -1)
+    switch (ch)
+      {
+      case 'B':
+	Message ("The compressed text size for this method cannot be\n"
+		 "estimated exactly, so an upper bound will be calculated.");
+	mode = ch;
+	break;
+      case 'M':
+	Message ("The compressed text size for this method cannot be\n"
+		 "estimated exactly, so an upper bound will be calculated.");
+	mode = 'Y';
+	break;
+      case 'D':
+      case 'Y':
+      case 'H':
+	mode = ch;
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-h] [B|M|D|Y|H] stats_dict comp_dict\n",
+		 argv[0]);
+	exit (1);
+      }
+
+  if (optind < argc)
+    stats_dict = argv[optind++];
+  else
+    stats_dict = NULL;
+  if (optind < argc)
+    comp_dict = argv[optind++];
+  else
+    {
+      fprintf (stderr, "usage: %s [-h] [B|M|D|Y|H] stats_dict comp_dict\n",
+	       argv[0]);
+      exit (1);
+    }
+
+
+
+  GetTime (&StartTime);
+
+  if (LoadCompressionDictionary (comp_dict) == COMPERROR)
+    exit (1);
+
+  if (mode == 'H')
+    bytes = ReadInWordsStandard (stats_dict, &num_bytes);
+  else
+    bytes = ReadInWordsSpecial (stats_dict, &num_bytes, &num_extra_bytes,
+				&aux_compressed);
+
+
+  Message ("Compressed Text %.2f Mb ( %.0f bytes   %0.3f %%)",
+	   bytes / 1024.0 / 1024, bytes, bytes * 100 / num_bytes);
+  Message ("Words portion of the dictionary %.2f Mb ( %.0f bytes   %0.3f %%)",
+	   Words_disk / 1024.0 / 1024, Words_disk * 1.0,
+	   (Words_disk * 100.0) / num_bytes);
+  if (mode == 'H')
+    Message ("Huffman info for chars in dictionary %.2f Mb ( %.0f bytes   %0.3f %%)",
+	     Chars_disk / 1024.0 / 1024, Chars_disk * 1.0,
+	     (Chars_disk * 100.0) / num_bytes);
+  else
+    {
+      Message ("Aux dictionary (Uncompressed) %.2f Mb ( %.0f bytes   %0.3f %%)",
+	       num_extra_bytes / 1024.0 / 1024, num_extra_bytes * 1.0,
+	       (num_extra_bytes * 100.0) / num_bytes);
+      Message ("Aux dictionary (Compressed)   %.2f Mb ( %.0f bytes   %0.3f %%)",
+	       aux_compressed / 1024.0 / 1024, aux_compressed * 1.0,
+	       (aux_compressed * 100.0) / num_bytes);
+    }
+  Message ("Words and non-words in the text %u", huff_count + escape_count);
+  Message ("Words and non-words coded by escapes %u ( %0.3f %%)", escape_count,
+	   escape_count * 100.0 / huff_count);
+  Message ("%s", ElapsedTime (&StartTime, NULL));
+  return 0;
+}
+
+
+
+
+
+static double 
+ReadInWordsStandard (char *name, double *num_bytes)
+{
+  compression_stats_header csh;
+  FILE *f;
+  u_long which;
+  double bytes, bits = 0;
+
+  f = open_named_file (name, "rb", MAGIC_STATS_DICT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+
+  fread (&csh, sizeof (csh), 1, f);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(csh.num_docs);
+  NTOHD(csh.num_bytes); /* [RJM 07/97: 4G limit] */
+
+  for (which = 0; which < 2; which++)
+    {
+      frags_stats_header fsh;
+      int j;
+      fread (&fsh, sizeof (fsh), 1, f);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fsh.num_frags);
+      NTOHUL(fsh.mem_for_frags);
+
+      for (j = 0; j < fsh.num_frags; j++)
+	{
+	  int len, freq, res, occur_num;
+	  u_char Word[16];
+	  fread (&freq, sizeof (freq), 1, f);
+	  fread (&occur_num, sizeof (occur_num), 1, f);
+	  len = fgetc (f);
+
+	  /* [RPAP - Jan 97: Endian Ordering] */
+	  NTOHSI(freq);
+	  NTOHSI(occur_num);
+
+	  Word[0] = len;
+	  fread (Word + 1, len, 1, f);
+
+	  /* Search the hash table for Word */
+	  if (ht[which])
+	    {
+	      register unsigned long hashval, step;
+	      register int tsize = ht[which]->size;
+	      register u_char **wptr;
+	      HASH (hashval, step, Word, tsize);
+	      for (;;)
+		{
+		  register u_char *s1;
+		  register u_char *s2;
+		  register int len;
+		  wptr = ht[which]->table[hashval];
+		  if (wptr == NULL)
+		    {
+		      res = COMPERROR;
+		      break;
+		    }
+
+		  /* Compare the words */
+		  s1 = Word;
+		  s2 = *wptr;
+		  len = *s1 + 1;
+		  for (; len; len--)
+		    if (*s1++ != *s2++)
+		      break;
+
+		  if (len)
+		    {
+		      hashval += step;
+		      if (hashval >= tsize)
+			hashval -= tsize;
+		    }
+		  else
+		    {
+		      res = ht[which]->table[hashval] - ht[which]->words;
+		      break;
+		    }
+		}
+	    }
+	  else
+	    res = COMPERROR;
+
+	  /* Check that the word was found in the dictionary */
+	  if (res == COMPERROR)
+	    {
+	      escape_count += freq;
+	      if (cdh.dict_type == MG_COMPLETE_DICTIONARY)
+		Message ("Unknown word \"%.*s\"\n", *Word, Word + 1);
+	      if (cdh.dict_type == MG_PARTIAL_DICTIONARY ||
+		  cdh.dict_type == MG_SEED_DICTIONARY)
+		{
+		  unsigned i;
+		  if (ht[which])
+		    {
+		      res = ht[which]->hd->num_codes - 1;
+		      bits += ht[which]->hd->clens[res] * freq;
+		    }
+		  bits += lens_huff[which].clens[Word[0]] * freq;
+		  for (i = 0; i < Word[0]; i++)
+		    bits += char_huff[which].clens[Word[i + 1]] * freq;
+		}
+	    }
+	  else
+	    {
+	      bits += ht[which]->hd->clens[res] * freq;
+	      huff_count += freq;
+	    }
+	}
+    }
+  fclose (f);
+
+
+  /*
+   * The 5.5 bits added on per document consist of
+   *   1 bit for the word/non-word flag at the start of the document.
+   *   1 bit on average for document termination.
+   *   3.5 bits on average for padding to a byte boundary.
+   */
+  bits += 5.5 * csh.num_docs;
+
+  bytes = bits / 8 + sizeof (compressed_text_header) + sizeof (u_long);
+
+  *num_bytes = csh.num_bytes;
+
+  return bytes;
+}
+
+
+static int 
+stats_comp (const void *a, const void *b)
+{
+  return ((stats_t *) a)->occur - ((stats_t *) b)->occur;
+}
+
+
+
+
+
+static double 
+ReadInWordsSpecial (char *name, double *num_bytes,
+		    double *num_extra_bytes,
+		    u_long *aux_compressed)
+{
+  compression_stats_header csh;
+  FILE *f;
+  u_long magic, which;
+  double bytes, bits = 0;
+  double size[2];
+
+  if (!(f = fopen (name, "rb")))  /* [RPAP - Feb 97: WIN32 Port] */
+    FatalError (1, "Unable to open file \"%s\"\n", name);
+
+  if (fread (&magic, sizeof (magic), 1, f) != 1)
+    FatalError (1, "Unable to read magic number");
+
+  NTOHUL(magic);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  if (magic != MAGIC_STATS_DICT)
+    FatalError (1, "Bad magic number");
+
+  fread (&csh, sizeof (csh), 1, f);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(csh.num_docs);
+  NTOHD(csh.num_bytes); /* [RJM 07/97: 4G limit] */
+
+  for (which = 0; which < 2; which++)
+    {
+      stats_t *stats;
+      frags_stats_header fsh;
+      int j, num = 0;
+      long lens[16], chars[256];
+
+      bzero ((char *) lens, sizeof (lens));
+      bzero ((char *) chars, sizeof (chars));
+
+      fread (&fsh, sizeof (fsh), 1, f);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fsh.num_frags);
+      NTOHUL(fsh.mem_for_frags);
+
+      stats = (stats_t *) Xmalloc (fsh.num_frags * sizeof (*stats));
+      if (!stats)
+	FatalError (1, "Unable to allocate memory for stats");
+      for (j = 0; j < fsh.num_frags; j++)
+	{
+	  int len, res;
+	  u_char Word[16];
+	  fread (&stats[num].freq, sizeof (stats[num].freq), 1, f);
+	  fread (&stats[num].occur, sizeof (stats[num].occur), 1, f);
+	  len = fgetc (f);
+
+	  /* [RPAP - Jan 97: Endian Ordering] */
+	  NTOHSI(stats[num].freq);
+	  NTOHSI(stats[num].occur);
+
+	  stats[num].len = len + 1;
+
+	  Word[0] = len;
+	  fread (Word + 1, len, 1, f);
+
+	  /* Search the hash table for Word */
+	  if (ht[which])
+	    {
+	      register unsigned long hashval, step;
+	      register int tsize = ht[which]->size;
+	      register u_char **wptr;
+	      HASH (hashval, step, Word, tsize);
+	      for (;;)
+		{
+		  register u_char *s1;
+		  register u_char *s2;
+		  register int len;
+		  wptr = ht[which]->table[hashval];
+		  if (wptr == NULL)
+		    {
+		      res = COMPERROR;
+		      break;
+		    }
+
+		  /* Compare the words */
+		  s1 = Word;
+		  s2 = *wptr;
+		  len = *s1 + 1;
+		  for (; len; len--)
+		    if (*s1++ != *s2++)
+		      break;
+
+		  if (len)
+		    {
+		      hashval += step;
+		      if (hashval >= tsize)
+			hashval -= tsize;
+		    }
+		  else
+		    {
+		      res = ht[which]->table[hashval] - ht[which]->words;
+		      break;
+		    }
+		}
+	    }
+	  else
+	    res = COMPERROR;
+
+	  assert (stats[num].freq > 0);
+
+	  /* Check that the word was found in the dictionary */
+	  if (res == COMPERROR)
+	    {
+	      if (cdh.dict_type == MG_COMPLETE_DICTIONARY)
+		Message ("Unknown word \"%.*s\"\n", *Word, Word + 1);
+	      if ((cdh.dict_type == MG_PARTIAL_DICTIONARY ||
+		   cdh.dict_type == MG_SEED_DICTIONARY) &&
+		  ht[which])
+		{
+		  int k;
+		  res = ht[which]->hd->num_codes - 1;
+		  bits += ht[which]->hd->clens[res] * stats[num].freq;
+		  lens[Word[0]]++;
+		  for (k = 0; k < (int) Word[0]; k++)
+		    chars[Word[k + 1]]++;
+		}
+	      num++;
+	    }
+	  else
+	    {
+	      bits += ht[which]->hd->clens[res] * stats[num].freq;
+	    }
+	}
+
+      {
+	int j, num_buckets = 0;
+	int start[10], end[10];
+	long flens[16], fchars[256];
+
+	for (j = 0; j < 256; j++)
+	  if (!chars[j] && PESINAWORD (j) == which)
+	    fchars[j] = 1;
+	  else
+	    fchars[j] = chars[j];
+	for (j = 0; j < 16; j++)
+	  if (!lens[j])
+	    flens[j] = 1;
+	  else
+	    flens[j] = lens[j];
+
+	size[which] = Calculate_Huffman_Size (16, flens, lens) +
+	  Calculate_Huffman_Size (256, fchars, chars);
+
+
+	qsort (stats, num, sizeof (*stats), stats_comp);
+
+
+	if (mode == 'Y')
+	  {
+	    int k;
+	    num_buckets = 1;
+	    start[0] = 0;
+	    end[0] = cdh.num_words[which] - 1;
+	    Message ("There are %d novel %swords", num, which ? "" : "non-");
+	    while (end[num_buckets - 1] < num)
+	      {
+		start[num_buckets] = end[num_buckets - 1] + 1;
+		end[num_buckets] = start[num_buckets] +
+		  (end[num_buckets - 1] - start[num_buckets - 1]) * 2;
+		num_buckets++;
+	      }
+	    for (k = 0; k < num_buckets; k++)
+	      Message ("Bucket %d   %d <= x <= %d\n", k + 1, start[k], end[k]);
+	  }
+
+	for (j = 0; j < num; j++)
+	  {
+	    (*num_extra_bytes) += stats[j].len;
+	    switch (mode)
+	      {
+	      case 'D':
+		bits += BIO_Delta_Length (j + 1) * stats[j].freq;
+		break;
+	      case 'B':
+		bits += BIO_Binary_Length (j + 1, num) * stats[j].freq;
+		break;
+	      case 'Y':
+		{
+		  int k = 0;
+		  while (j > end[k])
+		    k++;
+		  if (!(j - start[k] + 1 >= 1 &&
+			j - start[k] + 1 <= end[k] - start[k] + 1))
+		    {
+		      Message ("k = %d, j = %d", k, j);
+		      assert (j - start[k] + 1 >= 1 &&
+			      j - start[k] + 1 <= end[k] - start[k] + 1);
+		    }
+		  else
+		    {
+		      bits += (BIO_Gamma_Length (k + 1) +
+			       BIO_Binary_Length (j - start[k] + 1,
+						  end[k] - start[k] + 1)) *
+			stats[j].freq;
+		    }
+		}
+	      }
+	  }
+	if (mode == 'B' && num)
+	  bits += BIO_Delta_Length (num) * csh.num_docs;
+      }
+      Xfree (stats);
+    }
+  fclose (f);
+
+  *aux_compressed = (long unsigned int) ((size[0] + size[1]) / 8);
+
+  /*
+   * The 5.5 bits added on per document consist of
+   *   1 bit for the word/non-word flag at the start of the document.
+   *   1 bit on average for document termination.
+   *   3.5 bits on average for padding to a byte boundary.
+   */
+  bits += 5.5 * csh.num_docs;
+
+  bytes = bits / 8 + sizeof (compressed_text_header) + sizeof (u_long);
+
+  *num_bytes = csh.num_bytes;
+
+  return bytes;
+}
Index: /tags/gsdl-21/gsdl/src/mgpp/text/mg_text_merge.1
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/mg_text_merge.1	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/mg_text_merge.1	(revision 699)
@@ -0,0 +1,76 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mg_text_merge 1 \*(Dt CITRI
+.SH NAME
+mg_text_merge \- Append one compressed text file to another
+.SH SYNOPSIS
+.B mg_text_merge
+[
+.B \-h
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+This program takes two
+.BR mg (1)
+database inverted files and stemmed
+dictionaries, named "old" and "new" respectively, and creates
+a merged stemmed dictionary and inverted file.
+It is part of the
+.BR mgmerge (1)
+addition to the
+.BR mg (1)
+package.
+The two compressed text files are
+.IR "name" .old.text
+and
+.IR "name" .new.text.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.old.text
+The old compressed text.
+.TP
+.B *.new.text
+The new compressed text that will be appended
+.TP
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mgmerge (1),
+.BR mg_get_merge (1),
+.BR mg_invf_merge (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mgquery (1).
Index: /tags/gsdl-21/gsdl/src/mgpp/text/mg_text_merge.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/mg_text_merge.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/mg_text_merge.cpp	(revision 699)
@@ -0,0 +1,249 @@
+/**************************************************************************
+ *
+ * mg_text_merge.c --- merge  *.text, *.text.idx files
+ *                     part of the mgmerge utility
+ * Copyright (C) 1995 Shane Hudson (shane@cosc.canterbury.ac.nz)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ * Last edited:  November 11 1994
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "messages.h"
+#include "timing.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "locallib.h"
+#include "mg.h"
+#include "mg_merge.h"
+#include "mg_files.h"
+#include "text.h"
+
+
+/**** GLOBALS ****/
+FILE *text[3], *idx[3];
+
+typedef char FileName[256];
+FileName old_name, new_name, merge_name;
+
+long magicsize;			/* == where the header in a file begins */
+
+compressed_text_header cth[3];
+
+/*=======================================================================
+ * init_merge_text(): open files, set up global variables, etc
+ *=======================================================================*/
+int 
+init_merge_text ()
+{
+
+  /* open .text files */
+  text[OLD] = open_file (old_name, TEXT_SUFFIX, "r+b",
+			 MAGIC_TEXT, MG_ABORT);  /* [RPAP - Feb 97: WIN32 Port] */
+  magicsize = ftell (text[OLD]);
+  fread (&cth[OLD], sizeof (cth[OLD]), 1, text[OLD]);
+
+  text[NEW] = open_file (new_name, TEXT_SUFFIX, "rb+",
+			 MAGIC_TEXT, MG_ABORT);
+  fread (&cth[NEW], sizeof (cth[NEW]), 1, text[NEW]);
+
+  /* open .text.idx files */
+  idx[OLD] = open_file (old_name, TEXT_IDX_SUFFIX, "rb+",
+			MAGIC_TEXI, MG_ABORT);
+  fread (&cth[OLD], sizeof (cth[OLD]), 1, idx[OLD]);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(cth[OLD].num_of_docs);
+  NTOHD(cth[OLD].num_of_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(cth[OLD].num_of_words);
+  NTOHUL(cth[OLD].length_of_longest_doc);
+  NTOHD(cth[OLD].ratio);
+
+  idx[NEW] = open_file (new_name, TEXT_IDX_SUFFIX, "rb+",
+			MAGIC_TEXI, MG_ABORT);
+  fread (&cth[NEW], sizeof (cth[NEW]), 1, idx[NEW]);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(cth[NEW].num_of_docs);
+  NTOHD(cth[NEW].num_of_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(cth[NEW].num_of_words);
+  NTOHUL(cth[NEW].length_of_longest_doc);
+  NTOHD(cth[NEW].ratio);
+
+  idx[MERGE] = create_file (merge_name, TEXT_IDX_SUFFIX, "wb",
+			    MAGIC_TEXI, MG_ABORT);
+  return OK;
+}
+
+
+/*=======================================================================
+ * process_merge_text(): merge the files
+ *=======================================================================*/
+int 
+process_merge_text (void)
+{
+  int i;
+  u_long data, offset;
+  byte c;
+
+  /* update and write merged header to .text and .text.idx files */
+  /* they have the exact same header */
+  cth[MERGE].num_of_docs = cth[OLD].num_of_docs
+    + cth[NEW].num_of_docs;
+  cth[MERGE].num_of_bytes = cth[OLD].num_of_bytes
+    + cth[NEW].num_of_bytes;
+  cth[MERGE].num_of_words = cth[OLD].num_of_words;
+  cth[MERGE].length_of_longest_doc =
+    (cth[OLD].length_of_longest_doc > cth[NEW].length_of_longest_doc
+     ? cth[OLD].length_of_longest_doc
+     : cth[NEW].length_of_longest_doc);
+  cth[MERGE].ratio = ((cth[OLD].num_of_bytes * cth[OLD].ratio) +
+		      (cth[NEW].num_of_bytes * cth[NEW].ratio))
+    / cth[MERGE].num_of_bytes;
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  HTONUL(cth[MERGE].num_of_docs);
+  HTOND(cth[MERGE].num_of_bytes); /* [RJM 07/97: 4G limit] */
+  HTONUL(cth[MERGE].num_of_words);
+  HTONUL(cth[MERGE].length_of_longest_doc);
+  HTOND(cth[MERGE].ratio);
+
+  fwrite (&cth[MERGE], sizeof (cth[MERGE]), 1, idx[MERGE]);
+  fseek (text[OLD], magicsize, 0);
+  fwrite (&cth[MERGE], sizeof (cth[MERGE]), 1, text[OLD]);
+
+  /*
+   *  Update *.text.idx:  need to know where each new doc starts
+   *  in the appended .text file
+   */
+  for (i = 0; i < cth[OLD].num_of_docs; i++)
+    {
+      fread (&data, sizeof (u_long), 1, idx[OLD]);
+      fwrite (&data, sizeof (u_long), 1, idx[MERGE]);
+    }
+
+  /* offset is the amount to add to each entry from idx[NEW] */
+  fread (&offset, sizeof (u_long), 1, idx[OLD]);
+  NTOHUL(offset);  /* [RPAP - Jan 97: Endian Ordering] */
+  offset -= (4 + sizeof (cth[OLD]));	/* 4 for the magic number */
+
+  for (i = 0; i < cth[NEW].num_of_docs; i++)
+    {
+      fread (&data, sizeof (u_long), 1, idx[NEW]);
+      NTOHUL(data);  /* [RPAP - Jan 97: Endian Ordering] */
+      data += offset;
+      HTONUL(data);  /* [RPAP - Jan 97: Endian Ordering] */
+      fwrite (&data, sizeof (u_long), 1, idx[MERGE]);
+    }
+  /* write last u_long in idx[MERGE] (= length of file) */
+  fread (&data, sizeof (u_long), 1, idx[NEW]);
+  NTOHUL(data);  /* [RPAP - Jan 97: Endian Ordering] */
+  data += offset;
+  HTONUL(data);  /* [RPAP - Jan 97: Endian Ordering] */
+  fwrite (&data, sizeof (u_long), 1, idx[MERGE]);
+
+/******* update .text *******/
+  /* simply cat's the files together, except for the headers 
+   * and magic numbers, of course
+   */
+  fseek (text[OLD], 0L, 2);
+  while (!feof (text[NEW]))
+    {
+      fread (&c, sizeof (c), 1, text[NEW]);
+      if (!feof (text[NEW]))
+	fwrite (&c, sizeof (c), 1, text[OLD]);
+    }
+
+  return OK;
+}
+
+
+
+/*=======================================================================
+ * done_merge_text(): close files.
+ *=======================================================================*/
+int 
+done_merge_text (void)
+{
+  fclose (idx[MERGE]);
+  fclose (text[OLD]);
+  fclose (idx[OLD]);
+  fclose (text[NEW]);
+  fclose (idx[NEW]);
+
+  fprintf (stderr, "mg_text_merge: %ld documents added to %s\n",
+	   cth[NEW].num_of_docs, merge_name);
+
+  return OK;
+}
+
+
+/*=======================================================================
+ * usage()
+ *=======================================================================*/
+void 
+usage (char *progname)
+{
+  fprintf (stderr, "usage:  %s -f collection_name\n", progname);
+  exit (1);
+}
+
+
+/*=======================================================================
+ * main()
+ *=======================================================================*/
+int main (int argc, char *argv[])
+{
+  char *progname;
+  ProgTime start;
+  int ch;			/* for command line processing */
+
+  progname = argv[0];
+  msg_prefix = argv[0];
+  merge_name[0] = '\0';
+
+  while ((ch = getopt (argc, argv, "f:d:h")) != -1)
+    switch (ch)
+      {
+      case 'f':
+	strcpy (merge_name, optarg);
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 'h':
+      case '?':
+      default:
+	usage (progname);
+      }
+
+  if (merge_name[0] == '\0')
+    usage (progname);
+  strcpy (old_name, merge_name);
+  strcat (old_name, ".old");
+  strcpy (new_name, merge_name);
+  strcat (new_name, ".new");
+
+  GetTime (&start);
+  init_merge_text ();
+  process_merge_text ();
+  done_merge_text ();
+  Message ("%s\n", ElapsedTime (&start, NULL));
+  return 0;
+}
Index: /tags/gsdl-21/gsdl/src/mgpp/text/mgdictlist.1
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/mgdictlist.1	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/mgdictlist.1	(revision 699)
@@ -0,0 +1,72 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgdictlist 1 \*(Dt CITRI
+.SH NAME
+mgdictlist \- list the contents of a dictionary
+.SH SYNOPSIS
+.B mgdictlist
+[
+.B \-q
+]
+.I filename
+.SH DESCRIPTION
+.B mgdictlist
+lists the contents of either a compressed compression dictionary, a
+text statistics file, or a compressed stemmed dictionary.  Unlike all
+the other programs in the
+.BR mg (1)
+suite, it is necessary to specify the complete pathname of the
+dictionary to list.
+.SH OPTIONS
+The
+.I filename
+must be last.
+.TP "\w'\fIfilename\fP'u+2n"
+.B \-q
+Generate an abbreviated output (machine-readable rather than
+human-readable).
+.TP
+.I filename
+This is the name of the dictionary to list.
+.B mgdictlist
+automatically detects whether the file is a compressed compression
+dictionary, text statistics file, or a compressed stemmed dictionary.
+.SH FILES
+.TP 20
+.B *.text.dict
+Text statistics file.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.invf.build
+Compressed stemmed dictionary.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /tags/gsdl-21/gsdl/src/mgpp/text/mgdictlist.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/mgdictlist.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/mgdictlist.cpp	(revision 699)
@@ -0,0 +1,394 @@
+/**************************************************************************
+ *
+ * mgdictlist.c -- Program to list a dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "messages.h"
+#include "memlib.h"
+#include "local_strings.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "text.h"
+#include "invf.h"
+#include "locallib.h"
+#include "words.h"
+
+/*
+   $Log$
+   Revision 1.1  1999/10/11 02:58:08  cs025
+   Base install of MG-PP
+
+   Revision 1.1  1999/08/10 21:18:17  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:49  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:24  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.4  1994/11/29  00:32:07  tes
+   * Committing the new merged files and changes.
+   *
+   * Revision 1.3  1994/10/20  03:57:01  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:56  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+int quick = 0;
+int no_of_words[2];
+u_long maxcodelen[2];
+
+char *dictname = "";
+
+
+
+
+void 
+DumpStemDict (FILE * f)
+{
+  struct invf_dict_header idh;
+  int i;
+  u_char prev[MAXSTEMLEN + 1];
+
+  Invf_Header_read(&idh, f);
+  /*
+  fread (&idh, sizeof (idh), 1, f);
+
+  // [RPAP - Jan 97: Endian Ordering]
+  NTOHUL(idh.lookback);
+  NTOHUL(idh.dict_size);
+  NTOHUL(idh.total_bytes);
+  NTOHUL(idh.index_string_bytes);
+  NTOHD(idh.input_bytes); // [RJM 07/97: 4G limit] 
+  NTOHUL(idh.num_of_docs);
+  NTOHUL(idh.static_num_of_docs);
+  NTOHUL(idh.num_of_words);
+  NTOHUL(idh.stemmer_num);
+  NTOHUL(idh.stem_method);
+  */
+
+  if (quick)
+    printf ("%ld\n", idh.dict_size);
+  else
+    {
+      printf ("# lookback           = %lu\n", idh.lookback);
+      printf ("# dict size          = %lu\n", idh.dict_size);
+      printf ("# total bytes        = %lu\n", idh.total_bytes);
+      printf ("# index string bytes = %lu\n", idh.index_string_bytes);
+      printf ("# input bytes        = %.0f\n", idh.input_bytes); /* [RJM 07/97: 4G limit] */
+      printf ("# num of docs        = %lu\n", idh.num_of_docs);
+      printf ("# static num of docs = %lu\n", idh.static_num_of_docs);
+      printf ("# num of words       = %lu\n", idh.num_of_words);
+      printf ("#\n");
+    }
+
+  for (i = 0; i < idh.dict_size; i++)
+    {
+      register unsigned long copy, suff;
+      unsigned long wcnt, fcnt;
+
+      /* build a new word on top of prev */
+      copy = getc (f);
+      suff = getc (f);
+      *prev = copy + suff;
+      fread (prev + copy + 1, sizeof (u_char), suff, f);
+
+      /* read other data, but no need to store it */
+      fread (&fcnt, sizeof (fcnt), 1, f);
+      fread (&wcnt, sizeof (wcnt), 1, f);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fcnt);
+      NTOHUL(wcnt);
+
+      if (!quick)
+	{
+	  printf ("%d: %8ld ", i, wcnt);
+	  printf ("/ %5ld ", fcnt);
+	  printf ("%2d %2ld\t\"", *prev, copy);
+	}
+      printf ("%s", word2str (prev));
+      if (quick)
+	printf (" %ld %ld\n", wcnt, fcnt);
+      else
+	{
+	  putchar ('"');
+	  putchar ('\n');
+	}
+    }
+}
+
+
+
+
+void 
+ReadInWords (FILE * f)
+{
+  comp_frags_header cfh;
+  u_long *codes;
+  u_char prev[MAXSTEMLEN + 1];
+  int i;
+
+  if (Read_cfh (f, &cfh, NULL, NULL) == -1)
+    FatalError (1, "Unable to read in the dictionary");
+
+  printf ("#\n");
+  printf ("#   max code len       = %u\n", cfh.hd.maxcodelen);
+  printf ("#   total bytes        = %lu\n", cfh.uncompressed_size);
+  printf ("#\n");
+
+  if (!(codes = Generate_Huffman_Codes (&cfh.hd, NULL)))
+    FatalError (1, "no memory for huffman codes\n");
+
+  for (i = 0; i < cfh.hd.num_codes; i++)
+    {
+      register int val, copy, j, k;
+      char code[33];
+      val = fgetc (f);
+      copy = (val >> 4) & 0xf;
+      val &= 0xf;
+
+      fread (prev + copy + 1, sizeof (u_char), val, f);
+      *prev = val + copy;
+
+      for (k = 0, j = cfh.hd.clens[i] - 1; j >= 0; j--, k++)
+	code[k] = '0' + ((codes[i] >> j) & 1);
+      code[k] = '\0';
+
+      printf ("%d: %2d : %*s : \"%s\"\n", i, cfh.hd.clens[i],
+	      cfh.hd.maxcodelen, code, word2str (prev));
+    }
+  Xfree (codes);
+  Xfree (cfh.hd.clens);
+}
+
+void ReadCharsHuffman(FILE *f, char *title, int mode)
+{
+  int i;
+  huff_data hd;
+  u_long *codes;
+
+  if (Read_Huffman_Data (f, &hd, NULL, NULL) == -1)
+    FatalError (1, "Unable to read huffman data");
+
+  if (!(codes = Generate_Huffman_Codes (&hd, NULL)))
+    FatalError (1, "no memory for huffman codes\n");
+
+  printf ("#\n# %s\n#\n", title);
+  for (i = 0; i < hd.num_codes; i++)
+    if (hd.clens[i])
+      {
+	int j, k;
+	char code[33];
+	for (k = 0, j = hd.clens[i] - 1; j >= 0; j--, k++)
+	  code[k] = '0' + ((codes[i] >> j) & 1);
+	code[k] = '\0';
+
+	if (mode == 0)
+	  {
+	    printf ("%2d : %*s : \"%s\"\n", hd.clens[i],
+		    hd.maxcodelen, code, char2str (i));
+	  }
+	else
+	  {
+	    printf ("%2d : %*s : %d\n", hd.clens[i],
+		    hd.maxcodelen, code, i);
+	  }
+      }
+  Xfree (codes);
+  Xfree (hd.clens);
+}
+
+
+void 
+ReadCharHuffman (FILE * f, char *title)
+{
+  ReadCharsHuffman(f, title, 0);
+}
+
+
+void 
+ReadLenHuffman (FILE * f, char *title)
+{
+  ReadCharsHuffman(f, title, 1);
+}
+
+
+
+
+
+void 
+DumpTextDict (FILE * f)
+{
+  struct compression_dict_header cdh;
+  int which;
+
+  if (Read_cdh (f, &cdh, NULL, NULL) == -1)
+    FatalError (1, "Unable to read dictionary header");
+  switch (cdh.dict_type)
+    {
+    case MG_COMPLETE_DICTIONARY:
+      printf ("# COMPLETE DICTIONARY\n");
+      break;
+    case MG_PARTIAL_DICTIONARY:
+      printf ("# PARTIAL DICTIONARY\n");
+      break;
+    case MG_SEED_DICTIONARY:
+      printf ("# SEED DICTIONARY\n");
+      break;
+    }
+  printf ("# num words          = %lu\n", cdh.num_words[1]);
+  printf ("# num word chars     = %lu\n", cdh.num_word_chars[1]);
+  printf ("# num non-words      = %lu\n", cdh.num_words[0]);
+  printf ("# num non-word chars = %lu\n", cdh.num_word_chars[0]);
+  printf ("# lookback           = %lu\n", cdh.lookback);
+
+  for (which = 0; which < 2; which++)
+    switch (cdh.dict_type)
+      {
+      case MG_COMPLETE_DICTIONARY:
+	{
+	  ReadInWords (f);
+	}
+	break;
+      case MG_PARTIAL_DICTIONARY:
+      case MG_SEED_DICTIONARY:
+	{
+	  if (cdh.num_words[which])
+	    ReadInWords (f);
+
+	  ReadCharHuffman (f, "Characters");
+	  ReadLenHuffman (f, "Lengths");
+	}
+	break;
+      }
+}
+
+
+
+
+void 
+DumpStatsDict (FILE * f)
+{
+  int i;
+  compression_stats_header csh;
+
+  fread (&csh, sizeof (csh), 1, f);
+
+  for (i = 0; i < 2; i++)
+    {
+      int j;
+      frags_stats_header fsh;
+
+      fread (&fsh, sizeof (fsh), 1, f);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(fsh.num_frags);
+      NTOHUL(fsh.mem_for_frags);
+
+      if (!quick)
+	printf ("#\n# num %9s      = %lu\n#\n", i ? "words" : "non-words",
+		fsh.num_frags);
+
+      for (j = 0; j < fsh.num_frags; j++)
+	{
+	  u_char Word[16];
+	  u_long freq, occur_num;
+
+	  fread (&freq, sizeof (freq), 1, f);
+	  fread (&occur_num, sizeof (occur_num), 1, f);
+
+	  /* [RPAP - Jan 97: Endian Ordering] */
+	  NTOHUL(freq);
+	  NTOHUL(occur_num);
+
+	  Word[0] = fgetc (f);
+	  fread (Word + 1, Word[0], 1, f);
+	  printf ("%d: %7ld : %7ld : \"%s\"\n", j, freq,
+		  occur_num, word2str (Word));
+	}
+    }
+}
+
+
+int main (int argc, char **argv)
+{
+  FILE *fp;
+  unsigned long magic = 0;
+
+  if (argc < 2)
+    FatalError (1, "A file name must be specified");
+  dictname = argv[1];
+  if (strcmp (dictname, "-q") == 0)
+    {
+      quick = 1;
+      if (argc < 3)
+	FatalError (1, "A file name must be specified");
+      dictname = argv[2];
+    }
+  if (!(fp = fopen (dictname, "rb")))  /* [RPAP - Feb 97: WIN32 Port] */
+    FatalError (1, "Unable to open \"%s\"", dictname);
+
+  fread (&magic, sizeof (magic), 1, fp);
+
+  NTOHUL(magic);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  switch (magic)
+    {
+    case MAGIC_STEM_BUILD:
+      if (!quick)
+	printf ("# Contents of STEM file \"%s\"\n#\n", dictname);
+      DumpStemDict (fp);
+      break;
+    case MAGIC_DICT:
+      if (!quick)
+	printf ("# Contents of DICT file \"%s\"\n#\n", dictname);
+      DumpTextDict (fp);
+      break;
+    case MAGIC_STATS_DICT:
+      if (!quick)
+	printf ("# Contents of STATS file \"%s\"\n#\n", dictname);
+      DumpStatsDict (fp);
+      break;
+    default:
+      FatalError (1, "Bad magic number. \"%s\" cannot be dumped", dictname);
+    }
+  fclose (fp);
+  return 0;
+}
Index: /tags/gsdl-21/gsdl/src/mgpp/text/mgquery.1
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/mgquery.1	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/mgquery.1	(revision 699)
@@ -0,0 +1,693 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgquery 1 \*(Dt CITRI
+.SH NAME
+mgquery \- query program for the mg system
+.SH SYNOPSIS
+.B mgquery
+[
+.B \-h
+]
+[
+.B \-D
+]
+[
+.BI \-f " name"
+]
+[
+.BI \-d " directory"
+]
+.if n .ti +9n
+[
+.I collection-name
+]
+.SH DESCRIPTION
+.B mgquery
+enables users to make Boolean or ranked queries from a data base
+generated by the
+.BR mg (1)
+system.  It accepts queries from
+.I stdin
+and sends the retrieved documents to
+.IR stdout .
+Information on the resource usage of
+.B mgquery
+as it processes queries can be obtained interactively.
+.SH OPTIONS
+Options may appear in any order, but the
+.IR collection-name ,
+if specified, must be last.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B \-D
+This option causes the entire text to be decompressed and sent to
+.IR stdout .
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection that will be
+used.  If a collection with the specified base
+.I name
+does not exist, an error message will be displayed and
+.B mgquery
+will exit.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.SH USAGE
+Prior to processing the command line arguments, the
+.B mgquery
+program attempts to read in a startup script called
+.IR ./.mgrc .
+If that fails, it attempts to read in the file
+.IR $HOME/.mgrc .
+The startup file can only contain commands\(emno queries are
+permitted in the
+.I .mgrc
+file.  Lines starting with \*(lq\fB#\fP\*(rq in the file are comments.
+The most common use for the
+.I .mgrc
+file is to personalise the initial values of the predefined parameters
+with
+.B .set
+commands.
+.LP
+The input to
+.B mgquery
+consists of a series of input lines.  The backslash
+character
+.RB (\*(lq \e \*(rq)
+is used at the end of lines to indicate
+that input continues on the next line.
+.LP
+Input lines on which the first character is a dot
+.RB (\*(lq . \*(rq)
+are commands to the
+.B mgquery
+program.  Input lines that do not start with a dot are queries.
+.LP
+A query consists of two parts.  One part is a Boolean or ranked query
+that identifies documents.  The second part is a post-processing
+pattern matching operation.  Any text between the first speech mark
+(\*(lq) and the last speech mark (\*(rq) is considered to be a
+post-processing pattern.
+.SH COMMANDS
+The
+.B mgquery
+program can accept the following commands.
+.TP 17
+.B .help
+Display several pages of help text.
+.TP
+.B .quit
+Quit the program.
+.TP
+.B .warranty
+Display the
+.BR mg (1)
+warranty.
+.TP
+.B .conditions
+Display the conditions of use and distribution of
+.BR mg (1).
+.TP
+.BI ".set " "name value"
+Set the parameter
+.I name
+to the specified
+.IR value .
+If the parameter is a Boolean
+.I value
+and the
+.I value
+is omitted, the parameter will be inverted (i.e., if it was
+.IR true ,
+then it will change to
+.IR false ;
+if it was
+.IR false ,
+then it will change to
+.IR true ).
+.TP
+.BI ".unset " name
+Delete the parameter
+.I name
+from the currently-defined parameters.
+.TP
+.B .reset
+Reset the parameters to the state that they had after the processing
+of the
+.B mgquery
+command line.
+.TP
+.B .display
+Display the values of all the currently-defined parameters.
+.TP
+.B .push
+Push the currently-defined parameters onto a stack.
+.TP
+.B .pop
+Pops a set of parameters off the stack, replacing the currently-defined
+ones.
+.TP
+.BI ".output " arg
+This is used to specify where to send the text of the documents.  Once
+the
+.B .output
+command is specified, all subsequent output will be sent to the place
+specified by
+.IR arg .
+If
+.I arg
+is not specified subsequent output will be directed to
+.IR stdout .
+.I Arg
+may be any of the following.
+.RS
+.TP 13
+.BI "> " filename
+Send output to the specified file.
+.TP
+.BI ">> " filename
+Append output to the specified file.
+.TP
+.BI "| " command
+Pipe the output to
+.IR command ,
+which is executed by
+.IR sh .
+.RE
+.TP
+.BI ".input " arg
+This is used to specify where input (queries and commands) comes
+from.  Once the
+.B .input
+command is specified all subsequent input will be come from the place
+specified by
+.IR arg .
+If
+.I arg
+is not specified subsequent input will come from
+.IR stdin .
+.RS
+.TP 13
+.BI "< " filename
+Get input from the specified file.
+.TP
+.BI "| " command
+The input comes from the standard output of
+.IR command ,
+which is executed by
+.IR sh .
+.RE
+.SH PARAMETERS
+The following parameters are predefined and have special
+significance.  Each parameter will be followed by its default
+value.  Parameters are initialised before the
+.I .mgrc
+file is read or the command line arguments are processed.
+.TP 17
+.BI accumulator_method " `array'"
+This parameter is used during ranking, and specifies how the
+weight for each document should be accumulated.  The following
+methods are available:
+.IR array ,
+.IR splay_tree ,
+.IR hash_table ,
+and
+.IR list .
+.TP
+.BI briefstats " `off'"
+This is a Boolean parameter that determines whether the
+totals for disk, memory and time usage statistics will be
+displayed at the end of each query.
+.IR Note :
+this takes precedence over the parameters
+.BR diskstats ,
+.BR memstats " and " timestats .
+This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI buffer " `1048576'"
+When the documents are being read in, they are read into a
+buffer of this size and then displayed from this buffer.  If
+the documents are larger than this buffer, the buffer is
+expanded automatically.  Having a large buffer gives a very
+slight performance improvement, because it allows the order of
+disk operations to be optimised.  The buffer size is measured
+in bytes.
+.TP
+.BI diskstats " `off'"
+This is a Boolean parameter that determines whether the disk
+usage statistics for the preceding query will be displayed
+after each query.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI doc_sepstr " `---------------------------------- %n\en\'"
+This specifies the string that will be used to separate
+documents when they are displayed for `Boolean' or `docnums'
+queries.  The standard C escape character sequences
+may be used to place special characters in the
+string.  For example, a newline would be `\en'.  To include a `%',
+use the sequence `%%'.  To include the
+.BR mg (1)
+document number, use the sequence `%n'.  The following escape character
+sequences are available
+.nf
+.ta 1.7iL
+.B Sequence	Meaning
+`\e\e'	backslash
+`\eb'	backspace
+`\ef'	formfeed
+`\en'	newline
+`\er'	carriage return
+`\et'	tab
+`\e"'	speech marks
+`\e''	quote mark
+`\ex\fIhh\fP'	ASCII code in hexadecimal
+`\ennn'	ASCII code in octal
+.fi
+.TP
+.BI expert " `false'"
+If this is
+.IR true ,
+then much of the dialogue output is suppressed.  This parameter may
+take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI hash_tbl_size " `1000'"
+One of the options during ranking queries is to use a hash
+table to accumulate the weights for each document.  The hash
+table is a simple chained type.  This parameter specifies the
+size of the hash table and may take any value between 8 and
+268435456 (2^28).
+.TP
+.BI heads_length " `50'"
+When the mode is
+.BR heads ,
+this specifies the number of characters that will be output for each
+document.
+.TP
+.BI maxdocs " `all'"
+The maximum number of documents to display in response to a
+query.  This parameter may take on a numeric value between 1
+and 429467295 (2^32 - 1) or the word
+.IR all .
+.TP
+.BI maxparas " `1000'"
+The maximum number of paragraphs to identify during a ranked
+query with paragraph indexing.  After the paragraphs have been
+identified, the paragraphs are converted into documents, and
+because some of the paragraphs may refer to the same documents
+the final number of answers may be less than
+.BR maxparas .
+The
+.B maxdocs
+parameter will then be applied.  This parameter may take on a numeric
+value between 1 and 429467295 (2^32 - 1).
+.TP
+.BI max_accumulators " `50000'"
+This parameter limits the number of different paragraph and
+document numbers to be accumulated during ranked queries when
+the parameter
+.B accumulator_method
+is set to
+.IR splay_tree ,
+.IR hash_table ,
+or
+.IR list .
+This parameter may take any value between 8 and 268435456 (2^28).
+.TP
+.BI max_terms " `all'"
+This parameter limits the number of terms that will actually
+be used during a ranked query.  If more terms than the number
+specified by
+.B max_terms
+are entered, then the extra terms will be discarded.  If
+.B sorted_terms
+is on, then the limiting will be done after the terms have been
+sorted.  This parameter may take any value between 1 and 429467295
+(2^32 - 1), or the word
+.IR all.
+.TP
+.BI memstats " `off'"
+This is a Boolean parameter that determines whether the memory
+usage statistics for the preceding query will be displayed
+after each query.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI mgdir " `.'"
+This is set to the directory where the
+.BR mg (1)
+data files may be found.  If
+the environment variable
+.B MGDATA
+exists, then this is instead initialised to the value of
+.BR MGDATA .
+The value of this parameter may be changed, either in the
+.I .mgrc
+file with a
+.BI ".set mgdir "directory
+command, or from the command line using the
+.BI \-d " directory"
+option.  Once the \*(lq\fB>\fP\*(rq prompt appears, changing this
+parameter will have no effect.
+.TP
+.BI mgname " `bible'"
+This is set to the name of the
+.BR mg (1)
+collection that is to be used for the session.  The value of this
+parameter may be changed, either in the
+.I .mgrc
+file with a
+.BI ".set mgname "name
+command, or from the command line using the
+.BI \-f " name"
+option.  Once the \*(lq\fB>\fP\*(rq prompt appears, changing this
+parameter will have no effect.
+.TP
+.BI mode " `text'"
+This specifies how documents should be displayed when they
+are retrieved.  It may take six different values:
+.IR text ,
+.IR hilite ,
+.IR docnums ,
+.IR heads ,
+.IR silent ,
+or
+.IR count .
+.I text
+displays the contents of the document.
+.I hilite
+displays the contents of the document and highlights any of the 
+stemmed query terms.
+.I docnums
+displays only the document numbers.
+.I heads
+is used to print out the head of each document.
+.I silent
+retrieves all the documents but displays nothing except how many
+documents were retrieved.  This mode is intended to be used in timing
+experiments.
+.I count
+does the minimum
+amount of work required to determine how many documents would
+be retrieved, but does not retrieve them.
+.TP
+.BI optimise_type " `1'"
+There are three types of boolean query optimisation (parse tree
+rearrangement). Type 0 leaves parse tree unaltered. Type 1 optimises
+for AND of terms and AND of OR of terms. Type 2 converts the tree
+into DNF (an experiment :-).
+.TP
+.BI pager " `more'"
+This is the name of the program that will be used to display
+the help and the retrieved documents.  If the environment
+variable
+.B PAGER
+is defined, then
+.B pager
+takes on that value.
+.TP
+.BI hilite_style " `bold'"
+This specifies the type of highlighting method.
+It may take one of two different values:
+.IR bold,
+or
+.IR underline.
+.TP
+.BI para_sepstr " `\en######## PARAGRAPH %n ########\en'"
+This specifies the string that will be used to separate paragraphs.
+The standard C escape character sequences may be used to place special
+characters in the string.  For example, a newline would be written
+as `\en'.  To include a `%', use the sequence `%%'.  To include the
+paragraph number within the document, use the sequence `%n'.
+.TP
+.BI para_start " `***** Weight = %w *****\en'"
+This specifies the string that will be used at the head of paragraphs
+for a paragraph-level index following a ranked query.  The standard
+C-language escape character sequences may be used to place special
+characters in the string.  For example, a newline would be written as
+`\en'.  To include a `%', use the sequence `%%'.  To include the
+paragraph weight, use the sequence `%w'.
+.TP
+.BI qfreq " `true'"
+This determine whether the ranked queries will take into account the
+number of times each query term is specified.  When this is
+.IR true ,
+the number of times a term appears in the query is used in the
+ranking.  When this is
+.IR false ,
+all query terms are assumed to occur only once.  This parameter may
+take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI query  " `Boolean'"
+This specifies the type of queries that are to be specified.
+It can take four different values:
+.IR Boolean ,
+.IR ranked ,
+.IR docnums " or " approx-ranked.
+.I Boolean
+is for Boolean queries.
+The
+.BR yacc (1)
+grammar for Boolean queries is as follows.
+.IP
+.nf
+        query   : or;
+.IP
+        or      : or '|' and
+                | and ;
+.IP
+        and     : and '&' not
+                | and not
+                | not ;
+.IP
+        not     : term
+                | '!' not ;
+.IP
+        term    : TERM
+                | '(' or ')' ;
+.fi
+.IP
+.IR ranked " and " approx-ranked
+are for queries ranked by the cosine measure.
+.I approx-ranked
+uses only the low-precision document lengths, and therefore only
+produces an approximation to full cosine ranking.
+.IP
+.nf
+        query   : TERM
+                | query TERM ;
+.fi
+.IP
+.I docnums
+allows the entry of document numbers.  Multiple numbers separated by
+spaces may be specified, or ranges separated by hyphens.
+.IP
+.nf
+        query   : range
+                | query range ;
+.IP
+        range   : num
+                | num '-' num ;
+.fi
+.TP
+.BI ranked_doc_sepstr " `-------------------------------- %n %w\en'"
+This specifies the string that will be used to separate documents when
+they are displayed for `ranked' or `approx-ranked' queries.  The
+standard C escape character sequences may be used to place special
+characters in the string.  For example, a newline would be written as
+`\en'.  To include a `%', use the sequence `%%'.  To include the
+.BR mg (1)
+document number, use the sequence `%n'.  To include the document
+weight, use the sequence `%w'.
+.TP
+.BI sizestats " `false'"
+If this is
+.IR true ,
+then various numbers are output at the end of each query indicating
+what went on during the query.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI skip_dump " `skips.%d'"
+If this parameter is set, then a file will be produced in the current
+directory during ranked queries on skipped inverted files when
+.B accumulator_method
+is set to
+.IR splay_tree ,
+.IR hash_table ,
+or
+.IR list .
+The name of the file is the value of this parameter.  A `%d' in the
+file name will be replaced with the process id of
+.BR mgquery .
+This file will contain information about the usage of skips during the
+query processing.  This option is expensive; use
+.B .unset skip_dump
+to obtain optimal performance.
+.TP
+.BI sorted_terms " `on'"
+This specifies whether or not the terms should be sorted into
+decreasing occurrence in documents so that the least-often occurring
+terms are processed first when ranked queries are being done.  When
+this is
+.IR true ,
+the terms are sorted.  When this is
+.IR false ,
+the terms are not sorted, and are instead processed in order of
+occurrence.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI stop_at_max_accum " `on'"
+This specifies what should happen when the maximum number of
+accumulators set by
+.B max_accumulators
+is reached.  When this is
+.IR true ,
+the processing of terms is stopped at the completion of the current
+term.  When this is
+.IR false ,
+processing continues but no new accumulators are created.  This
+parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI terminator " `'"
+This specifies the string that will be output after the last document
+from the previous query has been output.  The standard C escape
+character sequences may be used to place special characters in the
+string.  For example, a newline would be written as `\en'.  To include
+a `%', use the sequence `%%'.
+.TP
+.BI timestats " `false'"
+If this is
+.IR true ,
+then the time to process a query is displayed in both real time and
+CPU time.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.TP
+.BI verbatim " `off'"
+This is a Boolean parameter that determines whether the program
+should attempt to do a regular-expression match on the retrieved
+text.  If verbatim is
+.I on
+and a post-processing string is specified with the query, then the
+post-processing string will be searched for in the documents just
+before they are displayed.  If the string is found, the document will
+be displayed; if not, the document will not be displayed.  If verbatim
+is
+.IR off ,
+the post-processing string will be considered a regular expression
+as in
+.BR egrep (1)
+or
+.BR vi (1).
+E.g., if verbatim is
+.I on,
+\*(lq\fBand.*the\fP\*(rq will look for the 8-character sequence
+\*(lq\fBand.*the\fP\*(rq.  If verbatim is
+.IR off ,
+\*(lq\fBand.*the\fP\*(rq will look for the sequence
+\*(lq\fBand\fP\*(rq followed somewhere later in the document by the
+sequence \*(lq\fBthe\fP\*(rq.  This parameter may take the values
+.IR yes ", " no ", "
+.IR true ", " false ", "
+.IR on " or " off .
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.I .mgrc
+.B mgquery
+startup file
+.TP
+.B help.mg
+Help file for
+.BR mgquery .
+The contents of this file is displayed with the
+.B .help
+command.
+.TP
+.B *.invf
+Inverted file.
+.TP
+.B *.invf.dict
+The `on-disk' stemmed dictionary.
+.TP
+.B *.text
+Compressed documents.
+.TP
+.B *.text.dict
+Compression dictionary.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.text.idx.wgt
+Interleaved index into the compressed documents and document weights.
+.TP
+.B *.weight.approx
+Approximate document weights.
+.SH "SEE ALSO"
+.na
+.BR egrep (1),
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgstat (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1),
+.BR vi (1),
+.BR yacc (1).
Index: /tags/gsdl-21/gsdl/src/mgpp/text/mgquery.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/mgquery.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/mgquery.cpp	(revision 699)
@@ -0,0 +1,1554 @@
+/**************************************************************************
+ *
+ * mgquery.c -- The   M G Q U E R Y   program
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.3  1999/10/12 04:12:46  cs025
+   Changes made to assist with eradication of GetEnv elsewhere in the
+   program.
+
+   Revision 1.2  1999/10/11 22:01:20  cs025
+   Got rid of regular expression calls.
+
+   Revision 1.1  1999/10/11 02:58:09  cs025
+   Base install of MG-PP
+
+   Revision 1.1  1999/08/10 21:18:18  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.3  1999/01/08 00:33:46  rjmcnab
+
+   Enabled mg and the library software to read in more than one index
+   at a time.
+
+   Revision 1.2  1998/11/25 07:55:49  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:29  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:57:02  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:41:58  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+#if defined(HAVE_SYS_PROCFS_H) && defined(HAVE_PR_BRKSIZE) && \
+    (__STDC__ == 0)
+/* STDC test was included to allow cc -Xc on SunOS 5 to work */
+#define USE_PROCESS_MEM
+#endif
+
+#ifdef USE_PROCESS_MEM
+# include <sys/procfs.h>
+#endif
+
+#ifdef HAVE_GETRUSAGE
+# ifdef HAVE_SYS_TIME_H
+#  include <sys/time.h>
+# endif
+# include <sys/resource.h>
+#endif
+
+#ifndef HAVE_GETPAGESIZE
+# include "getpagesize.h"
+#endif
+
+/* 
+extern "C" {
+#if WITH_REGEX
+# include <regex.h>
+#else
+# include <rx.h>
+#endif
+#include <stdio.h>
+}
+*/
+
+#include "non_ansi.h"
+
+#include <stdarg.h>
+#include <signal.h>
+#include <unistd.h>
+
+#include "messages.h"
+#include "timing.h"
+#include "memlib.h"
+#include "local_strings.h"  /* [RPAP - Feb 97: Term Frequency] */
+
+#include "filestats.h"
+#include "invf.h"
+#include "text.h"
+#include "mg.h"
+#include "lists.h"
+#include "backend.h"
+#include "DocEntry.h"
+#include "environment.h"
+#include "globals.h"
+#include "read_line.h"
+#include "mg_errors.h"
+#include "commands.h"
+#include "text_get.h"
+#include "term_lists.h"
+#include "query_term_list.h"
+
+FILE *OutFile = NULL, *InFile = NULL;
+int OutPipe = 0, InPipe = 0;
+int Quitting = 0;
+
+/* [RPAP - Feb 97: NZDL Additions] */
+#if defined(PARADOCNUM) ||  defined(NZDL)
+int GetDocNumFromParaNum(query_data *qd, int paranum) {
+  int Documents = qd->td->cth.num_of_docs;
+  int *Paragraph = qd->paragraph;
+  int low = 1, high = Documents;
+  int mid = (low+high)/2;
+
+  while ((mid = (low+high)/2) >=1 && mid <= Documents)
+    {
+      if (paranum > Paragraph[mid])
+	low = mid+1;
+      else if (paranum <= Paragraph[mid-1])
+	high = mid-1;
+      else
+	return mid;
+    }
+  FatalError(1, "Bad paragraph number.\n");
+}
+#endif
+
+#ifdef TREC_MODE
+char *trec_ids = NULL;
+long *trec_paras = NULL;
+#endif
+
+static volatile int PagerRunning = 0;
+static volatile int Ctrl_C = 0;
+
+
+/*****************************************************************************/
+
+typedef enum
+  {
+    S_Time, S_Mem, S_Size, S_File
+  }
+S_Type;
+
+static struct Stat
+  {
+    S_Type typ;
+    char *name;
+    char *text;
+  }
+ *Stats = NULL;
+static int NumStats = 0;
+
+static void 
+Clear_Stats (void)
+{
+  if (Stats)
+    {
+      int i;
+      for (i = 0; i < NumStats; i++)
+	{
+	  if (Stats[i].name)
+	    Xfree (Stats[i].name);
+	  if (Stats[i].text)
+	    Xfree (Stats[i].text);
+	}
+      Xfree (Stats);
+      Stats = NULL;
+      NumStats = 0;
+    }
+}
+
+static void 
+Add_Stats (S_Type typ, char *name, char *fmt,...)
+{
+  char buf[1024];
+  va_list args;
+  va_start (args, fmt);
+  vsprintf (buf, fmt, args);
+  if (Stats)
+    Stats = (struct Stat *) Xrealloc (Stats, (++NumStats) * sizeof (*Stats));
+  else
+    Stats = (struct Stat *) Xmalloc ((++NumStats) * sizeof (*Stats));
+  Stats[NumStats - 1].typ = typ;
+  Stats[NumStats - 1].name = Xstrdup (name);
+  Stats[NumStats - 1].text = Xstrdup (buf);
+}
+
+static void 
+Display_Stats (FILE * f)
+{
+  static char *sep = "-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-="
+  "-=-=-=-=-=-=-=-=-=-=-";
+  char *names[] =
+  {"Time:  ", "Memory:", "Sizes: ", "Disk:  ", "       "};
+  int i, last_typ = -1;
+  size_t len = 0;
+  if (NumStats == 0)
+    return;
+  fprintf (f, "%s\n", sep);
+  for (i = 0; i < NumStats; i++)
+    if (strlen (Stats[i].name) > len)
+      len = strlen (Stats[i].name);
+  for (i = 0; i < NumStats; i++)
+    {
+      int typ = 4;
+      if (Stats[i].typ != last_typ)
+	typ = last_typ = Stats[i].typ;
+      fprintf (f, "%s %-*s %s\n", names[typ], (int) len, Stats[i].name, Stats[i].text);
+    }
+  fprintf (f, "%s\n", sep);
+}
+
+/*****************************************************************************/
+
+
+static void 
+QueryTimeStats (ProgTime * Start, ProgTime * invf, ProgTime * text)
+{
+  if (!BooleanEnv (GetEnv ("briefstats"), 0))
+    {
+      Add_Stats (S_Time, "invf", ElapsedTime (Start, invf));
+      Add_Stats (S_Time, "text", ElapsedTime (invf, text));
+    }
+  Add_Stats (S_Time, "total", ElapsedTime (Start, text));
+}
+
+static void 
+StartUpTimeStats (InitQueryTimes * iqt)
+{
+  if (!BooleanEnv (GetEnv ("briefstats"), 0))
+    {
+      Add_Stats (S_Time, "dict [stem]", ElapsedTime (&iqt->Start,
+						     &iqt->StemDict));
+      Add_Stats (S_Time, "weights", ElapsedTime (&iqt->StemDict,
+						 &iqt->ApproxWeights));
+      Add_Stats (S_Time, "dict [text]", ElapsedTime (&iqt->ApproxWeights,
+						     &iqt->CompDict));
+      Add_Stats (S_Time, "Inverted", ElapsedTime (&iqt->CompDict,
+						  &iqt->Invf));
+      Add_Stats (S_Time, "Compressed", ElapsedTime (&iqt->Invf,
+						    &iqt->Text));
+    }
+  Add_Stats (S_Time, "total", ElapsedTime (&iqt->Start, &iqt->Text));
+}
+
+
+
+
+#ifdef USE_PROCESS_MEM
+static u_long 
+process_mem (void)
+{
+  prstatus_t pr;
+  static int fd = -1;
+  if (fd == -1)
+    {
+      char buf[128];
+      sprintf (buf, "/proc/%ld", (long) getpid ());
+      fd = open (buf, O_RDONLY);
+    }
+  if (fd == -1 || ioctl (fd, PIOCSTATUS, &pr) == -1)
+    return 0;
+  return pr.pr_brksize;
+}
+#endif
+
+
+
+
+static void 
+MemStats (query_data * qd)
+{
+  if (!BooleanEnv (GetEnv ("briefstats"), 0))
+    {
+#ifdef HAVE_GETRUSAGE
+      struct rusage rusage;
+      getrusage (RUSAGE_SELF, &rusage);
+
+      Add_Stats (S_Mem, "process mem", "%7.3f Mb",
+	    (double) (rusage.ru_maxrss * getpagesize () / 1024.0 / 1024.0));
+#endif
+#ifdef USE_PROCESS_MEM
+      Add_Stats (S_Mem, "process mem", "%7.3f Mb",
+		 (double) (process_mem () / 1024.0 / 1024.0));
+#endif
+      Add_Stats (S_Mem, "dict [stem]", "%7.1f kB",
+		 (double) qd->sd->MemForStemDict / 1024);
+      Add_Stats (S_Mem, "dict [text]", "%7.1f kB",
+		 (double) qd->cd->MemForCompDict / 1024);
+      if (qd->awd)
+	Add_Stats (S_Mem, "weights", "%7.1f kB",
+		   (double) qd->awd->MemForWeights / 1024);
+    }
+  if (qd->awd)
+    Add_Stats (S_Mem, "total [peak]", "%7.1f kB",
+	       (double) (qd->max_mem_in_use + qd->sd->MemForStemDict +
+		   qd->cd->MemForCompDict + qd->awd->MemForWeights) / 1024);
+  else
+    Add_Stats (S_Mem, "total [peak]", "%7.1f kB",
+	       (double) (qd->max_mem_in_use + qd->sd->MemForStemDict +
+			 qd->cd->MemForCompDict) / 1024);
+
+}
+
+
+
+static void 
+SizeStats (query_data * qd)
+{
+  Add_Stats (S_Size, "skips", "%7d", qd->hops_taken);
+  Add_Stats (S_Size, "pointers", "%7d", qd->num_of_ptrs);
+  Add_Stats (S_Size, "accumulators", "%7d", qd->num_of_accum);
+  Add_Stats (S_Size, "terms", "%7d", qd->num_of_terms);
+  Add_Stats (S_Size, "answers", "%7d", qd->num_of_ans);
+  Add_Stats (S_Size, "index lookups", "%7d", qd->text_idx_lookups);
+}
+
+static void 
+TotalSizeStats (query_data * qd)
+{
+  Add_Stats (S_Size, "skips", "%7d", qd->tot_hops_taken);
+  Add_Stats (S_Size, "pointers", "%7d", qd->tot_num_of_ptrs);
+  Add_Stats (S_Size, "accumulators", "%7d", qd->tot_num_of_accum);
+  Add_Stats (S_Size, "terms", "%7d", qd->tot_num_of_terms);
+  Add_Stats (S_Size, "answers", "%7d", qd->tot_num_of_ans);
+  Add_Stats (S_Size, "index lookups", "%7d", qd->tot_text_idx_lookups);
+}
+
+
+static void 
+StatFile (File * F)
+{
+  static unsigned long NumBytes = 0, NumSeeks = 0, NumReads = 0;
+  if (F)
+    if ((int) F != -1)
+      {
+	if (!BooleanEnv (GetEnv ("briefstats"), 0))
+	  Add_Stats (S_File, F->name, "%7.1f kB (%3d seeks, %7d reads)",
+		   (double) F->Current.NumBytes / 1024, F->Current.NumSeeks,
+		     F->Current.NumReads);
+	NumBytes += F->Current.NumBytes;
+	NumSeeks += F->Current.NumSeeks;
+	NumReads += F->Current.NumReads;
+      }
+    else
+      {
+	Add_Stats (S_File, "total", "%7.1f kB (%3d seeks, %7d reads)",
+		   (double) NumBytes / 1024, NumSeeks, NumReads);
+	NumSeeks = NumReads = NumBytes = 0;
+      }
+
+}
+
+
+static void 
+File_Stats (query_data * qd)
+{
+  StatFile (qd->File_comp_dict);
+  StatFile (qd->File_fast_comp_dict);
+  StatFile (qd->File_text_idx_wgt);
+  StatFile (qd->File_text);
+  StatFile (qd->File_stem);
+
+  /* [RPAP - Jan 97: Stem Index Change] */
+  if (qd->sd->sdh.indexed)
+    {
+      StatFile (qd->File_stem1);
+      StatFile (qd->File_stem2);
+      StatFile (qd->File_stem3);
+    }
+
+  StatFile (qd->File_invf);
+  StatFile (qd->File_weight_approx);
+  StatFile (qd->File_text_idx);
+  StatFile ((File *) (-1));
+}
+
+
+char *
+get_query (query_data * qd)
+{
+  char *line, *LinePtr;
+  WritePrompt ();
+  do
+    {
+      do
+	{
+	  line = GetMultiLine ();
+	  if (line == NULL)
+	    {
+	      if (stdin == InFile)
+		return (NULL);	/* EOF */
+	      if (InPipe)
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+		_pclose (InFile);
+#else
+		pclose (InFile);
+#endif
+	      else
+		fclose (InFile);
+	      InPipe = 0;
+	      InFile = stdin;
+	    }
+	}
+      while (line == NULL);
+      LinePtr = ProcessCommands (line, qd);
+      if (CommandsErrorStr)
+	fprintf (stderr, "%s\n", CommandsErrorStr);
+    }
+  while (*LinePtr == '\0' && !Quitting);
+  return (LinePtr);
+}
+
+
+/* This is executed when a SIGPIPE is detected
+   i.e. If some one quits out of the PAGER, this is executed */
+#ifdef HAVE_SIGCONTEXT
+static RETSIGTYPE 
+SIGPIPE_handler (int sig, int code,
+		 struct sigcontext *scp, char *addr)
+#else
+static RETSIGTYPE 
+SIGPIPE_handler (int sig)
+#endif
+{
+  /* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+  signal (sig, SIG_IGN);
+#else
+  signal (SIGPIPE, SIG_IGN);
+#endif
+  PagerRunning = 0;
+}
+
+/* This is executed when a SIGINT (i.e. CTRL-C) is detected */
+#ifdef  HAVE_SIGCONTEXT
+static RETSIGTYPE 
+SIGINT_handler (int sig, int code,
+		struct sigcontext *scp, char *addr)
+#else
+static RETSIGTYPE 
+SIGINT_handler (int sig)
+#endif
+{
+  Ctrl_C = 1;
+}
+
+
+
+static char *post_proc = NULL;
+
+
+
+void 
+GetPostProc (char *line)
+{
+  char *start, *finish;
+  if (post_proc)
+    {
+      Xfree (post_proc);
+      post_proc = NULL;
+    }
+  start = strchr (line, '\"');
+  finish = strrchr (line, '\"');
+  if (start != finish)
+    {
+      /* found a pattern */
+      *finish = '\0';
+      post_proc = Xstrdup (start + 1);
+      strcpy (start, finish + 1);
+      /**
+       * GRB: Elimination regular of expression support/dependency
+       *      removed the below from active use; assume verbatim
+       *      mode
+       if (BooleanEnv (GetEnv ("verbatim"), 1) == 0)
+	 {
+	   char *s;
+	   s = re_comp (post_proc);
+	   if (!s)
+	   {
+	   Xfree (post_proc);
+	   post_proc = NULL;
+	   }
+	 }
+       */
+    }
+  else if (start != NULL)
+    {
+      /* found a single speech mark. Delete It. */
+      strcpy (start, start + 1);
+    }
+}
+
+int 
+PostProc (char *UDoc, int verbatim)
+{
+  if (!post_proc)
+    return 1;
+
+  /**
+   * GRB: eliminating regular expressions; removed all the below and
+   *      replaced with a strstr construct only 
+   if (verbatim) {
+     return (strstr (UDoc, post_proc) != NULL);
+   }
+   return re_exec ((char *) UDoc);
+   */
+  return (strstr(UDoc, post_proc) != NULL);
+}
+
+
+
+
+static DocEntry *
+in_chain (int para, int ip, DocEntry * dc)
+{
+  while (dc)
+    {
+      if (dc->DocNum - ip == para)
+	return dc;
+      dc = dc->Next;
+    }
+  return NULL;
+}
+
+/* num should be greater than or equal to 1 */
+int 
+RawDocOutput (query_data * qd, u_long num, FILE * Output)
+{
+  static int last_pos = 0;
+  static u_char *c_buffer = 0;
+  static int buf_len = -1;
+  static u_char *uc_buffer = 0;
+  u_long pos, len;
+  int ULen;
+
+  FetchDocStart (qd, num, &pos, &len);
+
+  if ((int) len > buf_len)
+    {
+      if (c_buffer)
+	{
+	  Xfree (c_buffer);
+	  Xfree (uc_buffer);
+	}
+      if (!(c_buffer = (u_char *) Xmalloc (len)))
+	return -1;
+      if (!(uc_buffer = (u_char *) Xmalloc ((int) (qd->td->cth.ratio * 1.01 *
+					len) + 100)))
+	return -1;
+      buf_len = len;
+    }
+  if (last_pos != pos)
+    Fseek (qd->td->TextFile, pos, 0);
+  Fread (c_buffer, 1, len, qd->td->TextFile);
+  last_pos = pos + len;
+  DecodeText (qd->cd, c_buffer, len, uc_buffer, &ULen);
+  fwrite (uc_buffer, ULen, sizeof (u_char), Output);
+  return 0;
+}
+
+
+void 
+StringOut (FILE * Output, char *string,
+	   int intvalid, unsigned long intval,
+	   int floatvalid, double floatval)
+{
+  char *s;
+  for (s = string; *s; s++)
+    if (*s == '%' &&
+	(*(s + 1) == 'n' || *(s + 1) == 'w' || *(s + 1) == '%'))
+      {
+	s++;
+	switch (*s)
+	  {
+	  case 'n':
+	    if (intvalid)
+	      fprintf (Output, "%lu", intval);
+	    else
+	      fprintf (Output, "%%n");
+	    break;
+	  case 'w':
+	    if (floatvalid)
+	      fprintf (Output, "%f", floatval);
+	    else
+	      fprintf (Output, "%%w");
+	    break;
+	  case '%':
+	    fputc ('%', Output);
+	  }
+      }
+    else
+      fputc (*s, Output);
+}
+
+
+void 
+HeaderOut (FILE * Output, u_char * UDoc, unsigned long ULen, int heads_length)
+{
+  int i, space = 1, num = 0;
+  for (i = 0; i < ULen && num < heads_length; i++)
+    {
+      char c = UDoc[i];
+      if (c == '\02')
+	break;
+
+      if (isspace (c) || c == '\01' || c == '\03')
+	{
+	  if (!space)
+	    {
+	      fputc (' ', Output);
+	      num++;
+	    }
+	  space = 1;
+	}
+      else
+	{
+	  space = 0;
+	  fputc (c, Output);
+	  num++;
+	}
+    }
+}
+
+/* [RPAP - Feb 97: NZDL Additions] */
+#if defined(PARADOCNUM) || defined(NZDL)
+void PrintDocNum(FILE *output, char query_type,
+		 int docnum, int indexnum, float weight)
+{
+  if (query_type == 'R' || query_type == 'A')
+    fprintf(output, "%7d.%-7d  %6.4f\n", docnum, indexnum, weight);
+  else
+    fprintf(output, "%7d.%-7d\n", docnum, indexnum);
+}
+#endif
+
+/* --
+   -- George Buchanan - August 1999
+   --
+
+      ProcessDocs prints out the results of a query to the output
+      stream, respecting the configuration choices for display
+      format etc.
+*/
+static int 
+ProcessDocs (query_data * qd, int num, int verbatim,
+	     char OutputType, FILE * Output)
+{
+  int max_buf = 0;
+  int DocCount = 0;
+  char *doc_sepstr = NULL;
+  char *para_sepstr = NULL;
+  char *para_start = NULL;
+  int heads_length = atoi (GetDefEnv ("heads_length", "50"));
+  char QueryType = get_query_type ();
+  int need_text = (OutputType == OUTPUT_TEXT || OutputType == OUTPUT_HILITE ||
+		   OutputType == OUTPUT_HEADERS || OutputType == OUTPUT_SILENT || 
+		   post_proc); /* [RJM June 1997 -- fixing post retrieval scan] */
+
+  if (OutputType == OUTPUT_TEXT || OutputType == OUTPUT_HILITE)
+    {
+      if (QueryType == QUERY_APPROX || QueryType == QUERY_RANKED)
+	{
+	  doc_sepstr = de_escape_string (
+				    Xstrdup (GetDefEnv ("ranked_doc_sepstr",
+			   "---------------------------------- %n %w\\n")));
+	}
+      else
+	{
+	  doc_sepstr = de_escape_string (
+					  Xstrdup (GetDefEnv ("doc_sepstr",
+			      "---------------------------------- %n\\n")));
+	}
+      para_sepstr = de_escape_string (
+				       Xstrdup (GetDefEnv ("para_sepstr",
+				  "\\n######## PARAGRAPH %n ########\\n")));
+
+      para_start = de_escape_string (
+				      Xstrdup (GetDefEnv ("para_start",
+					    "***** Weight = %w *****\\n")));
+    }
+
+  if (need_text)
+    {
+      max_buf = atoi (GetDefEnv ("buffer", "1048576"));
+    }
+
+  do
+    {
+      u_char *UDoc = NULL;
+      unsigned long ULen;
+
+      if (need_text)
+	{ /* load the compressed text */
+          int _status;
+
+	  _status = LoadCompressedText (qd, max_buf);
+	  if (_status == -2)
+	    { /**
+	       * GRB: fallback wihen text requested, but text was not
+               *      loaded
+	       * 17/09/99
+	       */
+	      Message("Compressed text not loaded - falling back to Document numbers.");
+	      UDoc = NULL;
+	      OutputType = OUTPUT_DOCNUMS;
+	    }
+	  else if (_status != 0)
+	    {
+	      Message ("Unable to load compressed text.");
+	      FatalError (1, "This is probably due to lack of memory.");
+	    }
+	  else
+	    {
+	      /* uncompress the loaded text */
+	      UDoc = GetDocText (qd, &ULen);
+	      if (UDoc == NULL)
+		{
+		  FatalError (1, "UDoc is unexpectedly NULL");
+		}
+	    }
+	}
+
+      if (!UDoc || PostProc ((char *) UDoc, verbatim))
+	{
+	  switch (OutputType)
+	    {
+	    case OUTPUT_COUNT:
+	    case OUTPUT_SILENT:
+	      break;
+	    case OUTPUT_DOCNUMS:	/* This prints out the docnums string */
+	      if (PagerRunning)
+		{
+
+/* [RPAP - Feb 97: NZDL Additions] */
+#if defined(PARADOCNUM) || defined(NZDL)
+		  int doc_num = GetDocNum(qd);
+
+		  if (qd->paragraph)
+		    {
+		      if (qd->id->ifh.InvfLevel == 3 &&
+			  (QueryType == 'R' || QueryType == 'A'))
+			{
+			  /* Print weights for each paragraph in document */
+			  
+
+			  int true_doc_num = GetDocNumFromParaNum(qd, doc_num);
+
+			  /* Get number of paragraphs in this document */
+
+			  int num_paragraphs = 
+			    qd->paragraph[true_doc_num]-qd->paragraph[true_doc_num-1];
+
+			  int init_para = FetchInitialParagraph(qd->td,
+								doc_num);
+			  DocEntry *de, *doc_chain = GetDocChain(qd);
+			  int i;
+
+			  for (i = 0; i < num_paragraphs; i++)
+			    {
+			      if ((de = in_chain(i, init_para, doc_chain)))
+				PrintDocNum(Output, QueryType,
+					    true_doc_num, init_para+i,
+					    de->docWeight());
+			    }
+			}
+		      else
+			PrintDocNum(Output, QueryType,
+				    GetDocNumFromParaNum(qd, GetDocNum(qd)),
+				    GetDocNum(qd),
+				    GetDocWeight(qd));
+		    }
+		  else
+		    {
+		      PrintDocNum(Output, QueryType,
+				  doc_num, doc_num, GetDocWeight(qd));
+		    }
+#else
+		fprintf (Output, "%8d   %6.4f   %7lu\n", GetDocNum (qd),
+			 GetDocWeight (qd), GetDocCompLength (qd));
+#endif
+		}
+	      break;
+	    case OUTPUT_HEADERS:	/* This prints out the headers of the documents */
+	      if (PagerRunning)
+		fprintf (Output, "%d ", GetDocNum (qd));
+	      HeaderOut (Output, UDoc, ULen, heads_length);
+	      if (PagerRunning)
+		fputc ('\n', Output);
+	      break;
+#if TREC_MODE
+	    case OUTPUT_EXTRAS:	/* This prints out the docnums string */
+	      if (PagerRunning && trec_ids)
+		{
+		  long DN, PN = GetDocNum (qd) - 1;
+		  if (trec_paras)
+		    DN = trec_paras[PN];
+		  else
+		    DN = PN;
+		  fprintf (Output, "%-14.14s  %8ld  %10.5f\n",
+			   &trec_ids[DN * 14], PN + 1, GetDocWeight (qd));
+		}
+	      break;
+#endif
+	    case OUTPUT_TEXT:
+	    case OUTPUT_HILITE:
+	      {
+		int j, para = -1, curr_para = 0;
+		int init_para = -1;
+		DocEntry *de, *doc_chain = NULL;
+		register char ch = ' ';
+		register char lch = '\n';
+
+/* [RPAP - Feb 97: NZDL Additions] */
+#if defined(PARADOCNUM) || defined(NZDL)
+		if (qd->id->ifh.InvfLevel == 3)
+		  {
+		    init_para = FetchInitialParagraph(qd->td, GetDocNum(qd));
+		    
+		    StringOut(Output, para_sepstr, 
+			      1, init_para+curr_para,
+			      0, 0);
+
+		  }
+		else
+		  StringOut(Output, doc_sepstr, 
+			    1, GetDocNum(qd),
+			    QueryType == 'A' || QueryType == 'R',
+			    GetDocWeight(qd));
+
+#else
+		int p_on = 0;
+
+		if (PagerRunning)
+		  {
+		    StringOut (Output, doc_sepstr,
+			       1, GetDocNum (qd),
+			       QueryType == 'A' || QueryType == 'R',
+			       GetDocWeight (qd));
+		  }
+		if (qd->id->ifh.InvfLevel == 3)
+		  {
+		    init_para = FetchInitialParagraph (qd->td, GetDocNum (qd));
+		    doc_chain = GetDocChain (qd);
+		    para = GetDocNum (qd) - init_para;
+
+		    StringOut (Output, para_sepstr,
+			       1, curr_para + 1,
+			       0, 0);
+
+		    if ((de = in_chain (0, init_para, doc_chain)))
+		      StringOut (Output, para_start,
+				 0, 0,
+				 1, de->docWeight());
+
+		    if (doc_chain->DocNum - init_para == 0)
+		      p_on = 1;
+		  }
+#endif
+		for (j = 0; j < ULen; j++)
+		  {
+		    ch = UDoc[j];
+		    switch (ch)
+		      {
+		      case '\02':
+			break;
+		      case '\01':
+			ch = '\n';
+		      case '\03':
+/* [RPAP - Feb 97: NZDL Additions] */
+#if defined(PARADOCNUM) || defined(NZDL)
+			/* print paragraph numbers only if this is
+			   a level 3 index */
+			if (qd->id->ifh.InvfLevel == 3)
+			  {
+			    curr_para++;
+			    StringOut(Output, para_sepstr, 
+				      1, init_para+curr_para,
+				      0, 0);
+			  }
+#else
+			p_on = 0;
+			curr_para++;
+			StringOut (Output, para_sepstr,
+				   1, curr_para + 1,
+				   0, 0);
+			lch = *(strchr (para_sepstr, '\0') - 1);
+			if ((de = in_chain (curr_para, init_para, doc_chain)))
+			  StringOut (Output, para_start,
+				     0, 0,
+				     1, de->docWeight());
+			if (doc_chain &&
+			    doc_chain->DocNum - init_para == curr_para)
+			  p_on = 1;
+#endif
+			break;
+		      default:
+			{
+			  if (PagerRunning)
+			    {
+			      fputc (ch, Output);
+/* [RPAP - Feb 97: NZDL Additions] */
+#if !defined(PARADOCNUM) && !defined(NZDL)
+			      if (p_on && isprint (ch))
+				{
+				  fputc ('\b', Output);
+				  fputc ('_', Output);
+				}
+#endif
+			    }
+
+			  lch = ch;
+			}
+		      }
+		  }
+		if (PagerRunning && lch != '\n')
+		  fputc ('\n', Output);
+/* [RPAP - Feb 97: NZDL Additions] */
+#if !defined(PARADOCNUM) && !defined(NZDL)
+		p_on = 0;
+#endif
+	      }
+	    }
+	  if (PagerRunning)
+	    fflush (Output);
+
+	  DocCount++; /* moved within if statement [RJM June 1997 -- fixing post retrieval scan] */
+	}
+    }
+  while (NextDoc (qd) && PagerRunning && (!Ctrl_C));
+
+  if (need_text)
+    {
+      QueryData_FreeTextBuffer (qd);
+    }
+
+  if (OutputType == OUTPUT_TEXT || OutputType == OUTPUT_HILITE)
+    {
+      Xfree (doc_sepstr);
+      Xfree (para_sepstr);
+      Xfree (para_start);
+    }
+
+  return (DocCount);
+}
+
+
+void 
+output_terminator (FILE * out)
+{
+  char *terminator = Xstrdup (GetDefEnv ("terminator", ""));
+  de_escape_string (terminator);
+  fputs (terminator, out);
+  Xfree (terminator);
+}
+
+
+
+
+/* MoreDocs () */
+/* Displays all documents in list  DocList. */
+/* Documents are fetched, then decompressed and displayed according to the */
+/* format implied in  FormString(). */
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+# define HILITE_PAGER "mg_hilite_words.exe"
+#else
+# define HILITE_PAGER "mg_hilite_words"
+#endif
+
+#define MAX_HILITE_PAGER_STR 80	/* for command & its options */
+
+static void 
+MoreDocs (query_data * qd, char *Query, char OutputType)
+{
+  static char terms_str[MAXTERMSTRLEN + 1];
+  int DocCount = 0;		/* number of actual matches */
+  FILE *Output = NULL;
+  int using_pipe = 0;
+  char *pager = NULL;
+
+  Ctrl_C = 0;
+
+  qd->num_of_ans = qd->DL->size();
+
+  /* [RPAP - Feb 97: WIN32 Port] */
+#ifndef __WIN32__
+  signal (SIGPIPE, SIGPIPE_handler);
+#endif
+  signal (SIGINT, SIGINT_handler);
+
+  PagerRunning = 1;
+  if (isatty (fileno (OutFile)) && GetEnv ("pager") &&
+      OutputType != OUTPUT_HILITE &&
+      OutputType != OUTPUT_SILENT && OutputType != OUTPUT_COUNT)
+    {
+      pager = GetEnv ("pager");
+    }
+  else if (isatty (fileno (OutFile)) && OutputType == OUTPUT_HILITE)
+    {
+      /* concat the pager and its word argument strings */
+      TermList_toString (qd->TL, terms_str);
+      pager = (char *) Xmalloc (MAX_HILITE_PAGER_STR + strlen (terms_str) + 1);
+      if (!pager)
+	{
+	  fprintf (stderr, "Unable to allocate memory for highlighting\n");
+	  return;
+	}
+      sprintf (pager, "%s --style=%s --pager=%s --stem_method=%ld --stemmer=%ld %s",
+	       HILITE_PAGER,
+	       GetEnv ("hilite_style"),
+	       GetEnv ("pager"),
+	       qd->sd->sdh.stem_method,
+	       qd->sd->sdh.stemmer_num,
+	       terms_str);
+
+    }
+  else
+    {
+      Output = OutFile;
+    }
+
+/* [RPAP - Feb 97: NZDL Additions] */
+#if defined(OUTPUTSTEMMEDWORDS) || defined(NZDL)
+  if (!isatty(fileno(OutFile)) && get_query_type() != QUERY_DOCNUMS)
+    {
+      TermList_toString(qd->TL, terms_str);
+      fprintf(Output, "%s\n", terms_str);
+    }
+#endif
+  if (pager)
+    {
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+      Output = _popen (pager, "w");
+#else
+      Output = (FILE *) popen (pager, "w");
+#endif
+      using_pipe = (Output != NULL);
+      if (!using_pipe)
+	{
+	  fprintf (stderr, "Unable to run \"%s\"\n", pager);
+	  return;
+	}
+    }
+
+
+  if (qd->DL->size() > 0)
+    {
+      if (OutputType == OUTPUT_COUNT && !post_proc)
+	DocCount = qd->DL->size();
+      else {
+	DocCount = ProcessDocs (qd, qd->DL->size(),
+				BooleanEnv (GetEnv ("verbatim"), 1),
+				OutputType, Output);
+      }
+    }
+
+  if (PagerRunning)
+    {
+      output_terminator (Output);
+      fflush (Output);
+    }
+
+  if (OutputType == OUTPUT_HILITE && pager)
+    free (pager);		/* as needed to malloc to create the pager string */
+
+  if (using_pipe)
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+    _pclose (Output);
+#else
+    pclose (Output);
+#endif
+
+  if (qd->DL->size() == 0)
+    fprintf (stderr, "No entries correspond to that query.\n");
+  else
+    {
+      if (OutputType == OUTPUT_COUNT)
+	fprintf (stderr, "%d documents match.\n", DocCount);
+      else
+	fprintf (stderr, "%d documents retrieved.\n", DocCount);
+    }
+
+  signal (SIGINT, SIG_DFL);
+}
+
+
+void 
+start_up_stats (query_data * qd, InitQueryTimes iqt)
+{
+  Clear_Stats ();
+  if (BooleanEnv (GetEnv ("timestats"), 0) ||
+      BooleanEnv (GetEnv ("briefstats"), 0))
+    StartUpTimeStats (&iqt);
+
+  if (BooleanEnv (GetEnv ("diskstats"), 0) ||
+      BooleanEnv (GetEnv ("briefstats"), 0))
+    File_Stats (qd);
+
+  if (BooleanEnv (GetEnv ("memstats"), 0) ||
+      BooleanEnv (GetEnv ("briefstats"), 0))
+    MemStats (qd);
+
+}
+
+
+void 
+shut_down_stats (query_data * qd, ProgTime * start,
+		 ProgTime * invf, ProgTime * text)
+{
+  Clear_Stats ();
+  if (BooleanEnv (GetEnv ("timestats"), 0) ||
+      BooleanEnv (GetEnv ("briefstats"), 0))
+    QueryTimeStats (start, invf, text);
+
+  if (BooleanEnv (GetEnv ("diskstats"), 0) ||
+      BooleanEnv (GetEnv ("briefstats"), 0))
+    {
+      TransFileStats (qd);
+      File_Stats (qd);
+    }
+
+  if (BooleanEnv (GetEnv ("sizestats"), 0))
+    TotalSizeStats (qd);
+}
+
+
+
+char *wordfreqword2str (u_char * s)
+{
+  static char buf[1024];
+  int i, len = (int) *s++;
+
+  for (i = 0; i < len; i++)
+    {
+      buf[i] = (char)s[i];
+    }
+  buf[len] = '\0';
+
+  return buf;
+}
+
+
+/* [RPAP - Feb 97: Term Frequency] */
+/*********************************
+ * PrintQueryTermFreq
+ *
+ * Prints the query terms and their respective frequencies within the collection
+ *********************************/
+void
+PrintQueryTermFreqs (QueryTermList *qtl)
+{
+  int i;
+
+  /* Print the number of terms */
+  fprintf (OutFile, "%d\n", qtl->num);
+
+  /* Print the terms and their respective frequency within the collection */
+  for (i = 0; i < qtl->num; i++)
+    if (qtl->QTE[i].stem_method == -1)
+      /* Using default stem method - don't print stem method beside term */
+      fprintf (OutFile, "%s %d\n", wordfreqword2str (qtl->QTE[i].Term), qtl->QTE[i].Count);
+    else
+      /* Term was forced with a stem, print stem method with term */
+      fprintf (OutFile, "%s#%d %d\n", wordfreqword2str (qtl->QTE[i].Term), qtl->QTE[i].stem_method, qtl->QTE[i].Count);
+}
+
+
+void 
+query (void)
+{
+  ProgTime TotalStartTime, TotalInvfTime, TotalTextTime;
+  InitQueryTimes iqt;
+  query_data *qd;
+
+  TotalStartTime.RealTime = TotalStartTime.CPUTime = 0;
+  TotalInvfTime.RealTime = TotalInvfTime.CPUTime = 0;
+  TotalTextTime.RealTime = TotalTextTime.CPUTime = 0;
+
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+  qd = InitQuerySystem (GetDefEnv ("mgdir", ".\\"),
+                        GetDefEnv ("mgname", ""),
+			GetDefEnv ("textname", NULL), /* [RJM 06/97: text filename] */
+			&iqt);
+#else
+  qd = InitQuerySystem (GetDefEnv ("mgdir", "./"),
+                        GetDefEnv ("mgname", ""),
+			GetDefEnv ("textname", NULL), /* [RJM 06/97: text filename] */
+                        &iqt);
+#endif
+
+  if (!qd)
+    FatalError (1, mg_errorstrs[mg_errno], mg_error_data);
+
+  start_up_stats (qd, iqt);
+
+  while (1)
+    {
+      ProgTime StartTime, InvfTime, TextTime;
+      char QueryType;
+      char OutputType;
+      char *line;
+      ResetFileStats (qd);
+      qd->max_mem_in_use = qd->mem_in_use = 0;
+
+      qd->tot_hops_taken += qd->hops_taken;
+      qd->tot_num_of_ptrs += qd->num_of_ptrs;
+      qd->tot_num_of_accum += qd->num_of_accum;
+      qd->tot_num_of_terms += qd->num_of_terms;
+      qd->tot_num_of_ans += qd->num_of_ans;
+      qd->tot_text_idx_lookups += qd->text_idx_lookups;
+      qd->hops_taken = qd->num_of_ptrs = 0;
+      qd->num_of_accum = qd->num_of_ans = qd->num_of_terms = 0;
+      qd->text_idx_lookups = 0;
+
+      Display_Stats (stderr);
+      Clear_Stats ();
+      line = get_query (qd);
+      if (!line || Quitting)
+	break;
+
+      GetPostProc (line);
+
+      GetTime (&StartTime);
+
+      QueryData_FreeQueryDocs (qd);
+
+      QueryType = get_query_type ();
+      OutputType = get_output_type ();
+      /* No point in hiliting words on a docnum query */
+      if (OutputType == OUTPUT_HILITE && QueryType == QUERY_DOCNUMS)
+	OutputType = OUTPUT_TEXT;
+
+      switch (QueryType)
+	{
+	case QUERY_BOOLEAN:
+	  {
+	    char *maxdocs;
+	    BooleanQueryInfo bqi;
+
+	    maxdocs = GetDefEnv ("maxdocs", "all");
+	    bqi.MaxDocsToRetrieve = strcmp (maxdocs, "all") ? atoi (maxdocs) : -1;
+
+	    /* [RPAP - Jan 97: Stem Index Change] */
+	    if (qd->sd->sdh.indexed)
+	      bqi.setWordActions(BooleanEnv(GetEnv("casefold"), 0),
+				 BooleanEnv(GetEnv("stem"), 0));
+	    else
+	      bqi.setWordActions(0, qd->sd->sdh.stem_method);
+
+	    bqi.OptimiseType = atoi (GetEnv ("optimise_type"));
+
+	    BooleanQuery (qd, line, &bqi);
+	    break;
+	  }
+	case QUERY_APPROX:
+	case QUERY_RANKED:
+	  {
+	    char *maxdocs;
+	    char *maxterms;
+	    char *maxaccum;
+	    RankedQueryInfo rqi;
+
+	    maxdocs = GetDefEnv ("maxdocs", "all");
+	    maxterms = GetDefEnv ("max_terms", "all");
+	    maxaccum = GetDefEnv ("max_accumulators", "all");
+	    rqi.Sort = BooleanEnv (GetEnv ("sorted_terms"), 0);
+	    rqi.QueryFreqs = BooleanEnv (GetEnv ("qfreq"), 1);
+	    rqi.Exact = QueryType == QUERY_RANKED;
+	    rqi.MaxDocsToRetrieve = strcmp (maxdocs, "all") ? atoi (maxdocs) : -1;
+	    rqi.MaxTerms = strcmp (maxterms, "all") ? atoi (maxterms) : -1;
+	    rqi.MaxParasToRetrieve = rqi.MaxDocsToRetrieve;
+	    if (qd->id->ifh.InvfLevel == 3 && GetEnv ("maxparas"))
+	      rqi.MaxParasToRetrieve = atoi (GetEnv ("maxparas"));
+	    rqi.AccumMethod = toupper (*GetDefEnv ("accumulator_method", "A"));
+	    rqi.MaxAccums = strcmp (maxaccum, "all") ? atoi (maxaccum) : -1;
+	    rqi.HashTblSize = IntEnv (GetEnv ("hash_tbl_size"), 1000);
+	    rqi.StopAtMaxAccum = BooleanEnv (GetEnv ("stop_at_max_accum"), 0);
+	    rqi.skip_dump = GetEnv ("skip_dump");
+	    rqi.setWordActions(BooleanEnv(GetEnv("casefold"), 0),
+			       BooleanEnv(GetEnv("stem"), 0));
+	    RankedQuery (qd, line, &rqi);
+	    break;
+	  }
+	case QUERY_DOCNUMS:
+	  {
+	    DocnumsQuery (qd, line);
+	    break;
+	  }
+	}
+
+      GetTime (&InvfTime);
+
+      /* [RPAP - Feb 97: Term Frequency] */
+      if (qd->QTL && BooleanEnv (GetEnv ("term_freq"), 0))
+	PrintQueryTermFreqs (qd->QTL);
+
+      if (qd->DL)
+	MoreDocs (qd, line, OutputType);
+
+      GetTime (&TextTime);
+
+      if (BooleanEnv (GetEnv ("timestats"), 0) ||
+	  BooleanEnv (GetEnv ("briefstats"), 0))
+	QueryTimeStats (&StartTime, &InvfTime, &TextTime);
+
+      if (BooleanEnv (GetEnv ("diskstats"), 0) ||
+	  BooleanEnv (GetEnv ("briefstats"), 0))
+	File_Stats (qd);
+
+      if (BooleanEnv (GetEnv ("memstats"), 0) ||
+	  BooleanEnv (GetEnv ("briefstats"), 0))
+	MemStats (qd);
+
+      if (BooleanEnv (GetEnv ("sizestats"), 0))
+	SizeStats (qd);
+
+      TotalInvfTime.RealTime += InvfTime.RealTime - StartTime.RealTime;
+      TotalInvfTime.CPUTime += InvfTime.CPUTime - StartTime.CPUTime;
+      TotalTextTime.RealTime += TextTime.RealTime - StartTime.RealTime;
+      TotalTextTime.CPUTime += TextTime.CPUTime - StartTime.CPUTime;
+    }
+
+  if (isatty (fileno (InFile)) && !Quitting)
+    fprintf (stderr, "\n");
+
+  shut_down_stats (qd, &TotalStartTime, &TotalInvfTime, &TotalTextTime);
+
+  Display_Stats (stderr);
+
+}
+
+
+void 
+search_for_collection (char *name)
+{
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+  char *dir = GetDefEnv ("mgdir", ".\\");
+#else
+  char *dir = GetDefEnv ("mgdir", "./");
+#endif
+  char buffer[512];
+  struct stat stat_buf;
+  if (strrchr (dir, '/') && *(strrchr (dir, '/') + 1) != '\0')
+    {
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+      sprintf (buffer, "%s", dir);
+#else
+      sprintf (buffer, "%s/", dir);
+#endif
+      SetEnv ("mgdir", buffer, NULL);
+      dir = GetEnv ("mgdir");
+    }
+
+  sprintf (buffer, "%s.text", name);
+  if (stat (buffer, &stat_buf) != -1)
+    {
+      if (S_ISREG(stat_buf.st_mode))
+	{
+	  /* The name is a regular file */
+	  SetEnv ("mgname", name, NULL);
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+          SetEnv ("mgdir", ".\\", NULL);
+#else
+	  SetEnv ("mgdir", "./", NULL);
+#endif
+	  return;
+	}
+    }
+
+  sprintf (buffer, "%s%s", dir, name);
+  if (stat (buffer, &stat_buf) != -1)
+    {
+      if (S_ISDIR(stat_buf.st_mode))
+	{
+	  /* The name is a directory */
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+          sprintf (buffer, "%s%s", name, name);
+#else
+	  sprintf (buffer, "%s/%s", name, name);
+#endif
+	  SetEnv ("mgname", buffer, NULL);
+	  return;
+	}
+    }
+
+  /* Look in the current directory last */
+  if (stat (name, &stat_buf) != -1)
+    {
+      if (S_ISDIR(stat_buf.st_mode))
+	{
+	  /* The name is a directory */
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+          sprintf (buffer, "%s%s", name, name);
+          SetEnv ("mgdir", ".\\", NULL);
+#else
+	  sprintf (buffer, "%s/%s", name, name);
+          SetEnv ("mgdir", "./", NULL);
+#endif
+	  SetEnv ("mgname", buffer, NULL);
+	  return;
+	}
+    }
+
+  SetEnv ("mgname", name, NULL);
+}
+
+/* main () */
+/* Initialises global variables based on command line switches, and opens */
+/* files.  Then calls  query ()  to perform the querying. */
+int main (int argc, char **argv)
+{
+  ProgTime StartTime;
+  int decomp = 0;
+  int ch;
+
+  msg_prefix = argv[0];
+  GetTime (&StartTime);
+
+  /* Initialise the environment with default values */
+
+  InitEnv ();
+
+  read_mgrc_file ();
+
+  OutFile = stdout;
+  InFile = stdin;
+
+  opterr = 0;
+  /* [RJM 06/97: text filename] */
+  while ((ch = getopt (argc, argv, "Df:d:t:h")) != -1) {
+    switch (ch) {
+      case 'f':
+	SetEnv ("mgname", optarg, NULL);
+	break;
+      case 'd':
+	SetEnv ("mgdir", optarg, NULL);
+	break;
+      case 't': /* [RJM 06/97: text filename] */
+	SetEnv ("textname", optarg, NULL);
+	break;
+      case 'D':
+	decomp = 1;
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-D] [-f base name of collection] "
+		 "[-t base name of files for text] " /* [RJM 06/97: text filename] */
+		 "[-d data directory] [collection]\n", argv[0]);
+	exit (1);
+    }      
+  }
+
+  PushEnv ();
+
+  if (decomp == 0)
+    {
+
+      Init_ReadLine ();
+
+      /* write a first prompt, let the user start thinking */
+      if (!BooleanEnv (GetEnv ("expert"), 0) && isatty (fileno (InFile)))
+	{
+	  fprintf (stderr, "\n\n\t     FULL TEXT RETRIEVAL QUERY PROGRAM\n");
+	  fprintf (stderr, "%24s%s\n\n", "", *"21 Mar 1994" == '%' ? __DATE__ : "21 Mar 1994");
+	  fprintf (stderr, "\n");
+	  fprintf (stderr, "  mgquery version " VERSION ", Copyright (C) 1994 Neil Sharman\n");
+	  fprintf (stderr, "  mgquery comes with ABSOLUTELY NO WARRANTY; for details type `.warranty'\n");
+	  fprintf (stderr, "  This is free software, and you are welcome to redistribute it\n");
+	  fprintf (stderr, "  under certain conditions; type `.conditions' for details.\n");
+	  fprintf (stderr, "\n");
+	}
+    }
+  if (optind < argc)
+    search_for_collection (argv[optind]);
+
+  if (decomp == 0)
+    { 
+      query ();
+    }
+  else
+    { int i;
+      InitQueryTimes iqt;
+      query_data *qd;
+      
+/* [RPAP - Feb 97: WIN32 Port] */
+#ifdef __WIN32__
+      qd = InitQuerySystem (GetDefEnv ("mgdir", ".\\"),
+                            GetDefEnv ("mgname", ""),
+                            GetDefEnv ("textname", NULL), /* [RJM 06/97: text filename] */
+                            &iqt);
+#else
+      qd = InitQuerySystem (GetDefEnv ("mgdir", "./"),
+			    GetDefEnv ("mgname", ""),
+                            GetDefEnv ("textname", NULL), /* [RJM 06/97: text filename] */
+			    &iqt);
+#endif
+      if (!qd)
+	FatalError (1, mg_errorstrs[mg_errno], mg_error_data);
+
+
+      start_up_stats (qd, iqt);
+
+      Display_Stats (stderr);
+      for (i = 0; i < qd->td->cth.num_of_docs; i++)
+	{
+	  RawDocOutput (qd, i + 1, stdout);
+	  putc ('\2', stdout);
+	}
+      Message ("%s", ElapsedTime (&StartTime, NULL));
+
+      FinishQuerySystem (qd);
+    }
+
+  UninitEnv ();
+  return 0;
+}
Index: /tags/gsdl-21/gsdl/src/mgpp/text/mgquery.pp
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/mgquery.pp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/mgquery.pp	(revision 699)
@@ -0,0 +1,12394 @@
+# 1 "grbTest.cpp"
+ 
+ 
+# 1 "../../lib/sysfuncs.h" 1
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+# 1 "/usr/local/include/sys/stat.h" 1 3
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+# 1 "/usr/local/include/features.h" 1 3
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+ 
+
+
+
+ 
+
+
+
+
+
+
+
+ 
+# 137 "/usr/local/include/features.h" 3
+
+
+ 
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+ 
+
+# 165 "/usr/local/include/features.h" 3
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 195 "/usr/local/include/features.h" 3
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+ 
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+ 
+
+# 1 "/usr/local/include/sys/cdefs.h" 1 3
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+ 
+
+
+
+# 47 "/usr/local/include/sys/cdefs.h" 3
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 69 "/usr/local/include/sys/cdefs.h" 3
+
+
+
+
+ 
+
+
+
+
+
+
+
+ 
+
+
+
+# 103 "/usr/local/include/sys/cdefs.h" 3
+
+
+
+ 
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+# 138 "/usr/local/include/sys/cdefs.h" 3
+
+
+ 
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+# 249 "/usr/local/include/features.h" 2 3
+
+
+ 
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+ 
+
+ 
+
+
+
+
+
+
+
+
+# 1 "/usr/local/include/gnu/stubs.h" 1 3
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 277 "/usr/local/include/features.h" 2 3
+
+
+
+
+# 26 "/usr/local/include/sys/stat.h" 2 3
+
+
+# 1 "/usr/local/include/bits/types.h" 1 3
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+# 1 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 1 3
+
+
+
+
+
+
+ 
+
+
+# 19 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+
+ 
+
+
+ 
+
+
+
+
+
+ 
+
+
+# 61 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+ 
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+ 
+
+ 
+
+# 131 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+ 
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+typedef unsigned int size_t;
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+ 
+
+# 271 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+# 283 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+ 
+
+ 
+
+# 317 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 29 "/usr/local/include/bits/types.h" 2 3
+
+
+ 
+typedef unsigned char __u_char;
+typedef unsigned short __u_short;
+typedef unsigned int __u_int;
+typedef unsigned long __u_long;
+
+
+
+
+typedef struct
+  {
+    long int __val[2];
+  } __quad_t;
+typedef struct
+  {
+    __u_long __val[2];
+  } __u_quad_t;
+
+typedef signed char __int8_t;
+typedef unsigned char __uint8_t;
+typedef signed short int __int16_t;
+typedef unsigned short int __uint16_t;
+typedef signed int __int32_t;
+typedef unsigned int __uint32_t;
+
+
+
+
+typedef __quad_t *__qaddr_t;
+
+typedef __u_quad_t __dev_t;		 
+typedef __u_int __uid_t;		 
+typedef __u_int __gid_t;		 
+typedef __u_long __ino_t;		 
+typedef __u_int __mode_t;		 
+typedef __u_int __nlink_t; 		 
+typedef long int __off_t;		 
+typedef __quad_t __loff_t;		 
+typedef int __pid_t;			 
+typedef int __ssize_t;			 
+typedef long int __rlim_t;		 
+typedef __quad_t __rlim64_t;		 
+typedef __u_int __id_t;			 
+
+typedef struct
+  {
+    int __val[2];
+  } __fsid_t;				 
+
+ 
+typedef int __daddr_t;			 
+typedef char *__caddr_t;
+typedef long int __time_t;
+typedef long int __swblk_t;		 
+
+
+typedef long int __clock_t;
+
+ 
+typedef unsigned long int __fd_mask;
+
+ 
+
+
+ 
+
+
+
+
+ 
+typedef struct
+  {
+     
+
+
+
+
+
+    __fd_mask __fds_bits[1024  / (8 * sizeof (__fd_mask)) ];
+
+
+  } __fd_set;
+
+
+typedef int __key_t;
+
+ 
+typedef unsigned short int __ipc_pid_t;
+
+
+ 
+
+ 
+typedef long int __blkcnt_t;
+typedef __quad_t __blkcnt64_t;
+
+ 
+typedef __u_long __fsblkcnt_t;
+typedef __u_quad_t __fsblkcnt64_t;
+
+ 
+typedef __u_long __fsfilcnt_t;
+typedef __u_quad_t __fsfilcnt64_t;
+
+ 
+typedef __u_long __ino64_t;
+
+ 
+typedef __loff_t __off64_t;
+
+ 
+typedef int __t_scalar_t;
+typedef unsigned int __t_uscalar_t;
+
+ 
+typedef int __intptr_t;
+
+
+ 
+
+
+
+
+
+# 28 "/usr/local/include/sys/stat.h" 2 3
+
+
+# 78 "/usr/local/include/sys/stat.h" 3
+
+
+
+
+
+
+
+
+
+extern "C" { 
+
+# 1 "/usr/local/include/bits/stat.h" 1 3
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+ 
+
+
+
+
+
+struct stat
+  {
+    __dev_t st_dev;			 
+    unsigned short int __pad1;
+
+    __ino_t st_ino;			 
+
+
+
+    __mode_t st_mode;			 
+    __nlink_t st_nlink;			 
+    __uid_t st_uid;			 
+    __gid_t st_gid;			 
+    __dev_t st_rdev;			 
+    unsigned short int __pad2;
+
+    __off_t st_size;			 
+
+
+
+    unsigned long int st_blksize;	 
+
+
+    __blkcnt_t st_blocks;		 
+
+
+
+    __time_t st_atime;			 
+    unsigned long int __unused1;
+    __time_t st_mtime;			 
+    unsigned long int __unused2;
+    __time_t st_ctime;			 
+    unsigned long int __unused3;
+    unsigned long int __unused4;
+    unsigned long int __unused5;
+  };
+
+# 99 "/usr/local/include/bits/stat.h" 3
+
+
+ 
+
+
+
+ 
+
+
+
+ 
+
+
+
+
+
+
+ 
+
+
+
+ 
+
+
+
+
+
+
+
+# 89 "/usr/local/include/sys/stat.h" 2 3
+
+
+# 108 "/usr/local/include/sys/stat.h" 3
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+# 131 "/usr/local/include/sys/stat.h" 3
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+extern int stat  (const  char *__file, struct stat *__buf)  ;
+
+ 
+
+extern int fstat  (int __fd, struct stat *__buf)  ;
+# 195 "/usr/local/include/sys/stat.h" 3
+
+
+
+
+
+
+# 217 "/usr/local/include/sys/stat.h" 3
+
+
+ 
+
+
+extern int chmod  (const  char *__file, __mode_t __mode)  ;
+
+ 
+
+
+
+
+
+ 
+
+extern __mode_t umask  (__mode_t __mask)  ;
+
+
+
+
+
+
+
+ 
+extern int mkdir  (const  char *__path, __mode_t __mode)  ;
+
+ 
+
+
+
+
+
+
+
+
+ 
+extern int mkfifo  (const  char *__path, __mode_t __mode)  ;
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+extern int __fxstat  (int __ver, int __fildes,
+			  struct stat *__stat_buf)  ;
+extern int __xstat  (int __ver, const  char *__filename,
+			 struct stat *__stat_buf)  ;
+extern int __lxstat  (int __ver, const  char *__filename,
+			  struct stat *__stat_buf)  ;
+# 299 "/usr/local/include/sys/stat.h" 3
+
+
+
+
+
+
+
+
+
+
+extern int __xmknod  (int __ver, const  char *__path,
+			  __mode_t __mode, __dev_t *__dev)  ;
+
+# 363 "/usr/local/include/sys/stat.h" 3
+
+
+} 
+
+
+
+# 28 "../../lib/sysfuncs.h" 2
+
+
+ 
+
+
+
+
+# 1 "../../config.h" 1
+ 
+ 
+
+ 
+
+
+
+ 
+
+
+ 
+ 
+
+ 
+ 
+
+ 
+
+ 
+
+ 
+
+
+ 
+
+
+ 
+ 
+
+ 
+
+
+ 
+ 
+
+ 
+ 
+
+ 
+
+ 
+
+ 
+ 
+
+ 
+
+
+ 
+ 
+
+ 
+
+
+
+
+
+
+ 
+
+ 
+
+
+ 
+
+
+ 
+ 
+
+ 
+
+
+ 
+
+
+ 
+
+
+ 
+
+
+ 
+ 
+
+ 
+
+ 
+ 
+ 
+
+ 
+ 
+
+ 
+
+
+
+ 
+ 
+ 
+
+ 
+ 
+
+ 
+ 
+
+ 
+
+
+ 
+
+
+ 
+
+
+ 
+
+
+ 
+
+
+ 
+
+
+ 
+
+
+ 
+
+
+ 
+
+
+ 
+
+
+ 
+
+
+ 
+
+
+ 
+
+
+ 
+
+
+ 
+
+
+ 
+
+
+ 
+
+
+ 
+
+
+ 
+ 
+
+ 
+
+
+ 
+ 
+
+ 
+ 
+
+ 
+
+
+ 
+
+
+ 
+
+
+ 
+
+# 35 "../../lib/sysfuncs.h" 2
+
+
+
+
+ 
+
+
+
+
+
+# 1 "/usr/local/include/alloca.h" 1 3
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 1 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 1 3
+
+
+
+
+
+
+ 
+
+
+# 19 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+
+ 
+
+
+ 
+
+
+
+
+
+ 
+
+
+# 61 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+ 
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+ 
+
+ 
+
+# 131 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+ 
+
+ 
+
+
+# 188 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+
+
+
+ 
+
+
+
+
+ 
+
+# 271 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+# 283 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+ 
+
+ 
+
+# 317 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 25 "/usr/local/include/alloca.h" 2 3
+
+
+extern "C" { 
+
+ 
+
+
+ 
+extern void *  alloca  (size_t __size)  ;
+
+
+
+
+
+} 
+
+
+# 45 "../../lib/sysfuncs.h" 2
+
+# 54 "../../lib/sysfuncs.h"
+
+
+
+# 1 "/usr/local/include/sys/types.h" 1 3
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+extern "C" { 
+
+
+
+# 40 "/usr/local/include/sys/types.h" 3
+
+
+typedef __loff_t loff_t;
+
+
+
+typedef __ino_t ino_t;
+
+
+
+
+
+
+
+
+
+
+typedef __dev_t dev_t;
+
+
+
+
+typedef __gid_t gid_t;
+
+
+
+
+typedef __mode_t mode_t;
+
+
+
+
+typedef __nlink_t nlink_t;
+
+
+
+
+typedef __uid_t uid_t;
+
+
+
+
+
+typedef __off_t off_t;
+
+
+
+
+
+
+
+
+
+
+
+typedef __pid_t pid_t;
+
+
+
+
+
+
+
+
+typedef __ssize_t ssize_t;
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 1 "/usr/local/include/time.h" 1 3
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 51 "/usr/local/include/time.h" 3
+
+
+
+# 62 "/usr/local/include/time.h" 3
+
+
+
+
+
+
+
+
+ 
+typedef __time_t time_t;
+
+
+
+
+
+# 89 "/usr/local/include/time.h" 3
+
+
+
+
+# 279 "/usr/local/include/time.h" 3
+
+
+
+# 121 "/usr/local/include/sys/types.h" 2 3
+
+
+
+# 1 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 1 3
+
+
+
+
+
+
+ 
+
+
+# 19 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+
+ 
+
+
+ 
+
+
+
+
+
+ 
+
+
+# 61 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+ 
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+ 
+
+ 
+
+# 131 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+ 
+
+ 
+
+
+# 188 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+
+
+
+ 
+
+
+
+
+ 
+
+# 271 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+# 283 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+ 
+
+ 
+
+# 317 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 124 "/usr/local/include/sys/types.h" 2 3
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+ 
+
+
+typedef	char int8_t;
+typedef	short int int16_t;
+typedef	int int32_t;
+
+
+
+
+
+ 
+typedef	unsigned char u_int8_t;
+typedef	unsigned short int u_int16_t;
+typedef	unsigned int u_int32_t;
+
+
+
+
+typedef int register_t;
+
+# 184 "/usr/local/include/sys/types.h" 3
+
+
+
+
+# 197 "/usr/local/include/sys/types.h" 3
+
+
+
+ 
+
+typedef __blkcnt_t blkcnt_t;	  
+typedef __fsblkcnt_t fsblkcnt_t;  
+typedef __fsfilcnt_t fsfilcnt_t;  
+
+
+
+
+
+
+
+
+
+
+
+
+} 
+
+
+# 57 "../../lib/sysfuncs.h" 2
+
+
+ 
+ 
+
+
+# 1 "/usr/local/include/stdlib.h" 1 3
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+# 1 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 1 3
+
+
+
+
+
+
+ 
+
+
+# 19 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+
+ 
+
+
+ 
+
+
+
+
+
+ 
+
+
+# 61 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+ 
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+ 
+
+ 
+
+# 131 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+ 
+
+ 
+
+
+# 188 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+
+
+
+ 
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 283 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+ 
+
+ 
+
+# 317 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 33 "/usr/local/include/stdlib.h" 2 3
+
+
+extern "C" { 
+
+
+
+
+ 
+typedef struct
+  {
+    int quot;			 
+    int rem;			 
+  } div_t;
+
+ 
+
+typedef struct
+  {
+    long int quot;		 
+    long int rem;		 
+  } ldiv_t;
+
+
+
+# 65 "/usr/local/include/stdlib.h" 3
+
+
+
+ 
+
+
+
+ 
+
+
+
+
+
+ 
+
+extern size_t __ctype_get_mb_cur_max  (void)  ;
+
+
+ 
+extern double atof  (const  char *__nptr)  ;
+ 
+extern int atoi  (const  char *__nptr)  ;
+ 
+extern long int atol  (const  char *__nptr)  ;
+
+
+
+
+
+
+ 
+extern double strtod  (const  char *   __nptr,
+			   char **   __endptr)  ;
+
+
+
+
+
+
+
+
+
+
+ 
+extern long int strtol  (const  char *   __nptr,
+			     char **   __endptr, int __base)  ;
+ 
+extern unsigned long int strtoul  (const  char *   __nptr,
+				       char **   __endptr,
+				       int __base)  ;
+
+# 126 "/usr/local/include/stdlib.h" 3
+
+
+# 140 "/usr/local/include/stdlib.h" 3
+
+
+
+# 190 "/usr/local/include/stdlib.h" 3
+
+
+
+ 
+
+
+extern double __strtod_internal  (const  char *   __nptr,
+				      char **   __endptr,
+				      int __group)  ;
+extern float __strtof_internal  (const  char *   __nptr,
+				     char **   __endptr, int __group)  ;
+extern long double  __strtold_internal  (const  char *
+						  __nptr,
+						char **   __endptr,
+						int __group)  ;
+
+extern long int __strtol_internal  (const  char *   __nptr,
+					char **   __endptr,
+					int __base, int __group)  ;
+
+
+
+extern unsigned long int __strtoul_internal  (const  char *
+						    __nptr,
+						  char **   __endptr,
+						  int __base, int __group)  ;
+
+
+# 236 "/usr/local/include/stdlib.h" 3
+
+
+# 326 "/usr/local/include/stdlib.h" 3
+
+
+
+# 392 "/usr/local/include/stdlib.h" 3
+
+
+
+ 
+extern int rand  (void)  ;
+ 
+extern void srand  (unsigned int __seed)  ;
+
+
+
+
+
+
+
+# 467 "/usr/local/include/stdlib.h" 3
+
+
+
+
+
+
+ 
+extern void *  malloc  (size_t __size)  ;
+ 
+extern void *  calloc  (size_t __nmemb, size_t __size)  ;
+
+
+
+ 
+
+extern void *  realloc  (void *  __ptr, size_t __size)  ;
+ 
+extern void free  (void *  __ptr)  ;
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+extern void abort  (void)    ;
+
+
+ 
+extern int atexit  (void (*__func) (void))  ;
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+extern void exit  (int __status)    ;
+
+
+
+
+
+
+
+
+ 
+extern char *getenv  (const  char *__name)  ;
+
+ 
+
+extern char *__secure_getenv  (const  char *__name)  ;
+
+
+
+
+
+
+
+
+# 551 "/usr/local/include/stdlib.h" 3
+
+
+
+
+
+
+
+
+
+
+# 574 "/usr/local/include/stdlib.h" 3
+
+
+
+ 
+extern int system  (const  char *__command)  ;
+
+
+
+
+
+
+
+
+
+# 597 "/usr/local/include/stdlib.h" 3
+
+
+
+ 
+
+
+typedef int (*__compar_fn_t)  (const  void * , const  void * )  ;
+
+
+
+
+
+
+ 
+
+extern void *  bsearch  (const  void *  __key, const  void *  __base,
+			       size_t __nmemb, size_t __size,
+			       __compar_fn_t __compar)  ;
+
+ 
+
+extern void qsort  (void *  __base, size_t __nmemb, size_t __size,
+			  __compar_fn_t __compar)  ;
+
+
+ 
+extern int abs  (int __x)    ;
+extern long int labs  (long int __x)    ;
+
+
+
+
+
+
+ 
+
+ 
+extern div_t div  (int __numer, int __denom)    ;
+extern ldiv_t ldiv  (long int __numer, long int __denom)  
+      ;
+
+
+
+
+
+
+
+# 690 "/usr/local/include/stdlib.h" 3
+
+
+
+ 
+
+extern int mblen  (const  char *__s, size_t __n)  ;
+ 
+
+extern int mbtowc  (wchar_t *   __pwc,
+			const  char *   __s, size_t __n)  ;
+ 
+
+extern int wctomb  (char *__s, wchar_t __wchar)  ;
+
+
+ 
+extern size_t mbstowcs  (wchar_t *    __pwcs,
+			     const  char *   __s, size_t __n)  ;
+ 
+extern size_t wcstombs  (char *   __s,
+			     const  wchar_t *   __pwcs, size_t __n)  ;
+
+
+
+
+
+
+
+
+
+
+
+# 732 "/usr/local/include/stdlib.h" 3
+
+
+
+# 756 "/usr/local/include/stdlib.h" 3
+
+
+# 766 "/usr/local/include/stdlib.h" 3
+
+
+
+
+
+} 
+
+
+# 63 "../../lib/sysfuncs.h" 2
+
+# 1 "/usr/local/include/math.h" 1 3
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+extern "C" { 
+
+ 
+
+# 1 "/usr/local/include/bits/huge_val.h" 1 3
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+static union { unsigned char __c[8]; double __d; }  __huge_val = { { 0, 0, 0, 0, 0, 0, 0xf0, 0x7f }  };
+
+
+
+
+ 
+
+# 68 "/usr/local/include/bits/huge_val.h" 3
+
+# 33 "/usr/local/include/math.h" 2 3
+
+
+ 
+
+
+
+ 
+# 1 "/usr/local/include/bits/mathdef.h" 1 3
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 45 "/usr/local/include/bits/mathdef.h" 3
+
+# 40 "/usr/local/include/math.h" 2 3
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 1 "/usr/local/include/bits/mathcalls.h" 1 3
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+ 
+extern   double          acos          (double  __x)    ; extern   double         __acos          (double  __x)      ;
+ 
+extern   double          asin          (double  __x)    ; extern   double         __asin          (double  __x)      ;
+ 
+extern   double          atan          (double  __x)    ; extern   double         __atan          (double  __x)      ;
+ 
+extern   double          atan2          (double  __y, double  __x)    ; extern   double         __atan2          (double  __y, double  __x)      ;
+
+ 
+extern   double          cos          (double  __x)    ; extern   double         __cos          (double  __x)      ;
+ 
+extern   double          sin          (double  __x)    ; extern   double         __sin          (double  __x)      ;
+ 
+extern   double          tan          (double  __x)    ; extern   double         __tan          (double  __x)      ;
+
+
+
+
+
+
+
+ 
+
+ 
+extern   double          cosh          (double  __x)    ; extern   double         __cosh          (double  __x)      ;
+ 
+extern   double          sinh          (double  __x)    ; extern   double         __sinh          (double  __x)      ;
+ 
+extern   double          tanh          (double  __x)    ; extern   double         __tanh          (double  __x)      ;
+
+
+
+
+
+
+
+
+
+
+ 
+
+ 
+extern   double          exp          (double  __x)    ; extern   double         __exp          (double  __x)      ;
+
+
+
+
+
+
+
+
+ 
+extern   double          frexp          (double  __x, int *__exponent)    ; extern   double         __frexp          (double  __x, int *__exponent)      ;
+
+ 
+extern   double          ldexp          (double  __x, int __exponent)    ; extern   double         __ldexp          (double  __x, int __exponent)      ;
+
+ 
+extern   double          log          (double  __x)    ; extern   double         __log          (double  __x)      ;
+
+ 
+extern   double          log10          (double  __x)    ; extern   double         __log10          (double  __x)      ;
+
+ 
+extern   double          modf          (double  __x, double  *__iptr)    ; extern   double         __modf          (double  __x, double  *__iptr)      ;
+
+# 129 "/usr/local/include/bits/mathcalls.h" 3
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+ 
+extern   double          pow          (double  __x, double  __y)    ; extern   double         __pow          (double  __x, double  __y)      ;
+
+ 
+extern   double          sqrt          (double  __x)    ; extern   double         __sqrt          (double  __x)      ;
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+ 
+extern   double          ceil          (double  __x)    ; extern   double         __ceil          (double  __x)      ;
+
+ 
+extern   double          fabs          (double  __x)      ; extern   double         __fabs          (double  __x)        ;
+
+ 
+extern   double          floor          (double  __x)    ; extern   double         __floor          (double  __x)      ;
+
+ 
+extern   double          fmod          (double  __x, double  __y)    ; extern   double         __fmod          (double  __x, double  __y)      ;
+
+
+ 
+
+extern  int     __isinf      (double  __value)    ;
+
+# 200 "/usr/local/include/bits/mathcalls.h" 3
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 224 "/usr/local/include/bits/mathcalls.h" 3
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 272 "/usr/local/include/bits/mathcalls.h" 3
+
+
+# 330 "/usr/local/include/bits/mathcalls.h" 3
+
+# 63 "/usr/local/include/math.h" 2 3
+
+
+
+
+# 105 "/usr/local/include/math.h" 3
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+# 232 "/usr/local/include/math.h" 3
+
+
+# 249 "/usr/local/include/math.h" 3
+
+
+
+# 291 "/usr/local/include/math.h" 3
+
+
+
+
+
+
+
+
+
+
+
+ 
+# 317 "/usr/local/include/math.h" 3
+
+
+ 
+
+
+# 336 "/usr/local/include/math.h" 3
+
+
+
+ 
+
+
+
+
+
+
+ 
+
+
+
+
+
+# 407 "/usr/local/include/math.h" 3
+
+
+} 
+
+
+
+# 64 "../../lib/sysfuncs.h" 2
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+# 1 "/usr/local/include/assert.h" 1 3
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+# 32 "/usr/local/include/assert.h" 3
+
+
+
+
+
+ 
+
+
+
+
+# 56 "/usr/local/include/assert.h" 3
+
+
+extern "C" { 
+
+ 
+extern void __assert_fail  (const  char *__assertion,
+				const  char *__file,
+				unsigned int __line,
+				const  char *__function)  
+      ;
+
+ 
+extern void __assert_perror_fail  (int __errnum,
+				       const  char *__file,
+				       unsigned int __line,
+				       const  char *__function)  
+      ;
+
+} 
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 85 "../../lib/sysfuncs.h" 2
+
+
+ 
+
+ 
+
+
+
+
+
+
+ 
+
+# 1 "/usr/local/include/ctype.h" 1 3
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+extern "C" { 
+
+
+ 
+
+
+
+
+
+
+
+# 1 "/usr/local/include/endian.h" 1 3
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+ 
+# 1 "/usr/local/include/bits/endian.h" 1 3
+ 
+
+
+
+
+
+
+# 35 "/usr/local/include/endian.h" 2 3
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 40 "/usr/local/include/ctype.h" 2 3
+
+
+
+
+
+
+
+enum
+{
+  _ISupper = (( 0 ) < 8 ? ((1 << ( 0 )) << 8) : ((1 << ( 0 )) >> 8)) ,	 
+  _ISlower = (( 1 ) < 8 ? ((1 << ( 1 )) << 8) : ((1 << ( 1 )) >> 8)) ,	 
+  _ISalpha = (( 2 ) < 8 ? ((1 << ( 2 )) << 8) : ((1 << ( 2 )) >> 8)) ,	 
+  _ISdigit = (( 3 ) < 8 ? ((1 << ( 3 )) << 8) : ((1 << ( 3 )) >> 8)) ,	 
+  _ISxdigit = (( 4 ) < 8 ? ((1 << ( 4 )) << 8) : ((1 << ( 4 )) >> 8)) ,	 
+  _ISspace = (( 5 ) < 8 ? ((1 << ( 5 )) << 8) : ((1 << ( 5 )) >> 8)) ,	 
+  _ISprint = (( 6 ) < 8 ? ((1 << ( 6 )) << 8) : ((1 << ( 6 )) >> 8)) ,	 
+  _ISgraph = (( 7 ) < 8 ? ((1 << ( 7 )) << 8) : ((1 << ( 7 )) >> 8)) ,	 
+  _ISblank = (( 8 ) < 8 ? ((1 << ( 8 )) << 8) : ((1 << ( 8 )) >> 8)) ,	 
+  _IScntrl = (( 9 ) < 8 ? ((1 << ( 9 )) << 8) : ((1 << ( 9 )) >> 8)) ,	 
+  _ISpunct = (( 10 ) < 8 ? ((1 << ( 10 )) << 8) : ((1 << ( 10 )) >> 8)) ,	 
+  _ISalnum = (( 11 ) < 8 ? ((1 << ( 11 )) << 8) : ((1 << ( 11 )) >> 8)) 	 
+};
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+extern const  unsigned short int *__ctype_b;	 
+extern const  __int32_t *__ctype_tolower;  
+extern const  __int32_t *__ctype_toupper;  
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+extern int  isalnum   (int)   ;
+extern int  isalpha   (int)   ;
+extern int  iscntrl   (int)   ;
+extern int  isdigit   (int)   ;
+extern int  islower   (int)   ;
+extern int  isgraph   (int)   ;
+extern int  isprint   (int)   ;
+extern int  ispunct   (int)   ;
+extern int  isspace   (int)   ;
+extern int  isupper   (int)   ;
+extern int  isxdigit   (int)   ;
+
+
+
+
+
+
+ 
+extern int tolower  (int __c)  ;
+
+ 
+extern int toupper  (int __c)  ;
+
+
+# 125 "/usr/local/include/ctype.h" 3
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 164 "/usr/local/include/ctype.h" 3
+
+
+# 186 "/usr/local/include/ctype.h" 3
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 273 "/usr/local/include/ctype.h" 3
+
+
+} 
+
+
+# 98 "../../lib/sysfuncs.h" 2
+
+
+
+
+# 112 "../../lib/sysfuncs.h"
+
+
+
+ 
+ 
+
+
+
+
+
+# 1 "/usr/local/include/string.h" 1 3
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+extern "C" { 
+
+ 
+
+
+# 1 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 1 3
+
+
+
+
+
+
+ 
+
+
+# 19 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+
+ 
+
+
+ 
+
+
+
+
+
+ 
+
+
+# 61 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+ 
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+ 
+
+ 
+
+# 131 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+ 
+
+ 
+
+
+# 188 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+
+
+
+ 
+
+
+
+
+ 
+
+# 271 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+# 283 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+ 
+
+ 
+
+# 317 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 33 "/usr/local/include/string.h" 2 3
+
+
+
+ 
+extern void *  memcpy  (void *    __dest,
+			    const  void *    __src, size_t __n)  ;
+ 
+
+extern void *  memmove  (void *  __dest, const  void *  __src,
+			     size_t __n)  ;
+
+ 
+
+
+
+
+
+
+
+
+ 
+extern void *  memset  (void *  __s, int __c, size_t __n)  ;
+
+ 
+extern int memcmp  (const  void *  __s1, const  void *  __s2,
+			size_t __n)  ;
+
+ 
+extern void *  memchr  (const  void *  __s, int __c, size_t __n)  ;
+
+
+
+
+
+
+
+
+ 
+extern char *strcpy  (char *   __dest,
+			  const  char *   __src)  ;
+ 
+extern char *strncpy  (char *   __dest,
+			   const  char *   __src, size_t __n)  ;
+
+ 
+extern char *strcat  (char *   __dest,
+			  const  char *   __src)  ;
+ 
+extern char *strncat  (char *   __dest,
+			   const  char *   __src, size_t __n)  ;
+
+ 
+extern int strcmp  (const  char *__s1, const  char *__s2)  ;
+ 
+extern int strncmp  (const  char *__s1, const  char *__s2, size_t __n)  ;
+
+ 
+extern int strcoll  (const  char *__s1, const  char *__s2)  ;
+ 
+extern size_t strxfrm  (char *   __dest,
+			    const  char *   __src, size_t __n)  ;
+
+# 107 "/usr/local/include/string.h" 3
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+# 143 "/usr/local/include/string.h" 3
+
+
+ 
+extern char *strchr  (const  char *__s, int __c)  ;
+ 
+extern char *strrchr  (const  char *__s, int __c)  ;
+
+
+
+
+
+
+
+ 
+
+extern size_t strcspn  (const  char *__s, const  char *__reject)  ;
+ 
+
+extern size_t strspn  (const  char *__s, const  char *__accept)  ;
+ 
+extern char *strpbrk  (const  char *__s, const  char *__accept)  ;
+ 
+extern char *strstr  (const  char *__haystack, const  char *__needle)  ;
+
+
+
+
+
+
+
+
+
+ 
+extern char *strtok  (char *   __s,
+			  const  char *   __delim)  ;
+
+ 
+
+extern char *__strtok_r  (char *   __s,
+			      const  char *   __delim,
+			      char **   __save_ptr)  ;
+
+
+
+
+
+
+# 203 "/usr/local/include/string.h" 3
+
+
+
+ 
+extern size_t strlen  (const  char *__s)  ;
+
+
+
+
+
+
+
+
+ 
+extern char *strerror  (int __errnum)  ;
+
+
+
+
+
+
+
+ 
+
+extern void __bzero  (void *  __s, size_t __n)  ;
+
+# 267 "/usr/local/include/string.h" 3
+
+
+# 277 "/usr/local/include/string.h" 3
+
+
+
+
+
+
+
+
+
+# 319 "/usr/local/include/string.h" 3
+
+
+
+# 348 "/usr/local/include/string.h" 3
+
+
+} 
+
+
+# 122 "../../lib/sysfuncs.h" 2
+
+    
+
+
+
+
+
+
+
+
+
+
+
+
+
+inline void *bcopy(void *s, void *d, int n)
+{ return memcpy(d, s, n);
+}
+
+
+
+inline int bcmp(void *s1, void *s2, int n)
+{ return memcmp(s1, s2, n);
+}
+
+
+
+inline void bzero(void *m, int n)
+{ memset(m, 0, n);
+}
+
+
+# 164 "../../lib/sysfuncs.h"
+
+
+
+
+
+
+ 
+
+
+ 
+
+# 1 "/usr/local/include/errno.h" 1 3
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+ 
+
+
+
+
+
+
+extern "C" { 
+
+ 
+
+# 1 "/usr/local/include/bits/errno.h" 1 3
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 1 "/usr/include/linux/errno.h" 1 3
+
+
+
+# 1 "/usr/include/asm/errno.h" 1 3
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 4 "/usr/include/linux/errno.h" 2 3
+
+
+# 14 "/usr/include/linux/errno.h" 3
+
+
+
+# 25 "/usr/local/include/bits/errno.h" 2 3
+
+
+ 
+
+
+ 
+
+
+
+
+ 
+extern int errno;
+
+ 
+extern int *__errno_location  (void)    ;
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 36 "/usr/local/include/errno.h" 2 3
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+} 
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+# 175 "../../lib/sysfuncs.h" 2
+
+
+
+
+
+ 
+
+
+# 1 "/usr/local/include/fcntl.h" 1 3
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+ 
+extern "C" { 
+
+ 
+
+# 1 "/usr/local/include/bits/fcntl.h" 1 3
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+ 
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+struct flock
+  {
+    short int l_type;	 
+    short int l_whence;	 
+
+    __off_t l_start;	 
+    __off_t l_len;	 
+
+
+
+
+    __pid_t l_pid;	 
+  };
+
+# 131 "/usr/local/include/bits/fcntl.h" 3
+
+
+ 
+
+
+
+
+
+
+
+
+# 33 "/usr/local/include/fcntl.h" 2 3
+
+
+ 
+
+
+
+
+# 49 "/usr/local/include/fcntl.h" 3
+
+
+ 
+
+
+
+
+
+
+ 
+
+extern int __fcntl  (int __fd, int __cmd, ...)  ;
+extern int fcntl  (int __fd, int __cmd, ...)  ;
+
+ 
+
+
+extern int __open  (const  char *__file, int __oflag, ...)  ;
+
+extern int open  (const  char *__file, int __oflag, ...)  ;
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+extern int creat  (const  char *__file, __mode_t __mode)  ;
+
+
+
+
+
+
+
+
+
+
+
+
+# 126 "/usr/local/include/fcntl.h" 3
+
+
+} 
+
+
+# 183 "../../lib/sysfuncs.h" 2
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+# 217 "../../lib/sysfuncs.h"
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+# 1 "/usr/local/include/sys/param.h" 1 3
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 1 "/usr/local/include/limits.h" 1 3
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+ 
+
+
+
+ 
+
+
+
+
+
+ 
+
+
+ 
+
+
+ 
+
+
+ 
+
+
+
+ 
+
+
+ 
+
+
+
+
+
+
+
+
+ 
+
+
+
+ 
+
+
+ 
+
+
+
+ 
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  
+
+
+
+
+
+
+
+
+# 23 "/usr/local/include/sys/param.h" 2 3
+
+# 1 "/usr/include/linux/limits.h" 1 3
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 24 "/usr/local/include/sys/param.h" 2 3
+
+# 1 "/usr/include/linux/param.h" 1 3
+
+
+
+# 1 "/usr/include/asm/param.h" 1 3
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 4 "/usr/include/linux/param.h" 2 3
+
+
+
+# 25 "/usr/local/include/sys/param.h" 2 3
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+ 
+
+
+
+
+
+
+ 
+
+
+
+
+ 
+
+
+
+
+# 267 "../../lib/sysfuncs.h" 2
+
+
+
+ 
+
+# 1 "/usr/local/include/unistd.h" 1 3
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+extern "C" { 
+
+ 
+
+
+ 
+
+
+
+ 
+
+
+ 
+
+
+ 
+
+
+ 
+
+
+
+ 
+
+
+
+ 
+
+
+
+ 
+
+
+
+ 
+
+
+
+
+
+
+ 
+
+
+ 
+
+
+
+
+ 
+
+
+ 
+
+
+ 
+
+
+
+ 
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 1 "/usr/local/include/bits/posix_opt.h" 1 3
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+ 
+
+
+ 
+
+
+ 
+
+
+ 
+
+
+ 
+
+
+ 
+
+
+ 
+
+
+ 
+
+
+ 
+
+
+ 
+
+
+ 
+
+
+ 
+
+
+
+ 
+
+
+ 
+
+
+ 
+
+
+ 
+
+
+ 
+
+
+ 
+
+
+
+ 
+
+
+ 
+
+
+ 
+
+
+ 
+ 
+
+ 
+
+
+ 
+
+ 
+
+
+ 
+
+
+ 
+
+
+
+
+
+# 175 "/usr/local/include/unistd.h" 2 3
+
+
+ 
+
+
+
+
+ 
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+# 1 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 1 3
+
+
+
+
+
+
+ 
+
+
+# 19 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+
+ 
+
+
+ 
+
+
+
+
+
+ 
+
+
+# 61 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+ 
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+ 
+
+ 
+
+# 131 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+ 
+
+ 
+
+
+# 188 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+
+
+
+ 
+
+
+
+
+ 
+
+# 271 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+# 283 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+ 
+
+ 
+
+# 317 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 199 "/usr/local/include/unistd.h" 2 3
+
+
+# 231 "/usr/local/include/unistd.h" 3
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+ 
+extern int access  (const  char *__name, int __type)  ;
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+extern __off_t __lseek  (int __fd, __off_t __offset, int __whence)  ;
+
+extern __off_t lseek  (int __fd, __off_t __offset, int __whence)  ;
+# 289 "/usr/local/include/unistd.h" 3
+
+
+
+
+
+ 
+extern int __close  (int __fd)  ;
+extern int close  (int __fd)  ;
+
+ 
+
+extern ssize_t  __read  (int __fd, void *  __buf, size_t __nbytes)  ;
+extern ssize_t  read  (int __fd, void *  __buf, size_t __nbytes)  ;
+
+ 
+extern ssize_t  __write  (int __fd, const  void *  __buf, size_t __n)  ;
+extern ssize_t  write  (int __fd, const  void *  __buf, size_t __n)  ;
+
+# 343 "/usr/local/include/unistd.h" 3
+
+
+ 
+
+
+
+extern int pipe  (int __pipedes[2])  ;
+
+ 
+
+
+
+
+
+
+extern unsigned int alarm  (unsigned int __seconds)  ;
+
+ 
+
+
+
+
+
+
+extern unsigned int sleep  (unsigned int __seconds)  ;
+
+# 380 "/usr/local/include/unistd.h" 3
+
+
+
+ 
+
+extern int pause  (void)  ;
+
+
+ 
+extern int chown  (const  char *__file, __uid_t __owner,
+		       __gid_t __group)  ;
+
+# 402 "/usr/local/include/unistd.h" 3
+
+
+ 
+extern int chdir  (const  char *__path)  ;
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+extern char *getcwd  (char *__buf, size_t __size)  ;
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+extern int dup  (int __fd)  ;
+
+ 
+extern int dup2  (int __fd, int __fd2)  ;
+
+ 
+extern char **__environ;
+
+
+
+
+
+ 
+
+extern int execve  (const  char *__path, char * const  __argv[],
+			char * const  __envp[])  ;
+
+
+
+
+
+
+
+
+
+
+ 
+extern int execv  (const  char *__path, char * const  __argv[])  ;
+
+ 
+
+extern int execle  (const  char *__path, const  char *__arg, ...)  ;
+
+ 
+
+extern int execl  (const  char *__path, const  char *__arg, ...)  ;
+
+ 
+
+extern int execvp  (const  char *__file, char * const  __argv[])  ;
+
+ 
+
+
+extern int execlp  (const  char *__file, const  char *__arg, ...)  ;
+
+
+
+
+
+
+
+
+ 
+extern void _exit  (int __status)    ;
+
+
+ 
+
+
+# 1 "/usr/local/include/bits/confname.h" 1 3
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+enum
+  {
+    _PC_LINK_MAX,
+
+    _PC_MAX_CANON,
+
+    _PC_MAX_INPUT,
+
+    _PC_NAME_MAX,
+
+    _PC_PATH_MAX,
+
+    _PC_PIPE_BUF,
+
+    _PC_CHOWN_RESTRICTED,
+
+    _PC_NO_TRUNC,
+
+    _PC_VDISABLE,
+
+    _PC_SYNC_IO,
+
+    _PC_ASYNC_IO,
+
+    _PC_PRIO_IO,
+
+    _PC_SOCK_MAXBUF,
+
+    _PC_FILESIZEBITS
+
+  };
+
+ 
+enum
+  {
+    _SC_ARG_MAX,
+
+    _SC_CHILD_MAX,
+
+    _SC_CLK_TCK,
+
+    _SC_NGROUPS_MAX,
+
+    _SC_OPEN_MAX,
+
+    _SC_STREAM_MAX,
+
+    _SC_TZNAME_MAX,
+
+    _SC_JOB_CONTROL,
+
+    _SC_SAVED_IDS,
+
+    _SC_REALTIME_SIGNALS,
+
+    _SC_PRIORITY_SCHEDULING,
+
+    _SC_TIMERS,
+
+    _SC_ASYNCHRONOUS_IO,
+
+    _SC_PRIORITIZED_IO,
+
+    _SC_SYNCHRONIZED_IO,
+
+    _SC_FSYNC,
+
+    _SC_MAPPED_FILES,
+
+    _SC_MEMLOCK,
+
+    _SC_MEMLOCK_RANGE,
+
+    _SC_MEMORY_PROTECTION,
+
+    _SC_MESSAGE_PASSING,
+
+    _SC_SEMAPHORES,
+
+    _SC_SHARED_MEMORY_OBJECTS,
+
+    _SC_AIO_LISTIO_MAX,
+
+    _SC_AIO_MAX,
+
+    _SC_AIO_PRIO_DELTA_MAX,
+
+    _SC_DELAYTIMER_MAX,
+
+    _SC_MQ_OPEN_MAX,
+
+    _SC_MQ_PRIO_MAX,
+
+    _SC_VERSION,
+
+    _SC_PAGESIZE,
+
+
+    _SC_RTSIG_MAX,
+
+    _SC_SEM_NSEMS_MAX,
+
+    _SC_SEM_VALUE_MAX,
+
+    _SC_SIGQUEUE_MAX,
+
+    _SC_TIMER_MAX,
+
+
+     
+
+    _SC_BC_BASE_MAX,
+
+    _SC_BC_DIM_MAX,
+
+    _SC_BC_SCALE_MAX,
+
+    _SC_BC_STRING_MAX,
+
+    _SC_COLL_WEIGHTS_MAX,
+
+    _SC_EQUIV_CLASS_MAX,
+
+    _SC_EXPR_NEST_MAX,
+
+    _SC_LINE_MAX,
+
+    _SC_RE_DUP_MAX,
+
+    _SC_CHARCLASS_NAME_MAX,
+
+
+    _SC_2_VERSION,
+
+    _SC_2_C_BIND,
+
+    _SC_2_C_DEV,
+
+    _SC_2_FORT_DEV,
+
+    _SC_2_FORT_RUN,
+
+    _SC_2_SW_DEV,
+
+    _SC_2_LOCALEDEF,
+
+
+    _SC_PII,
+
+    _SC_PII_XTI,
+
+    _SC_PII_SOCKET,
+
+    _SC_PII_INTERNET,
+
+    _SC_PII_OSI,
+
+    _SC_POLL,
+
+    _SC_SELECT,
+
+    _SC_UIO_MAXIOV,
+
+    _SC_PII_INTERNET_STREAM,
+
+    _SC_PII_INTERNET_DGRAM,
+
+    _SC_PII_OSI_COTS,
+
+    _SC_PII_OSI_CLTS,
+
+    _SC_PII_OSI_M,
+
+    _SC_T_IOV_MAX,
+
+
+     
+    _SC_THREADS,
+
+    _SC_THREAD_SAFE_FUNCTIONS,
+
+    _SC_GETGR_R_SIZE_MAX,
+
+    _SC_GETPW_R_SIZE_MAX,
+
+    _SC_LOGIN_NAME_MAX,
+
+    _SC_TTY_NAME_MAX,
+
+    _SC_THREAD_DESTRUCTOR_ITERATIONS,
+
+    _SC_THREAD_KEYS_MAX,
+
+    _SC_THREAD_STACK_MIN,
+
+    _SC_THREAD_THREADS_MAX,
+
+    _SC_THREAD_ATTR_STACKADDR,
+
+    _SC_THREAD_ATTR_STACKSIZE,
+
+    _SC_THREAD_PRIORITY_SCHEDULING,
+
+    _SC_THREAD_PRIO_INHERIT,
+
+    _SC_THREAD_PRIO_PROTECT,
+
+    _SC_THREAD_PROCESS_SHARED,
+
+
+    _SC_NPROCESSORS_CONF,
+
+    _SC_NPROCESSORS_ONLN,
+
+    _SC_PHYS_PAGES,
+
+    _SC_AVPHYS_PAGES,
+
+    _SC_ATEXIT_MAX,
+
+    _SC_PASS_MAX,
+
+
+    _SC_XOPEN_VERSION,
+
+    _SC_XOPEN_XCU_VERSION,
+
+    _SC_XOPEN_UNIX,
+
+    _SC_XOPEN_CRYPT,
+
+    _SC_XOPEN_ENH_I18N,
+
+    _SC_XOPEN_SHM,
+
+
+    _SC_2_CHAR_TERM,
+
+    _SC_2_C_VERSION,
+
+    _SC_2_UPE,
+
+
+    _SC_XOPEN_XPG2,
+
+    _SC_XOPEN_XPG3,
+
+    _SC_XOPEN_XPG4,
+
+
+    _SC_CHAR_BIT,
+
+    _SC_CHAR_MAX,
+
+    _SC_CHAR_MIN,
+
+    _SC_INT_MAX,
+
+    _SC_INT_MIN,
+
+    _SC_LONG_BIT,
+
+    _SC_WORD_BIT,
+
+    _SC_MB_LEN_MAX,
+
+    _SC_NZERO,
+
+    _SC_SSIZE_MAX,
+
+    _SC_SCHAR_MAX,
+
+    _SC_SCHAR_MIN,
+
+    _SC_SHRT_MAX,
+
+    _SC_SHRT_MIN,
+
+    _SC_UCHAR_MAX,
+
+    _SC_UINT_MAX,
+
+    _SC_ULONG_MAX,
+
+    _SC_USHRT_MAX,
+
+
+    _SC_NL_ARGMAX,
+
+    _SC_NL_LANGMAX,
+
+    _SC_NL_MSGMAX,
+
+    _SC_NL_NMAX,
+
+    _SC_NL_SETMAX,
+
+    _SC_NL_TEXTMAX,
+
+
+    _SC_XBS5_ILP32_OFF32,
+
+    _SC_XBS5_ILP32_OFFBIG,
+
+    _SC_XBS5_LP64_OFF64,
+
+    _SC_XBS5_LPBIG_OFFBIG,
+
+
+    _SC_XOPEN_LEGACY,
+
+    _SC_XOPEN_REALTIME,
+
+    _SC_XOPEN_REALTIME_THREADS
+
+  };
+
+# 408 "/usr/local/include/bits/confname.h" 3
+
+# 497 "/usr/local/include/unistd.h" 2 3
+
+
+ 
+extern long int pathconf  (const  char *__path, int __name)  ;
+
+ 
+extern long int fpathconf  (int __fd, int __name)  ;
+
+ 
+extern long int sysconf  (int __name)  ;
+
+
+
+
+
+
+
+ 
+extern __pid_t __getpid  (void)  ;
+extern __pid_t getpid  (void)  ;
+
+ 
+extern __pid_t getppid  (void)  ;
+
+ 
+
+
+extern __pid_t getpgrp  (void)  ;
+
+
+
+
+
+
+
+
+ 
+extern __pid_t __getpgid  (__pid_t __pid)  ;
+
+
+
+
+
+ 
+
+
+extern int __setpgid  (__pid_t __pid, __pid_t __pgid)  ;
+extern int setpgid  (__pid_t __pid, __pid_t __pgid)  ;
+
+# 574 "/usr/local/include/unistd.h" 3
+
+
+ 
+
+
+extern __pid_t setsid  (void)  ;
+
+
+
+
+
+
+ 
+extern __uid_t getuid  (void)  ;
+
+ 
+extern __uid_t geteuid  (void)  ;
+
+ 
+extern __gid_t getgid  (void)  ;
+
+ 
+extern __gid_t getegid  (void)  ;
+
+ 
+
+
+extern int getgroups  (int __size, __gid_t __list[])  ;
+
+
+
+
+
+
+ 
+
+
+
+extern int setuid  (__uid_t __uid)  ;
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+extern int setgid  (__gid_t __gid)  ;
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+extern __pid_t __fork  (void)  ;
+extern __pid_t fork  (void)  ;
+
+
+
+
+
+
+
+
+
+
+ 
+
+extern char *ttyname  (int __fd)  ;
+
+ 
+
+extern int ttyname_r  (int __fd, char *__buf, size_t __buflen)  ;
+
+ 
+
+extern int isatty  (int __fd)  ;
+
+
+
+
+
+
+
+
+ 
+extern int link  (const  char *__from, const  char *__to)  ;
+
+# 688 "/usr/local/include/unistd.h" 3
+
+
+ 
+extern int unlink  (const  char *__name)  ;
+
+ 
+extern int rmdir  (const  char *__path)  ;
+
+
+ 
+extern __pid_t tcgetpgrp  (int __fd)  ;
+
+ 
+extern int tcsetpgrp  (int __fd, __pid_t __pgrp_id)  ;
+
+
+ 
+extern char *getlogin  (void)  ;
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 798 "/usr/local/include/unistd.h" 3
+
+
+
+# 853 "/usr/local/include/unistd.h" 3
+
+
+
+# 871 "/usr/local/include/unistd.h" 3
+
+
+
+# 887 "/usr/local/include/unistd.h" 3
+
+
+
+# 917 "/usr/local/include/unistd.h" 3
+
+
+
+# 940 "/usr/local/include/unistd.h" 3
+
+
+
+
+
+
+
+
+
+ 
+
+# 968 "/usr/local/include/unistd.h" 3
+
+
+
+ 
+
+# 983 "/usr/local/include/unistd.h" 3
+
+
+
+# 1002 "/usr/local/include/unistd.h" 3
+
+
+} 
+
+
+# 272 "../../lib/sysfuncs.h" 2
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+# 1 "/usr/local/include/stdio.h" 1 3
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+extern "C" { 
+
+
+
+# 1 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 1 3
+
+
+
+
+
+
+ 
+
+
+# 19 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+
+ 
+
+
+ 
+
+
+
+
+
+ 
+
+
+# 61 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+ 
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+ 
+
+ 
+
+# 131 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+ 
+
+ 
+
+
+# 188 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+
+
+
+ 
+
+
+
+
+ 
+
+# 271 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+# 283 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+ 
+
+ 
+
+# 317 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 33 "/usr/local/include/stdio.h" 2 3
+
+
+
+
+
+# 1 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stdarg.h" 1 3
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+typedef void *__gnuc_va_list;
+
+
+
+ 
+
+# 122 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stdarg.h" 3
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 209 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stdarg.h" 3
+
+
+
+
+# 38 "/usr/local/include/stdio.h" 2 3
+
+
+
+
+
+
+
+
+
+ 
+typedef struct _IO_FILE FILE;
+
+
+
+
+
+
+
+
+# 1 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/../../../../include/g++-3/libio.h" 1 3
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 1 "/usr/local/include/_G_config.h" 1 3
+ 
+
+
+
+
+
+ 
+
+
+
+
+
+
+# 1 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 1 3
+
+
+
+
+
+
+ 
+
+
+# 19 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+
+ 
+
+
+ 
+
+
+
+
+
+ 
+
+
+# 61 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+ 
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+ 
+
+ 
+
+# 131 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+ 
+
+ 
+
+
+# 188 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+
+
+
+ 
+
+
+
+
+ 
+
+
+# 269 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+
+
+
+
+
+
+
+
+
+typedef unsigned int  wint_t;
+
+
+
+
+ 
+
+ 
+
+# 317 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 14 "/usr/local/include/_G_config.h" 2 3
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+typedef int _G_int16_t  ;
+typedef int _G_int32_t  ;
+typedef unsigned int _G_uint16_t  ;
+typedef unsigned int _G_uint32_t  ;
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 30 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/../../../../include/g++-3/libio.h" 2 3
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+# 1 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stdarg.h" 1 3
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+ 
+
+# 122 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stdarg.h" 3
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 209 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stdarg.h" 3
+
+
+
+
+# 50 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/../../../../include/g++-3/libio.h" 2 3
+
+
+
+
+
+
+
+# 67 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/../../../../include/g++-3/libio.h" 3
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 104 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/../../../../include/g++-3/libio.h" 3
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+struct _IO_jump_t;  struct _IO_FILE;
+
+ 
+# 174 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/../../../../include/g++-3/libio.h" 3
+
+
+   typedef void _IO_lock_t;
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+struct _IO_marker {
+  struct _IO_marker *_next;
+  struct _IO_FILE *_sbuf;
+   
+
+   
+  int _pos;
+# 207 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/../../../../include/g++-3/libio.h" 3
+
+};
+
+struct _IO_FILE {
+  int _flags;		 
+
+
+   
+   
+  char* _IO_read_ptr;	 
+  char* _IO_read_end;	 
+  char* _IO_read_base;	 
+  char* _IO_write_base;	 
+  char* _IO_write_ptr;	 
+  char* _IO_write_end;	 
+  char* _IO_buf_base;	 
+  char* _IO_buf_end;	 
+   
+  char *_IO_save_base;  
+  char *_IO_backup_base;   
+  char *_IO_save_end;  
+
+  struct _IO_marker *_markers;
+
+  struct _IO_FILE *_chain;
+
+  int _fileno;
+  int _blksize;
+
+  __off_t   _old_offset;
+
+
+
+
+
+   
+  unsigned short _cur_column;
+  char _unused;
+  char _shortbuf[1];
+
+   
+
+
+  void *  _lock;
+
+
+  __off64_t   _offset;
+  int _unused2[16];	 
+
+};
+
+
+
+
+
+
+
+
+
+
+
+struct _IO_FILE_plus;
+extern struct _IO_FILE_plus _IO_2_1_stdin_ , _IO_2_1_stdout_ , _IO_2_1_stderr_ ;
+
+
+
+
+
+ 
+typedef struct
+{
+  __ssize_t   (*read)  (struct _IO_FILE *, void *, __ssize_t  )  ;
+  __ssize_t   (*write)  (struct _IO_FILE *, const void *, __ssize_t  )  ;
+  __off_t   (*seek)  (struct _IO_FILE *, __off_t  , int)  ;
+  int (*close)  (struct _IO_FILE *)  ;
+} _IO_cookie_io_functions_t;
+
+ 
+struct _IO_cookie_file
+{
+  struct _IO_FILE file;
+  const void *vtable;
+  void *cookie;
+  _IO_cookie_io_functions_t io_functions;
+};
+
+
+
+extern "C" {
+
+
+extern int __underflow  (_IO_FILE *)  ;
+extern int __uflow  (_IO_FILE *)  ;
+extern int __overflow  (_IO_FILE *, int)  ;
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+extern int _IO_getc  (_IO_FILE *__fp)  ;
+extern int _IO_putc  (int __c, _IO_FILE *__fp)  ;
+extern int _IO_feof  (_IO_FILE *__fp)  ;
+extern int _IO_ferror  (_IO_FILE *__fp)  ;
+
+extern int _IO_peekc_locked  (_IO_FILE *__fp)  ;
+
+ 
+
+
+
+extern void _IO_flockfile  (_IO_FILE *)  ;
+extern void _IO_funlockfile  (_IO_FILE *)  ;
+extern int _IO_ftrylockfile  (_IO_FILE *)  ;
+
+
+
+
+
+
+
+
+
+
+
+
+
+extern int _IO_vfscanf  (_IO_FILE *, const char *, __gnuc_va_list , int *)  ;
+extern int _IO_vfprintf  (_IO_FILE *, const char *, __gnuc_va_list )  ;
+extern __ssize_t   _IO_padn  (_IO_FILE *, int, __ssize_t  )  ;
+extern size_t   _IO_sgetn  (_IO_FILE *, void *, size_t  )  ;
+
+
+extern __off64_t   _IO_seekoff  (_IO_FILE *, __off64_t  , int, int)  ;
+extern __off64_t   _IO_seekpos  (_IO_FILE *, __off64_t  , int)  ;
+
+
+
+
+
+extern void _IO_free_backup_area  (_IO_FILE *)  ;
+
+
+}
+
+
+
+# 57 "/usr/local/include/stdio.h" 2 3
+
+
+ 
+
+typedef __off_t  fpos_t;
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+ 
+
+
+
+
+
+ 
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+# 1 "/usr/local/include/bits/stdio_lim.h" 1 3
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 110 "/usr/local/include/stdio.h" 2 3
+
+
+
+ 
+extern FILE *stdin;		 
+extern FILE *stdout;		 
+extern FILE *stderr;		 
+
+
+ 
+extern int remove  (const  char *__filename)  ;
+ 
+extern int rename  (const  char *__old, const  char *__new)  ;
+
+
+ 
+
+extern FILE *tmpfile  (void)  ;
+
+
+
+
+
+
+
+
+
+
+ 
+extern char *tmpnam  (char *__s)  ;
+
+
+
+
+
+
+
+
+# 157 "/usr/local/include/stdio.h" 3
+
+
+
+ 
+extern int fclose  (FILE *__stream)  ;
+ 
+extern int fflush  (FILE *__stream)  ;
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+extern FILE *fopen  (const  char *   __filename,
+			 const  char *   __modes)  ;
+ 
+extern FILE *freopen  (const  char *   __filename,
+			   const  char *   __modes,
+			   FILE *   __stream)  ;
+# 197 "/usr/local/include/stdio.h" 3
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 223 "/usr/local/include/stdio.h" 3
+
+
+
+ 
+
+extern void setbuf  (FILE *   __stream, char *   __buf)  ;
+ 
+
+
+extern int setvbuf  (FILE *   __stream, char *   __buf,
+			 int __modes, size_t __n)  ;
+
+# 243 "/usr/local/include/stdio.h" 3
+
+
+
+ 
+extern int fprintf  (FILE *   __stream,
+			 const  char *   __format, ...)  ;
+ 
+extern int printf  (const  char *   __format, ...)  ;
+ 
+extern int sprintf  (char *   __s,
+			 const  char *   __format, ...)  ;
+
+ 
+extern int vfprintf  (FILE *   __s,
+			  const  char *   __format,
+			  __gnuc_va_list  __arg)  ;
+ 
+extern int vprintf  (const  char *   __format,
+			 __gnuc_va_list  __arg)  ;
+ 
+extern int vsprintf  (char *   __s,
+			  const  char *   __format,
+			  __gnuc_va_list  __arg)  ;
+
+# 281 "/usr/local/include/stdio.h" 3
+
+
+# 302 "/usr/local/include/stdio.h" 3
+
+
+
+ 
+extern int fscanf  (FILE *   __stream,
+			const  char *   __format, ...)  ;
+ 
+extern int scanf  (const  char *   __format, ...)  ;
+ 
+extern int sscanf  (const  char *   __s,
+			const  char *   __format, ...)  ;
+
+# 330 "/usr/local/include/stdio.h" 3
+
+
+
+ 
+extern int fgetc  (FILE *__stream)  ;
+extern int getc  (FILE *__stream)  ;
+
+ 
+extern int getchar  (void)  ;
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+extern int fputc  (int __c, FILE *__stream)  ;
+extern int putc  (int __c, FILE *__stream)  ;
+
+ 
+extern int putchar  (int __c)  ;
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+extern char *fgets  (char *   __s, int __n,
+			 FILE *   __stream)  ;
+
+
+
+
+
+
+
+ 
+
+extern char *gets  (char *__s)  ;
+
+
+# 420 "/usr/local/include/stdio.h" 3
+
+
+
+ 
+extern int fputs  (const  char *   __s,
+		       FILE *   __stream)  ;
+
+
+
+
+
+
+
+ 
+extern int puts  (const  char *__s)  ;
+
+
+ 
+extern int ungetc  (int __c, FILE *__stream)  ;
+
+
+ 
+extern size_t fread  (void *   __ptr, size_t __size,
+			  size_t __n, FILE *   __stream)  ;
+ 
+extern size_t fwrite  (const  void *   __ptr, size_t __size,
+			   size_t __n, FILE *   __s)  ;
+
+
+
+
+
+
+
+
+
+
+
+ 
+extern int fseek  (FILE *__stream, long int __off, int __whence)  ;
+ 
+extern long int ftell  (FILE *__stream)  ;
+ 
+extern void rewind  (FILE *__stream)  ;
+
+ 
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+extern int fgetpos  (FILE *   __stream,
+			 fpos_t *   __pos)  ;
+ 
+extern int fsetpos  (FILE *__stream, const  fpos_t *__pos)  ;
+# 519 "/usr/local/include/stdio.h" 3
+
+
+# 529 "/usr/local/include/stdio.h" 3
+
+
+ 
+extern void clearerr  (FILE *__stream)  ;
+ 
+extern int feof  (FILE *__stream)  ;
+ 
+extern int ferror  (FILE *__stream)  ;
+
+
+
+
+
+
+
+
+
+ 
+extern void perror  (const  char *__s)  ;
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 603 "/usr/local/include/stdio.h" 3
+
+
+
+# 618 "/usr/local/include/stdio.h" 3
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+} 
+
+
+
+
+# 286 "../../lib/sysfuncs.h" 2
+
+
+
+
+
+
+
+
+
+
+# 1 "../../lib/pathmax.h" 1
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+# 1 "/usr/local/include/limits.h" 1 3
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+# 123 "/usr/local/include/limits.h" 3
+
+
+  
+
+
+
+
+
+
+
+
+# 29 "../../lib/pathmax.h" 2
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 296 "../../lib/sysfuncs.h" 2
+
+
+ 
+
+# 1 "../../lib/getopt.h" 1
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+extern "C"
+{
+
+
+ 
+
+
+
+
+
+  extern char *optarg;
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+  extern int optind;
+
+ 
+
+
+  extern int opterr;
+
+ 
+
+  extern int optopt;
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  struct option
+    {
+
+      const char *name;
+
+
+
+       
+
+      int has_arg;
+      int *flag;
+      int val;
+    };
+
+ 
+
+
+
+
+
+
+
+ 
+
+
+  extern int getopt (int argc, char *const *argv, const char *shortopts);
+
+
+
+  extern int getopt_long (int argc, char *const *argv, const char *shortopts,
+			  const struct option *longopts, int *longind);
+  extern int getopt_long_only (int argc, char *const *argv,
+			       const char *shortopts,
+			       const struct option *longopts, int *longind);
+
+ 
+  extern int _getopt_internal (int argc, char *const *argv,
+			       const char *shortopts,
+			       const struct option *longopts, int *longind,
+			       int long_only);
+
+
+
+
+
+
+
+
+
+}
+
+
+
+# 300 "../../lib/sysfuncs.h" 2
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+void error  (int, int, const char *,...)  ;
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+void *  xmalloc  (size_t)  ;
+void *  xrealloc  (void * , size_t)  ;
+char *xstrdup  (const char *)  ;
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+# 3 "grbTest.cpp" 2
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 1 "/usr/local/include/sys/time.h" 1 3
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 1 "/usr/local/include/time.h" 1 3
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+extern "C" { 
+
+
+
+
+ 
+
+
+# 1 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 1 3
+
+
+
+
+
+
+ 
+
+
+# 19 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+
+ 
+
+
+ 
+
+
+
+
+
+ 
+
+
+# 61 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+ 
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+ 
+
+ 
+
+# 131 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+ 
+
+ 
+
+
+# 188 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+
+
+
+ 
+
+
+
+
+ 
+
+# 271 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+# 283 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+ 
+
+ 
+
+# 317 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stddef.h" 3
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 38 "/usr/local/include/time.h" 2 3
+
+
+ 
+
+# 1 "/usr/local/include/bits/time.h" 1 3
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+ 
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 59 "/usr/local/include/bits/time.h" 3
+ 
+# 42 "/usr/local/include/time.h" 2 3
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+typedef __clock_t clock_t;
+
+
+
+
+# 73 "/usr/local/include/time.h" 3
+
+
+
+
+# 89 "/usr/local/include/time.h" 3
+
+
+
+
+
+ 
+struct tm
+{
+  int tm_sec;			 
+  int tm_min;			 
+  int tm_hour;			 
+  int tm_mday;			 
+  int tm_mon;			 
+  int tm_year;			 
+  int tm_wday;			 
+  int tm_yday;			 
+  int tm_isdst;			 
+
+
+
+
+
+  long int __tm_gmtoff;		 
+  const  char *__tm_zone;	 
+
+};
+
+
+ 
+
+extern clock_t clock  (void)  ;
+
+ 
+extern time_t time  (time_t *__timer)  ;
+
+ 
+extern double difftime  (time_t __time1, time_t __time0)  
+      ;
+
+ 
+extern time_t mktime  (struct tm *__tp)  ;
+
+
+ 
+
+
+extern size_t strftime  (char *   __s, size_t __maxsize,
+			     const  char *   __format,
+			     const  struct tm *   __tp)  ;
+
+
+
+
+
+
+
+
+
+ 
+
+extern struct tm *gmtime  (const  time_t *__timer)  ;
+
+ 
+
+extern struct tm *localtime  (const  time_t *__timer)  ;
+
+# 167 "/usr/local/include/time.h" 3
+
+
+ 
+
+extern char *asctime  (const  struct tm *__tp)  ;
+
+ 
+extern char *ctime  (const  time_t *__timer)  ;
+
+# 187 "/usr/local/include/time.h" 3
+
+
+
+ 
+extern char *__tzname[2];	 
+extern int __daylight;		 
+extern long int __timezone;	 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+# 235 "/usr/local/include/time.h" 3
+
+
+
+
+
+
+
+
+
+
+# 264 "/usr/local/include/time.h" 3
+
+
+# 274 "/usr/local/include/time.h" 3
+
+
+
+} 
+
+
+
+
+# 24 "/usr/local/include/sys/time.h" 2 3
+
+# 1 "/usr/local/include/sys/select.h" 1 3
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+ 
+
+
+ 
+# 1 "/usr/local/include/bits/select.h" 1 3
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 57 "/usr/local/include/bits/select.h" 3
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 31 "/usr/local/include/sys/select.h" 2 3
+
+
+ 
+# 1 "/usr/local/include/bits/sigset.h" 1 3
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+typedef int __sig_atomic_t;
+
+ 
+
+
+typedef struct
+  {
+    unsigned long int __val[(1024 / (8 * sizeof (unsigned long int))) ];
+  } __sigset_t;
+
+
+
+
+ 
+
+
+
+
+
+# 125 "/usr/local/include/bits/sigset.h" 3
+
+# 34 "/usr/local/include/sys/select.h" 2 3
+
+
+ 
+
+
+
+extern "C" { 
+
+ 
+
+
+
+struct timeval;
+
+typedef __fd_mask fd_mask;
+
+ 
+typedef __fd_set fd_set;
+
+ 
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+ 
+
+
+
+
+extern int __select  (int __nfds, __fd_set *__readfds,
+			  __fd_set *__writefds, __fd_set *__exceptfds,
+			  struct timeval *__timeout)  ;
+extern int select  (int __nfds, __fd_set *__readfds,
+			__fd_set *__writefds, __fd_set *__exceptfds,
+			struct timeval *__timeout)  ;
+
+# 91 "/usr/local/include/sys/select.h" 3
+
+
+} 
+
+
+# 25 "/usr/local/include/sys/time.h" 2 3
+
+
+
+# 1 "/usr/local/include/bits/time.h" 1 3
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+# 43 "/usr/local/include/bits/time.h" 3
+
+
+
+
+
+
+
+
+ 
+
+struct timeval
+  {
+    __time_t tv_sec;		 
+    __time_t tv_usec;		 
+  };
+
+ 
+# 28 "/usr/local/include/sys/time.h" 2 3
+
+
+
+extern "C" { 
+
+ 
+
+
+
+
+
+
+
+
+
+
+ 
+
+struct timezone
+  {
+    int tz_minuteswest;		 
+    int tz_dsttime;		 
+  };
+
+ 
+
+
+
+
+extern int gettimeofday  (struct timeval *__tv,
+			      struct timezone *__tz)  ;
+
+ 
+
+extern int settimeofday  (const  struct timeval *__tv,
+			      const  struct timezone *__tz)  ;
+
+ 
+
+
+
+extern int adjtime  (const  struct timeval *__delta,
+			 struct timeval *__olddelta)  ;
+
+
+ 
+enum __itimer_which
+  {
+     
+    ITIMER_REAL = 0,
+
+     
+    ITIMER_VIRTUAL = 1,
+
+     
+
+    ITIMER_PROF = 2
+
+  };
+
+ 
+
+struct itimerval
+  {
+     
+    struct timeval it_interval;
+     
+    struct timeval it_value;
+  };
+
+ 
+
+extern int getitimer  (enum __itimer_which __which,
+			   struct itimerval *__value)  ;
+
+ 
+
+
+extern int setitimer  (enum __itimer_which __which,
+			   const  struct itimerval *__new,
+			   struct itimerval *__old)  ;
+
+ 
+
+extern int utimes  (const  char *__file, struct timeval __tvp[2])  ;
+
+
+ 
+
+
+
+
+
+
+
+
+# 133 "/usr/local/include/sys/time.h" 3
+
+# 142 "/usr/local/include/sys/time.h" 3
+
+} 
+
+
+# 18 "grbTest.cpp" 2
+
+
+# 1 "/usr/local/include/sys/resource.h" 1 3
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+# 1 "/usr/local/include/bits/resource.h" 1 3
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 1 "/usr/include/asm/resource.h" 1 3
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 37 "/usr/include/asm/resource.h" 3
+
+
+
+# 24 "/usr/local/include/bits/resource.h" 2 3
+
+
+
+ 
+
+
+
+
+ 
+enum __rlimit_resource
+{
+   
+  _RLIMIT_CPU = 0 ,
+
+  RLIMIT_CPU = _RLIMIT_CPU,
+
+
+   
+  _RLIMIT_FSIZE = 1 ,
+
+  RLIMIT_FSIZE = _RLIMIT_FSIZE,
+
+
+   
+  _RLIMIT_DATA = 2 ,
+
+  RLIMIT_DATA = _RLIMIT_DATA,
+
+
+   
+  _RLIMIT_STACK = 3 ,
+
+  RLIMIT_STACK = _RLIMIT_STACK,
+
+
+   
+  _RLIMIT_CORE = 4 ,
+
+  RLIMIT_CORE = _RLIMIT_CORE,
+
+
+   
+
+
+
+  _RLIMIT_RSS = 5 ,
+
+  RLIMIT_RSS = _RLIMIT_RSS,
+
+
+   
+  _RLIMIT_NOFILE = 7 ,
+
+  RLIMIT_NOFILE = _RLIMIT_NOFILE,
+  RLIMIT_OFILE = RLIMIT_NOFILE,  
+
+
+
+   
+  _RLIMIT_AS = 9 ,
+
+  RLIMIT_AS = _RLIMIT_AS,
+
+
+   
+  _RLIMIT_NPROC = 6 ,
+
+  RLIMIT_NPROC = _RLIMIT_NPROC,
+
+
+   
+  _RLIMIT_MEMLOCK = 8 ,
+
+  RLIMIT_MEMLOCK = _RLIMIT_MEMLOCK,
+
+
+  RLIMIT_NLIMITS = 10 ,
+
+  RLIM_NLIMITS = RLIMIT_NLIMITS
+
+
+};
+
+ 
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+ 
+
+typedef __rlim_t rlim_t;
+
+
+
+
+
+
+
+struct rlimit
+  {
+     
+    rlim_t rlim_cur;
+     
+    rlim_t rlim_max;
+  };
+
+# 149 "/usr/local/include/bits/resource.h" 3
+
+
+ 
+enum __rusage_who
+{
+   
+  RUSAGE_SELF = 0,
+
+
+   
+  RUSAGE_CHILDREN = -1,
+
+
+   
+  RUSAGE_BOTH = -2
+
+};
+
+
+# 1 "/usr/local/include/bits/time.h" 1 3
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+# 43 "/usr/local/include/bits/time.h" 3
+
+
+
+
+# 58 "/usr/local/include/bits/time.h" 3
+
+ 
+# 168 "/usr/local/include/bits/resource.h" 2 3
+
+
+ 
+struct rusage
+  {
+     
+    struct timeval ru_utime;
+     
+    struct timeval ru_stime;
+     
+    long int ru_maxrss;
+     
+
+    long int ru_ixrss;
+     
+    long int ru_idrss;
+     
+    long int ru_isrss;
+     
+
+    long int ru_minflt;
+     
+    long int ru_majflt;
+     
+    long int ru_nswap;
+     
+
+    long int ru_inblock;
+     
+    long int ru_oublock;
+     
+    long int ru_msgsnd;
+     
+    long int ru_msgrcv;
+     
+    long int ru_nsignals;
+     
+
+
+    long int ru_nvcsw;
+     
+
+    long int ru_nivcsw;
+  };
+
+ 
+
+
+
+ 
+
+enum __priority_which
+{
+  PRIO_PROCESS = 0,		 
+  PRIO_PGRP = 1,		 
+  PRIO_USER = 2			 
+};
+# 25 "/usr/local/include/sys/resource.h" 2 3
+
+
+extern "C" { 
+
+ 
+
+
+extern int getrlimit  (enum __rlimit_resource __resource,
+			   struct rlimit *__rlimits)  ;
+# 42 "/usr/local/include/sys/resource.h" 3
+
+
+
+
+
+
+ 
+
+
+
+extern int setrlimit  (enum __rlimit_resource __resource,
+			   const  struct rlimit *__rlimits)  ;
+# 62 "/usr/local/include/sys/resource.h" 3
+
+
+
+
+
+
+ 
+
+extern int getrusage  (enum __rusage_who __who, struct rusage *__usage)  ;
+
+ 
+
+
+
+extern int getpriority  (enum __priority_which __which, int __who)  ;
+
+ 
+
+extern int setpriority  (enum __priority_which __which, int __who,
+			     int __prio)  ;
+
+} 
+
+
+# 20 "grbTest.cpp" 2
+
+
+
+
+
+
+ 
+extern "C" {
+
+
+
+# 1 "../../lib/rx.h" 1
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+typedef unsigned int RX_subset;
+
+
+
+
+typedef RX_subset * rx_Bitset;
+
+
+typedef void (*rx_bitset_iterator) (void *, int member_index);
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+typedef int (*rx_sp_comparer) (void * a, void * b);
+
+
+
+
+struct rx_sp_node 
+{
+  void * key;
+  void * data;
+  struct rx_sp_node * kids[2];
+};
+
+
+typedef void (*rx_sp_key_data_freer) (struct rx_sp_node *);
+
+
+
+
+
+
+ 
+
+struct rx_hash_item
+{
+  struct rx_hash_item * next_same_hash;
+  struct rx_hash * table;
+  unsigned long hash;
+  void * data;
+  void * binding;
+};
+
+struct rx_hash
+{
+  struct rx_hash * parent;
+  int refs;
+  struct rx_hash * children[13];
+  struct rx_hash_item * buckets [13];
+  int bucket_size [13];
+};
+
+struct rx_hash_rules;
+
+
+ 
+typedef int (*rx_hash_eq)(void *, void *);
+typedef struct rx_hash * (*rx_alloc_hash)(struct rx_hash_rules *);
+typedef void (*rx_free_hash)(struct rx_hash *,
+			    struct rx_hash_rules *);
+typedef struct rx_hash_item * (*rx_alloc_hash_item)(struct rx_hash_rules *,
+						    void *);
+typedef void (*rx_free_hash_item)(struct rx_hash_item *,
+				 struct rx_hash_rules *);
+
+
+
+
+
+
+
+
+struct rx_hash_rules
+{
+  rx_hash_eq eq;
+  rx_alloc_hash hash_alloc;
+  rx_free_hash free_hash;
+  rx_alloc_hash_item hash_item_alloc;
+  rx_free_hash_item free_hash_item;
+};
+
+
+
+ 
+
+struct rx_cache;
+struct rx_superset;
+struct rx;
+struct rx_se_list;
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+ 
+
+
+
+
+
+ 
+
+typedef int (*rx_se_list_order) (struct rx *,
+				 struct rx_se_list *, 
+				 struct rx_se_list *);
+
+
+
+
+
+
+ 
+
+
+
+
+struct rx
+{
+   
+
+
+
+
+
+
+
+
+
+
+
+
+
+  int rx_id;
+
+   
+
+
+  struct rx_cache * cache;
+
+   
+
+
+
+
+  int local_cset_size;
+
+   
+
+
+
+  void * buffer;
+  unsigned long allocated;
+
+   
+
+
+
+
+  unsigned long reserved;
+
+   
+   
+
+   
+
+
+  int nodec;
+
+   
+  int epsnodec;
+
+   
+
+
+
+   
+
+
+
+  struct rx_hash se_list_memo;
+
+   
+
+  struct rx_hash set_list_memo;
+
+
+
+
+   
+
+
+
+
+
+
+
+
+  void ** instruction_table;
+
+   
+
+
+
+
+
+  struct rx_nfa_state *nfa_states;
+
+   
+
+  struct rx_nfa_state *start;
+
+   
+
+
+  rx_se_list_order se_list_cmp;
+
+  struct rx_superset * start_set;
+};
+
+
+
+
+
+ 
+
+ 
+
+
+
+
+
+
+
+
+
+enum rexp_node_type
+{
+  r_cset,			 
+  r_concat,			 
+  r_alternate,			 
+  r_opt,			 
+  r_star,			 
+
+
+   
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  r_2phase_star,
+
+
+   
+  r_side_effect,
+
+   
+
+
+  r_data
+};
+
+ 
+
+
+
+
+
+typedef void * rx_side_effect;
+
+ 
+
+struct rexp_node
+{
+  enum rexp_node_type type;
+  union
+  {
+    rx_Bitset cset;
+    rx_side_effect side_effect;
+    struct
+      {
+	struct rexp_node *left;
+	struct rexp_node *right;
+      } pair;
+    void * data;
+  } params;
+};
+
+
+
+
+ 
+
+
+
+
+
+struct rx_nfa_state
+{
+   
+  struct rx_nfa_state *next;
+
+   
+
+
+
+
+  int id;
+
+   
+  struct rx_nfa_edge *edges;
+
+   
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  struct rx_possible_future *futures;
+
+
+   
+  unsigned int is_final:1;
+  unsigned int is_start:1;
+
+   
+  unsigned int eclosure_needed:1;
+  unsigned int mark:1;
+};
+
+
+ 
+enum rx_nfa_etype
+{
+   
+
+
+  ne_cset,
+
+   
+
+
+  ne_epsilon,
+
+   
+
+
+
+  ne_side_effect		 
+};
+
+struct rx_nfa_edge
+{
+  struct rx_nfa_edge *next;
+  enum rx_nfa_etype type;
+  struct rx_nfa_state *dest;
+  union
+  {
+    rx_Bitset cset;
+    rx_side_effect side_effect;
+  } params;
+};
+
+
+
+ 
+
+
+
+
+
+
+struct rx_nfa_state_set
+{
+  struct rx_nfa_state * car;
+  struct rx_nfa_state_set * cdr;
+};
+
+struct rx_se_list
+{
+  rx_side_effect car;
+  struct rx_se_list * cdr;
+};
+
+struct rx_possible_future
+{
+  struct rx_possible_future *next;
+  struct rx_se_list * effects;
+  struct rx_nfa_state_set * destset;
+};
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+enum rx_opcode
+{
+   
+
+
+
+
+
+
+
+
+
+
+
+  rx_backtrack_point = 0,	 
+
+   
+
+
+
+
+  rx_do_side_effects = rx_backtrack_point + 1,
+
+   
+
+   
+
+
+
+
+
+
+  rx_cache_miss = rx_do_side_effects + 1,
+   
+
+   
+
+
+
+
+
+  rx_next_char = rx_cache_miss + 1,  
+
+   
+
+  rx_backtrack = rx_next_char + 1,  
+
+   
+
+
+  rx_error_inx = rx_backtrack + 1,  
+
+  rx_num_instructions = rx_error_inx + 1
+};
+
+ 
+
+
+extern void * rx_id_instruction_table[rx_num_instructions];
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+struct rx_inx 
+{
+  void * data;
+  void * data_2;
+  void * inx;
+  void * fnord;
+};
+
+
+
+
+
+ 
+
+
+
+struct rx_superset
+{
+  int refs;			 
+
+   
+
+
+
+
+
+
+
+
+
+
+  int id;
+
+  struct rx_nfa_state * car;	 
+  struct rx_superset * cdr;
+
+   
+  struct rx_superstate * superstate;
+
+
+   
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  struct rx * starts_for;
+
+   
+
+
+  struct rx_hash_item hash_item;
+};
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+struct rx_super_edge
+{
+  struct rx_super_edge *next;
+  struct rx_inx rx_backtrack_frame;
+  int cset_size;
+  rx_Bitset cset;
+  struct rx_distinct_future *options;
+};
+
+ 
+
+
+
+
+struct rx_superstate
+{
+  int rx_id;			 
+  int locks;			 
+
+   
+
+
+
+
+  struct rx_superstate * next_recyclable;
+  struct rx_superstate * prev_recyclable;
+
+   
+
+
+  struct rx_distinct_future * transition_refs;
+
+   
+  struct rx_superset * contents;
+
+   
+  struct rx_super_edge * edges;
+
+   
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  int is_semifree;
+
+
+   
+
+
+
+  int trans_size;
+
+   
+  struct rx_inx transitions[1 ];
+};
+
+
+ 
+
+
+
+struct rx_distinct_future
+{
+  struct rx_distinct_future * next_same_super_edge[2];
+  struct rx_distinct_future * next_same_dest;
+  struct rx_distinct_future * prev_same_dest;
+  struct rx_superstate * present;	 
+  struct rx_superstate * future;	 
+  struct rx_super_edge * edge;
+
+
+   
+
+
+
+
+
+
+
+
+
+
+
+
+  struct rx_inx future_frame;
+
+  struct rx_inx side_effects_frame;
+  struct rx_se_list * effects;
+};
+
+
+
+
+
+
+ 
+
+struct rx_blocklist
+{
+  struct rx_blocklist * next;
+  int bytes;
+};
+
+struct rx_freelist
+{
+  struct rx_freelist * next;
+};
+
+struct rx_cache;
+
+
+typedef void (*rx_morecore_fn)(struct rx_cache *);
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+struct rx_cache
+{
+  struct rx_hash_rules superset_hash_rules;
+
+   
+
+
+  struct rx_blocklist * memory;
+  struct rx_blocklist * memory_pos;
+  int bytes_left;
+  char * memory_addr;
+  rx_morecore_fn morecore;
+
+   
+  struct rx_freelist * free_superstates;
+  struct rx_freelist * free_transition_classes;
+  struct rx_freelist * free_discernable_futures;
+  struct rx_freelist * free_supersets;
+  struct rx_freelist * free_hash;
+
+   
+
+
+  struct rx_superstate * lru_superstate;
+  struct rx_superstate * semifree_superstate;
+
+  struct rx_superset * empty_superset;
+
+  int superstates;
+  int semifree_superstates;
+  int hits;
+  int misses;
+  int superstates_allowed;
+
+  int local_cset_size;
+  void ** instruction_table;
+
+  struct rx_hash superset_table;
+};
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+struct rx_string_position
+{
+  const  unsigned char * pos;	 
+  const  unsigned char * string;  
+  const  unsigned char * end;	 
+  int offset;			 
+  int size;			 
+  int search_direction;		 
+  int search_end;		 
+};
+
+
+enum rx_get_burst_return
+{
+  rx_get_burst_continuation,
+  rx_get_burst_error,
+  rx_get_burst_ok,
+  rx_get_burst_no_more
+};
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+typedef enum rx_get_burst_return
+  (*rx_get_burst_fn) (struct rx_string_position * pos,
+		      void * app_closure,
+		      int stop);
+					       
+
+
+
+
+
+enum rx_back_check_return
+{
+  rx_back_check_continuation,
+  rx_back_check_error,
+  rx_back_check_pass,
+  rx_back_check_fail
+};
+
+ 
+
+
+
+
+
+
+
+
+
+typedef enum rx_back_check_return
+  (*rx_back_check_fn) (struct rx_string_position * pos,
+		       int lparen,
+		       int rparen,
+		       unsigned char * translate,
+		       void * app_closure,
+		       int stop);
+					       
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+typedef int (*rx_fetch_char_fn) (struct rx_string_position * pos,
+				 int offset,
+				 void * app_closure,
+				 int stop);
+
+
+
+
+
+enum rx_search_return
+{
+  rx_search_continuation = -4,
+  rx_search_error = -3,
+  rx_search_soft_fail = -2,	 
+  rx_search_fail = -1		 
+   
+};
+
+
+
+
+
+
+
+ 
+
+
+
+
+ 
+
+extern const  char *re_error_msg[];
+
+ 
+
+typedef enum
+{
+  REG_NOERROR = 0,	 
+  REG_NOMATCH,		 
+
+   
+
+  REG_BADPAT,		 
+  REG_ECOLLATE,		 
+  REG_ECTYPE,		 
+  REG_EESCAPE,		 
+  REG_ESUBREG,		 
+  REG_EBRACK,		 
+  REG_EPAREN,		  
+  REG_EBRACE,		 
+  REG_BADBR,		 
+  REG_ERANGE,		 
+  REG_ESPACE,		 
+  REG_BADRPT,		 
+
+   
+  REG_EEND,		 
+  REG_ESIZE,		 
+  REG_ERPAREN		 
+} reg_errcode_t;
+
+ 
+
+
+
+
+enum re_side_effects
+{
+
+
+
+
+
+# 1 "../../lib/rx.h" 1
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+# 3674 "../../lib/rx.h"
+
+   
+
+
+
+
+
+
+
+
+
+
+
+ 
+    re_se_try    = -1 , 		 
+
+    re_se_pushback    = re_se_try - 1 , 
+    re_se_push0    = re_se_pushback -1 , 
+    re_se_pushpos    = re_se_push0 - 1 , 
+    re_se_chkpos    = re_se_pushpos -1 , 
+    re_se_poppos    = re_se_chkpos - 1 , 
+
+    re_se_at_dot    = re_se_poppos - 1 , 	 
+    re_se_syntax    = re_se_at_dot - 1 ,   
+    re_se_not_syntax    = re_se_syntax - 1 ,   
+
+    re_se_begbuf    = re_se_not_syntax - 1 ,   
+    re_se_hat    = re_se_begbuf - 1 ,   
+
+    re_se_wordbeg    = re_se_hat - 1 ,  
+    re_se_wordbound    = re_se_wordbeg - 1 , 
+    re_se_notwordbound    = re_se_wordbound - 1 , 
+
+    re_se_wordend    = re_se_notwordbound - 1 , 
+    re_se_endbuf    = re_se_wordend - 1 , 
+
+   
+
+
+
+    re_se_dollar    = re_se_endbuf - 1 , 
+
+   
+    re_se_fail    = re_se_dollar - 1 , 
+
+   
+
+
+
+    re_se_win    = 0 , 
+    re_se_lparen    = re_se_win + 1 , 
+    re_se_rparen    = re_se_lparen + 1 , 
+    re_se_backref    = re_se_rparen + 1 , 
+    re_se_iter    = re_se_backref + 1 ,  
+    re_se_end_iter    = re_se_iter + 1 , 
+    re_se_tv    = re_se_end_iter + 1 , 
+
+
+
+
+# 1182 "../../lib/rx.h" 2
+
+
+
+
+   re_floogle_flap = 65533
+};
+
+ 
+
+
+
+
+struct re_se_params
+{
+  enum re_side_effects se;
+  int op1;
+  int op2;
+};
+
+typedef unsigned reg_syntax_t;
+
+struct re_pattern_buffer
+{
+  struct rx rx;
+  reg_syntax_t syntax;		 
+
+  unsigned int no_sub:1;	 
+  unsigned int not_bol:1;	 
+  unsigned int not_eol:1;	   
+  unsigned int newline_anchor:1; 
+  unsigned int least_subs:1;	 
+
+
+
+
+
+   
+
+
+
+  unsigned int match_regs_on_stack:1;
+  unsigned int search_regs_on_stack:1;
+
+   
+  unsigned int is_anchored:1;	 
+  unsigned int begbuf_only:1;	 
+
+  
+   
+
+
+
+
+
+
+
+  unsigned int regs_allocated:2;
+
+  
+   
+
+
+
+
+  unsigned char * translate;
+
+   
+
+
+
+
+
+
+
+
+
+  char * syntax_parens;
+
+	 
+  size_t re_nsub;
+
+  void * buffer;		 
+  unsigned long allocated;	 
+
+   
+
+
+  char *fastmap;
+
+  unsigned int fastmap_accurate:1;  
+  unsigned int can_match_empty:1;  
+  struct rx_nfa_state * start;	 
+
+   
+
+
+  struct re_se_params *se_params;
+
+   
+
+
+
+  rx_Bitset fastset;
+};
+
+ 
+typedef int regoff_t;
+
+ 
+
+struct re_registers
+{
+  unsigned num_regs;
+  regoff_t *start;
+  regoff_t *end;
+};
+
+typedef struct re_pattern_buffer regex_t;
+
+ 
+
+
+typedef struct
+{
+  regoff_t rm_so;   
+  regoff_t rm_eo;   
+} regmatch_t;
+
+
+
+ 
+
+
+
+
+
+ 
+
+
+
+ 
+
+
+
+
+ 
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+ 
+
+
+
+ 
+
+
+
+ 
+
+
+
+ 
+
+
+
+ 
+
+
+
+
+ 
+
+
+
+ 
+
+
+
+ 
+
+
+
+
+ 
+
+
+
+ 
+
+
+
+ 
+
+
+
+ 
+
+
+
+
+
+ 
+
+
+
+ 
+
+
+
+extern reg_syntax_t re_syntax_options;
+
+
+ 
+
+ 
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+ 
+
+ 
+
+
+
+
+
+
+
+
+
+ 
+
+ 
+
+
+
+ 
+
+
+ 
+ 
+
+
+
+
+ 
+
+
+
+
+ 
+
+ 
+
+
+
+
+
+
+ 
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+extern int rx_cache_bound;
+extern char rx_version_string[];
+
+
+
+
+# 1691 "../../lib/rx.h"
+
+
+
+
+
+
+extern int re_search_2 (struct re_pattern_buffer *rxb,
+			const  char * string1, int size1,
+			const  char * string2, int size2,
+			int startpos, int range,
+			struct re_registers *regs,
+			int stop);
+extern int re_search (struct re_pattern_buffer * rxb, const  char *string,
+		      int size, int startpos, int range,
+		      struct re_registers *regs);
+extern int re_match_2 (struct re_pattern_buffer * rxb,
+		       const  char * string1, int size1,
+		       const  char * string2, int size2,
+		       int pos, struct re_registers *regs, int stop);
+extern int re_match (struct re_pattern_buffer * rxb,
+		     const  char * string,
+		     int size, int pos,
+		     struct re_registers *regs);
+extern reg_syntax_t re_set_syntax (reg_syntax_t syntax);
+extern void re_set_registers (struct re_pattern_buffer *bufp,
+			      struct re_registers *regs,
+			      unsigned num_regs,
+			      regoff_t * starts, regoff_t * ends);
+extern const  char * re_compile_pattern (const  char *pattern,
+					int length,
+					struct re_pattern_buffer * rxb);
+extern int re_compile_fastmap (struct re_pattern_buffer * rxb);
+extern char * re_comp (const  char *s);
+extern int re_exec (const  char *s);
+extern int regcomp (regex_t * preg, const  char * pattern, int cflags);
+extern int regexec (const  regex_t *preg, const  char *string,
+		    size_t nmatch, regmatch_t pmatch[],
+		    int eflags);
+extern size_t regerror (int errcode, const  regex_t *preg,
+			char *errbuf, size_t errbuf_size);
+extern void regfree (regex_t *preg);
+
+# 1748 "../../lib/rx.h"
+
+
+
+
+
+# 3670 "../../lib/rx.h"
+
+
+
+
+# 3731 "../../lib/rx.h"
+
+
+
+# 31 "grbTest.cpp" 2
+
+
+
+}
+
+# 1 "../../lib/non_ansi.h" 1
+extern "C" int fileno(FILE *);
+extern "C" int ftruncate(int, long);
+
+extern "C" int pclose(FILE *);
+extern "C" FILE *popen(const char *command, const char *type);
+
+extern "C" int getpagesize();
+
+extern "C" int strcasecmp(const char *, const char *);
+# 36 "grbTest.cpp" 2
+
+
+# 1 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stdarg.h" 1 3
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+void va_end (__gnuc_va_list);		 
+
+
+ 
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 175 "/usr/local/lib/gcc-lib/i686-pc-linux-gnulibc1/2.95.1/include/stdarg.h" 3
+
+
+ 
+
+
+
+
+ 
+
+ 
+
+ 
+
+typedef __gnuc_va_list va_list;
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 38 "grbTest.cpp" 2
+
+# 1 "/usr/local/include/signal.h" 1 3
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+extern "C" { 
+
+# 1 "/usr/local/include/bits/sigset.h" 1 3
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 33 "/usr/local/include/bits/sigset.h" 3
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+ 
+
+
+# 98 "/usr/local/include/bits/sigset.h" 3
+
+
+ 
+
+
+extern int __sigismember (const  __sigset_t *, int);
+extern int __sigaddset (__sigset_t *, int);
+extern int __sigdelset (__sigset_t *, int);
+
+# 122 "/usr/local/include/bits/sigset.h" 3
+
+
+
+
+# 33 "/usr/local/include/signal.h" 2 3
+
+
+ 
+
+
+
+
+
+typedef __sig_atomic_t sig_atomic_t;
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 1 "/usr/local/include/bits/signum.h" 1 3
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+# 56 "/usr/local/include/signal.h" 2 3
+
+
+
+
+
+
+
+
+ 
+typedef void (*__sighandler_t)  (int)  ;
+
+ 
+
+
+extern __sighandler_t __sysv_signal  (int __sig,
+					  __sighandler_t __handler)  ;
+
+
+
+
+ 
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+extern int raise  (int __sig)  ;
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+extern int __sigpause  (int __sig_or_mask, int __is_sig)  ;
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 164 "/usr/local/include/signal.h" 3
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+# 259 "/usr/local/include/signal.h" 3
+
+
+# 299 "/usr/local/include/signal.h" 3
+
+
+
+# 322 "/usr/local/include/signal.h" 3
+
+
+# 342 "/usr/local/include/signal.h" 3
+
+
+ 
+
+
+ 
+extern int __libc_current_sigrtmin  (void)  ;
+ 
+extern int __libc_current_sigrtmax  (void)  ;
+
+
+
+} 
+
+
+# 39 "grbTest.cpp" 2
+
+
+
+# 1 "../../lib/messages.h" 1
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+extern char *msg_prefix;
+
+void   FatalError (int ExitCode, const char *fmt,...);
+
+
+extern "C" 
+
+void Message (const char *fmt,...);
+
+
+
+
+
+
+# 42 "grbTest.cpp" 2
+
+# 1 "../../lib/timing.h" 1
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 1 "/usr/local/include/sys/times.h" 1 3
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+extern "C" { 
+
+ 
+struct tms
+  {
+    clock_t tms_utime;		 
+    clock_t tms_stime;		 
+
+    clock_t tms_cutime;		 
+    clock_t tms_cstime;		 
+  };
+
+
+ 
+
+
+
+extern clock_t times  (struct tms *__buffer)  ;
+
+} 
+
+
+# 30 "../../lib/timing.h" 2
+
+
+
+
+
+
+
+
+
+
+
+
+# 50 "../../lib/timing.h"
+
+
+
+typedef struct
+  {
+    double RealTime, CPUTime;
+  }
+ProgTime;
+
+double RealTime (void);
+
+double CPUTime (double *user, double *sys);
+
+void GetTime (ProgTime * StartTime);
+
+ 
+char *ElapsedTime (ProgTime * StartTime, ProgTime * FinishTime);
+
+
+char *cputime_string (clock_t clk);
+
+
+
+
+
+
+# 43 "grbTest.cpp" 2
+
+# 1 "../../lib/memlib.h" 1
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+typedef void *(*Malloc_func) (size_t);
+typedef void *(*Realloc_func) (void *, size_t);
+typedef void (*Free_func) (void *);
+typedef char *(*Strdup_func) (const char *);
+
+extern Malloc_func Xmalloc;
+extern Realloc_func Xrealloc;
+extern Free_func Xfree;
+extern Strdup_func Xstrdup;
+
+
+
+# 44 "grbTest.cpp" 2
+
+# 1 "../../lib/local_strings.h" 1
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+int arg_atoi (char *str);
+int compare (unsigned char  * s1, unsigned char  * s2);
+int casecompare (unsigned char  * s1, unsigned char  * s2);    
+unsigned char  *copy_string (unsigned char  * src_str);
+char *str255_to_string (unsigned char  * str255, char *str);
+int prefixlen (unsigned char  * s1, unsigned char  * s2);
+char *char2str (unsigned char  c);
+char *word2str (unsigned char  * s);
+char *de_escape_string (char *s);
+
+
+# 45 "grbTest.cpp" 2
+
+
+# 1 "../../lib/filestats.h" 1
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+typedef struct FileStats
+  {
+    unsigned long NumSeeks;
+    unsigned long NumReads;
+    unsigned long NumBytes;
+  }
+FileStats;
+
+
+
+typedef struct
+  {
+    FILE *f;
+    char *pathname;
+    char *name;
+    FileStats Current;
+    FileStats Cumulative;
+  }
+File;
+
+ 
+File *Fopen (char *name, char *mode, unsigned long magic);
+
+size_t Fread (void *ptr, size_t size, size_t nitems, File * F);
+
+int Fseek (File * F, long offset, int ptrname);
+
+void Rewind (File * F);
+
+int Fclose (File * F);
+
+
+
+ 
+
+void ZeroFileStats (File * F);
+
+
+
+# 47 "grbTest.cpp" 2
+
+# 1 "invf.h" 1
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+struct invf_dict_header
+  {
+    unsigned long lookback;
+    unsigned long dict_size;
+    unsigned long total_bytes;
+    unsigned long index_string_bytes;
+    double input_bytes;  
+    unsigned long num_of_docs;
+    unsigned long static_num_of_docs;
+    unsigned long num_of_words;
+    unsigned long stemmer_num;
+    unsigned long stem_method;
+  };
+
+struct stem_dict_header
+  {
+    unsigned long lookback;
+    unsigned long block_size;
+    unsigned long num_blocks;
+    unsigned long blocks_start;
+    unsigned long index_chars;
+    unsigned long num_of_docs;
+    unsigned long static_num_of_docs;
+    unsigned long num_of_words;
+    unsigned long stemmer_num;
+    unsigned long stem_method;
+    unsigned long indexed;        
+  };
+
+struct invf_file_header
+  {
+    unsigned long no_of_words;
+    unsigned long no_of_ptrs;
+    unsigned long skip_mode;
+    unsigned long params[16];
+    unsigned long InvfLevel;
+  };
+
+ 
+struct stem_idx_header
+  {
+    unsigned long lookback;
+    unsigned long block_size;
+    unsigned long num_blocks;
+    unsigned long blocks_start;
+    unsigned long index_chars;
+    unsigned long num_of_words;
+  };
+
+
+# 48 "grbTest.cpp" 2
+
+# 1 "text.h" 1
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 1 "../../lib/huffman.h" 1
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+typedef struct huff_data
+  {
+    int num_codes;
+    int mincodelen;
+    int maxcodelen;
+    int lencount[31  + 1];
+    unsigned long min_code[31  + 1];
+    char *clens;
+  }
+huff_data;
+
+
+huff_data *Generate_Huffman_Data (int num, long *freqs, huff_data * data,
+				  unsigned long  * mem);
+
+unsigned long *Generate_Huffman_Codes (huff_data * data, unsigned long  * mem);
+
+unsigned long **Generate_Huffman_Vals (huff_data * data, unsigned long  * mem);
+
+int Write_Huffman_Data (FILE * f, huff_data * hd);
+
+int Read_Huffman_Data (FILE * f, huff_data * hd, unsigned long  * mem, unsigned long  * disk);
+
+int F_Read_Huffman_Data (File * f, huff_data * hd, unsigned long  * mem, unsigned long  * disk);
+
+ 
+
+
+double Calculate_Huffman_Size (int num, long *freqs, long *counts);
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 95 "../../lib/huffman.h"
+
+
+
+# 111 "../../lib/huffman.h"
+
+
+
+
+# 29 "text.h" 2
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+ 
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+ 
+
+
+
+
+ 
+
+
+
+
+ 
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+typedef struct compression_dict_header
+  {
+    unsigned long  dict_type;
+    unsigned long  novel_method;
+    unsigned long  params[15 ];
+    unsigned long  num_words[2];
+    unsigned long  num_word_chars[2];
+    unsigned long  lookback;
+  }
+compression_dict_header;
+
+
+typedef struct comp_frags_header
+  {
+    huff_data hd;
+    unsigned long  uncompressed_size;
+    unsigned long  huff_words_size[31  + 1];
+  }
+comp_frags_header;
+
+
+
+typedef struct compressed_text_header
+  {
+    unsigned long  num_of_docs;
+    unsigned long  num_of_words;	 
+    double num_of_bytes;  
+    double ratio;
+    unsigned long  length_of_longest_doc;	 
+    unsigned long  dummy;     
+  }
+compressed_text_header;
+
+
+typedef struct compression_stats_header
+  {
+    unsigned long  num_docs;
+    unsigned long  dummy;    
+    double num_bytes;  
+  }
+compression_stats_header;
+
+
+typedef struct frags_stats_header
+  {
+    unsigned long  num_frags;
+    unsigned long  mem_for_frags;
+  }
+frags_stats_header;
+
+
+typedef struct aux_frags_header
+  {
+    unsigned long  num_frags;
+    unsigned long  mem_for_frags;
+  }
+aux_frags_header;
+
+
+
+# 49 "grbTest.cpp" 2
+
+# 1 "mg.h" 1
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 50 "grbTest.cpp" 2
+
+# 1 "lists.h" 1
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+typedef struct DocEntry
+  {
+    float Weight;
+    int DocNum;
+    unsigned long SeekPos;	 
+    unsigned long Len;		 
+    char *CompTextBuffer;
+    struct DocEntry *Next;
+    short or_included;		 
+  }
+DocEntry;
+
+typedef struct DocList
+  {
+    int num;
+    int total_retrieved;  
+    int is_approx;        
+    DocEntry DE[1];
+  }
+DocList;
+
+DocList *MakeDocList (int num);
+DocList *ResizeDocList (DocList * d, int num);
+DocList *IntersectLists (DocList * List1, DocList * List2);
+DocList *DiffLists (DocList * List1, DocList * List2);
+DocList *MergeLists (DocList * List1, DocList * List2);
+DocList *AddToDocList(DocList *List, int res);
+void     ReduceList_OrIncluded(DocList *List);
+
+
+inline void DocEntry_init(DocEntry *d, int doc, float weight)
+{ d->DocNum         = doc;
+  d->Weight         = weight;
+  d->SeekPos        = 0;
+  d->Len            = 0;
+  d->CompTextBuffer = ((void *)0) ;
+  d->Next           = ((void *)0) ;
+  d->or_included    = 0;
+}
+
+inline int DocList_isEmpty(DocList *List)
+{ return (List == ((void *)0)  || List->num == 0);
+}
+
+inline int DocList_size(DocList *List)
+{ return List->num;
+}
+
+inline void DocList_resize(DocList *List, int n)
+{ List->num = n;
+}
+
+inline DocEntry *DocList_member(DocList *List, int at)
+{ return &List->DE[at];
+}
+
+inline DocEntry *DocList_nextMember(DocList *List, DocEntry *entry)
+{ entry ++;
+  return entry; 
+}
+
+inline int DocList_memberValid(DocList *List, DocEntry *entry)
+{ return (entry < (List->DE + List->num));
+}
+
+inline DocEntry *DocList_lastMember(DocList *List)
+{ return DocList_member(List, List->num - 1);
+}
+
+inline void DocList_setMember(DocList *List, int at, DocEntry entry)
+{ List->DE[at] = entry;
+}
+
+inline void DocList_moveMember(DocList *List, int to, int from)
+{ List->DE[to] = List->DE[from];
+}
+
+inline void DocList_addMember(DocList *List, DocEntry entry)
+{ List->DE[List->num] = entry;
+  List->num ++;
+}
+
+inline void DocList_addMember1(DocList *List, int doc, float weight)
+{ DocEntry_init(DocList_member(List, List->num), doc, weight);
+  List->num ++;
+}
+
+inline unsigned long *DocList_memberSeekPtr(DocList *List, int doc)
+{ return &List->DE[doc].SeekPos;
+}
+
+inline unsigned long *DocList_memberLenPtr(DocList *List, int doc)
+{ return &List->DE[doc].Len;
+}
+
+inline int DocList_memberDocNum(DocList *List, int doc)
+{ return List->DE[doc].DocNum;
+}
+
+
+# 51 "grbTest.cpp" 2
+
+# 1 "backend.h" 1
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# 1 "term_lists.h" 1
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+typedef struct WordEntry
+  {
+    int word_num;		 
+    unsigned long  count;		 
+    unsigned long  doc_count;		 
+    unsigned long  max_doc_count;        
+    unsigned long  invf_ptr;		 
+
+    unsigned long  invf_len;		 
+
+  }
+WordEntry;
+
+typedef struct TermEntry
+  {
+    WordEntry WE;
+    int Count;			 
+    unsigned char  *Word;		 
+    unsigned char  *Stem;                
+    int require_match;           
+  }
+TermEntry;
+
+typedef struct TermList
+  {
+    int list_size;
+    int num;
+    TermEntry TE[1];
+  }
+TermList;
+
+
+
+ 
+void ConvertTermsToString (TermList * query_term_list, char *str);
+int AddTermEntry (TermList ** query_term_list, TermEntry * te);
+int AddTerm (TermList ** query_term_list, unsigned char  * Word, int Count, int word_num,
+	     unsigned long  count, unsigned long  doc_count, unsigned long  invf_ptr, unsigned long  invf_len,  
+	     int stemmer_num);  
+
+extern "C"
+
+void ResetTermList (TermList ** tl);
+void FreeTermList (TermList ** the_tl);
+void PrintWordEntry (WordEntry * we, FILE * file);
+void PrintTermEntry (TermEntry * te, FILE * file);
+void PrintTermList (TermList * tl, FILE * file);
+TermList *MakeTermList (int n);
+
+
+
+# 32 "backend.h" 2
+
+# 1 "query_term_list.h" 1
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+typedef struct QueryTermEntry
+  {
+    int Count;			 
+    unsigned char  *Term;		 
+    int stem_method;             
+  }
+QueryTermEntry;
+
+typedef struct QueryTermList
+  {
+    int list_size;
+    int num;
+    QueryTermEntry QTE[1];
+  }
+QueryTermList;
+
+ 
+void ConvertQueryTermsToString (QueryTermList * query_term_list, char *str);
+
+
+extern "C"
+
+int AddQueryTerm (QueryTermList ** query_term_list, unsigned char  * Term, int Count, int stem_method);
+
+
+extern "C"
+
+void ResetQueryTermList (QueryTermList ** tl);
+void FreeQueryTermList (QueryTermList ** the_qtl);
+QueryTermList *MakeQueryTermList (int n);
+
+
+
+# 33 "backend.h" 2
+
+
+
+
+
+typedef struct invf_data
+  {
+    File *InvfFile;
+    unsigned long N;
+    unsigned long Nstatic;	 
+    struct invf_file_header ifh;
+  }
+invf_data;
+
+typedef struct __idx_data
+{
+  unsigned long Start;
+  float Weight;
+} __idx_data;
+     
+
+typedef struct text_data
+  {
+    File *TextFile;
+    File *TextIdxFile;
+    File *TextIdxWgtFile;
+    long current_pos;
+    __idx_data *idx_data;
+    compressed_text_header cth;
+  }
+text_data;
+
+
+typedef struct auxiliary_dict
+  {
+    aux_frags_header afh[2];
+    unsigned char  *word_data[2];
+    unsigned char  **words[2];
+    int blk_start[2][33], blk_end[2][33];	 
+
+  }
+auxiliary_dict;
+
+
+typedef struct compression_dict
+  {
+    compression_dict_header cdh;
+    comp_frags_header *cfh[2];
+    unsigned long MemForCompDict;
+    unsigned char  ***values[2];
+    unsigned char  *escape[2];
+    huff_data *chars_huff[2];
+    unsigned long  **chars_vals[2];
+    huff_data *lens_huff[2];
+    unsigned long  **lens_vals[2];
+    auxiliary_dict *ad;
+    int fast_loaded;
+  }
+compression_dict;
+
+
+typedef struct stemmed_idx    
+  {
+    File *stem_idx_file;
+    struct stem_idx_header sih;
+    unsigned char  ** strchr ;
+    unsigned long *pos;
+    int active;
+    unsigned char  *buffer;
+    unsigned long MemForStemIdx;
+  }
+stemmed_idx;
+
+
+typedef struct stemmed_dict
+  {
+    File *stem_file;
+    struct stem_dict_header sdh;
+    unsigned char  ** strchr ;
+    unsigned long *pos;
+    int active;
+    unsigned char  *buffer;
+    unsigned long MemForStemDict;
+
+     
+    stemmed_idx *stem1;
+    stemmed_idx *stem2;
+    stemmed_idx *stem3;
+  }
+stemmed_dict;
+
+
+typedef struct approx_weights_data
+  {
+    double L;
+    double B;
+    unsigned long *DocWeights;
+    char bits;
+    float *table;
+    unsigned long mask;
+    unsigned long MemForWeights;
+    unsigned long num_of_docs;
+  }
+approx_weights_data;
+
+
+typedef struct RankedQueryInfo
+  {
+    int QueryFreqs;
+    int Exact;            
+    long MaxDocsToRetrieve;	 
+    long MaxParasToRetrieve;
+    int Sort;        
+    char AccumMethod;		 
+    long MaxAccums;		 
+    long MaxTerms;		 
+    int StopAtMaxAccum;  
+    long HashTblSize;
+    char *skip_dump;
+  }
+RankedQueryInfo;
+
+
+
+typedef struct BooleanQueryInfo
+  {
+    long MaxDocsToRetrieve;
+  }
+BooleanQueryInfo;
+
+
+ 
+
+
+typedef struct query_data
+  {
+    stemmed_dict *sd;
+    compression_dict *cd;
+    approx_weights_data *awd;
+    invf_data *id;
+    text_data *td;
+
+
+
+    char *pathname;
+    char *textpathname;  
+    File *File_text;
+    File *File_comp_dict;
+    File *File_aux_dict;
+    File *File_fast_comp_dict;
+    File *File_text_idx_wgt;
+    File *File_text_idx;
+    File *File_stem;
+
+     
+    File *File_stem1;
+    File *File_stem2;
+    File *File_stem3;
+
+    File *File_invf;
+    File *File_weight_approx;
+    unsigned long mem_in_use, max_mem_in_use;
+    unsigned long num_of_ptrs, tot_num_of_ptrs;
+    unsigned long num_of_terms, tot_num_of_terms;
+    unsigned long num_of_accum, tot_num_of_accum;
+    unsigned long num_of_ans, tot_num_of_ans;
+    unsigned long hops_taken, tot_hops_taken;
+    unsigned long text_idx_lookups, tot_text_idx_lookups;
+    unsigned long max_buffers;
+    unsigned doc_pos;
+    unsigned buf_in_use;
+    DocList *DL;
+    TermList *TL;		 
+    unsigned char  *TextBuffer;
+    int TextBufferLen;
+    QueryTermList *QTL;     
+    int allfiles;           
+  }
+query_data;
+
+
+
+typedef struct InitQueryTimes
+  {
+    ProgTime Start;
+    ProgTime StemDict;
+    ProgTime ApproxWeights;
+    ProgTime CompDict;
+    ProgTime Invf;
+    ProgTime Text;
+  }
+InitQueryTimes;
+
+
+ 
+query_data *InitQuerySystem (char *dir, char *name, char *textname, InitQueryTimes * iqt);
+
+void ChangeMemInUse (query_data * qd, long delta);
+
+void FinishQuerySystem (query_data * qd);
+
+void ResetFileStats (query_data * qd);
+
+void TransFileStats (query_data * qd);
+
+void ChangeMemInUse (query_data * qd, long delta);
+
+void RankedQuery (query_data * qd, char *Query, RankedQueryInfo * rqi);
+
+void BooleanQuery (query_data * qd, char *Query, BooleanQueryInfo * bqi,
+		   int stem_method);
+
+void DocnumsQuery (query_data * qd, char *QueryLine);
+
+void FreeTextBuffer (query_data * qd);
+
+
+
+
+
+
+
+# 52 "grbTest.cpp" 2
+
+# 1 "DocEntry.h" 1
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+void FreeQueryDocs (query_data * qd);
+
+int LoadCompressedText (query_data * qd, int max_mem);
+
+int GetDocNum (query_data * qd);
+
+float GetDocWeight (query_data * qd);
+
+long GetDocCompLength (query_data * qd);
+
+unsigned char  *GetDocText (query_data * qd, unsigned long *len);
+
+DocEntry *GetDocChain (query_data * qd);
+
+int NextDoc (query_data * qd);
+
+int DocEntry_compare(void *a, void *b);
+
+
+# 53 "grbTest.cpp" 2
+
+# 1 "environment.h" 1
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+char get_output_type (void);
+char get_query_type (void);
+
+extern char *ConstraintErrorStr;
+int SetEnv (char *name, char *data, char *(*Constraint) (char *, char *));
+char *GetEnv (char *name);
+char *GetDefEnv (char *name, char *def);
+int UnsetEnv (char *name, int Force);
+char *de_escape_string (char *s);
+int PushEnv (void);
+int PopEnv (void);
+int EnvStackHeight (void);
+char *GetEnvName (int i);
+int BooleanEnv (char *data, int def);
+long IntEnv (char *data, long def);
+
+void InitEnv (void);
+void UninitEnv (void);
+
+
+# 54 "grbTest.cpp" 2
+
+# 1 "globals.h" 1
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+extern FILE *OutFile;
+extern FILE *InFile;
+
+
+# 55 "grbTest.cpp" 2
+
+# 1 "read_line.h" 1
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+void Init_ReadLine (void);
+void WritePrompt (void);
+char *GetMultiLine (void);
+
+
+# 56 "grbTest.cpp" 2
+
+# 1 "mg_errors.h" 1
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+extern int mg_errno;
+extern char *mg_error_data;
+extern char *mg_errorstrs[];
+
+
+
+
+
+
+
+
+
+
+void MgErrorData (char *s);
+void MgErrorDeinit (void);
+# 57 "grbTest.cpp" 2
+
+# 1 "commands.h" 1
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+extern char *CommandsErrorStr;
+char *ProcessCommands (char *line, query_data * qd);
+void read_mgrc_file (void);
+
+
+# 58 "grbTest.cpp" 2
+
+# 1 "text_get.h" 1
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+double FetchDocStart (query_data * qd, unsigned long  DN, unsigned long  * seek_pos, unsigned long  * len);
+
+unsigned long FetchInitialParagraph (text_data * td, unsigned long ParaNum);
+
+
+ 
+
+
+
+int FetchCompressed (query_data * qd, char **DocBuff, DocEntry * DocEnt);
+
+text_data *LoadTextData (File * text, File * text_idx_wgt, File * text_idx);
+
+void FreeTextData (text_data * td);
+
+int GetPosLens (query_data * qd, DocEntry * Docs, int num);
+
+int LoadBuffers (query_data * qd, DocEntry * Docs, int max_mem, int num);
+
+void FreeBuffers (query_data * qd, DocEntry * Docs, int num);
+
+
+
+compression_dict *LoadCompDict (File * text_comp_dict,
+				File * text_aux_dict,
+				File * text_fast_comp_dict);
+
+void FreeCompDict (compression_dict * cd);
+
+
+int DecodeText (compression_dict * cd,
+		unsigned char  * s_in, int l_in, unsigned char  * s_out, int *l_out);
+
+
+# 59 "grbTest.cpp" 2
+
+
+
+
+
+void main()
+{ int I;
+
+  if (((( ( 0 ) ) & 0170000 ) == (  0100000  ))   || S_IFREG)
+    { I = 0;
+    }
+  else
+    { I = 1;
+    }
+
+}
Index: /tags/gsdl-21/gsdl/src/mgpp/text/mgstat.1
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/mgstat.1	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/mgstat.1	(revision 699)
@@ -0,0 +1,131 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgstat 1 \*(Dt CITRI
+.SH NAME
+mgstat \- print out statistics about a document collection
+.SH SYNOPSIS
+.B mgstat
+[
+.B \-h
+]
+[
+.B \-E
+]
+[
+.BI \-d " directory"
+]
+.BI \-f " name"
+.SH DESCRIPTION
+.B mgstat
+prints out various statistics about an existing
+.BR mg (1)
+document collection.  Depending on the size of the collection, sizes
+will be printed in either kilobytes or megabytes.
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stdout .
+.TP
+.B \-E
+This option forces sizes to be printed in bytes rather than kilobytes
+or megabytes.
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 20
+.B *.text
+Compressed documents.
+.TP
+.B *.invf
+Inverted file.
+.TP
+.B *.text.idx.wgt
+Interleaved index into the compressed documents and document weights.
+.TP
+.B *.weight.approx
+Approximate document weights.
+.TP
+.B *.invf.dict.blocked
+Compressed stemmed dictionary and index into the inverted file merged
+into an inverted file.
+.TP
+.B *.text.dict.fast
+Fast loading compression dictionary.
+.TP
+.B *.text.dict
+Compressed compression dictionary.
+.TP
+.B *.invf.dict
+Compressed stemmed dictionary.
+.TP
+.B *.invf.idx
+The index into the inverted file.
+.TP
+.B *.text.stats
+Statistics about the text.
+.TP
+.B *.text.dict.aux
+Auxiliary compression dictionary.
+.TP
+.B *.text.idx
+Index into the compressed documents.
+.TP
+.B *.weight
+The exact weights file.
+.TP
+.B *.invf.chunk
+Maps stemmed terms from occurrence order to lexical order.
+.TP
+.B *.invf.chunk.trans
+Describes where the source text is broken up into chunks for the
+inversion pass.
+.TP
+.B *.invf.dict.hash
+A perfect hash function for the terms in the stemmed dictionary.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_compression_dict (1),
+.BR mg_fast_comp_dict (1),
+.BR mg_get (1),
+.BR mg_invf_dict (1),
+.BR mg_invf_dump (1),
+.BR mg_invf_rebuild (1),
+.BR mg_passes (1),
+.BR mg_perf_hash_build (1),
+.BR mg_text_estimate (1),
+.BR mg_weights_build (1),
+.BR mgbilevel (1),
+.BR mgbuild (1),
+.BR mgdictlist (1),
+.BR mgfelics (1),
+.BR mgquery (1),
+.BR mgtic (1),
+.BR mgticbuild (1),
+.BR mgticdump (1),
+.BR mgticprune (1),
+.BR mgticstat (1).
Index: /tags/gsdl-21/gsdl/src/mgpp/text/mgstat.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/mgstat.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/mgstat.cpp	(revision 699)
@@ -0,0 +1,402 @@
+/**************************************************************************
+ *
+ * mgstat.c -- Program to generate statistics on a text collection
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "locallib.h"
+#include "mg.h"
+#include "words.h"
+#include "invf.h"
+#include "text.h"
+#include "longlong.h"
+
+/*
+   $Log$
+   Revision 1.1  1999/10/11 02:58:12  cs025
+   Base install of MG-PP
+
+   Revision 1.1  1999/08/10 21:18:19  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:50  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:31  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.2  1994/09/20  04:41:59  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+char *search_for_collection (char *name);
+int process_file (char *, char *, int);
+void ProcessDict (char *);
+void ProcessStem (char *);
+long ProcessStemBlk (char *name);  /* [RPAP - Jan 97: Stem Index Change] */
+
+static mg_ullong inputbytes = 0;
+static mg_ullong total = 0;
+
+
+
+
+
+int main (int argc, char **argv)
+{
+  mg_ullong sub_total;
+  int fast;
+  char *file_name = "";
+  int ch;
+  int exact = 0;
+  long indexed = 0; /* [RPAP - Jan 97: Stem Index Change] */
+  opterr = 0;
+  while ((ch = getopt (argc, argv, "Ehf:d:")) != -1)
+    switch (ch)
+      {
+      case 'E':
+	exact = 1;
+	break;
+      case 'f':		/* input file */
+	file_name = optarg;
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-f input_file]"
+		 "[-d data directory] [-h]\n", argv[0]);
+	exit (1);
+      }
+
+  if (optind < argc)
+    file_name = search_for_collection (argv[optind]);
+
+  ProcessDict (file_name);
+  ProcessStem (file_name);
+  indexed = ProcessStemBlk (file_name); /* [RPAP - Jan 97: Stem Index Change] */
+
+  {
+    char Name[256];
+    char *s;
+    sprintf (Name, FILE_NAME_FORMAT, get_basepath (), file_name, "");
+    s = strrchr (Name, '/');
+    if (s)
+      *s = '\0';
+    printf ("\nThe collection is in \"%s\"\n", Name);
+  }
+
+
+  printf ("\n\t\tFiles required by mgquery\n");
+  process_file (file_name, TEXT_SUFFIX, exact);
+  process_file (file_name, INVF_SUFFIX, exact);
+  process_file (file_name, TEXT_IDX_WGT_SUFFIX, exact);
+  process_file (file_name, APPROX_WEIGHTS_SUFFIX, exact);
+  process_file (file_name, INVF_DICT_BLOCKED_SUFFIX, exact);
+
+  /* [RPAP - Jan 97: Stem Index Change] */
+  if (indexed)
+    {
+      process_file (file_name, INVF_DICT_BLOCKED_1_SUFFIX, exact);
+      process_file (file_name, INVF_DICT_BLOCKED_2_SUFFIX, exact);
+      process_file (file_name, INVF_DICT_BLOCKED_3_SUFFIX, exact);
+    }
+
+  fast = process_file (file_name, TEXT_DICT_FAST_SUFFIX, exact);
+  if (!fast)
+    {
+      process_file (file_name, TEXT_DICT_SUFFIX, exact);
+      process_file (file_name, TEXT_DICT_AUX_SUFFIX, exact);
+    }
+
+
+  process_file (NULL, "SUB TOTAL", exact);
+  sub_total = total;
+  total = 0;
+
+  printf ("\n\t\tFiles NOT required by mgquery\n");
+  if (fast)
+    {
+      process_file (file_name, TEXT_DICT_SUFFIX, exact);
+      process_file (file_name, TEXT_DICT_AUX_SUFFIX, exact);
+    }
+  process_file (file_name, INVF_DICT_SUFFIX, exact);
+  process_file (file_name, INVF_IDX_SUFFIX, exact);
+  process_file (file_name, TEXT_STATS_DICT_SUFFIX, exact);
+  process_file (file_name, TEXT_IDX_SUFFIX, exact);
+  process_file (file_name, WEIGHTS_SUFFIX, exact);
+  process_file (file_name, INVF_CHUNK_SUFFIX, exact);
+  process_file (file_name, INVF_CHUNK_TRANS_SUFFIX, exact);
+  process_file (file_name, INVF_DICT_HASH_SUFFIX, exact);
+  process_file (file_name, INVF_PARAGRAPH_SUFFIX, exact);
+  process_file (NULL, "SUB TOTAL", exact);
+  total += sub_total;
+  printf ("\n");
+  process_file (NULL, "TOTAL", exact);
+  exit (0);
+
+}
+
+
+
+char *
+search_for_collection (char *name)
+{
+  char *dir = get_basepath ();
+  static char buffer[512];
+  struct stat stat_buf;
+
+  /* Look in the current directory first */
+  if (stat (name, &stat_buf) != -1)
+    {
+      if (S_ISDIR(stat_buf.st_mode))
+	{
+	  /* The name is a directory */
+	  sprintf (buffer, "%s/%s", name, name);
+	  set_basepath (".");
+	  return buffer;
+	}
+    }
+
+  sprintf (buffer, "%s.text", name);
+  if (stat (buffer, &stat_buf) != -1)
+    {
+      if (S_ISREG(stat_buf.st_mode))
+	{
+	  /* The name is a directory */
+	  set_basepath (".");
+	  return name;
+	}
+    }
+  sprintf (buffer, "%s/%s", dir, name);
+  if (stat (buffer, &stat_buf) != -1)
+    {
+      if (S_ISDIR(stat_buf.st_mode))
+	{
+	  /* The name is a directory */
+	  sprintf (buffer, "%s/%s", name, name);
+	  return buffer;
+	}
+    }
+  return name;
+}
+
+
+
+
+
+
+void 
+ProcessDict (char *name)
+{
+  FILE *f;
+  compression_dict_header cdh;
+  int have_cdh = 0;
+  compressed_text_header cth;
+  int have_cth = 0;
+
+  if ((f = open_file (name, TEXT_DICT_SUFFIX, "rb", MAGIC_DICT, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      Read_cdh (f, &cdh, NULL, NULL);
+      fclose (f);
+      have_cdh = 1;
+    }
+
+  if ((f = open_file (name, TEXT_SUFFIX, "rb", MAGIC_TEXT, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    {
+      fread ((char *) &cth, sizeof (cth), 1, f);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(cth.num_of_docs);
+      NTOHD(cth.num_of_bytes); /* [RJM 07/97: 4G limit] */
+      NTOHUL(cth.num_of_words);
+      NTOHUL(cth.length_of_longest_doc);
+      NTOHD(cth.ratio);
+
+      fclose (f);
+      have_cth = 1;
+    }
+
+  if (have_cth)
+    {
+      inputbytes = (long unsigned int) cth.num_of_bytes;
+      printf ("Input bytes                        : %10" ULL_FS ", %8.2f Mbyte\n",
+	      cth.num_of_bytes, (double) cth.num_of_bytes / 1024 / 1024);
+      printf ("Documents                          : %10lu\n", cth.num_of_docs);
+      printf ("Words in collection [dict]         : %10lu\n", cth.num_of_words);
+      printf ("Longest doc in collection [dict]   : %10lu characters\n",
+	      cth.length_of_longest_doc);
+      printf ("Maximum ratio                      : %10.2f\n", cth.ratio);
+    }
+
+  if (have_cdh)
+    {
+      printf ("Words in dict                      : %10lu\n", cdh.num_words[1]);
+      printf ("Non-words in dict                  : %10lu\n", cdh.num_words[0]);
+      printf ("Total chars of distinct words      : %10lu\n", cdh.num_word_chars[1]);
+      printf ("Total chars of distinct non-words  : %10lu\n", cdh.num_word_chars[0]);
+    }
+
+}
+
+
+
+
+
+
+
+
+void 
+ProcessStem (char *name)
+{
+  FILE *f;
+  struct invf_dict_header idh;
+
+  if (!(f = open_file (name, INVF_DICT_SUFFIX, "rb",
+		       MAGIC_STEM_BUILD, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return;
+  fread ((char *) &idh, sizeof (idh), 1, f);
+  printf ("Words in collection [stem]         : %10ld\n", NTOHUL(idh.num_of_words));  /* [RPAP - Jan 97: Endian Ordering] */
+  printf ("Words in stem                      : %10ld\n", NTOHUL(idh.dict_size));  /* [RPAP - Jan 97: Endian Ordering] */
+  printf ("Indexed fragments                  : %10ld\n", NTOHUL(idh.num_of_docs));  /* [RPAP - Jan 97: Endian Ordering] */
+  printf ("Total chars of stem words          : %10ld\n", NTOHUL(idh.total_bytes));  /* [RPAP - Jan 97: Endian Ordering] */
+  fclose (f);
+}
+
+
+
+
+
+/* [RPAP - Jan 97: Stem Index Change] */
+long 
+ProcessStemBlk (char *name)
+{
+  FILE *f;
+  struct stem_dict_header sdh;
+
+  if (!(f = open_file (name, INVF_DICT_BLOCKED_SUFFIX, "rb",
+		       MAGIC_STEM, MG_MESSAGE)))  /* [RPAP - Feb 97: WIN32 Port] */
+    return 0;
+  fread ((char *) &sdh, sizeof (sdh), 1, f);
+  fclose (f);
+  return NTOHUL(sdh.indexed);  /* [RPAP - Jan 97: Endian Ordering] */
+}
+
+
+
+
+int 
+process_file (char *name, char *ext, int exact)
+{
+  static double scale = 0;
+  static char *units;
+  struct stat buf;
+  if (scale == 0)
+    {
+      /* So can get output in Mb or Kb, */
+      /* will divide by a scale of 1024(Kb) or 1024*1024(Mb) */
+      /* Note: if inputbytes==0, then use 1024 as default */
+      scale = inputbytes > 10 * 1000 * 1000 ? 1024 * 1024 : 1024;
+      units = (char *) ((scale == 1024) ? "Kb" : "Mb");
+    }
+  if (name)
+    {
+      char Name[256];
+      sprintf (Name, FILE_NAME_FORMAT, get_basepath (), name, ext);
+      if (!stat (Name, &buf))
+	{
+	  char fname[256];
+	  char *nam = strrchr (name, '/');
+	  nam = nam ? nam + 1 : name;
+	  sprintf (fname, "%s%s", nam, ext);
+
+	  if (inputbytes == 0.0)
+	    {
+	      if (exact)
+		printf ("%s%*s: %10ld bytes   %7.3f%%\n", fname,
+			35 - (int) strlen (fname), "",
+			buf.st_size,
+			100.0 * buf.st_size / inputbytes);
+	      else
+		printf ("%s%*s: %8.2f %s   %7.3f%%\n", fname,
+			35 - (int) strlen (fname), "",
+			buf.st_size / scale,
+			units,
+			100.0 * buf.st_size / inputbytes);
+	    }
+	  else
+	    {
+	      if (exact)
+		printf ("%s%*s: %10ld bytes\n", fname,
+			35 - (int) strlen (fname), "",
+			buf.st_size);
+	      else
+		printf ("%s%*s: %8.2f %s\n", fname,
+			35 - (int) strlen (fname), "",
+			buf.st_size / scale,
+			units);
+	    }
+	  total += buf.st_size;
+	  return 1;
+	}
+      else
+	return 0;
+    }
+  else
+    {
+      if (inputbytes == 0.0)
+	{
+	  if (exact)
+	    printf ("%-34s : %10" ULL_FS " bytes   %7.3f%%\n", ext,
+		    total,
+		    100.0 * total / inputbytes);
+	  else
+	    printf ("%-34s : %8.2f %s   %7.3f%%\n", ext,
+		    total / scale, units,
+		    100.0 * total / inputbytes);
+	}
+      else
+	{
+	  if (exact)
+	    printf ("%-34s : %10" ULL_FS " bytes\n", ext, total);
+	  else
+	    printf ("%-34s : %8.2f %s\n", ext,
+		    total / scale, units);
+	}
+    }
+  return 1;
+
+}
Index: /tags/gsdl-21/gsdl/src/mgpp/text/mgstemidxlist.1
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/mgstemidxlist.1	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/mgstemidxlist.1	(revision 699)
@@ -0,0 +1,86 @@
+.\"------------------------------------------------------------
+.\" Id - set Rv,revision, and Dt, Date using rcs-Id tag.
+.de Id
+.ds Rv \\$3
+.ds Dt \\$4
+..
+.Id $Id$
+.\"------------------------------------------------------------
+.TH mgstemidxlist 1 \*(Dt CITRI
+.SH NAME
+mgstemidxlist \- dumps the stem index file to STDOUT in ASCII format
+.SH SYNOPSIS
+.B mgstemidxlist
+[
+.B \-h
+]
+[
+.BI \-d " directory"
+]
+.B \-s 1|2|3
+.BI \-f " name"
+.SH DESCRIPTION
+.B mgstemidxlist
+is a utility program which passes through the stem index and displays
+the information.  (This program was used for debugging but is included
+for completeness!)  The header information is printed out, then for each
+block in the file the block header is displayed.  For each word in the
+stem index, the output is formatted as:
+
+.RS
+.I number_of_references \*(lqword\*(rq
+.br
+.RS
+.I "->  number_of_cases block 3-in-4_block offset"
+.br
+.I "->  ..."
+.RE
+.RE
+.SH OPTIONS
+Options may appear in any order.
+.TP "\w'\fB\-d\fP \fIdirectory\fP'u+2n"
+.B \-h
+This displays a usage line on
+.IR stderr .
+.TP
+.B -s 1|2|3
+The stem index file with the associated stem method.
+.br
+1 = casefolded and non-stemmed
+.br
+2 = non-casefolded and stemmed
+.br
+3 = casefolded and stemmed
+.TP
+.BI \-d " directory"
+This specifies the directory where the document collection can be found.
+.TP
+.BI \-f " name"
+This specifies the base name of the document collection.
+.SH ENVIRONMENT
+.TP "\w'\fBMGDATA\fP'u+2n"
+.SB MGDATA
+If this environment variable exists, then its value is used as the
+default directory where the
+.BR mg (1)
+collection files are.  If this variable does not exist, then the
+directory \*(lq\fB.\fP\*(rq is used by default.  The command line
+option
+.BI \-d " directory"
+overrides the directory in
+.BR MGDATA .
+.SH FILES
+.TP 25
+.B *.invf.dict.blocked.1
+Stem index with stem index method 1.
+.TP
+.B *.invf.dict.blocked.2
+Stem index with stem index method 2.
+.TP
+.B *.invf.dict.blocked.3
+Stem index with stem index method 3.
+.SH "SEE ALSO"
+.na
+.BR mg (1),
+.BR mg_invf_dict (1),
+.BR mg_stem_idx (1).
Index: /tags/gsdl-21/gsdl/src/mgpp/text/mgstemidxlist.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/mgstemidxlist.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/mgstemidxlist.cpp	(revision 699)
@@ -0,0 +1,209 @@
+/**************************************************************************
+ *
+ * mgstemidxlist.c -- Text dumper for the stem indexes
+ * Copyright (C) 1997  Ross Peeters
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "messages.h"
+#include "memlib.h"
+#include "local_strings.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg_files.h"
+#include "mg.h"
+#include "invf.h"
+#include "words.h"
+#include "backend.h"
+
+#include "StemIdx.h"
+
+size_t mgstemidxAbstractReader(void *ptr, size_t size, size_t nitems, void *file)
+{ return fread(ptr, size, nitems, (FILE *) file);
+}
+
+char mgstemidxAbstractGetc(void *file)
+{ return fgetc((FILE *) file);
+}
+
+int mgstemidxAbstractSeeker(void *file, long offset, int ptrname)
+{ return fseek((FILE *) file, offset, ptrname);
+}
+
+void
+read_3_in_4 (FILE * idbi)
+{
+  unsigned long i;
+  stemmed_idx *si;
+  u_char *buffer;
+  int block = 0;
+
+  si = StemIdx_header_read(idbi, 0, mgstemidxAbstractReader, mgstemidxAbstractGetc);
+  if (si == NULL)
+    {
+      return;
+    }
+
+  printf ("# lookback     = %lu\n", si->sih.lookback);
+  printf ("# block_size   = %lu\n", si->sih.block_size);
+  printf ("# num_blocks   = %lu\n", si->sih.num_blocks);
+  printf ("# blocks_start = %lu\n", si->sih.blocks_start);
+  printf ("# index_chars  = %lu\n", si->sih.index_chars);
+  printf ("# num_of_words = %lu\n", si->sih.num_of_words);
+
+  block = 0;
+  while (block < si->sih.num_blocks)
+    {
+      unsigned long *first_word;
+      unsigned short *num_words;
+      unsigned short *index;
+      long res;
+      u_char *base;
+      int num_indexes;
+
+      /* Read in next block */
+      StemIdx_block_readNext((void *) idbi, si, block, &first_word, &num_words, &index, &num_indexes,
+			     mgstemidxAbstractSeeker, mgstemidxAbstractReader);
+      /*
+      fseek (idbi, si->pos[block] + si->sih.blocks_start, 0);
+      fread (si->buffer, si->sih.block_size, sizeof (u_char), idbi);
+      si->active = si->pos[block];
+
+      first_word = (unsigned long *) (si->buffer);
+      NTOHUL(*first_word);  // [RPAP - Jan 97: Endian Ordering]
+      num_words = (unsigned short *) (first_word + 1);
+      NTOHUS(*num_words);  // [RPAP - Jan 97: Endian Ordering]
+      index = num_words + 1;
+      num_indexes = ((*num_words - 1) / si->sih.lookback) + 1;
+
+      // [RPAP - Jan 97: Endian Ordering]
+      for (i = 0; i < num_indexes; i++)
+	NTOHUS(index[i]);
+      */
+      base = (u_char *) (index + num_indexes);
+      base += index[0];
+
+      printf ("\n# block      = %d\n", block);
+      printf ("# first_word = %lu\n", *first_word);
+      printf ("# num_words  = %u\n", *num_words);
+
+      res = 0;
+      while (res < *num_words)
+	{
+	  unsigned copy, suff;
+	  u_char prev[MAXSTEMLEN + 1];
+	  unsigned int num_entries, num_cases, blk;
+	  unsigned short blk_index, offset;
+
+	  /* Read word entry */
+	  StemIdx_ReadWordString(&base, prev);
+	  /*
+	  copy = *base++;
+	  suff = *base++;
+	  bcopy ((char *) base, (char *) (prev + copy + 1), suff);
+	  *prev = copy + suff;
+	  base += suff;
+	  */
+	  bcopy ((char *) base, (char *) &num_entries, sizeof (num_entries));
+	  base += sizeof (num_entries);
+	  NTOHUI(num_entries);  // [RPAP - Jan 97: Endian Ordering]
+	  printf ("%u \"%s\"\n", num_entries, word2str (prev));
+
+	  /* For all the PosEntries for the word... */
+	  for (i = 0; i < num_entries; i++)
+	    {
+	      StemIdx_ReadPosEntry(&base, &num_cases, &blk, &blk_index, &offset);
+	      /*
+	      bcopy ((char *) base, (char *) &num_cases, sizeof (num_cases));
+	      NTOHUI(num_cases);  // [RPAP - Jan 97: Endian Ordering] 
+	      base += sizeof (num_cases);
+	      bcopy ((char *) base, (char *) &blk, sizeof (blk));
+	      NTOHUI(blk);  // [RPAP - Jan 97: Endian Ordering] 
+	      base += sizeof (blk);
+	      bcopy ((char *) base, (char *) &blk_index, sizeof (blk_index));
+	      NTOHUS(blk_index);  // [RPAP - Jan 97: Endian Ordering] 
+	      base += sizeof (blk_index);
+	      bcopy ((char *) base, (char *) &offset, sizeof (offset));
+	      NTOHUS(offset);  // [RPAP - Jan 97: Endian Ordering] 
+	      base += sizeof (offset);
+	      */
+
+	      printf ("    -> %4u %4u %4u %4u\n", num_cases, blk, blk_index, offset);
+	    }
+	  res++;
+	}
+      block++;
+    }
+  fclose (idbi);
+}
+
+int
+main (int argc, char **argv)
+{
+  FILE *idbi;
+  char *filename = "";
+  int ch;
+  int stem_method = 0;
+
+  msg_prefix = argv[0];
+  opterr = 0;
+  while ((ch = getopt (argc, argv, "f:d:hs:")) != -1)
+    switch (ch)
+      {
+      case 'f':		/* input file */
+	filename = optarg;
+	break;
+      case 'd':
+	set_basepath (optarg);
+	break;
+      case 's':
+	stem_method = atoi (optarg);
+	break;
+      case 'h':
+      case '?':
+	fprintf (stderr, "usage: %s [-d data directory] [-h] -s 1|2|3 -f name\n", argv[0]);
+	exit (1);
+      }
+
+  /* Open required files */
+  switch (stem_method)
+    {
+    case (1):
+      idbi = open_file (filename, INVF_DICT_BLOCKED_1_SUFFIX, "rb", MAGIC_STEM_1,
+			MG_ABORT);
+      break;
+    case (2):
+      idbi = open_file (filename, INVF_DICT_BLOCKED_2_SUFFIX, "rb", MAGIC_STEM_2,
+			MG_ABORT);
+      break;
+    case (3):
+      idbi = open_file (filename, INVF_DICT_BLOCKED_3_SUFFIX, "rb", MAGIC_STEM_3,
+			MG_ABORT);
+      break;
+    default:
+      FatalError (1, "Stem method must be 1, 2 or 3\n");
+    }
+
+  if (!idbi)
+    FatalError (1, "Could NOT open file");
+
+  read_3_in_4 (idbi);
+
+  return 0;
+}
Index: /tags/gsdl-21/gsdl/src/mgpp/text/query.docnums.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/query.docnums.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/query.docnums.cpp	(revision 699)
@@ -0,0 +1,78 @@
+/**************************************************************************
+ *
+ * query.docnums.c -- Docnums query evaluation
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+#include "filestats.h"
+#include "timing.h"
+
+#include "mg.h"
+#include "invf.h"
+#include "text.h"
+#include "lists.h"
+#include "backend.h"
+#include "DocEntry.h"
+
+
+void 
+DocnumsQuery (query_data * qd, char *QueryLine)
+{
+  DocList *Docs = DocList_create (0);
+  unsigned long start = 0, finish;
+  int range = 0; /* is there a range or not ? */
+
+  /* find the start of the first word */
+  while (*QueryLine)
+    {
+      char *end;
+      finish = strtol (QueryLine, &end, 10);
+      if (end != (char *) QueryLine && finish >= 1 &&
+	  finish <= qd->sd->sdh.num_of_docs)
+	{
+	  unsigned long i;
+	  if (!range)
+	    start = finish;
+	  else
+	    range = 0;
+	  for (i = start; i <= finish; i++)
+	    Docs = DocList_addDoc (Docs, i);
+	  while (*end == ' ' || *end == '\t' || *end == '\n')
+	    end++;
+	  if (*end == '-')
+	    {
+	      start = finish;
+	      range = 1;
+	      end++;
+	    }
+	}
+      else
+	break;
+      QueryLine = end;
+    }
+  QueryData_FreeQueryDocs (qd);
+
+  qd->DL = Docs;
+}
+
+
+
+
Index: /tags/gsdl-21/gsdl/src/mgpp/text/query.ranked.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/query.ranked.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/query.ranked.cpp	(revision 699)
@@ -0,0 +1,1100 @@
+/**************************************************************************
+ *
+ * query.ranked.c -- Ranked query evaluation
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "filestats.h"
+#include "messages.h"
+#include "timing.h"
+#include "sptree.h"
+#include "unitool.h"
+
+#include "mg.h"
+#include "invf.h"
+#include "text.h"
+#include "lists.h"
+#include "backend.h"
+#include "DocEntry.h"
+#include "stem_search.h"
+#include "weights.h"
+#include "text_get.h"
+#include "invf_get.h"
+#include "words.h"
+#include "stemmer.h"
+#include "locallib.h"
+#include "environment.h"
+#include "term_lists.h"
+#include "local_strings.h"
+#include "query_term_list.h"  /* [RPAP - Feb 97: Term Frequency] */
+
+/*
+   $Log$
+   Revision 1.2  1999/10/12 04:13:52  cs025
+   Calls to GetEnv eradicated; interface to parsing of query text
+   cleaned up.
+
+   Revision 1.1  1999/10/11 02:58:14  cs025
+   Base install of MG-PP
+
+   Revision 1.1  1999/08/10 21:18:20  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:50  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:34  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.4  1994/11/25  03:47:46  tes
+   * Committing files before adding the merge stuff.
+   *
+   * Revision 1.3  1994/10/20  03:57:03  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:42:04  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+/*************************************************************************/
+
+class HeapEntry : public loadableDocEntry
+  {
+  public:
+    float *OrgWeight;
+
+    HeapEntry()
+    {
+      // do nothing
+    }
+
+    inline void chain(int docno, float *weightPtr)
+    {
+      *OrgWeight = this->Weight;
+      DocNum     = docno;
+      Weight     = *weightPtr;
+      OrgWeight  = weightPtr;
+      *OrgWeight = 0;
+    }
+
+    inline void setValues(int docno, float _weight, u_long filepos, u_long _len, float *weightPtr)
+    {
+      DocNum = docno;
+      OrgWeight = weightPtr;
+      Weight = _weight;
+      SeekPos = filepos;
+      Len = _len;
+      *weightPtr = 0;
+    }
+  }
+;
+
+typedef int (*HeapComp) (HeapEntry *, HeapEntry *);
+
+typedef struct Heap
+  {
+    int NumItems;
+    int MaxSize;
+    HeapComp HC;
+    HeapEntry HE[1];
+  }
+Heap;
+
+
+Heap *
+Heap_Make (int size, HeapComp hc)
+{
+  Heap *H;
+  H = (Heap *) Xmalloc (sizeof (Heap) + (size - 1) * sizeof (HeapEntry));
+  if (!H)
+    return NULL;
+  H->NumItems = 0;
+  H->MaxSize = size;
+  H->HC = hc;
+  return H;
+}
+
+int 
+Heap_Size (Heap * H)
+{
+  return sizeof (Heap) + H->MaxSize * sizeof (HeapEntry);
+}
+
+HeapEntry *
+Heap_GetHead (Heap * H)
+{
+  if (H && H->NumItems)
+    return &H->HE[0];
+  else
+    return NULL;
+}
+
+
+
+void 
+Heap_Heapify (Heap * H, int i)
+{
+  register int curr, child;
+  curr = i;
+  child = curr * 2;
+  while (child <= H->NumItems)
+    {
+      if (child < H->NumItems && H->HC (&H->HE[child], &H->HE[child - 1]) > 0)
+	child++;
+      if (H->HC (&H->HE[curr - 1], &H->HE[child - 1]) < 0)
+	{
+	  HeapEntry temp = H->HE[child - 1];
+	  H->HE[child - 1] = H->HE[curr - 1];
+	  H->HE[curr - 1] = temp;
+	  curr = child;
+	  child = 2 * child;
+	}
+      else
+	break;
+    }
+}
+
+
+void 
+Heap_Build (Heap * H)
+{
+  register int i;
+  for (i = H->NumItems / 2; i > 0; i--)
+    Heap_Heapify (H, i);
+}
+
+
+void 
+Heap_Sort (Heap * H)
+{
+  register int i;
+  for (i = H->NumItems; i > 1; i--)
+    {
+      HeapEntry temp = H->HE[0];
+      H->HE[0] = H->HE[i - 1];
+      H->HE[i - 1] = temp;
+      H->NumItems--;
+      Heap_Heapify (H, 1);
+    }
+}
+
+
+void 
+Heap_DeleteHead (Heap * H)
+{
+  H->HE[0] = H->HE[--H->NumItems];
+  Heap_Heapify (H, 1);
+}
+
+int 
+Heap_Lesser (HeapEntry * a, HeapEntry * b)
+{
+  return (a->Weight > b->Weight ? -1 :
+	  (a->Weight == b->Weight ? 0 : 1));
+}
+
+
+int 
+Heap_Greater (HeapEntry * a, HeapEntry * b)
+{
+  return (a->Weight > b->Weight ? 1 :
+	  (a->Weight == b->Weight ? 0 : -1));
+}
+
+int 
+Make_Exact_Root (query_data * qd, Heap * H)
+{
+  int num = 0;
+  HeapEntry *he = H->HE;
+  while (he->SeekPos == 0)
+    {
+      he->Weight = he->Weight *
+	GetLowerApproxDocWeight (qd->awd, he->docNum() - 1) /
+	he->FetchStart (qd);
+      Heap_Heapify (H, 1);
+      num++;
+    }
+  return num;
+}
+
+void 
+Heap_Dump (Heap * H, int num)
+{
+  int i, l, lines, r;
+  if (num > H->NumItems)
+    num = H->NumItems;
+  lines = (num + 3) / 4;
+  for (l = 0; l < lines; l++)
+    for (r = 0; r < 4; r++)
+      {
+	i = lines * r + l;
+	if (i < num)
+	  fprintf (stderr, "[%2d] %7.4f", i, H->HE[i].Weight);
+	fprintf (stderr, r == 3 ? "\n" : "        ");
+      }
+  fprintf (stderr, "\n");
+}
+
+int Heap_chainWeight(Heap *H, HeapEntry *he, query_data *qd, int DocNum, float *SumPtr)
+{
+  *SumPtr = (*SumPtr) / GetLowerApproxDocWeight (qd->awd, DocNum - 1);
+  qd->num_of_accum ++;
+  if (*SumPtr <= he->docWeight())
+    return 0;
+
+  he->chain(DocNum, SumPtr);
+
+  Heap_Heapify (H, 1);
+  return 1;
+}
+
+int Heap_plainWeight(Heap *H, HeapEntry *he, query_data *qd, int DocNum, float *SumPtr)
+{ 
+  unsigned long SeekPos, Len;
+  float Weight;
+
+  if (*SumPtr == 0)
+    return 0;
+  if (*SumPtr <= he->docWeight())
+    return 0;
+
+  Weight = (*SumPtr) *
+	     GetLowerApproxDocWeight (qd->awd, DocNum-1) /
+	     FetchDocStart (qd, DocNum, &SeekPos, &Len);
+
+  if (Weight <= he->docWeight())
+    return 0;
+
+  he->setValues(DocNum, Weight, SeekPos, Len, SumPtr);
+  Heap_Heapify (H, 1);
+
+  return 1;
+}
+
+/*************************************************************************/
+
+
+
+
+
+
+int 
+doc_count_comp (const void *A, const void *B)
+{
+  const TermEntry *a = (TermEntry *) A;
+  const TermEntry *b = (TermEntry *) B;
+  return (a->WE.doc_count - b->WE.doc_count);
+}
+
+/* =========================================================================
+ * Function: ParseRankedQuery
+ * Description:
+ *      Takes a string query line and extracts the terms in the query
+ *      which exist in the stemmed dictionary.
+ *      Optionally sorts the terms into ascending order by doc. number.
+ * Input:
+ *      stemmed dictionary, query line, sort-flag
+ * Output:
+ *      A list of terms.
+ * ========================================================================= */
+
+/* [RPAP - Jan 97: Stem Index Change] */
+static TermList *
+ParseRankedQuery (query_data *qd, char *QueryLine, 
+		  RankedQueryInfo *rqi,
+		  QueryTermList **query_term_list)  /* [RPAP - Feb 97: Term Frequency] */
+{
+  u_char Word[MAXSTEMLEN + 1];
+  u_char sWord[MAXSTEMLEN + 1];
+  u_char *end, *s_in;
+  int default_stem_method = 0;
+  TermList *Terms = TermList_create(0);
+
+  s_in = (u_char *) QueryLine;
+  end = s_in + strlen ((char *) s_in) - 1;
+  *query_term_list = QueryTermList_create(0);  /* [RPAP - Feb 97: Term Frequency] */
+
+  /* [RPAP - Jan 97: Stem Index Change] */
+  if (qd->sd->sdh.indexed)
+    default_stem_method = rqi->getWordSpecial();
+      //BooleanEnv (GetEnv ("casefold"), 0) | (BooleanEnv (GetEnv ("stem"), 0) << 1);
+  else
+    default_stem_method = qd->sd->sdh.stem_method;
+
+  while (s_in <= end)
+    {
+      int j;
+      long num_entries, word_num;
+      unsigned long count, doc_count, invf_ptr, invf_len;
+      int weight_to_apply, stem_to_apply;
+      int method_using = -1;
+
+      /* 0=optional, 1=mustmatch */
+      int require_match = 0; /* [RJM 07/97: Ranked Required Terms] */
+
+      /* Skip over the non word separator taking note of any parameters */
+      PARSE_RANKED_NON_STEM_WORD (require_match, s_in, end); /* [RJM 07/97: Ranked Required Terms] */
+      if (s_in > end) break;
+
+      /* Get a word and stem it */
+      PARSE_STEM_WORD (Word, s_in, end);
+
+      /* Extract any parameters */
+      weight_to_apply = 1;
+      stem_to_apply = default_stem_method;
+      while (s_in <= end)
+	{
+	  int stem_param, weight_param, param_type;
+	  char param[MAXPARAMLEN + 1];
+
+	  param_type = 0;
+	  PARSE_OPT_TERM_PARAM (param, param_type, s_in, end);
+	  if (!param_type)
+	    break;
+
+	  switch (param_type)
+	    {
+	    case (WEIGHTPARAM):
+	      weight_param = atoi (param);
+	      if (errno != ERANGE && weight_param > 0)
+		weight_to_apply = weight_param;
+	      break;
+
+	    case (STEMPARAM):
+	      stem_param = atoi (param);
+	      if (errno != ERANGE && qd->sd->sdh.indexed && stem_param >= 0 && stem_param <= 3)
+		method_using = stem_to_apply = stem_param;
+	      break;
+	    }
+	}
+
+      bcopy ((void *) Word, (void *) sWord, *Word + 1);
+      stemmer (stem_to_apply, qd->sd->sdh.stemmer_num, sWord);
+
+      if (!qd->sd->sdh.indexed || stem_to_apply == 0)
+	{
+	  /* Look for the word in the already identified terms */
+	  for (j = 0; j < Terms->size(); j++)
+	    if (compare (Terms->TE[j].Word, Word) == 0)
+	      break;
+
+	  /* Increment the weight if the word is in the list */
+	  /* Update the require match attribute */
+	  if (j < Terms->size())
+	    {
+	      Terms->TE[j].Count = ((Terms->TE[j].Count + weight_to_apply > INT_MAX) ?
+				    INT_MAX : (Terms->TE[j].Count + weight_to_apply));
+	      Terms->TE[j].require_match = require_match; /* [RJM 07/97: Ranked Require match] */
+	      QueryTermList_AddQueryTerm (query_term_list, Word, Terms->TE[j].WE.count, method_using);  /* [RPAP - Feb 97: Term Frequency] */
+	    }
+	  else
+	    {
+	      /* Look for it in the stemmed dictionary */
+	      if ((word_num = FindWord (qd->sd, sWord, &count, &doc_count,
+					&invf_ptr, &invf_len)) != -1)
+		{
+		  /* Search the list for the word */
+		  for (j = 0; j < Terms->size(); j++)
+		    if (Terms->TE[j].WE.word_num == word_num)
+		      break;
+
+		  /* Increment the weight if the word is in the list */
+		  if (j < Terms->size())
+		    {
+		      Terms->TE[j].Count = ((Terms->TE[j].Count + weight_to_apply > INT_MAX) ?
+					    INT_MAX : (Terms->TE[j].Count + weight_to_apply));
+		      Terms->TE[j].require_match = require_match; /* [RJM 07/97: Ranked Require match] */
+		      QueryTermList_AddQueryTerm (query_term_list, Word, Terms->TE[j].WE.count, method_using);  /* [RPAP - Feb 97: Term Frequency] */
+		    }
+		  else
+		    {
+		      /* Create a new entry in the list for the new word */
+		      TermEntry te;
+
+		      te.WE.word_num = word_num;
+		      te.WE.count = count;
+		      te.WE.doc_count = doc_count;
+		      te.WE.max_doc_count = doc_count;
+		      te.WE.invf_ptr = invf_ptr;
+		      te.WE.invf_len = invf_len;
+		      te.Count = weight_to_apply;
+		      te.Word = copy_string (Word);
+		      if (!te.Word)
+			FatalError (1, "Could NOT create memory to add term");
+		      te.Stem = NULL;
+		      te.require_match = require_match;
+
+		      TermList_AddTermEntry (Terms, &te);
+
+		      /* [RPAP - Feb 97: Term Frequency] */
+		      QueryTermList_AddQueryTerm (query_term_list, Word, count, method_using);
+		    }
+		}
+	      /* [RPAP - Feb 97: Term Frequency] */
+	      else
+		QueryTermList_AddQueryTerm (query_term_list, Word, 0, method_using);
+	    }
+	}
+      else
+	{
+	  int total_count = 0;  /* [RPAP - Feb 97: Term Frequency] */
+	  TermList *tempList = TermList_create (0);
+
+	  if ((num_entries = FindWords (qd->sd, sWord, stem_to_apply, &tempList)) > 0)
+	    {
+	      int i;
+	      u_long max_doc_count = 0;
+
+	      /* get the maximum doc count */
+	      for (i = 0; i < tempList->size(); i++)
+		{
+		  if (tempList->TE[i].WE.doc_count > max_doc_count)
+		    max_doc_count = tempList->TE[i].WE.doc_count;
+		  total_count += tempList->TE[i].WE.count;  /* [RPAP - Feb 97: Term Frequency] */
+		}
+
+	      for (i = 0; i < tempList->size(); i++)
+		{
+		  /* Look for the word(s) in the already identified terms */
+		  for (j = 0; j < Terms->size(); j++) 
+		    {
+		      if (compare (Terms->TE[j].Word, tempList->TE[i].Word) == 0)
+			{
+			  /* found the word */
+			  /* Modify weight */
+			  Terms->TE[j].Count = ((Terms->TE[j].Count + weight_to_apply > INT_MAX) ?
+						INT_MAX : (Terms->TE[j].Count + weight_to_apply));
+			  if (Terms->TE[j].WE.max_doc_count < max_doc_count)
+			    Terms->TE[j].WE.max_doc_count = max_doc_count;
+			  break;
+			}
+		    }
+
+		  if (j == Terms->size())
+		    {
+		      /* word was not found */
+		      tempList->TE[i].WE.max_doc_count = max_doc_count;
+		      tempList->TE[i].Count = weight_to_apply;
+
+		      /* We cannot require a term to match if it is expanded */
+		      /* into multiple terms :-( */
+		      tempList->TE[i].require_match = 0; /* [RJM 07/97: Ranked Required Terms] */
+
+		      TermList_AddTermEntry (Terms, &tempList->TE[i]);
+		    }
+		}
+	    }
+	  
+	  /* [RPAP - Feb 97: Term Frequency] */
+	  QueryTermList_AddQueryTerm (query_term_list, Word, total_count, method_using);
+
+	  if (tempList != NULL) Xfree(tempList); /* [RJM 07/98: Memory Leak] */
+	} /* end indexed */
+    } /* end while */
+
+  if (rqi->Sort)
+    /* Sort the terms in ascending order by doc_count */
+    qsort (Terms->TE, Terms->size(), sizeof (TermEntry), doc_count_comp);
+  return (Terms);
+}
+
+
+/*
+ * This function is given a list of term numbers and it returns a list
+ * of document numbers based on the cosine document weighting system.
+ * This puts the entries in an array.
+ * inverted file.
+ * If MaxDocs == -1 then it means all 
+ */
+static DocList *
+CosineGet (query_data * qd, TermList * Terms, RankedQueryInfo * rqi) {
+  DocList *Docs;
+  float *AccumulatedWeights = NULL;
+  Splay_Tree *ST = NULL;
+  Splay_Tree *Para_ST = NULL;
+  Hash_Table *HT = NULL;
+  List_Table *LT = NULL;
+  Heap *H;
+  HeapEntry *he;
+  register float *fptr = NULL;
+  register Invf_Doc_Entry *ide = NULL;
+  register Invf_Doc_EntryH *ideh = NULL;
+  int BackEnd, NumExact, MaxExact, NumParas;
+  int MaxDocs = 0, MaxParas = 0;
+  int i;
+  Invf_Doc_Entry_Pool ide_pool;
+  ide_pool.pool = NULL;
+
+  qd->hops_taken = qd->num_of_ptrs = qd->num_of_accum = 0;
+
+  switch (rqi->AccumMethod)
+    {
+    case 'S':
+      ST = CosineDecodeSplay (qd, Terms, rqi, &ide_pool);
+      if (!ST)
+	return NULL;
+      break;
+    case 'A':
+      AccumulatedWeights = CosineDecode (qd, Terms, rqi);
+      if (!AccumulatedWeights)
+	return NULL;
+      break;
+    case 'H':
+      HT = CosineDecodeHash (qd, Terms, rqi);
+      if (!HT)
+	return NULL;
+      break;
+    case 'L':
+      LT = CosineDecodeList (qd, Terms, rqi);
+      if (!LT)
+	return NULL;
+      break;
+    }
+
+#if 0
+  if (rqi->UseSplayTree)
+    {
+
+      AccumulatedWeights = CosineDecode (qd, Terms, rqi);
+      fptr = AccumulatedWeights;
+      ide = SP_get_first (ST);
+      for (i = 0; i < qd->sd->sdh.num_of_docs; i++)
+	{
+	  if (AccumulatedWeights[i] != 0)
+	    {
+	      if (i != ide->DocNum)
+		fprintf (stderr, "Sum mismatch for %d %f %d %f\n", i + 1,
+			 AccumulatedWeights[i], ide->docNum() + 1, ide->docWeight());
+	      ide = SP_get_next (ST);
+	    }
+	}
+    }
+#endif
+
+  switch (rqi->AccumMethod)
+    {
+    case 'S':
+      MaxParas = ST->no_of_items;
+      break;
+    case 'A':
+      {				/* count the number of non-zero document weights */
+	register int i = qd->sd->sdh.num_of_docs;
+	register float *d;
+	MaxParas = 0;
+	for (d = AccumulatedWeights; i; i--, d++)
+	  {
+	    if (*d)
+	      {
+		MaxParas++;
+	      }
+	  }
+      }
+      break;
+    case 'H':
+      MaxParas = HT->num + HT->Suplimentary_Num;
+      break;
+    case 'L':
+      MaxParas = LT->num;
+      break;
+    }
+
+  if (rqi->MaxParasToRetrieve != -1 && MaxParas > rqi->MaxParasToRetrieve)
+    MaxParas = rqi->MaxParasToRetrieve;
+  MaxDocs = MaxParas;
+
+  /* Allocate memory for the heap */
+  Docs = DocList_create (MaxDocs);
+  ChangeMemInUse (qd, sizeof (DocEntry) * MaxDocs);
+
+  H = Heap_Make (MaxDocs, Heap_Lesser);
+
+
+  /* Get the sums from the array divide the sums by the 
+     document weights which we retrieve from the ".idx.wgt" file and put
+     the resulting data into a heap */
+
+
+  he = H->HE;
+  H->NumItems = MaxDocs;
+  switch (rqi->AccumMethod)
+    {
+    case 'S':
+      {
+	ide = (Invf_Doc_Entry *) SP_get_first (ST);
+	for (i = 0; i < H->NumItems; i++, ide = (Invf_Doc_Entry *) SP_get_next (ST), he++)
+	  {
+	    he->DocNum = ide->docNum() + 1;
+	    he->OrgWeight = &ide->Weight;
+	    qd->num_of_accum++;
+	  }
+      }
+      break;
+    case 'A':
+      {
+	fptr = AccumulatedWeights;
+	for (i = 0; i < H->NumItems; i++, fptr++, he++)
+	  {
+	    he->DocNum = i + 1;
+	    he->OrgWeight = fptr;
+	    if (*fptr)
+	      qd->num_of_accum++;
+	  }
+      }
+      break;
+    case 'H':
+      {
+	ideh = HT->Head;
+	for (i = 0; i < H->NumItems; i++, ideh = ideh->next, he++)
+	  {
+	    he->DocNum = ideh->DocNum + 1;
+	    he->OrgWeight = &ideh->Weight;
+	    qd->num_of_accum++;
+	  }
+      }
+      break;
+    case 'L':
+      {
+	ide = LT->IDE;
+	for (i = 0; i < H->NumItems; i++, ide++, he++)
+	  {
+	    he->DocNum = ide->DocNum + 1;
+	    he->OrgWeight = &ide->Weight;
+	    qd->num_of_accum++;
+	  }
+      }
+      break;
+    }
+
+  he = H->HE;
+  for (i = 0; i < H->NumItems; i++, he++)
+    {
+      *he->OrgWeight /= GetLowerApproxDocWeight (qd->awd, he->DocNum - 1);
+      he->Weight = *he->OrgWeight;
+      *he->OrgWeight = 0;
+      he->SeekPos = he->Len = 0;
+    }
+
+  Heap_Build (H);
+
+  he = H->HE;
+  switch (rqi->AccumMethod)
+    {
+    case 'S':
+      {
+	for (i = MaxDocs; i < ST->no_of_items; i++, ide = (Invf_Doc_Entry *) SP_get_next (ST))
+	  {
+	    /*
+	    ide->Sum /= GetLowerApproxDocWeight (qd->awd, ide->DocNum);
+	    qd->num_of_accum++;
+	    if (ide->Sum <= he->Weight)
+	      continue;
+	    */
+	    Heap_chainWeight(H, he, qd, ide->DocNum + 1, &ide->Weight);
+	    /*
+	    *he->OrgWeight = he->Weight;
+	    he->DocNum = ide->DocNum + 1;
+	    he->Weight = ide->Sum;
+	    he->OrgWeight = &ide->Sum;
+	    *he->OrgWeight = 0;
+	    Heap_Heapify (H, 1);
+	    */
+	  }
+      }
+      break;
+    case 'A':
+      {
+	for (i = MaxDocs; i < qd->sd->sdh.num_of_docs; i++, fptr++)
+	  {
+	    if (!*fptr)
+	      continue;
+	    /*
+	    *fptr /= GetLowerApproxDocWeight (qd->awd, i);
+	    qd->num_of_accum++;
+	    if (*fptr <= he->Weight)
+	      continue;
+	    */
+	    Heap_chainWeight(H, he, qd, i+1, fptr);
+	    /*
+	    *he->OrgWeight = he->Weight;
+	    he->DocNum = i + 1;
+	    he->Weight = *fptr;
+	    he->OrgWeight = fptr;
+	    *he->OrgWeight = 0;
+	    Heap_Heapify (H, 1);
+	    */
+	  }
+      }
+      break;
+    case 'H':
+      {
+	for (; ideh != NULL; ideh = ideh->next)
+	  {
+	    /*
+	    ideh->Sum /=
+	      GetLowerApproxDocWeight (qd->awd, ideh->DocNum);
+	    qd->num_of_accum++;
+	    if (ideh->Sum <= he->Weight)
+	      continue;
+	    */
+	    Heap_chainWeight(H, he, qd, ideh->DocNum + 1, &ideh->Weight);
+	    /*
+	    *he->OrgWeight = he->Weight;
+	    he->DocNum = ideh->DocNum + 1;
+	    he->Weight = ideh->Sum;
+	    he->OrgWeight = &ideh->Sum;
+	    *he->OrgWeight = 0;
+	    Heap_Heapify (H, 1);
+	    */
+	  }
+      }
+      break;
+    case 'L':
+      {
+	for (i = MaxDocs; i < LT->num; i++, ide++)
+	  {
+	    /*
+	    ide->Sum /=
+	      GetLowerApproxDocWeight (qd->awd, ide->DocNum);
+	    qd->num_of_accum++;
+	    if (ide->Sum <= he->Weight)
+	      continue;
+	    */
+	    Heap_chainWeight(H, he, qd, ide->DocNum + 1, &ide->Weight);
+	    /*
+	    *he->OrgWeight = he->Weight;
+	    he->DocNum = ide->DocNum + 1;
+	    he->Weight = ide->Sum;
+	    he->OrgWeight = &ide->Sum;
+	    *he->OrgWeight = 0;
+	    Heap_Heapify (H, 1);
+	    */
+	  }
+      }
+      break;
+    }
+
+
+  if (rqi->Exact && qd->id->ifh.InvfLevel != 3)
+    {
+      HeapEntry *he = H->HE;
+
+      for (i = 0; i < H->NumItems; i++, he++)
+	{
+	  he->Weight = he->docWeight() *
+	    GetLowerApproxDocWeight (qd->awd, he->docNum() - 1) /
+	    he->FetchStart (qd);
+	}
+
+      Heap_Build (H);
+
+      he = H->HE;
+
+      switch (rqi->AccumMethod)
+	{
+	case 'S':
+	  {
+	    ide = (Invf_Doc_Entry *) SP_get_first (ST);
+	    for (i = 0; i < ST->no_of_items; i++, ide = (Invf_Doc_Entry *) SP_get_next (ST))
+	      {
+		/*
+		u_long SeekPos, Len;
+		float Weight;
+		if (!ide->Sum)
+		  continue;
+		if (ide->Sum <= he->Weight)
+		  continue;
+		Weight = ide->Sum *
+		  GetLowerApproxDocWeight (qd->awd, ide->DocNum) /
+		  FetchDocStart (qd, ide->DocNum + 1, &SeekPos, &Len);
+		if (Weight <= he->Weight)
+		  continue;
+		*/
+		if (Heap_plainWeight(H, he, qd, ide->DocNum + 1, &ide->Weight) == 0)
+		  continue;
+		/*
+		he->DocNum = ide->DocNum + 1;
+		he->OrgWeight = &ide->Sum;
+		he->Weight = Weight;
+		he->SeekPos = SeekPos;
+		he->Len = Len;
+		ide->Sum = 0;
+		Heap_Heapify (H, 1);
+		*/
+	      }
+	  }
+	  break;
+
+	  /* up to here */
+
+	case 'A':
+	  {
+	    fptr = AccumulatedWeights;
+	    for (i = 0; i < qd->sd->sdh.num_of_docs; i++, fptr++)
+	      {
+		/*
+		u_long SeekPos, Len;
+		float Weight;
+		if (!*fptr)
+		  continue;
+		if (*fptr <= he->Weight)
+		  continue;
+		Weight = *fptr *
+		  GetLowerApproxDocWeight (qd->awd, i) /
+		  FetchDocStart (qd, i + 1, &SeekPos, &Len);
+		if (Weight <= he->Weight)
+		  continue;
+		*/
+		if (Heap_plainWeight(H, he, qd, i+1, fptr) == 0)
+		  continue;
+		/*
+		he->DocNum = i + 1;
+		he->OrgWeight = fptr;
+		he->Weight = Weight;
+		he->SeekPos = SeekPos;
+		he->Len = Len;
+		*fptr = 0;
+		Heap_Heapify (H, 1);
+		*/
+	      }
+	  }
+	  break;
+	case 'H':
+	  {
+	    ideh = HT->Head;
+	    for (ideh = HT->Head; ideh; ideh = ideh->next)
+	      {
+		/*
+		u_long SeekPos, Len;
+		float Weight;
+		if (!ideh->Sum)
+		  continue;
+		if (ideh->Sum <= he->Weight)
+		  continue;
+	      
+		Weight = ideh->Sum *
+		  GetLowerApproxDocWeight (qd->awd, ideh->DocNum) /
+		  FetchDocStart (qd, ideh->DocNum + 1, &SeekPos, &Len);
+		if (Weight <= he->Weight)
+		  continue;
+		*/
+		if (Heap_plainWeight(H, he, qd, ideh->docNum() + 1, &ideh->Weight) == 0)
+		  continue;
+		/*
+		he->DocNum = ideh->DocNum + 1;
+		he->OrgWeight = &ideh->Sum;
+		he->Weight = Weight;
+		he->SeekPos = SeekPos;
+		he->Len = Len;
+		ideh->Sum = 0;
+		Heap_Heapify (H, 1);
+		*/
+	      }
+	  }
+	  break;
+	case 'L':
+	  {
+	    ide = LT->IDE;
+	    for (i = 0; i < LT->num; i++, ide++)
+	      {
+		/*u_long SeekPos, Len;
+		float Weight;
+		if (!ide->Sum)
+		  continue;
+		if (ide->Sum <= he->Weight)
+		  continue;
+		
+		Weight = ide->Sum *
+		  GetLowerApproxDocWeight (qd->awd, ide->DocNum) /
+		  FetchDocStart (qd, ide->DocNum + 1, &SeekPos, &Len);
+		if (Weight <= he->Weight)
+		  continue;
+		*/
+		if (Heap_plainWeight(H, he, qd, ide->DocNum+1, &ide->Weight) == 0)
+		  continue;
+		/*
+		he->DocNum = ide->DocNum + 1;
+		he->OrgWeight = &ide->Sum;
+		he->Weight = Weight;
+		he->SeekPos = SeekPos;
+		he->Len = Len;
+		ide->Sum = 0;
+		Heap_Heapify (H, 1);
+		*/
+	      }
+	  }
+	  break;
+	}
+    }
+
+
+
+  H->HC = Heap_Greater;
+  Heap_Build (H);
+
+
+  MaxDocs = H->NumItems;
+  if (rqi->MaxDocsToRetrieve != -1 && MaxDocs > rqi->MaxDocsToRetrieve)
+    MaxDocs = rqi->MaxDocsToRetrieve;
+
+  /* Alarm */
+
+  he = H->HE;
+  BackEnd = H->NumItems - 1;
+  NumExact = 0;
+  MaxExact = H->NumItems;
+  NumParas = 0;
+  Para_ST = SP_createset (DocEntry_compare);
+  while (H->NumItems && Docs->size() < MaxDocs)
+    {
+      DocEntry DocEnt;
+      DocEntry *mem;
+
+      if (rqi->Exact)
+	{
+	  if (H->HE[0].SeekPos == 0)
+	    NumExact += Make_Exact_Root (qd, H);
+	}
+      else
+	he->FetchStart (qd);
+
+      NumParas++;
+
+      DocEnt.DocNum = he->DocNum;
+      DocEnt.Weight = he->Weight;
+      DocEnt.Len = he->Len;
+      DocEnt.SeekPos = he->SeekPos;
+      DocEnt.CompTextBuffer = NULL;
+      DocEnt.Next = NULL;
+
+      Heap_DeleteHead (H);
+
+      if (!(mem = (DocEntry *) SP_member (&DocEnt, Para_ST)))
+	{
+	  Docs->addMember(&DocEnt);
+	  SP_insert (Docs->lastMember(), Para_ST);
+	}
+      else
+	{
+	  DocEnt.Next = DocEntry_getNext(mem);
+	  Docs->setMember(BackEnd, &DocEnt);
+	  DocEntry_setNext(mem, Docs->member(BackEnd--));
+	}
+    }
+  SP_freeset (Para_ST);
+
+  if (qd->id->ifh.InvfLevel == 3)
+    {
+      Message ("%d Paragraphs were required to get %d documents",
+	       NumParas, Docs->size());
+      if (NumExact == MaxExact)
+	{
+	  Message ("The exact weights of all %d paragraphs had to be calculated", MaxExact);
+	  Message ("to obtain %d paragraphs. This may mean that the the documents", NumParas);
+	  Message ("returned do not necessarly represent an exact cosine ranking.");
+	  Message ("This problem may be corrected by increasing \'maxparas\'.");
+	}
+    }
+#if 0
+  {
+    int i;
+    FILE *f = fopen ("top.paras", "w");
+    fprintf (f, "=========================\nTop Paragraphs\n");
+    for (i = 0; i < Docs->size(); i++)
+      {
+	DocEntry *d;
+	fprintf (f, "<%d(%f)>  ", Heap[i].DocNum, Heap[i].Weight);
+	for (d = Heap[i].Next; d; d = d->Next)
+	  fprintf (f, "%d(%f)  ", d->docNum(), d->docWeight());
+	fprintf (f, "\n");
+      }
+    fprintf (f, "=========================\n");
+    fclose (f);
+  }
+#endif
+
+  if (AccumulatedWeights)
+    {
+      Xfree (AccumulatedWeights);
+      ChangeMemInUse (qd, -sizeof (float) * qd->sd->sdh.num_of_docs);
+    }
+  if (ST)
+    {
+      int mem = ST->mem_in_use;
+      SP_freeset (ST);
+      ChangeMemInUse (qd, -mem);
+      free_ide_pool (qd, &ide_pool);
+    }
+  if (HT)
+    HT_free (qd, HT);
+
+  if (LT)
+    LT_free (qd, LT);
+
+  if (H) Xfree (H); /* [RJM 07/98: Memory Leak] */
+
+  return (Docs);
+}
+
+
+
+
+/* if MaxDocs == -1 it means all */
+void 
+RankedQuery (query_data *qd, char *Query, RankedQueryInfo *rqi)
+{
+  DocList *dl;
+
+  if (qd->TL)
+    TermList_destroy (&(qd->TL));
+
+  /* [RPAP - Feb 97: Term Frequency] */
+  if (qd->QTL)
+    QueryTermList_free (&(qd->QTL));
+
+  qd->TL = ParseRankedQuery (qd, Query, rqi, &(qd->QTL));  /* [RPAP - Feb 97: Term Frequency] */
+
+  /*  TermList_print (qd->TL, stderr); */
+
+  dl = CosineGet (qd, qd->TL, rqi);
+
+  if (!dl)
+    FatalError (1, "Out of memory\n");
+
+  QueryData_FreeQueryDocs (qd);
+
+  qd->DL = dl;
+}
+
+
+
Index: /tags/gsdl-21/gsdl/src/mgpp/text/query_term_list.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/query_term_list.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/query_term_list.cpp	(revision 699)
@@ -0,0 +1,228 @@
+/**************************************************************************
+ *
+ * filename -- description
+ * Copyright (C) 1994  Authors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/10/11 02:58:15  cs025
+   Base install of MG-PP
+
+   Revision 1.1  1999/08/10 21:18:21  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:35:36  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/10/20  03:57:07  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+ */
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "local_strings.h"
+#include "query_term_list.h"
+#include "term_lists.h"  /* for MAXTERMSTRLEN */
+#include "messages.h"
+
+
+/* =========================================================================
+ * Function: QueryTermList_create
+ * Description:
+ * Input:
+ * Output:
+ * ========================================================================= */
+QueryTermList *
+QueryTermList_create (int n)
+{
+  QueryTermList *t;
+  int list_size = (n == 0 ? 1 : n);	/* always allocate at least one node */
+
+  t = (QueryTermList *) Xmalloc (sizeof (QueryTermList) + (list_size - 1) * sizeof (QueryTermEntry));
+  if (!t)
+    FatalError (1, "Unable to allocate query term list");
+
+  t->num = n;
+  t->list_size = list_size;
+
+  return t;
+}
+
+/* =========================================================================
+ * Function: ResizeQueryTermList
+ * Description:
+ * Input:
+ * Output:
+ * ========================================================================= */
+
+#define GROWTH_FACTOR 2
+#define MIN_SIZE 2
+
+static void
+QueryTermList_resize (QueryTermList ** query_list)
+{
+  QueryTermList *qtl = *query_list;
+
+  if (qtl->num > qtl->list_size)
+    {
+      if (qtl->list_size)
+	qtl->list_size *= GROWTH_FACTOR;
+      else
+	qtl->list_size = MIN_SIZE;
+    }
+  qtl = (QueryTermList *) Xrealloc (qtl, sizeof (QueryTermList) + (qtl->list_size - 1) * sizeof (QueryTermEntry));
+
+  if (!qtl)
+    FatalError (1, "Unable to resize query term list");
+
+  *query_list = qtl;
+}
+
+/* =========================================================================
+ * Function: ConvertQueryTermsToString
+ * Description:
+ *      Convert term list into null-terminated string
+ * Input:
+ *      query_term_list = query list
+ * Output:
+ *      str = term string
+ * ========================================================================= */
+
+void
+QueryTermList_toString (QueryTermList * query_term_list, char *str)
+{
+  int i = 0;
+  int total_len = 0;
+
+  /* terms_str should be preallocated */
+  if (!str)
+    return;
+
+  for (i = 0; i < query_term_list->num; i++)
+    {
+      unsigned char *word = query_term_list->QTE[i].Term;
+      int len = word[0];
+      total_len += len + 1;	/* +1 for space */
+      if (query_term_list->QTE[i].stem_method >= 0)
+	total_len += 2;         /* '#' and the stem number */
+      if (total_len > MAXTERMSTRLEN)
+	break;
+      strncpy (str, (char *) word + 1, len);
+      str += len;
+      if (query_term_list->QTE[i].stem_method >= 0)
+	{
+	  *str++ = '#';
+	  *str++ = '0' + query_term_list->QTE[i].stem_method;
+	}
+      if (i != (query_term_list->num) - 1)
+	{
+	  *str = ' ';
+	  str++;		/* add space gap */
+	}
+
+    }
+  *str = '\0';
+}
+
+/* =========================================================================
+ * Function: QueryTermList_reset
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+void
+QueryTermList_reset (QueryTermList ** qtl)
+{
+  if (*qtl)
+    QueryTermList_free (qtl);
+  *qtl = QueryTermList_create (0);
+}
+
+/* =========================================================================
+ * Function: AddQueryTermEntry
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+int
+QueryTermList_AddQueryTermEntry (QueryTermList ** query_term_list, QueryTermEntry * qte)
+{
+  QueryTermList *qtl = *query_term_list;
+
+  qtl->num++;
+  QueryTermList_resize (query_term_list);
+  qtl = *query_term_list;
+
+  /* copy the structure contents */
+  bcopy ((void *) qte, (void *) &(qtl->QTE[qtl->num - 1]), sizeof (QueryTermEntry));
+
+  return qtl->num - 1;
+}
+
+
+
+/* =========================================================================
+ * Function: AddQueryTerm
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+int
+QueryTermList_AddQueryTerm (QueryTermList ** query_term_list, u_char * Term, int Count, int stem_method)
+{
+  /* Create a new entry in the list for the new word */
+  QueryTermEntry qte;
+
+  qte.Count = Count;
+  qte.stem_method = stem_method;
+  qte.Term = copy_string (Term);
+  if (!qte.Term)
+    FatalError (1, "Could NOT create memory to add term");
+
+  return QueryTermList_AddQueryTermEntry (query_term_list, &qte);
+}
+
+/* =========================================================================
+ * Function: QueryTermList_free
+ * Description:
+ * Input:
+ * Output:
+ * ========================================================================= */
+
+void
+QueryTermList_free (QueryTermList ** the_qtl)
+{
+  int j;
+  QueryTermList *qtl = *the_qtl;
+
+  for (j = 0; j < qtl->num; j++)
+    if (qtl->QTE[j].Term)
+      Xfree (qtl->QTE[j].Term);
+  Xfree (qtl);
+
+  *the_qtl = NULL;
+}
Index: /tags/gsdl-21/gsdl/src/mgpp/text/query_term_list.h
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/query_term_list.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/query_term_list.h	(revision 699)
@@ -0,0 +1,78 @@
+/**************************************************************************
+ *
+ * term_lists.h -- description
+ * Copyright (C) 1994  Authors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/10/11 02:58:16  cs025
+   Base install of MG-PP
+
+   Revision 1.1  1999/08/10 21:18:21  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:35:37  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/10/20  03:57:08  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+ */
+
+#ifndef QUERY_TERM_LIST_H
+#define QUERY_TERM_LIST_H
+
+#include "sysfuncs.h"
+
+typedef struct QueryTermEntry
+  {
+    int Count;			/* The number of times the word occurs in the query */
+    u_char *Term;		/* The query term */
+    int stem_method;            /* The steming method to apply to the term : -1 is default */
+  }
+QueryTermEntry;
+
+typedef struct QueryTermList
+  {
+    int list_size;
+    int num;
+    QueryTermEntry QTE[1];
+  }
+QueryTermList;
+
+/* --- prototypes --- */
+void QueryTermList_toString (QueryTermList * query_term_list, char *str);
+
+#ifdef __cplusplus
+extern "C"
+#endif
+int QueryTermList_AddQueryTerm (QueryTermList ** query_term_list, u_char * Term, int Count, int stem_method);
+
+#ifdef __cplusplus
+extern "C"
+#endif
+void QueryTermList_reset (QueryTermList ** tl);
+void QueryTermList_free (QueryTermList ** the_qtl);
+QueryTermList *QueryTermList_create (int n);
+
+
+#endif
Index: /tags/gsdl-21/gsdl/src/mgpp/text/read_line.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/read_line.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/read_line.cpp	(revision 699)
@@ -0,0 +1,224 @@
+/**************************************************************************
+ *
+ * read_line.c -- Input line reading routines for mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include <stdio.h>
+#include "non_ansi.h"
+
+#include "sysfuncs.h"
+#include "memlib.h"
+
+#ifdef WITH_GNU_READLINE
+#include "readline/readline.h"
+#include "readline/chardefs.h"
+#if 0
+#include "readline/history.h"  /* dropped from recent versions of GNU readline */
+#endif
+#endif
+
+
+#include "globals.h"
+#include "environment.h"
+#include "read_line.h"
+
+
+
+#ifdef WITH_GNU_READLINE
+int rl_bind_key (int, int (*)());
+
+
+void 
+Init_ReadLine (void)
+{
+  /* Make TAB just insert itself rather than do a file name completion */
+  rl_bind_key (TAB, rl_insert);
+}
+#else
+static FILE *rl_instream;
+static FILE *rl_outstream;
+
+void 
+Init_ReadLine (void)
+{
+  rl_instream = stdin;
+  rl_outstream = stdout;
+}
+#endif
+
+
+
+
+/* WritePrompt() */
+/* Write out a prompt if user is a TTY */
+void 
+WritePrompt (void)
+{
+  if (isatty (fileno (InFile)))
+    {
+      if (!BooleanEnv (GetEnv ("expert"), 0))
+	fprintf (stderr, "Enter a command or query (.quit to terminate, .help for assistance).\n");
+    }
+}
+
+#ifdef WITH_GNU_READLINE
+static void memory_error_and_abort ();
+
+char *
+xmalloc (bytes)
+     int bytes;
+{
+  char *temp = (char *) Xmalloc (bytes);
+
+  if (!temp)
+    memory_error_and_abort ();
+  return (temp);
+}
+
+char *
+xrealloc (pointer, bytes)
+     char *pointer;
+     int bytes;
+{
+  char *temp;
+
+  if (!pointer)
+    temp = (char *) xmalloc (bytes);
+  else
+    temp = (char *) Xrealloc (pointer, bytes);
+
+  if (!temp)
+    memory_error_and_abort ();
+
+  return (temp);
+}
+
+static void
+memory_error_and_abort ()
+{
+  fprintf (stderr, "history: Out of virtual memory!\n");
+  abort ();
+}
+#endif
+
+
+#ifndef WITH_GNU_READLINE
+
+static char *
+readline (char *pmt)
+{
+  static char buf[MAXLINEBUFFERLEN + 1];
+  char *s;
+
+  fprintf (rl_outstream, "%s", pmt);
+  s = fgets (buf, sizeof (buf), rl_instream);
+  if (s)
+    {
+      char *s1 = strrchr (s, '\n');
+      if (s1 && *(s1 + 1) == '\0')
+	*s1 = '\0';
+    }
+  return s ? (char *) Xstrdup (s) : (char *) NULL;
+}
+#endif
+
+/*
+ * This routine returns a pointer to the users entered line 
+ *
+ */
+static char *
+GetLine (char *pmt)
+{
+  static char *the_line = NULL;
+  if (the_line)
+    Xfree (the_line);
+  the_line = NULL;
+  rl_instream = InFile;
+  if (isatty (fileno (InFile)))
+    {
+      fputc ('\r', stderr);
+      if (!isatty (fileno (OutFile)))
+	rl_outstream = stderr;
+      else
+	rl_outstream = OutFile;
+      the_line = readline (pmt);
+    }
+  else
+    {
+      if (isatty (fileno (OutFile)))
+	{
+	  the_line = readline (pmt);
+	  fprintf (stderr, "%s\n", the_line ? the_line : "");
+	}
+      else
+	{
+	  the_line = readline ("");
+	  if (the_line)
+	    fprintf (stderr, "%s%s\n", pmt, the_line);
+	}
+    }
+#ifdef WITH_GNU_READLINE
+  if (the_line && *the_line)
+    add_history (the_line);
+#endif
+  return (the_line);
+}
+
+
+
+
+
+
+char *
+GetMultiLine (void)
+{
+  static char *line = NULL;
+  char *s;
+  if (line)
+    Xfree (line);
+  line = NULL;
+  if (!(s = GetLine ("> ")))
+    return (NULL);
+  if (!(line = Xstrdup (s)))
+    {
+      fprintf (stderr, "Unable to allocate memory for the line\n");
+      abort ();
+    }
+
+  while ((s = strrchr (line, '\\')) && *(s + 1) == '\0')
+    {
+      char *newline;
+      *strrchr (line, '\\') = '\0';
+      if (!(s = GetLine ("? ")))
+	return (NULL);
+      if (!(newline = (char *) Xmalloc (strlen (line) + strlen (s) + 2)))
+	{
+	  fprintf (stderr, "Unable to allocate memory for the line\n");
+	  abort ();
+	}
+      strcpy (newline, line);
+      strcat (newline, "\n");
+      strcat (newline, s);
+      Xfree (line);
+      line = newline;
+    }
+  return (line);
+}
Index: /tags/gsdl-21/gsdl/src/mgpp/text/read_line.h
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/read_line.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/read_line.h	(revision 699)
@@ -0,0 +1,34 @@
+/**************************************************************************
+ *
+ * read_line.h -- Input line reading routines for mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef READ_LINE_H
+#define READ_LINE_H
+
+#define MAXLINEBUFFERLEN  65535
+
+void Init_ReadLine (void);
+void WritePrompt (void);
+char *GetMultiLine (void);
+
+#endif
Index: /tags/gsdl-21/gsdl/src/mgpp/text/stem_search.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/stem_search.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/stem_search.cpp	(revision 699)
@@ -0,0 +1,829 @@
+/**************************************************************************
+ *
+ * stem_search.c -- Functions for searching the blocked stemmed dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "messages.h"
+#include "filestats.h"
+#include "timing.h"
+#include "local_strings.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg.h"
+#include "invf.h"
+#include "text.h"
+#include "lists.h"
+#include "backend.h"
+#include "words.h"
+#include "locallib.h"
+#include "stem_search.h"
+#include "StemBlock.h"
+#include "StemIdx.h"
+#include "mg_errors.h"
+#include "term_lists.h"
+#include "stemmer.h"
+
+
+/*
+   $Log$
+   Revision 1.1  1999/10/11 02:58:32  cs025
+   Base install of MG-PP
+
+   Revision 1.1  1999/08/10 21:18:22  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.3  1999/07/02 00:18:55  rjmcnab
+   Changed so FindWords could be used in new ways.
+
+   Revision 1.2  1998/11/25 07:55:51  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:39  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:57:04  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:42:08  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+size_t stemAbstractReader(void *ptr, size_t size, size_t nitems, void *file)
+{ return Fread(ptr, size, nitems, (File *) file);
+}
+
+char stemAbstractGetc(void *file)
+{ return Getc(((File *) file));
+}
+
+int stemAbstractSeeker(void *file, long offset, int ptrname)
+{ return Fseek((File *) file, offset, ptrname);
+}
+
+/* [RPAP - Jan 97: Stem Index Change] */
+stemmed_idx *
+ReadStemIdxBlk (File * stem_idx_file)
+{
+  unsigned long i;
+  stemmed_idx *si;
+  u_char *buffer;
+
+  si = StemIdx_header_read((void *) stem_idx_file, 1, stemAbstractReader, stemAbstractGetc);
+  if (si == NULL)
+    {
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+  /*
+  si->stem_idx_file = stem_idx_file;
+  si->MemForStemIdx = 0;
+
+  Fread (&si->sih, sizeof (si->sih), 1, stem_idx_file);
+  // [RPAP - Jan 97: Endian Ordering]
+  NTOHUL(si->sih.lookback);
+  NTOHUL(si->sih.block_size);
+  NTOHUL(si->sih.num_blocks);
+  NTOHUL(si->sih.blocks_start);
+  NTOHUL(si->sih.index_chars);
+  NTOHUL(si->sih.num_of_words);
+
+  if (!(buffer = (u_char *) Xmalloc (si->sih.index_chars)))
+    {
+      Xfree (si);
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    };
+  si->MemForStemIdx += si->sih.index_chars;
+
+  if (!(si->index = (u_char **) Xmalloc (si->sih.num_blocks * sizeof (*si->index))))
+    {
+      Xfree (si);
+      Xfree (buffer);
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    };
+  si->MemForStemIdx += si->sih.num_blocks * sizeof (*si->index);
+
+  if (!(si->pos = (long unsigned int *) Xmalloc (si->sih.num_blocks * sizeof (*si->pos))))
+    {
+      Xfree (si->index);
+      Xfree (si);
+      Xfree (buffer);
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    };
+  si->MemForStemIdx += si->sih.num_blocks * sizeof (*si->pos);
+
+  if (!(si->buffer = (u_char *) Xmalloc (si->sih.block_size * sizeof (*si->buffer))))
+    {
+      Xfree (buffer);
+      Xfree (si->index);
+      Xfree (si->buffer);
+      Xfree (si);
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    };
+  si->MemForStemIdx += si->sih.block_size * sizeof (*si->buffer);
+
+  si->active = -1;
+
+  for (i = 0; i < si->sih.num_blocks; i++)
+    {
+      register u_char len;
+      si->index[i] = buffer;
+      len = Getc (stem_idx_file);
+      *buffer++ = len;
+      Fread (buffer, sizeof (u_char), len, stem_idx_file);
+      buffer += len;
+      Fread (&si->pos[i], sizeof (*si->pos), 1, stem_idx_file);
+      NTOHUL(si->pos[i]);  // [RPAP - Jan 97: Endian Ordering]
+    }
+  */
+  mg_errno = MG_NOERROR;
+
+  /*  fprintf (stderr, "mem for stem idx = %i\n", si->MemForStemIdx); */
+
+  return si;
+}
+
+static int 
+Word_BaseIndexNo(int num_indexes, u_char *base, unsigned short *index, u_char *word)
+{
+  register int lo, hi, mid, c;
+
+  lo = 0;
+  hi = num_indexes - 1;
+  while (lo <= hi)
+    {
+      mid = (lo + hi) / 2;
+      c = casecompare (word, base + index[mid] + 1);  /* [RPAP - Jan 97: Stem Index Change] */
+      if (c < 0)
+	hi = mid - 1;
+      else if (c > 0)
+	lo = mid + 1;
+      else
+	{
+	  hi = mid;
+	  break;
+	}
+    }
+  if (hi < 0)
+    hi = 0;
+
+  return hi;
+}
+
+
+/* [RPAP - Jan 97: Stem Index Change] */
+/* word should be appropriately stemed */
+static int
+Word_IdxBlockNo (stemmed_idx * si, u_char * word)
+{
+  register int lo = 0, hi = si->sih.num_blocks - 1;
+  register int mid = 0, c = 0;
+
+  while (lo <= hi)
+    {
+      mid = (lo + hi) / 2;
+      c = casecompare (word, si->index[mid]);
+      if (c < 0)
+	hi = mid - 1;
+      else if (c > 0)
+	lo = mid + 1;
+      else
+	return mid;
+    }
+  return hi < 0 ? 0 : (c < 0 ? mid - 1 : mid);
+}
+
+
+static int 
+Word_BlockNo (stemmed_dict * sd, u_char * word)
+{
+  register int lo = 0, hi = sd->sdh.num_blocks - 1;
+  register int mid = 0, c = 0;
+  while (lo <= hi)
+    {
+      mid = (lo + hi) / 2;
+      c = casecompare (word, sd->index[mid]);  /* [RPAP - Jan 97: Stem Index Change] */
+      if (c < 0)
+	hi = mid - 1;
+      else if (c > 0)
+	lo = mid + 1;
+      else
+	return mid;
+    }
+  return hi < 0 ? 0 : (c < 0 ? mid - 1 : mid);
+}
+
+unsigned long Word_SkipToNextInvfPtr(u_char **base, unsigned long invfp)
+{ 
+  unsigned suff;
+  unsigned long next_invfp;
+
+  *base  = *base + 1;
+  suff   = **base;
+  *base  = *base + 1;
+  *base += suff + sizeof (unsigned long) * 2;
+  bcopy ((void *) *base, (void *) &next_invfp, sizeof (next_invfp));
+  NTOHUL(next_invfp);  /* [RPAP - Jan 97: Endian Ordering] */
+  return next_invfp - invfp;
+}
+
+/*
+ * This function looks up a word in the stemmed dictionary, it returns -1
+ * if the word cound not be found, and 0 if it successfully finds the word.
+ * If count is non-null the ulong it is pointing to is set to the number of 
+ * occurances of the stemmed word in the collection. i.e wcnt.
+ * If doc_count is non-null the ulong it is pointing to is set to the number
+ * of documents that the word occurs in. i.e fcnt
+ * If invf_ptr is non-null the ulong it is pointing to is set to the position
+ * of the inverted file where the entry for this word start.
+ */
+int 
+FindWord (stemmed_dict * sd, u_char * Word, unsigned long *count,
+	  unsigned long *doc_count, unsigned long *invf_ptr,
+	  unsigned long *invf_len)
+{
+  register int lo, hi, mid, c;
+  register unsigned int res;
+  int block, num_indexes;
+  unsigned long *first_word, *last_invf_len;
+  unsigned short *num_words;
+  u_char *base;
+  unsigned short *index;
+  u_char prev[MAXSTEMLEN + 1];
+
+  /**
+   * GRB : Get the index for the block of words in the stemmed dictionary;
+   *       If the block required is not the current cached (active) block,
+   *         then load that block in now. 
+   */
+
+  block = Word_BlockNo (sd, Word);
+  /* [RPAP - Jan 97: Endian Ordering] */
+  if (sd->active != sd->pos[block])
+    {
+      StemBlock_read(sd, block, &first_word, &last_invf_len, &num_words, 
+		     &index, &num_indexes);
+      /*      int i;
+
+      Fseek (sd->stem_file, sd->pos[block] + sd->sdh.blocks_start, 0);
+      Fread (sd->buffer, sd->sdh.block_size, sizeof (u_char), sd->stem_file);
+      sd->active = sd->pos[block];
+
+      // [RPAP - Jan 97: Endian Ordering] 
+      first_word = (unsigned long *) (sd->buffer);
+      NTOHUL(*first_word);
+      last_invf_len = (unsigned long *) (first_word + 1);
+      NTOHUL(*last_invf_len);
+      num_words = (unsigned short *) (last_invf_len + 1);
+      NTOHUS(*num_words);
+      index = num_words + 1;
+      num_indexes = ((*num_words - 1) / sd->sdh.lookback) + 1;
+
+      for (i = 0; i < num_indexes; i++)
+      NTOHUS(index[i]);*/
+    }
+  else
+    {
+      first_word = (unsigned long *) (sd->buffer);
+      last_invf_len = (unsigned long *) (first_word + 1);
+      num_words = (unsigned short *) (last_invf_len + 1);
+      index = num_words + 1;
+      num_indexes = ((*num_words - 1) / sd->sdh.lookback) + 1;
+    }
+  base = (u_char *) (index + num_indexes);
+
+  /* --
+     -- GRB: search current block for the given word using binary chop
+     --
+  */
+  hi = Word_BaseIndexNo(num_indexes, base, index, Word);
+  /*
+  lo = 0;
+  hi = num_indexes - 1;
+  while (lo <= hi)
+    {
+      mid = (lo + hi) / 2;
+      c = casecompare (Word, base + index[mid] + 1);  // [RPAP - Jan 97: Stem Index Change]
+      if (c < 0)
+	hi = mid - 1;
+      else if (c > 0)
+	lo = mid + 1;
+      else
+	{
+	  hi = mid;
+	  break;
+	}
+    }
+  if (hi < 0)
+    hi = 0;
+  */
+
+  res = hi * sd->sdh.lookback;
+  base += index[hi];
+
+  /** 
+   *
+   * GRB: stream over the words which come from the same stem, and 
+   *      check for matches amongst them
+   */
+  while (res < *num_words)
+    {
+      //unsigned copy, suff;
+      unsigned long invfp;
+      /*      if (res >= *num_words)
+	      return (-1);*/
+      StemBlock_ReadWordString(&base, prev);
+      /*      
+      copy = *base++;
+      suff = *base++;
+      bcopy ((void *) base, (void *) (prev + copy + 1), suff);
+      base += suff;
+      *prev = copy + suff;
+      */
+
+      c = casecompare (Word, prev);   /* [RPAP - Jan 97: Stem Index Change] */
+      if (c < 0)
+	return (-1);
+
+      if (c == 0 && doc_count)
+	{
+	  bcopy ((void *) base, (void *) doc_count, sizeof (*doc_count));
+	  NTOHUL(*doc_count);  /* [RPAP - Jan 97: Endian Ordering] */
+	}
+      base += sizeof (*doc_count);
+
+      if (c == 0 && count)
+	{
+	  bcopy ((void *) base, (void *) count, sizeof (*count));
+	  NTOHUL(*count);  /* [RPAP - Jan 97: Endian Ordering] */
+	}
+      base += sizeof (*count);
+
+      if (c == 0 && invf_ptr)
+	{
+	  bcopy ((void *) base, (void *) &invfp, sizeof (invf_ptr));
+	  NTOHUL(invfp);  /* [RPAP - Jan 97: Endian Ordering] */
+	  *invf_ptr = invfp;
+	}
+      base += sizeof (*invf_ptr);
+
+      if (c == 0)
+	{
+	  /* Calculate invf_len is necessary */
+	  // unsigned long next_invfp;
+	  if (!invf_len)
+	    return (*first_word + res);
+
+	  /* If the current word is the last word of the block then get the 
+	     length from last_invf_len */
+	  if (res == *num_words - 1)
+	    {
+	      *invf_len = *last_invf_len;
+	      return (*first_word + res);
+	    }
+
+	  /* Skip over most of the next word to get to the invf_ptr */
+	  *invf_len = Word_SkipToNextInvfPtr(&base, invfp);
+	  /*
+	  base++;
+	  suff = *base++;
+	  base += suff + sizeof (unsigned long) * 2;
+	  bcopy ((void *) base, (void *) &next_invfp, sizeof (next_invfp));
+	  NTOHUL(next_invfp);  // [RPAP - Jan 97: Endian Ordering] 
+	  *invf_len = next_invfp - invfp;
+	  */
+	  return (*first_word + res);
+	}
+      res++;
+    }
+  /* -- GRB return a negative response; word was not actually found after all */
+  return (-1);
+}
+
+
+/* [RPAP - Jan 97: Stem Index Change] */
+int 
+FindWords (stemmed_dict * sd, u_char * sWord, int stem_method, TermList ** tl)
+{
+  register unsigned int res;
+  unsigned int idx_res;
+  unsigned copy, suff;
+  int j, k;
+
+  int block, num_indexes;
+  unsigned long *first_word, *last_invf_len;
+  unsigned short *num_words;
+  u_char *base;
+  unsigned short *index;
+  u_char prev[MAXSTEMLEN + 1];
+
+  int idx_block, idx_num_indexes;
+  unsigned long *idx_first_word;
+  unsigned short *idx_num_words;
+  u_char *idx_base;
+  unsigned short *idx_index;
+  u_char idx_prev[MAXSTEMLEN + 1];
+
+  unsigned int num_entries, num_cases;
+  unsigned short blk_index, offset;
+  stemmed_idx * si = NULL;
+
+  /* handle stem_method 0 seperately */
+  if (stem_method == 0) {
+    TermEntry te;
+    if ((te.WE.word_num = FindWord (sd, sWord, &te.WE.count, &te.WE.doc_count,
+				   &te.WE.invf_ptr, &te.WE.invf_len)) != -1) {
+      te.WE.max_doc_count = te.WE.doc_count;
+
+      te.Count = 1;
+      te.Word = copy_string (sWord);
+      if (!te.Word)
+	FatalError (1, "Could NOT create memory to add term");
+      te.Stem = copy_string (sWord);
+      if (!te.Stem)
+	FatalError (1, "Could NOT create memory to add term");
+      /*      te.query_mask = NULL;*/
+      
+      TermList_AddTermEntry (*tl, &te);
+      return (*tl)->num;
+      
+    } else {
+      /* didn't match */
+      return 0;
+    }
+  }
+  
+  if (stem_method == 1)
+    si = sd->stem1;
+  else if (stem_method == 2)
+    si = sd->stem2;
+  else
+    si = sd->stem3;
+
+  /**
+   * Locate block: If this block is not the active one, then make it active.
+   */
+  idx_block = Word_IdxBlockNo (si, sWord);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  if (si->active != si->pos[idx_block])
+    {
+      StemIdx_block_readNext((void *) si->stem_idx_file, si, idx_block, &idx_first_word, &idx_num_words,
+			     &idx_index, &idx_num_indexes, stemAbstractSeeker, stemAbstractReader);
+      /*
+      Fseek (si->stem_idx_file, si->pos[idx_block] + si->sih.blocks_start, 0);
+      Fread (si->buffer, si->sih.block_size, sizeof (u_char), si->stem_idx_file);
+      si->active = si->pos[idx_block];
+
+      idx_first_word = (unsigned long *) (si->buffer);
+      NTOHUL(*idx_first_word);  // [RPAP - Jan 97: Endian Ordering]
+      idx_num_words = (unsigned short *) (idx_first_word + 1);
+      NTOHUS(*idx_num_words);  // [RPAP - Jan 97: Endian Ordering]
+      idx_index = idx_num_words + 1;
+      idx_num_indexes = ((*idx_num_words - 1) / si->sih.lookback) + 1;
+
+      // [RPAP - Jan 97: Endian Ordering]
+      for (j = 0; j < idx_num_indexes; j++)
+	NTOHUS(idx_index[j]);
+      */
+    }
+  else
+    {
+      idx_first_word = (unsigned long *) (si->buffer);
+      idx_num_words = (unsigned short *) (idx_first_word + 1);
+      idx_index = idx_num_words + 1;
+      idx_num_indexes = ((*idx_num_words - 1) / si->sih.lookback) + 1;
+    }
+  idx_base = (u_char *) (idx_index + idx_num_indexes);
+
+  /**
+   * Search in this stem index for the word using binary chop
+   */
+  {
+    /* Locate 3-in-4 block */
+    //register int lo, hi, mid, c;
+    int hi;
+
+    hi = Word_BaseIndexNo(idx_num_indexes, idx_base, idx_index, sWord);
+    /*
+    lo = 0;
+    hi = idx_num_indexes - 1;
+    while (lo <= hi)
+      {
+	mid = (lo + hi) / 2;
+	c = casecompare (sWord, idx_base + idx_index[mid] + 1);
+	if (c < 0)
+	  hi = mid - 1;
+	else if (c > 0)
+	  lo = mid + 1;
+	else
+	  {
+	    hi = mid;
+	    break;
+	  }
+      }
+    if (hi < 0)
+      hi = 0;
+    */
+    idx_res = hi * si->sih.lookback;
+    idx_base += idx_index[hi];
+  }
+
+  /* Locate word entry by streaming forwards through the index */
+  for (;;)
+    {
+      int c;
+      if (idx_res >= *idx_num_words)
+	return (-1);
+      StemIdx_ReadWordString(&idx_base, idx_prev);
+
+      /*
+      copy = *idx_base++;
+      suff = *idx_base++;
+      bcopy ((void *) idx_base, (void *) (idx_prev + copy + 1), suff);
+      idx_base += suff;
+      *idx_prev = copy + suff;
+      */
+
+      c = casecompare (sWord, idx_prev);
+      if (c < 0)
+	return (-1);
+     
+      bcopy ((void *) idx_base, (void *) &num_entries, sizeof (num_entries));
+      NTOHUI(num_entries);  /* [RPAP - Jan 97: Endian Ordering] */
+      idx_base += sizeof (num_entries);
+      
+      if (c > 0)
+	idx_base += num_entries * (sizeof (num_cases) + sizeof (block) + 
+				   sizeof (blk_index) + sizeof (offset));
+      else
+	break;
+
+      idx_res++;
+    }
+
+  for (k = 0; k < num_entries; k++)
+    {
+      //unsigned copy, suff;
+      unsigned long invfp;
+      /**
+       * GRB: Read next stem index pos, getting the details of the next word
+       *      which is an extension of this stem (a wordform of this stem)
+       */
+      StemIdx_ReadPosEntry(&idx_base, &num_cases, (unsigned int *) &block, &blk_index, &offset);
+      /*
+      bcopy ((void *) idx_base, (void *) &num_cases, sizeof (num_cases));
+      NTOHUI(num_cases);  // [RPAP - Jan 97: Endian Ordering]
+      idx_base += sizeof (num_cases);
+      bcopy ((void *) idx_base, (void *) &block, sizeof (block));
+      NTOHUI(block);  // [RPAP - Jan 97: Endian Ordering]
+      idx_base += sizeof (block);
+      bcopy ((void *) idx_base, (void *) &blk_index, sizeof (blk_index));
+      NTOHUS(blk_index);  // [RPAP - Jan 97: Endian Ordering] 
+      idx_base += sizeof (blk_index);
+      bcopy ((void *) idx_base, (void *) &offset, sizeof (offset));
+      NTOHUS(offset);  // [RPAP - Jan 97: Endian Ordering]
+      idx_base += sizeof (offset);
+      */
+      
+
+      /**
+       * GRB: Now look up the given word form, as per FindWord above.
+       *      Firstly, we need to get to the right block; we read this
+       *      above [rather than using Word_BlockIndex as in FindWord()]
+       *      So we just need to make sure that it is the active block
+       */
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      if (sd->active != sd->pos[block])
+	{
+	  StemBlock_read(sd, block, &first_word, &last_invf_len, &num_words,
+			 &index, &num_indexes);
+	  /*
+	  Fseek (sd->stem_file, sd->pos[block] + sd->sdh.blocks_start, 0);
+	  Fread (sd->buffer, sd->sdh.block_size, sizeof (u_char), sd->stem_file);
+	  sd->active = sd->pos[block];
+
+	  first_word = (unsigned long *) (sd->buffer);
+	  NTOHUL(*first_word);  // [RPAP - Jan 97: Endian Ordering] 
+	  last_invf_len = (unsigned long *) (first_word + 1);
+	  NTOHUL(*last_invf_len);  // [RPAP - Jan 97: Endian Ordering] 
+	  num_words = (unsigned short *) (last_invf_len + 1);
+	  NTOHUS(*num_words);  // [RPAP - Jan 97: Endian Ordering]
+	  index = num_words + 1;
+	  num_indexes = ((*num_words - 1) / sd->sdh.lookback) + 1;
+
+	  // [RPAP - Jan 97: Endian Ordering] 
+	  for (j = 0; j < num_indexes; j++)
+	    NTOHUS(index[j]);
+	  */
+	}
+      else
+	{
+	  first_word = (unsigned long *) (sd->buffer);
+	  last_invf_len = (unsigned long *) (first_word + 1);
+	  num_words = (unsigned short *) (last_invf_len + 1);
+	  index = num_words + 1;
+	  num_indexes = ((*num_words - 1) / sd->sdh.lookback) + 1;
+	}
+      base = (u_char *) (index + num_indexes);
+      
+      res = blk_index * sd->sdh.lookback;
+      base += index[blk_index];
+
+      /**
+       * GRB: We know how many words to offset by, so let's scramble
+       *      past them
+       */      
+      for (j = 0; j < offset; j++)
+	{
+	  StemBlock_ReadWordString(&base, prev);
+	  
+	  /*
+	  copy = *base++;
+	  suff = *base++;
+	  bcopy ((void *) base, (void *) (prev + copy + 1), suff);
+	  base += suff;
+	  *prev = copy + suff;
+	  */
+	  base += sizeof (unsigned long);   /* skip doc_count */
+	  base += sizeof (unsigned long);   /* skip count */
+	  base += sizeof (unsigned long);   /* skip invf_ptr */
+	  res++;
+	}
+
+      /**
+       * GRB: Now take this wordform case by case
+       */
+      for (j = 0; j < num_cases; j++)
+	{
+	  TermEntry te;
+	  
+	  if (res >= *num_words)
+	    return (-1);
+	  StemBlock_ReadWordString(&base, prev);
+	  /*
+	  copy = *base++;
+	  suff = *base++;
+	  bcopy ((void *) base, (void *) (prev + copy + 1), suff);
+	  base += suff;
+	  *prev = copy + suff;
+	  */
+	      
+	  te.Word = copy_string (prev);
+	  if (!te.Word)
+	    FatalError (1, "Could NOT create memory to add term");
+	  te.Stem = copy_string (prev);
+	  if (!te.Stem)
+	    FatalError (1, "Could NOT create memory to add term");
+	  stemmer (2, sd->sdh.stemmer_num, te.Stem);
+
+	  te.Count = 1;
+	  te.WE.word_num = *first_word + res;
+	  bcopy ((void *) base, (void *) &te.WE.doc_count, sizeof (te.WE.doc_count));
+	  NTOHUL(te.WE.doc_count);  /* [RPAP - Jan 97: Endian Ordering] */
+	  te.WE.max_doc_count = te.WE.doc_count;
+	  base += sizeof (te.WE.doc_count);
+	      
+	  bcopy ((void *) base, (void *) &te.WE.count, sizeof (te.WE.count));
+	  NTOHUL(te.WE.count);
+	  base += sizeof (te.WE.count);
+	      
+	  bcopy ((void *) base, (void *) &invfp, sizeof (te.WE.invf_ptr));
+	  NTOHUL(invfp);  /* [RPAP - Jan 97: Endian Ordering] */
+	  te.WE.invf_ptr = invfp;
+	  base += sizeof (te.WE.invf_ptr);
+		
+	  /* If the current word is the last word of the block the get the 
+	     length from last_invf_len */
+	  if (res == *num_words - 1)
+	    te.WE.invf_len = *last_invf_len;
+	  else
+	    {
+	      //unsigned long next_invfp;
+	      u_char *oldbase = base;
+		  
+	      /* Skip over most of the next word to get to the invf_ptr */
+	      te.WE.invf_len = Word_SkipToNextInvfPtr(&base, invfp);
+	      /*
+	      base++;
+	      suff = *base++;
+	      base += suff + sizeof (unsigned long) * 2;
+	      bcopy ((void *) base, (void *) &next_invfp, sizeof (next_invfp));
+	      NTOHUL(next_invfp);  // [RPAP - Jan 97: Endian Ordering]
+	      te.WE.invf_len = next_invfp - invfp;
+	      */
+	      base = oldbase;
+	    }
+	      
+	  /* Add term entry to term list */
+	  /*	  te.query_mask = NULL;*/
+	  TermList_AddTermEntry (*tl, &te);
+
+          /**
+           * GRB: If this is the final case of this wordform in this block, 
+           *      and we have one or more remaining blocks containing the wordform, 
+           *      then read in next block
+           */ 	      
+	  if (res == *num_words - 1 && j + 1 < num_cases)
+	    {
+	      //int ii;
+	      /* Read in next block */
+	      block++;
+	      StemBlock_read(sd, block, &first_word, &last_invf_len, &num_words, 
+			     &index, &num_indexes);
+	      /*
+	      Fseek (sd->stem_file, sd->pos[block] + sd->sdh.blocks_start, 0);
+	      Fread (sd->buffer, sd->sdh.block_size, sizeof (u_char), sd->stem_file);
+	      sd->active = sd->pos[block];
+
+	      first_word = (unsigned long *) (sd->buffer);
+	      NTOHUL(*first_word);  // [RPAP - Jan 97: Endian Ordering] 
+	      last_invf_len = (unsigned long *) (first_word + 1);
+	      NTOHUL(*last_invf_len);  // [RPAP - Jan 97: Endian Ordering] 
+	      num_words = (unsigned short *) (last_invf_len + 1);
+	      NTOHUS(*num_words);  // [RPAP - Jan 97: Endian Ordering]
+	      index = num_words + 1;
+	      num_indexes = ((*num_words - 1) / sd->sdh.lookback) + 1;
+
+	      // [RPAP - Jan 97: Endian Ordering]
+	      for (ii = 0; ii < num_indexes; ii++)
+		NTOHUS(index[ii]);
+	      */
+	      base = (u_char *) (index + num_indexes);
+	      base += index[0];
+	      res = 0;
+	      blk_index = 0;
+	    }
+	  else
+	    res++;
+	} /* end for num_cases */
+    } /* end for num_entries */
+  return (*tl)->num;
+}
+
+
+void 
+FreeStemDict (stemmed_dict * sd)
+{
+  /* [RPAP - Jan 97: Stem Index Change] */
+  if (sd->stem1)
+    FreeStemIdx (sd->stem1);
+  if (sd->stem2)
+    FreeStemIdx (sd->stem2);
+  if (sd->stem3)
+    FreeStemIdx (sd->stem3);
+
+  Xfree (sd->index[0]);
+  Xfree (sd->index);
+  Xfree (sd->buffer);
+  Xfree (sd->pos);
+  Xfree (sd);
+}
+
+/* [RPAP - Jan 97: Stem Index Change] */
+void
+FreeStemIdx (stemmed_idx * si)
+{
+  Xfree (si->index[0]);
+  Xfree (si->index);
+  Xfree (si->buffer);
+  Xfree (si->pos);
+  Xfree (si);
+}
Index: /tags/gsdl-21/gsdl/src/mgpp/text/stem_search.h
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/stem_search.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/stem_search.h	(revision 699)
@@ -0,0 +1,49 @@
+/**************************************************************************
+ *
+ * stem_search.h -- Functions for searching the blocked stemmed dictionary
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef H_STEM_SEARCH
+#define H_STEM_SEARCH
+
+/*stemmed_dict *ReadStemDictBlk (File * stem_file);*/
+
+stemmed_idx *ReadStemIdxBlk (File * stem_idx_file);
+
+#ifdef __cplusplus
+extern "C"
+#endif
+int FindWord (stemmed_dict * sd, u_char * Word, unsigned long *count,
+	      unsigned long *doc_count, unsigned long *invf_ptr,
+	      unsigned long *invf_len);
+
+/* [RPAP - Jan 97: Stem Index Change] */
+#ifdef __cplusplus
+extern "C"
+#endif
+int FindWords (stemmed_dict * sd, u_char * sWord, int stem_method, TermList ** tl);
+
+void FreeStemDict (stemmed_dict * sd);
+
+/* [RPAP - Jan 97: Stem Index Change] */
+void FreeStemIdx (stemmed_idx * si);
+
+#endif
Index: /tags/gsdl-21/gsdl/src/mgpp/text/term_lists.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/term_lists.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/term_lists.cpp	(revision 699)
@@ -0,0 +1,334 @@
+/**************************************************************************
+ *
+ * filename -- description
+ * Copyright (C) 1994  Authors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/10/11 02:58:35  cs025
+   Base install of MG-PP
+
+   Revision 1.1  1999/08/10 21:18:24  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:52  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:43  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/10/20  03:57:07  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "local_strings.h"
+#include "term_lists.h"
+#include "messages.h"
+#include "stemmer.h"
+
+TermList *query_term_list = NULL;
+
+/* =========================================================================
+ * Function: TermList_create
+ * Description:
+ * Input:
+ * Output:
+ * ========================================================================= */
+TermList *
+TermList_create (int n)
+{
+  TermList *t;
+  int list_size = (n == 0 ? 1 : n);	/* always allocate at least one node */
+
+  t = (TermList *) Xmalloc (sizeof (TermList));
+  if (!t)
+    FatalError (1, "Unable to allocate term list");
+  t->TE = (TermEntry *) Xmalloc (list_size * sizeof (TermEntry));
+  if (!t)
+    FatalError (1, "Unable to allocate term list");
+
+  t->num = n;
+  t->list_size = list_size;
+
+  return t;
+}
+
+/* =========================================================================
+ * Function: TermList_resize
+ * Description:
+ * Input:
+ * Output:
+ * ========================================================================= */
+
+#define GROWTH_FACTOR 2
+#define MIN_SIZE 2
+
+static void
+TermList_resize (TermList * term_list)
+{
+  if (term_list->num > term_list->list_size)
+    {
+      if (term_list->list_size)
+	term_list->list_size *= GROWTH_FACTOR;
+      else
+	term_list->list_size = MIN_SIZE;
+    }
+  /* GRB: We don't need to test for list_size here as it is guaranteed by create
+          to be > 0
+  */
+  term_list->TE = 
+    (TermEntry *) Xrealloc (term_list->TE, term_list->list_size * sizeof (TermEntry));
+
+  if (!term_list->TE)
+    FatalError (1, "Unable to resize term list");
+}
+
+/* =========================================================================
+ * Function: TermList_toString
+ * Description:
+ *      Convert term list into null-terminated string
+ * Input:
+ *      query_term_list = term list
+ * Output:
+ *      str = term string
+ * ========================================================================= */
+
+void TermList_toString (TermList * query_term_list, char *str)
+{
+  int i = 0;
+  int total_len = 0;
+
+  /* terms_str should be preallocated */
+  if (!str)
+    return;
+
+  for (i = 0; i < query_term_list->num; i++)
+    {
+      unsigned char *word = query_term_list->TE[i].Word;
+      int len = word[0];
+      total_len += len + 1;	/* +1 for space */
+      if (total_len > MAXTERMSTRLEN)
+	break;
+      strncpy (str, (char *) word + 1, len);
+      str += len;
+      if (i != (query_term_list->num) - 1)
+	{
+	  *str = ' ';
+	  str++;		/* add space gap */
+	}
+
+    }
+  *str = '\0';
+}
+
+/* =========================================================================
+ * Function: TermList_reset
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+void
+TermList_reset (TermList ** tl)
+{
+  if (*tl)
+    TermList_destroy (tl);
+  *tl = TermList_create (0);
+}
+
+/* =========================================================================
+ * Function: TermList_AddTermEntry
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+int
+TermList_AddTermEntry (TermList * term_list, TermEntry * te)
+{
+  term_list->num++;
+  TermList_resize (term_list);
+
+  /* copy the structure contents */
+  bcopy ((void *) te, (void *) &(term_list->TE[term_list->num - 1]), sizeof (TermEntry));
+
+  return term_list->num - 1;
+}
+
+
+
+/* =========================================================================
+ * Function: TermList_AddTerm
+ * Description: Used in boolean parser - see bool_tree   [RPAP - Feb 97: Term Frequency] 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+int
+TermList_AddTerm (TermList * term_list, u_char * Word, int Count, int word_num,
+	 u_long count, u_long doc_count, u_long invf_ptr, u_long invf_len, int stemmer_num)  /* [RPAP - Feb 97: Term Frequency] */
+{
+  int j;
+
+  /* Look for the word in the already identified terms */
+  for (j = 0; j < term_list->num; j++)
+    {
+      TermEntry *te = &(term_list->TE[j]);
+      if (compare (te->Word, Word) == 0)
+	{
+	  te->Count++;
+	  return j;
+	}
+    }
+
+
+  {
+    /* Create a new entry in the list for the new word */
+    TermEntry te;
+
+    /* [RPAP - Feb 97: Term Frequency] */
+    te.WE.word_num = word_num;
+    te.WE.count = count;
+    te.WE.doc_count = doc_count;
+    te.WE.max_doc_count = doc_count;   /* [RPAP - Jan 97: Stem Index Change] */
+    te.WE.invf_ptr = invf_ptr;
+    te.WE.invf_len = invf_len;
+    te.Count = Count;
+    te.Word = copy_string (Word);
+    if (!te.Word)
+      FatalError (1, "Could NOT create memory to add term");
+
+    /* [RPAP - Jan 97: Stem Index Change] */
+    te.Stem = copy_string (Word);
+    if (!te.Stem)
+      FatalError (1, "Could NOT create memory to add term");
+    stemmer (2, stemmer_num, te.Stem);
+
+    te.require_match = 0;  /* [RJM - 07/97: Ranked Required Terms] */
+
+    return TermList_AddTermEntry (term_list, &te);
+  }
+
+}
+
+/* =========================================================================
+ * Function: TermList_destroy
+ * Description:
+ * Input:
+ * Output:
+ * ========================================================================= */
+
+void
+TermList_destroy (TermList ** the_tl)
+{
+  int j;
+  TermList *tl = *the_tl;
+
+  for (j = 0; j < tl->num; j++)
+    {
+      if (tl->TE[j].Word)
+	Xfree (tl->TE[j].Word);
+      /* [RPAP - Jan 97: Stem Index Change] */
+      if (tl->TE[j].Stem)
+	Xfree (tl->TE[j].Stem);
+    }
+  Xfree (tl->TE);
+  Xfree (tl);
+
+  *the_tl = NULL;
+}
+
+/* =========================================================================
+ * Function: WordEntry_print
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+void
+WordEntry_print (WordEntry * we, FILE * file)
+{
+  fprintf (file, "we->word_num = %d\n", we->word_num);
+  fprintf (file, "we->count = %ld\n", we->count);
+  fprintf (file, "we->doc_count = %ld\n", we->doc_count);
+  fprintf (file, "we->max_doc_count = %ld\n", we->max_doc_count);
+  fprintf (file, "we->invf_ptr = %ld\n", we->invf_ptr);
+  fprintf (file, "we->invf_len = %ld\n", we->invf_len);
+}
+
+/* =========================================================================
+ * Function: TermEntry_print
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+void
+TermEntry_print (TermEntry * te, FILE * file)
+{
+
+  fprintf (file, "Term Entry\n");
+  fprintf (file, "te->Count = %d\n", te->Count);
+  fprintf (file, "te->Word = %s\n", str255_to_string (te->Word, NULL));
+  if (te->Stem != NULL)
+    fprintf (file, "te->Stem = %s\n", str255_to_string (te->Stem, NULL));  /* [RPAP - Jan 97: Stem Index Change] */
+  fprintf (file, "te->require_match = %i\n", te->require_match);  /* [RJM 07/97: Ranked Required Terms] */
+  WordEntry_print (&(te->WE), file);
+
+}
+
+/* =========================================================================
+ * Function: TermList_print
+ * Description: 
+ * Input: 
+ * Output: 
+ * ========================================================================= */
+
+void
+TermList_print (TermList * tl, FILE * file)
+{
+  int i;
+
+  fprintf (file, "Term List\n");
+  fprintf (file, "tl->list_size = %d\n", tl->list_size);
+  fprintf (file, "tl->num = %d\n", TermList_size(tl));
+
+  for (i = 0; i < TermList_size(tl); i++)
+    {
+      fprintf (file, "[%d]\n", i);
+      TermEntry_print (&(tl->TE[i]), file);
+    }
+}
+
Index: /tags/gsdl-21/gsdl/src/mgpp/text/term_lists.h
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/term_lists.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/term_lists.h	(revision 699)
@@ -0,0 +1,130 @@
+/**************************************************************************
+ *
+ * term_lists.h -- description
+ * Copyright (C) 1994  Authors
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/10/11 02:58:36  cs025
+   Base install of MG-PP
+
+   Revision 1.1  1999/08/10 21:18:24  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.2  1998/11/25 07:55:52  rjmcnab
+
+   Modified mg to that you can specify the stemmer you want
+   to use via a command line option. You specify it to
+   mg_passes during the build process. The number of the
+   stemmer that you used is stored within the inverted
+   dictionary header and the stemmed dictionary header so
+   the correct stemmer is used in later stages of building
+   and querying.
+
+   Revision 1.1  1998/11/17 09:35:44  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.1  1994/10/20  03:57:08  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+ */
+
+#ifndef TERM_LISTS_H
+#define TERM_LISTS_H
+
+#include "sysfuncs.h"
+
+#define MAXTERMSTRLEN 1023	/* maximum number of characters in term string */
+
+typedef struct WordEntry
+  {
+    int word_num;		/* Unique number for each different word */
+    u_long count;		/* Number of times the word occurs in the text */
+    u_long doc_count;		/* Number of documents that contain the word */
+    u_long max_doc_count;       /* [RPAP - Jan 97: Stem Index Change] */
+    u_long invf_ptr;		/* This is a byte position of the  
+				   inverted file entry corresponding to the word */
+    u_long invf_len;		/* This is the length of the inverted 
+				   file entry in bytes */
+  }
+WordEntry;
+
+typedef struct TermEntry
+  {
+    WordEntry WE;
+    int Count;			/* The number of times the word occurs in the query */
+    u_char *Word;		/* The word. */
+    u_char *Stem;               /* [RPAP - Jan 97: Stem Index Change] - The stemmed non-cf version of Word */
+    int require_match;          /* [RJM 07/97: Ranked Required Terms] 0=optional match, 1=must match */
+
+    inline int wordLength()
+    { 
+      return Word[0];
+    }
+
+    inline unsigned char *word()
+    {
+      return Word + 1;
+    }
+  }
+TermEntry;
+
+typedef struct TermList
+  {
+    int list_size;
+    int num;
+    TermEntry *TE;
+
+    inline TermEntry *entry(int term)
+    { return &TE[term];
+    }
+
+    inline int size()
+    {
+      return num;
+    }
+  }
+TermList;
+
+#define GetNthWE(term_list, n) (&((term_list)->TE[(n)].WE))
+
+/* --- prototypes --- */
+void TermList_toString (TermList * query_term_list, char *str);
+int TermList_AddTermEntry (TermList * query_term_list, TermEntry * te);
+int TermList_AddTerm (TermList * query_term_list, u_char * Word, int Count, int word_num,
+	     u_long count, u_long doc_count, u_long invf_ptr, u_long invf_len, /* [RPAP - Feb 97: Term Frequency] */
+	     int stemmer_num);  
+#ifdef __cplusplus
+extern "C"
+#endif
+void TermList_reset(TermList **tl);
+void TermList_destroy (TermList ** the_tl);
+void TermList_print (TermList * tl, FILE * file);
+TermList *TermList_create (int n);
+
+#ifdef __cplusplus
+inline int TermList_size(TermList *tl)
+{ return tl->num;
+}
+#endif
+
+#endif
Index: /tags/gsdl-21/gsdl/src/mgpp/text/text_get.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/text_get.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/text_get.cpp	(revision 699)
@@ -0,0 +1,1163 @@
+/**************************************************************************
+ *
+ * text_get.c -- Function for reading documents from the compressed text
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "memlib.h"
+#include "filestats.h"
+#include "timing.h"
+#include "messages.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "huffman.h"
+#include "bitio_m_abstract.h"
+#include "bitio_m_mem.h"
+#include "bitio_m.h"
+#include "bitio_stdio.h"
+#include "huffman_stdio.h"
+
+#include "mg.h"
+#include "invf.h"
+#include "text.h"
+#include "lists.h"
+#include "backend.h"
+#include "text_get.h"
+#include "locallib.h"
+#include "words.h"
+#include "mg_errors.h"
+#include "local_strings.h"
+#include "DocEntry.h"
+
+/*
+   $Log$
+   Revision 1.1  1999/10/11 02:58:39  cs025
+   Base install of MG-PP
+
+   Revision 1.1  1999/08/10 21:18:26  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:35:48  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.3  1994/10/20  03:57:11  tes
+   * I have rewritten the boolean query optimiser and abstracted out the
+   * components of the boolean query.
+   *
+   * Revision 1.2  1994/09/20  04:42:15  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+
+
+
+/* FetchDocStart () 
+ * Reads into DocEnt the starting position of the document in the *.text file
+ * Where the first document is document number 1
+ * It returns the true weight of the document.
+ */
+
+
+
+
+static double 
+FetchDocStartLev1 (text_data * td, u_long DN,
+		   u_long * seek_pos, u_long * len)
+{
+  unsigned long data[2];
+  /* [TS:Sep/94] Fixed up the seek call to give the correct offset */
+  Fseek (td->TextIdxFile,
+	 sizeof (unsigned long) * (DN - 1) +	/* the doc offsets */
+	 sizeof (unsigned long) +	/* the magic number */
+	 sizeof (compressed_text_header),	/* the header */
+	 0);
+  Fread ((char *) &data, sizeof (data), 1, td->TextIdxFile);
+  
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(data[0]);
+  NTOHUL(data[1]);
+
+  *seek_pos = data[0];
+  *len = data[1] - data[0];
+  return (1.0);
+}
+
+#define MG_PAGE_SIZE 2048
+
+static int 
+LoadIdx (text_data * td, unsigned long DN)
+{
+  if (!td->idx_data)
+    {
+      td->idx_data = (__idx_data *) Xmalloc (sizeof (*(td->idx_data)) * MG_PAGE_SIZE);
+      if (!td->idx_data)
+	FatalError (1, "Out of memory in FDSL2");
+    }
+  if (td->current_pos == -1 || DN >= td->current_pos + MG_PAGE_SIZE - 1 ||
+      DN < td->current_pos)
+    {
+      int i, num;  /* [RPAP - Jan 97: Endian Ordering] */
+
+      long rn = (long) DN - (MG_PAGE_SIZE >> 1);
+      if (rn < 1)
+	rn = 1;
+      Fseek (td->TextIdxWgtFile, (sizeof (unsigned long) + sizeof (float)) *
+	       (rn - 1) + sizeof (unsigned long), 0);
+      num = Fread ((char *) td->idx_data, sizeof (*(td->idx_data)), MG_PAGE_SIZE, /* [RPAP - Jan 97: Endian Ordering] */
+		   td->TextIdxWgtFile);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      for (i = 0; i < num; i++)
+	{
+	  NTOHUL(td->idx_data[i].Start);
+	  NTOHF(td->idx_data[i].Weight);
+	}
+
+      td->current_pos = rn;
+    }
+  return DN - td->current_pos;
+}
+
+static double 
+FDSL2 (text_data * td, unsigned long DN, unsigned long *Pos)
+{
+  unsigned long pos = LoadIdx (td, DN);
+  *Pos = td->idx_data[pos].Start;
+  return (td->idx_data[pos].Weight);
+}
+
+
+static double 
+FetchDocStartLev2 (text_data * td, u_long DN,
+		   u_long * seek_pos, u_long * len)
+{
+  double Weight;
+  unsigned long s1, s2;
+
+  Weight = FDSL2 (td, DN, &s1);
+  do
+    {
+      DN++;
+      FDSL2 (td, DN, &s2);
+    }
+  while (s2 == s1);
+  *seek_pos = s1;
+  *len = s2 - s1;
+  return (Weight);
+}
+
+
+
+
+double 
+FetchDocStart (query_data * qd, u_long DN, u_long * seek_pos, u_long * len)
+{
+  qd->text_idx_lookups++;
+
+  if (qd->td->TextIdxWgtFile)
+    return FetchDocStartLev2 (qd->td, DN, seek_pos, len);
+  else
+    return FetchDocStartLev1 (qd->td, DN, seek_pos, len);
+}
+
+unsigned long 
+FetchInitialParagraph (text_data * td, unsigned long ParaNum)
+{
+  if (td->TextIdxWgtFile)
+    {
+      unsigned long pos;
+      unsigned long start;
+      int PN = ParaNum - 1;
+      pos = LoadIdx (td, ParaNum);
+      start = td->idx_data[pos].Start;
+      while (PN > 0)
+	{
+	  pos = LoadIdx (td, PN);
+	  if (td->idx_data[pos].Start != start)
+	    return PN + 1;
+	  PN--;
+	}
+      return PN + 1;
+    }
+  else
+    return ParaNum;
+}
+
+
+
+/* FetchCompressed () 
+ * Reads into buffer DocBuff the compressed form of document DocNum. 
+ * Where the first document is document number 1
+ */
+int 
+FetchCompressed (query_data * qd, char **DocBuff, DocEntry * DocEnt)
+{
+  if (!DocEnt->SeekPos)
+    DocEnt->FetchStart(qd);
+  //    FetchDocStart (qd, DocEnt->DocNum, &DocEnt->SeekPos, &DocEnt->Len);
+  if (!(*DocBuff = (char *) Xmalloc (DocEnt->Len)))
+    return (-1);
+
+  if (Fseek (qd->td->TextFile, DocEnt->SeekPos, 0) == -1)
+    FatalError (1, "Error when seeking into text file");
+#if 0
+  printf ("Loading compressed text %d %d\n", DocEnt->SeekPos, DocEnt->Len);
+#endif
+  if (Fread (*DocBuff, 1, DocEnt->Len, qd->td->TextFile) != DocEnt->Len)
+    FatalError (1, "Error when reading data");
+
+  return (DocEnt->Len);
+
+}
+
+
+text_data *
+LoadTextData (File * text, File * text_idx_wgt, File * text_idx)
+{
+  text_data *td;
+
+  if (!(td = (text_data *) Xmalloc (sizeof (text_data))))
+    {
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  td->TextFile = text;
+  td->TextIdxWgtFile = text_idx_wgt;
+  td->TextIdxFile = text_idx;
+  td->current_pos = -1;
+  td->idx_data = NULL;
+  Fread (&td->cth, sizeof (td->cth), 1, text);
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  NTOHUL(td->cth.num_of_docs);
+  NTOHD(td->cth.num_of_bytes); /* [RJM 07/97: 4G limit] */
+  NTOHUL(td->cth.num_of_words);
+  NTOHUL(td->cth.length_of_longest_doc);
+  NTOHD(td->cth.ratio);
+
+  return (td);
+}
+
+
+void 
+FreeTextData (text_data * td)
+{
+  if (td)
+    {
+      if (td->idx_data)
+	Xfree (td->idx_data);
+      Xfree (td);
+    }
+}
+
+
+static int 
+pts_comp (const void *A, const void *B)
+{
+  const DocEntry *const *a = (DocEntry **) A;
+  const DocEntry *const *b = (DocEntry **) B;
+  return (*a)->DocNum - (*b)->DocNum;
+}
+
+
+
+
+int 
+GetPosLens (query_data * qd, DocEntry * Docs, int num)
+{
+  DocEntry **pts;
+  int i, j;
+  if (!(pts = (DocEntry **) Xmalloc (num * sizeof (DocEntry *))))
+    {
+      mg_errno = MG_NOMEM;
+      return (-1);
+    }
+  for (i = j = 0; i < num; i++, Docs++)
+    if (!Docs->SeekPos)
+      pts[j++] = Docs;
+
+  if (j)
+    {
+      qsort (pts, j, sizeof (DocEntry *), pts_comp);
+      for (i = 0; i < j; i++)
+	pts[i]->FetchStart(qd);
+      //	FetchDocStart (qd, pts[i]->DocNum, &pts[i]->SeekPos, &pts[i]->Len);
+    }
+
+  Xfree (pts);
+  return (0);
+}
+
+
+
+
+
+int 
+LoadBuffers (query_data * qd, DocEntry * Docs, int max_mem, int num)
+{
+  DocEntry **pts;
+  int i, j;
+  int mem;
+
+  if (!num)
+    return (0);
+  if (!(pts = (DocEntry **) Xmalloc (num * sizeof (DocEntry *))))
+    {
+      mg_errno = MG_NOMEM;
+      return (-1);
+    }
+
+  mem = i = 0;
+  do
+    {
+      pts[i] = Docs;
+      mem += Docs->Len;
+      i++;
+      Docs++;
+    }
+  while (i < num && mem < max_mem);
+  if (i > 1)
+    qsort (pts, i, sizeof (DocEntry *), pts_comp);
+  for (j = 0; j < i; j++)
+    {
+      if (FetchCompressed (qd, &pts[j]->CompTextBuffer, pts[j]) == -1)
+	return (-1);
+      ChangeMemInUse (qd, pts[j]->Len);
+    }
+
+  Xfree (pts);
+
+  return (i);
+}
+
+
+/**
+ *
+ * GRB: Function removed 21/09/99 - wasn't being used; in any case used DocEntry_FreeTextBuffers instead
+ *
+ *
+void 
+FreeBuffers (query_data * qd, DocEntry * Docs, int num)
+{
+  int i;
+  
+  for (i = 0; i < num; i++, Docs++)
+    if (Docs->CompTextBuffer)
+      {
+	Xfree (Docs->CompTextBuffer);
+	Docs->CompTextBuffer = NULL;
+	ChangeMemInUse (qd, -Docs->Len);
+      }
+ 
+}
+*/
+
+
+/****************************************************************************/
+
+static void 
+FreeAuxDict (auxiliary_dict * ad)
+{
+  if (!ad)
+    return;
+  if (ad->word_data[0])
+    Xfree (ad->word_data[0]);
+  if (ad->word_data[1])
+    Xfree (ad->word_data[1]);
+  if (ad->words[0])
+    Xfree (ad->words[0]);
+  if (ad->words[1])
+    Xfree (ad->words[1]);
+  Xfree (ad);
+}
+
+static auxiliary_dict *
+LoadAuxDict (compression_dict * cd, File * text_aux_dict)
+{
+  auxiliary_dict *ad;
+  int i;
+
+  if (!(ad = (auxiliary_dict *) Xmalloc (sizeof (auxiliary_dict))))
+    {
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  bzero ((char *) ad, sizeof (*ad));
+
+  for (i = 0; i <= 1; i++)
+    {
+      int j;
+      u_char *pos;
+
+      Fread (&ad->afh[i], sizeof (aux_frags_header), 1, text_aux_dict);
+
+      /* [RPAP - Jan 97: Endian Ordering] */
+      NTOHUL(ad->afh[i].num_frags);
+      NTOHUL(ad->afh[i].mem_for_frags);
+
+      if (!(ad->word_data[i] = (u_char *) Xmalloc (ad->afh[i].mem_for_frags)))
+	{
+	  mg_errno = MG_NOMEM;
+	  FreeAuxDict (ad);
+	  return (NULL);
+	}
+      if (!(ad->words[i] = (u_char **) Xmalloc (ad->afh[i].num_frags * sizeof (u_char *))))
+	{
+	  mg_errno = MG_NOMEM;
+	  FreeAuxDict (ad);
+	  return (NULL);
+	}
+
+      Fread (ad->word_data[i], ad->afh[i].mem_for_frags, sizeof (u_char),
+	     text_aux_dict);
+
+      pos = ad->word_data[i];
+      for (j = 0; j < ad->afh[i].num_frags; j++)
+	{
+	  ad->words[i][j] = pos;
+	  pos += *pos + 1;
+	}
+      if (cd->cdh.novel_method == MG_NOVEL_HYBRID ||
+	  cd->cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+	{
+	  int num;
+	  num = 1;
+	  ad->blk_start[i][0] = 0;
+	  ad->blk_end[i][0] = cd->cdh.num_words[i] - 1;
+	  while (num < 33)
+	    {
+	      ad->blk_start[i][num] = ad->blk_end[i][num - 1] + 1;
+	      ad->blk_end[i][num] = ad->blk_start[i][num] +
+		(ad->blk_end[i][num - 1] - ad->blk_start[i][num - 1]) * 2;
+	      num++;
+	    }
+	}
+    }
+  return (ad);
+}
+
+
+
+
+
+
+static u_char ***
+ReadInWords (File * dict, compression_dict * cd,
+	     comp_frags_header * cfh, u_char ** escape)
+{
+  int i, lookback;
+  int ptrs_reqd = 0;
+  int mem_reqd = 0;
+  int num_set[MAX_HUFFCODE_LEN + 1];
+  u_char *next_word[MAX_HUFFCODE_LEN + 1];
+  u_char **vals;
+  u_char ***values;
+  u_char word[MAXWORDLEN + 1];
+  u_char last_word[MAX_HUFFCODE_LEN + 1][MAXWORDLEN + 1];
+
+  lookback = cd->cdh.lookback;
+
+  for (i = cfh->hd.mincodelen; i <= cfh->hd.maxcodelen; i++)
+    {
+      ptrs_reqd += (cfh->hd.lencount[i] + ((1 << lookback) - 1)) >> lookback;
+      mem_reqd += cfh->huff_words_size[i];
+    }
+
+  if (!(vals = (u_char **) Xmalloc (ptrs_reqd * sizeof (*vals))))
+    return (NULL);
+
+  if (!(values = (u_char ***) Xmalloc ((MAX_HUFFCODE_LEN + 1) * sizeof (u_char **))))
+    return (NULL);
+
+  if (!(next_word[0] = (u_char *) Xmalloc (mem_reqd))) return (NULL);
+
+  cd->MemForCompDict += ptrs_reqd * sizeof (*vals) +
+    (MAX_HUFFCODE_LEN + 1) * sizeof (u_char **) +
+    mem_reqd;
+
+  values[0] = vals;
+  values[0][0] = next_word[0];
+  for (i = 1; i <= cfh->hd.maxcodelen; i++)
+    {
+      int next_start = (values[i - 1] - vals) +
+      ((cfh->hd.lencount[i - 1] + ((1 << lookback) - 1)) >> lookback);
+      values[i] = &vals[next_start];
+      next_word[i] = next_word[i - 1] + cfh->huff_words_size[i - 1];
+      values[i][0] = next_word[i];
+    }
+
+  bzero ((char *) num_set, sizeof (num_set));
+
+  for (i = 0; i < cfh->hd.num_codes; i++)
+    {
+      register int val, copy;
+      register int len = cfh->hd.clens[i];
+      val = Getc (dict);
+      copy = (val >> 4) & 0xf;
+      val &= 0xf;
+
+      Fread (word + copy + 1, sizeof (u_char), val, dict);
+      *word = val + copy;
+
+      if ((num_set[len] & ((1 << lookback) - 1)) == 0)
+	{
+	  values[len][num_set[len] >> lookback] = next_word[len];
+	  memcpy (next_word[len], word, *word + 1);
+	  if (escape && i == cfh->hd.num_codes - 1)
+	    *escape = next_word[len];
+	  next_word[len] += *word + 1;
+	}
+      else
+	{
+	  copy = prefixlen (last_word[len], word);
+	  memcpy (next_word[len] + 1, word + copy + 1, *word - copy);
+	  *next_word[len] = (copy << 4) + (*word - copy);
+	  if (escape && i == cfh->hd.num_codes - 1)
+	    *escape = next_word[len];
+	  next_word[len] += (*word - copy) + 1;
+	}
+      memcpy (last_word[len], word, *word + 1);
+      num_set[len]++;
+    }
+  if (cfh->hd.clens)
+    Xfree (cfh->hd.clens);
+  cfh->hd.clens = NULL;
+  return values;
+}
+
+int Load_Comp_HuffData(compression_dict *cd, int which, File *dict)
+{
+  huff_data * hd;
+  u_long **   vals;
+
+  if (!(hd = (huff_data *) Xmalloc (sizeof (huff_data))))
+    return 1;
+  cd->MemForCompDict += sizeof (huff_data);
+  if (F_Read_Huffman_Data (dict, hd, &cd->MemForCompDict, NULL) == -1)
+    return 2;
+  if (!(vals = Generate_Huffman_Vals (hd, &cd->MemForCompDict)))
+    return 3;
+  if (hd->clens)
+    Xfree (hd->clens);
+  hd->clens = NULL;
+  cd->chars_huff[which] = hd;
+  cd->chars_vals[which] = vals;
+
+  return 0;
+}
+
+int Load_Comp_FragsHeader(compression_dict *cd, int which, int getEscape, File *dict)
+{ 
+  if (!(cd->cfh[which] = (comp_frags_header *) Xmalloc (sizeof (*cd->cfh[which]))))
+    return 1;
+  cd->MemForCompDict += sizeof (*cd->cfh[which]);
+  if (F_Read_cfh (dict, cd->cfh[which], &cd->MemForCompDict, NULL) == -1)
+    return 2;
+
+  if (!(cd->values[which] = ReadInWords (dict, cd, cd->cfh[which],
+					 getEscape == 0 ? NULL : &cd->escape[which])))
+    return 3;
+  return 0;
+}
+
+static compression_dict *
+Load_Comp_Dict (File * dict, File * aux_dict)
+{
+  int which;
+  compression_dict *cd;
+
+  if (!(cd = (compression_dict *) Xmalloc (sizeof (compression_dict))))
+    {
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  bzero ((char *) cd, sizeof (compression_dict));
+
+  cd->MemForCompDict = sizeof (compression_dict);
+
+  if (F_Read_cdh (dict, &cd->cdh, &cd->MemForCompDict, NULL) == -1)
+    return NULL;
+
+  for (which = 0; which < 2; which++)
+    switch (cd->cdh.dict_type)
+      {
+      case MG_COMPLETE_DICTIONARY:
+	{
+	  if (Load_Comp_FragsHeader(cd, which, 0, dict) != 0)
+	    return NULL;
+	  /*
+          if (!(cd->cfh[which] = (comp_frags_header *) Xmalloc (sizeof (*cd->cfh[which]))))
+	    return NULL;
+	  cd->MemForCompDict += sizeof (*cd->cfh[which]);
+	  if (F_Read_cfh (dict, cd->cfh[which], &cd->MemForCompDict, NULL) == -1)
+	    return NULL;
+
+	  if (!(cd->values[which] = ReadInWords (dict, cd, cd->cfh[which],
+						 NULL)))
+	    return NULL;
+	  */
+	  cd->escape[which] = NULL;
+
+	}
+	break;
+      case MG_PARTIAL_DICTIONARY:
+	{
+	  huff_data *hd;
+	  u_long **vals;
+	  if (cd->cdh.num_words[which])
+	    {
+	      if (Load_Comp_FragsHeader(cd, which, 1, dict) != 0)
+		return NULL;
+	      /*
+	      if (!(cd->cfh[which] = (comp_frags_header *) Xmalloc (sizeof (*cd->cfh[which]))))
+		return NULL;
+	      cd->MemForCompDict += sizeof (*cd->cfh[which]);
+	      if (F_Read_cfh (dict, cd->cfh[which], &cd->MemForCompDict, NULL) == -1)
+		return NULL;
+
+	      if (!(cd->values[which] = ReadInWords (dict, cd, cd->cfh[which],
+						     &cd->escape[which])))
+		return NULL;
+	      */
+	    }
+	  if (Load_Comp_HuffData(cd, which, dict) != 0)
+	    return NULL;
+	  /*
+	  if (!(hd = (huff_data *) Xmalloc (sizeof (huff_data))))
+	    return NULL;
+	  cd->MemForCompDict += sizeof (huff_data);
+	  if (F_Read_Huffman_Data (dict, hd, &cd->MemForCompDict, NULL) == -1)
+	    return NULL;
+	  if (!(vals = Generate_Huffman_Vals (hd, &cd->MemForCompDict)))
+	    return NULL;
+	  if (hd->clens)
+	    Xfree (hd->clens);
+	  hd->clens = NULL;
+	  cd->chars_huff[which] = hd;
+	  cd->chars_vals[which] = vals;
+	  */
+	  if (Load_Comp_HuffData(cd, which, dict) != 0)
+	    return NULL;
+	  /*
+	  if (!(hd = (huff_data *) Xmalloc (sizeof (huff_data))))
+	    return NULL;
+	  cd->MemForCompDict += sizeof (huff_data);
+	  if (F_Read_Huffman_Data (dict, hd, &cd->MemForCompDict, NULL) == -1)
+	    return NULL;
+	  if (!(vals = Generate_Huffman_Vals (hd, &cd->MemForCompDict)))
+	    return NULL;
+	  cd->lens_huff[which] = hd;
+	  cd->lens_vals[which] = vals;
+	  if (hd->clens)
+	    Xfree (hd->clens);
+	  hd->clens = NULL;
+	  */
+	}
+	break;
+      case MG_SEED_DICTIONARY:
+	{
+	  huff_data *hd;
+	  u_long **vals;
+	  if (cd->cdh.num_words[which])
+	    {
+	      if (Load_Comp_FragsHeader(cd, which, 1, dict))
+		return NULL;
+	      /*
+	      if (!(cd->cfh[which] = (comp_frags_header *) Xmalloc (sizeof (*cd->cfh[which]))))
+		return NULL;
+	      cd->MemForCompDict += sizeof (*cd->cfh[which]);
+	      if (F_Read_cfh (dict, cd->cfh[which], &cd->MemForCompDict, NULL) == -1)
+		return NULL;
+
+	      if (!(cd->values[which] = ReadInWords (dict, cd, cd->cfh[which],
+						     &cd->escape[which])))
+		return NULL;
+	      */
+	    }
+	  switch (cd->cdh.novel_method)
+	    {
+	    case MG_NOVEL_HUFFMAN_CHARS:
+	      if (Load_Comp_HuffData(cd, which, dict) != 0)
+		return NULL;
+	      /*
+	      if (!(hd = (huff_data *) Xmalloc (sizeof (huff_data))))
+		return NULL;
+	      cd->chars_huff[which] = hd;
+	      cd->MemForCompDict += sizeof (huff_data);
+	      if (F_Read_Huffman_Data (dict, hd, &cd->MemForCompDict,
+				       NULL) == -1)
+		return NULL;
+	      if (!(vals = Generate_Huffman_Vals (hd, &cd->MemForCompDict)))
+		return NULL;
+	      cd->chars_vals[which] = vals;
+	      if (hd->clens)
+		Xfree (hd->clens);
+	      hd->clens = NULL;
+	      */
+
+	      if (Load_Comp_HuffData(cd, which, dict) != 0)
+		return NULL;
+	      /*
+	      if (!(hd = (huff_data *) Xmalloc (sizeof (huff_data))))
+		return NULL;
+	      cd->MemForCompDict += sizeof (huff_data);
+	      cd->lens_huff[which] = hd;
+	      if (F_Read_Huffman_Data (dict, hd, &cd->MemForCompDict
+				       ,NULL) == -1)
+		return NULL;
+	      if (!(vals = Generate_Huffman_Vals (hd, &cd->MemForCompDict)))
+		return NULL;
+	      cd->lens_vals[which] = vals;
+	      if (hd->clens)
+		Xfree (hd->clens);
+	      hd->clens = NULL;
+	      */
+	      break;
+	    case MG_NOVEL_BINARY:
+	      break;
+	    case MG_NOVEL_DELTA:
+	      break;
+	    case MG_NOVEL_HYBRID:
+	      break;
+	    case MG_NOVEL_HYBRID_MTF:
+	      break;
+	    }
+	  break;
+	}
+      }
+
+  if (cd->cdh.novel_method == MG_NOVEL_BINARY ||
+      cd->cdh.novel_method == MG_NOVEL_DELTA ||
+      cd->cdh.novel_method == MG_NOVEL_HYBRID ||
+      cd->cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+    {
+      if (!aux_dict)
+	{
+	  mg_errno = MG_NOFILE;
+	  FreeCompDict (cd);
+	  return (NULL);
+	}
+
+      if (!(cd->ad = LoadAuxDict (cd, aux_dict)))
+	{
+	  FreeCompDict (cd);
+	  return (NULL);
+	}
+    }
+
+
+  mg_errno = MG_NOERROR;
+
+  cd->fast_loaded = 0;
+  return (cd);
+}
+
+#define WORDNO(p, base) ((((char*)(p))-((char*)(base)))/sizeof(u_char*))
+
+#define IS_FIXUP(p) ((fixup[WORDNO(p,cd)/8] & (1<<(WORDNO(p, cd) & 7))) != 0)
+
+
+static compression_dict *
+Load_Fast_Comp_Dict (File * text_fast_comp_dict)
+{
+  compression_dict *cd;
+  u_long *p, *end;
+  u_char *fixup;
+  u_long mem;
+  u_long fixup_mem;
+  int i;  /* [RPAP - Jan 97: Endian Ordering] */
+
+  Fread (&mem, sizeof (mem), 1, text_fast_comp_dict);
+  NTOHUL(mem);  /* [RPAP - Jan 97: Endian Ordering] */
+  Fread (&fixup_mem, sizeof (fixup_mem), 1, text_fast_comp_dict);
+  NTOHUL(fixup_mem);  /* [RPAP - Jan 97: Endian Ordering] */
+  if (!(cd = (compression_dict *) Xmalloc (mem)))
+    {
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  end = (u_long *) (((u_char *) cd) + mem);
+  Fread (cd, sizeof (u_char), mem, text_fast_comp_dict);
+
+  if (!(fixup = (u_char *) Xmalloc (fixup_mem)))
+    {
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  Fread (fixup, fixup_mem, sizeof (u_char), text_fast_comp_dict);
+
+  for (p = (u_long *) cd; (u_long) p < (u_long) end; p++)
+    if (IS_FIXUP (p))
+      {
+	NTOHUL(*p);  /* [RPAP - Jan 97: Endian Ordering] */
+	*p = *p + (u_long) cd;
+      }
+
+  /* [RPAP - Jan 97: Endian Ordering] */
+  /* cdh */
+  NTOHUL(cd->cdh.dict_type);
+  NTOHUL(cd->cdh.novel_method);
+  for (i = 0; i < TEXT_PARAMS; i++)
+    NTOHUL(cd->cdh.params[i]);
+  NTOHUL(cd->cdh.num_words[0]);
+  NTOHUL(cd->cdh.num_words[1]);
+  NTOHUL(cd->cdh.num_word_chars[0]);
+  NTOHUL(cd->cdh.num_word_chars[1]);
+  NTOHUL(cd->cdh.lookback);
+  /* cfh */
+  for (i = 0; i <= 1; i++)
+    {
+      int j;
+
+      NTOHSI(cd->cfh[i]->hd.num_codes);
+      NTOHSI(cd->cfh[i]->hd.mincodelen);
+      NTOHSI(cd->cfh[i]->hd.maxcodelen);
+      for (j = 0; j < MAX_HUFFCODE_LEN + 1; j++)
+	{
+	  NTOHSI(cd->cfh[i]->hd.lencount[j]);
+	  NTOHUL(cd->cfh[i]->hd.min_code[j]);
+	}
+      NTOHUL(cd->cfh[i]->uncompressed_size);
+      for (j = 0; j < MAX_HUFFCODE_LEN + 1; j++)
+	NTOHUL(cd->cfh[i]->huff_words_size[j]);
+    }
+  NTOHUL(cd->MemForCompDict);
+  /* ad */
+  if (cd->cdh.novel_method == MG_NOVEL_BINARY ||
+      cd->cdh.novel_method == MG_NOVEL_DELTA ||
+      cd->cdh.novel_method == MG_NOVEL_HYBRID ||
+      cd->cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+    for (i = 0; i <= 1; i++)
+      {
+	int j;
+	
+	NTOHUL(cd->ad->afh[i].num_frags);
+	NTOHUL(cd->ad->afh[i].mem_for_frags);
+	for (j = 0; j < 33; j++)
+	  {
+	    NTOHSI(cd->ad->blk_start[i][j]);
+	    NTOHSI(cd->ad->blk_end[i][j]);
+	  }
+      }
+  NTOHSI(cd->fast_loaded);
+
+  Xfree (fixup);
+  return (cd);
+}
+
+
+
+
+compression_dict *
+LoadCompDict (File * text_comp_dict,
+	      File * text_aux_dict,
+	      File * text_fast_comp_dict)
+{
+  return text_fast_comp_dict ?
+    Load_Fast_Comp_Dict (text_fast_comp_dict) :
+    Load_Comp_Dict (text_comp_dict, text_aux_dict);
+}
+
+
+
+
+void 
+FreeCompDict (compression_dict * cd)
+{
+  int which;
+
+  if (cd->fast_loaded)
+    {
+      Xfree (cd);
+      return;
+    }
+  for (which = 0; which < 2; which++)
+    {
+      if (cd->cfh[which])
+	Xfree (cd->cfh[which]);
+      if (cd->chars_huff[which])
+	Xfree (cd->chars_huff[which]);
+      if (cd->lens_huff[which])
+	Xfree (cd->lens_huff[which]);
+      if (cd->values[which])
+	{
+	  Xfree (cd->values[which][0][0]); /* [RJM 07/98: Memory Leak] */
+	  Xfree (cd->values[which][0]);
+	  Xfree (cd->values[which]);
+	}
+      if (cd->chars_vals[which])
+	{
+	  Xfree (cd->chars_vals[which][0]);
+	  Xfree (cd->chars_vals[which]);
+	}
+      if (cd->lens_vals[which])
+	{
+	  Xfree (cd->lens_vals[which][0]);
+	  Xfree (cd->lens_vals[which]);
+	}
+    }
+  if (cd->ad)
+    FreeAuxDict (cd->ad);
+  Xfree (cd);
+}
+
+
+
+
+
+#define MY_HUFF_DECODE(len, code, mcodes)				\
+  do {									\
+    register unsigned long *__min_code = (mcodes);			\
+    register unsigned long *__mclen = __min_code;			\
+    register unsigned long __code = 0;					\
+    do									\
+      {									\
+        __code += __code + buffer.bit();	                  	\
+      }									\
+    while (__code < *++__mclen);					\
+    (len) = __mclen - __min_code;					\
+    (code) = __code - *__mclen;						\
+  } while(0);
+
+
+/*#define DUMPDOC */
+
+#define MAX_SWAPS 10000
+
+int 
+DecodeText (compression_dict * cd,
+	    u_char * s_in, int l_in, u_char * s_out, int *l_out)
+{
+  auxiliary_dict *ad = cd->ad;
+  int which;
+  u_long num_bits, bits;
+  u_char *ptr = s_out;
+  static int num = 0;
+  u_long binary_start[2];
+  int novels_used[2];
+  int swaps[2][MAX_SWAPS];
+  novels_used[0] = novels_used[1] = 0;
+
+  {
+    unsigned char bf = s_in[l_in - 1];
+    num_bits = 1;
+    while ((bf & 1) != 1)
+      {
+	num_bits++;
+	bf >>= 1;
+      }
+    num_bits = l_in * 8 - num_bits;
+  }
+
+  DECODE_START (s_in, l_in)
+
+    which = buffer.bit();
+  bits = 1;
+
+  if (cd->cdh.novel_method == MG_NOVEL_BINARY)
+    {
+      binary_start[0] = buffer.delta_decode (&bits);
+      binary_start[1] = buffer.delta_decode (&bits);
+    }
+
+
+  while (bits < num_bits)
+    {
+      register unsigned code, len;
+      register int r;
+      register u_char *t, *b = NULL;
+      u_char word[MAXWORDLEN + 1];
+
+#ifdef DUMPDOC
+      printf ("\n%d %d  ", bits, num_bits);
+#endif
+      if (cd->cfh[which])
+	{
+	  MY_HUFF_DECODE (len, code, cd->cfh[which]->hd.min_code);
+	  bits += len;
+
+	  r = code & ((1 << cd->cdh.lookback) - 1);
+	  t = cd->values[which][len][code >> cd->cdh.lookback];
+
+	  /* step through from base pointer */
+	  b = word + 1;
+	  while (r--)
+	    {
+	      register int copy = *t >> 4;
+	      memcpy (word + copy + 1, t + 1, *t & 0xf);
+	      word[0] = copy + (*t & 0xf);
+	      t += ((*t) & 0xf) + 1;
+	    }
+	}
+      else
+	t = NULL;
+      if (t == cd->escape[which])
+	{
+	  switch (cd->cdh.novel_method)
+	    {
+	    case MG_NOVEL_HUFFMAN_CHARS:
+	      {
+		int len, i;
+		int c;
+		len = buffer.huff_decode(cd->lens_huff[which]->min_code,
+			                 cd->lens_vals[which], &bits);
+		for (i = 0; i < len; i++)
+		  {
+		    c = buffer.huff_decode(cd->chars_huff[which]->min_code,
+					   cd->chars_vals[which], &bits);
+		    *ptr++ = c;
+		  }
+	      }
+	      break;
+	    case MG_NOVEL_BINARY:
+	    case MG_NOVEL_DELTA:
+	    case MG_NOVEL_HYBRID:
+	    case MG_NOVEL_HYBRID_MTF:
+	      {
+		int idx = 0, len;
+		u_char *base;
+		switch (cd->cdh.novel_method)
+		  {
+		  case MG_NOVEL_BINARY:
+		    {
+		      idx = buffer.binary_decode(binary_start[which], &bits);
+		      if (idx == binary_start[which])
+			binary_start[which]++;
+		      idx--;
+		    }
+		    break;
+		  case MG_NOVEL_DELTA:
+		    {
+		      idx = buffer.delta_decode (&bits);
+		      idx--;
+		    }
+		    break;
+		  case MG_NOVEL_HYBRID:
+		    {
+		      int k;
+		      k = buffer.gamma_decode (&bits);
+		      k--;
+		      idx = buffer.binary_decode(ad->blk_end[which][k] -
+				                   ad->blk_start[which][k] + 1, 
+						 &bits);
+		      idx += ad->blk_start[which][k] - 1;
+		    }
+		    break;
+		  case MG_NOVEL_HYBRID_MTF:
+		    {
+		      int k;
+		      k = buffer.gamma_decode (&bits);
+		      k--;
+		      idx = buffer.binary_decode(ad->blk_end[which][k] -
+						   ad->blk_start[which][k] + 1, 
+						 &bits);
+		      idx += ad->blk_start[which][k] - 1;
+		      if (idx >= novels_used[which])
+			{
+			  u_char *temp;
+			  temp = ad->words[which][idx];
+			  ad->words[which][idx] =
+			    ad->words[which][novels_used[which]];
+			  ad->words[which][novels_used[which]] = temp;
+			  swaps[which][novels_used[which]] = idx;
+			  idx = novels_used[which]++;
+			}
+		    }
+		    break;
+		  }
+		base = ad->words[which][idx];
+		len = *base++;
+#ifdef DUMPDOC
+		printf ("[[");
+#endif
+		for (; len; len--)
+		  {
+		    *ptr++ = *base++;
+#ifdef DUMPDOC
+		    putchar (*(base - 1));
+#endif
+		  }
+#ifdef DUMPDOC
+		printf ("]]");
+#endif
+	      }
+	      break;
+	    }
+	}
+      else
+	{
+	  /* copy over the matching prefix */
+	  r = (*t >> 4);
+	  while (r--)
+#ifndef DUMPDOC
+	    *ptr++ = *b++;
+#else
+	    {
+	      *ptr = *b++;
+	      putchar (*ptr);
+	      ptr++;
+	    }
+#endif
+
+	  /* and the stored suffix */
+	  r = ((*t) & 0xf);
+	  while (r--)
+#ifndef DUMPDOC
+	    *ptr++ = *++t;
+#else
+	    {
+	      *ptr = *++t;
+	      putchar (*ptr);
+	      ptr++;
+	    }
+#endif
+	}
+      which = !which;
+    }
+
+  DECODE_DONE
+
+    * l_out = ptr - s_out;
+  num += *l_out + 1;
+
+  if (cd->cdh.novel_method == MG_NOVEL_HYBRID_MTF)
+    for (which = 0; which <= 1; which++)
+      for (novels_used[which]--; novels_used[which] >= 0; novels_used[which]--)
+	{
+	  int a = novels_used[which];
+	  int b = swaps[which][novels_used[which]];
+	  u_char *temp;
+	  temp = ad->words[which][a];
+	  ad->words[which][a] = ad->words[which][b];
+	  ad->words[which][b] = temp;
+	}
+  return (COMPALLOK);
+}
Index: /tags/gsdl-21/gsdl/src/mgpp/text/text_get.h
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/text_get.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/text_get.h	(revision 699)
@@ -0,0 +1,67 @@
+/**************************************************************************
+ *
+ * text_get.h -- Function for reading documents from the compressed text
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef H_TEXT_GET
+#define H_TEXT_GET
+
+
+
+/* FetchDocStart () 
+ * Reads into DocEnt the starting position of the document in the *.text file
+ * Where the first document is document number 1
+ * It returns the true weight of the document.
+ */
+double FetchDocStart (query_data * qd, u_long DN, u_long * seek_pos, u_long * len);
+
+unsigned long FetchInitialParagraph (text_data * td, unsigned long ParaNum);
+
+
+/* FetchCompressed () 
+ * Reads into buffer DocBuff the compressed form of document DocNum. 
+ * Where the first document is document number 1
+ */
+int FetchCompressed (query_data * qd, char **DocBuff, DocEntry * DocEnt);
+
+text_data *LoadTextData (File * text, File * text_idx_wgt, File * text_idx);
+
+void FreeTextData (text_data * td);
+
+int GetPosLens (query_data * qd, DocEntry * Docs, int num);
+
+int LoadBuffers (query_data * qd, DocEntry * Docs, int max_mem, int num);
+
+void FreeBuffers (query_data * qd, DocEntry * Docs, int num);
+
+
+
+compression_dict *LoadCompDict (File * text_comp_dict,
+				File * text_aux_dict,
+				File * text_fast_comp_dict);
+
+void FreeCompDict (compression_dict * cd);
+
+
+int DecodeText (compression_dict * cd,
+		u_char * s_in, int l_in, u_char * s_out, int *l_out);
+
+#endif
Index: /tags/gsdl-21/gsdl/src/mgpp/text/warranty.h
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/warranty.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/warranty.h	(revision 699)
@@ -0,0 +1,24 @@
+static char *warranty_str[] = {
+"			    NO WARRANTY\n",
+"\n",
+"  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY\n",
+"FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN\n",
+"OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES\n",
+"PROVIDE THE PROGRAM \"AS IS\" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED\n",
+"OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF\n",
+"MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS\n",
+"TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE\n",
+"PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,\n",
+"REPAIR OR CORRECTION.\n",
+"\n",
+"  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING\n",
+"WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR\n",
+"REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,\n",
+"INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING\n",
+"OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED\n",
+"TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY\n",
+"YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER\n",
+"PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE\n",
+"POSSIBILITY OF SUCH DAMAGES.\n",
+"\n",
+"		     END OF TERMS AND CONDITIONS\n"};
Index: /tags/gsdl-21/gsdl/src/mgpp/text/warranty.src
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/warranty.src	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/warranty.src	(revision 699)
@@ -0,0 +1,23 @@
+			    NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+		     END OF TERMS AND CONDITIONS
Index: /tags/gsdl-21/gsdl/src/mgpp/text/weights.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/weights.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/weights.cpp	(revision 699)
@@ -0,0 +1,170 @@
+/**************************************************************************
+ *
+ * weights.c -- Functions for reading the weights file in mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#include "sysfuncs.h"
+
+#include "filestats.h"
+#include "memlib.h"
+#include "messages.h"
+#include "timing.h"
+#include "netorder.h"  /* [RPAP - Jan 97: Endian Ordering] */
+
+#include "mg.h"
+#include "invf.h"
+#include "text.h"
+#include "lists.h"
+#include "backend.h"
+#include "weights.h"
+#include "locallib.h"
+#include "mg_errors.h"
+
+#define MAXBITS (sizeof(unsigned long) * 8)
+
+/*
+   $Log$
+   Revision 1.1  1999/10/11 02:58:41  cs025
+   Base install of MG-PP
+
+   Revision 1.1  1999/08/10 21:18:27  sjboddie
+   renamed mg-1.3d directory mg
+
+   Revision 1.1  1998/11/17 09:35:51  rjmcnab
+   *** empty log message ***
+
+   * Revision 1.2  1994/09/20  04:42:18  tes
+   * For version 1.1
+   *
+ */
+
+static char *RCSID = "$Id$";
+
+
+
+approx_weights_data *
+LoadDocWeights (File * weight_file,
+		unsigned long num_of_docs)
+{
+  approx_weights_data *awd;
+  int num;
+
+  if (!(awd = (approx_weights_data *) Xmalloc (sizeof (*awd))))
+    {
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  Fseek (weight_file, sizeof (long), 0);
+  Fread (&awd->bits, sizeof (awd->bits), 1, weight_file);
+  Fread (&awd->L, sizeof (awd->L), 1, weight_file);
+  NTOHD(awd->L);  /* [RPAP - Jan 97: Endian Ordering] */
+  Fread (&awd->B, sizeof (awd->B), 1, weight_file);
+  NTOHD(awd->B);  /* [RPAP - Jan 97: Endian Ordering] */
+
+  awd->mask = awd->bits == 32 ? 0xffffffff : (1 << awd->bits) - 1;
+
+  num = (num_of_docs * awd->bits + 31) / 32;
+  if (!(awd->DocWeights = (long unsigned int *) Xmalloc (sizeof (unsigned long) * num)))
+    {
+      Xfree (awd);
+      mg_errno = MG_NOMEM;
+      return (NULL);
+    }
+
+  Fread (awd->DocWeights, sizeof (unsigned long), num, weight_file);
+  {
+    /* [RPAP - Jan 97: Endian Ordering] */
+    unsigned long i;
+    for (i = 0; i < num; i++)
+      NTOHUL(awd->DocWeights[i]);
+  }
+
+  awd->MemForWeights = num * sizeof (unsigned long);
+  awd->num_of_docs = num_of_docs;
+
+  mg_errno = MG_NOERROR;
+
+  if (awd->bits <= 12)
+    {
+      int i, size = (1 << awd->bits);
+      if (!(awd->table = (float *) Xmalloc (size * sizeof (float))))
+	  return (awd);
+      awd->table[0] = awd->L;
+      for (i = 1; i < size; i++)
+	awd->table[i] = awd->table[i - 1] * awd->B;
+    }
+  else
+    awd->table = NULL;
+  return awd;
+}
+
+
+
+
+
+
+/*
+ * the first document in the collection has DocNum = 0
+ */
+float 
+GetLowerApproxDocWeight (approx_weights_data * awd, register int DocNum)
+{
+  register unsigned long c, Pos;
+  register unsigned long *dw;
+  if (awd == NULL)
+    return 1.0;
+#if 0
+  if (DocNum < 0 || DocNum >= awd->num_of_docs)
+    FatalError (1, "Something is wrong in \"GetDocWeight\" DocNum = %d\n",
+		DocNum);
+#endif
+  Pos = DocNum * awd->bits;
+  dw = &(awd->DocWeights[Pos / MAXBITS]);
+  Pos &= (MAXBITS - 1);
+  c = *dw >> Pos;
+  if (Pos + awd->bits > MAXBITS)
+    c |= *(dw + 1) << (MAXBITS - Pos);
+  c &= awd->mask;
+  if (awd->table)
+    return (awd->table[c]);
+  else
+    return (awd->L * pow (awd->B, (double) c));
+
+}
+
+void 
+FreeWeights (approx_weights_data * awd)
+{
+  Xfree (awd->DocWeights);
+  if (awd->table)
+    Xfree (awd->table);
+  Xfree (awd);
+}
+
+
+
+
+
+
+
+
+
Index: /tags/gsdl-21/gsdl/src/mgpp/text/weights.h
===================================================================
--- /tags/gsdl-21/gsdl/src/mgpp/text/weights.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/mgpp/text/weights.h	(revision 699)
@@ -0,0 +1,34 @@
+/**************************************************************************
+ *
+ * weights.h -- Functions for reading the weights file in mgquery
+ * Copyright (C) 1994  Neil Sharman
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+#ifndef H_WEIGHTS
+#define H_WEIGHTS
+
+approx_weights_data *LoadDocWeights (File * weight_file,
+				     unsigned long num_of_docs);
+
+float GetLowerApproxDocWeight (approx_weights_data * awd, register int DocNum);
+
+void FreeWeights (approx_weights_data * awd);
+
+#endif
Index: /tags/gsdl-21/gsdl/src/recpt/Makefile.in
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/Makefile.in	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/Makefile.in	(revision 699)
@@ -0,0 +1,174 @@
+###########################################################################
+#
+# Makefile -- src/recpt
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999  The New Zealand Digital Library Project
+#
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+AR = ar
+CC = @CC@
+CCFLAGS = @CFLAGS@ -Wall -Wunused -pedantic -W -Woverloaded-virtual
+DEFS = -DNZDL -DQUIET -DSHORT_SUFFIX -DPARADOCNUM -DGSDLSERVER \
+       -D_LITTLE_ENDIAN -DGSDLUSAGELOG @DEFS@
+
+USE_FASTCGI = @USE_FASTCGI@
+ifneq ($(USE_FASTCGI), 1)
+INCLUDES = -I../../lib -I. -I../colservr -I../../packages/mg \
+	   -I../../packages/mg/lib -I../../packages/mg/src/text 
+else 
+INCLUDES = -I../../lib -I. -I../colservr -I../../packages/mg \
+	   -I../../packages/mg/lib -I../../packages/mg/src/text \
+           -I../../packages/fcgi/include
+endif
+
+RANLIB = @RANLIB@
+LDFLAGS = @LDFLAGS@
+LIBS = -lgdbm -lstdc++ -lm -lcrypt
+
+GSDLOS = @gsdlos@
+
+
+COMPILE = $(CC) -c $(CCFLAGS) $(DEFS) $(INCLUDES)
+LINK = $(CC) $(LDFLAGS) -o $@
+
+.SUFFIXES:
+.SUFFIXES: .c .o
+.c.o:
+	$(COMPILE) $<
+
+.SUFFIXES: .cpp .o
+.cpp.o:
+	$(COMPILE) $< 
+
+COMMONHEADERS = receptionist.h cgiwrapper.h cgiargs.h action.h browserclass.h \
+                converter.h recptconfig.h cgiutils.h OIDtools.h browsetools.h htmlgen.h \
+		statusaction.h pageaction.h comtypes.h recptproto.h \
+		pingaction.h queryaction.h documentaction.h htmlutils.h \
+		querytools.h formattools.h tipaction.h authenaction.h infodbclass.h \
+                userdb.h usersaction.h vlistbrowserclass.h hlistbrowserclass.h \
+		datelistbrowserclass.h invbrowserclass.h pagedbrowserclass.h
+
+RECEPTHEADERS = 
+
+LIBRARYHEADERS = nullproto.h
+
+HEADERS = $(COMMONHEADERS) $(RECPTHEADERS) $(LIBRARYHEADERS)
+
+
+COMMONSOURCES = receptionist.cpp cgiwrapper.cpp cgiargs.cpp action.cpp browserclass.cpp \
+                converter.cpp recptconfig.cpp cgiutils.cpp OIDtools.cpp browsetools.cpp \
+		htmlgen.cpp statusaction.cpp pageaction.cpp comtypes.cpp \
+		recptproto.cpp pingaction.cpp queryaction.cpp documentaction.cpp \
+		htmlutils.cpp querytools.cpp formattools.cpp tipaction.cpp authenaction.cpp \
+                infodbclass.cpp userdb.cpp usersaction.cpp vlistbrowserclass.cpp \
+		hlistbrowserclass.cpp datelistbrowserclass.cpp invbrowserclass.cpp \
+		pagedbrowserclass.cpp
+
+RECPTSOURCES = recptmain.cpp
+
+LIBRARYSOURCES = nullproto.cpp librarymain.cpp
+
+SOURCES = $(COMMONSOURCES) $(RECPTSOURCES) $(LIBRARYSOURCES)
+
+
+COMMONOBJECTS = receptionist.o cgiwrapper.o cgiargs.o action.o browserclass.o \
+                converter.o recptconfig.o cgiutils.o OIDtools.o browsetools.o \
+		htmlgen.o statusaction.o pageaction.o comtypes.o \
+		recptproto.o pingaction.o queryaction.o documentaction.o \
+		htmlutils.o querytools.o formattools.o tipaction.o authenaction.o \
+                infodbclass.o userdb.o usersaction.o vlistbrowserclass.o \
+		hlistbrowserclass.o datelistbrowserclass.o invbrowserclass.o \
+		pagedbrowserclass.o
+
+RECPTOBJECTS = recptmain.o
+
+LIBRARYOBJECTS = nullproto.o librarymain.o
+
+OBJECTS = $(COMMONOBJECTS) $(RECPTOBJECTS) $(LIBRARYOBJECTS)
+
+
+COLSERVROBJECTS = ../colservr/mgq.o \
+                  ../colservr/mgsearch.o \
+                  ../colservr/querycache.o \
+                  ../colservr/queryinfo.o \
+                  ../colservr/collectserver.o \
+                  ../colservr/colservrconfig.o \
+                  ../colservr/maptools.o \
+                  ../colservr/filter.o \
+                  ../colservr/source.o \
+                  ../colservr/browsefilter.o \
+                  ../colservr/queryfilter.o \
+                  ../colservr/mggdbmsource.o \
+                  ../colservr/phrasesearch.o \
+                  ../colservr/phrasequeryfilter.o
+
+
+EXEC = recpt library
+
+all: $(EXEC)
+
+clean:
+	rm -f $(OBJECTS) $(EXEC)
+
+install:
+	cp recpt /home/nzdl/nzdl-1.2/cgi-bin/recpt; \
+	cp library /home/nzdl/nzdl-1.2/cgi-bin/library
+#	cp $(EXEC) /home/nzdl/nzdl-1.2/cgi-bin
+
+depend:
+	makedepend -Y -- $(DEFS) $(INCLUDES) $(CCFLAGS) -- $(SOURCES)
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+ifneq ($(USE_FASTCGI), 1)
+RECPT_OBJS = $(COMMONOBJECTS) $(RECPTOBJECTS) \
+    ../../lib/gsdllib.a  \
+    ../../packages/mg/src/text/libtextin.a \
+    ../../packages/mg/lib/libmg.a 
+else
+RECPT_OBJS = $(COMMONOBJECTS) $(RECPTOBJECTS) \
+    ../../lib/gsdllib.a  \
+    ../../packages/mg/src/text/libtextin.a \
+    ../../packages/mg/lib/libmg.a \
+    ../../packages/fcgi/libfcgi/libfcgi.a
+endif
+
+recpt: $(RECPT_OBJS)
+	$(LINK) $(RECPT_OBJS) $(LIBS) 
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+ifneq ($(USE_FASTCGI), 1)
+LIBRARY_OBJS = $(COMMONOBJECTS) $(LIBRARYOBJECTS) $(COLSERVROBJECTS) \
+    ../../lib/gsdllib.a  \
+    ../../packages/mg/src/text/libtextin.a \
+    ../../packages/mg/lib/libmg.a
+else
+LIBRARY_OBJS = $(COMMONOBJECTS) $(LIBRARYOBJECTS) $(COLSERVROBJECTS) \
+    ../../lib/gsdllib.a  \
+    ../../packages/mg/src/text/libtextin.a \
+    ../../packages/mg/lib/libmg.a \
+    ../../packages/fcgi/libfcgi/libfcgi.a
+endif
+
+library: $(LIBRARY_OBJS)
+	$(LINK) $(LIBRARY_OBJS) $(LIBS) 
Index: /tags/gsdl-21/gsdl/src/recpt/OIDtools.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/OIDtools.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/OIDtools.cpp	(revision 699)
@@ -0,0 +1,374 @@
+/**********************************************************************
+ *
+ * OIDtools.cpp -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.17  1999/10/10 08:14:02  sjboddie
+   - metadata now returns mp rather than array
+   - redesigned browsing support (although it's not finished so
+   won't currently work ;-)
+
+   Revision 1.16  1999/09/07 04:56:51  sjboddie
+   added GPL notice
+
+   Revision 1.15  1999/09/02 00:22:04  rjmcnab
+   added extra check in get_info
+
+   Revision 1.14  1999/08/25 04:45:25  sjboddie
+   changed FilterRequest_t::docSet into an array
+
+   Revision 1.13  1999/08/10 22:44:06  sjboddie
+   altered the get_contents function so it's now passed metadata
+   rather than having it hard-coded
+
+   Revision 1.12  1999/07/30 02:26:44  sjboddie
+   added overloaded get_info function for getting info on multiple OIDs
+
+   Revision 1.11  1999/07/20 02:59:03  sjboddie
+   get_children now takes a getParents argument
+
+   Revision 1.10  1999/07/07 05:47:41  sjboddie
+   changed around the way browsetools works
+
+   Revision 1.9  1999/06/16 23:51:53  sjboddie
+   added a strip_suffix function
+
+   Revision 1.8  1999/06/16 03:11:25  sjboddie
+   get_info() now takes a getParents argument
+
+   Revision 1.7  1999/06/16 02:05:23  sjboddie
+   just changed a comment that was confusing me
+
+   Revision 1.6  1999/05/10 03:40:25  sjboddie
+   lots of changes - slowly getting document action sorted out
+
+   Revision 1.5  1999/04/30 01:59:37  sjboddie
+   lots of stuff - getting documentaction working (documentaction replaces
+   old browseaction)
+
+   Revision 1.4  1999/03/29 02:14:25  sjboddie
+
+   More changes to browseaction
+
+   Revision 1.3  1999/03/25 03:13:42  sjboddie
+
+   More library functions for dealing with OIDs. Many of them just
+   return dummy data at present
+
+   Revision 1.2  1999/03/05 03:53:53  sjboddie
+
+   fixed some bugs
+
+   Revision 1.1  1999/03/04 22:38:20  sjboddie
+
+   Added subjectbrowseaction. - Doesn't do anything yet.
+
+ */
+
+#include "OIDtools.h"
+
+
+// returns (in top) the top level of OID (i.e. everything
+// up until the first dot)
+void get_top (const text_t &OID, text_t &top) {
+
+  top.clear();
+  if (OID.empty()) return;
+
+  text_t::const_iterator begin = OID.begin();
+  text_t::const_iterator end = OID.end();
+
+  top.appendrange (begin, findchar(begin, end, '.'));
+}	
+
+
+// checks if OID is top level (i.e. contains no dots)
+bool is_top (const text_t &OID) {
+
+  if (OID.empty()) return true;
+
+  text_t::const_iterator here = OID.begin();
+  text_t::const_iterator end = OID.end();
+  here = findchar (here, end, '.');
+
+  if (here == end) return true;
+  return false;
+}
+
+
+// get_parents_array loads the parents array with all the parents of the
+// document or classification specified by OID (not including OID itself)
+void get_parents_array (const text_t &OID, text_tarray &parents) {
+
+  parents.erase (parents.begin(), parents.end());
+
+  text_t::const_iterator here = OID.begin ();
+  text_t::const_iterator end = OID.end ();
+  text_t thisparent;
+
+  while (here != end) {
+    if (*here == '.') parents.push_back(thisparent);
+    thisparent.push_back(*here);
+    here ++;
+  }
+}
+
+
+// get_info does a protocol call and returns (in response) the metadata 
+// associated with OID. Metadata should be loaded with whatever
+// metadata fields are to be requested.
+
+bool get_info (const text_t &OID, const text_t &collection,
+	       const text_tset &metadata, bool getParents,
+	       recptproto *collectproto, FilterResponse_t &response, 
+	       ostream &logout) {
+
+  response.clear();
+
+  comerror_t err;
+  FilterRequest_t request;
+
+  request.filterName = "NullFilter";
+  request.filterResultOptions = FRmetadata;
+  request.getParents = getParents;
+  request.fields = metadata;
+  request.docSet.push_back (OID);
+  
+  collectproto->filter (collection, request, response, err, logout);
+  if (err != noError) {
+    outconvertclass text_t2ascii;
+    logout << text_t2ascii
+	   << "Error: call to filter failed for " << OID
+	   << " in OIDtools::get_info (" 
+	   << get_comerror_string (err) << ")\n";
+    return false;
+  }
+  
+  return true;
+}
+
+bool get_info (const text_tarray &OIDs, const text_t &collection,
+	       const text_tset &metadata, bool getParents,
+	       recptproto *collectproto, FilterResponse_t &response, 
+	       ostream &logout) {
+
+  response.clear();
+  if (OIDs.empty()) return true;
+
+  comerror_t err;
+  FilterRequest_t request;
+
+  request.filterName = "NullFilter";
+  request.filterResultOptions = FRmetadata;
+  request.getParents = getParents;
+  request.fields = metadata;
+
+  request.docSet = OIDs;
+  
+  collectproto->filter (collection, request, response, err, logout);
+  if (err != noError) {
+    outconvertclass text_t2ascii;
+    logout << text_t2ascii
+	   << "Error: call to filter failed in OIDtools::get_info (" 
+	   << get_comerror_string (err) << ")\n";
+    return false;
+  }
+
+  return true;
+}
+
+// has_children returns true if OID has children
+bool has_children (const text_t &OID, const text_t &collection, 
+		   recptproto *collectproto, ostream &logout) {
+
+  FilterResponse_t response;
+  text_tset metadata;
+  metadata.insert ("haschildren");
+
+  if (get_info (OID, collection, metadata, false, collectproto,	response, logout)) {
+    if (response.docInfo[0].metadata["haschildren"].values[0] == "1")
+      return true;
+  }
+  return false;
+}
+
+
+// get_children does a protocol call and returns (in response) the OIDs and 
+// metadata of all the children of OID. The metadata set should be loaded
+// with whatever metadata fields are to be requested.
+
+bool get_children (const text_t &OID, const text_t &collection, 
+		   const text_tset &metadata, bool getParents,
+		   recptproto *collectproto, FilterResponse_t &response, 
+		   ostream &logout) {
+  
+  response.clear();
+
+  comerror_t err;
+  FilterRequest_t request;
+  OptionValue_t option;
+
+  option.name = "ParentNode";
+  option.value = OID;
+  request.filterOptions.push_back (option);
+  request.filterName = "BrowseFilter";
+  request.filterResultOptions = FROID | FRmetadata;
+  request.fields = metadata;
+  request.getParents = getParents;
+
+  collectproto->filter (collection, request, response, err, logout);
+
+  if (err != noError) {
+    outconvertclass text_t2ascii;
+    logout << text_t2ascii
+	   << "Error: call to filter failed for " << OID 
+	   << " in OIDtools::get_children (" 
+	   << get_comerror_string (err) << ")\n";
+    return false;
+  }
+  return true;
+}
+
+// get_parent returns the parent of the document or classification
+// specified by OID
+text_t get_parent (text_t OID) {
+
+  if (OID.empty() || is_top (OID)) return "";
+
+  text_t::const_iterator begin = OID.begin();
+  text_t::const_iterator here = (OID.end() - 1);
+
+  while (here >= begin) {
+    OID.pop_back();
+    if (*here == '.') break;
+    here --;
+  }
+  return OID;
+}
+
+// takes an OID like ".2 and replaces the " with parent
+void translate_parent (text_t &OID, const text_t &parent) {
+
+  text_t::const_iterator here = OID.begin();
+  text_t::const_iterator end = OID.end();
+  text_t temp;
+
+  while (here != end) {
+    if (*here == '"') temp += parent;
+    else temp.push_back (*here);
+    here ++;
+  }
+  OID = temp;
+}
+
+// shrink_parent does the opposite to translate_parent
+void shrink_parent (text_t &OID) {
+  
+  text_tarray tmp;
+  splitchar (OID.begin(), OID.end(), '.', tmp);
+  OID = "\"." + tmp.back();
+}
+
+// checks if OID uses ".fc", ".lc", ".pr", ".ns",
+// or ".ps" syntax (first child, last child, parent,
+// next sibling, previous sibling)
+bool needs_translating (const text_t &OID) {
+
+  if (OID.size() < 4) return false;
+
+  text_t tail = substr (OID.end()-3, OID.end());
+  if (tail == ".fc" || tail == ".lc" || tail == ".pr" || 
+      tail == ".ns" || tail == ".ps") return true;
+
+  return false;
+}
+
+// strips the ".fc", ".lc", ".pr", ".ns",
+// or ".ps" suffix from the end of OID
+void strip_suffix (text_t &OID) {
+
+  text_t tail = substr (OID.end()-3, OID.end());
+  while (tail == ".fc" || tail == ".lc" || tail == ".pr" || 
+	 tail == ".ns" || tail == ".ps") {
+    OID.erase (OID.end()-3, OID.end());
+    tail = substr (OID.end()-3, OID.end());
+  }
+}
+
+static void recurse_contents (ResultDocInfo_t section, const bool &is_classify, 
+			      const text_t &collection, const text_tset &metadata, 
+			      recptproto *collectproto, FilterResponse_t &response, 
+			      ostream &logout) {
+
+  int haschildren = section.metadata["haschildren"].values[0].getint();
+  const text_t &doctype = section.metadata["doctype"].values[0];
+
+  if ((haschildren == 1) && ((!is_classify) || (doctype == "classify"))) {
+    FilterResponse_t tmp;
+    bool getParents = false;
+    get_children (section.OID, collection, metadata, getParents, collectproto, tmp, logout);
+    ResultDocInfo_tarray::iterator thisdoc = tmp.docInfo.begin();
+    ResultDocInfo_tarray::iterator lastdoc = tmp.docInfo.end();
+    while (thisdoc != lastdoc) {
+      response.docInfo.push_back (*thisdoc);
+      recurse_contents (*thisdoc, is_classify, collection, metadata, 
+			collectproto, response, logout);
+      thisdoc ++;
+    }
+  }
+}
+
+// get_contents returns OIDs and metadata of all contents 
+// below (and including) OID.
+void get_contents (const text_t &topOID, const bool &is_classify, 
+		   text_tset &metadata, const text_t &collection, 
+		   recptproto *collectproto, FilterResponse_t &response, 
+		   ostream &logout) {
+
+  if (topOID.empty()) return;
+  response.clear();
+
+  metadata.insert ("haschildren");
+  metadata.insert ("doctype");
+
+  // get topOIDs info
+  if (get_info (topOID, collection, metadata, false, collectproto, response, logout))
+      recurse_contents (response.docInfo[0], is_classify, collection, 
+			metadata, collectproto, response, logout);
+}
+
+// is_child_of returns true if OID2 is a child of OID1
+bool is_child_of(const text_t &OID1, const text_t &OID2) {
+
+  text_t parent = get_parent(OID2);
+
+  while (!parent.empty()) {
+    if (parent == OID1) return true;
+    parent = get_parent(parent);
+  }
+  return false;
+}
Index: /tags/gsdl-21/gsdl/src/recpt/OIDtools.h
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/OIDtools.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/OIDtools.h	(revision 699)
@@ -0,0 +1,100 @@
+/**********************************************************************
+ *
+ * OIDtools.h -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef OIDTOOLS_H
+#define OIDTOOLS_H
+
+#include "text_t.h"
+#include "recptproto.h"
+
+// returns (in top) the top level of OID (i.e. everything
+// up until the first dot)
+void get_top (const text_t &OID, text_t &top);
+
+// checks if OID is top level (i.e. contains no dots)
+bool is_top (const text_t &OID);
+
+// get_parents_array loads the parents array with all the parents of the
+// document or classification specified by OID (not including OID itself)
+void get_parents_array (const text_t &OID, text_tarray &parents);
+
+// get_info does a protocol call and returns (in response) the info 
+// associated with OID. Metadata should be loaded with whatever
+// metadata fields are to be requested
+bool get_info (const text_t &OID, const text_t &collection,
+	       const text_tset &metadata, bool getParents,
+	       recptproto *collectproto, FilterResponse_t &response, 
+	       ostream &logout);
+bool get_info (const text_tarray &OIDs, const text_t &collection,
+	       const text_tset &metadata, bool getParents,
+	       recptproto *collectproto, FilterResponse_t &response, 
+	       ostream &logout);
+
+// has_children returns true if OID has children
+bool has_children (const text_t &OID, const text_t &collection, 
+		   recptproto *collectproto, ostream &logout);
+
+// get_children does a protocol call and returns (in response) the OIDs and 
+// metadata of all the children of OID. The metadata set should be loaded
+// with whatever metadata fields are to be requested.
+bool get_children (const text_t &OID, const text_t &collection, 
+		   const text_tset &metadata, bool getParents,
+		   recptproto *collectproto, FilterResponse_t &response, 
+		   ostream &logout);
+
+// get_parent returns the parent of the document or classification
+// specified by OID
+text_t get_parent (text_t OID);
+
+// takes an OID like ".2.3 and replaces the " with parent
+void translate_parent (text_t &OID, const text_t &parent);
+
+// shrink_parent does the opposite to translate_parent
+void shrink_parent (text_t &OID);
+
+// checks if OID uses ".fc", ".lc", ".pr", ".ns",
+// or ".ps" syntax (first child, last child, parent,
+// next sibling, previous sibling)
+bool needs_translating (const text_t &OID);
+
+// strips the ".fc", ".lc", ".pr", ".ns",
+// or ".ps" suffix from the end of OID
+void strip_suffix (text_t &OID);
+
+// get_contents returns OIDs and metadata of all contents 
+// below (and including) OID.
+void get_contents (const text_t &topOID, const bool &is_classify, 
+		   text_tset &metadata, const text_t &collection, 
+		   recptproto *collectproto, FilterResponse_t &response, 
+		   ostream &logout);
+
+// is_child_of returns true if OID2 is a child of OID1
+bool is_child_of(const text_t &OID1, const text_t &OID2);
+
+
+#endif
Index: /tags/gsdl-21/gsdl/src/recpt/action.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/action.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/action.cpp	(revision 699)
@@ -0,0 +1,234 @@
+/**********************************************************************
+ *
+ * action.cpp -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.17  1999/09/07 23:04:03  rjmcnab
+   got rid of some compiler warnings
+
+   Revision 1.16  1999/09/07 04:56:51  sjboddie
+   added GPL notice
+
+   Revision 1.15  1999/09/03 09:51:47  rjmcnab
+   removed the argdefault configuration option (it should now be
+   done with cgiarg)
+
+   Revision 1.14  1999/09/02 00:22:42  rjmcnab
+   Changes to get it compiling on AIX
+
+   Revision 1.13  1999/07/30 02:24:43  sjboddie
+   added collectinfo argument to some functions
+
+   Revision 1.12  1999/07/10 22:15:30  rjmcnab
+   Added function check_external_cgiargs so that actions that
+   are not being called can override cgi arguments.
+
+   Revision 1.11  1999/06/08 04:29:41  sjboddie
+   added argsinfo to the call to check_cgiargs to make it easy to set
+   args to their default if they're found to be screwed up
+
+   Revision 1.10  1999/03/25 03:06:44  sjboddie
+
+   altered receptionist slightly so it now passes *collectproto to
+   define_internal_macros and define_external_macros - need it
+   for browseaction
+
+   Revision 1.9  1999/02/28 23:16:00  rjmcnab
+
+   Fixed a compiler warning.
+
+   Revision 1.8  1999/02/28 20:00:11  rjmcnab
+
+
+   Fixed a few things.
+
+   Revision 1.7  1999/02/25 21:58:58  rjmcnab
+
+   Merged sources.
+
+   Revision 1.6  1999/02/21 22:33:52  rjmcnab
+
+   Lots of stuff :-)
+
+   Revision 1.5  1999/02/11 01:24:04  rjmcnab
+
+   Fixed a few compiler warnings.
+
+   Revision 1.4  1999/02/08 01:27:59  rjmcnab
+
+   Got the receptionist producing something using the statusaction.
+
+   Revision 1.3  1999/02/05 10:42:41  rjmcnab
+
+   Continued working on receptionist
+
+   Revision 1.2  1999/02/04 10:00:53  rjmcnab
+
+   Developed the idea of an "action" and having them define the cgi arguments
+   which they need and how those cgi arguments function.
+
+   Revision 1.1  1999/01/08 08:40:52  rjmcnab
+
+   Moved from lib directory.
+
+   Revision 1.1  1999/01/08 03:57:44  rjmcnab
+
+   Initial revision
+
+ */
+
+
+#include "action.h"
+#include <assert.h>
+
+
+// define all the macros which are related to pages generated
+// by this action
+void action::define_internal_macros (const ColInfoResponse_t &/*collectinfo*/, displayclass &/*disp*/, 
+				     cgiargsclass &/*args*/, recptproto * /*collectproto*/, 
+				     ostream &/*logout*/) {
+}
+
+action::action () {
+}
+
+action::~action () {
+}
+
+// configure should be called once for each configuration line
+// the default version does nothing
+void action::configure (const text_t &/*key*/, const text_tarray &/*cfgline*/) {
+}
+
+// init should be called after all the configuration is done but
+// before any other methods are called
+bool action::init (ostream &/*logout*/) {
+  return true;
+}
+
+// returns the "a" argument value that will specify this action
+// this name should be short but does not have to be one character
+// long
+text_t action::get_action_name () {
+  return "nzdl";
+}
+
+// check_cgiargs should be called before get_cgihead_info, 
+// define_external_macros, and do_action. If an error is found
+// a message will be written to logout, if the error is severe
+// then the function will return false and no page content
+// should be produced based on the arguments.
+bool action::check_cgiargs (cgiargsinfoclass &/*argsinfo*/, cgiargsclass &/*args*/, 
+			    ostream &/*logout*/) {
+  return true;
+}
+
+// check_external_cgiargs should be called after check_cgiargs
+// for all actions. It should only be used to override some other
+// normal behaviour, for example, producing a login page when
+// the requested page needs authentication.
+bool action::check_external_cgiargs (cgiargsinfoclass &/*argsinfo*/,
+				     cgiargsclass &/*args*/,
+				     outconvertclass &/*outconvert*/,
+				     const text_t &/*saveconf*/,
+				     ostream &/*logout*/) {
+  return true;
+}
+
+// get_cgihead_info determines the cgi header information for
+// a set of cgi arguments. If response contains location then
+// response_data contains the redirect address. If reponse
+// contains content then reponse_data contains the content-type.
+// Note that images can now be produced by the receptionist.
+void action::get_cgihead_info (cgiargsclass &/*args*/, response_t &response,
+			       text_t &response_data, ostream &/*logout*/) {
+  response = location;
+  response_data = "http://www.nzdl.org";
+}
+  
+// uses_display should return true if the receptionist should return
+// true if the display class is needed to output the page content
+// The default is to return true.
+bool action::uses_display (cgiargsclass &/*args*/) {
+  return true;
+}
+
+
+// define all the macros which might be used by other actions
+// to produce pages. These macros should be well documented.
+void action::define_external_macros (const ColInfoResponse_t &/*collectinfo*/, displayclass &/*disp*/, 
+				     cgiargsclass &/*args*/, recptproto * /*collectproto*/, 
+				     ostream &/*logout*/) {
+}
+
+// returns false if there was an error which prevented the action
+// from outputing anything.
+bool action::do_action (cgiargsclass &/*args*/, const ColInfoResponse_t &/*collectinfo*/, 
+			recptproto * /*collectproto*/, displayclass &/*disp*/, 
+			outconvertclass &/*outconvert*/, ostream &/*textout*/, 
+			ostream &/*logout*/) {
+  return true;
+}
+
+
+bool operator==(const actionptr &x, const actionptr &y) {
+  return (x.a == y.a);
+}
+
+bool operator<(const actionptr &x, const actionptr &y) {
+  return (x.a < y.a);
+}
+
+
+// theaction remains the property of the calling code but
+// should not be deleted until it is removed from this list.
+void actionmapclass::addaction (action *theaction) {
+  // can't add a null action
+  assert (theaction != NULL);
+  if (theaction == NULL) return;
+  
+  // can't add an action with no name
+  assert (!(theaction->get_action_name()).empty());
+  if ((theaction->get_action_name()).empty()) return;
+
+  actionptr aptr;
+  aptr.a = theaction;
+  actionptrs[theaction->get_action_name()] = aptr;
+}
+
+// getaction will return NULL if the action could not be found
+action *actionmapclass::getaction (const text_t &key) {
+  // can't find an action with no name
+  assert (!key.empty());
+  if (key.empty()) return NULL;
+
+  iterator here = actionptrs.find (key);
+  if (here == actionptrs.end()) return NULL;
+  
+  return (*here).second.a;
+}
+
Index: /tags/gsdl-21/gsdl/src/recpt/action.h
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/action.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/action.h	(revision 699)
@@ -0,0 +1,181 @@
+/**********************************************************************
+ *
+ * action.h -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef ACTION_H
+#define ACTION_H
+
+#include "gsdlconf.h"
+#include "text_t.h"
+#include "cgiargs.h"
+#include "display.h"
+#include "recptproto.h"
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\iostream>
+#elif defined(GSDL_USE_IOS_H)
+#  include <iostream.h>
+#else
+#  include <iostream>
+#endif
+
+
+class action {
+protected:
+  cgiargsinfoclass argsinfo;
+
+
+public:
+  action ();
+  virtual ~action ();
+
+  // configure should be called once for each configuration line
+  virtual void configure (const text_t &key, const text_tarray &cfgline);
+
+  // init should be called after all the configuration is done but
+  // before any other methods are called
+  virtual bool init (ostream &logout);
+
+  // returns the "a" argument value that will specify this action
+  // this name should be short but does not have to be one character
+  // long
+  virtual text_t get_action_name ();
+
+  // check_cgiargs should be called before get_cgihead_info, 
+  // define_external_macros, and do_action. If an error is found
+  // a message will be written to logout, if the error is severe
+  // then the function will return false and no page content
+  // should be produced based on the arguments.
+  virtual bool check_cgiargs (cgiargsinfoclass &argsinfo, cgiargsclass &args, 
+			      ostream &logout);
+
+  // check_external_cgiargs should be called after check_cgiargs
+  // for all actions. It should only be used to override some other
+  // normal behaviour, for example, producing a login page when
+  // the requested page needs authentication.
+  virtual bool check_external_cgiargs (cgiargsinfoclass &argsinfo,
+				       cgiargsclass &args,
+				       outconvertclass &outconvert,
+				       const text_t &saveconf,
+				       ostream &logout);
+
+  // get_cgihead_info determines the cgi header information for
+  // a set of cgi arguments. If response contains location then
+  // response_data contains the redirect address. If reponse
+  // contains content then reponse_data contains the content-type.
+  // Note that images can now be produced by the receptionist.
+  virtual void get_cgihead_info (cgiargsclass &args, response_t &response,
+				 text_t &response_data, ostream &logout);
+  
+  // uses_display should return true if the receptionist should return
+  // true if the display class is needed to output the page content.
+  // The default is to return true.
+  virtual bool uses_display (cgiargsclass &args);
+
+  // define all the macros which are related to pages generated
+  // by this action
+  virtual void define_internal_macros (const ColInfoResponse_t &collectinfo, displayclass &disp, 
+				       cgiargsclass &args, recptproto *collectproto, 
+				       ostream &logout);
+
+  // define all the macros which might be used by other actions
+  // to produce pages. These macros should be well documented.
+  virtual void define_external_macros (const ColInfoResponse_t &collectinfo, displayclass &disp, 
+				       cgiargsclass &args, recptproto *collectproto, 
+				       ostream &logout);
+
+  // returns false if there was an error which prevented the action
+  // from outputing anything.
+  virtual bool do_action (cgiargsclass &args, const ColInfoResponse_t &collectinfo, 
+			  recptproto *collectproto, displayclass &disp, 
+			  outconvertclass &outconvert, ostream &textout, 
+			  ostream &logout);
+  
+  // getargsinfo should be called after all configuration files
+  // have been read
+  cgiargsinfoclass getargsinfo () {return argsinfo;};
+};
+
+
+// The actionptr function does not 'own' the action. The 
+// action should be deleted by the code which created it.
+class actionptr {
+public:
+  action *a;
+
+  actionptr () {a=NULL;}
+};
+
+bool operator==(const actionptr &x, const actionptr &y);
+bool operator<(const actionptr &x, const actionptr &y);
+
+typedef map<text_t, actionptr, lttext_t> actionptrmap;
+
+// contains a list of actions indexed by their name
+class actionmapclass {
+protected:
+  actionptrmap actionptrs;
+
+public:
+  // type support for actionptrmap
+  typedef actionptrmap::iterator iterator;
+  typedef actionptrmap::const_iterator const_iterator;
+  typedef actionptrmap::reference reference;
+  typedef actionptrmap::const_reference const_reference;
+  typedef actionptrmap::size_type size_type;
+
+  typedef actionptrmap::difference_type difference_type;
+  typedef actionptrmap::const_reverse_iterator const_reverse_iterator;
+  typedef actionptrmap::reverse_iterator reverse_iterator;
+  
+  // basic container support
+  iterator begin () {return actionptrs.begin();}
+  const_iterator begin () const {return actionptrs.begin();}
+  iterator end () {return actionptrs.end();}
+  const_iterator end () const {return actionptrs.end();}
+
+  void erase(iterator pos) {actionptrs.erase(pos);}
+  void erase(iterator first, iterator last) {actionptrs.erase(first, last);}
+  actionmapclass &operator=(const actionmapclass &x) {actionptrs=x.actionptrs;return *this;}
+
+  bool empty () const {return actionptrs.empty();}
+  size_type size() const {return actionptrs.size();}
+
+
+  // added functionality
+  void clear () {actionptrs.erase(actionptrs.begin(),actionptrs.end());}
+
+  // theaction remains the property of the calling code but
+  // should not be deleted until it is removed from this list.
+  void addaction (action *theaction);
+
+  // getaction will return NULL if the action could not be found
+  action *getaction (const text_t &key);
+};
+
+
+#endif
Index: /tags/gsdl-21/gsdl/src/recpt/authenaction.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/authenaction.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/authenaction.cpp	(revision 699)
@@ -0,0 +1,345 @@
+/**********************************************************************
+ *
+ * authenaction.cpp -- authenticating users
+ * Copyright (C) 1999  DigiLib Systems Limited, New Zealand
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.7  1999/09/07 23:04:29  rjmcnab
+   removed some compiler warnings
+
+   Revision 1.6  1999/09/07 04:56:52  sjboddie
+   added GPL notice
+
+   Revision 1.5  1999/09/02 00:23:24  rjmcnab
+   a couple of minor things
+
+   Revision 1.4  1999/07/30 02:24:43  sjboddie
+   added collectinfo argument to some functions
+
+   Revision 1.3  1999/07/13 23:23:26  rjmcnab
+   Put users in their own gdbm database. Moved a lot of functionality to usersdb
+
+   Revision 1.2  1999/07/11 10:47:32  rjmcnab
+   Got something basic working.
+
+   Revision 1.1  1999/07/10 22:19:29  rjmcnab
+   Initial revision.
+
+
+ */
+
+
+#include "authenaction.h"
+#include "fileutil.h"
+#include "cfgread.h"
+#include "cgiutils.h"
+#include "infodbclass.h"
+#include "gsdltimes.h"
+#include "userdb.h"
+
+
+///////////////
+// authenaction
+///////////////
+
+authenaction::authenaction () {
+  keydecay = 600; // 10 minutes
+  recpt = NULL;
+
+  // this action uses cgi variable "a"
+  cgiarginfo arg_ainfo;
+  arg_ainfo.shortname = "a";
+  arg_ainfo.longname = "action";
+  arg_ainfo.multiplechar = true;
+  arg_ainfo.defaultstatus = cgiarginfo::weak;
+  arg_ainfo.argdefault = "a";
+  arg_ainfo.savedarginfo = cgiarginfo::must;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+
+  // "us"
+  arg_ainfo.shortname = "us";
+  arg_ainfo.longname = "user account status";
+  arg_ainfo.multiplechar = true;
+  arg_ainfo.defaultstatus = cgiarginfo::weak;
+  arg_ainfo.argdefault = "invalid";
+  arg_ainfo.savedarginfo = cgiarginfo::mustnot;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+
+  // "ug"
+  arg_ainfo.shortname = "ug";
+  arg_ainfo.longname = "user groups"; // comma seperated list
+  arg_ainfo.multiplechar = true;
+  arg_ainfo.defaultstatus = cgiarginfo::weak;
+  arg_ainfo.argdefault = "";
+  arg_ainfo.savedarginfo = cgiarginfo::mustnot;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+
+  // "un"
+  arg_ainfo.shortname = "un";
+  arg_ainfo.longname = "user name";
+  arg_ainfo.multiplechar = true;
+  arg_ainfo.defaultstatus = cgiarginfo::weak;
+  arg_ainfo.argdefault = "";
+  arg_ainfo.savedarginfo = cgiarginfo::must;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+
+  // "pw"
+  arg_ainfo.shortname = "pw";
+  arg_ainfo.longname = "password";
+  arg_ainfo.multiplechar = true;
+  arg_ainfo.defaultstatus = cgiarginfo::weak;
+  arg_ainfo.argdefault = "";
+  arg_ainfo.savedarginfo = cgiarginfo::mustnot;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+
+  // "ky" - gives a specific user authentication for a
+  // limited amount of time
+  arg_ainfo.shortname = "ky";
+  arg_ainfo.longname = "user time key";
+  arg_ainfo.multiplechar = true;
+  arg_ainfo.defaultstatus = cgiarginfo::weak;
+  arg_ainfo.argdefault = "";
+  arg_ainfo.savedarginfo = cgiarginfo::must;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+
+  // "ua" - ""=no, "1"=yes 
+  arg_ainfo.shortname = "ua";
+  arg_ainfo.longname = "whether a user has been authenticated";
+  arg_ainfo.multiplechar = true;
+  arg_ainfo.defaultstatus = cgiarginfo::weak;
+  arg_ainfo.argdefault = "";
+  arg_ainfo.savedarginfo = cgiarginfo::mustnot;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+
+  // "er" - compressed arguments for the referer page
+  arg_ainfo.shortname = "er";
+  arg_ainfo.longname = "the compressed args of the refer page";
+  arg_ainfo.multiplechar = true;
+  arg_ainfo.defaultstatus = cgiarginfo::weak;
+  arg_ainfo.argdefault = "";
+  arg_ainfo.savedarginfo = cgiarginfo::mustnot;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+
+  // "uan" - whether user authentication is needed
+  arg_ainfo.shortname = "uan";
+  arg_ainfo.longname = "whether user authentication is needed";
+  arg_ainfo.multiplechar = true;
+  arg_ainfo.defaultstatus = cgiarginfo::weak;
+  arg_ainfo.argdefault = "";
+  arg_ainfo.savedarginfo = cgiarginfo::mustnot;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+}
+
+void authenaction::configure (const text_t &key, const text_tarray &cfgline) {
+  // get the password filename
+  if (cfgline.size() == 1) {
+    if (key == "usersfile") usersfile = cfgline[0];
+    else if (key == "keyfile") keyfile = cfgline[0];
+    else if (key == "keydecay") keydecay = cfgline[0].getint();
+    else if (key == "gsdlhome") {
+      if (usersfile.empty())
+	usersfile = filename_cat (cfgline[0], "etc", "users.db");
+      if (keyfile.empty())
+	keyfile = filename_cat (cfgline[0], "etc", "key.db");
+    }
+  }
+
+  action::configure (key, cfgline);
+}
+
+bool authenaction::init (ostream &logout) {
+  return action::init (logout);
+}
+
+bool authenaction::check_cgiargs (cgiargsinfoclass &/*argsinfo*/, cgiargsclass &/*args*/, 
+				  ostream &/*logout*/) {
+  return true;
+}
+
+// returns false if there is a major problem with the cgi arguments -- not
+// if authentication fails. If the authentication fails "un" will be empty
+bool authenaction::check_external_cgiargs (cgiargsinfoclass &argsinfo,
+					   cgiargsclass &args,
+					   outconvertclass &outconvert,
+					   const text_t &saveconf,
+					   ostream &logout) {
+  // failure means we have to redirect to this action to get authentication
+  // (if we are not already doing this)
+
+  userinfo_t thisuser;
+
+  text_t &args_uan = args["uan"];  text_t &args_un = args["un"];
+  text_t &args_pw = args["pw"];    text_t &args_us = args["us"];
+  text_t &args_ug = args["ug"];    text_t &args_ky = args["ky"];
+  text_t &args_ua = args["ua"];    text_t &args_a = args["a"];
+
+  // we must have a username and a password or a key to
+  // do any authentication
+  args_ua.clear(); // default = false;
+  if (args_un.empty() || args_pw.empty()) args_us = "invalid";
+  else args_us = "failed";
+
+  // make sure we have a username
+  if (!args_un.empty() && get_user_info (usersfile, args_un, thisuser)) {
+    if (!args_pw.empty()) {
+      // we are authenticating using a password
+      if (check_passwd (thisuser, args_pw)) args_ua = "1"; // succeeded
+      
+    } else if (!args_ky.empty()) {
+      // we are authenticating using a key
+      if (check_key (keyfile, thisuser, args_ky, args_ug, keydecay)) args_ua = "1";
+      else args_us = "stalekey";
+    }
+  }
+
+  args_pw.clear(); // password goes no further
+  if (!args_ua.empty()) {
+    if (thisuser.enabled) {
+      bool haspermission = true;
+      // check to make sure the user is in the required group
+      if (!args_ug.empty()) {
+	haspermission = false;
+	text_t::const_iterator group_here = thisuser.groups.begin();
+	text_t::const_iterator group_end = thisuser.groups.end();
+	text_t thisgroup;
+	while (group_here != group_end) {
+	  group_here = getdelimitstr (group_here, group_end, ',', thisgroup);
+	  if (thisgroup == args_ug) {
+	    haspermission = true;
+	    break;
+	  }
+	}
+      }
+
+      if (haspermission) {
+	// succeeded, get info about this user
+	// note: we don't need to set "ug" as it is already set to what it needs to be
+	args_us = "enabled";
+	args_ky = generate_key (keyfile, args_un); // new key
+
+	// delete old keys around every 50 accesses
+	if (rand()%50 == 1) remove_old_keys (keyfile, keydecay);
+	
+      } else {
+	// succeeded, however, the user is not in the correct group
+	args_ua.clear();
+	args_us = "permissiondenied";
+	args_ky.clear();
+      }
+
+    } else {
+      // succeeded, however, the account is disabled
+      args_ua.clear();
+      args_us = "disabled";
+      args_ky.clear();
+    }
+
+  } else {
+    // failure, reset info about the user
+    args_ky.clear();
+  }
+
+  // we will have to redirect the user if authentication is needed,
+  // it failed, and we weren't on our way to be authenticated anyway
+  if ((!args_uan.empty()) && (args_ua.empty()) && (args_a != "a")) {
+    // need to save the current arguments in "er"
+    text_t &arg_er = args["er"];
+    if (!compress_save_args(argsinfo, saveconf, args, arg_er, outconvert, logout))
+      arg_er.clear();
+    
+    // redirect to this action
+    args_a = "a";
+  }
+  
+  return true;
+}
+
+void authenaction::get_cgihead_info (cgiargsclass &/*args*/, response_t &response,
+				     text_t &response_data, ostream &/*logout*/) {
+  response = content;
+  response_data = "text/html";
+}
+
+void authenaction::define_internal_macros (const ColInfoResponse_t &/*collectinfo*/, displayclass &disp, 
+ 					   cgiargsclass &args, recptproto * /*collectproto*/, 
+					   ostream &/*logout*/) {
+  // sets _authen:messageextra_ based on the value of args["us"]
+  //      _authen:hiddenargs_   to contain all the arguments that were
+  //                            explicitly set
+  disp.setmacro ("messagestatus", "authen", ("_authen:message" + args["us"]
+					     + "_"));
+
+  // get a list of saved configuration arguments (if possible)
+  text_t saveconf;
+  text_tset saveconfset;
+  if (recpt != NULL) {
+    saveconf = recpt->get_configinfo().saveconf;
+    splitchar (saveconf.begin(), saveconf.end(), '-', saveconfset);
+  }
+  
+  text_t hiddenargs;
+  cgiargsclass::const_iterator args_here = args.begin();
+  cgiargsclass::const_iterator args_end = args.end();
+  while (args_here != args_end) {
+    // set this as a hidden argument if it came from the cgi arguments,
+    // its not the compressed arguments, the query string, a user name or
+    // password, and if it is not in the compressed arguments
+    if ((*args_here).second.source == cgiarg_t::cgi_arg &&
+	(*args_here).first != "e" && (*args_here).first != "q" &&
+	(*args_here).first != "un" && (*args_here).first != "pw" &&
+	saveconfset.find((*args_here).first) == saveconfset.end()) {
+      hiddenargs += "<input type=hidden name=\"" + (*args_here).first +
+	"\" value=\"_cgiarg" + (*args_here).first + "_\">\n";
+    }
+    
+    args_here++;
+  }
+
+  disp.setmacro ("hiddenargs", "authen", hiddenargs);
+}
+
+void authenaction::define_external_macros (const ColInfoResponse_t &/*collectinfo*/, displayclass &/*disp*/, 
+					   cgiargsclass &/*args*/, recptproto * /*collectproto*/, 
+					   ostream &/*logout*/) {
+}
+
+bool authenaction::do_action (cgiargsclass &args, const ColInfoResponse_t &/*collectinfo*/, 
+			      recptproto * /*collectproto*/, displayclass &disp, 
+			      outconvertclass &outconvert, ostream &textout, 
+			      ostream &/*logout*/) {
+  if (args["us"] == "enabled") {
+    // have been authenticated
+    textout << outconvert << disp
+	    << "_authenok:header_\n_authenok:content_\n_authenok:footer_\n";
+    return true;
+  }
+
+  // need to be authenticated
+  textout << outconvert << disp
+	  << "_authen:header_\n_authen:content_\n_authen:footer_\n";
+
+  return true;
+}
Index: /tags/gsdl-21/gsdl/src/recpt/authenaction.h
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/authenaction.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/authenaction.h	(revision 699)
@@ -0,0 +1,85 @@
+/**********************************************************************
+ *
+ * authenaction.h -- authenticating users
+ * Copyright (C) 1999  DigiLib Systems Limited, New Zealand
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef AUTHENACTION_H
+#define AUTHENACTION_H
+
+#include "gsdlconf.h"
+#include "action.h"
+#include "text_t.h"
+#include "receptionist.h"
+
+
+// note: you need to tell this action about the receptionist
+// using set_receptionist
+
+class authenaction : public action {
+protected:
+  text_t usersfile;
+  text_t keyfile;
+  int keydecay;
+
+  receptionist *recpt;
+  
+public:
+  authenaction ();
+  virtual ~authenaction () {}
+
+  void set_receptionist (receptionist *therecpt) {recpt=therecpt;}
+  
+  void configure (const text_t &key, const text_tarray &cfgline);
+  bool init (ostream &logout);
+  
+  text_t get_action_name () {return "a";}
+  
+  bool check_cgiargs (cgiargsinfoclass &argsinfo, cgiargsclass &args, 
+		      ostream &logout);
+  bool check_external_cgiargs (cgiargsinfoclass &argsinfo,
+			       cgiargsclass &args,
+			       outconvertclass &outconvert,
+			       const text_t &saveconf,
+			       ostream &logout);
+
+  void get_cgihead_info (cgiargsclass &args, response_t &response,
+			 text_t &response_data, ostream &logout);
+  
+  void define_internal_macros (const ColInfoResponse_t &collectinfo, displayclass &disp, 
+			       cgiargsclass &args, recptproto *collectproto, 
+			       ostream &logout);
+  void define_external_macros (const ColInfoResponse_t &collectinfo, displayclass &disp, 
+			       cgiargsclass &args, recptproto *collectproto, 
+			       ostream &logout);
+  
+  bool do_action (cgiargsclass &args, const ColInfoResponse_t &collectinfo, 
+		  recptproto *collectproto, displayclass &disp, 
+		  outconvertclass &outconvert, ostream &textout, 
+		  ostream &logout);
+};
+
+
+#endif
Index: /tags/gsdl-21/gsdl/src/recpt/browserclass.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/browserclass.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/browserclass.cpp	(revision 699)
@@ -0,0 +1,141 @@
+/**********************************************************************
+ *
+ * browserclass.cpp -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.2  1999/10/14 22:59:33  sjboddie
+   finished off browser classes
+
+   Revision 1.1  1999/10/10 08:14:03  sjboddie
+   - metadata now returns mp rather than array
+   - redesigned browsing support (although it's not finished so
+   won't currently work ;-)
+
+ */
+
+
+#include "browserclass.h"
+#include <assert.h>
+
+
+browserclass::browserclass () {
+}
+
+browserclass::~browserclass () {
+}
+
+// configure should be called once for each configuration line
+// the default version does nothing
+void browserclass::configure (const text_t &/*key*/, const text_tarray &/*cfgline*/) {
+}
+
+// init should be called after all the configuration is done but
+// before any other methods are called
+bool browserclass::init (ostream &/*logout*/) {
+  return true;
+}
+
+// returns the name that specifies the browserclass type
+text_t browserclass::get_browser_name () {
+  return "";
+}
+
+void browserclass::processOID (cgiargsclass &/*args*/, recptproto */*collectproto*/, ostream &/*logout*/) {
+}
+
+void browserclass::load_metadata_defaults (text_tset &/*metadata*/) {
+}
+
+text_t browserclass::get_default_formatstring () {
+  return "<td>[link][icon][/link]</td><td>[highlight]{Or}{[Title],Untitled}[/highlight]</td>";
+}
+
+int browserclass::output_section_group (ResultDocInfo_t &/*section*/, cgiargsclass &/*args*/,
+					int /*colnumber*/, format_t */*formatlistptr*/,
+					bool /*use_table*/, text_tset &/*metadata*/, bool &/*getParents*/,
+					recptproto */*collectproto*/, displayclass &/*disp*/, 
+					outconvertclass &/*outconvert*/, ostream &/*textout*/, ostream &/*logout*/) {
+  return 0;
+}
+
+int browserclass::output_section_group (FilterResponse_t &/*sections*/, cgiargsclass &/*args*/,
+					int /*colnumber*/, format_t */*formatlistptr*/, 
+					bool /*use_table*/, text_tset &/*metadata*/, bool &/*getParents*/,
+					recptproto */*collectproto*/, displayclass &/*disp*/, 
+					outconvertclass &/*outconvert*/, ostream &/*textout*/, ostream &/*logout*/) {
+  return 0;
+}
+
+bool operator==(const browserptr &x, const browserptr &y) {
+  return (x.b == y.b);
+}
+
+bool operator<(const browserptr &x, const browserptr &y) {
+  return (x.b < y.b);
+}
+
+// thebrowserclass remains the property of the calling code but
+// should not be deleted until it is removed from this list.
+void browsermapclass::addbrowser (browserclass *thebrowserclass) {
+  // can't add a null browserclass
+  assert (thebrowserclass != NULL);
+  if (thebrowserclass == NULL) return;
+  
+  // can't add a browserclass with no name
+  assert (!(thebrowserclass->get_browser_name()).empty());
+  if ((thebrowserclass->get_browser_name()).empty()) return;
+
+  browserptr aptr;
+  aptr.b = thebrowserclass;
+  browserptrs[thebrowserclass->get_browser_name()] = aptr;
+}
+
+// getbrowser will return the default browser if the browser 
+// could not be found
+browserclass *browsermapclass::getbrowser (const text_t &key) {
+  // can't find a browser with no name
+  if (key.empty()) return get_default_browser();
+
+  iterator here = browserptrs.find (key);
+  if (here == browserptrs.end()) return get_default_browser();
+  
+  return (*here).second.b;
+}
+
+void browsermapclass::setdefaultbrowser (const text_t &browsername) {
+  defaultbrowser = browsername;
+}
+
+browserclass *browsermapclass::get_default_browser () {
+
+  iterator here = browserptrs.find (defaultbrowser);
+  if (here == browserptrs.end())
+    // we'll just take the first in the list if
+    // no valid default was set
+    here = browserptrs.begin();
+  return (*here).second.b;
+}
Index: /tags/gsdl-21/gsdl/src/recpt/browserclass.h
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/browserclass.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/browserclass.h	(revision 699)
@@ -0,0 +1,155 @@
+/**********************************************************************
+ *
+ * browserclass.h -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef BROWSERCLASS_H
+#define BROWSERCLASS_H
+
+#include "gsdlconf.h"
+#include "text_t.h"
+#include "cgiargs.h"
+#include "display.h"
+#include "recptproto.h"
+#include "formattools.h"
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\iostream>
+#elif defined(GSDL_USE_IOS_H)
+#  include <iostream.h>
+#else
+#  include <iostream>
+#endif
+
+
+class browserclass {
+protected:
+
+public:
+  browserclass ();
+  virtual ~browserclass ();
+
+  // configure should be called once for each configuration line
+  virtual void configure (const text_t &key, const text_tarray &cfgline);
+
+  // init should be called after all the configuration is done but
+  // before any other methods are called
+  virtual bool init (ostream &logout);
+
+  // returns the name that specifies the browserclass type
+  virtual text_t get_browser_name ();
+  
+  virtual void processOID (cgiargsclass &args, recptproto *collectproto, ostream &logout);
+  
+  // load_metadata_defaults should be called before
+  // doing protocol call to get metadata for OID to
+  // be displayed by this browserclass. it adds fields
+  // to the metadata set that are needed by browserclass
+  // during display
+  virtual void load_metadata_defaults (text_tset &metadata);
+
+  virtual text_t get_default_formatstring ();
+  
+  // returns the number of tabs to add before displaying the
+  // next section group
+  virtual int output_section_group (ResultDocInfo_t &section, cgiargsclass &args,
+				    int colnumber, format_t *formatlistptr, 
+				    bool use_table, text_tset &metadata, bool &getParents,
+				    recptproto *collectproto, displayclass &disp, 
+				    outconvertclass &outconvert, ostream &textout, ostream &logout);
+
+  virtual int output_section_group (FilterResponse_t &sections, cgiargsclass &args,
+				    int colnumber, format_t *formatlistptr, 
+				    bool use_table, text_tset &metadata, bool &getParents,
+				    recptproto *collectproto, displayclass &disp, 
+				    outconvertclass &outconvert, ostream &textout, 
+				    ostream &logout);
+};
+
+
+// The browserptr function does not 'own' the browser. The 
+// browser should be deleted by the code which created it.
+class browserptr {
+public:
+  browserclass *b;
+
+  browserptr () {b=NULL;}
+};
+
+bool operator==(const browserptr &x, const browserptr &y);
+bool operator<(const browserptr &x, const browserptr &y);
+
+typedef map<text_t, browserptr, lttext_t> browserptrmap;
+
+// contains a list of browsers indexed by their name
+class browsermapclass {
+protected:
+  browserptrmap browserptrs;
+  text_t defaultbrowser;
+
+public:
+  // type support for browserptrmap
+  typedef browserptrmap::iterator iterator;
+  typedef browserptrmap::const_iterator const_iterator;
+  typedef browserptrmap::reference reference;
+  typedef browserptrmap::const_reference const_reference;
+  typedef browserptrmap::size_type size_type;
+
+  typedef browserptrmap::difference_type difference_type;
+  typedef browserptrmap::const_reverse_iterator const_reverse_iterator;
+  typedef browserptrmap::reverse_iterator reverse_iterator;
+  
+  // basic container support
+  iterator begin () {return browserptrs.begin();}
+  const_iterator begin () const {return browserptrs.begin();}
+  iterator end () {return browserptrs.end();}
+  const_iterator end () const {return browserptrs.end();}
+
+  void erase(iterator pos) {browserptrs.erase(pos);}
+  void erase(iterator first, iterator last) {browserptrs.erase(first, last);}
+  browsermapclass &operator=(const browsermapclass &x) {browserptrs=x.browserptrs;return *this;}
+
+  bool empty () const {return browserptrs.empty();}
+  size_type size() const {return browserptrs.size();}
+
+
+  // added functionality
+  void clear () {browserptrs.erase(browserptrs.begin(),browserptrs.end());}
+
+  // thebrowserclass remains the property of the calling code but
+  // should not be deleted until it is removed from this list.
+  void addbrowser (browserclass *thebrowserclass);
+
+  // getbrowser will return NULL if the browser could not be found
+  browserclass *getbrowser (const text_t &key);
+
+  void setdefaultbrowser (const text_t &browsername);
+
+  browserclass *get_default_browser ();
+};
+
+
+#endif
Index: /tags/gsdl-21/gsdl/src/recpt/browsetools.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/browsetools.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/browsetools.cpp	(revision 699)
@@ -0,0 +1,655 @@
+
+/**********************************************************************
+ *
+ * browsetools.cpp -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.27  1999/10/15 03:31:42  sjboddie
+   oops, left out an '&'
+
+   Revision 1.26  1999/10/14 22:58:05  sjboddie
+   finished up on changes to browseing support - may still need some
+   tidying up
+
+   Revision 1.25  1999/10/10 08:14:04  sjboddie
+   - metadata now returns mp rather than array
+   - redesigned browsing support (although it's not finished so
+   won't currently work ;-)
+
+   Revision 1.24  1999/09/28 01:46:55  rjmcnab
+   removed some unused stuff
+
+   Revision 1.23  1999/09/23 10:09:17  sjboddie
+   made some changes so AZLists within other classifications are
+   handled properly
+
+   Revision 1.22  1999/09/07 23:06:58  rjmcnab
+   removed some compiler warnings.
+
+   Revision 1.21  1999/09/07 04:56:52  sjboddie
+   added GPL notice
+
+   Revision 1.20  1999/08/25 04:46:58  sjboddie
+   fixed bug
+
+   Revision 1.19  1999/08/13 04:18:04  sjboddie
+   fixed some typos
+
+   Revision 1.18  1999/08/10 22:42:21  sjboddie
+   added more format options to document tocs - there are now just two
+   types of toc - standard (Hierarchical) and document (as in books)
+
+   Revision 1.17  1999/08/09 02:12:07  sjboddie
+   made it so dates may be only 4 digits (i.e. year only)
+
+   Revision 1.16  1999/07/30 02:16:10  sjboddie
+   -added ability to display nested classifications (expanded versions
+   of nested classifications has yet to be done).
+   -changed set_arrow_macros slightly to fit in with new showtoppage
+   format option
+
+   Revision 1.15  1999/07/21 05:01:56  sjboddie
+   wrote handler for DateList classification
+
+   Revision 1.14  1999/07/20 02:58:15  sjboddie
+   got List and AZList classifications using format strings - tidied
+   up a bit
+
+   Revision 1.13  1999/07/07 05:44:25  sjboddie
+   Made some changes to allow for new way classifiers work (i.e. you can
+   now have classifiers containing other classifiers). At present there's
+   only a special case for dealing with the hdl 'magazine' section. A bit
+   of a redesign is needed to get it completely flexible
+
+   Revision 1.12  1999/07/01 03:47:49  rjmcnab
+   Fixed a small warning.
+
+   Revision 1.11  1999/06/27 21:49:01  sjboddie
+   fixed a couple of version conflicts - tidied up some small things
+
+   Revision 1.10  1999/06/26 01:07:21  rjmcnab
+   Fixed a small "bug" -- well I probably just covered another one...
+
+   Revision 1.9  1999/06/24 05:12:15  sjboddie
+   lots of small changes
+
+   Revision 1.8  1999/06/17 03:06:53  sjboddie
+   got detach button working properly - the close book icon is now disabled
+   when page is detached as the javascript close() function I was using is
+   too unreliable over different browsers
+   note that in my last comment I meant the "cl" arg (not the "c" arg).
+
+   Revision 1.7  1999/06/16 23:53:14  sjboddie
+   tidied a few things up. documentaction::define_external_macros now
+   resets the "c" arg if it's set to something stupid by the .xx suffixes
+
+   Revision 1.6  1999/06/16 04:03:47  sjboddie
+   Now sets "cl" arg to "search" when going to a document from a search
+   results page. This allows the close book icon (in hierarchy toc) to
+   take you back to the results page if that's where you came from.
+   If you got to the document page somehow other than from a
+   classification or a search (i.e. if "cl" isn't set) then the close
+   book icon is disabled
+
+   Revision 1.5  1999/06/16 03:11:25  sjboddie
+   get_info() now takes a getParents argument
+
+   Revision 1.4  1999/05/10 03:40:26  sjboddie
+   lots of changes - slowly getting document action sorted out
+
+   Revision 1.3  1999/04/30 01:59:39  sjboddie
+   lots of stuff - getting documentaction working (documentaction replaces
+   old browseaction)
+
+   Revision 1.2  1999/03/29 02:14:29  sjboddie
+
+   More changes to browseaction
+
+   Revision 1.1  1999/03/25 03:10:15  sjboddie
+
+   new library for browse stuff
+
+ */
+
+#include "browsetools.h"
+#include "OIDtools.h"
+
+
+// simply checks to see if formatstring begins with a <td> tag
+static bool is_table_content (const text_t &formatstring) {
+  text_t::const_iterator here = formatstring.begin();
+  text_t::const_iterator end = formatstring.end();
+  
+  while (here != end) {
+    if (*here != ' ') {
+      if (*here == '<') {
+	if ((*(here+1) == 't' || *(here+1) == 'T') &&
+	    (*(here+2) == 'd' || *(here+2) == 'D') &&
+	    (*(here+3) == '>' || *(here+3) == ' '))
+	  return true;
+      } else return false;
+    }
+    here ++;
+  }
+  return false;
+}
+
+static bool is_table_content (const format_t *formatlistptr) {
+
+  if (formatlistptr == NULL) return false;
+  
+  if (formatlistptr->command == comText)
+    return is_table_content (formatlistptr->text);
+    
+  return false;
+}
+
+// output_controls displays the detach, expand/contract contents,
+// expand/contract text and highlighting/no highlighting buttons
+static void output_controls (cgiargsclass &args, const text_tarray &ibuttons,
+			     recptproto * /*collectproto*/, displayclass &disp, 
+			     outconvertclass &outconvert, ostream &textout, 
+			     ostream &/*logout*/) {
+
+  if (args["u"] != "1") {
+    
+    FilterResponse_t response;
+    text_tarray metadata;
+    text_tarray buttons;
+    
+    text_tarray::const_iterator here = ibuttons.begin();
+    text_tarray::const_iterator end = ibuttons.end();
+    
+    while (here != end) {
+      
+      if (*here == "Detach")
+	buttons.push_back ("_document:imagedetach_");
+      else if (*here == "Highlight") {
+	if (args["hl"] == "1") 
+	  buttons.push_back ("_document:imagenohighlight_");
+	else 
+	  buttons.push_back ("_document:imagehighlight_");
+      } else if (*here == "Expand Contents") {
+	if (args["gc"] == "1")
+	  buttons.push_back ("_document:imagecontracttoc_");
+	else 
+	  buttons.push_back ("_document:imageexpandtoc_");
+      } else if (*here == "Expand Text") {
+	if (args.getintarg("gt"))
+	  buttons.push_back ("_document:imagecontracttext_");
+	else 
+	  buttons.push_back ("_document:imageexpandtext_");
+      }
+      here ++;
+    }
+    
+    here = buttons.begin();
+    end = buttons.end();
+    int count = 0;
+    while (here != end) {
+      if ((count != 0) && ((count % 3) == 0)) textout << "<br>\n";
+      textout << outconvert << disp << *here;
+      count ++;
+      here ++;
+    }
+  }
+}
+
+
+// at the moment this just writes out the html to display 
+// the cover image (assuming it's called cover.jpg)
+// this whole thing should be done with a call to the collection
+// server which would send a link to the cover image if there
+// was one otherwise send title, author and stuff
+static void output_cover_image (cgiargsclass &args, recptproto */*collectproto*/, 
+				displayclass &disp, outconvertclass &outconvert, 
+				ostream &textout, ostream &/*logout*/) {
+  
+  if (args["d"].empty()) return;
+  
+  textout << outconvert << disp << 
+    "<img src=\"_httpcollection_/archives/_thisOID_/cover.jpg\"><br>\n";
+}
+
+static void output_titles (cgiargsclass &args, recptproto *collectproto, 
+			   formatinfo_t &formatinfo, displayclass &disp, 
+			   outconvertclass &outconvert, ostream &textout, 
+			   ostream &logout) {
+
+  if (args["d"].empty()) return;
+
+  text_tset metadata;
+  bool getParents;
+  FilterResponse_t response;
+
+  format_t *formatlistptr = new format_t();
+  parse_formatstring (formatinfo.DocumentHeading, formatlistptr, metadata, getParents);
+
+  text_t topOID;
+  get_top (args["d"], topOID);
+
+  if (!get_info (topOID, args["c"], metadata, getParents, collectproto, response, logout))
+    return;
+
+  textout << outconvert << disp << get_formatted_string (response.docInfo[0], formatlistptr);
+}
+
+
+static void recurse_contents (ResultDocInfo_t &section, cgiargsclass &args,
+			      browserclass *bptr, text_tset &metadata, bool &getParents, 
+			      format_t *formatlistptr, format_tmap &formatlistmap, 
+			      formatinfo_t &formatinfo, browsermapclass *browsermap, 
+			      int colnumber, recptproto *collectproto, displayclass &disp, 
+			      outconvertclass &outconvert, ostream &textout, ostream &logout) {
+  text_t formatstring;
+
+  bool is_classify = false;
+  if (args["d"].empty()) is_classify = true;
+
+  // output this section 
+  bool use_table = is_table_content (formatlistptr);
+  colnumber += bptr->output_section_group (section, args, colnumber, formatlistptr, use_table, 
+					   metadata, getParents, collectproto, disp, outconvert, 
+					   textout, logout);
+
+  text_t classification;
+  if (!is_classify) classification = "Document";
+  else get_top (section.OID, classification);
+
+  int haschildren = section.metadata["haschildren"].values[0].getint();
+  const text_t &doctype = section.metadata["doctype"].values[0];
+  text_t classifytype = section.metadata["classifytype"].values[0];
+  // HLists are displayed as VLists when contents are expanded,
+  // Paged documents are displayed as HLists
+  if (classifytype == "HList") classifytype = "VList";
+  if (classifytype == "Paged") classifytype = "HList";
+
+  // recurse through children
+  if ((haschildren == 1) && ((!is_classify) || (doctype == "classify"))) {
+
+    // get browser for displaying children
+    bptr = browsermap->getbrowser (classifytype);
+    bptr->load_metadata_defaults (metadata);
+
+    // get the formatstring if there is one 
+    if (!get_formatstring (classification, classifytype, 
+			   formatinfo.formatstrings, formatstring))
+      formatstring = bptr->get_default_formatstring();
+
+    format_tmap::const_iterator it = formatlistmap.find (formatstring);
+    // check if formatlistptr is cached
+    if (it != formatlistmap.end()) formatlistptr = (*it).second;
+    else {
+      formatlistptr = new format_t();
+      parse_formatstring (formatstring, formatlistptr, metadata, getParents);
+      formatlistmap[formatstring] = formatlistptr;
+    }
+
+    FilterResponse_t tmp;
+    get_children (section.OID, args["c"], metadata, getParents, collectproto, tmp, logout);
+    ResultDocInfo_tarray::iterator thisdoc = tmp.docInfo.begin();
+    ResultDocInfo_tarray::iterator lastdoc = tmp.docInfo.end();
+
+    while (thisdoc != lastdoc) {
+      recurse_contents (*thisdoc, args, bptr, metadata, getParents, 
+			formatlistptr, formatlistmap, formatinfo, browsermap, 
+			colnumber, collectproto, disp, outconvert, textout, logout);
+      thisdoc ++;
+    }
+  }
+}
+
+
+// expanded_contents recurses through all contents. there's a special case
+// for an HList when contents are expanded (i.e. it's displayed as a VList)
+//
+// if we're inside a document we expand all contents from the top,
+// if we're at classification level we'll just expand out those contents below 
+// the current one
+
+static void expanded_contents (cgiargsclass &args, browsermapclass *browsermap, 
+			       formatinfo_t &formatinfo, recptproto *collectproto, 
+			       displayclass &disp, outconvertclass &outconvert, 
+			       ostream &textout, ostream &logout) {
+  
+  if (args["d"].empty() && args["cl"].empty()) return;
+  text_t OID;
+  
+  FilterResponse_t response;
+  bool getParents = false;
+  text_tset metadata;
+  text_t classifytype, classification, formatstring;
+  int colnumber = 0;
+
+  if (!args["d"].empty()) {
+    // document level - expand from top
+    get_top (args["d"], OID);
+    classification = "Document";
+  } else {
+    // classification level
+    OID = args["cl"];
+    get_top (args["cl"], classification);
+  }
+
+  // get classifytype of this level
+  text_t tOID;
+  if (is_top(OID)) {
+    classifytype = "thistype";
+    tOID = OID;
+  } else {
+    classifytype = "childtype";
+    tOID = get_parent (OID);
+  }
+  metadata.insert (classifytype);
+
+  if (!get_info (tOID, args["c"], metadata, getParents, collectproto, response, logout))
+    return;
+  classifytype = response.docInfo[0].metadata[classifytype].values[0];
+  // if we still don't have a classifytype we'll use the default
+  if (classifytype.empty()) {
+    browserclass *bptr = browsermap->get_default_browser ();
+    classifytype = bptr->get_browser_name ();
+  }
+
+  // HLists are displayed as VLists when contents are expanded,
+  // Books are displayed as HLists
+  if (classifytype == "HList") classifytype = "VList";
+  if (classifytype == "Book") classifytype = "HList";
+
+  metadata.erase (metadata.begin(), metadata.end());
+		  
+  // metadata elements needed by recurse_contents
+  metadata.insert ("classifytype");
+  metadata.insert ("doctype");
+  metadata.insert ("haschildren");
+
+  // load up metadata array with browser defaults
+  browserclass *bptr = browsermap->getbrowser (classifytype);
+  bptr->load_metadata_defaults (metadata);
+
+  // get the formatstring if there is one or use the browsers default
+  if (!get_formatstring (classification, classifytype, 
+			 formatinfo.formatstrings, formatstring))
+    formatstring = bptr->get_default_formatstring();
+
+  format_t *formatlistptr = new format_t();
+  parse_formatstring (formatstring, formatlistptr, metadata, getParents);
+
+  // protocol call
+  if (!get_info (OID, args["c"], metadata, getParents, collectproto, response, logout))
+    return;
+
+  format_tmap formatlistmap;
+  formatlistmap[formatstring] = formatlistptr;
+
+  recurse_contents (response.docInfo[0], args, bptr, metadata, 
+		    getParents, formatlistptr, formatlistmap, formatinfo, browsermap, 
+		    colnumber, collectproto, disp, outconvert, textout, logout);
+
+  // clean up format list pointers
+  format_tmap::const_iterator here = formatlistmap.begin();
+  format_tmap::const_iterator end = formatlistmap.end();
+  while (here != end) {
+    delete (*here).second;
+    here ++;
+  }
+}
+
+
+static void load_formatstring (const text_t &classifytype, text_tset &metadata,
+			       bool &getParents, const text_t &classification, 
+			       browsermapclass *browsermap, formatinfo_t &formatinfo,
+			       format_tmap &formatlistmap) {
+  text_t formatstring;
+
+  // load up metadata array with browser defaults
+  browserclass *bptr = browsermap->getbrowser (classifytype);
+  bptr->load_metadata_defaults (metadata);
+
+  // get the formatstring if there is one or use the browsers default
+  if (!get_formatstring (classification, classifytype, 
+			 formatinfo.formatstrings, formatstring))
+    formatstring = bptr->get_default_formatstring();
+
+  // see if it's cached
+  format_tmap::const_iterator it = formatlistmap.find (formatstring);
+  if (it == formatlistmap.end()) {
+    format_t *formatlistptr = new format_t();
+    parse_formatstring (formatstring, formatlistptr, metadata, getParents);
+    formatlistmap[formatstring] = formatlistptr;
+  }
+}
+
+static void load_formatstrings (FilterResponse_t &response, text_tset &metadata,
+				bool &getParents, const text_t &classification,
+				browsermapclass *browsermap, formatinfo_t &formatinfo, 
+				format_tmap &formatlistmap) {
+
+  text_tset cache;
+
+  ResultDocInfo_tarray::iterator thisdoc = response.docInfo.begin();
+  ResultDocInfo_tarray::iterator lastdoc = response.docInfo.end();
+
+  while (thisdoc != lastdoc) {
+
+    if (is_top ((*thisdoc).OID))
+      load_formatstring ((*thisdoc).metadata["thistype"].values[0], metadata, 
+			 getParents, classification, browsermap, formatinfo, 
+			 formatlistmap);
+
+    text_t &childtype = (*thisdoc).metadata["childtype"].values[0];
+    text_tset::const_iterator it = cache.find (childtype);
+    if (it == cache.end()) {
+      load_formatstring (childtype, metadata, getParents, classification,
+			 browsermap, formatinfo, formatlistmap);
+      cache.insert (childtype);
+    }
+    thisdoc ++;
+  }
+}
+
+static void output_parents (FilterResponse_t &response, cgiargsclass &args, 
+			    browsermapclass *browsermap, formatinfo_t &formatinfo, 
+			    format_tmap &formatlistmap, const text_t &classification,
+			    int &colnumber, text_tset &metadata, bool &getParents,
+			    recptproto *collectproto, displayclass &disp, 
+			    outconvertclass &outconvert, ostream &textout, 
+			    ostream &logout) {
+
+  format_t *formatlistptr = NULL;
+  text_t classifytype, formatstring;
+  bool use_table, first = true;
+  ResultDocInfo_tarray::iterator thisparent = response.docInfo.begin();
+  ResultDocInfo_tarray::iterator lastparent = response.docInfo.end();
+  while (thisparent != lastparent) {
+
+    // get classifytype of this level
+    if (is_top ((*thisparent).OID)) classifytype = (*thisparent).metadata["thistype"].values[0];
+    else if (!first) classifytype = (*(thisparent-1)).metadata["childtype"].values[0];
+    
+    // if we still don't have a classifytype we'll use the default
+    if (classifytype.empty()) {
+      browserclass *bptr = browsermap->get_default_browser ();
+      classifytype = bptr->get_browser_name ();
+    }
+    
+    browserclass *bptr = browsermap->getbrowser (classifytype);
+
+    // get the formatstring if there is one or use the browsers default
+    if (!get_formatstring (classification, classifytype, 
+			   formatinfo.formatstrings, formatstring))
+      formatstring = bptr->get_default_formatstring();
+    
+    // see if it's cached
+    format_tmap::const_iterator it = formatlistmap.find (formatstring);
+    if (it != formatlistmap.end()) formatlistptr = (*it).second;
+    else {
+      logout << "browsetools error\n";
+      return;
+    }
+    
+    use_table = is_table_content (formatlistptr);
+    colnumber += bptr->output_section_group (*thisparent, args, colnumber, formatlistptr, use_table, 
+					     metadata, getParents, collectproto, disp, outconvert, 
+					     textout, logout);
+    first = false;
+    thisparent ++;
+  }
+}
+
+
+static void contracted_contents (cgiargsclass &args, browsermapclass *browsermap,
+				 formatinfo_t &formatinfo, recptproto *collectproto,
+				 displayclass &disp, outconvertclass &outconvert,
+				 ostream &textout, ostream &logout) {
+  FilterResponse_t response;
+  text_tset metadata;
+  bool getParents = false;
+  text_t formatstring;
+  text_tarray parents;
+  text_t OID = args["d"];
+  text_t classification = "Document";
+  if (OID.empty()) {
+    OID = args["cl"];
+    get_top (OID, classification);
+  } 
+  int colnumber = 0;
+
+  bool haschildren = has_children (OID, args["c"], collectproto, logout);
+
+  if (haschildren) get_parents_array (OID + ".fc", parents);
+  else get_parents_array (OID, parents);
+
+  if (!parents.empty()) {
+    // get classifytypes of each parent
+    metadata.insert ("thistype");
+    metadata.insert ("childtype");
+    
+    if (!get_info (parents, args["c"], metadata, getParents, collectproto, response, logout))
+      return;
+    
+    // get formatstrings for all parents
+    format_tmap formatlistmap;  
+    load_formatstrings (response, metadata, getParents, classification, 
+			browsermap, formatinfo, formatlistmap);
+
+    if (!get_info (parents, args["c"], metadata, getParents, collectproto, response, logout))
+      return;
+    
+    // display each parent
+    output_parents (response, args, browsermap, formatinfo, formatlistmap, 
+		    classification, colnumber, metadata, getParents, 
+		    collectproto, disp, outconvert, textout, logout);
+    
+    metadata.erase (metadata.begin(), metadata.end());
+    
+    // clean up cached format list pointers
+    format_tmap::const_iterator here = formatlistmap.begin();
+    format_tmap::const_iterator end = formatlistmap.end();
+    while (here != end) {
+      delete (*here).second;
+      here ++;
+    }
+  }
+
+  // get childrens classifytype
+  text_t classifytype;
+  int numparents = response.docInfo.size();
+  if (!parents.empty())
+    classifytype = response.docInfo[numparents-1].metadata["childtype"].values[0];
+  else {
+    // use the default
+    browserclass *bptr = browsermap->get_default_browser ();
+    classifytype = bptr->get_browser_name ();
+  }
+    
+  // load up metadata array with browser defaults
+  browserclass *bptr = browsermap->getbrowser (classifytype);
+  bptr->load_metadata_defaults (metadata);
+
+  // get the formatstring if there is one or use the browsers default
+  if (!get_formatstring (classification, classifytype, 
+			 formatinfo.formatstrings, formatstring))
+    formatstring = bptr->get_default_formatstring();
+
+  format_t *formatlistptr = new format_t();
+  parse_formatstring (formatstring, formatlistptr, metadata, getParents);
+
+  if (haschildren) 
+    get_children (OID, args["c"], metadata, getParents,
+		  collectproto, response, logout);
+  else 
+    get_children (OID + ".pr", args["c"], metadata, getParents, 
+		  collectproto, response, logout);
+
+  // display children
+  bool use_table = is_table_content (formatlistptr);
+  bptr->output_section_group (response, args, colnumber, formatlistptr, use_table, 
+			      metadata, getParents, collectproto, disp, outconvert, 
+			      textout, logout);
+  
+  delete formatlistptr;
+}
+
+void output_toc (cgiargsclass &args, browsermapclass *browsermap,
+                 formatinfo_t &formatinfo, recptproto *collectproto,
+                 displayclass &disp, outconvertclass &outconvert,
+                 ostream &textout, ostream &logout) {
+
+  bool havecontrols = false;
+  
+  // get the cover image (if there is one) and the control buttons
+  // if we're inside a book
+  if (!args["d"].empty()) {
+    textout << "<p><table cellpadding=0 cellspacing=0><tr>\n";
+    textout << "<td valign=top width=200>\n";
+    if (formatinfo.DocumentImages) 
+      output_cover_image (args, collectproto, disp, outconvert, textout, logout);
+    else if (formatinfo.DocumentTitles)
+      output_titles (args, collectproto, formatinfo, disp, outconvert, textout, logout);
+    output_controls (args, formatinfo.DocumentButtons, collectproto, disp, 
+		     outconvert, textout, logout);
+    textout << "</td><td valign=top>\n";
+    havecontrols = true;
+  }
+
+  if (formatinfo.DocumentContents) {
+    if (args.getintarg("gc") == 1) {
+      
+      // expanded table of contents
+      expanded_contents (args, browsermap, formatinfo, collectproto, 
+			 disp, outconvert, textout, logout);
+    } else {
+      
+      // contracted table of contents
+      contracted_contents (args, browsermap, formatinfo, collectproto, 
+			   disp, outconvert, textout, logout);
+    }
+  }
+
+  if (havecontrols) textout << "</td></tr></table>\n";
+}
Index: /tags/gsdl-21/gsdl/src/recpt/browsetools.h
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/browsetools.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/browsetools.h	(revision 699)
@@ -0,0 +1,47 @@
+/**********************************************************************
+ *
+ * browsetools.h --
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef BROWSETOOLS_H
+#define BROWSETOOLS_H
+
+#include "gsdlconf.h"
+#include "text_t.h"
+#include "cgiargs.h"
+#include "display.h"
+#include "recptproto.h"
+#include "formattools.h"
+#include "browserclass.h"
+
+typedef map<text_t, format_t*, lttext_t> format_tmap; 
+
+void output_toc (cgiargsclass &args, browsermapclass *browsermap,
+                 formatinfo_t &formatinfo, recptproto *collectproto,
+                 displayclass &disp, outconvertclass &outconvert,
+                 ostream &textout, ostream &logout);
+
+#endif
Index: /tags/gsdl-21/gsdl/src/recpt/cgiargs.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/cgiargs.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/cgiargs.cpp	(revision 699)
@@ -0,0 +1,307 @@
+/**********************************************************************
+ *
+ * cgiargs.cpp -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.9  1999/09/07 04:56:53  sjboddie
+   added GPL notice
+
+   Revision 1.8  1999/09/03 09:52:45  rjmcnab
+   Fixed a couple of small things
+
+   Revision 1.7  1999/09/02 00:23:55  rjmcnab
+   changes to get compiling on AIX
+
+   Revision 1.6  1999/07/11 01:02:13  rjmcnab
+   Stored information relating to the cgi argument's origin with the cgi argument.
+
+   Revision 1.5  1999/02/08 01:28:00  rjmcnab
+
+   Got the receptionist producing something using the statusaction.
+
+   Revision 1.4  1999/02/05 10:42:41  rjmcnab
+
+   Continued working on receptionist
+
+   Revision 1.3  1999/02/04 10:00:54  rjmcnab
+
+   Developed the idea of an "action" and having them define the cgi arguments
+   which they need and how those cgi arguments function.
+
+   Revision 1.2  1999/01/12 01:51:06  rjmcnab
+
+   Standard header.
+
+   Revision 1.1  1999/01/08 08:40:54  rjmcnab
+
+   Moved from lib directory.
+
+   Revision 1.1  1999/01/08 07:50:30  rjmcnab
+
+   Moved from src/library directory to lib directory.
+
+ */
+
+
+#include "cgiargs.h"
+#include "gsdlunicode.h"
+
+
+
+void cgiarg_t::clear () {
+  value.clear();
+  source = program_arg;
+}
+
+cgiarg_t &cgiarg_t::operator=(const cgiarg_t &x) {
+  value=x.value;
+  source=x.source;
+  return *this;
+}
+
+
+bool operator==(const cgiarg_t &x, const cgiarg_t &y) {
+  return ((x.value == y.value) && (x.source == y.source));
+}
+
+bool operator<(const cgiarg_t &x, const cgiarg_t &y) {
+  return ((x.value < y.value) ||
+	  ((x.value == y.value) && (x.source == y.source)));
+}
+
+
+
+
+// constructors
+cgiargsclass::cgiargsclass () {
+}
+
+
+// this returns NULL if there isn't an entry with a value of
+// 'key' already defined
+
+void cgiargsclass::setarg (const text_t &key, const text_t &value,
+			   cgiarg_t::source_t source) {
+  cgiarg_t temparg;
+  temparg.value = value;
+  temparg.source = source;
+  args[key] = temparg;
+}
+
+void cgiargsclass::setdefaultarg (const text_t &key, const text_t &value,
+				  cgiarg_t::source_t source) {
+  if (getarg(key) == NULL) setarg (key, value, source);
+}
+
+void cgiargsclass::setintarg (const text_t &key, int value,
+			      cgiarg_t::source_t source) {
+  setarg (key, value, source);
+}
+
+void cgiargsclass::setdefaultintarg (const text_t &key, int value,
+				     cgiarg_t::source_t source) {
+  if (getarg(key) == NULL) setintarg (key, value, source);
+}
+
+void cgiargsclass::setcarg (const text_t &key, unsigned short c,
+			    cgiarg_t::source_t source) {
+  text_t t; 
+  t.push_back(c);
+  setarg(key,t, source);
+}
+
+void cgiargsclass::setdefaultcarg (const text_t &key, unsigned short c,
+				   cgiarg_t::source_t source) {
+  if (getarg(key) == NULL) setcarg (key, c, source);
+}
+
+text_t *cgiargsclass::getarg (const text_t &key) {
+  iterator here = args.find (key);
+  if (here == args.end()) return NULL;
+
+  return &((*here).second.value);
+}
+
+int cgiargsclass::getintarg (const text_t &key) {
+  text_t *text = getarg (key);
+  if (text == NULL) return 0;
+  return text->getint();
+}
+
+
+
+// stream operators to print cgi arguments for debugging purposes
+ostream &operator<<(ostream &outs, const cgiargsclass &args) {
+  utf8outconvertclass text_t2utf8;
+  cgiargsclass::const_iterator here = args.begin ();
+  cgiargsclass::const_iterator end = args.end ();
+
+  outs << "*** cgiargsclass\n";
+  
+  while (here != end) {
+    outs << text_t2utf8 << " \"" << (*here).first << "\"=\"" << 
+      (*here).second.value << "\"\n";
+    here++;
+  }
+  outs << "\n";
+
+  return outs;
+}
+
+
+
+cgiarginfo::cgiarginfo () {
+  multiplechar = false;
+  defaultstatus = weak;
+  savedarginfo = can;
+}
+
+
+bool operator==(const cgiarginfo &x, const cgiarginfo &y) {
+  return ((x.shortname == y.shortname) &&
+	  (x.longname == y.longname) &&
+	  (x.multiplechar == y.multiplechar) &&
+	  (x.defaultstatus == y.defaultstatus) &&
+	  (x.argdefault == y.argdefault) &&
+	  (x.savedarginfo == y.savedarginfo));
+}
+
+bool operator<(const cgiarginfo &x, const cgiarginfo &y) {
+  return ((x.shortname < y.shortname) ||
+	  ((x.shortname == y.shortname) &&
+	   ((x.longname < y.longname) ||
+	    ((x.longname == y.longname) &&
+	     ((x.multiplechar < y.multiplechar) ||
+	      ((x.multiplechar == y.multiplechar) &&
+	       ((x.defaultstatus < y.defaultstatus) ||
+		((x.defaultstatus == y.defaultstatus) &&
+		 ((x.argdefault < y.argdefault) ||
+		  ((x.argdefault == y.argdefault) &&
+		   ((x.savedarginfo < y.savedarginfo))))))))))));
+}
+
+
+// constructor
+cgiargsinfoclass::cgiargsinfoclass () {
+}
+
+// addarginfo will combine the information with the present
+// information. If name clashes were detected then the information
+// will be written to logout and addarginfo will return false. No
+// processing with the arguments should be done if this happens
+// as the results will be meaningless.
+bool cgiargsinfoclass::addarginfo (ostream *logout, const cgiarginfo &info) {
+  outconvertclass text_t2ascii;
+
+  cgiarginfo *orginfo = getarginfo (info.shortname);
+  if (orginfo == NULL) {
+    argsinfo[info.shortname] = info;
+    return true; // no clashes
+  }
+
+  if (orginfo->longname != info.longname) {
+    if (logout != NULL) {
+      (*logout) << text_t2ascii << "Error: cgi argument name clash for argument \""
+		<< info.shortname << "\".\nOne long name was\n  \"" << orginfo->longname 
+		<< "\"\nand the other was\n  \"" << info.longname << "\".\n\n";
+    }
+    return false; // found a clash
+  }
+
+  if (orginfo->multiplechar != info.multiplechar) {
+    if (logout != NULL) {
+      (*logout) << text_t2ascii << "Error: cgi argument \"" << info.shortname 
+		<< "\" was given as being a single character option\n"
+		<< "and a multiple character option.\n\n";
+    }
+    return false; // found a clash
+  }
+  
+  if (!info.multiplechar && info.argdefault.size() > 1) {
+    if (logout != NULL) {
+      (*logout) << text_t2ascii << "Error: cgi argument \"" << info.shortname
+		<< "\" was defined as being a single character option\n"
+		<< "but a multiple character default was given.\n\n";
+    }
+    return false;  // found a problem
+  }
+
+  // make sure there is no clashes in the savedarginfo
+  if ((orginfo->savedarginfo==cgiarginfo::mustnot && 
+       info.savedarginfo==cgiarginfo::must) ||
+      (orginfo->savedarginfo==cgiarginfo::must && 
+       info.savedarginfo==cgiarginfo::mustnot)) {
+    if (logout != NULL) {
+      (*logout) << text_t2ascii << "Error: it was specified that cgi argument \"" 
+		<< info.shortname << "\" should be saved in the state\n"
+		<< "information and that it should not be save in the state information.\n\n";
+    }
+    return false; // found a clash
+  }
+  // the only time orginfo->savedarginfo can change is when it is set
+  // to "can"
+  if (orginfo->savedarginfo == cgiarginfo::can) 
+    orginfo->savedarginfo = info.savedarginfo;
+
+
+  if (orginfo->defaultstatus > info.defaultstatus) {
+    return true;
+  }
+  orginfo->defaultstatus = info.defaultstatus;
+  orginfo->argdefault = info.argdefault;
+
+  return true;
+}
+
+bool cgiargsinfoclass::addarginfo (ostream *logout, const cgiargsinfoclass &info) {
+  const_iterator here = info.begin ();
+  const_iterator end = info.end ();
+
+  while (here != end) {
+    if (!addarginfo (logout, (*here).second)) return false;
+    here++;
+  }
+  
+  return true; // made it, no clashes
+}
+
+cgiarginfo *cgiargsinfoclass::getarginfo (const text_t &key) {
+  iterator here = argsinfo.find (key);
+  if (here == argsinfo.end()) return NULL;
+
+  return &((*here).second);
+}
+
+const cgiarginfo *cgiargsinfoclass::getarginfo (const text_t &key) const {
+  const_iterator here = argsinfo.find (key);
+  if (here == argsinfo.end()) return NULL;
+
+  return &((*here).second);
+}
+
+
+
Index: /tags/gsdl-21/gsdl/src/recpt/cgiargs.h
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/cgiargs.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/cgiargs.h	(revision 699)
@@ -0,0 +1,230 @@
+/**********************************************************************
+ *
+ * cgiargs.h -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef CGIARGS_H
+#define CGIARGS_H
+
+#include "gsdlconf.h"
+#include "text_t.h"
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\iostream>
+#elif defined(GSDL_USE_IOS_H)
+#  include <iostream.h>
+#else
+#  include <iostream>
+#endif
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\map>
+#elif defined(GSDL_USE_STL_H)
+#  include <map.h>
+#else
+#  include <map>
+#endif
+
+
+// response_t is used to define what type of cgi header 
+// should be produced
+enum response_t {location, content};
+
+
+struct cgiarg_t {
+  void clear ();
+  cgiarg_t () {clear();}
+  cgiarg_t &operator=(const cgiarg_t &x);
+  
+  enum source_t {default_arg, compressed_arg, cgi_arg, program_arg};
+  
+  text_t value;
+  source_t source;
+};
+
+bool operator==(const cgiarg_t &x, const cgiarg_t &y);
+bool operator<(const cgiarg_t &x, const cgiarg_t &y);
+
+typedef map<text_t, cgiarg_t, lttext_t> cgiarg_tmap;
+
+// cgiargsclass is used to store a particular set
+// of cgi arguments.
+class cgiargsclass {
+protected:
+  cgiarg_tmap args;
+
+public:
+  //type support for cgiarg_tmap
+  typedef cgiarg_tmap::iterator iterator;
+  typedef cgiarg_tmap::const_iterator const_iterator;
+  typedef cgiarg_tmap::reference reference;
+  typedef cgiarg_tmap::const_reference const_reference;
+  typedef cgiarg_tmap::size_type size_type;
+  typedef cgiarg_tmap::difference_type difference_type;
+  typedef cgiarg_tmap::const_reverse_iterator const_reverse_iterator;
+  typedef cgiarg_tmap::reverse_iterator reverse_iterator;
+  
+  // constructors
+  cgiargsclass ();
+
+  // basic container support
+  iterator begin () {return args.begin();}
+  const_iterator begin () const {return args.begin();}
+  iterator end () {return args.end();}
+  const_iterator end () const {return args.end();}
+
+  void erase(iterator pos) {args.erase(pos);}
+  void erase(iterator first, iterator last) {args.erase(first, last);}
+  cgiargsclass &operator=(const cgiargsclass &x) {args=x.args;return *this;}
+
+  bool empty () const {return args.empty();}
+  size_type size() const {return args.size();}
+
+
+  // added functionality
+  void clear () {args.erase(args.begin(),args.end());}
+
+  // setdefaultarg, setdefaultintarg, and setdefaultcarg will 
+  // only set the argument if it is not already set
+  // getargs returns NULL if there isn't an entry with
+  // 'key' already defined, getintarg returns 0 if there wasn't an
+  // entry with 'key' defined and operator[] returns "" if
+  // 'key' wasn't already defined (and sets 'key'="").
+  void setarg (const text_t &key, const text_t &value,
+	       cgiarg_t::source_t source=cgiarg_t::program_arg);
+  void setdefaultarg (const text_t &key, const text_t &value,
+		      cgiarg_t::source_t source=cgiarg_t::default_arg);
+  void setintarg (const text_t &key, int value,
+		  cgiarg_t::source_t source=cgiarg_t::program_arg);
+  void setdefaultintarg (const text_t &key, int value,
+			 cgiarg_t::source_t source=cgiarg_t::default_arg);
+  void setcarg (const text_t &key, unsigned short c,
+		cgiarg_t::source_t source=cgiarg_t::program_arg);
+  void setdefaultcarg (const text_t &key, unsigned short c,
+		       cgiarg_t::source_t source=cgiarg_t::default_arg);
+  text_t *getarg (const text_t &key);
+  int getintarg (const text_t &key);
+  text_t &operator[] (const text_t &key) {return args[key].value;}
+  cgiarg_t &lookupcgiarg (const text_t &key) {return args[key];}
+};
+
+// stream operators to print cgi arguments for debugging purposes
+ostream &operator<<(ostream &outs, const cgiargsclass &args);
+
+
+
+// cgiarginfo holds information relating to a cgi argument
+class cgiarginfo {
+public:
+  cgiarginfo ();
+
+  // shortname is the name used in cgi arguments
+  text_t shortname;
+
+  // longname is the name used when giving information 
+  // about the argument and to spot name duplication
+  text_t longname;
+
+  // multiplechar should be set to true if the value can be
+  // more than one character long
+  bool multiplechar;
+
+  // defaultstatus_t indicates how good the default is when different
+  // defaults are given for one argument. "none" means there is no default
+  // (a default might be given elsewhere). "weak" is a basic guess that
+  // doesn't hold much weight, "good" is a better guess, "config" was a default
+  // given in a configuration file, and "imperative" means it must not
+  // be overriden at any costs (unless there is another "imperative"...)
+  enum defaultstatus_t {none=0, weak=1, good=2, config=3, imperative=4};
+
+  defaultstatus_t defaultstatus;
+  text_t argdefault;   // a default value
+
+  // savedarginfo_t indicates whether the argument should be saved
+  // between pages (e.g. using the compressed argument) or not. The
+  // value of an argument can change from "can" to "mustnot" and 
+  // "can" to "must" but not "mustnot" to "must" or "must" to "mustnot".
+  enum savedarginfo_t {mustnot, can, must};
+
+  savedarginfo_t savedarginfo;
+};
+
+bool operator==(const cgiarginfo &x, const cgiarginfo &y);
+bool operator<(const cgiarginfo &x, const cgiarginfo &y);
+
+typedef map<text_t, cgiarginfo, lttext_t> argsinfomap;
+
+// contains a list of cgi argument information
+class cgiargsinfoclass {
+protected:
+  argsinfomap argsinfo;
+
+public:
+  // type support for arginfomap
+  typedef argsinfomap::iterator iterator;
+  typedef argsinfomap::const_iterator const_iterator;
+  typedef argsinfomap::reference reference;
+  typedef argsinfomap::const_reference const_reference;
+  typedef argsinfomap::size_type size_type;
+  typedef argsinfomap::difference_type difference_type;
+  typedef argsinfomap::const_reverse_iterator const_reverse_iterator;
+  typedef argsinfomap::reverse_iterator reverse_iterator;
+  
+  // constructors
+  cgiargsinfoclass ();
+
+  // basic container support
+  iterator begin () {return argsinfo.begin();}
+  const_iterator begin () const {return argsinfo.begin();}
+  iterator end () {return argsinfo.end();}
+  const_iterator end () const {return argsinfo.end();}
+
+  void erase(iterator pos) {argsinfo.erase(pos);}
+  void erase(iterator first, iterator last) {argsinfo.erase(first, last);}
+  cgiargsinfoclass &operator=(const cgiargsinfoclass &x) {argsinfo=x.argsinfo;return *this;}
+
+  bool empty () const {return argsinfo.empty();}
+  size_type size() const {return argsinfo.size();}
+
+
+  // added functionality
+  void clear () {argsinfo.erase(argsinfo.begin(),argsinfo.end());}
+
+  // addarginfo will combine the information with the present
+  // information. If name clashes were detected then the information
+  // will be written to logout and addarginfo will return false. No
+  // processing with the arguments should be done if this happens
+  // as the results will be meaningless.
+  bool addarginfo (ostream *logout, const cgiarginfo &info);
+  bool addarginfo (ostream *logout, const cgiargsinfoclass &info);
+
+  cgiarginfo *getarginfo (const text_t &key);
+  const cgiarginfo *getarginfo (const text_t &key) const;
+  cgiarginfo &operator[] (const text_t &key) {return argsinfo[key];}
+};
+
+
+#endif
Index: /tags/gsdl-21/gsdl/src/recpt/cgiutils.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/cgiutils.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/cgiutils.cpp	(revision 699)
@@ -0,0 +1,486 @@
+/**********************************************************************
+ *
+ * cgiutils.cpp -- general cgi utilities
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.10  1999/09/21 21:41:17  sjboddie
+   fixed a couple of problems in what I committed last
+
+   Revision 1.9  1999/09/21 11:30:39  sjboddie
+   added ability to escape out '-' in saved args (currently
+   with a 'Z' ???)
+
+   Revision 1.8  1999/09/07 04:56:53  sjboddie
+   added GPL notice
+
+   Revision 1.7  1999/08/25 22:27:13  sjboddie
+   prevented cgi_safe from converting '+' and '-'. It was causing problems
+   with query strings containing spaces. The space was being converted
+   to '+', then %2b, then %xx2b over time when saved in compressed args.
+   I hope this won't cause problems elsewhere...
+
+   Revision 1.6  1999/07/11 01:05:19  rjmcnab
+   Stored origin of cgiarg with argument.
+
+   Revision 1.5  1999/06/26 01:08:36  rjmcnab
+   Added encoding and decoding of multibyte compresesd arguments.
+
+   Revision 1.4  1999/06/08 22:03:43  sjboddie
+   query string is now made cgi safe before being added to compressed args
+
+   Revision 1.3  1999/02/08 01:28:00  rjmcnab
+
+   Got the receptionist producing something using the statusaction.
+
+   Revision 1.2  1999/02/05 10:42:43  rjmcnab
+
+   Continued working on receptionist
+
+   Revision 1.1  1999/01/08 08:40:56  rjmcnab
+
+   Moved from lib directory.
+
+   Revision 1.1  1999/01/08 03:57:45  rjmcnab
+
+   Initial revision
+
+ */
+
+
+#include "cgiutils.h"
+
+
+static unsigned short hexdigit (unsigned short c) {
+  if (c >= '0' && c <= '9') return (c-'0');
+  if (c >= 'a' && c <= 'f') return (c-'a'+10);
+  if (c >= 'A' && c <= 'F') return (c-'A'+10);
+  return c;
+}
+
+
+static void c2hex (unsigned short c, text_t &t) {
+  t.clear();
+  
+  if (c >= 256) {
+    t = "20"; // ' '
+    return;
+  }
+  
+  unsigned short o1, o2;
+  
+  o1 = (c/16) % 16;
+  o2 = c % 16;
+  if (o1 >= 10) o1 += 'a' - 10;
+  else o1 += '0';
+  if (o2 >= 10) o2 += 'a' - 10;
+  else o2 += '0';
+  
+  t.push_back(o1);
+  t.push_back(o2);
+}
+
+// convert %xx and + to their appropriate equivalents
+void decode_cgi_arg (text_t &argstr) {
+  text_t::iterator in = argstr.begin();
+  text_t::iterator out = in;
+  text_t::iterator end = argstr.end();
+  
+  while (in != end) {
+    if (*in == '+') *out = ' ';
+    
+    else if (*in == '%') {
+      unsigned short c = '%';
+      in++;
+      if (in != end) {
+	c = hexdigit (*in);
+	in++;
+      }
+      if (in != end && c < 16) { // sanity check on the previous character
+	c = c*16 + hexdigit (*in);
+      }
+      
+      *out = c;
+    } else *out = *in;
+    
+    if (in != end) in++;
+    out++;
+  }
+  
+  // remove the excess characters
+  argstr.erase (out, end);
+}
+
+
+// split up the cgi arguments
+void split_cgi_args (text_t argstr, cgiargsclass &args) {
+  args.clear();
+  
+  text_t::iterator here = argstr.begin();
+  text_t::iterator end = argstr.end();
+  text_t key, value;
+  
+  // extract out the key=value pairs
+  while (here != end) {
+    // get the next key and value pair
+    here = getdelimitstr (here, end, '=', key);
+    here = getdelimitstr (here, end, '&', value);
+    
+    // convert %xx and + to their appropriate equivalents
+    decode_cgi_arg (value);
+
+    value.setencoding(1); // other encoding
+    // store this key=value pair
+    if (!key.empty()) args.setarg (key, value, cgiarg_t::cgi_arg);
+  }
+}
+
+text_t minus_safe (const text_t &intext) {
+
+  text_t outtext;
+
+  text_t::const_iterator here = intext.begin ();
+  text_t::const_iterator end = intext.end ();
+
+  while (here != end) {
+    if (*here == '-') outtext += "Z-";
+    else outtext.push_back (*here);
+    here ++;
+  }
+  outtext = cgi_safe (outtext);
+  return outtext;
+}
+
+text_t cgi_safe (const text_t &intext) {
+  text_t outtext;
+  
+  text_t::const_iterator here = intext.begin ();
+  text_t::const_iterator end = intext.end ();
+  unsigned short c;
+  text_t ttmp;
+  
+  while (here != end) {
+    c = *here;
+    if (((c >= 'a') && (c <= 'z')) ||
+	((c >= 'A') && (c <= 'Z')) ||
+	((c >= '0') && (c <= '9')) ||
+	(c == '+') || (c == '%') || (c == '-')) {
+      // alphanumeric character
+      outtext.push_back(c);
+    } else if (c == ' ') {
+      // space
+      outtext.push_back('+');
+    } else {
+      // everything else
+      outtext.push_back('%');
+      c2hex(c, ttmp);
+      outtext += ttmp;
+    }
+    
+    here++;
+  }
+  
+  return outtext;
+}
+
+
+
+
+static text_t::const_iterator get_next_save_arg (text_t::const_iterator first, 
+					   text_t::const_iterator last,
+					   text_t &argname) {
+  first = getdelimitstr (first, last, '-', argname);
+  return first;
+}
+
+
+// check_save_conf_str checks the configuration string for
+// the saved args and makes sure it does not conflict with
+// the information about the arguments. If an error is encountered
+// it will return false and the program should not produce any
+// output.
+bool check_save_conf_str (const text_t &saveconf, 
+			  const cgiargsinfoclass &argsinfo,
+			  ostream &logout) {
+  outconvertclass text_t2ascii;
+
+  text_tset argsset;
+  text_t::const_iterator saveconfhere = saveconf.begin ();
+  text_t::const_iterator saveconfend = saveconf.end ();
+  text_t argname;
+  const cgiarginfo *info;
+
+  // first check to make sure all saved arguments can be saved
+  
+  while (saveconfhere != saveconfend) {
+    saveconfhere = get_next_save_arg (saveconfhere, saveconfend, argname);
+
+    if (!argname.empty()) {
+      // save the argument name for later
+      argsset.insert (argname);
+
+      // check the argument
+      info =  argsinfo.getarginfo (argname);
+      if (info == NULL) {
+	logout << text_t2ascii << "Error: the cgi argument \"" << argname 
+	       << "\" is used in the configuration string for the\n"
+	       << "saved arguments but does not exist as a valid argument.\n\n";
+	return false;
+      }
+      if (info->savedarginfo == cgiarginfo::mustnot) {
+	logout << text_t2ascii << "Error: the cgi argument \"" << argname
+	       << "\" is used in the configuration string for the\n"
+	       << "saved arguments but has been specified as an argument whose\n"
+	       << "state must not be saved.\n\n";
+	return false;
+      }
+    }
+  }
+
+
+  // next check that all saved arguments that should be saved
+  // are saved
+  cgiargsinfoclass::const_iterator argsinfohere = argsinfo.begin ();
+  cgiargsinfoclass::const_iterator argsinfoend = argsinfo.end ();
+
+  while (argsinfohere != argsinfoend) {
+    if (((*argsinfohere).second.savedarginfo == cgiarginfo::must) &&
+	(argsset.find((*argsinfohere).second.shortname) == argsset.end())) {
+      logout << text_t2ascii << "Error: the cgi argument \"" 
+	     << (*argsinfohere).second.shortname << "\" was specified as needing to\n"
+	     << "be save but was not listed in the saved arguments.\n\n";
+      return false;
+    }
+
+    argsinfohere++;
+  }
+  
+  return true; // made it, no clashes
+}
+
+
+// create_save_conf_str will create a configuration string
+// based on the information in argsinfo. This method of configuration
+// is not recomended as small changes can produce large changes in
+// the resulting configuration string (for instance a totally different
+// ordering). Only arguments which "must" be saved are included in
+// the resulting string.
+text_t create_save_conf_str (const cgiargsinfoclass &argsinfo,
+			     ostream &/*logout*/) {
+  cgiargsinfoclass::const_iterator argsinfohere = argsinfo.begin ();
+  cgiargsinfoclass::const_iterator argsinfoend = argsinfo.end ();
+  text_t saveconf;
+  bool first = true;
+
+  while (argsinfohere != argsinfoend) {
+    // save this argument if it must be saved
+    if ((*argsinfohere).second.savedarginfo == cgiarginfo::must) {
+      if (!first) saveconf.push_back ('-');
+      else first = false;
+      saveconf += (*argsinfohere).second.shortname;
+    }
+
+    argsinfohere++;
+  }
+  
+  return saveconf;
+}
+
+
+// expand_save_args will expand the saved arguments based
+// on saveconf placing the results in args if they are not
+// already defined. If it encounters an error it will return false
+// and output more information to logout.
+bool expand_save_args (const cgiargsinfoclass &argsinfo,
+		       const text_t &saveconf, 
+		       cgiargsclass &args,
+		       ostream &logout) {
+  outconvertclass text_t2ascii;
+
+  text_t *arg_e = args.getarg("e");
+  if (arg_e == NULL) return true; // no compressed arguments
+  if (arg_e->empty()) return true; // no compressed arguments
+
+  text_t argname, argvalue;
+  const cgiarginfo *argnameinfo;
+  
+  text_t::const_iterator saveconfhere = saveconf.begin();
+  text_t::const_iterator saveconfend = saveconf.end();
+  
+  text_t::iterator arg_ehere = arg_e->begin();
+  text_t::iterator arg_eend = arg_e->end();
+  while (saveconfhere != saveconfend && arg_ehere != arg_eend) {
+    saveconfhere = get_next_save_arg (saveconfhere, saveconfend, argname);
+
+    if (!argname.empty()) {
+      // found another entry
+      argnameinfo = argsinfo.getarginfo (argname);
+
+      if (argnameinfo == NULL) {
+	// no information about the argument could be found
+	// we can't keep going because we don't know whether
+	// this argument is a single or multiple character value
+	logout << text_t2ascii << "Error: the cgi argument \"" << argname
+	       << "\" was specified as being a compressed argument\n"
+	       << "but no information about it could be found within the "
+	       << "cgiargsinfoclass.\n";
+	return false;
+
+      } else {
+
+	// found the argument information
+	if (argnameinfo->multiplechar) {
+	  text_t::const_iterator sav = arg_ehere;
+	  arg_ehere = getdelimitstr (arg_ehere, arg_eend, '-', argvalue);
+	  // replace any '-' chars escaped with 'Z'
+	  if (*(arg_ehere-2) == 'Z') {
+	    argvalue.clear();
+	    arg_ehere = (findchar (arg_ehere, arg_eend, '-')) + 1;
+	    while (sav != (arg_ehere-1)) {
+	      if (!((*sav == 'Z') && (*(sav+1) == '-'))) argvalue.push_back (*sav);
+	      sav ++;
+	    }
+	  }
+
+	  argvalue.setencoding(1); // other encoding
+	  if (!argvalue.empty()) args.setdefaultarg (argname, argvalue, cgiarg_t::compressed_arg);
+	} else {
+	  args.setdefaultcarg (argname,*arg_ehere, cgiarg_t::compressed_arg);
+	  arg_ehere++;
+	}
+      }
+    }
+  }
+
+  return true;
+}
+
+
+// adds the default values for those arguments which have not
+// been specified
+void add_default_args (const cgiargsinfoclass &argsinfo,
+		       cgiargsclass &args,
+		       ostream &/*logout*/) {
+  cgiargsinfoclass::const_iterator argsinfohere = argsinfo.begin ();
+  cgiargsinfoclass::const_iterator argsinfoend = argsinfo.end ();
+
+  while (argsinfohere != argsinfoend) {
+    if ((*argsinfohere).second.defaultstatus != cgiarginfo::none) {
+      args.setdefaultarg ((*argsinfohere).second.shortname,
+			  (*argsinfohere).second.argdefault, cgiarg_t::default_arg);
+    }
+    argsinfohere++;
+  }
+}
+
+
+// compress_save_args will compress the arguments and return
+// them in compressed_args. If an error was encountered
+// compressed_args will be set to to "", an error will be
+// written to logout, and the function will return false.
+bool compress_save_args (const cgiargsinfoclass &argsinfo,
+			 const text_t &saveconf,
+			 cgiargsclass &args,
+			 text_t &compressed_args,
+			 outconvertclass &outconvert,
+			 ostream &logout) {
+  outconvertclass text_t2ascii;
+
+  compressed_args.clear();
+
+  text_t argname, argvalue;
+  const cgiarginfo *argnameinfo;
+  
+  text_t::const_iterator saveconfhere = saveconf.begin();
+  text_t::const_iterator saveconfend = saveconf.end();
+  
+  while (saveconfhere != saveconfend) {
+    saveconfhere = get_next_save_arg (saveconfhere, saveconfend, argname);
+
+    if (!argname.empty()) {
+      // found another entry
+      argnameinfo = argsinfo.getarginfo (argname);
+
+      if (argnameinfo == NULL) {
+	// no information about the argument could be found
+	// we can't keep going because we don't know whether
+	// this argument is a single or multiple character value
+	logout << text_t2ascii << "Error: the cgi argument \"" << argname
+	       << "\" was specified as being a compressed argument\n"
+	       << "but no information about it could be found within the "
+	       << "cgiargsinfoclass.\n";
+	compressed_args.clear();
+	return false;
+
+      } else {
+	// found the argument information
+	if (argnameinfo->multiplechar) {
+	  // multiple character argument -- sort out any '-' chars
+	  compressed_args += minus_safe (outconvert.convert(args[argname]));
+	  if (saveconfhere != saveconfend) compressed_args.push_back ('-');
+
+	} else {
+	  // single character argument
+	  if (args[argname].size() == 0) {
+	    logout << text_t2ascii << "Error: the cgi argument \"" << argname
+		   << "\" was specified as being a compressed argument which\n"
+		   << "should have a one character value but it was empty.\n\n";
+	    compressed_args.clear ();
+	    return false;
+
+	  } else if (args[argname].size() > 1) {
+	    logout << text_t2ascii << "Error: the cgi argument \"" << argname
+		   << "\" was specified as being a compressed argument which\n"
+		   << "should have a one character value but it had multiple characters.\n\n";
+	    compressed_args.clear ();
+	    return false;
+	  }
+
+	  // everything is ok
+	  compressed_args += args[argname];
+	}
+      }
+    }
+  }
+
+  return true;
+}
+
+
+// args_tounicode converts any arguments which are not in unicode
+// to unicode using inconvert
+void args_tounicode (cgiargsclass &args, inconvertclass &inconvert) {
+  cgiargsclass::iterator here = args.begin();
+  cgiargsclass::iterator end = args.end();
+
+  while (here != end) {
+    if ((*here).second.value.getencoding() > 0) {
+      (*here).second.value = inconvert.convert((*here).second.value);
+    }
+    
+    here++;
+  }
+}
Index: /tags/gsdl-21/gsdl/src/recpt/cgiutils.h
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/cgiutils.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/cgiutils.h	(revision 699)
@@ -0,0 +1,97 @@
+/**********************************************************************
+ *
+ * cgiutils.h -- general cgi utilities
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef CGIUTILS_H
+#define CGIUTILS_H
+
+#include "gsdlconf.h"
+#include "text_t.h"
+#include "cgiargs.h"
+
+
+// convert %xx and + to their appropriate equivalents
+void decode_cgi_arg (text_t &argstr);
+
+// split up the cgi arguments
+void split_cgi_args (text_t argstr, cgiargsclass &args);
+
+// turns any '-' in a cgi argument into "(-)"
+text_t minus_safe (const text_t &intext);
+
+// returns the encoded version of a cgi argument
+text_t cgi_safe (const text_t &intext);
+
+// check_save_conf_str checks the configuration string for
+// the saved args and makes sure it does not conflict with
+// the information about the arguments. If an error is encountered
+// it will return false and the program should not produce any
+// output.
+bool check_save_conf_str (const text_t &saveconf, 
+			  const cgiargsinfoclass &argsinfo,
+			  ostream &logout);
+
+// create_save_conf_str will create a configuration string
+// based on the information in argsinfo. This method of configuration
+// is not recomended as small changes can produce large changes in
+// the resulting configuration string (for instance a totally different
+// ordering). Only arguments which "must" be saved are included in
+// the resulting string.
+text_t create_save_conf_str (const cgiargsinfoclass &argsinfo,
+			     ostream &logout);
+
+// expand_compress_args will expand the compressed arguments based
+// on compressconf placing the results in args if they are not
+// already defined. If it encounters an error it will return false
+// and output more information to logout.
+bool expand_save_args (const cgiargsinfoclass &argsinfo,
+		       const text_t &saveconf, 
+		       cgiargsclass &args,
+		       ostream &logout);
+
+// adds the default values for those arguments which have not
+// been specified
+void add_default_args (const cgiargsinfoclass &argsinfo,
+		       cgiargsclass &args,
+		       ostream &logout);
+
+// compress_save_args will compress the arguments and return
+// them in compressed_args. If an error was encountered
+// compressed_args will be set to to "", an error will be
+// written to logout, and the function will return false.
+bool compress_save_args (const cgiargsinfoclass &argsinfo,
+			 const text_t &saveconf,
+			 cgiargsclass &args,
+			 text_t &compressed_args,
+			 outconvertclass &outconvert,
+			 ostream &logout);
+
+// args_tounicode converts any arguments which are not in unicode
+// to unicode using inconvert
+void args_tounicode (cgiargsclass &args, inconvertclass &inconvert);
+
+#endif
Index: /tags/gsdl-21/gsdl/src/recpt/cgiwrapper.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/cgiwrapper.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/cgiwrapper.cpp	(revision 699)
@@ -0,0 +1,522 @@
+/**********************************************************************
+ *
+ * cgiwrapper.cpp -- output pages using the cgi protocol
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.20  1999/09/07 04:56:53  sjboddie
+   added GPL notice
+
+   Revision 1.19  1999/09/02 00:24:36  rjmcnab
+   fixed bug in getting POST arguments
+
+   Revision 1.18  1999/08/20 01:02:07  sjboddie
+   added some usage logging
+
+   Revision 1.17  1999/07/15 06:03:15  rjmcnab
+   Moved the adding of the actions to librarymain so that they can be
+   overriden easily.
+
+   Revision 1.16  1999/07/14 08:31:05  rjmcnab
+   Fixed a small bug in the POST implementation.
+
+   Revision 1.15  1999/07/13 23:32:17  rjmcnab
+   Added authenaction and usersaction
+
+   Revision 1.14  1999/07/11 01:03:37  rjmcnab
+   Added ability to receive POST cgi form data.
+
+   Revision 1.13  1999/06/24 05:12:18  sjboddie
+   lots of small changes
+
+   Revision 1.12  1999/04/30 01:59:40  sjboddie
+   lots of stuff - getting documentaction working (documentaction replaces
+   old browseaction)
+
+   Revision 1.11  1999/03/25 03:12:01  sjboddie
+
+   subjectbrowseaction was replaced with browseaction
+
+   Revision 1.10  1999/03/05 03:53:54  sjboddie
+
+   fixed some bugs
+
+   Revision 1.9  1999/03/04 22:38:21  sjboddie
+
+   Added subjectbrowseaction. - Doesn't do anything yet.
+
+   Revision 1.8  1999/02/28 20:00:13  rjmcnab
+
+
+   Fixed a few things.
+
+   Revision 1.7  1999/02/21 22:33:53  rjmcnab
+
+   Lots of stuff :-)
+
+   Revision 1.6  1999/02/12 02:40:17  sjboddie
+
+   Added page action
+
+   Revision 1.5  1999/02/11 01:24:04  rjmcnab
+
+   Fixed a few compiler warnings.
+
+   Revision 1.4  1999/02/08 01:28:01  rjmcnab
+
+   Got the receptionist producing something using the statusaction.
+
+   Revision 1.3  1999/02/05 10:42:44  rjmcnab
+
+   Continued working on receptionist
+
+   Revision 1.2  1999/02/04 10:00:56  rjmcnab
+
+   Developed the idea of an "action" and having them define the cgi arguments
+   which they need and how those cgi arguments function.
+
+   Revision 1.1  1999/02/04 01:16:17  rjmcnab
+
+   Initial revision.
+
+   Revision 1.5  1999/01/19 01:38:18  rjmcnab
+
+   Made the source more portable.
+
+   Revision 1.4  1999/01/12 01:51:04  rjmcnab
+
+   Standard header.
+
+ */
+
+
+#include "gsdlconf.h"
+#include "cgiwrapper.h"
+#include "recptconfig.h"
+#include <stdlib.h>
+
+
+// get GSDL_GSDLHOME
+#include "gsdlhome.h"
+
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace/std/iostream>
+#  include <ospace/std/fstream>
+#elif defined(GSDL_USE_IOS_H)
+#  include <iostream.h>
+#  include <fstream.h>
+#else
+#  include <iostream>
+#  include <fstream>
+#endif
+
+#ifdef USE_FASTCGI
+#include "fcgiapp.h"
+#endif
+
+
+#ifdef USE_FASTCGI
+// used to output the text from receptionist
+class fcgistreambuf : public streambuf {
+public:
+  fcgistreambuf ();
+  int sync ();
+  int overflow (int ch);
+  int underflow () {return EOF;}
+  
+  void fcgisbreset() {fcgx_stream = NULL; other_ostream = NULL;};
+  void set_fcgx_stream(FCGX_Stream *newone) {fcgx_stream=newone;};
+  void set_other_ostream(ostream *newone) {other_ostream=newone;};
+  
+private:
+  FCGX_Stream *fcgx_stream;
+  ostream *other_ostream;
+};
+
+fcgistreambuf::fcgistreambuf() {
+  fcgisbreset();
+  if (base() == ebuf()) allocate();
+  setp (base(), ebuf());
+};
+
+int fcgistreambuf::sync () {
+  if ((fcgx_stream != NULL) &&
+      (FCGX_PutStr (pbase(), out_waiting(), fcgx_stream) < 0)) {
+    fcgx_stream = NULL;
+  }
+
+  if (other_ostream != NULL) {
+    char *thepbase=pbase();
+    for (int i=0;i<out_waiting();i++) (*other_ostream).put(thepbase[i]);
+  }
+  
+  setp (pbase(), epptr());
+  
+  return 0;
+}
+
+int fcgistreambuf::overflow (int ch) {
+  if (sync () == EOF) return EOF;
+  if (ch != EOF) sputc (ch);
+  return 0;
+}
+
+#endif
+
+
+static void page_errorsitecfg (const text_t &gsdlhome, const text_t &collection,
+			       text_t &errorpage) {
+  errorpage += "Content-type: text/html\n\n";
+
+  errorpage += "<html>\n";
+  errorpage += "<head>\n";
+  errorpage += "<title>Error</title>\n";
+  errorpage += "</head>\n";
+  errorpage += "<body>\n";
+  errorpage += "<h2>Oops!</h2>\n";
+  errorpage += "The site.cfg configuration file could not be found. This file\n";
+  errorpage += "should contain configuration information relating to this\n";
+  errorpage += "site's setup. ";
+  if (collection.empty()) {
+    errorpage += "As this cgi script is not being run in collection specific mode,\n";
+    errorpage += "the file should reside at ";
+    errorpage += gsdlhome;
+    errorpage += "/etc/site.cfg.\n";
+  } else {
+    errorpage += "As this cgi script is being run in collection specific mode,\n";
+    errorpage += "the file can reside in ";
+    errorpage += gsdlhome;
+    errorpage += "/collect/";
+    errorpage += collection;
+    errorpage += "/etc/site.cfg or ";
+    errorpage += gsdlhome;
+    errorpage += "/etc/site.cfg.\n";
+  }
+  errorpage += "</body>\n";
+  errorpage += "</html>\n";
+}
+
+
+static void page_errormaincfg (const text_t &gsdlhome, const text_t &collection,
+			       text_t &errorpage) {
+  errorpage += "Content-type: text/html\n\n";
+
+  errorpage += "<html>\n";
+  errorpage += "<head>\n";
+  errorpage += "<title>Error</title>\n";
+  errorpage += "</head>\n";
+  errorpage += "<body>\n";
+  errorpage += "<h2>Oops!</h2>\n";
+  if (collection.empty()) {
+    errorpage += "The main.cfg configuration file could not be found. This file\n";
+    errorpage += "should contain configuration information relating to the\n";
+    errorpage += "setup of the interface. As this cgi script is not being run\n";
+    errorpage += "in collection specific mode the file should reside at\n";
+    errorpage += gsdlhome;
+    errorpage += "/etc/main.cfg.\n";
+  } else {
+    errorpage += "Neither the collect.cfg or main.cfg configuration files could\n";
+    errorpage += "not be found. This file should contain configuration information\n";
+    errorpage += "relating to the setup of the interface. As this cgi script is\n";
+    errorpage += "being run in collection specific mode the file should reside\n";
+    errorpage += "at either ";
+    errorpage += gsdlhome;
+    errorpage += "/collect/";
+    errorpage += collection;
+    errorpage += "/etc/collect.cfg, ";
+    errorpage += gsdlhome;
+    errorpage += "/etc/collect.cfg or ";
+    errorpage += gsdlhome;
+    errorpage += "/etc/main.cfg.\n";
+  }
+  errorpage += "</body>\n";
+  errorpage += "</html>\n";
+}
+
+
+static void page_errorinit (const text_t &/*gsdlhome*/, text_t &errorpage) {
+  errorpage += "Content-type: text/html\n\n";
+
+  errorpage += "<html>\n";
+  errorpage += "<head>\n";
+  errorpage += "<title>Error</title>\n";
+  errorpage += "</head>\n";
+  errorpage += "<body>\n";
+  errorpage += "<h2>Oops!</h2>\n";
+  errorpage += "An error occurred during the initialisation of the Greenstone Digital\n";
+  errorpage += "Library software. It is likely that the software has not been setup\n";
+  errorpage += "correctly.\n";
+
+  ifstream initin (GSDL_GSDLHOME "/etc/initout.txt");
+  if (initin) {
+    errorpage += "The initialisation error log, " GSDL_GSDLHOME "/etc/initout.txt, contains the\n";
+    errorpage += "following information:\n\n";
+    errorpage += "<pre>\n";
+
+    char c;
+    initin.get(c);
+    while (!initin.eof ()) {
+      errorpage.push_back(c);
+      initin.get(c);
+    }
+    
+    errorpage += "</pre>\n";
+
+    initin.close();
+
+  } else {
+    errorpage += "Please consult " GSDL_GSDLHOME "/etc/initout.txt for more information.\n";
+  }
+
+  errorpage += "</body>\n";
+  errorpage += "</html>\n";
+}
+
+static void page_errorparseargs (const text_t &/*gsdlhome*/, text_t &errorpage) {
+  errorpage += "Content-type: text/html\n\n";
+
+  errorpage += "<html>\n";
+  errorpage += "<head>\n";
+  errorpage += "<title>Error</title>\n";
+  errorpage += "</head>\n";
+  errorpage += "<body>\n";
+  errorpage += "<h2>Oops!</h2>\n";
+  errorpage += "An error occurred during the parsing of the cgi arguments.\n";
+
+  ifstream errin (GSDL_GSDLHOME "/etc/errout.txt");
+  if (errin) {
+    errorpage += "The error log, " GSDL_GSDLHOME "/etc/errout.txt, contains the\n";
+    errorpage += "following information:\n\n";
+    errorpage += "<pre>\n";
+
+    char c;
+    errin.get(c);
+    while (!errin.eof ()) {
+      errorpage.push_back(c);
+      errin.get(c);
+    }
+    errorpage += "</pre>\n";
+    errin.close();
+
+  } else {
+    errorpage += "Please consult " GSDL_GSDLHOME "/etc/errout.txt for more information.\n";
+  }
+
+  errorpage += "</body>\n";
+  errorpage += "</html>\n";
+}
+
+static void page_errorcgipage (const text_t &/*gsdlhome*/, text_t &errorpage) {
+  errorpage += "Content-type: text/html\n\n";
+
+  errorpage += "<html>\n";
+  errorpage += "<head>\n";
+  errorpage += "<title>Error</title>\n";
+  errorpage += "</head>\n";
+  errorpage += "<body>\n";
+  errorpage += "<h2>Oops!</h2>\n";
+  errorpage += "An error occurred during the construction of the cgi page.\n";
+
+  ifstream errin (GSDL_GSDLHOME "/etc/errout.txt");
+  if (errin) {
+    errorpage += "The error log, " GSDL_GSDLHOME "/etc/errout.txt, contains the\n";
+    errorpage += "following information:\n\n";
+    errorpage += "<pre>\n";
+
+    char c;
+    errin.get(c);
+    while (!errin.eof ()) {
+      errorpage.push_back(c);
+      errin.get(c);
+    }
+    errorpage += "</pre>\n";
+    errin.close();
+
+  } else {
+    errorpage += "Please consult " GSDL_GSDLHOME "/etc/errout.txt for more information.\n";
+  }
+
+  errorpage += "</body>\n";
+  errorpage += "</html>\n";
+}
+
+
+// cgiwrapper does everything necessary to output a page
+// using the cgi protocol. If this is being run for a particular
+// collection then "collection" should be set, otherwise it
+// should equal "".
+void cgiwrapper (receptionist &recpt, text_t collection) {
+#ifdef USE_FASTCGI
+  fcgistreambuf outbuf;
+#endif
+
+  // init stuff - we can't output error pages directly with
+  // fastcgi so the pages are stored until we can output them
+  text_t errorpage;
+  outconvertclass text_t2ascii;
+
+  // set defaults
+  int maxrequests = 10000;
+  recpt.configure ("gsdlhome", GSDL_GSDLHOME);
+  recpt.configure ("collection", collection);
+  recpt.configure ("httpimg", "/gsdl/images");
+  char *script_name = getenv("SCRIPT_NAME");
+  if (script_name != NULL) recpt.configure("gwcgi", script_name);
+  else recpt.configure("gwcgi", "/cgi-bin/gw");
+
+  // read in the configuration files.
+  if (!site_cfg_read (recpt, GSDL_GSDLHOME, collection, maxrequests)) {
+    // couldn't find the site configuration file
+    page_errorsitecfg (GSDL_GSDLHOME, collection, errorpage);
+  } else if (!main_cfg_read (recpt, GSDL_GSDLHOME, collection)) {
+    // couldn't find the main configuration file
+    page_errormaincfg (GSDL_GSDLHOME, collection, errorpage);
+  }
+
+  // initialise the library software
+  if (errorpage.empty()) {
+    ofstream initout (GSDL_GSDLHOME "/etc/initout.txt");
+    if (!recpt.init(initout)) {
+      // an error occurred during the initialisation
+      initout.close();
+      page_errorinit(GSDL_GSDLHOME, errorpage);
+    }
+    initout.close();
+  }
+  
+  // find out whether this is being run as a cgi-script
+  // or a fastcgi script
+  int numrequests = 0;
+#ifdef USE_FASTCGI
+  int isfastcgi = !FCGX_IsCGI();
+  FCGX_Stream *fcgiin, *fcgiout, *fcgierr;
+  FCGX_ParamArray fcgienvp;
+#else
+  int isfastcgi = 0;
+#endif
+
+  // get the query string if it is not being run as a fastcgi
+  // script
+  text_t argstr = "";
+  cgiargsclass args;
+  char *aURIStr;
+  if (!isfastcgi) {
+    char *request_method_str = getenv("REQUEST_METHOD");
+    char *content_length_str = getenv("CONTENT_LENGTH");
+    if (request_method_str != NULL && strcmp(request_method_str, "POST") == 0 &&
+	content_length_str != NULL)  {
+      // POST form data
+      int content_length = text_t(content_length_str).getint();
+      if (content_length > 0) {
+	char c;
+	do {
+	  cin.get(c);
+	  if (cin.eof()) break;
+	  argstr.push_back (c);
+	  content_length--;
+	} while (content_length > 0);
+      }
+      
+    } else {
+      aURIStr = getenv("QUERY_STRING");
+      if ((request_method_str != NULL && strcmp(request_method_str, "GET") == 0)
+	  || aURIStr != NULL) {
+	// GET form data
+	if (aURIStr != NULL) argstr = aURIStr;
+      } else {
+	// debugging from command line
+	char cinURIStr[1024];
+	cin.get(cinURIStr, 1024);
+	argstr = cinURIStr;
+      }
+    }
+
+    // cgi scripts only deal with one request
+    maxrequests = 1;
+  }
+
+  // Page-request loop. If this is not being run as a fastcgi
+  // process then only one request will be processed and then
+  // the process will exit.
+  while (numrequests < maxrequests) {
+#ifdef USE_FASTCGI
+    if (isfastcgi) {
+      if (FCGX_Accept(&fcgiin, &fcgiout, &fcgierr, &fcgienvp) < 0) break;
+      aURIStr = FCGX_GetParam("QUERY_STRING", fcgienvp);
+      if (aURIStr != NULL) argstr = aURIStr;
+      else argstr = "";
+    }
+#endif
+
+    // get output streams ready
+#ifdef USE_FASTCGI
+    outbuf.fcgisbreset ();
+    if (isfastcgi) outbuf.set_fcgx_stream (fcgiout);
+    else outbuf.set_other_ostream (&cout);
+    ostream pageout (&outbuf);
+#else
+#define pageout cout
+#endif
+
+    if (errorpage.empty()) {
+      ofstream errout (GSDL_GSDLHOME "/etc/errout.txt");
+      cerr = errout;
+    
+      // parse the cgi arguments and produce the resulting page if there
+      // has been no errors so far
+      if (!recpt.parse_cgi_args (argstr, args, errout)) {
+	errout.close ();
+	page_errorparseargs(GSDL_GSDLHOME, errorpage);
+      } else {
+	if (!recpt.produce_cgi_page (args, pageout, errout)) {
+	  errout.close ();
+	  page_errorcgipage(GSDL_GSDLHOME, errorpage);
+	} else {
+	  errout.close ();
+	}
+	recpt.log_cgi_args (args, errout);
+      }
+    }
+    // there was an error, output the error page
+    if (!errorpage.empty()) {
+      pageout << text_t2ascii << errorpage;
+      errorpage.clear();
+      numrequests = maxrequests; // make this the last page
+    }
+    pageout << flush;
+    
+    // finish with the output streams
+#ifdef USE_FASTCGI
+    if (isfastcgi) FCGX_Finish();
+#endif
+
+    numrequests++;
+  }
+
+  return;
+}
+
Index: /tags/gsdl-21/gsdl/src/recpt/cgiwrapper.h
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/cgiwrapper.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/cgiwrapper.h	(revision 699)
@@ -0,0 +1,42 @@
+/**********************************************************************
+ *
+ * cgiwrapper.h -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef CGIWRAPPER_H
+#define CGIWRAPPER_H
+
+
+#include "receptionist.h"
+
+// cgiwrapper does everything necessary to output a page
+// using the cgi protocol. If this is being run for a particular
+// collection then "collection" should be set, otherwise it
+// should equal "".
+void cgiwrapper (receptionist &recpt, text_t collection);
+
+
+#endif
Index: /tags/gsdl-21/gsdl/src/recpt/comproto.asn
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/comproto.asn	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/comproto.asn	(revision 699)
@@ -0,0 +1,376 @@
+-- The protocol data unit (PDU) is the
+-- message that is transmitted between receptionists and
+-- collectionservers (colservers?).
+--
+-- Comments in ASN.1 start with two dashes and continue to the end
+-- of a line (or to the next two dashes, whichever is first).
+-- 
+-- Z39.50 has inspired much of this but I have tried to keep
+-- it simple. Its hard to tell if I have succeeded ;-)
+
+
+GSDL-APDU DEFINITIONS ::=
+BEGIN
+
+
+-- PDU is the basic message type. Note: in ASN.1 the
+-- field name comes first and the field type second (opposite
+-- to C). Ignore the numbers in the square brackets (e.g. [0]),
+-- they're just tags to disambiguate certain circumstances and to
+-- make the protocol easier to extend. The keyword 'IMPLICIT' just
+-- makes the tags a little more efficient.
+--
+-- A SEQUENCE is a list of fields where the order is important
+
+PDU ::= SEQUENCE {
+  header  StandardHeader,
+  message Message
+}
+
+
+-- ????? Is there anything else we want in the header ?????
+--
+-- Anonymous transactions should leave out the idAuthentication field
+
+StandardHeader ::= SEQUENCE {
+  protocolVersion       [0] IMPLICIT INTEGER,
+  idAuthentication      [1] IMPLICIT IdAuthentication OPTIONAL
+}
+
+IdAuthentication ::= SEQUENCE {
+  userId   GeneralString,
+  groupId  GeneralString,
+  password GeneralString
+}
+
+
+-- A CHOICE is like a union in C.
+-- 
+-- Short forms: Auth (Authority), Mir (Mirror), Col (Collection),
+--   Doc (Document), Info (Information)
+
+Message ::= CHOICE {
+  -- General
+  error                      [0]  IMPLICIT Error,
+  pingRequest                [1]  IMPLICIT PingRequest,
+  pingResponse               [2]  IMPLICIT PingResponse,
+
+  -- Authority
+--  authListRequest            [12] IMPLICIT AuthListRequest,
+--  authListResponse           [13] IMPLICIT AuthListResponse,
+--  editAuthListRequest        [14] IMPLICIT EditAuthListRequest,
+--  editAuthListResponse       [15] IMPLICIT EditAuthListResponse,
+
+  -- an authority mirror is a mirror of THIS authority
+--  authMirListRequest         [16] IMPLICIT AuthMirListRequest,
+--  authMirListResponse        [17] IMPLICIT AuthMirListResponse,
+--  editAuthMirListRequest     [18] IMPLICIT EditAuthMirListRequest,
+--  editAuthMirListResponse    [19] IMPLICIT EditAuthMirListResponse,
+
+  colListRequest             [20] IMPLICIT ColListRequest,
+  colListResponse            [21] IMPLICIT ColListResponse,
+--  editColListRequest         [22] IMPLICIT EditColListRequest,
+--  editColListResponse        [23] IMPLICIT EditColListResponse,
+
+--  colMirListRequest          [24] IMPLICIT ColMirListRequest,
+--  colMirListResponse         [25] IMPLICIT ColMirListResponse, 
+--  editColMirListRequest      [26] IMPLICIT EditColMirListRequest, 
+--  editColMirListResponse     [27] IMPLICIT EditColMirListResponse,
+
+  -- Collection
+  colInfoRequest             [40] IMPLICIT ColInfoRequest,
+  colInfoResponse            [41] IMPLICIT ColInfoResponse,
+
+  -- Document
+  documentRequest            [52] IMPLICIT DocumentRequest,
+  documentResponse           [53] IMPLICIT DocumentResponse,
+
+  -- Access (searching, browsing, and metadata information)
+  infoFiltersRequest         [60] IMPLICIT InfoFiltersRequest,
+  infoFiltersResponse        [61] IMPLICIT InfoFiltersResponse,
+  infoFilterOptionsRequest   [62] IMPLICIT InfoFilterOptionsRequest,
+  infoFilterOptionsResponse  [63] IMPLICIT InfoFilterOptionsResponse,
+  filterRequest              [64] IMPLICIT FilterRequest,
+  filterResponse             [65] IMPLICIT FilterResponse
+}
+
+
+
+-- ------------------------------
+-- GENERAL
+-- ------------------------------
+
+
+-- The error list is not complete yet ...
+
+Error ::= ENUMERATED {
+  authenticationFailure(0), protocolError(1), systemProblem(2)
+}
+
+
+PingRequest ::= SEQUENCE {
+}
+
+PingResponse ::= SEQUENCE {
+}
+
+
+
+-- ------------------------------
+-- AUTHORITY
+-- ------------------------------
+
+
+-- Support for subscription is optional for both the clients and
+-- the servers. It is recomended that it is not implemented yet
+-- for simplicity.
+
+-- AuthListRequest ::= SEQUENCE {
+-- }
+
+-- AuthListResponse ::= SEQUENCE {
+--   authList  [1] IMPLICIT StringSet
+-- }
+
+
+-- EditAuthListRequest ::= SEQUENCE {
+--   editAction EditAction,
+--   authority  GeneralString
+-- }
+
+-- EditAuthListResponse ::= SEQUENCE {
+-- }
+
+
+-- AuthMirListRequest ::= SEQUENCE {
+--   limitNumber [1] IMPLICIT INTEGER OPTIONAL,
+--   limitDomain [2] IMPLICIT GeneralString OPTIONAL
+-- }
+
+-- AuthMirListResponse ::= SEQUENCE {
+--   authMirList [1] IMPLICIT HostPortSet
+-- }
+
+
+-- EditAuthMirListRequest ::= SEQUENCE {
+--   editAction EditAction,
+--   authMir    HostPort
+-- }
+
+-- EditAuthMirListResponse ::= SEQUENCE {
+-- }
+
+
+ColListRequest ::= SEQUENCE {
+}
+
+ColListResponse ::= SEQUENCE {
+  colList   [1] IMPLICIT ShortColInfoSet
+}
+
+
+-- EditColListRequest ::= SEQUENCE {
+--   editAction EditAction,
+--   colInfo    ShortColInfo
+-- }
+
+-- EditColListResponse ::= SEQUENCE {
+-- }
+
+
+-- ColMirListRequest ::= SEQUENCE {
+--   limitNumber [1] IMPLICIT INTEGER OPTIONAL,
+--   limitDomain [2] IMPLICIT GeneralString OPTIONAL
+-- }
+
+-- ColMirListResponse ::= SEQUENCE {
+--   colMirList [1] IMPLICIT ShortColInfoSet
+-- }
+
+
+-- EditColMirListRequest ::= SEQUENCE {
+--   editAction EditAction,
+--   colMirInfo ShortColInfo
+-- }
+
+-- EditColMirListResponse ::= SEQUENCE {
+-- }
+
+
+
+
+-- ------------------------------
+-- COLLECTION
+-- ------------------------------
+
+
+ColInfoRequest ::= SEQUENCE {
+}
+
+ColInfoResponse ::= SEQUENCE {
+  shortInfo   [0] IMPLICIT ShortCollectionInfo,
+  isPublic    [2] IMPLICIT BOOLEAN,    -- whether has anonymous access
+  isBeta      [3] IMPLICIT BOOLEAN,    -- beta if still under development
+  builddate   [4] IMPLICIT GeneralizedTime,
+  languages   [5] IMPLICIT StringSet,  -- languages in the collection
+  numDocs     [6] IMPLICIT INTEGER,
+  numWords    [7] IMPLICIT INTEGER OPTIONAL,
+  numBytes    [8] IMPLICIT INTEGER OPTIONAL
+}
+
+
+
+-- ------------------------------
+-- DOCUMENT
+-- ------------------------------
+
+
+
+DocumentRequest ::= SEQUENCE {
+  OID       GeneralString,
+  docType   GeneralString,
+  docFormat GeneralString
+}
+
+DocumentResponse ::= SEQUENCE {
+  doc OCTET STRING
+}
+
+
+-- ------------------------------
+-- ACCESS (SEARCH AND BROWSE)
+-- ------------------------------
+
+
+-- filter options which might be supported for the QueryFilter
+--
+-- onePerQuery StartResults   integer
+-- onePerQuery EndResults     integer
+-- onePerQuery QueryType      enumerated (boolean, ranked)
+-- onePerTerm  Term           string ???
+-- onePerTerm  Casefold       boolean
+-- onePerTerm  Stem           boolean
+-- onePerTerm  Index          enumerated
+-- onePerTerm  Subcollection  enumerated
+--
+-- filter options which might be supported for the BrowseFilter
+--
+-- onePerQuery StartResults   integer
+-- onePerQuery EndResults     integer
+-- onePerQuery ParentNode     string ("" will return the browsing available)
+--
+-- The NullFilter always returns the set it was given, it doesn't have
+-- any options
+
+InfoFiltersRequest ::= SEQUENCE {
+}
+
+InfoFiltersResponse ::= SEQUENCE {
+  filterNames StringSet
+}
+
+InfoFilterOptionsRequest ::= SEQUENCE {
+  filterName GeneralString
+}
+
+InfoFilterOptionsResponse ::= SEQUENCE {
+  filterOptions SET OF FilterOption
+}
+
+FilterOption ::= SEQUENCE {
+  name         GeneralString,
+  type         ENUMERATED {booleant(0), integert(1), enumeratedt(2), stringt(3)},
+  repeatable   ENUMERATED {onePerQuery(0), onePerTerm(1), nPerTerm(2)},
+  defaultValue GeneralString,
+  -- the interpretation of the validValues depends on the type
+  -- for boolean: the first value is the false value, the second is true value
+  -- for integer: the first value is the minimum, the second the maximum
+  -- for enumerated: all values a listed
+  -- for string: this value is ignored
+  validValues  StringSequence
+}
+
+
+-- Terms are presented in the same order that they are requested,
+-- any information relating to the terms is in reference to the
+-- index specified for that term.
+-- Metadata is presented in the same order that it is requested.
+
+FilterRequest ::= SEQUENCE {
+  filterName          [0] GeneralString,
+  filterOptions       [1] IMPLICIT SEQUENCE OF OptionValue,
+  docSet              [2] IMPLICIT StringSequence, -- the OID "" represents everything
+  filterResultOptions [3] IMPLICIT BIT STRING {termFreq(0), matchTerms(1), OID(2), 
+                                               subCol(3), ranking(4), docFreq(5), 
+                                               metadata(6)}
+
+  -- the next set of options are for the metadata request, 
+  -- they can be left blank if metadata is not wanted
+  requestParams       [4] IMPLICIT GeneralString, -- used to negotiate the metadata content
+  refParams           [5] IMPLICIT GeneralString, -- used to decide whether to return a 
+                                                  -- reference to the data or the actual data
+  fields              [6] IMPLICIT StringSequence
+}
+
+OptionValue ::= SEQUENCE {
+  name  GeneralString,
+  value GeneralString
+}
+
+FilterResponse ::= SEQUENCE {
+  numDocs   [0] IMPLICIT INTEGER,
+  isApprox  [1] IMPLICIT BOOLEAN,                  -- whether numDocs is approximate
+  termInfo  [2] IMPLICIT SEQUENCE OF TermInfo,     -- empty if not required
+  docInfo   [3] IMPLICIT SEQUENCE OF ResultDocInfo -- empty if not required
+}
+
+TermInfo ::= SEQUENCE {
+  term       [0] GeneralString,
+  freq       [1] IMPLICIT INTEGER,       -- 0 if not requested
+  matchTerms [2] IMPLICIT StringSequence -- empty if not requested
+}
+
+ResultDocInfo ::= SEQUENCE {
+  OID      [0] IMPLICIT GeneralString,
+  ranking  [1] IMPLICIT INTEGER,             -- 0 if not requested, range 0-10000
+  docFreq  [2] IMPLICIT SEQUENCE OF INTEGER, -- empty if not requested
+  metadata [3] IMPLICIT SEQUENCE OF MetadataInfo
+}
+
+
+MetadataInfo ::= SEQUENCE {
+  params  [0] IMPLICIT GeneralString,
+  isRef   [1] IMPLICIT BOOLEAN,
+  field   [3] IMPLICIT GeneralString,
+  value   [4] IMPLICIT GeneralString
+}
+
+
+-- ------------------------------
+-- Useful Types
+-- ------------------------------
+
+-- HostPortSet ::= SET OF HostPort
+
+-- HostPort ::= SEQUENCE {
+--   host [0] IMPLICIT GeneralString,
+--   port [1] IMPLICIT INTEGER
+-- }
+
+ShortColInfoSet ::= SET OF ShortColInfo
+
+ShortColInfo ::= SEQUENCE {
+  name GeneralString,
+  host GeneralString,
+  port INTEGER
+}
+
+-- EditAction ::= ENUMERATED {
+--   update(0), delete(1)
+-- }
+
+StringSequence ::= SEQUENCE OF GeneralString
+StringSet ::= SET OF GeneralString
+
+
+END
Index: /tags/gsdl-21/gsdl/src/recpt/comtypes.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/comtypes.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/comtypes.cpp	(revision 699)
@@ -0,0 +1,321 @@
+/**********************************************************************
+ *
+ * comtypes.cpp -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.20  1999/10/14 23:02:34  sjboddie
+   fixed bug in new parent metadata stuff
+
+   Revision 1.19  1999/10/10 07:49:35  sjboddie
+   - metadata now returns map rather than array
+   - redesigned browsing support (although it's not finished
+   so won't currently work ;-)
+
+   Revision 1.18  1999/09/07 04:56:54  sjboddie
+   added GPL notice
+
+   Revision 1.17  1999/09/02 00:25:26  rjmcnab
+   changes to get compiling on AIX
+
+   Revision 1.16  1999/08/13 04:19:05  sjboddie
+   added name attribute to metadata
+
+   Revision 1.15  1999/08/03 03:31:43  sjboddie
+   added ability to set receptionist from configuration files
+
+   Revision 1.14  1999/07/16 03:41:29  sjboddie
+   changed isApprox
+
+   Revision 1.13  1999/07/08 20:46:45  rjmcnab
+   Added the result number to the ResultDocInfo_t
+
+   Revision 1.12  1999/07/07 06:11:33  rjmcnab
+   Slight change for multiple phrase matching.
+
+   Revision 1.11  1999/07/01 22:47:49  sjboddie
+   added format option to collection info
+
+   Revision 1.10  1999/06/29 22:05:01  rjmcnab
+   Added a couple of fields to ResultDocInfo_t to handle a special
+   version of mg.
+
+   Revision 1.9  1999/06/16 02:01:22  sjboddie
+   Few changes to get getParents filter option to return metadata of parents
+   as well as current OID
+
+   Revision 1.8  1999/05/10 03:40:33  sjboddie
+   lots of changes - slowly getting document action sorted out
+
+   Revision 1.7  1999/04/21 05:23:14  sjboddie
+
+   Changed the way metadata is returned
+
+   Revision 1.6  1999/04/06 22:20:34  rjmcnab
+   Got browsefilter working.
+
+   Revision 1.5  1999/03/31 23:44:47  rjmcnab
+   Altered the protocol so that the metadata is part of the filter.
+
+   Revision 1.4  1999/03/11 00:11:19  rjmcnab
+   Added a function to get a string version of comerror_t
+
+   Revision 1.3  1999/03/09 21:00:47  rjmcnab
+   Reorganised the statusaction, added more functions to comtypes.
+
+   Revision 1.2  1999/03/04 21:19:15  rjmcnab
+
+   Added clear() functions to all the data types.
+
+   Revision 1.1  1999/02/21 22:35:25  rjmcnab
+
+   Initial revision.
+
+ */
+
+
+#include "comtypes.h"
+
+
+text_t get_comerror_string (comerror_t err) {
+  if (err == noError) return "no error";
+  else if (err == authenticationFailure) return "authentication failure";
+  else if (err == protocolError) return "protocol error";
+  else if (err == configurationError) return "configuration error";
+  else if (err == systemProblem) return "system problem";
+
+  return "unknown problem";
+}
+
+
+void ShortColInfo_t::clear() {
+  name.clear();
+  host.clear();
+  port = 0;
+}
+
+void ColInfoResponse_t::clear () {
+  shortInfo.clear();
+  isPublic=false;
+  isBeta=false;
+  buildDate=0;
+  numDocs=0;
+  numWords=0;
+  numBytes=0;
+  format.erase(format.begin(), format.end());
+  receptionist.clear();
+}
+
+void InfoFiltersResponse_t::clear () {
+  filterNames.erase(filterNames.begin(), filterNames.end());
+}
+
+
+void InfoFilterOptionsRequest_t::clear () {
+  filterName.clear();
+}
+
+
+void FilterOption_t::clear () {
+  name.clear();
+  type = booleant;
+  repeatable = onePerQuery;
+  defaultValue.clear();
+  validValues.erase(validValues.begin(), validValues.end());
+}
+
+void FilterOption_t::check_defaultValue () {
+  text_tarray::iterator here, end;
+
+  // how the default is interpreted depends on the option type
+  switch (type) {
+  case booleant: 
+  case enumeratedt: // has to be one of the validValues
+    here = validValues.begin ();
+    end = validValues.end ();
+    while (here != end) {
+      if (*here == defaultValue) return;
+      here++;
+    }
+    
+    break;
+
+  case integert: // has to be in the validValues range
+    if ((validValues.size() >= 2) &&
+	(validValues[0].getint() <= defaultValue.getint()) &&
+	(validValues[1].getint() >= defaultValue.getint()))
+      return;
+    break;
+
+  case stringt: // any value is valid
+    return; 
+  }
+
+  // did not find the defaultValue
+  if (validValues.empty()) defaultValue.clear();
+  else defaultValue = validValues[0];
+}
+
+
+bool operator==(const FilterOption_t &x, const FilterOption_t &y) {
+  return ((x.name == y.name) &&
+	  (x.type == y.type) &&
+	  (x.repeatable == y.repeatable) &&
+	  (x.defaultValue == y.defaultValue) &&
+	  (x.validValues == y.validValues));
+}
+
+bool operator<(const FilterOption_t &x, const FilterOption_t &y) {
+  return ((x.name < y.name) ||
+	  ((x.name == y.name) &&
+	   ((x.type < y.type) ||
+	    ((x.type == y.type) &&
+	     ((x.repeatable < y.repeatable) ||
+	      ((x.repeatable == y.repeatable) &&
+	       ((x.defaultValue < y.defaultValue) ||
+		((x.defaultValue == y.defaultValue) &&
+		 (x.validValues < y.validValues)))))))));
+}
+
+
+
+void InfoFilterOptionsResponse_t::clear () {
+  filterOptions.erase(filterOptions.begin(), filterOptions.end());
+}
+
+
+void OptionValue_t::clear () {
+  name.clear();
+  value.clear();
+}
+
+
+void FilterRequest_t::clear () {
+  filterName.clear();
+  filterOptions.erase(filterOptions.begin(), filterOptions.end());
+  docSet.erase(docSet.begin(), docSet.end());
+  filterResultOptions = 0;
+  requestParams.clear();
+  refParams.clear();
+  fields.erase(fields.begin(), fields.end());
+  getParents = false;
+}
+
+
+void TermInfo_t::clear () {
+  term.clear();
+  freq = 0;
+  matchTerms.erase (matchTerms.begin(), matchTerms.end());
+}
+
+
+void MetadataInfo_t::clear () {
+  params.clear();
+  isRef = false;
+  values.erase(values.begin(), values.end());
+  if (parent != NULL) {
+    delete parent;
+    parent = NULL;
+  }
+}
+
+MetadataInfo_t::MetadataInfo_t () {parent=NULL;clear();}
+
+// copy constructor
+MetadataInfo_t::MetadataInfo_t (const MetadataInfo_t &x) {
+  params = x.params;
+  isRef = x.isRef;
+  values = x.values;
+  if (x.parent == NULL) parent = NULL;
+  else {
+    parent = new MetadataInfo_t ();
+    *parent = *(x.parent);
+  }
+}
+
+MetadataInfo_t::~MetadataInfo_t () {
+  if (parent != NULL) {
+    delete parent;
+    parent = NULL;
+  }
+}
+
+MetadataInfo_t &MetadataInfo_t::operator=(const MetadataInfo_t &x) {
+  if (&x != this) {
+    params = x.params;
+    isRef = x.isRef;
+    values = x.values;
+    if (x.parent == NULL) parent = NULL;
+    else {
+      parent = new MetadataInfo_t ();
+      *parent = *(x.parent);
+    }
+  }
+  return *this;
+}
+
+void ResultDocInfo_t::clear () {
+  OID.clear ();
+  ranking = 0;
+  result_num = 0;
+  num_terms_matched = 0;
+  num_phrase_match = 0;
+  docFreq.erase(docFreq.begin(), docFreq.end());
+  metadata.erase(metadata.begin(), metadata.end());
+}
+
+ResultDocInfo_t &ResultDocInfo_t::operator=(const ResultDocInfo_t &x) {
+  OID=x.OID; 
+  ranking=x.ranking;
+  result_num=x.result_num;
+  num_terms_matched=x.num_terms_matched;
+  num_phrase_match = x.num_phrase_match;
+  docFreq=x.docFreq; 
+  metadata=x.metadata;
+  return *this;
+}
+
+
+void FilterResponse_t::clear () {
+  numDocs = 0;
+  isApprox = Exact;
+  termInfo.erase (termInfo.begin(), termInfo.end());
+  docInfo.erase (docInfo.begin(), docInfo.end());
+}
+
+
+void DocumentRequest_t::clear () {
+  OID.clear();
+  docType.clear();
+  docFormat.clear();
+}
+
+void DocumentResponse_t::clear () {
+  doc.clear();
+}
+
+
+
Index: /tags/gsdl-21/gsdl/src/recpt/comtypes.h
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/comtypes.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/comtypes.h	(revision 699)
@@ -0,0 +1,350 @@
+/**********************************************************************
+ *
+ * comtypes.h -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef COMTYPES_H
+#define COMTYPES_H
+
+#include "gsdlconf.h"
+#include "text_t.h"
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\vector>
+#  include <ospace\std\list>
+#  include <ospace\std\map>
+#elif defined(GSDL_USE_STL_H)
+#  include <vector.h>
+#  include <list.h>
+#  include <map.h>
+#else
+#  include <vector>
+#  include <list>
+#  include <map>
+#endif
+
+
+enum comerror_t {noError, authenticationFailure, protocolError, 
+		 configurationError, systemProblem};
+text_t get_comerror_string (comerror_t err);
+
+// ShortColInfo ::= SEQUENCE {
+//   name GeneralString,
+//   host GeneralString,
+//   port INTEGER
+// }
+struct ShortColInfo_t {
+  void clear ();
+  ShortColInfo_t () {clear();}
+
+  text_t name;
+  text_t host;
+  int port;
+};
+
+
+// ColInfoResponse ::= SEQUENCE {
+//   shortInfo     [0]  IMPLICIT ShortCollectionInfo,
+//   isPublic      [2]  IMPLICIT BOOLEAN,    -- whether has anonymous access
+//   isBeta        [3]  IMPLICIT BOOLEAN,    -- beta if still under development
+//   buildDate     [4]  IMPLICIT GeneralizedTime,
+//   languages     [5]  IMPLICIT StringSet,  -- languages in the collection
+//   numDocs       [6]  IMPLICIT INTEGER,
+//   numWords      [7]  IMPLICIT INTEGER OPTIONAL,
+//   numBytes      [8]  IMPLICIT INTEGER OPTIONAL
+//   format        [9]  IMPLICIT StringSet
+//   receptionist  [10] IMPLICIT GeneralString
+// }
+struct ColInfoResponse_t {
+  void clear ();
+  ColInfoResponse_t () {clear();}
+
+  ShortColInfo_t shortInfo;
+  bool isPublic;
+  bool isBeta;
+  unsigned long buildDate;
+  text_tarray languages;
+  unsigned long numDocs;  // 0 if not known
+  unsigned long numWords; // 0 if not known
+  unsigned long numBytes; // 0 if not known
+  text_tmap format;
+  text_t receptionist;
+};
+
+
+// -- filter options which might be supported for the QueryFilter
+// --
+// -- onePerQuery StartResults   integer
+// -- onePerQuery EndResults     integer
+// -- onePerQuery QueryType      enumerated (boolean, ranked)
+// -- onePerTerm  Term           string ???
+// -- onePerTerm  Casefold       boolean
+// -- onePerTerm  Stem           boolean
+// -- onePerTerm  Index          enumerated
+// -- onePerTerm  Subcollection  enumerated
+// --
+// -- filter options which might be supported for the BrowseFilter
+// --
+// -- onePerQuery StartResults   integer
+// -- onePerQuery EndResults     integer
+// -- onePerQuery ParentNode     string ("" will return the browsing available)
+// --
+// -- The NullFilter always returns the set it was given, it doesn't have
+// -- any options
+
+// InfoFiltersResponse ::= SEQUENCE {
+//   filterNames StringSet
+// }
+struct InfoFiltersResponse_t {
+  void clear ();
+
+  text_tset filterNames;
+};
+
+// InfoFilterOptionsRequest ::= SEQUENCE {
+//   filterName GeneralString
+// }
+struct InfoFilterOptionsRequest_t {
+  void clear ();
+
+  text_t filterName;
+};
+
+// FilterOption ::= SEQUENCE {
+//   name         GeneralString,
+//   type         ENUMERATED {booleant(0), integert(1), enumeratedt(2), stringt(3)},
+//   repeatable   ENUMERATED {onePerQuery(0), onePerTerm(1), nPerTerm(2)},
+//   defaultValue GeneralString,
+//   -- the interpretation of the validValues depends on the type
+//   -- for boolean: the first value is the false value, the second is true value
+//   -- for integer: the first value is the minimum, the second the maximum
+//   -- for enumerated: all values a listed
+//   -- for string: this value is ignored
+//   validValues  StringSequence
+// }
+struct FilterOption_t {
+  void clear ();
+  void check_defaultValue ();
+  FilterOption_t () {clear();}
+
+  text_t name;
+
+  enum type_t {booleant=0, integert=1, enumeratedt=2, stringt=3};
+  type_t type;
+
+  enum repeatable_t {onePerQuery=0, onePerTerm=1, nPerTerm=2};
+  repeatable_t repeatable;
+
+  text_t defaultValue;
+  text_tarray validValues;
+};
+
+bool operator==(const FilterOption_t &x, const FilterOption_t &y);
+bool operator<(const FilterOption_t &x, const FilterOption_t &y);
+
+
+typedef map<text_t, FilterOption_t, lttext_t> FilterOption_tmap;
+
+
+// InfoFilterOptionsResponse ::= SEQUENCE {
+//   filterOptions SET OF FilterOption
+// }
+struct InfoFilterOptionsResponse_t {
+  void clear ();
+
+  FilterOption_tmap filterOptions;
+};
+
+
+// OptionValue ::= SEQUENCE {
+//   name  GeneralString,
+//   value GeneralString
+// }
+struct OptionValue_t {
+  void clear ();
+
+  text_t name;
+  text_t value;
+};
+
+typedef vector<OptionValue_t> OptionValue_tarray;
+
+
+// -- Terms are presented in the same order that they are requested,
+// -- any information relating to the terms is in reference to the
+// -- index specified for that term.
+//
+// FilterRequest ::= SEQUENCE {
+//   filterName          [0] GeneralString,
+//   filterOptions       [1] IMPLICIT SEQUENCE OF OptionValue,
+//   docSet              [2] IMPLICIT StringSequence, -- the OID "" represents everything
+//   filterResultOptions [3] IMPLICIT BIT STRING {termFreq(0), matchTerms(1), OID(2), 
+//                                                subCol(3), ranking(4), docFreq(5), 
+//                                                metadata(6)}
+// 
+//   -- the next set of options are for the metadata request, 
+//   -- they can be left blank if metadata is not wanted
+//   requestParams       [4] IMPLICIT GeneralString, -- used to negotiate the metadata content
+//   refParams           [5] IMPLICIT GeneralString, -- used to decide whether to return a 
+//                                                   -- reference to the data or the actual data
+//   fields              [6] IMPLICIT StringSet
+//   getParents          [7] IMPLICIT BOOLEAN        -- gets metadata of all parents too
+// }
+#define FRtermFreq   1
+#define FRmatchTerms 2
+#define FROID        4
+#define FRsubCol     8
+#define FRranking    16
+#define FRdocFreq    32
+#define FRmetadata   64
+
+struct FilterRequest_t {
+  void clear ();
+  FilterRequest_t () {clear();}
+
+  text_t filterName;
+  OptionValue_tarray filterOptions;
+  text_tarray docSet;        // empty if not used
+  int filterResultOptions; // use the FR* defines above
+
+  text_t requestParams; // empty if not used
+  text_t refParams;     // empty if not used
+  text_tset fields;   // empty if not used
+  bool getParents;      // defaults to false
+};
+
+
+// TermInfo ::= SEQUENCE {
+//   term       [0] GeneralString,
+//   freq       [1] IMPLICIT INTEGER,       -- 0 if not requested
+//   matchTerms [2] IMPLICIT StringSequence -- empty if not requested
+// }
+struct TermInfo_t {
+  void clear ();
+  TermInfo_t () {clear();}
+
+  text_t term;
+  int freq; // 0 if not requested
+  text_tarray matchTerms; // empty if not requested
+};
+
+typedef vector<TermInfo_t> TermInfo_tarray;
+
+
+// MetadataInfo ::= SEQUENCE {
+//   params  [0] IMPLICIT GeneralString,
+//   isRef   [1] IMPLICIT BOOLEAN,
+//   values  [3] IMPLICIT SEQUENCE OF GeneralString,
+//   name    [4] IMPLICIT GeneralString
+// }
+struct MetadataInfo_t {
+  text_t params;
+  bool isRef;
+  text_tarray values;
+  MetadataInfo_t *parent;
+
+  void clear ();
+  MetadataInfo_t ();
+  MetadataInfo_t (const MetadataInfo_t &x); // copy constructor
+  ~MetadataInfo_t ();
+  MetadataInfo_t &operator=(const MetadataInfo_t &x);
+};
+
+typedef map<text_t, MetadataInfo_t, lttext_t> MetadataInfo_tmap;
+
+// ResultDocInfo ::= SEQUENCE {
+//   OID      [0] IMPLICIT GeneralString,
+//   ranking  [1] IMPLICIT INTEGER,             -- 0 if not requested, range 0-10000
+//   docFreq  [2] IMPLICIT SEQUENCE OF INTEGER, -- empty if not requested
+//   metadata [3] IMPLICIT SEQUENCE OF MetadataInfo -- no longer a SEQUENCE (SET maybe??)
+// }
+struct ResultDocInfo_t {
+  void clear ();
+  ResultDocInfo_t () {clear();}
+
+  text_t OID;
+  int result_num;          // place in results list
+  int ranking;             // 0 if not requested (real ranking*10000)
+  int num_terms_matched;   // not available on all versions of mg
+  int num_phrase_match;    // not available on all versions of mg
+
+  vector<int> docFreq; // empty if not requested
+  MetadataInfo_tmap metadata; // empty if not requested
+
+  ResultDocInfo_t &operator=(const ResultDocInfo_t &x);
+};
+
+typedef vector<ResultDocInfo_t> ResultDocInfo_tarray;
+
+
+// FilterResponse ::= SEQUENCE {
+//   numDocs   [0] IMPLICIT INTEGER,
+//   isApprox  [1] ENUMERATED {Exact(0), Approximate(1), MoreThan(2)}, -- whether numDocs is approximate
+//   termInfo  [2] IMPLICIT SEQUENCE OF TermInfo,                      -- empty if not required
+//   docInfo   [3] IMPLICIT SEQUENCE OF ResultDocInfo                  -- empty if not required
+// }
+
+enum isapprox {Exact=0, Approximate=1, MoreThan=2};
+
+struct FilterResponse_t {
+  void clear ();
+  FilterResponse_t () {clear();}
+
+  int numDocs;
+  isapprox isApprox;
+  TermInfo_tarray termInfo; // empty if not requested
+  ResultDocInfo_tarray docInfo; // empty if not requested
+};
+
+
+// DocumentRequest ::= SEQUENCE {
+//   OID       GeneralString,
+//   docType   GeneralString,
+//   docFormat GeneralString
+// }
+struct DocumentRequest_t {
+  void clear ();
+  DocumentRequest_t () {clear();}
+
+  text_t OID;
+  text_t docType;
+  text_t docFormat;
+};
+
+
+// DocumentResponse ::= SEQUENCE {
+//   doc OCTET STRING
+// }
+
+struct DocumentResponse_t {
+  void clear ();
+  DocumentResponse_t () {clear();}
+
+  text_t doc;
+};
+
+
+#endif
Index: /tags/gsdl-21/gsdl/src/recpt/converter.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/converter.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/converter.cpp	(revision 699)
@@ -0,0 +1,98 @@
+/**********************************************************************
+ *
+ * converter.cpp -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.4  1999/09/07 04:56:54  sjboddie
+   added GPL notice
+
+   Revision 1.3  1999/09/02 00:25:27  rjmcnab
+   changes to get compiling on AIX
+
+   Revision 1.2  1999/02/21 22:33:54  rjmcnab
+   Lots of stuff :-)
+
+   Revision 1.1  1999/02/05 10:42:45  rjmcnab
+
+   Continued working on receptionist
+
+ */
+
+
+#include "converter.h"
+
+
+
+// the converters within converterinfo become the property of
+// of this class after add_converter has been called. The converters
+// remain the responsability of the calling code and will not be
+// deleted by this class.
+void convertinfoclass::add_converter (const text_t &name, inconvertclass *inconverter,
+				      rzwsoutconvertclass *outconverter) {
+  // can't add a null converter or an action with no name
+  if (inconverter == NULL || outconverter == NULL || name.empty()) {
+    return;
+  }
+
+  converterinfo info;
+  info.name = name;
+  info.inconverter = inconverter;
+  info.outconverter = outconverter;
+  converters[name] = info;
+}
+
+
+bool operator==(const converterinfo &x, const converterinfo &y) {
+  return ((x.name == y.name) && (x.inconverter == y.inconverter) &&
+	  (x.outconverter == y.outconverter));
+}
+
+bool operator<(const converterinfo &x, const converterinfo &y) {
+  return ((x.name < y.name) ||
+	  ((x.name == y.name) &&
+	   ((x.inconverter < y.inconverter) ||
+	    ((x.inconverter == y.inconverter) &&
+	     ((x.outconverter < y.outconverter))))));
+}
+
+
+// get_inconverter will return NULL if the convert could not be found
+inconvertclass *convertinfoclass::get_inconverter (const text_t &name) {
+  iterator here = converters.find (name);
+  if (here == converters.end()) return NULL;
+
+  return (*here).second.inconverter;
+}
+
+// get_outconverter will return NULL if the convert could not be found
+rzwsoutconvertclass *convertinfoclass::get_outconverter (const text_t &name) {
+  iterator here = converters.find (name);
+  if (here == converters.end()) return NULL;
+
+  return (*here).second.outconverter;
+}
+
Index: /tags/gsdl-21/gsdl/src/recpt/converter.h
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/converter.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/converter.h	(revision 699)
@@ -0,0 +1,123 @@
+/**********************************************************************
+ *
+ * converter.h -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef CONVERTER_H
+#define CONVERTER_H
+
+
+#include "gsdlconf.h"
+#include "text_t.h"
+#include "gsdlunicode.h"
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\map>
+#elif defined(GSDL_USE_STL_H)
+#  include <map.h>
+#else
+#  include <map>
+#endif
+
+
+// inconverter and outconverter ARE NOT destroyed by this class
+// they should be destroyed by the code which created them after
+// no more pointers to the class exist.
+//
+// The above condition enables the class to be copied without
+// copying the converters which might contain a lot of data (and
+// be of an unknown derived type).
+class converterinfo {
+public:
+  text_t name;
+  inconvertclass *inconverter;
+  rzwsoutconvertclass *outconverter;
+
+  converterinfo() {inconverter=NULL; outconverter=NULL;}
+};
+
+bool operator==(const converterinfo &x, const converterinfo &y);
+bool operator<(const converterinfo &x, const converterinfo &y);
+
+
+typedef map<text_t, converterinfo, lttext_t> convertmap;
+
+// convertinfoclass is used to store a number of converters
+// indexing them by their name
+class convertinfoclass {
+protected:
+  convertmap converters;
+
+public:
+  // type support for convertmap
+  typedef convertmap::iterator iterator;
+  typedef convertmap::const_iterator const_iterator;
+  typedef convertmap::reference reference;
+  typedef convertmap::const_reference const_reference;
+  typedef convertmap::size_type size_type;
+  typedef convertmap::difference_type difference_type;
+  typedef convertmap::const_reverse_iterator const_reverse_iterator;
+  typedef convertmap::reverse_iterator reverse_iterator;
+  
+  // constructors
+  convertinfoclass () {}
+
+  // basic container support
+  iterator begin () {return converters.begin();}
+  const_iterator begin () const {return converters.begin();}
+  iterator end () {return converters.end();}
+  const_iterator end () const {return converters.end();}
+
+  void erase(iterator pos) {converters.erase(pos);}
+  void erase(iterator first, iterator last) {converters.erase(first, last);}
+  convertinfoclass &operator=(const convertinfoclass &x) 
+    {converters=x.converters;return *this;}
+
+  bool empty () const {return converters.empty();}
+  size_type size() const {return converters.size();}
+
+
+  // added functionality
+  void clear () {converters.erase(converters.begin(),converters.end());}
+
+  // the converters within converterinfo become the property of
+  // of this class after add_converter has been called. The converters
+  // remain the responsability of the calling code and will not be
+  // deleted by this class.
+  void add_converter (const text_t &name, inconvertclass *inconverter,
+		      rzwsoutconvertclass *outconverter);
+
+  // get_inconverter will return NULL if the convert could not be found
+  inconvertclass *get_inconverter (const text_t &name);
+
+  // get_outconverter will return NULL if the convert could not be found
+  rzwsoutconvertclass *get_outconverter (const text_t &name);
+};
+
+
+
+
+#endif
Index: /tags/gsdl-21/gsdl/src/recpt/datelistbrowserclass.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/datelistbrowserclass.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/datelistbrowserclass.cpp	(revision 699)
@@ -0,0 +1,147 @@
+/**********************************************************************
+ *
+ * datelistbrowserclass.cpp -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.2  1999/10/14 22:59:33  sjboddie
+   finished off browser classes
+
+   Revision 1.1  1999/10/10 08:14:05  sjboddie
+   - metadata now returns mp rather than array
+   - redesigned browsing support (although it's not finished so
+   won't currently work ;-)
+
+ */
+
+
+#include "datelistbrowserclass.h"
+#include <assert.h>
+
+
+datelistbrowserclass::datelistbrowserclass () {
+}
+
+datelistbrowserclass::~datelistbrowserclass () {
+}
+
+text_t datelistbrowserclass::get_browser_name () {
+  return "DateList";
+}
+
+void datelistbrowserclass::load_metadata_defaults (text_tset &metadata) {
+  metadata.insert ("Date");
+  metadata.insert ("doctype");
+}
+
+text_t datelistbrowserclass::get_default_formatstring () {
+  return "<td>[link][icon][/link]</td><td>[highlight]{Or}{[Title],Untitled}[/highlight]</td><td>[Date]</td>";
+}
+
+
+int datelistbrowserclass::output_section_group (FilterResponse_t &sections, cgiargsclass &args,
+						int colnumber, format_t *formatlistptr, 
+						bool use_table, text_tset &/*metadata*/, bool &/*getParents*/,
+						recptproto */*collectproto*/, displayclass &disp, 
+						outconvertclass &outconvert, ostream &textout, ostream &/*logout*/) {
+
+  text_t lastyear = "0000";
+  text_t lastmonth = "00";
+  text_t tabbing, endtabbing;
+
+  if (use_table || colnumber > 0) {
+    textout << "<table><tr><td>";
+    // get tab size
+    text_t tab; int itab;
+    disp.expandstring ("Global", "_tabwidth_", tab);
+    itab = tab.getint();
+    if (colnumber > 0) 
+      textout << "<img src=\"_httpimg_/space.gif\" width=" << (itab*colnumber) << ">";
+    textout<< "</td><td>";
+  }
+
+  textout << "<table>\n";
+
+  ResultDocInfo_tarray::iterator thissection = sections.docInfo.begin();
+  ResultDocInfo_tarray::iterator endsection = sections.docInfo.end();
+
+  while (thissection != endsection) {
+
+    text_t &doctype = (*thissection).metadata["doctype"].values[0];
+    text_t &date = (*thissection).metadata["Date"].values[0]; 
+
+    // bail on this document if it has no date
+    if (date.empty()) continue;
+
+    text_t::const_iterator datebegin = date.begin();
+    int datesize = date.size();
+    
+    if (datesize < 4) continue;
+    text_t thisyear = substr (datebegin, datebegin+4);
+    text_t thismonth = "00";
+    if (datesize >= 6)
+      thismonth = substr (datebegin+4, datebegin+6); 
+
+    text_t link = "<a href=\"_httpdocument_&cl=";
+    text_t icon = "_document:iconclosedbook_";
+      
+    if (doctype == "classify") {
+      icon = "_document:iconclosedbookshelf_";
+      link += (*thissection).OID + "\">";
+    } else link += args["cl"] + "&d=" + (*thissection).OID + "\">";
+
+    textout << "<tr valign=top>";
+
+    if (thisyear != lastyear) {
+      textout << outconvert << "<td><b>" << thisyear << "</b></td>";
+      lastyear = thisyear;
+    } else
+      textout << "<td></td>";
+    
+    if (thismonth != lastmonth) {
+      textout << outconvert << disp << ("<td><b>_textmonth" + thismonth + "_</b></td>");
+      lastmonth = thismonth;
+    } else
+      textout << "<td></td>";
+    
+    if (!use_table) textout << "<td>\n";
+    
+    textout << outconvert << disp 
+            << get_formatted_string (*thissection, formatlistptr, link, icon) << "\n";
+    
+    if (!use_table) textout << "</td>";
+    
+    textout << "</tr>\n";
+    
+    thissection ++;
+  }
+
+  textout << "</table>\n";
+
+  if (use_table  || colnumber > 0) textout << "</td></tr></table>\n";
+  return 1;
+}
+
Index: /tags/gsdl-21/gsdl/src/recpt/datelistbrowserclass.h
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/datelistbrowserclass.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/datelistbrowserclass.h	(revision 699)
@@ -0,0 +1,54 @@
+/**********************************************************************
+ *
+ * datelistbrowserclass.h -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef DATELISTBROWSERCLASS_H
+#define DATELISTBROWSERCLASS_H
+
+#include "browserclass.h"
+
+class datelistbrowserclass : public browserclass {
+protected:
+
+public:
+  datelistbrowserclass ();
+  virtual ~datelistbrowserclass ();
+
+  text_t get_browser_name ();
+
+  void load_metadata_defaults (text_tset &metadata);
+
+  text_t get_default_formatstring ();
+  
+  int output_section_group (FilterResponse_t &sections, cgiargsclass &args,
+			    int colnumber, format_t *formatlistptr, 
+			    bool use_table, text_tset &metadata, bool &getParents,
+			    recptproto *collectproto, displayclass &disp, 
+			    outconvertclass &outconvert, ostream &textout, ostream &logout);
+};
+
+#endif
Index: /tags/gsdl-21/gsdl/src/recpt/documentaction.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/documentaction.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/documentaction.cpp	(revision 699)
@@ -0,0 +1,878 @@
+/**********************************************************************
+ *
+ * documentaction.cpp -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.27  1999/10/14 23:00:53  sjboddie
+   finished changes to browsing support
+
+   Revision 1.26  1999/10/10 08:14:06  sjboddie
+   - metadata now returns mp rather than array
+   - redesigned browsing support (although it's not finished so
+   won't currently work ;-)
+
+   Revision 1.25  1999/09/17 04:46:05  sjboddie
+   fixed a couple of problems with 'unknown' classifier
+
+   Revision 1.24  1999/09/17 04:00:26  sjboddie
+   tidied up navigation bar to produce a default "unknown" button if
+   it comes across metadata it can't display
+
+   Revision 1.23  1999/09/07 04:56:54  sjboddie
+   added GPL notice
+
+   Revision 1.22  1999/08/20 01:00:22  sjboddie
+   split some of the larger functions up to simplify the creation of
+   collection specific receptionists
+
+   Revision 1.21  1999/08/11 23:28:03  sjboddie
+   -added support for html classifier
+   - removed _random_ macro - now use _blank for detach page
+
+   Revision 1.20  1999/08/10 23:16:39  sjboddie
+   a couple of small changes to get format options DocumentArrowsBottom
+   and DocumentArrowsTop working correctly
+
+   Revision 1.19  1999/08/10 22:40:01  sjboddie
+   added some more format options including the ability to format the
+   document text
+
+   Revision 1.18  1999/08/09 04:25:18  sjboddie
+   moved OID translation stuff from documentaction::define_external_macros
+   to receptionist
+
+   Revision 1.17  1999/08/09 02:13:53  sjboddie
+   fixed small bug in ShowSectionTitles
+
+   Revision 1.16  1999/08/03 03:30:47  sjboddie
+   a few small changes related to new format options
+
+   Revision 1.15  1999/07/30 02:18:49  sjboddie
+   -added collectinfo argument to some functions
+   -caught up with changes to browsetools (allowing nested classifications)
+   -added showtoppage format option
+
+   Revision 1.14  1999/07/21 05:01:09  sjboddie
+   fixed up wrapping of classificationlinks
+
+   Revision 1.13  1999/07/20 02:59:54  sjboddie
+   List and AZList classifications now use format strings
+
+   Revision 1.12  1999/07/16 00:19:01  sjboddie
+   some changes to the way quoted queries are handled
+
+   Revision 1.11  1999/07/07 05:47:41  sjboddie
+   changed around the way browsetools works
+
+   Revision 1.10  1999/06/29 01:46:44  sjboddie
+   now sets a _navigationbar_ macro even if there aren't any
+   classifications (it will just have the 'search' button
+
+   Revision 1.9  1999/06/27 22:05:58  sjboddie
+   now set a _thisOID_ macro for use in displaying images
+
+   Revision 1.8  1999/06/24 05:12:20  sjboddie
+   lots of small changes
+
+   Revision 1.7  1999/06/17 03:06:58  sjboddie
+   got detach button working properly - the close book icon is now disabled
+   when page is detached as the javascript close() function I was using is
+   too unreliable over different browsers
+   note that in my last comment I meant the "cl" arg (not the "c" arg).
+
+   Revision 1.6  1999/06/16 23:53:15  sjboddie
+   tidied a few things up. documentaction::define_external_macros now
+   resets the "c" arg if it's set to something stupid by the .xx suffixes
+
+   Revision 1.5  1999/06/16 03:10:49  sjboddie
+   define_internal_macros() now sets _pagetitle_ macro to document's title
+   (including parents of current document)
+
+   Revision 1.4  1999/06/15 01:55:29  sjboddie
+   - got text highlighting working
+   - got multiple collections working (now outputs error message if an
+     attempt is made to get a document when the "c" arg isn't set.
+
+   Revision 1.3  1999/06/08 04:29:37  sjboddie
+   added argsinfo to the call to check_cgiargs to make it easy to set
+   args to their default if they're found to be screwed up
+
+   Revision 1.2  1999/05/10 03:40:35  sjboddie
+   lots of changes - slowly getting document action sorted out
+
+   Revision 1.1  1999/04/30 01:59:40  sjboddie
+   lots of stuff - getting documentaction working (documentaction replaces
+   old browseaction)
+
+   Revision 1.2  1999/03/29 02:14:27  sjboddie
+
+   More changes to browseaction
+
+   Revision 1.1  1999/03/25 03:09:40  sjboddie
+
+   subjectbrowseaction became browseaction
+
+
+ */
+
+
+#include <string.h>
+#include "documentaction.h"
+#include "browsetools.h"
+#include "OIDtools.h"
+#include "querytools.h"
+#include "unitool.h"
+
+documentaction::documentaction () {
+
+  // this action uses cgi variables "a", "d", "cl",
+  // "x", "gc", "gt", "gp", and "hl"
+  cgiarginfo arg_ainfo;
+  arg_ainfo.shortname = "a";
+  arg_ainfo.longname = "action";
+  arg_ainfo.multiplechar = true;
+  arg_ainfo.defaultstatus = cgiarginfo::weak;
+  arg_ainfo.argdefault = "p";
+  arg_ainfo.savedarginfo = cgiarginfo::must;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+
+  arg_ainfo.shortname = "d";
+  arg_ainfo.longname = "document OID";
+  arg_ainfo.multiplechar = true;
+  arg_ainfo.defaultstatus = cgiarginfo::none;
+  arg_ainfo.argdefault = "";
+  arg_ainfo.savedarginfo = cgiarginfo::can;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+
+  arg_ainfo.shortname = "cl";
+  arg_ainfo.longname = "classification OID";
+  arg_ainfo.multiplechar = true;
+  arg_ainfo.defaultstatus = cgiarginfo::none;
+  arg_ainfo.argdefault = "";
+  arg_ainfo.savedarginfo = cgiarginfo::can;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+
+  // in this action "gc" controls the expand/contract
+  // contents function
+  arg_ainfo.shortname = "gc";
+  arg_ainfo.longname = "expand contents";
+  arg_ainfo.multiplechar = false;
+  arg_ainfo.defaultstatus = cgiarginfo::weak;
+  arg_ainfo.argdefault = "0";
+  arg_ainfo.savedarginfo = cgiarginfo::can;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+
+  // in this action "gt" controls the expand/contract
+  // text function 0 = not expanded, 1 = expand unless 
+  // there are more than 10 sections containing text,
+  // 2 = expand all
+  arg_ainfo.shortname = "gt";
+  arg_ainfo.longname = "expand text";
+  arg_ainfo.multiplechar = false;
+  arg_ainfo.defaultstatus = cgiarginfo::weak;
+  arg_ainfo.argdefault = "0";
+  arg_ainfo.savedarginfo = cgiarginfo::can;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+
+  // in this action "gp" is the "go to page" control
+  // used by the Book type of toc
+  arg_ainfo.shortname = "gp";
+  arg_ainfo.longname = "go to page";
+  arg_ainfo.multiplechar = true;
+  arg_ainfo.defaultstatus = cgiarginfo::none;
+  arg_ainfo.argdefault = "";
+  arg_ainfo.savedarginfo = cgiarginfo::mustnot;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+
+  // in this action "hl" is the "highlighting on/
+  // highlighting off control
+  arg_ainfo.shortname = "hl";
+  arg_ainfo.longname = "highlighting on/off";
+  arg_ainfo.multiplechar = false;
+  arg_ainfo.defaultstatus = cgiarginfo::weak;
+  arg_ainfo.argdefault = "1";
+  arg_ainfo.savedarginfo = cgiarginfo::must;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+
+  // "x" is 0 normally or 1 if page
+  // has been "detached"
+  arg_ainfo.shortname = "x";
+  arg_ainfo.longname = "detached page";
+  arg_ainfo.multiplechar = false;
+  arg_ainfo.defaultstatus = cgiarginfo::weak;
+  arg_ainfo.argdefault = "0";
+  arg_ainfo.savedarginfo = cgiarginfo::must;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+}
+
+documentaction::~documentaction () {
+}
+
+bool documentaction::check_cgiargs (cgiargsinfoclass &argsinfo, cgiargsclass &args, 
+				    ostream &logout) {
+
+  // check gc argument
+  int arg_gc = args.getintarg("gc");
+  if (arg_gc != 0 && arg_gc != 1) {
+    logout << "Warning: \"gc\" argument out of range (" << arg_gc << ")\n";
+    cgiarginfo *gcinfo = argsinfo.getarginfo ("gc");
+    if (gcinfo != NULL) args["gc"] = gcinfo->argdefault;
+  }
+
+  // check gt argument (may be either 0, 1 or 2)
+  int arg_gt = args.getintarg("gt");
+  if (arg_gt != 0 && arg_gt != 1 && arg_gt != 2) {
+    logout << "Warning: \"gt\" argument out of range (" << arg_gt << ")\n";
+    cgiarginfo *gtinfo = argsinfo.getarginfo ("gt");
+    if (gtinfo != NULL) args["gt"] = gtinfo->argdefault;
+  }
+
+  // check hl argument
+  int arg_hl = args.getintarg("hl");
+  if (arg_hl != 0 && arg_hl != 1) {
+    logout << "Warning: \"hl\" argument out of range (" << arg_hl << ")\n";
+    cgiarginfo *hlinfo = argsinfo.getarginfo ("hl");
+    if (hlinfo != NULL) args["hl"] = hlinfo->argdefault;
+  }
+
+  // check x argument
+  int arg_x = args.getintarg("x");
+  if (arg_x != 0 && arg_x != 1) {
+    logout << "Warning: \"x\" argument out of range (" << arg_x << ")\n";
+    cgiarginfo *xinfo = argsinfo.getarginfo ("x");
+    if (xinfo != NULL) args["x"] = xinfo->argdefault;
+  }
+
+  return true;
+}
+
+void documentaction::get_cgihead_info (cgiargsclass &/*args*/, response_t &response,
+				       text_t &response_data, ostream &/*logout*/) {
+  response = content;
+  response_data = "text/html";
+}
+
+// set_widthtspace calculates how wide the spaces in the nav bar should
+// be and sets the appropriate macro
+void documentaction::set_spacemacro (displayclass &disp, FilterResponse_t &response) {
+
+  text_t width;
+  int twidth, swidth, iwidth = 0;
+
+  int numc = response.docInfo.size();
+  ResultDocInfo_tarray::iterator dochere = response.docInfo.begin();
+  ResultDocInfo_tarray::iterator docend = response.docInfo.end();
+	
+  disp.expandstring ("Global", "_pagewidth_", width);
+  twidth = width.getint();
+	  
+  disp.expandstring ("query", "_searchwidth_", width);
+  iwidth += width.getint();
+  
+  while (dochere != docend) {
+    const text_t &title = (*dochere).metadata["Title"].values[0];
+
+    disp.expandstring ("document", "_" + title + "width_", width);
+    if (width == ("_" + title + "width_")) 
+      disp.expandstring ("document", "_defaultwidth_", width);
+    iwidth += width.getint();
+    dochere ++;
+  }
+  if ((twidth - iwidth) < numc) swidth = 2;
+  else {
+    swidth = twidth - iwidth;
+    if (numc > 0) swidth = swidth / numc;
+  }
+  disp.setmacro ("widthtspace", "Global", swidth);
+}
+
+// set_navbarmacros sets _navigationbar_, _javaimagesnavbar_ and _httpbrowseXXX_ macros
+// reponse contains 1 metadata field (Title)
+void documentaction::set_navbarmacros (displayclass &disp, FilterResponse_t &response, 
+				       cgiargsclass &args) {
+
+  text_t javaimagesnavbar, topparent;
+  text_t &arg_d = args["d"];
+  text_t navigationbar = "<!-- Navigation Bar -->\n";
+
+  get_top (args["cl"], topparent);
+  int numc = response.docInfo.size();
+  ResultDocInfo_tarray::iterator dochere = response.docInfo.begin();
+  ResultDocInfo_tarray::iterator docend = response.docInfo.end();
+
+  navigationbar += "<nobr>\n";
+  if (args["a"] == "q") {
+    navigationbar += "_icontabsearchgreen_";
+  } else {
+    navigationbar += "_imagesearch_";
+    javaimagesnavbar = "_javasearch_";
+  }
+  if (numc == 0) navigationbar += "_imagespacer_";
+	
+  while (dochere != docend) {
+    text_t title = (*dochere).metadata["Title"].values[0];
+
+    bool unknown = false;
+
+    // test the _XXXwidth_ macro to see if image macros are
+    // defined for this type of classification - if not we'll
+    // just display the text
+    text_t tmpwidth;
+    disp.expandstring ("document", "_" + title + "width_", tmpwidth);
+    if (tmpwidth == ("_" + title + "width_")) unknown = true;
+
+    // if we're inside a document all the classification buttons should be enabled
+    if (arg_d.empty() && ((*dochere).OID == topparent)) {
+      if (unknown) navigationbar += "_imagespacer_&nbsp;" + title + "&nbsp;";
+      else navigationbar += "_imagespacer__icontab" + title + "green_";
+    } else {
+
+      // set the _httpbrowseXXX_ macro for this classification
+      if (unknown) navigationbar += "_imagespacer_&nbsp;<a href=\"_httpdocument_&cl=" + 
+		     (*dochere).OID + "\">" + title + "</a>&nbsp;";
+      else {
+        navigationbar += "_imagespacer__image" + title + "_";
+        disp.setmacro ("httpbrowse" + title, "Global", "_httpdocument_&cl=" + (*dochere).OID);
+      }
+    }
+    if (!unknown) javaimagesnavbar += "_java" + title + "_";
+    dochere ++;
+  }
+  navigationbar += "\n</nobr>\n";
+  navigationbar += "<!-- End of Navigation Bar -->\n";
+  disp.setmacro ("navigationbar", "Global", navigationbar);
+  if (args.getintarg("v") == 0)
+    disp.setmacro ("javaimagesnavbar", "Global", javaimagesnavbar);
+}
+
+// define all the macros which might be used by other actions
+// to produce pages. 
+void documentaction::define_external_macros (const ColInfoResponse_t &/*collectinfo*/, 
+					     displayclass &disp, cgiargsclass &args, 
+					     recptproto *collectproto, ostream &logout) {
+  
+  // define_external_macros sets the following macros:
+
+  // _navigationbar_     this is the navigation bar containing the search button
+  //                     and any classification buttons - it goes at the top of 
+  //                     most pages. for now we're assuming that there'll always 
+  //                     be a search button - we should probably check that there
+  //                     is a query action before making this assumption
+
+  // _httpbrowseXXX_     the http macros for each classification (i.e. if there
+  //                     are Title and Creator classifications _httpbrowseTitle_
+  //                     and _httpbrowseCreator_ will be set
+
+  // _javaimagesnavbar_  this is the javascript code to shove in to make the 
+  //                     flashy images used by _navigationbar_ work
+
+  // _widthtspace_       the width of the spacers between buttons in navigation
+  //                     bar
+  
+
+  // can't do anything if collectproto is null (i.e. no collection was specified)
+  if (collectproto == NULL) return;
+
+  outconvertclass text_t2ascii;
+  comerror_t err;
+  InfoFiltersResponse_t filterinfo;
+  FilterResponse_t response;
+  text_tset metadata;
+  text_t &collection = args["c"];
+
+  // don't want navigation bar if page is 'detached'
+  if (!args.getintarg("x")) {
+
+    collectproto->get_filterinfo (collection, filterinfo, err, logout);
+    if (err == noError) {
+      // check that there's a browse filter
+      if (filterinfo.filterNames.find ("BrowseFilter") != filterinfo.filterNames.end()) {
+	
+	metadata.insert ("Title");
+	bool getParents = false;
+	get_children ("", collection, metadata, getParents, collectproto, response, logout);
+	
+	// calculate width of spacers and set _widthtspace_ macro
+	if (args.getintarg("v") == 0) set_spacemacro (disp, response);
+
+	// set _navigationbar_ and _javaimagesnavbar_ macros
+	set_navbarmacros (disp, response, args);
+      }
+    } else {
+      logout << text_t2ascii 
+	     << "Error (documentaction::define_external_macros()) in call to get_filterinfo() " 
+	     << get_comerror_string (err);
+    }
+  }
+}
+
+void documentaction::load_formatinfo (const text_tmap &colformat, int gt) {
+
+  formatinfo.clear();
+  text_tmap::const_iterator format_here = colformat.begin();
+  text_tmap::const_iterator format_end = colformat.end();
+
+  while (format_here != format_end) {
+    if (((*format_here).first == "DocumentImages") && 
+	((*format_here).second == "true"))
+      formatinfo.DocumentImages = true;
+    else if (((*format_here).first == "DocumentTitles") && 
+	     ((*format_here).second == "false"))
+      formatinfo.DocumentTitles = false;
+    else if ((*format_here).first == "DocumentHeading") 
+      formatinfo.DocumentHeading = (*format_here).second;
+    else if (((*format_here).first == "DocumentContents") && 
+	     ((*format_here).second == "false"))
+      formatinfo.DocumentContents = false;
+    else if (((*format_here).first == "DocumentArrowsBottom") && 
+	     ((*format_here).second == "false"))
+      formatinfo.DocumentArrowsBottom = false;
+    else if ((*format_here).first == "DocumentButtons")
+      splitchar ((*format_here).second.begin(), (*format_here).second.end(),
+		 '|', formatinfo.DocumentButtons);
+    else if ((*format_here).first == "DocumentText") 
+      formatinfo.DocumentText = (*format_here).second;
+    else 
+      formatinfo.formatstrings[(*format_here).first] = (*format_here).second;
+
+    format_here ++;
+  }
+
+  // never want arrows when text is expanded
+  if (gt)
+    formatinfo.DocumentArrowsBottom = false;
+}
+
+void documentaction::set_java_macros (cgiargsclass &args, displayclass &disp) {
+
+  text_t javaimagescontent = "_javaextras_";
+	
+  int arg_gt = args.getintarg("gt");
+  int arg_gc = args.getintarg("gc");
+  int arg_hl = args.getintarg("hl");
+
+  text_tarray::const_iterator button_here = formatinfo.DocumentButtons.begin();
+  text_tarray::const_iterator button_end = formatinfo.DocumentButtons.end();
+
+  while (button_here != button_end) {
+    if (*button_here == "Detach") 
+      javaimagescontent += "_javadetach_";
+    else if (*button_here == "Expand Text") {
+      if (arg_gt == 1)
+	javaimagescontent += "_javacontracttext__javacontinue_";
+      else if (arg_gt == 2)
+	javaimagescontent += "_javacontracttext_";
+      else
+	javaimagescontent += "_javaexpandtext_";
+    } else if (*button_here == "Expand Contents") {
+      if (arg_gc == 1)
+	javaimagescontent += "_javacontractcontents_";
+      else
+	javaimagescontent += "_javaexpandcontents_";
+    } else if (*button_here == "Highlight") {
+      if (arg_hl == 1)
+	javaimagescontent += "_javanohighlighting_";
+      else 
+	javaimagescontent += "_javahighlighting_";
+    }
+    button_here ++;
+  }
+  disp.setmacro ("javaimagescontent", "document", javaimagescontent);
+}
+
+
+// define all the macros which are related to pages generated
+// by this action. we also load up the formatinfo structure 
+// here (it's used in do_action as well as here)
+void documentaction::define_internal_macros (const ColInfoResponse_t &collectinfo, 
+					     displayclass &disp, cgiargsclass &args, 
+					     recptproto *collectproto, ostream &logout) {
+  
+  // define_internal_macros sets the following macros:
+  
+  // _pagetitle_            the title to be displayed at the top of the browser window
+  
+  // _imagethispage_        the title image to be displayed at top right of page
+
+  // _navarrows_            this may be overridden to "" when format option 
+  //                        DocumentArrowsBottom is false
+
+  // _header_               the header macro is overridden if we're not at a top level 
+  //                        classification to remove the title block
+
+  // _javaimagescontent_    this is the javascript code to shove in to make the 
+  //                        flashy buttons work
+
+  // _thisOID_              the OID (directory) of the current document - this corresponds
+  //                        to the archivedir metadata element
+
+  // can't do anything if collectproto is null (i.e. no collection was specified)
+  if (collectproto == NULL) return;
+  
+  text_tset metadata;
+  FilterResponse_t response;
+  text_t &arg_d = args["d"];
+  text_t &arg_cl = args["cl"];
+  text_t &collection = args["c"];
+  int gt = args.getintarg("gt");
+  load_formatinfo (collectinfo.format, gt);
+
+  if (!formatinfo.DocumentArrowsBottom)
+    disp.setmacro("navarrows", "document", "");
+
+  metadata.insert ("Title");
+  
+  if (!arg_d.empty()) {
+    // we're at document level
+
+    metadata.insert ("archivedir");
+    
+    // get metadata for this document and it's parents
+    if (get_info (arg_d, collection, metadata, true, collectproto, response, logout)) {
+      disp.setmacro ("header", "document", "_textheader_");
+
+      text_tarray pagetitlearray;
+      if (!response.docInfo[0].metadata["Title"].values[0].empty())
+	pagetitlearray.push_back (response.docInfo[0].metadata["Title"].values[0]);
+      if (!gt) {
+	MetadataInfo_t *parenttitle = response.docInfo[0].metadata["Title"].parent;
+	while (parenttitle != NULL) {
+	  if (!parenttitle->values[0].empty())
+	    pagetitlearray.push_back (parenttitle->values[0]);
+	  parenttitle = parenttitle->parent;
+	}
+      }
+      reverse (pagetitlearray.begin(), pagetitlearray.end());
+      text_t pagetitle;
+      joinchar (pagetitlearray, ": ", pagetitle);
+      disp.setmacro ("pagetitle", "document", pagetitle);
+
+      if (is_top (arg_d))
+	disp.setmacro ("thisOID", "Global", response.docInfo[0].metadata["archivedir"].values[0]);
+      else {
+	MetadataInfo_t *parentad = response.docInfo[0].metadata["archivedir"].parent;
+	text_t thisOID;
+	while (parentad != NULL) {
+	  thisOID = parentad->values[0];
+	  parentad = parentad->parent;
+	}
+	disp.setmacro ("thisOID", "Global", thisOID);
+      }
+      
+      if (args["u"] != "1")
+	set_java_macros (args, disp);
+
+    }
+  } else {
+    if (!arg_cl.empty()) {
+	
+      // get metadata for top level classification
+      text_t classtop;
+      get_top (arg_cl, classtop);
+      if (get_info (classtop, collection, metadata, false, collectproto, response, logout)) {
+	  
+	text_t &title = response.docInfo[0].metadata["Title"].values[0];
+	bool unknown = false;
+
+	// test the _XXXwidth_ macro to see if image macros are
+	// defined for this type of classification - if not we'll
+	// just display the text
+	text_t tmp;
+	disp.expandstring ("document", "_" + title + "width_", tmp);
+	if (tmp == ("_" + title + "width_")) unknown = true;
+
+	if (unknown) {
+	  disp.setmacro ("pagetitle", "document", title);
+	  disp.setmacro ("imagethispage", "document", "<h2>" + title + "</h2>");
+	} else {
+	  disp.setmacro ("pagetitle", "document", "_text" + title + "page_");
+	  disp.setmacro ("imagethispage", "document", "_icon" + title + "page_");
+	}
+      }
+    }
+  }
+}
+
+
+bool documentaction::do_action (cgiargsclass &args, const ColInfoResponse_t &/*collectinfo*/,
+				recptproto *collectproto, displayclass &disp, 
+				outconvertclass &outconvert, ostream &textout, 
+				ostream &logout) {
+
+  if (collectproto == NULL) {
+    logout << "documentaction::do_action called with NULL collectproto\n";
+    textout << outconvert << disp << "_document:header_\n" 
+	    << "Error: Attempt to get document without setting collection\n"
+	    << "_document:footer_\n";
+  } else {  
+
+    text_t OID = args["d"];
+    if (OID.empty()) OID = args["cl"];
+    if (OID.empty()) {
+      textout << outconvert << disp << "Document contains no data_document:footer_\n";
+      return true;
+    }
+
+    textout << outconvert << disp << "_document:header_\n"
+	    << "_document:content_\n";
+    
+    // output the table of contents
+    output_toc (args, recpt->get_browsermap_ptr(), formatinfo, 
+		collectproto, disp, outconvert, textout, logout);
+
+    // output the document text
+    textout << "<p>\n";
+    output_document (OID, args, collectproto, disp, outconvert, textout, logout);
+
+    textout << outconvert << disp << "_document:footer_\n";
+  }
+  return true;
+}
+
+void documentaction::output_text (ResultDocInfo_t &docinfo, format_t *formatlistptr,
+				  const TermInfo_tarray &terminfo, const text_t &OID,
+				  bool highlight, int hastxt, int wanttext, 
+				  text_t &collection, recptproto *collectproto, 
+				  displayclass &disp, outconvertclass &outconvert, 
+				  ostream &textout, ostream &logout) {
+  
+  DocumentRequest_t docrequest;
+  DocumentResponse_t docresponse;
+  comerror_t err;
+
+  if (hastxt == 1) {
+
+    if (wanttext) {
+      // get the text
+      docrequest.OID = OID;
+      collectproto->get_document (collection, docrequest, docresponse, err, logout);
+
+      // cut down on overhead by not using formattools if we only want the text
+      // (wanttext will equal 2 if we want text and other stuff too)
+      if (wanttext == 1)
+	if (highlight)
+	  highlighttext(docresponse.doc, terminfo, disp, outconvert, textout, logout);
+	else
+	  textout << outconvert << disp << docresponse.doc;
+    }
+  
+    if (wanttext != 1) {
+      text_t doctext = get_formatted_string (docinfo, formatlistptr, docresponse.doc); 
+      
+      if (highlight)
+	highlighttext(doctext, terminfo, disp, outconvert, textout, logout);
+      else
+	textout << outconvert << disp << doctext;
+    }
+  }
+}
+
+
+void documentaction::output_document (const text_t &OID, cgiargsclass &args, 
+				      recptproto *collectproto, displayclass &disp, 
+				      outconvertclass &outconvert, ostream &textout, 
+				      ostream &logout) {
+  FilterResponse_t inforesponse;
+  FilterResponse_t queryresponse;
+  text_tset metadata;
+  bool getParents = false;
+  bool highlight = false;
+  int wanttext = 0;
+  int arg_gt = args.getintarg("gt");
+  text_t &collection = args["c"];
+    
+  // if we have a query string and highlighting is turned on we need
+  // to redo the query to get the terms for highlighting
+  if (!args["q"].empty() && args.getintarg("hl")) {
+    FilterRequest_t request;
+    request.filterResultOptions = FRmatchTerms;
+    if (do_query (request, args, collectproto, queryresponse, logout))
+      highlight = true;
+  }
+
+  format_t *formatlistptr = new format_t();
+  parse_formatstring (formatinfo.DocumentText, formatlistptr, metadata, getParents);
+
+  metadata.insert ("hastxt");
+  metadata.insert ("haschildren");
+  
+  if (formatinfo.DocumentText == "[Text]")
+    wanttext = 1;
+  else {
+    char *docformat = formatinfo.DocumentText.getcstr();
+    if (strstr (docformat, "[Text]") != NULL)
+      wanttext = 2;
+    delete docformat;
+  }
+    
+  if (get_info (OID, collection, metadata, getParents, collectproto, inforesponse, logout)) {
+    int hastxt = inforesponse.docInfo[0].metadata["hastxt"].values[0].getint();
+    int haschildren = inforesponse.docInfo[0].metadata["haschildren"].values[0].getint();
+    
+    if (arg_gt == 0) {
+      output_text (inforesponse.docInfo[0], formatlistptr, queryresponse.termInfo, 
+		   OID, highlight, hastxt, wanttext, collection, collectproto, 
+		   disp, outconvert, textout, logout);
+
+
+    } else {
+
+      ResultDocInfo_t thisdocinfo = inforesponse.docInfo[0];
+	
+      // text is to be expanded
+      text_t exOID = OID;
+      if (haschildren != 1) exOID = get_parent (OID);
+      if (exOID.empty()) exOID = OID;
+	
+      // if we're not in a document (i.e. we're in a top level classification)
+      // we need to pass "is_classify = true" to get_contents so that it 
+      // doesn't recurse all the way through each document in the classification
+      bool is_classify = false;
+      if (args["d"].empty()) is_classify = true;
+
+      get_contents (exOID, is_classify, metadata, collection, 
+		    collectproto, inforesponse, logout);
+	
+      ResultDocInfo_tarray::iterator sechere = inforesponse.docInfo.begin();
+      ResultDocInfo_tarray::iterator secend = inforesponse.docInfo.end();
+	
+      if (arg_gt == 1) {
+	// check if there are more than 10 sections containing text to be expanded - 
+	// if there are output warning message - this isn't a great way to do this
+	// since the sections may be very large or very small - one day I'll fix it 
+	// -- Stefan.
+	int seccount = 0;
+	while (sechere != secend) {
+	  int shastxt = (*sechere).metadata["hastxt"].values[0].getint();
+	  if (shastxt == 1) seccount ++;
+	  if (seccount > 10) break;
+	  sechere ++;
+	}
+	if (seccount > 10) {
+	  // more than 10 sections so output warning message and text 
+	  // for current section only
+	  textout << outconvert << disp << "_document:textltwarning_";
+
+	  output_text (thisdocinfo, formatlistptr, queryresponse.termInfo, 
+		       OID, highlight, hastxt, wanttext, collection, 
+		       collectproto, disp, outconvert, textout, logout);
+	  
+	}
+	else arg_gt = 2;
+      }
+
+      if (arg_gt == 2) {
+	// get the text for each section
+	sechere = inforesponse.docInfo.begin();
+	int count = 0;
+	while (sechere != secend) {
+	  textout << outconvert << disp << "\n<p><a name=" << count << "></a>\n";
+
+	  int shastxt = (*sechere).metadata["hastxt"].values[0].getint();
+
+	  output_text (*sechere, formatlistptr, queryresponse.termInfo, 
+		       (*sechere).OID, highlight, shastxt, wanttext, collection, 
+		       collectproto, disp, outconvert, textout, logout);
+	  count ++;
+	  sechere ++;
+	}
+      }
+    }
+  }
+  delete formatlistptr;
+}
+
+// highlighttext highlights query terms in text string and outputs the resulting text string
+void documentaction::highlighttext(text_t &text, const TermInfo_tarray &terms, 
+				   displayclass &disp, outconvertclass &outconvert, 
+				   ostream &textout, ostream &/*logout*/) {
+
+  text_tmap allterms;
+  text_tmap::const_iterator it;
+
+  // first load all the term variations into a map
+  TermInfo_tarray::const_iterator this_term = terms.begin();
+  TermInfo_tarray::const_iterator last_term = terms.end();
+  while (this_term != last_term) {
+    text_tarray::const_iterator this_var = (*this_term).matchTerms.begin();
+    text_tarray::const_iterator last_var = (*this_term).matchTerms.end();
+    while (this_var != last_var) {
+      allterms[*this_var] = 1;
+      this_var ++;
+    }
+    this_term ++;
+  }
+
+  // get the text to start and end a hightlight
+  text_t starthighlight = "<b><u>";
+  text_t endhighlight = "</u></b>";
+  if (disp.isdefaultmacro("Global", "starthighlight"))
+    disp.expandstring("Global", "_starthighlight_", starthighlight);
+  if (disp.isdefaultmacro("Global", "endhighlight"))
+    disp.expandstring("Global", "_endhighlight_", endhighlight);
+
+
+  text_t::iterator here = text.begin(); 
+  text_t::iterator end = text.end();
+  text_t word, buffer;
+  while (here != end) {
+    if (is_unicode_letdig(*here)) {
+      // not word boundary
+      word.push_back(*here);
+      here++;
+
+    } else {
+      // found word boundary
+      // add last word if there was one
+      if (!word.empty()) {
+	it = allterms.find(word);
+	if (it != allterms.end()) {
+	  word = starthighlight + word + endhighlight;
+	}
+	buffer += word;
+        word.clear();
+      }
+
+      if (*here == '<') {
+        // skip over rest of html tag
+	while ((here != end) && (*here != '>')) {
+	  buffer.push_back(*here);
+	  here++;
+	}
+      }
+
+      buffer.push_back(*here);
+      here++;
+
+      if (buffer.size() > 1024) {
+	textout << outconvert << disp << buffer;
+	buffer.clear();
+      }
+    }
+  }
+  textout << outconvert << disp << buffer;
+}
Index: /tags/gsdl-21/gsdl/src/recpt/documentaction.h
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/documentaction.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/documentaction.h	(revision 699)
@@ -0,0 +1,107 @@
+/**********************************************************************
+ *
+ * documentaction.h -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef DOCUMENTACTION_H
+#define DOCUMENTACTION_H
+
+#include "action.h"
+#include "gsdlconf.h"
+#include "formattools.h"
+#include "receptionist.h"
+
+class documentaction : public action {
+
+protected:
+
+  receptionist *recpt;
+
+  void load_formatinfo (const text_tmap &colformat, int gt);
+
+  void set_spacemacro (displayclass &disp, FilterResponse_t &response);
+
+  void set_navbarmacros (displayclass &disp, FilterResponse_t &response, 
+			 cgiargsclass &args);
+
+  void set_java_macros (cgiargsclass &args, displayclass &disp);
+
+  void get_classificationlinks (const text_t &arg_cl, const text_t &collection,
+				recptproto *collectproto, displayclass &disp, 
+				ostream &logout);
+
+  virtual void output_text (ResultDocInfo_t &docinfo, format_t *formatlistptr,
+			    const TermInfo_tarray &terminfo, const text_t &OID,
+			    bool highlight, int hastxt, int wanttext, 
+			    text_t &collection, recptproto *collectproto,
+			    displayclass &disp, outconvertclass &outconvert,
+			    ostream &textout, ostream &logout);
+
+  virtual void output_document (const text_t &OID, cgiargsclass &args, 
+				recptproto *collectproto, displayclass &disp, 
+				outconvertclass &outconvert, ostream &textout, 
+				ostream &logout);
+
+  void highlighttext(text_t &text, const TermInfo_tarray &terms, 
+		     displayclass &disp, outconvertclass &outconvert, 
+		     ostream &textout, ostream &logout);
+
+  formatinfo_t formatinfo;
+
+public:
+  documentaction ();
+  virtual ~documentaction ();
+
+  text_t get_action_name () {return "d";}
+
+  void set_receptionist (receptionist *therecpt) {recpt=therecpt;}
+
+  bool check_cgiargs (cgiargsinfoclass &argsinfo, cgiargsclass &args, 
+		      ostream &logout);
+
+  void get_cgihead_info (cgiargsclass &args, response_t &response,
+                         text_t &response_data, ostream &logout);
+
+  void define_external_macros (const ColInfoResponse_t &collectinfo, 
+			       displayclass &disp, cgiargsclass &args, 
+			       recptproto *collectproto, ostream &logout);
+
+  void define_internal_macros (const ColInfoResponse_t &collectinfo, 
+			       displayclass &disp, cgiargsclass &args, 
+			       recptproto *collectproto, ostream &logout);
+
+  bool do_action (cgiargsclass &args, const ColInfoResponse_t &collectinfo, 
+		  recptproto *collectproto, displayclass &disp, 
+		  outconvertclass &outconvert, ostream &textout, 
+		  ostream &logout);
+
+};
+
+
+
+#endif
+
+
Index: /tags/gsdl-21/gsdl/src/recpt/formattools.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/formattools.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/formattools.cpp	(revision 699)
@@ -0,0 +1,656 @@
+/**********************************************************************
+ *
+ * formattools.cpp -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.13  1999/10/14 23:01:24  sjboddie
+   changes for new browsing support
+
+   Revision 1.12  1999/10/10 08:14:07  sjboddie
+   - metadata now returns mp rather than array
+   - redesigned browsing support (although it's not finished so
+   won't currently work ;-)
+
+   Revision 1.11  1999/09/28 20:38:19  rjmcnab
+   fixed a couple of bugs
+
+   Revision 1.10  1999/09/07 04:56:55  sjboddie
+   added GPL notice
+
+   Revision 1.9  1999/09/02 00:31:25  rjmcnab
+   fixed small error.
+
+   Revision 1.8  1999/08/20 00:56:38  sjboddie
+   added cgisafe option - you can now do something like [cgisafe:Title] if
+   you want Title to be entered safely into a url
+
+   Revision 1.7  1999/08/10 22:38:08  sjboddie
+   added some more format options
+
+   Revision 1.6  1999/07/30 02:25:42  sjboddie
+   made format_date function global
+
+   Revision 1.5  1999/07/21 05:00:00  sjboddie
+   added some date formatting
+
+   Revision 1.4  1999/07/20 03:02:15  sjboddie
+   added an [icon] option, added ability to call get_formatted_string
+   with icon and link arguments set
+
+   Revision 1.3  1999/07/09 02:44:35  sjboddie
+   fixed parent(All) function so it only outputs parents and not current
+   level meta
+
+   Revision 1.2  1999/07/08 20:48:33  rjmcnab
+   Added ability to print the result number
+
+   Revision 1.1  1999/07/07 05:49:34  sjboddie
+   had another crack at the format string code - created a new formattools
+   module. It can now handle {If} and {Or} statements although there's a
+   bug preventing nested if's and or's.
+
+ */
+
+
+#include "formattools.h"
+#include "cgiutils.h"
+
+// a few function prototypes
+static text_t format_string (ResultDocInfo_t &docinfo, format_t *formatlistptr,
+			     const text_t &link, const text_t &icon, 
+			     const text_t &text, bool highlight);
+
+static bool parse_action (text_t::const_iterator &here, const text_t::const_iterator &end, 
+			  format_t *formatlistptr, text_tset &metadata, bool &getParents);
+
+void metadata_t::clear() {
+  metaname.clear();
+  metacommand = mNone;
+  parentcommand = pNone;
+  parentoptions.clear();
+};
+
+void decision_t::clear() {
+  command = dMeta;
+  meta.clear();
+};
+
+void format_t::clear() {
+  command = comText;
+  decision.clear();
+  text.clear();
+  meta.clear();
+  nextptr = NULL;
+  ifptr = NULL;
+  elseptr = NULL;
+  orptr = NULL;
+};
+
+void formatinfo_t::clear() {
+  DocumentImages = false;
+  DocumentTitles = true;
+  DocumentHeading = "{Or}{[parent(Top):Title],[Title],untitled}<br>";
+  DocumentContents = true;
+  DocumentArrowsBottom = true;
+  DocumentButtons.erase (DocumentButtons.begin(), DocumentButtons.end());
+  DocumentButtons.push_back ("Detach");
+  DocumentButtons.push_back ("Highlight");
+  DocumentButtons.push_back ("Expand Text");
+  DocumentButtons.push_back ("Expand Contents");
+  DocumentText = "[Text]";
+  formatstrings.erase (formatstrings.begin(), formatstrings.end());
+}
+
+// returns false if key isn't in formatstringmap
+bool get_formatstring (const text_t &key, const text_tmap &formatstringmap,
+		       text_t &formatstring) {
+
+  formatstring.clear();
+  text_tmap::const_iterator it = formatstringmap.find(key);
+  if (it == formatstringmap.end()) return false;
+  formatstring = (*it).second;
+  return true;
+}
+// tries to find "key1key2" then "key1" then "key2"
+bool get_formatstring (const text_t &key1, const text_t &key2,  
+		       const text_tmap &formatstringmap,
+		       text_t &formatstring) {
+
+  formatstring.clear();
+  text_tmap::const_iterator it = formatstringmap.find(key1 + key2);
+  if (it != formatstringmap.end()) {
+    formatstring = (*it).second;
+    return true;
+  }
+  it = formatstringmap.find(key1);
+  if (it != formatstringmap.end()) {
+    formatstring = (*it).second;
+    return true;
+  }
+  it = formatstringmap.find(key2);
+  if (it != formatstringmap.end()) {
+    formatstring = (*it).second;
+    return true;
+  }
+  return false;
+}
+
+
+// returns a date of form _textmonthnn_ 31, 1999 
+// input is date of type 19991231
+// at least the year must be present in date
+text_t format_date (const text_t &date) {
+
+  if (date.size() < 4) return "";
+
+  text_t::const_iterator datebegin = date.begin();
+
+  text_t year = substr (datebegin, datebegin+4);
+
+  if (date.size() < 6) return year;
+
+  text_t month = "_textmonth" + substr (datebegin+4, datebegin+6) + "_";
+  int imonth = month.getint();
+  if (imonth < 0 || imonth > 12) return year;
+
+  if (date.size() < 8) return month + ", " + year;
+
+  text_t day = substr (datebegin+6, datebegin+8);
+  if (day[0] == '0') day = substr (day.begin()+1, day.end());
+  int iday = day.getint();
+  if (iday < 0 || iday > 31) return month + ", " + year;
+    
+  return month + " " + day + ", " + year;
+}  
+
+static void get_parent_options (text_t &instring, metadata_t &metaoption) {
+
+  text_t meta, com, op;
+  bool inbraces = false;
+  bool inquotes = false;
+  bool foundcolon = false;
+  text_t::const_iterator here = instring.begin()+6;
+  text_t::const_iterator end = instring.end();
+  while (here != end) {
+    if (*here == '(') inbraces = true;
+    else if (*here == ')') inbraces = false;
+    else if (*here == '\'' && !inquotes) inquotes = true;
+    else if (*here == '\'' && inquotes) inquotes = false;
+    else if (*here == ':' && !inbraces) foundcolon = true;
+    else if (foundcolon) meta.push_back (*here);
+    else if (inquotes) op.push_back (*here);
+    else com.push_back (*here);
+    here ++;
+  }
+  instring = meta;
+  if (com.empty())
+    metaoption.parentcommand = pImmediate;
+  else if (com == "Top")
+    metaoption.parentcommand = pTop;
+  else if (com == "All") {
+    metaoption.parentcommand = pAll;
+    metaoption.parentoptions = op;
+  }
+}
+
+static void parse_meta (text_t &meta, metadata_t &metaoption,
+			text_tset &metadata, bool &getParents) {
+
+  if (meta.size() > 8 && (substr(meta.begin(), meta.begin()+8) == "cgisafe:")) {
+    metaoption.metacommand = mCgiSafe;
+    meta = substr (meta.begin()+8, meta.end());
+  }
+
+  if (meta.size() > 7 && (substr (meta.begin(), meta.begin()+6) == "parent")) {
+    getParents = true;
+    get_parent_options (meta, metaoption);
+  } 
+
+  metadata.insert (meta);
+  metaoption.metaname = meta;
+}
+
+static void parse_meta (text_t &meta, format_t *formatlistptr,
+			text_tset &metadata, bool &getParents) {
+  
+  if (meta == "link")
+    formatlistptr->command = comLink;
+  else if (meta == "/link")
+    formatlistptr->command = comEndLink;
+
+  else if (meta == "num")
+    formatlistptr->command = comNum;
+
+  else if (meta == "icon")
+    formatlistptr->command = comIcon;
+
+  else if (meta == "Text")
+    formatlistptr->command = comDoc;
+
+  else if (meta == "highlight")
+    formatlistptr->command = comHighlight;
+
+  else if (meta == "/highlight")
+    formatlistptr->command = comEndHighlight;
+
+  else {
+    formatlistptr->command = comMeta;
+    parse_meta (meta, formatlistptr->meta, metadata, getParents);
+  }
+}
+
+static bool parse_string (const text_t &formatstring, format_t *formatlistptr, 
+			  text_tset &metadata, bool &getParents) {
+
+  text_t text;
+  text_t::const_iterator here = formatstring.begin();
+  text_t::const_iterator end = formatstring.end();
+
+  while (here != end) {
+
+    if (*here == '\\')
+      text.push_back (*(++here));
+
+    else if (*here == '{') {
+      if (!text.empty()) {
+	formatlistptr->command = comText;
+	formatlistptr->text = text;
+	formatlistptr->nextptr = new format_t();
+	formatlistptr = formatlistptr->nextptr;
+	
+	text.clear();
+      }
+      if (parse_action (++here, end, formatlistptr, metadata, getParents)) {
+	formatlistptr->nextptr = new format_t();
+	formatlistptr = formatlistptr->nextptr;
+	if (here == end) break;
+      }
+    } else if (*here == '[') {
+      if (!text.empty()) {
+	formatlistptr->command = comText;
+	formatlistptr->text = text;
+	formatlistptr->nextptr = new format_t();
+	formatlistptr = formatlistptr->nextptr;
+
+	text.clear();
+      }
+      text_t meta;
+      here ++;
+      while (*here != ']') {
+	if (here == end) return false;
+	meta.push_back (*here);
+	here ++;
+      }
+      parse_meta (meta, formatlistptr, metadata, getParents);
+      formatlistptr->nextptr = new format_t();
+      formatlistptr = formatlistptr->nextptr;
+
+    } else 
+      text.push_back (*here);
+
+    if (here != end) here ++;
+  }
+  if (!text.empty()) {
+    formatlistptr->command = comText;
+    formatlistptr->text = text;
+    formatlistptr->nextptr = new format_t();
+    formatlistptr = formatlistptr->nextptr;
+
+  }
+  return true;
+}
+
+
+static bool parse_action (text_t::const_iterator &here, const text_t::const_iterator &end, 
+			  format_t *formatlistptr, text_tset &metadata, bool &getParents) {
+
+  text_t::const_iterator it = findchar (here, end, '}');
+  if (it == end) return false;
+
+  text_t com = substr (here, it);
+  here = findchar (it, end, '{');
+  if (here == end) return false;
+  else here ++;
+
+  if (com == "If") formatlistptr->command = comIf;
+  else if (com == "Or") formatlistptr->command = comOr;
+  else return false;
+
+  int curlycount = 0;
+  int commacount = 0;
+  text_t text;
+  while (here != end) {
+
+    if (*here == '\\') {
+      here++;
+      if (here != end) text.push_back(*here);
+      
+    } else if (*here == '{') {curlycount ++; text.push_back(*here);}
+    else if (*here == '}' && curlycount > 0) {
+      curlycount --; 
+      text.push_back(*here);
+    }
+
+    else if ((*here == ',' || *here == '}') && curlycount <= 0) {
+
+      if (formatlistptr->command == comOr) {
+	// the {Or}{this, or this, or this, or this} statement
+	// or'ed statements may be either [metadata] or plain text
+	format_t *or_ptr;
+	
+	// find the next unused orptr
+	if (formatlistptr->orptr == NULL) {
+	  formatlistptr->orptr = new format_t();
+	  or_ptr = formatlistptr->orptr;
+	} else {
+	  or_ptr = formatlistptr->orptr;
+	  while (or_ptr->nextptr != NULL)
+	    or_ptr = or_ptr->nextptr;
+	  or_ptr->nextptr = new format_t();
+	  or_ptr = or_ptr->nextptr;
+	}
+
+	text_t::const_iterator beginbracket = text.begin();
+	text_t::const_iterator endbracket = (text.end() - 1);
+	if ((*beginbracket == '[') && (*endbracket == ']')) {
+	  // it's metadata
+	  text_t meta = substr (beginbracket+1, endbracket);
+	  parse_meta (meta, or_ptr, metadata, getParents);
+
+	} else {
+	  // assume it's plain text
+	  or_ptr->command = comText;
+	  or_ptr->text = text;
+	}
+	text.clear();
+
+      } else {
+	// the {If}{decide,do,else} statement
+	if (commacount == 0) {
+	  // If decision only supports metadata at present
+	  
+	  // remove the surrounding square brackets
+	  text_t::const_iterator beginbracket = text.begin();
+	  text_t::const_iterator endbracket = (text.end() - 1);
+	  if ((*beginbracket == '[') && (*endbracket == ']')) {
+	    text_t meta = substr (beginbracket+1, endbracket);
+	    parse_meta (meta, formatlistptr->decision.meta, metadata, getParents);
+	    commacount ++;
+	    text.clear();
+	  } 
+	  
+	} else if (commacount == 1) {
+	  formatlistptr->ifptr = new format_t();
+	  parse_string (text, formatlistptr->ifptr, metadata, getParents);
+	  commacount ++;
+	  text.clear();
+	  
+	} else if (commacount == 2) {
+	  formatlistptr->elseptr = new format_t();
+	  parse_string (text, formatlistptr->elseptr, metadata, getParents);
+	  commacount ++;
+	  text.clear();
+	}
+      }
+      if (*here == '}') break;
+      
+    } else text.push_back(*here);
+    
+    if (here != end) here ++;
+  }
+
+  return true;
+}
+
+
+bool parse_formatstring (const text_t &formatstring, format_t *formatlistptr, 
+			 text_tset &metadata, bool &getParents) {
+
+  formatlistptr->clear();
+  getParents = false;
+
+  return (parse_string (formatstring, formatlistptr, metadata, getParents));
+}
+
+
+// note: all the format_date stuff is assuming that all Date metadata is going to
+// be of the form yyyymmdd, this is of course, crap ;)
+
+static text_t get_meta (ResultDocInfo_t &docinfo, const metadata_t &meta) {
+  
+  // make sure we have the requested metadata
+  MetadataInfo_tmap::iterator it = docinfo.metadata.find (meta.metaname);
+  if (it == docinfo.metadata.end()) return "";
+
+  MetadataInfo_t *parent = docinfo.metadata[meta.metaname].parent;
+
+  switch (meta.parentcommand) {
+  case pNone:
+    if (meta.metaname == "Date")
+      return format_date (docinfo.metadata[meta.metaname].values[0]);
+    if (meta.metacommand == mCgiSafe) 
+      return cgi_safe (docinfo.metadata[meta.metaname].values[0]);
+    else return docinfo.metadata[meta.metaname].values[0];
+
+  case pImmediate:
+    if (parent != NULL) {
+      if (meta.metaname == "Date") 
+	return format_date (parent->values[0]);
+      if (meta.metacommand == mCgiSafe) 
+	return cgi_safe (parent->values[0]);
+      else return parent->values[0];
+    }
+    break;
+
+  case pTop:
+    if (parent != NULL) {
+      while (parent->parent != NULL) parent = parent->parent;
+
+      if (meta.metaname == "Date") 
+	return format_date (parent->values[0]);
+      if (meta.metacommand == mCgiSafe) 
+	return cgi_safe (parent->values[0]);
+      else return parent->values[0];
+    }
+    break;
+
+  case pAll:
+    MetadataInfo_t *parent = docinfo.metadata[meta.metaname].parent;
+    if (parent != NULL) {
+      text_tarray tmparray;
+      while (parent != NULL) {
+	tmparray.push_back (parent->values[0]);
+	parent = parent->parent;
+      }
+      bool first = true;
+      text_t tmp;
+      text_tarray::reverse_iterator here = tmparray.rbegin();
+      text_tarray::reverse_iterator end = tmparray.rend();
+      while (here != end) {
+	if (!first) tmp += meta.parentoptions;
+	if (meta.metaname == "Date") tmp += format_date (*here);
+	else tmp += *here;
+	first = false;
+	here ++;
+      }
+      if (meta.metacommand == mCgiSafe) return cgi_safe (tmp);
+      else return tmp;
+    }
+  }
+  return "";
+}
+
+static text_t get_or (ResultDocInfo_t &docinfo, format_t *orptr,
+		      const text_t &link, const text_t &icon, 
+		      const text_t &text, bool highlight) {
+
+  text_t tmp;
+  while (orptr != NULL) {
+
+    tmp = format_string (docinfo, orptr, link, icon, text, highlight);
+    if (!tmp.empty()) return tmp;
+
+    orptr = orptr->nextptr;
+  }
+  return "";
+}
+
+static text_t get_if (ResultDocInfo_t &docinfo, const decision_t &decision,
+		      format_t *ifptr, format_t *elseptr, const text_t &link, 
+		      const text_t &icon, const text_t &text, bool highlight) {
+
+  // not much of a choice yet ...
+  if (decision.command == dMeta) {
+    if (get_meta (docinfo, decision.meta) != "") {
+      if (ifptr != NULL)
+	return get_formatted_string (docinfo, ifptr, link, icon, text, highlight);
+    }
+    else {
+      if (elseptr != NULL)
+	return get_formatted_string (docinfo, elseptr, link, icon, text, highlight);
+    }
+  }
+  return "";
+}
+
+text_t format_string (ResultDocInfo_t &docinfo, format_t *formatlistptr,
+		      const text_t &link, const text_t &icon, 
+		      const text_t &text, bool highlight) {
+
+  if (formatlistptr == NULL) return "";
+
+  switch (formatlistptr->command) {
+  case comText:
+    return formatlistptr->text;
+  case comLink:
+    return link;
+  case comEndLink:
+    if (link.empty()) return "";
+    else return "</a>";
+  case comIcon:
+    return icon;
+  case comNum:
+    return docinfo.result_num;
+  case comMeta:
+    return get_meta (docinfo, formatlistptr->meta);
+  case comDoc:
+    return text;
+  case comHighlight:
+    if (highlight) return "<b>";
+    break;
+  case comEndHighlight:
+    if (highlight) return "</b>";
+    break;
+  case comIf:
+    return get_if (docinfo, formatlistptr->decision, formatlistptr->ifptr, 
+		   formatlistptr->elseptr, link, icon, text, highlight);
+  case comOr:
+    return get_or (docinfo, formatlistptr->orptr, link, icon, text, highlight);
+  }
+  return "";
+}
+
+
+text_t get_formatted_string (ResultDocInfo_t &docinfo, format_t *formatlistptr,
+			     const text_t &link, const text_t &icon) {
+
+  text_t text;
+
+  text_t ft;
+  while (formatlistptr != NULL) {
+    ft += format_string (docinfo, formatlistptr, link, icon, text, false);
+    formatlistptr = formatlistptr->nextptr;
+  }
+  return ft;
+}
+
+
+text_t get_formatted_string (ResultDocInfo_t &docinfo, format_t *formatlistptr) {
+
+  text_t link = "<a href=\"_httpdocument_&cl=search&d=" + docinfo.OID + "\">";
+  text_t icon = "_icontext_";
+  text_t text;
+
+  text_t ft;
+  while (formatlistptr != NULL) {
+    ft += format_string (docinfo, formatlistptr, link, icon, text, false);
+    formatlistptr = formatlistptr->nextptr;
+  }
+  return ft;
+}
+
+
+text_t get_formatted_string (ResultDocInfo_t &docinfo, format_t *formatlistptr,
+			     const text_t &text) {
+
+  text_t link = "<a href=\"_httpdocument_&cl=search&d=" + docinfo.OID + "\">";
+  text_t icon = "_icontext_";
+
+  text_t ft;
+  while (formatlistptr != NULL) {
+    ft += format_string (docinfo, formatlistptr, link, icon, text, false);
+    formatlistptr = formatlistptr->nextptr;
+  }
+  return ft;
+}
+
+
+text_t get_formatted_string (ResultDocInfo_t &docinfo, format_t *formatlistptr,
+			     const text_t &link, const text_t &icon, const text_t &text) {
+
+  text_t ft;
+  while (formatlistptr != NULL) {
+    ft += format_string (docinfo, formatlistptr, link, icon, text, false);
+    formatlistptr = formatlistptr->nextptr;
+  }
+  return ft;
+}
+
+text_t get_formatted_string (ResultDocInfo_t &docinfo, format_t *formatlistptr,
+			     const text_t &link, const text_t &icon, bool highlight) {
+
+  text_t text, ft;
+  while (formatlistptr != NULL) {
+    ft += format_string (docinfo, formatlistptr, link, icon, text, highlight);
+    formatlistptr = formatlistptr->nextptr;
+  }
+  return ft;
+}
+
+text_t get_formatted_string (ResultDocInfo_t &docinfo, format_t *formatlistptr,
+			     const text_t &link, const text_t &icon, 
+			     const text_t &text, bool highlight) {
+
+  text_t ft;
+  while (formatlistptr != NULL) {
+    ft += format_string (docinfo, formatlistptr, link, icon, text, highlight);
+    formatlistptr = formatlistptr->nextptr;
+  }
+  return ft;
+}
+
+
Index: /tags/gsdl-21/gsdl/src/recpt/formattools.h
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/formattools.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/formattools.h	(revision 699)
@@ -0,0 +1,118 @@
+/**********************************************************************
+ *
+ * formattools.h -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef FORMATTOOLS_H
+#define FORMATTOOLS_H
+
+#include "text_t.h"
+#include "comtypes.h"
+
+enum command_t {comIf, comOr, comMeta, comText, comLink, comEndLink, comNum, comIcon, 
+		comDoc, comHighlight, comEndHighlight};
+enum pcommand_t {pNone, pImmediate, pTop, pAll};
+enum dcommand_t {dMeta};
+enum mcommand_t {mNone, mCgiSafe};
+
+struct metadata_t {
+  void clear();
+  metadata_t () {clear();}
+
+  text_t metaname;
+  mcommand_t metacommand;
+  pcommand_t parentcommand;
+  text_t parentoptions;
+};
+
+struct decision_t {
+  void clear();
+  decision_t () {clear();}
+
+  dcommand_t command;
+  metadata_t meta;
+};
+
+struct format_t {
+  void clear();
+  format_t () {clear();}
+
+  command_t command;
+  decision_t decision;
+  text_t text;
+  metadata_t meta;
+  format_t *nextptr;
+  format_t *ifptr;
+  format_t *elseptr;
+  format_t *orptr;
+};
+
+// the format structure used by this receptionist
+struct formatinfo_t {
+  void clear();
+  formatinfo_t () {clear();}
+
+  bool DocumentImages;
+  bool DocumentTitles;
+  text_t DocumentHeading;
+  bool DocumentContents;
+  bool DocumentArrowsBottom;
+  text_tarray DocumentButtons;
+  text_t DocumentText;
+  text_tmap formatstrings;
+};
+
+bool get_formatstring (const text_t &key, const text_tmap &formatstringmap,
+		       text_t &formatstring);
+bool get_formatstring (const text_t &key1, const text_t &key2,  
+		       const text_tmap &formatstringmap,
+		       text_t &formatstring);
+
+text_t format_date (const text_t &date);
+
+bool parse_formatstring (const text_t &formatstring, format_t *formatlistptr, 
+			 text_tset &metadata, bool &getParents);
+
+text_t get_formatted_string (ResultDocInfo_t &docinfo, format_t *formatlistptr,
+			     const text_t &link, const text_t &icon);
+
+text_t get_formatted_string (ResultDocInfo_t &docinfo, format_t *formatlistptr);
+
+text_t get_formatted_string (ResultDocInfo_t &docinfo, format_t *formatlistptr,
+			     const text_t &text);
+
+text_t get_formatted_string (ResultDocInfo_t &docinfo, format_t *formatlistptr,
+			     const text_t &link, const text_t &icon, const text_t &text);
+
+text_t get_formatted_string (ResultDocInfo_t &docinfo, format_t *formatlistptr,
+			     const text_t &link, const text_t &icon, bool highlight);
+
+text_t get_formatted_string (ResultDocInfo_t &docinfo, format_t *formatlistptr,
+			     const text_t &link, const text_t &icon, 
+			     const text_t &text, bool highlight);
+
+
+#endif
Index: /tags/gsdl-21/gsdl/src/recpt/hlistbrowserclass.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/hlistbrowserclass.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/hlistbrowserclass.cpp	(revision 699)
@@ -0,0 +1,152 @@
+/**********************************************************************
+ *
+ * hlistbrowserclass.cpp -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.2  1999/10/14 22:59:34  sjboddie
+   finished off browser classes
+
+   Revision 1.1  1999/10/10 08:14:08  sjboddie
+   - metadata now returns mp rather than array
+   - redesigned browsing support (although it's not finished so
+   won't currently work ;-)
+
+ */
+
+
+#include "hlistbrowserclass.h"
+#include <assert.h>
+#include "OIDtools.h"
+
+hlistbrowserclass::hlistbrowserclass () {
+}
+
+hlistbrowserclass::~hlistbrowserclass () {
+}
+
+// returns the name that specifies the browserclass type
+text_t hlistbrowserclass::get_browser_name () {
+  return "HList";
+}
+
+text_t hlistbrowserclass::get_default_formatstring () {
+  return "[link][highlight][Title][/highlight][/link]";
+}
+
+// need to change OID to it's child 
+void hlistbrowserclass::processOID (cgiargsclass &args, recptproto *collectproto,
+				    ostream &logout) {
+
+  text_t OID = args["d"];
+  bool is_classify = false;
+  if (OID.empty()) {
+    is_classify = true;
+    OID = args["cl"];
+  }
+
+  FilterResponse_t response;
+  text_tset metadata;
+  metadata.insert ("childtype");
+  text_t classifytype = "Invisible";
+  text_t child_doctype;
+
+  text_tarray OIDs;
+
+  while (classifytype == "Invisible") {
+    OIDs.erase (OIDs.begin(), OIDs.end());
+    
+    OIDs.push_back (OID);
+    OIDs.push_back (OID + ".fc");
+
+    get_info (OIDs, args["c"], metadata, false, collectproto, response, logout);
+
+    OID = response.docInfo[1].OID;
+    classifytype = response.docInfo[0].metadata["childtype"].values[0];
+  }
+  if (is_classify) args["cl"] = OID;
+  else args["d"] = OID;
+}
+
+int hlistbrowserclass::output_section_group (ResultDocInfo_t &section, cgiargsclass &args,
+					     int colnumber, format_t *formatlistptr, 
+					     bool use_table, text_tset &metadata, bool &getParents,
+					     recptproto *collectproto, displayclass &disp, 
+					     outconvertclass &outconvert, ostream &textout, ostream &logout) {
+
+  // get all siblings
+  FilterResponse_t response;
+  text_t &arg_d = args["d"];
+  text_t &arg_cl = args["cl"];
+  get_children (section.OID + ".pr", args["c"], metadata, getParents, 
+		collectproto, response, logout);
+
+					
+  if (use_table || colnumber > 0) {
+    textout << "<table><tr><td>";
+    // get tab size
+    text_t tab; int itab;
+    disp.expandstring ("Global", "_tabwidth_", tab);
+    itab = tab.getint();
+    if (colnumber > 0) textout << outconvert << disp 
+			       << "<img src=\"_httpimg_/space.gif\" width=" 
+			       << (itab*colnumber) << ">";
+    textout << "</td>\n";
+  }
+
+  textout << "<table><tr>\n";
+
+  ResultDocInfo_tarray::iterator tsibling = response.docInfo.begin();
+  ResultDocInfo_tarray::iterator esibling = response.docInfo.end();
+
+  text_t icon, link;
+  while (tsibling != esibling) {
+
+    bool highlight = false;
+    textout << "<td>";
+
+    if ((*tsibling).OID == section.OID) {
+      link.clear();
+      highlight = true;
+    } else {
+      link = "<a href=\"_httpdocument_";
+      if (!arg_d.empty()) 
+	link += "&cl=" + arg_cl + "&d=" + (*tsibling).OID + "\">";
+      else 
+	link += "&cl=" + (*tsibling).OID + "\">";
+    }
+
+    textout << outconvert << disp << get_formatted_string (*tsibling, formatlistptr, link, icon, highlight);
+
+    textout << "</td>\n";
+    tsibling ++;
+  }
+
+  textout << "</tr></table>\n";
+  if (use_table  || colnumber > 0) textout << "</tr></table>\n";
+  return 0;
+}
+
Index: /tags/gsdl-21/gsdl/src/recpt/hlistbrowserclass.h
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/hlistbrowserclass.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/hlistbrowserclass.h	(revision 699)
@@ -0,0 +1,59 @@
+/**********************************************************************
+ *
+ * hlistbrowserclass.h -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef HLISTBROWSERCLASS_H
+#define HLISTBROWSERCLASS_H
+
+#include "browserclass.h"
+
+class hlistbrowserclass : public browserclass {
+protected:
+
+public:
+  hlistbrowserclass ();
+  virtual ~hlistbrowserclass ();
+
+  // returns the name that specifies the browserclass type
+  virtual text_t get_browser_name ();
+
+  text_t get_default_formatstring ();
+  
+  void processOID (cgiargsclass &args, recptproto *collectproto,
+		   ostream &logout);
+
+  int output_section_group (ResultDocInfo_t &section, cgiargsclass &args,
+			    int colnumber, format_t *formatlistptr, 
+			    bool use_table, text_tset &metadata, bool &getParents,
+			    recptproto *collectproto, displayclass &disp, 
+			    outconvertclass &outconvert, ostream &textout, 
+			    ostream &logout);
+
+};
+
+#endif
+
Index: /tags/gsdl-21/gsdl/src/recpt/htmlgen.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/htmlgen.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/htmlgen.cpp	(revision 699)
@@ -0,0 +1,124 @@
+/**********************************************************************
+ *
+ * htmlgen.cpp -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.4  1999/09/07 04:56:55  sjboddie
+   added GPL notice
+
+   Revision 1.3  1999/02/11 01:24:04  rjmcnab
+
+   Fixed a few compiler warnings.
+
+   Revision 1.2  1999/02/08 01:28:02  rjmcnab
+
+   Got the receptionist producing something using the statusaction.
+
+   Revision 1.1  1999/01/08 08:40:56  rjmcnab
+
+   Moved from lib directory.
+
+   Revision 1.1  1999/01/08 03:57:46  rjmcnab
+
+   Initial revision
+
+ */
+
+#include "htmlgen.h"
+#include "unitool.h"
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\map>
+#elif defined(GSDL_USE_STL_H)
+#  include <map.h>
+#else
+#  include <map>
+#endif
+
+
+
+// highlighttext highlights query terms in text string and 
+// outputs the resulting text string
+void highlighttext (text_tarray &termvars, displayclass &disp, text_t &text,
+		    outconvertclass &outconvert, ostream &textout, ostream &/*logout*/) {
+  map<text_t, int, lttext_t> terms;
+  map<text_t, int, lttext_t>::const_iterator it;
+  for (unsigned int i = 0; i < termvars.size(); i++) {
+    terms[termvars[i]] = 1;
+  }
+
+  // get the text to start and end a hightlight
+  text_t starthighlight = "<b><u>";
+  text_t endhighlight = "</u></b>";
+  if (disp.isdefaultmacro("Global", "starthighlight"))
+    disp.expandstring("Global", "_starthighlight_", starthighlight);
+  if (disp.isdefaultmacro("Global", "endhighlight"))
+    disp.expandstring("Global", "_endhighlight_", endhighlight);
+
+
+  text_t::iterator here = text.begin(); 
+  text_t::iterator end = text.end();
+  text_t word, buffer;
+  while (here != end) {
+    if (is_unicode_letdig(*here)) {
+      // not word boundary
+      word.push_back(*here);
+      here++;
+
+    } else {
+      // found word boundary
+      // add last word if there was one
+      if (!word.empty()) {
+	it = terms.find(word);
+	if (it != terms.end()) {
+	  word = starthighlight + word + endhighlight;
+	}
+	buffer += word;
+        word.clear();
+      }
+
+      if (*here == '<') {
+        // skip over rest of html tag
+	while ((here != end) && (*here != '>')) {
+	  buffer.push_back(*here);
+	  here++;
+	}
+      }
+
+      buffer.push_back(*here);
+      here++;
+
+      if (buffer.size() > 1024) {
+	textout << outconvert << buffer;
+	buffer.clear();
+      }
+    }
+  }
+  textout << outconvert << buffer;
+}
+
+
Index: /tags/gsdl-21/gsdl/src/recpt/htmlgen.h
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/htmlgen.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/htmlgen.h	(revision 699)
@@ -0,0 +1,52 @@
+/**********************************************************************
+ *
+ * htmlgen.h -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef HTMLGEN_H
+#define HTMLGEN_H
+
+#include "gsdlconf.h"
+#include "text_t.h"
+#include "display.h"
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\iostream>
+#elif defined(GSDL_USE_IOS_H)
+#  include <iostream.h>
+#else
+#  include <iostream>
+#endif
+
+
+
+// highlighttext highlights query terms in text string and 
+// outputs the resulting text string
+void highlighttext (text_tarray &termvars, displayclass &disp, text_t &text,
+		    outconvertclass &outconvert, ostream &textout, ostream &logout);
+
+
+#endif
Index: /tags/gsdl-21/gsdl/src/recpt/htmlutils.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/htmlutils.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/htmlutils.cpp	(revision 699)
@@ -0,0 +1,69 @@
+/**********************************************************************
+ *
+ * htmlutils.cpp -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.4  1999/09/07 04:56:56  sjboddie
+   added GPL notice
+
+   Revision 1.3  1999/06/09 00:09:30  sjboddie
+   Added a fairly minimal html_safe function
+
+   Revision 1.2  1999/02/08 01:28:02  rjmcnab
+
+   Got the receptionist producing something using the statusaction.
+
+   Revision 1.1  1999/01/08 08:40:58  rjmcnab
+
+   Moved from lib directory.
+
+   Revision 1.1  1999/01/08 03:57:47  rjmcnab
+
+   Initial revision
+
+ */
+
+
+#include "htmlutils.h"
+
+text_t html_safe (const text_t &instring) {
+  
+  text_t outstring;
+  text_t::const_iterator here = instring.begin();
+  text_t::const_iterator end = instring.end();
+  while (here != end) {
+    if (*here == '"') outstring += "&quot;";
+    else if (*here == '&') outstring += "&amp;";
+    else if (*here == '<') outstring += "&lt;";
+    else if (*here == '>') outstring += "&gt;";
+    else outstring.push_back(*here);
+    here ++;
+  }
+  return outstring;
+}
+
+
Index: /tags/gsdl-21/gsdl/src/recpt/htmlutils.h
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/htmlutils.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/htmlutils.h	(revision 699)
@@ -0,0 +1,37 @@
+/**********************************************************************
+ *
+ * htmlutils.h -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef HTMLUTILS_H
+#define HTMLUTILS_H
+
+#include "gsdlconf.h"
+#include "text_t.h"
+
+text_t html_safe (const text_t &instring);
+
+#endif
Index: /tags/gsdl-21/gsdl/src/recpt/infodbclass.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/infodbclass.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/infodbclass.cpp	(revision 699)
@@ -0,0 +1,541 @@
+/**********************************************************************
+ *
+ * infodbclass.cpp -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.4  1999/09/07 04:56:56  sjboddie
+   added GPL notice
+
+   Revision 1.3  1999/09/02 00:26:10  rjmcnab
+   now there can be multiple values for a single key
+
+   Revision 1.2  1999/07/13 23:24:05  rjmcnab
+   Added functionality to modify a gdbm database.
+
+   Revision 1.1  1999/07/11 08:27:52  rjmcnab
+   Moved from src/colservr and added capability to write out data.
+
+   Revision 1.4  1999/05/10 03:43:48  sjboddie
+   lots of changes to lots of files - getting document action going
+
+   Revision 1.3  1999/04/30 02:00:46  sjboddie
+   lots of stuff to do with getting documentaction working
+
+   Revision 1.2  1999/04/06 22:20:31  rjmcnab
+   Got browsefilter working.
+
+   Revision 1.1  1999/03/30 05:10:07  rjmcnab
+   Initial revision.
+
+ */
+
+
+#include "infodbclass.h"
+#include "unitool.h"
+#include "gsdlunicode.h"
+#include "fileutil.h"
+#include "OIDtools.h"
+#include <stdlib.h>
+
+
+
+
+// constructors
+infodbclass::infodbclass () {
+}
+
+void infodbclass::setinfo (const text_t &key, const text_t &value) {
+  text_tarray &tarr = info[key];
+  tarr.erase(tarr.begin(), tarr.end());
+  tarr.push_back(value);
+}
+
+void infodbclass::setintinfo (const text_t &key, int value) {
+  setinfo (key, value);
+}
+
+void infodbclass::setcinfo (const text_t &key, unsigned short c) {
+  text_t t;
+  t.push_back (c);
+  setinfo (key, t);
+}
+
+text_t *infodbclass::getinfo (const text_t &key) {
+  iterator here = info.find (key);
+  if (here == info.end()) return NULL;
+
+  if ((*here).second.empty()) return NULL;
+  
+  return &((*here).second[0]);
+}
+
+int infodbclass::getintinfo (const text_t &key) {
+  text_t *t = getinfo (key);
+  if (t == NULL) return 0;
+  return t->getint();
+}
+
+text_t &infodbclass::operator[] (const text_t &key) {
+  text_tarray &tarr = info[key];
+  if (tarr.empty()) {
+    text_t e;
+    tarr.push_back(e);
+  }
+  return tarr[0];
+}
+
+
+void infodbclass::addinfo (const text_t &key, const text_t &value) {
+  text_tarray &tarr = info[key];
+  tarr.push_back (value);
+}
+
+void infodbclass::addintinfo (const text_t &key, int value) {
+  addinfo (key, value);
+}
+
+void infodbclass::addcinfo (const text_t &key, unsigned short c) {
+  text_t t;
+  t.push_back(c);
+  addinfo (key, t);
+}
+
+text_tarray *infodbclass::getmultinfo (const text_t &key) {
+  iterator here = info.find (key);
+  if (here == info.end()) return NULL;
+
+  return &((*here).second);
+}
+
+
+
+
+// returns true if opened
+bool gdbmclass::opendatabase (const text_t &filename, int mode, int num_retrys, 
+			      bool need_filelock) {
+  text_t data_location;
+  int block_size = 512;
+  
+  if (gdbmfile != NULL) {
+    if (openfile == filename) return true;
+    else closedatabase ();
+  }
+
+  openfile = filename;
+
+  char *namebuffer = filename.getcstr();
+  do {
+#ifdef __WIN32__
+	  gdbmfile = gdbm_open (namebuffer, block_size, mode, 00664, NULL, (need_filelock) ? 1 : 0);
+#else
+    gdbmfile = gdbm_open (namebuffer, block_size, mode, 00664, NULL);
+#endif
+    num_retrys--;
+  } while (num_retrys>0 && gdbmfile==NULL &&
+	   (gdbm_errno==GDBM_CANT_BE_READER || gdbm_errno==GDBM_CANT_BE_WRITER));
+  delete namebuffer;
+
+  if (gdbmfile == NULL && logout != NULL) {
+    outconvertclass text_t2ascii;
+    (*logout) << text_t2ascii << "database open failed on: " << filename << "\n";
+  }
+
+  return (gdbmfile != NULL);
+}
+
+void gdbmclass::closedatabase () {
+  if (gdbmfile == NULL) return;
+  
+  gdbm_close (gdbmfile);
+  gdbmfile = NULL;
+  openfile.clear();
+}
+
+
+// returns true on success
+bool gdbmclass::setinfo (const text_t &key, const infodbclass &info) {
+  if (gdbmfile == NULL) return false;
+
+  text_t subkey;
+  text_t data;
+
+  // get all the keys and values
+  infodbclass::const_iterator info_here = info.begin();
+  infodbclass::const_iterator info_end = info.end();
+  while (info_here != info_end) {
+    // add the key
+    subkey.clear();
+    subkey.push_back('<');
+    text_t::const_iterator subkey_here = (*info_here).first.begin();
+    text_t::const_iterator subkey_end = (*info_here).first.end();
+    while (subkey_here != subkey_end) {
+      if (*subkey_here == '>') {
+	subkey.push_back('\\'); subkey.push_back('>');
+      } else if (*subkey_here == '\n') {
+	subkey.push_back('\\'); subkey.push_back('n');
+      } else if (*subkey_here == '\r') {
+	subkey.push_back('\\'); subkey.push_back('r');
+      } else {
+	subkey.push_back (*subkey_here);
+      }
+      subkey_here++;
+    }
+    subkey.push_back('>');
+
+    // add the values
+    text_tarray::const_iterator subvalue_here = (*info_here).second.begin();
+    text_tarray::const_iterator subvalue_end = (*info_here).second.end();
+    while (subvalue_here != subvalue_end) {
+      data += subkey;
+      
+      text_t::const_iterator thissubvalue_here = (*subvalue_here).begin();
+      text_t::const_iterator thissubvalue_end = (*subvalue_here).end();
+      while (thissubvalue_here != thissubvalue_end) {
+	if (*thissubvalue_here == '>') {
+	  data.push_back('\\'); data.push_back('>');
+	} else if (*thissubvalue_here == '\n') {
+	  data.push_back('\\'); data.push_back('n');
+	} else if (*thissubvalue_here == '\r') {
+	  data.push_back('\\'); data.push_back('r');
+	} else {
+	  data.push_back (*thissubvalue_here);
+	}
+	
+	thissubvalue_here++;
+      }
+      
+      data.push_back('\n');
+      subvalue_here++;
+    }
+
+    info_here++;
+  }
+
+  // store the value
+  datum key_data;
+  datum data_data;
+
+  // get a utf-8 encoded c string of the unicode key
+  key_data.dptr = (to_utf8(key)).getcstr();
+  if (key_data.dptr == NULL) {
+    if (logout != NULL) (*logout) << "gdbmclass: out of memory\n";
+    return false;
+  }
+  key_data.dsize = strlen (key_data.dptr);
+
+  data_data.dptr = (to_utf8(data)).getcstr();
+  if (data_data.dptr == NULL) {
+    if (logout != NULL) (*logout) << "gdbmclass: out of memory\n";
+    delete key_data.dptr;
+  }
+  data_data.dsize = strlen (data_data.dptr);
+
+  int ret = gdbm_store (gdbmfile, key_data, data_data, GDBM_REPLACE);
+  delete key_data.dptr;
+  delete data_data.dptr;
+
+  return (ret == 0);
+}
+
+
+void gdbmclass::deletekey (const text_t &key) {
+  if (gdbmfile == NULL) return;
+
+  // get a utf-8 encoded c string of the unicode key
+  datum key_data;
+  key_data.dptr = (to_utf8(key)).getcstr();
+  if (key_data.dptr == NULL) return;
+  key_data.dsize = strlen (key_data.dptr);
+
+  // delete the key
+  gdbm_delete (gdbmfile, key_data);
+
+  // free up the key memory
+  delete key_data.dptr;
+}
+
+
+// getfirstkey and getnextkey are used for traversing the database
+// no insertions or deletions should be carried out while traversing
+// the database. when there are no keys left to visit in the database
+// an empty string is returned.
+text_t gdbmclass::getfirstkey () {
+  if (gdbmfile == NULL) return "";
+
+  // get the first key
+  datum firstkey_data = gdbm_firstkey (gdbmfile);
+  if (firstkey_data.dptr == NULL) return "";
+
+  // convert it to text_t
+  text_t firstkey;
+  firstkey.setcarr (firstkey_data.dptr, firstkey_data.dsize);
+  free (firstkey_data.dptr);
+  return to_uni(firstkey);  // convert to unicode
+}
+
+text_t gdbmclass::getnextkey (const text_t &key) {
+  if (gdbmfile == NULL || key.empty()) return "";
+
+  // get a utf-8 encoded c string of the unicode key
+  datum key_data;
+  key_data.dptr = (to_utf8(key)).getcstr();
+  if (key_data.dptr == NULL) return "";
+  key_data.dsize = strlen (key_data.dptr);
+  
+  // get the next key
+  datum nextkey_data = gdbm_nextkey (gdbmfile, key_data);
+  if (nextkey_data.dptr == NULL) {
+    delete key_data.dptr;
+    return "";
+  }
+
+  // convert it to text_t
+  text_t nextkey;
+  nextkey.setcarr (nextkey_data.dptr, nextkey_data.dsize);
+  free (nextkey_data.dptr);
+  delete key_data.dptr;
+  return to_uni(nextkey);  // convert to unicode
+}
+
+
+// replaces the .fc, .lc, .pr, .ns and .ps syntax (first child, 
+// last child, parent, next sibling, previous sibling) 
+// it expects child, parent, etc. to exist if syntax has been used
+// so you should test before using
+text_t gdbmclass::translate_OID (const text_t &inOID, infodbclass &info) {
+
+  if (inOID.size() < 4) return inOID;
+  if (findchar (inOID.begin(), inOID.end(), '.') == inOID.end()) return inOID;
+
+  text_t OID = inOID;
+  text_tarray tailarray;
+  text_t tail = substr (OID.end()-3, OID.end());
+  while (tail == ".fc" || tail == ".lc" || tail == ".pr" || 
+	 tail == ".ns" || tail == ".ps") {
+    tailarray.push_back(tail);
+    OID.erase (OID.end()-3, OID.end());
+    tail = substr (OID.end()-3, OID.end());
+  }
+
+  if (!tailarray.size()) return inOID;
+  text_tarray::const_iterator begin = tailarray.begin();
+  text_tarray::const_iterator here = tailarray.end() - 1;
+
+  while (here >= begin) {
+
+    if (*here == ".fc")
+      get_first_child (OID, info);
+    else if (*here == ".lc")
+      get_last_child (OID, info);
+    else if (*here == ".pr")
+      OID = get_parent (OID);
+    else if (*here == ".ns")
+      get_next_sibling (OID, info);
+    else if (*here == ".ps")
+      get_previous_sibling (OID, info);
+    
+    here --;
+  }
+  return OID;
+}
+
+void gdbmclass::get_first_child (text_t &OID, infodbclass &info) {
+
+  text_t firstchild;
+  if (getinfo (OID, info)) {
+    text_t &contains = info["contains"];
+    if (!contains.empty()) {
+      text_t parent = OID;
+      getdelimitstr (contains.begin(), contains.end(), ';', firstchild);
+      if (firstchild.empty()) OID = contains;
+      else OID = firstchild;
+      if (*(OID.begin()) == '"') translate_parent (OID, parent);
+    }
+  }
+}
+
+void gdbmclass::get_last_child (text_t &OID, infodbclass &info) {
+
+  text_tarray children;
+  if (getinfo (OID, info)) {
+    text_t &contains = info["contains"];
+    if (!contains.empty()) {
+      text_t parent = OID;
+      splitchar (contains.begin(), contains.end(), ';', children);
+      OID = children.back();
+      if (*(OID.begin()) == '"') translate_parent (OID, parent);
+    }
+  }
+}
+  
+void gdbmclass::get_next_sibling (text_t &OID, infodbclass &info) {
+
+  text_tarray siblings;
+  text_t parent = get_parent (OID);
+	     
+  if (getinfo (parent, info)) {
+    text_t &contains = info["contains"];
+    if (!contains.empty()) {
+      splitchar (contains.begin(), contains.end(), ';', siblings);
+      text_tarray::const_iterator here = siblings.begin();
+      text_tarray::const_iterator end = siblings.end();
+      text_t shrunk_OID = OID;
+      shrink_parent (shrunk_OID);
+      while (here != end) {
+	if (*here == shrunk_OID && (here+1 != end)) {
+	  OID = *(here+1);
+	  if (*(OID.begin()) == '"') translate_parent (OID, parent);
+	  break;
+	}
+	here ++;
+      }
+    }
+  }
+}
+
+void gdbmclass::get_previous_sibling (text_t &OID, infodbclass &info) {
+
+  text_tarray siblings;
+  text_t parent = get_parent (OID);
+
+  if (getinfo (parent, info)) {
+    text_t &contains = info["contains"];
+    if (!contains.empty()) {
+      splitchar (contains.begin(), contains.end(), ';', siblings);
+      text_tarray::const_iterator here = siblings.begin();
+      text_tarray::const_iterator end = siblings.end();
+      text_t shrunk_OID = OID;
+      shrink_parent (shrunk_OID);
+      while (here != end) {
+	if (*here == shrunk_OID && (here != siblings.begin())) {
+	  OID = *(here-1);
+	  if (*(OID.begin()) == '"') translate_parent (OID, parent);
+	  break;
+	}
+	here ++;
+      }
+    }
+  }
+}
+
+// returns true on success
+bool gdbmclass::getinfo (text_t key, infodbclass &info) {
+  text_t data;
+  
+  if (!getkeydata (key, data)) return false;
+  text_t::iterator here = data.begin ();
+  text_t::iterator end = data.end ();
+  
+  text_t ikey, ivalue;
+  info.clear (); // reset info
+  
+  while (getinfoline (here, end, ikey, ivalue)) {
+    info.addinfo (ikey, ivalue);
+  }
+  
+  return true;
+}
+
+// returns true if exists
+bool gdbmclass::exists (text_t key) {
+  text_t data;
+  return getkeydata (key, data);
+}
+
+// returns true on success
+bool gdbmclass::getkeydata (text_t key, text_t &data) {
+  datum key_data;
+  datum return_data;
+
+  if (gdbmfile == NULL) return false;
+  
+  // get a utf-8 encoded c string of the unicode key
+  key_data.dptr = (to_utf8(key)).getcstr();
+  if (key_data.dptr == NULL) {
+    if (logout != NULL) (*logout) << "gdbmclass: out of memory\n";
+    return false;
+  }
+  key_data.dsize = strlen (key_data.dptr);
+  
+  // fetch the result
+  return_data = gdbm_fetch (gdbmfile, key_data);
+  delete key_data.dptr;
+  
+  if (return_data.dptr == NULL) return false;
+
+  data.setcarr (return_data.dptr, return_data.dsize);
+  free (return_data.dptr);
+  data = to_uni(data);  // convert to unicode
+
+  return true;
+}
+
+// returns true on success
+bool gdbmclass::getinfoline (text_t::iterator &here, text_t::iterator end, 
+			     text_t &key, text_t &value) {
+  key.clear();
+  value.clear();
+
+  // ignore white space
+  while (here != end && is_unicode_space (*here)) here++;
+
+  // get the '<'
+  if (here == end || *here != '<') return false;
+  here++;
+  
+  // get the key
+  while (here != end && *here != '>') {
+    key.push_back(*here);
+    here++;
+  }
+  
+  // get the '>'
+  if (here == end || *here != '>') return false;
+  here++;
+  
+  // get the value
+  while (here != end && *here != '\n') {
+    if (*here == '\\') { 
+      // found escape character
+      here++;
+      if (here != end) {
+	if (*here == 'n') value.push_back ('\n');
+	else if (*here == 'r') value.push_back ('\r');
+	else value.push_back(*here);
+      }
+
+    } else {
+      // a normal character
+      value.push_back(*here);
+    }
+
+    here++;
+  }
+
+  return true;
+}
+
Index: /tags/gsdl-21/gsdl/src/recpt/infodbclass.h
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/infodbclass.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/infodbclass.h	(revision 699)
@@ -0,0 +1,173 @@
+/**********************************************************************
+ *
+ * infodbclass.h -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef INFODBCLASS_H
+#define INFODBCLASS_H
+
+
+#include "gsdlconf.h"
+#include "text_t.h"
+
+#ifdef __WIN32__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include "autoconf.h"
+#include "systems.h"
+#include "gdbmconst.h"
+#include "gdbm.h"
+#ifdef __cplusplus
+}
+#endif
+
+#else
+#include <gdbm.h>
+#endif
+
+
+typedef map<text_t, text_tarray, lttext_t> text_tarraymap;
+
+
+// infodbclass is used to store information about a object
+class infodbclass {
+protected:
+  text_tarraymap info;
+
+public:
+  // type support for text_tarraymap
+  typedef text_tarraymap::iterator iterator;
+  typedef text_tarraymap::const_iterator const_iterator;
+  typedef text_tarraymap::reference reference;
+  typedef text_tarraymap::const_reference const_reference;
+  typedef text_tarraymap::size_type size_type;
+  typedef text_tarraymap::difference_type difference_type;
+  typedef text_tarraymap::const_reverse_iterator const_reverse_iterator;
+  typedef text_tarraymap::reverse_iterator reverse_iterator;
+  
+  // constructors
+  infodbclass ();
+
+  // basic container support
+  iterator begin () {return info.begin();}
+  const_iterator begin () const {return info.begin();}
+  iterator end () {return info.end();}
+  const_iterator end () const {return info.end();}
+
+  void erase(iterator pos) {info.erase(pos);}
+  void erase(iterator first, iterator last) {info.erase(first, last);}
+  infodbclass &operator=(const infodbclass &x) {info=x.info;return *this;}
+
+  bool empty () const {return info.empty();}
+  size_type size() const {return info.size();}
+
+
+  // added functionality
+  void clear () {info.erase(info.begin(),info.end());}
+
+  // the following functions deal with keys that can only
+  // have one value for compatibility
+  
+  // getinfo returns NULL if there isn't an entry with
+  // 'key' already defined, getintinfo returns 0 if there wasn't an
+  // entry with 'key' defined and operator[] returns "" if
+  // 'key' wasn't already defined (and sets 'key'="").
+  void setinfo (const text_t &key, const text_t &value);
+  void setintinfo (const text_t &key, int value);
+  void setcinfo (const text_t &key, unsigned short c);
+  text_t *getinfo (const text_t &key);
+  int getintinfo (const text_t &key);
+  text_t &operator[] (const text_t &key);
+
+
+  // the next set of functions allow you to set and access keys
+  // that can have more than one value
+
+  // getmultinfo returns NULL if there isn't an entry with
+  // 'key' already defined
+  void addinfo (const text_t &key, const text_t &value);
+  void addintinfo (const text_t &key, int value);
+  void addcinfo (const text_t &key, unsigned short c);
+  text_tarray *getmultinfo (const text_t &key);
+};
+
+
+class gdbmclass {
+public:
+  gdbmclass() {gdbmfile = NULL; logout = &cerr;};
+  ~gdbmclass() {};
+  
+  // returns true if opened
+  bool opendatabase (const text_t &filename, int mode, int num_retrys, 
+		     bool need_filelock);
+  void closedatabase ();
+
+  // replaces the .c, .p, .n, .l syntax (child, parent, next, previous) 
+  // it expects child, parent, etc. to exist if syntax has been used
+  // so you should test before using
+  text_t translate_OID (const text_t &OID, infodbclass &info);
+
+  // returns true on success
+  bool getinfo (text_t key, infodbclass &info);
+  void setlogout (ostream *thelogout) {logout = thelogout;}
+
+  // returns true if exists
+  bool exists (text_t key);
+
+  // returns true on success
+  bool setinfo (const text_t &key, const infodbclass &info);
+
+  void deletekey (const text_t &key);
+
+  // getfirstkey and getnextkey are used for traversing the database
+  // no insertions or deletions should be carried out while traversing
+  // the database. when there are no keys left to visit in the database
+  // an empty string is returned.
+  text_t getfirstkey ();
+  text_t getnextkey (const text_t &key);
+  
+protected:
+  text_t openfile;
+  GDBM_FILE gdbmfile;
+  ostream *logout;
+
+  void get_first_child (text_t &OID, infodbclass &info);
+  void get_last_child (text_t &OID, infodbclass &info);
+  void get_next_sibling (text_t &OID, infodbclass &info);
+  void get_previous_sibling (text_t &OID, infodbclass &info);
+
+  // returns true on success
+  bool getkeydata (text_t key, text_t &data);
+
+  // returns true on success
+  bool getinfoline (text_t::iterator &here, text_t::iterator end, 
+		   text_t &key, text_t &value);
+};
+
+
+#endif
Index: /tags/gsdl-21/gsdl/src/recpt/invbrowserclass.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/invbrowserclass.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/invbrowserclass.cpp	(revision 699)
@@ -0,0 +1,53 @@
+/**********************************************************************
+ *
+ * invbrowserclass.cpp -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.2  1999/10/14 22:59:34  sjboddie
+   finished off browser classes
+
+   Revision 1.1  1999/10/10 08:28:06  sjboddie
+   - metadata now returns map rather than array
+   - redesigned browsing support
+
+ */
+
+
+#include "invbrowserclass.h"
+#include <assert.h>
+#include "OIDtools.h"
+
+invbrowserclass::invbrowserclass () {
+}
+
+invbrowserclass::~invbrowserclass () {
+}
+
+// returns the name that specifies the browserclass type
+text_t invbrowserclass::get_browser_name () {
+  return "Invisible";
+}
Index: /tags/gsdl-21/gsdl/src/recpt/invbrowserclass.h
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/invbrowserclass.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/invbrowserclass.h	(revision 699)
@@ -0,0 +1,51 @@
+/**********************************************************************
+ *
+ * invbrowserclass.h -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef INVBROWSERCLASS_H
+#define INVBROWSERCLASS_H
+
+#include "browserclass.h"
+
+class invbrowserclass : public browserclass {
+protected:
+
+public:
+
+  // this browser class does nothing by default - it should
+  // be overridden for receptionists wanting to display 
+  // 'Invisible' classifications in some way
+
+  invbrowserclass ();
+  virtual ~invbrowserclass ();
+
+  // returns the name that specifies the browserclass type
+  virtual text_t get_browser_name ();
+
+};
+
+#endif
Index: /tags/gsdl-21/gsdl/src/recpt/librarymain.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/librarymain.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/librarymain.cpp	(revision 699)
@@ -0,0 +1,234 @@
+/**********************************************************************
+ *
+ * librarymain.cpp -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.15  1999/10/14 23:03:10  sjboddie
+   another browser
+
+   Revision 1.14  1999/10/10 08:14:09  sjboddie
+   - metadata now returns mp rather than array
+   - redesigned browsing support (although it's not finished so
+   won't currently work ;-)
+
+   Revision 1.13  1999/09/16 21:39:35  sjboddie
+   added gb converters
+
+   Revision 1.12  1999/09/14 22:42:06  sjboddie
+   included utf8 converter
+
+   Revision 1.11  1999/09/14 22:03:27  sjboddie
+   now reads in collections from a file to avoid recompiling
+
+   Revision 1.10  1999/09/08 00:51:11  sjboddie
+   removed old interface stuff
+
+   Revision 1.9  1999/09/07 04:56:56  sjboddie
+   added GPL notice
+
+   Revision 1.8  1999/07/15 06:08:55  rjmcnab
+   Moved the adding of the actions to librarymain so that they can
+   be overriden easier.
+
+   Revision 1.7  1999/06/15 01:56:12  sjboddie
+   Got multiple collections working
+
+   Revision 1.6  1999/05/10 03:40:39  sjboddie
+   lots of changes - slowly getting document action sorted out
+
+   Revision 1.5  1999/04/19 23:56:10  rjmcnab
+   Finished the gdbm metadata stuff
+
+   Revision 1.4  1999/04/12 03:45:05  rjmcnab
+   Finished the query filter.
+
+   Revision 1.3  1999/04/06 22:20:35  rjmcnab
+   Got browsefilter working.
+
+   Revision 1.2  1999/03/05 03:53:54  sjboddie
+
+   fixed some bugs
+
+   Revision 1.1  1999/02/21 22:35:22  rjmcnab
+
+   Initial revision.
+
+ */
+
+
+#include "receptionist.h"
+#include "cgiwrapper.h"
+#include "nullproto.h"
+#include "collectserver.h"
+#include "filter.h"
+#include "browsefilter.h"
+#include "queryfilter.h"
+#include "infodbclass.h"
+#include "mgsearch.h"
+#include "mggdbmsource.h"
+#include <assert.h>
+
+#include "action.h"
+#include "statusaction.h"
+#include "pageaction.h"
+#include "pingaction.h"
+#include "queryaction.h"
+#include "documentaction.h"
+#include "authenaction.h"
+#include "usersaction.h"
+#include "authenaction.h"
+
+#include "browserclass.h"
+#include "vlistbrowserclass.h"
+#include "hlistbrowserclass.h"
+#include "datelistbrowserclass.h"
+#include "invbrowserclass.h"
+#include "pagedbrowserclass.h"
+
+#include "gsdlhome.h"
+
+int main () {
+  receptionist recpt;
+  nullproto nproto;
+
+  text_tarray collections;
+
+  ifstream collist (GSDL_GSDLHOME "/etc/collections.txt");
+  if (!collist) {
+    exit(1);
+  }
+  char collection[100];
+  while (!collist.eof()) {
+    collist.getline (collection, 100);
+    if (collection[0] != '\0')
+      collections.push_back (collection);
+  }
+  collist.close();
+
+  text_tarray::const_iterator thiscol = collections.begin();
+  text_tarray::const_iterator endcol = collections.end();
+
+  while (thiscol != endcol) {
+
+    // this memory is created but never destroyed
+    // we're also not doing any error checking to make sure we didn't
+    // run out of memory
+    collectserver *cserver = new collectserver();
+    gdbmclass *gdbmhandler = new gdbmclass();
+    mgsearchclass *mgsearch = new mgsearchclass();
+    
+    // add a null filter
+    filterclass *filter = new filterclass ();
+    cserver->add_filter (filter);
+    
+    // add a browse filter
+    browsefilterclass *browsefilter = new browsefilterclass();
+    browsefilter->set_gdbmptr (gdbmhandler);
+    cserver->add_filter (browsefilter);
+    
+    // add a query filter
+    queryfilterclass *queryfilter = new queryfilterclass();
+    queryfilter->set_gdbmptr (gdbmhandler);
+    queryfilter->set_mgsearchptr (mgsearch);
+    cserver->add_filter (queryfilter);
+
+    // add a mg and gdbm source
+    mggdbmsourceclass *mggdbmsource = new mggdbmsourceclass ();
+    mggdbmsource->set_gdbmptr (gdbmhandler);
+    mggdbmsource->set_mgsearchptr (mgsearch);
+    cserver->add_source (mggdbmsource);
+    
+    // inform collection server and everything it contains about its
+    // collection name
+    cserver->configure ("collection", *thiscol);
+    
+    nproto.add_collectserver (cserver);
+    thiscol ++;
+  }
+  
+  // add the protocol to the receptionist
+  recpt.add_protocol (&nproto);
+
+  // add other converters
+  utf8inconvertclass utf8inconvert;
+  utf8outconvertclass utf8outconvert;
+  recpt.add_converter ("u", &utf8inconvert, &utf8outconvert);
+
+  mapinconvertclass gbinconvert;
+  gbinconvert.setmapfile (GSDL_GSDLHOME, "gbku", 0x25a1);
+  mapoutconvertclass gboutconvert;
+  gboutconvert.setmapfile (GSDL_GSDLHOME, "ugbk", 0xa1f5);
+  recpt.add_converter ("g", &gbinconvert, &gboutconvert);
+
+  // the list of actions. Note: these actions will become invalid
+  // at the end of this function.
+  statusaction astatusaction;
+  astatusaction.set_receptionist (&recpt);
+  recpt.add_action (&astatusaction);
+
+  pageaction apageaction;
+  apageaction.set_receptionist (&recpt);
+  recpt.add_action (&apageaction);
+
+  pingaction apingaction;
+  recpt.add_action (&apingaction);
+
+  queryaction aqueryaction;
+  recpt.add_action (&aqueryaction);
+
+  documentaction adocumentaction;
+  adocumentaction.set_receptionist(&recpt);
+  recpt.add_action (&adocumentaction);
+
+  usersaction ausersaction;
+  recpt.add_action (&ausersaction);
+
+  authenaction aauthenaction;
+  aauthenaction.set_receptionist(&recpt);
+  recpt.add_action (&aauthenaction);
+
+  // list of browsers
+  vlistbrowserclass avlistbrowserclass;
+  recpt.add_browser (&avlistbrowserclass);
+  recpt.setdefaultbrowser ("VList");
+
+  hlistbrowserclass ahlistbrowserclass;
+  recpt.add_browser (&ahlistbrowserclass);
+
+  datelistbrowserclass adatelistbrowserclass;
+  recpt.add_browser (&adatelistbrowserclass);
+
+  invbrowserclass ainvbrowserclass;
+  recpt.add_browser (&ainvbrowserclass);
+
+  pagedbrowserclass apagedbrowserclass;
+  recpt.add_browser (&apagedbrowserclass);
+
+  cgiwrapper (recpt, "");
+  return 0;
+}
+
Index: /tags/gsdl-21/gsdl/src/recpt/nullproto.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/nullproto.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/nullproto.cpp	(revision 699)
@@ -0,0 +1,210 @@
+/**********************************************************************
+ *
+ * nullproto.cpp -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.9  1999/09/07 04:56:56  sjboddie
+   added GPL notice
+
+   Revision 1.8  1999/06/27 22:04:47  sjboddie
+   now read in build.cfg before collect.cfg so that the indexmaps
+   are available if required to decode defaultindex, defaultsubcollection,
+   and defaultlanguage
+
+   Revision 1.7  1999/06/15 02:16:45  sjboddie
+   small change to prevent collectdir configuration string from being passed
+   to everything
+
+   Revision 1.6  1999/05/10 03:40:41  sjboddie
+   lots of changes - slowly getting document action sorted out
+
+   Revision 1.5  1999/04/30 01:59:41  sjboddie
+   lots of stuff - getting documentaction working (documentaction replaces
+   old browseaction)
+
+   Revision 1.4  1999/03/31 23:44:48  rjmcnab
+   Altered the protocol so that the metadata is part of the filter.
+
+   Revision 1.3  1999/03/03 23:26:35  sjboddie
+
+   Implemented more of the protocol
+
+   Revision 1.2  1999/02/25 21:58:58  rjmcnab
+
+   Merged sources.
+
+   Revision 1.1  1999/02/21 22:35:22  rjmcnab
+
+   Initial revision.
+
+ */
+
+
+#include "nullproto.h"
+#include "colservrconfig.h"
+#include <assert.h>
+
+
+// this configure will configure each of the collection servers
+void nullproto::configure (const text_t &key, const text_tarray &cfgline) {
+  // the naming of the collection should not be done here,
+  // it should be done just after the collection server has been 
+  // created
+  if (key == "collection" || key == "collectdir") return;
+
+  collectservermapclass::iterator here = cservers.begin();
+  collectservermapclass::iterator end = cservers.end();
+
+  while (here != end) {
+    assert ((*here).second.c != NULL);
+    if ((*here).second.c != NULL) {
+      (*here).second.c->configure (key, cfgline);
+    }
+
+    here++;
+  }
+}
+
+// this init will configure and init each of the collection servers
+bool nullproto::init (ostream &logout) {
+  collectservermapclass::iterator here = cservers.begin();
+  collectservermapclass::iterator end = cservers.end();
+
+  while (here != end) {
+    assert ((*here).second.c != NULL);
+    if ((*here).second.c != NULL) {
+      const colservrconf &configinfo = (*here).second.c->get_configinfo ();
+
+      // configure this collection server
+
+      // note that we read build.cfg before collect.cfg so that the indexmaps
+      // are available to decode defaultindex, defaultsubcollection, and
+      // defaultlanguage
+      if (!build_cfg_read (*((*here).second.c), configinfo.gsdlhome,
+			   configinfo.collection)) {
+	outconvertclass text_t2ascii;
+	logout << text_t2ascii 
+	       << "Error: couldn't read build.cfg file for collection \""
+	       << configinfo.collection << "\", gsdlhome=\""
+	       << configinfo.gsdlhome << "\"\n";
+	return false;
+      }
+
+      if (!collect_cfg_read (*((*here).second.c), configinfo.gsdlhome,
+			     configinfo.collection)) {
+	outconvertclass text_t2ascii;
+	logout << text_t2ascii 
+	       << "Error: couldn't read collect.cfg file for collection \""
+	       << configinfo.collection << "\", gsdlhome=\"" 
+	       << configinfo.gsdlhome << "\"\n";
+	return false;
+      }
+
+      if (!(*here).second.c->init (logout)) return false;
+    }
+    here++;
+  }
+
+  return true;
+}
+
+text_t nullproto::get_protocol_name () {
+  return "nullproto";
+}
+
+
+void nullproto::get_collection_list (text_tarray &collist, comerror_t &err, 
+				     ostream &/*logout*/) {
+  collist.erase(collist.begin(),collist.end());
+  err = noError;
+
+  collectservermapclass::iterator here = cservers.begin();
+  collectservermapclass::iterator end = cservers.end();
+  while (here != end) {
+    assert ((*here).second.c != NULL);
+    if ((*here).second.c != NULL) {
+      collist.push_back ((*here).second.c->get_collection_name());
+    }
+    here++;
+  }
+}
+
+void nullproto::has_collection (const text_t &collection, bool &hascollection, 
+				comerror_t &err, ostream &/*logout*/) {
+  hascollection = (cservers.getcollectserver(collection) != NULL);
+  err = noError;
+}
+
+void nullproto::ping (const text_t &collection, bool &wassuccess, 
+		      comerror_t &err, ostream &/*logout*/) {
+  wassuccess = (cservers.getcollectserver(collection) != NULL);
+  err = noError;
+}
+
+void nullproto::get_collectinfo (const text_t &collection, 
+				 ColInfoResponse_t &collectinfo,
+				 comerror_t &err, ostream &logout) {
+  collectserver *cserver = cservers.getcollectserver (collection);
+  if (cserver != NULL) cserver->get_collectinfo (collectinfo, err, logout);
+  else err = protocolError;
+}
+
+
+void nullproto::get_filterinfo (const text_t &collection,
+				InfoFiltersResponse_t &response,
+				comerror_t &err, ostream &logout) {
+  collectserver *cserver = cservers.getcollectserver (collection);
+  if (cserver != NULL) cserver->get_filterinfo (response, err, logout);
+  else err = protocolError;
+}
+
+void nullproto::get_filteroptions (const text_t &collection,
+				   const InfoFilterOptionsRequest_t &request,
+				   InfoFilterOptionsResponse_t &response, 
+				   comerror_t &err, ostream &logout) {
+  collectserver *cserver = cservers.getcollectserver (collection);
+  if (cserver != NULL) cserver->get_filteroptions (request, response, err, logout);
+  else err = protocolError;
+}
+
+void nullproto::filter (const text_t &collection,
+			FilterRequest_t &request,
+			FilterResponse_t &response,
+			comerror_t &err, ostream &logout) {
+  collectserver *cserver = cservers.getcollectserver (collection);
+  if (cserver != NULL) cserver->filter (request, response, err, logout);
+  else err = protocolError;
+}
+
+void nullproto::get_document (const text_t &collection,
+			      const DocumentRequest_t &request,
+			      DocumentResponse_t &response,
+			      comerror_t &err, ostream &logout) {
+  collectserver *cserver = cservers.getcollectserver (collection);
+  if (cserver != NULL) cserver->get_document (request, response, err, logout);
+  else err = protocolError;
+}
Index: /tags/gsdl-21/gsdl/src/recpt/nullproto.h
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/nullproto.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/nullproto.h	(revision 699)
@@ -0,0 +1,84 @@
+/**********************************************************************
+ *
+ * nullproto.h -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef NULLPROTO_H
+#define NULLPROTO_H
+
+#include "gsdlconf.h"
+#include "collectserver.h"
+#include "recptproto.h"
+
+
+class nullproto : public recptproto {
+protected:
+  collectservermapclass cservers;
+public:
+  virtual ~nullproto() {}
+
+  // add_collectserver should be called for each collection server
+  // before any configuration is done
+  void add_collectserver (collectserver *thecserver) 
+    {cservers.addcollectserver(thecserver);}
+
+  // this configure will configure each of the collection servers
+  void configure (const text_t &key, const text_tarray &cfgline);
+
+  // this init will init each of the collection servers
+  bool init (ostream &logout);
+
+  text_t get_protocol_name ();
+  void get_collection_list (text_tarray &collist, comerror_t &err, 
+			    ostream &logout);
+  void has_collection (const text_t &collection, bool &hascollection, 
+		       comerror_t &err, ostream &logout);
+  void ping (const text_t &collection, bool &wassuccess, 
+	     comerror_t &err, ostream &logout);
+  void get_collectinfo (const text_t &collection, 
+			ColInfoResponse_t &collectinfo,
+			comerror_t &err, ostream &logout);
+  void get_filterinfo (const text_t &collection,
+		       InfoFiltersResponse_t &response,
+		       comerror_t &err, ostream &logout);
+  void get_filteroptions (const text_t &collection,
+			  const InfoFilterOptionsRequest_t &request,
+			  InfoFilterOptionsResponse_t &response, 
+			  comerror_t &err, ostream &logout);
+  void filter (const text_t &collection,
+	       FilterRequest_t &request,
+	       FilterResponse_t &response,
+	       comerror_t &err, ostream &logout);
+
+  void get_document (const text_t &collection,
+		     const DocumentRequest_t &request,
+		     DocumentResponse_t &response,
+		     comerror_t &err, ostream &logout);
+
+};
+
+
+#endif
Index: /tags/gsdl-21/gsdl/src/recpt/pageaction.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/pageaction.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/pageaction.cpp	(revision 699)
@@ -0,0 +1,321 @@
+/**********************************************************************
+ *
+ * pageaction.cpp -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.17  1999/10/14 23:04:44  sjboddie
+   some hacks to make nzdl home page look right
+
+   Revision 1.16  1999/10/10 08:14:09  sjboddie
+   - metadata now returns mp rather than array
+   - redesigned browsing support (although it's not finished so
+   won't currently work ;-)
+
+   Revision 1.15  1999/09/17 04:46:05  sjboddie
+   fixed a couple of problems with 'unknown' classifier
+
+   Revision 1.14  1999/09/07 23:08:09  rjmcnab
+   Removed some compiler warnings
+
+   Revision 1.13  1999/09/07 04:56:57  sjboddie
+   added GPL notice
+
+   Revision 1.12  1999/09/02 00:26:43  rjmcnab
+   made the p cgi argument always saved in the compressed arguments
+
+   Revision 1.11  1999/08/25 04:48:43  sjboddie
+   changes to the home and about pages
+
+   Revision 1.10  1999/08/11 23:29:43  sjboddie
+   added support for html classifier (i.e. the hp argument)
+
+   Revision 1.9  1999/08/03 03:29:47  sjboddie
+   added ability to set receptionist from collect.cfg
+
+   Revision 1.8  1999/07/30 02:24:44  sjboddie
+   added collectinfo argument to some functions
+
+   Revision 1.7  1999/06/24 05:12:24  sjboddie
+   lots of small changes
+
+   Revision 1.6  1999/06/10 00:39:16  sjboddie
+   navigation bar is no longer written out for every page (it should
+   be included in the _content_ macro of pages wanting to display it).
+
+   Revision 1.5  1999/06/08 04:29:35  sjboddie
+   added argsinfo to the call to check_cgiargs to make it easy to set
+   args to their default if they're found to be screwed up
+
+   Revision 1.4  1999/02/28 20:00:14  rjmcnab
+
+
+   Fixed a few things.
+
+   Revision 1.3  1999/02/25 21:58:58  rjmcnab
+
+   Merged sources.
+
+   Revision 1.2  1999/02/21 22:33:54  rjmcnab
+
+   Lots of stuff :-)
+
+   Revision 1.1  1999/02/12 02:40:17  sjboddie
+
+   Added page action
+
+ */
+
+#include "OIDtools.h"
+#include "pageaction.h"
+#include "receptionist.h"
+#include <time.h>
+
+pageaction::pageaction () {
+
+  recpt = NULL;
+
+  // this action uses cgi variables "a", "p", and "hp"
+  cgiarginfo arg_ainfo;
+  arg_ainfo.shortname = "a";
+  arg_ainfo.longname = "action";
+  arg_ainfo.multiplechar = true;
+  arg_ainfo.defaultstatus = cgiarginfo::weak;
+  arg_ainfo.argdefault = "p";
+  arg_ainfo.savedarginfo = cgiarginfo::must;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+
+  arg_ainfo.shortname = "p";
+  arg_ainfo.longname = "page";
+  arg_ainfo.multiplechar = true;
+  arg_ainfo.defaultstatus = cgiarginfo::weak;
+  arg_ainfo.argdefault = "about";
+  arg_ainfo.savedarginfo = cgiarginfo::must;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+
+  arg_ainfo.shortname = "hp";
+  arg_ainfo.longname = "html page";
+  arg_ainfo.multiplechar = true;
+  arg_ainfo.defaultstatus = cgiarginfo::weak;
+  arg_ainfo.argdefault = "";
+  arg_ainfo.savedarginfo = cgiarginfo::mustnot;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+}
+
+pageaction::~pageaction () {
+}
+
+bool pageaction::check_cgiargs (cgiargsinfoclass &/*argsinfo*/, cgiargsclass &/*args*/, 
+				ostream &/*logout*/) {
+  // don't want to check anything yet.
+  return true;
+}
+
+void pageaction::get_cgihead_info (cgiargsclass &/*args*/, response_t &response,
+				   text_t &response_data, ostream &/*logout*/) {
+  response = content;
+  response_data = "text/html";
+}
+
+void pageaction::define_internal_macros (const ColInfoResponse_t &collectinfo, displayclass &disp, 
+					 cgiargsclass &args, recptproto *collectproto, 
+					 ostream &logout) {
+
+  // define_internal_macros sets the following macros:
+
+  // if page is "home"
+  // _homeextra_           this is the list of available collections and collection info
+  //                       to be displayed on the home page
+
+
+  // if page is "about"
+  // _numdocs_             the number of documents in the collection
+
+  // _builddate_           the date last built
+
+  // _textbrowseoptions_   the 'how to find information' text in the about and help pages
+
+  // _numbrowseoptions_    the number of browsing options 
+
+
+  // if page is "help"
+  // _textbrowseoptions_   the 'how to find information' text in the about and help pages
+
+  // _numbrowseoptions_    the number of browsing options 
+
+  
+  text_t &arg_p = args["p"];
+
+  if (arg_p == "home") {
+    // make sure we know about a receptionist
+    if (recpt == NULL) {
+      logout << "The page action does not contain information\n"
+	     << "about any receptionists. The method set_receptionist\n"
+	     <<	"was probably not called from the module which instantiated\n"
+	     <<	"this page action.\n";
+      return;
+    }
+
+    text_t homeextra = "<center><table width=_pagewidth_><tr valign=top>\n";
+
+    recptprotolistclass *rprotolist = recpt->get_recptprotolist_ptr ();
+    if (rprotolist == NULL) return;
+
+    recptprotolistclass::iterator rprotolist_here = rprotolist->begin();
+    recptprotolistclass::iterator rprotolist_end = rprotolist->end();
+    while (rprotolist_here != rprotolist_end) {
+      if ((*rprotolist_here).p != NULL) {
+
+	text_tarray collist;
+	comerror_t err;
+	(*rprotolist_here).p->get_collection_list (collist, err, logout);
+	if (err == noError) {
+	  text_tarray::iterator collist_here = collist.begin();
+	  text_tarray::iterator collist_end = collist.end();
+
+	  int row1 = 6;
+	  int row2 = 4;
+	  int count = 1;
+	  while (collist_here != collist_end) {
+
+	    if (*collist_here == "niupepa")
+	      homeextra += "<p><a href=\"_httpmusiclibrary_\">_iconmusiclibrary_</a>\n";
+
+
+	    ColInfoResponse_t cinfo;
+	    (*rprotolist_here).p->get_collectinfo (*collist_here, cinfo, err, logout);
+	    if (err == noError) {
+	      if (cinfo.isPublic) {
+
+		FilterResponse_t response;
+		text_tset metadata;
+		metadata.insert ("collectionname");
+		metadata.insert ("iconcollection");
+		metadata.insert ("iconcollectionsmall");
+		text_t collectionname = *collist_here;
+		text_t alt = collectionname;
+		if (get_info ("collection", *collist_here, metadata, false, 
+			      (*rprotolist_here).p, response, logout)) {
+		  if (!response.docInfo[0].metadata["collectionname"].values[0].empty())
+		    alt = response.docInfo[0].metadata["collectionname"].values[0];
+		  
+		  if (!response.docInfo[0].metadata["iconcollectionsmall"].values[0].empty())
+		    collectionname = "<img width=150 border=1 src=\""
+		      + response.docInfo[0].metadata["iconcollectionsmall"].values[0]
+		      + "\" alt=\"" + alt + "\">";
+		  else if (!response.docInfo[0].metadata["iconcollection"].values[0].empty())
+		    collectionname = "<img width=150 border=1 src=\""
+		      + response.docInfo[0].metadata["iconcollection"].values[0]
+		      + "\" alt=\"" + alt + "\">";
+		  else collectionname = alt;
+		}
+		
+		if ((count == 1) || (count == (row1+1)) || (count == ((row1+row2)+1))) 
+		  homeextra += "<td align=center>";
+		else homeextra += "<p>";
+		text_t link = "<a href=\"_gwcgi_?a=p&p=about&c=" + *collist_here + "\">";
+		if (*collist_here == "chinese")
+		  link = "<a href=\"_gwcgi_?a=p&p=about&l=zh&nw=u&c=" + *collist_here + "\">";
+
+		if (!cinfo.receptionist.empty()) 
+		  link = "<a href=\"" + cinfo.receptionist + "?a=p&p=about&c=" 
+		    + *collist_here + "\">";
+		
+		homeextra += link + collectionname + "</a>\n";
+		
+		if ((count == row1) || (count == (row1+row2))) 
+		  homeextra += "</td>";
+		
+		count ++;
+	      }
+	    }
+
+	    collist_here ++;
+	  }
+	  homeextra += "</tr></table></center>\n";
+	  disp.setmacro ("homeextra", "home", homeextra); 
+	}
+      }
+      rprotolist_here ++;
+    }
+  } else if (arg_p == "about") {
+    disp.setmacro ("numdocs", "about", collectinfo.numDocs);
+    unsigned long current_time = time(NULL);
+    unsigned long builddate = (current_time - collectinfo.buildDate) / 86400;
+    disp.setmacro ("builddate", "about", builddate);
+  }
+
+  if (arg_p == "about" || arg_p == "help") {
+
+    // _textbrowseoptions_ and _numbrowseoptions_
+
+    FilterResponse_t response;
+    text_tset metadata;
+    metadata.insert ("Title");
+    bool getParents = false;
+    get_children ("", args["c"], metadata, getParents, collectproto, response, logout);
+
+    disp.setmacro ("numbrowseoptions", "help", response.docInfo.size()+1);
+
+    ResultDocInfo_tarray::iterator here = response.docInfo.begin();
+    ResultDocInfo_tarray::iterator end = response.docInfo.end();
+
+    // we're assuming that we've always got a search button
+    text_t shorttext = "<ul><li>_textSearchshort_\n";
+    text_t longtext = "_textSearchlong_";
+
+    while (here != end) {
+      text_t title = (*here).metadata["Title"].values[0];
+
+      text_t stext, ltext;
+      disp.expandstring ("document", "_text" + title + "short_", stext);
+      if (stext == ("_text" + title + "short_")) {
+	stext = "<li>_textdefaultshorttext_";
+	ltext = "_textdefaultlongtext_";
+      } else {
+	shorttext += "<li>" + stext;
+	longtext += "_text" + title + "long_";
+      }
+      here ++;
+    }
+    shorttext += "</ul>\n";
+    disp.setmacro ("textbrowseoptions", "help", shorttext + longtext);
+  }
+}
+
+bool pageaction::do_action (cgiargsclass &args, const ColInfoResponse_t &/*collectinfo*/, 
+			    recptproto * /*collectproto*/, displayclass &disp, 
+			    outconvertclass &outconvert, ostream &textout, 
+			    ostream &/*logout*/) {
+
+  text_t &arg_p = args["p"];
+
+  textout << outconvert << disp << ("_" + arg_p + ":header_\n")
+	  << ("_" + arg_p + ":content_\n")
+	  << ("_" + arg_p + ":footer_\n");
+
+  return true;
+}
Index: /tags/gsdl-21/gsdl/src/recpt/pageaction.h
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/pageaction.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/pageaction.h	(revision 699)
@@ -0,0 +1,69 @@
+/**********************************************************************
+ *
+ * pageaction.h -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef PAGEACTION_H
+#define PAGEACTION_H
+
+
+#include "gsdlconf.h"
+#include "action.h"
+#include "receptionist.h"
+
+
+class pageaction : public action {
+  
+protected:
+  receptionist *recpt;
+
+  text_tset classify_meta;
+
+public:
+  pageaction ();
+  virtual ~pageaction ();
+
+  text_t get_action_name () {return "p";}
+
+  void set_receptionist (receptionist *therecpt) {recpt=therecpt;}
+  bool check_cgiargs (cgiargsinfoclass &argsinfo, cgiargsclass &args, 
+		      ostream &logout);
+
+  void get_cgihead_info (cgiargsclass &args, response_t &response,
+			 text_t &response_data, ostream &logout);
+
+  void define_internal_macros (const ColInfoResponse_t &collectinfo, displayclass &disp, 
+			       cgiargsclass &args, recptproto *collectproto, 
+			       ostream &logout);
+
+  bool do_action (cgiargsclass &args, const ColInfoResponse_t &collectinfo, 
+		  recptproto *collectproto, displayclass &disp, 
+		  outconvertclass &outconvert, ostream &textout, 
+		  ostream &logout);
+};
+
+
+#endif
Index: /tags/gsdl-21/gsdl/src/recpt/pagedbrowserclass.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/pagedbrowserclass.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/pagedbrowserclass.cpp	(revision 699)
@@ -0,0 +1,178 @@
+/**********************************************************************
+ *
+ * pagedbrowserclass.cpp -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1999/10/14 22:59:35  sjboddie
+   finished off browser classes
+
+ */
+
+
+#include "pagedbrowserclass.h"
+#include <assert.h>
+#include "OIDtools.h"
+
+pagedbrowserclass::pagedbrowserclass () {
+}
+
+pagedbrowserclass::~pagedbrowserclass () {
+}
+
+// returns the name that specifies the browserclass type
+text_t pagedbrowserclass::get_browser_name () {
+  return "Paged";
+}
+
+
+void pagedbrowserclass::load_metadata_defaults (text_tset &metadata) {
+  metadata.insert ("Title");
+}
+
+text_t pagedbrowserclass::get_default_formatstring () {
+  return "";
+}
+
+// if the "gp" (go to page) argument is set we need to set 
+// the "d" argument to the corresponding page
+void pagedbrowserclass::processOID (cgiargsclass &args, recptproto *collectproto,
+				    ostream &logout) {
+
+  text_t &arg_d = args["d"];
+  text_t &arg_gp = args["gp"];
+  
+  if ((!arg_d.empty()) && (!arg_gp.empty()) && (is_number (arg_gp))) {
+    text_t top;
+    get_top (arg_d, top);
+    text_tset metadata;
+    metadata.insert ("Title");
+    bool getParents = false;
+    FilterResponse_t response;
+    get_children (top, args["c"], metadata, getParents, collectproto, response, logout);
+    ResultDocInfo_tarray::iterator dochere = response.docInfo.begin();
+    ResultDocInfo_tarray::iterator docend = response.docInfo.end();
+    while (dochere != docend) {
+      if ((*dochere).metadata["Title"].values[0] == arg_gp) {
+	arg_d = (*dochere).OID;
+	break;
+      }
+      dochere ++;
+    }
+  }
+}
+
+
+int pagedbrowserclass::output_section_group (ResultDocInfo_t &section, cgiargsclass &args,
+					     int /*colnumber*/, format_t */*formatlistptr*/, 
+					     bool /*use_table*/, text_tset &/*metadata*/, bool &/*getParents*/,
+					     recptproto */*collectproto*/, displayclass &disp, 
+					     outconvertclass &outconvert, ostream &textout, ostream &/*logout*/) {
+
+  // this browser class only handles document levels
+  if (args["d"].empty()) return 0;
+
+
+
+  if (section.OID != args["d"]) {
+    // set the _parentarrow_ macro 
+    text_t parentarrow = "<a href=\"_httpdocument_&cl=" + args["cl"] +
+      "&d=" + section.OID + "\">_iconprev_</a>\n";
+    disp.setmacro ("parentarrow", "document", parentarrow);
+    return 0;
+  }
+
+  // must be at top level to get to here!
+  textout << outconvert << disp << "<table><tr valign=top><td colspan=3><center><b>"
+	  << section.metadata["Title"].values[0] << "_document:textintro_</b>\n";
+  
+  return 0;
+}
+
+int pagedbrowserclass::output_section_group (FilterResponse_t &sections, cgiargsclass &args,
+					     int /*colnumber*/, format_t */*formatlistptr*/, 
+					     bool /*use_table*/, text_tset &/*metadata*/, bool &/*getParents*/,
+					     recptproto */*collectproto*/, displayclass &disp, 
+					     outconvertclass &outconvert, ostream &textout, ostream &/*logout*/) {
+
+  text_t &arg_d = args["d"];
+
+  // this browser class only handles document levels
+  if (arg_d.empty()) return 0;
+
+  text_t previousOID, previoustitle, nextOID, nexttitle;
+  text_t previousarrow, nextarrow;
+  bool found = false;
+
+  // this should be our list of pages
+
+  ResultDocInfo_tarray::iterator thissection = sections.docInfo.begin();
+  ResultDocInfo_tarray::iterator endsection = sections.docInfo.end();
+
+  while (thissection != endsection) {
+    if (arg_d == (*thissection).OID) {
+      found = true;
+      textout << outconvert << disp << "<table><tr valign=top><td colspan=3><center><b>_document:textpage_"
+	      << (*thissection).metadata["Title"].values[0] << "</b>\n";
+      if (thissection != sections.docInfo.begin()) {
+	previousOID = (*(thissection-1)).OID;
+	previoustitle = (*(thissection-1)).metadata["Title"].values[0];
+      } else 
+	previousarrow = "_document:parentarrow_";
+
+      if (thissection != sections.docInfo.end()) {
+	nextOID = (*(thissection+1)).OID;
+	nexttitle = (*(thissection+1)).metadata["Title"].values[0];
+      }
+      break;
+    }
+    thissection ++;
+  }
+
+  int numpages = sections.docInfo.size();
+  textout << outconvert << disp 
+	  << ("_document:textnumpages_(" + text_t(numpages) + ")</center></td></tr>\n");
+
+  if (!found)
+    nextarrow = "<a href=\"_httpdocument_&cl=" + args["cl"] + "&d=" + sections.docInfo[0].OID + 
+      "\">" + sections.docInfo[0].metadata["Title"].values[0] + "_iconnext_";
+  else {
+    if (!previousOID.empty())
+      previousarrow = "<a href=\"_httpdocument_&cl=" + args["cl"] + "&d=" + 
+	previousOID + "\">_iconprev_" + previoustitle + "</a>\n";
+    if (!nextOID.empty())
+      nextarrow = "<a href=\"_httpdocument_&cl=" + args["cl"] + "&d=" + 
+	nextOID + "\">" + nexttitle + "_iconnext_</a>\n";
+  }
+
+  textout << outconvert << disp << "<tr valign=middle>\n"
+	  << "<td align=right>" << previousarrow << "</td>\n"
+	  << "<td align=center valign=top>_document:gotoform_</td>"
+	  << "<td align=left>" << nextarrow << "</td>\n"
+	  << "</tr></table>\n";
+
+  return 0;
+}
Index: /tags/gsdl-21/gsdl/src/recpt/pagedbrowserclass.h
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/pagedbrowserclass.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/pagedbrowserclass.h	(revision 699)
@@ -0,0 +1,66 @@
+/**********************************************************************
+ *
+ * pagedbrowserclass.h -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef PAGEDBROWSERCLASS_H
+#define PAGEDBROWSERCLASS_H
+
+#include "browserclass.h"
+
+class pagedbrowserclass : public browserclass {
+protected:
+
+public:
+
+  pagedbrowserclass ();
+  virtual ~pagedbrowserclass ();
+
+  // returns the name that specifies the browserclass type
+  virtual text_t get_browser_name ();
+
+  void load_metadata_defaults (text_tset &metadata);
+
+  text_t get_default_formatstring ();
+
+  void processOID (cgiargsclass &args, recptproto *collectproto,
+		   ostream &logout);
+
+  int output_section_group (ResultDocInfo_t &section, cgiargsclass &args,
+			    int colnumber, format_t *formatlistptr, 
+			    bool use_table, text_tset &metadata, bool &getParents,
+			    recptproto *collectproto, displayclass &disp, 
+			    outconvertclass &outconvert, ostream &textout, ostream &logout);
+
+  int output_section_group (FilterResponse_t &sections, cgiargsclass &args,
+			    int colnumber, format_t *formatlistptr, 
+			    bool use_table, text_tset &metadata, bool &getParents,
+			    recptproto *collectproto, displayclass &disp, 
+			    outconvertclass &outconvert, ostream &textout, ostream &logout);
+
+};
+
+#endif
Index: /tags/gsdl-21/gsdl/src/recpt/pingaction.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/pingaction.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/pingaction.cpp	(revision 699)
@@ -0,0 +1,101 @@
+/**********************************************************************
+ *
+ * pingaction.cpp -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.4  1999/09/07 04:56:57  sjboddie
+   added GPL notice
+
+   Revision 1.3  1999/07/30 02:24:45  sjboddie
+   added collectinfo argument to some functions
+
+   Revision 1.2  1999/02/25 21:58:59  rjmcnab
+
+   Merged sources.
+
+   Revision 1.1  1999/02/21 22:35:20  rjmcnab
+
+   Initial revision.
+
+ */
+
+
+#include "pingaction.h"
+
+pingaction::pingaction () {
+  // this action uses cgi variable "a"
+  cgiarginfo arg_ainfo;
+  arg_ainfo.shortname = "a";
+  arg_ainfo.longname = "action";
+  arg_ainfo.multiplechar = true;
+  arg_ainfo.defaultstatus = cgiarginfo::weak;
+  arg_ainfo.argdefault = "ping";
+  arg_ainfo.savedarginfo = cgiarginfo::must;
+
+  argsinfo.addarginfo (NULL, arg_ainfo);
+
+}
+
+void pingaction::get_cgihead_info (cgiargsclass &/*args*/, response_t &response,
+				   text_t &response_data, ostream &/*logout*/) {
+  response = content;
+  response_data = "text/html";
+}
+
+
+bool pingaction::do_action (cgiargsclass &args, const ColInfoResponse_t &/*collectinfo*/, 
+			    recptproto *collectproto, displayclass &/*disp*/, 
+			    outconvertclass &outconvert, ostream &textout, 
+			    ostream &logout) {
+  bool wassuccess = false;
+  comerror_t err;
+  
+  if (!args["c"].empty() && (collectproto != NULL)) {
+    collectproto->ping (args["c"], wassuccess, err, logout);
+    if (err != noError) wassuccess = false; // a communication error
+  }
+
+  textout << outconvert
+	  << "<html>\n"
+	  << "<head>\n"
+	  << "<title>Ping</title>\n"
+	  << "</head>\n"
+	  << "<body>\n"
+	  << "Ping for \"" << args["c"] << "\" ";
+
+  if (wassuccess) {
+    textout << outconvert << " succeeded.\n";
+  } else {
+    textout << outconvert << " did not succeed.\n";
+  }
+
+  textout << outconvert 
+	  << "</body>\n"
+	  << "</html>\n";
+
+  return true;
+};
Index: /tags/gsdl-21/gsdl/src/recpt/pingaction.h
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/pingaction.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/pingaction.h	(revision 699)
@@ -0,0 +1,51 @@
+/**********************************************************************
+ *
+ * pingaction.h -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef PINGACTION_H
+#define PINGACTION_H
+
+#include "gsdlconf.h"
+#include "action.h"
+
+
+class pingaction : public action {
+public:
+  pingaction ();
+  virtual ~pingaction () {}
+
+  text_t get_action_name () {return "ping";}
+  void get_cgihead_info (cgiargsclass &args, response_t &response,
+			 text_t &response_data, ostream &logout);
+  bool do_action (cgiargsclass &args, const ColInfoResponse_t &collectinfo, 
+		  recptproto *collectproto, displayclass &disp, 
+		  outconvertclass &outconvert, ostream &textout, 
+		  ostream &logout);
+};
+
+
+#endif
Index: /tags/gsdl-21/gsdl/src/recpt/queryaction.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/queryaction.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/queryaction.cpp	(revision 699)
@@ -0,0 +1,653 @@
+/**********************************************************************
+ *
+ * queryaction.cpp -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.26  1999/10/10 08:14:10  sjboddie
+   - metadata now returns mp rather than array
+   - redesigned browsing support (although it's not finished so
+   won't currently work ;-)
+
+   Revision 1.25  1999/09/24 04:49:39  sjboddie
+   fixed up the query selection boxes to display properly if there's only
+   a single index/sub-collection
+
+   Revision 1.24  1999/09/22 03:44:31  sjboddie
+   EndResults query filter option may now take '-1' for 'all'
+
+   Revision 1.23  1999/09/21 11:34:42  sjboddie
+   added Maxdocs queryfilter option which may be -1 for 'all'
+
+   Revision 1.22  1999/09/07 23:08:51  rjmcnab
+   removed some compiler warnings
+
+   Revision 1.21  1999/09/07 04:56:57  sjboddie
+   added GPL notice
+
+   Revision 1.20  1999/08/25 04:47:55  sjboddie
+   added advanced search option - other minor changes
+
+   Revision 1.19  1999/08/13 04:17:24  sjboddie
+   small change to do with new collection-level metadata
+
+   Revision 1.18  1999/08/10 22:46:33  sjboddie
+   changed format option result to QueryResults and added QueryLinks option
+
+   Revision 1.17  1999/07/30 02:24:42  sjboddie
+   added collectinfo argument to some functions
+
+   Revision 1.16  1999/07/19 00:16:58  sjboddie
+   no longer display documents that don't match all phrases in query string
+
+   Revision 1.15  1999/07/16 08:33:36  rjmcnab
+   Changed the logic for getting the results string slightly
+
+   Revision 1.14  1999/07/16 03:41:29  sjboddie
+   changed isApprox
+
+   Revision 1.13  1999/07/16 00:19:01  sjboddie
+   some changes to the way quoted queries are handled
+
+   Revision 1.12  1999/07/09 02:17:55  rjmcnab
+   Setting macros needed for a second query.
+
+   Revision 1.11  1999/07/07 06:13:10  rjmcnab
+   Added ability to combine two independant queries.
+
+   Revision 1.10  1999/07/07 05:49:35  sjboddie
+   had another crack at the format string code - created a new formattools
+   module. It can now handle {If} and {Or} statements although there's a
+   bug preventing nested if's and or's.
+
+   Revision 1.9  1999/07/01 22:48:46  sjboddie
+   had a go at getting a query result format string working
+
+   Revision 1.8  1999/06/27 22:02:11  sjboddie
+   author is added  to queryresults if there is one
+
+   Revision 1.7  1999/06/26 01:10:18  rjmcnab
+   Made h, i, and n arguments saved in the compressed arguments.
+
+   Revision 1.6  1999/06/24 05:12:25  sjboddie
+   lots of small changes
+
+   Revision 1.5  1999/06/16 04:03:48  sjboddie
+   Now sets "cl" arg to "search" when going to a document from a search
+   results page. This allows the close book icon (in hierarchy toc) to
+   take you back to the results page if that's where you came from.
+   If you got to the document page somehow other than from a
+   classification or a search (i.e. if "cl" isn't set) then the close
+   book icon is disabled
+
+   Revision 1.4  1999/06/16 02:08:38  sjboddie
+   got queryaction working
+
+   Revision 1.3  1999/03/25 03:06:45  sjboddie
+
+   altered receptionist slightly so it now passes *collectproto to
+   define_internal_macros and define_external_macros - need it
+   for browseaction
+
+   Revision 1.2  1999/03/03 20:26:50  rjmcnab
+
+   Modified stuff.
+
+   Revision 1.1  1999/02/28 22:45:21  rjmcnab
+
+   Initial revision.
+
+ */
+
+
+#include "queryaction.h"
+#include "querytools.h"
+#include "formattools.h"
+
+queryaction::queryaction () {
+
+  num_phrases = 0;
+
+  // this action uses cgi variable "a"
+  cgiarginfo arg_ainfo;
+  arg_ainfo.shortname = "a";
+  arg_ainfo.longname = "action";
+  arg_ainfo.multiplechar = true;
+  arg_ainfo.defaultstatus = cgiarginfo::weak;
+  arg_ainfo.argdefault = "q";
+  arg_ainfo.savedarginfo = cgiarginfo::must;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+
+  // "b" - 0 = simple, 1 = advanced
+  arg_ainfo.shortname = "b";
+  arg_ainfo.longname = "query mode";
+  arg_ainfo.multiplechar = false;
+  arg_ainfo.defaultstatus = cgiarginfo::weak;
+  arg_ainfo.argdefault = "0";
+  arg_ainfo.savedarginfo = cgiarginfo::must;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+
+  // "h"
+  arg_ainfo.shortname = "h";
+  arg_ainfo.longname = "main index";
+  arg_ainfo.multiplechar = true;
+  arg_ainfo.defaultstatus = cgiarginfo::weak;
+  arg_ainfo.argdefault = "";
+  arg_ainfo.savedarginfo = cgiarginfo::must;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+
+  // "h2"
+  arg_ainfo.shortname = "h2";
+  arg_ainfo.longname = "main index for second query";
+  arg_ainfo.multiplechar = true;
+  arg_ainfo.defaultstatus = cgiarginfo::weak;
+  arg_ainfo.argdefault = "";
+  arg_ainfo.savedarginfo = cgiarginfo::must;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+
+  // "j"
+  arg_ainfo.shortname = "j";
+  arg_ainfo.longname = "sub collection index";
+  arg_ainfo.multiplechar = true;
+  arg_ainfo.defaultstatus = cgiarginfo::weak;
+  arg_ainfo.argdefault = "";
+  arg_ainfo.savedarginfo = cgiarginfo::must;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+
+  // "j2"
+  arg_ainfo.shortname = "j2";
+  arg_ainfo.longname = "sub collection index for second query";
+  arg_ainfo.multiplechar = true;
+  arg_ainfo.defaultstatus = cgiarginfo::weak;
+  arg_ainfo.argdefault = "";
+  arg_ainfo.savedarginfo = cgiarginfo::must;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+
+  // "n"
+  arg_ainfo.shortname = "n";
+  arg_ainfo.longname = "language index";
+  arg_ainfo.multiplechar = true;
+  arg_ainfo.defaultstatus = cgiarginfo::weak;
+  arg_ainfo.argdefault = "";
+  arg_ainfo.savedarginfo = cgiarginfo::must;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+
+  // "n2"
+  arg_ainfo.shortname = "n2";
+  arg_ainfo.longname = "language index for second query";
+  arg_ainfo.multiplechar = true;
+  arg_ainfo.defaultstatus = cgiarginfo::weak;
+  arg_ainfo.argdefault = "";
+  arg_ainfo.savedarginfo = cgiarginfo::must;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+
+  // "q"
+  arg_ainfo.shortname = "q";
+  arg_ainfo.longname = "query string";
+  arg_ainfo.multiplechar = true;
+  arg_ainfo.defaultstatus = cgiarginfo::weak;
+  arg_ainfo.argdefault = "";
+  arg_ainfo.savedarginfo = cgiarginfo::must;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+
+  // "q2"
+  arg_ainfo.shortname = "q2";
+  arg_ainfo.longname = "query string for second query";
+  arg_ainfo.multiplechar = true;
+  arg_ainfo.defaultstatus = cgiarginfo::weak;
+  arg_ainfo.argdefault = "";
+  arg_ainfo.savedarginfo = cgiarginfo::must;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+
+  // "cq2" ""=don't combine, "and", "or", "not"
+  arg_ainfo.shortname = "cq2";
+  arg_ainfo.longname = "combine queries";
+  arg_ainfo.multiplechar = true;
+  arg_ainfo.defaultstatus = cgiarginfo::weak;
+  arg_ainfo.argdefault = "";
+  arg_ainfo.savedarginfo = cgiarginfo::must;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+
+  // "t" - 1 = ranked 0 = boolean
+  arg_ainfo.shortname = "t";
+  arg_ainfo.longname = "search type";
+  arg_ainfo.multiplechar = false;
+  arg_ainfo.defaultstatus = cgiarginfo::weak;
+  arg_ainfo.argdefault = "1";
+  arg_ainfo.savedarginfo = cgiarginfo::must;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+
+  // "k"
+  arg_ainfo.shortname = "k";
+  arg_ainfo.longname = "casefolding";
+  arg_ainfo.multiplechar = false;
+  arg_ainfo.defaultstatus = cgiarginfo::weak;
+  arg_ainfo.argdefault = "1";
+  arg_ainfo.savedarginfo = cgiarginfo::must;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+
+  // "s"
+  arg_ainfo.shortname = "s";
+  arg_ainfo.longname = "stemming";
+  arg_ainfo.multiplechar = false;
+  arg_ainfo.defaultstatus = cgiarginfo::weak;
+  arg_ainfo.argdefault ="0";
+  arg_ainfo.savedarginfo = cgiarginfo::must;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+
+  // "m"
+  arg_ainfo.shortname = "m";
+  arg_ainfo.longname = "maximum number of documents";
+  arg_ainfo.multiplechar = true;
+  arg_ainfo.defaultstatus = cgiarginfo::weak;
+  arg_ainfo.argdefault = "50";
+  arg_ainfo.savedarginfo = cgiarginfo::must;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+
+  // "o"
+  arg_ainfo.shortname = "o";
+  arg_ainfo.longname = "hits per page";
+  arg_ainfo.multiplechar = true;
+  arg_ainfo.defaultstatus = cgiarginfo::weak;
+  arg_ainfo.argdefault = "20";
+  arg_ainfo.savedarginfo = cgiarginfo::must;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+
+  // "r"
+  arg_ainfo.shortname = "r";
+  arg_ainfo.longname = "start results from";
+  arg_ainfo.multiplechar = true;
+  arg_ainfo.defaultstatus = cgiarginfo::weak;
+  arg_ainfo.argdefault = "1";
+  arg_ainfo.savedarginfo = cgiarginfo::must;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+}
+
+void queryaction::configure (const text_t &key, const text_tarray &cfgline) {
+  action::configure (key, cfgline);
+}
+
+bool queryaction::init (ostream &logout) {
+  return action::init (logout);
+}
+
+bool queryaction::check_cgiargs (cgiargsinfoclass &argsinfo, cgiargsclass &args, 
+				 ostream &logout) {
+
+  // check t argument
+  int arg_t = args.getintarg("t");
+  if (arg_t != 0 && arg_t != 1) {
+    logout << "Warning: \"t\" argument out of range (" << arg_t << ")\n";
+    cgiarginfo *tinfo = argsinfo.getarginfo ("t");
+    if (tinfo != NULL) args["t"] = tinfo->argdefault;
+  }
+
+  // check k argument
+  int arg_k = args.getintarg("k");
+  if (arg_k != 0 && arg_k != 1) {
+    logout << "Warning: \"k\" argument out of range (" << arg_k << ")\n";
+    cgiarginfo *kinfo = argsinfo.getarginfo ("k");
+    if (kinfo != NULL) args["k"] = kinfo->argdefault;
+  }
+
+  // check s argument
+  int arg_s = args.getintarg("s");
+  if (arg_s != 0 && arg_s != 1) {
+    logout << "Warning: \"s\" argument out of range (" << arg_s << ")\n";
+    cgiarginfo *sinfo = argsinfo.getarginfo ("s");
+    if (sinfo != NULL) args["s"] = sinfo->argdefault;
+  }
+
+  // check m argument
+  int arg_m = args.getintarg("m");
+  if (arg_m < -1) {
+    logout << "Warning: \"m\" argument less than -1 (" << arg_m << ")\n";
+    cgiarginfo *minfo = argsinfo.getarginfo ("m");
+    if (minfo != NULL) args["m"] = minfo->argdefault;
+  }
+
+  // check o argument
+  int arg_o = args.getintarg("o");
+  if (arg_o < -1) {
+    logout << "Warning: \"o\" argument less than -1 (" << arg_o << ")\n";
+    cgiarginfo *oinfo = argsinfo.getarginfo ("o");
+    if (oinfo != NULL) args["o"] = oinfo->argdefault;
+  }
+
+  // check r argument
+  int arg_r = args.getintarg("r");
+  if (arg_r < 1) {
+    logout << "Warning: \"r\" argument less than 1 (" << arg_r << ")\n";
+    cgiarginfo *rinfo = argsinfo.getarginfo ("r");
+    if (rinfo != NULL) args["r"] = rinfo->argdefault;
+  }
+
+  return true;
+}
+
+void queryaction::get_cgihead_info (cgiargsclass &/*args*/, response_t &response,
+				     text_t &response_data, ostream &/*logout*/) {
+  response = content;
+  response_data = "text/html";
+}
+
+void queryaction::define_internal_macros (const ColInfoResponse_t &/*collectinfo*/, displayclass &disp, 
+					  cgiargsclass &args, recptproto * /*collectproto*/, 
+					  ostream &/*logout*/) {
+
+  // define_internal_macros sets the following macros:
+
+  // _quotedquery_  the part of the query string that was quoted for post-processing
+
+
+
+  // The following macros are set later (in define_query_macros) as they can't be set until 
+  // the query has been done.
+
+  // _freqmsg_      the term frequency string
+
+  // _resultline_   the "x documents matched the query" string
+
+  // _prevfirst_    these are used when setting up the links to previous/next
+  // _prevlast_     pages of results (_thisfirst_ and _thislast_ are used to set
+  // _nextfirst_    the 'results x-x for query: xxxx' string in the title bar)
+  // _nextlast_
+  // _thisfirst_
+  // _thislast_
+
+
+  // get the quoted bits of the query string and set _quotedquery_
+  text_tarray phrases;
+  get_phrases (args["q"], phrases);
+  text_tarray::const_iterator phere = phrases.begin();
+  text_tarray::const_iterator pend = phrases.end();
+  bool first = true;
+  text_t quotedquery;
+  while (phere != pend) {
+    if (!first)
+      if ((phere +1) == pend) quotedquery += " and ";
+      else quotedquery += ", ";
+	
+    quotedquery += "\"" + *phere + "\"";
+    first = false;
+    phere ++;
+  }
+  if (args.getintarg("s")) quotedquery += "_textstemon_";
+  disp.setmacro ("quotedquery", "query", quotedquery);
+
+  // we'll also set num_phrases here so we don't have to parse the 
+  // querystring again later (we need to know this before outputting 
+  // results so we don't include results for documents not containing 
+  // all requested phrases).
+  num_phrases = phrases.size();
+
+}
+
+// sets the selection box macros _hselection_, _jselection_, and _nselection_.
+void queryaction::set_option_macro (const text_t &macroname, text_t current_value,
+				    const FilterOption_t &option, displayclass &disp) {
+  
+  if (option.validValues.empty()) return;
+  else if (option.validValues.size() == 1) {
+    disp.setmacro (macroname + "selection", "Global", "_" + option.defaultValue + "_");
+    return;
+  } 
+  if (option.validValues.size() < 2) return;
+  
+  text_t macrovalue = "<select name=\"" + macroname + "\">\n";
+  
+  if (current_value.empty()) current_value = option.defaultValue;
+  
+  text_tarray::const_iterator thisvalue = option.validValues.begin();
+  text_tarray::const_iterator endvalue = option.validValues.end();
+
+  while (thisvalue != endvalue) {
+    macrovalue += "<option value=\"" + *thisvalue + "\"";
+    if (*thisvalue == current_value)
+      macrovalue += " selected";
+    macrovalue += ">_" + *thisvalue + "_\n";
+    thisvalue ++;
+  }
+  macrovalue += "</select>\n";
+  disp.setmacro (macroname + "selection", "Global", macrovalue);
+}
+
+void queryaction::define_external_macros (const ColInfoResponse_t &/*collectinfo*/, displayclass &disp, 
+					  cgiargsclass &args, recptproto *collectproto, 
+					  ostream &logout) {
+
+  // define_external_macros sets the following macros:
+
+  // some or all of these may not be required to be set
+  // _hselection_, _h2selection_  the selection box for the main part of the index
+  // _jselection_, _j2selection_  the selection box for the subcollection part of the index
+  // _nselection_, _n2selection_  the selection box for the language part of the index
+  // _cq2selection                the selection box for combining two queries
+  
+  
+  // can't do anything if collectproto is null (i.e. no collection was specified)
+  if (collectproto == NULL) return;
+
+  comerror_t err;
+  InfoFilterOptionsResponse_t response;
+  InfoFilterOptionsRequest_t request;
+  request.filterName = "QueryFilter";
+  
+  collectproto->get_filteroptions (args["c"], request, response, err, logout);
+  if (err == noError) {
+    
+    FilterOption_tmap::const_iterator it;
+    FilterOption_tmap::const_iterator end = response.filterOptions.end();
+    
+    // _hselection_ and _h2selection_ (Index)
+    it = response.filterOptions.find ("Index");
+    if (it != end) set_option_macro ("h", args["h"], (*it).second, disp);
+    if (it != end) set_option_macro ("h2", args["h2"], (*it).second, disp);
+    
+    // _jselection_ and _j2selection_ (Subcollection)
+    it = response.filterOptions.find ("Subcollection");
+    if (it != end) set_option_macro ("j", args["j"], (*it).second, disp);
+    if (it != end) set_option_macro ("j2", args["j2"], (*it).second, disp);
+    
+    // _nselection_ and _n2selection_ (Language)
+    it = response.filterOptions.find ("Language");
+    if (it != end) set_option_macro ("n", args["n"], (*it).second, disp);
+    if (it != end) set_option_macro ("n2", args["n2"], (*it).second, disp);
+
+    // _cq2selection_ (CombineQuery)
+    it = response.filterOptions.find ("CombineQuery");
+    if (it != end) set_option_macro ("cq2", args["cq2"], (*it).second, disp);
+  }
+}
+
+bool queryaction::do_action (cgiargsclass &args, const ColInfoResponse_t &collectinfo, 
+			     recptproto *collectproto, displayclass &disp, 
+			     outconvertclass &outconvert, ostream &textout, 
+			     ostream &logout) {
+
+  if (formatstring.empty()) {
+    text_tmap::const_iterator result = collectinfo.format.find("QueryResults");
+    if (result != collectinfo.format.end())
+      formatstring = (*result).second;
+  }
+
+  // see if there's a QueryLinks format option
+  text_t querylinkmeta;
+  bool havequerylink = false;
+  text_tmap::const_iterator it = collectinfo.format.find("QueryLinks");
+  if (it != collectinfo.format.end()) {
+    querylinkmeta = (*it).second;
+    havequerylink = true;
+  }
+
+  // if we still don't have a format string use the default
+  if (formatstring.empty()) 
+    formatstring = "<td valign=top nowrap>[link]_icontext_[/link]</td><td>[Title]</td>";
+
+  if (collectproto == NULL) {
+    logout << "queryaction::do_action called with NULL collectproto\n";
+    textout << outconvert << disp << "_query:header_\n" 
+	    << "Error: Attempt to do query without setting collection\n"
+	    << "_query:footer_\n";
+  } else {  
+
+    FilterRequest_t request;
+    FilterResponse_t response;
+    format_t *formatlistptr = new format_t();
+
+    parse_formatstring (formatstring, formatlistptr, request.fields, request.getParents);
+
+    if (havequerylink)
+      request.fields.insert (querylinkmeta);
+
+    // do the query
+    request.filterResultOptions = FROID | FRmetadata | FRtermFreq;
+    if (!do_query (request, args, collectproto, response, logout))
+      return false;
+    
+    // set macros 
+    define_query_macros (args, disp, response);
+    
+    // output the header
+    textout << outconvert << disp << "_query:header_\n"
+	    << "_query:content_";
+
+    // output the results
+    textout << "<table cellspacing=4>\n";
+    ResultDocInfo_tarray::iterator this_doc = response.docInfo.begin();
+    ResultDocInfo_tarray::iterator end_doc = response.docInfo.end();
+
+    while (this_doc != end_doc) {
+      // don't include docs that didn't match phrases (if there were any)
+      // those that did match will have been sorted to the top
+      if ((*this_doc).num_phrase_match < num_phrases) break;
+      textout << "<tr>\n";
+      if (havequerylink) {
+	const text_t &qlmeta = (*this_doc).metadata[querylinkmeta].values[0];
+	if (qlmeta.empty())
+	  textout << outconvert << disp 
+		  << get_formatted_string (*this_doc, formatlistptr, "", "_iconblanktext_") << "\n";
+	else
+	  textout << outconvert << disp 
+		  << get_formatted_string (*this_doc, formatlistptr) << "\n";
+      } else {
+	textout << outconvert << disp 
+		<< get_formatted_string (*this_doc, formatlistptr) << "\n";
+      }
+      textout << "</tr>\n";
+
+      this_doc ++;
+    }
+    textout << "</table>\n";
+
+    delete (formatlistptr);
+
+    // output the footer
+    textout << outconvert << disp << "_query:footer_";
+  }
+  
+  return true;
+}
+  
+// define_query_macros sets the macros that couldn't be set until the
+// query had been done. Those macros are _freqmsg_, _quotedquery_,
+// _resultline_, _nextfirst_, _nextlast_, _prevfirst_, _prevlast_,
+// _thisfirst_, and _thislast_
+void queryaction::define_query_macros (cgiargsclass &args, displayclass &disp, 
+				       const FilterResponse_t &response) { 
+  // set up _freqmsg_ and _quotedquery_ macros
+  text_t freqmsg = "_textfreqmsg1_";
+  TermInfo_tarray::const_iterator this_term = response.termInfo.begin();
+  TermInfo_tarray::const_iterator end_term = response.termInfo.end();
+  while (this_term != end_term) {
+    freqmsg += (*this_term).term + ": " + (*this_term).freq;
+    if ((this_term + 1) != end_term)
+      freqmsg += ", ";
+    this_term ++;
+  }
+  
+  disp.setmacro ("freqmsg", "query", freqmsg);
+
+
+  // set up _resultline_ macro
+  text_t resline;
+  int maxdocs = args.getintarg("m");
+  int numdocs = response.numDocs;
+  if (maxdocs == -1) maxdocs = response.numDocs;
+  isapprox isApprox = response.isApprox;
+
+  // if there were phrases (post-processing) we're not going to include 
+  // those documents that didn't match 
+  if (num_phrases > 0) {
+    numdocs = 0;
+    isApprox = Exact;
+    ResultDocInfo_tarray::const_iterator this_doc = response.docInfo.begin();
+    ResultDocInfo_tarray::const_iterator end_doc = response.docInfo.end();
+    while (this_doc != end_doc) {
+      if ((*this_doc).num_phrase_match == num_phrases) numdocs ++;
+      else break; // we can bail here as matching docs are sorted to top
+      this_doc++;
+    }
+  }
+
+  if (isApprox == Approximate) resline = "_textapprox_";
+  else if (isApprox == MoreThan) resline = "_textmorethan_";
+  
+  if (numdocs == 0) resline = "_textnodocs_";
+  else if (numdocs == 1) resline += "_text1doc_";
+  else resline += text_t(numdocs) + " _textlotsdocs_";
+  
+  disp.setmacro("resultline", "query", resline);
+
+  int firstdoc = args.getintarg("r");
+  int hitsperpage = args.getintarg("o");
+  if (hitsperpage == -1) hitsperpage = numdocs;
+
+  // set up _thisfirst_ and _thislast_ macros
+  disp.setmacro ("thisfirst", "query", firstdoc);
+  int thislast = firstdoc + (hitsperpage - 1);
+  if (thislast > numdocs) thislast = numdocs;
+  disp.setmacro ("thislast", "query", thislast);
+
+  // set up _prevfirst_ and _prevlast_ macros
+  if (firstdoc > 1) {
+    disp.setmacro ("prevlast", "query", firstdoc - 1);
+    int prevfirst = firstdoc - hitsperpage;
+    if (prevfirst < 1) prevfirst = 1;
+    disp.setmacro ("prevfirst", "query", prevfirst);
+  }
+
+  // set up _nextfirst_ and _nextlast_ macros
+  if (thislast < numdocs) {
+    disp.setmacro ("nextfirst", "query", thislast + 1);
+    int nextlast = thislast + hitsperpage;
+    if (nextlast > numdocs) nextlast = numdocs;
+    disp.setmacro ("nextlast", "query", nextlast);
+  }
+}
+
+
+
Index: /tags/gsdl-21/gsdl/src/recpt/queryaction.h
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/queryaction.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/queryaction.h	(revision 699)
@@ -0,0 +1,75 @@
+/**********************************************************************
+ *
+ * queryaction.h -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef QUERYACTION_H
+#define QUERYACTION_H
+
+#include "gsdlconf.h"
+#include "action.h"
+
+
+class queryaction : public action {
+
+protected:
+  text_t formatstring;
+  int num_phrases;
+
+  void define_query_macros (cgiargsclass &args, displayclass &disp, 
+			    const FilterResponse_t &response); 
+
+  void set_option_macro (const text_t &macroname, text_t current_value,
+			 const FilterOption_t &option, displayclass &disp);
+
+
+public:
+  queryaction ();
+  virtual ~queryaction () {}
+  
+  void configure (const text_t &key, const text_tarray &cfgline);
+  bool init (ostream &logout);
+  
+  text_t get_action_name () {return "q";}
+  
+  bool check_cgiargs (cgiargsinfoclass &argsinfo, cgiargsclass &args, 
+		      ostream &logout);
+  void get_cgihead_info (cgiargsclass &args, response_t &response,
+			 text_t &response_data, ostream &logout);
+  
+  void define_internal_macros (const ColInfoResponse_t &collectinfo, displayclass &disp, 
+			       cgiargsclass &args, recptproto *collectproto, ostream &logout);
+  void define_external_macros (const ColInfoResponse_t &collectinfo, displayclass &disp, 
+			       cgiargsclass &args, recptproto *collectproto, ostream &logout);
+  
+  bool do_action (cgiargsclass &args, const ColInfoResponse_t &collectinfo, 
+		  recptproto *collectproto, displayclass &disp, 
+		  outconvertclass &outconvert, ostream &textout, 
+		  ostream &logout);
+};
+
+
+#endif
Index: /tags/gsdl-21/gsdl/src/recpt/querytools.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/querytools.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/querytools.cpp	(revision 699)
@@ -0,0 +1,243 @@
+/**********************************************************************
+ *
+ * querytools.cpp -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.7  1999/09/21 11:32:42  sjboddie
+   added Maxdocs queryfilter option which may be -1 for 'all'
+
+   Revision 1.6  1999/09/07 04:56:58  sjboddie
+   added GPL notice
+
+   Revision 1.5  1999/08/25 04:47:55  sjboddie
+   added advanced search option - other minor changes
+
+   Revision 1.4  1999/07/19 00:16:58  sjboddie
+   no longer display documents that don't match all phrases in query string
+
+   Revision 1.3  1999/07/16 00:19:02  sjboddie
+   some changes to the way quoted queries are handled
+
+   Revision 1.2  1999/07/07 06:12:21  rjmcnab
+   Added ability to combine two independant queries.
+
+   Revision 1.1  1999/06/15 02:24:06  sjboddie
+   moved do_query from queryaction to new querytools module (as do_action
+   is also called from documentaction when highlighting text)
+
+ */
+
+
+#include "querytools.h"
+
+
+// do_query sets the filter options and makes the protocol call to
+// do a query. The results are returned in response. 
+// request.filterResultOptions and request.fields (if required)
+// should be set by the calling function.
+
+bool do_query (FilterRequest_t &request, cgiargsclass &args, 
+	       recptproto *collectproto, FilterResponse_t &response, 
+	       ostream &logout) {
+
+  request.filterName = "QueryFilter";
+
+  comerror_t err;
+  OptionValue_t option;
+  text_t formattedstring = args["q"];
+  format_querystring (formattedstring, args.getintarg("b"));  
+
+  option.name = "Term";
+  option.value = formattedstring;
+  request.filterOptions.push_back (option);
+
+  option.name = "QueryType";
+  option.value = (args.getintarg("t")) ? "ranked" : "boolean";
+  request.filterOptions.push_back (option);
+
+  option.name = "Casefold";
+  option.value = (args.getintarg("k")) ? "true" : "false";
+  request.filterOptions.push_back (option);
+
+  option.name = "Stem";
+  option.value = (args.getintarg("s")) ? "true" : "false";
+  request.filterOptions.push_back (option);
+
+  if (!args["h"].empty()) {
+    option.name = "Index";
+    option.value = args["h"];
+    request.filterOptions.push_back (option);
+  }
+
+  if (!args["j"].empty()) {
+    option.name = "Subcollection";
+    option.value = args["j"];
+    request.filterOptions.push_back (option);
+  }
+
+  if (!args["n"].empty()) {
+    option.name = "Language";
+    option.value = args["n"];
+    request.filterOptions.push_back (option);
+  }
+
+  // fill in the second query if needed
+  if (!args["cq2"].empty()) {
+    option.name = "CombineQuery";
+    option.value = args["cq2"];
+    request.filterOptions.push_back (option);
+    
+    text_t formattedstring2 = args["q2"];
+    format_querystring (formattedstring2, args.getintarg("b"));
+
+    option.name = "Term";
+    option.value = formattedstring2;
+    request.filterOptions.push_back (option);
+
+    option.name = "QueryType";
+    option.value = (args.getintarg("t")) ? "ranked" : "boolean";
+    request.filterOptions.push_back (option);
+
+    option.name = "Casefold";
+    option.value = (args.getintarg("k")) ? "true" : "false";
+    request.filterOptions.push_back (option);
+
+    option.name = "Stem";
+    option.value = (args.getintarg("s")) ? "true" : "false";
+    request.filterOptions.push_back (option);
+
+    if (!args["h2"].empty()) {
+      option.name = "Index";
+      option.value = args["h2"];
+      request.filterOptions.push_back (option);
+    }
+
+    if (!args["j2"].empty()) {
+      option.name = "Subcollection";
+      option.value = args["j2"];
+      request.filterOptions.push_back (option);
+    }
+
+    if (!args["n2"].empty()) {
+      option.name = "Language";
+      option.value = args["n2"];
+      request.filterOptions.push_back (option);
+    }
+  }
+
+  int arg_m = args.getintarg("m");
+
+  option.name = "Maxdocs";
+  option.value = arg_m;
+  request.filterOptions.push_back (option);
+  
+  option.name = "StartResults";
+  option.value = args["r"];
+  request.filterOptions.push_back (option);
+
+  option.name = "EndResults";
+  int endresults = args.getintarg("o") + (args.getintarg("r") - 1);
+  if ((endresults > arg_m) && (arg_m != -1)) endresults = arg_m;
+  option.value = endresults;
+  request.filterOptions.push_back (option);
+
+  collectproto->filter (args["c"], request, response, err, logout);
+
+  if (err != noError) {
+    outconvertclass text_t2ascii;
+    logout << text_t2ascii
+	   << "Error: call to QueryFilter failed in queryaction ("
+	   << get_comerror_string (err) << ")\n";
+    return false;
+  }
+  return true;
+}
+
+void format_querystring (text_t &querystring, int querymode) {
+  text_t formattedstring;
+
+  text_t::const_iterator here = querystring.begin();
+  text_t::const_iterator end = querystring.end();
+
+  // space is used to insert spaces between Chinese
+  // characters. No space is needed before the first
+  // Chinese character.
+  bool space = false;
+
+  // want to remove ()|!& from querystring so boolean queries are just
+  // "all the words" queries (unless querymode is advanced)
+  while (here != end) {
+    if ((querymode == 0) && (*here == '(' || *here == ')' || *here == '|' ||
+			     *here == '!' || *here == '&')) {
+      formattedstring.push_back(' ');
+    } else {
+      if ((*here >= 0x4e00 && *here <= 0x9fa5) ||
+	  (*here >= 0xf900 && *here <= 0xfa2d)) {
+	// Chinese character
+	if (space) formattedstring.push_back (0x200b);
+	formattedstring.push_back (*here);
+	formattedstring.push_back (0x200b);
+	space = true;
+      } else {
+	// non-Chinese character
+	formattedstring.push_back (*here);
+	space = false;
+      }
+    }
+    here ++;
+  }
+  querystring = formattedstring;
+}
+
+void get_phrases (const text_t &querystring, text_tarray &phrases) {
+
+  phrases.erase (phrases.begin(), phrases.end());
+  if (!querystring.empty()) {
+    
+    text_t::const_iterator end = querystring.end();
+    text_t::const_iterator here = findchar (querystring.begin(), end, '"');
+    if (here != end) {
+      text_t tmptext;
+      bool foundquote = false;
+      while (here != end) {
+	if (*here == '"') {
+	  if (foundquote) {
+	    if (!tmptext.empty()) {
+	      phrases.push_back(tmptext);
+	      tmptext.clear();
+	    }
+	    foundquote = false;
+	  } else foundquote = true;
+	} else {
+	  if (foundquote) tmptext.push_back (*here);
+	}
+	here ++;
+      }
+    }
+  }
+}
+
Index: /tags/gsdl-21/gsdl/src/recpt/querytools.h
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/querytools.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/querytools.h	(revision 699)
@@ -0,0 +1,41 @@
+/**********************************************************************
+ *
+ * querytools.h -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef QUERYTOOLS_H
+#define QUERYTOOLS_H
+
+#include "cgiargs.h"
+#include "recptproto.h"
+
+bool do_query (FilterRequest_t &request, cgiargsclass &args, 
+	       recptproto *collectproto, FilterResponse_t &response, 
+	       ostream &logout);
+void format_querystring (text_t &querystring, int querymode);
+void get_phrases (const text_t &querystring, text_tarray &phrases);
+
+#endif
Index: /tags/gsdl-21/gsdl/src/recpt/receptionist.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/receptionist.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/receptionist.cpp	(revision 699)
@@ -0,0 +1,1152 @@
+/**********************************************************************
+ *
+ * receptionist.cpp -- a web interface for the gsdl
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.34  1999/10/14 23:00:52  sjboddie
+   finished changes to browsing support
+
+   Revision 1.33  1999/10/10 08:14:10  sjboddie
+   - metadata now returns mp rather than array
+   - redesigned browsing support (although it's not finished so
+   won't currently work ;-)
+
+   Revision 1.32  1999/09/21 11:28:45  sjboddie
+   tidied up file locking
+
+   Revision 1.31  1999/09/16 21:38:17  sjboddie
+   added some file locking stuff for logging. Windows still needs to
+   be done.
+
+   Revision 1.30  1999/09/07 04:56:58  sjboddie
+   added GPL notice
+
+   Revision 1.29  1999/09/03 10:02:30  rjmcnab
+   Made the page parameters configurable. Now the page parameters must
+   correspond to cgi arguments in name and value (ie language=zh should now
+   be l=zh) which makes things more consistent anyway. Removed a couple of
+   specialised NZDL page parameters.
+
+   Moved the combining of the cgi arguments so that the receptionist does
+   all the configuration now.
+
+   Made the macro precedence configurable.
+
+   Made cgi arguments totally configurable. Now any piece of information about
+   a cgi argument can be configured meaning that cgi arguments can be declared
+   from the configuration file.
+
+   Removed the argdefault configuration argument. This should now be done
+   using cgiarg.
+
+   Revision 1.28  1999/09/03 04:39:46  rjmcnab
+   Made cookies and logs optional (they are turned off by default). To
+   turn them on put
+
+   usecookies true
+   logcgiargs true
+
+   in your configuration file.
+
+   Revision 1.27  1999/09/02 00:27:21  rjmcnab
+   A few small things.
+
+   Revision 1.26  1999/08/25 04:43:06  sjboddie
+   made FilterRequest_t::docSet an array rather than a set
+
+   Revision 1.25  1999/08/20 00:59:01  sjboddie
+   -fixed up location redirection
+   -added some usage logging, also now set a GSDL_UID cookie. Logging
+   does NOT presently lock the log file while it's in use. That has yet
+   to be done.
+
+   Revision 1.24  1999/08/13 04:16:42  sjboddie
+   added some collection-level metadata stuff
+
+   Revision 1.23  1999/08/11 23:28:59  sjboddie
+   added support for html classifier (i.e. the hp argumant now must be
+   translated too).
+
+   Revision 1.22  1999/08/10 22:45:21  sjboddie
+   format option ShowTopPages is now called DocumentTopPages
+
+   Revision 1.21  1999/08/09 04:25:17  sjboddie
+   moved OID translation stuff from documentaction::define_external_macros
+   to receptionist
+
+   Revision 1.20  1999/07/30 02:13:09  sjboddie
+   -added collectinfo argument to some functions
+   -made some function prototypes virtual
+
+   Revision 1.19  1999/07/15 06:02:05  rjmcnab
+   Moved the setting of argsinfo into the constructor. Added the configuration
+   command argdefault (as used by the actions). Added code to output the
+   correct charset based on the page encoding so that the user does not need
+   to specify the encoding used for a particular page.
+
+   Revision 1.18  1999/07/11 01:05:20  rjmcnab
+   Stored origin of cgiarg with argument.
+
+   Revision 1.17  1999/07/10 22:18:26  rjmcnab
+   Added calls to define_external_cgiargs.
+
+   Revision 1.16  1999/06/27 21:49:03  sjboddie
+   fixed a couple of version conflicts - tidied up some small things
+
+   Revision 1.15  1999/06/26 01:14:32  rjmcnab
+   Made a couple of changes to handle different encodings.
+
+   Revision 1.14  1999/06/09 00:08:36  sjboddie
+   query string macro (_cgiargq_) is now made html safe before being set
+
+   Revision 1.13  1999/06/08 04:29:31  sjboddie
+   added argsinfo to the call to check_cgiargs to make it easy to set
+   args to their default if they're found to be screwed up
+
+   Revision 1.12  1999/04/30 01:59:42  sjboddie
+   lots of stuff - getting documentaction working (documentaction replaces
+   old browseaction)
+
+   Revision 1.11  1999/03/25 03:06:43  sjboddie
+
+   altered receptionist slightly so it now passes *collectproto to
+   define_internal_macros and define_external_macros - need it
+   for browseaction
+
+   Revision 1.10  1999/03/05 03:53:54  sjboddie
+
+   fixed some bugs
+
+   Revision 1.9  1999/02/28 20:00:16  rjmcnab
+
+
+   Fixed a few things.
+
+   Revision 1.8  1999/02/25 21:58:59  rjmcnab
+
+   Merged sources.
+
+   Revision 1.7  1999/02/21 22:33:55  rjmcnab
+
+   Lots of stuff :-)
+
+   Revision 1.6  1999/02/11 01:24:05  rjmcnab
+
+   Fixed a few compiler warnings.
+
+   Revision 1.5  1999/02/08 01:28:02  rjmcnab
+
+   Got the receptionist producing something using the statusaction.
+
+   Revision 1.4  1999/02/05 10:42:46  rjmcnab
+
+   Continued working on receptionist
+
+   Revision 1.3  1999/02/04 10:00:56  rjmcnab
+
+   Developed the idea of an "action" and having them define the cgi arguments
+   which they need and how those cgi arguments function.
+
+   Revision 1.2  1999/02/04 01:17:27  rjmcnab
+
+   Got it outputing something.
+
+
+ */
+
+
+#include "receptionist.h"
+#include "fileutil.h"
+#include "cgiutils.h"
+#include "htmlutils.h"
+#include "OIDtools.h"
+#include <assert.h>
+#include <time.h>
+#include <stdio.h>
+#include <fstream.h>
+
+#if defined (__WIN32_)
+#include "wincgiutils.h"
+#endif
+
+void recptconf::clear () {
+  gsdlhome.clear();
+  collection.clear();
+  collectdir.clear();
+  httpprefix.clear();
+  httpimg.clear();
+  gwcgi.clear();
+  macrofiles.erase(macrofiles.begin(), macrofiles.end());
+  saveconf.clear();
+  usecookies = false;
+  logcgiargs = false;
+
+  // these default page parameters can always be overriden
+  // in the configuration file
+  pageparams.erase(pageparams.begin(), pageparams.end());
+  pageparams["c"] = ""; 
+  pageparams["l"] = "en"; 
+
+#ifdef MACROPRECEDENCE
+  macroprecedence = MACROPRECEDENCE;
+#else
+  macroprecedence.clear();
+#endif
+}
+
+
+
+receptionist::receptionist () {
+  // create a list of cgi arguments
+  // this must be done before the configuration
+
+  cgiarginfo ainfo;
+
+  ainfo.shortname = "e";
+  ainfo.longname = "compressed arguments";
+  ainfo.multiplechar = true;
+  ainfo.defaultstatus = cgiarginfo::good;
+  ainfo.argdefault = "";
+  ainfo.savedarginfo = cgiarginfo::mustnot;
+  argsinfo.addarginfo (NULL, ainfo);
+
+  ainfo.shortname = "a";
+  ainfo.longname = "action";
+  ainfo.multiplechar = true;
+  ainfo.defaultstatus = cgiarginfo::none;
+  ainfo.argdefault = "";
+  ainfo.savedarginfo = cgiarginfo::must;
+  argsinfo.addarginfo (NULL, ainfo);
+
+  // w=western
+  ainfo.shortname = "w";
+  ainfo.longname = "encoding";
+  ainfo.multiplechar = true;
+  ainfo.defaultstatus = cgiarginfo::weak;
+  ainfo.argdefault = "w";
+  ainfo.savedarginfo = cgiarginfo::must;
+  argsinfo.addarginfo (NULL, ainfo);
+  
+  ainfo.shortname = "nw";
+  ainfo.longname = "new encoding";
+  ainfo.multiplechar = true;
+  ainfo.defaultstatus = cgiarginfo::none;
+  ainfo.argdefault = "";
+  ainfo.savedarginfo = cgiarginfo::mustnot;
+  argsinfo.addarginfo (NULL, ainfo);
+  
+  ainfo.shortname = "c";
+  ainfo.longname = "collection";
+  ainfo.multiplechar = true;
+  ainfo.defaultstatus = cgiarginfo::none;
+  ainfo.argdefault = "";
+  ainfo.savedarginfo = cgiarginfo::must;
+  argsinfo.addarginfo (NULL, ainfo);
+  
+  // the interface language name should use the ISO 639
+  // standard
+  ainfo.shortname = "l";
+  ainfo.longname = "interface language";
+  ainfo.multiplechar = true;
+  ainfo.defaultstatus = cgiarginfo::weak;
+  ainfo.argdefault = "en";
+  ainfo.savedarginfo = cgiarginfo::must;
+  argsinfo.addarginfo (NULL, ainfo);
+
+  // the GSDL_UID (cookie)
+  ainfo.shortname = "z";
+  ainfo.longname = "gsdl uid";
+  ainfo.multiplechar = true;
+  ainfo.defaultstatus = cgiarginfo::none;
+  ainfo.argdefault = "";
+  ainfo.savedarginfo = cgiarginfo::mustnot;
+  argsinfo.addarginfo (NULL, ainfo);
+}
+
+
+void receptionist::add_action (action *theaction) {
+  // make sure we have an action to add
+  if (theaction == NULL) return;
+
+  // add this action to the list of actions
+  actions.addaction(theaction);
+  
+  // add the cgi arguments from this action
+  argsinfo.addarginfo (NULL, theaction->getargsinfo());
+}
+
+
+void receptionist::add_browser (browserclass *thebrowser) {
+  // make sure we have a browser to add
+  if (thebrowser == NULL) return;
+
+  // add this browser to the list of browsers
+  browsers.addbrowser(thebrowser);
+}
+
+
+void receptionist::setdefaultbrowser (const text_t &browsername) {
+  browsers.setdefaultbrowser (browsername);
+}
+
+
+// configure should be called for each line in the
+// configuration files to configure the receptionist and everything
+// it contains. The configuration should take place after everything
+// has been added but before the initialisation.
+void receptionist::configure (const text_t &key, const text_tarray &cfgline) {
+  // configure the receptionist
+  if (cfgline.size() >= 1) {
+    cgiarginfo *info = NULL;
+    if (key == "gsdlhome") configinfo.gsdlhome = cfgline[0];
+    else if (key == "collection") {
+      configinfo.collection = cfgline[0];
+      // also need to set the default arg to this collection
+      if ((info = argsinfo.getarginfo("c")) != NULL) {
+	info->defaultstatus = cgiarginfo::good;
+	info->argdefault = cfgline[0];
+      }
+      
+    } else if (key == "collectdir") configinfo.collectdir = cfgline[0];
+    else if (key == "httpprefix") configinfo.httpprefix = cfgline[0];
+    else if (key == "httpimg") configinfo.httpimg = cfgline[0];
+    else if (key == "gwcgi") configinfo.gwcgi = cfgline[0];
+    else if (key == "macrofiles") configinfo.macrofiles = cfgline;
+    else if (key == "saveconf") configinfo.saveconf = cfgline[0];
+    else if (key == "usecookies") configinfo.usecookies = (cfgline[0] == "true");
+    else if (key == "logcgiargs") configinfo.logcgiargs = (cfgline[0] == "true");
+    else if (key == "pageparam") {
+      if (cfgline.size() >= 2) configinfo.pageparams[cfgline[0]] = cfgline[1];
+      else configinfo.pageparams[cfgline[0]] = "";
+    }
+    else if (key == "macroprecedence") configinfo.macroprecedence = cfgline[0];
+
+    
+    else if (key == "cgiarg") {
+      // get shortname
+      bool seen_defaultstatus = false;
+      text_t subkey, subvalue;
+      text_t shortname;
+      text_t::const_iterator cfglinesub_here;
+      text_tarray::const_iterator cfgline_here = cfgline.begin();
+      text_tarray::const_iterator cfgline_end = cfgline.end();
+      while (cfgline_here != cfgline_end) {
+	cfglinesub_here = getdelimitstr((*cfgline_here).begin(),
+					(*cfgline_here).end(), '=', subkey);
+	if (subkey == "shortname") {
+	  shortname = substr (cfglinesub_here, (*cfgline_here).end());
+	}
+	cfgline_here++;
+      }
+
+      // if we found the shortname process the line again filling in values
+      if (!shortname.empty()) {
+	cgiarginfo &chinfo = argsinfo[shortname];
+	chinfo.shortname = shortname; // in case this is a new argument
+	
+	cfgline_here = cfgline.begin();
+	while (cfgline_here != cfgline_end) {
+	  cfglinesub_here = getdelimitstr((*cfgline_here).begin(),
+					  (*cfgline_here).end(), '=', subkey);
+	  subvalue = substr (cfglinesub_here, (*cfgline_here).end());
+
+	  if (subkey == "longname") chinfo.longname = subvalue;
+	  else if (subkey == "multiplechar") chinfo.multiplechar = (subvalue == "true");
+	  else if (subkey == "defaultstatus") {
+	    seen_defaultstatus = true;
+	    if (subvalue == "none") chinfo.defaultstatus = cgiarginfo::none;
+	    else if (subvalue == "weak") chinfo.defaultstatus = cgiarginfo::weak;
+	    else if (subvalue == "good") chinfo.defaultstatus = cgiarginfo::good;
+	    else if (subvalue == "config") chinfo.defaultstatus = cgiarginfo::config;
+	    else if (subvalue == "imperative") chinfo.defaultstatus = cgiarginfo::imperative;
+	  }
+	  else if (subkey == "argdefault") {
+	    chinfo.argdefault = subvalue;
+	    if (!seen_defaultstatus) chinfo.defaultstatus = cgiarginfo::config;
+	  }
+	  else if (subkey == "savedarginfo") {
+	    if (subvalue == "mustnot") chinfo.savedarginfo = cgiarginfo::mustnot;
+	    else if (subvalue == "can") chinfo.savedarginfo = cgiarginfo::can;
+	    else if (subvalue == "must") chinfo.savedarginfo = cgiarginfo::must;
+	  }
+	  
+	  cfgline_here++;
+	}
+      }
+    }
+  }
+  
+  // configure the actions
+  actionptrmap::iterator actionhere = actions.begin ();
+  actionptrmap::iterator actionend = actions.end ();
+
+  while (actionhere != actionend) {
+    assert ((*actionhere).second.a != NULL);
+    if ((*actionhere).second.a != NULL) 
+      (*actionhere).second.a->configure(key, cfgline);
+
+    actionhere++;
+  }
+
+  // configure the protocols
+  recptprotolistclass::iterator protohere = protocols.begin ();
+  recptprotolistclass::iterator protoend = protocols.end ();
+
+  while (protohere != protoend) {
+    assert ((*protohere).p != NULL);
+    if ((*protohere).p != NULL) 
+      (*protohere).p->configure(key, cfgline);
+    
+    protohere++;
+  }
+
+  // configure the browsers
+  browserptrmap::iterator browserhere = browsers.begin ();
+  browserptrmap::iterator browserend = browsers.end ();
+
+  while (browserhere != browserend) {
+    assert ((*browserhere).second.b != NULL);
+    if ((*browserhere).second.b != NULL) 
+      (*browserhere).second.b->configure(key, cfgline);
+    
+    browserhere++;
+  }
+}
+
+
+void receptionist::configure (const text_t &key, const text_t &value) {
+  text_tarray cfgline;
+  cfgline.push_back (value);
+  configure(key, cfgline);
+}
+
+
+// init should be called after all the actions, protocols, and
+// converters have been added to the receptionist and after everything
+// has been configured but before any pages are created.
+// It returns true on success and false on failure. If false is
+// returned getpage should not be called (without producing
+// meaningless output), instead an error page should be
+// produced by the calling code.
+bool receptionist::init (ostream &logout) {
+  // first configure collectdir
+  text_t thecollectdir = configinfo.gsdlhome;
+  if (!configinfo.collection.empty()) {
+    // collection specific mode
+    if (!configinfo.collectdir.empty()) {
+      // has already been configured
+      thecollectdir = configinfo.collectdir;
+    } else {
+      // decide where collectdir is by searching for collect.cfg
+      // look in $GSDLHOME/collect/collection-name/etc/collect.cfg and
+      // then $GSDLHOME/etc/collect.cfg
+      thecollectdir = filename_cat (configinfo.gsdlhome, "collect");
+      thecollectdir = filename_cat (thecollectdir, configinfo.collection);
+      text_t filename = filename_cat (thecollectdir, "etc");
+      filename = filename_cat (filename, "collect.cfg");
+
+      if (!file_exists(filename)) thecollectdir = configinfo.gsdlhome;
+    }
+  }
+  configure("collectdir", thecollectdir);
+
+  // read in the macro files
+  if (!read_macrofiles (logout)) return false;
+
+  // there must be at least one action defined
+  if (actions.empty()) {
+    logout << "Error: no actions have been added to the receptionist\n";
+    return false;
+  }
+
+  // there must be at least one browser defined
+  if (browsers.empty()) {
+    logout << "Error: no browsers have been added to the receptionist\n";
+    return false;
+  }
+
+  // create a saveconf string if there isn't one already
+  if (configinfo.saveconf.empty())
+    configinfo.saveconf = create_save_conf_str (argsinfo, logout);
+
+  // check the saveconf string
+  if (!check_save_conf_str (configinfo.saveconf, argsinfo, logout))
+    return false;
+
+  // set a random seed
+  srand (time(NULL));
+
+  // make the output converters remove all the zero-width spaces
+  convertinfoclass::iterator converthere = converters.begin ();
+  convertinfoclass::iterator convertend = converters.end ();
+  text_t defaultconvertname;
+  while (converthere != convertend) {
+    assert ((*converthere).second.outconverter != NULL);
+    if ((*converthere).second.outconverter != NULL) {
+      (*converthere).second.outconverter->set_rzws(1);
+      if (defaultconvertname.empty())
+	defaultconvertname = (*converthere).second.name;
+    }
+    converthere++;
+  }
+  
+  // set default converter if no good one has been defined
+  if (!defaultconvertname.empty()) {
+    cgiarginfo *ainfo = argsinfo.getarginfo ("w");
+    if (ainfo->argdefault != "w") {
+      if ((ainfo != NULL) && (converters.get_outconverter(ainfo->argdefault) == NULL)) {
+	ainfo->defaultstatus = cgiarginfo::good;
+	ainfo->argdefault = defaultconvertname;
+      }
+    }
+  }
+
+  // init the actions
+  actionptrmap::iterator actionhere = actions.begin ();
+  actionptrmap::iterator actionend = actions.end ();
+  while (actionhere != actionend) {
+    if (((*actionhere).second.a == NULL) || 
+	!(*actionhere).second.a->init(logout)) return false;
+    actionhere++;
+  }
+
+  // init the protocols
+  recptprotolistclass::iterator protohere = protocols.begin ();
+  recptprotolistclass::iterator protoend = protocols.end ();
+  while (protohere != protoend) {
+    if (((*protohere).p == NULL) ||
+	!(*protohere).p->init(logout)) return false;
+    protohere++;
+  }
+
+  // init the browsers
+  browserptrmap::iterator browserhere = browsers.begin ();
+  browserptrmap::iterator browserend = browsers.end ();
+  while (browserhere != browserend) {
+    if (((*browserhere).second.b == NULL) || 
+	!(*browserhere).second.b->init(logout)) return false;
+    browserhere++;
+  }
+
+  return true;
+}
+
+
+// parse_cgi_args parses cgi arguments into an argument class.
+// This function should be called for each page request. It returns false
+// if there was a major problem with the cgi arguments.
+bool receptionist::parse_cgi_args (const text_t &argstr, cgiargsclass &args, 
+				   ostream &logout) {
+  outconvertclass text_t2ascii;
+
+  // get an initial list of cgi arguments
+  args.clear();
+  split_cgi_args (argstr, args);
+
+  // expand the compressed argument (if there was one)
+  if (!expand_save_args (argsinfo, configinfo.saveconf, args, logout)) return false;
+
+  // add the defaults
+  add_default_args (argsinfo, args, logout);
+
+  // get the cookie
+  if (configinfo.usecookies) get_cookie(args["z"]);
+  
+  // get the input encoding
+  text_t &arg_w = args["w"];
+  inconvertclass defaultinconvert;
+  inconvertclass *inconvert = converters.get_inconverter (arg_w);
+  if (inconvert == NULL) inconvert = &defaultinconvert;
+
+  // see if the next page will have a different encoding
+  if (args.getarg("nw") != NULL) arg_w = args["nw"];
+
+  // convert arguments which aren't in unicode to unicode
+  args_tounicode (args, *inconvert);
+
+
+  // decide on the output conversion class (needed for checking the external
+  // cgi arguments)
+  rzwsoutconvertclass defaultoutconverter;
+  rzwsoutconvertclass *outconverter = converters.get_outconverter (arg_w);
+  if (outconverter == NULL) outconverter = &defaultoutconverter;
+  outconverter->reset();
+
+  // check the main cgi arguments
+  if (!check_mainargs (args, logout)) return false;
+
+  // check the arguments for the action
+  action *a = actions.getaction (args["a"]);
+  if (a != NULL) {
+    if (!a->check_cgiargs (argsinfo, args, logout)) return false;
+  } else {
+    // the action was not found!!
+    logout << text_t2ascii << "Error: the action \"" << args["a"] 
+	   << "\" could not be found.\n";
+    return false;
+  }
+
+  // check external cgi arguments for each action
+  actionptrmap::iterator actionhere = actions.begin ();
+  actionptrmap::iterator actionend = actions.end ();
+  while (actionhere != actionend) {
+    assert ((*actionhere).second.a != NULL);
+    if ((*actionhere).second.a != NULL) {
+      if (!(*actionhere).second.a->check_external_cgiargs (argsinfo, args, *outconverter,
+							   configinfo.saveconf, logout))
+	return false;
+    }
+    actionhere++;
+  }
+
+  // the action might have changed but we will assume that
+  // the cgiargs were checked properly when the change was made
+
+  return true;
+}
+
+// returns true if cookie already existed, false
+// if it was generated
+bool receptionist::get_cookie (text_t &cookie) {
+
+  char *c = getenv("HTTP_COOKIE");
+  if (c != NULL) {
+    text_t cookiestring = c;
+    
+    text_t::const_iterator end = cookiestring.end();  
+    text_t::const_iterator here = findchar (cookiestring.begin(), end, 'G');
+    
+    if (substr(here, here+8) == "GSDL_UID") {
+      cookie = substr (here+9, findchar (here+9, end, ';')); 
+      return true;
+    }
+  }
+
+  cookie.clear();
+  char *host = getenv("REMOTE_ADDR");
+  time_t ttime = time(NULL);
+  if (host != NULL) {
+    cookie += host;
+    cookie.push_back ('-');
+  }
+  cookie += text_t(ttime);
+  
+  return false;
+}
+
+// as above but just tests if cookie exists
+bool receptionist::get_cookie () {
+
+  char *c = getenv("HTTP_COOKIE");
+  if (c != NULL) {
+    text_t cookiestring = c;
+    
+    text_t::const_iterator end = cookiestring.end();  
+    text_t::const_iterator here = findchar (cookiestring.begin(), end, 'G');
+    
+    if (substr(here, here+8) == "GSDL_UID")
+      return true;
+  }
+  return false;
+}
+
+bool receptionist::log_cgi_args (cgiargsclass &args, ostream &logout) {
+
+  // see if we want to log the cgi arguments
+  if (!configinfo.logcgiargs) return true;
+  
+  char *host;
+  host = getenv("REMOTE_HOST");
+  if (host == NULL) host = getenv ("REMOTE_ADDR");
+  if (host == NULL) host = "";
+  char *browser = getenv("HTTP_USER_AGENT");
+  if (browser == NULL) browser = "";
+  time_t ttime = time(NULL);
+
+  cgiargsclass::const_iterator args_here = args.begin();
+  cgiargsclass::const_iterator args_end = args.end();
+
+  text_t argstr;
+  bool first = true;
+  while (args_here != args_end) {
+    if (!first) argstr += ", ";
+    argstr += (*args_here).first + "=" + (*args_here).second.value;
+    first = false;
+    args_here ++;
+  }
+
+  text_t logfile = filename_cat (configinfo.gsdlhome, "etc");
+  logfile = filename_cat (logfile, "usage.txt");
+
+  text_t logstr = host;
+  logstr += " [";
+  logstr += ttime;
+  logstr += "] (" + argstr + ") \"";
+  logstr += browser;
+  logstr += "\"\n";
+
+  return append_logstr (logfile, logstr, logout);
+}
+
+bool receptionist::append_logstr (const text_t &filename, const text_t &logstr,
+				  ostream &logout) {
+
+  utf8outconvertclass text_t2utf8;
+  char *lfile = filename.getcstr();
+  ofstream log (lfile, ios::app);
+  
+  if (!log) {
+    logout << "Error: Couldn't open file " << lfile << "\n";
+    delete lfile;
+    return false;
+  }
+
+  int fd = GSDL_GET_FILEDESC(log);
+
+  // lock_val is set to 0 if file is locked successfully
+  int lock_val = 1;
+  GSDL_LOCK_FILE (fd);
+  if (lock_val == 0) {
+    log << text_t2utf8 << logstr;
+    GSDL_UNLOCK_FILE (fd);
+  } else {
+    logout << "Error: Couldn't lock file " << lfile << "\n";
+    log.close();
+    delete lfile;
+    return false;
+  }
+
+  log.close();
+		
+  delete lfile;
+  return true;
+}
+
+// produce_cgi_page will call get_cgihead_info and 
+// produce_content in the appropriate way to output a cgi header and
+// the page content (if needed). If a page could not be created it
+// will return false
+bool receptionist::produce_cgi_page (cgiargsclass &args, ostream &contentout, 
+				     ostream &logout) {
+  outconvertclass text_t2ascii;
+
+  response_t response;
+  text_t response_data;
+
+  // produce cgi header
+  get_cgihead_info (args, response, response_data, logout);
+  if (response == location) {
+    // location response
+    contentout << text_t2ascii << "Location: " << response_data << "\n\n";
+    contentout << flush;
+    return true;
+  } else if (response == content) {
+    // content response
+    contentout << text_t2ascii << "Content-type: " << response_data << "\n\n";
+  } else {
+    // unknown response
+    logout << "Error: get_cgihead_info returned an unknown response type.\n";
+    return false;
+  }
+
+  // produce cgi page
+  if (!produce_content (args, contentout, logout)) return false;
+
+  // flush contentout
+  contentout << flush;
+  return true;
+}
+
+
+// get_cgihead_info determines the cgi header information for
+// a set of cgi arguments. If response contains location then
+// response_data contains the redirect address. If reponse
+// contains content then reponse_data contains the content-type.
+// Note that images can now be produced by the receptionist.
+void receptionist::get_cgihead_info (cgiargsclass &args, response_t &response,
+				     text_t &response_data, ostream &logout) {
+  outconvertclass text_t2ascii;
+
+  // get the action
+  action *a = actions.getaction (args["a"]);
+  if (a != NULL) {
+    a->get_cgihead_info (args, response, response_data, logout);
+
+  } else {
+    // the action was not found!!
+    logout << text_t2ascii << "Error receptionist::get_cgihead_info: the action \"" 
+	   << args["a"] << "\" could not be found.\n";
+    response = content;
+    response_data = "text/html";
+  }
+
+  // add the encoding information
+  if (response == content) {
+    if (args["w"] == "u") {
+      response_data += "; charset=UTF-8";
+    } else if (args["w"] == "g") {
+      response_data += "; charset=GBK";
+    } else {
+      response_data += "; charset=ISO-8859-1";      
+    }
+
+    // add cookie if required
+    if (configinfo.usecookies && !get_cookie())
+      response_data += "\nSet-Cookie: GSDL_UID=" + args["z"]
+	+ "; expires=25-Dec-37 00:00:00 GMT";
+  }
+}
+
+
+// produce the page content
+bool receptionist::produce_content (cgiargsclass &args, ostream &contentout,
+				    ostream &logout) {
+  // decide on the output conversion class
+  text_t &arg_w = args["w"];
+  rzwsoutconvertclass defaultoutconverter;
+  rzwsoutconvertclass *outconverter = converters.get_outconverter (arg_w);
+  if (outconverter == NULL) outconverter = &defaultoutconverter;
+  outconverter->reset();
+
+  text_t &collection = args["c"];
+  recptproto *collectproto = NULL;
+  if (!collection.empty()) {
+    // decide on the protocol used for communicating with
+    // the collection server
+    collectproto = protocols.getrecptproto (collection, logout);
+
+    // get collection information 
+    collectinfo.clear();
+    if (!collection.empty()) {
+      comerror_t err;
+      collectproto->get_collectinfo (collection, collectinfo, err, logout);
+      if (err != noError) collectinfo.clear();
+    }
+
+    // get browsers to process OID
+    text_t OID = args["d"];
+    if (OID.empty()) OID = args["cl"];
+    if (!OID.empty()) {
+      text_tset metadata;
+      text_tarray OIDs;
+      OIDs.push_back (OID);
+      if (!is_top(OID)) OIDs.push_back (OID + ".pr");
+      FilterResponse_t response;
+      metadata.insert ("childtype");
+      if (get_info (OIDs, args["c"], metadata, false, collectproto, response, logout)) {
+	text_t classifytype = response.docInfo[0].metadata["childtype"].values[0];
+	if (classifytype.empty()) 
+	  classifytype = response.docInfo[1].metadata["childtype"].values[0];
+	browserclass *b = browsers.getbrowser (classifytype);
+	b->processOID (args, collectproto, logout);
+      }
+    }
+
+    // translate "d", "cl", and "hp" arguments if required
+    translate_OIDs (args, collectproto, logout);
+  }
+
+  // produce the page using the desired action
+  action *a = actions.getaction (args["a"]);
+  if (a != NULL) {
+    if (a->uses_display(args)) prepare_page (a, args, collectproto, (*outconverter), logout);
+    if (!a->do_action (args, collectinfo, collectproto, disp, 
+		       (*outconverter), contentout, logout))
+      return false;
+
+  } else {
+    // the action was not found!!
+    outconvertclass text_t2ascii;
+
+    logout << text_t2ascii << "Error receptionist::produce_content: the action \"" 
+	   << args["a"] << "\" could not be found.\n";
+    
+    contentout << (*outconverter) 
+	       << "<html>\n"
+	       << "<head>\n"
+	       << "<title>Error</title>\n"
+	       << "</head>\n"
+	       << "<body>\n"
+	       << "<h2>Oops!</h2>\n"
+	       << "Undefined Page. The action \"" 
+	       << args["a"] << "\" could not be found.\n"
+	       << "</body>\n"
+	       << "</html>\n";
+  }
+
+  return true;
+}
+
+
+// returns the compressed argument ("e") corresponding to the argument
+// list. This can be used to save preferences between sessions.
+text_t receptionist::get_compressed_arg (cgiargsclass &args, ostream &logout) {
+  // decide on the output conversion class
+  text_t &arg_w = args["w"];
+  rzwsoutconvertclass defaultoutconverter;
+  rzwsoutconvertclass *outconverter = converters.get_outconverter (arg_w);
+  if (outconverter == NULL) outconverter = &defaultoutconverter;
+  outconverter->reset();
+
+  text_t compressed_args;
+  if (compress_save_args (argsinfo, configinfo.saveconf, args, 
+			  compressed_args, *outconverter, logout))
+    return compressed_args;
+
+  return "";
+}
+
+
+// will read in all the macro files. If one is not found an
+// error message will be written to logout and the method will
+// return false.
+bool receptionist::read_macrofiles (ostream &logout) {
+  outconvertclass text_t2ascii;
+
+  // redirect the error output to logout
+  disp.setlogout (&logout);
+
+  // load up the default macro files, the collection directory
+  // is searched first for the file (if this is being used in
+  // collection specific mode) and then the main directory
+  text_t colmacrodir = filename_cat (configinfo.collectdir, "macros");
+  text_t gsdlmacrodir = filename_cat (configinfo.gsdlhome, "macros");
+  text_tarray::iterator arrhere = configinfo.macrofiles.begin();
+  text_tarray::iterator arrend = configinfo.macrofiles.end();
+  text_t filename;
+  while (arrhere != arrend) {
+    // filename is used as a flag to indicate whether
+    // the macro file has been found
+    filename.clear();
+
+    // try in the collection directory if this is being
+    // run in collection specific mode
+    if (!configinfo.collection.empty()) {
+      filename = filename_cat (colmacrodir, *arrhere);
+      if (!file_exists (filename)) filename.clear ();
+    }
+
+    // if we haven't found the macro file yet try in
+    // the main macro directory
+    if (filename.empty()) {
+      filename = filename_cat (gsdlmacrodir, *arrhere);
+      if (!file_exists (filename)) filename.clear ();
+    }
+
+    // see if we found the file or not
+    if (filename.empty()) {
+      logout << text_t2ascii 
+	     << "Error: the macro file \"" << *arrhere << "\" could not be found.\n";
+      if (configinfo.collection.empty()) {
+	logout << text_t2ascii
+	       << "It should be in " << gsdlmacrodir << ".\n\n";
+      } else {
+	logout << text_t2ascii
+	       << "It should be in either " << colmacrodir << " or in " 
+	       << gsdlmacrodir << ".\n\n";
+      }
+      return false;
+
+    } else { // found the file
+      disp.loaddefaultmacros(filename);
+    }
+
+    arrhere++;
+  }
+
+  // success
+  return true;
+}
+
+
+// check_mainargs will check all the main arguments. If a major
+// error is found it will return false and no cgi page should
+// be created using the arguments.
+bool receptionist::check_mainargs (cgiargsclass &args, ostream &/*logout*/) {
+  // if this receptionist is running in collection dependant mode
+  // then it should always set the collection argument to the
+  // collection
+  if (!configinfo.collection.empty()) args["c"] = configinfo.collection;
+
+  // argument "v" can only be 0 or 1. Use the default value
+  // if it is out of range
+  int arg_v = args.getintarg ("v");
+  if (arg_v != 0 && arg_v != 1) {
+    cgiarginfo *vinfo = argsinfo.getarginfo ("v");
+    if (vinfo != NULL) args["v"] = vinfo->argdefault;
+  }
+
+  // argument "f" can only be 0 or 1. Use the default value
+  // if it is out of range
+  int arg_f = args.getintarg ("f");
+  if (arg_f != 0 && arg_f != 1) {
+    cgiarginfo *finfo = argsinfo.getarginfo ("f");
+    if (finfo != NULL) args["f"] = finfo->argdefault;
+  }
+
+  return true;
+}
+
+// translate_OIDs translates the "d", "cl", and "hp" arguments to their correct values
+// if they use the tricky ".fc", ".lc" type syntax.
+void receptionist::translate_OIDs (cgiargsclass &args, recptproto *collectproto, 
+				   ostream &logout) {
+
+  FilterResponse_t response;
+  FilterRequest_t request;
+  comerror_t err;
+  text_t &arg_d = args["d"];
+  text_t &arg_cl = args["cl"];
+  text_t &arg_hp = args["hp"];
+  text_t &collection = args["c"];
+  
+  // do a call to translate OIDs if required
+  request.filterName = "NullFilter";
+  request.filterResultOptions = FROID;
+  if (!arg_d.empty() && needs_translating (arg_d)) {
+    request.docSet.push_back (arg_d);
+    collectproto->filter (collection, request, response, err, logout);
+    arg_d = response.docInfo[0].OID;
+    request.clear();
+  }
+  // we'll also check here that the "cl" argument has a "classify" doctype 
+  // (in case ".fc" or ".lc" have screwed up)
+  if (needs_translating (arg_cl)) {
+    request.fields.insert ("doctype");
+    request.docSet.push_back (arg_cl);
+    request.filterResultOptions = FRmetadata;
+    collectproto->filter (collection, request, response, err, logout);
+    // set to original value (without .xx stuff) if doctype isn't "classify"
+    if (response.docInfo[0].metadata["doctype"].values[0] != "classify")
+      strip_suffix (arg_cl);
+    else
+      arg_cl = response.docInfo[0].OID;
+  }
+
+  if (needs_translating (arg_hp)) {
+    request.docSet.push_back (arg_hp);
+    collectproto->filter (collection, request, response, err, logout);
+    arg_hp = response.docInfo[0].OID;
+  }
+}
+
+// prepare_page sets up page parameters, sets display macros
+// and opens the page ready for output 
+void receptionist::prepare_page (action *a, cgiargsclass &args, recptproto *collectproto,
+				 outconvertclass &outconvert, ostream &logout) {
+  // set up page parameters
+  text_t pageparams;
+  bool first = true;
+
+  text_tmap::iterator params_here = configinfo.pageparams.begin();
+  text_tmap::iterator params_end = configinfo.pageparams.end();
+  while (params_here != params_end) {
+    if (args[(*params_here).first] != (*params_here).second) {
+      if (!first) pageparams += ",";
+      first = false;
+      pageparams += (*params_here).first;
+      pageparams += "=";
+      pageparams += args[(*params_here).first];
+    }
+    
+    params_here++;
+  }
+  
+
+  // open the page
+  disp.openpage(pageparams, configinfo.macroprecedence);
+
+
+  // define external macros for each action
+  actionptrmap::iterator actionhere = actions.begin ();
+  actionptrmap::iterator actionend = actions.end ();
+
+  while (actionhere != actionend) {
+    assert ((*actionhere).second.a != NULL);
+    if ((*actionhere).second.a != NULL) 
+      (*actionhere).second.a->define_external_macros (collectinfo, disp, args, collectproto, logout);
+    actionhere++;
+  }
+
+  // define internal macros for the current action
+  a->define_internal_macros (collectinfo, disp, args, collectproto, logout);
+  
+  // define general macros. the defining of general macros is done here so that
+  // the last possible version of the cgi arguments are used
+  define_general_macros (args, collectproto, outconvert, logout);
+}
+
+void receptionist::define_general_macros (cgiargsclass &args, recptproto *collectproto,
+					  outconvertclass &/*outconvert*/, ostream &logout) {
+
+  text_t &collection = args["c"];
+
+  disp.setmacro ("gwcgi", "Global", configinfo.gwcgi);
+  disp.setmacro ("httpimg", "Global", configinfo.httpimg);
+  disp.setmacro ("httpprefix", "Global", configinfo.httpprefix);
+  disp.setmacro ("compressedoptions", "Global", get_compressed_arg(args, logout));
+
+  // set _cgiargX_ macros for each cgi argument
+  cgiargsclass::const_iterator argshere = args.begin();
+  cgiargsclass::const_iterator argsend = args.end();
+  while (argshere != argsend) {
+    if (((*argshere).first == "q") ||
+	((*argshere).first == "qa") ||
+	((*argshere).first == "qtt") ||
+	((*argshere).first == "qty") ||
+	((*argshere).first == "qp") ||
+	((*argshere).first == "qpl") ||
+	((*argshere).first == "qr") ||
+	((*argshere).first == "q2"))
+      // need to escape special characters from query string
+      disp.setmacro ("cgiarg" + (*argshere).first, 
+		     "Global", html_safe((*argshere).second.value));
+    else 
+      disp.setmacro ("cgiarg" + (*argshere).first, "Global", (*argshere).second.value);
+    argshere ++;
+  }
+
+  // set collection specific macros
+  if ((!collection.empty()) && (collectproto != NULL)) {
+    FilterResponse_t response;
+    text_tset metadata;
+    get_info ("collection", collection, metadata, false, 
+	      collectproto, response, logout);
+
+    if (!response.docInfo[0].metadata.empty()) {
+      MetadataInfo_tmap::const_iterator here = response.docInfo[0].metadata.begin();
+      MetadataInfo_tmap::const_iterator end = response.docInfo[0].metadata.end();
+      while (here != end) {
+	if (((*here).first != "haschildren") && ((*here).first != "hasnext") &&
+	    ((*here).first != "hasprevious")) {
+	  disp.setmacro ((*here).first, "Global", (*here).second.values[0]);
+	}
+	here ++;
+      }
+    }
+  }
+}
Index: /tags/gsdl-21/gsdl/src/recpt/receptionist.h
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/receptionist.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/receptionist.h	(revision 699)
@@ -0,0 +1,203 @@
+/**********************************************************************
+ *
+ * receptionist.h -- a web interface for the gsdl
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef RECEPTIONIST_H
+#define RECEPTIONIST_H
+
+#include "gsdlconf.h"
+#include "text_t.h"
+#include "cgiargs.h"
+#include "display.h"
+#include "action.h"
+#include "browserclass.h"
+#include "recptproto.h"
+#include "converter.h"
+
+// the MACROPRECEDENCE macro is used as a default. override
+// it using something like:
+//
+// macroprecedence c,l
+//
+// in your configuration file
+#ifndef MACROPRECEDENCE
+#define MACROPRECEDENCE "c,l"
+#endif
+
+
+struct recptconf {
+  text_t gsdlhome;
+  text_t collection; // will equal "" in 'general' mode
+  text_t collectdir; // will equal gsdlhome in 'general' mode
+  text_t httpprefix;
+  text_t httpimg;
+  text_t gwcgi;
+  text_tarray macrofiles;
+  text_t saveconf;
+  bool usecookies;  // true if we want cookies set
+  bool logcgiargs;  // true if we want to log cgi arguments
+
+  text_tmap pageparams;
+  text_t macroprecedence;
+  
+  void clear ();
+  recptconf () {clear();}
+};
+
+
+class receptionist {
+protected:
+  recptconf configinfo;
+
+  cgiargsinfoclass argsinfo;
+  actionmapclass actions;
+  browsermapclass browsers;
+  recptprotolistclass protocols;
+  displayclass disp;
+  convertinfoclass converters;
+  ColInfoResponse_t collectinfo;
+  
+  // prepare_page sets up page parameters, sets display macros
+  // and opens the page ready for output 
+  virtual void prepare_page (action *a, cgiargsclass &args, recptproto *collectproto,
+			     outconvertclass &outconvert, ostream &logout);
+  
+  virtual void translate_OIDs (cgiargsclass &args, recptproto *collectproto,
+			       ostream &logout);
+
+  bool get_cookie (text_t &cookie);
+  bool get_cookie ();
+
+  virtual void define_general_macros (cgiargsclass &args, recptproto *collectproto,
+				      outconvertclass &outconvert, ostream &logout);
+
+  bool append_logstr (const text_t &filename, const text_t &logstr,
+		      ostream &logout);
+
+public:
+  receptionist ();
+  virtual ~receptionist() {}
+
+  // add_action makes another action available to the receptionist
+  // the action remains the property of the calling code and that
+  // code should destroy the action after the recptionist has been
+  // destroyed.
+  void add_action (action *theaction);
+  actionmapclass *get_actionmap_ptr () {return &actions;}
+
+  // add_browser makes another browser available to the receptionist
+  // the browser remains the property of the calling code and that
+  // code should destroy it after the recptionist has been
+  // destroyed.
+  void add_browser (browserclass *thebrowser);
+  browsermapclass *get_browsermap_ptr () {return &browsers;}
+  void setdefaultbrowser (const text_t &browsername);
+
+  // add_protocol makes another protocol available to the receptionist
+  // without any protocols, no collections will be available. The
+  // protocols remain the property of the calling code.
+  void add_protocol (recptproto *theprotocol) {protocols.addrecptproto(theprotocol);}
+  recptprotolistclass *get_recptprotolist_ptr () {return &protocols;}
+  
+  // add_converter makes another converter available to the receptionist.
+  // Converters are needed to display pages using different encodings.
+  // The converters remain the property of the calling code.
+  void add_converter (const text_t &name, inconvertclass *inconverter,
+		      rzwsoutconvertclass *outconverter) {
+    converters.add_converter(name, inconverter, outconverter);}
+  convertinfoclass *get_convertinfo_ptr () {return &converters;}
+  
+  
+  // configure should be called for each line in the
+  // configuration files to configure the receptionist and everything
+  // it contains. The configuration should take place after everything
+  // has been added but before the initialisation.
+  virtual void configure (const text_t &key, const text_tarray &cfgline);
+  virtual void configure (const text_t &key, const text_t &value);
+  const recptconf &get_configinfo () {return configinfo;}
+  cgiargsinfoclass *get_cgiargsinfo_ptr () {return &argsinfo;}
+  
+
+  // init should be called after all the actions, protocols, and
+  // converters have been added to the receptionist and after everything
+  // has been configured but before any pages are created.
+  // It returns true on success and false on failure. If false is
+  // returned getpage should not be called (without producing
+  // meaningless output), instead an error page should be
+  // produced by the calling code.
+  virtual bool init (ostream &logout);
+  
+  // There are two ways to produce a page. You can either call parse_cgi_args,
+  // get_cgihead_info, and produce_content or you can just call parse_cgi_args and
+  // produce_cgi_page (which will be satisfactory in most cases). You might want to call
+  // parse_cgi_args, get_cgihead_info, and produce_content when you want to
+  // interface directly with a web server for which the standard header is inappropriate.
+
+  // parse_cgi_args parses cgi arguments into an argument class.
+  // This function should be called for each page request. It returns false
+  // if there was a major problem with the cgi arguments.
+  virtual bool parse_cgi_args (const text_t &argstr, cgiargsclass &args, ostream &logout);
+
+  virtual bool log_cgi_args (cgiargsclass &args, ostream &logout);
+
+  // produce_cgi_page will call get_cgihead_info and 
+  // produce_content in the appropriate way to output a cgi header and
+  // the page content (if needed). If a page could not be created it
+  // will return false
+  virtual bool produce_cgi_page (cgiargsclass &args, ostream &contentout, 
+				 ostream &logout);
+  
+  // get_cgihead_info determines the cgi header information for
+  // a set of cgi arguments. If response contains location then
+  // response_data contains the redirect address. If reponse
+  // contains content then reponse_data contains the content-type.
+  // Note that images can now be produced by the receptionist.
+  virtual void get_cgihead_info (cgiargsclass &args, response_t &response,
+				 text_t &response_data, ostream &logout);
+  
+  // produce the page content
+  virtual bool produce_content (cgiargsclass &args, ostream &contentout,
+				ostream &logout);
+  
+  // returns the compressed argument ("e") corresponding to the argument
+  // list. This can be used to save preferences between sessions.
+  text_t get_compressed_arg (cgiargsclass &args, ostream &logout);
+  
+protected:
+  // will read in all the macro files. If one is not found an
+  // error message will be written to logout and the method will
+  // return false.
+  virtual bool read_macrofiles (ostream &logout);
+  
+  // check_mainargs will check all the main arguments. If a major
+  // error is found it will return false and no cgi page should
+  // be created using the arguments.
+  virtual bool check_mainargs (cgiargsclass &args, ostream &logout);
+};
+
+
+#endif
Index: /tags/gsdl-21/gsdl/src/recpt/recptconfig.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/recptconfig.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/recptconfig.cpp	(revision 699)
@@ -0,0 +1,185 @@
+/**********************************************************************
+ *
+ * recptconfig.cpp -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.5  1999/09/07 04:56:59  sjboddie
+   added GPL notice
+
+   Revision 1.4  1999/09/07 00:09:31  sjboddie
+   now reads in both main.cfg and collect.cfg always
+
+   Revision 1.3  1999/02/21 22:33:57  rjmcnab
+
+   Lots of stuff :-)
+
+   Revision 1.2  1999/02/08 01:28:04  rjmcnab
+
+   Got the receptionist producing something using the statusaction.
+
+   Revision 1.1  1999/02/05 06:50:32  rjmcnab
+
+   Initial revision.
+
+ */
+
+
+#include "recptconfig.h"
+#include "fileutil.h"
+#include "cfgread.h"
+
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace/std/iostream>
+#  include <ospace/std/fstream>
+#elif defined(GSDL_USE_IOS_H)
+#  include <iostream.h>
+#  include <fstream.h>
+#else
+#  include <iostream>
+#  include <fstream>
+#endif
+
+
+// reads site configuration file returning true on success
+bool site_cfg_read (receptionist &recpt, const text_t &gsdlhome,
+		     const text_t &collection, int &maxrequests) {
+
+  maxrequests = 10000; // a reasonable default
+
+  // Look for site.cfg in GSDLHOME/collect/collection-name/etc directory 
+  // (if this is for a particular collection), and then GSDLHOME/etc.
+  text_t filename;
+  if (!collection.empty()) {
+    filename = filename_cat (gsdlhome, "collect");
+    filename = filename_cat (filename, collection);
+    filename = filename_cat (filename, "etc");
+    filename = filename_cat (filename, "site.cfg");
+    if (!file_exists (filename)) filename.clear();
+  }
+  if (filename.empty()) {
+    filename = filename_cat (gsdlhome, "etc");
+    filename = filename_cat (filename, "site.cfg");
+    if (!file_exists (filename)) return false;
+  }
+
+  // read in the site configuration file
+  text_tarray cfgline;
+  text_t key;
+  char *cstr = filename.getcstr();
+  ifstream confin (cstr);
+  delete cstr;
+
+  if (confin) {
+    while (read_cfg_line(confin, cfgline) >= 0) {
+      if (cfgline.size () >= 2) {
+	key = cfgline[0];
+	cfgline.erase(cfgline.begin());
+
+	if (key == "maxrequests") {
+	  maxrequests = cfgline[0].getint();
+	  if (maxrequests < 1) maxrequests = 1;
+	}
+
+	// configure the receptionist
+	recpt.configure (key, cfgline);
+      }
+    }
+    confin.close ();
+    return true;
+  }
+  return false;
+}
+
+
+// main_cfg_read reads both main.cfg and collect.cfg. It attempts
+// to read main.cfg first so values in collect.cfg override those
+// set earlier by main.cfg
+bool main_cfg_read (receptionist &recpt, const text_t &gsdlhome,
+		     const text_t &collection) {
+
+  // read in the main configuration file
+  bool rv = false;
+  text_t key, filename;
+  text_tarray cfgline;
+  filename = filename_cat (gsdlhome, "etc");
+  filename = filename_cat (filename, "main.cfg");
+  if (file_exists (filename)) {
+    char *cstr = filename.getcstr();
+    ifstream confin (cstr);
+    delete cstr;
+  
+    if (confin) {
+      while (read_cfg_line(confin, cfgline) >= 0) {
+	if (cfgline.size () >= 2) {
+	  key = cfgline[0];
+	  cfgline.erase(cfgline.begin());
+	
+	  // configure the receptionist
+	  recpt.configure (key, cfgline);
+	}
+      }
+      confin.close ();
+      rv = true;
+    }
+  }
+  
+  // Look for collect.cfg in GSDLHOME/collect/collection-name/etc directory 
+  // (if this is for a particular collection), and then GSDLHOME/etc.
+  if (!collection.empty()) {
+    filename = filename_cat (gsdlhome, "collect");
+    filename = filename_cat (filename, collection);
+    filename = filename_cat (filename, "etc");
+    filename = filename_cat (filename, "collect.cfg");
+    if (!file_exists (filename)) {
+      filename = filename_cat (gsdlhome, "etc");
+      filename = filename_cat (filename, "collect.cfg");
+      if (!file_exists (filename)) filename.clear();
+    }
+
+    if (!filename.empty()) {
+      char *cstr = filename.getcstr();
+      ifstream confin (cstr);
+      delete cstr;
+      
+      if (confin) {
+	while (read_cfg_line(confin, cfgline) >= 0) {
+	  if (cfgline.size () >= 2) {
+	    key = cfgline[0];
+	    cfgline.erase(cfgline.begin());
+	    
+	    // configure the receptionist
+	    recpt.configure (key, cfgline);
+	  }
+	}
+	confin.close ();
+	rv = true;
+      }
+    }
+  }
+  return rv;
+}
Index: /tags/gsdl-21/gsdl/src/recpt/recptconfig.h
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/recptconfig.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/recptconfig.h	(revision 699)
@@ -0,0 +1,49 @@
+/**********************************************************************
+ *
+ * recptconfig.h -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef RECPTCONFIG_H
+#define RECPTCONFIG_H
+
+
+#include "gsdlconf.h"
+#include "text_t.h"
+#include "receptionist.h"
+
+
+// reads site configuration file returning true on success
+bool site_cfg_read (receptionist &recpt, const text_t &gsdlhome,
+		     const text_t &collection, int &maxrequests);
+
+// main_cfg_read reads either collect.cfg or main.cfg and returning 
+// true on success. It attempts to read collect.cfg first if a
+// collection is specified and then it tries to read in main.cfg
+bool main_cfg_read (receptionist &recpt, const text_t &gsdlhome,
+		     const text_t &collection);
+
+
+#endif
Index: /tags/gsdl-21/gsdl/src/recpt/recptmain.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/recptmain.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/recptmain.cpp	(revision 699)
@@ -0,0 +1,53 @@
+/**********************************************************************
+ *
+ * recptmain.cpp -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.3  1999/09/07 04:56:59  sjboddie
+   added GPL notice
+
+   Revision 1.2  1999/02/21 22:33:58  rjmcnab
+
+   Lots of stuff :-)
+
+   Revision 1.1  1999/02/04 01:16:18  rjmcnab
+
+   Initial revision.
+
+ */
+
+
+#include "receptionist.h"
+#include "cgiwrapper.h"
+#include <assert.h>
+
+int main () {
+  receptionist recpt;
+  cgiwrapper (recpt, "");
+  return 0;
+}
+
Index: /tags/gsdl-21/gsdl/src/recpt/recptproto.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/recptproto.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/recptproto.cpp	(revision 699)
@@ -0,0 +1,184 @@
+/**********************************************************************
+ *
+ * recptproto.cpp -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.7  1999/09/07 04:56:59  sjboddie
+   added GPL notice
+
+   Revision 1.6  1999/05/10 03:40:43  sjboddie
+   lots of changes - slowly getting document action sorted out
+
+   Revision 1.5  1999/04/30 01:59:43  sjboddie
+   lots of stuff - getting documentaction working (documentaction replaces
+   old browseaction)
+
+   Revision 1.4  1999/03/31 23:44:48  rjmcnab
+   Altered the protocol so that the metadata is part of the filter.
+
+   Revision 1.3  1999/03/03 23:26:35  sjboddie
+
+   Implemented more of the protocol
+
+   Revision 1.2  1999/02/25 21:59:01  rjmcnab
+
+   Merged sources.
+
+   Revision 1.1  1999/02/21 22:35:24  rjmcnab
+
+   Initial revision.
+
+ */
+
+
+#include "recptproto.h"
+#include <assert.h>
+
+
+// configure should be called for each line in the configuration file
+void recptproto::configure (const text_t &/*key*/, const text_tarray &/*cfgline*/) {
+}
+
+// init should be called after the configuration but before any other
+// functions are called. If init returns false a message will be written
+// out to the log file and no other output should be produced.
+bool recptproto::init (ostream &/*logout*/) {
+  return true;
+}
+
+// get_protocol_name should return the name of this protocol (e.g. recptproto)
+// that can be used to do run time type identification and display information
+// about the protocol.
+text_t recptproto::get_protocol_name () {
+  return "recptproto";
+}
+
+// get_collection_list returns the list of collections that
+// this protocol knows about
+void recptproto::get_collection_list (text_tarray &collist, comerror_t &err, 
+				      ostream &/*logout*/) {
+  collist.erase(collist.begin(),collist.end());
+  err = noError;
+}
+
+// has_collection sets 'hascollection' to be true if the protocol
+// can communicate with the collection (i.e. it is within its 
+// collection list). This function should be implemented in as
+// efficient time as possible as it will be called for each page
+// access and for each protocol.
+void recptproto::has_collection (const text_t &/*collection*/, bool &hascollection, 
+				 comerror_t &err, ostream &/*logout*/) {
+  hascollection = false;
+  err = noError;
+}
+
+// sets 'wassuccess' to be true if a successful ping was done to
+// the given collection.
+void recptproto::ping (const text_t &/*collection*/, bool &wassuccess, 
+		       comerror_t &err, ostream &/*logout*/) {
+  wassuccess = false; // no collections are supported by this class
+  err = noError;
+}
+
+// obtains general information about the collection
+void recptproto::get_collectinfo (const text_t &/*collection*/, 
+				  ColInfoResponse_t &/*collectinfo*/,
+				  comerror_t &err, ostream &/*logout*/) {
+  err = protocolError;
+}
+
+// gets a list of all the filters
+void recptproto::get_filterinfo (const text_t &/*collection*/,
+				 InfoFiltersResponse_t &/*reponse*/,
+				 comerror_t &err, ostream &/*logout*/) {
+  err = protocolError;
+}
+
+// gets all the filter options for a particular filter
+void recptproto::get_filteroptions (const text_t &/*collection*/,
+				    const InfoFilterOptionsRequest_t &/*request*/,
+				    InfoFilterOptionsResponse_t &/*response*/, 
+				    comerror_t &err, ostream &/*logout*/) {
+  err = protocolError;
+}
+
+// filters (search or browse) a result set
+void recptproto::filter (const text_t &/*collection*/,
+			 FilterRequest_t &/*request*/,
+			 FilterResponse_t &/*response*/,
+			 comerror_t &err, ostream &/*logout*/) {
+  err = protocolError;
+}
+
+void recptproto::get_document (const text_t &/*collection*/,
+			       const DocumentRequest_t &/*request*/,
+			       DocumentResponse_t &/*response*/,
+			       comerror_t &err, ostream &/*logout*/) {
+  err = protocolError;
+}
+
+
+
+
+
+
+
+// therecptproto remains the property of the calling code but
+// should not be deleted until it is removed from this list.
+void recptprotolistclass::addrecptproto (recptproto *therecptproto) {
+  // can't add a protocol that doesn't exist
+  assert (therecptproto != NULL);
+  if (therecptproto == NULL) return;
+
+  recptprotoptr rpp;
+  rpp.p = therecptproto;
+
+  recptprotoptrs.push_back(rpp);
+}
+
+// getrecptproto will return NULL if a recptproto for the given colelction
+// could not be found
+recptproto *recptprotolistclass::getrecptproto (const text_t &collection, 
+						ostream &logout) {
+  iterator here = recptprotoptrs.begin ();
+  iterator end = recptprotoptrs.end ();
+  bool hascollection;
+  comerror_t err;
+
+  while (here != end) {
+    assert ((*here).p != NULL);
+    if ((*here).p != NULL) {
+      (*here).p->has_collection (collection, hascollection, err, logout);
+      if ((err == noError) && (hascollection == true)) return (*here).p;
+    }
+    
+    here++;
+  }
+  
+  return NULL;
+}
+
Index: /tags/gsdl-21/gsdl/src/recpt/recptproto.h
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/recptproto.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/recptproto.h	(revision 699)
@@ -0,0 +1,178 @@
+/**********************************************************************
+ *
+ * recptproto.h -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef RECPTPROTO_H
+#define RECPTPROTO_H
+
+#include "gsdlconf.h"
+#include "text_t.h"
+#include "comtypes.h"
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\vector>
+#elif defined(GSDL_USE_STL_H)
+#  include <vector.h>
+#else
+#  include <vector>
+#endif
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\iostream>
+#elif defined(GSDL_USE_IOS_H)
+#  include <iostream.h>
+#else
+#  include <iostream>
+#endif
+
+
+// recptproto is a generalisation of a protocol for communicating
+// with a collection server.
+class recptproto {
+public:
+  // configure should be called for each line in the configuration file
+  virtual void configure (const text_t &key, const text_tarray &cfgline);
+
+  // init should be called after the configuration but before any other
+  // functions are called. If init returns false a message will be written
+  // out to the log file and no other output should be produced.
+  virtual bool init (ostream &logout);
+
+  // get_protocol_name should return the name of this protocol (e.g. recptproto)
+  // that can be used to do run time type identification and display information
+  // about the protocol.
+  virtual text_t get_protocol_name ();
+
+  // get_collection_list returns the list of collections that
+  // this protocol knows about
+  virtual void get_collection_list (text_tarray &collist, comerror_t &err, 
+				    ostream &logout);
+
+  // has_collection sets 'hascollection' to be true if the protocol
+  // can communicate with the collection (i.e. it is within its 
+  // collection list). This function should be implemented in as
+  // efficient time as possible as it will be called for each page
+  // access and for each protocol.
+  virtual void has_collection (const text_t &collection, bool &hascollection, 
+			       comerror_t &err, ostream &logout);
+
+  // sets 'wassuccess' to be true if a successful ping was done to
+  // the given collection.
+  virtual void ping (const text_t &collection, bool &wassuccess, 
+		     comerror_t &err, ostream &logout);
+
+  // obtains general information about the collection
+  virtual void get_collectinfo (const text_t &collection, 
+				ColInfoResponse_t &collectinfo,
+				comerror_t &err, ostream &logout);
+
+  // gets a list of all the filters
+  virtual void get_filterinfo (const text_t &collection,
+			       InfoFiltersResponse_t &response,
+			       comerror_t &err, ostream &logout);
+
+  // gets all the filter options for a particular filter
+  virtual void get_filteroptions (const text_t &collection,
+				  const InfoFilterOptionsRequest_t &request,
+				  InfoFilterOptionsResponse_t &response, 
+				  comerror_t &err, ostream &logout);
+
+  // filters (search or browse) a result set and returns information
+  // about the filtered set including term frequency information and 
+  // metadata
+  virtual void filter (const text_t &collection,
+		       FilterRequest_t &request,
+		       FilterResponse_t &response,
+		       comerror_t &err, ostream &logout);
+
+  // gets a document (duh!)
+  virtual void get_document (const text_t &collection,
+			     const DocumentRequest_t &request,
+			     DocumentResponse_t &response,
+			     comerror_t &err, ostream &logout);
+};
+
+
+// The recptprotoptr function does not 'own' the recptproto. The 
+// recptproto should be deleted by the code which created it.
+class recptprotoptr {
+public:
+  recptproto *p;
+
+  recptprotoptr () {p=NULL;}
+  ~recptprotoptr () {}
+};
+
+typedef vector<recptprotoptr> recptprotoptrlist;
+
+
+// contains a list of recptprotos
+class recptprotolistclass {
+protected:
+  recptprotoptrlist recptprotoptrs;
+
+public:
+  // type support for recptprotolistclass
+  typedef recptprotoptrlist::iterator iterator;
+  typedef recptprotoptrlist::const_iterator const_iterator;
+  typedef recptprotoptrlist::reference reference;
+  typedef recptprotoptrlist::const_reference const_reference;
+  typedef recptprotoptrlist::size_type size_type;
+
+  typedef recptprotoptrlist::difference_type difference_type;
+  typedef recptprotoptrlist::const_reverse_iterator const_reverse_iterator;
+  typedef recptprotoptrlist::reverse_iterator reverse_iterator;
+  
+  // basic container support
+  iterator begin () {return recptprotoptrs.begin();}
+  const_iterator begin () const {return recptprotoptrs.begin();}
+  iterator end () {return recptprotoptrs.end();}
+  const_iterator end () const {return recptprotoptrs.end();}
+
+  void erase(iterator pos) {recptprotoptrs.erase(pos);}
+  void erase(iterator first, iterator last) {recptprotoptrs.erase(first, last);}
+  recptprotolistclass &operator=(const recptprotolistclass &x) 
+    {recptprotoptrs=x.recptprotoptrs;return *this;}
+
+  bool empty () const {return recptprotoptrs.empty();}
+  size_type size() const {return recptprotoptrs.size();}
+
+
+  // added functionality
+  void clear () {recptprotoptrs.erase(recptprotoptrs.begin(),recptprotoptrs.end());}
+
+  // therecptproto remains the property of the calling code but
+  // should not be deleted until it is removed from this list.
+  void addrecptproto (recptproto *therecptproto);
+
+  // getrecptproto will return NULL if a recptproto for the given colelction
+  // could not be found
+  recptproto *getrecptproto (const text_t &collection, ostream &logout);
+};
+
+
+#endif
Index: /tags/gsdl-21/gsdl/src/recpt/statusaction.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/statusaction.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/statusaction.cpp	(revision 699)
@@ -0,0 +1,860 @@
+/**********************************************************************
+ *
+ * statusaction.cpp -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.19  1999/09/07 23:09:40  rjmcnab
+   removed some compiler warnings
+
+   Revision 1.18  1999/09/07 04:57:00  sjboddie
+   added GPL notice
+
+   Revision 1.17  1999/09/03 09:53:46  rjmcnab
+   Output more configuration options for the receptionist.
+
+   Revision 1.16  1999/09/02 00:28:42  rjmcnab
+   removed dependancy on GSDL_GSDLHOME
+
+   Revision 1.15  1999/07/30 02:24:42  sjboddie
+   added collectinfo argument to some functions
+
+   Revision 1.14  1999/07/13 23:26:07  rjmcnab
+   Added support for authenaction and usersaction
+
+   Revision 1.13  1999/06/08 04:29:39  sjboddie
+   added argsinfo to the call to check_cgiargs to make it easy to set
+   args to their default if they're found to be screwed up
+
+   Revision 1.12  1999/04/06 22:20:35  rjmcnab
+   Got browsefilter working.
+
+   Revision 1.11  1999/03/31 23:44:49  rjmcnab
+   Altered the protocol so that the metadata is part of the filter.
+
+   Revision 1.10  1999/03/16 04:47:03  rjmcnab
+   Changed the list of arguments on the argument page to be the info list
+   not the current args list (there are some arguments which don't have
+   a default so they never make it onto the args list).
+
+   Revision 1.9  1999/03/11 01:26:02  rjmcnab
+   Fixed a few small formatting things.
+
+   Revision 1.8  1999/03/11 00:12:30  rjmcnab
+   Started using status.dm to handle the bulky text. Made the status
+   action use frames and added some more information.
+
+   Revision 1.7  1999/03/09 21:00:47  rjmcnab
+   Reorganised the statusaction, added more functions to comtypes.
+
+   Revision 1.6  1999/02/25 21:59:02  rjmcnab
+
+   Merged sources.
+
+   Revision 1.5  1999/02/21 22:33:58  rjmcnab
+
+   Lots of stuff :-)
+
+   Revision 1.4  1999/02/12 02:40:18  sjboddie
+
+   Added page action
+
+   Revision 1.3  1999/02/11 23:07:00  sjboddie
+
+   extended the status action
+
+   Revision 1.2  1999/02/11 01:24:06  rjmcnab
+
+   Fixed a few compiler warnings.
+
+   Revision 1.1  1999/02/11 01:01:27  rjmcnab
+
+   Initial revision.
+
+ */
+
+
+#include "statusaction.h"
+#include "fileutil.h"
+#include <assert.h>
+
+
+void statusaction::output_frameset (cgiargsclass &/*args*/, displayclass &disp, 
+				    outconvertclass &outconvert, 
+				    ostream &textout, ostream &/*logout*/) {
+  textout << outconvert << disp << "_status:frameset_\n";
+}
+
+void statusaction::output_select (cgiargsclass &/*args*/, displayclass &disp, 
+				  outconvertclass &outconvert, 
+				  ostream &textout, ostream &/*logout*/) {
+  textout << outconvert << disp << "_status:header_(selector)\n"
+    "_status:select_\n"
+    "_status:footer_\n";
+}
+
+void statusaction::output_generalinfo (cgiargsclass &/*args*/, displayclass &disp, 
+				       outconvertclass &outconvert, 
+				       ostream &textout, ostream &/*logout*/) {
+  if (recpt == NULL) return;
+  const recptconf &rcinfo = recpt->get_configinfo ();
+  
+  textout << outconvert << disp << "_status:header_(General Information)\n";
+  textout << outconvert 
+	  << "<h2>General information</h2>\n"
+	  << "<table>\n"
+	  << "<tr valign=top><th>gsdlhome</th><td>\"" << rcinfo.gsdlhome
+	  << "\"</td></tr>\n"
+	  << "<tr valign=top><th>collection</th><td>\"" << rcinfo.collection
+	  << "\"</td></tr>\n"
+	  << "<tr valign=top><th>collectdir</th><td>\"" << rcinfo.collectdir
+	  << "\"</td></tr>\n"
+	  << "<tr valign=top><th>httpprefix</th><td>\"" << rcinfo.httpprefix
+	  << "\"</td></tr>\n"
+	  << "<tr valign=top><th>httpimg</th><td>\"" << rcinfo.httpimg
+	  << "\"</td></tr>\n"
+	  << "<tr valign=top><th>gwcgi</th><td>\"" << rcinfo.gwcgi
+	  << "\"</td></tr>\n";
+    
+  // macrofiles
+  textout << outconvert << "<tr valign=top><th>macrofiles</th><td>";
+  text_tarray::const_iterator macrohere = rcinfo.macrofiles.begin ();
+  text_tarray::const_iterator macroend = rcinfo.macrofiles.end ();
+  bool macrofirst = true;
+  while (macrohere != macroend) {
+    if (!macrofirst) textout << outconvert << ", ";
+    macrofirst = false;
+    textout << outconvert << "\"" << *macrohere << "\"";
+    macrohere++;
+  }
+  textout << outconvert << "</td></tr>\n";
+  
+  // saveconf
+  textout << outconvert << "<tr valign=top><th>saveconf</th><td>\"" << rcinfo.saveconf
+	  << "\"</td></tr>\n";
+
+  // usecookies
+  textout << outconvert << "<tr valign=top><th>usecookies</th><td>\"";
+  if (rcinfo.usecookies) textout << outconvert << "true";
+  else textout << outconvert << "false";
+  textout << outconvert << "\"</td></tr>\n";
+  
+  // logcgiargs
+  textout << outconvert << "<tr valign=top><th>logcgiargs</th><td>\"";
+  if (rcinfo.logcgiargs) textout << outconvert << "true";
+  else textout << outconvert << "false";
+  textout << outconvert << "\"</td></tr>\n";
+
+  // pageparams
+  textout << outconvert << "<tr valign=top><th>pageparams</th><td>";
+  text_tmap::const_iterator params_here = rcinfo.pageparams.begin();
+  text_tmap::const_iterator params_end = rcinfo.pageparams.end();
+  bool params_first = true;
+  while (params_here != params_end) {
+    if (!params_first) textout << outconvert << ", ";
+    params_first = false;
+    textout << outconvert << "\"" << (*params_here).first << "\" != \""
+	    << (*params_here).second << "\"";
+    
+    params_here++;
+  }
+  textout << outconvert << "</td></tr>\n";
+
+  // macroprecedence
+  textout << outconvert << "<tr valign=top><th>macroprecedence</th><td>\"" << rcinfo.macroprecedence
+	  << "\"</td></tr>\n";
+
+  // arguments
+  cgiargsinfoclass *rcargsinfo = recpt->get_cgiargsinfo_ptr ();
+  if (rcargsinfo != NULL) {
+    textout << outconvert << "<tr valign=top><th>arguments</th><td>";
+    
+    cgiargsinfoclass::const_iterator argsinfohere = rcargsinfo->begin ();
+    cgiargsinfoclass::const_iterator argsinfoend = rcargsinfo->end ();
+    bool argsinfofirst = true;
+    while (argsinfohere != argsinfoend) {
+      if (!argsinfofirst) textout << outconvert << ", ";
+      argsinfofirst = false;
+      textout << outconvert << "\"" << (*argsinfohere).second.shortname << "\"";
+      argsinfohere++;
+    }
+    
+    textout << outconvert << "</td></tr>\n";
+  }      
+  
+  // actions
+  actionmapclass *actions = recpt->get_actionmap_ptr();
+  if (actions != NULL) {
+    textout << outconvert << "<tr valign=top><th>actions</th><td>";
+    
+    actionptrmap::iterator actionshere = actions->begin ();
+    actionptrmap::iterator actionsend = actions->end ();
+    bool actionsfirst = true;
+    while (actionshere != actionsend) {
+      if (!actionsfirst) textout << outconvert << ", ";
+      actionsfirst = false;
+      assert ((*actionshere).second.a != NULL);
+      if ((*actionshere).second.a != NULL) {
+	textout << outconvert << "\"" << (*actionshere).second.a->get_action_name() << "\"";
+      }
+      actionshere++;
+    }
+    
+    textout << outconvert << "</td></tr>\n";
+  }
+  
+  // protocols
+  recptprotolistclass *protocols = recpt->get_recptprotolist_ptr ();
+  if (protocols != NULL) {
+    textout << outconvert << "<tr valign=top><th>protocols</th><td>";
+    
+    recptprotolistclass::iterator protohere = protocols->begin ();
+    recptprotolistclass::iterator protoend = protocols->end ();
+    bool protofirst = true;
+    while (protohere != protoend) {
+      if (!protofirst) textout << outconvert << ", ";
+      protofirst = false;
+      if ((*protohere).p != NULL) {
+	textout << outconvert << "\"" << (*protohere).p->get_protocol_name() << "\"";
+      }
+      protohere++;
+    }
+    
+    textout << outconvert << "</td></tr>\n";
+  }
+  
+  // converters
+  convertinfoclass *converters = recpt->get_convertinfo_ptr ();
+  if (converters != NULL) {
+    textout << outconvert << "<tr valign=top><th>converters</th><td>";
+    
+    convertinfoclass::iterator converthere = converters->begin ();
+    convertinfoclass::iterator convertend = converters->end ();
+    bool convertfirst = true;
+    while (converthere != convertend) {
+      if (!convertfirst) textout << outconvert << ", ";
+      convertfirst = false;
+      textout << outconvert << "\"" << (*converthere).second.name << "\"";
+      converthere++;
+    }
+    
+    textout << outconvert << "</td></tr>\n";
+  }
+  
+  textout << outconvert << disp << "</table>\n_status:footer_\n";
+}
+
+void statusaction::output_argumentinfo (cgiargsclass &args, displayclass &disp, 
+					outconvertclass &outconvert, 
+					ostream &textout, ostream &/*logout*/) {
+  if (recpt == NULL) return;
+  cgiargsinfoclass *rcargsinfo = recpt->get_cgiargsinfo_ptr ();
+  if (rcargsinfo == NULL) return;
+
+  textout << outconvert << disp << "_status:header_(Argument Information)\n";
+  textout << outconvert
+	  << "<h2>Argument information</h2>\n"
+	  << "<table>";
+
+  // argument information
+  textout << outconvert << "<tr valign=top><th>short name</th><th>long name</th>"
+	  << "<th>multiple char?</th>"
+	  << "<th>default</th><th>default status</th><th>saved args</th>"
+	  << "<th>current value</th></tr>\n";
+  
+
+  cgiargsinfoclass::const_iterator argsinfohere = rcargsinfo->begin();
+  cgiargsinfoclass::const_iterator argsinfoend = rcargsinfo->end();
+  text_t *arg_value;
+  while (argsinfohere != argsinfoend) {
+    const cgiarginfo &ainfo = (*argsinfohere).second;
+    textout << outconvert 
+	    << "<tr valign=top><td>" << ainfo.shortname << "</td>\n";
+    
+    textout << outconvert << "<td>" << ainfo.longname << "</td>\n";
+    if (ainfo.multiplechar) textout << outconvert << "<td>yes</td>\n";
+    else textout << outconvert << "<td>no</td>\n";
+    textout << outconvert << "<td>" << ainfo.argdefault << "</td>\n";
+    switch (ainfo.defaultstatus) {
+    case cgiarginfo::none: textout << outconvert << "<td>none</td>\n"; break;
+    case cgiarginfo::weak: textout << outconvert << "<td>weak</td>\n"; break;
+    case cgiarginfo::good: textout << outconvert << "<td>good</td>\n"; break;
+    case cgiarginfo::config: textout << outconvert << "<td>config</td>\n"; break;
+    case cgiarginfo::imperative: textout << outconvert << "<td>imperative</td>\n"; break;
+    }
+    switch (ainfo.savedarginfo) {
+    case cgiarginfo::mustnot: textout << outconvert << "<td>mustnot</td>\n"; break;
+    case cgiarginfo::can: textout << outconvert << "<td>can</td>\n"; break;
+    case cgiarginfo::must: textout << outconvert << "<td>must</td>\n"; break;
+    }
+    
+    arg_value = args.getarg (ainfo.shortname);
+    if (arg_value == NULL) textout << outconvert << "<td></td></tr>\n";
+    else textout << outconvert << "<td>\"" << *arg_value << "\"</td></tr>\n";
+    
+    argsinfohere ++;
+  }
+  
+  textout << outconvert << disp << "</table>\n_status:footer_\n";
+}
+
+void statusaction::output_actioninfo (cgiargsclass &/*args*/, displayclass &disp, 
+				      outconvertclass &outconvert, 
+				      ostream &textout, ostream &/*logout*/) {
+  if (recpt == NULL) return;
+  actionmapclass *actions = recpt->get_actionmap_ptr();
+
+  textout << outconvert << disp << "_status:header_(Action Information)\n";
+  textout << outconvert
+	  << "<h2>Action information</h2>\n"
+	  << "<table>";
+
+  // action information
+  if (actions != NULL) {
+    textout << outconvert 
+	    << "<tr><th>action name</th><th>cgi arguments</th></tr>\n";
+	
+    actionptrmap::iterator actionshere = actions->begin ();
+    actionptrmap::iterator actionsend = actions->end ();
+    while (actionshere != actionsend) {
+      assert ((*actionshere).second.a != NULL);
+      if ((*actionshere).second.a != NULL) {
+	textout << outconvert 
+		<< "<tr><td>" << (*actionshere).second.a->get_action_name()
+		<< "</td><td>";
+	
+	cgiargsinfoclass argsinfo = (*actionshere).second.a->getargsinfo();
+	cgiargsinfoclass::const_iterator argsinfohere = argsinfo.begin ();
+	cgiargsinfoclass::const_iterator argsinfoend = argsinfo.end ();
+	bool aifirst = true;
+	while (argsinfohere != argsinfoend) {
+	  if (!aifirst) textout << outconvert << ", ";
+	  aifirst = false;
+	  textout << outconvert << (*argsinfohere).second.shortname;
+	  argsinfohere++;
+	}
+	
+	textout << outconvert << "</td></tr>\n";
+      }
+      actionshere++;
+    }
+  }
+  
+  textout << outconvert << disp << "</table>\n_status:footer_\n";
+}
+
+void statusaction::output_protocolinfo (cgiargsclass &/*args*/, displayclass &disp, 
+					outconvertclass &outconvert, 
+					ostream &textout, ostream &logout) {
+  if (recpt == NULL) return;
+  const recptconf &rcinfo = recpt->get_configinfo ();
+  bool colspecific = !rcinfo.collection.empty();
+  bool unreachablecol = false;
+
+  recptprotolistclass *rprotolist = recpt->get_recptprotolist_ptr ();
+  if (rprotolist == NULL) return;
+
+  textout << outconvert << disp << "_status:header_(Protocol Information)\n";
+  textout << outconvert
+	  << "<h2>Protocol information</h2>\n"
+	  << "<table>\n"
+	  << "<tr><th>protocol</th><th>collections</th></tr>\n";
+
+  text_t protoname;
+  recptprotolistclass::iterator rprotolist_here = rprotolist->begin();
+  recptprotolistclass::iterator rprotolist_end = rprotolist->end();
+  while (rprotolist_here != rprotolist_end) {
+    if ((*rprotolist_here).p != NULL) {
+      protoname = (*rprotolist_here).p->get_protocol_name();
+      textout << outconvert
+	      << "<tr><td>"
+	      << protoname
+	      << "</td><td>";
+
+      text_tarray collist;
+      comerror_t err;
+      (*rprotolist_here).p->get_collection_list (collist, err, logout);
+      if (err == noError) {
+	text_tarray::iterator collist_here = collist.begin();
+	text_tarray::iterator collist_end = collist.end();
+	bool first = true;
+	while (collist_here != collist_end) {
+	  if (!first) textout << outconvert << ", ";
+	  first = false;
+
+	  if (colspecific && *collist_here != rcinfo.collection) {
+	    unreachablecol = true;
+	    textout << outconvert << "\"" << *collist_here << "\"";
+	  } else {
+	    textout << outconvert << disp
+		    << "\"<a href=\"_gwcgi_?e=_compressedoptions_&a=status&sp=collectioninfo&pr="
+		    << protoname
+		    << "&c="
+		    << *collist_here
+		    << "\">"
+		    << *collist_here 
+		    << "</a>\"";
+	  }
+
+	  collist_here++;
+	}
+
+      } else {
+	textout << outconvert << "Error (" << get_comerror_string (err) 
+		<< ") while getting collect list\n";
+      }
+
+      textout << outconvert 
+	      << "</td></tr>\n";
+    }
+
+    rprotolist_here++;
+  }
+
+  textout << outconvert << "</table>\n";
+  if (unreachablecol) {
+    textout << outconvert 
+	    << "<b>Warning:</b> the receptionist is running in collection specific\n"
+	    << "mode making some of the collections unreachable.\n";
+  }
+  textout << outconvert << disp << "_status:footer_\n";
+}
+
+void statusaction::output_collectioninfo (cgiargsclass &args, displayclass &disp, 
+					  outconvertclass &outconvert, 
+					  ostream &textout, ostream &logout) {
+  if (recpt == NULL) return;
+
+  textout << outconvert << disp << "_status:header_(Collection info)\n";
+  textout << outconvert << "<h2>Collection info</h2>\n";
+
+  // get the list of protocols
+  recptprotolistclass *rprotolist = recpt->get_recptprotolist_ptr ();
+  if (rprotolist == NULL) return;
+
+  // look for the desired protocol
+  text_t &arg_pr = args["pr"];
+  text_t &arg_c = args["c"];
+  recptproto *rproto;
+  recptprotolistclass::iterator rprotolist_here = rprotolist->begin();
+  recptprotolistclass::iterator rprotolist_end = rprotolist->end();
+  while (rprotolist_here != rprotolist_end) {
+    rproto = (*rprotolist_here).p;
+    if (rproto != NULL && rproto->get_protocol_name() == arg_pr) {
+      // see if the protocol has the collection
+      bool hascollection;
+      comerror_t err;
+      rproto->has_collection (arg_c, hascollection, err, logout);
+      if (err == noError && hascollection) break;
+    }
+    rprotolist_here++;
+  }
+  
+  if (rprotolist_here == rprotolist_end) {
+    textout << outconvert << "Protocol \"" << arg_pr << "\" with collection \""
+	    << arg_c << "\" was not found\n";
+
+  } else {
+    ColInfoResponse_t collectinfo;
+    comerror_t err;
+
+    rproto->get_collectinfo (arg_c, collectinfo, err, logout);
+    if (err == noError) {
+      textout << outconvert << "<table>\n"
+	      << "<tr><th>collection name</th><td>\""
+	      << collectinfo.shortInfo.name
+	      << "\"</td></tr>\n"
+
+	      << "<tr><th>host</th><td>\""
+	      << collectinfo.shortInfo.host
+	      << "\"</td></tr>\n"
+
+	      << "<tr><th>port</th><td>\""
+	      << collectinfo.shortInfo.port
+	      << "\"</td></tr>\n"
+
+	      << "<tr><th>is public?</th><td>";
+      if (collectinfo.isPublic) textout << outconvert << "true";
+      else textout << outconvert << "false";
+      textout << outconvert 
+	      << "</td></tr>\n"
+
+	      << "<tr><th>is beta?</th><td>";
+      if (collectinfo.isBeta) textout << outconvert << "true";
+      else textout << outconvert << "false";
+      textout << outconvert 
+	      << "</td></tr>\n"
+
+	      << "<tr><th>build date</th><td>\""
+	      << collectinfo.buildDate
+	      << "\"</td></tr>\n"
+
+	      << "<tr><th>interface languages</th><td>";
+      text_tarray::iterator languages_here = collectinfo.languages.begin();
+      text_tarray::iterator languages_end = collectinfo.languages.end();
+      bool languages_first = true;
+      while (languages_here != languages_end) {
+	if (!languages_first) textout << outconvert << ", ";
+	languages_first = false;
+	textout << outconvert << "\"" << *languages_here << "\"";
+	languages_here++;
+      }
+      textout << outconvert 
+	      << "</td></tr>\n"
+
+	      << "<tr><th>number of documents</th><td>\""
+	      << collectinfo.numDocs
+	      << "\"</td></tr>\n"
+
+	      << "<tr><th>number of words</th><td>\""
+	      << collectinfo.numWords
+	      << "\"</td></tr>\n"
+
+	      << "<tr><th>number of bytes</th><td>\""
+	      << collectinfo.numBytes
+	      << "\"</td></tr>\n"
+
+	      << "</table>";
+      
+    } else {
+      textout << outconvert << "Error (" << get_comerror_string (err) 
+	      << ") while getting collect information\n";
+    }
+
+
+    InfoFiltersResponse_t filterinfo;
+    InfoFilterOptionsRequest_t filteroptions_request;
+    InfoFilterOptionsResponse_t filteroptions;
+    rproto->get_filterinfo (arg_c, filterinfo, err, logout);
+    if (err == noError) {
+      text_tset::iterator filternames_here = filterinfo.filterNames.begin();
+      text_tset::iterator filternames_end = filterinfo.filterNames.end();
+      while (filternames_here != filternames_end) {
+	textout << outconvert 
+		<< "<hr>\n"
+		<< "<h3>Filter options for \"" << (*filternames_here) << "\"</h3>\n"
+		<< "<table>\n"
+		<< "<tr><th>option name</th><th>type</th><th>repeatable</th>"
+		<< "<th>default value</th><th>valid values</th></tr>\n";
+
+	filteroptions_request.clear();
+	filteroptions_request.filterName = *filternames_here;
+	rproto->get_filteroptions (arg_c, filteroptions_request,
+				   filteroptions, err, logout);
+	if (err == noError) {
+	  FilterOption_tmap::iterator filteropt_here = 
+	    filteroptions.filterOptions.begin();
+	  FilterOption_tmap::iterator filteropt_end = 
+	    filteroptions.filterOptions.end();
+	  while (filteropt_here != filteropt_end) {
+	    textout << outconvert
+		    << "<tr><td>\""
+		    << (*filteropt_here).second.name
+		    << "\"</td>\n"
+	      
+		    << "<td>";
+	    text_t type_string;
+	    switch ((*filteropt_here).second.type) {
+	    case FilterOption_t::booleant: type_string = "boolean"; break;
+	    case FilterOption_t::integert: type_string = "integer"; break;
+	    case FilterOption_t::enumeratedt: type_string = "enumerated"; break;
+	    case FilterOption_t::stringt: type_string = "string"; break;
+	    }
+	    textout << outconvert
+		    << type_string
+		    << "</td>\n"
+	      
+		    << "<td>";
+	    text_t repeat_string;
+	    switch ((*filteropt_here).second.repeatable) {
+	    case FilterOption_t::onePerQuery: repeat_string = "one per query"; break;
+	    case FilterOption_t::onePerTerm: repeat_string = "one per term"; break;
+	    case FilterOption_t::nPerTerm: repeat_string = "n per term"; break;
+	    }
+	    textout << outconvert
+		    << repeat_string
+		    << "</td>\n"
+	      
+		    << "<td>\""
+		    << (*filteropt_here).second.defaultValue
+		    << "\"</td>\n"
+	      
+		    << "<td>";
+	    
+	    text_tarray::iterator valid_here = 
+	      (*filteropt_here).second.validValues.begin();
+	    text_tarray::iterator valid_end = 
+	      (*filteropt_here).second.validValues.end();
+	    bool valid_first = true;
+	    while (valid_here != valid_end) {
+	      if (!valid_first) textout << outconvert << ", ";
+	      valid_first = false;
+	      textout << outconvert << "\"" << *valid_here << "\"";
+	      valid_here++;
+	    }
+	    textout << outconvert
+		    << "</td></tr>\n";
+
+	    filteropt_here++;
+	  }
+
+	  textout << outconvert
+		  << "</table>\n";
+
+	} else {
+	  textout << outconvert << "Error (" << get_comerror_string (err) 
+		  << ") while getting filter option information\n";
+	}
+
+	filternames_here++;
+      }
+
+    } else {
+      textout << outconvert << "Error (" << get_comerror_string (err) 
+	      << ") while getting filter information\n";
+    }
+  }
+  
+  textout << outconvert << disp << "_status:footer_\n";
+}
+
+void statusaction::output_initlog (cgiargsclass &/*args*/, displayclass &disp,
+				   outconvertclass &outconvert, 
+				   ostream &textout, ostream &/*logout*/) {
+  if (recpt == NULL) return;
+  const recptconf &rcinfo = recpt->get_configinfo ();
+  text_t initfilename = filename_cat (rcinfo.gsdlhome, "etc", "initout.txt");
+  char *cinitfilename = initfilename.getcstr();
+  if (cinitfilename == NULL) return;
+
+  textout << outconvert << disp << "_status:header_(Init log)\n";
+  textout << outconvert << "<h2>Init log</h2>\n";
+
+  ifstream initin (cinitfilename, ios::in | ios::nocreate);
+  delete cinitfilename;
+  if (initin) {
+    textout << outconvert << "<p>The initialisation error log, "
+		<< initfilename << ", contains the\n";
+    textout << outconvert << "following information:\n\n";
+    text_t errorpage = "<p><pre>\n";
+    
+    char c;
+    initin.get(c);
+    while (!initin.eof ()) {
+      errorpage.push_back(c);
+      initin.get(c);
+    }
+    
+    errorpage += "</pre>\n";
+    initin.close();
+    textout << outconvert << errorpage;
+    
+  } else {
+    textout << outconvert << "Couldn't read initialisation error log, " 
+		<< initfilename << ".\n";
+  }
+
+  textout << outconvert << disp << "_status:footer_\n";
+}
+
+void statusaction::output_errorlog (cgiargsclass &/*args*/, displayclass &disp, 
+				    outconvertclass &outconvert, 
+				    ostream &textout, ostream &logout) {
+  if (recpt == NULL) return;
+  const recptconf &rcinfo = recpt->get_configinfo ();
+  text_t errfilename = filename_cat (rcinfo.gsdlhome, "etc", "errout.txt");
+  char *cerrfilename = errfilename.getcstr();
+  if (cerrfilename == NULL) return;
+
+  textout << outconvert << disp << "_status:header_(Error log)\n";
+  textout << outconvert << "<h2>Error log</h2>\n";
+  logout << flush;
+
+  ifstream errin (cerrfilename, ios::in | ios::nocreate);
+  delete cerrfilename;
+  if (errin) {
+    textout << outconvert << "<p>The error log, "
+		<< errfilename << ", contains the\n";
+    textout << outconvert << "following information:\n\n";
+    text_t errorpage = "<p><pre>\n";
+
+    char c;
+    errin.get(c);
+    while (!errin.eof ()) {
+      errorpage.push_back(c);
+      errin.get(c);
+    }
+    
+    errorpage += "</pre>\n";
+    errin.close();
+    textout << outconvert << errorpage;
+
+  } else {
+    textout << outconvert << "Couldn't read error log, " 
+		<< errfilename << ".\n";
+  }
+
+  textout << outconvert << disp << "_status:footer_\n";
+}
+
+void statusaction::output_errorpage (outconvertclass &outconvert, 
+				     ostream &textout, ostream &/*logout*/,
+				     text_t message) {
+  textout << outconvert 
+	  << "<html>\n"
+	  << "<head>\n"
+	  << "<title>Error</title>\n"
+	  << "</head>\n"
+	  << "<body bgcolor=\"#ffffff\" text=\"#000000\" link=\"#006666\" "
+	  << "alink=\"#cc9900\" vlink=\"#666633\">\n"
+	  << "<h2>Oops!</h2>\n" 
+	  << message
+	  << "\n</body>\n"
+	  << "</html>\n";
+  return;
+}
+
+
+
+statusaction::statusaction () {
+  disabled = true;
+  recpt = NULL;
+
+  // this action uses cgi variable "a"
+  cgiarginfo arg_ainfo;
+  arg_ainfo.shortname = "a";
+  arg_ainfo.longname = "action";
+  arg_ainfo.multiplechar = true;
+  arg_ainfo.defaultstatus = cgiarginfo::weak;
+  arg_ainfo.argdefault = "status";
+  arg_ainfo.savedarginfo = cgiarginfo::must;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+
+  // "sp" -- status page
+  arg_ainfo.shortname = "sp";
+  arg_ainfo.longname = "status page";
+  arg_ainfo.multiplechar = true;
+  arg_ainfo.defaultstatus = cgiarginfo::good;
+  arg_ainfo.argdefault = "frameset";
+  arg_ainfo.savedarginfo = cgiarginfo::can;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+
+  // "pr" -- protocol
+  arg_ainfo.shortname = "pr";
+  arg_ainfo.longname = "protocol";
+  arg_ainfo.multiplechar = true;
+  arg_ainfo.defaultstatus = cgiarginfo::none;
+  arg_ainfo.argdefault = "";
+  arg_ainfo.savedarginfo = cgiarginfo::can;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+}
+
+statusaction::~statusaction () {
+}
+
+bool statusaction::check_cgiargs (cgiargsinfoclass &/*argsinfo*/, cgiargsclass &args, 
+				  ostream &/*logout*/) {
+  // authenticate the user if authentication is avaiable
+  args["uan"] = 1;
+  args["ug"] = "administrator";
+  
+  return true;
+}
+
+void statusaction::get_cgihead_info (cgiargsclass &/*args*/, response_t &response,
+				     text_t &response_data, ostream &/*logout*/) {
+  response = content;
+  response_data = "text/html";
+}
+
+bool statusaction::do_action (cgiargsclass &args, const ColInfoResponse_t &/*collectinfo*/, 
+			      recptproto * /*collectproto*/, displayclass &disp, 
+			      outconvertclass &outconvert, ostream &textout, 
+			      ostream &logout) {
+  // make sure the status function is enabled
+  if (disabled) {
+    output_errorpage (outconvert, textout, logout, 
+		      "The status action is disabled. "
+		      "See the documentation (what documentation!) on how "
+		      "to enable it.");
+    return true;
+  }
+  
+  // make sure we know about a receptionist
+  if (recpt == NULL) {
+    output_errorpage (outconvert, textout, logout, 
+		      "The status action does not contain information\n"
+		      "about any receptionists. The method set_receptionist\n"
+		      "was probably not called from the module which instantiated\n"
+		      "this status action.\n");
+    return true;
+  }
+
+  // check to make sure status.dm was read in
+  const recptconf &rcinfo = recpt->get_configinfo ();
+  text_tarray::const_iterator macrohere = rcinfo.macrofiles.begin ();
+  text_tarray::const_iterator macroend = rcinfo.macrofiles.end ();
+  while (macrohere != macroend) {
+    if (*macrohere == "status.dm") break;
+    macrohere++;
+  }
+  if (macrohere == macroend) {
+    output_errorpage (outconvert, textout, logout, 
+		      "The status.dm file was not read in. This macro file is\n" 
+		      "needed to display configuration and status information.\n");
+    return true;
+  }
+
+  // output the required status page
+  text_t &arg_sp = args["sp"];
+  if (arg_sp == "frameset") output_frameset (args, disp, outconvert, textout, logout);
+  else if (arg_sp == "select") output_select (args, disp, outconvert, textout, logout);
+  else if (arg_sp == "generalinfo") output_generalinfo (args, disp, outconvert, textout, logout);
+  else if (arg_sp == "argumentinfo") output_argumentinfo (args, disp, outconvert, textout, logout);
+  else if (arg_sp == "actioninfo") output_actioninfo (args, disp, outconvert, textout, logout);
+  else if (arg_sp == "protocolinfo") output_protocolinfo (args, disp, outconvert, textout, logout);
+  else if (arg_sp == "collectioninfo") output_collectioninfo (args, disp, outconvert, textout, logout);
+  else if (arg_sp == "initlog") output_initlog (args, disp, outconvert, textout, logout);
+  else if (arg_sp == "errorlog") output_errorlog (args, disp, outconvert, textout, logout);
+  else {
+    output_errorpage (outconvert, textout, logout, 
+		      "Unknown page \"" + arg_sp + "\".\n");
+  }
+
+  return true;
+}
+
+void statusaction::configure (const text_t &key, const text_tarray &cfgline) {
+  if ((key == "status") && (cfgline.size() == 1) && 
+      (cfgline[0] == "true" || cfgline[0] == "on" || cfgline[0] == "enabled")) {
+    disabled = false;
+  } else {
+    // call the parent class to deal with the things which
+    // are not dealt with here
+    action::configure (key, cfgline);
+  }
+}
+
Index: /tags/gsdl-21/gsdl/src/recpt/statusaction.h
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/statusaction.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/statusaction.h	(revision 699)
@@ -0,0 +1,95 @@
+/**********************************************************************
+ *
+ * statusaction.h -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef STATUSACTION_H
+#define STATUSACTION_H
+
+
+#include "gsdlconf.h"
+#include "action.h"
+#include "receptionist.h"
+
+
+class statusaction : public action {
+protected:
+  bool disabled;
+  receptionist *recpt;
+
+  void output_frameset (cgiargsclass &args, displayclass &disp, 
+			outconvertclass &outconvert, 
+			ostream &textout, ostream &logout);
+  void output_select (cgiargsclass &args, displayclass &disp, 
+		      outconvertclass &outconvert, 
+		      ostream &textout, ostream &logout);
+
+  void output_generalinfo (cgiargsclass &args, displayclass &disp, 
+			   outconvertclass &outconvert, 
+			   ostream &textout, ostream &logout);
+  void output_argumentinfo (cgiargsclass &args, displayclass &disp, 
+			    outconvertclass &outconvert, 
+			    ostream &textout, ostream &logout);
+  void output_actioninfo (cgiargsclass &args, displayclass &disp, 
+			  outconvertclass &outconvert, 
+			  ostream &textout, ostream &logout);
+  void output_protocolinfo (cgiargsclass &args, displayclass &disp, 
+			    outconvertclass &outconvert, 
+			    ostream &textout, ostream &logout);
+  void output_collectioninfo (cgiargsclass &args, displayclass &disp, 
+			      outconvertclass &outconvert, 
+			      ostream &textout, ostream &logout);
+  void output_initlog (cgiargsclass &args, displayclass &disp, 
+		       outconvertclass &outconvert, 
+		       ostream &textout, ostream &logout);
+  void output_errorlog (cgiargsclass &args, displayclass &disp, 
+			outconvertclass &outconvert, 
+			ostream &textout, ostream &logout);
+
+  void output_errorpage (outconvertclass &outconvert, 
+			 ostream &textout, ostream &logout,
+			 text_t message);
+
+public:
+  statusaction ();
+  virtual ~statusaction ();
+
+  text_t get_action_name () {return "status";}
+  void set_receptionist (receptionist *therecpt) {recpt=therecpt;}
+  bool check_cgiargs (cgiargsinfoclass &argsinfo, cgiargsclass &args, 
+		      ostream &logout);
+  void get_cgihead_info (cgiargsclass &args, response_t &response,
+			 text_t &response_data, ostream &logout);
+  bool do_action (cgiargsclass &args, const ColInfoResponse_t &collectinfo, 
+		  recptproto *collectproto, displayclass &disp, 
+		  outconvertclass &outconvert, ostream &textout, 
+		  ostream &logout);
+
+  void configure (const text_t &key, const text_tarray &cfgline);
+};
+
+
+#endif
Index: /tags/gsdl-21/gsdl/src/recpt/tipaction.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/tipaction.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/tipaction.cpp	(revision 699)
@@ -0,0 +1,90 @@
+/**********************************************************************
+ *
+ * tipaction.cpp -- define random tip macros
+ * Copyright (C) 1999  DigiLib Systems Limited, New Zealand
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.5  1999/09/07 23:10:18  rjmcnab
+   removed some compiler warnings
+
+   Revision 1.4  1999/09/07 04:57:00  sjboddie
+   added GPL notice
+
+   Revision 1.3  1999/09/02 00:29:27  rjmcnab
+   added cgi argument "st" - show tips
+
+   Revision 1.2  1999/07/30 02:24:45  sjboddie
+   added collectinfo argument to some functions
+
+   Revision 1.1  1999/07/09 02:18:27  rjmcnab
+   Initial revision.
+
+
+ */
+
+
+#include "tipaction.h"
+#include <stdlib.h>
+
+
+tipaction::tipaction () {
+  // "st" show tips - "0"=no, "1"=yes
+  cgiarginfo arg_ainfo;
+  arg_ainfo.shortname = "st";
+  arg_ainfo.longname = "show tips";
+  arg_ainfo.multiplechar = true;
+  arg_ainfo.defaultstatus = cgiarginfo::weak;
+  arg_ainfo.argdefault = "1";
+  arg_ainfo.savedarginfo = cgiarginfo::must;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+}
+
+void tipaction::define_external_macros (const ColInfoResponse_t &/*collectinfo*/, displayclass &disp, 
+					cgiargsclass &args, recptproto * /*collectproto*/, 
+					ostream &logout) {
+  // define_external_macros sets the following macros:
+
+  // _tip:thistip_  - a random tip (defined if st == "1")
+
+  if (args["st"].getint()) {
+    // get the maximum tip number (_tip:tipmax_)
+    text_t tipmax_str;
+    disp.expandstring ("tip", "_tipmax_", tipmax_str);
+    if (tipmax_str.empty()) {
+      logout << "Warning: _tip:tipmax_ not set. This macro is needed for displaying tips.\n";
+      return;
+    }
+    
+    int tipmax = tipmax_str.getint();
+    if (tipmax < 1) {
+      logout << "Warning: _tip:tipmax_ was less than 1. No tips were set.\n";
+      return;
+    }
+    
+    disp.setmacro("thistip", "tip", "_tip" + text_t((rand()%tipmax) + 1) + "_");
+  }
+}
+
Index: /tags/gsdl-21/gsdl/src/recpt/tipaction.h
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/tipaction.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/tipaction.h	(revision 699)
@@ -0,0 +1,49 @@
+/**********************************************************************
+ *
+ * tipaction.h -- define tip macros
+ * Copyright (C) 1999  DigiLib Systems Limited, New Zealand
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef TIPACTION_H
+#define TIPACTION_H
+
+#include "gsdlconf.h"
+#include "action.h"
+
+
+// tipaction exists just for the benefit of defining external macros
+class tipaction : public action {
+public:
+  tipaction ();
+  virtual ~tipaction () {}
+  text_t get_action_name () {return "tip";}
+  
+  void define_external_macros (const ColInfoResponse_t &collectinfo, displayclass &disp, 
+			       cgiargsclass &args, recptproto *collectproto, 
+			       ostream &logout);
+};
+
+
+#endif
Index: /tags/gsdl-21/gsdl/src/recpt/userdb.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/userdb.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/userdb.cpp	(revision 699)
@@ -0,0 +1,376 @@
+/**********************************************************************
+ *
+ * userdb.cpp -- functions to handle a user database
+ * Copyright (C) 1999  DigiLib Systems Limited, New Zealand
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.4  1999/09/07 04:57:00  sjboddie
+   added GPL notice
+
+   Revision 1.3  1999/09/02 00:30:04  rjmcnab
+   added option for specifying whether the gdbm database should be locked
+
+   Revision 1.2  1999/07/14 08:30:01  rjmcnab
+   fixed an error to do with the way time is dealt with on different machines.
+   It seems that there is no direct corresponding function to mktime and
+   localtime or gmtime might not get you back to where you started.
+
+   Revision 1.1  1999/07/13 23:22:04  rjmcnab
+   Initial revision.
+
+ */
+
+
+#include "userdb.h"
+#include "gsdltimes.h"
+#include "fileutil.h"
+#include <stdlib.h>
+
+// include crypt
+//#include <crypt.h>
+#include <unistd.h>
+
+
+
+// few useful functions
+
+text_t crypt_text (const text_t &text) {
+  static const char *salt = "Tp";
+  text_t crypt_password;
+
+  if (text.empty()) return "";
+
+  // encrypt the password
+  char *text_cstr = text.getcstr();
+  if (text_cstr == NULL) return "";
+  crypt_password = crypt(text_cstr, salt);
+  delete text_cstr;
+
+  return crypt_password;
+}
+
+
+// username_ok tests to make sure a username is ok. a username
+// must be at least 2 characters long, but no longer than 30
+// characters long. it can contain the characters a-z A-Z 0-9
+// . and _
+bool username_ok (const text_t &username) {
+  if (username.size() < 2 || username.size() > 30) return false;
+
+  text_t::const_iterator here = username.begin();
+  text_t::const_iterator end = username.end();
+  while (here != end) {
+    if ((*here >= 'a' && *here <= 'z') ||
+	(*here >= 'A' && *here <= 'Z') ||
+	(*here >= '0' && *here <= '9') ||
+	*here == '.' ||
+	*here == '_') {
+      // ok
+    } else return false;
+    here++;
+  }
+
+  return true;
+}
+
+// password_ok tests to make sure a password is ok. a password
+// must be at least 3 characters long but no longer than 8 characters
+// long. it can contain any character in the range 0x20-0x7e
+bool password_ok (const text_t &password) {
+  if (password.size() < 3 || password.size() > 8) return false;
+
+  text_t::const_iterator here = password.begin();
+  text_t::const_iterator end = password.end();
+  while (here != end) {
+    if (*here >= 0x20 && *here <= 0x7e) {
+      // ok
+    } else return false;
+    here++;
+  }
+
+  return true;
+}
+
+
+
+
+/////////////
+// userinfo_t
+/////////////
+
+void userinfo_t::clear () {
+  username.clear();
+  password.clear();
+  enabled = false;
+  groups.clear();
+  comment.clear();
+}
+
+userinfo_t &userinfo_t::operator=(const userinfo_t &x) {
+  username = x.username;
+  password = x.password;
+  enabled = x.enabled;
+  groups = x.groups;
+  comment = x.comment;
+  
+  return *this;
+}
+
+
+
+// functions dealing with user databases
+
+// returns true on success (in which case userinfo will contain
+// the information for this user)
+bool get_user_info (gdbmclass &userdb, const text_t &username,
+		    userinfo_t &userinfo) {
+  userinfo.clear();
+  
+  infodbclass info;
+  if (userdb.getinfo (username, info)) {
+    userinfo.username = info["username"];
+    userinfo.password = info["password"];
+    userinfo.enabled = (info["enabled"] == "true");
+    userinfo.groups = info["groups"];
+    userinfo.comment = info["comment"];
+    return true;
+  }
+
+  return false;
+}
+
+bool get_user_info (const text_t &userdbfile, const text_t &username,
+		    userinfo_t &userinfo) {
+  gdbmclass userdb;
+  if (!userdb.opendatabase(userdbfile, GDBM_READER, 1000, true)) {
+/*    if (!userdbfile.empty() && !file_exists (userdbfile) &&
+	username == "admin") {*/
+    if (!userdbfile.empty() && username == "admin") {
+      // no database -- create a database with an initial account
+      userinfo.clear();
+      userinfo.username = "admin";
+      userinfo.password = crypt_text("admin");
+      userinfo.enabled = true;
+      userinfo.groups = "administrator";
+      userinfo.comment = "change the password for this account as soon as possible";
+      return set_user_info (userdbfile, username, userinfo);
+    }
+    return false;
+  }
+
+  bool success = get_user_info (userdb, username, userinfo);
+  userdb.closedatabase();
+
+  return success;
+}
+
+// returns true on success
+bool set_user_info (gdbmclass &userdb, const text_t &username,
+		    const userinfo_t &userinfo) {
+  infodbclass info;
+  info["username"] = userinfo.username;
+  info["password"] = userinfo.password;
+  info["enabled"] = userinfo.enabled ? "true" : "false";
+  info["groups"] = userinfo.groups;
+  info["comment"] = userinfo.comment;
+
+  return userdb.setinfo (username, info);
+}
+
+bool set_user_info (const text_t &userdbfile, const text_t &username,
+		    const userinfo_t &userinfo) {
+  gdbmclass userdb;
+  if (!userdb.opendatabase(userdbfile, GDBM_WRCREAT, 1000, true)) return false;
+
+  bool success = set_user_info (userdb, username, userinfo);
+  userdb.closedatabase();
+
+  return success;
+}
+
+// removes a user from the user database -- forever
+void delete_user (gdbmclass &userdb, const text_t &username) {
+  userdb.deletekey (username);
+}
+
+void delete_user (const text_t &userdbfile, const text_t &username) {
+  gdbmclass userdb;
+  if (!userdb.opendatabase(userdbfile, GDBM_WRCREAT, 1000, true)) return;
+
+  delete_user (userdb, username);
+  userdb.closedatabase();
+}
+
+// gets a list of all the users in the database. returns true
+// on success
+void get_user_list (gdbmclass &userdb, text_tarray &userlist) {
+  userlist.erase (userlist.begin(), userlist.end());
+  
+  text_t user = userdb.getfirstkey ();
+  while (!user.empty()) {
+    userlist.push_back(user);
+    user = userdb.getnextkey (user);
+  }
+}
+
+// returns true if the user's password is correct.
+bool check_passwd (const userinfo_t &thisuser, const text_t &password) {
+  // couple of basic checks
+  if (thisuser.username.empty() || thisuser.password.empty() ||
+      password.empty()) return false;
+
+  text_t crypt_password = crypt_text(password);
+  return (thisuser.password == crypt_password);
+}
+
+
+// functions dealing with databases of temporary keys
+
+// generates a random key for the user, stores it in the database and
+// returns it so that it can be used in page generation
+// returns "" on failure
+text_t generate_key (gdbmclass &keydb, const text_t &username) {
+  static const char *numconvert = "0123456789abcdefghijklmnopqrstuvwxyz"
+    "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
+
+  // loop looking for a suitable new key
+  text_t userkey;
+  text_t crypt_userkey;
+  do {
+    // convert to base 62 :-)
+    int userkey_int = rand ();
+    while (userkey_int > 0) {
+      userkey.push_back (numconvert[userkey_int%62]);
+      userkey_int /= 62;
+    }
+
+    // make sure this key is not in the database
+    crypt_userkey = crypt_text(userkey);
+    if (keydb.exists (crypt_userkey)) userkey.clear();
+  } while (userkey.empty());
+
+  // enter the key into the database
+  infodbclass keydata;
+  keydata["user"] = username;
+  keydata["time"] = time2text(time(NULL));
+  
+  if (!keydb.setinfo (crypt_userkey, keydata)) {
+    userkey.clear(); // failed
+  }
+  
+  return userkey;
+}
+
+text_t generate_key (const text_t &keydbfile, const text_t &username) {
+  gdbmclass keydb;
+  if (!keydb.opendatabase(keydbfile, GDBM_WRCREAT, 1000, true)) return "";
+
+  text_t key = generate_key (keydb, username);
+  keydb.closedatabase();
+
+  return key;
+}
+
+
+// checks to see if there is a key for this particular user in the
+// database that hasn't decayed. a short decay is used when group
+// is set to administrator
+bool check_key (gdbmclass &keydb, const userinfo_t &thisuser,
+		const text_t &key, const text_t &group, int keydecay) {
+  if (thisuser.username.empty() || key.empty()) return false;
+
+  // the keydecay is set to 5 minute for things requiring the
+  // administrator
+  if (group == "administrator") keydecay = 300;
+  
+  // success if there is a key in the key database that is owned by this
+  // user whose creation time is less that keydecay
+  text_t crypt_key = crypt_text(key);
+  infodbclass info;
+  if (keydb.getinfo (crypt_key, info))  {
+    if (info["user"] == thisuser.username) {
+      time_t keycreation = text2time (info["time"]);
+      if (keycreation != (time_t)-1 && difftime (text2time(time2text(time(NULL))),
+						 keycreation) <= keydecay) {
+	// succeeded, update the key's time
+	info["time"] = time2text(time(NULL));
+	keydb.setinfo (crypt_key, info);
+	return true;
+      }
+    }
+  }
+  
+  return false;;
+}
+
+bool check_key (const text_t &keydbfile, const userinfo_t &thisuser,
+		const text_t &key, const text_t &group, int keydecay) {
+  gdbmclass keydb;
+  if (!keydb.opendatabase(keydbfile, GDBM_WRCREAT, 1000, true)) return false;
+
+  bool success = check_key (keydb, thisuser, key, group, keydecay);
+  keydb.closedatabase();
+
+  return success;
+}
+
+
+// remove_old_keys will remove all keys created more than keydecay ago.
+// use sparingly, it can be quite an expensive function
+void remove_old_keys (const text_t &keydbfile, int keydecay) {
+  // open the key database
+  gdbmclass keydb;
+  if (!keydb.opendatabase(keydbfile, GDBM_WRCREAT, 1000, true)) return;
+
+  // get a list of keys created more than keydecay seconds agon
+  text_tarray oldkeys;
+  text_t key = keydb.getfirstkey ();
+  infodbclass info;
+  time_t timenow = text2time(time2text(time(NULL)));
+  time_t keycreation = (time_t)-1;
+  while (!key.empty()) {
+    if (keydb.getinfo (key, info))  {
+      keycreation = text2time (info["time"]);
+      if (keycreation != (time_t)-1 && difftime (timenow, keycreation) > keydecay) {
+	// found an old key
+	oldkeys.push_back(key);
+      }
+    }
+    
+    key = keydb.getnextkey (key);
+  }
+
+  // delete the keys
+  text_tarray::iterator keys_here = oldkeys.begin();
+  text_tarray::iterator keys_end = oldkeys.end();
+  while (keys_here != keys_end) {
+    keydb.deletekey(*keys_here);
+    keys_here++;
+  }
+  
+  // close the key database
+  keydb.closedatabase();
+}
Index: /tags/gsdl-21/gsdl/src/recpt/userdb.h
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/userdb.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/userdb.h	(revision 699)
@@ -0,0 +1,127 @@
+/**********************************************************************
+ *
+ * userdb.h -- functions to handle a user database
+ * Copyright (C) 1999  DigiLib Systems Limited, New Zealand
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef USERDB_H
+#define USERDB_H
+
+#include "infodbclass.h"
+
+
+// the password line in the userinfo_t must be encrypted using
+// this function
+text_t crypt_text (const text_t &text);
+
+// username_ok tests to make sure a username is ok. a username
+// must be at least 2 characters long, but no longer than 30
+// characters long. it can contain the characters a-z A-Z 0-9
+// . and _
+bool username_ok (const text_t &username);
+
+// password_ok tests to make sure a password is ok. a password
+// must be at least 3 characters long but no longer than 8 characters
+// long. it can contain any character in the range 0x20-0x7e
+bool password_ok (const text_t &password);
+
+
+// information about a single user
+struct userinfo_t {
+  void clear ();
+  userinfo_t () {clear();}
+  userinfo_t &operator=(const userinfo_t &x);
+  
+  text_t username;
+  text_t password;
+  bool enabled;
+  text_t groups; // comma separated list
+  text_t comment;
+};
+
+
+// these functions deal with user and key databases that are already open
+// they should have been opened with gdbmclass::opendatabase
+// as soon as the databases are finished with they should be closed with
+// gdbmclass::closedatabase so that another process can access the database
+//
+// some of the functions need the databases opened with write access, an
+// appropriate call would be something like:
+//
+//   success = keydb.opendatabase (keyfile, GDBM_WRCREAT, 1000);
+//
+// note: passwords and keys are encrypted, so you can't just compare passwords
+// in a userinfo_t with a password given by the user. use check_passwd
+
+
+// functions dealing with user databases
+
+// returns true on success (in which case userinfo will contain
+// the information for this user)
+bool get_user_info (gdbmclass &userdb, const text_t &username,
+		    userinfo_t &userinfo);
+bool get_user_info (const text_t &userdbfile, const text_t &username,
+		    userinfo_t &userinfo);
+
+// returns true on success
+bool set_user_info (gdbmclass &userdb, const text_t &username,
+		    const userinfo_t &userinfo);
+bool set_user_info (const text_t &userdbfile, const text_t &username,
+		    const userinfo_t &userinfo);
+
+// removes a user from the user database -- forever
+void delete_user (gdbmclass &userdb, const text_t &username);
+void delete_user (const text_t &userdbfile, const text_t &username);
+
+// gets a list of all the users in the database. returns true
+// on success
+void get_user_list (gdbmclass &userdb, text_tarray &userlist);
+
+// returns true if the user's password is correct.
+bool check_passwd (const userinfo_t &thisuser, const text_t &password);
+
+
+// functions dealing with databases of temporary keys
+
+// generates a random key for the user, stores it in the database and
+// returns it so that it can be used in page generation
+// returns "" on failure
+text_t generate_key (gdbmclass &keydb, const text_t &username);
+text_t generate_key (const text_t &keydbfile, const text_t &username);
+
+// checks to see if there is a key for this particular user in the
+// database that hasn't decayed. a short decay is used when group
+// is set to administrator
+bool check_key (gdbmclass &keydb, const userinfo_t &thisuser,
+		const text_t &key, const text_t &group, int keydecay);
+bool check_key (const text_t &keydbfile, const userinfo_t &thisuser,
+		const text_t &key, const text_t &group, int keydecay);
+
+// remove_old_keys will remove all keys created more than keydecay ago.
+// use sparingly, it can be quite an expensive function
+void remove_old_keys (const text_t &keydbfile, int keydecay);
+
+
+#endif
Index: /tags/gsdl-21/gsdl/src/recpt/usersaction.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/usersaction.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/usersaction.cpp	(revision 699)
@@ -0,0 +1,448 @@
+/**********************************************************************
+ *
+ * usersaction.cpp -- managing users
+ * Copyright (C) 1999  DigiLib Systems Limited, New Zealand
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.6  1999/09/08 03:44:24  cs025
+   Added a cast to (char *) of (..:..?..) on output to avoid type
+   error at compile time.
+
+   Revision 1.5  1999/09/07 23:12:34  rjmcnab
+   removed some compiler warnings
+
+   Revision 1.4  1999/09/07 04:57:01  sjboddie
+   added GPL notice
+
+   Revision 1.3  1999/09/02 00:30:56  rjmcnab
+   added option for specifying whether the gdbm database should be locked
+
+   Revision 1.2  1999/07/30 02:24:44  sjboddie
+   added collectinfo argument to some functions
+
+   Revision 1.1  1999/07/13 23:22:05  rjmcnab
+   Initial revision.
+
+
+ */
+
+
+#include "usersaction.h"
+#include "fileutil.h"
+#include "userdb.h"
+
+
+///////////////
+// usersaction
+///////////////
+
+usersaction::usersaction () {
+  // this action uses cgi variable "a"
+  cgiarginfo arg_ainfo;
+  arg_ainfo.shortname = "a";
+  arg_ainfo.longname = "action";
+  arg_ainfo.multiplechar = true;
+  arg_ainfo.defaultstatus = cgiarginfo::weak;
+  arg_ainfo.argdefault = "um"; // user management
+  arg_ainfo.savedarginfo = cgiarginfo::must;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+
+  // "uma"
+  arg_ainfo.shortname = "uma";
+  arg_ainfo.longname = "user management action";
+  arg_ainfo.multiplechar = true;
+  arg_ainfo.defaultstatus = cgiarginfo::weak;
+  arg_ainfo.argdefault = "listusers";
+  arg_ainfo.savedarginfo = cgiarginfo::mustnot;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+
+  // "umun"
+  arg_ainfo.shortname = "umun";
+  arg_ainfo.longname = "user management user name";
+  arg_ainfo.multiplechar = true;
+  arg_ainfo.defaultstatus = cgiarginfo::weak;
+  arg_ainfo.argdefault = "";
+  arg_ainfo.savedarginfo = cgiarginfo::mustnot;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+
+  // "umpw"
+  arg_ainfo.shortname = "umpw";
+  arg_ainfo.longname = "user management password";
+  arg_ainfo.multiplechar = true;
+  arg_ainfo.defaultstatus = cgiarginfo::weak;
+  arg_ainfo.argdefault = "";
+  arg_ainfo.savedarginfo = cgiarginfo::mustnot;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+
+  // "umnpw1"
+  arg_ainfo.shortname = "umnpw1";
+  arg_ainfo.longname = "user management new password 1";
+  arg_ainfo.multiplechar = true;
+  arg_ainfo.defaultstatus = cgiarginfo::weak;
+  arg_ainfo.argdefault = "";
+  arg_ainfo.savedarginfo = cgiarginfo::mustnot;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+
+  // "umnpw2"
+  arg_ainfo.shortname = "umnpw2";
+  arg_ainfo.longname = "user management new password 2";
+  arg_ainfo.multiplechar = true;
+  arg_ainfo.defaultstatus = cgiarginfo::weak;
+  arg_ainfo.argdefault = "";
+  arg_ainfo.savedarginfo = cgiarginfo::mustnot;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+
+  // "umus"
+  arg_ainfo.shortname = "umus";
+  arg_ainfo.longname = "user management account status";
+  arg_ainfo.multiplechar = true;
+  arg_ainfo.defaultstatus = cgiarginfo::weak;
+  arg_ainfo.argdefault = "";
+  arg_ainfo.savedarginfo = cgiarginfo::mustnot;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+
+  // "umug"
+  arg_ainfo.shortname = "umug";
+  arg_ainfo.longname = "user management groups"; // comma seperated list
+  arg_ainfo.multiplechar = true;
+  arg_ainfo.defaultstatus = cgiarginfo::weak;
+  arg_ainfo.argdefault = "";
+  arg_ainfo.savedarginfo = cgiarginfo::mustnot;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+
+  // "umc"
+  arg_ainfo.shortname = "umc";
+  arg_ainfo.longname = "user management comment";
+  arg_ainfo.multiplechar = true;
+  arg_ainfo.defaultstatus = cgiarginfo::weak;
+  arg_ainfo.argdefault = "";
+  arg_ainfo.savedarginfo = cgiarginfo::mustnot;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+
+  // "bcp"
+  arg_ainfo.shortname = "bcp";
+  arg_ainfo.longname = "change password submit button";
+  arg_ainfo.multiplechar = true;
+  arg_ainfo.defaultstatus = cgiarginfo::weak;
+  arg_ainfo.argdefault = "";
+  arg_ainfo.savedarginfo = cgiarginfo::mustnot;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+
+  // "beu"
+  arg_ainfo.shortname = "beu";
+  arg_ainfo.longname = "edit user submit button";
+  arg_ainfo.multiplechar = true;
+  arg_ainfo.defaultstatus = cgiarginfo::weak;
+  arg_ainfo.argdefault = "";
+  arg_ainfo.savedarginfo = cgiarginfo::mustnot;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+
+  // "cm"
+  arg_ainfo.shortname = "cm";
+  arg_ainfo.longname = "confirm an action";
+  arg_ainfo.multiplechar = true;
+  arg_ainfo.defaultstatus = cgiarginfo::weak;
+  arg_ainfo.argdefault = "";
+  arg_ainfo.savedarginfo = cgiarginfo::mustnot;
+  argsinfo.addarginfo (NULL, arg_ainfo);
+}
+
+void usersaction::configure (const text_t &key, const text_tarray &cfgline) {
+  // get the password filename
+  if (cfgline.size() == 1) {
+    if (key == "usersfile") usersfile = cfgline[0];
+    else if (key == "gsdlhome") {
+      if (usersfile.empty())
+	usersfile = filename_cat (cfgline[0], "etc", "users.db");
+    }
+  }
+
+  action::configure (key, cfgline);
+}
+
+bool usersaction::check_cgiargs (cgiargsinfoclass &/*argsinfo*/, cgiargsclass &args, 
+				  ostream &/*logout*/) {
+  args["uan"] = "1"; // user authentication is needed
+
+  if (args["uma"] == "changepasswd") {
+    // no particular group is needed to change a password
+    args["ug"].clear(); 
+  } else {
+    // administrator is needed for all other management tasks
+    args["ug"] = "administrator";
+  }
+  
+  return true;
+}
+
+void usersaction::get_cgihead_info (cgiargsclass &/*args*/, response_t &response,
+				     text_t &response_data, ostream &/*logout*/) {
+  response = content;
+  response_data = "text/html";
+}
+
+void usersaction::define_internal_macros (const ColInfoResponse_t &/*collectinfo*/, displayclass &/*disp*/, 
+					  cgiargsclass &/*args*/, recptproto * /*collectproto*/, 
+					  ostream &/*logout*/) {
+}
+
+void usersaction::define_external_macros (const ColInfoResponse_t &/*collectinfo*/, displayclass &/*disp*/, 
+					  cgiargsclass &/*args*/, recptproto * /*collectproto*/, 
+					  ostream &/*logout*/) {
+}
+
+bool usersaction::do_action (cgiargsclass &args, const ColInfoResponse_t &/*collectinfo*/, 
+			     recptproto *collectproto, displayclass &disp, 
+			     outconvertclass &outconvert, ostream &textout, 
+			     ostream &logout) {
+  if (args["uma"] == "adduser" || args["uma"] == "edituser") {
+    // adduser is handled by edituser
+    return do_edituser (args, collectproto, disp, outconvert, textout, logout);
+    
+  } else if (args["uma"] == "deleteuser") {
+    return do_deleteuser (args, collectproto, disp, outconvert, textout, logout);
+    
+  } else if (args["uma"] == "changepasswd") {
+    return do_changepasswd (args, collectproto, disp, outconvert, textout, logout);  
+  }
+
+  // default
+  return do_listusers (args, collectproto, disp, outconvert, textout, logout);
+}
+
+bool usersaction::do_listusers (cgiargsclass &/*args*/, recptproto * /*collectproto*/,
+				displayclass &disp, outconvertclass &outconvert, 
+				ostream &textout, ostream &/*logout*/) {
+  textout << outconvert << disp
+	  << "_userslistusers:header_\n_userslistusers:contentstart_\n";
+
+
+  // open the user database (it will be used a lot)
+  gdbmclass userdb;
+  text_tarray userlist;
+  if (userdb.opendatabase(usersfile, GDBM_READER, 1000, true)) {
+    // get user list
+    get_user_list (userdb, userlist);
+  }
+
+  // sort the list
+  sort(userlist.begin(), userlist.end());
+  
+  // output the information for each user
+  userinfo_t userinfo;
+  text_tarray::iterator users_here = userlist.begin();
+  text_tarray::iterator users_end = userlist.end();
+  while (users_here != users_end) {
+    if (get_user_info (userdb, *users_here, userinfo)) {
+      textout << outconvert << disp
+	      << "<tr><td bgcolor=\"\\#eeeeee\">" << userinfo.username << "</td>\n"
+       	      << "<td bgcolor=\"\\#eeeeee\">" << (char *) (userinfo.enabled ? "enabled" : "disabled") << "</td>\n"
+	      << "<td bgcolor=\"\\#eeeeee\">" << userinfo.groups << "&nbsp;</td>\n"
+	      << "<td bgcolor=\"\\#eeeeee\">" << userinfo.comment << "&nbsp;</td>\n"
+	      << "<td><a href=\"_httpcurrentdocument_&a=um&uma=edituser&umun="
+	      << userinfo.username << "\">_userslistusers:textedituser_</a> "
+	      << "<a href=\"_httpcurrentdocument_&a=um&uma=deleteuser&umun="
+	      << userinfo.username << "\">_userslistusers:textdeleteuser_</a>"
+	      << "</td></tr>\n\n";
+
+    } else {
+      textout << outconvert << disp
+	      << "<tr><td bgcolor=\"\\#eeeeee\">" << *users_here << "</td>\n"
+	      << "<td bgcolor=\"\\#eeeeee\">&nbsp;</td>\n"
+	      << "<td bgcolor=\"\\#eeeeee\">&nbsp;</td>\n"
+	      << "<td bgcolor=\"\\#eeeeee\">&nbsp;</td>\n"
+	      << "<td>&nbsp;</td></tr>\n\n";
+    }
+    
+    users_here++;
+  }
+
+  userdb.closedatabase();
+  
+  textout << outconvert << disp
+	  << "_userslistusers:contentend_\n_userslistusers:footer_\n";
+  return true;
+}
+
+void usersaction::define_user_macros (cgiargsclass &args, displayclass &disp) {
+  disp.setmacro ("usersargun", "users", args["umun"]);
+  disp.setmacro ("usersargpw", "users", args["umpw"]);
+  disp.setmacro ("usersargus", "users", args["umus"]);
+  disp.setmacro ("usersargug", "users", args["umug"]);
+  disp.setmacro ("usersargc", "users", args["umc"]);
+}
+
+
+bool usersaction::do_edituser (cgiargsclass &args, recptproto *collectproto,
+			       displayclass &disp, outconvertclass &outconvert, 
+			       ostream &textout, ostream &logout) {
+  userinfo_t userinfo;
+  text_t messagestatus;
+  bool noproblems = true;
+  
+  // fill in defaults from user database (if appropriate)
+  if (args["umun"].empty()) {
+    noproblems = false;
+    
+  } else if (!username_ok (args["umun"])) {
+    // problem with username
+    noproblems = false;
+    messagestatus += "_users:messageinvalidusername_";
+    
+  } else if (get_user_info (usersfile, args["umun"], userinfo)) {
+    if (args["uma"] == "adduser") {
+      // must not add a user that has the same name as another user
+      noproblems = false;
+      messagestatus += "_users:messageuserexists_";
+      
+    } else {
+      // only fill in the data if there is no user status defined
+      if (args["umus"].empty()) {
+	args["umus"] = userinfo.enabled ? "enabled" : "disabled";
+	if (args["umug"].empty()) args["umug"] = userinfo.groups;
+	if (args["umc"].empty()) args["umc"] = userinfo.comment;
+      }
+    }
+  }
+
+  // fill in the user status default
+  if (args["umus"].empty()) {
+    noproblems = false;
+    args["umus"] = "enabled";
+  }
+  
+  // make sure the password is ok
+  if (args["umpw"].empty()) {
+    // password must not be empty if none were supplied from database
+    // and we have had no other problems
+    if (userinfo.password.empty() && noproblems) {
+      noproblems = false;
+      messagestatus += "_users:messageemptypassword_";
+    }
+    
+  } else if (!password_ok(args["umpw"])) {
+    noproblems = false;
+    messagestatus += "_users:messageinvalidpassword_";
+  }
+
+  // set this info if no problems have been encounted
+  // and the submit button was pressed
+  if (noproblems && !args["beu"].empty()) {
+    userinfo.username = args["umun"];
+    if (!args["umpw"].empty()) {
+      // only set the password if it is not empty
+      userinfo.password = crypt_text(args["umpw"]);
+    }
+    userinfo.enabled = (args["umus"] == "enabled");
+    userinfo.groups = args["umug"];
+    userinfo.comment = args["umc"];
+    
+    set_user_info (usersfile, args["umun"], userinfo);
+    
+    // show list of users
+    return do_listusers (args, collectproto, disp, outconvert, textout, logout);
+  }
+  
+  // define the macros for the user
+  define_user_macros (args, disp);
+  disp.setmacro ("messagestatus", "users", messagestatus);
+  
+  textout << outconvert << disp
+	  << "_usersedituser:header_\n_usersedituser:content_\n_usersedituser:footer_\n";
+
+  return true;
+}
+
+bool usersaction::do_deleteuser (cgiargsclass &args, recptproto *collectproto,
+				 displayclass &disp, outconvertclass &outconvert, 
+				 ostream &textout, ostream &logout) {
+  if (!args["cm"].empty()) {
+    if (args["cm"] == "yes" && !args["umun"].empty()) {
+      // user confirmed the deletion of the user
+      delete_user (usersfile, args["umun"]);
+    }
+    
+    // redirect the user back to the listusers page
+    return do_listusers (args, collectproto, disp, outconvert, textout, logout);
+  }
+
+  define_user_macros (args, disp);
+  textout << outconvert << disp
+	  << "_usersdeleteuser:header_\n_usersdeleteuser:content_\n_usersdeleteuser:footer_\n";
+  
+  return true;
+}
+
+bool usersaction::do_changepasswd (cgiargsclass &args, recptproto * /*collectproto*/,
+				   displayclass &disp, outconvertclass &outconvert, 
+				   ostream &textout, ostream &/*logout*/) {
+  text_t messagestatus;
+  
+  if (!args["bcp"].empty()) {
+    if (args["un"].empty()) {
+      messagestatus = "_users:messageusernameempty_";
+    } else if (args["umpw"].empty()) {
+      messagestatus = "_users:messagepasswordempty_";
+    } else if (args["umnpw1"].empty()) {
+      messagestatus = "_users:messagenewpass1empty_";
+    } else if (args["umnpw2"].empty()) {
+      messagestatus = "_users:messagenewpass2empty_";
+    } else if (args["umnpw1"] != args["umnpw2"]) {
+      messagestatus = "_users:messagenewpassmismatch_";
+    } else if (!password_ok(args["umnpw1"])) {
+      messagestatus = "_users:messageinvalidpassword_";
+    } else {
+      userinfo_t userinfo;
+      if (get_user_info (usersfile, args["un"], userinfo)) {
+	// check old password
+	if (userinfo.password != crypt_text(args["umpw"])) {
+	  messagestatus = "_users:messagefailed_";
+	  
+	} else {
+	  userinfo.password = crypt_text(args["umnpw1"]);
+	  if (set_user_info (usersfile, args["un"], userinfo)) {
+	    // everything is ok
+	    textout << outconvert << disp
+		    << "_userschangepasswdok:header_\n"
+	      "_userschangepasswdok:content_\n"
+	      "_userschangepasswdok:footer_\n";
+	    return true;
+	  }
+	}
+      }
+    }
+  }
+
+  disp.setmacro ("messagestatus", "users", messagestatus);
+
+  textout << outconvert << disp
+	  << "_userschangepasswd:header_\n"
+    "_userschangepasswd:content_\n"
+    "_userschangepasswd:footer_\n";
+
+  return true;
+}
+
Index: /tags/gsdl-21/gsdl/src/recpt/usersaction.h
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/usersaction.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/usersaction.h	(revision 699)
@@ -0,0 +1,85 @@
+/**********************************************************************
+ *
+ * usersaction.h -- managing users
+ * Copyright (C) 1999  DigiLib Systems Limited, New Zealand
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef USERSACTION_H
+#define USERSACTION_H
+
+#include "gsdlconf.h"
+#include "action.h"
+#include "text_t.h"
+
+
+class usersaction : public action {
+protected:
+  text_t usersfile;
+
+public:
+  usersaction ();
+  virtual ~usersaction () {}
+
+  void configure (const text_t &key, const text_tarray &cfgline);
+  
+  text_t get_action_name () {return "um";}
+  
+  bool check_cgiargs (cgiargsinfoclass &argsinfo, cgiargsclass &args, 
+		      ostream &logout);
+
+  void get_cgihead_info (cgiargsclass &args, response_t &response,
+			 text_t &response_data, ostream &logout);
+  
+  void define_internal_macros (const ColInfoResponse_t &collectinfo, displayclass &disp, 
+			       cgiargsclass &args, recptproto *collectproto, 
+			       ostream &logout);
+  void define_external_macros (const ColInfoResponse_t &collectinfo, displayclass &disp, 
+			       cgiargsclass &args, recptproto *collectproto, 
+			       ostream &logout);
+  
+  bool do_action (cgiargsclass &args, const ColInfoResponse_t &collectinfo, 
+		  recptproto *collectproto, displayclass &disp, 
+		  outconvertclass &outconvert, ostream &textout, 
+		  ostream &logout);
+
+protected:
+  void define_user_macros (cgiargsclass &args, displayclass &disp);
+  
+  bool do_listusers (cgiargsclass &args, recptproto *collectproto,
+		     displayclass &disp, outconvertclass &outconvert, 
+		     ostream &textout, ostream &logout);
+  bool do_edituser (cgiargsclass &args, recptproto *collectproto,
+		    displayclass &disp, outconvertclass &outconvert, 
+		    ostream &textout, ostream &logout);
+  bool do_deleteuser (cgiargsclass &args, recptproto *collectproto,
+		      displayclass &disp, outconvertclass &outconvert, 
+		      ostream &textout, ostream &logout);
+  bool do_changepasswd (cgiargsclass &args, recptproto *collectproto,
+			displayclass &disp, outconvertclass &outconvert, 
+			ostream &textout, ostream &logout);
+};
+
+
+#endif
Index: /tags/gsdl-21/gsdl/src/recpt/vlistbrowserclass.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/vlistbrowserclass.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/vlistbrowserclass.cpp	(revision 699)
@@ -0,0 +1,209 @@
+/**********************************************************************
+ *
+ * vlistbrowserclass.cpp -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+/*
+   $Log$
+   Revision 1.2  1999/10/14 22:59:34  sjboddie
+   finished off browser classes
+
+   Revision 1.1  1999/10/10 08:14:11  sjboddie
+   - metadata now returns mp rather than array
+   - redesigned browsing support (although it's not finished so
+   won't currently work ;-)
+
+ */
+
+
+#include "vlistbrowserclass.h"
+#include <assert.h>
+#include "OIDtools.h"
+
+vlistbrowserclass::vlistbrowserclass () {
+}
+
+vlistbrowserclass::~vlistbrowserclass () {
+}
+
+// returns the name that specifies the browserclass type
+text_t vlistbrowserclass::get_browser_name () {
+  return "VList";
+}
+
+void vlistbrowserclass::load_metadata_defaults (text_tset &metadata) {
+  metadata.insert ("haschildren");
+  metadata.insert ("doctype");
+}
+
+text_t vlistbrowserclass::get_default_formatstring () {
+  return "<td>[link][icon][/link]</td><td>[highlight]{Or}{[Title],Untitled}[/highlight]</td>";
+}
+
+int vlistbrowserclass::output_section_group (ResultDocInfo_t &section, cgiargsclass &args,
+					     int colnumber, format_t *formatlistptr, 
+					     bool use_table, text_tset &/*metadata*/, bool &/*getParents*/,
+					     recptproto */*collectproto*/, displayclass &disp, 
+					     outconvertclass &outconvert, ostream &textout, ostream &/*logout*/) {
+
+  text_t link, icon;
+  get_link_icon (section, args, link, icon);
+  bool highlight = false;
+  if (!args["d"].empty()) {
+    if (args["d"] == section.OID) highlight = true;
+  } else 
+    if (args["cl"] == section.OID) highlight = true;
+
+
+  if (use_table || colnumber > 0) {
+    textout << "<table><tr>";
+    // get tab size
+    text_t tab; int itab;
+    disp.expandstring ("Global", "_tabwidth_", tab);
+    itab = tab.getint();
+    if (colnumber > 0) textout << outconvert << disp 
+			       << "<td><img src=\"_httpimg_/space.gif\" width=" 
+			       << (itab*colnumber) << "></td>";
+  }
+
+  textout << outconvert << disp << get_formatted_string (section, formatlistptr, 
+							 link, icon, highlight);
+  
+  if (use_table  || colnumber > 0) textout << "</tr></table>\n";
+
+  return 1;
+}
+
+int vlistbrowserclass::output_section_group (FilterResponse_t &sections, cgiargsclass &args,
+					     int colnumber, format_t *formatlistptr, 
+					     bool use_table, text_tset &/*metadata*/, bool &/*getParents*/,
+					     recptproto */*collectproto*/, displayclass &disp, 
+					     outconvertclass &outconvert, ostream &textout, ostream &/*logout*/) {
+
+  text_t link, icon;
+  text_t &arg_d = args["d"];
+  text_t &arg_cl = args["cl"];
+
+  if (colnumber > 0) {
+    textout << "<table><tr>";
+    // get tab size
+    text_t tab; int itab;
+    disp.expandstring ("Global", "_tabwidth_", tab);
+    itab = tab.getint();
+    if (colnumber > 0) textout << outconvert << disp 
+			       << "<td><img src=\"_httpimg_/space.gif\" width=" 
+			       << (itab*colnumber) << "></td>";
+    textout << "<td>";
+  }
+  if (use_table) textout << "<table>\n";
+
+  ResultDocInfo_tarray::iterator thissection = sections.docInfo.begin();
+  ResultDocInfo_tarray::iterator endsection = sections.docInfo.end();
+
+  while (thissection != endsection) {
+
+    get_link_icon (*thissection, args, link, icon);
+    bool highlight = false;
+    if (!arg_d.empty()) {
+      if (arg_d == (*thissection).OID) highlight = true;
+    } else 
+      if (arg_cl == (*thissection).OID) highlight = true;
+
+    if (use_table || colnumber > 0) textout << "<tr>";
+    textout << outconvert << disp << get_formatted_string (*thissection, formatlistptr, 
+							   link, icon, highlight);
+    if (use_table || colnumber > 0) textout << "</tr>";
+    textout << "\n";
+    thissection ++;
+  }
+
+  if (use_table) textout << "</table>\n";
+  if (colnumber > 0) textout << "</table></td></tr></table>\n";
+
+  return 1;
+}
+
+
+// get_link_icon attempts to work out what type of icon should be displayed for
+// the given section and what it should link to.
+void vlistbrowserclass::get_link_icon (ResultDocInfo_t &section, cgiargsclass &args,
+				       text_t &link, text_t &icon) {
+
+  link = "<a href=\"_httpdocument_";
+  icon = "_document:icontext_";
+  int haschildren = section.metadata["haschildren"].values[0].getint();
+  const text_t doctype = section.metadata["doctype"].values[0];
+
+  text_t &arg_d = args["d"];
+  text_t &arg_cl = args["cl"];
+
+  if (args["a"] == "q") {
+    // query results
+    if (is_top (section.OID)) icon = "_document:iconclosedbook_";
+    else if (haschildren == 1) icon = "_document:iconclosedfolder_";
+    link += "&cl=search&d=" + section.OID + "\">";
+
+  } else if (!arg_d.empty()) {
+    // document level
+    if (is_top(section.OID)) {
+      icon = "_document:iconopenbook_";
+      if (arg_cl == "search") link = "<a href=\"_httpquery_\">";
+      else link += "&cl=" + arg_cl + "\">";
+    
+    } else if (haschildren == 1) {
+      if ((args["gc"] == "1") || 
+	  (is_child_of (section.OID, arg_d)) ||
+	  (section.OID == arg_d)) {
+	icon = "_document:iconopenfolder_";
+	link += "&cl=" + arg_cl + "&d=" + section.OID + ".pr\">";
+      } else {
+	icon = "_document:iconclosedfolder_";
+	link += "&cl=" + arg_cl + "&d=" + section.OID + "\">";
+      }
+    } else link += "&cl=" + arg_cl + "&d=" + section.OID + "\">";
+
+  } else {
+    // classification level
+    if (haschildren == 1) {
+      if (doctype == "classify") {
+	if ((args["gc"] == "1") || 
+	    (is_child_of (section.OID, arg_cl)) || 
+	    (section.OID == arg_cl)) {
+	  icon = "_document:iconopenbookshelf_";
+	  link += "&cl=" + section.OID + ".pr\">";
+	} else {
+	  icon = "_document:iconclosedbookshelf_";
+	  link += "&cl=" + section.OID + "\">";
+	}
+      } else {
+	icon = "_document:iconclosedbook_";
+	link += "&cl=" + arg_cl + "&d=" + section.OID + "\">";
+      }
+    } else {
+      if (doctype == "classify") link = "";
+      else link += "&cl=" + arg_cl + "&d=" + section.OID + "\">";
+    }
+  }
+}
Index: /tags/gsdl-21/gsdl/src/recpt/vlistbrowserclass.h
===================================================================
--- /tags/gsdl-21/gsdl/src/recpt/vlistbrowserclass.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/recpt/vlistbrowserclass.h	(revision 699)
@@ -0,0 +1,65 @@
+/**********************************************************************
+ *
+ * vlistbrowserclass.h -- 
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ *********************************************************************/
+
+
+#ifndef VLISTBROWSERCLASS_H
+#define VLISTBROWSERCLASS_H
+
+#include "browserclass.h"
+
+class vlistbrowserclass : public browserclass {
+protected:
+
+public:
+  vlistbrowserclass ();
+  virtual ~vlistbrowserclass ();
+
+  virtual text_t get_browser_name ();
+
+  void load_metadata_defaults (text_tset &metadata);
+
+  text_t get_default_formatstring ();
+
+  int output_section_group (ResultDocInfo_t &section, cgiargsclass &args,
+			    int colnumber, format_t *formatlistptr, 
+			    bool use_table, text_tset &metadata, bool &getParents,
+			    recptproto *collectproto, displayclass &disp, 
+			    outconvertclass &outconvert, ostream &textout, ostream &logout);
+
+  int output_section_group (FilterResponse_t &sections, cgiargsclass &args,
+			    int colnumber, format_t *formatlistptr, 
+			    bool use_table, text_tset &metadata, bool &getParents,
+			    recptproto *collectproto, displayclass &disp, 
+			    outconvertclass &outconvert, ostream &textout, ostream &logout);
+
+protected:
+
+  void get_link_icon (ResultDocInfo_t &section, cgiargsclass &args,
+		      text_t &link, text_t &icon);
+};
+
+#endif
Index: /tags/gsdl-21/gsdl/src/txt2db/Makefile.in
===================================================================
--- /tags/gsdl-21/gsdl/src/txt2db/Makefile.in	(revision 699)
+++ /tags/gsdl-21/gsdl/src/txt2db/Makefile.in	(revision 699)
@@ -0,0 +1,85 @@
+###########################################################################
+#
+# Makefile -- Makefile for txt2db
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 Rodger McNab, Stefan Boddie
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+#       @(#)Makefile.in	1.3 16 Mar 1994
+#
+###########################################################################
+
+ 
+AR = ar
+CC = @CC@
+CCFLAGS = -Wall @CFLAGS@
+DEFS = @DEFS@
+RANLIB = @RANLIB@
+INCLUDES = -I../../lib
+LDFLAGS = @LDFLAGS@ 
+DPLIBS = ../../lib/gsdllib.a ../../packages/mg/lib/unitool.o
+LIBS = ../../lib/gsdllib.a ../../packages/mg/lib/unitool.o -lstdc++ -lgdbm
+GSDLOS = @gsdlos@
+
+ 
+COMPILE = $(CC) -c $(DEFS) $(INCLUDES) $(CCFLAGS)
+LINK = $(CC) $(LDFLAGS) -o $@
+ 
+.SUFFIXES:
+.SUFFIXES: .c .o
+.c.o:
+	$(COMPILE) $<
+
+.SUFFIXES: .cpp .o
+.cpp.o:
+	$(COMPILE) $< 
+ 
+
+HEADERS = 
+
+SOURCES = txt2db.cpp
+
+OBJECTS = txt2db.o
+
+EXEC    = txt2db
+ 
+all : $(EXEC)
+
+install: $(EXEC)
+	install -m 755 -s $(EXEC) ../../bin/$(GSDLOS)
+
+clean:
+	rm -f $(OBJECTS) $(EXEC)
+
+depend:
+	makedepend -Y -- $(DEFS) $(INCLUDES) $(CCFLAGS) -- $(SOURCES)
+
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+TXT2DB_OBJS = txt2db.o
+
+txt2db: $(TXT2DB_OBJS) $(DPLIBS)
+	$(LINK) $(TXT2DB_OBJS) $(LIBS)
+
+
+
+
+
+
Index: /tags/gsdl-21/gsdl/src/txt2db/txt2db.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/txt2db/txt2db.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/txt2db/txt2db.cpp	(revision 699)
@@ -0,0 +1,167 @@
+/**********************************************************************
+ *
+ * txt2db.cpp -- 
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the 
+ * University of Waikato, New Zealand.
+ *
+ * Copyright (C) 1999  The New Zealand Digital Library Project
+ * 
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ **********************************************************************/
+
+
+#ifdef __WIN32__
+#include "autoconf.h"
+#include "systems.h"
+#include "gdbmconst.h"
+#include "gdbm.h"
+
+#else
+#include <gdbm.h>
+#endif
+
+#include "gsdlconf.h"
+#include "text_t.h"
+#include <stdlib.h>
+
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\iostream>
+#elif defined(GSDL_USE_IOS_H)
+#  include <iostream.h>
+#else
+#  include <iostream>
+#endif
+
+
+void print_usage (char *program_name) {
+  cerr << "usage: " << program_name << " [options] database-name\n\n";
+  cerr << "options:\n";
+  cerr << " -append        append to existing database\n\n";
+}
+
+
+int main (int argc, char *argv[]) {
+  int block_size = 0;
+  GDBM_FILE dbf;
+  char c;
+  text_t key;
+  text_t value;
+  text_t tmp;
+  int num_dashes = 0;
+
+  // sanity check
+  if (argc != 2 && argc != 3) {
+    print_usage (argv[0]);
+    exit (0);
+  }
+  
+  char *dbname;
+  int append = 0;
+  if (argc == 3) {
+    if (strcmp (argv[1], "-append") == 0) {
+      append = 1;
+      dbname = argv[2];
+    } else {
+      cerr << argv[1] << " is not a valid option.\n\n";
+      print_usage (argv[0]);
+      exit (0);
+    }
+  } else dbname = argv[1];
+
+  
+  // open the database
+  // note that GDBM_FAST is obsolete on newer versions of gdbm
+  int read_write = GDBM_NEWDB | GDBM_FAST;
+  if (append) read_write = GDBM_WRCREAT | GDBM_FAST;
+  
+  dbf = gdbm_open  (dbname, block_size, read_write, 00664, NULL);
+  if (dbf == NULL) {
+    cerr << "couldn't create " << dbname << "\n";
+    exit (0);
+  }
+  
+  cin.get(c);
+  while (!cin.eof()) {
+    num_dashes = 0;
+    key = "";
+    value = "";
+    
+    // look for [key]\n
+    while (!cin.eof() && c != '[') cin.get(c);
+    if (!cin.eof()) cin.get(c); // skip [
+    while (!cin.eof() && c != ']') {
+      key.push_back ((unsigned char)c);
+      cin.get(c);
+    }
+    if (!cin.eof()) cin.get(c); // skip ]
+    while (!cin.eof() && (c == '\n' || c == 'r')) cin.get(c);
+    
+    // look for 70 dashes
+    tmp = "";
+    while (!cin.eof() && (num_dashes < 70)) {
+      if (c == '\n' || c == '\r') {
+	tmp.push_back ((unsigned char)c);
+	num_dashes = 0;
+	
+      } else if (c == '-') {
+	tmp.push_back ((unsigned char)c);
+	num_dashes++;
+	
+      } else {
+	value += tmp;
+	value.push_back ((unsigned char)c);
+	tmp = "";
+	num_dashes = 0;
+      }
+      cin.get(c);
+    }
+    
+    // if the key is not an empty string store this key-value pair
+    if (!key.empty()) {
+      // convert key to a datum datatype
+      datum key_data;
+      key_data.dptr = key.getcstr();
+      if (key_data.dptr == NULL) {
+	cerr << "NULL key_data.dptr\n";
+	exit (0);
+      }
+      key_data.dsize = strlen(key_data.dptr);
+      
+      // convert value to a datum datatype
+      datum value_data;
+      value_data.dptr = value.getcstr();
+      if (value_data.dptr == NULL) {
+	cerr << "NULL value_data.dptr\n";
+	exit (0);
+      }
+      value_data.dsize = strlen(value_data.dptr);
+      
+      // store the value
+      if (gdbm_store (dbf, key_data, value_data, GDBM_REPLACE) < 0) {
+	cerr << "gdbm_store returned an error\n";
+	exit (0);
+      }
+      
+      delete key_data.dptr;
+      delete value_data.dptr;
+    }
+  }
+  
+  gdbm_close (dbf);
+  
+  return 0;
+}
Index: /tags/gsdl-21/gsdl/src/w32server/cgiwrapper.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/w32server/cgiwrapper.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/w32server/cgiwrapper.cpp	(revision 699)
@@ -0,0 +1,672 @@
+#include <windows.h>
+#include <string.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <direct.h>
+#include "cgiwrapper.h"
+#include "netio.h"
+#include "wincgiutils.h"
+#include "settings.h"
+
+#include "gsdlconf.h"
+#include "gsdlhome.h"
+#include "recptconfig.h"
+
+#if defined (GSDL_USE_OBJECTSPACE)
+#include <ospace\std\iostream>
+#include <ospace\std\fstream>
+#elif defined (GSDL_USE_IOS_H)
+#include <iostream.h>
+#include <fstream.h>
+#else
+#include <iostream>
+#include <fstream>
+#endif
+
+#include "receptionist.h"
+#include "nullproto.h"
+#include "collectserver.h"
+#include "infodbclass.h"
+#include "mggdbmsource.h"
+
+
+// actions
+#include "pageaction.h"
+#include "pingaction.h"
+#include "queryaction.h"
+#include "documentaction.h"
+#include "statusaction.h"
+#include "usersaction.h"
+#include "authenaction.h"
+#include "tipaction.h"
+
+// filters
+#include "filter.h"
+#include "browsefilter.h"
+#include "queryfilter.h"
+#include "phrasequeryfilter.h"
+
+// the number of times the library has been accessed
+int libaccessnum = 0;
+
+// used to output the text from receptionist
+class textstreambuf : public streambuf
+{
+public:
+  textstreambuf ();
+  int sync ();
+  int overflow (int ch);
+  int underflow () {return EOF;}
+  
+  void tsbreset() {RInfo=NULL;casostr=NULL;}
+  void setrequestinfo (RequestInfoT *theRInfo) {RInfo=theRInfo;}
+  void cascadeoutput (ostream *thecasostr) {casostr=thecasostr;}
+  
+private:
+  RequestInfoT *RInfo;
+  ostream *casostr;
+};
+
+textstreambuf::textstreambuf() {
+  tsbreset();
+  if (base() == ebuf()) allocate();
+  setp (base(), ebuf());
+};
+
+int textstreambuf::sync () {
+  if ((RInfo != NULL) && 
+      (Send_String_N(pbase(), out_waiting(), RInfo) < 0)) {
+    RInfo = NULL;
+  }
+  
+  if (casostr != NULL) {
+    char *thepbase=pbase();
+		for (int i=0;i<out_waiting();i++) (*casostr).put(thepbase[i]);
+  }
+  
+  setp (pbase(), epptr());
+  
+  return 0;
+}
+
+int textstreambuf::overflow (int ch) {
+  if (sync () == EOF) return EOF;
+  if (ch != EOF) sputc (ch);
+  return 0;
+}
+
+
+// used to output all the log and error messages
+// from receptionist
+class logstreambuf : public streambuf
+{
+public:
+  logstreambuf ();
+  int sync ();
+  int overflow (int ch);
+  int underflow () {return EOF;}
+};
+
+logstreambuf::logstreambuf () {
+  if (base() == ebuf()) allocate();
+  setp (base(), ebuf());
+}
+
+int logstreambuf::sync () {
+  if (gs_keep_log || gs_show_console) {
+    log_message ("LOCAL LIB MESSAGE: ");
+		log_message_N (pbase(), out_waiting());
+  }
+
+  setp (pbase(), epptr());
+  return 0;
+}
+
+int logstreambuf::overflow (int ch) {
+  if (sync () == EOF) return EOF;
+  if (ch != EOF) sputc (ch);
+  return 0;
+}
+
+
+
+#ifndef MAX_FILENAME_SIZE
+#define MAX_FILENAME_SIZE 2048
+#endif
+
+
+receptionist recpt;
+nullproto nproto;
+textstreambuf textstream;
+logstreambuf logstream;
+DWORD lastlibaccesstime;
+DWORD baseavailvirtual;
+
+
+static void page_errorsitecfg (const text_t &gsdlhome, const text_t &collection) {
+	
+  text_t message = "Error\n\n"
+    "The site.cfg configuration file could not be found. This file\n"
+    "should contain configuration information relating to this sites\n"
+    "setup.\n";
+  
+  if (collection.empty()) {
+    message += "As this program is not being run in collection specific mode,\n"
+      "the file should reside at " + gsdlhome + "\\etc\\site.cfg.\n";
+  } else {
+    message += "As this program is being run in collection specific mode,\n"
+      "the file can reside at " + gsdlhome + "\\collect\\" + collection +
+      "\\etc\\site.cfg or " + gsdlhome + "\\etc\\site.cfg.\n";
+  }
+
+  MessageBox(NULL, message.getcstr(),
+	     "Greenstone Digital Library Software"
+	     ,MB_OK|MB_SYSTEMMODAL);
+}
+
+static void page_errormaincfg (const text_t &gsdlhome, const text_t &collection) {
+
+  if (collection.empty()) {
+    text_t message = "Error\n\n"
+      "The main.cfg configuration file could not be found. This file\n"
+      "should contain configuration information relating to the\n"
+      "setup of the interface. As this program is not being run\n"
+      "in collection specific mode the file should reside at\n" +
+      gsdlhome + "\\etc\\main.cfg.\n";
+
+    MessageBox(NULL, message.getcstr(),
+	       "Greenstone Digital Library Software"
+	       ,MB_OK|MB_SYSTEMMODAL);
+  } else {
+    text_t message = "Neither the collect.cfg or main.cfg configuration files could\n"
+      "be found. This file should contain configuration information\n"
+      "relating to the setup of the interface. As this cgi script is\n"
+      "being run in collection specific mode the file should reside\n"
+      "at either " + gsdlhome + "\\collect\\" + collection + "\\etc\\collect.cfg,\n" +
+      gsdlhome + "\\etc\\collect.cfg or " + gsdlhome + "\\etc\\main.cfg.\n";
+
+    MessageBox(NULL, message.getcstr(),
+	       "Greenstone Digital Library Software"
+	       ,MB_OK|MB_SYSTEMMODAL);
+  }
+}
+
+static void page_errorinit (const text_t &/*gsdlhome*/) {
+
+  text_t message = "Error\n\n"
+    "An error occurred during the initialisation of the Greenstone Digital\n"
+    "Library software. It is likely that the software has not been setup\n"
+    "correctly.\n";
+
+  MessageBox(NULL, message.getcstr(),
+	     "Greenstone Digital Library Software"
+	     ,MB_OK|MB_SYSTEMMODAL);
+}
+
+static void page_errorparseargs (const text_t &gsdlhome) {
+
+  text_t message = "Error\n\n"
+    "An error occurred during the parsing of the cgi arguments.\n";
+
+  MessageBox(NULL, message.getcstr(),
+	     "Greenstone Digital Library Software"
+	     ,MB_OK|MB_SYSTEMMODAL);
+}
+
+static void page_errorcgipage (const text_t &gsdlhome) {
+
+  text_t message = "Error\n\n"
+    "An error occurred during the construction of the cgi page.\n";
+
+  MessageBox(NULL, message.getcstr(),
+	     "Greenstone Digital Library Software"
+	     ,MB_OK|MB_SYSTEMMODAL);
+}
+
+// returns 0 if the directories can't be found
+// and the user wants to quit (it returns 1
+// if everything is ok)
+int checkdir (const text_t &thedir) {
+  UINT curerrormode;
+  int drive = _getdrive();
+  char cwd[1024];
+  char rootpath[4];
+  UINT drivetype;
+  char *cstrthedir = thedir.getcstr();
+  int returnvalue = 1;
+  
+  // make sure no rude error messages are presented to the user
+  curerrormode = SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOOPENFILEERRORBOX);
+  
+  // get the drive name
+  if (thedir.size() >= 2 && thedir[1] == ':') {
+    if (thedir[0] >= 'a' && thedir[0] <= 'z') {
+      drive = thedir[0] - 'a' + 1;
+      
+    } else if (thedir[0] >= 'A' && thedir[0] <= 'Z') {
+      drive = thedir[0] - 'A' + 1;
+    }
+  }
+  
+  // find the drive type
+  rootpath[0] = drive + 'A' - 1;
+  rootpath[1] = ':';
+  rootpath[2] = '\\';
+  rootpath[3] = '\0';
+  drivetype = GetDriveType (rootpath);
+  
+  // try and set this directory to be the current working
+  // directory
+  _getcwd(cwd, 1024);
+  while (_chdir(cstrthedir) != 0) {
+    // failed
+    if (drivetype == DRIVE_CDROM) {
+      // message to insert the cdrom
+      if (MessageBox (NULL, 
+		      "Please insert the Greenstone Digital Library\n"
+		      "CD-ROM into the CD-ROM drive and press 'OK'.\n\n"
+		      "If you don't have the CD-ROM press 'Cancel' to exit\n"
+		      "this program.", "Greenstone Digital Library Software", 
+		      MB_OKCANCEL | MB_TASKMODAL) == IDCANCEL) {
+	
+	returnvalue = 0;
+	break;
+      }
+      
+    } else {
+      // message saying that the system was unable
+      // to find a certain directory
+      text_t message = "Failed to find the directory:\n\n" + thedir;
+      message += "\n\n"
+	"This directory is needed for the successful operation\n"
+	"of this software. Make sure it hasn't been deleted or\n"
+	"moved, and restart the software. You may need to\n"
+	"reinstall this software to correct the problem.";
+      char *cstrmessage = message.getcstr();
+      
+      MessageBox (NULL, cstrmessage, "Greenstone Digital Library Software",
+		  MB_OK | MB_TASKMODAL);
+      
+      delete cstrmessage;
+      
+      returnvalue = 0;
+      break;
+    }
+  }
+  
+  // revert to the previous error and cwd states
+  _chdir(cwd);
+  SetErrorMode(curerrormode);
+  
+  // free the allocated C string
+  delete cstrthedir;
+  
+  return returnvalue;
+}
+
+
+// c-string version of checkdir for the outside
+// world
+int cstrcheckdir (char *cstrthedir) {
+  return checkdir (cstrthedir);
+}
+
+
+// returns 1 if successful, 0 if unsuccessful
+int gsdl_init () {
+  cerr = &logstream;
+  cout = &textstream;
+  
+  // collection should be set to "" unless in 
+  // collection specific mode
+  text_t collection = "";
+
+  // note the current time
+  lastlibaccesstime = GetTickCount();
+  
+  // before we do the init we should make sure
+  // that we can find the relevant directories
+  if (!checkdir (GSDL_GSDLHOME "\\")) return 0;
+  if (!checkdir (GSDL_GSDLHOME "\\macros\\")) return 0;
+  
+  // add a collection server for each collection
+  text_tarray collections;
+  ifstream collist (GSDL_GSDLHOME "/etc/collections.txt");
+  if (!collist) {
+    exit(1);
+  }
+  char col[100];
+  while (!collist.eof()) {
+    collist.getline (col, 100);
+    if (col[0] != '\0')
+      collections.push_back (col);
+  }
+  collist.close();
+
+  text_tarray::const_iterator thiscol = collections.begin();
+  text_tarray::const_iterator endcol = collections.end();
+  
+  while (thiscol != endcol) {
+    
+    // this memory is created but never destroyed
+    // we're also not doing any error checking to make sure we didn't
+    // run out of memory
+    collectserver *cserver = new collectserver();
+    gdbmclass *gdbmhandler = new gdbmclass();
+    mgsearchclass *mgsearch = new mgsearchclass();
+    
+    // add a null filter
+    filterclass *filter = new filterclass();
+    cserver->add_filter (filter);
+    
+    // add a browse filter
+    browsefilterclass *browsefilter = new browsefilterclass();
+    browsefilter->set_gdbmptr (gdbmhandler);
+    cserver->add_filter (browsefilter);
+    
+    // add a query filter
+    queryfilterclass *queryfilter = new queryfilterclass();
+    queryfilter->set_gdbmptr (gdbmhandler);
+    queryfilter->set_mgsearchptr (mgsearch);
+    cserver->add_filter (queryfilter);
+    
+    // add an mg and gdbm source
+    mggdbmsourceclass *mggdbmsource = new mggdbmsourceclass();
+    mggdbmsource->set_gdbmptr (gdbmhandler);
+    mggdbmsource->set_mgsearchptr (mgsearch);
+    cserver->add_source (mggdbmsource);
+    
+    // inform collection server and everything it contains about
+    // its collection name
+    cserver->configure ("collection", *thiscol);
+    
+    nproto.add_collectserver (cserver);
+    
+    thiscol ++;
+  }
+  
+  // add the protocol to the receptionist
+  recpt.add_protocol (&nproto);
+
+  // add other converters
+  utf8inconvertclass *utf8inconvert = new utf8inconvertclass();
+  utf8outconvertclass *utf8outconvert = new utf8outconvertclass();
+  recpt.add_converter ("u", utf8inconvert, utf8outconvert);
+  
+  mapinconvertclass *gbinconvert = new mapinconvertclass();
+  gbinconvert->setmapfile (GSDL_GSDLHOME, "gbku", 0x25a1);
+  mapoutconvertclass *gboutconvert = new mapoutconvertclass();
+  gboutconvert->setmapfile (GSDL_GSDLHOME, "ugbk", 0xa1f5);
+  recpt.add_converter ("g", gbinconvert, gboutconvert);
+  
+  // the list of actions.
+  statusaction *astatusaction = new statusaction();
+  astatusaction->set_receptionist (&recpt);
+  recpt.add_action (astatusaction);
+  
+  pageaction *apageaction = new pageaction();
+  apageaction->set_receptionist (&recpt);
+  recpt.add_action (apageaction);
+  
+  pingaction *apingaction = new pingaction();
+  recpt.add_action (apingaction);
+  
+  queryaction *aqueryaction = new queryaction();
+  recpt.add_action (aqueryaction);
+  
+  documentaction *adocumentaction = new documentaction();
+  recpt.add_action (adocumentaction);
+  
+  usersaction *ausersaction = new usersaction();
+  recpt.add_action (ausersaction);
+  
+  authenaction *aauthenaction = new authenaction();
+  aauthenaction->set_receptionist(&recpt);
+  recpt.add_action (aauthenaction);
+  
+  tipaction *atipaction = new tipaction();
+  recpt.add_action (atipaction);
+  
+  // set defaults
+  recpt.configure ("gsdlhome", GSDL_GSDLHOME);
+  recpt.configure ("collection", collection);
+
+  int maxrequests = 1;
+  if (!site_cfg_read (recpt, GSDL_GSDLHOME, collection, maxrequests)) {
+    // couldn't find the site configuration file
+    page_errorsitecfg (GSDL_GSDLHOME, collection);
+    return 0;
+  } else if (!main_cfg_read (recpt, GSDL_GSDLHOME, collection)) {
+    // couldn't find the main configuration file
+    page_errormaincfg (GSDL_GSDLHOME, collection);
+    return 0;
+  }
+
+  // w32server relies on gwcgi being set to "gw"
+  recpt.configure ("gwcgi", "gw");
+	
+  // initialise the library software
+  if (!recpt.init(cerr)) {
+    // an error occurred during the initialisation
+    page_errorinit(GSDL_GSDLHOME);
+    return 0;
+  }
+
+  // get memory information
+  MEMORYSTATUS memstatus;
+  memstatus.dwLength = sizeof(MEMORYSTATUS);
+  GlobalMemoryStatus(&memstatus);
+  baseavailvirtual = memstatus.dwAvailVirtual; // save for later comparison
+	
+  return 1;
+}
+
+
+static void rememberpref (char *tailstr) {
+  strcpy(gs_enterlib, tailstr);
+
+  // add "a=p&p=home"
+  strcat(gs_enterlib, "&a=p&p=home");
+}
+
+
+static void send_file_from_disk(char *filename,
+                                RequestInfoT *RInfo,
+                                RequestFieldsT *RFields)
+{
+  int len, nr;  char *tail, *kind;
+  FILE *thefile;  
+  
+  // select appropriate mime type from file name
+  len = strlen(filename);
+  while (len > 0 && filename[len-1] != '.') len--;
+  kind = "unknown";
+  if (len > 0) {
+    tail = &filename[len];
+    if (stricmp("gif",tail) == 0) kind = "image/gif";
+    else if (stricmp("jpg",tail) == 0 ||
+             stricmp("jpeg",tail) == 0) kind = "image/jpeg";
+    else if (stricmp("htm",tail) == 0 ||
+             stricmp("html",tail) == 0) kind = "text/html";
+  }
+
+  // set up file name
+  text_t filenamet = text_t(GSDL_GSDLHOME) + filename;
+  text_t::iterator here = filenamet.begin();
+  text_t::iterator end = filenamet.end();
+  while (here != end) {
+	  if (*here == '/') *here = '\\';
+	  here ++;
+  }
+
+  filename = filenamet.getcstr();
+	
+  // try to open it
+  thefile = fopen(filename, "rb");
+  if (thefile == NULL) {
+    log_message("file not found\n");
+    send_retrieve_error(404, "File not found",
+			"Could not find the local file requested", RInfo);
+    return;
+  }
+  
+  char buffer[2048];
+  // send back required information
+  if (send_header(kind, RInfo) >= 0) {
+    if (strcmpi(RFields->MethodStr, "HEAD") != 0) {
+      for (;;) {
+        nr = fread(buffer, 1, 2048, thefile);
+        if (nr <= 0) break;
+        if (SendData(RInfo->ClientSocket,
+                     (BYTE *)buffer, nr,
+		     RInfo->ThreadNum) < 0) break;        
+      }
+    }
+  }
+  fclose(thefile);
+}
+
+static void handle_library_request(char *TailStr, RequestInfoT *RInfo,
+				   RequestFieldsT *RequestFields) {
+  // check for a "?" at the start of the tail string
+  if (*TailStr == '?') TailStr++;
+
+  outconvertclass text_t2ascii;
+  
+  // parse the cgi arguments and produce the resulting page if there
+  // has been no errors so far
+  cgiargsclass args;
+  if (!recpt.parse_cgi_args (TailStr, args, cerr)) {
+    page_errorparseargs(GSDL_GSDLHOME);
+    return;
+  } 
+  
+  // produce cgi header
+  response_t response;
+  text_t response_data;
+
+  recpt.get_cgihead_info (args, response, response_data, cerr);
+  char *response_data_c = response_data.getcstr();
+
+  if (response == location) {
+    // location response
+    if (send_header(strcat("@", response_data_c), RInfo) < 0) return;
+    // textstream << text_t2ascii << "Location: " << response_data << "\n\n";
+  } else if (response == content) {
+    // content response
+    if (send_header(response_data_c, RInfo) < 0) return;
+  } else {
+    // unknown response
+    cerr << "Error: get_cgihead_info returned an unknown response type.\n";
+    return;
+  }
+
+  textstream.tsbreset();
+  textstream.setrequestinfo (RInfo);
+  delete response_data_c;
+  
+  if (!recpt.produce_content (args, cout, cerr)) {
+    page_errorcgipage(GSDL_GSDLHOME);
+    return;
+  } 
+  recpt.log_cgi_args (args, cerr);
+
+  cout << flush;
+  cerr << flush;
+  
+  libaccessnum++;
+}
+
+static void handle_server_request(char *initialTailStr, 
+				  RequestInfoT *RequestInfo,
+				  RequestFieldsT *RequestFields) {
+  char tmpstr[1024];
+  char tailstr[MAX_URL_SIZE];
+  
+  // do any url adjustments necessary
+  if (strcmp(initialTailStr, "/") == 0) {
+    strcpy (tailstr, "/cgi-bin/gw");
+  } else strcpy (tailstr, initialTailStr);
+  
+  // test to see if this is a library request or a local
+  // file request
+  if ((strncmp(tailstr, "/cgi-bin/gw", 11) == 0) &&
+      ((tailstr[11] == '\0') || (tailstr[11] == '?'))) {
+    // library request
+    
+    // log the difference in access times
+    DWORD thislibaccesstime = GetTickCount();
+    if (gs_keep_log||gs_show_console) {
+      sprintf(tmpstr, "DELTA LIB ACCESS TIME: %i\n", (int)(thislibaccesstime - lastlibaccesstime));
+      log_message (tmpstr);
+    }
+    lastlibaccesstime = thislibaccesstime;
+    
+    // log this request
+    if (gs_keep_log||gs_show_console) {
+      sprintf (tmpstr, "LOCAL LIB: %s\n", tailstr);
+      log_message (tmpstr);
+    }
+    
+    handle_library_request (&tailstr[11], RequestInfo, RequestFields);
+    
+    // remember the preferences
+    rememberpref (tailstr);
+    
+    // log memory information
+    if (gs_keep_log||gs_show_console) {
+      MEMORYSTATUS memstatus;
+      memstatus.dwLength = sizeof(MEMORYSTATUS);
+      GlobalMemoryStatus(&memstatus);
+      sprintf (tmpstr, "BDELTA AVAIL VIRTUAL: %i K\n", 
+	       (int)((baseavailvirtual - memstatus.dwAvailVirtual)/1024));
+      log_message (tmpstr);
+    }
+    
+  } else {
+    // local file
+    if (gs_keep_log||gs_show_console) {
+      sprintf (tmpstr, "LOCAL FILE: %s\n", tailstr);
+      log_message (tmpstr);
+    }
+    send_file_from_disk (tailstr, RequestInfo, RequestFields);
+  }
+}
+
+static void fix_prefix(char *dest, char *pref, char *suff)
+{
+  strcpy(dest,pref);
+  if (*suff != '/') strcat(dest,"/");
+  if (strlen(dest) + strlen(suff) + 1 > MAX_URL_SIZE)
+    strcpy(dest,"http://gsdl/name-too-long");
+  else
+    strcat(dest,suff);
+}
+
+int ExamineURIStr(char *URIStr,RequestInfoT *RequestInfo,
+		  RequestFieldsT *RequestFields)
+{
+  char *protocol, *machine, *rest;  int port;
+  char URICopyA[MAX_URL_SIZE], URICopyB[MAX_URL_SIZE];
+  strcpy(URICopyA,URIStr);
+  
+  if (parse_url(URICopyA,&protocol,&machine,&port,&rest)!=http_ok) {
+    /* Alter local file request to address 'gsdl' */
+    fix_prefix(URICopyB, "http://gsdl", URIStr);
+    URIStr = URICopyB;
+    strcpy(URICopyA, URICopyB);
+    parse_url(URICopyA,&protocol,&machine,&port,&rest);
+  }
+
+  if (strncmp(machine, "gsdl", 5) == 0) {
+    // a local file request
+    handle_server_request(rest, RequestInfo, RequestFields);
+    
+  } else {
+    send_retrieve_error(404, "File not found",
+			"Could not find the local file requested", RequestInfo);
+  }
+  
+  return 1;
+}
Index: /tags/gsdl-21/gsdl/src/w32server/cgiwrapper.h
===================================================================
--- /tags/gsdl-21/gsdl/src/w32server/cgiwrapper.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/w32server/cgiwrapper.h	(revision 699)
@@ -0,0 +1,23 @@
+#ifndef CGIWRAPPER_H
+#define CGIWRAPPER_H
+
+#include "httpreq.h"
+#include "locate.h"
+
+// the number of times the library has been accessed
+extern int libaccessnum;
+
+int ExamineURIStr(char *URIStr,RequestInfoT *RequestInfo,
+		  RequestFieldsT *RequestFields);
+
+// reads any configure file that has been set up and
+// sets up the library
+// returns 1 if successful, 0 if unsuccessful
+int gsdl_init ();
+
+// returns 0 if the directories cant be found
+// and the user wants to quit (it returns 1
+// if everything is ok)
+int cstrcheckdir (char *cstrthedir);
+
+#endif
Index: /tags/gsdl-21/gsdl/src/w32server/conftools.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/w32server/conftools.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/w32server/conftools.cpp	(revision 699)
@@ -0,0 +1,30 @@
+#include "text_t.h"
+#include "conftools.h"
+
+int write_confline (ofstream &fileout, const text_t &key, const text_t value) {
+  if (key.empty() || value.empty()) return -1;
+  outconvertclass text_t2ascii;
+  fileout << text_t2ascii << key << "=" << value << "\n";
+  return 0;
+}
+
+int get_confline (ifstream &filein, text_t &key, text_t &value) {
+  if (filein.eof()) return -1;
+
+  key.clear();
+  value.clear();
+  char c;
+  filein.get(c);
+
+  int foundeq = 0;
+  while (!filein.eof() && c != '\n') {
+    if (!foundeq && c == '=') {foundeq = 1; filein.get(c);}
+
+    if (foundeq) value.push_back(c);
+    else key.push_back(c);
+    filein.get(c);
+  }
+  if (key.empty()) return -1;
+  return 0;
+} 
+
Index: /tags/gsdl-21/gsdl/src/w32server/conftools.h
===================================================================
--- /tags/gsdl-21/gsdl/src/w32server/conftools.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/w32server/conftools.h	(revision 699)
@@ -0,0 +1,15 @@
+#ifndef CONFTOOLS_H
+#define CONFTOOLS_H
+
+#if defined (GSDL_USE_OBJECTSPACE)
+#include <ospace/std/fstream>
+#elif defined (GSDL_USE_IOS_H)
+#include <fstream.h>
+#else
+#include <fstream>
+#endif
+
+int write_confline (ofstream &fileout, const text_t &key, const text_t value);
+int get_confline (ifstream &filein, text_t &key, text_t &value);
+
+#endif
Index: /tags/gsdl-21/gsdl/src/w32server/d_winsock.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/w32server/d_winsock.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/w32server/d_winsock.cpp	(revision 699)
@@ -0,0 +1,642 @@
+#include <windows.h>
+#include "d_winsock.h"
+
+
+typedef SOCKET (PASCAL FAR *accept_Type)(SOCKET s, struct sockaddr FAR *addr, 
+					 int FAR *addrlen);
+typedef int (PASCAL FAR *bind_Type)(SOCKET s, const struct sockaddr FAR *addr, int namelen);
+typedef int (PASCAL FAR *closesocket_Type)(SOCKET s);
+typedef int (PASCAL FAR *connect_Type)(SOCKET s, const struct sockaddr FAR *name, 
+				       int namelen);
+typedef int (PASCAL FAR *ioctlsocket_Type)(SOCKET s, long cmd, u_long FAR *argp);
+typedef int (PASCAL FAR *getpeername_Type)(SOCKET s, struct sockaddr FAR *name,
+					   int FAR * namelen);
+typedef int (PASCAL FAR *getsockname_Type)(SOCKET s, struct sockaddr FAR *name,
+					   int FAR * namelen);
+typedef int (PASCAL FAR *getsockopt_Type)(SOCKET s, int level, int optname,
+					  char FAR * optval, int FAR *optlen);
+typedef u_long (PASCAL FAR *htonl_Type)(u_long hostlong);
+typedef u_short (PASCAL FAR *htons_Type)(u_short hostshort);
+typedef unsigned long (PASCAL FAR *inet_addr_Type)(const char FAR * cp);
+typedef char FAR * (PASCAL FAR *inet_ntoa_Type)(struct in_addr in);
+typedef int (PASCAL FAR *listen_Type)(SOCKET s, int backlog);
+typedef u_long (PASCAL FAR *ntohl_Type)(u_long netlong);
+typedef u_short (PASCAL FAR *ntohs_Type)(u_short netshort);
+typedef int (PASCAL FAR *recv_Type)(SOCKET s, char FAR * buf, int len, int flags);
+typedef int (PASCAL FAR *recvfrom_Type)(SOCKET s, char FAR * buf, int len, int flags,
+					struct sockaddr FAR *from, int FAR * fromlen);
+typedef int (PASCAL FAR *select_Type)(int nfds, fd_set FAR *readfds, fd_set FAR *writefds,
+				      fd_set FAR *exceptfds, const struct timeval FAR *timeout);
+typedef int (PASCAL FAR *send_Type)(SOCKET s, const char FAR * buf, int len, int flags);
+typedef int (PASCAL FAR *sendto_Type)(SOCKET s, const char FAR * buf, int len, int flags,
+				      const struct sockaddr FAR *to, int tolen);
+typedef int (PASCAL FAR *setsockopt_Type)(SOCKET s, int level, int optname,
+					  const char FAR * optval, int optlen);
+typedef int (PASCAL FAR *shutdown_Type)(SOCKET s, int how);
+typedef SOCKET (PASCAL FAR *socket_Type)(int af, int type, int protocol);
+typedef struct hostent FAR * (PASCAL FAR *gethostbyaddr_Type)(const char FAR * addr,
+							      int len, int type);
+typedef struct hostent FAR * (PASCAL FAR *gethostbyname_Type)(const char FAR * name);
+typedef int (PASCAL FAR *gethostname_Type) (char FAR * name, int namelen);
+typedef struct servent FAR * (PASCAL FAR *getservbyport_Type)(int port, const char FAR * proto);
+typedef struct servent FAR * (PASCAL FAR *getservbyname_Type)(const char FAR * name,
+							      const char FAR * proto);
+typedef struct protoent FAR * (PASCAL FAR *getprotobynumber_Type)(int proto);
+typedef struct protoent FAR * (PASCAL FAR *getprotobyname_Type)(const char FAR * name);
+typedef int (PASCAL FAR *WSAStartup_Type)(WORD wVersionRequired, LPWSADATA lpWSAData);
+typedef int (PASCAL FAR *WSACleanup_Type)(void);
+typedef void (PASCAL FAR *WSASetLastError_Type)(int iError);
+typedef int (PASCAL FAR *WSAGetLastError_Type)(void);
+typedef BOOL (PASCAL FAR *WSAIsBlocking_Type)(void);
+typedef int (PASCAL FAR *WSAUnhookBlockingHook_Type)(void);
+typedef FARPROC (PASCAL FAR *WSASetBlockingHook_Type)(FARPROC lpBlockFunc);
+typedef int (PASCAL FAR *WSACancelBlockingCall_Type)(void);
+typedef HANDLE (PASCAL FAR *WSAAsyncGetServByName_Type)(HWND hWnd, u_int wMsg,
+							const char FAR * name,
+							const char FAR * proto,
+							char FAR * buf, int buflen);
+typedef HANDLE (PASCAL FAR *WSAAsyncGetServByPort_Type)(HWND hWnd, u_int wMsg, int port,
+							const char FAR * proto, char FAR * buf,
+							int buflen);
+typedef HANDLE (PASCAL FAR *WSAAsyncGetProtoByName_Type)(HWND hWnd, u_int wMsg,
+							 const char FAR * name, char FAR * buf,
+							 int buflen);
+typedef HANDLE (PASCAL FAR *WSAAsyncGetProtoByNumber_Type)(HWND hWnd, u_int wMsg,
+							   int number, char FAR * buf,
+							   int buflen);
+typedef HANDLE (PASCAL FAR *WSAAsyncGetHostByName_Type)(HWND hWnd, u_int wMsg,
+							const char FAR * name, char FAR * buf,
+							int buflen);
+typedef HANDLE (PASCAL FAR *WSAAsyncGetHostByAddr_Type)(HWND hWnd, u_int wMsg,
+							const char FAR * addr, int len, int type,
+							char FAR * buf, int buflen);
+typedef int (PASCAL FAR *WSACancelAsyncRequest_Type)(HANDLE hAsyncTaskHandle);
+typedef int (PASCAL FAR *WSAAsyncSelect_Type)(SOCKET s, HWND hWnd, u_int wMsg,
+					      long lEvent);
+
+
+
+static accept_Type accept_Ptr = NULL;
+static bind_Type bind_Ptr = NULL;
+static closesocket_Type closesocket_Ptr = NULL;
+static connect_Type connect_Ptr = NULL;
+static ioctlsocket_Type ioctlsocket_Ptr = NULL;
+static getpeername_Type getpeername_Ptr = NULL;
+static getsockname_Type getsockname_Ptr = NULL;
+static getsockopt_Type getsockopt_Ptr = NULL;
+static htonl_Type htonl_Ptr = NULL;
+static htons_Type htons_Ptr = NULL;
+static inet_addr_Type inet_addr_Ptr = NULL;
+static inet_ntoa_Type inet_ntoa_Ptr = NULL;
+static listen_Type listen_Ptr = NULL;
+static ntohl_Type ntohl_Ptr = NULL;
+static ntohs_Type ntohs_Ptr = NULL;
+static recv_Type recv_Ptr = NULL;
+static recvfrom_Type recvfrom_Ptr = NULL;
+static select_Type select_Ptr = NULL;
+static send_Type send_Ptr = NULL;
+static sendto_Type sendto_Ptr = NULL;
+static setsockopt_Type setsockopt_Ptr = NULL;
+static shutdown_Type shutdown_Ptr = NULL;
+static socket_Type socket_Ptr = NULL;
+static gethostbyaddr_Type gethostbyaddr_Ptr = NULL;
+static gethostbyname_Type gethostbyname_Ptr = NULL;
+static gethostname_Type gethostname_Ptr = NULL;
+static getservbyport_Type getservbyport_Ptr = NULL;
+static getservbyname_Type getservbyname_Ptr = NULL;
+static getprotobynumber_Type getprotobynumber_Ptr = NULL;
+static getprotobyname_Type getprotobyname_Ptr = NULL;
+static WSAStartup_Type WSAStartup_Ptr = NULL;
+static WSACleanup_Type WSACleanup_Ptr = NULL;
+static WSASetLastError_Type WSASetLastError_Ptr = NULL;
+static WSAGetLastError_Type WSAGetLastError_Ptr = NULL;
+static WSAIsBlocking_Type WSAIsBlocking_Ptr = NULL;
+static WSAUnhookBlockingHook_Type WSAUnhookBlockingHook_Ptr = NULL;
+static WSASetBlockingHook_Type WSASetBlockingHook_Ptr = NULL;
+static WSACancelBlockingCall_Type WSACancelBlockingCall_Ptr = NULL;
+static WSAAsyncGetServByName_Type WSAAsyncGetServByName_Ptr = NULL;
+static WSAAsyncGetServByPort_Type WSAAsyncGetServByPort_Ptr = NULL;
+static WSAAsyncGetProtoByName_Type WSAAsyncGetProtoByName_Ptr = NULL;
+static WSAAsyncGetProtoByNumber_Type WSAAsyncGetProtoByNumber_Ptr = NULL;
+static WSAAsyncGetHostByName_Type WSAAsyncGetHostByName_Ptr = NULL;
+static WSAAsyncGetHostByAddr_Type WSAAsyncGetHostByAddr_Ptr = NULL;
+static WSACancelAsyncRequest_Type WSACancelAsyncRequest_Ptr = NULL;
+static WSAAsyncSelect_Type WSAAsyncSelect_Ptr = NULL;
+
+
+static HINSTANCE wsock32lib = NULL;
+static HINSTANCE winsocklib = NULL;
+
+
+
+// set all the pointers back to null
+static void ResetWSPtrs () {
+  accept_Ptr = NULL;
+  bind_Ptr = NULL;
+  closesocket_Ptr = NULL;
+  connect_Ptr = NULL;
+  ioctlsocket_Ptr = NULL;
+  getpeername_Ptr = NULL;
+  getsockname_Ptr = NULL;
+  getsockopt_Ptr = NULL;
+  htonl_Ptr = NULL;
+  htons_Ptr = NULL;
+  inet_addr_Ptr = NULL;
+  inet_ntoa_Ptr = NULL;
+  listen_Ptr = NULL;
+  ntohl_Ptr = NULL;
+  ntohs_Ptr = NULL;
+  recv_Ptr = NULL;
+  recvfrom_Ptr = NULL;
+  select_Ptr = NULL;
+  send_Ptr = NULL;
+  sendto_Ptr = NULL;
+  setsockopt_Ptr = NULL;
+  shutdown_Ptr = NULL;
+  socket_Ptr = NULL;
+  gethostbyaddr_Ptr = NULL;
+  gethostbyname_Ptr = NULL;
+  gethostname_Ptr = NULL;
+  getservbyport_Ptr = NULL;
+  getservbyname_Ptr = NULL;
+  getprotobynumber_Ptr = NULL;
+  getprotobyname_Ptr = NULL;
+  WSAStartup_Ptr = NULL;
+  WSACleanup_Ptr = NULL;
+  WSASetLastError_Ptr = NULL;
+  WSAGetLastError_Ptr = NULL;
+  WSAIsBlocking_Ptr = NULL;
+  WSAUnhookBlockingHook_Ptr = NULL;
+  WSASetBlockingHook_Ptr = NULL;
+  WSACancelBlockingCall_Ptr = NULL;
+  WSAAsyncGetServByName_Ptr = NULL;
+  WSAAsyncGetServByPort_Ptr = NULL;
+  WSAAsyncGetProtoByName_Ptr = NULL;
+  WSAAsyncGetProtoByNumber_Ptr = NULL;
+  WSAAsyncGetHostByName_Ptr = NULL;
+  WSAAsyncGetHostByAddr_Ptr = NULL;
+  WSACancelAsyncRequest_Ptr = NULL;
+  WSAAsyncSelect_Ptr = NULL;
+}
+
+// get the pointers to all the socket functions
+static void SetupWSPtrs (HINSTANCE socketlib) {
+  if (socketlib == NULL) return;
+
+  accept_Ptr = (accept_Type)GetProcAddress(socketlib,"accept");
+  bind_Ptr = (bind_Type)GetProcAddress(socketlib,"bind");
+  closesocket_Ptr = (closesocket_Type)GetProcAddress(socketlib,"closesocket");
+  connect_Ptr = (connect_Type)GetProcAddress(socketlib,"connect");
+  ioctlsocket_Ptr = (ioctlsocket_Type)GetProcAddress(socketlib,"ioctlsocket");
+  getpeername_Ptr = (getpeername_Type)GetProcAddress(socketlib,"getpeername");
+  getsockname_Ptr = (getsockname_Type)GetProcAddress(socketlib,"getsockname");
+  getsockopt_Ptr = (getsockopt_Type)GetProcAddress(socketlib,"getsockopt");
+  htonl_Ptr = (htonl_Type)GetProcAddress(socketlib,"htonl");
+  htons_Ptr = (htons_Type)GetProcAddress(socketlib,"htons");
+  inet_addr_Ptr = (inet_addr_Type)GetProcAddress(socketlib,"inet_addr");
+  inet_ntoa_Ptr = (inet_ntoa_Type)GetProcAddress(socketlib,"inet_ntoa");
+  listen_Ptr = (listen_Type)GetProcAddress(socketlib,"listen");
+  ntohl_Ptr = (ntohl_Type)GetProcAddress(socketlib,"ntohl");
+  ntohs_Ptr = (ntohs_Type)GetProcAddress(socketlib,"ntohs");
+  recv_Ptr = (recv_Type)GetProcAddress(socketlib,"recv");
+  recvfrom_Ptr = (recvfrom_Type)GetProcAddress(socketlib,"recvfrom");
+  select_Ptr = (select_Type)GetProcAddress(socketlib,"select");
+  send_Ptr = (send_Type)GetProcAddress(socketlib,"send");
+  sendto_Ptr = (sendto_Type)GetProcAddress(socketlib,"sendto");
+  setsockopt_Ptr = (setsockopt_Type)GetProcAddress(socketlib,"setsockopt");
+  shutdown_Ptr = (shutdown_Type)GetProcAddress(socketlib,"shutdown");
+  socket_Ptr = (socket_Type)GetProcAddress(socketlib,"socket");
+  gethostbyaddr_Ptr = (gethostbyaddr_Type)GetProcAddress(socketlib,"gethostbyaddr");
+  gethostbyname_Ptr = (gethostbyname_Type)GetProcAddress(socketlib,"gethostbyname");
+  gethostname_Ptr = (gethostname_Type)GetProcAddress(socketlib,"gethostname");
+  getservbyport_Ptr = (getservbyport_Type)GetProcAddress(socketlib,"getservbyport");
+  getservbyname_Ptr = (getservbyname_Type)GetProcAddress(socketlib,"getservbyname");
+  getprotobynumber_Ptr = (getprotobynumber_Type)GetProcAddress(socketlib,"getprotobynumber");
+  getprotobyname_Ptr = (getprotobyname_Type)GetProcAddress(socketlib,"getprotobyname");
+  WSAStartup_Ptr = (WSAStartup_Type)GetProcAddress(socketlib,"WSAStartup");
+  WSACleanup_Ptr = (WSACleanup_Type)GetProcAddress(socketlib,"WSACleanup");
+  WSASetLastError_Ptr = (WSASetLastError_Type)GetProcAddress(socketlib,"WSASetLastError");
+  WSAGetLastError_Ptr = (WSAGetLastError_Type)GetProcAddress(socketlib,"WSAGetLastError");
+  WSAIsBlocking_Ptr = (WSAIsBlocking_Type)GetProcAddress(socketlib,"WSAIsBlocking");
+  WSAUnhookBlockingHook_Ptr = 
+    (WSAUnhookBlockingHook_Type)GetProcAddress(socketlib,"WSAUnhookBlockingHook");
+  WSASetBlockingHook_Ptr = 
+    (WSASetBlockingHook_Type)GetProcAddress(socketlib,"WSASetBlockingHook");
+  WSACancelBlockingCall_Ptr = 
+    (WSACancelBlockingCall_Type)GetProcAddress(socketlib,"WSACancelBlockingCall");
+  WSAAsyncGetServByName_Ptr = 
+    (WSAAsyncGetServByName_Type)GetProcAddress(socketlib,"WSAAsyncGetServByName");
+  WSAAsyncGetServByPort_Ptr = 
+    (WSAAsyncGetServByPort_Type)GetProcAddress(socketlib,"WSAAsyncGetServByPort");
+  WSAAsyncGetProtoByName_Ptr = 
+    (WSAAsyncGetProtoByName_Type)GetProcAddress(socketlib,"WSAAsyncGetProtoByName");
+  WSAAsyncGetProtoByNumber_Ptr = 
+    (WSAAsyncGetProtoByNumber_Type)GetProcAddress(socketlib,"WSAAsyncGetProtoByNumber");
+  WSAAsyncGetHostByName_Ptr = 
+    (WSAAsyncGetHostByName_Type)GetProcAddress(socketlib,"WSAAsyncGetHostByName");
+  WSAAsyncGetHostByAddr_Ptr = 
+    (WSAAsyncGetHostByAddr_Type)GetProcAddress(socketlib,"WSAAsyncGetHostByAddr");
+  WSACancelAsyncRequest_Ptr = 
+    (WSACancelAsyncRequest_Type)GetProcAddress(socketlib,"WSACancelAsyncRequest");
+  WSAAsyncSelect_Ptr = (WSAAsyncSelect_Type)GetProcAddress(socketlib,"WSAAsyncSelect");
+}
+
+
+
+// expects path to be NULL, a empty string, or a path which ends in a slash
+// returns D_NOERROR on success,
+// D_ERR_UNEXPECTED or D_ERR_LOADFAILED on failure
+int d_LoadWinsock (char *path) {
+  char dllname[2048]; // HARD LIMIT!!!!
+  OSVERSIONINFO osver;
+
+  if ((wsock32lib != NULL) || (winsocklib != NULL)) return D_ERR_UNEXPECTED;
+
+  // set the error mode so that no nasty dialog boxes come
+  // up when things aren't installed
+  int olderrormode = SetErrorMode(SEM_NOOPENFILEERRORBOX|SEM_FAILCRITICALERRORS);
+
+  // get the operating system version information
+  memset(&osver, 0, sizeof(OSVERSIONINFO));
+  osver.dwOSVersionInfoSize = sizeof(OSVERSIONINFO);
+  if (!GetVersionEx(&osver)) {
+    SetErrorMode(olderrormode);
+    return D_ERR_UNEXPECTED;
+  }
+
+  // see if we should load winsock before wsock32
+  if ((osver.dwPlatformId == VER_PLATFORM_WIN32s) && (path != NULL)) {
+    strcpy (dllname, path);
+    strcat (dllname, "winsock.dll");
+
+    // attempt to load the library
+    winsocklib = LoadLibrary (dllname);
+    if (winsocklib == NULL) {
+      SetErrorMode(olderrormode);
+      return D_ERR_LOADFAILED;
+    }
+  }
+
+  // attempt to load wsock32
+  int err = D_NOERROR;
+  dllname[0] = '\0';
+  if ((osver.dwPlatformId != VER_PLATFORM_WIN32s) && (path != NULL)) {
+    strcpy (dllname, path);
+  }
+  strcat (dllname, "wsock32.dll");
+  wsock32lib = LoadLibrary (dllname);
+  if (wsock32lib == NULL) err = D_ERR_LOADFAILED;
+  else SetupWSPtrs (wsock32lib);
+
+
+  // perform a few tests
+
+  // first try starting-up (and check the szDescription field)
+  if (err == D_NOERROR) {
+    WSADATA Data;
+    memset(&Data, 0, sizeof(WSADATA));
+    if ((d_WSAStartup(257, &Data) != 0) ||
+	(Data.szDescription == NULL) || 
+	(Data.szDescription[0] == '\0'))
+      err = D_ERR_LOADFAILED;
+  }
+
+  // next try setting and getting an error
+  if (err == D_NOERROR) {
+    d_WSASetLastError (WSAENOTSOCK);
+    if (d_WSAGetLastError () != WSAENOTSOCK) err = D_ERR_LOADFAILED;
+    d_WSASetLastError (0);
+  }
+
+  // last try cleaning up (if an error occurs here we might
+  // not be able to unload anything)
+  if ((err == D_NOERROR) && (d_WSACleanup() == SOCKET_ERROR))
+    err = D_ERR_LOADFAILED;
+
+  // if an error occurred then we need to unload everything
+  if (err != D_NOERROR) d_UnloadWinsock ();
+
+  SetErrorMode(olderrormode);
+  return err;
+}
+
+// returns D_NOERROR on success,
+// D_ERR_FREELIBFAILED on failure
+int d_UnloadWinsock () {
+  int err = D_NOERROR;
+
+  // unload wsock32
+  if ((wsock32lib != NULL) && !FreeLibrary (wsock32lib)) {
+    err = D_ERR_FREELIBFAILED;
+  }
+  wsock32lib = NULL;
+
+  // unload winsock
+  if ((winsocklib != NULL) && !FreeLibrary (winsocklib)) {
+    err = D_ERR_FREELIBFAILED;
+  }
+  winsocklib = NULL;
+  
+  // reset all the function pointers
+  ResetWSPtrs ();
+
+  return err;
+}
+
+
+
+
+// socket functions, these just call the dynamically loaded version
+
+SOCKET d_accept (SOCKET s, struct sockaddr *addr, int *addrlen) {
+  if (accept_Ptr != NULL) 
+    return (*accept_Ptr)(s, addr, addrlen);
+  return INVALID_SOCKET; // error winsock not loaded
+}
+
+int d_bind (SOCKET s, const struct sockaddr *addr, int namelen) {
+  if (bind_Ptr != NULL) 
+    return (*bind_Ptr)(s, addr, namelen);
+  return WSANOTINITIALISED; // error winsock not loaded
+}
+
+int d_closesocket (SOCKET s) {
+  if (closesocket_Ptr != NULL) 
+    return (*closesocket_Ptr)(s);
+  return SOCKET_ERROR; // error winsock not loaded
+}
+
+int d_connect (SOCKET s, const struct sockaddr *name, int namelen) {
+  if (connect_Ptr != NULL) 
+    return (*connect_Ptr)(s, name, namelen);
+  return SOCKET_ERROR; // error winsock not loaded
+}
+
+int d_ioctlsocket (SOCKET s, long cmd, u_long *argp) {
+  if (ioctlsocket_Ptr != NULL) 
+    return (*ioctlsocket_Ptr)(s, cmd, argp);
+  return SOCKET_ERROR; // error winsock not loaded
+}
+
+int d_getpeername (SOCKET s, struct sockaddr *name, int *namelen) {
+  if (getpeername_Ptr != NULL) 
+    return (*getpeername_Ptr)(s, name, namelen);
+  return SOCKET_ERROR; // error winsock not loaded
+}
+
+int d_getsockname (SOCKET s, struct sockaddr *name, int *namelen) {
+  if (getsockname_Ptr != NULL) 
+    return (*getsockname_Ptr)(s, name, namelen);
+  return SOCKET_ERROR; // error winsock not loaded
+}
+
+int d_getsockopt (SOCKET s, int level, int optname, char *optval, int *optlen) {
+  if (getsockopt_Ptr != NULL) 
+    return (*getsockopt_Ptr)(s, level, optname, optval, optlen);
+  return SOCKET_ERROR; // error winsock not loaded
+}
+
+u_long d_htonl (u_long hostlong) {
+  if (htonl_Ptr != NULL) 
+    return (*htonl_Ptr)(hostlong);
+  return hostlong; // error winsock not loaded
+}
+
+u_short d_htons (u_short hostshort) {
+  if (htons_Ptr != NULL) 
+    return (*htons_Ptr)(hostshort);
+  return hostshort; // error winsock not loaded
+}
+
+unsigned long d_inet_addr (const char *cp) {
+  if (inet_addr_Ptr != NULL) 
+    return (*inet_addr_Ptr)(cp);
+  return INADDR_NONE; // error winsock not loaded
+}
+
+char *d_inet_ntoa (struct in_addr in) {
+  if (inet_ntoa_Ptr != NULL) 
+    return (*inet_ntoa_Ptr)(in);
+  return NULL; // error winsock not loaded
+}
+
+int d_listen (SOCKET s, int backlog) {
+  if (listen_Ptr != NULL) 
+    return (*listen_Ptr)(s, backlog);
+  return SOCKET_ERROR; // error winsock not loaded
+}
+
+u_long d_ntohl (u_long netlong) {
+  if (ntohl_Ptr != NULL) 
+    return (*ntohl_Ptr)(netlong);
+  return netlong; // error winsock not loaded
+}
+
+u_short d_ntohs (u_short netshort) {
+  if (ntohs_Ptr != NULL) 
+    return (*ntohs_Ptr)(netshort);
+  return netshort; // error winsock not loaded
+}
+
+int d_recv (SOCKET s, char *buf, int len, int flags) {
+  if (recv_Ptr != NULL) 
+    return (*recv_Ptr)(s, buf, len, flags);
+  return SOCKET_ERROR; // error winsock not loaded
+}
+
+int d_recvfrom (SOCKET s, char *buf, int len, int flags,
+		struct sockaddr *from, int *fromlen) {
+  if (recvfrom_Ptr != NULL) 
+    return (*recvfrom_Ptr)(s, buf, len, flags, from, fromlen);
+  return SOCKET_ERROR; // error winsock not loaded
+}
+
+int d_select (int nfds, fd_set *readfds, fd_set *writefds,
+	      fd_set *exceptfds, const struct timeval *timeout) {
+  if (select_Ptr != NULL) 
+    return (*select_Ptr)(nfds, readfds, writefds, exceptfds, timeout);
+  return SOCKET_ERROR; // error winsock not loaded
+}
+
+int d_send (SOCKET s, const char *buf, int len, int flags) {
+  if (send_Ptr != NULL) 
+    return (*send_Ptr)(s, buf, len, flags);
+  return SOCKET_ERROR; // error winsock not loaded
+}
+
+int d_sendto (SOCKET s, const char *buf, int len, int flags,
+	      const struct sockaddr *to, int tolen) {
+  if (sendto_Ptr != NULL) 
+    return (*sendto_Ptr)(s, buf, len, flags, to, tolen);
+  return SOCKET_ERROR; // error winsock not loaded
+}
+
+int d_setsockopt (SOCKET s, int level, int optname,
+		  const char *optval, int optlen) {
+  if (setsockopt_Ptr != NULL) 
+    return (*setsockopt_Ptr)(s, level, optname, optval, optlen);
+  return SOCKET_ERROR; // error winsock not loaded
+}
+
+int d_shutdown (SOCKET s, int how) {
+  if (shutdown_Ptr != NULL) 
+    return (*shutdown_Ptr)(s, how);
+  return SOCKET_ERROR; // error winsock not loaded
+}
+
+SOCKET d_socket (int af, int type, int protocol) {
+  if (socket_Ptr != NULL) 
+    return (*socket_Ptr)(af, type, protocol);
+  return INVALID_SOCKET; // error winsock not loaded
+}
+
+hostent *d_gethostbyaddr(const char *addr, int len, int type) {
+  if (gethostbyaddr_Ptr != NULL) 
+    return (*gethostbyaddr_Ptr)(addr, len, type);
+  return NULL; // error winsock not loaded
+}
+
+hostent *d_gethostbyname(const char *name) {
+  if (gethostbyname_Ptr != NULL) 
+    return (*gethostbyname_Ptr)(name);
+  return NULL; // error winsock not loaded
+}
+
+int d_gethostname (char *name, int namelen) {
+  if (gethostname_Ptr != NULL) 
+    return (*gethostname_Ptr)(name, namelen);
+  return SOCKET_ERROR; // error winsock not loaded
+}
+
+servent *d_getservbyport(int port, const char *proto) {
+  if (getservbyport_Ptr != NULL) 
+    return (*getservbyport_Ptr)(port, proto);
+  return NULL; // error winsock not loaded
+}
+
+servent *d_getservbyname(const char *name, const char *proto) {
+  if (getservbyname_Ptr != NULL) 
+    return (*getservbyname_Ptr)(name, proto);
+  return NULL; // error winsock not loaded
+}
+
+protoent *d_getprotobynumber(int proto) {
+  if (getprotobynumber_Ptr != NULL) 
+    return (*getprotobynumber_Ptr)(proto);
+  return NULL; // error winsock not loaded
+}
+
+protoent *d_getprotobyname(const char *name) {
+  if (getprotobyname_Ptr != NULL) 
+    return (*getprotobyname_Ptr)(name);
+  return NULL; // error winsock not loaded
+}
+
+int d_WSAStartup(WORD wVersionRequired, LPWSADATA lpWSAData) {
+  if (WSAStartup_Ptr != NULL) 
+    return (*WSAStartup_Ptr)(wVersionRequired, lpWSAData);
+  return WSASYSNOTREADY; // error winsock not loaded
+}
+
+int d_WSACleanup(void) {
+  if (WSACleanup_Ptr != NULL) 
+    return (*WSACleanup_Ptr)();
+  return WSANOTINITIALISED; // error winsock not loaded
+}
+
+void d_WSASetLastError(int iError) {
+  if (WSASetLastError_Ptr != NULL) (*WSASetLastError_Ptr)(iError);
+}
+
+int d_WSAGetLastError(void) {
+  if (WSAGetLastError_Ptr != NULL) 
+    return (*WSAGetLastError_Ptr)();
+  return WSANOTINITIALISED; // error winsock not loaded
+}
+
+BOOL d_WSAIsBlocking(void) {
+  if (WSAIsBlocking_Ptr != NULL) 
+    return (*WSAIsBlocking_Ptr)();
+  return FALSE; // error winsock not loaded
+}
+
+int d_WSAUnhookBlockingHook(void) {
+  if (WSAUnhookBlockingHook_Ptr != NULL) 
+    return (*WSAUnhookBlockingHook_Ptr)();
+  return SOCKET_ERROR; // error winsock not loaded
+}
+
+FARPROC d_WSASetBlockingHook(FARPROC lpBlockFunc) {
+  if (WSASetBlockingHook_Ptr != NULL) 
+    return (*WSASetBlockingHook_Ptr)(lpBlockFunc);
+  return NULL; // error winsock not loaded
+}
+
+int d_WSACancelBlockingCall(void) {
+  if (WSACancelBlockingCall_Ptr != NULL) 
+    return (*WSACancelBlockingCall_Ptr)();
+  return SOCKET_ERROR; // error winsock not loaded
+}
+
+HANDLE d_WSAAsyncGetServByName(HWND hWnd, u_int wMsg,
+			       const char *name,
+			       const char *proto,
+			       char *buf, int buflen) {
+  if (WSAAsyncGetServByName_Ptr != NULL) 
+    return (*WSAAsyncGetServByName_Ptr)(hWnd, wMsg, name, proto, buf, buflen);
+  return NULL; // error winsock not loaded
+}
+
+HANDLE d_WSAAsyncGetServByPort(HWND hWnd, u_int wMsg, int port,
+			       const char *proto, char *buf,
+			       int buflen) {
+  if (WSAAsyncGetServByPort_Ptr != NULL) 
+    return (*WSAAsyncGetServByPort_Ptr)(hWnd, wMsg, port, proto, buf, buflen);
+  return NULL; // error winsock not loaded
+}
+
+HANDLE d_WSAAsyncGetProtoByName(HWND hWnd, u_int wMsg,
+				const char *name, char *buf,
+				int buflen) {
+  if (WSAAsyncGetProtoByName_Ptr != NULL) 
+    return (*WSAAsyncGetProtoByName_Ptr)(hWnd, wMsg, name, buf, buflen);
+  return NULL; // error winsock not loaded
+}
+
+HANDLE d_WSAAsyncGetProtoByNumber(HWND hWnd, u_int wMsg,
+				  int number, char *buf,
+				  int buflen) {
+  if (WSAAsyncGetProtoByNumber_Ptr != NULL) 
+    return (*WSAAsyncGetProtoByNumber_Ptr)(hWnd, wMsg, number, buf, buflen);
+  return NULL; // error winsock not loaded
+}
+
+HANDLE d_WSAAsyncGetHostByName(HWND hWnd, u_int wMsg,
+			       const char *name, char *buf,
+			       int buflen) {
+  if (WSAAsyncGetHostByName_Ptr != NULL) 
+    return (*WSAAsyncGetHostByName_Ptr)(hWnd, wMsg, name, buf, buflen);
+  return NULL; // error winsock not loaded
+}
+
+HANDLE d_WSAAsyncGetHostByAddr(HWND hWnd, u_int wMsg,
+                                        const char *addr, int len, int type,
+                                        char *buf, int buflen) {
+  if (WSAAsyncGetHostByAddr_Ptr != NULL) 
+    return (*WSAAsyncGetHostByAddr_Ptr)(hWnd, wMsg, addr, len, type, buf, buflen);
+  return NULL; // error winsock not loaded
+}
+
+int d_WSACancelAsyncRequest(HANDLE hAsyncTaskHandle) {
+  if (WSACancelAsyncRequest_Ptr != NULL) 
+    return (*WSACancelAsyncRequest_Ptr)(hAsyncTaskHandle);
+  return SOCKET_ERROR; // error winsock not loaded
+}
+
+int d_WSAAsyncSelect(SOCKET s, HWND hWnd, u_int wMsg,
+                               long lEvent) {
+  if (WSAAsyncSelect_Ptr != NULL) 
+    return (*WSAAsyncSelect_Ptr)(s, hWnd, wMsg, lEvent);
+  return SOCKET_ERROR; // error winsock not loaded
+}
+
Index: /tags/gsdl-21/gsdl/src/w32server/d_winsock.h
===================================================================
--- /tags/gsdl-21/gsdl/src/w32server/d_winsock.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/w32server/d_winsock.h	(revision 699)
@@ -0,0 +1,103 @@
+#ifndef D_WINSOCK_H
+#define D_WINSOCK_H
+
+
+/* get most definitions from the real winsock */
+#include <winsock.h>
+
+
+#define D_NOERROR           0
+#define D_ERR_UNEXPECTED    1
+#define D_ERR_LOADFAILED    2
+#define D_ERR_FREELIBFAILED 3
+
+
+
+// d_LoadWinsock must be called before an normal winsock
+// functions
+
+
+// expects path to be NULL, a empty string, or a path which ends in a slash
+// returns D_NOERROR on success,
+// D_ERR_UNEXPECTED or D_ERR_LOADFAILED on failure
+int d_LoadWinsock (char *path);
+
+// returns D_NOERROR on success,
+// D_ERR_FREELIBFAILED on failure
+int d_UnloadWinsock ();
+
+
+// dynamically loaded versions of the winsock functions
+
+SOCKET d_accept (SOCKET s, struct sockaddr *addr, int *addrlen);
+int d_bind (SOCKET s, const struct sockaddr *addr, int namelen);
+int d_closesocket (SOCKET s);
+int d_connect (SOCKET s, const struct sockaddr *name, int namelen);
+int d_ioctlsocket (SOCKET s, long cmd, u_long *argp);
+int d_getpeername (SOCKET s, struct sockaddr *name, int *namelen);
+int d_getsockname (SOCKET s, struct sockaddr *name, int *namelen);
+int d_getsockopt (SOCKET s, int level, int optname, char *optval, int *optlen);
+u_long d_htonl (u_long hostlong);
+u_short d_htons (u_short hostshort);
+unsigned long d_inet_addr (const char *cp);
+char *d_inet_ntoa (struct in_addr in);
+int d_listen (SOCKET s, int backlog);
+u_long d_ntohl (u_long netlong);
+u_short d_ntohs (u_short netshort);
+int d_recv (SOCKET s, char *buf, int len, int flags);
+int d_recvfrom (SOCKET s, char *buf, int len, int flags,
+		struct sockaddr *from, int *fromlen);
+int d_select (int nfds, fd_set *readfds, fd_set *writefds,
+	      fd_set *exceptfds, const struct timeval *timeout);
+int d_send (SOCKET s, const char *buf, int len, int flags);
+int d_sendto (SOCKET s, const char *buf, int len, int flags,
+	      const struct sockaddr *to, int tolen);
+int d_setsockopt (SOCKET s, int level, int optname,
+		  const char *optval, int optlen);
+int d_shutdown (SOCKET s, int how);
+SOCKET d_socket (int af, int type, int protocol);
+
+hostent *d_gethostbyaddr(const char *addr, int len, int type);
+hostent *d_gethostbyname(const char *name);
+int d_gethostname (char *name, int namelen);
+servent *d_getservbyport(int port, const char *proto);
+servent *d_getservbyname(const char *name, const char *proto);
+protoent *d_getprotobynumber(int proto);
+protoent *d_getprotobyname(const char *name);
+
+int d_WSAStartup(WORD wVersionRequired, LPWSADATA lpWSAData);
+int d_WSACleanup(void);
+void d_WSASetLastError(int iError);
+int d_WSAGetLastError(void);
+BOOL d_WSAIsBlocking(void);
+int d_WSAUnhookBlockingHook(void);
+FARPROC d_WSASetBlockingHook(FARPROC lpBlockFunc);
+int d_WSACancelBlockingCall(void);
+HANDLE d_WSAAsyncGetServByName(HWND hWnd, u_int wMsg,
+			       const char *name,
+			       const char *proto,
+			       char *buf, int buflen);
+HANDLE d_WSAAsyncGetServByPort(HWND hWnd, u_int wMsg, int port,
+			       const char *proto, char *buf,
+			       int buflen);
+HANDLE d_WSAAsyncGetProtoByName(HWND hWnd, u_int wMsg,
+				const char *name, char *buf,
+				int buflen);
+HANDLE d_WSAAsyncGetProtoByNumber(HWND hWnd, u_int wMsg,
+				  int number, char *buf,
+				  int buflen);
+HANDLE d_WSAAsyncGetHostByName(HWND hWnd, u_int wMsg,
+			       const char *name, char *buf,
+			       int buflen);
+HANDLE d_WSAAsyncGetHostByAddr(HWND hWnd, u_int wMsg,
+			       const char *addr, int len, int type,
+			       char *buf, int buflen);
+int d_WSACancelAsyncRequest(HANDLE hAsyncTaskHandle);
+int d_WSAAsyncSelect(SOCKET s, HWND hWnd, u_int wMsg,
+		     long lEvent);
+
+
+
+
+
+#endif
Index: /tags/gsdl-21/gsdl/src/w32server/fnord.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/w32server/fnord.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/w32server/fnord.cpp	(revision 699)
@@ -0,0 +1,976 @@
+/*
+Copyright (C) 1996
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+The author can be contacted via Email at bmorin@wpi.edu
+*/
+#include <windows.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <memory.h>
+#pragma hdrstop
+#include "netio.h"
+#include "settings.h"
+#include "httpsrv.h"
+#include "resource.h"
+#include "locate.h"
+#include "locateP.h"
+#include "cgiwrapper.h"
+#include "startbrowser.h"
+#include "d_winsock.h"
+
+#define MAINWINDOWWIDTH 400
+#define MAINWINDOWHEIGHT 200
+
+#define RESTHEIGHT 45   // Extra window size reqd below logo
+#define WASTEHEIGHT 44  // Error in ht due to caption and menu bar
+#define WASTEWIDTH 6
+
+// size and position of the collection title
+#define COLTITLEWIDTH 351
+#define COLTITLEHEIGHT 35
+#define COLTITLEX ((MAINWINDOWWIDTH-COLTITLEWIDTH)/2)
+#define COLTITLEY 10
+
+// size and position of the version string
+#define VERSIONWIDTH MAINWINDOWWIDTH
+#define VERSIONHEIGHT 20
+#define VERSIONX 0
+#define VERSIONY (COLTITLEY+COLTITLEHEIGHT)
+
+// size and position of the Greenstone digital library logo
+#define LOGOWIDTH 135
+#define LOGOHEIGHT 55
+#define LOGOX (MAINWINDOWWIDTH-LOGOWIDTH-20)
+#define LOGOY (MAINWINDOWHEIGHT-LOGOHEIGHT-RESTHEIGHT)
+
+// size and position of the status string
+#define STATUSX 20
+#define STATUSY ((LOGOY-VERSIONY-VERSIONHEIGHT)/2+VERSIONY+VERSIONHEIGHT)
+#define STATUSWIDTH MAINWINDOWWIDTH-STATUSX
+#define STATUSHEIGHT (MAINWINDOWHEIGHT-RESTHEIGHT-STATUSY)
+
+// size and position of the info string (same position as the button)
+#define INFOWIDTH MAINWINDOWWIDTH
+#define INFOHEIGHT (MAINWINDOWHEIGHT-INFOY)
+#define INFOX 0
+#define INFOY ((MAINWINDOWHEIGHT-RESTHEIGHT)+10)
+
+// size and position of the button
+#define ENTERBUTTONWIDTH 150
+#define ENTERBUTTONHEIGHT 30
+#define ENTERBUTTONX ((MAINWINDOWWIDTH-ENTERBUTTONWIDTH)/2)
+#define ENTERBUTTONY ((MAINWINDOWHEIGHT-RESTHEIGHT)+5)
+
+#define VERSIONSTRING "version 1.1"
+const char versionstring[] = VERSIONSTRING;
+
+const char strnothing[] = "";
+const char strinit[] = "Initialising...";
+const char strsb[] = "Starting browser...";
+const char strenterlib[] = "Press the 'Enter Library' button to start a browser\n"
+"and begin using this library";
+const char strenterstaticlib[] = "Press the 'Enter Library' button to start a browser\n"
+"and begin using the static version\n"
+"of this library";
+const char *statusstring = NULL; // points to the current status string
+const char *infostring = NULL; // points to the current info string
+
+
+// globals to do with networking and browsers
+int have_networking = 0;
+int netscapeneeded = 0;
+char startbrowserdir[1024] = ""; // HARD LIMIT!!!!!
+
+
+// stats to do with the last time the enter library button was pressed
+// these stats are needed to make suggestions about the proxy
+int enterlib_libaccessnum = -1; // -1 = NA
+DWORD enterlib_time = 0;
+
+
+enum { undefined_window, console_window, plain_window }
+window_state;
+
+HDC coltitledc = NULL;
+HBITMAP defcoltitlebitmap = NULL, coltitlebitmap=NULL;
+
+HDC logodc=NULL;
+HBITMAP deflogobitmap=NULL, logobitmap=NULL;
+
+HWND GoButton = NULL;
+
+
+void finish_up() {
+  // remember the current preferences
+  write_settings();
+
+  // Shutdown the HTTP server
+  EndHTTPServer();
+  
+  // Shutdown the main server modules
+  CleanUpNetIO();
+  //    RemoveFnordFromTray();
+  
+  // Unload the wsock32 dll
+  d_UnloadWinsock();
+  
+  // Clean up graphics stuff from window
+  
+  if (defcoltitlebitmap != NULL)
+    SelectObject (coltitledc, defcoltitlebitmap);
+  if (coltitlebitmap != NULL)
+    DeleteObject (coltitlebitmap);
+  if (coltitledc != NULL)
+    DeleteDC (coltitledc);
+  
+  if (deflogobitmap != NULL)
+    SelectObject (logodc, deflogobitmap);
+  if (logobitmap != NULL)
+    DeleteObject (logobitmap);
+  if (logodc != NULL)
+    DeleteDC (logodc);
+  
+  // Shutdown the main window
+  PostQuitMessage(0);
+}
+
+
+int overlap(int left, int top, int right, int bottom,
+            RECT &r2)
+{
+  if (left <= r2.right && r2.left <= right &&
+      top <= r2.bottom && r2.top <= bottom)
+    return 1;
+  else
+    return 0;
+}
+
+
+void paint_window (HDC pdc, RECT &updateRect) {
+  // make sure the various DCs are set up
+  if (coltitledc == NULL) {
+    coltitledc = CreateCompatibleDC(pdc);
+    defcoltitlebitmap = SelectObject (coltitledc, coltitlebitmap);
+  }
+  
+  if (logodc == NULL) {
+    logodc = CreateCompatibleDC(pdc);
+    deflogobitmap = SelectObject (logodc, logobitmap);
+  }
+  
+  // update the collection title if needed
+  if (overlap(COLTITLEX, COLTITLEY,
+	      COLTITLEX+COLTITLEWIDTH, COLTITLEY+COLTITLEHEIGHT, 
+	      updateRect)) {
+    BitBlt (pdc, COLTITLEX, COLTITLEY, 
+	    COLTITLEX+COLTITLEWIDTH, COLTITLEY+COLTITLEHEIGHT,
+	    coltitledc, 0, 0, SRCCOPY);
+  }
+  
+  // update the version string if needed
+  if (overlap (VERSIONX, VERSIONY, 
+	       VERSIONX+VERSIONWIDTH, VERSIONY+VERSIONHEIGHT,
+	       updateRect)) {
+    RECT versionRect;
+    versionRect.left = VERSIONX;
+    versionRect.top = VERSIONY;
+    versionRect.right = VERSIONX+VERSIONWIDTH;
+    versionRect.bottom = VERSIONY+VERSIONHEIGHT;
+    DrawText(pdc, versionstring, -1, &versionRect, DT_CENTER);
+  }
+  
+  // decide what we want to draw
+  if (gs_show_console) {
+    // we want to draw a "console" window
+    if (window_state != console_window ||
+	overlap (text_rect.left, text_rect.top,
+		 text_rect.right, text_rect.bottom,
+		 updateRect)) {
+      refresh_console (pdc);
+    }
+    
+    window_state = console_window;
+    
+  } else {
+    // we want to draw a "plain" window
+    
+    // update the status string if needed
+    if ((statusstring != NULL) && (strlen(statusstring) > 0)
+	&& overlap (STATUSX, STATUSY, 
+		    STATUSX+STATUSWIDTH, STATUSY+STATUSHEIGHT,
+		    updateRect)) {
+      RECT statusRect;
+      statusRect.left = STATUSX;
+      statusRect.top = STATUSY;
+      statusRect.right = STATUSX+STATUSWIDTH;
+      statusRect.bottom = STATUSY+STATUSHEIGHT;
+      FillRect(pdc, &statusRect, GetStockObject(WHITE_BRUSH));
+      
+      int cury = STATUSY;
+      int startline = 0, here = 0;
+      while (statusstring[here] != '\0') {
+	if (statusstring[here] < ' ') {
+	  if (here - startline > 0) {
+	    // output the text
+	    TextOut (pdc, STATUSX, cury, 
+		     &statusstring[startline], here-startline);
+	    cury += line_spacing;
+	  }
+	  startline = here+1;
+	}
+	here++;
+      }
+      // output any remaining text
+      if (here - startline > 0) {
+	TextOut (pdc, STATUSX, cury, &statusstring[startline], here-startline);
+      }
+    }
+    
+    // update the logo (always)
+    BitBlt (pdc, LOGOX, LOGOY, LOGOX+LOGOWIDTH, LOGOY+LOGOHEIGHT,
+	    logodc, 0, 0, SRCCOPY);
+    
+    window_state = plain_window;
+  }
+  
+  
+  // update the info string if needed
+  if ((GoButton == NULL) && (infostring != NULL) &&
+      overlap (INFOX, INFOY, 
+	       INFOX+INFOWIDTH, INFOY+INFOHEIGHT,
+	       updateRect)) {
+    RECT infoRect;
+    infoRect.left = INFOX;
+    infoRect.top = INFOY;
+    infoRect.right = INFOX+INFOWIDTH;
+    infoRect.bottom = INFOY+INFOHEIGHT;
+    FillRect(pdc, &infoRect, GetStockObject(WHITE_BRUSH));
+    DrawText(pdc, infostring, -1, &infoRect, DT_CENTER);
+  }
+}
+
+
+void handle_painting (HWND Window) {
+  HDC pdc;  PAINTSTRUCT ps;
+  pdc = BeginPaint(Window, &ps);
+  
+  paint_window (pdc, ps.rcPaint);
+  
+  EndPaint(Window, &ps);
+}
+
+
+// returns 1 on success, 0 otherwise
+int enter_library (HWND Window) {
+  char url[256], numbuf[20];
+  int res;
+  
+  // get the url and attempt to start a browser
+  if (have_networking) {
+    strcpy(url, "http://");
+    strcat(url, GetLocalName(NULL));
+    if (gs_port_num != 80) {
+      itoa(gs_port_num,numbuf,10);
+      strcat(url,":");
+      strcat(url,numbuf);
+    }
+    strcat(url, gs_enterlib);
+    
+    // add a unique ending so it will always result in a request
+    if (strstr (gs_enterlib, "?") == NULL) strcat (url, "?");
+    else strcat (url, "&");
+    
+    strcat (url, "uq=");
+    itoa ((int)GetTickCount(), numbuf, 10);
+    strcat (url, numbuf);
+    
+    // remember the library access number for later proxy checking
+    enterlib_libaccessnum = libaccessnum;
+    
+    // do a quick check to make sure we're using netscape when
+    // we should be (it might have been just installed)
+    if (netscapeneeded) gs_check_browser_settings (netscapeneeded);
+    
+    if (strlen (startbrowserdir) <= 0) {
+      res = startbrowser (url, gs_browser_exe, NULL);
+      
+    } else res = startbrowser (url, gs_browser_exe, startbrowserdir);
+    
+  } else {
+    char staticpath[256];
+    
+    // try to find the static pages
+    strcpy (url, gs_staticpages);
+    
+    // remove any slashes from the static pathname
+    int len = strlen (url);
+    while ((len > 0) && ((url[len-1] == '\\') || (url[len-1] == '/'))) {
+      len--;
+    }
+    url[len] = '\0';
+    
+    // add one slash to the end
+    strcat (url, "\\");
+    
+    // check to make sure the static directory is available
+    strcpy (staticpath, url);
+    strcat (staticpath, "static\\");
+    
+    // check this directory
+    if (!cstrcheckdir(staticpath)) {
+      PostMessage(Window,WM_CLOSE,0,0);
+      return 0;
+    }
+    
+    // add filename of the first page
+    strcat (url, "niupepa.htm");
+    
+    res = startbrowser (url, gs_browser_exe, NULL);
+    startbrowserdir[0] = '\0'; // doesn't make sense for non-networked case
+  }
+  
+  if (res == SB_NOERROR) {
+    // success !!
+    
+    // remember the time the browser was started for later proxy checking
+    if (have_networking) enterlib_time = GetTickCount();
+    
+    return 1;
+  }
+  
+  // no browser was started
+  enterlib_libaccessnum = -1;
+  
+  if ((res == SB_FAIL_BADFORMAT) || (res == SB_FAIL_NOTFOUND)) {
+    if (!netscapeneeded) { // any browser
+      MessageBox(Window,
+		 "Failed to start your chosen browser. It seems that your\n"
+		 "chosen browser has been removed or corrupted.\n"
+		 "Please check you browser choice by going to the\n"
+		 "'Settings...' item under the 'File' menu.",
+		 "Greenstone Digital Library Software", MB_OK);
+    } else { // netscape used
+      MessageBox(Window,
+		 "Failed to start Netscape. It seems that Netscape has\n"
+		 "been removed or corrupted. You need Netscape\n"
+		 "installed to use the Greenstone Digital Library\n"
+		 "software on non-networked machines.\n\n"
+		 "You can install Netscape by choosing\n"
+		 "'Install Netscape 4.05' from the 'File' menu.",
+		 "Greenstone Digital Library Software", MB_OK);
+    }
+    
+  } else {
+    MessageBox(Window,
+	       "Failed to start your browser. This may have been\n"
+	       "because there was not enough memory available. Shut\n"
+	       "down all other applications you have running, and\n"
+	       "try again.",
+	       "Greenstone Digital Library Software", MB_OK);
+  }
+  
+  return 0; // failed
+}
+
+
+void install_netscape (HWND Window) {
+  char installpath[256];
+  
+  // try to find the browser directory
+  strcpy (installpath, gs_gsdlhome); // use gsdlhome to find the CD-ROM drive...
+  
+  // remove any slashes from the pathname
+  int len = strlen (installpath);
+  while ((len > 0) && ((installpath[len-1] == '\\') || (installpath[len-1] == '/'))) {
+    len--;
+  }
+  // remove '\gsdl'
+  len = len - 5;
+  installpath[len] = '\0';
+  
+  strcat (installpath, "\\netscape\\");
+  
+  // check this directory
+  if (!cstrcheckdir(installpath)) {
+    return;
+  }
+  
+  // get the operating system information
+  OSVERSIONINFO osver;
+  memset(&osver, 0, sizeof(OSVERSIONINFO));
+  osver.dwOSVersionInfoSize = sizeof(OSVERSIONINFO);
+  GetVersionEx(&osver);
+
+  // get the install program for this operating system
+  if (osver.dwPlatformId == VER_PLATFORM_WIN32s) {
+    strcat (installpath, "n16e405.exe");
+  } else {
+    strcat (installpath, "n32e405.exe");
+  }
+  
+  // run the install program
+  int res = WinExec (installpath, SW_SHOW);
+  if (res == 0) {
+    // out of resources or memory
+    MessageBox (Window,
+		"Did not have enough resources or memory to run the\n"
+		"install program. Try shutting down all other programs\n"
+		"and trying again.",
+		"Greenstone Digital Library Software",
+		MB_OK|MB_APPLMODAL);
+    
+  } else if (res == ERROR_BAD_FORMAT) {
+    // executable image is corrupt -- ????
+    MessageBox (Window,
+		"The install program seems to be corrupt.",
+		"Greenstone Digital Library Software",
+		MB_OK|MB_APPLMODAL);
+    
+  } else if ((res == ERROR_FILE_NOT_FOUND) || (res == ERROR_PATH_NOT_FOUND)) {
+    // couldn't find the executable -- ????
+    MessageBox (Window,
+		"Could not find the install program. Try installing\n"
+		"Netscape from the Greenstone Digital Library\n"
+		"software's install program.",
+		"Greenstone Digital Library Software",
+		MB_OK|MB_APPLMODAL);
+  }
+}
+
+void open_help () {
+  char topdir[256], cmd[256];
+  
+  // try to find the browser directory
+  strcpy (topdir, gs_gsdlhome); // use gsdlhome to find the CD-ROM drive...
+  strcpy (cmd, "notepad ");
+  
+  // remove any slashes from the pathname
+  int len = strlen (topdir);
+  while ((len > 0) && ((topdir[len-1] == '\\') || (topdir[len-1] == '/'))) {
+    len--;
+  }
+  // remove the '\gsdl'
+  len = len - 5;
+  topdir[len] = '\0';
+
+  strcat (topdir, "\\README.txt");
+  strcat (cmd, topdir);
+  
+  WinExec (cmd, SW_SHOW);
+}
+
+
+long __stdcall WndProc(HWND Window, UINT Message, WPARAM WParam, LPARAM LParam) {
+  switch(Message) {
+    // Constant Messages
+  case WM_CREATE:
+    break;
+    
+  case WM_CHAR:
+    // if the user hit the return key and if this is not a
+    // repeat key press then attempt to start a browser
+    if ((WParam == '\r') && !(LParam & 0xe0000000)) {
+      if (enter_library (Window) && !gs_show_console) 
+	ShowWindow(Window,SW_MINIMIZE);
+    }
+    
+  case WM_COMMAND:
+    if ((HWND)LParam == GoButton) {
+      // attempt to start a browser
+      if (enter_library (Window) && !gs_show_console) 
+	ShowWindow(Window,SW_MINIMIZE);
+
+    } else {
+      switch (LOWORD(WParam)) {
+      case ID_PROJECT_SETTINGS:
+	Settings_Dialog(Window, netscapeneeded);
+	break;
+      case ID_INSTALL_NETSCAPE:
+	install_netscape(Window);
+	break;
+      case IDHELP:
+	open_help();
+	break;
+      case ID_PROJECT_EXIT:
+        finish_up();
+        break;
+      default:
+        break;
+      }
+    }
+    break;
+    
+  case WM_PAINT:
+    handle_painting(Window);
+    break;
+    
+  case HTTP_SERVER_MSG:
+    ProcessHTTPServerMsg(WParam, LParam);
+    break;
+    
+  case WM_DESTROY:
+    finish_up();
+    break;
+    
+  default:
+    //Unhandled Messages end up here (DefWindowProc)
+    return DefWindowProc(Window, Message, WParam, LParam);
+  }
+  
+  return(0);
+}
+
+// returns 0 if successful,
+// otherwise a WSA error number
+int tryinitnetwork (HANDLE Instance, HWND MsgWindow, char *path) {
+  // try to load winsock
+  int err = d_LoadWinsock (path);
+  
+  if (err == D_NOERROR) {
+    // next, try to init netio
+    err = InitNetIO();
+    
+    if (err == 0) {
+      // next, try to start the http server
+      err = StartHTTPServer(MsgWindow);
+      
+      if (err == 0) {
+	// finally, get the host name (no error value
+	// is returned from this function)
+	GetLocalName(Instance);
+	
+      } else {
+	// couldn't start the http server, 
+	// deinit netio and unload winsock
+	CleanUpNetIO();
+	d_UnloadWinsock();
+      }
+      
+    } else {
+      // couldn't init netio, unload winsock
+      d_UnloadWinsock();
+    }
+    
+  } else {
+    // couldn't load winsock
+    err = WSASYSNOTREADY;
+  }
+  
+  return err;
+}
+
+
+// inits all the network functionality
+// returns 0 if an unrecoverable error occurred,
+// and 1 if everything successfully initialised.
+int initnetwork (HANDLE Instance, HWND MsgWindow) {
+  OSVERSIONINFO osver;
+  
+  // get the operating system information
+  memset(&osver, 0, sizeof(OSVERSIONINFO));
+  osver.dwOSVersionInfoSize = sizeof(OSVERSIONINFO);
+  GetVersionEx(&osver);
+  
+  // first try to initialise the network with system installed
+  // networking software
+  startbrowserdir[0] = '\0';
+  netscapeneeded = 0;
+  //	int err = tryinitnetwork (Instance, MsgWindow, NULL);
+  int err = 1;
+  
+  // if an error occurred, try again with billsock
+  if (err != 0) {
+    // get the path of billsock (current directory + ("net16" | "net32"))
+    char winsockpath[1024]; // HARD LIMIT!!!!
+    
+    find_location();
+    strcpy (winsockpath, data_location);
+    
+    // remove all the slashes at the end of billpath
+    int len = strlen (winsockpath);
+    while ((len > 0) && ((winsockpath[len-1] == '\\') || (winsockpath[len-1] == '/'))) {
+      len--;
+    }
+    winsockpath[len] = '\0';
+    
+    if (osver.dwPlatformId == VER_PLATFORM_WIN32s)
+      strcat (winsockpath, "\\net16\\");
+    else {
+      strcat (winsockpath, "\\net32\\");
+      strcpy (startbrowserdir, winsockpath);
+    }
+    
+    // try again
+    err = tryinitnetwork (Instance, MsgWindow, winsockpath);
+    
+    if (err == 0) {
+      // we will need to use netscape on 95/98/NT machines
+      if (osver.dwPlatformId != VER_PLATFORM_WIN32s) {
+	netscapeneeded = 1;
+	// make sure the browser state is in step with the network condition
+	gs_check_browser_settings (netscapeneeded);
+      }
+      
+      // check to see if a browser is running
+      if (browserrunning(gs_browser_exe) == NO_ERROR) {
+	MessageBox (NULL, 
+		    "The Greenstone Digital Library software has detected a\n"
+		    "running browser. To let our networking software\n"
+		    "correctly initialize, please shut down your browser\n"
+		    "and restart the Greenstone Digital Library software.",
+		    "Greenstone Digital Library Software", MB_OK|MB_APPLMODAL);
+	PostMessage(MsgWindow,WM_CLOSE,0,0);
+	finish_up(); // will unload winsock
+	exit(0);      // nothing more to do
+      }
+    }
+  }
+  
+  // if an error occurred display a nice error message
+  if (err != 0) {
+    
+    // WSASYSNOTREADY	Couldn't load winsock
+    // WSAVERNOTSUPPORTED	The version of Windows Sockets support requested 
+    //			is not provided by this particular Windows Sockets implementation.
+    // WSAEINVAL	The Windows Sockets version specified by the application 
+    //			is not supported by this DLL.
+    // WSAEPROTONOSUPPORT	The specified protocol is not supported.
+    // WSAEPROTOTYPE	The specified protocol is the wrong type for this socket.
+    // WSAESOCKTNOSUPPORT	The specified socket type is not supported in 
+    //		this address family.
+    // WSANOTINITIALISED	A successful WSAStartup must occur before 
+    //		using this function.
+    // WSAENETDOWN	The Windows Sockets implementation 
+    //		has detected that the network subsystem has failed.
+    // WSAEADDRINUSE 	The specified address 
+    //		is already in use. (See the SO_REUSEADDR socket option under setsockopt.)
+    // WSAEFAULT	The namelen argument is too small (less 
+    //		than the size of a struct sockaddr).
+    // WSAEINPROGRESS	A blocking Windows Sockets call is in progress.
+    // WSAEAFNOSUPPORT	The specified address family is not supported 
+    //		by this protocol.
+    // WSAENOBUFS	Not enough buffers available, too many connections.
+    // WSAENOTSOCK	The descriptor is not a socket.
+    // WSAEISCONN	The socket is already connected.
+    // WSAEMFILE	No more file descriptors are available.
+    // WSAEOPNOTSUPP	The referenced socket is not of a type that 
+    //		supports the listen operation.
+    
+    // get a text version of the error number
+    char errstr[256];
+    
+    switch (err) {
+    case WSASYSNOTREADY: strcpy(errstr, "WSASYSNOTREADY"); 
+      break;
+    case WSAVERNOTSUPPORTED: strcpy (errstr, "WSAVERNOTSUPPORTED");
+      break;
+    case WSAEINVAL: strcpy (errstr, "WSAEINVAL");
+      break;
+    case WSAEPROTONOSUPPORT: strcpy (errstr, "WSAEPROTONOSUPPORT");
+      break;
+    case WSAEPROTOTYPE: strcpy (errstr, "WSAEPROTOTYPE");
+      break;
+    case WSAESOCKTNOSUPPORT: strcpy (errstr, "WSAESOCKTNOSUPPORT");
+      break;
+    case WSANOTINITIALISED: strcpy (errstr, "WSANOTINITIALISED");
+      break;
+    case WSAEFAULT: strcpy (errstr, "WSAEFAULT");
+      break;
+    case WSAEAFNOSUPPORT: strcpy (errstr, "WSAEAFNOSUPPORT");
+      break;
+    case WSAENOTSOCK: strcpy (errstr, "WSAENOTSOCK");
+      break;
+    case WSAEISCONN: strcpy (errstr, "WSAEISCONN");
+      break;
+    case WSAEOPNOTSUPP: strcpy (errstr, "WSAEOPNOTSUPP");
+      break;
+    case WSAENETDOWN: strcpy (errstr, "WSAENETDOWN");
+      break;
+    case WSAEADDRINUSE: strcpy (errstr, "WSAEADDRINUSE");
+      break;
+    case WSAEINPROGRESS: strcpy (errstr, "WSAEINPROGRESS");
+      break;
+    case WSAENOBUFS: strcpy (errstr, "WSAENOBUFS");
+      break;
+    case WSAEMFILE: strcpy (errstr, "WSAEMFILE");
+      break;
+    default: sprintf (errstr, "WSA ERROR: %i", err);
+      break;
+    }
+    
+    // create a nice error message
+    char message[2048];
+    
+    switch (err) {
+    case WSAEADDRINUSE:
+      // couldn't bind socket
+      sprintf (message, "Could not find a free socket.", errstr);
+      break;
+      
+    default:
+      // cannot init winsock
+      sprintf (message, "Could not initialize the network\n"
+	       " (Reason: %s).", errstr);
+      break;
+    }
+    
+    strcat (message, "\n\n"
+	    // no static library for this collection
+	    //			"You will still be able to use the static version of this\n"
+	    //			"collection, however, searching will be unavailable.");
+	    "Please submit a bug report using the support.htm file\n"
+	    "on the CD-ROM.");
+    
+    MessageBox (NULL, message, "Greestone Digital Library Software", MB_OK);
+  }
+  
+  return (err == 0);
+}
+
+
+void log_computer_info () {
+  char tmpstr[1024];
+  
+  // get various information about this computer
+  if (gs_keep_log || gs_show_console) {
+    // get operating system information
+    OSVERSIONINFO osver;
+    osver.dwOSVersionInfoSize = sizeof(OSVERSIONINFO);
+    
+    if (GetVersionEx(&osver)) {
+      if (osver.dwPlatformId == VER_PLATFORM_WIN32s) {
+	log_message ("Platform: win32s\n");
+      } else if (osver.dwPlatformId == VER_PLATFORM_WIN32_WINDOWS) {
+	log_message ("Platform: windows 95\n");
+      } else if (osver.dwPlatformId == VER_PLATFORM_WIN32_NT) {
+	log_message ("Platform: windows NT\n");
+      }
+      
+      sprintf (tmpstr, "MajorVersion: %i\n", (int)osver.dwMajorVersion);
+      log_message(tmpstr);
+      sprintf (tmpstr, "MinorVersion: %i\n", (int)osver.dwMinorVersion);
+      log_message(tmpstr);
+      sprintf (tmpstr, "BuildNumber: %i\n", (int)osver.dwBuildNumber);
+      log_message(tmpstr);
+      sprintf (tmpstr, "CSDVersion: %s\n\n", osver.szCSDVersion);
+      log_message(tmpstr);
+    }
+    
+    // get memory information
+    MEMORYSTATUS memstatus;
+    memstatus.dwLength = sizeof(MEMORYSTATUS);
+    GlobalMemoryStatus(&memstatus);
+    
+    sprintf (tmpstr, "TotalPhys: %i Meg\n", (int)(memstatus.dwTotalPhys/1024/1024));
+    log_message (tmpstr);
+    sprintf (tmpstr, "AvailPhys: %i Meg\n", (int)(memstatus.dwAvailPhys/1024/1024));
+    log_message (tmpstr);
+    sprintf (tmpstr, "TotalPageFile: %i Meg\n", (int)(memstatus.dwTotalPageFile/1024/1024));
+    log_message (tmpstr);
+    sprintf (tmpstr, "AvailPageFile: %i Meg\n", (int)(memstatus.dwAvailPageFile/1024/1024));
+    log_message (tmpstr);
+    sprintf (tmpstr, "TotalVirtual: %i Meg\n", (int)(memstatus.dwTotalVirtual/1024/1024));
+    log_message (tmpstr);
+    sprintf (tmpstr, "AvailVirtual: %i Meg\n\n", (int)(memstatus.dwAvailVirtual/1024/1024));
+    log_message (tmpstr);
+    
+    // log the version number
+    log_message("GSDL version: " VERSIONSTRING "\n");
+  }
+  
+}
+
+
+int __stdcall WinMain(HANDLE Instance, HANDLE /*PrevInstance*/, LPSTR CmdLineStr, int /*CmdShow*/) {
+  HWND MainWindow;  MSG Message;  WNDCLASS MainClass;  
+  
+  //Create a window class
+  MainClass.style = CS_HREDRAW | CS_VREDRAW;
+  MainClass.lpfnWndProc = WndProc;
+  MainClass.cbClsExtra = 0;
+  MainClass.cbWndExtra = 0;
+  MainClass.hInstance = Instance;
+  MainClass.hIcon     = LoadIcon(Instance, MAKEINTRESOURCE(TRAY_ICON));
+  MainClass.hCursor   = LoadCursor(NULL, IDC_ARROW);
+  MainClass.hbrBackground = GetStockObject(WHITE_BRUSH);
+  MainClass.lpszMenuName = MAKEINTRESOURCE(Main_Menu);
+  MainClass.lpszClassName = "Greenstone Digital Library Software";
+  if (!RegisterClass(&MainClass))
+    return 0;
+  
+  // Load various bitmaps
+  coltitlebitmap = LoadBitmap (Instance, MAKEINTRESOURCE(GSDL_COL_TITLE));
+  logobitmap= LoadBitmap (Instance, MAKEINTRESOURCE(GSDL_LOGO));
+  
+  // Create the main window
+  MainWindow = CreateWindow("Greenstone Digital Library Software", 
+			    "Greenstone Digital Library Software",
+			    WS_OVERLAPPED | WS_CAPTION | WS_SYSMENU |
+			    WS_MINIMIZEBOX,
+			    CW_USEDEFAULT, CW_USEDEFAULT, 
+			    MAINWINDOWWIDTH + WASTEWIDTH, 
+			    MAINWINDOWHEIGHT + WASTEHEIGHT,
+			    NULL, NULL, Instance, NULL);
+  
+  text_rect.left = 0;  
+  text_rect.top = VERSIONY+VERSIONHEIGHT + 5;
+  text_rect.right = MAINWINDOWWIDTH;
+  text_rect.bottom = MAINWINDOWHEIGHT-RESTHEIGHT - 1;
+  
+  window_state = undefined_window;
+  
+  ShowWindow(MainWindow, SW_SHOW); 
+  set_location(CmdLineStr);
+  
+  // get ready to draw the main window
+  RECT windowRect;
+  GetClientRect (MainWindow, &windowRect);
+  HDC pdc = GetDC (MainWindow);
+  
+  // retrieve and save the text metrics
+  TEXTMETRIC tm;
+  GetTextMetrics(pdc, &tm);
+  line_spacing = tm.tmHeight + tm.tmExternalLeading;
+  GSDL_Window = MainWindow;
+  
+  // draw the main window containing an init message
+  infostring = strinit;
+  paint_window (pdc, windowRect);
+  DWORD lastcheck = GetTickCount();
+  
+  // init various modules
+  read_settings (0); // don't know if netscape is needed at this point
+  if (!gsdl_init()) // quit if can't initialise the library correctly
+    exit (0);
+  initstartbrowser();
+  log_computer_info ();
+  
+  // attempt to initialise the network
+  have_networking = initnetwork (Instance, MainWindow);
+  if (!have_networking) {
+    MessageBox(NULL,
+	       "Failed to initialize networking.",
+	       "Greenstone Digital Library Software",MB_OK|MB_SYSTEMMODAL);	
+    exit (0); // there is no static library for this collection
+  }
+  // show the initialising message for at least 1 second
+  while (DiffTickCounts (lastcheck, GetTickCount()) < 1000) {
+    Sleep (1);
+  }
+  
+  // if autoenter is enabled attempt to start a browser
+  if (have_networking && gs_auto_enter) {
+    infostring = strsb;
+    paint_window (pdc, windowRect);
+    lastcheck = GetTickCount();
+    if (enter_library (MainWindow) && !gs_show_console) {
+      // stay maximised for at least 1 second
+      while (DiffTickCounts (lastcheck, GetTickCount()) < 1000) Sleep (1);
+      ShowWindow(MainWindow,SW_MINIMIZE);
+    }
+  }
+  
+  // remove the last info message
+  infostring = strnothing;
+  paint_window (pdc, windowRect);
+  
+  // add the "enter library" button
+  GoButton = CreateWindow( 
+			  "BUTTON",             // predefined class 
+			  "Enter Library",      // button text 
+			  WS_VISIBLE | WS_CHILD | BS_DEFPUSHBUTTON,  // styles 
+			  ENTERBUTTONX,         // starting x position 
+			  ENTERBUTTONY,         // starting y position 
+			  ENTERBUTTONWIDTH,     // button width 
+			  ENTERBUTTONHEIGHT,    // button height 
+			  MainWindow,           // parent window 
+			  NULL,                 // No menu 
+			  Instance, 
+			  NULL);                // pointer not needed 
+  
+  if (have_networking) statusstring = strenterlib;
+  else statusstring = strenterstaticlib;
+  paint_window (pdc, windowRect);
+  
+  // release the DC used to display the init messages
+  ReleaseDC (MainWindow, pdc);
+  
+  //Message loop
+  lastcheck = GetTickCount();
+  int seenbrowser = 0;	// if we see a browser then it disappears
+  // we will ask if they want to close the
+  // the library down
+  char last_gs_browser_exe[MAX_FILENAME_SIZE]; // need to notice when the
+  // browser setting changes
+  strcpy (last_gs_browser_exe, gs_browser_exe);
+  
+  for (;;) {
+    if (PeekMessage(&Message, NULL, 0, 0, PM_REMOVE)) {
+      if (Message.message == WM_QUIT) break;
+      TranslateMessage(&Message); /* translate keyboard messages */
+      DispatchMessage(&Message);  /* return control to Windows NT */
+      
+    } else {
+      if (DiffTickCounts (lastcheck, GetTickCount()) > 500) {
+	// make sure the browser hasn't changed
+	if (strcmp (last_gs_browser_exe, gs_browser_exe) != 0) {
+	  seenbrowser = 0;
+	  strcpy (last_gs_browser_exe, gs_browser_exe);
+	}
+	
+	// do check
+	lastcheck = GetTickCount();
+	if (browserrunning(gs_browser_exe) == NO_ERROR) {
+	  // we were able to connect to a browser
+	  seenbrowser = 1;
+	  
+	  // see if the enter library button has been pressed
+	  // with nothing happening for 20 seconds
+	  if ((enterlib_libaccessnum >= 0) && 
+	      (enterlib_libaccessnum == libaccessnum) &&
+	      (DiffTickCounts (enterlib_time, GetTickCount()) > 20000)) {
+	    // something could be wrong!! (most likely cause is a proxy)
+	    MessageBox(NULL,
+		       "Your browser does not seem to be responding to the 'Enter Library'\n"
+		       "request. Try turning off all proxy servers in your browser settings.",
+		       "Greenstone Digital Library Software",MB_OK|MB_SYSTEMMODAL);
+	    enterlib_libaccessnum = -1;
+	  }
+	  
+	  
+	} else {
+	  // no browser was found
+	  if (seenbrowser) {
+	    // we have seen a browser in the past
+	    if (MessageBox(NULL,
+			   "The Greenstone Digital Library software has noticed that you\n"
+			   "shut down your browser. Would you also like to shut down the\n"
+			   "Greenstone Digital Library software?",
+			   "Greenstone Digital Library Software",MB_YESNO|MB_SYSTEMMODAL) == IDYES)
+	      PostMessage(MainWindow,WM_CLOSE,0,0);
+	  }
+	  seenbrowser = 0;
+	}
+      }
+      
+      Sleep (1); // millisecond
+    }
+  }
+  
+  return Message.wParam;
+}
Index: /tags/gsdl-21/gsdl/src/w32server/httpreq.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/w32server/httpreq.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/w32server/httpreq.cpp	(revision 699)
@@ -0,0 +1,439 @@
+/*
+Copyright (C) 1996
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+The author can be contacted via Email at bmorin@wpi.edu
+*/
+#include <windows.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <memory.h>
+#pragma hdrstop
+#include "parse.h"
+#include "netio.h"
+#include "settings.h"
+#include "httpreq.h" //Had to put myself here because of the global types...
+#include "httpsrv.h"
+#include "httpsend.h"
+#include "cgiwrapper.h"
+
+/*
+Implementation Notes:
+
+HTTP field names, method and version strings are converted to upper case
+right after being read from the client in order to allow case insensitive
+string comparisons to be done on them.  Since these fields are worked with a
+lot, this should help performance.
+*/
+
+//Private Data and declarations
+#define IO_BUFFER_SIZE 16384        //16K IO Buffer
+#define MAX_HTTP_LINE_LEN 1024      //Max length of line in a header of 1024
+#define MAX_HTTP_FIELD_NAME_LEN 128 //Max length of name field in line
+#define MAX_HTTP_FIELD_LEN 1024     //Max length of data in line
+
+//Private Function Declarations with Return Contstants
+
+/*
+Function Name: DispatchRequest
+Purpose: Manages having the request parsed, then sent to the right function
+         to send a response or handle an error.
+Parameters:
+         ClientSocket - Socket the client is on
+	 ClientSockAddr - Address of client
+	 AddrLen - Length of address of client
+	 IOBuffer - Pointer to buffer allocated for IO operations
+	 ThreadNum - Number of thread that called this function for debugging purposes
+Notes:   I'm still playing with the keep alive support.  I commented out
+	 the stuff for giving a client a timeout because I was unable to detect
+	 disconnects.
+More Notes: Not sure if this organization will allow me to easily add support
+	    for ISAPI filter DLLs.
+*/
+void DispatchRequest(SOCKET ClientSocket, SOCKADDR_IN ClientSockAddr, int AddrLen, BYTE *IOBuffer);
+
+/*
+Function Name: Get HTTP Headers
+Purpose: Manages having the request parsed, then sent to the right function
+         to send a response or handle an error.
+Parameters:
+	RequestInfo - Request information structure (see httpreq.h)
+	RequestFields - HTTP request fields structure (see httpreq.h)
+Returns: GH_ERROR on error (diconnect, bad data, Windows in a bad mood, etc.)
+			GH_UNKNOWN_VERSION if the version number is not HTTP/0.9 or HTTP/1.x
+			GH_SIMPLE_REQUEST on a properly formated HTTP/0.9 request
+			GH_10_REQUEST on a properly formated HTTP/1.x request
+*/
+int GetHTTPHeaders(RequestInfoT &RequestInfo, RequestFieldsT &RequestFields);
+#define GH_ERROR           -1
+#define GH_UNKNOWN_VERSION  0
+#define GH_SIMPLE_REQUEST   1
+#define GH_10_REQUEST       2
+
+/*
+Function Name: Clean Up HTTP Headers
+Purpose: Cleans up memory dynamicly allocated for headers
+Parameters:
+	RequestInfo - Request information structure (see httpreq.h)
+	RequestFields - HTTP request fields structure (see httpreq.h)
+Returns: Nothing
+*/
+void CleanUpHTTPHeaders(RequestInfoT &RequestInfo, RequestFieldsT &RequestFields);
+
+/*
+Function Name: Split Query
+Purpose: Splits the file and query part of a URI.  In other words, it
+	 puts the parts before and after the "?" in differnet strings.
+Parameters:
+	 URIStr - The requested URI
+	 FileStr - String to contain the name of the path + file part of the URI
+	 QueryStr - String to contain the query part of the URI
+Returns: TRUE if there is a query, else FALSE
+*/
+BOOL SplitQuery(char *URIStr, char *FileStr, char *QueryStr, int ThreadNum);
+
+/*
+Function Name: Get File
+Purpose: Attempts to find a given file, including looking for index.html.
+	 Updates the given URI string so it points to the true document location
+Parameters:
+	 FilePath - Path of file, may be modified to best reflect the retrived file
+				  or directory
+	 URIStr - URI string, minus the query
+Returns: GF_ERROR on error
+			GF_FILE_FOUND on success
+			GF_INDEX_FOUND if file is a directory with an index.html file in it
+			GF_DIRECTORY if file is a directory
+			GF_FILE_NOT_FOUND if file was found
+*/
+
+/*
+Function Name: Process Simple Request
+Purpose: Sends a reply to a HTTP 0.9 "simple" request
+Parameters:
+	ClientSocket - Socket the client is on
+	RequestInfo - Structure storing the parsed headers
+	IOBuffer - Pointer to buffer allocated for IO operations
+	TheadNum - Number of calling thread for debugging
+Notes: I should really test this and see if it works...
+*/
+void ProcessSimpleRequest(RequestInfoT &RequestInfo, RequestFieldsT &RequestFields);
+
+//Public Functions
+/******************************************************************************/
+void RequestThread(RequestThreadMessageT *Parameters) {
+  SOCKADDR_IN ClientSockAddr;
+  SOCKET ClientSocket;
+  int AddrLen;
+  //Allocate an IO buffer for this thread
+  BYTE *IOBuffer = new BYTE[IO_BUFFER_SIZE];
+
+  //Get the parameters for the request
+  ClientSocket = Parameters->ClientSocket;
+  ClientSockAddr = Parameters->ClientSockAddr;
+  AddrLen = Parameters->AddrLen;
+  DispatchRequest(ClientSocket, ClientSockAddr, AddrLen, IOBuffer);
+}
+/******************************************************************************/
+
+//Private Functions
+
+/******************************************************************************/
+void DispatchRequest(SOCKET ClientSocket, SOCKADDR_IN ClientSockAddr, int AddrLen,  BYTE *IOBuffer) {
+  RequestInfoT RequestInfo;
+  RequestFieldsT RequestFields;
+
+  // TrayAddConnection();
+
+  //Setup the RequestInfo structure
+  memset(&RequestInfo, 0, sizeof(RequestInfoT));
+  RequestInfo.ThreadNum = 0;
+  RequestInfo.IOBuffer = IOBuffer;
+  RequestInfo.IOBufferSize = IO_BUFFER_SIZE;
+  RequestInfo.ClientSocket = ClientSocket;
+  RequestInfo.ClientSockAddr = ClientSockAddr;
+  RequestInfo.AddrLen = AddrLen;
+  RequestInfo.KeepAlive = FALSE;
+  
+  int GetHeadersResult;
+  do {
+    //Get Headers
+    GetHeadersResult = GetHTTPHeaders(RequestInfo, RequestFields);
+    //Figure out what version we're dealing with and deal with it
+    switch (GetHeadersResult) {
+    case GH_SIMPLE_REQUEST :
+      SendHTTPError(400, "HTTP Request not supported", "Only 1.x requests supported", RequestInfo, RequestFields);
+      // TrayIncNumServed();
+      break;
+    case GH_10_REQUEST :
+      ExamineURIStr(RequestFields.URIStr,&RequestInfo,&RequestFields);
+      // TrayIncNumServed();
+      break;
+    case GH_UNKNOWN_VERSION :
+      SendHTTPError(400, "HTTP Version not supported", "Only 1.x requests supported", RequestInfo, RequestFields);
+      // TrayIncNumServed();
+      break;
+    case GH_ERROR:
+      //Disconnect
+      RequestInfo.KeepAlive = FALSE;
+      break;
+    }
+    CleanUpHTTPHeaders(RequestInfo, RequestFields);
+  } while (0/*RequestInfo.KeepAlive == TRUE*/);
+  //Close connection
+  CloseSocket(RequestInfo.ClientSocket);
+  // TrayRemoveConnection();
+}
+
+/******************************************************************************/
+int GetHTTPHeaders(RequestInfoT &RequestInfo, RequestFieldsT &RequestFields) {
+  //Parsing and IO buffers
+  char CurLine[NETIO_MAX_LINE];
+  char NextLine[NETIO_MAX_LINE];
+  char FieldNameStr[MAX_HTTP_FIELD_NAME_LEN];
+  char FieldValStr[MAX_HTTP_FIELD_LEN];
+  
+  //Parsing and IO working vars
+  int ReadBufferIndex;
+  int DataInBuffer;
+  int Start;
+  int End;
+  int Len;
+
+  //Clear all the fields
+  memset(&RequestFields, 0, sizeof(RequestFieldsT));
+
+  ReadBufferIndex = 0;
+  DataInBuffer = 0;
+  
+  //Get First Line
+  if (GetLine(CurLine, RequestInfo.ClientSocket, RequestInfo.IOBuffer,
+	      RequestInfo.IOBufferSize, ReadBufferIndex, DataInBuffer,
+	      RequestInfo.ThreadNum) != 0) return GH_ERROR;
+  do {//Get Next Line, append it if the first charactor is space
+    if(GetLine(NextLine, RequestInfo.ClientSocket, RequestInfo.IOBuffer,
+	       RequestInfo.IOBufferSize, ReadBufferIndex, DataInBuffer,
+	       RequestInfo.ThreadNum) != 0) return GH_ERROR;
+    if ((NextLine[0] == ' ') || (NextLine[0] == '\t'))
+      strcat(CurLine, NextLine);
+  } while ((NextLine[0] == ' ') || (NextLine[0] == '\t'));
+  //Method String (first word)
+  Start = 0;
+  GetWord(RequestFields.MethodStr, CurLine, Start, End);
+  CharUpper(RequestFields.MethodStr);
+  
+  //Version String (last word)
+  GetLastWord(RequestFields.VersionStr, CurLine, Start);
+  CharUpper(RequestFields.VersionStr);
+
+  if (strncmp(RequestFields.VersionStr, "HTTP/", 5) != 0) {
+    //No version, assume simple request
+    //part after method is URI
+    memcpy(RequestFields.URIStr, CurLine + End, strlen(CurLine) + 1 - End);
+    return GH_SIMPLE_REQUEST;
+  }
+
+  //URI String (in between End of first and Start of last)
+  //<Method> <WhiteSpace> <URI> <WhiteSpace> <Version> <CRLF>
+  //                  End^             Start^
+  int URIStrLen = Start - End;;
+  if (URIStrLen > ReqURIStrLen-1) URIStrLen = ReqURIStrLen - 1;
+  memcpy(RequestFields.URIStr, CurLine + End, URIStrLen);
+  RequestFields.URIStr[URIStrLen] = 0;
+  TrimRight(RequestFields.URIStr); //Remove trailing space
+  
+  //Only accept requests from HTTP/0.9 or HTTP/1.X clients, we'll
+  //assume that anything else will require an upgrade or patch
+  if (strncmp(RequestFields.VersionStr, "HTTP/1.", 7) != 0)
+    return GH_UNKNOWN_VERSION;
+  
+  //Get the rest of the lines
+  
+  strcpy(CurLine, NextLine);
+  
+  while (CurLine[0] != 0) {//Blank Line, we're done
+    do {//Get Next Line, append it if the first charactor is space
+      if (GetLine(NextLine, RequestInfo.ClientSocket, RequestInfo.IOBuffer,
+		  RequestInfo.IOBufferSize, ReadBufferIndex, DataInBuffer,
+		  RequestInfo.ThreadNum) != 0)
+	return GH_ERROR;
+      if ((NextLine[0] == ' ') || (NextLine[0] == '\t'))
+	strcat(CurLine, NextLine);
+    } while ((NextLine[0] == ' ') || (NextLine[0] == '\t'));
+    
+    Start = 0;
+    GetWord(FieldNameStr, CurLine, Start, End);
+    CharUpper(FieldNameStr);
+    
+    Len = strlen(CurLine) - End;
+    memcpy(FieldValStr, CurLine + End, Len);
+    FieldValStr[Len] = 0;
+    
+    //Process it
+    //In order of expected commonality
+    //All constants are in canonized, thus in upper case and case sensitive
+    //comparisons are used
+    //--Just About Always--
+    if (strcmp("ACCEPT:", FieldNameStr) == 0) {
+      if (RequestFields.AcceptStr[0] == '\0') {
+        strncpy(RequestFields.AcceptStr, FieldValStr, ReqAcceptStrLen - 1);
+	//  		if (Len >= ReqAcceptStrLen)
+	//        if (log_debug) log_message("Accept field truncated");
+      }
+      else {
+       	//Append it with a comma
+        int AcceptStrLen = strlen(RequestFields.AcceptStr);
+        if ((ReqAcceptStrLen - AcceptStrLen) >= 10) {
+          strncat(RequestFields.AcceptStr, ", ", ReqAcceptStrLen - AcceptStrLen - 1);
+          strncat(RequestFields.AcceptStr, FieldValStr, ReqAcceptStrLen - AcceptStrLen - 3);
+	}
+      }
+    }
+    else if (strcmp("DATE:", FieldNameStr) == 0) {
+      strncpy(RequestFields.DateStr, FieldValStr, ReqDateStrLen - 1);
+      //			if (Len >= ReqDateStrLen) LogError("Date field truncated");
+    }
+    else if (strcmp("USER-AGENT:", FieldNameStr) == 0) {
+      strncpy(RequestFields.UserAgentStr, FieldValStr, ReqUserAgentStrLen - 1);
+      //			if (Len >= ReqUserAgentStrLen) {
+      //         	LogError("User Agent field truncated, value follows");
+      //            LogError(RequestFields.UserAgentStr);
+      //            }
+    }
+    else if (strcmp("CONNECTION:", FieldNameStr) == 0) {
+      strncpy(RequestFields.ConnectionStr, FieldValStr, ReqConnectionStrLen - 1);
+      //			if (Len >= ReqConnectionStrLen) LogError("Connection field truncated");
+    }
+    //--Sometimes--
+    else if (strcmp("ACCEPT-LANGUAGE:", FieldNameStr) == 0) {
+      strncpy(RequestFields.AcceptLangStr, FieldValStr, ReqAcceptLangStrLen - 1);
+      //			if (Len >= ReqAcceptLangStrLen) LogError("Accept-Language field truncated");
+    }
+    else if (strcmp("REFERER:", FieldNameStr) == 0) {
+      strncpy(RequestFields.RefererStr, FieldValStr, ReqRefererStrLen - 1);
+      //			if (Len >= ReqRefererStrLen) LogError("Referer field truncated");
+    }
+    else if (strcmp("IF-MODIFIED-SINCE:", FieldNameStr) == 0) {
+      strncpy(RequestFields.IfModSinceStr, FieldValStr, ReqIfModSinceStrLen - 1);
+      //			if (Len >= ReqIfModSinceStrLen) LogError("If Modified Since field truncated");
+    }
+    //--Uncommon--
+    else if (strcmp("FROM:", FieldNameStr) == 0) {
+      strncpy(RequestFields.FromStr, FieldValStr, ReqFromStrLen - 1);
+      //			if (Len >= ReqFromStrLen) LogError("From field truncated");
+    }
+    else if (strcmp("MIME-VERSION:", FieldNameStr) == 0) {
+      strncpy(RequestFields.MIMEVerStr, FieldValStr, ReqMIMEVerStrLen - 1);
+      //			if (Len >= ReqMIMEVerStrLen) LogError("MIME Version field truncated");
+    }
+    else if (strcmp("PRAGMA:", FieldNameStr) == 0) {
+      strncpy(RequestFields.PragmaStr, FieldValStr, ReqPragmaStrLen - 1);
+      //			if (Len >= ReqPragmaStrLen) LogError("Pragma field truncated");
+    }
+    //--Special case--
+    else if (strcmp("AUTHORIZATION:", FieldNameStr) == 0) {
+      strncpy(RequestFields.AuthorizationStr, FieldValStr, ReqAuthorizationStrLen - 1);
+      //			if (Len >= ReqAuthorizationStrLen) LogError("Authorization field truncated");
+    }
+    else if (strcmp("CONTENT-LENGTH:", FieldNameStr) == 0) {
+      strncpy(RequestFields.ContentLengthStr, FieldValStr, ReqContentLengthStrLen - 1);
+      //			if (Len >= ReqContentLengthStrLen) LogError("Content Length field truncated");
+    }
+    else if (strcmp("CONTENT-TYPE:", FieldNameStr) == 0) {
+      strncpy(RequestFields.ContentTypeStr, FieldValStr, ReqContentTypeStrLen - 1);
+      //			if (Len >= ReqContentTypeStrLen) LogError("Content Type field truncated");
+    }
+    else if (strcmp("CONTENT-ENCODING:", FieldNameStr) == 0) {
+      strncpy(RequestFields.ContentEncodingStr, FieldValStr, ReqContentEncodingStrLen - 1);
+      //			if (Len >= ReqContentEncodingStrLen) LogError("Content Encoding field truncated");
+    }
+    else {
+      //Add it to the other headers
+      int VarLen = strlen(FieldNameStr);
+      if (FieldNameStr[VarLen - 1] == ':') {
+	//Remove the colon
+	FieldNameStr[VarLen - 1] = '\0';
+	VarLen--;
+      }
+      RequestFields.OtherHeaders[RequestFields.NumOtherHeaders].Var = new char[VarLen + 1];
+      RequestFields.OtherHeaders[RequestFields.NumOtherHeaders].Val = new char[Len + 1];
+      strcpy(RequestFields.OtherHeaders[RequestFields.NumOtherHeaders].Var, FieldNameStr);
+      strcpy(RequestFields.OtherHeaders[RequestFields.NumOtherHeaders].Val, FieldValStr);
+      RequestFields.NumOtherHeaders++;
+    }
+    strcpy(CurLine, NextLine);
+  }
+  
+  
+  if (RequestFields.ContentLengthStr[0] != 0) { //Do we have attached data?
+    unsigned int NumRecv;
+    
+    RequestFields.ContentLength = atol(RequestFields.ContentLengthStr);
+    if (RequestFields.ContentLength > 0) {
+      //			ThreadDebugMessage(RequestInfo.ThreadNum, "Getting content");
+      
+      //Allocate memory
+      RequestFields.Content = new BYTE[RequestFields.ContentLength];
+      
+      //Get rest of data from get lines
+      NumRecv = DataInBuffer - ReadBufferIndex;
+      
+      if (NumRecv >RequestFields.ContentLength) {
+	//Overflow, only read what they said they'd send
+	NumRecv = RequestFields.ContentLength;
+      }
+      memcpy(RequestFields.Content, RequestInfo.IOBuffer + ReadBufferIndex,
+	     NumRecv);
+      
+      while (NumRecv < RequestFields.ContentLength) {
+	NumRecv = GetData(RequestInfo.ClientSocket,
+			  RequestFields.Content + NumRecv,
+			  RequestFields.ContentLength - NumRecv,
+			  RequestInfo.ThreadNum);
+	if (NumRecv < 0) return GH_ERROR;
+      }
+    }
+    else {
+      RequestFields.Content = NULL;
+      RequestFields.ContentLength = 0;
+    }
+  }
+  else {
+    RequestFields.Content = NULL;
+    RequestFields.ContentLength = 0;
+  }
+  
+  return GH_10_REQUEST;
+}
+
+/******************************************************************************/
+void CleanUpHTTPHeaders(RequestInfoT &RequestInfo, RequestFieldsT &RequestFields) {
+  //Clean up memory allocated for the Content
+  if (RequestFields.Content != NULL)
+    delete[] RequestFields.Content;
+  while (RequestFields.NumOtherHeaders > 0) {
+    RequestFields.NumOtherHeaders--;
+    delete[] RequestFields.OtherHeaders[RequestFields.NumOtherHeaders].Var;
+    delete[] RequestFields.OtherHeaders[RequestFields.NumOtherHeaders].Val;
+  }
+  
+  // clean up memory allocated for the IOBuffer
+  if (RequestInfo.IOBuffer != NULL) {
+    delete[] RequestInfo.IOBuffer;
+    RequestInfo.IOBuffer = NULL;
+  }
+}
Index: /tags/gsdl-21/gsdl/src/w32server/httpreq.h
===================================================================
--- /tags/gsdl-21/gsdl/src/w32server/httpreq.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/w32server/httpreq.h	(revision 699)
@@ -0,0 +1,159 @@
+#ifndef HTTPREQ_H
+#define HTTPREQ_H
+
+/*
+Copyright (C) 1996
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+The author can be contacted via Email at bmorin@wpi.edu
+*/
+
+/*
+Module Name: HTTP Reqeust
+Purpose: Parses HTTP requests and then calls the appropriate function
+	to respond to the request
+Public Functions:
+	Request Thread
+*/
+
+// Public Data Structures
+
+//Used for sending information to the request thread
+struct RequestThreadMessageT {
+	SOCKADDR_IN ClientSockAddr;
+	SOCKET ClientSocket;
+	int AddrLen;
+	};
+
+//Length constants for RequestFieldsT
+#define ReqMethodStrLen           24
+#define ReqURIStrLen             255
+#define ReqVersionStrLen          24
+#define ReqDateStrLen             48
+#define ReqMIMEVerStrLen          24
+#define ReqPragmaStrLen          128
+#define ReqAuthorizationStrLen   512
+#define ReqFromStrLen            128
+#define ReqIfModSinceStrLen       48
+#define ReqRefererStrLen         512
+#define ReqUserAgentStrLen       256
+#define ReqContentEncodingStrLen 128
+#define ReqContentTypeStrLen     128
+#define ReqContentLengthStrLen    48
+#define ReqAcceptStrLen          256
+#define ReqAcceptLangStrLen       48
+#define ReqConnectionStrLen       24
+#define ReqAuthorizedUserStrLen   48
+#define ReqPathInfoStrLen        512
+#define ReqPathTranslatedStrLen  512
+#define ReqScriptNameStrLen      512
+//                               ---
+//                              4436
+
+#define MAX_OTHER_HEADERS 100
+
+struct RequestHeaderT {
+	char *Var;
+   char *Val;
+	};
+
+struct RequestFieldsT {
+	//Simple request line info v0.9
+	char MethodStr[ReqMethodStrLen];
+	char URIStr[ReqURIStrLen];
+	//added v1.0
+	char VersionStr[ReqVersionStrLen];
+	//General Header
+	char DateStr[ReqDateStrLen];
+	char MIMEVerStr[ReqMIMEVerStrLen];
+	char PragmaStr[ReqPragmaStrLen];
+	//Request Header
+	char AuthorizationStr[ReqAuthorizationStrLen];
+	char FromStr[ReqFromStrLen];
+	char IfModSinceStr[ReqIfModSinceStrLen];
+	char RefererStr[ReqRefererStrLen];
+	char UserAgentStr[ReqUserAgentStrLen];
+	//Entity Header (Only CGI stuff)
+	char ContentEncodingStr[ReqContentEncodingStrLen];
+	char ContentTypeStr[ReqContentTypeStrLen];
+	char ContentLengthStr[ReqContentLengthStrLen];
+   //v1.0 Optional (the more common ones)
+   char AcceptStr[ReqAcceptStrLen];
+   char AcceptLangStr[ReqAcceptLangStrLen];
+	//v1.1 Exentions
+	char ConnectionStr[ReqConnectionStrLen];
+	//Pointer to buffer containing the content
+	DWORD ContentLength;
+	BYTE *Content;
+
+	//Authorized user, filled in if a user is authorized from Auth..Str above
+	char AuthorizedUserStr[ReqAuthorizedUserStrLen];
+
+   //CGI style fields
+   char PathInfoStr[ReqPathInfoStrLen];
+   char PathTranslatedStr[ReqPathTranslatedStrLen];
+   char ScriptNameStr[ReqScriptNameStrLen];
+
+   //Other Headers
+   int NumOtherHeaders;
+   RequestHeaderT OtherHeaders[MAX_OTHER_HEADERS];
+	};
+
+struct RequestInfoT {
+	int ThreadNum;
+
+	//Buffer for IO operations (so we're not constantly reallocating buffers)
+	BYTE *IOBuffer;
+   int IOBufferSize;
+
+	//Socket the request is on and its address
+	SOCKET ClientSocket;
+	SOCKADDR_IN ClientSockAddr;
+	int AddrLen;
+
+	//Should we keep the connection alive?
+	BOOL KeepAlive;
+	};
+
+
+// Public Functions
+
+/*
+Function Name: RequestThread
+Purpose: Was the HTTP request processing thread
+         Now just a standard procedure called to process a request
+Parameters:
+	       Pointer to packed parameter structure
+*/
+void RequestThread(RequestThreadMessageT *Parameters);
+
+/*
+Function Name: Process 1.0 Request
+Purpose: Sends a HTTP 1.0 (plus some) reply to a HTTP 1.x request
+Parameters:
+	ClientSocket - Socket the client is on
+	ClientSockAddr - Address of client
+	AddrLen - Length of client address
+	RequestInfo - Structure storing the parsed headers
+	KeepAlive - To be set to true if we are maintainig the connection
+	IOBuffer - Pointer to buffer allocated for IO operations
+	TheadNum - Number of calling thread for debugging
+Notes: The function uses "Connection: Keep-Alive" as written in the HTTP/1.1
+	draft and implemented by Netscape and MSIE
+*/
+void Process10Request(RequestInfoT &RequestInfo, RequestFieldsT &RequestFields);
+
+#endif
Index: /tags/gsdl-21/gsdl/src/w32server/httpsend.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/w32server/httpsend.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/w32server/httpsend.cpp	(revision 699)
@@ -0,0 +1,100 @@
+/*
+Copyright (C) 1996
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+The author can be contacted via Email at bmorin@wpi.edu
+*/
+#include <windows.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <memory.h>
+#pragma hdrstop
+#include "parse.h"
+#include "netio.h"
+#include "httpreq.h"
+#include "httpsend.h"
+
+//Private Function Declarations
+/*
+Function Name: Check IMS Date
+Purpose: Checks the string date of an "If Modified Since" header against
+	a file date stamp.
+Parameters:
+	FileTime - Time stamp on file
+	IMSStr - "If Modified Since" date string
+Returns: TRUE if the file has been modified since the given date or
+	the data can not be resolved (the string to date function will log
+	any errors)
+*/
+BOOL CheckIMSDate(SYSTEMTIME &FileTime, char *IMSStr);
+
+//Public Functions
+/******************************************************************************/
+
+void SendHTTPError(int ErrorNum, char *ErrorTitleStr, char *ErrorDescStr, RequestInfoT &RequestInfo, RequestFieldsT &RequestFields)
+{
+	char Header[512];
+	char Body[512];
+
+	char ErrorNumStr[17];
+
+	DWORD BodyLength;
+	char BodyLengthStr[17];
+
+	int HeaderLength;
+
+	itoa(ErrorNum, ErrorNumStr, 10);
+
+		//Didn't find it, build the message manualry
+		Body[0] = 0;
+		strcat(Body, "<HTML><HEAD><TITLE>Server Error</TITLE></HEAD><BODY>\n");
+		strcat(Body, "<H1>Error ");
+		strcat(Body, ErrorNumStr);
+		strcat(Body, ": ");
+		strcat(Body, ErrorTitleStr);
+		strcat(Body, "</H1>\n");
+		strcat(Body, ErrorDescStr);
+		strcat(Body, "\n</BODY></HTML>");
+		BodyLength = (DWORD) strlen(Body);
+
+	//Build Header
+	Header[0] = 0;
+	//Status Line
+	strcat(Header, "HTTP/1.0 ");
+	strcat(Header, ErrorNumStr);
+	strcat(Header, " ");
+	strcat(Header, ErrorDescStr);
+	strcat(Header, "\r\n");
+	//Server
+	strcat(Header, "Server: GSDL\r\n");
+	//Content Type (assume HTML)
+	strcat(Header, "Content-Type: text/html\r\n");
+	//Content Length
+	itoa(BodyLength, BodyLengthStr, 10);
+	strcat(Header, "Content-Length: ");
+	strcat(Header, BodyLengthStr);
+	strcat(Header, "\r\n");
+	//Single CRLF to end header
+	strcat(Header, "\r\n");
+
+	HeaderLength = strlen(Header);
+
+  //Send header
+  SendData(RequestInfo.ClientSocket, (BYTE *) Header, strlen(Header), RequestInfo.ThreadNum);
+  //Send generated data
+  SendData(RequestInfo.ClientSocket, (BYTE *) Body, BodyLength, RequestInfo.ThreadNum);
+}
Index: /tags/gsdl-21/gsdl/src/w32server/httpsend.h
===================================================================
--- /tags/gsdl-21/gsdl/src/w32server/httpsend.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/w32server/httpsend.h	(revision 699)
@@ -0,0 +1,43 @@
+/*
+Copyright (C) 1996
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+The author can be contacted via Email at bmorin@wpi.edu
+*/
+
+/*
+Module Name: HTTP Send
+Purpose: Functions for sending HTTP responses to 1.X requests
+Public Functions:
+	SendHTTPError
+Notes: Considering adding a version number parameter to these functions.  Right
+	now the way people are implementing HTTP/1.1 is by adding the new stuff
+	to 1.0 requests.  I don't know of any browsers sending 1.1 requests yet.
+*/
+
+/*
+Function Name: Send HTTP Error
+Purpose: Sends a given error message to the client with either the error file
+			as a body or a default error message body.  Common and refenece log
+			entries are written, but not to the count log.
+Parameters:
+	ErrorNum - HTTP error number to send
+  ErrorTitleStr - Title of the error
+	ErrorDescStr - String describing the error
+	RequestInfo - HTTP request information (see httpreq.h)
+	RequestFields - Parsed HTTP header info (see httpreq.h)
+*/
+void SendHTTPError(int ErrorNum, char *ErrorTitleStr, char *ErrorDescStr, RequestInfoT &RequestInfo, RequestFieldsT &RequestFields);
Index: /tags/gsdl-21/gsdl/src/w32server/httpsrv.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/w32server/httpsrv.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/w32server/httpsrv.cpp	(revision 699)
@@ -0,0 +1,96 @@
+/*
+Copyright (C) 1996
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+The author can be contacted via Email at bmorin@wpi.edu
+*/
+#include <windows.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <memory.h>
+#pragma hdrstop
+#include "netio.h"
+#include "settings.h"
+#include "httpreq.h"
+#include "httpsrv.h"
+#include "locate.h"
+#include "settings.h"
+
+//Private Functions
+
+/*
+Function Name: HTTP Server Answer
+Purpose: Answers messages saying that a connection is ready to start
+*/
+void HTTPServerAnswer();
+
+//Module Vars
+static SOCKET ServerSocket = INVALID_SOCKET;
+
+static HWND MsgWindow;
+
+/******************************************************************************/
+// returns 0 on success, and a WSA error otherwise
+int StartHTTPServer(HWND PassedMsgWindow) {
+
+  MsgWindow = PassedMsgWindow;
+  //Create the server socket
+  return CreateListeningSocket(gs_port_num, MsgWindow, HTTP_SERVER_MSG, ServerSocket);
+}
+
+/******************************************************************************/
+void EndHTTPServer()
+{
+  //Stop the listening socket
+  DestroyListeningSocket(ServerSocket, MsgWindow);
+}
+
+/******************************************************************************/
+void ProcessHTTPServerMsg(WPARAM Socket, LPARAM MsgInfo) {
+	log_message("accept message arrived\n");
+	if (Socket != ServerSocket) {
+		if (gs_keep_log || gs_show_console) log_message("Incorrect socket sent in message");
+		}
+	switch (WSAGETSELECTEVENT(MsgInfo)) {
+		case FD_ACCEPT:
+			switch (WSAGETSELECTERROR(MsgInfo)) {
+				case WSAENETDOWN:
+					LogCriticalError("1","Network Down");
+					break;
+				default: //No Error
+					HTTPServerAnswer();
+				}
+			break;
+		}
+}
+
+/******************************************************************************/
+void HTTPServerAnswer() {
+	SOCKADDR_IN ClientSockAddr;
+	SOCKET ClientSocket;
+	int AddrLen = sizeof(SOCKADDR_IN);
+	RequestThreadMessageT Parameters;
+  while (AnswerListeningSocket(ServerSocket, ClientSocket, ClientSockAddr, AddrLen) == 0) {
+    log_message("Answering socket\n");
+	  //Call the thread procedure to handle the connection
+    Parameters.ClientSockAddr = ClientSockAddr;
+  	Parameters.ClientSocket = ClientSocket;
+  	Parameters.AddrLen = AddrLen;
+    RequestThread(&Parameters);
+  }
+}
+	
Index: /tags/gsdl-21/gsdl/src/w32server/httpsrv.h
===================================================================
--- /tags/gsdl-21/gsdl/src/w32server/httpsrv.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/w32server/httpsrv.h	(revision 699)
@@ -0,0 +1,47 @@
+/*
+Copyright (C) 1996
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+The author can be contacted via Email at bmorin@wpi.edu
+*/
+#define HTTP_SERVER_MSG WM_USER + 23
+
+/*
+Function Name: Start HTTP Server
+Purpose: Starts the HTTP server.  Inits Winsock, MIME, loads registry stuff,
+			etc.
+Parameters:
+	PassedMsgWindow - The handle of the window that handles messages.  Needed
+							to setup WSASelect notification of socket events and
+							to send a shutdown message on error.
+*/
+// returns 0 on success, and a WSA error otherwise
+int StartHTTPServer(HWND PassedMsgWindow);
+
+/*
+Function Name: End HTTP Server
+Purpose: Ends the HTTP server.  Cleans up MIME, users, etc.
+*/
+void EndHTTPServer();
+
+/*
+Function Name: Process HTTP Server Message
+Purpose: Processes HTTP_SERVER_MSG messages.
+Parameters:
+	Socket - The socket the message is regarding (WPARAM)
+	MsgInfo - The messsage (LPARAM)
+*/
+void ProcessHTTPServerMsg(WPARAM Socket, LPARAM MsgInfo);
Index: /tags/gsdl-21/gsdl/src/w32server/locate.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/w32server/locate.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/w32server/locate.cpp	(revision 699)
@@ -0,0 +1,240 @@
+#include <windows.h>
+#include <stdio.h>
+#include "locateP.h"
+#include "locate.h"
+#include "settings.h"
+
+/* The following values are inserted or used by FNORD.CPP */
+HWND GSDL_Window;
+int line_spacing;
+RECT text_rect;
+
+
+static FILE *log_file=NULL;
+
+static int location_found=0;
+char data_location[MAX_URL_SIZE];
+
+void set_location(char *possible)
+{
+  if (location_found != 0) return;
+  if (possible != NULL && *possible != 0) {
+    strcpy(data_location, possible);
+    if (data_location[strlen(data_location)-1] != '\\')
+      strcat(data_location,"\\");
+    location_found = 1;
+    }
+}
+
+void find_location(void)
+{
+  char *send;
+  if (location_found != 0) return;
+  if (GetModuleFileName(NULL,data_location,MAX_URL_SIZE) <= 0)
+    send = data_location;
+  else {
+    send = strrchr(data_location,'\\');
+    if (send != NULL)
+      send++;
+    else {
+      send = strchr(data_location,':');
+      if (send != NULL)
+        send++;
+      else
+        send == data_location;
+      }
+    }
+  *send = 0;
+  location_found = 1;
+}
+
+void open_log_file()
+{
+  if (log_file != NULL) fclose(log_file);
+  log_file = fopen(gs_log_name, "a");
+  if (log_file != NULL) {
+	  fprintf (log_file, "\n\n-----------\n");
+	  fprintf (log_file, "log started\n");
+	  fprintf (log_file, "-----------\n\n");
+  }
+}
+
+void close_log_file()
+{
+  if (log_file != NULL) {
+    fclose(log_file);
+    log_file = NULL;
+  }
+}
+
+#define CONSOLE_BUFFER_SIZE 2048
+char console_buffer[CONSOLE_BUFFER_SIZE];
+char *next_console, *last_console;
+int console_top, console_y;
+
+void activate_console(void)
+{
+  gs_show_console = 1;
+  next_console = console_buffer;
+  *next_console = 0;
+  last_console = next_console;
+  console_y = console_top = text_rect.top +
+    (text_rect.bottom - text_rect.top + 1) % line_spacing / 2;
+  InvalidateRect(GSDL_Window,NULL,TRUE);
+}
+
+void deactivate_console(void)
+{
+  gs_show_console = 0;
+  InvalidateRect(GSDL_Window,NULL,TRUE);
+}
+
+void refresh_console(HDC dc)
+{
+  int loc;  char *b, *e;
+  loc = console_top;
+  for (e = console_buffer;;) {
+    b = e;
+    while (*e >= ' ') e++;
+    TextOut(dc,0,loc,b,e-b);
+    loc += line_spacing;
+    if (*e == 0) break;
+    e++;
+    if (*e == 0) break;
+    }
+}
+
+void scroll(HDC dc, int nbits)
+{
+  RECT scroll, clip, update;
+
+  scroll.left = clip.left = text_rect.left;
+  scroll.right = clip.right = text_rect.right;
+  clip.top = console_top;
+  clip.bottom = text_rect.bottom;
+  scroll.top = clip.top + nbits;
+  scroll.bottom = text_rect.bottom;
+  ScrollDC(dc,0,-nbits,&scroll,&clip,NULL,&update);
+  FillRect(dc,&update, GetStockObject(WHITE_BRUSH));
+}
+
+int remove_line()
+{
+  char *bin, *bout;
+  bin = bout = console_buffer;
+  while (*bin >= ' ') bin++;
+  if (*bin != 0) {
+    bin++;
+    if (*bin != 0) {
+      while (*bin != 0) *bout++ = *bin++;
+      *bout = *bin;
+      last_console -= (bin - bout);
+      next_console = bout;
+      return line_spacing;
+      }
+    }
+  next_console = console_buffer;
+  *next_console = 0;
+  last_console = next_console;
+  return 0;
+}
+
+void display_line(HDC dc, char *line, int nch)
+{
+  int scroll_needed = 0;
+  while (next_console + nch + 1 
+         > console_buffer + CONSOLE_BUFFER_SIZE)
+    scroll_needed += remove_line();
+  console_y -= scroll_needed;
+  if (*last_console == '\n') {
+    console_y += line_spacing;
+    if (console_y + line_spacing > text_rect.bottom) {
+      scroll_needed += remove_line();
+      console_y -= line_spacing;
+      }
+    last_console = next_console;
+    }
+  if (scroll_needed != 0) scroll(dc, scroll_needed);
+  while (nch-- > 0) *next_console++ = *line++;
+  TextOut(dc,0,console_y,last_console,next_console-last_console);
+  *next_console = *line;
+  if (*next_console != 0) {
+    last_console = next_console;
+    next_console++;
+    *next_console = 0;
+    }
+}
+
+void display_text(HDC dc, char *buffer)
+{
+  char *bl, *bp;
+  for (bl = buffer;;) {
+    for (bp = bl;;) {
+      if (*bp == '\t') *bp = ' ';
+      if (*bp < ' ') break;
+      bp++;
+      }
+    if (*bp != 0) *bp = '\n';
+    display_line(dc, bl, bp-bl);
+    if (*bp == 0) break;
+    bl = bp+1;
+    if (*bl == 0) break;
+    }
+}
+
+void log_message(char *msg)
+{
+  if (log_file != NULL) {
+    fprintf(log_file,"%s",msg);
+    fflush(log_file);
+    }
+  if (gs_show_console) {
+    char buffer[1024];  HDC dc;
+    sprintf(buffer,"%s",msg);
+    dc = GetDC(GSDL_Window);
+    display_text(dc,buffer);
+    ReleaseDC(GSDL_Window,dc);
+    }
+}
+
+void log_message_N(char*msg, int n)
+{
+  if (log_file != NULL) {
+	  int len = 0;
+	  while (len < n) {
+		  fputc (msg[len], log_file);
+		  len++;
+	  }
+	  fflush(log_file);
+  }
+
+  // these messages are not send to the console
+  if (gs_show_console) {
+    HDC dc;
+    dc = GetDC(GSDL_Window);
+    display_text(dc,"message only written to log file\n");
+    ReleaseDC(GSDL_Window,dc);
+    }
+}
+
+void LogCriticalError(char *cderr, char *Msg)
+{
+	char MsgBoxStr[200];
+
+	strcpy(MsgBoxStr, "Critical Error Number ");
+	strcat(MsgBoxStr, cderr);
+	strcat(MsgBoxStr, "\n");
+	strcat(MsgBoxStr, Msg);
+	strcat(MsgBoxStr, "\nGreenstone Digital Library software\nshutting down");
+	MessageBox(GSDL_Window, MsgBoxStr, "Greenstone Digital Library Software",
+		MB_OK | MB_ICONERROR);
+	PostMessage(GSDL_Window, WM_DESTROY, 0, 0);
+}
+
+
+/* returns second-first taking into account wrap around */
+DWORD DiffTickCounts (DWORD first, DWORD second) 
+{
+	if (second >= first) return (second-first);
+	return (MAXDWORD-first+1+second);
+}
Index: /tags/gsdl-21/gsdl/src/w32server/locate.h
===================================================================
--- /tags/gsdl-21/gsdl/src/w32server/locate.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/w32server/locate.h	(revision 699)
@@ -0,0 +1,26 @@
+#define MAX_URL_SIZE 256
+
+extern HWND GSDL_Window;
+extern int line_spacing;
+extern RECT text_rect;
+
+void refresh_console(HDC dc);
+
+
+extern char data_location[MAX_URL_SIZE];
+
+extern void set_location(char *possible);
+extern void find_location(void);
+
+extern void open_log_file(void);
+extern void close_log_file(void);
+extern void log_message(char *msg);
+extern void log_message_N(char*msg, int n);
+
+extern void activate_console(void);
+extern void deactivate_console(void);
+
+extern void LogCriticalError(char *cderr, char *Msg);
+
+/* returns first-second taking into account wrap around */
+extern DWORD DiffTickCounts (DWORD first, DWORD second);
Index: /tags/gsdl-21/gsdl/src/w32server/netio.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/w32server/netio.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/w32server/netio.cpp	(revision 699)
@@ -0,0 +1,478 @@
+/*
+Copyright (C) 1996
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+The author can be contacted via Email at bmorin@wpi.edu
+*/
+
+#include <windows.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <memory.h>
+#pragma hdrstop
+#include "netio.h"
+#include "locate.h"
+#include "d_winsock.h"
+
+//Private Variables and Declarations
+
+//Max string size for a host name
+#define MAXHOSTNAME 100
+
+//Timeout for waiting between "packets" in seconds
+#define NETIO_CONN_TIMEOUT 20 //3 sec  was 180 @@@@@ WJR
+
+#define HTTP_DNS_MSG WM_USER + 1
+int dns_msg_received = 0;
+int dns_msg_error = 0;
+
+
+DWORD LocalIPNumber;
+char LocalName[MAXHOSTNAME];
+
+
+//Private Function Declarations
+DWORD GetHostID();
+
+
+// returns 0 on success,
+// WSASYSNOTREADY, WSAVERNOTSUPPORTED, or WSAEINVAL on failure
+int InitNetIO() {
+	WSADATA Data;
+
+	// Load Winsock
+	int err = d_WSAStartup(MAKEWORD(1, 1), &Data);
+
+	// get local IP number
+	if (err == 0) LocalIPNumber = GetHostID();
+	else LocalIPNumber = INADDR_ANY;
+
+	// set the Local Name to "" for now
+	LocalName[0] = 0;
+	
+	return err;
+}
+
+
+void CleanUpNetIO() {
+	//Experimental, heard this will better clean up, especialy if a call is
+	//blocking....
+	d_WSAUnhookBlockingHook();
+
+	//Clean up Winsock
+	d_WSACleanup() == SOCKET_ERROR;
+}
+
+
+void ResetNetIO() {
+	//Figure out the local name on first querry, set to "" until then...
+	LocalName[0] = 0;
+}
+
+
+DWORD GetLocalIP() {
+	return LocalIPNumber;
+}
+
+
+
+
+
+long __stdcall DNSWindowProc(HWND hWnd,UINT wMsg, WPARAM wParam, LPARAM lParam) {
+	if (wMsg == HTTP_DNS_MSG) {
+		dns_msg_received = 1;
+		dns_msg_error = WSAGETASYNCERROR(lParam);
+		return 0;
+	}
+
+	return DefWindowProc(hWnd, wMsg, wParam, lParam);
+}
+
+char *GetLocalName(HANDLE hInstance) {
+	// static in case it is written to after the function has finished
+	// (I did not error checking on WSACancelAsyncRequest)
+	static char buf[MAXGETHOSTSTRUCT];
+
+	
+	if (LocalName[0] == 0) {
+		hostent *DNSResult = NULL;
+		in_addr LocalInAddr;
+
+		// if we failed to get the local IP number
+		// use the loop-back device
+		if (LocalIPNumber == INADDR_ANY) {
+			strcpy(LocalName, "127.0.0.1"); // loop-back device
+			return LocalName;
+		}
+
+		//Convert the number to an in_addr struct
+		LocalInAddr.s_addr = LocalIPNumber;
+
+		// if we fail to find the domain name we will
+		// still want the IP address
+		strcpy(LocalName, d_inet_ntoa(LocalInAddr));
+
+
+		// make sure they actually passed in an instance handle
+		if (hInstance == NULL) return LocalName;
+
+		// do a async domain name lookup so that we
+		// can control the timeout
+
+		// create a window class and window to handle the async messages
+		WNDCLASS wc; HWND hwndDNS;
+	
+		wc.style = 0;
+		wc.lpfnWndProc = DNSWindowProc;
+		wc.cbClsExtra = 0;
+		wc.cbWndExtra = 0;
+		wc.hInstance = hInstance;
+		wc.hIcon = NULL;
+		wc.hCursor = NULL;
+		wc.hbrBackground = NULL;
+		wc.lpszMenuName = NULL;
+		wc.lpszClassName = "GSDL DNS Window";
+		if (!RegisterClass(&wc)) return LocalName;
+
+		hwndDNS = CreateWindow("GSDL DNS Window", "", 
+			WS_OVERLAPPEDWINDOW, 0, 0, 100, 100, 
+			NULL, NULL, hInstance, NULL);
+		if (!hwndDNS) return LocalName;
+
+		// process all messages currently on the queue
+		MSG Message;
+		while (PeekMessage(&Message, NULL, 0, 0, PM_REMOVE)) {
+			TranslateMessage(&Message); /* translate keyboard messages */
+			DispatchMessage(&Message);  /* return control to Windows NT */
+		}
+
+		//Do a async DNS lookup on the IP number
+		dns_msg_received = 0;
+		dns_msg_error = 0;
+		HANDLE asyncGetHostReq = d_WSAAsyncGetHostByAddr(
+		    hwndDNS, HTTP_DNS_MSG, (char *)&(LocalInAddr), 
+			4, PF_INET, buf, MAXGETHOSTSTRUCT);
+	
+		if (asyncGetHostReq != NULL) {
+			// wait 5 seconds for the request to complete
+			int now = GetTickCount();
+			while ((DiffTickCounts(now, GetTickCount()) < 5000) && !dns_msg_received) {
+				if (PeekMessage(&Message, NULL, 0, 0, PM_REMOVE)) {
+					TranslateMessage(&Message); /* translate keyboard messages */
+					DispatchMessage(&Message);  /* return control to Windows NT */
+				} else {
+					Sleep(1);
+				}
+			}
+			if (!dns_msg_received)
+				d_WSACancelAsyncRequest(asyncGetHostReq);
+		}
+
+		DestroyWindow(hwndDNS);
+
+		if (dns_msg_received && (dns_msg_error == 0)) {
+   	    	//Worked, use the primary name
+			strcpy(LocalName, ((hostent *)(buf))->h_name);
+      		//Convert it to lower case for cosmedic reasons
+			CharLower(LocalName);
+		}
+   	}
+	return LocalName;
+}
+
+
+// returns 0 on success, and a WSA error message on failure.
+// possible error messages include:
+// WSANOTINITIALISED, WSAENETDOWN, WSAEAFNOSUPPORT, WSAEINPROGRESS, WSAEMFILE,
+// WSAENOBUFS, WSAEPROTONOSUPPORT, WSAEPROTOTYPE, WSAESOCKTNOSUPPORT,
+// WSAEADDRINUSE, WSAEINVAL, WSAEISCONN, WSAENOTSOCK, WSAEOPNOTSUPP
+
+#define MAXBINDCOUNT 4096
+int CreateListeningSocket(int &PortNum, HWND MsgWindow, WORD SocketMsg, SOCKET &ServerSocket) {
+	int err = 0;
+	SOCKADDR_IN ServerSockAddr;
+	int bind_port, bind_count;
+
+	//Create the Server Socket
+	ServerSocket = d_socket(AF_INET, SOCK_STREAM, 0);
+	if (ServerSocket == INVALID_SOCKET) return d_WSAGetLastError();
+
+	bind_port = PortNum;
+	for (bind_count=0; bind_count < MAXBINDCOUNT; bind_count++) {
+		// Set up the Server Socket Address
+		memset(&ServerSockAddr, 0, sizeof(ServerSockAddr)); //Needed?
+		ServerSockAddr.sin_port = d_htons( (WORD) bind_port);
+		ServerSockAddr.sin_family = AF_INET;
+		ServerSockAddr.sin_addr.s_addr = d_htonl(INADDR_ANY);
+
+		// Try to bind the socket with the address
+		if (d_bind(ServerSocket, (LPSOCKADDR) &ServerSockAddr,
+				sizeof(ServerSockAddr)) != SOCKET_ERROR) {
+			PortNum = bind_port;
+			break;
+		}
+
+		// make sure it failed to bind because it was
+		// already bound
+		err = d_WSAGetLastError ();
+		if (err != WSAEADDRINUSE) return err;
+
+		// Or choose another port number to try
+		if (bind_port == 80) bind_port = IPPORT_RESERVED+1;
+		else if (bind_count == 0) bind_port = 80;
+		else bind_port++;
+	}
+
+	// return an error in we couldn't find a valid port
+	if (bind_count == MAXBINDCOUNT) return WSAEADDRINUSE;
+
+	// Start listening for connections
+	if (d_listen(ServerSocket, SOMAXCONN) == SOCKET_ERROR)
+		return d_WSAGetLastError ();	
+
+	// Set up event for new connections
+	if (d_WSAAsyncSelect(ServerSocket, MsgWindow, SocketMsg, FD_ACCEPT) == SOCKET_ERROR)
+		return d_WSAGetLastError ();
+
+	return 0;
+}
+
+
+int AnswerListeningSocket(SOCKET ServerSocket, SOCKET &ClientSocket, SOCKADDR_IN &ClientSockAddr, int AddrLen) {
+	if (d_WSAIsBlocking()) {
+		log_message("rejected connect due blocking\n");
+		return -1;
+	}
+
+	ClientSocket = d_accept(ServerSocket, (LPSOCKADDR) &ClientSockAddr, &AddrLen);
+	if (ClientSocket == INVALID_SOCKET) {
+		log_message("accept failed - connection lost\n");
+		return -1;
+	}
+
+	log_message("accept success - connection made\n");
+	return 0;
+}
+
+
+void DestroyListeningSocket(SOCKET &ServerSocket, HWND MsgWindow) {
+	//Remove any message notification
+	d_WSAAsyncSelect(ServerSocket, MsgWindow, 0, 0);
+
+	//Close the socket
+	CloseSocket(ServerSocket);
+}
+
+
+void CloseSocket(SOCKET &TargetSocket) {
+	if (TargetSocket != INVALID_SOCKET) {
+		//Since we're closing the socket, there's not much I can do about errors
+		//now so I'm not gonna bother checking...
+
+		//Shutdown both ends, assume we have all data...
+		d_shutdown(TargetSocket, 2);
+
+		d_closesocket(TargetSocket);
+
+		//Make sure we can't use the old handle again...
+		TargetSocket = INVALID_SOCKET;
+	}
+}
+
+
+int GetData(SOCKET ClientSocket, BYTE *IOBuffer, int IOBufferSize,
+													int ThreadNum) {
+
+	int NumRecv;
+	int Error;
+	struct timeval Timeout;
+	fd_set SocketSet;
+
+	//Set up a socket set structure with just ClientSocket for  select(..)
+	FD_ZERO(&SocketSet);
+	FD_SET(ClientSocket, &SocketSet);
+
+	//set timeout
+	Timeout.tv_sec = NETIO_CONN_TIMEOUT;
+	Timeout.tv_usec = 0;
+
+	do {
+		NumRecv = d_recv(ClientSocket, (char *) IOBuffer, IOBufferSize, 0);
+		if (NumRecv == 0) {
+			//Lost connect
+			return -1;
+
+		} else if (NumRecv == SOCKET_ERROR) {
+			Error = d_WSAGetLastError();
+			if (Error == WSAEWOULDBLOCK) {
+				NumRecv = 0;
+				//Wait for socket to be readable
+				if (d_select(0, &SocketSet, NULL, NULL, &Timeout) != 1) {
+					//Timeout
+					return -1;
+				}
+
+			} else {
+			//Assume connection terminated
+			return -1;
+			}
+		}
+	} while(NumRecv == 0);
+	return NumRecv;
+}
+
+
+int GetLine(char *OutStr, SOCKET ClientSocket, BYTE *IOBuffer, int IOBufferSize,
+				int &BufferIndex, int &DataInBuffer, int ThreadNum) {
+
+	int i;
+	char CurChar;
+
+	i = 0;
+	do {
+		if (BufferIndex == DataInBuffer) { //Need more data
+			DataInBuffer = GetData(ClientSocket, IOBuffer, IOBufferSize, ThreadNum);
+			if (DataInBuffer == -1) {
+				//Lost connect
+				return -1;
+			}
+			BufferIndex = 0;
+		}
+		CurChar = IOBuffer[BufferIndex];
+		BufferIndex++;
+		if ((CurChar != 10) && (CurChar != 13))  {
+			OutStr[i] = CurChar;
+			i++;
+		}
+	} while ((CurChar != 10) && (i < NETIO_MAX_LINE));
+	if (i == NETIO_MAX_LINE) {
+		return -1;
+	}
+
+	OutStr[i] = 0;
+	return 0;
+}
+
+
+int SendData(SOCKET ClientSocket, BYTE *SendBuffer, int NumToSend,
+								int ThreadNum) {
+
+	int NumSent = 0;
+	int Error;
+	struct timeval Timeout;
+	fd_set SocketSet;
+
+	char NumSentStr[50];
+	itoa(NumToSend, NumSentStr, 10);
+
+	//Set up a socket set structure with just ClientSocket for  select(..)
+	FD_ZERO(&SocketSet);
+	FD_SET(ClientSocket, &SocketSet);
+	//set timeout
+	Timeout.tv_sec = NETIO_CONN_TIMEOUT;
+	Timeout.tv_usec = 0;
+
+	while (NumToSend > 0) {
+		NumSent = d_send(ClientSocket, (char *) SendBuffer + NumSent, NumToSend, 0);
+		if (NumSent == 0) {
+			//Lost connect
+			return -1;
+		}
+		else if (NumSent == SOCKET_ERROR) {
+			Error = d_WSAGetLastError();
+			if (Error == WSAEWOULDBLOCK) {
+				NumSent = 0;
+				if (d_select(0, NULL, &SocketSet, NULL, &Timeout) != 1) {
+					//Timeout
+					return -1;
+				}
+			}
+			else {
+				//Lost Connection
+				return -1;
+			}
+		}
+		NumToSend -= NumSent;
+	}
+	return 0;
+}
+
+
+
+/******************************************************************************/
+//Private Functions
+/******************************************************************************/
+/*-----------------------------------------------------------
+ * Function: GetHostID()
+ *
+ * Description:
+ *  Get the Local IP address using the following algorithm:
+ *    - get local hostname with gethostname()
+ *    - attempt to resolve local hostname with gethostbyname()
+ *    if that fails:
+ *    - get a UDP socket
+ *    - connect UDP socket to arbitrary address and port
+ *    - use getsockname() to get local address
+ *
+ * Notes: Copyright by Bob Quinn, 1995, taken from his Winsock library
+ *        was removed from its original module unmodified
+ */
+DWORD GetHostID () {
+	 char szLclHost [MAXHOSTNAME];
+	 LPHOSTENT lpstHostent;
+	 SOCKADDR_IN stLclAddr;
+	 SOCKADDR_IN stRmtAddr;
+	 int nAddrSize = sizeof(SOCKADDR);
+	 SOCKET hSock;
+	 int nRet;
+
+	 /* Init local address (to zero) */
+	 stLclAddr.sin_addr.s_addr = INADDR_ANY;
+
+	 /* Get the local hostname */
+	 nRet = d_gethostname(szLclHost, MAXHOSTNAME);
+	 if (nRet != SOCKET_ERROR) {
+		/* Resolve hostname for local address */
+		lpstHostent = d_gethostbyname((LPSTR)szLclHost);
+		if (lpstHostent)
+		  stLclAddr.sin_addr.s_addr = *((u_long FAR*) (lpstHostent->h_addr));
+	 }
+
+	 /* If still not resolved, then try second strategy */
+	 if (stLclAddr.sin_addr.s_addr == INADDR_ANY) {
+		/* Get a UDP socket */
+		hSock = d_socket(AF_INET, SOCK_DGRAM, 0);
+		if (hSock != INVALID_SOCKET)  {
+		  /* Connect to arbitrary port and address (NOT loopback) */
+		  stRmtAddr.sin_family = AF_INET;
+		  stRmtAddr.sin_port   = d_htons(IPPORT_ECHO);
+		  stRmtAddr.sin_addr.s_addr = d_inet_addr("128.127.50.1");
+		  nRet = d_connect(hSock,
+							  (LPSOCKADDR)&stRmtAddr,
+							  sizeof(SOCKADDR));
+		  if (nRet != SOCKET_ERROR) {
+			 /* Get local address */
+			 d_getsockname(hSock,
+							 (LPSOCKADDR)&stLclAddr,
+							 (int FAR*)&nAddrSize);
+		  }
+		  d_closesocket(hSock);   /* we're done with the socket */
+		}
+	 }
+	 return (stLclAddr.sin_addr.s_addr);
+}
+
Index: /tags/gsdl-21/gsdl/src/w32server/netio.h
===================================================================
--- /tags/gsdl-21/gsdl/src/w32server/netio.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/w32server/netio.h	(revision 699)
@@ -0,0 +1,151 @@
+/*
+Copyright (C) 1996
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+The author can be contacted via Email at bmorin@wpi.edu
+*/
+
+/*
+Module Name: Net IO
+Purpose: High level Socket IO and Misc socket routines
+Public Functions:
+	InitNetIO
+   CleanUpNetIO
+
+	GetLocalIP
+	GetLocalName
+
+   CreateListeningSocket
+   AnswerListeningSocket
+   DestroyListeningSocket
+
+   CloseSocket
+
+	GetData
+	GetLine
+
+	SendData
+	SendFile
+*/
+
+//Public definitions
+//Max line size for GetLine
+#define NETIO_MAX_LINE 300
+
+//Public Functions
+
+/*
+Function Name: Init Net IO
+Purpose: Initializes the module so calls can be made querying network
+	information for the local macnine.
+Notes: MUST be called before GetLocalIP and GetLocalName
+*/
+
+// returns 0 on success,
+// WSASYSNOTREADY, WSAVERNOTSUPPORTED, or WSAEINVAL on failure
+int InitNetIO();
+
+/*
+Function Name: Cleanup Net IO
+Purpose: Cleans up the module and unloads Winsock
+*/
+void CleanUpNetIO();
+
+/*
+Function Name: Reset Net IO
+Purpose: Resets internal data in NetIO to reflect registry changes, etc.
+*/
+void ResetNetIO();
+
+/*
+Function Name: Get Local IP
+Purpose: Gets the local IP number
+Returns: The local IP number as an double word (4 bytes)
+*/
+DWORD GetLocalIP();
+
+/*
+Function Name: Get Local Name
+Purpose: Gets the local Domain name, or if there is none, then the server
+address in dotted format.
+Returns: A string containing the local address
+*/
+char *GetLocalName(HANDLE hInstance);
+
+// returns 0 on success, and a WSA error message on failure.
+// possible error messages include:
+// WSANOTINITIALISED, WSAENETDOWN, WSAEAFNOSUPPORT, WSAEINPROGRESS, WSAEMFILE,
+// WSAENOBUFS, WSAEPROTONOSUPPORT, WSAEPROTOTYPE, WSAESOCKTNOSUPPORT,
+// WSAEADDRINUSE, WSAEINVAL, WSAEISCONN, WSAENOTSOCK, WSAEOPNOTSUPP
+int CreateListeningSocket(int &PortNum, HWND MsgWindow, WORD SocketMsg, SOCKET &ServerSocket);
+
+int AnswerListeningSocket(SOCKET ServerSocket, SOCKET &ClientSocket, SOCKADDR_IN &ClientSockAddr, int AddrLen);
+
+void DestroyListeningSocket(SOCKET &ServerSocket, HWND MsgWindow);
+
+void CloseSocket(SOCKET &TargetSocket);
+
+/*
+Function Name: Get Data
+Purpose: Gets data from a socket with muli-threaded debugging
+Parameters:
+	ClientSocket - The socket to get data from
+	IOBuffer - A pointer to a chunk of memory to put data in
+	IOBufferSize - Size of the buffer
+	ThreadNum - Number of the thread that called this functions used for debugging
+
+Returns: -1 on error, else the number of bytes received
+
+Notes:
+Removed socket error checking because I was unable to isolate what
+errors I was getting when getting a "rude" disconnect
+*/
+int GetData(SOCKET ClientSocket, BYTE *IOBuffer, int IOBufferSize, int ThreadNum);
+
+/*
+Function Name: Get Line
+Purpose: Gets a line of text (ended by a LF (10)) from a socket with buffered
+			input.  If the buffer is empty this function fills the buffer, else it
+			just changes an index. Also has the ability to do multi-threaded debuging
+Parameters:
+	OutStr - Varible were output is put (a length of NETIO_MAX_LINE is assumed)
+	ClientSocket - The socket to get data from
+	IOBuffer - A pointer to a chunk of memory to put data in
+	IOBufferSize - Size of the buffer
+	BufferIndex - A zero based index of what charactor in the buffer we're at
+	DataInBuffer - Amount of data in the buffer
+	ThreadNum - Number of the thread that called this functions used for debugging
+
+Returns: -1 on disconnect or error (abort connection), 0 on success
+*/
+int GetLine(char *OutStr, SOCKET ClientSocket, BYTE *IOBuffer, int IOBufferSize, int &BufferIndex, int &DataInBuffer, int ThreadNum);
+
+/*
+Function Name: Send Data
+Purpose: Sends data to a socket with muli-thread debugging
+Parameters:
+	ClientSocket - The socket to get data from
+	SendBuffer - A pointer to a chunk of memory to put data in
+	NumToSend - Number of bytes to send from buffer
+	ThreadNum - Number of the thread that called this functions used for debugging
+
+Returns: -1 on error, 0 on success
+
+Notes:
+Removed socket error checking because I was unable to isolate what
+errors I was getting when getting a "rude" disconnect
+*/
+int SendData(SOCKET ClientSocket, BYTE *SendBuffer, int NumToSend, int ThreadNum);
Index: /tags/gsdl-21/gsdl/src/w32server/parse.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/w32server/parse.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/w32server/parse.cpp	(revision 699)
@@ -0,0 +1,194 @@
+/*
+Copyright (C) 1996
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+The author can be contacted via Email at bmorin@wpi.edu
+*/
+#include <windows.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <memory.h>
+#pragma hdrstop
+
+#include "parse.h"
+
+//Public Functions
+/******************************************************************************/
+void Left(char *DestStr, const char *SourceStr, const int Len) {
+  memcpy(DestStr, SourceStr, Len);
+  DestStr[Len] = 0;
+}
+/******************************************************************************/
+void Right(char *DestStr, const char *SourceStr, const int Len) {
+  int SourceLen, Offset;
+  
+  SourceLen = strlen(SourceStr);
+  Offset = SourceLen - Len;
+  
+  memcpy(DestStr, SourceStr + Offset, Len + 1);
+  //The null is already copied
+}
+/******************************************************************************/
+void TrimLeft(char *TargetStr) {
+  int i, NewStrSize;
+  
+  i = 0;
+  //Find the first non-space charactor
+  while ( (TargetStr[i] == ' ') || (TargetStr[i] == '\t') ) {
+    i++;
+  }
+  //If we have space to get rid of, move the string
+  if (i > 0) {
+    NewStrSize = strlen(TargetStr) - i + 1; //Include the NULL in the size
+    memmove(TargetStr, TargetStr + i, NewStrSize);
+  }
+}
+/******************************************************************************/
+void TrimRight(char *TargetStr) {
+  int i;
+  
+  //Find the last non-space charactor (index i)
+  i = strlen(TargetStr) - 1;
+  while ((i >= 0) && ((TargetStr[i] == ' ') || (TargetStr[i] == '\t')) ) {
+    i--;
+  }
+  //Set the NULL 1 after the last non-space charactor
+  TargetStr[i+1] = 0;
+}
+/******************************************************************************/
+void Trim(char *TargetStr) {
+  TrimRight(TargetStr);
+  TrimLeft(TargetStr);
+}
+/******************************************************************************/
+void GetWord(char *DestStr, const char *SourceStr, const int Start, int &End) {
+  int i, Len;
+  
+  Len = 0;
+  i = Start;
+  while ((SourceStr[i] != ' ') && (SourceStr[i] != '\t') && (SourceStr[i] != 0)) {
+    Len++;
+    i++;
+  }
+  //Create the substring
+  memcpy(DestStr, SourceStr + Start, Len);
+  DestStr[Len] = 0;
+  //Set End
+  while ((SourceStr[i] != 0) &&
+	 ((SourceStr[i] == ' ') || (SourceStr[i] == '\t'))) {
+    i ++;
+  }
+  End = i;
+}
+/******************************************************************************/
+void GetLastWord(char *DestString, const char *SourceStr, int &Start) {
+  int SourceLen, i, Len;
+  
+  Len = 0;
+  SourceLen = strlen(SourceStr);
+  i = SourceLen - 1;
+  while ((i >= 0) && (SourceStr[i] != ' ') && (SourceStr[i] != '\t')) {
+    Len++;
+    i--;
+  }
+  //Create the substring
+  memcpy(DestString, SourceStr + SourceLen - Len, Len);
+  DestString[Len] = 0;
+  Start = i + 1;
+}
+/******************************************************************************/
+void SplitPath(char *Path, char *Dir, char *FileName) {
+  int SplitPoint;
+  SplitPoint = strlen(Path) - 1;
+  while ((SplitPoint > 0) && (Path[SplitPoint] != '\\')) {
+    SplitPoint--;
+  }
+  if ((SplitPoint == 0) && (Path[SplitPoint] != '\\')) {
+    //No directory, just a file name
+    Dir[0] = 0;
+    strcpy(FileName, Path);
+  }
+  else {
+    //Move past the last slash
+    SplitPoint++;
+    
+    //Directory
+    int i = 0;
+    while (i < SplitPoint) {
+      Dir[i] = Path[i];
+      i++;
+    }
+    Dir[i] = 0;
+    
+    //File name
+    int j = 0;
+    while (Path[i] != 0) {
+      FileName[j] = Path[i];
+      j++;
+      i++;
+    }
+    FileName[j] = 0;
+  }
+}
+/******************************************************************************/
+void GetExtention(char *Path, char *Extention) {
+  int Dot;
+  int PathLen;
+  
+  PathLen = strlen(Path) - 1;
+  Dot = PathLen;
+  while ((Dot > 0) && (Path[Dot] != '.')) {
+    Dot--;
+  }
+  if (Dot > 0) {
+    strcpy(Extention, Path + Dot + 1);
+  }
+  else {
+    Extention[0] = 0;
+  }
+}
+/******************************************************************************/
+void TranslateEscapeString(char *TargetStr) {
+  int i = 0;
+  int j = 0;
+  
+  while (TargetStr[i] != 0) {
+    if ((TargetStr[i] == '%') && (TargetStr[i+1] != 0) && (TargetStr[i+2] != 0)) {
+      //Escaped value (did checking to make sure there are 2 more chars)
+      //         #pragma warn -sig //Suppress significant digit loss warning (BC++)
+      TargetStr[j] = 16 * HexVal(TargetStr[i+1]) + HexVal(TargetStr[i+2]);
+      //         #pragma warn +sig   //Restore significant digit loss warning (BC++)
+      i += 3;
+    }
+    else {
+      TargetStr[j] = TargetStr[i];
+      i++;
+    }
+    j++;
+  }
+  TargetStr[j] = 0;
+}
+/******************************************************************************/
+int HexVal(char c) {
+  //   #pragma warn -sig //Suppress significant digit loss warning (BC++)
+  if (c >= '0' && c <= '9') return c - '0';
+  else if (c >= 'a' && c <= 'f') return c - 'a' + 10;
+  else if (c >= 'A' && c <= 'F') return c - 'A' + 10;
+  else return 0;
+  //   #pragma warn +sig   //Restore significant digit loss warning (BC++)
+}
+/******************************************************************************/
Index: /tags/gsdl-21/gsdl/src/w32server/parse.h
===================================================================
--- /tags/gsdl-21/gsdl/src/w32server/parse.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/w32server/parse.h	(revision 699)
@@ -0,0 +1,153 @@
+/*
+Copyright (C) 1996
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+The author can be contacted via Email at bmorin@wpi.edu
+*/
+
+/*
+Module Name: Parse
+Purpose: General string parsing.  These are functions that are not in string.h,
+	but are found in many other libraries and high level languages.
+Public Functions:
+	Left
+	Right
+
+	Trim
+	TrimLeft
+	TrimRight
+
+	GetWord
+	GetLastWord
+
+   Split Path
+   GetExtention
+*/
+
+/*
+Function Name: Left
+Purpose: Copies the leftmost x number of charactors of a string to a buffer
+Parameters:
+	DestStr - Destination buffer for substring
+	SourceStr - Source string to make substing from
+	Len - Length of substring
+
+Notes: If Len is greater than strlen(SourceStr) a memory error may occur.
+*/
+void Left(char *DestStr, const char *SourceStr, const int Len);
+
+/*
+Function Name: Right
+Purpose: Copies the rightmost x number of charactors of a string to a buffer
+Parameters:
+	DestStr - Destination buffer for substring
+	SourceStr - Source string to make substing from
+	Len - Length of substring
+
+Notes: If Len is greater than strlen(SourceStr) then a memory error may
+		 occur.
+*/
+void Right(char *DestStr, const char *SourceStr, const int Len);
+
+/*
+Function Name: TrimLeft
+Purpose: Removes spaces and tabs from the left end of a string.
+Parameters:
+	TargetString - String to be trimmed
+*/
+void TrimLeft(char *TargetStr);
+
+/*
+Function Name: TrimRight
+Purpose: Removes spaces and tabs from the right end of a string.
+Parameters:
+	TargetString - String to be trimmed
+*/
+void TrimRight(char *TargetStr);
+
+/*
+Function Name: Trim
+Purpose: Removes spaces and tabs from both ends of a string.
+Parameters:
+	TargetString - String to be trimmed
+*/
+void Trim(char *TargetStr);
+
+/*
+Function Name: GetWord
+Purpose: Gets leftmost substring without a space, starts at any position at
+	 the start string and sets End to be the next non-space charactor in the
+	 string.  ' ' and '\t' and considered spaces
+Parameters:
+	 DestString - Destination buffer for substring
+	 SourceString - Source string to make substing from
+	 Start - 0 based string index to start at
+	 End - At end of function call is set to the location of next non-space
+               charactor
+
+Notes: If Start is greater than strlen(SourceStr) then a memory error may
+		 occur.
+*/
+void GetWord(char *DestStr, const char *SourceStr, const int Start, int &End);
+
+/*
+Function Name: GetLastWord
+Purpose: Gets rightmost substring without a space sets Start to be the 0 based
+	index of where the substring starts in the original string.
+Parameters:
+	DestString - Destination buffer for substring
+	SourceString - Source string to make substing from
+	Start - 0 based string index where the DestString starts at
+*/
+void GetLastWord(char *DestString, const char *SourceStr, int &Start);
+
+
+/*
+Function Name: Split Path
+Purpose: Splits a path string into directory and file name
+Parameters:
+	Path - Path string (example "c:\Fnord\SomeFile.Ext")
+	Dir - Directory part of the Path (example "c:\Fnord\")
+	FileName - File name part of the path (example "SomeFile.Ext")
+*/
+void SplitPath(char *Path, char *Dir, char *FileName);
+
+/*
+Function Name: Get Extention
+Purpose: Splits a path string into directory and file name
+Parameters:
+	Path - Path string or file name (example "c:\Fnord\SomeFile.Ext")
+	Extention - Extention part of the path (example "Ext")
+*/
+void GetExtention(char *Path, char *Extention);
+
+/*
+Function Name: Translate Escape String
+Purpose: Translates all URL escape sequences, such as "%20" where "20" is
+	the resulting charactor in hexidecimal
+Parameters:
+	TargetStr - String that may contain URL escape sequences.  All sequences
+   	are decoded on return.
+*/
+void TranslateEscapeString(char *TargetStr);
+
+/*
+Function Name: Hex Value
+Purpose: Returns the value of a hexidecimal digit character as an integer
+Parameters:
+	c - Charactor to translate
+*/
+int HexVal(char c);
Index: /tags/gsdl-21/gsdl/src/w32server/resource.h
===================================================================
--- /tags/gsdl-21/gsdl/src/w32server/resource.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/w32server/resource.h	(revision 699)
@@ -0,0 +1,48 @@
+//{{NO_DEPENDENCIES}}
+// Microsoft Developer Studio generated include file.
+// Used by server.rc
+//
+#define TRAY_ICON                       101
+#define COMMS_DIALOG                    102
+#define COMMS_DIALOG_DEBUG              102
+#define Main_Menu                       103
+#define COMMS_DIALOG2                   103
+#define GSDL_LOGO                       108
+#define GSDL_COL_TITLE                  109
+#define APPLY_BUTTON                    1000
+#define REMOTE_CHECK                    1001
+#define PROXY_CHECK                     1002
+#define CONSOLE_CHECK                   1003
+#define PROXY_EDIT_BOX                  1004
+#define PROXY_PORT_EDIT_BOX             1005
+#define TrayID                          1006
+#define LOG_DEBUG_CHECK                 1007
+#define SERVER_PORT_EDIT_BOX            1008
+#define LOG_LOCAL_CHECK                 1009
+#define LOG_REMOTE_CHECK                1010
+#define LOG_NAME_BOX                    1011
+#define LOG_CHECK                       1012
+#define SAVE_NAME_BOX                   1013
+#define SAVE_CHECK                      1014
+#define ID_AUTO_ENTER_LIBRARY           1015
+#define ID_RADIO_NETSCAPE               1016
+#define ID_RADIO_IE                     1017
+#define ID_RADIO_OTHER                  1018
+#define ID_OTHER_NAME                   1019
+#define ID_BROWSE_BUTTON                1020
+#define ID_OK_BUTTON                    1021
+#define ID_CANCEL_BUTTON                1022
+#define ID_PROJECT_SETTINGS             40001
+#define ID_PROJECT_EXIT                 40002
+#define ID_INSTALL_NETSCAPE             40007
+
+// Next default values for new objects
+// 
+#ifdef APSTUDIO_INVOKED
+#ifndef APSTUDIO_READONLY_SYMBOLS
+#define _APS_NEXT_RESOURCE_VALUE        118
+#define _APS_NEXT_COMMAND_VALUE         40008
+#define _APS_NEXT_CONTROL_VALUE         1024
+#define _APS_NEXT_SYMED_VALUE           102
+#endif
+#endif
Index: /tags/gsdl-21/gsdl/src/w32server/settings.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/w32server/settings.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/w32server/settings.cpp	(revision 699)
@@ -0,0 +1,787 @@
+#if defined(GSDL_USE_OBJECTSPACE)
+#  include <ospace\std\fstream>
+#elif defined(GSDL_USE_IOS_H)
+#  include <fstream.h>
+#else
+#  include <fstream>
+#endif
+
+#include <windows.h>
+//#include <stdlib.h>
+//#include <stdio.h>
+//#include <string.h>
+//#include <memory.h>
+//#pragma hdrstop
+#include "httpsrv.h"
+#include "settings.h"
+#include "locate.h"
+#include "resource.h"
+
+#include "text_t.h"
+#include "conftools.h"
+
+
+// library settings
+char gs_enterlib[MAX_FILENAME_SIZE] = "";
+char gs_gsdlhome[MAX_FILENAME_SIZE] = "";
+char gs_macroshome[MAX_FILENAME_SIZE] = "";
+char gs_gdbmhome[MAX_FILENAME_SIZE] = "";
+char gs_indexhome[MAX_FILENAME_SIZE] = "";
+
+char gs_staticpages[MAX_FILENAME_SIZE] = "d:\\static";
+
+
+// debug settings
+char gs_log_name[MAX_FILENAME_SIZE] = "c:\\gsdl.log";
+int gs_keep_log = 0;
+int gs_show_console = 0;
+
+// general settings
+int gs_port_num = 80;
+int gs_auto_enter = 0;
+int gs_browser = GS_NETSCAPE;
+char gs_browser_exe[MAX_FILENAME_SIZE]="";
+
+
+
+// private data
+
+// pathnames of the latest installed versions of netscape and
+// internet explorer
+static char netscape_exe[MAX_FILENAME_SIZE]="";
+static char iexplore_exe[MAX_FILENAME_SIZE]="";
+
+// pathname of the default browser
+static char default_browser_exe[MAX_FILENAME_SIZE]="";
+
+// whether netscape is needed (used in the settings dialog box)
+static int gs_netscapeneeded = 0;
+
+
+static void remove_end_slashes(char *str) {
+	int len = strlen (str);
+	while ((len > 0) && ((str[len-1] == '\\') || (str[len-1] == '/'))) {
+		len --;
+	}
+	str[len] = '\0';
+}
+
+
+// returns 1 if successful, 0 otherwise
+// checks for a file's existence
+static int check_program_exe (char *path) {
+	UINT curerrormode;
+	OFSTRUCT reOpenBuff;
+	int retvalue = 0;
+
+	reOpenBuff.cBytes = sizeof (OFSTRUCT);
+
+	curerrormode = SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOOPENFILEERRORBOX);
+	if (OpenFile(path, &reOpenBuff, OF_EXIST) != HFILE_ERROR) retvalue = 1;
+	SetErrorMode(curerrormode);
+
+	return retvalue;
+}
+
+
+// returns 1 if successful, 0 otherwise
+static int get_default_browser_path (char *providedhtmltype, char *path, int maxsize) {
+	HKEY hk1, hk2;
+	char htmltype[256];
+	long size;
+	int retval = 0;
+
+	htmltype[0] = '\0';
+	path[0] = '\0';
+
+
+	// try and get information about the default browser from the 
+	// registry
+
+	if (providedhtmltype != NULL) strcpy (htmltype, providedhtmltype);
+
+	else if (RegOpenKey(HKEY_CLASSES_ROOT, ".htm", &hk1) == ERROR_SUCCESS) {
+		size = 256;
+		RegQueryValue(hk1, NULL, htmltype, &size);
+		RegCloseKey(hk1);
+	}
+
+	if (RegOpenKey(HKEY_CLASSES_ROOT, htmltype, &hk1) == ERROR_SUCCESS) {
+		if (RegOpenKey(hk1, "shell\\open", &hk2) == ERROR_SUCCESS) {
+			// look for the command to start the browser
+			size = maxsize;
+			if (RegQueryValue(hk2,"command",path,&size) == ERROR_SUCCESS) {
+				// strip all excess stuff off the command
+				char endchar = ' ';
+				int i = 0, len = 0;
+				int state = 0; // 0=strip initial white-space and quotes, 1=grab path
+				while (path[i] != '\0') {
+					if (state == 0) {
+						if (path[i] == ' ') {
+							// do nothing
+						} else if (path[i] == '"') {
+							endchar = '"';
+							state = 1;
+						} else {
+							state = 1;
+							path[len] = path[i];
+							len++;
+						}
+					} else {
+						if (path[i] == endchar) break;
+						path[len] = path[i];
+						len++;
+					}
+					i++;
+				}
+				path[len] = '\0';
+
+				// check the resulting pathname
+				if (check_program_exe (path)) {
+					retval = 1; // succeeded
+				} else {
+					path[0] = '\0'; // failed (must have been deleted or something...)
+				}
+			}
+		    RegCloseKey(hk2);
+		}
+		RegCloseKey(hk1);
+	}
+
+	return retval;
+}
+
+
+// returns 1 if successful, 0 otherwise
+// note that appname should include .exe (e.g. netscape.exe)
+static int registry_get_app_paths (char *appname, char *path, int maxsize) {
+	HKEY hk1;
+	long size;
+	int retval = 0;
+
+	path[0] = '\0';
+
+	if (RegOpenKey(HKEY_LOCAL_MACHINE, 
+			"Software\\Microsoft\\Windows\\CurrentVersion\\App Paths", 
+			&hk1) == ERROR_SUCCESS) {
+		size = maxsize;
+		if (RegQueryValue (hk1, appname, path, &size) == ERROR_SUCCESS &&
+				check_program_exe (path));
+			retval = 1;
+
+		RegCloseKey(hk1);
+	}
+
+	return retval;
+}
+
+static void check_installed_browsers (int netscapeneeded) {
+	netscape_exe[0] = '\0';
+	iexplore_exe[0] = '\0';
+	default_browser_exe[0] = '\0';
+
+	// update the default browser information (if netscape isn't needed)
+	if (!netscapeneeded) {
+		get_default_browser_path (NULL, default_browser_exe, MAX_FILENAME_SIZE);
+		_strlwr (default_browser_exe);
+	}
+
+
+	// look for netscape
+
+	// first look for netscape in the registry
+	if (!registry_get_app_paths("netscape.exe", netscape_exe, MAX_FILENAME_SIZE)) {
+		// next look for "NetscapeMarkup"
+		if (get_default_browser_path ("NetscapeMarkup", netscape_exe, MAX_FILENAME_SIZE)) {
+			_strlwr (netscape_exe);
+			if (!strstr (netscape_exe, "netscape.exe")) {
+				netscape_exe[0] = '\0';
+
+				// next look for \netscape\netscape.exe
+
+				// get drive letter of the windows directory
+				GetWindowsDirectory(netscape_exe, MAX_FILENAME_SIZE);
+				netscape_exe[1] = '\0';
+				strcat (netscape_exe, ":\\netscape\\netscape.exe");
+
+				if (!check_program_exe (netscape_exe)) {
+					// lastly look for \netscape\program\netscape.exe
+					netscape_exe[1] = '\0';
+					strcat (netscape_exe, ":\\netscape\\program\\netscape.exe");
+					
+				}
+			}
+		}
+	}
+	_strlwr (netscape_exe);
+	if ((strlen (netscape_exe) > 0) && !check_program_exe (netscape_exe)) {
+		netscape_exe[0] = '\0';  // couldn't find netscape
+	}
+
+
+	// look for internet explorer (if netscape isn't needed)
+
+	// first look for iexplore in the registry
+	if (!netscapeneeded && !registry_get_app_paths("iexplore.exe", 
+			iexplore_exe, MAX_FILENAME_SIZE)) {
+		// next look for "htmlfile"
+		if (get_default_browser_path ("htmlfile", iexplore_exe, MAX_FILENAME_SIZE)) {
+			_strlwr (iexplore_exe);
+			if (!strstr (iexplore_exe, "iexplore.exe")) {
+				iexplore_exe[0] = '\0';
+
+				// next look for \iexplore\iexplore.exe
+
+				// get drive letter of the windows directory
+				GetWindowsDirectory(iexplore_exe, MAX_FILENAME_SIZE);
+				iexplore_exe[1] = '\0';
+				strcat (iexplore_exe, ":\\iexplore\\iexplore.exe");
+			}
+		}
+	}
+	_strlwr (iexplore_exe);
+	if ((strlen (iexplore_exe) > 0) && !check_program_exe (iexplore_exe)) {
+		iexplore_exe[0] = '\0';  // couldn't find internet explorer
+	}
+
+}
+
+
+void write_settings () {
+	char conffile[MAX_FILENAME_SIZE];
+
+	strcpy (conffile, data_location);
+	remove_end_slashes (conffile);
+	strcat (conffile, "\\gsdl.ini");
+
+	ofstream fout (conffile);
+	if (fout) {
+		write_confline(fout, "enterlib", gs_enterlib);
+		write_confline(fout, "gsdlhome", gs_gsdlhome);
+		write_confline(fout, "macroshome", gs_macroshome);
+		write_confline(fout, "gdbmhome", gs_gdbmhome);
+		write_confline(fout, "indexhome", gs_indexhome);
+
+		write_confline(fout, "staticpages", gs_staticpages);
+
+		write_confline(fout, "logfilename", gs_log_name);
+		write_confline(fout, "keeplog", text_t(gs_keep_log));
+		write_confline(fout, "consolelog", text_t(gs_show_console));
+
+		write_confline(fout, "portnumber", text_t(gs_port_num));
+		write_confline(fout, "autoenter", text_t(gs_auto_enter));
+		write_confline(fout, "browser", text_t(gs_browser));
+		write_confline(fout, "browserexe", gs_browser_exe);
+		
+		fout.close();
+	}
+}
+
+
+// browser_compare assumes that check_exe is in lowercase
+static int browser_compare (char *browser_exe_path, char *check_exe) {
+	char tmp_exe[MAX_FILENAME_SIZE];
+	strcpy (tmp_exe, browser_exe_path);
+	_strlwr (tmp_exe);
+	return (strstr (tmp_exe, check_exe) != NULL);
+}
+
+// This function assumes that check_installed_browsers
+// has been recently called
+// It also assumes that browser_exe is large
+// enough to receive any path names to any browser
+static void check_browser_settings (int &browser, char *browser_exe, 
+							 int netscapeneeded) {
+
+	// sort out which browser to use
+	if ((browser == GS_NETSCAPE) && (netscape_exe[0] == '\0'))
+		browser = GS_IEXPLORE;
+	if ((browser == GS_IEXPLORE) && (netscapeneeded || (iexplore_exe[0] == '\0'))) {
+		if (netscape_exe[0] != '\0') browser = GS_NETSCAPE;
+		else browser = GS_OTHER;
+	}
+	if ((browser == GS_OTHER) && netscapeneeded && 
+		!browser_compare (browser_exe, "netscape.exe") && (netscape_exe[0] != '\0')) {
+		// this other browser isn't netscape and netscape is available
+		browser = GS_NETSCAPE;
+	}
+
+
+	// get the browser's executable
+	if (browser == GS_NETSCAPE) {
+		strcpy (browser_exe, netscape_exe);
+	} else if (browser == GS_IEXPLORE) {
+		strcpy (browser_exe, iexplore_exe);
+	} else {
+		browser = GS_OTHER;
+		
+		// if the browser should be netscape then check to
+		// make sure it is
+		if (netscapeneeded && !browser_compare (browser_exe, "netscape.exe")) {
+			// not netscape, set to no browser
+			browser_exe[0] = '\0';
+		}
+	}
+}
+
+
+static void revert_defaults (int netscapeneeded) {
+	find_location();
+	
+	// library settings
+	strcpy (gs_enterlib,"/cgi-bin/gw");
+	strcpy (gs_gsdlhome, data_location);
+	strcpy (gs_macroshome, data_location);
+	strcpy (gs_gdbmhome, data_location);
+	strcpy (gs_indexhome, data_location);
+
+	strcpy (gs_staticpages, "d:\\static");
+
+	// debug settings
+	gs_keep_log = 0;
+	strcpy (gs_log_name, "c:\\gsdl.log");
+	gs_show_console = 0;
+
+	// general settings
+	gs_port_num = 80;
+	gs_auto_enter = 0;
+
+	check_installed_browsers (netscapeneeded);
+	gs_browser = GS_NETSCAPE;
+	strcpy (gs_browser_exe, default_browser_exe);
+	check_browser_settings (gs_browser, gs_browser_exe, netscapeneeded);
+}
+
+
+// debug settings
+
+// general settings
+extern int gs_port_num;                        // portnumber
+extern int gs_auto_enter;                      // autoenter
+extern int gs_browser;                         // browser
+extern char gs_browser_exe[MAX_FILENAME_SIZE]; // browserexe
+
+
+
+void read_settings (int netscapeneeded) {
+	char conffile[MAX_FILENAME_SIZE];
+
+	find_location();
+
+	strcpy (conffile, data_location);
+	remove_end_slashes (conffile);
+	strcat (conffile, "\\gsdl.ini");
+
+	// set up defaults
+	revert_defaults (netscapeneeded);
+
+	text_t key, value;
+	char *cstr_value;
+	ifstream conf (conffile, ios::nocreate);
+	if (conf) {
+		while (get_confline(conf, key, value) >= 0) {
+			cstr_value = value.getcstr ();
+
+			if (key == "enterlib") {
+				strcpy (gs_enterlib, cstr_value);
+
+			} else if (key == "gsdlhome") {
+				strcpy (gs_gsdlhome, cstr_value);
+				strcpy (gs_macroshome, cstr_value);
+				strcpy (gs_gdbmhome, cstr_value);
+				strcpy (gs_indexhome, cstr_value);
+
+			} else if (key == "macroshome") {
+				strcpy (gs_macroshome, cstr_value);
+
+			} else if (key == "gdbmhome") {
+				strcpy (gs_gdbmhome, cstr_value);
+
+			} else if (key == "indexhome") {
+				strcpy (gs_indexhome, cstr_value);
+
+
+			} else if (key == "staticpages") {
+				strcpy (gs_staticpages, cstr_value);
+
+
+			// 'logfilename' should occur in the configuration file
+			// before 'keeplog'
+			} else if (key == "logfilename") {
+				strcpy (gs_log_name, cstr_value);
+
+			} else if (key == "keeplog") {
+				gs_keep_log = value.getint();
+				if (gs_keep_log) open_log_file();
+
+			} else if (key == "consolelog") {
+				if (value.getint()) activate_console();
+				else deactivate_console();
+
+			
+			} else if (key == "portnumber") {
+				gs_port_num = value.getint();
+
+			} else if (key == "autoenter") {
+				gs_auto_enter = value.getint();
+
+			} else if (key == "browser") {
+				gs_browser = value.getint();
+
+			} else if (key == "browserexe") {
+				strcpy (gs_browser_exe, cstr_value);
+			}
+
+			delete cstr_value;
+		}
+		conf.close();
+	}
+
+	// check the homes to make sure they don't contain
+	// extra slashes at the end
+	remove_end_slashes (gs_gsdlhome);
+	remove_end_slashes (gs_macroshome);
+	remove_end_slashes (gs_gdbmhome);
+	remove_end_slashes (gs_indexhome);
+	remove_end_slashes (gs_staticpages);
+
+	// check the browser settings
+	check_browser_settings (gs_browser, gs_browser_exe, netscapeneeded);
+}
+
+
+
+
+void gs_check_browser_settings (int netscapeneeded) {
+	check_installed_browsers (netscapeneeded);
+	check_browser_settings (gs_browser, gs_browser_exe, netscapeneeded);
+}
+
+
+static void dialog_update_enables (HWND hwndDlg) {
+	int res;
+
+	// if they want to use another browser, they should be able
+	// edit its filename
+	res = SendDlgItemMessage(hwndDlg, ID_RADIO_OTHER, BM_GETCHECK,0,0);
+	SendDlgItemMessage(hwndDlg, ID_OTHER_NAME, WM_ENABLE, (res == 1), 0);
+}
+
+
+static int read_dialog_browser_field (HWND hwndDlg) {
+	if (SendDlgItemMessage(hwndDlg, ID_RADIO_NETSCAPE,
+			BM_GETCHECK, 0, 0) == 1) return GS_NETSCAPE;
+	if (SendDlgItemMessage(hwndDlg, ID_RADIO_IE,
+			BM_GETCHECK, 0, 0) == 1) return GS_IEXPLORE;
+	if (SendDlgItemMessage(hwndDlg, ID_RADIO_OTHER,
+			BM_GETCHECK, 0, 0) == 1) return GS_OTHER;
+	return GS_NETSCAPE;
+}
+
+
+
+static void set_dialog_browser_field (HWND hwndDlg, int browser, char *othername) {
+	// if we are trying to set the browser to netscape or
+	// internet explorer and we can't find them, set the browser
+	// to 'other'
+	if ((browser == GS_NETSCAPE) && (netscape_exe[0] == '\0'))
+		browser = GS_IEXPLORE;
+	if ((browser == GS_IEXPLORE) && (iexplore_exe[0] == '\0')) {
+		if (netscape_exe[0] != '\0') browser = GS_NETSCAPE;
+		else browser = GS_OTHER;
+	}
+
+	// update the radio buttons
+	SendDlgItemMessage(hwndDlg, ID_RADIO_NETSCAPE, BM_SETCHECK,
+		(browser == GS_NETSCAPE) ? BST_CHECKED : BST_UNCHECKED, 0);
+	SendDlgItemMessage(hwndDlg, ID_RADIO_IE, BM_SETCHECK,
+		(browser == GS_IEXPLORE) ? BST_CHECKED : BST_UNCHECKED, 0);
+	SendDlgItemMessage(hwndDlg, ID_RADIO_OTHER, BM_SETCHECK,
+		(browser == GS_OTHER) ? BST_CHECKED : BST_UNCHECKED, 0);
+
+	// update the other name field
+	if (browser == GS_NETSCAPE) {
+		SetDlgItemText(hwndDlg, ID_OTHER_NAME, netscape_exe);
+	} else if (browser == GS_IEXPLORE) {
+		SetDlgItemText(hwndDlg, ID_OTHER_NAME, iexplore_exe);
+	} else if (othername != NULL) {
+		SetDlgItemText(hwndDlg, ID_OTHER_NAME, othername);
+	}
+}
+
+
+static void read_browser_exe_field (HWND hwndDlg, char *filename, int maxsize) {
+	filename[0] = '\0';
+	GetDlgItemText(hwndDlg, ID_OTHER_NAME, filename, maxsize);
+}
+
+
+static void read_dialog_fields (HWND hwndDlg) {
+	int res;  BOOL bres;
+
+	// port number
+	gs_port_num = GetDlgItemInt(hwndDlg, SERVER_PORT_EDIT_BOX, &bres, 0);
+	if (!bres) gs_port_num = 80;
+	
+	// whether to enter the library automatically on startup
+	res = SendDlgItemMessage(hwndDlg, ID_AUTO_ENTER_LIBRARY, BM_GETCHECK, 0, 0);
+	gs_auto_enter = (res == 1);
+
+	// which browser to use
+	gs_browser = read_dialog_browser_field (hwndDlg);
+
+	// find out where the browser exe lives
+	read_browser_exe_field (hwndDlg, gs_browser_exe, MAX_FILENAME_SIZE);
+}
+
+
+
+
+static BOOL CALLBACK dialog_proc(HWND hwndDlg, UINT uMsg, 
+								WPARAM wParam,LPARAM  lParam) {
+	int wNotifyCode = HIWORD(wParam);
+	int wID = LOWORD(wParam);
+	char *filefilter = "Program Files (*.exe)\0*.exe\0All Files (*.*)\0*.*\0\0";
+	OPENFILENAME fname;
+	char filename[MAX_FILENAME_SIZE];
+
+	switch (uMsg) {
+	case WM_INITDIALOG:
+		// set current values
+		SetDlgItemInt(hwndDlg, SERVER_PORT_EDIT_BOX, gs_port_num, FALSE);
+		SendDlgItemMessage(hwndDlg, ID_AUTO_ENTER_LIBRARY, BM_SETCHECK,
+			gs_auto_enter ? 1 : 0, 0);
+		set_dialog_browser_field (hwndDlg, gs_browser, gs_browser_exe);
+		dialog_update_enables(hwndDlg);
+
+		// make sure that the netscape and internet explorer
+		// radio buttons are only enabled if they could be found
+		if (netscape_exe[0] == '\0')
+			EnableWindow (GetDlgItem (hwndDlg, ID_RADIO_NETSCAPE), FALSE);
+		if (iexplore_exe[0] == '\0')
+			EnableWindow (GetDlgItem (hwndDlg, ID_RADIO_IE), FALSE);
+
+		return 1;
+
+	case WM_COMMAND:
+		if (wNotifyCode == BN_CLICKED) {
+			switch (wID) {
+			case ID_CANCEL_BUTTON:
+				EndDialog(hwndDlg, 0);
+				return 1;
+
+			case ID_OK_BUTTON:
+				// check to make sure we can find the browser
+				filename[0] = '\0';
+				read_browser_exe_field (hwndDlg, filename, MAX_FILENAME_SIZE);
+				if (check_program_exe (filename)) { 
+					// the file exists
+					// if netscape is needed, make sure the browser is netscape
+					_strlwr(filename);
+					if (!gs_netscapeneeded || strstr (filename, "netscape.exe") != NULL) {
+						// everything is ok
+						read_dialog_fields(hwndDlg);
+						EndDialog(hwndDlg, 1);
+
+					} else {
+						// message: must use netscape with this version
+						MessageBox(hwndDlg,
+								"You are using this software on a machine which\n"
+								"currently does not have a working TCP/IP network layer.\n"
+								"To allow the Greenstone Digital Library software to use\n"
+								"its own TCP/IP network layer you must use a Netscape\n"
+								"browser. If you do not have Netscape installed, please\n"
+								"choose 'Install Netscape 4.05' from the 'Files' menu.",
+								"Greenstone Digital Library Software",
+								MB_OK|MB_APPLMODAL);
+					}
+				} else {
+					// message: the file did not exist
+					MessageBox(hwndDlg,
+							"Could not find the selected browser. If you do not\n"
+							"have a browser installed, please choose\n"
+							"'Install Netscape 4.05' from the 'Files' menu.",
+							"Greenstone Digital Library Software",
+							MB_OK|MB_APPLMODAL);
+				}
+				return 1;
+
+			case ID_RADIO_NETSCAPE:
+				set_dialog_browser_field (hwndDlg, GS_NETSCAPE, NULL);
+				dialog_update_enables(hwndDlg);
+				return 1;
+
+			case ID_RADIO_IE:
+				set_dialog_browser_field (hwndDlg, GS_IEXPLORE, NULL);
+				dialog_update_enables(hwndDlg);
+				return 1;
+
+			case ID_RADIO_OTHER:
+				set_dialog_browser_field (hwndDlg, GS_OTHER, NULL);
+				dialog_update_enables(hwndDlg);
+				return 1;
+
+			case ID_BROWSE_BUTTON:
+				filename[0] = '\0';
+				memset(&fname, 0, sizeof(OPENFILENAME));
+				fname.lStructSize = sizeof(OPENFILENAME);
+				fname.hwndOwner = hwndDlg;
+				fname.hInstance = NULL;
+				fname.lpstrFilter = filefilter;
+				fname.lpstrCustomFilter = NULL;
+				fname.nMaxCustFilter = 0;
+				fname.nFilterIndex = 1;
+				fname.lpstrFile = filename;
+				fname.nMaxFile = MAX_FILENAME_SIZE-1;
+				fname.lpstrFileTitle = NULL;
+				fname.nMaxFileTitle = 0;
+				fname.lpstrInitialDir = NULL;
+				fname.lpstrTitle = "Choose Web Browser";
+				fname.Flags = OFN_FILEMUSTEXIST | OFN_HIDEREADONLY;
+
+				if(GetOpenFileName(&fname)) {
+					set_dialog_browser_field (hwndDlg, GS_OTHER, filename);
+					dialog_update_enables(hwndDlg);
+				}
+				return 1;
+			}
+		}
+		return 0;
+
+	case WM_CLOSE:
+	case WM_DESTROY:
+	    EndDialog(hwndDlg, 0);
+		return 1;
+	}
+
+	return 0;
+}
+
+
+
+static void dialog_debug_update_enables (HWND hwndDlg) {
+	int res;
+
+	// if they want a log of program use allow them to
+	// edit its filename
+	res = SendDlgItemMessage(hwndDlg, LOG_CHECK, BM_GETCHECK,0,0);
+	SendDlgItemMessage(hwndDlg, LOG_NAME_BOX, WM_ENABLE, (res == 1), 0);
+}
+
+
+static void read_dialog_debug_fields (HWND hwndDlg) {
+	int res;
+
+	// whether to log program use
+	res = SendDlgItemMessage(hwndDlg, LOG_CHECK, BM_GETCHECK,0,0);
+	gs_keep_log = (res == 1);
+
+	// log filename
+	res = SendDlgItemMessage(hwndDlg, LOG_NAME_BOX, EM_GETMODIFY,0,0);
+	if (res != 0) {
+		res = GetDlgItemText(hwndDlg, LOG_NAME_BOX, gs_log_name, MAX_FILENAME_SIZE);
+	}
+
+	// whether to display log information on console
+	res = SendDlgItemMessage(hwndDlg, CONSOLE_CHECK, BM_GETCHECK,0,0);
+	gs_show_console = (res == 1);
+}
+
+
+static BOOL CALLBACK dialog_debug_proc(HWND hwndDlg, UINT uMsg, 
+										WPARAM wParam,LPARAM  lParam) {
+	int wNotifyCode = HIWORD(wParam);
+	int wID = LOWORD(wParam);
+
+	switch (uMsg) {
+	case WM_INITDIALOG:
+		// set current values
+		SendDlgItemMessage(hwndDlg, LOG_CHECK, BM_SETCHECK,
+			gs_keep_log ? 1 : 0, 0);
+		SetDlgItemText(hwndDlg, LOG_NAME_BOX, gs_log_name);
+		SendDlgItemMessage(hwndDlg, CONSOLE_CHECK, BM_SETCHECK,
+			gs_show_console ? 1 : 0, 0);
+		dialog_debug_update_enables(hwndDlg);
+		return 1;
+
+	case WM_COMMAND:
+		if (wNotifyCode == BN_CLICKED) {
+			switch (wID) {
+			case ID_CANCEL_BUTTON:
+				EndDialog(hwndDlg, 0);
+				return 1;
+
+			case ID_OK_BUTTON:
+				read_dialog_debug_fields(hwndDlg);
+				EndDialog(hwndDlg, 1);
+				return 1;
+
+			case LOG_CHECK:
+				dialog_debug_update_enables(hwndDlg);
+				return 1;
+			}
+		}
+		return 0;
+
+	case WM_CLOSE:
+	case WM_DESTROY:
+	    EndDialog(hwndDlg, 0);
+		return 1;
+	}
+
+	return 0;
+}
+
+
+void Settings_Dialog(HWND window, int netscapeneeded)
+{
+	int old_gs_show_console = gs_show_console;
+	int old_gs_port_num = gs_port_num;
+
+	int res = 0;
+	
+
+	gs_netscapeneeded = netscapeneeded;
+
+	if ((GetAsyncKeyState(VK_CONTROL) & 0x8000) && 
+		(GetAsyncKeyState(VkKeyScan('d')) & 0x8000)) {
+		// control-D was pressed, display debug settings
+		// dialog box
+		res = DialogBox (NULL, MAKEINTRESOURCE(COMMS_DIALOG_DEBUG),
+			window, (DLGPROC)dialog_debug_proc);
+
+		if (res == 1) {
+			// only update if exited via the OK button
+			if (gs_keep_log) open_log_file(); 
+			else close_log_file();
+
+			// turn the console on/off if we need to
+			if (gs_show_console != old_gs_show_console) {
+				if (gs_show_console) activate_console ();
+				else deactivate_console ();
+			}
+		}
+
+	} else {
+		// display normal settings dialog box
+
+		// make sure everything is up-to-date
+		gs_check_browser_settings (netscapeneeded);
+
+		res = DialogBox (NULL, MAKEINTRESOURCE(COMMS_DIALOG2),
+			window, (DLGPROC)dialog_proc);
+
+		if (res == 1) {
+			// only update if exited via the OK button
+			if (gs_port_num != old_gs_port_num) {
+				EndHTTPServer();
+				StartHTTPServer(window);
+			}
+		}
+	}
+
+	// save the settings if we exited via the ok button
+	if (res == 1) {
+		write_settings();
+	}
+}
Index: /tags/gsdl-21/gsdl/src/w32server/settings.h
===================================================================
--- /tags/gsdl-21/gsdl/src/w32server/settings.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/w32server/settings.h	(revision 699)
@@ -0,0 +1,41 @@
+#ifndef SETTINGS_H
+#define SETTINGS_H
+
+#ifndef MAX_FILENAME_SIZE
+#define MAX_FILENAME_SIZE 1024
+#endif
+
+#define GS_NETSCAPE 0
+#define GS_IEXPLORE 1
+#define GS_OTHER    2
+
+
+// library settings
+extern char gs_enterlib[MAX_FILENAME_SIZE];    // enterlib
+extern char gs_gsdlhome[MAX_FILENAME_SIZE];    // gsdlhome
+extern char gs_macroshome[MAX_FILENAME_SIZE];  // macroshome
+extern char gs_gdbmhome[MAX_FILENAME_SIZE];    // gdbmhome
+extern char gs_indexhome[MAX_FILENAME_SIZE];   // indexhome
+
+extern char gs_staticpages[MAX_FILENAME_SIZE]; // staticpages
+
+// debug settings
+extern char gs_log_name[MAX_FILENAME_SIZE];    // logfilename
+extern int gs_keep_log;                        // keeplog
+extern int gs_show_console;                    // consolelog
+
+// general settings
+extern int gs_port_num;                        // portnumber
+extern int gs_auto_enter;                      // autoenter
+extern int gs_browser;                         // browser
+extern char gs_browser_exe[MAX_FILENAME_SIZE]; // browserexe
+
+
+void Settings_Dialog(HWND window, int netscapeneeded);
+
+void read_settings (int netscapeneeded);
+void write_settings ();
+
+void gs_check_browser_settings (int netscapeneeded);
+
+#endif
Index: /tags/gsdl-21/gsdl/src/w32server/startbrowser.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/w32server/startbrowser.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/w32server/startbrowser.cpp	(revision 699)
@@ -0,0 +1,277 @@
+#include "startbrowser.h"
+#include <windows.h>
+#include <DDE.H>
+#include <DDEML.H>
+#include <string.h>
+#include <direct.h>
+#include "locate.h"
+
+#define SB_COMLEN 1024
+#define SB_DDETIMEOUT 60000UL  // One minute in milliseconds
+
+int numbrowserstarted = 0;
+
+static int inited = 0;
+
+static DWORD idInst = 0;
+static HINSTANCE hinst = NULL;
+static HSZ hszNetscapeName = NULL;
+static HSZ hszIExploreName = NULL;
+static HSZ hszOpenURLTopic = NULL;
+static HSZ hszActivateTopic = NULL;
+
+
+// budget callback
+HDDEDATA CALLBACK DdeCallback(UINT uType,     /* transaction type                 */
+							  UINT uFmt,      /* clipboard data format            */
+							  HCONV hconv,    /* handle of conversation           */
+							  HSZ hsz1,       /* handle of string                 */
+							  HSZ hsz2,       /* handle of string                 */
+							  HDDEDATA hdata, /* handle of global memory object   */
+							  DWORD dwData1,  /* transaction-specific data        */
+							  DWORD dwData2)  /* transaction-specific data        */ 
+{
+	return (HDDEDATA) NULL; 
+}
+
+
+// returns 0 on failure
+static int talkdde (HSZ hszServName, HSZ hszSysTopic, char *requeststr) {
+	HSZ hszRequest;
+	HCONV hConv;
+	HDDEDATA result;
+
+	// try to connect to the newly started browser
+	hConv = DdeConnect (idInst, hszServName, hszSysTopic, (PCONVCONTEXT) NULL);
+	if (hConv == NULL) return 0;
+
+	hszRequest = DdeCreateStringHandle (idInst, requeststr, CP_WINANSI);
+
+//HDDEDATA hRetVal = DdeClientTransaction(NULL, 0ul, m_hConv, 
+//								hszItem, CF_TEXT, 
+//								XTYP_REQUEST, 
+//								DDETIMEOUT, NULL);
+	result = DdeClientTransaction (
+		NULL,			// pointer to data to pass to server
+		0,				// length of data
+		hConv,			// handle to conversation
+		hszRequest,		// handle to item name string
+		CF_TEXT,		// clipboard data format
+		XTYP_REQUEST,	// transaction type
+		SB_DDETIMEOUT,	// time-out duration
+		NULL		 	// pointer to transaction result
+	);
+
+	if (result != NULL) DdeFreeDataHandle (result);
+	DdeDisconnect(hConv);
+	DdeFreeStringHandle (idInst, hszRequest);
+
+	return 1;
+}
+
+
+// returns 0 on failure
+static int openurl(HSZ hszServName, HSZ hszActivateTopic, HSZ hszOpenURLTopic, char *url) {
+	char requeststr[65600];
+
+	// bring the browser to the front, don't worry if this fails
+	talkdde (hszServName, hszActivateTopic, "0xFFFFFFFF,0x0");
+
+	// tell the browser to open a url
+	strcpy (requeststr, "\"");
+	strcat (requeststr, url);
+	strcat (requeststr, "\",,0xFFFFFFFF,0x0,,,");
+
+	return talkdde (hszServName, hszOpenURLTopic, requeststr);
+}
+
+// returns 0 on failure
+static int tryconnect(HSZ hszServName, HSZ hszSysTopic) {
+	// try to connect to a running browser
+	HCONV hConv = DdeConnect (idInst, hszServName, hszSysTopic, (PCONVCONTEXT) NULL);
+	if (hConv == NULL) return 0;
+	DdeDisconnect(hConv);
+	return 1;
+}
+
+static void freestrings () {
+	// free up any currently allocated strings
+	if (hszNetscapeName != NULL) DdeFreeStringHandle (idInst, hszNetscapeName);
+	hszNetscapeName = NULL;
+	if (hszIExploreName != NULL) DdeFreeStringHandle (idInst, hszIExploreName);
+	hszIExploreName = NULL;
+	if (hszOpenURLTopic != NULL) DdeFreeStringHandle (idInst, hszOpenURLTopic);
+	hszOpenURLTopic = NULL;
+	if (hszActivateTopic != NULL) DdeFreeStringHandle (idInst, hszActivateTopic);
+	hszActivateTopic = NULL;
+}
+
+
+
+static void getbrowserinfo() {
+	freestrings();
+
+	hszNetscapeName = DdeCreateStringHandle(idInst,"NETSCAPE",CP_WINANSI);
+	hszIExploreName = DdeCreateStringHandle(idInst,"IExplore",CP_WINANSI);
+
+	hszOpenURLTopic = DdeCreateStringHandle(idInst,"WWW_OpenURL",CP_WINANSI);
+	hszActivateTopic = DdeCreateStringHandle(idInst,"WWW_Activate",CP_WINANSI);
+}
+
+
+
+// returns SB_NOERROR on success, 
+// SB_ALREADYINIT or SB_DDE_FAILINIT on failure
+int initstartbrowser () {
+	if (inited) return SB_ALREADYINIT;
+
+// from nstest if(DdeInitialize(&CDDEObject::m_dwidInst, NstestDdeCallBack, APPCLASS_STANDARD, 0ul))
+	if (DdeInitialize (&idInst, (PFNCALLBACK) DdeCallback, 
+		APPCLASS_STANDARD, 0)
+//		CBF_FAIL_EXECUTES | CBF_SKIP_ALLNOTIFICATIONS, 0)
+			!= DMLERR_NO_ERROR) {
+		freestrings();
+		inited = 0;
+		return SB_DDE_FAILINIT;
+	}
+
+	getbrowserinfo();
+	
+	inited = 1;
+	return SB_NOERROR;
+}
+
+
+// returns SB_NOERROR on success,
+// SB_DDE_FAILDEINIT of failure
+int deinitstartbrowser () {
+	// make sure this has been inited
+	// not being inited here, however, is not
+	// important
+	if (!inited) return SB_NOERROR;
+	inited = 0;
+
+	freestrings();
+
+	if (DdeUninitialize(idInst)) return SB_NOERROR;
+
+	return SB_DDE_FAILDEINIT;
+}
+
+
+// startbrowser will try to start a browser (or connect to a running browser).
+//
+// Returns SB_NOERROR on success,
+// SB_NOTINITED, SB_FAIL_BADFORMAT, SB_FAIL_NOTFOUND, SB_FAIL_NORESOURCES, or
+// SB_FAILCONNECTBROWSER on failure
+int startbrowser (char *url, char *browser_exe, char *startdir) {
+	char newcommand[SB_COMLEN];
+	int usingnetscape = 0;
+	int usingiexplore = 0;
+
+	if (!inited) return SB_NOTINITED; // has to be inited first
+
+	// find out which browser we are dealing with
+	strcpy (newcommand, browser_exe);
+	_strlwr (newcommand);
+	if (strstr (newcommand, "netscape.exe") != NULL) { 
+		// netscape
+		usingnetscape = 1;
+	} else if (strstr (newcommand, "iexplore.exe") != NULL) { 
+		// internet explorer
+		usingiexplore = 1;
+	}
+
+	// only try to communicate with a running browser if a startdir
+	// is not specified
+	if (startdir == NULL) {
+		if (usingnetscape && hszNetscapeName != NULL && 
+					openurl(hszNetscapeName, hszActivateTopic, hszOpenURLTopic, url)) {
+			return SB_NOERROR;
+		}
+		if (usingiexplore && hszIExploreName != NULL && 
+					openurl(hszIExploreName, hszActivateTopic, hszOpenURLTopic, url)) {
+			return SB_NOERROR;
+		}
+	}
+
+	strcpy (newcommand, browser_exe);
+
+	// don't put the url on the command line for internet explorer
+	if (usingiexplore) {
+		strcat (newcommand, " -nohome");
+
+	} else {
+		// will have to put the url on the command line
+		strcat (newcommand, " \"");
+		strcat (newcommand, url);
+		strcat (newcommand, "\"");
+	}
+
+	// change the working directory (if needed)
+	char cwd[1024];
+	cwd[0] = '\0';
+	if (startdir != NULL) {
+		_getcwd(cwd, 1024);
+		_chdir(startdir);
+	}
+
+	// attempt to start the browser
+	int res = WinExec(newcommand, SW_SHOW);
+
+	// change the working directory back (if needed)
+	if (startdir != NULL) _chdir(cwd);
+
+	if (res == ERROR_BAD_FORMAT) return SB_FAIL_BADFORMAT;
+	if ((res == ERROR_FILE_NOT_FOUND) || (res == ERROR_PATH_NOT_FOUND)) 
+		return SB_FAIL_NOTFOUND;
+	if (res <= 31) return SB_FAIL_NORESOURCES;
+
+	numbrowserstarted++;
+
+	// if we aren't using iexplore then the url was
+	// put on the command line and there is nothing left to do
+	if (!usingiexplore) return SB_NOERROR;
+
+	// connect to the browser and get it to open a page,
+	// time out after 1 minute
+	DWORD lastcheck = GetTickCount ();
+	while (DiffTickCounts (lastcheck, GetTickCount()) <= 60000) {
+		if (usingnetscape && hszNetscapeName != NULL && 
+					openurl(hszNetscapeName, hszActivateTopic, hszOpenURLTopic, url)) {
+			return SB_NOERROR;
+		}
+		if (usingiexplore && hszIExploreName != NULL && 
+					openurl(hszIExploreName, hszActivateTopic, hszOpenURLTopic, url)) {
+			return SB_NOERROR;
+		}
+	}
+
+	// timed out trying to connect to the browser
+	return SB_FAILCONNECTBROWSER;
+}
+
+
+
+// returns SB_NOERROR on success,
+// SB_NOTINITED, SB_FAILCONNECTBROWSER on failure
+int browserrunning (char *browser_exe) {
+	char newcommand[SB_COMLEN];
+	if (!inited) return SB_NOTINITED; // has to be inited first
+
+	strcpy (newcommand, browser_exe);
+	_strlwr (newcommand);
+
+	if ((strstr (newcommand, "netscape.exe") != NULL) && (hszNetscapeName != NULL) && 
+			tryconnect(hszNetscapeName, hszOpenURLTopic)) {
+		// succeeded connecting to netscape
+		return SB_NOERROR;
+
+	} else if ((strstr (newcommand, "iexplore.exe") != NULL) && (hszIExploreName != NULL) && 
+			tryconnect(hszIExploreName, hszOpenURLTopic)) {
+		// succeeded connecting to internet explorer
+		return SB_NOERROR;
+	}
+
+	return SB_FAILCONNECTBROWSER;
+}
Index: /tags/gsdl-21/gsdl/src/w32server/startbrowser.h
===================================================================
--- /tags/gsdl-21/gsdl/src/w32server/startbrowser.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/w32server/startbrowser.h	(revision 699)
@@ -0,0 +1,33 @@
+#define SB_NOERROR            0
+#define SB_ALREADYINIT        1
+#define SB_DDE_FAILINIT       2
+#define SB_DDE_FAILDEINIT     3
+#define SB_NOTINITED          4
+#define SB_FAIL_BADFORMAT     5
+#define SB_FAIL_NOTFOUND      6
+#define SB_FAIL_NORESOURCES   7
+#define SB_FAILCONNECTBROWSER 9
+
+
+extern int numbrowserstarted;
+
+// initstartbrowser should be called before startbrowser
+//
+// returns SB_NOERROR on success, 
+// SB_ALREADYINIT or SB_DDE_FAILINIT on failure
+int initstartbrowser ();
+
+// returns SB_NOERROR on success,
+// SB_DDE_FAILDEINIT of failure
+int deinitstartbrowser ();
+
+// startbrowser will try to start a browser (or connect to a running browser).
+//
+// Returns SB_NOERROR on success,
+// SB_NOTINITED, SB_FAIL_BADFORMAT, SB_FAIL_NOTFOUND, or SB_FAIL_NORESOURCES, 
+//  on failure
+int startbrowser (char *url, char *browser_exe, char *startdir);
+
+// returns SB_NOERROR on success,
+// SB_NOTINITED, SB_FAILCONNECTBROWSER on failure
+int browserrunning (char *browser_exe);
Index: /tags/gsdl-21/gsdl/src/w32server/wincgiutils.cpp
===================================================================
--- /tags/gsdl-21/gsdl/src/w32server/wincgiutils.cpp	(revision 699)
+++ /tags/gsdl-21/gsdl/src/w32server/wincgiutils.cpp	(revision 699)
@@ -0,0 +1,156 @@
+#include <windows.h>
+#include <string.h>
+#include "netio.h"
+#include "httpreq.h"
+// #include "locate.h"
+#include "wincgiutils.h"
+
+
+#define url_limit 256
+#define file_limit 102400   /*should be large enough for a header line*/
+#define default_http_port 80
+
+
+int parse_url(char *url,
+	      char **protocol, char **machine, int *port, char **name)
+/* Attempt to break a url into segments.  If the parsing
+   fails, 'name' is left pointing to the unparsed portion.
+   This routine may extend the url string by one character */
+{
+  char next, thiss;
+  *protocol = *machine = "";
+  *port = default_http_port;
+  
+  *name = url;       /* In case there is no protocol */
+  while (*url != ':' && *url != 0) url++;
+  if (*url == 0) return http_error_url_invalid;
+  url++;
+  if (*url != '/') return http_error_url_invalid;
+  *url++ = 0;
+  if (*url != '/') return http_error_url_invalid;
+  url++;
+  
+  *protocol = *name;
+  *machine = url;
+  while (*url != ':' && *url != '/' && *url != 0) url++;
+  if (*url == ':') {
+    *url = 0;
+    url++; *port = 0;
+    while (*url >= '0' && *url <= '9')
+      *port = *port * 10 + (*url++ - '0');
+    }
+  if (*url == '/') {
+    *url++ = 0;
+    next = '/';
+    *name = url;
+    while (*url != 0) {
+      thiss = *url;  *url++ = next;  next = thiss; }
+    *url++ = next;
+    *url = 0;
+    }
+  else if (*url == 0) {
+    url++;
+    *name = url;
+    *url++ = '/';
+    *url = 0;
+    }
+  else {
+    *name = url;
+    return http_error_url_invalid;
+  }
+  return http_ok;
+}
+
+
+
+
+int Send_String(char *str, RequestInfoT *RInfo)
+{
+  return
+    SendData(RInfo->ClientSocket, (BYTE *)str, strlen(str), RInfo->ThreadNum);
+}
+
+int Send_String_N(char *str, int n,  RequestInfoT *RInfo)
+{
+  return
+    SendData(RInfo->ClientSocket, (BYTE *)str, n, RInfo->ThreadNum);
+}
+
+int send_header(char *content, RequestInfoT *RequestInfo)
+{
+  char Header[1536];  int len, lenc;
+	//Build Header
+	Header[0] = 0;
+	//Status Line
+	if (content[0] != '@')          /*No redirection needed*/
+	  strcat(Header, "HTTP/1.0 200 OK\r\n");
+	else                            /*Redirection*/
+	  strcat(Header, "HTTP/1.0 302 Relocation\r\n");
+	//Server
+	strcat(Header, "Server: GSDL\r\n");
+	//Content Type
+	strcat(Header, "Content-type: ");
+	if (content[0] != '@')          /*No redirection needed*/
+	  strcat(Header, content);
+	else {                          /*Redirection entry*/
+	  strcat(Header, "text/html \r\n");
+	  strcat(Header, "Location: ");
+	  len = strlen(Header);
+	  lenc = strlen(content) - 1;
+	  strncpy(&Header[len], &content[1], lenc);
+	  Header[len+lenc] = 0;
+	  }
+	strcat(Header, " \r\n");
+	//Single CRLF to end header
+	strcat(Header, "\r\n");
+
+   return Send_String(Header, RequestInfo);
+}
+
+void send_retrieve_error(int error_num, char *msg1, char *msg2, RequestInfoT *RequestInfo)
+{
+  char Header[512], Body[512], ErrorNumStr[17], BodyLengthStr[17];
+  DWORD BodyLength; int HeaderLength;
+
+  itoa(error_num, ErrorNumStr, 10);
+
+  //Build Data (so we can determine length for the header)
+  Body[0] = 0;
+  strcat(Body, "<HTML><HEAD><TITLE>Server Error</TITLE></HEAD><BODY>\n");
+  strcat(Body, "<H1>Error ");
+  strcat(Body, ErrorNumStr);
+  strcat(Body, ": ");
+  strcat(Body, msg1);
+  strcat(Body, "</H1>\n");
+  strcat(Body, msg2);
+  strcat(Body, "\n</BODY></HTML>");
+  BodyLength = (DWORD) strlen(Body);
+
+  //Build Header
+  Header[0] = 0;
+  //Status Line
+  strcat(Header, "HTTP/1.0 ");
+  strcat(Header, ErrorNumStr);
+  strcat(Header, " ");
+  strcat(Header, msg1);
+  strcat(Header, "\r\n");
+  //Server
+  strcat(Header, "Server: GSDL\r\n");
+  strcat(Header, "Content-Type: text/html\r\n");
+  //Content Length
+  itoa(BodyLength, BodyLengthStr, 10);
+  strcat(Header, "Content-Length: ");
+  strcat(Header, BodyLengthStr);
+  strcat(Header, "\r\n");
+  //Single CRLF to end header
+  strcat(Header, "\r\n");
+  HeaderLength = strlen(Header);
+
+   //Send the file if we have one (and if it's not a HEAD request)
+  SendData(RequestInfo->ClientSocket, (BYTE *) Header, strlen(Header), RequestInfo->ThreadNum);
+//   if ((FileFound == FALSE) && (strcmpi(RequestFields.MethodStr, "HEAD") != 0)) {
+	//Send generated data
+  SendData(RequestInfo->ClientSocket, (BYTE *) Body, BodyLength, RequestInfo->ThreadNum);
+
+}
+
Index: /tags/gsdl-21/gsdl/src/w32server/wincgiutils.h
===================================================================
--- /tags/gsdl-21/gsdl/src/w32server/wincgiutils.h	(revision 699)
+++ /tags/gsdl-21/gsdl/src/w32server/wincgiutils.h	(revision 699)
@@ -0,0 +1,37 @@
+#define http_ok 0
+#define http_error_url_invalid 11
+
+
+int parse_url(char *url,
+	      char **protocol, char **machine, int *port, char **name);
+
+
+struct param_vals {char *name, *value; int vsize; char *dfault; };
+
+
+int Send_String(char *str, RequestInfoT *RInfo);
+int Send_String_N(char *str, int n,  RequestInfoT *RInfo);
+void send_retrieve_error(int error_num, char *msg1, char *msg2, RequestInfoT *RequestInfo);
+int send_header(char *content, RequestInfoT *RequestInfo);
+void decode(char *st);
+void decode_template(int ntemp, param_vals tmplate[]);
+void parse_query_string(char *qstring, int ntemp, param_vals tmplate[]);
+int mg_sender(char *UDoc, int ULen, int DocNum, float Weight, void *RInfo);
+
+#define OUT_CONST_A(t) \
+{if (Send_String_N(t, sizeof(t), RInfo) < 0) return;}
+
+#define OUT_CONST_B(t) \
+{if (Send_String_N(t, sizeof(t), (RequestInfoT *)RInfo) < 0) return -1;}
+
+#define OUT_STRING_A(t) \
+{if (Send_String(t, RInfo) < 0) return;}
+
+#define OUT_STRING_B(t) \
+{if (Send_String(t, (RequestInfoT *)RInfo) < 0) return -1;}
+
+#define OUT_STRING_N_A(t,n) \
+{if (Send_String_N(t, n, RInfo) < 0) return;}
+
+#define OUT_STRING_N_B(t,n) \
+{if (Send_String_N(t, n, (RequestInfoT *)RInfo) < 0) return -1;}
Index: /tags/gsdl-21/gsdl/unicode/MAPPINGS/EASTASIA/GB/GB12345.TXT
===================================================================
--- /tags/gsdl-21/gsdl/unicode/MAPPINGS/EASTASIA/GB/GB12345.TXT	(revision 699)
+++ /tags/gsdl-21/gsdl/unicode/MAPPINGS/EASTASIA/GB/GB12345.TXT	(revision 699)
@@ -0,0 +1,7604 @@
+#
+#	Name:             GB12345-80 to Unicode table (complete, hex format)
+#	Unicode version:  1.1
+#	Table version:    0.0d1
+#	Table format:     Format A
+#	Date:             6 December 1993
+#	Author:           Glenn Adams <glenn@metis.com>
+#                     John H. Jenkins <John_Jenkins@taligent.com>
+#
+#	Copyright (c) 1991-1994 Unicode, Inc.  All Rights reserved.
+#
+#	This file is provided as-is by Unicode, Inc. (The Unicode Consortium).
+#	No claims are made as to fitness for any particular purpose.  No
+#	warranties of any kind are expressed or implied.  The recipient
+#	agrees to determine applicability of information provided.  If this
+#	file has been provided on magnetic media by Unicode, Inc., the sole
+#	remedy for any claim will be exchange of defective media within 90
+#	days of receipt.
+#
+#	Recipient is granted the right to make copies in any form for
+#	internal distribution and to freely use the information supplied
+#	in the creation of products supporting Unicode.  Unicode, Inc.
+#	specifically excludes the right to re-distribute this file directly
+#	to third parties or other organizations whether for profit or not.
+#
+#	General notes:
+#
+#	This table contains the data Metis and Taligent currently have on how
+#       GB12345-90 characters map into Unicode.
+#
+#	Format:  Three tab-separated columns
+#		 Column #1 is the GB12345 code (in hex as 0xXXXX)
+#		 Column #2 is the Unicode (in hex as 0xXXXX)
+#		 Column #3 the Unicode name (follows a comment sign, '#')
+#					The official names for Unicode characters U+4E00
+#					to U+9FA5, inclusive, is "CJK UNIFIED IDEOGRAPH-XXXX",
+#					where XXXX is the code point.  Including all these
+#					names in this file increases its size substantially
+#					and needlessly.  The token "<CJK>" is used for the
+#					name of these characters.  If necessary, it can be
+#					expanded algorithmically by a parser or editor.
+#
+#	The entries are in GB12345 order
+#
+#	The following algorithms can be used to change the hex form
+#		of GB12345 to other standard forms:
+#
+#		To change hex to EUC form, add 0x8080
+#		To change hex to kuten form, first subtract 0x2020.  Then
+#			the high and low bytes correspond to the ku and ten of
+#			the kuten form.  For example, 0x2121 -> 0x0101 -> 0101;
+#			0x777E -> 0x575E -> 8794
+#
+#	Any comments or problems, contact <John_Jenkins@taligent.com>
+#
+#
+0x2121	0x3000	# IDEOGRAPHIC SPACE
+0x2122	0x3001	# IDEOGRAPHIC COMMA
+0x2123	0x3002	# IDEOGRAPHIC FULL STOP
+0x2124	0x30FB	# KATAKANA MIDDLE DOT
+0x2125	0x02C9	# MODIFIER LETTER MACRON (Mandarin Chinese first tone)
+0x2126	0x02C7	# CARON (Mandarin Chinese third tone)
+0x2127	0x00A8	# DIAERESIS
+0x2128	0x3003	# DITTO MARK
+0x2129	0x3005	# IDEOGRAPHIC ITERATION MARK
+0x212A	0x2015	# HORIZONTAL BAR
+0x212B	0xFF5E	# FULLWIDTH TILDE
+0x212C	0x2225	# PARALLEL TO
+0x212D	0x2026	# HORIZONTAL ELLIPSIS
+0x212E	0x2018	# LEFT SINGLE QUOTATION MARK
+0x212F	0x2019	# RIGHT SINGLE QUOTATION MARK
+0x2130	0x201C	# LEFT DOUBLE QUOTATION MARK
+0x2131	0x201D	# RIGHT DOUBLE QUOTATION MARK
+0x2132	0x3014	# LEFT TORTOISE SHELL BRACKET
+0x2133	0x3015	# RIGHT TORTOISE SHELL BRACKET
+0x2134	0x3008	# LEFT ANGLE BRACKET
+0x2135	0x3009	# RIGHT ANGLE BRACKET
+0x2136	0x300A	# LEFT DOUBLE ANGLE BRACKET
+0x2137	0x300B	# RIGHT DOUBLE ANGLE BRACKET
+0x2138	0x300C	# LEFT CORNER BRACKET
+0x2139	0x300D	# RIGHT CORNER BRACKET
+0x213A	0x300E	# LEFT WHITE CORNER BRACKET
+0x213B	0x300F	# RIGHT WHITE CORNER BRACKET
+0x213C	0x3016	# LEFT WHITE LENTICULAR BRACKET
+0x213D	0x3017	# RIGHT WHITE LENTICULAR BRACKET
+0x213E	0x3010	# LEFT BLACK LENTICULAR BRACKET
+0x213F	0x3011	# RIGHT BLACK LENTICULAR BRACKET
+0x2140	0x00B1	# PLUS-MINUS SIGN
+0x2141	0x00D7	# MULTIPLICATION SIGN
+0x2142	0x00F7	# DIVISION SIGN
+0x2143	0x2236	# RATIO
+0x2144	0x2227	# LOGICAL AND
+0x2145	0x2228	# LOGICAL OR
+0x2146	0x2211	# N-ARY SUMMATION
+0x2147	0x220F	# N-ARY PRODUCT
+0x2148	0x222A	# UNION
+0x2149	0x2229	# INTERSECTION
+0x214A	0x2208	# ELEMENT OF
+0x214B	0x2237	# PROPORTION
+0x214C	0x221A	# SQUARE ROOT
+0x214D	0x22A5	# UP TACK
+0x214E	0x2225	# PARALLEL TO
+0x214F	0x2220	# ANGLE
+0x2150	0x2312	# ARC
+0x2151	0x2299	# CIRCLED DOT OPERATOR
+0x2152	0x222B	# INTEGRAL
+0x2153	0x222E	# CONTOUR INTEGRAL
+0x2154	0x2261	# IDENTICAL TO
+0x2155	0x224C	# ALL EQUAL TO
+0x2156	0x2248	# ALMOST EQUAL TO
+0x2157	0x223D	# REVERSED TILDE
+0x2158	0x221D	# PROPORTIONAL TO
+0x2159	0x2260	# NOT EQUAL TO
+0x215A	0x226E	# NOT LESS-THAN
+0x215B	0x226F	# NOT GREATER-THAN
+0x215C	0x2264	# LESS-THAN OR EQUAL TO
+0x215D	0x2265	# GREATER-THAN OR EQUAL TO
+0x215E	0x221E	# INFINITY
+0x215F	0x2235	# BECAUSE
+0x2160	0x2234	# THEREFORE
+0x2161	0x2642	# MALE SIGN
+0x2162	0x2640	# FEMALE SIGN
+0x2163	0x00B0	# DEGREE SIGN
+0x2164	0x2032	# PRIME
+0x2165	0x2033	# DOUBLE PRIME
+0x2166	0x2103	# DEGREE CELSIUS
+0x2167	0xFF04	# FULLWIDTH DOLLAR SIGN
+0x2168	0x00A4	# CURRENCY SIGN
+0x2169	0xFFE0	# FULLWIDTH CENT SIGN
+0x216A	0xFFE1	# FULLWIDTH POUND SIGN
+0x216B	0x2030	# PER MILLE SIGN
+0x216C	0x00A7	# SECTION SIGN
+0x216D	0x2116	# NUMERO SIGN
+0x216E	0x2606	# WHITE STAR
+0x216F	0x2605	# BLACK STAR
+0x2170	0x25CB	# WHITE CIRCLE
+0x2171	0x25CF	# BLACK CIRCLE
+0x2172	0x25CE	# BULLSEYE
+0x2173	0x25C7	# WHITE DIAMOND
+0x2174	0x25C6	# BLACK DIAMOND
+0x2175	0x25A1	# WHITE SQUARE
+0x2176	0x25A0	# BLACK SQUARE
+0x2177	0x25B3	# WHITE UP-POINTING TRIANGLE
+0x2178	0x25B2	# BLACK UP-POINTING TRIANGLE
+0x2179	0x203B	# REFERENCE MARK
+0x217A	0x2192	# RIGHTWARDS ARROW
+0x217B	0x2190	# LEFTWARDS ARROW
+0x217C	0x2191	# UPWARDS ARROW
+0x217D	0x2193	# DOWNWARDS ARROW
+0x217E	0x3013	# GETA MARK
+0x2231	0x2488	# DIGIT ONE FULL STOP
+0x2232	0x2489	# DIGIT TWO FULL STOP
+0x2233	0x248A	# DIGIT THREE FULL STOP
+0x2234	0x248B	# DIGIT FOUR FULL STOP
+0x2235	0x248C	# DIGIT FIVE FULL STOP
+0x2236	0x248D	# DIGIT SIX FULL STOP
+0x2237	0x248E	# DIGIT SEVEN FULL STOP
+0x2238	0x248F	# DIGIT EIGHT FULL STOP
+0x2239	0x2490	# DIGIT NINE FULL STOP
+0x223A	0x2491	# NUMBER TEN FULL STOP
+0x223B	0x2492	# NUMBER ELEVEN FULL STOP
+0x223C	0x2493	# NUMBER TWELVE FULL STOP
+0x223D	0x2494	# NUMBER THIRTEEN FULL STOP
+0x223E	0x2495	# NUMBER FOURTEEN FULL STOP
+0x223F	0x2496	# NUMBER FIFTEEN FULL STOP
+0x2240	0x2497	# NUMBER SIXTEEN FULL STOP
+0x2241	0x2498	# NUMBER SEVENTEEN FULL STOP
+0x2242	0x2499	# NUMBER EIGHTEEN FULL STOP
+0x2243	0x249A	# NUMBER NINETEEN FULL STOP
+0x2244	0x249B	# NUMBER TWENTY FULL STOP
+0x2245	0x2474	# PARENTHESIZED DIGIT ONE
+0x2246	0x2475	# PARENTHESIZED DIGIT TWO
+0x2247	0x2476	# PARENTHESIZED DIGIT THREE
+0x2248	0x2477	# PARENTHESIZED DIGIT FOUR
+0x2249	0x2478	# PARENTHESIZED DIGIT FIVE
+0x224A	0x2479	# PARENTHESIZED DIGIT SIX
+0x224B	0x247A	# PARENTHESIZED DIGIT SEVEN
+0x224C	0x247B	# PARENTHESIZED DIGIT EIGHT
+0x224D	0x247C	# PARENTHESIZED DIGIT NINE
+0x224E	0x247D	# PARENTHESIZED NUMBER TEN
+0x224F	0x247E	# PARENTHESIZED NUMBER ELEVEN
+0x2250	0x247F	# PARENTHESIZED NUMBER TWELVE
+0x2251	0x2480	# PARENTHESIZED NUMBER THIRTEEN
+0x2252	0x2481	# PARENTHESIZED NUMBER FOURTEEN
+0x2253	0x2482	# PARENTHESIZED NUMBER FIFTEEN
+0x2254	0x2483	# PARENTHESIZED NUMBER SIXTEEN
+0x2255	0x2484	# PARENTHESIZED NUMBER SEVENTEEN
+0x2256	0x2485	# PARENTHESIZED NUMBER EIGHTEEN
+0x2257	0x2486	# PARENTHESIZED NUMBER NINETEEN
+0x2258	0x2487	# PARENTHESIZED NUMBER TWENTY
+0x2259	0x2460	# CIRCLED DIGIT ONE
+0x225A	0x2461	# CIRCLED DIGIT TWO
+0x225B	0x2462	# CIRCLED DIGIT THREE
+0x225C	0x2463	# CIRCLED DIGIT FOUR
+0x225D	0x2464	# CIRCLED DIGIT FIVE
+0x225E	0x2465	# CIRCLED DIGIT SIX
+0x225F	0x2466	# CIRCLED DIGIT SEVEN
+0x2260	0x2467	# CIRCLED DIGIT EIGHT
+0x2261	0x2468	# CIRCLED DIGIT NINE
+0x2262	0x2469	# CIRCLED NUMBER TEN
+0x2265	0x3220	# PARENTHESIZED IDEOGRAPH ONE
+0x2266	0x3221	# PARENTHESIZED IDEOGRAPH TWO
+0x2267	0x3222	# PARENTHESIZED IDEOGRAPH THREE
+0x2268	0x3223	# PARENTHESIZED IDEOGRAPH FOUR
+0x2269	0x3224	# PARENTHESIZED IDEOGRAPH FIVE
+0x226A	0x3225	# PARENTHESIZED IDEOGRAPH SIX
+0x226B	0x3226	# PARENTHESIZED IDEOGRAPH SEVEN
+0x226C	0x3227	# PARENTHESIZED IDEOGRAPH EIGHT
+0x226D	0x3228	# PARENTHESIZED IDEOGRAPH NINE
+0x226E	0x3229	# PARENTHESIZED IDEOGRAPH TEN
+0x2271	0x2160	# ROMAN NUMERAL ONE
+0x2272	0x2161	# ROMAN NUMERAL TWO
+0x2273	0x2162	# ROMAN NUMERAL THREE
+0x2274	0x2163	# ROMAN NUMERAL FOUR
+0x2275	0x2164	# ROMAN NUMERAL FIVE
+0x2276	0x2165	# ROMAN NUMERAL SIX
+0x2277	0x2166	# ROMAN NUMERAL SEVEN
+0x2278	0x2167	# ROMAN NUMERAL EIGHT
+0x2279	0x2168	# ROMAN NUMERAL NINE
+0x227A	0x2169	# ROMAN NUMERAL TEN
+0x227B	0x216A	# ROMAN NUMERAL ELEVEN
+0x227C	0x216B	# ROMAN NUMERAL TWELVE
+0x2321	0xFF01	# FULLWIDTH EXCLAMATION MARK
+0x2322	0xFF02	# FULLWIDTH QUOTATION MARK
+0x2323	0xFF03	# FULLWIDTH NUMBER SIGN
+0x2324	0xFFE5	# FULLWIDTH YEN SIGN
+0x2325	0xFF05	# FULLWIDTH PERCENT SIGN
+0x2326	0xFF06	# FULLWIDTH AMPERSAND
+0x2327	0xFF07	# FULLWIDTH APOSTROPHE
+0x2328	0xFF08	# FULLWIDTH LEFT PARENTHESIS
+0x2329	0xFF09	# FULLWIDTH RIGHT PARENTHESIS
+0x232A	0xFF0A	# FULLWIDTH ASTERISK
+0x232B	0xFF0B	# FULLWIDTH PLUS SIGN
+0x232C	0xFF0C	# FULLWIDTH COMMA
+0x232D	0xFF0D	# FULLWIDTH HYPHEN-MINUS
+0x232E	0xFF0E	# FULLWIDTH FULL STOP
+0x232F	0xFF0F	# FULLWIDTH SOLIDUS
+0x2330	0xFF10	# FULLWIDTH DIGIT ZERO
+0x2331	0xFF11	# FULLWIDTH DIGIT ONE
+0x2332	0xFF12	# FULLWIDTH DIGIT TWO
+0x2333	0xFF13	# FULLWIDTH DIGIT THREE
+0x2334	0xFF14	# FULLWIDTH DIGIT FOUR
+0x2335	0xFF15	# FULLWIDTH DIGIT FIVE
+0x2336	0xFF16	# FULLWIDTH DIGIT SIX
+0x2337	0xFF17	# FULLWIDTH DIGIT SEVEN
+0x2338	0xFF18	# FULLWIDTH DIGIT EIGHT
+0x2339	0xFF19	# FULLWIDTH DIGIT NINE
+0x233A	0xFF1A	# FULLWIDTH COLON
+0x233B	0xFF1B	# FULLWIDTH SEMICOLON
+0x233C	0xFF1C	# FULLWIDTH LESS-THAN SIGN
+0x233D	0xFF1D	# FULLWIDTH EQUALS SIGN
+0x233E	0xFF1E	# FULLWIDTH GREATER-THAN SIGN
+0x233F	0xFF1F	# FULLWIDTH QUESTION MARK
+0x2340	0xFF20	# FULLWIDTH COMMERCIAL AT
+0x2341	0xFF21	# FULLWIDTH LATIN CAPITAL LETTER A
+0x2342	0xFF22	# FULLWIDTH LATIN CAPITAL LETTER B
+0x2343	0xFF23	# FULLWIDTH LATIN CAPITAL LETTER C
+0x2344	0xFF24	# FULLWIDTH LATIN CAPITAL LETTER D
+0x2345	0xFF25	# FULLWIDTH LATIN CAPITAL LETTER E
+0x2346	0xFF26	# FULLWIDTH LATIN CAPITAL LETTER F
+0x2347	0xFF27	# FULLWIDTH LATIN CAPITAL LETTER G
+0x2348	0xFF28	# FULLWIDTH LATIN CAPITAL LETTER H
+0x2349	0xFF29	# FULLWIDTH LATIN CAPITAL LETTER I
+0x234A	0xFF2A	# FULLWIDTH LATIN CAPITAL LETTER J
+0x234B	0xFF2B	# FULLWIDTH LATIN CAPITAL LETTER K
+0x234C	0xFF2C	# FULLWIDTH LATIN CAPITAL LETTER L
+0x234D	0xFF2D	# FULLWIDTH LATIN CAPITAL LETTER M
+0x234E	0xFF2E	# FULLWIDTH LATIN CAPITAL LETTER N
+0x234F	0xFF2F	# FULLWIDTH LATIN CAPITAL LETTER O
+0x2350	0xFF30	# FULLWIDTH LATIN CAPITAL LETTER P
+0x2351	0xFF31	# FULLWIDTH LATIN CAPITAL LETTER Q
+0x2352	0xFF32	# FULLWIDTH LATIN CAPITAL LETTER R
+0x2353	0xFF33	# FULLWIDTH LATIN CAPITAL LETTER S
+0x2354	0xFF34	# FULLWIDTH LATIN CAPITAL LETTER T
+0x2355	0xFF35	# FULLWIDTH LATIN CAPITAL LETTER U
+0x2356	0xFF36	# FULLWIDTH LATIN CAPITAL LETTER V
+0x2357	0xFF37	# FULLWIDTH LATIN CAPITAL LETTER W
+0x2358	0xFF38	# FULLWIDTH LATIN CAPITAL LETTER X
+0x2359	0xFF39	# FULLWIDTH LATIN CAPITAL LETTER Y
+0x235A	0xFF3A	# FULLWIDTH LATIN CAPITAL LETTER Z
+0x235B	0xFF3B	# FULLWIDTH LEFT SQUARE BRACKET
+0x235C	0xFF3C	# FULLWIDTH REVERSE SOLIDUS
+0x235D	0xFF3D	# FULLWIDTH RIGHT SQUARE BRACKET
+0x235E	0xFF3E	# FULLWIDTH CIRCUMFLEX ACCENT
+0x235F	0xFF3F	# FULLWIDTH LOW LINE
+0x2360	0xFF40	# FULLWIDTH GRAVE ACCENT
+0x2361	0xFF41	# FULLWIDTH LATIN SMALL LETTER A
+0x2362	0xFF42	# FULLWIDTH LATIN SMALL LETTER B
+0x2363	0xFF43	# FULLWIDTH LATIN SMALL LETTER C
+0x2364	0xFF44	# FULLWIDTH LATIN SMALL LETTER D
+0x2365	0xFF45	# FULLWIDTH LATIN SMALL LETTER E
+0x2366	0xFF46	# FULLWIDTH LATIN SMALL LETTER F
+0x2367	0xFF47	# FULLWIDTH LATIN SMALL LETTER G
+0x2368	0xFF48	# FULLWIDTH LATIN SMALL LETTER H
+0x2369	0xFF49	# FULLWIDTH LATIN SMALL LETTER I
+0x236A	0xFF4A	# FULLWIDTH LATIN SMALL LETTER J
+0x236B	0xFF4B	# FULLWIDTH LATIN SMALL LETTER K
+0x236C	0xFF4C	# FULLWIDTH LATIN SMALL LETTER L
+0x236D	0xFF4D	# FULLWIDTH LATIN SMALL LETTER M
+0x236E	0xFF4E	# FULLWIDTH LATIN SMALL LETTER N
+0x236F	0xFF4F	# FULLWIDTH LATIN SMALL LETTER O
+0x2370	0xFF50	# FULLWIDTH LATIN SMALL LETTER P
+0x2371	0xFF51	# FULLWIDTH LATIN SMALL LETTER Q
+0x2372	0xFF52	# FULLWIDTH LATIN SMALL LETTER R
+0x2373	0xFF53	# FULLWIDTH LATIN SMALL LETTER S
+0x2374	0xFF54	# FULLWIDTH LATIN SMALL LETTER T
+0x2375	0xFF55	# FULLWIDTH LATIN SMALL LETTER U
+0x2376	0xFF56	# FULLWIDTH LATIN SMALL LETTER V
+0x2377	0xFF57	# FULLWIDTH LATIN SMALL LETTER W
+0x2378	0xFF58	# FULLWIDTH LATIN SMALL LETTER X
+0x2379	0xFF59	# FULLWIDTH LATIN SMALL LETTER Y
+0x237A	0xFF5A	# FULLWIDTH LATIN SMALL LETTER Z
+0x237B	0xFF5B	# FULLWIDTH LEFT CURLY BRACKET
+0x237C	0xFF5C	# FULLWIDTH VERTICAL LINE
+0x237D	0xFF5D	# FULLWIDTH RIGHT CURLY BRACKET
+0x237E	0xFFE3	# FULLWIDTH MACRON
+0x2421	0x3041	# HIRAGANA LETTER SMALL A
+0x2422	0x3042	# HIRAGANA LETTER A
+0x2423	0x3043	# HIRAGANA LETTER SMALL I
+0x2424	0x3044	# HIRAGANA LETTER I
+0x2425	0x3045	# HIRAGANA LETTER SMALL U
+0x2426	0x3046	# HIRAGANA LETTER U
+0x2427	0x3047	# HIRAGANA LETTER SMALL E
+0x2428	0x3048	# HIRAGANA LETTER E
+0x2429	0x3049	# HIRAGANA LETTER SMALL O
+0x242A	0x304A	# HIRAGANA LETTER O
+0x242B	0x304B	# HIRAGANA LETTER KA
+0x242C	0x304C	# HIRAGANA LETTER GA
+0x242D	0x304D	# HIRAGANA LETTER KI
+0x242E	0x304E	# HIRAGANA LETTER GI
+0x242F	0x304F	# HIRAGANA LETTER KU
+0x2430	0x3050	# HIRAGANA LETTER GU
+0x2431	0x3051	# HIRAGANA LETTER KE
+0x2432	0x3052	# HIRAGANA LETTER GE
+0x2433	0x3053	# HIRAGANA LETTER KO
+0x2434	0x3054	# HIRAGANA LETTER GO
+0x2435	0x3055	# HIRAGANA LETTER SA
+0x2436	0x3056	# HIRAGANA LETTER ZA
+0x2437	0x3057	# HIRAGANA LETTER SI
+0x2438	0x3058	# HIRAGANA LETTER ZI
+0x2439	0x3059	# HIRAGANA LETTER SU
+0x243A	0x305A	# HIRAGANA LETTER ZU
+0x243B	0x305B	# HIRAGANA LETTER SE
+0x243C	0x305C	# HIRAGANA LETTER ZE
+0x243D	0x305D	# HIRAGANA LETTER SO
+0x243E	0x305E	# HIRAGANA LETTER ZO
+0x243F	0x305F	# HIRAGANA LETTER TA
+0x2440	0x3060	# HIRAGANA LETTER DA
+0x2441	0x3061	# HIRAGANA LETTER TI
+0x2442	0x3062	# HIRAGANA LETTER DI
+0x2443	0x3063	# HIRAGANA LETTER SMALL TU
+0x2444	0x3064	# HIRAGANA LETTER TU
+0x2445	0x3065	# HIRAGANA LETTER DU
+0x2446	0x3066	# HIRAGANA LETTER TE
+0x2447	0x3067	# HIRAGANA LETTER DE
+0x2448	0x3068	# HIRAGANA LETTER TO
+0x2449	0x3069	# HIRAGANA LETTER DO
+0x244A	0x306A	# HIRAGANA LETTER NA
+0x244B	0x306B	# HIRAGANA LETTER NI
+0x244C	0x306C	# HIRAGANA LETTER NU
+0x244D	0x306D	# HIRAGANA LETTER NE
+0x244E	0x306E	# HIRAGANA LETTER NO
+0x244F	0x306F	# HIRAGANA LETTER HA
+0x2450	0x3070	# HIRAGANA LETTER BA
+0x2451	0x3071	# HIRAGANA LETTER PA
+0x2452	0x3072	# HIRAGANA LETTER HI
+0x2453	0x3073	# HIRAGANA LETTER BI
+0x2454	0x3074	# HIRAGANA LETTER PI
+0x2455	0x3075	# HIRAGANA LETTER HU
+0x2456	0x3076	# HIRAGANA LETTER BU
+0x2457	0x3077	# HIRAGANA LETTER PU
+0x2458	0x3078	# HIRAGANA LETTER HE
+0x2459	0x3079	# HIRAGANA LETTER BE
+0x245A	0x307A	# HIRAGANA LETTER PE
+0x245B	0x307B	# HIRAGANA LETTER HO
+0x245C	0x307C	# HIRAGANA LETTER BO
+0x245D	0x307D	# HIRAGANA LETTER PO
+0x245E	0x307E	# HIRAGANA LETTER MA
+0x245F	0x307F	# HIRAGANA LETTER MI
+0x2460	0x3080	# HIRAGANA LETTER MU
+0x2461	0x3081	# HIRAGANA LETTER ME
+0x2462	0x3082	# HIRAGANA LETTER MO
+0x2463	0x3083	# HIRAGANA LETTER SMALL YA
+0x2464	0x3084	# HIRAGANA LETTER YA
+0x2465	0x3085	# HIRAGANA LETTER SMALL YU
+0x2466	0x3086	# HIRAGANA LETTER YU
+0x2467	0x3087	# HIRAGANA LETTER SMALL YO
+0x2468	0x3088	# HIRAGANA LETTER YO
+0x2469	0x3089	# HIRAGANA LETTER RA
+0x246A	0x308A	# HIRAGANA LETTER RI
+0x246B	0x308B	# HIRAGANA LETTER RU
+0x246C	0x308C	# HIRAGANA LETTER RE
+0x246D	0x308D	# HIRAGANA LETTER RO
+0x246E	0x308E	# HIRAGANA LETTER SMALL WA
+0x246F	0x308F	# HIRAGANA LETTER WA
+0x2470	0x3090	# HIRAGANA LETTER WI
+0x2471	0x3091	# HIRAGANA LETTER WE
+0x2472	0x3092	# HIRAGANA LETTER WO
+0x2473	0x3093	# HIRAGANA LETTER N
+0x2521	0x30A1	# KATAKANA LETTER SMALL A
+0x2522	0x30A2	# KATAKANA LETTER A
+0x2523	0x30A3	# KATAKANA LETTER SMALL I
+0x2524	0x30A4	# KATAKANA LETTER I
+0x2525	0x30A5	# KATAKANA LETTER SMALL U
+0x2526	0x30A6	# KATAKANA LETTER U
+0x2527	0x30A7	# KATAKANA LETTER SMALL E
+0x2528	0x30A8	# KATAKANA LETTER E
+0x2529	0x30A9	# KATAKANA LETTER SMALL O
+0x252A	0x30AA	# KATAKANA LETTER O
+0x252B	0x30AB	# KATAKANA LETTER KA
+0x252C	0x30AC	# KATAKANA LETTER GA
+0x252D	0x30AD	# KATAKANA LETTER KI
+0x252E	0x30AE	# KATAKANA LETTER GI
+0x252F	0x30AF	# KATAKANA LETTER KU
+0x2530	0x30B0	# KATAKANA LETTER GU
+0x2531	0x30B1	# KATAKANA LETTER KE
+0x2532	0x30B2	# KATAKANA LETTER GE
+0x2533	0x30B3	# KATAKANA LETTER KO
+0x2534	0x30B4	# KATAKANA LETTER GO
+0x2535	0x30B5	# KATAKANA LETTER SA
+0x2536	0x30B6	# KATAKANA LETTER ZA
+0x2537	0x30B7	# KATAKANA LETTER SI
+0x2538	0x30B8	# KATAKANA LETTER ZI
+0x2539	0x30B9	# KATAKANA LETTER SU
+0x253A	0x30BA	# KATAKANA LETTER ZU
+0x253B	0x30BB	# KATAKANA LETTER SE
+0x253C	0x30BC	# KATAKANA LETTER ZE
+0x253D	0x30BD	# KATAKANA LETTER SO
+0x253E	0x30BE	# KATAKANA LETTER ZO
+0x253F	0x30BF	# KATAKANA LETTER TA
+0x2540	0x30C0	# KATAKANA LETTER DA
+0x2541	0x30C1	# KATAKANA LETTER TI
+0x2542	0x30C2	# KATAKANA LETTER DI
+0x2543	0x30C3	# KATAKANA LETTER SMALL TU
+0x2544	0x30C4	# KATAKANA LETTER TU
+0x2545	0x30C5	# KATAKANA LETTER DU
+0x2546	0x30C6	# KATAKANA LETTER TE
+0x2547	0x30C7	# KATAKANA LETTER DE
+0x2548	0x30C8	# KATAKANA LETTER TO
+0x2549	0x30C9	# KATAKANA LETTER DO
+0x254A	0x30CA	# KATAKANA LETTER NA
+0x254B	0x30CB	# KATAKANA LETTER NI
+0x254C	0x30CC	# KATAKANA LETTER NU
+0x254D	0x30CD	# KATAKANA LETTER NE
+0x254E	0x30CE	# KATAKANA LETTER NO
+0x254F	0x30CF	# KATAKANA LETTER HA
+0x2550	0x30D0	# KATAKANA LETTER BA
+0x2551	0x30D1	# KATAKANA LETTER PA
+0x2552	0x30D2	# KATAKANA LETTER HI
+0x2553	0x30D3	# KATAKANA LETTER BI
+0x2554	0x30D4	# KATAKANA LETTER PI
+0x2555	0x30D5	# KATAKANA LETTER HU
+0x2556	0x30D6	# KATAKANA LETTER BU
+0x2557	0x30D7	# KATAKANA LETTER PU
+0x2558	0x30D8	# KATAKANA LETTER HE
+0x2559	0x30D9	# KATAKANA LETTER BE
+0x255A	0x30DA	# KATAKANA LETTER PE
+0x255B	0x30DB	# KATAKANA LETTER HO
+0x255C	0x30DC	# KATAKANA LETTER BO
+0x255D	0x30DD	# KATAKANA LETTER PO
+0x255E	0x30DE	# KATAKANA LETTER MA
+0x255F	0x30DF	# KATAKANA LETTER MI
+0x2560	0x30E0	# KATAKANA LETTER MU
+0x2561	0x30E1	# KATAKANA LETTER ME
+0x2562	0x30E2	# KATAKANA LETTER MO
+0x2563	0x30E3	# KATAKANA LETTER SMALL YA
+0x2564	0x30E4	# KATAKANA LETTER YA
+0x2565	0x30E5	# KATAKANA LETTER SMALL YU
+0x2566	0x30E6	# KATAKANA LETTER YU
+0x2567	0x30E7	# KATAKANA LETTER SMALL YO
+0x2568	0x30E8	# KATAKANA LETTER YO
+0x2569	0x30E9	# KATAKANA LETTER RA
+0x256A	0x30EA	# KATAKANA LETTER RI
+0x256B	0x30EB	# KATAKANA LETTER RU
+0x256C	0x30EC	# KATAKANA LETTER RE
+0x256D	0x30ED	# KATAKANA LETTER RO
+0x256E	0x30EE	# KATAKANA LETTER SMALL WA
+0x256F	0x30EF	# KATAKANA LETTER WA
+0x2570	0x30F0	# KATAKANA LETTER WI
+0x2571	0x30F1	# KATAKANA LETTER WE
+0x2572	0x30F2	# KATAKANA LETTER WO
+0x2573	0x30F3	# KATAKANA LETTER N
+0x2574	0x30F4	# KATAKANA LETTER VU
+0x2575	0x30F5	# KATAKANA LETTER SMALL KA
+0x2576	0x30F6	# KATAKANA LETTER SMALL KE
+0x2621	0x0391	# GREEK CAPITAL LETTER ALPHA
+0x2622	0x0392	# GREEK CAPITAL LETTER BETA
+0x2623	0x0393	# GREEK CAPITAL LETTER GAMMA
+0x2624	0x0394	# GREEK CAPITAL LETTER DELTA
+0x2625	0x0395	# GREEK CAPITAL LETTER EPSILON
+0x2626	0x0396	# GREEK CAPITAL LETTER ZETA
+0x2627	0x0397	# GREEK CAPITAL LETTER ETA
+0x2628	0x0398	# GREEK CAPITAL LETTER THETA
+0x2629	0x0399	# GREEK CAPITAL LETTER IOTA
+0x262A	0x039A	# GREEK CAPITAL LETTER KAPPA
+0x262B	0x039B	# GREEK CAPITAL LETTER LAMDA
+0x262C	0x039C	# GREEK CAPITAL LETTER MU
+0x262D	0x039D	# GREEK CAPITAL LETTER NU
+0x262E	0x039E	# GREEK CAPITAL LETTER XI
+0x262F	0x039F	# GREEK CAPITAL LETTER OMICRON
+0x2630	0x03A0	# GREEK CAPITAL LETTER PI
+0x2631	0x03A1	# GREEK CAPITAL LETTER RHO
+0x2632	0x03A3	# GREEK CAPITAL LETTER SIGMA
+0x2633	0x03A4	# GREEK CAPITAL LETTER TAU
+0x2634	0x03A5	# GREEK CAPITAL LETTER UPSILON
+0x2635	0x03A6	# GREEK CAPITAL LETTER PHI
+0x2636	0x03A7	# GREEK CAPITAL LETTER CHI
+0x2637	0x03A8	# GREEK CAPITAL LETTER PSI
+0x2638	0x03A9	# GREEK CAPITAL LETTER OMEGA
+0x2641	0x03B1	# GREEK SMALL LETTER ALPHA
+0x2642	0x03B2	# GREEK SMALL LETTER BETA
+0x2643	0x03B3	# GREEK SMALL LETTER GAMMA
+0x2644	0x03B4	# GREEK SMALL LETTER DELTA
+0x2645	0x03B5	# GREEK SMALL LETTER EPSILON
+0x2646	0x03B6	# GREEK SMALL LETTER ZETA
+0x2647	0x03B7	# GREEK SMALL LETTER ETA
+0x2648	0x03B8	# GREEK SMALL LETTER THETA
+0x2649	0x03B9	# GREEK SMALL LETTER IOTA
+0x264A	0x03BA	# GREEK SMALL LETTER KAPPA
+0x264B	0x03BB	# GREEK SMALL LETTER LAMDA
+0x264C	0x03BC	# GREEK SMALL LETTER MU
+0x264D	0x03BD	# GREEK SMALL LETTER NU
+0x264E	0x03BE	# GREEK SMALL LETTER XI
+0x264F	0x03BF	# GREEK SMALL LETTER OMICRON
+0x2650	0x03C0	# GREEK SMALL LETTER PI
+0x2651	0x03C1	# GREEK SMALL LETTER RHO
+0x2652	0x03C3	# GREEK SMALL LETTER SIGMA
+0x2653	0x03C4	# GREEK SMALL LETTER TAU
+0x2654	0x03C5	# GREEK SMALL LETTER UPSILON
+0x2655	0x03C6	# GREEK SMALL LETTER PHI
+0x2656	0x03C7	# GREEK SMALL LETTER CHI
+0x2657	0x03C8	# GREEK SMALL LETTER PSI
+0x2658	0x03C9	# GREEK SMALL LETTER OMEGA
+0x2721	0x0410	# CYRILLIC CAPITAL LETTER A
+0x2722	0x0411	# CYRILLIC CAPITAL LETTER BE
+0x2723	0x0412	# CYRILLIC CAPITAL LETTER VE
+0x2724	0x0413	# CYRILLIC CAPITAL LETTER GHE
+0x2725	0x0414	# CYRILLIC CAPITAL LETTER DE
+0x2726	0x0415	# CYRILLIC CAPITAL LETTER IE
+0x2727	0x0401	# CYRILLIC CAPITAL LETTER IO
+0x2728	0x0416	# CYRILLIC CAPITAL LETTER ZHE
+0x2729	0x0417	# CYRILLIC CAPITAL LETTER ZE
+0x272A	0x0418	# CYRILLIC CAPITAL LETTER I
+0x272B	0x0419	# CYRILLIC CAPITAL LETTER SHORT I
+0x272C	0x041A	# CYRILLIC CAPITAL LETTER KA
+0x272D	0x041B	# CYRILLIC CAPITAL LETTER EL
+0x272E	0x041C	# CYRILLIC CAPITAL LETTER EM
+0x272F	0x041D	# CYRILLIC CAPITAL LETTER EN
+0x2730	0x041E	# CYRILLIC CAPITAL LETTER O
+0x2731	0x041F	# CYRILLIC CAPITAL LETTER PE
+0x2732	0x0420	# CYRILLIC CAPITAL LETTER ER
+0x2733	0x0421	# CYRILLIC CAPITAL LETTER ES
+0x2734	0x0422	# CYRILLIC CAPITAL LETTER TE
+0x2735	0x0423	# CYRILLIC CAPITAL LETTER U
+0x2736	0x0424	# CYRILLIC CAPITAL LETTER EF
+0x2737	0x0425	# CYRILLIC CAPITAL LETTER HA
+0x2738	0x0426	# CYRILLIC CAPITAL LETTER TSE
+0x2739	0x0427	# CYRILLIC CAPITAL LETTER CHE
+0x273A	0x0428	# CYRILLIC CAPITAL LETTER SHA
+0x273B	0x0429	# CYRILLIC CAPITAL LETTER SHCHA
+0x273C	0x042A	# CYRILLIC CAPITAL LETTER HARD SIGN
+0x273D	0x042B	# CYRILLIC CAPITAL LETTER YERU
+0x273E	0x042C	# CYRILLIC CAPITAL LETTER SOFT SIGN
+0x273F	0x042D	# CYRILLIC CAPITAL LETTER E
+0x2740	0x042E	# CYRILLIC CAPITAL LETTER YU
+0x2741	0x042F	# CYRILLIC CAPITAL LETTER YA
+0x2751	0x0430	# CYRILLIC SMALL LETTER A
+0x2752	0x0431	# CYRILLIC SMALL LETTER BE
+0x2753	0x0432	# CYRILLIC SMALL LETTER VE
+0x2754	0x0433	# CYRILLIC SMALL LETTER GHE
+0x2755	0x0434	# CYRILLIC SMALL LETTER DE
+0x2756	0x0435	# CYRILLIC SMALL LETTER IE
+0x2757	0x0451	# CYRILLIC SMALL LETTER IO
+0x2758	0x0436	# CYRILLIC SMALL LETTER ZHE
+0x2759	0x0437	# CYRILLIC SMALL LETTER ZE
+0x275A	0x0438	# CYRILLIC SMALL LETTER I
+0x275B	0x0439	# CYRILLIC SMALL LETTER SHORT I
+0x275C	0x043A	# CYRILLIC SMALL LETTER KA
+0x275D	0x043B	# CYRILLIC SMALL LETTER EL
+0x275E	0x043C	# CYRILLIC SMALL LETTER EM
+0x275F	0x043D	# CYRILLIC SMALL LETTER EN
+0x2760	0x043E	# CYRILLIC SMALL LETTER O
+0x2761	0x043F	# CYRILLIC SMALL LETTER PE
+0x2762	0x0440	# CYRILLIC SMALL LETTER ER
+0x2763	0x0441	# CYRILLIC SMALL LETTER ES
+0x2764	0x0442	# CYRILLIC SMALL LETTER TE
+0x2765	0x0443	# CYRILLIC SMALL LETTER U
+0x2766	0x0444	# CYRILLIC SMALL LETTER EF
+0x2767	0x0445	# CYRILLIC SMALL LETTER HA
+0x2768	0x0446	# CYRILLIC SMALL LETTER TSE
+0x2769	0x0447	# CYRILLIC SMALL LETTER CHE
+0x276A	0x0448	# CYRILLIC SMALL LETTER SHA
+0x276B	0x0449	# CYRILLIC SMALL LETTER SHCHA
+0x276C	0x044A	# CYRILLIC SMALL LETTER HARD SIGN
+0x276D	0x044B	# CYRILLIC SMALL LETTER YERU
+0x276E	0x044C	# CYRILLIC SMALL LETTER SOFT SIGN
+0x276F	0x044D	# CYRILLIC SMALL LETTER E
+0x2770	0x044E	# CYRILLIC SMALL LETTER YU
+0x2771	0x044F	# CYRILLIC SMALL LETTER YA
+0x2821	0x0101	# LATIN SMALL LETTER A WITH MACRON
+0x2822	0x00E1	# LATIN SMALL LETTER A WITH ACUTE
+0x2823	0x01CE	# LATIN SMALL LETTER A WITH CARON
+0x2824	0x00E0	# LATIN SMALL LETTER A WITH GRAVE
+0x2825	0x0113	# LATIN SMALL LETTER E WITH MACRON
+0x2826	0x00E9	# LATIN SMALL LETTER E WITH ACUTE
+0x2827	0x011B	# LATIN SMALL LETTER E WITH CARON
+0x2828	0x00E8	# LATIN SMALL LETTER E WITH GRAVE
+0x2829	0x012B	# LATIN SMALL LETTER I WITH MACRON
+0x282A	0x00ED	# LATIN SMALL LETTER I WITH ACUTE
+0x282B	0x01D0	# LATIN SMALL LETTER I WITH CARON
+0x282C	0x00EC	# LATIN SMALL LETTER I WITH GRAVE
+0x282D	0x014D	# LATIN SMALL LETTER O WITH MACRON
+0x282E	0x00F3	# LATIN SMALL LETTER O WITH ACUTE
+0x282F	0x01D2	# LATIN SMALL LETTER O WITH CARON
+0x2830	0x00F2	# LATIN SMALL LETTER O WITH GRAVE
+0x2831	0x016B	# LATIN SMALL LETTER U WITH MACRON
+0x2832	0x00FA	# LATIN SMALL LETTER U WITH ACUTE
+0x2833	0x01D4	# LATIN SMALL LETTER U WITH CARON
+0x2834	0x00F9	# LATIN SMALL LETTER U WITH GRAVE
+0x2835	0x01D6	# LATIN SMALL LETTER U WITH DIAERESIS AND MACRON
+0x2836	0x01D8	# LATIN SMALL LETTER U WITH DIAERESIS AND ACUTE
+0x2837	0x01DA	# LATIN SMALL LETTER U WITH DIAERESIS AND CARON
+0x2838	0x01DC	# LATIN SMALL LETTER U WITH DIAERESIS AND GRAVE
+0x2839	0x00FC	# LATIN SMALL LETTER U WITH DIAERESIS
+0x283A	0x00EA	# LATIN SMALL LETTER E WITH CIRCUMFLEX
+0x2845	0x3105	# BOPOMOFO LETTER B
+0x2846	0x3106	# BOPOMOFO LETTER P
+0x2847	0x3107	# BOPOMOFO LETTER M
+0x2848	0x3108	# BOPOMOFO LETTER F
+0x2849	0x3109	# BOPOMOFO LETTER D
+0x284A	0x310A	# BOPOMOFO LETTER T
+0x284B	0x310B	# BOPOMOFO LETTER N
+0x284C	0x310C	# BOPOMOFO LETTER L
+0x284D	0x310D	# BOPOMOFO LETTER G
+0x284E	0x310E	# BOPOMOFO LETTER K
+0x284F	0x310F	# BOPOMOFO LETTER H
+0x2850	0x3110	# BOPOMOFO LETTER J
+0x2851	0x3111	# BOPOMOFO LETTER Q
+0x2852	0x3112	# BOPOMOFO LETTER X
+0x2853	0x3113	# BOPOMOFO LETTER ZH
+0x2854	0x3114	# BOPOMOFO LETTER CH
+0x2855	0x3115	# BOPOMOFO LETTER SH
+0x2856	0x3116	# BOPOMOFO LETTER R
+0x2857	0x3117	# BOPOMOFO LETTER Z
+0x2858	0x3118	# BOPOMOFO LETTER C
+0x2859	0x3119	# BOPOMOFO LETTER S
+0x285A	0x311A	# BOPOMOFO LETTER A
+0x285B	0x311B	# BOPOMOFO LETTER O
+0x285C	0x311C	# BOPOMOFO LETTER E
+0x285D	0x311D	# BOPOMOFO LETTER EH
+0x285E	0x311E	# BOPOMOFO LETTER AI
+0x285F	0x311F	# BOPOMOFO LETTER EI
+0x2860	0x3120	# BOPOMOFO LETTER AU
+0x2861	0x3121	# BOPOMOFO LETTER OU
+0x2862	0x3122	# BOPOMOFO LETTER AN
+0x2863	0x3123	# BOPOMOFO LETTER EN
+0x2864	0x3124	# BOPOMOFO LETTER ANG
+0x2865	0x3125	# BOPOMOFO LETTER ENG
+0x2866	0x3126	# BOPOMOFO LETTER ER
+0x2867	0x3127	# BOPOMOFO LETTER I
+0x2868	0x3128	# BOPOMOFO LETTER U
+0x2869	0x3129	# BOPOMOFO LETTER IU
+0x2924	0x2500	# BOX DRAWINGS LIGHT HORIZONTAL
+0x2925	0x2501	# BOX DRAWINGS HEAVY HORIZONTAL
+0x2926	0x2502	# BOX DRAWINGS LIGHT VERTICAL
+0x2927	0x2503	# BOX DRAWINGS HEAVY VERTICAL
+0x2928	0x2504	# BOX DRAWINGS LIGHT TRIPLE DASH HORIZONTAL
+0x2929	0x2505	# BOX DRAWINGS HEAVY TRIPLE DASH HORIZONTAL
+0x292A	0x2506	# BOX DRAWINGS LIGHT TRIPLE DASH VERTICAL
+0x292B	0x2507	# BOX DRAWINGS HEAVY TRIPLE DASH VERTICAL
+0x292C	0x2508	# BOX DRAWINGS LIGHT QUADRUPLE DASH HORIZONTAL
+0x292D	0x2509	# BOX DRAWINGS HEAVY QUADRUPLE DASH HORIZONTAL
+0x292E	0x250A	# BOX DRAWINGS LIGHT QUADRUPLE DASH VERTICAL
+0x292F	0x250B	# BOX DRAWINGS HEAVY QUADRUPLE DASH VERTICAL
+0x2930	0x250C	# BOX DRAWINGS LIGHT DOWN AND RIGHT
+0x2931	0x250D	# BOX DRAWINGS DOWN LIGHT AND RIGHT HEAVY
+0x2932	0x250E	# BOX DRAWINGS DOWN HEAVY AND RIGHT LIGHT
+0x2933	0x250F	# BOX DRAWINGS HEAVY DOWN AND RIGHT
+0x2934	0x2510	# BOX DRAWINGS LIGHT DOWN AND LEFT
+0x2935	0x2511	# BOX DRAWINGS DOWN LIGHT AND LEFT HEAVY
+0x2936	0x2512	# BOX DRAWINGS DOWN HEAVY AND LEFT LIGHT
+0x2937	0x2513	# BOX DRAWINGS HEAVY DOWN AND LEFT
+0x2938	0x2514	# BOX DRAWINGS LIGHT UP AND RIGHT
+0x2939	0x2515	# BOX DRAWINGS UP LIGHT AND RIGHT HEAVY
+0x293A	0x2516	# BOX DRAWINGS UP HEAVY AND RIGHT LIGHT
+0x293B	0x2517	# BOX DRAWINGS HEAVY UP AND RIGHT
+0x293C	0x2518	# BOX DRAWINGS LIGHT UP AND LEFT
+0x293D	0x2519	# BOX DRAWINGS UP LIGHT AND LEFT HEAVY
+0x293E	0x251A	# BOX DRAWINGS UP HEAVY AND LEFT LIGHT
+0x293F	0x251B	# BOX DRAWINGS HEAVY UP AND LEFT
+0x2940	0x251C	# BOX DRAWINGS LIGHT VERTICAL AND RIGHT
+0x2941	0x251D	# BOX DRAWINGS VERTICAL LIGHT AND RIGHT HEAVY
+0x2942	0x251E	# BOX DRAWINGS UP HEAVY AND RIGHT DOWN LIGHT
+0x2943	0x251F	# BOX DRAWINGS DOWN HEAVY AND RIGHT UP LIGHT
+0x2944	0x2520	# BOX DRAWINGS VERTICAL HEAVY AND RIGHT LIGHT
+0x2945	0x2521	# BOX DRAWINGS DOWN LIGHT AND RIGHT UP HEAVY
+0x2946	0x2522	# BOX DRAWINGS UP LIGHT AND RIGHT DOWN HEAVY
+0x2947	0x2523	# BOX DRAWINGS HEAVY VERTICAL AND RIGHT
+0x2948	0x2524	# BOX DRAWINGS LIGHT VERTICAL AND LEFT
+0x2949	0x2525	# BOX DRAWINGS VERTICAL LIGHT AND LEFT HEAVY
+0x294A	0x2526	# BOX DRAWINGS UP HEAVY AND LEFT DOWN LIGHT
+0x294B	0x2527	# BOX DRAWINGS DOWN HEAVY AND LEFT UP LIGHT
+0x294C	0x2528	# BOX DRAWINGS VERTICAL HEAVY AND LEFT LIGHT
+0x294D	0x2529	# BOX DRAWINGS DOWN LIGHT AND LEFT UP HEAVY
+0x294E	0x252A	# BOX DRAWINGS UP LIGHT AND LEFT DOWN HEAVY
+0x294F	0x252B	# BOX DRAWINGS HEAVY VERTICAL AND LEFT
+0x2950	0x252C	# BOX DRAWINGS LIGHT DOWN AND HORIZONTAL
+0x2951	0x252D	# BOX DRAWINGS LEFT HEAVY AND RIGHT DOWN LIGHT
+0x2952	0x252E	# BOX DRAWINGS RIGHT HEAVY AND LEFT DOWN LIGHT
+0x2953	0x252F	# BOX DRAWINGS DOWN LIGHT AND HORIZONTAL HEAVY
+0x2954	0x2530	# BOX DRAWINGS DOWN HEAVY AND HORIZONTAL LIGHT
+0x2955	0x2531	# BOX DRAWINGS RIGHT LIGHT AND LEFT DOWN HEAVY
+0x2956	0x2532	# BOX DRAWINGS LEFT LIGHT AND RIGHT DOWN HEAVY
+0x2957	0x2533	# BOX DRAWINGS HEAVY DOWN AND HORIZONTAL
+0x2958	0x2534	# BOX DRAWINGS LIGHT UP AND HORIZONTAL
+0x2959	0x2535	# BOX DRAWINGS LEFT HEAVY AND RIGHT UP LIGHT
+0x295A	0x2536	# BOX DRAWINGS RIGHT HEAVY AND LEFT UP LIGHT
+0x295B	0x2537	# BOX DRAWINGS UP LIGHT AND HORIZONTAL HEAVY
+0x295C	0x2538	# BOX DRAWINGS UP HEAVY AND HORIZONTAL LIGHT
+0x295D	0x2539	# BOX DRAWINGS RIGHT LIGHT AND LEFT UP HEAVY
+0x295E	0x253A	# BOX DRAWINGS LEFT LIGHT AND RIGHT UP HEAVY
+0x295F	0x253B	# BOX DRAWINGS HEAVY UP AND HORIZONTAL
+0x2960	0x253C	# BOX DRAWINGS LIGHT VERTICAL AND HORIZONTAL
+0x2961	0x253D	# BOX DRAWINGS LEFT HEAVY AND RIGHT VERTICAL LIGHT
+0x2962	0x253E	# BOX DRAWINGS RIGHT HEAVY AND LEFT VERTICAL LIGHT
+0x2963	0x253F	# BOX DRAWINGS VERTICAL LIGHT AND HORIZONTAL HEAVY
+0x2964	0x2540	# BOX DRAWINGS UP HEAVY AND DOWN HORIZONTAL LIGHT
+0x2965	0x2541	# BOX DRAWINGS DOWN HEAVY AND UP HORIZONTAL LIGHT
+0x2966	0x2542	# BOX DRAWINGS VERTICAL HEAVY AND HORIZONTAL LIGHT
+0x2967	0x2543	# BOX DRAWINGS LEFT UP HEAVY AND RIGHT DOWN LIGHT
+0x2968	0x2544	# BOX DRAWINGS RIGHT UP HEAVY AND LEFT DOWN LIGHT
+0x2969	0x2545	# BOX DRAWINGS LEFT DOWN HEAVY AND RIGHT UP LIGHT
+0x296A	0x2546	# BOX DRAWINGS RIGHT DOWN HEAVY AND LEFT UP LIGHT
+0x296B	0x2547	# BOX DRAWINGS DOWN LIGHT AND UP HORIZONTAL HEAVY
+0x296C	0x2548	# BOX DRAWINGS UP LIGHT AND DOWN HORIZONTAL HEAVY
+0x296D	0x2549	# BOX DRAWINGS RIGHT LIGHT AND LEFT VERTICAL HEAVY
+0x296E	0x254A	# BOX DRAWINGS LEFT LIGHT AND RIGHT VERTICAL HEAVY
+0x296F	0x254B	# BOX DRAWINGS HEAVY VERTICAL AND HORIZONTAL
+0x3021	0x554A	# <CJK>
+0x3022	0x963F	# <CJK>
+0x3023	0x57C3	# <CJK>
+0x3024	0x6328	# <CJK>
+0x3025	0x54CE	# <CJK>
+0x3026	0x5509	# <CJK>
+0x3027	0x54C0	# <CJK>
+0x3028	0x769A	# <CJK>
+0x3029	0x764C	# <CJK>
+0x302A	0x85F9	# <CJK>
+0x302B	0x77EE	# <CJK>
+0x302C	0x827E	# <CJK>
+0x302D	0x7919	# <CJK>
+0x302E	0x611B	# <CJK>
+0x302F	0x9698	# <CJK>
+0x3030	0x978D	# <CJK>
+0x3031	0x6C28	# <CJK>
+0x3032	0x5B89	# <CJK>
+0x3033	0x4FFA	# <CJK>
+0x3034	0x6309	# <CJK>
+0x3035	0x6697	# <CJK>
+0x3036	0x5CB8	# <CJK>
+0x3037	0x80FA	# <CJK>
+0x3038	0x6848	# <CJK>
+0x3039	0x9AAF	# <CJK>
+0x303A	0x6602	# <CJK>
+0x303B	0x76CE	# <CJK>
+0x303C	0x51F9	# <CJK>
+0x303D	0x6556	# <CJK>
+0x303E	0x71AC	# <CJK>
+0x303F	0x7FF1	# <CJK>
+0x3040	0x8956	# <CJK>
+0x3041	0x50B2	# <CJK>
+0x3042	0x5965	# <CJK>
+0x3043	0x61CA	# <CJK>
+0x3044	0x6FB3	# <CJK>
+0x3045	0x82AD	# <CJK>
+0x3046	0x634C	# <CJK>
+0x3047	0x6252	# <CJK>
+0x3048	0x53ED	# <CJK>
+0x3049	0x5427	# <CJK>
+0x304A	0x7B06	# <CJK>
+0x304B	0x516B	# <CJK>
+0x304C	0x75A4	# <CJK>
+0x304D	0x5DF4	# <CJK>
+0x304E	0x62D4	# <CJK>
+0x304F	0x8DCB	# <CJK>
+0x3050	0x9776	# <CJK>
+0x3051	0x628A	# <CJK>
+0x3052	0x8019	# <CJK>
+0x3053	0x58E9	# <CJK>
+0x3054	0x9738	# <CJK>
+0x3055	0x7F77	# <CJK>
+0x3056	0x7238	# <CJK>
+0x3057	0x767D	# <CJK>
+0x3058	0x67CF	# <CJK>
+0x3059	0x767E	# <CJK>
+0x305A	0x64FA	# <CJK>
+0x305B	0x4F70	# <CJK>
+0x305C	0x6557	# <CJK>
+0x305D	0x62DC	# <CJK>
+0x305E	0x7A17	# <CJK>
+0x305F	0x6591	# <CJK>
+0x3060	0x73ED	# <CJK>
+0x3061	0x642C	# <CJK>
+0x3062	0x6273	# <CJK>
+0x3063	0x822C	# <CJK>
+0x3064	0x9812	# <CJK>
+0x3065	0x677F	# <CJK>
+0x3066	0x7248	# <CJK>
+0x3067	0x626E	# <CJK>
+0x3068	0x62CC	# <CJK>
+0x3069	0x4F34	# <CJK>
+0x306A	0x74E3	# <CJK>
+0x306B	0x534A	# <CJK>
+0x306C	0x8FA6	# <CJK>
+0x306D	0x7D46	# <CJK>
+0x306E	0x90A6	# <CJK>
+0x306F	0x5E6B	# <CJK>
+0x3070	0x6886	# <CJK>
+0x3071	0x699C	# <CJK>
+0x3072	0x8180	# <CJK>
+0x3073	0x7D81	# <CJK>
+0x3074	0x68D2	# <CJK>
+0x3075	0x78C5	# <CJK>
+0x3076	0x868C	# <CJK>
+0x3077	0x938A	# <CJK>
+0x3078	0x508D	# <CJK>
+0x3079	0x8B17	# <CJK>
+0x307A	0x82DE	# <CJK>
+0x307B	0x80DE	# <CJK>
+0x307C	0x5305	# <CJK>
+0x307D	0x8912	# <CJK>
+0x307E	0x5265	# <CJK>
+0x3121	0x8584	# <CJK>
+0x3122	0x96F9	# <CJK>
+0x3123	0x4FDD	# <CJK>
+0x3124	0x5821	# <CJK>
+0x3125	0x98FD	# <CJK>
+0x3126	0x5BF6	# <CJK>
+0x3127	0x62B1	# <CJK>
+0x3128	0x5831	# <CJK>
+0x3129	0x66B4	# <CJK>
+0x312A	0x8C79	# <CJK>
+0x312B	0x9B91	# <CJK>
+0x312C	0x7206	# <CJK>
+0x312D	0x676F	# <CJK>
+0x312E	0x7891	# <CJK>
+0x312F	0x60B2	# <CJK>
+0x3130	0x5351	# <CJK>
+0x3131	0x5317	# <CJK>
+0x3132	0x8F29	# <CJK>
+0x3133	0x80CC	# <CJK>
+0x3134	0x8C9D	# <CJK>
+0x3135	0x92C7	# <CJK>
+0x3136	0x500D	# <CJK>
+0x3137	0x72FD	# <CJK>
+0x3138	0x5099	# <CJK>
+0x3139	0x618A	# <CJK>
+0x313A	0x7119	# <CJK>
+0x313B	0x88AB	# <CJK>
+0x313C	0x5954	# <CJK>
+0x313D	0x82EF	# <CJK>
+0x313E	0x672C	# <CJK>
+0x313F	0x7B28	# <CJK>
+0x3140	0x5D29	# <CJK>
+0x3141	0x7DB3	# <CJK>
+0x3142	0x752D	# <CJK>
+0x3143	0x6CF5	# <CJK>
+0x3144	0x8E66	# <CJK>
+0x3145	0x8FF8	# <CJK>
+0x3146	0x903C	# <CJK>
+0x3147	0x9F3B	# <CJK>
+0x3148	0x6BD4	# <CJK>
+0x3149	0x9119	# <CJK>
+0x314A	0x7B46	# <CJK>
+0x314B	0x5F7C	# <CJK>
+0x314C	0x78A7	# <CJK>
+0x314D	0x84D6	# <CJK>
+0x314E	0x853D	# <CJK>
+0x314F	0x7562	# <CJK>
+0x3150	0x6583	# <CJK>
+0x3151	0x6BD6	# <CJK>
+0x3152	0x5E63	# <CJK>
+0x3153	0x5E87	# <CJK>
+0x3154	0x75F9	# <CJK>
+0x3155	0x9589	# <CJK>
+0x3156	0x655D	# <CJK>
+0x3157	0x5F0A	# <CJK>
+0x3158	0x5FC5	# <CJK>
+0x3159	0x8F9F	# <CJK>
+0x315A	0x58C1	# <CJK>
+0x315B	0x81C2	# <CJK>
+0x315C	0x907F	# <CJK>
+0x315D	0x965B	# <CJK>
+0x315E	0x97AD	# <CJK>
+0x315F	0x908A	# <CJK>
+0x3160	0x7DE8	# <CJK>
+0x3161	0x8CB6	# <CJK>
+0x3162	0x6241	# <CJK>
+0x3163	0x4FBF	# <CJK>
+0x3164	0x8B8A	# <CJK>
+0x3165	0x535E	# <CJK>
+0x3166	0x8FA8	# <CJK>
+0x3167	0x8FAF	# <CJK>
+0x3168	0x8FAE	# <CJK>
+0x3169	0x904D	# <CJK>
+0x316A	0x6A19	# <CJK>
+0x316B	0x5F6A	# <CJK>
+0x316C	0x8198	# <CJK>
+0x316D	0x8868	# <CJK>
+0x316E	0x9C49	# <CJK>
+0x316F	0x618B	# <CJK>
+0x3170	0x522B	# <CJK>
+0x3171	0x765F	# <CJK>
+0x3172	0x5F6C	# <CJK>
+0x3173	0x658C	# <CJK>
+0x3174	0x7015	# <CJK>
+0x3175	0x6FF1	# <CJK>
+0x3176	0x8CD3	# <CJK>
+0x3177	0x64EF	# <CJK>
+0x3178	0x5175	# <CJK>
+0x3179	0x51B0	# <CJK>
+0x317A	0x67C4	# <CJK>
+0x317B	0x4E19	# <CJK>
+0x317C	0x79C9	# <CJK>
+0x317D	0x9905	# <CJK>
+0x317E	0x70B3	# <CJK>
+0x3221	0x75C5	# <CJK>
+0x3222	0x5E76	# <CJK>
+0x3223	0x73BB	# <CJK>
+0x3224	0x83E0	# <CJK>
+0x3225	0x64AD	# <CJK>
+0x3226	0x64A5	# <CJK>
+0x3227	0x9262	# <CJK>
+0x3228	0x6CE2	# <CJK>
+0x3229	0x535A	# <CJK>
+0x322A	0x52C3	# <CJK>
+0x322B	0x640F	# <CJK>
+0x322C	0x9251	# <CJK>
+0x322D	0x7B94	# <CJK>
+0x322E	0x4F2F	# <CJK>
+0x322F	0x5E1B	# <CJK>
+0x3230	0x8236	# <CJK>
+0x3231	0x8116	# <CJK>
+0x3232	0x818A	# <CJK>
+0x3233	0x6E24	# <CJK>
+0x3234	0x6CCA	# <CJK>
+0x3235	0x99C1	# <CJK>
+0x3236	0x6355	# <CJK>
+0x3237	0x535C	# <CJK>
+0x3238	0x54FA	# <CJK>
+0x3239	0x88DC	# <CJK>
+0x323A	0x57E0	# <CJK>
+0x323B	0x4E0D	# <CJK>
+0x323C	0x5E03	# <CJK>
+0x323D	0x6B65	# <CJK>
+0x323E	0x7C3F	# <CJK>
+0x323F	0x90E8	# <CJK>
+0x3240	0x6016	# <CJK>
+0x3241	0x64E6	# <CJK>
+0x3242	0x731C	# <CJK>
+0x3243	0x88C1	# <CJK>
+0x3244	0x6750	# <CJK>
+0x3245	0x624D	# <CJK>
+0x3246	0x8CA1	# <CJK>
+0x3247	0x776C	# <CJK>
+0x3248	0x8E29	# <CJK>
+0x3249	0x91C7	# <CJK>
+0x324A	0x5F69	# <CJK>
+0x324B	0x83DC	# <CJK>
+0x324C	0x8521	# <CJK>
+0x324D	0x9910	# <CJK>
+0x324E	0x53C3	# <CJK>
+0x324F	0x8836	# <CJK>
+0x3250	0x6B98	# <CJK>
+0x3251	0x615A	# <CJK>
+0x3252	0x6158	# <CJK>
+0x3253	0x71E6	# <CJK>
+0x3254	0x84BC	# <CJK>
+0x3255	0x8259	# <CJK>
+0x3256	0x5009	# <CJK>
+0x3257	0x6EC4	# <CJK>
+0x3258	0x85CF	# <CJK>
+0x3259	0x64CD	# <CJK>
+0x325A	0x7CD9	# <CJK>
+0x325B	0x69FD	# <CJK>
+0x325C	0x66F9	# <CJK>
+0x325D	0x8349	# <CJK>
+0x325E	0x53A0	# <CJK>
+0x325F	0x7B56	# <CJK>
+0x3260	0x5074	# <CJK>
+0x3261	0x518C	# <CJK>
+0x3262	0x6E2C	# <CJK>
+0x3263	0x5C64	# <CJK>
+0x3264	0x8E6D	# <CJK>
+0x3265	0x63D2	# <CJK>
+0x3266	0x53C9	# <CJK>
+0x3267	0x832C	# <CJK>
+0x3268	0x8336	# <CJK>
+0x3269	0x67E5	# <CJK>
+0x326A	0x78B4	# <CJK>
+0x326B	0x643D	# <CJK>
+0x326C	0x5BDF	# <CJK>
+0x326D	0x5C94	# <CJK>
+0x326E	0x5DEE	# <CJK>
+0x326F	0x8A6B	# <CJK>
+0x3270	0x62C6	# <CJK>
+0x3271	0x67F4	# <CJK>
+0x3272	0x8C7A	# <CJK>
+0x3273	0x6519	# <CJK>
+0x3274	0x647B	# <CJK>
+0x3275	0x87EC	# <CJK>
+0x3276	0x995E	# <CJK>
+0x3277	0x8B92	# <CJK>
+0x3278	0x7E8F	# <CJK>
+0x3279	0x93DF	# <CJK>
+0x327A	0x7523	# <CJK>
+0x327B	0x95E1	# <CJK>
+0x327C	0x986B	# <CJK>
+0x327D	0x660C	# <CJK>
+0x327E	0x7316	# <CJK>
+0x3321	0x5834	# <CJK>
+0x3322	0x5617	# <CJK>
+0x3323	0x5E38	# <CJK>
+0x3324	0x9577	# <CJK>
+0x3325	0x511F	# <CJK>
+0x3326	0x8178	# <CJK>
+0x3327	0x5EE0	# <CJK>
+0x3328	0x655E	# <CJK>
+0x3329	0x66A2	# <CJK>
+0x332A	0x5531	# <CJK>
+0x332B	0x5021	# <CJK>
+0x332C	0x8D85	# <CJK>
+0x332D	0x6284	# <CJK>
+0x332E	0x9214	# <CJK>
+0x332F	0x671D	# <CJK>
+0x3330	0x5632	# <CJK>
+0x3331	0x6F6E	# <CJK>
+0x3332	0x5DE2	# <CJK>
+0x3333	0x5435	# <CJK>
+0x3334	0x7092	# <CJK>
+0x3335	0x8ECA	# <CJK>
+0x3336	0x626F	# <CJK>
+0x3337	0x64A4	# <CJK>
+0x3338	0x63A3	# <CJK>
+0x3339	0x5FB9	# <CJK>
+0x333A	0x6F88	# <CJK>
+0x333B	0x90F4	# <CJK>
+0x333C	0x81E3	# <CJK>
+0x333D	0x8FB0	# <CJK>
+0x333E	0x5875	# <CJK>
+0x333F	0x6668	# <CJK>
+0x3340	0x5FF1	# <CJK>
+0x3341	0x6C89	# <CJK>
+0x3342	0x9673	# <CJK>
+0x3343	0x8D81	# <CJK>
+0x3344	0x896F	# <CJK>
+0x3345	0x6491	# <CJK>
+0x3346	0x7A31	# <CJK>
+0x3347	0x57CE	# <CJK>
+0x3348	0x6A59	# <CJK>
+0x3349	0x6210	# <CJK>
+0x334A	0x5448	# <CJK>
+0x334B	0x4E58	# <CJK>
+0x334C	0x7A0B	# <CJK>
+0x334D	0x61F2	# <CJK>
+0x334E	0x6F84	# <CJK>
+0x334F	0x8AA0	# <CJK>
+0x3350	0x627F	# <CJK>
+0x3351	0x901E	# <CJK>
+0x3352	0x9A01	# <CJK>
+0x3353	0x79E4	# <CJK>
+0x3354	0x5403	# <CJK>
+0x3355	0x75F4	# <CJK>
+0x3356	0x6301	# <CJK>
+0x3357	0x5319	# <CJK>
+0x3358	0x6C60	# <CJK>
+0x3359	0x9072	# <CJK>
+0x335A	0x5F1B	# <CJK>
+0x335B	0x99B3	# <CJK>
+0x335C	0x803B	# <CJK>
+0x335D	0x9F52	# <CJK>
+0x335E	0x4F88	# <CJK>
+0x335F	0x5C3A	# <CJK>
+0x3360	0x8D64	# <CJK>
+0x3361	0x7FC5	# <CJK>
+0x3362	0x65A5	# <CJK>
+0x3363	0x71BE	# <CJK>
+0x3364	0x5145	# <CJK>
+0x3365	0x885D	# <CJK>
+0x3366	0x87F2	# <CJK>
+0x3367	0x5D07	# <CJK>
+0x3368	0x5BF5	# <CJK>
+0x3369	0x62BD	# <CJK>
+0x336A	0x916C	# <CJK>
+0x336B	0x7587	# <CJK>
+0x336C	0x8E8A	# <CJK>
+0x336D	0x7A20	# <CJK>
+0x336E	0x6101	# <CJK>
+0x336F	0x7C4C	# <CJK>
+0x3370	0x4EC7	# <CJK>
+0x3371	0x7DA2	# <CJK>
+0x3372	0x7785	# <CJK>
+0x3373	0x919C	# <CJK>
+0x3374	0x81ED	# <CJK>
+0x3375	0x521D	# <CJK>
+0x3376	0x51FA	# <CJK>
+0x3377	0x6A71	# <CJK>
+0x3378	0x53A8	# <CJK>
+0x3379	0x8E87	# <CJK>
+0x337A	0x92E4	# <CJK>
+0x337B	0x96DB	# <CJK>
+0x337C	0x6EC1	# <CJK>
+0x337D	0x9664	# <CJK>
+0x337E	0x695A	# <CJK>
+0x3421	0x790E	# <CJK>
+0x3422	0x5132	# <CJK>
+0x3423	0x77D7	# <CJK>
+0x3424	0x6410	# <CJK>
+0x3425	0x89F8	# <CJK>
+0x3426	0x8655	# <CJK>
+0x3427	0x63E3	# <CJK>
+0x3428	0x5DDD	# <CJK>
+0x3429	0x7A7F	# <CJK>
+0x342A	0x693D	# <CJK>
+0x342B	0x50B3	# <CJK>
+0x342C	0x8239	# <CJK>
+0x342D	0x5598	# <CJK>
+0x342E	0x4E32	# <CJK>
+0x342F	0x7621	# <CJK>
+0x3430	0x7A97	# <CJK>
+0x3431	0x5E62	# <CJK>
+0x3432	0x5E8A	# <CJK>
+0x3433	0x95D6	# <CJK>
+0x3434	0x5275	# <CJK>
+0x3435	0x5439	# <CJK>
+0x3436	0x708A	# <CJK>
+0x3437	0x6376	# <CJK>
+0x3438	0x9318	# <CJK>
+0x3439	0x5782	# <CJK>
+0x343A	0x6625	# <CJK>
+0x343B	0x693F	# <CJK>
+0x343C	0x9187	# <CJK>
+0x343D	0x5507	# <CJK>
+0x343E	0x6DF3	# <CJK>
+0x343F	0x7D14	# <CJK>
+0x3440	0x8822	# <CJK>
+0x3441	0x6233	# <CJK>
+0x3442	0x7DBD	# <CJK>
+0x3443	0x75B5	# <CJK>
+0x3444	0x8328	# <CJK>
+0x3445	0x78C1	# <CJK>
+0x3446	0x96CC	# <CJK>
+0x3447	0x8FAD	# <CJK>
+0x3448	0x6148	# <CJK>
+0x3449	0x74F7	# <CJK>
+0x344A	0x8A5E	# <CJK>
+0x344B	0x6B64	# <CJK>
+0x344C	0x523A	# <CJK>
+0x344D	0x8CDC	# <CJK>
+0x344E	0x6B21	# <CJK>
+0x344F	0x8070	# <CJK>
+0x3450	0x8471	# <CJK>
+0x3451	0x56F1	# <CJK>
+0x3452	0x5306	# <CJK>
+0x3453	0x5F9E	# <CJK>
+0x3454	0x53E2	# <CJK>
+0x3455	0x51D1	# <CJK>
+0x3456	0x7C97	# <CJK>
+0x3457	0x918B	# <CJK>
+0x3458	0x7C07	# <CJK>
+0x3459	0x4FC3	# <CJK>
+0x345A	0x8EA5	# <CJK>
+0x345B	0x7BE1	# <CJK>
+0x345C	0x7AC4	# <CJK>
+0x345D	0x6467	# <CJK>
+0x345E	0x5D14	# <CJK>
+0x345F	0x50AC	# <CJK>
+0x3460	0x8106	# <CJK>
+0x3461	0x7601	# <CJK>
+0x3462	0x7CB9	# <CJK>
+0x3463	0x6DEC	# <CJK>
+0x3464	0x7FE0	# <CJK>
+0x3465	0x6751	# <CJK>
+0x3466	0x5B58	# <CJK>
+0x3467	0x5BF8	# <CJK>
+0x3468	0x78CB	# <CJK>
+0x3469	0x64AE	# <CJK>
+0x346A	0x6413	# <CJK>
+0x346B	0x63AA	# <CJK>
+0x346C	0x632B	# <CJK>
+0x346D	0x932F	# <CJK>
+0x346E	0x642D	# <CJK>
+0x346F	0x9054	# <CJK>
+0x3470	0x7B54	# <CJK>
+0x3471	0x7629	# <CJK>
+0x3472	0x6253	# <CJK>
+0x3473	0x5927	# <CJK>
+0x3474	0x5446	# <CJK>
+0x3475	0x6B79	# <CJK>
+0x3476	0x50A3	# <CJK>
+0x3477	0x6234	# <CJK>
+0x3478	0x5E36	# <CJK>
+0x3479	0x6B86	# <CJK>
+0x347A	0x4EE3	# <CJK>
+0x347B	0x8CB8	# <CJK>
+0x347C	0x888B	# <CJK>
+0x347D	0x5F85	# <CJK>
+0x347E	0x902E	# <CJK>
+0x3521	0x6020	# <CJK>
+0x3522	0x803D	# <CJK>
+0x3523	0x64D4	# <CJK>
+0x3524	0x4E39	# <CJK>
+0x3525	0x55AE	# <CJK>
+0x3526	0x9132	# <CJK>
+0x3527	0x64A3	# <CJK>
+0x3528	0x81BD	# <CJK>
+0x3529	0x65E6	# <CJK>
+0x352A	0x6C2E	# <CJK>
+0x352B	0x4F46	# <CJK>
+0x352C	0x619A	# <CJK>
+0x352D	0x6DE1	# <CJK>
+0x352E	0x8A95	# <CJK>
+0x352F	0x5F48	# <CJK>
+0x3530	0x86CB	# <CJK>
+0x3531	0x7576	# <CJK>
+0x3532	0x64CB	# <CJK>
+0x3533	0x9EE8	# <CJK>
+0x3534	0x8569	# <CJK>
+0x3535	0x6A94	# <CJK>
+0x3536	0x5200	# <CJK>
+0x3537	0x6417	# <CJK>
+0x3538	0x8E48	# <CJK>
+0x3539	0x5012	# <CJK>
+0x353A	0x5CF6	# <CJK>
+0x353B	0x79B1	# <CJK>
+0x353C	0x5C0E	# <CJK>
+0x353D	0x5230	# <CJK>
+0x353E	0x7A3B	# <CJK>
+0x353F	0x60BC	# <CJK>
+0x3540	0x9053	# <CJK>
+0x3541	0x76D7	# <CJK>
+0x3542	0x5FB7	# <CJK>
+0x3543	0x5F97	# <CJK>
+0x3544	0x7684	# <CJK>
+0x3545	0x8E6C	# <CJK>
+0x3546	0x71C8	# <CJK>
+0x3547	0x767B	# <CJK>
+0x3548	0x7B49	# <CJK>
+0x3549	0x77AA	# <CJK>
+0x354A	0x51F3	# <CJK>
+0x354B	0x9127	# <CJK>
+0x354C	0x5824	# <CJK>
+0x354D	0x4F4E	# <CJK>
+0x354E	0x6EF4	# <CJK>
+0x354F	0x8FEA	# <CJK>
+0x3550	0x6575	# <CJK>
+0x3551	0x7B1B	# <CJK>
+0x3552	0x72C4	# <CJK>
+0x3553	0x6ECC	# <CJK>
+0x3554	0x7FDF	# <CJK>
+0x3555	0x5AE1	# <CJK>
+0x3556	0x62B5	# <CJK>
+0x3557	0x5E95	# <CJK>
+0x3558	0x5730	# <CJK>
+0x3559	0x8482	# <CJK>
+0x355A	0x7B2C	# <CJK>
+0x355B	0x5E1D	# <CJK>
+0x355C	0x5F1F	# <CJK>
+0x355D	0x905E	# <CJK>
+0x355E	0x7DE0	# <CJK>
+0x355F	0x985B	# <CJK>
+0x3560	0x6382	# <CJK>
+0x3561	0x6EC7	# <CJK>
+0x3562	0x7898	# <CJK>
+0x3563	0x9EDE	# <CJK>
+0x3564	0x5178	# <CJK>
+0x3565	0x975B	# <CJK>
+0x3566	0x588A	# <CJK>
+0x3567	0x96FB	# <CJK>
+0x3568	0x4F43	# <CJK>
+0x3569	0x7538	# <CJK>
+0x356A	0x5E97	# <CJK>
+0x356B	0x60E6	# <CJK>
+0x356C	0x5960	# <CJK>
+0x356D	0x6FB1	# <CJK>
+0x356E	0x6BBF	# <CJK>
+0x356F	0x7889	# <CJK>
+0x3570	0x53FC	# <CJK>
+0x3571	0x96D5	# <CJK>
+0x3572	0x51CB	# <CJK>
+0x3573	0x5201	# <CJK>
+0x3574	0x6389	# <CJK>
+0x3575	0x540A	# <CJK>
+0x3576	0x91E3	# <CJK>
+0x3577	0x8ABF	# <CJK>
+0x3578	0x8DCC	# <CJK>
+0x3579	0x7239	# <CJK>
+0x357A	0x789F	# <CJK>
+0x357B	0x8776	# <CJK>
+0x357C	0x8FED	# <CJK>
+0x357D	0x8ADC	# <CJK>
+0x357E	0x758A	# <CJK>
+0x3621	0x4E01	# <CJK>
+0x3622	0x76EF	# <CJK>
+0x3623	0x53EE	# <CJK>
+0x3624	0x91D8	# <CJK>
+0x3625	0x9802	# <CJK>
+0x3626	0x9F0E	# <CJK>
+0x3627	0x9320	# <CJK>
+0x3628	0x5B9A	# <CJK>
+0x3629	0x8A02	# <CJK>
+0x362A	0x4E22	# <CJK>
+0x362B	0x6771	# <CJK>
+0x362C	0x51AC	# <CJK>
+0x362D	0x8463	# <CJK>
+0x362E	0x61C2	# <CJK>
+0x362F	0x52D5	# <CJK>
+0x3630	0x68DF	# <CJK>
+0x3631	0x4F97	# <CJK>
+0x3632	0x606B	# <CJK>
+0x3633	0x51CD	# <CJK>
+0x3634	0x6D1E	# <CJK>
+0x3635	0x515C	# <CJK>
+0x3636	0x6296	# <CJK>
+0x3637	0x9B25	# <CJK>
+0x3638	0x9661	# <CJK>
+0x3639	0x8C46	# <CJK>
+0x363A	0x9017	# <CJK>
+0x363B	0x75D8	# <CJK>
+0x363C	0x90FD	# <CJK>
+0x363D	0x7763	# <CJK>
+0x363E	0x6BD2	# <CJK>
+0x363F	0x72A2	# <CJK>
+0x3640	0x7368	# <CJK>
+0x3641	0x8B80	# <CJK>
+0x3642	0x5835	# <CJK>
+0x3643	0x7779	# <CJK>
+0x3644	0x8CED	# <CJK>
+0x3645	0x675C	# <CJK>
+0x3646	0x934D	# <CJK>
+0x3647	0x809A	# <CJK>
+0x3648	0x5EA6	# <CJK>
+0x3649	0x6E21	# <CJK>
+0x364A	0x5992	# <CJK>
+0x364B	0x7AEF	# <CJK>
+0x364C	0x77ED	# <CJK>
+0x364D	0x935B	# <CJK>
+0x364E	0x6BB5	# <CJK>
+0x364F	0x65B7	# <CJK>
+0x3650	0x7DDE	# <CJK>
+0x3651	0x5806	# <CJK>
+0x3652	0x5151	# <CJK>
+0x3653	0x968A	# <CJK>
+0x3654	0x5C0D	# <CJK>
+0x3655	0x58A9	# <CJK>
+0x3656	0x5678	# <CJK>
+0x3657	0x8E72	# <CJK>
+0x3658	0x6566	# <CJK>
+0x3659	0x9813	# <CJK>
+0x365A	0x56E4	# <CJK>
+0x365B	0x920D	# <CJK>
+0x365C	0x76FE	# <CJK>
+0x365D	0x9041	# <CJK>
+0x365E	0x6387	# <CJK>
+0x365F	0x54C6	# <CJK>
+0x3660	0x591A	# <CJK>
+0x3661	0x596A	# <CJK>
+0x3662	0x579B	# <CJK>
+0x3663	0x8EB2	# <CJK>
+0x3664	0x6735	# <CJK>
+0x3665	0x8DFA	# <CJK>
+0x3666	0x8235	# <CJK>
+0x3667	0x5241	# <CJK>
+0x3668	0x60F0	# <CJK>
+0x3669	0x58AE	# <CJK>
+0x366A	0x86FE	# <CJK>
+0x366B	0x5CE8	# <CJK>
+0x366C	0x9D5D	# <CJK>
+0x366D	0x4FC4	# <CJK>
+0x366E	0x984D	# <CJK>
+0x366F	0x8A1B	# <CJK>
+0x3670	0x5A25	# <CJK>
+0x3671	0x60E1	# <CJK>
+0x3672	0x5384	# <CJK>
+0x3673	0x627C	# <CJK>
+0x3674	0x904F	# <CJK>
+0x3675	0x9102	# <CJK>
+0x3676	0x9913	# <CJK>
+0x3677	0x6069	# <CJK>
+0x3678	0x800C	# <CJK>
+0x3679	0x5152	# <CJK>
+0x367A	0x8033	# <CJK>
+0x367B	0x723E	# <CJK>
+0x367C	0x990C	# <CJK>
+0x367D	0x6D31	# <CJK>
+0x367E	0x4E8C	# <CJK>
+0x3721	0x8CB3	# <CJK>
+0x3722	0x767C	# <CJK>
+0x3723	0x7F70	# <CJK>
+0x3724	0x7B4F	# <CJK>
+0x3725	0x4F10	# <CJK>
+0x3726	0x4E4F	# <CJK>
+0x3727	0x95A5	# <CJK>
+0x3728	0x6CD5	# <CJK>
+0x3729	0x73D0	# <CJK>
+0x372A	0x85E9	# <CJK>
+0x372B	0x5E06	# <CJK>
+0x372C	0x756A	# <CJK>
+0x372D	0x7FFB	# <CJK>
+0x372E	0x6A0A	# <CJK>
+0x372F	0x792C	# <CJK>
+0x3730	0x91E9	# <CJK>
+0x3731	0x7E41	# <CJK>
+0x3732	0x51E1	# <CJK>
+0x3733	0x7169	# <CJK>
+0x3734	0x53CD	# <CJK>
+0x3735	0x8FD4	# <CJK>
+0x3736	0x7BC4	# <CJK>
+0x3737	0x8CA9	# <CJK>
+0x3738	0x72AF	# <CJK>
+0x3739	0x98EF	# <CJK>
+0x373A	0x6CDB	# <CJK>
+0x373B	0x574A	# <CJK>
+0x373C	0x82B3	# <CJK>
+0x373D	0x65B9	# <CJK>
+0x373E	0x80AA	# <CJK>
+0x373F	0x623F	# <CJK>
+0x3740	0x9632	# <CJK>
+0x3741	0x59A8	# <CJK>
+0x3742	0x4EFF	# <CJK>
+0x3743	0x8A2A	# <CJK>
+0x3744	0x7D21	# <CJK>
+0x3745	0x653E	# <CJK>
+0x3746	0x83F2	# <CJK>
+0x3747	0x975E	# <CJK>
+0x3748	0x5561	# <CJK>
+0x3749	0x98DB	# <CJK>
+0x374A	0x80A5	# <CJK>
+0x374B	0x532A	# <CJK>
+0x374C	0x8AB9	# <CJK>
+0x374D	0x5420	# <CJK>
+0x374E	0x80BA	# <CJK>
+0x374F	0x5EE2	# <CJK>
+0x3750	0x6CB8	# <CJK>
+0x3751	0x8CBB	# <CJK>
+0x3752	0x82AC	# <CJK>
+0x3753	0x915A	# <CJK>
+0x3754	0x5429	# <CJK>
+0x3755	0x6C1B	# <CJK>
+0x3756	0x5206	# <CJK>
+0x3757	0x7D1B	# <CJK>
+0x3758	0x58B3	# <CJK>
+0x3759	0x711A	# <CJK>
+0x375A	0x6C7E	# <CJK>
+0x375B	0x7C89	# <CJK>
+0x375C	0x596E	# <CJK>
+0x375D	0x4EFD	# <CJK>
+0x375E	0x5FFF	# <CJK>
+0x375F	0x61A4	# <CJK>
+0x3760	0x7CDE	# <CJK>
+0x3761	0x8C50	# <CJK>
+0x3762	0x5C01	# <CJK>
+0x3763	0x6953	# <CJK>
+0x3764	0x8702	# <CJK>
+0x3765	0x5CF0	# <CJK>
+0x3766	0x92D2	# <CJK>
+0x3767	0x98A8	# <CJK>
+0x3768	0x760B	# <CJK>
+0x3769	0x70FD	# <CJK>
+0x376A	0x9022	# <CJK>
+0x376B	0x99AE	# <CJK>
+0x376C	0x7E2B	# <CJK>
+0x376D	0x8AF7	# <CJK>
+0x376E	0x5949	# <CJK>
+0x376F	0x9CF3	# <CJK>
+0x3770	0x4F5B	# <CJK>
+0x3771	0x5426	# <CJK>
+0x3772	0x592B	# <CJK>
+0x3773	0x6577	# <CJK>
+0x3774	0x819A	# <CJK>
+0x3775	0x5B75	# <CJK>
+0x3776	0x6276	# <CJK>
+0x3777	0x62C2	# <CJK>
+0x3778	0x8F3B	# <CJK>
+0x3779	0x5E45	# <CJK>
+0x377A	0x6C1F	# <CJK>
+0x377B	0x7B26	# <CJK>
+0x377C	0x4F0F	# <CJK>
+0x377D	0x4FD8	# <CJK>
+0x377E	0x670D	# <CJK>
+0x3821	0x6D6E	# <CJK>
+0x3822	0x6DAA	# <CJK>
+0x3823	0x798F	# <CJK>
+0x3824	0x88B1	# <CJK>
+0x3825	0x5F17	# <CJK>
+0x3826	0x752B	# <CJK>
+0x3827	0x64AB	# <CJK>
+0x3828	0x8F14	# <CJK>
+0x3829	0x4FEF	# <CJK>
+0x382A	0x91DC	# <CJK>
+0x382B	0x65A7	# <CJK>
+0x382C	0x812F	# <CJK>
+0x382D	0x8151	# <CJK>
+0x382E	0x5E9C	# <CJK>
+0x382F	0x8150	# <CJK>
+0x3830	0x8D74	# <CJK>
+0x3831	0x526F	# <CJK>
+0x3832	0x8986	# <CJK>
+0x3833	0x8CE6	# <CJK>
+0x3834	0x5FA9	# <CJK>
+0x3835	0x5085	# <CJK>
+0x3836	0x4ED8	# <CJK>
+0x3837	0x961C	# <CJK>
+0x3838	0x7236	# <CJK>
+0x3839	0x8179	# <CJK>
+0x383A	0x8CA0	# <CJK>
+0x383B	0x5BCC	# <CJK>
+0x383C	0x8A03	# <CJK>
+0x383D	0x9644	# <CJK>
+0x383E	0x5A66	# <CJK>
+0x383F	0x7E1B	# <CJK>
+0x3840	0x5490	# <CJK>
+0x3841	0x5676	# <CJK>
+0x3842	0x560E	# <CJK>
+0x3843	0x8A72	# <CJK>
+0x3844	0x6539	# <CJK>
+0x3845	0x6982	# <CJK>
+0x3846	0x9223	# <CJK>
+0x3847	0x84CB	# <CJK>
+0x3848	0x6E89	# <CJK>
+0x3849	0x5E79	# <CJK>
+0x384A	0x7518	# <CJK>
+0x384B	0x6746	# <CJK>
+0x384C	0x67D1	# <CJK>
+0x384D	0x7AFF	# <CJK>
+0x384E	0x809D	# <CJK>
+0x384F	0x8D95	# <CJK>
+0x3850	0x611F	# <CJK>
+0x3851	0x79C6	# <CJK>
+0x3852	0x6562	# <CJK>
+0x3853	0x8D1B	# <CJK>
+0x3854	0x5CA1	# <CJK>
+0x3855	0x525B	# <CJK>
+0x3856	0x92FC	# <CJK>
+0x3857	0x7F38	# <CJK>
+0x3858	0x809B	# <CJK>
+0x3859	0x7DB1	# <CJK>
+0x385A	0x5D17	# <CJK>
+0x385B	0x6E2F	# <CJK>
+0x385C	0x6760	# <CJK>
+0x385D	0x7BD9	# <CJK>
+0x385E	0x768B	# <CJK>
+0x385F	0x9AD8	# <CJK>
+0x3860	0x818F	# <CJK>
+0x3861	0x7F94	# <CJK>
+0x3862	0x7CD5	# <CJK>
+0x3863	0x641E	# <CJK>
+0x3864	0x93AC	# <CJK>
+0x3865	0x7A3F	# <CJK>
+0x3866	0x544A	# <CJK>
+0x3867	0x54E5	# <CJK>
+0x3868	0x6B4C	# <CJK>
+0x3869	0x64F1	# <CJK>
+0x386A	0x6208	# <CJK>
+0x386B	0x9D3F	# <CJK>
+0x386C	0x80F3	# <CJK>
+0x386D	0x7599	# <CJK>
+0x386E	0x5272	# <CJK>
+0x386F	0x9769	# <CJK>
+0x3870	0x845B	# <CJK>
+0x3871	0x683C	# <CJK>
+0x3872	0x86E4	# <CJK>
+0x3873	0x95A3	# <CJK>
+0x3874	0x9694	# <CJK>
+0x3875	0x927B	# <CJK>
+0x3876	0x500B	# <CJK>
+0x3877	0x5404	# <CJK>
+0x3878	0x7D66	# <CJK>
+0x3879	0x6839	# <CJK>
+0x387A	0x8DDF	# <CJK>
+0x387B	0x8015	# <CJK>
+0x387C	0x66F4	# <CJK>
+0x387D	0x5E9A	# <CJK>
+0x387E	0x7FB9	# <CJK>
+0x3921	0x57C2	# <CJK>
+0x3922	0x803F	# <CJK>
+0x3923	0x6897	# <CJK>
+0x3924	0x5DE5	# <CJK>
+0x3925	0x653B	# <CJK>
+0x3926	0x529F	# <CJK>
+0x3927	0x606D	# <CJK>
+0x3928	0x9F94	# <CJK>
+0x3929	0x4F9B	# <CJK>
+0x392A	0x8EAC	# <CJK>
+0x392B	0x516C	# <CJK>
+0x392C	0x5BAB	# <CJK>
+0x392D	0x5F13	# <CJK>
+0x392E	0x978F	# <CJK>
+0x392F	0x6C5E	# <CJK>
+0x3930	0x62F1	# <CJK>
+0x3931	0x8CA2	# <CJK>
+0x3932	0x5171	# <CJK>
+0x3933	0x920E	# <CJK>
+0x3934	0x52FE	# <CJK>
+0x3935	0x6E9D	# <CJK>
+0x3936	0x82DF	# <CJK>
+0x3937	0x72D7	# <CJK>
+0x3938	0x57A2	# <CJK>
+0x3939	0x69CB	# <CJK>
+0x393A	0x8CFC	# <CJK>
+0x393B	0x591F	# <CJK>
+0x393C	0x8F9C	# <CJK>
+0x393D	0x83C7	# <CJK>
+0x393E	0x5495	# <CJK>
+0x393F	0x7B8D	# <CJK>
+0x3940	0x4F30	# <CJK>
+0x3941	0x6CBD	# <CJK>
+0x3942	0x5B64	# <CJK>
+0x3943	0x59D1	# <CJK>
+0x3944	0x9F13	# <CJK>
+0x3945	0x53E4	# <CJK>
+0x3946	0x8831	# <CJK>
+0x3947	0x9AA8	# <CJK>
+0x3948	0x8C37	# <CJK>
+0x3949	0x80A1	# <CJK>
+0x394A	0x6545	# <CJK>
+0x394B	0x9867	# <CJK>
+0x394C	0x56FA	# <CJK>
+0x394D	0x96C7	# <CJK>
+0x394E	0x522E	# <CJK>
+0x394F	0x74DC	# <CJK>
+0x3950	0x526E	# <CJK>
+0x3951	0x5BE1	# <CJK>
+0x3952	0x6302	# <CJK>
+0x3953	0x8902	# <CJK>
+0x3954	0x4E56	# <CJK>
+0x3955	0x62D0	# <CJK>
+0x3956	0x602A	# <CJK>
+0x3957	0x68FA	# <CJK>
+0x3958	0x95DC	# <CJK>
+0x3959	0x5B98	# <CJK>
+0x395A	0x51A0	# <CJK>
+0x395B	0x89C0	# <CJK>
+0x395C	0x7BA1	# <CJK>
+0x395D	0x9928	# <CJK>
+0x395E	0x7F50	# <CJK>
+0x395F	0x6163	# <CJK>
+0x3960	0x704C	# <CJK>
+0x3961	0x8CAB	# <CJK>
+0x3962	0x5149	# <CJK>
+0x3963	0x5EE3	# <CJK>
+0x3964	0x901B	# <CJK>
+0x3965	0x7470	# <CJK>
+0x3966	0x898F	# <CJK>
+0x3967	0x572D	# <CJK>
+0x3968	0x7845	# <CJK>
+0x3969	0x6B78	# <CJK>
+0x396A	0x9F9C	# <CJK>
+0x396B	0x95A8	# <CJK>
+0x396C	0x8ECC	# <CJK>
+0x396D	0x9B3C	# <CJK>
+0x396E	0x8A6D	# <CJK>
+0x396F	0x7678	# <CJK>
+0x3970	0x6842	# <CJK>
+0x3971	0x6AC3	# <CJK>
+0x3972	0x8DEA	# <CJK>
+0x3973	0x8CB4	# <CJK>
+0x3974	0x528A	# <CJK>
+0x3975	0x8F25	# <CJK>
+0x3976	0x6EDA	# <CJK>
+0x3977	0x68CD	# <CJK>
+0x3978	0x934B	# <CJK>
+0x3979	0x90ED	# <CJK>
+0x397A	0x570B	# <CJK>
+0x397B	0x679C	# <CJK>
+0x397C	0x88F9	# <CJK>
+0x397D	0x904E	# <CJK>
+0x397E	0x54C8	# <CJK>
+0x3A21	0x9AB8	# <CJK>
+0x3A22	0x5B69	# <CJK>
+0x3A23	0x6D77	# <CJK>
+0x3A24	0x6C26	# <CJK>
+0x3A25	0x4EA5	# <CJK>
+0x3A26	0x5BB3	# <CJK>
+0x3A27	0x99ED	# <CJK>
+0x3A28	0x9163	# <CJK>
+0x3A29	0x61A8	# <CJK>
+0x3A2A	0x90AF	# <CJK>
+0x3A2B	0x97D3	# <CJK>
+0x3A2C	0x542B	# <CJK>
+0x3A2D	0x6DB5	# <CJK>
+0x3A2E	0x5BD2	# <CJK>
+0x3A2F	0x51FD	# <CJK>
+0x3A30	0x558A	# <CJK>
+0x3A31	0x7F55	# <CJK>
+0x3A32	0x7FF0	# <CJK>
+0x3A33	0x64BC	# <CJK>
+0x3A34	0x634D	# <CJK>
+0x3A35	0x65F1	# <CJK>
+0x3A36	0x61BE	# <CJK>
+0x3A37	0x608D	# <CJK>
+0x3A38	0x710A	# <CJK>
+0x3A39	0x6C57	# <CJK>
+0x3A3A	0x6F22	# <CJK>
+0x3A3B	0x592F	# <CJK>
+0x3A3C	0x676D	# <CJK>
+0x3A3D	0x822A	# <CJK>
+0x3A3E	0x58D5	# <CJK>
+0x3A3F	0x568E	# <CJK>
+0x3A40	0x8C6A	# <CJK>
+0x3A41	0x6BEB	# <CJK>
+0x3A42	0x90DD	# <CJK>
+0x3A43	0x597D	# <CJK>
+0x3A44	0x8017	# <CJK>
+0x3A45	0x865F	# <CJK>
+0x3A46	0x6D69	# <CJK>
+0x3A47	0x5475	# <CJK>
+0x3A48	0x559D	# <CJK>
+0x3A49	0x8377	# <CJK>
+0x3A4A	0x83CF	# <CJK>
+0x3A4B	0x6838	# <CJK>
+0x3A4C	0x79BE	# <CJK>
+0x3A4D	0x548C	# <CJK>
+0x3A4E	0x4F55	# <CJK>
+0x3A4F	0x5408	# <CJK>
+0x3A50	0x76D2	# <CJK>
+0x3A51	0x8C89	# <CJK>
+0x3A52	0x95A1	# <CJK>
+0x3A53	0x6CB3	# <CJK>
+0x3A54	0x6DB8	# <CJK>
+0x3A55	0x8D6B	# <CJK>
+0x3A56	0x8910	# <CJK>
+0x3A57	0x9DB4	# <CJK>
+0x3A58	0x8CC0	# <CJK>
+0x3A59	0x563F	# <CJK>
+0x3A5A	0x9ED1	# <CJK>
+0x3A5B	0x75D5	# <CJK>
+0x3A5C	0x5F88	# <CJK>
+0x3A5D	0x72E0	# <CJK>
+0x3A5E	0x6068	# <CJK>
+0x3A5F	0x54FC	# <CJK>
+0x3A60	0x4EA8	# <CJK>
+0x3A61	0x6A2A	# <CJK>
+0x3A62	0x8861	# <CJK>
+0x3A63	0x6052	# <CJK>
+0x3A64	0x8F5F	# <CJK>
+0x3A65	0x54C4	# <CJK>
+0x3A66	0x70D8	# <CJK>
+0x3A67	0x8679	# <CJK>
+0x3A68	0x9D3B	# <CJK>
+0x3A69	0x6D2A	# <CJK>
+0x3A6A	0x5B8F	# <CJK>
+0x3A6B	0x5F18	# <CJK>
+0x3A6C	0x7D05	# <CJK>
+0x3A6D	0x5589	# <CJK>
+0x3A6E	0x4FAF	# <CJK>
+0x3A6F	0x7334	# <CJK>
+0x3A70	0x543C	# <CJK>
+0x3A71	0x539A	# <CJK>
+0x3A72	0x5019	# <CJK>
+0x3A73	0x5F8C	# <CJK>
+0x3A74	0x547C	# <CJK>
+0x3A75	0x4E4E	# <CJK>
+0x3A76	0x5FFD	# <CJK>
+0x3A77	0x745A	# <CJK>
+0x3A78	0x58FA	# <CJK>
+0x3A79	0x846B	# <CJK>
+0x3A7A	0x80E1	# <CJK>
+0x3A7B	0x8774	# <CJK>
+0x3A7C	0x72D0	# <CJK>
+0x3A7D	0x7CCA	# <CJK>
+0x3A7E	0x6E56	# <CJK>
+0x3B21	0x5F27	# <CJK>
+0x3B22	0x864E	# <CJK>
+0x3B23	0x552C	# <CJK>
+0x3B24	0x8B77	# <CJK>
+0x3B25	0x4E92	# <CJK>
+0x3B26	0x6EEC	# <CJK>
+0x3B27	0x6237	# <CJK>
+0x3B28	0x82B1	# <CJK>
+0x3B29	0x5629	# <CJK>
+0x3B2A	0x83EF	# <CJK>
+0x3B2B	0x733E	# <CJK>
+0x3B2C	0x6ED1	# <CJK>
+0x3B2D	0x756B	# <CJK>
+0x3B2E	0x5283	# <CJK>
+0x3B2F	0x5316	# <CJK>
+0x3B30	0x8A71	# <CJK>
+0x3B31	0x69D0	# <CJK>
+0x3B32	0x5F8A	# <CJK>
+0x3B33	0x61F7	# <CJK>
+0x3B34	0x6DEE	# <CJK>
+0x3B35	0x58DE	# <CJK>
+0x3B36	0x6B61	# <CJK>
+0x3B37	0x74B0	# <CJK>
+0x3B38	0x6853	# <CJK>
+0x3B39	0x9084	# <CJK>
+0x3B3A	0x7DE9	# <CJK>
+0x3B3B	0x63DB	# <CJK>
+0x3B3C	0x60A3	# <CJK>
+0x3B3D	0x559A	# <CJK>
+0x3B3E	0x7613	# <CJK>
+0x3B3F	0x8C62	# <CJK>
+0x3B40	0x7165	# <CJK>
+0x3B41	0x6E19	# <CJK>
+0x3B42	0x5BA6	# <CJK>
+0x3B43	0x5E7B	# <CJK>
+0x3B44	0x8352	# <CJK>
+0x3B45	0x614C	# <CJK>
+0x3B46	0x9EC4	# <CJK>
+0x3B47	0x78FA	# <CJK>
+0x3B48	0x8757	# <CJK>
+0x3B49	0x7C27	# <CJK>
+0x3B4A	0x7687	# <CJK>
+0x3B4B	0x51F0	# <CJK>
+0x3B4C	0x60F6	# <CJK>
+0x3B4D	0x714C	# <CJK>
+0x3B4E	0x6643	# <CJK>
+0x3B4F	0x5E4C	# <CJK>
+0x3B50	0x604D	# <CJK>
+0x3B51	0x8B0A	# <CJK>
+0x3B52	0x7070	# <CJK>
+0x3B53	0x63EE	# <CJK>
+0x3B54	0x8F1D	# <CJK>
+0x3B55	0x5FBD	# <CJK>
+0x3B56	0x6062	# <CJK>
+0x3B57	0x86D4	# <CJK>
+0x3B58	0x56DE	# <CJK>
+0x3B59	0x6BC1	# <CJK>
+0x3B5A	0x6094	# <CJK>
+0x3B5B	0x6167	# <CJK>
+0x3B5C	0x5349	# <CJK>
+0x3B5D	0x60E0	# <CJK>
+0x3B5E	0x6666	# <CJK>
+0x3B5F	0x8CC4	# <CJK>
+0x3B60	0x7A62	# <CJK>
+0x3B61	0x6703	# <CJK>
+0x3B62	0x71F4	# <CJK>
+0x3B63	0x532F	# <CJK>
+0x3B64	0x8AF1	# <CJK>
+0x3B65	0x8AA8	# <CJK>
+0x3B66	0x7E6A	# <CJK>
+0x3B67	0x8477	# <CJK>
+0x3B68	0x660F	# <CJK>
+0x3B69	0x5A5A	# <CJK>
+0x3B6A	0x9B42	# <CJK>
+0x3B6B	0x6E3E	# <CJK>
+0x3B6C	0x6DF7	# <CJK>
+0x3B6D	0x8C41	# <CJK>
+0x3B6E	0x6D3B	# <CJK>
+0x3B6F	0x4F19	# <CJK>
+0x3B70	0x706B	# <CJK>
+0x3B71	0x7372	# <CJK>
+0x3B72	0x6216	# <CJK>
+0x3B73	0x60D1	# <CJK>
+0x3B74	0x970D	# <CJK>
+0x3B75	0x8CA8	# <CJK>
+0x3B76	0x798D	# <CJK>
+0x3B77	0x64CA	# <CJK>
+0x3B78	0x573E	# <CJK>
+0x3B79	0x57FA	# <CJK>
+0x3B7A	0x6A5F	# <CJK>
+0x3B7B	0x7578	# <CJK>
+0x3B7C	0x7A3D	# <CJK>
+0x3B7D	0x7A4D	# <CJK>
+0x3B7E	0x7B95	# <CJK>
+0x3C21	0x808C	# <CJK>
+0x3C22	0x9951	# <CJK>
+0x3C23	0x8FF9	# <CJK>
+0x3C24	0x6FC0	# <CJK>
+0x3C25	0x8B4F	# <CJK>
+0x3C26	0x9DC4	# <CJK>
+0x3C27	0x59EC	# <CJK>
+0x3C28	0x7E3E	# <CJK>
+0x3C29	0x7DDD	# <CJK>
+0x3C2A	0x5409	# <CJK>
+0x3C2B	0x6975	# <CJK>
+0x3C2C	0x68D8	# <CJK>
+0x3C2D	0x8F2F	# <CJK>
+0x3C2E	0x7C4D	# <CJK>
+0x3C2F	0x96C6	# <CJK>
+0x3C30	0x53CA	# <CJK>
+0x3C31	0x6025	# <CJK>
+0x3C32	0x75BE	# <CJK>
+0x3C33	0x6C72	# <CJK>
+0x3C34	0x5373	# <CJK>
+0x3C35	0x5AC9	# <CJK>
+0x3C36	0x7D1A	# <CJK>
+0x3C37	0x64E0	# <CJK>
+0x3C38	0x5E7E	# <CJK>
+0x3C39	0x810A	# <CJK>
+0x3C3A	0x5DF1	# <CJK>
+0x3C3B	0x858A	# <CJK>
+0x3C3C	0x6280	# <CJK>
+0x3C3D	0x5180	# <CJK>
+0x3C3E	0x5B63	# <CJK>
+0x3C3F	0x4F0E	# <CJK>
+0x3C40	0x796D	# <CJK>
+0x3C41	0x5291	# <CJK>
+0x3C42	0x60B8	# <CJK>
+0x3C43	0x6FDF	# <CJK>
+0x3C44	0x5BC4	# <CJK>
+0x3C45	0x5BC2	# <CJK>
+0x3C46	0x8A08	# <CJK>
+0x3C47	0x8A18	# <CJK>
+0x3C48	0x65E2	# <CJK>
+0x3C49	0x5FCC	# <CJK>
+0x3C4A	0x969B	# <CJK>
+0x3C4B	0x5993	# <CJK>
+0x3C4C	0x7E7C	# <CJK>
+0x3C4D	0x7D00	# <CJK>
+0x3C4E	0x5609	# <CJK>
+0x3C4F	0x67B7	# <CJK>
+0x3C50	0x593E	# <CJK>
+0x3C51	0x4F73	# <CJK>
+0x3C52	0x5BB6	# <CJK>
+0x3C53	0x52A0	# <CJK>
+0x3C54	0x83A2	# <CJK>
+0x3C55	0x9830	# <CJK>
+0x3C56	0x8CC8	# <CJK>
+0x3C57	0x7532	# <CJK>
+0x3C58	0x9240	# <CJK>
+0x3C59	0x5047	# <CJK>
+0x3C5A	0x7A3C	# <CJK>
+0x3C5B	0x50F9	# <CJK>
+0x3C5C	0x67B6	# <CJK>
+0x3C5D	0x99D5	# <CJK>
+0x3C5E	0x5AC1	# <CJK>
+0x3C5F	0x6BB2	# <CJK>
+0x3C60	0x76E3	# <CJK>
+0x3C61	0x5805	# <CJK>
+0x3C62	0x5C16	# <CJK>
+0x3C63	0x7B8B	# <CJK>
+0x3C64	0x9593	# <CJK>
+0x3C65	0x714E	# <CJK>
+0x3C66	0x517C	# <CJK>
+0x3C67	0x80A9	# <CJK>
+0x3C68	0x8271	# <CJK>
+0x3C69	0x5978	# <CJK>
+0x3C6A	0x7DD8	# <CJK>
+0x3C6B	0x7E6D	# <CJK>
+0x3C6C	0x6AA2	# <CJK>
+0x3C6D	0x67EC	# <CJK>
+0x3C6E	0x78B1	# <CJK>
+0x3C6F	0x9E7C	# <CJK>
+0x3C70	0x63C0	# <CJK>
+0x3C71	0x64BF	# <CJK>
+0x3C72	0x7C21	# <CJK>
+0x3C73	0x5109	# <CJK>
+0x3C74	0x526A	# <CJK>
+0x3C75	0x51CF	# <CJK>
+0x3C76	0x85A6	# <CJK>
+0x3C77	0x6ABB	# <CJK>
+0x3C78	0x9452	# <CJK>
+0x3C79	0x8E10	# <CJK>
+0x3C7A	0x8CE4	# <CJK>
+0x3C7B	0x898B	# <CJK>
+0x3C7C	0x9375	# <CJK>
+0x3C7D	0x7BAD	# <CJK>
+0x3C7E	0x4EF6	# <CJK>
+0x3D21	0x5065	# <CJK>
+0x3D22	0x8266	# <CJK>
+0x3D23	0x528D	# <CJK>
+0x3D24	0x991E	# <CJK>
+0x3D25	0x6F38	# <CJK>
+0x3D26	0x6FFA	# <CJK>
+0x3D27	0x6F97	# <CJK>
+0x3D28	0x5EFA	# <CJK>
+0x3D29	0x50F5	# <CJK>
+0x3D2A	0x59DC	# <CJK>
+0x3D2B	0x5C07	# <CJK>
+0x3D2C	0x6F3F	# <CJK>
+0x3D2D	0x6C5F	# <CJK>
+0x3D2E	0x7586	# <CJK>
+0x3D2F	0x8523	# <CJK>
+0x3D30	0x69F3	# <CJK>
+0x3D31	0x596C	# <CJK>
+0x3D32	0x8B1B	# <CJK>
+0x3D33	0x5320	# <CJK>
+0x3D34	0x91AC	# <CJK>
+0x3D35	0x964D	# <CJK>
+0x3D36	0x8549	# <CJK>
+0x3D37	0x6912	# <CJK>
+0x3D38	0x7901	# <CJK>
+0x3D39	0x7126	# <CJK>
+0x3D3A	0x81A0	# <CJK>
+0x3D3B	0x4EA4	# <CJK>
+0x3D3C	0x90CA	# <CJK>
+0x3D3D	0x6F86	# <CJK>
+0x3D3E	0x9A55	# <CJK>
+0x3D3F	0x5B0C	# <CJK>
+0x3D40	0x56BC	# <CJK>
+0x3D41	0x652A	# <CJK>
+0x3D42	0x9278	# <CJK>
+0x3D43	0x77EF	# <CJK>
+0x3D44	0x50E5	# <CJK>
+0x3D45	0x811A	# <CJK>
+0x3D46	0x72E1	# <CJK>
+0x3D47	0x89D2	# <CJK>
+0x3D48	0x9903	# <CJK>
+0x3D49	0x7E73	# <CJK>
+0x3D4A	0x7D5E	# <CJK>
+0x3D4B	0x527F	# <CJK>
+0x3D4C	0x6559	# <CJK>
+0x3D4D	0x9175	# <CJK>
+0x3D4E	0x8F4E	# <CJK>
+0x3D4F	0x8F03	# <CJK>
+0x3D50	0x53EB	# <CJK>
+0x3D51	0x7A96	# <CJK>
+0x3D52	0x63ED	# <CJK>
+0x3D53	0x63A5	# <CJK>
+0x3D54	0x7686	# <CJK>
+0x3D55	0x79F8	# <CJK>
+0x3D56	0x8857	# <CJK>
+0x3D57	0x968E	# <CJK>
+0x3D58	0x622A	# <CJK>
+0x3D59	0x52AB	# <CJK>
+0x3D5A	0x7BC0	# <CJK>
+0x3D5B	0x6854	# <CJK>
+0x3D5C	0x6770	# <CJK>
+0x3D5D	0x6377	# <CJK>
+0x3D5E	0x776B	# <CJK>
+0x3D5F	0x7AED	# <CJK>
+0x3D60	0x6F54	# <CJK>
+0x3D61	0x7D50	# <CJK>
+0x3D62	0x89E3	# <CJK>
+0x3D63	0x59D0	# <CJK>
+0x3D64	0x6212	# <CJK>
+0x3D65	0x85C9	# <CJK>
+0x3D66	0x82A5	# <CJK>
+0x3D67	0x754C	# <CJK>
+0x3D68	0x501F	# <CJK>
+0x3D69	0x4ECB	# <CJK>
+0x3D6A	0x75A5	# <CJK>
+0x3D6B	0x8AA1	# <CJK>
+0x3D6C	0x5C4A	# <CJK>
+0x3D6D	0x5DFE	# <CJK>
+0x3D6E	0x7B4B	# <CJK>
+0x3D6F	0x65A4	# <CJK>
+0x3D70	0x91D1	# <CJK>
+0x3D71	0x4ECA	# <CJK>
+0x3D72	0x6D25	# <CJK>
+0x3D73	0x895F	# <CJK>
+0x3D74	0x7DCA	# <CJK>
+0x3D75	0x9326	# <CJK>
+0x3D76	0x50C5	# <CJK>
+0x3D77	0x8B39	# <CJK>
+0x3D78	0x9032	# <CJK>
+0x3D79	0x9773	# <CJK>
+0x3D7A	0x6649	# <CJK>
+0x3D7B	0x7981	# <CJK>
+0x3D7C	0x8FD1	# <CJK>
+0x3D7D	0x71FC	# <CJK>
+0x3D7E	0x6D78	# <CJK>
+0x3E21	0x76E1	# <CJK>
+0x3E22	0x52C1	# <CJK>
+0x3E23	0x8346	# <CJK>
+0x3E24	0x5162	# <CJK>
+0x3E25	0x8396	# <CJK>
+0x3E26	0x775B	# <CJK>
+0x3E27	0x6676	# <CJK>
+0x3E28	0x9BE8	# <CJK>
+0x3E29	0x4EAC	# <CJK>
+0x3E2A	0x9A5A	# <CJK>
+0x3E2B	0x7CBE	# <CJK>
+0x3E2C	0x7CB3	# <CJK>
+0x3E2D	0x7D93	# <CJK>
+0x3E2E	0x4E95	# <CJK>
+0x3E2F	0x8B66	# <CJK>
+0x3E30	0x666F	# <CJK>
+0x3E31	0x9838	# <CJK>
+0x3E32	0x975C	# <CJK>
+0x3E33	0x5883	# <CJK>
+0x3E34	0x656C	# <CJK>
+0x3E35	0x93E1	# <CJK>
+0x3E36	0x5F91	# <CJK>
+0x3E37	0x75D9	# <CJK>
+0x3E38	0x9756	# <CJK>
+0x3E39	0x7ADF	# <CJK>
+0x3E3A	0x7AF6	# <CJK>
+0x3E3B	0x51C8	# <CJK>
+0x3E3C	0x70AF	# <CJK>
+0x3E3D	0x7A98	# <CJK>
+0x3E3E	0x63EA	# <CJK>
+0x3E3F	0x7A76	# <CJK>
+0x3E40	0x7CFE	# <CJK>
+0x3E41	0x7396	# <CJK>
+0x3E42	0x97ED	# <CJK>
+0x3E43	0x4E45	# <CJK>
+0x3E44	0x7078	# <CJK>
+0x3E45	0x4E5D	# <CJK>
+0x3E46	0x9152	# <CJK>
+0x3E47	0x53A9	# <CJK>
+0x3E48	0x6551	# <CJK>
+0x3E49	0x820A	# <CJK>
+0x3E4A	0x81FC	# <CJK>
+0x3E4B	0x8205	# <CJK>
+0x3E4C	0x548E	# <CJK>
+0x3E4D	0x5C31	# <CJK>
+0x3E4E	0x759A	# <CJK>
+0x3E4F	0x97A0	# <CJK>
+0x3E50	0x62D8	# <CJK>
+0x3E51	0x72D9	# <CJK>
+0x3E52	0x75BD	# <CJK>
+0x3E53	0x5C45	# <CJK>
+0x3E54	0x99D2	# <CJK>
+0x3E55	0x83CA	# <CJK>
+0x3E56	0x5C40	# <CJK>
+0x3E57	0x5480	# <CJK>
+0x3E58	0x77E9	# <CJK>
+0x3E59	0x8209	# <CJK>
+0x3E5A	0x6CAE	# <CJK>
+0x3E5B	0x805A	# <CJK>
+0x3E5C	0x62D2	# <CJK>
+0x3E5D	0x64DA	# <CJK>
+0x3E5E	0x5DE8	# <CJK>
+0x3E5F	0x5177	# <CJK>
+0x3E60	0x8DDD	# <CJK>
+0x3E61	0x8E1E	# <CJK>
+0x3E62	0x92F8	# <CJK>
+0x3E63	0x4FF1	# <CJK>
+0x3E64	0x53E5	# <CJK>
+0x3E65	0x61FC	# <CJK>
+0x3E66	0x70AC	# <CJK>
+0x3E67	0x5287	# <CJK>
+0x3E68	0x6350	# <CJK>
+0x3E69	0x9D51	# <CJK>
+0x3E6A	0x5A1F	# <CJK>
+0x3E6B	0x5026	# <CJK>
+0x3E6C	0x7737	# <CJK>
+0x3E6D	0x5377	# <CJK>
+0x3E6E	0x7D79	# <CJK>
+0x3E6F	0x6485	# <CJK>
+0x3E70	0x652B	# <CJK>
+0x3E71	0x6289	# <CJK>
+0x3E72	0x6398	# <CJK>
+0x3E73	0x5014	# <CJK>
+0x3E74	0x7235	# <CJK>
+0x3E75	0x89BA	# <CJK>
+0x3E76	0x51B3	# <CJK>
+0x3E77	0x8A23	# <CJK>
+0x3E78	0x7D76	# <CJK>
+0x3E79	0x5747	# <CJK>
+0x3E7A	0x83CC	# <CJK>
+0x3E7B	0x921E	# <CJK>
+0x3E7C	0x8ECD	# <CJK>
+0x3E7D	0x541B	# <CJK>
+0x3E7E	0x5CFB	# <CJK>
+0x3F21	0x4FCA	# <CJK>
+0x3F22	0x7AE3	# <CJK>
+0x3F23	0x6D5A	# <CJK>
+0x3F24	0x90E1	# <CJK>
+0x3F25	0x99FF	# <CJK>
+0x3F26	0x5580	# <CJK>
+0x3F27	0x5496	# <CJK>
+0x3F28	0x5361	# <CJK>
+0x3F29	0x54AF	# <CJK>
+0x3F2A	0x958B	# <CJK>
+0x3F2B	0x63E9	# <CJK>
+0x3F2C	0x6977	# <CJK>
+0x3F2D	0x51F1	# <CJK>
+0x3F2E	0x6168	# <CJK>
+0x3F2F	0x520A	# <CJK>
+0x3F30	0x582A	# <CJK>
+0x3F31	0x52D8	# <CJK>
+0x3F32	0x574E	# <CJK>
+0x3F33	0x780D	# <CJK>
+0x3F34	0x770B	# <CJK>
+0x3F35	0x5EB7	# <CJK>
+0x3F36	0x6177	# <CJK>
+0x3F37	0x7CE0	# <CJK>
+0x3F38	0x625B	# <CJK>
+0x3F39	0x6297	# <CJK>
+0x3F3A	0x4EA2	# <CJK>
+0x3F3B	0x7095	# <CJK>
+0x3F3C	0x8003	# <CJK>
+0x3F3D	0x62F7	# <CJK>
+0x3F3E	0x70E4	# <CJK>
+0x3F3F	0x9760	# <CJK>
+0x3F40	0x5777	# <CJK>
+0x3F41	0x82DB	# <CJK>
+0x3F42	0x67EF	# <CJK>
+0x3F43	0x68F5	# <CJK>
+0x3F44	0x78D5	# <CJK>
+0x3F45	0x9846	# <CJK>
+0x3F46	0x79D1	# <CJK>
+0x3F47	0x6BBB	# <CJK>
+0x3F48	0x54B3	# <CJK>
+0x3F49	0x53EF	# <CJK>
+0x3F4A	0x6E34	# <CJK>
+0x3F4B	0x514B	# <CJK>
+0x3F4C	0x523B	# <CJK>
+0x3F4D	0x5BA2	# <CJK>
+0x3F4E	0x8AB2	# <CJK>
+0x3F4F	0x80AF	# <CJK>
+0x3F50	0x5543	# <CJK>
+0x3F51	0x58BE	# <CJK>
+0x3F52	0x61C7	# <CJK>
+0x3F53	0x5751	# <CJK>
+0x3F54	0x542D	# <CJK>
+0x3F55	0x7A7A	# <CJK>
+0x3F56	0x6050	# <CJK>
+0x3F57	0x5B54	# <CJK>
+0x3F58	0x63A7	# <CJK>
+0x3F59	0x6473	# <CJK>
+0x3F5A	0x53E3	# <CJK>
+0x3F5B	0x6263	# <CJK>
+0x3F5C	0x5BC7	# <CJK>
+0x3F5D	0x67AF	# <CJK>
+0x3F5E	0x54ED	# <CJK>
+0x3F5F	0x7A9F	# <CJK>
+0x3F60	0x82E6	# <CJK>
+0x3F61	0x9177	# <CJK>
+0x3F62	0x5EAB	# <CJK>
+0x3F63	0x8932	# <CJK>
+0x3F64	0x8A87	# <CJK>
+0x3F65	0x57AE	# <CJK>
+0x3F66	0x630E	# <CJK>
+0x3F67	0x8DE8	# <CJK>
+0x3F68	0x80EF	# <CJK>
+0x3F69	0x584A	# <CJK>
+0x3F6A	0x7B77	# <CJK>
+0x3F6B	0x5108	# <CJK>
+0x3F6C	0x5FEB	# <CJK>
+0x3F6D	0x5BEC	# <CJK>
+0x3F6E	0x6B3E	# <CJK>
+0x3F6F	0x5321	# <CJK>
+0x3F70	0x7B50	# <CJK>
+0x3F71	0x72C2	# <CJK>
+0x3F72	0x6846	# <CJK>
+0x3F73	0x7926	# <CJK>
+0x3F74	0x7736	# <CJK>
+0x3F75	0x66E0	# <CJK>
+0x3F76	0x51B5	# <CJK>
+0x3F77	0x8667	# <CJK>
+0x3F78	0x76D4	# <CJK>
+0x3F79	0x5DCB	# <CJK>
+0x3F7A	0x7ABA	# <CJK>
+0x3F7B	0x8475	# <CJK>
+0x3F7C	0x594E	# <CJK>
+0x3F7D	0x9B41	# <CJK>
+0x3F7E	0x5080	# <CJK>
+0x4021	0x994B	# <CJK>
+0x4022	0x6127	# <CJK>
+0x4023	0x6F70	# <CJK>
+0x4024	0x5764	# <CJK>
+0x4025	0x6606	# <CJK>
+0x4026	0x6346	# <CJK>
+0x4027	0x56F0	# <CJK>
+0x4028	0x62EC	# <CJK>
+0x4029	0x64F4	# <CJK>
+0x402A	0x5ED3	# <CJK>
+0x402B	0x95CA	# <CJK>
+0x402C	0x5783	# <CJK>
+0x402D	0x62C9	# <CJK>
+0x402E	0x5587	# <CJK>
+0x402F	0x881F	# <CJK>
+0x4030	0x81D8	# <CJK>
+0x4031	0x8FA3	# <CJK>
+0x4032	0x5566	# <CJK>
+0x4033	0x840A	# <CJK>
+0x4034	0x4F86	# <CJK>
+0x4035	0x8CF4	# <CJK>
+0x4036	0x85CD	# <CJK>
+0x4037	0x5A6A	# <CJK>
+0x4038	0x6B04	# <CJK>
+0x4039	0x6514	# <CJK>
+0x403A	0x7C43	# <CJK>
+0x403B	0x95CC	# <CJK>
+0x403C	0x862D	# <CJK>
+0x403D	0x703E	# <CJK>
+0x403E	0x8B95	# <CJK>
+0x403F	0x652C	# <CJK>
+0x4040	0x89BD	# <CJK>
+0x4041	0x61F6	# <CJK>
+0x4042	0x7E9C	# <CJK>
+0x4043	0x721B	# <CJK>
+0x4044	0x6FEB	# <CJK>
+0x4045	0x7405	# <CJK>
+0x4046	0x6994	# <CJK>
+0x4047	0x72FC	# <CJK>
+0x4048	0x5ECA	# <CJK>
+0x4049	0x90CE	# <CJK>
+0x404A	0x6717	# <CJK>
+0x404B	0x6D6A	# <CJK>
+0x404C	0x6488	# <CJK>
+0x404D	0x52DE	# <CJK>
+0x404E	0x7262	# <CJK>
+0x404F	0x8001	# <CJK>
+0x4050	0x4F6C	# <CJK>
+0x4051	0x59E5	# <CJK>
+0x4052	0x916A	# <CJK>
+0x4053	0x70D9	# <CJK>
+0x4054	0x6F87	# <CJK>
+0x4055	0x52D2	# <CJK>
+0x4056	0x6A02	# <CJK>
+0x4057	0x96F7	# <CJK>
+0x4058	0x9433	# <CJK>
+0x4059	0x857E	# <CJK>
+0x405A	0x78CA	# <CJK>
+0x405B	0x7D2F	# <CJK>
+0x405C	0x5121	# <CJK>
+0x405D	0x58D8	# <CJK>
+0x405E	0x64C2	# <CJK>
+0x405F	0x808B	# <CJK>
+0x4060	0x985E	# <CJK>
+0x4061	0x6CEA	# <CJK>
+0x4062	0x68F1	# <CJK>
+0x4063	0x695E	# <CJK>
+0x4064	0x51B7	# <CJK>
+0x4065	0x5398	# <CJK>
+0x4066	0x68A8	# <CJK>
+0x4067	0x7281	# <CJK>
+0x4068	0x9ECE	# <CJK>
+0x4069	0x7C6C	# <CJK>
+0x406A	0x72F8	# <CJK>
+0x406B	0x96E2	# <CJK>
+0x406C	0x7055	# <CJK>
+0x406D	0x7406	# <CJK>
+0x406E	0x674E	# <CJK>
+0x406F	0x88CF	# <CJK>
+0x4070	0x9BC9	# <CJK>
+0x4071	0x79AE	# <CJK>
+0x4072	0x8389	# <CJK>
+0x4073	0x8354	# <CJK>
+0x4074	0x540F	# <CJK>
+0x4075	0x6817	# <CJK>
+0x4076	0x9E97	# <CJK>
+0x4077	0x53B2	# <CJK>
+0x4078	0x52F5	# <CJK>
+0x4079	0x792B	# <CJK>
+0x407A	0x6B77	# <CJK>
+0x407B	0x5229	# <CJK>
+0x407C	0x5088	# <CJK>
+0x407D	0x4F8B	# <CJK>
+0x407E	0x4FD0	# <CJK>
+0x4121	0x75E2	# <CJK>
+0x4122	0x7ACB	# <CJK>
+0x4123	0x7C92	# <CJK>
+0x4124	0x701D	# <CJK>
+0x4125	0x96B8	# <CJK>
+0x4126	0x529B	# <CJK>
+0x4127	0x7483	# <CJK>
+0x4128	0x54E9	# <CJK>
+0x4129	0x5006	# <CJK>
+0x412A	0x806F	# <CJK>
+0x412B	0x84EE	# <CJK>
+0x412C	0x9023	# <CJK>
+0x412D	0x942E	# <CJK>
+0x412E	0x5EC9	# <CJK>
+0x412F	0x6190	# <CJK>
+0x4130	0x6F23	# <CJK>
+0x4131	0x7C3E	# <CJK>
+0x4132	0x6582	# <CJK>
+0x4133	0x81C9	# <CJK>
+0x4134	0x93C8	# <CJK>
+0x4135	0x6200	# <CJK>
+0x4136	0x7149	# <CJK>
+0x4137	0x7DF4	# <CJK>
+0x4138	0x7CE7	# <CJK>
+0x4139	0x51C9	# <CJK>
+0x413A	0x6881	# <CJK>
+0x413B	0x7CB1	# <CJK>
+0x413C	0x826F	# <CJK>
+0x413D	0x5169	# <CJK>
+0x413E	0x8F1B	# <CJK>
+0x413F	0x91CF	# <CJK>
+0x4140	0x667E	# <CJK>
+0x4141	0x4EAE	# <CJK>
+0x4142	0x8AD2	# <CJK>
+0x4143	0x64A9	# <CJK>
+0x4144	0x804A	# <CJK>
+0x4145	0x50DA	# <CJK>
+0x4146	0x7642	# <CJK>
+0x4147	0x71CE	# <CJK>
+0x4148	0x5BE5	# <CJK>
+0x4149	0x907C	# <CJK>
+0x414A	0x6F66	# <CJK>
+0x414B	0x4E86	# <CJK>
+0x414C	0x6482	# <CJK>
+0x414D	0x9410	# <CJK>
+0x414E	0x5ED6	# <CJK>
+0x414F	0x6599	# <CJK>
+0x4150	0x5217	# <CJK>
+0x4151	0x88C2	# <CJK>
+0x4152	0x70C8	# <CJK>
+0x4153	0x52A3	# <CJK>
+0x4154	0x7375	# <CJK>
+0x4155	0x7433	# <CJK>
+0x4156	0x6797	# <CJK>
+0x4157	0x78F7	# <CJK>
+0x4158	0x9716	# <CJK>
+0x4159	0x81E8	# <CJK>
+0x415A	0x9130	# <CJK>
+0x415B	0x9C57	# <CJK>
+0x415C	0x6DCB	# <CJK>
+0x415D	0x51DB	# <CJK>
+0x415E	0x8CC3	# <CJK>
+0x415F	0x541D	# <CJK>
+0x4160	0x62CE	# <CJK>
+0x4161	0x73B2	# <CJK>
+0x4162	0x83F1	# <CJK>
+0x4163	0x96F6	# <CJK>
+0x4164	0x9F61	# <CJK>
+0x4165	0x9234	# <CJK>
+0x4166	0x4F36	# <CJK>
+0x4167	0x7F9A	# <CJK>
+0x4168	0x51CC	# <CJK>
+0x4169	0x9748	# <CJK>
+0x416A	0x9675	# <CJK>
+0x416B	0x5DBA	# <CJK>
+0x416C	0x9818	# <CJK>
+0x416D	0x53E6	# <CJK>
+0x416E	0x4EE4	# <CJK>
+0x416F	0x6E9C	# <CJK>
+0x4170	0x7409	# <CJK>
+0x4171	0x69B4	# <CJK>
+0x4172	0x786B	# <CJK>
+0x4173	0x993E	# <CJK>
+0x4174	0x7559	# <CJK>
+0x4175	0x5289	# <CJK>
+0x4176	0x7624	# <CJK>
+0x4177	0x6D41	# <CJK>
+0x4178	0x67F3	# <CJK>
+0x4179	0x516D	# <CJK>
+0x417A	0x9F8D	# <CJK>
+0x417B	0x807E	# <CJK>
+0x417C	0x56A8	# <CJK>
+0x417D	0x7C60	# <CJK>
+0x417E	0x7ABF	# <CJK>
+0x4221	0x9686	# <CJK>
+0x4222	0x58DF	# <CJK>
+0x4223	0x650F	# <CJK>
+0x4224	0x96B4	# <CJK>
+0x4225	0x6A13	# <CJK>
+0x4226	0x5A41	# <CJK>
+0x4227	0x645F	# <CJK>
+0x4228	0x7C0D	# <CJK>
+0x4229	0x6F0F	# <CJK>
+0x422A	0x964B	# <CJK>
+0x422B	0x8606	# <CJK>
+0x422C	0x76E7	# <CJK>
+0x422D	0x9871	# <CJK>
+0x422E	0x5EEC	# <CJK>
+0x422F	0x7210	# <CJK>
+0x4230	0x64C4	# <CJK>
+0x4231	0x6EF7	# <CJK>
+0x4232	0x865C	# <CJK>
+0x4233	0x9B6F	# <CJK>
+0x4234	0x9E93	# <CJK>
+0x4235	0x788C	# <CJK>
+0x4236	0x9732	# <CJK>
+0x4237	0x8DEF	# <CJK>
+0x4238	0x8CC2	# <CJK>
+0x4239	0x9E7F	# <CJK>
+0x423A	0x6F5E	# <CJK>
+0x423B	0x7984	# <CJK>
+0x423C	0x9332	# <CJK>
+0x423D	0x9678	# <CJK>
+0x423E	0x622E	# <CJK>
+0x423F	0x9A62	# <CJK>
+0x4240	0x5415	# <CJK>
+0x4241	0x92C1	# <CJK>
+0x4242	0x4FA3	# <CJK>
+0x4243	0x65C5	# <CJK>
+0x4244	0x5C65	# <CJK>
+0x4245	0x5C62	# <CJK>
+0x4246	0x7E37	# <CJK>
+0x4247	0x616E	# <CJK>
+0x4248	0x6C2F	# <CJK>
+0x4249	0x5F8B	# <CJK>
+0x424A	0x7387	# <CJK>
+0x424B	0x6FFE	# <CJK>
+0x424C	0x7DD1	# <CJK>
+0x424D	0x5DD2	# <CJK>
+0x424E	0x6523	# <CJK>
+0x424F	0x5B7F	# <CJK>
+0x4250	0x7064	# <CJK>
+0x4251	0x5375	# <CJK>
+0x4252	0x4E82	# <CJK>
+0x4253	0x63A0	# <CJK>
+0x4254	0x7565	# <CJK>
+0x4255	0x6384	# <CJK>
+0x4256	0x8F2A	# <CJK>
+0x4257	0x502B	# <CJK>
+0x4258	0x4F96	# <CJK>
+0x4259	0x6DEA	# <CJK>
+0x425A	0x7DB8	# <CJK>
+0x425B	0x8AD6	# <CJK>
+0x425C	0x863F	# <CJK>
+0x425D	0x87BA	# <CJK>
+0x425E	0x7F85	# <CJK>
+0x425F	0x908F	# <CJK>
+0x4260	0x947C	# <CJK>
+0x4261	0x7C6E	# <CJK>
+0x4262	0x9A3E	# <CJK>
+0x4263	0x88F8	# <CJK>
+0x4264	0x843D	# <CJK>
+0x4265	0x6D1B	# <CJK>
+0x4266	0x99F1	# <CJK>
+0x4267	0x7D61	# <CJK>
+0x4268	0x5ABD	# <CJK>
+0x4269	0x9EBB	# <CJK>
+0x426A	0x746A	# <CJK>
+0x426B	0x78BC	# <CJK>
+0x426C	0x879E	# <CJK>
+0x426D	0x99AC	# <CJK>
+0x426E	0x99E1	# <CJK>
+0x426F	0x561B	# <CJK>
+0x4270	0x55CE	# <CJK>
+0x4271	0x57CB	# <CJK>
+0x4272	0x8CB7	# <CJK>
+0x4273	0x9EA5	# <CJK>
+0x4274	0x8CE3	# <CJK>
+0x4275	0x9081	# <CJK>
+0x4276	0x8109	# <CJK>
+0x4277	0x779E	# <CJK>
+0x4278	0x9945	# <CJK>
+0x4279	0x883B	# <CJK>
+0x427A	0x6EFF	# <CJK>
+0x427B	0x8513	# <CJK>
+0x427C	0x66FC	# <CJK>
+0x427D	0x6162	# <CJK>
+0x427E	0x6F2B	# <CJK>
+0x4321	0x8B3E	# <CJK>
+0x4322	0x8292	# <CJK>
+0x4323	0x832B	# <CJK>
+0x4324	0x76F2	# <CJK>
+0x4325	0x6C13	# <CJK>
+0x4326	0x5FD9	# <CJK>
+0x4327	0x83BD	# <CJK>
+0x4328	0x732B	# <CJK>
+0x4329	0x8305	# <CJK>
+0x432A	0x9328	# <CJK>
+0x432B	0x6BDB	# <CJK>
+0x432C	0x77DB	# <CJK>
+0x432D	0x925A	# <CJK>
+0x432E	0x536F	# <CJK>
+0x432F	0x8302	# <CJK>
+0x4330	0x5192	# <CJK>
+0x4331	0x5E3D	# <CJK>
+0x4332	0x8C8C	# <CJK>
+0x4333	0x8CBF	# <CJK>
+0x4334	0x9EBD	# <CJK>
+0x4335	0x73AB	# <CJK>
+0x4336	0x679A	# <CJK>
+0x4337	0x6885	# <CJK>
+0x4338	0x9176	# <CJK>
+0x4339	0x9709	# <CJK>
+0x433A	0x7164	# <CJK>
+0x433B	0x6CA1	# <CJK>
+0x433C	0x7709	# <CJK>
+0x433D	0x5A92	# <CJK>
+0x433E	0x9382	# <CJK>
+0x433F	0x6BCF	# <CJK>
+0x4340	0x7F8E	# <CJK>
+0x4341	0x6627	# <CJK>
+0x4342	0x5BD0	# <CJK>
+0x4343	0x59B9	# <CJK>
+0x4344	0x5A9A	# <CJK>
+0x4345	0x9580	# <CJK>
+0x4346	0x60B6	# <CJK>
+0x4347	0x5011	# <CJK>
+0x4348	0x840C	# <CJK>
+0x4349	0x8499	# <CJK>
+0x434A	0x6AAC	# <CJK>
+0x434B	0x76DF	# <CJK>
+0x434C	0x9333	# <CJK>
+0x434D	0x731B	# <CJK>
+0x434E	0x5922	# <CJK>
+0x434F	0x5B5F	# <CJK>
+0x4350	0x772F	# <CJK>
+0x4351	0x919A	# <CJK>
+0x4352	0x9761	# <CJK>
+0x4353	0x7CDC	# <CJK>
+0x4354	0x8FF7	# <CJK>
+0x4355	0x8B0E	# <CJK>
+0x4356	0x5F4C	# <CJK>
+0x4357	0x7C73	# <CJK>
+0x4358	0x79D8	# <CJK>
+0x4359	0x8993	# <CJK>
+0x435A	0x6CCC	# <CJK>
+0x435B	0x871C	# <CJK>
+0x435C	0x5BC6	# <CJK>
+0x435D	0x5E42	# <CJK>
+0x435E	0x68C9	# <CJK>
+0x435F	0x7720	# <CJK>
+0x4360	0x7DBF	# <CJK>
+0x4361	0x5195	# <CJK>
+0x4362	0x514D	# <CJK>
+0x4363	0x52C9	# <CJK>
+0x4364	0x5A29	# <CJK>
+0x4365	0x7DEC	# <CJK>
+0x4366	0x9762	# <CJK>
+0x4367	0x82D7	# <CJK>
+0x4368	0x63CF	# <CJK>
+0x4369	0x7784	# <CJK>
+0x436A	0x85D0	# <CJK>
+0x436B	0x79D2	# <CJK>
+0x436C	0x6E3A	# <CJK>
+0x436D	0x5EDF	# <CJK>
+0x436E	0x5999	# <CJK>
+0x436F	0x8511	# <CJK>
+0x4370	0x6EC5	# <CJK>
+0x4371	0x6C11	# <CJK>
+0x4372	0x62BF	# <CJK>
+0x4373	0x76BF	# <CJK>
+0x4374	0x654F	# <CJK>
+0x4375	0x61AB	# <CJK>
+0x4376	0x95A9	# <CJK>
+0x4377	0x660E	# <CJK>
+0x4378	0x879F	# <CJK>
+0x4379	0x9CF4	# <CJK>
+0x437A	0x9298	# <CJK>
+0x437B	0x540D	# <CJK>
+0x437C	0x547D	# <CJK>
+0x437D	0x8B2C	# <CJK>
+0x437E	0x6478	# <CJK>
+0x4421	0x6479	# <CJK>
+0x4422	0x8611	# <CJK>
+0x4423	0x6A21	# <CJK>
+0x4424	0x819C	# <CJK>
+0x4425	0x78E8	# <CJK>
+0x4426	0x6469	# <CJK>
+0x4427	0x9B54	# <CJK>
+0x4428	0x62B9	# <CJK>
+0x4429	0x672B	# <CJK>
+0x442A	0x83AB	# <CJK>
+0x442B	0x58A8	# <CJK>
+0x442C	0x9ED8	# <CJK>
+0x442D	0x6CAB	# <CJK>
+0x442E	0x6F20	# <CJK>
+0x442F	0x5BDE	# <CJK>
+0x4430	0x964C	# <CJK>
+0x4431	0x8B00	# <CJK>
+0x4432	0x725F	# <CJK>
+0x4433	0x67D0	# <CJK>
+0x4434	0x62C7	# <CJK>
+0x4435	0x7261	# <CJK>
+0x4436	0x755D	# <CJK>
+0x4437	0x59C6	# <CJK>
+0x4438	0x6BCD	# <CJK>
+0x4439	0x5893	# <CJK>
+0x443A	0x66AE	# <CJK>
+0x443B	0x5E55	# <CJK>
+0x443C	0x52DF	# <CJK>
+0x443D	0x6155	# <CJK>
+0x443E	0x6728	# <CJK>
+0x443F	0x76EE	# <CJK>
+0x4440	0x7766	# <CJK>
+0x4441	0x7267	# <CJK>
+0x4442	0x7A46	# <CJK>
+0x4443	0x62FF	# <CJK>
+0x4444	0x54EA	# <CJK>
+0x4445	0x5450	# <CJK>
+0x4446	0x9209	# <CJK>
+0x4447	0x90A3	# <CJK>
+0x4448	0x5A1C	# <CJK>
+0x4449	0x7D0D	# <CJK>
+0x444A	0x6C16	# <CJK>
+0x444B	0x4E43	# <CJK>
+0x444C	0x5976	# <CJK>
+0x444D	0x8010	# <CJK>
+0x444E	0x5948	# <CJK>
+0x444F	0x5357	# <CJK>
+0x4450	0x7537	# <CJK>
+0x4451	0x96E3	# <CJK>
+0x4452	0x56CA	# <CJK>
+0x4453	0x6493	# <CJK>
+0x4454	0x8166	# <CJK>
+0x4455	0x60F1	# <CJK>
+0x4456	0x9B27	# <CJK>
+0x4457	0x6DD6	# <CJK>
+0x4458	0x5462	# <CJK>
+0x4459	0x9912	# <CJK>
+0x445A	0x5185	# <CJK>
+0x445B	0x5AE9	# <CJK>
+0x445C	0x80FD	# <CJK>
+0x445D	0x59AE	# <CJK>
+0x445E	0x9713	# <CJK>
+0x445F	0x502A	# <CJK>
+0x4460	0x6CE5	# <CJK>
+0x4461	0x5C3C	# <CJK>
+0x4462	0x64EC	# <CJK>
+0x4463	0x4F60	# <CJK>
+0x4464	0x533F	# <CJK>
+0x4465	0x81A9	# <CJK>
+0x4466	0x9006	# <CJK>
+0x4467	0x6EBA	# <CJK>
+0x4468	0x852B	# <CJK>
+0x4469	0x62C8	# <CJK>
+0x446A	0x5E74	# <CJK>
+0x446B	0x78BE	# <CJK>
+0x446C	0x6506	# <CJK>
+0x446D	0x637B	# <CJK>
+0x446E	0x5FF5	# <CJK>
+0x446F	0x5A18	# <CJK>
+0x4470	0x91C0	# <CJK>
+0x4471	0x9CE5	# <CJK>
+0x4472	0x5C3F	# <CJK>
+0x4473	0x634F	# <CJK>
+0x4474	0x8076	# <CJK>
+0x4475	0x5B7D	# <CJK>
+0x4476	0x5699	# <CJK>
+0x4477	0x9477	# <CJK>
+0x4478	0x93B3	# <CJK>
+0x4479	0x6D85	# <CJK>
+0x447A	0x60A8	# <CJK>
+0x447B	0x6AB8	# <CJK>
+0x447C	0x7370	# <CJK>
+0x447D	0x51DD	# <CJK>
+0x447E	0x5BE7	# <CJK>
+0x4521	0x64F0	# <CJK>
+0x4522	0x6FD8	# <CJK>
+0x4523	0x725B	# <CJK>
+0x4524	0x626D	# <CJK>
+0x4525	0x9215	# <CJK>
+0x4526	0x7D10	# <CJK>
+0x4527	0x81BF	# <CJK>
+0x4528	0x6FC3	# <CJK>
+0x4529	0x8FB2	# <CJK>
+0x452A	0x5F04	# <CJK>
+0x452B	0x5974	# <CJK>
+0x452C	0x52AA	# <CJK>
+0x452D	0x6012	# <CJK>
+0x452E	0x5973	# <CJK>
+0x452F	0x6696	# <CJK>
+0x4530	0x8650	# <CJK>
+0x4531	0x7627	# <CJK>
+0x4532	0x632A	# <CJK>
+0x4533	0x61E6	# <CJK>
+0x4534	0x7CEF	# <CJK>
+0x4535	0x8AFE	# <CJK>
+0x4536	0x54E6	# <CJK>
+0x4537	0x6B50	# <CJK>
+0x4538	0x9DD7	# <CJK>
+0x4539	0x6BC6	# <CJK>
+0x453A	0x85D5	# <CJK>
+0x453B	0x5614	# <CJK>
+0x453C	0x5076	# <CJK>
+0x453D	0x6F1A	# <CJK>
+0x453E	0x556A	# <CJK>
+0x453F	0x8DB4	# <CJK>
+0x4540	0x722C	# <CJK>
+0x4541	0x5E15	# <CJK>
+0x4542	0x6015	# <CJK>
+0x4543	0x7436	# <CJK>
+0x4544	0x62CD	# <CJK>
+0x4545	0x6392	# <CJK>
+0x4546	0x724C	# <CJK>
+0x4547	0x5F98	# <CJK>
+0x4548	0x6E43	# <CJK>
+0x4549	0x6D3E	# <CJK>
+0x454A	0x6500	# <CJK>
+0x454B	0x6F58	# <CJK>
+0x454C	0x76E4	# <CJK>
+0x454D	0x78D0	# <CJK>
+0x454E	0x76FC	# <CJK>
+0x454F	0x7554	# <CJK>
+0x4550	0x5224	# <CJK>
+0x4551	0x53DB	# <CJK>
+0x4552	0x4E53	# <CJK>
+0x4553	0x9F90	# <CJK>
+0x4554	0x65C1	# <CJK>
+0x4555	0x802A	# <CJK>
+0x4556	0x80D6	# <CJK>
+0x4557	0x629B	# <CJK>
+0x4558	0x5486	# <CJK>
+0x4559	0x5228	# <CJK>
+0x455A	0x70AE	# <CJK>
+0x455B	0x888D	# <CJK>
+0x455C	0x8DD1	# <CJK>
+0x455D	0x6CE1	# <CJK>
+0x455E	0x5478	# <CJK>
+0x455F	0x80DA	# <CJK>
+0x4560	0x57F9	# <CJK>
+0x4561	0x88F4	# <CJK>
+0x4562	0x8CE0	# <CJK>
+0x4563	0x966A	# <CJK>
+0x4564	0x914D	# <CJK>
+0x4565	0x4F69	# <CJK>
+0x4566	0x6C9B	# <CJK>
+0x4567	0x5674	# <CJK>
+0x4568	0x76C6	# <CJK>
+0x4569	0x7830	# <CJK>
+0x456A	0x62A8	# <CJK>
+0x456B	0x70F9	# <CJK>
+0x456C	0x6F8E	# <CJK>
+0x456D	0x5F6D	# <CJK>
+0x456E	0x84EC	# <CJK>
+0x456F	0x68DA	# <CJK>
+0x4570	0x787C	# <CJK>
+0x4571	0x7BF7	# <CJK>
+0x4572	0x81A8	# <CJK>
+0x4573	0x670B	# <CJK>
+0x4574	0x9D6C	# <CJK>
+0x4575	0x6367	# <CJK>
+0x4576	0x78B0	# <CJK>
+0x4577	0x576F	# <CJK>
+0x4578	0x7812	# <CJK>
+0x4579	0x9739	# <CJK>
+0x457A	0x6279	# <CJK>
+0x457B	0x62AB	# <CJK>
+0x457C	0x5288	# <CJK>
+0x457D	0x7435	# <CJK>
+0x457E	0x6BD7	# <CJK>
+0x4621	0x5564	# <CJK>
+0x4622	0x813E	# <CJK>
+0x4623	0x75B2	# <CJK>
+0x4624	0x76AE	# <CJK>
+0x4625	0x5339	# <CJK>
+0x4626	0x75DE	# <CJK>
+0x4627	0x50FB	# <CJK>
+0x4628	0x5C41	# <CJK>
+0x4629	0x8B6C	# <CJK>
+0x462A	0x7BC7	# <CJK>
+0x462B	0x504F	# <CJK>
+0x462C	0x7247	# <CJK>
+0x462D	0x9A19	# <CJK>
+0x462E	0x98C4	# <CJK>
+0x462F	0x6F02	# <CJK>
+0x4630	0x74E2	# <CJK>
+0x4631	0x7968	# <CJK>
+0x4632	0x6487	# <CJK>
+0x4633	0x77A5	# <CJK>
+0x4634	0x62FC	# <CJK>
+0x4635	0x983B	# <CJK>
+0x4636	0x8CA7	# <CJK>
+0x4637	0x54C1	# <CJK>
+0x4638	0x8058	# <CJK>
+0x4639	0x4E52	# <CJK>
+0x463A	0x576A	# <CJK>
+0x463B	0x860B	# <CJK>
+0x463C	0x840D	# <CJK>
+0x463D	0x5E73	# <CJK>
+0x463E	0x6191	# <CJK>
+0x463F	0x74F6	# <CJK>
+0x4640	0x8A55	# <CJK>
+0x4641	0x5C4F	# <CJK>
+0x4642	0x5761	# <CJK>
+0x4643	0x6F51	# <CJK>
+0x4644	0x9817	# <CJK>
+0x4645	0x5A46	# <CJK>
+0x4646	0x7834	# <CJK>
+0x4647	0x9B44	# <CJK>
+0x4648	0x8FEB	# <CJK>
+0x4649	0x7C95	# <CJK>
+0x464A	0x5256	# <CJK>
+0x464B	0x64B2	# <CJK>
+0x464C	0x92EA	# <CJK>
+0x464D	0x50D5	# <CJK>
+0x464E	0x8386	# <CJK>
+0x464F	0x8461	# <CJK>
+0x4650	0x83E9	# <CJK>
+0x4651	0x84B2	# <CJK>
+0x4652	0x57D4	# <CJK>
+0x4653	0x6A38	# <CJK>
+0x4654	0x5703	# <CJK>
+0x4655	0x666E	# <CJK>
+0x4656	0x6D66	# <CJK>
+0x4657	0x8B5C	# <CJK>
+0x4658	0x66DD	# <CJK>
+0x4659	0x7011	# <CJK>
+0x465A	0x671F	# <CJK>
+0x465B	0x6B3A	# <CJK>
+0x465C	0x68F2	# <CJK>
+0x465D	0x621A	# <CJK>
+0x465E	0x59BB	# <CJK>
+0x465F	0x4E03	# <CJK>
+0x4660	0x51C4	# <CJK>
+0x4661	0x6F06	# <CJK>
+0x4662	0x67D2	# <CJK>
+0x4663	0x6C8F	# <CJK>
+0x4664	0x5176	# <CJK>
+0x4665	0x68CB	# <CJK>
+0x4666	0x5947	# <CJK>
+0x4667	0x6B67	# <CJK>
+0x4668	0x7566	# <CJK>
+0x4669	0x5D0E	# <CJK>
+0x466A	0x81CD	# <CJK>
+0x466B	0x9F4A	# <CJK>
+0x466C	0x65D7	# <CJK>
+0x466D	0x7948	# <CJK>
+0x466E	0x7941	# <CJK>
+0x466F	0x9A0E	# <CJK>
+0x4670	0x8D77	# <CJK>
+0x4671	0x8C48	# <CJK>
+0x4672	0x4E5E	# <CJK>
+0x4673	0x4F01	# <CJK>
+0x4674	0x5553	# <CJK>
+0x4675	0x5951	# <CJK>
+0x4676	0x780C	# <CJK>
+0x4677	0x5668	# <CJK>
+0x4678	0x6C23	# <CJK>
+0x4679	0x8FC4	# <CJK>
+0x467A	0x68C4	# <CJK>
+0x467B	0x6C7D	# <CJK>
+0x467C	0x6CE3	# <CJK>
+0x467D	0x8A16	# <CJK>
+0x467E	0x6390	# <CJK>
+0x4721	0x6070	# <CJK>
+0x4722	0x6D3D	# <CJK>
+0x4723	0x727D	# <CJK>
+0x4724	0x6266	# <CJK>
+0x4725	0x91FA	# <CJK>
+0x4726	0x925B	# <CJK>
+0x4727	0x5343	# <CJK>
+0x4728	0x9077	# <CJK>
+0x4729	0x7C3D	# <CJK>
+0x472A	0x4EDF	# <CJK>
+0x472B	0x8B19	# <CJK>
+0x472C	0x4E7E	# <CJK>
+0x472D	0x9ED4	# <CJK>
+0x472E	0x9322	# <CJK>
+0x472F	0x9257	# <CJK>
+0x4730	0x524D	# <CJK>
+0x4731	0x6F5B	# <CJK>
+0x4732	0x9063	# <CJK>
+0x4733	0x6DFA	# <CJK>
+0x4734	0x8B74	# <CJK>
+0x4735	0x5879	# <CJK>
+0x4736	0x5D4C	# <CJK>
+0x4737	0x6B20	# <CJK>
+0x4738	0x6B49	# <CJK>
+0x4739	0x69CD	# <CJK>
+0x473A	0x55C6	# <CJK>
+0x473B	0x8154	# <CJK>
+0x473C	0x7F8C	# <CJK>
+0x473D	0x58BB	# <CJK>
+0x473E	0x8594	# <CJK>
+0x473F	0x5F3A	# <CJK>
+0x4740	0x6436	# <CJK>
+0x4741	0x6A47	# <CJK>
+0x4742	0x936C	# <CJK>
+0x4743	0x6572	# <CJK>
+0x4744	0x6084	# <CJK>
+0x4745	0x6A4B	# <CJK>
+0x4746	0x77A7	# <CJK>
+0x4747	0x55AC	# <CJK>
+0x4748	0x50D1	# <CJK>
+0x4749	0x5DE7	# <CJK>
+0x474A	0x9798	# <CJK>
+0x474B	0x64AC	# <CJK>
+0x474C	0x7FF9	# <CJK>
+0x474D	0x5CED	# <CJK>
+0x474E	0x4FCF	# <CJK>
+0x474F	0x7AC5	# <CJK>
+0x4750	0x5207	# <CJK>
+0x4751	0x8304	# <CJK>
+0x4752	0x4E14	# <CJK>
+0x4753	0x602F	# <CJK>
+0x4754	0x7ACA	# <CJK>
+0x4755	0x6B3D	# <CJK>
+0x4756	0x4FB5	# <CJK>
+0x4757	0x89AA	# <CJK>
+0x4758	0x79E6	# <CJK>
+0x4759	0x7434	# <CJK>
+0x475A	0x52E4	# <CJK>
+0x475B	0x82B9	# <CJK>
+0x475C	0x64D2	# <CJK>
+0x475D	0x79BD	# <CJK>
+0x475E	0x5BE2	# <CJK>
+0x475F	0x6C81	# <CJK>
+0x4760	0x9752	# <CJK>
+0x4761	0x8F15	# <CJK>
+0x4762	0x6C2B	# <CJK>
+0x4763	0x50BE	# <CJK>
+0x4764	0x537F	# <CJK>
+0x4765	0x6E05	# <CJK>
+0x4766	0x64CE	# <CJK>
+0x4767	0x6674	# <CJK>
+0x4768	0x6C30	# <CJK>
+0x4769	0x60C5	# <CJK>
+0x476A	0x9803	# <CJK>
+0x476B	0x8ACB	# <CJK>
+0x476C	0x6176	# <CJK>
+0x476D	0x74CA	# <CJK>
+0x476E	0x7AAE	# <CJK>
+0x476F	0x79CB	# <CJK>
+0x4770	0x4E18	# <CJK>
+0x4771	0x90B1	# <CJK>
+0x4772	0x7403	# <CJK>
+0x4773	0x6C42	# <CJK>
+0x4774	0x56DA	# <CJK>
+0x4775	0x914B	# <CJK>
+0x4776	0x6CC5	# <CJK>
+0x4777	0x8DA8	# <CJK>
+0x4778	0x5340	# <CJK>
+0x4779	0x86C6	# <CJK>
+0x477A	0x66F2	# <CJK>
+0x477B	0x8EC0	# <CJK>
+0x477C	0x5C48	# <CJK>
+0x477D	0x9A45	# <CJK>
+0x477E	0x6E20	# <CJK>
+0x4821	0x53D6	# <CJK>
+0x4822	0x5A36	# <CJK>
+0x4823	0x9F72	# <CJK>
+0x4824	0x8DA3	# <CJK>
+0x4825	0x53BB	# <CJK>
+0x4826	0x5708	# <CJK>
+0x4827	0x9874	# <CJK>
+0x4828	0x6B0A	# <CJK>
+0x4829	0x919B	# <CJK>
+0x482A	0x6CC9	# <CJK>
+0x482B	0x5168	# <CJK>
+0x482C	0x75CA	# <CJK>
+0x482D	0x62F3	# <CJK>
+0x482E	0x72AC	# <CJK>
+0x482F	0x5238	# <CJK>
+0x4830	0x52F8	# <CJK>
+0x4831	0x7F3A	# <CJK>
+0x4832	0x7094	# <CJK>
+0x4833	0x7638	# <CJK>
+0x4834	0x5374	# <CJK>
+0x4835	0x9D72	# <CJK>
+0x4836	0x69B7	# <CJK>
+0x4837	0x78BA	# <CJK>
+0x4838	0x96C0	# <CJK>
+0x4839	0x88D9	# <CJK>
+0x483A	0x7FA4	# <CJK>
+0x483B	0x7136	# <CJK>
+0x483C	0x71C3	# <CJK>
+0x483D	0x5189	# <CJK>
+0x483E	0x67D3	# <CJK>
+0x483F	0x74E4	# <CJK>
+0x4840	0x58E4	# <CJK>
+0x4841	0x6518	# <CJK>
+0x4842	0x56B7	# <CJK>
+0x4843	0x8B93	# <CJK>
+0x4844	0x9952	# <CJK>
+0x4845	0x64FE	# <CJK>
+0x4846	0x7E5E	# <CJK>
+0x4847	0x60F9	# <CJK>
+0x4848	0x71B1	# <CJK>
+0x4849	0x58EC	# <CJK>
+0x484A	0x4EC1	# <CJK>
+0x484B	0x4EBA	# <CJK>
+0x484C	0x5FCD	# <CJK>
+0x484D	0x97CC	# <CJK>
+0x484E	0x4EFB	# <CJK>
+0x484F	0x8A8D	# <CJK>
+0x4850	0x5203	# <CJK>
+0x4851	0x598A	# <CJK>
+0x4852	0x7D09	# <CJK>
+0x4853	0x6254	# <CJK>
+0x4854	0x4ECD	# <CJK>
+0x4855	0x65E5	# <CJK>
+0x4856	0x620E	# <CJK>
+0x4857	0x8338	# <CJK>
+0x4858	0x84C9	# <CJK>
+0x4859	0x69AE	# <CJK>
+0x485A	0x878D	# <CJK>
+0x485B	0x7194	# <CJK>
+0x485C	0x6EB6	# <CJK>
+0x485D	0x5BB9	# <CJK>
+0x485E	0x7D68	# <CJK>
+0x485F	0x5197	# <CJK>
+0x4860	0x63C9	# <CJK>
+0x4861	0x67D4	# <CJK>
+0x4862	0x8089	# <CJK>
+0x4863	0x8339	# <CJK>
+0x4864	0x8815	# <CJK>
+0x4865	0x5112	# <CJK>
+0x4866	0x5B7A	# <CJK>
+0x4867	0x5982	# <CJK>
+0x4868	0x8FB1	# <CJK>
+0x4869	0x4E73	# <CJK>
+0x486A	0x6C5D	# <CJK>
+0x486B	0x5165	# <CJK>
+0x486C	0x8925	# <CJK>
+0x486D	0x8EDF	# <CJK>
+0x486E	0x962E	# <CJK>
+0x486F	0x854A	# <CJK>
+0x4870	0x745E	# <CJK>
+0x4871	0x92ED	# <CJK>
+0x4872	0x958F	# <CJK>
+0x4873	0x6F64	# <CJK>
+0x4874	0x82E5	# <CJK>
+0x4875	0x5F31	# <CJK>
+0x4876	0x6492	# <CJK>
+0x4877	0x7051	# <CJK>
+0x4878	0x85A9	# <CJK>
+0x4879	0x816E	# <CJK>
+0x487A	0x9C13	# <CJK>
+0x487B	0x585E	# <CJK>
+0x487C	0x8CFD	# <CJK>
+0x487D	0x4E09	# <CJK>
+0x487E	0x53C1	# <CJK>
+0x4921	0x5098	# <CJK>
+0x4922	0x6563	# <CJK>
+0x4923	0x6851	# <CJK>
+0x4924	0x55D3	# <CJK>
+0x4925	0x55AA	# <CJK>
+0x4926	0x6414	# <CJK>
+0x4927	0x9A37	# <CJK>
+0x4928	0x6383	# <CJK>
+0x4929	0x5AC2	# <CJK>
+0x492A	0x745F	# <CJK>
+0x492B	0x8272	# <CJK>
+0x492C	0x6F80	# <CJK>
+0x492D	0x68EE	# <CJK>
+0x492E	0x50E7	# <CJK>
+0x492F	0x838E	# <CJK>
+0x4930	0x7802	# <CJK>
+0x4931	0x6BBA	# <CJK>
+0x4932	0x5239	# <CJK>
+0x4933	0x6C99	# <CJK>
+0x4934	0x7D17	# <CJK>
+0x4935	0x50BB	# <CJK>
+0x4936	0x5565	# <CJK>
+0x4937	0x715E	# <CJK>
+0x4938	0x7BE9	# <CJK>
+0x4939	0x66EC	# <CJK>
+0x493A	0x73CA	# <CJK>
+0x493B	0x82EB	# <CJK>
+0x493C	0x6749	# <CJK>
+0x493D	0x5C71	# <CJK>
+0x493E	0x5220	# <CJK>
+0x493F	0x717D	# <CJK>
+0x4940	0x886B	# <CJK>
+0x4941	0x9583	# <CJK>
+0x4942	0x965D	# <CJK>
+0x4943	0x64C5	# <CJK>
+0x4944	0x8D0D	# <CJK>
+0x4945	0x81B3	# <CJK>
+0x4946	0x5584	# <CJK>
+0x4947	0x6C55	# <CJK>
+0x4948	0x6247	# <CJK>
+0x4949	0x7E55	# <CJK>
+0x494A	0x5892	# <CJK>
+0x494B	0x50B7	# <CJK>
+0x494C	0x5546	# <CJK>
+0x494D	0x8CDE	# <CJK>
+0x494E	0x664C	# <CJK>
+0x494F	0x4E0A	# <CJK>
+0x4950	0x5C1A	# <CJK>
+0x4951	0x88F3	# <CJK>
+0x4952	0x68A2	# <CJK>
+0x4953	0x634E	# <CJK>
+0x4954	0x7A0D	# <CJK>
+0x4955	0x71D2	# <CJK>
+0x4956	0x828D	# <CJK>
+0x4957	0x52FA	# <CJK>
+0x4958	0x97F6	# <CJK>
+0x4959	0x5C11	# <CJK>
+0x495A	0x54E8	# <CJK>
+0x495B	0x90B5	# <CJK>
+0x495C	0x7D39	# <CJK>
+0x495D	0x5962	# <CJK>
+0x495E	0x8CD2	# <CJK>
+0x495F	0x86C7	# <CJK>
+0x4960	0x820C	# <CJK>
+0x4961	0x6368	# <CJK>
+0x4962	0x8D66	# <CJK>
+0x4963	0x651D	# <CJK>
+0x4964	0x5C04	# <CJK>
+0x4965	0x61FE	# <CJK>
+0x4966	0x6D89	# <CJK>
+0x4967	0x793E	# <CJK>
+0x4968	0x8A2D	# <CJK>
+0x4969	0x7837	# <CJK>
+0x496A	0x7533	# <CJK>
+0x496B	0x547B	# <CJK>
+0x496C	0x4F38	# <CJK>
+0x496D	0x8EAB	# <CJK>
+0x496E	0x6DF1	# <CJK>
+0x496F	0x5A20	# <CJK>
+0x4970	0x7D33	# <CJK>
+0x4971	0x795E	# <CJK>
+0x4972	0x6C88	# <CJK>
+0x4973	0x5BE9	# <CJK>
+0x4974	0x5B38	# <CJK>
+0x4975	0x751A	# <CJK>
+0x4976	0x814E	# <CJK>
+0x4977	0x614E	# <CJK>
+0x4978	0x6EF2	# <CJK>
+0x4979	0x8072	# <CJK>
+0x497A	0x751F	# <CJK>
+0x497B	0x7525	# <CJK>
+0x497C	0x7272	# <CJK>
+0x497D	0x5347	# <CJK>
+0x497E	0x7E69	# <CJK>
+0x4A21	0x7701	# <CJK>
+0x4A22	0x76DB	# <CJK>
+0x4A23	0x5269	# <CJK>
+0x4A24	0x52DD	# <CJK>
+0x4A25	0x8056	# <CJK>
+0x4A26	0x5E2B	# <CJK>
+0x4A27	0x5931	# <CJK>
+0x4A28	0x7345	# <CJK>
+0x4A29	0x65BD	# <CJK>
+0x4A2A	0x6FD5	# <CJK>
+0x4A2B	0x8A69	# <CJK>
+0x4A2C	0x5C38	# <CJK>
+0x4A2D	0x8671	# <CJK>
+0x4A2E	0x5341	# <CJK>
+0x4A2F	0x77F3	# <CJK>
+0x4A30	0x62FE	# <CJK>
+0x4A31	0x6642	# <CJK>
+0x4A32	0x4EC0	# <CJK>
+0x4A33	0x98DF	# <CJK>
+0x4A34	0x8755	# <CJK>
+0x4A35	0x5BE6	# <CJK>
+0x4A36	0x8B58	# <CJK>
+0x4A37	0x53F2	# <CJK>
+0x4A38	0x77E2	# <CJK>
+0x4A39	0x4F7F	# <CJK>
+0x4A3A	0x5C4E	# <CJK>
+0x4A3B	0x99DB	# <CJK>
+0x4A3C	0x59CB	# <CJK>
+0x4A3D	0x5F0F	# <CJK>
+0x4A3E	0x793A	# <CJK>
+0x4A3F	0x58EB	# <CJK>
+0x4A40	0x4E16	# <CJK>
+0x4A41	0x67FF	# <CJK>
+0x4A42	0x4E8B	# <CJK>
+0x4A43	0x62ED	# <CJK>
+0x4A44	0x8A93	# <CJK>
+0x4A45	0x901D	# <CJK>
+0x4A46	0x52E2	# <CJK>
+0x4A47	0x662F	# <CJK>
+0x4A48	0x55DC	# <CJK>
+0x4A49	0x566C	# <CJK>
+0x4A4A	0x9069	# <CJK>
+0x4A4B	0x4ED5	# <CJK>
+0x4A4C	0x4F8D	# <CJK>
+0x4A4D	0x91CB	# <CJK>
+0x4A4E	0x98FE	# <CJK>
+0x4A4F	0x6C0F	# <CJK>
+0x4A50	0x5E02	# <CJK>
+0x4A51	0x6043	# <CJK>
+0x4A52	0x5BA4	# <CJK>
+0x4A53	0x8996	# <CJK>
+0x4A54	0x8A66	# <CJK>
+0x4A55	0x6536	# <CJK>
+0x4A56	0x624B	# <CJK>
+0x4A57	0x9996	# <CJK>
+0x4A58	0x5B88	# <CJK>
+0x4A59	0x58FD	# <CJK>
+0x4A5A	0x6388	# <CJK>
+0x4A5B	0x552E	# <CJK>
+0x4A5C	0x53D7	# <CJK>
+0x4A5D	0x7626	# <CJK>
+0x4A5E	0x7378	# <CJK>
+0x4A5F	0x852C	# <CJK>
+0x4A60	0x6A1E	# <CJK>
+0x4A61	0x68B3	# <CJK>
+0x4A62	0x6B8A	# <CJK>
+0x4A63	0x6292	# <CJK>
+0x4A64	0x8F38	# <CJK>
+0x4A65	0x53D4	# <CJK>
+0x4A66	0x8212	# <CJK>
+0x4A67	0x6DD1	# <CJK>
+0x4A68	0x758F	# <CJK>
+0x4A69	0x66F8	# <CJK>
+0x4A6A	0x8D16	# <CJK>
+0x4A6B	0x5B70	# <CJK>
+0x4A6C	0x719F	# <CJK>
+0x4A6D	0x85AF	# <CJK>
+0x4A6E	0x6691	# <CJK>
+0x4A6F	0x66D9	# <CJK>
+0x4A70	0x7F72	# <CJK>
+0x4A71	0x8700	# <CJK>
+0x4A72	0x9ECD	# <CJK>
+0x4A73	0x9F20	# <CJK>
+0x4A74	0x5C6C	# <CJK>
+0x4A75	0x8853	# <CJK>
+0x4A76	0x8FF0	# <CJK>
+0x4A77	0x6A39	# <CJK>
+0x4A78	0x675F	# <CJK>
+0x4A79	0x620D	# <CJK>
+0x4A7A	0x7AEA	# <CJK>
+0x4A7B	0x5885	# <CJK>
+0x4A7C	0x5EB6	# <CJK>
+0x4A7D	0x6578	# <CJK>
+0x4A7E	0x6F31	# <CJK>
+0x4B21	0x6055	# <CJK>
+0x4B22	0x5237	# <CJK>
+0x4B23	0x800D	# <CJK>
+0x4B24	0x6454	# <CJK>
+0x4B25	0x8870	# <CJK>
+0x4B26	0x7529	# <CJK>
+0x4B27	0x5E25	# <CJK>
+0x4B28	0x6813	# <CJK>
+0x4B29	0x62F4	# <CJK>
+0x4B2A	0x971C	# <CJK>
+0x4B2B	0x96D9	# <CJK>
+0x4B2C	0x723D	# <CJK>
+0x4B2D	0x8AB0	# <CJK>
+0x4B2E	0x6C34	# <CJK>
+0x4B2F	0x7761	# <CJK>
+0x4B30	0x7A0E	# <CJK>
+0x4B31	0x542E	# <CJK>
+0x4B32	0x77AC	# <CJK>
+0x4B33	0x9806	# <CJK>
+0x4B34	0x821C	# <CJK>
+0x4B35	0x8AAC	# <CJK>
+0x4B36	0x78A9	# <CJK>
+0x4B37	0x6714	# <CJK>
+0x4B38	0x720D	# <CJK>
+0x4B39	0x65AF	# <CJK>
+0x4B3A	0x6495	# <CJK>
+0x4B3B	0x5636	# <CJK>
+0x4B3C	0x601D	# <CJK>
+0x4B3D	0x79C1	# <CJK>
+0x4B3E	0x53F8	# <CJK>
+0x4B3F	0x7D72	# <CJK>
+0x4B40	0x6B7B	# <CJK>
+0x4B41	0x8086	# <CJK>
+0x4B42	0x5BFA	# <CJK>
+0x4B43	0x55E3	# <CJK>
+0x4B44	0x56DB	# <CJK>
+0x4B45	0x4F3A	# <CJK>
+0x4B46	0x4F3C	# <CJK>
+0x4B47	0x98FC	# <CJK>
+0x4B48	0x5DF3	# <CJK>
+0x4B49	0x9B06	# <CJK>
+0x4B4A	0x8073	# <CJK>
+0x4B4B	0x616B	# <CJK>
+0x4B4C	0x980C	# <CJK>
+0x4B4D	0x9001	# <CJK>
+0x4B4E	0x5B8B	# <CJK>
+0x4B4F	0x8A1F	# <CJK>
+0x4B50	0x8AA6	# <CJK>
+0x4B51	0x641C	# <CJK>
+0x4B52	0x8258	# <CJK>
+0x4B53	0x64FB	# <CJK>
+0x4B54	0x55FD	# <CJK>
+0x4B55	0x8607	# <CJK>
+0x4B56	0x9165	# <CJK>
+0x4B57	0x4FD7	# <CJK>
+0x4B58	0x7D20	# <CJK>
+0x4B59	0x901F	# <CJK>
+0x4B5A	0x7C9F	# <CJK>
+0x4B5B	0x50F3	# <CJK>
+0x4B5C	0x5851	# <CJK>
+0x4B5D	0x6EAF	# <CJK>
+0x4B5E	0x5BBF	# <CJK>
+0x4B5F	0x8A34	# <CJK>
+0x4B60	0x8085	# <CJK>
+0x4B61	0x9178	# <CJK>
+0x4B62	0x849C	# <CJK>
+0x4B63	0x7B97	# <CJK>
+0x4B64	0x96D6	# <CJK>
+0x4B65	0x968B	# <CJK>
+0x4B66	0x96A8	# <CJK>
+0x4B67	0x7D8F	# <CJK>
+0x4B68	0x9AD3	# <CJK>
+0x4B69	0x788E	# <CJK>
+0x4B6A	0x6B72	# <CJK>
+0x4B6B	0x7A57	# <CJK>
+0x4B6C	0x9042	# <CJK>
+0x4B6D	0x96A7	# <CJK>
+0x4B6E	0x795F	# <CJK>
+0x4B6F	0x5B6B	# <CJK>
+0x4B70	0x640D	# <CJK>
+0x4B71	0x7B0B	# <CJK>
+0x4B72	0x84D1	# <CJK>
+0x4B73	0x68AD	# <CJK>
+0x4B74	0x5506	# <CJK>
+0x4B75	0x7E2E	# <CJK>
+0x4B76	0x7463	# <CJK>
+0x4B77	0x7D22	# <CJK>
+0x4B78	0x9396	# <CJK>
+0x4B79	0x6240	# <CJK>
+0x4B7A	0x584C	# <CJK>
+0x4B7B	0x4ED6	# <CJK>
+0x4B7C	0x5B83	# <CJK>
+0x4B7D	0x5979	# <CJK>
+0x4B7E	0x5854	# <CJK>
+0x4C21	0x737A	# <CJK>
+0x4C22	0x64BB	# <CJK>
+0x4C23	0x8E4B	# <CJK>
+0x4C24	0x8E0F	# <CJK>
+0x4C25	0x80CE	# <CJK>
+0x4C26	0x82D4	# <CJK>
+0x4C27	0x62AC	# <CJK>
+0x4C28	0x81FA	# <CJK>
+0x4C29	0x6CF0	# <CJK>
+0x4C2A	0x915E	# <CJK>
+0x4C2B	0x592A	# <CJK>
+0x4C2C	0x614B	# <CJK>
+0x4C2D	0x6C70	# <CJK>
+0x4C2E	0x574D	# <CJK>
+0x4C2F	0x6524	# <CJK>
+0x4C30	0x8CAA	# <CJK>
+0x4C31	0x7671	# <CJK>
+0x4C32	0x7058	# <CJK>
+0x4C33	0x58C7	# <CJK>
+0x4C34	0x6A80	# <CJK>
+0x4C35	0x75F0	# <CJK>
+0x4C36	0x6F6D	# <CJK>
+0x4C37	0x8B5A	# <CJK>
+0x4C38	0x8AC7	# <CJK>
+0x4C39	0x5766	# <CJK>
+0x4C3A	0x6BEF	# <CJK>
+0x4C3B	0x8892	# <CJK>
+0x4C3C	0x78B3	# <CJK>
+0x4C3D	0x63A2	# <CJK>
+0x4C3E	0x5606	# <CJK>
+0x4C3F	0x70AD	# <CJK>
+0x4C40	0x6E6F	# <CJK>
+0x4C41	0x5858	# <CJK>
+0x4C42	0x642A	# <CJK>
+0x4C43	0x5802	# <CJK>
+0x4C44	0x68E0	# <CJK>
+0x4C45	0x819B	# <CJK>
+0x4C46	0x5510	# <CJK>
+0x4C47	0x7CD6	# <CJK>
+0x4C48	0x5018	# <CJK>
+0x4C49	0x8EBA	# <CJK>
+0x4C4A	0x6DCC	# <CJK>
+0x4C4B	0x8D9F	# <CJK>
+0x4C4C	0x71D9	# <CJK>
+0x4C4D	0x638F	# <CJK>
+0x4C4E	0x6FE4	# <CJK>
+0x4C4F	0x6ED4	# <CJK>
+0x4C50	0x7E27	# <CJK>
+0x4C51	0x8404	# <CJK>
+0x4C52	0x6843	# <CJK>
+0x4C53	0x9003	# <CJK>
+0x4C54	0x6DD8	# <CJK>
+0x4C55	0x9676	# <CJK>
+0x4C56	0x8A0E	# <CJK>
+0x4C57	0x5957	# <CJK>
+0x4C58	0x7279	# <CJK>
+0x4C59	0x85E4	# <CJK>
+0x4C5A	0x9A30	# <CJK>
+0x4C5B	0x75BC	# <CJK>
+0x4C5C	0x8B04	# <CJK>
+0x4C5D	0x68AF	# <CJK>
+0x4C5E	0x5254	# <CJK>
+0x4C5F	0x8E22	# <CJK>
+0x4C60	0x92BB	# <CJK>
+0x4C61	0x63D0	# <CJK>
+0x4C62	0x984C	# <CJK>
+0x4C63	0x8E44	# <CJK>
+0x4C64	0x557C	# <CJK>
+0x4C65	0x9AD4	# <CJK>
+0x4C66	0x66FF	# <CJK>
+0x4C67	0x568F	# <CJK>
+0x4C68	0x60D5	# <CJK>
+0x4C69	0x6D95	# <CJK>
+0x4C6A	0x5243	# <CJK>
+0x4C6B	0x5C49	# <CJK>
+0x4C6C	0x5929	# <CJK>
+0x4C6D	0x6DFB	# <CJK>
+0x4C6E	0x586B	# <CJK>
+0x4C6F	0x7530	# <CJK>
+0x4C70	0x751C	# <CJK>
+0x4C71	0x606C	# <CJK>
+0x4C72	0x8214	# <CJK>
+0x4C73	0x8146	# <CJK>
+0x4C74	0x6311	# <CJK>
+0x4C75	0x689D	# <CJK>
+0x4C76	0x8FE2	# <CJK>
+0x4C77	0x773A	# <CJK>
+0x4C78	0x8DF3	# <CJK>
+0x4C79	0x8CBC	# <CJK>
+0x4C7A	0x9435	# <CJK>
+0x4C7B	0x5E16	# <CJK>
+0x4C7C	0x5EF3	# <CJK>
+0x4C7D	0x807D	# <CJK>
+0x4C7E	0x70F4	# <CJK>
+0x4D21	0x6C40	# <CJK>
+0x4D22	0x5EF7	# <CJK>
+0x4D23	0x505C	# <CJK>
+0x4D24	0x4EAD	# <CJK>
+0x4D25	0x5EAD	# <CJK>
+0x4D26	0x633A	# <CJK>
+0x4D27	0x8247	# <CJK>
+0x4D28	0x901A	# <CJK>
+0x4D29	0x6850	# <CJK>
+0x4D2A	0x916E	# <CJK>
+0x4D2B	0x77B3	# <CJK>
+0x4D2C	0x540C	# <CJK>
+0x4D2D	0x9285	# <CJK>
+0x4D2E	0x5F64	# <CJK>
+0x4D2F	0x7AE5	# <CJK>
+0x4D30	0x6876	# <CJK>
+0x4D31	0x6345	# <CJK>
+0x4D32	0x7B52	# <CJK>
+0x4D33	0x7D71	# <CJK>
+0x4D34	0x75DB	# <CJK>
+0x4D35	0x5077	# <CJK>
+0x4D36	0x6295	# <CJK>
+0x4D37	0x982D	# <CJK>
+0x4D38	0x900F	# <CJK>
+0x4D39	0x51F8	# <CJK>
+0x4D3A	0x79C3	# <CJK>
+0x4D3B	0x7A81	# <CJK>
+0x4D3C	0x5716	# <CJK>
+0x4D3D	0x5F92	# <CJK>
+0x4D3E	0x9014	# <CJK>
+0x4D3F	0x5857	# <CJK>
+0x4D40	0x5C60	# <CJK>
+0x4D41	0x571F	# <CJK>
+0x4D42	0x5410	# <CJK>
+0x4D43	0x5154	# <CJK>
+0x4D44	0x6E4D	# <CJK>
+0x4D45	0x5718	# <CJK>
+0x4D46	0x63A8	# <CJK>
+0x4D47	0x983D	# <CJK>
+0x4D48	0x817F	# <CJK>
+0x4D49	0x8715	# <CJK>
+0x4D4A	0x892A	# <CJK>
+0x4D4B	0x9000	# <CJK>
+0x4D4C	0x541E	# <CJK>
+0x4D4D	0x5C6F	# <CJK>
+0x4D4E	0x81C0	# <CJK>
+0x4D4F	0x62D6	# <CJK>
+0x4D50	0x6258	# <CJK>
+0x4D51	0x8131	# <CJK>
+0x4D52	0x9D15	# <CJK>
+0x4D53	0x9640	# <CJK>
+0x4D54	0x99B1	# <CJK>
+0x4D55	0x99DD	# <CJK>
+0x4D56	0x6A62	# <CJK>
+0x4D57	0x59A5	# <CJK>
+0x4D58	0x62D3	# <CJK>
+0x4D59	0x553E	# <CJK>
+0x4D5A	0x6316	# <CJK>
+0x4D5B	0x54C7	# <CJK>
+0x4D5C	0x86D9	# <CJK>
+0x4D5D	0x7AAA	# <CJK>
+0x4D5E	0x5A03	# <CJK>
+0x4D5F	0x74E6	# <CJK>
+0x4D60	0x896A	# <CJK>
+0x4D61	0x6B6A	# <CJK>
+0x4D62	0x5916	# <CJK>
+0x4D63	0x8C4C	# <CJK>
+0x4D64	0x5F4E	# <CJK>
+0x4D65	0x7063	# <CJK>
+0x4D66	0x73A9	# <CJK>
+0x4D67	0x9811	# <CJK>
+0x4D68	0x4E38	# <CJK>
+0x4D69	0x70F7	# <CJK>
+0x4D6A	0x5B8C	# <CJK>
+0x4D6B	0x7897	# <CJK>
+0x4D6C	0x633D	# <CJK>
+0x4D6D	0x665A	# <CJK>
+0x4D6E	0x7696	# <CJK>
+0x4D6F	0x60CB	# <CJK>
+0x4D70	0x5B9B	# <CJK>
+0x4D71	0x5A49	# <CJK>
+0x4D72	0x842C	# <CJK>
+0x4D73	0x8155	# <CJK>
+0x4D74	0x6C6A	# <CJK>
+0x4D75	0x738B	# <CJK>
+0x4D76	0x4EA1	# <CJK>
+0x4D77	0x6789	# <CJK>
+0x4D78	0x7DB2	# <CJK>
+0x4D79	0x5F80	# <CJK>
+0x4D7A	0x65FA	# <CJK>
+0x4D7B	0x671B	# <CJK>
+0x4D7C	0x5FD8	# <CJK>
+0x4D7D	0x5984	# <CJK>
+0x4D7E	0x5A01	# <CJK>
+0x4E21	0x5DCD	# <CJK>
+0x4E22	0x5FAE	# <CJK>
+0x4E23	0x5371	# <CJK>
+0x4E24	0x97CB	# <CJK>
+0x4E25	0x9055	# <CJK>
+0x4E26	0x6845	# <CJK>
+0x4E27	0x570D	# <CJK>
+0x4E28	0x552F	# <CJK>
+0x4E29	0x60DF	# <CJK>
+0x4E2A	0x7232	# <CJK>
+0x4E2B	0x6FF0	# <CJK>
+0x4E2C	0x7DAD	# <CJK>
+0x4E2D	0x8466	# <CJK>
+0x4E2E	0x840E	# <CJK>
+0x4E2F	0x59D4	# <CJK>
+0x4E30	0x5049	# <CJK>
+0x4E31	0x50DE	# <CJK>
+0x4E32	0x5C3E	# <CJK>
+0x4E33	0x7DEF	# <CJK>
+0x4E34	0x672A	# <CJK>
+0x4E35	0x851A	# <CJK>
+0x4E36	0x5473	# <CJK>
+0x4E37	0x754F	# <CJK>
+0x4E38	0x80C3	# <CJK>
+0x4E39	0x5582	# <CJK>
+0x4E3A	0x9B4F	# <CJK>
+0x4E3B	0x4F4D	# <CJK>
+0x4E3C	0x6E2D	# <CJK>
+0x4E3D	0x8B02	# <CJK>
+0x4E3E	0x5C09	# <CJK>
+0x4E3F	0x6170	# <CJK>
+0x4E40	0x885B	# <CJK>
+0x4E41	0x761F	# <CJK>
+0x4E42	0x6E29	# <CJK>
+0x4E43	0x868A	# <CJK>
+0x4E44	0x6587	# <CJK>
+0x4E45	0x805E	# <CJK>
+0x4E46	0x7D0B	# <CJK>
+0x4E47	0x543B	# <CJK>
+0x4E48	0x7A69	# <CJK>
+0x4E49	0x7D0A	# <CJK>
+0x4E4A	0x554F	# <CJK>
+0x4E4B	0x55E1	# <CJK>
+0x4E4C	0x7FC1	# <CJK>
+0x4E4D	0x74EE	# <CJK>
+0x4E4E	0x64BE	# <CJK>
+0x4E4F	0x8778	# <CJK>
+0x4E50	0x6E26	# <CJK>
+0x4E51	0x7AA9	# <CJK>
+0x4E52	0x6211	# <CJK>
+0x4E53	0x65A1	# <CJK>
+0x4E54	0x5367	# <CJK>
+0x4E55	0x63E1	# <CJK>
+0x4E56	0x6C83	# <CJK>
+0x4E57	0x5DEB	# <CJK>
+0x4E58	0x55DA	# <CJK>
+0x4E59	0x93A2	# <CJK>
+0x4E5A	0x70CF	# <CJK>
+0x4E5B	0x6C61	# <CJK>
+0x4E5C	0x8AA3	# <CJK>
+0x4E5D	0x5C4B	# <CJK>
+0x4E5E	0x7121	# <CJK>
+0x4E5F	0x856A	# <CJK>
+0x4E60	0x68A7	# <CJK>
+0x4E61	0x543E	# <CJK>
+0x4E62	0x5434	# <CJK>
+0x4E63	0x6BCB	# <CJK>
+0x4E64	0x6B66	# <CJK>
+0x4E65	0x4E94	# <CJK>
+0x4E66	0x6342	# <CJK>
+0x4E67	0x5348	# <CJK>
+0x4E68	0x821E	# <CJK>
+0x4E69	0x4F0D	# <CJK>
+0x4E6A	0x4FAE	# <CJK>
+0x4E6B	0x5862	# <CJK>
+0x4E6C	0x620A	# <CJK>
+0x4E6D	0x9727	# <CJK>
+0x4E6E	0x6664	# <CJK>
+0x4E6F	0x7269	# <CJK>
+0x4E70	0x52FF	# <CJK>
+0x4E71	0x52D9	# <CJK>
+0x4E72	0x609F	# <CJK>
+0x4E73	0x8AA4	# <CJK>
+0x4E74	0x6614	# <CJK>
+0x4E75	0x7199	# <CJK>
+0x4E76	0x6790	# <CJK>
+0x4E77	0x897F	# <CJK>
+0x4E78	0x7852	# <CJK>
+0x4E79	0x77FD	# <CJK>
+0x4E7A	0x6670	# <CJK>
+0x4E7B	0x563B	# <CJK>
+0x4E7C	0x5438	# <CJK>
+0x4E7D	0x932B	# <CJK>
+0x4E7E	0x72A7	# <CJK>
+0x4F21	0x7A00	# <CJK>
+0x4F22	0x606F	# <CJK>
+0x4F23	0x5E0C	# <CJK>
+0x4F24	0x6089	# <CJK>
+0x4F25	0x819D	# <CJK>
+0x4F26	0x5915	# <CJK>
+0x4F27	0x60DC	# <CJK>
+0x4F28	0x7184	# <CJK>
+0x4F29	0x70EF	# <CJK>
+0x4F2A	0x6EAA	# <CJK>
+0x4F2B	0x6C50	# <CJK>
+0x4F2C	0x7280	# <CJK>
+0x4F2D	0x6A84	# <CJK>
+0x4F2E	0x8972	# <CJK>
+0x4F2F	0x5E2D	# <CJK>
+0x4F30	0x7FD2	# <CJK>
+0x4F31	0x5AB3	# <CJK>
+0x4F32	0x559C	# <CJK>
+0x4F33	0x9291	# <CJK>
+0x4F34	0x6D17	# <CJK>
+0x4F35	0x7CFB	# <CJK>
+0x4F36	0x9699	# <CJK>
+0x4F37	0x6232	# <CJK>
+0x4F38	0x7D30	# <CJK>
+0x4F39	0x778E	# <CJK>
+0x4F3A	0x8766	# <CJK>
+0x4F3B	0x5323	# <CJK>
+0x4F3C	0x971E	# <CJK>
+0x4F3D	0x8F44	# <CJK>
+0x4F3E	0x6687	# <CJK>
+0x4F3F	0x5CFD	# <CJK>
+0x4F40	0x4FE0	# <CJK>
+0x4F41	0x72F9	# <CJK>
+0x4F42	0x4E0B	# <CJK>
+0x4F43	0x53A6	# <CJK>
+0x4F44	0x590F	# <CJK>
+0x4F45	0x5687	# <CJK>
+0x4F46	0x6380	# <CJK>
+0x4F47	0x9341	# <CJK>
+0x4F48	0x5148	# <CJK>
+0x4F49	0x4ED9	# <CJK>
+0x4F4A	0x9BAE	# <CJK>
+0x4F4B	0x7E96	# <CJK>
+0x4F4C	0x54B8	# <CJK>
+0x4F4D	0x8CE2	# <CJK>
+0x4F4E	0x929C	# <CJK>
+0x4F4F	0x8237	# <CJK>
+0x4F50	0x9591	# <CJK>
+0x4F51	0x6D8E	# <CJK>
+0x4F52	0x5F26	# <CJK>
+0x4F53	0x5ACC	# <CJK>
+0x4F54	0x986F	# <CJK>
+0x4F55	0x96AA	# <CJK>
+0x4F56	0x73FE	# <CJK>
+0x4F57	0x737B	# <CJK>
+0x4F58	0x7E23	# <CJK>
+0x4F59	0x817A	# <CJK>
+0x4F5A	0x9921	# <CJK>
+0x4F5B	0x7FA1	# <CJK>
+0x4F5C	0x61B2	# <CJK>
+0x4F5D	0x9677	# <CJK>
+0x4F5E	0x9650	# <CJK>
+0x4F5F	0x7DAB	# <CJK>
+0x4F60	0x76F8	# <CJK>
+0x4F61	0x53A2	# <CJK>
+0x4F62	0x9472	# <CJK>
+0x4F63	0x9999	# <CJK>
+0x4F64	0x7BB1	# <CJK>
+0x4F65	0x8944	# <CJK>
+0x4F66	0x6E58	# <CJK>
+0x4F67	0x9109	# <CJK>
+0x4F68	0x7FD4	# <CJK>
+0x4F69	0x7965	# <CJK>
+0x4F6A	0x8A73	# <CJK>
+0x4F6B	0x60F3	# <CJK>
+0x4F6C	0x97FF	# <CJK>
+0x4F6D	0x4EAB	# <CJK>
+0x4F6E	0x9805	# <CJK>
+0x4F6F	0x5DF7	# <CJK>
+0x4F70	0x6A61	# <CJK>
+0x4F71	0x50CF	# <CJK>
+0x4F72	0x5411	# <CJK>
+0x4F73	0x8C61	# <CJK>
+0x4F74	0x856D	# <CJK>
+0x4F75	0x785D	# <CJK>
+0x4F76	0x9704	# <CJK>
+0x4F77	0x524A	# <CJK>
+0x4F78	0x54EE	# <CJK>
+0x4F79	0x56C2	# <CJK>
+0x4F7A	0x92B7	# <CJK>
+0x4F7B	0x6D88	# <CJK>
+0x4F7C	0x5BB5	# <CJK>
+0x4F7D	0x6DC6	# <CJK>
+0x4F7E	0x66C9	# <CJK>
+0x5021	0x5C0F	# <CJK>
+0x5022	0x5B5D	# <CJK>
+0x5023	0x6821	# <CJK>
+0x5024	0x8096	# <CJK>
+0x5025	0x562F	# <CJK>
+0x5026	0x7B11	# <CJK>
+0x5027	0x6548	# <CJK>
+0x5028	0x6954	# <CJK>
+0x5029	0x4E9B	# <CJK>
+0x502A	0x6B47	# <CJK>
+0x502B	0x874E	# <CJK>
+0x502C	0x978B	# <CJK>
+0x502D	0x5354	# <CJK>
+0x502E	0x633E	# <CJK>
+0x502F	0x643A	# <CJK>
+0x5030	0x90AA	# <CJK>
+0x5031	0x659C	# <CJK>
+0x5032	0x8105	# <CJK>
+0x5033	0x8AE7	# <CJK>
+0x5034	0x5BEB	# <CJK>
+0x5035	0x68B0	# <CJK>
+0x5036	0x5378	# <CJK>
+0x5037	0x87F9	# <CJK>
+0x5038	0x61C8	# <CJK>
+0x5039	0x6CC4	# <CJK>
+0x503A	0x7009	# <CJK>
+0x503B	0x8B1D	# <CJK>
+0x503C	0x5C51	# <CJK>
+0x503D	0x85AA	# <CJK>
+0x503E	0x82AF	# <CJK>
+0x503F	0x92C5	# <CJK>
+0x5040	0x6B23	# <CJK>
+0x5041	0x8F9B	# <CJK>
+0x5042	0x65B0	# <CJK>
+0x5043	0x5FFB	# <CJK>
+0x5044	0x5FC3	# <CJK>
+0x5045	0x4FE1	# <CJK>
+0x5046	0x91C1	# <CJK>
+0x5047	0x661F	# <CJK>
+0x5048	0x8165	# <CJK>
+0x5049	0x7329	# <CJK>
+0x504A	0x60FA	# <CJK>
+0x504B	0x8208	# <CJK>
+0x504C	0x5211	# <CJK>
+0x504D	0x578B	# <CJK>
+0x504E	0x5F62	# <CJK>
+0x504F	0x90A2	# <CJK>
+0x5050	0x884C	# <CJK>
+0x5051	0x9192	# <CJK>
+0x5052	0x5E78	# <CJK>
+0x5053	0x674F	# <CJK>
+0x5054	0x6027	# <CJK>
+0x5055	0x59D3	# <CJK>
+0x5056	0x5144	# <CJK>
+0x5057	0x51F6	# <CJK>
+0x5058	0x80F8	# <CJK>
+0x5059	0x5308	# <CJK>
+0x505A	0x6C79	# <CJK>
+0x505B	0x96C4	# <CJK>
+0x505C	0x718A	# <CJK>
+0x505D	0x4F11	# <CJK>
+0x505E	0x4FEE	# <CJK>
+0x505F	0x7F9E	# <CJK>
+0x5060	0x673D	# <CJK>
+0x5061	0x55C5	# <CJK>
+0x5062	0x92B9	# <CJK>
+0x5063	0x79C0	# <CJK>
+0x5064	0x8896	# <CJK>
+0x5065	0x7D89	# <CJK>
+0x5066	0x589F	# <CJK>
+0x5067	0x620C	# <CJK>
+0x5068	0x9700	# <CJK>
+0x5069	0x865A	# <CJK>
+0x506A	0x5618	# <CJK>
+0x506B	0x9808	# <CJK>
+0x506C	0x5F90	# <CJK>
+0x506D	0x8A31	# <CJK>
+0x506E	0x84C4	# <CJK>
+0x506F	0x9157	# <CJK>
+0x5070	0x53D9	# <CJK>
+0x5071	0x65ED	# <CJK>
+0x5072	0x5E8F	# <CJK>
+0x5073	0x755C	# <CJK>
+0x5074	0x6064	# <CJK>
+0x5075	0x7D6E	# <CJK>
+0x5076	0x5A7F	# <CJK>
+0x5077	0x7DD2	# <CJK>
+0x5078	0x7E8C	# <CJK>
+0x5079	0x8ED2	# <CJK>
+0x507A	0x55A7	# <CJK>
+0x507B	0x5BA3	# <CJK>
+0x507C	0x61F8	# <CJK>
+0x507D	0x65CB	# <CJK>
+0x507E	0x7384	# <CJK>
+0x5121	0x9078	# <CJK>
+0x5122	0x766C	# <CJK>
+0x5123	0x7729	# <CJK>
+0x5124	0x7D62	# <CJK>
+0x5125	0x9774	# <CJK>
+0x5126	0x859B	# <CJK>
+0x5127	0x5B78	# <CJK>
+0x5128	0x7A74	# <CJK>
+0x5129	0x96EA	# <CJK>
+0x512A	0x8840	# <CJK>
+0x512B	0x52DB	# <CJK>
+0x512C	0x718F	# <CJK>
+0x512D	0x5FAA	# <CJK>
+0x512E	0x65EC	# <CJK>
+0x512F	0x8A62	# <CJK>
+0x5130	0x5C0B	# <CJK>
+0x5131	0x99B4	# <CJK>
+0x5132	0x5DE1	# <CJK>
+0x5133	0x6B89	# <CJK>
+0x5134	0x6C5B	# <CJK>
+0x5135	0x8A13	# <CJK>
+0x5136	0x8A0A	# <CJK>
+0x5137	0x905C	# <CJK>
+0x5138	0x8FC5	# <CJK>
+0x5139	0x58D3	# <CJK>
+0x513A	0x62BC	# <CJK>
+0x513B	0x9D09	# <CJK>
+0x513C	0x9D28	# <CJK>
+0x513D	0x5440	# <CJK>
+0x513E	0x4E2B	# <CJK>
+0x513F	0x82BD	# <CJK>
+0x5140	0x7259	# <CJK>
+0x5141	0x869C	# <CJK>
+0x5142	0x5D16	# <CJK>
+0x5143	0x8859	# <CJK>
+0x5144	0x6DAF	# <CJK>
+0x5145	0x96C5	# <CJK>
+0x5146	0x555E	# <CJK>
+0x5147	0x4E9E	# <CJK>
+0x5148	0x8A1D	# <CJK>
+0x5149	0x7109	# <CJK>
+0x514A	0x54BD	# <CJK>
+0x514B	0x95B9	# <CJK>
+0x514C	0x70DF	# <CJK>
+0x514D	0x6DF9	# <CJK>
+0x514E	0x9E7D	# <CJK>
+0x514F	0x56B4	# <CJK>
+0x5150	0x7814	# <CJK>
+0x5151	0x8712	# <CJK>
+0x5152	0x5CA9	# <CJK>
+0x5153	0x5EF6	# <CJK>
+0x5154	0x8A00	# <CJK>
+0x5155	0x9854	# <CJK>
+0x5156	0x95BB	# <CJK>
+0x5157	0x708E	# <CJK>
+0x5158	0x6CBF	# <CJK>
+0x5159	0x5944	# <CJK>
+0x515A	0x63A9	# <CJK>
+0x515B	0x773C	# <CJK>
+0x515C	0x884D	# <CJK>
+0x515D	0x6F14	# <CJK>
+0x515E	0x8277	# <CJK>
+0x515F	0x5830	# <CJK>
+0x5160	0x71D5	# <CJK>
+0x5161	0x53AD	# <CJK>
+0x5162	0x786F	# <CJK>
+0x5163	0x96C1	# <CJK>
+0x5164	0x5501	# <CJK>
+0x5165	0x5F66	# <CJK>
+0x5166	0x7130	# <CJK>
+0x5167	0x5BB4	# <CJK>
+0x5168	0x8AFA	# <CJK>
+0x5169	0x9A57	# <CJK>
+0x516A	0x6B83	# <CJK>
+0x516B	0x592E	# <CJK>
+0x516C	0x9D26	# <CJK>
+0x516D	0x79E7	# <CJK>
+0x516E	0x694A	# <CJK>
+0x516F	0x63DA	# <CJK>
+0x5170	0x4F6F	# <CJK>
+0x5171	0x760D	# <CJK>
+0x5172	0x7F8A	# <CJK>
+0x5173	0x6D0B	# <CJK>
+0x5174	0x967D	# <CJK>
+0x5175	0x6C27	# <CJK>
+0x5176	0x4EF0	# <CJK>
+0x5177	0x7662	# <CJK>
+0x5178	0x990A	# <CJK>
+0x5179	0x6A23	# <CJK>
+0x517A	0x6F3E	# <CJK>
+0x517B	0x9080	# <CJK>
+0x517C	0x8170	# <CJK>
+0x517D	0x5996	# <CJK>
+0x517E	0x7476	# <CJK>
+0x5221	0x6447	# <CJK>
+0x5222	0x582F	# <CJK>
+0x5223	0x9065	# <CJK>
+0x5224	0x7A91	# <CJK>
+0x5225	0x8B21	# <CJK>
+0x5226	0x59DA	# <CJK>
+0x5227	0x54AC	# <CJK>
+0x5228	0x8200	# <CJK>
+0x5229	0x85E5	# <CJK>
+0x522A	0x8981	# <CJK>
+0x522B	0x8000	# <CJK>
+0x522C	0x6930	# <CJK>
+0x522D	0x564E	# <CJK>
+0x522E	0x8036	# <CJK>
+0x522F	0x723A	# <CJK>
+0x5230	0x91CE	# <CJK>
+0x5231	0x51B6	# <CJK>
+0x5232	0x4E5F	# <CJK>
+0x5233	0x9801	# <CJK>
+0x5234	0x6396	# <CJK>
+0x5235	0x696D	# <CJK>
+0x5236	0x8449	# <CJK>
+0x5237	0x66F3	# <CJK>
+0x5238	0x814B	# <CJK>
+0x5239	0x591C	# <CJK>
+0x523A	0x6DB2	# <CJK>
+0x523B	0x4E00	# <CJK>
+0x523C	0x58F9	# <CJK>
+0x523D	0x91AB	# <CJK>
+0x523E	0x63D6	# <CJK>
+0x523F	0x92A5	# <CJK>
+0x5240	0x4F9D	# <CJK>
+0x5241	0x4F0A	# <CJK>
+0x5242	0x8863	# <CJK>
+0x5243	0x9824	# <CJK>
+0x5244	0x5937	# <CJK>
+0x5245	0x907A	# <CJK>
+0x5246	0x79FB	# <CJK>
+0x5247	0x5100	# <CJK>
+0x5248	0x80F0	# <CJK>
+0x5249	0x7591	# <CJK>
+0x524A	0x6C82	# <CJK>
+0x524B	0x5B9C	# <CJK>
+0x524C	0x59E8	# <CJK>
+0x524D	0x5F5D	# <CJK>
+0x524E	0x6905	# <CJK>
+0x524F	0x87FB	# <CJK>
+0x5250	0x501A	# <CJK>
+0x5251	0x5DF2	# <CJK>
+0x5252	0x4E59	# <CJK>
+0x5253	0x77E3	# <CJK>
+0x5254	0x4EE5	# <CJK>
+0x5255	0x85DD	# <CJK>
+0x5256	0x6291	# <CJK>
+0x5257	0x6613	# <CJK>
+0x5258	0x9091	# <CJK>
+0x5259	0x5C79	# <CJK>
+0x525A	0x5104	# <CJK>
+0x525B	0x5F79	# <CJK>
+0x525C	0x81C6	# <CJK>
+0x525D	0x9038	# <CJK>
+0x525E	0x8084	# <CJK>
+0x525F	0x75AB	# <CJK>
+0x5260	0x4EA6	# <CJK>
+0x5261	0x88D4	# <CJK>
+0x5262	0x610F	# <CJK>
+0x5263	0x6BC5	# <CJK>
+0x5264	0x61B6	# <CJK>
+0x5265	0x7FA9	# <CJK>
+0x5266	0x76CA	# <CJK>
+0x5267	0x6EA2	# <CJK>
+0x5268	0x8A63	# <CJK>
+0x5269	0x8B70	# <CJK>
+0x526A	0x8ABC	# <CJK>
+0x526B	0x8B6F	# <CJK>
+0x526C	0x5F02	# <CJK>
+0x526D	0x7FFC	# <CJK>
+0x526E	0x7FCC	# <CJK>
+0x526F	0x7E79	# <CJK>
+0x5270	0x8335	# <CJK>
+0x5271	0x852D	# <CJK>
+0x5272	0x56E0	# <CJK>
+0x5273	0x6BB7	# <CJK>
+0x5274	0x97F3	# <CJK>
+0x5275	0x9670	# <CJK>
+0x5276	0x59FB	# <CJK>
+0x5277	0x541F	# <CJK>
+0x5278	0x9280	# <CJK>
+0x5279	0x6DEB	# <CJK>
+0x527A	0x5BC5	# <CJK>
+0x527B	0x98F2	# <CJK>
+0x527C	0x5C39	# <CJK>
+0x527D	0x5F15	# <CJK>
+0x527E	0x96B1	# <CJK>
+0x5321	0x5370	# <CJK>
+0x5322	0x82F1	# <CJK>
+0x5323	0x6AFB	# <CJK>
+0x5324	0x5B30	# <CJK>
+0x5325	0x9DF9	# <CJK>
+0x5326	0x61C9	# <CJK>
+0x5327	0x7E93	# <CJK>
+0x5328	0x7469	# <CJK>
+0x5329	0x87A2	# <CJK>
+0x532A	0x71DF	# <CJK>
+0x532B	0x7192	# <CJK>
+0x532C	0x8805	# <CJK>
+0x532D	0x8FCE	# <CJK>
+0x532E	0x8D0F	# <CJK>
+0x532F	0x76C8	# <CJK>
+0x5330	0x5F71	# <CJK>
+0x5331	0x7A4E	# <CJK>
+0x5332	0x786C	# <CJK>
+0x5333	0x6620	# <CJK>
+0x5334	0x55B2	# <CJK>
+0x5335	0x64C1	# <CJK>
+0x5336	0x50AD	# <CJK>
+0x5337	0x81C3	# <CJK>
+0x5338	0x7670	# <CJK>
+0x5339	0x5EB8	# <CJK>
+0x533A	0x96CD	# <CJK>
+0x533B	0x8E34	# <CJK>
+0x533C	0x86F9	# <CJK>
+0x533D	0x548F	# <CJK>
+0x533E	0x6CF3	# <CJK>
+0x533F	0x6D8C	# <CJK>
+0x5340	0x6C38	# <CJK>
+0x5341	0x607F	# <CJK>
+0x5342	0x52C7	# <CJK>
+0x5343	0x7528	# <CJK>
+0x5344	0x5E7D	# <CJK>
+0x5345	0x512A	# <CJK>
+0x5346	0x60A0	# <CJK>
+0x5347	0x6182	# <CJK>
+0x5348	0x5C24	# <CJK>
+0x5349	0x7531	# <CJK>
+0x534A	0x90F5	# <CJK>
+0x534B	0x923E	# <CJK>
+0x534C	0x7336	# <CJK>
+0x534D	0x6CB9	# <CJK>
+0x534E	0x6E38	# <CJK>
+0x534F	0x9149	# <CJK>
+0x5350	0x6709	# <CJK>
+0x5351	0x53CB	# <CJK>
+0x5352	0x53F3	# <CJK>
+0x5353	0x4F51	# <CJK>
+0x5354	0x91C9	# <CJK>
+0x5355	0x8A98	# <CJK>
+0x5356	0x53C8	# <CJK>
+0x5357	0x5E7C	# <CJK>
+0x5358	0x8FC2	# <CJK>
+0x5359	0x6DE4	# <CJK>
+0x535A	0x4E8E	# <CJK>
+0x535B	0x76C2	# <CJK>
+0x535C	0x6986	# <CJK>
+0x535D	0x865E	# <CJK>
+0x535E	0x611A	# <CJK>
+0x535F	0x8F3F	# <CJK>
+0x5360	0x9918	# <CJK>
+0x5361	0x4FDE	# <CJK>
+0x5362	0x903E	# <CJK>
+0x5363	0x9B5A	# <CJK>
+0x5364	0x6109	# <CJK>
+0x5365	0x6E1D	# <CJK>
+0x5366	0x6F01	# <CJK>
+0x5367	0x9685	# <CJK>
+0x5368	0x4E88	# <CJK>
+0x5369	0x5A31	# <CJK>
+0x536A	0x96E8	# <CJK>
+0x536B	0x8207	# <CJK>
+0x536C	0x5DBC	# <CJK>
+0x536D	0x79B9	# <CJK>
+0x536E	0x5B87	# <CJK>
+0x536F	0x8A9E	# <CJK>
+0x5370	0x7FBD	# <CJK>
+0x5371	0x7389	# <CJK>
+0x5372	0x57DF	# <CJK>
+0x5373	0x828B	# <CJK>
+0x5374	0x9B31	# <CJK>
+0x5375	0x5401	# <CJK>
+0x5376	0x9047	# <CJK>
+0x5377	0x55BB	# <CJK>
+0x5378	0x5CEA	# <CJK>
+0x5379	0x5FA1	# <CJK>
+0x537A	0x6108	# <CJK>
+0x537B	0x6B32	# <CJK>
+0x537C	0x7344	# <CJK>
+0x537D	0x80B2	# <CJK>
+0x537E	0x8B7D	# <CJK>
+0x5421	0x6D74	# <CJK>
+0x5422	0x5BD3	# <CJK>
+0x5423	0x88D5	# <CJK>
+0x5424	0x9810	# <CJK>
+0x5425	0x8C6B	# <CJK>
+0x5426	0x99AD	# <CJK>
+0x5427	0x9D1B	# <CJK>
+0x5428	0x6DF5	# <CJK>
+0x5429	0x51A4	# <CJK>
+0x542A	0x5143	# <CJK>
+0x542B	0x57A3	# <CJK>
+0x542C	0x8881	# <CJK>
+0x542D	0x539F	# <CJK>
+0x542E	0x63F4	# <CJK>
+0x542F	0x8F45	# <CJK>
+0x5430	0x5712	# <CJK>
+0x5431	0x54E1	# <CJK>
+0x5432	0x5713	# <CJK>
+0x5433	0x733F	# <CJK>
+0x5434	0x6E90	# <CJK>
+0x5435	0x7DE3	# <CJK>
+0x5436	0x9060	# <CJK>
+0x5437	0x82D1	# <CJK>
+0x5438	0x9858	# <CJK>
+0x5439	0x6028	# <CJK>
+0x543A	0x9662	# <CJK>
+0x543B	0x66F0	# <CJK>
+0x543C	0x7D04	# <CJK>
+0x543D	0x8D8A	# <CJK>
+0x543E	0x8E8D	# <CJK>
+0x543F	0x9470	# <CJK>
+0x5440	0x5CB3	# <CJK>
+0x5441	0x7CA4	# <CJK>
+0x5442	0x6708	# <CJK>
+0x5443	0x60A6	# <CJK>
+0x5444	0x95B2	# <CJK>
+0x5445	0x8018	# <CJK>
+0x5446	0x96F2	# <CJK>
+0x5447	0x9116	# <CJK>
+0x5448	0x5300	# <CJK>
+0x5449	0x9695	# <CJK>
+0x544A	0x5141	# <CJK>
+0x544B	0x904B	# <CJK>
+0x544C	0x85F4	# <CJK>
+0x544D	0x9196	# <CJK>
+0x544E	0x6688	# <CJK>
+0x544F	0x97F5	# <CJK>
+0x5450	0x5B55	# <CJK>
+0x5451	0x531D	# <CJK>
+0x5452	0x7838	# <CJK>
+0x5453	0x96DC	# <CJK>
+0x5454	0x683D	# <CJK>
+0x5455	0x54C9	# <CJK>
+0x5456	0x707E	# <CJK>
+0x5457	0x5BB0	# <CJK>
+0x5458	0x8F09	# <CJK>
+0x5459	0x518D	# <CJK>
+0x545A	0x5728	# <CJK>
+0x545B	0x54B1	# <CJK>
+0x545C	0x6522	# <CJK>
+0x545D	0x66AB	# <CJK>
+0x545E	0x8D0A	# <CJK>
+0x545F	0x8D1C	# <CJK>
+0x5460	0x81DF	# <CJK>
+0x5461	0x846C	# <CJK>
+0x5462	0x906D	# <CJK>
+0x5463	0x7CDF	# <CJK>
+0x5464	0x947F	# <CJK>
+0x5465	0x85FB	# <CJK>
+0x5466	0x68D7	# <CJK>
+0x5467	0x65E9	# <CJK>
+0x5468	0x6FA1	# <CJK>
+0x5469	0x86A4	# <CJK>
+0x546A	0x8E81	# <CJK>
+0x546B	0x566A	# <CJK>
+0x546C	0x9020	# <CJK>
+0x546D	0x7682	# <CJK>
+0x546E	0x7AC8	# <CJK>
+0x546F	0x71E5	# <CJK>
+0x5470	0x8CAC	# <CJK>
+0x5471	0x64C7	# <CJK>
+0x5472	0x5247	# <CJK>
+0x5473	0x6FA4	# <CJK>
+0x5474	0x8CCA	# <CJK>
+0x5475	0x600E	# <CJK>
+0x5476	0x589E	# <CJK>
+0x5477	0x618E	# <CJK>
+0x5478	0x66FE	# <CJK>
+0x5479	0x8D08	# <CJK>
+0x547A	0x624E	# <CJK>
+0x547B	0x55B3	# <CJK>
+0x547C	0x6E23	# <CJK>
+0x547D	0x672D	# <CJK>
+0x547E	0x8ECB	# <CJK>
+0x5521	0x9358	# <CJK>
+0x5522	0x9598	# <CJK>
+0x5523	0x7728	# <CJK>
+0x5524	0x6805	# <CJK>
+0x5525	0x69A8	# <CJK>
+0x5526	0x548B	# <CJK>
+0x5527	0x4E4D	# <CJK>
+0x5528	0x70B8	# <CJK>
+0x5529	0x8A50	# <CJK>
+0x552A	0x6458	# <CJK>
+0x552B	0x9F4B	# <CJK>
+0x552C	0x5B85	# <CJK>
+0x552D	0x7A84	# <CJK>
+0x552E	0x50B5	# <CJK>
+0x552F	0x5BE8	# <CJK>
+0x5530	0x77BB	# <CJK>
+0x5531	0x6C08	# <CJK>
+0x5532	0x8A79	# <CJK>
+0x5533	0x7C98	# <CJK>
+0x5534	0x6CBE	# <CJK>
+0x5535	0x76DE	# <CJK>
+0x5536	0x65AC	# <CJK>
+0x5537	0x8F3E	# <CJK>
+0x5538	0x5D84	# <CJK>
+0x5539	0x5C55	# <CJK>
+0x553A	0x8638	# <CJK>
+0x553B	0x68E7	# <CJK>
+0x553C	0x5360	# <CJK>
+0x553D	0x6230	# <CJK>
+0x553E	0x7AD9	# <CJK>
+0x553F	0x6E5B	# <CJK>
+0x5540	0x7DBB	# <CJK>
+0x5541	0x6A1F	# <CJK>
+0x5542	0x7AE0	# <CJK>
+0x5543	0x5F70	# <CJK>
+0x5544	0x6F33	# <CJK>
+0x5545	0x5F35	# <CJK>
+0x5546	0x638C	# <CJK>
+0x5547	0x6F32	# <CJK>
+0x5548	0x6756	# <CJK>
+0x5549	0x4E08	# <CJK>
+0x554A	0x5E33	# <CJK>
+0x554B	0x8CEC	# <CJK>
+0x554C	0x4ED7	# <CJK>
+0x554D	0x8139	# <CJK>
+0x554E	0x7634	# <CJK>
+0x554F	0x969C	# <CJK>
+0x5550	0x62DB	# <CJK>
+0x5551	0x662D	# <CJK>
+0x5552	0x627E	# <CJK>
+0x5553	0x6CBC	# <CJK>
+0x5554	0x8D99	# <CJK>
+0x5555	0x7167	# <CJK>
+0x5556	0x7F69	# <CJK>
+0x5557	0x5146	# <CJK>
+0x5558	0x8087	# <CJK>
+0x5559	0x53EC	# <CJK>
+0x555A	0x906E	# <CJK>
+0x555B	0x6298	# <CJK>
+0x555C	0x54F2	# <CJK>
+0x555D	0x87C4	# <CJK>
+0x555E	0x8F4D	# <CJK>
+0x555F	0x8005	# <CJK>
+0x5560	0x937A	# <CJK>
+0x5561	0x8517	# <CJK>
+0x5562	0x9019	# <CJK>
+0x5563	0x6D59	# <CJK>
+0x5564	0x73CD	# <CJK>
+0x5565	0x659F	# <CJK>
+0x5566	0x771F	# <CJK>
+0x5567	0x7504	# <CJK>
+0x5568	0x7827	# <CJK>
+0x5569	0x81FB	# <CJK>
+0x556A	0x8C9E	# <CJK>
+0x556B	0x91DD	# <CJK>
+0x556C	0x5075	# <CJK>
+0x556D	0x6795	# <CJK>
+0x556E	0x75B9	# <CJK>
+0x556F	0x8A3A	# <CJK>
+0x5570	0x9707	# <CJK>
+0x5571	0x632F	# <CJK>
+0x5572	0x93AE	# <CJK>
+0x5573	0x9663	# <CJK>
+0x5574	0x84B8	# <CJK>
+0x5575	0x6399	# <CJK>
+0x5576	0x775C	# <CJK>
+0x5577	0x5F81	# <CJK>
+0x5578	0x7319	# <CJK>
+0x5579	0x722D	# <CJK>
+0x557A	0x6014	# <CJK>
+0x557B	0x6574	# <CJK>
+0x557C	0x62EF	# <CJK>
+0x557D	0x6B63	# <CJK>
+0x557E	0x653F	# <CJK>
+0x5621	0x5E40	# <CJK>
+0x5622	0x7665	# <CJK>
+0x5623	0x912D	# <CJK>
+0x5624	0x8B49	# <CJK>
+0x5625	0x829D	# <CJK>
+0x5626	0x679D	# <CJK>
+0x5627	0x652F	# <CJK>
+0x5628	0x5431	# <CJK>
+0x5629	0x8718	# <CJK>
+0x562A	0x77E5	# <CJK>
+0x562B	0x80A2	# <CJK>
+0x562C	0x8102	# <CJK>
+0x562D	0x6C41	# <CJK>
+0x562E	0x4E4B	# <CJK>
+0x562F	0x7E54	# <CJK>
+0x5630	0x8077	# <CJK>
+0x5631	0x76F4	# <CJK>
+0x5632	0x690D	# <CJK>
+0x5633	0x6B96	# <CJK>
+0x5634	0x57F7	# <CJK>
+0x5635	0x503C	# <CJK>
+0x5636	0x4F84	# <CJK>
+0x5637	0x5740	# <CJK>
+0x5638	0x6307	# <CJK>
+0x5639	0x6B62	# <CJK>
+0x563A	0x8DBE	# <CJK>
+0x563B	0x8879	# <CJK>
+0x563C	0x65E8	# <CJK>
+0x563D	0x7D19	# <CJK>
+0x563E	0x5FD7	# <CJK>
+0x563F	0x646F	# <CJK>
+0x5640	0x64F2	# <CJK>
+0x5641	0x81F3	# <CJK>
+0x5642	0x81F4	# <CJK>
+0x5643	0x7F6E	# <CJK>
+0x5644	0x5E5F	# <CJK>
+0x5645	0x5CD9	# <CJK>
+0x5646	0x5236	# <CJK>
+0x5647	0x667A	# <CJK>
+0x5648	0x79E9	# <CJK>
+0x5649	0x7A1A	# <CJK>
+0x564A	0x8CEA	# <CJK>
+0x564B	0x7099	# <CJK>
+0x564C	0x75D4	# <CJK>
+0x564D	0x6EEF	# <CJK>
+0x564E	0x6CBB	# <CJK>
+0x564F	0x7A92	# <CJK>
+0x5650	0x4E2D	# <CJK>
+0x5651	0x76C5	# <CJK>
+0x5652	0x5FE0	# <CJK>
+0x5653	0x9418	# <CJK>
+0x5654	0x8877	# <CJK>
+0x5655	0x7D42	# <CJK>
+0x5656	0x7A2E	# <CJK>
+0x5657	0x816B	# <CJK>
+0x5658	0x91CD	# <CJK>
+0x5659	0x4EF2	# <CJK>
+0x565A	0x8846	# <CJK>
+0x565B	0x821F	# <CJK>
+0x565C	0x5468	# <CJK>
+0x565D	0x5DDE	# <CJK>
+0x565E	0x6D32	# <CJK>
+0x565F	0x8B05	# <CJK>
+0x5660	0x7CA5	# <CJK>
+0x5661	0x8EF8	# <CJK>
+0x5662	0x8098	# <CJK>
+0x5663	0x5E1A	# <CJK>
+0x5664	0x5492	# <CJK>
+0x5665	0x76BA	# <CJK>
+0x5666	0x5B99	# <CJK>
+0x5667	0x665D	# <CJK>
+0x5668	0x9A5F	# <CJK>
+0x5669	0x73E0	# <CJK>
+0x566A	0x682A	# <CJK>
+0x566B	0x86DB	# <CJK>
+0x566C	0x6731	# <CJK>
+0x566D	0x732A	# <CJK>
+0x566E	0x8AF8	# <CJK>
+0x566F	0x8A85	# <CJK>
+0x5670	0x9010	# <CJK>
+0x5671	0x7AF9	# <CJK>
+0x5672	0x71ED	# <CJK>
+0x5673	0x716E	# <CJK>
+0x5674	0x62C4	# <CJK>
+0x5675	0x77DA	# <CJK>
+0x5676	0x56D1	# <CJK>
+0x5677	0x4E3B	# <CJK>
+0x5678	0x8457	# <CJK>
+0x5679	0x67F1	# <CJK>
+0x567A	0x52A9	# <CJK>
+0x567B	0x86C0	# <CJK>
+0x567C	0x8CAF	# <CJK>
+0x567D	0x9444	# <CJK>
+0x567E	0x7BC9	# <CJK>
+0x5721	0x4F4F	# <CJK>
+0x5722	0x6CE8	# <CJK>
+0x5723	0x795D	# <CJK>
+0x5724	0x99D0	# <CJK>
+0x5725	0x6293	# <CJK>
+0x5726	0x722A	# <CJK>
+0x5727	0x62FD	# <CJK>
+0x5728	0x5C08	# <CJK>
+0x5729	0x78DA	# <CJK>
+0x572A	0x8F49	# <CJK>
+0x572B	0x64B0	# <CJK>
+0x572C	0x8CFA	# <CJK>
+0x572D	0x7BC6	# <CJK>
+0x572E	0x6A01	# <CJK>
+0x572F	0x838A	# <CJK>
+0x5730	0x88DD	# <CJK>
+0x5731	0x599D	# <CJK>
+0x5732	0x649E	# <CJK>
+0x5733	0x58EF	# <CJK>
+0x5734	0x72C0	# <CJK>
+0x5735	0x690E	# <CJK>
+0x5736	0x9310	# <CJK>
+0x5737	0x8FFD	# <CJK>
+0x5738	0x8D05	# <CJK>
+0x5739	0x589C	# <CJK>
+0x573A	0x7DB4	# <CJK>
+0x573B	0x8AC4	# <CJK>
+0x573C	0x6E96	# <CJK>
+0x573D	0x6349	# <CJK>
+0x573E	0x62D9	# <CJK>
+0x573F	0x5353	# <CJK>
+0x5740	0x684C	# <CJK>
+0x5741	0x7422	# <CJK>
+0x5742	0x8301	# <CJK>
+0x5743	0x914C	# <CJK>
+0x5744	0x5544	# <CJK>
+0x5745	0x7740	# <CJK>
+0x5746	0x707C	# <CJK>
+0x5747	0x6FC1	# <CJK>
+0x5748	0x5179	# <CJK>
+0x5749	0x54A8	# <CJK>
+0x574A	0x8CC7	# <CJK>
+0x574B	0x59FF	# <CJK>
+0x574C	0x6ECB	# <CJK>
+0x574D	0x6DC4	# <CJK>
+0x574E	0x5B5C	# <CJK>
+0x574F	0x7D2B	# <CJK>
+0x5750	0x4ED4	# <CJK>
+0x5751	0x7C7D	# <CJK>
+0x5752	0x6ED3	# <CJK>
+0x5753	0x5B50	# <CJK>
+0x5754	0x81EA	# <CJK>
+0x5755	0x6F2C	# <CJK>
+0x5756	0x5B57	# <CJK>
+0x5757	0x9B03	# <CJK>
+0x5758	0x68D5	# <CJK>
+0x5759	0x8E2A	# <CJK>
+0x575A	0x5B97	# <CJK>
+0x575B	0x7D9C	# <CJK>
+0x575C	0x7E3D	# <CJK>
+0x575D	0x7E31	# <CJK>
+0x575E	0x9112	# <CJK>
+0x575F	0x8D70	# <CJK>
+0x5760	0x594F	# <CJK>
+0x5761	0x63CD	# <CJK>
+0x5762	0x79DF	# <CJK>
+0x5763	0x8DB3	# <CJK>
+0x5764	0x5352	# <CJK>
+0x5765	0x65CF	# <CJK>
+0x5766	0x7956	# <CJK>
+0x5767	0x8A5B	# <CJK>
+0x5768	0x963B	# <CJK>
+0x5769	0x7D44	# <CJK>
+0x576A	0x947D	# <CJK>
+0x576B	0x7E82	# <CJK>
+0x576C	0x5634	# <CJK>
+0x576D	0x9189	# <CJK>
+0x576E	0x6700	# <CJK>
+0x576F	0x7F6A	# <CJK>
+0x5770	0x5C0A	# <CJK>
+0x5771	0x9075	# <CJK>
+0x5772	0x6628	# <CJK>
+0x5773	0x5DE6	# <CJK>
+0x5774	0x4F50	# <CJK>
+0x5775	0x67DE	# <CJK>
+0x5776	0x505A	# <CJK>
+0x5777	0x4F5C	# <CJK>
+0x5778	0x5750	# <CJK>
+0x5779	0x5EA7	# <CJK>
+0x5821	0x4E8D	# <CJK>
+0x5822	0x4E0C	# <CJK>
+0x5823	0x5140	# <CJK>
+0x5824	0x4E10	# <CJK>
+0x5825	0x5EFF	# <CJK>
+0x5826	0x5345	# <CJK>
+0x5827	0x4E15	# <CJK>
+0x5828	0x4E98	# <CJK>
+0x5829	0x4E1E	# <CJK>
+0x582A	0x9B32	# <CJK>
+0x582B	0x5B6C	# <CJK>
+0x582C	0x5669	# <CJK>
+0x582D	0x4E28	# <CJK>
+0x582E	0x79BA	# <CJK>
+0x582F	0x4E3F	# <CJK>
+0x5830	0x5315	# <CJK>
+0x5831	0x4E47	# <CJK>
+0x5832	0x592D	# <CJK>
+0x5833	0x723B	# <CJK>
+0x5834	0x536E	# <CJK>
+0x5835	0x6C10	# <CJK>
+0x5836	0x56DF	# <CJK>
+0x5837	0x80E4	# <CJK>
+0x5838	0x9997	# <CJK>
+0x5839	0x6BD3	# <CJK>
+0x583A	0x777E	# <CJK>
+0x583B	0x9F17	# <CJK>
+0x583C	0x4E36	# <CJK>
+0x583D	0x4E9F	# <CJK>
+0x583E	0x9F10	# <CJK>
+0x583F	0x4E5C	# <CJK>
+0x5840	0x4E69	# <CJK>
+0x5841	0x4E93	# <CJK>
+0x5842	0x8288	# <CJK>
+0x5843	0x5B5B	# <CJK>
+0x5844	0x55C7	# <CJK>
+0x5845	0x560F	# <CJK>
+0x5846	0x4EC4	# <CJK>
+0x5847	0x5399	# <CJK>
+0x5848	0x539D	# <CJK>
+0x5849	0x53B4	# <CJK>
+0x584A	0x53A5	# <CJK>
+0x584B	0x53AE	# <CJK>
+0x584C	0x9768	# <CJK>
+0x584D	0x8D0B	# <CJK>
+0x584E	0x531A	# <CJK>
+0x584F	0x53F5	# <CJK>
+0x5850	0x532D	# <CJK>
+0x5851	0x5331	# <CJK>
+0x5852	0x533E	# <CJK>
+0x5853	0x8CFE	# <CJK>
+0x5854	0x5366	# <CJK>
+0x5855	0x5363	# <CJK>
+0x5856	0x5202	# <CJK>
+0x5857	0x5208	# <CJK>
+0x5858	0x520E	# <CJK>
+0x5859	0x5244	# <CJK>
+0x585A	0x5233	# <CJK>
+0x585B	0x528C	# <CJK>
+0x585C	0x5274	# <CJK>
+0x585D	0x524C	# <CJK>
+0x585E	0x525E	# <CJK>
+0x585F	0x5261	# <CJK>
+0x5860	0x525C	# <CJK>
+0x5861	0x84AF	# <CJK>
+0x5862	0x527D	# <CJK>
+0x5863	0x5282	# <CJK>
+0x5864	0x5281	# <CJK>
+0x5865	0x5290	# <CJK>
+0x5866	0x5293	# <CJK>
+0x5867	0x5182	# <CJK>
+0x5868	0x7F54	# <CJK>
+0x5869	0x4EBB	# <CJK>
+0x586A	0x4EC3	# <CJK>
+0x586B	0x4EC9	# <CJK>
+0x586C	0x4EC2	# <CJK>
+0x586D	0x4EE8	# <CJK>
+0x586E	0x4EE1	# <CJK>
+0x586F	0x4EEB	# <CJK>
+0x5870	0x4EDE	# <CJK>
+0x5871	0x50B4	# <CJK>
+0x5872	0x4EF3	# <CJK>
+0x5873	0x4F22	# <CJK>
+0x5874	0x4F64	# <CJK>
+0x5875	0x4EF5	# <CJK>
+0x5876	0x5000	# <CJK>
+0x5877	0x5096	# <CJK>
+0x5878	0x4F09	# <CJK>
+0x5879	0x4F47	# <CJK>
+0x587A	0x4F5E	# <CJK>
+0x587B	0x4F67	# <CJK>
+0x587C	0x6538	# <CJK>
+0x587D	0x4F5A	# <CJK>
+0x587E	0x4F5D	# <CJK>
+0x5921	0x4F5F	# <CJK>
+0x5922	0x4F57	# <CJK>
+0x5923	0x4F32	# <CJK>
+0x5924	0x4F3D	# <CJK>
+0x5925	0x4F76	# <CJK>
+0x5926	0x4F74	# <CJK>
+0x5927	0x4F91	# <CJK>
+0x5928	0x4F89	# <CJK>
+0x5929	0x4F83	# <CJK>
+0x592A	0x4F8F	# <CJK>
+0x592B	0x4F7E	# <CJK>
+0x592C	0x4F7B	# <CJK>
+0x592D	0x5115	# <CJK>
+0x592E	0x4F7C	# <CJK>
+0x592F	0x5102	# <CJK>
+0x5930	0x4F94	# <CJK>
+0x5931	0x5114	# <CJK>
+0x5932	0x513C	# <CJK>
+0x5933	0x5137	# <CJK>
+0x5934	0x4FC5	# <CJK>
+0x5935	0x4FDA	# <CJK>
+0x5936	0x4FE3	# <CJK>
+0x5937	0x4FDC	# <CJK>
+0x5938	0x4FD1	# <CJK>
+0x5939	0x4FDF	# <CJK>
+0x593A	0x4FF8	# <CJK>
+0x593B	0x5029	# <CJK>
+0x593C	0x504C	# <CJK>
+0x593D	0x4FF3	# <CJK>
+0x593E	0x502C	# <CJK>
+0x593F	0x500F	# <CJK>
+0x5940	0x502E	# <CJK>
+0x5941	0x502D	# <CJK>
+0x5942	0x4FFE	# <CJK>
+0x5943	0x501C	# <CJK>
+0x5944	0x500C	# <CJK>
+0x5945	0x5025	# <CJK>
+0x5946	0x5028	# <CJK>
+0x5947	0x50E8	# <CJK>
+0x5948	0x5043	# <CJK>
+0x5949	0x5055	# <CJK>
+0x594A	0x5048	# <CJK>
+0x594B	0x504E	# <CJK>
+0x594C	0x506C	# <CJK>
+0x594D	0x50C2	# <CJK>
+0x594E	0x513B	# <CJK>
+0x594F	0x5110	# <CJK>
+0x5950	0x513A	# <CJK>
+0x5951	0x50BA	# <CJK>
+0x5952	0x50D6	# <CJK>
+0x5953	0x5106	# <CJK>
+0x5954	0x50ED	# <CJK>
+0x5955	0x50EC	# <CJK>
+0x5956	0x50E6	# <CJK>
+0x5957	0x50EE	# <CJK>
+0x5958	0x5107	# <CJK>
+0x5959	0x510B	# <CJK>
+0x595A	0x4EDD	# <CJK>
+0x595B	0x6C3D	# <CJK>
+0x595C	0x4F58	# <CJK>
+0x595D	0x50C9	# <CJK>
+0x595E	0x4FCE	# <CJK>
+0x595F	0x9FA0	# <CJK>
+0x5960	0x6C46	# <CJK>
+0x5961	0x7CF4	# <CJK>
+0x5962	0x516E	# <CJK>
+0x5963	0x5DFD	# <CJK>
+0x5964	0x9ECC	# <CJK>
+0x5965	0x9998	# <CJK>
+0x5966	0x56C5	# <CJK>
+0x5967	0x5914	# <CJK>
+0x5968	0x52F9	# <CJK>
+0x5969	0x530D	# <CJK>
+0x596A	0x8A07	# <CJK>
+0x596B	0x5310	# <CJK>
+0x596C	0x9CEC	# <CJK>
+0x596D	0x5919	# <CJK>
+0x596E	0x5155	# <CJK>
+0x596F	0x4EA0	# <CJK>
+0x5970	0x5156	# <CJK>
+0x5971	0x4EB3	# <CJK>
+0x5972	0x886E	# <CJK>
+0x5973	0x88A4	# <CJK>
+0x5974	0x893B	# <CJK>
+0x5975	0x81E0	# <CJK>
+0x5976	0x88D2	# <CJK>
+0x5977	0x7980	# <CJK>
+0x5978	0x5B34	# <CJK>
+0x5979	0x8803	# <CJK>
+0x597A	0x7FB8	# <CJK>
+0x597B	0x51AB	# <CJK>
+0x597C	0x51B1	# <CJK>
+0x597D	0x51BD	# <CJK>
+0x597E	0x51BC	# <CJK>
+0x5A21	0x51C7	# <CJK>
+0x5A22	0x5196	# <CJK>
+0x5A23	0x51A2	# <CJK>
+0x5A24	0x51A5	# <CJK>
+0x5A25	0x8A01	# <CJK>
+0x5A26	0x8A10	# <CJK>
+0x5A27	0x8A0C	# <CJK>
+0x5A28	0x8A15	# <CJK>
+0x5A29	0x8B33	# <CJK>
+0x5A2A	0x8A4E	# <CJK>
+0x5A2B	0x8A25	# <CJK>
+0x5A2C	0x8A41	# <CJK>
+0x5A2D	0x8A36	# <CJK>
+0x5A2E	0x8A46	# <CJK>
+0x5A2F	0x8A54	# <CJK>
+0x5A30	0x8A58	# <CJK>
+0x5A31	0x8A52	# <CJK>
+0x5A32	0x8A86	# <CJK>
+0x5A33	0x8A84	# <CJK>
+0x5A34	0x8A7F	# <CJK>
+0x5A35	0x8A70	# <CJK>
+0x5A36	0x8A7C	# <CJK>
+0x5A37	0x8A75	# <CJK>
+0x5A38	0x8A6C	# <CJK>
+0x5A39	0x8A6E	# <CJK>
+0x5A3A	0x8ACD	# <CJK>
+0x5A3B	0x8AE2	# <CJK>
+0x5A3C	0x8A61	# <CJK>
+0x5A3D	0x8A9A	# <CJK>
+0x5A3E	0x8AA5	# <CJK>
+0x5A3F	0x8A91	# <CJK>
+0x5A40	0x8A92	# <CJK>
+0x5A41	0x8ACF	# <CJK>
+0x5A42	0x8AD1	# <CJK>
+0x5A43	0x8AC9	# <CJK>
+0x5A44	0x8ADB	# <CJK>
+0x5A45	0x8AD7	# <CJK>
+0x5A46	0x8AC2	# <CJK>
+0x5A47	0x8AB6	# <CJK>
+0x5A48	0x8AF6	# <CJK>
+0x5A49	0x8AEB	# <CJK>
+0x5A4A	0x8B14	# <CJK>
+0x5A4B	0x8B01	# <CJK>
+0x5A4C	0x8AE4	# <CJK>
+0x5A4D	0x8AED	# <CJK>
+0x5A4E	0x8AFC	# <CJK>
+0x5A4F	0x8AF3	# <CJK>
+0x5A50	0x8AE6	# <CJK>
+0x5A51	0x8AEE	# <CJK>
+0x5A52	0x8ADE	# <CJK>
+0x5A53	0x8B28	# <CJK>
+0x5A54	0x8B9C	# <CJK>
+0x5A55	0x8B16	# <CJK>
+0x5A56	0x8B1A	# <CJK>
+0x5A57	0x8B10	# <CJK>
+0x5A58	0x8B2B	# <CJK>
+0x5A59	0x8B2D	# <CJK>
+0x5A5A	0x8B56	# <CJK>
+0x5A5B	0x8B59	# <CJK>
+0x5A5C	0x8B4E	# <CJK>
+0x5A5D	0x8B9E	# <CJK>
+0x5A5E	0x8B6B	# <CJK>
+0x5A5F	0x8B96	# <CJK>
+0x5A60	0x5369	# <CJK>
+0x5A61	0x537A	# <CJK>
+0x5A62	0x961D	# <CJK>
+0x5A63	0x9622	# <CJK>
+0x5A64	0x9621	# <CJK>
+0x5A65	0x9631	# <CJK>
+0x5A66	0x962A	# <CJK>
+0x5A67	0x963D	# <CJK>
+0x5A68	0x963C	# <CJK>
+0x5A69	0x9642	# <CJK>
+0x5A6A	0x9658	# <CJK>
+0x5A6B	0x9654	# <CJK>
+0x5A6C	0x965F	# <CJK>
+0x5A6D	0x9689	# <CJK>
+0x5A6E	0x966C	# <CJK>
+0x5A6F	0x9672	# <CJK>
+0x5A70	0x9674	# <CJK>
+0x5A71	0x9688	# <CJK>
+0x5A72	0x968D	# <CJK>
+0x5A73	0x9697	# <CJK>
+0x5A74	0x96B0	# <CJK>
+0x5A75	0x9097	# <CJK>
+0x5A76	0x909B	# <CJK>
+0x5A77	0x913A	# <CJK>
+0x5A78	0x9099	# <CJK>
+0x5A79	0x9114	# <CJK>
+0x5A7A	0x90A1	# <CJK>
+0x5A7B	0x90B4	# <CJK>
+0x5A7C	0x90B3	# <CJK>
+0x5A7D	0x90B6	# <CJK>
+0x5A7E	0x9134	# <CJK>
+0x5B21	0x90B8	# <CJK>
+0x5B22	0x90B0	# <CJK>
+0x5B23	0x90DF	# <CJK>
+0x5B24	0x90C5	# <CJK>
+0x5B25	0x90BE	# <CJK>
+0x5B26	0x9136	# <CJK>
+0x5B27	0x90C4	# <CJK>
+0x5B28	0x90C7	# <CJK>
+0x5B29	0x9106	# <CJK>
+0x5B2A	0x9148	# <CJK>
+0x5B2B	0x90E2	# <CJK>
+0x5B2C	0x90DC	# <CJK>
+0x5B2D	0x90D7	# <CJK>
+0x5B2E	0x90DB	# <CJK>
+0x5B2F	0x90EB	# <CJK>
+0x5B30	0x90EF	# <CJK>
+0x5B31	0x90FE	# <CJK>
+0x5B32	0x9104	# <CJK>
+0x5B33	0x9122	# <CJK>
+0x5B34	0x911E	# <CJK>
+0x5B35	0x9123	# <CJK>
+0x5B36	0x9131	# <CJK>
+0x5B37	0x912F	# <CJK>
+0x5B38	0x9139	# <CJK>
+0x5B39	0x9143	# <CJK>
+0x5B3A	0x9146	# <CJK>
+0x5B3B	0x82BB	# <CJK>
+0x5B3C	0x5950	# <CJK>
+0x5B3D	0x52F1	# <CJK>
+0x5B3E	0x52AC	# <CJK>
+0x5B3F	0x52AD	# <CJK>
+0x5B40	0x52BE	# <CJK>
+0x5B41	0x54FF	# <CJK>
+0x5B42	0x52D0	# <CJK>
+0x5B43	0x52D6	# <CJK>
+0x5B44	0x52F0	# <CJK>
+0x5B45	0x53DF	# <CJK>
+0x5B46	0x71EE	# <CJK>
+0x5B47	0x77CD	# <CJK>
+0x5B48	0x5EF4	# <CJK>
+0x5B49	0x51F5	# <CJK>
+0x5B4A	0x51FC	# <CJK>
+0x5B4B	0x9B2F	# <CJK>
+0x5B4C	0x53B6	# <CJK>
+0x5B4D	0x5F01	# <CJK>
+0x5B4E	0x755A	# <CJK>
+0x5B4F	0x5DF0	# <CJK>
+0x5B50	0x574C	# <CJK>
+0x5B51	0x580A	# <CJK>
+0x5B52	0x57A1	# <CJK>
+0x5B53	0x587E	# <CJK>
+0x5B54	0x58BC	# <CJK>
+0x5B55	0x58C5	# <CJK>
+0x5B56	0x58D1	# <CJK>
+0x5B57	0x5729	# <CJK>
+0x5B58	0x572C	# <CJK>
+0x5B59	0x572A	# <CJK>
+0x5B5A	0x5733	# <CJK>
+0x5B5B	0x58D9	# <CJK>
+0x5B5C	0x572E	# <CJK>
+0x5B5D	0x572F	# <CJK>
+0x5B5E	0x58E2	# <CJK>
+0x5B5F	0x573B	# <CJK>
+0x5B60	0x5742	# <CJK>
+0x5B61	0x5769	# <CJK>
+0x5B62	0x58E0	# <CJK>
+0x5B63	0x576B	# <CJK>
+0x5B64	0x58DA	# <CJK>
+0x5B65	0x577C	# <CJK>
+0x5B66	0x577B	# <CJK>
+0x5B67	0x5768	# <CJK>
+0x5B68	0x576D	# <CJK>
+0x5B69	0x5776	# <CJK>
+0x5B6A	0x5773	# <CJK>
+0x5B6B	0x57E1	# <CJK>
+0x5B6C	0x57A4	# <CJK>
+0x5B6D	0x578C	# <CJK>
+0x5B6E	0x584F	# <CJK>
+0x5B6F	0x57CF	# <CJK>
+0x5B70	0x57A7	# <CJK>
+0x5B71	0x5816	# <CJK>
+0x5B72	0x5793	# <CJK>
+0x5B73	0x57A0	# <CJK>
+0x5B74	0x57D5	# <CJK>
+0x5B75	0x5852	# <CJK>
+0x5B76	0x581D	# <CJK>
+0x5B77	0x5864	# <CJK>
+0x5B78	0x57D2	# <CJK>
+0x5B79	0x57B8	# <CJK>
+0x5B7A	0x57F4	# <CJK>
+0x5B7B	0x57EF	# <CJK>
+0x5B7C	0x57F8	# <CJK>
+0x5B7D	0x57E4	# <CJK>
+0x5B7E	0x57DD	# <CJK>
+0x5C21	0x580B	# <CJK>
+0x5C22	0x580D	# <CJK>
+0x5C23	0x57FD	# <CJK>
+0x5C24	0x57ED	# <CJK>
+0x5C25	0x5800	# <CJK>
+0x5C26	0x581E	# <CJK>
+0x5C27	0x5819	# <CJK>
+0x5C28	0x5844	# <CJK>
+0x5C29	0x5820	# <CJK>
+0x5C2A	0x5865	# <CJK>
+0x5C2B	0x586C	# <CJK>
+0x5C2C	0x5881	# <CJK>
+0x5C2D	0x5889	# <CJK>
+0x5C2E	0x589A	# <CJK>
+0x5C2F	0x5880	# <CJK>
+0x5C30	0x99A8	# <CJK>
+0x5C31	0x9F19	# <CJK>
+0x5C32	0x61FF	# <CJK>
+0x5C33	0x8279	# <CJK>
+0x5C34	0x827D	# <CJK>
+0x5C35	0x827F	# <CJK>
+0x5C36	0x828F	# <CJK>
+0x5C37	0x828A	# <CJK>
+0x5C38	0x82A8	# <CJK>
+0x5C39	0x8284	# <CJK>
+0x5C3A	0x828E	# <CJK>
+0x5C3B	0x8291	# <CJK>
+0x5C3C	0x858C	# <CJK>
+0x5C3D	0x8299	# <CJK>
+0x5C3E	0x82AB	# <CJK>
+0x5C3F	0x8553	# <CJK>
+0x5C40	0x82BE	# <CJK>
+0x5C41	0x82B0	# <CJK>
+0x5C42	0x85F6	# <CJK>
+0x5C43	0x82CA	# <CJK>
+0x5C44	0x82E3	# <CJK>
+0x5C45	0x8298	# <CJK>
+0x5C46	0x82B7	# <CJK>
+0x5C47	0x82AE	# <CJK>
+0x5C48	0x83A7	# <CJK>
+0x5C49	0x8407	# <CJK>
+0x5C4A	0x84EF	# <CJK>
+0x5C4B	0x82A9	# <CJK>
+0x5C4C	0x82B4	# <CJK>
+0x5C4D	0x82A1	# <CJK>
+0x5C4E	0x82AA	# <CJK>
+0x5C4F	0x829F	# <CJK>
+0x5C50	0x82C4	# <CJK>
+0x5C51	0x82E7	# <CJK>
+0x5C52	0x82A4	# <CJK>
+0x5C53	0x82E1	# <CJK>
+0x5C54	0x8309	# <CJK>
+0x5C55	0x82F7	# <CJK>
+0x5C56	0x82E4	# <CJK>
+0x5C57	0x8622	# <CJK>
+0x5C58	0x8307	# <CJK>
+0x5C59	0x82DC	# <CJK>
+0x5C5A	0x82F4	# <CJK>
+0x5C5B	0x82D2	# <CJK>
+0x5C5C	0x82D8	# <CJK>
+0x5C5D	0x830C	# <CJK>
+0x5C5E	0x82FB	# <CJK>
+0x5C5F	0x82D3	# <CJK>
+0x5C60	0x8526	# <CJK>
+0x5C61	0x831A	# <CJK>
+0x5C62	0x8306	# <CJK>
+0x5C63	0x584B	# <CJK>
+0x5C64	0x7162	# <CJK>
+0x5C65	0x82E0	# <CJK>
+0x5C66	0x82D5	# <CJK>
+0x5C67	0x831C	# <CJK>
+0x5C68	0x8351	# <CJK>
+0x5C69	0x8558	# <CJK>
+0x5C6A	0x84FD	# <CJK>
+0x5C6B	0x8308	# <CJK>
+0x5C6C	0x8392	# <CJK>
+0x5C6D	0x833C	# <CJK>
+0x5C6E	0x8334	# <CJK>
+0x5C6F	0x8331	# <CJK>
+0x5C70	0x839B	# <CJK>
+0x5C71	0x854E	# <CJK>
+0x5C72	0x832F	# <CJK>
+0x5C73	0x834F	# <CJK>
+0x5C74	0x8347	# <CJK>
+0x5C75	0x8343	# <CJK>
+0x5C76	0x8588	# <CJK>
+0x5C77	0x8340	# <CJK>
+0x5C78	0x8317	# <CJK>
+0x5C79	0x85BA	# <CJK>
+0x5C7A	0x832D	# <CJK>
+0x5C7B	0x833A	# <CJK>
+0x5C7C	0x8333	# <CJK>
+0x5C7D	0x7296	# <CJK>
+0x5C7E	0x6ECE	# <CJK>
+0x5D21	0x8541	# <CJK>
+0x5D22	0x831B	# <CJK>
+0x5D23	0x85CE	# <CJK>
+0x5D24	0x8552	# <CJK>
+0x5D25	0x84C0	# <CJK>
+0x5D26	0x8452	# <CJK>
+0x5D27	0x8464	# <CJK>
+0x5D28	0x83B0	# <CJK>
+0x5D29	0x8378	# <CJK>
+0x5D2A	0x8494	# <CJK>
+0x5D2B	0x8435	# <CJK>
+0x5D2C	0x83A0	# <CJK>
+0x5D2D	0x83AA	# <CJK>
+0x5D2E	0x8393	# <CJK>
+0x5D2F	0x839C	# <CJK>
+0x5D30	0x8385	# <CJK>
+0x5D31	0x837C	# <CJK>
+0x5D32	0x859F	# <CJK>
+0x5D33	0x83A9	# <CJK>
+0x5D34	0x837D	# <CJK>
+0x5D35	0x8555	# <CJK>
+0x5D36	0x837B	# <CJK>
+0x5D37	0x8398	# <CJK>
+0x5D38	0x839E	# <CJK>
+0x5D39	0x83A8	# <CJK>
+0x5D3A	0x9DAF	# <CJK>
+0x5D3B	0x8493	# <CJK>
+0x5D3C	0x83C1	# <CJK>
+0x5D3D	0x8401	# <CJK>
+0x5D3E	0x83E5	# <CJK>
+0x5D3F	0x83D8	# <CJK>
+0x5D40	0x5807	# <CJK>
+0x5D41	0x8418	# <CJK>
+0x5D42	0x840B	# <CJK>
+0x5D43	0x83DD	# <CJK>
+0x5D44	0x83FD	# <CJK>
+0x5D45	0x83D6	# <CJK>
+0x5D46	0x841C	# <CJK>
+0x5D47	0x8438	# <CJK>
+0x5D48	0x8411	# <CJK>
+0x5D49	0x8406	# <CJK>
+0x5D4A	0x83D4	# <CJK>
+0x5D4B	0x83DF	# <CJK>
+0x5D4C	0x840F	# <CJK>
+0x5D4D	0x8403	# <CJK>
+0x5D4E	0x83F8	# <CJK>
+0x5D4F	0x83F9	# <CJK>
+0x5D50	0x83EA	# <CJK>
+0x5D51	0x83C5	# <CJK>
+0x5D52	0x83C0	# <CJK>
+0x5D53	0x7E08	# <CJK>
+0x5D54	0x83F0	# <CJK>
+0x5D55	0x83E1	# <CJK>
+0x5D56	0x845C	# <CJK>
+0x5D57	0x8451	# <CJK>
+0x5D58	0x845A	# <CJK>
+0x5D59	0x8459	# <CJK>
+0x5D5A	0x8473	# <CJK>
+0x5D5B	0x8546	# <CJK>
+0x5D5C	0x8488	# <CJK>
+0x5D5D	0x847A	# <CJK>
+0x5D5E	0x8562	# <CJK>
+0x5D5F	0x8478	# <CJK>
+0x5D60	0x843C	# <CJK>
+0x5D61	0x8446	# <CJK>
+0x5D62	0x8469	# <CJK>
+0x5D63	0x8476	# <CJK>
+0x5D64	0x851E	# <CJK>
+0x5D65	0x848E	# <CJK>
+0x5D66	0x8431	# <CJK>
+0x5D67	0x846D	# <CJK>
+0x5D68	0x84C1	# <CJK>
+0x5D69	0x84CD	# <CJK>
+0x5D6A	0x84D0	# <CJK>
+0x5D6B	0x9A40	# <CJK>
+0x5D6C	0x84BD	# <CJK>
+0x5D6D	0x84D3	# <CJK>
+0x5D6E	0x84CA	# <CJK>
+0x5D6F	0x84BF	# <CJK>
+0x5D70	0x84BA	# <CJK>
+0x5D71	0x863A	# <CJK>
+0x5D72	0x84A1	# <CJK>
+0x5D73	0x84B9	# <CJK>
+0x5D74	0x84B4	# <CJK>
+0x5D75	0x8497	# <CJK>
+0x5D76	0x93A3	# <CJK>
+0x5D77	0x8577	# <CJK>
+0x5D78	0x850C	# <CJK>
+0x5D79	0x750D	# <CJK>
+0x5D7A	0x8538	# <CJK>
+0x5D7B	0x84F0	# <CJK>
+0x5D7C	0x861E	# <CJK>
+0x5D7D	0x851F	# <CJK>
+0x5D7E	0x85FA	# <CJK>
+0x5E21	0x8556	# <CJK>
+0x5E22	0x853B	# <CJK>
+0x5E23	0x84FF	# <CJK>
+0x5E24	0x84FC	# <CJK>
+0x5E25	0x8559	# <CJK>
+0x5E26	0x8548	# <CJK>
+0x5E27	0x8568	# <CJK>
+0x5E28	0x8564	# <CJK>
+0x5E29	0x855E	# <CJK>
+0x5E2A	0x857A	# <CJK>
+0x5E2B	0x77A2	# <CJK>
+0x5E2C	0x8543	# <CJK>
+0x5E2D	0x8604	# <CJK>
+0x5E2E	0x857B	# <CJK>
+0x5E2F	0x85A4	# <CJK>
+0x5E30	0x85A8	# <CJK>
+0x5E31	0x8587	# <CJK>
+0x5E32	0x858F	# <CJK>
+0x5E33	0x8579	# <CJK>
+0x5E34	0x85EA	# <CJK>
+0x5E35	0x859C	# <CJK>
+0x5E36	0x8585	# <CJK>
+0x5E37	0x85B9	# <CJK>
+0x5E38	0x85B7	# <CJK>
+0x5E39	0x85B0	# <CJK>
+0x5E3A	0x861A	# <CJK>
+0x5E3B	0x85C1	# <CJK>
+0x5E3C	0x85DC	# <CJK>
+0x5E3D	0x85FF	# <CJK>
+0x5E3E	0x8627	# <CJK>
+0x5E3F	0x8605	# <CJK>
+0x5E40	0x8629	# <CJK>
+0x5E41	0x8616	# <CJK>
+0x5E42	0x863C	# <CJK>
+0x5E43	0x5EFE	# <CJK>
+0x5E44	0x5F08	# <CJK>
+0x5E45	0x593C	# <CJK>
+0x5E46	0x5969	# <CJK>
+0x5E47	0x8037	# <CJK>
+0x5E48	0x5955	# <CJK>
+0x5E49	0x595A	# <CJK>
+0x5E4A	0x5958	# <CJK>
+0x5E4B	0x530F	# <CJK>
+0x5E4C	0x5C22	# <CJK>
+0x5E4D	0x5C25	# <CJK>
+0x5E4E	0x5C2C	# <CJK>
+0x5E4F	0x5C37	# <CJK>
+0x5E50	0x624C	# <CJK>
+0x5E51	0x636B	# <CJK>
+0x5E52	0x6476	# <CJK>
+0x5E53	0x62BB	# <CJK>
+0x5E54	0x62CA	# <CJK>
+0x5E55	0x62DA	# <CJK>
+0x5E56	0x62D7	# <CJK>
+0x5E57	0x62EE	# <CJK>
+0x5E58	0x649F	# <CJK>
+0x5E59	0x62F6	# <CJK>
+0x5E5A	0x6339	# <CJK>
+0x5E5B	0x634B	# <CJK>
+0x5E5C	0x6343	# <CJK>
+0x5E5D	0x63AD	# <CJK>
+0x5E5E	0x63F6	# <CJK>
+0x5E5F	0x6371	# <CJK>
+0x5E60	0x637A	# <CJK>
+0x5E61	0x638E	# <CJK>
+0x5E62	0x6451	# <CJK>
+0x5E63	0x636D	# <CJK>
+0x5E64	0x63AC	# <CJK>
+0x5E65	0x638A	# <CJK>
+0x5E66	0x6369	# <CJK>
+0x5E67	0x63AE	# <CJK>
+0x5E68	0x645C	# <CJK>
+0x5E69	0x63F2	# <CJK>
+0x5E6A	0x63F8	# <CJK>
+0x5E6B	0x63E0	# <CJK>
+0x5E6C	0x64B3	# <CJK>
+0x5E6D	0x63C4	# <CJK>
+0x5E6E	0x63DE	# <CJK>
+0x5E6F	0x63CE	# <CJK>
+0x5E70	0x6452	# <CJK>
+0x5E71	0x63C6	# <CJK>
+0x5E72	0x63BE	# <CJK>
+0x5E73	0x6504	# <CJK>
+0x5E74	0x6441	# <CJK>
+0x5E75	0x640B	# <CJK>
+0x5E76	0x641B	# <CJK>
+0x5E77	0x6420	# <CJK>
+0x5E78	0x640C	# <CJK>
+0x5E79	0x6426	# <CJK>
+0x5E7A	0x6421	# <CJK>
+0x5E7B	0x645E	# <CJK>
+0x5E7C	0x6516	# <CJK>
+0x5E7D	0x646D	# <CJK>
+0x5E7E	0x6496	# <CJK>
+0x5F21	0x647A	# <CJK>
+0x5F22	0x64F7	# <CJK>
+0x5F23	0x64FC	# <CJK>
+0x5F24	0x6499	# <CJK>
+0x5F25	0x651B	# <CJK>
+0x5F26	0x64C0	# <CJK>
+0x5F27	0x64D0	# <CJK>
+0x5F28	0x64D7	# <CJK>
+0x5F29	0x64E4	# <CJK>
+0x5F2A	0x64E2	# <CJK>
+0x5F2B	0x6509	# <CJK>
+0x5F2C	0x6525	# <CJK>
+0x5F2D	0x652E	# <CJK>
+0x5F2E	0x5F0B	# <CJK>
+0x5F2F	0x5FD2	# <CJK>
+0x5F30	0x7519	# <CJK>
+0x5F31	0x5F11	# <CJK>
+0x5F32	0x535F	# <CJK>
+0x5F33	0x53F1	# <CJK>
+0x5F34	0x5630	# <CJK>
+0x5F35	0x53E9	# <CJK>
+0x5F36	0x53E8	# <CJK>
+0x5F37	0x53FB	# <CJK>
+0x5F38	0x5412	# <CJK>
+0x5F39	0x5416	# <CJK>
+0x5F3A	0x5406	# <CJK>
+0x5F3B	0x544B	# <CJK>
+0x5F3C	0x5638	# <CJK>
+0x5F3D	0x56C8	# <CJK>
+0x5F3E	0x5454	# <CJK>
+0x5F3F	0x56A6	# <CJK>
+0x5F40	0x5443	# <CJK>
+0x5F41	0x5421	# <CJK>
+0x5F42	0x5504	# <CJK>
+0x5F43	0x54BC	# <CJK>
+0x5F44	0x5423	# <CJK>
+0x5F45	0x5432	# <CJK>
+0x5F46	0x5482	# <CJK>
+0x5F47	0x5494	# <CJK>
+0x5F48	0x5477	# <CJK>
+0x5F49	0x5471	# <CJK>
+0x5F4A	0x5464	# <CJK>
+0x5F4B	0x549A	# <CJK>
+0x5F4C	0x5680	# <CJK>
+0x5F4D	0x5484	# <CJK>
+0x5F4E	0x5476	# <CJK>
+0x5F4F	0x5466	# <CJK>
+0x5F50	0x565D	# <CJK>
+0x5F51	0x54D0	# <CJK>
+0x5F52	0x54AD	# <CJK>
+0x5F53	0x54C2	# <CJK>
+0x5F54	0x54B4	# <CJK>
+0x5F55	0x5660	# <CJK>
+0x5F56	0x54A7	# <CJK>
+0x5F57	0x54A6	# <CJK>
+0x5F58	0x5635	# <CJK>
+0x5F59	0x55F6	# <CJK>
+0x5F5A	0x5472	# <CJK>
+0x5F5B	0x54A3	# <CJK>
+0x5F5C	0x5666	# <CJK>
+0x5F5D	0x54BB	# <CJK>
+0x5F5E	0x54BF	# <CJK>
+0x5F5F	0x54CC	# <CJK>
+0x5F60	0x5672	# <CJK>
+0x5F61	0x54DA	# <CJK>
+0x5F62	0x568C	# <CJK>
+0x5F63	0x54A9	# <CJK>
+0x5F64	0x54AA	# <CJK>
+0x5F65	0x54A4	# <CJK>
+0x5F66	0x5665	# <CJK>
+0x5F67	0x54CF	# <CJK>
+0x5F68	0x54DE	# <CJK>
+0x5F69	0x561C	# <CJK>
+0x5F6A	0x54E7	# <CJK>
+0x5F6B	0x562E	# <CJK>
+0x5F6C	0x54FD	# <CJK>
+0x5F6D	0x5514	# <CJK>
+0x5F6E	0x54F3	# <CJK>
+0x5F6F	0x55E9	# <CJK>
+0x5F70	0x5523	# <CJK>
+0x5F71	0x550F	# <CJK>
+0x5F72	0x5511	# <CJK>
+0x5F73	0x5527	# <CJK>
+0x5F74	0x552A	# <CJK>
+0x5F75	0x5616	# <CJK>
+0x5F76	0x558F	# <CJK>
+0x5F77	0x55B5	# <CJK>
+0x5F78	0x5549	# <CJK>
+0x5F79	0x56C0	# <CJK>
+0x5F7A	0x5541	# <CJK>
+0x5F7B	0x5555	# <CJK>
+0x5F7C	0x553F	# <CJK>
+0x5F7D	0x5550	# <CJK>
+0x5F7E	0x553C	# <CJK>
+0x6021	0x5537	# <CJK>
+0x6022	0x5556	# <CJK>
+0x6023	0x5575	# <CJK>
+0x6024	0x5576	# <CJK>
+0x6025	0x5577	# <CJK>
+0x6026	0x5533	# <CJK>
+0x6027	0x5530	# <CJK>
+0x6028	0x555C	# <CJK>
+0x6029	0x558B	# <CJK>
+0x602A	0x55D2	# <CJK>
+0x602B	0x5583	# <CJK>
+0x602C	0x55B1	# <CJK>
+0x602D	0x55B9	# <CJK>
+0x602E	0x5588	# <CJK>
+0x602F	0x5581	# <CJK>
+0x6030	0x559F	# <CJK>
+0x6031	0x557E	# <CJK>
+0x6032	0x55D6	# <CJK>
+0x6033	0x5591	# <CJK>
+0x6034	0x557B	# <CJK>
+0x6035	0x55DF	# <CJK>
+0x6036	0x560D	# <CJK>
+0x6037	0x56B3	# <CJK>
+0x6038	0x5594	# <CJK>
+0x6039	0x5599	# <CJK>
+0x603A	0x55EA	# <CJK>
+0x603B	0x55F7	# <CJK>
+0x603C	0x55C9	# <CJK>
+0x603D	0x561F	# <CJK>
+0x603E	0x55D1	# <CJK>
+0x603F	0x56C1	# <CJK>
+0x6040	0x55EC	# <CJK>
+0x6041	0x55D4	# <CJK>
+0x6042	0x55E6	# <CJK>
+0x6043	0x55DD	# <CJK>
+0x6044	0x55C4	# <CJK>
+0x6045	0x55EF	# <CJK>
+0x6046	0x55E5	# <CJK>
+0x6047	0x55F2	# <CJK>
+0x6048	0x566F	# <CJK>
+0x6049	0x55CC	# <CJK>
+0x604A	0x55CD	# <CJK>
+0x604B	0x55E8	# <CJK>
+0x604C	0x55F5	# <CJK>
+0x604D	0x55E4	# <CJK>
+0x604E	0x8F61	# <CJK>
+0x604F	0x561E	# <CJK>
+0x6050	0x5608	# <CJK>
+0x6051	0x560C	# <CJK>
+0x6052	0x5601	# <CJK>
+0x6053	0x56B6	# <CJK>
+0x6054	0x5623	# <CJK>
+0x6055	0x55FE	# <CJK>
+0x6056	0x5600	# <CJK>
+0x6057	0x5627	# <CJK>
+0x6058	0x562D	# <CJK>
+0x6059	0x5658	# <CJK>
+0x605A	0x5639	# <CJK>
+0x605B	0x5657	# <CJK>
+0x605C	0x562C	# <CJK>
+0x605D	0x564D	# <CJK>
+0x605E	0x5662	# <CJK>
+0x605F	0x5659	# <CJK>
+0x6060	0x5695	# <CJK>
+0x6061	0x564C	# <CJK>
+0x6062	0x5654	# <CJK>
+0x6063	0x5686	# <CJK>
+0x6064	0x5664	# <CJK>
+0x6065	0x5671	# <CJK>
+0x6066	0x566B	# <CJK>
+0x6067	0x567B	# <CJK>
+0x6068	0x567C	# <CJK>
+0x6069	0x5685	# <CJK>
+0x606A	0x5693	# <CJK>
+0x606B	0x56AF	# <CJK>
+0x606C	0x56D4	# <CJK>
+0x606D	0x56D7	# <CJK>
+0x606E	0x56DD	# <CJK>
+0x606F	0x56E1	# <CJK>
+0x6070	0x5707	# <CJK>
+0x6071	0x56EB	# <CJK>
+0x6072	0x56F9	# <CJK>
+0x6073	0x56FF	# <CJK>
+0x6074	0x5704	# <CJK>
+0x6075	0x570A	# <CJK>
+0x6076	0x5709	# <CJK>
+0x6077	0x571C	# <CJK>
+0x6078	0x5E43	# <CJK>
+0x6079	0x5E19	# <CJK>
+0x607A	0x5E14	# <CJK>
+0x607B	0x5E11	# <CJK>
+0x607C	0x5E6C	# <CJK>
+0x607D	0x5E58	# <CJK>
+0x607E	0x5E57	# <CJK>
+0x6121	0x5E37	# <CJK>
+0x6122	0x5E44	# <CJK>
+0x6123	0x5E54	# <CJK>
+0x6124	0x5E5B	# <CJK>
+0x6125	0x5E5E	# <CJK>
+0x6126	0x5E61	# <CJK>
+0x6127	0x5C8C	# <CJK>
+0x6128	0x5C7A	# <CJK>
+0x6129	0x5C8D	# <CJK>
+0x612A	0x5C90	# <CJK>
+0x612B	0x5D87	# <CJK>
+0x612C	0x5C88	# <CJK>
+0x612D	0x5CF4	# <CJK>
+0x612E	0x5C99	# <CJK>
+0x612F	0x5C91	# <CJK>
+0x6130	0x5D50	# <CJK>
+0x6131	0x5C9C	# <CJK>
+0x6132	0x5CB5	# <CJK>
+0x6133	0x5CA2	# <CJK>
+0x6134	0x5D2C	# <CJK>
+0x6135	0x5CAC	# <CJK>
+0x6136	0x5CAB	# <CJK>
+0x6137	0x5CB1	# <CJK>
+0x6138	0x5CA3	# <CJK>
+0x6139	0x5CC1	# <CJK>
+0x613A	0x5CB7	# <CJK>
+0x613B	0x5DA7	# <CJK>
+0x613C	0x5CD2	# <CJK>
+0x613D	0x5DA0	# <CJK>
+0x613E	0x5CCB	# <CJK>
+0x613F	0x5D22	# <CJK>
+0x6140	0x5D97	# <CJK>
+0x6141	0x5D0D	# <CJK>
+0x6142	0x5D27	# <CJK>
+0x6143	0x5D26	# <CJK>
+0x6144	0x5D2E	# <CJK>
+0x6145	0x5D24	# <CJK>
+0x6146	0x5D1E	# <CJK>
+0x6147	0x5D06	# <CJK>
+0x6148	0x5D1B	# <CJK>
+0x6149	0x5DB8	# <CJK>
+0x614A	0x5D3E	# <CJK>
+0x614B	0x5D34	# <CJK>
+0x614C	0x5D3D	# <CJK>
+0x614D	0x5D6C	# <CJK>
+0x614E	0x5D5B	# <CJK>
+0x614F	0x5D6F	# <CJK>
+0x6150	0x5D81	# <CJK>
+0x6151	0x5D6B	# <CJK>
+0x6152	0x5D4B	# <CJK>
+0x6153	0x5D4A	# <CJK>
+0x6154	0x5D69	# <CJK>
+0x6155	0x5D74	# <CJK>
+0x6156	0x5D82	# <CJK>
+0x6157	0x5D99	# <CJK>
+0x6158	0x5D9D	# <CJK>
+0x6159	0x8C73	# <CJK>
+0x615A	0x5DB7	# <CJK>
+0x615B	0x5DD4	# <CJK>
+0x615C	0x5F73	# <CJK>
+0x615D	0x5F77	# <CJK>
+0x615E	0x5F82	# <CJK>
+0x615F	0x5F87	# <CJK>
+0x6160	0x5F89	# <CJK>
+0x6161	0x540E	# <CJK>
+0x6162	0x5FA0	# <CJK>
+0x6163	0x5F99	# <CJK>
+0x6164	0x5F9C	# <CJK>
+0x6165	0x5FA8	# <CJK>
+0x6166	0x5FAD	# <CJK>
+0x6167	0x5FB5	# <CJK>
+0x6168	0x5FBC	# <CJK>
+0x6169	0x8862	# <CJK>
+0x616A	0x5F61	# <CJK>
+0x616B	0x72AD	# <CJK>
+0x616C	0x72B0	# <CJK>
+0x616D	0x72B4	# <CJK>
+0x616E	0x7377	# <CJK>
+0x616F	0x7341	# <CJK>
+0x6170	0x72C3	# <CJK>
+0x6171	0x72C1	# <CJK>
+0x6172	0x72CE	# <CJK>
+0x6173	0x72CD	# <CJK>
+0x6174	0x72D2	# <CJK>
+0x6175	0x72E8	# <CJK>
+0x6176	0x736A	# <CJK>
+0x6177	0x72E9	# <CJK>
+0x6178	0x733B	# <CJK>
+0x6179	0x72F4	# <CJK>
+0x617A	0x72F7	# <CJK>
+0x617B	0x7301	# <CJK>
+0x617C	0x72F3	# <CJK>
+0x617D	0x736B	# <CJK>
+0x617E	0x72FA	# <CJK>
+0x6221	0x72FB	# <CJK>
+0x6222	0x7317	# <CJK>
+0x6223	0x7313	# <CJK>
+0x6224	0x7380	# <CJK>
+0x6225	0x730A	# <CJK>
+0x6226	0x731E	# <CJK>
+0x6227	0x731D	# <CJK>
+0x6228	0x737C	# <CJK>
+0x6229	0x7322	# <CJK>
+0x622A	0x7339	# <CJK>
+0x622B	0x7325	# <CJK>
+0x622C	0x732C	# <CJK>
+0x622D	0x7338	# <CJK>
+0x622E	0x7331	# <CJK>
+0x622F	0x7350	# <CJK>
+0x6230	0x734D	# <CJK>
+0x6231	0x7357	# <CJK>
+0x6232	0x7360	# <CJK>
+0x6233	0x736C	# <CJK>
+0x6234	0x736F	# <CJK>
+0x6235	0x737E	# <CJK>
+0x6236	0x821B	# <CJK>
+0x6237	0x5925	# <CJK>
+0x6238	0x98E7	# <CJK>
+0x6239	0x5924	# <CJK>
+0x623A	0x5902	# <CJK>
+0x623B	0x98E0	# <CJK>
+0x623C	0x9933	# <CJK>
+0x623D	0x98E9	# <CJK>
+0x623E	0x993C	# <CJK>
+0x623F	0x98EA	# <CJK>
+0x6240	0x98EB	# <CJK>
+0x6241	0x98ED	# <CJK>
+0x6242	0x98F4	# <CJK>
+0x6243	0x9909	# <CJK>
+0x6244	0x9911	# <CJK>
+0x6245	0x4F59	# <CJK>
+0x6246	0x991B	# <CJK>
+0x6247	0x9937	# <CJK>
+0x6248	0x993F	# <CJK>
+0x6249	0x9943	# <CJK>
+0x624A	0x9948	# <CJK>
+0x624B	0x9949	# <CJK>
+0x624C	0x994A	# <CJK>
+0x624D	0x994C	# <CJK>
+0x624E	0x9962	# <CJK>
+0x624F	0x5E80	# <CJK>
+0x6250	0x5EE1	# <CJK>
+0x6251	0x5E8B	# <CJK>
+0x6252	0x5E96	# <CJK>
+0x6253	0x5EA5	# <CJK>
+0x6254	0x5EA0	# <CJK>
+0x6255	0x5EB9	# <CJK>
+0x6256	0x5EB5	# <CJK>
+0x6257	0x5EBE	# <CJK>
+0x6258	0x5EB3	# <CJK>
+0x6259	0x8CE1	# <CJK>
+0x625A	0x5ED2	# <CJK>
+0x625B	0x5ED1	# <CJK>
+0x625C	0x5EDB	# <CJK>
+0x625D	0x5EE8	# <CJK>
+0x625E	0x5EEA	# <CJK>
+0x625F	0x81BA	# <CJK>
+0x6260	0x5FC4	# <CJK>
+0x6261	0x5FC9	# <CJK>
+0x6262	0x5FD6	# <CJK>
+0x6263	0x61FA	# <CJK>
+0x6264	0x61AE	# <CJK>
+0x6265	0x5FEE	# <CJK>
+0x6266	0x616A	# <CJK>
+0x6267	0x5FE1	# <CJK>
+0x6268	0x5FE4	# <CJK>
+0x6269	0x613E	# <CJK>
+0x626A	0x60B5	# <CJK>
+0x626B	0x6134	# <CJK>
+0x626C	0x5FEA	# <CJK>
+0x626D	0x5FED	# <CJK>
+0x626E	0x5FF8	# <CJK>
+0x626F	0x6019	# <CJK>
+0x6270	0x6035	# <CJK>
+0x6271	0x6026	# <CJK>
+0x6272	0x601B	# <CJK>
+0x6273	0x600F	# <CJK>
+0x6274	0x600D	# <CJK>
+0x6275	0x6029	# <CJK>
+0x6276	0x602B	# <CJK>
+0x6277	0x600A	# <CJK>
+0x6278	0x61CC	# <CJK>
+0x6279	0x6021	# <CJK>
+0x627A	0x615F	# <CJK>
+0x627B	0x61E8	# <CJK>
+0x627C	0x60FB	# <CJK>
+0x627D	0x6137	# <CJK>
+0x627E	0x6042	# <CJK>
+0x6321	0x606A	# <CJK>
+0x6322	0x60F2	# <CJK>
+0x6323	0x6096	# <CJK>
+0x6324	0x609A	# <CJK>
+0x6325	0x6173	# <CJK>
+0x6326	0x609D	# <CJK>
+0x6327	0x6083	# <CJK>
+0x6328	0x6092	# <CJK>
+0x6329	0x608C	# <CJK>
+0x632A	0x609B	# <CJK>
+0x632B	0x611C	# <CJK>
+0x632C	0x60BB	# <CJK>
+0x632D	0x60B1	# <CJK>
+0x632E	0x60DD	# <CJK>
+0x632F	0x60D8	# <CJK>
+0x6330	0x60C6	# <CJK>
+0x6331	0x60DA	# <CJK>
+0x6332	0x60B4	# <CJK>
+0x6333	0x6120	# <CJK>
+0x6334	0x6192	# <CJK>
+0x6335	0x6115	# <CJK>
+0x6336	0x6123	# <CJK>
+0x6337	0x60F4	# <CJK>
+0x6338	0x6100	# <CJK>
+0x6339	0x610E	# <CJK>
+0x633A	0x612B	# <CJK>
+0x633B	0x614A	# <CJK>
+0x633C	0x6175	# <CJK>
+0x633D	0x61AC	# <CJK>
+0x633E	0x6194	# <CJK>
+0x633F	0x61A7	# <CJK>
+0x6340	0x61B7	# <CJK>
+0x6341	0x61D4	# <CJK>
+0x6342	0x61F5	# <CJK>
+0x6343	0x5FDD	# <CJK>
+0x6344	0x96B3	# <CJK>
+0x6345	0x9582	# <CJK>
+0x6346	0x9586	# <CJK>
+0x6347	0x95C8	# <CJK>
+0x6348	0x958E	# <CJK>
+0x6349	0x9594	# <CJK>
+0x634A	0x958C	# <CJK>
+0x634B	0x95E5	# <CJK>
+0x634C	0x95AD	# <CJK>
+0x634D	0x95AB	# <CJK>
+0x634E	0x9B2E	# <CJK>
+0x634F	0x95AC	# <CJK>
+0x6350	0x95BE	# <CJK>
+0x6351	0x95B6	# <CJK>
+0x6352	0x9B29	# <CJK>
+0x6353	0x95BF	# <CJK>
+0x6354	0x95BD	# <CJK>
+0x6355	0x95BC	# <CJK>
+0x6356	0x95C3	# <CJK>
+0x6357	0x95CB	# <CJK>
+0x6358	0x95D4	# <CJK>
+0x6359	0x95D0	# <CJK>
+0x635A	0x95D5	# <CJK>
+0x635B	0x95DE	# <CJK>
+0x635C	0x4E2C	# <CJK>
+0x635D	0x723F	# <CJK>
+0x635E	0x6215	# <CJK>
+0x635F	0x6C35	# <CJK>
+0x6360	0x6C54	# <CJK>
+0x6361	0x6C5C	# <CJK>
+0x6362	0x6C4A	# <CJK>
+0x6363	0x7043	# <CJK>
+0x6364	0x6C85	# <CJK>
+0x6365	0x6C90	# <CJK>
+0x6366	0x6C94	# <CJK>
+0x6367	0x6C8C	# <CJK>
+0x6368	0x6C68	# <CJK>
+0x6369	0x6C69	# <CJK>
+0x636A	0x6C74	# <CJK>
+0x636B	0x6C76	# <CJK>
+0x636C	0x6C86	# <CJK>
+0x636D	0x6F59	# <CJK>
+0x636E	0x6CD0	# <CJK>
+0x636F	0x6CD4	# <CJK>
+0x6370	0x6CAD	# <CJK>
+0x6371	0x7027	# <CJK>
+0x6372	0x7018	# <CJK>
+0x6373	0x6CF1	# <CJK>
+0x6374	0x6CD7	# <CJK>
+0x6375	0x6CB2	# <CJK>
+0x6376	0x6CE0	# <CJK>
+0x6377	0x6CD6	# <CJK>
+0x6378	0x6FFC	# <CJK>
+0x6379	0x6CEB	# <CJK>
+0x637A	0x6CEE	# <CJK>
+0x637B	0x6CB1	# <CJK>
+0x637C	0x6CD3	# <CJK>
+0x637D	0x6CEF	# <CJK>
+0x637E	0x6D87	# <CJK>
+0x6421	0x6D39	# <CJK>
+0x6422	0x6D27	# <CJK>
+0x6423	0x6D0C	# <CJK>
+0x6424	0x6D79	# <CJK>
+0x6425	0x6E5E	# <CJK>
+0x6426	0x6D07	# <CJK>
+0x6427	0x6D04	# <CJK>
+0x6428	0x6D19	# <CJK>
+0x6429	0x6D0E	# <CJK>
+0x642A	0x6D2B	# <CJK>
+0x642B	0x6FAE	# <CJK>
+0x642C	0x6D2E	# <CJK>
+0x642D	0x6D35	# <CJK>
+0x642E	0x6D1A	# <CJK>
+0x642F	0x700F	# <CJK>
+0x6430	0x6EF8	# <CJK>
+0x6431	0x6F6F	# <CJK>
+0x6432	0x6D33	# <CJK>
+0x6433	0x6D91	# <CJK>
+0x6434	0x6D6F	# <CJK>
+0x6435	0x6DF6	# <CJK>
+0x6436	0x6F7F	# <CJK>
+0x6437	0x6D5E	# <CJK>
+0x6438	0x6D93	# <CJK>
+0x6439	0x6D94	# <CJK>
+0x643A	0x6D5C	# <CJK>
+0x643B	0x6D60	# <CJK>
+0x643C	0x6D7C	# <CJK>
+0x643D	0x6D63	# <CJK>
+0x643E	0x6E1A	# <CJK>
+0x643F	0x6DC7	# <CJK>
+0x6440	0x6DC5	# <CJK>
+0x6441	0x6DDE	# <CJK>
+0x6442	0x7006	# <CJK>
+0x6443	0x6DBF	# <CJK>
+0x6444	0x6DE0	# <CJK>
+0x6445	0x6FA0	# <CJK>
+0x6446	0x6DE6	# <CJK>
+0x6447	0x6DDD	# <CJK>
+0x6448	0x6DD9	# <CJK>
+0x6449	0x700B	# <CJK>
+0x644A	0x6DAB	# <CJK>
+0x644B	0x6E0C	# <CJK>
+0x644C	0x6DAE	# <CJK>
+0x644D	0x6E2B	# <CJK>
+0x644E	0x6E6E	# <CJK>
+0x644F	0x6E4E	# <CJK>
+0x6450	0x6E6B	# <CJK>
+0x6451	0x6EB2	# <CJK>
+0x6452	0x6E5F	# <CJK>
+0x6453	0x6E86	# <CJK>
+0x6454	0x6E53	# <CJK>
+0x6455	0x6E54	# <CJK>
+0x6456	0x6E32	# <CJK>
+0x6457	0x6E25	# <CJK>
+0x6458	0x6E44	# <CJK>
+0x6459	0x7067	# <CJK>
+0x645A	0x6EB1	# <CJK>
+0x645B	0x6E98	# <CJK>
+0x645C	0x7044	# <CJK>
+0x645D	0x6F2D	# <CJK>
+0x645E	0x7005	# <CJK>
+0x645F	0x6EA5	# <CJK>
+0x6460	0x6EA7	# <CJK>
+0x6461	0x6EBD	# <CJK>
+0x6462	0x6EBB	# <CJK>
+0x6463	0x6EB7	# <CJK>
+0x6464	0x6F77	# <CJK>
+0x6465	0x6EB4	# <CJK>
+0x6466	0x6ECF	# <CJK>
+0x6467	0x6E8F	# <CJK>
+0x6468	0x6EC2	# <CJK>
+0x6469	0x6E9F	# <CJK>
+0x646A	0x6F62	# <CJK>
+0x646B	0x7020	# <CJK>
+0x646C	0x701F	# <CJK>
+0x646D	0x6F24	# <CJK>
+0x646E	0x6F15	# <CJK>
+0x646F	0x6EF9	# <CJK>
+0x6470	0x6F2F	# <CJK>
+0x6471	0x6F36	# <CJK>
+0x6472	0x7032	# <CJK>
+0x6473	0x6F74	# <CJK>
+0x6474	0x6F2A	# <CJK>
+0x6475	0x6F09	# <CJK>
+0x6476	0x6F29	# <CJK>
+0x6477	0x6F89	# <CJK>
+0x6478	0x6F8D	# <CJK>
+0x6479	0x6F8C	# <CJK>
+0x647A	0x6F78	# <CJK>
+0x647B	0x6F72	# <CJK>
+0x647C	0x6F7C	# <CJK>
+0x647D	0x6F7A	# <CJK>
+0x647E	0x7028	# <CJK>
+0x6521	0x6FC9	# <CJK>
+0x6522	0x6FA7	# <CJK>
+0x6523	0x6FB9	# <CJK>
+0x6524	0x6FB6	# <CJK>
+0x6525	0x6FC2	# <CJK>
+0x6526	0x6FE1	# <CJK>
+0x6527	0x6FEE	# <CJK>
+0x6528	0x6FDE	# <CJK>
+0x6529	0x6FE0	# <CJK>
+0x652A	0x6FEF	# <CJK>
+0x652B	0x701A	# <CJK>
+0x652C	0x7023	# <CJK>
+0x652D	0x701B	# <CJK>
+0x652E	0x7039	# <CJK>
+0x652F	0x7035	# <CJK>
+0x6530	0x705D	# <CJK>
+0x6531	0x705E	# <CJK>
+0x6532	0x5B80	# <CJK>
+0x6533	0x5B84	# <CJK>
+0x6534	0x5B95	# <CJK>
+0x6535	0x5B93	# <CJK>
+0x6536	0x5BA5	# <CJK>
+0x6537	0x5BB8	# <CJK>
+0x6538	0x752F	# <CJK>
+0x6539	0x9A2B	# <CJK>
+0x653A	0x6434	# <CJK>
+0x653B	0x5BE4	# <CJK>
+0x653C	0x5BEE	# <CJK>
+0x653D	0x8930	# <CJK>
+0x653E	0x5BF0	# <CJK>
+0x653F	0x8E47	# <CJK>
+0x6540	0x8B07	# <CJK>
+0x6541	0x8FB6	# <CJK>
+0x6542	0x8FD3	# <CJK>
+0x6543	0x8FD5	# <CJK>
+0x6544	0x8FE5	# <CJK>
+0x6545	0x8FEE	# <CJK>
+0x6546	0x8FE4	# <CJK>
+0x6547	0x9087	# <CJK>
+0x6548	0x8FE6	# <CJK>
+0x6549	0x9015	# <CJK>
+0x654A	0x8FE8	# <CJK>
+0x654B	0x9005	# <CJK>
+0x654C	0x9004	# <CJK>
+0x654D	0x900B	# <CJK>
+0x654E	0x9090	# <CJK>
+0x654F	0x9011	# <CJK>
+0x6550	0x900D	# <CJK>
+0x6551	0x9016	# <CJK>
+0x6552	0x9021	# <CJK>
+0x6553	0x9035	# <CJK>
+0x6554	0x9036	# <CJK>
+0x6555	0x902D	# <CJK>
+0x6556	0x902F	# <CJK>
+0x6557	0x9044	# <CJK>
+0x6558	0x9051	# <CJK>
+0x6559	0x9052	# <CJK>
+0x655A	0x9050	# <CJK>
+0x655B	0x9068	# <CJK>
+0x655C	0x9058	# <CJK>
+0x655D	0x9062	# <CJK>
+0x655E	0x905B	# <CJK>
+0x655F	0x66B9	# <CJK>
+0x6560	0x9074	# <CJK>
+0x6561	0x907D	# <CJK>
+0x6562	0x9082	# <CJK>
+0x6563	0x9088	# <CJK>
+0x6564	0x9083	# <CJK>
+0x6565	0x908B	# <CJK>
+0x6566	0x5F50	# <CJK>
+0x6567	0x5F57	# <CJK>
+0x6568	0x5F56	# <CJK>
+0x6569	0x5F58	# <CJK>
+0x656A	0x5C3B	# <CJK>
+0x656B	0x54AB	# <CJK>
+0x656C	0x5C50	# <CJK>
+0x656D	0x5C59	# <CJK>
+0x656E	0x5B71	# <CJK>
+0x656F	0x5C63	# <CJK>
+0x6570	0x5C68	# <CJK>
+0x6571	0x7FBC	# <CJK>
+0x6572	0x5F33	# <CJK>
+0x6573	0x5F29	# <CJK>
+0x6574	0x5F2D	# <CJK>
+0x6575	0x8274	# <CJK>
+0x6576	0x5F3C	# <CJK>
+0x6577	0x9B3B	# <CJK>
+0x6578	0x5C6E	# <CJK>
+0x6579	0x5981	# <CJK>
+0x657A	0x5983	# <CJK>
+0x657B	0x598D	# <CJK>
+0x657C	0x5AF5	# <CJK>
+0x657D	0x5AD7	# <CJK>
+0x657E	0x59A3	# <CJK>
+0x6621	0x5997	# <CJK>
+0x6622	0x59CA	# <CJK>
+0x6623	0x5B00	# <CJK>
+0x6624	0x599E	# <CJK>
+0x6625	0x59A4	# <CJK>
+0x6626	0x59D2	# <CJK>
+0x6627	0x59B2	# <CJK>
+0x6628	0x59AF	# <CJK>
+0x6629	0x59D7	# <CJK>
+0x662A	0x59BE	# <CJK>
+0x662B	0x5A6D	# <CJK>
+0x662C	0x5B08	# <CJK>
+0x662D	0x59DD	# <CJK>
+0x662E	0x5B4C	# <CJK>
+0x662F	0x59E3	# <CJK>
+0x6630	0x59D8	# <CJK>
+0x6631	0x59F9	# <CJK>
+0x6632	0x5A0C	# <CJK>
+0x6633	0x5A09	# <CJK>
+0x6634	0x5AA7	# <CJK>
+0x6635	0x5AFB	# <CJK>
+0x6636	0x5A11	# <CJK>
+0x6637	0x5A23	# <CJK>
+0x6638	0x5A13	# <CJK>
+0x6639	0x5A40	# <CJK>
+0x663A	0x5A67	# <CJK>
+0x663B	0x5A4A	# <CJK>
+0x663C	0x5A55	# <CJK>
+0x663D	0x5A3C	# <CJK>
+0x663E	0x5A62	# <CJK>
+0x663F	0x5B0B	# <CJK>
+0x6640	0x80EC	# <CJK>
+0x6641	0x5AAA	# <CJK>
+0x6642	0x5A9B	# <CJK>
+0x6643	0x5A77	# <CJK>
+0x6644	0x5A7A	# <CJK>
+0x6645	0x5ABE	# <CJK>
+0x6646	0x5AEB	# <CJK>
+0x6647	0x5AB2	# <CJK>
+0x6648	0x5B21	# <CJK>
+0x6649	0x5B2A	# <CJK>
+0x664A	0x5AB8	# <CJK>
+0x664B	0x5AE0	# <CJK>
+0x664C	0x5AE3	# <CJK>
+0x664D	0x5B19	# <CJK>
+0x664E	0x5AD6	# <CJK>
+0x664F	0x5AE6	# <CJK>
+0x6650	0x5AD8	# <CJK>
+0x6651	0x5ADC	# <CJK>
+0x6652	0x5B09	# <CJK>
+0x6653	0x5B17	# <CJK>
+0x6654	0x5B16	# <CJK>
+0x6655	0x5B32	# <CJK>
+0x6656	0x5B37	# <CJK>
+0x6657	0x5B40	# <CJK>
+0x6658	0x5C15	# <CJK>
+0x6659	0x5C1C	# <CJK>
+0x665A	0x5B5A	# <CJK>
+0x665B	0x5B65	# <CJK>
+0x665C	0x5B73	# <CJK>
+0x665D	0x5B51	# <CJK>
+0x665E	0x5B53	# <CJK>
+0x665F	0x5B62	# <CJK>
+0x6660	0x99D4	# <CJK>
+0x6661	0x99DF	# <CJK>
+0x6662	0x99D9	# <CJK>
+0x6663	0x9A36	# <CJK>
+0x6664	0x9A5B	# <CJK>
+0x6665	0x99D1	# <CJK>
+0x6666	0x99D8	# <CJK>
+0x6667	0x9A4D	# <CJK>
+0x6668	0x9A4A	# <CJK>
+0x6669	0x99E2	# <CJK>
+0x666A	0x9A6A	# <CJK>
+0x666B	0x9A0F	# <CJK>
+0x666C	0x9A0D	# <CJK>
+0x666D	0x9A05	# <CJK>
+0x666E	0x9A42	# <CJK>
+0x666F	0x9A2D	# <CJK>
+0x6670	0x9A16	# <CJK>
+0x6671	0x9A41	# <CJK>
+0x6672	0x9A2E	# <CJK>
+0x6673	0x9A38	# <CJK>
+0x6674	0x9A43	# <CJK>
+0x6675	0x9A44	# <CJK>
+0x6676	0x9A4F	# <CJK>
+0x6677	0x9A65	# <CJK>
+0x6678	0x9A64	# <CJK>
+0x6679	0x7CF9	# <CJK>
+0x667A	0x7D06	# <CJK>
+0x667B	0x7D02	# <CJK>
+0x667C	0x7D07	# <CJK>
+0x667D	0x7D08	# <CJK>
+0x667E	0x7E8A	# <CJK>
+0x6721	0x7D1C	# <CJK>
+0x6722	0x7D15	# <CJK>
+0x6723	0x7D13	# <CJK>
+0x6724	0x7D3A	# <CJK>
+0x6725	0x7D32	# <CJK>
+0x6726	0x7D31	# <CJK>
+0x6727	0x7E10	# <CJK>
+0x6728	0x7D3C	# <CJK>
+0x6729	0x7D40	# <CJK>
+0x672A	0x7D3F	# <CJK>
+0x672B	0x7D5D	# <CJK>
+0x672C	0x7D4E	# <CJK>
+0x672D	0x7D73	# <CJK>
+0x672E	0x7D86	# <CJK>
+0x672F	0x7D83	# <CJK>
+0x6730	0x7D88	# <CJK>
+0x6731	0x7DBE	# <CJK>
+0x6732	0x7DBA	# <CJK>
+0x6733	0x7DCB	# <CJK>
+0x6734	0x7DD4	# <CJK>
+0x6735	0x7DC4	# <CJK>
+0x6736	0x7D9E	# <CJK>
+0x6737	0x7DAC	# <CJK>
+0x6738	0x7DB9	# <CJK>
+0x6739	0x7DA3	# <CJK>
+0x673A	0x7DB0	# <CJK>
+0x673B	0x7DC7	# <CJK>
+0x673C	0x7DD9	# <CJK>
+0x673D	0x7DD7	# <CJK>
+0x673E	0x7DF9	# <CJK>
+0x673F	0x7DF2	# <CJK>
+0x6740	0x7E62	# <CJK>
+0x6741	0x7DE6	# <CJK>
+0x6742	0x7DF6	# <CJK>
+0x6743	0x7DF1	# <CJK>
+0x6744	0x7E0B	# <CJK>
+0x6745	0x7DE1	# <CJK>
+0x6746	0x7E09	# <CJK>
+0x6747	0x7E1D	# <CJK>
+0x6748	0x7E1F	# <CJK>
+0x6749	0x7E1E	# <CJK>
+0x674A	0x7E2D	# <CJK>
+0x674B	0x7E0A	# <CJK>
+0x674C	0x7E11	# <CJK>
+0x674D	0x7E7D	# <CJK>
+0x674E	0x7E39	# <CJK>
+0x674F	0x7E35	# <CJK>
+0x6750	0x7E32	# <CJK>
+0x6751	0x7E46	# <CJK>
+0x6752	0x7E45	# <CJK>
+0x6753	0x7E88	# <CJK>
+0x6754	0x7E5A	# <CJK>
+0x6755	0x7E52	# <CJK>
+0x6756	0x7E6E	# <CJK>
+0x6757	0x7E7E	# <CJK>
+0x6758	0x7E70	# <CJK>
+0x6759	0x7E6F	# <CJK>
+0x675A	0x7E98	# <CJK>
+0x675B	0x5E7A	# <CJK>
+0x675C	0x757F	# <CJK>
+0x675D	0x5DDB	# <CJK>
+0x675E	0x753E	# <CJK>
+0x675F	0x9095	# <CJK>
+0x6760	0x738E	# <CJK>
+0x6761	0x74A3	# <CJK>
+0x6762	0x744B	# <CJK>
+0x6763	0x73A2	# <CJK>
+0x6764	0x739F	# <CJK>
+0x6765	0x73CF	# <CJK>
+0x6766	0x73C2	# <CJK>
+0x6767	0x74CF	# <CJK>
+0x6768	0x73B7	# <CJK>
+0x6769	0x73B3	# <CJK>
+0x676A	0x73C0	# <CJK>
+0x676B	0x73C9	# <CJK>
+0x676C	0x73C8	# <CJK>
+0x676D	0x73E5	# <CJK>
+0x676E	0x73D9	# <CJK>
+0x676F	0x980A	# <CJK>
+0x6770	0x740A	# <CJK>
+0x6771	0x73E9	# <CJK>
+0x6772	0x73E7	# <CJK>
+0x6773	0x73DE	# <CJK>
+0x6774	0x74BD	# <CJK>
+0x6775	0x743F	# <CJK>
+0x6776	0x7489	# <CJK>
+0x6777	0x742A	# <CJK>
+0x6778	0x745B	# <CJK>
+0x6779	0x7426	# <CJK>
+0x677A	0x7425	# <CJK>
+0x677B	0x7428	# <CJK>
+0x677C	0x7430	# <CJK>
+0x677D	0x742E	# <CJK>
+0x677E	0x742C	# <CJK>
+0x6821	0x741B	# <CJK>
+0x6822	0x741A	# <CJK>
+0x6823	0x7441	# <CJK>
+0x6824	0x745C	# <CJK>
+0x6825	0x7457	# <CJK>
+0x6826	0x7455	# <CJK>
+0x6827	0x7459	# <CJK>
+0x6828	0x74A6	# <CJK>
+0x6829	0x746D	# <CJK>
+0x682A	0x747E	# <CJK>
+0x682B	0x749C	# <CJK>
+0x682C	0x74D4	# <CJK>
+0x682D	0x7480	# <CJK>
+0x682E	0x7481	# <CJK>
+0x682F	0x7487	# <CJK>
+0x6830	0x748B	# <CJK>
+0x6831	0x749E	# <CJK>
+0x6832	0x74A8	# <CJK>
+0x6833	0x74A9	# <CJK>
+0x6834	0x7490	# <CJK>
+0x6835	0x74A7	# <CJK>
+0x6836	0x74DA	# <CJK>
+0x6837	0x74BA	# <CJK>
+0x6838	0x97D9	# <CJK>
+0x6839	0x97DE	# <CJK>
+0x683A	0x97DC	# <CJK>
+0x683B	0x674C	# <CJK>
+0x683C	0x6753	# <CJK>
+0x683D	0x675E	# <CJK>
+0x683E	0x6748	# <CJK>
+0x683F	0x69AA	# <CJK>
+0x6840	0x6AEA	# <CJK>
+0x6841	0x6787	# <CJK>
+0x6842	0x676A	# <CJK>
+0x6843	0x6773	# <CJK>
+0x6844	0x6798	# <CJK>
+0x6845	0x6898	# <CJK>
+0x6846	0x6775	# <CJK>
+0x6847	0x68D6	# <CJK>
+0x6848	0x6A05	# <CJK>
+0x6849	0x689F	# <CJK>
+0x684A	0x678B	# <CJK>
+0x684B	0x6777	# <CJK>
+0x684C	0x677C	# <CJK>
+0x684D	0x67F0	# <CJK>
+0x684E	0x6ADB	# <CJK>
+0x684F	0x67D8	# <CJK>
+0x6850	0x6AF3	# <CJK>
+0x6851	0x67E9	# <CJK>
+0x6852	0x67B0	# <CJK>
+0x6853	0x6AE8	# <CJK>
+0x6854	0x67D9	# <CJK>
+0x6855	0x67B5	# <CJK>
+0x6856	0x67DA	# <CJK>
+0x6857	0x67B3	# <CJK>
+0x6858	0x67DD	# <CJK>
+0x6859	0x6800	# <CJK>
+0x685A	0x67C3	# <CJK>
+0x685B	0x67B8	# <CJK>
+0x685C	0x67E2	# <CJK>
+0x685D	0x6ADF	# <CJK>
+0x685E	0x67C1	# <CJK>
+0x685F	0x6A89	# <CJK>
+0x6860	0x6832	# <CJK>
+0x6861	0x6833	# <CJK>
+0x6862	0x690F	# <CJK>
+0x6863	0x6A48	# <CJK>
+0x6864	0x684E	# <CJK>
+0x6865	0x6968	# <CJK>
+0x6866	0x6844	# <CJK>
+0x6867	0x69BF	# <CJK>
+0x6868	0x6883	# <CJK>
+0x6869	0x681D	# <CJK>
+0x686A	0x6855	# <CJK>
+0x686B	0x6A3A	# <CJK>
+0x686C	0x6841	# <CJK>
+0x686D	0x6A9C	# <CJK>
+0x686E	0x6840	# <CJK>
+0x686F	0x6B12	# <CJK>
+0x6870	0x684A	# <CJK>
+0x6871	0x6849	# <CJK>
+0x6872	0x6829	# <CJK>
+0x6873	0x68B5	# <CJK>
+0x6874	0x688F	# <CJK>
+0x6875	0x6874	# <CJK>
+0x6876	0x6877	# <CJK>
+0x6877	0x6893	# <CJK>
+0x6878	0x686B	# <CJK>
+0x6879	0x6B1E	# <CJK>
+0x687A	0x696E	# <CJK>
+0x687B	0x68FC	# <CJK>
+0x687C	0x6ADD	# <CJK>
+0x687D	0x69E7	# <CJK>
+0x687E	0x68F9	# <CJK>
+0x6921	0x6B0F	# <CJK>
+0x6922	0x68F0	# <CJK>
+0x6923	0x690B	# <CJK>
+0x6924	0x6901	# <CJK>
+0x6925	0x6957	# <CJK>
+0x6926	0x68E3	# <CJK>
+0x6927	0x6910	# <CJK>
+0x6928	0x6971	# <CJK>
+0x6929	0x6939	# <CJK>
+0x692A	0x6960	# <CJK>
+0x692B	0x6942	# <CJK>
+0x692C	0x695D	# <CJK>
+0x692D	0x6B16	# <CJK>
+0x692E	0x696B	# <CJK>
+0x692F	0x6980	# <CJK>
+0x6930	0x6998	# <CJK>
+0x6931	0x6978	# <CJK>
+0x6932	0x6934	# <CJK>
+0x6933	0x69CC	# <CJK>
+0x6934	0x6AEC	# <CJK>
+0x6935	0x6ADA	# <CJK>
+0x6936	0x69CE	# <CJK>
+0x6937	0x6AF8	# <CJK>
+0x6938	0x6966	# <CJK>
+0x6939	0x6963	# <CJK>
+0x693A	0x6979	# <CJK>
+0x693B	0x699B	# <CJK>
+0x693C	0x69A7	# <CJK>
+0x693D	0x69BB	# <CJK>
+0x693E	0x69AB	# <CJK>
+0x693F	0x69AD	# <CJK>
+0x6940	0x69D4	# <CJK>
+0x6941	0x69B1	# <CJK>
+0x6942	0x69C1	# <CJK>
+0x6943	0x69CA	# <CJK>
+0x6944	0x6AB3	# <CJK>
+0x6945	0x6995	# <CJK>
+0x6946	0x6AE7	# <CJK>
+0x6947	0x698D	# <CJK>
+0x6948	0x69FF	# <CJK>
+0x6949	0x6AA3	# <CJK>
+0x694A	0x69ED	# <CJK>
+0x694B	0x6A17	# <CJK>
+0x694C	0x6A18	# <CJK>
+0x694D	0x6A65	# <CJK>
+0x694E	0x69F2	# <CJK>
+0x694F	0x6A44	# <CJK>
+0x6950	0x6A3E	# <CJK>
+0x6951	0x6AA0	# <CJK>
+0x6952	0x6A50	# <CJK>
+0x6953	0x6A5B	# <CJK>
+0x6954	0x6A35	# <CJK>
+0x6955	0x6A8E	# <CJK>
+0x6956	0x6AD3	# <CJK>
+0x6957	0x6A3D	# <CJK>
+0x6958	0x6A28	# <CJK>
+0x6959	0x6A58	# <CJK>
+0x695A	0x6ADE	# <CJK>
+0x695B	0x6A91	# <CJK>
+0x695C	0x6A90	# <CJK>
+0x695D	0x6AA9	# <CJK>
+0x695E	0x6A97	# <CJK>
+0x695F	0x6AAB	# <CJK>
+0x6960	0x7337	# <CJK>
+0x6961	0x7352	# <CJK>
+0x6962	0x6B81	# <CJK>
+0x6963	0x6B82	# <CJK>
+0x6964	0x6BA4	# <CJK>
+0x6965	0x6B84	# <CJK>
+0x6966	0x6B9E	# <CJK>
+0x6967	0x6BAE	# <CJK>
+0x6968	0x6B8D	# <CJK>
+0x6969	0x6BAB	# <CJK>
+0x696A	0x6B9B	# <CJK>
+0x696B	0x6BAF	# <CJK>
+0x696C	0x6BAA	# <CJK>
+0x696D	0x8ED4	# <CJK>
+0x696E	0x8EDB	# <CJK>
+0x696F	0x8EF2	# <CJK>
+0x6970	0x8EFB	# <CJK>
+0x6971	0x8F64	# <CJK>
+0x6972	0x8EF9	# <CJK>
+0x6973	0x8EFC	# <CJK>
+0x6974	0x8EEB	# <CJK>
+0x6975	0x8EE4	# <CJK>
+0x6976	0x8F62	# <CJK>
+0x6977	0x8EFA	# <CJK>
+0x6978	0x8EFE	# <CJK>
+0x6979	0x8F0A	# <CJK>
+0x697A	0x8F07	# <CJK>
+0x697B	0x8F05	# <CJK>
+0x697C	0x8F12	# <CJK>
+0x697D	0x8F26	# <CJK>
+0x697E	0x8F1E	# <CJK>
+0x6A21	0x8F1F	# <CJK>
+0x6A22	0x8F1C	# <CJK>
+0x6A23	0x8F33	# <CJK>
+0x6A24	0x8F46	# <CJK>
+0x6A25	0x8F54	# <CJK>
+0x6A26	0x8ECE	# <CJK>
+0x6A27	0x6214	# <CJK>
+0x6A28	0x6227	# <CJK>
+0x6A29	0x621B	# <CJK>
+0x6A2A	0x621F	# <CJK>
+0x6A2B	0x6222	# <CJK>
+0x6A2C	0x6221	# <CJK>
+0x6A2D	0x6225	# <CJK>
+0x6A2E	0x6224	# <CJK>
+0x6A2F	0x6229	# <CJK>
+0x6A30	0x81E7	# <CJK>
+0x6A31	0x750C	# <CJK>
+0x6A32	0x74F4	# <CJK>
+0x6A33	0x74FF	# <CJK>
+0x6A34	0x750F	# <CJK>
+0x6A35	0x7511	# <CJK>
+0x6A36	0x7513	# <CJK>
+0x6A37	0x6534	# <CJK>
+0x6A38	0x65EE	# <CJK>
+0x6A39	0x65EF	# <CJK>
+0x6A3A	0x65F0	# <CJK>
+0x6A3B	0x660A	# <CJK>
+0x6A3C	0x66C7	# <CJK>
+0x6A3D	0x6772	# <CJK>
+0x6A3E	0x6603	# <CJK>
+0x6A3F	0x6615	# <CJK>
+0x6A40	0x6600	# <CJK>
+0x6A41	0x7085	# <CJK>
+0x6A42	0x66F7	# <CJK>
+0x6A43	0x661D	# <CJK>
+0x6A44	0x6634	# <CJK>
+0x6A45	0x6631	# <CJK>
+0x6A46	0x6636	# <CJK>
+0x6A47	0x6635	# <CJK>
+0x6A48	0x8006	# <CJK>
+0x6A49	0x665F	# <CJK>
+0x6A4A	0x66C4	# <CJK>
+0x6A4B	0x6641	# <CJK>
+0x6A4C	0x664F	# <CJK>
+0x6A4D	0x6689	# <CJK>
+0x6A4E	0x6661	# <CJK>
+0x6A4F	0x6657	# <CJK>
+0x6A50	0x6677	# <CJK>
+0x6A51	0x6684	# <CJK>
+0x6A52	0x668C	# <CJK>
+0x6A53	0x66D6	# <CJK>
+0x6A54	0x669D	# <CJK>
+0x6A55	0x66BE	# <CJK>
+0x6A56	0x66DB	# <CJK>
+0x6A57	0x66DC	# <CJK>
+0x6A58	0x66E6	# <CJK>
+0x6A59	0x66E9	# <CJK>
+0x6A5A	0x8CC1	# <CJK>
+0x6A5B	0x8CB0	# <CJK>
+0x6A5C	0x8CBA	# <CJK>
+0x6A5D	0x8CBD	# <CJK>
+0x6A5E	0x8D04	# <CJK>
+0x6A5F	0x8CB2	# <CJK>
+0x6A60	0x8CC5	# <CJK>
+0x6A61	0x8D10	# <CJK>
+0x6A62	0x8CD1	# <CJK>
+0x6A63	0x8CDA	# <CJK>
+0x6A64	0x8CD5	# <CJK>
+0x6A65	0x8CEB	# <CJK>
+0x6A66	0x8CE7	# <CJK>
+0x6A67	0x8CFB	# <CJK>
+0x6A68	0x8998	# <CJK>
+0x6A69	0x89AC	# <CJK>
+0x6A6A	0x89A1	# <CJK>
+0x6A6B	0x89BF	# <CJK>
+0x6A6C	0x89A6	# <CJK>
+0x6A6D	0x89AF	# <CJK>
+0x6A6E	0x89B2	# <CJK>
+0x6A6F	0x89B7	# <CJK>
+0x6A70	0x726E	# <CJK>
+0x6A71	0x729F	# <CJK>
+0x6A72	0x725D	# <CJK>
+0x6A73	0x7266	# <CJK>
+0x6A74	0x726F	# <CJK>
+0x6A75	0x727E	# <CJK>
+0x6A76	0x727F	# <CJK>
+0x6A77	0x7284	# <CJK>
+0x6A78	0x728B	# <CJK>
+0x6A79	0x728D	# <CJK>
+0x6A7A	0x728F	# <CJK>
+0x6A7B	0x7292	# <CJK>
+0x6A7C	0x6308	# <CJK>
+0x6A7D	0x6332	# <CJK>
+0x6A7E	0x63B0	# <CJK>
+0x6B21	0x643F	# <CJK>
+0x6B22	0x64D8	# <CJK>
+0x6B23	0x8004	# <CJK>
+0x6B24	0x6BEA	# <CJK>
+0x6B25	0x6BF3	# <CJK>
+0x6B26	0x6BFD	# <CJK>
+0x6B27	0x6BFF	# <CJK>
+0x6B28	0x6BF9	# <CJK>
+0x6B29	0x6C05	# <CJK>
+0x6B2A	0x6C0C	# <CJK>
+0x6B2B	0x6C06	# <CJK>
+0x6B2C	0x6C0D	# <CJK>
+0x6B2D	0x6C15	# <CJK>
+0x6B2E	0x6C18	# <CJK>
+0x6B2F	0x6C19	# <CJK>
+0x6B30	0x6C1A	# <CJK>
+0x6B31	0x6C21	# <CJK>
+0x6B32	0x6C2C	# <CJK>
+0x6B33	0x6C24	# <CJK>
+0x6B34	0x6C2A	# <CJK>
+0x6B35	0x6C32	# <CJK>
+0x6B36	0x6535	# <CJK>
+0x6B37	0x6555	# <CJK>
+0x6B38	0x656B	# <CJK>
+0x6B39	0x7258	# <CJK>
+0x6B3A	0x7252	# <CJK>
+0x6B3B	0x7256	# <CJK>
+0x6B3C	0x7230	# <CJK>
+0x6B3D	0x8662	# <CJK>
+0x6B3E	0x5216	# <CJK>
+0x6B3F	0x809F	# <CJK>
+0x6B40	0x809C	# <CJK>
+0x6B41	0x8093	# <CJK>
+0x6B42	0x80BC	# <CJK>
+0x6B43	0x670A	# <CJK>
+0x6B44	0x80BD	# <CJK>
+0x6B45	0x80B1	# <CJK>
+0x6B46	0x80AB	# <CJK>
+0x6B47	0x80AD	# <CJK>
+0x6B48	0x80B4	# <CJK>
+0x6B49	0x80B7	# <CJK>
+0x6B4A	0x6727	# <CJK>
+0x6B4B	0x8156	# <CJK>
+0x6B4C	0x80E9	# <CJK>
+0x6B4D	0x81DA	# <CJK>
+0x6B4E	0x80DB	# <CJK>
+0x6B4F	0x80C2	# <CJK>
+0x6B50	0x80C4	# <CJK>
+0x6B51	0x80D9	# <CJK>
+0x6B52	0x80CD	# <CJK>
+0x6B53	0x80D7	# <CJK>
+0x6B54	0x6710	# <CJK>
+0x6B55	0x80DD	# <CJK>
+0x6B56	0x811B	# <CJK>
+0x6B57	0x80F1	# <CJK>
+0x6B58	0x80F4	# <CJK>
+0x6B59	0x80ED	# <CJK>
+0x6B5A	0x81BE	# <CJK>
+0x6B5B	0x810E	# <CJK>
+0x6B5C	0x80F2	# <CJK>
+0x6B5D	0x80FC	# <CJK>
+0x6B5E	0x6715	# <CJK>
+0x6B5F	0x8112	# <CJK>
+0x6B60	0x8C5A	# <CJK>
+0x6B61	0x8161	# <CJK>
+0x6B62	0x811E	# <CJK>
+0x6B63	0x812C	# <CJK>
+0x6B64	0x8118	# <CJK>
+0x6B65	0x8132	# <CJK>
+0x6B66	0x8148	# <CJK>
+0x6B67	0x814C	# <CJK>
+0x6B68	0x8153	# <CJK>
+0x6B69	0x8174	# <CJK>
+0x6B6A	0x8159	# <CJK>
+0x6B6B	0x815A	# <CJK>
+0x6B6C	0x8171	# <CJK>
+0x6B6D	0x8160	# <CJK>
+0x6B6E	0x8169	# <CJK>
+0x6B6F	0x817C	# <CJK>
+0x6B70	0x817D	# <CJK>
+0x6B71	0x816D	# <CJK>
+0x6B72	0x8167	# <CJK>
+0x6B73	0x584D	# <CJK>
+0x6B74	0x5AB5	# <CJK>
+0x6B75	0x8188	# <CJK>
+0x6B76	0x8182	# <CJK>
+0x6B77	0x81CF	# <CJK>
+0x6B78	0x6ED5	# <CJK>
+0x6B79	0x81A3	# <CJK>
+0x6B7A	0x81AA	# <CJK>
+0x6B7B	0x81CC	# <CJK>
+0x6B7C	0x6726	# <CJK>
+0x6B7D	0x81CA	# <CJK>
+0x6B7E	0x81BB	# <CJK>
+0x6C21	0x81C1	# <CJK>
+0x6C22	0x81A6	# <CJK>
+0x6C23	0x6B5F	# <CJK>
+0x6C24	0x6B37	# <CJK>
+0x6C25	0x6B39	# <CJK>
+0x6C26	0x6B43	# <CJK>
+0x6C27	0x6B46	# <CJK>
+0x6C28	0x6B59	# <CJK>
+0x6C29	0x98AE	# <CJK>
+0x6C2A	0x98AF	# <CJK>
+0x6C2B	0x98B6	# <CJK>
+0x6C2C	0x98BC	# <CJK>
+0x6C2D	0x98C6	# <CJK>
+0x6C2E	0x98C8	# <CJK>
+0x6C2F	0x6BB3	# <CJK>
+0x6C30	0x5F40	# <CJK>
+0x6C31	0x8F42	# <CJK>
+0x6C32	0x89F3	# <CJK>
+0x6C33	0x6590	# <CJK>
+0x6C34	0x9F4F	# <CJK>
+0x6C35	0x6595	# <CJK>
+0x6C36	0x65BC	# <CJK>
+0x6C37	0x65C6	# <CJK>
+0x6C38	0x65C4	# <CJK>
+0x6C39	0x65C3	# <CJK>
+0x6C3A	0x65CC	# <CJK>
+0x6C3B	0x65CE	# <CJK>
+0x6C3C	0x65D2	# <CJK>
+0x6C3D	0x65D6	# <CJK>
+0x6C3E	0x716C	# <CJK>
+0x6C3F	0x7152	# <CJK>
+0x6C40	0x7096	# <CJK>
+0x6C41	0x7197	# <CJK>
+0x6C42	0x70BB	# <CJK>
+0x6C43	0x70C0	# <CJK>
+0x6C44	0x70B7	# <CJK>
+0x6C45	0x70AB	# <CJK>
+0x6C46	0x70B1	# <CJK>
+0x6C47	0x71C1	# <CJK>
+0x6C48	0x70CA	# <CJK>
+0x6C49	0x7110	# <CJK>
+0x6C4A	0x7113	# <CJK>
+0x6C4B	0x71DC	# <CJK>
+0x6C4C	0x712F	# <CJK>
+0x6C4D	0x7131	# <CJK>
+0x6C4E	0x7173	# <CJK>
+0x6C4F	0x715C	# <CJK>
+0x6C50	0x7168	# <CJK>
+0x6C51	0x7145	# <CJK>
+0x6C52	0x7172	# <CJK>
+0x6C53	0x714A	# <CJK>
+0x6C54	0x7178	# <CJK>
+0x6C55	0x717A	# <CJK>
+0x6C56	0x7198	# <CJK>
+0x6C57	0x71B3	# <CJK>
+0x6C58	0x71B5	# <CJK>
+0x6C59	0x71A8	# <CJK>
+0x6C5A	0x71A0	# <CJK>
+0x6C5B	0x71E0	# <CJK>
+0x6C5C	0x71D4	# <CJK>
+0x6C5D	0x71E7	# <CJK>
+0x6C5E	0x71F9	# <CJK>
+0x6C5F	0x721D	# <CJK>
+0x6C60	0x7228	# <CJK>
+0x6C61	0x706C	# <CJK>
+0x6C62	0x71FE	# <CJK>
+0x6C63	0x7166	# <CJK>
+0x6C64	0x71B9	# <CJK>
+0x6C65	0x623E	# <CJK>
+0x6C66	0x623D	# <CJK>
+0x6C67	0x6243	# <CJK>
+0x6C68	0x6248	# <CJK>
+0x6C69	0x6249	# <CJK>
+0x6C6A	0x793B	# <CJK>
+0x6C6B	0x7940	# <CJK>
+0x6C6C	0x7946	# <CJK>
+0x6C6D	0x7949	# <CJK>
+0x6C6E	0x795B	# <CJK>
+0x6C6F	0x795C	# <CJK>
+0x6C70	0x7953	# <CJK>
+0x6C71	0x795A	# <CJK>
+0x6C72	0x79B0	# <CJK>
+0x6C73	0x7957	# <CJK>
+0x6C74	0x7960	# <CJK>
+0x6C75	0x798E	# <CJK>
+0x6C76	0x7967	# <CJK>
+0x6C77	0x797A	# <CJK>
+0x6C78	0x79AA	# <CJK>
+0x6C79	0x798A	# <CJK>
+0x6C7A	0x799A	# <CJK>
+0x6C7B	0x79A7	# <CJK>
+0x6C7C	0x79B3	# <CJK>
+0x6C7D	0x5FD1	# <CJK>
+0x6C7E	0x5FD0	# <CJK>
+0x6D21	0x61DF	# <CJK>
+0x6D22	0x605D	# <CJK>
+0x6D23	0x605A	# <CJK>
+0x6D24	0x6067	# <CJK>
+0x6D25	0x6041	# <CJK>
+0x6D26	0x6059	# <CJK>
+0x6D27	0x6063	# <CJK>
+0x6D28	0x6164	# <CJK>
+0x6D29	0x6106	# <CJK>
+0x6D2A	0x610D	# <CJK>
+0x6D2B	0x615D	# <CJK>
+0x6D2C	0x61A9	# <CJK>
+0x6D2D	0x619D	# <CJK>
+0x6D2E	0x61CB	# <CJK>
+0x6D2F	0x61E3	# <CJK>
+0x6D30	0x6207	# <CJK>
+0x6D31	0x8080	# <CJK>
+0x6D32	0x807F	# <CJK>
+0x6D33	0x6C93	# <CJK>
+0x6D34	0x6FA9	# <CJK>
+0x6D35	0x6DFC	# <CJK>
+0x6D36	0x78EF	# <CJK>
+0x6D37	0x77F8	# <CJK>
+0x6D38	0x78AD	# <CJK>
+0x6D39	0x7809	# <CJK>
+0x6D3A	0x7868	# <CJK>
+0x6D3B	0x7818	# <CJK>
+0x6D3C	0x7811	# <CJK>
+0x6D3D	0x65AB	# <CJK>
+0x6D3E	0x782D	# <CJK>
+0x6D3F	0x78B8	# <CJK>
+0x6D40	0x781D	# <CJK>
+0x6D41	0x7839	# <CJK>
+0x6D42	0x792A	# <CJK>
+0x6D43	0x7931	# <CJK>
+0x6D44	0x781F	# <CJK>
+0x6D45	0x783C	# <CJK>
+0x6D46	0x7825	# <CJK>
+0x6D47	0x782C	# <CJK>
+0x6D48	0x7823	# <CJK>
+0x6D49	0x7829	# <CJK>
+0x6D4A	0x784E	# <CJK>
+0x6D4B	0x786D	# <CJK>
+0x6D4C	0x7864	# <CJK>
+0x6D4D	0x78FD	# <CJK>
+0x6D4E	0x7826	# <CJK>
+0x6D4F	0x7850	# <CJK>
+0x6D50	0x7847	# <CJK>
+0x6D51	0x784C	# <CJK>
+0x6D52	0x786A	# <CJK>
+0x6D53	0x78E7	# <CJK>
+0x6D54	0x7893	# <CJK>
+0x6D55	0x789A	# <CJK>
+0x6D56	0x7887	# <CJK>
+0x6D57	0x78E3	# <CJK>
+0x6D58	0x78A1	# <CJK>
+0x6D59	0x78A3	# <CJK>
+0x6D5A	0x78B2	# <CJK>
+0x6D5B	0x78B9	# <CJK>
+0x6D5C	0x78A5	# <CJK>
+0x6D5D	0x78D4	# <CJK>
+0x6D5E	0x78D9	# <CJK>
+0x6D5F	0x78C9	# <CJK>
+0x6D60	0x78EC	# <CJK>
+0x6D61	0x78F2	# <CJK>
+0x6D62	0x7905	# <CJK>
+0x6D63	0x78F4	# <CJK>
+0x6D64	0x7913	# <CJK>
+0x6D65	0x7924	# <CJK>
+0x6D66	0x791E	# <CJK>
+0x6D67	0x7934	# <CJK>
+0x6D68	0x9F95	# <CJK>
+0x6D69	0x9EF9	# <CJK>
+0x6D6A	0x9EFB	# <CJK>
+0x6D6B	0x9EFC	# <CJK>
+0x6D6C	0x76F1	# <CJK>
+0x6D6D	0x7704	# <CJK>
+0x6D6E	0x7798	# <CJK>
+0x6D6F	0x76F9	# <CJK>
+0x6D70	0x7707	# <CJK>
+0x6D71	0x7708	# <CJK>
+0x6D72	0x771A	# <CJK>
+0x6D73	0x7722	# <CJK>
+0x6D74	0x7719	# <CJK>
+0x6D75	0x772D	# <CJK>
+0x6D76	0x7726	# <CJK>
+0x6D77	0x7735	# <CJK>
+0x6D78	0x7738	# <CJK>
+0x6D79	0x775E	# <CJK>
+0x6D7A	0x77BC	# <CJK>
+0x6D7B	0x7747	# <CJK>
+0x6D7C	0x7743	# <CJK>
+0x6D7D	0x775A	# <CJK>
+0x6D7E	0x7768	# <CJK>
+0x6E21	0x7762	# <CJK>
+0x6E22	0x7765	# <CJK>
+0x6E23	0x777F	# <CJK>
+0x6E24	0x778D	# <CJK>
+0x6E25	0x777D	# <CJK>
+0x6E26	0x7780	# <CJK>
+0x6E27	0x778C	# <CJK>
+0x6E28	0x7791	# <CJK>
+0x6E29	0x779F	# <CJK>
+0x6E2A	0x77A0	# <CJK>
+0x6E2B	0x77B0	# <CJK>
+0x6E2C	0x77B5	# <CJK>
+0x6E2D	0x77BD	# <CJK>
+0x6E2E	0x753A	# <CJK>
+0x6E2F	0x7540	# <CJK>
+0x6E30	0x754E	# <CJK>
+0x6E31	0x754B	# <CJK>
+0x6E32	0x7548	# <CJK>
+0x6E33	0x755B	# <CJK>
+0x6E34	0x7572	# <CJK>
+0x6E35	0x7579	# <CJK>
+0x6E36	0x7583	# <CJK>
+0x6E37	0x7F58	# <CJK>
+0x6E38	0x7F61	# <CJK>
+0x6E39	0x7F5F	# <CJK>
+0x6E3A	0x8A48	# <CJK>
+0x6E3B	0x7F68	# <CJK>
+0x6E3C	0x7F86	# <CJK>
+0x6E3D	0x7F71	# <CJK>
+0x6E3E	0x7F79	# <CJK>
+0x6E3F	0x7F88	# <CJK>
+0x6E40	0x7F7E	# <CJK>
+0x6E41	0x76CD	# <CJK>
+0x6E42	0x76E5	# <CJK>
+0x6E43	0x8832	# <CJK>
+0x6E44	0x91D2	# <CJK>
+0x6E45	0x91D3	# <CJK>
+0x6E46	0x91D4	# <CJK>
+0x6E47	0x91D9	# <CJK>
+0x6E48	0x91D7	# <CJK>
+0x6E49	0x91D5	# <CJK>
+0x6E4A	0x91F7	# <CJK>
+0x6E4B	0x91E7	# <CJK>
+0x6E4C	0x91E4	# <CJK>
+0x6E4D	0x9346	# <CJK>
+0x6E4E	0x91F5	# <CJK>
+0x6E4F	0x91F9	# <CJK>
+0x6E50	0x9208	# <CJK>
+0x6E51	0x9226	# <CJK>
+0x6E52	0x9245	# <CJK>
+0x6E53	0x9211	# <CJK>
+0x6E54	0x9210	# <CJK>
+0x6E55	0x9201	# <CJK>
+0x6E56	0x9227	# <CJK>
+0x6E57	0x9204	# <CJK>
+0x6E58	0x9225	# <CJK>
+0x6E59	0x9200	# <CJK>
+0x6E5A	0x923A	# <CJK>
+0x6E5B	0x9266	# <CJK>
+0x6E5C	0x9237	# <CJK>
+0x6E5D	0x9233	# <CJK>
+0x6E5E	0x9255	# <CJK>
+0x6E5F	0x923D	# <CJK>
+0x6E60	0x9238	# <CJK>
+0x6E61	0x925E	# <CJK>
+0x6E62	0x926C	# <CJK>
+0x6E63	0x926D	# <CJK>
+0x6E64	0x923F	# <CJK>
+0x6E65	0x9460	# <CJK>
+0x6E66	0x9230	# <CJK>
+0x6E67	0x9249	# <CJK>
+0x6E68	0x9248	# <CJK>
+0x6E69	0x924D	# <CJK>
+0x6E6A	0x922E	# <CJK>
+0x6E6B	0x9239	# <CJK>
+0x6E6C	0x9438	# <CJK>
+0x6E6D	0x92AC	# <CJK>
+0x6E6E	0x92A0	# <CJK>
+0x6E6F	0x927A	# <CJK>
+0x6E70	0x92AA	# <CJK>
+0x6E71	0x92EE	# <CJK>
+0x6E72	0x92CF	# <CJK>
+0x6E73	0x9403	# <CJK>
+0x6E74	0x92E3	# <CJK>
+0x6E75	0x943A	# <CJK>
+0x6E76	0x92B1	# <CJK>
+0x6E77	0x92A6	# <CJK>
+0x6E78	0x93A7	# <CJK>
+0x6E79	0x9296	# <CJK>
+0x6E7A	0x92CC	# <CJK>
+0x6E7B	0x92A9	# <CJK>
+0x6E7C	0x93F5	# <CJK>
+0x6E7D	0x9293	# <CJK>
+0x6E7E	0x927F	# <CJK>
+0x6F21	0x93A9	# <CJK>
+0x6F22	0x929A	# <CJK>
+0x6F23	0x931A	# <CJK>
+0x6F24	0x92AB	# <CJK>
+0x6F25	0x9283	# <CJK>
+0x6F26	0x940B	# <CJK>
+0x6F27	0x92A8	# <CJK>
+0x6F28	0x92A3	# <CJK>
+0x6F29	0x9412	# <CJK>
+0x6F2A	0x9338	# <CJK>
+0x6F2B	0x92F1	# <CJK>
+0x6F2C	0x93D7	# <CJK>
+0x6F2D	0x92E5	# <CJK>
+0x6F2E	0x92F0	# <CJK>
+0x6F2F	0x92EF	# <CJK>
+0x6F30	0x92E8	# <CJK>
+0x6F31	0x92BC	# <CJK>
+0x6F32	0x92DD	# <CJK>
+0x6F33	0x92F6	# <CJK>
+0x6F34	0x9426	# <CJK>
+0x6F35	0x9427	# <CJK>
+0x6F36	0x92C3	# <CJK>
+0x6F37	0x92DF	# <CJK>
+0x6F38	0x92E6	# <CJK>
+0x6F39	0x9312	# <CJK>
+0x6F3A	0x9306	# <CJK>
+0x6F3B	0x9369	# <CJK>
+0x6F3C	0x931B	# <CJK>
+0x6F3D	0x9340	# <CJK>
+0x6F3E	0x9301	# <CJK>
+0x6F3F	0x9315	# <CJK>
+0x6F40	0x932E	# <CJK>
+0x6F41	0x9343	# <CJK>
+0x6F42	0x9307	# <CJK>
+0x6F43	0x9308	# <CJK>
+0x6F44	0x931F	# <CJK>
+0x6F45	0x9319	# <CJK>
+0x6F46	0x9365	# <CJK>
+0x6F47	0x9347	# <CJK>
+0x6F48	0x9376	# <CJK>
+0x6F49	0x9354	# <CJK>
+0x6F4A	0x9364	# <CJK>
+0x6F4B	0x93AA	# <CJK>
+0x6F4C	0x9370	# <CJK>
+0x6F4D	0x9384	# <CJK>
+0x6F4E	0x93E4	# <CJK>
+0x6F4F	0x93D8	# <CJK>
+0x6F50	0x9428	# <CJK>
+0x6F51	0x9387	# <CJK>
+0x6F52	0x93CC	# <CJK>
+0x6F53	0x9398	# <CJK>
+0x6F54	0x93B8	# <CJK>
+0x6F55	0x93BF	# <CJK>
+0x6F56	0x93A6	# <CJK>
+0x6F57	0x93B0	# <CJK>
+0x6F58	0x93B5	# <CJK>
+0x6F59	0x944C	# <CJK>
+0x6F5A	0x93E2	# <CJK>
+0x6F5B	0x93DC	# <CJK>
+0x6F5C	0x93DD	# <CJK>
+0x6F5D	0x93CD	# <CJK>
+0x6F5E	0x93DE	# <CJK>
+0x6F5F	0x93C3	# <CJK>
+0x6F60	0x93C7	# <CJK>
+0x6F61	0x93D1	# <CJK>
+0x6F62	0x9414	# <CJK>
+0x6F63	0x941D	# <CJK>
+0x6F64	0x93F7	# <CJK>
+0x6F65	0x9465	# <CJK>
+0x6F66	0x9413	# <CJK>
+0x6F67	0x946D	# <CJK>
+0x6F68	0x9420	# <CJK>
+0x6F69	0x9479	# <CJK>
+0x6F6A	0x93F9	# <CJK>
+0x6F6B	0x9419	# <CJK>
+0x6F6C	0x944A	# <CJK>
+0x6F6D	0x9432	# <CJK>
+0x6F6E	0x943F	# <CJK>
+0x6F6F	0x9454	# <CJK>
+0x6F70	0x9463	# <CJK>
+0x6F71	0x937E	# <CJK>
+0x6F72	0x77E7	# <CJK>
+0x6F73	0x77EC	# <CJK>
+0x6F74	0x96C9	# <CJK>
+0x6F75	0x79D5	# <CJK>
+0x6F76	0x79ED	# <CJK>
+0x6F77	0x79E3	# <CJK>
+0x6F78	0x79EB	# <CJK>
+0x6F79	0x7A06	# <CJK>
+0x6F7A	0x5D47	# <CJK>
+0x6F7B	0x7A03	# <CJK>
+0x6F7C	0x7A02	# <CJK>
+0x6F7D	0x7A1E	# <CJK>
+0x6F7E	0x7A14	# <CJK>
+0x7021	0x7A39	# <CJK>
+0x7022	0x7A37	# <CJK>
+0x7023	0x7A61	# <CJK>
+0x7024	0x9ECF	# <CJK>
+0x7025	0x99A5	# <CJK>
+0x7026	0x7A70	# <CJK>
+0x7027	0x7688	# <CJK>
+0x7028	0x768E	# <CJK>
+0x7029	0x7693	# <CJK>
+0x702A	0x7699	# <CJK>
+0x702B	0x76A4	# <CJK>
+0x702C	0x74DE	# <CJK>
+0x702D	0x74E0	# <CJK>
+0x702E	0x752C	# <CJK>
+0x702F	0x9CE9	# <CJK>
+0x7030	0x9CF6	# <CJK>
+0x7031	0x9D07	# <CJK>
+0x7032	0x9D06	# <CJK>
+0x7033	0x9D23	# <CJK>
+0x7034	0x9D87	# <CJK>
+0x7035	0x9E15	# <CJK>
+0x7036	0x9D1D	# <CJK>
+0x7037	0x9D1F	# <CJK>
+0x7038	0x9DE5	# <CJK>
+0x7039	0x9D2F	# <CJK>
+0x703A	0x9DD9	# <CJK>
+0x703B	0x9D30	# <CJK>
+0x703C	0x9D42	# <CJK>
+0x703D	0x9E1E	# <CJK>
+0x703E	0x9D53	# <CJK>
+0x703F	0x9E1D	# <CJK>
+0x7040	0x9D60	# <CJK>
+0x7041	0x9D52	# <CJK>
+0x7042	0x9DF3	# <CJK>
+0x7043	0x9D5C	# <CJK>
+0x7044	0x9D61	# <CJK>
+0x7045	0x9D93	# <CJK>
+0x7046	0x9D6A	# <CJK>
+0x7047	0x9D6F	# <CJK>
+0x7048	0x9D89	# <CJK>
+0x7049	0x9D98	# <CJK>
+0x704A	0x9D9A	# <CJK>
+0x704B	0x9DC0	# <CJK>
+0x704C	0x9DA5	# <CJK>
+0x704D	0x9DA9	# <CJK>
+0x704E	0x9DC2	# <CJK>
+0x704F	0x9DBC	# <CJK>
+0x7050	0x9E1A	# <CJK>
+0x7051	0x9DD3	# <CJK>
+0x7052	0x9DDA	# <CJK>
+0x7053	0x9DEF	# <CJK>
+0x7054	0x9DE6	# <CJK>
+0x7055	0x9DF2	# <CJK>
+0x7056	0x9DF8	# <CJK>
+0x7057	0x9E0C	# <CJK>
+0x7058	0x9DFA	# <CJK>
+0x7059	0x9E1B	# <CJK>
+0x705A	0x7592	# <CJK>
+0x705B	0x7594	# <CJK>
+0x705C	0x7664	# <CJK>
+0x705D	0x7658	# <CJK>
+0x705E	0x759D	# <CJK>
+0x705F	0x7667	# <CJK>
+0x7060	0x75A3	# <CJK>
+0x7061	0x75B3	# <CJK>
+0x7062	0x75B4	# <CJK>
+0x7063	0x75B8	# <CJK>
+0x7064	0x75C4	# <CJK>
+0x7065	0x75B1	# <CJK>
+0x7066	0x75B0	# <CJK>
+0x7067	0x75C3	# <CJK>
+0x7068	0x75C2	# <CJK>
+0x7069	0x7602	# <CJK>
+0x706A	0x75CD	# <CJK>
+0x706B	0x75E3	# <CJK>
+0x706C	0x7646	# <CJK>
+0x706D	0x75E6	# <CJK>
+0x706E	0x75E4	# <CJK>
+0x706F	0x7647	# <CJK>
+0x7070	0x75E7	# <CJK>
+0x7071	0x7603	# <CJK>
+0x7072	0x75F1	# <CJK>
+0x7073	0x75FC	# <CJK>
+0x7074	0x75FF	# <CJK>
+0x7075	0x7610	# <CJK>
+0x7076	0x7600	# <CJK>
+0x7077	0x7649	# <CJK>
+0x7078	0x760C	# <CJK>
+0x7079	0x761E	# <CJK>
+0x707A	0x760A	# <CJK>
+0x707B	0x7625	# <CJK>
+0x707C	0x763B	# <CJK>
+0x707D	0x7615	# <CJK>
+0x707E	0x7619	# <CJK>
+0x7121	0x761B	# <CJK>
+0x7122	0x763C	# <CJK>
+0x7123	0x7622	# <CJK>
+0x7124	0x7620	# <CJK>
+0x7125	0x7640	# <CJK>
+0x7126	0x762D	# <CJK>
+0x7127	0x7630	# <CJK>
+0x7128	0x766D	# <CJK>
+0x7129	0x7635	# <CJK>
+0x712A	0x7643	# <CJK>
+0x712B	0x766E	# <CJK>
+0x712C	0x7633	# <CJK>
+0x712D	0x764D	# <CJK>
+0x712E	0x7669	# <CJK>
+0x712F	0x7654	# <CJK>
+0x7130	0x765C	# <CJK>
+0x7131	0x7656	# <CJK>
+0x7132	0x7672	# <CJK>
+0x7133	0x766F	# <CJK>
+0x7134	0x7FCA	# <CJK>
+0x7135	0x7AE6	# <CJK>
+0x7136	0x7A78	# <CJK>
+0x7137	0x7A79	# <CJK>
+0x7138	0x7A80	# <CJK>
+0x7139	0x7A86	# <CJK>
+0x713A	0x7A88	# <CJK>
+0x713B	0x7A95	# <CJK>
+0x713C	0x7AC7	# <CJK>
+0x713D	0x7AA0	# <CJK>
+0x713E	0x7AAC	# <CJK>
+0x713F	0x7AA8	# <CJK>
+0x7140	0x7AB6	# <CJK>
+0x7141	0x7AB3	# <CJK>
+0x7142	0x8864	# <CJK>
+0x7143	0x8869	# <CJK>
+0x7144	0x8872	# <CJK>
+0x7145	0x887D	# <CJK>
+0x7146	0x887F	# <CJK>
+0x7147	0x8882	# <CJK>
+0x7148	0x88A2	# <CJK>
+0x7149	0x8960	# <CJK>
+0x714A	0x88B7	# <CJK>
+0x714B	0x88BC	# <CJK>
+0x714C	0x88C9	# <CJK>
+0x714D	0x8933	# <CJK>
+0x714E	0x88CE	# <CJK>
+0x714F	0x895D	# <CJK>
+0x7150	0x8947	# <CJK>
+0x7151	0x88F1	# <CJK>
+0x7152	0x891A	# <CJK>
+0x7153	0x88FC	# <CJK>
+0x7154	0x88E8	# <CJK>
+0x7155	0x88FE	# <CJK>
+0x7156	0x88F0	# <CJK>
+0x7157	0x8921	# <CJK>
+0x7158	0x8919	# <CJK>
+0x7159	0x8913	# <CJK>
+0x715A	0x8938	# <CJK>
+0x715B	0x890A	# <CJK>
+0x715C	0x8964	# <CJK>
+0x715D	0x892B	# <CJK>
+0x715E	0x8936	# <CJK>
+0x715F	0x8941	# <CJK>
+0x7160	0x8966	# <CJK>
+0x7161	0x897B	# <CJK>
+0x7162	0x758B	# <CJK>
+0x7163	0x80E5	# <CJK>
+0x7164	0x76B8	# <CJK>
+0x7165	0x76B4	# <CJK>
+0x7166	0x77DC	# <CJK>
+0x7167	0x8012	# <CJK>
+0x7168	0x8014	# <CJK>
+0x7169	0x8016	# <CJK>
+0x716A	0x801C	# <CJK>
+0x716B	0x8020	# <CJK>
+0x716C	0x802E	# <CJK>
+0x716D	0x8025	# <CJK>
+0x716E	0x8026	# <CJK>
+0x716F	0x802C	# <CJK>
+0x7170	0x8029	# <CJK>
+0x7171	0x8028	# <CJK>
+0x7172	0x8031	# <CJK>
+0x7173	0x800B	# <CJK>
+0x7174	0x8035	# <CJK>
+0x7175	0x8043	# <CJK>
+0x7176	0x8046	# <CJK>
+0x7177	0x8079	# <CJK>
+0x7178	0x8052	# <CJK>
+0x7179	0x8075	# <CJK>
+0x717A	0x8071	# <CJK>
+0x717B	0x8983	# <CJK>
+0x717C	0x9807	# <CJK>
+0x717D	0x980E	# <CJK>
+0x717E	0x980F	# <CJK>
+0x7221	0x9821	# <CJK>
+0x7222	0x981C	# <CJK>
+0x7223	0x6F41	# <CJK>
+0x7224	0x9826	# <CJK>
+0x7225	0x9837	# <CJK>
+0x7226	0x984E	# <CJK>
+0x7227	0x9853	# <CJK>
+0x7228	0x9873	# <CJK>
+0x7229	0x9862	# <CJK>
+0x722A	0x9859	# <CJK>
+0x722B	0x9865	# <CJK>
+0x722C	0x986C	# <CJK>
+0x722D	0x9870	# <CJK>
+0x722E	0x864D	# <CJK>
+0x722F	0x8654	# <CJK>
+0x7230	0x866C	# <CJK>
+0x7231	0x87E3	# <CJK>
+0x7232	0x8806	# <CJK>
+0x7233	0x867A	# <CJK>
+0x7234	0x867C	# <CJK>
+0x7235	0x867B	# <CJK>
+0x7236	0x86A8	# <CJK>
+0x7237	0x868D	# <CJK>
+0x7238	0x868B	# <CJK>
+0x7239	0x8706	# <CJK>
+0x723A	0x869D	# <CJK>
+0x723B	0x86A7	# <CJK>
+0x723C	0x86A3	# <CJK>
+0x723D	0x86AA	# <CJK>
+0x723E	0x8693	# <CJK>
+0x723F	0x86A9	# <CJK>
+0x7240	0x86B6	# <CJK>
+0x7241	0x86C4	# <CJK>
+0x7242	0x86B5	# <CJK>
+0x7243	0x8823	# <CJK>
+0x7244	0x86B0	# <CJK>
+0x7245	0x86BA	# <CJK>
+0x7246	0x86B1	# <CJK>
+0x7247	0x86AF	# <CJK>
+0x7248	0x86C9	# <CJK>
+0x7249	0x87F6	# <CJK>
+0x724A	0x86B4	# <CJK>
+0x724B	0x86E9	# <CJK>
+0x724C	0x86FA	# <CJK>
+0x724D	0x87EF	# <CJK>
+0x724E	0x86ED	# <CJK>
+0x724F	0x8784	# <CJK>
+0x7250	0x86D0	# <CJK>
+0x7251	0x8713	# <CJK>
+0x7252	0x86DE	# <CJK>
+0x7253	0x8810	# <CJK>
+0x7254	0x86DF	# <CJK>
+0x7255	0x86D8	# <CJK>
+0x7256	0x86D1	# <CJK>
+0x7257	0x8703	# <CJK>
+0x7258	0x8707	# <CJK>
+0x7259	0x86F8	# <CJK>
+0x725A	0x8708	# <CJK>
+0x725B	0x870A	# <CJK>
+0x725C	0x870D	# <CJK>
+0x725D	0x8709	# <CJK>
+0x725E	0x8723	# <CJK>
+0x725F	0x873B	# <CJK>
+0x7260	0x871E	# <CJK>
+0x7261	0x8725	# <CJK>
+0x7262	0x872E	# <CJK>
+0x7263	0x871A	# <CJK>
+0x7264	0x873E	# <CJK>
+0x7265	0x87C8	# <CJK>
+0x7266	0x8734	# <CJK>
+0x7267	0x8731	# <CJK>
+0x7268	0x8729	# <CJK>
+0x7269	0x8737	# <CJK>
+0x726A	0x873F	# <CJK>
+0x726B	0x8782	# <CJK>
+0x726C	0x8722	# <CJK>
+0x726D	0x877D	# <CJK>
+0x726E	0x8811	# <CJK>
+0x726F	0x877B	# <CJK>
+0x7270	0x8760	# <CJK>
+0x7271	0x8770	# <CJK>
+0x7272	0x874C	# <CJK>
+0x7273	0x876E	# <CJK>
+0x7274	0x878B	# <CJK>
+0x7275	0x8753	# <CJK>
+0x7276	0x8763	# <CJK>
+0x7277	0x87BB	# <CJK>
+0x7278	0x8764	# <CJK>
+0x7279	0x8759	# <CJK>
+0x727A	0x8765	# <CJK>
+0x727B	0x8793	# <CJK>
+0x727C	0x87AF	# <CJK>
+0x727D	0x87CE	# <CJK>
+0x727E	0x87D2	# <CJK>
+0x7321	0x87C6	# <CJK>
+0x7322	0x8788	# <CJK>
+0x7323	0x8785	# <CJK>
+0x7324	0x87AD	# <CJK>
+0x7325	0x8797	# <CJK>
+0x7326	0x8783	# <CJK>
+0x7327	0x87AB	# <CJK>
+0x7328	0x87E5	# <CJK>
+0x7329	0x87AC	# <CJK>
+0x732A	0x87B5	# <CJK>
+0x732B	0x87B3	# <CJK>
+0x732C	0x87CB	# <CJK>
+0x732D	0x87D3	# <CJK>
+0x732E	0x87BD	# <CJK>
+0x732F	0x87D1	# <CJK>
+0x7330	0x87C0	# <CJK>
+0x7331	0x87CA	# <CJK>
+0x7332	0x87DB	# <CJK>
+0x7333	0x87EA	# <CJK>
+0x7334	0x87E0	# <CJK>
+0x7335	0x87EE	# <CJK>
+0x7336	0x8816	# <CJK>
+0x7337	0x8813	# <CJK>
+0x7338	0x87FE	# <CJK>
+0x7339	0x880A	# <CJK>
+0x733A	0x881B	# <CJK>
+0x733B	0x8821	# <CJK>
+0x733C	0x8839	# <CJK>
+0x733D	0x883C	# <CJK>
+0x733E	0x7F36	# <CJK>
+0x733F	0x7F4C	# <CJK>
+0x7340	0x7F44	# <CJK>
+0x7341	0x7F45	# <CJK>
+0x7342	0x8210	# <CJK>
+0x7343	0x7AFA	# <CJK>
+0x7344	0x7AFD	# <CJK>
+0x7345	0x7B08	# <CJK>
+0x7346	0x7BE4	# <CJK>
+0x7347	0x7B04	# <CJK>
+0x7348	0x7B67	# <CJK>
+0x7349	0x7B0A	# <CJK>
+0x734A	0x7B2B	# <CJK>
+0x734B	0x7B0F	# <CJK>
+0x734C	0x7B47	# <CJK>
+0x734D	0x7B38	# <CJK>
+0x734E	0x7B2A	# <CJK>
+0x734F	0x7B19	# <CJK>
+0x7350	0x7B2E	# <CJK>
+0x7351	0x7B31	# <CJK>
+0x7352	0x7B20	# <CJK>
+0x7353	0x7B25	# <CJK>
+0x7354	0x7B24	# <CJK>
+0x7355	0x7B33	# <CJK>
+0x7356	0x7C69	# <CJK>
+0x7357	0x7B1E	# <CJK>
+0x7358	0x7B58	# <CJK>
+0x7359	0x7BF3	# <CJK>
+0x735A	0x7B45	# <CJK>
+0x735B	0x7B75	# <CJK>
+0x735C	0x7B4C	# <CJK>
+0x735D	0x7B8F	# <CJK>
+0x735E	0x7B60	# <CJK>
+0x735F	0x7B6E	# <CJK>
+0x7360	0x7B7B	# <CJK>
+0x7361	0x7B62	# <CJK>
+0x7362	0x7B72	# <CJK>
+0x7363	0x7B71	# <CJK>
+0x7364	0x7B90	# <CJK>
+0x7365	0x7C00	# <CJK>
+0x7366	0x7BCB	# <CJK>
+0x7367	0x7BB8	# <CJK>
+0x7368	0x7BAC	# <CJK>
+0x7369	0x7B9D	# <CJK>
+0x736A	0x7C5C	# <CJK>
+0x736B	0x7B85	# <CJK>
+0x736C	0x7C1E	# <CJK>
+0x736D	0x7B9C	# <CJK>
+0x736E	0x7BA2	# <CJK>
+0x736F	0x7C2B	# <CJK>
+0x7370	0x7BB4	# <CJK>
+0x7371	0x7C23	# <CJK>
+0x7372	0x7BC1	# <CJK>
+0x7373	0x7BCC	# <CJK>
+0x7374	0x7BDD	# <CJK>
+0x7375	0x7BDA	# <CJK>
+0x7376	0x7BE5	# <CJK>
+0x7377	0x7BE6	# <CJK>
+0x7378	0x7BEA	# <CJK>
+0x7379	0x7C0C	# <CJK>
+0x737A	0x7BFE	# <CJK>
+0x737B	0x7BFC	# <CJK>
+0x737C	0x7C0F	# <CJK>
+0x737D	0x7C6A	# <CJK>
+0x737E	0x7C0B	# <CJK>
+0x7421	0x7C1F	# <CJK>
+0x7422	0x7C2A	# <CJK>
+0x7423	0x7C26	# <CJK>
+0x7424	0x7C38	# <CJK>
+0x7425	0x7C5F	# <CJK>
+0x7426	0x7C40	# <CJK>
+0x7427	0x81FE	# <CJK>
+0x7428	0x8201	# <CJK>
+0x7429	0x8202	# <CJK>
+0x742A	0x8204	# <CJK>
+0x742B	0x81EC	# <CJK>
+0x742C	0x8844	# <CJK>
+0x742D	0x8221	# <CJK>
+0x742E	0x8222	# <CJK>
+0x742F	0x8264	# <CJK>
+0x7430	0x822D	# <CJK>
+0x7431	0x822F	# <CJK>
+0x7432	0x8228	# <CJK>
+0x7433	0x822B	# <CJK>
+0x7434	0x8238	# <CJK>
+0x7435	0x826B	# <CJK>
+0x7436	0x8233	# <CJK>
+0x7437	0x8234	# <CJK>
+0x7438	0x823E	# <CJK>
+0x7439	0x8244	# <CJK>
+0x743A	0x8249	# <CJK>
+0x743B	0x824B	# <CJK>
+0x743C	0x824F	# <CJK>
+0x743D	0x825A	# <CJK>
+0x743E	0x825F	# <CJK>
+0x743F	0x8268	# <CJK>
+0x7440	0x887E	# <CJK>
+0x7441	0x88CA	# <CJK>
+0x7442	0x8888	# <CJK>
+0x7443	0x88D8	# <CJK>
+0x7444	0x88DF	# <CJK>
+0x7445	0x895E	# <CJK>
+0x7446	0x7F9D	# <CJK>
+0x7447	0x7FA5	# <CJK>
+0x7448	0x7FA7	# <CJK>
+0x7449	0x7FAF	# <CJK>
+0x744A	0x7FB0	# <CJK>
+0x744B	0x7FB2	# <CJK>
+0x744C	0x7C7C	# <CJK>
+0x744D	0x6549	# <CJK>
+0x744E	0x7C91	# <CJK>
+0x744F	0x7CF2	# <CJK>
+0x7450	0x7CF6	# <CJK>
+0x7451	0x7C9E	# <CJK>
+0x7452	0x7CA2	# <CJK>
+0x7453	0x7CB2	# <CJK>
+0x7454	0x7CBC	# <CJK>
+0x7455	0x7CBD	# <CJK>
+0x7456	0x7CDD	# <CJK>
+0x7457	0x7CC7	# <CJK>
+0x7458	0x7CCC	# <CJK>
+0x7459	0x7CCD	# <CJK>
+0x745A	0x7CC8	# <CJK>
+0x745B	0x7CC5	# <CJK>
+0x745C	0x7CD7	# <CJK>
+0x745D	0x7CE8	# <CJK>
+0x745E	0x826E	# <CJK>
+0x745F	0x66A8	# <CJK>
+0x7460	0x7FBF	# <CJK>
+0x7461	0x7FCE	# <CJK>
+0x7462	0x7FD5	# <CJK>
+0x7463	0x7FE5	# <CJK>
+0x7464	0x7FE1	# <CJK>
+0x7465	0x7FE6	# <CJK>
+0x7466	0x7FE9	# <CJK>
+0x7467	0x7FEE	# <CJK>
+0x7468	0x7FF3	# <CJK>
+0x7469	0x7CF8	# <CJK>
+0x746A	0x7E36	# <CJK>
+0x746B	0x7DA6	# <CJK>
+0x746C	0x7DAE	# <CJK>
+0x746D	0x7E47	# <CJK>
+0x746E	0x7E9B	# <CJK>
+0x746F	0x9EA9	# <CJK>
+0x7470	0x9EB4	# <CJK>
+0x7471	0x8D73	# <CJK>
+0x7472	0x8D84	# <CJK>
+0x7473	0x8D94	# <CJK>
+0x7474	0x8D91	# <CJK>
+0x7475	0x8DB2	# <CJK>
+0x7476	0x8D67	# <CJK>
+0x7477	0x8D6D	# <CJK>
+0x7478	0x8C47	# <CJK>
+0x7479	0x8C49	# <CJK>
+0x747A	0x914A	# <CJK>
+0x747B	0x9150	# <CJK>
+0x747C	0x914E	# <CJK>
+0x747D	0x914F	# <CJK>
+0x747E	0x9164	# <CJK>
+0x7521	0x9162	# <CJK>
+0x7522	0x9161	# <CJK>
+0x7523	0x9170	# <CJK>
+0x7524	0x9169	# <CJK>
+0x7525	0x916F	# <CJK>
+0x7526	0x91C5	# <CJK>
+0x7527	0x91C3	# <CJK>
+0x7528	0x9172	# <CJK>
+0x7529	0x9174	# <CJK>
+0x752A	0x9179	# <CJK>
+0x752B	0x918C	# <CJK>
+0x752C	0x9185	# <CJK>
+0x752D	0x9190	# <CJK>
+0x752E	0x918D	# <CJK>
+0x752F	0x9191	# <CJK>
+0x7530	0x91A2	# <CJK>
+0x7531	0x91A3	# <CJK>
+0x7532	0x91AA	# <CJK>
+0x7533	0x91AD	# <CJK>
+0x7534	0x91AE	# <CJK>
+0x7535	0x91AF	# <CJK>
+0x7536	0x91B5	# <CJK>
+0x7537	0x91B4	# <CJK>
+0x7538	0x91BA	# <CJK>
+0x7539	0x8C55	# <CJK>
+0x753A	0x9E7A	# <CJK>
+0x753B	0x8E89	# <CJK>
+0x753C	0x8DEB	# <CJK>
+0x753D	0x8E05	# <CJK>
+0x753E	0x8E59	# <CJK>
+0x753F	0x8E69	# <CJK>
+0x7540	0x8DB5	# <CJK>
+0x7541	0x8DBF	# <CJK>
+0x7542	0x8DBC	# <CJK>
+0x7543	0x8DBA	# <CJK>
+0x7544	0x8E4C	# <CJK>
+0x7545	0x8DD6	# <CJK>
+0x7546	0x8DD7	# <CJK>
+0x7547	0x8DDA	# <CJK>
+0x7548	0x8E92	# <CJK>
+0x7549	0x8DCE	# <CJK>
+0x754A	0x8DCF	# <CJK>
+0x754B	0x8DDB	# <CJK>
+0x754C	0x8DC6	# <CJK>
+0x754D	0x8DEC	# <CJK>
+0x754E	0x8E7A	# <CJK>
+0x754F	0x8E55	# <CJK>
+0x7550	0x8DE3	# <CJK>
+0x7551	0x8E9A	# <CJK>
+0x7552	0x8E8B	# <CJK>
+0x7553	0x8DE4	# <CJK>
+0x7554	0x8E09	# <CJK>
+0x7555	0x8DFD	# <CJK>
+0x7556	0x8E14	# <CJK>
+0x7557	0x8E1D	# <CJK>
+0x7558	0x8E1F	# <CJK>
+0x7559	0x8E93	# <CJK>
+0x755A	0x8E2E	# <CJK>
+0x755B	0x8E23	# <CJK>
+0x755C	0x8E91	# <CJK>
+0x755D	0x8E3A	# <CJK>
+0x755E	0x8E40	# <CJK>
+0x755F	0x8E39	# <CJK>
+0x7560	0x8E35	# <CJK>
+0x7561	0x8E3D	# <CJK>
+0x7562	0x8E31	# <CJK>
+0x7563	0x8E49	# <CJK>
+0x7564	0x8E41	# <CJK>
+0x7565	0x8E42	# <CJK>
+0x7566	0x8EA1	# <CJK>
+0x7567	0x8E63	# <CJK>
+0x7568	0x8E4A	# <CJK>
+0x7569	0x8E70	# <CJK>
+0x756A	0x8E76	# <CJK>
+0x756B	0x8E7C	# <CJK>
+0x756C	0x8E6F	# <CJK>
+0x756D	0x8E74	# <CJK>
+0x756E	0x8E85	# <CJK>
+0x756F	0x8EAA	# <CJK>
+0x7570	0x8E94	# <CJK>
+0x7571	0x8E90	# <CJK>
+0x7572	0x8EA6	# <CJK>
+0x7573	0x8E9E	# <CJK>
+0x7574	0x8C78	# <CJK>
+0x7575	0x8C82	# <CJK>
+0x7576	0x8C8A	# <CJK>
+0x7577	0x8C85	# <CJK>
+0x7578	0x8C98	# <CJK>
+0x7579	0x8C94	# <CJK>
+0x757A	0x659B	# <CJK>
+0x757B	0x89D6	# <CJK>
+0x757C	0x89F4	# <CJK>
+0x757D	0x89DA	# <CJK>
+0x757E	0x89DC	# <CJK>
+0x7621	0x89E5	# <CJK>
+0x7622	0x89EB	# <CJK>
+0x7623	0x89F6	# <CJK>
+0x7624	0x8A3E	# <CJK>
+0x7625	0x8B26	# <CJK>
+0x7626	0x975A	# <CJK>
+0x7627	0x96E9	# <CJK>
+0x7628	0x9742	# <CJK>
+0x7629	0x96EF	# <CJK>
+0x762A	0x9706	# <CJK>
+0x762B	0x973D	# <CJK>
+0x762C	0x9708	# <CJK>
+0x762D	0x970F	# <CJK>
+0x762E	0x970E	# <CJK>
+0x762F	0x972A	# <CJK>
+0x7630	0x9744	# <CJK>
+0x7631	0x9730	# <CJK>
+0x7632	0x973E	# <CJK>
+0x7633	0x9F54	# <CJK>
+0x7634	0x9F5F	# <CJK>
+0x7635	0x9F59	# <CJK>
+0x7636	0x9F60	# <CJK>
+0x7637	0x9F5C	# <CJK>
+0x7638	0x9F66	# <CJK>
+0x7639	0x9F6C	# <CJK>
+0x763A	0x9F6A	# <CJK>
+0x763B	0x9F77	# <CJK>
+0x763C	0x9EFD	# <CJK>
+0x763D	0x9EFF	# <CJK>
+0x763E	0x9F09	# <CJK>
+0x763F	0x96B9	# <CJK>
+0x7640	0x96BC	# <CJK>
+0x7641	0x96BD	# <CJK>
+0x7642	0x96CE	# <CJK>
+0x7643	0x96D2	# <CJK>
+0x7644	0x77BF	# <CJK>
+0x7645	0x8B8E	# <CJK>
+0x7646	0x928E	# <CJK>
+0x7647	0x947E	# <CJK>
+0x7648	0x92C8	# <CJK>
+0x7649	0x93E8	# <CJK>
+0x764A	0x936A	# <CJK>
+0x764B	0x93CA	# <CJK>
+0x764C	0x938F	# <CJK>
+0x764D	0x943E	# <CJK>
+0x764E	0x946B	# <CJK>
+0x764F	0x9B77	# <CJK>
+0x7650	0x9B74	# <CJK>
+0x7651	0x9B81	# <CJK>
+0x7652	0x9B83	# <CJK>
+0x7653	0x9B8E	# <CJK>
+0x7654	0x9C78	# <CJK>
+0x7655	0x7A4C	# <CJK>
+0x7656	0x9B92	# <CJK>
+0x7657	0x9C5F	# <CJK>
+0x7658	0x9B90	# <CJK>
+0x7659	0x9BAD	# <CJK>
+0x765A	0x9B9A	# <CJK>
+0x765B	0x9BAA	# <CJK>
+0x765C	0x9B9E	# <CJK>
+0x765D	0x9C6D	# <CJK>
+0x765E	0x9BAB	# <CJK>
+0x765F	0x9B9D	# <CJK>
+0x7660	0x9C58	# <CJK>
+0x7661	0x9BC1	# <CJK>
+0x7662	0x9C7A	# <CJK>
+0x7663	0x9C31	# <CJK>
+0x7664	0x9C39	# <CJK>
+0x7665	0x9C23	# <CJK>
+0x7666	0x9C37	# <CJK>
+0x7667	0x9BC0	# <CJK>
+0x7668	0x9BCA	# <CJK>
+0x7669	0x9BC7	# <CJK>
+0x766A	0x9BFD	# <CJK>
+0x766B	0x9BD6	# <CJK>
+0x766C	0x9BEA	# <CJK>
+0x766D	0x9BEB	# <CJK>
+0x766E	0x9BE1	# <CJK>
+0x766F	0x9BE4	# <CJK>
+0x7670	0x9BE7	# <CJK>
+0x7671	0x9BDD	# <CJK>
+0x7672	0x9BE2	# <CJK>
+0x7673	0x9BF0	# <CJK>
+0x7674	0x9BDB	# <CJK>
+0x7675	0x9BF4	# <CJK>
+0x7676	0x9BD4	# <CJK>
+0x7677	0x9C5D	# <CJK>
+0x7678	0x9C08	# <CJK>
+0x7679	0x9C10	# <CJK>
+0x767A	0x9C0D	# <CJK>
+0x767B	0x9C12	# <CJK>
+0x767C	0x9C09	# <CJK>
+0x767D	0x9BFF	# <CJK>
+0x767E	0x9C20	# <CJK>
+0x7721	0x9C32	# <CJK>
+0x7722	0x9C2D	# <CJK>
+0x7723	0x9C28	# <CJK>
+0x7724	0x9C25	# <CJK>
+0x7725	0x9C29	# <CJK>
+0x7726	0x9C33	# <CJK>
+0x7727	0x9C3E	# <CJK>
+0x7728	0x9C48	# <CJK>
+0x7729	0x9C3B	# <CJK>
+0x772A	0x9C35	# <CJK>
+0x772B	0x9C45	# <CJK>
+0x772C	0x9C56	# <CJK>
+0x772D	0x9C54	# <CJK>
+0x772E	0x9C52	# <CJK>
+0x772F	0x9C67	# <CJK>
+0x7730	0x977C	# <CJK>
+0x7731	0x9785	# <CJK>
+0x7732	0x97C3	# <CJK>
+0x7733	0x97BD	# <CJK>
+0x7734	0x9794	# <CJK>
+0x7735	0x97C9	# <CJK>
+0x7736	0x97AB	# <CJK>
+0x7737	0x97A3	# <CJK>
+0x7738	0x97B2	# <CJK>
+0x7739	0x97B4	# <CJK>
+0x773A	0x9AB1	# <CJK>
+0x773B	0x9AB0	# <CJK>
+0x773C	0x9AB7	# <CJK>
+0x773D	0x9DBB	# <CJK>
+0x773E	0x9AB6	# <CJK>
+0x773F	0x9ABA	# <CJK>
+0x7740	0x9ABC	# <CJK>
+0x7741	0x9AC1	# <CJK>
+0x7742	0x9AC0	# <CJK>
+0x7743	0x9ACF	# <CJK>
+0x7744	0x9AC2	# <CJK>
+0x7745	0x9AD6	# <CJK>
+0x7746	0x9AD5	# <CJK>
+0x7747	0x9AD1	# <CJK>
+0x7748	0x9B45	# <CJK>
+0x7749	0x9B43	# <CJK>
+0x774A	0x9B58	# <CJK>
+0x774B	0x9B4E	# <CJK>
+0x774C	0x9B48	# <CJK>
+0x774D	0x9B4D	# <CJK>
+0x774E	0x9B51	# <CJK>
+0x774F	0x9957	# <CJK>
+0x7750	0x995C	# <CJK>
+0x7751	0x992E	# <CJK>
+0x7752	0x9955	# <CJK>
+0x7753	0x9954	# <CJK>
+0x7754	0x9ADF	# <CJK>
+0x7755	0x9AE1	# <CJK>
+0x7756	0x9AE6	# <CJK>
+0x7757	0x9AEF	# <CJK>
+0x7758	0x9AEB	# <CJK>
+0x7759	0x9AFB	# <CJK>
+0x775A	0x9AED	# <CJK>
+0x775B	0x9AF9	# <CJK>
+0x775C	0x9B08	# <CJK>
+0x775D	0x9B0F	# <CJK>
+0x775E	0x9B22	# <CJK>
+0x775F	0x9B1F	# <CJK>
+0x7760	0x9B23	# <CJK>
+0x7761	0x4E48	# <CJK>
+0x7762	0x9EBE	# <CJK>
+0x7763	0x7E3B	# <CJK>
+0x7764	0x9E82	# <CJK>
+0x7765	0x9E87	# <CJK>
+0x7766	0x9E88	# <CJK>
+0x7767	0x9E8B	# <CJK>
+0x7768	0x9E92	# <CJK>
+0x7769	0x93D6	# <CJK>
+0x776A	0x9E9D	# <CJK>
+0x776B	0x9E9F	# <CJK>
+0x776C	0x9EDB	# <CJK>
+0x776D	0x9EDC	# <CJK>
+0x776E	0x9EDD	# <CJK>
+0x776F	0x9EE0	# <CJK>
+0x7770	0x9EDF	# <CJK>
+0x7771	0x9EE2	# <CJK>
+0x7772	0x9EF7	# <CJK>
+0x7773	0x9EE7	# <CJK>
+0x7774	0x9EE5	# <CJK>
+0x7775	0x9EF2	# <CJK>
+0x7776	0x9EEF	# <CJK>
+0x7777	0x9F22	# <CJK>
+0x7778	0x9F2C	# <CJK>
+0x7779	0x9F2F	# <CJK>
+0x777A	0x9F39	# <CJK>
+0x777B	0x9F37	# <CJK>
+0x777C	0x9F3D	# <CJK>
+0x777D	0x9F3E	# <CJK>
+0x777E	0x9F44	# <CJK>
+0x7821	0x896C	# <CJK>
+0x7822	0x95C6	# <CJK>
+0x7823	0x9336	# <CJK>
+0x7824	0x5F46	# <CJK>
+0x7825	0x8514	# <CJK>
+0x7826	0x7E94	# <CJK>
+0x7827	0x5382	# <CJK>
+0x7828	0x51B2	# <CJK>
+0x7829	0x4E11	# <CJK>
+0x782A	0x9F63	# <CJK>
+0x782B	0x5679	# <CJK>
+0x782C	0x515A	# <CJK>
+0x782D	0x6DC0	# <CJK>
+0x782E	0x9F15	# <CJK>
+0x782F	0x6597	# <CJK>
+0x7830	0x5641	# <CJK>
+0x7831	0x9AEE	# <CJK>
+0x7832	0x8303	# <CJK>
+0x7833	0x4E30	# <CJK>
+0x7834	0x8907	# <CJK>
+0x7835	0x5E72	# <CJK>
+0x7836	0x7A40	# <CJK>
+0x7837	0x98B3	# <CJK>
+0x7838	0x5E7F	# <CJK>
+0x7839	0x95A4	# <CJK>
+0x783A	0x9B0D	# <CJK>
+0x783B	0x5212	# <CJK>
+0x783C	0x8FF4	# <CJK>
+0x783D	0x5F59	# <CJK>
+0x783E	0x7A6B	# <CJK>
+0x783F	0x98E2	# <CJK>
+0x7840	0x51E0	# <CJK>
+0x7841	0x50A2	# <CJK>
+0x7842	0x4EF7	# <CJK>
+0x7843	0x8350	# <CJK>
+0x7844	0x8591	# <CJK>
+0x7845	0x5118	# <CJK>
+0x7846	0x636E	# <CJK>
+0x7847	0x6372	# <CJK>
+0x7848	0x524B	# <CJK>
+0x7849	0x5938	# <CJK>
+0x784A	0x774F	# <CJK>
+0x784B	0x8721	# <CJK>
+0x784C	0x814A	# <CJK>
+0x784D	0x7E8D	# <CJK>
+0x784E	0x91CC	# <CJK>
+0x784F	0x66C6	# <CJK>
+0x7850	0x5E18	# <CJK>
+0x7851	0x77AD	# <CJK>
+0x7852	0x9E75	# <CJK>
+0x7853	0x56C9	# <CJK>
+0x7854	0x9EF4	# <CJK>
+0x7855	0x6FDB	# <CJK>
+0x7856	0x61DE	# <CJK>
+0x7857	0x77C7	# <CJK>
+0x7858	0x7030	# <CJK>
+0x7859	0x9EB5	# <CJK>
+0x785A	0x884A	# <CJK>
+0x785B	0x95E2	# <CJK>
+0x785C	0x82F9	# <CJK>
+0x785D	0x51ED	# <CJK>
+0x785E	0x6251	# <CJK>
+0x785F	0x4EC6	# <CJK>
+0x7860	0x6734	# <CJK>
+0x7861	0x97C6	# <CJK>
+0x7862	0x7C64	# <CJK>
+0x7863	0x7E34	# <CJK>
+0x7864	0x97A6	# <CJK>
+0x7865	0x9EAF	# <CJK>
+0x7866	0x786E	# <CJK>
+0x7867	0x820D	# <CJK>
+0x7868	0x672F	# <CJK>
+0x7869	0x677E	# <CJK>
+0x786A	0x56CC	# <CJK>
+0x786B	0x53F0	# <CJK>
+0x786C	0x98B1	# <CJK>
+0x786D	0x6AAF	# <CJK>
+0x786E	0x7F4E	# <CJK>
+0x786F	0x6D82	# <CJK>
+0x7870	0x7CF0	# <CJK>
+0x7871	0x4E07	# <CJK>
+0x7872	0x4FC2	# <CJK>
+0x7873	0x7E6B	# <CJK>
+0x7874	0x9E79	# <CJK>
+0x7875	0x56AE	# <CJK>
+0x7876	0x9B1A	# <CJK>
+0x7877	0x846F	# <CJK>
+0x7878	0x53F6	# <CJK>
+0x7879	0x90C1	# <CJK>
+0x787A	0x79A6	# <CJK>
+0x787B	0x7C72	# <CJK>
+0x787C	0x613F	# <CJK>
+0x787D	0x4E91	# <CJK>
+0x787E	0x9AD2	# <CJK>
+0x7921	0x75C7	# <CJK>
+0x7922	0x96BB	# <CJK>
+0x7923	0x53EA	# <CJK>
+0x7924	0x7DFB	# <CJK>
+0x7925	0x88FD	# <CJK>
+0x7926	0x79CD	# <CJK>
+0x7927	0x7843	# <CJK>
+0x7928	0x7B51	# <CJK>
+0x7929	0x51C6	# <CJK>
Index: /tags/gsdl-21/gsdl/unicode/MAPPINGS/EASTASIA/GB/GB2312.TXT
===================================================================
--- /tags/gsdl-21/gsdl/unicode/MAPPINGS/EASTASIA/GB/GB2312.TXT	(revision 699)
+++ /tags/gsdl-21/gsdl/unicode/MAPPINGS/EASTASIA/GB/GB2312.TXT	(revision 699)
@@ -0,0 +1,7501 @@
+#
+#	Name:             GB2312-80 to Unicode table (complete, hex format)
+#	Unicode version:  1.1
+#	Table version:    0.0d2
+#	Table format:     Format A
+#	Date:             6 December 1993
+#	Author:           Glenn Adams <glenn@metis.com>
+#                     John H. Jenkins <John_Jenkins@taligent.com>
+#
+#	Copyright (c) 1991-1994 Unicode, Inc.  All Rights reserved.
+#
+#	This file is provided as-is by Unicode, Inc. (The Unicode Consortium).
+#	No claims are made as to fitness for any particular purpose.  No
+#	warranties of any kind are expressed or implied.  The recipient
+#	agrees to determine applicability of information provided.  If this
+#	file has been provided on magnetic media by Unicode, Inc., the sole
+#	remedy for any claim will be exchange of defective media within 90
+#	days of receipt.
+#
+#	Recipient is granted the right to make copies in any form for
+#	internal distribution and to freely use the information supplied
+#	in the creation of products supporting Unicode.  Unicode, Inc.
+#	specifically excludes the right to re-distribute this file directly
+#	to third parties or other organizations whether for profit or not.
+#
+#	General notes:
+#
+#	This table contains the data Metis and Taligent currently have on how
+#       GB2312-80 characters map into Unicode.
+#
+#	Format:  Three tab-separated columns
+#		 Column #1 is the GB2312 code (in hex as 0xXXXX)
+#		 Column #2 is the Unicode (in hex as 0xXXXX)
+#		 Column #3 the Unicode name (follows a comment sign, '#')
+#					The official names for Unicode characters U+4E00
+#					to U+9FA5, inclusive, is "CJK UNIFIED IDEOGRAPH-XXXX",
+#					where XXXX is the code point.  Including all these
+#					names in this file increases its size substantially
+#					and needlessly.  The token "<CJK>" is used for the
+#					name of these characters.  If necessary, it can be
+#					expanded algorithmically by a parser or editor.
+#
+#	The entries are in GB2312 order
+#
+#	The following algorithms can be used to change the hex form
+#		of GB2312 to other standard forms:
+#
+#		To change hex to EUC form, add 0x8080
+#		To change hex to kuten form, first subtract 0x2020.  Then
+#			the high and low bytes correspond to the ku and ten of
+#			the kuten form.  For example, 0x2121 -> 0x0101 -> 0101;
+#			0x777E -> 0x575E -> 8794
+#
+#	Any comments or problems, contact <John_Jenkins@taligent.com>
+#
+#
+0x2121	0x3000	# IDEOGRAPHIC SPACE
+0x2122	0x3001	# IDEOGRAPHIC COMMA
+0x2123	0x3002	# IDEOGRAPHIC FULL STOP
+0x2124	0x30FB	# KATAKANA MIDDLE DOT
+0x2125	0x02C9	# MODIFIER LETTER MACRON (Mandarin Chinese first tone)
+0x2126	0x02C7	# CARON (Mandarin Chinese third tone)
+0x2127	0x00A8	# DIAERESIS
+0x2128	0x3003	# DITTO MARK
+0x2129	0x3005	# IDEOGRAPHIC ITERATION MARK
+0x212A	0x2015	# HORIZONTAL BAR
+0x212B	0xFF5E	# FULLWIDTH TILDE
+0x212C	0x2225	# PARALLEL TO
+0x212D	0x2026	# HORIZONTAL ELLIPSIS
+0x212E	0x2018	# LEFT SINGLE QUOTATION MARK
+0x212F	0x2019	# RIGHT SINGLE QUOTATION MARK
+0x2130	0x201C	# LEFT DOUBLE QUOTATION MARK
+0x2131	0x201D	# RIGHT DOUBLE QUOTATION MARK
+0x2132	0x3014	# LEFT TORTOISE SHELL BRACKET
+0x2133	0x3015	# RIGHT TORTOISE SHELL BRACKET
+0x2134	0x3008	# LEFT ANGLE BRACKET
+0x2135	0x3009	# RIGHT ANGLE BRACKET
+0x2136	0x300A	# LEFT DOUBLE ANGLE BRACKET
+0x2137	0x300B	# RIGHT DOUBLE ANGLE BRACKET
+0x2138	0x300C	# LEFT CORNER BRACKET
+0x2139	0x300D	# RIGHT CORNER BRACKET
+0x213A	0x300E	# LEFT WHITE CORNER BRACKET
+0x213B	0x300F	# RIGHT WHITE CORNER BRACKET
+0x213C	0x3016	# LEFT WHITE LENTICULAR BRACKET
+0x213D	0x3017	# RIGHT WHITE LENTICULAR BRACKET
+0x213E	0x3010	# LEFT BLACK LENTICULAR BRACKET
+0x213F	0x3011	# RIGHT BLACK LENTICULAR BRACKET
+0x2140	0x00B1	# PLUS-MINUS SIGN
+0x2141	0x00D7	# MULTIPLICATION SIGN
+0x2142	0x00F7	# DIVISION SIGN
+0x2143	0x2236	# RATIO
+0x2144	0x2227	# LOGICAL AND
+0x2145	0x2228	# LOGICAL OR
+0x2146	0x2211	# N-ARY SUMMATION
+0x2147	0x220F	# N-ARY PRODUCT
+0x2148	0x222A	# UNION
+0x2149	0x2229	# INTERSECTION
+0x214A	0x2208	# ELEMENT OF
+0x214B	0x2237	# PROPORTION
+0x214C	0x221A	# SQUARE ROOT
+0x214D	0x22A5	# UP TACK
+0x214E	0x2225	# PARALLEL TO
+0x214F	0x2220	# ANGLE
+0x2150	0x2312	# ARC
+0x2151	0x2299	# CIRCLED DOT OPERATOR
+0x2152	0x222B	# INTEGRAL
+0x2153	0x222E	# CONTOUR INTEGRAL
+0x2154	0x2261	# IDENTICAL TO
+0x2155	0x224C	# ALL EQUAL TO
+0x2156	0x2248	# ALMOST EQUAL TO
+0x2157	0x223D	# REVERSED TILDE
+0x2158	0x221D	# PROPORTIONAL TO
+0x2159	0x2260	# NOT EQUAL TO
+0x215A	0x226E	# NOT LESS-THAN
+0x215B	0x226F	# NOT GREATER-THAN
+0x215C	0x2264	# LESS-THAN OR EQUAL TO
+0x215D	0x2265	# GREATER-THAN OR EQUAL TO
+0x215E	0x221E	# INFINITY
+0x215F	0x2235	# BECAUSE
+0x2160	0x2234	# THEREFORE
+0x2161	0x2642	# MALE SIGN
+0x2162	0x2640	# FEMALE SIGN
+0x2163	0x00B0	# DEGREE SIGN
+0x2164	0x2032	# PRIME
+0x2165	0x2033	# DOUBLE PRIME
+0x2166	0x2103	# DEGREE CELSIUS
+0x2167	0xFF04	# FULLWIDTH DOLLAR SIGN
+0x2168	0x00A4	# CURRENCY SIGN
+0x2169	0xFFE0	# FULLWIDTH CENT SIGN
+0x216A	0xFFE1	# FULLWIDTH POUND SIGN
+0x216B	0x2030	# PER MILLE SIGN
+0x216C	0x00A7	# SECTION SIGN
+0x216D	0x2116	# NUMERO SIGN
+0x216E	0x2606	# WHITE STAR
+0x216F	0x2605	# BLACK STAR
+0x2170	0x25CB	# WHITE CIRCLE
+0x2171	0x25CF	# BLACK CIRCLE
+0x2172	0x25CE	# BULLSEYE
+0x2173	0x25C7	# WHITE DIAMOND
+0x2174	0x25C6	# BLACK DIAMOND
+0x2175	0x25A1	# WHITE SQUARE
+0x2176	0x25A0	# BLACK SQUARE
+0x2177	0x25B3	# WHITE UP-POINTING TRIANGLE
+0x2178	0x25B2	# BLACK UP-POINTING TRIANGLE
+0x2179	0x203B	# REFERENCE MARK
+0x217A	0x2192	# RIGHTWARDS ARROW
+0x217B	0x2190	# LEFTWARDS ARROW
+0x217C	0x2191	# UPWARDS ARROW
+0x217D	0x2193	# DOWNWARDS ARROW
+0x217E	0x3013	# GETA MARK
+0x2231	0x2488	# DIGIT ONE FULL STOP
+0x2232	0x2489	# DIGIT TWO FULL STOP
+0x2233	0x248A	# DIGIT THREE FULL STOP
+0x2234	0x248B	# DIGIT FOUR FULL STOP
+0x2235	0x248C	# DIGIT FIVE FULL STOP
+0x2236	0x248D	# DIGIT SIX FULL STOP
+0x2237	0x248E	# DIGIT SEVEN FULL STOP
+0x2238	0x248F	# DIGIT EIGHT FULL STOP
+0x2239	0x2490	# DIGIT NINE FULL STOP
+0x223A	0x2491	# NUMBER TEN FULL STOP
+0x223B	0x2492	# NUMBER ELEVEN FULL STOP
+0x223C	0x2493	# NUMBER TWELVE FULL STOP
+0x223D	0x2494	# NUMBER THIRTEEN FULL STOP
+0x223E	0x2495	# NUMBER FOURTEEN FULL STOP
+0x223F	0x2496	# NUMBER FIFTEEN FULL STOP
+0x2240	0x2497	# NUMBER SIXTEEN FULL STOP
+0x2241	0x2498	# NUMBER SEVENTEEN FULL STOP
+0x2242	0x2499	# NUMBER EIGHTEEN FULL STOP
+0x2243	0x249A	# NUMBER NINETEEN FULL STOP
+0x2244	0x249B	# NUMBER TWENTY FULL STOP
+0x2245	0x2474	# PARENTHESIZED DIGIT ONE
+0x2246	0x2475	# PARENTHESIZED DIGIT TWO
+0x2247	0x2476	# PARENTHESIZED DIGIT THREE
+0x2248	0x2477	# PARENTHESIZED DIGIT FOUR
+0x2249	0x2478	# PARENTHESIZED DIGIT FIVE
+0x224A	0x2479	# PARENTHESIZED DIGIT SIX
+0x224B	0x247A	# PARENTHESIZED DIGIT SEVEN
+0x224C	0x247B	# PARENTHESIZED DIGIT EIGHT
+0x224D	0x247C	# PARENTHESIZED DIGIT NINE
+0x224E	0x247D	# PARENTHESIZED NUMBER TEN
+0x224F	0x247E	# PARENTHESIZED NUMBER ELEVEN
+0x2250	0x247F	# PARENTHESIZED NUMBER TWELVE
+0x2251	0x2480	# PARENTHESIZED NUMBER THIRTEEN
+0x2252	0x2481	# PARENTHESIZED NUMBER FOURTEEN
+0x2253	0x2482	# PARENTHESIZED NUMBER FIFTEEN
+0x2254	0x2483	# PARENTHESIZED NUMBER SIXTEEN
+0x2255	0x2484	# PARENTHESIZED NUMBER SEVENTEEN
+0x2256	0x2485	# PARENTHESIZED NUMBER EIGHTEEN
+0x2257	0x2486	# PARENTHESIZED NUMBER NINETEEN
+0x2258	0x2487	# PARENTHESIZED NUMBER TWENTY
+0x2259	0x2460	# CIRCLED DIGIT ONE
+0x225A	0x2461	# CIRCLED DIGIT TWO
+0x225B	0x2462	# CIRCLED DIGIT THREE
+0x225C	0x2463	# CIRCLED DIGIT FOUR
+0x225D	0x2464	# CIRCLED DIGIT FIVE
+0x225E	0x2465	# CIRCLED DIGIT SIX
+0x225F	0x2466	# CIRCLED DIGIT SEVEN
+0x2260	0x2467	# CIRCLED DIGIT EIGHT
+0x2261	0x2468	# CIRCLED DIGIT NINE
+0x2262	0x2469	# CIRCLED NUMBER TEN
+0x2265	0x3220	# PARENTHESIZED IDEOGRAPH ONE
+0x2266	0x3221	# PARENTHESIZED IDEOGRAPH TWO
+0x2267	0x3222	# PARENTHESIZED IDEOGRAPH THREE
+0x2268	0x3223	# PARENTHESIZED IDEOGRAPH FOUR
+0x2269	0x3224	# PARENTHESIZED IDEOGRAPH FIVE
+0x226A	0x3225	# PARENTHESIZED IDEOGRAPH SIX
+0x226B	0x3226	# PARENTHESIZED IDEOGRAPH SEVEN
+0x226C	0x3227	# PARENTHESIZED IDEOGRAPH EIGHT
+0x226D	0x3228	# PARENTHESIZED IDEOGRAPH NINE
+0x226E	0x3229	# PARENTHESIZED IDEOGRAPH TEN
+0x2271	0x2160	# ROMAN NUMERAL ONE
+0x2272	0x2161	# ROMAN NUMERAL TWO
+0x2273	0x2162	# ROMAN NUMERAL THREE
+0x2274	0x2163	# ROMAN NUMERAL FOUR
+0x2275	0x2164	# ROMAN NUMERAL FIVE
+0x2276	0x2165	# ROMAN NUMERAL SIX
+0x2277	0x2166	# ROMAN NUMERAL SEVEN
+0x2278	0x2167	# ROMAN NUMERAL EIGHT
+0x2279	0x2168	# ROMAN NUMERAL NINE
+0x227A	0x2169	# ROMAN NUMERAL TEN
+0x227B	0x216A	# ROMAN NUMERAL ELEVEN
+0x227C	0x216B	# ROMAN NUMERAL TWELVE
+0x2321	0xFF01	# FULLWIDTH EXCLAMATION MARK
+0x2322	0xFF02	# FULLWIDTH QUOTATION MARK
+0x2323	0xFF03	# FULLWIDTH NUMBER SIGN
+0x2324	0xFFE5	# FULLWIDTH YEN SIGN
+0x2325	0xFF05	# FULLWIDTH PERCENT SIGN
+0x2326	0xFF06	# FULLWIDTH AMPERSAND
+0x2327	0xFF07	# FULLWIDTH APOSTROPHE
+0x2328	0xFF08	# FULLWIDTH LEFT PARENTHESIS
+0x2329	0xFF09	# FULLWIDTH RIGHT PARENTHESIS
+0x232A	0xFF0A	# FULLWIDTH ASTERISK
+0x232B	0xFF0B	# FULLWIDTH PLUS SIGN
+0x232C	0xFF0C	# FULLWIDTH COMMA
+0x232D	0xFF0D	# FULLWIDTH HYPHEN-MINUS
+0x232E	0xFF0E	# FULLWIDTH FULL STOP
+0x232F	0xFF0F	# FULLWIDTH SOLIDUS
+0x2330	0xFF10	# FULLWIDTH DIGIT ZERO
+0x2331	0xFF11	# FULLWIDTH DIGIT ONE
+0x2332	0xFF12	# FULLWIDTH DIGIT TWO
+0x2333	0xFF13	# FULLWIDTH DIGIT THREE
+0x2334	0xFF14	# FULLWIDTH DIGIT FOUR
+0x2335	0xFF15	# FULLWIDTH DIGIT FIVE
+0x2336	0xFF16	# FULLWIDTH DIGIT SIX
+0x2337	0xFF17	# FULLWIDTH DIGIT SEVEN
+0x2338	0xFF18	# FULLWIDTH DIGIT EIGHT
+0x2339	0xFF19	# FULLWIDTH DIGIT NINE
+0x233A	0xFF1A	# FULLWIDTH COLON
+0x233B	0xFF1B	# FULLWIDTH SEMICOLON
+0x233C	0xFF1C	# FULLWIDTH LESS-THAN SIGN
+0x233D	0xFF1D	# FULLWIDTH EQUALS SIGN
+0x233E	0xFF1E	# FULLWIDTH GREATER-THAN SIGN
+0x233F	0xFF1F	# FULLWIDTH QUESTION MARK
+0x2340	0xFF20	# FULLWIDTH COMMERCIAL AT
+0x2341	0xFF21	# FULLWIDTH LATIN CAPITAL LETTER A
+0x2342	0xFF22	# FULLWIDTH LATIN CAPITAL LETTER B
+0x2343	0xFF23	# FULLWIDTH LATIN CAPITAL LETTER C
+0x2344	0xFF24	# FULLWIDTH LATIN CAPITAL LETTER D
+0x2345	0xFF25	# FULLWIDTH LATIN CAPITAL LETTER E
+0x2346	0xFF26	# FULLWIDTH LATIN CAPITAL LETTER F
+0x2347	0xFF27	# FULLWIDTH LATIN CAPITAL LETTER G
+0x2348	0xFF28	# FULLWIDTH LATIN CAPITAL LETTER H
+0x2349	0xFF29	# FULLWIDTH LATIN CAPITAL LETTER I
+0x234A	0xFF2A	# FULLWIDTH LATIN CAPITAL LETTER J
+0x234B	0xFF2B	# FULLWIDTH LATIN CAPITAL LETTER K
+0x234C	0xFF2C	# FULLWIDTH LATIN CAPITAL LETTER L
+0x234D	0xFF2D	# FULLWIDTH LATIN CAPITAL LETTER M
+0x234E	0xFF2E	# FULLWIDTH LATIN CAPITAL LETTER N
+0x234F	0xFF2F	# FULLWIDTH LATIN CAPITAL LETTER O
+0x2350	0xFF30	# FULLWIDTH LATIN CAPITAL LETTER P
+0x2351	0xFF31	# FULLWIDTH LATIN CAPITAL LETTER Q
+0x2352	0xFF32	# FULLWIDTH LATIN CAPITAL LETTER R
+0x2353	0xFF33	# FULLWIDTH LATIN CAPITAL LETTER S
+0x2354	0xFF34	# FULLWIDTH LATIN CAPITAL LETTER T
+0x2355	0xFF35	# FULLWIDTH LATIN CAPITAL LETTER U
+0x2356	0xFF36	# FULLWIDTH LATIN CAPITAL LETTER V
+0x2357	0xFF37	# FULLWIDTH LATIN CAPITAL LETTER W
+0x2358	0xFF38	# FULLWIDTH LATIN CAPITAL LETTER X
+0x2359	0xFF39	# FULLWIDTH LATIN CAPITAL LETTER Y
+0x235A	0xFF3A	# FULLWIDTH LATIN CAPITAL LETTER Z
+0x235B	0xFF3B	# FULLWIDTH LEFT SQUARE BRACKET
+0x235C	0xFF3C	# FULLWIDTH REVERSE SOLIDUS
+0x235D	0xFF3D	# FULLWIDTH RIGHT SQUARE BRACKET
+0x235E	0xFF3E	# FULLWIDTH CIRCUMFLEX ACCENT
+0x235F	0xFF3F	# FULLWIDTH LOW LINE
+0x2360	0xFF40	# FULLWIDTH GRAVE ACCENT
+0x2361	0xFF41	# FULLWIDTH LATIN SMALL LETTER A
+0x2362	0xFF42	# FULLWIDTH LATIN SMALL LETTER B
+0x2363	0xFF43	# FULLWIDTH LATIN SMALL LETTER C
+0x2364	0xFF44	# FULLWIDTH LATIN SMALL LETTER D
+0x2365	0xFF45	# FULLWIDTH LATIN SMALL LETTER E
+0x2366	0xFF46	# FULLWIDTH LATIN SMALL LETTER F
+0x2367	0xFF47	# FULLWIDTH LATIN SMALL LETTER G
+0x2368	0xFF48	# FULLWIDTH LATIN SMALL LETTER H
+0x2369	0xFF49	# FULLWIDTH LATIN SMALL LETTER I
+0x236A	0xFF4A	# FULLWIDTH LATIN SMALL LETTER J
+0x236B	0xFF4B	# FULLWIDTH LATIN SMALL LETTER K
+0x236C	0xFF4C	# FULLWIDTH LATIN SMALL LETTER L
+0x236D	0xFF4D	# FULLWIDTH LATIN SMALL LETTER M
+0x236E	0xFF4E	# FULLWIDTH LATIN SMALL LETTER N
+0x236F	0xFF4F	# FULLWIDTH LATIN SMALL LETTER O
+0x2370	0xFF50	# FULLWIDTH LATIN SMALL LETTER P
+0x2371	0xFF51	# FULLWIDTH LATIN SMALL LETTER Q
+0x2372	0xFF52	# FULLWIDTH LATIN SMALL LETTER R
+0x2373	0xFF53	# FULLWIDTH LATIN SMALL LETTER S
+0x2374	0xFF54	# FULLWIDTH LATIN SMALL LETTER T
+0x2375	0xFF55	# FULLWIDTH LATIN SMALL LETTER U
+0x2376	0xFF56	# FULLWIDTH LATIN SMALL LETTER V
+0x2377	0xFF57	# FULLWIDTH LATIN SMALL LETTER W
+0x2378	0xFF58	# FULLWIDTH LATIN SMALL LETTER X
+0x2379	0xFF59	# FULLWIDTH LATIN SMALL LETTER Y
+0x237A	0xFF5A	# FULLWIDTH LATIN SMALL LETTER Z
+0x237B	0xFF5B	# FULLWIDTH LEFT CURLY BRACKET
+0x237C	0xFF5C	# FULLWIDTH VERTICAL LINE
+0x237D	0xFF5D	# FULLWIDTH RIGHT CURLY BRACKET
+0x237E	0xFFE3	# FULLWIDTH MACRON
+0x2421	0x3041	# HIRAGANA LETTER SMALL A
+0x2422	0x3042	# HIRAGANA LETTER A
+0x2423	0x3043	# HIRAGANA LETTER SMALL I
+0x2424	0x3044	# HIRAGANA LETTER I
+0x2425	0x3045	# HIRAGANA LETTER SMALL U
+0x2426	0x3046	# HIRAGANA LETTER U
+0x2427	0x3047	# HIRAGANA LETTER SMALL E
+0x2428	0x3048	# HIRAGANA LETTER E
+0x2429	0x3049	# HIRAGANA LETTER SMALL O
+0x242A	0x304A	# HIRAGANA LETTER O
+0x242B	0x304B	# HIRAGANA LETTER KA
+0x242C	0x304C	# HIRAGANA LETTER GA
+0x242D	0x304D	# HIRAGANA LETTER KI
+0x242E	0x304E	# HIRAGANA LETTER GI
+0x242F	0x304F	# HIRAGANA LETTER KU
+0x2430	0x3050	# HIRAGANA LETTER GU
+0x2431	0x3051	# HIRAGANA LETTER KE
+0x2432	0x3052	# HIRAGANA LETTER GE
+0x2433	0x3053	# HIRAGANA LETTER KO
+0x2434	0x3054	# HIRAGANA LETTER GO
+0x2435	0x3055	# HIRAGANA LETTER SA
+0x2436	0x3056	# HIRAGANA LETTER ZA
+0x2437	0x3057	# HIRAGANA LETTER SI
+0x2438	0x3058	# HIRAGANA LETTER ZI
+0x2439	0x3059	# HIRAGANA LETTER SU
+0x243A	0x305A	# HIRAGANA LETTER ZU
+0x243B	0x305B	# HIRAGANA LETTER SE
+0x243C	0x305C	# HIRAGANA LETTER ZE
+0x243D	0x305D	# HIRAGANA LETTER SO
+0x243E	0x305E	# HIRAGANA LETTER ZO
+0x243F	0x305F	# HIRAGANA LETTER TA
+0x2440	0x3060	# HIRAGANA LETTER DA
+0x2441	0x3061	# HIRAGANA LETTER TI
+0x2442	0x3062	# HIRAGANA LETTER DI
+0x2443	0x3063	# HIRAGANA LETTER SMALL TU
+0x2444	0x3064	# HIRAGANA LETTER TU
+0x2445	0x3065	# HIRAGANA LETTER DU
+0x2446	0x3066	# HIRAGANA LETTER TE
+0x2447	0x3067	# HIRAGANA LETTER DE
+0x2448	0x3068	# HIRAGANA LETTER TO
+0x2449	0x3069	# HIRAGANA LETTER DO
+0x244A	0x306A	# HIRAGANA LETTER NA
+0x244B	0x306B	# HIRAGANA LETTER NI
+0x244C	0x306C	# HIRAGANA LETTER NU
+0x244D	0x306D	# HIRAGANA LETTER NE
+0x244E	0x306E	# HIRAGANA LETTER NO
+0x244F	0x306F	# HIRAGANA LETTER HA
+0x2450	0x3070	# HIRAGANA LETTER BA
+0x2451	0x3071	# HIRAGANA LETTER PA
+0x2452	0x3072	# HIRAGANA LETTER HI
+0x2453	0x3073	# HIRAGANA LETTER BI
+0x2454	0x3074	# HIRAGANA LETTER PI
+0x2455	0x3075	# HIRAGANA LETTER HU
+0x2456	0x3076	# HIRAGANA LETTER BU
+0x2457	0x3077	# HIRAGANA LETTER PU
+0x2458	0x3078	# HIRAGANA LETTER HE
+0x2459	0x3079	# HIRAGANA LETTER BE
+0x245A	0x307A	# HIRAGANA LETTER PE
+0x245B	0x307B	# HIRAGANA LETTER HO
+0x245C	0x307C	# HIRAGANA LETTER BO
+0x245D	0x307D	# HIRAGANA LETTER PO
+0x245E	0x307E	# HIRAGANA LETTER MA
+0x245F	0x307F	# HIRAGANA LETTER MI
+0x2460	0x3080	# HIRAGANA LETTER MU
+0x2461	0x3081	# HIRAGANA LETTER ME
+0x2462	0x3082	# HIRAGANA LETTER MO
+0x2463	0x3083	# HIRAGANA LETTER SMALL YA
+0x2464	0x3084	# HIRAGANA LETTER YA
+0x2465	0x3085	# HIRAGANA LETTER SMALL YU
+0x2466	0x3086	# HIRAGANA LETTER YU
+0x2467	0x3087	# HIRAGANA LETTER SMALL YO
+0x2468	0x3088	# HIRAGANA LETTER YO
+0x2469	0x3089	# HIRAGANA LETTER RA
+0x246A	0x308A	# HIRAGANA LETTER RI
+0x246B	0x308B	# HIRAGANA LETTER RU
+0x246C	0x308C	# HIRAGANA LETTER RE
+0x246D	0x308D	# HIRAGANA LETTER RO
+0x246E	0x308E	# HIRAGANA LETTER SMALL WA
+0x246F	0x308F	# HIRAGANA LETTER WA
+0x2470	0x3090	# HIRAGANA LETTER WI
+0x2471	0x3091	# HIRAGANA LETTER WE
+0x2472	0x3092	# HIRAGANA LETTER WO
+0x2473	0x3093	# HIRAGANA LETTER N
+0x2521	0x30A1	# KATAKANA LETTER SMALL A
+0x2522	0x30A2	# KATAKANA LETTER A
+0x2523	0x30A3	# KATAKANA LETTER SMALL I
+0x2524	0x30A4	# KATAKANA LETTER I
+0x2525	0x30A5	# KATAKANA LETTER SMALL U
+0x2526	0x30A6	# KATAKANA LETTER U
+0x2527	0x30A7	# KATAKANA LETTER SMALL E
+0x2528	0x30A8	# KATAKANA LETTER E
+0x2529	0x30A9	# KATAKANA LETTER SMALL O
+0x252A	0x30AA	# KATAKANA LETTER O
+0x252B	0x30AB	# KATAKANA LETTER KA
+0x252C	0x30AC	# KATAKANA LETTER GA
+0x252D	0x30AD	# KATAKANA LETTER KI
+0x252E	0x30AE	# KATAKANA LETTER GI
+0x252F	0x30AF	# KATAKANA LETTER KU
+0x2530	0x30B0	# KATAKANA LETTER GU
+0x2531	0x30B1	# KATAKANA LETTER KE
+0x2532	0x30B2	# KATAKANA LETTER GE
+0x2533	0x30B3	# KATAKANA LETTER KO
+0x2534	0x30B4	# KATAKANA LETTER GO
+0x2535	0x30B5	# KATAKANA LETTER SA
+0x2536	0x30B6	# KATAKANA LETTER ZA
+0x2537	0x30B7	# KATAKANA LETTER SI
+0x2538	0x30B8	# KATAKANA LETTER ZI
+0x2539	0x30B9	# KATAKANA LETTER SU
+0x253A	0x30BA	# KATAKANA LETTER ZU
+0x253B	0x30BB	# KATAKANA LETTER SE
+0x253C	0x30BC	# KATAKANA LETTER ZE
+0x253D	0x30BD	# KATAKANA LETTER SO
+0x253E	0x30BE	# KATAKANA LETTER ZO
+0x253F	0x30BF	# KATAKANA LETTER TA
+0x2540	0x30C0	# KATAKANA LETTER DA
+0x2541	0x30C1	# KATAKANA LETTER TI
+0x2542	0x30C2	# KATAKANA LETTER DI
+0x2543	0x30C3	# KATAKANA LETTER SMALL TU
+0x2544	0x30C4	# KATAKANA LETTER TU
+0x2545	0x30C5	# KATAKANA LETTER DU
+0x2546	0x30C6	# KATAKANA LETTER TE
+0x2547	0x30C7	# KATAKANA LETTER DE
+0x2548	0x30C8	# KATAKANA LETTER TO
+0x2549	0x30C9	# KATAKANA LETTER DO
+0x254A	0x30CA	# KATAKANA LETTER NA
+0x254B	0x30CB	# KATAKANA LETTER NI
+0x254C	0x30CC	# KATAKANA LETTER NU
+0x254D	0x30CD	# KATAKANA LETTER NE
+0x254E	0x30CE	# KATAKANA LETTER NO
+0x254F	0x30CF	# KATAKANA LETTER HA
+0x2550	0x30D0	# KATAKANA LETTER BA
+0x2551	0x30D1	# KATAKANA LETTER PA
+0x2552	0x30D2	# KATAKANA LETTER HI
+0x2553	0x30D3	# KATAKANA LETTER BI
+0x2554	0x30D4	# KATAKANA LETTER PI
+0x2555	0x30D5	# KATAKANA LETTER HU
+0x2556	0x30D6	# KATAKANA LETTER BU
+0x2557	0x30D7	# KATAKANA LETTER PU
+0x2558	0x30D8	# KATAKANA LETTER HE
+0x2559	0x30D9	# KATAKANA LETTER BE
+0x255A	0x30DA	# KATAKANA LETTER PE
+0x255B	0x30DB	# KATAKANA LETTER HO
+0x255C	0x30DC	# KATAKANA LETTER BO
+0x255D	0x30DD	# KATAKANA LETTER PO
+0x255E	0x30DE	# KATAKANA LETTER MA
+0x255F	0x30DF	# KATAKANA LETTER MI
+0x2560	0x30E0	# KATAKANA LETTER MU
+0x2561	0x30E1	# KATAKANA LETTER ME
+0x2562	0x30E2	# KATAKANA LETTER MO
+0x2563	0x30E3	# KATAKANA LETTER SMALL YA
+0x2564	0x30E4	# KATAKANA LETTER YA
+0x2565	0x30E5	# KATAKANA LETTER SMALL YU
+0x2566	0x30E6	# KATAKANA LETTER YU
+0x2567	0x30E7	# KATAKANA LETTER SMALL YO
+0x2568	0x30E8	# KATAKANA LETTER YO
+0x2569	0x30E9	# KATAKANA LETTER RA
+0x256A	0x30EA	# KATAKANA LETTER RI
+0x256B	0x30EB	# KATAKANA LETTER RU
+0x256C	0x30EC	# KATAKANA LETTER RE
+0x256D	0x30ED	# KATAKANA LETTER RO
+0x256E	0x30EE	# KATAKANA LETTER SMALL WA
+0x256F	0x30EF	# KATAKANA LETTER WA
+0x2570	0x30F0	# KATAKANA LETTER WI
+0x2571	0x30F1	# KATAKANA LETTER WE
+0x2572	0x30F2	# KATAKANA LETTER WO
+0x2573	0x30F3	# KATAKANA LETTER N
+0x2574	0x30F4	# KATAKANA LETTER VU
+0x2575	0x30F5	# KATAKANA LETTER SMALL KA
+0x2576	0x30F6	# KATAKANA LETTER SMALL KE
+0x2621	0x0391	# GREEK CAPITAL LETTER ALPHA
+0x2622	0x0392	# GREEK CAPITAL LETTER BETA
+0x2623	0x0393	# GREEK CAPITAL LETTER GAMMA
+0x2624	0x0394	# GREEK CAPITAL LETTER DELTA
+0x2625	0x0395	# GREEK CAPITAL LETTER EPSILON
+0x2626	0x0396	# GREEK CAPITAL LETTER ZETA
+0x2627	0x0397	# GREEK CAPITAL LETTER ETA
+0x2628	0x0398	# GREEK CAPITAL LETTER THETA
+0x2629	0x0399	# GREEK CAPITAL LETTER IOTA
+0x262A	0x039A	# GREEK CAPITAL LETTER KAPPA
+0x262B	0x039B	# GREEK CAPITAL LETTER LAMDA
+0x262C	0x039C	# GREEK CAPITAL LETTER MU
+0x262D	0x039D	# GREEK CAPITAL LETTER NU
+0x262E	0x039E	# GREEK CAPITAL LETTER XI
+0x262F	0x039F	# GREEK CAPITAL LETTER OMICRON
+0x2630	0x03A0	# GREEK CAPITAL LETTER PI
+0x2631	0x03A1	# GREEK CAPITAL LETTER RHO
+0x2632	0x03A3	# GREEK CAPITAL LETTER SIGMA
+0x2633	0x03A4	# GREEK CAPITAL LETTER TAU
+0x2634	0x03A5	# GREEK CAPITAL LETTER UPSILON
+0x2635	0x03A6	# GREEK CAPITAL LETTER PHI
+0x2636	0x03A7	# GREEK CAPITAL LETTER CHI
+0x2637	0x03A8	# GREEK CAPITAL LETTER PSI
+0x2638	0x03A9	# GREEK CAPITAL LETTER OMEGA
+0x2641	0x03B1	# GREEK SMALL LETTER ALPHA
+0x2642	0x03B2	# GREEK SMALL LETTER BETA
+0x2643	0x03B3	# GREEK SMALL LETTER GAMMA
+0x2644	0x03B4	# GREEK SMALL LETTER DELTA
+0x2645	0x03B5	# GREEK SMALL LETTER EPSILON
+0x2646	0x03B6	# GREEK SMALL LETTER ZETA
+0x2647	0x03B7	# GREEK SMALL LETTER ETA
+0x2648	0x03B8	# GREEK SMALL LETTER THETA
+0x2649	0x03B9	# GREEK SMALL LETTER IOTA
+0x264A	0x03BA	# GREEK SMALL LETTER KAPPA
+0x264B	0x03BB	# GREEK SMALL LETTER LAMDA
+0x264C	0x03BC	# GREEK SMALL LETTER MU
+0x264D	0x03BD	# GREEK SMALL LETTER NU
+0x264E	0x03BE	# GREEK SMALL LETTER XI
+0x264F	0x03BF	# GREEK SMALL LETTER OMICRON
+0x2650	0x03C0	# GREEK SMALL LETTER PI
+0x2651	0x03C1	# GREEK SMALL LETTER RHO
+0x2652	0x03C3	# GREEK SMALL LETTER SIGMA
+0x2653	0x03C4	# GREEK SMALL LETTER TAU
+0x2654	0x03C5	# GREEK SMALL LETTER UPSILON
+0x2655	0x03C6	# GREEK SMALL LETTER PHI
+0x2656	0x03C7	# GREEK SMALL LETTER CHI
+0x2657	0x03C8	# GREEK SMALL LETTER PSI
+0x2658	0x03C9	# GREEK SMALL LETTER OMEGA
+0x2721	0x0410	# CYRILLIC CAPITAL LETTER A
+0x2722	0x0411	# CYRILLIC CAPITAL LETTER BE
+0x2723	0x0412	# CYRILLIC CAPITAL LETTER VE
+0x2724	0x0413	# CYRILLIC CAPITAL LETTER GHE
+0x2725	0x0414	# CYRILLIC CAPITAL LETTER DE
+0x2726	0x0415	# CYRILLIC CAPITAL LETTER IE
+0x2727	0x0401	# CYRILLIC CAPITAL LETTER IO
+0x2728	0x0416	# CYRILLIC CAPITAL LETTER ZHE
+0x2729	0x0417	# CYRILLIC CAPITAL LETTER ZE
+0x272A	0x0418	# CYRILLIC CAPITAL LETTER I
+0x272B	0x0419	# CYRILLIC CAPITAL LETTER SHORT I
+0x272C	0x041A	# CYRILLIC CAPITAL LETTER KA
+0x272D	0x041B	# CYRILLIC CAPITAL LETTER EL
+0x272E	0x041C	# CYRILLIC CAPITAL LETTER EM
+0x272F	0x041D	# CYRILLIC CAPITAL LETTER EN
+0x2730	0x041E	# CYRILLIC CAPITAL LETTER O
+0x2731	0x041F	# CYRILLIC CAPITAL LETTER PE
+0x2732	0x0420	# CYRILLIC CAPITAL LETTER ER
+0x2733	0x0421	# CYRILLIC CAPITAL LETTER ES
+0x2734	0x0422	# CYRILLIC CAPITAL LETTER TE
+0x2735	0x0423	# CYRILLIC CAPITAL LETTER U
+0x2736	0x0424	# CYRILLIC CAPITAL LETTER EF
+0x2737	0x0425	# CYRILLIC CAPITAL LETTER HA
+0x2738	0x0426	# CYRILLIC CAPITAL LETTER TSE
+0x2739	0x0427	# CYRILLIC CAPITAL LETTER CHE
+0x273A	0x0428	# CYRILLIC CAPITAL LETTER SHA
+0x273B	0x0429	# CYRILLIC CAPITAL LETTER SHCHA
+0x273C	0x042A	# CYRILLIC CAPITAL LETTER HARD SIGN
+0x273D	0x042B	# CYRILLIC CAPITAL LETTER YERU
+0x273E	0x042C	# CYRILLIC CAPITAL LETTER SOFT SIGN
+0x273F	0x042D	# CYRILLIC CAPITAL LETTER E
+0x2740	0x042E	# CYRILLIC CAPITAL LETTER YU
+0x2741	0x042F	# CYRILLIC CAPITAL LETTER YA
+0x2751	0x0430	# CYRILLIC SMALL LETTER A
+0x2752	0x0431	# CYRILLIC SMALL LETTER BE
+0x2753	0x0432	# CYRILLIC SMALL LETTER VE
+0x2754	0x0433	# CYRILLIC SMALL LETTER GHE
+0x2755	0x0434	# CYRILLIC SMALL LETTER DE
+0x2756	0x0435	# CYRILLIC SMALL LETTER IE
+0x2757	0x0451	# CYRILLIC SMALL LETTER IO
+0x2758	0x0436	# CYRILLIC SMALL LETTER ZHE
+0x2759	0x0437	# CYRILLIC SMALL LETTER ZE
+0x275A	0x0438	# CYRILLIC SMALL LETTER I
+0x275B	0x0439	# CYRILLIC SMALL LETTER SHORT I
+0x275C	0x043A	# CYRILLIC SMALL LETTER KA
+0x275D	0x043B	# CYRILLIC SMALL LETTER EL
+0x275E	0x043C	# CYRILLIC SMALL LETTER EM
+0x275F	0x043D	# CYRILLIC SMALL LETTER EN
+0x2760	0x043E	# CYRILLIC SMALL LETTER O
+0x2761	0x043F	# CYRILLIC SMALL LETTER PE
+0x2762	0x0440	# CYRILLIC SMALL LETTER ER
+0x2763	0x0441	# CYRILLIC SMALL LETTER ES
+0x2764	0x0442	# CYRILLIC SMALL LETTER TE
+0x2765	0x0443	# CYRILLIC SMALL LETTER U
+0x2766	0x0444	# CYRILLIC SMALL LETTER EF
+0x2767	0x0445	# CYRILLIC SMALL LETTER HA
+0x2768	0x0446	# CYRILLIC SMALL LETTER TSE
+0x2769	0x0447	# CYRILLIC SMALL LETTER CHE
+0x276A	0x0448	# CYRILLIC SMALL LETTER SHA
+0x276B	0x0449	# CYRILLIC SMALL LETTER SHCHA
+0x276C	0x044A	# CYRILLIC SMALL LETTER HARD SIGN
+0x276D	0x044B	# CYRILLIC SMALL LETTER YERU
+0x276E	0x044C	# CYRILLIC SMALL LETTER SOFT SIGN
+0x276F	0x044D	# CYRILLIC SMALL LETTER E
+0x2770	0x044E	# CYRILLIC SMALL LETTER YU
+0x2771	0x044F	# CYRILLIC SMALL LETTER YA
+0x2821	0x0101	# LATIN SMALL LETTER A WITH MACRON
+0x2822	0x00E1	# LATIN SMALL LETTER A WITH ACUTE
+0x2823	0x01CE	# LATIN SMALL LETTER A WITH CARON
+0x2824	0x00E0	# LATIN SMALL LETTER A WITH GRAVE
+0x2825	0x0113	# LATIN SMALL LETTER E WITH MACRON
+0x2826	0x00E9	# LATIN SMALL LETTER E WITH ACUTE
+0x2827	0x011B	# LATIN SMALL LETTER E WITH CARON
+0x2828	0x00E8	# LATIN SMALL LETTER E WITH GRAVE
+0x2829	0x012B	# LATIN SMALL LETTER I WITH MACRON
+0x282A	0x00ED	# LATIN SMALL LETTER I WITH ACUTE
+0x282B	0x01D0	# LATIN SMALL LETTER I WITH CARON
+0x282C	0x00EC	# LATIN SMALL LETTER I WITH GRAVE
+0x282D	0x014D	# LATIN SMALL LETTER O WITH MACRON
+0x282E	0x00F3	# LATIN SMALL LETTER O WITH ACUTE
+0x282F	0x01D2	# LATIN SMALL LETTER O WITH CARON
+0x2830	0x00F2	# LATIN SMALL LETTER O WITH GRAVE
+0x2831	0x016B	# LATIN SMALL LETTER U WITH MACRON
+0x2832	0x00FA	# LATIN SMALL LETTER U WITH ACUTE
+0x2833	0x01D4	# LATIN SMALL LETTER U WITH CARON
+0x2834	0x00F9	# LATIN SMALL LETTER U WITH GRAVE
+0x2835	0x01D6	# LATIN SMALL LETTER U WITH DIAERESIS AND MACRON
+0x2836	0x01D8	# LATIN SMALL LETTER U WITH DIAERESIS AND ACUTE
+0x2837	0x01DA	# LATIN SMALL LETTER U WITH DIAERESIS AND CARON
+0x2838	0x01DC	# LATIN SMALL LETTER U WITH DIAERESIS AND GRAVE
+0x2839	0x00FC	# LATIN SMALL LETTER U WITH DIAERESIS
+0x283A	0x00EA	# LATIN SMALL LETTER E WITH CIRCUMFLEX
+0x2845	0x3105	# BOPOMOFO LETTER B
+0x2846	0x3106	# BOPOMOFO LETTER P
+0x2847	0x3107	# BOPOMOFO LETTER M
+0x2848	0x3108	# BOPOMOFO LETTER F
+0x2849	0x3109	# BOPOMOFO LETTER D
+0x284A	0x310A	# BOPOMOFO LETTER T
+0x284B	0x310B	# BOPOMOFO LETTER N
+0x284C	0x310C	# BOPOMOFO LETTER L
+0x284D	0x310D	# BOPOMOFO LETTER G
+0x284E	0x310E	# BOPOMOFO LETTER K
+0x284F	0x310F	# BOPOMOFO LETTER H
+0x2850	0x3110	# BOPOMOFO LETTER J
+0x2851	0x3111	# BOPOMOFO LETTER Q
+0x2852	0x3112	# BOPOMOFO LETTER X
+0x2853	0x3113	# BOPOMOFO LETTER ZH
+0x2854	0x3114	# BOPOMOFO LETTER CH
+0x2855	0x3115	# BOPOMOFO LETTER SH
+0x2856	0x3116	# BOPOMOFO LETTER R
+0x2857	0x3117	# BOPOMOFO LETTER Z
+0x2858	0x3118	# BOPOMOFO LETTER C
+0x2859	0x3119	# BOPOMOFO LETTER S
+0x285A	0x311A	# BOPOMOFO LETTER A
+0x285B	0x311B	# BOPOMOFO LETTER O
+0x285C	0x311C	# BOPOMOFO LETTER E
+0x285D	0x311D	# BOPOMOFO LETTER EH
+0x285E	0x311E	# BOPOMOFO LETTER AI
+0x285F	0x311F	# BOPOMOFO LETTER EI
+0x2860	0x3120	# BOPOMOFO LETTER AU
+0x2861	0x3121	# BOPOMOFO LETTER OU
+0x2862	0x3122	# BOPOMOFO LETTER AN
+0x2863	0x3123	# BOPOMOFO LETTER EN
+0x2864	0x3124	# BOPOMOFO LETTER ANG
+0x2865	0x3125	# BOPOMOFO LETTER ENG
+0x2866	0x3126	# BOPOMOFO LETTER ER
+0x2867	0x3127	# BOPOMOFO LETTER I
+0x2868	0x3128	# BOPOMOFO LETTER U
+0x2869	0x3129	# BOPOMOFO LETTER IU
+0x2924	0x2500	# BOX DRAWINGS LIGHT HORIZONTAL
+0x2925	0x2501	# BOX DRAWINGS HEAVY HORIZONTAL
+0x2926	0x2502	# BOX DRAWINGS LIGHT VERTICAL
+0x2927	0x2503	# BOX DRAWINGS HEAVY VERTICAL
+0x2928	0x2504	# BOX DRAWINGS LIGHT TRIPLE DASH HORIZONTAL
+0x2929	0x2505	# BOX DRAWINGS HEAVY TRIPLE DASH HORIZONTAL
+0x292A	0x2506	# BOX DRAWINGS LIGHT TRIPLE DASH VERTICAL
+0x292B	0x2507	# BOX DRAWINGS HEAVY TRIPLE DASH VERTICAL
+0x292C	0x2508	# BOX DRAWINGS LIGHT QUADRUPLE DASH HORIZONTAL
+0x292D	0x2509	# BOX DRAWINGS HEAVY QUADRUPLE DASH HORIZONTAL
+0x292E	0x250A	# BOX DRAWINGS LIGHT QUADRUPLE DASH VERTICAL
+0x292F	0x250B	# BOX DRAWINGS HEAVY QUADRUPLE DASH VERTICAL
+0x2930	0x250C	# BOX DRAWINGS LIGHT DOWN AND RIGHT
+0x2931	0x250D	# BOX DRAWINGS DOWN LIGHT AND RIGHT HEAVY
+0x2932	0x250E	# BOX DRAWINGS DOWN HEAVY AND RIGHT LIGHT
+0x2933	0x250F	# BOX DRAWINGS HEAVY DOWN AND RIGHT
+0x2934	0x2510	# BOX DRAWINGS LIGHT DOWN AND LEFT
+0x2935	0x2511	# BOX DRAWINGS DOWN LIGHT AND LEFT HEAVY
+0x2936	0x2512	# BOX DRAWINGS DOWN HEAVY AND LEFT LIGHT
+0x2937	0x2513	# BOX DRAWINGS HEAVY DOWN AND LEFT
+0x2938	0x2514	# BOX DRAWINGS LIGHT UP AND RIGHT
+0x2939	0x2515	# BOX DRAWINGS UP LIGHT AND RIGHT HEAVY
+0x293A	0x2516	# BOX DRAWINGS UP HEAVY AND RIGHT LIGHT
+0x293B	0x2517	# BOX DRAWINGS HEAVY UP AND RIGHT
+0x293C	0x2518	# BOX DRAWINGS LIGHT UP AND LEFT
+0x293D	0x2519	# BOX DRAWINGS UP LIGHT AND LEFT HEAVY
+0x293E	0x251A	# BOX DRAWINGS UP HEAVY AND LEFT LIGHT
+0x293F	0x251B	# BOX DRAWINGS HEAVY UP AND LEFT
+0x2940	0x251C	# BOX DRAWINGS LIGHT VERTICAL AND RIGHT
+0x2941	0x251D	# BOX DRAWINGS VERTICAL LIGHT AND RIGHT HEAVY
+0x2942	0x251E	# BOX DRAWINGS UP HEAVY AND RIGHT DOWN LIGHT
+0x2943	0x251F	# BOX DRAWINGS DOWN HEAVY AND RIGHT UP LIGHT
+0x2944	0x2520	# BOX DRAWINGS VERTICAL HEAVY AND RIGHT LIGHT
+0x2945	0x2521	# BOX DRAWINGS DOWN LIGHT AND RIGHT UP HEAVY
+0x2946	0x2522	# BOX DRAWINGS UP LIGHT AND RIGHT DOWN HEAVY
+0x2947	0x2523	# BOX DRAWINGS HEAVY VERTICAL AND RIGHT
+0x2948	0x2524	# BOX DRAWINGS LIGHT VERTICAL AND LEFT
+0x2949	0x2525	# BOX DRAWINGS VERTICAL LIGHT AND LEFT HEAVY
+0x294A	0x2526	# BOX DRAWINGS UP HEAVY AND LEFT DOWN LIGHT
+0x294B	0x2527	# BOX DRAWINGS DOWN HEAVY AND LEFT UP LIGHT
+0x294C	0x2528	# BOX DRAWINGS VERTICAL HEAVY AND LEFT LIGHT
+0x294D	0x2529	# BOX DRAWINGS DOWN LIGHT AND LEFT UP HEAVY
+0x294E	0x252A	# BOX DRAWINGS UP LIGHT AND LEFT DOWN HEAVY
+0x294F	0x252B	# BOX DRAWINGS HEAVY VERTICAL AND LEFT
+0x2950	0x252C	# BOX DRAWINGS LIGHT DOWN AND HORIZONTAL
+0x2951	0x252D	# BOX DRAWINGS LEFT HEAVY AND RIGHT DOWN LIGHT
+0x2952	0x252E	# BOX DRAWINGS RIGHT HEAVY AND LEFT DOWN LIGHT
+0x2953	0x252F	# BOX DRAWINGS DOWN LIGHT AND HORIZONTAL HEAVY
+0x2954	0x2530	# BOX DRAWINGS DOWN HEAVY AND HORIZONTAL LIGHT
+0x2955	0x2531	# BOX DRAWINGS RIGHT LIGHT AND LEFT DOWN HEAVY
+0x2956	0x2532	# BOX DRAWINGS LEFT LIGHT AND RIGHT DOWN HEAVY
+0x2957	0x2533	# BOX DRAWINGS HEAVY DOWN AND HORIZONTAL
+0x2958	0x2534	# BOX DRAWINGS LIGHT UP AND HORIZONTAL
+0x2959	0x2535	# BOX DRAWINGS LEFT HEAVY AND RIGHT UP LIGHT
+0x295A	0x2536	# BOX DRAWINGS RIGHT HEAVY AND LEFT UP LIGHT
+0x295B	0x2537	# BOX DRAWINGS UP LIGHT AND HORIZONTAL HEAVY
+0x295C	0x2538	# BOX DRAWINGS UP HEAVY AND HORIZONTAL LIGHT
+0x295D	0x2539	# BOX DRAWINGS RIGHT LIGHT AND LEFT UP HEAVY
+0x295E	0x253A	# BOX DRAWINGS LEFT LIGHT AND RIGHT UP HEAVY
+0x295F	0x253B	# BOX DRAWINGS HEAVY UP AND HORIZONTAL
+0x2960	0x253C	# BOX DRAWINGS LIGHT VERTICAL AND HORIZONTAL
+0x2961	0x253D	# BOX DRAWINGS LEFT HEAVY AND RIGHT VERTICAL LIGHT
+0x2962	0x253E	# BOX DRAWINGS RIGHT HEAVY AND LEFT VERTICAL LIGHT
+0x2963	0x253F	# BOX DRAWINGS VERTICAL LIGHT AND HORIZONTAL HEAVY
+0x2964	0x2540	# BOX DRAWINGS UP HEAVY AND DOWN HORIZONTAL LIGHT
+0x2965	0x2541	# BOX DRAWINGS DOWN HEAVY AND UP HORIZONTAL LIGHT
+0x2966	0x2542	# BOX DRAWINGS VERTICAL HEAVY AND HORIZONTAL LIGHT
+0x2967	0x2543	# BOX DRAWINGS LEFT UP HEAVY AND RIGHT DOWN LIGHT
+0x2968	0x2544	# BOX DRAWINGS RIGHT UP HEAVY AND LEFT DOWN LIGHT
+0x2969	0x2545	# BOX DRAWINGS LEFT DOWN HEAVY AND RIGHT UP LIGHT
+0x296A	0x2546	# BOX DRAWINGS RIGHT DOWN HEAVY AND LEFT UP LIGHT
+0x296B	0x2547	# BOX DRAWINGS DOWN LIGHT AND UP HORIZONTAL HEAVY
+0x296C	0x2548	# BOX DRAWINGS UP LIGHT AND DOWN HORIZONTAL HEAVY
+0x296D	0x2549	# BOX DRAWINGS RIGHT LIGHT AND LEFT VERTICAL HEAVY
+0x296E	0x254A	# BOX DRAWINGS LEFT LIGHT AND RIGHT VERTICAL HEAVY
+0x296F	0x254B	# BOX DRAWINGS HEAVY VERTICAL AND HORIZONTAL
+0x3021	0x554A	# <CJK>
+0x3022	0x963F	# <CJK>
+0x3023	0x57C3	# <CJK>
+0x3024	0x6328	# <CJK>
+0x3025	0x54CE	# <CJK>
+0x3026	0x5509	# <CJK>
+0x3027	0x54C0	# <CJK>
+0x3028	0x7691	# <CJK>
+0x3029	0x764C	# <CJK>
+0x302A	0x853C	# <CJK>
+0x302B	0x77EE	# <CJK>
+0x302C	0x827E	# <CJK>
+0x302D	0x788D	# <CJK>
+0x302E	0x7231	# <CJK>
+0x302F	0x9698	# <CJK>
+0x3030	0x978D	# <CJK>
+0x3031	0x6C28	# <CJK>
+0x3032	0x5B89	# <CJK>
+0x3033	0x4FFA	# <CJK>
+0x3034	0x6309	# <CJK>
+0x3035	0x6697	# <CJK>
+0x3036	0x5CB8	# <CJK>
+0x3037	0x80FA	# <CJK>
+0x3038	0x6848	# <CJK>
+0x3039	0x80AE	# <CJK>
+0x303A	0x6602	# <CJK>
+0x303B	0x76CE	# <CJK>
+0x303C	0x51F9	# <CJK>
+0x303D	0x6556	# <CJK>
+0x303E	0x71AC	# <CJK>
+0x303F	0x7FF1	# <CJK>
+0x3040	0x8884	# <CJK>
+0x3041	0x50B2	# <CJK>
+0x3042	0x5965	# <CJK>
+0x3043	0x61CA	# <CJK>
+0x3044	0x6FB3	# <CJK>
+0x3045	0x82AD	# <CJK>
+0x3046	0x634C	# <CJK>
+0x3047	0x6252	# <CJK>
+0x3048	0x53ED	# <CJK>
+0x3049	0x5427	# <CJK>
+0x304A	0x7B06	# <CJK>
+0x304B	0x516B	# <CJK>
+0x304C	0x75A4	# <CJK>
+0x304D	0x5DF4	# <CJK>
+0x304E	0x62D4	# <CJK>
+0x304F	0x8DCB	# <CJK>
+0x3050	0x9776	# <CJK>
+0x3051	0x628A	# <CJK>
+0x3052	0x8019	# <CJK>
+0x3053	0x575D	# <CJK>
+0x3054	0x9738	# <CJK>
+0x3055	0x7F62	# <CJK>
+0x3056	0x7238	# <CJK>
+0x3057	0x767D	# <CJK>
+0x3058	0x67CF	# <CJK>
+0x3059	0x767E	# <CJK>
+0x305A	0x6446	# <CJK>
+0x305B	0x4F70	# <CJK>
+0x305C	0x8D25	# <CJK>
+0x305D	0x62DC	# <CJK>
+0x305E	0x7A17	# <CJK>
+0x305F	0x6591	# <CJK>
+0x3060	0x73ED	# <CJK>
+0x3061	0x642C	# <CJK>
+0x3062	0x6273	# <CJK>
+0x3063	0x822C	# <CJK>
+0x3064	0x9881	# <CJK>
+0x3065	0x677F	# <CJK>
+0x3066	0x7248	# <CJK>
+0x3067	0x626E	# <CJK>
+0x3068	0x62CC	# <CJK>
+0x3069	0x4F34	# <CJK>
+0x306A	0x74E3	# <CJK>
+0x306B	0x534A	# <CJK>
+0x306C	0x529E	# <CJK>
+0x306D	0x7ECA	# <CJK>
+0x306E	0x90A6	# <CJK>
+0x306F	0x5E2E	# <CJK>
+0x3070	0x6886	# <CJK>
+0x3071	0x699C	# <CJK>
+0x3072	0x8180	# <CJK>
+0x3073	0x7ED1	# <CJK>
+0x3074	0x68D2	# <CJK>
+0x3075	0x78C5	# <CJK>
+0x3076	0x868C	# <CJK>
+0x3077	0x9551	# <CJK>
+0x3078	0x508D	# <CJK>
+0x3079	0x8C24	# <CJK>
+0x307A	0x82DE	# <CJK>
+0x307B	0x80DE	# <CJK>
+0x307C	0x5305	# <CJK>
+0x307D	0x8912	# <CJK>
+0x307E	0x5265	# <CJK>
+0x3121	0x8584	# <CJK>
+0x3122	0x96F9	# <CJK>
+0x3123	0x4FDD	# <CJK>
+0x3124	0x5821	# <CJK>
+0x3125	0x9971	# <CJK>
+0x3126	0x5B9D	# <CJK>
+0x3127	0x62B1	# <CJK>
+0x3128	0x62A5	# <CJK>
+0x3129	0x66B4	# <CJK>
+0x312A	0x8C79	# <CJK>
+0x312B	0x9C8D	# <CJK>
+0x312C	0x7206	# <CJK>
+0x312D	0x676F	# <CJK>
+0x312E	0x7891	# <CJK>
+0x312F	0x60B2	# <CJK>
+0x3130	0x5351	# <CJK>
+0x3131	0x5317	# <CJK>
+0x3132	0x8F88	# <CJK>
+0x3133	0x80CC	# <CJK>
+0x3134	0x8D1D	# <CJK>
+0x3135	0x94A1	# <CJK>
+0x3136	0x500D	# <CJK>
+0x3137	0x72C8	# <CJK>
+0x3138	0x5907	# <CJK>
+0x3139	0x60EB	# <CJK>
+0x313A	0x7119	# <CJK>
+0x313B	0x88AB	# <CJK>
+0x313C	0x5954	# <CJK>
+0x313D	0x82EF	# <CJK>
+0x313E	0x672C	# <CJK>
+0x313F	0x7B28	# <CJK>
+0x3140	0x5D29	# <CJK>
+0x3141	0x7EF7	# <CJK>
+0x3142	0x752D	# <CJK>
+0x3143	0x6CF5	# <CJK>
+0x3144	0x8E66	# <CJK>
+0x3145	0x8FF8	# <CJK>
+0x3146	0x903C	# <CJK>
+0x3147	0x9F3B	# <CJK>
+0x3148	0x6BD4	# <CJK>
+0x3149	0x9119	# <CJK>
+0x314A	0x7B14	# <CJK>
+0x314B	0x5F7C	# <CJK>
+0x314C	0x78A7	# <CJK>
+0x314D	0x84D6	# <CJK>
+0x314E	0x853D	# <CJK>
+0x314F	0x6BD5	# <CJK>
+0x3150	0x6BD9	# <CJK>
+0x3151	0x6BD6	# <CJK>
+0x3152	0x5E01	# <CJK>
+0x3153	0x5E87	# <CJK>
+0x3154	0x75F9	# <CJK>
+0x3155	0x95ED	# <CJK>
+0x3156	0x655D	# <CJK>
+0x3157	0x5F0A	# <CJK>
+0x3158	0x5FC5	# <CJK>
+0x3159	0x8F9F	# <CJK>
+0x315A	0x58C1	# <CJK>
+0x315B	0x81C2	# <CJK>
+0x315C	0x907F	# <CJK>
+0x315D	0x965B	# <CJK>
+0x315E	0x97AD	# <CJK>
+0x315F	0x8FB9	# <CJK>
+0x3160	0x7F16	# <CJK>
+0x3161	0x8D2C	# <CJK>
+0x3162	0x6241	# <CJK>
+0x3163	0x4FBF	# <CJK>
+0x3164	0x53D8	# <CJK>
+0x3165	0x535E	# <CJK>
+0x3166	0x8FA8	# <CJK>
+0x3167	0x8FA9	# <CJK>
+0x3168	0x8FAB	# <CJK>
+0x3169	0x904D	# <CJK>
+0x316A	0x6807	# <CJK>
+0x316B	0x5F6A	# <CJK>
+0x316C	0x8198	# <CJK>
+0x316D	0x8868	# <CJK>
+0x316E	0x9CD6	# <CJK>
+0x316F	0x618B	# <CJK>
+0x3170	0x522B	# <CJK>
+0x3171	0x762A	# <CJK>
+0x3172	0x5F6C	# <CJK>
+0x3173	0x658C	# <CJK>
+0x3174	0x6FD2	# <CJK>
+0x3175	0x6EE8	# <CJK>
+0x3176	0x5BBE	# <CJK>
+0x3177	0x6448	# <CJK>
+0x3178	0x5175	# <CJK>
+0x3179	0x51B0	# <CJK>
+0x317A	0x67C4	# <CJK>
+0x317B	0x4E19	# <CJK>
+0x317C	0x79C9	# <CJK>
+0x317D	0x997C	# <CJK>
+0x317E	0x70B3	# <CJK>
+0x3221	0x75C5	# <CJK>
+0x3222	0x5E76	# <CJK>
+0x3223	0x73BB	# <CJK>
+0x3224	0x83E0	# <CJK>
+0x3225	0x64AD	# <CJK>
+0x3226	0x62E8	# <CJK>
+0x3227	0x94B5	# <CJK>
+0x3228	0x6CE2	# <CJK>
+0x3229	0x535A	# <CJK>
+0x322A	0x52C3	# <CJK>
+0x322B	0x640F	# <CJK>
+0x322C	0x94C2	# <CJK>
+0x322D	0x7B94	# <CJK>
+0x322E	0x4F2F	# <CJK>
+0x322F	0x5E1B	# <CJK>
+0x3230	0x8236	# <CJK>
+0x3231	0x8116	# <CJK>
+0x3232	0x818A	# <CJK>
+0x3233	0x6E24	# <CJK>
+0x3234	0x6CCA	# <CJK>
+0x3235	0x9A73	# <CJK>
+0x3236	0x6355	# <CJK>
+0x3237	0x535C	# <CJK>
+0x3238	0x54FA	# <CJK>
+0x3239	0x8865	# <CJK>
+0x323A	0x57E0	# <CJK>
+0x323B	0x4E0D	# <CJK>
+0x323C	0x5E03	# <CJK>
+0x323D	0x6B65	# <CJK>
+0x323E	0x7C3F	# <CJK>
+0x323F	0x90E8	# <CJK>
+0x3240	0x6016	# <CJK>
+0x3241	0x64E6	# <CJK>
+0x3242	0x731C	# <CJK>
+0x3243	0x88C1	# <CJK>
+0x3244	0x6750	# <CJK>
+0x3245	0x624D	# <CJK>
+0x3246	0x8D22	# <CJK>
+0x3247	0x776C	# <CJK>
+0x3248	0x8E29	# <CJK>
+0x3249	0x91C7	# <CJK>
+0x324A	0x5F69	# <CJK>
+0x324B	0x83DC	# <CJK>
+0x324C	0x8521	# <CJK>
+0x324D	0x9910	# <CJK>
+0x324E	0x53C2	# <CJK>
+0x324F	0x8695	# <CJK>
+0x3250	0x6B8B	# <CJK>
+0x3251	0x60ED	# <CJK>
+0x3252	0x60E8	# <CJK>
+0x3253	0x707F	# <CJK>
+0x3254	0x82CD	# <CJK>
+0x3255	0x8231	# <CJK>
+0x3256	0x4ED3	# <CJK>
+0x3257	0x6CA7	# <CJK>
+0x3258	0x85CF	# <CJK>
+0x3259	0x64CD	# <CJK>
+0x325A	0x7CD9	# <CJK>
+0x325B	0x69FD	# <CJK>
+0x325C	0x66F9	# <CJK>
+0x325D	0x8349	# <CJK>
+0x325E	0x5395	# <CJK>
+0x325F	0x7B56	# <CJK>
+0x3260	0x4FA7	# <CJK>
+0x3261	0x518C	# <CJK>
+0x3262	0x6D4B	# <CJK>
+0x3263	0x5C42	# <CJK>
+0x3264	0x8E6D	# <CJK>
+0x3265	0x63D2	# <CJK>
+0x3266	0x53C9	# <CJK>
+0x3267	0x832C	# <CJK>
+0x3268	0x8336	# <CJK>
+0x3269	0x67E5	# <CJK>
+0x326A	0x78B4	# <CJK>
+0x326B	0x643D	# <CJK>
+0x326C	0x5BDF	# <CJK>
+0x326D	0x5C94	# <CJK>
+0x326E	0x5DEE	# <CJK>
+0x326F	0x8BE7	# <CJK>
+0x3270	0x62C6	# <CJK>
+0x3271	0x67F4	# <CJK>
+0x3272	0x8C7A	# <CJK>
+0x3273	0x6400	# <CJK>
+0x3274	0x63BA	# <CJK>
+0x3275	0x8749	# <CJK>
+0x3276	0x998B	# <CJK>
+0x3277	0x8C17	# <CJK>
+0x3278	0x7F20	# <CJK>
+0x3279	0x94F2	# <CJK>
+0x327A	0x4EA7	# <CJK>
+0x327B	0x9610	# <CJK>
+0x327C	0x98A4	# <CJK>
+0x327D	0x660C	# <CJK>
+0x327E	0x7316	# <CJK>
+0x3321	0x573A	# <CJK>
+0x3322	0x5C1D	# <CJK>
+0x3323	0x5E38	# <CJK>
+0x3324	0x957F	# <CJK>
+0x3325	0x507F	# <CJK>
+0x3326	0x80A0	# <CJK>
+0x3327	0x5382	# <CJK>
+0x3328	0x655E	# <CJK>
+0x3329	0x7545	# <CJK>
+0x332A	0x5531	# <CJK>
+0x332B	0x5021	# <CJK>
+0x332C	0x8D85	# <CJK>
+0x332D	0x6284	# <CJK>
+0x332E	0x949E	# <CJK>
+0x332F	0x671D	# <CJK>
+0x3330	0x5632	# <CJK>
+0x3331	0x6F6E	# <CJK>
+0x3332	0x5DE2	# <CJK>
+0x3333	0x5435	# <CJK>
+0x3334	0x7092	# <CJK>
+0x3335	0x8F66	# <CJK>
+0x3336	0x626F	# <CJK>
+0x3337	0x64A4	# <CJK>
+0x3338	0x63A3	# <CJK>
+0x3339	0x5F7B	# <CJK>
+0x333A	0x6F88	# <CJK>
+0x333B	0x90F4	# <CJK>
+0x333C	0x81E3	# <CJK>
+0x333D	0x8FB0	# <CJK>
+0x333E	0x5C18	# <CJK>
+0x333F	0x6668	# <CJK>
+0x3340	0x5FF1	# <CJK>
+0x3341	0x6C89	# <CJK>
+0x3342	0x9648	# <CJK>
+0x3343	0x8D81	# <CJK>
+0x3344	0x886C	# <CJK>
+0x3345	0x6491	# <CJK>
+0x3346	0x79F0	# <CJK>
+0x3347	0x57CE	# <CJK>
+0x3348	0x6A59	# <CJK>
+0x3349	0x6210	# <CJK>
+0x334A	0x5448	# <CJK>
+0x334B	0x4E58	# <CJK>
+0x334C	0x7A0B	# <CJK>
+0x334D	0x60E9	# <CJK>
+0x334E	0x6F84	# <CJK>
+0x334F	0x8BDA	# <CJK>
+0x3350	0x627F	# <CJK>
+0x3351	0x901E	# <CJK>
+0x3352	0x9A8B	# <CJK>
+0x3353	0x79E4	# <CJK>
+0x3354	0x5403	# <CJK>
+0x3355	0x75F4	# <CJK>
+0x3356	0x6301	# <CJK>
+0x3357	0x5319	# <CJK>
+0x3358	0x6C60	# <CJK>
+0x3359	0x8FDF	# <CJK>
+0x335A	0x5F1B	# <CJK>
+0x335B	0x9A70	# <CJK>
+0x335C	0x803B	# <CJK>
+0x335D	0x9F7F	# <CJK>
+0x335E	0x4F88	# <CJK>
+0x335F	0x5C3A	# <CJK>
+0x3360	0x8D64	# <CJK>
+0x3361	0x7FC5	# <CJK>
+0x3362	0x65A5	# <CJK>
+0x3363	0x70BD	# <CJK>
+0x3364	0x5145	# <CJK>
+0x3365	0x51B2	# <CJK>
+0x3366	0x866B	# <CJK>
+0x3367	0x5D07	# <CJK>
+0x3368	0x5BA0	# <CJK>
+0x3369	0x62BD	# <CJK>
+0x336A	0x916C	# <CJK>
+0x336B	0x7574	# <CJK>
+0x336C	0x8E0C	# <CJK>
+0x336D	0x7A20	# <CJK>
+0x336E	0x6101	# <CJK>
+0x336F	0x7B79	# <CJK>
+0x3370	0x4EC7	# <CJK>
+0x3371	0x7EF8	# <CJK>
+0x3372	0x7785	# <CJK>
+0x3373	0x4E11	# <CJK>
+0x3374	0x81ED	# <CJK>
+0x3375	0x521D	# <CJK>
+0x3376	0x51FA	# <CJK>
+0x3377	0x6A71	# <CJK>
+0x3378	0x53A8	# <CJK>
+0x3379	0x8E87	# <CJK>
+0x337A	0x9504	# <CJK>
+0x337B	0x96CF	# <CJK>
+0x337C	0x6EC1	# <CJK>
+0x337D	0x9664	# <CJK>
+0x337E	0x695A	# <CJK>
+0x3421	0x7840	# <CJK>
+0x3422	0x50A8	# <CJK>
+0x3423	0x77D7	# <CJK>
+0x3424	0x6410	# <CJK>
+0x3425	0x89E6	# <CJK>
+0x3426	0x5904	# <CJK>
+0x3427	0x63E3	# <CJK>
+0x3428	0x5DDD	# <CJK>
+0x3429	0x7A7F	# <CJK>
+0x342A	0x693D	# <CJK>
+0x342B	0x4F20	# <CJK>
+0x342C	0x8239	# <CJK>
+0x342D	0x5598	# <CJK>
+0x342E	0x4E32	# <CJK>
+0x342F	0x75AE	# <CJK>
+0x3430	0x7A97	# <CJK>
+0x3431	0x5E62	# <CJK>
+0x3432	0x5E8A	# <CJK>
+0x3433	0x95EF	# <CJK>
+0x3434	0x521B	# <CJK>
+0x3435	0x5439	# <CJK>
+0x3436	0x708A	# <CJK>
+0x3437	0x6376	# <CJK>
+0x3438	0x9524	# <CJK>
+0x3439	0x5782	# <CJK>
+0x343A	0x6625	# <CJK>
+0x343B	0x693F	# <CJK>
+0x343C	0x9187	# <CJK>
+0x343D	0x5507	# <CJK>
+0x343E	0x6DF3	# <CJK>
+0x343F	0x7EAF	# <CJK>
+0x3440	0x8822	# <CJK>
+0x3441	0x6233	# <CJK>
+0x3442	0x7EF0	# <CJK>
+0x3443	0x75B5	# <CJK>
+0x3444	0x8328	# <CJK>
+0x3445	0x78C1	# <CJK>
+0x3446	0x96CC	# <CJK>
+0x3447	0x8F9E	# <CJK>
+0x3448	0x6148	# <CJK>
+0x3449	0x74F7	# <CJK>
+0x344A	0x8BCD	# <CJK>
+0x344B	0x6B64	# <CJK>
+0x344C	0x523A	# <CJK>
+0x344D	0x8D50	# <CJK>
+0x344E	0x6B21	# <CJK>
+0x344F	0x806A	# <CJK>
+0x3450	0x8471	# <CJK>
+0x3451	0x56F1	# <CJK>
+0x3452	0x5306	# <CJK>
+0x3453	0x4ECE	# <CJK>
+0x3454	0x4E1B	# <CJK>
+0x3455	0x51D1	# <CJK>
+0x3456	0x7C97	# <CJK>
+0x3457	0x918B	# <CJK>
+0x3458	0x7C07	# <CJK>
+0x3459	0x4FC3	# <CJK>
+0x345A	0x8E7F	# <CJK>
+0x345B	0x7BE1	# <CJK>
+0x345C	0x7A9C	# <CJK>
+0x345D	0x6467	# <CJK>
+0x345E	0x5D14	# <CJK>
+0x345F	0x50AC	# <CJK>
+0x3460	0x8106	# <CJK>
+0x3461	0x7601	# <CJK>
+0x3462	0x7CB9	# <CJK>
+0x3463	0x6DEC	# <CJK>
+0x3464	0x7FE0	# <CJK>
+0x3465	0x6751	# <CJK>
+0x3466	0x5B58	# <CJK>
+0x3467	0x5BF8	# <CJK>
+0x3468	0x78CB	# <CJK>
+0x3469	0x64AE	# <CJK>
+0x346A	0x6413	# <CJK>
+0x346B	0x63AA	# <CJK>
+0x346C	0x632B	# <CJK>
+0x346D	0x9519	# <CJK>
+0x346E	0x642D	# <CJK>
+0x346F	0x8FBE	# <CJK>
+0x3470	0x7B54	# <CJK>
+0x3471	0x7629	# <CJK>
+0x3472	0x6253	# <CJK>
+0x3473	0x5927	# <CJK>
+0x3474	0x5446	# <CJK>
+0x3475	0x6B79	# <CJK>
+0x3476	0x50A3	# <CJK>
+0x3477	0x6234	# <CJK>
+0x3478	0x5E26	# <CJK>
+0x3479	0x6B86	# <CJK>
+0x347A	0x4EE3	# <CJK>
+0x347B	0x8D37	# <CJK>
+0x347C	0x888B	# <CJK>
+0x347D	0x5F85	# <CJK>
+0x347E	0x902E	# <CJK>
+0x3521	0x6020	# <CJK>
+0x3522	0x803D	# <CJK>
+0x3523	0x62C5	# <CJK>
+0x3524	0x4E39	# <CJK>
+0x3525	0x5355	# <CJK>
+0x3526	0x90F8	# <CJK>
+0x3527	0x63B8	# <CJK>
+0x3528	0x80C6	# <CJK>
+0x3529	0x65E6	# <CJK>
+0x352A	0x6C2E	# <CJK>
+0x352B	0x4F46	# <CJK>
+0x352C	0x60EE	# <CJK>
+0x352D	0x6DE1	# <CJK>
+0x352E	0x8BDE	# <CJK>
+0x352F	0x5F39	# <CJK>
+0x3530	0x86CB	# <CJK>
+0x3531	0x5F53	# <CJK>
+0x3532	0x6321	# <CJK>
+0x3533	0x515A	# <CJK>
+0x3534	0x8361	# <CJK>
+0x3535	0x6863	# <CJK>
+0x3536	0x5200	# <CJK>
+0x3537	0x6363	# <CJK>
+0x3538	0x8E48	# <CJK>
+0x3539	0x5012	# <CJK>
+0x353A	0x5C9B	# <CJK>
+0x353B	0x7977	# <CJK>
+0x353C	0x5BFC	# <CJK>
+0x353D	0x5230	# <CJK>
+0x353E	0x7A3B	# <CJK>
+0x353F	0x60BC	# <CJK>
+0x3540	0x9053	# <CJK>
+0x3541	0x76D7	# <CJK>
+0x3542	0x5FB7	# <CJK>
+0x3543	0x5F97	# <CJK>
+0x3544	0x7684	# <CJK>
+0x3545	0x8E6C	# <CJK>
+0x3546	0x706F	# <CJK>
+0x3547	0x767B	# <CJK>
+0x3548	0x7B49	# <CJK>
+0x3549	0x77AA	# <CJK>
+0x354A	0x51F3	# <CJK>
+0x354B	0x9093	# <CJK>
+0x354C	0x5824	# <CJK>
+0x354D	0x4F4E	# <CJK>
+0x354E	0x6EF4	# <CJK>
+0x354F	0x8FEA	# <CJK>
+0x3550	0x654C	# <CJK>
+0x3551	0x7B1B	# <CJK>
+0x3552	0x72C4	# <CJK>
+0x3553	0x6DA4	# <CJK>
+0x3554	0x7FDF	# <CJK>
+0x3555	0x5AE1	# <CJK>
+0x3556	0x62B5	# <CJK>
+0x3557	0x5E95	# <CJK>
+0x3558	0x5730	# <CJK>
+0x3559	0x8482	# <CJK>
+0x355A	0x7B2C	# <CJK>
+0x355B	0x5E1D	# <CJK>
+0x355C	0x5F1F	# <CJK>
+0x355D	0x9012	# <CJK>
+0x355E	0x7F14	# <CJK>
+0x355F	0x98A0	# <CJK>
+0x3560	0x6382	# <CJK>
+0x3561	0x6EC7	# <CJK>
+0x3562	0x7898	# <CJK>
+0x3563	0x70B9	# <CJK>
+0x3564	0x5178	# <CJK>
+0x3565	0x975B	# <CJK>
+0x3566	0x57AB	# <CJK>
+0x3567	0x7535	# <CJK>
+0x3568	0x4F43	# <CJK>
+0x3569	0x7538	# <CJK>
+0x356A	0x5E97	# <CJK>
+0x356B	0x60E6	# <CJK>
+0x356C	0x5960	# <CJK>
+0x356D	0x6DC0	# <CJK>
+0x356E	0x6BBF	# <CJK>
+0x356F	0x7889	# <CJK>
+0x3570	0x53FC	# <CJK>
+0x3571	0x96D5	# <CJK>
+0x3572	0x51CB	# <CJK>
+0x3573	0x5201	# <CJK>
+0x3574	0x6389	# <CJK>
+0x3575	0x540A	# <CJK>
+0x3576	0x9493	# <CJK>
+0x3577	0x8C03	# <CJK>
+0x3578	0x8DCC	# <CJK>
+0x3579	0x7239	# <CJK>
+0x357A	0x789F	# <CJK>
+0x357B	0x8776	# <CJK>
+0x357C	0x8FED	# <CJK>
+0x357D	0x8C0D	# <CJK>
+0x357E	0x53E0	# <CJK>
+0x3621	0x4E01	# <CJK>
+0x3622	0x76EF	# <CJK>
+0x3623	0x53EE	# <CJK>
+0x3624	0x9489	# <CJK>
+0x3625	0x9876	# <CJK>
+0x3626	0x9F0E	# <CJK>
+0x3627	0x952D	# <CJK>
+0x3628	0x5B9A	# <CJK>
+0x3629	0x8BA2	# <CJK>
+0x362A	0x4E22	# <CJK>
+0x362B	0x4E1C	# <CJK>
+0x362C	0x51AC	# <CJK>
+0x362D	0x8463	# <CJK>
+0x362E	0x61C2	# <CJK>
+0x362F	0x52A8	# <CJK>
+0x3630	0x680B	# <CJK>
+0x3631	0x4F97	# <CJK>
+0x3632	0x606B	# <CJK>
+0x3633	0x51BB	# <CJK>
+0x3634	0x6D1E	# <CJK>
+0x3635	0x515C	# <CJK>
+0x3636	0x6296	# <CJK>
+0x3637	0x6597	# <CJK>
+0x3638	0x9661	# <CJK>
+0x3639	0x8C46	# <CJK>
+0x363A	0x9017	# <CJK>
+0x363B	0x75D8	# <CJK>
+0x363C	0x90FD	# <CJK>
+0x363D	0x7763	# <CJK>
+0x363E	0x6BD2	# <CJK>
+0x363F	0x728A	# <CJK>
+0x3640	0x72EC	# <CJK>
+0x3641	0x8BFB	# <CJK>
+0x3642	0x5835	# <CJK>
+0x3643	0x7779	# <CJK>
+0x3644	0x8D4C	# <CJK>
+0x3645	0x675C	# <CJK>
+0x3646	0x9540	# <CJK>
+0x3647	0x809A	# <CJK>
+0x3648	0x5EA6	# <CJK>
+0x3649	0x6E21	# <CJK>
+0x364A	0x5992	# <CJK>
+0x364B	0x7AEF	# <CJK>
+0x364C	0x77ED	# <CJK>
+0x364D	0x953B	# <CJK>
+0x364E	0x6BB5	# <CJK>
+0x364F	0x65AD	# <CJK>
+0x3650	0x7F0E	# <CJK>
+0x3651	0x5806	# <CJK>
+0x3652	0x5151	# <CJK>
+0x3653	0x961F	# <CJK>
+0x3654	0x5BF9	# <CJK>
+0x3655	0x58A9	# <CJK>
+0x3656	0x5428	# <CJK>
+0x3657	0x8E72	# <CJK>
+0x3658	0x6566	# <CJK>
+0x3659	0x987F	# <CJK>
+0x365A	0x56E4	# <CJK>
+0x365B	0x949D	# <CJK>
+0x365C	0x76FE	# <CJK>
+0x365D	0x9041	# <CJK>
+0x365E	0x6387	# <CJK>
+0x365F	0x54C6	# <CJK>
+0x3660	0x591A	# <CJK>
+0x3661	0x593A	# <CJK>
+0x3662	0x579B	# <CJK>
+0x3663	0x8EB2	# <CJK>
+0x3664	0x6735	# <CJK>
+0x3665	0x8DFA	# <CJK>
+0x3666	0x8235	# <CJK>
+0x3667	0x5241	# <CJK>
+0x3668	0x60F0	# <CJK>
+0x3669	0x5815	# <CJK>
+0x366A	0x86FE	# <CJK>
+0x366B	0x5CE8	# <CJK>
+0x366C	0x9E45	# <CJK>
+0x366D	0x4FC4	# <CJK>
+0x366E	0x989D	# <CJK>
+0x366F	0x8BB9	# <CJK>
+0x3670	0x5A25	# <CJK>
+0x3671	0x6076	# <CJK>
+0x3672	0x5384	# <CJK>
+0x3673	0x627C	# <CJK>
+0x3674	0x904F	# <CJK>
+0x3675	0x9102	# <CJK>
+0x3676	0x997F	# <CJK>
+0x3677	0x6069	# <CJK>
+0x3678	0x800C	# <CJK>
+0x3679	0x513F	# <CJK>
+0x367A	0x8033	# <CJK>
+0x367B	0x5C14	# <CJK>
+0x367C	0x9975	# <CJK>
+0x367D	0x6D31	# <CJK>
+0x367E	0x4E8C	# <CJK>
+0x3721	0x8D30	# <CJK>
+0x3722	0x53D1	# <CJK>
+0x3723	0x7F5A	# <CJK>
+0x3724	0x7B4F	# <CJK>
+0x3725	0x4F10	# <CJK>
+0x3726	0x4E4F	# <CJK>
+0x3727	0x9600	# <CJK>
+0x3728	0x6CD5	# <CJK>
+0x3729	0x73D0	# <CJK>
+0x372A	0x85E9	# <CJK>
+0x372B	0x5E06	# <CJK>
+0x372C	0x756A	# <CJK>
+0x372D	0x7FFB	# <CJK>
+0x372E	0x6A0A	# <CJK>
+0x372F	0x77FE	# <CJK>
+0x3730	0x9492	# <CJK>
+0x3731	0x7E41	# <CJK>
+0x3732	0x51E1	# <CJK>
+0x3733	0x70E6	# <CJK>
+0x3734	0x53CD	# <CJK>
+0x3735	0x8FD4	# <CJK>
+0x3736	0x8303	# <CJK>
+0x3737	0x8D29	# <CJK>
+0x3738	0x72AF	# <CJK>
+0x3739	0x996D	# <CJK>
+0x373A	0x6CDB	# <CJK>
+0x373B	0x574A	# <CJK>
+0x373C	0x82B3	# <CJK>
+0x373D	0x65B9	# <CJK>
+0x373E	0x80AA	# <CJK>
+0x373F	0x623F	# <CJK>
+0x3740	0x9632	# <CJK>
+0x3741	0x59A8	# <CJK>
+0x3742	0x4EFF	# <CJK>
+0x3743	0x8BBF	# <CJK>
+0x3744	0x7EBA	# <CJK>
+0x3745	0x653E	# <CJK>
+0x3746	0x83F2	# <CJK>
+0x3747	0x975E	# <CJK>
+0x3748	0x5561	# <CJK>
+0x3749	0x98DE	# <CJK>
+0x374A	0x80A5	# <CJK>
+0x374B	0x532A	# <CJK>
+0x374C	0x8BFD	# <CJK>
+0x374D	0x5420	# <CJK>
+0x374E	0x80BA	# <CJK>
+0x374F	0x5E9F	# <CJK>
+0x3750	0x6CB8	# <CJK>
+0x3751	0x8D39	# <CJK>
+0x3752	0x82AC	# <CJK>
+0x3753	0x915A	# <CJK>
+0x3754	0x5429	# <CJK>
+0x3755	0x6C1B	# <CJK>
+0x3756	0x5206	# <CJK>
+0x3757	0x7EB7	# <CJK>
+0x3758	0x575F	# <CJK>
+0x3759	0x711A	# <CJK>
+0x375A	0x6C7E	# <CJK>
+0x375B	0x7C89	# <CJK>
+0x375C	0x594B	# <CJK>
+0x375D	0x4EFD	# <CJK>
+0x375E	0x5FFF	# <CJK>
+0x375F	0x6124	# <CJK>
+0x3760	0x7CAA	# <CJK>
+0x3761	0x4E30	# <CJK>
+0x3762	0x5C01	# <CJK>
+0x3763	0x67AB	# <CJK>
+0x3764	0x8702	# <CJK>
+0x3765	0x5CF0	# <CJK>
+0x3766	0x950B	# <CJK>
+0x3767	0x98CE	# <CJK>
+0x3768	0x75AF	# <CJK>
+0x3769	0x70FD	# <CJK>
+0x376A	0x9022	# <CJK>
+0x376B	0x51AF	# <CJK>
+0x376C	0x7F1D	# <CJK>
+0x376D	0x8BBD	# <CJK>
+0x376E	0x5949	# <CJK>
+0x376F	0x51E4	# <CJK>
+0x3770	0x4F5B	# <CJK>
+0x3771	0x5426	# <CJK>
+0x3772	0x592B	# <CJK>
+0x3773	0x6577	# <CJK>
+0x3774	0x80A4	# <CJK>
+0x3775	0x5B75	# <CJK>
+0x3776	0x6276	# <CJK>
+0x3777	0x62C2	# <CJK>
+0x3778	0x8F90	# <CJK>
+0x3779	0x5E45	# <CJK>
+0x377A	0x6C1F	# <CJK>
+0x377B	0x7B26	# <CJK>
+0x377C	0x4F0F	# <CJK>
+0x377D	0x4FD8	# <CJK>
+0x377E	0x670D	# <CJK>
+0x3821	0x6D6E	# <CJK>
+0x3822	0x6DAA	# <CJK>
+0x3823	0x798F	# <CJK>
+0x3824	0x88B1	# <CJK>
+0x3825	0x5F17	# <CJK>
+0x3826	0x752B	# <CJK>
+0x3827	0x629A	# <CJK>
+0x3828	0x8F85	# <CJK>
+0x3829	0x4FEF	# <CJK>
+0x382A	0x91DC	# <CJK>
+0x382B	0x65A7	# <CJK>
+0x382C	0x812F	# <CJK>
+0x382D	0x8151	# <CJK>
+0x382E	0x5E9C	# <CJK>
+0x382F	0x8150	# <CJK>
+0x3830	0x8D74	# <CJK>
+0x3831	0x526F	# <CJK>
+0x3832	0x8986	# <CJK>
+0x3833	0x8D4B	# <CJK>
+0x3834	0x590D	# <CJK>
+0x3835	0x5085	# <CJK>
+0x3836	0x4ED8	# <CJK>
+0x3837	0x961C	# <CJK>
+0x3838	0x7236	# <CJK>
+0x3839	0x8179	# <CJK>
+0x383A	0x8D1F	# <CJK>
+0x383B	0x5BCC	# <CJK>
+0x383C	0x8BA3	# <CJK>
+0x383D	0x9644	# <CJK>
+0x383E	0x5987	# <CJK>
+0x383F	0x7F1A	# <CJK>
+0x3840	0x5490	# <CJK>
+0x3841	0x5676	# <CJK>
+0x3842	0x560E	# <CJK>
+0x3843	0x8BE5	# <CJK>
+0x3844	0x6539	# <CJK>
+0x3845	0x6982	# <CJK>
+0x3846	0x9499	# <CJK>
+0x3847	0x76D6	# <CJK>
+0x3848	0x6E89	# <CJK>
+0x3849	0x5E72	# <CJK>
+0x384A	0x7518	# <CJK>
+0x384B	0x6746	# <CJK>
+0x384C	0x67D1	# <CJK>
+0x384D	0x7AFF	# <CJK>
+0x384E	0x809D	# <CJK>
+0x384F	0x8D76	# <CJK>
+0x3850	0x611F	# <CJK>
+0x3851	0x79C6	# <CJK>
+0x3852	0x6562	# <CJK>
+0x3853	0x8D63	# <CJK>
+0x3854	0x5188	# <CJK>
+0x3855	0x521A	# <CJK>
+0x3856	0x94A2	# <CJK>
+0x3857	0x7F38	# <CJK>
+0x3858	0x809B	# <CJK>
+0x3859	0x7EB2	# <CJK>
+0x385A	0x5C97	# <CJK>
+0x385B	0x6E2F	# <CJK>
+0x385C	0x6760	# <CJK>
+0x385D	0x7BD9	# <CJK>
+0x385E	0x768B	# <CJK>
+0x385F	0x9AD8	# <CJK>
+0x3860	0x818F	# <CJK>
+0x3861	0x7F94	# <CJK>
+0x3862	0x7CD5	# <CJK>
+0x3863	0x641E	# <CJK>
+0x3864	0x9550	# <CJK>
+0x3865	0x7A3F	# <CJK>
+0x3866	0x544A	# <CJK>
+0x3867	0x54E5	# <CJK>
+0x3868	0x6B4C	# <CJK>
+0x3869	0x6401	# <CJK>
+0x386A	0x6208	# <CJK>
+0x386B	0x9E3D	# <CJK>
+0x386C	0x80F3	# <CJK>
+0x386D	0x7599	# <CJK>
+0x386E	0x5272	# <CJK>
+0x386F	0x9769	# <CJK>
+0x3870	0x845B	# <CJK>
+0x3871	0x683C	# <CJK>
+0x3872	0x86E4	# <CJK>
+0x3873	0x9601	# <CJK>
+0x3874	0x9694	# <CJK>
+0x3875	0x94EC	# <CJK>
+0x3876	0x4E2A	# <CJK>
+0x3877	0x5404	# <CJK>
+0x3878	0x7ED9	# <CJK>
+0x3879	0x6839	# <CJK>
+0x387A	0x8DDF	# <CJK>
+0x387B	0x8015	# <CJK>
+0x387C	0x66F4	# <CJK>
+0x387D	0x5E9A	# <CJK>
+0x387E	0x7FB9	# <CJK>
+0x3921	0x57C2	# <CJK>
+0x3922	0x803F	# <CJK>
+0x3923	0x6897	# <CJK>
+0x3924	0x5DE5	# <CJK>
+0x3925	0x653B	# <CJK>
+0x3926	0x529F	# <CJK>
+0x3927	0x606D	# <CJK>
+0x3928	0x9F9A	# <CJK>
+0x3929	0x4F9B	# <CJK>
+0x392A	0x8EAC	# <CJK>
+0x392B	0x516C	# <CJK>
+0x392C	0x5BAB	# <CJK>
+0x392D	0x5F13	# <CJK>
+0x392E	0x5DE9	# <CJK>
+0x392F	0x6C5E	# <CJK>
+0x3930	0x62F1	# <CJK>
+0x3931	0x8D21	# <CJK>
+0x3932	0x5171	# <CJK>
+0x3933	0x94A9	# <CJK>
+0x3934	0x52FE	# <CJK>
+0x3935	0x6C9F	# <CJK>
+0x3936	0x82DF	# <CJK>
+0x3937	0x72D7	# <CJK>
+0x3938	0x57A2	# <CJK>
+0x3939	0x6784	# <CJK>
+0x393A	0x8D2D	# <CJK>
+0x393B	0x591F	# <CJK>
+0x393C	0x8F9C	# <CJK>
+0x393D	0x83C7	# <CJK>
+0x393E	0x5495	# <CJK>
+0x393F	0x7B8D	# <CJK>
+0x3940	0x4F30	# <CJK>
+0x3941	0x6CBD	# <CJK>
+0x3942	0x5B64	# <CJK>
+0x3943	0x59D1	# <CJK>
+0x3944	0x9F13	# <CJK>
+0x3945	0x53E4	# <CJK>
+0x3946	0x86CA	# <CJK>
+0x3947	0x9AA8	# <CJK>
+0x3948	0x8C37	# <CJK>
+0x3949	0x80A1	# <CJK>
+0x394A	0x6545	# <CJK>
+0x394B	0x987E	# <CJK>
+0x394C	0x56FA	# <CJK>
+0x394D	0x96C7	# <CJK>
+0x394E	0x522E	# <CJK>
+0x394F	0x74DC	# <CJK>
+0x3950	0x5250	# <CJK>
+0x3951	0x5BE1	# <CJK>
+0x3952	0x6302	# <CJK>
+0x3953	0x8902	# <CJK>
+0x3954	0x4E56	# <CJK>
+0x3955	0x62D0	# <CJK>
+0x3956	0x602A	# <CJK>
+0x3957	0x68FA	# <CJK>
+0x3958	0x5173	# <CJK>
+0x3959	0x5B98	# <CJK>
+0x395A	0x51A0	# <CJK>
+0x395B	0x89C2	# <CJK>
+0x395C	0x7BA1	# <CJK>
+0x395D	0x9986	# <CJK>
+0x395E	0x7F50	# <CJK>
+0x395F	0x60EF	# <CJK>
+0x3960	0x704C	# <CJK>
+0x3961	0x8D2F	# <CJK>
+0x3962	0x5149	# <CJK>
+0x3963	0x5E7F	# <CJK>
+0x3964	0x901B	# <CJK>
+0x3965	0x7470	# <CJK>
+0x3966	0x89C4	# <CJK>
+0x3967	0x572D	# <CJK>
+0x3968	0x7845	# <CJK>
+0x3969	0x5F52	# <CJK>
+0x396A	0x9F9F	# <CJK>
+0x396B	0x95FA	# <CJK>
+0x396C	0x8F68	# <CJK>
+0x396D	0x9B3C	# <CJK>
+0x396E	0x8BE1	# <CJK>
+0x396F	0x7678	# <CJK>
+0x3970	0x6842	# <CJK>
+0x3971	0x67DC	# <CJK>
+0x3972	0x8DEA	# <CJK>
+0x3973	0x8D35	# <CJK>
+0x3974	0x523D	# <CJK>
+0x3975	0x8F8A	# <CJK>
+0x3976	0x6EDA	# <CJK>
+0x3977	0x68CD	# <CJK>
+0x3978	0x9505	# <CJK>
+0x3979	0x90ED	# <CJK>
+0x397A	0x56FD	# <CJK>
+0x397B	0x679C	# <CJK>
+0x397C	0x88F9	# <CJK>
+0x397D	0x8FC7	# <CJK>
+0x397E	0x54C8	# <CJK>
+0x3A21	0x9AB8	# <CJK>
+0x3A22	0x5B69	# <CJK>
+0x3A23	0x6D77	# <CJK>
+0x3A24	0x6C26	# <CJK>
+0x3A25	0x4EA5	# <CJK>
+0x3A26	0x5BB3	# <CJK>
+0x3A27	0x9A87	# <CJK>
+0x3A28	0x9163	# <CJK>
+0x3A29	0x61A8	# <CJK>
+0x3A2A	0x90AF	# <CJK>
+0x3A2B	0x97E9	# <CJK>
+0x3A2C	0x542B	# <CJK>
+0x3A2D	0x6DB5	# <CJK>
+0x3A2E	0x5BD2	# <CJK>
+0x3A2F	0x51FD	# <CJK>
+0x3A30	0x558A	# <CJK>
+0x3A31	0x7F55	# <CJK>
+0x3A32	0x7FF0	# <CJK>
+0x3A33	0x64BC	# <CJK>
+0x3A34	0x634D	# <CJK>
+0x3A35	0x65F1	# <CJK>
+0x3A36	0x61BE	# <CJK>
+0x3A37	0x608D	# <CJK>
+0x3A38	0x710A	# <CJK>
+0x3A39	0x6C57	# <CJK>
+0x3A3A	0x6C49	# <CJK>
+0x3A3B	0x592F	# <CJK>
+0x3A3C	0x676D	# <CJK>
+0x3A3D	0x822A	# <CJK>
+0x3A3E	0x58D5	# <CJK>
+0x3A3F	0x568E	# <CJK>
+0x3A40	0x8C6A	# <CJK>
+0x3A41	0x6BEB	# <CJK>
+0x3A42	0x90DD	# <CJK>
+0x3A43	0x597D	# <CJK>
+0x3A44	0x8017	# <CJK>
+0x3A45	0x53F7	# <CJK>
+0x3A46	0x6D69	# <CJK>
+0x3A47	0x5475	# <CJK>
+0x3A48	0x559D	# <CJK>
+0x3A49	0x8377	# <CJK>
+0x3A4A	0x83CF	# <CJK>
+0x3A4B	0x6838	# <CJK>
+0x3A4C	0x79BE	# <CJK>
+0x3A4D	0x548C	# <CJK>
+0x3A4E	0x4F55	# <CJK>
+0x3A4F	0x5408	# <CJK>
+0x3A50	0x76D2	# <CJK>
+0x3A51	0x8C89	# <CJK>
+0x3A52	0x9602	# <CJK>
+0x3A53	0x6CB3	# <CJK>
+0x3A54	0x6DB8	# <CJK>
+0x3A55	0x8D6B	# <CJK>
+0x3A56	0x8910	# <CJK>
+0x3A57	0x9E64	# <CJK>
+0x3A58	0x8D3A	# <CJK>
+0x3A59	0x563F	# <CJK>
+0x3A5A	0x9ED1	# <CJK>
+0x3A5B	0x75D5	# <CJK>
+0x3A5C	0x5F88	# <CJK>
+0x3A5D	0x72E0	# <CJK>
+0x3A5E	0x6068	# <CJK>
+0x3A5F	0x54FC	# <CJK>
+0x3A60	0x4EA8	# <CJK>
+0x3A61	0x6A2A	# <CJK>
+0x3A62	0x8861	# <CJK>
+0x3A63	0x6052	# <CJK>
+0x3A64	0x8F70	# <CJK>
+0x3A65	0x54C4	# <CJK>
+0x3A66	0x70D8	# <CJK>
+0x3A67	0x8679	# <CJK>
+0x3A68	0x9E3F	# <CJK>
+0x3A69	0x6D2A	# <CJK>
+0x3A6A	0x5B8F	# <CJK>
+0x3A6B	0x5F18	# <CJK>
+0x3A6C	0x7EA2	# <CJK>
+0x3A6D	0x5589	# <CJK>
+0x3A6E	0x4FAF	# <CJK>
+0x3A6F	0x7334	# <CJK>
+0x3A70	0x543C	# <CJK>
+0x3A71	0x539A	# <CJK>
+0x3A72	0x5019	# <CJK>
+0x3A73	0x540E	# <CJK>
+0x3A74	0x547C	# <CJK>
+0x3A75	0x4E4E	# <CJK>
+0x3A76	0x5FFD	# <CJK>
+0x3A77	0x745A	# <CJK>
+0x3A78	0x58F6	# <CJK>
+0x3A79	0x846B	# <CJK>
+0x3A7A	0x80E1	# <CJK>
+0x3A7B	0x8774	# <CJK>
+0x3A7C	0x72D0	# <CJK>
+0x3A7D	0x7CCA	# <CJK>
+0x3A7E	0x6E56	# <CJK>
+0x3B21	0x5F27	# <CJK>
+0x3B22	0x864E	# <CJK>
+0x3B23	0x552C	# <CJK>
+0x3B24	0x62A4	# <CJK>
+0x3B25	0x4E92	# <CJK>
+0x3B26	0x6CAA	# <CJK>
+0x3B27	0x6237	# <CJK>
+0x3B28	0x82B1	# <CJK>
+0x3B29	0x54D7	# <CJK>
+0x3B2A	0x534E	# <CJK>
+0x3B2B	0x733E	# <CJK>
+0x3B2C	0x6ED1	# <CJK>
+0x3B2D	0x753B	# <CJK>
+0x3B2E	0x5212	# <CJK>
+0x3B2F	0x5316	# <CJK>
+0x3B30	0x8BDD	# <CJK>
+0x3B31	0x69D0	# <CJK>
+0x3B32	0x5F8A	# <CJK>
+0x3B33	0x6000	# <CJK>
+0x3B34	0x6DEE	# <CJK>
+0x3B35	0x574F	# <CJK>
+0x3B36	0x6B22	# <CJK>
+0x3B37	0x73AF	# <CJK>
+0x3B38	0x6853	# <CJK>
+0x3B39	0x8FD8	# <CJK>
+0x3B3A	0x7F13	# <CJK>
+0x3B3B	0x6362	# <CJK>
+0x3B3C	0x60A3	# <CJK>
+0x3B3D	0x5524	# <CJK>
+0x3B3E	0x75EA	# <CJK>
+0x3B3F	0x8C62	# <CJK>
+0x3B40	0x7115	# <CJK>
+0x3B41	0x6DA3	# <CJK>
+0x3B42	0x5BA6	# <CJK>
+0x3B43	0x5E7B	# <CJK>
+0x3B44	0x8352	# <CJK>
+0x3B45	0x614C	# <CJK>
+0x3B46	0x9EC4	# <CJK>
+0x3B47	0x78FA	# <CJK>
+0x3B48	0x8757	# <CJK>
+0x3B49	0x7C27	# <CJK>
+0x3B4A	0x7687	# <CJK>
+0x3B4B	0x51F0	# <CJK>
+0x3B4C	0x60F6	# <CJK>
+0x3B4D	0x714C	# <CJK>
+0x3B4E	0x6643	# <CJK>
+0x3B4F	0x5E4C	# <CJK>
+0x3B50	0x604D	# <CJK>
+0x3B51	0x8C0E	# <CJK>
+0x3B52	0x7070	# <CJK>
+0x3B53	0x6325	# <CJK>
+0x3B54	0x8F89	# <CJK>
+0x3B55	0x5FBD	# <CJK>
+0x3B56	0x6062	# <CJK>
+0x3B57	0x86D4	# <CJK>
+0x3B58	0x56DE	# <CJK>
+0x3B59	0x6BC1	# <CJK>
+0x3B5A	0x6094	# <CJK>
+0x3B5B	0x6167	# <CJK>
+0x3B5C	0x5349	# <CJK>
+0x3B5D	0x60E0	# <CJK>
+0x3B5E	0x6666	# <CJK>
+0x3B5F	0x8D3F	# <CJK>
+0x3B60	0x79FD	# <CJK>
+0x3B61	0x4F1A	# <CJK>
+0x3B62	0x70E9	# <CJK>
+0x3B63	0x6C47	# <CJK>
+0x3B64	0x8BB3	# <CJK>
+0x3B65	0x8BF2	# <CJK>
+0x3B66	0x7ED8	# <CJK>
+0x3B67	0x8364	# <CJK>
+0x3B68	0x660F	# <CJK>
+0x3B69	0x5A5A	# <CJK>
+0x3B6A	0x9B42	# <CJK>
+0x3B6B	0x6D51	# <CJK>
+0x3B6C	0x6DF7	# <CJK>
+0x3B6D	0x8C41	# <CJK>
+0x3B6E	0x6D3B	# <CJK>
+0x3B6F	0x4F19	# <CJK>
+0x3B70	0x706B	# <CJK>
+0x3B71	0x83B7	# <CJK>
+0x3B72	0x6216	# <CJK>
+0x3B73	0x60D1	# <CJK>
+0x3B74	0x970D	# <CJK>
+0x3B75	0x8D27	# <CJK>
+0x3B76	0x7978	# <CJK>
+0x3B77	0x51FB	# <CJK>
+0x3B78	0x573E	# <CJK>
+0x3B79	0x57FA	# <CJK>
+0x3B7A	0x673A	# <CJK>
+0x3B7B	0x7578	# <CJK>
+0x3B7C	0x7A3D	# <CJK>
+0x3B7D	0x79EF	# <CJK>
+0x3B7E	0x7B95	# <CJK>
+0x3C21	0x808C	# <CJK>
+0x3C22	0x9965	# <CJK>
+0x3C23	0x8FF9	# <CJK>
+0x3C24	0x6FC0	# <CJK>
+0x3C25	0x8BA5	# <CJK>
+0x3C26	0x9E21	# <CJK>
+0x3C27	0x59EC	# <CJK>
+0x3C28	0x7EE9	# <CJK>
+0x3C29	0x7F09	# <CJK>
+0x3C2A	0x5409	# <CJK>
+0x3C2B	0x6781	# <CJK>
+0x3C2C	0x68D8	# <CJK>
+0x3C2D	0x8F91	# <CJK>
+0x3C2E	0x7C4D	# <CJK>
+0x3C2F	0x96C6	# <CJK>
+0x3C30	0x53CA	# <CJK>
+0x3C31	0x6025	# <CJK>
+0x3C32	0x75BE	# <CJK>
+0x3C33	0x6C72	# <CJK>
+0x3C34	0x5373	# <CJK>
+0x3C35	0x5AC9	# <CJK>
+0x3C36	0x7EA7	# <CJK>
+0x3C37	0x6324	# <CJK>
+0x3C38	0x51E0	# <CJK>
+0x3C39	0x810A	# <CJK>
+0x3C3A	0x5DF1	# <CJK>
+0x3C3B	0x84DF	# <CJK>
+0x3C3C	0x6280	# <CJK>
+0x3C3D	0x5180	# <CJK>
+0x3C3E	0x5B63	# <CJK>
+0x3C3F	0x4F0E	# <CJK>
+0x3C40	0x796D	# <CJK>
+0x3C41	0x5242	# <CJK>
+0x3C42	0x60B8	# <CJK>
+0x3C43	0x6D4E	# <CJK>
+0x3C44	0x5BC4	# <CJK>
+0x3C45	0x5BC2	# <CJK>
+0x3C46	0x8BA1	# <CJK>
+0x3C47	0x8BB0	# <CJK>
+0x3C48	0x65E2	# <CJK>
+0x3C49	0x5FCC	# <CJK>
+0x3C4A	0x9645	# <CJK>
+0x3C4B	0x5993	# <CJK>
+0x3C4C	0x7EE7	# <CJK>
+0x3C4D	0x7EAA	# <CJK>
+0x3C4E	0x5609	# <CJK>
+0x3C4F	0x67B7	# <CJK>
+0x3C50	0x5939	# <CJK>
+0x3C51	0x4F73	# <CJK>
+0x3C52	0x5BB6	# <CJK>
+0x3C53	0x52A0	# <CJK>
+0x3C54	0x835A	# <CJK>
+0x3C55	0x988A	# <CJK>
+0x3C56	0x8D3E	# <CJK>
+0x3C57	0x7532	# <CJK>
+0x3C58	0x94BE	# <CJK>
+0x3C59	0x5047	# <CJK>
+0x3C5A	0x7A3C	# <CJK>
+0x3C5B	0x4EF7	# <CJK>
+0x3C5C	0x67B6	# <CJK>
+0x3C5D	0x9A7E	# <CJK>
+0x3C5E	0x5AC1	# <CJK>
+0x3C5F	0x6B7C	# <CJK>
+0x3C60	0x76D1	# <CJK>
+0x3C61	0x575A	# <CJK>
+0x3C62	0x5C16	# <CJK>
+0x3C63	0x7B3A	# <CJK>
+0x3C64	0x95F4	# <CJK>
+0x3C65	0x714E	# <CJK>
+0x3C66	0x517C	# <CJK>
+0x3C67	0x80A9	# <CJK>
+0x3C68	0x8270	# <CJK>
+0x3C69	0x5978	# <CJK>
+0x3C6A	0x7F04	# <CJK>
+0x3C6B	0x8327	# <CJK>
+0x3C6C	0x68C0	# <CJK>
+0x3C6D	0x67EC	# <CJK>
+0x3C6E	0x78B1	# <CJK>
+0x3C6F	0x7877	# <CJK>
+0x3C70	0x62E3	# <CJK>
+0x3C71	0x6361	# <CJK>
+0x3C72	0x7B80	# <CJK>
+0x3C73	0x4FED	# <CJK>
+0x3C74	0x526A	# <CJK>
+0x3C75	0x51CF	# <CJK>
+0x3C76	0x8350	# <CJK>
+0x3C77	0x69DB	# <CJK>
+0x3C78	0x9274	# <CJK>
+0x3C79	0x8DF5	# <CJK>
+0x3C7A	0x8D31	# <CJK>
+0x3C7B	0x89C1	# <CJK>
+0x3C7C	0x952E	# <CJK>
+0x3C7D	0x7BAD	# <CJK>
+0x3C7E	0x4EF6	# <CJK>
+0x3D21	0x5065	# <CJK>
+0x3D22	0x8230	# <CJK>
+0x3D23	0x5251	# <CJK>
+0x3D24	0x996F	# <CJK>
+0x3D25	0x6E10	# <CJK>
+0x3D26	0x6E85	# <CJK>
+0x3D27	0x6DA7	# <CJK>
+0x3D28	0x5EFA	# <CJK>
+0x3D29	0x50F5	# <CJK>
+0x3D2A	0x59DC	# <CJK>
+0x3D2B	0x5C06	# <CJK>
+0x3D2C	0x6D46	# <CJK>
+0x3D2D	0x6C5F	# <CJK>
+0x3D2E	0x7586	# <CJK>
+0x3D2F	0x848B	# <CJK>
+0x3D30	0x6868	# <CJK>
+0x3D31	0x5956	# <CJK>
+0x3D32	0x8BB2	# <CJK>
+0x3D33	0x5320	# <CJK>
+0x3D34	0x9171	# <CJK>
+0x3D35	0x964D	# <CJK>
+0x3D36	0x8549	# <CJK>
+0x3D37	0x6912	# <CJK>
+0x3D38	0x7901	# <CJK>
+0x3D39	0x7126	# <CJK>
+0x3D3A	0x80F6	# <CJK>
+0x3D3B	0x4EA4	# <CJK>
+0x3D3C	0x90CA	# <CJK>
+0x3D3D	0x6D47	# <CJK>
+0x3D3E	0x9A84	# <CJK>
+0x3D3F	0x5A07	# <CJK>
+0x3D40	0x56BC	# <CJK>
+0x3D41	0x6405	# <CJK>
+0x3D42	0x94F0	# <CJK>
+0x3D43	0x77EB	# <CJK>
+0x3D44	0x4FA5	# <CJK>
+0x3D45	0x811A	# <CJK>
+0x3D46	0x72E1	# <CJK>
+0x3D47	0x89D2	# <CJK>
+0x3D48	0x997A	# <CJK>
+0x3D49	0x7F34	# <CJK>
+0x3D4A	0x7EDE	# <CJK>
+0x3D4B	0x527F	# <CJK>
+0x3D4C	0x6559	# <CJK>
+0x3D4D	0x9175	# <CJK>
+0x3D4E	0x8F7F	# <CJK>
+0x3D4F	0x8F83	# <CJK>
+0x3D50	0x53EB	# <CJK>
+0x3D51	0x7A96	# <CJK>
+0x3D52	0x63ED	# <CJK>
+0x3D53	0x63A5	# <CJK>
+0x3D54	0x7686	# <CJK>
+0x3D55	0x79F8	# <CJK>
+0x3D56	0x8857	# <CJK>
+0x3D57	0x9636	# <CJK>
+0x3D58	0x622A	# <CJK>
+0x3D59	0x52AB	# <CJK>
+0x3D5A	0x8282	# <CJK>
+0x3D5B	0x6854	# <CJK>
+0x3D5C	0x6770	# <CJK>
+0x3D5D	0x6377	# <CJK>
+0x3D5E	0x776B	# <CJK>
+0x3D5F	0x7AED	# <CJK>
+0x3D60	0x6D01	# <CJK>
+0x3D61	0x7ED3	# <CJK>
+0x3D62	0x89E3	# <CJK>
+0x3D63	0x59D0	# <CJK>
+0x3D64	0x6212	# <CJK>
+0x3D65	0x85C9	# <CJK>
+0x3D66	0x82A5	# <CJK>
+0x3D67	0x754C	# <CJK>
+0x3D68	0x501F	# <CJK>
+0x3D69	0x4ECB	# <CJK>
+0x3D6A	0x75A5	# <CJK>
+0x3D6B	0x8BEB	# <CJK>
+0x3D6C	0x5C4A	# <CJK>
+0x3D6D	0x5DFE	# <CJK>
+0x3D6E	0x7B4B	# <CJK>
+0x3D6F	0x65A4	# <CJK>
+0x3D70	0x91D1	# <CJK>
+0x3D71	0x4ECA	# <CJK>
+0x3D72	0x6D25	# <CJK>
+0x3D73	0x895F	# <CJK>
+0x3D74	0x7D27	# <CJK>
+0x3D75	0x9526	# <CJK>
+0x3D76	0x4EC5	# <CJK>
+0x3D77	0x8C28	# <CJK>
+0x3D78	0x8FDB	# <CJK>
+0x3D79	0x9773	# <CJK>
+0x3D7A	0x664B	# <CJK>
+0x3D7B	0x7981	# <CJK>
+0x3D7C	0x8FD1	# <CJK>
+0x3D7D	0x70EC	# <CJK>
+0x3D7E	0x6D78	# <CJK>
+0x3E21	0x5C3D	# <CJK>
+0x3E22	0x52B2	# <CJK>
+0x3E23	0x8346	# <CJK>
+0x3E24	0x5162	# <CJK>
+0x3E25	0x830E	# <CJK>
+0x3E26	0x775B	# <CJK>
+0x3E27	0x6676	# <CJK>
+0x3E28	0x9CB8	# <CJK>
+0x3E29	0x4EAC	# <CJK>
+0x3E2A	0x60CA	# <CJK>
+0x3E2B	0x7CBE	# <CJK>
+0x3E2C	0x7CB3	# <CJK>
+0x3E2D	0x7ECF	# <CJK>
+0x3E2E	0x4E95	# <CJK>
+0x3E2F	0x8B66	# <CJK>
+0x3E30	0x666F	# <CJK>
+0x3E31	0x9888	# <CJK>
+0x3E32	0x9759	# <CJK>
+0x3E33	0x5883	# <CJK>
+0x3E34	0x656C	# <CJK>
+0x3E35	0x955C	# <CJK>
+0x3E36	0x5F84	# <CJK>
+0x3E37	0x75C9	# <CJK>
+0x3E38	0x9756	# <CJK>
+0x3E39	0x7ADF	# <CJK>
+0x3E3A	0x7ADE	# <CJK>
+0x3E3B	0x51C0	# <CJK>
+0x3E3C	0x70AF	# <CJK>
+0x3E3D	0x7A98	# <CJK>
+0x3E3E	0x63EA	# <CJK>
+0x3E3F	0x7A76	# <CJK>
+0x3E40	0x7EA0	# <CJK>
+0x3E41	0x7396	# <CJK>
+0x3E42	0x97ED	# <CJK>
+0x3E43	0x4E45	# <CJK>
+0x3E44	0x7078	# <CJK>
+0x3E45	0x4E5D	# <CJK>
+0x3E46	0x9152	# <CJK>
+0x3E47	0x53A9	# <CJK>
+0x3E48	0x6551	# <CJK>
+0x3E49	0x65E7	# <CJK>
+0x3E4A	0x81FC	# <CJK>
+0x3E4B	0x8205	# <CJK>
+0x3E4C	0x548E	# <CJK>
+0x3E4D	0x5C31	# <CJK>
+0x3E4E	0x759A	# <CJK>
+0x3E4F	0x97A0	# <CJK>
+0x3E50	0x62D8	# <CJK>
+0x3E51	0x72D9	# <CJK>
+0x3E52	0x75BD	# <CJK>
+0x3E53	0x5C45	# <CJK>
+0x3E54	0x9A79	# <CJK>
+0x3E55	0x83CA	# <CJK>
+0x3E56	0x5C40	# <CJK>
+0x3E57	0x5480	# <CJK>
+0x3E58	0x77E9	# <CJK>
+0x3E59	0x4E3E	# <CJK>
+0x3E5A	0x6CAE	# <CJK>
+0x3E5B	0x805A	# <CJK>
+0x3E5C	0x62D2	# <CJK>
+0x3E5D	0x636E	# <CJK>
+0x3E5E	0x5DE8	# <CJK>
+0x3E5F	0x5177	# <CJK>
+0x3E60	0x8DDD	# <CJK>
+0x3E61	0x8E1E	# <CJK>
+0x3E62	0x952F	# <CJK>
+0x3E63	0x4FF1	# <CJK>
+0x3E64	0x53E5	# <CJK>
+0x3E65	0x60E7	# <CJK>
+0x3E66	0x70AC	# <CJK>
+0x3E67	0x5267	# <CJK>
+0x3E68	0x6350	# <CJK>
+0x3E69	0x9E43	# <CJK>
+0x3E6A	0x5A1F	# <CJK>
+0x3E6B	0x5026	# <CJK>
+0x3E6C	0x7737	# <CJK>
+0x3E6D	0x5377	# <CJK>
+0x3E6E	0x7EE2	# <CJK>
+0x3E6F	0x6485	# <CJK>
+0x3E70	0x652B	# <CJK>
+0x3E71	0x6289	# <CJK>
+0x3E72	0x6398	# <CJK>
+0x3E73	0x5014	# <CJK>
+0x3E74	0x7235	# <CJK>
+0x3E75	0x89C9	# <CJK>
+0x3E76	0x51B3	# <CJK>
+0x3E77	0x8BC0	# <CJK>
+0x3E78	0x7EDD	# <CJK>
+0x3E79	0x5747	# <CJK>
+0x3E7A	0x83CC	# <CJK>
+0x3E7B	0x94A7	# <CJK>
+0x3E7C	0x519B	# <CJK>
+0x3E7D	0x541B	# <CJK>
+0x3E7E	0x5CFB	# <CJK>
+0x3F21	0x4FCA	# <CJK>
+0x3F22	0x7AE3	# <CJK>
+0x3F23	0x6D5A	# <CJK>
+0x3F24	0x90E1	# <CJK>
+0x3F25	0x9A8F	# <CJK>
+0x3F26	0x5580	# <CJK>
+0x3F27	0x5496	# <CJK>
+0x3F28	0x5361	# <CJK>
+0x3F29	0x54AF	# <CJK>
+0x3F2A	0x5F00	# <CJK>
+0x3F2B	0x63E9	# <CJK>
+0x3F2C	0x6977	# <CJK>
+0x3F2D	0x51EF	# <CJK>
+0x3F2E	0x6168	# <CJK>
+0x3F2F	0x520A	# <CJK>
+0x3F30	0x582A	# <CJK>
+0x3F31	0x52D8	# <CJK>
+0x3F32	0x574E	# <CJK>
+0x3F33	0x780D	# <CJK>
+0x3F34	0x770B	# <CJK>
+0x3F35	0x5EB7	# <CJK>
+0x3F36	0x6177	# <CJK>
+0x3F37	0x7CE0	# <CJK>
+0x3F38	0x625B	# <CJK>
+0x3F39	0x6297	# <CJK>
+0x3F3A	0x4EA2	# <CJK>
+0x3F3B	0x7095	# <CJK>
+0x3F3C	0x8003	# <CJK>
+0x3F3D	0x62F7	# <CJK>
+0x3F3E	0x70E4	# <CJK>
+0x3F3F	0x9760	# <CJK>
+0x3F40	0x5777	# <CJK>
+0x3F41	0x82DB	# <CJK>
+0x3F42	0x67EF	# <CJK>
+0x3F43	0x68F5	# <CJK>
+0x3F44	0x78D5	# <CJK>
+0x3F45	0x9897	# <CJK>
+0x3F46	0x79D1	# <CJK>
+0x3F47	0x58F3	# <CJK>
+0x3F48	0x54B3	# <CJK>
+0x3F49	0x53EF	# <CJK>
+0x3F4A	0x6E34	# <CJK>
+0x3F4B	0x514B	# <CJK>
+0x3F4C	0x523B	# <CJK>
+0x3F4D	0x5BA2	# <CJK>
+0x3F4E	0x8BFE	# <CJK>
+0x3F4F	0x80AF	# <CJK>
+0x3F50	0x5543	# <CJK>
+0x3F51	0x57A6	# <CJK>
+0x3F52	0x6073	# <CJK>
+0x3F53	0x5751	# <CJK>
+0x3F54	0x542D	# <CJK>
+0x3F55	0x7A7A	# <CJK>
+0x3F56	0x6050	# <CJK>
+0x3F57	0x5B54	# <CJK>
+0x3F58	0x63A7	# <CJK>
+0x3F59	0x62A0	# <CJK>
+0x3F5A	0x53E3	# <CJK>
+0x3F5B	0x6263	# <CJK>
+0x3F5C	0x5BC7	# <CJK>
+0x3F5D	0x67AF	# <CJK>
+0x3F5E	0x54ED	# <CJK>
+0x3F5F	0x7A9F	# <CJK>
+0x3F60	0x82E6	# <CJK>
+0x3F61	0x9177	# <CJK>
+0x3F62	0x5E93	# <CJK>
+0x3F63	0x88E4	# <CJK>
+0x3F64	0x5938	# <CJK>
+0x3F65	0x57AE	# <CJK>
+0x3F66	0x630E	# <CJK>
+0x3F67	0x8DE8	# <CJK>
+0x3F68	0x80EF	# <CJK>
+0x3F69	0x5757	# <CJK>
+0x3F6A	0x7B77	# <CJK>
+0x3F6B	0x4FA9	# <CJK>
+0x3F6C	0x5FEB	# <CJK>
+0x3F6D	0x5BBD	# <CJK>
+0x3F6E	0x6B3E	# <CJK>
+0x3F6F	0x5321	# <CJK>
+0x3F70	0x7B50	# <CJK>
+0x3F71	0x72C2	# <CJK>
+0x3F72	0x6846	# <CJK>
+0x3F73	0x77FF	# <CJK>
+0x3F74	0x7736	# <CJK>
+0x3F75	0x65F7	# <CJK>
+0x3F76	0x51B5	# <CJK>
+0x3F77	0x4E8F	# <CJK>
+0x3F78	0x76D4	# <CJK>
+0x3F79	0x5CBF	# <CJK>
+0x3F7A	0x7AA5	# <CJK>
+0x3F7B	0x8475	# <CJK>
+0x3F7C	0x594E	# <CJK>
+0x3F7D	0x9B41	# <CJK>
+0x3F7E	0x5080	# <CJK>
+0x4021	0x9988	# <CJK>
+0x4022	0x6127	# <CJK>
+0x4023	0x6E83	# <CJK>
+0x4024	0x5764	# <CJK>
+0x4025	0x6606	# <CJK>
+0x4026	0x6346	# <CJK>
+0x4027	0x56F0	# <CJK>
+0x4028	0x62EC	# <CJK>
+0x4029	0x6269	# <CJK>
+0x402A	0x5ED3	# <CJK>
+0x402B	0x9614	# <CJK>
+0x402C	0x5783	# <CJK>
+0x402D	0x62C9	# <CJK>
+0x402E	0x5587	# <CJK>
+0x402F	0x8721	# <CJK>
+0x4030	0x814A	# <CJK>
+0x4031	0x8FA3	# <CJK>
+0x4032	0x5566	# <CJK>
+0x4033	0x83B1	# <CJK>
+0x4034	0x6765	# <CJK>
+0x4035	0x8D56	# <CJK>
+0x4036	0x84DD	# <CJK>
+0x4037	0x5A6A	# <CJK>
+0x4038	0x680F	# <CJK>
+0x4039	0x62E6	# <CJK>
+0x403A	0x7BEE	# <CJK>
+0x403B	0x9611	# <CJK>
+0x403C	0x5170	# <CJK>
+0x403D	0x6F9C	# <CJK>
+0x403E	0x8C30	# <CJK>
+0x403F	0x63FD	# <CJK>
+0x4040	0x89C8	# <CJK>
+0x4041	0x61D2	# <CJK>
+0x4042	0x7F06	# <CJK>
+0x4043	0x70C2	# <CJK>
+0x4044	0x6EE5	# <CJK>
+0x4045	0x7405	# <CJK>
+0x4046	0x6994	# <CJK>
+0x4047	0x72FC	# <CJK>
+0x4048	0x5ECA	# <CJK>
+0x4049	0x90CE	# <CJK>
+0x404A	0x6717	# <CJK>
+0x404B	0x6D6A	# <CJK>
+0x404C	0x635E	# <CJK>
+0x404D	0x52B3	# <CJK>
+0x404E	0x7262	# <CJK>
+0x404F	0x8001	# <CJK>
+0x4050	0x4F6C	# <CJK>
+0x4051	0x59E5	# <CJK>
+0x4052	0x916A	# <CJK>
+0x4053	0x70D9	# <CJK>
+0x4054	0x6D9D	# <CJK>
+0x4055	0x52D2	# <CJK>
+0x4056	0x4E50	# <CJK>
+0x4057	0x96F7	# <CJK>
+0x4058	0x956D	# <CJK>
+0x4059	0x857E	# <CJK>
+0x405A	0x78CA	# <CJK>
+0x405B	0x7D2F	# <CJK>
+0x405C	0x5121	# <CJK>
+0x405D	0x5792	# <CJK>
+0x405E	0x64C2	# <CJK>
+0x405F	0x808B	# <CJK>
+0x4060	0x7C7B	# <CJK>
+0x4061	0x6CEA	# <CJK>
+0x4062	0x68F1	# <CJK>
+0x4063	0x695E	# <CJK>
+0x4064	0x51B7	# <CJK>
+0x4065	0x5398	# <CJK>
+0x4066	0x68A8	# <CJK>
+0x4067	0x7281	# <CJK>
+0x4068	0x9ECE	# <CJK>
+0x4069	0x7BF1	# <CJK>
+0x406A	0x72F8	# <CJK>
+0x406B	0x79BB	# <CJK>
+0x406C	0x6F13	# <CJK>
+0x406D	0x7406	# <CJK>
+0x406E	0x674E	# <CJK>
+0x406F	0x91CC	# <CJK>
+0x4070	0x9CA4	# <CJK>
+0x4071	0x793C	# <CJK>
+0x4072	0x8389	# <CJK>
+0x4073	0x8354	# <CJK>
+0x4074	0x540F	# <CJK>
+0x4075	0x6817	# <CJK>
+0x4076	0x4E3D	# <CJK>
+0x4077	0x5389	# <CJK>
+0x4078	0x52B1	# <CJK>
+0x4079	0x783E	# <CJK>
+0x407A	0x5386	# <CJK>
+0x407B	0x5229	# <CJK>
+0x407C	0x5088	# <CJK>
+0x407D	0x4F8B	# <CJK>
+0x407E	0x4FD0	# <CJK>
+0x4121	0x75E2	# <CJK>
+0x4122	0x7ACB	# <CJK>
+0x4123	0x7C92	# <CJK>
+0x4124	0x6CA5	# <CJK>
+0x4125	0x96B6	# <CJK>
+0x4126	0x529B	# <CJK>
+0x4127	0x7483	# <CJK>
+0x4128	0x54E9	# <CJK>
+0x4129	0x4FE9	# <CJK>
+0x412A	0x8054	# <CJK>
+0x412B	0x83B2	# <CJK>
+0x412C	0x8FDE	# <CJK>
+0x412D	0x9570	# <CJK>
+0x412E	0x5EC9	# <CJK>
+0x412F	0x601C	# <CJK>
+0x4130	0x6D9F	# <CJK>
+0x4131	0x5E18	# <CJK>
+0x4132	0x655B	# <CJK>
+0x4133	0x8138	# <CJK>
+0x4134	0x94FE	# <CJK>
+0x4135	0x604B	# <CJK>
+0x4136	0x70BC	# <CJK>
+0x4137	0x7EC3	# <CJK>
+0x4138	0x7CAE	# <CJK>
+0x4139	0x51C9	# <CJK>
+0x413A	0x6881	# <CJK>
+0x413B	0x7CB1	# <CJK>
+0x413C	0x826F	# <CJK>
+0x413D	0x4E24	# <CJK>
+0x413E	0x8F86	# <CJK>
+0x413F	0x91CF	# <CJK>
+0x4140	0x667E	# <CJK>
+0x4141	0x4EAE	# <CJK>
+0x4142	0x8C05	# <CJK>
+0x4143	0x64A9	# <CJK>
+0x4144	0x804A	# <CJK>
+0x4145	0x50DA	# <CJK>
+0x4146	0x7597	# <CJK>
+0x4147	0x71CE	# <CJK>
+0x4148	0x5BE5	# <CJK>
+0x4149	0x8FBD	# <CJK>
+0x414A	0x6F66	# <CJK>
+0x414B	0x4E86	# <CJK>
+0x414C	0x6482	# <CJK>
+0x414D	0x9563	# <CJK>
+0x414E	0x5ED6	# <CJK>
+0x414F	0x6599	# <CJK>
+0x4150	0x5217	# <CJK>
+0x4151	0x88C2	# <CJK>
+0x4152	0x70C8	# <CJK>
+0x4153	0x52A3	# <CJK>
+0x4154	0x730E	# <CJK>
+0x4155	0x7433	# <CJK>
+0x4156	0x6797	# <CJK>
+0x4157	0x78F7	# <CJK>
+0x4158	0x9716	# <CJK>
+0x4159	0x4E34	# <CJK>
+0x415A	0x90BB	# <CJK>
+0x415B	0x9CDE	# <CJK>
+0x415C	0x6DCB	# <CJK>
+0x415D	0x51DB	# <CJK>
+0x415E	0x8D41	# <CJK>
+0x415F	0x541D	# <CJK>
+0x4160	0x62CE	# <CJK>
+0x4161	0x73B2	# <CJK>
+0x4162	0x83F1	# <CJK>
+0x4163	0x96F6	# <CJK>
+0x4164	0x9F84	# <CJK>
+0x4165	0x94C3	# <CJK>
+0x4166	0x4F36	# <CJK>
+0x4167	0x7F9A	# <CJK>
+0x4168	0x51CC	# <CJK>
+0x4169	0x7075	# <CJK>
+0x416A	0x9675	# <CJK>
+0x416B	0x5CAD	# <CJK>
+0x416C	0x9886	# <CJK>
+0x416D	0x53E6	# <CJK>
+0x416E	0x4EE4	# <CJK>
+0x416F	0x6E9C	# <CJK>
+0x4170	0x7409	# <CJK>
+0x4171	0x69B4	# <CJK>
+0x4172	0x786B	# <CJK>
+0x4173	0x998F	# <CJK>
+0x4174	0x7559	# <CJK>
+0x4175	0x5218	# <CJK>
+0x4176	0x7624	# <CJK>
+0x4177	0x6D41	# <CJK>
+0x4178	0x67F3	# <CJK>
+0x4179	0x516D	# <CJK>
+0x417A	0x9F99	# <CJK>
+0x417B	0x804B	# <CJK>
+0x417C	0x5499	# <CJK>
+0x417D	0x7B3C	# <CJK>
+0x417E	0x7ABF	# <CJK>
+0x4221	0x9686	# <CJK>
+0x4222	0x5784	# <CJK>
+0x4223	0x62E2	# <CJK>
+0x4224	0x9647	# <CJK>
+0x4225	0x697C	# <CJK>
+0x4226	0x5A04	# <CJK>
+0x4227	0x6402	# <CJK>
+0x4228	0x7BD3	# <CJK>
+0x4229	0x6F0F	# <CJK>
+0x422A	0x964B	# <CJK>
+0x422B	0x82A6	# <CJK>
+0x422C	0x5362	# <CJK>
+0x422D	0x9885	# <CJK>
+0x422E	0x5E90	# <CJK>
+0x422F	0x7089	# <CJK>
+0x4230	0x63B3	# <CJK>
+0x4231	0x5364	# <CJK>
+0x4232	0x864F	# <CJK>
+0x4233	0x9C81	# <CJK>
+0x4234	0x9E93	# <CJK>
+0x4235	0x788C	# <CJK>
+0x4236	0x9732	# <CJK>
+0x4237	0x8DEF	# <CJK>
+0x4238	0x8D42	# <CJK>
+0x4239	0x9E7F	# <CJK>
+0x423A	0x6F5E	# <CJK>
+0x423B	0x7984	# <CJK>
+0x423C	0x5F55	# <CJK>
+0x423D	0x9646	# <CJK>
+0x423E	0x622E	# <CJK>
+0x423F	0x9A74	# <CJK>
+0x4240	0x5415	# <CJK>
+0x4241	0x94DD	# <CJK>
+0x4242	0x4FA3	# <CJK>
+0x4243	0x65C5	# <CJK>
+0x4244	0x5C65	# <CJK>
+0x4245	0x5C61	# <CJK>
+0x4246	0x7F15	# <CJK>
+0x4247	0x8651	# <CJK>
+0x4248	0x6C2F	# <CJK>
+0x4249	0x5F8B	# <CJK>
+0x424A	0x7387	# <CJK>
+0x424B	0x6EE4	# <CJK>
+0x424C	0x7EFF	# <CJK>
+0x424D	0x5CE6	# <CJK>
+0x424E	0x631B	# <CJK>
+0x424F	0x5B6A	# <CJK>
+0x4250	0x6EE6	# <CJK>
+0x4251	0x5375	# <CJK>
+0x4252	0x4E71	# <CJK>
+0x4253	0x63A0	# <CJK>
+0x4254	0x7565	# <CJK>
+0x4255	0x62A1	# <CJK>
+0x4256	0x8F6E	# <CJK>
+0x4257	0x4F26	# <CJK>
+0x4258	0x4ED1	# <CJK>
+0x4259	0x6CA6	# <CJK>
+0x425A	0x7EB6	# <CJK>
+0x425B	0x8BBA	# <CJK>
+0x425C	0x841D	# <CJK>
+0x425D	0x87BA	# <CJK>
+0x425E	0x7F57	# <CJK>
+0x425F	0x903B	# <CJK>
+0x4260	0x9523	# <CJK>
+0x4261	0x7BA9	# <CJK>
+0x4262	0x9AA1	# <CJK>
+0x4263	0x88F8	# <CJK>
+0x4264	0x843D	# <CJK>
+0x4265	0x6D1B	# <CJK>
+0x4266	0x9A86	# <CJK>
+0x4267	0x7EDC	# <CJK>
+0x4268	0x5988	# <CJK>
+0x4269	0x9EBB	# <CJK>
+0x426A	0x739B	# <CJK>
+0x426B	0x7801	# <CJK>
+0x426C	0x8682	# <CJK>
+0x426D	0x9A6C	# <CJK>
+0x426E	0x9A82	# <CJK>
+0x426F	0x561B	# <CJK>
+0x4270	0x5417	# <CJK>
+0x4271	0x57CB	# <CJK>
+0x4272	0x4E70	# <CJK>
+0x4273	0x9EA6	# <CJK>
+0x4274	0x5356	# <CJK>
+0x4275	0x8FC8	# <CJK>
+0x4276	0x8109	# <CJK>
+0x4277	0x7792	# <CJK>
+0x4278	0x9992	# <CJK>
+0x4279	0x86EE	# <CJK>
+0x427A	0x6EE1	# <CJK>
+0x427B	0x8513	# <CJK>
+0x427C	0x66FC	# <CJK>
+0x427D	0x6162	# <CJK>
+0x427E	0x6F2B	# <CJK>
+0x4321	0x8C29	# <CJK>
+0x4322	0x8292	# <CJK>
+0x4323	0x832B	# <CJK>
+0x4324	0x76F2	# <CJK>
+0x4325	0x6C13	# <CJK>
+0x4326	0x5FD9	# <CJK>
+0x4327	0x83BD	# <CJK>
+0x4328	0x732B	# <CJK>
+0x4329	0x8305	# <CJK>
+0x432A	0x951A	# <CJK>
+0x432B	0x6BDB	# <CJK>
+0x432C	0x77DB	# <CJK>
+0x432D	0x94C6	# <CJK>
+0x432E	0x536F	# <CJK>
+0x432F	0x8302	# <CJK>
+0x4330	0x5192	# <CJK>
+0x4331	0x5E3D	# <CJK>
+0x4332	0x8C8C	# <CJK>
+0x4333	0x8D38	# <CJK>
+0x4334	0x4E48	# <CJK>
+0x4335	0x73AB	# <CJK>
+0x4336	0x679A	# <CJK>
+0x4337	0x6885	# <CJK>
+0x4338	0x9176	# <CJK>
+0x4339	0x9709	# <CJK>
+0x433A	0x7164	# <CJK>
+0x433B	0x6CA1	# <CJK>
+0x433C	0x7709	# <CJK>
+0x433D	0x5A92	# <CJK>
+0x433E	0x9541	# <CJK>
+0x433F	0x6BCF	# <CJK>
+0x4340	0x7F8E	# <CJK>
+0x4341	0x6627	# <CJK>
+0x4342	0x5BD0	# <CJK>
+0x4343	0x59B9	# <CJK>
+0x4344	0x5A9A	# <CJK>
+0x4345	0x95E8	# <CJK>
+0x4346	0x95F7	# <CJK>
+0x4347	0x4EEC	# <CJK>
+0x4348	0x840C	# <CJK>
+0x4349	0x8499	# <CJK>
+0x434A	0x6AAC	# <CJK>
+0x434B	0x76DF	# <CJK>
+0x434C	0x9530	# <CJK>
+0x434D	0x731B	# <CJK>
+0x434E	0x68A6	# <CJK>
+0x434F	0x5B5F	# <CJK>
+0x4350	0x772F	# <CJK>
+0x4351	0x919A	# <CJK>
+0x4352	0x9761	# <CJK>
+0x4353	0x7CDC	# <CJK>
+0x4354	0x8FF7	# <CJK>
+0x4355	0x8C1C	# <CJK>
+0x4356	0x5F25	# <CJK>
+0x4357	0x7C73	# <CJK>
+0x4358	0x79D8	# <CJK>
+0x4359	0x89C5	# <CJK>
+0x435A	0x6CCC	# <CJK>
+0x435B	0x871C	# <CJK>
+0x435C	0x5BC6	# <CJK>
+0x435D	0x5E42	# <CJK>
+0x435E	0x68C9	# <CJK>
+0x435F	0x7720	# <CJK>
+0x4360	0x7EF5	# <CJK>
+0x4361	0x5195	# <CJK>
+0x4362	0x514D	# <CJK>
+0x4363	0x52C9	# <CJK>
+0x4364	0x5A29	# <CJK>
+0x4365	0x7F05	# <CJK>
+0x4366	0x9762	# <CJK>
+0x4367	0x82D7	# <CJK>
+0x4368	0x63CF	# <CJK>
+0x4369	0x7784	# <CJK>
+0x436A	0x85D0	# <CJK>
+0x436B	0x79D2	# <CJK>
+0x436C	0x6E3A	# <CJK>
+0x436D	0x5E99	# <CJK>
+0x436E	0x5999	# <CJK>
+0x436F	0x8511	# <CJK>
+0x4370	0x706D	# <CJK>
+0x4371	0x6C11	# <CJK>
+0x4372	0x62BF	# <CJK>
+0x4373	0x76BF	# <CJK>
+0x4374	0x654F	# <CJK>
+0x4375	0x60AF	# <CJK>
+0x4376	0x95FD	# <CJK>
+0x4377	0x660E	# <CJK>
+0x4378	0x879F	# <CJK>
+0x4379	0x9E23	# <CJK>
+0x437A	0x94ED	# <CJK>
+0x437B	0x540D	# <CJK>
+0x437C	0x547D	# <CJK>
+0x437D	0x8C2C	# <CJK>
+0x437E	0x6478	# <CJK>
+0x4421	0x6479	# <CJK>
+0x4422	0x8611	# <CJK>
+0x4423	0x6A21	# <CJK>
+0x4424	0x819C	# <CJK>
+0x4425	0x78E8	# <CJK>
+0x4426	0x6469	# <CJK>
+0x4427	0x9B54	# <CJK>
+0x4428	0x62B9	# <CJK>
+0x4429	0x672B	# <CJK>
+0x442A	0x83AB	# <CJK>
+0x442B	0x58A8	# <CJK>
+0x442C	0x9ED8	# <CJK>
+0x442D	0x6CAB	# <CJK>
+0x442E	0x6F20	# <CJK>
+0x442F	0x5BDE	# <CJK>
+0x4430	0x964C	# <CJK>
+0x4431	0x8C0B	# <CJK>
+0x4432	0x725F	# <CJK>
+0x4433	0x67D0	# <CJK>
+0x4434	0x62C7	# <CJK>
+0x4435	0x7261	# <CJK>
+0x4436	0x4EA9	# <CJK>
+0x4437	0x59C6	# <CJK>
+0x4438	0x6BCD	# <CJK>
+0x4439	0x5893	# <CJK>
+0x443A	0x66AE	# <CJK>
+0x443B	0x5E55	# <CJK>
+0x443C	0x52DF	# <CJK>
+0x443D	0x6155	# <CJK>
+0x443E	0x6728	# <CJK>
+0x443F	0x76EE	# <CJK>
+0x4440	0x7766	# <CJK>
+0x4441	0x7267	# <CJK>
+0x4442	0x7A46	# <CJK>
+0x4443	0x62FF	# <CJK>
+0x4444	0x54EA	# <CJK>
+0x4445	0x5450	# <CJK>
+0x4446	0x94A0	# <CJK>
+0x4447	0x90A3	# <CJK>
+0x4448	0x5A1C	# <CJK>
+0x4449	0x7EB3	# <CJK>
+0x444A	0x6C16	# <CJK>
+0x444B	0x4E43	# <CJK>
+0x444C	0x5976	# <CJK>
+0x444D	0x8010	# <CJK>
+0x444E	0x5948	# <CJK>
+0x444F	0x5357	# <CJK>
+0x4450	0x7537	# <CJK>
+0x4451	0x96BE	# <CJK>
+0x4452	0x56CA	# <CJK>
+0x4453	0x6320	# <CJK>
+0x4454	0x8111	# <CJK>
+0x4455	0x607C	# <CJK>
+0x4456	0x95F9	# <CJK>
+0x4457	0x6DD6	# <CJK>
+0x4458	0x5462	# <CJK>
+0x4459	0x9981	# <CJK>
+0x445A	0x5185	# <CJK>
+0x445B	0x5AE9	# <CJK>
+0x445C	0x80FD	# <CJK>
+0x445D	0x59AE	# <CJK>
+0x445E	0x9713	# <CJK>
+0x445F	0x502A	# <CJK>
+0x4460	0x6CE5	# <CJK>
+0x4461	0x5C3C	# <CJK>
+0x4462	0x62DF	# <CJK>
+0x4463	0x4F60	# <CJK>
+0x4464	0x533F	# <CJK>
+0x4465	0x817B	# <CJK>
+0x4466	0x9006	# <CJK>
+0x4467	0x6EBA	# <CJK>
+0x4468	0x852B	# <CJK>
+0x4469	0x62C8	# <CJK>
+0x446A	0x5E74	# <CJK>
+0x446B	0x78BE	# <CJK>
+0x446C	0x64B5	# <CJK>
+0x446D	0x637B	# <CJK>
+0x446E	0x5FF5	# <CJK>
+0x446F	0x5A18	# <CJK>
+0x4470	0x917F	# <CJK>
+0x4471	0x9E1F	# <CJK>
+0x4472	0x5C3F	# <CJK>
+0x4473	0x634F	# <CJK>
+0x4474	0x8042	# <CJK>
+0x4475	0x5B7D	# <CJK>
+0x4476	0x556E	# <CJK>
+0x4477	0x954A	# <CJK>
+0x4478	0x954D	# <CJK>
+0x4479	0x6D85	# <CJK>
+0x447A	0x60A8	# <CJK>
+0x447B	0x67E0	# <CJK>
+0x447C	0x72DE	# <CJK>
+0x447D	0x51DD	# <CJK>
+0x447E	0x5B81	# <CJK>
+0x4521	0x62E7	# <CJK>
+0x4522	0x6CDE	# <CJK>
+0x4523	0x725B	# <CJK>
+0x4524	0x626D	# <CJK>
+0x4525	0x94AE	# <CJK>
+0x4526	0x7EBD	# <CJK>
+0x4527	0x8113	# <CJK>
+0x4528	0x6D53	# <CJK>
+0x4529	0x519C	# <CJK>
+0x452A	0x5F04	# <CJK>
+0x452B	0x5974	# <CJK>
+0x452C	0x52AA	# <CJK>
+0x452D	0x6012	# <CJK>
+0x452E	0x5973	# <CJK>
+0x452F	0x6696	# <CJK>
+0x4530	0x8650	# <CJK>
+0x4531	0x759F	# <CJK>
+0x4532	0x632A	# <CJK>
+0x4533	0x61E6	# <CJK>
+0x4534	0x7CEF	# <CJK>
+0x4535	0x8BFA	# <CJK>
+0x4536	0x54E6	# <CJK>
+0x4537	0x6B27	# <CJK>
+0x4538	0x9E25	# <CJK>
+0x4539	0x6BB4	# <CJK>
+0x453A	0x85D5	# <CJK>
+0x453B	0x5455	# <CJK>
+0x453C	0x5076	# <CJK>
+0x453D	0x6CA4	# <CJK>
+0x453E	0x556A	# <CJK>
+0x453F	0x8DB4	# <CJK>
+0x4540	0x722C	# <CJK>
+0x4541	0x5E15	# <CJK>
+0x4542	0x6015	# <CJK>
+0x4543	0x7436	# <CJK>
+0x4544	0x62CD	# <CJK>
+0x4545	0x6392	# <CJK>
+0x4546	0x724C	# <CJK>
+0x4547	0x5F98	# <CJK>
+0x4548	0x6E43	# <CJK>
+0x4549	0x6D3E	# <CJK>
+0x454A	0x6500	# <CJK>
+0x454B	0x6F58	# <CJK>
+0x454C	0x76D8	# <CJK>
+0x454D	0x78D0	# <CJK>
+0x454E	0x76FC	# <CJK>
+0x454F	0x7554	# <CJK>
+0x4550	0x5224	# <CJK>
+0x4551	0x53DB	# <CJK>
+0x4552	0x4E53	# <CJK>
+0x4553	0x5E9E	# <CJK>
+0x4554	0x65C1	# <CJK>
+0x4555	0x802A	# <CJK>
+0x4556	0x80D6	# <CJK>
+0x4557	0x629B	# <CJK>
+0x4558	0x5486	# <CJK>
+0x4559	0x5228	# <CJK>
+0x455A	0x70AE	# <CJK>
+0x455B	0x888D	# <CJK>
+0x455C	0x8DD1	# <CJK>
+0x455D	0x6CE1	# <CJK>
+0x455E	0x5478	# <CJK>
+0x455F	0x80DA	# <CJK>
+0x4560	0x57F9	# <CJK>
+0x4561	0x88F4	# <CJK>
+0x4562	0x8D54	# <CJK>
+0x4563	0x966A	# <CJK>
+0x4564	0x914D	# <CJK>
+0x4565	0x4F69	# <CJK>
+0x4566	0x6C9B	# <CJK>
+0x4567	0x55B7	# <CJK>
+0x4568	0x76C6	# <CJK>
+0x4569	0x7830	# <CJK>
+0x456A	0x62A8	# <CJK>
+0x456B	0x70F9	# <CJK>
+0x456C	0x6F8E	# <CJK>
+0x456D	0x5F6D	# <CJK>
+0x456E	0x84EC	# <CJK>
+0x456F	0x68DA	# <CJK>
+0x4570	0x787C	# <CJK>
+0x4571	0x7BF7	# <CJK>
+0x4572	0x81A8	# <CJK>
+0x4573	0x670B	# <CJK>
+0x4574	0x9E4F	# <CJK>
+0x4575	0x6367	# <CJK>
+0x4576	0x78B0	# <CJK>
+0x4577	0x576F	# <CJK>
+0x4578	0x7812	# <CJK>
+0x4579	0x9739	# <CJK>
+0x457A	0x6279	# <CJK>
+0x457B	0x62AB	# <CJK>
+0x457C	0x5288	# <CJK>
+0x457D	0x7435	# <CJK>
+0x457E	0x6BD7	# <CJK>
+0x4621	0x5564	# <CJK>
+0x4622	0x813E	# <CJK>
+0x4623	0x75B2	# <CJK>
+0x4624	0x76AE	# <CJK>
+0x4625	0x5339	# <CJK>
+0x4626	0x75DE	# <CJK>
+0x4627	0x50FB	# <CJK>
+0x4628	0x5C41	# <CJK>
+0x4629	0x8B6C	# <CJK>
+0x462A	0x7BC7	# <CJK>
+0x462B	0x504F	# <CJK>
+0x462C	0x7247	# <CJK>
+0x462D	0x9A97	# <CJK>
+0x462E	0x98D8	# <CJK>
+0x462F	0x6F02	# <CJK>
+0x4630	0x74E2	# <CJK>
+0x4631	0x7968	# <CJK>
+0x4632	0x6487	# <CJK>
+0x4633	0x77A5	# <CJK>
+0x4634	0x62FC	# <CJK>
+0x4635	0x9891	# <CJK>
+0x4636	0x8D2B	# <CJK>
+0x4637	0x54C1	# <CJK>
+0x4638	0x8058	# <CJK>
+0x4639	0x4E52	# <CJK>
+0x463A	0x576A	# <CJK>
+0x463B	0x82F9	# <CJK>
+0x463C	0x840D	# <CJK>
+0x463D	0x5E73	# <CJK>
+0x463E	0x51ED	# <CJK>
+0x463F	0x74F6	# <CJK>
+0x4640	0x8BC4	# <CJK>
+0x4641	0x5C4F	# <CJK>
+0x4642	0x5761	# <CJK>
+0x4643	0x6CFC	# <CJK>
+0x4644	0x9887	# <CJK>
+0x4645	0x5A46	# <CJK>
+0x4646	0x7834	# <CJK>
+0x4647	0x9B44	# <CJK>
+0x4648	0x8FEB	# <CJK>
+0x4649	0x7C95	# <CJK>
+0x464A	0x5256	# <CJK>
+0x464B	0x6251	# <CJK>
+0x464C	0x94FA	# <CJK>
+0x464D	0x4EC6	# <CJK>
+0x464E	0x8386	# <CJK>
+0x464F	0x8461	# <CJK>
+0x4650	0x83E9	# <CJK>
+0x4651	0x84B2	# <CJK>
+0x4652	0x57D4	# <CJK>
+0x4653	0x6734	# <CJK>
+0x4654	0x5703	# <CJK>
+0x4655	0x666E	# <CJK>
+0x4656	0x6D66	# <CJK>
+0x4657	0x8C31	# <CJK>
+0x4658	0x66DD	# <CJK>
+0x4659	0x7011	# <CJK>
+0x465A	0x671F	# <CJK>
+0x465B	0x6B3A	# <CJK>
+0x465C	0x6816	# <CJK>
+0x465D	0x621A	# <CJK>
+0x465E	0x59BB	# <CJK>
+0x465F	0x4E03	# <CJK>
+0x4660	0x51C4	# <CJK>
+0x4661	0x6F06	# <CJK>
+0x4662	0x67D2	# <CJK>
+0x4663	0x6C8F	# <CJK>
+0x4664	0x5176	# <CJK>
+0x4665	0x68CB	# <CJK>
+0x4666	0x5947	# <CJK>
+0x4667	0x6B67	# <CJK>
+0x4668	0x7566	# <CJK>
+0x4669	0x5D0E	# <CJK>
+0x466A	0x8110	# <CJK>
+0x466B	0x9F50	# <CJK>
+0x466C	0x65D7	# <CJK>
+0x466D	0x7948	# <CJK>
+0x466E	0x7941	# <CJK>
+0x466F	0x9A91	# <CJK>
+0x4670	0x8D77	# <CJK>
+0x4671	0x5C82	# <CJK>
+0x4672	0x4E5E	# <CJK>
+0x4673	0x4F01	# <CJK>
+0x4674	0x542F	# <CJK>
+0x4675	0x5951	# <CJK>
+0x4676	0x780C	# <CJK>
+0x4677	0x5668	# <CJK>
+0x4678	0x6C14	# <CJK>
+0x4679	0x8FC4	# <CJK>
+0x467A	0x5F03	# <CJK>
+0x467B	0x6C7D	# <CJK>
+0x467C	0x6CE3	# <CJK>
+0x467D	0x8BAB	# <CJK>
+0x467E	0x6390	# <CJK>
+0x4721	0x6070	# <CJK>
+0x4722	0x6D3D	# <CJK>
+0x4723	0x7275	# <CJK>
+0x4724	0x6266	# <CJK>
+0x4725	0x948E	# <CJK>
+0x4726	0x94C5	# <CJK>
+0x4727	0x5343	# <CJK>
+0x4728	0x8FC1	# <CJK>
+0x4729	0x7B7E	# <CJK>
+0x472A	0x4EDF	# <CJK>
+0x472B	0x8C26	# <CJK>
+0x472C	0x4E7E	# <CJK>
+0x472D	0x9ED4	# <CJK>
+0x472E	0x94B1	# <CJK>
+0x472F	0x94B3	# <CJK>
+0x4730	0x524D	# <CJK>
+0x4731	0x6F5C	# <CJK>
+0x4732	0x9063	# <CJK>
+0x4733	0x6D45	# <CJK>
+0x4734	0x8C34	# <CJK>
+0x4735	0x5811	# <CJK>
+0x4736	0x5D4C	# <CJK>
+0x4737	0x6B20	# <CJK>
+0x4738	0x6B49	# <CJK>
+0x4739	0x67AA	# <CJK>
+0x473A	0x545B	# <CJK>
+0x473B	0x8154	# <CJK>
+0x473C	0x7F8C	# <CJK>
+0x473D	0x5899	# <CJK>
+0x473E	0x8537	# <CJK>
+0x473F	0x5F3A	# <CJK>
+0x4740	0x62A2	# <CJK>
+0x4741	0x6A47	# <CJK>
+0x4742	0x9539	# <CJK>
+0x4743	0x6572	# <CJK>
+0x4744	0x6084	# <CJK>
+0x4745	0x6865	# <CJK>
+0x4746	0x77A7	# <CJK>
+0x4747	0x4E54	# <CJK>
+0x4748	0x4FA8	# <CJK>
+0x4749	0x5DE7	# <CJK>
+0x474A	0x9798	# <CJK>
+0x474B	0x64AC	# <CJK>
+0x474C	0x7FD8	# <CJK>
+0x474D	0x5CED	# <CJK>
+0x474E	0x4FCF	# <CJK>
+0x474F	0x7A8D	# <CJK>
+0x4750	0x5207	# <CJK>
+0x4751	0x8304	# <CJK>
+0x4752	0x4E14	# <CJK>
+0x4753	0x602F	# <CJK>
+0x4754	0x7A83	# <CJK>
+0x4755	0x94A6	# <CJK>
+0x4756	0x4FB5	# <CJK>
+0x4757	0x4EB2	# <CJK>
+0x4758	0x79E6	# <CJK>
+0x4759	0x7434	# <CJK>
+0x475A	0x52E4	# <CJK>
+0x475B	0x82B9	# <CJK>
+0x475C	0x64D2	# <CJK>
+0x475D	0x79BD	# <CJK>
+0x475E	0x5BDD	# <CJK>
+0x475F	0x6C81	# <CJK>
+0x4760	0x9752	# <CJK>
+0x4761	0x8F7B	# <CJK>
+0x4762	0x6C22	# <CJK>
+0x4763	0x503E	# <CJK>
+0x4764	0x537F	# <CJK>
+0x4765	0x6E05	# <CJK>
+0x4766	0x64CE	# <CJK>
+0x4767	0x6674	# <CJK>
+0x4768	0x6C30	# <CJK>
+0x4769	0x60C5	# <CJK>
+0x476A	0x9877	# <CJK>
+0x476B	0x8BF7	# <CJK>
+0x476C	0x5E86	# <CJK>
+0x476D	0x743C	# <CJK>
+0x476E	0x7A77	# <CJK>
+0x476F	0x79CB	# <CJK>
+0x4770	0x4E18	# <CJK>
+0x4771	0x90B1	# <CJK>
+0x4772	0x7403	# <CJK>
+0x4773	0x6C42	# <CJK>
+0x4774	0x56DA	# <CJK>
+0x4775	0x914B	# <CJK>
+0x4776	0x6CC5	# <CJK>
+0x4777	0x8D8B	# <CJK>
+0x4778	0x533A	# <CJK>
+0x4779	0x86C6	# <CJK>
+0x477A	0x66F2	# <CJK>
+0x477B	0x8EAF	# <CJK>
+0x477C	0x5C48	# <CJK>
+0x477D	0x9A71	# <CJK>
+0x477E	0x6E20	# <CJK>
+0x4821	0x53D6	# <CJK>
+0x4822	0x5A36	# <CJK>
+0x4823	0x9F8B	# <CJK>
+0x4824	0x8DA3	# <CJK>
+0x4825	0x53BB	# <CJK>
+0x4826	0x5708	# <CJK>
+0x4827	0x98A7	# <CJK>
+0x4828	0x6743	# <CJK>
+0x4829	0x919B	# <CJK>
+0x482A	0x6CC9	# <CJK>
+0x482B	0x5168	# <CJK>
+0x482C	0x75CA	# <CJK>
+0x482D	0x62F3	# <CJK>
+0x482E	0x72AC	# <CJK>
+0x482F	0x5238	# <CJK>
+0x4830	0x529D	# <CJK>
+0x4831	0x7F3A	# <CJK>
+0x4832	0x7094	# <CJK>
+0x4833	0x7638	# <CJK>
+0x4834	0x5374	# <CJK>
+0x4835	0x9E4A	# <CJK>
+0x4836	0x69B7	# <CJK>
+0x4837	0x786E	# <CJK>
+0x4838	0x96C0	# <CJK>
+0x4839	0x88D9	# <CJK>
+0x483A	0x7FA4	# <CJK>
+0x483B	0x7136	# <CJK>
+0x483C	0x71C3	# <CJK>
+0x483D	0x5189	# <CJK>
+0x483E	0x67D3	# <CJK>
+0x483F	0x74E4	# <CJK>
+0x4840	0x58E4	# <CJK>
+0x4841	0x6518	# <CJK>
+0x4842	0x56B7	# <CJK>
+0x4843	0x8BA9	# <CJK>
+0x4844	0x9976	# <CJK>
+0x4845	0x6270	# <CJK>
+0x4846	0x7ED5	# <CJK>
+0x4847	0x60F9	# <CJK>
+0x4848	0x70ED	# <CJK>
+0x4849	0x58EC	# <CJK>
+0x484A	0x4EC1	# <CJK>
+0x484B	0x4EBA	# <CJK>
+0x484C	0x5FCD	# <CJK>
+0x484D	0x97E7	# <CJK>
+0x484E	0x4EFB	# <CJK>
+0x484F	0x8BA4	# <CJK>
+0x4850	0x5203	# <CJK>
+0x4851	0x598A	# <CJK>
+0x4852	0x7EAB	# <CJK>
+0x4853	0x6254	# <CJK>
+0x4854	0x4ECD	# <CJK>
+0x4855	0x65E5	# <CJK>
+0x4856	0x620E	# <CJK>
+0x4857	0x8338	# <CJK>
+0x4858	0x84C9	# <CJK>
+0x4859	0x8363	# <CJK>
+0x485A	0x878D	# <CJK>
+0x485B	0x7194	# <CJK>
+0x485C	0x6EB6	# <CJK>
+0x485D	0x5BB9	# <CJK>
+0x485E	0x7ED2	# <CJK>
+0x485F	0x5197	# <CJK>
+0x4860	0x63C9	# <CJK>
+0x4861	0x67D4	# <CJK>
+0x4862	0x8089	# <CJK>
+0x4863	0x8339	# <CJK>
+0x4864	0x8815	# <CJK>
+0x4865	0x5112	# <CJK>
+0x4866	0x5B7A	# <CJK>
+0x4867	0x5982	# <CJK>
+0x4868	0x8FB1	# <CJK>
+0x4869	0x4E73	# <CJK>
+0x486A	0x6C5D	# <CJK>
+0x486B	0x5165	# <CJK>
+0x486C	0x8925	# <CJK>
+0x486D	0x8F6F	# <CJK>
+0x486E	0x962E	# <CJK>
+0x486F	0x854A	# <CJK>
+0x4870	0x745E	# <CJK>
+0x4871	0x9510	# <CJK>
+0x4872	0x95F0	# <CJK>
+0x4873	0x6DA6	# <CJK>
+0x4874	0x82E5	# <CJK>
+0x4875	0x5F31	# <CJK>
+0x4876	0x6492	# <CJK>
+0x4877	0x6D12	# <CJK>
+0x4878	0x8428	# <CJK>
+0x4879	0x816E	# <CJK>
+0x487A	0x9CC3	# <CJK>
+0x487B	0x585E	# <CJK>
+0x487C	0x8D5B	# <CJK>
+0x487D	0x4E09	# <CJK>
+0x487E	0x53C1	# <CJK>
+0x4921	0x4F1E	# <CJK>
+0x4922	0x6563	# <CJK>
+0x4923	0x6851	# <CJK>
+0x4924	0x55D3	# <CJK>
+0x4925	0x4E27	# <CJK>
+0x4926	0x6414	# <CJK>
+0x4927	0x9A9A	# <CJK>
+0x4928	0x626B	# <CJK>
+0x4929	0x5AC2	# <CJK>
+0x492A	0x745F	# <CJK>
+0x492B	0x8272	# <CJK>
+0x492C	0x6DA9	# <CJK>
+0x492D	0x68EE	# <CJK>
+0x492E	0x50E7	# <CJK>
+0x492F	0x838E	# <CJK>
+0x4930	0x7802	# <CJK>
+0x4931	0x6740	# <CJK>
+0x4932	0x5239	# <CJK>
+0x4933	0x6C99	# <CJK>
+0x4934	0x7EB1	# <CJK>
+0x4935	0x50BB	# <CJK>
+0x4936	0x5565	# <CJK>
+0x4937	0x715E	# <CJK>
+0x4938	0x7B5B	# <CJK>
+0x4939	0x6652	# <CJK>
+0x493A	0x73CA	# <CJK>
+0x493B	0x82EB	# <CJK>
+0x493C	0x6749	# <CJK>
+0x493D	0x5C71	# <CJK>
+0x493E	0x5220	# <CJK>
+0x493F	0x717D	# <CJK>
+0x4940	0x886B	# <CJK>
+0x4941	0x95EA	# <CJK>
+0x4942	0x9655	# <CJK>
+0x4943	0x64C5	# <CJK>
+0x4944	0x8D61	# <CJK>
+0x4945	0x81B3	# <CJK>
+0x4946	0x5584	# <CJK>
+0x4947	0x6C55	# <CJK>
+0x4948	0x6247	# <CJK>
+0x4949	0x7F2E	# <CJK>
+0x494A	0x5892	# <CJK>
+0x494B	0x4F24	# <CJK>
+0x494C	0x5546	# <CJK>
+0x494D	0x8D4F	# <CJK>
+0x494E	0x664C	# <CJK>
+0x494F	0x4E0A	# <CJK>
+0x4950	0x5C1A	# <CJK>
+0x4951	0x88F3	# <CJK>
+0x4952	0x68A2	# <CJK>
+0x4953	0x634E	# <CJK>
+0x4954	0x7A0D	# <CJK>
+0x4955	0x70E7	# <CJK>
+0x4956	0x828D	# <CJK>
+0x4957	0x52FA	# <CJK>
+0x4958	0x97F6	# <CJK>
+0x4959	0x5C11	# <CJK>
+0x495A	0x54E8	# <CJK>
+0x495B	0x90B5	# <CJK>
+0x495C	0x7ECD	# <CJK>
+0x495D	0x5962	# <CJK>
+0x495E	0x8D4A	# <CJK>
+0x495F	0x86C7	# <CJK>
+0x4960	0x820C	# <CJK>
+0x4961	0x820D	# <CJK>
+0x4962	0x8D66	# <CJK>
+0x4963	0x6444	# <CJK>
+0x4964	0x5C04	# <CJK>
+0x4965	0x6151	# <CJK>
+0x4966	0x6D89	# <CJK>
+0x4967	0x793E	# <CJK>
+0x4968	0x8BBE	# <CJK>
+0x4969	0x7837	# <CJK>
+0x496A	0x7533	# <CJK>
+0x496B	0x547B	# <CJK>
+0x496C	0x4F38	# <CJK>
+0x496D	0x8EAB	# <CJK>
+0x496E	0x6DF1	# <CJK>
+0x496F	0x5A20	# <CJK>
+0x4970	0x7EC5	# <CJK>
+0x4971	0x795E	# <CJK>
+0x4972	0x6C88	# <CJK>
+0x4973	0x5BA1	# <CJK>
+0x4974	0x5A76	# <CJK>
+0x4975	0x751A	# <CJK>
+0x4976	0x80BE	# <CJK>
+0x4977	0x614E	# <CJK>
+0x4978	0x6E17	# <CJK>
+0x4979	0x58F0	# <CJK>
+0x497A	0x751F	# <CJK>
+0x497B	0x7525	# <CJK>
+0x497C	0x7272	# <CJK>
+0x497D	0x5347	# <CJK>
+0x497E	0x7EF3	# <CJK>
+0x4A21	0x7701	# <CJK>
+0x4A22	0x76DB	# <CJK>
+0x4A23	0x5269	# <CJK>
+0x4A24	0x80DC	# <CJK>
+0x4A25	0x5723	# <CJK>
+0x4A26	0x5E08	# <CJK>
+0x4A27	0x5931	# <CJK>
+0x4A28	0x72EE	# <CJK>
+0x4A29	0x65BD	# <CJK>
+0x4A2A	0x6E7F	# <CJK>
+0x4A2B	0x8BD7	# <CJK>
+0x4A2C	0x5C38	# <CJK>
+0x4A2D	0x8671	# <CJK>
+0x4A2E	0x5341	# <CJK>
+0x4A2F	0x77F3	# <CJK>
+0x4A30	0x62FE	# <CJK>
+0x4A31	0x65F6	# <CJK>
+0x4A32	0x4EC0	# <CJK>
+0x4A33	0x98DF	# <CJK>
+0x4A34	0x8680	# <CJK>
+0x4A35	0x5B9E	# <CJK>
+0x4A36	0x8BC6	# <CJK>
+0x4A37	0x53F2	# <CJK>
+0x4A38	0x77E2	# <CJK>
+0x4A39	0x4F7F	# <CJK>
+0x4A3A	0x5C4E	# <CJK>
+0x4A3B	0x9A76	# <CJK>
+0x4A3C	0x59CB	# <CJK>
+0x4A3D	0x5F0F	# <CJK>
+0x4A3E	0x793A	# <CJK>
+0x4A3F	0x58EB	# <CJK>
+0x4A40	0x4E16	# <CJK>
+0x4A41	0x67FF	# <CJK>
+0x4A42	0x4E8B	# <CJK>
+0x4A43	0x62ED	# <CJK>
+0x4A44	0x8A93	# <CJK>
+0x4A45	0x901D	# <CJK>
+0x4A46	0x52BF	# <CJK>
+0x4A47	0x662F	# <CJK>
+0x4A48	0x55DC	# <CJK>
+0x4A49	0x566C	# <CJK>
+0x4A4A	0x9002	# <CJK>
+0x4A4B	0x4ED5	# <CJK>
+0x4A4C	0x4F8D	# <CJK>
+0x4A4D	0x91CA	# <CJK>
+0x4A4E	0x9970	# <CJK>
+0x4A4F	0x6C0F	# <CJK>
+0x4A50	0x5E02	# <CJK>
+0x4A51	0x6043	# <CJK>
+0x4A52	0x5BA4	# <CJK>
+0x4A53	0x89C6	# <CJK>
+0x4A54	0x8BD5	# <CJK>
+0x4A55	0x6536	# <CJK>
+0x4A56	0x624B	# <CJK>
+0x4A57	0x9996	# <CJK>
+0x4A58	0x5B88	# <CJK>
+0x4A59	0x5BFF	# <CJK>
+0x4A5A	0x6388	# <CJK>
+0x4A5B	0x552E	# <CJK>
+0x4A5C	0x53D7	# <CJK>
+0x4A5D	0x7626	# <CJK>
+0x4A5E	0x517D	# <CJK>
+0x4A5F	0x852C	# <CJK>
+0x4A60	0x67A2	# <CJK>
+0x4A61	0x68B3	# <CJK>
+0x4A62	0x6B8A	# <CJK>
+0x4A63	0x6292	# <CJK>
+0x4A64	0x8F93	# <CJK>
+0x4A65	0x53D4	# <CJK>
+0x4A66	0x8212	# <CJK>
+0x4A67	0x6DD1	# <CJK>
+0x4A68	0x758F	# <CJK>
+0x4A69	0x4E66	# <CJK>
+0x4A6A	0x8D4E	# <CJK>
+0x4A6B	0x5B70	# <CJK>
+0x4A6C	0x719F	# <CJK>
+0x4A6D	0x85AF	# <CJK>
+0x4A6E	0x6691	# <CJK>
+0x4A6F	0x66D9	# <CJK>
+0x4A70	0x7F72	# <CJK>
+0x4A71	0x8700	# <CJK>
+0x4A72	0x9ECD	# <CJK>
+0x4A73	0x9F20	# <CJK>
+0x4A74	0x5C5E	# <CJK>
+0x4A75	0x672F	# <CJK>
+0x4A76	0x8FF0	# <CJK>
+0x4A77	0x6811	# <CJK>
+0x4A78	0x675F	# <CJK>
+0x4A79	0x620D	# <CJK>
+0x4A7A	0x7AD6	# <CJK>
+0x4A7B	0x5885	# <CJK>
+0x4A7C	0x5EB6	# <CJK>
+0x4A7D	0x6570	# <CJK>
+0x4A7E	0x6F31	# <CJK>
+0x4B21	0x6055	# <CJK>
+0x4B22	0x5237	# <CJK>
+0x4B23	0x800D	# <CJK>
+0x4B24	0x6454	# <CJK>
+0x4B25	0x8870	# <CJK>
+0x4B26	0x7529	# <CJK>
+0x4B27	0x5E05	# <CJK>
+0x4B28	0x6813	# <CJK>
+0x4B29	0x62F4	# <CJK>
+0x4B2A	0x971C	# <CJK>
+0x4B2B	0x53CC	# <CJK>
+0x4B2C	0x723D	# <CJK>
+0x4B2D	0x8C01	# <CJK>
+0x4B2E	0x6C34	# <CJK>
+0x4B2F	0x7761	# <CJK>
+0x4B30	0x7A0E	# <CJK>
+0x4B31	0x542E	# <CJK>
+0x4B32	0x77AC	# <CJK>
+0x4B33	0x987A	# <CJK>
+0x4B34	0x821C	# <CJK>
+0x4B35	0x8BF4	# <CJK>
+0x4B36	0x7855	# <CJK>
+0x4B37	0x6714	# <CJK>
+0x4B38	0x70C1	# <CJK>
+0x4B39	0x65AF	# <CJK>
+0x4B3A	0x6495	# <CJK>
+0x4B3B	0x5636	# <CJK>
+0x4B3C	0x601D	# <CJK>
+0x4B3D	0x79C1	# <CJK>
+0x4B3E	0x53F8	# <CJK>
+0x4B3F	0x4E1D	# <CJK>
+0x4B40	0x6B7B	# <CJK>
+0x4B41	0x8086	# <CJK>
+0x4B42	0x5BFA	# <CJK>
+0x4B43	0x55E3	# <CJK>
+0x4B44	0x56DB	# <CJK>
+0x4B45	0x4F3A	# <CJK>
+0x4B46	0x4F3C	# <CJK>
+0x4B47	0x9972	# <CJK>
+0x4B48	0x5DF3	# <CJK>
+0x4B49	0x677E	# <CJK>
+0x4B4A	0x8038	# <CJK>
+0x4B4B	0x6002	# <CJK>
+0x4B4C	0x9882	# <CJK>
+0x4B4D	0x9001	# <CJK>
+0x4B4E	0x5B8B	# <CJK>
+0x4B4F	0x8BBC	# <CJK>
+0x4B50	0x8BF5	# <CJK>
+0x4B51	0x641C	# <CJK>
+0x4B52	0x8258	# <CJK>
+0x4B53	0x64DE	# <CJK>
+0x4B54	0x55FD	# <CJK>
+0x4B55	0x82CF	# <CJK>
+0x4B56	0x9165	# <CJK>
+0x4B57	0x4FD7	# <CJK>
+0x4B58	0x7D20	# <CJK>
+0x4B59	0x901F	# <CJK>
+0x4B5A	0x7C9F	# <CJK>
+0x4B5B	0x50F3	# <CJK>
+0x4B5C	0x5851	# <CJK>
+0x4B5D	0x6EAF	# <CJK>
+0x4B5E	0x5BBF	# <CJK>
+0x4B5F	0x8BC9	# <CJK>
+0x4B60	0x8083	# <CJK>
+0x4B61	0x9178	# <CJK>
+0x4B62	0x849C	# <CJK>
+0x4B63	0x7B97	# <CJK>
+0x4B64	0x867D	# <CJK>
+0x4B65	0x968B	# <CJK>
+0x4B66	0x968F	# <CJK>
+0x4B67	0x7EE5	# <CJK>
+0x4B68	0x9AD3	# <CJK>
+0x4B69	0x788E	# <CJK>
+0x4B6A	0x5C81	# <CJK>
+0x4B6B	0x7A57	# <CJK>
+0x4B6C	0x9042	# <CJK>
+0x4B6D	0x96A7	# <CJK>
+0x4B6E	0x795F	# <CJK>
+0x4B6F	0x5B59	# <CJK>
+0x4B70	0x635F	# <CJK>
+0x4B71	0x7B0B	# <CJK>
+0x4B72	0x84D1	# <CJK>
+0x4B73	0x68AD	# <CJK>
+0x4B74	0x5506	# <CJK>
+0x4B75	0x7F29	# <CJK>
+0x4B76	0x7410	# <CJK>
+0x4B77	0x7D22	# <CJK>
+0x4B78	0x9501	# <CJK>
+0x4B79	0x6240	# <CJK>
+0x4B7A	0x584C	# <CJK>
+0x4B7B	0x4ED6	# <CJK>
+0x4B7C	0x5B83	# <CJK>
+0x4B7D	0x5979	# <CJK>
+0x4B7E	0x5854	# <CJK>
+0x4C21	0x736D	# <CJK>
+0x4C22	0x631E	# <CJK>
+0x4C23	0x8E4B	# <CJK>
+0x4C24	0x8E0F	# <CJK>
+0x4C25	0x80CE	# <CJK>
+0x4C26	0x82D4	# <CJK>
+0x4C27	0x62AC	# <CJK>
+0x4C28	0x53F0	# <CJK>
+0x4C29	0x6CF0	# <CJK>
+0x4C2A	0x915E	# <CJK>
+0x4C2B	0x592A	# <CJK>
+0x4C2C	0x6001	# <CJK>
+0x4C2D	0x6C70	# <CJK>
+0x4C2E	0x574D	# <CJK>
+0x4C2F	0x644A	# <CJK>
+0x4C30	0x8D2A	# <CJK>
+0x4C31	0x762B	# <CJK>
+0x4C32	0x6EE9	# <CJK>
+0x4C33	0x575B	# <CJK>
+0x4C34	0x6A80	# <CJK>
+0x4C35	0x75F0	# <CJK>
+0x4C36	0x6F6D	# <CJK>
+0x4C37	0x8C2D	# <CJK>
+0x4C38	0x8C08	# <CJK>
+0x4C39	0x5766	# <CJK>
+0x4C3A	0x6BEF	# <CJK>
+0x4C3B	0x8892	# <CJK>
+0x4C3C	0x78B3	# <CJK>
+0x4C3D	0x63A2	# <CJK>
+0x4C3E	0x53F9	# <CJK>
+0x4C3F	0x70AD	# <CJK>
+0x4C40	0x6C64	# <CJK>
+0x4C41	0x5858	# <CJK>
+0x4C42	0x642A	# <CJK>
+0x4C43	0x5802	# <CJK>
+0x4C44	0x68E0	# <CJK>
+0x4C45	0x819B	# <CJK>
+0x4C46	0x5510	# <CJK>
+0x4C47	0x7CD6	# <CJK>
+0x4C48	0x5018	# <CJK>
+0x4C49	0x8EBA	# <CJK>
+0x4C4A	0x6DCC	# <CJK>
+0x4C4B	0x8D9F	# <CJK>
+0x4C4C	0x70EB	# <CJK>
+0x4C4D	0x638F	# <CJK>
+0x4C4E	0x6D9B	# <CJK>
+0x4C4F	0x6ED4	# <CJK>
+0x4C50	0x7EE6	# <CJK>
+0x4C51	0x8404	# <CJK>
+0x4C52	0x6843	# <CJK>
+0x4C53	0x9003	# <CJK>
+0x4C54	0x6DD8	# <CJK>
+0x4C55	0x9676	# <CJK>
+0x4C56	0x8BA8	# <CJK>
+0x4C57	0x5957	# <CJK>
+0x4C58	0x7279	# <CJK>
+0x4C59	0x85E4	# <CJK>
+0x4C5A	0x817E	# <CJK>
+0x4C5B	0x75BC	# <CJK>
+0x4C5C	0x8A8A	# <CJK>
+0x4C5D	0x68AF	# <CJK>
+0x4C5E	0x5254	# <CJK>
+0x4C5F	0x8E22	# <CJK>
+0x4C60	0x9511	# <CJK>
+0x4C61	0x63D0	# <CJK>
+0x4C62	0x9898	# <CJK>
+0x4C63	0x8E44	# <CJK>
+0x4C64	0x557C	# <CJK>
+0x4C65	0x4F53	# <CJK>
+0x4C66	0x66FF	# <CJK>
+0x4C67	0x568F	# <CJK>
+0x4C68	0x60D5	# <CJK>
+0x4C69	0x6D95	# <CJK>
+0x4C6A	0x5243	# <CJK>
+0x4C6B	0x5C49	# <CJK>
+0x4C6C	0x5929	# <CJK>
+0x4C6D	0x6DFB	# <CJK>
+0x4C6E	0x586B	# <CJK>
+0x4C6F	0x7530	# <CJK>
+0x4C70	0x751C	# <CJK>
+0x4C71	0x606C	# <CJK>
+0x4C72	0x8214	# <CJK>
+0x4C73	0x8146	# <CJK>
+0x4C74	0x6311	# <CJK>
+0x4C75	0x6761	# <CJK>
+0x4C76	0x8FE2	# <CJK>
+0x4C77	0x773A	# <CJK>
+0x4C78	0x8DF3	# <CJK>
+0x4C79	0x8D34	# <CJK>
+0x4C7A	0x94C1	# <CJK>
+0x4C7B	0x5E16	# <CJK>
+0x4C7C	0x5385	# <CJK>
+0x4C7D	0x542C	# <CJK>
+0x4C7E	0x70C3	# <CJK>
+0x4D21	0x6C40	# <CJK>
+0x4D22	0x5EF7	# <CJK>
+0x4D23	0x505C	# <CJK>
+0x4D24	0x4EAD	# <CJK>
+0x4D25	0x5EAD	# <CJK>
+0x4D26	0x633A	# <CJK>
+0x4D27	0x8247	# <CJK>
+0x4D28	0x901A	# <CJK>
+0x4D29	0x6850	# <CJK>
+0x4D2A	0x916E	# <CJK>
+0x4D2B	0x77B3	# <CJK>
+0x4D2C	0x540C	# <CJK>
+0x4D2D	0x94DC	# <CJK>
+0x4D2E	0x5F64	# <CJK>
+0x4D2F	0x7AE5	# <CJK>
+0x4D30	0x6876	# <CJK>
+0x4D31	0x6345	# <CJK>
+0x4D32	0x7B52	# <CJK>
+0x4D33	0x7EDF	# <CJK>
+0x4D34	0x75DB	# <CJK>
+0x4D35	0x5077	# <CJK>
+0x4D36	0x6295	# <CJK>
+0x4D37	0x5934	# <CJK>
+0x4D38	0x900F	# <CJK>
+0x4D39	0x51F8	# <CJK>
+0x4D3A	0x79C3	# <CJK>
+0x4D3B	0x7A81	# <CJK>
+0x4D3C	0x56FE	# <CJK>
+0x4D3D	0x5F92	# <CJK>
+0x4D3E	0x9014	# <CJK>
+0x4D3F	0x6D82	# <CJK>
+0x4D40	0x5C60	# <CJK>
+0x4D41	0x571F	# <CJK>
+0x4D42	0x5410	# <CJK>
+0x4D43	0x5154	# <CJK>
+0x4D44	0x6E4D	# <CJK>
+0x4D45	0x56E2	# <CJK>
+0x4D46	0x63A8	# <CJK>
+0x4D47	0x9893	# <CJK>
+0x4D48	0x817F	# <CJK>
+0x4D49	0x8715	# <CJK>
+0x4D4A	0x892A	# <CJK>
+0x4D4B	0x9000	# <CJK>
+0x4D4C	0x541E	# <CJK>
+0x4D4D	0x5C6F	# <CJK>
+0x4D4E	0x81C0	# <CJK>
+0x4D4F	0x62D6	# <CJK>
+0x4D50	0x6258	# <CJK>
+0x4D51	0x8131	# <CJK>
+0x4D52	0x9E35	# <CJK>
+0x4D53	0x9640	# <CJK>
+0x4D54	0x9A6E	# <CJK>
+0x4D55	0x9A7C	# <CJK>
+0x4D56	0x692D	# <CJK>
+0x4D57	0x59A5	# <CJK>
+0x4D58	0x62D3	# <CJK>
+0x4D59	0x553E	# <CJK>
+0x4D5A	0x6316	# <CJK>
+0x4D5B	0x54C7	# <CJK>
+0x4D5C	0x86D9	# <CJK>
+0x4D5D	0x6D3C	# <CJK>
+0x4D5E	0x5A03	# <CJK>
+0x4D5F	0x74E6	# <CJK>
+0x4D60	0x889C	# <CJK>
+0x4D61	0x6B6A	# <CJK>
+0x4D62	0x5916	# <CJK>
+0x4D63	0x8C4C	# <CJK>
+0x4D64	0x5F2F	# <CJK>
+0x4D65	0x6E7E	# <CJK>
+0x4D66	0x73A9	# <CJK>
+0x4D67	0x987D	# <CJK>
+0x4D68	0x4E38	# <CJK>
+0x4D69	0x70F7	# <CJK>
+0x4D6A	0x5B8C	# <CJK>
+0x4D6B	0x7897	# <CJK>
+0x4D6C	0x633D	# <CJK>
+0x4D6D	0x665A	# <CJK>
+0x4D6E	0x7696	# <CJK>
+0x4D6F	0x60CB	# <CJK>
+0x4D70	0x5B9B	# <CJK>
+0x4D71	0x5A49	# <CJK>
+0x4D72	0x4E07	# <CJK>
+0x4D73	0x8155	# <CJK>
+0x4D74	0x6C6A	# <CJK>
+0x4D75	0x738B	# <CJK>
+0x4D76	0x4EA1	# <CJK>
+0x4D77	0x6789	# <CJK>
+0x4D78	0x7F51	# <CJK>
+0x4D79	0x5F80	# <CJK>
+0x4D7A	0x65FA	# <CJK>
+0x4D7B	0x671B	# <CJK>
+0x4D7C	0x5FD8	# <CJK>
+0x4D7D	0x5984	# <CJK>
+0x4D7E	0x5A01	# <CJK>
+0x4E21	0x5DCD	# <CJK>
+0x4E22	0x5FAE	# <CJK>
+0x4E23	0x5371	# <CJK>
+0x4E24	0x97E6	# <CJK>
+0x4E25	0x8FDD	# <CJK>
+0x4E26	0x6845	# <CJK>
+0x4E27	0x56F4	# <CJK>
+0x4E28	0x552F	# <CJK>
+0x4E29	0x60DF	# <CJK>
+0x4E2A	0x4E3A	# <CJK>
+0x4E2B	0x6F4D	# <CJK>
+0x4E2C	0x7EF4	# <CJK>
+0x4E2D	0x82C7	# <CJK>
+0x4E2E	0x840E	# <CJK>
+0x4E2F	0x59D4	# <CJK>
+0x4E30	0x4F1F	# <CJK>
+0x4E31	0x4F2A	# <CJK>
+0x4E32	0x5C3E	# <CJK>
+0x4E33	0x7EAC	# <CJK>
+0x4E34	0x672A	# <CJK>
+0x4E35	0x851A	# <CJK>
+0x4E36	0x5473	# <CJK>
+0x4E37	0x754F	# <CJK>
+0x4E38	0x80C3	# <CJK>
+0x4E39	0x5582	# <CJK>
+0x4E3A	0x9B4F	# <CJK>
+0x4E3B	0x4F4D	# <CJK>
+0x4E3C	0x6E2D	# <CJK>
+0x4E3D	0x8C13	# <CJK>
+0x4E3E	0x5C09	# <CJK>
+0x4E3F	0x6170	# <CJK>
+0x4E40	0x536B	# <CJK>
+0x4E41	0x761F	# <CJK>
+0x4E42	0x6E29	# <CJK>
+0x4E43	0x868A	# <CJK>
+0x4E44	0x6587	# <CJK>
+0x4E45	0x95FB	# <CJK>
+0x4E46	0x7EB9	# <CJK>
+0x4E47	0x543B	# <CJK>
+0x4E48	0x7A33	# <CJK>
+0x4E49	0x7D0A	# <CJK>
+0x4E4A	0x95EE	# <CJK>
+0x4E4B	0x55E1	# <CJK>
+0x4E4C	0x7FC1	# <CJK>
+0x4E4D	0x74EE	# <CJK>
+0x4E4E	0x631D	# <CJK>
+0x4E4F	0x8717	# <CJK>
+0x4E50	0x6DA1	# <CJK>
+0x4E51	0x7A9D	# <CJK>
+0x4E52	0x6211	# <CJK>
+0x4E53	0x65A1	# <CJK>
+0x4E54	0x5367	# <CJK>
+0x4E55	0x63E1	# <CJK>
+0x4E56	0x6C83	# <CJK>
+0x4E57	0x5DEB	# <CJK>
+0x4E58	0x545C	# <CJK>
+0x4E59	0x94A8	# <CJK>
+0x4E5A	0x4E4C	# <CJK>
+0x4E5B	0x6C61	# <CJK>
+0x4E5C	0x8BEC	# <CJK>
+0x4E5D	0x5C4B	# <CJK>
+0x4E5E	0x65E0	# <CJK>
+0x4E5F	0x829C	# <CJK>
+0x4E60	0x68A7	# <CJK>
+0x4E61	0x543E	# <CJK>
+0x4E62	0x5434	# <CJK>
+0x4E63	0x6BCB	# <CJK>
+0x4E64	0x6B66	# <CJK>
+0x4E65	0x4E94	# <CJK>
+0x4E66	0x6342	# <CJK>
+0x4E67	0x5348	# <CJK>
+0x4E68	0x821E	# <CJK>
+0x4E69	0x4F0D	# <CJK>
+0x4E6A	0x4FAE	# <CJK>
+0x4E6B	0x575E	# <CJK>
+0x4E6C	0x620A	# <CJK>
+0x4E6D	0x96FE	# <CJK>
+0x4E6E	0x6664	# <CJK>
+0x4E6F	0x7269	# <CJK>
+0x4E70	0x52FF	# <CJK>
+0x4E71	0x52A1	# <CJK>
+0x4E72	0x609F	# <CJK>
+0x4E73	0x8BEF	# <CJK>
+0x4E74	0x6614	# <CJK>
+0x4E75	0x7199	# <CJK>
+0x4E76	0x6790	# <CJK>
+0x4E77	0x897F	# <CJK>
+0x4E78	0x7852	# <CJK>
+0x4E79	0x77FD	# <CJK>
+0x4E7A	0x6670	# <CJK>
+0x4E7B	0x563B	# <CJK>
+0x4E7C	0x5438	# <CJK>
+0x4E7D	0x9521	# <CJK>
+0x4E7E	0x727A	# <CJK>
+0x4F21	0x7A00	# <CJK>
+0x4F22	0x606F	# <CJK>
+0x4F23	0x5E0C	# <CJK>
+0x4F24	0x6089	# <CJK>
+0x4F25	0x819D	# <CJK>
+0x4F26	0x5915	# <CJK>
+0x4F27	0x60DC	# <CJK>
+0x4F28	0x7184	# <CJK>
+0x4F29	0x70EF	# <CJK>
+0x4F2A	0x6EAA	# <CJK>
+0x4F2B	0x6C50	# <CJK>
+0x4F2C	0x7280	# <CJK>
+0x4F2D	0x6A84	# <CJK>
+0x4F2E	0x88AD	# <CJK>
+0x4F2F	0x5E2D	# <CJK>
+0x4F30	0x4E60	# <CJK>
+0x4F31	0x5AB3	# <CJK>
+0x4F32	0x559C	# <CJK>
+0x4F33	0x94E3	# <CJK>
+0x4F34	0x6D17	# <CJK>
+0x4F35	0x7CFB	# <CJK>
+0x4F36	0x9699	# <CJK>
+0x4F37	0x620F	# <CJK>
+0x4F38	0x7EC6	# <CJK>
+0x4F39	0x778E	# <CJK>
+0x4F3A	0x867E	# <CJK>
+0x4F3B	0x5323	# <CJK>
+0x4F3C	0x971E	# <CJK>
+0x4F3D	0x8F96	# <CJK>
+0x4F3E	0x6687	# <CJK>
+0x4F3F	0x5CE1	# <CJK>
+0x4F40	0x4FA0	# <CJK>
+0x4F41	0x72ED	# <CJK>
+0x4F42	0x4E0B	# <CJK>
+0x4F43	0x53A6	# <CJK>
+0x4F44	0x590F	# <CJK>
+0x4F45	0x5413	# <CJK>
+0x4F46	0x6380	# <CJK>
+0x4F47	0x9528	# <CJK>
+0x4F48	0x5148	# <CJK>
+0x4F49	0x4ED9	# <CJK>
+0x4F4A	0x9C9C	# <CJK>
+0x4F4B	0x7EA4	# <CJK>
+0x4F4C	0x54B8	# <CJK>
+0x4F4D	0x8D24	# <CJK>
+0x4F4E	0x8854	# <CJK>
+0x4F4F	0x8237	# <CJK>
+0x4F50	0x95F2	# <CJK>
+0x4F51	0x6D8E	# <CJK>
+0x4F52	0x5F26	# <CJK>
+0x4F53	0x5ACC	# <CJK>
+0x4F54	0x663E	# <CJK>
+0x4F55	0x9669	# <CJK>
+0x4F56	0x73B0	# <CJK>
+0x4F57	0x732E	# <CJK>
+0x4F58	0x53BF	# <CJK>
+0x4F59	0x817A	# <CJK>
+0x4F5A	0x9985	# <CJK>
+0x4F5B	0x7FA1	# <CJK>
+0x4F5C	0x5BAA	# <CJK>
+0x4F5D	0x9677	# <CJK>
+0x4F5E	0x9650	# <CJK>
+0x4F5F	0x7EBF	# <CJK>
+0x4F60	0x76F8	# <CJK>
+0x4F61	0x53A2	# <CJK>
+0x4F62	0x9576	# <CJK>
+0x4F63	0x9999	# <CJK>
+0x4F64	0x7BB1	# <CJK>
+0x4F65	0x8944	# <CJK>
+0x4F66	0x6E58	# <CJK>
+0x4F67	0x4E61	# <CJK>
+0x4F68	0x7FD4	# <CJK>
+0x4F69	0x7965	# <CJK>
+0x4F6A	0x8BE6	# <CJK>
+0x4F6B	0x60F3	# <CJK>
+0x4F6C	0x54CD	# <CJK>
+0x4F6D	0x4EAB	# <CJK>
+0x4F6E	0x9879	# <CJK>
+0x4F6F	0x5DF7	# <CJK>
+0x4F70	0x6A61	# <CJK>
+0x4F71	0x50CF	# <CJK>
+0x4F72	0x5411	# <CJK>
+0x4F73	0x8C61	# <CJK>
+0x4F74	0x8427	# <CJK>
+0x4F75	0x785D	# <CJK>
+0x4F76	0x9704	# <CJK>
+0x4F77	0x524A	# <CJK>
+0x4F78	0x54EE	# <CJK>
+0x4F79	0x56A3	# <CJK>
+0x4F7A	0x9500	# <CJK>
+0x4F7B	0x6D88	# <CJK>
+0x4F7C	0x5BB5	# <CJK>
+0x4F7D	0x6DC6	# <CJK>
+0x4F7E	0x6653	# <CJK>
+0x5021	0x5C0F	# <CJK>
+0x5022	0x5B5D	# <CJK>
+0x5023	0x6821	# <CJK>
+0x5024	0x8096	# <CJK>
+0x5025	0x5578	# <CJK>
+0x5026	0x7B11	# <CJK>
+0x5027	0x6548	# <CJK>
+0x5028	0x6954	# <CJK>
+0x5029	0x4E9B	# <CJK>
+0x502A	0x6B47	# <CJK>
+0x502B	0x874E	# <CJK>
+0x502C	0x978B	# <CJK>
+0x502D	0x534F	# <CJK>
+0x502E	0x631F	# <CJK>
+0x502F	0x643A	# <CJK>
+0x5030	0x90AA	# <CJK>
+0x5031	0x659C	# <CJK>
+0x5032	0x80C1	# <CJK>
+0x5033	0x8C10	# <CJK>
+0x5034	0x5199	# <CJK>
+0x5035	0x68B0	# <CJK>
+0x5036	0x5378	# <CJK>
+0x5037	0x87F9	# <CJK>
+0x5038	0x61C8	# <CJK>
+0x5039	0x6CC4	# <CJK>
+0x503A	0x6CFB	# <CJK>
+0x503B	0x8C22	# <CJK>
+0x503C	0x5C51	# <CJK>
+0x503D	0x85AA	# <CJK>
+0x503E	0x82AF	# <CJK>
+0x503F	0x950C	# <CJK>
+0x5040	0x6B23	# <CJK>
+0x5041	0x8F9B	# <CJK>
+0x5042	0x65B0	# <CJK>
+0x5043	0x5FFB	# <CJK>
+0x5044	0x5FC3	# <CJK>
+0x5045	0x4FE1	# <CJK>
+0x5046	0x8845	# <CJK>
+0x5047	0x661F	# <CJK>
+0x5048	0x8165	# <CJK>
+0x5049	0x7329	# <CJK>
+0x504A	0x60FA	# <CJK>
+0x504B	0x5174	# <CJK>
+0x504C	0x5211	# <CJK>
+0x504D	0x578B	# <CJK>
+0x504E	0x5F62	# <CJK>
+0x504F	0x90A2	# <CJK>
+0x5050	0x884C	# <CJK>
+0x5051	0x9192	# <CJK>
+0x5052	0x5E78	# <CJK>
+0x5053	0x674F	# <CJK>
+0x5054	0x6027	# <CJK>
+0x5055	0x59D3	# <CJK>
+0x5056	0x5144	# <CJK>
+0x5057	0x51F6	# <CJK>
+0x5058	0x80F8	# <CJK>
+0x5059	0x5308	# <CJK>
+0x505A	0x6C79	# <CJK>
+0x505B	0x96C4	# <CJK>
+0x505C	0x718A	# <CJK>
+0x505D	0x4F11	# <CJK>
+0x505E	0x4FEE	# <CJK>
+0x505F	0x7F9E	# <CJK>
+0x5060	0x673D	# <CJK>
+0x5061	0x55C5	# <CJK>
+0x5062	0x9508	# <CJK>
+0x5063	0x79C0	# <CJK>
+0x5064	0x8896	# <CJK>
+0x5065	0x7EE3	# <CJK>
+0x5066	0x589F	# <CJK>
+0x5067	0x620C	# <CJK>
+0x5068	0x9700	# <CJK>
+0x5069	0x865A	# <CJK>
+0x506A	0x5618	# <CJK>
+0x506B	0x987B	# <CJK>
+0x506C	0x5F90	# <CJK>
+0x506D	0x8BB8	# <CJK>
+0x506E	0x84C4	# <CJK>
+0x506F	0x9157	# <CJK>
+0x5070	0x53D9	# <CJK>
+0x5071	0x65ED	# <CJK>
+0x5072	0x5E8F	# <CJK>
+0x5073	0x755C	# <CJK>
+0x5074	0x6064	# <CJK>
+0x5075	0x7D6E	# <CJK>
+0x5076	0x5A7F	# <CJK>
+0x5077	0x7EEA	# <CJK>
+0x5078	0x7EED	# <CJK>
+0x5079	0x8F69	# <CJK>
+0x507A	0x55A7	# <CJK>
+0x507B	0x5BA3	# <CJK>
+0x507C	0x60AC	# <CJK>
+0x507D	0x65CB	# <CJK>
+0x507E	0x7384	# <CJK>
+0x5121	0x9009	# <CJK>
+0x5122	0x7663	# <CJK>
+0x5123	0x7729	# <CJK>
+0x5124	0x7EDA	# <CJK>
+0x5125	0x9774	# <CJK>
+0x5126	0x859B	# <CJK>
+0x5127	0x5B66	# <CJK>
+0x5128	0x7A74	# <CJK>
+0x5129	0x96EA	# <CJK>
+0x512A	0x8840	# <CJK>
+0x512B	0x52CB	# <CJK>
+0x512C	0x718F	# <CJK>
+0x512D	0x5FAA	# <CJK>
+0x512E	0x65EC	# <CJK>
+0x512F	0x8BE2	# <CJK>
+0x5130	0x5BFB	# <CJK>
+0x5131	0x9A6F	# <CJK>
+0x5132	0x5DE1	# <CJK>
+0x5133	0x6B89	# <CJK>
+0x5134	0x6C5B	# <CJK>
+0x5135	0x8BAD	# <CJK>
+0x5136	0x8BAF	# <CJK>
+0x5137	0x900A	# <CJK>
+0x5138	0x8FC5	# <CJK>
+0x5139	0x538B	# <CJK>
+0x513A	0x62BC	# <CJK>
+0x513B	0x9E26	# <CJK>
+0x513C	0x9E2D	# <CJK>
+0x513D	0x5440	# <CJK>
+0x513E	0x4E2B	# <CJK>
+0x513F	0x82BD	# <CJK>
+0x5140	0x7259	# <CJK>
+0x5141	0x869C	# <CJK>
+0x5142	0x5D16	# <CJK>
+0x5143	0x8859	# <CJK>
+0x5144	0x6DAF	# <CJK>
+0x5145	0x96C5	# <CJK>
+0x5146	0x54D1	# <CJK>
+0x5147	0x4E9A	# <CJK>
+0x5148	0x8BB6	# <CJK>
+0x5149	0x7109	# <CJK>
+0x514A	0x54BD	# <CJK>
+0x514B	0x9609	# <CJK>
+0x514C	0x70DF	# <CJK>
+0x514D	0x6DF9	# <CJK>
+0x514E	0x76D0	# <CJK>
+0x514F	0x4E25	# <CJK>
+0x5150	0x7814	# <CJK>
+0x5151	0x8712	# <CJK>
+0x5152	0x5CA9	# <CJK>
+0x5153	0x5EF6	# <CJK>
+0x5154	0x8A00	# <CJK>
+0x5155	0x989C	# <CJK>
+0x5156	0x960E	# <CJK>
+0x5157	0x708E	# <CJK>
+0x5158	0x6CBF	# <CJK>
+0x5159	0x5944	# <CJK>
+0x515A	0x63A9	# <CJK>
+0x515B	0x773C	# <CJK>
+0x515C	0x884D	# <CJK>
+0x515D	0x6F14	# <CJK>
+0x515E	0x8273	# <CJK>
+0x515F	0x5830	# <CJK>
+0x5160	0x71D5	# <CJK>
+0x5161	0x538C	# <CJK>
+0x5162	0x781A	# <CJK>
+0x5163	0x96C1	# <CJK>
+0x5164	0x5501	# <CJK>
+0x5165	0x5F66	# <CJK>
+0x5166	0x7130	# <CJK>
+0x5167	0x5BB4	# <CJK>
+0x5168	0x8C1A	# <CJK>
+0x5169	0x9A8C	# <CJK>
+0x516A	0x6B83	# <CJK>
+0x516B	0x592E	# <CJK>
+0x516C	0x9E2F	# <CJK>
+0x516D	0x79E7	# <CJK>
+0x516E	0x6768	# <CJK>
+0x516F	0x626C	# <CJK>
+0x5170	0x4F6F	# <CJK>
+0x5171	0x75A1	# <CJK>
+0x5172	0x7F8A	# <CJK>
+0x5173	0x6D0B	# <CJK>
+0x5174	0x9633	# <CJK>
+0x5175	0x6C27	# <CJK>
+0x5176	0x4EF0	# <CJK>
+0x5177	0x75D2	# <CJK>
+0x5178	0x517B	# <CJK>
+0x5179	0x6837	# <CJK>
+0x517A	0x6F3E	# <CJK>
+0x517B	0x9080	# <CJK>
+0x517C	0x8170	# <CJK>
+0x517D	0x5996	# <CJK>
+0x517E	0x7476	# <CJK>
+0x5221	0x6447	# <CJK>
+0x5222	0x5C27	# <CJK>
+0x5223	0x9065	# <CJK>
+0x5224	0x7A91	# <CJK>
+0x5225	0x8C23	# <CJK>
+0x5226	0x59DA	# <CJK>
+0x5227	0x54AC	# <CJK>
+0x5228	0x8200	# <CJK>
+0x5229	0x836F	# <CJK>
+0x522A	0x8981	# <CJK>
+0x522B	0x8000	# <CJK>
+0x522C	0x6930	# <CJK>
+0x522D	0x564E	# <CJK>
+0x522E	0x8036	# <CJK>
+0x522F	0x7237	# <CJK>
+0x5230	0x91CE	# <CJK>
+0x5231	0x51B6	# <CJK>
+0x5232	0x4E5F	# <CJK>
+0x5233	0x9875	# <CJK>
+0x5234	0x6396	# <CJK>
+0x5235	0x4E1A	# <CJK>
+0x5236	0x53F6	# <CJK>
+0x5237	0x66F3	# <CJK>
+0x5238	0x814B	# <CJK>
+0x5239	0x591C	# <CJK>
+0x523A	0x6DB2	# <CJK>
+0x523B	0x4E00	# <CJK>
+0x523C	0x58F9	# <CJK>
+0x523D	0x533B	# <CJK>
+0x523E	0x63D6	# <CJK>
+0x523F	0x94F1	# <CJK>
+0x5240	0x4F9D	# <CJK>
+0x5241	0x4F0A	# <CJK>
+0x5242	0x8863	# <CJK>
+0x5243	0x9890	# <CJK>
+0x5244	0x5937	# <CJK>
+0x5245	0x9057	# <CJK>
+0x5246	0x79FB	# <CJK>
+0x5247	0x4EEA	# <CJK>
+0x5248	0x80F0	# <CJK>
+0x5249	0x7591	# <CJK>
+0x524A	0x6C82	# <CJK>
+0x524B	0x5B9C	# <CJK>
+0x524C	0x59E8	# <CJK>
+0x524D	0x5F5D	# <CJK>
+0x524E	0x6905	# <CJK>
+0x524F	0x8681	# <CJK>
+0x5250	0x501A	# <CJK>
+0x5251	0x5DF2	# <CJK>
+0x5252	0x4E59	# <CJK>
+0x5253	0x77E3	# <CJK>
+0x5254	0x4EE5	# <CJK>
+0x5255	0x827A	# <CJK>
+0x5256	0x6291	# <CJK>
+0x5257	0x6613	# <CJK>
+0x5258	0x9091	# <CJK>
+0x5259	0x5C79	# <CJK>
+0x525A	0x4EBF	# <CJK>
+0x525B	0x5F79	# <CJK>
+0x525C	0x81C6	# <CJK>
+0x525D	0x9038	# <CJK>
+0x525E	0x8084	# <CJK>
+0x525F	0x75AB	# <CJK>
+0x5260	0x4EA6	# <CJK>
+0x5261	0x88D4	# <CJK>
+0x5262	0x610F	# <CJK>
+0x5263	0x6BC5	# <CJK>
+0x5264	0x5FC6	# <CJK>
+0x5265	0x4E49	# <CJK>
+0x5266	0x76CA	# <CJK>
+0x5267	0x6EA2	# <CJK>
+0x5268	0x8BE3	# <CJK>
+0x5269	0x8BAE	# <CJK>
+0x526A	0x8C0A	# <CJK>
+0x526B	0x8BD1	# <CJK>
+0x526C	0x5F02	# <CJK>
+0x526D	0x7FFC	# <CJK>
+0x526E	0x7FCC	# <CJK>
+0x526F	0x7ECE	# <CJK>
+0x5270	0x8335	# <CJK>
+0x5271	0x836B	# <CJK>
+0x5272	0x56E0	# <CJK>
+0x5273	0x6BB7	# <CJK>
+0x5274	0x97F3	# <CJK>
+0x5275	0x9634	# <CJK>
+0x5276	0x59FB	# <CJK>
+0x5277	0x541F	# <CJK>
+0x5278	0x94F6	# <CJK>
+0x5279	0x6DEB	# <CJK>
+0x527A	0x5BC5	# <CJK>
+0x527B	0x996E	# <CJK>
+0x527C	0x5C39	# <CJK>
+0x527D	0x5F15	# <CJK>
+0x527E	0x9690	# <CJK>
+0x5321	0x5370	# <CJK>
+0x5322	0x82F1	# <CJK>
+0x5323	0x6A31	# <CJK>
+0x5324	0x5A74	# <CJK>
+0x5325	0x9E70	# <CJK>
+0x5326	0x5E94	# <CJK>
+0x5327	0x7F28	# <CJK>
+0x5328	0x83B9	# <CJK>
+0x5329	0x8424	# <CJK>
+0x532A	0x8425	# <CJK>
+0x532B	0x8367	# <CJK>
+0x532C	0x8747	# <CJK>
+0x532D	0x8FCE	# <CJK>
+0x532E	0x8D62	# <CJK>
+0x532F	0x76C8	# <CJK>
+0x5330	0x5F71	# <CJK>
+0x5331	0x9896	# <CJK>
+0x5332	0x786C	# <CJK>
+0x5333	0x6620	# <CJK>
+0x5334	0x54DF	# <CJK>
+0x5335	0x62E5	# <CJK>
+0x5336	0x4F63	# <CJK>
+0x5337	0x81C3	# <CJK>
+0x5338	0x75C8	# <CJK>
+0x5339	0x5EB8	# <CJK>
+0x533A	0x96CD	# <CJK>
+0x533B	0x8E0A	# <CJK>
+0x533C	0x86F9	# <CJK>
+0x533D	0x548F	# <CJK>
+0x533E	0x6CF3	# <CJK>
+0x533F	0x6D8C	# <CJK>
+0x5340	0x6C38	# <CJK>
+0x5341	0x607F	# <CJK>
+0x5342	0x52C7	# <CJK>
+0x5343	0x7528	# <CJK>
+0x5344	0x5E7D	# <CJK>
+0x5345	0x4F18	# <CJK>
+0x5346	0x60A0	# <CJK>
+0x5347	0x5FE7	# <CJK>
+0x5348	0x5C24	# <CJK>
+0x5349	0x7531	# <CJK>
+0x534A	0x90AE	# <CJK>
+0x534B	0x94C0	# <CJK>
+0x534C	0x72B9	# <CJK>
+0x534D	0x6CB9	# <CJK>
+0x534E	0x6E38	# <CJK>
+0x534F	0x9149	# <CJK>
+0x5350	0x6709	# <CJK>
+0x5351	0x53CB	# <CJK>
+0x5352	0x53F3	# <CJK>
+0x5353	0x4F51	# <CJK>
+0x5354	0x91C9	# <CJK>
+0x5355	0x8BF1	# <CJK>
+0x5356	0x53C8	# <CJK>
+0x5357	0x5E7C	# <CJK>
+0x5358	0x8FC2	# <CJK>
+0x5359	0x6DE4	# <CJK>
+0x535A	0x4E8E	# <CJK>
+0x535B	0x76C2	# <CJK>
+0x535C	0x6986	# <CJK>
+0x535D	0x865E	# <CJK>
+0x535E	0x611A	# <CJK>
+0x535F	0x8206	# <CJK>
+0x5360	0x4F59	# <CJK>
+0x5361	0x4FDE	# <CJK>
+0x5362	0x903E	# <CJK>
+0x5363	0x9C7C	# <CJK>
+0x5364	0x6109	# <CJK>
+0x5365	0x6E1D	# <CJK>
+0x5366	0x6E14	# <CJK>
+0x5367	0x9685	# <CJK>
+0x5368	0x4E88	# <CJK>
+0x5369	0x5A31	# <CJK>
+0x536A	0x96E8	# <CJK>
+0x536B	0x4E0E	# <CJK>
+0x536C	0x5C7F	# <CJK>
+0x536D	0x79B9	# <CJK>
+0x536E	0x5B87	# <CJK>
+0x536F	0x8BED	# <CJK>
+0x5370	0x7FBD	# <CJK>
+0x5371	0x7389	# <CJK>
+0x5372	0x57DF	# <CJK>
+0x5373	0x828B	# <CJK>
+0x5374	0x90C1	# <CJK>
+0x5375	0x5401	# <CJK>
+0x5376	0x9047	# <CJK>
+0x5377	0x55BB	# <CJK>
+0x5378	0x5CEA	# <CJK>
+0x5379	0x5FA1	# <CJK>
+0x537A	0x6108	# <CJK>
+0x537B	0x6B32	# <CJK>
+0x537C	0x72F1	# <CJK>
+0x537D	0x80B2	# <CJK>
+0x537E	0x8A89	# <CJK>
+0x5421	0x6D74	# <CJK>
+0x5422	0x5BD3	# <CJK>
+0x5423	0x88D5	# <CJK>
+0x5424	0x9884	# <CJK>
+0x5425	0x8C6B	# <CJK>
+0x5426	0x9A6D	# <CJK>
+0x5427	0x9E33	# <CJK>
+0x5428	0x6E0A	# <CJK>
+0x5429	0x51A4	# <CJK>
+0x542A	0x5143	# <CJK>
+0x542B	0x57A3	# <CJK>
+0x542C	0x8881	# <CJK>
+0x542D	0x539F	# <CJK>
+0x542E	0x63F4	# <CJK>
+0x542F	0x8F95	# <CJK>
+0x5430	0x56ED	# <CJK>
+0x5431	0x5458	# <CJK>
+0x5432	0x5706	# <CJK>
+0x5433	0x733F	# <CJK>
+0x5434	0x6E90	# <CJK>
+0x5435	0x7F18	# <CJK>
+0x5436	0x8FDC	# <CJK>
+0x5437	0x82D1	# <CJK>
+0x5438	0x613F	# <CJK>
+0x5439	0x6028	# <CJK>
+0x543A	0x9662	# <CJK>
+0x543B	0x66F0	# <CJK>
+0x543C	0x7EA6	# <CJK>
+0x543D	0x8D8A	# <CJK>
+0x543E	0x8DC3	# <CJK>
+0x543F	0x94A5	# <CJK>
+0x5440	0x5CB3	# <CJK>
+0x5441	0x7CA4	# <CJK>
+0x5442	0x6708	# <CJK>
+0x5443	0x60A6	# <CJK>
+0x5444	0x9605	# <CJK>
+0x5445	0x8018	# <CJK>
+0x5446	0x4E91	# <CJK>
+0x5447	0x90E7	# <CJK>
+0x5448	0x5300	# <CJK>
+0x5449	0x9668	# <CJK>
+0x544A	0x5141	# <CJK>
+0x544B	0x8FD0	# <CJK>
+0x544C	0x8574	# <CJK>
+0x544D	0x915D	# <CJK>
+0x544E	0x6655	# <CJK>
+0x544F	0x97F5	# <CJK>
+0x5450	0x5B55	# <CJK>
+0x5451	0x531D	# <CJK>
+0x5452	0x7838	# <CJK>
+0x5453	0x6742	# <CJK>
+0x5454	0x683D	# <CJK>
+0x5455	0x54C9	# <CJK>
+0x5456	0x707E	# <CJK>
+0x5457	0x5BB0	# <CJK>
+0x5458	0x8F7D	# <CJK>
+0x5459	0x518D	# <CJK>
+0x545A	0x5728	# <CJK>
+0x545B	0x54B1	# <CJK>
+0x545C	0x6512	# <CJK>
+0x545D	0x6682	# <CJK>
+0x545E	0x8D5E	# <CJK>
+0x545F	0x8D43	# <CJK>
+0x5460	0x810F	# <CJK>
+0x5461	0x846C	# <CJK>
+0x5462	0x906D	# <CJK>
+0x5463	0x7CDF	# <CJK>
+0x5464	0x51FF	# <CJK>
+0x5465	0x85FB	# <CJK>
+0x5466	0x67A3	# <CJK>
+0x5467	0x65E9	# <CJK>
+0x5468	0x6FA1	# <CJK>
+0x5469	0x86A4	# <CJK>
+0x546A	0x8E81	# <CJK>
+0x546B	0x566A	# <CJK>
+0x546C	0x9020	# <CJK>
+0x546D	0x7682	# <CJK>
+0x546E	0x7076	# <CJK>
+0x546F	0x71E5	# <CJK>
+0x5470	0x8D23	# <CJK>
+0x5471	0x62E9	# <CJK>
+0x5472	0x5219	# <CJK>
+0x5473	0x6CFD	# <CJK>
+0x5474	0x8D3C	# <CJK>
+0x5475	0x600E	# <CJK>
+0x5476	0x589E	# <CJK>
+0x5477	0x618E	# <CJK>
+0x5478	0x66FE	# <CJK>
+0x5479	0x8D60	# <CJK>
+0x547A	0x624E	# <CJK>
+0x547B	0x55B3	# <CJK>
+0x547C	0x6E23	# <CJK>
+0x547D	0x672D	# <CJK>
+0x547E	0x8F67	# <CJK>
+0x5521	0x94E1	# <CJK>
+0x5522	0x95F8	# <CJK>
+0x5523	0x7728	# <CJK>
+0x5524	0x6805	# <CJK>
+0x5525	0x69A8	# <CJK>
+0x5526	0x548B	# <CJK>
+0x5527	0x4E4D	# <CJK>
+0x5528	0x70B8	# <CJK>
+0x5529	0x8BC8	# <CJK>
+0x552A	0x6458	# <CJK>
+0x552B	0x658B	# <CJK>
+0x552C	0x5B85	# <CJK>
+0x552D	0x7A84	# <CJK>
+0x552E	0x503A	# <CJK>
+0x552F	0x5BE8	# <CJK>
+0x5530	0x77BB	# <CJK>
+0x5531	0x6BE1	# <CJK>
+0x5532	0x8A79	# <CJK>
+0x5533	0x7C98	# <CJK>
+0x5534	0x6CBE	# <CJK>
+0x5535	0x76CF	# <CJK>
+0x5536	0x65A9	# <CJK>
+0x5537	0x8F97	# <CJK>
+0x5538	0x5D2D	# <CJK>
+0x5539	0x5C55	# <CJK>
+0x553A	0x8638	# <CJK>
+0x553B	0x6808	# <CJK>
+0x553C	0x5360	# <CJK>
+0x553D	0x6218	# <CJK>
+0x553E	0x7AD9	# <CJK>
+0x553F	0x6E5B	# <CJK>
+0x5540	0x7EFD	# <CJK>
+0x5541	0x6A1F	# <CJK>
+0x5542	0x7AE0	# <CJK>
+0x5543	0x5F70	# <CJK>
+0x5544	0x6F33	# <CJK>
+0x5545	0x5F20	# <CJK>
+0x5546	0x638C	# <CJK>
+0x5547	0x6DA8	# <CJK>
+0x5548	0x6756	# <CJK>
+0x5549	0x4E08	# <CJK>
+0x554A	0x5E10	# <CJK>
+0x554B	0x8D26	# <CJK>
+0x554C	0x4ED7	# <CJK>
+0x554D	0x80C0	# <CJK>
+0x554E	0x7634	# <CJK>
+0x554F	0x969C	# <CJK>
+0x5550	0x62DB	# <CJK>
+0x5551	0x662D	# <CJK>
+0x5552	0x627E	# <CJK>
+0x5553	0x6CBC	# <CJK>
+0x5554	0x8D75	# <CJK>
+0x5555	0x7167	# <CJK>
+0x5556	0x7F69	# <CJK>
+0x5557	0x5146	# <CJK>
+0x5558	0x8087	# <CJK>
+0x5559	0x53EC	# <CJK>
+0x555A	0x906E	# <CJK>
+0x555B	0x6298	# <CJK>
+0x555C	0x54F2	# <CJK>
+0x555D	0x86F0	# <CJK>
+0x555E	0x8F99	# <CJK>
+0x555F	0x8005	# <CJK>
+0x5560	0x9517	# <CJK>
+0x5561	0x8517	# <CJK>
+0x5562	0x8FD9	# <CJK>
+0x5563	0x6D59	# <CJK>
+0x5564	0x73CD	# <CJK>
+0x5565	0x659F	# <CJK>
+0x5566	0x771F	# <CJK>
+0x5567	0x7504	# <CJK>
+0x5568	0x7827	# <CJK>
+0x5569	0x81FB	# <CJK>
+0x556A	0x8D1E	# <CJK>
+0x556B	0x9488	# <CJK>
+0x556C	0x4FA6	# <CJK>
+0x556D	0x6795	# <CJK>
+0x556E	0x75B9	# <CJK>
+0x556F	0x8BCA	# <CJK>
+0x5570	0x9707	# <CJK>
+0x5571	0x632F	# <CJK>
+0x5572	0x9547	# <CJK>
+0x5573	0x9635	# <CJK>
+0x5574	0x84B8	# <CJK>
+0x5575	0x6323	# <CJK>
+0x5576	0x7741	# <CJK>
+0x5577	0x5F81	# <CJK>
+0x5578	0x72F0	# <CJK>
+0x5579	0x4E89	# <CJK>
+0x557A	0x6014	# <CJK>
+0x557B	0x6574	# <CJK>
+0x557C	0x62EF	# <CJK>
+0x557D	0x6B63	# <CJK>
+0x557E	0x653F	# <CJK>
+0x5621	0x5E27	# <CJK>
+0x5622	0x75C7	# <CJK>
+0x5623	0x90D1	# <CJK>
+0x5624	0x8BC1	# <CJK>
+0x5625	0x829D	# <CJK>
+0x5626	0x679D	# <CJK>
+0x5627	0x652F	# <CJK>
+0x5628	0x5431	# <CJK>
+0x5629	0x8718	# <CJK>
+0x562A	0x77E5	# <CJK>
+0x562B	0x80A2	# <CJK>
+0x562C	0x8102	# <CJK>
+0x562D	0x6C41	# <CJK>
+0x562E	0x4E4B	# <CJK>
+0x562F	0x7EC7	# <CJK>
+0x5630	0x804C	# <CJK>
+0x5631	0x76F4	# <CJK>
+0x5632	0x690D	# <CJK>
+0x5633	0x6B96	# <CJK>
+0x5634	0x6267	# <CJK>
+0x5635	0x503C	# <CJK>
+0x5636	0x4F84	# <CJK>
+0x5637	0x5740	# <CJK>
+0x5638	0x6307	# <CJK>
+0x5639	0x6B62	# <CJK>
+0x563A	0x8DBE	# <CJK>
+0x563B	0x53EA	# <CJK>
+0x563C	0x65E8	# <CJK>
+0x563D	0x7EB8	# <CJK>
+0x563E	0x5FD7	# <CJK>
+0x563F	0x631A	# <CJK>
+0x5640	0x63B7	# <CJK>
+0x5641	0x81F3	# <CJK>
+0x5642	0x81F4	# <CJK>
+0x5643	0x7F6E	# <CJK>
+0x5644	0x5E1C	# <CJK>
+0x5645	0x5CD9	# <CJK>
+0x5646	0x5236	# <CJK>
+0x5647	0x667A	# <CJK>
+0x5648	0x79E9	# <CJK>
+0x5649	0x7A1A	# <CJK>
+0x564A	0x8D28	# <CJK>
+0x564B	0x7099	# <CJK>
+0x564C	0x75D4	# <CJK>
+0x564D	0x6EDE	# <CJK>
+0x564E	0x6CBB	# <CJK>
+0x564F	0x7A92	# <CJK>
+0x5650	0x4E2D	# <CJK>
+0x5651	0x76C5	# <CJK>
+0x5652	0x5FE0	# <CJK>
+0x5653	0x949F	# <CJK>
+0x5654	0x8877	# <CJK>
+0x5655	0x7EC8	# <CJK>
+0x5656	0x79CD	# <CJK>
+0x5657	0x80BF	# <CJK>
+0x5658	0x91CD	# <CJK>
+0x5659	0x4EF2	# <CJK>
+0x565A	0x4F17	# <CJK>
+0x565B	0x821F	# <CJK>
+0x565C	0x5468	# <CJK>
+0x565D	0x5DDE	# <CJK>
+0x565E	0x6D32	# <CJK>
+0x565F	0x8BCC	# <CJK>
+0x5660	0x7CA5	# <CJK>
+0x5661	0x8F74	# <CJK>
+0x5662	0x8098	# <CJK>
+0x5663	0x5E1A	# <CJK>
+0x5664	0x5492	# <CJK>
+0x5665	0x76B1	# <CJK>
+0x5666	0x5B99	# <CJK>
+0x5667	0x663C	# <CJK>
+0x5668	0x9AA4	# <CJK>
+0x5669	0x73E0	# <CJK>
+0x566A	0x682A	# <CJK>
+0x566B	0x86DB	# <CJK>
+0x566C	0x6731	# <CJK>
+0x566D	0x732A	# <CJK>
+0x566E	0x8BF8	# <CJK>
+0x566F	0x8BDB	# <CJK>
+0x5670	0x9010	# <CJK>
+0x5671	0x7AF9	# <CJK>
+0x5672	0x70DB	# <CJK>
+0x5673	0x716E	# <CJK>
+0x5674	0x62C4	# <CJK>
+0x5675	0x77A9	# <CJK>
+0x5676	0x5631	# <CJK>
+0x5677	0x4E3B	# <CJK>
+0x5678	0x8457	# <CJK>
+0x5679	0x67F1	# <CJK>
+0x567A	0x52A9	# <CJK>
+0x567B	0x86C0	# <CJK>
+0x567C	0x8D2E	# <CJK>
+0x567D	0x94F8	# <CJK>
+0x567E	0x7B51	# <CJK>
+0x5721	0x4F4F	# <CJK>
+0x5722	0x6CE8	# <CJK>
+0x5723	0x795D	# <CJK>
+0x5724	0x9A7B	# <CJK>
+0x5725	0x6293	# <CJK>
+0x5726	0x722A	# <CJK>
+0x5727	0x62FD	# <CJK>
+0x5728	0x4E13	# <CJK>
+0x5729	0x7816	# <CJK>
+0x572A	0x8F6C	# <CJK>
+0x572B	0x64B0	# <CJK>
+0x572C	0x8D5A	# <CJK>
+0x572D	0x7BC6	# <CJK>
+0x572E	0x6869	# <CJK>
+0x572F	0x5E84	# <CJK>
+0x5730	0x88C5	# <CJK>
+0x5731	0x5986	# <CJK>
+0x5732	0x649E	# <CJK>
+0x5733	0x58EE	# <CJK>
+0x5734	0x72B6	# <CJK>
+0x5735	0x690E	# <CJK>
+0x5736	0x9525	# <CJK>
+0x5737	0x8FFD	# <CJK>
+0x5738	0x8D58	# <CJK>
+0x5739	0x5760	# <CJK>
+0x573A	0x7F00	# <CJK>
+0x573B	0x8C06	# <CJK>
+0x573C	0x51C6	# <CJK>
+0x573D	0x6349	# <CJK>
+0x573E	0x62D9	# <CJK>
+0x573F	0x5353	# <CJK>
+0x5740	0x684C	# <CJK>
+0x5741	0x7422	# <CJK>
+0x5742	0x8301	# <CJK>
+0x5743	0x914C	# <CJK>
+0x5744	0x5544	# <CJK>
+0x5745	0x7740	# <CJK>
+0x5746	0x707C	# <CJK>
+0x5747	0x6D4A	# <CJK>
+0x5748	0x5179	# <CJK>
+0x5749	0x54A8	# <CJK>
+0x574A	0x8D44	# <CJK>
+0x574B	0x59FF	# <CJK>
+0x574C	0x6ECB	# <CJK>
+0x574D	0x6DC4	# <CJK>
+0x574E	0x5B5C	# <CJK>
+0x574F	0x7D2B	# <CJK>
+0x5750	0x4ED4	# <CJK>
+0x5751	0x7C7D	# <CJK>
+0x5752	0x6ED3	# <CJK>
+0x5753	0x5B50	# <CJK>
+0x5754	0x81EA	# <CJK>
+0x5755	0x6E0D	# <CJK>
+0x5756	0x5B57	# <CJK>
+0x5757	0x9B03	# <CJK>
+0x5758	0x68D5	# <CJK>
+0x5759	0x8E2A	# <CJK>
+0x575A	0x5B97	# <CJK>
+0x575B	0x7EFC	# <CJK>
+0x575C	0x603B	# <CJK>
+0x575D	0x7EB5	# <CJK>
+0x575E	0x90B9	# <CJK>
+0x575F	0x8D70	# <CJK>
+0x5760	0x594F	# <CJK>
+0x5761	0x63CD	# <CJK>
+0x5762	0x79DF	# <CJK>
+0x5763	0x8DB3	# <CJK>
+0x5764	0x5352	# <CJK>
+0x5765	0x65CF	# <CJK>
+0x5766	0x7956	# <CJK>
+0x5767	0x8BC5	# <CJK>
+0x5768	0x963B	# <CJK>
+0x5769	0x7EC4	# <CJK>
+0x576A	0x94BB	# <CJK>
+0x576B	0x7E82	# <CJK>
+0x576C	0x5634	# <CJK>
+0x576D	0x9189	# <CJK>
+0x576E	0x6700	# <CJK>
+0x576F	0x7F6A	# <CJK>
+0x5770	0x5C0A	# <CJK>
+0x5771	0x9075	# <CJK>
+0x5772	0x6628	# <CJK>
+0x5773	0x5DE6	# <CJK>
+0x5774	0x4F50	# <CJK>
+0x5775	0x67DE	# <CJK>
+0x5776	0x505A	# <CJK>
+0x5777	0x4F5C	# <CJK>
+0x5778	0x5750	# <CJK>
+0x5779	0x5EA7	# <CJK>
+0x5821	0x4E8D	# <CJK>
+0x5822	0x4E0C	# <CJK>
+0x5823	0x5140	# <CJK>
+0x5824	0x4E10	# <CJK>
+0x5825	0x5EFF	# <CJK>
+0x5826	0x5345	# <CJK>
+0x5827	0x4E15	# <CJK>
+0x5828	0x4E98	# <CJK>
+0x5829	0x4E1E	# <CJK>
+0x582A	0x9B32	# <CJK>
+0x582B	0x5B6C	# <CJK>
+0x582C	0x5669	# <CJK>
+0x582D	0x4E28	# <CJK>
+0x582E	0x79BA	# <CJK>
+0x582F	0x4E3F	# <CJK>
+0x5830	0x5315	# <CJK>
+0x5831	0x4E47	# <CJK>
+0x5832	0x592D	# <CJK>
+0x5833	0x723B	# <CJK>
+0x5834	0x536E	# <CJK>
+0x5835	0x6C10	# <CJK>
+0x5836	0x56DF	# <CJK>
+0x5837	0x80E4	# <CJK>
+0x5838	0x9997	# <CJK>
+0x5839	0x6BD3	# <CJK>
+0x583A	0x777E	# <CJK>
+0x583B	0x9F17	# <CJK>
+0x583C	0x4E36	# <CJK>
+0x583D	0x4E9F	# <CJK>
+0x583E	0x9F10	# <CJK>
+0x583F	0x4E5C	# <CJK>
+0x5840	0x4E69	# <CJK>
+0x5841	0x4E93	# <CJK>
+0x5842	0x8288	# <CJK>
+0x5843	0x5B5B	# <CJK>
+0x5844	0x556C	# <CJK>
+0x5845	0x560F	# <CJK>
+0x5846	0x4EC4	# <CJK>
+0x5847	0x538D	# <CJK>
+0x5848	0x539D	# <CJK>
+0x5849	0x53A3	# <CJK>
+0x584A	0x53A5	# <CJK>
+0x584B	0x53AE	# <CJK>
+0x584C	0x9765	# <CJK>
+0x584D	0x8D5D	# <CJK>
+0x584E	0x531A	# <CJK>
+0x584F	0x53F5	# <CJK>
+0x5850	0x5326	# <CJK>
+0x5851	0x532E	# <CJK>
+0x5852	0x533E	# <CJK>
+0x5853	0x8D5C	# <CJK>
+0x5854	0x5366	# <CJK>
+0x5855	0x5363	# <CJK>
+0x5856	0x5202	# <CJK>
+0x5857	0x5208	# <CJK>
+0x5858	0x520E	# <CJK>
+0x5859	0x522D	# <CJK>
+0x585A	0x5233	# <CJK>
+0x585B	0x523F	# <CJK>
+0x585C	0x5240	# <CJK>
+0x585D	0x524C	# <CJK>
+0x585E	0x525E	# <CJK>
+0x585F	0x5261	# <CJK>
+0x5860	0x525C	# <CJK>
+0x5861	0x84AF	# <CJK>
+0x5862	0x527D	# <CJK>
+0x5863	0x5282	# <CJK>
+0x5864	0x5281	# <CJK>
+0x5865	0x5290	# <CJK>
+0x5866	0x5293	# <CJK>
+0x5867	0x5182	# <CJK>
+0x5868	0x7F54	# <CJK>
+0x5869	0x4EBB	# <CJK>
+0x586A	0x4EC3	# <CJK>
+0x586B	0x4EC9	# <CJK>
+0x586C	0x4EC2	# <CJK>
+0x586D	0x4EE8	# <CJK>
+0x586E	0x4EE1	# <CJK>
+0x586F	0x4EEB	# <CJK>
+0x5870	0x4EDE	# <CJK>
+0x5871	0x4F1B	# <CJK>
+0x5872	0x4EF3	# <CJK>
+0x5873	0x4F22	# <CJK>
+0x5874	0x4F64	# <CJK>
+0x5875	0x4EF5	# <CJK>
+0x5876	0x4F25	# <CJK>
+0x5877	0x4F27	# <CJK>
+0x5878	0x4F09	# <CJK>
+0x5879	0x4F2B	# <CJK>
+0x587A	0x4F5E	# <CJK>
+0x587B	0x4F67	# <CJK>
+0x587C	0x6538	# <CJK>
+0x587D	0x4F5A	# <CJK>
+0x587E	0x4F5D	# <CJK>
+0x5921	0x4F5F	# <CJK>
+0x5922	0x4F57	# <CJK>
+0x5923	0x4F32	# <CJK>
+0x5924	0x4F3D	# <CJK>
+0x5925	0x4F76	# <CJK>
+0x5926	0x4F74	# <CJK>
+0x5927	0x4F91	# <CJK>
+0x5928	0x4F89	# <CJK>
+0x5929	0x4F83	# <CJK>
+0x592A	0x4F8F	# <CJK>
+0x592B	0x4F7E	# <CJK>
+0x592C	0x4F7B	# <CJK>
+0x592D	0x4FAA	# <CJK>
+0x592E	0x4F7C	# <CJK>
+0x592F	0x4FAC	# <CJK>
+0x5930	0x4F94	# <CJK>
+0x5931	0x4FE6	# <CJK>
+0x5932	0x4FE8	# <CJK>
+0x5933	0x4FEA	# <CJK>
+0x5934	0x4FC5	# <CJK>
+0x5935	0x4FDA	# <CJK>
+0x5936	0x4FE3	# <CJK>
+0x5937	0x4FDC	# <CJK>
+0x5938	0x4FD1	# <CJK>
+0x5939	0x4FDF	# <CJK>
+0x593A	0x4FF8	# <CJK>
+0x593B	0x5029	# <CJK>
+0x593C	0x504C	# <CJK>
+0x593D	0x4FF3	# <CJK>
+0x593E	0x502C	# <CJK>
+0x593F	0x500F	# <CJK>
+0x5940	0x502E	# <CJK>
+0x5941	0x502D	# <CJK>
+0x5942	0x4FFE	# <CJK>
+0x5943	0x501C	# <CJK>
+0x5944	0x500C	# <CJK>
+0x5945	0x5025	# <CJK>
+0x5946	0x5028	# <CJK>
+0x5947	0x507E	# <CJK>
+0x5948	0x5043	# <CJK>
+0x5949	0x5055	# <CJK>
+0x594A	0x5048	# <CJK>
+0x594B	0x504E	# <CJK>
+0x594C	0x506C	# <CJK>
+0x594D	0x507B	# <CJK>
+0x594E	0x50A5	# <CJK>
+0x594F	0x50A7	# <CJK>
+0x5950	0x50A9	# <CJK>
+0x5951	0x50BA	# <CJK>
+0x5952	0x50D6	# <CJK>
+0x5953	0x5106	# <CJK>
+0x5954	0x50ED	# <CJK>
+0x5955	0x50EC	# <CJK>
+0x5956	0x50E6	# <CJK>
+0x5957	0x50EE	# <CJK>
+0x5958	0x5107	# <CJK>
+0x5959	0x510B	# <CJK>
+0x595A	0x4EDD	# <CJK>
+0x595B	0x6C3D	# <CJK>
+0x595C	0x4F58	# <CJK>
+0x595D	0x4F65	# <CJK>
+0x595E	0x4FCE	# <CJK>
+0x595F	0x9FA0	# <CJK>
+0x5960	0x6C46	# <CJK>
+0x5961	0x7C74	# <CJK>
+0x5962	0x516E	# <CJK>
+0x5963	0x5DFD	# <CJK>
+0x5964	0x9EC9	# <CJK>
+0x5965	0x9998	# <CJK>
+0x5966	0x5181	# <CJK>
+0x5967	0x5914	# <CJK>
+0x5968	0x52F9	# <CJK>
+0x5969	0x530D	# <CJK>
+0x596A	0x8A07	# <CJK>
+0x596B	0x5310	# <CJK>
+0x596C	0x51EB	# <CJK>
+0x596D	0x5919	# <CJK>
+0x596E	0x5155	# <CJK>
+0x596F	0x4EA0	# <CJK>
+0x5970	0x5156	# <CJK>
+0x5971	0x4EB3	# <CJK>
+0x5972	0x886E	# <CJK>
+0x5973	0x88A4	# <CJK>
+0x5974	0x4EB5	# <CJK>
+0x5975	0x8114	# <CJK>
+0x5976	0x88D2	# <CJK>
+0x5977	0x7980	# <CJK>
+0x5978	0x5B34	# <CJK>
+0x5979	0x8803	# <CJK>
+0x597A	0x7FB8	# <CJK>
+0x597B	0x51AB	# <CJK>
+0x597C	0x51B1	# <CJK>
+0x597D	0x51BD	# <CJK>
+0x597E	0x51BC	# <CJK>
+0x5A21	0x51C7	# <CJK>
+0x5A22	0x5196	# <CJK>
+0x5A23	0x51A2	# <CJK>
+0x5A24	0x51A5	# <CJK>
+0x5A25	0x8BA0	# <CJK>
+0x5A26	0x8BA6	# <CJK>
+0x5A27	0x8BA7	# <CJK>
+0x5A28	0x8BAA	# <CJK>
+0x5A29	0x8BB4	# <CJK>
+0x5A2A	0x8BB5	# <CJK>
+0x5A2B	0x8BB7	# <CJK>
+0x5A2C	0x8BC2	# <CJK>
+0x5A2D	0x8BC3	# <CJK>
+0x5A2E	0x8BCB	# <CJK>
+0x5A2F	0x8BCF	# <CJK>
+0x5A30	0x8BCE	# <CJK>
+0x5A31	0x8BD2	# <CJK>
+0x5A32	0x8BD3	# <CJK>
+0x5A33	0x8BD4	# <CJK>
+0x5A34	0x8BD6	# <CJK>
+0x5A35	0x8BD8	# <CJK>
+0x5A36	0x8BD9	# <CJK>
+0x5A37	0x8BDC	# <CJK>
+0x5A38	0x8BDF	# <CJK>
+0x5A39	0x8BE0	# <CJK>
+0x5A3A	0x8BE4	# <CJK>
+0x5A3B	0x8BE8	# <CJK>
+0x5A3C	0x8BE9	# <CJK>
+0x5A3D	0x8BEE	# <CJK>
+0x5A3E	0x8BF0	# <CJK>
+0x5A3F	0x8BF3	# <CJK>
+0x5A40	0x8BF6	# <CJK>
+0x5A41	0x8BF9	# <CJK>
+0x5A42	0x8BFC	# <CJK>
+0x5A43	0x8BFF	# <CJK>
+0x5A44	0x8C00	# <CJK>
+0x5A45	0x8C02	# <CJK>
+0x5A46	0x8C04	# <CJK>
+0x5A47	0x8C07	# <CJK>
+0x5A48	0x8C0C	# <CJK>
+0x5A49	0x8C0F	# <CJK>
+0x5A4A	0x8C11	# <CJK>
+0x5A4B	0x8C12	# <CJK>
+0x5A4C	0x8C14	# <CJK>
+0x5A4D	0x8C15	# <CJK>
+0x5A4E	0x8C16	# <CJK>
+0x5A4F	0x8C19	# <CJK>
+0x5A50	0x8C1B	# <CJK>
+0x5A51	0x8C18	# <CJK>
+0x5A52	0x8C1D	# <CJK>
+0x5A53	0x8C1F	# <CJK>
+0x5A54	0x8C20	# <CJK>
+0x5A55	0x8C21	# <CJK>
+0x5A56	0x8C25	# <CJK>
+0x5A57	0x8C27	# <CJK>
+0x5A58	0x8C2A	# <CJK>
+0x5A59	0x8C2B	# <CJK>
+0x5A5A	0x8C2E	# <CJK>
+0x5A5B	0x8C2F	# <CJK>
+0x5A5C	0x8C32	# <CJK>
+0x5A5D	0x8C33	# <CJK>
+0x5A5E	0x8C35	# <CJK>
+0x5A5F	0x8C36	# <CJK>
+0x5A60	0x5369	# <CJK>
+0x5A61	0x537A	# <CJK>
+0x5A62	0x961D	# <CJK>
+0x5A63	0x9622	# <CJK>
+0x5A64	0x9621	# <CJK>
+0x5A65	0x9631	# <CJK>
+0x5A66	0x962A	# <CJK>
+0x5A67	0x963D	# <CJK>
+0x5A68	0x963C	# <CJK>
+0x5A69	0x9642	# <CJK>
+0x5A6A	0x9649	# <CJK>
+0x5A6B	0x9654	# <CJK>
+0x5A6C	0x965F	# <CJK>
+0x5A6D	0x9667	# <CJK>
+0x5A6E	0x966C	# <CJK>
+0x5A6F	0x9672	# <CJK>
+0x5A70	0x9674	# <CJK>
+0x5A71	0x9688	# <CJK>
+0x5A72	0x968D	# <CJK>
+0x5A73	0x9697	# <CJK>
+0x5A74	0x96B0	# <CJK>
+0x5A75	0x9097	# <CJK>
+0x5A76	0x909B	# <CJK>
+0x5A77	0x909D	# <CJK>
+0x5A78	0x9099	# <CJK>
+0x5A79	0x90AC	# <CJK>
+0x5A7A	0x90A1	# <CJK>
+0x5A7B	0x90B4	# <CJK>
+0x5A7C	0x90B3	# <CJK>
+0x5A7D	0x90B6	# <CJK>
+0x5A7E	0x90BA	# <CJK>
+0x5B21	0x90B8	# <CJK>
+0x5B22	0x90B0	# <CJK>
+0x5B23	0x90CF	# <CJK>
+0x5B24	0x90C5	# <CJK>
+0x5B25	0x90BE	# <CJK>
+0x5B26	0x90D0	# <CJK>
+0x5B27	0x90C4	# <CJK>
+0x5B28	0x90C7	# <CJK>
+0x5B29	0x90D3	# <CJK>
+0x5B2A	0x90E6	# <CJK>
+0x5B2B	0x90E2	# <CJK>
+0x5B2C	0x90DC	# <CJK>
+0x5B2D	0x90D7	# <CJK>
+0x5B2E	0x90DB	# <CJK>
+0x5B2F	0x90EB	# <CJK>
+0x5B30	0x90EF	# <CJK>
+0x5B31	0x90FE	# <CJK>
+0x5B32	0x9104	# <CJK>
+0x5B33	0x9122	# <CJK>
+0x5B34	0x911E	# <CJK>
+0x5B35	0x9123	# <CJK>
+0x5B36	0x9131	# <CJK>
+0x5B37	0x912F	# <CJK>
+0x5B38	0x9139	# <CJK>
+0x5B39	0x9143	# <CJK>
+0x5B3A	0x9146	# <CJK>
+0x5B3B	0x520D	# <CJK>
+0x5B3C	0x5942	# <CJK>
+0x5B3D	0x52A2	# <CJK>
+0x5B3E	0x52AC	# <CJK>
+0x5B3F	0x52AD	# <CJK>
+0x5B40	0x52BE	# <CJK>
+0x5B41	0x54FF	# <CJK>
+0x5B42	0x52D0	# <CJK>
+0x5B43	0x52D6	# <CJK>
+0x5B44	0x52F0	# <CJK>
+0x5B45	0x53DF	# <CJK>
+0x5B46	0x71EE	# <CJK>
+0x5B47	0x77CD	# <CJK>
+0x5B48	0x5EF4	# <CJK>
+0x5B49	0x51F5	# <CJK>
+0x5B4A	0x51FC	# <CJK>
+0x5B4B	0x9B2F	# <CJK>
+0x5B4C	0x53B6	# <CJK>
+0x5B4D	0x5F01	# <CJK>
+0x5B4E	0x755A	# <CJK>
+0x5B4F	0x5DEF	# <CJK>
+0x5B50	0x574C	# <CJK>
+0x5B51	0x57A9	# <CJK>
+0x5B52	0x57A1	# <CJK>
+0x5B53	0x587E	# <CJK>
+0x5B54	0x58BC	# <CJK>
+0x5B55	0x58C5	# <CJK>
+0x5B56	0x58D1	# <CJK>
+0x5B57	0x5729	# <CJK>
+0x5B58	0x572C	# <CJK>
+0x5B59	0x572A	# <CJK>
+0x5B5A	0x5733	# <CJK>
+0x5B5B	0x5739	# <CJK>
+0x5B5C	0x572E	# <CJK>
+0x5B5D	0x572F	# <CJK>
+0x5B5E	0x575C	# <CJK>
+0x5B5F	0x573B	# <CJK>
+0x5B60	0x5742	# <CJK>
+0x5B61	0x5769	# <CJK>
+0x5B62	0x5785	# <CJK>
+0x5B63	0x576B	# <CJK>
+0x5B64	0x5786	# <CJK>
+0x5B65	0x577C	# <CJK>
+0x5B66	0x577B	# <CJK>
+0x5B67	0x5768	# <CJK>
+0x5B68	0x576D	# <CJK>
+0x5B69	0x5776	# <CJK>
+0x5B6A	0x5773	# <CJK>
+0x5B6B	0x57AD	# <CJK>
+0x5B6C	0x57A4	# <CJK>
+0x5B6D	0x578C	# <CJK>
+0x5B6E	0x57B2	# <CJK>
+0x5B6F	0x57CF	# <CJK>
+0x5B70	0x57A7	# <CJK>
+0x5B71	0x57B4	# <CJK>
+0x5B72	0x5793	# <CJK>
+0x5B73	0x57A0	# <CJK>
+0x5B74	0x57D5	# <CJK>
+0x5B75	0x57D8	# <CJK>
+0x5B76	0x57DA	# <CJK>
+0x5B77	0x57D9	# <CJK>
+0x5B78	0x57D2	# <CJK>
+0x5B79	0x57B8	# <CJK>
+0x5B7A	0x57F4	# <CJK>
+0x5B7B	0x57EF	# <CJK>
+0x5B7C	0x57F8	# <CJK>
+0x5B7D	0x57E4	# <CJK>
+0x5B7E	0x57DD	# <CJK>
+0x5C21	0x580B	# <CJK>
+0x5C22	0x580D	# <CJK>
+0x5C23	0x57FD	# <CJK>
+0x5C24	0x57ED	# <CJK>
+0x5C25	0x5800	# <CJK>
+0x5C26	0x581E	# <CJK>
+0x5C27	0x5819	# <CJK>
+0x5C28	0x5844	# <CJK>
+0x5C29	0x5820	# <CJK>
+0x5C2A	0x5865	# <CJK>
+0x5C2B	0x586C	# <CJK>
+0x5C2C	0x5881	# <CJK>
+0x5C2D	0x5889	# <CJK>
+0x5C2E	0x589A	# <CJK>
+0x5C2F	0x5880	# <CJK>
+0x5C30	0x99A8	# <CJK>
+0x5C31	0x9F19	# <CJK>
+0x5C32	0x61FF	# <CJK>
+0x5C33	0x8279	# <CJK>
+0x5C34	0x827D	# <CJK>
+0x5C35	0x827F	# <CJK>
+0x5C36	0x828F	# <CJK>
+0x5C37	0x828A	# <CJK>
+0x5C38	0x82A8	# <CJK>
+0x5C39	0x8284	# <CJK>
+0x5C3A	0x828E	# <CJK>
+0x5C3B	0x8291	# <CJK>
+0x5C3C	0x8297	# <CJK>
+0x5C3D	0x8299	# <CJK>
+0x5C3E	0x82AB	# <CJK>
+0x5C3F	0x82B8	# <CJK>
+0x5C40	0x82BE	# <CJK>
+0x5C41	0x82B0	# <CJK>
+0x5C42	0x82C8	# <CJK>
+0x5C43	0x82CA	# <CJK>
+0x5C44	0x82E3	# <CJK>
+0x5C45	0x8298	# <CJK>
+0x5C46	0x82B7	# <CJK>
+0x5C47	0x82AE	# <CJK>
+0x5C48	0x82CB	# <CJK>
+0x5C49	0x82CC	# <CJK>
+0x5C4A	0x82C1	# <CJK>
+0x5C4B	0x82A9	# <CJK>
+0x5C4C	0x82B4	# <CJK>
+0x5C4D	0x82A1	# <CJK>
+0x5C4E	0x82AA	# <CJK>
+0x5C4F	0x829F	# <CJK>
+0x5C50	0x82C4	# <CJK>
+0x5C51	0x82CE	# <CJK>
+0x5C52	0x82A4	# <CJK>
+0x5C53	0x82E1	# <CJK>
+0x5C54	0x8309	# <CJK>
+0x5C55	0x82F7	# <CJK>
+0x5C56	0x82E4	# <CJK>
+0x5C57	0x830F	# <CJK>
+0x5C58	0x8307	# <CJK>
+0x5C59	0x82DC	# <CJK>
+0x5C5A	0x82F4	# <CJK>
+0x5C5B	0x82D2	# <CJK>
+0x5C5C	0x82D8	# <CJK>
+0x5C5D	0x830C	# <CJK>
+0x5C5E	0x82FB	# <CJK>
+0x5C5F	0x82D3	# <CJK>
+0x5C60	0x8311	# <CJK>
+0x5C61	0x831A	# <CJK>
+0x5C62	0x8306	# <CJK>
+0x5C63	0x8314	# <CJK>
+0x5C64	0x8315	# <CJK>
+0x5C65	0x82E0	# <CJK>
+0x5C66	0x82D5	# <CJK>
+0x5C67	0x831C	# <CJK>
+0x5C68	0x8351	# <CJK>
+0x5C69	0x835B	# <CJK>
+0x5C6A	0x835C	# <CJK>
+0x5C6B	0x8308	# <CJK>
+0x5C6C	0x8392	# <CJK>
+0x5C6D	0x833C	# <CJK>
+0x5C6E	0x8334	# <CJK>
+0x5C6F	0x8331	# <CJK>
+0x5C70	0x839B	# <CJK>
+0x5C71	0x835E	# <CJK>
+0x5C72	0x832F	# <CJK>
+0x5C73	0x834F	# <CJK>
+0x5C74	0x8347	# <CJK>
+0x5C75	0x8343	# <CJK>
+0x5C76	0x835F	# <CJK>
+0x5C77	0x8340	# <CJK>
+0x5C78	0x8317	# <CJK>
+0x5C79	0x8360	# <CJK>
+0x5C7A	0x832D	# <CJK>
+0x5C7B	0x833A	# <CJK>
+0x5C7C	0x8333	# <CJK>
+0x5C7D	0x8366	# <CJK>
+0x5C7E	0x8365	# <CJK>
+0x5D21	0x8368	# <CJK>
+0x5D22	0x831B	# <CJK>
+0x5D23	0x8369	# <CJK>
+0x5D24	0x836C	# <CJK>
+0x5D25	0x836A	# <CJK>
+0x5D26	0x836D	# <CJK>
+0x5D27	0x836E	# <CJK>
+0x5D28	0x83B0	# <CJK>
+0x5D29	0x8378	# <CJK>
+0x5D2A	0x83B3	# <CJK>
+0x5D2B	0x83B4	# <CJK>
+0x5D2C	0x83A0	# <CJK>
+0x5D2D	0x83AA	# <CJK>
+0x5D2E	0x8393	# <CJK>
+0x5D2F	0x839C	# <CJK>
+0x5D30	0x8385	# <CJK>
+0x5D31	0x837C	# <CJK>
+0x5D32	0x83B6	# <CJK>
+0x5D33	0x83A9	# <CJK>
+0x5D34	0x837D	# <CJK>
+0x5D35	0x83B8	# <CJK>
+0x5D36	0x837B	# <CJK>
+0x5D37	0x8398	# <CJK>
+0x5D38	0x839E	# <CJK>
+0x5D39	0x83A8	# <CJK>
+0x5D3A	0x83BA	# <CJK>
+0x5D3B	0x83BC	# <CJK>
+0x5D3C	0x83C1	# <CJK>
+0x5D3D	0x8401	# <CJK>
+0x5D3E	0x83E5	# <CJK>
+0x5D3F	0x83D8	# <CJK>
+0x5D40	0x5807	# <CJK>
+0x5D41	0x8418	# <CJK>
+0x5D42	0x840B	# <CJK>
+0x5D43	0x83DD	# <CJK>
+0x5D44	0x83FD	# <CJK>
+0x5D45	0x83D6	# <CJK>
+0x5D46	0x841C	# <CJK>
+0x5D47	0x8438	# <CJK>
+0x5D48	0x8411	# <CJK>
+0x5D49	0x8406	# <CJK>
+0x5D4A	0x83D4	# <CJK>
+0x5D4B	0x83DF	# <CJK>
+0x5D4C	0x840F	# <CJK>
+0x5D4D	0x8403	# <CJK>
+0x5D4E	0x83F8	# <CJK>
+0x5D4F	0x83F9	# <CJK>
+0x5D50	0x83EA	# <CJK>
+0x5D51	0x83C5	# <CJK>
+0x5D52	0x83C0	# <CJK>
+0x5D53	0x8426	# <CJK>
+0x5D54	0x83F0	# <CJK>
+0x5D55	0x83E1	# <CJK>
+0x5D56	0x845C	# <CJK>
+0x5D57	0x8451	# <CJK>
+0x5D58	0x845A	# <CJK>
+0x5D59	0x8459	# <CJK>
+0x5D5A	0x8473	# <CJK>
+0x5D5B	0x8487	# <CJK>
+0x5D5C	0x8488	# <CJK>
+0x5D5D	0x847A	# <CJK>
+0x5D5E	0x8489	# <CJK>
+0x5D5F	0x8478	# <CJK>
+0x5D60	0x843C	# <CJK>
+0x5D61	0x8446	# <CJK>
+0x5D62	0x8469	# <CJK>
+0x5D63	0x8476	# <CJK>
+0x5D64	0x848C	# <CJK>
+0x5D65	0x848E	# <CJK>
+0x5D66	0x8431	# <CJK>
+0x5D67	0x846D	# <CJK>
+0x5D68	0x84C1	# <CJK>
+0x5D69	0x84CD	# <CJK>
+0x5D6A	0x84D0	# <CJK>
+0x5D6B	0x84E6	# <CJK>
+0x5D6C	0x84BD	# <CJK>
+0x5D6D	0x84D3	# <CJK>
+0x5D6E	0x84CA	# <CJK>
+0x5D6F	0x84BF	# <CJK>
+0x5D70	0x84BA	# <CJK>
+0x5D71	0x84E0	# <CJK>
+0x5D72	0x84A1	# <CJK>
+0x5D73	0x84B9	# <CJK>
+0x5D74	0x84B4	# <CJK>
+0x5D75	0x8497	# <CJK>
+0x5D76	0x84E5	# <CJK>
+0x5D77	0x84E3	# <CJK>
+0x5D78	0x850C	# <CJK>
+0x5D79	0x750D	# <CJK>
+0x5D7A	0x8538	# <CJK>
+0x5D7B	0x84F0	# <CJK>
+0x5D7C	0x8539	# <CJK>
+0x5D7D	0x851F	# <CJK>
+0x5D7E	0x853A	# <CJK>
+0x5E21	0x8556	# <CJK>
+0x5E22	0x853B	# <CJK>
+0x5E23	0x84FF	# <CJK>
+0x5E24	0x84FC	# <CJK>
+0x5E25	0x8559	# <CJK>
+0x5E26	0x8548	# <CJK>
+0x5E27	0x8568	# <CJK>
+0x5E28	0x8564	# <CJK>
+0x5E29	0x855E	# <CJK>
+0x5E2A	0x857A	# <CJK>
+0x5E2B	0x77A2	# <CJK>
+0x5E2C	0x8543	# <CJK>
+0x5E2D	0x8572	# <CJK>
+0x5E2E	0x857B	# <CJK>
+0x5E2F	0x85A4	# <CJK>
+0x5E30	0x85A8	# <CJK>
+0x5E31	0x8587	# <CJK>
+0x5E32	0x858F	# <CJK>
+0x5E33	0x8579	# <CJK>
+0x5E34	0x85AE	# <CJK>
+0x5E35	0x859C	# <CJK>
+0x5E36	0x8585	# <CJK>
+0x5E37	0x85B9	# <CJK>
+0x5E38	0x85B7	# <CJK>
+0x5E39	0x85B0	# <CJK>
+0x5E3A	0x85D3	# <CJK>
+0x5E3B	0x85C1	# <CJK>
+0x5E3C	0x85DC	# <CJK>
+0x5E3D	0x85FF	# <CJK>
+0x5E3E	0x8627	# <CJK>
+0x5E3F	0x8605	# <CJK>
+0x5E40	0x8629	# <CJK>
+0x5E41	0x8616	# <CJK>
+0x5E42	0x863C	# <CJK>
+0x5E43	0x5EFE	# <CJK>
+0x5E44	0x5F08	# <CJK>
+0x5E45	0x593C	# <CJK>
+0x5E46	0x5941	# <CJK>
+0x5E47	0x8037	# <CJK>
+0x5E48	0x5955	# <CJK>
+0x5E49	0x595A	# <CJK>
+0x5E4A	0x5958	# <CJK>
+0x5E4B	0x530F	# <CJK>
+0x5E4C	0x5C22	# <CJK>
+0x5E4D	0x5C25	# <CJK>
+0x5E4E	0x5C2C	# <CJK>
+0x5E4F	0x5C34	# <CJK>
+0x5E50	0x624C	# <CJK>
+0x5E51	0x626A	# <CJK>
+0x5E52	0x629F	# <CJK>
+0x5E53	0x62BB	# <CJK>
+0x5E54	0x62CA	# <CJK>
+0x5E55	0x62DA	# <CJK>
+0x5E56	0x62D7	# <CJK>
+0x5E57	0x62EE	# <CJK>
+0x5E58	0x6322	# <CJK>
+0x5E59	0x62F6	# <CJK>
+0x5E5A	0x6339	# <CJK>
+0x5E5B	0x634B	# <CJK>
+0x5E5C	0x6343	# <CJK>
+0x5E5D	0x63AD	# <CJK>
+0x5E5E	0x63F6	# <CJK>
+0x5E5F	0x6371	# <CJK>
+0x5E60	0x637A	# <CJK>
+0x5E61	0x638E	# <CJK>
+0x5E62	0x63B4	# <CJK>
+0x5E63	0x636D	# <CJK>
+0x5E64	0x63AC	# <CJK>
+0x5E65	0x638A	# <CJK>
+0x5E66	0x6369	# <CJK>
+0x5E67	0x63AE	# <CJK>
+0x5E68	0x63BC	# <CJK>
+0x5E69	0x63F2	# <CJK>
+0x5E6A	0x63F8	# <CJK>
+0x5E6B	0x63E0	# <CJK>
+0x5E6C	0x63FF	# <CJK>
+0x5E6D	0x63C4	# <CJK>
+0x5E6E	0x63DE	# <CJK>
+0x5E6F	0x63CE	# <CJK>
+0x5E70	0x6452	# <CJK>
+0x5E71	0x63C6	# <CJK>
+0x5E72	0x63BE	# <CJK>
+0x5E73	0x6445	# <CJK>
+0x5E74	0x6441	# <CJK>
+0x5E75	0x640B	# <CJK>
+0x5E76	0x641B	# <CJK>
+0x5E77	0x6420	# <CJK>
+0x5E78	0x640C	# <CJK>
+0x5E79	0x6426	# <CJK>
+0x5E7A	0x6421	# <CJK>
+0x5E7B	0x645E	# <CJK>
+0x5E7C	0x6484	# <CJK>
+0x5E7D	0x646D	# <CJK>
+0x5E7E	0x6496	# <CJK>
+0x5F21	0x647A	# <CJK>
+0x5F22	0x64B7	# <CJK>
+0x5F23	0x64B8	# <CJK>
+0x5F24	0x6499	# <CJK>
+0x5F25	0x64BA	# <CJK>
+0x5F26	0x64C0	# <CJK>
+0x5F27	0x64D0	# <CJK>
+0x5F28	0x64D7	# <CJK>
+0x5F29	0x64E4	# <CJK>
+0x5F2A	0x64E2	# <CJK>
+0x5F2B	0x6509	# <CJK>
+0x5F2C	0x6525	# <CJK>
+0x5F2D	0x652E	# <CJK>
+0x5F2E	0x5F0B	# <CJK>
+0x5F2F	0x5FD2	# <CJK>
+0x5F30	0x7519	# <CJK>
+0x5F31	0x5F11	# <CJK>
+0x5F32	0x535F	# <CJK>
+0x5F33	0x53F1	# <CJK>
+0x5F34	0x53FD	# <CJK>
+0x5F35	0x53E9	# <CJK>
+0x5F36	0x53E8	# <CJK>
+0x5F37	0x53FB	# <CJK>
+0x5F38	0x5412	# <CJK>
+0x5F39	0x5416	# <CJK>
+0x5F3A	0x5406	# <CJK>
+0x5F3B	0x544B	# <CJK>
+0x5F3C	0x5452	# <CJK>
+0x5F3D	0x5453	# <CJK>
+0x5F3E	0x5454	# <CJK>
+0x5F3F	0x5456	# <CJK>
+0x5F40	0x5443	# <CJK>
+0x5F41	0x5421	# <CJK>
+0x5F42	0x5457	# <CJK>
+0x5F43	0x5459	# <CJK>
+0x5F44	0x5423	# <CJK>
+0x5F45	0x5432	# <CJK>
+0x5F46	0x5482	# <CJK>
+0x5F47	0x5494	# <CJK>
+0x5F48	0x5477	# <CJK>
+0x5F49	0x5471	# <CJK>
+0x5F4A	0x5464	# <CJK>
+0x5F4B	0x549A	# <CJK>
+0x5F4C	0x549B	# <CJK>
+0x5F4D	0x5484	# <CJK>
+0x5F4E	0x5476	# <CJK>
+0x5F4F	0x5466	# <CJK>
+0x5F50	0x549D	# <CJK>
+0x5F51	0x54D0	# <CJK>
+0x5F52	0x54AD	# <CJK>
+0x5F53	0x54C2	# <CJK>
+0x5F54	0x54B4	# <CJK>
+0x5F55	0x54D2	# <CJK>
+0x5F56	0x54A7	# <CJK>
+0x5F57	0x54A6	# <CJK>
+0x5F58	0x54D3	# <CJK>
+0x5F59	0x54D4	# <CJK>
+0x5F5A	0x5472	# <CJK>
+0x5F5B	0x54A3	# <CJK>
+0x5F5C	0x54D5	# <CJK>
+0x5F5D	0x54BB	# <CJK>
+0x5F5E	0x54BF	# <CJK>
+0x5F5F	0x54CC	# <CJK>
+0x5F60	0x54D9	# <CJK>
+0x5F61	0x54DA	# <CJK>
+0x5F62	0x54DC	# <CJK>
+0x5F63	0x54A9	# <CJK>
+0x5F64	0x54AA	# <CJK>
+0x5F65	0x54A4	# <CJK>
+0x5F66	0x54DD	# <CJK>
+0x5F67	0x54CF	# <CJK>
+0x5F68	0x54DE	# <CJK>
+0x5F69	0x551B	# <CJK>
+0x5F6A	0x54E7	# <CJK>
+0x5F6B	0x5520	# <CJK>
+0x5F6C	0x54FD	# <CJK>
+0x5F6D	0x5514	# <CJK>
+0x5F6E	0x54F3	# <CJK>
+0x5F6F	0x5522	# <CJK>
+0x5F70	0x5523	# <CJK>
+0x5F71	0x550F	# <CJK>
+0x5F72	0x5511	# <CJK>
+0x5F73	0x5527	# <CJK>
+0x5F74	0x552A	# <CJK>
+0x5F75	0x5567	# <CJK>
+0x5F76	0x558F	# <CJK>
+0x5F77	0x55B5	# <CJK>
+0x5F78	0x5549	# <CJK>
+0x5F79	0x556D	# <CJK>
+0x5F7A	0x5541	# <CJK>
+0x5F7B	0x5555	# <CJK>
+0x5F7C	0x553F	# <CJK>
+0x5F7D	0x5550	# <CJK>
+0x5F7E	0x553C	# <CJK>
+0x6021	0x5537	# <CJK>
+0x6022	0x5556	# <CJK>
+0x6023	0x5575	# <CJK>
+0x6024	0x5576	# <CJK>
+0x6025	0x5577	# <CJK>
+0x6026	0x5533	# <CJK>
+0x6027	0x5530	# <CJK>
+0x6028	0x555C	# <CJK>
+0x6029	0x558B	# <CJK>
+0x602A	0x55D2	# <CJK>
+0x602B	0x5583	# <CJK>
+0x602C	0x55B1	# <CJK>
+0x602D	0x55B9	# <CJK>
+0x602E	0x5588	# <CJK>
+0x602F	0x5581	# <CJK>
+0x6030	0x559F	# <CJK>
+0x6031	0x557E	# <CJK>
+0x6032	0x55D6	# <CJK>
+0x6033	0x5591	# <CJK>
+0x6034	0x557B	# <CJK>
+0x6035	0x55DF	# <CJK>
+0x6036	0x55BD	# <CJK>
+0x6037	0x55BE	# <CJK>
+0x6038	0x5594	# <CJK>
+0x6039	0x5599	# <CJK>
+0x603A	0x55EA	# <CJK>
+0x603B	0x55F7	# <CJK>
+0x603C	0x55C9	# <CJK>
+0x603D	0x561F	# <CJK>
+0x603E	0x55D1	# <CJK>
+0x603F	0x55EB	# <CJK>
+0x6040	0x55EC	# <CJK>
+0x6041	0x55D4	# <CJK>
+0x6042	0x55E6	# <CJK>
+0x6043	0x55DD	# <CJK>
+0x6044	0x55C4	# <CJK>
+0x6045	0x55EF	# <CJK>
+0x6046	0x55E5	# <CJK>
+0x6047	0x55F2	# <CJK>
+0x6048	0x55F3	# <CJK>
+0x6049	0x55CC	# <CJK>
+0x604A	0x55CD	# <CJK>
+0x604B	0x55E8	# <CJK>
+0x604C	0x55F5	# <CJK>
+0x604D	0x55E4	# <CJK>
+0x604E	0x8F94	# <CJK>
+0x604F	0x561E	# <CJK>
+0x6050	0x5608	# <CJK>
+0x6051	0x560C	# <CJK>
+0x6052	0x5601	# <CJK>
+0x6053	0x5624	# <CJK>
+0x6054	0x5623	# <CJK>
+0x6055	0x55FE	# <CJK>
+0x6056	0x5600	# <CJK>
+0x6057	0x5627	# <CJK>
+0x6058	0x562D	# <CJK>
+0x6059	0x5658	# <CJK>
+0x605A	0x5639	# <CJK>
+0x605B	0x5657	# <CJK>
+0x605C	0x562C	# <CJK>
+0x605D	0x564D	# <CJK>
+0x605E	0x5662	# <CJK>
+0x605F	0x5659	# <CJK>
+0x6060	0x565C	# <CJK>
+0x6061	0x564C	# <CJK>
+0x6062	0x5654	# <CJK>
+0x6063	0x5686	# <CJK>
+0x6064	0x5664	# <CJK>
+0x6065	0x5671	# <CJK>
+0x6066	0x566B	# <CJK>
+0x6067	0x567B	# <CJK>
+0x6068	0x567C	# <CJK>
+0x6069	0x5685	# <CJK>
+0x606A	0x5693	# <CJK>
+0x606B	0x56AF	# <CJK>
+0x606C	0x56D4	# <CJK>
+0x606D	0x56D7	# <CJK>
+0x606E	0x56DD	# <CJK>
+0x606F	0x56E1	# <CJK>
+0x6070	0x56F5	# <CJK>
+0x6071	0x56EB	# <CJK>
+0x6072	0x56F9	# <CJK>
+0x6073	0x56FF	# <CJK>
+0x6074	0x5704	# <CJK>
+0x6075	0x570A	# <CJK>
+0x6076	0x5709	# <CJK>
+0x6077	0x571C	# <CJK>
+0x6078	0x5E0F	# <CJK>
+0x6079	0x5E19	# <CJK>
+0x607A	0x5E14	# <CJK>
+0x607B	0x5E11	# <CJK>
+0x607C	0x5E31	# <CJK>
+0x607D	0x5E3B	# <CJK>
+0x607E	0x5E3C	# <CJK>
+0x6121	0x5E37	# <CJK>
+0x6122	0x5E44	# <CJK>
+0x6123	0x5E54	# <CJK>
+0x6124	0x5E5B	# <CJK>
+0x6125	0x5E5E	# <CJK>
+0x6126	0x5E61	# <CJK>
+0x6127	0x5C8C	# <CJK>
+0x6128	0x5C7A	# <CJK>
+0x6129	0x5C8D	# <CJK>
+0x612A	0x5C90	# <CJK>
+0x612B	0x5C96	# <CJK>
+0x612C	0x5C88	# <CJK>
+0x612D	0x5C98	# <CJK>
+0x612E	0x5C99	# <CJK>
+0x612F	0x5C91	# <CJK>
+0x6130	0x5C9A	# <CJK>
+0x6131	0x5C9C	# <CJK>
+0x6132	0x5CB5	# <CJK>
+0x6133	0x5CA2	# <CJK>
+0x6134	0x5CBD	# <CJK>
+0x6135	0x5CAC	# <CJK>
+0x6136	0x5CAB	# <CJK>
+0x6137	0x5CB1	# <CJK>
+0x6138	0x5CA3	# <CJK>
+0x6139	0x5CC1	# <CJK>
+0x613A	0x5CB7	# <CJK>
+0x613B	0x5CC4	# <CJK>
+0x613C	0x5CD2	# <CJK>
+0x613D	0x5CE4	# <CJK>
+0x613E	0x5CCB	# <CJK>
+0x613F	0x5CE5	# <CJK>
+0x6140	0x5D02	# <CJK>
+0x6141	0x5D03	# <CJK>
+0x6142	0x5D27	# <CJK>
+0x6143	0x5D26	# <CJK>
+0x6144	0x5D2E	# <CJK>
+0x6145	0x5D24	# <CJK>
+0x6146	0x5D1E	# <CJK>
+0x6147	0x5D06	# <CJK>
+0x6148	0x5D1B	# <CJK>
+0x6149	0x5D58	# <CJK>
+0x614A	0x5D3E	# <CJK>
+0x614B	0x5D34	# <CJK>
+0x614C	0x5D3D	# <CJK>
+0x614D	0x5D6C	# <CJK>
+0x614E	0x5D5B	# <CJK>
+0x614F	0x5D6F	# <CJK>
+0x6150	0x5D5D	# <CJK>
+0x6151	0x5D6B	# <CJK>
+0x6152	0x5D4B	# <CJK>
+0x6153	0x5D4A	# <CJK>
+0x6154	0x5D69	# <CJK>
+0x6155	0x5D74	# <CJK>
+0x6156	0x5D82	# <CJK>
+0x6157	0x5D99	# <CJK>
+0x6158	0x5D9D	# <CJK>
+0x6159	0x8C73	# <CJK>
+0x615A	0x5DB7	# <CJK>
+0x615B	0x5DC5	# <CJK>
+0x615C	0x5F73	# <CJK>
+0x615D	0x5F77	# <CJK>
+0x615E	0x5F82	# <CJK>
+0x615F	0x5F87	# <CJK>
+0x6160	0x5F89	# <CJK>
+0x6161	0x5F8C	# <CJK>
+0x6162	0x5F95	# <CJK>
+0x6163	0x5F99	# <CJK>
+0x6164	0x5F9C	# <CJK>
+0x6165	0x5FA8	# <CJK>
+0x6166	0x5FAD	# <CJK>
+0x6167	0x5FB5	# <CJK>
+0x6168	0x5FBC	# <CJK>
+0x6169	0x8862	# <CJK>
+0x616A	0x5F61	# <CJK>
+0x616B	0x72AD	# <CJK>
+0x616C	0x72B0	# <CJK>
+0x616D	0x72B4	# <CJK>
+0x616E	0x72B7	# <CJK>
+0x616F	0x72B8	# <CJK>
+0x6170	0x72C3	# <CJK>
+0x6171	0x72C1	# <CJK>
+0x6172	0x72CE	# <CJK>
+0x6173	0x72CD	# <CJK>
+0x6174	0x72D2	# <CJK>
+0x6175	0x72E8	# <CJK>
+0x6176	0x72EF	# <CJK>
+0x6177	0x72E9	# <CJK>
+0x6178	0x72F2	# <CJK>
+0x6179	0x72F4	# <CJK>
+0x617A	0x72F7	# <CJK>
+0x617B	0x7301	# <CJK>
+0x617C	0x72F3	# <CJK>
+0x617D	0x7303	# <CJK>
+0x617E	0x72FA	# <CJK>
+0x6221	0x72FB	# <CJK>
+0x6222	0x7317	# <CJK>
+0x6223	0x7313	# <CJK>
+0x6224	0x7321	# <CJK>
+0x6225	0x730A	# <CJK>
+0x6226	0x731E	# <CJK>
+0x6227	0x731D	# <CJK>
+0x6228	0x7315	# <CJK>
+0x6229	0x7322	# <CJK>
+0x622A	0x7339	# <CJK>
+0x622B	0x7325	# <CJK>
+0x622C	0x732C	# <CJK>
+0x622D	0x7338	# <CJK>
+0x622E	0x7331	# <CJK>
+0x622F	0x7350	# <CJK>
+0x6230	0x734D	# <CJK>
+0x6231	0x7357	# <CJK>
+0x6232	0x7360	# <CJK>
+0x6233	0x736C	# <CJK>
+0x6234	0x736F	# <CJK>
+0x6235	0x737E	# <CJK>
+0x6236	0x821B	# <CJK>
+0x6237	0x5925	# <CJK>
+0x6238	0x98E7	# <CJK>
+0x6239	0x5924	# <CJK>
+0x623A	0x5902	# <CJK>
+0x623B	0x9963	# <CJK>
+0x623C	0x9967	# <CJK>
+0x623D	0x9968	# <CJK>
+0x623E	0x9969	# <CJK>
+0x623F	0x996A	# <CJK>
+0x6240	0x996B	# <CJK>
+0x6241	0x996C	# <CJK>
+0x6242	0x9974	# <CJK>
+0x6243	0x9977	# <CJK>
+0x6244	0x997D	# <CJK>
+0x6245	0x9980	# <CJK>
+0x6246	0x9984	# <CJK>
+0x6247	0x9987	# <CJK>
+0x6248	0x998A	# <CJK>
+0x6249	0x998D	# <CJK>
+0x624A	0x9990	# <CJK>
+0x624B	0x9991	# <CJK>
+0x624C	0x9993	# <CJK>
+0x624D	0x9994	# <CJK>
+0x624E	0x9995	# <CJK>
+0x624F	0x5E80	# <CJK>
+0x6250	0x5E91	# <CJK>
+0x6251	0x5E8B	# <CJK>
+0x6252	0x5E96	# <CJK>
+0x6253	0x5EA5	# <CJK>
+0x6254	0x5EA0	# <CJK>
+0x6255	0x5EB9	# <CJK>
+0x6256	0x5EB5	# <CJK>
+0x6257	0x5EBE	# <CJK>
+0x6258	0x5EB3	# <CJK>
+0x6259	0x8D53	# <CJK>
+0x625A	0x5ED2	# <CJK>
+0x625B	0x5ED1	# <CJK>
+0x625C	0x5EDB	# <CJK>
+0x625D	0x5EE8	# <CJK>
+0x625E	0x5EEA	# <CJK>
+0x625F	0x81BA	# <CJK>
+0x6260	0x5FC4	# <CJK>
+0x6261	0x5FC9	# <CJK>
+0x6262	0x5FD6	# <CJK>
+0x6263	0x5FCF	# <CJK>
+0x6264	0x6003	# <CJK>
+0x6265	0x5FEE	# <CJK>
+0x6266	0x6004	# <CJK>
+0x6267	0x5FE1	# <CJK>
+0x6268	0x5FE4	# <CJK>
+0x6269	0x5FFE	# <CJK>
+0x626A	0x6005	# <CJK>
+0x626B	0x6006	# <CJK>
+0x626C	0x5FEA	# <CJK>
+0x626D	0x5FED	# <CJK>
+0x626E	0x5FF8	# <CJK>
+0x626F	0x6019	# <CJK>
+0x6270	0x6035	# <CJK>
+0x6271	0x6026	# <CJK>
+0x6272	0x601B	# <CJK>
+0x6273	0x600F	# <CJK>
+0x6274	0x600D	# <CJK>
+0x6275	0x6029	# <CJK>
+0x6276	0x602B	# <CJK>
+0x6277	0x600A	# <CJK>
+0x6278	0x603F	# <CJK>
+0x6279	0x6021	# <CJK>
+0x627A	0x6078	# <CJK>
+0x627B	0x6079	# <CJK>
+0x627C	0x607B	# <CJK>
+0x627D	0x607A	# <CJK>
+0x627E	0x6042	# <CJK>
+0x6321	0x606A	# <CJK>
+0x6322	0x607D	# <CJK>
+0x6323	0x6096	# <CJK>
+0x6324	0x609A	# <CJK>
+0x6325	0x60AD	# <CJK>
+0x6326	0x609D	# <CJK>
+0x6327	0x6083	# <CJK>
+0x6328	0x6092	# <CJK>
+0x6329	0x608C	# <CJK>
+0x632A	0x609B	# <CJK>
+0x632B	0x60EC	# <CJK>
+0x632C	0x60BB	# <CJK>
+0x632D	0x60B1	# <CJK>
+0x632E	0x60DD	# <CJK>
+0x632F	0x60D8	# <CJK>
+0x6330	0x60C6	# <CJK>
+0x6331	0x60DA	# <CJK>
+0x6332	0x60B4	# <CJK>
+0x6333	0x6120	# <CJK>
+0x6334	0x6126	# <CJK>
+0x6335	0x6115	# <CJK>
+0x6336	0x6123	# <CJK>
+0x6337	0x60F4	# <CJK>
+0x6338	0x6100	# <CJK>
+0x6339	0x610E	# <CJK>
+0x633A	0x612B	# <CJK>
+0x633B	0x614A	# <CJK>
+0x633C	0x6175	# <CJK>
+0x633D	0x61AC	# <CJK>
+0x633E	0x6194	# <CJK>
+0x633F	0x61A7	# <CJK>
+0x6340	0x61B7	# <CJK>
+0x6341	0x61D4	# <CJK>
+0x6342	0x61F5	# <CJK>
+0x6343	0x5FDD	# <CJK>
+0x6344	0x96B3	# <CJK>
+0x6345	0x95E9	# <CJK>
+0x6346	0x95EB	# <CJK>
+0x6347	0x95F1	# <CJK>
+0x6348	0x95F3	# <CJK>
+0x6349	0x95F5	# <CJK>
+0x634A	0x95F6	# <CJK>
+0x634B	0x95FC	# <CJK>
+0x634C	0x95FE	# <CJK>
+0x634D	0x9603	# <CJK>
+0x634E	0x9604	# <CJK>
+0x634F	0x9606	# <CJK>
+0x6350	0x9608	# <CJK>
+0x6351	0x960A	# <CJK>
+0x6352	0x960B	# <CJK>
+0x6353	0x960C	# <CJK>
+0x6354	0x960D	# <CJK>
+0x6355	0x960F	# <CJK>
+0x6356	0x9612	# <CJK>
+0x6357	0x9615	# <CJK>
+0x6358	0x9616	# <CJK>
+0x6359	0x9617	# <CJK>
+0x635A	0x9619	# <CJK>
+0x635B	0x961A	# <CJK>
+0x635C	0x4E2C	# <CJK>
+0x635D	0x723F	# <CJK>
+0x635E	0x6215	# <CJK>
+0x635F	0x6C35	# <CJK>
+0x6360	0x6C54	# <CJK>
+0x6361	0x6C5C	# <CJK>
+0x6362	0x6C4A	# <CJK>
+0x6363	0x6CA3	# <CJK>
+0x6364	0x6C85	# <CJK>
+0x6365	0x6C90	# <CJK>
+0x6366	0x6C94	# <CJK>
+0x6367	0x6C8C	# <CJK>
+0x6368	0x6C68	# <CJK>
+0x6369	0x6C69	# <CJK>
+0x636A	0x6C74	# <CJK>
+0x636B	0x6C76	# <CJK>
+0x636C	0x6C86	# <CJK>
+0x636D	0x6CA9	# <CJK>
+0x636E	0x6CD0	# <CJK>
+0x636F	0x6CD4	# <CJK>
+0x6370	0x6CAD	# <CJK>
+0x6371	0x6CF7	# <CJK>
+0x6372	0x6CF8	# <CJK>
+0x6373	0x6CF1	# <CJK>
+0x6374	0x6CD7	# <CJK>
+0x6375	0x6CB2	# <CJK>
+0x6376	0x6CE0	# <CJK>
+0x6377	0x6CD6	# <CJK>
+0x6378	0x6CFA	# <CJK>
+0x6379	0x6CEB	# <CJK>
+0x637A	0x6CEE	# <CJK>
+0x637B	0x6CB1	# <CJK>
+0x637C	0x6CD3	# <CJK>
+0x637D	0x6CEF	# <CJK>
+0x637E	0x6CFE	# <CJK>
+0x6421	0x6D39	# <CJK>
+0x6422	0x6D27	# <CJK>
+0x6423	0x6D0C	# <CJK>
+0x6424	0x6D43	# <CJK>
+0x6425	0x6D48	# <CJK>
+0x6426	0x6D07	# <CJK>
+0x6427	0x6D04	# <CJK>
+0x6428	0x6D19	# <CJK>
+0x6429	0x6D0E	# <CJK>
+0x642A	0x6D2B	# <CJK>
+0x642B	0x6D4D	# <CJK>
+0x642C	0x6D2E	# <CJK>
+0x642D	0x6D35	# <CJK>
+0x642E	0x6D1A	# <CJK>
+0x642F	0x6D4F	# <CJK>
+0x6430	0x6D52	# <CJK>
+0x6431	0x6D54	# <CJK>
+0x6432	0x6D33	# <CJK>
+0x6433	0x6D91	# <CJK>
+0x6434	0x6D6F	# <CJK>
+0x6435	0x6D9E	# <CJK>
+0x6436	0x6DA0	# <CJK>
+0x6437	0x6D5E	# <CJK>
+0x6438	0x6D93	# <CJK>
+0x6439	0x6D94	# <CJK>
+0x643A	0x6D5C	# <CJK>
+0x643B	0x6D60	# <CJK>
+0x643C	0x6D7C	# <CJK>
+0x643D	0x6D63	# <CJK>
+0x643E	0x6E1A	# <CJK>
+0x643F	0x6DC7	# <CJK>
+0x6440	0x6DC5	# <CJK>
+0x6441	0x6DDE	# <CJK>
+0x6442	0x6E0E	# <CJK>
+0x6443	0x6DBF	# <CJK>
+0x6444	0x6DE0	# <CJK>
+0x6445	0x6E11	# <CJK>
+0x6446	0x6DE6	# <CJK>
+0x6447	0x6DDD	# <CJK>
+0x6448	0x6DD9	# <CJK>
+0x6449	0x6E16	# <CJK>
+0x644A	0x6DAB	# <CJK>
+0x644B	0x6E0C	# <CJK>
+0x644C	0x6DAE	# <CJK>
+0x644D	0x6E2B	# <CJK>
+0x644E	0x6E6E	# <CJK>
+0x644F	0x6E4E	# <CJK>
+0x6450	0x6E6B	# <CJK>
+0x6451	0x6EB2	# <CJK>
+0x6452	0x6E5F	# <CJK>
+0x6453	0x6E86	# <CJK>
+0x6454	0x6E53	# <CJK>
+0x6455	0x6E54	# <CJK>
+0x6456	0x6E32	# <CJK>
+0x6457	0x6E25	# <CJK>
+0x6458	0x6E44	# <CJK>
+0x6459	0x6EDF	# <CJK>
+0x645A	0x6EB1	# <CJK>
+0x645B	0x6E98	# <CJK>
+0x645C	0x6EE0	# <CJK>
+0x645D	0x6F2D	# <CJK>
+0x645E	0x6EE2	# <CJK>
+0x645F	0x6EA5	# <CJK>
+0x6460	0x6EA7	# <CJK>
+0x6461	0x6EBD	# <CJK>
+0x6462	0x6EBB	# <CJK>
+0x6463	0x6EB7	# <CJK>
+0x6464	0x6ED7	# <CJK>
+0x6465	0x6EB4	# <CJK>
+0x6466	0x6ECF	# <CJK>
+0x6467	0x6E8F	# <CJK>
+0x6468	0x6EC2	# <CJK>
+0x6469	0x6E9F	# <CJK>
+0x646A	0x6F62	# <CJK>
+0x646B	0x6F46	# <CJK>
+0x646C	0x6F47	# <CJK>
+0x646D	0x6F24	# <CJK>
+0x646E	0x6F15	# <CJK>
+0x646F	0x6EF9	# <CJK>
+0x6470	0x6F2F	# <CJK>
+0x6471	0x6F36	# <CJK>
+0x6472	0x6F4B	# <CJK>
+0x6473	0x6F74	# <CJK>
+0x6474	0x6F2A	# <CJK>
+0x6475	0x6F09	# <CJK>
+0x6476	0x6F29	# <CJK>
+0x6477	0x6F89	# <CJK>
+0x6478	0x6F8D	# <CJK>
+0x6479	0x6F8C	# <CJK>
+0x647A	0x6F78	# <CJK>
+0x647B	0x6F72	# <CJK>
+0x647C	0x6F7C	# <CJK>
+0x647D	0x6F7A	# <CJK>
+0x647E	0x6FD1	# <CJK>
+0x6521	0x6FC9	# <CJK>
+0x6522	0x6FA7	# <CJK>
+0x6523	0x6FB9	# <CJK>
+0x6524	0x6FB6	# <CJK>
+0x6525	0x6FC2	# <CJK>
+0x6526	0x6FE1	# <CJK>
+0x6527	0x6FEE	# <CJK>
+0x6528	0x6FDE	# <CJK>
+0x6529	0x6FE0	# <CJK>
+0x652A	0x6FEF	# <CJK>
+0x652B	0x701A	# <CJK>
+0x652C	0x7023	# <CJK>
+0x652D	0x701B	# <CJK>
+0x652E	0x7039	# <CJK>
+0x652F	0x7035	# <CJK>
+0x6530	0x704F	# <CJK>
+0x6531	0x705E	# <CJK>
+0x6532	0x5B80	# <CJK>
+0x6533	0x5B84	# <CJK>
+0x6534	0x5B95	# <CJK>
+0x6535	0x5B93	# <CJK>
+0x6536	0x5BA5	# <CJK>
+0x6537	0x5BB8	# <CJK>
+0x6538	0x752F	# <CJK>
+0x6539	0x9A9E	# <CJK>
+0x653A	0x6434	# <CJK>
+0x653B	0x5BE4	# <CJK>
+0x653C	0x5BEE	# <CJK>
+0x653D	0x8930	# <CJK>
+0x653E	0x5BF0	# <CJK>
+0x653F	0x8E47	# <CJK>
+0x6540	0x8B07	# <CJK>
+0x6541	0x8FB6	# <CJK>
+0x6542	0x8FD3	# <CJK>
+0x6543	0x8FD5	# <CJK>
+0x6544	0x8FE5	# <CJK>
+0x6545	0x8FEE	# <CJK>
+0x6546	0x8FE4	# <CJK>
+0x6547	0x8FE9	# <CJK>
+0x6548	0x8FE6	# <CJK>
+0x6549	0x8FF3	# <CJK>
+0x654A	0x8FE8	# <CJK>
+0x654B	0x9005	# <CJK>
+0x654C	0x9004	# <CJK>
+0x654D	0x900B	# <CJK>
+0x654E	0x9026	# <CJK>
+0x654F	0x9011	# <CJK>
+0x6550	0x900D	# <CJK>
+0x6551	0x9016	# <CJK>
+0x6552	0x9021	# <CJK>
+0x6553	0x9035	# <CJK>
+0x6554	0x9036	# <CJK>
+0x6555	0x902D	# <CJK>
+0x6556	0x902F	# <CJK>
+0x6557	0x9044	# <CJK>
+0x6558	0x9051	# <CJK>
+0x6559	0x9052	# <CJK>
+0x655A	0x9050	# <CJK>
+0x655B	0x9068	# <CJK>
+0x655C	0x9058	# <CJK>
+0x655D	0x9062	# <CJK>
+0x655E	0x905B	# <CJK>
+0x655F	0x66B9	# <CJK>
+0x6560	0x9074	# <CJK>
+0x6561	0x907D	# <CJK>
+0x6562	0x9082	# <CJK>
+0x6563	0x9088	# <CJK>
+0x6564	0x9083	# <CJK>
+0x6565	0x908B	# <CJK>
+0x6566	0x5F50	# <CJK>
+0x6567	0x5F57	# <CJK>
+0x6568	0x5F56	# <CJK>
+0x6569	0x5F58	# <CJK>
+0x656A	0x5C3B	# <CJK>
+0x656B	0x54AB	# <CJK>
+0x656C	0x5C50	# <CJK>
+0x656D	0x5C59	# <CJK>
+0x656E	0x5B71	# <CJK>
+0x656F	0x5C63	# <CJK>
+0x6570	0x5C66	# <CJK>
+0x6571	0x7FBC	# <CJK>
+0x6572	0x5F2A	# <CJK>
+0x6573	0x5F29	# <CJK>
+0x6574	0x5F2D	# <CJK>
+0x6575	0x8274	# <CJK>
+0x6576	0x5F3C	# <CJK>
+0x6577	0x9B3B	# <CJK>
+0x6578	0x5C6E	# <CJK>
+0x6579	0x5981	# <CJK>
+0x657A	0x5983	# <CJK>
+0x657B	0x598D	# <CJK>
+0x657C	0x59A9	# <CJK>
+0x657D	0x59AA	# <CJK>
+0x657E	0x59A3	# <CJK>
+0x6621	0x5997	# <CJK>
+0x6622	0x59CA	# <CJK>
+0x6623	0x59AB	# <CJK>
+0x6624	0x599E	# <CJK>
+0x6625	0x59A4	# <CJK>
+0x6626	0x59D2	# <CJK>
+0x6627	0x59B2	# <CJK>
+0x6628	0x59AF	# <CJK>
+0x6629	0x59D7	# <CJK>
+0x662A	0x59BE	# <CJK>
+0x662B	0x5A05	# <CJK>
+0x662C	0x5A06	# <CJK>
+0x662D	0x59DD	# <CJK>
+0x662E	0x5A08	# <CJK>
+0x662F	0x59E3	# <CJK>
+0x6630	0x59D8	# <CJK>
+0x6631	0x59F9	# <CJK>
+0x6632	0x5A0C	# <CJK>
+0x6633	0x5A09	# <CJK>
+0x6634	0x5A32	# <CJK>
+0x6635	0x5A34	# <CJK>
+0x6636	0x5A11	# <CJK>
+0x6637	0x5A23	# <CJK>
+0x6638	0x5A13	# <CJK>
+0x6639	0x5A40	# <CJK>
+0x663A	0x5A67	# <CJK>
+0x663B	0x5A4A	# <CJK>
+0x663C	0x5A55	# <CJK>
+0x663D	0x5A3C	# <CJK>
+0x663E	0x5A62	# <CJK>
+0x663F	0x5A75	# <CJK>
+0x6640	0x80EC	# <CJK>
+0x6641	0x5AAA	# <CJK>
+0x6642	0x5A9B	# <CJK>
+0x6643	0x5A77	# <CJK>
+0x6644	0x5A7A	# <CJK>
+0x6645	0x5ABE	# <CJK>
+0x6646	0x5AEB	# <CJK>
+0x6647	0x5AB2	# <CJK>
+0x6648	0x5AD2	# <CJK>
+0x6649	0x5AD4	# <CJK>
+0x664A	0x5AB8	# <CJK>
+0x664B	0x5AE0	# <CJK>
+0x664C	0x5AE3	# <CJK>
+0x664D	0x5AF1	# <CJK>
+0x664E	0x5AD6	# <CJK>
+0x664F	0x5AE6	# <CJK>
+0x6650	0x5AD8	# <CJK>
+0x6651	0x5ADC	# <CJK>
+0x6652	0x5B09	# <CJK>
+0x6653	0x5B17	# <CJK>
+0x6654	0x5B16	# <CJK>
+0x6655	0x5B32	# <CJK>
+0x6656	0x5B37	# <CJK>
+0x6657	0x5B40	# <CJK>
+0x6658	0x5C15	# <CJK>
+0x6659	0x5C1C	# <CJK>
+0x665A	0x5B5A	# <CJK>
+0x665B	0x5B65	# <CJK>
+0x665C	0x5B73	# <CJK>
+0x665D	0x5B51	# <CJK>
+0x665E	0x5B53	# <CJK>
+0x665F	0x5B62	# <CJK>
+0x6660	0x9A75	# <CJK>
+0x6661	0x9A77	# <CJK>
+0x6662	0x9A78	# <CJK>
+0x6663	0x9A7A	# <CJK>
+0x6664	0x9A7F	# <CJK>
+0x6665	0x9A7D	# <CJK>
+0x6666	0x9A80	# <CJK>
+0x6667	0x9A81	# <CJK>
+0x6668	0x9A85	# <CJK>
+0x6669	0x9A88	# <CJK>
+0x666A	0x9A8A	# <CJK>
+0x666B	0x9A90	# <CJK>
+0x666C	0x9A92	# <CJK>
+0x666D	0x9A93	# <CJK>
+0x666E	0x9A96	# <CJK>
+0x666F	0x9A98	# <CJK>
+0x6670	0x9A9B	# <CJK>
+0x6671	0x9A9C	# <CJK>
+0x6672	0x9A9D	# <CJK>
+0x6673	0x9A9F	# <CJK>
+0x6674	0x9AA0	# <CJK>
+0x6675	0x9AA2	# <CJK>
+0x6676	0x9AA3	# <CJK>
+0x6677	0x9AA5	# <CJK>
+0x6678	0x9AA7	# <CJK>
+0x6679	0x7E9F	# <CJK>
+0x667A	0x7EA1	# <CJK>
+0x667B	0x7EA3	# <CJK>
+0x667C	0x7EA5	# <CJK>
+0x667D	0x7EA8	# <CJK>
+0x667E	0x7EA9	# <CJK>
+0x6721	0x7EAD	# <CJK>
+0x6722	0x7EB0	# <CJK>
+0x6723	0x7EBE	# <CJK>
+0x6724	0x7EC0	# <CJK>
+0x6725	0x7EC1	# <CJK>
+0x6726	0x7EC2	# <CJK>
+0x6727	0x7EC9	# <CJK>
+0x6728	0x7ECB	# <CJK>
+0x6729	0x7ECC	# <CJK>
+0x672A	0x7ED0	# <CJK>
+0x672B	0x7ED4	# <CJK>
+0x672C	0x7ED7	# <CJK>
+0x672D	0x7EDB	# <CJK>
+0x672E	0x7EE0	# <CJK>
+0x672F	0x7EE1	# <CJK>
+0x6730	0x7EE8	# <CJK>
+0x6731	0x7EEB	# <CJK>
+0x6732	0x7EEE	# <CJK>
+0x6733	0x7EEF	# <CJK>
+0x6734	0x7EF1	# <CJK>
+0x6735	0x7EF2	# <CJK>
+0x6736	0x7F0D	# <CJK>
+0x6737	0x7EF6	# <CJK>
+0x6738	0x7EFA	# <CJK>
+0x6739	0x7EFB	# <CJK>
+0x673A	0x7EFE	# <CJK>
+0x673B	0x7F01	# <CJK>
+0x673C	0x7F02	# <CJK>
+0x673D	0x7F03	# <CJK>
+0x673E	0x7F07	# <CJK>
+0x673F	0x7F08	# <CJK>
+0x6740	0x7F0B	# <CJK>
+0x6741	0x7F0C	# <CJK>
+0x6742	0x7F0F	# <CJK>
+0x6743	0x7F11	# <CJK>
+0x6744	0x7F12	# <CJK>
+0x6745	0x7F17	# <CJK>
+0x6746	0x7F19	# <CJK>
+0x6747	0x7F1C	# <CJK>
+0x6748	0x7F1B	# <CJK>
+0x6749	0x7F1F	# <CJK>
+0x674A	0x7F21	# <CJK>
+0x674B	0x7F22	# <CJK>
+0x674C	0x7F23	# <CJK>
+0x674D	0x7F24	# <CJK>
+0x674E	0x7F25	# <CJK>
+0x674F	0x7F26	# <CJK>
+0x6750	0x7F27	# <CJK>
+0x6751	0x7F2A	# <CJK>
+0x6752	0x7F2B	# <CJK>
+0x6753	0x7F2C	# <CJK>
+0x6754	0x7F2D	# <CJK>
+0x6755	0x7F2F	# <CJK>
+0x6756	0x7F30	# <CJK>
+0x6757	0x7F31	# <CJK>
+0x6758	0x7F32	# <CJK>
+0x6759	0x7F33	# <CJK>
+0x675A	0x7F35	# <CJK>
+0x675B	0x5E7A	# <CJK>
+0x675C	0x757F	# <CJK>
+0x675D	0x5DDB	# <CJK>
+0x675E	0x753E	# <CJK>
+0x675F	0x9095	# <CJK>
+0x6760	0x738E	# <CJK>
+0x6761	0x7391	# <CJK>
+0x6762	0x73AE	# <CJK>
+0x6763	0x73A2	# <CJK>
+0x6764	0x739F	# <CJK>
+0x6765	0x73CF	# <CJK>
+0x6766	0x73C2	# <CJK>
+0x6767	0x73D1	# <CJK>
+0x6768	0x73B7	# <CJK>
+0x6769	0x73B3	# <CJK>
+0x676A	0x73C0	# <CJK>
+0x676B	0x73C9	# <CJK>
+0x676C	0x73C8	# <CJK>
+0x676D	0x73E5	# <CJK>
+0x676E	0x73D9	# <CJK>
+0x676F	0x987C	# <CJK>
+0x6770	0x740A	# <CJK>
+0x6771	0x73E9	# <CJK>
+0x6772	0x73E7	# <CJK>
+0x6773	0x73DE	# <CJK>
+0x6774	0x73BA	# <CJK>
+0x6775	0x73F2	# <CJK>
+0x6776	0x740F	# <CJK>
+0x6777	0x742A	# <CJK>
+0x6778	0x745B	# <CJK>
+0x6779	0x7426	# <CJK>
+0x677A	0x7425	# <CJK>
+0x677B	0x7428	# <CJK>
+0x677C	0x7430	# <CJK>
+0x677D	0x742E	# <CJK>
+0x677E	0x742C	# <CJK>
+0x6821	0x741B	# <CJK>
+0x6822	0x741A	# <CJK>
+0x6823	0x7441	# <CJK>
+0x6824	0x745C	# <CJK>
+0x6825	0x7457	# <CJK>
+0x6826	0x7455	# <CJK>
+0x6827	0x7459	# <CJK>
+0x6828	0x7477	# <CJK>
+0x6829	0x746D	# <CJK>
+0x682A	0x747E	# <CJK>
+0x682B	0x749C	# <CJK>
+0x682C	0x748E	# <CJK>
+0x682D	0x7480	# <CJK>
+0x682E	0x7481	# <CJK>
+0x682F	0x7487	# <CJK>
+0x6830	0x748B	# <CJK>
+0x6831	0x749E	# <CJK>
+0x6832	0x74A8	# <CJK>
+0x6833	0x74A9	# <CJK>
+0x6834	0x7490	# <CJK>
+0x6835	0x74A7	# <CJK>
+0x6836	0x74D2	# <CJK>
+0x6837	0x74BA	# <CJK>
+0x6838	0x97EA	# <CJK>
+0x6839	0x97EB	# <CJK>
+0x683A	0x97EC	# <CJK>
+0x683B	0x674C	# <CJK>
+0x683C	0x6753	# <CJK>
+0x683D	0x675E	# <CJK>
+0x683E	0x6748	# <CJK>
+0x683F	0x6769	# <CJK>
+0x6840	0x67A5	# <CJK>
+0x6841	0x6787	# <CJK>
+0x6842	0x676A	# <CJK>
+0x6843	0x6773	# <CJK>
+0x6844	0x6798	# <CJK>
+0x6845	0x67A7	# <CJK>
+0x6846	0x6775	# <CJK>
+0x6847	0x67A8	# <CJK>
+0x6848	0x679E	# <CJK>
+0x6849	0x67AD	# <CJK>
+0x684A	0x678B	# <CJK>
+0x684B	0x6777	# <CJK>
+0x684C	0x677C	# <CJK>
+0x684D	0x67F0	# <CJK>
+0x684E	0x6809	# <CJK>
+0x684F	0x67D8	# <CJK>
+0x6850	0x680A	# <CJK>
+0x6851	0x67E9	# <CJK>
+0x6852	0x67B0	# <CJK>
+0x6853	0x680C	# <CJK>
+0x6854	0x67D9	# <CJK>
+0x6855	0x67B5	# <CJK>
+0x6856	0x67DA	# <CJK>
+0x6857	0x67B3	# <CJK>
+0x6858	0x67DD	# <CJK>
+0x6859	0x6800	# <CJK>
+0x685A	0x67C3	# <CJK>
+0x685B	0x67B8	# <CJK>
+0x685C	0x67E2	# <CJK>
+0x685D	0x680E	# <CJK>
+0x685E	0x67C1	# <CJK>
+0x685F	0x67FD	# <CJK>
+0x6860	0x6832	# <CJK>
+0x6861	0x6833	# <CJK>
+0x6862	0x6860	# <CJK>
+0x6863	0x6861	# <CJK>
+0x6864	0x684E	# <CJK>
+0x6865	0x6862	# <CJK>
+0x6866	0x6844	# <CJK>
+0x6867	0x6864	# <CJK>
+0x6868	0x6883	# <CJK>
+0x6869	0x681D	# <CJK>
+0x686A	0x6855	# <CJK>
+0x686B	0x6866	# <CJK>
+0x686C	0x6841	# <CJK>
+0x686D	0x6867	# <CJK>
+0x686E	0x6840	# <CJK>
+0x686F	0x683E	# <CJK>
+0x6870	0x684A	# <CJK>
+0x6871	0x6849	# <CJK>
+0x6872	0x6829	# <CJK>
+0x6873	0x68B5	# <CJK>
+0x6874	0x688F	# <CJK>
+0x6875	0x6874	# <CJK>
+0x6876	0x6877	# <CJK>
+0x6877	0x6893	# <CJK>
+0x6878	0x686B	# <CJK>
+0x6879	0x68C2	# <CJK>
+0x687A	0x696E	# <CJK>
+0x687B	0x68FC	# <CJK>
+0x687C	0x691F	# <CJK>
+0x687D	0x6920	# <CJK>
+0x687E	0x68F9	# <CJK>
+0x6921	0x6924	# <CJK>
+0x6922	0x68F0	# <CJK>
+0x6923	0x690B	# <CJK>
+0x6924	0x6901	# <CJK>
+0x6925	0x6957	# <CJK>
+0x6926	0x68E3	# <CJK>
+0x6927	0x6910	# <CJK>
+0x6928	0x6971	# <CJK>
+0x6929	0x6939	# <CJK>
+0x692A	0x6960	# <CJK>
+0x692B	0x6942	# <CJK>
+0x692C	0x695D	# <CJK>
+0x692D	0x6984	# <CJK>
+0x692E	0x696B	# <CJK>
+0x692F	0x6980	# <CJK>
+0x6930	0x6998	# <CJK>
+0x6931	0x6978	# <CJK>
+0x6932	0x6934	# <CJK>
+0x6933	0x69CC	# <CJK>
+0x6934	0x6987	# <CJK>
+0x6935	0x6988	# <CJK>
+0x6936	0x69CE	# <CJK>
+0x6937	0x6989	# <CJK>
+0x6938	0x6966	# <CJK>
+0x6939	0x6963	# <CJK>
+0x693A	0x6979	# <CJK>
+0x693B	0x699B	# <CJK>
+0x693C	0x69A7	# <CJK>
+0x693D	0x69BB	# <CJK>
+0x693E	0x69AB	# <CJK>
+0x693F	0x69AD	# <CJK>
+0x6940	0x69D4	# <CJK>
+0x6941	0x69B1	# <CJK>
+0x6942	0x69C1	# <CJK>
+0x6943	0x69CA	# <CJK>
+0x6944	0x69DF	# <CJK>
+0x6945	0x6995	# <CJK>
+0x6946	0x69E0	# <CJK>
+0x6947	0x698D	# <CJK>
+0x6948	0x69FF	# <CJK>
+0x6949	0x6A2F	# <CJK>
+0x694A	0x69ED	# <CJK>
+0x694B	0x6A17	# <CJK>
+0x694C	0x6A18	# <CJK>
+0x694D	0x6A65	# <CJK>
+0x694E	0x69F2	# <CJK>
+0x694F	0x6A44	# <CJK>
+0x6950	0x6A3E	# <CJK>
+0x6951	0x6AA0	# <CJK>
+0x6952	0x6A50	# <CJK>
+0x6953	0x6A5B	# <CJK>
+0x6954	0x6A35	# <CJK>
+0x6955	0x6A8E	# <CJK>
+0x6956	0x6A79	# <CJK>
+0x6957	0x6A3D	# <CJK>
+0x6958	0x6A28	# <CJK>
+0x6959	0x6A58	# <CJK>
+0x695A	0x6A7C	# <CJK>
+0x695B	0x6A91	# <CJK>
+0x695C	0x6A90	# <CJK>
+0x695D	0x6AA9	# <CJK>
+0x695E	0x6A97	# <CJK>
+0x695F	0x6AAB	# <CJK>
+0x6960	0x7337	# <CJK>
+0x6961	0x7352	# <CJK>
+0x6962	0x6B81	# <CJK>
+0x6963	0x6B82	# <CJK>
+0x6964	0x6B87	# <CJK>
+0x6965	0x6B84	# <CJK>
+0x6966	0x6B92	# <CJK>
+0x6967	0x6B93	# <CJK>
+0x6968	0x6B8D	# <CJK>
+0x6969	0x6B9A	# <CJK>
+0x696A	0x6B9B	# <CJK>
+0x696B	0x6BA1	# <CJK>
+0x696C	0x6BAA	# <CJK>
+0x696D	0x8F6B	# <CJK>
+0x696E	0x8F6D	# <CJK>
+0x696F	0x8F71	# <CJK>
+0x6970	0x8F72	# <CJK>
+0x6971	0x8F73	# <CJK>
+0x6972	0x8F75	# <CJK>
+0x6973	0x8F76	# <CJK>
+0x6974	0x8F78	# <CJK>
+0x6975	0x8F77	# <CJK>
+0x6976	0x8F79	# <CJK>
+0x6977	0x8F7A	# <CJK>
+0x6978	0x8F7C	# <CJK>
+0x6979	0x8F7E	# <CJK>
+0x697A	0x8F81	# <CJK>
+0x697B	0x8F82	# <CJK>
+0x697C	0x8F84	# <CJK>
+0x697D	0x8F87	# <CJK>
+0x697E	0x8F8B	# <CJK>
+0x6A21	0x8F8D	# <CJK>
+0x6A22	0x8F8E	# <CJK>
+0x6A23	0x8F8F	# <CJK>
+0x6A24	0x8F98	# <CJK>
+0x6A25	0x8F9A	# <CJK>
+0x6A26	0x8ECE	# <CJK>
+0x6A27	0x620B	# <CJK>
+0x6A28	0x6217	# <CJK>
+0x6A29	0x621B	# <CJK>
+0x6A2A	0x621F	# <CJK>
+0x6A2B	0x6222	# <CJK>
+0x6A2C	0x6221	# <CJK>
+0x6A2D	0x6225	# <CJK>
+0x6A2E	0x6224	# <CJK>
+0x6A2F	0x622C	# <CJK>
+0x6A30	0x81E7	# <CJK>
+0x6A31	0x74EF	# <CJK>
+0x6A32	0x74F4	# <CJK>
+0x6A33	0x74FF	# <CJK>
+0x6A34	0x750F	# <CJK>
+0x6A35	0x7511	# <CJK>
+0x6A36	0x7513	# <CJK>
+0x6A37	0x6534	# <CJK>
+0x6A38	0x65EE	# <CJK>
+0x6A39	0x65EF	# <CJK>
+0x6A3A	0x65F0	# <CJK>
+0x6A3B	0x660A	# <CJK>
+0x6A3C	0x6619	# <CJK>
+0x6A3D	0x6772	# <CJK>
+0x6A3E	0x6603	# <CJK>
+0x6A3F	0x6615	# <CJK>
+0x6A40	0x6600	# <CJK>
+0x6A41	0x7085	# <CJK>
+0x6A42	0x66F7	# <CJK>
+0x6A43	0x661D	# <CJK>
+0x6A44	0x6634	# <CJK>
+0x6A45	0x6631	# <CJK>
+0x6A46	0x6636	# <CJK>
+0x6A47	0x6635	# <CJK>
+0x6A48	0x8006	# <CJK>
+0x6A49	0x665F	# <CJK>
+0x6A4A	0x6654	# <CJK>
+0x6A4B	0x6641	# <CJK>
+0x6A4C	0x664F	# <CJK>
+0x6A4D	0x6656	# <CJK>
+0x6A4E	0x6661	# <CJK>
+0x6A4F	0x6657	# <CJK>
+0x6A50	0x6677	# <CJK>
+0x6A51	0x6684	# <CJK>
+0x6A52	0x668C	# <CJK>
+0x6A53	0x66A7	# <CJK>
+0x6A54	0x669D	# <CJK>
+0x6A55	0x66BE	# <CJK>
+0x6A56	0x66DB	# <CJK>
+0x6A57	0x66DC	# <CJK>
+0x6A58	0x66E6	# <CJK>
+0x6A59	0x66E9	# <CJK>
+0x6A5A	0x8D32	# <CJK>
+0x6A5B	0x8D33	# <CJK>
+0x6A5C	0x8D36	# <CJK>
+0x6A5D	0x8D3B	# <CJK>
+0x6A5E	0x8D3D	# <CJK>
+0x6A5F	0x8D40	# <CJK>
+0x6A60	0x8D45	# <CJK>
+0x6A61	0x8D46	# <CJK>
+0x6A62	0x8D48	# <CJK>
+0x6A63	0x8D49	# <CJK>
+0x6A64	0x8D47	# <CJK>
+0x6A65	0x8D4D	# <CJK>
+0x6A66	0x8D55	# <CJK>
+0x6A67	0x8D59	# <CJK>
+0x6A68	0x89C7	# <CJK>
+0x6A69	0x89CA	# <CJK>
+0x6A6A	0x89CB	# <CJK>
+0x6A6B	0x89CC	# <CJK>
+0x6A6C	0x89CE	# <CJK>
+0x6A6D	0x89CF	# <CJK>
+0x6A6E	0x89D0	# <CJK>
+0x6A6F	0x89D1	# <CJK>
+0x6A70	0x726E	# <CJK>
+0x6A71	0x729F	# <CJK>
+0x6A72	0x725D	# <CJK>
+0x6A73	0x7266	# <CJK>
+0x6A74	0x726F	# <CJK>
+0x6A75	0x727E	# <CJK>
+0x6A76	0x727F	# <CJK>
+0x6A77	0x7284	# <CJK>
+0x6A78	0x728B	# <CJK>
+0x6A79	0x728D	# <CJK>
+0x6A7A	0x728F	# <CJK>
+0x6A7B	0x7292	# <CJK>
+0x6A7C	0x6308	# <CJK>
+0x6A7D	0x6332	# <CJK>
+0x6A7E	0x63B0	# <CJK>
+0x6B21	0x643F	# <CJK>
+0x6B22	0x64D8	# <CJK>
+0x6B23	0x8004	# <CJK>
+0x6B24	0x6BEA	# <CJK>
+0x6B25	0x6BF3	# <CJK>
+0x6B26	0x6BFD	# <CJK>
+0x6B27	0x6BF5	# <CJK>
+0x6B28	0x6BF9	# <CJK>
+0x6B29	0x6C05	# <CJK>
+0x6B2A	0x6C07	# <CJK>
+0x6B2B	0x6C06	# <CJK>
+0x6B2C	0x6C0D	# <CJK>
+0x6B2D	0x6C15	# <CJK>
+0x6B2E	0x6C18	# <CJK>
+0x6B2F	0x6C19	# <CJK>
+0x6B30	0x6C1A	# <CJK>
+0x6B31	0x6C21	# <CJK>
+0x6B32	0x6C29	# <CJK>
+0x6B33	0x6C24	# <CJK>
+0x6B34	0x6C2A	# <CJK>
+0x6B35	0x6C32	# <CJK>
+0x6B36	0x6535	# <CJK>
+0x6B37	0x6555	# <CJK>
+0x6B38	0x656B	# <CJK>
+0x6B39	0x724D	# <CJK>
+0x6B3A	0x7252	# <CJK>
+0x6B3B	0x7256	# <CJK>
+0x6B3C	0x7230	# <CJK>
+0x6B3D	0x8662	# <CJK>
+0x6B3E	0x5216	# <CJK>
+0x6B3F	0x809F	# <CJK>
+0x6B40	0x809C	# <CJK>
+0x6B41	0x8093	# <CJK>
+0x6B42	0x80BC	# <CJK>
+0x6B43	0x670A	# <CJK>
+0x6B44	0x80BD	# <CJK>
+0x6B45	0x80B1	# <CJK>
+0x6B46	0x80AB	# <CJK>
+0x6B47	0x80AD	# <CJK>
+0x6B48	0x80B4	# <CJK>
+0x6B49	0x80B7	# <CJK>
+0x6B4A	0x80E7	# <CJK>
+0x6B4B	0x80E8	# <CJK>
+0x6B4C	0x80E9	# <CJK>
+0x6B4D	0x80EA	# <CJK>
+0x6B4E	0x80DB	# <CJK>
+0x6B4F	0x80C2	# <CJK>
+0x6B50	0x80C4	# <CJK>
+0x6B51	0x80D9	# <CJK>
+0x6B52	0x80CD	# <CJK>
+0x6B53	0x80D7	# <CJK>
+0x6B54	0x6710	# <CJK>
+0x6B55	0x80DD	# <CJK>
+0x6B56	0x80EB	# <CJK>
+0x6B57	0x80F1	# <CJK>
+0x6B58	0x80F4	# <CJK>
+0x6B59	0x80ED	# <CJK>
+0x6B5A	0x810D	# <CJK>
+0x6B5B	0x810E	# <CJK>
+0x6B5C	0x80F2	# <CJK>
+0x6B5D	0x80FC	# <CJK>
+0x6B5E	0x6715	# <CJK>
+0x6B5F	0x8112	# <CJK>
+0x6B60	0x8C5A	# <CJK>
+0x6B61	0x8136	# <CJK>
+0x6B62	0x811E	# <CJK>
+0x6B63	0x812C	# <CJK>
+0x6B64	0x8118	# <CJK>
+0x6B65	0x8132	# <CJK>
+0x6B66	0x8148	# <CJK>
+0x6B67	0x814C	# <CJK>
+0x6B68	0x8153	# <CJK>
+0x6B69	0x8174	# <CJK>
+0x6B6A	0x8159	# <CJK>
+0x6B6B	0x815A	# <CJK>
+0x6B6C	0x8171	# <CJK>
+0x6B6D	0x8160	# <CJK>
+0x6B6E	0x8169	# <CJK>
+0x6B6F	0x817C	# <CJK>
+0x6B70	0x817D	# <CJK>
+0x6B71	0x816D	# <CJK>
+0x6B72	0x8167	# <CJK>
+0x6B73	0x584D	# <CJK>
+0x6B74	0x5AB5	# <CJK>
+0x6B75	0x8188	# <CJK>
+0x6B76	0x8182	# <CJK>
+0x6B77	0x8191	# <CJK>
+0x6B78	0x6ED5	# <CJK>
+0x6B79	0x81A3	# <CJK>
+0x6B7A	0x81AA	# <CJK>
+0x6B7B	0x81CC	# <CJK>
+0x6B7C	0x6726	# <CJK>
+0x6B7D	0x81CA	# <CJK>
+0x6B7E	0x81BB	# <CJK>
+0x6C21	0x81C1	# <CJK>
+0x6C22	0x81A6	# <CJK>
+0x6C23	0x6B24	# <CJK>
+0x6C24	0x6B37	# <CJK>
+0x6C25	0x6B39	# <CJK>
+0x6C26	0x6B43	# <CJK>
+0x6C27	0x6B46	# <CJK>
+0x6C28	0x6B59	# <CJK>
+0x6C29	0x98D1	# <CJK>
+0x6C2A	0x98D2	# <CJK>
+0x6C2B	0x98D3	# <CJK>
+0x6C2C	0x98D5	# <CJK>
+0x6C2D	0x98D9	# <CJK>
+0x6C2E	0x98DA	# <CJK>
+0x6C2F	0x6BB3	# <CJK>
+0x6C30	0x5F40	# <CJK>
+0x6C31	0x6BC2	# <CJK>
+0x6C32	0x89F3	# <CJK>
+0x6C33	0x6590	# <CJK>
+0x6C34	0x9F51	# <CJK>
+0x6C35	0x6593	# <CJK>
+0x6C36	0x65BC	# <CJK>
+0x6C37	0x65C6	# <CJK>
+0x6C38	0x65C4	# <CJK>
+0x6C39	0x65C3	# <CJK>
+0x6C3A	0x65CC	# <CJK>
+0x6C3B	0x65CE	# <CJK>
+0x6C3C	0x65D2	# <CJK>
+0x6C3D	0x65D6	# <CJK>
+0x6C3E	0x7080	# <CJK>
+0x6C3F	0x709C	# <CJK>
+0x6C40	0x7096	# <CJK>
+0x6C41	0x709D	# <CJK>
+0x6C42	0x70BB	# <CJK>
+0x6C43	0x70C0	# <CJK>
+0x6C44	0x70B7	# <CJK>
+0x6C45	0x70AB	# <CJK>
+0x6C46	0x70B1	# <CJK>
+0x6C47	0x70E8	# <CJK>
+0x6C48	0x70CA	# <CJK>
+0x6C49	0x7110	# <CJK>
+0x6C4A	0x7113	# <CJK>
+0x6C4B	0x7116	# <CJK>
+0x6C4C	0x712F	# <CJK>
+0x6C4D	0x7131	# <CJK>
+0x6C4E	0x7173	# <CJK>
+0x6C4F	0x715C	# <CJK>
+0x6C50	0x7168	# <CJK>
+0x6C51	0x7145	# <CJK>
+0x6C52	0x7172	# <CJK>
+0x6C53	0x714A	# <CJK>
+0x6C54	0x7178	# <CJK>
+0x6C55	0x717A	# <CJK>
+0x6C56	0x7198	# <CJK>
+0x6C57	0x71B3	# <CJK>
+0x6C58	0x71B5	# <CJK>
+0x6C59	0x71A8	# <CJK>
+0x6C5A	0x71A0	# <CJK>
+0x6C5B	0x71E0	# <CJK>
+0x6C5C	0x71D4	# <CJK>
+0x6C5D	0x71E7	# <CJK>
+0x6C5E	0x71F9	# <CJK>
+0x6C5F	0x721D	# <CJK>
+0x6C60	0x7228	# <CJK>
+0x6C61	0x706C	# <CJK>
+0x6C62	0x7118	# <CJK>
+0x6C63	0x7166	# <CJK>
+0x6C64	0x71B9	# <CJK>
+0x6C65	0x623E	# <CJK>
+0x6C66	0x623D	# <CJK>
+0x6C67	0x6243	# <CJK>
+0x6C68	0x6248	# <CJK>
+0x6C69	0x6249	# <CJK>
+0x6C6A	0x793B	# <CJK>
+0x6C6B	0x7940	# <CJK>
+0x6C6C	0x7946	# <CJK>
+0x6C6D	0x7949	# <CJK>
+0x6C6E	0x795B	# <CJK>
+0x6C6F	0x795C	# <CJK>
+0x6C70	0x7953	# <CJK>
+0x6C71	0x795A	# <CJK>
+0x6C72	0x7962	# <CJK>
+0x6C73	0x7957	# <CJK>
+0x6C74	0x7960	# <CJK>
+0x6C75	0x796F	# <CJK>
+0x6C76	0x7967	# <CJK>
+0x6C77	0x797A	# <CJK>
+0x6C78	0x7985	# <CJK>
+0x6C79	0x798A	# <CJK>
+0x6C7A	0x799A	# <CJK>
+0x6C7B	0x79A7	# <CJK>
+0x6C7C	0x79B3	# <CJK>
+0x6C7D	0x5FD1	# <CJK>
+0x6C7E	0x5FD0	# <CJK>
+0x6D21	0x603C	# <CJK>
+0x6D22	0x605D	# <CJK>
+0x6D23	0x605A	# <CJK>
+0x6D24	0x6067	# <CJK>
+0x6D25	0x6041	# <CJK>
+0x6D26	0x6059	# <CJK>
+0x6D27	0x6063	# <CJK>
+0x6D28	0x60AB	# <CJK>
+0x6D29	0x6106	# <CJK>
+0x6D2A	0x610D	# <CJK>
+0x6D2B	0x615D	# <CJK>
+0x6D2C	0x61A9	# <CJK>
+0x6D2D	0x619D	# <CJK>
+0x6D2E	0x61CB	# <CJK>
+0x6D2F	0x61D1	# <CJK>
+0x6D30	0x6206	# <CJK>
+0x6D31	0x8080	# <CJK>
+0x6D32	0x807F	# <CJK>
+0x6D33	0x6C93	# <CJK>
+0x6D34	0x6CF6	# <CJK>
+0x6D35	0x6DFC	# <CJK>
+0x6D36	0x77F6	# <CJK>
+0x6D37	0x77F8	# <CJK>
+0x6D38	0x7800	# <CJK>
+0x6D39	0x7809	# <CJK>
+0x6D3A	0x7817	# <CJK>
+0x6D3B	0x7818	# <CJK>
+0x6D3C	0x7811	# <CJK>
+0x6D3D	0x65AB	# <CJK>
+0x6D3E	0x782D	# <CJK>
+0x6D3F	0x781C	# <CJK>
+0x6D40	0x781D	# <CJK>
+0x6D41	0x7839	# <CJK>
+0x6D42	0x783A	# <CJK>
+0x6D43	0x783B	# <CJK>
+0x6D44	0x781F	# <CJK>
+0x6D45	0x783C	# <CJK>
+0x6D46	0x7825	# <CJK>
+0x6D47	0x782C	# <CJK>
+0x6D48	0x7823	# <CJK>
+0x6D49	0x7829	# <CJK>
+0x6D4A	0x784E	# <CJK>
+0x6D4B	0x786D	# <CJK>
+0x6D4C	0x7856	# <CJK>
+0x6D4D	0x7857	# <CJK>
+0x6D4E	0x7826	# <CJK>
+0x6D4F	0x7850	# <CJK>
+0x6D50	0x7847	# <CJK>
+0x6D51	0x784C	# <CJK>
+0x6D52	0x786A	# <CJK>
+0x6D53	0x789B	# <CJK>
+0x6D54	0x7893	# <CJK>
+0x6D55	0x789A	# <CJK>
+0x6D56	0x7887	# <CJK>
+0x6D57	0x789C	# <CJK>
+0x6D58	0x78A1	# <CJK>
+0x6D59	0x78A3	# <CJK>
+0x6D5A	0x78B2	# <CJK>
+0x6D5B	0x78B9	# <CJK>
+0x6D5C	0x78A5	# <CJK>
+0x6D5D	0x78D4	# <CJK>
+0x6D5E	0x78D9	# <CJK>
+0x6D5F	0x78C9	# <CJK>
+0x6D60	0x78EC	# <CJK>
+0x6D61	0x78F2	# <CJK>
+0x6D62	0x7905	# <CJK>
+0x6D63	0x78F4	# <CJK>
+0x6D64	0x7913	# <CJK>
+0x6D65	0x7924	# <CJK>
+0x6D66	0x791E	# <CJK>
+0x6D67	0x7934	# <CJK>
+0x6D68	0x9F9B	# <CJK>
+0x6D69	0x9EF9	# <CJK>
+0x6D6A	0x9EFB	# <CJK>
+0x6D6B	0x9EFC	# <CJK>
+0x6D6C	0x76F1	# <CJK>
+0x6D6D	0x7704	# <CJK>
+0x6D6E	0x770D	# <CJK>
+0x6D6F	0x76F9	# <CJK>
+0x6D70	0x7707	# <CJK>
+0x6D71	0x7708	# <CJK>
+0x6D72	0x771A	# <CJK>
+0x6D73	0x7722	# <CJK>
+0x6D74	0x7719	# <CJK>
+0x6D75	0x772D	# <CJK>
+0x6D76	0x7726	# <CJK>
+0x6D77	0x7735	# <CJK>
+0x6D78	0x7738	# <CJK>
+0x6D79	0x7750	# <CJK>
+0x6D7A	0x7751	# <CJK>
+0x6D7B	0x7747	# <CJK>
+0x6D7C	0x7743	# <CJK>
+0x6D7D	0x775A	# <CJK>
+0x6D7E	0x7768	# <CJK>
+0x6E21	0x7762	# <CJK>
+0x6E22	0x7765	# <CJK>
+0x6E23	0x777F	# <CJK>
+0x6E24	0x778D	# <CJK>
+0x6E25	0x777D	# <CJK>
+0x6E26	0x7780	# <CJK>
+0x6E27	0x778C	# <CJK>
+0x6E28	0x7791	# <CJK>
+0x6E29	0x779F	# <CJK>
+0x6E2A	0x77A0	# <CJK>
+0x6E2B	0x77B0	# <CJK>
+0x6E2C	0x77B5	# <CJK>
+0x6E2D	0x77BD	# <CJK>
+0x6E2E	0x753A	# <CJK>
+0x6E2F	0x7540	# <CJK>
+0x6E30	0x754E	# <CJK>
+0x6E31	0x754B	# <CJK>
+0x6E32	0x7548	# <CJK>
+0x6E33	0x755B	# <CJK>
+0x6E34	0x7572	# <CJK>
+0x6E35	0x7579	# <CJK>
+0x6E36	0x7583	# <CJK>
+0x6E37	0x7F58	# <CJK>
+0x6E38	0x7F61	# <CJK>
+0x6E39	0x7F5F	# <CJK>
+0x6E3A	0x8A48	# <CJK>
+0x6E3B	0x7F68	# <CJK>
+0x6E3C	0x7F74	# <CJK>
+0x6E3D	0x7F71	# <CJK>
+0x6E3E	0x7F79	# <CJK>
+0x6E3F	0x7F81	# <CJK>
+0x6E40	0x7F7E	# <CJK>
+0x6E41	0x76CD	# <CJK>
+0x6E42	0x76E5	# <CJK>
+0x6E43	0x8832	# <CJK>
+0x6E44	0x9485	# <CJK>
+0x6E45	0x9486	# <CJK>
+0x6E46	0x9487	# <CJK>
+0x6E47	0x948B	# <CJK>
+0x6E48	0x948A	# <CJK>
+0x6E49	0x948C	# <CJK>
+0x6E4A	0x948D	# <CJK>
+0x6E4B	0x948F	# <CJK>
+0x6E4C	0x9490	# <CJK>
+0x6E4D	0x9494	# <CJK>
+0x6E4E	0x9497	# <CJK>
+0x6E4F	0x9495	# <CJK>
+0x6E50	0x949A	# <CJK>
+0x6E51	0x949B	# <CJK>
+0x6E52	0x949C	# <CJK>
+0x6E53	0x94A3	# <CJK>
+0x6E54	0x94A4	# <CJK>
+0x6E55	0x94AB	# <CJK>
+0x6E56	0x94AA	# <CJK>
+0x6E57	0x94AD	# <CJK>
+0x6E58	0x94AC	# <CJK>
+0x6E59	0x94AF	# <CJK>
+0x6E5A	0x94B0	# <CJK>
+0x6E5B	0x94B2	# <CJK>
+0x6E5C	0x94B4	# <CJK>
+0x6E5D	0x94B6	# <CJK>
+0x6E5E	0x94B7	# <CJK>
+0x6E5F	0x94B8	# <CJK>
+0x6E60	0x94B9	# <CJK>
+0x6E61	0x94BA	# <CJK>
+0x6E62	0x94BC	# <CJK>
+0x6E63	0x94BD	# <CJK>
+0x6E64	0x94BF	# <CJK>
+0x6E65	0x94C4	# <CJK>
+0x6E66	0x94C8	# <CJK>
+0x6E67	0x94C9	# <CJK>
+0x6E68	0x94CA	# <CJK>
+0x6E69	0x94CB	# <CJK>
+0x6E6A	0x94CC	# <CJK>
+0x6E6B	0x94CD	# <CJK>
+0x6E6C	0x94CE	# <CJK>
+0x6E6D	0x94D0	# <CJK>
+0x6E6E	0x94D1	# <CJK>
+0x6E6F	0x94D2	# <CJK>
+0x6E70	0x94D5	# <CJK>
+0x6E71	0x94D6	# <CJK>
+0x6E72	0x94D7	# <CJK>
+0x6E73	0x94D9	# <CJK>
+0x6E74	0x94D8	# <CJK>
+0x6E75	0x94DB	# <CJK>
+0x6E76	0x94DE	# <CJK>
+0x6E77	0x94DF	# <CJK>
+0x6E78	0x94E0	# <CJK>
+0x6E79	0x94E2	# <CJK>
+0x6E7A	0x94E4	# <CJK>
+0x6E7B	0x94E5	# <CJK>
+0x6E7C	0x94E7	# <CJK>
+0x6E7D	0x94E8	# <CJK>
+0x6E7E	0x94EA	# <CJK>
+0x6F21	0x94E9	# <CJK>
+0x6F22	0x94EB	# <CJK>
+0x6F23	0x94EE	# <CJK>
+0x6F24	0x94EF	# <CJK>
+0x6F25	0x94F3	# <CJK>
+0x6F26	0x94F4	# <CJK>
+0x6F27	0x94F5	# <CJK>
+0x6F28	0x94F7	# <CJK>
+0x6F29	0x94F9	# <CJK>
+0x6F2A	0x94FC	# <CJK>
+0x6F2B	0x94FD	# <CJK>
+0x6F2C	0x94FF	# <CJK>
+0x6F2D	0x9503	# <CJK>
+0x6F2E	0x9502	# <CJK>
+0x6F2F	0x9506	# <CJK>
+0x6F30	0x9507	# <CJK>
+0x6F31	0x9509	# <CJK>
+0x6F32	0x950A	# <CJK>
+0x6F33	0x950D	# <CJK>
+0x6F34	0x950E	# <CJK>
+0x6F35	0x950F	# <CJK>
+0x6F36	0x9512	# <CJK>
+0x6F37	0x9513	# <CJK>
+0x6F38	0x9514	# <CJK>
+0x6F39	0x9515	# <CJK>
+0x6F3A	0x9516	# <CJK>
+0x6F3B	0x9518	# <CJK>
+0x6F3C	0x951B	# <CJK>
+0x6F3D	0x951D	# <CJK>
+0x6F3E	0x951E	# <CJK>
+0x6F3F	0x951F	# <CJK>
+0x6F40	0x9522	# <CJK>
+0x6F41	0x952A	# <CJK>
+0x6F42	0x952B	# <CJK>
+0x6F43	0x9529	# <CJK>
+0x6F44	0x952C	# <CJK>
+0x6F45	0x9531	# <CJK>
+0x6F46	0x9532	# <CJK>
+0x6F47	0x9534	# <CJK>
+0x6F48	0x9536	# <CJK>
+0x6F49	0x9537	# <CJK>
+0x6F4A	0x9538	# <CJK>
+0x6F4B	0x953C	# <CJK>
+0x6F4C	0x953E	# <CJK>
+0x6F4D	0x953F	# <CJK>
+0x6F4E	0x9542	# <CJK>
+0x6F4F	0x9535	# <CJK>
+0x6F50	0x9544	# <CJK>
+0x6F51	0x9545	# <CJK>
+0x6F52	0x9546	# <CJK>
+0x6F53	0x9549	# <CJK>
+0x6F54	0x954C	# <CJK>
+0x6F55	0x954E	# <CJK>
+0x6F56	0x954F	# <CJK>
+0x6F57	0x9552	# <CJK>
+0x6F58	0x9553	# <CJK>
+0x6F59	0x9554	# <CJK>
+0x6F5A	0x9556	# <CJK>
+0x6F5B	0x9557	# <CJK>
+0x6F5C	0x9558	# <CJK>
+0x6F5D	0x9559	# <CJK>
+0x6F5E	0x955B	# <CJK>
+0x6F5F	0x955E	# <CJK>
+0x6F60	0x955F	# <CJK>
+0x6F61	0x955D	# <CJK>
+0x6F62	0x9561	# <CJK>
+0x6F63	0x9562	# <CJK>
+0x6F64	0x9564	# <CJK>
+0x6F65	0x9565	# <CJK>
+0x6F66	0x9566	# <CJK>
+0x6F67	0x9567	# <CJK>
+0x6F68	0x9568	# <CJK>
+0x6F69	0x9569	# <CJK>
+0x6F6A	0x956A	# <CJK>
+0x6F6B	0x956B	# <CJK>
+0x6F6C	0x956C	# <CJK>
+0x6F6D	0x956F	# <CJK>
+0x6F6E	0x9571	# <CJK>
+0x6F6F	0x9572	# <CJK>
+0x6F70	0x9573	# <CJK>
+0x6F71	0x953A	# <CJK>
+0x6F72	0x77E7	# <CJK>
+0x6F73	0x77EC	# <CJK>
+0x6F74	0x96C9	# <CJK>
+0x6F75	0x79D5	# <CJK>
+0x6F76	0x79ED	# <CJK>
+0x6F77	0x79E3	# <CJK>
+0x6F78	0x79EB	# <CJK>
+0x6F79	0x7A06	# <CJK>
+0x6F7A	0x5D47	# <CJK>
+0x6F7B	0x7A03	# <CJK>
+0x6F7C	0x7A02	# <CJK>
+0x6F7D	0x7A1E	# <CJK>
+0x6F7E	0x7A14	# <CJK>
+0x7021	0x7A39	# <CJK>
+0x7022	0x7A37	# <CJK>
+0x7023	0x7A51	# <CJK>
+0x7024	0x9ECF	# <CJK>
+0x7025	0x99A5	# <CJK>
+0x7026	0x7A70	# <CJK>
+0x7027	0x7688	# <CJK>
+0x7028	0x768E	# <CJK>
+0x7029	0x7693	# <CJK>
+0x702A	0x7699	# <CJK>
+0x702B	0x76A4	# <CJK>
+0x702C	0x74DE	# <CJK>
+0x702D	0x74E0	# <CJK>
+0x702E	0x752C	# <CJK>
+0x702F	0x9E20	# <CJK>
+0x7030	0x9E22	# <CJK>
+0x7031	0x9E28	# <CJK>
+0x7032	0x9E29	# <CJK>
+0x7033	0x9E2A	# <CJK>
+0x7034	0x9E2B	# <CJK>
+0x7035	0x9E2C	# <CJK>
+0x7036	0x9E32	# <CJK>
+0x7037	0x9E31	# <CJK>
+0x7038	0x9E36	# <CJK>
+0x7039	0x9E38	# <CJK>
+0x703A	0x9E37	# <CJK>
+0x703B	0x9E39	# <CJK>
+0x703C	0x9E3A	# <CJK>
+0x703D	0x9E3E	# <CJK>
+0x703E	0x9E41	# <CJK>
+0x703F	0x9E42	# <CJK>
+0x7040	0x9E44	# <CJK>
+0x7041	0x9E46	# <CJK>
+0x7042	0x9E47	# <CJK>
+0x7043	0x9E48	# <CJK>
+0x7044	0x9E49	# <CJK>
+0x7045	0x9E4B	# <CJK>
+0x7046	0x9E4C	# <CJK>
+0x7047	0x9E4E	# <CJK>
+0x7048	0x9E51	# <CJK>
+0x7049	0x9E55	# <CJK>
+0x704A	0x9E57	# <CJK>
+0x704B	0x9E5A	# <CJK>
+0x704C	0x9E5B	# <CJK>
+0x704D	0x9E5C	# <CJK>
+0x704E	0x9E5E	# <CJK>
+0x704F	0x9E63	# <CJK>
+0x7050	0x9E66	# <CJK>
+0x7051	0x9E67	# <CJK>
+0x7052	0x9E68	# <CJK>
+0x7053	0x9E69	# <CJK>
+0x7054	0x9E6A	# <CJK>
+0x7055	0x9E6B	# <CJK>
+0x7056	0x9E6C	# <CJK>
+0x7057	0x9E71	# <CJK>
+0x7058	0x9E6D	# <CJK>
+0x7059	0x9E73	# <CJK>
+0x705A	0x7592	# <CJK>
+0x705B	0x7594	# <CJK>
+0x705C	0x7596	# <CJK>
+0x705D	0x75A0	# <CJK>
+0x705E	0x759D	# <CJK>
+0x705F	0x75AC	# <CJK>
+0x7060	0x75A3	# <CJK>
+0x7061	0x75B3	# <CJK>
+0x7062	0x75B4	# <CJK>
+0x7063	0x75B8	# <CJK>
+0x7064	0x75C4	# <CJK>
+0x7065	0x75B1	# <CJK>
+0x7066	0x75B0	# <CJK>
+0x7067	0x75C3	# <CJK>
+0x7068	0x75C2	# <CJK>
+0x7069	0x75D6	# <CJK>
+0x706A	0x75CD	# <CJK>
+0x706B	0x75E3	# <CJK>
+0x706C	0x75E8	# <CJK>
+0x706D	0x75E6	# <CJK>
+0x706E	0x75E4	# <CJK>
+0x706F	0x75EB	# <CJK>
+0x7070	0x75E7	# <CJK>
+0x7071	0x7603	# <CJK>
+0x7072	0x75F1	# <CJK>
+0x7073	0x75FC	# <CJK>
+0x7074	0x75FF	# <CJK>
+0x7075	0x7610	# <CJK>
+0x7076	0x7600	# <CJK>
+0x7077	0x7605	# <CJK>
+0x7078	0x760C	# <CJK>
+0x7079	0x7617	# <CJK>
+0x707A	0x760A	# <CJK>
+0x707B	0x7625	# <CJK>
+0x707C	0x7618	# <CJK>
+0x707D	0x7615	# <CJK>
+0x707E	0x7619	# <CJK>
+0x7121	0x761B	# <CJK>
+0x7122	0x763C	# <CJK>
+0x7123	0x7622	# <CJK>
+0x7124	0x7620	# <CJK>
+0x7125	0x7640	# <CJK>
+0x7126	0x762D	# <CJK>
+0x7127	0x7630	# <CJK>
+0x7128	0x763F	# <CJK>
+0x7129	0x7635	# <CJK>
+0x712A	0x7643	# <CJK>
+0x712B	0x763E	# <CJK>
+0x712C	0x7633	# <CJK>
+0x712D	0x764D	# <CJK>
+0x712E	0x765E	# <CJK>
+0x712F	0x7654	# <CJK>
+0x7130	0x765C	# <CJK>
+0x7131	0x7656	# <CJK>
+0x7132	0x766B	# <CJK>
+0x7133	0x766F	# <CJK>
+0x7134	0x7FCA	# <CJK>
+0x7135	0x7AE6	# <CJK>
+0x7136	0x7A78	# <CJK>
+0x7137	0x7A79	# <CJK>
+0x7138	0x7A80	# <CJK>
+0x7139	0x7A86	# <CJK>
+0x713A	0x7A88	# <CJK>
+0x713B	0x7A95	# <CJK>
+0x713C	0x7AA6	# <CJK>
+0x713D	0x7AA0	# <CJK>
+0x713E	0x7AAC	# <CJK>
+0x713F	0x7AA8	# <CJK>
+0x7140	0x7AAD	# <CJK>
+0x7141	0x7AB3	# <CJK>
+0x7142	0x8864	# <CJK>
+0x7143	0x8869	# <CJK>
+0x7144	0x8872	# <CJK>
+0x7145	0x887D	# <CJK>
+0x7146	0x887F	# <CJK>
+0x7147	0x8882	# <CJK>
+0x7148	0x88A2	# <CJK>
+0x7149	0x88C6	# <CJK>
+0x714A	0x88B7	# <CJK>
+0x714B	0x88BC	# <CJK>
+0x714C	0x88C9	# <CJK>
+0x714D	0x88E2	# <CJK>
+0x714E	0x88CE	# <CJK>
+0x714F	0x88E3	# <CJK>
+0x7150	0x88E5	# <CJK>
+0x7151	0x88F1	# <CJK>
+0x7152	0x891A	# <CJK>
+0x7153	0x88FC	# <CJK>
+0x7154	0x88E8	# <CJK>
+0x7155	0x88FE	# <CJK>
+0x7156	0x88F0	# <CJK>
+0x7157	0x8921	# <CJK>
+0x7158	0x8919	# <CJK>
+0x7159	0x8913	# <CJK>
+0x715A	0x891B	# <CJK>
+0x715B	0x890A	# <CJK>
+0x715C	0x8934	# <CJK>
+0x715D	0x892B	# <CJK>
+0x715E	0x8936	# <CJK>
+0x715F	0x8941	# <CJK>
+0x7160	0x8966	# <CJK>
+0x7161	0x897B	# <CJK>
+0x7162	0x758B	# <CJK>
+0x7163	0x80E5	# <CJK>
+0x7164	0x76B2	# <CJK>
+0x7165	0x76B4	# <CJK>
+0x7166	0x77DC	# <CJK>
+0x7167	0x8012	# <CJK>
+0x7168	0x8014	# <CJK>
+0x7169	0x8016	# <CJK>
+0x716A	0x801C	# <CJK>
+0x716B	0x8020	# <CJK>
+0x716C	0x8022	# <CJK>
+0x716D	0x8025	# <CJK>
+0x716E	0x8026	# <CJK>
+0x716F	0x8027	# <CJK>
+0x7170	0x8029	# <CJK>
+0x7171	0x8028	# <CJK>
+0x7172	0x8031	# <CJK>
+0x7173	0x800B	# <CJK>
+0x7174	0x8035	# <CJK>
+0x7175	0x8043	# <CJK>
+0x7176	0x8046	# <CJK>
+0x7177	0x804D	# <CJK>
+0x7178	0x8052	# <CJK>
+0x7179	0x8069	# <CJK>
+0x717A	0x8071	# <CJK>
+0x717B	0x8983	# <CJK>
+0x717C	0x9878	# <CJK>
+0x717D	0x9880	# <CJK>
+0x717E	0x9883	# <CJK>
+0x7221	0x9889	# <CJK>
+0x7222	0x988C	# <CJK>
+0x7223	0x988D	# <CJK>
+0x7224	0x988F	# <CJK>
+0x7225	0x9894	# <CJK>
+0x7226	0x989A	# <CJK>
+0x7227	0x989B	# <CJK>
+0x7228	0x989E	# <CJK>
+0x7229	0x989F	# <CJK>
+0x722A	0x98A1	# <CJK>
+0x722B	0x98A2	# <CJK>
+0x722C	0x98A5	# <CJK>
+0x722D	0x98A6	# <CJK>
+0x722E	0x864D	# <CJK>
+0x722F	0x8654	# <CJK>
+0x7230	0x866C	# <CJK>
+0x7231	0x866E	# <CJK>
+0x7232	0x867F	# <CJK>
+0x7233	0x867A	# <CJK>
+0x7234	0x867C	# <CJK>
+0x7235	0x867B	# <CJK>
+0x7236	0x86A8	# <CJK>
+0x7237	0x868D	# <CJK>
+0x7238	0x868B	# <CJK>
+0x7239	0x86AC	# <CJK>
+0x723A	0x869D	# <CJK>
+0x723B	0x86A7	# <CJK>
+0x723C	0x86A3	# <CJK>
+0x723D	0x86AA	# <CJK>
+0x723E	0x8693	# <CJK>
+0x723F	0x86A9	# <CJK>
+0x7240	0x86B6	# <CJK>
+0x7241	0x86C4	# <CJK>
+0x7242	0x86B5	# <CJK>
+0x7243	0x86CE	# <CJK>
+0x7244	0x86B0	# <CJK>
+0x7245	0x86BA	# <CJK>
+0x7246	0x86B1	# <CJK>
+0x7247	0x86AF	# <CJK>
+0x7248	0x86C9	# <CJK>
+0x7249	0x86CF	# <CJK>
+0x724A	0x86B4	# <CJK>
+0x724B	0x86E9	# <CJK>
+0x724C	0x86F1	# <CJK>
+0x724D	0x86F2	# <CJK>
+0x724E	0x86ED	# <CJK>
+0x724F	0x86F3	# <CJK>
+0x7250	0x86D0	# <CJK>
+0x7251	0x8713	# <CJK>
+0x7252	0x86DE	# <CJK>
+0x7253	0x86F4	# <CJK>
+0x7254	0x86DF	# <CJK>
+0x7255	0x86D8	# <CJK>
+0x7256	0x86D1	# <CJK>
+0x7257	0x8703	# <CJK>
+0x7258	0x8707	# <CJK>
+0x7259	0x86F8	# <CJK>
+0x725A	0x8708	# <CJK>
+0x725B	0x870A	# <CJK>
+0x725C	0x870D	# <CJK>
+0x725D	0x8709	# <CJK>
+0x725E	0x8723	# <CJK>
+0x725F	0x873B	# <CJK>
+0x7260	0x871E	# <CJK>
+0x7261	0x8725	# <CJK>
+0x7262	0x872E	# <CJK>
+0x7263	0x871A	# <CJK>
+0x7264	0x873E	# <CJK>
+0x7265	0x8748	# <CJK>
+0x7266	0x8734	# <CJK>
+0x7267	0x8731	# <CJK>
+0x7268	0x8729	# <CJK>
+0x7269	0x8737	# <CJK>
+0x726A	0x873F	# <CJK>
+0x726B	0x8782	# <CJK>
+0x726C	0x8722	# <CJK>
+0x726D	0x877D	# <CJK>
+0x726E	0x877E	# <CJK>
+0x726F	0x877B	# <CJK>
+0x7270	0x8760	# <CJK>
+0x7271	0x8770	# <CJK>
+0x7272	0x874C	# <CJK>
+0x7273	0x876E	# <CJK>
+0x7274	0x878B	# <CJK>
+0x7275	0x8753	# <CJK>
+0x7276	0x8763	# <CJK>
+0x7277	0x877C	# <CJK>
+0x7278	0x8764	# <CJK>
+0x7279	0x8759	# <CJK>
+0x727A	0x8765	# <CJK>
+0x727B	0x8793	# <CJK>
+0x727C	0x87AF	# <CJK>
+0x727D	0x87A8	# <CJK>
+0x727E	0x87D2	# <CJK>
+0x7321	0x87C6	# <CJK>
+0x7322	0x8788	# <CJK>
+0x7323	0x8785	# <CJK>
+0x7324	0x87AD	# <CJK>
+0x7325	0x8797	# <CJK>
+0x7326	0x8783	# <CJK>
+0x7327	0x87AB	# <CJK>
+0x7328	0x87E5	# <CJK>
+0x7329	0x87AC	# <CJK>
+0x732A	0x87B5	# <CJK>
+0x732B	0x87B3	# <CJK>
+0x732C	0x87CB	# <CJK>
+0x732D	0x87D3	# <CJK>
+0x732E	0x87BD	# <CJK>
+0x732F	0x87D1	# <CJK>
+0x7330	0x87C0	# <CJK>
+0x7331	0x87CA	# <CJK>
+0x7332	0x87DB	# <CJK>
+0x7333	0x87EA	# <CJK>
+0x7334	0x87E0	# <CJK>
+0x7335	0x87EE	# <CJK>
+0x7336	0x8816	# <CJK>
+0x7337	0x8813	# <CJK>
+0x7338	0x87FE	# <CJK>
+0x7339	0x880A	# <CJK>
+0x733A	0x881B	# <CJK>
+0x733B	0x8821	# <CJK>
+0x733C	0x8839	# <CJK>
+0x733D	0x883C	# <CJK>
+0x733E	0x7F36	# <CJK>
+0x733F	0x7F42	# <CJK>
+0x7340	0x7F44	# <CJK>
+0x7341	0x7F45	# <CJK>
+0x7342	0x8210	# <CJK>
+0x7343	0x7AFA	# <CJK>
+0x7344	0x7AFD	# <CJK>
+0x7345	0x7B08	# <CJK>
+0x7346	0x7B03	# <CJK>
+0x7347	0x7B04	# <CJK>
+0x7348	0x7B15	# <CJK>
+0x7349	0x7B0A	# <CJK>
+0x734A	0x7B2B	# <CJK>
+0x734B	0x7B0F	# <CJK>
+0x734C	0x7B47	# <CJK>
+0x734D	0x7B38	# <CJK>
+0x734E	0x7B2A	# <CJK>
+0x734F	0x7B19	# <CJK>
+0x7350	0x7B2E	# <CJK>
+0x7351	0x7B31	# <CJK>
+0x7352	0x7B20	# <CJK>
+0x7353	0x7B25	# <CJK>
+0x7354	0x7B24	# <CJK>
+0x7355	0x7B33	# <CJK>
+0x7356	0x7B3E	# <CJK>
+0x7357	0x7B1E	# <CJK>
+0x7358	0x7B58	# <CJK>
+0x7359	0x7B5A	# <CJK>
+0x735A	0x7B45	# <CJK>
+0x735B	0x7B75	# <CJK>
+0x735C	0x7B4C	# <CJK>
+0x735D	0x7B5D	# <CJK>
+0x735E	0x7B60	# <CJK>
+0x735F	0x7B6E	# <CJK>
+0x7360	0x7B7B	# <CJK>
+0x7361	0x7B62	# <CJK>
+0x7362	0x7B72	# <CJK>
+0x7363	0x7B71	# <CJK>
+0x7364	0x7B90	# <CJK>
+0x7365	0x7BA6	# <CJK>
+0x7366	0x7BA7	# <CJK>
+0x7367	0x7BB8	# <CJK>
+0x7368	0x7BAC	# <CJK>
+0x7369	0x7B9D	# <CJK>
+0x736A	0x7BA8	# <CJK>
+0x736B	0x7B85	# <CJK>
+0x736C	0x7BAA	# <CJK>
+0x736D	0x7B9C	# <CJK>
+0x736E	0x7BA2	# <CJK>
+0x736F	0x7BAB	# <CJK>
+0x7370	0x7BB4	# <CJK>
+0x7371	0x7BD1	# <CJK>
+0x7372	0x7BC1	# <CJK>
+0x7373	0x7BCC	# <CJK>
+0x7374	0x7BDD	# <CJK>
+0x7375	0x7BDA	# <CJK>
+0x7376	0x7BE5	# <CJK>
+0x7377	0x7BE6	# <CJK>
+0x7378	0x7BEA	# <CJK>
+0x7379	0x7C0C	# <CJK>
+0x737A	0x7BFE	# <CJK>
+0x737B	0x7BFC	# <CJK>
+0x737C	0x7C0F	# <CJK>
+0x737D	0x7C16	# <CJK>
+0x737E	0x7C0B	# <CJK>
+0x7421	0x7C1F	# <CJK>
+0x7422	0x7C2A	# <CJK>
+0x7423	0x7C26	# <CJK>
+0x7424	0x7C38	# <CJK>
+0x7425	0x7C41	# <CJK>
+0x7426	0x7C40	# <CJK>
+0x7427	0x81FE	# <CJK>
+0x7428	0x8201	# <CJK>
+0x7429	0x8202	# <CJK>
+0x742A	0x8204	# <CJK>
+0x742B	0x81EC	# <CJK>
+0x742C	0x8844	# <CJK>
+0x742D	0x8221	# <CJK>
+0x742E	0x8222	# <CJK>
+0x742F	0x8223	# <CJK>
+0x7430	0x822D	# <CJK>
+0x7431	0x822F	# <CJK>
+0x7432	0x8228	# <CJK>
+0x7433	0x822B	# <CJK>
+0x7434	0x8238	# <CJK>
+0x7435	0x823B	# <CJK>
+0x7436	0x8233	# <CJK>
+0x7437	0x8234	# <CJK>
+0x7438	0x823E	# <CJK>
+0x7439	0x8244	# <CJK>
+0x743A	0x8249	# <CJK>
+0x743B	0x824B	# <CJK>
+0x743C	0x824F	# <CJK>
+0x743D	0x825A	# <CJK>
+0x743E	0x825F	# <CJK>
+0x743F	0x8268	# <CJK>
+0x7440	0x887E	# <CJK>
+0x7441	0x8885	# <CJK>
+0x7442	0x8888	# <CJK>
+0x7443	0x88D8	# <CJK>
+0x7444	0x88DF	# <CJK>
+0x7445	0x895E	# <CJK>
+0x7446	0x7F9D	# <CJK>
+0x7447	0x7F9F	# <CJK>
+0x7448	0x7FA7	# <CJK>
+0x7449	0x7FAF	# <CJK>
+0x744A	0x7FB0	# <CJK>
+0x744B	0x7FB2	# <CJK>
+0x744C	0x7C7C	# <CJK>
+0x744D	0x6549	# <CJK>
+0x744E	0x7C91	# <CJK>
+0x744F	0x7C9D	# <CJK>
+0x7450	0x7C9C	# <CJK>
+0x7451	0x7C9E	# <CJK>
+0x7452	0x7CA2	# <CJK>
+0x7453	0x7CB2	# <CJK>
+0x7454	0x7CBC	# <CJK>
+0x7455	0x7CBD	# <CJK>
+0x7456	0x7CC1	# <CJK>
+0x7457	0x7CC7	# <CJK>
+0x7458	0x7CCC	# <CJK>
+0x7459	0x7CCD	# <CJK>
+0x745A	0x7CC8	# <CJK>
+0x745B	0x7CC5	# <CJK>
+0x745C	0x7CD7	# <CJK>
+0x745D	0x7CE8	# <CJK>
+0x745E	0x826E	# <CJK>
+0x745F	0x66A8	# <CJK>
+0x7460	0x7FBF	# <CJK>
+0x7461	0x7FCE	# <CJK>
+0x7462	0x7FD5	# <CJK>
+0x7463	0x7FE5	# <CJK>
+0x7464	0x7FE1	# <CJK>
+0x7465	0x7FE6	# <CJK>
+0x7466	0x7FE9	# <CJK>
+0x7467	0x7FEE	# <CJK>
+0x7468	0x7FF3	# <CJK>
+0x7469	0x7CF8	# <CJK>
+0x746A	0x7D77	# <CJK>
+0x746B	0x7DA6	# <CJK>
+0x746C	0x7DAE	# <CJK>
+0x746D	0x7E47	# <CJK>
+0x746E	0x7E9B	# <CJK>
+0x746F	0x9EB8	# <CJK>
+0x7470	0x9EB4	# <CJK>
+0x7471	0x8D73	# <CJK>
+0x7472	0x8D84	# <CJK>
+0x7473	0x8D94	# <CJK>
+0x7474	0x8D91	# <CJK>
+0x7475	0x8DB1	# <CJK>
+0x7476	0x8D67	# <CJK>
+0x7477	0x8D6D	# <CJK>
+0x7478	0x8C47	# <CJK>
+0x7479	0x8C49	# <CJK>
+0x747A	0x914A	# <CJK>
+0x747B	0x9150	# <CJK>
+0x747C	0x914E	# <CJK>
+0x747D	0x914F	# <CJK>
+0x747E	0x9164	# <CJK>
+0x7521	0x9162	# <CJK>
+0x7522	0x9161	# <CJK>
+0x7523	0x9170	# <CJK>
+0x7524	0x9169	# <CJK>
+0x7525	0x916F	# <CJK>
+0x7526	0x917D	# <CJK>
+0x7527	0x917E	# <CJK>
+0x7528	0x9172	# <CJK>
+0x7529	0x9174	# <CJK>
+0x752A	0x9179	# <CJK>
+0x752B	0x918C	# <CJK>
+0x752C	0x9185	# <CJK>
+0x752D	0x9190	# <CJK>
+0x752E	0x918D	# <CJK>
+0x752F	0x9191	# <CJK>
+0x7530	0x91A2	# <CJK>
+0x7531	0x91A3	# <CJK>
+0x7532	0x91AA	# <CJK>
+0x7533	0x91AD	# <CJK>
+0x7534	0x91AE	# <CJK>
+0x7535	0x91AF	# <CJK>
+0x7536	0x91B5	# <CJK>
+0x7537	0x91B4	# <CJK>
+0x7538	0x91BA	# <CJK>
+0x7539	0x8C55	# <CJK>
+0x753A	0x9E7E	# <CJK>
+0x753B	0x8DB8	# <CJK>
+0x753C	0x8DEB	# <CJK>
+0x753D	0x8E05	# <CJK>
+0x753E	0x8E59	# <CJK>
+0x753F	0x8E69	# <CJK>
+0x7540	0x8DB5	# <CJK>
+0x7541	0x8DBF	# <CJK>
+0x7542	0x8DBC	# <CJK>
+0x7543	0x8DBA	# <CJK>
+0x7544	0x8DC4	# <CJK>
+0x7545	0x8DD6	# <CJK>
+0x7546	0x8DD7	# <CJK>
+0x7547	0x8DDA	# <CJK>
+0x7548	0x8DDE	# <CJK>
+0x7549	0x8DCE	# <CJK>
+0x754A	0x8DCF	# <CJK>
+0x754B	0x8DDB	# <CJK>
+0x754C	0x8DC6	# <CJK>
+0x754D	0x8DEC	# <CJK>
+0x754E	0x8DF7	# <CJK>
+0x754F	0x8DF8	# <CJK>
+0x7550	0x8DE3	# <CJK>
+0x7551	0x8DF9	# <CJK>
+0x7552	0x8DFB	# <CJK>
+0x7553	0x8DE4	# <CJK>
+0x7554	0x8E09	# <CJK>
+0x7555	0x8DFD	# <CJK>
+0x7556	0x8E14	# <CJK>
+0x7557	0x8E1D	# <CJK>
+0x7558	0x8E1F	# <CJK>
+0x7559	0x8E2C	# <CJK>
+0x755A	0x8E2E	# <CJK>
+0x755B	0x8E23	# <CJK>
+0x755C	0x8E2F	# <CJK>
+0x755D	0x8E3A	# <CJK>
+0x755E	0x8E40	# <CJK>
+0x755F	0x8E39	# <CJK>
+0x7560	0x8E35	# <CJK>
+0x7561	0x8E3D	# <CJK>
+0x7562	0x8E31	# <CJK>
+0x7563	0x8E49	# <CJK>
+0x7564	0x8E41	# <CJK>
+0x7565	0x8E42	# <CJK>
+0x7566	0x8E51	# <CJK>
+0x7567	0x8E52	# <CJK>
+0x7568	0x8E4A	# <CJK>
+0x7569	0x8E70	# <CJK>
+0x756A	0x8E76	# <CJK>
+0x756B	0x8E7C	# <CJK>
+0x756C	0x8E6F	# <CJK>
+0x756D	0x8E74	# <CJK>
+0x756E	0x8E85	# <CJK>
+0x756F	0x8E8F	# <CJK>
+0x7570	0x8E94	# <CJK>
+0x7571	0x8E90	# <CJK>
+0x7572	0x8E9C	# <CJK>
+0x7573	0x8E9E	# <CJK>
+0x7574	0x8C78	# <CJK>
+0x7575	0x8C82	# <CJK>
+0x7576	0x8C8A	# <CJK>
+0x7577	0x8C85	# <CJK>
+0x7578	0x8C98	# <CJK>
+0x7579	0x8C94	# <CJK>
+0x757A	0x659B	# <CJK>
+0x757B	0x89D6	# <CJK>
+0x757C	0x89DE	# <CJK>
+0x757D	0x89DA	# <CJK>
+0x757E	0x89DC	# <CJK>
+0x7621	0x89E5	# <CJK>
+0x7622	0x89EB	# <CJK>
+0x7623	0x89EF	# <CJK>
+0x7624	0x8A3E	# <CJK>
+0x7625	0x8B26	# <CJK>
+0x7626	0x9753	# <CJK>
+0x7627	0x96E9	# <CJK>
+0x7628	0x96F3	# <CJK>
+0x7629	0x96EF	# <CJK>
+0x762A	0x9706	# <CJK>
+0x762B	0x9701	# <CJK>
+0x762C	0x9708	# <CJK>
+0x762D	0x970F	# <CJK>
+0x762E	0x970E	# <CJK>
+0x762F	0x972A	# <CJK>
+0x7630	0x972D	# <CJK>
+0x7631	0x9730	# <CJK>
+0x7632	0x973E	# <CJK>
+0x7633	0x9F80	# <CJK>
+0x7634	0x9F83	# <CJK>
+0x7635	0x9F85	# <CJK>
+0x7636	0x9F86	# <CJK>
+0x7637	0x9F87	# <CJK>
+0x7638	0x9F88	# <CJK>
+0x7639	0x9F89	# <CJK>
+0x763A	0x9F8A	# <CJK>
+0x763B	0x9F8C	# <CJK>
+0x763C	0x9EFE	# <CJK>
+0x763D	0x9F0B	# <CJK>
+0x763E	0x9F0D	# <CJK>
+0x763F	0x96B9	# <CJK>
+0x7640	0x96BC	# <CJK>
+0x7641	0x96BD	# <CJK>
+0x7642	0x96CE	# <CJK>
+0x7643	0x96D2	# <CJK>
+0x7644	0x77BF	# <CJK>
+0x7645	0x96E0	# <CJK>
+0x7646	0x928E	# <CJK>
+0x7647	0x92AE	# <CJK>
+0x7648	0x92C8	# <CJK>
+0x7649	0x933E	# <CJK>
+0x764A	0x936A	# <CJK>
+0x764B	0x93CA	# <CJK>
+0x764C	0x938F	# <CJK>
+0x764D	0x943E	# <CJK>
+0x764E	0x946B	# <CJK>
+0x764F	0x9C7F	# <CJK>
+0x7650	0x9C82	# <CJK>
+0x7651	0x9C85	# <CJK>
+0x7652	0x9C86	# <CJK>
+0x7653	0x9C87	# <CJK>
+0x7654	0x9C88	# <CJK>
+0x7655	0x7A23	# <CJK>
+0x7656	0x9C8B	# <CJK>
+0x7657	0x9C8E	# <CJK>
+0x7658	0x9C90	# <CJK>
+0x7659	0x9C91	# <CJK>
+0x765A	0x9C92	# <CJK>
+0x765B	0x9C94	# <CJK>
+0x765C	0x9C95	# <CJK>
+0x765D	0x9C9A	# <CJK>
+0x765E	0x9C9B	# <CJK>
+0x765F	0x9C9E	# <CJK>
+0x7660	0x9C9F	# <CJK>
+0x7661	0x9CA0	# <CJK>
+0x7662	0x9CA1	# <CJK>
+0x7663	0x9CA2	# <CJK>
+0x7664	0x9CA3	# <CJK>
+0x7665	0x9CA5	# <CJK>
+0x7666	0x9CA6	# <CJK>
+0x7667	0x9CA7	# <CJK>
+0x7668	0x9CA8	# <CJK>
+0x7669	0x9CA9	# <CJK>
+0x766A	0x9CAB	# <CJK>
+0x766B	0x9CAD	# <CJK>
+0x766C	0x9CAE	# <CJK>
+0x766D	0x9CB0	# <CJK>
+0x766E	0x9CB1	# <CJK>
+0x766F	0x9CB2	# <CJK>
+0x7670	0x9CB3	# <CJK>
+0x7671	0x9CB4	# <CJK>
+0x7672	0x9CB5	# <CJK>
+0x7673	0x9CB6	# <CJK>
+0x7674	0x9CB7	# <CJK>
+0x7675	0x9CBA	# <CJK>
+0x7676	0x9CBB	# <CJK>
+0x7677	0x9CBC	# <CJK>
+0x7678	0x9CBD	# <CJK>
+0x7679	0x9CC4	# <CJK>
+0x767A	0x9CC5	# <CJK>
+0x767B	0x9CC6	# <CJK>
+0x767C	0x9CC7	# <CJK>
+0x767D	0x9CCA	# <CJK>
+0x767E	0x9CCB	# <CJK>
+0x7721	0x9CCC	# <CJK>
+0x7722	0x9CCD	# <CJK>
+0x7723	0x9CCE	# <CJK>
+0x7724	0x9CCF	# <CJK>
+0x7725	0x9CD0	# <CJK>
+0x7726	0x9CD3	# <CJK>
+0x7727	0x9CD4	# <CJK>
+0x7728	0x9CD5	# <CJK>
+0x7729	0x9CD7	# <CJK>
+0x772A	0x9CD8	# <CJK>
+0x772B	0x9CD9	# <CJK>
+0x772C	0x9CDC	# <CJK>
+0x772D	0x9CDD	# <CJK>
+0x772E	0x9CDF	# <CJK>
+0x772F	0x9CE2	# <CJK>
+0x7730	0x977C	# <CJK>
+0x7731	0x9785	# <CJK>
+0x7732	0x9791	# <CJK>
+0x7733	0x9792	# <CJK>
+0x7734	0x9794	# <CJK>
+0x7735	0x97AF	# <CJK>
+0x7736	0x97AB	# <CJK>
+0x7737	0x97A3	# <CJK>
+0x7738	0x97B2	# <CJK>
+0x7739	0x97B4	# <CJK>
+0x773A	0x9AB1	# <CJK>
+0x773B	0x9AB0	# <CJK>
+0x773C	0x9AB7	# <CJK>
+0x773D	0x9E58	# <CJK>
+0x773E	0x9AB6	# <CJK>
+0x773F	0x9ABA	# <CJK>
+0x7740	0x9ABC	# <CJK>
+0x7741	0x9AC1	# <CJK>
+0x7742	0x9AC0	# <CJK>
+0x7743	0x9AC5	# <CJK>
+0x7744	0x9AC2	# <CJK>
+0x7745	0x9ACB	# <CJK>
+0x7746	0x9ACC	# <CJK>
+0x7747	0x9AD1	# <CJK>
+0x7748	0x9B45	# <CJK>
+0x7749	0x9B43	# <CJK>
+0x774A	0x9B47	# <CJK>
+0x774B	0x9B49	# <CJK>
+0x774C	0x9B48	# <CJK>
+0x774D	0x9B4D	# <CJK>
+0x774E	0x9B51	# <CJK>
+0x774F	0x98E8	# <CJK>
+0x7750	0x990D	# <CJK>
+0x7751	0x992E	# <CJK>
+0x7752	0x9955	# <CJK>
+0x7753	0x9954	# <CJK>
+0x7754	0x9ADF	# <CJK>
+0x7755	0x9AE1	# <CJK>
+0x7756	0x9AE6	# <CJK>
+0x7757	0x9AEF	# <CJK>
+0x7758	0x9AEB	# <CJK>
+0x7759	0x9AFB	# <CJK>
+0x775A	0x9AED	# <CJK>
+0x775B	0x9AF9	# <CJK>
+0x775C	0x9B08	# <CJK>
+0x775D	0x9B0F	# <CJK>
+0x775E	0x9B13	# <CJK>
+0x775F	0x9B1F	# <CJK>
+0x7760	0x9B23	# <CJK>
+0x7761	0x9EBD	# <CJK>
+0x7762	0x9EBE	# <CJK>
+0x7763	0x7E3B	# <CJK>
+0x7764	0x9E82	# <CJK>
+0x7765	0x9E87	# <CJK>
+0x7766	0x9E88	# <CJK>
+0x7767	0x9E8B	# <CJK>
+0x7768	0x9E92	# <CJK>
+0x7769	0x93D6	# <CJK>
+0x776A	0x9E9D	# <CJK>
+0x776B	0x9E9F	# <CJK>
+0x776C	0x9EDB	# <CJK>
+0x776D	0x9EDC	# <CJK>
+0x776E	0x9EDD	# <CJK>
+0x776F	0x9EE0	# <CJK>
+0x7770	0x9EDF	# <CJK>
+0x7771	0x9EE2	# <CJK>
+0x7772	0x9EE9	# <CJK>
+0x7773	0x9EE7	# <CJK>
+0x7774	0x9EE5	# <CJK>
+0x7775	0x9EEA	# <CJK>
+0x7776	0x9EEF	# <CJK>
+0x7777	0x9F22	# <CJK>
+0x7778	0x9F2C	# <CJK>
+0x7779	0x9F2F	# <CJK>
+0x777A	0x9F39	# <CJK>
+0x777B	0x9F37	# <CJK>
+0x777C	0x9F3D	# <CJK>
+0x777D	0x9F3E	# <CJK>
+0x777E	0x9F44	# <CJK>
Index: /tags/gsdl-21/gsdl/unicode/MAPPINGS/EASTASIA/GB/GBK.TXT
===================================================================
--- /tags/gsdl-21/gsdl/unicode/MAPPINGS/EASTASIA/GB/GBK.TXT	(revision 699)
+++ /tags/gsdl-21/gsdl/unicode/MAPPINGS/EASTASIA/GB/GBK.TXT	(revision 699)
@@ -0,0 +1,23941 @@
+0x8140	0x4E02	
+0x8141	0x4E04	
+0x8142	0x4E05	
+0x8143	0x4E06	
+0x8144	0x4E0F	
+0x8145	0x4E12	
+0x8146	0x4E17	
+0x8147	0x4E1F	
+0x8148	0x4E20	
+0x8149	0x4E21	
+0x814A	0x4E23	
+0x814B	0x4E26	
+0x814C	0x4E29	
+0x814D	0x4E2E	
+0x814E	0x4E2F	
+0x814F	0x4E31	
+0x8150	0x4E33	
+0x8151	0x4E35	
+0x8152	0x4E37	
+0x8153	0x4E3C	
+0x8154	0x4E40	
+0x8155	0x4E41	
+0x8156	0x4E42	
+0x8157	0x4E44	
+0x8158	0x4E46	
+0x8159	0x4E4A	
+0x815A	0x4E51	
+0x815B	0x4E55	
+0x815C	0x4E57	
+0x815D	0x4E5A	
+0x815E	0x4E5B	
+0x815F	0x4E62	
+0x8160	0x4E63	
+0x8161	0x4E64	
+0x8162	0x4E65	
+0x8163	0x4E67	
+0x8164	0x4E68	
+0x8165	0x4E6A	
+0x8166	0x4E6B	
+0x8167	0x4E6C	
+0x8168	0x4E6D	
+0x8169	0x4E6E	
+0x816A	0x4E6F	
+0x816B	0x4E72	
+0x816C	0x4E74	
+0x816D	0x4E75	
+0x816E	0x4E76	
+0x816F	0x4E77	
+0x8170	0x4E78	
+0x8171	0x4E79	
+0x8172	0x4E7A	
+0x8173	0x4E7B	
+0x8174	0x4E7C	
+0x8175	0x4E7D	
+0x8176	0x4E7F	
+0x8177	0x4E80	
+0x8178	0x4E81	
+0x8179	0x4E82	
+0x817A	0x4E83	
+0x817B	0x4E84	
+0x817C	0x4E85	
+0x817D	0x4E87	
+0x817E	0x4E8A	
+0x8180	0x4E90	
+0x8181	0x4E96	
+0x8182	0x4E97	
+0x8183	0x4E99	
+0x8184	0x4E9C	
+0x8185	0x4E9D	
+0x8186	0x4E9E	
+0x8187	0x4EA3	
+0x8188	0x4EAA	
+0x8189	0x4EAF	
+0x818A	0x4EB0	
+0x818B	0x4EB1	
+0x818C	0x4EB4	
+0x818D	0x4EB6	
+0x818E	0x4EB7	
+0x818F	0x4EB8	
+0x8190	0x4EB9	
+0x8191	0x4EBC	
+0x8192	0x4EBD	
+0x8193	0x4EBE	
+0x8194	0x4EC8	
+0x8195	0x4ECC	
+0x8196	0x4ECF	
+0x8197	0x4ED0	
+0x8198	0x4ED2	
+0x8199	0x4EDA	
+0x819A	0x4EDB	
+0x819B	0x4EDC	
+0x819C	0x4EE0	
+0x819D	0x4EE2	
+0x819E	0x4EE6	
+0x819F	0x4EE7	
+0x81A0	0x4EE9	
+0x81A1	0x4EED	
+0x81A2	0x4EEE	
+0x81A3	0x4EEF	
+0x81A4	0x4EF1	
+0x81A5	0x4EF4	
+0x81A6	0x4EF8	
+0x81A7	0x4EF9	
+0x81A8	0x4EFA	
+0x81A9	0x4EFC	
+0x81AA	0x4EFE	
+0x81AB	0x4F00	
+0x81AC	0x4F02	
+0x81AD	0x4F03	
+0x81AE	0x4F04	
+0x81AF	0x4F05	
+0x81B0	0x4F06	
+0x81B1	0x4F07	
+0x81B2	0x4F08	
+0x81B3	0x4F0B	
+0x81B4	0x4F0C	
+0x81B5	0x4F12	
+0x81B6	0x4F13	
+0x81B7	0x4F14	
+0x81B8	0x4F15	
+0x81B9	0x4F16	
+0x81BA	0x4F1C	
+0x81BB	0x4F1D	
+0x81BC	0x4F21	
+0x81BD	0x4F23	
+0x81BE	0x4F28	
+0x81BF	0x4F29	
+0x81C0	0x4F2C	
+0x81C1	0x4F2D	
+0x81C2	0x4F2E	
+0x81C3	0x4F31	
+0x81C4	0x4F33	
+0x81C5	0x4F35	
+0x81C6	0x4F37	
+0x81C7	0x4F39	
+0x81C8	0x4F3B	
+0x81C9	0x4F3E	
+0x81CA	0x4F3F	
+0x81CB	0x4F40	
+0x81CC	0x4F41	
+0x81CD	0x4F42	
+0x81CE	0x4F44	
+0x81CF	0x4F45	
+0x81D0	0x4F47	
+0x81D1	0x4F48	
+0x81D2	0x4F49	
+0x81D3	0x4F4A	
+0x81D4	0x4F4B	
+0x81D5	0x4F4C	
+0x81D6	0x4F52	
+0x81D7	0x4F54	
+0x81D8	0x4F56	
+0x81D9	0x4F61	
+0x81DA	0x4F62	
+0x81DB	0x4F66	
+0x81DC	0x4F68	
+0x81DD	0x4F6A	
+0x81DE	0x4F6B	
+0x81DF	0x4F6D	
+0x81E0	0x4F6E	
+0x81E1	0x4F71	
+0x81E2	0x4F72	
+0x81E3	0x4F75	
+0x81E4	0x4F77	
+0x81E5	0x4F78	
+0x81E6	0x4F79	
+0x81E7	0x4F7A	
+0x81E8	0x4F7D	
+0x81E9	0x4F80	
+0x81EA	0x4F81	
+0x81EB	0x4F82	
+0x81EC	0x4F85	
+0x81ED	0x4F86	
+0x81EE	0x4F87	
+0x81EF	0x4F8A	
+0x81F0	0x4F8C	
+0x81F1	0x4F8E	
+0x81F2	0x4F90	
+0x81F3	0x4F92	
+0x81F4	0x4F93	
+0x81F5	0x4F95	
+0x81F6	0x4F96	
+0x81F7	0x4F98	
+0x81F8	0x4F99	
+0x81F9	0x4F9A	
+0x81FA	0x4F9C	
+0x81FB	0x4F9E	
+0x81FC	0x4F9F	
+0x81FD	0x4FA1	
+0x81FE	0x4FA2	
+0x8240	0x4FA4	
+0x8241	0x4FAB	
+0x8242	0x4FAD	
+0x8243	0x4FB0	
+0x8244	0x4FB1	
+0x8245	0x4FB2	
+0x8246	0x4FB3	
+0x8247	0x4FB4	
+0x8248	0x4FB6	
+0x8249	0x4FB7	
+0x824A	0x4FB8	
+0x824B	0x4FB9	
+0x824C	0x4FBA	
+0x824D	0x4FBB	
+0x824E	0x4FBC	
+0x824F	0x4FBD	
+0x8250	0x4FBE	
+0x8251	0x4FC0	
+0x8252	0x4FC1	
+0x8253	0x4FC2	
+0x8254	0x4FC6	
+0x8255	0x4FC7	
+0x8256	0x4FC8	
+0x8257	0x4FC9	
+0x8258	0x4FCB	
+0x8259	0x4FCC	
+0x825A	0x4FCD	
+0x825B	0x4FD2	
+0x825C	0x4FD3	
+0x825D	0x4FD4	
+0x825E	0x4FD5	
+0x825F	0x4FD6	
+0x8260	0x4FD9	
+0x8261	0x4FDB	
+0x8262	0x4FE0	
+0x8263	0x4FE2	
+0x8264	0x4FE4	
+0x8265	0x4FE5	
+0x8266	0x4FE7	
+0x8267	0x4FEB	
+0x8268	0x4FEC	
+0x8269	0x4FF0	
+0x826A	0x4FF2	
+0x826B	0x4FF4	
+0x826C	0x4FF5	
+0x826D	0x4FF6	
+0x826E	0x4FF7	
+0x826F	0x4FF9	
+0x8270	0x4FFB	
+0x8271	0x4FFC	
+0x8272	0x4FFD	
+0x8273	0x4FFF	
+0x8274	0x5000	
+0x8275	0x5001	
+0x8276	0x5002	
+0x8277	0x5003	
+0x8278	0x5004	
+0x8279	0x5005	
+0x827A	0x5006	
+0x827B	0x5007	
+0x827C	0x5008	
+0x827D	0x5009	
+0x827E	0x500A	
+0x8280	0x500B	
+0x8281	0x500E	
+0x8282	0x5010	
+0x8283	0x5011	
+0x8284	0x5013	
+0x8285	0x5015	
+0x8286	0x5016	
+0x8287	0x5017	
+0x8288	0x501B	
+0x8289	0x501D	
+0x828A	0x501E	
+0x828B	0x5020	
+0x828C	0x5022	
+0x828D	0x5023	
+0x828E	0x5024	
+0x828F	0x5027	
+0x8290	0x502B	
+0x8291	0x502F	
+0x8292	0x5030	
+0x8293	0x5031	
+0x8294	0x5032	
+0x8295	0x5033	
+0x8296	0x5034	
+0x8297	0x5035	
+0x8298	0x5036	
+0x8299	0x5037	
+0x829A	0x5038	
+0x829B	0x5039	
+0x829C	0x503B	
+0x829D	0x503D	
+0x829E	0x503F	
+0x829F	0x5040	
+0x82A0	0x5041	
+0x82A1	0x5042	
+0x82A2	0x5044	
+0x82A3	0x5045	
+0x82A4	0x5046	
+0x82A5	0x5049	
+0x82A6	0x504A	
+0x82A7	0x504B	
+0x82A8	0x504D	
+0x82A9	0x5050	
+0x82AA	0x5051	
+0x82AB	0x5052	
+0x82AC	0x5053	
+0x82AD	0x5054	
+0x82AE	0x5056	
+0x82AF	0x5057	
+0x82B0	0x5058	
+0x82B1	0x5059	
+0x82B2	0x505B	
+0x82B3	0x505D	
+0x82B4	0x505E	
+0x82B5	0x505F	
+0x82B6	0x5060	
+0x82B7	0x5061	
+0x82B8	0x5062	
+0x82B9	0x5063	
+0x82BA	0x5064	
+0x82BB	0x5066	
+0x82BC	0x5067	
+0x82BD	0x5068	
+0x82BE	0x5069	
+0x82BF	0x506A	
+0x82C0	0x506B	
+0x82C1	0x506D	
+0x82C2	0x506E	
+0x82C3	0x506F	
+0x82C4	0x5070	
+0x82C5	0x5071	
+0x82C6	0x5072	
+0x82C7	0x5073	
+0x82C8	0x5074	
+0x82C9	0x5075	
+0x82CA	0x5078	
+0x82CB	0x5079	
+0x82CC	0x507A	
+0x82CD	0x507C	
+0x82CE	0x507D	
+0x82CF	0x5081	
+0x82D0	0x5082	
+0x82D1	0x5083	
+0x82D2	0x5084	
+0x82D3	0x5086	
+0x82D4	0x5087	
+0x82D5	0x5089	
+0x82D6	0x508A	
+0x82D7	0x508B	
+0x82D8	0x508C	
+0x82D9	0x508E	
+0x82DA	0x508F	
+0x82DB	0x5090	
+0x82DC	0x5091	
+0x82DD	0x5092	
+0x82DE	0x5093	
+0x82DF	0x5094	
+0x82E0	0x5095	
+0x82E1	0x5096	
+0x82E2	0x5097	
+0x82E3	0x5098	
+0x82E4	0x5099	
+0x82E5	0x509A	
+0x82E6	0x509B	
+0x82E7	0x509C	
+0x82E8	0x509D	
+0x82E9	0x509E	
+0x82EA	0x509F	
+0x82EB	0x50A0	
+0x82EC	0x50A1	
+0x82ED	0x50A2	
+0x82EE	0x50A4	
+0x82EF	0x50A6	
+0x82F0	0x50AA	
+0x82F1	0x50AB	
+0x82F2	0x50AD	
+0x82F3	0x50AE	
+0x82F4	0x50AF	
+0x82F5	0x50B0	
+0x82F6	0x50B1	
+0x82F7	0x50B3	
+0x82F8	0x50B4	
+0x82F9	0x50B5	
+0x82FA	0x50B6	
+0x82FB	0x50B7	
+0x82FC	0x50B8	
+0x82FD	0x50B9	
+0x82FE	0x50BC	
+0x8340	0x50BD	
+0x8341	0x50BE	
+0x8342	0x50BF	
+0x8343	0x50C0	
+0x8344	0x50C1	
+0x8345	0x50C2	
+0x8346	0x50C3	
+0x8347	0x50C4	
+0x8348	0x50C5	
+0x8349	0x50C6	
+0x834A	0x50C7	
+0x834B	0x50C8	
+0x834C	0x50C9	
+0x834D	0x50CA	
+0x834E	0x50CB	
+0x834F	0x50CC	
+0x8350	0x50CD	
+0x8351	0x50CE	
+0x8352	0x50D0	
+0x8353	0x50D1	
+0x8354	0x50D2	
+0x8355	0x50D3	
+0x8356	0x50D4	
+0x8357	0x50D5	
+0x8358	0x50D7	
+0x8359	0x50D8	
+0x835A	0x50D9	
+0x835B	0x50DB	
+0x835C	0x50DC	
+0x835D	0x50DD	
+0x835E	0x50DE	
+0x835F	0x50DF	
+0x8360	0x50E0	
+0x8361	0x50E1	
+0x8362	0x50E2	
+0x8363	0x50E3	
+0x8364	0x50E4	
+0x8365	0x50E5	
+0x8366	0x50E8	
+0x8367	0x50E9	
+0x8368	0x50EA	
+0x8369	0x50EB	
+0x836A	0x50EF	
+0x836B	0x50F0	
+0x836C	0x50F1	
+0x836D	0x50F2	
+0x836E	0x50F4	
+0x836F	0x50F6	
+0x8370	0x50F7	
+0x8371	0x50F8	
+0x8372	0x50F9	
+0x8373	0x50FA	
+0x8374	0x50FC	
+0x8375	0x50FD	
+0x8376	0x50FE	
+0x8377	0x50FF	
+0x8378	0x5100	
+0x8379	0x5101	
+0x837A	0x5102	
+0x837B	0x5103	
+0x837C	0x5104	
+0x837D	0x5105	
+0x837E	0x5108	
+0x8380	0x5109	
+0x8381	0x510A	
+0x8382	0x510C	
+0x8383	0x510D	
+0x8384	0x510E	
+0x8385	0x510F	
+0x8386	0x5110	
+0x8387	0x5111	
+0x8388	0x5113	
+0x8389	0x5114	
+0x838A	0x5115	
+0x838B	0x5116	
+0x838C	0x5117	
+0x838D	0x5118	
+0x838E	0x5119	
+0x838F	0x511A	
+0x8390	0x511B	
+0x8391	0x511C	
+0x8392	0x511D	
+0x8393	0x511E	
+0x8394	0x511F	
+0x8395	0x5120	
+0x8396	0x5122	
+0x8397	0x5123	
+0x8398	0x5124	
+0x8399	0x5125	
+0x839A	0x5126	
+0x839B	0x5127	
+0x839C	0x5128	
+0x839D	0x5129	
+0x839E	0x512A	
+0x839F	0x512B	
+0x83A0	0x512C	
+0x83A1	0x512D	
+0x83A2	0x512E	
+0x83A3	0x512F	
+0x83A4	0x5130	
+0x83A5	0x5131	
+0x83A6	0x5132	
+0x83A7	0x5133	
+0x83A8	0x5134	
+0x83A9	0x5135	
+0x83AA	0x5136	
+0x83AB	0x5137	
+0x83AC	0x5138	
+0x83AD	0x5139	
+0x83AE	0x513A	
+0x83AF	0x513B	
+0x83B0	0x513C	
+0x83B1	0x513D	
+0x83B2	0x513E	
+0x83B3	0x5142	
+0x83B4	0x5147	
+0x83B5	0x514A	
+0x83B6	0x514C	
+0x83B7	0x514E	
+0x83B8	0x514F	
+0x83B9	0x5150	
+0x83BA	0x5152	
+0x83BB	0x5153	
+0x83BC	0x5157	
+0x83BD	0x5158	
+0x83BE	0x5159	
+0x83BF	0x515B	
+0x83C0	0x515D	
+0x83C1	0x515E	
+0x83C2	0x515F	
+0x83C3	0x5160	
+0x83C4	0x5161	
+0x83C5	0x5163	
+0x83C6	0x5164	
+0x83C7	0x5166	
+0x83C8	0x5167	
+0x83C9	0x5169	
+0x83CA	0x516A	
+0x83CB	0x516F	
+0x83CC	0x5172	
+0x83CD	0x517A	
+0x83CE	0x517E	
+0x83CF	0x517F	
+0x83D0	0x5183	
+0x83D1	0x5184	
+0x83D2	0x5186	
+0x83D3	0x5187	
+0x83D4	0x518A	
+0x83D5	0x518B	
+0x83D6	0x518E	
+0x83D7	0x518F	
+0x83D8	0x5190	
+0x83D9	0x5191	
+0x83DA	0x5193	
+0x83DB	0x5194	
+0x83DC	0x5198	
+0x83DD	0x519A	
+0x83DE	0x519D	
+0x83DF	0x519E	
+0x83E0	0x519F	
+0x83E1	0x51A1	
+0x83E2	0x51A3	
+0x83E3	0x51A6	
+0x83E4	0x51A7	
+0x83E5	0x51A8	
+0x83E6	0x51A9	
+0x83E7	0x51AA	
+0x83E8	0x51AD	
+0x83E9	0x51AE	
+0x83EA	0x51B4	
+0x83EB	0x51B8	
+0x83EC	0x51B9	
+0x83ED	0x51BA	
+0x83EE	0x51BE	
+0x83EF	0x51BF	
+0x83F0	0x51C1	
+0x83F1	0x51C2	
+0x83F2	0x51C3	
+0x83F3	0x51C5	
+0x83F4	0x51C8	
+0x83F5	0x51CA	
+0x83F6	0x51CD	
+0x83F7	0x51CE	
+0x83F8	0x51D0	
+0x83F9	0x51D2	
+0x83FA	0x51D3	
+0x83FB	0x51D4	
+0x83FC	0x51D5	
+0x83FD	0x51D6	
+0x83FE	0x51D7	
+0x8440	0x51D8	
+0x8441	0x51D9	
+0x8442	0x51DA	
+0x8443	0x51DC	
+0x8444	0x51DE	
+0x8445	0x51DF	
+0x8446	0x51E2	
+0x8447	0x51E3	
+0x8448	0x51E5	
+0x8449	0x51E6	
+0x844A	0x51E7	
+0x844B	0x51E8	
+0x844C	0x51E9	
+0x844D	0x51EA	
+0x844E	0x51EC	
+0x844F	0x51EE	
+0x8450	0x51F1	
+0x8451	0x51F2	
+0x8452	0x51F4	
+0x8453	0x51F7	
+0x8454	0x51FE	
+0x8455	0x5204	
+0x8456	0x5205	
+0x8457	0x5209	
+0x8458	0x520B	
+0x8459	0x520C	
+0x845A	0x520F	
+0x845B	0x5210	
+0x845C	0x5213	
+0x845D	0x5214	
+0x845E	0x5215	
+0x845F	0x521C	
+0x8460	0x521E	
+0x8461	0x521F	
+0x8462	0x5221	
+0x8463	0x5222	
+0x8464	0x5223	
+0x8465	0x5225	
+0x8466	0x5226	
+0x8467	0x5227	
+0x8468	0x522A	
+0x8469	0x522C	
+0x846A	0x522F	
+0x846B	0x5231	
+0x846C	0x5232	
+0x846D	0x5234	
+0x846E	0x5235	
+0x846F	0x523C	
+0x8470	0x523E	
+0x8471	0x5244	
+0x8472	0x5245	
+0x8473	0x5246	
+0x8474	0x5247	
+0x8475	0x5248	
+0x8476	0x5249	
+0x8477	0x524B	
+0x8478	0x524E	
+0x8479	0x524F	
+0x847A	0x5252	
+0x847B	0x5253	
+0x847C	0x5255	
+0x847D	0x5257	
+0x847E	0x5258	
+0x8480	0x5259	
+0x8481	0x525A	
+0x8482	0x525B	
+0x8483	0x525D	
+0x8484	0x525F	
+0x8485	0x5260	
+0x8486	0x5262	
+0x8487	0x5263	
+0x8488	0x5264	
+0x8489	0x5266	
+0x848A	0x5268	
+0x848B	0x526B	
+0x848C	0x526C	
+0x848D	0x526D	
+0x848E	0x526E	
+0x848F	0x5270	
+0x8490	0x5271	
+0x8491	0x5273	
+0x8492	0x5274	
+0x8493	0x5275	
+0x8494	0x5276	
+0x8495	0x5277	
+0x8496	0x5278	
+0x8497	0x5279	
+0x8498	0x527A	
+0x8499	0x527B	
+0x849A	0x527C	
+0x849B	0x527E	
+0x849C	0x5280	
+0x849D	0x5283	
+0x849E	0x5284	
+0x849F	0x5285	
+0x84A0	0x5286	
+0x84A1	0x5287	
+0x84A2	0x5289	
+0x84A3	0x528A	
+0x84A4	0x528B	
+0x84A5	0x528C	
+0x84A6	0x528D	
+0x84A7	0x528E	
+0x84A8	0x528F	
+0x84A9	0x5291	
+0x84AA	0x5292	
+0x84AB	0x5294	
+0x84AC	0x5295	
+0x84AD	0x5296	
+0x84AE	0x5297	
+0x84AF	0x5298	
+0x84B0	0x5299	
+0x84B1	0x529A	
+0x84B2	0x529C	
+0x84B3	0x52A4	
+0x84B4	0x52A5	
+0x84B5	0x52A6	
+0x84B6	0x52A7	
+0x84B7	0x52AE	
+0x84B8	0x52AF	
+0x84B9	0x52B0	
+0x84BA	0x52B4	
+0x84BB	0x52B5	
+0x84BC	0x52B6	
+0x84BD	0x52B7	
+0x84BE	0x52B8	
+0x84BF	0x52B9	
+0x84C0	0x52BA	
+0x84C1	0x52BB	
+0x84C2	0x52BC	
+0x84C3	0x52BD	
+0x84C4	0x52C0	
+0x84C5	0x52C1	
+0x84C6	0x52C2	
+0x84C7	0x52C4	
+0x84C8	0x52C5	
+0x84C9	0x52C6	
+0x84CA	0x52C8	
+0x84CB	0x52CA	
+0x84CC	0x52CC	
+0x84CD	0x52CD	
+0x84CE	0x52CE	
+0x84CF	0x52CF	
+0x84D0	0x52D1	
+0x84D1	0x52D3	
+0x84D2	0x52D4	
+0x84D3	0x52D5	
+0x84D4	0x52D7	
+0x84D5	0x52D9	
+0x84D6	0x52DA	
+0x84D7	0x52DB	
+0x84D8	0x52DC	
+0x84D9	0x52DD	
+0x84DA	0x52DE	
+0x84DB	0x52E0	
+0x84DC	0x52E1	
+0x84DD	0x52E2	
+0x84DE	0x52E3	
+0x84DF	0x52E5	
+0x84E0	0x52E6	
+0x84E1	0x52E7	
+0x84E2	0x52E8	
+0x84E3	0x52E9	
+0x84E4	0x52EA	
+0x84E5	0x52EB	
+0x84E6	0x52EC	
+0x84E7	0x52ED	
+0x84E8	0x52EE	
+0x84E9	0x52EF	
+0x84EA	0x52F1	
+0x84EB	0x52F2	
+0x84EC	0x52F3	
+0x84ED	0x52F4	
+0x84EE	0x52F5	
+0x84EF	0x52F6	
+0x84F0	0x52F7	
+0x84F1	0x52F8	
+0x84F2	0x52FB	
+0x84F3	0x52FC	
+0x84F4	0x52FD	
+0x84F5	0x5301	
+0x84F6	0x5302	
+0x84F7	0x5303	
+0x84F8	0x5304	
+0x84F9	0x5307	
+0x84FA	0x5309	
+0x84FB	0x530A	
+0x84FC	0x530B	
+0x84FD	0x530C	
+0x84FE	0x530E	
+0x8540	0x5311	
+0x8541	0x5312	
+0x8542	0x5313	
+0x8543	0x5314	
+0x8544	0x5318	
+0x8545	0x531B	
+0x8546	0x531C	
+0x8547	0x531E	
+0x8548	0x531F	
+0x8549	0x5322	
+0x854A	0x5324	
+0x854B	0x5325	
+0x854C	0x5327	
+0x854D	0x5328	
+0x854E	0x5329	
+0x854F	0x532B	
+0x8550	0x532C	
+0x8551	0x532D	
+0x8552	0x532F	
+0x8553	0x5330	
+0x8554	0x5331	
+0x8555	0x5332	
+0x8556	0x5333	
+0x8557	0x5334	
+0x8558	0x5335	
+0x8559	0x5336	
+0x855A	0x5337	
+0x855B	0x5338	
+0x855C	0x533C	
+0x855D	0x533D	
+0x855E	0x5340	
+0x855F	0x5342	
+0x8560	0x5344	
+0x8561	0x5346	
+0x8562	0x534B	
+0x8563	0x534C	
+0x8564	0x534D	
+0x8565	0x5350	
+0x8566	0x5354	
+0x8567	0x5358	
+0x8568	0x5359	
+0x8569	0x535B	
+0x856A	0x535D	
+0x856B	0x5365	
+0x856C	0x5368	
+0x856D	0x536A	
+0x856E	0x536C	
+0x856F	0x536D	
+0x8570	0x5372	
+0x8571	0x5376	
+0x8572	0x5379	
+0x8573	0x537B	
+0x8574	0x537C	
+0x8575	0x537D	
+0x8576	0x537E	
+0x8577	0x5380	
+0x8578	0x5381	
+0x8579	0x5383	
+0x857A	0x5387	
+0x857B	0x5388	
+0x857C	0x538A	
+0x857D	0x538E	
+0x857E	0x538F	
+0x8580	0x5390	
+0x8581	0x5391	
+0x8582	0x5392	
+0x8583	0x5393	
+0x8584	0x5394	
+0x8585	0x5396	
+0x8586	0x5397	
+0x8587	0x5399	
+0x8588	0x539B	
+0x8589	0x539C	
+0x858A	0x539E	
+0x858B	0x53A0	
+0x858C	0x53A1	
+0x858D	0x53A4	
+0x858E	0x53A7	
+0x858F	0x53AA	
+0x8590	0x53AB	
+0x8591	0x53AC	
+0x8592	0x53AD	
+0x8593	0x53AF	
+0x8594	0x53B0	
+0x8595	0x53B1	
+0x8596	0x53B2	
+0x8597	0x53B3	
+0x8598	0x53B4	
+0x8599	0x53B5	
+0x859A	0x53B7	
+0x859B	0x53B8	
+0x859C	0x53B9	
+0x859D	0x53BA	
+0x859E	0x53BC	
+0x859F	0x53BD	
+0x85A0	0x53BE	
+0x85A1	0x53C0	
+0x85A2	0x53C3	
+0x85A3	0x53C4	
+0x85A4	0x53C5	
+0x85A5	0x53C6	
+0x85A6	0x53C7	
+0x85A7	0x53CE	
+0x85A8	0x53CF	
+0x85A9	0x53D0	
+0x85AA	0x53D2	
+0x85AB	0x53D3	
+0x85AC	0x53D5	
+0x85AD	0x53DA	
+0x85AE	0x53DC	
+0x85AF	0x53DD	
+0x85B0	0x53DE	
+0x85B1	0x53E1	
+0x85B2	0x53E2	
+0x85B3	0x53E7	
+0x85B4	0x53F4	
+0x85B5	0x53FA	
+0x85B6	0x53FE	
+0x85B7	0x53FF	
+0x85B8	0x5400	
+0x85B9	0x5402	
+0x85BA	0x5405	
+0x85BB	0x5407	
+0x85BC	0x540B	
+0x85BD	0x5414	
+0x85BE	0x5418	
+0x85BF	0x5419	
+0x85C0	0x541A	
+0x85C1	0x541C	
+0x85C2	0x5422	
+0x85C3	0x5424	
+0x85C4	0x5425	
+0x85C5	0x542A	
+0x85C6	0x5430	
+0x85C7	0x5433	
+0x85C8	0x5436	
+0x85C9	0x5437	
+0x85CA	0x543A	
+0x85CB	0x543D	
+0x85CC	0x543F	
+0x85CD	0x5441	
+0x85CE	0x5442	
+0x85CF	0x5444	
+0x85D0	0x5445	
+0x85D1	0x5447	
+0x85D2	0x5449	
+0x85D3	0x544C	
+0x85D4	0x544D	
+0x85D5	0x544E	
+0x85D6	0x544F	
+0x85D7	0x5451	
+0x85D8	0x545A	
+0x85D9	0x545D	
+0x85DA	0x545E	
+0x85DB	0x545F	
+0x85DC	0x5460	
+0x85DD	0x5461	
+0x85DE	0x5463	
+0x85DF	0x5465	
+0x85E0	0x5467	
+0x85E1	0x5469	
+0x85E2	0x546A	
+0x85E3	0x546B	
+0x85E4	0x546C	
+0x85E5	0x546D	
+0x85E6	0x546E	
+0x85E7	0x546F	
+0x85E8	0x5470	
+0x85E9	0x5474	
+0x85EA	0x5479	
+0x85EB	0x547A	
+0x85EC	0x547E	
+0x85ED	0x547F	
+0x85EE	0x5481	
+0x85EF	0x5483	
+0x85F0	0x5485	
+0x85F1	0x5487	
+0x85F2	0x5488	
+0x85F3	0x5489	
+0x85F4	0x548A	
+0x85F5	0x548D	
+0x85F6	0x5491	
+0x85F7	0x5493	
+0x85F8	0x5497	
+0x85F9	0x5498	
+0x85FA	0x549C	
+0x85FB	0x549E	
+0x85FC	0x549F	
+0x85FD	0x54A0	
+0x85FE	0x54A1	
+0x8640	0x54A2	
+0x8641	0x54A5	
+0x8642	0x54AE	
+0x8643	0x54B0	
+0x8644	0x54B2	
+0x8645	0x54B5	
+0x8646	0x54B6	
+0x8647	0x54B7	
+0x8648	0x54B9	
+0x8649	0x54BA	
+0x864A	0x54BC	
+0x864B	0x54BE	
+0x864C	0x54C3	
+0x864D	0x54C5	
+0x864E	0x54CA	
+0x864F	0x54CB	
+0x8650	0x54D6	
+0x8651	0x54D8	
+0x8652	0x54DB	
+0x8653	0x54E0	
+0x8654	0x54E1	
+0x8655	0x54E2	
+0x8656	0x54E3	
+0x8657	0x54E4	
+0x8658	0x54EB	
+0x8659	0x54EC	
+0x865A	0x54EF	
+0x865B	0x54F0	
+0x865C	0x54F1	
+0x865D	0x54F4	
+0x865E	0x54F5	
+0x865F	0x54F6	
+0x8660	0x54F7	
+0x8661	0x54F8	
+0x8662	0x54F9	
+0x8663	0x54FB	
+0x8664	0x54FE	
+0x8665	0x5500	
+0x8666	0x5502	
+0x8667	0x5503	
+0x8668	0x5504	
+0x8669	0x5505	
+0x866A	0x5508	
+0x866B	0x550A	
+0x866C	0x550B	
+0x866D	0x550C	
+0x866E	0x550D	
+0x866F	0x550E	
+0x8670	0x5512	
+0x8671	0x5513	
+0x8672	0x5515	
+0x8673	0x5516	
+0x8674	0x5517	
+0x8675	0x5518	
+0x8676	0x5519	
+0x8677	0x551A	
+0x8678	0x551C	
+0x8679	0x551D	
+0x867A	0x551E	
+0x867B	0x551F	
+0x867C	0x5521	
+0x867D	0x5525	
+0x867E	0x5526	
+0x8680	0x5528	
+0x8681	0x5529	
+0x8682	0x552B	
+0x8683	0x552D	
+0x8684	0x5532	
+0x8685	0x5534	
+0x8686	0x5535	
+0x8687	0x5536	
+0x8688	0x5538	
+0x8689	0x5539	
+0x868A	0x553A	
+0x868B	0x553B	
+0x868C	0x553D	
+0x868D	0x5540	
+0x868E	0x5542	
+0x868F	0x5545	
+0x8690	0x5547	
+0x8691	0x5548	
+0x8692	0x554B	
+0x8693	0x554C	
+0x8694	0x554D	
+0x8695	0x554E	
+0x8696	0x554F	
+0x8697	0x5551	
+0x8698	0x5552	
+0x8699	0x5553	
+0x869A	0x5554	
+0x869B	0x5557	
+0x869C	0x5558	
+0x869D	0x5559	
+0x869E	0x555A	
+0x869F	0x555B	
+0x86A0	0x555D	
+0x86A1	0x555E	
+0x86A2	0x555F	
+0x86A3	0x5560	
+0x86A4	0x5562	
+0x86A5	0x5563	
+0x86A6	0x5568	
+0x86A7	0x5569	
+0x86A8	0x556B	
+0x86A9	0x556F	
+0x86AA	0x5570	
+0x86AB	0x5571	
+0x86AC	0x5572	
+0x86AD	0x5573	
+0x86AE	0x5574	
+0x86AF	0x5579	
+0x86B0	0x557A	
+0x86B1	0x557D	
+0x86B2	0x557F	
+0x86B3	0x5585	
+0x86B4	0x5586	
+0x86B5	0x558C	
+0x86B6	0x558D	
+0x86B7	0x558E	
+0x86B8	0x5590	
+0x86B9	0x5592	
+0x86BA	0x5593	
+0x86BB	0x5595	
+0x86BC	0x5596	
+0x86BD	0x5597	
+0x86BE	0x559A	
+0x86BF	0x559B	
+0x86C0	0x559E	
+0x86C1	0x55A0	
+0x86C2	0x55A1	
+0x86C3	0x55A2	
+0x86C4	0x55A3	
+0x86C5	0x55A4	
+0x86C6	0x55A5	
+0x86C7	0x55A6	
+0x86C8	0x55A8	
+0x86C9	0x55A9	
+0x86CA	0x55AA	
+0x86CB	0x55AB	
+0x86CC	0x55AC	
+0x86CD	0x55AD	
+0x86CE	0x55AE	
+0x86CF	0x55AF	
+0x86D0	0x55B0	
+0x86D1	0x55B2	
+0x86D2	0x55B4	
+0x86D3	0x55B6	
+0x86D4	0x55B8	
+0x86D5	0x55BA	
+0x86D6	0x55BC	
+0x86D7	0x55BF	
+0x86D8	0x55C0	
+0x86D9	0x55C1	
+0x86DA	0x55C2	
+0x86DB	0x55C3	
+0x86DC	0x55C6	
+0x86DD	0x55C7	
+0x86DE	0x55C8	
+0x86DF	0x55CA	
+0x86E0	0x55CB	
+0x86E1	0x55CE	
+0x86E2	0x55CF	
+0x86E3	0x55D0	
+0x86E4	0x55D5	
+0x86E5	0x55D7	
+0x86E6	0x55D8	
+0x86E7	0x55D9	
+0x86E8	0x55DA	
+0x86E9	0x55DB	
+0x86EA	0x55DE	
+0x86EB	0x55E0	
+0x86EC	0x55E2	
+0x86ED	0x55E7	
+0x86EE	0x55E9	
+0x86EF	0x55ED	
+0x86F0	0x55EE	
+0x86F1	0x55F0	
+0x86F2	0x55F1	
+0x86F3	0x55F4	
+0x86F4	0x55F6	
+0x86F5	0x55F8	
+0x86F6	0x55F9	
+0x86F7	0x55FA	
+0x86F8	0x55FB	
+0x86F9	0x55FC	
+0x86FA	0x55FF	
+0x86FB	0x5602	
+0x86FC	0x5603	
+0x86FD	0x5604	
+0x86FE	0x5605	
+0x8740	0x5606	
+0x8741	0x5607	
+0x8742	0x560A	
+0x8743	0x560B	
+0x8744	0x560D	
+0x8745	0x5610	
+0x8746	0x5611	
+0x8747	0x5612	
+0x8748	0x5613	
+0x8749	0x5614	
+0x874A	0x5615	
+0x874B	0x5616	
+0x874C	0x5617	
+0x874D	0x5619	
+0x874E	0x561A	
+0x874F	0x561C	
+0x8750	0x561D	
+0x8751	0x5620	
+0x8752	0x5621	
+0x8753	0x5622	
+0x8754	0x5625	
+0x8755	0x5626	
+0x8756	0x5628	
+0x8757	0x5629	
+0x8758	0x562A	
+0x8759	0x562B	
+0x875A	0x562E	
+0x875B	0x562F	
+0x875C	0x5630	
+0x875D	0x5633	
+0x875E	0x5635	
+0x875F	0x5637	
+0x8760	0x5638	
+0x8761	0x563A	
+0x8762	0x563C	
+0x8763	0x563D	
+0x8764	0x563E	
+0x8765	0x5640	
+0x8766	0x5641	
+0x8767	0x5642	
+0x8768	0x5643	
+0x8769	0x5644	
+0x876A	0x5645	
+0x876B	0x5646	
+0x876C	0x5647	
+0x876D	0x5648	
+0x876E	0x5649	
+0x876F	0x564A	
+0x8770	0x564B	
+0x8771	0x564F	
+0x8772	0x5650	
+0x8773	0x5651	
+0x8774	0x5652	
+0x8775	0x5653	
+0x8776	0x5655	
+0x8777	0x5656	
+0x8778	0x565A	
+0x8779	0x565B	
+0x877A	0x565D	
+0x877B	0x565E	
+0x877C	0x565F	
+0x877D	0x5660	
+0x877E	0x5661	
+0x8780	0x5663	
+0x8781	0x5665	
+0x8782	0x5666	
+0x8783	0x5667	
+0x8784	0x566D	
+0x8785	0x566E	
+0x8786	0x566F	
+0x8787	0x5670	
+0x8788	0x5672	
+0x8789	0x5673	
+0x878A	0x5674	
+0x878B	0x5675	
+0x878C	0x5677	
+0x878D	0x5678	
+0x878E	0x5679	
+0x878F	0x567A	
+0x8790	0x567D	
+0x8791	0x567E	
+0x8792	0x567F	
+0x8793	0x5680	
+0x8794	0x5681	
+0x8795	0x5682	
+0x8796	0x5683	
+0x8797	0x5684	
+0x8798	0x5687	
+0x8799	0x5688	
+0x879A	0x5689	
+0x879B	0x568A	
+0x879C	0x568B	
+0x879D	0x568C	
+0x879E	0x568D	
+0x879F	0x5690	
+0x87A0	0x5691	
+0x87A1	0x5692	
+0x87A2	0x5694	
+0x87A3	0x5695	
+0x87A4	0x5696	
+0x87A5	0x5697	
+0x87A6	0x5698	
+0x87A7	0x5699	
+0x87A8	0x569A	
+0x87A9	0x569B	
+0x87AA	0x569C	
+0x87AB	0x569D	
+0x87AC	0x569E	
+0x87AD	0x569F	
+0x87AE	0x56A0	
+0x87AF	0x56A1	
+0x87B0	0x56A2	
+0x87B1	0x56A4	
+0x87B2	0x56A5	
+0x87B3	0x56A6	
+0x87B4	0x56A7	
+0x87B5	0x56A8	
+0x87B6	0x56A9	
+0x87B7	0x56AA	
+0x87B8	0x56AB	
+0x87B9	0x56AC	
+0x87BA	0x56AD	
+0x87BB	0x56AE	
+0x87BC	0x56B0	
+0x87BD	0x56B1	
+0x87BE	0x56B2	
+0x87BF	0x56B3	
+0x87C0	0x56B4	
+0x87C1	0x56B5	
+0x87C2	0x56B6	
+0x87C3	0x56B8	
+0x87C4	0x56B9	
+0x87C5	0x56BA	
+0x87C6	0x56BB	
+0x87C7	0x56BD	
+0x87C8	0x56BE	
+0x87C9	0x56BF	
+0x87CA	0x56C0	
+0x87CB	0x56C1	
+0x87CC	0x56C2	
+0x87CD	0x56C3	
+0x87CE	0x56C4	
+0x87CF	0x56C5	
+0x87D0	0x56C6	
+0x87D1	0x56C7	
+0x87D2	0x56C8	
+0x87D3	0x56C9	
+0x87D4	0x56CB	
+0x87D5	0x56CC	
+0x87D6	0x56CD	
+0x87D7	0x56CE	
+0x87D8	0x56CF	
+0x87D9	0x56D0	
+0x87DA	0x56D1	
+0x87DB	0x56D2	
+0x87DC	0x56D3	
+0x87DD	0x56D5	
+0x87DE	0x56D6	
+0x87DF	0x56D8	
+0x87E0	0x56D9	
+0x87E1	0x56DC	
+0x87E2	0x56E3	
+0x87E3	0x56E5	
+0x87E4	0x56E6	
+0x87E5	0x56E7	
+0x87E6	0x56E8	
+0x87E7	0x56E9	
+0x87E8	0x56EA	
+0x87E9	0x56EC	
+0x87EA	0x56EE	
+0x87EB	0x56EF	
+0x87EC	0x56F2	
+0x87ED	0x56F3	
+0x87EE	0x56F6	
+0x87EF	0x56F7	
+0x87F0	0x56F8	
+0x87F1	0x56FB	
+0x87F2	0x56FC	
+0x87F3	0x5700	
+0x87F4	0x5701	
+0x87F5	0x5702	
+0x87F6	0x5705	
+0x87F7	0x5707	
+0x87F8	0x570B	
+0x87F9	0x570C	
+0x87FA	0x570D	
+0x87FB	0x570E	
+0x87FC	0x570F	
+0x87FD	0x5710	
+0x87FE	0x5711	
+0x8840	0x5712	
+0x8841	0x5713	
+0x8842	0x5714	
+0x8843	0x5715	
+0x8844	0x5716	
+0x8845	0x5717	
+0x8846	0x5718	
+0x8847	0x5719	
+0x8848	0x571A	
+0x8849	0x571B	
+0x884A	0x571D	
+0x884B	0x571E	
+0x884C	0x5720	
+0x884D	0x5721	
+0x884E	0x5722	
+0x884F	0x5724	
+0x8850	0x5725	
+0x8851	0x5726	
+0x8852	0x5727	
+0x8853	0x572B	
+0x8854	0x5731	
+0x8855	0x5732	
+0x8856	0x5734	
+0x8857	0x5735	
+0x8858	0x5736	
+0x8859	0x5737	
+0x885A	0x5738	
+0x885B	0x573C	
+0x885C	0x573D	
+0x885D	0x573F	
+0x885E	0x5741	
+0x885F	0x5743	
+0x8860	0x5744	
+0x8861	0x5745	
+0x8862	0x5746	
+0x8863	0x5748	
+0x8864	0x5749	
+0x8865	0x574B	
+0x8866	0x5752	
+0x8867	0x5753	
+0x8868	0x5754	
+0x8869	0x5755	
+0x886A	0x5756	
+0x886B	0x5758	
+0x886C	0x5759	
+0x886D	0x5762	
+0x886E	0x5763	
+0x886F	0x5765	
+0x8870	0x5767	
+0x8871	0x576C	
+0x8872	0x576E	
+0x8873	0x5770	
+0x8874	0x5771	
+0x8875	0x5772	
+0x8876	0x5774	
+0x8877	0x5775	
+0x8878	0x5778	
+0x8879	0x5779	
+0x887A	0x577A	
+0x887B	0x577D	
+0x887C	0x577E	
+0x887D	0x577F	
+0x887E	0x5780	
+0x8880	0x5781	
+0x8881	0x5787	
+0x8882	0x5788	
+0x8883	0x5789	
+0x8884	0x578A	
+0x8885	0x578D	
+0x8886	0x578E	
+0x8887	0x578F	
+0x8888	0x5790	
+0x8889	0x5791	
+0x888A	0x5794	
+0x888B	0x5795	
+0x888C	0x5796	
+0x888D	0x5797	
+0x888E	0x5798	
+0x888F	0x5799	
+0x8890	0x579A	
+0x8891	0x579C	
+0x8892	0x579D	
+0x8893	0x579E	
+0x8894	0x579F	
+0x8895	0x57A5	
+0x8896	0x57A8	
+0x8897	0x57AA	
+0x8898	0x57AC	
+0x8899	0x57AF	
+0x889A	0x57B0	
+0x889B	0x57B1	
+0x889C	0x57B3	
+0x889D	0x57B5	
+0x889E	0x57B6	
+0x889F	0x57B7	
+0x88A0	0x57B9	
+0x88A1	0x57BA	
+0x88A2	0x57BB	
+0x88A3	0x57BC	
+0x88A4	0x57BD	
+0x88A5	0x57BE	
+0x88A6	0x57BF	
+0x88A7	0x57C0	
+0x88A8	0x57C1	
+0x88A9	0x57C4	
+0x88AA	0x57C5	
+0x88AB	0x57C6	
+0x88AC	0x57C7	
+0x88AD	0x57C8	
+0x88AE	0x57C9	
+0x88AF	0x57CA	
+0x88B0	0x57CC	
+0x88B1	0x57CD	
+0x88B2	0x57D0	
+0x88B3	0x57D1	
+0x88B4	0x57D3	
+0x88B5	0x57D6	
+0x88B6	0x57D7	
+0x88B7	0x57DB	
+0x88B8	0x57DC	
+0x88B9	0x57DE	
+0x88BA	0x57E1	
+0x88BB	0x57E2	
+0x88BC	0x57E3	
+0x88BD	0x57E5	
+0x88BE	0x57E6	
+0x88BF	0x57E7	
+0x88C0	0x57E8	
+0x88C1	0x57E9	
+0x88C2	0x57EA	
+0x88C3	0x57EB	
+0x88C4	0x57EC	
+0x88C5	0x57EE	
+0x88C6	0x57F0	
+0x88C7	0x57F1	
+0x88C8	0x57F2	
+0x88C9	0x57F3	
+0x88CA	0x57F5	
+0x88CB	0x57F6	
+0x88CC	0x57F7	
+0x88CD	0x57FB	
+0x88CE	0x57FC	
+0x88CF	0x57FE	
+0x88D0	0x57FF	
+0x88D1	0x5801	
+0x88D2	0x5803	
+0x88D3	0x5804	
+0x88D4	0x5805	
+0x88D5	0x5808	
+0x88D6	0x5809	
+0x88D7	0x580A	
+0x88D8	0x580C	
+0x88D9	0x580E	
+0x88DA	0x580F	
+0x88DB	0x5810	
+0x88DC	0x5812	
+0x88DD	0x5813	
+0x88DE	0x5814	
+0x88DF	0x5816	
+0x88E0	0x5817	
+0x88E1	0x5818	
+0x88E2	0x581A	
+0x88E3	0x581B	
+0x88E4	0x581C	
+0x88E5	0x581D	
+0x88E6	0x581F	
+0x88E7	0x5822	
+0x88E8	0x5823	
+0x88E9	0x5825	
+0x88EA	0x5826	
+0x88EB	0x5827	
+0x88EC	0x5828	
+0x88ED	0x5829	
+0x88EE	0x582B	
+0x88EF	0x582C	
+0x88F0	0x582D	
+0x88F1	0x582E	
+0x88F2	0x582F	
+0x88F3	0x5831	
+0x88F4	0x5832	
+0x88F5	0x5833	
+0x88F6	0x5834	
+0x88F7	0x5836	
+0x88F8	0x5837	
+0x88F9	0x5838	
+0x88FA	0x5839	
+0x88FB	0x583A	
+0x88FC	0x583B	
+0x88FD	0x583C	
+0x88FE	0x583D	
+0x8940	0x583E	
+0x8941	0x583F	
+0x8942	0x5840	
+0x8943	0x5841	
+0x8944	0x5842	
+0x8945	0x5843	
+0x8946	0x5845	
+0x8947	0x5846	
+0x8948	0x5847	
+0x8949	0x5848	
+0x894A	0x5849	
+0x894B	0x584A	
+0x894C	0x584B	
+0x894D	0x584E	
+0x894E	0x584F	
+0x894F	0x5850	
+0x8950	0x5852	
+0x8951	0x5853	
+0x8952	0x5855	
+0x8953	0x5856	
+0x8954	0x5857	
+0x8955	0x5859	
+0x8956	0x585A	
+0x8957	0x585B	
+0x8958	0x585C	
+0x8959	0x585D	
+0x895A	0x585F	
+0x895B	0x5860	
+0x895C	0x5861	
+0x895D	0x5862	
+0x895E	0x5863	
+0x895F	0x5864	
+0x8960	0x5866	
+0x8961	0x5867	
+0x8962	0x5868	
+0x8963	0x5869	
+0x8964	0x586A	
+0x8965	0x586D	
+0x8966	0x586E	
+0x8967	0x586F	
+0x8968	0x5870	
+0x8969	0x5871	
+0x896A	0x5872	
+0x896B	0x5873	
+0x896C	0x5874	
+0x896D	0x5875	
+0x896E	0x5876	
+0x896F	0x5877	
+0x8970	0x5878	
+0x8971	0x5879	
+0x8972	0x587A	
+0x8973	0x587B	
+0x8974	0x587C	
+0x8975	0x587D	
+0x8976	0x587F	
+0x8977	0x5882	
+0x8978	0x5884	
+0x8979	0x5886	
+0x897A	0x5887	
+0x897B	0x5888	
+0x897C	0x588A	
+0x897D	0x588B	
+0x897E	0x588C	
+0x8980	0x588D	
+0x8981	0x588E	
+0x8982	0x588F	
+0x8983	0x5890	
+0x8984	0x5891	
+0x8985	0x5894	
+0x8986	0x5895	
+0x8987	0x5896	
+0x8988	0x5897	
+0x8989	0x5898	
+0x898A	0x589B	
+0x898B	0x589C	
+0x898C	0x589D	
+0x898D	0x58A0	
+0x898E	0x58A1	
+0x898F	0x58A2	
+0x8990	0x58A3	
+0x8991	0x58A4	
+0x8992	0x58A5	
+0x8993	0x58A6	
+0x8994	0x58A7	
+0x8995	0x58AA	
+0x8996	0x58AB	
+0x8997	0x58AC	
+0x8998	0x58AD	
+0x8999	0x58AE	
+0x899A	0x58AF	
+0x899B	0x58B0	
+0x899C	0x58B1	
+0x899D	0x58B2	
+0x899E	0x58B3	
+0x899F	0x58B4	
+0x89A0	0x58B5	
+0x89A1	0x58B6	
+0x89A2	0x58B7	
+0x89A3	0x58B8	
+0x89A4	0x58B9	
+0x89A5	0x58BA	
+0x89A6	0x58BB	
+0x89A7	0x58BD	
+0x89A8	0x58BE	
+0x89A9	0x58BF	
+0x89AA	0x58C0	
+0x89AB	0x58C2	
+0x89AC	0x58C3	
+0x89AD	0x58C4	
+0x89AE	0x58C6	
+0x89AF	0x58C7	
+0x89B0	0x58C8	
+0x89B1	0x58C9	
+0x89B2	0x58CA	
+0x89B3	0x58CB	
+0x89B4	0x58CC	
+0x89B5	0x58CD	
+0x89B6	0x58CE	
+0x89B7	0x58CF	
+0x89B8	0x58D0	
+0x89B9	0x58D2	
+0x89BA	0x58D3	
+0x89BB	0x58D4	
+0x89BC	0x58D6	
+0x89BD	0x58D7	
+0x89BE	0x58D8	
+0x89BF	0x58D9	
+0x89C0	0x58DA	
+0x89C1	0x58DB	
+0x89C2	0x58DC	
+0x89C3	0x58DD	
+0x89C4	0x58DE	
+0x89C5	0x58DF	
+0x89C6	0x58E0	
+0x89C7	0x58E1	
+0x89C8	0x58E2	
+0x89C9	0x58E3	
+0x89CA	0x58E5	
+0x89CB	0x58E6	
+0x89CC	0x58E7	
+0x89CD	0x58E8	
+0x89CE	0x58E9	
+0x89CF	0x58EA	
+0x89D0	0x58ED	
+0x89D1	0x58EF	
+0x89D2	0x58F1	
+0x89D3	0x58F2	
+0x89D4	0x58F4	
+0x89D5	0x58F5	
+0x89D6	0x58F7	
+0x89D7	0x58F8	
+0x89D8	0x58FA	
+0x89D9	0x58FB	
+0x89DA	0x58FC	
+0x89DB	0x58FD	
+0x89DC	0x58FE	
+0x89DD	0x58FF	
+0x89DE	0x5900	
+0x89DF	0x5901	
+0x89E0	0x5903	
+0x89E1	0x5905	
+0x89E2	0x5906	
+0x89E3	0x5908	
+0x89E4	0x5909	
+0x89E5	0x590A	
+0x89E6	0x590B	
+0x89E7	0x590C	
+0x89E8	0x590E	
+0x89E9	0x5910	
+0x89EA	0x5911	
+0x89EB	0x5912	
+0x89EC	0x5913	
+0x89ED	0x5917	
+0x89EE	0x5918	
+0x89EF	0x591B	
+0x89F0	0x591D	
+0x89F1	0x591E	
+0x89F2	0x5920	
+0x89F3	0x5921	
+0x89F4	0x5922	
+0x89F5	0x5923	
+0x89F6	0x5926	
+0x89F7	0x5928	
+0x89F8	0x592C	
+0x89F9	0x5930	
+0x89FA	0x5932	
+0x89FB	0x5933	
+0x89FC	0x5935	
+0x89FD	0x5936	
+0x89FE	0x593B	
+0x8A40	0x593D	
+0x8A41	0x593E	
+0x8A42	0x593F	
+0x8A43	0x5940	
+0x8A44	0x5943	
+0x8A45	0x5945	
+0x8A46	0x5946	
+0x8A47	0x594A	
+0x8A48	0x594C	
+0x8A49	0x594D	
+0x8A4A	0x5950	
+0x8A4B	0x5952	
+0x8A4C	0x5953	
+0x8A4D	0x5959	
+0x8A4E	0x595B	
+0x8A4F	0x595C	
+0x8A50	0x595D	
+0x8A51	0x595E	
+0x8A52	0x595F	
+0x8A53	0x5961	
+0x8A54	0x5963	
+0x8A55	0x5964	
+0x8A56	0x5966	
+0x8A57	0x5967	
+0x8A58	0x5968	
+0x8A59	0x5969	
+0x8A5A	0x596A	
+0x8A5B	0x596B	
+0x8A5C	0x596C	
+0x8A5D	0x596D	
+0x8A5E	0x596E	
+0x8A5F	0x596F	
+0x8A60	0x5970	
+0x8A61	0x5971	
+0x8A62	0x5972	
+0x8A63	0x5975	
+0x8A64	0x5977	
+0x8A65	0x597A	
+0x8A66	0x597B	
+0x8A67	0x597C	
+0x8A68	0x597E	
+0x8A69	0x597F	
+0x8A6A	0x5980	
+0x8A6B	0x5985	
+0x8A6C	0x5989	
+0x8A6D	0x598B	
+0x8A6E	0x598C	
+0x8A6F	0x598E	
+0x8A70	0x598F	
+0x8A71	0x5990	
+0x8A72	0x5991	
+0x8A73	0x5994	
+0x8A74	0x5995	
+0x8A75	0x5998	
+0x8A76	0x599A	
+0x8A77	0x599B	
+0x8A78	0x599C	
+0x8A79	0x599D	
+0x8A7A	0x599F	
+0x8A7B	0x59A0	
+0x8A7C	0x59A1	
+0x8A7D	0x59A2	
+0x8A7E	0x59A6	
+0x8A80	0x59A7	
+0x8A81	0x59AC	
+0x8A82	0x59AD	
+0x8A83	0x59B0	
+0x8A84	0x59B1	
+0x8A85	0x59B3	
+0x8A86	0x59B4	
+0x8A87	0x59B5	
+0x8A88	0x59B6	
+0x8A89	0x59B7	
+0x8A8A	0x59B8	
+0x8A8B	0x59BA	
+0x8A8C	0x59BC	
+0x8A8D	0x59BD	
+0x8A8E	0x59BF	
+0x8A8F	0x59C0	
+0x8A90	0x59C1	
+0x8A91	0x59C2	
+0x8A92	0x59C3	
+0x8A93	0x59C4	
+0x8A94	0x59C5	
+0x8A95	0x59C7	
+0x8A96	0x59C8	
+0x8A97	0x59C9	
+0x8A98	0x59CC	
+0x8A99	0x59CD	
+0x8A9A	0x59CE	
+0x8A9B	0x59CF	
+0x8A9C	0x59D5	
+0x8A9D	0x59D6	
+0x8A9E	0x59D9	
+0x8A9F	0x59DB	
+0x8AA0	0x59DE	
+0x8AA1	0x59DF	
+0x8AA2	0x59E0	
+0x8AA3	0x59E1	
+0x8AA4	0x59E2	
+0x8AA5	0x59E4	
+0x8AA6	0x59E6	
+0x8AA7	0x59E7	
+0x8AA8	0x59E9	
+0x8AA9	0x59EA	
+0x8AAA	0x59EB	
+0x8AAB	0x59ED	
+0x8AAC	0x59EE	
+0x8AAD	0x59EF	
+0x8AAE	0x59F0	
+0x8AAF	0x59F1	
+0x8AB0	0x59F2	
+0x8AB1	0x59F3	
+0x8AB2	0x59F4	
+0x8AB3	0x59F5	
+0x8AB4	0x59F6	
+0x8AB5	0x59F7	
+0x8AB6	0x59F8	
+0x8AB7	0x59FA	
+0x8AB8	0x59FC	
+0x8AB9	0x59FD	
+0x8ABA	0x59FE	
+0x8ABB	0x5A00	
+0x8ABC	0x5A02	
+0x8ABD	0x5A0A	
+0x8ABE	0x5A0B	
+0x8ABF	0x5A0D	
+0x8AC0	0x5A0E	
+0x8AC1	0x5A0F	
+0x8AC2	0x5A10	
+0x8AC3	0x5A12	
+0x8AC4	0x5A14	
+0x8AC5	0x5A15	
+0x8AC6	0x5A16	
+0x8AC7	0x5A17	
+0x8AC8	0x5A19	
+0x8AC9	0x5A1A	
+0x8ACA	0x5A1B	
+0x8ACB	0x5A1D	
+0x8ACC	0x5A1E	
+0x8ACD	0x5A21	
+0x8ACE	0x5A22	
+0x8ACF	0x5A24	
+0x8AD0	0x5A26	
+0x8AD1	0x5A27	
+0x8AD2	0x5A28	
+0x8AD3	0x5A2A	
+0x8AD4	0x5A2B	
+0x8AD5	0x5A2C	
+0x8AD6	0x5A2D	
+0x8AD7	0x5A2E	
+0x8AD8	0x5A2F	
+0x8AD9	0x5A30	
+0x8ADA	0x5A33	
+0x8ADB	0x5A35	
+0x8ADC	0x5A37	
+0x8ADD	0x5A38	
+0x8ADE	0x5A39	
+0x8ADF	0x5A3A	
+0x8AE0	0x5A3B	
+0x8AE1	0x5A3D	
+0x8AE2	0x5A3E	
+0x8AE3	0x5A3F	
+0x8AE4	0x5A41	
+0x8AE5	0x5A42	
+0x8AE6	0x5A43	
+0x8AE7	0x5A44	
+0x8AE8	0x5A45	
+0x8AE9	0x5A47	
+0x8AEA	0x5A48	
+0x8AEB	0x5A4B	
+0x8AEC	0x5A4C	
+0x8AED	0x5A4D	
+0x8AEE	0x5A4E	
+0x8AEF	0x5A4F	
+0x8AF0	0x5A50	
+0x8AF1	0x5A51	
+0x8AF2	0x5A52	
+0x8AF3	0x5A53	
+0x8AF4	0x5A54	
+0x8AF5	0x5A56	
+0x8AF6	0x5A57	
+0x8AF7	0x5A58	
+0x8AF8	0x5A59	
+0x8AF9	0x5A5B	
+0x8AFA	0x5A5C	
+0x8AFB	0x5A5D	
+0x8AFC	0x5A5E	
+0x8AFD	0x5A5F	
+0x8AFE	0x5A60	
+0x8B40	0x5A61	
+0x8B41	0x5A63	
+0x8B42	0x5A64	
+0x8B43	0x5A65	
+0x8B44	0x5A66	
+0x8B45	0x5A68	
+0x8B46	0x5A69	
+0x8B47	0x5A6B	
+0x8B48	0x5A6C	
+0x8B49	0x5A6D	
+0x8B4A	0x5A6E	
+0x8B4B	0x5A6F	
+0x8B4C	0x5A70	
+0x8B4D	0x5A71	
+0x8B4E	0x5A72	
+0x8B4F	0x5A73	
+0x8B50	0x5A78	
+0x8B51	0x5A79	
+0x8B52	0x5A7B	
+0x8B53	0x5A7C	
+0x8B54	0x5A7D	
+0x8B55	0x5A7E	
+0x8B56	0x5A80	
+0x8B57	0x5A81	
+0x8B58	0x5A82	
+0x8B59	0x5A83	
+0x8B5A	0x5A84	
+0x8B5B	0x5A85	
+0x8B5C	0x5A86	
+0x8B5D	0x5A87	
+0x8B5E	0x5A88	
+0x8B5F	0x5A89	
+0x8B60	0x5A8A	
+0x8B61	0x5A8B	
+0x8B62	0x5A8C	
+0x8B63	0x5A8D	
+0x8B64	0x5A8E	
+0x8B65	0x5A8F	
+0x8B66	0x5A90	
+0x8B67	0x5A91	
+0x8B68	0x5A93	
+0x8B69	0x5A94	
+0x8B6A	0x5A95	
+0x8B6B	0x5A96	
+0x8B6C	0x5A97	
+0x8B6D	0x5A98	
+0x8B6E	0x5A99	
+0x8B6F	0x5A9C	
+0x8B70	0x5A9D	
+0x8B71	0x5A9E	
+0x8B72	0x5A9F	
+0x8B73	0x5AA0	
+0x8B74	0x5AA1	
+0x8B75	0x5AA2	
+0x8B76	0x5AA3	
+0x8B77	0x5AA4	
+0x8B78	0x5AA5	
+0x8B79	0x5AA6	
+0x8B7A	0x5AA7	
+0x8B7B	0x5AA8	
+0x8B7C	0x5AA9	
+0x8B7D	0x5AAB	
+0x8B7E	0x5AAC	
+0x8B80	0x5AAD	
+0x8B81	0x5AAE	
+0x8B82	0x5AAF	
+0x8B83	0x5AB0	
+0x8B84	0x5AB1	
+0x8B85	0x5AB4	
+0x8B86	0x5AB6	
+0x8B87	0x5AB7	
+0x8B88	0x5AB9	
+0x8B89	0x5ABA	
+0x8B8A	0x5ABB	
+0x8B8B	0x5ABC	
+0x8B8C	0x5ABD	
+0x8B8D	0x5ABF	
+0x8B8E	0x5AC0	
+0x8B8F	0x5AC3	
+0x8B90	0x5AC4	
+0x8B91	0x5AC5	
+0x8B92	0x5AC6	
+0x8B93	0x5AC7	
+0x8B94	0x5AC8	
+0x8B95	0x5ACA	
+0x8B96	0x5ACB	
+0x8B97	0x5ACD	
+0x8B98	0x5ACE	
+0x8B99	0x5ACF	
+0x8B9A	0x5AD0	
+0x8B9B	0x5AD1	
+0x8B9C	0x5AD3	
+0x8B9D	0x5AD5	
+0x8B9E	0x5AD7	
+0x8B9F	0x5AD9	
+0x8BA0	0x5ADA	
+0x8BA1	0x5ADB	
+0x8BA2	0x5ADD	
+0x8BA3	0x5ADE	
+0x8BA4	0x5ADF	
+0x8BA5	0x5AE2	
+0x8BA6	0x5AE4	
+0x8BA7	0x5AE5	
+0x8BA8	0x5AE7	
+0x8BA9	0x5AE8	
+0x8BAA	0x5AEA	
+0x8BAB	0x5AEC	
+0x8BAC	0x5AED	
+0x8BAD	0x5AEE	
+0x8BAE	0x5AEF	
+0x8BAF	0x5AF0	
+0x8BB0	0x5AF2	
+0x8BB1	0x5AF3	
+0x8BB2	0x5AF4	
+0x8BB3	0x5AF5	
+0x8BB4	0x5AF6	
+0x8BB5	0x5AF7	
+0x8BB6	0x5AF8	
+0x8BB7	0x5AF9	
+0x8BB8	0x5AFA	
+0x8BB9	0x5AFB	
+0x8BBA	0x5AFC	
+0x8BBB	0x5AFD	
+0x8BBC	0x5AFE	
+0x8BBD	0x5AFF	
+0x8BBE	0x5B00	
+0x8BBF	0x5B01	
+0x8BC0	0x5B02	
+0x8BC1	0x5B03	
+0x8BC2	0x5B04	
+0x8BC3	0x5B05	
+0x8BC4	0x5B06	
+0x8BC5	0x5B07	
+0x8BC6	0x5B08	
+0x8BC7	0x5B0A	
+0x8BC8	0x5B0B	
+0x8BC9	0x5B0C	
+0x8BCA	0x5B0D	
+0x8BCB	0x5B0E	
+0x8BCC	0x5B0F	
+0x8BCD	0x5B10	
+0x8BCE	0x5B11	
+0x8BCF	0x5B12	
+0x8BD0	0x5B13	
+0x8BD1	0x5B14	
+0x8BD2	0x5B15	
+0x8BD3	0x5B18	
+0x8BD4	0x5B19	
+0x8BD5	0x5B1A	
+0x8BD6	0x5B1B	
+0x8BD7	0x5B1C	
+0x8BD8	0x5B1D	
+0x8BD9	0x5B1E	
+0x8BDA	0x5B1F	
+0x8BDB	0x5B20	
+0x8BDC	0x5B21	
+0x8BDD	0x5B22	
+0x8BDE	0x5B23	
+0x8BDF	0x5B24	
+0x8BE0	0x5B25	
+0x8BE1	0x5B26	
+0x8BE2	0x5B27	
+0x8BE3	0x5B28	
+0x8BE4	0x5B29	
+0x8BE5	0x5B2A	
+0x8BE6	0x5B2B	
+0x8BE7	0x5B2C	
+0x8BE8	0x5B2D	
+0x8BE9	0x5B2E	
+0x8BEA	0x5B2F	
+0x8BEB	0x5B30	
+0x8BEC	0x5B31	
+0x8BED	0x5B33	
+0x8BEE	0x5B35	
+0x8BEF	0x5B36	
+0x8BF0	0x5B38	
+0x8BF1	0x5B39	
+0x8BF2	0x5B3A	
+0x8BF3	0x5B3B	
+0x8BF4	0x5B3C	
+0x8BF5	0x5B3D	
+0x8BF6	0x5B3E	
+0x8BF7	0x5B3F	
+0x8BF8	0x5B41	
+0x8BF9	0x5B42	
+0x8BFA	0x5B43	
+0x8BFB	0x5B44	
+0x8BFC	0x5B45	
+0x8BFD	0x5B46	
+0x8BFE	0x5B47	
+0x8C40	0x5B48	
+0x8C41	0x5B49	
+0x8C42	0x5B4A	
+0x8C43	0x5B4B	
+0x8C44	0x5B4C	
+0x8C45	0x5B4D	
+0x8C46	0x5B4E	
+0x8C47	0x5B4F	
+0x8C48	0x5B52	
+0x8C49	0x5B56	
+0x8C4A	0x5B5E	
+0x8C4B	0x5B60	
+0x8C4C	0x5B61	
+0x8C4D	0x5B67	
+0x8C4E	0x5B68	
+0x8C4F	0x5B6B	
+0x8C50	0x5B6D	
+0x8C51	0x5B6E	
+0x8C52	0x5B6F	
+0x8C53	0x5B72	
+0x8C54	0x5B74	
+0x8C55	0x5B76	
+0x8C56	0x5B77	
+0x8C57	0x5B78	
+0x8C58	0x5B79	
+0x8C59	0x5B7B	
+0x8C5A	0x5B7C	
+0x8C5B	0x5B7E	
+0x8C5C	0x5B7F	
+0x8C5D	0x5B82	
+0x8C5E	0x5B86	
+0x8C5F	0x5B8A	
+0x8C60	0x5B8D	
+0x8C61	0x5B8E	
+0x8C62	0x5B90	
+0x8C63	0x5B91	
+0x8C64	0x5B92	
+0x8C65	0x5B94	
+0x8C66	0x5B96	
+0x8C67	0x5B9F	
+0x8C68	0x5BA7	
+0x8C69	0x5BA8	
+0x8C6A	0x5BA9	
+0x8C6B	0x5BAC	
+0x8C6C	0x5BAD	
+0x8C6D	0x5BAE	
+0x8C6E	0x5BAF	
+0x8C6F	0x5BB1	
+0x8C70	0x5BB2	
+0x8C71	0x5BB7	
+0x8C72	0x5BBA	
+0x8C73	0x5BBB	
+0x8C74	0x5BBC	
+0x8C75	0x5BC0	
+0x8C76	0x5BC1	
+0x8C77	0x5BC3	
+0x8C78	0x5BC8	
+0x8C79	0x5BC9	
+0x8C7A	0x5BCA	
+0x8C7B	0x5BCB	
+0x8C7C	0x5BCD	
+0x8C7D	0x5BCE	
+0x8C7E	0x5BCF	
+0x8C80	0x5BD1	
+0x8C81	0x5BD4	
+0x8C82	0x5BD5	
+0x8C83	0x5BD6	
+0x8C84	0x5BD7	
+0x8C85	0x5BD8	
+0x8C86	0x5BD9	
+0x8C87	0x5BDA	
+0x8C88	0x5BDB	
+0x8C89	0x5BDC	
+0x8C8A	0x5BE0	
+0x8C8B	0x5BE2	
+0x8C8C	0x5BE3	
+0x8C8D	0x5BE6	
+0x8C8E	0x5BE7	
+0x8C8F	0x5BE9	
+0x8C90	0x5BEA	
+0x8C91	0x5BEB	
+0x8C92	0x5BEC	
+0x8C93	0x5BED	
+0x8C94	0x5BEF	
+0x8C95	0x5BF1	
+0x8C96	0x5BF2	
+0x8C97	0x5BF3	
+0x8C98	0x5BF4	
+0x8C99	0x5BF5	
+0x8C9A	0x5BF6	
+0x8C9B	0x5BF7	
+0x8C9C	0x5BFD	
+0x8C9D	0x5BFE	
+0x8C9E	0x5C00	
+0x8C9F	0x5C02	
+0x8CA0	0x5C03	
+0x8CA1	0x5C05	
+0x8CA2	0x5C07	
+0x8CA3	0x5C08	
+0x8CA4	0x5C0B	
+0x8CA5	0x5C0C	
+0x8CA6	0x5C0D	
+0x8CA7	0x5C0E	
+0x8CA8	0x5C10	
+0x8CA9	0x5C12	
+0x8CAA	0x5C13	
+0x8CAB	0x5C17	
+0x8CAC	0x5C19	
+0x8CAD	0x5C1B	
+0x8CAE	0x5C1E	
+0x8CAF	0x5C1F	
+0x8CB0	0x5C20	
+0x8CB1	0x5C21	
+0x8CB2	0x5C23	
+0x8CB3	0x5C26	
+0x8CB4	0x5C28	
+0x8CB5	0x5C29	
+0x8CB6	0x5C2A	
+0x8CB7	0x5C2B	
+0x8CB8	0x5C2D	
+0x8CB9	0x5C2E	
+0x8CBA	0x5C2F	
+0x8CBB	0x5C30	
+0x8CBC	0x5C32	
+0x8CBD	0x5C33	
+0x8CBE	0x5C35	
+0x8CBF	0x5C36	
+0x8CC0	0x5C37	
+0x8CC1	0x5C43	
+0x8CC2	0x5C44	
+0x8CC3	0x5C46	
+0x8CC4	0x5C47	
+0x8CC5	0x5C4C	
+0x8CC6	0x5C4D	
+0x8CC7	0x5C52	
+0x8CC8	0x5C53	
+0x8CC9	0x5C54	
+0x8CCA	0x5C56	
+0x8CCB	0x5C57	
+0x8CCC	0x5C58	
+0x8CCD	0x5C5A	
+0x8CCE	0x5C5B	
+0x8CCF	0x5C5C	
+0x8CD0	0x5C5D	
+0x8CD1	0x5C5F	
+0x8CD2	0x5C62	
+0x8CD3	0x5C64	
+0x8CD4	0x5C67	
+0x8CD5	0x5C68	
+0x8CD6	0x5C69	
+0x8CD7	0x5C6A	
+0x8CD8	0x5C6B	
+0x8CD9	0x5C6C	
+0x8CDA	0x5C6D	
+0x8CDB	0x5C70	
+0x8CDC	0x5C72	
+0x8CDD	0x5C73	
+0x8CDE	0x5C74	
+0x8CDF	0x5C75	
+0x8CE0	0x5C76	
+0x8CE1	0x5C77	
+0x8CE2	0x5C78	
+0x8CE3	0x5C7B	
+0x8CE4	0x5C7C	
+0x8CE5	0x5C7D	
+0x8CE6	0x5C7E	
+0x8CE7	0x5C80	
+0x8CE8	0x5C83	
+0x8CE9	0x5C84	
+0x8CEA	0x5C85	
+0x8CEB	0x5C86	
+0x8CEC	0x5C87	
+0x8CED	0x5C89	
+0x8CEE	0x5C8A	
+0x8CEF	0x5C8B	
+0x8CF0	0x5C8E	
+0x8CF1	0x5C8F	
+0x8CF2	0x5C92	
+0x8CF3	0x5C93	
+0x8CF4	0x5C95	
+0x8CF5	0x5C9D	
+0x8CF6	0x5C9E	
+0x8CF7	0x5C9F	
+0x8CF8	0x5CA0	
+0x8CF9	0x5CA1	
+0x8CFA	0x5CA4	
+0x8CFB	0x5CA5	
+0x8CFC	0x5CA6	
+0x8CFD	0x5CA7	
+0x8CFE	0x5CA8	
+0x8D40	0x5CAA	
+0x8D41	0x5CAE	
+0x8D42	0x5CAF	
+0x8D43	0x5CB0	
+0x8D44	0x5CB2	
+0x8D45	0x5CB4	
+0x8D46	0x5CB6	
+0x8D47	0x5CB9	
+0x8D48	0x5CBA	
+0x8D49	0x5CBB	
+0x8D4A	0x5CBC	
+0x8D4B	0x5CBE	
+0x8D4C	0x5CC0	
+0x8D4D	0x5CC2	
+0x8D4E	0x5CC3	
+0x8D4F	0x5CC5	
+0x8D50	0x5CC6	
+0x8D51	0x5CC7	
+0x8D52	0x5CC8	
+0x8D53	0x5CC9	
+0x8D54	0x5CCA	
+0x8D55	0x5CCC	
+0x8D56	0x5CCD	
+0x8D57	0x5CCE	
+0x8D58	0x5CCF	
+0x8D59	0x5CD0	
+0x8D5A	0x5CD1	
+0x8D5B	0x5CD3	
+0x8D5C	0x5CD4	
+0x8D5D	0x5CD5	
+0x8D5E	0x5CD6	
+0x8D5F	0x5CD7	
+0x8D60	0x5CD8	
+0x8D61	0x5CDA	
+0x8D62	0x5CDB	
+0x8D63	0x5CDC	
+0x8D64	0x5CDD	
+0x8D65	0x5CDE	
+0x8D66	0x5CDF	
+0x8D67	0x5CE0	
+0x8D68	0x5CE2	
+0x8D69	0x5CE3	
+0x8D6A	0x5CE7	
+0x8D6B	0x5CE9	
+0x8D6C	0x5CEB	
+0x8D6D	0x5CEC	
+0x8D6E	0x5CEE	
+0x8D6F	0x5CEF	
+0x8D70	0x5CF1	
+0x8D71	0x5CF2	
+0x8D72	0x5CF3	
+0x8D73	0x5CF4	
+0x8D74	0x5CF5	
+0x8D75	0x5CF6	
+0x8D76	0x5CF7	
+0x8D77	0x5CF8	
+0x8D78	0x5CF9	
+0x8D79	0x5CFA	
+0x8D7A	0x5CFC	
+0x8D7B	0x5CFD	
+0x8D7C	0x5CFE	
+0x8D7D	0x5CFF	
+0x8D7E	0x5D00	
+0x8D80	0x5D01	
+0x8D81	0x5D04	
+0x8D82	0x5D05	
+0x8D83	0x5D08	
+0x8D84	0x5D09	
+0x8D85	0x5D0A	
+0x8D86	0x5D0B	
+0x8D87	0x5D0C	
+0x8D88	0x5D0D	
+0x8D89	0x5D0F	
+0x8D8A	0x5D10	
+0x8D8B	0x5D11	
+0x8D8C	0x5D12	
+0x8D8D	0x5D13	
+0x8D8E	0x5D15	
+0x8D8F	0x5D17	
+0x8D90	0x5D18	
+0x8D91	0x5D19	
+0x8D92	0x5D1A	
+0x8D93	0x5D1C	
+0x8D94	0x5D1D	
+0x8D95	0x5D1F	
+0x8D96	0x5D20	
+0x8D97	0x5D21	
+0x8D98	0x5D22	
+0x8D99	0x5D23	
+0x8D9A	0x5D25	
+0x8D9B	0x5D28	
+0x8D9C	0x5D2A	
+0x8D9D	0x5D2B	
+0x8D9E	0x5D2C	
+0x8D9F	0x5D2F	
+0x8DA0	0x5D30	
+0x8DA1	0x5D31	
+0x8DA2	0x5D32	
+0x8DA3	0x5D33	
+0x8DA4	0x5D35	
+0x8DA5	0x5D36	
+0x8DA6	0x5D37	
+0x8DA7	0x5D38	
+0x8DA8	0x5D39	
+0x8DA9	0x5D3A	
+0x8DAA	0x5D3B	
+0x8DAB	0x5D3C	
+0x8DAC	0x5D3F	
+0x8DAD	0x5D40	
+0x8DAE	0x5D41	
+0x8DAF	0x5D42	
+0x8DB0	0x5D43	
+0x8DB1	0x5D44	
+0x8DB2	0x5D45	
+0x8DB3	0x5D46	
+0x8DB4	0x5D48	
+0x8DB5	0x5D49	
+0x8DB6	0x5D4D	
+0x8DB7	0x5D4E	
+0x8DB8	0x5D4F	
+0x8DB9	0x5D50	
+0x8DBA	0x5D51	
+0x8DBB	0x5D52	
+0x8DBC	0x5D53	
+0x8DBD	0x5D54	
+0x8DBE	0x5D55	
+0x8DBF	0x5D56	
+0x8DC0	0x5D57	
+0x8DC1	0x5D59	
+0x8DC2	0x5D5A	
+0x8DC3	0x5D5C	
+0x8DC4	0x5D5E	
+0x8DC5	0x5D5F	
+0x8DC6	0x5D60	
+0x8DC7	0x5D61	
+0x8DC8	0x5D62	
+0x8DC9	0x5D63	
+0x8DCA	0x5D64	
+0x8DCB	0x5D65	
+0x8DCC	0x5D66	
+0x8DCD	0x5D67	
+0x8DCE	0x5D68	
+0x8DCF	0x5D6A	
+0x8DD0	0x5D6D	
+0x8DD1	0x5D6E	
+0x8DD2	0x5D70	
+0x8DD3	0x5D71	
+0x8DD4	0x5D72	
+0x8DD5	0x5D73	
+0x8DD6	0x5D75	
+0x8DD7	0x5D76	
+0x8DD8	0x5D77	
+0x8DD9	0x5D78	
+0x8DDA	0x5D79	
+0x8DDB	0x5D7A	
+0x8DDC	0x5D7B	
+0x8DDD	0x5D7C	
+0x8DDE	0x5D7D	
+0x8DDF	0x5D7E	
+0x8DE0	0x5D7F	
+0x8DE1	0x5D80	
+0x8DE2	0x5D81	
+0x8DE3	0x5D83	
+0x8DE4	0x5D84	
+0x8DE5	0x5D85	
+0x8DE6	0x5D86	
+0x8DE7	0x5D87	
+0x8DE8	0x5D88	
+0x8DE9	0x5D89	
+0x8DEA	0x5D8A	
+0x8DEB	0x5D8B	
+0x8DEC	0x5D8C	
+0x8DED	0x5D8D	
+0x8DEE	0x5D8E	
+0x8DEF	0x5D8F	
+0x8DF0	0x5D90	
+0x8DF1	0x5D91	
+0x8DF2	0x5D92	
+0x8DF3	0x5D93	
+0x8DF4	0x5D94	
+0x8DF5	0x5D95	
+0x8DF6	0x5D96	
+0x8DF7	0x5D97	
+0x8DF8	0x5D98	
+0x8DF9	0x5D9A	
+0x8DFA	0x5D9B	
+0x8DFB	0x5D9C	
+0x8DFC	0x5D9E	
+0x8DFD	0x5D9F	
+0x8DFE	0x5DA0	
+0x8E40	0x5DA1	
+0x8E41	0x5DA2	
+0x8E42	0x5DA3	
+0x8E43	0x5DA4	
+0x8E44	0x5DA5	
+0x8E45	0x5DA6	
+0x8E46	0x5DA7	
+0x8E47	0x5DA8	
+0x8E48	0x5DA9	
+0x8E49	0x5DAA	
+0x8E4A	0x5DAB	
+0x8E4B	0x5DAC	
+0x8E4C	0x5DAD	
+0x8E4D	0x5DAE	
+0x8E4E	0x5DAF	
+0x8E4F	0x5DB0	
+0x8E50	0x5DB1	
+0x8E51	0x5DB2	
+0x8E52	0x5DB3	
+0x8E53	0x5DB4	
+0x8E54	0x5DB5	
+0x8E55	0x5DB6	
+0x8E56	0x5DB8	
+0x8E57	0x5DB9	
+0x8E58	0x5DBA	
+0x8E59	0x5DBB	
+0x8E5A	0x5DBC	
+0x8E5B	0x5DBD	
+0x8E5C	0x5DBE	
+0x8E5D	0x5DBF	
+0x8E5E	0x5DC0	
+0x8E5F	0x5DC1	
+0x8E60	0x5DC2	
+0x8E61	0x5DC3	
+0x8E62	0x5DC4	
+0x8E63	0x5DC6	
+0x8E64	0x5DC7	
+0x8E65	0x5DC8	
+0x8E66	0x5DC9	
+0x8E67	0x5DCA	
+0x8E68	0x5DCB	
+0x8E69	0x5DCC	
+0x8E6A	0x5DCE	
+0x8E6B	0x5DCF	
+0x8E6C	0x5DD0	
+0x8E6D	0x5DD1	
+0x8E6E	0x5DD2	
+0x8E6F	0x5DD3	
+0x8E70	0x5DD4	
+0x8E71	0x5DD5	
+0x8E72	0x5DD6	
+0x8E73	0x5DD7	
+0x8E74	0x5DD8	
+0x8E75	0x5DD9	
+0x8E76	0x5DDA	
+0x8E77	0x5DDC	
+0x8E78	0x5DDF	
+0x8E79	0x5DE0	
+0x8E7A	0x5DE3	
+0x8E7B	0x5DE4	
+0x8E7C	0x5DEA	
+0x8E7D	0x5DEC	
+0x8E7E	0x5DED	
+0x8E80	0x5DF0	
+0x8E81	0x5DF5	
+0x8E82	0x5DF6	
+0x8E83	0x5DF8	
+0x8E84	0x5DF9	
+0x8E85	0x5DFA	
+0x8E86	0x5DFB	
+0x8E87	0x5DFC	
+0x8E88	0x5DFF	
+0x8E89	0x5E00	
+0x8E8A	0x5E04	
+0x8E8B	0x5E07	
+0x8E8C	0x5E09	
+0x8E8D	0x5E0A	
+0x8E8E	0x5E0B	
+0x8E8F	0x5E0D	
+0x8E90	0x5E0E	
+0x8E91	0x5E12	
+0x8E92	0x5E13	
+0x8E93	0x5E17	
+0x8E94	0x5E1E	
+0x8E95	0x5E1F	
+0x8E96	0x5E20	
+0x8E97	0x5E21	
+0x8E98	0x5E22	
+0x8E99	0x5E23	
+0x8E9A	0x5E24	
+0x8E9B	0x5E25	
+0x8E9C	0x5E28	
+0x8E9D	0x5E29	
+0x8E9E	0x5E2A	
+0x8E9F	0x5E2B	
+0x8EA0	0x5E2C	
+0x8EA1	0x5E2F	
+0x8EA2	0x5E30	
+0x8EA3	0x5E32	
+0x8EA4	0x5E33	
+0x8EA5	0x5E34	
+0x8EA6	0x5E35	
+0x8EA7	0x5E36	
+0x8EA8	0x5E39	
+0x8EA9	0x5E3A	
+0x8EAA	0x5E3E	
+0x8EAB	0x5E3F	
+0x8EAC	0x5E40	
+0x8EAD	0x5E41	
+0x8EAE	0x5E43	
+0x8EAF	0x5E46	
+0x8EB0	0x5E47	
+0x8EB1	0x5E48	
+0x8EB2	0x5E49	
+0x8EB3	0x5E4A	
+0x8EB4	0x5E4B	
+0x8EB5	0x5E4D	
+0x8EB6	0x5E4E	
+0x8EB7	0x5E4F	
+0x8EB8	0x5E50	
+0x8EB9	0x5E51	
+0x8EBA	0x5E52	
+0x8EBB	0x5E53	
+0x8EBC	0x5E56	
+0x8EBD	0x5E57	
+0x8EBE	0x5E58	
+0x8EBF	0x5E59	
+0x8EC0	0x5E5A	
+0x8EC1	0x5E5C	
+0x8EC2	0x5E5D	
+0x8EC3	0x5E5F	
+0x8EC4	0x5E60	
+0x8EC5	0x5E63	
+0x8EC6	0x5E64	
+0x8EC7	0x5E65	
+0x8EC8	0x5E66	
+0x8EC9	0x5E67	
+0x8ECA	0x5E68	
+0x8ECB	0x5E69	
+0x8ECC	0x5E6A	
+0x8ECD	0x5E6B	
+0x8ECE	0x5E6C	
+0x8ECF	0x5E6D	
+0x8ED0	0x5E6E	
+0x8ED1	0x5E6F	
+0x8ED2	0x5E70	
+0x8ED3	0x5E71	
+0x8ED4	0x5E75	
+0x8ED5	0x5E77	
+0x8ED6	0x5E79	
+0x8ED7	0x5E7E	
+0x8ED8	0x5E81	
+0x8ED9	0x5E82	
+0x8EDA	0x5E83	
+0x8EDB	0x5E85	
+0x8EDC	0x5E88	
+0x8EDD	0x5E89	
+0x8EDE	0x5E8C	
+0x8EDF	0x5E8D	
+0x8EE0	0x5E8E	
+0x8EE1	0x5E92	
+0x8EE2	0x5E98	
+0x8EE3	0x5E9B	
+0x8EE4	0x5E9D	
+0x8EE5	0x5EA1	
+0x8EE6	0x5EA2	
+0x8EE7	0x5EA3	
+0x8EE8	0x5EA4	
+0x8EE9	0x5EA8	
+0x8EEA	0x5EA9	
+0x8EEB	0x5EAA	
+0x8EEC	0x5EAB	
+0x8EED	0x5EAC	
+0x8EEE	0x5EAE	
+0x8EEF	0x5EAF	
+0x8EF0	0x5EB0	
+0x8EF1	0x5EB1	
+0x8EF2	0x5EB2	
+0x8EF3	0x5EB4	
+0x8EF4	0x5EBA	
+0x8EF5	0x5EBB	
+0x8EF6	0x5EBC	
+0x8EF7	0x5EBD	
+0x8EF8	0x5EBF	
+0x8EF9	0x5EC0	
+0x8EFA	0x5EC1	
+0x8EFB	0x5EC2	
+0x8EFC	0x5EC3	
+0x8EFD	0x5EC4	
+0x8EFE	0x5EC5	
+0x8F40	0x5EC6	
+0x8F41	0x5EC7	
+0x8F42	0x5EC8	
+0x8F43	0x5ECB	
+0x8F44	0x5ECC	
+0x8F45	0x5ECD	
+0x8F46	0x5ECE	
+0x8F47	0x5ECF	
+0x8F48	0x5ED0	
+0x8F49	0x5ED4	
+0x8F4A	0x5ED5	
+0x8F4B	0x5ED7	
+0x8F4C	0x5ED8	
+0x8F4D	0x5ED9	
+0x8F4E	0x5EDA	
+0x8F4F	0x5EDC	
+0x8F50	0x5EDD	
+0x8F51	0x5EDE	
+0x8F52	0x5EDF	
+0x8F53	0x5EE0	
+0x8F54	0x5EE1	
+0x8F55	0x5EE2	
+0x8F56	0x5EE3	
+0x8F57	0x5EE4	
+0x8F58	0x5EE5	
+0x8F59	0x5EE6	
+0x8F5A	0x5EE7	
+0x8F5B	0x5EE9	
+0x8F5C	0x5EEB	
+0x8F5D	0x5EEC	
+0x8F5E	0x5EED	
+0x8F5F	0x5EEE	
+0x8F60	0x5EEF	
+0x8F61	0x5EF0	
+0x8F62	0x5EF1	
+0x8F63	0x5EF2	
+0x8F64	0x5EF3	
+0x8F65	0x5EF5	
+0x8F66	0x5EF8	
+0x8F67	0x5EF9	
+0x8F68	0x5EFB	
+0x8F69	0x5EFC	
+0x8F6A	0x5EFD	
+0x8F6B	0x5F05	
+0x8F6C	0x5F06	
+0x8F6D	0x5F07	
+0x8F6E	0x5F09	
+0x8F6F	0x5F0C	
+0x8F70	0x5F0D	
+0x8F71	0x5F0E	
+0x8F72	0x5F10	
+0x8F73	0x5F12	
+0x8F74	0x5F14	
+0x8F75	0x5F16	
+0x8F76	0x5F19	
+0x8F77	0x5F1A	
+0x8F78	0x5F1C	
+0x8F79	0x5F1D	
+0x8F7A	0x5F1E	
+0x8F7B	0x5F21	
+0x8F7C	0x5F22	
+0x8F7D	0x5F23	
+0x8F7E	0x5F24	
+0x8F80	0x5F28	
+0x8F81	0x5F2B	
+0x8F82	0x5F2C	
+0x8F83	0x5F2E	
+0x8F84	0x5F30	
+0x8F85	0x5F32	
+0x8F86	0x5F33	
+0x8F87	0x5F34	
+0x8F88	0x5F35	
+0x8F89	0x5F36	
+0x8F8A	0x5F37	
+0x8F8B	0x5F38	
+0x8F8C	0x5F3B	
+0x8F8D	0x5F3D	
+0x8F8E	0x5F3E	
+0x8F8F	0x5F3F	
+0x8F90	0x5F41	
+0x8F91	0x5F42	
+0x8F92	0x5F43	
+0x8F93	0x5F44	
+0x8F94	0x5F45	
+0x8F95	0x5F46	
+0x8F96	0x5F47	
+0x8F97	0x5F48	
+0x8F98	0x5F49	
+0x8F99	0x5F4A	
+0x8F9A	0x5F4B	
+0x8F9B	0x5F4C	
+0x8F9C	0x5F4D	
+0x8F9D	0x5F4E	
+0x8F9E	0x5F4F	
+0x8F9F	0x5F51	
+0x8FA0	0x5F54	
+0x8FA1	0x5F59	
+0x8FA2	0x5F5A	
+0x8FA3	0x5F5B	
+0x8FA4	0x5F5C	
+0x8FA5	0x5F5E	
+0x8FA6	0x5F5F	
+0x8FA7	0x5F60	
+0x8FA8	0x5F63	
+0x8FA9	0x5F65	
+0x8FAA	0x5F67	
+0x8FAB	0x5F68	
+0x8FAC	0x5F6B	
+0x8FAD	0x5F6E	
+0x8FAE	0x5F6F	
+0x8FAF	0x5F72	
+0x8FB0	0x5F74	
+0x8FB1	0x5F75	
+0x8FB2	0x5F76	
+0x8FB3	0x5F78	
+0x8FB4	0x5F7A	
+0x8FB5	0x5F7D	
+0x8FB6	0x5F7E	
+0x8FB7	0x5F7F	
+0x8FB8	0x5F83	
+0x8FB9	0x5F86	
+0x8FBA	0x5F8D	
+0x8FBB	0x5F8E	
+0x8FBC	0x5F8F	
+0x8FBD	0x5F91	
+0x8FBE	0x5F93	
+0x8FBF	0x5F94	
+0x8FC0	0x5F96	
+0x8FC1	0x5F9A	
+0x8FC2	0x5F9B	
+0x8FC3	0x5F9D	
+0x8FC4	0x5F9E	
+0x8FC5	0x5F9F	
+0x8FC6	0x5FA0	
+0x8FC7	0x5FA2	
+0x8FC8	0x5FA3	
+0x8FC9	0x5FA4	
+0x8FCA	0x5FA5	
+0x8FCB	0x5FA6	
+0x8FCC	0x5FA7	
+0x8FCD	0x5FA9	
+0x8FCE	0x5FAB	
+0x8FCF	0x5FAC	
+0x8FD0	0x5FAF	
+0x8FD1	0x5FB0	
+0x8FD2	0x5FB1	
+0x8FD3	0x5FB2	
+0x8FD4	0x5FB3	
+0x8FD5	0x5FB4	
+0x8FD6	0x5FB6	
+0x8FD7	0x5FB8	
+0x8FD8	0x5FB9	
+0x8FD9	0x5FBA	
+0x8FDA	0x5FBB	
+0x8FDB	0x5FBE	
+0x8FDC	0x5FBF	
+0x8FDD	0x5FC0	
+0x8FDE	0x5FC1	
+0x8FDF	0x5FC2	
+0x8FE0	0x5FC7	
+0x8FE1	0x5FC8	
+0x8FE2	0x5FCA	
+0x8FE3	0x5FCB	
+0x8FE4	0x5FCE	
+0x8FE5	0x5FD3	
+0x8FE6	0x5FD4	
+0x8FE7	0x5FD5	
+0x8FE8	0x5FDA	
+0x8FE9	0x5FDB	
+0x8FEA	0x5FDC	
+0x8FEB	0x5FDE	
+0x8FEC	0x5FDF	
+0x8FED	0x5FE2	
+0x8FEE	0x5FE3	
+0x8FEF	0x5FE5	
+0x8FF0	0x5FE6	
+0x8FF1	0x5FE8	
+0x8FF2	0x5FE9	
+0x8FF3	0x5FEC	
+0x8FF4	0x5FEF	
+0x8FF5	0x5FF0	
+0x8FF6	0x5FF2	
+0x8FF7	0x5FF3	
+0x8FF8	0x5FF4	
+0x8FF9	0x5FF6	
+0x8FFA	0x5FF7	
+0x8FFB	0x5FF9	
+0x8FFC	0x5FFA	
+0x8FFD	0x5FFC	
+0x8FFE	0x6007	
+0x9040	0x6008	
+0x9041	0x6009	
+0x9042	0x600B	
+0x9043	0x600C	
+0x9044	0x6010	
+0x9045	0x6011	
+0x9046	0x6013	
+0x9047	0x6017	
+0x9048	0x6018	
+0x9049	0x601A	
+0x904A	0x601E	
+0x904B	0x601F	
+0x904C	0x6022	
+0x904D	0x6023	
+0x904E	0x6024	
+0x904F	0x602C	
+0x9050	0x602D	
+0x9051	0x602E	
+0x9052	0x6030	
+0x9053	0x6031	
+0x9054	0x6032	
+0x9055	0x6033	
+0x9056	0x6034	
+0x9057	0x6036	
+0x9058	0x6037	
+0x9059	0x6038	
+0x905A	0x6039	
+0x905B	0x603A	
+0x905C	0x603D	
+0x905D	0x603E	
+0x905E	0x6040	
+0x905F	0x6044	
+0x9060	0x6045	
+0x9061	0x6046	
+0x9062	0x6047	
+0x9063	0x6048	
+0x9064	0x6049	
+0x9065	0x604A	
+0x9066	0x604C	
+0x9067	0x604E	
+0x9068	0x604F	
+0x9069	0x6051	
+0x906A	0x6053	
+0x906B	0x6054	
+0x906C	0x6056	
+0x906D	0x6057	
+0x906E	0x6058	
+0x906F	0x605B	
+0x9070	0x605C	
+0x9071	0x605E	
+0x9072	0x605F	
+0x9073	0x6060	
+0x9074	0x6061	
+0x9075	0x6065	
+0x9076	0x6066	
+0x9077	0x606E	
+0x9078	0x6071	
+0x9079	0x6072	
+0x907A	0x6074	
+0x907B	0x6075	
+0x907C	0x6077	
+0x907D	0x607E	
+0x907E	0x6080	
+0x9080	0x6081	
+0x9081	0x6082	
+0x9082	0x6085	
+0x9083	0x6086	
+0x9084	0x6087	
+0x9085	0x6088	
+0x9086	0x608A	
+0x9087	0x608B	
+0x9088	0x608E	
+0x9089	0x608F	
+0x908A	0x6090	
+0x908B	0x6091	
+0x908C	0x6093	
+0x908D	0x6095	
+0x908E	0x6097	
+0x908F	0x6098	
+0x9090	0x6099	
+0x9091	0x609C	
+0x9092	0x609E	
+0x9093	0x60A1	
+0x9094	0x60A2	
+0x9095	0x60A4	
+0x9096	0x60A5	
+0x9097	0x60A7	
+0x9098	0x60A9	
+0x9099	0x60AA	
+0x909A	0x60AE	
+0x909B	0x60B0	
+0x909C	0x60B3	
+0x909D	0x60B5	
+0x909E	0x60B6	
+0x909F	0x60B7	
+0x90A0	0x60B9	
+0x90A1	0x60BA	
+0x90A2	0x60BD	
+0x90A3	0x60BE	
+0x90A4	0x60BF	
+0x90A5	0x60C0	
+0x90A6	0x60C1	
+0x90A7	0x60C2	
+0x90A8	0x60C3	
+0x90A9	0x60C4	
+0x90AA	0x60C7	
+0x90AB	0x60C8	
+0x90AC	0x60C9	
+0x90AD	0x60CC	
+0x90AE	0x60CD	
+0x90AF	0x60CE	
+0x90B0	0x60CF	
+0x90B1	0x60D0	
+0x90B2	0x60D2	
+0x90B3	0x60D3	
+0x90B4	0x60D4	
+0x90B5	0x60D6	
+0x90B6	0x60D7	
+0x90B7	0x60D9	
+0x90B8	0x60DB	
+0x90B9	0x60DE	
+0x90BA	0x60E1	
+0x90BB	0x60E2	
+0x90BC	0x60E3	
+0x90BD	0x60E4	
+0x90BE	0x60E5	
+0x90BF	0x60EA	
+0x90C0	0x60F1	
+0x90C1	0x60F2	
+0x90C2	0x60F5	
+0x90C3	0x60F7	
+0x90C4	0x60F8	
+0x90C5	0x60FB	
+0x90C6	0x60FC	
+0x90C7	0x60FD	
+0x90C8	0x60FE	
+0x90C9	0x60FF	
+0x90CA	0x6102	
+0x90CB	0x6103	
+0x90CC	0x6104	
+0x90CD	0x6105	
+0x90CE	0x6107	
+0x90CF	0x610A	
+0x90D0	0x610B	
+0x90D1	0x610C	
+0x90D2	0x6110	
+0x90D3	0x6111	
+0x90D4	0x6112	
+0x90D5	0x6113	
+0x90D6	0x6114	
+0x90D7	0x6116	
+0x90D8	0x6117	
+0x90D9	0x6118	
+0x90DA	0x6119	
+0x90DB	0x611B	
+0x90DC	0x611C	
+0x90DD	0x611D	
+0x90DE	0x611E	
+0x90DF	0x6121	
+0x90E0	0x6122	
+0x90E1	0x6125	
+0x90E2	0x6128	
+0x90E3	0x6129	
+0x90E4	0x612A	
+0x90E5	0x612C	
+0x90E6	0x612D	
+0x90E7	0x612E	
+0x90E8	0x612F	
+0x90E9	0x6130	
+0x90EA	0x6131	
+0x90EB	0x6132	
+0x90EC	0x6133	
+0x90ED	0x6134	
+0x90EE	0x6135	
+0x90EF	0x6136	
+0x90F0	0x6137	
+0x90F1	0x6138	
+0x90F2	0x6139	
+0x90F3	0x613A	
+0x90F4	0x613B	
+0x90F5	0x613C	
+0x90F6	0x613D	
+0x90F7	0x613E	
+0x90F8	0x6140	
+0x90F9	0x6141	
+0x90FA	0x6142	
+0x90FB	0x6143	
+0x90FC	0x6144	
+0x90FD	0x6145	
+0x90FE	0x6146	
+0x9140	0x6147	
+0x9141	0x6149	
+0x9142	0x614B	
+0x9143	0x614D	
+0x9144	0x614F	
+0x9145	0x6150	
+0x9146	0x6152	
+0x9147	0x6153	
+0x9148	0x6154	
+0x9149	0x6156	
+0x914A	0x6157	
+0x914B	0x6158	
+0x914C	0x6159	
+0x914D	0x615A	
+0x914E	0x615B	
+0x914F	0x615C	
+0x9150	0x615E	
+0x9151	0x615F	
+0x9152	0x6160	
+0x9153	0x6161	
+0x9154	0x6163	
+0x9155	0x6164	
+0x9156	0x6165	
+0x9157	0x6166	
+0x9158	0x6169	
+0x9159	0x616A	
+0x915A	0x616B	
+0x915B	0x616C	
+0x915C	0x616D	
+0x915D	0x616E	
+0x915E	0x616F	
+0x915F	0x6171	
+0x9160	0x6172	
+0x9161	0x6173	
+0x9162	0x6174	
+0x9163	0x6176	
+0x9164	0x6178	
+0x9165	0x6179	
+0x9166	0x617A	
+0x9167	0x617B	
+0x9168	0x617C	
+0x9169	0x617D	
+0x916A	0x617E	
+0x916B	0x617F	
+0x916C	0x6180	
+0x916D	0x6181	
+0x916E	0x6182	
+0x916F	0x6183	
+0x9170	0x6184	
+0x9171	0x6185	
+0x9172	0x6186	
+0x9173	0x6187	
+0x9174	0x6188	
+0x9175	0x6189	
+0x9176	0x618A	
+0x9177	0x618C	
+0x9178	0x618D	
+0x9179	0x618F	
+0x917A	0x6190	
+0x917B	0x6191	
+0x917C	0x6192	
+0x917D	0x6193	
+0x917E	0x6195	
+0x9180	0x6196	
+0x9181	0x6197	
+0x9182	0x6198	
+0x9183	0x6199	
+0x9184	0x619A	
+0x9185	0x619B	
+0x9186	0x619C	
+0x9187	0x619E	
+0x9188	0x619F	
+0x9189	0x61A0	
+0x918A	0x61A1	
+0x918B	0x61A2	
+0x918C	0x61A3	
+0x918D	0x61A4	
+0x918E	0x61A5	
+0x918F	0x61A6	
+0x9190	0x61AA	
+0x9191	0x61AB	
+0x9192	0x61AD	
+0x9193	0x61AE	
+0x9194	0x61AF	
+0x9195	0x61B0	
+0x9196	0x61B1	
+0x9197	0x61B2	
+0x9198	0x61B3	
+0x9199	0x61B4	
+0x919A	0x61B5	
+0x919B	0x61B6	
+0x919C	0x61B8	
+0x919D	0x61B9	
+0x919E	0x61BA	
+0x919F	0x61BB	
+0x91A0	0x61BC	
+0x91A1	0x61BD	
+0x91A2	0x61BF	
+0x91A3	0x61C0	
+0x91A4	0x61C1	
+0x91A5	0x61C3	
+0x91A6	0x61C4	
+0x91A7	0x61C5	
+0x91A8	0x61C6	
+0x91A9	0x61C7	
+0x91AA	0x61C9	
+0x91AB	0x61CC	
+0x91AC	0x61CD	
+0x91AD	0x61CE	
+0x91AE	0x61CF	
+0x91AF	0x61D0	
+0x91B0	0x61D3	
+0x91B1	0x61D5	
+0x91B2	0x61D6	
+0x91B3	0x61D7	
+0x91B4	0x61D8	
+0x91B5	0x61D9	
+0x91B6	0x61DA	
+0x91B7	0x61DB	
+0x91B8	0x61DC	
+0x91B9	0x61DD	
+0x91BA	0x61DE	
+0x91BB	0x61DF	
+0x91BC	0x61E0	
+0x91BD	0x61E1	
+0x91BE	0x61E2	
+0x91BF	0x61E3	
+0x91C0	0x61E4	
+0x91C1	0x61E5	
+0x91C2	0x61E7	
+0x91C3	0x61E8	
+0x91C4	0x61E9	
+0x91C5	0x61EA	
+0x91C6	0x61EB	
+0x91C7	0x61EC	
+0x91C8	0x61ED	
+0x91C9	0x61EE	
+0x91CA	0x61EF	
+0x91CB	0x61F0	
+0x91CC	0x61F1	
+0x91CD	0x61F2	
+0x91CE	0x61F3	
+0x91CF	0x61F4	
+0x91D0	0x61F6	
+0x91D1	0x61F7	
+0x91D2	0x61F8	
+0x91D3	0x61F9	
+0x91D4	0x61FA	
+0x91D5	0x61FB	
+0x91D6	0x61FC	
+0x91D7	0x61FD	
+0x91D8	0x61FE	
+0x91D9	0x6200	
+0x91DA	0x6201	
+0x91DB	0x6202	
+0x91DC	0x6203	
+0x91DD	0x6204	
+0x91DE	0x6205	
+0x91DF	0x6207	
+0x91E0	0x6209	
+0x91E1	0x6213	
+0x91E2	0x6214	
+0x91E3	0x6219	
+0x91E4	0x621C	
+0x91E5	0x621D	
+0x91E6	0x621E	
+0x91E7	0x6220	
+0x91E8	0x6223	
+0x91E9	0x6226	
+0x91EA	0x6227	
+0x91EB	0x6228	
+0x91EC	0x6229	
+0x91ED	0x622B	
+0x91EE	0x622D	
+0x91EF	0x622F	
+0x91F0	0x6230	
+0x91F1	0x6231	
+0x91F2	0x6232	
+0x91F3	0x6235	
+0x91F4	0x6236	
+0x91F5	0x6238	
+0x91F6	0x6239	
+0x91F7	0x623A	
+0x91F8	0x623B	
+0x91F9	0x623C	
+0x91FA	0x6242	
+0x91FB	0x6244	
+0x91FC	0x6245	
+0x91FD	0x6246	
+0x91FE	0x624A	
+0x9240	0x624F	
+0x9241	0x6250	
+0x9242	0x6255	
+0x9243	0x6256	
+0x9244	0x6257	
+0x9245	0x6259	
+0x9246	0x625A	
+0x9247	0x625C	
+0x9248	0x625D	
+0x9249	0x625E	
+0x924A	0x625F	
+0x924B	0x6260	
+0x924C	0x6261	
+0x924D	0x6262	
+0x924E	0x6264	
+0x924F	0x6265	
+0x9250	0x6268	
+0x9251	0x6271	
+0x9252	0x6272	
+0x9253	0x6274	
+0x9254	0x6275	
+0x9255	0x6277	
+0x9256	0x6278	
+0x9257	0x627A	
+0x9258	0x627B	
+0x9259	0x627D	
+0x925A	0x6281	
+0x925B	0x6282	
+0x925C	0x6283	
+0x925D	0x6285	
+0x925E	0x6286	
+0x925F	0x6287	
+0x9260	0x6288	
+0x9261	0x628B	
+0x9262	0x628C	
+0x9263	0x628D	
+0x9264	0x628E	
+0x9265	0x628F	
+0x9266	0x6290	
+0x9267	0x6294	
+0x9268	0x6299	
+0x9269	0x629C	
+0x926A	0x629D	
+0x926B	0x629E	
+0x926C	0x62A3	
+0x926D	0x62A6	
+0x926E	0x62A7	
+0x926F	0x62A9	
+0x9270	0x62AA	
+0x9271	0x62AD	
+0x9272	0x62AE	
+0x9273	0x62AF	
+0x9274	0x62B0	
+0x9275	0x62B2	
+0x9276	0x62B3	
+0x9277	0x62B4	
+0x9278	0x62B6	
+0x9279	0x62B7	
+0x927A	0x62B8	
+0x927B	0x62BA	
+0x927C	0x62BE	
+0x927D	0x62C0	
+0x927E	0x62C1	
+0x9280	0x62C3	
+0x9281	0x62CB	
+0x9282	0x62CF	
+0x9283	0x62D1	
+0x9284	0x62D5	
+0x9285	0x62DD	
+0x9286	0x62DE	
+0x9287	0x62E0	
+0x9288	0x62E1	
+0x9289	0x62E4	
+0x928A	0x62EA	
+0x928B	0x62EB	
+0x928C	0x62F0	
+0x928D	0x62F2	
+0x928E	0x62F5	
+0x928F	0x62F8	
+0x9290	0x62F9	
+0x9291	0x62FA	
+0x9292	0x62FB	
+0x9293	0x6300	
+0x9294	0x6303	
+0x9295	0x6304	
+0x9296	0x6305	
+0x9297	0x6306	
+0x9298	0x630A	
+0x9299	0x630B	
+0x929A	0x630C	
+0x929B	0x630D	
+0x929C	0x630F	
+0x929D	0x6310	
+0x929E	0x6312	
+0x929F	0x6313	
+0x92A0	0x6314	
+0x92A1	0x6315	
+0x92A2	0x6317	
+0x92A3	0x6318	
+0x92A4	0x6319	
+0x92A5	0x631C	
+0x92A6	0x6326	
+0x92A7	0x6327	
+0x92A8	0x6329	
+0x92A9	0x632C	
+0x92AA	0x632D	
+0x92AB	0x632E	
+0x92AC	0x6330	
+0x92AD	0x6331	
+0x92AE	0x6333	
+0x92AF	0x6334	
+0x92B0	0x6335	
+0x92B1	0x6336	
+0x92B2	0x6337	
+0x92B3	0x6338	
+0x92B4	0x633B	
+0x92B5	0x633C	
+0x92B6	0x633E	
+0x92B7	0x633F	
+0x92B8	0x6340	
+0x92B9	0x6341	
+0x92BA	0x6344	
+0x92BB	0x6347	
+0x92BC	0x6348	
+0x92BD	0x634A	
+0x92BE	0x6351	
+0x92BF	0x6352	
+0x92C0	0x6353	
+0x92C1	0x6354	
+0x92C2	0x6356	
+0x92C3	0x6357	
+0x92C4	0x6358	
+0x92C5	0x6359	
+0x92C6	0x635A	
+0x92C7	0x635B	
+0x92C8	0x635C	
+0x92C9	0x635D	
+0x92CA	0x6360	
+0x92CB	0x6364	
+0x92CC	0x6365	
+0x92CD	0x6366	
+0x92CE	0x6368	
+0x92CF	0x636A	
+0x92D0	0x636B	
+0x92D1	0x636C	
+0x92D2	0x636F	
+0x92D3	0x6370	
+0x92D4	0x6372	
+0x92D5	0x6373	
+0x92D6	0x6374	
+0x92D7	0x6375	
+0x92D8	0x6378	
+0x92D9	0x6379	
+0x92DA	0x637C	
+0x92DB	0x637D	
+0x92DC	0x637E	
+0x92DD	0x637F	
+0x92DE	0x6381	
+0x92DF	0x6383	
+0x92E0	0x6384	
+0x92E1	0x6385	
+0x92E2	0x6386	
+0x92E3	0x638B	
+0x92E4	0x638D	
+0x92E5	0x6391	
+0x92E6	0x6393	
+0x92E7	0x6394	
+0x92E8	0x6395	
+0x92E9	0x6397	
+0x92EA	0x6399	
+0x92EB	0x639A	
+0x92EC	0x639B	
+0x92ED	0x639C	
+0x92EE	0x639D	
+0x92EF	0x639E	
+0x92F0	0x639F	
+0x92F1	0x63A1	
+0x92F2	0x63A4	
+0x92F3	0x63A6	
+0x92F4	0x63AB	
+0x92F5	0x63AF	
+0x92F6	0x63B1	
+0x92F7	0x63B2	
+0x92F8	0x63B5	
+0x92F9	0x63B6	
+0x92FA	0x63B9	
+0x92FB	0x63BB	
+0x92FC	0x63BD	
+0x92FD	0x63BF	
+0x92FE	0x63C0	
+0x9340	0x63C1	
+0x9341	0x63C2	
+0x9342	0x63C3	
+0x9343	0x63C5	
+0x9344	0x63C7	
+0x9345	0x63C8	
+0x9346	0x63CA	
+0x9347	0x63CB	
+0x9348	0x63CC	
+0x9349	0x63D1	
+0x934A	0x63D3	
+0x934B	0x63D4	
+0x934C	0x63D5	
+0x934D	0x63D7	
+0x934E	0x63D8	
+0x934F	0x63D9	
+0x9350	0x63DA	
+0x9351	0x63DB	
+0x9352	0x63DC	
+0x9353	0x63DD	
+0x9354	0x63DF	
+0x9355	0x63E2	
+0x9356	0x63E4	
+0x9357	0x63E5	
+0x9358	0x63E6	
+0x9359	0x63E7	
+0x935A	0x63E8	
+0x935B	0x63EB	
+0x935C	0x63EC	
+0x935D	0x63EE	
+0x935E	0x63EF	
+0x935F	0x63F0	
+0x9360	0x63F1	
+0x9361	0x63F3	
+0x9362	0x63F5	
+0x9363	0x63F7	
+0x9364	0x63F9	
+0x9365	0x63FA	
+0x9366	0x63FB	
+0x9367	0x63FC	
+0x9368	0x63FE	
+0x9369	0x6403	
+0x936A	0x6404	
+0x936B	0x6406	
+0x936C	0x6407	
+0x936D	0x6408	
+0x936E	0x6409	
+0x936F	0x640A	
+0x9370	0x640D	
+0x9371	0x640E	
+0x9372	0x6411	
+0x9373	0x6412	
+0x9374	0x6415	
+0x9375	0x6416	
+0x9376	0x6417	
+0x9377	0x6418	
+0x9378	0x6419	
+0x9379	0x641A	
+0x937A	0x641D	
+0x937B	0x641F	
+0x937C	0x6422	
+0x937D	0x6423	
+0x937E	0x6424	
+0x9380	0x6425	
+0x9381	0x6427	
+0x9382	0x6428	
+0x9383	0x6429	
+0x9384	0x642B	
+0x9385	0x642E	
+0x9386	0x642F	
+0x9387	0x6430	
+0x9388	0x6431	
+0x9389	0x6432	
+0x938A	0x6433	
+0x938B	0x6435	
+0x938C	0x6436	
+0x938D	0x6437	
+0x938E	0x6438	
+0x938F	0x6439	
+0x9390	0x643B	
+0x9391	0x643C	
+0x9392	0x643E	
+0x9393	0x6440	
+0x9394	0x6442	
+0x9395	0x6443	
+0x9396	0x6449	
+0x9397	0x644B	
+0x9398	0x644C	
+0x9399	0x644D	
+0x939A	0x644E	
+0x939B	0x644F	
+0x939C	0x6450	
+0x939D	0x6451	
+0x939E	0x6453	
+0x939F	0x6455	
+0x93A0	0x6456	
+0x93A1	0x6457	
+0x93A2	0x6459	
+0x93A3	0x645A	
+0x93A4	0x645B	
+0x93A5	0x645C	
+0x93A6	0x645D	
+0x93A7	0x645F	
+0x93A8	0x6460	
+0x93A9	0x6461	
+0x93AA	0x6462	
+0x93AB	0x6463	
+0x93AC	0x6464	
+0x93AD	0x6465	
+0x93AE	0x6466	
+0x93AF	0x6468	
+0x93B0	0x646A	
+0x93B1	0x646B	
+0x93B2	0x646C	
+0x93B3	0x646E	
+0x93B4	0x646F	
+0x93B5	0x6470	
+0x93B6	0x6471	
+0x93B7	0x6472	
+0x93B8	0x6473	
+0x93B9	0x6474	
+0x93BA	0x6475	
+0x93BB	0x6476	
+0x93BC	0x6477	
+0x93BD	0x647B	
+0x93BE	0x647C	
+0x93BF	0x647D	
+0x93C0	0x647E	
+0x93C1	0x647F	
+0x93C2	0x6480	
+0x93C3	0x6481	
+0x93C4	0x6483	
+0x93C5	0x6486	
+0x93C6	0x6488	
+0x93C7	0x6489	
+0x93C8	0x648A	
+0x93C9	0x648B	
+0x93CA	0x648C	
+0x93CB	0x648D	
+0x93CC	0x648E	
+0x93CD	0x648F	
+0x93CE	0x6490	
+0x93CF	0x6493	
+0x93D0	0x6494	
+0x93D1	0x6497	
+0x93D2	0x6498	
+0x93D3	0x649A	
+0x93D4	0x649B	
+0x93D5	0x649C	
+0x93D6	0x649D	
+0x93D7	0x649F	
+0x93D8	0x64A0	
+0x93D9	0x64A1	
+0x93DA	0x64A2	
+0x93DB	0x64A3	
+0x93DC	0x64A5	
+0x93DD	0x64A6	
+0x93DE	0x64A7	
+0x93DF	0x64A8	
+0x93E0	0x64AA	
+0x93E1	0x64AB	
+0x93E2	0x64AF	
+0x93E3	0x64B1	
+0x93E4	0x64B2	
+0x93E5	0x64B3	
+0x93E6	0x64B4	
+0x93E7	0x64B6	
+0x93E8	0x64B9	
+0x93E9	0x64BB	
+0x93EA	0x64BD	
+0x93EB	0x64BE	
+0x93EC	0x64BF	
+0x93ED	0x64C1	
+0x93EE	0x64C3	
+0x93EF	0x64C4	
+0x93F0	0x64C6	
+0x93F1	0x64C7	
+0x93F2	0x64C8	
+0x93F3	0x64C9	
+0x93F4	0x64CA	
+0x93F5	0x64CB	
+0x93F6	0x64CC	
+0x93F7	0x64CF	
+0x93F8	0x64D1	
+0x93F9	0x64D3	
+0x93FA	0x64D4	
+0x93FB	0x64D5	
+0x93FC	0x64D6	
+0x93FD	0x64D9	
+0x93FE	0x64DA	
+0x9440	0x64DB	
+0x9441	0x64DC	
+0x9442	0x64DD	
+0x9443	0x64DF	
+0x9444	0x64E0	
+0x9445	0x64E1	
+0x9446	0x64E3	
+0x9447	0x64E5	
+0x9448	0x64E7	
+0x9449	0x64E8	
+0x944A	0x64E9	
+0x944B	0x64EA	
+0x944C	0x64EB	
+0x944D	0x64EC	
+0x944E	0x64ED	
+0x944F	0x64EE	
+0x9450	0x64EF	
+0x9451	0x64F0	
+0x9452	0x64F1	
+0x9453	0x64F2	
+0x9454	0x64F3	
+0x9455	0x64F4	
+0x9456	0x64F5	
+0x9457	0x64F6	
+0x9458	0x64F7	
+0x9459	0x64F8	
+0x945A	0x64F9	
+0x945B	0x64FA	
+0x945C	0x64FB	
+0x945D	0x64FC	
+0x945E	0x64FD	
+0x945F	0x64FE	
+0x9460	0x64FF	
+0x9461	0x6501	
+0x9462	0x6502	
+0x9463	0x6503	
+0x9464	0x6504	
+0x9465	0x6505	
+0x9466	0x6506	
+0x9467	0x6507	
+0x9468	0x6508	
+0x9469	0x650A	
+0x946A	0x650B	
+0x946B	0x650C	
+0x946C	0x650D	
+0x946D	0x650E	
+0x946E	0x650F	
+0x946F	0x6510	
+0x9470	0x6511	
+0x9471	0x6513	
+0x9472	0x6514	
+0x9473	0x6515	
+0x9474	0x6516	
+0x9475	0x6517	
+0x9476	0x6519	
+0x9477	0x651A	
+0x9478	0x651B	
+0x9479	0x651C	
+0x947A	0x651D	
+0x947B	0x651E	
+0x947C	0x651F	
+0x947D	0x6520	
+0x947E	0x6521	
+0x9480	0x6522	
+0x9481	0x6523	
+0x9482	0x6524	
+0x9483	0x6526	
+0x9484	0x6527	
+0x9485	0x6528	
+0x9486	0x6529	
+0x9487	0x652A	
+0x9488	0x652C	
+0x9489	0x652D	
+0x948A	0x6530	
+0x948B	0x6531	
+0x948C	0x6532	
+0x948D	0x6533	
+0x948E	0x6537	
+0x948F	0x653A	
+0x9490	0x653C	
+0x9491	0x653D	
+0x9492	0x6540	
+0x9493	0x6541	
+0x9494	0x6542	
+0x9495	0x6543	
+0x9496	0x6544	
+0x9497	0x6546	
+0x9498	0x6547	
+0x9499	0x654A	
+0x949A	0x654B	
+0x949B	0x654D	
+0x949C	0x654E	
+0x949D	0x6550	
+0x949E	0x6552	
+0x949F	0x6553	
+0x94A0	0x6554	
+0x94A1	0x6557	
+0x94A2	0x6558	
+0x94A3	0x655A	
+0x94A4	0x655C	
+0x94A5	0x655F	
+0x94A6	0x6560	
+0x94A7	0x6561	
+0x94A8	0x6564	
+0x94A9	0x6565	
+0x94AA	0x6567	
+0x94AB	0x6568	
+0x94AC	0x6569	
+0x94AD	0x656A	
+0x94AE	0x656D	
+0x94AF	0x656E	
+0x94B0	0x656F	
+0x94B1	0x6571	
+0x94B2	0x6573	
+0x94B3	0x6575	
+0x94B4	0x6576	
+0x94B5	0x6578	
+0x94B6	0x6579	
+0x94B7	0x657A	
+0x94B8	0x657B	
+0x94B9	0x657C	
+0x94BA	0x657D	
+0x94BB	0x657E	
+0x94BC	0x657F	
+0x94BD	0x6580	
+0x94BE	0x6581	
+0x94BF	0x6582	
+0x94C0	0x6583	
+0x94C1	0x6584	
+0x94C2	0x6585	
+0x94C3	0x6586	
+0x94C4	0x6588	
+0x94C5	0x6589	
+0x94C6	0x658A	
+0x94C7	0x658D	
+0x94C8	0x658E	
+0x94C9	0x658F	
+0x94CA	0x6592	
+0x94CB	0x6594	
+0x94CC	0x6595	
+0x94CD	0x6596	
+0x94CE	0x6598	
+0x94CF	0x659A	
+0x94D0	0x659D	
+0x94D1	0x659E	
+0x94D2	0x65A0	
+0x94D3	0x65A2	
+0x94D4	0x65A3	
+0x94D5	0x65A6	
+0x94D6	0x65A8	
+0x94D7	0x65AA	
+0x94D8	0x65AC	
+0x94D9	0x65AE	
+0x94DA	0x65B1	
+0x94DB	0x65B2	
+0x94DC	0x65B3	
+0x94DD	0x65B4	
+0x94DE	0x65B5	
+0x94DF	0x65B6	
+0x94E0	0x65B7	
+0x94E1	0x65B8	
+0x94E2	0x65BA	
+0x94E3	0x65BB	
+0x94E4	0x65BE	
+0x94E5	0x65BF	
+0x94E6	0x65C0	
+0x94E7	0x65C2	
+0x94E8	0x65C7	
+0x94E9	0x65C8	
+0x94EA	0x65C9	
+0x94EB	0x65CA	
+0x94EC	0x65CD	
+0x94ED	0x65D0	
+0x94EE	0x65D1	
+0x94EF	0x65D3	
+0x94F0	0x65D4	
+0x94F1	0x65D5	
+0x94F2	0x65D8	
+0x94F3	0x65D9	
+0x94F4	0x65DA	
+0x94F5	0x65DB	
+0x94F6	0x65DC	
+0x94F7	0x65DD	
+0x94F8	0x65DE	
+0x94F9	0x65DF	
+0x94FA	0x65E1	
+0x94FB	0x65E3	
+0x94FC	0x65E4	
+0x94FD	0x65EA	
+0x94FE	0x65EB	
+0x9540	0x65F2	
+0x9541	0x65F3	
+0x9542	0x65F4	
+0x9543	0x65F5	
+0x9544	0x65F8	
+0x9545	0x65F9	
+0x9546	0x65FB	
+0x9547	0x65FC	
+0x9548	0x65FD	
+0x9549	0x65FE	
+0x954A	0x65FF	
+0x954B	0x6601	
+0x954C	0x6604	
+0x954D	0x6605	
+0x954E	0x6607	
+0x954F	0x6608	
+0x9550	0x6609	
+0x9551	0x660B	
+0x9552	0x660D	
+0x9553	0x6610	
+0x9554	0x6611	
+0x9555	0x6612	
+0x9556	0x6616	
+0x9557	0x6617	
+0x9558	0x6618	
+0x9559	0x661A	
+0x955A	0x661B	
+0x955B	0x661C	
+0x955C	0x661E	
+0x955D	0x6621	
+0x955E	0x6622	
+0x955F	0x6623	
+0x9560	0x6624	
+0x9561	0x6626	
+0x9562	0x6629	
+0x9563	0x662A	
+0x9564	0x662B	
+0x9565	0x662C	
+0x9566	0x662E	
+0x9567	0x6630	
+0x9568	0x6632	
+0x9569	0x6633	
+0x956A	0x6637	
+0x956B	0x6638	
+0x956C	0x6639	
+0x956D	0x663A	
+0x956E	0x663B	
+0x956F	0x663D	
+0x9570	0x663F	
+0x9571	0x6640	
+0x9572	0x6642	
+0x9573	0x6644	
+0x9574	0x6645	
+0x9575	0x6646	
+0x9576	0x6647	
+0x9577	0x6648	
+0x9578	0x6649	
+0x9579	0x664A	
+0x957A	0x664D	
+0x957B	0x664E	
+0x957C	0x6650	
+0x957D	0x6651	
+0x957E	0x6658	
+0x9580	0x6659	
+0x9581	0x665B	
+0x9582	0x665C	
+0x9583	0x665D	
+0x9584	0x665E	
+0x9585	0x6660	
+0x9586	0x6662	
+0x9587	0x6663	
+0x9588	0x6665	
+0x9589	0x6667	
+0x958A	0x6669	
+0x958B	0x666A	
+0x958C	0x666B	
+0x958D	0x666C	
+0x958E	0x666D	
+0x958F	0x6671	
+0x9590	0x6672	
+0x9591	0x6673	
+0x9592	0x6675	
+0x9593	0x6678	
+0x9594	0x6679	
+0x9595	0x667B	
+0x9596	0x667C	
+0x9597	0x667D	
+0x9598	0x667F	
+0x9599	0x6680	
+0x959A	0x6681	
+0x959B	0x6683	
+0x959C	0x6685	
+0x959D	0x6686	
+0x959E	0x6688	
+0x959F	0x6689	
+0x95A0	0x668A	
+0x95A1	0x668B	
+0x95A2	0x668D	
+0x95A3	0x668E	
+0x95A4	0x668F	
+0x95A5	0x6690	
+0x95A6	0x6692	
+0x95A7	0x6693	
+0x95A8	0x6694	
+0x95A9	0x6695	
+0x95AA	0x6698	
+0x95AB	0x6699	
+0x95AC	0x669A	
+0x95AD	0x669B	
+0x95AE	0x669C	
+0x95AF	0x669E	
+0x95B0	0x669F	
+0x95B1	0x66A0	
+0x95B2	0x66A1	
+0x95B3	0x66A2	
+0x95B4	0x66A3	
+0x95B5	0x66A4	
+0x95B6	0x66A5	
+0x95B7	0x66A6	
+0x95B8	0x66A9	
+0x95B9	0x66AA	
+0x95BA	0x66AB	
+0x95BB	0x66AC	
+0x95BC	0x66AD	
+0x95BD	0x66AF	
+0x95BE	0x66B0	
+0x95BF	0x66B1	
+0x95C0	0x66B2	
+0x95C1	0x66B3	
+0x95C2	0x66B5	
+0x95C3	0x66B6	
+0x95C4	0x66B7	
+0x95C5	0x66B8	
+0x95C6	0x66BA	
+0x95C7	0x66BB	
+0x95C8	0x66BC	
+0x95C9	0x66BD	
+0x95CA	0x66BF	
+0x95CB	0x66C0	
+0x95CC	0x66C1	
+0x95CD	0x66C2	
+0x95CE	0x66C3	
+0x95CF	0x66C4	
+0x95D0	0x66C5	
+0x95D1	0x66C6	
+0x95D2	0x66C7	
+0x95D3	0x66C8	
+0x95D4	0x66C9	
+0x95D5	0x66CA	
+0x95D6	0x66CB	
+0x95D7	0x66CC	
+0x95D8	0x66CD	
+0x95D9	0x66CE	
+0x95DA	0x66CF	
+0x95DB	0x66D0	
+0x95DC	0x66D1	
+0x95DD	0x66D2	
+0x95DE	0x66D3	
+0x95DF	0x66D4	
+0x95E0	0x66D5	
+0x95E1	0x66D6	
+0x95E2	0x66D7	
+0x95E3	0x66D8	
+0x95E4	0x66DA	
+0x95E5	0x66DE	
+0x95E6	0x66DF	
+0x95E7	0x66E0	
+0x95E8	0x66E1	
+0x95E9	0x66E2	
+0x95EA	0x66E3	
+0x95EB	0x66E4	
+0x95EC	0x66E5	
+0x95ED	0x66E7	
+0x95EE	0x66E8	
+0x95EF	0x66EA	
+0x95F0	0x66EB	
+0x95F1	0x66EC	
+0x95F2	0x66ED	
+0x95F3	0x66EE	
+0x95F4	0x66EF	
+0x95F5	0x66F1	
+0x95F6	0x66F5	
+0x95F7	0x66F6	
+0x95F8	0x66F8	
+0x95F9	0x66FA	
+0x95FA	0x66FB	
+0x95FB	0x66FD	
+0x95FC	0x6701	
+0x95FD	0x6702	
+0x95FE	0x6703	
+0x9640	0x6704	
+0x9641	0x6705	
+0x9642	0x6706	
+0x9643	0x6707	
+0x9644	0x670C	
+0x9645	0x670E	
+0x9646	0x670F	
+0x9647	0x6711	
+0x9648	0x6712	
+0x9649	0x6713	
+0x964A	0x6716	
+0x964B	0x6718	
+0x964C	0x6719	
+0x964D	0x671A	
+0x964E	0x671C	
+0x964F	0x671E	
+0x9650	0x6720	
+0x9651	0x6721	
+0x9652	0x6722	
+0x9653	0x6723	
+0x9654	0x6724	
+0x9655	0x6725	
+0x9656	0x6727	
+0x9657	0x6729	
+0x9658	0x672E	
+0x9659	0x6730	
+0x965A	0x6732	
+0x965B	0x6733	
+0x965C	0x6736	
+0x965D	0x6737	
+0x965E	0x6738	
+0x965F	0x6739	
+0x9660	0x673B	
+0x9661	0x673C	
+0x9662	0x673E	
+0x9663	0x673F	
+0x9664	0x6741	
+0x9665	0x6744	
+0x9666	0x6745	
+0x9667	0x6747	
+0x9668	0x674A	
+0x9669	0x674B	
+0x966A	0x674D	
+0x966B	0x6752	
+0x966C	0x6754	
+0x966D	0x6755	
+0x966E	0x6757	
+0x966F	0x6758	
+0x9670	0x6759	
+0x9671	0x675A	
+0x9672	0x675B	
+0x9673	0x675D	
+0x9674	0x6762	
+0x9675	0x6763	
+0x9676	0x6764	
+0x9677	0x6766	
+0x9678	0x6767	
+0x9679	0x676B	
+0x967A	0x676C	
+0x967B	0x676E	
+0x967C	0x6771	
+0x967D	0x6774	
+0x967E	0x6776	
+0x9680	0x6778	
+0x9681	0x6779	
+0x9682	0x677A	
+0x9683	0x677B	
+0x9684	0x677D	
+0x9685	0x6780	
+0x9686	0x6782	
+0x9687	0x6783	
+0x9688	0x6785	
+0x9689	0x6786	
+0x968A	0x6788	
+0x968B	0x678A	
+0x968C	0x678C	
+0x968D	0x678D	
+0x968E	0x678E	
+0x968F	0x678F	
+0x9690	0x6791	
+0x9691	0x6792	
+0x9692	0x6793	
+0x9693	0x6794	
+0x9694	0x6796	
+0x9695	0x6799	
+0x9696	0x679B	
+0x9697	0x679F	
+0x9698	0x67A0	
+0x9699	0x67A1	
+0x969A	0x67A4	
+0x969B	0x67A6	
+0x969C	0x67A9	
+0x969D	0x67AC	
+0x969E	0x67AE	
+0x969F	0x67B1	
+0x96A0	0x67B2	
+0x96A1	0x67B4	
+0x96A2	0x67B9	
+0x96A3	0x67BA	
+0x96A4	0x67BB	
+0x96A5	0x67BC	
+0x96A6	0x67BD	
+0x96A7	0x67BE	
+0x96A8	0x67BF	
+0x96A9	0x67C0	
+0x96AA	0x67C2	
+0x96AB	0x67C5	
+0x96AC	0x67C6	
+0x96AD	0x67C7	
+0x96AE	0x67C8	
+0x96AF	0x67C9	
+0x96B0	0x67CA	
+0x96B1	0x67CB	
+0x96B2	0x67CC	
+0x96B3	0x67CD	
+0x96B4	0x67CE	
+0x96B5	0x67D5	
+0x96B6	0x67D6	
+0x96B7	0x67D7	
+0x96B8	0x67DB	
+0x96B9	0x67DF	
+0x96BA	0x67E1	
+0x96BB	0x67E3	
+0x96BC	0x67E4	
+0x96BD	0x67E6	
+0x96BE	0x67E7	
+0x96BF	0x67E8	
+0x96C0	0x67EA	
+0x96C1	0x67EB	
+0x96C2	0x67ED	
+0x96C3	0x67EE	
+0x96C4	0x67F2	
+0x96C5	0x67F5	
+0x96C6	0x67F6	
+0x96C7	0x67F7	
+0x96C8	0x67F8	
+0x96C9	0x67F9	
+0x96CA	0x67FA	
+0x96CB	0x67FB	
+0x96CC	0x67FC	
+0x96CD	0x67FE	
+0x96CE	0x6801	
+0x96CF	0x6802	
+0x96D0	0x6803	
+0x96D1	0x6804	
+0x96D2	0x6806	
+0x96D3	0x680D	
+0x96D4	0x6810	
+0x96D5	0x6812	
+0x96D6	0x6814	
+0x96D7	0x6815	
+0x96D8	0x6818	
+0x96D9	0x6819	
+0x96DA	0x681A	
+0x96DB	0x681B	
+0x96DC	0x681C	
+0x96DD	0x681E	
+0x96DE	0x681F	
+0x96DF	0x6820	
+0x96E0	0x6822	
+0x96E1	0x6823	
+0x96E2	0x6824	
+0x96E3	0x6825	
+0x96E4	0x6826	
+0x96E5	0x6827	
+0x96E6	0x6828	
+0x96E7	0x682B	
+0x96E8	0x682C	
+0x96E9	0x682D	
+0x96EA	0x682E	
+0x96EB	0x682F	
+0x96EC	0x6830	
+0x96ED	0x6831	
+0x96EE	0x6834	
+0x96EF	0x6835	
+0x96F0	0x6836	
+0x96F1	0x683A	
+0x96F2	0x683B	
+0x96F3	0x683F	
+0x96F4	0x6847	
+0x96F5	0x684B	
+0x96F6	0x684D	
+0x96F7	0x684F	
+0x96F8	0x6852	
+0x96F9	0x6856	
+0x96FA	0x6857	
+0x96FB	0x6858	
+0x96FC	0x6859	
+0x96FD	0x685A	
+0x96FE	0x685B	
+0x9740	0x685C	
+0x9741	0x685D	
+0x9742	0x685E	
+0x9743	0x685F	
+0x9744	0x686A	
+0x9745	0x686C	
+0x9746	0x686D	
+0x9747	0x686E	
+0x9748	0x686F	
+0x9749	0x6870	
+0x974A	0x6871	
+0x974B	0x6872	
+0x974C	0x6873	
+0x974D	0x6875	
+0x974E	0x6878	
+0x974F	0x6879	
+0x9750	0x687A	
+0x9751	0x687B	
+0x9752	0x687C	
+0x9753	0x687D	
+0x9754	0x687E	
+0x9755	0x687F	
+0x9756	0x6880	
+0x9757	0x6882	
+0x9758	0x6884	
+0x9759	0x6887	
+0x975A	0x6888	
+0x975B	0x6889	
+0x975C	0x688A	
+0x975D	0x688B	
+0x975E	0x688C	
+0x975F	0x688D	
+0x9760	0x688E	
+0x9761	0x6890	
+0x9762	0x6891	
+0x9763	0x6892	
+0x9764	0x6894	
+0x9765	0x6895	
+0x9766	0x6896	
+0x9767	0x6898	
+0x9768	0x6899	
+0x9769	0x689A	
+0x976A	0x689B	
+0x976B	0x689C	
+0x976C	0x689D	
+0x976D	0x689E	
+0x976E	0x689F	
+0x976F	0x68A0	
+0x9770	0x68A1	
+0x9771	0x68A3	
+0x9772	0x68A4	
+0x9773	0x68A5	
+0x9774	0x68A9	
+0x9775	0x68AA	
+0x9776	0x68AB	
+0x9777	0x68AC	
+0x9778	0x68AE	
+0x9779	0x68B1	
+0x977A	0x68B2	
+0x977B	0x68B4	
+0x977C	0x68B6	
+0x977D	0x68B7	
+0x977E	0x68B8	
+0x9780	0x68B9	
+0x9781	0x68BA	
+0x9782	0x68BB	
+0x9783	0x68BC	
+0x9784	0x68BD	
+0x9785	0x68BE	
+0x9786	0x68BF	
+0x9787	0x68C1	
+0x9788	0x68C3	
+0x9789	0x68C4	
+0x978A	0x68C5	
+0x978B	0x68C6	
+0x978C	0x68C7	
+0x978D	0x68C8	
+0x978E	0x68CA	
+0x978F	0x68CC	
+0x9790	0x68CE	
+0x9791	0x68CF	
+0x9792	0x68D0	
+0x9793	0x68D1	
+0x9794	0x68D3	
+0x9795	0x68D4	
+0x9796	0x68D6	
+0x9797	0x68D7	
+0x9798	0x68D9	
+0x9799	0x68DB	
+0x979A	0x68DC	
+0x979B	0x68DD	
+0x979C	0x68DE	
+0x979D	0x68DF	
+0x979E	0x68E1	
+0x979F	0x68E2	
+0x97A0	0x68E4	
+0x97A1	0x68E5	
+0x97A2	0x68E6	
+0x97A3	0x68E7	
+0x97A4	0x68E8	
+0x97A5	0x68E9	
+0x97A6	0x68EA	
+0x97A7	0x68EB	
+0x97A8	0x68EC	
+0x97A9	0x68ED	
+0x97AA	0x68EF	
+0x97AB	0x68F2	
+0x97AC	0x68F3	
+0x97AD	0x68F4	
+0x97AE	0x68F6	
+0x97AF	0x68F7	
+0x97B0	0x68F8	
+0x97B1	0x68FB	
+0x97B2	0x68FD	
+0x97B3	0x68FE	
+0x97B4	0x68FF	
+0x97B5	0x6900	
+0x97B6	0x6902	
+0x97B7	0x6903	
+0x97B8	0x6904	
+0x97B9	0x6906	
+0x97BA	0x6907	
+0x97BB	0x6908	
+0x97BC	0x6909	
+0x97BD	0x690A	
+0x97BE	0x690C	
+0x97BF	0x690F	
+0x97C0	0x6911	
+0x97C1	0x6913	
+0x97C2	0x6914	
+0x97C3	0x6915	
+0x97C4	0x6916	
+0x97C5	0x6917	
+0x97C6	0x6918	
+0x97C7	0x6919	
+0x97C8	0x691A	
+0x97C9	0x691B	
+0x97CA	0x691C	
+0x97CB	0x691D	
+0x97CC	0x691E	
+0x97CD	0x6921	
+0x97CE	0x6922	
+0x97CF	0x6923	
+0x97D0	0x6925	
+0x97D1	0x6926	
+0x97D2	0x6927	
+0x97D3	0x6928	
+0x97D4	0x6929	
+0x97D5	0x692A	
+0x97D6	0x692B	
+0x97D7	0x692C	
+0x97D8	0x692E	
+0x97D9	0x692F	
+0x97DA	0x6931	
+0x97DB	0x6932	
+0x97DC	0x6933	
+0x97DD	0x6935	
+0x97DE	0x6936	
+0x97DF	0x6937	
+0x97E0	0x6938	
+0x97E1	0x693A	
+0x97E2	0x693B	
+0x97E3	0x693C	
+0x97E4	0x693E	
+0x97E5	0x6940	
+0x97E6	0x6941	
+0x97E7	0x6943	
+0x97E8	0x6944	
+0x97E9	0x6945	
+0x97EA	0x6946	
+0x97EB	0x6947	
+0x97EC	0x6948	
+0x97ED	0x6949	
+0x97EE	0x694A	
+0x97EF	0x694B	
+0x97F0	0x694C	
+0x97F1	0x694D	
+0x97F2	0x694E	
+0x97F3	0x694F	
+0x97F4	0x6950	
+0x97F5	0x6951	
+0x97F6	0x6952	
+0x97F7	0x6953	
+0x97F8	0x6955	
+0x97F9	0x6956	
+0x97FA	0x6958	
+0x97FB	0x6959	
+0x97FC	0x695B	
+0x97FD	0x695C	
+0x97FE	0x695F	
+0x9840	0x6961	
+0x9841	0x6962	
+0x9842	0x6964	
+0x9843	0x6965	
+0x9844	0x6967	
+0x9845	0x6968	
+0x9846	0x6969	
+0x9847	0x696A	
+0x9848	0x696C	
+0x9849	0x696D	
+0x984A	0x696F	
+0x984B	0x6970	
+0x984C	0x6972	
+0x984D	0x6973	
+0x984E	0x6974	
+0x984F	0x6975	
+0x9850	0x6976	
+0x9851	0x697A	
+0x9852	0x697B	
+0x9853	0x697D	
+0x9854	0x697E	
+0x9855	0x697F	
+0x9856	0x6981	
+0x9857	0x6983	
+0x9858	0x6985	
+0x9859	0x698A	
+0x985A	0x698B	
+0x985B	0x698C	
+0x985C	0x698E	
+0x985D	0x698F	
+0x985E	0x6990	
+0x985F	0x6991	
+0x9860	0x6992	
+0x9861	0x6993	
+0x9862	0x6996	
+0x9863	0x6997	
+0x9864	0x6999	
+0x9865	0x699A	
+0x9866	0x699D	
+0x9867	0x699E	
+0x9868	0x699F	
+0x9869	0x69A0	
+0x986A	0x69A1	
+0x986B	0x69A2	
+0x986C	0x69A3	
+0x986D	0x69A4	
+0x986E	0x69A5	
+0x986F	0x69A6	
+0x9870	0x69A9	
+0x9871	0x69AA	
+0x9872	0x69AC	
+0x9873	0x69AE	
+0x9874	0x69AF	
+0x9875	0x69B0	
+0x9876	0x69B2	
+0x9877	0x69B3	
+0x9878	0x69B5	
+0x9879	0x69B6	
+0x987A	0x69B8	
+0x987B	0x69B9	
+0x987C	0x69BA	
+0x987D	0x69BC	
+0x987E	0x69BD	
+0x9880	0x69BE	
+0x9881	0x69BF	
+0x9882	0x69C0	
+0x9883	0x69C2	
+0x9884	0x69C3	
+0x9885	0x69C4	
+0x9886	0x69C5	
+0x9887	0x69C6	
+0x9888	0x69C7	
+0x9889	0x69C8	
+0x988A	0x69C9	
+0x988B	0x69CB	
+0x988C	0x69CD	
+0x988D	0x69CF	
+0x988E	0x69D1	
+0x988F	0x69D2	
+0x9890	0x69D3	
+0x9891	0x69D5	
+0x9892	0x69D6	
+0x9893	0x69D7	
+0x9894	0x69D8	
+0x9895	0x69D9	
+0x9896	0x69DA	
+0x9897	0x69DC	
+0x9898	0x69DD	
+0x9899	0x69DE	
+0x989A	0x69E1	
+0x989B	0x69E2	
+0x989C	0x69E3	
+0x989D	0x69E4	
+0x989E	0x69E5	
+0x989F	0x69E6	
+0x98A0	0x69E7	
+0x98A1	0x69E8	
+0x98A2	0x69E9	
+0x98A3	0x69EA	
+0x98A4	0x69EB	
+0x98A5	0x69EC	
+0x98A6	0x69EE	
+0x98A7	0x69EF	
+0x98A8	0x69F0	
+0x98A9	0x69F1	
+0x98AA	0x69F3	
+0x98AB	0x69F4	
+0x98AC	0x69F5	
+0x98AD	0x69F6	
+0x98AE	0x69F7	
+0x98AF	0x69F8	
+0x98B0	0x69F9	
+0x98B1	0x69FA	
+0x98B2	0x69FB	
+0x98B3	0x69FC	
+0x98B4	0x69FE	
+0x98B5	0x6A00	
+0x98B6	0x6A01	
+0x98B7	0x6A02	
+0x98B8	0x6A03	
+0x98B9	0x6A04	
+0x98BA	0x6A05	
+0x98BB	0x6A06	
+0x98BC	0x6A07	
+0x98BD	0x6A08	
+0x98BE	0x6A09	
+0x98BF	0x6A0B	
+0x98C0	0x6A0C	
+0x98C1	0x6A0D	
+0x98C2	0x6A0E	
+0x98C3	0x6A0F	
+0x98C4	0x6A10	
+0x98C5	0x6A11	
+0x98C6	0x6A12	
+0x98C7	0x6A13	
+0x98C8	0x6A14	
+0x98C9	0x6A15	
+0x98CA	0x6A16	
+0x98CB	0x6A19	
+0x98CC	0x6A1A	
+0x98CD	0x6A1B	
+0x98CE	0x6A1C	
+0x98CF	0x6A1D	
+0x98D0	0x6A1E	
+0x98D1	0x6A20	
+0x98D2	0x6A22	
+0x98D3	0x6A23	
+0x98D4	0x6A24	
+0x98D5	0x6A25	
+0x98D6	0x6A26	
+0x98D7	0x6A27	
+0x98D8	0x6A29	
+0x98D9	0x6A2B	
+0x98DA	0x6A2C	
+0x98DB	0x6A2D	
+0x98DC	0x6A2E	
+0x98DD	0x6A30	
+0x98DE	0x6A32	
+0x98DF	0x6A33	
+0x98E0	0x6A34	
+0x98E1	0x6A36	
+0x98E2	0x6A37	
+0x98E3	0x6A38	
+0x98E4	0x6A39	
+0x98E5	0x6A3A	
+0x98E6	0x6A3B	
+0x98E7	0x6A3C	
+0x98E8	0x6A3F	
+0x98E9	0x6A40	
+0x98EA	0x6A41	
+0x98EB	0x6A42	
+0x98EC	0x6A43	
+0x98ED	0x6A45	
+0x98EE	0x6A46	
+0x98EF	0x6A48	
+0x98F0	0x6A49	
+0x98F1	0x6A4A	
+0x98F2	0x6A4B	
+0x98F3	0x6A4C	
+0x98F4	0x6A4D	
+0x98F5	0x6A4E	
+0x98F6	0x6A4F	
+0x98F7	0x6A51	
+0x98F8	0x6A52	
+0x98F9	0x6A53	
+0x98FA	0x6A54	
+0x98FB	0x6A55	
+0x98FC	0x6A56	
+0x98FD	0x6A57	
+0x98FE	0x6A5A	
+0x9940	0x6A5C	
+0x9941	0x6A5D	
+0x9942	0x6A5E	
+0x9943	0x6A5F	
+0x9944	0x6A60	
+0x9945	0x6A62	
+0x9946	0x6A63	
+0x9947	0x6A64	
+0x9948	0x6A66	
+0x9949	0x6A67	
+0x994A	0x6A68	
+0x994B	0x6A69	
+0x994C	0x6A6A	
+0x994D	0x6A6B	
+0x994E	0x6A6C	
+0x994F	0x6A6D	
+0x9950	0x6A6E	
+0x9951	0x6A6F	
+0x9952	0x6A70	
+0x9953	0x6A72	
+0x9954	0x6A73	
+0x9955	0x6A74	
+0x9956	0x6A75	
+0x9957	0x6A76	
+0x9958	0x6A77	
+0x9959	0x6A78	
+0x995A	0x6A7A	
+0x995B	0x6A7B	
+0x995C	0x6A7D	
+0x995D	0x6A7E	
+0x995E	0x6A7F	
+0x995F	0x6A81	
+0x9960	0x6A82	
+0x9961	0x6A83	
+0x9962	0x6A85	
+0x9963	0x6A86	
+0x9964	0x6A87	
+0x9965	0x6A88	
+0x9966	0x6A89	
+0x9967	0x6A8A	
+0x9968	0x6A8B	
+0x9969	0x6A8C	
+0x996A	0x6A8D	
+0x996B	0x6A8F	
+0x996C	0x6A92	
+0x996D	0x6A93	
+0x996E	0x6A94	
+0x996F	0x6A95	
+0x9970	0x6A96	
+0x9971	0x6A98	
+0x9972	0x6A99	
+0x9973	0x6A9A	
+0x9974	0x6A9B	
+0x9975	0x6A9C	
+0x9976	0x6A9D	
+0x9977	0x6A9E	
+0x9978	0x6A9F	
+0x9979	0x6AA1	
+0x997A	0x6AA2	
+0x997B	0x6AA3	
+0x997C	0x6AA4	
+0x997D	0x6AA5	
+0x997E	0x6AA6	
+0x9980	0x6AA7	
+0x9981	0x6AA8	
+0x9982	0x6AAA	
+0x9983	0x6AAD	
+0x9984	0x6AAE	
+0x9985	0x6AAF	
+0x9986	0x6AB0	
+0x9987	0x6AB1	
+0x9988	0x6AB2	
+0x9989	0x6AB3	
+0x998A	0x6AB4	
+0x998B	0x6AB5	
+0x998C	0x6AB6	
+0x998D	0x6AB7	
+0x998E	0x6AB8	
+0x998F	0x6AB9	
+0x9990	0x6ABA	
+0x9991	0x6ABB	
+0x9992	0x6ABC	
+0x9993	0x6ABD	
+0x9994	0x6ABE	
+0x9995	0x6ABF	
+0x9996	0x6AC0	
+0x9997	0x6AC1	
+0x9998	0x6AC2	
+0x9999	0x6AC3	
+0x999A	0x6AC4	
+0x999B	0x6AC5	
+0x999C	0x6AC6	
+0x999D	0x6AC7	
+0x999E	0x6AC8	
+0x999F	0x6AC9	
+0x99A0	0x6ACA	
+0x99A1	0x6ACB	
+0x99A2	0x6ACC	
+0x99A3	0x6ACD	
+0x99A4	0x6ACE	
+0x99A5	0x6ACF	
+0x99A6	0x6AD0	
+0x99A7	0x6AD1	
+0x99A8	0x6AD2	
+0x99A9	0x6AD3	
+0x99AA	0x6AD4	
+0x99AB	0x6AD5	
+0x99AC	0x6AD6	
+0x99AD	0x6AD7	
+0x99AE	0x6AD8	
+0x99AF	0x6AD9	
+0x99B0	0x6ADA	
+0x99B1	0x6ADB	
+0x99B2	0x6ADC	
+0x99B3	0x6ADD	
+0x99B4	0x6ADE	
+0x99B5	0x6ADF	
+0x99B6	0x6AE0	
+0x99B7	0x6AE1	
+0x99B8	0x6AE2	
+0x99B9	0x6AE3	
+0x99BA	0x6AE4	
+0x99BB	0x6AE5	
+0x99BC	0x6AE6	
+0x99BD	0x6AE7	
+0x99BE	0x6AE8	
+0x99BF	0x6AE9	
+0x99C0	0x6AEA	
+0x99C1	0x6AEB	
+0x99C2	0x6AEC	
+0x99C3	0x6AED	
+0x99C4	0x6AEE	
+0x99C5	0x6AEF	
+0x99C6	0x6AF0	
+0x99C7	0x6AF1	
+0x99C8	0x6AF2	
+0x99C9	0x6AF3	
+0x99CA	0x6AF4	
+0x99CB	0x6AF5	
+0x99CC	0x6AF6	
+0x99CD	0x6AF7	
+0x99CE	0x6AF8	
+0x99CF	0x6AF9	
+0x99D0	0x6AFA	
+0x99D1	0x6AFB	
+0x99D2	0x6AFC	
+0x99D3	0x6AFD	
+0x99D4	0x6AFE	
+0x99D5	0x6AFF	
+0x99D6	0x6B00	
+0x99D7	0x6B01	
+0x99D8	0x6B02	
+0x99D9	0x6B03	
+0x99DA	0x6B04	
+0x99DB	0x6B05	
+0x99DC	0x6B06	
+0x99DD	0x6B07	
+0x99DE	0x6B08	
+0x99DF	0x6B09	
+0x99E0	0x6B0A	
+0x99E1	0x6B0B	
+0x99E2	0x6B0C	
+0x99E3	0x6B0D	
+0x99E4	0x6B0E	
+0x99E5	0x6B0F	
+0x99E6	0x6B10	
+0x99E7	0x6B11	
+0x99E8	0x6B12	
+0x99E9	0x6B13	
+0x99EA	0x6B14	
+0x99EB	0x6B15	
+0x99EC	0x6B16	
+0x99ED	0x6B17	
+0x99EE	0x6B18	
+0x99EF	0x6B19	
+0x99F0	0x6B1A	
+0x99F1	0x6B1B	
+0x99F2	0x6B1C	
+0x99F3	0x6B1D	
+0x99F4	0x6B1E	
+0x99F5	0x6B1F	
+0x99F6	0x6B25	
+0x99F7	0x6B26	
+0x99F8	0x6B28	
+0x99F9	0x6B29	
+0x99FA	0x6B2A	
+0x99FB	0x6B2B	
+0x99FC	0x6B2C	
+0x99FD	0x6B2D	
+0x99FE	0x6B2E	
+0x9A40	0x6B2F	
+0x9A41	0x6B30	
+0x9A42	0x6B31	
+0x9A43	0x6B33	
+0x9A44	0x6B34	
+0x9A45	0x6B35	
+0x9A46	0x6B36	
+0x9A47	0x6B38	
+0x9A48	0x6B3B	
+0x9A49	0x6B3C	
+0x9A4A	0x6B3D	
+0x9A4B	0x6B3F	
+0x9A4C	0x6B40	
+0x9A4D	0x6B41	
+0x9A4E	0x6B42	
+0x9A4F	0x6B44	
+0x9A50	0x6B45	
+0x9A51	0x6B48	
+0x9A52	0x6B4A	
+0x9A53	0x6B4B	
+0x9A54	0x6B4D	
+0x9A55	0x6B4E	
+0x9A56	0x6B4F	
+0x9A57	0x6B50	
+0x9A58	0x6B51	
+0x9A59	0x6B52	
+0x9A5A	0x6B53	
+0x9A5B	0x6B54	
+0x9A5C	0x6B55	
+0x9A5D	0x6B56	
+0x9A5E	0x6B57	
+0x9A5F	0x6B58	
+0x9A60	0x6B5A	
+0x9A61	0x6B5B	
+0x9A62	0x6B5C	
+0x9A63	0x6B5D	
+0x9A64	0x6B5E	
+0x9A65	0x6B5F	
+0x9A66	0x6B60	
+0x9A67	0x6B61	
+0x9A68	0x6B68	
+0x9A69	0x6B69	
+0x9A6A	0x6B6B	
+0x9A6B	0x6B6C	
+0x9A6C	0x6B6D	
+0x9A6D	0x6B6E	
+0x9A6E	0x6B6F	
+0x9A6F	0x6B70	
+0x9A70	0x6B71	
+0x9A71	0x6B72	
+0x9A72	0x6B73	
+0x9A73	0x6B74	
+0x9A74	0x6B75	
+0x9A75	0x6B76	
+0x9A76	0x6B77	
+0x9A77	0x6B78	
+0x9A78	0x6B7A	
+0x9A79	0x6B7D	
+0x9A7A	0x6B7E	
+0x9A7B	0x6B7F	
+0x9A7C	0x6B80	
+0x9A7D	0x6B85	
+0x9A7E	0x6B88	
+0x9A80	0x6B8C	
+0x9A81	0x6B8E	
+0x9A82	0x6B8F	
+0x9A83	0x6B90	
+0x9A84	0x6B91	
+0x9A85	0x6B94	
+0x9A86	0x6B95	
+0x9A87	0x6B97	
+0x9A88	0x6B98	
+0x9A89	0x6B99	
+0x9A8A	0x6B9C	
+0x9A8B	0x6B9D	
+0x9A8C	0x6B9E	
+0x9A8D	0x6B9F	
+0x9A8E	0x6BA0	
+0x9A8F	0x6BA2	
+0x9A90	0x6BA3	
+0x9A91	0x6BA4	
+0x9A92	0x6BA5	
+0x9A93	0x6BA6	
+0x9A94	0x6BA7	
+0x9A95	0x6BA8	
+0x9A96	0x6BA9	
+0x9A97	0x6BAB	
+0x9A98	0x6BAC	
+0x9A99	0x6BAD	
+0x9A9A	0x6BAE	
+0x9A9B	0x6BAF	
+0x9A9C	0x6BB0	
+0x9A9D	0x6BB1	
+0x9A9E	0x6BB2	
+0x9A9F	0x6BB6	
+0x9AA0	0x6BB8	
+0x9AA1	0x6BB9	
+0x9AA2	0x6BBA	
+0x9AA3	0x6BBB	
+0x9AA4	0x6BBC	
+0x9AA5	0x6BBD	
+0x9AA6	0x6BBE	
+0x9AA7	0x6BC0	
+0x9AA8	0x6BC3	
+0x9AA9	0x6BC4	
+0x9AAA	0x6BC6	
+0x9AAB	0x6BC7	
+0x9AAC	0x6BC8	
+0x9AAD	0x6BC9	
+0x9AAE	0x6BCA	
+0x9AAF	0x6BCC	
+0x9AB0	0x6BCE	
+0x9AB1	0x6BD0	
+0x9AB2	0x6BD1	
+0x9AB3	0x6BD8	
+0x9AB4	0x6BDA	
+0x9AB5	0x6BDC	
+0x9AB6	0x6BDD	
+0x9AB7	0x6BDE	
+0x9AB8	0x6BDF	
+0x9AB9	0x6BE0	
+0x9ABA	0x6BE2	
+0x9ABB	0x6BE3	
+0x9ABC	0x6BE4	
+0x9ABD	0x6BE5	
+0x9ABE	0x6BE6	
+0x9ABF	0x6BE7	
+0x9AC0	0x6BE8	
+0x9AC1	0x6BE9	
+0x9AC2	0x6BEC	
+0x9AC3	0x6BED	
+0x9AC4	0x6BEE	
+0x9AC5	0x6BF0	
+0x9AC6	0x6BF1	
+0x9AC7	0x6BF2	
+0x9AC8	0x6BF4	
+0x9AC9	0x6BF6	
+0x9ACA	0x6BF7	
+0x9ACB	0x6BF8	
+0x9ACC	0x6BFA	
+0x9ACD	0x6BFB	
+0x9ACE	0x6BFC	
+0x9ACF	0x6BFE	
+0x9AD0	0x6BFF	
+0x9AD1	0x6C00	
+0x9AD2	0x6C01	
+0x9AD3	0x6C02	
+0x9AD4	0x6C03	
+0x9AD5	0x6C04	
+0x9AD6	0x6C08	
+0x9AD7	0x6C09	
+0x9AD8	0x6C0A	
+0x9AD9	0x6C0B	
+0x9ADA	0x6C0C	
+0x9ADB	0x6C0E	
+0x9ADC	0x6C12	
+0x9ADD	0x6C17	
+0x9ADE	0x6C1C	
+0x9ADF	0x6C1D	
+0x9AE0	0x6C1E	
+0x9AE1	0x6C20	
+0x9AE2	0x6C23	
+0x9AE3	0x6C25	
+0x9AE4	0x6C2B	
+0x9AE5	0x6C2C	
+0x9AE6	0x6C2D	
+0x9AE7	0x6C31	
+0x9AE8	0x6C33	
+0x9AE9	0x6C36	
+0x9AEA	0x6C37	
+0x9AEB	0x6C39	
+0x9AEC	0x6C3A	
+0x9AED	0x6C3B	
+0x9AEE	0x6C3C	
+0x9AEF	0x6C3E	
+0x9AF0	0x6C3F	
+0x9AF1	0x6C43	
+0x9AF2	0x6C44	
+0x9AF3	0x6C45	
+0x9AF4	0x6C48	
+0x9AF5	0x6C4B	
+0x9AF6	0x6C4C	
+0x9AF7	0x6C4D	
+0x9AF8	0x6C4E	
+0x9AF9	0x6C4F	
+0x9AFA	0x6C51	
+0x9AFB	0x6C52	
+0x9AFC	0x6C53	
+0x9AFD	0x6C56	
+0x9AFE	0x6C58	
+0x9B40	0x6C59	
+0x9B41	0x6C5A	
+0x9B42	0x6C62	
+0x9B43	0x6C63	
+0x9B44	0x6C65	
+0x9B45	0x6C66	
+0x9B46	0x6C67	
+0x9B47	0x6C6B	
+0x9B48	0x6C6C	
+0x9B49	0x6C6D	
+0x9B4A	0x6C6E	
+0x9B4B	0x6C6F	
+0x9B4C	0x6C71	
+0x9B4D	0x6C73	
+0x9B4E	0x6C75	
+0x9B4F	0x6C77	
+0x9B50	0x6C78	
+0x9B51	0x6C7A	
+0x9B52	0x6C7B	
+0x9B53	0x6C7C	
+0x9B54	0x6C7F	
+0x9B55	0x6C80	
+0x9B56	0x6C84	
+0x9B57	0x6C87	
+0x9B58	0x6C8A	
+0x9B59	0x6C8B	
+0x9B5A	0x6C8D	
+0x9B5B	0x6C8E	
+0x9B5C	0x6C91	
+0x9B5D	0x6C92	
+0x9B5E	0x6C95	
+0x9B5F	0x6C96	
+0x9B60	0x6C97	
+0x9B61	0x6C98	
+0x9B62	0x6C9A	
+0x9B63	0x6C9C	
+0x9B64	0x6C9D	
+0x9B65	0x6C9E	
+0x9B66	0x6CA0	
+0x9B67	0x6CA2	
+0x9B68	0x6CA8	
+0x9B69	0x6CAC	
+0x9B6A	0x6CAF	
+0x9B6B	0x6CB0	
+0x9B6C	0x6CB4	
+0x9B6D	0x6CB5	
+0x9B6E	0x6CB6	
+0x9B6F	0x6CB7	
+0x9B70	0x6CBA	
+0x9B71	0x6CC0	
+0x9B72	0x6CC1	
+0x9B73	0x6CC2	
+0x9B74	0x6CC3	
+0x9B75	0x6CC6	
+0x9B76	0x6CC7	
+0x9B77	0x6CC8	
+0x9B78	0x6CCB	
+0x9B79	0x6CCD	
+0x9B7A	0x6CCE	
+0x9B7B	0x6CCF	
+0x9B7C	0x6CD1	
+0x9B7D	0x6CD2	
+0x9B7E	0x6CD8	
+0x9B80	0x6CD9	
+0x9B81	0x6CDA	
+0x9B82	0x6CDC	
+0x9B83	0x6CDD	
+0x9B84	0x6CDF	
+0x9B85	0x6CE4	
+0x9B86	0x6CE6	
+0x9B87	0x6CE7	
+0x9B88	0x6CE9	
+0x9B89	0x6CEC	
+0x9B8A	0x6CED	
+0x9B8B	0x6CF2	
+0x9B8C	0x6CF4	
+0x9B8D	0x6CF9	
+0x9B8E	0x6CFF	
+0x9B8F	0x6D00	
+0x9B90	0x6D02	
+0x9B91	0x6D03	
+0x9B92	0x6D05	
+0x9B93	0x6D06	
+0x9B94	0x6D08	
+0x9B95	0x6D09	
+0x9B96	0x6D0A	
+0x9B97	0x6D0D	
+0x9B98	0x6D0F	
+0x9B99	0x6D10	
+0x9B9A	0x6D11	
+0x9B9B	0x6D13	
+0x9B9C	0x6D14	
+0x9B9D	0x6D15	
+0x9B9E	0x6D16	
+0x9B9F	0x6D18	
+0x9BA0	0x6D1C	
+0x9BA1	0x6D1D	
+0x9BA2	0x6D1F	
+0x9BA3	0x6D20	
+0x9BA4	0x6D21	
+0x9BA5	0x6D22	
+0x9BA6	0x6D23	
+0x9BA7	0x6D24	
+0x9BA8	0x6D26	
+0x9BA9	0x6D28	
+0x9BAA	0x6D29	
+0x9BAB	0x6D2C	
+0x9BAC	0x6D2D	
+0x9BAD	0x6D2F	
+0x9BAE	0x6D30	
+0x9BAF	0x6D34	
+0x9BB0	0x6D36	
+0x9BB1	0x6D37	
+0x9BB2	0x6D38	
+0x9BB3	0x6D3A	
+0x9BB4	0x6D3F	
+0x9BB5	0x6D40	
+0x9BB6	0x6D42	
+0x9BB7	0x6D44	
+0x9BB8	0x6D49	
+0x9BB9	0x6D4C	
+0x9BBA	0x6D50	
+0x9BBB	0x6D55	
+0x9BBC	0x6D56	
+0x9BBD	0x6D57	
+0x9BBE	0x6D58	
+0x9BBF	0x6D5B	
+0x9BC0	0x6D5D	
+0x9BC1	0x6D5F	
+0x9BC2	0x6D61	
+0x9BC3	0x6D62	
+0x9BC4	0x6D64	
+0x9BC5	0x6D65	
+0x9BC6	0x6D67	
+0x9BC7	0x6D68	
+0x9BC8	0x6D6B	
+0x9BC9	0x6D6C	
+0x9BCA	0x6D6D	
+0x9BCB	0x6D70	
+0x9BCC	0x6D71	
+0x9BCD	0x6D72	
+0x9BCE	0x6D73	
+0x9BCF	0x6D75	
+0x9BD0	0x6D76	
+0x9BD1	0x6D79	
+0x9BD2	0x6D7A	
+0x9BD3	0x6D7B	
+0x9BD4	0x6D7D	
+0x9BD5	0x6D7E	
+0x9BD6	0x6D7F	
+0x9BD7	0x6D80	
+0x9BD8	0x6D81	
+0x9BD9	0x6D83	
+0x9BDA	0x6D84	
+0x9BDB	0x6D86	
+0x9BDC	0x6D87	
+0x9BDD	0x6D8A	
+0x9BDE	0x6D8B	
+0x9BDF	0x6D8D	
+0x9BE0	0x6D8F	
+0x9BE1	0x6D90	
+0x9BE2	0x6D92	
+0x9BE3	0x6D96	
+0x9BE4	0x6D97	
+0x9BE5	0x6D98	
+0x9BE6	0x6D99	
+0x9BE7	0x6D9A	
+0x9BE8	0x6D9C	
+0x9BE9	0x6DA2	
+0x9BEA	0x6DA5	
+0x9BEB	0x6DAC	
+0x9BEC	0x6DAD	
+0x9BED	0x6DB0	
+0x9BEE	0x6DB1	
+0x9BEF	0x6DB3	
+0x9BF0	0x6DB4	
+0x9BF1	0x6DB6	
+0x9BF2	0x6DB7	
+0x9BF3	0x6DB9	
+0x9BF4	0x6DBA	
+0x9BF5	0x6DBB	
+0x9BF6	0x6DBC	
+0x9BF7	0x6DBD	
+0x9BF8	0x6DBE	
+0x9BF9	0x6DC1	
+0x9BFA	0x6DC2	
+0x9BFB	0x6DC3	
+0x9BFC	0x6DC8	
+0x9BFD	0x6DC9	
+0x9BFE	0x6DCA	
+0x9C40	0x6DCD	
+0x9C41	0x6DCE	
+0x9C42	0x6DCF	
+0x9C43	0x6DD0	
+0x9C44	0x6DD2	
+0x9C45	0x6DD3	
+0x9C46	0x6DD4	
+0x9C47	0x6DD5	
+0x9C48	0x6DD7	
+0x9C49	0x6DDA	
+0x9C4A	0x6DDB	
+0x9C4B	0x6DDC	
+0x9C4C	0x6DDF	
+0x9C4D	0x6DE2	
+0x9C4E	0x6DE3	
+0x9C4F	0x6DE5	
+0x9C50	0x6DE7	
+0x9C51	0x6DE8	
+0x9C52	0x6DE9	
+0x9C53	0x6DEA	
+0x9C54	0x6DED	
+0x9C55	0x6DEF	
+0x9C56	0x6DF0	
+0x9C57	0x6DF2	
+0x9C58	0x6DF4	
+0x9C59	0x6DF5	
+0x9C5A	0x6DF6	
+0x9C5B	0x6DF8	
+0x9C5C	0x6DFA	
+0x9C5D	0x6DFD	
+0x9C5E	0x6DFE	
+0x9C5F	0x6DFF	
+0x9C60	0x6E00	
+0x9C61	0x6E01	
+0x9C62	0x6E02	
+0x9C63	0x6E03	
+0x9C64	0x6E04	
+0x9C65	0x6E06	
+0x9C66	0x6E07	
+0x9C67	0x6E08	
+0x9C68	0x6E09	
+0x9C69	0x6E0B	
+0x9C6A	0x6E0F	
+0x9C6B	0x6E12	
+0x9C6C	0x6E13	
+0x9C6D	0x6E15	
+0x9C6E	0x6E18	
+0x9C6F	0x6E19	
+0x9C70	0x6E1B	
+0x9C71	0x6E1C	
+0x9C72	0x6E1E	
+0x9C73	0x6E1F	
+0x9C74	0x6E22	
+0x9C75	0x6E26	
+0x9C76	0x6E27	
+0x9C77	0x6E28	
+0x9C78	0x6E2A	
+0x9C79	0x6E2C	
+0x9C7A	0x6E2E	
+0x9C7B	0x6E30	
+0x9C7C	0x6E31	
+0x9C7D	0x6E33	
+0x9C7E	0x6E35	
+0x9C80	0x6E36	
+0x9C81	0x6E37	
+0x9C82	0x6E39	
+0x9C83	0x6E3B	
+0x9C84	0x6E3C	
+0x9C85	0x6E3D	
+0x9C86	0x6E3E	
+0x9C87	0x6E3F	
+0x9C88	0x6E40	
+0x9C89	0x6E41	
+0x9C8A	0x6E42	
+0x9C8B	0x6E45	
+0x9C8C	0x6E46	
+0x9C8D	0x6E47	
+0x9C8E	0x6E48	
+0x9C8F	0x6E49	
+0x9C90	0x6E4A	
+0x9C91	0x6E4B	
+0x9C92	0x6E4C	
+0x9C93	0x6E4F	
+0x9C94	0x6E50	
+0x9C95	0x6E51	
+0x9C96	0x6E52	
+0x9C97	0x6E55	
+0x9C98	0x6E57	
+0x9C99	0x6E59	
+0x9C9A	0x6E5A	
+0x9C9B	0x6E5C	
+0x9C9C	0x6E5D	
+0x9C9D	0x6E5E	
+0x9C9E	0x6E60	
+0x9C9F	0x6E61	
+0x9CA0	0x6E62	
+0x9CA1	0x6E63	
+0x9CA2	0x6E64	
+0x9CA3	0x6E65	
+0x9CA4	0x6E66	
+0x9CA5	0x6E67	
+0x9CA6	0x6E68	
+0x9CA7	0x6E69	
+0x9CA8	0x6E6A	
+0x9CA9	0x6E6C	
+0x9CAA	0x6E6D	
+0x9CAB	0x6E6F	
+0x9CAC	0x6E70	
+0x9CAD	0x6E71	
+0x9CAE	0x6E72	
+0x9CAF	0x6E73	
+0x9CB0	0x6E74	
+0x9CB1	0x6E75	
+0x9CB2	0x6E76	
+0x9CB3	0x6E77	
+0x9CB4	0x6E78	
+0x9CB5	0x6E79	
+0x9CB6	0x6E7A	
+0x9CB7	0x6E7B	
+0x9CB8	0x6E7C	
+0x9CB9	0x6E7D	
+0x9CBA	0x6E80	
+0x9CBB	0x6E81	
+0x9CBC	0x6E82	
+0x9CBD	0x6E84	
+0x9CBE	0x6E87	
+0x9CBF	0x6E88	
+0x9CC0	0x6E8A	
+0x9CC1	0x6E8B	
+0x9CC2	0x6E8C	
+0x9CC3	0x6E8D	
+0x9CC4	0x6E8E	
+0x9CC5	0x6E91	
+0x9CC6	0x6E92	
+0x9CC7	0x6E93	
+0x9CC8	0x6E94	
+0x9CC9	0x6E95	
+0x9CCA	0x6E96	
+0x9CCB	0x6E97	
+0x9CCC	0x6E99	
+0x9CCD	0x6E9A	
+0x9CCE	0x6E9B	
+0x9CCF	0x6E9D	
+0x9CD0	0x6E9E	
+0x9CD1	0x6EA0	
+0x9CD2	0x6EA1	
+0x9CD3	0x6EA3	
+0x9CD4	0x6EA4	
+0x9CD5	0x6EA6	
+0x9CD6	0x6EA8	
+0x9CD7	0x6EA9	
+0x9CD8	0x6EAB	
+0x9CD9	0x6EAC	
+0x9CDA	0x6EAD	
+0x9CDB	0x6EAE	
+0x9CDC	0x6EB0	
+0x9CDD	0x6EB3	
+0x9CDE	0x6EB5	
+0x9CDF	0x6EB8	
+0x9CE0	0x6EB9	
+0x9CE1	0x6EBC	
+0x9CE2	0x6EBE	
+0x9CE3	0x6EBF	
+0x9CE4	0x6EC0	
+0x9CE5	0x6EC3	
+0x9CE6	0x6EC4	
+0x9CE7	0x6EC5	
+0x9CE8	0x6EC6	
+0x9CE9	0x6EC8	
+0x9CEA	0x6EC9	
+0x9CEB	0x6ECA	
+0x9CEC	0x6ECC	
+0x9CED	0x6ECD	
+0x9CEE	0x6ECE	
+0x9CEF	0x6ED0	
+0x9CF0	0x6ED2	
+0x9CF1	0x6ED6	
+0x9CF2	0x6ED8	
+0x9CF3	0x6ED9	
+0x9CF4	0x6EDB	
+0x9CF5	0x6EDC	
+0x9CF6	0x6EDD	
+0x9CF7	0x6EE3	
+0x9CF8	0x6EE7	
+0x9CF9	0x6EEA	
+0x9CFA	0x6EEB	
+0x9CFB	0x6EEC	
+0x9CFC	0x6EED	
+0x9CFD	0x6EEE	
+0x9CFE	0x6EEF	
+0x9D40	0x6EF0	
+0x9D41	0x6EF1	
+0x9D42	0x6EF2	
+0x9D43	0x6EF3	
+0x9D44	0x6EF5	
+0x9D45	0x6EF6	
+0x9D46	0x6EF7	
+0x9D47	0x6EF8	
+0x9D48	0x6EFA	
+0x9D49	0x6EFB	
+0x9D4A	0x6EFC	
+0x9D4B	0x6EFD	
+0x9D4C	0x6EFE	
+0x9D4D	0x6EFF	
+0x9D4E	0x6F00	
+0x9D4F	0x6F01	
+0x9D50	0x6F03	
+0x9D51	0x6F04	
+0x9D52	0x6F05	
+0x9D53	0x6F07	
+0x9D54	0x6F08	
+0x9D55	0x6F0A	
+0x9D56	0x6F0B	
+0x9D57	0x6F0C	
+0x9D58	0x6F0D	
+0x9D59	0x6F0E	
+0x9D5A	0x6F10	
+0x9D5B	0x6F11	
+0x9D5C	0x6F12	
+0x9D5D	0x6F16	
+0x9D5E	0x6F17	
+0x9D5F	0x6F18	
+0x9D60	0x6F19	
+0x9D61	0x6F1A	
+0x9D62	0x6F1B	
+0x9D63	0x6F1C	
+0x9D64	0x6F1D	
+0x9D65	0x6F1E	
+0x9D66	0x6F1F	
+0x9D67	0x6F21	
+0x9D68	0x6F22	
+0x9D69	0x6F23	
+0x9D6A	0x6F25	
+0x9D6B	0x6F26	
+0x9D6C	0x6F27	
+0x9D6D	0x6F28	
+0x9D6E	0x6F2C	
+0x9D6F	0x6F2E	
+0x9D70	0x6F30	
+0x9D71	0x6F32	
+0x9D72	0x6F34	
+0x9D73	0x6F35	
+0x9D74	0x6F37	
+0x9D75	0x6F38	
+0x9D76	0x6F39	
+0x9D77	0x6F3A	
+0x9D78	0x6F3B	
+0x9D79	0x6F3C	
+0x9D7A	0x6F3D	
+0x9D7B	0x6F3F	
+0x9D7C	0x6F40	
+0x9D7D	0x6F41	
+0x9D7E	0x6F42	
+0x9D80	0x6F43	
+0x9D81	0x6F44	
+0x9D82	0x6F45	
+0x9D83	0x6F48	
+0x9D84	0x6F49	
+0x9D85	0x6F4A	
+0x9D86	0x6F4C	
+0x9D87	0x6F4E	
+0x9D88	0x6F4F	
+0x9D89	0x6F50	
+0x9D8A	0x6F51	
+0x9D8B	0x6F52	
+0x9D8C	0x6F53	
+0x9D8D	0x6F54	
+0x9D8E	0x6F55	
+0x9D8F	0x6F56	
+0x9D90	0x6F57	
+0x9D91	0x6F59	
+0x9D92	0x6F5A	
+0x9D93	0x6F5B	
+0x9D94	0x6F5D	
+0x9D95	0x6F5F	
+0x9D96	0x6F60	
+0x9D97	0x6F61	
+0x9D98	0x6F63	
+0x9D99	0x6F64	
+0x9D9A	0x6F65	
+0x9D9B	0x6F67	
+0x9D9C	0x6F68	
+0x9D9D	0x6F69	
+0x9D9E	0x6F6A	
+0x9D9F	0x6F6B	
+0x9DA0	0x6F6C	
+0x9DA1	0x6F6F	
+0x9DA2	0x6F70	
+0x9DA3	0x6F71	
+0x9DA4	0x6F73	
+0x9DA5	0x6F75	
+0x9DA6	0x6F76	
+0x9DA7	0x6F77	
+0x9DA8	0x6F79	
+0x9DA9	0x6F7B	
+0x9DAA	0x6F7D	
+0x9DAB	0x6F7E	
+0x9DAC	0x6F7F	
+0x9DAD	0x6F80	
+0x9DAE	0x6F81	
+0x9DAF	0x6F82	
+0x9DB0	0x6F83	
+0x9DB1	0x6F85	
+0x9DB2	0x6F86	
+0x9DB3	0x6F87	
+0x9DB4	0x6F8A	
+0x9DB5	0x6F8B	
+0x9DB6	0x6F8F	
+0x9DB7	0x6F90	
+0x9DB8	0x6F91	
+0x9DB9	0x6F92	
+0x9DBA	0x6F93	
+0x9DBB	0x6F94	
+0x9DBC	0x6F95	
+0x9DBD	0x6F96	
+0x9DBE	0x6F97	
+0x9DBF	0x6F98	
+0x9DC0	0x6F99	
+0x9DC1	0x6F9A	
+0x9DC2	0x6F9B	
+0x9DC3	0x6F9D	
+0x9DC4	0x6F9E	
+0x9DC5	0x6F9F	
+0x9DC6	0x6FA0	
+0x9DC7	0x6FA2	
+0x9DC8	0x6FA3	
+0x9DC9	0x6FA4	
+0x9DCA	0x6FA5	
+0x9DCB	0x6FA6	
+0x9DCC	0x6FA8	
+0x9DCD	0x6FA9	
+0x9DCE	0x6FAA	
+0x9DCF	0x6FAB	
+0x9DD0	0x6FAC	
+0x9DD1	0x6FAD	
+0x9DD2	0x6FAE	
+0x9DD3	0x6FAF	
+0x9DD4	0x6FB0	
+0x9DD5	0x6FB1	
+0x9DD6	0x6FB2	
+0x9DD7	0x6FB4	
+0x9DD8	0x6FB5	
+0x9DD9	0x6FB7	
+0x9DDA	0x6FB8	
+0x9DDB	0x6FBA	
+0x9DDC	0x6FBB	
+0x9DDD	0x6FBC	
+0x9DDE	0x6FBD	
+0x9DDF	0x6FBE	
+0x9DE0	0x6FBF	
+0x9DE1	0x6FC1	
+0x9DE2	0x6FC3	
+0x9DE3	0x6FC4	
+0x9DE4	0x6FC5	
+0x9DE5	0x6FC6	
+0x9DE6	0x6FC7	
+0x9DE7	0x6FC8	
+0x9DE8	0x6FCA	
+0x9DE9	0x6FCB	
+0x9DEA	0x6FCC	
+0x9DEB	0x6FCD	
+0x9DEC	0x6FCE	
+0x9DED	0x6FCF	
+0x9DEE	0x6FD0	
+0x9DEF	0x6FD3	
+0x9DF0	0x6FD4	
+0x9DF1	0x6FD5	
+0x9DF2	0x6FD6	
+0x9DF3	0x6FD7	
+0x9DF4	0x6FD8	
+0x9DF5	0x6FD9	
+0x9DF6	0x6FDA	
+0x9DF7	0x6FDB	
+0x9DF8	0x6FDC	
+0x9DF9	0x6FDD	
+0x9DFA	0x6FDF	
+0x9DFB	0x6FE2	
+0x9DFC	0x6FE3	
+0x9DFD	0x6FE4	
+0x9DFE	0x6FE5	
+0x9E40	0x6FE6	
+0x9E41	0x6FE7	
+0x9E42	0x6FE8	
+0x9E43	0x6FE9	
+0x9E44	0x6FEA	
+0x9E45	0x6FEB	
+0x9E46	0x6FEC	
+0x9E47	0x6FED	
+0x9E48	0x6FF0	
+0x9E49	0x6FF1	
+0x9E4A	0x6FF2	
+0x9E4B	0x6FF3	
+0x9E4C	0x6FF4	
+0x9E4D	0x6FF5	
+0x9E4E	0x6FF6	
+0x9E4F	0x6FF7	
+0x9E50	0x6FF8	
+0x9E51	0x6FF9	
+0x9E52	0x6FFA	
+0x9E53	0x6FFB	
+0x9E54	0x6FFC	
+0x9E55	0x6FFD	
+0x9E56	0x6FFE	
+0x9E57	0x6FFF	
+0x9E58	0x7000	
+0x9E59	0x7001	
+0x9E5A	0x7002	
+0x9E5B	0x7003	
+0x9E5C	0x7004	
+0x9E5D	0x7005	
+0x9E5E	0x7006	
+0x9E5F	0x7007	
+0x9E60	0x7008	
+0x9E61	0x7009	
+0x9E62	0x700A	
+0x9E63	0x700B	
+0x9E64	0x700C	
+0x9E65	0x700D	
+0x9E66	0x700E	
+0x9E67	0x700F	
+0x9E68	0x7010	
+0x9E69	0x7012	
+0x9E6A	0x7013	
+0x9E6B	0x7014	
+0x9E6C	0x7015	
+0x9E6D	0x7016	
+0x9E6E	0x7017	
+0x9E6F	0x7018	
+0x9E70	0x7019	
+0x9E71	0x701C	
+0x9E72	0x701D	
+0x9E73	0x701E	
+0x9E74	0x701F	
+0x9E75	0x7020	
+0x9E76	0x7021	
+0x9E77	0x7022	
+0x9E78	0x7024	
+0x9E79	0x7025	
+0x9E7A	0x7026	
+0x9E7B	0x7027	
+0x9E7C	0x7028	
+0x9E7D	0x7029	
+0x9E7E	0x702A	
+0x9E80	0x702B	
+0x9E81	0x702C	
+0x9E82	0x702D	
+0x9E83	0x702E	
+0x9E84	0x702F	
+0x9E85	0x7030	
+0x9E86	0x7031	
+0x9E87	0x7032	
+0x9E88	0x7033	
+0x9E89	0x7034	
+0x9E8A	0x7036	
+0x9E8B	0x7037	
+0x9E8C	0x7038	
+0x9E8D	0x703A	
+0x9E8E	0x703B	
+0x9E8F	0x703C	
+0x9E90	0x703D	
+0x9E91	0x703E	
+0x9E92	0x703F	
+0x9E93	0x7040	
+0x9E94	0x7041	
+0x9E95	0x7042	
+0x9E96	0x7043	
+0x9E97	0x7044	
+0x9E98	0x7045	
+0x9E99	0x7046	
+0x9E9A	0x7047	
+0x9E9B	0x7048	
+0x9E9C	0x7049	
+0x9E9D	0x704A	
+0x9E9E	0x704B	
+0x9E9F	0x704D	
+0x9EA0	0x704E	
+0x9EA1	0x7050	
+0x9EA2	0x7051	
+0x9EA3	0x7052	
+0x9EA4	0x7053	
+0x9EA5	0x7054	
+0x9EA6	0x7055	
+0x9EA7	0x7056	
+0x9EA8	0x7057	
+0x9EA9	0x7058	
+0x9EAA	0x7059	
+0x9EAB	0x705A	
+0x9EAC	0x705B	
+0x9EAD	0x705C	
+0x9EAE	0x705D	
+0x9EAF	0x705F	
+0x9EB0	0x7060	
+0x9EB1	0x7061	
+0x9EB2	0x7062	
+0x9EB3	0x7063	
+0x9EB4	0x7064	
+0x9EB5	0x7065	
+0x9EB6	0x7066	
+0x9EB7	0x7067	
+0x9EB8	0x7068	
+0x9EB9	0x7069	
+0x9EBA	0x706A	
+0x9EBB	0x706E	
+0x9EBC	0x7071	
+0x9EBD	0x7072	
+0x9EBE	0x7073	
+0x9EBF	0x7074	
+0x9EC0	0x7077	
+0x9EC1	0x7079	
+0x9EC2	0x707A	
+0x9EC3	0x707B	
+0x9EC4	0x707D	
+0x9EC5	0x7081	
+0x9EC6	0x7082	
+0x9EC7	0x7083	
+0x9EC8	0x7084	
+0x9EC9	0x7086	
+0x9ECA	0x7087	
+0x9ECB	0x7088	
+0x9ECC	0x708B	
+0x9ECD	0x708C	
+0x9ECE	0x708D	
+0x9ECF	0x708F	
+0x9ED0	0x7090	
+0x9ED1	0x7091	
+0x9ED2	0x7093	
+0x9ED3	0x7097	
+0x9ED4	0x7098	
+0x9ED5	0x709A	
+0x9ED6	0x709B	
+0x9ED7	0x709E	
+0x9ED8	0x709F	
+0x9ED9	0x70A0	
+0x9EDA	0x70A1	
+0x9EDB	0x70A2	
+0x9EDC	0x70A3	
+0x9EDD	0x70A4	
+0x9EDE	0x70A5	
+0x9EDF	0x70A6	
+0x9EE0	0x70A7	
+0x9EE1	0x70A8	
+0x9EE2	0x70A9	
+0x9EE3	0x70AA	
+0x9EE4	0x70B0	
+0x9EE5	0x70B2	
+0x9EE6	0x70B4	
+0x9EE7	0x70B5	
+0x9EE8	0x70B6	
+0x9EE9	0x70BA	
+0x9EEA	0x70BE	
+0x9EEB	0x70BF	
+0x9EEC	0x70C4	
+0x9EED	0x70C5	
+0x9EEE	0x70C6	
+0x9EEF	0x70C7	
+0x9EF0	0x70C9	
+0x9EF1	0x70CB	
+0x9EF2	0x70CC	
+0x9EF3	0x70CD	
+0x9EF4	0x70CE	
+0x9EF5	0x70CF	
+0x9EF6	0x70D0	
+0x9EF7	0x70D1	
+0x9EF8	0x70D2	
+0x9EF9	0x70D3	
+0x9EFA	0x70D4	
+0x9EFB	0x70D5	
+0x9EFC	0x70D6	
+0x9EFD	0x70D7	
+0x9EFE	0x70DA	
+0x9F40	0x70DC	
+0x9F41	0x70DD	
+0x9F42	0x70DE	
+0x9F43	0x70E0	
+0x9F44	0x70E1	
+0x9F45	0x70E2	
+0x9F46	0x70E3	
+0x9F47	0x70E5	
+0x9F48	0x70EA	
+0x9F49	0x70EE	
+0x9F4A	0x70F0	
+0x9F4B	0x70F1	
+0x9F4C	0x70F2	
+0x9F4D	0x70F3	
+0x9F4E	0x70F4	
+0x9F4F	0x70F5	
+0x9F50	0x70F6	
+0x9F51	0x70F8	
+0x9F52	0x70FA	
+0x9F53	0x70FB	
+0x9F54	0x70FC	
+0x9F55	0x70FE	
+0x9F56	0x70FF	
+0x9F57	0x7100	
+0x9F58	0x7101	
+0x9F59	0x7102	
+0x9F5A	0x7103	
+0x9F5B	0x7104	
+0x9F5C	0x7105	
+0x9F5D	0x7106	
+0x9F5E	0x7107	
+0x9F5F	0x7108	
+0x9F60	0x710B	
+0x9F61	0x710C	
+0x9F62	0x710D	
+0x9F63	0x710E	
+0x9F64	0x710F	
+0x9F65	0x7111	
+0x9F66	0x7112	
+0x9F67	0x7114	
+0x9F68	0x7117	
+0x9F69	0x711B	
+0x9F6A	0x711C	
+0x9F6B	0x711D	
+0x9F6C	0x711E	
+0x9F6D	0x711F	
+0x9F6E	0x7120	
+0x9F6F	0x7121	
+0x9F70	0x7122	
+0x9F71	0x7123	
+0x9F72	0x7124	
+0x9F73	0x7125	
+0x9F74	0x7127	
+0x9F75	0x7128	
+0x9F76	0x7129	
+0x9F77	0x712A	
+0x9F78	0x712B	
+0x9F79	0x712C	
+0x9F7A	0x712D	
+0x9F7B	0x712E	
+0x9F7C	0x7132	
+0x9F7D	0x7133	
+0x9F7E	0x7134	
+0x9F80	0x7135	
+0x9F81	0x7137	
+0x9F82	0x7138	
+0x9F83	0x7139	
+0x9F84	0x713A	
+0x9F85	0x713B	
+0x9F86	0x713C	
+0x9F87	0x713D	
+0x9F88	0x713E	
+0x9F89	0x713F	
+0x9F8A	0x7140	
+0x9F8B	0x7141	
+0x9F8C	0x7142	
+0x9F8D	0x7143	
+0x9F8E	0x7144	
+0x9F8F	0x7146	
+0x9F90	0x7147	
+0x9F91	0x7148	
+0x9F92	0x7149	
+0x9F93	0x714B	
+0x9F94	0x714D	
+0x9F95	0x714F	
+0x9F96	0x7150	
+0x9F97	0x7151	
+0x9F98	0x7152	
+0x9F99	0x7153	
+0x9F9A	0x7154	
+0x9F9B	0x7155	
+0x9F9C	0x7156	
+0x9F9D	0x7157	
+0x9F9E	0x7158	
+0x9F9F	0x7159	
+0x9FA0	0x715A	
+0x9FA1	0x715B	
+0x9FA2	0x715D	
+0x9FA3	0x715F	
+0x9FA4	0x7160	
+0x9FA5	0x7161	
+0x9FA6	0x7162	
+0x9FA7	0x7163	
+0x9FA8	0x7165	
+0x9FA9	0x7169	
+0x9FAA	0x716A	
+0x9FAB	0x716B	
+0x9FAC	0x716C	
+0x9FAD	0x716D	
+0x9FAE	0x716F	
+0x9FAF	0x7170	
+0x9FB0	0x7171	
+0x9FB1	0x7174	
+0x9FB2	0x7175	
+0x9FB3	0x7176	
+0x9FB4	0x7177	
+0x9FB5	0x7179	
+0x9FB6	0x717B	
+0x9FB7	0x717C	
+0x9FB8	0x717E	
+0x9FB9	0x717F	
+0x9FBA	0x7180	
+0x9FBB	0x7181	
+0x9FBC	0x7182	
+0x9FBD	0x7183	
+0x9FBE	0x7185	
+0x9FBF	0x7186	
+0x9FC0	0x7187	
+0x9FC1	0x7188	
+0x9FC2	0x7189	
+0x9FC3	0x718B	
+0x9FC4	0x718C	
+0x9FC5	0x718D	
+0x9FC6	0x718E	
+0x9FC7	0x7190	
+0x9FC8	0x7191	
+0x9FC9	0x7192	
+0x9FCA	0x7193	
+0x9FCB	0x7195	
+0x9FCC	0x7196	
+0x9FCD	0x7197	
+0x9FCE	0x719A	
+0x9FCF	0x719B	
+0x9FD0	0x719C	
+0x9FD1	0x719D	
+0x9FD2	0x719E	
+0x9FD3	0x71A1	
+0x9FD4	0x71A2	
+0x9FD5	0x71A3	
+0x9FD6	0x71A4	
+0x9FD7	0x71A5	
+0x9FD8	0x71A6	
+0x9FD9	0x71A7	
+0x9FDA	0x71A9	
+0x9FDB	0x71AA	
+0x9FDC	0x71AB	
+0x9FDD	0x71AD	
+0x9FDE	0x71AE	
+0x9FDF	0x71AF	
+0x9FE0	0x71B0	
+0x9FE1	0x71B1	
+0x9FE2	0x71B2	
+0x9FE3	0x71B4	
+0x9FE4	0x71B6	
+0x9FE5	0x71B7	
+0x9FE6	0x71B8	
+0x9FE7	0x71BA	
+0x9FE8	0x71BB	
+0x9FE9	0x71BC	
+0x9FEA	0x71BD	
+0x9FEB	0x71BE	
+0x9FEC	0x71BF	
+0x9FED	0x71C0	
+0x9FEE	0x71C1	
+0x9FEF	0x71C2	
+0x9FF0	0x71C4	
+0x9FF1	0x71C5	
+0x9FF2	0x71C6	
+0x9FF3	0x71C7	
+0x9FF4	0x71C8	
+0x9FF5	0x71C9	
+0x9FF6	0x71CA	
+0x9FF7	0x71CB	
+0x9FF8	0x71CC	
+0x9FF9	0x71CD	
+0x9FFA	0x71CF	
+0x9FFB	0x71D0	
+0x9FFC	0x71D1	
+0x9FFD	0x71D2	
+0x9FFE	0x71D3	
+0xA040	0x71D6	
+0xA041	0x71D7	
+0xA042	0x71D8	
+0xA043	0x71D9	
+0xA044	0x71DA	
+0xA045	0x71DB	
+0xA046	0x71DC	
+0xA047	0x71DD	
+0xA048	0x71DE	
+0xA049	0x71DF	
+0xA04A	0x71E1	
+0xA04B	0x71E2	
+0xA04C	0x71E3	
+0xA04D	0x71E4	
+0xA04E	0x71E6	
+0xA04F	0x71E8	
+0xA050	0x71E9	
+0xA051	0x71EA	
+0xA052	0x71EB	
+0xA053	0x71EC	
+0xA054	0x71ED	
+0xA055	0x71EF	
+0xA056	0x71F0	
+0xA057	0x71F1	
+0xA058	0x71F2	
+0xA059	0x71F3	
+0xA05A	0x71F4	
+0xA05B	0x71F5	
+0xA05C	0x71F6	
+0xA05D	0x71F7	
+0xA05E	0x71F8	
+0xA05F	0x71FA	
+0xA060	0x71FB	
+0xA061	0x71FC	
+0xA062	0x71FD	
+0xA063	0x71FE	
+0xA064	0x71FF	
+0xA065	0x7200	
+0xA066	0x7201	
+0xA067	0x7202	
+0xA068	0x7203	
+0xA069	0x7204	
+0xA06A	0x7205	
+0xA06B	0x7207	
+0xA06C	0x7208	
+0xA06D	0x7209	
+0xA06E	0x720A	
+0xA06F	0x720B	
+0xA070	0x720C	
+0xA071	0x720D	
+0xA072	0x720E	
+0xA073	0x720F	
+0xA074	0x7210	
+0xA075	0x7211	
+0xA076	0x7212	
+0xA077	0x7213	
+0xA078	0x7214	
+0xA079	0x7215	
+0xA07A	0x7216	
+0xA07B	0x7217	
+0xA07C	0x7218	
+0xA07D	0x7219	
+0xA07E	0x721A	
+0xA080	0x721B	
+0xA081	0x721C	
+0xA082	0x721E	
+0xA083	0x721F	
+0xA084	0x7220	
+0xA085	0x7221	
+0xA086	0x7222	
+0xA087	0x7223	
+0xA088	0x7224	
+0xA089	0x7225	
+0xA08A	0x7226	
+0xA08B	0x7227	
+0xA08C	0x7229	
+0xA08D	0x722B	
+0xA08E	0x722D	
+0xA08F	0x722E	
+0xA090	0x722F	
+0xA091	0x7232	
+0xA092	0x7233	
+0xA093	0x7234	
+0xA094	0x723A	
+0xA095	0x723C	
+0xA096	0x723E	
+0xA097	0x7240	
+0xA098	0x7241	
+0xA099	0x7242	
+0xA09A	0x7243	
+0xA09B	0x7244	
+0xA09C	0x7245	
+0xA09D	0x7246	
+0xA09E	0x7249	
+0xA09F	0x724A	
+0xA0A0	0x724B	
+0xA0A1	0x724E	
+0xA0A2	0x724F	
+0xA0A3	0x7250	
+0xA0A4	0x7251	
+0xA0A5	0x7253	
+0xA0A6	0x7254	
+0xA0A7	0x7255	
+0xA0A8	0x7257	
+0xA0A9	0x7258	
+0xA0AA	0x725A	
+0xA0AB	0x725C	
+0xA0AC	0x725E	
+0xA0AD	0x7260	
+0xA0AE	0x7263	
+0xA0AF	0x7264	
+0xA0B0	0x7265	
+0xA0B1	0x7268	
+0xA0B2	0x726A	
+0xA0B3	0x726B	
+0xA0B4	0x726C	
+0xA0B5	0x726D	
+0xA0B6	0x7270	
+0xA0B7	0x7271	
+0xA0B8	0x7273	
+0xA0B9	0x7274	
+0xA0BA	0x7276	
+0xA0BB	0x7277	
+0xA0BC	0x7278	
+0xA0BD	0x727B	
+0xA0BE	0x727C	
+0xA0BF	0x727D	
+0xA0C0	0x7282	
+0xA0C1	0x7283	
+0xA0C2	0x7285	
+0xA0C3	0x7286	
+0xA0C4	0x7287	
+0xA0C5	0x7288	
+0xA0C6	0x7289	
+0xA0C7	0x728C	
+0xA0C8	0x728E	
+0xA0C9	0x7290	
+0xA0CA	0x7291	
+0xA0CB	0x7293	
+0xA0CC	0x7294	
+0xA0CD	0x7295	
+0xA0CE	0x7296	
+0xA0CF	0x7297	
+0xA0D0	0x7298	
+0xA0D1	0x7299	
+0xA0D2	0x729A	
+0xA0D3	0x729B	
+0xA0D4	0x729C	
+0xA0D5	0x729D	
+0xA0D6	0x729E	
+0xA0D7	0x72A0	
+0xA0D8	0x72A1	
+0xA0D9	0x72A2	
+0xA0DA	0x72A3	
+0xA0DB	0x72A4	
+0xA0DC	0x72A5	
+0xA0DD	0x72A6	
+0xA0DE	0x72A7	
+0xA0DF	0x72A8	
+0xA0E0	0x72A9	
+0xA0E1	0x72AA	
+0xA0E2	0x72AB	
+0xA0E3	0x72AE	
+0xA0E4	0x72B1	
+0xA0E5	0x72B2	
+0xA0E6	0x72B3	
+0xA0E7	0x72B5	
+0xA0E8	0x72BA	
+0xA0E9	0x72BB	
+0xA0EA	0x72BC	
+0xA0EB	0x72BD	
+0xA0EC	0x72BE	
+0xA0ED	0x72BF	
+0xA0EE	0x72C0	
+0xA0EF	0x72C5	
+0xA0F0	0x72C6	
+0xA0F1	0x72C7	
+0xA0F2	0x72C9	
+0xA0F3	0x72CA	
+0xA0F4	0x72CB	
+0xA0F5	0x72CC	
+0xA0F6	0x72CF	
+0xA0F7	0x72D1	
+0xA0F8	0x72D3	
+0xA0F9	0x72D4	
+0xA0FA	0x72D5	
+0xA0FB	0x72D6	
+0xA0FC	0x72D8	
+0xA0FD	0x72DA	
+0xA0FE	0x72DB	
+# 0xA140	 	
+# 0xA141	 	
+# 0xA142	 	
+# 0xA143	 	
+# 0xA144	 	
+# 0xA145	 	
+# 0xA146	 	
+# 0xA147	 	
+# 0xA148	 	
+# 0xA149	 	
+# 0xA14A	 	
+# 0xA14B	 	
+# 0xA14C	 	
+# 0xA14D	 	
+# 0xA14E	 	
+# 0xA14F	 	
+# 0xA150	 	
+# 0xA151	 	
+# 0xA152	 	
+# 0xA153	 	
+# 0xA154	 	
+# 0xA155	 	
+# 0xA156	 	
+# 0xA157	 	
+# 0xA158	 	
+# 0xA159	 	
+# 0xA15A	 	
+# 0xA15B	 	
+# 0xA15C	 	
+# 0xA15D	 	
+# 0xA15E	 	
+# 0xA15F	 	
+# 0xA160	 	
+# 0xA161	 	
+# 0xA162	 	
+# 0xA163	 	
+# 0xA164	 	
+# 0xA165	 	
+# 0xA166	 	
+# 0xA167	 	
+# 0xA168	 	
+# 0xA169	 	
+# 0xA16A	 	
+# 0xA16B	 	
+# 0xA16C	 	
+# 0xA16D	 	
+# 0xA16E	 	
+# 0xA16F	 	
+# 0xA170	 	
+# 0xA171	 	
+# 0xA172	 	
+# 0xA173	 	
+# 0xA174	 	
+# 0xA175	 	
+# 0xA176	 	
+# 0xA177	 	
+# 0xA178	 	
+# 0xA179	 	
+# 0xA17A	 	
+# 0xA17B	 	
+# 0xA17C	 	
+# 0xA17D	 	
+# 0xA17E	 	
+# 0xA180	 	
+# 0xA181	 	
+# 0xA182	 	
+# 0xA183	 	
+# 0xA184	 	
+# 0xA185	 	
+# 0xA186	 	
+# 0xA187	 	
+# 0xA188	 	
+# 0xA189	 	
+# 0xA18A	 	
+# 0xA18B	 	
+# 0xA18C	 	
+# 0xA18D	 	
+# 0xA18E	 	
+# 0xA18F	 	
+# 0xA190	 	
+# 0xA191	 	
+# 0xA192	 	
+# 0xA193	 	
+# 0xA194	 	
+# 0xA195	 	
+# 0xA196	 	
+# 0xA197	 	
+# 0xA198	 	
+# 0xA199	 	
+# 0xA19A	 	
+# 0xA19B	 	
+# 0xA19C	 	
+# 0xA19D	 	
+# 0xA19E	 	
+# 0xA19F	 	
+# 0xA1A0	 	
+0xA1A1	0x3000	
+0xA1A2	0x3001	
+0xA1A3	0x3002	
+0xA1A4	0x30FB
+0xA1A5	0x02C9	
+0xA1A6	0x02C7	
+0xA1A7	0x00A8	
+0xA1A8	0x3003	
+0xA1A9	0x3005	
+0xA1AA	0x2015	
+0xA1AB	0xFF5E	
+0xA1AC	0x2016	
+0xA1AD	0x2026	
+0xA1AE	0x2018	
+0xA1AF	0x2019	
+0xA1B0	0x201C	
+0xA1B1	0x201D	
+0xA1B2	0x3014	
+0xA1B3	0x3015	
+0xA1B4	0x3008	
+0xA1B5	0x3009	
+0xA1B6	0x300A	
+0xA1B7	0x300B	
+0xA1B8	0x300C	
+0xA1B9	0x300D	
+0xA1BA	0x300E	
+0xA1BB	0x300F	
+0xA1BC	0x3016	
+0xA1BD	0x3017	
+0xA1BE	0x3010	
+0xA1BF	0x3011	
+0xA1C0	0x00B1	
+0xA1C1	0x00D7	
+0xA1C2	0x00F7	
+0xA1C3	0x2236	
+0xA1C4	0x2227	
+0xA1C5	0x2228	
+0xA1C6	0x2211	
+0xA1C7	0x220F	
+0xA1C8	0x222A	
+0xA1C9	0x2229	
+0xA1CA	0x2208	
+0xA1CB	0x2237	
+0xA1CC	0x221A	
+0xA1CD	0x22A5	
+0xA1CE	0x2225	
+0xA1CF	0x2220	
+0xA1D0	0x2312	
+0xA1D1	0x2299	
+0xA1D2	0x222B	
+0xA1D3	0x222E	
+0xA1D4	0x2261	
+0xA1D5	0x224C	
+0xA1D6	0x2248	
+0xA1D7	0x223D	
+0xA1D8	0x221D	
+0xA1D9	0x2260	
+0xA1DA	0x226E	
+0xA1DB	0x226F	
+0xA1DC	0x2264	
+0xA1DD	0x2265	
+0xA1DE	0x221E	
+0xA1DF	0x2235	
+0xA1E0	0x2234	
+0xA1E1	0x2642	
+0xA1E2	0x2640	
+0xA1E3	0x00B0	
+0xA1E4	0x2032	
+0xA1E5	0x2033	
+0xA1E6	0x2103	
+0xA1E7	0xFF04	
+0xA1E8	0x00A4	
+0xA1E9	0xFFE0	
+0xA1EA	0xFFE1	
+0xA1EB	0x2030	
+0xA1EC	0x00A7	
+0xA1ED	0x2116	
+0xA1EE	0x2606	
+0xA1EF	0x2605	
+0xA1F0	0x25CB	
+0xA1F1	0x25CF	
+0xA1F2	0x25CE	
+0xA1F3	0x25C7	
+0xA1F4	0x25C6	
+0xA1F5	0x25A1	
+0xA1F6	0x25A0	
+0xA1F7	0x25B3	
+0xA1F8	0x25B2	
+0xA1F9	0x203B	
+0xA1FA	0x2192	
+0xA1FB	0x2190	
+0xA1FC	0x2191	
+0xA1FD	0x2193	
+0xA1FE	0x3013	
+# 0xA240	 	
+# 0xA241	 	
+# 0xA242	 	
+# 0xA243	 	
+# 0xA244	 	
+# 0xA245	 	
+# 0xA246	 	
+# 0xA247	 	
+# 0xA248	 	
+# 0xA249	 	
+# 0xA24A	 	
+# 0xA24B	 	
+# 0xA24C	 	
+# 0xA24D	 	
+# 0xA24E	 	
+# 0xA24F	 	
+# 0xA250	 	
+# 0xA251	 	
+# 0xA252	 	
+# 0xA253	 	
+# 0xA254	 	
+# 0xA255	 	
+# 0xA256	 	
+# 0xA257	 	
+# 0xA258	 	
+# 0xA259	 	
+# 0xA25A	 	
+# 0xA25B	 	
+# 0xA25C	 	
+# 0xA25D	 	
+# 0xA25E	 	
+# 0xA25F	 	
+# 0xA260	 	
+# 0xA261	 	
+# 0xA262	 	
+# 0xA263	 	
+# 0xA264	 	
+# 0xA265	 	
+# 0xA266	 	
+# 0xA267	 	
+# 0xA268	 	
+# 0xA269	 	
+# 0xA26A	 	
+# 0xA26B	 	
+# 0xA26C	 	
+# 0xA26D	 	
+# 0xA26E	 	
+# 0xA26F	 	
+# 0xA270	 	
+# 0xA271	 	
+# 0xA272	 	
+# 0xA273	 	
+# 0xA274	 	
+# 0xA275	 	
+# 0xA276	 	
+# 0xA277	 	
+# 0xA278	 	
+# 0xA279	 	
+# 0xA27A	 	
+# 0xA27B	 	
+# 0xA27C	 	
+# 0xA27D	 	
+# 0xA27E	 	
+# 0xA280	 	
+# 0xA281	 	
+# 0xA282	 	
+# 0xA283	 	
+# 0xA284	 	
+# 0xA285	 	
+# 0xA286	 	
+# 0xA287	 	
+# 0xA288	 	
+# 0xA289	 	
+# 0xA28A	 	
+# 0xA28B	 	
+# 0xA28C	 	
+# 0xA28D	 	
+# 0xA28E	 	
+# 0xA28F	 	
+# 0xA290	 	
+# 0xA291	 	
+# 0xA292	 	
+# 0xA293	 	
+# 0xA294	 	
+# 0xA295	 	
+# 0xA296	 	
+# 0xA297	 	
+# 0xA298	 	
+# 0xA299	 	
+# 0xA29A	 	
+# 0xA29B	 	
+# 0xA29C	 	
+# 0xA29D	 	
+# 0xA29E	 	
+# 0xA29F	 	
+# 0xA2A0	 	
+0xA2A1	0x2170	
+0xA2A2	0x2171	
+0xA2A3	0x2172	
+0xA2A4	0x2173	
+0xA2A5	0x2174	
+0xA2A6	0x2175	
+0xA2A7	0x2176	
+0xA2A8	0x2177	
+0xA2A9	0x2178	
+0xA2AA	0x2179	
+0xA2AB	0xE586	
+0xA2AC	0xE587	
+0xA2AD	0xE588	
+0xA2AE	0xE589	
+0xA2AF	0xE58A	
+0xA2B0	0xE58B	
+0xA2B1	0x2488	
+0xA2B2	0x2489	
+0xA2B3	0x248A	
+0xA2B4	0x248B	
+0xA2B5	0x248C	
+0xA2B6	0x248D	
+0xA2B7	0x248E	
+0xA2B8	0x248F	
+0xA2B9	0x2490	
+0xA2BA	0x2491	
+0xA2BB	0x2492	
+0xA2BC	0x2493	
+0xA2BD	0x2494	
+0xA2BE	0x2495	
+0xA2BF	0x2496	
+0xA2C0	0x2497	
+0xA2C1	0x2498	
+0xA2C2	0x2499	
+0xA2C3	0x249A	
+0xA2C4	0x249B	
+0xA2C5	0x2474	
+0xA2C6	0x2475	
+0xA2C7	0x2476	
+0xA2C8	0x2477	
+0xA2C9	0x2478	
+0xA2CA	0x2479	
+0xA2CB	0x247A	
+0xA2CC	0x247B	
+0xA2CD	0x247C	
+0xA2CE	0x247D	
+0xA2CF	0x247E	
+0xA2D0	0x247F	
+0xA2D1	0x2480	
+0xA2D2	0x2481	
+0xA2D3	0x2482	
+0xA2D4	0x2483	
+0xA2D5	0x2484	
+0xA2D6	0x2485	
+0xA2D7	0x2486	
+0xA2D8	0x2487	
+0xA2D9	0x2460	
+0xA2DA	0x2461	
+0xA2DB	0x2462	
+0xA2DC	0x2463	
+0xA2DD	0x2464	
+0xA2DE	0x2465	
+0xA2DF	0x2466	
+0xA2E0	0x2467	
+0xA2E1	0x2468	
+0xA2E2	0x2469	
+0xA2E3	0xE58C	
+0xA2E4	0xE58D	
+0xA2E5	0x3220	
+0xA2E6	0x3221	
+0xA2E7	0x3222	
+0xA2E8	0x3223	
+0xA2E9	0x3224	
+0xA2EA	0x3225	
+0xA2EB	0x3226	
+0xA2EC	0x3227	
+0xA2ED	0x3228	
+0xA2EE	0x3229	
+0xA2EF	0xE58E	
+0xA2F0	0xE58F	
+0xA2F1	0x2160	
+0xA2F2	0x2161	
+0xA2F3	0x2162	
+0xA2F4	0x2163	
+0xA2F5	0x2164	
+0xA2F6	0x2165	
+0xA2F7	0x2166	
+0xA2F8	0x2167	
+0xA2F9	0x2168	
+0xA2FA	0x2169	
+0xA2FB	0x216A	
+0xA2FC	0x216B	
+0xA2FD	0xE590	
+0xA2FE	0xE591	
+# 0xA340	 	
+# 0xA341	 	
+# 0xA342	 	
+# 0xA343	 	
+# 0xA344	 	
+# 0xA345	 	
+# 0xA346	 	
+# 0xA347	 	
+# 0xA348	 	
+# 0xA349	 	
+# 0xA34A	 	
+# 0xA34B	 	
+# 0xA34C	 	
+# 0xA34D	 	
+# 0xA34E	 	
+# 0xA34F	 	
+# 0xA350	 	
+# 0xA351	 	
+# 0xA352	 	
+# 0xA353	 	
+# 0xA354	 	
+# 0xA355	 	
+# 0xA356	 	
+# 0xA357	 	
+# 0xA358	 	
+# 0xA359	 	
+# 0xA35A	 	
+# 0xA35B	 	
+# 0xA35C	 	
+# 0xA35D	 	
+# 0xA35E	 	
+# 0xA35F	 	
+# 0xA360	 	
+# 0xA361	 	
+# 0xA362	 	
+# 0xA363	 	
+# 0xA364	 	
+# 0xA365	 	
+# 0xA366	 	
+# 0xA367	 	
+# 0xA368	 	
+# 0xA369	 	
+# 0xA36A	 	
+# 0xA36B	 	
+# 0xA36C	 	
+# 0xA36D	 	
+# 0xA36E	 	
+# 0xA36F	 	
+# 0xA370	 	
+# 0xA371	 	
+# 0xA372	 	
+# 0xA373	 	
+# 0xA374	 	
+# 0xA375	 	
+# 0xA376	 	
+# 0xA377	 	
+# 0xA378	 	
+# 0xA379	 	
+# 0xA37A	 	
+# 0xA37B	 	
+# 0xA37C	 	
+# 0xA37D	 	
+# 0xA37E	 	
+# 0xA380	 	
+# 0xA381	 	
+# 0xA382	 	
+# 0xA383	 	
+# 0xA384	 	
+# 0xA385	 	
+# 0xA386	 	
+# 0xA387	 	
+# 0xA388	 	
+# 0xA389	 	
+# 0xA38A	 	
+# 0xA38B	 	
+# 0xA38C	 	
+# 0xA38D	 	
+# 0xA38E	 	
+# 0xA38F	 	
+# 0xA390	 	
+# 0xA391	 	
+# 0xA392	 	
+# 0xA393	 	
+# 0xA394	 	
+# 0xA395	 	
+# 0xA396	 	
+# 0xA397	 	
+# 0xA398	 	
+# 0xA399	 	
+# 0xA39A	 	
+# 0xA39B	 	
+# 0xA39C	 	
+# 0xA39D	 	
+# 0xA39E	 	
+# 0xA39F	 	
+# 0xA3A0	 	
+0xA3A1	0xFF01	
+0xA3A2	0xFF02	
+0xA3A3	0xFF03	
+0xA3A4	0xFFE5	
+0xA3A5	0xFF05	
+0xA3A6	0xFF06	
+0xA3A7	0xFF07	
+0xA3A8	0xFF08	
+0xA3A9	0xFF09	
+0xA3AA	0xFF0A	
+0xA3AB	0xFF0B	
+0xA3AC	0xFF0C	
+0xA3AD	0xFF0D	
+0xA3AE	0xFF0E	
+0xA3AF	0xFF0F	
+0xA3B0	0xFF10	
+0xA3B1	0xFF11	
+0xA3B2	0xFF12	
+0xA3B3	0xFF13	
+0xA3B4	0xFF14	
+0xA3B5	0xFF15	
+0xA3B6	0xFF16	
+0xA3B7	0xFF17	
+0xA3B8	0xFF18	
+0xA3B9	0xFF19	
+0xA3BA	0xFF1A	
+0xA3BB	0xFF1B	
+0xA3BC	0xFF1C	
+0xA3BD	0xFF1D	
+0xA3BE	0xFF1E	
+0xA3BF	0xFF1F	
+0xA3C0	0xFF20	
+0xA3C1	0xFF21	
+0xA3C2	0xFF22	
+0xA3C3	0xFF23	
+0xA3C4	0xFF24	
+0xA3C5	0xFF25	
+0xA3C6	0xFF26	
+0xA3C7	0xFF27	
+0xA3C8	0xFF28	
+0xA3C9	0xFF29	
+0xA3CA	0xFF2A	
+0xA3CB	0xFF2B	
+0xA3CC	0xFF2C	
+0xA3CD	0xFF2D	
+0xA3CE	0xFF2E	
+0xA3CF	0xFF2F	
+0xA3D0	0xFF30	
+0xA3D1	0xFF31	
+0xA3D2	0xFF32	
+0xA3D3	0xFF33	
+0xA3D4	0xFF34	
+0xA3D5	0xFF35	
+0xA3D6	0xFF36	
+0xA3D7	0xFF37	
+0xA3D8	0xFF38	
+0xA3D9	0xFF39	
+0xA3DA	0xFF3A	
+0xA3DB	0xFF3B	
+0xA3DC	0xFF3C	
+0xA3DD	0xFF3D	
+0xA3DE	0xFF3E	
+0xA3DF	0xFF3F	
+0xA3E0	0xFF40	
+0xA3E1	0xFF41	
+0xA3E2	0xFF42	
+0xA3E3	0xFF43	
+0xA3E4	0xFF44	
+0xA3E5	0xFF45	
+0xA3E6	0xFF46	
+0xA3E7	0xFF47	
+0xA3E8	0xFF48	
+0xA3E9	0xFF49	
+0xA3EA	0xFF4A	
+0xA3EB	0xFF4B	
+0xA3EC	0xFF4C	
+0xA3ED	0xFF4D	
+0xA3EE	0xFF4E	
+0xA3EF	0xFF4F	
+0xA3F0	0xFF50	
+0xA3F1	0xFF51	
+0xA3F2	0xFF52	
+0xA3F3	0xFF53	
+0xA3F4	0xFF54	
+0xA3F5	0xFF55	
+0xA3F6	0xFF56	
+0xA3F7	0xFF57	
+0xA3F8	0xFF58	
+0xA3F9	0xFF59	
+0xA3FA	0xFF5A	
+0xA3FB	0xFF5B	
+0xA3FC	0xFF5C	
+0xA3FD	0xFF5D	
+0xA3FE	0xFFE3	
+# 0xA440	 	
+# 0xA441	 	
+# 0xA442	 	
+# 0xA443	 	
+# 0xA444	 	
+# 0xA445	 	
+# 0xA446	 	
+# 0xA447	 	
+# 0xA448	 	
+# 0xA449	 	
+# 0xA44A	 	
+# 0xA44B	 	
+# 0xA44C	 	
+# 0xA44D	 	
+# 0xA44E	 	
+# 0xA44F	 	
+# 0xA450	 	
+# 0xA451	 	
+# 0xA452	 	
+# 0xA453	 	
+# 0xA454	 	
+# 0xA455	 	
+# 0xA456	 	
+# 0xA457	 	
+# 0xA458	 	
+# 0xA459	 	
+# 0xA45A	 	
+# 0xA45B	 	
+# 0xA45C	 	
+# 0xA45D	 	
+# 0xA45E	 	
+# 0xA45F	 	
+# 0xA460	 	
+# 0xA461	 	
+# 0xA462	 	
+# 0xA463	 	
+# 0xA464	 	
+# 0xA465	 	
+# 0xA466	 	
+# 0xA467	 	
+# 0xA468	 	
+# 0xA469	 	
+# 0xA46A	 	
+# 0xA46B	 	
+# 0xA46C	 	
+# 0xA46D	 	
+# 0xA46E	 	
+# 0xA46F	 	
+# 0xA470	 	
+# 0xA471	 	
+# 0xA472	 	
+# 0xA473	 	
+# 0xA474	 	
+# 0xA475	 	
+# 0xA476	 	
+# 0xA477	 	
+# 0xA478	 	
+# 0xA479	 	
+# 0xA47A	 	
+# 0xA47B	 	
+# 0xA47C	 	
+# 0xA47D	 	
+# 0xA47E	 	
+# 0xA480	 	
+# 0xA481	 	
+# 0xA482	 	
+# 0xA483	 	
+# 0xA484	 	
+# 0xA485	 	
+# 0xA486	 	
+# 0xA487	 	
+# 0xA488	 	
+# 0xA489	 	
+# 0xA48A	 	
+# 0xA48B	 	
+# 0xA48C	 	
+# 0xA48D	 	
+# 0xA48E	 	
+# 0xA48F	 	
+# 0xA490	 	
+# 0xA491	 	
+# 0xA492	 	
+# 0xA493	 	
+# 0xA494	 	
+# 0xA495	 	
+# 0xA496	 	
+# 0xA497	 	
+# 0xA498	 	
+# 0xA499	 	
+# 0xA49A	 	
+# 0xA49B	 	
+# 0xA49C	 	
+# 0xA49D	 	
+# 0xA49E	 	
+# 0xA49F	 	
+# 0xA4A0	 	
+0xA4A1	0x3041	
+0xA4A2	0x3042	
+0xA4A3	0x3043	
+0xA4A4	0x3044	
+0xA4A5	0x3045	
+0xA4A6	0x3046	
+0xA4A7	0x3047	
+0xA4A8	0x3048	
+0xA4A9	0x3049	
+0xA4AA	0x304A	
+0xA4AB	0x304B	
+0xA4AC	0x304C	
+0xA4AD	0x304D	
+0xA4AE	0x304E	
+0xA4AF	0x304F	
+0xA4B0	0x3050	
+0xA4B1	0x3051	
+0xA4B2	0x3052	
+0xA4B3	0x3053	
+0xA4B4	0x3054	
+0xA4B5	0x3055	
+0xA4B6	0x3056	
+0xA4B7	0x3057	
+0xA4B8	0x3058	
+0xA4B9	0x3059	
+0xA4BA	0x305A	
+0xA4BB	0x305B	
+0xA4BC	0x305C	
+0xA4BD	0x305D	
+0xA4BE	0x305E	
+0xA4BF	0x305F	
+0xA4C0	0x3060	
+0xA4C1	0x3061	
+0xA4C2	0x3062	
+0xA4C3	0x3063	
+0xA4C4	0x3064	
+0xA4C5	0x3065	
+0xA4C6	0x3066	
+0xA4C7	0x3067	
+0xA4C8	0x3068	
+0xA4C9	0x3069	
+0xA4CA	0x306A	
+0xA4CB	0x306B	
+0xA4CC	0x306C	
+0xA4CD	0x306D	
+0xA4CE	0x306E	
+0xA4CF	0x306F	
+0xA4D0	0x3070	
+0xA4D1	0x3071	
+0xA4D2	0x3072	
+0xA4D3	0x3073	
+0xA4D4	0x3074	
+0xA4D5	0x3075	
+0xA4D6	0x3076	
+0xA4D7	0x3077	
+0xA4D8	0x3078	
+0xA4D9	0x3079	
+0xA4DA	0x307A	
+0xA4DB	0x307B	
+0xA4DC	0x307C	
+0xA4DD	0x307D	
+0xA4DE	0x307E	
+0xA4DF	0x307F	
+0xA4E0	0x3080	
+0xA4E1	0x3081	
+0xA4E2	0x3082	
+0xA4E3	0x3083	
+0xA4E4	0x3084	
+0xA4E5	0x3085	
+0xA4E6	0x3086	
+0xA4E7	0x3087	
+0xA4E8	0x3088	
+0xA4E9	0x3089	
+0xA4EA	0x308A	
+0xA4EB	0x308B	
+0xA4EC	0x308C	
+0xA4ED	0x308D	
+0xA4EE	0x308E	
+0xA4EF	0x308F	
+0xA4F0	0x3090	
+0xA4F1	0x3091	
+0xA4F2	0x3092
+0xA4F3	0x3093	
+0xA4F4	0xE652	
+0xA4F5	0xE653	
+0xA4F6	0xE654	
+0xA4F7	0xE655	
+0xA4F8	0xE656	
+0xA4F9	0xE657	
+0xA4FA	0xE658	
+0xA4FB	0xE659	
+0xA4FC	0xE65A	
+0xA4FD	0xE65B	
+0xA4FE	0xE65C	
+# 0xA540	 	
+# 0xA541	 	
+# 0xA542	 	
+# 0xA543	 	
+# 0xA544	 	
+# 0xA545	 	
+# 0xA546	 	
+# 0xA547	 	
+# 0xA548	 	
+# 0xA549	 	
+# 0xA54A	 	
+# 0xA54B	 	
+# 0xA54C	 	
+# 0xA54D	 	
+# 0xA54E	 	
+# 0xA54F	 	
+# 0xA550	 	
+# 0xA551	 	
+# 0xA552	 	
+# 0xA553	 	
+# 0xA554	 	
+# 0xA555	 	
+# 0xA556	 	
+# 0xA557	 	
+# 0xA558	 	
+# 0xA559	 	
+# 0xA55A	 	
+# 0xA55B	 	
+# 0xA55C	 	
+# 0xA55D	 	
+# 0xA55E	 	
+# 0xA55F	 	
+# 0xA560	 	
+# 0xA561	 	
+# 0xA562	 	
+# 0xA563	 	
+# 0xA564	 	
+# 0xA565	 	
+# 0xA566	 	
+# 0xA567	 	
+# 0xA568	 	
+# 0xA569	 	
+# 0xA56A	 	
+# 0xA56B	 	
+# 0xA56C	 	
+# 0xA56D	 	
+# 0xA56E	 	
+# 0xA56F	 	
+# 0xA570	 	
+# 0xA571	 	
+# 0xA572	 	
+# 0xA573	 	
+# 0xA574	 	
+# 0xA575	 	
+# 0xA576	 	
+# 0xA577	 	
+# 0xA578	 	
+# 0xA579	 	
+# 0xA57A	 	
+# 0xA57B	 	
+# 0xA57C	 	
+# 0xA57D	 	
+# 0xA57E	 	
+# 0xA580	 	
+# 0xA581	 	
+# 0xA582	 	
+# 0xA583	 	
+# 0xA584	 	
+# 0xA585	 	
+# 0xA586	 	
+# 0xA587	 	
+# 0xA588	 	
+# 0xA589	 	
+# 0xA58A	 	
+# 0xA58B	 	
+# 0xA58C	 	
+# 0xA58D	 	
+# 0xA58E	 	
+# 0xA58F	 	
+# 0xA590	 	
+# 0xA591	 	
+# 0xA592	 	
+# 0xA593	 	
+# 0xA594	 	
+# 0xA595	 	
+# 0xA596	 	
+# 0xA597	 	
+# 0xA598	 	
+# 0xA599	 	
+# 0xA59A	 	
+# 0xA59B	 	
+# 0xA59C	 	
+# 0xA59D	 	
+# 0xA59E	 	
+# 0xA59F	 	
+# 0xA5A0	 	
+0xA5A1	0x30A1	
+0xA5A2	0x30A2	
+0xA5A3	0x30A3	
+0xA5A4	0x30A4	
+0xA5A5	0x30A5	
+0xA5A6	0x30A6	
+0xA5A7	0x30A7	
+0xA5A8	0x30A8	
+0xA5A9	0x30A9	
+0xA5AA	0x30AA	
+0xA5AB	0x30AB	
+0xA5AC	0x30AC	
+0xA5AD	0x30AD	
+0xA5AE	0x30AE	
+0xA5AF	0x30AF	
+0xA5B0	0x30B0	
+0xA5B1	0x30B1	
+0xA5B2	0x30B2	
+0xA5B3	0x30B3	
+0xA5B4	0x30B4	
+0xA5B5	0x30B5	
+0xA5B6	0x30B6	
+0xA5B7	0x30B7	
+0xA5B8	0x30B8	
+0xA5B9	0x30B9	
+0xA5BA	0x30BA	
+0xA5BB	0x30BB	
+0xA5BC	0x30BC	
+0xA5BD	0x30BD	
+0xA5BE	0x30BE	
+0xA5BF	0x30BF	
+0xA5C0	0x30C0	
+0xA5C1	0x30C1	
+0xA5C2	0x30C2	
+0xA5C3	0x30C3	
+0xA5C4	0x30C4	
+0xA5C5	0x30C5	
+0xA5C6	0x30C6	
+0xA5C7	0x30C7	
+0xA5C8	0x30C8	
+0xA5C9	0x30C9	
+0xA5CA	0x30CA	
+0xA5CB	0x30CB	
+0xA5CC	0x30CC	
+0xA5CD	0x30CD	
+0xA5CE	0x30CE	
+0xA5CF	0x30CF	
+0xA5D0	0x30D0	
+0xA5D1	0x30D1	
+0xA5D2	0x30D2	
+0xA5D3	0x30D3	
+0xA5D4	0x30D4	
+0xA5D5	0x30D5	
+0xA5D6	0x30D6	
+0xA5D7	0x30D7	
+0xA5D8	0x30D8	
+0xA5D9	0x30D9	
+0xA5DA	0x30DA	
+0xA5DB	0x30DB	
+0xA5DC	0x30DC	
+0xA5DD	0x30DD	
+0xA5DE	0x30DE	
+0xA5DF	0x30DF	
+0xA5E0	0x30E0	
+0xA5E1	0x30E1	
+0xA5E2	0x30E2	
+0xA5E3	0x30E3	
+0xA5E4	0x30E4	
+0xA5E5	0x30E5	
+0xA5E6	0x30E6	
+0xA5E7	0x30E7	
+0xA5E8	0x30E8	
+0xA5E9	0x30E9	
+0xA5EA	0x30EA	
+0xA5EB	0x30EB	
+0xA5EC	0x30EC	
+0xA5ED	0x30ED	
+0xA5EE	0x30EE	
+0xA5EF	0x30EF	
+0xA5F0	0x30F0	
+0xA5F1	0x30F1	
+0xA5F2	0x30F2	
+0xA5F3	0x30F3	
+0xA5F4	0x30F4	
+0xA5F5	0x30F5	
+0xA5F6	0x30F6	
+0xA5F7	0xE6BD	
+0xA5F8	0xE6BE	
+0xA5F9	0xE6BF	
+0xA5FA	0xE6C0	
+0xA5FB	0xE6C1	
+0xA5FC	0xE6C2	
+0xA5FD	0xE6C3	
+0xA5FE	0xE6C4	
+# 0xA640	 	
+# 0xA641	 	
+# 0xA642	 	
+# 0xA643	 	
+# 0xA644	 	
+# 0xA645	 	
+# 0xA646	 	
+# 0xA647	 	
+# 0xA648	 	
+# 0xA649	 	
+# 0xA64A	 	
+# 0xA64B	 	
+# 0xA64C	 	
+# 0xA64D	 	
+# 0xA64E	 	
+# 0xA64F	 	
+# 0xA650	 	
+# 0xA651	 	
+# 0xA652	 	
+# 0xA653	 	
+# 0xA654	 	
+# 0xA655	 	
+# 0xA656	 	
+# 0xA657	 	
+# 0xA658	 	
+# 0xA659	 	
+# 0xA65A	 	
+# 0xA65B	 	
+# 0xA65C	 	
+# 0xA65D	 	
+# 0xA65E	 	
+# 0xA65F	 	
+# 0xA660	 	
+# 0xA661	 	
+# 0xA662	 	
+# 0xA663	 	
+# 0xA664	 	
+# 0xA665	 	
+# 0xA666	 	
+# 0xA667	 	
+# 0xA668	 	
+# 0xA669	 	
+# 0xA66A	 	
+# 0xA66B	 	
+# 0xA66C	 	
+# 0xA66D	 	
+# 0xA66E	 	
+# 0xA66F	 	
+# 0xA670	 	
+# 0xA671	 	
+# 0xA672	 	
+# 0xA673	 	
+# 0xA674	 	
+# 0xA675	 	
+# 0xA676	 	
+# 0xA677	 	
+# 0xA678	 	
+# 0xA679	 	
+# 0xA67A	 	
+# 0xA67B	 	
+# 0xA67C	 	
+# 0xA67D	 	
+# 0xA67E	 	
+# 0xA680	 	
+# 0xA681	 	
+# 0xA682	 	
+# 0xA683	 	
+# 0xA684	 	
+# 0xA685	 	
+# 0xA686	 	
+# 0xA687	 	
+# 0xA688	 	
+# 0xA689	 	
+# 0xA68A	 	
+# 0xA68B	 	
+# 0xA68C	 	
+# 0xA68D	 	
+# 0xA68E	 	
+# 0xA68F	 	
+# 0xA690	 	
+# 0xA691	 	
+# 0xA692	 	
+# 0xA693	 	
+# 0xA694	 	
+# 0xA695	 	
+# 0xA696	 	
+# 0xA697	 	
+# 0xA698	 	
+# 0xA699	 	
+# 0xA69A	 	
+# 0xA69B	 	
+# 0xA69C	 	
+# 0xA69D	 	
+# 0xA69E	 	
+# 0xA69F	 	
+# 0xA6A0	 	
+0xA6A1	0x0391	
+0xA6A2	0x0392	
+0xA6A3	0x0393	
+0xA6A4	0x0394	
+0xA6A5	0x0395	
+0xA6A6	0x0396	
+0xA6A7	0x0397	
+0xA6A8	0x0398	
+0xA6A9	0x0399	
+0xA6AA	0x039A	
+0xA6AB	0x039B	
+0xA6AC	0x039C	
+0xA6AD	0x039D	
+0xA6AE	0x039E	
+0xA6AF	0x039F	
+0xA6B0	0x03A0	
+0xA6B1	0x03A1	
+0xA6B2	0x03A3	
+0xA6B3	0x03A4	
+0xA6B4	0x03A5	
+0xA6B5	0x03A6	
+0xA6B6	0x03A7	
+0xA6B7	0x03A8	
+0xA6B8	0x03A9	
+0xA6B9	0xE725	
+0xA6BA	0xE726	
+0xA6BB	0xE727	
+0xA6BC	0xE728	
+0xA6BD	0xE729	
+0xA6BE	0xE72A	
+0xA6BF	0xE72B	
+0xA6C0	0xE72C	
+0xA6C1	0x03B1	
+0xA6C2	0x03B2	
+0xA6C3	0x03B3	
+0xA6C4	0x03B4	
+0xA6C5	0x03B5	
+0xA6C6	0x03B6	
+0xA6C7	0x03B7	
+0xA6C8	0x03B8	
+0xA6C9	0x03B9	
+0xA6CA	0x03BA	
+0xA6CB	0x03BB	
+0xA6CC	0x03BC	
+0xA6CD	0x03BD	
+0xA6CE	0x03BE	
+0xA6CF	0x03BF	
+0xA6D0	0x03C0	
+0xA6D1	0x03C1	
+0xA6D2	0x03C3	
+0xA6D3	0x03C4	
+0xA6D4	0x03C5	
+0xA6D5	0x03C6	
+0xA6D6	0x03C7	
+0xA6D7	0x03C8	
+0xA6D8	0x03C9	
+0xA6D9	0xE72D	
+0xA6DA	0xE72E	
+0xA6DB	0xE72F	
+0xA6DC	0xE730	
+0xA6DD	0xE731	
+0xA6DE	0xE732	
+0xA6DF	0xE733	
+0xA6E0	0xFE35	
+0xA6E1	0xFE36	
+0xA6E2	0xFE39	
+0xA6E3	0xFE3A	
+0xA6E4	0xFE3F	
+0xA6E5	0xFE40	
+0xA6E6	0xFE3D	
+0xA6E7	0xFE3E	
+0xA6E8	0xFE41	
+0xA6E9	0xFE42	
+0xA6EA	0xFE43	
+0xA6EB	0xFE44	
+0xA6EC	0xE734	
+0xA6ED	0xE735	
+0xA6EE	0xFE3B	
+0xA6EF	0xFE3C	
+0xA6F0	0xFE37	
+0xA6F1	0xFE38	
+0xA6F2	0xFE31	
+0xA6F3	0xE736	
+0xA6F4	0xFE33	
+0xA6F5	0xFE34	
+0xA6F6	0xE737	
+0xA6F7	0xE738	
+0xA6F8	0xE739	
+0xA6F9	0xE73A	
+0xA6FA	0xE73B	
+0xA6FB	0xE73C	
+0xA6FC	0xE73D	
+0xA6FD	0xE73E	
+0xA6FE	0xE73F	
+# 0xA740	 	
+# 0xA741	 	
+# 0xA742	 	
+# 0xA743	 	
+# 0xA744	 	
+# 0xA745	 	
+# 0xA746	 	
+# 0xA747	 	
+# 0xA748	 	
+# 0xA749	 	
+# 0xA74A	 	
+# 0xA74B	 	
+# 0xA74C	 	
+# 0xA74D	 	
+# 0xA74E	 	
+# 0xA74F	 	
+# 0xA750	 	
+# 0xA751	 	
+# 0xA752	 	
+# 0xA753	 	
+# 0xA754	 	
+# 0xA755	 	
+# 0xA756	 	
+# 0xA757	 	
+# 0xA758	 	
+# 0xA759	 	
+# 0xA75A	 	
+# 0xA75B	 	
+# 0xA75C	 	
+# 0xA75D	 	
+# 0xA75E	 	
+# 0xA75F	 	
+# 0xA760	 	
+# 0xA761	 	
+# 0xA762	 	
+# 0xA763	 	
+# 0xA764	 	
+# 0xA765	 	
+# 0xA766	 	
+# 0xA767	 	
+# 0xA768	 	
+# 0xA769	 	
+# 0xA76A	 	
+# 0xA76B	 	
+# 0xA76C	 	
+# 0xA76D	 	
+# 0xA76E	 	
+# 0xA76F	 	
+# 0xA770	 	
+# 0xA771	 	
+# 0xA772	 	
+# 0xA773	 	
+# 0xA774	 	
+# 0xA775	 	
+# 0xA776	 	
+# 0xA777	 	
+# 0xA778	 	
+# 0xA779	 	
+# 0xA77A	 	
+# 0xA77B	 	
+# 0xA77C	 	
+# 0xA77D	 	
+# 0xA77E	 	
+# 0xA780	 	
+# 0xA781	 	
+# 0xA782	 	
+# 0xA783	 	
+# 0xA784	 	
+# 0xA785	 	
+# 0xA786	 	
+# 0xA787	 	
+# 0xA788	 	
+# 0xA789	 	
+# 0xA78A	 	
+# 0xA78B	 	
+# 0xA78C	 	
+# 0xA78D	 	
+# 0xA78E	 	
+# 0xA78F	 	
+# 0xA790	 	
+# 0xA791	 	
+# 0xA792	 	
+# 0xA793	 	
+# 0xA794	 	
+# 0xA795	 	
+# 0xA796	 	
+# 0xA797	 	
+# 0xA798	 	
+# 0xA799	 	
+# 0xA79A	 	
+# 0xA79B	 	
+# 0xA79C	 	
+# 0xA79D	 	
+# 0xA79E	 	
+# 0xA79F	 	
+# 0xA7A0	 	
+0xA7A1	0x0410	
+0xA7A2	0x0411	
+0xA7A3	0x0412	
+0xA7A4	0x0413	
+0xA7A5	0x0414	
+0xA7A6	0x0415	
+0xA7A7	0x0401	
+0xA7A8	0x0416	
+0xA7A9	0x0417	
+0xA7AA	0x0418	
+0xA7AB	0x0419	
+0xA7AC	0x041A	
+0xA7AD	0x041B	
+0xA7AE	0x041C	
+0xA7AF	0x041D	
+0xA7B0	0x041E	
+0xA7B1	0x041F	
+0xA7B2	0x0420	
+0xA7B3	0x0421	
+0xA7B4	0x0422	
+0xA7B5	0x0423	
+0xA7B6	0x0424	
+0xA7B7	0x0425	
+0xA7B8	0x0426	
+0xA7B9	0x0427	
+0xA7BA	0x0428	
+0xA7BB	0x0429	
+0xA7BC	0x042A	
+0xA7BD	0x042B	
+0xA7BE	0x042C	
+0xA7BF	0x042D	
+0xA7C0	0x042E	
+0xA7C1	0x042F	
+0xA7C2	0xE7A0	
+0xA7C3	0xE7A1	
+0xA7C4	0xE7A2	
+0xA7C5	0xE7A3	
+0xA7C6	0xE7A4	
+0xA7C7	0xE7A5	
+0xA7C8	0xE7A6	
+0xA7C9	0xE7A7	
+0xA7CA	0xE7A8	
+0xA7CB	0xE7A9	
+0xA7CC	0xE7AA	
+0xA7CD	0xE7AB	
+0xA7CE	0xE7AC	
+0xA7CF	0xE7AD	
+0xA7D0	0xE7AE	
+0xA7D1	0x0430	
+0xA7D2	0x0431	
+0xA7D3	0x0432	
+0xA7D4	0x0433	
+0xA7D5	0x0434	
+0xA7D6	0x0435	
+0xA7D7	0x0451	
+0xA7D8	0x0436	
+0xA7D9	0x0437	
+0xA7DA	0x0438	
+0xA7DB	0x0439	
+0xA7DC	0x043A	
+0xA7DD	0x043B	
+0xA7DE	0x043C	
+0xA7DF	0x043D	
+0xA7E0	0x043E	
+0xA7E1	0x043F	
+0xA7E2	0x0440	
+0xA7E3	0x0441	
+0xA7E4	0x0442	
+0xA7E5	0x0443	
+0xA7E6	0x0444	
+0xA7E7	0x0445	
+0xA7E8	0x0446	
+0xA7E9	0x0447	
+0xA7EA	0x0448	
+0xA7EB	0x0449	
+0xA7EC	0x044A	
+0xA7ED	0x044B	
+0xA7EE	0x044C	
+0xA7EF	0x044D	
+0xA7F0	0x044E	
+0xA7F1	0x044F	
+0xA7F2	0xE7AF	
+0xA7F3	0xE7B0	
+0xA7F4	0xE7B1	
+0xA7F5	0xE7B2	
+0xA7F6	0xE7B3	
+0xA7F7	0xE7B4	
+0xA7F8	0xE7B5	
+0xA7F9	0xE7B6	
+0xA7FA	0xE7B7	
+0xA7FB	0xE7B8	
+0xA7FC	0xE7B9	
+0xA7FD	0xE7BA	
+0xA7FE	0xE7BB	
+0xA840	0x02CA	
+0xA841	0x02CB	
+0xA842	0x02D9	
+0xA843	0x2013	
+0xA844	0x2015	
+0xA845	0x2025	
+0xA846	0x2035	
+0xA847	0x2105	
+0xA848	0x2109	
+0xA849	0x2196	
+0xA84A	0x2197	
+0xA84B	0x2198	
+0xA84C	0x2199	
+0xA84D	0x2215	
+0xA84E	0x221F	
+0xA84F	0x2223	
+0xA850	0x2252	
+0xA851	0x2266	
+0xA852	0x2267	
+0xA853	0x22BF	
+0xA854	0x2550	
+0xA855	0x2551	
+0xA856	0x2552	
+0xA857	0x2553	
+0xA858	0x2554	
+0xA859	0x2555	
+0xA85A	0x2556	
+0xA85B	0x2557	
+0xA85C	0x2558	
+0xA85D	0x2559	
+0xA85E	0x255A	
+0xA85F	0x255B	
+0xA860	0x255C	
+0xA861	0x255D	
+0xA862	0x255E	
+0xA863	0x255F	
+0xA864	0x2560	
+0xA865	0x2561	
+0xA866	0x2562	
+0xA867	0x2563	
+0xA868	0x2564	
+0xA869	0x2565	
+0xA86A	0x2566	
+0xA86B	0x2567	
+0xA86C	0x2568	
+0xA86D	0x2569	
+0xA86E	0x256A	
+0xA86F	0x256B	
+0xA870	0x256C	
+0xA871	0x256D	
+0xA872	0x256E	
+0xA873	0x256F	
+0xA874	0x2570	
+0xA875	0x2571	
+0xA876	0x2572	
+0xA877	0x2573	
+0xA878	0x2581	
+0xA879	0x2582	
+0xA87A	0x2583	
+0xA87B	0x2584	
+0xA87C	0x2585	
+0xA87D	0x2586	
+0xA87E	0x2587	
+0xA880	0x2588	
+0xA881	0x2589	
+0xA882	0x258A	
+0xA883	0x258B	
+0xA884	0x258C	
+0xA885	0x258D	
+0xA886	0x258E	
+0xA887	0x258F	
+0xA888	0x2593	
+0xA889	0x2594	
+0xA88A	0x2595	
+0xA88B	0x25BC	
+0xA88C	0x25BD	
+0xA88D	0x25E2	
+0xA88E	0x25E3	
+0xA88F	0x25E4	
+0xA890	0x25E5	
+0xA891	0x2609	
+0xA892	0x2295	
+0xA893	0x3012	
+0xA894	0x301D	
+0xA895	0x301E	
+0xA896	0xE7BC	
+0xA897	0xE7BD	
+0xA898	0xE7BE	
+0xA899	0xE7BF	
+0xA89A	0xE7C0	
+0xA89B	0xE7C1	
+0xA89C	0xE7C2	
+0xA89D	0xE7C3	
+0xA89E	0xE7C4	
+0xA89F	0xE7C5	
+0xA8A0	0xE7C6	
+0xA8A1	0x0101	
+0xA8A2	0x00E1	
+0xA8A3	0x01CE	
+0xA8A4	0x00E0	
+0xA8A5	0x0113	
+0xA8A6	0x00E9	
+0xA8A7	0x011B	
+0xA8A8	0x00E8	
+0xA8A9	0x012B	
+0xA8AA	0x00ED	
+0xA8AB	0x01D0	
+0xA8AC	0x00EC	
+0xA8AD	0x014D	
+0xA8AE	0x00F3	
+0xA8AF	0x01D2	
+0xA8B0	0x00F2	
+0xA8B1	0x016B	
+0xA8B2	0x00FA	
+0xA8B3	0x01D4	
+0xA8B4	0x00F9	
+0xA8B5	0x01D6	
+0xA8B6	0x01D8	
+0xA8B7	0x01DA	
+0xA8B8	0x01DC	
+0xA8B9	0x00FC	
+0xA8BA	0x00EA	
+0xA8BB	0x0251	
+0xA8BC	0xE7C7	
+0xA8BD	0x0144	
+0xA8BE	0x0148	
+0xA8BF	0xE7C8	
+0xA8C0	0x0261	
+0xA8C1	0xE7C9	
+0xA8C2	0xE7CA	
+0xA8C3	0xE7CB	
+0xA8C4	0xE7CC	
+0xA8C5	0x3105	
+0xA8C6	0x3106	
+0xA8C7	0x3107	
+0xA8C8	0x3108	
+0xA8C9	0x3109	
+0xA8CA	0x310A	
+0xA8CB	0x310B	
+0xA8CC	0x310C	
+0xA8CD	0x310D	
+0xA8CE	0x310E	
+0xA8CF	0x310F	
+0xA8D0	0x3110	
+0xA8D1	0x3111	
+0xA8D2	0x3112	
+0xA8D3	0x3113	
+0xA8D4	0x3114	
+0xA8D5	0x3115	
+0xA8D6	0x3116	
+0xA8D7	0x3117	
+0xA8D8	0x3118	
+0xA8D9	0x3119	
+0xA8DA	0x311A	
+0xA8DB	0x311B	
+0xA8DC	0x311C	
+0xA8DD	0x311D	
+0xA8DE	0x311E	
+0xA8DF	0x311F	
+0xA8E0	0x3120	
+0xA8E1	0x3121	
+0xA8E2	0x3122	
+0xA8E3	0x3123	
+0xA8E4	0x3124	
+0xA8E5	0x3125	
+0xA8E6	0x3126	
+0xA8E7	0x3127	
+0xA8E8	0x3128	
+0xA8E9	0x3129	
+0xA8EA	0xE7CD	
+0xA8EB	0xE7CE	
+0xA8EC	0xE7CF	
+0xA8ED	0xE7D0	
+0xA8EE	0xE7D1	
+0xA8EF	0xE7D2	
+0xA8F0	0xE7D3	
+0xA8F1	0xE7D4	
+0xA8F2	0xE7D5	
+0xA8F3	0xE7D6	
+0xA8F4	0xE7D7	
+0xA8F5	0xE7D8	
+0xA8F6	0xE7D9	
+0xA8F7	0xE7DA	
+0xA8F8	0xE7DB	
+0xA8F9	0xE7DC	
+0xA8FA	0xE7DD	
+0xA8FB	0xE7DE	
+0xA8FC	0xE7DF	
+0xA8FD	0xE7E0	
+0xA8FE	0xE7E1	
+0xA940	0x3021	
+0xA941	0x3022	
+0xA942	0x3023	
+0xA943	0x3024	
+0xA944	0x3025	
+0xA945	0x3026	
+0xA946	0x3027	
+0xA947	0x3028	
+0xA948	0x3029	
+0xA949	0x32A3	
+0xA94A	0x338E	
+0xA94B	0x338F	
+0xA94C	0x339C	
+0xA94D	0x339D	
+0xA94E	0x339E	
+0xA94F	0x33A1	
+0xA950	0x33C4	
+0xA951	0x33CE	
+0xA952	0x33D1	
+0xA953	0x33D2	
+0xA954	0x33D5	
+0xA955	0xFE30	
+0xA956	0xFFE2	
+0xA957	0xFFE4	
+0xA958	0xE7E2	
+0xA959	0x2121	
+0xA95A	0x3231	
+0xA95B	0xE7E3	
+0xA95C	0x2010	
+0xA95D	0xE7E4	
+0xA95E	0xE7E5	
+0xA95F	0xE7E6	
+0xA960	0x30FC	
+0xA961	0x309B	
+0xA962	0x309C	
+0xA963	0x30FD	
+0xA964	0x30FE	
+0xA965	0x3006	
+0xA966	0x309D	
+0xA967	0x309E	
+0xA968	0xFE49	
+0xA969	0xFE4A	
+0xA96A	0xFE4B	
+0xA96B	0xFE4C	
+0xA96C	0xFE4D	
+0xA96D	0xFE4E	
+0xA96E	0xFE4F	
+0xA96F	0xFE50	
+0xA970	0xFE51	
+0xA971	0xFE52	
+0xA972	0xFE54	
+0xA973	0xFE55	
+0xA974	0xFE56	
+0xA975	0xFE57	
+0xA976	0xFE59	
+0xA977	0xFE5A	
+0xA978	0xFE5B	
+0xA979	0xFE5C	
+0xA97A	0xFE5D	
+0xA97B	0xFE5E	
+0xA97C	0xFE5F	
+0xA97D	0xFE60	
+0xA97E	0xFE61	
+0xA980	0xFE62	
+0xA981	0xFE63	
+0xA982	0xFE64	
+0xA983	0xFE65	
+0xA984	0xFE66	
+0xA985	0xFE68	
+0xA986	0xFE69	
+0xA987	0xFE6A	
+0xA988	0xFE6B	
+0xA989	0xE7E7	
+0xA98A	0xE7E8	
+0xA98B	0xE7E9	
+0xA98C	0xE7EA	
+0xA98D	0xE7EB	
+0xA98E	0xE7EC	
+0xA98F	0xE7ED	
+0xA990	0xE7EE	
+0xA991	0xE7EF	
+0xA992	0xE7F0	
+0xA993	0xE7F1	
+0xA994	0xE7F2	
+0xA995	0xE7F3	
+0xA996	0x3007	
+0xA997	0xE7F4	
+0xA998	0xE7F5	
+0xA999	0xE7F6	
+0xA99A	0xE7F7	
+0xA99B	0xE7F8	
+0xA99C	0xE7F9	
+0xA99D	0xE7FA	
+0xA99E	0xE7FB	
+0xA99F	0xE7FC	
+0xA9A0	0xE7FD	
+0xA9A1	0xE7FE	
+0xA9A2	0xE7FF	
+0xA9A3	0xE800	
+0xA9A4	0x2500	
+0xA9A5	0x2501	
+0xA9A6	0x2502	
+0xA9A7	0x2503	
+0xA9A8	0x2504	
+0xA9A9	0x2505	
+0xA9AA	0x2506	
+0xA9AB	0x2507	
+0xA9AC	0x2508	
+0xA9AD	0x2509	
+0xA9AE	0x250A	
+0xA9AF	0x250B	
+0xA9B0	0x250C	
+0xA9B1	0x250D	
+0xA9B2	0x250E	
+0xA9B3	0x250F	
+0xA9B4	0x2510	
+0xA9B5	0x2511	
+0xA9B6	0x2512	
+0xA9B7	0x2513	
+0xA9B8	0x2514	
+0xA9B9	0x2515	
+0xA9BA	0x2516	
+0xA9BB	0x2517	
+0xA9BC	0x2518	
+0xA9BD	0x2519	
+0xA9BE	0x251A	
+0xA9BF	0x251B	
+0xA9C0	0x251C	
+0xA9C1	0x251D	
+0xA9C2	0x251E	
+0xA9C3	0x251F	
+0xA9C4	0x2520	
+0xA9C5	0x2521	
+0xA9C6	0x2522	
+0xA9C7	0x2523	
+0xA9C8	0x2524	
+0xA9C9	0x2525	
+0xA9CA	0x2526	
+0xA9CB	0x2527	
+0xA9CC	0x2528	
+0xA9CD	0x2529	
+0xA9CE	0x252A	
+0xA9CF	0x252B	
+0xA9D0	0x252C	
+0xA9D1	0x252D	
+0xA9D2	0x252E	
+0xA9D3	0x252F	
+0xA9D4	0x2530	
+0xA9D5	0x2531	
+0xA9D6	0x2532	
+0xA9D7	0x2533	
+0xA9D8	0x2534	
+0xA9D9	0x2535	
+0xA9DA	0x2536	
+0xA9DB	0x2537	
+0xA9DC	0x2538	
+0xA9DD	0x2539	
+0xA9DE	0x253A	
+0xA9DF	0x253B	
+0xA9E0	0x253C	
+0xA9E1	0x253D	
+0xA9E2	0x253E	
+0xA9E3	0x253F	
+0xA9E4	0x2540	
+0xA9E5	0x2541	
+0xA9E6	0x2542	
+0xA9E7	0x2543	
+0xA9E8	0x2544	
+0xA9E9	0x2545	
+0xA9EA	0x2546	
+0xA9EB	0x2547	
+0xA9EC	0x2548	
+0xA9ED	0x2549	
+0xA9EE	0x254A	
+0xA9EF	0x254B	
+0xA9F0	0xE801	
+0xA9F1	0xE802	
+0xA9F2	0xE803	
+0xA9F3	0xE804	
+0xA9F4	0xE805	
+0xA9F5	0xE806	
+0xA9F6	0xE807	
+0xA9F7	0xE808	
+0xA9F8	0xE809	
+0xA9F9	0xE80A	
+0xA9FA	0xE80B	
+0xA9FB	0xE80C	
+0xA9FC	0xE80D	
+0xA9FD	0xE80E	
+0xA9FE	0xE80F	
+0xAA40	0x72DC	
+0xAA41	0x72DD	
+0xAA42	0x72DF	
+0xAA43	0x72E2	
+0xAA44	0x72E3	
+0xAA45	0x72E4	
+0xAA46	0x72E5	
+0xAA47	0x72E6	
+0xAA48	0x72E7	
+0xAA49	0x72EA	
+0xAA4A	0x72EB	
+0xAA4B	0x72F5	
+0xAA4C	0x72F6	
+0xAA4D	0x72F9	
+0xAA4E	0x72FD	
+0xAA4F	0x72FE	
+0xAA50	0x72FF	
+0xAA51	0x7300	
+0xAA52	0x7302	
+0xAA53	0x7304	
+0xAA54	0x7305	
+0xAA55	0x7306	
+0xAA56	0x7307	
+0xAA57	0x7308	
+0xAA58	0x7309	
+0xAA59	0x730B	
+0xAA5A	0x730C	
+0xAA5B	0x730D	
+0xAA5C	0x730F	
+0xAA5D	0x7310	
+0xAA5E	0x7311	
+0xAA5F	0x7312	
+0xAA60	0x7314	
+0xAA61	0x7318	
+0xAA62	0x7319	
+0xAA63	0x731A	
+0xAA64	0x731F	
+0xAA65	0x7320	
+0xAA66	0x7323	
+0xAA67	0x7324	
+0xAA68	0x7326	
+0xAA69	0x7327	
+0xAA6A	0x7328	
+0xAA6B	0x732D	
+0xAA6C	0x732F	
+0xAA6D	0x7330	
+0xAA6E	0x7332	
+0xAA6F	0x7333	
+0xAA70	0x7335	
+0xAA71	0x7336	
+0xAA72	0x733A	
+0xAA73	0x733B	
+0xAA74	0x733C	
+0xAA75	0x733D	
+0xAA76	0x7340	
+0xAA77	0x7341	
+0xAA78	0x7342	
+0xAA79	0x7343	
+0xAA7A	0x7344	
+0xAA7B	0x7345	
+0xAA7C	0x7346	
+0xAA7D	0x7347	
+0xAA7E	0x7348	
+0xAA80	0x7349	
+0xAA81	0x734A	
+0xAA82	0x734B	
+0xAA83	0x734C	
+0xAA84	0x734E	
+0xAA85	0x734F	
+0xAA86	0x7351	
+0xAA87	0x7353	
+0xAA88	0x7354	
+0xAA89	0x7355	
+0xAA8A	0x7356	
+0xAA8B	0x7358	
+0xAA8C	0x7359	
+0xAA8D	0x735A	
+0xAA8E	0x735B	
+0xAA8F	0x735C	
+0xAA90	0x735D	
+0xAA91	0x735E	
+0xAA92	0x735F	
+0xAA93	0x7361	
+0xAA94	0x7362	
+0xAA95	0x7363	
+0xAA96	0x7364	
+0xAA97	0x7365	
+0xAA98	0x7366	
+0xAA99	0x7367	
+0xAA9A	0x7368	
+0xAA9B	0x7369	
+0xAA9C	0x736A	
+0xAA9D	0x736B	
+0xAA9E	0x736E	
+0xAA9F	0x7370	
+0xAAA0	0x7371	
+# 0xAAA1	 	
+# 0xAAA2	 	
+# 0xAAA3	 	
+# 0xAAA4	 	
+# 0xAAA5	 	
+# 0xAAA6	 	
+# 0xAAA7	 	
+# 0xAAA8	 	
+# 0xAAA9	 	
+# 0xAAAA	 	
+# 0xAAAB	 	
+# 0xAAAC	 	
+# 0xAAAD	 	
+# 0xAAAE	 	
+# 0xAAAF	 	
+# 0xAAB0	 	
+# 0xAAB1	 	
+# 0xAAB2	 	
+# 0xAAB3	 	
+# 0xAAB4	 	
+# 0xAAB5	 	
+# 0xAAB6	 	
+# 0xAAB7	 	
+# 0xAAB8	 	
+# 0xAAB9	 	
+# 0xAABA	 	
+# 0xAABB	 	
+# 0xAABC	 	
+# 0xAABD	 	
+# 0xAABE	 	
+# 0xAABF	 	
+# 0xAAC0	 	
+# 0xAAC1	 	
+# 0xAAC2	 	
+# 0xAAC3	 	
+# 0xAAC4	 	
+# 0xAAC5	 	
+# 0xAAC6	 	
+# 0xAAC7	 	
+# 0xAAC8	 	
+# 0xAAC9	 	
+# 0xAACA	 	
+# 0xAACB	 	
+# 0xAACC	 	
+# 0xAACD	 	
+# 0xAACE	 	
+# 0xAACF	 	
+# 0xAAD0	 	
+# 0xAAD1	 	
+# 0xAAD2	 	
+# 0xAAD3	 	
+# 0xAAD4	 	
+# 0xAAD5	 	
+# 0xAAD6	 	
+# 0xAAD7	 	
+# 0xAAD8	 	
+# 0xAAD9	 	
+# 0xAADA	 	
+# 0xAADB	 	
+# 0xAADC	 	
+# 0xAADD	 	
+# 0xAADE	 	
+# 0xAADF	 	
+# 0xAAE0	 	
+# 0xAAE1	 	
+# 0xAAE2	 	
+# 0xAAE3	 	
+# 0xAAE4	 	
+# 0xAAE5	 	
+# 0xAAE6	 	
+# 0xAAE7	 	
+# 0xAAE8	 	
+# 0xAAE9	 	
+# 0xAAEA	 	
+# 0xAAEB	 	
+# 0xAAEC	 	
+# 0xAAED	 	
+# 0xAAEE	 	
+# 0xAAEF	 	
+# 0xAAF0	 	
+# 0xAAF1	 	
+# 0xAAF2	 	
+# 0xAAF3	 	
+# 0xAAF4	 	
+# 0xAAF5	 	
+# 0xAAF6	 	
+# 0xAAF7	 	
+# 0xAAF8	 	
+# 0xAAF9	 	
+# 0xAAFA	 	
+# 0xAAFB	 	
+# 0xAAFC	 	
+# 0xAAFD	 	
+# 0xAAFE	 	
+0xAB40	0x7372	
+0xAB41	0x7373	
+0xAB42	0x7374	
+0xAB43	0x7375	
+0xAB44	0x7376	
+0xAB45	0x7377	
+0xAB46	0x7378	
+0xAB47	0x7379	
+0xAB48	0x737A	
+0xAB49	0x737B	
+0xAB4A	0x737C	
+0xAB4B	0x737D	
+0xAB4C	0x737F	
+0xAB4D	0x7380	
+0xAB4E	0x7381	
+0xAB4F	0x7382	
+0xAB50	0x7383	
+0xAB51	0x7385	
+0xAB52	0x7386	
+0xAB53	0x7388	
+0xAB54	0x738A	
+0xAB55	0x738C	
+0xAB56	0x738D	
+0xAB57	0x738F	
+0xAB58	0x7390	
+0xAB59	0x7392	
+0xAB5A	0x7393	
+0xAB5B	0x7394	
+0xAB5C	0x7395	
+0xAB5D	0x7397	
+0xAB5E	0x7398	
+0xAB5F	0x7399	
+0xAB60	0x739A	
+0xAB61	0x739C	
+0xAB62	0x739D	
+0xAB63	0x739E	
+0xAB64	0x73A0	
+0xAB65	0x73A1	
+0xAB66	0x73A3	
+0xAB67	0x73A4	
+0xAB68	0x73A5	
+0xAB69	0x73A6	
+0xAB6A	0x73A7	
+0xAB6B	0x73A8	
+0xAB6C	0x73AA	
+0xAB6D	0x73AC	
+0xAB6E	0x73AD	
+0xAB6F	0x73B1	
+0xAB70	0x73B4	
+0xAB71	0x73B5	
+0xAB72	0x73B6	
+0xAB73	0x73B8	
+0xAB74	0x73B9	
+0xAB75	0x73BC	
+0xAB76	0x73BD	
+0xAB77	0x73BE	
+0xAB78	0x73BF	
+0xAB79	0x73C1	
+0xAB7A	0x73C3	
+0xAB7B	0x73C4	
+0xAB7C	0x73C5	
+0xAB7D	0x73C6	
+0xAB7E	0x73C7	
+0xAB80	0x73CB	
+0xAB81	0x73CC	
+0xAB82	0x73CE	
+0xAB83	0x73D2	
+0xAB84	0x73D3	
+0xAB85	0x73D4	
+0xAB86	0x73D5	
+0xAB87	0x73D6	
+0xAB88	0x73D7	
+0xAB89	0x73D8	
+0xAB8A	0x73DA	
+0xAB8B	0x73DB	
+0xAB8C	0x73DC	
+0xAB8D	0x73DD	
+0xAB8E	0x73DF	
+0xAB8F	0x73E1	
+0xAB90	0x73E2	
+0xAB91	0x73E3	
+0xAB92	0x73E4	
+0xAB93	0x73E6	
+0xAB94	0x73E8	
+0xAB95	0x73EA	
+0xAB96	0x73EB	
+0xAB97	0x73EC	
+0xAB98	0x73EE	
+0xAB99	0x73EF	
+0xAB9A	0x73F0	
+0xAB9B	0x73F1	
+0xAB9C	0x73F3	
+0xAB9D	0x73F4	
+0xAB9E	0x73F5	
+0xAB9F	0x73F6	
+0xABA0	0x73F7	
+# 0xABA1	 	
+# 0xABA2	 	
+# 0xABA3	 	
+# 0xABA4	 	
+# 0xABA5	 	
+# 0xABA6	 	
+# 0xABA7	 	
+# 0xABA8	 	
+# 0xABA9	 	
+# 0xABAA	 	
+# 0xABAB	 	
+# 0xABAC	 	
+# 0xABAD	 	
+# 0xABAE	 	
+# 0xABAF	 	
+# 0xABB0	 	
+# 0xABB1	 	
+# 0xABB2	 	
+# 0xABB3	 	
+# 0xABB4	 	
+# 0xABB5	 	
+# 0xABB6	 	
+# 0xABB7	 	
+# 0xABB8	 	
+# 0xABB9	 	
+# 0xABBA	 	
+# 0xABBB	 	
+# 0xABBC	 	
+# 0xABBD	 	
+# 0xABBE	 	
+# 0xABBF	 	
+# 0xABC0	 	
+# 0xABC1	 	
+# 0xABC2	 	
+# 0xABC3	 	
+# 0xABC4	 	
+# 0xABC5	 	
+# 0xABC6	 	
+# 0xABC7	 	
+# 0xABC8	 	
+# 0xABC9	 	
+# 0xABCA	 	
+# 0xABCB	 	
+# 0xABCC	 	
+# 0xABCD	 	
+# 0xABCE	 	
+# 0xABCF	 	
+# 0xABD0	 	
+# 0xABD1	 	
+# 0xABD2	 	
+# 0xABD3	 	
+# 0xABD4	 	
+# 0xABD5	 	
+# 0xABD6	 	
+# 0xABD7	 	
+# 0xABD8	 	
+# 0xABD9	 	
+# 0xABDA	 	
+# 0xABDB	 	
+# 0xABDC	 	
+# 0xABDD	 	
+# 0xABDE	 	
+# 0xABDF	 	
+# 0xABE0	 	
+# 0xABE1	 	
+# 0xABE2	 	
+# 0xABE3	 	
+# 0xABE4	 	
+# 0xABE5	 	
+# 0xABE6	 	
+# 0xABE7	 	
+# 0xABE8	 	
+# 0xABE9	 	
+# 0xABEA	 	
+# 0xABEB	 	
+# 0xABEC	 	
+# 0xABED	 	
+# 0xABEE	 	
+# 0xABEF	 	
+# 0xABF0	 	
+# 0xABF1	 	
+# 0xABF2	 	
+# 0xABF3	 	
+# 0xABF4	 	
+# 0xABF5	 	
+# 0xABF6	 	
+# 0xABF7	 	
+# 0xABF8	 	
+# 0xABF9	 	
+# 0xABFA	 	
+# 0xABFB	 	
+# 0xABFC	 	
+# 0xABFD	 	
+# 0xABFE	 	
+0xAC40	0x73F8	
+0xAC41	0x73F9	
+0xAC42	0x73FA	
+0xAC43	0x73FB	
+0xAC44	0x73FC	
+0xAC45	0x73FD	
+0xAC46	0x73FE	
+0xAC47	0x73FF	
+0xAC48	0x7400	
+0xAC49	0x7401	
+0xAC4A	0x7402	
+0xAC4B	0x7404	
+0xAC4C	0x7407	
+0xAC4D	0x7408	
+0xAC4E	0x740B	
+0xAC4F	0x740C	
+0xAC50	0x740D	
+0xAC51	0x740E	
+0xAC52	0x7411	
+0xAC53	0x7412	
+0xAC54	0x7413	
+0xAC55	0x7414	
+0xAC56	0x7415	
+0xAC57	0x7416	
+0xAC58	0x7417	
+0xAC59	0x7418	
+0xAC5A	0x7419	
+0xAC5B	0x741C	
+0xAC5C	0x741D	
+0xAC5D	0x741E	
+0xAC5E	0x741F	
+0xAC5F	0x7420	
+0xAC60	0x7421	
+0xAC61	0x7423	
+0xAC62	0x7424	
+0xAC63	0x7427	
+0xAC64	0x7429	
+0xAC65	0x742B	
+0xAC66	0x742D	
+0xAC67	0x742F	
+0xAC68	0x7431	
+0xAC69	0x7432	
+0xAC6A	0x7437	
+0xAC6B	0x7438	
+0xAC6C	0x7439	
+0xAC6D	0x743A	
+0xAC6E	0x743B	
+0xAC6F	0x743D	
+0xAC70	0x743E	
+0xAC71	0x743F	
+0xAC72	0x7440	
+0xAC73	0x7442	
+0xAC74	0x7443	
+0xAC75	0x7444	
+0xAC76	0x7445	
+0xAC77	0x7446	
+0xAC78	0x7447	
+0xAC79	0x7448	
+0xAC7A	0x7449	
+0xAC7B	0x744A	
+0xAC7C	0x744B	
+0xAC7D	0x744C	
+0xAC7E	0x744D	
+0xAC80	0x744E	
+0xAC81	0x744F	
+0xAC82	0x7450	
+0xAC83	0x7451	
+0xAC84	0x7452	
+0xAC85	0x7453	
+0xAC86	0x7454	
+0xAC87	0x7456	
+0xAC88	0x7458	
+0xAC89	0x745D	
+0xAC8A	0x7460	
+0xAC8B	0x7461	
+0xAC8C	0x7462	
+0xAC8D	0x7463	
+0xAC8E	0x7464	
+0xAC8F	0x7465	
+0xAC90	0x7466	
+0xAC91	0x7467	
+0xAC92	0x7468	
+0xAC93	0x7469	
+0xAC94	0x746A	
+0xAC95	0x746B	
+0xAC96	0x746C	
+0xAC97	0x746E	
+0xAC98	0x746F	
+0xAC99	0x7471	
+0xAC9A	0x7472	
+0xAC9B	0x7473	
+0xAC9C	0x7474	
+0xAC9D	0x7475	
+0xAC9E	0x7478	
+0xAC9F	0x7479	
+0xACA0	0x747A	
+# 0xACA1	 	
+# 0xACA2	 	
+# 0xACA3	 	
+# 0xACA4	 	
+# 0xACA5	 	
+# 0xACA6	 	
+# 0xACA7	 	
+# 0xACA8	 	
+# 0xACA9	 	
+# 0xACAA	 	
+# 0xACAB	 	
+# 0xACAC	 	
+# 0xACAD	 	
+# 0xACAE	 	
+# 0xACAF	 	
+# 0xACB0	 	
+# 0xACB1	 	
+# 0xACB2	 	
+# 0xACB3	 	
+# 0xACB4	 	
+# 0xACB5	 	
+# 0xACB6	 	
+# 0xACB7	 	
+# 0xACB8	 	
+# 0xACB9	 	
+# 0xACBA	 	
+# 0xACBB	 	
+# 0xACBC	 	
+# 0xACBD	 	
+# 0xACBE	 	
+# 0xACBF	 	
+# 0xACC0	 	
+# 0xACC1	 	
+# 0xACC2	 	
+# 0xACC3	 	
+# 0xACC4	 	
+# 0xACC5	 	
+# 0xACC6	 	
+# 0xACC7	 	
+# 0xACC8	 	
+# 0xACC9	 	
+# 0xACCA	 	
+# 0xACCB	 	
+# 0xACCC	 	
+# 0xACCD	 	
+# 0xACCE	 	
+# 0xACCF	 	
+# 0xACD0	 	
+# 0xACD1	 	
+# 0xACD2	 	
+# 0xACD3	 	
+# 0xACD4	 	
+# 0xACD5	 	
+# 0xACD6	 	
+# 0xACD7	 	
+# 0xACD8	 	
+# 0xACD9	 	
+# 0xACDA	 	
+# 0xACDB	 	
+# 0xACDC	 	
+# 0xACDD	 	
+# 0xACDE	 	
+# 0xACDF	 	
+# 0xACE0	 	
+# 0xACE1	 	
+# 0xACE2	 	
+# 0xACE3	 	
+# 0xACE4	 	
+# 0xACE5	 	
+# 0xACE6	 	
+# 0xACE7	 	
+# 0xACE8	 	
+# 0xACE9	 	
+# 0xACEA	 	
+# 0xACEB	 	
+# 0xACEC	 	
+# 0xACED	 	
+# 0xACEE	 	
+# 0xACEF	 	
+# 0xACF0	 	
+# 0xACF1	 	
+# 0xACF2	 	
+# 0xACF3	 	
+# 0xACF4	 	
+# 0xACF5	 	
+# 0xACF6	 	
+# 0xACF7	 	
+# 0xACF8	 	
+# 0xACF9	 	
+# 0xACFA	 	
+# 0xACFB	 	
+# 0xACFC	 	
+# 0xACFD	 	
+# 0xACFE	 	
+0xAD40	0x747B	
+0xAD41	0x747C	
+0xAD42	0x747D	
+0xAD43	0x747F	
+0xAD44	0x7482	
+0xAD45	0x7484	
+0xAD46	0x7485	
+0xAD47	0x7486	
+0xAD48	0x7488	
+0xAD49	0x7489	
+0xAD4A	0x748A	
+0xAD4B	0x748C	
+0xAD4C	0x748D	
+0xAD4D	0x748F	
+0xAD4E	0x7491	
+0xAD4F	0x7492	
+0xAD50	0x7493	
+0xAD51	0x7494	
+0xAD52	0x7495	
+0xAD53	0x7496	
+0xAD54	0x7497	
+0xAD55	0x7498	
+0xAD56	0x7499	
+0xAD57	0x749A	
+0xAD58	0x749B	
+0xAD59	0x749D	
+0xAD5A	0x749F	
+0xAD5B	0x74A0	
+0xAD5C	0x74A1	
+0xAD5D	0x74A2	
+0xAD5E	0x74A3	
+0xAD5F	0x74A4	
+0xAD60	0x74A5	
+0xAD61	0x74A6	
+0xAD62	0x74AA	
+0xAD63	0x74AB	
+0xAD64	0x74AC	
+0xAD65	0x74AD	
+0xAD66	0x74AE	
+0xAD67	0x74AF	
+0xAD68	0x74B0	
+0xAD69	0x74B1	
+0xAD6A	0x74B2	
+0xAD6B	0x74B3	
+0xAD6C	0x74B4	
+0xAD6D	0x74B5	
+0xAD6E	0x74B6	
+0xAD6F	0x74B7	
+0xAD70	0x74B8	
+0xAD71	0x74B9	
+0xAD72	0x74BB	
+0xAD73	0x74BC	
+0xAD74	0x74BD	
+0xAD75	0x74BE	
+0xAD76	0x74BF	
+0xAD77	0x74C0	
+0xAD78	0x74C1	
+0xAD79	0x74C2	
+0xAD7A	0x74C3	
+0xAD7B	0x74C4	
+0xAD7C	0x74C5	
+0xAD7D	0x74C6	
+0xAD7E	0x74C7	
+0xAD80	0x74C8	
+0xAD81	0x74C9	
+0xAD82	0x74CA	
+0xAD83	0x74CB	
+0xAD84	0x74CC	
+0xAD85	0x74CD	
+0xAD86	0x74CE	
+0xAD87	0x74CF	
+0xAD88	0x74D0	
+0xAD89	0x74D1	
+0xAD8A	0x74D3	
+0xAD8B	0x74D4	
+0xAD8C	0x74D5	
+0xAD8D	0x74D6	
+0xAD8E	0x74D7	
+0xAD8F	0x74D8	
+0xAD90	0x74D9	
+0xAD91	0x74DA	
+0xAD92	0x74DB	
+0xAD93	0x74DD	
+0xAD94	0x74DF	
+0xAD95	0x74E1	
+0xAD96	0x74E5	
+0xAD97	0x74E7	
+0xAD98	0x74E8	
+0xAD99	0x74E9	
+0xAD9A	0x74EA	
+0xAD9B	0x74EB	
+0xAD9C	0x74EC	
+0xAD9D	0x74ED	
+0xAD9E	0x74F0	
+0xAD9F	0x74F1	
+0xADA0	0x74F2	
+# 0xADA1	 	
+# 0xADA2	 	
+# 0xADA3	 	
+# 0xADA4	 	
+# 0xADA5	 	
+# 0xADA6	 	
+# 0xADA7	 	
+# 0xADA8	 	
+# 0xADA9	 	
+# 0xADAA	 	
+# 0xADAB	 	
+# 0xADAC	 	
+# 0xADAD	 	
+# 0xADAE	 	
+# 0xADAF	 	
+# 0xADB0	 	
+# 0xADB1	 	
+# 0xADB2	 	
+# 0xADB3	 	
+# 0xADB4	 	
+# 0xADB5	 	
+# 0xADB6	 	
+# 0xADB7	 	
+# 0xADB8	 	
+# 0xADB9	 	
+# 0xADBA	 	
+# 0xADBB	 	
+# 0xADBC	 	
+# 0xADBD	 	
+# 0xADBE	 	
+# 0xADBF	 	
+# 0xADC0	 	
+# 0xADC1	 	
+# 0xADC2	 	
+# 0xADC3	 	
+# 0xADC4	 	
+# 0xADC5	 	
+# 0xADC6	 	
+# 0xADC7	 	
+# 0xADC8	 	
+# 0xADC9	 	
+# 0xADCA	 	
+# 0xADCB	 	
+# 0xADCC	 	
+# 0xADCD	 	
+# 0xADCE	 	
+# 0xADCF	 	
+# 0xADD0	 	
+# 0xADD1	 	
+# 0xADD2	 	
+# 0xADD3	 	
+# 0xADD4	 	
+# 0xADD5	 	
+# 0xADD6	 	
+# 0xADD7	 	
+# 0xADD8	 	
+# 0xADD9	 	
+# 0xADDA	 	
+# 0xADDB	 	
+# 0xADDC	 	
+# 0xADDD	 	
+# 0xADDE	 	
+# 0xADDF	 	
+# 0xADE0	 	
+# 0xADE1	 	
+# 0xADE2	 	
+# 0xADE3	 	
+# 0xADE4	 	
+# 0xADE5	 	
+# 0xADE6	 	
+# 0xADE7	 	
+# 0xADE8	 	
+# 0xADE9	 	
+# 0xADEA	 	
+# 0xADEB	 	
+# 0xADEC	 	
+# 0xADED	 	
+# 0xADEE	 	
+# 0xADEF	 	
+# 0xADF0	 	
+# 0xADF1	 	
+# 0xADF2	 	
+# 0xADF3	 	
+# 0xADF4	 	
+# 0xADF5	 	
+# 0xADF6	 	
+# 0xADF7	 	
+# 0xADF8	 	
+# 0xADF9	 	
+# 0xADFA	 	
+# 0xADFB	 	
+# 0xADFC	 	
+# 0xADFD	 	
+# 0xADFE	 	
+0xAE40	0x74F3	
+0xAE41	0x74F5	
+0xAE42	0x74F8	
+0xAE43	0x74F9	
+0xAE44	0x74FA	
+0xAE45	0x74FB	
+0xAE46	0x74FC	
+0xAE47	0x74FD	
+0xAE48	0x74FE	
+0xAE49	0x7500	
+0xAE4A	0x7501	
+0xAE4B	0x7502	
+0xAE4C	0x7503	
+0xAE4D	0x7505	
+0xAE4E	0x7506	
+0xAE4F	0x7507	
+0xAE50	0x7508	
+0xAE51	0x7509	
+0xAE52	0x750A	
+0xAE53	0x750B	
+0xAE54	0x750C	
+0xAE55	0x750E	
+0xAE56	0x7510	
+0xAE57	0x7512	
+0xAE58	0x7514	
+0xAE59	0x7515	
+0xAE5A	0x7516	
+0xAE5B	0x7517	
+0xAE5C	0x751B	
+0xAE5D	0x751D	
+0xAE5E	0x751E	
+0xAE5F	0x7520	
+0xAE60	0x7521	
+0xAE61	0x7522	
+0xAE62	0x7523	
+0xAE63	0x7524	
+0xAE64	0x7526	
+0xAE65	0x7527	
+0xAE66	0x752A	
+0xAE67	0x752E	
+0xAE68	0x7534	
+0xAE69	0x7536	
+0xAE6A	0x7539	
+0xAE6B	0x753C	
+0xAE6C	0x753D	
+0xAE6D	0x753F	
+0xAE6E	0x7541	
+0xAE6F	0x7542	
+0xAE70	0x7543	
+0xAE71	0x7544	
+0xAE72	0x7546	
+0xAE73	0x7547	
+0xAE74	0x7549	
+0xAE75	0x754A	
+0xAE76	0x754D	
+0xAE77	0x7550	
+0xAE78	0x7551	
+0xAE79	0x7552	
+0xAE7A	0x7553	
+0xAE7B	0x7555	
+0xAE7C	0x7556	
+0xAE7D	0x7557	
+0xAE7E	0x7558	
+0xAE80	0x755D	
+0xAE81	0x755E	
+0xAE82	0x755F	
+0xAE83	0x7560	
+0xAE84	0x7561	
+0xAE85	0x7562	
+0xAE86	0x7563	
+0xAE87	0x7564	
+0xAE88	0x7567	
+0xAE89	0x7568	
+0xAE8A	0x7569	
+0xAE8B	0x756B	
+0xAE8C	0x756C	
+0xAE8D	0x756D	
+0xAE8E	0x756E	
+0xAE8F	0x756F	
+0xAE90	0x7570	
+0xAE91	0x7571	
+0xAE92	0x7573	
+0xAE93	0x7575	
+0xAE94	0x7576	
+0xAE95	0x7577	
+0xAE96	0x757A	
+0xAE97	0x757B	
+0xAE98	0x757C	
+0xAE99	0x757D	
+0xAE9A	0x757E	
+0xAE9B	0x7580	
+0xAE9C	0x7581	
+0xAE9D	0x7582	
+0xAE9E	0x7584	
+0xAE9F	0x7585	
+0xAEA0	0x7587	
+# 0xAEA1	 	
+# 0xAEA2	 	
+# 0xAEA3	 	
+# 0xAEA4	 	
+# 0xAEA5	 	
+# 0xAEA6	 	
+# 0xAEA7	 	
+# 0xAEA8	 	
+# 0xAEA9	 	
+# 0xAEAA	 	
+# 0xAEAB	 	
+# 0xAEAC	 	
+# 0xAEAD	 	
+# 0xAEAE	 	
+# 0xAEAF	 	
+# 0xAEB0	 	
+# 0xAEB1	 	
+# 0xAEB2	 	
+# 0xAEB3	 	
+# 0xAEB4	 	
+# 0xAEB5	 	
+# 0xAEB6	 	
+# 0xAEB7	 	
+# 0xAEB8	 	
+# 0xAEB9	 	
+# 0xAEBA	 	
+# 0xAEBB	 	
+# 0xAEBC	 	
+# 0xAEBD	 	
+# 0xAEBE	 	
+# 0xAEBF	 	
+# 0xAEC0	 	
+# 0xAEC1	 	
+# 0xAEC2	 	
+# 0xAEC3	 	
+# 0xAEC4	 	
+# 0xAEC5	 	
+# 0xAEC6	 	
+# 0xAEC7	 	
+# 0xAEC8	 	
+# 0xAEC9	 	
+# 0xAECA	 	
+# 0xAECB	 	
+# 0xAECC	 	
+# 0xAECD	 	
+# 0xAECE	 	
+# 0xAECF	 	
+# 0xAED0	 	
+# 0xAED1	 	
+# 0xAED2	 	
+# 0xAED3	 	
+# 0xAED4	 	
+# 0xAED5	 	
+# 0xAED6	 	
+# 0xAED7	 	
+# 0xAED8	 	
+# 0xAED9	 	
+# 0xAEDA	 	
+# 0xAEDB	 	
+# 0xAEDC	 	
+# 0xAEDD	 	
+# 0xAEDE	 	
+# 0xAEDF	 	
+# 0xAEE0	 	
+# 0xAEE1	 	
+# 0xAEE2	 	
+# 0xAEE3	 	
+# 0xAEE4	 	
+# 0xAEE5	 	
+# 0xAEE6	 	
+# 0xAEE7	 	
+# 0xAEE8	 	
+# 0xAEE9	 	
+# 0xAEEA	 	
+# 0xAEEB	 	
+# 0xAEEC	 	
+# 0xAEED	 	
+# 0xAEEE	 	
+# 0xAEEF	 	
+# 0xAEF0	 	
+# 0xAEF1	 	
+# 0xAEF2	 	
+# 0xAEF3	 	
+# 0xAEF4	 	
+# 0xAEF5	 	
+# 0xAEF6	 	
+# 0xAEF7	 	
+# 0xAEF8	 	
+# 0xAEF9	 	
+# 0xAEFA	 	
+# 0xAEFB	 	
+# 0xAEFC	 	
+# 0xAEFD	 	
+# 0xAEFE	 	
+0xAF40	0x7588	
+0xAF41	0x7589	
+0xAF42	0x758A	
+0xAF43	0x758C	
+0xAF44	0x758D	
+0xAF45	0x758E	
+0xAF46	0x7590	
+0xAF47	0x7593	
+0xAF48	0x7595	
+0xAF49	0x7598	
+0xAF4A	0x759B	
+0xAF4B	0x759C	
+0xAF4C	0x759E	
+0xAF4D	0x75A2	
+0xAF4E	0x75A6	
+0xAF4F	0x75A7	
+0xAF50	0x75A8	
+0xAF51	0x75A9	
+0xAF52	0x75AA	
+0xAF53	0x75AD	
+0xAF54	0x75B6	
+0xAF55	0x75B7	
+0xAF56	0x75BA	
+0xAF57	0x75BB	
+0xAF58	0x75BF	
+0xAF59	0x75C0	
+0xAF5A	0x75C1	
+0xAF5B	0x75C6	
+0xAF5C	0x75CB	
+0xAF5D	0x75CC	
+0xAF5E	0x75CE	
+0xAF5F	0x75CF	
+0xAF60	0x75D0	
+0xAF61	0x75D1	
+0xAF62	0x75D3	
+0xAF63	0x75D7	
+0xAF64	0x75D9	
+0xAF65	0x75DA	
+0xAF66	0x75DC	
+0xAF67	0x75DD	
+0xAF68	0x75DF	
+0xAF69	0x75E0	
+0xAF6A	0x75E1	
+0xAF6B	0x75E5	
+0xAF6C	0x75E9	
+0xAF6D	0x75EC	
+0xAF6E	0x75ED	
+0xAF6F	0x75EE	
+0xAF70	0x75EF	
+0xAF71	0x75F2	
+0xAF72	0x75F3	
+0xAF73	0x75F5	
+0xAF74	0x75F6	
+0xAF75	0x75F7	
+0xAF76	0x75F8	
+0xAF77	0x75FA	
+0xAF78	0x75FB	
+0xAF79	0x75FD	
+0xAF7A	0x75FE	
+0xAF7B	0x7602	
+0xAF7C	0x7604	
+0xAF7D	0x7606	
+0xAF7E	0x7607	
+0xAF80	0x7608	
+0xAF81	0x7609	
+0xAF82	0x760B	
+0xAF83	0x760D	
+0xAF84	0x760E	
+0xAF85	0x760F	
+0xAF86	0x7611	
+0xAF87	0x7612	
+0xAF88	0x7613	
+0xAF89	0x7614	
+0xAF8A	0x7616	
+0xAF8B	0x761A	
+0xAF8C	0x761C	
+0xAF8D	0x761D	
+0xAF8E	0x761E	
+0xAF8F	0x7621	
+0xAF90	0x7623	
+0xAF91	0x7627	
+0xAF92	0x7628	
+0xAF93	0x762C	
+0xAF94	0x762E	
+0xAF95	0x762F	
+0xAF96	0x7631	
+0xAF97	0x7632	
+0xAF98	0x7636	
+0xAF99	0x7637	
+0xAF9A	0x7639	
+0xAF9B	0x763A	
+0xAF9C	0x763B	
+0xAF9D	0x763D	
+0xAF9E	0x7641	
+0xAF9F	0x7642	
+0xAFA0	0x7644	
+# 0xAFA1	 	
+# 0xAFA2	 	
+# 0xAFA3	 	
+# 0xAFA4	 	
+# 0xAFA5	 	
+# 0xAFA6	 	
+# 0xAFA7	 	
+# 0xAFA8	 	
+# 0xAFA9	 	
+# 0xAFAA	 	
+# 0xAFAB	 	
+# 0xAFAC	 	
+# 0xAFAD	 	
+# 0xAFAE	 	
+# 0xAFAF	 	
+# 0xAFB0	 	
+# 0xAFB1	 	
+# 0xAFB2	 	
+# 0xAFB3	 	
+# 0xAFB4	 	
+# 0xAFB5	 	
+# 0xAFB6	 	
+# 0xAFB7	 	
+# 0xAFB8	 	
+# 0xAFB9	 	
+# 0xAFBA	 	
+# 0xAFBB	 	
+# 0xAFBC	 	
+# 0xAFBD	 	
+# 0xAFBE	 	
+# 0xAFBF	 	
+# 0xAFC0	 	
+# 0xAFC1	 	
+# 0xAFC2	 	
+# 0xAFC3	 	
+# 0xAFC4	 	
+# 0xAFC5	 	
+# 0xAFC6	 	
+# 0xAFC7	 	
+# 0xAFC8	 	
+# 0xAFC9	 	
+# 0xAFCA	 	
+# 0xAFCB	 	
+# 0xAFCC	 	
+# 0xAFCD	 	
+# 0xAFCE	 	
+# 0xAFCF	 	
+# 0xAFD0	 	
+# 0xAFD1	 	
+# 0xAFD2	 	
+# 0xAFD3	 	
+# 0xAFD4	 	
+# 0xAFD5	 	
+# 0xAFD6	 	
+# 0xAFD7	 	
+# 0xAFD8	 	
+# 0xAFD9	 	
+# 0xAFDA	 	
+# 0xAFDB	 	
+# 0xAFDC	 	
+# 0xAFDD	 	
+# 0xAFDE	 	
+# 0xAFDF	 	
+# 0xAFE0	 	
+# 0xAFE1	 	
+# 0xAFE2	 	
+# 0xAFE3	 	
+# 0xAFE4	 	
+# 0xAFE5	 	
+# 0xAFE6	 	
+# 0xAFE7	 	
+# 0xAFE8	 	
+# 0xAFE9	 	
+# 0xAFEA	 	
+# 0xAFEB	 	
+# 0xAFEC	 	
+# 0xAFED	 	
+# 0xAFEE	 	
+# 0xAFEF	 	
+# 0xAFF0	 	
+# 0xAFF1	 	
+# 0xAFF2	 	
+# 0xAFF3	 	
+# 0xAFF4	 	
+# 0xAFF5	 	
+# 0xAFF6	 	
+# 0xAFF7	 	
+# 0xAFF8	 	
+# 0xAFF9	 	
+# 0xAFFA	 	
+# 0xAFFB	 	
+# 0xAFFC	 	
+# 0xAFFD	 	
+# 0xAFFE	 	
+0xB040	0x7645	
+0xB041	0x7646	
+0xB042	0x7647	
+0xB043	0x7648	
+0xB044	0x7649	
+0xB045	0x764A	
+0xB046	0x764B	
+0xB047	0x764E	
+0xB048	0x764F	
+0xB049	0x7650	
+0xB04A	0x7651	
+0xB04B	0x7652	
+0xB04C	0x7653	
+0xB04D	0x7655	
+0xB04E	0x7657	
+0xB04F	0x7658	
+0xB050	0x7659	
+0xB051	0x765A	
+0xB052	0x765B	
+0xB053	0x765D	
+0xB054	0x765F	
+0xB055	0x7660	
+0xB056	0x7661	
+0xB057	0x7662	
+0xB058	0x7664	
+0xB059	0x7665	
+0xB05A	0x7666	
+0xB05B	0x7667	
+0xB05C	0x7668	
+0xB05D	0x7669	
+0xB05E	0x766A	
+0xB05F	0x766C	
+0xB060	0x766D	
+0xB061	0x766E	
+0xB062	0x7670	
+0xB063	0x7671	
+0xB064	0x7672	
+0xB065	0x7673	
+0xB066	0x7674	
+0xB067	0x7675	
+0xB068	0x7676	
+0xB069	0x7677	
+0xB06A	0x7679	
+0xB06B	0x767A	
+0xB06C	0x767C	
+0xB06D	0x767F	
+0xB06E	0x7680	
+0xB06F	0x7681	
+0xB070	0x7683	
+0xB071	0x7685	
+0xB072	0x7689	
+0xB073	0x768A	
+0xB074	0x768C	
+0xB075	0x768D	
+0xB076	0x768F	
+0xB077	0x7690	
+0xB078	0x7692	
+0xB079	0x7694	
+0xB07A	0x7695	
+0xB07B	0x7697	
+0xB07C	0x7698	
+0xB07D	0x769A	
+0xB07E	0x769B	
+0xB080	0x769C	
+0xB081	0x769D	
+0xB082	0x769E	
+0xB083	0x769F	
+0xB084	0x76A0	
+0xB085	0x76A1	
+0xB086	0x76A2	
+0xB087	0x76A3	
+0xB088	0x76A5	
+0xB089	0x76A6	
+0xB08A	0x76A7	
+0xB08B	0x76A8	
+0xB08C	0x76A9	
+0xB08D	0x76AA	
+0xB08E	0x76AB	
+0xB08F	0x76AC	
+0xB090	0x76AD	
+0xB091	0x76AF	
+0xB092	0x76B0	
+0xB093	0x76B3	
+0xB094	0x76B5	
+0xB095	0x76B6	
+0xB096	0x76B7	
+0xB097	0x76B8	
+0xB098	0x76B9	
+0xB099	0x76BA	
+0xB09A	0x76BB	
+0xB09B	0x76BC	
+0xB09C	0x76BD	
+0xB09D	0x76BE	
+0xB09E	0x76C0	
+0xB09F	0x76C1	
+0xB0A0	0x76C3	
+0xB0A1	0x554A	
+0xB0A2	0x963F	
+0xB0A3	0x57C3	
+0xB0A4	0x6328	
+0xB0A5	0x54CE	
+0xB0A6	0x5509	
+0xB0A7	0x54C0	
+0xB0A8	0x7691	
+0xB0A9	0x764C	
+0xB0AA	0x853C	
+0xB0AB	0x77EE	
+0xB0AC	0x827E	
+0xB0AD	0x788D	
+0xB0AE	0x7231	
+0xB0AF	0x9698	
+0xB0B0	0x978D	
+0xB0B1	0x6C28	
+0xB0B2	0x5B89	
+0xB0B3	0x4FFA	
+0xB0B4	0x6309	
+0xB0B5	0x6697	
+0xB0B6	0x5CB8	
+0xB0B7	0x80FA	
+0xB0B8	0x6848	
+0xB0B9	0x80AE	
+0xB0BA	0x6602	
+0xB0BB	0x76CE	
+0xB0BC	0x51F9	
+0xB0BD	0x6556	
+0xB0BE	0x71AC	
+0xB0BF	0x7FF1	
+0xB0C0	0x8884	
+0xB0C1	0x50B2	
+0xB0C2	0x5965	
+0xB0C3	0x61CA	
+0xB0C4	0x6FB3	
+0xB0C5	0x82AD	
+0xB0C6	0x634C	
+0xB0C7	0x6252	
+0xB0C8	0x53ED	
+0xB0C9	0x5427	
+0xB0CA	0x7B06	
+0xB0CB	0x516B	
+0xB0CC	0x75A4	
+0xB0CD	0x5DF4	
+0xB0CE	0x62D4	
+0xB0CF	0x8DCB	
+0xB0D0	0x9776	
+0xB0D1	0x628A	
+0xB0D2	0x8019	
+0xB0D3	0x575D	
+0xB0D4	0x9738	
+0xB0D5	0x7F62	
+0xB0D6	0x7238	
+0xB0D7	0x767D	
+0xB0D8	0x67CF	
+0xB0D9	0x767E	
+0xB0DA	0x6446	
+0xB0DB	0x4F70	
+0xB0DC	0x8D25	
+0xB0DD	0x62DC	
+0xB0DE	0x7A17	
+0xB0DF	0x6591	
+0xB0E0	0x73ED	
+0xB0E1	0x642C	
+0xB0E2	0x6273	
+0xB0E3	0x822C	
+0xB0E4	0x9881	
+0xB0E5	0x677F	
+0xB0E6	0x7248	
+0xB0E7	0x626E	
+0xB0E8	0x62CC	
+0xB0E9	0x4F34	
+0xB0EA	0x74E3	
+0xB0EB	0x534A	
+0xB0EC	0x529E	
+0xB0ED	0x7ECA	
+0xB0EE	0x90A6	
+0xB0EF	0x5E2E	
+0xB0F0	0x6886	
+0xB0F1	0x699C	
+0xB0F2	0x8180	
+0xB0F3	0x7ED1	
+0xB0F4	0x68D2	
+0xB0F5	0x78C5	
+0xB0F6	0x868C	
+0xB0F7	0x9551	
+0xB0F8	0x508D	
+0xB0F9	0x8C24	
+0xB0FA	0x82DE	
+0xB0FB	0x80DE	
+0xB0FC	0x5305	
+0xB0FD	0x8912	
+0xB0FE	0x5265	
+0xB140	0x76C4	
+0xB141	0x76C7	
+0xB142	0x76C9	
+0xB143	0x76CB	
+0xB144	0x76CC	
+0xB145	0x76D3	
+0xB146	0x76D5	
+0xB147	0x76D9	
+0xB148	0x76DA	
+0xB149	0x76DC	
+0xB14A	0x76DD	
+0xB14B	0x76DE	
+0xB14C	0x76E0	
+0xB14D	0x76E1	
+0xB14E	0x76E2	
+0xB14F	0x76E3	
+0xB150	0x76E4	
+0xB151	0x76E6	
+0xB152	0x76E7	
+0xB153	0x76E8	
+0xB154	0x76E9	
+0xB155	0x76EA	
+0xB156	0x76EB	
+0xB157	0x76EC	
+0xB158	0x76ED	
+0xB159	0x76F0	
+0xB15A	0x76F3	
+0xB15B	0x76F5	
+0xB15C	0x76F6	
+0xB15D	0x76F7	
+0xB15E	0x76FA	
+0xB15F	0x76FB	
+0xB160	0x76FD	
+0xB161	0x76FF	
+0xB162	0x7700	
+0xB163	0x7702	
+0xB164	0x7703	
+0xB165	0x7705	
+0xB166	0x7706	
+0xB167	0x770A	
+0xB168	0x770C	
+0xB169	0x770E	
+0xB16A	0x770F	
+0xB16B	0x7710	
+0xB16C	0x7711	
+0xB16D	0x7712	
+0xB16E	0x7713	
+0xB16F	0x7714	
+0xB170	0x7715	
+0xB171	0x7716	
+0xB172	0x7717	
+0xB173	0x7718	
+0xB174	0x771B	
+0xB175	0x771C	
+0xB176	0x771D	
+0xB177	0x771E	
+0xB178	0x7721	
+0xB179	0x7723	
+0xB17A	0x7724	
+0xB17B	0x7725	
+0xB17C	0x7727	
+0xB17D	0x772A	
+0xB17E	0x772B	
+0xB180	0x772C	
+0xB181	0x772E	
+0xB182	0x7730	
+0xB183	0x7731	
+0xB184	0x7732	
+0xB185	0x7733	
+0xB186	0x7734	
+0xB187	0x7739	
+0xB188	0x773B	
+0xB189	0x773D	
+0xB18A	0x773E	
+0xB18B	0x773F	
+0xB18C	0x7742	
+0xB18D	0x7744	
+0xB18E	0x7745	
+0xB18F	0x7746	
+0xB190	0x7748	
+0xB191	0x7749	
+0xB192	0x774A	
+0xB193	0x774B	
+0xB194	0x774C	
+0xB195	0x774D	
+0xB196	0x774E	
+0xB197	0x774F	
+0xB198	0x7752	
+0xB199	0x7753	
+0xB19A	0x7754	
+0xB19B	0x7755	
+0xB19C	0x7756	
+0xB19D	0x7757	
+0xB19E	0x7758	
+0xB19F	0x7759	
+0xB1A0	0x775C	
+0xB1A1	0x8584	
+0xB1A2	0x96F9	
+0xB1A3	0x4FDD	
+0xB1A4	0x5821	
+0xB1A5	0x9971	
+0xB1A6	0x5B9D	
+0xB1A7	0x62B1	
+0xB1A8	0x62A5	
+0xB1A9	0x66B4	
+0xB1AA	0x8C79	
+0xB1AB	0x9C8D	
+0xB1AC	0x7206	
+0xB1AD	0x676F	
+0xB1AE	0x7891	
+0xB1AF	0x60B2	
+0xB1B0	0x5351	
+0xB1B1	0x5317	
+0xB1B2	0x8F88	
+0xB1B3	0x80CC	
+0xB1B4	0x8D1D	
+0xB1B5	0x94A1	
+0xB1B6	0x500D	
+0xB1B7	0x72C8	
+0xB1B8	0x5907	
+0xB1B9	0x60EB	
+0xB1BA	0x7119	
+0xB1BB	0x88AB	
+0xB1BC	0x5954	
+0xB1BD	0x82EF	
+0xB1BE	0x672C	
+0xB1BF	0x7B28	
+0xB1C0	0x5D29	
+0xB1C1	0x7EF7	
+0xB1C2	0x752D	
+0xB1C3	0x6CF5	
+0xB1C4	0x8E66	
+0xB1C5	0x8FF8	
+0xB1C6	0x903C	
+0xB1C7	0x9F3B	
+0xB1C8	0x6BD4	
+0xB1C9	0x9119	
+0xB1CA	0x7B14	
+0xB1CB	0x5F7C	
+0xB1CC	0x78A7	
+0xB1CD	0x84D6	
+0xB1CE	0x853D	
+0xB1CF	0x6BD5	
+0xB1D0	0x6BD9	
+0xB1D1	0x6BD6	
+0xB1D2	0x5E01	
+0xB1D3	0x5E87	
+0xB1D4	0x75F9	
+0xB1D5	0x95ED	
+0xB1D6	0x655D	
+0xB1D7	0x5F0A	
+0xB1D8	0x5FC5	
+0xB1D9	0x8F9F	
+0xB1DA	0x58C1	
+0xB1DB	0x81C2	
+0xB1DC	0x907F	
+0xB1DD	0x965B	
+0xB1DE	0x97AD	
+0xB1DF	0x8FB9	
+0xB1E0	0x7F16	
+0xB1E1	0x8D2C	
+0xB1E2	0x6241	
+0xB1E3	0x4FBF	
+0xB1E4	0x53D8	
+0xB1E5	0x535E	
+0xB1E6	0x8FA8	
+0xB1E7	0x8FA9	
+0xB1E8	0x8FAB	
+0xB1E9	0x904D	
+0xB1EA	0x6807	
+0xB1EB	0x5F6A	
+0xB1EC	0x8198	
+0xB1ED	0x8868	
+0xB1EE	0x9CD6	
+0xB1EF	0x618B	
+0xB1F0	0x522B	
+0xB1F1	0x762A	
+0xB1F2	0x5F6C	
+0xB1F3	0x658C	
+0xB1F4	0x6FD2	
+0xB1F5	0x6EE8	
+0xB1F6	0x5BBE	
+0xB1F7	0x6448	
+0xB1F8	0x5175	
+0xB1F9	0x51B0	
+0xB1FA	0x67C4	
+0xB1FB	0x4E19	
+0xB1FC	0x79C9	
+0xB1FD	0x997C	
+0xB1FE	0x70B3	
+0xB240	0x775D	
+0xB241	0x775E	
+0xB242	0x775F	
+0xB243	0x7760	
+0xB244	0x7764	
+0xB245	0x7767	
+0xB246	0x7769	
+0xB247	0x776A	
+0xB248	0x776D	
+0xB249	0x776E	
+0xB24A	0x776F	
+0xB24B	0x7770	
+0xB24C	0x7771	
+0xB24D	0x7772	
+0xB24E	0x7773	
+0xB24F	0x7774	
+0xB250	0x7775	
+0xB251	0x7776	
+0xB252	0x7777	
+0xB253	0x7778	
+0xB254	0x777A	
+0xB255	0x777B	
+0xB256	0x777C	
+0xB257	0x7781	
+0xB258	0x7782	
+0xB259	0x7783	
+0xB25A	0x7786	
+0xB25B	0x7787	
+0xB25C	0x7788	
+0xB25D	0x7789	
+0xB25E	0x778A	
+0xB25F	0x778B	
+0xB260	0x778F	
+0xB261	0x7790	
+0xB262	0x7793	
+0xB263	0x7794	
+0xB264	0x7795	
+0xB265	0x7796	
+0xB266	0x7797	
+0xB267	0x7798	
+0xB268	0x7799	
+0xB269	0x779A	
+0xB26A	0x779B	
+0xB26B	0x779C	
+0xB26C	0x779D	
+0xB26D	0x779E	
+0xB26E	0x77A1	
+0xB26F	0x77A3	
+0xB270	0x77A4	
+0xB271	0x77A6	
+0xB272	0x77A8	
+0xB273	0x77AB	
+0xB274	0x77AD	
+0xB275	0x77AE	
+0xB276	0x77AF	
+0xB277	0x77B1	
+0xB278	0x77B2	
+0xB279	0x77B4	
+0xB27A	0x77B6	
+0xB27B	0x77B7	
+0xB27C	0x77B8	
+0xB27D	0x77B9	
+0xB27E	0x77BA	
+0xB280	0x77BC	
+0xB281	0x77BE	
+0xB282	0x77C0	
+0xB283	0x77C1	
+0xB284	0x77C2	
+0xB285	0x77C3	
+0xB286	0x77C4	
+0xB287	0x77C5	
+0xB288	0x77C6	
+0xB289	0x77C7	
+0xB28A	0x77C8	
+0xB28B	0x77C9	
+0xB28C	0x77CA	
+0xB28D	0x77CB	
+0xB28E	0x77CC	
+0xB28F	0x77CE	
+0xB290	0x77CF	
+0xB291	0x77D0	
+0xB292	0x77D1	
+0xB293	0x77D2	
+0xB294	0x77D3	
+0xB295	0x77D4	
+0xB296	0x77D5	
+0xB297	0x77D6	
+0xB298	0x77D8	
+0xB299	0x77D9	
+0xB29A	0x77DA	
+0xB29B	0x77DD	
+0xB29C	0x77DE	
+0xB29D	0x77DF	
+0xB29E	0x77E0	
+0xB29F	0x77E1	
+0xB2A0	0x77E4	
+0xB2A1	0x75C5	
+0xB2A2	0x5E76	
+0xB2A3	0x73BB	
+0xB2A4	0x83E0	
+0xB2A5	0x64AD	
+0xB2A6	0x62E8	
+0xB2A7	0x94B5	
+0xB2A8	0x6CE2	
+0xB2A9	0x535A	
+0xB2AA	0x52C3	
+0xB2AB	0x640F	
+0xB2AC	0x94C2	
+0xB2AD	0x7B94	
+0xB2AE	0x4F2F	
+0xB2AF	0x5E1B	
+0xB2B0	0x8236	
+0xB2B1	0x8116	
+0xB2B2	0x818A	
+0xB2B3	0x6E24	
+0xB2B4	0x6CCA	
+0xB2B5	0x9A73	
+0xB2B6	0x6355	
+0xB2B7	0x535C	
+0xB2B8	0x54FA	
+0xB2B9	0x8865	
+0xB2BA	0x57E0	
+0xB2BB	0x4E0D	
+0xB2BC	0x5E03	
+0xB2BD	0x6B65	
+0xB2BE	0x7C3F	
+0xB2BF	0x90E8	
+0xB2C0	0x6016	
+0xB2C1	0x64E6	
+0xB2C2	0x731C	
+0xB2C3	0x88C1	
+0xB2C4	0x6750	
+0xB2C5	0x624D	
+0xB2C6	0x8D22	
+0xB2C7	0x776C	
+0xB2C8	0x8E29	
+0xB2C9	0x91C7	
+0xB2CA	0x5F69	
+0xB2CB	0x83DC	
+0xB2CC	0x8521	
+0xB2CD	0x9910	
+0xB2CE	0x53C2	
+0xB2CF	0x8695	
+0xB2D0	0x6B8B	
+0xB2D1	0x60ED	
+0xB2D2	0x60E8	
+0xB2D3	0x707F	
+0xB2D4	0x82CD	
+0xB2D5	0x8231	
+0xB2D6	0x4ED3	
+0xB2D7	0x6CA7	
+0xB2D8	0x85CF	
+0xB2D9	0x64CD	
+0xB2DA	0x7CD9	
+0xB2DB	0x69FD	
+0xB2DC	0x66F9	
+0xB2DD	0x8349	
+0xB2DE	0x5395	
+0xB2DF	0x7B56	
+0xB2E0	0x4FA7	
+0xB2E1	0x518C	
+0xB2E2	0x6D4B	
+0xB2E3	0x5C42	
+0xB2E4	0x8E6D	
+0xB2E5	0x63D2	
+0xB2E6	0x53C9	
+0xB2E7	0x832C	
+0xB2E8	0x8336	
+0xB2E9	0x67E5	
+0xB2EA	0x78B4	
+0xB2EB	0x643D	
+0xB2EC	0x5BDF	
+0xB2ED	0x5C94	
+0xB2EE	0x5DEE	
+0xB2EF	0x8BE7	
+0xB2F0	0x62C6	
+0xB2F1	0x67F4	
+0xB2F2	0x8C7A	
+0xB2F3	0x6400	
+0xB2F4	0x63BA	
+0xB2F5	0x8749	
+0xB2F6	0x998B	
+0xB2F7	0x8C17	
+0xB2F8	0x7F20	
+0xB2F9	0x94F2	
+0xB2FA	0x4EA7	
+0xB2FB	0x9610	
+0xB2FC	0x98A4	
+0xB2FD	0x660C	
+0xB2FE	0x7316	
+0xB340	0x77E6	
+0xB341	0x77E8	
+0xB342	0x77EA	
+0xB343	0x77EF	
+0xB344	0x77F0	
+0xB345	0x77F1	
+0xB346	0x77F2	
+0xB347	0x77F4	
+0xB348	0x77F5	
+0xB349	0x77F7	
+0xB34A	0x77F9	
+0xB34B	0x77FA	
+0xB34C	0x77FB	
+0xB34D	0x77FC	
+0xB34E	0x7803	
+0xB34F	0x7804	
+0xB350	0x7805	
+0xB351	0x7806	
+0xB352	0x7807	
+0xB353	0x7808	
+0xB354	0x780A	
+0xB355	0x780B	
+0xB356	0x780E	
+0xB357	0x780F	
+0xB358	0x7810	
+0xB359	0x7813	
+0xB35A	0x7815	
+0xB35B	0x7819	
+0xB35C	0x781B	
+0xB35D	0x781E	
+0xB35E	0x7820	
+0xB35F	0x7821	
+0xB360	0x7822	
+0xB361	0x7824	
+0xB362	0x7828	
+0xB363	0x782A	
+0xB364	0x782B	
+0xB365	0x782E	
+0xB366	0x782F	
+0xB367	0x7831	
+0xB368	0x7832	
+0xB369	0x7833	
+0xB36A	0x7835	
+0xB36B	0x7836	
+0xB36C	0x783D	
+0xB36D	0x783F	
+0xB36E	0x7841	
+0xB36F	0x7842	
+0xB370	0x7843	
+0xB371	0x7844	
+0xB372	0x7846	
+0xB373	0x7848	
+0xB374	0x7849	
+0xB375	0x784A	
+0xB376	0x784B	
+0xB377	0x784D	
+0xB378	0x784F	
+0xB379	0x7851	
+0xB37A	0x7853	
+0xB37B	0x7854	
+0xB37C	0x7858	
+0xB37D	0x7859	
+0xB37E	0x785A	
+0xB380	0x785B	
+0xB381	0x785C	
+0xB382	0x785E	
+0xB383	0x785F	
+0xB384	0x7860	
+0xB385	0x7861	
+0xB386	0x7862	
+0xB387	0x7863	
+0xB388	0x7864	
+0xB389	0x7865	
+0xB38A	0x7866	
+0xB38B	0x7867	
+0xB38C	0x7868	
+0xB38D	0x7869	
+0xB38E	0x786F	
+0xB38F	0x7870	
+0xB390	0x7871	
+0xB391	0x7872	
+0xB392	0x7873	
+0xB393	0x7874	
+0xB394	0x7875	
+0xB395	0x7876	
+0xB396	0x7878	
+0xB397	0x7879	
+0xB398	0x787A	
+0xB399	0x787B	
+0xB39A	0x787D	
+0xB39B	0x787E	
+0xB39C	0x787F	
+0xB39D	0x7880	
+0xB39E	0x7881	
+0xB39F	0x7882	
+0xB3A0	0x7883	
+0xB3A1	0x573A	
+0xB3A2	0x5C1D	
+0xB3A3	0x5E38	
+0xB3A4	0x957F	
+0xB3A5	0x507F	
+0xB3A6	0x80A0	
+0xB3A7	0x5382	
+0xB3A8	0x655E	
+0xB3A9	0x7545	
+0xB3AA	0x5531	
+0xB3AB	0x5021	
+0xB3AC	0x8D85	
+0xB3AD	0x6284	
+0xB3AE	0x949E	
+0xB3AF	0x671D	
+0xB3B0	0x5632	
+0xB3B1	0x6F6E	
+0xB3B2	0x5DE2	
+0xB3B3	0x5435	
+0xB3B4	0x7092	
+0xB3B5	0x8F66	
+0xB3B6	0x626F	
+0xB3B7	0x64A4	
+0xB3B8	0x63A3	
+0xB3B9	0x5F7B	
+0xB3BA	0x6F88	
+0xB3BB	0x90F4	
+0xB3BC	0x81E3	
+0xB3BD	0x8FB0	
+0xB3BE	0x5C18	
+0xB3BF	0x6668	
+0xB3C0	0x5FF1	
+0xB3C1	0x6C89	
+0xB3C2	0x9648	
+0xB3C3	0x8D81	
+0xB3C4	0x886C	
+0xB3C5	0x6491	
+0xB3C6	0x79F0	
+0xB3C7	0x57CE	
+0xB3C8	0x6A59	
+0xB3C9	0x6210	
+0xB3CA	0x5448	
+0xB3CB	0x4E58	
+0xB3CC	0x7A0B	
+0xB3CD	0x60E9	
+0xB3CE	0x6F84	
+0xB3CF	0x8BDA	
+0xB3D0	0x627F	
+0xB3D1	0x901E	
+0xB3D2	0x9A8B	
+0xB3D3	0x79E4	
+0xB3D4	0x5403	
+0xB3D5	0x75F4	
+0xB3D6	0x6301	
+0xB3D7	0x5319	
+0xB3D8	0x6C60	
+0xB3D9	0x8FDF	
+0xB3DA	0x5F1B	
+0xB3DB	0x9A70	
+0xB3DC	0x803B	
+0xB3DD	0x9F7F	
+0xB3DE	0x4F88	
+0xB3DF	0x5C3A	
+0xB3E0	0x8D64	
+0xB3E1	0x7FC5	
+0xB3E2	0x65A5	
+0xB3E3	0x70BD	
+0xB3E4	0x5145	
+0xB3E5	0x51B2	
+0xB3E6	0x866B	
+0xB3E7	0x5D07	
+0xB3E8	0x5BA0	
+0xB3E9	0x62BD	
+0xB3EA	0x916C	
+0xB3EB	0x7574	
+0xB3EC	0x8E0C	
+0xB3ED	0x7A20	
+0xB3EE	0x6101	
+0xB3EF	0x7B79	
+0xB3F0	0x4EC7	
+0xB3F1	0x7EF8	
+0xB3F2	0x7785	
+0xB3F3	0x4E11	
+0xB3F4	0x81ED	
+0xB3F5	0x521D	
+0xB3F6	0x51FA	
+0xB3F7	0x6A71	
+0xB3F8	0x53A8	
+0xB3F9	0x8E87	
+0xB3FA	0x9504	
+0xB3FB	0x96CF	
+0xB3FC	0x6EC1	
+0xB3FD	0x9664	
+0xB3FE	0x695A	
+0xB440	0x7884	
+0xB441	0x7885	
+0xB442	0x7886	
+0xB443	0x7888	
+0xB444	0x788A	
+0xB445	0x788B	
+0xB446	0x788F	
+0xB447	0x7890	
+0xB448	0x7892	
+0xB449	0x7894	
+0xB44A	0x7895	
+0xB44B	0x7896	
+0xB44C	0x7899	
+0xB44D	0x789D	
+0xB44E	0x789E	
+0xB44F	0x78A0	
+0xB450	0x78A2	
+0xB451	0x78A4	
+0xB452	0x78A6	
+0xB453	0x78A8	
+0xB454	0x78A9	
+0xB455	0x78AA	
+0xB456	0x78AB	
+0xB457	0x78AC	
+0xB458	0x78AD	
+0xB459	0x78AE	
+0xB45A	0x78AF	
+0xB45B	0x78B5	
+0xB45C	0x78B6	
+0xB45D	0x78B7	
+0xB45E	0x78B8	
+0xB45F	0x78BA	
+0xB460	0x78BB	
+0xB461	0x78BC	
+0xB462	0x78BD	
+0xB463	0x78BF	
+0xB464	0x78C0	
+0xB465	0x78C2	
+0xB466	0x78C3	
+0xB467	0x78C4	
+0xB468	0x78C6	
+0xB469	0x78C7	
+0xB46A	0x78C8	
+0xB46B	0x78CC	
+0xB46C	0x78CD	
+0xB46D	0x78CE	
+0xB46E	0x78CF	
+0xB46F	0x78D1	
+0xB470	0x78D2	
+0xB471	0x78D3	
+0xB472	0x78D6	
+0xB473	0x78D7	
+0xB474	0x78D8	
+0xB475	0x78DA	
+0xB476	0x78DB	
+0xB477	0x78DC	
+0xB478	0x78DD	
+0xB479	0x78DE	
+0xB47A	0x78DF	
+0xB47B	0x78E0	
+0xB47C	0x78E1	
+0xB47D	0x78E2	
+0xB47E	0x78E3	
+0xB480	0x78E4	
+0xB481	0x78E5	
+0xB482	0x78E6	
+0xB483	0x78E7	
+0xB484	0x78E9	
+0xB485	0x78EA	
+0xB486	0x78EB	
+0xB487	0x78ED	
+0xB488	0x78EE	
+0xB489	0x78EF	
+0xB48A	0x78F0	
+0xB48B	0x78F1	
+0xB48C	0x78F3	
+0xB48D	0x78F5	
+0xB48E	0x78F6	
+0xB48F	0x78F8	
+0xB490	0x78F9	
+0xB491	0x78FB	
+0xB492	0x78FC	
+0xB493	0x78FD	
+0xB494	0x78FE	
+0xB495	0x78FF	
+0xB496	0x7900	
+0xB497	0x7902	
+0xB498	0x7903	
+0xB499	0x7904	
+0xB49A	0x7906	
+0xB49B	0x7907	
+0xB49C	0x7908	
+0xB49D	0x7909	
+0xB49E	0x790A	
+0xB49F	0x790B	
+0xB4A0	0x790C	
+0xB4A1	0x7840	
+0xB4A2	0x50A8	
+0xB4A3	0x77D7	
+0xB4A4	0x6410	
+0xB4A5	0x89E6	
+0xB4A6	0x5904	
+0xB4A7	0x63E3	
+0xB4A8	0x5DDD	
+0xB4A9	0x7A7F	
+0xB4AA	0x693D	
+0xB4AB	0x4F20	
+0xB4AC	0x8239	
+0xB4AD	0x5598	
+0xB4AE	0x4E32	
+0xB4AF	0x75AE	
+0xB4B0	0x7A97	
+0xB4B1	0x5E62	
+0xB4B2	0x5E8A	
+0xB4B3	0x95EF	
+0xB4B4	0x521B	
+0xB4B5	0x5439	
+0xB4B6	0x708A	
+0xB4B7	0x6376	
+0xB4B8	0x9524	
+0xB4B9	0x5782	
+0xB4BA	0x6625	
+0xB4BB	0x693F	
+0xB4BC	0x9187	
+0xB4BD	0x5507	
+0xB4BE	0x6DF3	
+0xB4BF	0x7EAF	
+0xB4C0	0x8822	
+0xB4C1	0x6233	
+0xB4C2	0x7EF0	
+0xB4C3	0x75B5	
+0xB4C4	0x8328	
+0xB4C5	0x78C1	
+0xB4C6	0x96CC	
+0xB4C7	0x8F9E	
+0xB4C8	0x6148	
+0xB4C9	0x74F7	
+0xB4CA	0x8BCD	
+0xB4CB	0x6B64	
+0xB4CC	0x523A	
+0xB4CD	0x8D50	
+0xB4CE	0x6B21	
+0xB4CF	0x806A	
+0xB4D0	0x8471	
+0xB4D1	0x56F1	
+0xB4D2	0x5306	
+0xB4D3	0x4ECE	
+0xB4D4	0x4E1B	
+0xB4D5	0x51D1	
+0xB4D6	0x7C97	
+0xB4D7	0x918B	
+0xB4D8	0x7C07	
+0xB4D9	0x4FC3	
+0xB4DA	0x8E7F	
+0xB4DB	0x7BE1	
+0xB4DC	0x7A9C	
+0xB4DD	0x6467	
+0xB4DE	0x5D14	
+0xB4DF	0x50AC	
+0xB4E0	0x8106	
+0xB4E1	0x7601	
+0xB4E2	0x7CB9	
+0xB4E3	0x6DEC	
+0xB4E4	0x7FE0	
+0xB4E5	0x6751	
+0xB4E6	0x5B58	
+0xB4E7	0x5BF8	
+0xB4E8	0x78CB	
+0xB4E9	0x64AE	
+0xB4EA	0x6413	
+0xB4EB	0x63AA	
+0xB4EC	0x632B	
+0xB4ED	0x9519	
+0xB4EE	0x642D	
+0xB4EF	0x8FBE	
+0xB4F0	0x7B54	
+0xB4F1	0x7629	
+0xB4F2	0x6253	
+0xB4F3	0x5927	
+0xB4F4	0x5446	
+0xB4F5	0x6B79	
+0xB4F6	0x50A3	
+0xB4F7	0x6234	
+0xB4F8	0x5E26	
+0xB4F9	0x6B86	
+0xB4FA	0x4EE3	
+0xB4FB	0x8D37	
+0xB4FC	0x888B	
+0xB4FD	0x5F85	
+0xB4FE	0x902E	
+0xB540	0x790D	
+0xB541	0x790E	
+0xB542	0x790F	
+0xB543	0x7910	
+0xB544	0x7911	
+0xB545	0x7912	
+0xB546	0x7914	
+0xB547	0x7915	
+0xB548	0x7916	
+0xB549	0x7917	
+0xB54A	0x7918	
+0xB54B	0x7919	
+0xB54C	0x791A	
+0xB54D	0x791B	
+0xB54E	0x791C	
+0xB54F	0x791D	
+0xB550	0x791F	
+0xB551	0x7920	
+0xB552	0x7921	
+0xB553	0x7922	
+0xB554	0x7923	
+0xB555	0x7925	
+0xB556	0x7926	
+0xB557	0x7927	
+0xB558	0x7928	
+0xB559	0x7929	
+0xB55A	0x792A	
+0xB55B	0x792B	
+0xB55C	0x792C	
+0xB55D	0x792D	
+0xB55E	0x792E	
+0xB55F	0x792F	
+0xB560	0x7930	
+0xB561	0x7931	
+0xB562	0x7932	
+0xB563	0x7933	
+0xB564	0x7935	
+0xB565	0x7936	
+0xB566	0x7937	
+0xB567	0x7938	
+0xB568	0x7939	
+0xB569	0x793D	
+0xB56A	0x793F	
+0xB56B	0x7942	
+0xB56C	0x7943	
+0xB56D	0x7944	
+0xB56E	0x7945	
+0xB56F	0x7947	
+0xB570	0x794A	
+0xB571	0x794B	
+0xB572	0x794C	
+0xB573	0x794D	
+0xB574	0x794E	
+0xB575	0x794F	
+0xB576	0x7950	
+0xB577	0x7951	
+0xB578	0x7952	
+0xB579	0x7954	
+0xB57A	0x7955	
+0xB57B	0x7958	
+0xB57C	0x7959	
+0xB57D	0x7961	
+0xB57E	0x7963	
+0xB580	0x7964	
+0xB581	0x7966	
+0xB582	0x7969	
+0xB583	0x796A	
+0xB584	0x796B	
+0xB585	0x796C	
+0xB586	0x796E	
+0xB587	0x7970	
+0xB588	0x7971	
+0xB589	0x7972	
+0xB58A	0x7973	
+0xB58B	0x7974	
+0xB58C	0x7975	
+0xB58D	0x7976	
+0xB58E	0x7979	
+0xB58F	0x797B	
+0xB590	0x797C	
+0xB591	0x797D	
+0xB592	0x797E	
+0xB593	0x797F	
+0xB594	0x7982	
+0xB595	0x7983	
+0xB596	0x7986	
+0xB597	0x7987	
+0xB598	0x7988	
+0xB599	0x7989	
+0xB59A	0x798B	
+0xB59B	0x798C	
+0xB59C	0x798D	
+0xB59D	0x798E	
+0xB59E	0x7990	
+0xB59F	0x7991	
+0xB5A0	0x7992	
+0xB5A1	0x6020	
+0xB5A2	0x803D	
+0xB5A3	0x62C5	
+0xB5A4	0x4E39	
+0xB5A5	0x5355	
+0xB5A6	0x90F8	
+0xB5A7	0x63B8	
+0xB5A8	0x80C6	
+0xB5A9	0x65E6	
+0xB5AA	0x6C2E	
+0xB5AB	0x4F46	
+0xB5AC	0x60EE	
+0xB5AD	0x6DE1	
+0xB5AE	0x8BDE	
+0xB5AF	0x5F39	
+0xB5B0	0x86CB	
+0xB5B1	0x5F53	
+0xB5B2	0x6321	
+0xB5B3	0x515A	
+0xB5B4	0x8361	
+0xB5B5	0x6863	
+0xB5B6	0x5200	
+0xB5B7	0x6363	
+0xB5B8	0x8E48	
+0xB5B9	0x5012	
+0xB5BA	0x5C9B	
+0xB5BB	0x7977	
+0xB5BC	0x5BFC	
+0xB5BD	0x5230	
+0xB5BE	0x7A3B	
+0xB5BF	0x60BC	
+0xB5C0	0x9053	
+0xB5C1	0x76D7	
+0xB5C2	0x5FB7	
+0xB5C3	0x5F97	
+0xB5C4	0x7684	
+0xB5C5	0x8E6C	
+0xB5C6	0x706F	
+0xB5C7	0x767B	
+0xB5C8	0x7B49	
+0xB5C9	0x77AA	
+0xB5CA	0x51F3	
+0xB5CB	0x9093	
+0xB5CC	0x5824	
+0xB5CD	0x4F4E	
+0xB5CE	0x6EF4	
+0xB5CF	0x8FEA	
+0xB5D0	0x654C	
+0xB5D1	0x7B1B	
+0xB5D2	0x72C4	
+0xB5D3	0x6DA4	
+0xB5D4	0x7FDF	
+0xB5D5	0x5AE1	
+0xB5D6	0x62B5	
+0xB5D7	0x5E95	
+0xB5D8	0x5730	
+0xB5D9	0x8482	
+0xB5DA	0x7B2C	
+0xB5DB	0x5E1D	
+0xB5DC	0x5F1F	
+0xB5DD	0x9012	
+0xB5DE	0x7F14	
+0xB5DF	0x98A0	
+0xB5E0	0x6382	
+0xB5E1	0x6EC7	
+0xB5E2	0x7898	
+0xB5E3	0x70B9	
+0xB5E4	0x5178	
+0xB5E5	0x975B	
+0xB5E6	0x57AB	
+0xB5E7	0x7535	
+0xB5E8	0x4F43	
+0xB5E9	0x7538	
+0xB5EA	0x5E97	
+0xB5EB	0x60E6	
+0xB5EC	0x5960	
+0xB5ED	0x6DC0	
+0xB5EE	0x6BBF	
+0xB5EF	0x7889	
+0xB5F0	0x53FC	
+0xB5F1	0x96D5	
+0xB5F2	0x51CB	
+0xB5F3	0x5201	
+0xB5F4	0x6389	
+0xB5F5	0x540A	
+0xB5F6	0x9493	
+0xB5F7	0x8C03	
+0xB5F8	0x8DCC	
+0xB5F9	0x7239	
+0xB5FA	0x789F	
+0xB5FB	0x8776	
+0xB5FC	0x8FED	
+0xB5FD	0x8C0D	
+0xB5FE	0x53E0	
+0xB640	0x7993	
+0xB641	0x7994	
+0xB642	0x7995	
+0xB643	0x7996	
+0xB644	0x7997	
+0xB645	0x7998	
+0xB646	0x7999	
+0xB647	0x799B	
+0xB648	0x799C	
+0xB649	0x799D	
+0xB64A	0x799E	
+0xB64B	0x799F	
+0xB64C	0x79A0	
+0xB64D	0x79A1	
+0xB64E	0x79A2	
+0xB64F	0x79A3	
+0xB650	0x79A4	
+0xB651	0x79A5	
+0xB652	0x79A6	
+0xB653	0x79A8	
+0xB654	0x79A9	
+0xB655	0x79AA	
+0xB656	0x79AB	
+0xB657	0x79AC	
+0xB658	0x79AD	
+0xB659	0x79AE	
+0xB65A	0x79AF	
+0xB65B	0x79B0	
+0xB65C	0x79B1	
+0xB65D	0x79B2	
+0xB65E	0x79B4	
+0xB65F	0x79B5	
+0xB660	0x79B6	
+0xB661	0x79B7	
+0xB662	0x79B8	
+0xB663	0x79BC	
+0xB664	0x79BF	
+0xB665	0x79C2	
+0xB666	0x79C4	
+0xB667	0x79C5	
+0xB668	0x79C7	
+0xB669	0x79C8	
+0xB66A	0x79CA	
+0xB66B	0x79CC	
+0xB66C	0x79CE	
+0xB66D	0x79CF	
+0xB66E	0x79D0	
+0xB66F	0x79D3	
+0xB670	0x79D4	
+0xB671	0x79D6	
+0xB672	0x79D7	
+0xB673	0x79D9	
+0xB674	0x79DA	
+0xB675	0x79DB	
+0xB676	0x79DC	
+0xB677	0x79DD	
+0xB678	0x79DE	
+0xB679	0x79E0	
+0xB67A	0x79E1	
+0xB67B	0x79E2	
+0xB67C	0x79E5	
+0xB67D	0x79E8	
+0xB67E	0x79EA	
+0xB680	0x79EC	
+0xB681	0x79EE	
+0xB682	0x79F1	
+0xB683	0x79F2	
+0xB684	0x79F3	
+0xB685	0x79F4	
+0xB686	0x79F5	
+0xB687	0x79F6	
+0xB688	0x79F7	
+0xB689	0x79F9	
+0xB68A	0x79FA	
+0xB68B	0x79FC	
+0xB68C	0x79FE	
+0xB68D	0x79FF	
+0xB68E	0x7A01	
+0xB68F	0x7A04	
+0xB690	0x7A05	
+0xB691	0x7A07	
+0xB692	0x7A08	
+0xB693	0x7A09	
+0xB694	0x7A0A	
+0xB695	0x7A0C	
+0xB696	0x7A0F	
+0xB697	0x7A10	
+0xB698	0x7A11	
+0xB699	0x7A12	
+0xB69A	0x7A13	
+0xB69B	0x7A15	
+0xB69C	0x7A16	
+0xB69D	0x7A18	
+0xB69E	0x7A19	
+0xB69F	0x7A1B	
+0xB6A0	0x7A1C	
+0xB6A1	0x4E01	
+0xB6A2	0x76EF	
+0xB6A3	0x53EE	
+0xB6A4	0x9489	
+0xB6A5	0x9876	
+0xB6A6	0x9F0E	
+0xB6A7	0x952D	
+0xB6A8	0x5B9A	
+0xB6A9	0x8BA2	
+0xB6AA	0x4E22	
+0xB6AB	0x4E1C	
+0xB6AC	0x51AC	
+0xB6AD	0x8463	
+0xB6AE	0x61C2	
+0xB6AF	0x52A8	
+0xB6B0	0x680B	
+0xB6B1	0x4F97	
+0xB6B2	0x606B	
+0xB6B3	0x51BB	
+0xB6B4	0x6D1E	
+0xB6B5	0x515C	
+0xB6B6	0x6296	
+0xB6B7	0x6597	
+0xB6B8	0x9661	
+0xB6B9	0x8C46	
+0xB6BA	0x9017	
+0xB6BB	0x75D8	
+0xB6BC	0x90FD	
+0xB6BD	0x7763	
+0xB6BE	0x6BD2	
+0xB6BF	0x728A	
+0xB6C0	0x72EC	
+0xB6C1	0x8BFB	
+0xB6C2	0x5835	
+0xB6C3	0x7779	
+0xB6C4	0x8D4C	
+0xB6C5	0x675C	
+0xB6C6	0x9540	
+0xB6C7	0x809A	
+0xB6C8	0x5EA6	
+0xB6C9	0x6E21	
+0xB6CA	0x5992	
+0xB6CB	0x7AEF	
+0xB6CC	0x77ED	
+0xB6CD	0x953B	
+0xB6CE	0x6BB5	
+0xB6CF	0x65AD	
+0xB6D0	0x7F0E	
+0xB6D1	0x5806	
+0xB6D2	0x5151	
+0xB6D3	0x961F	
+0xB6D4	0x5BF9	
+0xB6D5	0x58A9	
+0xB6D6	0x5428	
+0xB6D7	0x8E72	
+0xB6D8	0x6566	
+0xB6D9	0x987F	
+0xB6DA	0x56E4	
+0xB6DB	0x949D	
+0xB6DC	0x76FE	
+0xB6DD	0x9041	
+0xB6DE	0x6387	
+0xB6DF	0x54C6	
+0xB6E0	0x591A	
+0xB6E1	0x593A	
+0xB6E2	0x579B	
+0xB6E3	0x8EB2	
+0xB6E4	0x6735	
+0xB6E5	0x8DFA	
+0xB6E6	0x8235	
+0xB6E7	0x5241	
+0xB6E8	0x60F0	
+0xB6E9	0x5815	
+0xB6EA	0x86FE	
+0xB6EB	0x5CE8	
+0xB6EC	0x9E45	
+0xB6ED	0x4FC4	
+0xB6EE	0x989D	
+0xB6EF	0x8BB9	
+0xB6F0	0x5A25	
+0xB6F1	0x6076	
+0xB6F2	0x5384	
+0xB6F3	0x627C	
+0xB6F4	0x904F	
+0xB6F5	0x9102	
+0xB6F6	0x997F	
+0xB6F7	0x6069	
+0xB6F8	0x800C	
+0xB6F9	0x513F	
+0xB6FA	0x8033	
+0xB6FB	0x5C14	
+0xB6FC	0x9975	
+0xB6FD	0x6D31	
+0xB6FE	0x4E8C	
+0xB740	0x7A1D	
+0xB741	0x7A1F	
+0xB742	0x7A21	
+0xB743	0x7A22	
+0xB744	0x7A24	
+0xB745	0x7A25	
+0xB746	0x7A26	
+0xB747	0x7A27	
+0xB748	0x7A28	
+0xB749	0x7A29	
+0xB74A	0x7A2A	
+0xB74B	0x7A2B	
+0xB74C	0x7A2C	
+0xB74D	0x7A2D	
+0xB74E	0x7A2E	
+0xB74F	0x7A2F	
+0xB750	0x7A30	
+0xB751	0x7A31	
+0xB752	0x7A32	
+0xB753	0x7A34	
+0xB754	0x7A35	
+0xB755	0x7A36	
+0xB756	0x7A38	
+0xB757	0x7A3A	
+0xB758	0x7A3E	
+0xB759	0x7A40	
+0xB75A	0x7A41	
+0xB75B	0x7A42	
+0xB75C	0x7A43	
+0xB75D	0x7A44	
+0xB75E	0x7A45	
+0xB75F	0x7A47	
+0xB760	0x7A48	
+0xB761	0x7A49	
+0xB762	0x7A4A	
+0xB763	0x7A4B	
+0xB764	0x7A4C	
+0xB765	0x7A4D	
+0xB766	0x7A4E	
+0xB767	0x7A4F	
+0xB768	0x7A50	
+0xB769	0x7A52	
+0xB76A	0x7A53	
+0xB76B	0x7A54	
+0xB76C	0x7A55	
+0xB76D	0x7A56	
+0xB76E	0x7A58	
+0xB76F	0x7A59	
+0xB770	0x7A5A	
+0xB771	0x7A5B	
+0xB772	0x7A5C	
+0xB773	0x7A5D	
+0xB774	0x7A5E	
+0xB775	0x7A5F	
+0xB776	0x7A60	
+0xB777	0x7A61	
+0xB778	0x7A62	
+0xB779	0x7A63	
+0xB77A	0x7A64	
+0xB77B	0x7A65	
+0xB77C	0x7A66	
+0xB77D	0x7A67	
+0xB77E	0x7A68	
+0xB780	0x7A69	
+0xB781	0x7A6A	
+0xB782	0x7A6B	
+0xB783	0x7A6C	
+0xB784	0x7A6D	
+0xB785	0x7A6E	
+0xB786	0x7A6F	
+0xB787	0x7A71	
+0xB788	0x7A72	
+0xB789	0x7A73	
+0xB78A	0x7A75	
+0xB78B	0x7A7B	
+0xB78C	0x7A7C	
+0xB78D	0x7A7D	
+0xB78E	0x7A7E	
+0xB78F	0x7A82	
+0xB790	0x7A85	
+0xB791	0x7A87	
+0xB792	0x7A89	
+0xB793	0x7A8A	
+0xB794	0x7A8B	
+0xB795	0x7A8C	
+0xB796	0x7A8E	
+0xB797	0x7A8F	
+0xB798	0x7A90	
+0xB799	0x7A93	
+0xB79A	0x7A94	
+0xB79B	0x7A99	
+0xB79C	0x7A9A	
+0xB79D	0x7A9B	
+0xB79E	0x7A9E	
+0xB79F	0x7AA1	
+0xB7A0	0x7AA2	
+0xB7A1	0x8D30	
+0xB7A2	0x53D1	
+0xB7A3	0x7F5A	
+0xB7A4	0x7B4F	
+0xB7A5	0x4F10	
+0xB7A6	0x4E4F	
+0xB7A7	0x9600	
+0xB7A8	0x6CD5	
+0xB7A9	0x73D0	
+0xB7AA	0x85E9	
+0xB7AB	0x5E06	
+0xB7AC	0x756A	
+0xB7AD	0x7FFB	
+0xB7AE	0x6A0A	
+0xB7AF	0x77FE	
+0xB7B0	0x9492	
+0xB7B1	0x7E41	
+0xB7B2	0x51E1	
+0xB7B3	0x70E6	
+0xB7B4	0x53CD	
+0xB7B5	0x8FD4	
+0xB7B6	0x8303	
+0xB7B7	0x8D29	
+0xB7B8	0x72AF	
+0xB7B9	0x996D	
+0xB7BA	0x6CDB	
+0xB7BB	0x574A	
+0xB7BC	0x82B3	
+0xB7BD	0x65B9	
+0xB7BE	0x80AA	
+0xB7BF	0x623F	
+0xB7C0	0x9632	
+0xB7C1	0x59A8	
+0xB7C2	0x4EFF	
+0xB7C3	0x8BBF	
+0xB7C4	0x7EBA	
+0xB7C5	0x653E	
+0xB7C6	0x83F2	
+0xB7C7	0x975E	
+0xB7C8	0x5561	
+0xB7C9	0x98DE	
+0xB7CA	0x80A5	
+0xB7CB	0x532A	
+0xB7CC	0x8BFD	
+0xB7CD	0x5420	
+0xB7CE	0x80BA	
+0xB7CF	0x5E9F	
+0xB7D0	0x6CB8	
+0xB7D1	0x8D39	
+0xB7D2	0x82AC	
+0xB7D3	0x915A	
+0xB7D4	0x5429	
+0xB7D5	0x6C1B	
+0xB7D6	0x5206	
+0xB7D7	0x7EB7	
+0xB7D8	0x575F	
+0xB7D9	0x711A	
+0xB7DA	0x6C7E	
+0xB7DB	0x7C89	
+0xB7DC	0x594B	
+0xB7DD	0x4EFD	
+0xB7DE	0x5FFF	
+0xB7DF	0x6124	
+0xB7E0	0x7CAA	
+0xB7E1	0x4E30	
+0xB7E2	0x5C01	
+0xB7E3	0x67AB	
+0xB7E4	0x8702	
+0xB7E5	0x5CF0	
+0xB7E6	0x950B	
+0xB7E7	0x98CE	
+0xB7E8	0x75AF	
+0xB7E9	0x70FD	
+0xB7EA	0x9022	
+0xB7EB	0x51AF	
+0xB7EC	0x7F1D	
+0xB7ED	0x8BBD	
+0xB7EE	0x5949	
+0xB7EF	0x51E4	
+0xB7F0	0x4F5B	
+0xB7F1	0x5426	
+0xB7F2	0x592B	
+0xB7F3	0x6577	
+0xB7F4	0x80A4	
+0xB7F5	0x5B75	
+0xB7F6	0x6276	
+0xB7F7	0x62C2	
+0xB7F8	0x8F90	
+0xB7F9	0x5E45	
+0xB7FA	0x6C1F	
+0xB7FB	0x7B26	
+0xB7FC	0x4F0F	
+0xB7FD	0x4FD8	
+0xB7FE	0x670D	
+0xB840	0x7AA3	
+0xB841	0x7AA4	
+0xB842	0x7AA7	
+0xB843	0x7AA9	
+0xB844	0x7AAA	
+0xB845	0x7AAB	
+0xB846	0x7AAE	
+0xB847	0x7AAF	
+0xB848	0x7AB0	
+0xB849	0x7AB1	
+0xB84A	0x7AB2	
+0xB84B	0x7AB4	
+0xB84C	0x7AB5	
+0xB84D	0x7AB6	
+0xB84E	0x7AB7	
+0xB84F	0x7AB8	
+0xB850	0x7AB9	
+0xB851	0x7ABA	
+0xB852	0x7ABB	
+0xB853	0x7ABC	
+0xB854	0x7ABD	
+0xB855	0x7ABE	
+0xB856	0x7AC0	
+0xB857	0x7AC1	
+0xB858	0x7AC2	
+0xB859	0x7AC3	
+0xB85A	0x7AC4	
+0xB85B	0x7AC5	
+0xB85C	0x7AC6	
+0xB85D	0x7AC7	
+0xB85E	0x7AC8	
+0xB85F	0x7AC9	
+0xB860	0x7ACA	
+0xB861	0x7ACC	
+0xB862	0x7ACD	
+0xB863	0x7ACE	
+0xB864	0x7ACF	
+0xB865	0x7AD0	
+0xB866	0x7AD1	
+0xB867	0x7AD2	
+0xB868	0x7AD3	
+0xB869	0x7AD4	
+0xB86A	0x7AD5	
+0xB86B	0x7AD7	
+0xB86C	0x7AD8	
+0xB86D	0x7ADA	
+0xB86E	0x7ADB	
+0xB86F	0x7ADC	
+0xB870	0x7ADD	
+0xB871	0x7AE1	
+0xB872	0x7AE2	
+0xB873	0x7AE4	
+0xB874	0x7AE7	
+0xB875	0x7AE8	
+0xB876	0x7AE9	
+0xB877	0x7AEA	
+0xB878	0x7AEB	
+0xB879	0x7AEC	
+0xB87A	0x7AEE	
+0xB87B	0x7AF0	
+0xB87C	0x7AF1	
+0xB87D	0x7AF2	
+0xB87E	0x7AF3	
+0xB880	0x7AF4	
+0xB881	0x7AF5	
+0xB882	0x7AF6	
+0xB883	0x7AF7	
+0xB884	0x7AF8	
+0xB885	0x7AFB	
+0xB886	0x7AFC	
+0xB887	0x7AFE	
+0xB888	0x7B00	
+0xB889	0x7B01	
+0xB88A	0x7B02	
+0xB88B	0x7B05	
+0xB88C	0x7B07	
+0xB88D	0x7B09	
+0xB88E	0x7B0C	
+0xB88F	0x7B0D	
+0xB890	0x7B0E	
+0xB891	0x7B10	
+0xB892	0x7B12	
+0xB893	0x7B13	
+0xB894	0x7B16	
+0xB895	0x7B17	
+0xB896	0x7B18	
+0xB897	0x7B1A	
+0xB898	0x7B1C	
+0xB899	0x7B1D	
+0xB89A	0x7B1F	
+0xB89B	0x7B21	
+0xB89C	0x7B22	
+0xB89D	0x7B23	
+0xB89E	0x7B27	
+0xB89F	0x7B29	
+0xB8A0	0x7B2D	
+0xB8A1	0x6D6E	
+0xB8A2	0x6DAA	
+0xB8A3	0x798F	
+0xB8A4	0x88B1	
+0xB8A5	0x5F17	
+0xB8A6	0x752B	
+0xB8A7	0x629A	
+0xB8A8	0x8F85	
+0xB8A9	0x4FEF	
+0xB8AA	0x91DC	
+0xB8AB	0x65A7	
+0xB8AC	0x812F	
+0xB8AD	0x8151	
+0xB8AE	0x5E9C	
+0xB8AF	0x8150	
+0xB8B0	0x8D74	
+0xB8B1	0x526F	
+0xB8B2	0x8986	
+0xB8B3	0x8D4B	
+0xB8B4	0x590D	
+0xB8B5	0x5085	
+0xB8B6	0x4ED8	
+0xB8B7	0x961C	
+0xB8B8	0x7236	
+0xB8B9	0x8179	
+0xB8BA	0x8D1F	
+0xB8BB	0x5BCC	
+0xB8BC	0x8BA3	
+0xB8BD	0x9644	
+0xB8BE	0x5987	
+0xB8BF	0x7F1A	
+0xB8C0	0x5490	
+0xB8C1	0x5676	
+0xB8C2	0x560E	
+0xB8C3	0x8BE5	
+0xB8C4	0x6539	
+0xB8C5	0x6982	
+0xB8C6	0x9499	
+0xB8C7	0x76D6	
+0xB8C8	0x6E89	
+0xB8C9	0x5E72	
+0xB8CA	0x7518	
+0xB8CB	0x6746	
+0xB8CC	0x67D1	
+0xB8CD	0x7AFF	
+0xB8CE	0x809D	
+0xB8CF	0x8D76	
+0xB8D0	0x611F	
+0xB8D1	0x79C6	
+0xB8D2	0x6562	
+0xB8D3	0x8D63	
+0xB8D4	0x5188	
+0xB8D5	0x521A	
+0xB8D6	0x94A2	
+0xB8D7	0x7F38	
+0xB8D8	0x809B	
+0xB8D9	0x7EB2	
+0xB8DA	0x5C97	
+0xB8DB	0x6E2F	
+0xB8DC	0x6760	
+0xB8DD	0x7BD9	
+0xB8DE	0x768B	
+0xB8DF	0x9AD8	
+0xB8E0	0x818F	
+0xB8E1	0x7F94	
+0xB8E2	0x7CD5	
+0xB8E3	0x641E	
+0xB8E4	0x9550	
+0xB8E5	0x7A3F	
+0xB8E6	0x544A	
+0xB8E7	0x54E5	
+0xB8E8	0x6B4C	
+0xB8E9	0x6401	
+0xB8EA	0x6208	
+0xB8EB	0x9E3D	
+0xB8EC	0x80F3	
+0xB8ED	0x7599	
+0xB8EE	0x5272	
+0xB8EF	0x9769	
+0xB8F0	0x845B	
+0xB8F1	0x683C	
+0xB8F2	0x86E4	
+0xB8F3	0x9601	
+0xB8F4	0x9694	
+0xB8F5	0x94EC	
+0xB8F6	0x4E2A	
+0xB8F7	0x5404	
+0xB8F8	0x7ED9	
+0xB8F9	0x6839	
+0xB8FA	0x8DDF	
+0xB8FB	0x8015	
+0xB8FC	0x66F4	
+0xB8FD	0x5E9A	
+0xB8FE	0x7FB9	
+0xB940	0x7B2F	
+0xB941	0x7B30	
+0xB942	0x7B32	
+0xB943	0x7B34	
+0xB944	0x7B35	
+0xB945	0x7B36	
+0xB946	0x7B37	
+0xB947	0x7B39	
+0xB948	0x7B3B	
+0xB949	0x7B3D	
+0xB94A	0x7B3F	
+0xB94B	0x7B40	
+0xB94C	0x7B41	
+0xB94D	0x7B42	
+0xB94E	0x7B43	
+0xB94F	0x7B44	
+0xB950	0x7B46	
+0xB951	0x7B48	
+0xB952	0x7B4A	
+0xB953	0x7B4D	
+0xB954	0x7B4E	
+0xB955	0x7B53	
+0xB956	0x7B55	
+0xB957	0x7B57	
+0xB958	0x7B59	
+0xB959	0x7B5C	
+0xB95A	0x7B5E	
+0xB95B	0x7B5F	
+0xB95C	0x7B61	
+0xB95D	0x7B63	
+0xB95E	0x7B64	
+0xB95F	0x7B65	
+0xB960	0x7B66	
+0xB961	0x7B67	
+0xB962	0x7B68	
+0xB963	0x7B69	
+0xB964	0x7B6A	
+0xB965	0x7B6B	
+0xB966	0x7B6C	
+0xB967	0x7B6D	
+0xB968	0x7B6F	
+0xB969	0x7B70	
+0xB96A	0x7B73	
+0xB96B	0x7B74	
+0xB96C	0x7B76	
+0xB96D	0x7B78	
+0xB96E	0x7B7A	
+0xB96F	0x7B7C	
+0xB970	0x7B7D	
+0xB971	0x7B7F	
+0xB972	0x7B81	
+0xB973	0x7B82	
+0xB974	0x7B83	
+0xB975	0x7B84	
+0xB976	0x7B86	
+0xB977	0x7B87	
+0xB978	0x7B88	
+0xB979	0x7B89	
+0xB97A	0x7B8A	
+0xB97B	0x7B8B	
+0xB97C	0x7B8C	
+0xB97D	0x7B8E	
+0xB97E	0x7B8F	
+0xB980	0x7B91	
+0xB981	0x7B92	
+0xB982	0x7B93	
+0xB983	0x7B96	
+0xB984	0x7B98	
+0xB985	0x7B99	
+0xB986	0x7B9A	
+0xB987	0x7B9B	
+0xB988	0x7B9E	
+0xB989	0x7B9F	
+0xB98A	0x7BA0	
+0xB98B	0x7BA3	
+0xB98C	0x7BA4	
+0xB98D	0x7BA5	
+0xB98E	0x7BAE	
+0xB98F	0x7BAF	
+0xB990	0x7BB0	
+0xB991	0x7BB2	
+0xB992	0x7BB3	
+0xB993	0x7BB5	
+0xB994	0x7BB6	
+0xB995	0x7BB7	
+0xB996	0x7BB9	
+0xB997	0x7BBA	
+0xB998	0x7BBB	
+0xB999	0x7BBC	
+0xB99A	0x7BBD	
+0xB99B	0x7BBE	
+0xB99C	0x7BBF	
+0xB99D	0x7BC0	
+0xB99E	0x7BC2	
+0xB99F	0x7BC3	
+0xB9A0	0x7BC4	
+0xB9A1	0x57C2	
+0xB9A2	0x803F	
+0xB9A3	0x6897	
+0xB9A4	0x5DE5	
+0xB9A5	0x653B	
+0xB9A6	0x529F	
+0xB9A7	0x606D	
+0xB9A8	0x9F9A	
+0xB9A9	0x4F9B	
+0xB9AA	0x8EAC	
+0xB9AB	0x516C	
+0xB9AC	0x5BAB	
+0xB9AD	0x5F13	
+0xB9AE	0x5DE9	
+0xB9AF	0x6C5E	
+0xB9B0	0x62F1	
+0xB9B1	0x8D21	
+0xB9B2	0x5171	
+0xB9B3	0x94A9	
+0xB9B4	0x52FE	
+0xB9B5	0x6C9F	
+0xB9B6	0x82DF	
+0xB9B7	0x72D7	
+0xB9B8	0x57A2	
+0xB9B9	0x6784	
+0xB9BA	0x8D2D	
+0xB9BB	0x591F	
+0xB9BC	0x8F9C	
+0xB9BD	0x83C7	
+0xB9BE	0x5495	
+0xB9BF	0x7B8D	
+0xB9C0	0x4F30	
+0xB9C1	0x6CBD	
+0xB9C2	0x5B64	
+0xB9C3	0x59D1	
+0xB9C4	0x9F13	
+0xB9C5	0x53E4	
+0xB9C6	0x86CA	
+0xB9C7	0x9AA8	
+0xB9C8	0x8C37	
+0xB9C9	0x80A1	
+0xB9CA	0x6545	
+0xB9CB	0x987E	
+0xB9CC	0x56FA	
+0xB9CD	0x96C7	
+0xB9CE	0x522E	
+0xB9CF	0x74DC	
+0xB9D0	0x5250	
+0xB9D1	0x5BE1	
+0xB9D2	0x6302	
+0xB9D3	0x8902	
+0xB9D4	0x4E56	
+0xB9D5	0x62D0	
+0xB9D6	0x602A	
+0xB9D7	0x68FA	
+0xB9D8	0x5173	
+0xB9D9	0x5B98	
+0xB9DA	0x51A0	
+0xB9DB	0x89C2	
+0xB9DC	0x7BA1	
+0xB9DD	0x9986	
+0xB9DE	0x7F50	
+0xB9DF	0x60EF	
+0xB9E0	0x704C	
+0xB9E1	0x8D2F	
+0xB9E2	0x5149	
+0xB9E3	0x5E7F	
+0xB9E4	0x901B	
+0xB9E5	0x7470	
+0xB9E6	0x89C4	
+0xB9E7	0x572D	
+0xB9E8	0x7845	
+0xB9E9	0x5F52	
+0xB9EA	0x9F9F	
+0xB9EB	0x95FA	
+0xB9EC	0x8F68	
+0xB9ED	0x9B3C	
+0xB9EE	0x8BE1	
+0xB9EF	0x7678	
+0xB9F0	0x6842	
+0xB9F1	0x67DC	
+0xB9F2	0x8DEA	
+0xB9F3	0x8D35	
+0xB9F4	0x523D	
+0xB9F5	0x8F8A	
+0xB9F6	0x6EDA	
+0xB9F7	0x68CD	
+0xB9F8	0x9505	
+0xB9F9	0x90ED	
+0xB9FA	0x56FD	
+0xB9FB	0x679C	
+0xB9FC	0x88F9	
+0xB9FD	0x8FC7	
+0xB9FE	0x54C8	
+0xBA40	0x7BC5	
+0xBA41	0x7BC8	
+0xBA42	0x7BC9	
+0xBA43	0x7BCA	
+0xBA44	0x7BCB	
+0xBA45	0x7BCD	
+0xBA46	0x7BCE	
+0xBA47	0x7BCF	
+0xBA48	0x7BD0	
+0xBA49	0x7BD2	
+0xBA4A	0x7BD4	
+0xBA4B	0x7BD5	
+0xBA4C	0x7BD6	
+0xBA4D	0x7BD7	
+0xBA4E	0x7BD8	
+0xBA4F	0x7BDB	
+0xBA50	0x7BDC	
+0xBA51	0x7BDE	
+0xBA52	0x7BDF	
+0xBA53	0x7BE0	
+0xBA54	0x7BE2	
+0xBA55	0x7BE3	
+0xBA56	0x7BE4	
+0xBA57	0x7BE7	
+0xBA58	0x7BE8	
+0xBA59	0x7BE9	
+0xBA5A	0x7BEB	
+0xBA5B	0x7BEC	
+0xBA5C	0x7BED	
+0xBA5D	0x7BEF	
+0xBA5E	0x7BF0	
+0xBA5F	0x7BF2	
+0xBA60	0x7BF3	
+0xBA61	0x7BF4	
+0xBA62	0x7BF5	
+0xBA63	0x7BF6	
+0xBA64	0x7BF8	
+0xBA65	0x7BF9	
+0xBA66	0x7BFA	
+0xBA67	0x7BFB	
+0xBA68	0x7BFD	
+0xBA69	0x7BFF	
+0xBA6A	0x7C00	
+0xBA6B	0x7C01	
+0xBA6C	0x7C02	
+0xBA6D	0x7C03	
+0xBA6E	0x7C04	
+0xBA6F	0x7C05	
+0xBA70	0x7C06	
+0xBA71	0x7C08	
+0xBA72	0x7C09	
+0xBA73	0x7C0A	
+0xBA74	0x7C0D	
+0xBA75	0x7C0E	
+0xBA76	0x7C10	
+0xBA77	0x7C11	
+0xBA78	0x7C12	
+0xBA79	0x7C13	
+0xBA7A	0x7C14	
+0xBA7B	0x7C15	
+0xBA7C	0x7C17	
+0xBA7D	0x7C18	
+0xBA7E	0x7C19	
+0xBA80	0x7C1A	
+0xBA81	0x7C1B	
+0xBA82	0x7C1C	
+0xBA83	0x7C1D	
+0xBA84	0x7C1E	
+0xBA85	0x7C20	
+0xBA86	0x7C21	
+0xBA87	0x7C22	
+0xBA88	0x7C23	
+0xBA89	0x7C24	
+0xBA8A	0x7C25	
+0xBA8B	0x7C28	
+0xBA8C	0x7C29	
+0xBA8D	0x7C2B	
+0xBA8E	0x7C2C	
+0xBA8F	0x7C2D	
+0xBA90	0x7C2E	
+0xBA91	0x7C2F	
+0xBA92	0x7C30	
+0xBA93	0x7C31	
+0xBA94	0x7C32	
+0xBA95	0x7C33	
+0xBA96	0x7C34	
+0xBA97	0x7C35	
+0xBA98	0x7C36	
+0xBA99	0x7C37	
+0xBA9A	0x7C39	
+0xBA9B	0x7C3A	
+0xBA9C	0x7C3B	
+0xBA9D	0x7C3C	
+0xBA9E	0x7C3D	
+0xBA9F	0x7C3E	
+0xBAA0	0x7C42	
+0xBAA1	0x9AB8	
+0xBAA2	0x5B69	
+0xBAA3	0x6D77	
+0xBAA4	0x6C26	
+0xBAA5	0x4EA5	
+0xBAA6	0x5BB3	
+0xBAA7	0x9A87	
+0xBAA8	0x9163	
+0xBAA9	0x61A8	
+0xBAAA	0x90AF	
+0xBAAB	0x97E9	
+0xBAAC	0x542B	
+0xBAAD	0x6DB5	
+0xBAAE	0x5BD2	
+0xBAAF	0x51FD	
+0xBAB0	0x558A	
+0xBAB1	0x7F55	
+0xBAB2	0x7FF0	
+0xBAB3	0x64BC	
+0xBAB4	0x634D	
+0xBAB5	0x65F1	
+0xBAB6	0x61BE	
+0xBAB7	0x608D	
+0xBAB8	0x710A	
+0xBAB9	0x6C57	
+0xBABA	0x6C49	
+0xBABB	0x592F	
+0xBABC	0x676D	
+0xBABD	0x822A	
+0xBABE	0x58D5	
+0xBABF	0x568E	
+0xBAC0	0x8C6A	
+0xBAC1	0x6BEB	
+0xBAC2	0x90DD	
+0xBAC3	0x597D	
+0xBAC4	0x8017	
+0xBAC5	0x53F7	
+0xBAC6	0x6D69	
+0xBAC7	0x5475	
+0xBAC8	0x559D	
+0xBAC9	0x8377	
+0xBACA	0x83CF	
+0xBACB	0x6838	
+0xBACC	0x79BE	
+0xBACD	0x548C	
+0xBACE	0x4F55	
+0xBACF	0x5408	
+0xBAD0	0x76D2	
+0xBAD1	0x8C89	
+0xBAD2	0x9602	
+0xBAD3	0x6CB3	
+0xBAD4	0x6DB8	
+0xBAD5	0x8D6B	
+0xBAD6	0x8910	
+0xBAD7	0x9E64	
+0xBAD8	0x8D3A	
+0xBAD9	0x563F	
+0xBADA	0x9ED1	
+0xBADB	0x75D5	
+0xBADC	0x5F88	
+0xBADD	0x72E0	
+0xBADE	0x6068	
+0xBADF	0x54FC	
+0xBAE0	0x4EA8	
+0xBAE1	0x6A2A	
+0xBAE2	0x8861	
+0xBAE3	0x6052	
+0xBAE4	0x8F70	
+0xBAE5	0x54C4	
+0xBAE6	0x70D8	
+0xBAE7	0x8679	
+0xBAE8	0x9E3F	
+0xBAE9	0x6D2A	
+0xBAEA	0x5B8F	
+0xBAEB	0x5F18	
+0xBAEC	0x7EA2	
+0xBAED	0x5589	
+0xBAEE	0x4FAF	
+0xBAEF	0x7334	
+0xBAF0	0x543C	
+0xBAF1	0x539A	
+0xBAF2	0x5019	
+0xBAF3	0x540E	
+0xBAF4	0x547C	
+0xBAF5	0x4E4E	
+0xBAF6	0x5FFD	
+0xBAF7	0x745A	
+0xBAF8	0x58F6	
+0xBAF9	0x846B	
+0xBAFA	0x80E1	
+0xBAFB	0x8774	
+0xBAFC	0x72D0	
+0xBAFD	0x7CCA	
+0xBAFE	0x6E56	
+0xBB40	0x7C43	
+0xBB41	0x7C44	
+0xBB42	0x7C45	
+0xBB43	0x7C46	
+0xBB44	0x7C47	
+0xBB45	0x7C48	
+0xBB46	0x7C49	
+0xBB47	0x7C4A	
+0xBB48	0x7C4B	
+0xBB49	0x7C4C	
+0xBB4A	0x7C4E	
+0xBB4B	0x7C4F	
+0xBB4C	0x7C50	
+0xBB4D	0x7C51	
+0xBB4E	0x7C52	
+0xBB4F	0x7C53	
+0xBB50	0x7C54	
+0xBB51	0x7C55	
+0xBB52	0x7C56	
+0xBB53	0x7C57	
+0xBB54	0x7C58	
+0xBB55	0x7C59	
+0xBB56	0x7C5A	
+0xBB57	0x7C5B	
+0xBB58	0x7C5C	
+0xBB59	0x7C5D	
+0xBB5A	0x7C5E	
+0xBB5B	0x7C5F	
+0xBB5C	0x7C60	
+0xBB5D	0x7C61	
+0xBB5E	0x7C62	
+0xBB5F	0x7C63	
+0xBB60	0x7C64	
+0xBB61	0x7C65	
+0xBB62	0x7C66	
+0xBB63	0x7C67	
+0xBB64	0x7C68	
+0xBB65	0x7C69	
+0xBB66	0x7C6A	
+0xBB67	0x7C6B	
+0xBB68	0x7C6C	
+0xBB69	0x7C6D	
+0xBB6A	0x7C6E	
+0xBB6B	0x7C6F	
+0xBB6C	0x7C70	
+0xBB6D	0x7C71	
+0xBB6E	0x7C72	
+0xBB6F	0x7C75	
+0xBB70	0x7C76	
+0xBB71	0x7C77	
+0xBB72	0x7C78	
+0xBB73	0x7C79	
+0xBB74	0x7C7A	
+0xBB75	0x7C7E	
+0xBB76	0x7C7F	
+0xBB77	0x7C80	
+0xBB78	0x7C81	
+0xBB79	0x7C82	
+0xBB7A	0x7C83	
+0xBB7B	0x7C84	
+0xBB7C	0x7C85	
+0xBB7D	0x7C86	
+0xBB7E	0x7C87	
+0xBB80	0x7C88	
+0xBB81	0x7C8A	
+0xBB82	0x7C8B	
+0xBB83	0x7C8C	
+0xBB84	0x7C8D	
+0xBB85	0x7C8E	
+0xBB86	0x7C8F	
+0xBB87	0x7C90	
+0xBB88	0x7C93	
+0xBB89	0x7C94	
+0xBB8A	0x7C96	
+0xBB8B	0x7C99	
+0xBB8C	0x7C9A	
+0xBB8D	0x7C9B	
+0xBB8E	0x7CA0	
+0xBB8F	0x7CA1	
+0xBB90	0x7CA3	
+0xBB91	0x7CA6	
+0xBB92	0x7CA7	
+0xBB93	0x7CA8	
+0xBB94	0x7CA9	
+0xBB95	0x7CAB	
+0xBB96	0x7CAC	
+0xBB97	0x7CAD	
+0xBB98	0x7CAF	
+0xBB99	0x7CB0	
+0xBB9A	0x7CB4	
+0xBB9B	0x7CB5	
+0xBB9C	0x7CB6	
+0xBB9D	0x7CB7	
+0xBB9E	0x7CB8	
+0xBB9F	0x7CBA	
+0xBBA0	0x7CBB	
+0xBBA1	0x5F27	
+0xBBA2	0x864E	
+0xBBA3	0x552C	
+0xBBA4	0x62A4	
+0xBBA5	0x4E92	
+0xBBA6	0x6CAA	
+0xBBA7	0x6237	
+0xBBA8	0x82B1	
+0xBBA9	0x54D7	
+0xBBAA	0x534E	
+0xBBAB	0x733E	
+0xBBAC	0x6ED1	
+0xBBAD	0x753B	
+0xBBAE	0x5212	
+0xBBAF	0x5316	
+0xBBB0	0x8BDD	
+0xBBB1	0x69D0	
+0xBBB2	0x5F8A	
+0xBBB3	0x6000	
+0xBBB4	0x6DEE	
+0xBBB5	0x574F	
+0xBBB6	0x6B22	
+0xBBB7	0x73AF	
+0xBBB8	0x6853	
+0xBBB9	0x8FD8	
+0xBBBA	0x7F13	
+0xBBBB	0x6362	
+0xBBBC	0x60A3	
+0xBBBD	0x5524	
+0xBBBE	0x75EA	
+0xBBBF	0x8C62	
+0xBBC0	0x7115	
+0xBBC1	0x6DA3	
+0xBBC2	0x5BA6	
+0xBBC3	0x5E7B	
+0xBBC4	0x8352	
+0xBBC5	0x614C	
+0xBBC6	0x9EC4	
+0xBBC7	0x78FA	
+0xBBC8	0x8757	
+0xBBC9	0x7C27	
+0xBBCA	0x7687	
+0xBBCB	0x51F0	
+0xBBCC	0x60F6	
+0xBBCD	0x714C	
+0xBBCE	0x6643	
+0xBBCF	0x5E4C	
+0xBBD0	0x604D	
+0xBBD1	0x8C0E	
+0xBBD2	0x7070	
+0xBBD3	0x6325	
+0xBBD4	0x8F89	
+0xBBD5	0x5FBD	
+0xBBD6	0x6062	
+0xBBD7	0x86D4	
+0xBBD8	0x56DE	
+0xBBD9	0x6BC1	
+0xBBDA	0x6094	
+0xBBDB	0x6167	
+0xBBDC	0x5349	
+0xBBDD	0x60E0	
+0xBBDE	0x6666	
+0xBBDF	0x8D3F	
+0xBBE0	0x79FD	
+0xBBE1	0x4F1A	
+0xBBE2	0x70E9	
+0xBBE3	0x6C47	
+0xBBE4	0x8BB3	
+0xBBE5	0x8BF2	
+0xBBE6	0x7ED8	
+0xBBE7	0x8364	
+0xBBE8	0x660F	
+0xBBE9	0x5A5A	
+0xBBEA	0x9B42	
+0xBBEB	0x6D51	
+0xBBEC	0x6DF7	
+0xBBED	0x8C41	
+0xBBEE	0x6D3B	
+0xBBEF	0x4F19	
+0xBBF0	0x706B	
+0xBBF1	0x83B7	
+0xBBF2	0x6216	
+0xBBF3	0x60D1	
+0xBBF4	0x970D	
+0xBBF5	0x8D27	
+0xBBF6	0x7978	
+0xBBF7	0x51FB	
+0xBBF8	0x573E	
+0xBBF9	0x57FA	
+0xBBFA	0x673A	
+0xBBFB	0x7578	
+0xBBFC	0x7A3D	
+0xBBFD	0x79EF	
+0xBBFE	0x7B95	
+0xBC40	0x7CBF	
+0xBC41	0x7CC0	
+0xBC42	0x7CC2	
+0xBC43	0x7CC3	
+0xBC44	0x7CC4	
+0xBC45	0x7CC6	
+0xBC46	0x7CC9	
+0xBC47	0x7CCB	
+0xBC48	0x7CCE	
+0xBC49	0x7CCF	
+0xBC4A	0x7CD0	
+0xBC4B	0x7CD1	
+0xBC4C	0x7CD2	
+0xBC4D	0x7CD3	
+0xBC4E	0x7CD4	
+0xBC4F	0x7CD8	
+0xBC50	0x7CDA	
+0xBC51	0x7CDB	
+0xBC52	0x7CDD	
+0xBC53	0x7CDE	
+0xBC54	0x7CE1	
+0xBC55	0x7CE2	
+0xBC56	0x7CE3	
+0xBC57	0x7CE4	
+0xBC58	0x7CE5	
+0xBC59	0x7CE6	
+0xBC5A	0x7CE7	
+0xBC5B	0x7CE9	
+0xBC5C	0x7CEA	
+0xBC5D	0x7CEB	
+0xBC5E	0x7CEC	
+0xBC5F	0x7CED	
+0xBC60	0x7CEE	
+0xBC61	0x7CF0	
+0xBC62	0x7CF1	
+0xBC63	0x7CF2	
+0xBC64	0x7CF3	
+0xBC65	0x7CF4	
+0xBC66	0x7CF5	
+0xBC67	0x7CF6	
+0xBC68	0x7CF7	
+0xBC69	0x7CF9	
+0xBC6A	0x7CFA	
+0xBC6B	0x7CFC	
+0xBC6C	0x7CFD	
+0xBC6D	0x7CFE	
+0xBC6E	0x7CFF	
+0xBC6F	0x7D00	
+0xBC70	0x7D01	
+0xBC71	0x7D02	
+0xBC72	0x7D03	
+0xBC73	0x7D04	
+0xBC74	0x7D05	
+0xBC75	0x7D06	
+0xBC76	0x7D07	
+0xBC77	0x7D08	
+0xBC78	0x7D09	
+0xBC79	0x7D0B	
+0xBC7A	0x7D0C	
+0xBC7B	0x7D0D	
+0xBC7C	0x7D0E	
+0xBC7D	0x7D0F	
+0xBC7E	0x7D10	
+0xBC80	0x7D11	
+0xBC81	0x7D12	
+0xBC82	0x7D13	
+0xBC83	0x7D14	
+0xBC84	0x7D15	
+0xBC85	0x7D16	
+0xBC86	0x7D17	
+0xBC87	0x7D18	
+0xBC88	0x7D19	
+0xBC89	0x7D1A	
+0xBC8A	0x7D1B	
+0xBC8B	0x7D1C	
+0xBC8C	0x7D1D	
+0xBC8D	0x7D1E	
+0xBC8E	0x7D1F	
+0xBC8F	0x7D21	
+0xBC90	0x7D23	
+0xBC91	0x7D24	
+0xBC92	0x7D25	
+0xBC93	0x7D26	
+0xBC94	0x7D28	
+0xBC95	0x7D29	
+0xBC96	0x7D2A	
+0xBC97	0x7D2C	
+0xBC98	0x7D2D	
+0xBC99	0x7D2E	
+0xBC9A	0x7D30	
+0xBC9B	0x7D31	
+0xBC9C	0x7D32	
+0xBC9D	0x7D33	
+0xBC9E	0x7D34	
+0xBC9F	0x7D35	
+0xBCA0	0x7D36	
+0xBCA1	0x808C	
+0xBCA2	0x9965	
+0xBCA3	0x8FF9	
+0xBCA4	0x6FC0	
+0xBCA5	0x8BA5	
+0xBCA6	0x9E21	
+0xBCA7	0x59EC	
+0xBCA8	0x7EE9	
+0xBCA9	0x7F09	
+0xBCAA	0x5409	
+0xBCAB	0x6781	
+0xBCAC	0x68D8	
+0xBCAD	0x8F91	
+0xBCAE	0x7C4D	
+0xBCAF	0x96C6	
+0xBCB0	0x53CA	
+0xBCB1	0x6025	
+0xBCB2	0x75BE	
+0xBCB3	0x6C72	
+0xBCB4	0x5373	
+0xBCB5	0x5AC9	
+0xBCB6	0x7EA7	
+0xBCB7	0x6324	
+0xBCB8	0x51E0	
+0xBCB9	0x810A	
+0xBCBA	0x5DF1	
+0xBCBB	0x84DF	
+0xBCBC	0x6280	
+0xBCBD	0x5180	
+0xBCBE	0x5B63	
+0xBCBF	0x4F0E	
+0xBCC0	0x796D	
+0xBCC1	0x5242	
+0xBCC2	0x60B8	
+0xBCC3	0x6D4E	
+0xBCC4	0x5BC4	
+0xBCC5	0x5BC2	
+0xBCC6	0x8BA1	
+0xBCC7	0x8BB0	
+0xBCC8	0x65E2	
+0xBCC9	0x5FCC	
+0xBCCA	0x9645	
+0xBCCB	0x5993	
+0xBCCC	0x7EE7	
+0xBCCD	0x7EAA	
+0xBCCE	0x5609	
+0xBCCF	0x67B7	
+0xBCD0	0x5939	
+0xBCD1	0x4F73	
+0xBCD2	0x5BB6	
+0xBCD3	0x52A0	
+0xBCD4	0x835A	
+0xBCD5	0x988A	
+0xBCD6	0x8D3E	
+0xBCD7	0x7532	
+0xBCD8	0x94BE	
+0xBCD9	0x5047	
+0xBCDA	0x7A3C	
+0xBCDB	0x4EF7	
+0xBCDC	0x67B6	
+0xBCDD	0x9A7E	
+0xBCDE	0x5AC1	
+0xBCDF	0x6B7C	
+0xBCE0	0x76D1	
+0xBCE1	0x575A	
+0xBCE2	0x5C16	
+0xBCE3	0x7B3A	
+0xBCE4	0x95F4	
+0xBCE5	0x714E	
+0xBCE6	0x517C	
+0xBCE7	0x80A9	
+0xBCE8	0x8270	
+0xBCE9	0x5978	
+0xBCEA	0x7F04	
+0xBCEB	0x8327	
+0xBCEC	0x68C0	
+0xBCED	0x67EC	
+0xBCEE	0x78B1	
+0xBCEF	0x7877	
+0xBCF0	0x62E3	
+0xBCF1	0x6361	
+0xBCF2	0x7B80	
+0xBCF3	0x4FED	
+0xBCF4	0x526A	
+0xBCF5	0x51CF	
+0xBCF6	0x8350	
+0xBCF7	0x69DB	
+0xBCF8	0x9274	
+0xBCF9	0x8DF5	
+0xBCFA	0x8D31	
+0xBCFB	0x89C1	
+0xBCFC	0x952E	
+0xBCFD	0x7BAD	
+0xBCFE	0x4EF6	
+0xBD40	0x7D37	
+0xBD41	0x7D38	
+0xBD42	0x7D39	
+0xBD43	0x7D3A	
+0xBD44	0x7D3B	
+0xBD45	0x7D3C	
+0xBD46	0x7D3D	
+0xBD47	0x7D3E	
+0xBD48	0x7D3F	
+0xBD49	0x7D40	
+0xBD4A	0x7D41	
+0xBD4B	0x7D42	
+0xBD4C	0x7D43	
+0xBD4D	0x7D44	
+0xBD4E	0x7D45	
+0xBD4F	0x7D46	
+0xBD50	0x7D47	
+0xBD51	0x7D48	
+0xBD52	0x7D49	
+0xBD53	0x7D4A	
+0xBD54	0x7D4B	
+0xBD55	0x7D4C	
+0xBD56	0x7D4D	
+0xBD57	0x7D4E	
+0xBD58	0x7D4F	
+0xBD59	0x7D50	
+0xBD5A	0x7D51	
+0xBD5B	0x7D52	
+0xBD5C	0x7D53	
+0xBD5D	0x7D54	
+0xBD5E	0x7D55	
+0xBD5F	0x7D56	
+0xBD60	0x7D57	
+0xBD61	0x7D58	
+0xBD62	0x7D59	
+0xBD63	0x7D5A	
+0xBD64	0x7D5B	
+0xBD65	0x7D5C	
+0xBD66	0x7D5D	
+0xBD67	0x7D5E	
+0xBD68	0x7D5F	
+0xBD69	0x7D60	
+0xBD6A	0x7D61	
+0xBD6B	0x7D62	
+0xBD6C	0x7D63	
+0xBD6D	0x7D64	
+0xBD6E	0x7D65	
+0xBD6F	0x7D66	
+0xBD70	0x7D67	
+0xBD71	0x7D68	
+0xBD72	0x7D69	
+0xBD73	0x7D6A	
+0xBD74	0x7D6B	
+0xBD75	0x7D6C	
+0xBD76	0x7D6D	
+0xBD77	0x7D6F	
+0xBD78	0x7D70	
+0xBD79	0x7D71	
+0xBD7A	0x7D72	
+0xBD7B	0x7D73	
+0xBD7C	0x7D74	
+0xBD7D	0x7D75	
+0xBD7E	0x7D76	
+0xBD80	0x7D78	
+0xBD81	0x7D79	
+0xBD82	0x7D7A	
+0xBD83	0x7D7B	
+0xBD84	0x7D7C	
+0xBD85	0x7D7D	
+0xBD86	0x7D7E	
+0xBD87	0x7D7F	
+0xBD88	0x7D80	
+0xBD89	0x7D81	
+0xBD8A	0x7D82	
+0xBD8B	0x7D83	
+0xBD8C	0x7D84	
+0xBD8D	0x7D85	
+0xBD8E	0x7D86	
+0xBD8F	0x7D87	
+0xBD90	0x7D88	
+0xBD91	0x7D89	
+0xBD92	0x7D8A	
+0xBD93	0x7D8B	
+0xBD94	0x7D8C	
+0xBD95	0x7D8D	
+0xBD96	0x7D8E	
+0xBD97	0x7D8F	
+0xBD98	0x7D90	
+0xBD99	0x7D91	
+0xBD9A	0x7D92	
+0xBD9B	0x7D93	
+0xBD9C	0x7D94	
+0xBD9D	0x7D95	
+0xBD9E	0x7D96	
+0xBD9F	0x7D97	
+0xBDA0	0x7D98	
+0xBDA1	0x5065	
+0xBDA2	0x8230	
+0xBDA3	0x5251	
+0xBDA4	0x996F	
+0xBDA5	0x6E10	
+0xBDA6	0x6E85	
+0xBDA7	0x6DA7	
+0xBDA8	0x5EFA	
+0xBDA9	0x50F5	
+0xBDAA	0x59DC	
+0xBDAB	0x5C06	
+0xBDAC	0x6D46	
+0xBDAD	0x6C5F	
+0xBDAE	0x7586	
+0xBDAF	0x848B	
+0xBDB0	0x6868	
+0xBDB1	0x5956	
+0xBDB2	0x8BB2	
+0xBDB3	0x5320	
+0xBDB4	0x9171	
+0xBDB5	0x964D	
+0xBDB6	0x8549	
+0xBDB7	0x6912	
+0xBDB8	0x7901	
+0xBDB9	0x7126	
+0xBDBA	0x80F6	
+0xBDBB	0x4EA4	
+0xBDBC	0x90CA	
+0xBDBD	0x6D47	
+0xBDBE	0x9A84	
+0xBDBF	0x5A07	
+0xBDC0	0x56BC	
+0xBDC1	0x6405	
+0xBDC2	0x94F0	
+0xBDC3	0x77EB	
+0xBDC4	0x4FA5	
+0xBDC5	0x811A	
+0xBDC6	0x72E1	
+0xBDC7	0x89D2	
+0xBDC8	0x997A	
+0xBDC9	0x7F34	
+0xBDCA	0x7EDE	
+0xBDCB	0x527F	
+0xBDCC	0x6559	
+0xBDCD	0x9175	
+0xBDCE	0x8F7F	
+0xBDCF	0x8F83	
+0xBDD0	0x53EB	
+0xBDD1	0x7A96	
+0xBDD2	0x63ED	
+0xBDD3	0x63A5	
+0xBDD4	0x7686	
+0xBDD5	0x79F8	
+0xBDD6	0x8857	
+0xBDD7	0x9636	
+0xBDD8	0x622A	
+0xBDD9	0x52AB	
+0xBDDA	0x8282	
+0xBDDB	0x6854	
+0xBDDC	0x6770	
+0xBDDD	0x6377	
+0xBDDE	0x776B	
+0xBDDF	0x7AED	
+0xBDE0	0x6D01	
+0xBDE1	0x7ED3	
+0xBDE2	0x89E3	
+0xBDE3	0x59D0	
+0xBDE4	0x6212	
+0xBDE5	0x85C9	
+0xBDE6	0x82A5	
+0xBDE7	0x754C	
+0xBDE8	0x501F	
+0xBDE9	0x4ECB	
+0xBDEA	0x75A5	
+0xBDEB	0x8BEB	
+0xBDEC	0x5C4A	
+0xBDED	0x5DFE	
+0xBDEE	0x7B4B	
+0xBDEF	0x65A4	
+0xBDF0	0x91D1	
+0xBDF1	0x4ECA	
+0xBDF2	0x6D25	
+0xBDF3	0x895F	
+0xBDF4	0x7D27	
+0xBDF5	0x9526	
+0xBDF6	0x4EC5	
+0xBDF7	0x8C28	
+0xBDF8	0x8FDB	
+0xBDF9	0x9773	
+0xBDFA	0x664B	
+0xBDFB	0x7981	
+0xBDFC	0x8FD1	
+0xBDFD	0x70EC	
+0xBDFE	0x6D78	
+0xBE40	0x7D99	
+0xBE41	0x7D9A	
+0xBE42	0x7D9B	
+0xBE43	0x7D9C	
+0xBE44	0x7D9D	
+0xBE45	0x7D9E	
+0xBE46	0x7D9F	
+0xBE47	0x7DA0	
+0xBE48	0x7DA1	
+0xBE49	0x7DA2	
+0xBE4A	0x7DA3	
+0xBE4B	0x7DA4	
+0xBE4C	0x7DA5	
+0xBE4D	0x7DA7	
+0xBE4E	0x7DA8	
+0xBE4F	0x7DA9	
+0xBE50	0x7DAA	
+0xBE51	0x7DAB	
+0xBE52	0x7DAC	
+0xBE53	0x7DAD	
+0xBE54	0x7DAF	
+0xBE55	0x7DB0	
+0xBE56	0x7DB1	
+0xBE57	0x7DB2	
+0xBE58	0x7DB3	
+0xBE59	0x7DB4	
+0xBE5A	0x7DB5	
+0xBE5B	0x7DB6	
+0xBE5C	0x7DB7	
+0xBE5D	0x7DB8	
+0xBE5E	0x7DB9	
+0xBE5F	0x7DBA	
+0xBE60	0x7DBB	
+0xBE61	0x7DBC	
+0xBE62	0x7DBD	
+0xBE63	0x7DBE	
+0xBE64	0x7DBF	
+0xBE65	0x7DC0	
+0xBE66	0x7DC1	
+0xBE67	0x7DC2	
+0xBE68	0x7DC3	
+0xBE69	0x7DC4	
+0xBE6A	0x7DC5	
+0xBE6B	0x7DC6	
+0xBE6C	0x7DC7	
+0xBE6D	0x7DC8	
+0xBE6E	0x7DC9	
+0xBE6F	0x7DCA	
+0xBE70	0x7DCB	
+0xBE71	0x7DCC	
+0xBE72	0x7DCD	
+0xBE73	0x7DCE	
+0xBE74	0x7DCF	
+0xBE75	0x7DD0	
+0xBE76	0x7DD1	
+0xBE77	0x7DD2	
+0xBE78	0x7DD3	
+0xBE79	0x7DD4	
+0xBE7A	0x7DD5	
+0xBE7B	0x7DD6	
+0xBE7C	0x7DD7	
+0xBE7D	0x7DD8	
+0xBE7E	0x7DD9	
+0xBE80	0x7DDA	
+0xBE81	0x7DDB	
+0xBE82	0x7DDC	
+0xBE83	0x7DDD	
+0xBE84	0x7DDE	
+0xBE85	0x7DDF	
+0xBE86	0x7DE0	
+0xBE87	0x7DE1	
+0xBE88	0x7DE2	
+0xBE89	0x7DE3	
+0xBE8A	0x7DE4	
+0xBE8B	0x7DE5	
+0xBE8C	0x7DE6	
+0xBE8D	0x7DE7	
+0xBE8E	0x7DE8	
+0xBE8F	0x7DE9	
+0xBE90	0x7DEA	
+0xBE91	0x7DEB	
+0xBE92	0x7DEC	
+0xBE93	0x7DED	
+0xBE94	0x7DEE	
+0xBE95	0x7DEF	
+0xBE96	0x7DF0	
+0xBE97	0x7DF1	
+0xBE98	0x7DF2	
+0xBE99	0x7DF3	
+0xBE9A	0x7DF4	
+0xBE9B	0x7DF5	
+0xBE9C	0x7DF6	
+0xBE9D	0x7DF7	
+0xBE9E	0x7DF8	
+0xBE9F	0x7DF9	
+0xBEA0	0x7DFA	
+0xBEA1	0x5C3D	
+0xBEA2	0x52B2	
+0xBEA3	0x8346	
+0xBEA4	0x5162	
+0xBEA5	0x830E	
+0xBEA6	0x775B	
+0xBEA7	0x6676	
+0xBEA8	0x9CB8	
+0xBEA9	0x4EAC	
+0xBEAA	0x60CA	
+0xBEAB	0x7CBE	
+0xBEAC	0x7CB3	
+0xBEAD	0x7ECF	
+0xBEAE	0x4E95	
+0xBEAF	0x8B66	
+0xBEB0	0x666F	
+0xBEB1	0x9888	
+0xBEB2	0x9759	
+0xBEB3	0x5883	
+0xBEB4	0x656C	
+0xBEB5	0x955C	
+0xBEB6	0x5F84	
+0xBEB7	0x75C9	
+0xBEB8	0x9756	
+0xBEB9	0x7ADF	
+0xBEBA	0x7ADE	
+0xBEBB	0x51C0	
+0xBEBC	0x70AF	
+0xBEBD	0x7A98	
+0xBEBE	0x63EA	
+0xBEBF	0x7A76	
+0xBEC0	0x7EA0	
+0xBEC1	0x7396	
+0xBEC2	0x97ED	
+0xBEC3	0x4E45	
+0xBEC4	0x7078	
+0xBEC5	0x4E5D	
+0xBEC6	0x9152	
+0xBEC7	0x53A9	
+0xBEC8	0x6551	
+0xBEC9	0x65E7	
+0xBECA	0x81FC	
+0xBECB	0x8205	
+0xBECC	0x548E	
+0xBECD	0x5C31	
+0xBECE	0x759A	
+0xBECF	0x97A0	
+0xBED0	0x62D8	
+0xBED1	0x72D9	
+0xBED2	0x75BD	
+0xBED3	0x5C45	
+0xBED4	0x9A79	
+0xBED5	0x83CA	
+0xBED6	0x5C40	
+0xBED7	0x5480	
+0xBED8	0x77E9	
+0xBED9	0x4E3E	
+0xBEDA	0x6CAE	
+0xBEDB	0x805A	
+0xBEDC	0x62D2	
+0xBEDD	0x636E	
+0xBEDE	0x5DE8	
+0xBEDF	0x5177	
+0xBEE0	0x8DDD	
+0xBEE1	0x8E1E	
+0xBEE2	0x952F	
+0xBEE3	0x4FF1	
+0xBEE4	0x53E5	
+0xBEE5	0x60E7	
+0xBEE6	0x70AC	
+0xBEE7	0x5267	
+0xBEE8	0x6350	
+0xBEE9	0x9E43	
+0xBEEA	0x5A1F	
+0xBEEB	0x5026	
+0xBEEC	0x7737	
+0xBEED	0x5377	
+0xBEEE	0x7EE2	
+0xBEEF	0x6485	
+0xBEF0	0x652B	
+0xBEF1	0x6289	
+0xBEF2	0x6398	
+0xBEF3	0x5014	
+0xBEF4	0x7235	
+0xBEF5	0x89C9	
+0xBEF6	0x51B3	
+0xBEF7	0x8BC0	
+0xBEF8	0x7EDD	
+0xBEF9	0x5747	
+0xBEFA	0x83CC	
+0xBEFB	0x94A7	
+0xBEFC	0x519B	
+0xBEFD	0x541B	
+0xBEFE	0x5CFB	
+0xBF40	0x7DFB	
+0xBF41	0x7DFC	
+0xBF42	0x7DFD	
+0xBF43	0x7DFE	
+0xBF44	0x7DFF	
+0xBF45	0x7E00	
+0xBF46	0x7E01	
+0xBF47	0x7E02	
+0xBF48	0x7E03	
+0xBF49	0x7E04	
+0xBF4A	0x7E05	
+0xBF4B	0x7E06	
+0xBF4C	0x7E07	
+0xBF4D	0x7E08	
+0xBF4E	0x7E09	
+0xBF4F	0x7E0A	
+0xBF50	0x7E0B	
+0xBF51	0x7E0C	
+0xBF52	0x7E0D	
+0xBF53	0x7E0E	
+0xBF54	0x7E0F	
+0xBF55	0x7E10	
+0xBF56	0x7E11	
+0xBF57	0x7E12	
+0xBF58	0x7E13	
+0xBF59	0x7E14	
+0xBF5A	0x7E15	
+0xBF5B	0x7E16	
+0xBF5C	0x7E17	
+0xBF5D	0x7E18	
+0xBF5E	0x7E19	
+0xBF5F	0x7E1A	
+0xBF60	0x7E1B	
+0xBF61	0x7E1C	
+0xBF62	0x7E1D	
+0xBF63	0x7E1E	
+0xBF64	0x7E1F	
+0xBF65	0x7E20	
+0xBF66	0x7E21	
+0xBF67	0x7E22	
+0xBF68	0x7E23	
+0xBF69	0x7E24	
+0xBF6A	0x7E25	
+0xBF6B	0x7E26	
+0xBF6C	0x7E27	
+0xBF6D	0x7E28	
+0xBF6E	0x7E29	
+0xBF6F	0x7E2A	
+0xBF70	0x7E2B	
+0xBF71	0x7E2C	
+0xBF72	0x7E2D	
+0xBF73	0x7E2E	
+0xBF74	0x7E2F	
+0xBF75	0x7E30	
+0xBF76	0x7E31	
+0xBF77	0x7E32	
+0xBF78	0x7E33	
+0xBF79	0x7E34	
+0xBF7A	0x7E35	
+0xBF7B	0x7E36	
+0xBF7C	0x7E37	
+0xBF7D	0x7E38	
+0xBF7E	0x7E39	
+0xBF80	0x7E3A	
+0xBF81	0x7E3C	
+0xBF82	0x7E3D	
+0xBF83	0x7E3E	
+0xBF84	0x7E3F	
+0xBF85	0x7E40	
+0xBF86	0x7E42	
+0xBF87	0x7E43	
+0xBF88	0x7E44	
+0xBF89	0x7E45	
+0xBF8A	0x7E46	
+0xBF8B	0x7E48	
+0xBF8C	0x7E49	
+0xBF8D	0x7E4A	
+0xBF8E	0x7E4B	
+0xBF8F	0x7E4C	
+0xBF90	0x7E4D	
+0xBF91	0x7E4E	
+0xBF92	0x7E4F	
+0xBF93	0x7E50	
+0xBF94	0x7E51	
+0xBF95	0x7E52	
+0xBF96	0x7E53	
+0xBF97	0x7E54	
+0xBF98	0x7E55	
+0xBF99	0x7E56	
+0xBF9A	0x7E57	
+0xBF9B	0x7E58	
+0xBF9C	0x7E59	
+0xBF9D	0x7E5A	
+0xBF9E	0x7E5B	
+0xBF9F	0x7E5C	
+0xBFA0	0x7E5D	
+0xBFA1	0x4FCA	
+0xBFA2	0x7AE3	
+0xBFA3	0x6D5A	
+0xBFA4	0x90E1	
+0xBFA5	0x9A8F	
+0xBFA6	0x5580	
+0xBFA7	0x5496	
+0xBFA8	0x5361	
+0xBFA9	0x54AF	
+0xBFAA	0x5F00	
+0xBFAB	0x63E9	
+0xBFAC	0x6977	
+0xBFAD	0x51EF	
+0xBFAE	0x6168	
+0xBFAF	0x520A	
+0xBFB0	0x582A	
+0xBFB1	0x52D8	
+0xBFB2	0x574E	
+0xBFB3	0x780D	
+0xBFB4	0x770B	
+0xBFB5	0x5EB7	
+0xBFB6	0x6177	
+0xBFB7	0x7CE0	
+0xBFB8	0x625B	
+0xBFB9	0x6297	
+0xBFBA	0x4EA2	
+0xBFBB	0x7095	
+0xBFBC	0x8003	
+0xBFBD	0x62F7	
+0xBFBE	0x70E4	
+0xBFBF	0x9760	
+0xBFC0	0x5777	
+0xBFC1	0x82DB	
+0xBFC2	0x67EF	
+0xBFC3	0x68F5	
+0xBFC4	0x78D5	
+0xBFC5	0x9897	
+0xBFC6	0x79D1	
+0xBFC7	0x58F3	
+0xBFC8	0x54B3	
+0xBFC9	0x53EF	
+0xBFCA	0x6E34	
+0xBFCB	0x514B	
+0xBFCC	0x523B	
+0xBFCD	0x5BA2	
+0xBFCE	0x8BFE	
+0xBFCF	0x80AF	
+0xBFD0	0x5543	
+0xBFD1	0x57A6	
+0xBFD2	0x6073	
+0xBFD3	0x5751	
+0xBFD4	0x542D	
+0xBFD5	0x7A7A	
+0xBFD6	0x6050	
+0xBFD7	0x5B54	
+0xBFD8	0x63A7	
+0xBFD9	0x62A0	
+0xBFDA	0x53E3	
+0xBFDB	0x6263	
+0xBFDC	0x5BC7	
+0xBFDD	0x67AF	
+0xBFDE	0x54ED	
+0xBFDF	0x7A9F	
+0xBFE0	0x82E6	
+0xBFE1	0x9177	
+0xBFE2	0x5E93	
+0xBFE3	0x88E4	
+0xBFE4	0x5938	
+0xBFE5	0x57AE	
+0xBFE6	0x630E	
+0xBFE7	0x8DE8	
+0xBFE8	0x80EF	
+0xBFE9	0x5757	
+0xBFEA	0x7B77	
+0xBFEB	0x4FA9	
+0xBFEC	0x5FEB	
+0xBFED	0x5BBD	
+0xBFEE	0x6B3E	
+0xBFEF	0x5321	
+0xBFF0	0x7B50	
+0xBFF1	0x72C2	
+0xBFF2	0x6846	
+0xBFF3	0x77FF	
+0xBFF4	0x7736	
+0xBFF5	0x65F7	
+0xBFF6	0x51B5	
+0xBFF7	0x4E8F	
+0xBFF8	0x76D4	
+0xBFF9	0x5CBF	
+0xBFFA	0x7AA5	
+0xBFFB	0x8475	
+0xBFFC	0x594E	
+0xBFFD	0x9B41	
+0xBFFE	0x5080	
+0xC040	0x7E5E	
+0xC041	0x7E5F	
+0xC042	0x7E60	
+0xC043	0x7E61	
+0xC044	0x7E62	
+0xC045	0x7E63	
+0xC046	0x7E64	
+0xC047	0x7E65	
+0xC048	0x7E66	
+0xC049	0x7E67	
+0xC04A	0x7E68	
+0xC04B	0x7E69	
+0xC04C	0x7E6A	
+0xC04D	0x7E6B	
+0xC04E	0x7E6C	
+0xC04F	0x7E6D	
+0xC050	0x7E6E	
+0xC051	0x7E6F	
+0xC052	0x7E70	
+0xC053	0x7E71	
+0xC054	0x7E72	
+0xC055	0x7E73	
+0xC056	0x7E74	
+0xC057	0x7E75	
+0xC058	0x7E76	
+0xC059	0x7E77	
+0xC05A	0x7E78	
+0xC05B	0x7E79	
+0xC05C	0x7E7A	
+0xC05D	0x7E7B	
+0xC05E	0x7E7C	
+0xC05F	0x7E7D	
+0xC060	0x7E7E	
+0xC061	0x7E7F	
+0xC062	0x7E80	
+0xC063	0x7E81	
+0xC064	0x7E83	
+0xC065	0x7E84	
+0xC066	0x7E85	
+0xC067	0x7E86	
+0xC068	0x7E87	
+0xC069	0x7E88	
+0xC06A	0x7E89	
+0xC06B	0x7E8A	
+0xC06C	0x7E8B	
+0xC06D	0x7E8C	
+0xC06E	0x7E8D	
+0xC06F	0x7E8E	
+0xC070	0x7E8F	
+0xC071	0x7E90	
+0xC072	0x7E91	
+0xC073	0x7E92	
+0xC074	0x7E93	
+0xC075	0x7E94	
+0xC076	0x7E95	
+0xC077	0x7E96	
+0xC078	0x7E97	
+0xC079	0x7E98	
+0xC07A	0x7E99	
+0xC07B	0x7E9A	
+0xC07C	0x7E9C	
+0xC07D	0x7E9D	
+0xC07E	0x7E9E	
+0xC080	0x7EAE	
+0xC081	0x7EB4	
+0xC082	0x7EBB	
+0xC083	0x7EBC	
+0xC084	0x7ED6	
+0xC085	0x7EE4	
+0xC086	0x7EEC	
+0xC087	0x7EF9	
+0xC088	0x7F0A	
+0xC089	0x7F10	
+0xC08A	0x7F1E	
+0xC08B	0x7F37	
+0xC08C	0x7F39	
+0xC08D	0x7F3B	
+0xC08E	0x7F3C	
+0xC08F	0x7F3D	
+0xC090	0x7F3E	
+0xC091	0x7F3F	
+0xC092	0x7F40	
+0xC093	0x7F41	
+0xC094	0x7F43	
+0xC095	0x7F46	
+0xC096	0x7F47	
+0xC097	0x7F48	
+0xC098	0x7F49	
+0xC099	0x7F4A	
+0xC09A	0x7F4B	
+0xC09B	0x7F4C	
+0xC09C	0x7F4D	
+0xC09D	0x7F4E	
+0xC09E	0x7F4F	
+0xC09F	0x7F52	
+0xC0A0	0x7F53	
+0xC0A1	0x9988	
+0xC0A2	0x6127	
+0xC0A3	0x6E83	
+0xC0A4	0x5764	
+0xC0A5	0x6606	
+0xC0A6	0x6346	
+0xC0A7	0x56F0	
+0xC0A8	0x62EC	
+0xC0A9	0x6269	
+0xC0AA	0x5ED3	
+0xC0AB	0x9614	
+0xC0AC	0x5783	
+0xC0AD	0x62C9	
+0xC0AE	0x5587	
+0xC0AF	0x8721	
+0xC0B0	0x814A	
+0xC0B1	0x8FA3	
+0xC0B2	0x5566	
+0xC0B3	0x83B1	
+0xC0B4	0x6765	
+0xC0B5	0x8D56	
+0xC0B6	0x84DD	
+0xC0B7	0x5A6A	
+0xC0B8	0x680F	
+0xC0B9	0x62E6	
+0xC0BA	0x7BEE	
+0xC0BB	0x9611	
+0xC0BC	0x5170	
+0xC0BD	0x6F9C	
+0xC0BE	0x8C30	
+0xC0BF	0x63FD	
+0xC0C0	0x89C8	
+0xC0C1	0x61D2	
+0xC0C2	0x7F06	
+0xC0C3	0x70C2	
+0xC0C4	0x6EE5	
+0xC0C5	0x7405	
+0xC0C6	0x6994	
+0xC0C7	0x72FC	
+0xC0C8	0x5ECA	
+0xC0C9	0x90CE	
+0xC0CA	0x6717	
+0xC0CB	0x6D6A	
+0xC0CC	0x635E	
+0xC0CD	0x52B3	
+0xC0CE	0x7262	
+0xC0CF	0x8001	
+0xC0D0	0x4F6C	
+0xC0D1	0x59E5	
+0xC0D2	0x916A	
+0xC0D3	0x70D9	
+0xC0D4	0x6D9D	
+0xC0D5	0x52D2	
+0xC0D6	0x4E50	
+0xC0D7	0x96F7	
+0xC0D8	0x956D	
+0xC0D9	0x857E	
+0xC0DA	0x78CA	
+0xC0DB	0x7D2F	
+0xC0DC	0x5121	
+0xC0DD	0x5792	
+0xC0DE	0x64C2	
+0xC0DF	0x808B	
+0xC0E0	0x7C7B	
+0xC0E1	0x6CEA	
+0xC0E2	0x68F1	
+0xC0E3	0x695E	
+0xC0E4	0x51B7	
+0xC0E5	0x5398	
+0xC0E6	0x68A8	
+0xC0E7	0x7281	
+0xC0E8	0x9ECE	
+0xC0E9	0x7BF1	
+0xC0EA	0x72F8	
+0xC0EB	0x79BB	
+0xC0EC	0x6F13	
+0xC0ED	0x7406	
+0xC0EE	0x674E	
+0xC0EF	0x91CC	
+0xC0F0	0x9CA4	
+0xC0F1	0x793C	
+0xC0F2	0x8389	
+0xC0F3	0x8354	
+0xC0F4	0x540F	
+0xC0F5	0x6817	
+0xC0F6	0x4E3D	
+0xC0F7	0x5389	
+0xC0F8	0x52B1	
+0xC0F9	0x783E	
+0xC0FA	0x5386	
+0xC0FB	0x5229	
+0xC0FC	0x5088	
+0xC0FD	0x4F8B	
+0xC0FE	0x4FD0	
+0xC140	0x7F56	
+0xC141	0x7F59	
+0xC142	0x7F5B	
+0xC143	0x7F5C	
+0xC144	0x7F5D	
+0xC145	0x7F5E	
+0xC146	0x7F60	
+0xC147	0x7F63	
+0xC148	0x7F64	
+0xC149	0x7F65	
+0xC14A	0x7F66	
+0xC14B	0x7F67	
+0xC14C	0x7F6B	
+0xC14D	0x7F6C	
+0xC14E	0x7F6D	
+0xC14F	0x7F6F	
+0xC150	0x7F70	
+0xC151	0x7F73	
+0xC152	0x7F75	
+0xC153	0x7F76	
+0xC154	0x7F77	
+0xC155	0x7F78	
+0xC156	0x7F7A	
+0xC157	0x7F7B	
+0xC158	0x7F7C	
+0xC159	0x7F7D	
+0xC15A	0x7F7F	
+0xC15B	0x7F80	
+0xC15C	0x7F82	
+0xC15D	0x7F83	
+0xC15E	0x7F84	
+0xC15F	0x7F85	
+0xC160	0x7F86	
+0xC161	0x7F87	
+0xC162	0x7F88	
+0xC163	0x7F89	
+0xC164	0x7F8B	
+0xC165	0x7F8D	
+0xC166	0x7F8F	
+0xC167	0x7F90	
+0xC168	0x7F91	
+0xC169	0x7F92	
+0xC16A	0x7F93	
+0xC16B	0x7F95	
+0xC16C	0x7F96	
+0xC16D	0x7F97	
+0xC16E	0x7F98	
+0xC16F	0x7F99	
+0xC170	0x7F9B	
+0xC171	0x7F9C	
+0xC172	0x7FA0	
+0xC173	0x7FA2	
+0xC174	0x7FA3	
+0xC175	0x7FA5	
+0xC176	0x7FA6	
+0xC177	0x7FA8	
+0xC178	0x7FA9	
+0xC179	0x7FAA	
+0xC17A	0x7FAB	
+0xC17B	0x7FAC	
+0xC17C	0x7FAD	
+0xC17D	0x7FAE	
+0xC17E	0x7FB1	
+0xC180	0x7FB3	
+0xC181	0x7FB4	
+0xC182	0x7FB5	
+0xC183	0x7FB6	
+0xC184	0x7FB7	
+0xC185	0x7FBA	
+0xC186	0x7FBB	
+0xC187	0x7FBE	
+0xC188	0x7FC0	
+0xC189	0x7FC2	
+0xC18A	0x7FC3	
+0xC18B	0x7FC4	
+0xC18C	0x7FC6	
+0xC18D	0x7FC7	
+0xC18E	0x7FC8	
+0xC18F	0x7FC9	
+0xC190	0x7FCB	
+0xC191	0x7FCD	
+0xC192	0x7FCF	
+0xC193	0x7FD0	
+0xC194	0x7FD1	
+0xC195	0x7FD2	
+0xC196	0x7FD3	
+0xC197	0x7FD6	
+0xC198	0x7FD7	
+0xC199	0x7FD9	
+0xC19A	0x7FDA	
+0xC19B	0x7FDB	
+0xC19C	0x7FDC	
+0xC19D	0x7FDD	
+0xC19E	0x7FDE	
+0xC19F	0x7FE2	
+0xC1A0	0x7FE3	
+0xC1A1	0x75E2	
+0xC1A2	0x7ACB	
+0xC1A3	0x7C92	
+0xC1A4	0x6CA5	
+0xC1A5	0x96B6	
+0xC1A6	0x529B	
+0xC1A7	0x7483	
+0xC1A8	0x54E9	
+0xC1A9	0x4FE9	
+0xC1AA	0x8054	
+0xC1AB	0x83B2	
+0xC1AC	0x8FDE	
+0xC1AD	0x9570	
+0xC1AE	0x5EC9	
+0xC1AF	0x601C	
+0xC1B0	0x6D9F	
+0xC1B1	0x5E18	
+0xC1B2	0x655B	
+0xC1B3	0x8138	
+0xC1B4	0x94FE	
+0xC1B5	0x604B	
+0xC1B6	0x70BC	
+0xC1B7	0x7EC3	
+0xC1B8	0x7CAE	
+0xC1B9	0x51C9	
+0xC1BA	0x6881	
+0xC1BB	0x7CB1	
+0xC1BC	0x826F	
+0xC1BD	0x4E24	
+0xC1BE	0x8F86	
+0xC1BF	0x91CF	
+0xC1C0	0x667E	
+0xC1C1	0x4EAE	
+0xC1C2	0x8C05	
+0xC1C3	0x64A9	
+0xC1C4	0x804A	
+0xC1C5	0x50DA	
+0xC1C6	0x7597	
+0xC1C7	0x71CE	
+0xC1C8	0x5BE5	
+0xC1C9	0x8FBD	
+0xC1CA	0x6F66	
+0xC1CB	0x4E86	
+0xC1CC	0x6482	
+0xC1CD	0x9563	
+0xC1CE	0x5ED6	
+0xC1CF	0x6599	
+0xC1D0	0x5217	
+0xC1D1	0x88C2	
+0xC1D2	0x70C8	
+0xC1D3	0x52A3	
+0xC1D4	0x730E	
+0xC1D5	0x7433	
+0xC1D6	0x6797	
+0xC1D7	0x78F7	
+0xC1D8	0x9716	
+0xC1D9	0x4E34	
+0xC1DA	0x90BB	
+0xC1DB	0x9CDE	
+0xC1DC	0x6DCB	
+0xC1DD	0x51DB	
+0xC1DE	0x8D41	
+0xC1DF	0x541D	
+0xC1E0	0x62CE	
+0xC1E1	0x73B2	
+0xC1E2	0x83F1	
+0xC1E3	0x96F6	
+0xC1E4	0x9F84	
+0xC1E5	0x94C3	
+0xC1E6	0x4F36	
+0xC1E7	0x7F9A	
+0xC1E8	0x51CC	
+0xC1E9	0x7075	
+0xC1EA	0x9675	
+0xC1EB	0x5CAD	
+0xC1EC	0x9886	
+0xC1ED	0x53E6	
+0xC1EE	0x4EE4	
+0xC1EF	0x6E9C	
+0xC1F0	0x7409	
+0xC1F1	0x69B4	
+0xC1F2	0x786B	
+0xC1F3	0x998F	
+0xC1F4	0x7559	
+0xC1F5	0x5218	
+0xC1F6	0x7624	
+0xC1F7	0x6D41	
+0xC1F8	0x67F3	
+0xC1F9	0x516D	
+0xC1FA	0x9F99	
+0xC1FB	0x804B	
+0xC1FC	0x5499	
+0xC1FD	0x7B3C	
+0xC1FE	0x7ABF	
+0xC240	0x7FE4	
+0xC241	0x7FE7	
+0xC242	0x7FE8	
+0xC243	0x7FEA	
+0xC244	0x7FEB	
+0xC245	0x7FEC	
+0xC246	0x7FED	
+0xC247	0x7FEF	
+0xC248	0x7FF2	
+0xC249	0x7FF4	
+0xC24A	0x7FF5	
+0xC24B	0x7FF6	
+0xC24C	0x7FF7	
+0xC24D	0x7FF8	
+0xC24E	0x7FF9	
+0xC24F	0x7FFA	
+0xC250	0x7FFD	
+0xC251	0x7FFE	
+0xC252	0x7FFF	
+0xC253	0x8002	
+0xC254	0x8007	
+0xC255	0x8008	
+0xC256	0x8009	
+0xC257	0x800A	
+0xC258	0x800E	
+0xC259	0x800F	
+0xC25A	0x8011	
+0xC25B	0x8013	
+0xC25C	0x801A	
+0xC25D	0x801B	
+0xC25E	0x801D	
+0xC25F	0x801E	
+0xC260	0x801F	
+0xC261	0x8021	
+0xC262	0x8023	
+0xC263	0x8024	
+0xC264	0x802B	
+0xC265	0x802C	
+0xC266	0x802D	
+0xC267	0x802E	
+0xC268	0x802F	
+0xC269	0x8030	
+0xC26A	0x8032	
+0xC26B	0x8034	
+0xC26C	0x8039	
+0xC26D	0x803A	
+0xC26E	0x803C	
+0xC26F	0x803E	
+0xC270	0x8040	
+0xC271	0x8041	
+0xC272	0x8044	
+0xC273	0x8045	
+0xC274	0x8047	
+0xC275	0x8048	
+0xC276	0x8049	
+0xC277	0x804E	
+0xC278	0x804F	
+0xC279	0x8050	
+0xC27A	0x8051	
+0xC27B	0x8053	
+0xC27C	0x8055	
+0xC27D	0x8056	
+0xC27E	0x8057	
+0xC280	0x8059	
+0xC281	0x805B	
+0xC282	0x805C	
+0xC283	0x805D	
+0xC284	0x805E	
+0xC285	0x805F	
+0xC286	0x8060	
+0xC287	0x8061	
+0xC288	0x8062	
+0xC289	0x8063	
+0xC28A	0x8064	
+0xC28B	0x8065	
+0xC28C	0x8066	
+0xC28D	0x8067	
+0xC28E	0x8068	
+0xC28F	0x806B	
+0xC290	0x806C	
+0xC291	0x806D	
+0xC292	0x806E	
+0xC293	0x806F	
+0xC294	0x8070	
+0xC295	0x8072	
+0xC296	0x8073	
+0xC297	0x8074	
+0xC298	0x8075	
+0xC299	0x8076	
+0xC29A	0x8077	
+0xC29B	0x8078	
+0xC29C	0x8079	
+0xC29D	0x807A	
+0xC29E	0x807B	
+0xC29F	0x807C	
+0xC2A0	0x807D	
+0xC2A1	0x9686	
+0xC2A2	0x5784	
+0xC2A3	0x62E2	
+0xC2A4	0x9647	
+0xC2A5	0x697C	
+0xC2A6	0x5A04	
+0xC2A7	0x6402	
+0xC2A8	0x7BD3	
+0xC2A9	0x6F0F	
+0xC2AA	0x964B	
+0xC2AB	0x82A6	
+0xC2AC	0x5362	
+0xC2AD	0x9885	
+0xC2AE	0x5E90	
+0xC2AF	0x7089	
+0xC2B0	0x63B3	
+0xC2B1	0x5364	
+0xC2B2	0x864F	
+0xC2B3	0x9C81	
+0xC2B4	0x9E93	
+0xC2B5	0x788C	
+0xC2B6	0x9732	
+0xC2B7	0x8DEF	
+0xC2B8	0x8D42	
+0xC2B9	0x9E7F	
+0xC2BA	0x6F5E	
+0xC2BB	0x7984	
+0xC2BC	0x5F55	
+0xC2BD	0x9646	
+0xC2BE	0x622E	
+0xC2BF	0x9A74	
+0xC2C0	0x5415	
+0xC2C1	0x94DD	
+0xC2C2	0x4FA3	
+0xC2C3	0x65C5	
+0xC2C4	0x5C65	
+0xC2C5	0x5C61	
+0xC2C6	0x7F15	
+0xC2C7	0x8651	
+0xC2C8	0x6C2F	
+0xC2C9	0x5F8B	
+0xC2CA	0x7387	
+0xC2CB	0x6EE4	
+0xC2CC	0x7EFF	
+0xC2CD	0x5CE6	
+0xC2CE	0x631B	
+0xC2CF	0x5B6A	
+0xC2D0	0x6EE6	
+0xC2D1	0x5375	
+0xC2D2	0x4E71	
+0xC2D3	0x63A0	
+0xC2D4	0x7565	
+0xC2D5	0x62A1	
+0xC2D6	0x8F6E	
+0xC2D7	0x4F26	
+0xC2D8	0x4ED1	
+0xC2D9	0x6CA6	
+0xC2DA	0x7EB6	
+0xC2DB	0x8BBA	
+0xC2DC	0x841D	
+0xC2DD	0x87BA	
+0xC2DE	0x7F57	
+0xC2DF	0x903B	
+0xC2E0	0x9523	
+0xC2E1	0x7BA9	
+0xC2E2	0x9AA1	
+0xC2E3	0x88F8	
+0xC2E4	0x843D	
+0xC2E5	0x6D1B	
+0xC2E6	0x9A86	
+0xC2E7	0x7EDC	
+0xC2E8	0x5988	
+0xC2E9	0x9EBB	
+0xC2EA	0x739B	
+0xC2EB	0x7801	
+0xC2EC	0x8682	
+0xC2ED	0x9A6C	
+0xC2EE	0x9A82	
+0xC2EF	0x561B	
+0xC2F0	0x5417	
+0xC2F1	0x57CB	
+0xC2F2	0x4E70	
+0xC2F3	0x9EA6	
+0xC2F4	0x5356	
+0xC2F5	0x8FC8	
+0xC2F6	0x8109	
+0xC2F7	0x7792	
+0xC2F8	0x9992	
+0xC2F9	0x86EE	
+0xC2FA	0x6EE1	
+0xC2FB	0x8513	
+0xC2FC	0x66FC	
+0xC2FD	0x6162	
+0xC2FE	0x6F2B	
+0xC340	0x807E	
+0xC341	0x8081	
+0xC342	0x8082	
+0xC343	0x8085	
+0xC344	0x8088	
+0xC345	0x808A	
+0xC346	0x808D	
+0xC347	0x808E	
+0xC348	0x808F	
+0xC349	0x8090	
+0xC34A	0x8091	
+0xC34B	0x8092	
+0xC34C	0x8094	
+0xC34D	0x8095	
+0xC34E	0x8097	
+0xC34F	0x8099	
+0xC350	0x809E	
+0xC351	0x80A3	
+0xC352	0x80A6	
+0xC353	0x80A7	
+0xC354	0x80A8	
+0xC355	0x80AC	
+0xC356	0x80B0	
+0xC357	0x80B3	
+0xC358	0x80B5	
+0xC359	0x80B6	
+0xC35A	0x80B8	
+0xC35B	0x80B9	
+0xC35C	0x80BB	
+0xC35D	0x80C5	
+0xC35E	0x80C7	
+0xC35F	0x80C8	
+0xC360	0x80C9	
+0xC361	0x80CA	
+0xC362	0x80CB	
+0xC363	0x80CF	
+0xC364	0x80D0	
+0xC365	0x80D1	
+0xC366	0x80D2	
+0xC367	0x80D3	
+0xC368	0x80D4	
+0xC369	0x80D5	
+0xC36A	0x80D8	
+0xC36B	0x80DF	
+0xC36C	0x80E0	
+0xC36D	0x80E2	
+0xC36E	0x80E3	
+0xC36F	0x80E6	
+0xC370	0x80EE	
+0xC371	0x80F5	
+0xC372	0x80F7	
+0xC373	0x80F9	
+0xC374	0x80FB	
+0xC375	0x80FE	
+0xC376	0x80FF	
+0xC377	0x8100	
+0xC378	0x8101	
+0xC379	0x8103	
+0xC37A	0x8104	
+0xC37B	0x8105	
+0xC37C	0x8107	
+0xC37D	0x8108	
+0xC37E	0x810B	
+0xC380	0x810C	
+0xC381	0x8115	
+0xC382	0x8117	
+0xC383	0x8119	
+0xC384	0x811B	
+0xC385	0x811C	
+0xC386	0x811D	
+0xC387	0x811F	
+0xC388	0x8120	
+0xC389	0x8121	
+0xC38A	0x8122	
+0xC38B	0x8123	
+0xC38C	0x8124	
+0xC38D	0x8125	
+0xC38E	0x8126	
+0xC38F	0x8127	
+0xC390	0x8128	
+0xC391	0x8129	
+0xC392	0x812A	
+0xC393	0x812B	
+0xC394	0x812D	
+0xC395	0x812E	
+0xC396	0x8130	
+0xC397	0x8133	
+0xC398	0x8134	
+0xC399	0x8135	
+0xC39A	0x8137	
+0xC39B	0x8139	
+0xC39C	0x813A	
+0xC39D	0x813B	
+0xC39E	0x813C	
+0xC39F	0x813D	
+0xC3A0	0x813F	
+0xC3A1	0x8C29	
+0xC3A2	0x8292	
+0xC3A3	0x832B	
+0xC3A4	0x76F2	
+0xC3A5	0x6C13	
+0xC3A6	0x5FD9	
+0xC3A7	0x83BD	
+0xC3A8	0x732B	
+0xC3A9	0x8305	
+0xC3AA	0x951A	
+0xC3AB	0x6BDB	
+0xC3AC	0x77DB	
+0xC3AD	0x94C6	
+0xC3AE	0x536F	
+0xC3AF	0x8302	
+0xC3B0	0x5192	
+0xC3B1	0x5E3D	
+0xC3B2	0x8C8C	
+0xC3B3	0x8D38	
+0xC3B4	0x4E48	
+0xC3B5	0x73AB	
+0xC3B6	0x679A	
+0xC3B7	0x6885	
+0xC3B8	0x9176	
+0xC3B9	0x9709	
+0xC3BA	0x7164	
+0xC3BB	0x6CA1	
+0xC3BC	0x7709	
+0xC3BD	0x5A92	
+0xC3BE	0x9541	
+0xC3BF	0x6BCF	
+0xC3C0	0x7F8E	
+0xC3C1	0x6627	
+0xC3C2	0x5BD0	
+0xC3C3	0x59B9	
+0xC3C4	0x5A9A	
+0xC3C5	0x95E8	
+0xC3C6	0x95F7	
+0xC3C7	0x4EEC	
+0xC3C8	0x840C	
+0xC3C9	0x8499	
+0xC3CA	0x6AAC	
+0xC3CB	0x76DF	
+0xC3CC	0x9530	
+0xC3CD	0x731B	
+0xC3CE	0x68A6	
+0xC3CF	0x5B5F	
+0xC3D0	0x772F	
+0xC3D1	0x919A	
+0xC3D2	0x9761	
+0xC3D3	0x7CDC	
+0xC3D4	0x8FF7	
+0xC3D5	0x8C1C	
+0xC3D6	0x5F25	
+0xC3D7	0x7C73	
+0xC3D8	0x79D8	
+0xC3D9	0x89C5	
+0xC3DA	0x6CCC	
+0xC3DB	0x871C	
+0xC3DC	0x5BC6	
+0xC3DD	0x5E42	
+0xC3DE	0x68C9	
+0xC3DF	0x7720	
+0xC3E0	0x7EF5	
+0xC3E1	0x5195	
+0xC3E2	0x514D	
+0xC3E3	0x52C9	
+0xC3E4	0x5A29	
+0xC3E5	0x7F05	
+0xC3E6	0x9762	
+0xC3E7	0x82D7	
+0xC3E8	0x63CF	
+0xC3E9	0x7784	
+0xC3EA	0x85D0	
+0xC3EB	0x79D2	
+0xC3EC	0x6E3A	
+0xC3ED	0x5E99	
+0xC3EE	0x5999	
+0xC3EF	0x8511	
+0xC3F0	0x706D	
+0xC3F1	0x6C11	
+0xC3F2	0x62BF	
+0xC3F3	0x76BF	
+0xC3F4	0x654F	
+0xC3F5	0x60AF	
+0xC3F6	0x95FD	
+0xC3F7	0x660E	
+0xC3F8	0x879F	
+0xC3F9	0x9E23	
+0xC3FA	0x94ED	
+0xC3FB	0x540D	
+0xC3FC	0x547D	
+0xC3FD	0x8C2C	
+0xC3FE	0x6478	
+0xC440	0x8140	
+0xC441	0x8141	
+0xC442	0x8142	
+0xC443	0x8143	
+0xC444	0x8144	
+0xC445	0x8145	
+0xC446	0x8147	
+0xC447	0x8149	
+0xC448	0x814D	
+0xC449	0x814E	
+0xC44A	0x814F	
+0xC44B	0x8152	
+0xC44C	0x8156	
+0xC44D	0x8157	
+0xC44E	0x8158	
+0xC44F	0x815B	
+0xC450	0x815C	
+0xC451	0x815D	
+0xC452	0x815E	
+0xC453	0x815F	
+0xC454	0x8161	
+0xC455	0x8162	
+0xC456	0x8163	
+0xC457	0x8164	
+0xC458	0x8166	
+0xC459	0x8168	
+0xC45A	0x816A	
+0xC45B	0x816B	
+0xC45C	0x816C	
+0xC45D	0x816F	
+0xC45E	0x8172	
+0xC45F	0x8173	
+0xC460	0x8175	
+0xC461	0x8176	
+0xC462	0x8177	
+0xC463	0x8178	
+0xC464	0x8181	
+0xC465	0x8183	
+0xC466	0x8184	
+0xC467	0x8185	
+0xC468	0x8186	
+0xC469	0x8187	
+0xC46A	0x8189	
+0xC46B	0x818B	
+0xC46C	0x818C	
+0xC46D	0x818D	
+0xC46E	0x818E	
+0xC46F	0x8190	
+0xC470	0x8192	
+0xC471	0x8193	
+0xC472	0x8194	
+0xC473	0x8195	
+0xC474	0x8196	
+0xC475	0x8197	
+0xC476	0x8199	
+0xC477	0x819A	
+0xC478	0x819E	
+0xC479	0x819F	
+0xC47A	0x81A0	
+0xC47B	0x81A1	
+0xC47C	0x81A2	
+0xC47D	0x81A4	
+0xC47E	0x81A5	
+0xC480	0x81A7	
+0xC481	0x81A9	
+0xC482	0x81AB	
+0xC483	0x81AC	
+0xC484	0x81AD	
+0xC485	0x81AE	
+0xC486	0x81AF	
+0xC487	0x81B0	
+0xC488	0x81B1	
+0xC489	0x81B2	
+0xC48A	0x81B4	
+0xC48B	0x81B5	
+0xC48C	0x81B6	
+0xC48D	0x81B7	
+0xC48E	0x81B8	
+0xC48F	0x81B9	
+0xC490	0x81BC	
+0xC491	0x81BD	
+0xC492	0x81BE	
+0xC493	0x81BF	
+0xC494	0x81C4	
+0xC495	0x81C5	
+0xC496	0x81C7	
+0xC497	0x81C8	
+0xC498	0x81C9	
+0xC499	0x81CB	
+0xC49A	0x81CD	
+0xC49B	0x81CE	
+0xC49C	0x81CF	
+0xC49D	0x81D0	
+0xC49E	0x81D1	
+0xC49F	0x81D2	
+0xC4A0	0x81D3	
+0xC4A1	0x6479	
+0xC4A2	0x8611	
+0xC4A3	0x6A21	
+0xC4A4	0x819C	
+0xC4A5	0x78E8	
+0xC4A6	0x6469	
+0xC4A7	0x9B54	
+0xC4A8	0x62B9	
+0xC4A9	0x672B	
+0xC4AA	0x83AB	
+0xC4AB	0x58A8	
+0xC4AC	0x9ED8	
+0xC4AD	0x6CAB	
+0xC4AE	0x6F20	
+0xC4AF	0x5BDE	
+0xC4B0	0x964C	
+0xC4B1	0x8C0B	
+0xC4B2	0x725F	
+0xC4B3	0x67D0	
+0xC4B4	0x62C7	
+0xC4B5	0x7261	
+0xC4B6	0x4EA9	
+0xC4B7	0x59C6	
+0xC4B8	0x6BCD	
+0xC4B9	0x5893	
+0xC4BA	0x66AE	
+0xC4BB	0x5E55	
+0xC4BC	0x52DF	
+0xC4BD	0x6155	
+0xC4BE	0x6728	
+0xC4BF	0x76EE	
+0xC4C0	0x7766	
+0xC4C1	0x7267	
+0xC4C2	0x7A46	
+0xC4C3	0x62FF	
+0xC4C4	0x54EA	
+0xC4C5	0x5450	
+0xC4C6	0x94A0	
+0xC4C7	0x90A3	
+0xC4C8	0x5A1C	
+0xC4C9	0x7EB3	
+0xC4CA	0x6C16	
+0xC4CB	0x4E43	
+0xC4CC	0x5976	
+0xC4CD	0x8010	
+0xC4CE	0x5948	
+0xC4CF	0x5357	
+0xC4D0	0x7537	
+0xC4D1	0x96BE	
+0xC4D2	0x56CA	
+0xC4D3	0x6320	
+0xC4D4	0x8111	
+0xC4D5	0x607C	
+0xC4D6	0x95F9	
+0xC4D7	0x6DD6	
+0xC4D8	0x5462	
+0xC4D9	0x9981	
+0xC4DA	0x5185	
+0xC4DB	0x5AE9	
+0xC4DC	0x80FD	
+0xC4DD	0x59AE	
+0xC4DE	0x9713	
+0xC4DF	0x502A	
+0xC4E0	0x6CE5	
+0xC4E1	0x5C3C	
+0xC4E2	0x62DF	
+0xC4E3	0x4F60	
+0xC4E4	0x533F	
+0xC4E5	0x817B	
+0xC4E6	0x9006	
+0xC4E7	0x6EBA	
+0xC4E8	0x852B	
+0xC4E9	0x62C8	
+0xC4EA	0x5E74	
+0xC4EB	0x78BE	
+0xC4EC	0x64B5	
+0xC4ED	0x637B	
+0xC4EE	0x5FF5	
+0xC4EF	0x5A18	
+0xC4F0	0x917F	
+0xC4F1	0x9E1F	
+0xC4F2	0x5C3F	
+0xC4F3	0x634F	
+0xC4F4	0x8042	
+0xC4F5	0x5B7D	
+0xC4F6	0x556E	
+0xC4F7	0x954A	
+0xC4F8	0x954D	
+0xC4F9	0x6D85	
+0xC4FA	0x60A8	
+0xC4FB	0x67E0	
+0xC4FC	0x72DE	
+0xC4FD	0x51DD	
+0xC4FE	0x5B81	
+0xC540	0x81D4	
+0xC541	0x81D5	
+0xC542	0x81D6	
+0xC543	0x81D7	
+0xC544	0x81D8	
+0xC545	0x81D9	
+0xC546	0x81DA	
+0xC547	0x81DB	
+0xC548	0x81DC	
+0xC549	0x81DD	
+0xC54A	0x81DE	
+0xC54B	0x81DF	
+0xC54C	0x81E0	
+0xC54D	0x81E1	
+0xC54E	0x81E2	
+0xC54F	0x81E4	
+0xC550	0x81E5	
+0xC551	0x81E6	
+0xC552	0x81E8	
+0xC553	0x81E9	
+0xC554	0x81EB	
+0xC555	0x81EE	
+0xC556	0x81EF	
+0xC557	0x81F0	
+0xC558	0x81F1	
+0xC559	0x81F2	
+0xC55A	0x81F5	
+0xC55B	0x81F6	
+0xC55C	0x81F7	
+0xC55D	0x81F8	
+0xC55E	0x81F9	
+0xC55F	0x81FA	
+0xC560	0x81FD	
+0xC561	0x81FF	
+0xC562	0x8203	
+0xC563	0x8207	
+0xC564	0x8208	
+0xC565	0x8209	
+0xC566	0x820A	
+0xC567	0x820B	
+0xC568	0x820E	
+0xC569	0x820F	
+0xC56A	0x8211	
+0xC56B	0x8213	
+0xC56C	0x8215	
+0xC56D	0x8216	
+0xC56E	0x8217	
+0xC56F	0x8218	
+0xC570	0x8219	
+0xC571	0x821A	
+0xC572	0x821D	
+0xC573	0x8220	
+0xC574	0x8224	
+0xC575	0x8225	
+0xC576	0x8226	
+0xC577	0x8227	
+0xC578	0x8229	
+0xC579	0x822E	
+0xC57A	0x8232	
+0xC57B	0x823A	
+0xC57C	0x823C	
+0xC57D	0x823D	
+0xC57E	0x823F	
+0xC580	0x8240	
+0xC581	0x8241	
+0xC582	0x8242	
+0xC583	0x8243	
+0xC584	0x8245	
+0xC585	0x8246	
+0xC586	0x8248	
+0xC587	0x824A	
+0xC588	0x824C	
+0xC589	0x824D	
+0xC58A	0x824E	
+0xC58B	0x8250	
+0xC58C	0x8251	
+0xC58D	0x8252	
+0xC58E	0x8253	
+0xC58F	0x8254	
+0xC590	0x8255	
+0xC591	0x8256	
+0xC592	0x8257	
+0xC593	0x8259	
+0xC594	0x825B	
+0xC595	0x825C	
+0xC596	0x825D	
+0xC597	0x825E	
+0xC598	0x8260	
+0xC599	0x8261	
+0xC59A	0x8262	
+0xC59B	0x8263	
+0xC59C	0x8264	
+0xC59D	0x8265	
+0xC59E	0x8266	
+0xC59F	0x8267	
+0xC5A0	0x8269	
+0xC5A1	0x62E7	
+0xC5A2	0x6CDE	
+0xC5A3	0x725B	
+0xC5A4	0x626D	
+0xC5A5	0x94AE	
+0xC5A6	0x7EBD	
+0xC5A7	0x8113	
+0xC5A8	0x6D53	
+0xC5A9	0x519C	
+0xC5AA	0x5F04	
+0xC5AB	0x5974	
+0xC5AC	0x52AA	
+0xC5AD	0x6012	
+0xC5AE	0x5973	
+0xC5AF	0x6696	
+0xC5B0	0x8650	
+0xC5B1	0x759F	
+0xC5B2	0x632A	
+0xC5B3	0x61E6	
+0xC5B4	0x7CEF	
+0xC5B5	0x8BFA	
+0xC5B6	0x54E6	
+0xC5B7	0x6B27	
+0xC5B8	0x9E25	
+0xC5B9	0x6BB4	
+0xC5BA	0x85D5	
+0xC5BB	0x5455	
+0xC5BC	0x5076	
+0xC5BD	0x6CA4	
+0xC5BE	0x556A	
+0xC5BF	0x8DB4	
+0xC5C0	0x722C	
+0xC5C1	0x5E15	
+0xC5C2	0x6015	
+0xC5C3	0x7436	
+0xC5C4	0x62CD	
+0xC5C5	0x6392	
+0xC5C6	0x724C	
+0xC5C7	0x5F98	
+0xC5C8	0x6E43	
+0xC5C9	0x6D3E	
+0xC5CA	0x6500	
+0xC5CB	0x6F58	
+0xC5CC	0x76D8	
+0xC5CD	0x78D0	
+0xC5CE	0x76FC	
+0xC5CF	0x7554	
+0xC5D0	0x5224	
+0xC5D1	0x53DB	
+0xC5D2	0x4E53	
+0xC5D3	0x5E9E	
+0xC5D4	0x65C1	
+0xC5D5	0x802A	
+0xC5D6	0x80D6	
+0xC5D7	0x629B	
+0xC5D8	0x5486	
+0xC5D9	0x5228	
+0xC5DA	0x70AE	
+0xC5DB	0x888D	
+0xC5DC	0x8DD1	
+0xC5DD	0x6CE1	
+0xC5DE	0x5478	
+0xC5DF	0x80DA	
+0xC5E0	0x57F9	
+0xC5E1	0x88F4	
+0xC5E2	0x8D54	
+0xC5E3	0x966A	
+0xC5E4	0x914D	
+0xC5E5	0x4F69	
+0xC5E6	0x6C9B	
+0xC5E7	0x55B7	
+0xC5E8	0x76C6	
+0xC5E9	0x7830	
+0xC5EA	0x62A8	
+0xC5EB	0x70F9	
+0xC5EC	0x6F8E	
+0xC5ED	0x5F6D	
+0xC5EE	0x84EC	
+0xC5EF	0x68DA	
+0xC5F0	0x787C	
+0xC5F1	0x7BF7	
+0xC5F2	0x81A8	
+0xC5F3	0x670B	
+0xC5F4	0x9E4F	
+0xC5F5	0x6367	
+0xC5F6	0x78B0	
+0xC5F7	0x576F	
+0xC5F8	0x7812	
+0xC5F9	0x9739	
+0xC5FA	0x6279	
+0xC5FB	0x62AB	
+0xC5FC	0x5288	
+0xC5FD	0x7435	
+0xC5FE	0x6BD7	
+0xC640	0x826A	
+0xC641	0x826B	
+0xC642	0x826C	
+0xC643	0x826D	
+0xC644	0x8271	
+0xC645	0x8275	
+0xC646	0x8276	
+0xC647	0x8277	
+0xC648	0x8278	
+0xC649	0x827B	
+0xC64A	0x827C	
+0xC64B	0x8280	
+0xC64C	0x8281	
+0xC64D	0x8283	
+0xC64E	0x8285	
+0xC64F	0x8286	
+0xC650	0x8287	
+0xC651	0x8289	
+0xC652	0x828C	
+0xC653	0x8290	
+0xC654	0x8293	
+0xC655	0x8294	
+0xC656	0x8295	
+0xC657	0x8296	
+0xC658	0x829A	
+0xC659	0x829B	
+0xC65A	0x829E	
+0xC65B	0x82A0	
+0xC65C	0x82A2	
+0xC65D	0x82A3	
+0xC65E	0x82A7	
+0xC65F	0x82B2	
+0xC660	0x82B5	
+0xC661	0x82B6	
+0xC662	0x82BA	
+0xC663	0x82BB	
+0xC664	0x82BC	
+0xC665	0x82BF	
+0xC666	0x82C0	
+0xC667	0x82C2	
+0xC668	0x82C3	
+0xC669	0x82C5	
+0xC66A	0x82C6	
+0xC66B	0x82C9	
+0xC66C	0x82D0	
+0xC66D	0x82D6	
+0xC66E	0x82D9	
+0xC66F	0x82DA	
+0xC670	0x82DD	
+0xC671	0x82E2	
+0xC672	0x82E7	
+0xC673	0x82E8	
+0xC674	0x82E9	
+0xC675	0x82EA	
+0xC676	0x82EC	
+0xC677	0x82ED	
+0xC678	0x82EE	
+0xC679	0x82F0	
+0xC67A	0x82F2	
+0xC67B	0x82F3	
+0xC67C	0x82F5	
+0xC67D	0x82F6	
+0xC67E	0x82F8	
+0xC680	0x82FA	
+0xC681	0x82FC	
+0xC682	0x82FD	
+0xC683	0x82FE	
+0xC684	0x82FF	
+0xC685	0x8300	
+0xC686	0x830A	
+0xC687	0x830B	
+0xC688	0x830D	
+0xC689	0x8310	
+0xC68A	0x8312	
+0xC68B	0x8313	
+0xC68C	0x8316	
+0xC68D	0x8318	
+0xC68E	0x8319	
+0xC68F	0x831D	
+0xC690	0x831E	
+0xC691	0x831F	
+0xC692	0x8320	
+0xC693	0x8321	
+0xC694	0x8322	
+0xC695	0x8323	
+0xC696	0x8324	
+0xC697	0x8325	
+0xC698	0x8326	
+0xC699	0x8329	
+0xC69A	0x832A	
+0xC69B	0x832E	
+0xC69C	0x8330	
+0xC69D	0x8332	
+0xC69E	0x8337	
+0xC69F	0x833B	
+0xC6A0	0x833D	
+0xC6A1	0x5564	
+0xC6A2	0x813E	
+0xC6A3	0x75B2	
+0xC6A4	0x76AE	
+0xC6A5	0x5339	
+0xC6A6	0x75DE	
+0xC6A7	0x50FB	
+0xC6A8	0x5C41	
+0xC6A9	0x8B6C	
+0xC6AA	0x7BC7	
+0xC6AB	0x504F	
+0xC6AC	0x7247	
+0xC6AD	0x9A97	
+0xC6AE	0x98D8	
+0xC6AF	0x6F02	
+0xC6B0	0x74E2	
+0xC6B1	0x7968	
+0xC6B2	0x6487	
+0xC6B3	0x77A5	
+0xC6B4	0x62FC	
+0xC6B5	0x9891	
+0xC6B6	0x8D2B	
+0xC6B7	0x54C1	
+0xC6B8	0x8058	
+0xC6B9	0x4E52	
+0xC6BA	0x576A	
+0xC6BB	0x82F9	
+0xC6BC	0x840D	
+0xC6BD	0x5E73	
+0xC6BE	0x51ED	
+0xC6BF	0x74F6	
+0xC6C0	0x8BC4	
+0xC6C1	0x5C4F	
+0xC6C2	0x5761	
+0xC6C3	0x6CFC	
+0xC6C4	0x9887	
+0xC6C5	0x5A46	
+0xC6C6	0x7834	
+0xC6C7	0x9B44	
+0xC6C8	0x8FEB	
+0xC6C9	0x7C95	
+0xC6CA	0x5256	
+0xC6CB	0x6251	
+0xC6CC	0x94FA	
+0xC6CD	0x4EC6	
+0xC6CE	0x8386	
+0xC6CF	0x8461	
+0xC6D0	0x83E9	
+0xC6D1	0x84B2	
+0xC6D2	0x57D4	
+0xC6D3	0x6734	
+0xC6D4	0x5703	
+0xC6D5	0x666E	
+0xC6D6	0x6D66	
+0xC6D7	0x8C31	
+0xC6D8	0x66DD	
+0xC6D9	0x7011	
+0xC6DA	0x671F	
+0xC6DB	0x6B3A	
+0xC6DC	0x6816	
+0xC6DD	0x621A	
+0xC6DE	0x59BB	
+0xC6DF	0x4E03	
+0xC6E0	0x51C4	
+0xC6E1	0x6F06	
+0xC6E2	0x67D2	
+0xC6E3	0x6C8F	
+0xC6E4	0x5176	
+0xC6E5	0x68CB	
+0xC6E6	0x5947	
+0xC6E7	0x6B67	
+0xC6E8	0x7566	
+0xC6E9	0x5D0E	
+0xC6EA	0x8110	
+0xC6EB	0x9F50	
+0xC6EC	0x65D7	
+0xC6ED	0x7948	
+0xC6EE	0x7941	
+0xC6EF	0x9A91	
+0xC6F0	0x8D77	
+0xC6F1	0x5C82	
+0xC6F2	0x4E5E	
+0xC6F3	0x4F01	
+0xC6F4	0x542F	
+0xC6F5	0x5951	
+0xC6F6	0x780C	
+0xC6F7	0x5668	
+0xC6F8	0x6C14	
+0xC6F9	0x8FC4	
+0xC6FA	0x5F03	
+0xC6FB	0x6C7D	
+0xC6FC	0x6CE3	
+0xC6FD	0x8BAB	
+0xC6FE	0x6390	
+0xC740	0x833E	
+0xC741	0x833F	
+0xC742	0x8341	
+0xC743	0x8342	
+0xC744	0x8344	
+0xC745	0x8345	
+0xC746	0x8348	
+0xC747	0x834A	
+0xC748	0x834B	
+0xC749	0x834C	
+0xC74A	0x834D	
+0xC74B	0x834E	
+0xC74C	0x8353	
+0xC74D	0x8355	
+0xC74E	0x8356	
+0xC74F	0x8357	
+0xC750	0x8358	
+0xC751	0x8359	
+0xC752	0x835D	
+0xC753	0x8362	
+0xC754	0x8370	
+0xC755	0x8371	
+0xC756	0x8372	
+0xC757	0x8373	
+0xC758	0x8374	
+0xC759	0x8375	
+0xC75A	0x8376	
+0xC75B	0x8379	
+0xC75C	0x837A	
+0xC75D	0x837E	
+0xC75E	0x837F	
+0xC75F	0x8380	
+0xC760	0x8381	
+0xC761	0x8382	
+0xC762	0x8383	
+0xC763	0x8384	
+0xC764	0x8387	
+0xC765	0x8388	
+0xC766	0x838A	
+0xC767	0x838B	
+0xC768	0x838C	
+0xC769	0x838D	
+0xC76A	0x838F	
+0xC76B	0x8390	
+0xC76C	0x8391	
+0xC76D	0x8394	
+0xC76E	0x8395	
+0xC76F	0x8396	
+0xC770	0x8397	
+0xC771	0x8399	
+0xC772	0x839A	
+0xC773	0x839D	
+0xC774	0x839F	
+0xC775	0x83A1	
+0xC776	0x83A2	
+0xC777	0x83A3	
+0xC778	0x83A4	
+0xC779	0x83A5	
+0xC77A	0x83A6	
+0xC77B	0x83A7	
+0xC77C	0x83AC	
+0xC77D	0x83AD	
+0xC77E	0x83AE	
+0xC780	0x83AF	
+0xC781	0x83B5	
+0xC782	0x83BB	
+0xC783	0x83BE	
+0xC784	0x83BF	
+0xC785	0x83C2	
+0xC786	0x83C3	
+0xC787	0x83C4	
+0xC788	0x83C6	
+0xC789	0x83C8	
+0xC78A	0x83C9	
+0xC78B	0x83CB	
+0xC78C	0x83CD	
+0xC78D	0x83CE	
+0xC78E	0x83D0	
+0xC78F	0x83D1	
+0xC790	0x83D2	
+0xC791	0x83D3	
+0xC792	0x83D5	
+0xC793	0x83D7	
+0xC794	0x83D9	
+0xC795	0x83DA	
+0xC796	0x83DB	
+0xC797	0x83DE	
+0xC798	0x83E2	
+0xC799	0x83E3	
+0xC79A	0x83E4	
+0xC79B	0x83E6	
+0xC79C	0x83E7	
+0xC79D	0x83E8	
+0xC79E	0x83EB	
+0xC79F	0x83EC	
+0xC7A0	0x83ED	
+0xC7A1	0x6070	
+0xC7A2	0x6D3D	
+0xC7A3	0x7275	
+0xC7A4	0x6266	
+0xC7A5	0x948E	
+0xC7A6	0x94C5	
+0xC7A7	0x5343	
+0xC7A8	0x8FC1	
+0xC7A9	0x7B7E	
+0xC7AA	0x4EDF	
+0xC7AB	0x8C26	
+0xC7AC	0x4E7E	
+0xC7AD	0x9ED4	
+0xC7AE	0x94B1	
+0xC7AF	0x94B3	
+0xC7B0	0x524D	
+0xC7B1	0x6F5C	
+0xC7B2	0x9063	
+0xC7B3	0x6D45	
+0xC7B4	0x8C34	
+0xC7B5	0x5811	
+0xC7B6	0x5D4C	
+0xC7B7	0x6B20	
+0xC7B8	0x6B49	
+0xC7B9	0x67AA	
+0xC7BA	0x545B	
+0xC7BB	0x8154	
+0xC7BC	0x7F8C	
+0xC7BD	0x5899	
+0xC7BE	0x8537	
+0xC7BF	0x5F3A	
+0xC7C0	0x62A2	
+0xC7C1	0x6A47	
+0xC7C2	0x9539	
+0xC7C3	0x6572	
+0xC7C4	0x6084	
+0xC7C5	0x6865	
+0xC7C6	0x77A7	
+0xC7C7	0x4E54	
+0xC7C8	0x4FA8	
+0xC7C9	0x5DE7	
+0xC7CA	0x9798	
+0xC7CB	0x64AC	
+0xC7CC	0x7FD8	
+0xC7CD	0x5CED	
+0xC7CE	0x4FCF	
+0xC7CF	0x7A8D	
+0xC7D0	0x5207	
+0xC7D1	0x8304	
+0xC7D2	0x4E14	
+0xC7D3	0x602F	
+0xC7D4	0x7A83	
+0xC7D5	0x94A6	
+0xC7D6	0x4FB5	
+0xC7D7	0x4EB2	
+0xC7D8	0x79E6	
+0xC7D9	0x7434	
+0xC7DA	0x52E4	
+0xC7DB	0x82B9	
+0xC7DC	0x64D2	
+0xC7DD	0x79BD	
+0xC7DE	0x5BDD	
+0xC7DF	0x6C81	
+0xC7E0	0x9752	
+0xC7E1	0x8F7B	
+0xC7E2	0x6C22	
+0xC7E3	0x503E	
+0xC7E4	0x537F	
+0xC7E5	0x6E05	
+0xC7E6	0x64CE	
+0xC7E7	0x6674	
+0xC7E8	0x6C30	
+0xC7E9	0x60C5	
+0xC7EA	0x9877	
+0xC7EB	0x8BF7	
+0xC7EC	0x5E86	
+0xC7ED	0x743C	
+0xC7EE	0x7A77	
+0xC7EF	0x79CB	
+0xC7F0	0x4E18	
+0xC7F1	0x90B1	
+0xC7F2	0x7403	
+0xC7F3	0x6C42	
+0xC7F4	0x56DA	
+0xC7F5	0x914B	
+0xC7F6	0x6CC5	
+0xC7F7	0x8D8B	
+0xC7F8	0x533A	
+0xC7F9	0x86C6	
+0xC7FA	0x66F2	
+0xC7FB	0x8EAF	
+0xC7FC	0x5C48	
+0xC7FD	0x9A71	
+0xC7FE	0x6E20	
+0xC840	0x83EE	
+0xC841	0x83EF	
+0xC842	0x83F3	
+0xC843	0x83F4	
+0xC844	0x83F5	
+0xC845	0x83F6	
+0xC846	0x83F7	
+0xC847	0x83FA	
+0xC848	0x83FB	
+0xC849	0x83FC	
+0xC84A	0x83FE	
+0xC84B	0x83FF	
+0xC84C	0x8400	
+0xC84D	0x8402	
+0xC84E	0x8405	
+0xC84F	0x8407	
+0xC850	0x8408	
+0xC851	0x8409	
+0xC852	0x840A	
+0xC853	0x8410	
+0xC854	0x8412	
+0xC855	0x8413	
+0xC856	0x8414	
+0xC857	0x8415	
+0xC858	0x8416	
+0xC859	0x8417	
+0xC85A	0x8419	
+0xC85B	0x841A	
+0xC85C	0x841B	
+0xC85D	0x841E	
+0xC85E	0x841F	
+0xC85F	0x8420	
+0xC860	0x8421	
+0xC861	0x8422	
+0xC862	0x8423	
+0xC863	0x8429	
+0xC864	0x842A	
+0xC865	0x842B	
+0xC866	0x842C	
+0xC867	0x842D	
+0xC868	0x842E	
+0xC869	0x842F	
+0xC86A	0x8430	
+0xC86B	0x8432	
+0xC86C	0x8433	
+0xC86D	0x8434	
+0xC86E	0x8435	
+0xC86F	0x8436	
+0xC870	0x8437	
+0xC871	0x8439	
+0xC872	0x843A	
+0xC873	0x843B	
+0xC874	0x843E	
+0xC875	0x843F	
+0xC876	0x8440	
+0xC877	0x8441	
+0xC878	0x8442	
+0xC879	0x8443	
+0xC87A	0x8444	
+0xC87B	0x8445	
+0xC87C	0x8447	
+0xC87D	0x8448	
+0xC87E	0x8449	
+0xC880	0x844A	
+0xC881	0x844B	
+0xC882	0x844C	
+0xC883	0x844D	
+0xC884	0x844E	
+0xC885	0x844F	
+0xC886	0x8450	
+0xC887	0x8452	
+0xC888	0x8453	
+0xC889	0x8454	
+0xC88A	0x8455	
+0xC88B	0x8456	
+0xC88C	0x8458	
+0xC88D	0x845D	
+0xC88E	0x845E	
+0xC88F	0x845F	
+0xC890	0x8460	
+0xC891	0x8462	
+0xC892	0x8464	
+0xC893	0x8465	
+0xC894	0x8466	
+0xC895	0x8467	
+0xC896	0x8468	
+0xC897	0x846A	
+0xC898	0x846E	
+0xC899	0x846F	
+0xC89A	0x8470	
+0xC89B	0x8472	
+0xC89C	0x8474	
+0xC89D	0x8477	
+0xC89E	0x8479	
+0xC89F	0x847B	
+0xC8A0	0x847C	
+0xC8A1	0x53D6	
+0xC8A2	0x5A36	
+0xC8A3	0x9F8B	
+0xC8A4	0x8DA3	
+0xC8A5	0x53BB	
+0xC8A6	0x5708	
+0xC8A7	0x98A7	
+0xC8A8	0x6743	
+0xC8A9	0x919B	
+0xC8AA	0x6CC9	
+0xC8AB	0x5168	
+0xC8AC	0x75CA	
+0xC8AD	0x62F3	
+0xC8AE	0x72AC	
+0xC8AF	0x5238	
+0xC8B0	0x529D	
+0xC8B1	0x7F3A	
+0xC8B2	0x7094	
+0xC8B3	0x7638	
+0xC8B4	0x5374	
+0xC8B5	0x9E4A	
+0xC8B6	0x69B7	
+0xC8B7	0x786E	
+0xC8B8	0x96C0	
+0xC8B9	0x88D9	
+0xC8BA	0x7FA4	
+0xC8BB	0x7136	
+0xC8BC	0x71C3	
+0xC8BD	0x5189	
+0xC8BE	0x67D3	
+0xC8BF	0x74E4	
+0xC8C0	0x58E4	
+0xC8C1	0x6518	
+0xC8C2	0x56B7	
+0xC8C3	0x8BA9	
+0xC8C4	0x9976	
+0xC8C5	0x6270	
+0xC8C6	0x7ED5	
+0xC8C7	0x60F9	
+0xC8C8	0x70ED	
+0xC8C9	0x58EC	
+0xC8CA	0x4EC1	
+0xC8CB	0x4EBA	
+0xC8CC	0x5FCD	
+0xC8CD	0x97E7	
+0xC8CE	0x4EFB	
+0xC8CF	0x8BA4	
+0xC8D0	0x5203	
+0xC8D1	0x598A	
+0xC8D2	0x7EAB	
+0xC8D3	0x6254	
+0xC8D4	0x4ECD	
+0xC8D5	0x65E5	
+0xC8D6	0x620E	
+0xC8D7	0x8338	
+0xC8D8	0x84C9	
+0xC8D9	0x8363	
+0xC8DA	0x878D	
+0xC8DB	0x7194	
+0xC8DC	0x6EB6	
+0xC8DD	0x5BB9	
+0xC8DE	0x7ED2	
+0xC8DF	0x5197	
+0xC8E0	0x63C9	
+0xC8E1	0x67D4	
+0xC8E2	0x8089	
+0xC8E3	0x8339	
+0xC8E4	0x8815	
+0xC8E5	0x5112	
+0xC8E6	0x5B7A	
+0xC8E7	0x5982	
+0xC8E8	0x8FB1	
+0xC8E9	0x4E73	
+0xC8EA	0x6C5D	
+0xC8EB	0x5165	
+0xC8EC	0x8925	
+0xC8ED	0x8F6F	
+0xC8EE	0x962E	
+0xC8EF	0x854A	
+0xC8F0	0x745E	
+0xC8F1	0x9510	
+0xC8F2	0x95F0	
+0xC8F3	0x6DA6	
+0xC8F4	0x82E5	
+0xC8F5	0x5F31	
+0xC8F6	0x6492	
+0xC8F7	0x6D12	
+0xC8F8	0x8428	
+0xC8F9	0x816E	
+0xC8FA	0x9CC3	
+0xC8FB	0x585E	
+0xC8FC	0x8D5B	
+0xC8FD	0x4E09	
+0xC8FE	0x53C1	
+0xC940	0x847D	
+0xC941	0x847E	
+0xC942	0x847F	
+0xC943	0x8480	
+0xC944	0x8481	
+0xC945	0x8483	
+0xC946	0x8484	
+0xC947	0x8485	
+0xC948	0x8486	
+0xC949	0x848A	
+0xC94A	0x848D	
+0xC94B	0x848F	
+0xC94C	0x8490	
+0xC94D	0x8491	
+0xC94E	0x8492	
+0xC94F	0x8493	
+0xC950	0x8494	
+0xC951	0x8495	
+0xC952	0x8496	
+0xC953	0x8498	
+0xC954	0x849A	
+0xC955	0x849B	
+0xC956	0x849D	
+0xC957	0x849E	
+0xC958	0x849F	
+0xC959	0x84A0	
+0xC95A	0x84A2	
+0xC95B	0x84A3	
+0xC95C	0x84A4	
+0xC95D	0x84A5	
+0xC95E	0x84A6	
+0xC95F	0x84A7	
+0xC960	0x84A8	
+0xC961	0x84A9	
+0xC962	0x84AA	
+0xC963	0x84AB	
+0xC964	0x84AC	
+0xC965	0x84AD	
+0xC966	0x84AE	
+0xC967	0x84B0	
+0xC968	0x84B1	
+0xC969	0x84B3	
+0xC96A	0x84B5	
+0xC96B	0x84B6	
+0xC96C	0x84B7	
+0xC96D	0x84BB	
+0xC96E	0x84BC	
+0xC96F	0x84BE	
+0xC970	0x84C0	
+0xC971	0x84C2	
+0xC972	0x84C3	
+0xC973	0x84C5	
+0xC974	0x84C6	
+0xC975	0x84C7	
+0xC976	0x84C8	
+0xC977	0x84CB	
+0xC978	0x84CC	
+0xC979	0x84CE	
+0xC97A	0x84CF	
+0xC97B	0x84D2	
+0xC97C	0x84D4	
+0xC97D	0x84D5	
+0xC97E	0x84D7	
+0xC980	0x84D8	
+0xC981	0x84D9	
+0xC982	0x84DA	
+0xC983	0x84DB	
+0xC984	0x84DC	
+0xC985	0x84DE	
+0xC986	0x84E1	
+0xC987	0x84E2	
+0xC988	0x84E4	
+0xC989	0x84E7	
+0xC98A	0x84E8	
+0xC98B	0x84E9	
+0xC98C	0x84EA	
+0xC98D	0x84EB	
+0xC98E	0x84ED	
+0xC98F	0x84EE	
+0xC990	0x84EF	
+0xC991	0x84F1	
+0xC992	0x84F2	
+0xC993	0x84F3	
+0xC994	0x84F4	
+0xC995	0x84F5	
+0xC996	0x84F6	
+0xC997	0x84F7	
+0xC998	0x84F8	
+0xC999	0x84F9	
+0xC99A	0x84FA	
+0xC99B	0x84FB	
+0xC99C	0x84FD	
+0xC99D	0x84FE	
+0xC99E	0x8500	
+0xC99F	0x8501	
+0xC9A0	0x8502	
+0xC9A1	0x4F1E	
+0xC9A2	0x6563	
+0xC9A3	0x6851	
+0xC9A4	0x55D3	
+0xC9A5	0x4E27	
+0xC9A6	0x6414	
+0xC9A7	0x9A9A	
+0xC9A8	0x626B	
+0xC9A9	0x5AC2	
+0xC9AA	0x745F	
+0xC9AB	0x8272	
+0xC9AC	0x6DA9	
+0xC9AD	0x68EE	
+0xC9AE	0x50E7	
+0xC9AF	0x838E	
+0xC9B0	0x7802	
+0xC9B1	0x6740	
+0xC9B2	0x5239	
+0xC9B3	0x6C99	
+0xC9B4	0x7EB1	
+0xC9B5	0x50BB	
+0xC9B6	0x5565	
+0xC9B7	0x715E	
+0xC9B8	0x7B5B	
+0xC9B9	0x6652	
+0xC9BA	0x73CA	
+0xC9BB	0x82EB	
+0xC9BC	0x6749	
+0xC9BD	0x5C71	
+0xC9BE	0x5220	
+0xC9BF	0x717D	
+0xC9C0	0x886B	
+0xC9C1	0x95EA	
+0xC9C2	0x9655	
+0xC9C3	0x64C5	
+0xC9C4	0x8D61	
+0xC9C5	0x81B3	
+0xC9C6	0x5584	
+0xC9C7	0x6C55	
+0xC9C8	0x6247	
+0xC9C9	0x7F2E	
+0xC9CA	0x5892	
+0xC9CB	0x4F24	
+0xC9CC	0x5546	
+0xC9CD	0x8D4F	
+0xC9CE	0x664C	
+0xC9CF	0x4E0A	
+0xC9D0	0x5C1A	
+0xC9D1	0x88F3	
+0xC9D2	0x68A2	
+0xC9D3	0x634E	
+0xC9D4	0x7A0D	
+0xC9D5	0x70E7	
+0xC9D6	0x828D	
+0xC9D7	0x52FA	
+0xC9D8	0x97F6	
+0xC9D9	0x5C11	
+0xC9DA	0x54E8	
+0xC9DB	0x90B5	
+0xC9DC	0x7ECD	
+0xC9DD	0x5962	
+0xC9DE	0x8D4A	
+0xC9DF	0x86C7	
+0xC9E0	0x820C	
+0xC9E1	0x820D	
+0xC9E2	0x8D66	
+0xC9E3	0x6444	
+0xC9E4	0x5C04	
+0xC9E5	0x6151	
+0xC9E6	0x6D89	
+0xC9E7	0x793E	
+0xC9E8	0x8BBE	
+0xC9E9	0x7837	
+0xC9EA	0x7533	
+0xC9EB	0x547B	
+0xC9EC	0x4F38	
+0xC9ED	0x8EAB	
+0xC9EE	0x6DF1	
+0xC9EF	0x5A20	
+0xC9F0	0x7EC5	
+0xC9F1	0x795E	
+0xC9F2	0x6C88	
+0xC9F3	0x5BA1	
+0xC9F4	0x5A76	
+0xC9F5	0x751A	
+0xC9F6	0x80BE	
+0xC9F7	0x614E	
+0xC9F8	0x6E17	
+0xC9F9	0x58F0	
+0xC9FA	0x751F	
+0xC9FB	0x7525	
+0xC9FC	0x7272	
+0xC9FD	0x5347	
+0xC9FE	0x7EF3	
+0xCA40	0x8503	
+0xCA41	0x8504	
+0xCA42	0x8505	
+0xCA43	0x8506	
+0xCA44	0x8507	
+0xCA45	0x8508	
+0xCA46	0x8509	
+0xCA47	0x850A	
+0xCA48	0x850B	
+0xCA49	0x850D	
+0xCA4A	0x850E	
+0xCA4B	0x850F	
+0xCA4C	0x8510	
+0xCA4D	0x8512	
+0xCA4E	0x8514	
+0xCA4F	0x8515	
+0xCA50	0x8516	
+0xCA51	0x8518	
+0xCA52	0x8519	
+0xCA53	0x851B	
+0xCA54	0x851C	
+0xCA55	0x851D	
+0xCA56	0x851E	
+0xCA57	0x8520	
+0xCA58	0x8522	
+0xCA59	0x8523	
+0xCA5A	0x8524	
+0xCA5B	0x8525	
+0xCA5C	0x8526	
+0xCA5D	0x8527	
+0xCA5E	0x8528	
+0xCA5F	0x8529	
+0xCA60	0x852A	
+0xCA61	0x852D	
+0xCA62	0x852E	
+0xCA63	0x852F	
+0xCA64	0x8530	
+0xCA65	0x8531	
+0xCA66	0x8532	
+0xCA67	0x8533	
+0xCA68	0x8534	
+0xCA69	0x8535	
+0xCA6A	0x8536	
+0xCA6B	0x853E	
+0xCA6C	0x853F	
+0xCA6D	0x8540	
+0xCA6E	0x8541	
+0xCA6F	0x8542	
+0xCA70	0x8544	
+0xCA71	0x8545	
+0xCA72	0x8546	
+0xCA73	0x8547	
+0xCA74	0x854B	
+0xCA75	0x854C	
+0xCA76	0x854D	
+0xCA77	0x854E	
+0xCA78	0x854F	
+0xCA79	0x8550	
+0xCA7A	0x8551	
+0xCA7B	0x8552	
+0xCA7C	0x8553	
+0xCA7D	0x8554	
+0xCA7E	0x8555	
+0xCA80	0x8557	
+0xCA81	0x8558	
+0xCA82	0x855A	
+0xCA83	0x855B	
+0xCA84	0x855C	
+0xCA85	0x855D	
+0xCA86	0x855F	
+0xCA87	0x8560	
+0xCA88	0x8561	
+0xCA89	0x8562	
+0xCA8A	0x8563	
+0xCA8B	0x8565	
+0xCA8C	0x8566	
+0xCA8D	0x8567	
+0xCA8E	0x8569	
+0xCA8F	0x856A	
+0xCA90	0x856B	
+0xCA91	0x856C	
+0xCA92	0x856D	
+0xCA93	0x856E	
+0xCA94	0x856F	
+0xCA95	0x8570	
+0xCA96	0x8571	
+0xCA97	0x8573	
+0xCA98	0x8575	
+0xCA99	0x8576	
+0xCA9A	0x8577	
+0xCA9B	0x8578	
+0xCA9C	0x857C	
+0xCA9D	0x857D	
+0xCA9E	0x857F	
+0xCA9F	0x8580	
+0xCAA0	0x8581	
+0xCAA1	0x7701	
+0xCAA2	0x76DB	
+0xCAA3	0x5269	
+0xCAA4	0x80DC	
+0xCAA5	0x5723	
+0xCAA6	0x5E08	
+0xCAA7	0x5931	
+0xCAA8	0x72EE	
+0xCAA9	0x65BD	
+0xCAAA	0x6E7F	
+0xCAAB	0x8BD7	
+0xCAAC	0x5C38	
+0xCAAD	0x8671	
+0xCAAE	0x5341	
+0xCAAF	0x77F3	
+0xCAB0	0x62FE	
+0xCAB1	0x65F6	
+0xCAB2	0x4EC0	
+0xCAB3	0x98DF	
+0xCAB4	0x8680	
+0xCAB5	0x5B9E	
+0xCAB6	0x8BC6	
+0xCAB7	0x53F2	
+0xCAB8	0x77E2	
+0xCAB9	0x4F7F	
+0xCABA	0x5C4E	
+0xCABB	0x9A76	
+0xCABC	0x59CB	
+0xCABD	0x5F0F	
+0xCABE	0x793A	
+0xCABF	0x58EB	
+0xCAC0	0x4E16	
+0xCAC1	0x67FF	
+0xCAC2	0x4E8B	
+0xCAC3	0x62ED	
+0xCAC4	0x8A93	
+0xCAC5	0x901D	
+0xCAC6	0x52BF	
+0xCAC7	0x662F	
+0xCAC8	0x55DC	
+0xCAC9	0x566C	
+0xCACA	0x9002	
+0xCACB	0x4ED5	
+0xCACC	0x4F8D	
+0xCACD	0x91CA	
+0xCACE	0x9970	
+0xCACF	0x6C0F	
+0xCAD0	0x5E02	
+0xCAD1	0x6043	
+0xCAD2	0x5BA4	
+0xCAD3	0x89C6	
+0xCAD4	0x8BD5	
+0xCAD5	0x6536	
+0xCAD6	0x624B	
+0xCAD7	0x9996	
+0xCAD8	0x5B88	
+0xCAD9	0x5BFF	
+0xCADA	0x6388	
+0xCADB	0x552E	
+0xCADC	0x53D7	
+0xCADD	0x7626	
+0xCADE	0x517D	
+0xCADF	0x852C	
+0xCAE0	0x67A2	
+0xCAE1	0x68B3	
+0xCAE2	0x6B8A	
+0xCAE3	0x6292	
+0xCAE4	0x8F93	
+0xCAE5	0x53D4	
+0xCAE6	0x8212	
+0xCAE7	0x6DD1	
+0xCAE8	0x758F	
+0xCAE9	0x4E66	
+0xCAEA	0x8D4E	
+0xCAEB	0x5B70	
+0xCAEC	0x719F	
+0xCAED	0x85AF	
+0xCAEE	0x6691	
+0xCAEF	0x66D9	
+0xCAF0	0x7F72	
+0xCAF1	0x8700	
+0xCAF2	0x9ECD	
+0xCAF3	0x9F20	
+0xCAF4	0x5C5E	
+0xCAF5	0x672F	
+0xCAF6	0x8FF0	
+0xCAF7	0x6811	
+0xCAF8	0x675F	
+0xCAF9	0x620D	
+0xCAFA	0x7AD6	
+0xCAFB	0x5885	
+0xCAFC	0x5EB6	
+0xCAFD	0x6570	
+0xCAFE	0x6F31	
+0xCB40	0x8582	
+0xCB41	0x8583	
+0xCB42	0x8586	
+0xCB43	0x8588	
+0xCB44	0x8589	
+0xCB45	0x858A	
+0xCB46	0x858B	
+0xCB47	0x858C	
+0xCB48	0x858D	
+0xCB49	0x858E	
+0xCB4A	0x8590	
+0xCB4B	0x8591	
+0xCB4C	0x8592	
+0xCB4D	0x8593	
+0xCB4E	0x8594	
+0xCB4F	0x8595	
+0xCB50	0x8596	
+0xCB51	0x8597	
+0xCB52	0x8598	
+0xCB53	0x8599	
+0xCB54	0x859A	
+0xCB55	0x859D	
+0xCB56	0x859E	
+0xCB57	0x859F	
+0xCB58	0x85A0	
+0xCB59	0x85A1	
+0xCB5A	0x85A2	
+0xCB5B	0x85A3	
+0xCB5C	0x85A5	
+0xCB5D	0x85A6	
+0xCB5E	0x85A7	
+0xCB5F	0x85A9	
+0xCB60	0x85AB	
+0xCB61	0x85AC	
+0xCB62	0x85AD	
+0xCB63	0x85B1	
+0xCB64	0x85B2	
+0xCB65	0x85B3	
+0xCB66	0x85B4	
+0xCB67	0x85B5	
+0xCB68	0x85B6	
+0xCB69	0x85B8	
+0xCB6A	0x85BA	
+0xCB6B	0x85BB	
+0xCB6C	0x85BC	
+0xCB6D	0x85BD	
+0xCB6E	0x85BE	
+0xCB6F	0x85BF	
+0xCB70	0x85C0	
+0xCB71	0x85C2	
+0xCB72	0x85C3	
+0xCB73	0x85C4	
+0xCB74	0x85C5	
+0xCB75	0x85C6	
+0xCB76	0x85C7	
+0xCB77	0x85C8	
+0xCB78	0x85CA	
+0xCB79	0x85CB	
+0xCB7A	0x85CC	
+0xCB7B	0x85CD	
+0xCB7C	0x85CE	
+0xCB7D	0x85D1	
+0xCB7E	0x85D2	
+0xCB80	0x85D4	
+0xCB81	0x85D6	
+0xCB82	0x85D7	
+0xCB83	0x85D8	
+0xCB84	0x85D9	
+0xCB85	0x85DA	
+0xCB86	0x85DB	
+0xCB87	0x85DD	
+0xCB88	0x85DE	
+0xCB89	0x85DF	
+0xCB8A	0x85E0	
+0xCB8B	0x85E1	
+0xCB8C	0x85E2	
+0xCB8D	0x85E3	
+0xCB8E	0x85E5	
+0xCB8F	0x85E6	
+0xCB90	0x85E7	
+0xCB91	0x85E8	
+0xCB92	0x85EA	
+0xCB93	0x85EB	
+0xCB94	0x85EC	
+0xCB95	0x85ED	
+0xCB96	0x85EE	
+0xCB97	0x85EF	
+0xCB98	0x85F0	
+0xCB99	0x85F1	
+0xCB9A	0x85F2	
+0xCB9B	0x85F3	
+0xCB9C	0x85F4	
+0xCB9D	0x85F5	
+0xCB9E	0x85F6	
+0xCB9F	0x85F7	
+0xCBA0	0x85F8	
+0xCBA1	0x6055	
+0xCBA2	0x5237	
+0xCBA3	0x800D	
+0xCBA4	0x6454	
+0xCBA5	0x8870	
+0xCBA6	0x7529	
+0xCBA7	0x5E05	
+0xCBA8	0x6813	
+0xCBA9	0x62F4	
+0xCBAA	0x971C	
+0xCBAB	0x53CC	
+0xCBAC	0x723D	
+0xCBAD	0x8C01	
+0xCBAE	0x6C34	
+0xCBAF	0x7761	
+0xCBB0	0x7A0E	
+0xCBB1	0x542E	
+0xCBB2	0x77AC	
+0xCBB3	0x987A	
+0xCBB4	0x821C	
+0xCBB5	0x8BF4	
+0xCBB6	0x7855	
+0xCBB7	0x6714	
+0xCBB8	0x70C1	
+0xCBB9	0x65AF	
+0xCBBA	0x6495	
+0xCBBB	0x5636	
+0xCBBC	0x601D	
+0xCBBD	0x79C1	
+0xCBBE	0x53F8	
+0xCBBF	0x4E1D	
+0xCBC0	0x6B7B	
+0xCBC1	0x8086	
+0xCBC2	0x5BFA	
+0xCBC3	0x55E3	
+0xCBC4	0x56DB	
+0xCBC5	0x4F3A	
+0xCBC6	0x4F3C	
+0xCBC7	0x9972	
+0xCBC8	0x5DF3	
+0xCBC9	0x677E	
+0xCBCA	0x8038	
+0xCBCB	0x6002	
+0xCBCC	0x9882	
+0xCBCD	0x9001	
+0xCBCE	0x5B8B	
+0xCBCF	0x8BBC	
+0xCBD0	0x8BF5	
+0xCBD1	0x641C	
+0xCBD2	0x8258	
+0xCBD3	0x64DE	
+0xCBD4	0x55FD	
+0xCBD5	0x82CF	
+0xCBD6	0x9165	
+0xCBD7	0x4FD7	
+0xCBD8	0x7D20	
+0xCBD9	0x901F	
+0xCBDA	0x7C9F	
+0xCBDB	0x50F3	
+0xCBDC	0x5851	
+0xCBDD	0x6EAF	
+0xCBDE	0x5BBF	
+0xCBDF	0x8BC9	
+0xCBE0	0x8083	
+0xCBE1	0x9178	
+0xCBE2	0x849C	
+0xCBE3	0x7B97	
+0xCBE4	0x867D	
+0xCBE5	0x968B	
+0xCBE6	0x968F	
+0xCBE7	0x7EE5	
+0xCBE8	0x9AD3	
+0xCBE9	0x788E	
+0xCBEA	0x5C81	
+0xCBEB	0x7A57	
+0xCBEC	0x9042	
+0xCBED	0x96A7	
+0xCBEE	0x795F	
+0xCBEF	0x5B59	
+0xCBF0	0x635F	
+0xCBF1	0x7B0B	
+0xCBF2	0x84D1	
+0xCBF3	0x68AD	
+0xCBF4	0x5506	
+0xCBF5	0x7F29	
+0xCBF6	0x7410	
+0xCBF7	0x7D22	
+0xCBF8	0x9501	
+0xCBF9	0x6240	
+0xCBFA	0x584C	
+0xCBFB	0x4ED6	
+0xCBFC	0x5B83	
+0xCBFD	0x5979	
+0xCBFE	0x5854	
+0xCC40	0x85F9	
+0xCC41	0x85FA	
+0xCC42	0x85FC	
+0xCC43	0x85FD	
+0xCC44	0x85FE	
+0xCC45	0x8600	
+0xCC46	0x8601	
+0xCC47	0x8602	
+0xCC48	0x8603	
+0xCC49	0x8604	
+0xCC4A	0x8606	
+0xCC4B	0x8607	
+0xCC4C	0x8608	
+0xCC4D	0x8609	
+0xCC4E	0x860A	
+0xCC4F	0x860B	
+0xCC50	0x860C	
+0xCC51	0x860D	
+0xCC52	0x860E	
+0xCC53	0x860F	
+0xCC54	0x8610	
+0xCC55	0x8612	
+0xCC56	0x8613	
+0xCC57	0x8614	
+0xCC58	0x8615	
+0xCC59	0x8617	
+0xCC5A	0x8618	
+0xCC5B	0x8619	
+0xCC5C	0x861A	
+0xCC5D	0x861B	
+0xCC5E	0x861C	
+0xCC5F	0x861D	
+0xCC60	0x861E	
+0xCC61	0x861F	
+0xCC62	0x8620	
+0xCC63	0x8621	
+0xCC64	0x8622	
+0xCC65	0x8623	
+0xCC66	0x8624	
+0xCC67	0x8625	
+0xCC68	0x8626	
+0xCC69	0x8628	
+0xCC6A	0x862A	
+0xCC6B	0x862B	
+0xCC6C	0x862C	
+0xCC6D	0x862D	
+0xCC6E	0x862E	
+0xCC6F	0x862F	
+0xCC70	0x8630	
+0xCC71	0x8631	
+0xCC72	0x8632	
+0xCC73	0x8633	
+0xCC74	0x8634	
+0xCC75	0x8635	
+0xCC76	0x8636	
+0xCC77	0x8637	
+0xCC78	0x8639	
+0xCC79	0x863A	
+0xCC7A	0x863B	
+0xCC7B	0x863D	
+0xCC7C	0x863E	
+0xCC7D	0x863F	
+0xCC7E	0x8640	
+0xCC80	0x8641	
+0xCC81	0x8642	
+0xCC82	0x8643	
+0xCC83	0x8644	
+0xCC84	0x8645	
+0xCC85	0x8646	
+0xCC86	0x8647	
+0xCC87	0x8648	
+0xCC88	0x8649	
+0xCC89	0x864A	
+0xCC8A	0x864B	
+0xCC8B	0x864C	
+0xCC8C	0x8652	
+0xCC8D	0x8653	
+0xCC8E	0x8655	
+0xCC8F	0x8656	
+0xCC90	0x8657	
+0xCC91	0x8658	
+0xCC92	0x8659	
+0xCC93	0x865B	
+0xCC94	0x865C	
+0xCC95	0x865D	
+0xCC96	0x865F	
+0xCC97	0x8660	
+0xCC98	0x8661	
+0xCC99	0x8663	
+0xCC9A	0x8664	
+0xCC9B	0x8665	
+0xCC9C	0x8666	
+0xCC9D	0x8667	
+0xCC9E	0x8668	
+0xCC9F	0x8669	
+0xCCA0	0x866A	
+0xCCA1	0x736D	
+0xCCA2	0x631E	
+0xCCA3	0x8E4B	
+0xCCA4	0x8E0F	
+0xCCA5	0x80CE	
+0xCCA6	0x82D4	
+0xCCA7	0x62AC	
+0xCCA8	0x53F0	
+0xCCA9	0x6CF0	
+0xCCAA	0x915E	
+0xCCAB	0x592A	
+0xCCAC	0x6001	
+0xCCAD	0x6C70	
+0xCCAE	0x574D	
+0xCCAF	0x644A	
+0xCCB0	0x8D2A	
+0xCCB1	0x762B	
+0xCCB2	0x6EE9	
+0xCCB3	0x575B	
+0xCCB4	0x6A80	
+0xCCB5	0x75F0	
+0xCCB6	0x6F6D	
+0xCCB7	0x8C2D	
+0xCCB8	0x8C08	
+0xCCB9	0x5766	
+0xCCBA	0x6BEF	
+0xCCBB	0x8892	
+0xCCBC	0x78B3	
+0xCCBD	0x63A2	
+0xCCBE	0x53F9	
+0xCCBF	0x70AD	
+0xCCC0	0x6C64	
+0xCCC1	0x5858	
+0xCCC2	0x642A	
+0xCCC3	0x5802	
+0xCCC4	0x68E0	
+0xCCC5	0x819B	
+0xCCC6	0x5510	
+0xCCC7	0x7CD6	
+0xCCC8	0x5018	
+0xCCC9	0x8EBA	
+0xCCCA	0x6DCC	
+0xCCCB	0x8D9F	
+0xCCCC	0x70EB	
+0xCCCD	0x638F	
+0xCCCE	0x6D9B	
+0xCCCF	0x6ED4	
+0xCCD0	0x7EE6	
+0xCCD1	0x8404	
+0xCCD2	0x6843	
+0xCCD3	0x9003	
+0xCCD4	0x6DD8	
+0xCCD5	0x9676	
+0xCCD6	0x8BA8	
+0xCCD7	0x5957	
+0xCCD8	0x7279	
+0xCCD9	0x85E4	
+0xCCDA	0x817E	
+0xCCDB	0x75BC	
+0xCCDC	0x8A8A	
+0xCCDD	0x68AF	
+0xCCDE	0x5254	
+0xCCDF	0x8E22	
+0xCCE0	0x9511	
+0xCCE1	0x63D0	
+0xCCE2	0x9898	
+0xCCE3	0x8E44	
+0xCCE4	0x557C	
+0xCCE5	0x4F53	
+0xCCE6	0x66FF	
+0xCCE7	0x568F	
+0xCCE8	0x60D5	
+0xCCE9	0x6D95	
+0xCCEA	0x5243	
+0xCCEB	0x5C49	
+0xCCEC	0x5929	
+0xCCED	0x6DFB	
+0xCCEE	0x586B	
+0xCCEF	0x7530	
+0xCCF0	0x751C	
+0xCCF1	0x606C	
+0xCCF2	0x8214	
+0xCCF3	0x8146	
+0xCCF4	0x6311	
+0xCCF5	0x6761	
+0xCCF6	0x8FE2	
+0xCCF7	0x773A	
+0xCCF8	0x8DF3	
+0xCCF9	0x8D34	
+0xCCFA	0x94C1	
+0xCCFB	0x5E16	
+0xCCFC	0x5385	
+0xCCFD	0x542C	
+0xCCFE	0x70C3	
+0xCD40	0x866D	
+0xCD41	0x866F	
+0xCD42	0x8670	
+0xCD43	0x8672	
+0xCD44	0x8673	
+0xCD45	0x8674	
+0xCD46	0x8675	
+0xCD47	0x8676	
+0xCD48	0x8677	
+0xCD49	0x8678	
+0xCD4A	0x8683	
+0xCD4B	0x8684	
+0xCD4C	0x8685	
+0xCD4D	0x8686	
+0xCD4E	0x8687	
+0xCD4F	0x8688	
+0xCD50	0x8689	
+0xCD51	0x868E	
+0xCD52	0x868F	
+0xCD53	0x8690	
+0xCD54	0x8691	
+0xCD55	0x8692	
+0xCD56	0x8694	
+0xCD57	0x8696	
+0xCD58	0x8697	
+0xCD59	0x8698	
+0xCD5A	0x8699	
+0xCD5B	0x869A	
+0xCD5C	0x869B	
+0xCD5D	0x869E	
+0xCD5E	0x869F	
+0xCD5F	0x86A0	
+0xCD60	0x86A1	
+0xCD61	0x86A2	
+0xCD62	0x86A5	
+0xCD63	0x86A6	
+0xCD64	0x86AB	
+0xCD65	0x86AD	
+0xCD66	0x86AE	
+0xCD67	0x86B2	
+0xCD68	0x86B3	
+0xCD69	0x86B7	
+0xCD6A	0x86B8	
+0xCD6B	0x86B9	
+0xCD6C	0x86BB	
+0xCD6D	0x86BC	
+0xCD6E	0x86BD	
+0xCD6F	0x86BE	
+0xCD70	0x86BF	
+0xCD71	0x86C1	
+0xCD72	0x86C2	
+0xCD73	0x86C3	
+0xCD74	0x86C5	
+0xCD75	0x86C8	
+0xCD76	0x86CC	
+0xCD77	0x86CD	
+0xCD78	0x86D2	
+0xCD79	0x86D3	
+0xCD7A	0x86D5	
+0xCD7B	0x86D6	
+0xCD7C	0x86D7	
+0xCD7D	0x86DA	
+0xCD7E	0x86DC	
+0xCD80	0x86DD	
+0xCD81	0x86E0	
+0xCD82	0x86E1	
+0xCD83	0x86E2	
+0xCD84	0x86E3	
+0xCD85	0x86E5	
+0xCD86	0x86E6	
+0xCD87	0x86E7	
+0xCD88	0x86E8	
+0xCD89	0x86EA	
+0xCD8A	0x86EB	
+0xCD8B	0x86EC	
+0xCD8C	0x86EF	
+0xCD8D	0x86F5	
+0xCD8E	0x86F6	
+0xCD8F	0x86F7	
+0xCD90	0x86FA	
+0xCD91	0x86FB	
+0xCD92	0x86FC	
+0xCD93	0x86FD	
+0xCD94	0x86FF	
+0xCD95	0x8701	
+0xCD96	0x8704	
+0xCD97	0x8705	
+0xCD98	0x8706	
+0xCD99	0x870B	
+0xCD9A	0x870C	
+0xCD9B	0x870E	
+0xCD9C	0x870F	
+0xCD9D	0x8710	
+0xCD9E	0x8711	
+0xCD9F	0x8714	
+0xCDA0	0x8716	
+0xCDA1	0x6C40	
+0xCDA2	0x5EF7	
+0xCDA3	0x505C	
+0xCDA4	0x4EAD	
+0xCDA5	0x5EAD	
+0xCDA6	0x633A	
+0xCDA7	0x8247	
+0xCDA8	0x901A	
+0xCDA9	0x6850	
+0xCDAA	0x916E	
+0xCDAB	0x77B3	
+0xCDAC	0x540C	
+0xCDAD	0x94DC	
+0xCDAE	0x5F64	
+0xCDAF	0x7AE5	
+0xCDB0	0x6876	
+0xCDB1	0x6345	
+0xCDB2	0x7B52	
+0xCDB3	0x7EDF	
+0xCDB4	0x75DB	
+0xCDB5	0x5077	
+0xCDB6	0x6295	
+0xCDB7	0x5934	
+0xCDB8	0x900F	
+0xCDB9	0x51F8	
+0xCDBA	0x79C3	
+0xCDBB	0x7A81	
+0xCDBC	0x56FE	
+0xCDBD	0x5F92	
+0xCDBE	0x9014	
+0xCDBF	0x6D82	
+0xCDC0	0x5C60	
+0xCDC1	0x571F	
+0xCDC2	0x5410	
+0xCDC3	0x5154	
+0xCDC4	0x6E4D	
+0xCDC5	0x56E2	
+0xCDC6	0x63A8	
+0xCDC7	0x9893	
+0xCDC8	0x817F	
+0xCDC9	0x8715	
+0xCDCA	0x892A	
+0xCDCB	0x9000	
+0xCDCC	0x541E	
+0xCDCD	0x5C6F	
+0xCDCE	0x81C0	
+0xCDCF	0x62D6	
+0xCDD0	0x6258	
+0xCDD1	0x8131	
+0xCDD2	0x9E35	
+0xCDD3	0x9640	
+0xCDD4	0x9A6E	
+0xCDD5	0x9A7C	
+0xCDD6	0x692D	
+0xCDD7	0x59A5	
+0xCDD8	0x62D3	
+0xCDD9	0x553E	
+0xCDDA	0x6316	
+0xCDDB	0x54C7	
+0xCDDC	0x86D9	
+0xCDDD	0x6D3C	
+0xCDDE	0x5A03	
+0xCDDF	0x74E6	
+0xCDE0	0x889C	
+0xCDE1	0x6B6A	
+0xCDE2	0x5916	
+0xCDE3	0x8C4C	
+0xCDE4	0x5F2F	
+0xCDE5	0x6E7E	
+0xCDE6	0x73A9	
+0xCDE7	0x987D	
+0xCDE8	0x4E38	
+0xCDE9	0x70F7	
+0xCDEA	0x5B8C	
+0xCDEB	0x7897	
+0xCDEC	0x633D	
+0xCDED	0x665A	
+0xCDEE	0x7696	
+0xCDEF	0x60CB	
+0xCDF0	0x5B9B	
+0xCDF1	0x5A49	
+0xCDF2	0x4E07	
+0xCDF3	0x8155	
+0xCDF4	0x6C6A	
+0xCDF5	0x738B	
+0xCDF6	0x4EA1	
+0xCDF7	0x6789	
+0xCDF8	0x7F51	
+0xCDF9	0x5F80	
+0xCDFA	0x65FA	
+0xCDFB	0x671B	
+0xCDFC	0x5FD8	
+0xCDFD	0x5984	
+0xCDFE	0x5A01	
+0xCE40	0x8719	
+0xCE41	0x871B	
+0xCE42	0x871D	
+0xCE43	0x871F	
+0xCE44	0x8720	
+0xCE45	0x8724	
+0xCE46	0x8726	
+0xCE47	0x8727	
+0xCE48	0x8728	
+0xCE49	0x872A	
+0xCE4A	0x872B	
+0xCE4B	0x872C	
+0xCE4C	0x872D	
+0xCE4D	0x872F	
+0xCE4E	0x8730	
+0xCE4F	0x8732	
+0xCE50	0x8733	
+0xCE51	0x8735	
+0xCE52	0x8736	
+0xCE53	0x8738	
+0xCE54	0x8739	
+0xCE55	0x873A	
+0xCE56	0x873C	
+0xCE57	0x873D	
+0xCE58	0x8740	
+0xCE59	0x8741	
+0xCE5A	0x8742	
+0xCE5B	0x8743	
+0xCE5C	0x8744	
+0xCE5D	0x8745	
+0xCE5E	0x8746	
+0xCE5F	0x874A	
+0xCE60	0x874B	
+0xCE61	0x874D	
+0xCE62	0x874F	
+0xCE63	0x8750	
+0xCE64	0x8751	
+0xCE65	0x8752	
+0xCE66	0x8754	
+0xCE67	0x8755	
+0xCE68	0x8756	
+0xCE69	0x8758	
+0xCE6A	0x875A	
+0xCE6B	0x875B	
+0xCE6C	0x875C	
+0xCE6D	0x875D	
+0xCE6E	0x875E	
+0xCE6F	0x875F	
+0xCE70	0x8761	
+0xCE71	0x8762	
+0xCE72	0x8766	
+0xCE73	0x8767	
+0xCE74	0x8768	
+0xCE75	0x8769	
+0xCE76	0x876A	
+0xCE77	0x876B	
+0xCE78	0x876C	
+0xCE79	0x876D	
+0xCE7A	0x876F	
+0xCE7B	0x8771	
+0xCE7C	0x8772	
+0xCE7D	0x8773	
+0xCE7E	0x8775	
+0xCE80	0x8777	
+0xCE81	0x8778	
+0xCE82	0x8779	
+0xCE83	0x877A	
+0xCE84	0x877F	
+0xCE85	0x8780	
+0xCE86	0x8781	
+0xCE87	0x8784	
+0xCE88	0x8786	
+0xCE89	0x8787	
+0xCE8A	0x8789	
+0xCE8B	0x878A	
+0xCE8C	0x878C	
+0xCE8D	0x878E	
+0xCE8E	0x878F	
+0xCE8F	0x8790	
+0xCE90	0x8791	
+0xCE91	0x8792	
+0xCE92	0x8794	
+0xCE93	0x8795	
+0xCE94	0x8796	
+0xCE95	0x8798	
+0xCE96	0x8799	
+0xCE97	0x879A	
+0xCE98	0x879B	
+0xCE99	0x879C	
+0xCE9A	0x879D	
+0xCE9B	0x879E	
+0xCE9C	0x87A0	
+0xCE9D	0x87A1	
+0xCE9E	0x87A2	
+0xCE9F	0x87A3	
+0xCEA0	0x87A4	
+0xCEA1	0x5DCD	
+0xCEA2	0x5FAE	
+0xCEA3	0x5371	
+0xCEA4	0x97E6	
+0xCEA5	0x8FDD	
+0xCEA6	0x6845	
+0xCEA7	0x56F4	
+0xCEA8	0x552F	
+0xCEA9	0x60DF	
+0xCEAA	0x4E3A	
+0xCEAB	0x6F4D	
+0xCEAC	0x7EF4	
+0xCEAD	0x82C7	
+0xCEAE	0x840E	
+0xCEAF	0x59D4	
+0xCEB0	0x4F1F	
+0xCEB1	0x4F2A	
+0xCEB2	0x5C3E	
+0xCEB3	0x7EAC	
+0xCEB4	0x672A	
+0xCEB5	0x851A	
+0xCEB6	0x5473	
+0xCEB7	0x754F	
+0xCEB8	0x80C3	
+0xCEB9	0x5582	
+0xCEBA	0x9B4F	
+0xCEBB	0x4F4D	
+0xCEBC	0x6E2D	
+0xCEBD	0x8C13	
+0xCEBE	0x5C09	
+0xCEBF	0x6170	
+0xCEC0	0x536B	
+0xCEC1	0x761F	
+0xCEC2	0x6E29	
+0xCEC3	0x868A	
+0xCEC4	0x6587	
+0xCEC5	0x95FB	
+0xCEC6	0x7EB9	
+0xCEC7	0x543B	
+0xCEC8	0x7A33	
+0xCEC9	0x7D0A	
+0xCECA	0x95EE	
+0xCECB	0x55E1	
+0xCECC	0x7FC1	
+0xCECD	0x74EE	
+0xCECE	0x631D	
+0xCECF	0x8717	
+0xCED0	0x6DA1	
+0xCED1	0x7A9D	
+0xCED2	0x6211	
+0xCED3	0x65A1	
+0xCED4	0x5367	
+0xCED5	0x63E1	
+0xCED6	0x6C83	
+0xCED7	0x5DEB	
+0xCED8	0x545C	
+0xCED9	0x94A8	
+0xCEDA	0x4E4C	
+0xCEDB	0x6C61	
+0xCEDC	0x8BEC	
+0xCEDD	0x5C4B	
+0xCEDE	0x65E0	
+0xCEDF	0x829C	
+0xCEE0	0x68A7	
+0xCEE1	0x543E	
+0xCEE2	0x5434	
+0xCEE3	0x6BCB	
+0xCEE4	0x6B66	
+0xCEE5	0x4E94	
+0xCEE6	0x6342	
+0xCEE7	0x5348	
+0xCEE8	0x821E	
+0xCEE9	0x4F0D	
+0xCEEA	0x4FAE	
+0xCEEB	0x575E	
+0xCEEC	0x620A	
+0xCEED	0x96FE	
+0xCEEE	0x6664	
+0xCEEF	0x7269	
+0xCEF0	0x52FF	
+0xCEF1	0x52A1	
+0xCEF2	0x609F	
+0xCEF3	0x8BEF	
+0xCEF4	0x6614	
+0xCEF5	0x7199	
+0xCEF6	0x6790	
+0xCEF7	0x897F	
+0xCEF8	0x7852	
+0xCEF9	0x77FD	
+0xCEFA	0x6670	
+0xCEFB	0x563B	
+0xCEFC	0x5438	
+0xCEFD	0x9521	
+0xCEFE	0x727A	
+0xCF40	0x87A5	
+0xCF41	0x87A6	
+0xCF42	0x87A7	
+0xCF43	0x87A9	
+0xCF44	0x87AA	
+0xCF45	0x87AE	
+0xCF46	0x87B0	
+0xCF47	0x87B1	
+0xCF48	0x87B2	
+0xCF49	0x87B4	
+0xCF4A	0x87B6	
+0xCF4B	0x87B7	
+0xCF4C	0x87B8	
+0xCF4D	0x87B9	
+0xCF4E	0x87BB	
+0xCF4F	0x87BC	
+0xCF50	0x87BE	
+0xCF51	0x87BF	
+0xCF52	0x87C1	
+0xCF53	0x87C2	
+0xCF54	0x87C3	
+0xCF55	0x87C4	
+0xCF56	0x87C5	
+0xCF57	0x87C7	
+0xCF58	0x87C8	
+0xCF59	0x87C9	
+0xCF5A	0x87CC	
+0xCF5B	0x87CD	
+0xCF5C	0x87CE	
+0xCF5D	0x87CF	
+0xCF5E	0x87D0	
+0xCF5F	0x87D4	
+0xCF60	0x87D5	
+0xCF61	0x87D6	
+0xCF62	0x87D7	
+0xCF63	0x87D8	
+0xCF64	0x87D9	
+0xCF65	0x87DA	
+0xCF66	0x87DC	
+0xCF67	0x87DD	
+0xCF68	0x87DE	
+0xCF69	0x87DF	
+0xCF6A	0x87E1	
+0xCF6B	0x87E2	
+0xCF6C	0x87E3	
+0xCF6D	0x87E4	
+0xCF6E	0x87E6	
+0xCF6F	0x87E7	
+0xCF70	0x87E8	
+0xCF71	0x87E9	
+0xCF72	0x87EB	
+0xCF73	0x87EC	
+0xCF74	0x87ED	
+0xCF75	0x87EF	
+0xCF76	0x87F0	
+0xCF77	0x87F1	
+0xCF78	0x87F2	
+0xCF79	0x87F3	
+0xCF7A	0x87F4	
+0xCF7B	0x87F5	
+0xCF7C	0x87F6	
+0xCF7D	0x87F7	
+0xCF7E	0x87F8	
+0xCF80	0x87FA	
+0xCF81	0x87FB	
+0xCF82	0x87FC	
+0xCF83	0x87FD	
+0xCF84	0x87FF	
+0xCF85	0x8800	
+0xCF86	0x8801	
+0xCF87	0x8802	
+0xCF88	0x8804	
+0xCF89	0x8805	
+0xCF8A	0x8806	
+0xCF8B	0x8807	
+0xCF8C	0x8808	
+0xCF8D	0x8809	
+0xCF8E	0x880B	
+0xCF8F	0x880C	
+0xCF90	0x880D	
+0xCF91	0x880E	
+0xCF92	0x880F	
+0xCF93	0x8810	
+0xCF94	0x8811	
+0xCF95	0x8812	
+0xCF96	0x8814	
+0xCF97	0x8817	
+0xCF98	0x8818	
+0xCF99	0x8819	
+0xCF9A	0x881A	
+0xCF9B	0x881C	
+0xCF9C	0x881D	
+0xCF9D	0x881E	
+0xCF9E	0x881F	
+0xCF9F	0x8820	
+0xCFA0	0x8823	
+0xCFA1	0x7A00	
+0xCFA2	0x606F	
+0xCFA3	0x5E0C	
+0xCFA4	0x6089	
+0xCFA5	0x819D	
+0xCFA6	0x5915	
+0xCFA7	0x60DC	
+0xCFA8	0x7184	
+0xCFA9	0x70EF	
+0xCFAA	0x6EAA	
+0xCFAB	0x6C50	
+0xCFAC	0x7280	
+0xCFAD	0x6A84	
+0xCFAE	0x88AD	
+0xCFAF	0x5E2D	
+0xCFB0	0x4E60	
+0xCFB1	0x5AB3	
+0xCFB2	0x559C	
+0xCFB3	0x94E3	
+0xCFB4	0x6D17	
+0xCFB5	0x7CFB	
+0xCFB6	0x9699	
+0xCFB7	0x620F	
+0xCFB8	0x7EC6	
+0xCFB9	0x778E	
+0xCFBA	0x867E	
+0xCFBB	0x5323	
+0xCFBC	0x971E	
+0xCFBD	0x8F96	
+0xCFBE	0x6687	
+0xCFBF	0x5CE1	
+0xCFC0	0x4FA0	
+0xCFC1	0x72ED	
+0xCFC2	0x4E0B	
+0xCFC3	0x53A6	
+0xCFC4	0x590F	
+0xCFC5	0x5413	
+0xCFC6	0x6380	
+0xCFC7	0x9528	
+0xCFC8	0x5148	
+0xCFC9	0x4ED9	
+0xCFCA	0x9C9C	
+0xCFCB	0x7EA4	
+0xCFCC	0x54B8	
+0xCFCD	0x8D24	
+0xCFCE	0x8854	
+0xCFCF	0x8237	
+0xCFD0	0x95F2	
+0xCFD1	0x6D8E	
+0xCFD2	0x5F26	
+0xCFD3	0x5ACC	
+0xCFD4	0x663E	
+0xCFD5	0x9669	
+0xCFD6	0x73B0	
+0xCFD7	0x732E	
+0xCFD8	0x53BF	
+0xCFD9	0x817A	
+0xCFDA	0x9985	
+0xCFDB	0x7FA1	
+0xCFDC	0x5BAA	
+0xCFDD	0x9677	
+0xCFDE	0x9650	
+0xCFDF	0x7EBF	
+0xCFE0	0x76F8	
+0xCFE1	0x53A2	
+0xCFE2	0x9576	
+0xCFE3	0x9999	
+0xCFE4	0x7BB1	
+0xCFE5	0x8944	
+0xCFE6	0x6E58	
+0xCFE7	0x4E61	
+0xCFE8	0x7FD4	
+0xCFE9	0x7965	
+0xCFEA	0x8BE6	
+0xCFEB	0x60F3	
+0xCFEC	0x54CD	
+0xCFED	0x4EAB	
+0xCFEE	0x9879	
+0xCFEF	0x5DF7	
+0xCFF0	0x6A61	
+0xCFF1	0x50CF	
+0xCFF2	0x5411	
+0xCFF3	0x8C61	
+0xCFF4	0x8427	
+0xCFF5	0x785D	
+0xCFF6	0x9704	
+0xCFF7	0x524A	
+0xCFF8	0x54EE	
+0xCFF9	0x56A3	
+0xCFFA	0x9500	
+0xCFFB	0x6D88	
+0xCFFC	0x5BB5	
+0xCFFD	0x6DC6	
+0xCFFE	0x6653	
+0xD040	0x8824	
+0xD041	0x8825	
+0xD042	0x8826	
+0xD043	0x8827	
+0xD044	0x8828	
+0xD045	0x8829	
+0xD046	0x882A	
+0xD047	0x882B	
+0xD048	0x882C	
+0xD049	0x882D	
+0xD04A	0x882E	
+0xD04B	0x882F	
+0xD04C	0x8830	
+0xD04D	0x8831	
+0xD04E	0x8833	
+0xD04F	0x8834	
+0xD050	0x8835	
+0xD051	0x8836	
+0xD052	0x8837	
+0xD053	0x8838	
+0xD054	0x883A	
+0xD055	0x883B	
+0xD056	0x883D	
+0xD057	0x883E	
+0xD058	0x883F	
+0xD059	0x8841	
+0xD05A	0x8842	
+0xD05B	0x8843	
+0xD05C	0x8846	
+0xD05D	0x8847	
+0xD05E	0x8848	
+0xD05F	0x8849	
+0xD060	0x884A	
+0xD061	0x884B	
+0xD062	0x884E	
+0xD063	0x884F	
+0xD064	0x8850	
+0xD065	0x8851	
+0xD066	0x8852	
+0xD067	0x8853	
+0xD068	0x8855	
+0xD069	0x8856	
+0xD06A	0x8858	
+0xD06B	0x885A	
+0xD06C	0x885B	
+0xD06D	0x885C	
+0xD06E	0x885D	
+0xD06F	0x885E	
+0xD070	0x885F	
+0xD071	0x8860	
+0xD072	0x8866	
+0xD073	0x8867	
+0xD074	0x886A	
+0xD075	0x886D	
+0xD076	0x886F	
+0xD077	0x8871	
+0xD078	0x8873	
+0xD079	0x8874	
+0xD07A	0x8875	
+0xD07B	0x8876	
+0xD07C	0x8878	
+0xD07D	0x8879	
+0xD07E	0x887A	
+0xD080	0x887B	
+0xD081	0x887C	
+0xD082	0x8880	
+0xD083	0x8883	
+0xD084	0x8886	
+0xD085	0x8887	
+0xD086	0x8889	
+0xD087	0x888A	
+0xD088	0x888C	
+0xD089	0x888E	
+0xD08A	0x888F	
+0xD08B	0x8890	
+0xD08C	0x8891	
+0xD08D	0x8893	
+0xD08E	0x8894	
+0xD08F	0x8895	
+0xD090	0x8897	
+0xD091	0x8898	
+0xD092	0x8899	
+0xD093	0x889A	
+0xD094	0x889B	
+0xD095	0x889D	
+0xD096	0x889E	
+0xD097	0x889F	
+0xD098	0x88A0	
+0xD099	0x88A1	
+0xD09A	0x88A3	
+0xD09B	0x88A5	
+0xD09C	0x88A6	
+0xD09D	0x88A7	
+0xD09E	0x88A8	
+0xD09F	0x88A9	
+0xD0A0	0x88AA	
+0xD0A1	0x5C0F	
+0xD0A2	0x5B5D	
+0xD0A3	0x6821	
+0xD0A4	0x8096	
+0xD0A5	0x5578	
+0xD0A6	0x7B11	
+0xD0A7	0x6548	
+0xD0A8	0x6954	
+0xD0A9	0x4E9B	
+0xD0AA	0x6B47	
+0xD0AB	0x874E	
+0xD0AC	0x978B	
+0xD0AD	0x534F	
+0xD0AE	0x631F	
+0xD0AF	0x643A	
+0xD0B0	0x90AA	
+0xD0B1	0x659C	
+0xD0B2	0x80C1	
+0xD0B3	0x8C10	
+0xD0B4	0x5199	
+0xD0B5	0x68B0	
+0xD0B6	0x5378	
+0xD0B7	0x87F9	
+0xD0B8	0x61C8	
+0xD0B9	0x6CC4	
+0xD0BA	0x6CFB	
+0xD0BB	0x8C22	
+0xD0BC	0x5C51	
+0xD0BD	0x85AA	
+0xD0BE	0x82AF	
+0xD0BF	0x950C	
+0xD0C0	0x6B23	
+0xD0C1	0x8F9B	
+0xD0C2	0x65B0	
+0xD0C3	0x5FFB	
+0xD0C4	0x5FC3	
+0xD0C5	0x4FE1	
+0xD0C6	0x8845	
+0xD0C7	0x661F	
+0xD0C8	0x8165	
+0xD0C9	0x7329	
+0xD0CA	0x60FA	
+0xD0CB	0x5174	
+0xD0CC	0x5211	
+0xD0CD	0x578B	
+0xD0CE	0x5F62	
+0xD0CF	0x90A2	
+0xD0D0	0x884C	
+0xD0D1	0x9192	
+0xD0D2	0x5E78	
+0xD0D3	0x674F	
+0xD0D4	0x6027	
+0xD0D5	0x59D3	
+0xD0D6	0x5144	
+0xD0D7	0x51F6	
+0xD0D8	0x80F8	
+0xD0D9	0x5308	
+0xD0DA	0x6C79	
+0xD0DB	0x96C4	
+0xD0DC	0x718A	
+0xD0DD	0x4F11	
+0xD0DE	0x4FEE	
+0xD0DF	0x7F9E	
+0xD0E0	0x673D	
+0xD0E1	0x55C5	
+0xD0E2	0x9508	
+0xD0E3	0x79C0	
+0xD0E4	0x8896	
+0xD0E5	0x7EE3	
+0xD0E6	0x589F	
+0xD0E7	0x620C	
+0xD0E8	0x9700	
+0xD0E9	0x865A	
+0xD0EA	0x5618	
+0xD0EB	0x987B	
+0xD0EC	0x5F90	
+0xD0ED	0x8BB8	
+0xD0EE	0x84C4	
+0xD0EF	0x9157	
+0xD0F0	0x53D9	
+0xD0F1	0x65ED	
+0xD0F2	0x5E8F	
+0xD0F3	0x755C	
+0xD0F4	0x6064	
+0xD0F5	0x7D6E	
+0xD0F6	0x5A7F	
+0xD0F7	0x7EEA	
+0xD0F8	0x7EED	
+0xD0F9	0x8F69	
+0xD0FA	0x55A7	
+0xD0FB	0x5BA3	
+0xD0FC	0x60AC	
+0xD0FD	0x65CB	
+0xD0FE	0x7384	
+0xD140	0x88AC	
+0xD141	0x88AE	
+0xD142	0x88AF	
+0xD143	0x88B0	
+0xD144	0x88B2	
+0xD145	0x88B3	
+0xD146	0x88B4	
+0xD147	0x88B5	
+0xD148	0x88B6	
+0xD149	0x88B8	
+0xD14A	0x88B9	
+0xD14B	0x88BA	
+0xD14C	0x88BB	
+0xD14D	0x88BD	
+0xD14E	0x88BE	
+0xD14F	0x88BF	
+0xD150	0x88C0	
+0xD151	0x88C3	
+0xD152	0x88C4	
+0xD153	0x88C7	
+0xD154	0x88C8	
+0xD155	0x88CA	
+0xD156	0x88CB	
+0xD157	0x88CC	
+0xD158	0x88CD	
+0xD159	0x88CF	
+0xD15A	0x88D0	
+0xD15B	0x88D1	
+0xD15C	0x88D3	
+0xD15D	0x88D6	
+0xD15E	0x88D7	
+0xD15F	0x88DA	
+0xD160	0x88DB	
+0xD161	0x88DC	
+0xD162	0x88DD	
+0xD163	0x88DE	
+0xD164	0x88E0	
+0xD165	0x88E1	
+0xD166	0x88E6	
+0xD167	0x88E7	
+0xD168	0x88E9	
+0xD169	0x88EA	
+0xD16A	0x88EB	
+0xD16B	0x88EC	
+0xD16C	0x88ED	
+0xD16D	0x88EE	
+0xD16E	0x88EF	
+0xD16F	0x88F2	
+0xD170	0x88F5	
+0xD171	0x88F6	
+0xD172	0x88F7	
+0xD173	0x88FA	
+0xD174	0x88FB	
+0xD175	0x88FD	
+0xD176	0x88FF	
+0xD177	0x8900	
+0xD178	0x8901	
+0xD179	0x8903	
+0xD17A	0x8904	
+0xD17B	0x8905	
+0xD17C	0x8906	
+0xD17D	0x8907	
+0xD17E	0x8908	
+0xD180	0x8909	
+0xD181	0x890B	
+0xD182	0x890C	
+0xD183	0x890D	
+0xD184	0x890E	
+0xD185	0x890F	
+0xD186	0x8911	
+0xD187	0x8914	
+0xD188	0x8915	
+0xD189	0x8916	
+0xD18A	0x8917	
+0xD18B	0x8918	
+0xD18C	0x891C	
+0xD18D	0x891D	
+0xD18E	0x891E	
+0xD18F	0x891F	
+0xD190	0x8920	
+0xD191	0x8922	
+0xD192	0x8923	
+0xD193	0x8924	
+0xD194	0x8926	
+0xD195	0x8927	
+0xD196	0x8928	
+0xD197	0x8929	
+0xD198	0x892C	
+0xD199	0x892D	
+0xD19A	0x892E	
+0xD19B	0x892F	
+0xD19C	0x8931	
+0xD19D	0x8932	
+0xD19E	0x8933	
+0xD19F	0x8935	
+0xD1A0	0x8937	
+0xD1A1	0x9009	
+0xD1A2	0x7663	
+0xD1A3	0x7729	
+0xD1A4	0x7EDA	
+0xD1A5	0x9774	
+0xD1A6	0x859B	
+0xD1A7	0x5B66	
+0xD1A8	0x7A74	
+0xD1A9	0x96EA	
+0xD1AA	0x8840	
+0xD1AB	0x52CB	
+0xD1AC	0x718F	
+0xD1AD	0x5FAA	
+0xD1AE	0x65EC	
+0xD1AF	0x8BE2	
+0xD1B0	0x5BFB	
+0xD1B1	0x9A6F	
+0xD1B2	0x5DE1	
+0xD1B3	0x6B89	
+0xD1B4	0x6C5B	
+0xD1B5	0x8BAD	
+0xD1B6	0x8BAF	
+0xD1B7	0x900A	
+0xD1B8	0x8FC5	
+0xD1B9	0x538B	
+0xD1BA	0x62BC	
+0xD1BB	0x9E26	
+0xD1BC	0x9E2D	
+0xD1BD	0x5440	
+0xD1BE	0x4E2B	
+0xD1BF	0x82BD	
+0xD1C0	0x7259	
+0xD1C1	0x869C	
+0xD1C2	0x5D16	
+0xD1C3	0x8859	
+0xD1C4	0x6DAF	
+0xD1C5	0x96C5	
+0xD1C6	0x54D1	
+0xD1C7	0x4E9A	
+0xD1C8	0x8BB6	
+0xD1C9	0x7109	
+0xD1CA	0x54BD	
+0xD1CB	0x9609	
+0xD1CC	0x70DF	
+0xD1CD	0x6DF9	
+0xD1CE	0x76D0	
+0xD1CF	0x4E25	
+0xD1D0	0x7814	
+0xD1D1	0x8712	
+0xD1D2	0x5CA9	
+0xD1D3	0x5EF6	
+0xD1D4	0x8A00	
+0xD1D5	0x989C	
+0xD1D6	0x960E	
+0xD1D7	0x708E	
+0xD1D8	0x6CBF	
+0xD1D9	0x5944	
+0xD1DA	0x63A9	
+0xD1DB	0x773C	
+0xD1DC	0x884D	
+0xD1DD	0x6F14	
+0xD1DE	0x8273	
+0xD1DF	0x5830	
+0xD1E0	0x71D5	
+0xD1E1	0x538C	
+0xD1E2	0x781A	
+0xD1E3	0x96C1	
+0xD1E4	0x5501	
+0xD1E5	0x5F66	
+0xD1E6	0x7130	
+0xD1E7	0x5BB4	
+0xD1E8	0x8C1A	
+0xD1E9	0x9A8C	
+0xD1EA	0x6B83	
+0xD1EB	0x592E	
+0xD1EC	0x9E2F	
+0xD1ED	0x79E7	
+0xD1EE	0x6768	
+0xD1EF	0x626C	
+0xD1F0	0x4F6F	
+0xD1F1	0x75A1	
+0xD1F2	0x7F8A	
+0xD1F3	0x6D0B	
+0xD1F4	0x9633	
+0xD1F5	0x6C27	
+0xD1F6	0x4EF0	
+0xD1F7	0x75D2	
+0xD1F8	0x517B	
+0xD1F9	0x6837	
+0xD1FA	0x6F3E	
+0xD1FB	0x9080	
+0xD1FC	0x8170	
+0xD1FD	0x5996	
+0xD1FE	0x7476	
+0xD240	0x8938	
+0xD241	0x8939	
+0xD242	0x893A	
+0xD243	0x893B	
+0xD244	0x893C	
+0xD245	0x893D	
+0xD246	0x893E	
+0xD247	0x893F	
+0xD248	0x8940	
+0xD249	0x8942	
+0xD24A	0x8943	
+0xD24B	0x8945	
+0xD24C	0x8946	
+0xD24D	0x8947	
+0xD24E	0x8948	
+0xD24F	0x8949	
+0xD250	0x894A	
+0xD251	0x894B	
+0xD252	0x894C	
+0xD253	0x894D	
+0xD254	0x894E	
+0xD255	0x894F	
+0xD256	0x8950	
+0xD257	0x8951	
+0xD258	0x8952	
+0xD259	0x8953	
+0xD25A	0x8954	
+0xD25B	0x8955	
+0xD25C	0x8956	
+0xD25D	0x8957	
+0xD25E	0x8958	
+0xD25F	0x8959	
+0xD260	0x895A	
+0xD261	0x895B	
+0xD262	0x895C	
+0xD263	0x895D	
+0xD264	0x8960	
+0xD265	0x8961	
+0xD266	0x8962	
+0xD267	0x8963	
+0xD268	0x8964	
+0xD269	0x8965	
+0xD26A	0x8967	
+0xD26B	0x8968	
+0xD26C	0x8969	
+0xD26D	0x896A	
+0xD26E	0x896B	
+0xD26F	0x896C	
+0xD270	0x896D	
+0xD271	0x896E	
+0xD272	0x896F	
+0xD273	0x8970	
+0xD274	0x8971	
+0xD275	0x8972	
+0xD276	0x8973	
+0xD277	0x8974	
+0xD278	0x8975	
+0xD279	0x8976	
+0xD27A	0x8977	
+0xD27B	0x8978	
+0xD27C	0x8979	
+0xD27D	0x897A	
+0xD27E	0x897C	
+0xD280	0x897D	
+0xD281	0x897E	
+0xD282	0x8980	
+0xD283	0x8982	
+0xD284	0x8984	
+0xD285	0x8985	
+0xD286	0x8987	
+0xD287	0x8988	
+0xD288	0x8989	
+0xD289	0x898A	
+0xD28A	0x898B	
+0xD28B	0x898C	
+0xD28C	0x898D	
+0xD28D	0x898E	
+0xD28E	0x898F	
+0xD28F	0x8990	
+0xD290	0x8991	
+0xD291	0x8992	
+0xD292	0x8993	
+0xD293	0x8994	
+0xD294	0x8995	
+0xD295	0x8996	
+0xD296	0x8997	
+0xD297	0x8998	
+0xD298	0x8999	
+0xD299	0x899A	
+0xD29A	0x899B	
+0xD29B	0x899C	
+0xD29C	0x899D	
+0xD29D	0x899E	
+0xD29E	0x899F	
+0xD29F	0x89A0	
+0xD2A0	0x89A1	
+0xD2A1	0x6447	
+0xD2A2	0x5C27	
+0xD2A3	0x9065	
+0xD2A4	0x7A91	
+0xD2A5	0x8C23	
+0xD2A6	0x59DA	
+0xD2A7	0x54AC	
+0xD2A8	0x8200	
+0xD2A9	0x836F	
+0xD2AA	0x8981	
+0xD2AB	0x8000	
+0xD2AC	0x6930	
+0xD2AD	0x564E	
+0xD2AE	0x8036	
+0xD2AF	0x7237	
+0xD2B0	0x91CE	
+0xD2B1	0x51B6	
+0xD2B2	0x4E5F	
+0xD2B3	0x9875	
+0xD2B4	0x6396	
+0xD2B5	0x4E1A	
+0xD2B6	0x53F6	
+0xD2B7	0x66F3	
+0xD2B8	0x814B	
+0xD2B9	0x591C	
+0xD2BA	0x6DB2	
+0xD2BB	0x4E00	
+0xD2BC	0x58F9	
+0xD2BD	0x533B	
+0xD2BE	0x63D6	
+0xD2BF	0x94F1	
+0xD2C0	0x4F9D	
+0xD2C1	0x4F0A	
+0xD2C2	0x8863	
+0xD2C3	0x9890	
+0xD2C4	0x5937	
+0xD2C5	0x9057	
+0xD2C6	0x79FB	
+0xD2C7	0x4EEA	
+0xD2C8	0x80F0	
+0xD2C9	0x7591	
+0xD2CA	0x6C82	
+0xD2CB	0x5B9C	
+0xD2CC	0x59E8	
+0xD2CD	0x5F5D	
+0xD2CE	0x6905	
+0xD2CF	0x8681	
+0xD2D0	0x501A	
+0xD2D1	0x5DF2	
+0xD2D2	0x4E59	
+0xD2D3	0x77E3	
+0xD2D4	0x4EE5	
+0xD2D5	0x827A	
+0xD2D6	0x6291	
+0xD2D7	0x6613	
+0xD2D8	0x9091	
+0xD2D9	0x5C79	
+0xD2DA	0x4EBF	
+0xD2DB	0x5F79	
+0xD2DC	0x81C6	
+0xD2DD	0x9038	
+0xD2DE	0x8084	
+0xD2DF	0x75AB	
+0xD2E0	0x4EA6	
+0xD2E1	0x88D4	
+0xD2E2	0x610F	
+0xD2E3	0x6BC5	
+0xD2E4	0x5FC6	
+0xD2E5	0x4E49	
+0xD2E6	0x76CA	
+0xD2E7	0x6EA2	
+0xD2E8	0x8BE3	
+0xD2E9	0x8BAE	
+0xD2EA	0x8C0A	
+0xD2EB	0x8BD1	
+0xD2EC	0x5F02	
+0xD2ED	0x7FFC	
+0xD2EE	0x7FCC	
+0xD2EF	0x7ECE	
+0xD2F0	0x8335	
+0xD2F1	0x836B	
+0xD2F2	0x56E0	
+0xD2F3	0x6BB7	
+0xD2F4	0x97F3	
+0xD2F5	0x9634	
+0xD2F6	0x59FB	
+0xD2F7	0x541F	
+0xD2F8	0x94F6	
+0xD2F9	0x6DEB	
+0xD2FA	0x5BC5	
+0xD2FB	0x996E	
+0xD2FC	0x5C39	
+0xD2FD	0x5F15	
+0xD2FE	0x9690	
+0xD340	0x89A2	
+0xD341	0x89A3	
+0xD342	0x89A4	
+0xD343	0x89A5	
+0xD344	0x89A6	
+0xD345	0x89A7	
+0xD346	0x89A8	
+0xD347	0x89A9	
+0xD348	0x89AA	
+0xD349	0x89AB	
+0xD34A	0x89AC	
+0xD34B	0x89AD	
+0xD34C	0x89AE	
+0xD34D	0x89AF	
+0xD34E	0x89B0	
+0xD34F	0x89B1	
+0xD350	0x89B2	
+0xD351	0x89B3	
+0xD352	0x89B4	
+0xD353	0x89B5	
+0xD354	0x89B6	
+0xD355	0x89B7	
+0xD356	0x89B8	
+0xD357	0x89B9	
+0xD358	0x89BA	
+0xD359	0x89BB	
+0xD35A	0x89BC	
+0xD35B	0x89BD	
+0xD35C	0x89BE	
+0xD35D	0x89BF	
+0xD35E	0x89C0	
+0xD35F	0x89C3	
+0xD360	0x89CD	
+0xD361	0x89D3	
+0xD362	0x89D4	
+0xD363	0x89D5	
+0xD364	0x89D7	
+0xD365	0x89D8	
+0xD366	0x89D9	
+0xD367	0x89DB	
+0xD368	0x89DD	
+0xD369	0x89DF	
+0xD36A	0x89E0	
+0xD36B	0x89E1	
+0xD36C	0x89E2	
+0xD36D	0x89E4	
+0xD36E	0x89E7	
+0xD36F	0x89E8	
+0xD370	0x89E9	
+0xD371	0x89EA	
+0xD372	0x89EC	
+0xD373	0x89ED	
+0xD374	0x89EE	
+0xD375	0x89F0	
+0xD376	0x89F1	
+0xD377	0x89F2	
+0xD378	0x89F4	
+0xD379	0x89F5	
+0xD37A	0x89F6	
+0xD37B	0x89F7	
+0xD37C	0x89F8	
+0xD37D	0x89F9	
+0xD37E	0x89FA	
+0xD380	0x89FB	
+0xD381	0x89FC	
+0xD382	0x89FD	
+0xD383	0x89FE	
+0xD384	0x89FF	
+0xD385	0x8A01	
+0xD386	0x8A02	
+0xD387	0x8A03	
+0xD388	0x8A04	
+0xD389	0x8A05	
+0xD38A	0x8A06	
+0xD38B	0x8A08	
+0xD38C	0x8A09	
+0xD38D	0x8A0A	
+0xD38E	0x8A0B	
+0xD38F	0x8A0C	
+0xD390	0x8A0D	
+0xD391	0x8A0E	
+0xD392	0x8A0F	
+0xD393	0x8A10	
+0xD394	0x8A11	
+0xD395	0x8A12	
+0xD396	0x8A13	
+0xD397	0x8A14	
+0xD398	0x8A15	
+0xD399	0x8A16	
+0xD39A	0x8A17	
+0xD39B	0x8A18	
+0xD39C	0x8A19	
+0xD39D	0x8A1A	
+0xD39E	0x8A1B	
+0xD39F	0x8A1C	
+0xD3A0	0x8A1D	
+0xD3A1	0x5370	
+0xD3A2	0x82F1	
+0xD3A3	0x6A31	
+0xD3A4	0x5A74	
+0xD3A5	0x9E70	
+0xD3A6	0x5E94	
+0xD3A7	0x7F28	
+0xD3A8	0x83B9	
+0xD3A9	0x8424	
+0xD3AA	0x8425	
+0xD3AB	0x8367	
+0xD3AC	0x8747	
+0xD3AD	0x8FCE	
+0xD3AE	0x8D62	
+0xD3AF	0x76C8	
+0xD3B0	0x5F71	
+0xD3B1	0x9896	
+0xD3B2	0x786C	
+0xD3B3	0x6620	
+0xD3B4	0x54DF	
+0xD3B5	0x62E5	
+0xD3B6	0x4F63	
+0xD3B7	0x81C3	
+0xD3B8	0x75C8	
+0xD3B9	0x5EB8	
+0xD3BA	0x96CD	
+0xD3BB	0x8E0A	
+0xD3BC	0x86F9	
+0xD3BD	0x548F	
+0xD3BE	0x6CF3	
+0xD3BF	0x6D8C	
+0xD3C0	0x6C38	
+0xD3C1	0x607F	
+0xD3C2	0x52C7	
+0xD3C3	0x7528	
+0xD3C4	0x5E7D	
+0xD3C5	0x4F18	
+0xD3C6	0x60A0	
+0xD3C7	0x5FE7	
+0xD3C8	0x5C24	
+0xD3C9	0x7531	
+0xD3CA	0x90AE	
+0xD3CB	0x94C0	
+0xD3CC	0x72B9	
+0xD3CD	0x6CB9	
+0xD3CE	0x6E38	
+0xD3CF	0x9149	
+0xD3D0	0x6709	
+0xD3D1	0x53CB	
+0xD3D2	0x53F3	
+0xD3D3	0x4F51	
+0xD3D4	0x91C9	
+0xD3D5	0x8BF1	
+0xD3D6	0x53C8	
+0xD3D7	0x5E7C	
+0xD3D8	0x8FC2	
+0xD3D9	0x6DE4	
+0xD3DA	0x4E8E	
+0xD3DB	0x76C2	
+0xD3DC	0x6986	
+0xD3DD	0x865E	
+0xD3DE	0x611A	
+0xD3DF	0x8206	
+0xD3E0	0x4F59	
+0xD3E1	0x4FDE	
+0xD3E2	0x903E	
+0xD3E3	0x9C7C	
+0xD3E4	0x6109	
+0xD3E5	0x6E1D	
+0xD3E6	0x6E14	
+0xD3E7	0x9685	
+0xD3E8	0x4E88	
+0xD3E9	0x5A31	
+0xD3EA	0x96E8	
+0xD3EB	0x4E0E	
+0xD3EC	0x5C7F	
+0xD3ED	0x79B9	
+0xD3EE	0x5B87	
+0xD3EF	0x8BED	
+0xD3F0	0x7FBD	
+0xD3F1	0x7389	
+0xD3F2	0x57DF	
+0xD3F3	0x828B	
+0xD3F4	0x90C1	
+0xD3F5	0x5401	
+0xD3F6	0x9047	
+0xD3F7	0x55BB	
+0xD3F8	0x5CEA	
+0xD3F9	0x5FA1	
+0xD3FA	0x6108	
+0xD3FB	0x6B32	
+0xD3FC	0x72F1	
+0xD3FD	0x80B2	
+0xD3FE	0x8A89	
+0xD440	0x8A1E	
+0xD441	0x8A1F	
+0xD442	0x8A20	
+0xD443	0x8A21	
+0xD444	0x8A22	
+0xD445	0x8A23	
+0xD446	0x8A24	
+0xD447	0x8A25	
+0xD448	0x8A26	
+0xD449	0x8A27	
+0xD44A	0x8A28	
+0xD44B	0x8A29	
+0xD44C	0x8A2A	
+0xD44D	0x8A2B	
+0xD44E	0x8A2C	
+0xD44F	0x8A2D	
+0xD450	0x8A2E	
+0xD451	0x8A2F	
+0xD452	0x8A30	
+0xD453	0x8A31	
+0xD454	0x8A32	
+0xD455	0x8A33	
+0xD456	0x8A34	
+0xD457	0x8A35	
+0xD458	0x8A36	
+0xD459	0x8A37	
+0xD45A	0x8A38	
+0xD45B	0x8A39	
+0xD45C	0x8A3A	
+0xD45D	0x8A3B	
+0xD45E	0x8A3C	
+0xD45F	0x8A3D	
+0xD460	0x8A3F	
+0xD461	0x8A40	
+0xD462	0x8A41	
+0xD463	0x8A42	
+0xD464	0x8A43	
+0xD465	0x8A44	
+0xD466	0x8A45	
+0xD467	0x8A46	
+0xD468	0x8A47	
+0xD469	0x8A49	
+0xD46A	0x8A4A	
+0xD46B	0x8A4B	
+0xD46C	0x8A4C	
+0xD46D	0x8A4D	
+0xD46E	0x8A4E	
+0xD46F	0x8A4F	
+0xD470	0x8A50	
+0xD471	0x8A51	
+0xD472	0x8A52	
+0xD473	0x8A53	
+0xD474	0x8A54	
+0xD475	0x8A55	
+0xD476	0x8A56	
+0xD477	0x8A57	
+0xD478	0x8A58	
+0xD479	0x8A59	
+0xD47A	0x8A5A	
+0xD47B	0x8A5B	
+0xD47C	0x8A5C	
+0xD47D	0x8A5D	
+0xD47E	0x8A5E	
+0xD480	0x8A5F	
+0xD481	0x8A60	
+0xD482	0x8A61	
+0xD483	0x8A62	
+0xD484	0x8A63	
+0xD485	0x8A64	
+0xD486	0x8A65	
+0xD487	0x8A66	
+0xD488	0x8A67	
+0xD489	0x8A68	
+0xD48A	0x8A69	
+0xD48B	0x8A6A	
+0xD48C	0x8A6B	
+0xD48D	0x8A6C	
+0xD48E	0x8A6D	
+0xD48F	0x8A6E	
+0xD490	0x8A6F	
+0xD491	0x8A70	
+0xD492	0x8A71	
+0xD493	0x8A72	
+0xD494	0x8A73	
+0xD495	0x8A74	
+0xD496	0x8A75	
+0xD497	0x8A76	
+0xD498	0x8A77	
+0xD499	0x8A78	
+0xD49A	0x8A7A	
+0xD49B	0x8A7B	
+0xD49C	0x8A7C	
+0xD49D	0x8A7D	
+0xD49E	0x8A7E	
+0xD49F	0x8A7F	
+0xD4A0	0x8A80	
+0xD4A1	0x6D74	
+0xD4A2	0x5BD3	
+0xD4A3	0x88D5	
+0xD4A4	0x9884	
+0xD4A5	0x8C6B	
+0xD4A6	0x9A6D	
+0xD4A7	0x9E33	
+0xD4A8	0x6E0A	
+0xD4A9	0x51A4	
+0xD4AA	0x5143	
+0xD4AB	0x57A3	
+0xD4AC	0x8881	
+0xD4AD	0x539F	
+0xD4AE	0x63F4	
+0xD4AF	0x8F95	
+0xD4B0	0x56ED	
+0xD4B1	0x5458	
+0xD4B2	0x5706	
+0xD4B3	0x733F	
+0xD4B4	0x6E90	
+0xD4B5	0x7F18	
+0xD4B6	0x8FDC	
+0xD4B7	0x82D1	
+0xD4B8	0x613F	
+0xD4B9	0x6028	
+0xD4BA	0x9662	
+0xD4BB	0x66F0	
+0xD4BC	0x7EA6	
+0xD4BD	0x8D8A	
+0xD4BE	0x8DC3	
+0xD4BF	0x94A5	
+0xD4C0	0x5CB3	
+0xD4C1	0x7CA4	
+0xD4C2	0x6708	
+0xD4C3	0x60A6	
+0xD4C4	0x9605	
+0xD4C5	0x8018	
+0xD4C6	0x4E91	
+0xD4C7	0x90E7	
+0xD4C8	0x5300	
+0xD4C9	0x9668	
+0xD4CA	0x5141	
+0xD4CB	0x8FD0	
+0xD4CC	0x8574	
+0xD4CD	0x915D	
+0xD4CE	0x6655	
+0xD4CF	0x97F5	
+0xD4D0	0x5B55	
+0xD4D1	0x531D	
+0xD4D2	0x7838	
+0xD4D3	0x6742	
+0xD4D4	0x683D	
+0xD4D5	0x54C9	
+0xD4D6	0x707E	
+0xD4D7	0x5BB0	
+0xD4D8	0x8F7D	
+0xD4D9	0x518D	
+0xD4DA	0x5728	
+0xD4DB	0x54B1	
+0xD4DC	0x6512	
+0xD4DD	0x6682	
+0xD4DE	0x8D5E	
+0xD4DF	0x8D43	
+0xD4E0	0x810F	
+0xD4E1	0x846C	
+0xD4E2	0x906D	
+0xD4E3	0x7CDF	
+0xD4E4	0x51FF	
+0xD4E5	0x85FB	
+0xD4E6	0x67A3	
+0xD4E7	0x65E9	
+0xD4E8	0x6FA1	
+0xD4E9	0x86A4	
+0xD4EA	0x8E81	
+0xD4EB	0x566A	
+0xD4EC	0x9020	
+0xD4ED	0x7682	
+0xD4EE	0x7076	
+0xD4EF	0x71E5	
+0xD4F0	0x8D23	
+0xD4F1	0x62E9	
+0xD4F2	0x5219	
+0xD4F3	0x6CFD	
+0xD4F4	0x8D3C	
+0xD4F5	0x600E	
+0xD4F6	0x589E	
+0xD4F7	0x618E	
+0xD4F8	0x66FE	
+0xD4F9	0x8D60	
+0xD4FA	0x624E	
+0xD4FB	0x55B3	
+0xD4FC	0x6E23	
+0xD4FD	0x672D	
+0xD4FE	0x8F67	
+0xD540	0x8A81	
+0xD541	0x8A82	
+0xD542	0x8A83	
+0xD543	0x8A84	
+0xD544	0x8A85	
+0xD545	0x8A86	
+0xD546	0x8A87	
+0xD547	0x8A88	
+0xD548	0x8A8B	
+0xD549	0x8A8C	
+0xD54A	0x8A8D	
+0xD54B	0x8A8E	
+0xD54C	0x8A8F	
+0xD54D	0x8A90	
+0xD54E	0x8A91	
+0xD54F	0x8A92	
+0xD550	0x8A94	
+0xD551	0x8A95	
+0xD552	0x8A96	
+0xD553	0x8A97	
+0xD554	0x8A98	
+0xD555	0x8A99	
+0xD556	0x8A9A	
+0xD557	0x8A9B	
+0xD558	0x8A9C	
+0xD559	0x8A9D	
+0xD55A	0x8A9E	
+0xD55B	0x8A9F	
+0xD55C	0x8AA0	
+0xD55D	0x8AA1	
+0xD55E	0x8AA2	
+0xD55F	0x8AA3	
+0xD560	0x8AA4	
+0xD561	0x8AA5	
+0xD562	0x8AA6	
+0xD563	0x8AA7	
+0xD564	0x8AA8	
+0xD565	0x8AA9	
+0xD566	0x8AAA	
+0xD567	0x8AAB	
+0xD568	0x8AAC	
+0xD569	0x8AAD	
+0xD56A	0x8AAE	
+0xD56B	0x8AAF	
+0xD56C	0x8AB0	
+0xD56D	0x8AB1	
+0xD56E	0x8AB2	
+0xD56F	0x8AB3	
+0xD570	0x8AB4	
+0xD571	0x8AB5	
+0xD572	0x8AB6	
+0xD573	0x8AB7	
+0xD574	0x8AB8	
+0xD575	0x8AB9	
+0xD576	0x8ABA	
+0xD577	0x8ABB	
+0xD578	0x8ABC	
+0xD579	0x8ABD	
+0xD57A	0x8ABE	
+0xD57B	0x8ABF	
+0xD57C	0x8AC0	
+0xD57D	0x8AC1	
+0xD57E	0x8AC2	
+0xD580	0x8AC3	
+0xD581	0x8AC4	
+0xD582	0x8AC5	
+0xD583	0x8AC6	
+0xD584	0x8AC7	
+0xD585	0x8AC8	
+0xD586	0x8AC9	
+0xD587	0x8ACA	
+0xD588	0x8ACB	
+0xD589	0x8ACC	
+0xD58A	0x8ACD	
+0xD58B	0x8ACE	
+0xD58C	0x8ACF	
+0xD58D	0x8AD0	
+0xD58E	0x8AD1	
+0xD58F	0x8AD2	
+0xD590	0x8AD3	
+0xD591	0x8AD4	
+0xD592	0x8AD5	
+0xD593	0x8AD6	
+0xD594	0x8AD7	
+0xD595	0x8AD8	
+0xD596	0x8AD9	
+0xD597	0x8ADA	
+0xD598	0x8ADB	
+0xD599	0x8ADC	
+0xD59A	0x8ADD	
+0xD59B	0x8ADE	
+0xD59C	0x8ADF	
+0xD59D	0x8AE0	
+0xD59E	0x8AE1	
+0xD59F	0x8AE2	
+0xD5A0	0x8AE3	
+0xD5A1	0x94E1	
+0xD5A2	0x95F8	
+0xD5A3	0x7728	
+0xD5A4	0x6805	
+0xD5A5	0x69A8	
+0xD5A6	0x548B	
+0xD5A7	0x4E4D	
+0xD5A8	0x70B8	
+0xD5A9	0x8BC8	
+0xD5AA	0x6458	
+0xD5AB	0x658B	
+0xD5AC	0x5B85	
+0xD5AD	0x7A84	
+0xD5AE	0x503A	
+0xD5AF	0x5BE8	
+0xD5B0	0x77BB	
+0xD5B1	0x6BE1	
+0xD5B2	0x8A79	
+0xD5B3	0x7C98	
+0xD5B4	0x6CBE	
+0xD5B5	0x76CF	
+0xD5B6	0x65A9	
+0xD5B7	0x8F97	
+0xD5B8	0x5D2D	
+0xD5B9	0x5C55	
+0xD5BA	0x8638	
+0xD5BB	0x6808	
+0xD5BC	0x5360	
+0xD5BD	0x6218	
+0xD5BE	0x7AD9	
+0xD5BF	0x6E5B	
+0xD5C0	0x7EFD	
+0xD5C1	0x6A1F	
+0xD5C2	0x7AE0	
+0xD5C3	0x5F70	
+0xD5C4	0x6F33	
+0xD5C5	0x5F20	
+0xD5C6	0x638C	
+0xD5C7	0x6DA8	
+0xD5C8	0x6756	
+0xD5C9	0x4E08	
+0xD5CA	0x5E10	
+0xD5CB	0x8D26	
+0xD5CC	0x4ED7	
+0xD5CD	0x80C0	
+0xD5CE	0x7634	
+0xD5CF	0x969C	
+0xD5D0	0x62DB	
+0xD5D1	0x662D	
+0xD5D2	0x627E	
+0xD5D3	0x6CBC	
+0xD5D4	0x8D75	
+0xD5D5	0x7167	
+0xD5D6	0x7F69	
+0xD5D7	0x5146	
+0xD5D8	0x8087	
+0xD5D9	0x53EC	
+0xD5DA	0x906E	
+0xD5DB	0x6298	
+0xD5DC	0x54F2	
+0xD5DD	0x86F0	
+0xD5DE	0x8F99	
+0xD5DF	0x8005	
+0xD5E0	0x9517	
+0xD5E1	0x8517	
+0xD5E2	0x8FD9	
+0xD5E3	0x6D59	
+0xD5E4	0x73CD	
+0xD5E5	0x659F	
+0xD5E6	0x771F	
+0xD5E7	0x7504	
+0xD5E8	0x7827	
+0xD5E9	0x81FB	
+0xD5EA	0x8D1E	
+0xD5EB	0x9488	
+0xD5EC	0x4FA6	
+0xD5ED	0x6795	
+0xD5EE	0x75B9	
+0xD5EF	0x8BCA	
+0xD5F0	0x9707	
+0xD5F1	0x632F	
+0xD5F2	0x9547	
+0xD5F3	0x9635	
+0xD5F4	0x84B8	
+0xD5F5	0x6323	
+0xD5F6	0x7741	
+0xD5F7	0x5F81	
+0xD5F8	0x72F0	
+0xD5F9	0x4E89	
+0xD5FA	0x6014	
+0xD5FB	0x6574	
+0xD5FC	0x62EF	
+0xD5FD	0x6B63	
+0xD5FE	0x653F	
+0xD640	0x8AE4	
+0xD641	0x8AE5	
+0xD642	0x8AE6	
+0xD643	0x8AE7	
+0xD644	0x8AE8	
+0xD645	0x8AE9	
+0xD646	0x8AEA	
+0xD647	0x8AEB	
+0xD648	0x8AEC	
+0xD649	0x8AED	
+0xD64A	0x8AEE	
+0xD64B	0x8AEF	
+0xD64C	0x8AF0	
+0xD64D	0x8AF1	
+0xD64E	0x8AF2	
+0xD64F	0x8AF3	
+0xD650	0x8AF4	
+0xD651	0x8AF5	
+0xD652	0x8AF6	
+0xD653	0x8AF7	
+0xD654	0x8AF8	
+0xD655	0x8AF9	
+0xD656	0x8AFA	
+0xD657	0x8AFB	
+0xD658	0x8AFC	
+0xD659	0x8AFD	
+0xD65A	0x8AFE	
+0xD65B	0x8AFF	
+0xD65C	0x8B00	
+0xD65D	0x8B01	
+0xD65E	0x8B02	
+0xD65F	0x8B03	
+0xD660	0x8B04	
+0xD661	0x8B05	
+0xD662	0x8B06	
+0xD663	0x8B08	
+0xD664	0x8B09	
+0xD665	0x8B0A	
+0xD666	0x8B0B	
+0xD667	0x8B0C	
+0xD668	0x8B0D	
+0xD669	0x8B0E	
+0xD66A	0x8B0F	
+0xD66B	0x8B10	
+0xD66C	0x8B11	
+0xD66D	0x8B12	
+0xD66E	0x8B13	
+0xD66F	0x8B14	
+0xD670	0x8B15	
+0xD671	0x8B16	
+0xD672	0x8B17	
+0xD673	0x8B18	
+0xD674	0x8B19	
+0xD675	0x8B1A	
+0xD676	0x8B1B	
+0xD677	0x8B1C	
+0xD678	0x8B1D	
+0xD679	0x8B1E	
+0xD67A	0x8B1F	
+0xD67B	0x8B20	
+0xD67C	0x8B21	
+0xD67D	0x8B22	
+0xD67E	0x8B23	
+0xD680	0x8B24	
+0xD681	0x8B25	
+0xD682	0x8B27	
+0xD683	0x8B28	
+0xD684	0x8B29	
+0xD685	0x8B2A	
+0xD686	0x8B2B	
+0xD687	0x8B2C	
+0xD688	0x8B2D	
+0xD689	0x8B2E	
+0xD68A	0x8B2F	
+0xD68B	0x8B30	
+0xD68C	0x8B31	
+0xD68D	0x8B32	
+0xD68E	0x8B33	
+0xD68F	0x8B34	
+0xD690	0x8B35	
+0xD691	0x8B36	
+0xD692	0x8B37	
+0xD693	0x8B38	
+0xD694	0x8B39	
+0xD695	0x8B3A	
+0xD696	0x8B3B	
+0xD697	0x8B3C	
+0xD698	0x8B3D	
+0xD699	0x8B3E	
+0xD69A	0x8B3F	
+0xD69B	0x8B40	
+0xD69C	0x8B41	
+0xD69D	0x8B42	
+0xD69E	0x8B43	
+0xD69F	0x8B44	
+0xD6A0	0x8B45	
+0xD6A1	0x5E27	
+0xD6A2	0x75C7	
+0xD6A3	0x90D1	
+0xD6A4	0x8BC1	
+0xD6A5	0x829D	
+0xD6A6	0x679D	
+0xD6A7	0x652F	
+0xD6A8	0x5431	
+0xD6A9	0x8718	
+0xD6AA	0x77E5	
+0xD6AB	0x80A2	
+0xD6AC	0x8102	
+0xD6AD	0x6C41	
+0xD6AE	0x4E4B	
+0xD6AF	0x7EC7	
+0xD6B0	0x804C	
+0xD6B1	0x76F4	
+0xD6B2	0x690D	
+0xD6B3	0x6B96	
+0xD6B4	0x6267	
+0xD6B5	0x503C	
+0xD6B6	0x4F84	
+0xD6B7	0x5740	
+0xD6B8	0x6307	
+0xD6B9	0x6B62	
+0xD6BA	0x8DBE	
+0xD6BB	0x53EA	
+0xD6BC	0x65E8	
+0xD6BD	0x7EB8	
+0xD6BE	0x5FD7	
+0xD6BF	0x631A	
+0xD6C0	0x63B7	
+0xD6C1	0x81F3	
+0xD6C2	0x81F4	
+0xD6C3	0x7F6E	
+0xD6C4	0x5E1C	
+0xD6C5	0x5CD9	
+0xD6C6	0x5236	
+0xD6C7	0x667A	
+0xD6C8	0x79E9	
+0xD6C9	0x7A1A	
+0xD6CA	0x8D28	
+0xD6CB	0x7099	
+0xD6CC	0x75D4	
+0xD6CD	0x6EDE	
+0xD6CE	0x6CBB	
+0xD6CF	0x7A92	
+0xD6D0	0x4E2D	
+0xD6D1	0x76C5	
+0xD6D2	0x5FE0	
+0xD6D3	0x949F	
+0xD6D4	0x8877	
+0xD6D5	0x7EC8	
+0xD6D6	0x79CD	
+0xD6D7	0x80BF	
+0xD6D8	0x91CD	
+0xD6D9	0x4EF2	
+0xD6DA	0x4F17	
+0xD6DB	0x821F	
+0xD6DC	0x5468	
+0xD6DD	0x5DDE	
+0xD6DE	0x6D32	
+0xD6DF	0x8BCC	
+0xD6E0	0x7CA5	
+0xD6E1	0x8F74	
+0xD6E2	0x8098	
+0xD6E3	0x5E1A	
+0xD6E4	0x5492	
+0xD6E5	0x76B1	
+0xD6E6	0x5B99	
+0xD6E7	0x663C	
+0xD6E8	0x9AA4	
+0xD6E9	0x73E0	
+0xD6EA	0x682A	
+0xD6EB	0x86DB	
+0xD6EC	0x6731	
+0xD6ED	0x732A	
+0xD6EE	0x8BF8	
+0xD6EF	0x8BDB	
+0xD6F0	0x9010	
+0xD6F1	0x7AF9	
+0xD6F2	0x70DB	
+0xD6F3	0x716E	
+0xD6F4	0x62C4	
+0xD6F5	0x77A9	
+0xD6F6	0x5631	
+0xD6F7	0x4E3B	
+0xD6F8	0x8457	
+0xD6F9	0x67F1	
+0xD6FA	0x52A9	
+0xD6FB	0x86C0	
+0xD6FC	0x8D2E	
+0xD6FD	0x94F8	
+0xD6FE	0x7B51	
+0xD740	0x8B46	
+0xD741	0x8B47	
+0xD742	0x8B48	
+0xD743	0x8B49	
+0xD744	0x8B4A	
+0xD745	0x8B4B	
+0xD746	0x8B4C	
+0xD747	0x8B4D	
+0xD748	0x8B4E	
+0xD749	0x8B4F	
+0xD74A	0x8B50	
+0xD74B	0x8B51	
+0xD74C	0x8B52	
+0xD74D	0x8B53	
+0xD74E	0x8B54	
+0xD74F	0x8B55	
+0xD750	0x8B56	
+0xD751	0x8B57	
+0xD752	0x8B58	
+0xD753	0x8B59	
+0xD754	0x8B5A	
+0xD755	0x8B5B	
+0xD756	0x8B5C	
+0xD757	0x8B5D	
+0xD758	0x8B5E	
+0xD759	0x8B5F	
+0xD75A	0x8B60	
+0xD75B	0x8B61	
+0xD75C	0x8B62	
+0xD75D	0x8B63	
+0xD75E	0x8B64	
+0xD75F	0x8B65	
+0xD760	0x8B67	
+0xD761	0x8B68	
+0xD762	0x8B69	
+0xD763	0x8B6A	
+0xD764	0x8B6B	
+0xD765	0x8B6D	
+0xD766	0x8B6E	
+0xD767	0x8B6F	
+0xD768	0x8B70	
+0xD769	0x8B71	
+0xD76A	0x8B72	
+0xD76B	0x8B73	
+0xD76C	0x8B74	
+0xD76D	0x8B75	
+0xD76E	0x8B76	
+0xD76F	0x8B77	
+0xD770	0x8B78	
+0xD771	0x8B79	
+0xD772	0x8B7A	
+0xD773	0x8B7B	
+0xD774	0x8B7C	
+0xD775	0x8B7D	
+0xD776	0x8B7E	
+0xD777	0x8B7F	
+0xD778	0x8B80	
+0xD779	0x8B81	
+0xD77A	0x8B82	
+0xD77B	0x8B83	
+0xD77C	0x8B84	
+0xD77D	0x8B85	
+0xD77E	0x8B86	
+0xD780	0x8B87	
+0xD781	0x8B88	
+0xD782	0x8B89	
+0xD783	0x8B8A	
+0xD784	0x8B8B	
+0xD785	0x8B8C	
+0xD786	0x8B8D	
+0xD787	0x8B8E	
+0xD788	0x8B8F	
+0xD789	0x8B90	
+0xD78A	0x8B91	
+0xD78B	0x8B92	
+0xD78C	0x8B93	
+0xD78D	0x8B94	
+0xD78E	0x8B95	
+0xD78F	0x8B96	
+0xD790	0x8B97	
+0xD791	0x8B98	
+0xD792	0x8B99	
+0xD793	0x8B9A	
+0xD794	0x8B9B	
+0xD795	0x8B9C	
+0xD796	0x8B9D	
+0xD797	0x8B9E	
+0xD798	0x8B9F	
+0xD799	0x8BAC	
+0xD79A	0x8BB1	
+0xD79B	0x8BBB	
+0xD79C	0x8BC7	
+0xD79D	0x8BD0	
+0xD79E	0x8BEA	
+0xD79F	0x8C09	
+0xD7A0	0x8C1E	
+0xD7A1	0x4F4F	
+0xD7A2	0x6CE8	
+0xD7A3	0x795D	
+0xD7A4	0x9A7B	
+0xD7A5	0x6293	
+0xD7A6	0x722A	
+0xD7A7	0x62FD	
+0xD7A8	0x4E13	
+0xD7A9	0x7816	
+0xD7AA	0x8F6C	
+0xD7AB	0x64B0	
+0xD7AC	0x8D5A	
+0xD7AD	0x7BC6	
+0xD7AE	0x6869	
+0xD7AF	0x5E84	
+0xD7B0	0x88C5	
+0xD7B1	0x5986	
+0xD7B2	0x649E	
+0xD7B3	0x58EE	
+0xD7B4	0x72B6	
+0xD7B5	0x690E	
+0xD7B6	0x9525	
+0xD7B7	0x8FFD	
+0xD7B8	0x8D58	
+0xD7B9	0x5760	
+0xD7BA	0x7F00	
+0xD7BB	0x8C06	
+0xD7BC	0x51C6	
+0xD7BD	0x6349	
+0xD7BE	0x62D9	
+0xD7BF	0x5353	
+0xD7C0	0x684C	
+0xD7C1	0x7422	
+0xD7C2	0x8301	
+0xD7C3	0x914C	
+0xD7C4	0x5544	
+0xD7C5	0x7740	
+0xD7C6	0x707C	
+0xD7C7	0x6D4A	
+0xD7C8	0x5179	
+0xD7C9	0x54A8	
+0xD7CA	0x8D44	
+0xD7CB	0x59FF	
+0xD7CC	0x6ECB	
+0xD7CD	0x6DC4	
+0xD7CE	0x5B5C	
+0xD7CF	0x7D2B	
+0xD7D0	0x4ED4	
+0xD7D1	0x7C7D	
+0xD7D2	0x6ED3	
+0xD7D3	0x5B50	
+0xD7D4	0x81EA	
+0xD7D5	0x6E0D	
+0xD7D6	0x5B57	
+0xD7D7	0x9B03	
+0xD7D8	0x68D5	
+0xD7D9	0x8E2A	
+0xD7DA	0x5B97	
+0xD7DB	0x7EFC	
+0xD7DC	0x603B	
+0xD7DD	0x7EB5	
+0xD7DE	0x90B9	
+0xD7DF	0x8D70	
+0xD7E0	0x594F	
+0xD7E1	0x63CD	
+0xD7E2	0x79DF	
+0xD7E3	0x8DB3	
+0xD7E4	0x5352	
+0xD7E5	0x65CF	
+0xD7E6	0x7956	
+0xD7E7	0x8BC5	
+0xD7E8	0x963B	
+0xD7E9	0x7EC4	
+0xD7EA	0x94BB	
+0xD7EB	0x7E82	
+0xD7EC	0x5634	
+0xD7ED	0x9189	
+0xD7EE	0x6700	
+0xD7EF	0x7F6A	
+0xD7F0	0x5C0A	
+0xD7F1	0x9075	
+0xD7F2	0x6628	
+0xD7F3	0x5DE6	
+0xD7F4	0x4F50	
+0xD7F5	0x67DE	
+0xD7F6	0x505A	
+0xD7F7	0x4F5C	
+0xD7F8	0x5750	
+0xD7F9	0x5EA7	
+# 0xD7FA	 	
+# 0xD7FB	 	
+# 0xD7FC	 	
+# 0xD7FD	 	
+# 0xD7FE	 	
+0xD840	0x8C38	
+0xD841	0x8C39	
+0xD842	0x8C3A	
+0xD843	0x8C3B	
+0xD844	0x8C3C	
+0xD845	0x8C3D	
+0xD846	0x8C3E	
+0xD847	0x8C3F	
+0xD848	0x8C40	
+0xD849	0x8C42	
+0xD84A	0x8C43	
+0xD84B	0x8C44	
+0xD84C	0x8C45	
+0xD84D	0x8C48	
+0xD84E	0x8C4A	
+0xD84F	0x8C4B	
+0xD850	0x8C4D	
+0xD851	0x8C4E	
+0xD852	0x8C4F	
+0xD853	0x8C50	
+0xD854	0x8C51	
+0xD855	0x8C52	
+0xD856	0x8C53	
+0xD857	0x8C54	
+0xD858	0x8C56	
+0xD859	0x8C57	
+0xD85A	0x8C58	
+0xD85B	0x8C59	
+0xD85C	0x8C5B	
+0xD85D	0x8C5C	
+0xD85E	0x8C5D	
+0xD85F	0x8C5E	
+0xD860	0x8C5F	
+0xD861	0x8C60	
+0xD862	0x8C63	
+0xD863	0x8C64	
+0xD864	0x8C65	
+0xD865	0x8C66	
+0xD866	0x8C67	
+0xD867	0x8C68	
+0xD868	0x8C69	
+0xD869	0x8C6C	
+0xD86A	0x8C6D	
+0xD86B	0x8C6E	
+0xD86C	0x8C6F	
+0xD86D	0x8C70	
+0xD86E	0x8C71	
+0xD86F	0x8C72	
+0xD870	0x8C74	
+0xD871	0x8C75	
+0xD872	0x8C76	
+0xD873	0x8C77	
+0xD874	0x8C7B	
+0xD875	0x8C7C	
+0xD876	0x8C7D	
+0xD877	0x8C7E	
+0xD878	0x8C7F	
+0xD879	0x8C80	
+0xD87A	0x8C81	
+0xD87B	0x8C83	
+0xD87C	0x8C84	
+0xD87D	0x8C86	
+0xD87E	0x8C87	
+0xD880	0x8C88	
+0xD881	0x8C8B	
+0xD882	0x8C8D	
+0xD883	0x8C8E	
+0xD884	0x8C8F	
+0xD885	0x8C90	
+0xD886	0x8C91	
+0xD887	0x8C92	
+0xD888	0x8C93	
+0xD889	0x8C95	
+0xD88A	0x8C96	
+0xD88B	0x8C97	
+0xD88C	0x8C99	
+0xD88D	0x8C9A	
+0xD88E	0x8C9B	
+0xD88F	0x8C9C	
+0xD890	0x8C9D	
+0xD891	0x8C9E	
+0xD892	0x8C9F	
+0xD893	0x8CA0	
+0xD894	0x8CA1	
+0xD895	0x8CA2	
+0xD896	0x8CA3	
+0xD897	0x8CA4	
+0xD898	0x8CA5	
+0xD899	0x8CA6	
+0xD89A	0x8CA7	
+0xD89B	0x8CA8	
+0xD89C	0x8CA9	
+0xD89D	0x8CAA	
+0xD89E	0x8CAB	
+0xD89F	0x8CAC	
+0xD8A0	0x8CAD	
+0xD8A1	0x4E8D	
+0xD8A2	0x4E0C	
+0xD8A3	0x5140	
+0xD8A4	0x4E10	
+0xD8A5	0x5EFF	
+0xD8A6	0x5345	
+0xD8A7	0x4E15	
+0xD8A8	0x4E98	
+0xD8A9	0x4E1E	
+0xD8AA	0x9B32	
+0xD8AB	0x5B6C	
+0xD8AC	0x5669	
+0xD8AD	0x4E28	
+0xD8AE	0x79BA	
+0xD8AF	0x4E3F	
+0xD8B0	0x5315	
+0xD8B1	0x4E47	
+0xD8B2	0x592D	
+0xD8B3	0x723B	
+0xD8B4	0x536E	
+0xD8B5	0x6C10	
+0xD8B6	0x56DF	
+0xD8B7	0x80E4	
+0xD8B8	0x9997	
+0xD8B9	0x6BD3	
+0xD8BA	0x777E	
+0xD8BB	0x9F17	
+0xD8BC	0x4E36	
+0xD8BD	0x4E9F	
+0xD8BE	0x9F10	
+0xD8BF	0x4E5C	
+0xD8C0	0x4E69	
+0xD8C1	0x4E93	
+0xD8C2	0x8288	
+0xD8C3	0x5B5B	
+0xD8C4	0x556C	
+0xD8C5	0x560F	
+0xD8C6	0x4EC4	
+0xD8C7	0x538D	
+0xD8C8	0x539D	
+0xD8C9	0x53A3	
+0xD8CA	0x53A5	
+0xD8CB	0x53AE	
+0xD8CC	0x9765	
+0xD8CD	0x8D5D	
+0xD8CE	0x531A	
+0xD8CF	0x53F5	
+0xD8D0	0x5326	
+0xD8D1	0x532E	
+0xD8D2	0x533E	
+0xD8D3	0x8D5C	
+0xD8D4	0x5366	
+0xD8D5	0x5363	
+0xD8D6	0x5202	
+0xD8D7	0x5208	
+0xD8D8	0x520E	
+0xD8D9	0x522D	
+0xD8DA	0x5233	
+0xD8DB	0x523F	
+0xD8DC	0x5240	
+0xD8DD	0x524C	
+0xD8DE	0x525E	
+0xD8DF	0x5261	
+0xD8E0	0x525C	
+0xD8E1	0x84AF	
+0xD8E2	0x527D	
+0xD8E3	0x5282	
+0xD8E4	0x5281	
+0xD8E5	0x5290	
+0xD8E6	0x5293	
+0xD8E7	0x5182	
+0xD8E8	0x7F54	
+0xD8E9	0x4EBB	
+0xD8EA	0x4EC3	
+0xD8EB	0x4EC9	
+0xD8EC	0x4EC2	
+0xD8ED	0x4EE8	
+0xD8EE	0x4EE1	
+0xD8EF	0x4EEB	
+0xD8F0	0x4EDE	
+0xD8F1	0x4F1B	
+0xD8F2	0x4EF3	
+0xD8F3	0x4F22	
+0xD8F4	0x4F64	
+0xD8F5	0x4EF5	
+0xD8F6	0x4F25	
+0xD8F7	0x4F27	
+0xD8F8	0x4F09	
+0xD8F9	0x4F2B	
+0xD8FA	0x4F5E	
+0xD8FB	0x4F67	
+0xD8FC	0x6538	
+0xD8FD	0x4F5A	
+0xD8FE	0x4F5D	
+0xD940	0x8CAE	
+0xD941	0x8CAF	
+0xD942	0x8CB0	
+0xD943	0x8CB1	
+0xD944	0x8CB2	
+0xD945	0x8CB3	
+0xD946	0x8CB4	
+0xD947	0x8CB5	
+0xD948	0x8CB6	
+0xD949	0x8CB7	
+0xD94A	0x8CB8	
+0xD94B	0x8CB9	
+0xD94C	0x8CBA	
+0xD94D	0x8CBB	
+0xD94E	0x8CBC	
+0xD94F	0x8CBD	
+0xD950	0x8CBE	
+0xD951	0x8CBF	
+0xD952	0x8CC0	
+0xD953	0x8CC1	
+0xD954	0x8CC2	
+0xD955	0x8CC3	
+0xD956	0x8CC4	
+0xD957	0x8CC5	
+0xD958	0x8CC6	
+0xD959	0x8CC7	
+0xD95A	0x8CC8	
+0xD95B	0x8CC9	
+0xD95C	0x8CCA	
+0xD95D	0x8CCB	
+0xD95E	0x8CCC	
+0xD95F	0x8CCD	
+0xD960	0x8CCE	
+0xD961	0x8CCF	
+0xD962	0x8CD0	
+0xD963	0x8CD1	
+0xD964	0x8CD2	
+0xD965	0x8CD3	
+0xD966	0x8CD4	
+0xD967	0x8CD5	
+0xD968	0x8CD6	
+0xD969	0x8CD7	
+0xD96A	0x8CD8	
+0xD96B	0x8CD9	
+0xD96C	0x8CDA	
+0xD96D	0x8CDB	
+0xD96E	0x8CDC	
+0xD96F	0x8CDD	
+0xD970	0x8CDE	
+0xD971	0x8CDF	
+0xD972	0x8CE0	
+0xD973	0x8CE1	
+0xD974	0x8CE2	
+0xD975	0x8CE3	
+0xD976	0x8CE4	
+0xD977	0x8CE5	
+0xD978	0x8CE6	
+0xD979	0x8CE7	
+0xD97A	0x8CE8	
+0xD97B	0x8CE9	
+0xD97C	0x8CEA	
+0xD97D	0x8CEB	
+0xD97E	0x8CEC	
+0xD980	0x8CED	
+0xD981	0x8CEE	
+0xD982	0x8CEF	
+0xD983	0x8CF0	
+0xD984	0x8CF1	
+0xD985	0x8CF2	
+0xD986	0x8CF3	
+0xD987	0x8CF4	
+0xD988	0x8CF5	
+0xD989	0x8CF6	
+0xD98A	0x8CF7	
+0xD98B	0x8CF8	
+0xD98C	0x8CF9	
+0xD98D	0x8CFA	
+0xD98E	0x8CFB	
+0xD98F	0x8CFC	
+0xD990	0x8CFD	
+0xD991	0x8CFE	
+0xD992	0x8CFF	
+0xD993	0x8D00	
+0xD994	0x8D01	
+0xD995	0x8D02	
+0xD996	0x8D03	
+0xD997	0x8D04	
+0xD998	0x8D05	
+0xD999	0x8D06	
+0xD99A	0x8D07	
+0xD99B	0x8D08	
+0xD99C	0x8D09	
+0xD99D	0x8D0A	
+0xD99E	0x8D0B	
+0xD99F	0x8D0C	
+0xD9A0	0x8D0D	
+0xD9A1	0x4F5F	
+0xD9A2	0x4F57	
+0xD9A3	0x4F32	
+0xD9A4	0x4F3D	
+0xD9A5	0x4F76	
+0xD9A6	0x4F74	
+0xD9A7	0x4F91	
+0xD9A8	0x4F89	
+0xD9A9	0x4F83	
+0xD9AA	0x4F8F	
+0xD9AB	0x4F7E	
+0xD9AC	0x4F7B	
+0xD9AD	0x4FAA	
+0xD9AE	0x4F7C	
+0xD9AF	0x4FAC	
+0xD9B0	0x4F94	
+0xD9B1	0x4FE6	
+0xD9B2	0x4FE8	
+0xD9B3	0x4FEA	
+0xD9B4	0x4FC5	
+0xD9B5	0x4FDA	
+0xD9B6	0x4FE3	
+0xD9B7	0x4FDC	
+0xD9B8	0x4FD1	
+0xD9B9	0x4FDF	
+0xD9BA	0x4FF8	
+0xD9BB	0x5029	
+0xD9BC	0x504C	
+0xD9BD	0x4FF3	
+0xD9BE	0x502C	
+0xD9BF	0x500F	
+0xD9C0	0x502E	
+0xD9C1	0x502D	
+0xD9C2	0x4FFE	
+0xD9C3	0x501C	
+0xD9C4	0x500C	
+0xD9C5	0x5025	
+0xD9C6	0x5028	
+0xD9C7	0x507E	
+0xD9C8	0x5043	
+0xD9C9	0x5055	
+0xD9CA	0x5048	
+0xD9CB	0x504E	
+0xD9CC	0x506C	
+0xD9CD	0x507B	
+0xD9CE	0x50A5	
+0xD9CF	0x50A7	
+0xD9D0	0x50A9	
+0xD9D1	0x50BA	
+0xD9D2	0x50D6	
+0xD9D3	0x5106	
+0xD9D4	0x50ED	
+0xD9D5	0x50EC	
+0xD9D6	0x50E6	
+0xD9D7	0x50EE	
+0xD9D8	0x5107	
+0xD9D9	0x510B	
+0xD9DA	0x4EDD	
+0xD9DB	0x6C3D	
+0xD9DC	0x4F58	
+0xD9DD	0x4F65	
+0xD9DE	0x4FCE	
+0xD9DF	0x9FA0	
+0xD9E0	0x6C46	
+0xD9E1	0x7C74	
+0xD9E2	0x516E	
+0xD9E3	0x5DFD	
+0xD9E4	0x9EC9	
+0xD9E5	0x9998	
+0xD9E6	0x5181	
+0xD9E7	0x5914	
+0xD9E8	0x52F9	
+0xD9E9	0x530D	
+0xD9EA	0x8A07	
+0xD9EB	0x5310	
+0xD9EC	0x51EB	
+0xD9ED	0x5919	
+0xD9EE	0x5155	
+0xD9EF	0x4EA0	
+0xD9F0	0x5156	
+0xD9F1	0x4EB3	
+0xD9F2	0x886E	
+0xD9F3	0x88A4	
+0xD9F4	0x4EB5	
+0xD9F5	0x8114	
+0xD9F6	0x88D2	
+0xD9F7	0x7980	
+0xD9F8	0x5B34	
+0xD9F9	0x8803	
+0xD9FA	0x7FB8	
+0xD9FB	0x51AB	
+0xD9FC	0x51B1	
+0xD9FD	0x51BD	
+0xD9FE	0x51BC	
+0xDA40	0x8D0E	
+0xDA41	0x8D0F	
+0xDA42	0x8D10	
+0xDA43	0x8D11	
+0xDA44	0x8D12	
+0xDA45	0x8D13	
+0xDA46	0x8D14	
+0xDA47	0x8D15	
+0xDA48	0x8D16	
+0xDA49	0x8D17	
+0xDA4A	0x8D18	
+0xDA4B	0x8D19	
+0xDA4C	0x8D1A	
+0xDA4D	0x8D1B	
+0xDA4E	0x8D1C	
+0xDA4F	0x8D20	
+0xDA50	0x8D51	
+0xDA51	0x8D52	
+0xDA52	0x8D57	
+0xDA53	0x8D5F	
+0xDA54	0x8D65	
+0xDA55	0x8D68	
+0xDA56	0x8D69	
+0xDA57	0x8D6A	
+0xDA58	0x8D6C	
+0xDA59	0x8D6E	
+0xDA5A	0x8D6F	
+0xDA5B	0x8D71	
+0xDA5C	0x8D72	
+0xDA5D	0x8D78	
+0xDA5E	0x8D79	
+0xDA5F	0x8D7A	
+0xDA60	0x8D7B	
+0xDA61	0x8D7C	
+0xDA62	0x8D7D	
+0xDA63	0x8D7E	
+0xDA64	0x8D7F	
+0xDA65	0x8D80	
+0xDA66	0x8D82	
+0xDA67	0x8D83	
+0xDA68	0x8D86	
+0xDA69	0x8D87	
+0xDA6A	0x8D88	
+0xDA6B	0x8D89	
+0xDA6C	0x8D8C	
+0xDA6D	0x8D8D	
+0xDA6E	0x8D8E	
+0xDA6F	0x8D8F	
+0xDA70	0x8D90	
+0xDA71	0x8D92	
+0xDA72	0x8D93	
+0xDA73	0x8D95	
+0xDA74	0x8D96	
+0xDA75	0x8D97	
+0xDA76	0x8D98	
+0xDA77	0x8D99	
+0xDA78	0x8D9A	
+0xDA79	0x8D9B	
+0xDA7A	0x8D9C	
+0xDA7B	0x8D9D	
+0xDA7C	0x8D9E	
+0xDA7D	0x8DA0	
+0xDA7E	0x8DA1	
+0xDA80	0x8DA2	
+0xDA81	0x8DA4	
+0xDA82	0x8DA5	
+0xDA83	0x8DA6	
+0xDA84	0x8DA7	
+0xDA85	0x8DA8	
+0xDA86	0x8DA9	
+0xDA87	0x8DAA	
+0xDA88	0x8DAB	
+0xDA89	0x8DAC	
+0xDA8A	0x8DAD	
+0xDA8B	0x8DAE	
+0xDA8C	0x8DAF	
+0xDA8D	0x8DB0	
+0xDA8E	0x8DB2	
+0xDA8F	0x8DB6	
+0xDA90	0x8DB7	
+0xDA91	0x8DB9	
+0xDA92	0x8DBB	
+0xDA93	0x8DBD	
+0xDA94	0x8DC0	
+0xDA95	0x8DC1	
+0xDA96	0x8DC2	
+0xDA97	0x8DC5	
+0xDA98	0x8DC7	
+0xDA99	0x8DC8	
+0xDA9A	0x8DC9	
+0xDA9B	0x8DCA	
+0xDA9C	0x8DCD	
+0xDA9D	0x8DD0	
+0xDA9E	0x8DD2	
+0xDA9F	0x8DD3	
+0xDAA0	0x8DD4	
+0xDAA1	0x51C7	
+0xDAA2	0x5196	
+0xDAA3	0x51A2	
+0xDAA4	0x51A5	
+0xDAA5	0x8BA0	
+0xDAA6	0x8BA6	
+0xDAA7	0x8BA7	
+0xDAA8	0x8BAA	
+0xDAA9	0x8BB4	
+0xDAAA	0x8BB5	
+0xDAAB	0x8BB7	
+0xDAAC	0x8BC2	
+0xDAAD	0x8BC3	
+0xDAAE	0x8BCB	
+0xDAAF	0x8BCF	
+0xDAB0	0x8BCE	
+0xDAB1	0x8BD2	
+0xDAB2	0x8BD3	
+0xDAB3	0x8BD4	
+0xDAB4	0x8BD6	
+0xDAB5	0x8BD8	
+0xDAB6	0x8BD9	
+0xDAB7	0x8BDC	
+0xDAB8	0x8BDF	
+0xDAB9	0x8BE0	
+0xDABA	0x8BE4	
+0xDABB	0x8BE8	
+0xDABC	0x8BE9	
+0xDABD	0x8BEE	
+0xDABE	0x8BF0	
+0xDABF	0x8BF3	
+0xDAC0	0x8BF6	
+0xDAC1	0x8BF9	
+0xDAC2	0x8BFC	
+0xDAC3	0x8BFF	
+0xDAC4	0x8C00	
+0xDAC5	0x8C02	
+0xDAC6	0x8C04	
+0xDAC7	0x8C07	
+0xDAC8	0x8C0C	
+0xDAC9	0x8C0F	
+0xDACA	0x8C11	
+0xDACB	0x8C12	
+0xDACC	0x8C14	
+0xDACD	0x8C15	
+0xDACE	0x8C16	
+0xDACF	0x8C19	
+0xDAD0	0x8C1B	
+0xDAD1	0x8C18	
+0xDAD2	0x8C1D	
+0xDAD3	0x8C1F	
+0xDAD4	0x8C20	
+0xDAD5	0x8C21	
+0xDAD6	0x8C25	
+0xDAD7	0x8C27	
+0xDAD8	0x8C2A	
+0xDAD9	0x8C2B	
+0xDADA	0x8C2E	
+0xDADB	0x8C2F	
+0xDADC	0x8C32	
+0xDADD	0x8C33	
+0xDADE	0x8C35	
+0xDADF	0x8C36	
+0xDAE0	0x5369	
+0xDAE1	0x537A	
+0xDAE2	0x961D	
+0xDAE3	0x9622	
+0xDAE4	0x9621	
+0xDAE5	0x9631	
+0xDAE6	0x962A	
+0xDAE7	0x963D	
+0xDAE8	0x963C	
+0xDAE9	0x9642	
+0xDAEA	0x9649	
+0xDAEB	0x9654	
+0xDAEC	0x965F	
+0xDAED	0x9667	
+0xDAEE	0x966C	
+0xDAEF	0x9672	
+0xDAF0	0x9674	
+0xDAF1	0x9688	
+0xDAF2	0x968D	
+0xDAF3	0x9697	
+0xDAF4	0x96B0	
+0xDAF5	0x9097	
+0xDAF6	0x909B	
+0xDAF7	0x909D	
+0xDAF8	0x9099	
+0xDAF9	0x90AC	
+0xDAFA	0x90A1	
+0xDAFB	0x90B4	
+0xDAFC	0x90B3	
+0xDAFD	0x90B6	
+0xDAFE	0x90BA	
+0xDB40	0x8DD5	
+0xDB41	0x8DD8	
+0xDB42	0x8DD9	
+0xDB43	0x8DDC	
+0xDB44	0x8DE0	
+0xDB45	0x8DE1	
+0xDB46	0x8DE2	
+0xDB47	0x8DE5	
+0xDB48	0x8DE6	
+0xDB49	0x8DE7	
+0xDB4A	0x8DE9	
+0xDB4B	0x8DED	
+0xDB4C	0x8DEE	
+0xDB4D	0x8DF0	
+0xDB4E	0x8DF1	
+0xDB4F	0x8DF2	
+0xDB50	0x8DF4	
+0xDB51	0x8DF6	
+0xDB52	0x8DFC	
+0xDB53	0x8DFE	
+0xDB54	0x8DFF	
+0xDB55	0x8E00	
+0xDB56	0x8E01	
+0xDB57	0x8E02	
+0xDB58	0x8E03	
+0xDB59	0x8E04	
+0xDB5A	0x8E06	
+0xDB5B	0x8E07	
+0xDB5C	0x8E08	
+0xDB5D	0x8E0B	
+0xDB5E	0x8E0D	
+0xDB5F	0x8E0E	
+0xDB60	0x8E10	
+0xDB61	0x8E11	
+0xDB62	0x8E12	
+0xDB63	0x8E13	
+0xDB64	0x8E15	
+0xDB65	0x8E16	
+0xDB66	0x8E17	
+0xDB67	0x8E18	
+0xDB68	0x8E19	
+0xDB69	0x8E1A	
+0xDB6A	0x8E1B	
+0xDB6B	0x8E1C	
+0xDB6C	0x8E20	
+0xDB6D	0x8E21	
+0xDB6E	0x8E24	
+0xDB6F	0x8E25	
+0xDB70	0x8E26	
+0xDB71	0x8E27	
+0xDB72	0x8E28	
+0xDB73	0x8E2B	
+0xDB74	0x8E2D	
+0xDB75	0x8E30	
+0xDB76	0x8E32	
+0xDB77	0x8E33	
+0xDB78	0x8E34	
+0xDB79	0x8E36	
+0xDB7A	0x8E37	
+0xDB7B	0x8E38	
+0xDB7C	0x8E3B	
+0xDB7D	0x8E3C	
+0xDB7E	0x8E3E	
+0xDB80	0x8E3F	
+0xDB81	0x8E43	
+0xDB82	0x8E45	
+0xDB83	0x8E46	
+0xDB84	0x8E4C	
+0xDB85	0x8E4D	
+0xDB86	0x8E4E	
+0xDB87	0x8E4F	
+0xDB88	0x8E50	
+0xDB89	0x8E53	
+0xDB8A	0x8E54	
+0xDB8B	0x8E55	
+0xDB8C	0x8E56	
+0xDB8D	0x8E57	
+0xDB8E	0x8E58	
+0xDB8F	0x8E5A	
+0xDB90	0x8E5B	
+0xDB91	0x8E5C	
+0xDB92	0x8E5D	
+0xDB93	0x8E5E	
+0xDB94	0x8E5F	
+0xDB95	0x8E60	
+0xDB96	0x8E61	
+0xDB97	0x8E62	
+0xDB98	0x8E63	
+0xDB99	0x8E64	
+0xDB9A	0x8E65	
+0xDB9B	0x8E67	
+0xDB9C	0x8E68	
+0xDB9D	0x8E6A	
+0xDB9E	0x8E6B	
+0xDB9F	0x8E6E	
+0xDBA0	0x8E71	
+0xDBA1	0x90B8	
+0xDBA2	0x90B0	
+0xDBA3	0x90CF	
+0xDBA4	0x90C5	
+0xDBA5	0x90BE	
+0xDBA6	0x90D0	
+0xDBA7	0x90C4	
+0xDBA8	0x90C7	
+0xDBA9	0x90D3	
+0xDBAA	0x90E6	
+0xDBAB	0x90E2	
+0xDBAC	0x90DC	
+0xDBAD	0x90D7	
+0xDBAE	0x90DB	
+0xDBAF	0x90EB	
+0xDBB0	0x90EF	
+0xDBB1	0x90FE	
+0xDBB2	0x9104	
+0xDBB3	0x9122	
+0xDBB4	0x911E	
+0xDBB5	0x9123	
+0xDBB6	0x9131	
+0xDBB7	0x912F	
+0xDBB8	0x9139	
+0xDBB9	0x9143	
+0xDBBA	0x9146	
+0xDBBB	0x520D	
+0xDBBC	0x5942	
+0xDBBD	0x52A2	
+0xDBBE	0x52AC	
+0xDBBF	0x52AD	
+0xDBC0	0x52BE	
+0xDBC1	0x54FF	
+0xDBC2	0x52D0	
+0xDBC3	0x52D6	
+0xDBC4	0x52F0	
+0xDBC5	0x53DF	
+0xDBC6	0x71EE	
+0xDBC7	0x77CD	
+0xDBC8	0x5EF4	
+0xDBC9	0x51F5	
+0xDBCA	0x51FC	
+0xDBCB	0x9B2F	
+0xDBCC	0x53B6	
+0xDBCD	0x5F01	
+0xDBCE	0x755A	
+0xDBCF	0x5DEF	
+0xDBD0	0x574C	
+0xDBD1	0x57A9	
+0xDBD2	0x57A1	
+0xDBD3	0x587E	
+0xDBD4	0x58BC	
+0xDBD5	0x58C5	
+0xDBD6	0x58D1	
+0xDBD7	0x5729	
+0xDBD8	0x572C	
+0xDBD9	0x572A	
+0xDBDA	0x5733	
+0xDBDB	0x5739	
+0xDBDC	0x572E	
+0xDBDD	0x572F	
+0xDBDE	0x575C	
+0xDBDF	0x573B	
+0xDBE0	0x5742	
+0xDBE1	0x5769	
+0xDBE2	0x5785	
+0xDBE3	0x576B	
+0xDBE4	0x5786	
+0xDBE5	0x577C	
+0xDBE6	0x577B	
+0xDBE7	0x5768	
+0xDBE8	0x576D	
+0xDBE9	0x5776	
+0xDBEA	0x5773	
+0xDBEB	0x57AD	
+0xDBEC	0x57A4	
+0xDBED	0x578C	
+0xDBEE	0x57B2	
+0xDBEF	0x57CF	
+0xDBF0	0x57A7	
+0xDBF1	0x57B4	
+0xDBF2	0x5793	
+0xDBF3	0x57A0	
+0xDBF4	0x57D5	
+0xDBF5	0x57D8	
+0xDBF6	0x57DA	
+0xDBF7	0x57D9	
+0xDBF8	0x57D2	
+0xDBF9	0x57B8	
+0xDBFA	0x57F4	
+0xDBFB	0x57EF	
+0xDBFC	0x57F8	
+0xDBFD	0x57E4	
+0xDBFE	0x57DD	
+0xDC40	0x8E73	
+0xDC41	0x8E75	
+0xDC42	0x8E77	
+0xDC43	0x8E78	
+0xDC44	0x8E79	
+0xDC45	0x8E7A	
+0xDC46	0x8E7B	
+0xDC47	0x8E7D	
+0xDC48	0x8E7E	
+0xDC49	0x8E80	
+0xDC4A	0x8E82	
+0xDC4B	0x8E83	
+0xDC4C	0x8E84	
+0xDC4D	0x8E86	
+0xDC4E	0x8E88	
+0xDC4F	0x8E89	
+0xDC50	0x8E8A	
+0xDC51	0x8E8B	
+0xDC52	0x8E8C	
+0xDC53	0x8E8D	
+0xDC54	0x8E8E	
+0xDC55	0x8E91	
+0xDC56	0x8E92	
+0xDC57	0x8E93	
+0xDC58	0x8E95	
+0xDC59	0x8E96	
+0xDC5A	0x8E97	
+0xDC5B	0x8E98	
+0xDC5C	0x8E99	
+0xDC5D	0x8E9A	
+0xDC5E	0x8E9B	
+0xDC5F	0x8E9D	
+0xDC60	0x8E9F	
+0xDC61	0x8EA0	
+0xDC62	0x8EA1	
+0xDC63	0x8EA2	
+0xDC64	0x8EA3	
+0xDC65	0x8EA4	
+0xDC66	0x8EA5	
+0xDC67	0x8EA6	
+0xDC68	0x8EA7	
+0xDC69	0x8EA8	
+0xDC6A	0x8EA9	
+0xDC6B	0x8EAA	
+0xDC6C	0x8EAD	
+0xDC6D	0x8EAE	
+0xDC6E	0x8EB0	
+0xDC6F	0x8EB1	
+0xDC70	0x8EB3	
+0xDC71	0x8EB4	
+0xDC72	0x8EB5	
+0xDC73	0x8EB6	
+0xDC74	0x8EB7	
+0xDC75	0x8EB8	
+0xDC76	0x8EB9	
+0xDC77	0x8EBB	
+0xDC78	0x8EBC	
+0xDC79	0x8EBD	
+0xDC7A	0x8EBE	
+0xDC7B	0x8EBF	
+0xDC7C	0x8EC0	
+0xDC7D	0x8EC1	
+0xDC7E	0x8EC2	
+0xDC80	0x8EC3	
+0xDC81	0x8EC4	
+0xDC82	0x8EC5	
+0xDC83	0x8EC6	
+0xDC84	0x8EC7	
+0xDC85	0x8EC8	
+0xDC86	0x8EC9	
+0xDC87	0x8ECA	
+0xDC88	0x8ECB	
+0xDC89	0x8ECC	
+0xDC8A	0x8ECD	
+0xDC8B	0x8ECF	
+0xDC8C	0x8ED0	
+0xDC8D	0x8ED1	
+0xDC8E	0x8ED2	
+0xDC8F	0x8ED3	
+0xDC90	0x8ED4	
+0xDC91	0x8ED5	
+0xDC92	0x8ED6	
+0xDC93	0x8ED7	
+0xDC94	0x8ED8	
+0xDC95	0x8ED9	
+0xDC96	0x8EDA	
+0xDC97	0x8EDB	
+0xDC98	0x8EDC	
+0xDC99	0x8EDD	
+0xDC9A	0x8EDE	
+0xDC9B	0x8EDF	
+0xDC9C	0x8EE0	
+0xDC9D	0x8EE1	
+0xDC9E	0x8EE2	
+0xDC9F	0x8EE3	
+0xDCA0	0x8EE4	
+0xDCA1	0x580B	
+0xDCA2	0x580D	
+0xDCA3	0x57FD	
+0xDCA4	0x57ED	
+0xDCA5	0x5800	
+0xDCA6	0x581E	
+0xDCA7	0x5819	
+0xDCA8	0x5844	
+0xDCA9	0x5820	
+0xDCAA	0x5865	
+0xDCAB	0x586C	
+0xDCAC	0x5881	
+0xDCAD	0x5889	
+0xDCAE	0x589A	
+0xDCAF	0x5880	
+0xDCB0	0x99A8	
+0xDCB1	0x9F19	
+0xDCB2	0x61FF	
+0xDCB3	0x8279	
+0xDCB4	0x827D	
+0xDCB5	0x827F	
+0xDCB6	0x828F	
+0xDCB7	0x828A	
+0xDCB8	0x82A8	
+0xDCB9	0x8284	
+0xDCBA	0x828E	
+0xDCBB	0x8291	
+0xDCBC	0x8297	
+0xDCBD	0x8299	
+0xDCBE	0x82AB	
+0xDCBF	0x82B8	
+0xDCC0	0x82BE	
+0xDCC1	0x82B0	
+0xDCC2	0x82C8	
+0xDCC3	0x82CA	
+0xDCC4	0x82E3	
+0xDCC5	0x8298	
+0xDCC6	0x82B7	
+0xDCC7	0x82AE	
+0xDCC8	0x82CB	
+0xDCC9	0x82CC	
+0xDCCA	0x82C1	
+0xDCCB	0x82A9	
+0xDCCC	0x82B4	
+0xDCCD	0x82A1	
+0xDCCE	0x82AA	
+0xDCCF	0x829F	
+0xDCD0	0x82C4	
+0xDCD1	0x82CE	
+0xDCD2	0x82A4	
+0xDCD3	0x82E1	
+0xDCD4	0x8309	
+0xDCD5	0x82F7	
+0xDCD6	0x82E4	
+0xDCD7	0x830F	
+0xDCD8	0x8307	
+0xDCD9	0x82DC	
+0xDCDA	0x82F4	
+0xDCDB	0x82D2	
+0xDCDC	0x82D8	
+0xDCDD	0x830C	
+0xDCDE	0x82FB	
+0xDCDF	0x82D3	
+0xDCE0	0x8311	
+0xDCE1	0x831A	
+0xDCE2	0x8306	
+0xDCE3	0x8314	
+0xDCE4	0x8315	
+0xDCE5	0x82E0	
+0xDCE6	0x82D5	
+0xDCE7	0x831C	
+0xDCE8	0x8351	
+0xDCE9	0x835B	
+0xDCEA	0x835C	
+0xDCEB	0x8308	
+0xDCEC	0x8392	
+0xDCED	0x833C	
+0xDCEE	0x8334	
+0xDCEF	0x8331	
+0xDCF0	0x839B	
+0xDCF1	0x835E	
+0xDCF2	0x832F	
+0xDCF3	0x834F	
+0xDCF4	0x8347	
+0xDCF5	0x8343	
+0xDCF6	0x835F	
+0xDCF7	0x8340	
+0xDCF8	0x8317	
+0xDCF9	0x8360	
+0xDCFA	0x832D	
+0xDCFB	0x833A	
+0xDCFC	0x8333	
+0xDCFD	0x8366	
+0xDCFE	0x8365	
+0xDD40	0x8EE5	
+0xDD41	0x8EE6	
+0xDD42	0x8EE7	
+0xDD43	0x8EE8	
+0xDD44	0x8EE9	
+0xDD45	0x8EEA	
+0xDD46	0x8EEB	
+0xDD47	0x8EEC	
+0xDD48	0x8EED	
+0xDD49	0x8EEE	
+0xDD4A	0x8EEF	
+0xDD4B	0x8EF0	
+0xDD4C	0x8EF1	
+0xDD4D	0x8EF2	
+0xDD4E	0x8EF3	
+0xDD4F	0x8EF4	
+0xDD50	0x8EF5	
+0xDD51	0x8EF6	
+0xDD52	0x8EF7	
+0xDD53	0x8EF8	
+0xDD54	0x8EF9	
+0xDD55	0x8EFA	
+0xDD56	0x8EFB	
+0xDD57	0x8EFC	
+0xDD58	0x8EFD	
+0xDD59	0x8EFE	
+0xDD5A	0x8EFF	
+0xDD5B	0x8F00	
+0xDD5C	0x8F01	
+0xDD5D	0x8F02	
+0xDD5E	0x8F03	
+0xDD5F	0x8F04	
+0xDD60	0x8F05	
+0xDD61	0x8F06	
+0xDD62	0x8F07	
+0xDD63	0x8F08	
+0xDD64	0x8F09	
+0xDD65	0x8F0A	
+0xDD66	0x8F0B	
+0xDD67	0x8F0C	
+0xDD68	0x8F0D	
+0xDD69	0x8F0E	
+0xDD6A	0x8F0F	
+0xDD6B	0x8F10	
+0xDD6C	0x8F11	
+0xDD6D	0x8F12	
+0xDD6E	0x8F13	
+0xDD6F	0x8F14	
+0xDD70	0x8F15	
+0xDD71	0x8F16	
+0xDD72	0x8F17	
+0xDD73	0x8F18	
+0xDD74	0x8F19	
+0xDD75	0x8F1A	
+0xDD76	0x8F1B	
+0xDD77	0x8F1C	
+0xDD78	0x8F1D	
+0xDD79	0x8F1E	
+0xDD7A	0x8F1F	
+0xDD7B	0x8F20	
+0xDD7C	0x8F21	
+0xDD7D	0x8F22	
+0xDD7E	0x8F23	
+0xDD80	0x8F24	
+0xDD81	0x8F25	
+0xDD82	0x8F26	
+0xDD83	0x8F27	
+0xDD84	0x8F28	
+0xDD85	0x8F29	
+0xDD86	0x8F2A	
+0xDD87	0x8F2B	
+0xDD88	0x8F2C	
+0xDD89	0x8F2D	
+0xDD8A	0x8F2E	
+0xDD8B	0x8F2F	
+0xDD8C	0x8F30	
+0xDD8D	0x8F31	
+0xDD8E	0x8F32	
+0xDD8F	0x8F33	
+0xDD90	0x8F34	
+0xDD91	0x8F35	
+0xDD92	0x8F36	
+0xDD93	0x8F37	
+0xDD94	0x8F38	
+0xDD95	0x8F39	
+0xDD96	0x8F3A	
+0xDD97	0x8F3B	
+0xDD98	0x8F3C	
+0xDD99	0x8F3D	
+0xDD9A	0x8F3E	
+0xDD9B	0x8F3F	
+0xDD9C	0x8F40	
+0xDD9D	0x8F41	
+0xDD9E	0x8F42	
+0xDD9F	0x8F43	
+0xDDA0	0x8F44	
+0xDDA1	0x8368	
+0xDDA2	0x831B	
+0xDDA3	0x8369	
+0xDDA4	0x836C	
+0xDDA5	0x836A	
+0xDDA6	0x836D	
+0xDDA7	0x836E	
+0xDDA8	0x83B0	
+0xDDA9	0x8378	
+0xDDAA	0x83B3	
+0xDDAB	0x83B4	
+0xDDAC	0x83A0	
+0xDDAD	0x83AA	
+0xDDAE	0x8393	
+0xDDAF	0x839C	
+0xDDB0	0x8385	
+0xDDB1	0x837C	
+0xDDB2	0x83B6	
+0xDDB3	0x83A9	
+0xDDB4	0x837D	
+0xDDB5	0x83B8	
+0xDDB6	0x837B	
+0xDDB7	0x8398	
+0xDDB8	0x839E	
+0xDDB9	0x83A8	
+0xDDBA	0x83BA	
+0xDDBB	0x83BC	
+0xDDBC	0x83C1	
+0xDDBD	0x8401	
+0xDDBE	0x83E5	
+0xDDBF	0x83D8	
+0xDDC0	0x5807	
+0xDDC1	0x8418	
+0xDDC2	0x840B	
+0xDDC3	0x83DD	
+0xDDC4	0x83FD	
+0xDDC5	0x83D6	
+0xDDC6	0x841C	
+0xDDC7	0x8438	
+0xDDC8	0x8411	
+0xDDC9	0x8406	
+0xDDCA	0x83D4	
+0xDDCB	0x83DF	
+0xDDCC	0x840F	
+0xDDCD	0x8403	
+0xDDCE	0x83F8	
+0xDDCF	0x83F9	
+0xDDD0	0x83EA	
+0xDDD1	0x83C5	
+0xDDD2	0x83C0	
+0xDDD3	0x8426	
+0xDDD4	0x83F0	
+0xDDD5	0x83E1	
+0xDDD6	0x845C	
+0xDDD7	0x8451	
+0xDDD8	0x845A	
+0xDDD9	0x8459	
+0xDDDA	0x8473	
+0xDDDB	0x8487	
+0xDDDC	0x8488	
+0xDDDD	0x847A	
+0xDDDE	0x8489	
+0xDDDF	0x8478	
+0xDDE0	0x843C	
+0xDDE1	0x8446	
+0xDDE2	0x8469	
+0xDDE3	0x8476	
+0xDDE4	0x848C	
+0xDDE5	0x848E	
+0xDDE6	0x8431	
+0xDDE7	0x846D	
+0xDDE8	0x84C1	
+0xDDE9	0x84CD	
+0xDDEA	0x84D0	
+0xDDEB	0x84E6	
+0xDDEC	0x84BD	
+0xDDED	0x84D3	
+0xDDEE	0x84CA	
+0xDDEF	0x84BF	
+0xDDF0	0x84BA	
+0xDDF1	0x84E0	
+0xDDF2	0x84A1	
+0xDDF3	0x84B9	
+0xDDF4	0x84B4	
+0xDDF5	0x8497	
+0xDDF6	0x84E5	
+0xDDF7	0x84E3	
+0xDDF8	0x850C	
+0xDDF9	0x750D	
+0xDDFA	0x8538	
+0xDDFB	0x84F0	
+0xDDFC	0x8539	
+0xDDFD	0x851F	
+0xDDFE	0x853A	
+0xDE40	0x8F45	
+0xDE41	0x8F46	
+0xDE42	0x8F47	
+0xDE43	0x8F48	
+0xDE44	0x8F49	
+0xDE45	0x8F4A	
+0xDE46	0x8F4B	
+0xDE47	0x8F4C	
+0xDE48	0x8F4D	
+0xDE49	0x8F4E	
+0xDE4A	0x8F4F	
+0xDE4B	0x8F50	
+0xDE4C	0x8F51	
+0xDE4D	0x8F52	
+0xDE4E	0x8F53	
+0xDE4F	0x8F54	
+0xDE50	0x8F55	
+0xDE51	0x8F56	
+0xDE52	0x8F57	
+0xDE53	0x8F58	
+0xDE54	0x8F59	
+0xDE55	0x8F5A	
+0xDE56	0x8F5B	
+0xDE57	0x8F5C	
+0xDE58	0x8F5D	
+0xDE59	0x8F5E	
+0xDE5A	0x8F5F	
+0xDE5B	0x8F60	
+0xDE5C	0x8F61	
+0xDE5D	0x8F62	
+0xDE5E	0x8F63	
+0xDE5F	0x8F64	
+0xDE60	0x8F65	
+0xDE61	0x8F6A	
+0xDE62	0x8F80	
+0xDE63	0x8F8C	
+0xDE64	0x8F92	
+0xDE65	0x8F9D	
+0xDE66	0x8FA0	
+0xDE67	0x8FA1	
+0xDE68	0x8FA2	
+0xDE69	0x8FA4	
+0xDE6A	0x8FA5	
+0xDE6B	0x8FA6	
+0xDE6C	0x8FA7	
+0xDE6D	0x8FAA	
+0xDE6E	0x8FAC	
+0xDE6F	0x8FAD	
+0xDE70	0x8FAE	
+0xDE71	0x8FAF	
+0xDE72	0x8FB2	
+0xDE73	0x8FB3	
+0xDE74	0x8FB4	
+0xDE75	0x8FB5	
+0xDE76	0x8FB7	
+0xDE77	0x8FB8	
+0xDE78	0x8FBA	
+0xDE79	0x8FBB	
+0xDE7A	0x8FBC	
+0xDE7B	0x8FBF	
+0xDE7C	0x8FC0	
+0xDE7D	0x8FC3	
+0xDE7E	0x8FC6	
+0xDE80	0x8FC9	
+0xDE81	0x8FCA	
+0xDE82	0x8FCB	
+0xDE83	0x8FCC	
+0xDE84	0x8FCD	
+0xDE85	0x8FCF	
+0xDE86	0x8FD2	
+0xDE87	0x8FD6	
+0xDE88	0x8FD7	
+0xDE89	0x8FDA	
+0xDE8A	0x8FE0	
+0xDE8B	0x8FE1	
+0xDE8C	0x8FE3	
+0xDE8D	0x8FE7	
+0xDE8E	0x8FEC	
+0xDE8F	0x8FEF	
+0xDE90	0x8FF1	
+0xDE91	0x8FF2	
+0xDE92	0x8FF4	
+0xDE93	0x8FF5	
+0xDE94	0x8FF6	
+0xDE95	0x8FFA	
+0xDE96	0x8FFB	
+0xDE97	0x8FFC	
+0xDE98	0x8FFE	
+0xDE99	0x8FFF	
+0xDE9A	0x9007	
+0xDE9B	0x9008	
+0xDE9C	0x900C	
+0xDE9D	0x900E	
+0xDE9E	0x9013	
+0xDE9F	0x9015	
+0xDEA0	0x9018	
+0xDEA1	0x8556	
+0xDEA2	0x853B	
+0xDEA3	0x84FF	
+0xDEA4	0x84FC	
+0xDEA5	0x8559	
+0xDEA6	0x8548	
+0xDEA7	0x8568	
+0xDEA8	0x8564	
+0xDEA9	0x855E	
+0xDEAA	0x857A	
+0xDEAB	0x77A2	
+0xDEAC	0x8543	
+0xDEAD	0x8572	
+0xDEAE	0x857B	
+0xDEAF	0x85A4	
+0xDEB0	0x85A8	
+0xDEB1	0x8587	
+0xDEB2	0x858F	
+0xDEB3	0x8579	
+0xDEB4	0x85AE	
+0xDEB5	0x859C	
+0xDEB6	0x8585	
+0xDEB7	0x85B9	
+0xDEB8	0x85B7	
+0xDEB9	0x85B0	
+0xDEBA	0x85D3	
+0xDEBB	0x85C1	
+0xDEBC	0x85DC	
+0xDEBD	0x85FF	
+0xDEBE	0x8627	
+0xDEBF	0x8605	
+0xDEC0	0x8629	
+0xDEC1	0x8616	
+0xDEC2	0x863C	
+0xDEC3	0x5EFE	
+0xDEC4	0x5F08	
+0xDEC5	0x593C	
+0xDEC6	0x5941	
+0xDEC7	0x8037	
+0xDEC8	0x5955	
+0xDEC9	0x595A	
+0xDECA	0x5958	
+0xDECB	0x530F	
+0xDECC	0x5C22	
+0xDECD	0x5C25	
+0xDECE	0x5C2C	
+0xDECF	0x5C34	
+0xDED0	0x624C	
+0xDED1	0x626A	
+0xDED2	0x629F	
+0xDED3	0x62BB	
+0xDED4	0x62CA	
+0xDED5	0x62DA	
+0xDED6	0x62D7	
+0xDED7	0x62EE	
+0xDED8	0x6322	
+0xDED9	0x62F6	
+0xDEDA	0x6339	
+0xDEDB	0x634B	
+0xDEDC	0x6343	
+0xDEDD	0x63AD	
+0xDEDE	0x63F6	
+0xDEDF	0x6371	
+0xDEE0	0x637A	
+0xDEE1	0x638E	
+0xDEE2	0x63B4	
+0xDEE3	0x636D	
+0xDEE4	0x63AC	
+0xDEE5	0x638A	
+0xDEE6	0x6369	
+0xDEE7	0x63AE	
+0xDEE8	0x63BC	
+0xDEE9	0x63F2	
+0xDEEA	0x63F8	
+0xDEEB	0x63E0	
+0xDEEC	0x63FF	
+0xDEED	0x63C4	
+0xDEEE	0x63DE	
+0xDEEF	0x63CE	
+0xDEF0	0x6452	
+0xDEF1	0x63C6	
+0xDEF2	0x63BE	
+0xDEF3	0x6445	
+0xDEF4	0x6441	
+0xDEF5	0x640B	
+0xDEF6	0x641B	
+0xDEF7	0x6420	
+0xDEF8	0x640C	
+0xDEF9	0x6426	
+0xDEFA	0x6421	
+0xDEFB	0x645E	
+0xDEFC	0x6484	
+0xDEFD	0x646D	
+0xDEFE	0x6496	
+0xDF40	0x9019	
+0xDF41	0x901C	
+0xDF42	0x9023	
+0xDF43	0x9024	
+0xDF44	0x9025	
+0xDF45	0x9027	
+0xDF46	0x9028	
+0xDF47	0x9029	
+0xDF48	0x902A	
+0xDF49	0x902B	
+0xDF4A	0x902C	
+0xDF4B	0x9030	
+0xDF4C	0x9031	
+0xDF4D	0x9032	
+0xDF4E	0x9033	
+0xDF4F	0x9034	
+0xDF50	0x9037	
+0xDF51	0x9039	
+0xDF52	0x903A	
+0xDF53	0x903D	
+0xDF54	0x903F	
+0xDF55	0x9040	
+0xDF56	0x9043	
+0xDF57	0x9045	
+0xDF58	0x9046	
+0xDF59	0x9048	
+0xDF5A	0x9049	
+0xDF5B	0x904A	
+0xDF5C	0x904B	
+0xDF5D	0x904C	
+0xDF5E	0x904E	
+0xDF5F	0x9054	
+0xDF60	0x9055	
+0xDF61	0x9056	
+0xDF62	0x9059	
+0xDF63	0x905A	
+0xDF64	0x905C	
+0xDF65	0x905D	
+0xDF66	0x905E	
+0xDF67	0x905F	
+0xDF68	0x9060	
+0xDF69	0x9061	
+0xDF6A	0x9064	
+0xDF6B	0x9066	
+0xDF6C	0x9067	
+0xDF6D	0x9069	
+0xDF6E	0x906A	
+0xDF6F	0x906B	
+0xDF70	0x906C	
+0xDF71	0x906F	
+0xDF72	0x9070	
+0xDF73	0x9071	
+0xDF74	0x9072	
+0xDF75	0x9073	
+0xDF76	0x9076	
+0xDF77	0x9077	
+0xDF78	0x9078	
+0xDF79	0x9079	
+0xDF7A	0x907A	
+0xDF7B	0x907B	
+0xDF7C	0x907C	
+0xDF7D	0x907E	
+0xDF7E	0x9081	
+0xDF80	0x9084	
+0xDF81	0x9085	
+0xDF82	0x9086	
+0xDF83	0x9087	
+0xDF84	0x9089	
+0xDF85	0x908A	
+0xDF86	0x908C	
+0xDF87	0x908D	
+0xDF88	0x908E	
+0xDF89	0x908F	
+0xDF8A	0x9090	
+0xDF8B	0x9092	
+0xDF8C	0x9094	
+0xDF8D	0x9096	
+0xDF8E	0x9098	
+0xDF8F	0x909A	
+0xDF90	0x909C	
+0xDF91	0x909E	
+0xDF92	0x909F	
+0xDF93	0x90A0	
+0xDF94	0x90A4	
+0xDF95	0x90A5	
+0xDF96	0x90A7	
+0xDF97	0x90A8	
+0xDF98	0x90A9	
+0xDF99	0x90AB	
+0xDF9A	0x90AD	
+0xDF9B	0x90B2	
+0xDF9C	0x90B7	
+0xDF9D	0x90BC	
+0xDF9E	0x90BD	
+0xDF9F	0x90BF	
+0xDFA0	0x90C0	
+0xDFA1	0x647A	
+0xDFA2	0x64B7	
+0xDFA3	0x64B8	
+0xDFA4	0x6499	
+0xDFA5	0x64BA	
+0xDFA6	0x64C0	
+0xDFA7	0x64D0	
+0xDFA8	0x64D7	
+0xDFA9	0x64E4	
+0xDFAA	0x64E2	
+0xDFAB	0x6509	
+0xDFAC	0x6525	
+0xDFAD	0x652E	
+0xDFAE	0x5F0B	
+0xDFAF	0x5FD2	
+0xDFB0	0x7519	
+0xDFB1	0x5F11	
+0xDFB2	0x535F	
+0xDFB3	0x53F1	
+0xDFB4	0x53FD	
+0xDFB5	0x53E9	
+0xDFB6	0x53E8	
+0xDFB7	0x53FB	
+0xDFB8	0x5412	
+0xDFB9	0x5416	
+0xDFBA	0x5406	
+0xDFBB	0x544B	
+0xDFBC	0x5452	
+0xDFBD	0x5453	
+0xDFBE	0x5454	
+0xDFBF	0x5456	
+0xDFC0	0x5443	
+0xDFC1	0x5421	
+0xDFC2	0x5457	
+0xDFC3	0x5459	
+0xDFC4	0x5423	
+0xDFC5	0x5432	
+0xDFC6	0x5482	
+0xDFC7	0x5494	
+0xDFC8	0x5477	
+0xDFC9	0x5471	
+0xDFCA	0x5464	
+0xDFCB	0x549A	
+0xDFCC	0x549B	
+0xDFCD	0x5484	
+0xDFCE	0x5476	
+0xDFCF	0x5466	
+0xDFD0	0x549D	
+0xDFD1	0x54D0	
+0xDFD2	0x54AD	
+0xDFD3	0x54C2	
+0xDFD4	0x54B4	
+0xDFD5	0x54D2	
+0xDFD6	0x54A7	
+0xDFD7	0x54A6	
+0xDFD8	0x54D3	
+0xDFD9	0x54D4	
+0xDFDA	0x5472	
+0xDFDB	0x54A3	
+0xDFDC	0x54D5	
+0xDFDD	0x54BB	
+0xDFDE	0x54BF	
+0xDFDF	0x54CC	
+0xDFE0	0x54D9	
+0xDFE1	0x54DA	
+0xDFE2	0x54DC	
+0xDFE3	0x54A9	
+0xDFE4	0x54AA	
+0xDFE5	0x54A4	
+0xDFE6	0x54DD	
+0xDFE7	0x54CF	
+0xDFE8	0x54DE	
+0xDFE9	0x551B	
+0xDFEA	0x54E7	
+0xDFEB	0x5520	
+0xDFEC	0x54FD	
+0xDFED	0x5514	
+0xDFEE	0x54F3	
+0xDFEF	0x5522	
+0xDFF0	0x5523	
+0xDFF1	0x550F	
+0xDFF2	0x5511	
+0xDFF3	0x5527	
+0xDFF4	0x552A	
+0xDFF5	0x5567	
+0xDFF6	0x558F	
+0xDFF7	0x55B5	
+0xDFF8	0x5549	
+0xDFF9	0x556D	
+0xDFFA	0x5541	
+0xDFFB	0x5555	
+0xDFFC	0x553F	
+0xDFFD	0x5550	
+0xDFFE	0x553C	
+0xE040	0x90C2	
+0xE041	0x90C3	
+0xE042	0x90C6	
+0xE043	0x90C8	
+0xE044	0x90C9	
+0xE045	0x90CB	
+0xE046	0x90CC	
+0xE047	0x90CD	
+0xE048	0x90D2	
+0xE049	0x90D4	
+0xE04A	0x90D5	
+0xE04B	0x90D6	
+0xE04C	0x90D8	
+0xE04D	0x90D9	
+0xE04E	0x90DA	
+0xE04F	0x90DE	
+0xE050	0x90DF	
+0xE051	0x90E0	
+0xE052	0x90E3	
+0xE053	0x90E4	
+0xE054	0x90E5	
+0xE055	0x90E9	
+0xE056	0x90EA	
+0xE057	0x90EC	
+0xE058	0x90EE	
+0xE059	0x90F0	
+0xE05A	0x90F1	
+0xE05B	0x90F2	
+0xE05C	0x90F3	
+0xE05D	0x90F5	
+0xE05E	0x90F6	
+0xE05F	0x90F7	
+0xE060	0x90F9	
+0xE061	0x90FA	
+0xE062	0x90FB	
+0xE063	0x90FC	
+0xE064	0x90FF	
+0xE065	0x9100	
+0xE066	0x9101	
+0xE067	0x9103	
+0xE068	0x9105	
+0xE069	0x9106	
+0xE06A	0x9107	
+0xE06B	0x9108	
+0xE06C	0x9109	
+0xE06D	0x910A	
+0xE06E	0x910B	
+0xE06F	0x910C	
+0xE070	0x910D	
+0xE071	0x910E	
+0xE072	0x910F	
+0xE073	0x9110	
+0xE074	0x9111	
+0xE075	0x9112	
+0xE076	0x9113	
+0xE077	0x9114	
+0xE078	0x9115	
+0xE079	0x9116	
+0xE07A	0x9117	
+0xE07B	0x9118	
+0xE07C	0x911A	
+0xE07D	0x911B	
+0xE07E	0x911C	
+0xE080	0x911D	
+0xE081	0x911F	
+0xE082	0x9120	
+0xE083	0x9121	
+0xE084	0x9124	
+0xE085	0x9125	
+0xE086	0x9126	
+0xE087	0x9127	
+0xE088	0x9128	
+0xE089	0x9129	
+0xE08A	0x912A	
+0xE08B	0x912B	
+0xE08C	0x912C	
+0xE08D	0x912D	
+0xE08E	0x912E	
+0xE08F	0x9130	
+0xE090	0x9132	
+0xE091	0x9133	
+0xE092	0x9134	
+0xE093	0x9135	
+0xE094	0x9136	
+0xE095	0x9137	
+0xE096	0x9138	
+0xE097	0x913A	
+0xE098	0x913B	
+0xE099	0x913C	
+0xE09A	0x913D	
+0xE09B	0x913E	
+0xE09C	0x913F	
+0xE09D	0x9140	
+0xE09E	0x9141	
+0xE09F	0x9142	
+0xE0A0	0x9144	
+0xE0A1	0x5537	
+0xE0A2	0x5556	
+0xE0A3	0x5575	
+0xE0A4	0x5576	
+0xE0A5	0x5577	
+0xE0A6	0x5533	
+0xE0A7	0x5530	
+0xE0A8	0x555C	
+0xE0A9	0x558B	
+0xE0AA	0x55D2	
+0xE0AB	0x5583	
+0xE0AC	0x55B1	
+0xE0AD	0x55B9	
+0xE0AE	0x5588	
+0xE0AF	0x5581	
+0xE0B0	0x559F	
+0xE0B1	0x557E	
+0xE0B2	0x55D6	
+0xE0B3	0x5591	
+0xE0B4	0x557B	
+0xE0B5	0x55DF	
+0xE0B6	0x55BD	
+0xE0B7	0x55BE	
+0xE0B8	0x5594	
+0xE0B9	0x5599	
+0xE0BA	0x55EA	
+0xE0BB	0x55F7	
+0xE0BC	0x55C9	
+0xE0BD	0x561F	
+0xE0BE	0x55D1	
+0xE0BF	0x55EB	
+0xE0C0	0x55EC	
+0xE0C1	0x55D4	
+0xE0C2	0x55E6	
+0xE0C3	0x55DD	
+0xE0C4	0x55C4	
+0xE0C5	0x55EF	
+0xE0C6	0x55E5	
+0xE0C7	0x55F2	
+0xE0C8	0x55F3	
+0xE0C9	0x55CC	
+0xE0CA	0x55CD	
+0xE0CB	0x55E8	
+0xE0CC	0x55F5	
+0xE0CD	0x55E4	
+0xE0CE	0x8F94	
+0xE0CF	0x561E	
+0xE0D0	0x5608	
+0xE0D1	0x560C	
+0xE0D2	0x5601	
+0xE0D3	0x5624	
+0xE0D4	0x5623	
+0xE0D5	0x55FE	
+0xE0D6	0x5600	
+0xE0D7	0x5627	
+0xE0D8	0x562D	
+0xE0D9	0x5658	
+0xE0DA	0x5639	
+0xE0DB	0x5657	
+0xE0DC	0x562C	
+0xE0DD	0x564D	
+0xE0DE	0x5662	
+0xE0DF	0x5659	
+0xE0E0	0x565C	
+0xE0E1	0x564C	
+0xE0E2	0x5654	
+0xE0E3	0x5686	
+0xE0E4	0x5664	
+0xE0E5	0x5671	
+0xE0E6	0x566B	
+0xE0E7	0x567B	
+0xE0E8	0x567C	
+0xE0E9	0x5685	
+0xE0EA	0x5693	
+0xE0EB	0x56AF	
+0xE0EC	0x56D4	
+0xE0ED	0x56D7	
+0xE0EE	0x56DD	
+0xE0EF	0x56E1	
+0xE0F0	0x56F5	
+0xE0F1	0x56EB	
+0xE0F2	0x56F9	
+0xE0F3	0x56FF	
+0xE0F4	0x5704	
+0xE0F5	0x570A	
+0xE0F6	0x5709	
+0xE0F7	0x571C	
+0xE0F8	0x5E0F	
+0xE0F9	0x5E19	
+0xE0FA	0x5E14	
+0xE0FB	0x5E11	
+0xE0FC	0x5E31	
+0xE0FD	0x5E3B	
+0xE0FE	0x5E3C	
+0xE140	0x9145	
+0xE141	0x9147	
+0xE142	0x9148	
+0xE143	0x9151	
+0xE144	0x9153	
+0xE145	0x9154	
+0xE146	0x9155	
+0xE147	0x9156	
+0xE148	0x9158	
+0xE149	0x9159	
+0xE14A	0x915B	
+0xE14B	0x915C	
+0xE14C	0x915F	
+0xE14D	0x9160	
+0xE14E	0x9166	
+0xE14F	0x9167	
+0xE150	0x9168	
+0xE151	0x916B	
+0xE152	0x916D	
+0xE153	0x9173	
+0xE154	0x917A	
+0xE155	0x917B	
+0xE156	0x917C	
+0xE157	0x9180	
+0xE158	0x9181	
+0xE159	0x9182	
+0xE15A	0x9183	
+0xE15B	0x9184	
+0xE15C	0x9186	
+0xE15D	0x9188	
+0xE15E	0x918A	
+0xE15F	0x918E	
+0xE160	0x918F	
+0xE161	0x9193	
+0xE162	0x9194	
+0xE163	0x9195	
+0xE164	0x9196	
+0xE165	0x9197	
+0xE166	0x9198	
+0xE167	0x9199	
+0xE168	0x919C	
+0xE169	0x919D	
+0xE16A	0x919E	
+0xE16B	0x919F	
+0xE16C	0x91A0	
+0xE16D	0x91A1	
+0xE16E	0x91A4	
+0xE16F	0x91A5	
+0xE170	0x91A6	
+0xE171	0x91A7	
+0xE172	0x91A8	
+0xE173	0x91A9	
+0xE174	0x91AB	
+0xE175	0x91AC	
+0xE176	0x91B0	
+0xE177	0x91B1	
+0xE178	0x91B2	
+0xE179	0x91B3	
+0xE17A	0x91B6	
+0xE17B	0x91B7	
+0xE17C	0x91B8	
+0xE17D	0x91B9	
+0xE17E	0x91BB	
+0xE180	0x91BC	
+0xE181	0x91BD	
+0xE182	0x91BE	
+0xE183	0x91BF	
+0xE184	0x91C0	
+0xE185	0x91C1	
+0xE186	0x91C2	
+0xE187	0x91C3	
+0xE188	0x91C4	
+0xE189	0x91C5	
+0xE18A	0x91C6	
+0xE18B	0x91C8	
+0xE18C	0x91CB	
+0xE18D	0x91D0	
+0xE18E	0x91D2	
+0xE18F	0x91D3	
+0xE190	0x91D4	
+0xE191	0x91D5	
+0xE192	0x91D6	
+0xE193	0x91D7	
+0xE194	0x91D8	
+0xE195	0x91D9	
+0xE196	0x91DA	
+0xE197	0x91DB	
+0xE198	0x91DD	
+0xE199	0x91DE	
+0xE19A	0x91DF	
+0xE19B	0x91E0	
+0xE19C	0x91E1	
+0xE19D	0x91E2	
+0xE19E	0x91E3	
+0xE19F	0x91E4	
+0xE1A0	0x91E5	
+0xE1A1	0x5E37	
+0xE1A2	0x5E44	
+0xE1A3	0x5E54	
+0xE1A4	0x5E5B	
+0xE1A5	0x5E5E	
+0xE1A6	0x5E61	
+0xE1A7	0x5C8C	
+0xE1A8	0x5C7A	
+0xE1A9	0x5C8D	
+0xE1AA	0x5C90	
+0xE1AB	0x5C96	
+0xE1AC	0x5C88	
+0xE1AD	0x5C98	
+0xE1AE	0x5C99	
+0xE1AF	0x5C91	
+0xE1B0	0x5C9A	
+0xE1B1	0x5C9C	
+0xE1B2	0x5CB5	
+0xE1B3	0x5CA2	
+0xE1B4	0x5CBD	
+0xE1B5	0x5CAC	
+0xE1B6	0x5CAB	
+0xE1B7	0x5CB1	
+0xE1B8	0x5CA3	
+0xE1B9	0x5CC1	
+0xE1BA	0x5CB7	
+0xE1BB	0x5CC4	
+0xE1BC	0x5CD2	
+0xE1BD	0x5CE4	
+0xE1BE	0x5CCB	
+0xE1BF	0x5CE5	
+0xE1C0	0x5D02	
+0xE1C1	0x5D03	
+0xE1C2	0x5D27	
+0xE1C3	0x5D26	
+0xE1C4	0x5D2E	
+0xE1C5	0x5D24	
+0xE1C6	0x5D1E	
+0xE1C7	0x5D06	
+0xE1C8	0x5D1B	
+0xE1C9	0x5D58	
+0xE1CA	0x5D3E	
+0xE1CB	0x5D34	
+0xE1CC	0x5D3D	
+0xE1CD	0x5D6C	
+0xE1CE	0x5D5B	
+0xE1CF	0x5D6F	
+0xE1D0	0x5D5D	
+0xE1D1	0x5D6B	
+0xE1D2	0x5D4B	
+0xE1D3	0x5D4A	
+0xE1D4	0x5D69	
+0xE1D5	0x5D74	
+0xE1D6	0x5D82	
+0xE1D7	0x5D99	
+0xE1D8	0x5D9D	
+0xE1D9	0x8C73	
+0xE1DA	0x5DB7	
+0xE1DB	0x5DC5	
+0xE1DC	0x5F73	
+0xE1DD	0x5F77	
+0xE1DE	0x5F82	
+0xE1DF	0x5F87	
+0xE1E0	0x5F89	
+0xE1E1	0x5F8C	
+0xE1E2	0x5F95	
+0xE1E3	0x5F99	
+0xE1E4	0x5F9C	
+0xE1E5	0x5FA8	
+0xE1E6	0x5FAD	
+0xE1E7	0x5FB5	
+0xE1E8	0x5FBC	
+0xE1E9	0x8862	
+0xE1EA	0x5F61	
+0xE1EB	0x72AD	
+0xE1EC	0x72B0	
+0xE1ED	0x72B4	
+0xE1EE	0x72B7	
+0xE1EF	0x72B8	
+0xE1F0	0x72C3	
+0xE1F1	0x72C1	
+0xE1F2	0x72CE	
+0xE1F3	0x72CD	
+0xE1F4	0x72D2	
+0xE1F5	0x72E8	
+0xE1F6	0x72EF	
+0xE1F7	0x72E9	
+0xE1F8	0x72F2	
+0xE1F9	0x72F4	
+0xE1FA	0x72F7	
+0xE1FB	0x7301	
+0xE1FC	0x72F3	
+0xE1FD	0x7303	
+0xE1FE	0x72FA	
+0xE240	0x91E6	
+0xE241	0x91E7	
+0xE242	0x91E8	
+0xE243	0x91E9	
+0xE244	0x91EA	
+0xE245	0x91EB	
+0xE246	0x91EC	
+0xE247	0x91ED	
+0xE248	0x91EE	
+0xE249	0x91EF	
+0xE24A	0x91F0	
+0xE24B	0x91F1	
+0xE24C	0x91F2	
+0xE24D	0x91F3	
+0xE24E	0x91F4	
+0xE24F	0x91F5	
+0xE250	0x91F6	
+0xE251	0x91F7	
+0xE252	0x91F8	
+0xE253	0x91F9	
+0xE254	0x91FA	
+0xE255	0x91FB	
+0xE256	0x91FC	
+0xE257	0x91FD	
+0xE258	0x91FE	
+0xE259	0x91FF	
+0xE25A	0x9200	
+0xE25B	0x9201	
+0xE25C	0x9202	
+0xE25D	0x9203	
+0xE25E	0x9204	
+0xE25F	0x9205	
+0xE260	0x9206	
+0xE261	0x9207	
+0xE262	0x9208	
+0xE263	0x9209	
+0xE264	0x920A	
+0xE265	0x920B	
+0xE266	0x920C	
+0xE267	0x920D	
+0xE268	0x920E	
+0xE269	0x920F	
+0xE26A	0x9210	
+0xE26B	0x9211	
+0xE26C	0x9212	
+0xE26D	0x9213	
+0xE26E	0x9214	
+0xE26F	0x9215	
+0xE270	0x9216	
+0xE271	0x9217	
+0xE272	0x9218	
+0xE273	0x9219	
+0xE274	0x921A	
+0xE275	0x921B	
+0xE276	0x921C	
+0xE277	0x921D	
+0xE278	0x921E	
+0xE279	0x921F	
+0xE27A	0x9220	
+0xE27B	0x9221	
+0xE27C	0x9222	
+0xE27D	0x9223	
+0xE27E	0x9224	
+0xE280	0x9225	
+0xE281	0x9226	
+0xE282	0x9227	
+0xE283	0x9228	
+0xE284	0x9229	
+0xE285	0x922A	
+0xE286	0x922B	
+0xE287	0x922C	
+0xE288	0x922D	
+0xE289	0x922E	
+0xE28A	0x922F	
+0xE28B	0x9230	
+0xE28C	0x9231	
+0xE28D	0x9232	
+0xE28E	0x9233	
+0xE28F	0x9234	
+0xE290	0x9235	
+0xE291	0x9236	
+0xE292	0x9237	
+0xE293	0x9238	
+0xE294	0x9239	
+0xE295	0x923A	
+0xE296	0x923B	
+0xE297	0x923C	
+0xE298	0x923D	
+0xE299	0x923E	
+0xE29A	0x923F	
+0xE29B	0x9240	
+0xE29C	0x9241	
+0xE29D	0x9242	
+0xE29E	0x9243	
+0xE29F	0x9244	
+0xE2A0	0x9245	
+0xE2A1	0x72FB	
+0xE2A2	0x7317	
+0xE2A3	0x7313	
+0xE2A4	0x7321	
+0xE2A5	0x730A	
+0xE2A6	0x731E	
+0xE2A7	0x731D	
+0xE2A8	0x7315	
+0xE2A9	0x7322	
+0xE2AA	0x7339	
+0xE2AB	0x7325	
+0xE2AC	0x732C	
+0xE2AD	0x7338	
+0xE2AE	0x7331	
+0xE2AF	0x7350	
+0xE2B0	0x734D	
+0xE2B1	0x7357	
+0xE2B2	0x7360	
+0xE2B3	0x736C	
+0xE2B4	0x736F	
+0xE2B5	0x737E	
+0xE2B6	0x821B	
+0xE2B7	0x5925	
+0xE2B8	0x98E7	
+0xE2B9	0x5924	
+0xE2BA	0x5902	
+0xE2BB	0x9963	
+0xE2BC	0x9967	
+0xE2BD	0x9968	
+0xE2BE	0x9969	
+0xE2BF	0x996A	
+0xE2C0	0x996B	
+0xE2C1	0x996C	
+0xE2C2	0x9974	
+0xE2C3	0x9977	
+0xE2C4	0x997D	
+0xE2C5	0x9980	
+0xE2C6	0x9984	
+0xE2C7	0x9987	
+0xE2C8	0x998A	
+0xE2C9	0x998D	
+0xE2CA	0x9990	
+0xE2CB	0x9991	
+0xE2CC	0x9993	
+0xE2CD	0x9994	
+0xE2CE	0x9995	
+0xE2CF	0x5E80	
+0xE2D0	0x5E91	
+0xE2D1	0x5E8B	
+0xE2D2	0x5E96	
+0xE2D3	0x5EA5	
+0xE2D4	0x5EA0	
+0xE2D5	0x5EB9	
+0xE2D6	0x5EB5	
+0xE2D7	0x5EBE	
+0xE2D8	0x5EB3	
+0xE2D9	0x8D53	
+0xE2DA	0x5ED2	
+0xE2DB	0x5ED1	
+0xE2DC	0x5EDB	
+0xE2DD	0x5EE8	
+0xE2DE	0x5EEA	
+0xE2DF	0x81BA	
+0xE2E0	0x5FC4	
+0xE2E1	0x5FC9	
+0xE2E2	0x5FD6	
+0xE2E3	0x5FCF	
+0xE2E4	0x6003	
+0xE2E5	0x5FEE	
+0xE2E6	0x6004	
+0xE2E7	0x5FE1	
+0xE2E8	0x5FE4	
+0xE2E9	0x5FFE	
+0xE2EA	0x6005	
+0xE2EB	0x6006	
+0xE2EC	0x5FEA	
+0xE2ED	0x5FED	
+0xE2EE	0x5FF8	
+0xE2EF	0x6019	
+0xE2F0	0x6035	
+0xE2F1	0x6026	
+0xE2F2	0x601B	
+0xE2F3	0x600F	
+0xE2F4	0x600D	
+0xE2F5	0x6029	
+0xE2F6	0x602B	
+0xE2F7	0x600A	
+0xE2F8	0x603F	
+0xE2F9	0x6021	
+0xE2FA	0x6078	
+0xE2FB	0x6079	
+0xE2FC	0x607B	
+0xE2FD	0x607A	
+0xE2FE	0x6042	
+0xE340	0x9246	
+0xE341	0x9247	
+0xE342	0x9248	
+0xE343	0x9249	
+0xE344	0x924A	
+0xE345	0x924B	
+0xE346	0x924C	
+0xE347	0x924D	
+0xE348	0x924E	
+0xE349	0x924F	
+0xE34A	0x9250	
+0xE34B	0x9251	
+0xE34C	0x9252	
+0xE34D	0x9253	
+0xE34E	0x9254	
+0xE34F	0x9255	
+0xE350	0x9256	
+0xE351	0x9257	
+0xE352	0x9258	
+0xE353	0x9259	
+0xE354	0x925A	
+0xE355	0x925B	
+0xE356	0x925C	
+0xE357	0x925D	
+0xE358	0x925E	
+0xE359	0x925F	
+0xE35A	0x9260	
+0xE35B	0x9261	
+0xE35C	0x9262	
+0xE35D	0x9263	
+0xE35E	0x9264	
+0xE35F	0x9265	
+0xE360	0x9266	
+0xE361	0x9267	
+0xE362	0x9268	
+0xE363	0x9269	
+0xE364	0x926A	
+0xE365	0x926B	
+0xE366	0x926C	
+0xE367	0x926D	
+0xE368	0x926E	
+0xE369	0x926F	
+0xE36A	0x9270	
+0xE36B	0x9271	
+0xE36C	0x9272	
+0xE36D	0x9273	
+0xE36E	0x9275	
+0xE36F	0x9276	
+0xE370	0x9277	
+0xE371	0x9278	
+0xE372	0x9279	
+0xE373	0x927A	
+0xE374	0x927B	
+0xE375	0x927C	
+0xE376	0x927D	
+0xE377	0x927E	
+0xE378	0x927F	
+0xE379	0x9280	
+0xE37A	0x9281	
+0xE37B	0x9282	
+0xE37C	0x9283	
+0xE37D	0x9284	
+0xE37E	0x9285	
+0xE380	0x9286	
+0xE381	0x9287	
+0xE382	0x9288	
+0xE383	0x9289	
+0xE384	0x928A	
+0xE385	0x928B	
+0xE386	0x928C	
+0xE387	0x928D	
+0xE388	0x928F	
+0xE389	0x9290	
+0xE38A	0x9291	
+0xE38B	0x9292	
+0xE38C	0x9293	
+0xE38D	0x9294	
+0xE38E	0x9295	
+0xE38F	0x9296	
+0xE390	0x9297	
+0xE391	0x9298	
+0xE392	0x9299	
+0xE393	0x929A	
+0xE394	0x929B	
+0xE395	0x929C	
+0xE396	0x929D	
+0xE397	0x929E	
+0xE398	0x929F	
+0xE399	0x92A0	
+0xE39A	0x92A1	
+0xE39B	0x92A2	
+0xE39C	0x92A3	
+0xE39D	0x92A4	
+0xE39E	0x92A5	
+0xE39F	0x92A6	
+0xE3A0	0x92A7	
+0xE3A1	0x606A	
+0xE3A2	0x607D	
+0xE3A3	0x6096	
+0xE3A4	0x609A	
+0xE3A5	0x60AD	
+0xE3A6	0x609D	
+0xE3A7	0x6083	
+0xE3A8	0x6092	
+0xE3A9	0x608C	
+0xE3AA	0x609B	
+0xE3AB	0x60EC	
+0xE3AC	0x60BB	
+0xE3AD	0x60B1	
+0xE3AE	0x60DD	
+0xE3AF	0x60D8	
+0xE3B0	0x60C6	
+0xE3B1	0x60DA	
+0xE3B2	0x60B4	
+0xE3B3	0x6120	
+0xE3B4	0x6126	
+0xE3B5	0x6115	
+0xE3B6	0x6123	
+0xE3B7	0x60F4	
+0xE3B8	0x6100	
+0xE3B9	0x610E	
+0xE3BA	0x612B	
+0xE3BB	0x614A	
+0xE3BC	0x6175	
+0xE3BD	0x61AC	
+0xE3BE	0x6194	
+0xE3BF	0x61A7	
+0xE3C0	0x61B7	
+0xE3C1	0x61D4	
+0xE3C2	0x61F5	
+0xE3C3	0x5FDD	
+0xE3C4	0x96B3	
+0xE3C5	0x95E9	
+0xE3C6	0x95EB	
+0xE3C7	0x95F1	
+0xE3C8	0x95F3	
+0xE3C9	0x95F5	
+0xE3CA	0x95F6	
+0xE3CB	0x95FC	
+0xE3CC	0x95FE	
+0xE3CD	0x9603	
+0xE3CE	0x9604	
+0xE3CF	0x9606	
+0xE3D0	0x9608	
+0xE3D1	0x960A	
+0xE3D2	0x960B	
+0xE3D3	0x960C	
+0xE3D4	0x960D	
+0xE3D5	0x960F	
+0xE3D6	0x9612	
+0xE3D7	0x9615	
+0xE3D8	0x9616	
+0xE3D9	0x9617	
+0xE3DA	0x9619	
+0xE3DB	0x961A	
+0xE3DC	0x4E2C	
+0xE3DD	0x723F	
+0xE3DE	0x6215	
+0xE3DF	0x6C35	
+0xE3E0	0x6C54	
+0xE3E1	0x6C5C	
+0xE3E2	0x6C4A	
+0xE3E3	0x6CA3	
+0xE3E4	0x6C85	
+0xE3E5	0x6C90	
+0xE3E6	0x6C94	
+0xE3E7	0x6C8C	
+0xE3E8	0x6C68	
+0xE3E9	0x6C69	
+0xE3EA	0x6C74	
+0xE3EB	0x6C76	
+0xE3EC	0x6C86	
+0xE3ED	0x6CA9	
+0xE3EE	0x6CD0	
+0xE3EF	0x6CD4	
+0xE3F0	0x6CAD	
+0xE3F1	0x6CF7	
+0xE3F2	0x6CF8	
+0xE3F3	0x6CF1	
+0xE3F4	0x6CD7	
+0xE3F5	0x6CB2	
+0xE3F6	0x6CE0	
+0xE3F7	0x6CD6	
+0xE3F8	0x6CFA	
+0xE3F9	0x6CEB	
+0xE3FA	0x6CEE	
+0xE3FB	0x6CB1	
+0xE3FC	0x6CD3	
+0xE3FD	0x6CEF	
+0xE3FE	0x6CFE	
+0xE440	0x92A8	
+0xE441	0x92A9	
+0xE442	0x92AA	
+0xE443	0x92AB	
+0xE444	0x92AC	
+0xE445	0x92AD	
+0xE446	0x92AF	
+0xE447	0x92B0	
+0xE448	0x92B1	
+0xE449	0x92B2	
+0xE44A	0x92B3	
+0xE44B	0x92B4	
+0xE44C	0x92B5	
+0xE44D	0x92B6	
+0xE44E	0x92B7	
+0xE44F	0x92B8	
+0xE450	0x92B9	
+0xE451	0x92BA	
+0xE452	0x92BB	
+0xE453	0x92BC	
+0xE454	0x92BD	
+0xE455	0x92BE	
+0xE456	0x92BF	
+0xE457	0x92C0	
+0xE458	0x92C1	
+0xE459	0x92C2	
+0xE45A	0x92C3	
+0xE45B	0x92C4	
+0xE45C	0x92C5	
+0xE45D	0x92C6	
+0xE45E	0x92C7	
+0xE45F	0x92C9	
+0xE460	0x92CA	
+0xE461	0x92CB	
+0xE462	0x92CC	
+0xE463	0x92CD	
+0xE464	0x92CE	
+0xE465	0x92CF	
+0xE466	0x92D0	
+0xE467	0x92D1	
+0xE468	0x92D2	
+0xE469	0x92D3	
+0xE46A	0x92D4	
+0xE46B	0x92D5	
+0xE46C	0x92D6	
+0xE46D	0x92D7	
+0xE46E	0x92D8	
+0xE46F	0x92D9	
+0xE470	0x92DA	
+0xE471	0x92DB	
+0xE472	0x92DC	
+0xE473	0x92DD	
+0xE474	0x92DE	
+0xE475	0x92DF	
+0xE476	0x92E0	
+0xE477	0x92E1	
+0xE478	0x92E2	
+0xE479	0x92E3	
+0xE47A	0x92E4	
+0xE47B	0x92E5	
+0xE47C	0x92E6	
+0xE47D	0x92E7	
+0xE47E	0x92E8	
+0xE480	0x92E9	
+0xE481	0x92EA	
+0xE482	0x92EB	
+0xE483	0x92EC	
+0xE484	0x92ED	
+0xE485	0x92EE	
+0xE486	0x92EF	
+0xE487	0x92F0	
+0xE488	0x92F1	
+0xE489	0x92F2	
+0xE48A	0x92F3	
+0xE48B	0x92F4	
+0xE48C	0x92F5	
+0xE48D	0x92F6	
+0xE48E	0x92F7	
+0xE48F	0x92F8	
+0xE490	0x92F9	
+0xE491	0x92FA	
+0xE492	0x92FB	
+0xE493	0x92FC	
+0xE494	0x92FD	
+0xE495	0x92FE	
+0xE496	0x92FF	
+0xE497	0x9300	
+0xE498	0x9301	
+0xE499	0x9302	
+0xE49A	0x9303	
+0xE49B	0x9304	
+0xE49C	0x9305	
+0xE49D	0x9306	
+0xE49E	0x9307	
+0xE49F	0x9308	
+0xE4A0	0x9309	
+0xE4A1	0x6D39	
+0xE4A2	0x6D27	
+0xE4A3	0x6D0C	
+0xE4A4	0x6D43	
+0xE4A5	0x6D48	
+0xE4A6	0x6D07	
+0xE4A7	0x6D04	
+0xE4A8	0x6D19	
+0xE4A9	0x6D0E	
+0xE4AA	0x6D2B	
+0xE4AB	0x6D4D	
+0xE4AC	0x6D2E	
+0xE4AD	0x6D35	
+0xE4AE	0x6D1A	
+0xE4AF	0x6D4F	
+0xE4B0	0x6D52	
+0xE4B1	0x6D54	
+0xE4B2	0x6D33	
+0xE4B3	0x6D91	
+0xE4B4	0x6D6F	
+0xE4B5	0x6D9E	
+0xE4B6	0x6DA0	
+0xE4B7	0x6D5E	
+0xE4B8	0x6D93	
+0xE4B9	0x6D94	
+0xE4BA	0x6D5C	
+0xE4BB	0x6D60	
+0xE4BC	0x6D7C	
+0xE4BD	0x6D63	
+0xE4BE	0x6E1A	
+0xE4BF	0x6DC7	
+0xE4C0	0x6DC5	
+0xE4C1	0x6DDE	
+0xE4C2	0x6E0E	
+0xE4C3	0x6DBF	
+0xE4C4	0x6DE0	
+0xE4C5	0x6E11	
+0xE4C6	0x6DE6	
+0xE4C7	0x6DDD	
+0xE4C8	0x6DD9	
+0xE4C9	0x6E16	
+0xE4CA	0x6DAB	
+0xE4CB	0x6E0C	
+0xE4CC	0x6DAE	
+0xE4CD	0x6E2B	
+0xE4CE	0x6E6E	
+0xE4CF	0x6E4E	
+0xE4D0	0x6E6B	
+0xE4D1	0x6EB2	
+0xE4D2	0x6E5F	
+0xE4D3	0x6E86	
+0xE4D4	0x6E53	
+0xE4D5	0x6E54	
+0xE4D6	0x6E32	
+0xE4D7	0x6E25	
+0xE4D8	0x6E44	
+0xE4D9	0x6EDF	
+0xE4DA	0x6EB1	
+0xE4DB	0x6E98	
+0xE4DC	0x6EE0	
+0xE4DD	0x6F2D	
+0xE4DE	0x6EE2	
+0xE4DF	0x6EA5	
+0xE4E0	0x6EA7	
+0xE4E1	0x6EBD	
+0xE4E2	0x6EBB	
+0xE4E3	0x6EB7	
+0xE4E4	0x6ED7	
+0xE4E5	0x6EB4	
+0xE4E6	0x6ECF	
+0xE4E7	0x6E8F	
+0xE4E8	0x6EC2	
+0xE4E9	0x6E9F	
+0xE4EA	0x6F62	
+0xE4EB	0x6F46	
+0xE4EC	0x6F47	
+0xE4ED	0x6F24	
+0xE4EE	0x6F15	
+0xE4EF	0x6EF9	
+0xE4F0	0x6F2F	
+0xE4F1	0x6F36	
+0xE4F2	0x6F4B	
+0xE4F3	0x6F74	
+0xE4F4	0x6F2A	
+0xE4F5	0x6F09	
+0xE4F6	0x6F29	
+0xE4F7	0x6F89	
+0xE4F8	0x6F8D	
+0xE4F9	0x6F8C	
+0xE4FA	0x6F78	
+0xE4FB	0x6F72	
+0xE4FC	0x6F7C	
+0xE4FD	0x6F7A	
+0xE4FE	0x6FD1	
+0xE540	0x930A	
+0xE541	0x930B	
+0xE542	0x930C	
+0xE543	0x930D	
+0xE544	0x930E	
+0xE545	0x930F	
+0xE546	0x9310	
+0xE547	0x9311	
+0xE548	0x9312	
+0xE549	0x9313	
+0xE54A	0x9314	
+0xE54B	0x9315	
+0xE54C	0x9316	
+0xE54D	0x9317	
+0xE54E	0x9318	
+0xE54F	0x9319	
+0xE550	0x931A	
+0xE551	0x931B	
+0xE552	0x931C	
+0xE553	0x931D	
+0xE554	0x931E	
+0xE555	0x931F	
+0xE556	0x9320	
+0xE557	0x9321	
+0xE558	0x9322	
+0xE559	0x9323	
+0xE55A	0x9324	
+0xE55B	0x9325	
+0xE55C	0x9326	
+0xE55D	0x9327	
+0xE55E	0x9328	
+0xE55F	0x9329	
+0xE560	0x932A	
+0xE561	0x932B	
+0xE562	0x932C	
+0xE563	0x932D	
+0xE564	0x932E	
+0xE565	0x932F	
+0xE566	0x9330	
+0xE567	0x9331	
+0xE568	0x9332	
+0xE569	0x9333	
+0xE56A	0x9334	
+0xE56B	0x9335	
+0xE56C	0x9336	
+0xE56D	0x9337	
+0xE56E	0x9338	
+0xE56F	0x9339	
+0xE570	0x933A	
+0xE571	0x933B	
+0xE572	0x933C	
+0xE573	0x933D	
+0xE574	0x933F	
+0xE575	0x9340	
+0xE576	0x9341	
+0xE577	0x9342	
+0xE578	0x9343	
+0xE579	0x9344	
+0xE57A	0x9345	
+0xE57B	0x9346	
+0xE57C	0x9347	
+0xE57D	0x9348	
+0xE57E	0x9349	
+0xE580	0x934A	
+0xE581	0x934B	
+0xE582	0x934C	
+0xE583	0x934D	
+0xE584	0x934E	
+0xE585	0x934F	
+0xE586	0x9350	
+0xE587	0x9351	
+0xE588	0x9352	
+0xE589	0x9353	
+0xE58A	0x9354	
+0xE58B	0x9355	
+0xE58C	0x9356	
+0xE58D	0x9357	
+0xE58E	0x9358	
+0xE58F	0x9359	
+0xE590	0x935A	
+0xE591	0x935B	
+0xE592	0x935C	
+0xE593	0x935D	
+0xE594	0x935E	
+0xE595	0x935F	
+0xE596	0x9360	
+0xE597	0x9361	
+0xE598	0x9362	
+0xE599	0x9363	
+0xE59A	0x9364	
+0xE59B	0x9365	
+0xE59C	0x9366	
+0xE59D	0x9367	
+0xE59E	0x9368	
+0xE59F	0x9369	
+0xE5A0	0x936B	
+0xE5A1	0x6FC9	
+0xE5A2	0x6FA7	
+0xE5A3	0x6FB9	
+0xE5A4	0x6FB6	
+0xE5A5	0x6FC2	
+0xE5A6	0x6FE1	
+0xE5A7	0x6FEE	
+0xE5A8	0x6FDE	
+0xE5A9	0x6FE0	
+0xE5AA	0x6FEF	
+0xE5AB	0x701A	
+0xE5AC	0x7023	
+0xE5AD	0x701B	
+0xE5AE	0x7039	
+0xE5AF	0x7035	
+0xE5B0	0x704F	
+0xE5B1	0x705E	
+0xE5B2	0x5B80	
+0xE5B3	0x5B84	
+0xE5B4	0x5B95	
+0xE5B5	0x5B93	
+0xE5B6	0x5BA5	
+0xE5B7	0x5BB8	
+0xE5B8	0x752F	
+0xE5B9	0x9A9E	
+0xE5BA	0x6434	
+0xE5BB	0x5BE4	
+0xE5BC	0x5BEE	
+0xE5BD	0x8930	
+0xE5BE	0x5BF0	
+0xE5BF	0x8E47	
+0xE5C0	0x8B07	
+0xE5C1	0x8FB6	
+0xE5C2	0x8FD3	
+0xE5C3	0x8FD5	
+0xE5C4	0x8FE5	
+0xE5C5	0x8FEE	
+0xE5C6	0x8FE4	
+0xE5C7	0x8FE9	
+0xE5C8	0x8FE6	
+0xE5C9	0x8FF3	
+0xE5CA	0x8FE8	
+0xE5CB	0x9005	
+0xE5CC	0x9004	
+0xE5CD	0x900B	
+0xE5CE	0x9026	
+0xE5CF	0x9011	
+0xE5D0	0x900D	
+0xE5D1	0x9016	
+0xE5D2	0x9021	
+0xE5D3	0x9035	
+0xE5D4	0x9036	
+0xE5D5	0x902D	
+0xE5D6	0x902F	
+0xE5D7	0x9044	
+0xE5D8	0x9051	
+0xE5D9	0x9052	
+0xE5DA	0x9050	
+0xE5DB	0x9068	
+0xE5DC	0x9058	
+0xE5DD	0x9062	
+0xE5DE	0x905B	
+0xE5DF	0x66B9	
+0xE5E0	0x9074	
+0xE5E1	0x907D	
+0xE5E2	0x9082	
+0xE5E3	0x9088	
+0xE5E4	0x9083	
+0xE5E5	0x908B	
+0xE5E6	0x5F50	
+0xE5E7	0x5F57	
+0xE5E8	0x5F56	
+0xE5E9	0x5F58	
+0xE5EA	0x5C3B	
+0xE5EB	0x54AB	
+0xE5EC	0x5C50	
+0xE5ED	0x5C59	
+0xE5EE	0x5B71	
+0xE5EF	0x5C63	
+0xE5F0	0x5C66	
+0xE5F1	0x7FBC	
+0xE5F2	0x5F2A	
+0xE5F3	0x5F29	
+0xE5F4	0x5F2D	
+0xE5F5	0x8274	
+0xE5F6	0x5F3C	
+0xE5F7	0x9B3B	
+0xE5F8	0x5C6E	
+0xE5F9	0x5981	
+0xE5FA	0x5983	
+0xE5FB	0x598D	
+0xE5FC	0x59A9	
+0xE5FD	0x59AA	
+0xE5FE	0x59A3	
+0xE640	0x936C	
+0xE641	0x936D	
+0xE642	0x936E	
+0xE643	0x936F	
+0xE644	0x9370	
+0xE645	0x9371	
+0xE646	0x9372	
+0xE647	0x9373	
+0xE648	0x9374	
+0xE649	0x9375	
+0xE64A	0x9376	
+0xE64B	0x9377	
+0xE64C	0x9378	
+0xE64D	0x9379	
+0xE64E	0x937A	
+0xE64F	0x937B	
+0xE650	0x937C	
+0xE651	0x937D	
+0xE652	0x937E	
+0xE653	0x937F	
+0xE654	0x9380	
+0xE655	0x9381	
+0xE656	0x9382	
+0xE657	0x9383	
+0xE658	0x9384	
+0xE659	0x9385	
+0xE65A	0x9386	
+0xE65B	0x9387	
+0xE65C	0x9388	
+0xE65D	0x9389	
+0xE65E	0x938A	
+0xE65F	0x938B	
+0xE660	0x938C	
+0xE661	0x938D	
+0xE662	0x938E	
+0xE663	0x9390	
+0xE664	0x9391	
+0xE665	0x9392	
+0xE666	0x9393	
+0xE667	0x9394	
+0xE668	0x9395	
+0xE669	0x9396	
+0xE66A	0x9397	
+0xE66B	0x9398	
+0xE66C	0x9399	
+0xE66D	0x939A	
+0xE66E	0x939B	
+0xE66F	0x939C	
+0xE670	0x939D	
+0xE671	0x939E	
+0xE672	0x939F	
+0xE673	0x93A0	
+0xE674	0x93A1	
+0xE675	0x93A2	
+0xE676	0x93A3	
+0xE677	0x93A4	
+0xE678	0x93A5	
+0xE679	0x93A6	
+0xE67A	0x93A7	
+0xE67B	0x93A8	
+0xE67C	0x93A9	
+0xE67D	0x93AA	
+0xE67E	0x93AB	
+0xE680	0x93AC	
+0xE681	0x93AD	
+0xE682	0x93AE	
+0xE683	0x93AF	
+0xE684	0x93B0	
+0xE685	0x93B1	
+0xE686	0x93B2	
+0xE687	0x93B3	
+0xE688	0x93B4	
+0xE689	0x93B5	
+0xE68A	0x93B6	
+0xE68B	0x93B7	
+0xE68C	0x93B8	
+0xE68D	0x93B9	
+0xE68E	0x93BA	
+0xE68F	0x93BB	
+0xE690	0x93BC	
+0xE691	0x93BD	
+0xE692	0x93BE	
+0xE693	0x93BF	
+0xE694	0x93C0	
+0xE695	0x93C1	
+0xE696	0x93C2	
+0xE697	0x93C3	
+0xE698	0x93C4	
+0xE699	0x93C5	
+0xE69A	0x93C6	
+0xE69B	0x93C7	
+0xE69C	0x93C8	
+0xE69D	0x93C9	
+0xE69E	0x93CB	
+0xE69F	0x93CC	
+0xE6A0	0x93CD	
+0xE6A1	0x5997	
+0xE6A2	0x59CA	
+0xE6A3	0x59AB	
+0xE6A4	0x599E	
+0xE6A5	0x59A4	
+0xE6A6	0x59D2	
+0xE6A7	0x59B2	
+0xE6A8	0x59AF	
+0xE6A9	0x59D7	
+0xE6AA	0x59BE	
+0xE6AB	0x5A05	
+0xE6AC	0x5A06	
+0xE6AD	0x59DD	
+0xE6AE	0x5A08	
+0xE6AF	0x59E3	
+0xE6B0	0x59D8	
+0xE6B1	0x59F9	
+0xE6B2	0x5A0C	
+0xE6B3	0x5A09	
+0xE6B4	0x5A32	
+0xE6B5	0x5A34	
+0xE6B6	0x5A11	
+0xE6B7	0x5A23	
+0xE6B8	0x5A13	
+0xE6B9	0x5A40	
+0xE6BA	0x5A67	
+0xE6BB	0x5A4A	
+0xE6BC	0x5A55	
+0xE6BD	0x5A3C	
+0xE6BE	0x5A62	
+0xE6BF	0x5A75	
+0xE6C0	0x80EC	
+0xE6C1	0x5AAA	
+0xE6C2	0x5A9B	
+0xE6C3	0x5A77	
+0xE6C4	0x5A7A	
+0xE6C5	0x5ABE	
+0xE6C6	0x5AEB	
+0xE6C7	0x5AB2	
+0xE6C8	0x5AD2	
+0xE6C9	0x5AD4	
+0xE6CA	0x5AB8	
+0xE6CB	0x5AE0	
+0xE6CC	0x5AE3	
+0xE6CD	0x5AF1	
+0xE6CE	0x5AD6	
+0xE6CF	0x5AE6	
+0xE6D0	0x5AD8	
+0xE6D1	0x5ADC	
+0xE6D2	0x5B09	
+0xE6D3	0x5B17	
+0xE6D4	0x5B16	
+0xE6D5	0x5B32	
+0xE6D6	0x5B37	
+0xE6D7	0x5B40	
+0xE6D8	0x5C15	
+0xE6D9	0x5C1C	
+0xE6DA	0x5B5A	
+0xE6DB	0x5B65	
+0xE6DC	0x5B73	
+0xE6DD	0x5B51	
+0xE6DE	0x5B53	
+0xE6DF	0x5B62	
+0xE6E0	0x9A75	
+0xE6E1	0x9A77	
+0xE6E2	0x9A78	
+0xE6E3	0x9A7A	
+0xE6E4	0x9A7F	
+0xE6E5	0x9A7D	
+0xE6E6	0x9A80	
+0xE6E7	0x9A81	
+0xE6E8	0x9A85	
+0xE6E9	0x9A88	
+0xE6EA	0x9A8A	
+0xE6EB	0x9A90	
+0xE6EC	0x9A92	
+0xE6ED	0x9A93	
+0xE6EE	0x9A96	
+0xE6EF	0x9A98	
+0xE6F0	0x9A9B	
+0xE6F1	0x9A9C	
+0xE6F2	0x9A9D	
+0xE6F3	0x9A9F	
+0xE6F4	0x9AA0	
+0xE6F5	0x9AA2	
+0xE6F6	0x9AA3	
+0xE6F7	0x9AA5	
+0xE6F8	0x9AA7	
+0xE6F9	0x7E9F	
+0xE6FA	0x7EA1	
+0xE6FB	0x7EA3	
+0xE6FC	0x7EA5	
+0xE6FD	0x7EA8	
+0xE6FE	0x7EA9	
+0xE740	0x93CE	
+0xE741	0x93CF	
+0xE742	0x93D0	
+0xE743	0x93D1	
+0xE744	0x93D2	
+0xE745	0x93D3	
+0xE746	0x93D4	
+0xE747	0x93D5	
+0xE748	0x93D7	
+0xE749	0x93D8	
+0xE74A	0x93D9	
+0xE74B	0x93DA	
+0xE74C	0x93DB	
+0xE74D	0x93DC	
+0xE74E	0x93DD	
+0xE74F	0x93DE	
+0xE750	0x93DF	
+0xE751	0x93E0	
+0xE752	0x93E1	
+0xE753	0x93E2	
+0xE754	0x93E3	
+0xE755	0x93E4	
+0xE756	0x93E5	
+0xE757	0x93E6	
+0xE758	0x93E7	
+0xE759	0x93E8	
+0xE75A	0x93E9	
+0xE75B	0x93EA	
+0xE75C	0x93EB	
+0xE75D	0x93EC	
+0xE75E	0x93ED	
+0xE75F	0x93EE	
+0xE760	0x93EF	
+0xE761	0x93F0	
+0xE762	0x93F1	
+0xE763	0x93F2	
+0xE764	0x93F3	
+0xE765	0x93F4	
+0xE766	0x93F5	
+0xE767	0x93F6	
+0xE768	0x93F7	
+0xE769	0x93F8	
+0xE76A	0x93F9	
+0xE76B	0x93FA	
+0xE76C	0x93FB	
+0xE76D	0x93FC	
+0xE76E	0x93FD	
+0xE76F	0x93FE	
+0xE770	0x93FF	
+0xE771	0x9400	
+0xE772	0x9401	
+0xE773	0x9402	
+0xE774	0x9403	
+0xE775	0x9404	
+0xE776	0x9405	
+0xE777	0x9406	
+0xE778	0x9407	
+0xE779	0x9408	
+0xE77A	0x9409	
+0xE77B	0x940A	
+0xE77C	0x940B	
+0xE77D	0x940C	
+0xE77E	0x940D	
+0xE780	0x940E	
+0xE781	0x940F	
+0xE782	0x9410	
+0xE783	0x9411	
+0xE784	0x9412	
+0xE785	0x9413	
+0xE786	0x9414	
+0xE787	0x9415	
+0xE788	0x9416	
+0xE789	0x9417	
+0xE78A	0x9418	
+0xE78B	0x9419	
+0xE78C	0x941A	
+0xE78D	0x941B	
+0xE78E	0x941C	
+0xE78F	0x941D	
+0xE790	0x941E	
+0xE791	0x941F	
+0xE792	0x9420	
+0xE793	0x9421	
+0xE794	0x9422	
+0xE795	0x9423	
+0xE796	0x9424	
+0xE797	0x9425	
+0xE798	0x9426	
+0xE799	0x9427	
+0xE79A	0x9428	
+0xE79B	0x9429	
+0xE79C	0x942A	
+0xE79D	0x942B	
+0xE79E	0x942C	
+0xE79F	0x942D	
+0xE7A0	0x942E	
+0xE7A1	0x7EAD	
+0xE7A2	0x7EB0	
+0xE7A3	0x7EBE	
+0xE7A4	0x7EC0	
+0xE7A5	0x7EC1	
+0xE7A6	0x7EC2	
+0xE7A7	0x7EC9	
+0xE7A8	0x7ECB	
+0xE7A9	0x7ECC	
+0xE7AA	0x7ED0	
+0xE7AB	0x7ED4	
+0xE7AC	0x7ED7	
+0xE7AD	0x7EDB	
+0xE7AE	0x7EE0	
+0xE7AF	0x7EE1	
+0xE7B0	0x7EE8	
+0xE7B1	0x7EEB	
+0xE7B2	0x7EEE	
+0xE7B3	0x7EEF	
+0xE7B4	0x7EF1	
+0xE7B5	0x7EF2	
+0xE7B6	0x7F0D	
+0xE7B7	0x7EF6	
+0xE7B8	0x7EFA	
+0xE7B9	0x7EFB	
+0xE7BA	0x7EFE	
+0xE7BB	0x7F01	
+0xE7BC	0x7F02	
+0xE7BD	0x7F03	
+0xE7BE	0x7F07	
+0xE7BF	0x7F08	
+0xE7C0	0x7F0B	
+0xE7C1	0x7F0C	
+0xE7C2	0x7F0F	
+0xE7C3	0x7F11	
+0xE7C4	0x7F12	
+0xE7C5	0x7F17	
+0xE7C6	0x7F19	
+0xE7C7	0x7F1C	
+0xE7C8	0x7F1B	
+0xE7C9	0x7F1F	
+0xE7CA	0x7F21	
+0xE7CB	0x7F22	
+0xE7CC	0x7F23	
+0xE7CD	0x7F24	
+0xE7CE	0x7F25	
+0xE7CF	0x7F26	
+0xE7D0	0x7F27	
+0xE7D1	0x7F2A	
+0xE7D2	0x7F2B	
+0xE7D3	0x7F2C	
+0xE7D4	0x7F2D	
+0xE7D5	0x7F2F	
+0xE7D6	0x7F30	
+0xE7D7	0x7F31	
+0xE7D8	0x7F32	
+0xE7D9	0x7F33	
+0xE7DA	0x7F35	
+0xE7DB	0x5E7A	
+0xE7DC	0x757F	
+0xE7DD	0x5DDB	
+0xE7DE	0x753E	
+0xE7DF	0x9095	
+0xE7E0	0x738E	
+0xE7E1	0x7391	
+0xE7E2	0x73AE	
+0xE7E3	0x73A2	
+0xE7E4	0x739F	
+0xE7E5	0x73CF	
+0xE7E6	0x73C2	
+0xE7E7	0x73D1	
+0xE7E8	0x73B7	
+0xE7E9	0x73B3	
+0xE7EA	0x73C0	
+0xE7EB	0x73C9	
+0xE7EC	0x73C8	
+0xE7ED	0x73E5	
+0xE7EE	0x73D9	
+0xE7EF	0x987C	
+0xE7F0	0x740A	
+0xE7F1	0x73E9	
+0xE7F2	0x73E7	
+0xE7F3	0x73DE	
+0xE7F4	0x73BA	
+0xE7F5	0x73F2	
+0xE7F6	0x740F	
+0xE7F7	0x742A	
+0xE7F8	0x745B	
+0xE7F9	0x7426	
+0xE7FA	0x7425	
+0xE7FB	0x7428	
+0xE7FC	0x7430	
+0xE7FD	0x742E	
+0xE7FE	0x742C	
+0xE840	0x942F	
+0xE841	0x9430	
+0xE842	0x9431	
+0xE843	0x9432	
+0xE844	0x9433	
+0xE845	0x9434	
+0xE846	0x9435	
+0xE847	0x9436	
+0xE848	0x9437	
+0xE849	0x9438	
+0xE84A	0x9439	
+0xE84B	0x943A	
+0xE84C	0x943B	
+0xE84D	0x943C	
+0xE84E	0x943D	
+0xE84F	0x943F	
+0xE850	0x9440	
+0xE851	0x9441	
+0xE852	0x9442	
+0xE853	0x9443	
+0xE854	0x9444	
+0xE855	0x9445	
+0xE856	0x9446	
+0xE857	0x9447	
+0xE858	0x9448	
+0xE859	0x9449	
+0xE85A	0x944A	
+0xE85B	0x944B	
+0xE85C	0x944C	
+0xE85D	0x944D	
+0xE85E	0x944E	
+0xE85F	0x944F	
+0xE860	0x9450	
+0xE861	0x9451	
+0xE862	0x9452	
+0xE863	0x9453	
+0xE864	0x9454	
+0xE865	0x9455	
+0xE866	0x9456	
+0xE867	0x9457	
+0xE868	0x9458	
+0xE869	0x9459	
+0xE86A	0x945A	
+0xE86B	0x945B	
+0xE86C	0x945C	
+0xE86D	0x945D	
+0xE86E	0x945E	
+0xE86F	0x945F	
+0xE870	0x9460	
+0xE871	0x9461	
+0xE872	0x9462	
+0xE873	0x9463	
+0xE874	0x9464	
+0xE875	0x9465	
+0xE876	0x9466	
+0xE877	0x9467	
+0xE878	0x9468	
+0xE879	0x9469	
+0xE87A	0x946A	
+0xE87B	0x946C	
+0xE87C	0x946D	
+0xE87D	0x946E	
+0xE87E	0x946F	
+0xE880	0x9470	
+0xE881	0x9471	
+0xE882	0x9472	
+0xE883	0x9473	
+0xE884	0x9474	
+0xE885	0x9475	
+0xE886	0x9476	
+0xE887	0x9477	
+0xE888	0x9478	
+0xE889	0x9479	
+0xE88A	0x947A	
+0xE88B	0x947B	
+0xE88C	0x947C	
+0xE88D	0x947D	
+0xE88E	0x947E	
+0xE88F	0x947F	
+0xE890	0x9480	
+0xE891	0x9481	
+0xE892	0x9482	
+0xE893	0x9483	
+0xE894	0x9484	
+0xE895	0x9491	
+0xE896	0x9496	
+0xE897	0x9498	
+0xE898	0x94C7	
+0xE899	0x94CF	
+0xE89A	0x94D3	
+0xE89B	0x94D4	
+0xE89C	0x94DA	
+0xE89D	0x94E6	
+0xE89E	0x94FB	
+0xE89F	0x951C	
+0xE8A0	0x9520	
+0xE8A1	0x741B	
+0xE8A2	0x741A	
+0xE8A3	0x7441	
+0xE8A4	0x745C	
+0xE8A5	0x7457	
+0xE8A6	0x7455	
+0xE8A7	0x7459	
+0xE8A8	0x7477	
+0xE8A9	0x746D	
+0xE8AA	0x747E	
+0xE8AB	0x749C	
+0xE8AC	0x748E	
+0xE8AD	0x7480	
+0xE8AE	0x7481	
+0xE8AF	0x7487	
+0xE8B0	0x748B	
+0xE8B1	0x749E	
+0xE8B2	0x74A8	
+0xE8B3	0x74A9	
+0xE8B4	0x7490	
+0xE8B5	0x74A7	
+0xE8B6	0x74D2	
+0xE8B7	0x74BA	
+0xE8B8	0x97EA	
+0xE8B9	0x97EB	
+0xE8BA	0x97EC	
+0xE8BB	0x674C	
+0xE8BC	0x6753	
+0xE8BD	0x675E	
+0xE8BE	0x6748	
+0xE8BF	0x6769	
+0xE8C0	0x67A5	
+0xE8C1	0x6787	
+0xE8C2	0x676A	
+0xE8C3	0x6773	
+0xE8C4	0x6798	
+0xE8C5	0x67A7	
+0xE8C6	0x6775	
+0xE8C7	0x67A8	
+0xE8C8	0x679E	
+0xE8C9	0x67AD	
+0xE8CA	0x678B	
+0xE8CB	0x6777	
+0xE8CC	0x677C	
+0xE8CD	0x67F0	
+0xE8CE	0x6809	
+0xE8CF	0x67D8	
+0xE8D0	0x680A	
+0xE8D1	0x67E9	
+0xE8D2	0x67B0	
+0xE8D3	0x680C	
+0xE8D4	0x67D9	
+0xE8D5	0x67B5	
+0xE8D6	0x67DA	
+0xE8D7	0x67B3	
+0xE8D8	0x67DD	
+0xE8D9	0x6800	
+0xE8DA	0x67C3	
+0xE8DB	0x67B8	
+0xE8DC	0x67E2	
+0xE8DD	0x680E	
+0xE8DE	0x67C1	
+0xE8DF	0x67FD	
+0xE8E0	0x6832	
+0xE8E1	0x6833	
+0xE8E2	0x6860	
+0xE8E3	0x6861	
+0xE8E4	0x684E	
+0xE8E5	0x6862	
+0xE8E6	0x6844	
+0xE8E7	0x6864	
+0xE8E8	0x6883	
+0xE8E9	0x681D	
+0xE8EA	0x6855	
+0xE8EB	0x6866	
+0xE8EC	0x6841	
+0xE8ED	0x6867	
+0xE8EE	0x6840	
+0xE8EF	0x683E	
+0xE8F0	0x684A	
+0xE8F1	0x6849	
+0xE8F2	0x6829	
+0xE8F3	0x68B5	
+0xE8F4	0x688F	
+0xE8F5	0x6874	
+0xE8F6	0x6877	
+0xE8F7	0x6893	
+0xE8F8	0x686B	
+0xE8F9	0x68C2	
+0xE8FA	0x696E	
+0xE8FB	0x68FC	
+0xE8FC	0x691F	
+0xE8FD	0x6920	
+0xE8FE	0x68F9	
+0xE940	0x9527	
+0xE941	0x9533	
+0xE942	0x953D	
+0xE943	0x9543	
+0xE944	0x9548	
+0xE945	0x954B	
+0xE946	0x9555	
+0xE947	0x955A	
+0xE948	0x9560	
+0xE949	0x956E	
+0xE94A	0x9574	
+0xE94B	0x9575	
+0xE94C	0x9577	
+0xE94D	0x9578	
+0xE94E	0x9579	
+0xE94F	0x957A	
+0xE950	0x957B	
+0xE951	0x957C	
+0xE952	0x957D	
+0xE953	0x957E	
+0xE954	0x9580	
+0xE955	0x9581	
+0xE956	0x9582	
+0xE957	0x9583	
+0xE958	0x9584	
+0xE959	0x9585	
+0xE95A	0x9586	
+0xE95B	0x9587	
+0xE95C	0x9588	
+0xE95D	0x9589	
+0xE95E	0x958A	
+0xE95F	0x958B	
+0xE960	0x958C	
+0xE961	0x958D	
+0xE962	0x958E	
+0xE963	0x958F	
+0xE964	0x9590	
+0xE965	0x9591	
+0xE966	0x9592	
+0xE967	0x9593	
+0xE968	0x9594	
+0xE969	0x9595	
+0xE96A	0x9596	
+0xE96B	0x9597	
+0xE96C	0x9598	
+0xE96D	0x9599	
+0xE96E	0x959A	
+0xE96F	0x959B	
+0xE970	0x959C	
+0xE971	0x959D	
+0xE972	0x959E	
+0xE973	0x959F	
+0xE974	0x95A0	
+0xE975	0x95A1	
+0xE976	0x95A2	
+0xE977	0x95A3	
+0xE978	0x95A4	
+0xE979	0x95A5	
+0xE97A	0x95A6	
+0xE97B	0x95A7	
+0xE97C	0x95A8	
+0xE97D	0x95A9	
+0xE97E	0x95AA	
+0xE980	0x95AB	
+0xE981	0x95AC	
+0xE982	0x95AD	
+0xE983	0x95AE	
+0xE984	0x95AF	
+0xE985	0x95B0	
+0xE986	0x95B1	
+0xE987	0x95B2	
+0xE988	0x95B3	
+0xE989	0x95B4	
+0xE98A	0x95B5	
+0xE98B	0x95B6	
+0xE98C	0x95B7	
+0xE98D	0x95B8	
+0xE98E	0x95B9	
+0xE98F	0x95BA	
+0xE990	0x95BB	
+0xE991	0x95BC	
+0xE992	0x95BD	
+0xE993	0x95BE	
+0xE994	0x95BF	
+0xE995	0x95C0	
+0xE996	0x95C1	
+0xE997	0x95C2	
+0xE998	0x95C3	
+0xE999	0x95C4	
+0xE99A	0x95C5	
+0xE99B	0x95C6	
+0xE99C	0x95C7	
+0xE99D	0x95C8	
+0xE99E	0x95C9	
+0xE99F	0x95CA	
+0xE9A0	0x95CB	
+0xE9A1	0x6924	
+0xE9A2	0x68F0	
+0xE9A3	0x690B	
+0xE9A4	0x6901	
+0xE9A5	0x6957	
+0xE9A6	0x68E3	
+0xE9A7	0x6910	
+0xE9A8	0x6971	
+0xE9A9	0x6939	
+0xE9AA	0x6960	
+0xE9AB	0x6942	
+0xE9AC	0x695D	
+0xE9AD	0x6984	
+0xE9AE	0x696B	
+0xE9AF	0x6980	
+0xE9B0	0x6998	
+0xE9B1	0x6978	
+0xE9B2	0x6934	
+0xE9B3	0x69CC	
+0xE9B4	0x6987	
+0xE9B5	0x6988	
+0xE9B6	0x69CE	
+0xE9B7	0x6989	
+0xE9B8	0x6966	
+0xE9B9	0x6963	
+0xE9BA	0x6979	
+0xE9BB	0x699B	
+0xE9BC	0x69A7	
+0xE9BD	0x69BB	
+0xE9BE	0x69AB	
+0xE9BF	0x69AD	
+0xE9C0	0x69D4	
+0xE9C1	0x69B1	
+0xE9C2	0x69C1	
+0xE9C3	0x69CA	
+0xE9C4	0x69DF	
+0xE9C5	0x6995	
+0xE9C6	0x69E0	
+0xE9C7	0x698D	
+0xE9C8	0x69FF	
+0xE9C9	0x6A2F	
+0xE9CA	0x69ED	
+0xE9CB	0x6A17	
+0xE9CC	0x6A18	
+0xE9CD	0x6A65	
+0xE9CE	0x69F2	
+0xE9CF	0x6A44	
+0xE9D0	0x6A3E	
+0xE9D1	0x6AA0	
+0xE9D2	0x6A50	
+0xE9D3	0x6A5B	
+0xE9D4	0x6A35	
+0xE9D5	0x6A8E	
+0xE9D6	0x6A79	
+0xE9D7	0x6A3D	
+0xE9D8	0x6A28	
+0xE9D9	0x6A58	
+0xE9DA	0x6A7C	
+0xE9DB	0x6A91	
+0xE9DC	0x6A90	
+0xE9DD	0x6AA9	
+0xE9DE	0x6A97	
+0xE9DF	0x6AAB	
+0xE9E0	0x7337	
+0xE9E1	0x7352	
+0xE9E2	0x6B81	
+0xE9E3	0x6B82	
+0xE9E4	0x6B87	
+0xE9E5	0x6B84	
+0xE9E6	0x6B92	
+0xE9E7	0x6B93	
+0xE9E8	0x6B8D	
+0xE9E9	0x6B9A	
+0xE9EA	0x6B9B	
+0xE9EB	0x6BA1	
+0xE9EC	0x6BAA	
+0xE9ED	0x8F6B	
+0xE9EE	0x8F6D	
+0xE9EF	0x8F71	
+0xE9F0	0x8F72	
+0xE9F1	0x8F73	
+0xE9F2	0x8F75	
+0xE9F3	0x8F76	
+0xE9F4	0x8F78	
+0xE9F5	0x8F77	
+0xE9F6	0x8F79	
+0xE9F7	0x8F7A	
+0xE9F8	0x8F7C	
+0xE9F9	0x8F7E	
+0xE9FA	0x8F81	
+0xE9FB	0x8F82	
+0xE9FC	0x8F84	
+0xE9FD	0x8F87	
+0xE9FE	0x8F8B	
+0xEA40	0x95CC	
+0xEA41	0x95CD	
+0xEA42	0x95CE	
+0xEA43	0x95CF	
+0xEA44	0x95D0	
+0xEA45	0x95D1	
+0xEA46	0x95D2	
+0xEA47	0x95D3	
+0xEA48	0x95D4	
+0xEA49	0x95D5	
+0xEA4A	0x95D6	
+0xEA4B	0x95D7	
+0xEA4C	0x95D8	
+0xEA4D	0x95D9	
+0xEA4E	0x95DA	
+0xEA4F	0x95DB	
+0xEA50	0x95DC	
+0xEA51	0x95DD	
+0xEA52	0x95DE	
+0xEA53	0x95DF	
+0xEA54	0x95E0	
+0xEA55	0x95E1	
+0xEA56	0x95E2	
+0xEA57	0x95E3	
+0xEA58	0x95E4	
+0xEA59	0x95E5	
+0xEA5A	0x95E6	
+0xEA5B	0x95E7	
+0xEA5C	0x95EC	
+0xEA5D	0x95FF	
+0xEA5E	0x9607	
+0xEA5F	0x9613	
+0xEA60	0x9618	
+0xEA61	0x961B	
+0xEA62	0x961E	
+0xEA63	0x9620	
+0xEA64	0x9623	
+0xEA65	0x9624	
+0xEA66	0x9625	
+0xEA67	0x9626	
+0xEA68	0x9627	
+0xEA69	0x9628	
+0xEA6A	0x9629	
+0xEA6B	0x962B	
+0xEA6C	0x962C	
+0xEA6D	0x962D	
+0xEA6E	0x962F	
+0xEA6F	0x9630	
+0xEA70	0x9637	
+0xEA71	0x9638	
+0xEA72	0x9639	
+0xEA73	0x963A	
+0xEA74	0x963E	
+0xEA75	0x9641	
+0xEA76	0x9643	
+0xEA77	0x964A	
+0xEA78	0x964E	
+0xEA79	0x964F	
+0xEA7A	0x9651	
+0xEA7B	0x9652	
+0xEA7C	0x9653	
+0xEA7D	0x9656	
+0xEA7E	0x9657	
+0xEA80	0x9658	
+0xEA81	0x9659	
+0xEA82	0x965A	
+0xEA83	0x965C	
+0xEA84	0x965D	
+0xEA85	0x965E	
+0xEA86	0x9660	
+0xEA87	0x9663	
+0xEA88	0x9665	
+0xEA89	0x9666	
+0xEA8A	0x966B	
+0xEA8B	0x966D	
+0xEA8C	0x966E	
+0xEA8D	0x966F	
+0xEA8E	0x9670	
+0xEA8F	0x9671	
+0xEA90	0x9673	
+0xEA91	0x9678	
+0xEA92	0x9679	
+0xEA93	0x967A	
+0xEA94	0x967B	
+0xEA95	0x967C	
+0xEA96	0x967D	
+0xEA97	0x967E	
+0xEA98	0x967F	
+0xEA99	0x9680	
+0xEA9A	0x9681	
+0xEA9B	0x9682	
+0xEA9C	0x9683	
+0xEA9D	0x9684	
+0xEA9E	0x9687	
+0xEA9F	0x9689	
+0xEAA0	0x968A	
+0xEAA1	0x8F8D	
+0xEAA2	0x8F8E	
+0xEAA3	0x8F8F	
+0xEAA4	0x8F98	
+0xEAA5	0x8F9A	
+0xEAA6	0x8ECE	
+0xEAA7	0x620B	
+0xEAA8	0x6217	
+0xEAA9	0x621B	
+0xEAAA	0x621F	
+0xEAAB	0x6222	
+0xEAAC	0x6221	
+0xEAAD	0x6225	
+0xEAAE	0x6224	
+0xEAAF	0x622C	
+0xEAB0	0x81E7	
+0xEAB1	0x74EF	
+0xEAB2	0x74F4	
+0xEAB3	0x74FF	
+0xEAB4	0x750F	
+0xEAB5	0x7511	
+0xEAB6	0x7513	
+0xEAB7	0x6534	
+0xEAB8	0x65EE	
+0xEAB9	0x65EF	
+0xEABA	0x65F0	
+0xEABB	0x660A	
+0xEABC	0x6619	
+0xEABD	0x6772	
+0xEABE	0x6603	
+0xEABF	0x6615	
+0xEAC0	0x6600	
+0xEAC1	0x7085	
+0xEAC2	0x66F7	
+0xEAC3	0x661D	
+0xEAC4	0x6634	
+0xEAC5	0x6631	
+0xEAC6	0x6636	
+0xEAC7	0x6635	
+0xEAC8	0x8006	
+0xEAC9	0x665F	
+0xEACA	0x6654	
+0xEACB	0x6641	
+0xEACC	0x664F	
+0xEACD	0x6656	
+0xEACE	0x6661	
+0xEACF	0x6657	
+0xEAD0	0x6677	
+0xEAD1	0x6684	
+0xEAD2	0x668C	
+0xEAD3	0x66A7	
+0xEAD4	0x669D	
+0xEAD5	0x66BE	
+0xEAD6	0x66DB	
+0xEAD7	0x66DC	
+0xEAD8	0x66E6	
+0xEAD9	0x66E9	
+0xEADA	0x8D32	
+0xEADB	0x8D33	
+0xEADC	0x8D36	
+0xEADD	0x8D3B	
+0xEADE	0x8D3D	
+0xEADF	0x8D40	
+0xEAE0	0x8D45	
+0xEAE1	0x8D46	
+0xEAE2	0x8D48	
+0xEAE3	0x8D49	
+0xEAE4	0x8D47	
+0xEAE5	0x8D4D	
+0xEAE6	0x8D55	
+0xEAE7	0x8D59	
+0xEAE8	0x89C7	
+0xEAE9	0x89CA	
+0xEAEA	0x89CB	
+0xEAEB	0x89CC	
+0xEAEC	0x89CE	
+0xEAED	0x89CF	
+0xEAEE	0x89D0	
+0xEAEF	0x89D1	
+0xEAF0	0x726E	
+0xEAF1	0x729F	
+0xEAF2	0x725D	
+0xEAF3	0x7266	
+0xEAF4	0x726F	
+0xEAF5	0x727E	
+0xEAF6	0x727F	
+0xEAF7	0x7284	
+0xEAF8	0x728B	
+0xEAF9	0x728D	
+0xEAFA	0x728F	
+0xEAFB	0x7292	
+0xEAFC	0x6308	
+0xEAFD	0x6332	
+0xEAFE	0x63B0	
+0xEB40	0x968C	
+0xEB41	0x968E	
+0xEB42	0x9691	
+0xEB43	0x9692	
+0xEB44	0x9693	
+0xEB45	0x9695	
+0xEB46	0x9696	
+0xEB47	0x969A	
+0xEB48	0x969B	
+0xEB49	0x969D	
+0xEB4A	0x969E	
+0xEB4B	0x969F	
+0xEB4C	0x96A0	
+0xEB4D	0x96A1	
+0xEB4E	0x96A2	
+0xEB4F	0x96A3	
+0xEB50	0x96A4	
+0xEB51	0x96A5	
+0xEB52	0x96A6	
+0xEB53	0x96A8	
+0xEB54	0x96A9	
+0xEB55	0x96AA	
+0xEB56	0x96AB	
+0xEB57	0x96AC	
+0xEB58	0x96AD	
+0xEB59	0x96AE	
+0xEB5A	0x96AF	
+0xEB5B	0x96B1	
+0xEB5C	0x96B2	
+0xEB5D	0x96B4	
+0xEB5E	0x96B5	
+0xEB5F	0x96B7	
+0xEB60	0x96B8	
+0xEB61	0x96BA	
+0xEB62	0x96BB	
+0xEB63	0x96BF	
+0xEB64	0x96C2	
+0xEB65	0x96C3	
+0xEB66	0x96C8	
+0xEB67	0x96CA	
+0xEB68	0x96CB	
+0xEB69	0x96D0	
+0xEB6A	0x96D1	
+0xEB6B	0x96D3	
+0xEB6C	0x96D4	
+0xEB6D	0x96D6	
+0xEB6E	0x96D7	
+0xEB6F	0x96D8	
+0xEB70	0x96D9	
+0xEB71	0x96DA	
+0xEB72	0x96DB	
+0xEB73	0x96DC	
+0xEB74	0x96DD	
+0xEB75	0x96DE	
+0xEB76	0x96DF	
+0xEB77	0x96E1	
+0xEB78	0x96E2	
+0xEB79	0x96E3	
+0xEB7A	0x96E4	
+0xEB7B	0x96E5	
+0xEB7C	0x96E6	
+0xEB7D	0x96E7	
+0xEB7E	0x96EB	
+0xEB80	0x96EC	
+0xEB81	0x96ED	
+0xEB82	0x96EE	
+0xEB83	0x96F0	
+0xEB84	0x96F1	
+0xEB85	0x96F2	
+0xEB86	0x96F4	
+0xEB87	0x96F5	
+0xEB88	0x96F8	
+0xEB89	0x96FA	
+0xEB8A	0x96FB	
+0xEB8B	0x96FC	
+0xEB8C	0x96FD	
+0xEB8D	0x96FF	
+0xEB8E	0x9702	
+0xEB8F	0x9703	
+0xEB90	0x9705	
+0xEB91	0x970A	
+0xEB92	0x970B	
+0xEB93	0x970C	
+0xEB94	0x9710	
+0xEB95	0x9711	
+0xEB96	0x9712	
+0xEB97	0x9714	
+0xEB98	0x9715	
+0xEB99	0x9717	
+0xEB9A	0x9718	
+0xEB9B	0x9719	
+0xEB9C	0x971A	
+0xEB9D	0x971B	
+0xEB9E	0x971D	
+0xEB9F	0x971F	
+0xEBA0	0x9720	
+0xEBA1	0x643F	
+0xEBA2	0x64D8	
+0xEBA3	0x8004	
+0xEBA4	0x6BEA	
+0xEBA5	0x6BF3	
+0xEBA6	0x6BFD	
+0xEBA7	0x6BF5	
+0xEBA8	0x6BF9	
+0xEBA9	0x6C05	
+0xEBAA	0x6C07	
+0xEBAB	0x6C06	
+0xEBAC	0x6C0D	
+0xEBAD	0x6C15	
+0xEBAE	0x6C18	
+0xEBAF	0x6C19	
+0xEBB0	0x6C1A	
+0xEBB1	0x6C21	
+0xEBB2	0x6C29	
+0xEBB3	0x6C24	
+0xEBB4	0x6C2A	
+0xEBB5	0x6C32	
+0xEBB6	0x6535	
+0xEBB7	0x6555	
+0xEBB8	0x656B	
+0xEBB9	0x724D	
+0xEBBA	0x7252	
+0xEBBB	0x7256	
+0xEBBC	0x7230	
+0xEBBD	0x8662	
+0xEBBE	0x5216	
+0xEBBF	0x809F	
+0xEBC0	0x809C	
+0xEBC1	0x8093	
+0xEBC2	0x80BC	
+0xEBC3	0x670A	
+0xEBC4	0x80BD	
+0xEBC5	0x80B1	
+0xEBC6	0x80AB	
+0xEBC7	0x80AD	
+0xEBC8	0x80B4	
+0xEBC9	0x80B7	
+0xEBCA	0x80E7	
+0xEBCB	0x80E8	
+0xEBCC	0x80E9	
+0xEBCD	0x80EA	
+0xEBCE	0x80DB	
+0xEBCF	0x80C2	
+0xEBD0	0x80C4	
+0xEBD1	0x80D9	
+0xEBD2	0x80CD	
+0xEBD3	0x80D7	
+0xEBD4	0x6710	
+0xEBD5	0x80DD	
+0xEBD6	0x80EB	
+0xEBD7	0x80F1	
+0xEBD8	0x80F4	
+0xEBD9	0x80ED	
+0xEBDA	0x810D	
+0xEBDB	0x810E	
+0xEBDC	0x80F2	
+0xEBDD	0x80FC	
+0xEBDE	0x6715	
+0xEBDF	0x8112	
+0xEBE0	0x8C5A	
+0xEBE1	0x8136	
+0xEBE2	0x811E	
+0xEBE3	0x812C	
+0xEBE4	0x8118	
+0xEBE5	0x8132	
+0xEBE6	0x8148	
+0xEBE7	0x814C	
+0xEBE8	0x8153	
+0xEBE9	0x8174	
+0xEBEA	0x8159	
+0xEBEB	0x815A	
+0xEBEC	0x8171	
+0xEBED	0x8160	
+0xEBEE	0x8169	
+0xEBEF	0x817C	
+0xEBF0	0x817D	
+0xEBF1	0x816D	
+0xEBF2	0x8167	
+0xEBF3	0x584D	
+0xEBF4	0x5AB5	
+0xEBF5	0x8188	
+0xEBF6	0x8182	
+0xEBF7	0x8191	
+0xEBF8	0x6ED5	
+0xEBF9	0x81A3	
+0xEBFA	0x81AA	
+0xEBFB	0x81CC	
+0xEBFC	0x6726	
+0xEBFD	0x81CA	
+0xEBFE	0x81BB	
+0xEC40	0x9721	
+0xEC41	0x9722	
+0xEC42	0x9723	
+0xEC43	0x9724	
+0xEC44	0x9725	
+0xEC45	0x9726	
+0xEC46	0x9727	
+0xEC47	0x9728	
+0xEC48	0x9729	
+0xEC49	0x972B	
+0xEC4A	0x972C	
+0xEC4B	0x972E	
+0xEC4C	0x972F	
+0xEC4D	0x9731	
+0xEC4E	0x9733	
+0xEC4F	0x9734	
+0xEC50	0x9735	
+0xEC51	0x9736	
+0xEC52	0x9737	
+0xEC53	0x973A	
+0xEC54	0x973B	
+0xEC55	0x973C	
+0xEC56	0x973D	
+0xEC57	0x973F	
+0xEC58	0x9740	
+0xEC59	0x9741	
+0xEC5A	0x9742	
+0xEC5B	0x9743	
+0xEC5C	0x9744	
+0xEC5D	0x9745	
+0xEC5E	0x9746	
+0xEC5F	0x9747	
+0xEC60	0x9748	
+0xEC61	0x9749	
+0xEC62	0x974A	
+0xEC63	0x974B	
+0xEC64	0x974C	
+0xEC65	0x974D	
+0xEC66	0x974E	
+0xEC67	0x974F	
+0xEC68	0x9750	
+0xEC69	0x9751	
+0xEC6A	0x9754	
+0xEC6B	0x9755	
+0xEC6C	0x9757	
+0xEC6D	0x9758	
+0xEC6E	0x975A	
+0xEC6F	0x975C	
+0xEC70	0x975D	
+0xEC71	0x975F	
+0xEC72	0x9763	
+0xEC73	0x9764	
+0xEC74	0x9766	
+0xEC75	0x9767	
+0xEC76	0x9768	
+0xEC77	0x976A	
+0xEC78	0x976B	
+0xEC79	0x976C	
+0xEC7A	0x976D	
+0xEC7B	0x976E	
+0xEC7C	0x976F	
+0xEC7D	0x9770	
+0xEC7E	0x9771	
+0xEC80	0x9772	
+0xEC81	0x9775	
+0xEC82	0x9777	
+0xEC83	0x9778	
+0xEC84	0x9779	
+0xEC85	0x977A	
+0xEC86	0x977B	
+0xEC87	0x977D	
+0xEC88	0x977E	
+0xEC89	0x977F	
+0xEC8A	0x9780	
+0xEC8B	0x9781	
+0xEC8C	0x9782	
+0xEC8D	0x9783	
+0xEC8E	0x9784	
+0xEC8F	0x9786	
+0xEC90	0x9787	
+0xEC91	0x9788	
+0xEC92	0x9789	
+0xEC93	0x978A	
+0xEC94	0x978C	
+0xEC95	0x978E	
+0xEC96	0x978F	
+0xEC97	0x9790	
+0xEC98	0x9793	
+0xEC99	0x9795	
+0xEC9A	0x9796	
+0xEC9B	0x9797	
+0xEC9C	0x9799	
+0xEC9D	0x979A	
+0xEC9E	0x979B	
+0xEC9F	0x979C	
+0xECA0	0x979D	
+0xECA1	0x81C1	
+0xECA2	0x81A6	
+0xECA3	0x6B24	
+0xECA4	0x6B37	
+0xECA5	0x6B39	
+0xECA6	0x6B43	
+0xECA7	0x6B46	
+0xECA8	0x6B59	
+0xECA9	0x98D1	
+0xECAA	0x98D2	
+0xECAB	0x98D3	
+0xECAC	0x98D5	
+0xECAD	0x98D9	
+0xECAE	0x98DA	
+0xECAF	0x6BB3	
+0xECB0	0x5F40	
+0xECB1	0x6BC2	
+0xECB2	0x89F3	
+0xECB3	0x6590	
+0xECB4	0x9F51	
+0xECB5	0x6593	
+0xECB6	0x65BC	
+0xECB7	0x65C6	
+0xECB8	0x65C4	
+0xECB9	0x65C3	
+0xECBA	0x65CC	
+0xECBB	0x65CE	
+0xECBC	0x65D2	
+0xECBD	0x65D6	
+0xECBE	0x7080	
+0xECBF	0x709C	
+0xECC0	0x7096	
+0xECC1	0x709D	
+0xECC2	0x70BB	
+0xECC3	0x70C0	
+0xECC4	0x70B7	
+0xECC5	0x70AB	
+0xECC6	0x70B1	
+0xECC7	0x70E8	
+0xECC8	0x70CA	
+0xECC9	0x7110	
+0xECCA	0x7113	
+0xECCB	0x7116	
+0xECCC	0x712F	
+0xECCD	0x7131	
+0xECCE	0x7173	
+0xECCF	0x715C	
+0xECD0	0x7168	
+0xECD1	0x7145	
+0xECD2	0x7172	
+0xECD3	0x714A	
+0xECD4	0x7178	
+0xECD5	0x717A	
+0xECD6	0x7198	
+0xECD7	0x71B3	
+0xECD8	0x71B5	
+0xECD9	0x71A8	
+0xECDA	0x71A0	
+0xECDB	0x71E0	
+0xECDC	0x71D4	
+0xECDD	0x71E7	
+0xECDE	0x71F9	
+0xECDF	0x721D	
+0xECE0	0x7228	
+0xECE1	0x706C	
+0xECE2	0x7118	
+0xECE3	0x7166	
+0xECE4	0x71B9	
+0xECE5	0x623E	
+0xECE6	0x623D	
+0xECE7	0x6243	
+0xECE8	0x6248	
+0xECE9	0x6249	
+0xECEA	0x793B	
+0xECEB	0x7940	
+0xECEC	0x7946	
+0xECED	0x7949	
+0xECEE	0x795B	
+0xECEF	0x795C	
+0xECF0	0x7953	
+0xECF1	0x795A	
+0xECF2	0x7962	
+0xECF3	0x7957	
+0xECF4	0x7960	
+0xECF5	0x796F	
+0xECF6	0x7967	
+0xECF7	0x797A	
+0xECF8	0x7985	
+0xECF9	0x798A	
+0xECFA	0x799A	
+0xECFB	0x79A7	
+0xECFC	0x79B3	
+0xECFD	0x5FD1	
+0xECFE	0x5FD0	
+0xED40	0x979E	
+0xED41	0x979F	
+0xED42	0x97A1	
+0xED43	0x97A2	
+0xED44	0x97A4	
+0xED45	0x97A5	
+0xED46	0x97A6	
+0xED47	0x97A7	
+0xED48	0x97A8	
+0xED49	0x97A9	
+0xED4A	0x97AA	
+0xED4B	0x97AC	
+0xED4C	0x97AE	
+0xED4D	0x97B0	
+0xED4E	0x97B1	
+0xED4F	0x97B3	
+0xED50	0x97B5	
+0xED51	0x97B6	
+0xED52	0x97B7	
+0xED53	0x97B8	
+0xED54	0x97B9	
+0xED55	0x97BA	
+0xED56	0x97BB	
+0xED57	0x97BC	
+0xED58	0x97BD	
+0xED59	0x97BE	
+0xED5A	0x97BF	
+0xED5B	0x97C0	
+0xED5C	0x97C1	
+0xED5D	0x97C2	
+0xED5E	0x97C3	
+0xED5F	0x97C4	
+0xED60	0x97C5	
+0xED61	0x97C6	
+0xED62	0x97C7	
+0xED63	0x97C8	
+0xED64	0x97C9	
+0xED65	0x97CA	
+0xED66	0x97CB	
+0xED67	0x97CC	
+0xED68	0x97CD	
+0xED69	0x97CE	
+0xED6A	0x97CF	
+0xED6B	0x97D0	
+0xED6C	0x97D1	
+0xED6D	0x97D2	
+0xED6E	0x97D3	
+0xED6F	0x97D4	
+0xED70	0x97D5	
+0xED71	0x97D6	
+0xED72	0x97D7	
+0xED73	0x97D8	
+0xED74	0x97D9	
+0xED75	0x97DA	
+0xED76	0x97DB	
+0xED77	0x97DC	
+0xED78	0x97DD	
+0xED79	0x97DE	
+0xED7A	0x97DF	
+0xED7B	0x97E0	
+0xED7C	0x97E1	
+0xED7D	0x97E2	
+0xED7E	0x97E3	
+0xED80	0x97E4	
+0xED81	0x97E5	
+0xED82	0x97E8	
+0xED83	0x97EE	
+0xED84	0x97EF	
+0xED85	0x97F0	
+0xED86	0x97F1	
+0xED87	0x97F2	
+0xED88	0x97F4	
+0xED89	0x97F7	
+0xED8A	0x97F8	
+0xED8B	0x97F9	
+0xED8C	0x97FA	
+0xED8D	0x97FB	
+0xED8E	0x97FC	
+0xED8F	0x97FD	
+0xED90	0x97FE	
+0xED91	0x97FF	
+0xED92	0x9800	
+0xED93	0x9801	
+0xED94	0x9802	
+0xED95	0x9803	
+0xED96	0x9804	
+0xED97	0x9805	
+0xED98	0x9806	
+0xED99	0x9807	
+0xED9A	0x9808	
+0xED9B	0x9809	
+0xED9C	0x980A	
+0xED9D	0x980B	
+0xED9E	0x980C	
+0xED9F	0x980D	
+0xEDA0	0x980E	
+0xEDA1	0x603C	
+0xEDA2	0x605D	
+0xEDA3	0x605A	
+0xEDA4	0x6067	
+0xEDA5	0x6041	
+0xEDA6	0x6059	
+0xEDA7	0x6063	
+0xEDA8	0x60AB	
+0xEDA9	0x6106	
+0xEDAA	0x610D	
+0xEDAB	0x615D	
+0xEDAC	0x61A9	
+0xEDAD	0x619D	
+0xEDAE	0x61CB	
+0xEDAF	0x61D1	
+0xEDB0	0x6206	
+0xEDB1	0x8080	
+0xEDB2	0x807F	
+0xEDB3	0x6C93	
+0xEDB4	0x6CF6	
+0xEDB5	0x6DFC	
+0xEDB6	0x77F6	
+0xEDB7	0x77F8	
+0xEDB8	0x7800	
+0xEDB9	0x7809	
+0xEDBA	0x7817	
+0xEDBB	0x7818	
+0xEDBC	0x7811	
+0xEDBD	0x65AB	
+0xEDBE	0x782D	
+0xEDBF	0x781C	
+0xEDC0	0x781D	
+0xEDC1	0x7839	
+0xEDC2	0x783A	
+0xEDC3	0x783B	
+0xEDC4	0x781F	
+0xEDC5	0x783C	
+0xEDC6	0x7825	
+0xEDC7	0x782C	
+0xEDC8	0x7823	
+0xEDC9	0x7829	
+0xEDCA	0x784E	
+0xEDCB	0x786D	
+0xEDCC	0x7856	
+0xEDCD	0x7857	
+0xEDCE	0x7826	
+0xEDCF	0x7850	
+0xEDD0	0x7847	
+0xEDD1	0x784C	
+0xEDD2	0x786A	
+0xEDD3	0x789B	
+0xEDD4	0x7893	
+0xEDD5	0x789A	
+0xEDD6	0x7887	
+0xEDD7	0x789C	
+0xEDD8	0x78A1	
+0xEDD9	0x78A3	
+0xEDDA	0x78B2	
+0xEDDB	0x78B9	
+0xEDDC	0x78A5	
+0xEDDD	0x78D4	
+0xEDDE	0x78D9	
+0xEDDF	0x78C9	
+0xEDE0	0x78EC	
+0xEDE1	0x78F2	
+0xEDE2	0x7905	
+0xEDE3	0x78F4	
+0xEDE4	0x7913	
+0xEDE5	0x7924	
+0xEDE6	0x791E	
+0xEDE7	0x7934	
+0xEDE8	0x9F9B	
+0xEDE9	0x9EF9	
+0xEDEA	0x9EFB	
+0xEDEB	0x9EFC	
+0xEDEC	0x76F1	
+0xEDED	0x7704	
+0xEDEE	0x770D	
+0xEDEF	0x76F9	
+0xEDF0	0x7707	
+0xEDF1	0x7708	
+0xEDF2	0x771A	
+0xEDF3	0x7722	
+0xEDF4	0x7719	
+0xEDF5	0x772D	
+0xEDF6	0x7726	
+0xEDF7	0x7735	
+0xEDF8	0x7738	
+0xEDF9	0x7750	
+0xEDFA	0x7751	
+0xEDFB	0x7747	
+0xEDFC	0x7743	
+0xEDFD	0x775A	
+0xEDFE	0x7768	
+0xEE40	0x980F	
+0xEE41	0x9810	
+0xEE42	0x9811	
+0xEE43	0x9812	
+0xEE44	0x9813	
+0xEE45	0x9814	
+0xEE46	0x9815	
+0xEE47	0x9816	
+0xEE48	0x9817	
+0xEE49	0x9818	
+0xEE4A	0x9819	
+0xEE4B	0x981A	
+0xEE4C	0x981B	
+0xEE4D	0x981C	
+0xEE4E	0x981D	
+0xEE4F	0x981E	
+0xEE50	0x981F	
+0xEE51	0x9820	
+0xEE52	0x9821	
+0xEE53	0x9822	
+0xEE54	0x9823	
+0xEE55	0x9824	
+0xEE56	0x9825	
+0xEE57	0x9826	
+0xEE58	0x9827	
+0xEE59	0x9828	
+0xEE5A	0x9829	
+0xEE5B	0x982A	
+0xEE5C	0x982B	
+0xEE5D	0x982C	
+0xEE5E	0x982D	
+0xEE5F	0x982E	
+0xEE60	0x982F	
+0xEE61	0x9830	
+0xEE62	0x9831	
+0xEE63	0x9832	
+0xEE64	0x9833	
+0xEE65	0x9834	
+0xEE66	0x9835	
+0xEE67	0x9836	
+0xEE68	0x9837	
+0xEE69	0x9838	
+0xEE6A	0x9839	
+0xEE6B	0x983A	
+0xEE6C	0x983B	
+0xEE6D	0x983C	
+0xEE6E	0x983D	
+0xEE6F	0x983E	
+0xEE70	0x983F	
+0xEE71	0x9840	
+0xEE72	0x9841	
+0xEE73	0x9842	
+0xEE74	0x9843	
+0xEE75	0x9844	
+0xEE76	0x9845	
+0xEE77	0x9846	
+0xEE78	0x9847	
+0xEE79	0x9848	
+0xEE7A	0x9849	
+0xEE7B	0x984A	
+0xEE7C	0x984B	
+0xEE7D	0x984C	
+0xEE7E	0x984D	
+0xEE80	0x984E	
+0xEE81	0x984F	
+0xEE82	0x9850	
+0xEE83	0x9851	
+0xEE84	0x9852	
+0xEE85	0x9853	
+0xEE86	0x9854	
+0xEE87	0x9855	
+0xEE88	0x9856	
+0xEE89	0x9857	
+0xEE8A	0x9858	
+0xEE8B	0x9859	
+0xEE8C	0x985A	
+0xEE8D	0x985B	
+0xEE8E	0x985C	
+0xEE8F	0x985D	
+0xEE90	0x985E	
+0xEE91	0x985F	
+0xEE92	0x9860	
+0xEE93	0x9861	
+0xEE94	0x9862	
+0xEE95	0x9863	
+0xEE96	0x9864	
+0xEE97	0x9865	
+0xEE98	0x9866	
+0xEE99	0x9867	
+0xEE9A	0x9868	
+0xEE9B	0x9869	
+0xEE9C	0x986A	
+0xEE9D	0x986B	
+0xEE9E	0x986C	
+0xEE9F	0x986D	
+0xEEA0	0x986E	
+0xEEA1	0x7762	
+0xEEA2	0x7765	
+0xEEA3	0x777F	
+0xEEA4	0x778D	
+0xEEA5	0x777D	
+0xEEA6	0x7780	
+0xEEA7	0x778C	
+0xEEA8	0x7791	
+0xEEA9	0x779F	
+0xEEAA	0x77A0	
+0xEEAB	0x77B0	
+0xEEAC	0x77B5	
+0xEEAD	0x77BD	
+0xEEAE	0x753A	
+0xEEAF	0x7540	
+0xEEB0	0x754E	
+0xEEB1	0x754B	
+0xEEB2	0x7548	
+0xEEB3	0x755B	
+0xEEB4	0x7572	
+0xEEB5	0x7579	
+0xEEB6	0x7583	
+0xEEB7	0x7F58	
+0xEEB8	0x7F61	
+0xEEB9	0x7F5F	
+0xEEBA	0x8A48	
+0xEEBB	0x7F68	
+0xEEBC	0x7F74	
+0xEEBD	0x7F71	
+0xEEBE	0x7F79	
+0xEEBF	0x7F81	
+0xEEC0	0x7F7E	
+0xEEC1	0x76CD	
+0xEEC2	0x76E5	
+0xEEC3	0x8832	
+0xEEC4	0x9485	
+0xEEC5	0x9486	
+0xEEC6	0x9487	
+0xEEC7	0x948B	
+0xEEC8	0x948A	
+0xEEC9	0x948C	
+0xEECA	0x948D	
+0xEECB	0x948F	
+0xEECC	0x9490	
+0xEECD	0x9494	
+0xEECE	0x9497	
+0xEECF	0x9495	
+0xEED0	0x949A	
+0xEED1	0x949B	
+0xEED2	0x949C	
+0xEED3	0x94A3	
+0xEED4	0x94A4	
+0xEED5	0x94AB	
+0xEED6	0x94AA	
+0xEED7	0x94AD	
+0xEED8	0x94AC	
+0xEED9	0x94AF	
+0xEEDA	0x94B0	
+0xEEDB	0x94B2	
+0xEEDC	0x94B4	
+0xEEDD	0x94B6	
+0xEEDE	0x94B7	
+0xEEDF	0x94B8	
+0xEEE0	0x94B9	
+0xEEE1	0x94BA	
+0xEEE2	0x94BC	
+0xEEE3	0x94BD	
+0xEEE4	0x94BF	
+0xEEE5	0x94C4	
+0xEEE6	0x94C8	
+0xEEE7	0x94C9	
+0xEEE8	0x94CA	
+0xEEE9	0x94CB	
+0xEEEA	0x94CC	
+0xEEEB	0x94CD	
+0xEEEC	0x94CE	
+0xEEED	0x94D0	
+0xEEEE	0x94D1	
+0xEEEF	0x94D2	
+0xEEF0	0x94D5	
+0xEEF1	0x94D6	
+0xEEF2	0x94D7	
+0xEEF3	0x94D9	
+0xEEF4	0x94D8	
+0xEEF5	0x94DB	
+0xEEF6	0x94DE	
+0xEEF7	0x94DF	
+0xEEF8	0x94E0	
+0xEEF9	0x94E2	
+0xEEFA	0x94E4	
+0xEEFB	0x94E5	
+0xEEFC	0x94E7	
+0xEEFD	0x94E8	
+0xEEFE	0x94EA	
+0xEF40	0x986F	
+0xEF41	0x9870	
+0xEF42	0x9871	
+0xEF43	0x9872	
+0xEF44	0x9873	
+0xEF45	0x9874	
+0xEF46	0x988B	
+0xEF47	0x988E	
+0xEF48	0x9892	
+0xEF49	0x9895	
+0xEF4A	0x9899	
+0xEF4B	0x98A3	
+0xEF4C	0x98A8	
+0xEF4D	0x98A9	
+0xEF4E	0x98AA	
+0xEF4F	0x98AB	
+0xEF50	0x98AC	
+0xEF51	0x98AD	
+0xEF52	0x98AE	
+0xEF53	0x98AF	
+0xEF54	0x98B0	
+0xEF55	0x98B1	
+0xEF56	0x98B2	
+0xEF57	0x98B3	
+0xEF58	0x98B4	
+0xEF59	0x98B5	
+0xEF5A	0x98B6	
+0xEF5B	0x98B7	
+0xEF5C	0x98B8	
+0xEF5D	0x98B9	
+0xEF5E	0x98BA	
+0xEF5F	0x98BB	
+0xEF60	0x98BC	
+0xEF61	0x98BD	
+0xEF62	0x98BE	
+0xEF63	0x98BF	
+0xEF64	0x98C0	
+0xEF65	0x98C1	
+0xEF66	0x98C2	
+0xEF67	0x98C3	
+0xEF68	0x98C4	
+0xEF69	0x98C5	
+0xEF6A	0x98C6	
+0xEF6B	0x98C7	
+0xEF6C	0x98C8	
+0xEF6D	0x98C9	
+0xEF6E	0x98CA	
+0xEF6F	0x98CB	
+0xEF70	0x98CC	
+0xEF71	0x98CD	
+0xEF72	0x98CF	
+0xEF73	0x98D0	
+0xEF74	0x98D4	
+0xEF75	0x98D6	
+0xEF76	0x98D7	
+0xEF77	0x98DB	
+0xEF78	0x98DC	
+0xEF79	0x98DD	
+0xEF7A	0x98E0	
+0xEF7B	0x98E1	
+0xEF7C	0x98E2	
+0xEF7D	0x98E3	
+0xEF7E	0x98E4	
+0xEF80	0x98E5	
+0xEF81	0x98E6	
+0xEF82	0x98E9	
+0xEF83	0x98EA	
+0xEF84	0x98EB	
+0xEF85	0x98EC	
+0xEF86	0x98ED	
+0xEF87	0x98EE	
+0xEF88	0x98EF	
+0xEF89	0x98F0	
+0xEF8A	0x98F1	
+0xEF8B	0x98F2	
+0xEF8C	0x98F3	
+0xEF8D	0x98F4	
+0xEF8E	0x98F5	
+0xEF8F	0x98F6	
+0xEF90	0x98F7	
+0xEF91	0x98F8	
+0xEF92	0x98F9	
+0xEF93	0x98FA	
+0xEF94	0x98FB	
+0xEF95	0x98FC	
+0xEF96	0x98FD	
+0xEF97	0x98FE	
+0xEF98	0x98FF	
+0xEF99	0x9900	
+0xEF9A	0x9901	
+0xEF9B	0x9902	
+0xEF9C	0x9903	
+0xEF9D	0x9904	
+0xEF9E	0x9905	
+0xEF9F	0x9906	
+0xEFA0	0x9907	
+0xEFA1	0x94E9	
+0xEFA2	0x94EB	
+0xEFA3	0x94EE	
+0xEFA4	0x94EF	
+0xEFA5	0x94F3	
+0xEFA6	0x94F4	
+0xEFA7	0x94F5	
+0xEFA8	0x94F7	
+0xEFA9	0x94F9	
+0xEFAA	0x94FC	
+0xEFAB	0x94FD	
+0xEFAC	0x94FF	
+0xEFAD	0x9503	
+0xEFAE	0x9502	
+0xEFAF	0x9506	
+0xEFB0	0x9507	
+0xEFB1	0x9509	
+0xEFB2	0x950A	
+0xEFB3	0x950D	
+0xEFB4	0x950E	
+0xEFB5	0x950F	
+0xEFB6	0x9512	
+0xEFB7	0x9513	
+0xEFB8	0x9514	
+0xEFB9	0x9515	
+0xEFBA	0x9516	
+0xEFBB	0x9518	
+0xEFBC	0x951B	
+0xEFBD	0x951D	
+0xEFBE	0x951E	
+0xEFBF	0x951F	
+0xEFC0	0x9522	
+0xEFC1	0x952A	
+0xEFC2	0x952B	
+0xEFC3	0x9529	
+0xEFC4	0x952C	
+0xEFC5	0x9531	
+0xEFC6	0x9532	
+0xEFC7	0x9534	
+0xEFC8	0x9536	
+0xEFC9	0x9537	
+0xEFCA	0x9538	
+0xEFCB	0x953C	
+0xEFCC	0x953E	
+0xEFCD	0x953F	
+0xEFCE	0x9542	
+0xEFCF	0x9535	
+0xEFD0	0x9544	
+0xEFD1	0x9545	
+0xEFD2	0x9546	
+0xEFD3	0x9549	
+0xEFD4	0x954C	
+0xEFD5	0x954E	
+0xEFD6	0x954F	
+0xEFD7	0x9552	
+0xEFD8	0x9553	
+0xEFD9	0x9554	
+0xEFDA	0x9556	
+0xEFDB	0x9557	
+0xEFDC	0x9558	
+0xEFDD	0x9559	
+0xEFDE	0x955B	
+0xEFDF	0x955E	
+0xEFE0	0x955F	
+0xEFE1	0x955D	
+0xEFE2	0x9561	
+0xEFE3	0x9562	
+0xEFE4	0x9564	
+0xEFE5	0x9565	
+0xEFE6	0x9566	
+0xEFE7	0x9567	
+0xEFE8	0x9568	
+0xEFE9	0x9569	
+0xEFEA	0x956A	
+0xEFEB	0x956B	
+0xEFEC	0x956C	
+0xEFED	0x956F	
+0xEFEE	0x9571	
+0xEFEF	0x9572	
+0xEFF0	0x9573	
+0xEFF1	0x953A	
+0xEFF2	0x77E7	
+0xEFF3	0x77EC	
+0xEFF4	0x96C9	
+0xEFF5	0x79D5	
+0xEFF6	0x79ED	
+0xEFF7	0x79E3	
+0xEFF8	0x79EB	
+0xEFF9	0x7A06	
+0xEFFA	0x5D47	
+0xEFFB	0x7A03	
+0xEFFC	0x7A02	
+0xEFFD	0x7A1E	
+0xEFFE	0x7A14	
+0xF040	0x9908	
+0xF041	0x9909	
+0xF042	0x990A	
+0xF043	0x990B	
+0xF044	0x990C	
+0xF045	0x990E	
+0xF046	0x990F	
+0xF047	0x9911	
+0xF048	0x9912	
+0xF049	0x9913	
+0xF04A	0x9914	
+0xF04B	0x9915	
+0xF04C	0x9916	
+0xF04D	0x9917	
+0xF04E	0x9918	
+0xF04F	0x9919	
+0xF050	0x991A	
+0xF051	0x991B	
+0xF052	0x991C	
+0xF053	0x991D	
+0xF054	0x991E	
+0xF055	0x991F	
+0xF056	0x9920	
+0xF057	0x9921	
+0xF058	0x9922	
+0xF059	0x9923	
+0xF05A	0x9924	
+0xF05B	0x9925	
+0xF05C	0x9926	
+0xF05D	0x9927	
+0xF05E	0x9928	
+0xF05F	0x9929	
+0xF060	0x992A	
+0xF061	0x992B	
+0xF062	0x992C	
+0xF063	0x992D	
+0xF064	0x992F	
+0xF065	0x9930	
+0xF066	0x9931	
+0xF067	0x9932	
+0xF068	0x9933	
+0xF069	0x9934	
+0xF06A	0x9935	
+0xF06B	0x9936	
+0xF06C	0x9937	
+0xF06D	0x9938	
+0xF06E	0x9939	
+0xF06F	0x993A	
+0xF070	0x993B	
+0xF071	0x993C	
+0xF072	0x993D	
+0xF073	0x993E	
+0xF074	0x993F	
+0xF075	0x9940	
+0xF076	0x9941	
+0xF077	0x9942	
+0xF078	0x9943	
+0xF079	0x9944	
+0xF07A	0x9945	
+0xF07B	0x9946	
+0xF07C	0x9947	
+0xF07D	0x9948	
+0xF07E	0x9949	
+0xF080	0x994A	
+0xF081	0x994B	
+0xF082	0x994C	
+0xF083	0x994D	
+0xF084	0x994E	
+0xF085	0x994F	
+0xF086	0x9950	
+0xF087	0x9951	
+0xF088	0x9952	
+0xF089	0x9953	
+0xF08A	0x9956	
+0xF08B	0x9957	
+0xF08C	0x9958	
+0xF08D	0x9959	
+0xF08E	0x995A	
+0xF08F	0x995B	
+0xF090	0x995C	
+0xF091	0x995D	
+0xF092	0x995E	
+0xF093	0x995F	
+0xF094	0x9960	
+0xF095	0x9961	
+0xF096	0x9962	
+0xF097	0x9964	
+0xF098	0x9966	
+0xF099	0x9973	
+0xF09A	0x9978	
+0xF09B	0x9979	
+0xF09C	0x997B	
+0xF09D	0x997E	
+0xF09E	0x9982	
+0xF09F	0x9983	
+0xF0A0	0x9989	
+0xF0A1	0x7A39	
+0xF0A2	0x7A37	
+0xF0A3	0x7A51	
+0xF0A4	0x9ECF	
+0xF0A5	0x99A5	
+0xF0A6	0x7A70	
+0xF0A7	0x7688	
+0xF0A8	0x768E	
+0xF0A9	0x7693	
+0xF0AA	0x7699	
+0xF0AB	0x76A4	
+0xF0AC	0x74DE	
+0xF0AD	0x74E0	
+0xF0AE	0x752C	
+0xF0AF	0x9E20	
+0xF0B0	0x9E22	
+0xF0B1	0x9E28	
+0xF0B2	0x9E29	
+0xF0B3	0x9E2A	
+0xF0B4	0x9E2B	
+0xF0B5	0x9E2C	
+0xF0B6	0x9E32	
+0xF0B7	0x9E31	
+0xF0B8	0x9E36	
+0xF0B9	0x9E38	
+0xF0BA	0x9E37	
+0xF0BB	0x9E39	
+0xF0BC	0x9E3A	
+0xF0BD	0x9E3E	
+0xF0BE	0x9E41	
+0xF0BF	0x9E42	
+0xF0C0	0x9E44	
+0xF0C1	0x9E46	
+0xF0C2	0x9E47	
+0xF0C3	0x9E48	
+0xF0C4	0x9E49	
+0xF0C5	0x9E4B	
+0xF0C6	0x9E4C	
+0xF0C7	0x9E4E	
+0xF0C8	0x9E51	
+0xF0C9	0x9E55	
+0xF0CA	0x9E57	
+0xF0CB	0x9E5A	
+0xF0CC	0x9E5B	
+0xF0CD	0x9E5C	
+0xF0CE	0x9E5E	
+0xF0CF	0x9E63	
+0xF0D0	0x9E66	
+0xF0D1	0x9E67	
+0xF0D2	0x9E68	
+0xF0D3	0x9E69	
+0xF0D4	0x9E6A	
+0xF0D5	0x9E6B	
+0xF0D6	0x9E6C	
+0xF0D7	0x9E71	
+0xF0D8	0x9E6D	
+0xF0D9	0x9E73	
+0xF0DA	0x7592	
+0xF0DB	0x7594	
+0xF0DC	0x7596	
+0xF0DD	0x75A0	
+0xF0DE	0x759D	
+0xF0DF	0x75AC	
+0xF0E0	0x75A3	
+0xF0E1	0x75B3	
+0xF0E2	0x75B4	
+0xF0E3	0x75B8	
+0xF0E4	0x75C4	
+0xF0E5	0x75B1	
+0xF0E6	0x75B0	
+0xF0E7	0x75C3	
+0xF0E8	0x75C2	
+0xF0E9	0x75D6	
+0xF0EA	0x75CD	
+0xF0EB	0x75E3	
+0xF0EC	0x75E8	
+0xF0ED	0x75E6	
+0xF0EE	0x75E4	
+0xF0EF	0x75EB	
+0xF0F0	0x75E7	
+0xF0F1	0x7603	
+0xF0F2	0x75F1	
+0xF0F3	0x75FC	
+0xF0F4	0x75FF	
+0xF0F5	0x7610	
+0xF0F6	0x7600	
+0xF0F7	0x7605	
+0xF0F8	0x760C	
+0xF0F9	0x7617	
+0xF0FA	0x760A	
+0xF0FB	0x7625	
+0xF0FC	0x7618	
+0xF0FD	0x7615	
+0xF0FE	0x7619	
+0xF140	0x998C	
+0xF141	0x998E	
+0xF142	0x999A	
+0xF143	0x999B	
+0xF144	0x999C	
+0xF145	0x999D	
+0xF146	0x999E	
+0xF147	0x999F	
+0xF148	0x99A0	
+0xF149	0x99A1	
+0xF14A	0x99A2	
+0xF14B	0x99A3	
+0xF14C	0x99A4	
+0xF14D	0x99A6	
+0xF14E	0x99A7	
+0xF14F	0x99A9	
+0xF150	0x99AA	
+0xF151	0x99AB	
+0xF152	0x99AC	
+0xF153	0x99AD	
+0xF154	0x99AE	
+0xF155	0x99AF	
+0xF156	0x99B0	
+0xF157	0x99B1	
+0xF158	0x99B2	
+0xF159	0x99B3	
+0xF15A	0x99B4	
+0xF15B	0x99B5	
+0xF15C	0x99B6	
+0xF15D	0x99B7	
+0xF15E	0x99B8	
+0xF15F	0x99B9	
+0xF160	0x99BA	
+0xF161	0x99BB	
+0xF162	0x99BC	
+0xF163	0x99BD	
+0xF164	0x99BE	
+0xF165	0x99BF	
+0xF166	0x99C0	
+0xF167	0x99C1	
+0xF168	0x99C2	
+0xF169	0x99C3	
+0xF16A	0x99C4	
+0xF16B	0x99C5	
+0xF16C	0x99C6	
+0xF16D	0x99C7	
+0xF16E	0x99C8	
+0xF16F	0x99C9	
+0xF170	0x99CA	
+0xF171	0x99CB	
+0xF172	0x99CC	
+0xF173	0x99CD	
+0xF174	0x99CE	
+0xF175	0x99CF	
+0xF176	0x99D0	
+0xF177	0x99D1	
+0xF178	0x99D2	
+0xF179	0x99D3	
+0xF17A	0x99D4	
+0xF17B	0x99D5	
+0xF17C	0x99D6	
+0xF17D	0x99D7	
+0xF17E	0x99D8	
+0xF180	0x99D9	
+0xF181	0x99DA	
+0xF182	0x99DB	
+0xF183	0x99DC	
+0xF184	0x99DD	
+0xF185	0x99DE	
+0xF186	0x99DF	
+0xF187	0x99E0	
+0xF188	0x99E1	
+0xF189	0x99E2	
+0xF18A	0x99E3	
+0xF18B	0x99E4	
+0xF18C	0x99E5	
+0xF18D	0x99E6	
+0xF18E	0x99E7	
+0xF18F	0x99E8	
+0xF190	0x99E9	
+0xF191	0x99EA	
+0xF192	0x99EB	
+0xF193	0x99EC	
+0xF194	0x99ED	
+0xF195	0x99EE	
+0xF196	0x99EF	
+0xF197	0x99F0	
+0xF198	0x99F1	
+0xF199	0x99F2	
+0xF19A	0x99F3	
+0xF19B	0x99F4	
+0xF19C	0x99F5	
+0xF19D	0x99F6	
+0xF19E	0x99F7	
+0xF19F	0x99F8	
+0xF1A0	0x99F9	
+0xF1A1	0x761B	
+0xF1A2	0x763C	
+0xF1A3	0x7622	
+0xF1A4	0x7620	
+0xF1A5	0x7640	
+0xF1A6	0x762D	
+0xF1A7	0x7630	
+0xF1A8	0x763F	
+0xF1A9	0x7635	
+0xF1AA	0x7643	
+0xF1AB	0x763E	
+0xF1AC	0x7633	
+0xF1AD	0x764D	
+0xF1AE	0x765E	
+0xF1AF	0x7654	
+0xF1B0	0x765C	
+0xF1B1	0x7656	
+0xF1B2	0x766B	
+0xF1B3	0x766F	
+0xF1B4	0x7FCA	
+0xF1B5	0x7AE6	
+0xF1B6	0x7A78	
+0xF1B7	0x7A79	
+0xF1B8	0x7A80	
+0xF1B9	0x7A86	
+0xF1BA	0x7A88	
+0xF1BB	0x7A95	
+0xF1BC	0x7AA6	
+0xF1BD	0x7AA0	
+0xF1BE	0x7AAC	
+0xF1BF	0x7AA8	
+0xF1C0	0x7AAD	
+0xF1C1	0x7AB3	
+0xF1C2	0x8864	
+0xF1C3	0x8869	
+0xF1C4	0x8872	
+0xF1C5	0x887D	
+0xF1C6	0x887F	
+0xF1C7	0x8882	
+0xF1C8	0x88A2	
+0xF1C9	0x88C6	
+0xF1CA	0x88B7	
+0xF1CB	0x88BC	
+0xF1CC	0x88C9	
+0xF1CD	0x88E2	
+0xF1CE	0x88CE	
+0xF1CF	0x88E3	
+0xF1D0	0x88E5	
+0xF1D1	0x88F1	
+0xF1D2	0x891A	
+0xF1D3	0x88FC	
+0xF1D4	0x88E8	
+0xF1D5	0x88FE	
+0xF1D6	0x88F0	
+0xF1D7	0x8921	
+0xF1D8	0x8919	
+0xF1D9	0x8913	
+0xF1DA	0x891B	
+0xF1DB	0x890A	
+0xF1DC	0x8934	
+0xF1DD	0x892B	
+0xF1DE	0x8936	
+0xF1DF	0x8941	
+0xF1E0	0x8966	
+0xF1E1	0x897B	
+0xF1E2	0x758B	
+0xF1E3	0x80E5	
+0xF1E4	0x76B2	
+0xF1E5	0x76B4	
+0xF1E6	0x77DC	
+0xF1E7	0x8012	
+0xF1E8	0x8014	
+0xF1E9	0x8016	
+0xF1EA	0x801C	
+0xF1EB	0x8020	
+0xF1EC	0x8022	
+0xF1ED	0x8025	
+0xF1EE	0x8026	
+0xF1EF	0x8027	
+0xF1F0	0x8029	
+0xF1F1	0x8028	
+0xF1F2	0x8031	
+0xF1F3	0x800B	
+0xF1F4	0x8035	
+0xF1F5	0x8043	
+0xF1F6	0x8046	
+0xF1F7	0x804D	
+0xF1F8	0x8052	
+0xF1F9	0x8069	
+0xF1FA	0x8071	
+0xF1FB	0x8983	
+0xF1FC	0x9878	
+0xF1FD	0x9880	
+0xF1FE	0x9883	
+0xF240	0x99FA	
+0xF241	0x99FB	
+0xF242	0x99FC	
+0xF243	0x99FD	
+0xF244	0x99FE	
+0xF245	0x99FF	
+0xF246	0x9A00	
+0xF247	0x9A01	
+0xF248	0x9A02	
+0xF249	0x9A03	
+0xF24A	0x9A04	
+0xF24B	0x9A05	
+0xF24C	0x9A06	
+0xF24D	0x9A07	
+0xF24E	0x9A08	
+0xF24F	0x9A09	
+0xF250	0x9A0A	
+0xF251	0x9A0B	
+0xF252	0x9A0C	
+0xF253	0x9A0D	
+0xF254	0x9A0E	
+0xF255	0x9A0F	
+0xF256	0x9A10	
+0xF257	0x9A11	
+0xF258	0x9A12	
+0xF259	0x9A13	
+0xF25A	0x9A14	
+0xF25B	0x9A15	
+0xF25C	0x9A16	
+0xF25D	0x9A17	
+0xF25E	0x9A18	
+0xF25F	0x9A19	
+0xF260	0x9A1A	
+0xF261	0x9A1B	
+0xF262	0x9A1C	
+0xF263	0x9A1D	
+0xF264	0x9A1E	
+0xF265	0x9A1F	
+0xF266	0x9A20	
+0xF267	0x9A21	
+0xF268	0x9A22	
+0xF269	0x9A23	
+0xF26A	0x9A24	
+0xF26B	0x9A25	
+0xF26C	0x9A26	
+0xF26D	0x9A27	
+0xF26E	0x9A28	
+0xF26F	0x9A29	
+0xF270	0x9A2A	
+0xF271	0x9A2B	
+0xF272	0x9A2C	
+0xF273	0x9A2D	
+0xF274	0x9A2E	
+0xF275	0x9A2F	
+0xF276	0x9A30	
+0xF277	0x9A31	
+0xF278	0x9A32	
+0xF279	0x9A33	
+0xF27A	0x9A34	
+0xF27B	0x9A35	
+0xF27C	0x9A36	
+0xF27D	0x9A37	
+0xF27E	0x9A38	
+0xF280	0x9A39	
+0xF281	0x9A3A	
+0xF282	0x9A3B	
+0xF283	0x9A3C	
+0xF284	0x9A3D	
+0xF285	0x9A3E	
+0xF286	0x9A3F	
+0xF287	0x9A40	
+0xF288	0x9A41	
+0xF289	0x9A42	
+0xF28A	0x9A43	
+0xF28B	0x9A44	
+0xF28C	0x9A45	
+0xF28D	0x9A46	
+0xF28E	0x9A47	
+0xF28F	0x9A48	
+0xF290	0x9A49	
+0xF291	0x9A4A	
+0xF292	0x9A4B	
+0xF293	0x9A4C	
+0xF294	0x9A4D	
+0xF295	0x9A4E	
+0xF296	0x9A4F	
+0xF297	0x9A50	
+0xF298	0x9A51	
+0xF299	0x9A52	
+0xF29A	0x9A53	
+0xF29B	0x9A54	
+0xF29C	0x9A55	
+0xF29D	0x9A56	
+0xF29E	0x9A57	
+0xF29F	0x9A58	
+0xF2A0	0x9A59	
+0xF2A1	0x9889	
+0xF2A2	0x988C	
+0xF2A3	0x988D	
+0xF2A4	0x988F	
+0xF2A5	0x9894	
+0xF2A6	0x989A	
+0xF2A7	0x989B	
+0xF2A8	0x989E	
+0xF2A9	0x989F	
+0xF2AA	0x98A1	
+0xF2AB	0x98A2	
+0xF2AC	0x98A5	
+0xF2AD	0x98A6	
+0xF2AE	0x864D	
+0xF2AF	0x8654	
+0xF2B0	0x866C	
+0xF2B1	0x866E	
+0xF2B2	0x867F	
+0xF2B3	0x867A	
+0xF2B4	0x867C	
+0xF2B5	0x867B	
+0xF2B6	0x86A8	
+0xF2B7	0x868D	
+0xF2B8	0x868B	
+0xF2B9	0x86AC	
+0xF2BA	0x869D	
+0xF2BB	0x86A7	
+0xF2BC	0x86A3	
+0xF2BD	0x86AA	
+0xF2BE	0x8693	
+0xF2BF	0x86A9	
+0xF2C0	0x86B6	
+0xF2C1	0x86C4	
+0xF2C2	0x86B5	
+0xF2C3	0x86CE	
+0xF2C4	0x86B0	
+0xF2C5	0x86BA	
+0xF2C6	0x86B1	
+0xF2C7	0x86AF	
+0xF2C8	0x86C9	
+0xF2C9	0x86CF	
+0xF2CA	0x86B4	
+0xF2CB	0x86E9	
+0xF2CC	0x86F1	
+0xF2CD	0x86F2	
+0xF2CE	0x86ED	
+0xF2CF	0x86F3	
+0xF2D0	0x86D0	
+0xF2D1	0x8713	
+0xF2D2	0x86DE	
+0xF2D3	0x86F4	
+0xF2D4	0x86DF	
+0xF2D5	0x86D8	
+0xF2D6	0x86D1	
+0xF2D7	0x8703	
+0xF2D8	0x8707	
+0xF2D9	0x86F8	
+0xF2DA	0x8708	
+0xF2DB	0x870A	
+0xF2DC	0x870D	
+0xF2DD	0x8709	
+0xF2DE	0x8723	
+0xF2DF	0x873B	
+0xF2E0	0x871E	
+0xF2E1	0x8725	
+0xF2E2	0x872E	
+0xF2E3	0x871A	
+0xF2E4	0x873E	
+0xF2E5	0x8748	
+0xF2E6	0x8734	
+0xF2E7	0x8731	
+0xF2E8	0x8729	
+0xF2E9	0x8737	
+0xF2EA	0x873F	
+0xF2EB	0x8782	
+0xF2EC	0x8722	
+0xF2ED	0x877D	
+0xF2EE	0x877E	
+0xF2EF	0x877B	
+0xF2F0	0x8760	
+0xF2F1	0x8770	
+0xF2F2	0x874C	
+0xF2F3	0x876E	
+0xF2F4	0x878B	
+0xF2F5	0x8753	
+0xF2F6	0x8763	
+0xF2F7	0x877C	
+0xF2F8	0x8764	
+0xF2F9	0x8759	
+0xF2FA	0x8765	
+0xF2FB	0x8793	
+0xF2FC	0x87AF	
+0xF2FD	0x87A8	
+0xF2FE	0x87D2	
+0xF340	0x9A5A	
+0xF341	0x9A5B	
+0xF342	0x9A5C	
+0xF343	0x9A5D	
+0xF344	0x9A5E	
+0xF345	0x9A5F	
+0xF346	0x9A60	
+0xF347	0x9A61	
+0xF348	0x9A62	
+0xF349	0x9A63	
+0xF34A	0x9A64	
+0xF34B	0x9A65	
+0xF34C	0x9A66	
+0xF34D	0x9A67	
+0xF34E	0x9A68	
+0xF34F	0x9A69	
+0xF350	0x9A6A	
+0xF351	0x9A6B	
+0xF352	0x9A72	
+0xF353	0x9A83	
+0xF354	0x9A89	
+0xF355	0x9A8D	
+0xF356	0x9A8E	
+0xF357	0x9A94	
+0xF358	0x9A95	
+0xF359	0x9A99	
+0xF35A	0x9AA6	
+0xF35B	0x9AA9	
+0xF35C	0x9AAA	
+0xF35D	0x9AAB	
+0xF35E	0x9AAC	
+0xF35F	0x9AAD	
+0xF360	0x9AAE	
+0xF361	0x9AAF	
+0xF362	0x9AB2	
+0xF363	0x9AB3	
+0xF364	0x9AB4	
+0xF365	0x9AB5	
+0xF366	0x9AB9	
+0xF367	0x9ABB	
+0xF368	0x9ABD	
+0xF369	0x9ABE	
+0xF36A	0x9ABF	
+0xF36B	0x9AC3	
+0xF36C	0x9AC4	
+0xF36D	0x9AC6	
+0xF36E	0x9AC7	
+0xF36F	0x9AC8	
+0xF370	0x9AC9	
+0xF371	0x9ACA	
+0xF372	0x9ACD	
+0xF373	0x9ACE	
+0xF374	0x9ACF	
+0xF375	0x9AD0	
+0xF376	0x9AD2	
+0xF377	0x9AD4	
+0xF378	0x9AD5	
+0xF379	0x9AD6	
+0xF37A	0x9AD7	
+0xF37B	0x9AD9	
+0xF37C	0x9ADA	
+0xF37D	0x9ADB	
+0xF37E	0x9ADC	
+0xF380	0x9ADD	
+0xF381	0x9ADE	
+0xF382	0x9AE0	
+0xF383	0x9AE2	
+0xF384	0x9AE3	
+0xF385	0x9AE4	
+0xF386	0x9AE5	
+0xF387	0x9AE7	
+0xF388	0x9AE8	
+0xF389	0x9AE9	
+0xF38A	0x9AEA	
+0xF38B	0x9AEC	
+0xF38C	0x9AEE	
+0xF38D	0x9AF0	
+0xF38E	0x9AF1	
+0xF38F	0x9AF2	
+0xF390	0x9AF3	
+0xF391	0x9AF4	
+0xF392	0x9AF5	
+0xF393	0x9AF6	
+0xF394	0x9AF7	
+0xF395	0x9AF8	
+0xF396	0x9AFA	
+0xF397	0x9AFC	
+0xF398	0x9AFD	
+0xF399	0x9AFE	
+0xF39A	0x9AFF	
+0xF39B	0x9B00	
+0xF39C	0x9B01	
+0xF39D	0x9B02	
+0xF39E	0x9B04	
+0xF39F	0x9B05	
+0xF3A0	0x9B06	
+0xF3A1	0x87C6	
+0xF3A2	0x8788	
+0xF3A3	0x8785	
+0xF3A4	0x87AD	
+0xF3A5	0x8797	
+0xF3A6	0x8783	
+0xF3A7	0x87AB	
+0xF3A8	0x87E5	
+0xF3A9	0x87AC	
+0xF3AA	0x87B5	
+0xF3AB	0x87B3	
+0xF3AC	0x87CB	
+0xF3AD	0x87D3	
+0xF3AE	0x87BD	
+0xF3AF	0x87D1	
+0xF3B0	0x87C0	
+0xF3B1	0x87CA	
+0xF3B2	0x87DB	
+0xF3B3	0x87EA	
+0xF3B4	0x87E0	
+0xF3B5	0x87EE	
+0xF3B6	0x8816	
+0xF3B7	0x8813	
+0xF3B8	0x87FE	
+0xF3B9	0x880A	
+0xF3BA	0x881B	
+0xF3BB	0x8821	
+0xF3BC	0x8839	
+0xF3BD	0x883C	
+0xF3BE	0x7F36	
+0xF3BF	0x7F42	
+0xF3C0	0x7F44	
+0xF3C1	0x7F45	
+0xF3C2	0x8210	
+0xF3C3	0x7AFA	
+0xF3C4	0x7AFD	
+0xF3C5	0x7B08	
+0xF3C6	0x7B03	
+0xF3C7	0x7B04	
+0xF3C8	0x7B15	
+0xF3C9	0x7B0A	
+0xF3CA	0x7B2B	
+0xF3CB	0x7B0F	
+0xF3CC	0x7B47	
+0xF3CD	0x7B38	
+0xF3CE	0x7B2A	
+0xF3CF	0x7B19	
+0xF3D0	0x7B2E	
+0xF3D1	0x7B31	
+0xF3D2	0x7B20	
+0xF3D3	0x7B25	
+0xF3D4	0x7B24	
+0xF3D5	0x7B33	
+0xF3D6	0x7B3E	
+0xF3D7	0x7B1E	
+0xF3D8	0x7B58	
+0xF3D9	0x7B5A	
+0xF3DA	0x7B45	
+0xF3DB	0x7B75	
+0xF3DC	0x7B4C	
+0xF3DD	0x7B5D	
+0xF3DE	0x7B60	
+0xF3DF	0x7B6E	
+0xF3E0	0x7B7B	
+0xF3E1	0x7B62	
+0xF3E2	0x7B72	
+0xF3E3	0x7B71	
+0xF3E4	0x7B90	
+0xF3E5	0x7BA6	
+0xF3E6	0x7BA7	
+0xF3E7	0x7BB8	
+0xF3E8	0x7BAC	
+0xF3E9	0x7B9D	
+0xF3EA	0x7BA8	
+0xF3EB	0x7B85	
+0xF3EC	0x7BAA	
+0xF3ED	0x7B9C	
+0xF3EE	0x7BA2	
+0xF3EF	0x7BAB	
+0xF3F0	0x7BB4	
+0xF3F1	0x7BD1	
+0xF3F2	0x7BC1	
+0xF3F3	0x7BCC	
+0xF3F4	0x7BDD	
+0xF3F5	0x7BDA	
+0xF3F6	0x7BE5	
+0xF3F7	0x7BE6	
+0xF3F8	0x7BEA	
+0xF3F9	0x7C0C	
+0xF3FA	0x7BFE	
+0xF3FB	0x7BFC	
+0xF3FC	0x7C0F	
+0xF3FD	0x7C16	
+0xF3FE	0x7C0B	
+0xF440	0x9B07	
+0xF441	0x9B09	
+0xF442	0x9B0A	
+0xF443	0x9B0B	
+0xF444	0x9B0C	
+0xF445	0x9B0D	
+0xF446	0x9B0E	
+0xF447	0x9B10	
+0xF448	0x9B11	
+0xF449	0x9B12	
+0xF44A	0x9B14	
+0xF44B	0x9B15	
+0xF44C	0x9B16	
+0xF44D	0x9B17	
+0xF44E	0x9B18	
+0xF44F	0x9B19	
+0xF450	0x9B1A	
+0xF451	0x9B1B	
+0xF452	0x9B1C	
+0xF453	0x9B1D	
+0xF454	0x9B1E	
+0xF455	0x9B20	
+0xF456	0x9B21	
+0xF457	0x9B22	
+0xF458	0x9B24	
+0xF459	0x9B25	
+0xF45A	0x9B26	
+0xF45B	0x9B27	
+0xF45C	0x9B28	
+0xF45D	0x9B29	
+0xF45E	0x9B2A	
+0xF45F	0x9B2B	
+0xF460	0x9B2C	
+0xF461	0x9B2D	
+0xF462	0x9B2E	
+0xF463	0x9B30	
+0xF464	0x9B31	
+0xF465	0x9B33	
+0xF466	0x9B34	
+0xF467	0x9B35	
+0xF468	0x9B36	
+0xF469	0x9B37	
+0xF46A	0x9B38	
+0xF46B	0x9B39	
+0xF46C	0x9B3A	
+0xF46D	0x9B3D	
+0xF46E	0x9B3E	
+0xF46F	0x9B3F	
+0xF470	0x9B40	
+0xF471	0x9B46	
+0xF472	0x9B4A	
+0xF473	0x9B4B	
+0xF474	0x9B4C	
+0xF475	0x9B4E	
+0xF476	0x9B50	
+0xF477	0x9B52	
+0xF478	0x9B53	
+0xF479	0x9B55	
+0xF47A	0x9B56	
+0xF47B	0x9B57	
+0xF47C	0x9B58	
+0xF47D	0x9B59	
+0xF47E	0x9B5A	
+0xF480	0x9B5B	
+0xF481	0x9B5C	
+0xF482	0x9B5D	
+0xF483	0x9B5E	
+0xF484	0x9B5F	
+0xF485	0x9B60	
+0xF486	0x9B61	
+0xF487	0x9B62	
+0xF488	0x9B63	
+0xF489	0x9B64	
+0xF48A	0x9B65	
+0xF48B	0x9B66	
+0xF48C	0x9B67	
+0xF48D	0x9B68	
+0xF48E	0x9B69	
+0xF48F	0x9B6A	
+0xF490	0x9B6B	
+0xF491	0x9B6C	
+0xF492	0x9B6D	
+0xF493	0x9B6E	
+0xF494	0x9B6F	
+0xF495	0x9B70	
+0xF496	0x9B71	
+0xF497	0x9B72	
+0xF498	0x9B73	
+0xF499	0x9B74	
+0xF49A	0x9B75	
+0xF49B	0x9B76	
+0xF49C	0x9B77	
+0xF49D	0x9B78	
+0xF49E	0x9B79	
+0xF49F	0x9B7A	
+0xF4A0	0x9B7B	
+0xF4A1	0x7C1F	
+0xF4A2	0x7C2A	
+0xF4A3	0x7C26	
+0xF4A4	0x7C38	
+0xF4A5	0x7C41	
+0xF4A6	0x7C40	
+0xF4A7	0x81FE	
+0xF4A8	0x8201	
+0xF4A9	0x8202	
+0xF4AA	0x8204	
+0xF4AB	0x81EC	
+0xF4AC	0x8844	
+0xF4AD	0x8221	
+0xF4AE	0x8222	
+0xF4AF	0x8223	
+0xF4B0	0x822D	
+0xF4B1	0x822F	
+0xF4B2	0x8228	
+0xF4B3	0x822B	
+0xF4B4	0x8238	
+0xF4B5	0x823B	
+0xF4B6	0x8233	
+0xF4B7	0x8234	
+0xF4B8	0x823E	
+0xF4B9	0x8244	
+0xF4BA	0x8249	
+0xF4BB	0x824B	
+0xF4BC	0x824F	
+0xF4BD	0x825A	
+0xF4BE	0x825F	
+0xF4BF	0x8268	
+0xF4C0	0x887E	
+0xF4C1	0x8885	
+0xF4C2	0x8888	
+0xF4C3	0x88D8	
+0xF4C4	0x88DF	
+0xF4C5	0x895E	
+0xF4C6	0x7F9D	
+0xF4C7	0x7F9F	
+0xF4C8	0x7FA7	
+0xF4C9	0x7FAF	
+0xF4CA	0x7FB0	
+0xF4CB	0x7FB2	
+0xF4CC	0x7C7C	
+0xF4CD	0x6549	
+0xF4CE	0x7C91	
+0xF4CF	0x7C9D	
+0xF4D0	0x7C9C	
+0xF4D1	0x7C9E	
+0xF4D2	0x7CA2	
+0xF4D3	0x7CB2	
+0xF4D4	0x7CBC	
+0xF4D5	0x7CBD	
+0xF4D6	0x7CC1	
+0xF4D7	0x7CC7	
+0xF4D8	0x7CCC	
+0xF4D9	0x7CCD	
+0xF4DA	0x7CC8	
+0xF4DB	0x7CC5	
+0xF4DC	0x7CD7	
+0xF4DD	0x7CE8	
+0xF4DE	0x826E	
+0xF4DF	0x66A8	
+0xF4E0	0x7FBF	
+0xF4E1	0x7FCE	
+0xF4E2	0x7FD5	
+0xF4E3	0x7FE5	
+0xF4E4	0x7FE1	
+0xF4E5	0x7FE6	
+0xF4E6	0x7FE9	
+0xF4E7	0x7FEE	
+0xF4E8	0x7FF3	
+0xF4E9	0x7CF8	
+0xF4EA	0x7D77	
+0xF4EB	0x7DA6	
+0xF4EC	0x7DAE	
+0xF4ED	0x7E47	
+0xF4EE	0x7E9B	
+0xF4EF	0x9EB8	
+0xF4F0	0x9EB4	
+0xF4F1	0x8D73	
+0xF4F2	0x8D84	
+0xF4F3	0x8D94	
+0xF4F4	0x8D91	
+0xF4F5	0x8DB1	
+0xF4F6	0x8D67	
+0xF4F7	0x8D6D	
+0xF4F8	0x8C47	
+0xF4F9	0x8C49	
+0xF4FA	0x914A	
+0xF4FB	0x9150	
+0xF4FC	0x914E	
+0xF4FD	0x914F	
+0xF4FE	0x9164	
+0xF540	0x9B7C	
+0xF541	0x9B7D	
+0xF542	0x9B7E	
+0xF543	0x9B7F	
+0xF544	0x9B80	
+0xF545	0x9B81	
+0xF546	0x9B82	
+0xF547	0x9B83	
+0xF548	0x9B84	
+0xF549	0x9B85	
+0xF54A	0x9B86	
+0xF54B	0x9B87	
+0xF54C	0x9B88	
+0xF54D	0x9B89	
+0xF54E	0x9B8A	
+0xF54F	0x9B8B	
+0xF550	0x9B8C	
+0xF551	0x9B8D	
+0xF552	0x9B8E	
+0xF553	0x9B8F	
+0xF554	0x9B90	
+0xF555	0x9B91	
+0xF556	0x9B92	
+0xF557	0x9B93	
+0xF558	0x9B94	
+0xF559	0x9B95	
+0xF55A	0x9B96	
+0xF55B	0x9B97	
+0xF55C	0x9B98	
+0xF55D	0x9B99	
+0xF55E	0x9B9A	
+0xF55F	0x9B9B	
+0xF560	0x9B9C	
+0xF561	0x9B9D	
+0xF562	0x9B9E	
+0xF563	0x9B9F	
+0xF564	0x9BA0	
+0xF565	0x9BA1	
+0xF566	0x9BA2	
+0xF567	0x9BA3	
+0xF568	0x9BA4	
+0xF569	0x9BA5	
+0xF56A	0x9BA6	
+0xF56B	0x9BA7	
+0xF56C	0x9BA8	
+0xF56D	0x9BA9	
+0xF56E	0x9BAA	
+0xF56F	0x9BAB	
+0xF570	0x9BAC	
+0xF571	0x9BAD	
+0xF572	0x9BAE	
+0xF573	0x9BAF	
+0xF574	0x9BB0	
+0xF575	0x9BB1	
+0xF576	0x9BB2	
+0xF577	0x9BB3	
+0xF578	0x9BB4	
+0xF579	0x9BB5	
+0xF57A	0x9BB6	
+0xF57B	0x9BB7	
+0xF57C	0x9BB8	
+0xF57D	0x9BB9	
+0xF57E	0x9BBA	
+0xF580	0x9BBB	
+0xF581	0x9BBC	
+0xF582	0x9BBD	
+0xF583	0x9BBE	
+0xF584	0x9BBF	
+0xF585	0x9BC0	
+0xF586	0x9BC1	
+0xF587	0x9BC2	
+0xF588	0x9BC3	
+0xF589	0x9BC4	
+0xF58A	0x9BC5	
+0xF58B	0x9BC6	
+0xF58C	0x9BC7	
+0xF58D	0x9BC8	
+0xF58E	0x9BC9	
+0xF58F	0x9BCA	
+0xF590	0x9BCB	
+0xF591	0x9BCC	
+0xF592	0x9BCD	
+0xF593	0x9BCE	
+0xF594	0x9BCF	
+0xF595	0x9BD0	
+0xF596	0x9BD1	
+0xF597	0x9BD2	
+0xF598	0x9BD3	
+0xF599	0x9BD4	
+0xF59A	0x9BD5	
+0xF59B	0x9BD6	
+0xF59C	0x9BD7	
+0xF59D	0x9BD8	
+0xF59E	0x9BD9	
+0xF59F	0x9BDA	
+0xF5A0	0x9BDB	
+0xF5A1	0x9162	
+0xF5A2	0x9161	
+0xF5A3	0x9170	
+0xF5A4	0x9169	
+0xF5A5	0x916F	
+0xF5A6	0x917D	
+0xF5A7	0x917E	
+0xF5A8	0x9172	
+0xF5A9	0x9174	
+0xF5AA	0x9179	
+0xF5AB	0x918C	
+0xF5AC	0x9185	
+0xF5AD	0x9190	
+0xF5AE	0x918D	
+0xF5AF	0x9191	
+0xF5B0	0x91A2	
+0xF5B1	0x91A3	
+0xF5B2	0x91AA	
+0xF5B3	0x91AD	
+0xF5B4	0x91AE	
+0xF5B5	0x91AF	
+0xF5B6	0x91B5	
+0xF5B7	0x91B4	
+0xF5B8	0x91BA	
+0xF5B9	0x8C55	
+0xF5BA	0x9E7E	
+0xF5BB	0x8DB8	
+0xF5BC	0x8DEB	
+0xF5BD	0x8E05	
+0xF5BE	0x8E59	
+0xF5BF	0x8E69	
+0xF5C0	0x8DB5	
+0xF5C1	0x8DBF	
+0xF5C2	0x8DBC	
+0xF5C3	0x8DBA	
+0xF5C4	0x8DC4	
+0xF5C5	0x8DD6	
+0xF5C6	0x8DD7	
+0xF5C7	0x8DDA	
+0xF5C8	0x8DDE	
+0xF5C9	0x8DCE	
+0xF5CA	0x8DCF	
+0xF5CB	0x8DDB	
+0xF5CC	0x8DC6	
+0xF5CD	0x8DEC	
+0xF5CE	0x8DF7	
+0xF5CF	0x8DF8	
+0xF5D0	0x8DE3	
+0xF5D1	0x8DF9	
+0xF5D2	0x8DFB	
+0xF5D3	0x8DE4	
+0xF5D4	0x8E09	
+0xF5D5	0x8DFD	
+0xF5D6	0x8E14	
+0xF5D7	0x8E1D	
+0xF5D8	0x8E1F	
+0xF5D9	0x8E2C	
+0xF5DA	0x8E2E	
+0xF5DB	0x8E23	
+0xF5DC	0x8E2F	
+0xF5DD	0x8E3A	
+0xF5DE	0x8E40	
+0xF5DF	0x8E39	
+0xF5E0	0x8E35	
+0xF5E1	0x8E3D	
+0xF5E2	0x8E31	
+0xF5E3	0x8E49	
+0xF5E4	0x8E41	
+0xF5E5	0x8E42	
+0xF5E6	0x8E51	
+0xF5E7	0x8E52	
+0xF5E8	0x8E4A	
+0xF5E9	0x8E70	
+0xF5EA	0x8E76	
+0xF5EB	0x8E7C	
+0xF5EC	0x8E6F	
+0xF5ED	0x8E74	
+0xF5EE	0x8E85	
+0xF5EF	0x8E8F	
+0xF5F0	0x8E94	
+0xF5F1	0x8E90	
+0xF5F2	0x8E9C	
+0xF5F3	0x8E9E	
+0xF5F4	0x8C78	
+0xF5F5	0x8C82	
+0xF5F6	0x8C8A	
+0xF5F7	0x8C85	
+0xF5F8	0x8C98	
+0xF5F9	0x8C94	
+0xF5FA	0x659B	
+0xF5FB	0x89D6	
+0xF5FC	0x89DE	
+0xF5FD	0x89DA	
+0xF5FE	0x89DC	
+0xF640	0x9BDC	
+0xF641	0x9BDD	
+0xF642	0x9BDE	
+0xF643	0x9BDF	
+0xF644	0x9BE0	
+0xF645	0x9BE1	
+0xF646	0x9BE2	
+0xF647	0x9BE3	
+0xF648	0x9BE4	
+0xF649	0x9BE5	
+0xF64A	0x9BE6	
+0xF64B	0x9BE7	
+0xF64C	0x9BE8	
+0xF64D	0x9BE9	
+0xF64E	0x9BEA	
+0xF64F	0x9BEB	
+0xF650	0x9BEC	
+0xF651	0x9BED	
+0xF652	0x9BEE	
+0xF653	0x9BEF	
+0xF654	0x9BF0	
+0xF655	0x9BF1	
+0xF656	0x9BF2	
+0xF657	0x9BF3	
+0xF658	0x9BF4	
+0xF659	0x9BF5	
+0xF65A	0x9BF6	
+0xF65B	0x9BF7	
+0xF65C	0x9BF8	
+0xF65D	0x9BF9	
+0xF65E	0x9BFA	
+0xF65F	0x9BFB	
+0xF660	0x9BFC	
+0xF661	0x9BFD	
+0xF662	0x9BFE	
+0xF663	0x9BFF	
+0xF664	0x9C00	
+0xF665	0x9C01	
+0xF666	0x9C02	
+0xF667	0x9C03	
+0xF668	0x9C04	
+0xF669	0x9C05	
+0xF66A	0x9C06	
+0xF66B	0x9C07	
+0xF66C	0x9C08	
+0xF66D	0x9C09	
+0xF66E	0x9C0A	
+0xF66F	0x9C0B	
+0xF670	0x9C0C	
+0xF671	0x9C0D	
+0xF672	0x9C0E	
+0xF673	0x9C0F	
+0xF674	0x9C10	
+0xF675	0x9C11	
+0xF676	0x9C12	
+0xF677	0x9C13	
+0xF678	0x9C14	
+0xF679	0x9C15	
+0xF67A	0x9C16	
+0xF67B	0x9C17	
+0xF67C	0x9C18	
+0xF67D	0x9C19	
+0xF67E	0x9C1A	
+0xF680	0x9C1B	
+0xF681	0x9C1C	
+0xF682	0x9C1D	
+0xF683	0x9C1E	
+0xF684	0x9C1F	
+0xF685	0x9C20	
+0xF686	0x9C21	
+0xF687	0x9C22	
+0xF688	0x9C23	
+0xF689	0x9C24	
+0xF68A	0x9C25	
+0xF68B	0x9C26	
+0xF68C	0x9C27	
+0xF68D	0x9C28	
+0xF68E	0x9C29	
+0xF68F	0x9C2A	
+0xF690	0x9C2B	
+0xF691	0x9C2C	
+0xF692	0x9C2D	
+0xF693	0x9C2E	
+0xF694	0x9C2F	
+0xF695	0x9C30	
+0xF696	0x9C31	
+0xF697	0x9C32	
+0xF698	0x9C33	
+0xF699	0x9C34	
+0xF69A	0x9C35	
+0xF69B	0x9C36	
+0xF69C	0x9C37	
+0xF69D	0x9C38	
+0xF69E	0x9C39	
+0xF69F	0x9C3A	
+0xF6A0	0x9C3B	
+0xF6A1	0x89E5	
+0xF6A2	0x89EB	
+0xF6A3	0x89EF	
+0xF6A4	0x8A3E	
+0xF6A5	0x8B26	
+0xF6A6	0x9753	
+0xF6A7	0x96E9	
+0xF6A8	0x96F3	
+0xF6A9	0x96EF	
+0xF6AA	0x9706	
+0xF6AB	0x9701	
+0xF6AC	0x9708	
+0xF6AD	0x970F	
+0xF6AE	0x970E	
+0xF6AF	0x972A	
+0xF6B0	0x972D	
+0xF6B1	0x9730	
+0xF6B2	0x973E	
+0xF6B3	0x9F80	
+0xF6B4	0x9F83	
+0xF6B5	0x9F85	
+0xF6B6	0x9F86	
+0xF6B7	0x9F87	
+0xF6B8	0x9F88	
+0xF6B9	0x9F89	
+0xF6BA	0x9F8A	
+0xF6BB	0x9F8C	
+0xF6BC	0x9EFE	
+0xF6BD	0x9F0B	
+0xF6BE	0x9F0D	
+0xF6BF	0x96B9	
+0xF6C0	0x96BC	
+0xF6C1	0x96BD	
+0xF6C2	0x96CE	
+0xF6C3	0x96D2	
+0xF6C4	0x77BF	
+0xF6C5	0x96E0	
+0xF6C6	0x928E	
+0xF6C7	0x92AE	
+0xF6C8	0x92C8	
+0xF6C9	0x933E	
+0xF6CA	0x936A	
+0xF6CB	0x93CA	
+0xF6CC	0x938F	
+0xF6CD	0x943E	
+0xF6CE	0x946B	
+0xF6CF	0x9C7F	
+0xF6D0	0x9C82	
+0xF6D1	0x9C85	
+0xF6D2	0x9C86	
+0xF6D3	0x9C87	
+0xF6D4	0x9C88	
+0xF6D5	0x7A23	
+0xF6D6	0x9C8B	
+0xF6D7	0x9C8E	
+0xF6D8	0x9C90	
+0xF6D9	0x9C91	
+0xF6DA	0x9C92	
+0xF6DB	0x9C94	
+0xF6DC	0x9C95	
+0xF6DD	0x9C9A	
+0xF6DE	0x9C9B	
+0xF6DF	0x9C9E	
+0xF6E0	0x9C9F	
+0xF6E1	0x9CA0	
+0xF6E2	0x9CA1	
+0xF6E3	0x9CA2	
+0xF6E4	0x9CA3	
+0xF6E5	0x9CA5	
+0xF6E6	0x9CA6	
+0xF6E7	0x9CA7	
+0xF6E8	0x9CA8	
+0xF6E9	0x9CA9	
+0xF6EA	0x9CAB	
+0xF6EB	0x9CAD	
+0xF6EC	0x9CAE	
+0xF6ED	0x9CB0	
+0xF6EE	0x9CB1	
+0xF6EF	0x9CB2	
+0xF6F0	0x9CB3	
+0xF6F1	0x9CB4	
+0xF6F2	0x9CB5	
+0xF6F3	0x9CB6	
+0xF6F4	0x9CB7	
+0xF6F5	0x9CBA	
+0xF6F6	0x9CBB	
+0xF6F7	0x9CBC	
+0xF6F8	0x9CBD	
+0xF6F9	0x9CC4	
+0xF6FA	0x9CC5	
+0xF6FB	0x9CC6	
+0xF6FC	0x9CC7	
+0xF6FD	0x9CCA	
+0xF6FE	0x9CCB	
+0xF740	0x9C3C	
+0xF741	0x9C3D	
+0xF742	0x9C3E	
+0xF743	0x9C3F	
+0xF744	0x9C40	
+0xF745	0x9C41	
+0xF746	0x9C42	
+0xF747	0x9C43	
+0xF748	0x9C44	
+0xF749	0x9C45	
+0xF74A	0x9C46	
+0xF74B	0x9C47	
+0xF74C	0x9C48	
+0xF74D	0x9C49	
+0xF74E	0x9C4A	
+0xF74F	0x9C4B	
+0xF750	0x9C4C	
+0xF751	0x9C4D	
+0xF752	0x9C4E	
+0xF753	0x9C4F	
+0xF754	0x9C50	
+0xF755	0x9C51	
+0xF756	0x9C52	
+0xF757	0x9C53	
+0xF758	0x9C54	
+0xF759	0x9C55	
+0xF75A	0x9C56	
+0xF75B	0x9C57	
+0xF75C	0x9C58	
+0xF75D	0x9C59	
+0xF75E	0x9C5A	
+0xF75F	0x9C5B	
+0xF760	0x9C5C	
+0xF761	0x9C5D	
+0xF762	0x9C5E	
+0xF763	0x9C5F	
+0xF764	0x9C60	
+0xF765	0x9C61	
+0xF766	0x9C62	
+0xF767	0x9C63	
+0xF768	0x9C64	
+0xF769	0x9C65	
+0xF76A	0x9C66	
+0xF76B	0x9C67	
+0xF76C	0x9C68	
+0xF76D	0x9C69	
+0xF76E	0x9C6A	
+0xF76F	0x9C6B	
+0xF770	0x9C6C	
+0xF771	0x9C6D	
+0xF772	0x9C6E	
+0xF773	0x9C6F	
+0xF774	0x9C70	
+0xF775	0x9C71	
+0xF776	0x9C72	
+0xF777	0x9C73	
+0xF778	0x9C74	
+0xF779	0x9C75	
+0xF77A	0x9C76	
+0xF77B	0x9C77	
+0xF77C	0x9C78	
+0xF77D	0x9C79	
+0xF77E	0x9C7A	
+0xF780	0x9C7B	
+0xF781	0x9C7D	
+0xF782	0x9C7E	
+0xF783	0x9C80	
+0xF784	0x9C83	
+0xF785	0x9C84	
+0xF786	0x9C89	
+0xF787	0x9C8A	
+0xF788	0x9C8C	
+0xF789	0x9C8F	
+0xF78A	0x9C93	
+0xF78B	0x9C96	
+0xF78C	0x9C97	
+0xF78D	0x9C98	
+0xF78E	0x9C99	
+0xF78F	0x9C9D	
+0xF790	0x9CAA	
+0xF791	0x9CAC	
+0xF792	0x9CAF	
+0xF793	0x9CB9	
+0xF794	0x9CBE	
+0xF795	0x9CBF	
+0xF796	0x9CC0	
+0xF797	0x9CC1	
+0xF798	0x9CC2	
+0xF799	0x9CC8	
+0xF79A	0x9CC9	
+0xF79B	0x9CD1	
+0xF79C	0x9CD2	
+0xF79D	0x9CDA	
+0xF79E	0x9CDB	
+0xF79F	0x9CE0	
+0xF7A0	0x9CE1	
+0xF7A1	0x9CCC	
+0xF7A2	0x9CCD	
+0xF7A3	0x9CCE	
+0xF7A4	0x9CCF	
+0xF7A5	0x9CD0	
+0xF7A6	0x9CD3	
+0xF7A7	0x9CD4	
+0xF7A8	0x9CD5	
+0xF7A9	0x9CD7	
+0xF7AA	0x9CD8	
+0xF7AB	0x9CD9	
+0xF7AC	0x9CDC	
+0xF7AD	0x9CDD	
+0xF7AE	0x9CDF	
+0xF7AF	0x9CE2	
+0xF7B0	0x977C	
+0xF7B1	0x9785	
+0xF7B2	0x9791	
+0xF7B3	0x9792	
+0xF7B4	0x9794	
+0xF7B5	0x97AF	
+0xF7B6	0x97AB	
+0xF7B7	0x97A3	
+0xF7B8	0x97B2	
+0xF7B9	0x97B4	
+0xF7BA	0x9AB1	
+0xF7BB	0x9AB0	
+0xF7BC	0x9AB7	
+0xF7BD	0x9E58	
+0xF7BE	0x9AB6	
+0xF7BF	0x9ABA	
+0xF7C0	0x9ABC	
+0xF7C1	0x9AC1	
+0xF7C2	0x9AC0	
+0xF7C3	0x9AC5	
+0xF7C4	0x9AC2	
+0xF7C5	0x9ACB	
+0xF7C6	0x9ACC	
+0xF7C7	0x9AD1	
+0xF7C8	0x9B45	
+0xF7C9	0x9B43	
+0xF7CA	0x9B47	
+0xF7CB	0x9B49	
+0xF7CC	0x9B48	
+0xF7CD	0x9B4D	
+0xF7CE	0x9B51	
+0xF7CF	0x98E8	
+0xF7D0	0x990D	
+0xF7D1	0x992E	
+0xF7D2	0x9955	
+0xF7D3	0x9954	
+0xF7D4	0x9ADF	
+0xF7D5	0x9AE1	
+0xF7D6	0x9AE6	
+0xF7D7	0x9AEF	
+0xF7D8	0x9AEB	
+0xF7D9	0x9AFB	
+0xF7DA	0x9AED	
+0xF7DB	0x9AF9	
+0xF7DC	0x9B08	
+0xF7DD	0x9B0F	
+0xF7DE	0x9B13	
+0xF7DF	0x9B1F	
+0xF7E0	0x9B23	
+0xF7E1	0x9EBD	
+0xF7E2	0x9EBE	
+0xF7E3	0x7E3B	
+0xF7E4	0x9E82	
+0xF7E5	0x9E87	
+0xF7E6	0x9E88	
+0xF7E7	0x9E8B	
+0xF7E8	0x9E92	
+0xF7E9	0x93D6	
+0xF7EA	0x9E9D	
+0xF7EB	0x9E9F	
+0xF7EC	0x9EDB	
+0xF7ED	0x9EDC	
+0xF7EE	0x9EDD	
+0xF7EF	0x9EE0	
+0xF7F0	0x9EDF	
+0xF7F1	0x9EE2	
+0xF7F2	0x9EE9	
+0xF7F3	0x9EE7	
+0xF7F4	0x9EE5	
+0xF7F5	0x9EEA	
+0xF7F6	0x9EEF	
+0xF7F7	0x9F22	
+0xF7F8	0x9F2C	
+0xF7F9	0x9F2F	
+0xF7FA	0x9F39	
+0xF7FB	0x9F37	
+0xF7FC	0x9F3D	
+0xF7FD	0x9F3E	
+0xF7FE	0x9F44	
+0xF840	0x9CE3	
+0xF841	0x9CE4	
+0xF842	0x9CE5	
+0xF843	0x9CE6	
+0xF844	0x9CE7	
+0xF845	0x9CE8	
+0xF846	0x9CE9	
+0xF847	0x9CEA	
+0xF848	0x9CEB	
+0xF849	0x9CEC	
+0xF84A	0x9CED	
+0xF84B	0x9CEE	
+0xF84C	0x9CEF	
+0xF84D	0x9CF0	
+0xF84E	0x9CF1	
+0xF84F	0x9CF2	
+0xF850	0x9CF3	
+0xF851	0x9CF4	
+0xF852	0x9CF5	
+0xF853	0x9CF6	
+0xF854	0x9CF7	
+0xF855	0x9CF8	
+0xF856	0x9CF9	
+0xF857	0x9CFA	
+0xF858	0x9CFB	
+0xF859	0x9CFC	
+0xF85A	0x9CFD	
+0xF85B	0x9CFE	
+0xF85C	0x9CFF	
+0xF85D	0x9D00	
+0xF85E	0x9D01	
+0xF85F	0x9D02	
+0xF860	0x9D03	
+0xF861	0x9D04	
+0xF862	0x9D05	
+0xF863	0x9D06	
+0xF864	0x9D07	
+0xF865	0x9D08	
+0xF866	0x9D09	
+0xF867	0x9D0A	
+0xF868	0x9D0B	
+0xF869	0x9D0C	
+0xF86A	0x9D0D	
+0xF86B	0x9D0E	
+0xF86C	0x9D0F	
+0xF86D	0x9D10	
+0xF86E	0x9D11	
+0xF86F	0x9D12	
+0xF870	0x9D13	
+0xF871	0x9D14	
+0xF872	0x9D15	
+0xF873	0x9D16	
+0xF874	0x9D17	
+0xF875	0x9D18	
+0xF876	0x9D19	
+0xF877	0x9D1A	
+0xF878	0x9D1B	
+0xF879	0x9D1C	
+0xF87A	0x9D1D	
+0xF87B	0x9D1E	
+0xF87C	0x9D1F	
+0xF87D	0x9D20	
+0xF87E	0x9D21	
+0xF880	0x9D22	
+0xF881	0x9D23	
+0xF882	0x9D24	
+0xF883	0x9D25	
+0xF884	0x9D26	
+0xF885	0x9D27	
+0xF886	0x9D28	
+0xF887	0x9D29	
+0xF888	0x9D2A	
+0xF889	0x9D2B	
+0xF88A	0x9D2C	
+0xF88B	0x9D2D	
+0xF88C	0x9D2E	
+0xF88D	0x9D2F	
+0xF88E	0x9D30	
+0xF88F	0x9D31	
+0xF890	0x9D32	
+0xF891	0x9D33	
+0xF892	0x9D34	
+0xF893	0x9D35	
+0xF894	0x9D36	
+0xF895	0x9D37	
+0xF896	0x9D38	
+0xF897	0x9D39	
+0xF898	0x9D3A	
+0xF899	0x9D3B	
+0xF89A	0x9D3C	
+0xF89B	0x9D3D	
+0xF89C	0x9D3E	
+0xF89D	0x9D3F	
+0xF89E	0x9D40	
+0xF89F	0x9D41	
+0xF8A0	0x9D42	
+# 0xF8A1	 	
+# 0xF8A2	 	
+# 0xF8A3	 	
+# 0xF8A4	 	
+# 0xF8A5	 	
+# 0xF8A6	 	
+# 0xF8A7	 	
+# 0xF8A8	 	
+# 0xF8A9	 	
+# 0xF8AA	 	
+# 0xF8AB	 	
+# 0xF8AC	 	
+# 0xF8AD	 	
+# 0xF8AE	 	
+# 0xF8AF	 	
+# 0xF8B0	 	
+# 0xF8B1	 	
+# 0xF8B2	 	
+# 0xF8B3	 	
+# 0xF8B4	 	
+# 0xF8B5	 	
+# 0xF8B6	 	
+# 0xF8B7	 	
+# 0xF8B8	 	
+# 0xF8B9	 	
+# 0xF8BA	 	
+# 0xF8BB	 	
+# 0xF8BC	 	
+# 0xF8BD	 	
+# 0xF8BE	 	
+# 0xF8BF	 	
+# 0xF8C0	 	
+# 0xF8C1	 	
+# 0xF8C2	 	
+# 0xF8C3	 	
+# 0xF8C4	 	
+# 0xF8C5	 	
+# 0xF8C6	 	
+# 0xF8C7	 	
+# 0xF8C8	 	
+# 0xF8C9	 	
+# 0xF8CA	 	
+# 0xF8CB	 	
+# 0xF8CC	 	
+# 0xF8CD	 	
+# 0xF8CE	 	
+# 0xF8CF	 	
+# 0xF8D0	 	
+# 0xF8D1	 	
+# 0xF8D2	 	
+# 0xF8D3	 	
+# 0xF8D4	 	
+# 0xF8D5	 	
+# 0xF8D6	 	
+# 0xF8D7	 	
+# 0xF8D8	 	
+# 0xF8D9	 	
+# 0xF8DA	 	
+# 0xF8DB	 	
+# 0xF8DC	 	
+# 0xF8DD	 	
+# 0xF8DE	 	
+# 0xF8DF	 	
+# 0xF8E0	 	
+# 0xF8E1	 	
+# 0xF8E2	 	
+# 0xF8E3	 	
+# 0xF8E4	 	
+# 0xF8E5	 	
+# 0xF8E6	 	
+# 0xF8E7	 	
+# 0xF8E8	 	
+# 0xF8E9	 	
+# 0xF8EA	 	
+# 0xF8EB	 	
+# 0xF8EC	 	
+# 0xF8ED	 	
+# 0xF8EE	 	
+# 0xF8EF	 	
+# 0xF8F0	 	
+# 0xF8F1	 	
+# 0xF8F2	 	
+# 0xF8F3	 	
+# 0xF8F4	 	
+# 0xF8F5	 	
+# 0xF8F6	 	
+# 0xF8F7	 	
+# 0xF8F8	 	
+# 0xF8F9	 	
+# 0xF8FA	 	
+# 0xF8FB	 	
+# 0xF8FC	 	
+# 0xF8FD	 	
+# 0xF8FE	 	
+0xF940	0x9D43	
+0xF941	0x9D44	
+0xF942	0x9D45	
+0xF943	0x9D46	
+0xF944	0x9D47	
+0xF945	0x9D48	
+0xF946	0x9D49	
+0xF947	0x9D4A	
+0xF948	0x9D4B	
+0xF949	0x9D4C	
+0xF94A	0x9D4D	
+0xF94B	0x9D4E	
+0xF94C	0x9D4F	
+0xF94D	0x9D50	
+0xF94E	0x9D51	
+0xF94F	0x9D52	
+0xF950	0x9D53	
+0xF951	0x9D54	
+0xF952	0x9D55	
+0xF953	0x9D56	
+0xF954	0x9D57	
+0xF955	0x9D58	
+0xF956	0x9D59	
+0xF957	0x9D5A	
+0xF958	0x9D5B	
+0xF959	0x9D5C	
+0xF95A	0x9D5D	
+0xF95B	0x9D5E	
+0xF95C	0x9D5F	
+0xF95D	0x9D60	
+0xF95E	0x9D61	
+0xF95F	0x9D62	
+0xF960	0x9D63	
+0xF961	0x9D64	
+0xF962	0x9D65	
+0xF963	0x9D66	
+0xF964	0x9D67	
+0xF965	0x9D68	
+0xF966	0x9D69	
+0xF967	0x9D6A	
+0xF968	0x9D6B	
+0xF969	0x9D6C	
+0xF96A	0x9D6D	
+0xF96B	0x9D6E	
+0xF96C	0x9D6F	
+0xF96D	0x9D70	
+0xF96E	0x9D71	
+0xF96F	0x9D72	
+0xF970	0x9D73	
+0xF971	0x9D74	
+0xF972	0x9D75	
+0xF973	0x9D76	
+0xF974	0x9D77	
+0xF975	0x9D78	
+0xF976	0x9D79	
+0xF977	0x9D7A	
+0xF978	0x9D7B	
+0xF979	0x9D7C	
+0xF97A	0x9D7D	
+0xF97B	0x9D7E	
+0xF97C	0x9D7F	
+0xF97D	0x9D80	
+0xF97E	0x9D81	
+0xF980	0x9D82	
+0xF981	0x9D83	
+0xF982	0x9D84	
+0xF983	0x9D85	
+0xF984	0x9D86	
+0xF985	0x9D87	
+0xF986	0x9D88	
+0xF987	0x9D89	
+0xF988	0x9D8A	
+0xF989	0x9D8B	
+0xF98A	0x9D8C	
+0xF98B	0x9D8D	
+0xF98C	0x9D8E	
+0xF98D	0x9D8F	
+0xF98E	0x9D90	
+0xF98F	0x9D91	
+0xF990	0x9D92	
+0xF991	0x9D93	
+0xF992	0x9D94	
+0xF993	0x9D95	
+0xF994	0x9D96	
+0xF995	0x9D97	
+0xF996	0x9D98	
+0xF997	0x9D99	
+0xF998	0x9D9A	
+0xF999	0x9D9B	
+0xF99A	0x9D9C	
+0xF99B	0x9D9D	
+0xF99C	0x9D9E	
+0xF99D	0x9D9F	
+0xF99E	0x9DA0	
+0xF99F	0x9DA1	
+0xF9A0	0x9DA2	
+# 0xF9A1	 	
+# 0xF9A2	 	
+# 0xF9A3	 	
+# 0xF9A4	 	
+# 0xF9A5	 	
+# 0xF9A6	 	
+# 0xF9A7	 	
+# 0xF9A8	 	
+# 0xF9A9	 	
+# 0xF9AA	 	
+# 0xF9AB	 	
+# 0xF9AC	 	
+# 0xF9AD	 	
+# 0xF9AE	 	
+# 0xF9AF	 	
+# 0xF9B0	 	
+# 0xF9B1	 	
+# 0xF9B2	 	
+# 0xF9B3	 	
+# 0xF9B4	 	
+# 0xF9B5	 	
+# 0xF9B6	 	
+# 0xF9B7	 	
+# 0xF9B8	 	
+# 0xF9B9	 	
+# 0xF9BA	 	
+# 0xF9BB	 	
+# 0xF9BC	 	
+# 0xF9BD	 	
+# 0xF9BE	 	
+# 0xF9BF	 	
+# 0xF9C0	 	
+# 0xF9C1	 	
+# 0xF9C2	 	
+# 0xF9C3	 	
+# 0xF9C4	 	
+# 0xF9C5	 	
+# 0xF9C6	 	
+# 0xF9C7	 	
+# 0xF9C8	 	
+# 0xF9C9	 	
+# 0xF9CA	 	
+# 0xF9CB	 	
+# 0xF9CC	 	
+# 0xF9CD	 	
+# 0xF9CE	 	
+# 0xF9CF	 	
+# 0xF9D0	 	
+# 0xF9D1	 	
+# 0xF9D2	 	
+# 0xF9D3	 	
+# 0xF9D4	 	
+# 0xF9D5	 	
+# 0xF9D6	 	
+# 0xF9D7	 	
+# 0xF9D8	 	
+# 0xF9D9	 	
+# 0xF9DA	 	
+# 0xF9DB	 	
+# 0xF9DC	 	
+# 0xF9DD	 	
+# 0xF9DE	 	
+# 0xF9DF	 	
+# 0xF9E0	 	
+# 0xF9E1	 	
+# 0xF9E2	 	
+# 0xF9E3	 	
+# 0xF9E4	 	
+# 0xF9E5	 	
+# 0xF9E6	 	
+# 0xF9E7	 	
+# 0xF9E8	 	
+# 0xF9E9	 	
+# 0xF9EA	 	
+# 0xF9EB	 	
+# 0xF9EC	 	
+# 0xF9ED	 	
+# 0xF9EE	 	
+# 0xF9EF	 	
+# 0xF9F0	 	
+# 0xF9F1	 	
+# 0xF9F2	 	
+# 0xF9F3	 	
+# 0xF9F4	 	
+# 0xF9F5	 	
+# 0xF9F6	 	
+# 0xF9F7	 	
+# 0xF9F8	 	
+# 0xF9F9	 	
+# 0xF9FA	 	
+# 0xF9FB	 	
+# 0xF9FC	 	
+# 0xF9FD	 	
+# 0xF9FE	 	
+0xFA40	0x9DA3	
+0xFA41	0x9DA4	
+0xFA42	0x9DA5	
+0xFA43	0x9DA6	
+0xFA44	0x9DA7	
+0xFA45	0x9DA8	
+0xFA46	0x9DA9	
+0xFA47	0x9DAA	
+0xFA48	0x9DAB	
+0xFA49	0x9DAC	
+0xFA4A	0x9DAD	
+0xFA4B	0x9DAE	
+0xFA4C	0x9DAF	
+0xFA4D	0x9DB0	
+0xFA4E	0x9DB1	
+0xFA4F	0x9DB2	
+0xFA50	0x9DB3	
+0xFA51	0x9DB4	
+0xFA52	0x9DB5	
+0xFA53	0x9DB6	
+0xFA54	0x9DB7	
+0xFA55	0x9DB8	
+0xFA56	0x9DB9	
+0xFA57	0x9DBA	
+0xFA58	0x9DBB	
+0xFA59	0x9DBC	
+0xFA5A	0x9DBD	
+0xFA5B	0x9DBE	
+0xFA5C	0x9DBF	
+0xFA5D	0x9DC0	
+0xFA5E	0x9DC1	
+0xFA5F	0x9DC2	
+0xFA60	0x9DC3	
+0xFA61	0x9DC4	
+0xFA62	0x9DC5	
+0xFA63	0x9DC6	
+0xFA64	0x9DC7	
+0xFA65	0x9DC8	
+0xFA66	0x9DC9	
+0xFA67	0x9DCA	
+0xFA68	0x9DCB	
+0xFA69	0x9DCC	
+0xFA6A	0x9DCD	
+0xFA6B	0x9DCE	
+0xFA6C	0x9DCF	
+0xFA6D	0x9DD0	
+0xFA6E	0x9DD1	
+0xFA6F	0x9DD2	
+0xFA70	0x9DD3	
+0xFA71	0x9DD4	
+0xFA72	0x9DD5	
+0xFA73	0x9DD6	
+0xFA74	0x9DD7	
+0xFA75	0x9DD8	
+0xFA76	0x9DD9	
+0xFA77	0x9DDA	
+0xFA78	0x9DDB	
+0xFA79	0x9DDC	
+0xFA7A	0x9DDD	
+0xFA7B	0x9DDE	
+0xFA7C	0x9DDF	
+0xFA7D	0x9DE0	
+0xFA7E	0x9DE1	
+0xFA80	0x9DE2	
+0xFA81	0x9DE3	
+0xFA82	0x9DE4	
+0xFA83	0x9DE5	
+0xFA84	0x9DE6	
+0xFA85	0x9DE7	
+0xFA86	0x9DE8	
+0xFA87	0x9DE9	
+0xFA88	0x9DEA	
+0xFA89	0x9DEB	
+0xFA8A	0x9DEC	
+0xFA8B	0x9DED	
+0xFA8C	0x9DEE	
+0xFA8D	0x9DEF	
+0xFA8E	0x9DF0	
+0xFA8F	0x9DF1	
+0xFA90	0x9DF2	
+0xFA91	0x9DF3	
+0xFA92	0x9DF4	
+0xFA93	0x9DF5	
+0xFA94	0x9DF6	
+0xFA95	0x9DF7	
+0xFA96	0x9DF8	
+0xFA97	0x9DF9	
+0xFA98	0x9DFA	
+0xFA99	0x9DFB	
+0xFA9A	0x9DFC	
+0xFA9B	0x9DFD	
+0xFA9C	0x9DFE	
+0xFA9D	0x9DFF	
+0xFA9E	0x9E00	
+0xFA9F	0x9E01	
+0xFAA0	0x9E02	
+# 0xFAA1	 	
+# 0xFAA2	 	
+# 0xFAA3	 	
+# 0xFAA4	 	
+# 0xFAA5	 	
+# 0xFAA6	 	
+# 0xFAA7	 	
+# 0xFAA8	 	
+# 0xFAA9	 	
+# 0xFAAA	 	
+# 0xFAAB	 	
+# 0xFAAC	 	
+# 0xFAAD	 	
+# 0xFAAE	 	
+# 0xFAAF	 	
+# 0xFAB0	 	
+# 0xFAB1	 	
+# 0xFAB2	 	
+# 0xFAB3	 	
+# 0xFAB4	 	
+# 0xFAB5	 	
+# 0xFAB6	 	
+# 0xFAB7	 	
+# 0xFAB8	 	
+# 0xFAB9	 	
+# 0xFABA	 	
+# 0xFABB	 	
+# 0xFABC	 	
+# 0xFABD	 	
+# 0xFABE	 	
+# 0xFABF	 	
+# 0xFAC0	 	
+# 0xFAC1	 	
+# 0xFAC2	 	
+# 0xFAC3	 	
+# 0xFAC4	 	
+# 0xFAC5	 	
+# 0xFAC6	 	
+# 0xFAC7	 	
+# 0xFAC8	 	
+# 0xFAC9	 	
+# 0xFACA	 	
+# 0xFACB	 	
+# 0xFACC	 	
+# 0xFACD	 	
+# 0xFACE	 	
+# 0xFACF	 	
+# 0xFAD0	 	
+# 0xFAD1	 	
+# 0xFAD2	 	
+# 0xFAD3	 	
+# 0xFAD4	 	
+# 0xFAD5	 	
+# 0xFAD6	 	
+# 0xFAD7	 	
+# 0xFAD8	 	
+# 0xFAD9	 	
+# 0xFADA	 	
+# 0xFADB	 	
+# 0xFADC	 	
+# 0xFADD	 	
+# 0xFADE	 	
+# 0xFADF	 	
+# 0xFAE0	 	
+# 0xFAE1	 	
+# 0xFAE2	 	
+# 0xFAE3	 	
+# 0xFAE4	 	
+# 0xFAE5	 	
+# 0xFAE6	 	
+# 0xFAE7	 	
+# 0xFAE8	 	
+# 0xFAE9	 	
+# 0xFAEA	 	
+# 0xFAEB	 	
+# 0xFAEC	 	
+# 0xFAED	 	
+# 0xFAEE	 	
+# 0xFAEF	 	
+# 0xFAF0	 	
+# 0xFAF1	 	
+# 0xFAF2	 	
+# 0xFAF3	 	
+# 0xFAF4	 	
+# 0xFAF5	 	
+# 0xFAF6	 	
+# 0xFAF7	 	
+# 0xFAF8	 	
+# 0xFAF9	 	
+# 0xFAFA	 	
+# 0xFAFB	 	
+# 0xFAFC	 	
+# 0xFAFD	 	
+# 0xFAFE	 	
+0xFB40	0x9E03	
+0xFB41	0x9E04	
+0xFB42	0x9E05	
+0xFB43	0x9E06	
+0xFB44	0x9E07	
+0xFB45	0x9E08	
+0xFB46	0x9E09	
+0xFB47	0x9E0A	
+0xFB48	0x9E0B	
+0xFB49	0x9E0C	
+0xFB4A	0x9E0D	
+0xFB4B	0x9E0E	
+0xFB4C	0x9E0F	
+0xFB4D	0x9E10	
+0xFB4E	0x9E11	
+0xFB4F	0x9E12	
+0xFB50	0x9E13	
+0xFB51	0x9E14	
+0xFB52	0x9E15	
+0xFB53	0x9E16	
+0xFB54	0x9E17	
+0xFB55	0x9E18	
+0xFB56	0x9E19	
+0xFB57	0x9E1A	
+0xFB58	0x9E1B	
+0xFB59	0x9E1C	
+0xFB5A	0x9E1D	
+0xFB5B	0x9E1E	
+0xFB5C	0x9E24	
+0xFB5D	0x9E27	
+0xFB5E	0x9E2E	
+0xFB5F	0x9E30	
+0xFB60	0x9E34	
+0xFB61	0x9E3B	
+0xFB62	0x9E3C	
+0xFB63	0x9E40	
+0xFB64	0x9E4D	
+0xFB65	0x9E50	
+0xFB66	0x9E52	
+0xFB67	0x9E53	
+0xFB68	0x9E54	
+0xFB69	0x9E56	
+0xFB6A	0x9E59	
+0xFB6B	0x9E5D	
+0xFB6C	0x9E5F	
+0xFB6D	0x9E60	
+0xFB6E	0x9E61	
+0xFB6F	0x9E62	
+0xFB70	0x9E65	
+0xFB71	0x9E6E	
+0xFB72	0x9E6F	
+0xFB73	0x9E72	
+0xFB74	0x9E74	
+0xFB75	0x9E75	
+0xFB76	0x9E76	
+0xFB77	0x9E77	
+0xFB78	0x9E78	
+0xFB79	0x9E79	
+0xFB7A	0x9E7A	
+0xFB7B	0x9E7B	
+0xFB7C	0x9E7C	
+0xFB7D	0x9E7D	
+0xFB7E	0x9E80	
+0xFB80	0x9E81	
+0xFB81	0x9E83	
+0xFB82	0x9E84	
+0xFB83	0x9E85	
+0xFB84	0x9E86	
+0xFB85	0x9E89	
+0xFB86	0x9E8A	
+0xFB87	0x9E8C	
+0xFB88	0x9E8D	
+0xFB89	0x9E8E	
+0xFB8A	0x9E8F	
+0xFB8B	0x9E90	
+0xFB8C	0x9E91	
+0xFB8D	0x9E94	
+0xFB8E	0x9E95	
+0xFB8F	0x9E96	
+0xFB90	0x9E97	
+0xFB91	0x9E98	
+0xFB92	0x9E99	
+0xFB93	0x9E9A	
+0xFB94	0x9E9B	
+0xFB95	0x9E9C	
+0xFB96	0x9E9E	
+0xFB97	0x9EA0	
+0xFB98	0x9EA1	
+0xFB99	0x9EA2	
+0xFB9A	0x9EA3	
+0xFB9B	0x9EA4	
+0xFB9C	0x9EA5	
+0xFB9D	0x9EA7	
+0xFB9E	0x9EA8	
+0xFB9F	0x9EA9	
+0xFBA0	0x9EAA	
+# 0xFBA1	 	
+# 0xFBA2	 	
+# 0xFBA3	 	
+# 0xFBA4	 	
+# 0xFBA5	 	
+# 0xFBA6	 	
+# 0xFBA7	 	
+# 0xFBA8	 	
+# 0xFBA9	 	
+# 0xFBAA	 	
+# 0xFBAB	 	
+# 0xFBAC	 	
+# 0xFBAD	 	
+# 0xFBAE	 	
+# 0xFBAF	 	
+# 0xFBB0	 	
+# 0xFBB1	 	
+# 0xFBB2	 	
+# 0xFBB3	 	
+# 0xFBB4	 	
+# 0xFBB5	 	
+# 0xFBB6	 	
+# 0xFBB7	 	
+# 0xFBB8	 	
+# 0xFBB9	 	
+# 0xFBBA	 	
+# 0xFBBB	 	
+# 0xFBBC	 	
+# 0xFBBD	 	
+# 0xFBBE	 	
+# 0xFBBF	 	
+# 0xFBC0	 	
+# 0xFBC1	 	
+# 0xFBC2	 	
+# 0xFBC3	 	
+# 0xFBC4	 	
+# 0xFBC5	 	
+# 0xFBC6	 	
+# 0xFBC7	 	
+# 0xFBC8	 	
+# 0xFBC9	 	
+# 0xFBCA	 	
+# 0xFBCB	 	
+# 0xFBCC	 	
+# 0xFBCD	 	
+# 0xFBCE	 	
+# 0xFBCF	 	
+# 0xFBD0	 	
+# 0xFBD1	 	
+# 0xFBD2	 	
+# 0xFBD3	 	
+# 0xFBD4	 	
+# 0xFBD5	 	
+# 0xFBD6	 	
+# 0xFBD7	 	
+# 0xFBD8	 	
+# 0xFBD9	 	
+# 0xFBDA	 	
+# 0xFBDB	 	
+# 0xFBDC	 	
+# 0xFBDD	 	
+# 0xFBDE	 	
+# 0xFBDF	 	
+# 0xFBE0	 	
+# 0xFBE1	 	
+# 0xFBE2	 	
+# 0xFBE3	 	
+# 0xFBE4	 	
+# 0xFBE5	 	
+# 0xFBE6	 	
+# 0xFBE7	 	
+# 0xFBE8	 	
+# 0xFBE9	 	
+# 0xFBEA	 	
+# 0xFBEB	 	
+# 0xFBEC	 	
+# 0xFBED	 	
+# 0xFBEE	 	
+# 0xFBEF	 	
+# 0xFBF0	 	
+# 0xFBF1	 	
+# 0xFBF2	 	
+# 0xFBF3	 	
+# 0xFBF4	 	
+# 0xFBF5	 	
+# 0xFBF6	 	
+# 0xFBF7	 	
+# 0xFBF8	 	
+# 0xFBF9	 	
+# 0xFBFA	 	
+# 0xFBFB	 	
+# 0xFBFC	 	
+# 0xFBFD	 	
+# 0xFBFE	 	
+0xFC40	0x9EAB	
+0xFC41	0x9EAC	
+0xFC42	0x9EAD	
+0xFC43	0x9EAE	
+0xFC44	0x9EAF	
+0xFC45	0x9EB0	
+0xFC46	0x9EB1	
+0xFC47	0x9EB2	
+0xFC48	0x9EB3	
+0xFC49	0x9EB5	
+0xFC4A	0x9EB6	
+0xFC4B	0x9EB7	
+0xFC4C	0x9EB9	
+0xFC4D	0x9EBA	
+0xFC4E	0x9EBC	
+0xFC4F	0x9EBF	
+0xFC50	0x9EC0	
+0xFC51	0x9EC1	
+0xFC52	0x9EC2	
+0xFC53	0x9EC3	
+0xFC54	0x9EC5	
+0xFC55	0x9EC6	
+0xFC56	0x9EC7	
+0xFC57	0x9EC8	
+0xFC58	0x9ECA	
+0xFC59	0x9ECB	
+0xFC5A	0x9ECC	
+0xFC5B	0x9ED0	
+0xFC5C	0x9ED2	
+0xFC5D	0x9ED3	
+0xFC5E	0x9ED5	
+0xFC5F	0x9ED6	
+0xFC60	0x9ED7	
+0xFC61	0x9ED9	
+0xFC62	0x9EDA	
+0xFC63	0x9EDE	
+0xFC64	0x9EE1	
+0xFC65	0x9EE3	
+0xFC66	0x9EE4	
+0xFC67	0x9EE6	
+0xFC68	0x9EE8	
+0xFC69	0x9EEB	
+0xFC6A	0x9EEC	
+0xFC6B	0x9EED	
+0xFC6C	0x9EEE	
+0xFC6D	0x9EF0	
+0xFC6E	0x9EF1	
+0xFC6F	0x9EF2	
+0xFC70	0x9EF3	
+0xFC71	0x9EF4	
+0xFC72	0x9EF5	
+0xFC73	0x9EF6	
+0xFC74	0x9EF7	
+0xFC75	0x9EF8	
+0xFC76	0x9EFA	
+0xFC77	0x9EFD	
+0xFC78	0x9EFF	
+0xFC79	0x9F00	
+0xFC7A	0x9F01	
+0xFC7B	0x9F02	
+0xFC7C	0x9F03	
+0xFC7D	0x9F04	
+0xFC7E	0x9F05	
+0xFC80	0x9F06	
+0xFC81	0x9F07	
+0xFC82	0x9F08	
+0xFC83	0x9F09	
+0xFC84	0x9F0A	
+0xFC85	0x9F0C	
+0xFC86	0x9F0F	
+0xFC87	0x9F11	
+0xFC88	0x9F12	
+0xFC89	0x9F14	
+0xFC8A	0x9F15	
+0xFC8B	0x9F16	
+0xFC8C	0x9F18	
+0xFC8D	0x9F1A	
+0xFC8E	0x9F1B	
+0xFC8F	0x9F1C	
+0xFC90	0x9F1D	
+0xFC91	0x9F1E	
+0xFC92	0x9F1F	
+0xFC93	0x9F21	
+0xFC94	0x9F23	
+0xFC95	0x9F24	
+0xFC96	0x9F25	
+0xFC97	0x9F26	
+0xFC98	0x9F27	
+0xFC99	0x9F28	
+0xFC9A	0x9F29	
+0xFC9B	0x9F2A	
+0xFC9C	0x9F2B	
+0xFC9D	0x9F2D	
+0xFC9E	0x9F2E	
+0xFC9F	0x9F30	
+0xFCA0	0x9F31	
+# 0xFCA1	 	
+# 0xFCA2	 	
+# 0xFCA3	 	
+# 0xFCA4	 	
+# 0xFCA5	 	
+# 0xFCA6	 	
+# 0xFCA7	 	
+# 0xFCA8	 	
+# 0xFCA9	 	
+# 0xFCAA	 	
+# 0xFCAB	 	
+# 0xFCAC	 	
+# 0xFCAD	 	
+# 0xFCAE	 	
+# 0xFCAF	 	
+# 0xFCB0	 	
+# 0xFCB1	 	
+# 0xFCB2	 	
+# 0xFCB3	 	
+# 0xFCB4	 	
+# 0xFCB5	 	
+# 0xFCB6	 	
+# 0xFCB7	 	
+# 0xFCB8	 	
+# 0xFCB9	 	
+# 0xFCBA	 	
+# 0xFCBB	 	
+# 0xFCBC	 	
+# 0xFCBD	 	
+# 0xFCBE	 	
+# 0xFCBF	 	
+# 0xFCC0	 	
+# 0xFCC1	 	
+# 0xFCC2	 	
+# 0xFCC3	 	
+# 0xFCC4	 	
+# 0xFCC5	 	
+# 0xFCC6	 	
+# 0xFCC7	 	
+# 0xFCC8	 	
+# 0xFCC9	 	
+# 0xFCCA	 	
+# 0xFCCB	 	
+# 0xFCCC	 	
+# 0xFCCD	 	
+# 0xFCCE	 	
+# 0xFCCF	 	
+# 0xFCD0	 	
+# 0xFCD1	 	
+# 0xFCD2	 	
+# 0xFCD3	 	
+# 0xFCD4	 	
+# 0xFCD5	 	
+# 0xFCD6	 	
+# 0xFCD7	 	
+# 0xFCD8	 	
+# 0xFCD9	 	
+# 0xFCDA	 	
+# 0xFCDB	 	
+# 0xFCDC	 	
+# 0xFCDD	 	
+# 0xFCDE	 	
+# 0xFCDF	 	
+# 0xFCE0	 	
+# 0xFCE1	 	
+# 0xFCE2	 	
+# 0xFCE3	 	
+# 0xFCE4	 	
+# 0xFCE5	 	
+# 0xFCE6	 	
+# 0xFCE7	 	
+# 0xFCE8	 	
+# 0xFCE9	 	
+# 0xFCEA	 	
+# 0xFCEB	 	
+# 0xFCEC	 	
+# 0xFCED	 	
+# 0xFCEE	 	
+# 0xFCEF	 	
+# 0xFCF0	 	
+# 0xFCF1	 	
+# 0xFCF2	 	
+# 0xFCF3	 	
+# 0xFCF4	 	
+# 0xFCF5	 	
+# 0xFCF6	 	
+# 0xFCF7	 	
+# 0xFCF8	 	
+# 0xFCF9	 	
+# 0xFCFA	 	
+# 0xFCFB	 	
+# 0xFCFC	 	
+# 0xFCFD	 	
+# 0xFCFE	 	
+0xFD40	0x9F32	
+0xFD41	0x9F33	
+0xFD42	0x9F34	
+0xFD43	0x9F35	
+0xFD44	0x9F36	
+0xFD45	0x9F38	
+0xFD46	0x9F3A	
+0xFD47	0x9F3C	
+0xFD48	0x9F3F	
+0xFD49	0x9F40	
+0xFD4A	0x9F41	
+0xFD4B	0x9F42	
+0xFD4C	0x9F43	
+0xFD4D	0x9F45	
+0xFD4E	0x9F46	
+0xFD4F	0x9F47	
+0xFD50	0x9F48	
+0xFD51	0x9F49	
+0xFD52	0x9F4A	
+0xFD53	0x9F4B	
+0xFD54	0x9F4C	
+0xFD55	0x9F4D	
+0xFD56	0x9F4E	
+0xFD57	0x9F4F	
+0xFD58	0x9F52	
+0xFD59	0x9F53	
+0xFD5A	0x9F54	
+0xFD5B	0x9F55	
+0xFD5C	0x9F56	
+0xFD5D	0x9F57	
+0xFD5E	0x9F58	
+0xFD5F	0x9F59	
+0xFD60	0x9F5A	
+0xFD61	0x9F5B	
+0xFD62	0x9F5C	
+0xFD63	0x9F5D	
+0xFD64	0x9F5E	
+0xFD65	0x9F5F	
+0xFD66	0x9F60	
+0xFD67	0x9F61	
+0xFD68	0x9F62	
+0xFD69	0x9F63	
+0xFD6A	0x9F64	
+0xFD6B	0x9F65	
+0xFD6C	0x9F66	
+0xFD6D	0x9F67	
+0xFD6E	0x9F68	
+0xFD6F	0x9F69	
+0xFD70	0x9F6A	
+0xFD71	0x9F6B	
+0xFD72	0x9F6C	
+0xFD73	0x9F6D	
+0xFD74	0x9F6E	
+0xFD75	0x9F6F	
+0xFD76	0x9F70	
+0xFD77	0x9F71	
+0xFD78	0x9F72	
+0xFD79	0x9F73	
+0xFD7A	0x9F74	
+0xFD7B	0x9F75	
+0xFD7C	0x9F76	
+0xFD7D	0x9F77	
+0xFD7E	0x9F78	
+0xFD80	0x9F79	
+0xFD81	0x9F7A	
+0xFD82	0x9F7B	
+0xFD83	0x9F7C	
+0xFD84	0x9F7D	
+0xFD85	0x9F7E	
+0xFD86	0x9F81	
+0xFD87	0x9F82	
+0xFD88	0x9F8D	
+0xFD89	0x9F8E	
+0xFD8A	0x9F8F	
+0xFD8B	0x9F90	
+0xFD8C	0x9F91	
+0xFD8D	0x9F92	
+0xFD8E	0x9F93	
+0xFD8F	0x9F94	
+0xFD90	0x9F95	
+0xFD91	0x9F96	
+0xFD92	0x9F97	
+0xFD93	0x9F98	
+0xFD94	0x9F9C	
+0xFD95	0x9F9D	
+0xFD96	0x9F9E	
+0xFD97	0x9FA1	
+0xFD98	0x9FA2	
+0xFD99	0x9FA3	
+0xFD9A	0x9FA4	
+0xFD9B	0x9FA5	
+0xFD9C	0xF92C	
+0xFD9D	0xF979	
+0xFD9E	0xF995	
+0xFD9F	0xF9E7	
+0xFDA0	0xF9F1	
+# 0xFDA1	 	
+# 0xFDA2	 	
+# 0xFDA3	 	
+# 0xFDA4	 	
+# 0xFDA5	 	
+# 0xFDA6	 	
+# 0xFDA7	 	
+# 0xFDA8	 	
+# 0xFDA9	 	
+# 0xFDAA	 	
+# 0xFDAB	 	
+# 0xFDAC	 	
+# 0xFDAD	 	
+# 0xFDAE	 	
+# 0xFDAF	 	
+# 0xFDB0	 	
+# 0xFDB1	 	
+# 0xFDB2	 	
+# 0xFDB3	 	
+# 0xFDB4	 	
+# 0xFDB5	 	
+# 0xFDB6	 	
+# 0xFDB7	 	
+# 0xFDB8	 	
+# 0xFDB9	 	
+# 0xFDBA	 	
+# 0xFDBB	 	
+# 0xFDBC	 	
+# 0xFDBD	 	
+# 0xFDBE	 	
+# 0xFDBF	 	
+# 0xFDC0	 	
+# 0xFDC1	 	
+# 0xFDC2	 	
+# 0xFDC3	 	
+# 0xFDC4	 	
+# 0xFDC5	 	
+# 0xFDC6	 	
+# 0xFDC7	 	
+# 0xFDC8	 	
+# 0xFDC9	 	
+# 0xFDCA	 	
+# 0xFDCB	 	
+# 0xFDCC	 	
+# 0xFDCD	 	
+# 0xFDCE	 	
+# 0xFDCF	 	
+# 0xFDD0	 	
+# 0xFDD1	 	
+# 0xFDD2	 	
+# 0xFDD3	 	
+# 0xFDD4	 	
+# 0xFDD5	 	
+# 0xFDD6	 	
+# 0xFDD7	 	
+# 0xFDD8	 	
+# 0xFDD9	 	
+# 0xFDDA	 	
+# 0xFDDB	 	
+# 0xFDDC	 	
+# 0xFDDD	 	
+# 0xFDDE	 	
+# 0xFDDF	 	
+# 0xFDE0	 	
+# 0xFDE1	 	
+# 0xFDE2	 	
+# 0xFDE3	 	
+# 0xFDE4	 	
+# 0xFDE5	 	
+# 0xFDE6	 	
+# 0xFDE7	 	
+# 0xFDE8	 	
+# 0xFDE9	 	
+# 0xFDEA	 	
+# 0xFDEB	 	
+# 0xFDEC	 	
+# 0xFDED	 	
+# 0xFDEE	 	
+# 0xFDEF	 	
+# 0xFDF0	 	
+# 0xFDF1	 	
+# 0xFDF2	 	
+# 0xFDF3	 	
+# 0xFDF4	 	
+# 0xFDF5	 	
+# 0xFDF6	 	
+# 0xFDF7	 	
+# 0xFDF8	 	
+# 0xFDF9	 	
+# 0xFDFA	 	
+# 0xFDFB	 	
+# 0xFDFC	 	
+# 0xFDFD	 	
+# 0xFDFE	 	
+0xFE40	0xFA0C	
+0xFE41	0xFA0D	
+0xFE42	0xFA0E	
+0xFE43	0xFA0F	
+0xFE44	0xFA11	
+0xFE45	0xFA13	
+0xFE46	0xFA14	
+0xFE47	0xFA18	
+0xFE48	0xFA1F	
+0xFE49	0xFA20	
+0xFE4A	0xFA21	
+0xFE4B	0xFA23	
+0xFE4C	0xFA24	
+0xFE4D	0xFA27	
+0xFE4E	0xFA28	
+0xFE4F	0xFA29	
+0xFE50	0xE815	
+0xFE51	0xE816	
+0xFE52	0xE817	
+0xFE53	0xE818	
+0xFE54	0xE819	
+0xFE55	0xE81A	
+0xFE56	0xE81B	
+0xFE57	0xE81C	
+0xFE58	0xE81D	
+0xFE59	0xE81E	
+0xFE5A	0xE81F	
+0xFE5B	0xE820	
+0xFE5C	0xE821	
+0xFE5D	0xE822	
+0xFE5E	0xE823	
+0xFE5F	0xE824	
+0xFE60	0xE825	
+0xFE61	0xE826	
+0xFE62	0xE827	
+0xFE63	0xE828	
+0xFE64	0xE829	
+0xFE65	0xE82A	
+0xFE66	0xE82B	
+0xFE67	0xE82C	
+0xFE68	0xE82D	
+0xFE69	0xE82E	
+0xFE6A	0xE82F	
+0xFE6B	0xE830	
+0xFE6C	0xE831	
+0xFE6D	0xE832	
+0xFE6E	0xE833	
+0xFE6F	0xE834	
+0xFE70	0xE835	
+0xFE71	0xE836	
+0xFE72	0xE837	
+0xFE73	0xE838	
+0xFE74	0xE839	
+0xFE75	0xE83A	
+0xFE76	0xE83B	
+0xFE77	0xE83C	
+0xFE78	0xE83D	
+0xFE79	0xE83E	
+0xFE7A	0xE83F	
+0xFE7B	0xE840	
+0xFE7C	0xE841	
+0xFE7D	0xE842	
+0xFE7E	0xE843	
+0xFE80	0xE844	
+0xFE81	0xE845	
+0xFE82	0xE846	
+0xFE83	0xE847	
+0xFE84	0xE848	
+0xFE85	0xE849	
+0xFE86	0xE84A	
+0xFE87	0xE84B	
+0xFE88	0xE84C	
+0xFE89	0xE84D	
+0xFE8A	0xE84E	
+0xFE8B	0xE84F	
+0xFE8C	0xE850	
+0xFE8D	0xE851	
+0xFE8E	0xE852	
+0xFE8F	0xE853	
+0xFE90	0xE854	
+0xFE91	0xE855	
+0xFE92	0xE856	
+0xFE93	0xE857	
+0xFE94	0xE858	
+0xFE95	0xE859	
+0xFE96	0xE85A	
+0xFE97	0xE85B	
+0xFE98	0xE85C	
+0xFE99	0xE85D	
+0xFE9A	0xE85E	
+0xFE9B	0xE85F	
+0xFE9C	0xE860	
+0xFE9D	0xE861	
+0xFE9E	0xE862	
+0xFE9F	0xE863	
+0xFEA0	0xE864	
+# 0xFEA1	 	
+# 0xFEA2	 	
+# 0xFEA3	 	
+# 0xFEA4	 	
+# 0xFEA5	 	
+# 0xFEA6	 	
+# 0xFEA7	 	
+# 0xFEA8	 	
+# 0xFEA9	 	
+# 0xFEAA	 	
+# 0xFEAB	 	
+# 0xFEAC	 	
+# 0xFEAD	 	
+# 0xFEAE	 	
+# 0xFEAF	 	
+# 0xFEB0	 	
+# 0xFEB1	 	
+# 0xFEB2	 	
+# 0xFEB3	 	
+# 0xFEB4	 	
+# 0xFEB5	 	
+# 0xFEB6	 	
+# 0xFEB7	 	
+# 0xFEB8	 	
+# 0xFEB9	 	
+# 0xFEBA	 	
+# 0xFEBB	 	
+# 0xFEBC	 	
+# 0xFEBD	 	
+# 0xFEBE	 	
+# 0xFEBF	 	
+# 0xFEC0	 	
+# 0xFEC1	 	
+# 0xFEC2	 	
+# 0xFEC3	 	
+# 0xFEC4	 	
+# 0xFEC5	 	
+# 0xFEC6	 	
+# 0xFEC7	 	
+# 0xFEC8	 	
+# 0xFEC9	 	
+# 0xFECA	 	
+# 0xFECB	 	
+# 0xFECC	 	
+# 0xFECD	 	
+# 0xFECE	 	
+# 0xFECF	 	
+# 0xFED0	 	
+# 0xFED1	 	
+# 0xFED2	 	
+# 0xFED3	 	
+# 0xFED4	 	
+# 0xFED5	 	
+# 0xFED6	 	
+# 0xFED7	 	
+# 0xFED8	 	
+# 0xFED9	 	
+# 0xFEDA	 	
+# 0xFEDB	 	
+# 0xFEDC	 	
+# 0xFEDD	 	
+# 0xFEDE	 	
+# 0xFEDF	 	
+# 0xFEE0	 	
+# 0xFEE1	 	
+# 0xFEE2	 	
+# 0xFEE3	 	
+# 0xFEE4	 	
+# 0xFEE5	 	
+# 0xFEE6	 	
+# 0xFEE7	 	
+# 0xFEE8	 	
+# 0xFEE9	 	
+# 0xFEEA	 	
+# 0xFEEB	 	
+# 0xFEEC	 	
+# 0xFEED	 	
+# 0xFEEE	 	
+# 0xFEEF	 	
+# 0xFEF0	 	
+# 0xFEF1	 	
+# 0xFEF2	 	
+# 0xFEF3	 	
+# 0xFEF4	 	
+# 0xFEF5	 	
+# 0xFEF6	 	
+# 0xFEF7	 	
+# 0xFEF8	 	
+# 0xFEF9	 	
+# 0xFEFA	 	
+# 0xFEFB	 	
+# 0xFEFC	 	
+# 0xFEFD	 	
+# 0xFEFE	 	
+	
Index: /tags/gsdl-21/gsdl/unicode/MAPPINGS/EASTASIA/GB/makemapfile.pl
===================================================================
--- /tags/gsdl-21/gsdl/unicode/MAPPINGS/EASTASIA/GB/makemapfile.pl	(revision 699)
+++ /tags/gsdl-21/gsdl/unicode/MAPPINGS/EASTASIA/GB/makemapfile.pl	(revision 699)
@@ -0,0 +1,329 @@
+#! /usr/local/bin/perl5 -w
+
+# useful methods for dealing with the GuoBiao (GB) encoding
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+}
+
+use unicode;
+use util;
+
+# %encodings is of the form:
+#
+# encodings{encodingname-encodingname}->blocktranslation
+# blocktranslation->[[0-255],[256-511], ..., [65280-65535]]
+#
+# Any of the top translation blocks can point to an undefined
+# value. This data structure aims to allow fast translation and 
+# efficient storage.
+
+# @array256 is used for initialisation, there must be
+# a better way...
+@array256 = (0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+	     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0);
+
+
+%translations = ();
+
+&main ();
+
+
+
+# gb2unicode uses the GBK translation table
+sub gb2unicode {
+    generalgb2unicode ("GBK", @_);
+}
+
+# returns a pointer to unicode array
+sub generalgb2unicode {
+    my ($encoding, $intext) = @_;
+    my $mapfile = &util::filename_cat($ENV{'GSDLHOME'}, 
+				      "unicode/MAPPINGS/EASTASIA/GB",
+				      "$encoding.TXT");
+    if (!&loadencoding ($encoding, $mapfile)) {
+	print STDERR "gb: ERROR - could not load encoding $encoding\n";
+	return [];
+    }
+    
+    my @outtext = ();
+    my $encodename = "$encoding-unicode";
+    my $len = length($intext);
+    my ($c1, $c2);
+    my $i = 0;
+
+    while ($i < $len) {
+	if (($c1 = ord(substr($intext, $i, 1))) >= 0x81) {
+	    if ($i+1 < $len) {
+		# found a GB character, put black square if cannot translate
+		$c2 = ord(substr($intext, $i+1, 1));
+		my $c = &transchar ($encodename, ($c1 << 8) | $c2);
+		$c = 0x25a1 if $c == 0;
+		push (@outtext, $c);
+		$i += 2;
+		
+	    } else {
+		# error
+		print STDERR "gb: ERROR missing second half of GB character\n";
+		$i++;
+	    }
+	    
+	} else {
+	    # normal ascii character
+	    push (@outtext, $c1) if ($c1 < 0x80);
+	    $i++;
+	}
+    }
+    
+    return \@outtext;
+}
+
+
+# unicode2gb uses the GBK translation table
+sub unicode2gb {
+    generalunicode2gb ("GBK", @_);
+}
+
+# returns a gb string. Note that this will not translate
+# traditional forms of characters to their simplified forms
+# Any traditional characters in the unicode will not be translated
+sub generalunicode2gb {
+    my ($encoding, $intext) = @_;
+
+    # load the encoding (if it is not already loaded)
+    my $mapfile = &util::filename_cat($ENV{'GSDLHOME'}, 
+				      "unicode/MAPPINGS/EASTASIA/GB",
+				      "$encoding.TXT");
+    if (!&loadencoding ($encoding, $mapfile)) {
+	print STDERR "gb: ERROR - could not load encoding $encoding\n";
+	return "";
+    }
+    
+    # translate the string
+    my ($num, $gbc);
+    my $outtext = "";
+    my $encodename = "unicode-$encoding";
+    foreach $num (@$intext) {
+	if ($num < 0x80) {
+	    # normal ascii character
+	    $outtext .= chr ($num);
+	} else {
+	    # gb character
+	    my $gbc = &transchar ($encodename, $num);
+	    $gbc = 0xa1f5 if $gbc == 0;
+	    $outtext .= chr ($gbc >> 8);
+	    $outtext .= chr ($gbc & 0xff);
+	}
+    }
+
+    return $outtext;
+}
+
+
+# loadmapfile expects the mapfile to contain (at least) two
+# tab-seperated fields. The first field is the mapped value
+# and the second field is the unicode value.
+#
+# It returns 1 if successful, 0 if unsuccessful
+sub loadencoding {
+    my ($encoding, $mapfile) = @_;
+    
+    my $to = "$encoding-unicode";
+    my $from = "unicode-$encoding";
+
+    # check to see if the encoding has already been loaded
+    if (defined $translations{$to} && defined $translations{$from}) {
+	return 1;
+    }
+
+    return 0 unless open (MAPFILE, $mapfile);
+
+    my ($line, @line);
+    $translations{$to} = [@array256];
+    $translations{$from} = [@array256];
+    while (defined ($line = <MAPFILE>)) {
+	# remove comments
+	$line =~ s/\#.*$//;
+	next unless $line =~ /\S/;
+
+	# split the line into fields and do a few
+	# simple sanity checks
+	@line = split (/\t/, $line);
+	next unless (scalar(@line) >= 2 &&
+		     $line[0] =~ /^0x/ &&
+		     $line[1] =~ /^0x/);
+
+	my $gbc = hex($line[0]);
+	my $unic = hex($line[1]);
+
+	# gbk has already had the necessary most significant
+	# bits set (it is used in the encoding
+	$gbc = $gbc | 0x8080 unless ($encoding =~ /gbk/i);
+
+	&addchartrans ($translations{$to}, $gbc, $unic);
+	&addchartrans ($translations{$from}, $unic, $gbc);
+    }
+
+    close (MAPFILE);
+
+    return 1;
+}
+
+sub showfilled {
+    my ($block) = @_;
+    my $count = 0;
+    foreach $el (@$block) {
+	if (ref ($el) eq "ARRAY") {
+	    print "1";
+	    $count++;
+	} else {
+	    print "0";
+	}
+    }
+    print "\n";
+    print "total: $count\n";
+}
+
+
+# addchartrans adds one character translation to a translation block.
+# It also simplifies the translation block if possible.
+sub addchartrans {
+    my ($block, $from, $to) = @_;
+    my $i = 0;
+
+    my $high = ($from / 256) % 256;
+    my $low = $from % 256;
+
+    if (ref ($block->[$high]) ne "ARRAY") {
+	$block->[$high] = [@array256];
+    }
+    $block->[$high]->[$low] = $to;
+}
+
+
+sub transchar {
+    my ($encoding, $from) = @_;
+    my $high = ($from / 256) % 256;
+    my $low = $from % 256;
+
+    return 0 unless defined $translations{$encoding};
+
+    my $block = $translations{$encoding};
+
+    if (ref ($block->[$high]) ne "ARRAY") {
+	return 0;
+    }
+    return $block->[$high]->[$low];
+}
+
+# It returns 1 if successful, 0 if unsuccessful
+sub loadmapencoding {
+    my ($encoding) = @_;
+    
+    my $to = "$encoding-unicode";
+    my $from = "unicode-$encoding";
+
+    # check to see if the encoding has already been loaded
+    if (defined $translations{$to} && defined $translations{$from}) {
+	return 1;
+    }
+    
+    &readmapfile ($to);
+    &readmapfile ($from);
+    return 1;
+}
+
+sub readmapfile {
+    my ($encoding) = @_;
+    my $filename = $encoding;
+
+    $filename =~ s/\-//;
+    $filename =~ s/unicode/u/i;
+    $filename =~ tr/A-Z/a-z/;
+    $filename .= ".ump"; # unicode map file
+
+    $translations{$encoding} = [@array256];
+    my $block = $translations{$encoding};
+
+    open (MAPFILE, "$ENV{'GSDLHOME'}/unicode/$filename") || die;
+
+    my ($in,$i,$j);
+    while (read(MAPFILE, $in, 1) == 1) {
+	$i = unpack ("C", $in);
+	$block->[$i] = [@array256];
+	for ($j=0; $j<256 && read(MAPFILE, $in, 2)==2; $j++) {
+	    my ($n1, $n2) = unpack ("CC", $in);
+	    $block->[$i]->[$j] = ($n1*256) + $n2;
+	}
+    }
+
+    close (MAPFILE);
+}
+
+sub writemapfile {
+    my ($encoding) = @_;
+    my $filename = $encoding;
+
+    $filename =~ s/\-//;
+    $filename =~ s/unicode/u/i;
+    $filename =~ tr/A-Z/a-z/;
+    $filename .= ".ump"; # unicode map file
+
+    die "translation not defined" if (!defined $translations{$encoding});
+    my $block = $translations{$encoding};
+
+    print "writing $ENV{'GSDLHOME'}/unicode/$filename\n";
+    open (MAPFILE, ">$ENV{'GSDLHOME'}/unicode/$filename") || die;
+
+    my ($i, $j);
+    for ($i=0; $i<256; $i++) {
+	if (ref ($block->[$i]) eq "ARRAY") {
+	    print MAPFILE pack ("C", $i);
+	    for ($j=0; $j<256; $j++) {
+		# unsigned short in network order
+		print MAPFILE pack ("CC", int($block->[$i]->[$j] / 256), 
+				    $block->[$i]->[$j] % 256);
+	    }
+	}
+    }
+
+    close (MAPFILE);
+}
+
+sub makeencodingmapfile {
+    my ($encoding) = @_;
+
+    my $mapfile = &util::filename_cat($ENV{'GSDLHOME'}, 
+				      "unicode/MAPPINGS/EASTASIA/GB",
+				      "$encoding.TXT");
+    if (!&loadencoding ($encoding, $mapfile)) {
+#    if (!&loadmapencoding ($encoding)) {
+	die "couldn't load encoding $encoding";
+    }
+
+    # write out map files
+    &writemapfile ("$encoding-unicode");
+    &writemapfile ("unicode-$encoding");
+}
+
+sub main {
+    makeencodingmapfile ("BIG5");
+#    makeencodingmapfile ("GBK");
+#    makeencodingmapfile ("GB2312");
+#    makeencodingmapfile ("GB12345");
+}
