Index: /trunk/gsdl/bin/script/buildcol.pl
===================================================================
--- /trunk/gsdl/bin/script/buildcol.pl	(revision 4)
+++ /trunk/gsdl/bin/script/buildcol.pl	(revision 4)
@@ -0,0 +1,111 @@
+#!/usr/local/bin/perl5 -w
+
+# This program will build a particular collection
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/lib");
+    unshift (@INC, "$ENV{'GSDLHOME'}/lib/plugins");
+}
+
+use parsargv;
+use util;
+
+&main();
+
+sub print_usage {
+    print STDERR "\n  usage: $0 [options] collection-name\n\n";
+    print STDERR "  options:\n";
+    print STDERR "   -verbosity number     0=none, 3=lots\n";
+    print STDERR "   -archivedir directory Where the archives live\n";
+    print STDERR "   -cachedir directory   Where to cache the archives\n";
+    print STDERR "   -builddir directory   Where to put the built indexes\n\n";
+}
+
+
+sub main
+{
+    if (!parsargv::parse(\@ARGV, 
+			 'verbosity/\d+/2', \$verbosity,
+			 'archivedir/.*/', \$archivedir,
+			 'cachedir/.*/', \$cachedir,
+			 'builddir/.*/', \$builddir)) {
+	&print_usage();
+	die "\n";
+    }
+
+    # get and check the collection
+    ($collection) = @ARGV;
+    if (!defined ($collection)) {
+	&print_usage();
+	die "\n";
+    }
+    if (!-e "$ENV{'GSDLHOME'}/collect/$collection") {
+	die "Invalid collection name ($collection).\n";
+    }
+
+    # fill in the default archives and building directories if none
+    # were supplied, turn all \ into / and remove trailing /
+    $archivedir = "$ENV{'GSDLHOME'}/collect/$collection/archives" if $archivedir eq "";
+    $archivedir =~ s/[\\\/]+/\//g;
+    $archivedir =~ s/\/$//;
+    $builddir = "$ENV{'GSDLHOME'}/collect/$collection/building" if $builddir eq "";
+    $builddir =~ s/[\\\/]+/\//g;
+    $builddir =~ s/\/$//;
+
+    # update the archive cache if needed
+    if ($cachedir) {
+	print STDERR "Updating archive cache\n" if ($verbosity >= 1);
+
+	$cachedir =~ s/[\\\/]+$//;
+	$cachedir .= "/collect/$collection" unless 
+	    $cachedir =~ /collect\/$collection/;
+
+	$realarchivedir = "$cachedir/archives";
+	$realbuilddir = "$cachedir/building";
+	&util::mk_all_dir ($realarchivedir);
+	&util::mk_all_dir ($realbuilddir);
+	&util::cachedir ($archivedir, $realarchivedir, $verbosity);
+
+    } else {
+	$realarchivedir = $archivedir;
+	$realbuilddir = $builddir;
+    }
+
+    # build it in realbuilddir
+    &util::mk_all_dir ($realbuilddir);
+
+
+    # if a builder class has been created for this collection, use it
+    # otherwise, use the mg builder
+    if (-e "$ENV{'GSDLHOME'}/collect/$collection/lib/${collection}builder.pm") {
+	$builderdir = "$ENV{'GSDLHOME'}/collect/$collection/lib";
+	$buildertype = "${collection}builder";
+    } else {
+	$builderdir = "$ENV{'GSDLHOME'}/lib";
+	$buildertype = "mgbuilder";
+    }
+	
+    require "$builderdir/$buildertype.pm";
+
+    eval("\$builder = new $buildertype(\$collection, " .
+	 "\$realarchivedir, \$realbuilddir, \$verbosity)");
+    die "$@" if $@;
+
+    $builder->init();
+    $builder->compress_text();
+    $builder->build_indexes();
+    $builder->make_infodatabase();
+    $builder->make_auxiliary_files();
+    $builder->deinit();
+
+
+    if ($realbuilddir ne $builddir) {
+	print STDERR "Copying back the cached build\n" if ($verbosity >= 1);
+	&util::rm_r ($builddir);
+	&util::cp_r ($realbuilddir, $builddir);
+    }
+}
+
+
Index: /trunk/gsdl/bin/script/gettext.pl
===================================================================
--- /trunk/gsdl/bin/script/gettext.pl	(revision 4)
+++ /trunk/gsdl/bin/script/gettext.pl	(revision 4)
@@ -0,0 +1,91 @@
+#!/usr/local/bin/perl5 -w
+
+# This program will output the text for a particular index
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/lib");
+    unshift (@INC, "$ENV{'GSDLHOME'}/lib/plugins");
+}
+
+use parsargv;
+use plugin;
+use colcfg;
+
+sub print_usage {
+    print STDERR "\n  usage: $0 collection-name\n";
+    print STDERR "  options:\n";
+    print STDERR "   -index index   The index to output\n\n";
+}
+
+
+&main ();
+
+sub main {
+
+    if (!parsargv::parse(\@ARGV, 
+			 'index/.*/section:text', \$index,
+			 'mode/.*/text', \$mode)) {
+	&print_usage();
+	die "\n";
+    }
+
+    # get and check the collection name
+    ($collection) = @ARGV;
+
+    if (!defined($collection)) {
+	&print_usage();
+	die "\n";
+    }
+    
+    # get the list of plugins for this collection
+    @plugins = ("RecPlug", "HTMLPlug", "TXTPlug");
+    if (-e "$ENV{'GSDLHOME'}/collect/$collection/collect.cfg") {
+	$collectcfg = &colcfg::read_collect_cfg ("$ENV{'GSDLHOME'}/collect/$collection/collect.cfg");
+	if (defined $collectcfg->{'plugins'}) {
+	    @plugins = @{$collectcfg->{'plugins'}};
+	}
+    }
+    
+
+    # load all the plugins
+    $pluginfo = &plugin::load_plugins ($collection, \@plugins);
+    if (scalar(@$pluginfo) == 0) {
+	print STDERR "No plugins were loaded.\n";
+	die "\n";
+    }
+
+    # load up the document processor for building
+    # if a buildproc class has been created for this collection, use it
+    # otherwise, use the mg buildproc
+    my ($buildprocdir, $buildproctype);
+    if (-e "$ENV{'GSDLHOME'}/collect/$collection/lib/${collection}buildproc.pm") {
+	$buildprocdir = "$ENV{'GSDLHOME'}/collect/$collection/lib";
+	$buildproctype = "${collection}buildproc";
+    } else {
+	$buildprocdir = "$ENV{'GSDLHOME'}/lib";
+	$buildproctype = "mgbuildproc";
+    }
+    require "$buildprocdir/$buildproctype.pm";
+
+    my $source_dir = "$ENV{'GSDLHOME'}/collect/$collection/archives";
+    my $build_dir = "$ENV{'GSDLHOME'}/collect/$collection/building";
+    my $verbosity = 1;
+
+    eval("\$buildproc = new $buildproctype(\$collection, " .
+	 "\$source_dir, \$build_dir, \$verbosity)");
+    die "$@" if $@;
+
+    # set up the document processor
+    $buildproc->set_output_handle ('STDOUT');
+    $buildproc->set_mode ($mode);
+    $buildproc->set_index ($index);
+
+    $buildproc->reset();
+
+    # process the import directory
+    &plugin::read ($pluginfo, $source_dir, 
+		   "", {}, $buildproc);
+
+}
Index: /trunk/gsdl/bin/script/gsdlinfo.pl
===================================================================
--- /trunk/gsdl/bin/script/gsdlinfo.pl	(revision 4)
+++ /trunk/gsdl/bin/script/gsdlinfo.pl	(revision 4)
@@ -0,0 +1,110 @@
+#!/usr/local/bin/perl5 -w
+
+# This program will look to see what collections are installed
+# under this file system
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/lib");
+}
+
+use parsargv;
+use colcfg;
+
+
+# the collection information is stored as an array of hashes
+# (one hash for each collection). The hash contains all the 
+# information in the collect and build configuration files
+
+
+# read in all the collection configuration information
+
+opendir (DIR, "$ENV{'GSDLHOME'}/collect") ||
+    die "ERROR - couldn't read the directory $ENV{'GSDLHOME'}/collect\n";
+@dir = readdir (DIR);
+closedir (DIR);
+
+@collections = ();
+foreach $dir (@dir) {
+    if ($dir =~ /^(\..*|modelcol)$/) {
+	# do nothing
+
+    } else {
+	# read in the collection configuration file
+	my $collectcfg = {};
+	if (-e "$ENV{'GSDLHOME'}/collect/$dir/collect.cfg") {
+	    $collectcfg = &colcfg::read_collect_cfg ("$ENV{'GSDLHOME'}/collect/$dir/collect.cfg");
+	}
+	$collectcfg->{'collection'} = $dir;
+
+	# read in the build configuration file
+	my $buildcfg = {};
+	if (-e "$ENV{'GSDLHOME'}/collect/$dir/index/build.cfg") {
+	    $buildcfg = &colcfg::read_build_cfg ("$ENV{'GSDLHOME'}/collect/$dir/index/build.cfg");
+	}
+
+	# add the merged configuration files to the current list of collections
+	push (@collections, {%$collectcfg, %$buildcfg});
+    }
+}
+
+
+# do any sorting requested
+
+
+# print out the collection information
+
+# {'creator'}->string
+# {'maintainer'}->array of strings
+# {'public'}->string
+# {'beta'}->string
+# {'key'}->string
+# {'indexes'}->array of strings
+# {'defaultindex'}->string
+# {'builddate'}->string
+# {'metadata'}->array of strings
+# {'languages'}->array of strings
+# {'numdocs'}->string
+# {'numwords'}->string
+# {'numbytes'}->string
+
+print "\n";
+foreach $info (@collections) {
+    if (defined $info->{'collection'}) {
+	$collection = $info->{'collection'};
+	if (defined $info->{'public'}) {
+	    $public = $info->{'public'};
+	} else {
+	    $public = "?";
+	}
+	if (defined $info->{'beta'}) {
+	    $beta = $info->{'beta'};
+	} else {
+	    $beta = "?";
+	}
+	if (defined $info->{'numdocs'}) {
+	    $numdocs = $info->{'numdocs'};
+	} else {
+	    $numdocs = "?";
+	}
+	if (defined $info->{'numbytes'}) {
+	    $numMbytes = int(($info->{'numbytes'}/1024.0/1024.0)*1000)/1000.0;
+	} else {
+	    $numMbytes = "?";
+	}
+	write;
+    }
+}
+print "\n";
+
+exit;
+
+format STDOUT_TOP = 
+  name   public  beta   documents  size (Mbytes)
+-------- ------ ------ ----------- -------------
+.
+
+format STDOUT = 
+@>>>>>>> @>>>>> @>>>>> @>>>>>>>>>> @>>>>>>>>>>>>
+$collection, $public, $beta, $numdocs, $numMbytes
+.
Index: /trunk/gsdl/bin/script/import.pl
===================================================================
--- /trunk/gsdl/bin/script/import.pl	(revision 4)
+++ /trunk/gsdl/bin/script/import.pl	(revision 4)
@@ -0,0 +1,69 @@
+#!/usr/local/bin/perl5 -w
+
+# This program will import a number of files into a particular collection
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/lib");
+    unshift (@INC, "$ENV{'GSDLHOME'}/lib/plugins");
+}
+
+use arcinfo;
+use colcfg;
+use plugin;
+use docsave;
+
+
+sub print_usage {
+    print STDERR "\n  usage: $0 collection-name\n";
+}
+
+
+&main ();
+
+sub main {
+    # get and check the collection name
+    ($collection) = @ARGV;
+    if (!defined($collection)) {
+	&print_usage();
+	die "\n";
+    }
+    
+    if ($collection eq "modelcol") {
+	print STDERR "You can't import documents into modelcol.\n";
+	die "\n";
+    }
+    
+    # get the list of plugins for this collection
+    @plugins = ("RecPlug", "HTMLPlug", "TXTPlug");
+    if (-e "$ENV{'GSDLHOME'}/collect/$collection/collect.cfg") {
+	$collectcfg = &colcfg::read_collect_cfg ("$ENV{'GSDLHOME'}/collect/$collection/collect.cfg");
+	if (defined $collectcfg->{'plugins'}) {
+	    @plugins = @{$collectcfg->{'plugins'}};
+	}
+    }
+    
+
+    # load all the plugins
+    $pluginfo = &plugin::load_plugins ($collection, \@plugins);
+    if (scalar(@$pluginfo) == 0) {
+	print STDERR "No plugins were loaded.\n";
+	die "\n";
+    }
+    
+    # read the archive information file
+    $archive_info_filename = "$ENV{'GSDLHOME'}/collect/$collection/archives/archives.inf";
+    $archive_info = new arcinfo ();
+    $archive_info->load_info ($archive_info_filename);
+
+    # create a docsave object to process the documents
+    $processor = new docsave ($collection, $archive_info);
+
+    # process the import directory
+    &plugin::read ($pluginfo, "$ENV{'GSDLHOME'}/collect/$collection/import/", 
+		   "", {}, $processor);
+    
+    # write out the archive information file
+    $archive_info->save_info($archive_info_filename);
+}
Index: /trunk/gsdl/bin/script/mkcol.pl
===================================================================
--- /trunk/gsdl/bin/script/mkcol.pl	(revision 4)
+++ /trunk/gsdl/bin/script/mkcol.pl	(revision 4)
@@ -0,0 +1,159 @@
+#!/usr/local/bin/perl5 -w
+
+# This program will setup a new collection from a model one. It does this by
+# copying the model, moving files to have the correct names, and replacing
+# text within the files to match the parameters.
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/lib");
+}
+
+use parsargv;
+
+sub print_usage {
+    print STDERR "\n  usage: $0 [options] collection-name\n\n";
+    print STDERR "  options:\n";
+    print STDERR "   -creator email      Your email address\n";
+    print STDERR "   -maintainer email   The current maintainer's email address\n";
+    print STDERR "   -public true|false  If this collection has anonymous access\n";
+    print STDERR "   -beta true|false    If this collection is still under development\n";
+    print STDERR "   -index type         The indexes which should be made\n";
+    print STDERR "   -indextext name     The index description\n";
+    print STDERR "   -defaultindex type  The index to use if no others are supplied\n\n";
+}
+
+sub traverse_dir
+{
+    my ($modeldir, $coldir) = @_;
+    my ($newfile, @filetext);
+
+    if (!(-e $coldir)) {
+	if (!mkdir ($coldir, 0775)) {
+	    die "$!";
+	}
+    }
+
+    opendir(DIR, $modeldir) || die "Can't read $modeldir";
+    my @files = grep(!/^\.\.?$/, readdir(DIR));
+    closedir(DIR);
+
+    foreach $file (@files)
+    {
+	if (-d "$modeldir/$file") {
+	    traverse_dir ("$modeldir/$file", "$coldir/$file");
+
+	} else {
+	    my $destfile = $file;
+	    $destfile =~ s/^\modelcol/$collection/;
+	    $destfile =~ s/^\MODELCOL/$capcollection/;
+
+	    print STDERR "doing replacements for $modeldir/$file\n";
+	    open (INFILE, "$modeldir/$file") || 
+		die "Can't read file $modeldir/$file";
+	    open (OUTFILE, ">$coldir/$destfile") ||
+		die "Can't create file $coldir/destfile";
+
+	    while (defined ($line = <INFILE>)) {
+		$line =~ s/\*\*collection\*\*/$collection/g;
+		$line =~ s/\*\*COLLECTION\*\*/$capcollection/g;
+		$line =~ s/\*\*creator\*\*/$creator/g;
+		$line =~ s/\*\*maintainer\*\*/$maintainer/g;
+		$line =~ s/\*\*public\*\*/$public/g;
+		$line =~ s/\*\*beta\*\*/$beta/g;
+		$line =~ s/\*\*key\*\*/$key/g;
+		$line =~ s/\*\*indexes\*\*/$indexesstr/g;
+		$line =~ s/\*\*indexestext\*\*/$indexestextstr/g;
+		$line =~ s/\*\*defaultindex\*\*/$defaultindex/g;
+
+		print OUTFILE $line;
+	    }
+	    
+	    close (OUTFILE);
+	    close (INFILE);
+	}
+    }
+}
+
+
+my (@indexes, @indexestext);
+
+# get and check options
+if (!&parsargv::parse(\@ARGV,
+		       'creator/\w+\@[\w\.]+/', \$creator,
+		       'maintainer/\w+\@[\w\.]+/', \$maintainer,
+		       'public/true|false/true', \$public,
+		       'beta/true|false/true', \$beta,
+		       'index/.*/document:all', \@indexes,
+		       'indextext/\.*/Terms must appear within the same document', \@indexestext,
+		       'defaultindex/.*/document:all', \$defaultindex
+		       )) {
+    &print_usage();
+    die "\n";
+}
+    
+# get and check the collection name
+($collection) = @ARGV;
+if (!defined($collection)) {
+    &print_usage();
+    die "\n";
+}
+
+if (length($collection) > 8) {
+    print STDERR "The collection name must be less than 8 characters\n";
+    print STDERR "so compatibility with earlier filesystems can be\n";
+    print STDERR "maintained.\n";
+    die "\n";
+}
+
+if ($collection eq "modelcol") {
+    print STDERR "No collection can be named modelcol as this is the\n";
+    print STDERR "name of the model collection.\n";
+    die "\n";
+}
+
+if (!defined($creator) || $creator eq "") {
+    print STDERR "The creator was not defined. This variable is\n";
+    print STDERR "needed to recognise duplicate collection names.\n";
+    die "\n";
+}
+
+if (!defined($maintainer) || $maintainer eq "") {
+    $maintainer = $creator;
+}
+
+$public = "true" unless defined $public;
+$beta = "true" unless defined $beta;
+
+srand(time);
+$key = int(rand()*100000000.0) + int(rand()*100000000.0);
+
+# get capitalised version of the collection
+$capcollection = $collection;
+$capcollection =~ tr/a-z/A-Z/;
+
+
+# get the strings to include.
+$indexesstr = join ("\t", @indexes);
+$indexestextstr = "";
+for ($i=0; $i<scalar(@indexestext); $i++) {
+    $indexestextstr .= "_$indexes[$i]_[] {$indexestext[$i]}\n";
+}
+
+
+# make sure the model collection exists
+die "Cannot find the model collection $ENV{'GSDLHOME'}/collect/modelcol" unless
+    (-d "$ENV{'GSDLHOME'}/collect/modelcol");
+
+# make sure this collection does not already exist
+if (-e "$ENV{'GSDLHOME'}/collect/$collection") {
+    print STDERR "This collection already exists\n";
+    die "\n";
+}
+
+
+# start creating the collection
+print STDERR "Creating the collection $collection\n\n";
+&traverse_dir ("$ENV{'GSDLHOME'}/collect/modelcol", "$ENV{'GSDLHOME'}/collect/$collection");
+print STDERR "\n\nThe new collection is in $ENV{'GSDLHOME'}/collect/$collection.\n\n";
+
Index: /trunk/gsdl/bin/script/perlcalc.pl
===================================================================
--- /trunk/gsdl/bin/script/perlcalc.pl	(revision 4)
+++ /trunk/gsdl/bin/script/perlcalc.pl	(revision 4)
@@ -0,0 +1,5 @@
+#! /usr/local/bin/perl5 -w
+
+while (defined($line=<STDIN>)) {
+	print eval($line), "\n";
+}
Index: /trunk/gsdl/collect/modelcol/collect.cfg
===================================================================
--- /trunk/gsdl/collect/modelcol/collect.cfg	(revision 4)
+++ /trunk/gsdl/collect/modelcol/collect.cfg	(revision 4)
@@ -0,0 +1,8 @@
+creator	**creator**
+maintainer	**maintainer**
+public	**public**
+beta	**beta**
+key	**key**
+indexes	**indexes**
+defaultindex	**defaultindex**
+plugins	GMLPlug	ArcPlug	TOCPlug	HBPlug
Index: /trunk/gsdl/perllib/arcinfo.pm
===================================================================
--- /trunk/gsdl/perllib/arcinfo.pm	(revision 4)
+++ /trunk/gsdl/perllib/arcinfo.pm	(revision 4)
@@ -0,0 +1,110 @@
+# This module stores information about the archives. At the moment
+# this information just consists of the file name (relative to the
+# directory the archives information file is in) and its OID.
+
+# This module assumes there is a one to one correspondance between
+# a file in the archives directory and an OID.
+
+
+package arcinfo;
+
+sub new {
+    my ($class) = @_;
+    my $self = {'info'=>{}};
+
+    return bless $self, $class;
+}
+
+sub load_info {
+    my $self = shift (@_);
+    my ($filename) = @_;
+
+    $self->{'info'} = {};
+
+    if (-e $filename) {
+	open (INFILE, $filename) || 
+	    die "arcinfo::load_info couldn't read $filename\n";
+
+	my ($line, @line);
+	while (defined ($line = <INFILE>)) {
+	    $line =~ s/\cM|\cJ//g; # remove end-of-line characters
+	    @line = split ("\t", $line); # filename, 
+	    if (scalar(@line) >= 2) {
+		$self->add_info ($line[0], $line[1]);
+	    }
+	}
+
+	close (INFILE);
+    }
+}
+
+sub save_info {
+    my $self = shift (@_);
+    my ($filename) = @_;
+
+    my ($OID);
+
+    open (OUTFILE, ">$filename") || 
+	die "arcinfo::save_info couldn't write $filename\n";
+
+    foreach $OID (keys(%{$self->{'info'}})) {
+	my $info = $self->get_info($OID);
+	if (defined $info) {
+	    print OUTFILE "$OID\t", join("\t", @$info), "\n";
+	}
+    }
+    
+    close (OUTFILE);
+}
+
+sub add_info {
+    my $self = shift (@_);
+    my ($OID, $doc_file) = @_;
+
+    $self->{'info'}->{$OID} = [$doc_file];
+}
+
+# returns a list of the form [[OID, doc_file], ...]
+sub get_OID_list {
+    my $self = shift (@_);
+
+    my ($OID);
+    my @list = ();
+
+    foreach $OID (sort(keys(%{$self->{'info'}}))) {
+	push (@list, [$OID, $self->{'info'}->{$OID}->[0]]);
+    }
+
+    return \@list;
+}
+
+# returns a list of the form [[doc_file, OID], ...]
+sub get_file_list {
+    my $self = shift (@_);
+
+    my ($OID);
+    my @list = ();
+
+    foreach $OID (sort(keys(%{$self->{'info'}}))) {
+	push (@list, [$self->{'info'}->{$OID}->[0], $OID]);
+    }
+
+    return \@list;
+}
+
+
+# returns a list of the form [doc_file]
+sub get_info {
+    my $self = shift (@_);
+    my ($OID) = @_;
+
+    if (defined $self->{'info'}->{$OID}) {
+	return $self->{'info'}->{$OID};
+    }
+
+    return undef;
+}
+
+
+1;
+
Index: /trunk/gsdl/perllib/cfgread.pm
===================================================================
--- /trunk/gsdl/perllib/cfgread.pm	(revision 4)
+++ /trunk/gsdl/perllib/cfgread.pm	(revision 4)
@@ -0,0 +1,75 @@
+# reads in configuration files
+
+package cfgread;
+
+
+sub read_cfg_line {
+    my ($handle) = @_;
+    my $line = "";
+
+    while (defined($line = <$handle>)) {
+	$line =~ s/^\#.*$//; # remove comments
+	$line =~ s/\cM|\cJ//g; # remove end-of-line characters
+	if ($line =~ /\w/) {
+	    my @line = split(/\t/, $line);
+	    return \@line;
+	}
+    }
+    return undef;
+}
+
+sub write_cfg_line {
+    my ($handle, $line) = @_;
+    print $handle join ("\t", @$line), "\n";
+}
+
+
+# stringexp and arrayexp should be something like
+#   '^(this|that)$'
+sub read_cfg_file {
+    my ($filename, $stringexp, $arrayexp) = @_;
+    my ($line);
+    my $data = {};
+
+    if (open (COLCFG, $filename)) {
+	while (defined ($line = &read_cfg_line('COLCFG'))) {
+	    if (scalar(@$line) >= 2) {
+		my $key = shift (@$line);
+		if ($key =~ /$stringexp/) {
+		    $data->{$key} = shift (@$line);
+
+		} elsif ($key =~ /$arrayexp/) {
+		    push (@{$data->{$key}}, @$line);
+		}
+	    }
+	}
+	close (COLCFG);
+    } else {
+	print STDERR "cfgread::read_cfg_file couldn't read the cfg file $filename\n";
+    }
+
+    return $data;
+}
+
+
+# stringexp and arrayexp should be something like
+#   '^(this|that)$'
+sub write_cfg_file {
+    my ($filename, $data, $stringexp, $arrayexp) = @_;
+
+    if (open (COLCFG, ">$filename")) {
+	foreach $key (sort(keys(%$data))) {
+	    if ($key =~ /$stringexp/) {
+		&write_cfg_line ('COLCFG', [$key, $data->{$key}]);
+	    } elsif ($key =~ /$arrayexp/) {
+		&write_cfg_line ('COLCFG', [$key, @{$data->{$key}}]);
+	    }
+	}
+	close (COLCFG);
+    } else {
+	print STDERR "cfgread::write_cfg_file couldn't write the cfg file $filename\n";
+    }
+}
+
+
+1;
Index: /trunk/gsdl/perllib/colcfg.pm
===================================================================
--- /trunk/gsdl/perllib/colcfg.pm	(revision 4)
+++ /trunk/gsdl/perllib/colcfg.pm	(revision 4)
@@ -0,0 +1,62 @@
+# reads in a collection configuration file
+
+package colcfg;
+
+use cfgread;
+
+
+# the collection configuration file data is stored in the form
+#
+# {'creator'}->string
+# {'maintainer'}->array of strings
+# {'public'}->string
+# {'beta'}->string
+# {'key'}->string
+# {'indexes'}->array of strings
+# {'defaultindex'}->string
+# {'plugins'}->array of strings
+
+sub read_collect_cfg {
+    my ($filename) = @_;
+
+    return &cfgread::read_cfg_file ($filename, 
+		    '^(creator|public|beta|key|defaultindex)$', 
+		    '^(maintainer|indexes|plugins)$');
+}
+
+sub write_collect_cfg {
+    my ($filename, $data) = @_;
+
+    &cfgread::write_cfg_file($filename, $data,
+		    '^(creator|public|beta|key|defaultindex)$', 
+		    '^(maintainer|indexes|plugins)$');
+}
+
+
+# the build configuration file data is stored in the form
+#
+# {'builddate'}->string
+# {'metadata'}->array of strings
+# {'languages'}->array of strings
+# {'numdocs'}->string
+# {'numwords'}->string
+# {'numbytes'}->string
+
+sub read_build_cfg {
+    my ($filename) = @_;
+
+    return &cfgread::read_cfg_file ($filename, 
+		    '^(builddate|numdocs|numwords|numbytes)$', 
+		    '^(metadata|languages)$');
+}
+
+sub write_build_cfg {
+    my ($filename, $data) = @_;
+
+    &cfgread::write_cfg_file($filename, $data,
+		    '^(builddate|numdocs|numwords|numbytes)$', 
+		    '^(metadata|languages)$');
+}
+
+
+1;
Index: /trunk/gsdl/perllib/doc.pm
===================================================================
--- /trunk/gsdl/perllib/doc.pm	(revision 4)
+++ /trunk/gsdl/perllib/doc.pm	(revision 4)
@@ -0,0 +1,677 @@
+# class to hold documents
+
+package doc;
+
+use util;
+use html;
+
+# the document type may be indexed_doc, nonindexed_doc, or
+# classification
+
+sub new {
+    my $class = shift (@_);
+    my ($source_filename, $doc_type) = @_;
+    
+    my $self = bless {'associated_files'=>[],
+		      'subsection_order'=>[],
+		      'next_subsection'=>1,
+		      'subsections'=>{},
+		      'metadata'=>[],
+		      'text'=>""}, $class;
+
+    $self->set_source_filename ($source_filename) if defined $source_filename;
+    $self->set_doc_type ($doc_type) if defined $doc_type;
+
+    return $self;
+}
+
+sub set_source_filename {
+    my $self = shift (@_);
+    my ($source_filename) = @_;
+
+    $self->set_metadata_element ($self->get_top_section(), 
+				 "gsdlsourcefilename", 
+				 $source_filename);
+}
+
+# returns the source_filename as it was provided
+sub get_source_filename {
+    my $self = shift (@_);
+
+    return $self->get_metadata_element ($self->get_top_section(), "gsdlsourcefilename");
+}
+
+sub set_doc_type {
+    my $self = shift (@_);
+    my ($doc_type) = @_;
+
+    $self->set_metadata_element ($self->get_top_section(), 
+				 "gsdldoctype", 
+				 $doc_type);
+}
+
+# returns the source_filename as it was provided
+sub get_doc_type {
+    my $self = shift (@_);
+
+    return $self->get_metadata_element ($self->get_top_section(), "gsdldoctype");
+}
+
+sub output_section {
+    my $self = shift (@_);
+    my ($handle, $section, $suppress_subject_info) = @_;
+    $suppress_subject_info = 0 unless defined $suppress_subject_info;
+    my ($data, $subsection);
+    
+    my $section_ptr = $self->_lookup_section ($section);
+    my ($section_num) = $section =~ /(\d+)$/;
+    
+    return unless defined $section_ptr;
+
+    # output the section header (including the section number
+    # and metadata)
+    print $handle "<gsdlsection";
+    print $handle " gsdlnum=\"$section_num\"" if defined $section_num;
+    foreach $data (@{$section_ptr->{'metadata'}}) {
+	print $handle " $data->[0]=\"$data->[1]\"" 
+	    unless $suppress_subject_info && $data->[0] eq "Subject";
+    }
+    print $handle ">";
+
+    # output the text
+    print $handle $section_ptr->{'text'};
+
+    # output all the subsections
+    foreach $subsection (@{$section_ptr->{'subsection_order'}}) {
+	$self->output_section($handle, "$section.$subsection", $suppress_subject_info);
+    }
+    
+    # output the closing tag
+    print $handle "</gsdlsection>\n";
+}
+
+# look up the reference to the a particular section
+sub _lookup_section {
+    my $self = shift (@_);
+    my ($section) = @_;
+
+    my ($num);
+    my $sectionref = $self;
+
+    while ($section ne "") {
+	($num, $section) = $section =~ /^\.?(\d+)(.*)$/;
+	$num =~ s/^0+(\d)/$1/; # remove leading 0s
+	$section = "" unless defined $section;
+	
+	if (defined $num && defined $sectionref->{'subsections'}->{$num}) {
+	    $sectionref = $sectionref->{'subsections'}->{$num};
+	} else {
+	    return undef;
+	}
+    }
+    
+    return $sectionref;
+}
+
+sub _calc_OID {
+    my $self = shift (@_);
+    my ($filename) = @_;
+
+    my $hashfile_exe = "$ENV{'GSDLHOME'}/bin/$ENV{'GSDLOS'}/hashfile.exe";
+    my $result = "NULL";
+
+    if (-e "$hashfile_exe") {
+	$result = `$hashfile_exe \"$filename\"`;
+	($result) = $result =~ /:\s*([0-9a-f]+)/i;
+
+    } else {
+	print STDERR "doc::_calc_OID hashfile.exe could not be found\n";
+    }
+
+    return "HASH$result";
+}
+
+# methods dealing with OID, not groups of them.
+
+# if $OID is not provided one is calculated from hashing the
+# current contents of the document
+# An OID are actually stored as metadata of the document
+sub set_OID {
+    my $self = shift (@_);
+    my ($OID) = @_;
+
+    # if an OID wasn't provided then feed this document to 
+    # hashfile.exe
+    if (!defined $OID) {
+	$OID = "NULL";
+	my $tmp_filename = &util::get_tmp_filename();
+	if (!open (OUTFILE, ">$tmp_filename")) {
+	    print STDERR "doc::set_OID could not write to $tmp_filename\n";
+	} else {
+	    $self->output_section('OUTFILE', $self->get_top_section(), 1);
+	    close (OUTFILE);
+
+	    $OID = $self->_calc_OID ($tmp_filename);
+	    &util::rm ($tmp_filename);
+	}
+    }
+
+    $self->set_metadata_element ($self->get_top_section(), "Identifier", $OID);
+}
+
+# returns the OID for this document
+sub get_OID {
+    my $self = shift (@_);
+    
+    return $self->get_metadata_element ($self->get_top_section(), "Identifier");
+}
+
+sub delete_OID {
+    my $self = shift (@_);
+    
+    $self->set_metadata_element ($self->get_top_section(), "Identifier", "NULL");
+}
+
+
+# methods for manipulating section names
+
+# returns the name of the top-most section (the top
+# level of the document
+sub get_top_section {
+    my $self = shift (@_);
+    
+    return "";
+}
+
+# returns a section
+sub get_parent_section {
+    my $self = shift (@_);
+    my ($section) = @_;
+
+    $section =~ s/(^|\.)\d+$//;
+
+    return $section;
+}
+
+# returns the first child section (or the end child 
+# if there isn't any)
+sub get_begin_child {
+    my $self = shift (@_);
+    my ($section) = @_;
+
+    my $section_ptr = $self->_lookup_section($section);
+    return "" unless defined $section_ptr;
+
+    if (defined $section_ptr->{'subsection_order'}->[0]) {
+	return "$section.$section_ptr->{'subsection_order'}->[0]";
+    }
+
+    return $self->get_end_child ($section);
+}
+
+# returns the next child of a parent section
+sub get_next_child {
+    my $self = shift (@_);
+    my ($section) = @_;
+    
+    my $parent_section = $self->get_parent_section($section);
+    my $parent_section_ptr = $self->_lookup_section($parent_section);
+    return undef unless defined $parent_section_ptr;
+
+    my ($section_num) = $section =~ /(\d+)$/;
+    return undef unless defined $section_num;
+
+    my $i = 0;
+    my $section_order = $parent_section_ptr->{'subsection_order'};
+    while ($i < scalar(@$section_order)) {
+	last if $section_order->[$i] eq $section_num;
+	$i++;
+    }
+
+    $i++; # the next child
+    if ($i < scalar(@$section_order)) {
+	return $section_order->[$i] if $parent_section eq "";
+	return "$parent_section.$section_order->[$i]";
+    }
+
+    # no more sections in this level
+    return undef;
+}
+
+# returns a reference to a list of children
+
+sub get_children {
+    my $self = shift (@_);
+    my ($section) = @_;
+
+    my $section_ptr = $self->_lookup_section($section);
+    return [] unless defined $section_ptr;
+
+    my @children = @{$section_ptr->{'subsection_order'}};
+
+    map {$_ = "$section.$_"; $_ =~ s/^\.+//;} @children;
+    return \@children;
+}
+
+# returns the child section one past the last one (which
+# is coded as "0")
+sub get_end_child {
+    my $self = shift (@_);
+    my ($section) = @_;
+
+    return $section . ".0" unless $section eq "";
+    return "0";
+}
+
+# returns the next section in book order
+sub get_next_section {
+    my $self = shift (@_);
+    my ($section) = @_;
+
+    return undef unless defined $section;
+
+    my $section_ptr = $self->_lookup_section($section);
+    return undef unless defined $section_ptr;
+
+    # first try to find first child
+    if (defined $section_ptr->{'subsection_order'}->[0]) {
+	return $section_ptr->{'subsection_order'}->[0] if ($section eq "");
+	return "$section.$section_ptr->{'subsection_order'}->[0]";
+    }
+
+    do {
+	# try to find sibling
+	my $next_child = $self->get_next_child ($section);
+	return $next_child if (defined $next_child);
+
+	# move up one level
+	$section = $self->get_parent_section ($section);
+    } while $section =~ /\d/;
+
+    return undef;
+}
+
+sub is_leaf_section {
+    my $self = shift (@_);
+    my ($section) = @_;
+
+    my $section_ptr = $self->_lookup_section($section);
+    return 1 unless defined $section_ptr;
+
+    return (scalar (@{$section_ptr->{'subsection_order'}}) == 0);
+}
+
+# methods for dealing with sections
+
+# returns the name of the inserted section
+sub insert_section {
+    my $self = shift (@_);
+    my ($before_section) = @_;
+
+    # get the child to insert before and its parent section
+    my $parent_section = "";
+    my $before_child = "0";
+    my @before_section = split (/\./, $before_section);
+    if (scalar(@before_section) > 0) {
+	$before_child = pop (@before_section);
+	$parent_section = join (".", @before_section);
+    }
+
+    my $parent_section_ptr = $self->_lookup_section($parent_section);
+    if (!defined $parent_section_ptr) {
+	print STDERR "doc::insert_section couldn't find parent section " .
+	    "$parent_section\n";
+	return;
+    }
+
+    # get the next section number
+    my $section_num = $parent_section_ptr->{'next_subsection'}++;
+
+    my $i = 0;
+    while ($i < scalar(@{$parent_section_ptr->{'subsection_order'}}) &&
+	   $parent_section_ptr->{'subsection_order'}->[$i] ne $before_child) {
+	$i++;
+    }
+    
+    # insert the section number into the order list
+    splice (@{$parent_section_ptr->{'subsection_order'}}, $i, 0, $section_num);
+
+    # add this section to the parent section
+    my $section_ptr = {'subsection_order'=>[],
+		       'next_subsection'=>1,
+		       'subsections'=>{},
+		       'metadata'=>[],
+		       'text'=>""};
+    $parent_section_ptr->{'subsections'}->{$section_num} = $section_ptr;
+
+    # work out the full section number
+    my $section = $parent_section;
+    $section .= "." unless $section eq "";
+    $section .= $section_num;
+    
+    return $section;
+}
+
+# creates a pre-named section
+sub create_named_section {
+    my $self = shift (@_);
+    my ($mastersection) = @_;
+
+    my ($num);
+    my $section = $mastersection;
+    my $sectionref = $self;
+
+    while ($section ne "") {
+	($num, $section) = $section =~ /^\.?(\d+)(.*)$/;
+	$num =~ s/^0+(\d)/$1/; # remove leading 0s
+	$section = "" unless defined $section;
+	
+	if (defined $num) {
+	    if (!defined $sectionref->{'subsections'}->{$num}) {
+		push (@{$sectionref->{'subsection_order'}}, $num);
+		$sectionref->{'subsections'}->{$num} = {'subsection_order'=>[],
+							'next_subsection'=>1,
+							'subsections'=>{},
+							'metadata'=>[],
+							'text'=>""};
+		if ($num >= $sectionref->{'next_subsection'}) {
+		    $sectionref->{'next_subsection'} = $num + 1;
+		}
+	    }
+	    $sectionref = $sectionref->{'subsections'}->{$num};
+
+	} else {
+	    print STDERR "doc::create_named_section couldn't create section " .
+		"$mastersection\n";
+	    last;
+	}
+    }
+}
+
+# returns a reference to a list of subsections
+sub list_subsections {
+    my $self = shift (@_);
+    my ($section) = @_;
+
+    my $section_ptr = $self->_lookup_section ($section);
+    if (!defined $section_ptr) {
+	print STDERR "doc::list_subsections couldn't find section $section\n";
+	return [];
+    }
+
+    return [@{$section_ptr->{'subsection_order'}}];
+}
+
+sub delete_section {
+    my $self = shift (@_);
+    my ($section) = @_;
+
+#    my $section_ptr = {'subsection_order'=>[],
+#		       'next_subsection'=>1,
+#		       'subsections'=>{},
+#		       'metadata'=>[],
+#		       'text'=>""};
+
+    # if this is the top section reset everything
+    if ($section eq "") {
+	$self->{'subsection_order'} = [];
+	$self->{'subsections'} = {};
+	$self->{'metadata'} = [];
+	$self->{'text'} = "";
+	return;
+    }
+
+    # find the parent of the section to delete
+    my $parent_section = "";
+    my $child = "0";
+    my @section = split (".", $section);
+    if (scalar(@section) > 0) {
+	$child = pop (@section);
+	$parent_section = join (".", @section);
+    }
+
+    my $parent_section_ptr = $self->_lookup_section($parent_section);
+    if (!defined $parent_section_ptr) {
+	print STDERR "doc::delete_section couldn't find parent section " .
+	    "$parent_section\n";
+	return;
+    }
+
+    # remove this section from the subsection_order list
+    my $i = 0;
+    while ($i < scalar (@{$parent_section_ptr->{'subsection_order'}})) {
+	if ($parent_section_ptr->{'subsection_order'}->[$i] eq $child) {
+	    splice (@{$parent_section_ptr->{'subsection_order'}}, $i, 1);
+	    last;
+	}
+	$i++;
+    }
+
+    # remove this section from the subsection hash
+    if (defined ($parent_section_ptr->{'subsections'}->{$child})) {
+	undef $parent_section_ptr->{'subsections'}->{$child};
+    }
+}
+
+
+# methods for dealing with metadata
+
+# set_metadata_element and get_metadata_element are for metadata
+# which should only have one value. add_meta_data and get_metadata
+# are for metadata which can have more than one value.
+
+sub set_metadata_element {
+    my $self = shift (@_);
+    my ($section, $field, $value) = @_;
+
+    $self->delete_metadata ($section, $field);
+    $self->add_metadata ($section, $field, $value);
+}
+
+# returns the first metadata value which matches field
+sub get_metadata_element {
+    my $self = shift (@_);
+    my ($section, $field) = @_;
+    my ($data);
+
+    my $section_ptr = $self->_lookup_section($section);
+    if (!defined $section_ptr) {
+	print STDERR "doc::get_metadata_element couldn't find section " .
+	    "$section\n";
+	return;
+    }
+
+    foreach $data (@{$section_ptr->{'metadata'}}) {
+	return $data->[1] if ($data->[0] eq $field);
+    }
+	
+    return undef; # was not found
+}
+
+sub add_metadata {
+    my $self = shift (@_);
+    my ($section, $field, $value) = @_;
+
+    my $section_ptr = $self->_lookup_section($section);
+    if (!defined $section_ptr) {
+	print STDERR "doc::add_metadata couldn't find section " .
+	    "$section\n";
+	return;
+    }
+    
+    push (@{$section_ptr->{'metadata'}}, [$field, $value]);
+}
+
+# returns a list of the form [value1, value2, ...]
+sub get_metadata {
+    my $self = shift (@_);
+    my ($section, $field) = @_;
+    my ($data);
+
+    my $section_ptr = $self->_lookup_section($section);
+    if (!defined $section_ptr) {
+	print STDERR "doc::get_metadata couldn't find section " .
+	    "$section\n";
+	return;
+    }
+
+    my @metadata = ();
+    foreach $data (@{$section_ptr->{'metadata'}}) {
+	push (@metadata, $data->[1]) if ($data->[0] eq $field);
+    }
+	
+    return \@metadata;
+}
+
+# returns a list of the form [[field,value],[field,value],...]
+sub get_all_metadata {
+    my $self = shift (@_);
+    my ($section) = @_;
+
+    my $section_ptr = $self->_lookup_section($section);
+    if (!defined $section_ptr) {
+	print STDERR "doc::get_all_metadata couldn't find section " .
+	    "$section\n";
+	return;
+    }
+    
+    return $section_ptr->{'metadata'};
+}
+
+# $value is optional
+sub delete_metadata {
+    my $self = shift (@_);
+    my ($section, $field, $value) = @_;
+
+    my $section_ptr = $self->_lookup_section($section);
+    if (!defined $section_ptr) {
+	print STDERR "doc::delete_metadata couldn't find section " .
+	    "$section\n";
+	return;
+    }
+
+    my $i = 0;
+    while ($i < scalar (@{$section_ptr->{'metadata'}})) {
+	if (($section_ptr->{'metadata'}->[$i]->[0] eq $field) &&
+	    (!defined $value || $section_ptr->{'metadata'}->[$i]->[1] eq $value)) {
+	    splice (@{$section_ptr->{'metadata'}}, $i, 1);
+	} else {
+	    $i++;
+	}
+    }
+}
+
+sub delete_all_metadata {
+    my $self = shift (@_);
+    my ($section) = @_;
+
+    my $section_ptr = $self->_lookup_section($section);
+    if (!defined $section_ptr) {
+	print STDERR "doc::delete_all_metadata couldn't find section " .
+	    "$section\n";
+	return;
+    }
+    
+    $section_ptr->{'metadata'} = [];
+}
+
+
+# methods for dealing with associated files
+
+sub associate_file {
+    my $self = shift (@_);
+    my ($real_filename, $assoc_filename, $mime_type) = @_;
+    $mime_type = &html::guess_mime_type ($real_filename) unless defined $mime_type;
+
+    # remove all associated files with the same name
+    $self->delete_assoc_file ($assoc_filename);
+
+    push (@{$self->{'associated_files'}}, 
+	  [$real_filename, $assoc_filename, $mime_type]);
+}
+
+# returns a list of associated files in the form
+#   [[real_filename, assoc_filename, mimetype], ...]
+sub get_assoc_files {
+    my $self = shift (@_);
+
+    return $self->{'associated_files'};
+}
+
+sub delete_assoc_file {
+    my $self = shift (@_);
+    my ($assoc_filename) = @_;
+
+    my $i=0;
+    while ($i < scalar (@{$self->{'associated_files'}})) {
+	if ($self->{'associated_files'}->[$i]->[1] eq $assoc_filename) {
+	    splice (@{$self->{'associated_files'}}, $i, 1);
+	} else {
+	    $i++;
+	}
+    }
+}
+
+
+# methods for dealing with text
+
+sub add_text {
+    my $self = shift (@_);
+    my ($section, $text) = @_;
+
+    my $section_ptr = $self->_lookup_section($section);
+    if (!defined $section_ptr) {
+	print STDERR "doc::add_text couldn't find section " .
+	    "$section\n";
+	return;
+    }
+
+    $section_ptr->{'text'} .= $text;
+}
+
+# returns the text for a section
+sub get_text {
+    my $self = shift (@_);
+    my ($section) = @_;
+
+    my $section_ptr = $self->_lookup_section($section);
+    if (!defined $section_ptr) {
+	print STDERR "doc::get_text couldn't find section " .
+	    "$section\n";
+	return "";
+    }
+
+    return $section_ptr->{'text'};
+}
+
+# returns the length of the text for a section
+sub get_text_length {
+    my $self = shift (@_);
+    my ($section) = @_;
+
+    my $section_ptr = $self->_lookup_section($section);
+    if (!defined $section_ptr) {
+	print STDERR "doc::get_text_length couldn't find section " .
+	    "$section\n";
+	return 0;
+    }
+
+    return length ($section_ptr->{'text'});
+}
+
+sub delete_text {
+    my $self = shift (@_);
+    my ($section) = @_;
+
+    my $section_ptr = $self->_lookup_section($section);
+    if (!defined $section_ptr) {
+	print STDERR "doc::delete_text couldn't find section " .
+	    "$section\n";
+	return;
+    }
+
+    $section_ptr->{'text'} = "";
+}
+
+
+1;
Index: /trunk/gsdl/perllib/docproc.pm
===================================================================
--- /trunk/gsdl/perllib/docproc.pm	(revision 4)
+++ /trunk/gsdl/perllib/docproc.pm	(revision 4)
@@ -0,0 +1,21 @@
+# document processors are used by the document reader plugins
+# to do some processing on some documents
+
+package docproc;
+
+sub new {
+    my ($class) = @_;
+    my $self = {};
+
+    return bless $self, $class;
+}
+
+sub process {
+    my $self = shift (@_);
+    my ($doc_obj) = @_;
+
+    die "docproc::process function must be implemented in sub classes\n";
+}
+
+1;
+
Index: /trunk/gsdl/perllib/docsave.pm
===================================================================
--- /trunk/gsdl/perllib/docsave.pm	(revision 4)
+++ /trunk/gsdl/perllib/docsave.pm	(revision 4)
@@ -0,0 +1,73 @@
+# This document processor saves a document in the 
+# archives directory of a collection
+
+
+package docsave;
+
+use arcinfo;
+use docproc;
+use util;
+
+
+sub BEGIN {
+    @ISA = ('docproc');
+}
+
+sub new {
+    my ($class, $collection, $archive_info) = @_;
+    my $self = new docproc ();
+
+    $self->{'collection'} = $collection;
+    $self->{'archive_info'} = $archive_info;
+
+    return bless $self, $class;
+}
+
+sub process {
+    my $self = shift (@_);
+    my ($doc_obj) = @_;
+
+    my $archive_dir = "$ENV{'GSDLHOME'}/collect/$self->{'collection'}/archives";
+    my $OID = $doc_obj->get_OID();
+    $OID = "NULL" unless defined $OID;
+
+    # get the document's directory by inserting a / every 1-8 characters
+    # and removing the last /
+    my $doc_dir = $OID;
+    $doc_dir =~ s/(.{1,8})/$1\//g;
+    $doc_dir =~ s/\/$//;
+    $doc_dir .= ".dir";
+    &util::mk_all_dir ("$archive_dir/$doc_dir");
+
+    # copy all the associated files, add this information as metadata
+    # to the document
+    my @assoc_files = ();
+    foreach $assoc_file (@{$doc_obj->get_assoc_files()}) {
+	if (-e $assoc_file->[0]) {
+	    &util::cp ($assoc_file->[0], "$archive_dir/$doc_dir/$assoc_file->[1]");
+	    $doc_obj->add_metadata ($doc_obj->get_top_section(), 
+				    "gsdlassocfile", 
+				    "$assoc_file->[1]:$assoc_file->[2]");
+	} else {
+	    print STDERR "docsave::process couldn't copy the associated file " .
+		"$assoc_file->[0] to $archive_dir/$doc_dir/$assoc_file->[1]\n"
+	}
+    }
+
+    # save this document
+    if (!open (OUTDOC, ">$archive_dir/$doc_dir/doc.gml")) {
+	print STDERR "docsave::process could not write to file " . 
+	    "$archive_dir/$doc_dir/doc.gml\n";
+	return;
+    }
+    $doc_obj->output_section('docsave::OUTDOC', $doc_obj->get_top_section());
+    close OUTDOC;
+
+    # store reference in the archive_info
+    $self->{'archive_info'}->add_info($OID, 
+				      "$doc_dir/doc.gml");
+    
+}
+
+
+1;
Index: /trunk/gsdl/perllib/html.pm
===================================================================
--- /trunk/gsdl/perllib/html.pm	(revision 4)
+++ /trunk/gsdl/perllib/html.pm	(revision 4)
@@ -0,0 +1,248 @@
+package html;
+
+# htmlsafe(TEXT)
+# 
+# Converts SGML meta characters in TEXT to entity references.
+# 
+sub htmlsafe
+{
+    $_[0] =~ s/&/&amp;/og;
+    $_[0] =~ s/</&lt;/og;
+    $_[0] =~ s/>/&gt;/og;
+}
+
+# urlsafe(TEXT)
+# 
+# Converts characters not allowed in a URL to their hex representation.
+# 
+sub urlsafe
+{
+    $_[0] =~ s/[\x09\x20\x22\x3c\x3e\x5b\x5c\x5d\x5e\x60\x7b\x7c\x7d\x7e\?\=\&\+_]/sprintf("%%%2x", ord($&))/ge;
+}
+
+# dmsafe
+#
+# Converts characters which could make display.pl or html crack to their entity references.
+# Don't use this on dm macros as they will no longer be recognisable by dm (which
+# by the way, is the whole idea).
+# -- Rodger 12/1/98
+sub dmsafe {
+    my ($s) = $_[0];
+    $s =~ s/&/&amp;/og; # for html
+    $s =~ s/</&lt;/og;  # for html
+    $s =~ s/>/&gt;/og;  # for html
+    $s =~ s/\_/&#095;/og; # for dm (we have a convention of starting macros with _
+    $s =~ s/\"/&quot;/og; # for html (don't want to be interpreted as a quote)
+    $s =~ s/\{/&#123;/og; # for dm blocks
+    $s =~ s/\}/&#125;/og; # for dm blocks
+    return $s;
+}
+
+
+# named entry to the standard html font
+%charnetosf = ("Agrave"=> "192",  "Aacute"=> "193",  "Acirc" => "194",  "Atilde"=> "195",
+	       "Auml"  => "196",  "Aring" => "197",  "AElig" => "198",  "Ccedil"=> "199",
+	       "Egrave"=> "200",  "Eacute"=> "201",  "Ecirc" => "202",  "Euml"  => "203",
+	       "Igrave"=> "204",  "Iacute"=> "205",  "Icirc" => "206",  "Iuml"  => "207",
+	       "ETH"   => "208",  "Ntilde"=> "209",  "Ograve"=> "210",  "Oacute"=> "211",
+	       "Ocirc" => "212",  "Otilde"=> "213",  "Ouml"  => "214",  
+	       "Oslash"=> "216",  "Ugrave"=> "217",  "Uacute"=> "218",  "Ucirc" => "219",
+	       "Uuml"  => "220",  "Yacute"=> "221",  "THORN" => "222",  "szlig" => "223",
+	       "agrave"=> "224",  "aacute"=> "225",  "acirc" => "226",  "atilde"=> "227",
+	       "auml"  => "228",  "aring" => "229",  "aelig" => "230",  "ccedil"=> "231",
+	       "egrave"=> "232",  "eacute"=> "233",  "ecirc" => "234",  "euml"  => "235",
+	       "igrave"=> "236",  "iacute"=> "237",  "icirc" => "238",  "iuml"  => "239",
+	       "eth"   => "240",  "ntidle"=> "241",  "ograve"=> "242",  "oacute"=> "243",
+	       "ocirc" => "244",  "otilde"=> "245",  "ouml"  => "246",  
+	       "oslash"=> "248",  "ugrave"=> "249",  "uacute"=> "250",  "ucirc" => "251",
+	       "uuml"  => "252",  "yacute"=> "253",  "thorn" => "254",  "yuml"  => "255");
+
+%symnetosf = ("quot"  => "34",   "amp"   => "38",   "lt"    => "60",   "gt"    => "62",
+	      "nbsp"  => "160",  "iexcl" => "161",  "cent"  => "162",  "pound" => "163",
+	      "curren"=> "164",  "yen"   => "165",  "brvbar"=> "166",  "sect"  => "167",
+	      "uml"   => "168",  "copy"  => "169",  "ordf"  => "170",  "laquo" => "171",
+	      "not"   => "172",  "shy"   => "173",  "reg"   => "174",  "macr"  => "175",
+	      "deg"   => "176",  "plusmn"=> "177",  "sup2"  => "178",  "sup3"  => "179",
+	      "acute" => "180",  "micro" => "181",  "para"  => "182",  "middot"=> "183",
+	      "cedil" => "184",  "sup1"  => "185",  "ordm"  => "186",  "raquo" => "187",
+	      "frac14"=> "188",  "frac12"=> "189",  "frac34"=> "190",  "iquest"=> "191",
+	      "times" => "215",  "divide"=> "247");
+
+
+
+# standard font to plain text
+%sftotxt = ("32"  => " ",  "33"  => "!",  "34"  => "\"",  "35"  => "\#",  "36"  => "\$",
+	    "37"  => "\%", "38"  => "&",  "39"  => "'",   "40"  => "(",   "41"  => ")",
+	    "42"  => "*",  "43"  => "+",  "44"  => ",",   "45"  => "-",   "46"  => ".",
+	    "47"  => "/",  "48"  => "0",  "49"  => "1",   "50"  => "2",   "51"  => "3",
+	    "52"  => "4",  "53"  => "5",  "54"  => "6",   "55"  => "7",   "56"  => "8",
+	    "57"  => "9",  "58"  => ":",  "59"  => ";",   "60"  => "<",   "61"  => "=",
+	    "62"  => ">",  "63"  => "?",  "64"  => "\@",  "65"  => "A",   "66"  => "B",
+	    "57"  => "9",  "58"  => ":",  "59"  => ";",   "61"  => "=",
+	    "63"  => "?",  "64"  => "\@", "65"  => "A",  "66"  => "B",
+	    "67"  => "C",  "68"  => "D",  "69"  => "E",   "70"  => "F",   "71"  => "G",
+	    "72"  => "H",  "73"  => "I",  "74"  => "J",   "75"  => "K",   "76"  => "L",
+	    "77"  => "M",  "78"  => "N",  "79"  => "O",   "80"  => "P",   "81"  => "Q",
+	    "82"  => "R",  "83"  => "S",  "84"  => "T",   "85"  => "U",   "86"  => "V",
+	    "87"  => "W",  "88"  => "X",  "89"  => "Y",   "90"  => "Z",   "91"  => "[",
+	    "92"  => "\\", "93"  => "]",  "94"  => "^",   "95"  => "_",   "96"  => "`",
+	    "97"  => "a",  "98"  => "b",  "99"  => "c",   "100" => "d",   "101" => "e",
+	    "102" => "f",  "103" => "g",  "104" => "h",   "105" => "i",   "106" => "j",
+	    "107" => "k",  "108" => "l",  "109" => "m",   "110" => "n",   "111" => "o",
+	    "112" => "p",  "113" => "q",  "114" => "r",   "115" => "s",   "116" => "t",
+	    "117" => "u",  "118" => "v",  "119" => "w",   "120" => "x",   "121" => "y",
+	    "122" => "z",  "123" => "{",  "124" => "|",   "125" => "}",   "126" => "~",
+	    "130" => ",",  "131" => "f",  "132" => "\"",   "133" => "...", "139" => "<",
+	    "140" => "OE", "145" => "'",  "146" => "'",   "147" => "\"",   "148" => "\"",
+	    "149" => "o",  "150" => "--", "151" => "-",   "152" => "~",   "153" => "TM",
+	    "155" => ">",  "156" => "oe", "159" => "Y",   "160" => " ",   "178" => "2",
+	    "179" => "3",  "185" => "1",  "188" => "1/4", "189" => "1/2", "190" => "3/4",
+	    "192" => "A",  "193" => "A",  "194" => "A",   "195" => "A",   "196" => "A",
+	    "197" => "A",  "198" => "AE", "199" => "C",   "200" => "E",   "201" => "E",
+	    "202" => "E",  "203" => "E",  "204" => "I",   "205" => "I",   "206" => "I",
+	    "207" => "I",  "208" => "D",  "209" => "N",   "210" => "O",   "211" => "O",
+	    "212" => "O",  "213" => "O",  "214" => "O",   "215" => "*",   "216" => "O",
+	    "217" => "U",  "218" => "U",  "219" => "U",   "220" => "U",   "221" => "Y",
+	    "223" => "ss", "224" => "a",  "225" => "a",   "226" => "a",   "227" => "a",
+	    "228" => "a",  "229" => "a",  "230" => "ae",  "231" => "c",   "232" => "e",
+	    "233" => "e",  "234" => "e",  "235" => "e",   "236" => "i",   "237" => "i",
+	    "238" => "i",  "239" => "i",  "241" => "n",   "242" => "o",   "243" => "o",
+	    "244" => "o",  "245" => "o",  "246" => "o",   "247" => "/",   "248" => "o",
+	    "249" => "u",  "250" => "u",  "251" => "u",   "252" => "u",   "253" => "y",
+	    "255" => "y",  "8218" => ",");
+
+
+# returns the character in the standard html font. It assumes that the
+# & and ; have been stripped off the string.
+sub getcharequiv {
+    my ($entity, $convertsymbols) = @_;
+
+    # replace &#8218 with comma (",")
+    $entity =~ s/8218/044/;
+
+    # a numeric entity
+    if ($entity =~ /^\#0*(\d+)/) {
+	return pack("c", $1);
+    }
+    
+    # a named character entity
+    if (defined $charnetosf{$entity}) {
+	return pack("c", $charnetosf{$entity});
+    }
+
+    # a named symbol entity
+    if ($convertsymbols && defined $symnetosf{$entity}) {
+	return pack("c", $symnetosf{$entity});
+    }
+
+    return "&$entity;"; # unknown character
+}
+
+# convert character entities from named equivalents to html font
+sub convertcharentities {
+    # args: the text that you want to convert
+
+    $_[0] =~ s/&([^;]+);/&getcharequiv($1,0)/ge;
+}
+
+# convert any entities from named equivalents to html font
+sub convertallentities {
+    # args: the text that you want to convert
+
+    $_[0] =~ s/&([^;]+);/&getcharequiv($1,1)/ge;
+}
+
+sub html2txt {
+    # args: the text that you want converted to ascii, 
+    # and whether to strip out sgml tags
+
+    # strip out sgml tags if needed
+    $_[0] =~ s/<[^>]>//g if $_[1];
+
+    # convert the char entities to the standard html font
+    &convertcharentities($_[0]); 
+    
+    # convert the html character set to a plain ascii character set
+    my $pos = 0;
+    while ($pos < length($_[0])) {
+	my $charnum = ord(substr($_[0], $pos, 1));
+	if ($charnum >= 32) { # only convert characters above #32
+	    my $replacechars = " ";
+	    $replacechars = $sftotxt{$charnum} if defined $sftotxt{$charnum};
+	    substr($_[0], $pos, 1) = $replacechars;
+	    $pos += length ($replacechars);
+
+	} else {
+	    $pos ++;
+	}
+    }
+}  
+
+
+sub guess_mime_type {
+    my ($filename) = @_;
+
+    my ($fileext) = $filename =~ /\.(\w+)$/;
+    return "unknown" unless defined $fileext;
+
+    my %mime_type = ("ai"=>"application/postscript", "aif"=>"audio/x-aiff", 
+		     "aifc"=>"audio/x-aiff", "aiff"=>"audio/x-aiff", 
+		     "au"=>"audio/basic", "avi"=>"video/x-msvideo",
+		     "bcpio"=>"application/x-bcpio", "bin"=>"application/octet-stream", 
+		     "cdf"=>"application/x-netcdf", "class"=>"application/octet-stream", 
+		     "cpio"=>"application/x-cpio", "cpt"=>"application/mac-compactpro",
+		     "csh"=>"application/x-csh", "dcr"=>"application/x-director", 
+		     "dir"=>"application/x-director", "dms"=>"application/octet-stream", 
+		     "doc"=>"application/msword", "dvi"=>"application/x-dvi",
+		     "dxr"=>"application/x-director", "eps"=>"application/postscript", 
+		     "etx"=>"text/x-setext",
+		     "exe"=>"application/octet-stream", "gif"=>"image/gif",
+		     "gtar"=>"application/x-gtar", "hdf"=>"application/x-hdf",
+		     "hqx"=>"application/mac-binhex40", "htm"=>"text/html",
+		     "html"=>"text/html", "ice"=>"x-conference/x-cooltalk",
+		     "ief"=>"image/ief", "jpe"=>"image/jpeg",
+		     "jpeg"=>"image/jpeg", "jpg"=>"image/jpeg",
+		     "kar"=>"audio/midi", "latex"=>"application/x-latex",
+		     "lha"=>"application/octet-stream", "lzh"=>"application/octet-stream",
+		     "man"=>"application/x-troff-man", "mcf"=>"image/vasa",
+		     "me"=>"application/x-troff-me", "mid"=>"audio/midi",
+		     "midi"=>"audio/midi", "mif"=>"application/x-mif",
+		     "mov"=>"video/quicktime", "movie"=>"video/x-sgi-movie",
+		     "mp2"=>"audio/mpeg", "mpe"=>"video/mpeg",
+		     "mpeg"=>"video/mpeg", "mpg"=>"video/mpeg",
+		     "mpga"=>"audio/mpeg", "ms"=>"application/x-troff-ms",
+		     "nc"=>"application/x-netcdf", "oda"=>"application/oda",
+		     "pbm"=>"image/x-portable-bitmap", "pdb"=>"chemical/x-pdb",
+		     "pdf"=>"application/pdf", "pgm"=>"image/x-portable-graymap",
+		     "png"=>"image/png", "pnm"=>"image/x-portable-anymap",
+		     "ppm"=>"image/x-portable-pixmap", "ppt"=>"application/powerpoint",
+		     "ps"=>"application/postscript", "qt"=>"video/quicktime",
+		     "ra"=>"audio/x-realaudio", "ram"=>"audio/x-pn-realaudio",
+		     "ras"=>"image/x-cmu-raster", "rgb"=>"image/x-rgb",
+		     "roff"=>"application/x-troff", "rpm"=>"audio/x-pn-realaudio-plugin",
+		     "rtf"=>"application/rtf", "rtx"=>"text/richtext",
+		     "sgm"=>"text/x-sgml", "sgml"=>"text/x-sgml",
+		     "sh"=>"application/x-sh", "shar"=>"application/x-shar",
+		     "sit"=>"application/x-stuffit", "skd"=>"application/x-koan",
+		     "skm"=>"application/x-koan", "skp"=>"application/x-koan",
+		     "skt"=>"application/x-koan", "snd"=>"audio/basic",
+		     "src"=>"application/x-wais-source", "sv4cpio"=>"application/x-sv4cpio",
+		     "sv4crc"=>"application/x-sv4crc", "t"=>"application/x-troff",
+		     "tar"=>"application/x-tar", "tcl"=>"application/x-tcl",
+		     "tex"=>"application/x-tex", "texi"=>"application/x-texinfo",
+		     "texinfo"=>"application/x-texinfo", "tif"=>"image/tiff",
+		     "tiff"=>"image/tiff", "tr"=>"application/x-troff",
+		     "tsv"=>"text/tab-separated-values", "txt"=>"text/plain",
+		     "ustar"=>"application/x-ustar", "vcd"=>"application/x-cdlink",
+		     "vrml"=>"x-world/x-vrml", "wav"=>"audio/x-wav",
+		     "wrl"=>"x-world/x-vrml", "xbm"=>"image/x-xbitmap",
+		     "xpm"=>"image/x-xpixmap", "xwd"=>"image/x-xwindowdump",
+		     "xyz"=>"chemical/x-pdb", "zip"=>"application/zip");
+
+    return $mime_type{$fileext} if (defined $mime_type{$fileext});
+
+    return "unknown";
+}
+
+
+1;
Index: /trunk/gsdl/perllib/lang.pm
===================================================================
--- /trunk/gsdl/perllib/lang.pm	(revision 4)
+++ /trunk/gsdl/perllib/lang.pm	(revision 4)
@@ -0,0 +1,176 @@
+# this module translates language descriptions to and from
+# the ISO 639 standard
+
+package lang;
+
+@trans = (["ab", "Abkhazian"],      ["om", "Afan Oromo"],
+	  ["om", "Oromo"],          ["aa", "Afar"],
+	  ["af", "Afrikaans"],      ["sq", "Albanian"],
+	  ["am", "Amharic"],        ["ar", "Arabic"],
+	  ["hy", "Armenian"],       ["as", "Assamese"],
+	  ["ay", "Aymara"],         ["az", "Azerbaijani"],
+
+	  ["ba", "Bashkir"],        ["eu", "Basque"],
+	  ["bn", "Bengali"],        ["bn", "Bangla"],
+	  ["dz", "Bhutani"],        ["bh", "Bihari"],
+	  ["bi", "Bislama"],        ["br", "Breton"],
+	  ["bg", "Bulgarian"],      ["my", "Burmese"],
+	  ["be", "Byelorussian"],
+
+	  ["km", "Cambodian"],      ["ca", "Catalan"],
+	  ["zh", "Chinese"],        ["co", "Corsican"],
+	  ["hr", "Croatian"],       ["cs", "Czech"],
+
+	  ["da", "Danish"],         ["nl", "Dutch"],
+
+	  ["en", "English"],        ["eo", "Esperanto"],
+	  ["et", "Estonian"],       ["fo", "Faroese"],
+	  ["fj", "Fijian"],         ["fi", "Finnish"],
+	  ["fr", "French"],         ["fy", "Frisian"],
+
+	  ["gl", "Galician"],       ["ka", "Georgian"],
+	  ["de", "German"],         ["el", "Greek"],
+	  ["kl", "Greenlandic"],    ["gn", "Guarani"],
+	  ["gu", "Gujarati"],
+
+	  ["ha", "Hausa"],          ["iw", "Hebrew"],
+	  ["hi", "Hindi"],          ["hu", "Hungarian"],
+
+	  ["is", "Icelandic"],      ["in", "Indonesian"],
+	  ["ia", "Interlingua"],    ["ie", "Interlingue"],
+	  ["ik", "Inupiak"],        ["ga", "Irish"],
+	  ["it", "Italian"],
+
+	  ["ja", "Japanese"],       ["jv", "Javanese"],
+
+	  ["kn", "Kannada"],        ["ks", "Kashmiri"],
+	  ["kk", "Kazakh"],         ["rw", "Kinyarwanda"],
+	  ["ky", "Kirghiz"],        ["rn", "Kirundi"],
+	  ["ko", "Korean"],         ["ku", "Kurdish"],
+
+	  ["lo", "Laothian"],       ["la", "Latin"],
+	  ["lv", "Latvian"],        ["lv", "Lettish"],
+	  ["ln", "Lingala"],        ["lt", "Lithuanian"],
+
+	  ["mk", "Macedonian"],     ["mg", "Malagasy"],
+	  ["ms", "Malay"],          ["ml", "Malayalam"],
+	  ["mt", "Maltese"],        ["mi", "Maori"],
+	  ["mr", "Marathi"],        ["mo", "Moldavian"],
+	  ["mn", "Mongolian"],
+
+	  ["na", "Nauru"],          ["ne", "Nepali"],
+	  ["no", "Norwegian"],
+
+	  ["oc", "Occitan"],        ["or", "Oriya"],
+
+	  ["ps", "Pashto"],         ["ps", "Pushto"],
+	  ["fa", "Persian"],        ["pl", "Polish"],
+	  ["pt", "Portuguese"],     ["pa", "Punjabi"],
+
+	  ["qu", "Quechua"],
+
+	  ["rm", "Rhaeto-Romance"], ["ro", "Romanian"],
+	  ["ru", "Russian"],
+
+	  ["sm", "Samoan"],         ["sg", "Sangho"],
+	  ["sa", "Sanskrit"],       ["gd", "Scots Gaelic"],
+	  ["sr", "Serbian"],        ["sh", "Serbo-Croatian"],
+	  ["st", "Sesotho"],        ["tn", "Setswana"],
+	  ["sn", "Shona"],          ["sd", "Sindhi"],
+	  ["si", "Singhalese"],     ["ss", "Siswati"],
+	  ["sk", "Slovak"],         ["sl", "Slovenian"],
+	  ["so", "Somali"],         ["es", "Spanish"],
+	  ["su", "Sundanese"],      ["sw", "Swahili"],
+	  ["sv", "Swedish"],
+
+	  ["tl", "Tagalog"],        ["tg", "Tajik"],
+	  ["ta", "Tamil"],          ["tt", "Tatar"],
+	  ["te", "Telugu"],         ["th", "Thai"],
+	  ["bo", "Tibetan"],        ["ti", "Tigrinya"],
+	  ["to", "Tonga"],          ["ts", "Tsonga"],
+	  ["tr", "Turkish"],        ["tk", "Turkmen"],
+	  ["tw", "Twi"],
+
+	  ["uk", "Ukrainian"],      ["ur", "Urdu"],
+	  ["uz", "Uzbek"],
+
+	  ["vi", "Vietnamese"],     ["vo", "Volapuk"],
+
+	  ["cy", "Welsh"],          ["wo", "Wolof"],
+
+	  ["xh", "Xhosa"],
+
+	  ["ji", "Yiddish"],        ["yo", "Yoruba"],
+
+	  ["zu", "Zulu"]);
+
+
+sub one_english_to_iso639 {
+    my ($english) = @_;
+    my $iso639 = "";
+
+    foreach $lang (@trans) {
+	my $code = $lang->[0];
+	my $enname = $lang->[1];
+	if ($english =~ /\s*($code|$enname)\s*$/i) {
+	    # found the language
+	    $iso639 = $code;
+	    last;
+
+	}
+    }
+
+    return $iso639;
+}
+
+
+sub english_to_iso639 {
+    my ($englishlist) = @_;
+    my ($english);
+
+    my @englishlist = split (",", $englishlist);
+    my @iso639list = ();
+
+    foreach $english (@englishlist) {
+	push (@iso639list, &one_english_to_iso639($english));
+    }
+
+    return join (",", @iso639list);
+}
+
+
+sub one_iso639_to_english {
+    my ($iso639) = @_;
+    my $english = "";
+
+    foreach $lang (@trans) {
+	my $code = $lang->[0];
+	my $enname = $lang->[1];
+	if ($iso639 =~ /\s*($code|$enname)\s*$/i) {
+	    # found the language
+	    $english = $enname;
+	    last;
+
+	}
+    }
+
+    return $enname;
+}
+
+
+sub iso639_to_english {
+    my ($iso639list) = @_;
+    my ($iso639);
+
+    my @iso639list = split (",", $iso639list);
+    my @englishlist = ();
+
+    foreach $iso639 (@iso639list) {
+	push (@englishlist, &one_iso639_to_english($iso639));
+    }
+
+    return join (",", @englishlist);
+}
+
+
+1;
Index: /trunk/gsdl/perllib/mgbuilder.pm
===================================================================
--- /trunk/gsdl/perllib/mgbuilder.pm	(revision 4)
+++ /trunk/gsdl/perllib/mgbuilder.pm	(revision 4)
@@ -0,0 +1,383 @@
+# MGBuilder object
+#
+
+package mgbuilder;
+
+use cfgread;
+use colcfg;
+use plugin;
+use util;
+
+$maxdocsize = 12000;
+
+%wanted_index_files = ('td'=>1,
+		       't'=>1,
+		       'idb'=>1,
+		       'ib1'=>1,
+		       'ib2'=>1,
+		       'ib3'=>1,
+		       'i'=>1,
+		       'ip'=>1,
+		       'tiw'=>1,
+		       'wa'=>1);
+
+
+sub new {
+    my ($class, $collection, $source_dir, $build_dir, $verbosity) = @_;
+
+    # create an mgbuilder object
+    my $self = bless {'collection'=>$collection,
+		      'source_dir'=>$source_dir,
+		      'build_dir'=>$build_dir,
+		      'verbosity'=>$verbosity}, $class;
+
+
+    # read in the collection configuration file
+    if (!-e "$ENV{'GSDLHOME'}/collect/$collection/collect.cfg") {
+	die "mgbuilder::new - couldn't find collect.cfg for collection $collection\n";
+    }
+    $self->{'collect_cfg'} = &colcfg::read_collect_cfg ("$ENV{'GSDLHOME'}/collect/" .
+							"$collection/collect.cfg");
+
+
+    # get the list of plugins for this collection
+    my @plugins = (); # some good choice of plugins .... ????
+    if (defined $self->{'collect_cfg'}->{'plugins'}) {
+	@plugins = @{$self->{'collect_cfg'}->{'plugins'}};
+    }
+    
+
+    # load all the plugins
+    $self->{'pluginfo'} = &plugin::load_plugins ($collection, \@plugins);
+    if (scalar(@{$self->{'pluginfo'}}) == 0) {
+	print STDERR "No plugins were loaded.\n";
+	die "\n";
+    }
+    
+
+    # load up the document processor for building
+    # if a buildproc class has been created for this collection, use it
+    # otherwise, use the mg buildproc
+    my ($buildprocdir, $buildproctype);
+    if (-e "$ENV{'GSDLHOME'}/collect/$collection/lib/${collection}buildproc.pm") {
+	$buildprocdir = "$ENV{'GSDLHOME'}/collect/$collection/lib";
+	$buildproctype = "${collection}buildproc";
+    } else {
+	$buildprocdir = "$ENV{'GSDLHOME'}/lib";
+	$buildproctype = "mgbuildproc";
+    }
+    require "$buildprocdir/$buildproctype.pm";
+
+    eval("\$self->{'buildproc'} = new $buildproctype(\$collection, " .
+	 "\$source_dir, \$build_dir, \$verbosity)");
+    die "$@" if $@;
+
+
+    return $self;
+}
+
+sub init {
+    my $self = shift (@_);
+
+    # remove any old builds
+    &util::rm_r($self->{'build_dir'});
+    &util::mk_all_dir($self->{'build_dir'});
+        
+    # make the text directory
+    my $textdir = "$self->{'build_dir'}/text";
+    &util::mk_all_dir($textdir);
+}
+
+sub compress_text {
+    my $self = shift (@_);
+    my $exedir = "$ENV{'GSDLHOME'}/bin/$ENV{'GSDLOS'}";
+    my $exe = &util::get_os_exe ();
+
+    &util::mk_all_dir (&util::filename_cat($self->{'build_dir'}, "text"));
+    my $basefilename = "text/$self->{'collection'}";
+    my $fulltextprefix = &util::filename_cat ($self->{'build_dir'}, $basefilename);
+
+    my $osextra = "";
+    if ($ENV{'GSDLOS'} =~ /^windows$/i) {
+	$fulltextprefix =~ s/\//\\/g;
+    } else {
+	$osextra = " -d /";
+    }
+
+    print STDERR "\n*** creating the compressed text\n" if ($self->{'verbosity'} >= 1);
+
+    # set up the document processor
+    $self->{'buildproc'}->set_output_handle ('mgbuilder::PIPEOUT');
+    $self->{'buildproc'}->set_mode ('text');
+    $self->{'buildproc'}->set_index ('section:text');
+    
+    # collect the statistics for the text
+    # -b $maxdocsize sets the maximum document size to be 12 meg
+    print STDERR "\n    collecting text statistics\n"  if ($self->{'verbosity'} >= 1);
+    if (!-e "$exedir/mg_passes$exe" || !open (PIPEOUT, 
+            "| $exedir/mg_passes$exe -f $fulltextprefix -b $maxdocsize -T1 $osextra")) {
+	die "mgbuilder::compress_text - couldn't run $exedir/mg_passes$exe\n";
+    }
+    $self->{'buildproc'}->reset();
+    &plugin::read ($self->{'pluginfo'}, $self->{'source_dir'}, 
+		   "", {}, $self->{'buildproc'});
+    close (PIPEOUT);
+
+    # create the compression dictionary
+    # the compression dictionary is built by assuming the stats are from a seed
+    # dictionary (-S), if a novel word is encountered it is spelled out (-H),
+    # and the resulting dictionary must be less than 5 meg with the most frequent
+    # words being put into the dictionary first (-2 -k 5120)
+    print STDERR "\n    creating the compression dictionary\n"  if ($self->{'verbosity'} >= 1);
+    if (!-e "$exedir/mg_compression_dict$exe") {
+	die "mgbuilder::compress_text - couldn't run $exedir/mg_compression_dict$exe\n";
+    }
+    system ("$exedir/mg_compression_dict$exe -f $fulltextprefix -S -H -2 -k 5120 $osextra");
+
+    # compress the text
+    # -b $maxdocsize sets the maximum document size to be 12 meg
+    print STDERR "\n    compressing the text\n"  if ($self->{'verbosity'} >= 1);
+    if (!-e "$exedir/mg_passes$exe" || !open (PIPEOUT, 
+            "| $exedir/mg_passes$exe -f $fulltextprefix -b $maxdocsize -T2 $osextra")) {
+	die "mgbuilder::compress_text - couldn't run $exedir/mg_passes$exe\n";
+    }
+    $self->{'buildproc'}->reset();
+    &plugin::read ($self->{'pluginfo'}, $self->{'source_dir'}, 
+		   "", {}, $self->{'buildproc'});
+    close (PIPEOUT);
+}
+
+sub build_indexes {
+    my $self = shift (@_);
+    my $indexes = $self->{'collect_cfg'}->{'indexes'};
+
+    # create the mapping between the index descriptions 
+    # and their directory names
+    $self->{'index_mapping'} = $self->create_index_mapping ($indexes);
+
+    # build each of the indexes
+    foreach $index (@$indexes) {
+	print STDERR "\n*** building index $index in subdirectory " . 
+	    "$self->{'index_mapping'}->{$index}\n" if ($self->{'verbosity'} >= 1);
+	$self->build_index($index);
+    }
+}
+
+# creates directory names for each of the index descriptions
+sub create_index_mapping {
+    my $self = shift (@_);
+    my ($indexes) = @_;
+
+    my %mapping = ();
+    
+    # dirnames is used to check for collisions. Start this off
+    # with the manditory directory names
+    my %dirnames = ('text'=>'text',
+		    'extra'=>'extra');
+    
+    foreach $index (@$indexes) {
+	my ($level, $fields) = split (":", $index);
+	my @fields = split (",", $fields);
+	splice (@fields, 2); # just want first two fields
+	
+	# the directory names starts with the first character of the index level
+	my ($dirname) = $level =~ /^(.)/;
+
+	# next comes a processed version of the first two fields in the index
+	# the processed version contains the first character and the next
+	# consonant
+	map {s/^(.).*?([bcdfghjklmnpqrstvwxyz]).*$/$1$2/i;} @fields;
+	$dirname .= join("", @fields);
+
+	# convert the directory name to lowercase
+	$dirname = lc ($dirname);
+
+	# add a number to make this directory name unique
+	if (defined $dirnames{$dirname}) {
+	    my $num = 1;
+	    while (defined $dirnames{"$dirname$num"}) {
+		$num++;
+	    }
+	    $dirname .= $num;
+	}
+	
+	$mapping{$index} = $dirname;
+	$dirnames{$dirname} = $index;
+    }
+
+    return \%mapping;
+}
+
+
+sub build_index {
+    my $self = shift (@_);
+    my ($index) = @_;
+
+    # get the full index directory path and make sure it exists
+    my $indexdir = $self->{'index_mapping'}->{$index};
+    &util::mk_all_dir (&util::filename_cat($self->{'build_dir'}, $indexdir));
+    my $fullindexprefix = &util::filename_cat ($self->{'build_dir'}, $indexdir, 
+					       $self->{'collection'});
+    my $fulltextprefix = &util::filename_cat ($self->{'build_dir'}, "text", 
+					       $self->{'collection'});
+
+    # get any os specific stuff
+    my $exedir = "$ENV{'GSDLHOME'}/bin/$ENV{'GSDLOS'}";
+    my $exe = &util::get_os_exe ();
+    my $osextra = "";
+    if ($ENV{'GSDLOS'} =~ /^windows$/i) {
+	$fullindexprefix =~ s/\//\\/g;
+    } else {
+	$osextra = " -d /";
+    }
+
+    # get the index level from the index description
+    # the index will be level 2 unless we are building a
+    # paragraph level index
+    my $index_level = 2;
+    $index_level = 3 if $index =~ /^paragraph/i;
+
+    # set up the document processor
+    $self->{'buildproc'}->set_output_handle ('mgbuilder::PIPEOUT');
+    $self->{'buildproc'}->set_mode ('text');
+    $self->{'buildproc'}->set_index ($index);
+
+
+    # Build index dictionary. Uses verbatim stem method
+    print STDERR "\n    creating index dictionary\n"  if ($self->{'verbosity'} >= 1);
+    if (!-e "$exedir/mg_passes$exe" || !open (PIPEOUT, 
+	    "| $exedir/mg_passes$exe -f $fullindexprefix -b $maxdocsize " .
+	    "-$index_level -m 32 -s 0 -G -t 10 -N1 $osextra")) {
+	die "mgbuilder::build_index - couldn't run $exedir/mg_passes$exe\n";
+    }
+    $self->{'buildproc'}->reset();
+    &plugin::read ($self->{'pluginfo'}, $self->{'source_dir'}, 
+		   "", {}, $self->{'buildproc'});
+    close (PIPEOUT);
+
+    # create the perfect hash function
+    if (!-e "$exedir/mg_perf_hash_build$exe") {
+	die "mgbuilder::build_index - couldn't run $exedir/mg_perf_hash_build$exe\n";
+    }
+    system ("$exedir/mg_perf_hash_build$exe -f $fullindexprefix $osextra");
+
+    # invert the text
+    print STDERR "\n    inverting the text\n"  if ($self->{'verbosity'} >= 1);
+    if (!-e "$exedir/mg_passes$exe" || !open (PIPEOUT, 
+	    "| $exedir/mg_passes$exe -f $fullindexprefix -b $maxdocsize " .
+	    "-$index_level -c 3 -G -t 10 -N2 $osextra")) {
+	die "mgbuilder::build_index - couldn't run $exedir/mg_passes$exe\n";
+    }
+    $self->{'buildproc'}->reset();
+    &plugin::read ($self->{'pluginfo'}, $self->{'source_dir'}, 
+		   "", {}, $self->{'buildproc'});
+    close (PIPEOUT);
+
+    # create the weights file
+    print STDERR "\n    create the weights file\n"  if ($self->{'verbosity'} >= 1);
+    if (!-e "$exedir/mg_weights_build$exe") {
+	die "mgbuilder::build_index - couldn't run $exedir/mg_weights_build$exe\n";
+    }
+    system ("$exedir/mg_weights_build$exe -f $fullindexprefix -t $fulltextprefix $osextra");
+
+    # create 'on-disk' stemmed dictionary
+    print STDERR "\n    creating 'on-disk' stemmed dictionary\n"  if ($self->{'verbosity'} >= 1);
+    if (!-e "$exedir/mg_invf_dict$exe") {
+	die "mgbuilder::build_index - couldn't run $exedir/mg_invf_dict$exe\n";
+    }
+    system ("$exedir/mg_invf_dict$exe -f $fullindexprefix $osextra");
+
+
+    # creates stem index files for the various stemming methods
+    print STDERR "\n    creating stem indexes\n"  if ($self->{'verbosity'} >= 1);
+    if (!-e "$exedir/mg_stem_idx$exe") {
+	die "mgbuilder::build_index - couldn't run $exedir/mg_stem_idx$exe\n";
+    }
+    system ("$exedir/mg_stem_idx$exe -b 4096 -s1 -f $fullindexprefix $osextra");
+    system ("$exedir/mg_stem_idx$exe -b 4096 -s2 -f $fullindexprefix $osextra");
+    system ("$exedir/mg_stem_idx$exe -b 4096 -s3 -f $fullindexprefix $osextra");
+
+    
+    # remove unwanted files
+    my $tmpdir = &util::filename_cat ($self->{'build_dir'}, $indexdir);
+    opendir (DIR, $tmpdir) || die
+	"mgbuilder::build_index - couldn't read directory $tmpdir\n";
+    foreach $file (readdir(DIR)) {
+	next if $file =~ /^\./;
+	my ($suffix) = $file =~ /\.([^\.]+)$/;
+	if (defined $suffix && !defined $wanted_index_files{$suffix}) {
+	    # delete it!
+#	    print STDERR "deleting $file\n";
+	    &util::rm (&util::filename_cat ($tmpdir, $file));
+	}
+    }
+    closedir (DIR);
+}
+
+sub make_infodatabase {
+    my $self = shift (@_);
+    my $textdir = &util::filename_cat($self->{'build_dir'}, "text");
+    &util::mk_all_dir ($textdir);
+
+    # assume little-endian for now :-)
+    my $fulldbname = &util::filename_cat ($textdir, "$self->{'collection'}.ldb");
+    $fulldbname =~ s/\//\\/g if ($ENV{'GSDLOS'} =~ /^windows$/i);
+    my $exedir = "$ENV{'GSDLHOME'}/bin/$ENV{'GSDLOS'}";
+    my $exe = &util::get_os_exe ();
+
+    print STDERR "\n*** creating the info database\n" if ($self->{'verbosity'} >= 1);
+
+    # set up the document processor
+    $self->{'buildproc'}->set_output_handle ('mgbuilder::PIPEOUT');
+    $self->{'buildproc'}->set_mode ('infodb');
+    
+    # collect the statistics for the text
+    # -b $maxdocsize sets the maximum document size to be 12 meg
+    if (!-e "$exedir/txt2db$exe" || !open (PIPEOUT, 
+            "| $exedir/txt2db$exe $fulldbname")) {
+	die "mgbuilder::make_infodatabase - couldn't run $exedir/txt2db$exe\n";
+    }
+    $self->{'buildproc'}->reset();
+    &plugin::read ($self->{'pluginfo'}, $self->{'source_dir'}, 
+		   "", {}, $self->{'buildproc'});
+    close (PIPEOUT);
+}
+
+sub make_auxiliary_files {
+    my $self = shift (@_);
+    my ($index);
+    my %build_cfg = ();
+
+    print STDERR "\n*** creating auxiliary files \n" if ($self->{'verbosity'} >= 1);
+
+    # get the text directory
+    &util::mk_all_dir ($self->{'build_dir'});
+
+    # store the build date
+    $build_cfg->{'builddate'} = time;
+
+    # store the number of documents and number of bytes
+    $build_cfg->{'numdocs'} = $self->{'buildproc'}->get_num_docs();
+    $build_cfg->{'numbytes'} = $self->{'buildproc'}->get_num_bytes();
+
+    # store the mapping between the index names and the directory names
+    my @indexmap = ();
+    foreach $index (@{$self->{'collect_cfg'}->{'indexes'}}) {
+	push (@indexmap, "$index\-\>$self->{'index_mapping'}->{$index}");
+    }
+    $build_cfg->{'indexmap'} = \@indexmap;
+
+    # write out the build information
+    &cfgread::write_cfg_file("$self->{'build_dir'}/build.cfg", $build_cfg,
+			     '^(builddate|numdocs|numbytes)$', '^(indexmap)$');
+
+}
+
+sub deinit {
+    my $self = shift (@_);
+}
+
+
+1;
+
+
Index: /trunk/gsdl/perllib/mgbuildproc.pm
===================================================================
--- /trunk/gsdl/perllib/mgbuildproc.pm	(revision 4)
+++ /trunk/gsdl/perllib/mgbuildproc.pm	(revision 4)
@@ -0,0 +1,467 @@
+# This document processor saves a document in the 
+# archives directory of a collection
+
+
+package mgbuildproc;
+
+use docproc;
+use util;
+
+
+BEGIN {
+    @ISA = ('docproc');
+}
+
+
+sub new {
+    my ($class, $collection, $source_dir, $build_dir, $verbosity) = @_;
+    my $self = new docproc ();
+
+    $self->{'collection'} = $collection;
+    $self->{'source_dir'} = $source_dir;
+    $self->{'build_dir'} = $build_dir;
+    $self->{'verbosity'} = $verbosity;
+    $self->{'mode'} = "text";
+    $self->{'index'} = "section:text";
+    $self->{'output_handle'} = "STDOUT";
+    $self->{'num_docs'} = 0;
+    $self->{'num_sections'} = 0;
+    $self->{'num_bytes'} = 0;
+
+    return bless $self, $class;
+}
+
+sub reset {
+    my $self = shift (@_);
+    
+    $self->{'num_docs'} = 0;
+    $self->{'num_sections'} = 0;
+    $self->{'num_bytes'} = 0;
+}
+
+sub get_num_docs {
+    my $self = shift (@_);
+
+    return $self->{'num_docs'};
+}
+
+sub get_num_sections {
+    my $self = shift (@_);
+
+    return $self->{'num_sections'};
+}
+
+sub get_num_bytes {
+    my $self = shift (@_);
+
+    return $self->{'num_bytes'};
+}
+
+sub set_output_handle {
+    my $self = shift (@_);
+    my ($handle) = @_;
+
+    $self->{'output_handle'} = $handle;
+}
+
+sub set_mode {
+    my $self = shift (@_);
+    my ($mode) = @_;
+
+    $self->{'mode'} = $mode;
+}
+
+sub set_index {
+    my $self = shift (@_);
+    my ($index) = @_;
+
+    $self->{'index'} = $index;
+}
+
+sub process {
+    my $self = shift (@_);
+    my ($doc_obj) = @_;
+    my $method = $self->{'mode'};
+
+    $self->$method($doc_obj);
+}
+
+sub infodb {
+    my $self = shift (@_);
+    my ($doc_obj) = @_;
+    my $handle = $self->{'output_handle'};
+
+    if ($doc_obj eq 'classifications') {
+	# output classifications if all books have been processed
+	foreach $key (keys %$saved_classifications) {
+	    $saved_classifications->{$key}->{'contains'} = undef if 
+		$saved_classifications->{$key}->{'contains'} eq "";
+	    $saved_classifications->{$key}->{'parent'} = undef if 
+		(!defined $saved_classifications->{$key}->{'parent'}) || 
+		    ($saved_classifications->{$key}->{'parent'} eq "");
+	    $self->write_to_gdbm ($handle, $key, $saved_classifications->{$key}->{'title'}, 
+				  undef, undef, 
+				  $saved_classifications->{$key}->{'contains'}, undef, 
+				  $saved_classifications->{$key}->{'parent'}, undef);
+	}
+	return;
+    }
+
+    my $doctype = $doc_obj->get_doc_type();
+
+    # only output this document if it is one to be indexed
+    return if ($doctype ne "indexed_doc" && 
+	       $doctype ne "classification");
+
+    # found classification document
+    $saved_classifications = {} if ($doctype eq "classification");
+    $sectionmap = {};
+
+    # this is another document
+    $self->{'num_docs'} += 1 unless ($doctype eq "classification");
+
+    my $section = $doc_obj->get_top_section();
+    while (defined $section) {
+	# update a few statistics
+	$self->{'num_bytes'} += $doc_obj->get_text_length ($section);
+	$self->{'num_sections'} += 1 unless ($doctype eq "classification");
+
+	my $title = $doc_obj->get_metadata_element($section, "Title");
+	my $jobnumber = $doc_obj->get_source_filename();
+
+	my $mapped_section = $self->map_section($doctype, $section);
+	if ($doctype eq "classification") {
+	    $mapped_section = $self->char_classification($mapped_section);
+	} else {
+	    $mapped_section = "B.$self->{'num_docs'}.$mapped_section";
+	}
+	$mapped_section =~ s/\.+$//;
+
+	my ($parent, $classification, $creator, $OID);
+
+	if ($section ne $doc_obj->get_top_section()) {
+	    $parent = $self->map_section($doctype, $doc_obj->get_parent_section($section));
+	    if ($doctype eq "classification") {
+		$parent = $self->char_classification($parent);
+	    } else {
+		$parent = "B.$self->{'num_docs'}.$parent";
+	    }
+	    $parent =~ s/\.+$//;
+	} else {
+	    $classification = $self->get_classifications($doc_obj, $section, $mapped_section)
+		unless $doctype eq "classification";
+	    $creator = $doc_obj->get_metadata_element($section, "Creator");
+	    $OID = $doc_obj->get_OID();
+	    # this is a hack at getting OID to look like the directory path - I'm sure there's 
+	    # a better way to do it but it's late... Stefan
+	    my @OIDchars = split //, $OID;
+	    $OID = "";
+	    my $count = 0;
+	    foreach $i (@OIDchars) {
+		if ($count == 7) {
+		    $OID .= "$i/";
+		    $count = 0;
+		} else {
+		    $OID .= $i;
+		    $count ++;
+		}
+	    }
+	    $OID =~ s/\/$//;
+	    $OID .= ".dir";
+	}
+
+	if ($doc_obj->is_leaf_section($section)) {
+	    if ($doctype eq "classification") {
+		if (defined $saved_classifications->{$mapped_section}) {
+		    print STDERR "mgbuildproc:warning: classification $mapped_section " .
+			"declared multiple times\n";
+		} else {
+		    $saved_classifications->{$mapped_section}  = {'title' => $title, 'contains' => "",
+								  'parent' => $parent};
+		}
+	    } else {
+		$self->write_to_gdbm($handle, $mapped_section, $title, $creator, $jobnumber, undef,
+				     $self->{'num_sections'}, $parent, $classification, $OID);
+	    }
+	} else {
+
+	    # add the introductory section if it exists
+	    my $contains = "";
+	    if ($doc_obj->get_text_length($section) > 0) {
+		$contains .= "B.$self->{'num_docs'}." . $self->map_section ($doctype, "$section.0");
+	    }
+
+	    # add the rest of the children
+	    my @children = @{$doc_obj->get_children($section)};
+	    if ($doctype eq "classification") {
+		map {$_ = $self->char_classification($_);} @children;
+		if (defined $saved_classifications->{$mapped_section}) {
+		    print STDERR "mgbuildproc:warning: classification $mapped_section " .
+			"declared multiple times\n";
+		} else {
+		    $saved_classifications->{$mapped_section}  = {'title' => $title, 'contains' => ""};
+		    $saved_classifications->{$mapped_section}->{'contains'} = join ":", @children;
+		    $saved_classifications->{$mapped_section}->{'parent'} = $parent;
+		}
+	    } else {
+		map {$_ = "B.$self->{'num_docs'}." . $self->map_section($doctype, $_);} @children;
+		$contains .= ":" if $contains ne "";
+		$contains .= join ":", @children;
+		$self->write_to_gdbm ($handle, $mapped_section, $title, $creator,
+				      $jobnumber, $contains, $self->{'num_sections'},
+				      $parent, $classification, $OID);
+
+		if ($doc_obj->get_text_length($section) > 0) {
+		    my $intromapsection = "B.$self->{'num_docs'}." . 
+			$self->map_section($doctype, "$section.0");
+		    $self->write_to_gdbm ($handle, $intromapsection, "<i>(introductory text)</i>", $creator,
+					  $jobnumber, undef, $self->{'num_sections'},
+					  $mapped_section, undef, $OID);
+		}
+	    }
+	}
+	$section = $doc_obj->get_next_section($section);
+    }
+    
+    # update classification list with those books that 
+    # were processed before classification list
+    if ($doctype eq "classification" && defined $temp_classifications) {
+	foreach $key (keys(%$temp_classifications)) {
+	    if (!defined $saved_classifications->{$key}) {
+		print STDERR "mgbuildproc:$temp_classifications->{$key} belong to " .
+		    "undefined classification $key\n";
+	    } 
+	    $saved_classifications->{$key}->{'contains'} = $temp_classifications->{$key}->{'contains'};
+	}
+	$temp_classifications = undef;
+    }
+}
+
+
+sub write_to_gdbm {
+    my $self = shift (@_);
+    my ($handle, $section, $title, $creator, $jobnumber, $contains,
+	$docnum, $parent, $classification, $OID) = @_;
+
+    print $handle "[$section]\n";
+    print $handle "<t>$title\n" if defined $title;
+    print $handle "<a>$creator\n" if defined $creator;
+    print $handle "<j>$jobnumber\n" if defined $jobnumber;
+    print $handle "<c>$contains\n" if defined $contains;
+    print $handle "<d>$docnum\n" if defined $docnum;
+    print $handle "<p>$parent\n" if defined $parent;
+    print $handle "<x>$classification\n" if defined $classification;
+    print $handle "<o>$OID\n" if defined $OID;
+    print $handle '-' x 70, "\n";
+    
+    if (defined $docnum) {
+	print $handle "[$docnum]\n";
+	print $handle "<x>$section\n";
+	print $handle '-' x 70, "\n";
+    }
+}
+
+sub get_classifications {
+    my $self = shift (@_);
+    my ($doc_obj, $section, $mapped_section) = @_;
+
+    my ($classificationsref);
+    if (defined $saved_classifications) {
+	# classification list has been processed
+	$classificationsref = $saved_classifications;
+    } else {
+	# classification list has yet to be processed, save
+	# books in temp_classifications until list is processed
+	# and they can be moved into saved_classifications
+	$temp_classifications = {} unless defined $temp_classifications;
+	$classificationsref = $temp_classifications;
+    }
+
+    my $classifications = $doc_obj->get_metadata($section, "Subject");
+
+    # need to save which books belong in each classification
+    # to output later
+    foreach $classification (@$classifications) {
+
+	if (!defined $classificationsref->{$classification}) {
+	    $classificationsref->{$classification}->{'parent'} = $doc_obj->get_parent_section($section);
+	    $classificationsref->{$classification}->{'contains'} = "";
+	}
+	$classificationsref->{$classification}->{'contains'} .= ":" unless
+	    $classificationsref->{$classification}->{'contains'} eq "";
+	$classificationsref->{$classification}->{'contains'} .= $mapped_section;
+    }
+
+    return (join ":", @$classifications);
+}
+
+sub map_section {
+    my $self = shift (@_);
+    my ($doctype, $section) = @_;
+
+    # classifications should never need to be mapped
+    return $section if $doctype eq "classification";
+
+    return "" unless (defined $section) && ($section ne "");
+
+    $sectionmap = {} unless defined $sectionmap;
+
+
+
+    # get the section into a standard format
+    $section =~ s/^\.+|\.+$//g;
+
+    # return the mapped section if it has been seen before
+    if (defined $sectionmap->{$section}) {
+	return $sectionmap->{$section};
+    }
+
+    # find out the parent section
+    my ($parentsection, $num);
+    if ($section =~ /^(.+)\.(\d+)$/) {
+	$parentsection = $1;
+	$num = $2;
+    } elsif ($section =~ /^(\d+)$/) {
+	$parentsection = "";
+	$num = $1;
+    } else {
+	print STDERR "mgbuildproc:map_section - misformed section $section\n";
+    }
+    $parentsection = "" unless defined $parentsection;
+
+    if ($parentsection eq "") {
+	$num ++ if $num == 1 and defined $sectionmap->{'0'};
+    } else {
+	$num ++ if $num == 1 and defined $sectionmap->{"$parentsection.0"};
+    }
+
+    # find out the mapped parent section
+    my $mappedparentsection = $self->map_section ($doctype, $parentsection);
+
+    # find the next unused child section
+    my $previousnum = $num - 1;
+    my $previoussection = "";
+    if ($parentsection eq "") {
+	$previoussection = $previousnum;
+	$previoussection = 0 if ($section == 1) && (defined $sectionmap->{'0'});
+    } else{
+	$previoussection = "$parentsection.$previousnum";
+	$previoussection = "$parentsection.0" if ($section =~ /\.1$/) && (defined $sectionmap->{"$parentsection.0"});
+    }
+    while (($previousnum > 0) && (!defined $sectionmap->{$previoussection})) {
+	$previousnum--;
+	$previoussection = "$parentsection.$previousnum";
+	$previoussection = $previousnum if $parentsection eq "";
+    }
+
+    # there has been no children under this parent, this section will be number 1
+    if ($previousnum <= 0) {
+	if ($mappedparentsection eq "") {
+	    $sectionmap->{$section} = "1";
+	} else {
+	    $sectionmap->{$section} = "$mappedparentsection.1";
+	}
+
+    } else {
+	# get the previous mapped number
+	my $previousmapnum = 0;
+	if ($sectionmap->{$previoussection} =~ /(^|\.)?(\d+)$/) {
+	    $previousmapnum = $2;
+	}
+	
+	# increment it to get this mapped child
+	my $mappednum = $previousmapnum+1;
+
+	if ($mappedparentsection eq "") {
+	    $sectionmap->{$section} = $mappednum;
+	} else {
+	    $sectionmap->{$section} = "$mappedparentsection.$mappednum";
+	}
+    }
+
+    return $sectionmap->{$section};
+}
+
+sub text {
+    my $self = shift (@_);
+    my ($doc_obj) = @_;
+    my $handle = $self->{'output_handle'};
+
+    # only output this document if it is one to be indexed
+    return if ($doc_obj->get_doc_type() ne "indexed_doc");
+
+    # this is another document
+    $self->{'num_docs'} += 1;
+
+    # get the parameters for the output
+    my ($level, $fields) = split (/:/, $self->{'index'});
+    $fields =~ s/\ball\b/Title,Creator,text/;
+    $fields =~ s/\btopall\b/topTitle,topCreator,toptext/;
+
+    my $doc_section = 0; # just for this document
+    my $text = "";
+    my $text_extra = "";
+
+    # get the text for this document
+    my $section = $doc_obj->get_top_section();
+    while (defined $section) {
+	# update a few statistics
+	$doc_section++;
+	$self->{'num_bytes'} += $doc_obj->get_text_length ($section);
+	$self->{'num_sections'} += 1;
+	
+	foreach $field (split (/,/, $fields)) {
+	    # only deal with this field if it doesn't start with top or
+	    # this is the first section
+	    my $real_field = $field;
+	    if (!($real_field =~ s/^top//) || ($doc_section == 1)) {
+		my $new_text = "";
+		if ($real_field eq "text") {
+		    $new_text = $doc_obj->get_text ($section);
+		    $new_text =~ s/[\cB\cC]//g;
+		    $new_text =~ s/(<p\b)/\cC$1/gi;
+		    
+		} else {
+		    $new_text = join ("\cC", @{$doc_obj->get_metadata ($section, $real_field)});
+		}
+	    
+		$text .= "$new_text\cC";
+	    }
+	}
+	
+	if ($level eq "document") { $text_extra .= "\cB"; }
+	else { $text .= "\cB"; }
+	
+	$section = $doc_obj->get_next_section($section);
+    }
+
+    print $handle "$text$text_extra";
+}
+
+# converts leading number in classification back 
+# to letter it represents
+# i.e 67.2.4 becomes C.2.4
+sub char_classification {
+    my $self = shift (@_);
+    my ($classification) = @_;
+
+    return $classification if $classification eq "";
+    my ($c) = $classification =~ /^\.?(\d+)/;
+    $c = chr($c);
+    $classification =~ s/^\d+/$c/;
+
+    return $classification;
+}
+
+# converts leading letter of a classification into its ascii equivalent
+# i.e C.2.4 becomes 67.2.4
+sub int_classification {
+    my $self = shift (@_);
+    my ($classification) = @_;
+    my $c = ord($classification);
+    $classification =~ s/^./$c/;
+
+    return $classification;
+}
+1;
+
Index: /trunk/gsdl/perllib/parsargv.pm
===================================================================
--- /trunk/gsdl/perllib/parsargv.pm	(revision 4)
+++ /trunk/gsdl/perllib/parsargv.pm	(revision 4)
@@ -0,0 +1,181 @@
+package parsargv;
+# 
+# parse(ARGVREF, [SPEC, VARREF] ...)
+# 
+# Parse command line arguments.
+# 
+# ARGVREF is an array reference, usually to @ARGV.  The remaining
+# arguments are paired (SPEC, VARREF).  SPEC is a specification string
+# for a particular argument; VARREF is a variable reference that will
+# receive the argument.
+# 
+# SPEC is in one of the following forms:
+# 
+# ARG/REGEX/DEFAULT ARG is the name of command line argument.  REGEX is
+#     a regular expression that gives legal values for the argument.
+#     DEFAULT is the default value assigned to VARREF if the option does
+#     not appear on the command line.  Example
+# 
+# 
+# ARG/REGEX ARG and REGEX are as above.  Since no default is given, ARG
+#     must appear on the command line; if it doesn't, parse() returns 0.
+# 
+# ARG ARG is as above.  ARG is a boolean option.  VARREF is assigned 0 if ARG
+#     is not on the command line; 1 otherwise.
+# 
+# SPEC may start with a punctuation character, in which case this
+# character will be used instead of '/' as a delimiter.  Useful when '/'
+# is needed in the REGEX part.
+# 
+# VARREF is a reference to a scalar or an array.  If VARREF is an array
+# reference, then multiple command line options are allowed an append.  Example:
+# 
+# 	Command line: -day mon -day fri
+# 
+# 	parse(\@ARGV, "day/(mon|tue|wed|thu|fri)", \@days)
+# 
+# 	days => ('mon', 'fri')
+# 
+# Returns 0 if there was an error, nonzero otherwise.
+# 
+sub parse
+{
+    my $arglist = shift;
+    my ($spec, $var);
+    my %option;
+
+    while (($spec, $var) = splice(@_, 0, 2))
+    {
+	die "Variable for $spec is not a valid type."
+	    unless ref($var) eq 'SCALAR' || ref($var) eq 'ARRAY';
+
+	my $delimiter;
+	if ($spec !~ /^\w/)
+	{
+	    $delimiter = substr($spec, 0, 1);
+	    $spec = substr($spec, 1);
+	}
+	else
+	{
+	    $delimiter = '/';
+	}
+	my ($name, $regex, $default) = split(/$delimiter/, $spec, 3);
+	
+	if ($name)
+	{
+	    if ($default && $default !~ /$regex/)
+	    {
+		die "Default value for $name doesn't match regex ($spec).";
+	    }
+	    $option{$name} = {'name' => $name,
+			      'regex' => $regex,
+			      'default' => $default,
+			      'varref' => $var,
+			      'set' => 0};
+	}
+	else
+	{
+	    die "Invalid argument ($spec) for parsargv.";
+	}
+    }
+
+    my @argv;
+    my $arg;
+    my $parse_options = 1;
+    my $errors = 0;
+
+    while ($arg = shift(@$arglist))
+    {
+	if ($parse_options && $arg eq '--')
+	{
+	    $parse_options = 0;
+	    next;
+	}
+
+	if ($parse_options && $arg =~ /^-+\w/)
+	{
+	    $arg =~ s/^-+//;
+
+	    if (defined $option{$arg})
+	    {
+		&process_arg($option{$arg}, $arglist, \$errors);
+	    }
+	    else
+	    {
+		print STDERR "$arg is not a valid option.\n";
+		$errors++;
+	    }
+	}
+	else
+	{
+	    push(@argv, $arg);
+	}
+    }
+    @$arglist = @argv;
+
+    foreach $arg (keys %option)
+    {
+	if ($option{$arg}->{'set'} == 0)
+	{
+	    if (defined $option{$arg}->{'default'})
+	    {
+		&set_var($option{$arg}, $option{$arg}->{'default'});
+	    }
+	    elsif (!$option{$arg}->{'regex'})
+	    {
+		&set_var($option{$arg}, 0)
+	    }
+	    elsif (ref($option{$arg}->{'varref'}) ne 'ARRAY')
+	    {
+		print STDERR "Missing command line argument -$arg.\n";
+		$errors++;
+	    }
+	}
+    }
+    return $errors == 0;
+}
+
+sub process_arg
+{
+    my ($option, $arglist, $errors) = @_;
+
+    if ($option->{'regex'} && @$arglist > 0 && $arglist->[0] !~ /^-+\w/)
+    {
+	if ($arglist->[0] =~ /$option->{'regex'}/)
+	{
+	    &set_var($option, shift(@$arglist));
+	}
+	else
+	{
+	    print STDERR "Bad value for -$option->{'name'} argument.\n";
+	    $$errors++;
+	}
+    }
+    elsif (!$option->{'regex'})
+    {
+	&set_var($option, 1);
+    }
+    else
+    {
+	print STDERR "No value given for -$option->{'name'}.\n";
+	$$errors++;
+    }
+}
+
+sub set_var
+{
+    my ($option, $value) = @_;
+    my $type = ref($option->{'varref'});
+
+    if ($type eq 'SCALAR')
+    {
+	${$option->{'varref'}} = $value;
+    }
+    elsif ($type eq 'ARRAY')
+    {
+	push(@{$option->{'varref'}}, $value);
+    }
+    $option->{'set'} = 1;
+}
+
+1;
Index: /trunk/gsdl/perllib/plugin.pm
===================================================================
--- /trunk/gsdl/perllib/plugin.pm	(revision 4)
+++ /trunk/gsdl/perllib/plugin.pm	(revision 4)
@@ -0,0 +1,50 @@
+# functions to handle using the plugins
+
+package plugin;
+
+
+sub load_plugins {
+    my ($collection, $plugin_list) = @_;
+    my @plugin_objects = ();
+
+    foreach $plugin (@$plugin_list) {
+	# find the plugin
+	if (-e "$ENV{'GSDLHOME'}/collect/$collection/lib/plugins/${plugin}.pm") {
+	    require "$ENV{'GSDLHOME'}/collect/$collection/lib/plugins/${plugin}.pm";
+
+	} elsif (-e "$ENV{'GSDLHOME'}/lib/plugins/${plugin}.pm") {
+	    require "$ENV{'GSDLHOME'}/lib/plugins/${plugin}.pm";
+
+	} else {
+	    die "ERROR - couldn't find plugin $plugin\n";
+	}
+
+	# create a plugin object
+	my ($plugobj);
+	eval ("\$plugobj = new $plugin()");
+	die "$@" if $@;
+
+	# add this object to the list
+	push (@plugin_objects, $plugobj);
+    }
+
+    return \@plugin_objects;
+}
+
+sub read {
+    my ($pluginfo, $base_dir, $file, $metadata, $processor) = @_;
+
+    # pass this file by each of the plugins in turn until one
+    # is found which will process it
+    foreach $plugobj (@$pluginfo) {
+	if ($plugobj->read($pluginfo, $base_dir, $file, 
+			   $metadata, $processor)) {
+	    return;
+	}
+    }
+
+    print STDERR "WARNING - no plugin could process $base_dir$file\n";
+}
+
+
+1;
Index: /trunk/gsdl/perllib/plugins/ArcPlug.pm
===================================================================
--- /trunk/gsdl/perllib/plugins/ArcPlug.pm	(revision 4)
+++ /trunk/gsdl/perllib/plugins/ArcPlug.pm	(revision 4)
@@ -0,0 +1,104 @@
+# plugin which recurses through directories processing
+# each file it finds
+
+package ArcPlug;
+
+use util;
+use BasPlug;
+use plugin;
+use arcinfo;
+
+BEGIN {
+    @ISA = ('BasPlug');
+}
+
+sub new {
+    my ($class) = @_;
+    my $self = new BasPlug ();
+
+    return bless $self, $class;
+}
+
+# return 1 if this class might recurse using $pluginfo
+sub is_recursive {
+    my $self = shift (@_);
+
+    return 1;
+}
+
+# return 1 if processed, 0 if not processed
+# Note that $base_dir might be "" and that $file might 
+# include directories
+sub read {
+    my $self = shift (@_);
+    ($pluginfo, $base_dir, $file, $metadata, $processor) = @_;
+
+    # see if this has a archives information file within it
+    $archive_info_filename = &util::filename_cat($base_dir,$file,"archives.inf");
+
+    if (-e $archive_info_filename) {
+
+	# read in the archives information file
+	my $archive_info = new arcinfo ();
+	$archive_info->load_info ($archive_info_filename);
+	
+	my $file_list = $archive_info->get_file_list();
+
+	# process each file
+	foreach $subfile (sort alphabetically @$file_list) {
+	    # note: metadata is not carried on to the next level
+	    my $tmp = &util::filename_cat ($file, $subfile->[0]);
+	    next if $tmp eq $file;
+	    &plugin::read ($pluginfo, $base_dir, $tmp, {}, $processor);
+	}
+	
+	# all books have been processed so need to output classifications
+	# to infodb - note that at present you have to import before building
+	if ($processor->{'mode'} eq 'infodb') {
+	    print STDERR "ArcPlug: Adding classifications to infodb\n";
+	    $processor->process('classifications');
+	}
+
+	return 1;
+    }
+
+    # wasn't an archives directory, someone else will have to process it
+    return 0;
+}
+
+# sort files so they're built (and displayed) in alphabetical order
+sub alphabetically {
+    open (A, &util::filename_cat($base_dir, $file, $a->[0])) || 
+	die "couldn't open " . &util::filename_cat($base_dir, $file, $a->[0]) . "\n";
+    open (B, &util::filename_cat($base_dir, $file, $b->[0])) || 
+	die "couldn't open " . &util::filename_cat($base_dir, $file, $b->[0]) . "\n";
+    my ($atitle, $btitle);
+    my $line = "";
+    while (defined ($line .= <A>)) {
+	if ($line =~ /Title=\"([^\"]*)\"/) {$atitle = $1; last;}
+    } 
+    $line = "";
+    while (defined ($line .= <B>)) {
+	if ($line =~ /Title=\"([^\"]*)\"/) {$btitle = $1; last;}
+    } 
+    close A;
+    close B;
+    
+    $atitle =~ tr/A-Z/a-z/;
+    $atitle =~ s/[^a-z0-9 ]//g;
+    $atitle =~ s/^\s+//;
+    $atitle =~ s/^(the|a|an)\b//;
+    $atitle =~ s/^\s+//;
+    $atitle =~ s/^(.).*$/$1/;
+    
+    $btitle =~ tr/A-Z/a-z/;
+    $btitle =~ s/[^a-z0-9 ]//g;
+    $btitle =~ s/^\s+//;
+    $btitle =~ s/^(the|a|an)\b//;
+    $btitle =~ s/^\s+//;
+    $btitle =~ s/^(.).*$/$1/;
+
+    $atitle cmp $btitle;
+}
+
+1;
Index: /trunk/gsdl/perllib/plugins/BasPlug.pm
===================================================================
--- /trunk/gsdl/perllib/plugins/BasPlug.pm	(revision 4)
+++ /trunk/gsdl/perllib/plugins/BasPlug.pm	(revision 4)
@@ -0,0 +1,32 @@
+# base class for all the import plugins
+
+package BasPlug;
+
+
+sub new {
+    my ($class) = @_;
+
+    return bless {}, $class;
+}
+
+# return 1 if this class might recurse using $pluginfo
+sub is_recursive {
+    my $self = shift (@_);
+
+    die "BasPlug::is_recursive function must be implemented in sub classes\n";
+}
+
+# return 1 if processed, 0 if not processed
+# Note that $base_dir might be "" and that $file might 
+# include directories
+sub read {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $metadata, $processor) = @_;
+
+    die "BasPlug::read function must be implemented in sub classes\n";
+
+    return 0; # will never get here
+}
+
+
+1;
Index: /trunk/gsdl/perllib/plugins/GMLPlug.pm
===================================================================
--- /trunk/gsdl/perllib/plugins/GMLPlug.pm	(revision 4)
+++ /trunk/gsdl/perllib/plugins/GMLPlug.pm	(revision 4)
@@ -0,0 +1,117 @@
+# plugin which process an HTML book directory
+
+package GMLPlug;
+
+use html;
+use BasPlug;
+use util;
+use doc;
+
+
+sub BEGIN {
+    @ISA = ('BasPlug');
+}
+
+sub new {
+    my ($class) = @_;
+    $self = new BasPlug ();
+
+    return bless $self, $class;
+}
+
+sub is_recursive {
+    my $self = shift (@_);
+
+    return 0; # this is not a recursive plugin
+}
+
+# return 1 if processed, 0 if not processed
+# Note that $base_dir might be "" and that $file might 
+# include directories
+sub read {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $metadata, $processor) = @_;
+    my $fullname = &util::filename_cat ($base_dir, $file);
+
+    # see if this is a gml book
+    return 0 unless (-f $fullname && $fullname =~ /\.gml$/i);
+
+    my ($parent_dir) = $fullname =~ /^(.*)\/[^\/]+.gml$/;
+
+    # create a new document
+    my $doc_obj = new doc ();
+    my $section = $doc_obj->get_top_section();
+
+    print STDERR "GMLPlug: processing $file\n";
+
+    # read in the document
+    my $gml = "";
+    my $line = "";
+    if (!open (INFILE, $fullname)) {
+	print STDERR "GMLPlug::read - couldn't read $fullname\n";
+	return 0;
+    }
+    while (defined ($line = <INFILE>)) {
+	$gml .= $line;
+    }
+    close (INFILE);
+
+    # process the document
+    my $firstsection = 1;
+    while ($gml =~ /\S/) {
+	if ($gml =~ s/^\s*<gsdlsection([^>]*)>(.*?)(<\/?gsdlsection)/$3/is) {
+	    my $tags = $1;
+	    $tags = "" unless defined $tags;
+	    my $text = $2;
+
+	    # create the section (unless this is the first section)
+	    if (!$firstsection) {
+		$tags =~ s/gsdlnum\s*=\s*\"?(\d+)\"?//i;
+		if (defined $1) {
+		    $section .= ".$1";
+		    $doc_obj->create_named_section($section);
+		} else {
+		    $section = $doc_obj->insert_section($doc_obj->get_end_child($section));
+		}
+	    }
+	    $firstsection = 0;
+
+	    # add the tags
+	    while ((defined $tags) && ($tags =~ s/^\s*(\w+)\s*=\s*\"([^\"]*)\"//)) {
+		$doc_obj->add_metadata($section, $1 , $2) if (defined $1 and defined $2);
+	    }
+
+	    # add the text
+	    $doc_obj->add_text($section, $text) if ((defined $text) && ($text ne ""));
+
+	} elsif ($gml =~ s/^\s*<\/gsdlsection>//) {
+	    $section = $doc_obj->get_parent_section ($section);
+
+	} else {
+	    print STDERR "GMLPlug::read - error in file $fullname\n";
+	    print STDERR "text: \"$gml\"\n";
+	    last;
+	}
+    }
+
+    # add the associated files
+    $assoc_files = $doc_obj->get_metadata($doc_obj->get_top_section(), "gsdlassocfile");
+    my ($assoc_file_info);
+    foreach $assoc_file_info (@$assoc_files) {
+	my ($assoc_file, $mime_type) = split (":", $assoc_file_info);
+	$doc_obj->associate_file(&util::filename_cat($parent_dir, $assoc_file),
+				 $assoc_file, $mime_type);
+				 
+    }
+    $doc_obj->delete_metadata($doc_obj->get_top_section(), "gsdlassocfile");
+
+    # assume the document has an OID
+
+    # process the document
+    $processor->process($doc_obj);
+        
+    return 1; # processed the file
+}
+
+
+1;
Index: /trunk/gsdl/perllib/plugins/HBPlug.pm
===================================================================
--- /trunk/gsdl/perllib/plugins/HBPlug.pm	(revision 4)
+++ /trunk/gsdl/perllib/plugins/HBPlug.pm	(revision 4)
@@ -0,0 +1,244 @@
+# plugin which process an HTML book directory
+
+package HBPlug;
+
+use html;
+use BasPlug;
+
+
+sub BEGIN {
+    @ISA = ('BasPlug');
+}
+
+sub new {
+    my ($class) = @_;
+    $self = new BasPlug ();
+
+    return bless $self, $class;
+}
+
+sub is_recursive {
+    my $self = shift (@_);
+
+    return 0; # this is not a recursive plugin
+}
+
+sub HB_read_html_file {
+    my $self = shift (@_);
+    my ($htmlfile) = @_;
+
+    # load in the file
+    if (!open (FILE, $htmlfile)) {
+	print STDERR "ERROR - could not open $htmlfile\n";
+	return "";
+    }
+
+    my $line = "";
+    my $file = "";
+    my $foundbody = 0;
+    while (defined ($line = <FILE>)) {
+	# look for body tag
+	if (!$foundbody) {
+	    if ($line =~ s/^.*<body[^>]*>//i) {
+		$foundbody = 1;
+	    } else {
+		next;
+	    }
+	}
+
+	# check for symbol fonts
+	if ($line =~ /<font [^>]*?face\s*=\s*\"?(\w+)\"?/i) {
+	    my $font = $1;
+	    print STDERR "HBPlug::HB_read_html_file - warning font $font used\n" 
+		if ($font !~ /^arial$/i);
+	}
+
+
+	$line =~ s/<\/p>//ig;   # remove </p> tags
+	$line =~ s/<\/?(body|html|font)\b[^>]*>//ig; # remove any unwanted tags
+
+	# convert any alphanumeric character entities to their extended
+	# ascii equivalent for indexing purposes
+	&html::convertcharentities ($line);
+
+	$file .= $line;
+    }
+    close FILE;
+
+    $file =~ s/\s+/ /g; # remove \n's
+
+    return $file;
+}
+
+sub HB_clean_section {
+    my $self = shift (@_);
+    my ($section) = @_;
+
+    # remove tags without a starting tag from the section
+    my ($tag, $tagstart);
+    while ($section =~ /<\/([^>]{1,10})>/) {
+	$tag = $1;
+	$tagstart = index($section, "<$tag");
+	last if (($tagstart >= 0) && ($tagstart < index($section, "<\/$tag")));
+	$section =~ s/<\/$tag>//;
+    }
+    
+    # remove extra paragraph tags
+    while ($section =~ s/<p\b[^>]*>\s*<p\b/<p/ig) {}
+    
+    # remove extra stuff at the end of the section
+    while ($section =~ s/(<u>|<i>|<b>|<p\b[^>]*>|&nbsp;|\s)$//i) {}
+    
+    # add a newline at the beginning of each paragraph
+    $section =~ s/(.)\s*<p\b/$1\n\n<p/gi;
+    
+    # add a newline every 80 characters at a word boundary
+    # Note: this regular expression puts a line feed before
+    # the last word in each section, even when it is not
+    # needed.
+    $section =~ s/(.{1,80})\s/$1\n/g;
+    
+    # fix up the image links -- not sure what Rodger intended here - Stefan.
+#    $section =~ s/<img[^>]*?src=\"?([^\">]+)\"?[^>]*>/<img src=\"_linkOID_(_thisOID_\/$1)\"><br>/ig;
+#    $section =~ s/&lt;&lt;I&gt;&gt;\s*([^\.]+\.(png|jpg|gif))/<img src=\"_linkOID_(_thisOID_\/$1)\"><br>/ig;
+
+    $section =~ s/<img[^>]*?src=\"?([^\">]+)\"?[^>]*>/<center><img src=\"_httpcollection_\/archives\/_thisOID_\/$1\"><\/center><br>/ig;
+    $section =~ s/&lt;&lt;I&gt;&gt;\s*([^\.]+\.(png|jpg|gif))/<center><img src=\"_httpcollection_\/archives\/_thisOID_\/$1\"><\/center><br>/ig;
+
+    return $section;
+}
+
+
+sub shorten {
+    my $self = shift (@_);
+    my ($text) = @_;
+
+    return "\"$text\"" if (length($text) < 100);
+
+    return "\"" . substr ($text, 0, 50) . "\" ... \"" . 
+	substr ($text, length($text)-50) . "\"";
+}
+
+
+# return 1 if processed, 0 if not processed
+# Note that $base_dir might be "" and that $file might 
+# include directories
+sub read {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $metadata, $processor) = @_;
+
+    # get the html filename and see if this is a HTML Book...
+    my ($jobnumber);
+    if ($file =~ /[\\\/]/) {
+	($jobnumber) = $file =~ /[\\\/]([^\\\/]+)$/;
+    } else {
+	($jobnumber) = $file =~ /^([^\\\/]+)$/;
+    }
+    return 0 unless defined $jobnumber;
+    my $htmlfile = "$base_dir$file/$jobnumber.htm";
+    return 0 unless -e $htmlfile;
+
+    print STDERR "HBPlug: processing $file\n";
+
+    # read in the file and do basic html cleaning (removing header etc)
+    my $html = $self->HB_read_html_file ($htmlfile);
+
+    # create a new document
+    my $doc_obj = new doc ($file, "indexed_doc");
+
+    # copy the book cover if it exists
+    my $bookcover = "$base_dir$file/$jobnumber.jpg";
+    $doc_obj->associate_file($bookcover, "cover.jpg", "image/jpeg") 
+	if -e $bookcover;
+
+    # process the file one section at a time
+    my $curtoclevel = 1;
+    my $cursection = $doc_obj->get_top_section();
+    my $firstsection = 1;
+    while (length ($html) > 0) {
+	if ($html =~ s/^.*?<p\b[^>]*>((<b>|<i>|<u>|\s)*)&lt;&lt;TOC(\d+)&gt;&gt;\s*(.*?)<p\b/<p/i) {
+	    my $toclevel = $3;
+	    my $title = $4;
+	    my $sectiontext = "";
+	    if ($html =~ s/^(.*?)(<p\b[^>]*>((<b>|<i>|<u>|\s)*)&lt;&lt;TOC\d+&gt;&gt;)/$2/i) {
+		$sectiontext = $1;
+	    } else {
+		$sectiontext = $html;
+		$html = "";
+	    }
+
+	    # remove tags and extra spaces from the title
+	    $title =~ s/<\/?[^>]+>//g;
+	    $title =~ s/^\s+|\s+$//g;
+
+	    # close any sections below the current level and
+	    # create a new section (special case for the firstsection)
+	    while (($curtoclevel > $toclevel) ||
+		   (!$firstsection && $curtoclevel == $toclevel)) {
+		$cursection = $doc_obj->get_parent_section ($cursection);
+		$curtoclevel--;
+	    }
+	    if ($curtoclevel+1 < $toclevel) {
+		print STDERR "WARNING - jump in toc levels in $htmlfile " . 
+		    "from $curtoclevel to $toclevel\n";
+	    }
+	    while ($curtoclevel < $toclevel) {
+		$curtoclevel++;
+		$cursection = 
+		    $doc_obj->insert_section($doc_obj->get_end_child($cursection));
+	    }
+
+	    # add the metadata to this section
+	    if (!$firstsection || !defined($metadata->{'Title'})) {
+		$doc_obj->add_metadata ($cursection, "Title", $title);
+	    }
+	    if ($firstsection) {
+		foreach $field (keys(%$metadata)) {
+		    # Subjects may be a colon separated list
+		    if ($field eq "Subject") {
+			my @subjects = split /:/, $metadata->{'Subject'};
+			foreach $subject (@subjects) {
+			    $doc_obj->add_metadata ($cursection, 'Subject', $subject); 
+			}
+		    } else {
+			$doc_obj->add_metadata ($cursection, $field, $metadata->{$field});
+		    }
+		}
+		$firstsection = 0;
+	    }
+
+	    # clean up the section html
+	    $sectiontext = $self->HB_clean_section($sectiontext);
+
+	    # associate any files
+#	    map { $doc_obj->associate_file("$base_dir$file/$1", $1) 
+#		      if /_linkOID_\(_thisOID_\/([^\)]+)\)/; 0; }
+#		 split (/(_linkOID_\(_thisOID_\/[^\)]+\))/, $sectiontext);
+
+	    map { $doc_obj->associate_file("$base_dir$file/$1", $1)
+		      if /_httpcollection_\/archives\/_thisOID_\/([^\"]+)\"/; 0; }
+	         split (/(_httpcollection_\/archives\/_thisOID_\/[^\"]+\")/, $sectiontext);
+
+	    # add the text for this section
+	    $doc_obj->add_text ($cursection, $sectiontext);
+	    
+	} else {
+	    print STDERR "WARNING - leftover text\n" , $self->shorten($html), 
+	    "\nin $htmlfile\n";
+	    last;
+	}
+    }
+
+    # add a OID
+    $doc_obj->set_OID ();
+
+    print STDERR "OID: ", $doc_obj->get_OID(), "\n";
+
+    # process the document
+    $processor->process($doc_obj);
+        
+    return 1; # processed the file
+}
+
+
+1;
Index: /trunk/gsdl/perllib/plugins/RecPlug.pm
===================================================================
--- /trunk/gsdl/perllib/plugins/RecPlug.pm	(revision 4)
+++ /trunk/gsdl/perllib/plugins/RecPlug.pm	(revision 4)
@@ -0,0 +1,65 @@
+# plugin which recurses through directories processing
+# each file it finds
+
+package RecPlug;
+
+use BasPlug;
+use plugin;
+
+
+BEGIN {
+    @ISA = ('BasPlug');
+}
+
+sub new {
+    my ($class) = @_;
+    my $self = new BasPlug ();
+
+    return bless $self, $class;
+}
+
+# return 1 if this class might recurse using $pluginfo
+sub is_recursive {
+    my $self = shift (@_);
+
+    return 1;
+}
+
+# return 1 if processed, 0 if not processed
+# Note that $base_dir might be "" and that $file might 
+# include directories
+sub read {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $metadata, $processor) = @_;
+
+
+    my (@dir, $subfile);
+
+    # see if this is a directory
+    if (-d "$base_dir$file") {
+
+	# read all the files in the directory
+	opendir (DIR, "$base_dir$file") ||
+	    die "ERROR - couldn't read directory $base_dir$file\n";
+	@dir = readdir (DIR);
+	closedir (DIR);
+
+	# process each file
+	foreach $subfile (@dir) {
+	    if ($subfile !~ /^\.\.?$/) {
+		# note: metadata is not carried on to the next level
+		my $tmp = "$file/$subfile";
+		$tmp =~ s/^[\\\/]+|[\\\/]+$//g; # remove extra slashes
+		&plugin::read ($pluginfo, $base_dir, $tmp, {}, $processor);
+	    }
+	}
+
+	return 1;
+    }
+
+    # wasn't a directory, someone else will have to process it
+    return 0;
+}
+
+
+1;
Index: /trunk/gsdl/perllib/plugins/TOCPlug.pm
===================================================================
--- /trunk/gsdl/perllib/plugins/TOCPlug.pm	(revision 4)
+++ /trunk/gsdl/perllib/plugins/TOCPlug.pm	(revision 4)
@@ -0,0 +1,142 @@
+# This plugin processes all books in a table of contents file.
+# A toc.txt file contains two things: a subject classification and a
+# list of file names.
+
+package TOCPlug;
+
+use plugin;
+use BasPlug;
+use lang;
+use doc;
+
+sub BEGIN {
+    @ISA = ('BasPlug');
+}
+
+sub new {
+    my ($class) = @_;
+    $self = new BasPlug ();
+
+    return bless $self, $class;
+}
+
+# return 1 if this class might recurse using $pluginfo
+sub is_recursive {
+    my $self = shift (@_);
+    
+    return 1;
+}
+
+sub read_toc_subject {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $metadata, $processor) = @_;
+
+    my ($infoline, $line, @line);
+
+    open (TOCFILE, "$base_dir$file") || 
+	die "TOCPlug::read_toc_subject couldn't open $tocfile\n";
+    $infoline = <TOCFILE>;     # first line is an info line
+    my $doc_obj = new doc($file, "classification");
+    $doc_obj->set_OID ("CLSU");
+
+    while (defined ($line = <TOCFILE>)) {
+	$line =~ s/^\#.*$//;   # remove comments
+	$line =~ s/\cM|\cJ//g; # remove end-of-line characters
+	@line = split(/\t/, $line);
+
+	# remove spaces at the start and end of each field
+	map { s/^\s+|\s+$//g; } @line;
+
+	# if this is a classification entry add it to the classification file
+	if ((scalar(@line) >= 2) && ($line[0] ne "") && ($line[1] ne "")) {
+	    my $classifier = $self->int_classification ($line[0]); # convert leading letter to int
+	    $doc_obj->create_named_section($classifier);
+	    $doc_obj->add_metadata($classifier, "Title", $line[1]);
+	}
+    }
+
+    close (TOCFILE);
+    
+    # process the classification file
+    $processor->process($doc_obj);
+}
+
+# converts leading letter of a classification into its ascii equivalent
+# i.e C.2.4 becomes 67.2.4
+sub int_classification {
+    my $self = shift (@_);
+    my ($classification) = @_;
+    my $c = ord($classification);
+    $classification =~ s/^./$c/;
+
+    return $classification;
+}
+
+sub read_toc_files {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $metadata, $processor) = @_;
+
+    my ($infoline, $line);
+# the fields are in the following order (separated by tabs):
+# Classification, Classification title, Title, 
+# Language, Creator
+
+    open (TOCFILE, "$base_dir$file") || 
+	die "TOCPlug::read_toc_files couldn't open $tocfile\n";
+    $infoline = <TOCFILE>;     # first line is an info line
+
+    while (defined ($line = <TOCFILE>)) {
+	next if ($line =~ $line =~ /^\#.*$/) || ($line !~ /\w/);
+	$line =~ s/\cM|\cJ//g; # remove end-of-line characters
+	@line = split(/\t/, $line);
+
+	# remove spaces at the start and end of each field
+	map { s/^\s+|\s+$//g; } @line;
+
+	# if this is a file entry process it
+	if ((scalar(@line) >= 5) && ($line[1] eq "") && ($line[4] ne "")) {
+	    my $metadata = {'Subject' => $line[0],
+			    'Title'   => $line[2],
+			    'Language'=> &lang::english_to_iso639($line[3])};
+
+	    $metadata->{'Creator'} = $line[5] if defined $line[5]; 
+
+
+	    my $subimportfile = "$base_dir$line[4]";
+
+	    &plugin::read ($pluginfo, $base_dir, $line[4], $metadata, $processor);
+	}
+    }
+
+    close (TOCFILE);
+}
+
+
+# return 1 if processed, 0 if not processed
+# Note that $base_dir might be "" and that $file might 
+# include directories
+sub read {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $metadata, $processor) = @_;
+
+    my $tocfile = "$file/toc.txt";
+    if (!(-f "$base_dir$tocfile")) {
+	# not a directory containing a toc file
+	return 0;
+    }
+
+    # found a toc.txt file
+
+    print STDERR "TOCPlug: processing $tocfile\n";
+
+    # create the subject classification document
+    $self->read_toc_subject ($pluginfo, $base_dir, $tocfile, {}, $processor);
+    
+    # process each file within this table of contents file
+    $self->read_toc_files ($pluginfo, $base_dir, $tocfile, {}, $processor);
+
+    return 1; # was processed
+}
+
+
+1;
Index: /trunk/gsdl/perllib/util.pm
===================================================================
--- /trunk/gsdl/perllib/util.pm	(revision 4)
+++ /trunk/gsdl/perllib/util.pm	(revision 4)
@@ -0,0 +1,359 @@
+# various useful utilities
+
+package util;
+
+use File::Copy;
+
+
+# removes files (but not directories)
+sub rm {
+    my (@files) = @_;
+    my @filefiles = ();
+
+    # make sure the files we want to delete exist 
+    # and are regular files
+    foreach $file (@files) {
+	if (!-e $file) {
+	    print STDERR "util::rm $file does not exist\n";
+	} elsif (!-f $file) {
+	    print STDERR "util::rm $file is not a regular file\n";
+	} else {
+	    push (@filefiles, $file);
+	}
+    }
+    
+    # remove the files
+    my $numremoved = unlink @filefiles;
+
+    # check to make sure all of them were removed
+    if ($numremoved != scalar(@filefiles)) {
+	print STDERR "util::rm Not all files were removed\n";
+    }
+}
+
+
+# recursive removal
+sub rm_r {
+    my (@files) = @_;
+
+    # recursively remove the files
+    foreach $file (@files) {
+	$file =~ s/[\/\\]+$//; # remove trailing slashes
+	
+	if (!-e $file) {
+	    print STDERR "util::rm_r $file does not exist\n";
+
+	} elsif (-d $file) {
+	    # get the contents of this directory
+	    if (!opendir (INDIR, $file)) {
+		print STDERR "util::rm_r could not open directory $file\n";
+	    } else {
+		my @filedir = grep (!/^\.\.?$/, readdir (INDIR));
+		closedir (INDIR);
+
+		# remove all the files in this directory
+		&rm_r (map {$_="$file/$_";} @filedir);
+
+		# remove this directory
+		if (!rmdir $file) {
+		    print STDERR "util::rm_r couldn't remove directory $file\n";
+		}
+	    }
+
+	} else {
+	    # remove this file
+	    &rm ($file);
+	}
+    }
+}
+
+
+# copies a file or a group of files
+sub cp {
+    my $dest = pop (@_);
+    my (@srcfiles) = @_;
+
+    # remove trailing slashes from source and destination files
+    $dest =~ s/[\\\/]+$//;
+    map {$_ =~ s/[\\\/]+$//;} @srcfiles;
+
+    # a few sanity checks
+    if (scalar (@srcfiles) == 0) {
+	print STDERR "util::cp no destination directory given\n";
+	return;
+    } elsif ((scalar (@srcfiles) > 1) && (!-d $dest)) {
+	print STDERR "util::cp if multiple source files are given the ".
+	    "destination must be a directory\n";
+	return;
+    }
+
+    # copy the files
+    foreach $file (@srcfiles) {
+	my $tempdest = $dest;
+	if (-d $tempdest) {
+	    my ($filename) = $file =~ /([^\\\/]+)$/;
+	    $tempdest .= "/$filename";
+	}
+	if (!-e $file) {
+	    print STDERR "util::cp $file does not exist\n";
+	} elsif (!-f $file) {
+	    print STDERR "util::cp $file is not a plain file\n";
+	} else {
+	    &File::Copy::copy ($file, $tempdest);
+	}
+    }
+}
+
+
+# recursively copies a file or group of files
+# syntax: cp_r (sourcefiles, destination file or directory)
+sub cp_r {
+    my $dest = pop (@_);
+    my (@srcfiles) = @_;
+
+    # remove trailing slashes from source and destination files
+    $dest =~ s/[\\\/]+$//;
+    map {$_ =~ s/[\\\/]+$//;} @srcfiles;
+
+    # a few sanity checks
+    if (scalar (@srcfiles) == 0) {
+	print STDERR "util::cp no destination directory given\n";
+	return;
+    } elsif ((scalar (@srcfiles) > 1) && (!-d $dest)) {
+	print STDERR "util::cp if multiple source files are given the ".
+	    "destination must be a directory\n";
+	return;
+    }
+    
+    # copy the files
+    foreach $file (@srcfiles) {
+	# copy the file to within dest if dest is a directory
+	# exception: if there is only one source file and that
+	# source file is a directory
+	my $tempdest = $dest;
+	if (-d $tempdest && !(scalar(@srcfiles) == 1 && -d $file)) {
+	    my ($filename) = $file =~ /([^\\\/]+)$/;
+	    $tempdest .= "/$filename";
+	}
+
+	if (!-e $file) {
+	    print STDERR "util::cp $file does not exist\n";
+
+	} elsif (-d $file) {
+	    # make a new directory (if needed)
+	    mkdir ($tempdest, 0775) unless -e $tempdest;
+	    
+	    # get the contents of this directory
+	    if (!opendir (INDIR, $file)) {
+		print STDERR "util::cp_r could not open directory $file\n";
+	    } else {
+		my @filedir = grep (!/^\.\.?$/, readdir (INDIR));
+		closedir (INDIR);
+
+		# copy all the files in this directory
+		&cp_r (map {$_="$file/$_";} @filedir, $tempdest);
+	    }
+
+	} else {
+	    &cp($file, $tempdest);
+	}
+    }
+}
+
+
+sub mk_all_dir {
+    my ($dir) = @_;
+
+    # use / for the directory separator, remove duplicate and
+    # trailing slashes
+    $dir=~s/[\\\/]+/\//g; 
+    $dir=~s/[\\\/]+$//;
+
+    # make sure the cache directory exists
+    my $dirsofar = "";
+    my $first = 1;
+    foreach $dirname (split ("/", $dir)) {
+	$dirsofar .= "/" unless $first;
+	$first = 0;
+
+	$dirsofar .= $dirname;
+
+	next if $dirname =~ /^(|[a-z]:)$/i;
+	if (!-e $dirsofar && !mkdir ($dirsofar, 0775)) {
+	    print STDERR "util::mk_all_dir could not create directory $dirsofar\n";
+	    return;
+	}
+    }
+}
+
+
+# updates a copy of a directory in some other part of the filesystem
+# verbosity settings are: 0=low, 1=normal, 2=high
+# both $fromdir and $todir should be absolute paths
+sub cachedir {
+    my ($fromdir, $todir, $verbosity) = @_;
+    $verbosity = 1 unless defined $verbosity;
+
+    # use / for the directory separator, remove duplicate and
+    # trailing slashes
+    $fromdir=~s/[\\\/]+/\//g; 
+    $fromdir=~s/[\\\/]+$//;
+    $todir=~s/[\\\/]+/\//g; 
+    $todir=~s/[\\\/]+$//;
+
+    &mk_all_dir ($todir);
+
+    # get the directories in ascending order
+    if (!opendir (FROMDIR, $fromdir)) {
+	print STDERR "util::cachedir could not read directory $fromdir\n";
+	return;
+    }
+    my @fromdir = grep (!/^\.\.?$/, sort(readdir (FROMDIR)));
+    closedir (FROMDIR);
+
+    if (!opendir (TODIR, $todir)) {
+	print STDERR "util::cacedir could not read directory $todir\n";
+	return;
+    }
+    my @todir = grep (!/^\.\.?$/, sort(readdir (TODIR)));
+    closedir (TODIR);
+
+    my $fromi = 0;
+    my $toi = 0;
+		    
+    while ($fromi < scalar(@fromdir) || $toi < scalar(@todir)) {
+#	print "fromi: $fromi toi: $toi\n";
+
+	# see if we should delete a file/directory
+	# this should happen if the file/directory
+	# is not in the from list or if its a different
+	# size, or has an older timestamp
+	if ($toi < scalar(@todir)) {
+	    if (($fromi >= scalar(@fromdir)) ||
+		($todir[$toi] lt $fromdir[$fromi] || 
+		 ($todir[$toi] eq $fromdir[$fromi] && 
+		  &differentfiles("$fromdir/$fromdir[$fromi]","$todir/$todir[$toi]",
+				  $verbosity)))) {
+
+		# the files are different
+		&rm_r("$todir/$todir[$toi]");
+		splice(@todir, $toi, 1); # $toi stays the same
+
+	    } elsif ($todir[$toi] eq $fromdir[$fromi]) {
+		# the files are the same
+		# if it is a directory, check its contents
+		if (-d "$todir/$todir[$toi]") {
+		    &cachedir ("$fromdir/$fromdir[$fromi]",
+			       "$todir/$todir[$toi]", $verbosity);
+		}
+
+		$toi++;
+		$fromi++;
+		next;
+	    }
+	}
+  
+	# see if we should insert a file/directory
+	# we should insert a file/directory if there
+	# is no tofiles left or if the tofile does not exist
+	if ($fromi < scalar(@fromdir) && ($toi >= scalar(@todir) || 
+					  $todir[$toi] gt $fromdir[$fromi])) {
+	    &cp_r ("$fromdir/$fromdir[$fromi]", "$todir/$fromdir[$fromi]");
+	    splice (@todir, $toi, 0, $fromdir[$fromi]);
+
+	    $toi++;
+	    $fromi++;
+	}
+    }
+}
+
+# this function returns -1 if either file is not found
+# assumes that $file1 and $file2 are absolute file names or
+# in the current directory
+# $file2 is allowed to be newer than $file1
+sub differentfiles {
+    my ($file1, $file2, $verbosity) = @_;
+    $verbosity = 1 unless defined $verbosity;
+
+    $file1 =~ s/\/+$//;
+    $file2 =~ s/\/+$//;
+    
+    my ($file1name) = $file1 =~ /\/([^\/]*)$/;
+    my ($file2name) = $file2 =~ /\/([^\/]*)$/;
+
+    return -1 unless (-e $file1 && -e $file2);
+    if ($file1name ne $file2name) {
+	print STDERR "filenames are not the same\n" if ($verbosity >= 2);
+	return 1;
+    }
+
+    @file1stat = stat ($file1);
+    @file2stat = stat ($file2);
+
+    if (-d $file1) {
+	if (! -d $file2) {
+	    print STDERR "one file is a directory\n" if ($verbosity >= 2);
+	    return 1;
+	}
+	return 0;
+    }
+
+    # both must be regular files
+    unless (-f $file1 && -f $file2) {
+	print STDERR "one file is not a regular file\n" if ($verbosity >= 2);
+	return 1;
+    }
+
+    # the size of the files must be the same
+    if ($file1stat[7] != $file2stat[7]) {
+	print STDERR "different sized files\n" if ($verbosity >= 2);
+	return 1;
+    }
+
+    # the second file cannot be older than the first
+    if ($file1stat[9] > $file2stat[9]) {
+	print STDERR "file is older\n" if ($verbosity >= 2);
+	return 1;
+    }
+
+    return 0;
+}
+
+
+sub get_tmp_filename {
+    my $tmpdir = "$ENV{'GSDLHOME'}/tmp";
+    &mk_all_dir ($tmpdir) unless -e $tmpdir;
+
+    my $count = 1000;
+    my $rand = int(rand $count);
+    while (-e "$tmpdir/F$rand") {
+	$rand = int(rand $count);
+	$count++;
+    }
+
+    return "$tmpdir/F$rand";
+}
+
+
+sub filename_cat {
+    my (@filenames) = @_;
+    my $filename = join("/", @filenames);
+
+    # remove duplicate slashes and remove the last slash
+    $filename =~ s/[\\\/]+/\//g;
+    $filename =~ s/\/$//;
+
+    return $filename;
+}
+
+
+# if this is running on windows we want binaries to end in
+# .exe, otherwise they don't have to end in any extension
+sub get_os_exe {
+    return ".exe" if $ENV{'GSDLOS'} =~ /^windows$/i;
+    return "";
+}
+
+
+
+1;
Index: /trunk/gsdl/src/hashfile/hashfile.cpp
===================================================================
--- /trunk/gsdl/src/hashfile/hashfile.cpp	(revision 4)
+++ /trunk/gsdl/src/hashfile/hashfile.cpp	(revision 4)
@@ -0,0 +1,192 @@
+#include <stdio.h>
+
+#define MAXNUMLEN  100   // bytes
+
+
+// The number class is used to store very long unsigned integers.
+// The least significant of each number is stored first. Numbers can 
+// be no longer than MAXNUMLEN bytes long. A number with length=0
+// is considered to be equal to 0. There should be no extra zeros
+// at the front of the number.
+
+struct number {
+  unsigned int len;
+  unsigned char num[MAXNUMLEN];
+
+  number () {len = 0;}
+};
+
+
+// evaluates a = a + b
+// a and b can be the same number
+void my_inc (number &a, number &b) {
+  unsigned int carry = 0;
+  unsigned int num = 0;
+  unsigned int i = 0;
+
+  while ((carry || (i < a.len) || (i < b.len)) && (i < MAXNUMLEN)) {
+    num = carry;
+    if (i < a.len) num += a.num[i];
+    if (i < b.len) num += b.num[i];
+
+    if (num >= 256) {
+      num -= 256; // assumes that each number in the list < 256
+      carry = 1;
+    } else {
+      carry = 0;
+    }
+    a.num[i] = (unsigned char)num;
+
+    i++;
+  }
+
+  a.len = i;
+}
+
+
+// evaluates a = a - b if a >= b
+// returns true if the subtraction was possible
+int my_ifpos_dec (number &a, number &b) {
+  int carry = 0;
+  int num = 0;
+  unsigned int i = 0;
+  unsigned int len = 0; // the last non-zero digit
+
+  // make sure a >= b
+  if (b.len > a.len) return 0;
+  if (b.len == a.len) {
+    i = a.len - 1;
+    while (i >= 0) {
+      if (a.num[i] > b.num[i]) break;
+      if (a.num[i] < b.num[i]) return 0;
+      i--;
+    }
+  }
+
+  i = 0;
+  len = 0;
+  while ((i < a.len) || (i < b.len)) {
+    num = -carry;
+    if (i < a.len) num += a.num[i];
+    if (i < b.len) num -= b.num[i];
+
+    if (num < 0) {
+      num += 256; // assumes each number in the list < 256
+      carry = 1;
+    } else {
+      carry = 0;
+    }
+
+    a.num[i] = (unsigned char) num;
+    
+    i++;
+    if (num != 0) len = i;
+  }
+
+  a.len = len;
+
+  if (carry) {
+    // panic, this should NOT happen
+    fprintf (stderr, "panic: error in my_ifpos_dec\n");
+  }
+
+  return 1;
+}
+
+
+char *my_convert_num (number &a) {
+  char *result = new char[a.len+2];
+  int i,len;
+  char convert[16] = {'0','1','2','3','4','5','6','7',
+		      '8','9','a','b','c','d','e','f'};
+
+  // zero might be a special case
+  if (a.len == 0) {
+    result[0] = '0';
+    result[1] = '\0';
+    return result;
+  }
+
+  for (i=a.len-1, len=0; i >= 0; i--) {
+    result[len++] = convert[a.num[i]/16];
+    result[len++] = convert[a.num[i]%16];
+  }
+  result[len] = '\0';
+
+  return result;
+}
+
+
+char *hashfile (char *filename) {
+  FILE *infile = NULL;
+  int i;
+
+  // calculate the 8 multiples of the prime number to use
+  // in the long division
+  number primepow[8];
+  number pow;
+  number remainder;
+  unsigned char c;
+
+  // our prime number (618970019642690137449562111)
+  pow.num[0] = 255;  pow.num[1] = 255;
+  pow.num[2] = 255;  pow.num[3] = 255;
+  pow.num[4] = 255;  pow.num[5] = 255;
+  pow.num[6] = 255;  pow.num[7] = 255;
+  pow.num[8] = 255;  pow.num[9] = 255;
+  pow.num[10] = 255; pow.num[11] = 1;
+  pow.len = 12;
+
+  for (i=0; i<8; i++) {
+    primepow[i] = pow;
+    my_inc (pow, pow);
+  }
+
+  infile = fopen (filename, "rb");
+  if (infile == NULL) {
+    return NULL;
+  }
+  
+  remainder.len = 0;
+  c = (unsigned char)fgetc(infile);
+  while (!feof (infile)) {
+    // remainder = remainder * 256 + c
+    if (remainder.len == MAXNUMLEN-1) {
+      fprintf (stderr, "ERROR - number overflow\n");
+      return NULL;
+    }
+    for (i=remainder.len; i>0; i--) {
+      remainder.num[i] = remainder.num[i-1];
+    }
+    remainder.num[0] = c;
+    remainder.len = remainder.len+1;
+    
+    for (i=7; i>=0; i--) {
+      my_ifpos_dec (remainder, primepow[i]);
+    }
+
+    c = (unsigned char)fgetc(infile);
+  }
+
+  fclose (infile);
+
+  return my_convert_num (remainder);
+}
+
+
+int main (int argc, char *argv[]) {
+  int i;
+  char *result;
+
+  for (i = 1; i < argc; i++) {
+    result = hashfile (argv[i]);
+    if (result == NULL) {
+      printf ("ERROR - could not process %s\n", argv[i]);
+    } else {
+      printf ("%s: %s\n", argv[i], result);
+      delete [] result;
+    }
+  }
+
+  return 0;
+}
Index: /trunk/gsdl/src/library/browse.cpp
===================================================================
--- /trunk/gsdl/src/library/browse.cpp	(revision 4)
+++ /trunk/gsdl/src/library/browse.cpp	(revision 4)
@@ -0,0 +1,1 @@
+#include "browse.h"
Index: /trunk/gsdl/src/library/browse.h
===================================================================
--- /trunk/gsdl/src/library/browse.h	(revision 4)
+++ /trunk/gsdl/src/library/browse.h	(revision 4)
@@ -0,0 +1,41 @@
+#ifndef BROWSE_H
+#define BROWSE_H
+
+#include "text_t.h"
+#include "gdbmclass.h"
+#include "cgiargs.h"
+
+class browseclass 
+{
+  
+public:
+
+  browseclass () {}
+  virtual ~browseclass () {}
+
+  virtual void get_browse_bar(const text_t &targetdoc, text_t &return_text) {}
+
+  virtual void get_top_locator(cgiargsclass &args, gdbmclass &gdbm, int oversize, text_t &return_text) {}
+  virtual void get_links(cgiargsclass &args, gdbmclass &gdbm, text_t &return_text) {}
+  virtual void get_contents_arr(cgiargsclass &args, gdbmclass &gdbm, vector<text_t> &contents_arr) {}
+
+protected:
+
+  virtual void get_expanded_toc(cgiargsclass &args, text_t &booksection, 
+				const text_t &classification, gdbmclass &gdbm, 
+				text_t &return_text) {}
+  virtual void get_standard_toc(cgiargsclass &args, text_t &booksection, 
+				const text_t &classification, gdbmclass &gdbm, 
+				text_t &return_text) {}
+  virtual void get_alphabet_links (const text_t &booksection, gdbmclass &gdbm, 
+				   const text_t &collection, text_t &return_text) {}
+  virtual void get_title_arrows(text_t classification, gdbmclass &gdbm, 
+				const text_t &collection, text_t &alphabet_arrows) {}
+  virtual void get_contents (text_t expandlevel, gdbmclass &gdbm, 
+			     const text_t &collection, vector<text_t> &contents, 
+			     int &totalcols) {}
+
+};
+
+
+#endif
Index: /trunk/gsdl/src/library/cgiargs.cpp
===================================================================
--- /trunk/gsdl/src/library/cgiargs.cpp	(revision 4)
+++ /trunk/gsdl/src/library/cgiargs.cpp	(revision 4)
@@ -0,0 +1,83 @@
+#include "cgiargs.h"
+
+
+  
+// constructors
+cgiargsclass::cgiargsclass ()
+{
+}
+
+
+// this returns NULL if there isn't an entry with a value of
+// 'key' already defined
+
+void cgiargsclass::setarg (const text_t &key, const text_t &value)
+{
+  args[key] = value;
+}
+
+void cgiargsclass::setdefaultarg (const text_t &key, const text_t &value)
+{
+  if (getarg(key) == NULL) setarg (key, value);
+}
+
+void cgiargsclass::setintarg (const text_t &key, int value)
+{
+  setarg (key, value);
+}
+
+void cgiargsclass::setdefaultintarg (const text_t &key, int value)
+{
+  if (getarg(key) == NULL) setintarg (key, value);
+}
+
+void cgiargsclass::setcarg (const text_t &key, unsigned short c)
+{
+  text_t t; 
+  t.push_back(c);
+  setarg(key,t);
+}
+
+void cgiargsclass::setdefaultcarg (const text_t &key, unsigned short c)
+{
+  if (getarg(key) == NULL) setcarg (key, c);
+}
+
+text_t *cgiargsclass::getarg (const text_t &key)
+{
+  iterator here = args.find (key);
+  if (here == args.end()) return NULL;
+
+  return &((*here).second);
+}
+
+int cgiargsclass::getintarg (const text_t &key)
+{
+  text_t *text = getarg (key);
+  if (text == NULL) return 0;
+  return text->getint();
+}
+
+
+
+// stream operators to print cgi arguments for debugging purposes
+ostream &operator<<(ostream &outs, const cgiargsclass &args)
+{
+  outconvertclass text_t2ascii;
+  cgiargsclass::const_iterator here = args.begin ();
+  cgiargsclass::const_iterator end = args.end ();
+
+  outs << "*** cgiargsclass\n";
+  
+  while (here != end)
+    {
+      outs << text_t2ascii << " \"" << (*here).first << "\"=\"" << 
+	(*here).second << "\"\n";
+      here++;
+    }
+  outs << "\n";
+
+  return outs;
+}
+
+
Index: /trunk/gsdl/src/library/cgiargs.h
===================================================================
--- /trunk/gsdl/src/library/cgiargs.h	(revision 4)
+++ /trunk/gsdl/src/library/cgiargs.h	(revision 4)
@@ -0,0 +1,85 @@
+#ifndef CGIARGS_H
+#define CGIARGS_H
+
+
+#ifdef __GNUG__
+#  include <ostream.h>
+#else
+#  ifndef USE_OBJECTSPACE
+#    include <ostream>
+#  else
+#    include <ospace\std\ostream>
+#  endif
+#endif
+
+#ifndef USE_OBJECTSPACE
+#  include <map>
+#else
+#  include <ospace\std\map>
+#endif
+
+
+#include "text_t.h"
+
+typedef map<text_t, text_t, lttext_t> textmap;
+
+
+class cgiargsclass
+{
+protected:
+  textmap args;
+
+public:
+  //type support for textmap
+  typedef textmap::iterator iterator;
+  typedef textmap::const_iterator const_iterator;
+  typedef textmap::reference reference;
+  typedef textmap::const_reference const_reference;
+  typedef textmap::size_type size_type;
+  typedef textmap::difference_type difference_type;
+  typedef textmap::const_reverse_iterator const_reverse_iterator;
+  typedef textmap::reverse_iterator reverse_iterator;
+  
+  // constructors
+  cgiargsclass ();
+
+  // basic container support
+  iterator begin () {return args.begin();}
+  const_iterator begin () const {return args.begin();}
+  iterator end () {return args.end();}
+  const_iterator end () const {return args.end();}
+
+  void erase(iterator pos) {args.erase(pos);}
+  void erase(iterator first, iterator last) {args.erase(first, last);}
+  cgiargsclass &operator=(const cgiargsclass &x) {args=x.args;return *this;}
+
+  bool empty () const {return args.empty();}
+  size_type size() const {return args.size();}
+
+
+  // added functionality
+  void clear () {args.erase(args.begin(),args.end());}
+
+  // setdefaultarg, setdefaultintarg, and setdefaultcarg will 
+  // only set the argument if it is not already set
+  // getargs NULLs if there isn't an entry with
+  // 'key' already defined, getintarg returns 0 if there wasn't an
+  // entry with 'key' defined and operator[] returns "" if
+  // 'key' wasn't already defined (and sets 'key'="").
+  void setarg (const text_t &key, const text_t &value);
+  void setdefaultarg (const text_t &key, const text_t &value); 
+  void setintarg (const text_t &key, int value);
+  void setdefaultintarg (const text_t &key, int value);
+  void setcarg (const text_t &key, unsigned short c);
+  void setdefaultcarg (const text_t &key, unsigned short c);
+  text_t *getarg (const text_t &key);
+  int getintarg (const text_t &key);
+  text_t &operator[] (const text_t &key) {return args[key];}
+};
+
+
+// stream operators to print cgi arguments for debugging purposes
+ostream &operator<<(ostream &outs, const cgiargsclass &args);
+
+
+#endif
Index: /trunk/gsdl/src/library/conftools.cpp
===================================================================
--- /trunk/gsdl/src/library/conftools.cpp	(revision 4)
+++ /trunk/gsdl/src/library/conftools.cpp	(revision 4)
@@ -0,0 +1,43 @@
+#ifdef __GNUG__
+#  include <iostream.h>
+#  include <fstream.h>
+#else
+#  ifndef USE_OBJECTSPACE
+#    include <iostream>
+#    include <fstream>
+#  else
+#    include <ospace\std\iostream>
+#    include <ospace\std\fstream>
+#  endif
+#endif
+
+#include "text_t.h"
+#include "conftools.h"
+
+int write_confline (ofstream &fileout, const text_t &key, const text_t value) {
+  if (key.empty() || value.empty()) return -1;
+  outconvertclass text_t2ascii;
+  fileout << text_t2ascii << key << "=" << value << "\n";
+  return 0;
+}
+
+int get_confline (ifstream &filein, text_t &key, text_t &value) {
+  if (filein.eof()) return -1;
+
+  key.clear();
+  value.clear();
+  char c;
+  filein.get(c);
+
+  int foundeq = 0;
+  while (!filein.eof() && c != '\n') {
+    if (!foundeq && c == '=') {foundeq = 1; filein.get(c);}
+
+    if (foundeq) value.push_back(c);
+    else key.push_back(c);
+    filein.get(c);
+  }
+  if (key.empty()) return -1;
+  return 0;
+} 
+
Index: /trunk/gsdl/src/library/conftools.h
===================================================================
--- /trunk/gsdl/src/library/conftools.h	(revision 4)
+++ /trunk/gsdl/src/library/conftools.h	(revision 4)
@@ -0,0 +1,11 @@
+#ifndef CONFTOOLS_H
+#define CONFTOOLS_H
+
+#ifdef __WIN32__
+#include <ospace/std/fstream>
+#endif
+
+int write_confline (ofstream &fileout, const text_t &key, const text_t value);
+int get_confline (ifstream &filein, text_t &key, text_t &value);
+
+#endif
Index: /trunk/gsdl/src/library/display.cpp
===================================================================
--- /trunk/gsdl/src/library/display.cpp	(revision 4)
+++ /trunk/gsdl/src/library/display.cpp	(revision 4)
@@ -0,0 +1,1527 @@
+/**************************************************************************
+ *
+ * display.cpp -- Context sensitive macro language
+ * Copyright (c) 1998 -- Rodger McNab and Hong Chen
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*fopen
+   $Log$
+   Revision 1.1  1998/11/17 01:11:03  sjboddie
+   Initial revision
+
+   Revision 1.8  1998/06/24 01:49:28  rjmcnab
+   Added stuff for the objectspace libraries
+
+   Revision 1.7  1998/06/22 01:05:49  rjmcnab
+   Added a warning when a macro is not found
+
+   Revision 1.6  1998/06/18 23:58:03  rjmcnab
+   changed 'display' to 'displayclass'
+   added support to stream text through display with <<
+
+   Revision 1.5  1998/06/18 00:14:21  rjmcnab
+   fixed a bug in "macro" and added a comment about what
+   the use of macro names
+
+   Revision 1.4  1998/06/17 10:30:00  rjmcnab
+   added a simple _If_ statement
+
+   Revision 1.3  1998/06/17 08:51:08  rjmcnab
+   Added the expansion of macro options, some checking
+   for recursive macro calls, and changed 'set' to
+   'setmacro' in a few places as the MVC++ version 4.2
+   was getting 'set' confused with the STL 'set'
+
+   Revision 1.2  1998/06/15 22:00:29  rjmcnab
+   a lot of restructuring and simplifying
+
+   Revision 1.1  1998/06/13 20:06:05  rjmcnab
+   Initial revision
+
+*/
+
+static char *RCSID = "$Id$";
+
+
+#include "display.h"
+#include <assert.h>
+
+// include to get NULL
+#include <stdlib.h>
+
+#ifdef __GNUG__
+#  include <iostream.h>
+#  include <fstream.h>
+#else
+#  ifndef USE_OBJECTSPACE
+#    include <iostream>
+#    include <fstream>
+#  else
+#    include <ospace\std\iostream>
+#    include <ospace\std\fstream>
+#  endif
+#endif
+
+
+
+
+/////////////////////////////////////
+// misc classes
+/////////////////////////////////////
+
+
+// precedencetype defines a 'precedence value' for each parameter
+typedef map<text_t, double, lttext_t> precedencetype;
+
+
+// macro value structure
+struct mvalue
+{
+  text_t filename;
+  text_t value;
+};
+
+
+
+/////////////////////////////////////
+// stuff for defaultmacros_t
+/////////////////////////////////////
+  
+typedef map<text_t, mvalue, lttext_t> defparammap;
+typedef map<text_t, defparammap, lttext_t> defmacromap;
+typedef map<text_t, defmacromap, lttext_t> defpackagemap;
+
+// all methods in defaultmacros_t assume the parameters
+// and packages in a standard form and not empty
+class defaultmacros_t
+{
+protected:
+  defpackagemap macros;
+
+public:
+  typedef defpackagemap::const_iterator const_package_iterator;
+  typedef defpackagemap::size_type package_size_type;
+
+  // setmacro returns 0 if there was no error,
+  //            -1 if it redefined a macro
+  //            -2 if it hid a Global macro
+  //            -3 if it redefined a macro and hid a Global macro
+  //            -4 if either a package, macroname, or params were not supplied
+  int setmacro(const text_t &package,
+	  const text_t &macroname, 
+	  const text_t &params, 
+	  const text_t &filename, 
+	  const text_t &macrovalue);
+
+  void delmacro(const text_t &package,
+	      const text_t &macroname, 
+	      const text_t &params);
+
+  void clear () {macros.erase(macros.begin(), macros.end());}
+
+  // top level stuff
+  const_package_iterator package_begin () const {return macros.begin();}
+  const_package_iterator package_end () const {return macros.end();}
+  bool package_empty () const {return macros.empty();}
+  package_size_type package_size() const {return macros.size();}
+
+  // methods to find things
+  defmacromap *package_find (const text_t &packagename);
+  defparammap *macro_find (const text_t &packagename, 
+			   const text_t &macroname);
+  mvalue *parameter_find (const text_t &packagename, 
+			  const text_t &macroname,
+			  const text_t &parametername);
+};
+
+
+// setmacro returns 0 if there was no error,
+//            -1 if it redefined a macro
+//            -2 if it hid a Global macro
+//            -3 if it redefined a macro and hid a Global macro
+//            -4 if either a package, macroname, or params were not supplied
+int defaultmacros_t::setmacro(const text_t &package,
+			 const text_t &macroname, 
+			 const text_t &params, 
+			 const text_t &filename, 
+			 const text_t &macrovalue)
+{
+  text_t constGlobal("Global");
+  paramhashtype paramhash;
+
+  defmacromap *macromapptr;
+  defparammap *parammapptr;
+  mvalue *mvalueptr;
+
+  int warning = 0;
+
+  // check to see if we would be hiding a Global macro with the 
+  // same name. Note: it doesn't matter what parameters are used
+  // a Global macro will still be hid by a non-Global one with
+  // the same name.
+  if ((package != constGlobal) && 
+      (macro_find (constGlobal, macroname) != NULL))
+    {
+      warning -= 2; // found the macroname
+    }
+
+  // define this macro
+  macromapptr = &(macros[package]); // -- package
+  parammapptr = &((*macromapptr)[macroname]); // -- macro name
+  // -- parameters
+  if ((*parammapptr).find(params) != (*parammapptr).end()) {
+    warning -= 1; // found the parameters
+  }
+  mvalueptr = &((*parammapptr)[params]);
+
+  // -- value
+  (*mvalueptr).filename = filename;
+  (*mvalueptr).value = macrovalue;
+
+  return warning;
+}
+
+void defaultmacros_t::delmacro(const text_t &package,
+			     const text_t &macroname, 
+			     const text_t &params)
+{
+  // make sure everything was supplied
+  if (package.empty() || macroname.empty() || params.empty()) return;
+
+  // find the package and macroname
+  defparammap *parammapptr = macro_find(package, macroname);
+  if (parammapptr == NULL) return;
+
+  // find the parameters
+  defparammap::iterator parammapit = parammapptr->find(params);
+  if (parammapit == parammapptr->end()) return;
+
+  // finally delete this element
+  parammapptr->erase(parammapit);
+}
+
+defmacromap *defaultmacros_t::package_find (const text_t &packagename)
+{
+  if (packagename.empty()) return NULL;
+
+  defpackagemap::iterator it = macros.find(packagename);
+  if (it == macros.end()) return NULL;
+
+  return &((*it).second);
+}
+
+defparammap *defaultmacros_t::macro_find (const text_t &packagename, 
+					       const text_t &macroname)
+{
+  defmacromap *macromapptr = package_find(packagename);
+  if (macromapptr == NULL || macroname.empty()) return NULL;
+
+  defmacromap::iterator it = (*macromapptr).find(macroname);
+  if (it == (*macromapptr).end()) return NULL;
+
+  return &((*it).second);
+}
+
+mvalue *defaultmacros_t::parameter_find (const text_t &packagename, 
+					 const text_t &macroname,
+					 const text_t &parametername)
+{
+  defparammap *parammapptr = macro_find(packagename, macroname);
+  if (parammapptr == NULL || parametername.empty()) return NULL;
+
+  defparammap::iterator it = (*parammapptr).find(parametername);
+  if (it == (*parammapptr).end()) return NULL;
+
+  return &((*it).second);
+}
+
+
+
+/////////////////////////////////////
+// stuff for currentmacros_t
+/////////////////////////////////////
+
+typedef map<text_t, mvalue, lttext_t> curmacromap;
+typedef map<text_t, curmacromap, lttext_t> curpackagemap;
+
+// all methods in currentmacros_t assume the parameters
+// and packages in a standard form and not empty
+class currentmacros_t 
+{
+protected:
+  curpackagemap macros;
+
+public:
+  typedef curpackagemap::const_iterator const_package_iterator;
+  typedef curpackagemap::size_type package_size_type;
+
+  // setmacro returns 0 if there was no error,
+  //            -1 if it redefined a macro
+  //            -4 if either a package, or macroname were not supplied
+  int setmacro(const text_t &package,
+	  const text_t &macroname, 
+	  const text_t &filename, 
+	  const text_t &macrovalue);
+
+  void delmacro(const text_t &package,
+		const text_t &macroname);
+
+  void clear () {macros.erase(macros.begin(), macros.end());}
+
+  // top level stuff
+  const_package_iterator package_begin () const {return macros.begin();}
+  const_package_iterator package_end () const {return macros.end();}
+  bool package_empty () const {return macros.empty();}
+  package_size_type package_size() const {return macros.size();}
+
+  // methods to find things
+  curmacromap *package_find (const text_t &packagename);
+  mvalue *macro_find (const text_t &packagename, 
+		      const text_t &macroname);
+};
+
+
+// setmacro returns 0 if there was no error,
+//            -1 if it redefined a macro
+//            -4 if either a package, or macroname were not supplied
+int currentmacros_t::setmacro(const text_t &package,
+			 const text_t &macroname, 
+			 const text_t &filename, 
+			 const text_t &macrovalue)
+{
+  int warning = 0;
+
+  // make sure everything was supplied
+  if (package.empty() || macroname.empty()) return -4;
+
+  // define this macro
+  curmacromap *macromapptr = &(macros[package]);
+  if ((*macromapptr).find(macroname) != (*macromapptr).end()) 
+    {
+      warning -= 1; // found the macroname
+    }
+  mvalue *mvalueptr = &((*macromapptr)[macroname]);
+
+  // -- value
+  (*mvalueptr).filename = filename;
+  (*mvalueptr).value = macrovalue;
+
+  return warning;
+}
+
+void currentmacros_t::delmacro(const text_t &package,
+			     const text_t &macroname)
+{
+  // make sure everything was supplied
+  if (package.empty() || macroname.empty()) return;
+
+  // find the package
+  curmacromap *macromapptr = package_find(package);
+  if (macromapptr == NULL) return;
+
+  // find the macroname
+  curmacromap::iterator macromapit = macromapptr->find(macroname);
+  if (macromapit == macromapptr->end()) return;
+
+  // finally delete this element
+  macromapptr->erase(macromapit);
+}
+
+curmacromap *currentmacros_t::package_find (const text_t &packagename)
+{
+  if (packagename.empty()) return NULL;
+
+  curpackagemap::iterator it = macros.find(packagename);
+  if (it == macros.end()) return NULL;
+
+  return &((*it).second);
+}
+
+mvalue *currentmacros_t::macro_find (const text_t &packagename, 
+				     const text_t &macroname)
+{
+  curmacromap *macromapptr = package_find(packagename);
+  if (macromapptr == NULL || macroname.empty()) return NULL;
+
+  curmacromap::iterator it = (*macromapptr).find(macroname);
+  if (it == (*macromapptr).end()) return NULL;
+
+  return &((*it).second);
+}
+
+
+
+
+/////////////////////////////////////
+// support functions
+/////////////////////////////////////
+
+// this calculation of precedence will make one occurance of a high
+// precedence item override many occurances of lower precedence items
+void calcprecedence (const text_t &precedstr, precedencetype &precedhash) 
+{
+  text_t param;
+  double multiple = 5.0;
+  double nextprec = multiple;
+
+  text_t::const_iterator here, end;
+
+  // set precedhash to an empty hash
+  precedhash.erase(precedhash.begin(), precedhash.end());
+
+  // iterate through paramstring ignoring space and extracting
+  // out key value pairs.
+  here = precedstr.begin();
+  end = precedstr.end();
+  while (here != end) 
+    {
+      // get the next parameter
+      param.clear(); // set param to an empty string
+      
+      while (here != end) 
+	{
+	  if (*here == ',') 
+	    {
+	      // found the end of the parameter
+	      here++;
+	      break;
+	    } 
+	  else if (*here == ' ') 
+	    {
+	      // do nothing (ignore spaces)
+	    } 
+	  else 
+	    {
+	      // character in parameter
+	      param.push_back (*here); // copy this character
+	    }
+	  
+	  here++;
+	}
+
+      // if a parameter was found insert its value
+      if (!param.empty()) 
+	{
+	  precedhash[param] = nextprec;
+	  nextprec *= multiple;
+	}
+    }
+}
+
+
+// decides how specific any given parameter is to the page parameters. Returns 
+// negative if any options are different or else the sum of the precedence for
+// the options which match. 
+double getspecificness(const paramhashtype &pageparams, 
+		       const paramhashtype &theseparams, 
+		       const precedencetype &precedhash) 
+{
+  double specificness = 0.0;
+  paramhashtype::const_iterator here, pagevalueit;
+  precedencetype::const_iterator precedit;
+  text_t thesekey, thesevalue, constignore("ignore");
+  
+  here = theseparams.begin();
+  while (here != theseparams.end()) 
+    {
+      thesekey = (*here).first;
+      thesevalue = (*here).second;
+      
+      if (thesekey == constignore) 
+	{
+	  // do nothing, this is a placeholder
+	} 
+      else 
+	{
+	  pagevalueit = pageparams.find(thesekey);
+	  if ((pagevalueit != pageparams.end()) && 
+	      ((*pagevalueit).second == thesevalue)) 
+	    {
+	      // this parameter fits, add its precedence value
+	      precedit = precedhash.find(thesekey);
+	      if (precedit == precedhash.end()) 
+		specificness += 1.0; // no precedence defined
+	      else 
+		  specificness += (*precedit).second;
+	    }
+	  else 
+	    {
+	      return -1.0; // did not match
+	    }
+	}
+
+      here++;
+    }
+  
+  return specificness;
+}
+
+
+void splitparams (const text_t &paramstring, paramhashtype &paramhash)
+{
+  text_t key;
+  text_t value;
+
+  text_t::const_iterator here, end;
+
+  // set paramhash to an empty hash
+  paramhash.erase(paramhash.begin(), paramhash.end());
+
+  // iterate through paramstring ignoring space and extracting
+  // out key value pairs.
+  here = paramstring.begin();
+  end  = paramstring.end();
+  while (here != end) 
+    {
+      // reset key and value
+      key.clear();
+      value.clear();
+
+      // get key
+      while (here != end) 
+	{
+	  if (*here == ',') 
+	    {
+	      // have a key with no value
+	      break;
+	    } 
+	  else if (*here == '=') 
+	    {
+	      // found the end of the key
+	      here ++; 
+	      break;
+	    } 
+	  else if (*here == ' ') 
+	    {
+	      // do nothing (ignore spaces)
+	    } 
+	  else 
+	    {
+	      // character in key
+	      key.push_back (*here); // copy this character
+	    }
+	  here ++;
+	}
+
+      // get value
+      while (here != end) 
+	{
+	  if (*here == '=') 
+	    {
+	      // multiple values for one key, ignore previous
+	      // values
+	      value.clear();
+	    } 
+	  else if (*here == ',') 
+	    {
+	      // found the end of the value
+	      here ++;
+	      break;
+	    } 
+	  else if (*here == ' ') 
+	    {
+	      // do nothing (ignore spaces)
+	    } 
+	  else 
+	    {
+	      // character in value
+	      value.push_back (*here); // copy this character
+	    }
+	  here ++;
+	}
+    
+      // if a key was found insert the key/value pair in 
+      // the map 
+      if (!key.empty()) 
+	{
+	  paramhash[key] = value;
+	}
+    }
+}
+
+
+void joinparams (const paramhashtype &paramhash, text_t &paramstring)
+{
+  //set paramstring to an empty string
+  paramstring.clear();
+
+  //iterate through the paramhash
+  paramhashtype::const_iterator thepair;
+  int firstparam = 1;
+
+  for (thepair=paramhash.begin();thepair!=paramhash.end(); thepair++)
+    {
+      if (!firstparam) paramstring.push_back(',');
+      firstparam = 0;
+      
+      // insert pairs of "key=value"
+      text_t thevalue;
+      paramstring.append((*thepair).first);
+      paramstring.push_back('=');
+      paramstring.append((*thepair).second);
+    }
+}
+
+
+
+/////////////////////////////////////
+// parsing support functions
+/////////////////////////////////////
+
+inline int my_isspace (char ch) 
+{
+  unsigned char c = ch;
+  return (((c > 0) && (c <= 31)) || (c == ' '));
+}
+
+
+inline int my_isalpha (char c) 
+{
+  return ((c >= 'A' && c <= 'Z') ||
+	  (c >= 'a' && c <= 'z'));
+} 
+
+
+// as we are using one character lookahead the
+// value of line might be off by one.
+inline char my_get (istream &fin, int &line) 
+{
+  char c;
+  fin.get(c);
+  if (c == '\n') line++;
+  return c;
+}
+
+
+inline unsigned short my_ttnextchar (text_t::const_iterator &here, 
+				     text_t::const_iterator &end) 
+{
+  if (here != end) 
+    {
+      here++;
+      if (here != end) return (*here);
+    }
+  return '\0';
+}
+
+
+
+/////////////////////////////////////
+// methods for display
+/////////////////////////////////////
+
+// public methods for display
+
+displayclass::displayclass ()
+{
+  defaultmacros = new defaultmacros_t;
+  defaultfiles = new fileinfomap;
+  
+  orderparamlist = new paramspeclist;
+  currentmacros = new currentmacros_t;
+
+  outc = NULL;
+}
+
+
+displayclass::~displayclass ()
+{
+  delete defaultmacros;
+  delete defaultfiles;
+  
+  delete orderparamlist;
+  delete currentmacros;
+}
+  
+// setdefaultmacro adds an entry to the list of default macros 
+// returns 0 if there was no error,
+//        -1 if it redefined a macro
+//        -2 if it hid a Global macro
+//        -3 if it redefined a macro and hid a Global macro
+//        -4 if no macroname was supplied
+int displayclass::setdefaultmacro (text_t package, const text_t &macroname, 
+			      text_t params, const text_t &macrovalue) 
+{
+  return setdefaultmacro (macroname, package, params, macrovalue, "memory");
+}
+
+
+
+// loads a default macro file (if it isn't already loaded)
+// returns 0 if didn't need to load the file (it was already loaded)
+//         1 if was (re)loaded
+//        -1 an error occurred while trying to load the file
+int displayclass::loaddefaultmacros (text_t thisfilename)
+{
+  // convert the filename to a C string
+  char *filenamestr = thisfilename.getcstr();
+  outconvertclass text_t2ascii;
+
+  // see if we need to open this file -- this isn't implemented yet
+  
+  // open the file
+  ifstream fin(filenamestr);
+  if (fin.fail()) return -1; // read failed
+
+  text_t package = "Global";
+  int line  = 1;
+  char c = my_get(fin, line); // pre-fetch the next character
+
+  text_t macropackage, macroname, macroparameters, macrovalue;
+  int err; // for keeping track of whether an error occurred somewhere
+
+  while (!fin.eof()) 
+    {
+      // expect: white space, comment, "package", or macroname
+      if (my_isspace(c)) 
+	{
+	  // found some white-space
+	  c = my_get(fin, line);
+	}
+      else if (c == '#') 
+	{
+	  // found the start of a comment
+	  // skip all characters up to the end of the line
+	  c = my_get(fin, line); // skip the '#'
+	  while (!fin.eof ()) 
+	    {
+	      if (c == '\n') break;
+	      c = my_get(fin, line);
+	    }
+
+	} 
+      else if (c == 'p') 
+	{
+	  // found the start of 'package' (hopefully)
+	  // get everything up to the next space
+	  text_t tmp;
+	  while (!fin.eof() && my_isalpha(c)) 
+	    {
+	      tmp.push_back((unsigned char)c);
+	      c = my_get(fin, line);
+	    }
+	  // see if we have a package name
+	  if (tmp == "package") 
+	    {
+	      // skip all white space
+	      while (!fin.eof() && my_isspace(c)) 
+		c = my_get(fin, line);
+
+	      // get the package name
+	      tmp.clear(); // init tmp
+	      while (!fin.eof() && my_isalpha(c)) 
+		{
+		  tmp.push_back((unsigned char)c);
+		  c = my_get(fin, line);
+		}
+	      package = tmp;
+	      if (package.empty()) package = "Global";
+
+	    } 
+	  else 
+	    {
+	      // error
+	      cerr << "Expected 'package' on line " << line << "\n";
+	    }
+
+	} 
+      else if (c == '_') 
+	{
+	  // found the start of a macro (hopefully)
+	  c = my_get(fin, line); // skip the _
+
+	  // init variables
+	  err = 0;
+	  macropackage = package;
+	  macroname.clear(); // init macroname
+	  macroparameters.clear(); // init macroname
+	  macrovalue.clear(); // init macroname
+
+	  // get the macro name
+	  while ((!fin.eof()) && (!my_isspace(c)) && 
+		 (c != '\\') && (c != '_') &&(c != ':') && 
+		 (macroname.size() < 80)) 
+	    {
+	      macroname.push_back((unsigned char)c);
+	      c = my_get(fin, line);
+	    }
+	  
+	  if (c == ':') 
+	    {
+	      // we actually had the macro package
+	      c = my_get(fin, line); // skip :
+	      macropackage = macroname;
+	      macroname.clear ();
+
+	      // get the macro name (honest!)
+	      while ((!fin.eof()) && (!my_isspace(c)) && 
+		     (c != '\\') && (c != '_') &&(c != ':') && 
+		     (macroname.size() < 80)) 
+		{
+		  macroname.push_back((unsigned char)c);
+		  c = my_get(fin, line);
+		}
+	    }
+
+	  if (!err && c == '_') 
+	    {
+	      c = my_get(fin, line); // skip the _
+
+	      // skip all white space
+	      while (!fin.eof() && my_isspace(c)) c = my_get(fin, line);
+	    } 
+	  else err = 1;
+
+	  // get the macro parameters (optional)
+	  if (!err && c == '[') 
+	    {
+	      c = my_get(fin, line); // skip the [
+	      while ((!fin.eof()) && (c != '\n') && (c != '\\') && (c != ']')) 
+		{
+		  macroparameters.push_back((unsigned char)c);
+		  c = my_get(fin, line);
+		}
+
+	      if (c == ']') 
+		{
+		  c = my_get(fin, line); // skip the ]
+	  
+		  // skip all white space
+		  while (!fin.eof() && my_isspace(c)) c = my_get(fin, line);
+		} 
+	      else err = 2;
+	    }
+
+	  // get the macro value
+	  if (!err && c == '{') 
+	    {
+	      c = my_get(fin, line); // skip the {
+	      while ((!fin.eof()) && (c != '}')) 
+		{
+		  if (c == '\\') 
+		    {
+		      macrovalue.push_back((unsigned char)c); // keep the '\'
+		      c = my_get(fin, line); // store the *next* value regardless
+		      if (!fin.eof()) macrovalue.push_back((unsigned char)c);
+		      c = my_get(fin, line);
+		    }
+		  macrovalue.push_back((unsigned char)c);
+		  c = my_get(fin, line);
+		}
+
+	      if (c == '}') 
+		{
+		  c = my_get(fin, line); // skip the }
+	  
+		  // define the macro
+		  err = setdefaultmacro (macropackage, macroname, macroparameters,
+					 thisfilename, macrovalue);
+		  if (err == -1 || err == -3) 
+		    {
+		      cerr << text_t2ascii << "Warning: redefinition of _" << 
+			package << ":" << macroname << "_[" << macroparameters <<
+			"] on line ";
+		      cerr << line;
+		      cerr << text_t2ascii << " of " << thisfilename << "\n";
+	    
+		    } 
+		  else if (err == -2) 
+		    {
+		      cerr << text_t2ascii << "Warning: _" << 
+			package << ":" << macroname << "_[" << macroparameters <<
+			"] on line ";
+		      cerr << line;
+		      cerr << text_t2ascii << " of " << 
+			thisfilename << " hides a Global macro with the same name\n";
+	    
+		    } 
+		  else if (err == -4) 
+		    {
+		      cerr << text_t2ascii << "Error: macro name expected on line ";
+		      cerr << line ;
+		      cerr << text_t2ascii << " of " << thisfilename << "\n";
+		    }		  
+	  
+		  err = 0; // for the test below
+		} 
+	      else err = 3;
+	    } 
+	  else err = 4;
+
+	  if (err) 
+	    {
+	      // found an error, skip to the end of the line
+	      cerr << text_t2ascii << "Err: Unexpected input on line ";
+	      cerr << line;
+	      cerr << text_t2ascii << " (";
+	      cerr << err;
+	      cerr << text_t2ascii << ")\n";
+	      while (!fin.eof ()) 
+		{
+		  if (c == '\n') break;
+		  c = my_get(fin, line);
+		}
+	    }
+
+	} 
+      else 
+	{
+	  // found an error, skip to the end of the line
+	  cerr << "Error: Unexpected input on line " << line << "\n";
+	  while (!fin.eof ()) 
+	    {
+	      if (c == '\n') break;
+	      c = my_get(fin, line);
+	    }
+      
+	}
+    }
+  
+  fin.close ();
+  
+  // free up memory
+  delete filenamestr;
+  return 0;
+}
+
+
+// prepares to create a page.
+void displayclass::openpage (const text_t &thispageparams, 
+			const text_t &thisprecedence)
+{
+  // init variables
+  currentmacros->clear();
+
+  // reload any default macro files which have changed.
+  if (checkdefaultmacrofiles() < 0) 
+    {
+      // print error message
+    }
+
+  setpageparams (thispageparams, thisprecedence);
+}
+
+
+// changes the parameters for the current page.
+void displayclass::setpageparams (text_t thispageparams, 
+			     text_t thisprecedence)
+{
+  paramhashtype thissplitparams, splittheseparams;
+  precedencetype precedhash;
+  text_tset::iterator text_tsetit;
+  paramspec tmpps;
+
+  precedence = thisprecedence;
+  calcprecedence(thisprecedence, precedhash);
+
+  splitparams(thispageparams, thissplitparams);
+  joinparams(thissplitparams, thispageparams);
+  params = thispageparams;
+
+  // get a list of parameters with their specificness
+  orderparamlist->erase(orderparamlist->begin(), orderparamlist->end());
+  for (text_tsetit=allparams.begin();
+       text_tsetit!=allparams.end();
+       text_tsetit++) 
+    {
+      tmpps.param = (*text_tsetit);
+      splitparams(tmpps.param, splittheseparams);
+      tmpps.spec = getspecificness(thissplitparams, splittheseparams, precedhash);
+      if (tmpps.spec >= 0) 
+	{
+	  orderparamlist->push_back (tmpps);
+	}
+    }
+
+  // sort the list
+  sort(orderparamlist->begin(), orderparamlist->end());
+
+//   paramspeclist::iterator pshere = orderparamlist->begin();
+//   paramspeclist::iterator psend = orderparamlist->end();
+//   while (pshere != psend) 
+//     {
+//       cerr << text_t2ascii << "param=" << (*pshere).param;
+//       cerr << " spec=" << (int)((*pshere).spec) << "\n";
+//       pshere++;
+//     }
+}
+
+
+// overrides (or sets) a macro for the current page.
+// returns 0 if there was no error,
+//        -1 if it redefined a macro
+//        -4 if no macroname was supplied
+int displayclass::setmacro (const text_t &macroname, 
+		       text_t package, 
+		       const text_t &macrovalue)
+{
+  // make sure a macroname was supplied
+  if (macroname.empty()) return -4;
+
+  // make package "Global" if it doesn't point to anything yet
+  if (package.empty()) package = "Global";
+
+  // set the macro
+  return currentmacros->setmacro(package, macroname, "memory", macrovalue);
+}
+
+
+void displayclass::expandstring (const text_t &inputtext, text_t &outputtext)
+{
+  expandstring("", inputtext, outputtext);
+}
+
+
+void displayclass::expandstring (text_t package, const text_t &inputtext,
+			    text_t &outputtext, int recursiondepth)
+{
+  text_t macroname, macropackage, macroargs;
+  text_t::const_iterator tthere = inputtext.begin();
+  text_t::const_iterator ttend = inputtext.end();
+  unsigned short c;
+
+  if (package.empty()) package = "Global";
+
+  outputtext.clear();
+      
+  // use one-character lookahead
+  if (tthere != ttend) c = (*tthere);
+  while (tthere != ttend) 
+    {
+      if (c == '\\')
+	{
+	  // found an escape character
+	  c = my_ttnextchar (tthere, ttend); // skip the escape character
+	  if (tthere != ttend) 
+	    {
+	      if (c == 'n') outputtext.push_back('\n');
+	      else if (c == 't') outputtext.push_back('\t');
+	      else outputtext.push_back(c);
+	    }
+	  c = my_ttnextchar (tthere, ttend);
+	  
+	} 
+      else if (c == '_') 
+	{
+	  // might have found the start of a macro
+      
+	  // embedded macros (macros within the names
+	  // of other macros) are no longer supported -- Rodger.
+      
+	  // save the current state (in case we need to back out)
+	  text_t::const_iterator savedtthere = tthere;
+	  unsigned short savedc = c;
+	  
+	  macroname.clear();
+	  macropackage = package;
+	  macroargs.clear();
+      
+	  c = my_ttnextchar (tthere, ttend); // skip the '_'
+	  
+	  // get the macroname
+	  while (tthere != ttend && (!my_isspace(c)) && 
+		 (c != '\\') && (c != '_') &&(c != ':') && 
+		 (macroname.size() < 80)) 
+	    {
+	      macroname.push_back((unsigned char)c);
+	      c = my_ttnextchar (tthere, ttend);
+	    }
+      
+	  if (c == ':') 
+	    {
+	      // we actually had the macro package
+	      c = my_ttnextchar (tthere, ttend);
+	      macropackage = macroname;
+	      macroname.clear ();
+	
+	      // get the macro name (honest!)
+	      while ((tthere != ttend) && (!my_isspace(c)) && 
+		     (c != '\\') && (c != '_') &&(c != ':') && 
+		     (macroname.size() < 80)) 
+		{
+		  macroname.push_back((unsigned char)c);
+		  c = my_ttnextchar (tthere, ttend);
+		}
+	    }
+      
+	  if ((tthere != ttend) && (c == '_') &&
+	      (macroname.size() > 0)) 
+	    {
+	      // found a macro!!!! (maybe)
+	
+	      c = my_ttnextchar (tthere, ttend); // skip the '_'
+	
+	      // get the parameters (if there are any)
+	      if ((tthere != ttend) && (c == '(')) 
+		{
+		  c = my_ttnextchar (tthere, ttend); // skip '('
+	  
+		  // have to be careful of quotes
+		  unsigned short quote = '\0';
+		  while (tthere != ttend) 
+		    {
+		      if (c == '\\') 
+			{
+			  // have an escape character, save escape 
+			  // character and next character as well
+			  macroargs.push_back (c);
+			  c = my_ttnextchar (tthere, ttend); 
+	      
+			} 
+		      else if (quote == '\0') 
+			{
+			  // not in a quote at the moment
+			  if (c == ')') 
+			    {
+			      // found end of the arguments
+			      c = my_ttnextchar (tthere, ttend); // skip ')'
+			      break;
+		
+			    } 
+			  else if (c == '\'' || c == '\"') 
+			    {
+			      // found a quote
+			      quote = c;
+			    }
+	      
+			} 
+		      else 
+			{
+			  // we are in a quote
+			  if (c == quote) quote = '\0';
+			}
+	    
+		      // save this character
+		      if (tthere != ttend) macroargs.push_back (c);
+		      c = my_ttnextchar (tthere, ttend);
+		    }
+		}
+	
+	      //
+	      // we now have the macropackage, macroname, and macroargs
+	      //
+	
+	      // try to expand out this macro
+	      text_t expandedmacro;
+	      if (macro (macroname, macropackage, macroargs, expandedmacro, recursiondepth)) 
+		{
+		  // append the expanded macro
+		  outputtext += expandedmacro;
+		} 
+	      else
+		{
+		  // wasn't a macro
+		  tthere = savedtthere;
+		  c = savedc;
+		  outputtext.push_back(c); // the '_'
+		  c = my_ttnextchar (tthere, ttend);
+		}
+	
+
+	    }
+	  else
+	    {
+	      // wasn't a macro
+	      tthere = savedtthere;
+	      c = savedc;
+	      outputtext.push_back(c); // the '_'
+	      c = my_ttnextchar (tthere, ttend);
+	    }
+
+	}
+      else 
+	{
+	  // nothing of interest
+	  outputtext.push_back(c); // the '_'
+	  c = my_ttnextchar (tthere, ttend);
+	}
+    }
+}
+
+
+// protected methods for display
+
+// reloads any default macro files which have changed
+// returns 0 no errors occurred
+//        -1 an error occurred while trying to load one of the files
+int displayclass::checkdefaultmacrofiles ()
+{
+  // this isn't implemented yet
+  return 0;
+}
+
+// setdefaultmacro adds an entry to the list of default macros 
+// returns 0 if there was no error,
+//        -1 if it redefined a macro
+//        -2 if it hid a Global macro
+//        -3 if it redefined a macro and hid a Global macro
+//        -4 if no macroname was supplied
+int displayclass::setdefaultmacro (text_t package, const text_t &macroname, 
+			      text_t params, const text_t &filename,
+			      const text_t &macrovalue) 
+{
+  // make sure a macroname was supplied
+  if (macroname.empty()) return -4;
+
+  // put the parameters in a standard form
+  paramhashtype paramhash;
+  if (params.empty()) params = "ignore=yes";
+  splitparams (params, paramhash);
+  joinparams (paramhash, params);
+
+  // make package "Global" if it doesn't point to anything yet
+  if (package.empty()) package = "Global";
+
+  // remember these parameters
+  allparams.insert (params);
+
+  // remember this filename (this part isn't finished yet -- Rodger).
+
+  // set the macro
+  return defaultmacros->setmacro(package, macroname, params, filename, macrovalue);
+}
+
+// (recursively) expand out a macro
+// returns true  if the macro was found
+//         false if the macro was not found
+bool displayclass::macro (const text_t &macroname, text_t macropackage, 
+		     const text_t &macroparam, text_t &outputtext,
+		     int recursiondepth)
+{
+  outconvertclass text_t2ascii;
+  text_tlist splitmacroparam;
+  text_t::const_iterator hereit, endit;
+  text_t aparam;
+  unsigned short c, quote;
+
+  //  cerr << "r: " << recursiondepth << "\n";
+
+  // check for deep recursion
+  if (recursiondepth >= MAXRECURSIONDEPTH)
+    {
+      cerr << "Warning: deep recursion, limiting macro expansion\n";
+      return false;
+    }
+
+  // check the macropackage
+  if (macropackage.empty()) macropackage = "Global";
+
+  // get the parameters (but don't expand them)
+  if (macroparam.size() > 0) {
+    // set up for loop
+    hereit = macroparam.begin();
+    endit = macroparam.end();
+    if (hereit != endit) c = (*hereit);
+
+    while (hereit != endit) {
+      // get the next parameter
+      aparam.clear();
+      quote = '\0'; // not-quoted unless proven quoted
+      
+      // ignore initial whitespace
+      while ((hereit!=endit)&&my_isspace(c)) c=my_ttnextchar(hereit,endit);
+      
+      // look for the end of the parameter
+      while (hereit != endit) {
+	if (c == '\\') {
+	  // found escape character, also get next character
+	  aparam.push_back(c);
+	  c = my_ttnextchar (hereit, endit);
+	  if (hereit != endit) aparam.push_back(c);
+	  
+	} else if (quote=='\0' && (c=='\'' || c=='"')) {
+	  // found a quoted section
+	  quote = c;
+
+	} else if (quote!='\0' && c==quote) {
+	  // found the end of a quote
+	  quote = '\0';
+
+	} else if (quote=='\0' && c==',') {
+	  // found the end of a parameter
+	  c = my_ttnextchar (hereit, endit);
+	  break;
+
+	} else {
+	  // ordinary character
+	  aparam.push_back(c);
+	}
+
+	c = my_ttnextchar (hereit, endit);
+      }
+
+      // add this parameter to the list
+      splitmacroparam.push_back(aparam);
+    }
+  }
+
+  // I'm only supporting a simple version of _If_ at the moment
+  if (macroname == "If")
+    {
+      // get the condition, then clause and else clause
+      text_tlist::iterator paramcond = splitmacroparam.begin();
+      text_tlist::iterator paramend = splitmacroparam.end();
+      text_tlist::iterator paramthen = paramend;
+      text_tlist::iterator paramelse = paramend;
+
+      if (paramcond != paramend)
+	{
+	  paramthen = paramcond;
+	  paramthen++;
+	}
+      if (paramthen != paramend)
+	{
+	  paramelse = paramthen;
+	  paramelse++;
+	}
+
+      // will always output something
+      outputtext.clear();
+
+      // expand out the first parameter
+      if (paramcond != paramend)
+	{
+	  text_t tmpoutput;
+	  expandstring (macropackage, *paramcond, tmpoutput, recursiondepth+1);
+	  
+	  // test the expanded string
+	  if (tmpoutput.size() > 1 || 
+	      (tmpoutput.size() == 1 && tmpoutput[0] != '0'))
+	    {
+	      // true
+	      if (paramthen != paramend)
+		expandstring (macropackage, *paramthen, outputtext, recursiondepth+1);
+	    }
+	  else
+	    {
+	      // false
+	      if (paramelse != paramend)
+		expandstring (macropackage, *paramelse, outputtext, recursiondepth+1);
+	    }
+	}
+
+      return true;
+    }
+
+  // try and find this macro
+
+  // this list might be replaced by something a little more
+  // sophisticated in the future.
+  text_tlist packagelist;
+  packagelist.push_back (macropackage);
+  packagelist.push_back ("Global");
+
+  paramspeclist::iterator paramhere, paramend;
+  text_tlist::iterator packagehere = packagelist.begin();
+  text_tlist::iterator packageend = packagelist.end();
+  mvalue *macrovalue = NULL;
+  while ((macrovalue == NULL) && (packagehere != packageend)) 
+    {
+      // first look in the currentmacros
+      macrovalue = currentmacros->macro_find(*packagehere, macroname);
+      if (macrovalue == NULL)
+	macrovalue = currentmacros->macro_find("Style", macroname);
+
+      // look in the default macros
+      if (macrovalue == NULL) 
+	{
+	  // next look in the defaultmacros
+	  if (defaultmacros->macro_find (*packagehere, macroname) != NULL)
+	    {
+	      paramhere = orderparamlist->begin(); paramend = orderparamlist->end();
+	      while ((macrovalue == NULL) && (paramhere != paramend))
+		{
+// 		  cerr << text_t2ascii << "\npackage: " << *packagehere << "\n";
+// 		  cerr << text_t2ascii << "macroname: " << macroname << "\n";
+// 		  cerr << text_t2ascii << "param: " << (*paramhere).param << "\n";
+// 		  cerr << "spec: " << (int)((*paramhere).spec) << "\n";
+		  macrovalue = defaultmacros->parameter_find(*packagehere, macroname,
+							     (*paramhere).param);
+		  paramhere++;
+		}
+	    }
+	}
+
+      // and again in the package "Style"
+      if (macrovalue == NULL) 
+	{
+	  // next look in the defaultmacros
+	  if (defaultmacros->macro_find ("Style", macroname) != NULL)
+	    {
+	      paramhere = orderparamlist->begin(); paramend = orderparamlist->end();
+	      while ((macrovalue == NULL) && (paramhere != paramend))
+		{
+		  macrovalue = defaultmacros->parameter_find("Style", macroname,
+							     (*paramhere).param);
+		  paramhere++;
+		}
+	    }
+	}
+      if (macrovalue == NULL) packagehere++;
+    }
+
+  if (macrovalue != NULL)
+    {
+      // we found a macro
+
+      // replace all the option macros (_1_, _2_, ...) in the 
+      // value of this macro
+      text_t tempmacrovalue;
+      tempmacrovalue.clear();
+      hereit = macrovalue->value.begin();
+      endit = macrovalue->value.end();
+      if (hereit != endit) c = (*hereit);
+      while (hereit != endit)
+	{
+	  if (c == '\\')
+	    {
+	      // found an escape character
+	      tempmacrovalue.push_back(c);
+	      c = my_ttnextchar (hereit, endit);
+	      if (hereit != endit) tempmacrovalue.push_back(c);
+	      c = my_ttnextchar (hereit, endit);
+	    }
+	  else if (c == '_')
+	    {
+	      text_t::const_iterator savedhereit = hereit;
+	      unsigned short c2 = my_ttnextchar (hereit, endit);
+	      unsigned short c3 = my_ttnextchar (hereit, endit);
+	      if ((c2 >= '1') && (c2 <= '9') && (c3 == '_'))
+		{
+		  // found an option macro, append the appropriate text
+		  text_tlist::iterator ttlisthere = splitmacroparam.begin();
+		  text_tlist::iterator ttlistend = splitmacroparam.end();
+		  while ((c2 > '1') && (ttlisthere != ttlistend))
+		    {
+		      c2--;
+		      ttlisthere++;
+		    }
+		  if (ttlisthere != ttlistend)
+		    tempmacrovalue.append(*ttlisthere);
+
+		  c = my_ttnextchar (hereit, endit);
+
+		}
+	      else
+		{
+		  // wasn't a option macro
+		  tempmacrovalue.push_back(c);
+		  hereit = savedhereit;
+		  c = my_ttnextchar (hereit, endit);
+		}
+	    }
+	  else
+	    {
+	      tempmacrovalue.push_back(c);
+	      c = my_ttnextchar (hereit, endit);
+	    }
+	}
+      
+      // recursively replace this macro
+      expandstring (*packagehere, tempmacrovalue, outputtext, recursiondepth+1);
+
+      return true;
+    }
+
+  cerr << text_t2ascii << "Warning: _" << 
+    macropackage << ":" << macroname << "_ not found\n";
+  
+  return false; // didn't find the macro
+}
+
+
+void displayclass::printdefaultmacros ()
+{
+  defpackagemap::const_iterator packagemapit;
+  defmacromap::const_iterator macromapit;
+  defparammap::const_iterator parammapit;
+  const mvalue *mvalueptr;
+  outconvertclass text_t2ascii;
+
+  text_t packagename, macroname, macroparam;
+  
+  for (packagemapit = defaultmacros->package_begin();
+       packagemapit != defaultmacros->package_end();
+       packagemapit++) 
+    {
+      packagename = (*packagemapit).first;
+      //     cerr << "***package   : \"" << packagename << "\"\n";
+    
+      for (macromapit = (*packagemapit).second.begin();
+	   macromapit != (*packagemapit).second.end();
+	   macromapit++) 
+	{
+	  macroname = (*macromapit).first;
+	  //       cerr << "***macroname : \"" << macroname << "\"\n";
+
+	  for (parammapit = (*macromapit).second.begin();
+	       parammapit != (*macromapit).second.end();
+	       parammapit++) 
+	    {
+	      macroparam = (*parammapit).first;
+	      mvalueptr = &((*parammapit).second);
+	      cerr << text_t2ascii << "package   : \"" << packagename << "\"\n";
+	      cerr << text_t2ascii << "macroname : \"" << macroname << "\"\n";
+	      cerr << text_t2ascii << "parameters: [" << macroparam << "]\n";
+	      cerr << text_t2ascii << "filename  : \"" << mvalueptr->filename << "\"\n";
+	      cerr << text_t2ascii << "value     : {" << mvalueptr->value << "}\n\n";
+	    }
+	}
+    }
+}
+
+void displayclass::printallparams ()
+{
+  text_tset::iterator text_tsetit;
+  outconvertclass text_t2ascii;
+
+  cerr << text_t2ascii << "** allparams\n";
+  for (text_tsetit=allparams.begin();
+       text_tsetit!=allparams.end();
+       text_tsetit++) {
+    cerr << text_t2ascii << (*text_tsetit) << "\n";
+  }
+  cerr << text_t2ascii << "\n";
+}
+
+
+
+/////////////////////////////////////
+// stuff to do tricky output
+/////////////////////////////////////
+
+displayclass &operator<< (outconvertclass &theoutc, displayclass &display)
+{
+  display.setconvertclass (&theoutc);
+  return display;
+}
+
+displayclass &operator<< (displayclass &display, const text_t &t)
+{
+  outconvertclass *theoutc = display.getconvertclass();
+
+  if (theoutc == NULL) return display;
+  
+  text_t output;
+  display.expandstring (t, output);
+  (*theoutc) << output;
+
+  return display;
+}
Index: /trunk/gsdl/src/library/display.h
===================================================================
--- /trunk/gsdl/src/library/display.h	(revision 4)
+++ /trunk/gsdl/src/library/display.h	(revision 4)
@@ -0,0 +1,229 @@
+/**************************************************************************
+ *
+ * display.h
+ *
+ * This is an implementation of a macro language that is context 
+ * sensitive. It is used to generate pages which have different
+ * 'versions'. Many variants of a macro are possible, macro
+ * resolution is done at expansion time by finding the best
+ * macro for the 'situation'.
+ *
+ * This implementation relies heavily on the Standard Template
+ * Library.
+ *
+ * Copyright (c) 1998 -- Rodger McNab and Hong Chen
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+#ifndef DISPLAY_H
+#define DISPLAY_H
+
+#include "text_t.h"
+
+
+#ifndef USE_OBJECTSPACE
+#  include <vector>
+#  include <list>
+#  include <set>
+#  include <map>
+#  include <algorithm>
+#else
+#  include <ospace\std\vector>
+#  include <ospace\std\list>
+#  include <ospace\std\set>
+#  include <ospace\std\map>
+#  include <ospace\std\algorithm>
+#endif
+
+// use the standard namespace
+#ifdef __GNUG__
+// namespaces are broken in current version of gcc
+#else
+#  ifndef USE_OBJECTSPACE
+using namespace std;
+#  else
+#    ifndef OS_NO_NAMESPACE
+using namespace ospace::std;
+#    endif
+#  endif
+#endif
+
+
+
+// MAXRECURSIONDEPTH is a cutoff to catch
+// cyclic macros (a includes b and b includes a)
+#define MAXRECURSIONDEPTH 30
+
+
+// class prototypes
+class defaultmacros_t;
+class currentmacros_t;
+
+// a few supporting types
+// fileelement isn't finished yet, I have to find out
+// more about getting information about files
+struct fileinfoelement 
+{
+  int otherinfo;
+};
+
+typedef map<text_t, fileinfoelement, lttext_t> fileinfomap;
+
+
+typedef map<text_t, text_t, lttext_t> paramhashtype;
+
+
+struct paramspec 
+{
+  double spec;
+  text_t param;
+};
+
+inline bool operator==(const paramspec& x, const paramspec& y) 
+{
+  return ((x.spec == y.spec) && (x.param == y.param));
+}
+
+inline bool operator!=(const paramspec& x, const paramspec& y) 
+{
+  return ((x.spec != y.spec) || (x.param != y.param));
+}
+
+// note: paramspec is sorted by reverse spec and then param
+
+inline bool operator<(const paramspec& x, const paramspec& y) 
+{
+  return ((x.spec > y.spec) ||
+	  ((x.spec == y.spec) && (x.param < y.param)));
+}
+
+inline bool operator>(const paramspec& x, const paramspec& y) 
+{
+  return ((x.spec < y.spec) ||
+	  ((x.spec == y.spec) && (x.param > y.param)));
+}
+
+typedef vector<paramspec> paramspeclist;
+
+
+// NOTE: when macros are used within text they should have
+// underscores '_' on both sides, however, in calls to
+// setdefaultmacro and setmacro they should not have the
+// underscores. So you might have
+//
+// setmacro ("aname", "query", " some text ");
+// expandstring ("_aname_", expandedtext);
+//
+// because the input to expandstring is a block of text, not
+// a macroname
+
+class displayclass 
+{
+public:
+  displayclass ();
+  ~displayclass ();
+  
+  // setdefaultmacro adds an entry to the list of default macros 
+  // returns 0 if there was no error,
+  //        -1 if it redefined a macro
+  //        -2 if it hid a Global macro
+  //        -3 if it redefined a macro and hid a Global macro
+  //        -4 if no macroname was supplied
+  int setdefaultmacro (text_t package, const text_t &macroname, 
+		       text_t params, const text_t &macrovalue);
+  
+  // loads a default macro file (if it isn't already loaded)
+  // returns 0 if didn't need to load the file (it was already loaded)
+  //         1 if was (re)loaded
+  //        -1 an error occurred while trying to load the file
+  int loaddefaultmacros (text_t thisfilename);
+  
+  // prepares to create a page: deletes all macros set with
+  // 'setmacro', checks all the default macro files to see
+  // if any need reloading, and sets the page parameters.
+  void openpage (const text_t &thispageparams, 
+		 const text_t &thisprecedence);
+  
+  // changes the parameters for the current page.
+  void setpageparams (text_t thispageparams, 
+		      text_t thisprecedence);
+  
+  // overrides (or sets) a macro for the current page.
+  // returns 0 if there was no error,
+  //        -1 if it redefined a macro
+  //        -4 if no macroname was supplied
+  int setmacro (const text_t &macroname, 
+		text_t package, 
+		const text_t &macrovalue);
+  
+  void expandstring (const text_t &inputtext, text_t &outputtext);
+  void expandstring (text_t package, const text_t &inputtext,
+		     text_t &outputtext, int recursiondepth = 0);
+
+  // these functions are not ment to be used directly, they are used to permit 
+  // concise stream output like:
+  // cout << text_t2ascii << display << "_amacro_" << "_anothermacro_";
+  void setconvertclass (outconvertclass *theoutc) {outc = theoutc;}
+  outconvertclass *getconvertclass () {return outc;}
+
+
+  // debug stuff
+  void printdefaultmacros ();
+  void printallparams ();
+
+protected:
+  // special variables to permit trickly output
+  outconvertclass *outc;
+
+  // general variables
+  text_tset allparams;  // possible parameter combinations
+  defaultmacros_t *defaultmacros;
+  fileinfomap *defaultfiles;
+  
+  // variables to do with a page expansion
+  text_t params;
+  text_t precedence;
+  paramspeclist *orderparamlist; // ordered by specificness
+  currentmacros_t *currentmacros; // only macros set by setmacro
+  
+  // reloads any default macro files which have changed
+  // returns 0 no errors occurred
+  //        -1 an error occurred while trying to load one of the files
+  int checkdefaultmacrofiles ();
+
+  int setdefaultmacro (text_t package, const text_t &macroname, 
+		       text_t params, const text_t &filename,
+		       const text_t &macrovalue); 
+
+  // (recursively) expand out a macro
+  // returns true  if the macro was found
+  //         false if the macro was not found
+  bool macro (const text_t &macroname, text_t macropackage, 
+	      const text_t &macroparam, text_t &outputtext,
+	      int recursiondepth);
+};
+
+
+displayclass &operator<< (outconvertclass &theoutc, displayclass &display);
+displayclass &operator<< (displayclass &display, const text_t &t);
+
+
+#endif
+
Index: /trunk/gsdl/src/library/gdbmclass.cpp
===================================================================
--- /trunk/gsdl/src/library/gdbmclass.cpp	(revision 4)
+++ /trunk/gsdl/src/library/gdbmclass.cpp	(revision 4)
@@ -0,0 +1,575 @@
+#include "text_t.h"
+#include "gdbmclass.h"
+#include <ctype.h>
+
+#ifndef USE_OBJECTSPACE
+#  include <algorithm>
+#else
+#  include <ospace\std\algorithm>
+#endif
+
+
+void gdbm_info::clear () {
+  d = 0;
+  t.clear();
+  p.clear();
+  x.clear();
+  c.clear();
+  j.clear();
+  o.clear();
+  a.clear();
+}
+
+// checks if key is present in infodb.
+// returns 1 on success 0 on failure
+int gdbmclass::exists (text_t key, text_t collection) {
+
+  datum key_data;
+  datum return_data;
+
+  // try to open the database if it's not already open
+  if (!open_database(collection)) {
+    if (logout != NULL) (*logout) << "gdbmclass: can't open database\n"; 
+    return 0;
+  }
+  
+  // sanity check
+  if (gdbm_file == NULL) {
+    if (logout != NULL) (*logout) << "gdbmclass: failed sanity check\n";	
+    return 0;
+  }
+
+  char *db_key = key.getcstr();
+
+  if (db_key == NULL) {
+    if (logout != NULL) (*logout) << "gdbmclass: failed sanity check\n";
+    return 0;
+  }
+
+  key_data.dptr = db_key;
+  key_data.dsize = strlen (db_key);
+  return_data = gdbm_fetch (gdbm_file, key_data);
+  delete db_key;
+  db_key = NULL;
+
+  if (return_data.dptr == NULL) {
+    if (logout != NULL) {
+      (*logout) << "gdbmclass: null data pointer\n";
+      outconvertclass text_t2ascii;
+      (*logout) << text_t2ascii << "gdbmclass: key was \"" << key << "\"\n";
+    }
+    return 0;
+  }
+  free (return_data.dptr);
+  return 1;
+}
+
+
+// returns 0 on success, -1 on failure
+// key and collection aren't references as they might be aliases to
+// something in info
+int gdbmclass::getinfo (text_t key, text_t collection, gdbm_info &info) {
+
+  info.clear (); // reset info
+  
+  datum key_data;
+  datum return_data;
+  char ikey[256];
+  char ivalue[16384];
+  int pos = 0;
+
+  // try to open the database if it's not already open
+  if (!open_database(collection)) {
+    if (logout != NULL) (*logout) << "gdbmclass: can't open database\n"; 
+    return -1;
+  }
+  
+  // sanity check
+  if (gdbm_file == NULL) {
+    if (logout != NULL) (*logout) << "gdbmclass: failed sanity check\n";	
+    return -1;
+  }
+
+  char *db_key = key.getcstr();
+
+  if (db_key == NULL) {
+    if (logout != NULL) (*logout) << "gdbmclass: failed sanity check\n";
+    return -1;
+  }
+
+  key_data.dptr = db_key;
+  key_data.dsize = strlen (db_key);
+  return_data = gdbm_fetch (gdbm_file, key_data);
+  delete db_key;
+  db_key = NULL;
+
+  if (return_data.dptr == NULL) {
+    if (logout != NULL) {
+      (*logout) << "gdbmclass: null data pointer\n";
+	  outconvertclass text_t2ascii;
+      (*logout) << text_t2ascii << "gdbmclass: key was \"" << key << "\"\n";
+    }
+    return -1;
+  }
+  
+  while ((pos = get_infoline(return_data.dptr, return_data.dsize, 
+			     pos, ikey, ivalue)) >= 0) {
+
+    if (strcmp (ikey, "d") == 0) {
+      info.d = atoi(ivalue);
+      
+    } else if (strcmp (ikey, "p") == 0) {
+      info.p = ivalue;
+
+    } else if (strcmp (ikey, "t") == 0) {
+      info.t = ivalue;
+      
+    } else if (strcmp (ikey, "x") == 0) {
+      info.x = ivalue;
+      
+    } else if (strcmp (ikey, "c") == 0) {
+      info.c = ivalue;
+      
+    } else if (strcmp (ikey, "j") == 0) {
+      info.j = ivalue;
+
+	} else if (strcmp (ikey, "o") == 0) {
+      info.o = ivalue;
+
+	} else if (strcmp (ikey, "a") == 0) {
+      info.a = ivalue;
+    }
+  }
+  
+  free (return_data.dptr);
+  
+  return 0;
+}
+
+void gdbmclass::setgdbmhome (const text_t &thegdbmhome)
+{
+  gdbmhome = thegdbmhome;
+}
+
+// parses a line of the form <key>value\n
+// returns next position, -1 if there was nothing left to process
+// in the string, -2 if there was an error
+int gdbmclass::get_infoline (char *str, int len, int pos, char *key, char *value) {
+  int keylen = 0;
+  int valuelen = 0;
+  
+  key[0] = '\0';
+  value[0] = '\0';
+  
+  // ignore white space
+  while (pos < len && isspace(str[pos])) pos++;
+  
+  // get the '<'
+  if (pos >= len) return -1;
+  if (str[pos] != '<') return -2;
+  pos++;
+  
+  // get the key
+  while (pos < len && str[pos] != '>')
+    key[keylen++] = str[pos++];
+  
+  key[keylen] = '\0';
+  
+  // get the '>'
+  if (pos >= len || str[pos] != '>') return -2;
+  pos++;
+  
+  // get the value
+  while (pos < len && str[pos] != '\n') 
+    value[valuelen++] = str[pos++];
+  
+  value[valuelen] = '\0';
+  
+  return pos; 
+}
+
+
+// returns 0 if failed, 1 if opened
+int gdbmclass::open_database (const text_t &collection) {
+  text_t data_location, text_dir;
+  int block_size = 0;
+  
+  if (gdbm_file != NULL) {
+    if (collection != gdbm_open_name) {
+      close_database();
+    }
+  }
+  if (gdbm_file == NULL) {
+    gdbm_open_name = collection;
+    gettextsuffix (collection, text_dir);
+
+#ifdef __WIN32__
+    data_location = gdbmhome + "\\" + text_dir;
+#else
+    data_location = gdbmhome + "/" + text_dir;
+#endif
+        
+#ifdef _LITTLE_ENDIAN
+    data_location += ".ldb"; // little endian version of the gdbm database
+#else
+    data_location += ".bdb"; // big endian version on the gdbm database
+#endif
+    char *namebuffer = data_location.getcstr();	
+    gdbm_file = gdbm_open (namebuffer, block_size, GDBM_READER, 00664, NULL);
+    delete namebuffer;
+  }
+  return (gdbm_file != NULL);
+}
+
+
+void gdbmclass::close_database () {
+  
+  if (gdbm_file == NULL) return;
+  
+  gdbm_close (gdbm_file);
+  gdbm_file = NULL;
+}
+
+
+
+// a few useful functions
+
+//////////////////////////////////////////////////////////////////////////////////////////
+// functions for testing classification strings
+
+
+// returns 1 if targetdoc is top level of a book (i.e. =~ /B\.\d+$/) - otherwise 0;
+int is_top_level (const text_t &targetdoc) {
+
+  text_t::const_iterator here = targetdoc.begin();
+  text_t::const_iterator end = targetdoc.end();
+
+  // look for the 'B'
+  here = findchar (here, end, 'B');
+
+  // there must be exactly one dot after the 'B'
+  if ((here != end) && (countchar (here, end, '.') == 1))
+    return 1;
+
+  return 0;
+}
+
+// returns 1 if targetdoc is any level of a book (i.e. contains 'B') - otherwise 0 
+int is_book (const text_t &targetdoc) {
+
+  text_t::const_iterator here = targetdoc.begin();
+  text_t::const_iterator end = targetdoc.end();
+  
+  here = findchar (here, end, 'B');
+  if (here != end) return 1;
+  return 0;
+}
+
+// returns (in book_top) the top level of the book in targetdoc 
+void get_book_top (const text_t &targetdoc, text_t &book_top) {
+
+  text_t::const_iterator here = targetdoc.begin();
+  text_t::const_iterator end = targetdoc.end();
+
+  book_top.clear();
+
+  // look for the 'B'
+  here = findchar (here, end, 'B');
+
+  // copy up to the second '.'
+  int founddot = 0;
+  while (here != end) {
+    if (*here == '.') {
+      if (founddot) return;
+      founddot = 1;
+    }
+    book_top.push_back(*here);
+    here++;
+  }
+}	
+
+// returns (in book) the book section part of the classification
+// contained in targetdoc
+void get_book (const text_t &targetdoc, text_t &book) {
+
+  text_t::const_iterator here = targetdoc.begin();
+  text_t::const_iterator end = targetdoc.end();
+  
+  book.clear ();
+
+  // look for the 'B'
+  here = findchar (here, end, 'B');
+
+  // copy the rest of the string
+  while (here != end) {
+    book.push_back(*here);
+    here ++;
+  }
+}
+
+// get_parent_section removes the last part from section (i.e.=~ s/\.\d+$//)  
+void get_parent_section (text_t &section) {
+  int founddot = 0;
+  text_t::iterator end;
+  while (!founddot && !section.empty()) {
+    end = section.end();
+    end --;
+    if (*end == '.') founddot = 1;
+    section.pop_back();
+  }
+}
+
+// same as above but also returns ths child section that's removed
+void get_parent_section (text_t &parentsection, text_t &childsection) {
+ int founddot = 0;
+ text_t tmp;
+ childsection.clear();
+  text_t::iterator end;
+  while (!founddot && !parentsection.empty()) {
+    end = parentsection.end();
+    end --;
+    if (*end == '.') founddot = 1;
+    else tmp.push_back(*end); childsection = tmp + childsection; tmp.clear();
+    parentsection.pop_back();
+  }
+}
+
+// count_dots returns the number of dots ('.') there are
+// in a range of a targetdoc string
+int count_dots(text_t::const_iterator first, text_t::const_iterator last) {
+  return countchar (first, last, '.');
+}
+
+int count_dots (const text_t &targetdoc) {
+  return count_dots(targetdoc.begin(), targetdoc.end());
+}
+
+// returns 1 if targetdoc is a first level descendant
+// (i.e. B.n.1, B.n.1.1, B.n.1.1.1 etc.) - otherwise 0
+int is_section_top(const text_t &targetdoc) {
+  text_t::const_iterator here = targetdoc.begin();
+  text_t::const_iterator end = targetdoc.end();
+  
+  // look for the 'B'
+  here = findchar (here, end, 'B');
+  here = findchar (here, end, '.');
+  if (here != end) here++; // skip over the '.'
+  here = findchar (here, end, '.');
+
+  // make sure that all '.' are followed by a '1'
+  while (here != end) {
+    if (*here != '.') return 0;
+    here ++;
+
+    if (here != end) {
+      if (*here != '1') return 0;
+      here ++;
+    }
+  }
+  return 1;
+}
+
+// seperate_parts seperates targetdoc into its classification and booksection
+// if classification isn't supplied it gets the first classification for the 
+// book from the gdbm
+// if booksection doesn't exist it remains blank
+void seperate_parts(const text_t &targetdoc, gdbmclass &gdbm, const text_t &collection, 
+		    text_t &classification, text_t &booksection) {
+  
+  split_targetdoc (targetdoc, classification, booksection);
+
+  if (classification.empty()) {
+    // no classification included so get first one for this book
+    gdbm_info info;
+    text_t book_top;
+    vector<text_t> classarray;
+    get_book_top (targetdoc, book_top);
+    gdbm.getinfo(book_top, collection, info);
+    splitstring (info.x, classarray);
+    if (!classarray.empty()) classification = classarray[0];
+	else classification = "C.1";
+  }
+}
+
+// split_targetdoc splits up a string containing a classification
+// and book (or one or the other)
+void split_targetdoc(const text_t &targetdoc, text_t &classification, 
+		     text_t &booksection) {
+  
+  classification.clear ();
+  booksection.clear();
+
+  text_t::const_iterator here = targetdoc.begin();
+  text_t::const_iterator end = targetdoc.end();
+
+  // copy everything up to the first 'B'
+  while (here != end) {
+    if (*here == 'B') break;
+    classification.push_back(*here);
+    here++;
+  }
+
+  // remove middle '.'
+  if (!classification.empty() && 
+      classification[classification.size()-1] == '.') 
+    classification.pop_back();
+
+  // copy the rest of the string
+  while (here != end) {
+    booksection.push_back(*here);
+    here++;
+  }
+}
+
+// splitstring splits a colon seperated string into an array
+void splitstring (const text_t &string, vector<text_t> &array) {
+  splitchar (string.begin(), string.end(), ':', array);
+}
+
+// get_parents returns the parents array containing all the parents of the
+// document specified by classification and booksection
+void get_parents (const text_t &targetdoc, vector<text_t> &parents)
+{
+  text_t::const_iterator here = targetdoc.begin ();
+  text_t::const_iterator end = targetdoc.end ();
+
+  text_t currentparent;
+  text_t newsuffixpart;
+  text_t newsuffix;
+  bool first = true;
+  while (here != end)
+    {
+      // if there is a newsuffix add it to the current parent
+      // and add that parent to the parents vector
+      if (!newsuffix.empty())
+	{
+	  currentparent += newsuffix;
+	  parents.push_back (currentparent);
+	}
+      
+      // keep getting suffixes until one is found which starts with
+      // a number
+      newsuffix.clear();
+      do 
+	{
+	  here = getdelimitstr (here, end, '.', newsuffixpart);
+	  if (!first) newsuffix.push_back ('.');
+	  first = false;
+	  newsuffix += newsuffixpart;
+	} 
+      while ((here != end) && !newsuffixpart.empty() && 
+	     (newsuffixpart[0] < '0' || newsuffixpart[0] > '9'));
+    }
+}
+
+
+// get_siblings returns the siblings array containing all the siblings of the current
+// classification or booksection
+void get_siblings (const text_t &classification, const text_t &booksection, 
+		   gdbmclass &gdbm, const text_t &collection, 
+		   vector<text_t> &siblings) {
+  
+  gdbm_info info;
+  
+  if (booksection.empty() &&  classification.size() == 1) {
+    // top level classification has no siblings
+    return;
+
+  } else if (booksection.empty()) {
+    // get classification siblings
+    gdbm.getinfo(classification, collection, info);
+    gdbm.getinfo(info.p, collection, info); // info is now parent info
+    splitstring(info.c, siblings);
+    return;
+
+  } else {
+    // get book section siblings
+    if (is_top_level(booksection)) {
+      // top level of book so siblings are children of classification
+      gdbm.getinfo(classification, collection, info);
+      splitstring(info.c, siblings);
+
+      // add classifications to book sections
+      for (unsigned int i = 0; i < siblings.size(); i++) {
+	if (is_book(siblings[i])) siblings[i] = classification + "." + siblings[i];
+      } 
+
+    } else {
+      // siblings come from immediate parent
+      gdbm.getinfo(booksection, collection, info);
+      gdbm.getinfo(info.p, collection, info); // info is now parent info
+      splitstring(info.c, siblings);
+
+      // add classifications to book sections
+      for (unsigned int i = 0; i < siblings.size(); i++) {
+	if (is_book(siblings[i])) siblings[i] = classification + "." + siblings[i];
+      } 
+    }
+  }
+}
+
+// compares section 1 and section 2 and returns 1 if section2 belongs to 
+// the same chapter as section1 (i.e. is sibling of or child of or child of sibling) 
+int are_same_chapter(text_t section1, text_t section2)
+{
+  get_parent_section(section1);
+
+  while (!section2.empty()) {
+    get_parent_section(section2);
+    if (section2 == section1) return 1;
+  }
+  return 0;
+}
+
+// get_first_section gets the first section from a colon separated
+// list (instring)
+void get_first_section(const text_t &instring, text_t &returnstring) {
+
+  returnstring.clear();
+  
+  text_t::const_iterator here = instring.begin();
+  text_t::const_iterator end = instring.end();
+  
+  while (here != end) {
+    if (*here == ':') return;
+    returnstring.push_back(*here);
+	here ++;
+  }
+}
+
+
+// removes html tags from string - everything after < will be removed
+// if < occurs without >
+void remove_tags (text_t &text) 
+{
+  text_t::const_iterator here = text.begin ();
+  text_t::const_iterator end = text.end ();
+  int found = 0;
+  text_t tmp;
+
+  while (here != end) {
+    if (*here == '<') {found = 1; here ++; continue;}
+    if (*here == '>') {found = 0; here ++; continue;}
+
+    if (!found) tmp.push_back(*here);
+    here ++;
+  }
+  text = tmp;
+}
+
+// checks text to see if it is a number (i.e. contains only 0-9)
+// returns 1 if true, 0 if false
+int is_number (text_t &text) {
+
+  text_t::const_iterator here = text.begin();
+  text_t::const_iterator end = text.end();
+
+  while (here != end) {
+    if ((*here!='0') && (*here!='1') && (*here!='2') &&
+	(*here!='3') && (*here!='4') && (*here!='5') &&
+	(*here!='6') && (*here!='7') && (*here!='8') &&
+	(*here!='9')) return 0;
+    here ++;
+  }
+  return 1;
+}
Index: /trunk/gsdl/src/library/gdbmclass.h
===================================================================
--- /trunk/gsdl/src/library/gdbmclass.h	(revision 4)
+++ /trunk/gsdl/src/library/gdbmclass.h	(revision 4)
@@ -0,0 +1,84 @@
+#ifndef GDBMCLASS_H
+#define GDBMCLASS_H
+
+#ifdef __WIN32__
+#include "autoconf.h"
+#include "systems.h"
+#include "gdbmconst.h"
+#include "gdbm.h"
+
+#else
+#include <gdbm.h>
+#endif
+
+#include "locateinfo.h"
+
+
+class gdbm_info 
+{
+public:
+  int docnum;
+  int d;
+  text_t t;
+  text_t p;
+  text_t x;
+  text_t c;
+  text_t j;
+  text_t o;
+  text_t a;
+
+  void clear ();
+};
+
+
+class gdbmclass 
+{
+public:
+  gdbmclass() {gdbm_file = NULL; logout = &cerr;};
+  ~gdbmclass() {};
+  
+  // returns 0 on success, -1 on failure
+  int getinfo (text_t key, text_t collection, gdbm_info &info);
+  void setgdbmhome (const text_t &thegdbmhome);
+  void setlogout (ostream *thelogout) {logout = thelogout;}
+  int exists (text_t key, text_t collection);
+
+protected:
+  text_t gdbmhome; // top level directory for gdbm files
+  GDBM_FILE gdbm_file;
+  text_t gdbm_open_name;
+  ostream *logout;
+  
+  int open_database (const text_t &collection);
+  void close_database ();
+  
+  int get_infoline (char *str, int len, int pos, char *key, char *value);
+};
+
+
+// a few useful functions
+
+int is_top_level (const text_t &targetdoc);
+int is_book (const text_t &targetdoc);
+void get_book_top (const text_t &targetdoc, text_t &book_top);
+void get_book (const text_t &targetdoc, text_t &book);
+void get_parent_section (text_t &section);
+void get_parent_section (text_t &parentsection, text_t &childsection);
+int count_dots (text_t::const_iterator first, text_t::const_iterator last);
+int count_dots (const text_t &targetdoc);
+int is_section_top (const text_t &targetdoc);
+void remove_tags (text_t &text);
+
+void seperate_parts (const text_t &targetdoc, gdbmclass &gdbm, const text_t &collection, 
+		     text_t &classification, text_t &booksection);
+void split_targetdoc (const text_t &targetdoc, text_t &classification, 
+		     text_t &booksection);
+void splitstring (const text_t &string, vector<text_t> &array);
+void get_parents (const text_t &targetdoc, vector<text_t> &parents);
+void get_siblings (const text_t &classification, const text_t &booksection, 
+		   gdbmclass &gdbm, const text_t &collection, 
+		   vector<text_t> &siblings);
+int are_same_chapter(text_t section1, text_t section2);
+void get_first_section(const text_t &instring, text_t &returnstring);
+int is_number (text_t &text);
+#endif
Index: /trunk/gsdl/src/library/libinterface.cpp
===================================================================
--- /trunk/gsdl/src/library/libinterface.cpp	(revision 4)
+++ /trunk/gsdl/src/library/libinterface.cpp	(revision 4)
@@ -0,0 +1,1012 @@
+#include <string.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <ctype.h>
+#include <time.h>
+#include "libinterface.h"
+#include "cgiargs.h"
+
+#include <assert.h>
+
+///////////////////////
+// support functions //
+///////////////////////
+
+unsigned short hexdigit (unsigned short c)
+{
+  if (c >= '0' && c <= '9') return (c-'0');
+  if (c >= 'a' && c <= 'f') return (c-'a'+10);
+  if (c >= 'A' && c <= 'F') return (c-'A'+10);
+  return c;
+}
+
+
+void c2hex (unsigned short c, text_t &t)
+{
+  t.clear();
+
+  if (c >= 256) 
+    {
+      t = "20"; // ' '
+      return;
+    }
+
+  unsigned short o1, o2;
+ 
+  o1 = (c/16) % 16;
+  o2 = c % 16;
+  if (o1 >= 10) o1 += 'a' - 10;
+  else o1 += '0';
+  if (o2 >= 10) o2 += 'a' - 10;
+  else o2 += '0';
+
+  t.push_back(o1);
+  t.push_back(o2);
+}
+
+// convert %xx and + to their appropriate equivalents
+void decode (text_t &argstr)
+{
+  text_t::iterator in = argstr.begin();
+  text_t::iterator out = in;
+  text_t::iterator end = argstr.end();
+
+  while (in != end)
+    {
+      if (*in == '+') *out = ' ';
+
+      else if (*in == '%')
+	{
+	  unsigned short c = '%';
+	  in++;
+	  if (in != end)
+	    {
+	      c = hexdigit (*in);
+	      in++;
+	    }
+	  if (in != end && c < 16) // sanity check on the previous character
+	    {
+	      c = c*16 + hexdigit (*in);
+	    }
+
+	  *out = c;
+	}
+      else *out = *in;
+
+      if (in != end) in++;
+      out++;
+    }
+  
+  // remove the excess characters
+  argstr.erase (out, end);
+}
+
+
+// split up the cgi arguments
+void parse_cgi_args (text_t argstr, cgiargsclass &args)
+{
+  args.clear();
+
+  text_t::iterator here = argstr.begin();
+  text_t::iterator end = argstr.end();
+  text_t key, value;
+
+  // extract out the key=value pairs
+  while (here != end) 
+    {
+      // get the next key and value pair
+      here = getdelimitstr (here, end, '=', key);
+      here = getdelimitstr (here, end, '&', value);
+
+      // convert %xx and + to their appropriate equivalents
+      decode (value);
+      // store this key=value pair
+      if (!key.empty()) args.setarg (key, value);
+    }
+}
+
+text_t cgisafe (text_t &intext)
+{
+  text_t outtext;
+
+  text_t::iterator here = intext.begin ();
+  text_t::iterator end = intext.end ();
+  unsigned short c;
+  text_t ttmp;
+
+  while (here != end)
+    {
+      c = *here;
+      if (((c >= 'a') && (c <= 'z')) ||
+	  ((c >= 'A') && (c <= 'Z')) ||
+	  ((c >= '0') && (c <= '9'))) {
+	// alphanumeric character
+	outtext.push_back(c);
+      } else {
+	  // non-alphnumeric character
+	  outtext.push_back('%');
+	  c2hex(c, ttmp);
+	  outtext += ttmp;
+      }
+	  
+      here++;
+    }
+
+  return outtext;
+}
+
+//////////////////////////////
+// methods for libinterface //
+//////////////////////////////
+
+// constructor
+
+libinterface::libinterface() 
+{
+  browse = NULL;
+}
+
+void libinterface::setgsdlhome (const text_t &thegsdlhome)
+{
+  gsdlhome = thegsdlhome;
+  setmacroshome (thegsdlhome);
+  setgdbmhome (thegsdlhome);
+  setindexhome (thegsdlhome);
+}
+
+void libinterface::setmacroshome (const text_t &themacroshome)
+{
+  macroshome = themacroshome;
+}
+
+void libinterface::setgdbmhome (const text_t &thegdbmhome)
+{
+  gdbmhome = thegdbmhome;
+  gdbm.setgdbmhome(gdbmhome);
+}
+
+void libinterface::setindexhome (const text_t &theindexhome)
+{
+  indexhome = theindexhome;
+  search.setindexhome(theindexhome);
+}
+
+void libinterface::sethttpprefix (const text_t &thehttpprefix)
+{
+  httpprefix = thehttpprefix;
+}
+
+void libinterface::setgwcgi (const text_t &thegwcgi)
+{
+  gwcgi = thegwcgi;
+}
+
+// init should be called after the various homes are set,
+// it returns 'false' on failure and 'true' on success
+bool libinterface::init ()
+{
+  set_default_index();
+  text_t collection = get_collection_name();
+
+  // load up default macro files -- these shouldn't have absolute file names
+#ifdef __WIN32__
+  disp.loaddefaultmacros(macroshome + "\\macros\\base.dm");
+  disp.loaddefaultmacros(macroshome + "\\macros\\browse.dm");
+  disp.loaddefaultmacros(macroshome + "\\macros\\text.dm");
+  disp.loaddefaultmacros(macroshome + "\\macros\\query.dm");
+  disp.loaddefaultmacros(macroshome + "\\macros\\help.dm");
+  disp.loaddefaultmacros(macroshome + "\\macros\\gsdl.dm");
+  disp.loaddefaultmacros(macroshome + "\\macros\\pref.dm");
+#else
+  disp.loaddefaultmacros(macroshome + "/macros/base.dm");
+  disp.loaddefaultmacros(macroshome + "/macros/browse.dm");
+  disp.loaddefaultmacros(macroshome + "/macros/text.dm");
+  disp.loaddefaultmacros(macroshome + "/macros/query.dm");
+  disp.loaddefaultmacros(macroshome + "/macros/help.dm");
+  disp.loaddefaultmacros(macroshome + "/macros/gsdl.dm");
+  disp.loaddefaultmacros(macroshome + "/macros/pref.dm");
+#endif
+
+  srand(time(NULL));
+
+  return collection_init(collection);
+}
+
+
+// examine the cgi arguments and create the appropriate page, 
+// outputing the page to textout and any debug information to logout
+//
+// returns LI_NOERROR on success, LI_LOADDATABASEFAILED on failure
+int libinterface::getpage (const text_t &argstr, ostream &textout, ostream &logout)
+{
+  int err = LI_NOERROR;
+
+  gdbm.setlogout (&logout); // make the output go where they want!
+
+  cgiargsclass args;
+
+  parse_cgi_args (argstr, args);
+  expand_compressed_args (args);
+  add_default_args (args);
+  check_args (args);
+
+  lastcomparg = get_compressed_args (args);
+
+  logout << args;
+
+  text_t &arg_a = args["a"];
+  if (arg_a == "q") err = query_action (args, textout, logout);
+  else if (arg_a == "b") browse_action (args, textout, logout);
+  else if (arg_a == "t") document_action (args, textout, logout);
+  else if (arg_a == "p") page_action (args, textout, logout);
+  else 
+    {
+      // output error page
+    }
+
+  return err;
+}
+
+// the arg config string is used to do processing on the arguments
+// entries take the form argname[defaultvalue]
+// if the argument name is preceeded by a "+" it means that the
+// value may be more than one character long
+// the main state variable missed on this list is "q" the query string
+void libinterface::get_arg_config (text_t &argconfigstr)
+{
+  argconfigstr = 
+    "a[p]"     // action: q=query, b=browse, t=targetdoc, p=page
+    "t[1]"     // query type: 0=boolean, 1=ranked
+    "i[c]"     // index: c=chapter, p=paragraph, t=title, b=book
+    "k[1]"     // casefolding: 0=off, 1=on
+    "s[0]"     // stemming: 0=off, 1=on
+    "+p[home]" // page
+    "+c[unu2]" // collection
+    "+r[1]"    // results from
+    "+d[C.1]"  // the target document
+    "+j[11]"   // partial index: 11=all, 10=f&n, 01=other
+    "+m[100]"   // maxdocs
+    "+o[20]"   // hits per page
+    "v[0]"     // version: 0=text+graphics, 1=text
+    "f[0]";    // query box size: 0=normal, 1=big
+  
+}
+
+text_t::iterator libinterface::get_next_config_arg (text_t::iterator first, 
+						    text_t::iterator last,
+						    text_t &argname, 
+						    text_t &defaultvalue, 
+						    bool &longarg)
+{
+  first = getdelimitstr (first, last, '[', argname);
+  first = getdelimitstr (first, last, ']', defaultvalue);
+  longarg = false;
+
+  if (!argname.empty() && (argname[0] == '+'))
+    {
+      argname.erase(argname.begin(), argname.begin()+1);
+      longarg = true;
+    }
+
+  return first;
+}
+
+text_t libinterface::get_compressed_args (cgiargsclass &args)
+{
+  text_t argconfigstr; get_arg_config (argconfigstr);
+  text_t arg_e;
+  text_t argname, defaultvalue;
+  text_t *argvalue;
+  bool longarg;
+
+  text_t::iterator here = argconfigstr.begin();
+  text_t::iterator end = argconfigstr.end();
+  while (here != end)
+    {
+      here = get_next_config_arg (here, end, argname, defaultvalue, longarg);
+
+      if (!argname.empty())
+	{
+	  argvalue = args.getarg (argname);
+	  if (argvalue == NULL) arg_e += defaultvalue;
+	  else arg_e += *argvalue;
+	    
+	  if (longarg) arg_e += "-";
+	}
+    }
+  
+  return arg_e;
+}
+
+
+// the compressed options should never override explicit options
+// but they should always be expanded before add_default_args is
+// called
+void libinterface::expand_compressed_args (cgiargsclass &args)
+{
+  text_t *arg_e = args.getarg("e");
+
+  // see if there is compressed options
+  if (arg_e != NULL)
+    {
+      text_t argconfigstr; get_arg_config (argconfigstr);
+      text_t argname, defaultvalue, argvalue;
+      bool longarg;
+
+      text_t::iterator confighere = argconfigstr.begin();
+      text_t::iterator configend = argconfigstr.end();
+
+      text_t::iterator arghere = arg_e->begin();
+      text_t::iterator argend = arg_e->end();
+      while (confighere != configend && arghere != argend)
+	{
+	  confighere = get_next_config_arg (confighere, configend, argname, 
+					    defaultvalue, longarg);
+	  if (!argname.empty())
+	    {
+	      if (longarg)
+		{
+		  arghere = getdelimitstr (arghere, argend, '-', argvalue);
+		  if (!argvalue.empty()) args.setdefaultarg (argname, argvalue);
+		}
+	      else
+		{
+		  args.setdefaultcarg (argname,*arghere);
+		  arghere++;
+		}
+	    }
+	}
+    }
+}
+
+
+void libinterface::add_default_args (cgiargsclass &args)
+{ 
+  text_t argconfigstr; get_arg_config (argconfigstr);
+  text_t argname, defaultvalue;
+  bool longarg;
+
+  text_t::iterator confighere = argconfigstr.begin();
+  text_t::iterator configend = argconfigstr.end();
+  while (confighere != configend)
+    {
+      confighere = get_next_config_arg (confighere, configend, argname, 
+					defaultvalue, longarg);
+      if (!argname.empty()) args.setdefaultarg (argname, defaultvalue);
+    }
+  
+  // the query string and format string are not included in the argument configuration string
+  args.setdefaultarg ("q", ""); // the default query string is ""
+  args.setdefaultarg ("g", "00");
+  args.setdefaultarg ("x", "0");
+}
+
+
+// check and attempt to fix an problems encountered in the list
+// of cgi arguments
+void libinterface::check_args (cgiargsclass &args)
+{
+  text_t collection = get_collection_name();
+  args.setarg("c", collection);
+}
+
+
+void libinterface::define_general_macros (cgiargsclass &args, ostream &logout)
+{
+  disp.setmacro("httpprefix", "Global", httpprefix);
+  disp.setmacro("gwcgi", "Global", gwcgi);
+
+  disp.setmacro("collection", "Global", cgisafe(args["c"]));
+  disp.setmacro("compressedoptions", "Global", get_compressed_args(args));
+  disp.setmacro("urlsafequerystring", "Global", cgisafe(args["q"]));
+
+  // need to escape any quotes in querystring to prevent them upsetting the html
+  text_t querystring;
+  text_t::iterator here = args["q"].begin();
+  text_t::iterator end = args["q"].end();
+  while (here != end) {
+    if (*here == '"') querystring += "&quot;";
+    else querystring.push_back(*here);
+    here ++;
+  }
+  disp.setmacro("querystring", "Global", querystring);
+
+
+  if (args.getintarg("x") == 0) disp.setmacro("notdetached", "Global", "1");
+  if (args["d"][0] == 'T') disp.setmacro("istitle", "Global", "1");
+
+  int i = rand();
+  disp.setmacro("pagedest", "Global", text_t(i));
+
+  // define the macro for the "g" argument
+  disp.setmacro("g", "Global", args["g"]);
+}
+
+
+// prepare_page prepares to write out a page using the current 
+// page parameters and defines any general macros
+void libinterface::prepare_page (cgiargsclass &args, ostream &logout)
+{
+  // get page parameters
+  text_t pageparams = text_t("collection=") + args["c"];
+  if (args.getintarg("u") == 1) pageparams += ",style=htmlonly";
+  if (args.getintarg("v") == 1) pageparams += ",version=text";
+  if (args.getintarg("f") == 1) pageparams += ",queryversion=big";
+
+  disp.openpage(pageparams, MACROPRECEDENCE);
+  define_general_macros(args, logout);
+  define_collection_macros(args, logout);
+}
+
+void libinterface::set_query_params (cgiargsclass &args, queryparamclass &queryparams)
+{
+  queryparams.collection = args["c"];
+  assemble_index (args, queryparams.search_index);
+  queryparams.querystring = args["q"];
+  format_querystring (queryparams.querystring);
+  queryparams.search_type = args.getintarg ("t");
+  queryparams.casefolding = args.getintarg ("k");
+  queryparams.stemming = args.getintarg ("s");
+  queryparams.maxdocs = args.getintarg ("m");
+}
+
+void libinterface::format_querystring (text_t &querystring)
+{
+  text_t formattedstring;
+  quotedstring.clear();
+
+  text_t::iterator here = querystring.begin();
+  text_t::iterator end = querystring.end();
+  int foundquote = 0;
+
+  // want to remove ()|!& from querystring so boolean queries are just
+  // "all the words" queries
+  while (here != end) {
+    if (*here == '(' || *here == ')' || *here == '|' ||
+	*here == '!' || *here == '&') {
+      formattedstring += " ";
+    } else {
+      if (*here == '"') {
+	if (foundquote) {foundquote = 0; quotedstring.push_back(*here);}
+	else foundquote = 1;
+      } else {
+	formattedstring.push_back(*here);
+      }
+      if (foundquote) quotedstring.push_back(*here);
+    }
+    here ++;
+  }
+  querystring = formattedstring + quotedstring;
+}
+
+void libinterface::define_query_macros (cgiargsclass &args, 
+					queryparamclass &queryparams,
+					queryresultsclass &queryresults, 
+					ostream &logout) 
+{
+  int numdocs = queryresults.getnumdocs();
+  int numterms = queryresults.getnumterms();
+  disp.setmacro("querysize", "query", args["f"]);
+  disp.setmacro("haveresults", "query", numdocs);
+  
+  // set the display frequency macro
+  text_t freqmsg = "_textfm1_";
+
+  int first = 1;
+  for (int i = 0; i < numterms; i++) {
+    if (first == 0) freqmsg += "; ";
+    first = 0;
+    freqmsg += queryresults.terms[i].termstr + ": " + queryresults.terms[i].termfreq;
+  }
+  if (!quotedstring.empty()) freqmsg += "<br><i>post-processed to find " + quotedstring + "</i>\n";
+  disp.setmacro("freqmsg", "query", freqmsg);
+
+  // set the result line macro
+
+  text_t resline;
+  
+  if (numdocs >= queryparams.maxdocs)
+    resline.setcstr("_textmt2_");
+  
+  if (numdocs == 0) {
+    resline.setcstr("_textndmtq_");
+  } else if (numdocs == 1) {
+    resline += text_t(numdocs) + text_t(" _textdmtq2_.");
+  } else {
+    resline += text_t(numdocs) + text_t(" _textdmtq3_.");
+  }
+
+  disp.setmacro("resultline", "query", resline);
+
+  //  define_collection_macros (args, logout);
+
+  if (queryresults.getnumdocs() > 0) {
+    docLinks(args, queryresults, logout);
+  }
+}
+
+
+// set the _links_ macro to create the links between pages of query results
+void libinterface::docLinks (cgiargsclass &args, 
+			     queryresultsclass &queryresults, 
+			     ostream &logout) 
+{
+  text_t links;
+  int a, b, documents, nextfirst, nextlast, prevfirst, prevlast;
+  int results_from = args.getintarg("r");
+  int hitsperpage = args.getintarg("o");
+  
+  documents = queryresults.getnumdocs();
+  
+  a = results_from;
+  b = a + (hitsperpage - 1);
+  
+  // make sure a and b are in range
+  if (a < 1) a = 1;
+  if (b < 1) b = 1;
+  if (a > documents) a = documents;
+  if (b > documents) b = documents;
+  
+  links.setcstr("<table cellspacing=0 cellpadding=0 border=0 width=\"100%\">\n");
+  links += "<tr valign=bottom>\n";
+  links += "<td align=left>\n";
+  links += "<_font_>\n";
+  
+  // previous page link
+  if (a > 1) {
+    prevlast = a - 1;
+    prevfirst = a - hitsperpage;
+    
+    links += "<p><a href=\"_httpqueryresults_&r="; 
+    links += prevfirst;
+    links += "\">_iconprev__textmatches_ ";
+    links += prevfirst;
+    links += " - ";
+    links += prevlast;
+    links += "</a>\n";
+  }
+  
+  links += "</font>\n";
+  links += "</td><td width=100></td><td align=right>\n";
+  links += "<_font_>\n";
+  
+  // next page link
+  if (b < documents) {
+    nextfirst = b + 1;
+    nextlast = b + hitsperpage;
+    if (nextlast > documents) nextlast = documents;
+    
+    links += "<p><a href=\"_httpqueryresults_&r=";
+    links += nextfirst;
+    links += "\">_textmatches_ ";
+    links += nextfirst;
+    links += " - ";
+    links += nextlast ;
+    links += "_iconnext_</a>\n";
+  }
+  
+  links += "</font>\n";
+  links += "</td></tr></table>\n";
+  
+  disp.setmacro("links", "query", links);
+}
+
+
+// returns LI_NOERROR on success, LI_LOADDATABASEFAILED on failure
+int libinterface::do_query(cgiargsclass &args, queryparamclass &queryparams, 
+			    queryresultsclass &queryresults, ostream &logout) 
+{
+  set_query_params(args, queryparams);
+
+  if (!queryparams.querystring.empty()) {
+    // do the query - the results are returned in queryresults
+    if (!search.search(queryparams, queryresults)) {
+      logout << "ERROR: database didn't load\n";
+	  return LI_LOADDATABASEFAILED;
+    }
+  }
+  return LI_NOERROR;
+}
+
+////////////////////////////////////////////////////////////////////////////////////////
+// query_action is called whenever a search is to be carried out  (i.e. when the 
+// 'a' parameter == 'q') - query calls the mgsearch search() function (via do_query()) to 
+// carry out the search then displays the first page of results.
+//
+// returns LI_NOERROR on success, LI_LOADDATABASEFAILED on failure
+int libinterface::query_action (cgiargsclass &args, ostream &textout, ostream &logout)
+{
+  int err = LI_NOERROR;
+
+  queryparamclass queryparams;
+  queryresultsclass queryresults;
+
+  err = do_query(args, queryparams, queryresults, logout);
+
+  // prepare to print out the page
+  prepare_page(args, logout);
+  define_query_macros(args, queryparams, queryresults, logout);
+
+  // print out the query page
+  textout << text_t2ascii << disp << "_query:header_\n";
+
+  // output query results if there is a query string - 
+  // otherwise output help text
+  if (!queryparams.querystring.empty()) 
+    {
+      displayresults (args, textout, logout, queryresults);
+    } 
+  else 
+    {
+      textout << text_t2ascii << disp << "_query:noqueryheader_\n";
+    }
+  
+  textout << text_t2ascii << disp << "_query:footer_\n";
+
+  return err;
+}
+
+void libinterface::displayresults (cgiargsclass &args, ostream &textout, ostream &logout,
+				   queryresultsclass &queryresults) 
+{
+  textout << text_t2ascii << disp << "_query:queryheader_";
+
+
+  int startresults = args.getintarg("r") - 1;
+  int numresults = args.getintarg("o");
+
+  textout << text_t2ascii << "<table cellspacing=4>\n";
+  for (int i=startresults; i < startresults+numresults; i++)
+    {
+      displaydocsummary (args, textout, logout, queryresults, i);
+    }
+  textout << text_t2ascii << "</table>\n\n";
+
+  textout << text_t2ascii << disp << "_query:queryfooter_";
+}
+
+
+///////////////////////////////////////////////////////////////////////////////////////////////
+// browse_action writes out the browse pages (i.e. the top level hierarchy pages)
+void libinterface::browse_action (cgiargsclass &args, ostream &textout, ostream &logout)
+{
+  text_t browse_bar, locator, output;
+  gdbm_info info;
+  
+  prepare_page(args, logout);
+  
+  // get browse bar unless page has been detached 
+  if (args.getintarg("x") == 0) {
+    browse->get_browse_bar(args["d"], browse_bar);
+  }
+  
+  // get top locator
+  browse->get_top_locator(args, gdbm, 0, locator);	
+
+  // expand and output page 
+  // Note: we need to expand these out using package 'browse'
+  // so we can't use the tricky '<<' syntax
+  disp.expandstring("browse", "_header_", output);
+  textout << text_t2ascii << output;
+  disp.expandstring("browse", browse_bar, output);
+  textout << text_t2ascii << output;
+  disp.expandstring("browse", locator, output);
+  textout << text_t2ascii << output;
+  disp.expandstring("browse", "_footer_", output); 
+  textout << text_t2ascii << output;
+}
+
+
+
+/////////////////////////////////////////////////////////////////////////////////////////
+// document_action is called to retrieve and display collection documents
+// It calls the mgsearch function docTargetDocument() to retrieve 
+// a document. 
+
+void libinterface::document_action (cgiargsclass &args, ostream &textout, ostream &logout) {
+  
+  text_t locator, content, links, output;
+  gdbm_info info;
+  queryparamclass queryparams;
+  queryresultsclass queryresults;
+  int oversize = 0;
+
+  // have to redo the query to get queryterms for highlight text
+  do_query(args, queryparams, queryresults, logout);
+
+  prepare_page(args, logout);
+  
+  if (args["g"][1] == '0') {
+    // get docnum from gdbm
+    text_t docref;
+    if (args["d"][0] != 'B') get_book(args["d"], docref);
+    else docref = args["d"];
+    if (gdbm.getinfo(docref, queryparams.collection, info) != 0) {
+      logout << text_t2ascii << "info_db wasn't opened - " << docref << "\n";
+      return;
+    }
+    
+    // get document text if there is any
+    if (info.c.empty()) {
+        search.docTargetDocument(default_index, queryparams.collection,
+				 info.d, content);
+	if (info.t != "<i>(introductory text)</i>")
+	  content = "<h3>" + info.t + "</h3>\n" + content;
+    }
+  } 
+
+  if (args["g"][1] == '1') {
+    // want to get expanded out text
+    vector<text_t> contents_arr;
+    text_t booksection;
+    int levelcount;
+
+    get_book (args["d"], booksection);
+    levelcount = count_dots(booksection);
+
+    browse->get_contents_arr(args, gdbm, contents_arr);
+    
+
+    // get text for each section of book
+    vector<text_t>::const_iterator thiscontent = contents_arr.begin();
+    vector<text_t>::const_iterator end = contents_arr.end();
+
+    int first = 1;
+    int count = 1;
+    while (thiscontent != end) {
+      text_t text;
+
+      // get docnum from gdbm
+      if (gdbm.getinfo(*thiscontent, queryparams.collection, info) != 0) { 
+	logout << text_t2ascii << "info_db wasn't opened - " << args["d"] << "\n";
+	return;
+      }
+
+      // if section has text get it, otherwise output section title
+      if (info.c.empty()) {
+
+	// output <a name= > tags for all text sections currently displayed in toc (all text sections
+	// if contents are expanded
+	if (count_dots(*thiscontent) == levelcount || args["g"][0] == '1') {
+	  content += "<a name=\"";
+	  content += count;
+	  content += "\"></a>\n";
+	  count ++;
+	}
+
+	search.docTargetDocument(default_index, queryparams.collection,
+				 info.d, text);					
+	if (info.t != "<i>(introductory text)</i>") 
+	  content += "<h3>" + info.t + "</h3>\n";
+	//	content += text + "<hr><br>\n";
+	content += text + "<p>\n"; // no longer want <hr> between sections
+      } else {
+	content += "<h3>" + info.t + "</h3>\n";
+      }
+
+      if (args["n"] == 1) {
+	if (first) {
+	  browse->get_top_locator(args, gdbm, 0, locator);
+	  disp.expandstring("text", "_header_", output);
+	  textout << text_t2ascii << output;
+	  disp.expandstring("text", locator, output);
+	  textout << text_t2ascii << output;
+	}
+	
+	disp.expandstring("text", content, output);
+	if (!queryparams.querystring.empty())
+	  highlighttext(queryresults.termvariants, textout, logout, output);
+	else
+	  textout << text_t2ascii << output;
+   	first = 0;
+	content.clear();
+      }
+
+      thiscontent ++;
+      if (content.size() > 200000 && args["n"] != 1) {
+	content.clear();
+	oversize = 1;
+	args["g"][1] = '0';
+	break;
+      }
+    }
+
+    if (args["g"][1] == '0') {
+      // get docnum from gdbm
+      text_t docref;
+      if (args["d"][0] != 'B') get_book(args["d"], docref);
+      else docref = args["d"];
+      if (gdbm.getinfo(docref, queryparams.collection, info) != 0) {
+        logout << text_t2ascii << "info_db wasn't opened - " << docref << "\n";
+        return;
+      }
+    
+      // get document text if there is any
+      if (info.c.empty()) {
+        search.docTargetDocument(default_index, queryparams.collection,
+                                 info.d, content);
+	if (info.t != "<i>(introductory text)</i>")
+	  content = "<h3>" + info.t + "</h3>\n" + content;
+      }
+    }
+  }
+
+  if (args["n"] != 1) {
+    // get top locator
+    browse->get_top_locator(args, gdbm, oversize, locator);
+
+    // expand and output page 
+    // Note: we need to expand these out using package 'text'
+    // so we can't use the tricky '<<' syntax
+    disp.expandstring("text", "_header_", output);
+    textout << text_t2ascii << output;
+
+    disp.expandstring("text", locator, output);
+    textout << text_t2ascii << output;
+
+    disp.expandstring("text", content, output);
+    if (!queryparams.querystring.empty())
+      highlighttext(queryresults.termvariants, textout, logout, output);
+    else
+      textout << text_t2ascii << output;
+  }
+
+  // get links to next and previous sections unless in expand text mode
+  if (args["g"][1] == '0') {
+    browse->get_links(args, gdbm, links);
+    disp.expandstring("text", links, output);
+    textout << text_t2ascii << output;
+  }
+
+  disp.expandstring("text", "_footer_", output); 
+  textout << text_t2ascii << output;
+}
+
+
+/////////////////////////////////////////////////////////////////////////////////////////
+// page is called when a standard html page is to be displayed
+void libinterface::page_action (cgiargsclass &args, ostream &textout, ostream &logout)
+{
+
+  text_t &arg_p = args["p"];
+
+  prepare_page(args, logout);
+
+  if (arg_p == "preferences")
+    define_pref_macros(args, logout);
+
+  textout << text_t2ascii << disp << ("_" + arg_p + ":header_\n")
+	  << ("_" + arg_p + ":imagestandardbar_\n") << ("_" + arg_p + ":content_\n")
+	  << ("_" + arg_p + ":footer_\n");
+}
+
+// highlighttext highlights query terms in text string and outputs the resulting text string
+void libinterface::highlighttext(vector<text_t> &termvars, ostream &textout, 
+				 ostream &logout, text_t &text) 
+{
+  map<text_t, int, lttext_t> terms;
+  map<text_t, int, lttext_t>::const_iterator it;
+  for (unsigned int i = 0; i < termvars.size(); i++) {
+    terms[termvars[i]] = 1;
+  }
+
+  text_t::iterator here = text.begin(); 
+  text_t::iterator end = text.end();
+  text_t word, buffer;
+  while (here != end) {
+    if (((*here >= 65) && (*here <= 90)) ||
+	((*here >= 97) && (*here <= 122)) ||
+	((*here >= '0') && (*here <= '9')) ||
+	((*here >= 192) && (*here <= 214)) ||
+	((*here >= 216) && (*here <= 246)) ||
+	((*here >= 248) && (*here <= 255))) {
+      // not word boundary
+      word.push_back(*here);
+      here++;
+
+    } else {
+      // found word boundary
+      // add last word if there was one
+      if (!word.empty()) {
+	it = terms.find(word);
+	if (it != terms.end()) {
+	  word = "<b><u>" + word + "</u></b>";
+	}
+	buffer += word;
+        word.clear();
+      }
+
+      if (*here == '<') {
+        // skip over rest of html tag
+	while ((here != end) && (*here != '>')) {
+	  buffer.push_back(*here);
+	  here++;
+	}
+      }
+
+      buffer.push_back(*here);
+      here++;
+
+      if (buffer.size() > 1024) {
+	textout << text_t2ascii << buffer;
+	buffer.clear();
+      }
+    }
+  }
+  textout << text_t2ascii << buffer;
+}
+
+void libinterface::define_pref_macros (cgiargsclass &args, ostream &logout)
+{
+  // the caseoption macro
+  text_t caseoption;
+  int arg_k = args.getintarg("k");
+
+  caseoption += "\n<input type=radio name=k value=1";
+  if (arg_k) caseoption += " checked";
+  caseoption += "> ignore case differences<br>\n";
+  caseoption += "<input type=radio name=k value=0";
+  if (!arg_k) caseoption += " checked";
+  caseoption += "> upper/lower case must match\n";
+
+  disp.setmacro ("caseoption", "preferences", caseoption);
+
+  // the stemoption macro
+  text_t stemoption;
+  int arg_s = args.getintarg("s");
+
+  stemoption += "\n<input type=radio name=s value=1";
+  if (arg_s) stemoption += " checked";
+  stemoption += "> ignore word endings<br>\n";
+  stemoption += "<input type=radio name=s value=0";
+  if (!arg_s) stemoption += " checked";
+  stemoption += "> whole word must match\n";
+
+  disp.setmacro ("stemoption", "preferences", stemoption);
+
+
+  // the maxdocoption
+  text_t maxdocoption;
+  int arg_m = args.getintarg("m");
+
+  maxdocoption += "\n<select name=m>\n";
+  maxdocoption += "  <option value=\"50\"";
+  if (arg_m < 100) maxdocoption += " selected";
+  maxdocoption += ">50\n";
+  maxdocoption += "  <option value=\"100\"";
+  if (arg_m >= 100 && arg_m < 200) maxdocoption += " selected";
+  maxdocoption += ">100\n";
+  maxdocoption += "  <option value=\"200\"";
+  if (arg_m >= 200 && arg_m < 500) maxdocoption += " selected";
+  maxdocoption += ">200\n";
+  maxdocoption += "  <option value=\"500\"";
+  if (arg_m >= 500) maxdocoption += " selected";
+  maxdocoption += ">500\n";
+  maxdocoption += "</select>\n";
+
+  disp.setmacro ("maxdocoption", "preferences", maxdocoption);
+
+  // the hitsperpageoption
+  text_t hitsoption;
+  int arg_o = args.getintarg("o");
+
+  hitsoption += "\n<select name=o>\n";
+  hitsoption += "  <option value=\"10\"";
+  if (arg_o < 20) hitsoption += " selected";
+  hitsoption += ">10\n";
+  hitsoption += "  <option value=\"20\"";
+  if (arg_o >= 20 && arg_o < 50) hitsoption += " selected";
+  hitsoption += ">20\n";
+  hitsoption += "  <option value=\"50\"";
+  if (arg_o >= 50 && arg_o < 100) hitsoption += " selected";
+  hitsoption += ">50\n";
+  hitsoption += "  <option value=\"100\"";
+  if (arg_o >= 100 && arg_o < 500) hitsoption += " selected";
+  hitsoption += ">100\n";
+  hitsoption += "  <option value=\"500\"";
+  if (arg_o >= 500) hitsoption += " selected";
+  hitsoption += ">all\n";
+  hitsoption += "  </select>\n";
+
+  disp.setmacro ("hitsperpageoption", "preferences", hitsoption);
+}
Index: /trunk/gsdl/src/library/libinterface.h
===================================================================
--- /trunk/gsdl/src/library/libinterface.h	(revision 4)
+++ /trunk/gsdl/src/library/libinterface.h	(revision 4)
@@ -0,0 +1,146 @@
+#ifndef LIBINTERFACE_H
+#define LIBINTERFACE_H
+
+#include "browse.h"
+#include "queryinfo.h"
+#include "mgsearch.h"
+#include "display.h"
+#include "gdbmclass.h"
+
+#ifdef __GNUG__
+#  include <iostream.h>
+#  include <map>
+#else
+#  ifndef USE_OBJECTSPACE
+#    include <iostream>
+#  else
+#    include <ospace\std\iostream>
+#    include <ospace\std\map>
+#  endif
+#endif
+
+#include "cgiargs.h"
+
+// This is the precedence given to various parameters 
+// by the display class - note that at present language
+// is excluded but will be required for multi-language
+// collections
+#define MACROPRECEDENCE "style,collection,queryversion,version"
+
+
+// errors related to this module
+#define LI_NOERROR            0
+#define LI_LOADDATABASEFAILED 1
+
+
+
+class libinterface
+{
+public:
+  libinterface ();
+  virtual ~libinterface() {}
+
+  // setgsdlhome will set the macros, gdbm, index, and text homes
+  void setgsdlhome (const text_t &thegsdlhome);
+
+  void setmacroshome (const text_t &themacroshome);
+  void setgdbmhome (const text_t &thegdbmhome);
+  void setindexhome (const text_t &theindexhome);
+
+  void sethttpprefix (const text_t &thehttpprefix);
+  void setgwcgi (const text_t &thegwcgi);
+
+  // init should be called after the various homes are set,
+  // it returns 'false' on failure and 'true' on success
+  bool init ();
+  
+  // examine the cgi arguments and create the appropriate page, 
+  // outputing the page to textout and any debug information to logout
+  //
+  // returns LI_NOERROR on success, LI_LOADDATABASEFAILED on failure
+  virtual int getpage (const text_t &argstr, ostream &textout, ostream &logout);
+  
+  // returns the compressed arguments for the last getpage
+  const text_t &getlastcomparg() {return lastcomparg;}
+
+protected:
+  text_t gsdlhome;
+  text_t macroshome;
+  text_t gdbmhome;
+  text_t indexhome;
+
+  text_t books;
+  text_t hrefs;
+  text_t titles;
+  vector<text_t> bookarr;
+  vector<text_t> hrefarr;
+  vector<text_t> titlearr;
+  int numbooks;
+
+  text_t default_index;
+
+  text_t httpprefix;
+  text_t gwcgi;
+  text_t quotedstring;
+
+  text_t lastcomparg;
+
+  gdbmclass gdbm;
+  outconvertclass text_t2ascii;
+  displayclass disp;
+
+
+  browseclass *browse;
+  mgsearchclass search;
+  
+  virtual void get_arg_config (text_t &argconfigstr);
+  text_t::iterator get_next_config_arg (text_t::iterator first, text_t::iterator last,
+					text_t &argname, text_t &defaultvalue, bool &longarg);
+
+  virtual text_t get_compressed_args (cgiargsclass &args);
+  virtual void expand_compressed_args (cgiargsclass &args);
+  virtual void add_default_args (cgiargsclass &args);
+  virtual void check_args (cgiargsclass &args);
+
+  virtual void define_general_macros (cgiargsclass &args, ostream &logout);
+  virtual void prepare_page (cgiargsclass &args, ostream &logout);
+
+
+  virtual void set_query_params (cgiargsclass &args, queryparamclass &queryparams);
+  virtual void define_query_macros (cgiargsclass &args, 
+				    queryparamclass &queryparams,
+				    queryresultsclass &queryresults, 
+				    ostream &logout);
+  virtual void docLinks (cgiargsclass &args, queryresultsclass &queryresults, 
+			 ostream &logout);
+
+  virtual int query_action (cgiargsclass &args, ostream &textout, ostream &logout);
+  virtual void browse_action (cgiargsclass &args, ostream &textout, ostream &logout);
+  virtual void document_action (cgiargsclass &args, ostream &textout, ostream &logout);
+  virtual void page_action (cgiargsclass &args, ostream &textout, ostream &logout);
+  
+  virtual void displayresults (cgiargsclass &args, ostream &textout, ostream &logout,
+			       queryresultsclass &queryresults);
+
+  virtual void define_pref_macros (cgiargsclass &args, ostream &logout);
+  virtual int do_query(cgiargsclass &args, queryparamclass &queryparams, 
+			queryresultsclass &queryresults, ostream &logout);
+  virtual void highlighttext(vector<text_t> &termvars, ostream &textout, 
+			     ostream &logout, text_t &text);
+  void libinterface::format_querystring (text_t &querystring);
+
+
+
+  virtual void set_default_index(void) {}
+  virtual text_t get_collection_name(void) {return "";}
+  virtual void assemble_index (cgiargsclass &args, text_t &index) {}
+  virtual void define_collection_macros (cgiargsclass &args, ostream &logout) {}
+  virtual bool collection_init(const text_t &collection) {return true;}
+  virtual void displaydocsummary (cgiargsclass &args, ostream &textout,
+                                  ostream &logout, queryresultsclass &queryresults,
+								  int resultnum) {}
+
+
+};
+
+#endif
Index: /trunk/gsdl/src/library/locateinfo.cpp
===================================================================
--- /trunk/gsdl/src/library/locateinfo.cpp	(revision 4)
+++ /trunk/gsdl/src/library/locateinfo.cpp	(revision 4)
@@ -0,0 +1,60 @@
+#include "locateinfo.h"
+
+void getindexsuffix (const text_t &index,
+		     const text_t &collection,
+		     text_t &ttidxsuffix)
+{
+  ttidxsuffix = "collect";
+
+#ifdef __WIN32__
+  ttidxsuffix += "\\";
+#else
+  ttidxsuffix += "/";
+#endif
+
+  ttidxsuffix += collection;
+
+#ifdef __WIN32__
+  ttidxsuffix += "\\index\\";
+#else
+  ttidxsuffix += "/index/";
+#endif
+
+//           ttidxsuffix += collection; //this changed with new build software
+  ttidxsuffix += index;
+#ifdef __WIN32__
+  ttidxsuffix += "\\";
+#else
+  ttidxsuffix += "/";
+#endif
+  ttidxsuffix += collection;
+}
+
+void gettextsuffix (const text_t &collection, 
+		    text_t &tttxtsuffix)
+{
+  tttxtsuffix = "collect";
+
+#ifdef __WIN32__
+  tttxtsuffix += "\\";
+#else
+  tttxtsuffix += "/";
+#endif
+
+  tttxtsuffix += collection;
+
+#ifdef __WIN32__
+  tttxtsuffix += "\\index\\";
+#else
+  tttxtsuffix += "/index/";
+#endif
+
+//                tttxtsuffix += collection; // path changed with new building software - Stefan.
+  tttxtsuffix += "text";
+#ifdef __WIN32__
+  tttxtsuffix += "\\";
+#else
+  tttxtsuffix += "/";
+#endif
+  tttxtsuffix += collection;
+}
Index: /trunk/gsdl/src/library/locateinfo.h
===================================================================
--- /trunk/gsdl/src/library/locateinfo.h	(revision 4)
+++ /trunk/gsdl/src/library/locateinfo.h	(revision 4)
@@ -0,0 +1,13 @@
+#ifndef LOCATEINFO_H
+#define LOCATEINFO_H
+
+#include "text_t.h"
+
+void getindexsuffix (const text_t &index,
+		     const text_t &collection,
+		     text_t &ttidxsuffix);
+
+void gettextsuffix (const text_t &collection, 
+		    text_t &tttxtsuffix);
+
+#endif
Index: /trunk/gsdl/src/library/mgq.c
===================================================================
--- /trunk/gsdl/src/library/mgq.c	(revision 4)
+++ /trunk/gsdl/src/library/mgq.c	(revision 4)
@@ -0,0 +1,347 @@
+/* cut-down version of mgquery */
+
+
+#include <stdio.h>
+#include <string.h>
+/* #include <io.h> */
+#include <fcntl.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+  
+#include "sysfuncs.h"
+  
+#include "messages.h"
+#include "memlib.h"
+  
+#include "invf.h"
+#include "text.h"
+#include "mg.h"
+#include "lists.h"
+#include "backend.h"
+#include "environment.h"
+#include "globals.h"
+#include "mg_errors.h"
+#include "commands.h"
+#include "text_get.h"
+#include "term_lists.h"
+#include "local_strings.h"
+  
+#ifdef __cplusplus
+}
+#endif
+
+
+#include "mgq.h"
+
+/* globals needed by some vague part of mg... */
+FILE *OutFile = NULL, *InFile = NULL;
+int OutPipe = 0, InPipe = 0;
+int Quitting = 0;
+
+static query_data *qd;
+char db_loaded_name[256] = "";
+
+#if defined(PARADOCNUM) ||  defined(NZDL)
+int *Paragraph = NULL;
+int Documents = 0;
+
+int GetDocNumFromParaNum(int paranum)
+{
+  int low = 1, high = Documents;
+  int mid = (low+high)/2;
+
+  while ((mid = (low+high)/2) >=1 && mid <= Documents)
+    {
+      if (paranum > Paragraph[mid])
+        low = mid+1;
+      else if (paranum <= Paragraph[mid-1])
+        high = mid-1;
+      else
+        return mid;
+    }
+  FatalError(1, "Bad paragraph number.\n");
+  return 0;
+}
+#endif
+
+
+/*****************************************************************************/
+
+static void MGQError(char *emsg)
+{
+  fprintf(stderr,"Fatal error: %s\n", emsg);
+  exit(1);
+}
+
+static int ProcessDocs (query_data * qd, int num, int skip, int howmany, int verbatim,
+			enum result_kinds kind, FILE * Output,
+			int (*sender)(char *,int,int,float,void *), void *ptr)
+{
+  int max_buf = 0, output_failure = 0;
+  int DocCount = 0;
+  int need_text = (kind == result_docs);
+  
+  for (;;) {
+    if (skip <= 0) break;
+    skip--;
+    if (!NextDoc(qd)) return 0;
+  }
+  
+  if (need_text)
+    {
+      max_buf = atoi (GetDefEnv ("buffer", "1048576"));
+    }
+  
+  do
+    {
+      u_char *UDoc = NULL;
+      unsigned long ULen;
+      
+      if (need_text)
+	{
+	  /* load the compressed text */
+	  if (LoadCompressedText (qd, max_buf))
+	    {
+	      MGQError("Unable to load compressed text(memory?).");
+	    }
+	  /* uncompress the loaded text */
+	  UDoc = GetDocText (qd, &ULen);
+	  if (UDoc == NULL)
+	    MGQError("UDoc is unexpectedly NULL");
+	}
+      
+      if (UDoc != NULL  ||  kind == result_docnums)
+	{
+	  int docnum = GetDocNum(qd);
+#if defined(PARADOCNUM) ||  defined(NZDL)
+	  if (qd->id->ifh.InvfLevel == 3) docnum = GetDocNumFromParaNum(docnum);
+#endif
+	  switch (kind)
+	    {
+	    case result_docnums:
+	      if (sender != NULL)
+		output_failure = (*sender)("",0,docnum,GetDocWeight(qd),ptr);
+	      break;
+	    case result_docs:
+	      if (sender != NULL)
+		output_failure = (*sender)(UDoc,ULen,docnum,GetDocWeight(qd),ptr);
+	      break;
+	    default:
+	      break;
+	    }
+	}
+      DocCount++;
+      
+    }
+  while (NextDoc (qd) && output_failure == 0 && --howmany > 0);
+  
+  if (need_text)
+    {
+      FreeTextBuffer (qd);
+    }
+  
+  return (DocCount);
+}
+
+
+void
+send_query_term_freqs(QueryTermList *qtl,
+		      int (*sender)(char *,int,int,float,void *), void *ptr)
+{
+  int i;  char *word;
+  for (i = 0; i < qtl->num; i++)
+    if (sender != NULL) {
+      word = word2str(qtl->QTE[i].Term);
+      (* sender)(word, strlen(word), qtl->QTE[i].Count, (float)0.0, ptr);
+    }
+}
+
+
+void
+send_terms (TermList * qtl, int (*sender)(char *,int,int,float,void *), void *ptr)
+{
+  int i = 0;
+  char *word;
+
+  if (sender == NULL) return;
+  for (i = 0; i < qtl->num; i++)
+    {
+      word = word2str(qtl->TE[i].Word);
+      (* sender)(word, strlen(word), qtl->TE[i].Count, (float)0.0, ptr);
+    }
+}
+
+
+/* MoreDocs () */
+/* Displays all documents in list  DocList. */
+/* Documents are fetched, then decompressed and displayed according to the */
+/* format implied in  FormString(). */
+
+static void
+MoreDocs (query_data * qd, enum result_kinds kind, 
+	  int skip, int howmany,
+	  int (*sender)(char *,int,int,float,void *), void *ptr)
+{
+  qd->num_of_ans = qd->DL->num;
+  switch (kind) {
+  case result_docs:
+  case result_docnums:
+    if (qd->num_of_ans > 0)
+      ProcessDocs (qd, qd->DL->num, skip, howmany,
+		   BooleanEnv (GetEnv ("verbatim"), 1),
+		   kind, NULL, sender, ptr);
+    break;
+  case result_termfreqs:
+    send_query_term_freqs(qd->QTL, sender, ptr);
+    break;
+  case result_terms:
+    send_terms(qd->TL, sender, ptr);
+    break;
+  }
+}
+
+
+int mgq_init(char *mgdir, char *gensuffix, char *textsuffix)
+{
+  /* Initialise the environment with default values */
+  InitEnv ();
+  
+  SetEnv("mgdir",mgdir,NULL);
+  SetEnv("mgname",gensuffix,NULL);
+  SetEnv("textname",textsuffix,NULL);
+  
+  PushEnv ();
+  
+  qd = InitQuerySystem (GetDefEnv ("mgdir", ""),
+			GetDefEnv ("mgname", ""),
+			GetDefEnv ("textname", NULL),
+			NULL);
+  
+  if (qd == NULL) return 0;  else return 1;
+}
+
+int mgq_ask(char *line)
+{
+  char QueryType;  char OutputType;  char *LinePtr;
+  
+  ResetFileStats (qd);
+  qd->max_mem_in_use = qd->mem_in_use = 0;
+  qd->tot_hops_taken += qd->hops_taken;
+  qd->tot_num_of_ptrs += qd->num_of_ptrs;
+  qd->tot_num_of_accum += qd->num_of_accum;
+  qd->tot_num_of_terms += qd->num_of_terms;
+  qd->tot_num_of_ans += qd->num_of_ans;
+  qd->tot_text_idx_lookups += qd->text_idx_lookups;
+  qd->hops_taken = qd->num_of_ptrs = 0;
+  qd->num_of_accum = qd->num_of_ans = qd->num_of_terms = 0;
+  qd->text_idx_lookups = 0;
+  
+  LinePtr = ProcessCommands (line, qd);
+  if (CommandsErrorStr) {
+    fprintf (stderr, "%s\n", CommandsErrorStr);
+    return 0;
+  }
+  if (*LinePtr == '\0') return 1;
+  
+  FreeQueryDocs (qd);
+  
+  QueryType = get_query_type ();
+  OutputType = get_output_type ();
+  /* No point in hiliting words on a docnum query */
+  if (OutputType == OUTPUT_HILITE && QueryType == QUERY_DOCNUMS)
+    OutputType = OUTPUT_TEXT;
+  
+  switch (QueryType)
+    {
+    case QUERY_BOOLEAN:
+      {
+	char *maxdocs;
+	BooleanQueryInfo bqi;
+	maxdocs = GetDefEnv ("maxdocs", "all");
+	bqi.MaxDocsToRetrieve = strcmp (maxdocs, "all") ? atoi (maxdocs) : -1;
+        if (qd->sd->sdh.indexed)
+	  BooleanQuery (qd, line, &bqi, (BooleanEnv (GetEnv ("casefold"), 0) |
+					 (BooleanEnv (GetEnv ("stem"), 0) << 1)));
+	else
+	  BooleanQuery (qd, line, &bqi, qd->sd->sdh.stem_method);
+	/*	if (qd->sd->sdh.indexed) BooleanQuery (qd, line, &bqi, 3);
+	else BooleanQuery (qd, line, &bqi, qd->sd->sdh.stem_method); */
+	break;
+      }
+    case QUERY_APPROX:
+    case QUERY_RANKED:
+      {
+	char *maxdocs;
+	char *maxterms;
+	char *maxaccum;
+	RankedQueryInfo rqi;
+	maxdocs = GetDefEnv ("maxdocs", "all");
+	maxterms = GetDefEnv ("max_terms", "all");
+	maxaccum = GetDefEnv ("max_accumulators", "all");
+	rqi.Sort = BooleanEnv (GetEnv ("sorted_terms"), 0);
+	rqi.QueryFreqs = BooleanEnv (GetEnv ("qfreq"), 1);
+	rqi.Exact = QueryType == QUERY_RANKED;
+	rqi.MaxDocsToRetrieve = strcmp (maxdocs, "all") ? atoi (maxdocs) : -1;
+	rqi.MaxTerms = strcmp (maxterms, "all") ? atoi (maxterms) : -1;
+	rqi.MaxParasToRetrieve = rqi.MaxDocsToRetrieve;
+	if (qd->id->ifh.InvfLevel == 3 && GetEnv ("maxparas"))
+	  rqi.MaxParasToRetrieve = atoi (GetEnv ("maxparas"));
+	rqi.AccumMethod = toupper (*GetDefEnv ("accumulator_method", "A"));
+	rqi.MaxAccums = strcmp (maxaccum, "all") ? atoi (maxaccum) : -1;
+	rqi.HashTblSize = IntEnv (GetEnv ("hash_tbl_size"), 1000);
+	rqi.StopAtMaxAccum = BooleanEnv (GetEnv ("stop_at_max_accum"), 0);
+	rqi.skip_dump = GetEnv ("skip_dump");
+	RankedQuery (qd, line, &rqi);
+	break;
+      }
+    case QUERY_DOCNUMS:
+      {
+	DocnumsQuery (qd, line);
+	break;
+      }
+    }
+
+  return 1;
+}
+
+int mgq_numdocs(void)
+{
+  if (qd->DL) return qd->DL->num;
+  else return 0;
+}
+
+int mgq_numterms(void)
+{
+  if (qd->QTL) return qd->QTL->num;
+  else return 0;
+}
+
+int mgq_results(enum result_kinds kind,int skip,int howmany, int (*sender)(char *, int, int, float, void *), void *ptr)
+{
+  if (qd->DL) {
+    qd->doc_pos = 0;
+    MoreDocs(qd, kind, skip, howmany, sender, ptr);
+  }
+  return 0;
+}
+
+
+void mgq_finish(void)
+{
+  FinishQuerySystem(qd);
+  UninitEnv ();
+}
+
+int load_database(char *mgdir, char *gensuffix, char *textsuffix)
+{
+  if ((db_loaded_name[0]) != '\0' && (strcmp(gensuffix,db_loaded_name) != 0)) {
+    mgq_finish();
+    db_loaded_name[0] = '\0';
+  }
+  if ((db_loaded_name[0] == '\0') && mgq_init(mgdir, gensuffix, textsuffix)) {
+    strcpy(db_loaded_name, gensuffix);
+  }
+  return (db_loaded_name[0] != '\0');
+}
Index: /trunk/gsdl/src/library/mgq.h
===================================================================
--- /trunk/gsdl/src/library/mgq.h	(revision 4)
+++ /trunk/gsdl/src/library/mgq.h	(revision 4)
@@ -0,0 +1,25 @@
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+enum result_kinds {
+	result_docs,      /* Return the documents found in last search */
+	result_docnums,   /* Return document id numbers and weights */
+	result_termfreqs, /* Return terms and frequencies */
+	result_terms      /* Return matching query terms */
+    };
+
+int mgq_ask(char *line);
+int mgq_results(enum result_kinds kind,int skip, int howmany, int (*sender)(char *, int, int, float, void *), void *ptr);
+int mgq_numdocs(void);
+int mgq_numterms(void);
+int load_database(char *mgdir, char *gensuffix, char *textsuffix);
+
+/* this will equal "" when a database is not loaded */
+/* if a database is loaded it will equal its gensuffix */
+extern char db_loaded_name[256];
+
+
+#ifdef __cplusplus
+}
+#endif
Index: /trunk/gsdl/src/library/mgsearch.cpp
===================================================================
--- /trunk/gsdl/src/library/mgsearch.cpp	(revision 4)
+++ /trunk/gsdl/src/library/mgsearch.cpp	(revision 4)
@@ -0,0 +1,467 @@
+#include <string.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <ctype.h>
+
+#ifdef __GNUG__
+#  include <iostream.h>
+#  include <gdbm.h>
+
+#else
+#  ifndef USE_OBJECTSPACE
+#    include <iostream>
+#  else
+#    include <ospace\std\iostream>
+#  endif
+
+// gdbm stuff
+#  include "autoconf.h"
+#  include "systems.h"
+#  include "gdbmconst.h"
+#  include "gdbm.h"
+#endif
+  
+#include <assert.h>
+
+#include "mgq.h"
+#include "mgsearch.h"
+#include "locateinfo.h"
+
+/////////////
+// globals //
+/////////////
+
+static char *quotedquery = NULL;
+
+
+
+////////////////////////
+// callback functions //
+////////////////////////
+
+// This routine is called for each document found in a search
+// it assumes that cache_num is set up correctly to point to
+// a suitable result cache
+int ourquerycallback(char *UDoc, int ULen, int DocNum, 
+		     float Weight, void *info) {
+
+  
+  queryresultsclass *queryresults = (queryresultsclass * )info;
+
+  // check the returned document for the presence of the
+  // quoted part of the query, if there was one
+ 
+  if (UDoc != NULL && quotedquery != NULL &&
+      quotedquery[0] != '\0' && strstr (UDoc, quotedquery) == NULL) return 0;
+  
+  // append this entry to the document results
+  docresultclass docresult;
+  docresult.docnum = DocNum;
+  docresult.docweight = Weight;
+
+  queryresults->docs.push_back(docresult);
+  
+  return 0;
+}
+
+// This callback is called once for each term in the query
+int termfreqcallback(char *Word, int ULen,  int Freq, 
+		     float Weight,  void *info) {
+  queryresultsclass *queryresults = (queryresultsclass *)info;
+
+  termfreqclass termfreq;
+  termfreq.termstr.setcarr(Word, ULen);
+  termfreq.termfreq = Freq;
+  queryresults->terms.push_back(termfreq);
+  
+  return 0;
+}
+
+// this callback is called once for each variation of each term
+int termscallback(char *Word, int ULen, int Freq,
+		  float Weight, void *info) {
+
+  queryresultsclass *queryresults = (queryresultsclass *)info;
+  queryresults->termvariants.push_back(Word);
+
+  return 0;
+}
+
+// This callback is for getting document text
+int doctextcallback(char *Word, int ULen,  int Freq, 
+		    float Weight,  void *info) {
+  text_t *output = (text_t *)info;
+  if (output == NULL) return 0;
+
+  output->setcarr(Word, ULen);
+  
+  // replace all control-Cs with spaces
+  text_t::iterator here = output->begin();
+  text_t::iterator end = output->end();
+  while (here != end)
+    {
+      if (*here == '\x3') *here = ' ';
+      here++;
+    }
+  
+  return 0;
+}
+
+
+
+////////////////////
+// mgsearch class //
+////////////////////
+
+mgsearchclass::mgsearchclass ()
+{
+  cache = new querycache (RESULTCACHESIZE);
+}
+
+mgsearchclass::~mgsearchclass () 
+{
+  if (cache != NULL) 
+    {
+      delete cache;
+      cache = NULL;
+    }
+}
+
+
+void mgsearchclass::setindexhome (const text_t &theindexhome)
+{
+  indexhome = theindexhome;
+}
+
+
+bool mgsearchclass::search(const queryparamclass &queryparams, 
+		      queryresultsclass &queryresults)
+{
+  bool databaseloaded = true;
+
+  assert (cache != NULL);
+
+  queryresults.clear();
+
+  // first check the cache
+  if (cache->find(queryparams, queryresults))
+    {
+      return true;
+    }
+
+  // make sure there is a query to be processed
+  text_t::const_iterator queryhere = queryparams.querystring.begin();
+  text_t::const_iterator queryend = queryparams.querystring.end();
+  while (queryhere != queryend) {
+    if (((*queryhere >= 65) && (*queryhere <= 90)) ||
+      ((*queryhere >= 97) && (*queryhere <= 122)) ||
+      ((*queryhere >= 192) && (*queryhere <= 214)) ||
+      ((*queryhere >= 216) && (*queryhere <= 246)) ||
+      ((*queryhere >= 248) && (*queryhere <= 255)) ||
+      ((*queryhere >= '0') && (*queryhere <= '9'))) break;
+    queryhere++;
+  }
+
+  // if we reached the end of the query string without finding
+  // any alphanumeric characters then return no results (and say
+  // the database was loaded)
+  if (queryhere == queryend) return true;
+
+
+
+  // get the names of the index and text suffixes
+  text_t ttidxsuffix, tttxtsuffix;
+  getindexsuffix (queryparams.search_index, 
+		  queryparams.collection, ttidxsuffix);
+  gettextsuffix (queryparams.collection, tttxtsuffix);
+  char *idxsuffix = ttidxsuffix.getcstr(); assert (idxsuffix != NULL);
+  char *txtsuffix = tttxtsuffix.getcstr(); assert (txtsuffix != NULL);
+
+#ifdef __WIN32__
+  char *cindexhome = (indexhome+"\\").getcstr(); assert (cindexhome != NULL);
+#else
+  char *cindexhome = indexhome.getcstr(); assert (cindexhome != NULL);
+#endif
+
+  if (load_database(cindexhome, idxsuffix, txtsuffix))
+    {
+      setsearchmode (queryparams);
+      submitquery (queryparams);
+      getresults (queryresults);
+    }
+  else databaseloaded = false;
+
+  // free up the c strings
+  delete idxsuffix;
+  delete txtsuffix;
+  delete cindexhome;
+
+  return databaseloaded;
+}
+
+
+void mgsearchclass::setsearchmode (const queryparamclass &queryparams)
+{
+  mgq_ask(".set expert true");
+  mgq_ask(".set accumulator_method list");
+  mgq_ask(".set max_accumulators 50000");
+  mgq_ask(".set verbatim true");
+  mgq_ask(".unset skip_dump");
+  mgq_ask(".set mode docnums");
+
+  switch (queryparams.search_type) 
+    {
+    case 0: mgq_ask(".set query boolean");  break;
+    case 1:  mgq_ask(".set query ranked"); break;
+    }
+  switch (queryparams.casefolding) 
+    {
+    case 1: mgq_ask(".set casefold on");  break;
+    case 0: mgq_ask(".set casefold off"); break;
+    }
+  switch (queryparams.stemming) 
+    {
+    case 1: mgq_ask(".set stem on");  break;
+    case 0: mgq_ask(".set stem off"); break;
+    }
+  mgq_ask(".set heads_length 150");
+  
+  char maxdocstr[32];
+  sprintf(maxdocstr, ".set maxdocs %i", queryparams.maxdocs);
+  mgq_ask(maxdocstr);
+}
+
+
+void mgsearchclass::submitquery (const queryparamclass &queryparams)
+{
+  // sort out the query string
+  text_t ttquerystring = queryparams.querystring;
+  text_t ttquotedquery;
+  extractquoted (ttquerystring, ttquotedquery);
+  filterquery (ttquerystring);
+  
+  // turn the strings into c strings for mg
+  if (quotedquery != NULL) // quotedquery is a global
+    {
+      delete quotedquery;
+      quotedquery = NULL;
+    }
+
+  // quotedquery will be deleted on the next call to this function
+  quotedquery = ttquotedquery.getcstr ();
+  char *querystring = ttquerystring.getcstr();
+  
+  // submit the query
+  mgq_ask(querystring);
+
+  delete querystring;
+}
+
+
+void mgsearchclass::getresults (queryresultsclass &queryresults)
+{
+  if (quotedquery[0] == '\0') 
+    { 
+      // don't need the text
+      mgq_results(result_docnums, 0, MAXNUMDOCS, 
+		  ourquerycallback, (void *)(&queryresults));
+    }
+  else
+    {
+      // we need the text for this one
+      mgq_results(result_docs, 0, MAXNUMDOCS, 
+		  ourquerycallback, (void *)(&queryresults));
+    }
+  
+  // get the term frequencies
+  mgq_results(result_termfreqs, 0, MAXNUMTERMS, 
+	      termfreqcallback, (void *)(&queryresults));
+  mgq_results(result_terms, 0, MAXNUMTERMS,
+              termscallback, (void *)(&queryresults));
+  queryresults.sortqueryterms();
+  queryresults.uniqqueryterms();
+}
+
+
+void mgsearchclass::extractquoted (text_t &ttquerystring, text_t &ttquotedquery)
+{
+  ttquotedquery.clear();
+
+  text_t::iterator ithere = ttquerystring.begin ();
+  text_t::iterator itend = ttquerystring.end ();
+
+  bool inquote = false;
+
+  while (ithere != itend)
+    {
+      if ((*ithere) == '\"')
+	{
+	  if (!inquote) ttquotedquery.clear ();
+	  inquote = !inquote;
+	  *ithere = ' '; // delete the quote
+	}
+      else if (inquote)
+	{
+	  ttquotedquery.push_back(*ithere);
+	  *ithere = ' ';
+	}
+      
+      ithere++;
+    }
+}
+
+
+void mgsearchclass::filterquery (text_t &ttquerystring)
+{
+
+  text_t::iterator ithere = ttquerystring.begin ();
+  text_t::iterator itend = ttquerystring.end ();
+  unsigned short c;
+
+  // remove all non alphanumeric characters below 127
+  while (ithere != itend)
+    {
+      c = *ithere;
+
+      //      if ((c <= 127) && !((c >= '0' && c <= '9') ||
+      //			  (c >= 'A' && c <= 'Z') ||
+      //			  (c >= 'a' && c <= 'z')))
+      if (!(((c >= 65) && (c <= 90)) ||
+	    ((c >= 97) && (c <= 122)) ||
+	    ((c >= 192) && (c <= 214)) ||
+	    ((c >= 216) && (c <= 246)) ||
+	    ((c >= 248) && (c <= 255)) ||
+	    ((c >= '0') && (c <= '9')) ||
+	    (c == 176)))
+	(*ithere) = ' ';
+      
+      ithere++;
+    }
+}
+
+
+// the document text for 'docnum' is placed in 'output'
+// docTargetDocument returns 'true' if it was able to
+// try to get a document
+// collection is needed to see if an index from the 
+// collection is loaded. If no index has been loaded
+// defaultindex is needed to load one
+bool mgsearchclass::docTargetDocument(const text_t &defaultindex, 
+				 const text_t &collection,
+				 int docnum,
+				 text_t &output)
+{
+  bool databaseloaded = true;
+
+  output.clear();
+
+
+  // make sure index is level 2
+
+  ////// this changed with new naming scheme in new building software
+  /////  i.e paragraph level index no longer contain number '3' but begin
+  /////  with letter 'p'
+
+  text_t db_loaded = db_loaded_name;
+
+  if (!db_loaded.empty()) {
+	  text_t::const_iterator here = db_loaded.begin();
+	  text_t::const_iterator end = db_loaded.end();
+  
+	  
+	  //while (here != end) {
+	  //  if (*here == '3')
+	  //    databaseloaded = false;
+	  //  here ++;
+	  //}
+
+	  char separator = '/';
+	  text_t db;
+	  int found = 0;
+#ifdef __WIN32__
+	  separator = '\\';
+#endif;
+	  // strip away path to db and following collection name
+	  end --;
+	  while (end != here) {
+		  if (*end == separator) {
+			  if (found) break;
+			  else {db.clear(); found = 1; end--; continue;}
+		  }
+		  db.push_back(*end);
+		  end --;
+	  }
+
+	  // string will have been reversed above so see if last 
+	  // character is 'p'
+	  if (db[db.size()-1] == 'p') databaseloaded = false;
+  }
+
+  // find out if the database is already loaded
+  // this is needed because a different index (but valid one)
+  // might be already loaded.
+  // this comparison is needed because 'load_database'
+  // is now more oriented towards indexes
+  if (databaseloaded == true) {
+	  text_t::const_iterator here = collection.begin();
+	  text_t::const_iterator end = collection.end();
+	  char *dbhere = &db_loaded_name[strlen(db_loaded_name) - collection.size()]; // assumes collection shorter than db_loaded_name
+	  while (here != end)
+	  {
+		  if (*here != *dbhere)
+		  {
+			  databaseloaded = false;
+			  break;
+		  }
+		  here++;
+		  dbhere++;
+	  }
+  }
+
+  // try and load the database
+  if (!databaseloaded)
+    {
+      // get the names of the index and text suffixes
+      text_t ttidxsuffix, tttxtsuffix;
+      getindexsuffix (defaultindex, collection, ttidxsuffix);
+      gettextsuffix (collection, tttxtsuffix);
+	  char *idxsuffix = ttidxsuffix.getcstr(); assert (idxsuffix != NULL);
+      char *txtsuffix = tttxtsuffix.getcstr(); assert (txtsuffix != NULL);
+
+#ifdef __WIN32__
+      char *cindexhome = (indexhome+"\\").getcstr(); assert (cindexhome != NULL);
+#else
+      char *cindexhome = indexhome.getcstr(); assert (cindexhome != NULL);
+#endif
+      
+
+      if (load_database(cindexhome, idxsuffix, txtsuffix))
+	{
+	  databaseloaded = true;
+	}
+      else 
+	{
+	  databaseloaded = false;
+	}
+	  
+      // free up the c strings
+      delete idxsuffix;
+      delete txtsuffix;
+      delete cindexhome;
+    }
+  
+  if (databaseloaded)
+    {
+      // retrieve the document from mg
+      char docstr[32];
+      sprintf(docstr, "%i", docnum);
+
+      mgq_ask(".set mode text");
+      mgq_ask(".set query docnums");
+      mgq_ask(docstr);
+      mgq_results (result_docs, 0, 1, doctextcallback, (void *)&output);
+    }
+
+  return databaseloaded;
+}
+
Index: /trunk/gsdl/src/library/mgsearch.h
===================================================================
--- /trunk/gsdl/src/library/mgsearch.h	(revision 4)
+++ /trunk/gsdl/src/library/mgsearch.h	(revision 4)
@@ -0,0 +1,51 @@
+#ifndef MGSEARCH_H
+#define MGSEARCH_H
+
+
+#include "text_t.h"
+#include "querycache.h"
+
+#define RESULTCACHESIZE 10
+#define MAXNUMDOCS      500
+#define MAXNUMTERMS     100
+
+class mgsearchclass
+{
+public:
+  mgsearchclass ();
+  virtual ~mgsearchclass ();
+  
+  // the index directory must be set before any searching
+  // is done
+  void setindexhome (const text_t &theindexhome);
+  
+  // the search results are returned in queryresults
+  // search returns 'true' if it was able to do a search
+  bool search(const queryparamclass &queryparams, 
+	      queryresultsclass &queryresults);
+  
+  // the document text for 'docnum' is placed in 'output'
+  // docTargetDocument returns 'true' if it was able to
+  // try to get a document
+  // collection is needed to see if an index from the 
+  // collection is loaded. If no index has been loaded
+  // defaultindex is needed to load one
+  bool docTargetDocument(const text_t &defaultindex, 
+			 const text_t &collection,
+			 int docnum,
+			 text_t &output);
+  
+protected:
+  querycache *cache;
+  text_t indexhome; // top level directory for indexes
+
+  void setsearchmode (const queryparamclass &queryparams);
+  void submitquery (const queryparamclass &queryparams);
+  void getresults (queryresultsclass &queryresults);
+
+  virtual void extractquoted (text_t &ttquerystring, text_t &ttquotedquery);
+  virtual void filterquery (text_t &ttquerystring);
+};
+
+#endif
+
Index: /trunk/gsdl/src/library/querycache.cpp
===================================================================
--- /trunk/gsdl/src/library/querycache.cpp	(revision 4)
+++ /trunk/gsdl/src/library/querycache.cpp	(revision 4)
@@ -0,0 +1,77 @@
+#include "querycache.h"
+
+
+
+resultcacheel::resultcacheel () 
+{
+  accessnum = -1;
+}
+
+
+
+querycache::querycache (int themaxcachesize)
+{
+  if (themaxcachesize < 1) themaxcachesize = 1;
+  
+  resultcache = new resultcacheel[themaxcachesize];
+
+  maxcachesize = themaxcachesize;
+  nextaccessnum = 1;
+}
+
+querycache::~querycache ()
+{
+  delete [] resultcache;
+}
+
+// returns true if the query was found in the cache
+// if the query was found then queryresults contains
+// the results
+bool querycache::find (const queryparamclass &queryparams, 
+		       queryresultsclass &queryresults)
+{
+  int i;
+
+  for (i=0; i < maxcachesize; i++)
+    {
+      if (resultcache[i].queryparameters == queryparams)
+	{
+	  queryresults = resultcache[i].queryresults;
+	  resultcache[i].accessnum = getnextaccessnum ();
+	  return true;
+	}
+    }
+
+  return false;
+}
+
+
+void querycache::cache (const queryparamclass &queryparams,
+	  const queryresultsclass &queryresults)
+{
+  int i = getfreecachenum();
+  resultcache[i].queryparameters = queryparams;
+  resultcache[i].queryresults = queryresults;
+  resultcache[i].accessnum = getnextaccessnum ();
+}
+
+int querycache::getnextaccessnum ()
+{
+  return nextaccessnum++;
+}
+
+int querycache::getfreecachenum ()
+{
+  int i;
+  int minaccessnum = 0;
+  int minaccessi = 0;
+  
+  for (i=0; i < maxcachesize; i++)
+    {
+      if (resultcache[i].accessnum < minaccessnum)
+	minaccessi = i;
+    }
+
+  return i;
+}
+
Index: /trunk/gsdl/src/library/querycache.h
===================================================================
--- /trunk/gsdl/src/library/querycache.h	(revision 4)
+++ /trunk/gsdl/src/library/querycache.h	(revision 4)
@@ -0,0 +1,44 @@
+#ifndef QUERYCACHE_H
+#define QUERYCACHE_H
+
+#include "text_t.h"
+#include "queryinfo.h"
+
+class resultcacheel
+{
+public:
+  resultcacheel ();
+
+  int accessnum;
+  queryparamclass queryparameters;
+  queryresultsclass queryresults;
+};
+
+
+class querycache {
+public:
+  querycache (int themaxcachesize);
+  ~querycache ();
+
+  // returns true if the query was found in the cache
+  // if the query was found then queryresults contains
+  // the results
+  bool find (const queryparamclass &queryparams, 
+	     queryresultsclass &queryresults);
+
+  void cache (const queryparamclass &queryparams,
+	    const queryresultsclass &queryresults);
+
+protected:
+  resultcacheel *resultcache;
+
+  int maxcachesize;
+  int nextaccessnum;
+  
+  int getnextaccessnum ();
+  int getfreecachenum ();
+};
+
+
+
+#endif
Index: /trunk/gsdl/src/library/queryinfo.cpp
===================================================================
--- /trunk/gsdl/src/library/queryinfo.cpp	(revision 4)
+++ /trunk/gsdl/src/library/queryinfo.cpp	(revision 4)
@@ -0,0 +1,170 @@
+#include "queryinfo.h"
+
+
+// query parameters
+
+queryparamclass &queryparamclass::operator=(const queryparamclass &q)
+{
+  collection = q.collection;
+  search_index = q.search_index;
+  querystring = q.querystring;
+  search_type = q.search_type;
+  casefolding = q.casefolding;
+  stemming = q.stemming;
+  maxdocs = q.maxdocs;
+
+  return *this;
+}
+
+
+bool operator==(const queryparamclass &x, const queryparamclass &y)
+{
+  return ((x.collection == y.collection) &&
+	  (x.search_index == y.search_index) &&
+	  (x.querystring == y.querystring) &&
+	  (x.search_type == y.search_type) &&
+	  (x.casefolding == y.casefolding) &&
+	  (x.stemming == y.stemming) &&
+	  (x.maxdocs == y.maxdocs));
+}
+
+bool operator!=(const queryparamclass &x, const queryparamclass &y)
+{
+  return !(x == y);
+}
+
+
+ostream &operator<< (ostream &outs, queryparamclass &q)
+{
+  outconvertclass text_t2ascii;
+
+  outs << "*** queryparamclass\n";
+  outs << text_t2ascii << " collection = \"" << q.collection << "\"\n";
+  outs << text_t2ascii << " search_index = \"" << q.search_index << "\"\n";
+  outs << text_t2ascii << " querystring = \"" << q.querystring << "\"\n";
+  outs << " search_type = \"" << q.search_type << "\"\n";
+  outs << " casefolding = \"" << q.casefolding << "\"\n";
+  outs << " stemming = \"" << q.stemming << "\"\n";
+  outs << " maxdocs = \"" << q.maxdocs << "\"\n";
+  outs << "\n";
+
+  return outs;
+}
+
+
+
+
+// term frequencies
+
+termfreqclass &termfreqclass::operator=(const termfreqclass &t) 
+{
+  termstr = t.termstr;
+  termfreq = t.termfreq;
+
+  return *this;
+}
+	
+bool operator==(const termfreqclass &x, const termfreqclass &y)
+{
+  return ((x.termstr == y.termstr) &&
+	  (x.termfreq == y.termfreq));
+}
+
+bool operator!=(const termfreqclass &x, const termfreqclass &y)
+{
+  return !(x == y);
+}
+
+// ordered by termfreq and then by termstr
+bool operator<(const termfreqclass &x, const termfreqclass &y)
+{
+  return ((x.termfreq < y.termfreq) ||
+	  ((x.termfreq == y.termfreq) && (x.termstr < y.termstr)));
+
+}
+
+bool operator>(const termfreqclass &x, const termfreqclass &y)
+{
+  return ((x.termfreq > y.termfreq) ||
+	  ((x.termfreq == y.termfreq) && (x.termstr > y.termstr)));
+
+}
+
+// stream output for debugging purposes
+ostream &operator<< (ostream &outs, termfreqclass &t)
+{
+  outconvertclass text_t2ascii;
+
+  outs << text_t2ascii << " t:\"" << t.termstr << "\"";
+  outs << " f:" << t.termfreq << "\n";
+
+  return outs;
+}
+
+
+
+// one query result
+
+// stream output for debugging purposes
+ostream &operator<< (ostream &outs, docresultclass &a)
+{
+  outs << " d:" << a.docnum << " w:" << a.docweight << "\n";
+  return outs;
+}
+
+
+
+// query results
+
+void queryresultsclass::clear ()
+{
+  docs.erase(docs.begin(),docs.end());
+  terms.erase(terms.begin(),terms.end());
+}
+
+queryresultsclass &queryresultsclass::operator=(const queryresultsclass &q)
+{
+  docs = q.docs;
+  terms = q.terms;
+
+  return *this;
+}
+
+void queryresultsclass::sortqueryterms()
+{
+  sort (terms.begin(), terms.end());
+}
+
+void queryresultsclass::uniqqueryterms()
+{
+  vector<termfreqclass>::iterator new_end = unique (terms.begin(), terms.end());
+  terms.erase(new_end, terms.end());
+}
+
+
+
+// stream output for debugging purposes
+ostream &operator<< (ostream &outs, queryresultsclass &q)
+{
+  outs << "*** queryresultsclass\n";
+  outs << "docs\n";
+
+  vector<docresultclass>::iterator docshere = q.docs.begin();
+  vector<docresultclass>::iterator docsend = q.docs.end();
+  while (docshere != docsend) {
+    outs << (*docshere);
+    docshere++;
+  }
+
+  outs << "terms\n";
+  vector<termfreqclass>::iterator termshere = q.terms.begin();
+  vector<termfreqclass>::iterator termsend = q.terms.end();
+  while (termshere != termsend) {
+    outs << (*termshere);
+    termshere++;
+  }
+
+  outs << "\n";
+
+  return outs;
+}
Index: /trunk/gsdl/src/library/queryinfo.h
===================================================================
--- /trunk/gsdl/src/library/queryinfo.h	(revision 4)
+++ /trunk/gsdl/src/library/queryinfo.h	(revision 4)
@@ -0,0 +1,96 @@
+#ifndef QUERYINFO_H
+#define QUERYINFO_H
+
+#include "text_t.h"
+
+#ifndef USE_OBJECTSPACE
+#  include <vector>
+#  include <algorithm>
+#else
+#  include <ospace\std\vector>
+#  include <ospace\std\algorithm>
+#endif
+
+// query parameters
+
+class queryparamclass 
+{
+public:
+  text_t collection;
+  text_t search_index;
+  text_t querystring;
+  int search_type; // 0 = boolean, 1 = ranked
+  int casefolding;
+  int stemming;
+  int maxdocs;
+
+  queryparamclass &operator=(const queryparamclass &q);
+};
+
+bool operator==(const queryparamclass &x, const queryparamclass &y);
+bool operator!=(const queryparamclass &x, const queryparamclass &y);
+
+// stream output for debugging purposes
+ostream &operator<< (ostream &outs, queryparamclass &q);
+
+
+
+// term frequencies
+
+class termfreqclass 
+{
+public:
+  text_t termstr;
+  unsigned int termfreq;
+  
+  termfreqclass &operator=(const termfreqclass &t);
+};
+
+bool operator==(const termfreqclass &x, const termfreqclass &y);
+bool operator!=(const termfreqclass &x, const termfreqclass &y);
+bool operator<(const termfreqclass &x, const termfreqclass &y);
+bool operator>(const termfreqclass &x, const termfreqclass &y);
+
+// stream output for debugging purposes
+ostream &operator<< (ostream &outs, termfreqclass &q);
+
+
+
+// one query result
+
+class docresultclass
+{
+public:
+  int docnum;
+  float docweight;
+};
+
+// stream output for debugging purposes
+ostream &operator<< (ostream &outs, docresultclass &a);
+
+
+
+// query results
+
+class queryresultsclass 
+{
+public:
+  vector<docresultclass> docs;
+  vector<termfreqclass> terms;
+  vector<text_t> termvariants;
+
+  void clear ();
+  queryresultsclass &operator=(const queryresultsclass &q);
+
+  int getnumdocs () {return docs.size();}
+  int getnumterms () {return terms.size();}
+
+  void sortqueryterms();
+  void uniqqueryterms();
+};
+
+// stream output for debugging purposes
+ostream &operator<< (ostream &outs, queryresultsclass &q);
+
+
+#endif
Index: /trunk/gsdl/src/library/text_t.cpp
===================================================================
--- /trunk/gsdl/src/library/text_t.cpp	(revision 4)
+++ /trunk/gsdl/src/library/text_t.cpp	(revision 4)
@@ -0,0 +1,555 @@
+/**************************************************************************
+ *
+ * text_t.cpp -- a simple 16-bit charater string class
+ * Copyright (c) 1998 -- Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+/*
+   $Log$
+   Revision 1.1  1998/11/17 01:11:06  sjboddie
+   Initial revision
+
+   Revision 1.7  1998/06/24 01:50:08  rjmcnab
+   Added stuff for the objectspace libraries
+
+   Revision 1.6  1998/06/22 22:11:14  rjmcnab
+   Added copy operator = for ints and c-strings for efficiency
+   Added some general functions for dealing with text_ts
+
+   Revision 1.5  1998/06/18 23:53:52  rjmcnab
+   added '+' operator for concatenation
+   removed global declaration of ascii2text_t and text_t2ascii
+     (too unfriendly for threads)
+   added support for integers
+
+   Revision 1.4  1998/06/17 04:52:32  rjmcnab
+   added support for arrays of characters
+
+   Revision 1.3  1998/06/17 02:09:42  rjmcnab
+   added getcstr
+
+   Revision 1.2  1998/06/15 22:01:19  rjmcnab
+   fixed a few bugs
+
+   Revision 1.1  1998/06/13 20:08:34  rjmcnab
+   Initial revision
+
+*/
+
+static char *RCSID = "$Id$";
+
+
+#ifndef USE_OBJECTSPACE
+#  include <algorithm>
+#else
+#  include <ospace\std\algorithm>
+#endif
+
+
+#include "text_t.h"
+
+////////////////////////////////////
+// text_t methods
+////////////////////////////////////
+
+text_t::text_t () 
+{
+  clear ();
+}
+
+text_t::text_t (int i) 
+{
+  clear ();
+  appendint (i);
+}
+
+text_t::text_t (char *s) 
+{
+  clear ();
+  appendcstr (s);
+}
+
+void text_t::append (const text_t &t) 
+{
+  const_iterator here, end=t.end();
+  for (here=t.begin(); here!=end;here++)
+    {
+      text.push_back(*here);
+    }
+}
+
+void text_t::appendrange (iterator first, iterator last) 
+{
+  while (first != last) 
+    {
+      text.push_back (*first);
+      first++;
+    }
+}
+
+void text_t::appendrange (const_iterator first, const_iterator last) 
+{
+  while (first != last) 
+    {
+      text.push_back (*first);
+      first++;
+    }
+}
+
+void text_t::appendint (int i)
+{
+  // deal with zeros and negatives
+  if (i == 0) 
+    {
+      text.push_back('0');
+      return;
+    }
+  else if (i < 0)
+    {
+      text.push_back('-');
+      i *= -1;
+    }
+
+  // get a buffer for the conversion
+  int maxbuflen = sizeof(int)*3;
+  char *buf = new char[maxbuflen];
+  int len = 0;
+  
+  // get the number in reverse
+  while (i > 0)
+    {
+      buf[len++] = '0'+ (i%10);
+      i = i/10;
+    }
+
+  // reverse the number
+  while (len > 0)
+    {
+      text.push_back(buf[--len]);
+    }
+
+  delete buf;
+}
+
+int text_t::getint ()
+{
+  int i = 0;
+  int mult = 1; // become -1 for negative numbers
+
+  iterator here = text.begin();
+  iterator end = text.end();
+  
+  // do plus and minus signs
+  if (here != end)
+    {
+      if (*here == '-')
+	{
+	  mult = -1;
+	  here++;
+	}
+      else if (*here == '+')
+	{
+	  mult = 1;
+	  here++;
+	}
+    }
+
+  // deal with the number
+  while ((here != end) && (*here >= '0') && (*here <= '9'))
+    {
+      i = 10*i + (*here - '0');
+      here++;
+    }
+
+  i *= mult;
+  return i;
+}
+
+
+
+void text_t::appendcarr (char *s, size_type len)
+{
+  unsigned char *us = (unsigned char *)s;
+  while (len > 0) 
+    {
+      text.push_back (*us); // append this character
+      us++;
+      len--;
+    }
+}
+
+void text_t::appendcstr (char *s) 
+{
+  unsigned char *us = (unsigned char *)s;
+  while (*us != '\0') 
+    {
+      text.push_back (*us); // append this character
+      us++;
+    }
+}
+
+
+// strings returned from getcarr and getcstr become the callers 
+// responsibility and should be deallocated with "delete"
+
+char *text_t::getcarr(size_type &len) const
+{
+  unsigned char *cstr = new unsigned char[size()];
+  const_iterator ithere = begin();
+  const_iterator itend = end();
+
+  while (ithere != itend)
+    {
+      if (*ithere >= 256) cstr[len] = ' ';
+      else cstr[len] = (*ithere);
+      len++;
+      ithere++;
+    }
+
+  return (char *)cstr;
+}
+
+char *text_t::getcstr() const
+{
+  unsigned char *cstr = new unsigned char[size() + 1];
+  const_iterator ithere = begin();
+  const_iterator itend = end();
+  int len = 0;
+
+  while (ithere != itend)
+    {
+      if (*ithere >= 256) cstr[len] = ' ';
+      else cstr[len] = (*ithere);
+      len++;
+      ithere++;
+    }
+
+  cstr[len] = '\0';
+
+  return (char *)cstr;
+}
+
+
+// general functions which work on text_ts
+
+// find a character within a range
+text_t::const_iterator findchar (text_t::const_iterator first, text_t::const_iterator last, 
+				 unsigned short c)
+{
+  while (first != last)
+    {
+      if (*first == c) break;
+      first++;
+    }
+  return first;
+}
+
+text_t::iterator findchar (text_t::iterator first, text_t::iterator last, 
+			   unsigned short c)
+{
+  while (first != last)
+    {
+      if (*first == c) break;
+      first++;
+    }
+  return first;
+}
+
+// get a string up to the next delimiter (which is skipped)
+text_t::const_iterator getdelimitstr (text_t::const_iterator first, 
+				      text_t::const_iterator last,
+				      unsigned short c, text_t &outstr)
+{
+  text_t::const_iterator here = first;
+  here = findchar (first, last, c);
+  outstr.clear();
+  outstr.appendrange (first, here);
+  if (here != last) here++; // skip c
+  return here;
+}
+
+text_t::iterator getdelimitstr (text_t::iterator first, text_t::iterator last,
+				unsigned short c, text_t &outstr)
+{
+  text_t::iterator here = first;
+  here = findchar (first, last, c);
+  outstr.clear();
+  outstr.appendrange (first, here);
+  if (here != last) here++; // skip c
+  return here;
+}
+
+// split a string with a character
+void splitchar (text_t::const_iterator first, text_t::const_iterator last,
+		unsigned short c, text_tlist &outlist)
+{
+  outlist.erase(outlist.begin(), outlist.end());
+
+  text_t t;
+
+  while (first != last)
+    {
+      first = getdelimitstr (first, last, c, t);
+      outlist.push_back (t);
+    }
+}
+
+void splitchar (text_t::const_iterator first, text_t::const_iterator last,
+		unsigned short c, text_tarray &outlist)
+{
+  outlist.erase(outlist.begin(), outlist.end());
+
+  text_t t;
+
+  while (first != last)
+    {
+      first = getdelimitstr (first, last, c, t);
+      outlist.push_back (t);
+    }
+}
+
+// join a string using a character
+void joinchar (const text_tlist &inlist, unsigned short c, text_t &outtext)
+{
+  outtext.clear ();
+
+  text_tlist::const_iterator here = inlist.begin ();
+  text_tlist::const_iterator end = inlist.end ();
+  bool first = true;
+  while (here != end)
+    {
+      if (!first) outtext.push_back (c);
+      first = false;
+      outtext += *here;
+      here++;
+    }
+}
+
+void joinchar (const text_tarray &inlist, unsigned short c, text_t &outtext)
+{
+  outtext.clear ();
+
+  text_tarray::const_iterator here = inlist.begin ();
+  text_tarray::const_iterator end = inlist.end ();
+  bool first = true;
+  while (here != end)
+    {
+      if (!first) outtext.push_back (c);
+      first = false;
+      outtext += *here;
+      here++;
+    }
+}
+
+// count the occurances of a character within a range
+int countchar (text_t::const_iterator first, text_t::const_iterator last,
+	       unsigned short c)
+{
+  int count = 0;
+  while (first != last) {
+    if (*first == c) count ++;
+    first ++;
+  }
+  return count;
+}
+
+
+
+////////////////////////////////////
+// convertclass methods
+////////////////////////////////////
+
+// conversion classes used for getting information in to and out of
+// the text_t class.
+
+convertclass::convertclass () 
+{
+  // nothing to do
+}
+
+void convertclass::reset ()
+{
+  // nothing to do
+}
+
+
+////////////////////////////////////
+// inconvertclass methods
+////////////////////////////////////
+
+// convert from a char stream to the text_t class
+// the default version assumes the input is a ascii
+// character array
+
+inconvertclass::inconvertclass () 
+{
+  start = NULL;
+  len = 0;
+}
+
+
+void inconvertclass::reset ()
+{
+  start = NULL;
+  len = 0;
+}
+
+void inconvertclass::setinput (char *thestart, size_t thelen)
+{
+  start = thestart;
+  len = thelen;
+}
+
+void inconvertclass::convert (text_t &output, status_t &status)
+{
+  if (start == NULL || len == 0)
+    {
+      status = finished;
+      return;
+    }
+
+  // don't want any funny sign conversions happening
+  unsigned char *here = (unsigned char *)start;
+  while (len > 0) 
+    {
+      output.push_back (*here); // append this character
+      ++here;
+      --len;
+    }
+
+  start = (char *)here; // save current position
+  status = finished;
+}
+
+// an instance of the default inconvertclass to do simple
+// conversions. Note that any functions that use this are
+// not reentrant. If a function needs to be reentrant it
+// should declare its own instance.
+inconvertclass ascii2text_t;
+
+
+////////////////////////////////////
+// outconvertclass methods
+////////////////////////////////////
+
+// Convert from a text_t class to a char stream
+// This default version assumes the output is a ascii
+// character array. If you set the output stream you
+// can use this class to output to a stream using the
+// << operator. The << operator can also be conveniently
+// used to set the output stream by doing something like
+//
+// cout << text_t2ascii << text_tstr << anothertext_tstr;
+//
+outconvertclass::outconvertclass ()
+{
+  input = NULL;
+  outs = NULL;
+}
+
+void outconvertclass::reset ()
+{
+  input = NULL;
+  outs = NULL;
+}
+
+void outconvertclass::setinput (text_t *theinput)
+{
+  input = theinput;
+  if (input != NULL) texthere = input->begin();
+}
+
+void outconvertclass::convert (char *output, size_t maxlen, 
+		      size_t &len, status_t &status)
+{
+  if (input == NULL || output == NULL)
+    {
+      status = finished;
+      return;
+    }
+
+  // don't want any funny sign conversions happening
+  unsigned char *uoutput = (unsigned char *)output;
+  text_t::iterator textend = input->end();
+  len = 0;
+  while ((len < maxlen) && (texthere != textend)) 
+    {
+      if (*texthere < 256) *uoutput = (unsigned char)(*texthere);
+      else *uoutput = 32; // put a space where a char >= 256 exists
+      ++uoutput;
+      ++len;
+      ++texthere;
+    }
+  
+  if (texthere == textend) status = finished;
+  else status = unfinished;
+}
+
+void outconvertclass::setostream (ostream *theouts)
+{
+  outs = theouts;
+}
+
+ostream *outconvertclass::getostream ()
+{
+  return outs;
+}
+
+
+
+
+// an instance of the default outconvertclass to do simple
+// conversions
+outconvertclass text_t2ascii;
+
+
+
+// stream operators for the output class
+
+outconvertclass &operator<< (ostream &theouts, outconvertclass &outconverter)
+{
+  outconverter.setostream(&theouts);
+  return outconverter;
+}
+
+
+#define STREAMBUFSIZE 256
+outconvertclass &operator<< (outconvertclass &outconverter, const text_t &t)
+{
+  ostream *outstream = outconverter.getostream();
+
+  if (outstream == NULL) return outconverter;
+
+  char outbuf[STREAMBUFSIZE];
+  size_t len;
+  outconvertclass::status_t status = outconvertclass::unfinished;
+
+  // assume that there is no data needing converting
+  // left in the converter
+  outconverter.setinput ((text_t *)(&t)); // note the const -> nonconst conversion
+
+  while (status == outconvertclass::unfinished)
+    {
+      outconverter.convert (outbuf, STREAMBUFSIZE, len, status);
+      if (len > 0) outstream->write(outbuf, len);
+    }
+
+  return outconverter;
+}
Index: /trunk/gsdl/src/library/text_t.h
===================================================================
--- /trunk/gsdl/src/library/text_t.h	(revision 4)
+++ /trunk/gsdl/src/library/text_t.h	(revision 4)
@@ -0,0 +1,313 @@
+/**************************************************************************
+ *
+ * text_t.h -- a simple 16-bit charater string class
+ * Copyright (c) 1998 -- Rodger McNab
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * $Id$
+ *
+ **************************************************************************/
+
+
+
+#ifndef TEXT_T_H
+#define TEXT_T_H
+
+#ifndef USE_OBJECTSPACE
+#  include <vector>
+#  include <list>
+#  include <set>
+#else
+#  include <ospace\std\vector>
+#  include <ospace\std\list>
+#  include <ospace\std\set>
+#endif
+
+// use the standard namespace
+#ifdef __GNUG__
+// namespaces are broken in current version of gcc
+#else
+#  ifndef USE_OBJECTSPACE
+using namespace std;
+#  else
+#    ifndef OS_NO_NAMESPACE
+using namespace ospace::std;
+#    endif
+#  endif
+#endif
+
+
+// class prototypes
+class inconvertclass;
+class outconvertclass;
+
+
+typedef vector<unsigned short> usvector;
+
+// The class text_t can handle long strings which may contain
+// null characters. It uses unsigned shorts to represent up to
+// 64K character values.
+class text_t {
+  public:
+  //type support for ucvector
+  typedef usvector::iterator iterator;
+  typedef usvector::const_iterator const_iterator;
+  typedef usvector::reference reference;
+  typedef usvector::const_reference const_reference;
+  typedef usvector::size_type size_type;
+  typedef usvector::difference_type difference_type;
+  typedef usvector::const_reverse_iterator const_reverse_iterator;
+  typedef usvector::reverse_iterator reverse_iterator;
+  
+protected:
+  usvector text;
+  
+public:
+  // constructors
+  text_t ();
+  text_t (int i);
+  text_t (char *s); // assumed to be a normal c string
+
+  // basic container support
+  iterator begin () {return text.begin();}
+  const_iterator begin () const {return text.begin();}
+  iterator end () {return text.end();}
+  const_iterator end () const {return text.end();}
+
+  void erase(iterator pos) {text.erase(pos);}
+  void erase(iterator first, iterator last) {text.erase(first, last);}
+  void push_back(unsigned short c) {text.push_back(c);}
+  void pop_back() {text.pop_back();}
+  text_t &operator=(const text_t &x) {text=x.text;return *this;}
+  reference operator[](size_type n) {return text[n];};
+  const_reference operator[](size_type n) const {return text[n];};
+
+  bool empty () const {return text.empty();}
+  size_type size() const {return text.size();}
+  friend bool operator!=(const text_t& x, const text_t& y);
+  friend bool operator==(const text_t& x, const text_t& y);
+  friend bool operator<(const text_t& x, const text_t& y);
+  friend bool operator>(const text_t& x, const text_t& y);
+  
+  // added functionality
+  void clear () {text.erase(text.begin(),text.end());}
+  void append (const text_t &t);
+  void appendrange (iterator first, iterator last);
+  void appendrange (const_iterator first, const_iterator last);
+  text_t &operator+= (const text_t &t) {append(t);return *this;}
+
+  // support for integers
+  void appendint (int i);
+  void setint (int i) {clear();appendint(i);}
+  text_t &operator=(int i) {setint (i);return *this;}
+  text_t &operator+= (int i) {appendint(i);return *this;}
+  int getint ();
+
+  // support for arrays of chars
+  void appendcarr (char *s, size_type len);
+  void setcarr (char *s, size_type len) {clear();appendcarr(s,len);}
+
+  // support for null-terminated C strings
+  void appendcstr (char *s);
+  void setcstr (char *s) {clear();appendcstr(s);}
+  text_t &operator= (char *s) {setcstr(s);return *this;} // c string
+  text_t &operator+= (char *s) {appendcstr(s);return *this;} // c string
+
+  // strings returned from getcarr and getcstr become the callers 
+  // responsibility and should be deallocated with "delete"
+  char *getcarr(size_type &len) const;
+  char *getcstr() const;
+};
+
+
+inline text_t operator+(const text_t &t1, const text_t &t2)
+{
+  text_t tnew = t1;
+  tnew.append(t2);
+  return tnew;
+}
+
+inline text_t operator+(const text_t &t1, int i1)
+{
+  text_t tnew = t1;
+  tnew.appendint(i1);
+  return tnew;
+}
+
+inline text_t operator+(const text_t &t1, char *s1)
+{
+  text_t tnew = t1;
+  tnew.appendcstr(s1);
+  return tnew;
+}
+
+
+inline bool operator!=(const text_t& x, const text_t& y) 
+{
+  return (x.text != y.text);
+}
+
+inline bool operator==(const text_t& x, const text_t& y) 
+{
+  return (x.text == y.text);
+}
+
+inline bool operator<(const text_t& x, const text_t& y) 
+{
+  return (x.text < y.text);
+}
+
+inline bool operator>(const text_t& x, const text_t& y) 
+{
+  return (x.text > y.text);
+}
+
+
+struct eqtext_t 
+{
+  bool operator()(const text_t t1, const text_t t2) const
+  { return t1 == t2; }
+};
+
+struct lttext_t 
+{
+  bool operator()(const text_t t1, const text_t t2) const
+  { return t1 < t2; }
+};
+
+
+// frequently used derived types
+typedef set<text_t,lttext_t> text_tset;
+typedef list<text_t> text_tlist;  // more efficient for insertions/deletions
+typedef vector<text_t> text_tarray; // more space efficient than text_tlist
+
+
+// general functions which work on text_ts
+
+// find a character within a range
+text_t::const_iterator findchar (text_t::const_iterator first, text_t::const_iterator last, 
+				 unsigned short c);
+text_t::iterator findchar (text_t::iterator first, text_t::iterator last, 
+			   unsigned short c);
+
+// get a string up to the next delimiter (which is skipped)
+text_t::const_iterator getdelimitstr (text_t::const_iterator first, 
+				      text_t::const_iterator last,
+				      unsigned short c, text_t &outstr);
+text_t::iterator getdelimitstr (text_t::iterator first, text_t::iterator last,
+				unsigned short c, text_t &outstr);
+
+// split a string with a character
+void splitchar (text_t::const_iterator first, text_t::const_iterator last,
+		unsigned short c, text_tlist &outlist);
+void splitchar (text_t::const_iterator first, text_t::const_iterator last,
+		unsigned short c, text_tarray &outlist);
+
+// join a string using a character
+void joinchar (const text_tlist &inlist, unsigned short c, text_t &outtext);
+void joinchar (const text_tarray &inlist, unsigned short c, text_t &outtext);
+
+// count the occurances of a character within a range
+int countchar (text_t::const_iterator first, text_t::const_iterator last,
+	       unsigned short c);
+
+
+
+
+
+
+
+
+// conversion classes used for getting information in to and out of
+// the text_t class.
+
+class convertclass 
+{
+public:
+  enum status_t {finished, stopped, unfinished};
+  
+  convertclass ();
+  virtual void reset ();
+};
+
+
+
+// convert from a char stream to the text_t class
+// the default version assumes the input is a ascii
+// character array
+class inconvertclass : public convertclass
+{
+public:
+  inconvertclass ();
+  void reset ();
+  void setinput (char *thestart, size_t thelen);
+  virtual void convert (text_t &output, status_t &status);
+
+protected:
+  char *start;
+  size_t len;
+};
+
+// to get something which will do the conversion
+// to ascii declare a (non global!) instance like
+// this
+// inconvertclass ascii2text_t;
+
+
+// Convert from a text_t class to a char stream
+// This default version assumes the output is a ascii
+// character array. If you set the output stream you
+// can use this class to output to a stream using the
+// << operator. The << operator can also be conveniently
+// used to set the output stream by doing something like
+//
+// cout << text_t2ascii << textstr << anothertextstr;
+//
+// this class assumes that the input text doesn't change
+// while the conversion takes place
+class outconvertclass : public convertclass
+{
+public:
+  outconvertclass ();
+  void reset ();
+  void setinput (text_t *theinput);
+  // note that convert does not null-terminate the
+  // output array of characters
+  virtual void convert (char *output, size_t maxlen, 
+			size_t &len, status_t &status);
+
+  void setostream (ostream *theouts);
+  ostream *getostream ();
+
+protected:
+  text_t *input;
+  text_t::iterator texthere; // only valid if input is valid
+
+  ostream *outs;
+};
+
+// to get something which will do the conversion
+// to text_t declare a (non global!) instance like
+// this
+// outconvertclass text_t2ascii;
+
+
+// stream operators for the output class
+outconvertclass &operator<< (ostream &theouts, outconvertclass &outconverter);
+outconvertclass &operator<< (outconvertclass &outconverter, const text_t &t);
+
+
+#endif
Index: /trunk/gsdl/src/txt2db/txt2db.cpp
===================================================================
--- /trunk/gsdl/src/txt2db/txt2db.cpp	(revision 4)
+++ /trunk/gsdl/src/txt2db/txt2db.cpp	(revision 4)
@@ -0,0 +1,124 @@
+#ifdef __WIN32__
+#include "autoconf.h"
+#include "systems.h"
+#include "gdbmconst.h"
+#include "gdbm.h"
+
+#else
+#include <gdbm.h>
+#endif
+
+
+#ifdef __GNUG__
+#  include <iostream.h>
+#else
+#  ifndef USE_OBJECTSPACE
+#    include <iostream>
+#  else
+#    include <ospace\std\iostream>
+#  endif
+#endif
+
+#include "text_t.h"
+
+
+void print_usage (char *program_name) {
+	cerr << "usage: " << program_name << "database-name\n\n";
+}
+
+
+int main (int argc, char *argv[]) {
+	int block_size = 0;
+	GDBM_FILE dbf;
+	char c;
+	text_t key;
+	text_t value;
+	text_t tmp;
+	int num_dashes = 0;
+
+	// sanity check
+	if (argc != 2) {
+		print_usage (argv[0]);
+		exit (0);
+	}
+
+	// open the database
+	dbf = gdbm_open  (argv[1], block_size, GDBM_NEWDB, 00664, NULL);
+    if (dbf == NULL) {
+		cerr << "couldn't create " << argv[1] << "\n";
+		exit (0);
+	}
+
+	cin.get(c);
+	while (!cin.eof()) {
+		num_dashes = 0;
+		key = "";
+		value = "";
+
+		// look for [key]\n
+		while (!cin.eof() && c != '[') cin.get(c);
+		if (!cin.eof()) cin.get(c); // skip [
+		while (!cin.eof() && c != ']') {
+			key.push_back ((unsigned char)c);
+			cin.get(c);
+		}
+		if (!cin.eof()) cin.get(c); // skip ]
+		while (!cin.eof() && (c == '\n' || c == 'r')) cin.get(c);
+
+		// look for 70 dashes
+		tmp = "";
+		while (!cin.eof() && (num_dashes < 70)) {
+			if (c == '\n' || c == '\r') {
+				tmp.push_back ((unsigned char)c);
+				num_dashes = 0;
+
+			} else if (c == '-') {
+				tmp.push_back ((unsigned char)c);
+				num_dashes++;
+
+			} else {
+				value += tmp;
+				value.push_back ((unsigned char)c);
+				tmp = "";
+				num_dashes = 0;
+			}
+			cin.get(c);
+		}
+		
+		// if the key is not an empty string store this key-value pair
+		if (!key.empty()) {
+			// convert key to a datum datatype
+			datum key_data;
+			key_data.dptr = key.getcstr();
+			if (key_data.dptr == NULL) {
+				cerr << "NULL key_data.dptr\n";
+				exit (0);
+			}
+			key_data.dsize = strlen(key_data.dptr);
+
+			// convert value to a datum datatype
+			datum value_data;
+			value_data.dptr = value.getcstr();
+			if (value_data.dptr == NULL) {
+				cerr << "NULL value_data.dptr\n";
+				exit (0);
+			}
+			value_data.dsize = strlen(value_data.dptr);
+
+			// store the value
+			if (gdbm_store (dbf, key_data, value_data, GDBM_REPLACE) < 0) {
+				cerr << "gdbm_store returned an error\n";
+				exit (0);
+			}
+
+			delete key_data.dptr;
+			delete value_data.dptr;
+		}
+	}
+
+
+	gdbm_close (dbf);
+
+
+	return 0;
+}
