Index: trunk/gsdl/macros/phind.dm
===================================================================
--- trunk/gsdl/macros/phind.dm	(revision 1643)
+++ trunk/gsdl/macros/phind.dm	(revision 1643)
@@ -0,0 +1,30 @@
+package phind
+
+#######################################################################
+# page content
+#######################################################################
+
+_imagethispage_{<img src="_httpimg_/h\_topic.gif">}
+
+_content_ {
+<center>
+_navigationbar_
+<p>
+_phindapplet_
+</center>
+}
+
+
+_phindapplet_ {_If_("_cgiargc_" ne "",
+<APPLET CODEBASE="_httpprefix_/src/phind/client" CODE="Phind.class" WIDTH=500 HEIGHT=400>
+  <PARAM NAME=library      VALUE="_gwcgi_">
+  <PARAM NAME=phindcgi     VALUE="phindcgi">
+  <PARAM NAME=backdrop     VALUE="/~paynter/transfer/phindtest/green1.jpg">
+  <PARAM NAME=collection   VALUE="_cgiargc_">
+  <PARAM NAME=orientation  VALUE="vertical">
+  <PARAM NAME=depth        VALUE="2">
+  <PARAM NAME=fontsize     VALUE="12">
+  <PARAM NAME=blocksize    VALUE="10">
+  The Phind java applet.
+</APPLET>,
+The Phind applet requires a collection.}
Index: trunk/gsdl/perllib/classify/phind.pm
===================================================================
--- trunk/gsdl/perllib/classify/phind.pm	(revision 1643)
+++ trunk/gsdl/perllib/classify/phind.pm	(revision 1643)
@@ -0,0 +1,899 @@
+###########################################################################
+#
+# phind.pm -- the Phind classifier
+#
+# Copyright (C) 2000 Gordon W. Paynter
+# Copyright (C) 2000 New Zealand Digital Library Project
+#
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# The phind clasifier plugin
+#
+# options are:
+#   title=Title           The title field for this classification 
+#   text=fields           The text used to build the phrase hierarchy
+#   untidy=true           Do not clean up intermediate files
+#   phindexdir=directory  Location of phind index files
+#   verbosity=num         Control amount of output
+
+
+# How a classifier works.  
+#
+# When a classifier is requested in the collect.cfg file, buildcol creates a
+# new classifier object (such as the one defined in theis file) and later
+# passes each document object to the classifier in turn.  Four functions are
+# used:
+#
+# 1. "new" is called before the documents are processed to set up the
+#    classifier.
+#
+# 2. "init" is called after buildcol.pl has created the indexes etc but
+#    before the documents are classified in order that the classifier might
+#    set any varioables it requiers, etc.
+#
+# 3. "classify" is called once for each document object.  The classifier
+#    "classifies" each document and updates its local data accordingly.
+#
+# 4. "get_classify_info" is called after every document has been
+#    classified.  It collates the information about the documents and
+#    stores a reference to the classifier so that Greenstone can later
+#    display it.
+
+
+package phind;
+
+use BasClas;
+use util;
+
+sub BEGIN {
+    @ISA = ('BasClas');
+}
+
+# Define delimiter symbols - this should be abstracted out someplace
+my $colstart = "COLLECTIONSTART";
+my $colend   = "COLLECTIONEND";
+my $doclimit = "DOCUMENTLIMIT";
+my $senlimit = "SENTENCELIMIT";
+my @delimiters = ($colstart, $colend, $doclimit, $senlimit);
+
+# Create a new phind browser based on the options in collect.cfg
+
+sub new {
+    my ($class, @options) = @_;
+    my $self = new BasClas ($class, @_);
+    my $out = pop @options;
+
+    # Phind installation check
+    # The phind phrase browser is research software and is not installed
+    # by defualt.  If the user attepts to use it we warn them that it's a
+    # bit dodgy, then tell them how to install it.  If they can do that 
+    # and get all the files in place, then we let them proceed.
+	
+    print $out "The Phind classifier for Greenstone.\n";
+    print $out "Checking the phind phrase browser requirements...\n";
+
+    # Make sure we're not in windows
+    if ($ENV{'GSDLOS'} =~ /windows/i) {
+	print STDERR "Phind currently only works under Unix";
+	exit(1);
+    }
+
+    # Ensure the Phind generate scripts are in place
+    my $file1 = &util::filename_cat($ENV{'GSDLHOME'}, "bin", $ENV{'GSDLOS'}, "suffix");
+    my $src = &util::filename_cat($ENV{'GSDLHOME'}, "src", "phind", "generate");
+
+    if (!(-e $file1)) {
+	print STDERR "The phind \"suffix\" program is not installed.  ";
+	print STDERR "To install it, change to the directory\n";
+	print STDERR "   $src\n";
+	print STDERR "and type \"make install-phind\".\n\n";
+	exit(1);
+    }
+
+    # Ensure the Phind CGI script is in place
+    $file1 = &util::filename_cat($ENV{'GSDLHOME'}, "cgi-bin", "phindcgi");
+    $src =  &util::filename_cat($ENV{'GSDLHOME'}, "src", "phind", "host");
+
+    if (!(-e $file1)) {
+	print STDERR "The phind CGI program is not installed.  ";
+	print STDERR "To install it, change to the directory\n";
+	print STDERR "   $src\n";
+	print STDERR "and type \"make install-phind\".\n\n";
+	exit(1);
+    }
+
+    # Ensure the Phind Java applet is in place
+    $src =  &util::filename_cat($ENV{'GSDLHOME'}, "src", "phind", "client");
+    $file1 = &util::filename_cat($src, "Phind.class");
+
+    if (!(-e $file1)) {
+	print STDERR "The phind Java classes are not compiled.  ";
+	print STDERR "To compile them, change to the directory\n";
+	print STDERR "   $src\n";
+	print STDERR "and use your Java compiler to compile Phind.java.\n";
+	print STDERR "(if you have Java 1.2 installed, type \"javac Phind.java\")\n\n";
+	exit(1);
+    }
+	
+
+    # The installation appears OK - set up the classifier
+    my $collection = $ENV{'GSDLCOLLECTION'};
+    my $phindexdir = &util::filename_cat($ENV{'GSDLCOLLECTDIR'},"phindex");
+    my $language = "english";
+
+    my $title = "Topic";
+    my $indexes = "section:Title,section:text";
+
+    my $verbosity = 2;
+    my $untidy = 0;
+
+    # parse the options
+    foreach $option (@options) {
+
+	if ($option =~ /^text=(.*)$/i) {
+	    $indexes = $1;
+	} elsif ($option =~ /^title=(.*)$/i) {
+	    $title = $1;
+	} elsif ($option =~ /^phindexdir=(.*)$/i) {
+	    $phindexdir = $1;
+	} elsif ($option =~ /^verbosity=(.*)$/i) {
+	    $verbosity = $1;
+	} elsif ($option =~ /^untidy/i) {
+	    $untidy = 1;
+	}
+    }
+
+
+    $self->{'collection'} = $collection;
+    $self->{'title'} = $title;
+    $self->{'indexes'} = $indexes;
+    $self->{'verbosity'} = $verbosity;
+    $self->{'untidy'} = $untidy;
+
+    # limit languages
+    $language =~ s/,/\|/g;
+    $self->{'language_exp'} = $language;
+    $self->{'delimiter'} = $delimiter;
+
+    # reset phindex directory
+    if (-e "$phindexdir") {
+	&util::rm_r("$phindexdir");
+    }
+    &util::mk_dir("$phindexdir");
+    $self->{'phindexdir'} = $phindexdir;
+
+    return bless $self, $class;
+}
+
+
+# Initialise the phind classifier
+
+sub init {
+    my $self = shift (@_);
+
+    # open filehandles for documents and text
+    my $phindexdir = $self->{'phindexdir'};
+
+    my $clausefile =  &util::filename_cat("$phindexdir", "clauses");
+    &util::rm($clausefile) if (-e $clausefile);
+    open(TEXT, ">$clausefile") || die "Cannot open $clausefile: $!";
+    $self->{'txthandle'} = TEXT;
+
+    my $docfile = &util::filename_cat("$phindexdir", "docs.txt");
+    &util::rm($docfile) if (-e $docfile);
+    open(DOCS, ">$docfile") || die "Cannot open $docfile: $!";
+    $self->{'dochandle'} = DOCS;
+    
+}
+
+# Classify each document.
+#
+# Each document is passed here in turn.  The classifier extracts the 
+# text of each and stores it in the clauses file.  Document details are
+# stored in the docs.txt file. 
+
+sub classify {
+    my $self = shift (@_);
+    my ($doc_obj) = @_;
+
+    my $verbosity = $self->{'verbosity'};
+    my $top_section = $doc_obj->get_top_section();
+
+    my $title = $doc_obj->get_metadata_element ($top_section, "Title");
+    print "process: $title\n" if ($verbosity > 2);
+
+
+    # only consider english-language files
+    my $doclanguage = $doc_obj->get_metadata_element ($top_section, "Language");
+    my $phrlanguage = $self->{'language_exp'};
+    return if ($doclanguage && ($doclanguage !~ /$phrlanguage/i));
+    
+    # record this file
+    my $total++;
+    print "file $total: $file\n" if ($self->{'$verbosity'});
+    
+
+    # Store document details
+    my $OID = $doc_obj->get_OID();
+    $OID = "NULL" unless defined $OID;
+    my $dochandle = $self->{'dochandle'};
+    print $dochandle "<Document>\t$OID\t$title\n";
+    
+    # Store the text occuring in this object
+
+    # output the document delimiter
+    my $txthandle = $self->{'txthandle'};
+    print $txthandle "$doclimit\n";
+
+    # iterarate over the required indexes and store their text
+    my $indexes = $self->{'indexes'};
+    my $text = "";
+    my ($part, $level, $field, $section, $data);
+
+    foreach $part (split(/,/, $indexes)) {
+
+	# Each field has a level and a data element ((e.g. document:Title)
+	($level, $field) = split(/:/, $part);
+	die unless ($level && $field);
+
+	# Extract the text from every section
+	# (In phind, document:text and section:text are equivalent)
+	if ($field eq "text") {
+	    $data = "";
+	    $section = $doc_obj->get_top_section();
+	    while (defined($section)) {
+		$data .= $doc_obj->get_text($section) . "\n";
+		$section = $doc_obj->get_next_section($section);
+	    }
+	    $text .= convert_gml_to_tokens($data) . "\n";
+	}
+	
+	# Extract a metadata field from a document
+	elsif ($level eq "document") {
+	    $data = $doc_obj->get_metadata_element($doc_obj->get_top_section(), $field);
+	    $text .= convert_gml_to_tokens($data) . "\n";
+	} 
+	
+	# Extract metadata from every section in a document
+	elsif ($level eq "section") {
+	    $data = "";
+	    $section = $doc_obj->get_top_section();
+	    while (defined($section)) {
+		$data .= $doc_obj->get_metadata_element($section, $field) . "\n";
+		$section = $doc_obj->get_next_section($section);
+	    }
+	    $text .= convert_gml_to_tokens($data) . "\n";
+	} 
+	
+	# Some sort of specification which I don't understand
+	else {
+	    die "Unknown level ($level) in phind index ($part)\n";
+	}
+	
+    }
+    
+    # output the text
+    $text =~ tr/\n//s;
+    print $txthandle "$text";
+    
+}
+
+
+
+# Construct the classifier from the information already gathered
+#
+# When get_classify_info is called, the clauses and docs.txt files have
+# already been constructed in the phindex directory.  This function will
+# translate them into compressed, indexed MGPP files that can be read by
+# the phindcgi script.  It will also register our classifier so that it
+# shows up in thenavigation bar.
+
+sub get_classify_info {
+    my $self = shift (@_);
+
+    print STDERR "\n\n**** phind.pm get_classify_info ***\n";
+
+    my $verbosity = $self->{'verbosity'};
+    my $phindexdir = $self->{'phindexdir'};
+    my $language = "english";
+    
+    # Construct phind indexes
+    my $mode = 1;
+    my $symbol_limit = 40000000;
+    my ($command, $status);
+    
+    # Generate the vocabulary, symbol statistics, and numbers file
+    # from the clauses file
+    print "\nExtracting vocabulary and statistics\n" if $verbosity;
+    &extract_vocabulary($phindexdir, $language, $verbosity);
+
+    # Use the suffix program to generate the phindex/phrases file
+    &execute("suffix $phindexdir $symbol_limit $mode", $verbosity);
+
+    # Create the phrase file and put phrase numbers in phindex/phrases
+    print "\nSorting and Renumbering phrases for input to mgpp\n" if $verbosity;
+    &renumber_phrases("$phindexdir", $verbosity);
+    
+    # Create the mg phrase database
+    my $mgpp = &util::filename_cat($ENV{'GSDLHOME'}, "src", "mgpp");
+    my $mg_passes = &util::filename_cat($mgpp, "text", "mg_passes");
+    my $mg_compression_dict = &util::filename_cat($mgpp, "text", "mg_compression_dict");
+
+    my $mg_perf_hash_build = &util::filename_cat($mgpp, "text", "mg_perf_hash_build");
+    my $mg_weights_build = &util::filename_cat($mgpp, "text", "mg_weights_build");
+    my $mg_invf_dict = &util::filename_cat($mgpp, "text", "mg_invf_dict");
+    my $mg_stem_idx = &util::filename_cat($mgpp, "text", "mg_stem_idx");
+
+    print "\nCreating phrase databases\n";
+    my $mg_input = &util::filename_cat($phindexdir, "pdata.txt");
+    my $mg_stem = "pdata";
+
+    &execute("$mg_passes -d $phindexdir -f $mg_stem -T1 $mg_input", $verbosity);
+    &execute("$mg_compression_dict -d $phindexdir -f $mg_stem", $verbosity);
+    &execute("$mg_passes -d $phindexdir -f $mg_stem -T2 $mg_input", $verbosity);
+
+    # create the mg index of words
+    print "\nCreating word-level search indexes\n";
+    $mg_input = &util::filename_cat($phindexdir, "pword.txt");
+    $mg_stem = "pword";
+
+    &execute("$mg_passes -d $phindexdir -f $mg_stem -T1 -I1 $mg_input", $verbosity);
+    &execute("$mg_compression_dict -d $phindexdir -f $mg_stem", $verbosity);
+    &execute("$mg_perf_hash_build -d $phindexdir -f $mg_stem", $verbosity);
+    &execute("$mg_passes -d $phindexdir -f $mg_stem -T2 -I2 $mg_input", $verbosity);
+    &execute("$mg_weights_build -d $phindexdir -f $mg_stem", $verbosity);
+    &execute("$mg_invf_dict -d $phindexdir -f $mg_stem", $verbosity);
+
+    &execute("$mg_stem_idx -d $phindexdir -f $mg_stem -s 1", $verbosity);
+    &execute("$mg_stem_idx -d $phindexdir -f $mg_stem -s 2", $verbosity);
+    &execute("$mg_stem_idx -d $phindexdir -f $mg_stem -s 3", $verbosity);
+
+    # create the mg document information database
+    print "\nCreating document information databases\n";
+    $mg_input = &util::filename_cat($phindexdir, "docs.txt");
+    $mg_stem = "docs";
+
+    &execute("$mg_passes -d $phindexdir -f $mg_stem -T1 $mg_input", $verbosity);
+    &execute("$mg_compression_dict -d $phindexdir -f $mg_stem", $verbosity);
+    &execute("$mg_passes -d $phindexdir -f $mg_stem -T2 $mg_input", $verbosity);
+
+
+    # Tidy up stray files
+    if (!$untidy) {
+	print "\nCleaning up\n" if $verbosity;
+	&util::rm("$phindexdir/clauses", "$phindexdir/clauses.numbers", 
+		  "$phindexdir/clauses.vocab", "$phindexdir/clauses.stats", 
+		  "$phindexdir/phrases", "$phindexdir/docs.txt",
+		  "$phindexdir/pdata.txt", "$phindexdir/pword.txt");
+	my $outfile = 1;
+	while (-e "$phindexdir/outPhrase.$outfile") {
+	    &util::rm("$phindexdir/outPhrase.$outfile");
+	    $outfile++;
+	}
+    }
+
+
+    # Insert the classifier into.... what?
+    my $collection = $self->{'collection'};
+    my $url = "library?a=p&p=phind&c=$collection";
+
+    my %classifyinfo = ('thistype'=>'Invisible',
+			'childtype'=>'Phind',
+			'Title'=>$self->{'title'},
+			'contains'=>[]);
+    
+    push (@{$classifyinfo{'contains'}}, {'OID'=>$url});
+    return \%classifyinfo;
+}
+
+
+
+sub convert_gml_to_tokens {
+    
+    $_ = shift @_;
+
+    # FIRST, remove GML tags
+
+    # Replace all whitespace with a simple space
+    s/\s+/ /gs;
+
+    # Remove everything that is in a tag
+    s/\s*<p>\s*/ PARAGRAPHBREAK /isg;
+    s/\s*<br>\s*/ LINEBREAK /isg;
+    s/<[^>]*>/ /sg;
+
+    # Now we have the text, but it may contain HTML 
+    # elements coded as &gt; etc.  Remove these tags. 
+    s/&lt;/</sg;
+    s/&gt;/>/sg;
+
+    s/\s+/ /sg;
+    s/\s*<p>\s*/ PARAGRAPHBREAK /isg;
+    s/\s*<br>\s*/ LINEBREAK /isg;
+    s/<[^>]*>/ /sg;
+
+    # remove &amp; and other miscellaneous markup tags
+    s/&amp;/&/sg;
+    s/&lt;/</sg;
+    s/&gt;/>/sg;
+    s/&amp;/&/sg;
+
+    # replace<p> and <br> placeholders with carriage returns
+    s/PARAGRAPHBREAK/\n/sg;
+    s/LINEBREAK/\n/sg;
+
+
+    # Exceptional punctuation
+    # 
+    # We make special cases of some punctuation
+
+    # remove any apostrophe that indicates omitted letters 
+    s/(\w+)\'(\w*\s)/ $1$2 /g;
+
+    # remove period that appears in a person's initals
+    s/\s([A-Z])\./ $1 /g;
+
+    # replace hyphens in hypheanted words and names with a space
+    s/([A-Za-z])-\s*([A-Za-z])/$1 $2/g;
+
+
+    # Convert the remaining text to "clause format",
+    # This means removing all excess punctuation and garbage text,
+    # normalising valid punctuation to fullstops and commas,
+    # then putting one cluse on each line.
+
+    # Insert newline when the end of a sentence is detected
+    # (delimter is:  "[\.\?\!]\s")
+    s/\s*[\.\?\!]\s+/\n/g; 
+
+    # split numbers after four digits
+    s/(\d\d\d\d)/$1 /g;
+
+    # split words after 32 characters
+
+    # squash repeated punctuation 
+    tr/A-Za-z0-9 //cs;
+
+    # save email addresses
+    # s/\w+@\w+\.[\w\.]+/EMAIL/g;
+
+    # normalise clause breaks (mostly punctuation symbols) to commas
+    s/[^A-Za-z0-9 \n]+/ , /g;
+
+    # Remove repeated commas, and replace with newline
+    s/\s*,[, ]+/\n/g;
+
+    # remove extra whitespace
+    s/ +/ /sg;
+    s/^\s+//mg;
+    s/\s*$/\n/mg;
+
+    # remove lines that contain one word or less
+    s/^\w*$//mg;
+    s/^\s*$//mg;
+    tr/\n//s;
+
+    return $_;
+}
+
+
+# Execute a system command
+
+sub execute {
+    my ($command, $verbosity) = @_;
+    print "Executing: $command\n" if $verbosity;
+    my $status = system($command);
+    if ($status != 0) {
+	print STDERR "phindgen.pl - Error executing $command: $!\n";
+	exit($status);
+    }
+}
+
+
+# Generate the vocabulary, symbol statistics, and numbers file from the
+# clauses file.  This is legacy code, so is a bit messy and probably wont
+# run under windows.
+
+sub extract_vocabulary {
+    my ($phindex_dir, $language, $verbosity) = @_;
+
+    my ($w, $l, $line, $word);
+
+    my ($first_delimiter, $last_delimiter,
+	$first_stopword, $last_stopword,
+	$first_extractword, $last_extractword,
+	$first_contentword, $last_contentword,
+	$phrasedelimiter);
+
+    my ($use_thesaurus, %thesaurus, $first_thesaurusword, $last_thesaurusword);
+
+
+    my %symbol;
+    my (%freq);
+
+    print "Calculating vocabulary\n" if ($verbosity > 1);
+
+    # Read and store the stopwords
+    my $words = `find $ENV{'GSDLHOME'}/etc/phind/$language -name "*.sw" | xargs cat`;
+    my %stopwords;
+    foreach my $w (split(/\s+/, $words)) {
+	$l = lc($w);
+	$stopwords{$l} = $w;
+    }
+    
+    # Read and store the thesaurus terms
+    $use_thesaurus = 0;
+    my $lex_file = &util::filename_cat("$ENV{'GSDLHOME'}", "etc", "phind", 
+				       "$language", "agrovoc.lex");
+    if (-e "$lex_file") {
+	open(TH, "<$lex_file");
+	while(<TH>) {
+	    s/^\d+ //;
+	    s/\(.*\)//;
+	    foreach my $w (split(/\s+/, $_)) {
+		$thesaurus{lc($w)} = $w;
+	    }
+	}
+	close TH;
+	$use_thesaurus = 1;
+    }
+
+    # Read words in the text and count occurences
+    open(TXT, "<$phindex_dir/clauses");
+    my @words;
+    
+    while(<TXT>) {
+	$line = $_;
+	next unless ($line =~ /./);
+	
+	@words = split(/\s+/, $line);
+	foreach $w (@words) {
+	    $l = lc($w);
+	    $w = $l if ((defined $stopwords{$l}) || (defined $thesaurus{$l}));
+	    $freq{$w}++;
+	}
+	$freq{$senlimit}++;
+    }
+    
+    # Calculate the "best" form of each word
+    my (%bestform, %totalfreq, %bestfreq);
+
+    foreach $w (sort (keys %freq)) {
+	$l = lc($w);
+	
+	# totalfreq is the number of times a term appears in any form
+	$totalfreq{$l} += $freq{$w};
+	
+	if (defined $stopwords{$l}) {
+	    $bestform{$l} = $stopwords{$l};
+	    
+	} elsif (defined $thesaurus{$l}) {
+	    $bestform{$l} = $thesaurus{$l};
+	    
+	} elsif (!$bestform{$l} || ($freq{$w} > $bestfreq{$l})) {
+	    $bestfreq{$l} = $freq{$w};
+	    $bestform{$l} = $w;
+	}
+    }
+    
+    undef %freq;
+    undef %bestfreq;
+    
+
+    # Assign symbol numbers to tokens
+    my $nextsymbol = 1;
+    my (@vocab);
+    
+    # Delimiters
+    $first_delimiter = 1;
+    
+    foreach $word (@delimiters) {
+
+	$word = lc($word);
+	$bestform{$word} = uc($word);
+	$vocab[$nextsymbol] = $word;
+	$symbol{$word} = $nextsymbol;
+	$nextsymbol++;
+    }
+    $last_delimiter = $nextsymbol - 1;
+    
+    # Stopwords
+    $first_stopword = $nextsymbol;
+    
+    foreach my $word (sort keys %stopwords) {
+	
+	# don't incluse stopword unless it occurs in the text
+	$word = lc($word);
+	next unless ($totalfreq{$word});
+	next if ($symbol{$word});
+	
+	$vocab[$nextsymbol] = $word;
+	$symbol{$word} = $nextsymbol;
+	$nextsymbol++;
+    }
+    $last_stopword = $nextsymbol - 1;
+    $first_contentword = $nextsymbol;
+    
+    # Thesaurus terms
+    if ($use_thesaurus) {
+	$first_thesaurusword = $nextsymbol;
+    
+	foreach my $word (sort keys %thesaurus) {
+	    
+	    $word = lc($word);
+	    next if ($symbol{$word});
+	    $bestform{$word} = $thesaurus{$word};
+	    
+	    $vocab[$nextsymbol] = $word;
+	    $symbol{$word} = $nextsymbol;
+	    $nextsymbol++;
+	    
+	}
+	$last_thesaurusword = $nextsymbol - 1;
+    }
+
+    # Other content words
+    $first_extractword = $nextsymbol;
+    
+    foreach my $word (sort (keys %bestform)) {
+	
+	next if ($symbol{$word});
+	
+	$vocab[$nextsymbol] = $word;
+	$symbol{$word} = $nextsymbol;
+	$nextsymbol++;
+    }
+    $last_extractword = $nextsymbol - 1;
+    $last_contentword = $nextsymbol - 1;
+    
+
+    # Outut the words
+    print "Saving vocabulary in $phindex_dir/clauses.vocab\n" if ($verbosity > 1);
+    open(VOC, ">$phindex_dir/clauses.vocab");
+
+    for (my $i = 1; $i < $nextsymbol; $i++) {
+	$w = $vocab[$i];
+
+	print VOC "$bestform{$w}\n";
+	$totalfreq{$w} = 0 unless ($totalfreq{$w});
+    }
+    close VOC;
+
+
+    # Output statistics about the vocablary
+    print "Saving statistics in $phindex_dir/clauses.stats\n" if ($verbosity > 1);
+    &util::rm("$phindex_dir/clauses.stats") if (-e "$phindex_dir/clauses.stats");
+    open(STAT, ">$phindex_dir/clauses.stats")
+	|| die "Cannot open $phindex_dir/clauses.stats: $!";
+
+    print STAT "first_delimiter $first_delimiter\n";
+    print STAT "last_delimiter $last_delimiter\n";
+    print STAT "first_stopword $first_stopword\n";
+    print STAT "last_stopword $last_stopword\n";
+    if ($use_thesaurus) {
+	print STAT "first_thesaurusword $first_thesaurusword\n";
+	print STAT "last_thesaurusword $last_thesaurusword\n";
+    }
+    print STAT "first_extractword $first_extractword\n";
+    print STAT "last_extractword $last_extractword\n";
+    print STAT "first_contentword $first_contentword\n";
+    print STAT "last_contentword $last_contentword\n";
+    print STAT "first_symbol $first_delimiter\n";
+    print STAT "last_symbol $last_contentword\n";
+    print STAT "first_word $first_stopword\n";
+    print STAT "last_word $last_contentword\n";
+    close STAT;
+
+    undef @vocab;
+
+
+    # Save text as symbol numbers
+    print "Saving text as numbers in $phindex_dir/clauses.numbers\n" if ($verbosity > 1);
+    
+    open(TXT, "<$phindex_dir/clauses");
+    open(NUM, ">$phindex_dir/clauses.numbers");
+    
+    $phrasedelimiter = $symbol{lc($senlimit)};
+    print NUM "$symbol{lc($colstart)}\n";
+    
+    # set up the special symbols that delimit documents and sentences
+    while(<TXT>) {
+	
+	# split sentence into a list of tokens
+	$line = $_;
+	next unless ($line =~ /./);
+	@words = split(/\s+/, $line);
+	
+	# output one token at a time
+	foreach $word (@words) {
+	    $word = lc($word);
+	    print NUM "$symbol{$word}\n";
+	}
+	
+	# output phrase delimiter
+	print NUM "$phrasedelimiter\n";
+    }
+    
+    print NUM "$symbol{lc($colend)}\n";
+
+}
+
+
+# Prepare the phrases file to be input to mgpp.
+# This means renumbering the phrases in order of decreasing frequency.
+
+
+# This is legacy code, and a little ugly, and may be unix-specific
+# (particularly the sort command).
+
+sub renumber_phrases {
+    my ($phindex_dir, $verbosity) = @_;
+
+    # Sort the phrases into order of increasing frequency
+    # This means the expansions will be sorted correctly later on.
+    print "Sorting phrases into freq order\n" if ($verbosity);
+    system("sort -rnt ':' +2 -o $phindex_dir/phrases $phindex_dir/phrases");
+
+    my @symbol;
+
+    # Read the vocabulary
+    print "Reading the vocabulary\n" if ($verbosity);
+    open(V, "<$phindex_dir/clauses.vocab")
+	|| die "Cannot open $phindex_dir/clauses.vocab: $!";
+    
+    my $i = 1;
+    while(<V>) {
+	chomp;
+	$symbol[$i++] = $_;
+    }
+
+    # Create file for phrase data
+    #
+    # The phrases file looks something like this
+    #  159396-1:s5175:4:1:116149-2:3:d2240,2;d2253;d2254
+    #  159409-1:s5263:6:1:159410-2:6:d2122;d2128;d2129;d2130;d2215;d2380
+    #  159415-1:s5267:9:1:159418-2:8:d3,2;d632;d633;d668;d1934;d2010;d2281;d2374
+    #  159426-1:s5273:5:2:159429-2,115168-17:5:d252;d815;d938;d939;d2361
+
+    # The first field on each line is a unique phrase identifier.  
+    # We need to calculate phrase numbers for each phrase
+    print "Calculate phrase numbers\n" if ($verbosity);
+    
+    my %phrasenumber;
+    my $nextphrase = 1;
+    my ($line);
+    
+    open(IN, "<$phindex_dir/phrases");
+    while(<IN>) {
+	
+	# read the line
+	chomp;
+	$line = $_;
+	
+	# we're only interested in the first field
+	$line =~ s/:.*//;
+	
+	# get a phrase number for this line
+	$phrasenumber{$line} = $nextphrase;
+	$nextphrase++;
+    }
+    
+
+    # Now we create a new phrase file using phrase numbers, not the old IDs.
+    print "Format phrase data for MGPP\n" if ($verbosity);
+    
+    open(IN, "<$phindex_dir/phrases");
+    open(DATA, ">$phindex_dir/pdata.txt");
+    open(IDX, ">$phindex_dir/pword.txt");
+    
+    my ($key, $tf, $num, $countexp, $expansions, $countdocs, $documents, $text, $word);
+    my @fields;
+    my @documents;
+    my (@newexp, $k, $n);
+    
+    my $linenumber = 0;
+
+    while(<IN>) {
+	
+	# read the line
+	chomp;
+	$line = $_;
+	@fields = split(/:/, $line);
+	
+	# get a phrase number for this line
+	$key = shift @fields;
+	die unless (defined($phrasenumber{$key}));
+	$num = $phrasenumber{$key};
+	
+	# get the text of the phrase
+	$text = shift @fields;
+	$text =~ s/s(\d+)/$symbol[$1]/g;
+	if ($text =~ / /) {
+	    $word = "";
+	} else {
+	    $word = $text;
+	}
+
+	$linenumber++;
+	if ($linenumber % 1000 == 0) {
+	    print "line $linenumber:\t$num\t$key\t($text)\n"  if ($verbosity > 2);
+	}
+	print "$num: $key\t($text)\n" if ($verbosity > 3);
+	
+	# get the phrase frequency
+	$tf = shift @fields;
+	
+	# get the number of expansions
+	$countexp = shift @fields;
+	
+	# get the expansions and convert them into phrase numbers
+	$expansions = shift @fields;
+	@newexp = ();
+	foreach $k (split(/,/, $expansions)) {
+	    die "ERROR - no phrase number for: $k" unless (defined($phrasenumber{$k}));
+	    $n = $phrasenumber{$k};
+	    push @newexp, $n;
+	}
+	@newexp = sort numerically @newexp;
+
+	# get the number of documents
+	$countdocs = shift @fields;
+	
+	# get the documents
+	$documents = shift @fields;
+	$documents =~ s/d//g;
+	@documents = split(/;/, $documents);
+	@documents = sort by_frequency @documents;
+
+	# output the phrase data
+	print DATA "<Document>";
+ 	print DATA "$num:$text:$tf:$countexp:$countdocs:";
+	print DATA join(",", @newexp), ":", join(";", @documents), "\n";
+	
+	# output the word index search data
+	print IDX "<Document>$word\n";
+
+	
+    }
+}
+
+# sort routines used to renumber phrases
+
+sub numerically { $a <=> $b }
+
+sub by_frequency { 
+    my $fa = 1;
+    if ($a =~ /,/) {
+	$fa = $a;
+	$fa =~ s/\d+,//;
+    }
+    my $fb = 1;
+    if ($b =~ /,/) {
+	$fb = $b;
+	$fb =~ s/\d+,//;
+    }
+
+    return ($fb <=> $fa);
+}
+
+
+1;
Index: trunk/gsdl/src/phind/generate/Makefile.in
===================================================================
--- trunk/gsdl/src/phind/generate/Makefile.in	(revision 1642)
+++ trunk/gsdl/src/phind/generate/Makefile.in	(revision 1643)
@@ -54,7 +54,4 @@
 EXEC    = suffix
 
-PERL_SCRIPTS = phindgen.pl
-PERL_MODULES = phproc.pm
-
 LEXICON_ENGLISH = lexicons/english/brown.sw lexicons/english/fao.sw lexicons/english/punctuation.sw
 
@@ -69,8 +66,4 @@
 	install -d $(GSDLHOME)/bin/$(GSDLOS)
 	install -m 775 -s $(EXEC) $(GSDLHOME)/bin/$(GSDLOS)
-	install -d $(GSDLHOME)/bin/script
-	install -m 775 $(PERL_SCRIPTS) $(GSDLHOME)/bin/script
-	install -d $(GSDLHOME)/perllib
-	install -m 664 $(PERL_MODULES) $(GSDLHOME)/perllib
 	install -d $(GSDLHOME)/etc/phind/english
 	install -m 664 $(LEXICON_ENGLISH) $(GSDLHOME)/etc/phind/english
Index: trunk/gsdl/src/phind/generate/phindgen.pl
===================================================================
--- trunk/gsdl/src/phind/generate/phindgen.pl	(revision 1642)
+++ 	(revision )
@@ -1,643 +1,0 @@
-#! /usr/bin/perl -w
-
-###########################################################################
-#
-# phindgen.pl -- produce Phind index files for a GSDL collection.
-#
-# Copyright 2000 Gordon W. Paynter
-# Copyright 2000 The New Zealand Digital Library Project
-#
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-
-# This program generates phind phrase browsing indexes for a Greenstone
-# digital library collection.  
-#
-# The GML files that have been imported to the archives directory are 
-# read, and then the phind indexes are created in the phindex directory.
-#
-# This version od Phind uses suffix arrays to extract phrases.
-
-
-BEGIN {
-    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
-    die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'};
-    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
-    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/plugins");
-}
-
-
-use strict;
-use colcfg;
-use util;
-use parsargv;
-use plugin;
-use phproc;
-
-
-sub print_usage {
-    print STDERR "\n  usage: $0 [options] collection-name\n\n";
-    print STDERR "  options:\n";
-    print STDERR "   -verbosity number      0=none, 3=lots\n";
-    print STDERR "   -archivedir directory  Where the converted material ends up\n";
-    print STDERR "   -phindexdir directory  Where to place the phind indexes\n";
-    print STDERR "   -languages names       The languages to consider in the browser\n";
-    print STDERR "   -maxdocs number        Maximum number of documents to process\n";
-    print STDERR "   -untidy                Leave working files in phindex directory\n";
-    print STDERR "\n";
-}
-
-
-# Define delimiter symbols - this should be abstracted out someplace
-my $colstart = "COLLECTIONSTART";
-my $colend   = "COLLECTIONEND";
-my $doclimit = "DOCUMENTLIMIT";
-my $senlimit = "SENTENCELIMIT";
-my @delimiters = ($colstart, $colend, $doclimit, $senlimit);
-
-&main ();
-
-
-sub main {
-    
-    my ($verbosity, $archivedir, $phindexdir, 
-	$phindcfg, $language, $maxdocs, $untidy,
-	$collection, $configfilename, $collectcfg);
-    
-    # Parse command-line arguments and get the collection name
-    if (!parsargv::parse(\@ARGV, 
-			 'verbosity/\d+/1', \$verbosity,
-			 'archivedir/.*/', \$archivedir,
-			 'phindexdir/.*/', \$phindexdir,
-			 'languages/\w+/english', \$language,
-			 'maxdocs/^\-?\d+/-1', \$maxdocs,
-			 'untidy', \$untidy )) {
-	&print_usage();
-	die "\n";
-    }
-    $collection = &util::use_collection(@ARGV);
-    if ($collection eq "") {
-	&print_usage();
-	die "\n";
-    }
-
-    # Read the collection configuration file
-    $configfilename = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, "etc", "collect.cfg");
-    if (-e $configfilename) {
-	$collectcfg = &colcfg::read_collect_cfg ($configfilename);
-	if (defined $collectcfg->{'archivedir'} && $archivedir eq "") {
-	    $archivedir = $collectcfg->{'archivedir'};
-	}
-	if (defined $collectcfg->{'phindexdir'} && $phindexdir eq "") {
-	    $phindexdir = $collectcfg->{'phindexdir'};
-	}
-	if (defined $collectcfg->{'phind'}) {
-	    $phindcfg = $collectcfg->{'phind'};
-	}
-    } else {
-	die "Couldn't find the configuration file $configfilename\n";
-    }
-
-    # Calculate default archive and phindex directories
-    if ($archivedir eq "") {
-	$archivedir = &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, "archives");
-    }
-    if ($phindexdir eq "") {
-	$phindexdir = &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, "phindex");
-    }
-
-    # Make sure theuser has in fact requested phind indexes
-    if (!defined($phindcfg)) {
-	print "No phind information in $configfilename\n";
-	exit;
-    }
-
-
-    # Construct phind indexes
-    my $mode = 1;
-    my $symbol_limit = 40000000;
-    my ($command, $status);
-    
-    # Make a clean phindex directory
-    if (-e "$phindexdir") {
-	&util::rm_r("$phindexdir");
-    }
-    &util::mk_dir("$phindexdir");
-
-    # Read the archives directory and build the clauses file
-    print "\nReading archive directory\n" if $verbosity;
-    &build_clauses($archivedir, $phindexdir, $language, $phindcfg, $verbosity, $maxdocs);
-
-    # Generate the vocabulary, symbol statistics, and numbers file
-    # from the clauses file
-    print "\nExtracting vocabulary and statistics\n" if $verbosity;
-    &extract_vocabulary($phindexdir, $language, $verbosity);
-
-    # Use the suffix program to generate the phindex/phrases file
-    &execute("suffix $phindexdir $symbol_limit $mode", $verbosity);
-
-    # Create the phrase file and put phrase numbers in phindex/phrases
-    print "\nSorting and Renumbering phrases for input to mgpp\n" if $verbosity;
-    &renumber_phrases("$phindexdir", $verbosity);
-    
-    # Create the mg phrase database
-    my $mgpp = &util::filename_cat($ENV{'GSDLHOME'}, "src", "mgpp");
-    my $mg_passes = &util::filename_cat($mgpp, "text", "mg_passes");
-    my $mg_compression_dict = &util::filename_cat($mgpp, "text", "mg_compression_dict");
-
-    my $mg_perf_hash_build = &util::filename_cat($mgpp, "text", "mg_perf_hash_build");
-    my $mg_weights_build = &util::filename_cat($mgpp, "text", "mg_weights_build");
-    my $mg_invf_dict = &util::filename_cat($mgpp, "text", "mg_invf_dict");
-    my $mg_stem_idx = &util::filename_cat($mgpp, "text", "mg_stem_idx");
-
-    print "\nCreating phrase databases\n";
-    my $mg_input = &util::filename_cat($phindexdir, "pdata.txt");
-    my $mg_stem = "pdata";
-
-    &execute("$mg_passes -d $phindexdir -f $mg_stem -T1 $mg_input", $verbosity);
-    &execute("$mg_compression_dict -d $phindexdir -f $mg_stem", $verbosity);
-    &execute("$mg_passes -d $phindexdir -f $mg_stem -T2 $mg_input", $verbosity);
-
-    # create the mg index of words
-    print "\nCreating word-level search indexes\n";
-    $mg_input = &util::filename_cat($phindexdir, "pword.txt");
-    $mg_stem = "pword";
-
-    &execute("$mg_passes -d $phindexdir -f $mg_stem -T1 -I1 $mg_input", $verbosity);
-    &execute("$mg_compression_dict -d $phindexdir -f $mg_stem", $verbosity);
-    &execute("$mg_perf_hash_build -d $phindexdir -f $mg_stem", $verbosity);
-    &execute("$mg_passes -d $phindexdir -f $mg_stem -T2 -I2 $mg_input", $verbosity);
-    &execute("$mg_weights_build -d $phindexdir -f $mg_stem", $verbosity);
-    &execute("$mg_invf_dict -d $phindexdir -f $mg_stem", $verbosity);
-
-    &execute("$mg_stem_idx -d $phindexdir -f $mg_stem -s 1", $verbosity);
-    &execute("$mg_stem_idx -d $phindexdir -f $mg_stem -s 2", $verbosity);
-    &execute("$mg_stem_idx -d $phindexdir -f $mg_stem -s 3", $verbosity);
-
-    # create the mg document information database
-    print "\nCreating document information databases\n";
-    $mg_input = &util::filename_cat($phindexdir, "docs.txt");
-    $mg_stem = "docs";
-
-    &execute("$mg_passes -d $phindexdir -f $mg_stem -T1 $mg_input", $verbosity);
-    &execute("$mg_compression_dict -d $phindexdir -f $mg_stem", $verbosity);
-    &execute("$mg_passes -d $phindexdir -f $mg_stem -T2 $mg_input", $verbosity);
-
-
-    # Tidy up stray files
-    if (!$untidy) {
-	print "\nCleaning up\n" if $verbosity;
-	&util::rm("$phindexdir/clauses", "$phindexdir/clauses.numbers", 
-		  "$phindexdir/clauses.vocab", "$phindexdir/clauses.stats", 
-		  "$phindexdir/phrases", "$phindexdir/docs.txt",
-		  "$phindexdir/pdata.txt", "$phindexdir/pword.txt");
-	my $outfile = 1;
-	while (-e "$phindexdir/outPhrase.$outfile") {
-	    &util::rm("$phindexdir/outPhrase.$outfile");
-	    $outfile++;
-	}
-    }
-}
-
-# Execute a system command
-
-sub execute {
-    my ($command, $verbosity) = @_;
-    print "Executing: $command\n" if $verbosity;
-    my $status = system($command);
-    if ($status != 0) {
-	print STDERR "phindgen.pl - Error executing $command: $!\n";
-	exit($status);
-    }
-}
-
-
-
-
-
-# Get the text for the collection.  
-# Save the document names and identifiers as we go.
-
-sub build_clauses {
-    my ($archive_dir, $phindex_dir, $language, $phindcfg, $verbosity, $maxdocs) = @_;
-
-    # create a "pluginfo" for ArcPlug and RecPlug
-    my $plugins = [["GMLPlug"], ["ArcPlug"]];
-    my $pluginfo = &plugin::load_plugins ($plugins, $verbosity);
-    die "No plugins were loaded." if (scalar(@$pluginfo) == 0);
-    
-    
-    # create a phind document processor object to process the documents
-    my $processor = new phproc ($archive_dir, $phindex_dir, $phindcfg, $language,
-				$doclimit, $verbosity, "STDOUT");
-
-    # process the archives directory
-    &plugin::read ($pluginfo, $archive_dir, "", {}, $processor, $maxdocs);
-}
-
-
-
-# Generate the vocabulary, symbol statistics, and numbers file from the
-# clauses file.  This is legacy code, so is a bit messy and probably wont
-# run under windows.
-
-sub extract_vocabulary {
-    my ($phindex_dir, $language, $verbosity) = @_;
-
-    my ($w, $l, $line, $word);
-
-    my ($first_delimiter, $last_delimiter,
-	$first_stopword, $last_stopword,
-	$first_extractword, $last_extractword,
-	$first_contentword, $last_contentword,
-	$phrasedelimiter);
-
-    my ($use_thesaurus, %thesaurus, $first_thesaurusword, $last_thesaurusword);
-
-
-    my %symbol;
-    my (%freq);
-
-    print "Calculating vocabulary\n" if ($verbosity > 1);
-
-    # Read and store the stopwords
-    my $words = `find $ENV{'GSDLHOME'}/etc/phind/$language -name "*.sw" | xargs cat`;
-    my %stopwords;
-    foreach my $w (split(/\s+/, $words)) {
-	$l = lc($w);
-	$stopwords{$l} = $w;
-    }
-    
-    # Read and store the thesaurus terms
-    $use_thesaurus = 0;
-    my $lex_file = &util::filename_cat("$ENV{'GSDLHOME'}", "etc", "phind", 
-				       "$language", "agrovoc.lex");
-    if (-e "$lex_file") {
-	open(TH, "<$lex_file");
-	while(<TH>) {
-	    s/^\d+ //;
-	    s/\(.*\)//;
-	    foreach my $w (split(/\s+/, $_)) {
-		$thesaurus{lc($w)} = $w;
-	    }
-	}
-	close TH;
-	$use_thesaurus = 1;
-    }
-
-    # Read words in the text and count occurences
-    open(TXT, "<$phindex_dir/clauses");
-    my @words;
-    
-    while(<TXT>) {
-	$line = $_;
-	next unless ($line =~ /./);
-	
-	@words = split(/\s+/, $line);
-	foreach $w (@words) {
-	    $l = lc($w);
-	    $w = $l if ((defined $stopwords{$l}) || (defined $thesaurus{$l}));
-	    $freq{$w}++;
-	}
-	$freq{$senlimit}++;
-    }
-    
-    # Calculate the "best" form of each word
-    my (%bestform, %totalfreq, %bestfreq);
-
-    foreach $w (sort (keys %freq)) {
-	$l = lc($w);
-	
-	# totalfreq is the number of times a term appears in any form
-	$totalfreq{$l} += $freq{$w};
-	
-	if (defined $stopwords{$l}) {
-	    $bestform{$l} = $stopwords{$l};
-	    
-	} elsif (defined $thesaurus{$l}) {
-	    $bestform{$l} = $thesaurus{$l};
-	    
-	} elsif (!$bestform{$l} || ($freq{$w} > $bestfreq{$l})) {
-	    $bestfreq{$l} = $freq{$w};
-	    $bestform{$l} = $w;
-	}
-    }
-    
-    undef %freq;
-    undef %bestfreq;
-    
-
-    # Assign symbol numbers to tokens
-    my $nextsymbol = 1;
-    my (@vocab);
-    
-    # Delimiters
-    $first_delimiter = 1;
-    
-    foreach $word (@delimiters) {
-
-	$word = lc($word);
-	$bestform{$word} = uc($word);
-	$vocab[$nextsymbol] = $word;
-	$symbol{$word} = $nextsymbol;
-	$nextsymbol++;
-    }
-    $last_delimiter = $nextsymbol - 1;
-    
-    # Stopwords
-    $first_stopword = $nextsymbol;
-    
-    foreach my $word (sort keys %stopwords) {
-	
-	# don't incluse stopword unless it occurs in the text
-	$word = lc($word);
-	next unless ($totalfreq{$word});
-	next if ($symbol{$word});
-	
-	$vocab[$nextsymbol] = $word;
-	$symbol{$word} = $nextsymbol;
-	$nextsymbol++;
-    }
-    $last_stopword = $nextsymbol - 1;
-    $first_contentword = $nextsymbol;
-    
-    # Thesaurus terms
-    if ($use_thesaurus) {
-	$first_thesaurusword = $nextsymbol;
-    
-	foreach my $word (sort keys %thesaurus) {
-	    
-	    $word = lc($word);
-	    next if ($symbol{$word});
-	    $bestform{$word} = $thesaurus{$word};
-	    
-	    $vocab[$nextsymbol] = $word;
-	    $symbol{$word} = $nextsymbol;
-	    $nextsymbol++;
-	    
-	}
-	$last_thesaurusword = $nextsymbol - 1;
-    }
-
-    # Other content words
-    $first_extractword = $nextsymbol;
-    
-    foreach my $word (sort (keys %bestform)) {
-	
-	next if ($symbol{$word});
-	
-	$vocab[$nextsymbol] = $word;
-	$symbol{$word} = $nextsymbol;
-	$nextsymbol++;
-    }
-    $last_extractword = $nextsymbol - 1;
-    $last_contentword = $nextsymbol - 1;
-    
-
-    # Outut the words
-    print "Saving vocabulary in $phindex_dir/clauses.vocab\n" if ($verbosity > 1);
-    open(VOC, ">$phindex_dir/clauses.vocab");
-
-    for (my $i = 1; $i < $nextsymbol; $i++) {
-	$w = $vocab[$i];
-
-	print VOC "$bestform{$w}\n";
-	$totalfreq{$w} = 0 unless ($totalfreq{$w});
-    }
-    close VOC;
-
-
-    # Output statistics about the vocablary
-    print "Saving statistics in $phindex_dir/clauses.stats\n" if ($verbosity > 1);
-    &util::rm("$phindex_dir/clauses.stats") if (-e "$phindex_dir/clauses.stats");
-    open(STAT, ">$phindex_dir/clauses.stats")
-	|| die "Cannot open $phindex_dir/clauses.stats: $!";
-
-    print STAT "first_delimiter $first_delimiter\n";
-    print STAT "last_delimiter $last_delimiter\n";
-    print STAT "first_stopword $first_stopword\n";
-    print STAT "last_stopword $last_stopword\n";
-    if ($use_thesaurus) {
-	print STAT "first_thesaurusword $first_thesaurusword\n";
-	print STAT "last_thesaurusword $last_thesaurusword\n";
-    }
-    print STAT "first_extractword $first_extractword\n";
-    print STAT "last_extractword $last_extractword\n";
-    print STAT "first_contentword $first_contentword\n";
-    print STAT "last_contentword $last_contentword\n";
-    print STAT "first_symbol $first_delimiter\n";
-    print STAT "last_symbol $last_contentword\n";
-    print STAT "first_word $first_stopword\n";
-    print STAT "last_word $last_contentword\n";
-    close STAT;
-
-    undef @vocab;
-
-
-    # Save text as symbol numbers
-    print "Saving text as numbers in $phindex_dir/clauses.numbers\n" if ($verbosity > 1);
-    
-    open(TXT, "<$phindex_dir/clauses");
-    open(NUM, ">$phindex_dir/clauses.numbers");
-    
-    $phrasedelimiter = $symbol{lc($senlimit)};
-    print NUM "$symbol{lc($colstart)}\n";
-    
-    # set up the special symbols that delimit documents and sentences
-    while(<TXT>) {
-	
-	# split sentence into a list of tokens
-	$line = $_;
-	next unless ($line =~ /./);
-	@words = split(/\s+/, $line);
-	
-	# output one token at a time
-	foreach $word (@words) {
-	    $word = lc($word);
-	    print NUM "$symbol{$word}\n";
-	}
-	
-	# output phrase delimiter
-	print NUM "$phrasedelimiter\n";
-    }
-    
-    print NUM "$symbol{lc($colend)}\n";
-
-}
-
-
-# Prepare the phrases file to be input to mgpp.
-# This means renumbering the phrases in order of decreasing frequency.
-
-
-# This is legacy code, and a little ugly, and may be unix-specific
-# (particularly the sort command).
-
-sub renumber_phrases {
-    my ($phindex_dir, $verbosity) = @_;
-
-    # Sort the phrases into order of increasing frequency
-    # This means the expansions will be sorted correctly later on.
-    print "Sorting phrases into freq order\n" if ($verbosity);
-    system("sort -rnt ':' +2 -o $phindex_dir/phrases $phindex_dir/phrases");
-
-    my @symbol;
-
-    # Read the vocabulary
-    print "Reading the vocabulary\n" if ($verbosity);
-    open(V, "<$phindex_dir/clauses.vocab")
-	|| die "Cannot open $phindex_dir/clauses.vocab: $!";
-    
-    my $i = 1;
-    while(<V>) {
-	chomp;
-	$symbol[$i++] = $_;
-    }
-
-    # Create file for phrase data
-    #
-    # The phrases file looks something like this
-    #  159396-1:s5175:4:1:116149-2:3:d2240,2;d2253;d2254
-    #  159409-1:s5263:6:1:159410-2:6:d2122;d2128;d2129;d2130;d2215;d2380
-    #  159415-1:s5267:9:1:159418-2:8:d3,2;d632;d633;d668;d1934;d2010;d2281;d2374
-    #  159426-1:s5273:5:2:159429-2,115168-17:5:d252;d815;d938;d939;d2361
-
-    # The first field on each line is a unique phrase identifier.  
-    # We need to calculate phrase numbers for each phrase
-    print "Calculate phrase numbers\n" if ($verbosity);
-    
-    my %phrasenumber;
-    my $nextphrase = 1;
-    my ($line);
-    
-    open(IN, "<$phindex_dir/phrases");
-    while(<IN>) {
-	
-	# read the line
-	chomp;
-	$line = $_;
-	
-	# we're only interested in the first field
-	$line =~ s/:.*//;
-	
-	# get a phrase number for this line
-	$phrasenumber{$line} = $nextphrase;
-	$nextphrase++;
-    }
-    
-
-    # Now we create a new phrase file using phrase numbers, not the old IDs.
-    print "Format phrase data for MGPP\n" if ($verbosity);
-    
-    open(IN, "<$phindex_dir/phrases");
-    open(DATA, ">$phindex_dir/pdata.txt");
-    open(IDX, ">$phindex_dir/pword.txt");
-    
-    my ($key, $tf, $num, $countexp, $expansions, $countdocs, $documents, $text, $word);
-    my @fields;
-    my @documents;
-    my (@newexp, $k, $n);
-    
-    my $linenumber = 0;
-
-    while(<IN>) {
-	
-	# read the line
-	chomp;
-	$line = $_;
-	@fields = split(/:/, $line);
-	
-	# get a phrase number for this line
-	$key = shift @fields;
-	die unless (defined($phrasenumber{$key}));
-	$num = $phrasenumber{$key};
-	
-	# get the text of the phrase
-	$text = shift @fields;
-	$text =~ s/s(\d+)/$symbol[$1]/g;
-	if ($text =~ / /) {
-	    $word = "";
-	} else {
-	    $word = $text;
-	}
-
-	$linenumber++;
-	if ($linenumber % 1000 == 0) {
-	    print "line $linenumber:\t$num\t$key\t($text)\n"  if ($verbosity > 2);
-	}
-	print "$num: $key\t($text)\n" if ($verbosity > 3);
-	
-	# get the phrase frequency
-	$tf = shift @fields;
-	
-	# get the number of expansions
-	$countexp = shift @fields;
-	
-	# get the expansions and convert them into phrase numbers
-	$expansions = shift @fields;
-	@newexp = ();
-	foreach $k (split(/,/, $expansions)) {
-	    die "ERROR - no phrase number for: $k" unless (defined($phrasenumber{$k}));
-	    $n = $phrasenumber{$k};
-	    push @newexp, $n;
-	}
-	@newexp = sort numerically @newexp;
-
-	# get the number of documents
-	$countdocs = shift @fields;
-	
-	# get the documents
-	$documents = shift @fields;
-	$documents =~ s/d//g;
-	@documents = split(/;/, $documents);
-	@documents = sort by_frequency @documents;
-
-	# output the phrase data
-	print DATA "<Document>";
- 	print DATA "$num:$text:$tf:$countexp:$countdocs:";
-	print DATA join(",", @newexp), ":", join(";", @documents), "\n";
-	
-	# output the word index search data
-	print IDX "<Document>$word\n";
-
-	
-    }
-}
-
-# sort routines used to renumber phrases
-
-sub numerically { $a <=> $b }
-
-sub by_frequency { 
-    my $fa = 1;
-    if ($a =~ /,/) {
-	$fa = $a;
-	$fa =~ s/\d+,//;
-    }
-    my $fb = 1;
-    if ($b =~ /,/) {
-	$fb = $b;
-	$fb =~ s/\d+,//;
-    }
-
-    return ($fb <=> $fa);
-}
-
Index: trunk/gsdl/src/phind/generate/phproc.pm
===================================================================
--- trunk/gsdl/src/phind/generate/phproc.pm	(revision 1642)
+++ 	(revision )
@@ -1,258 +1,0 @@
-###########################################################################
-#
-# phproc.pm -- the Phind document processor
-#
-# Copyright (C) 2000 Gordon W. Paynter
-# Copyright 2000 The New Zealand Digital Library Project
-#
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-# This is the phind document processor object.  It is used by the document
-# reader plugins to extract the clauses from each document.
-
-package phproc;
-
-use docproc;
-use util;
-
-sub BEGIN {
-    @ISA = ('docproc');
-}
-
-sub new {
-    my ($class, $archive_dir, $phindex_dir, $phindcfg,
-	$language, $delimiter, $verbosity, $outhandle) = @_;
-
-    my $self = new docproc ();
-    
-    $self->{'archive_dir'} = $archive_dir;
-    $self->{'phindex_dir'} = $phindex_dir;
-    $self->{'indexes'} = $phindcfg;
-
-    $language =~ s/,/\|/g;
-    $self->{'language_exp'} = $language;
-    $self->{'delimiter'} = $delimiter;
-
-    $self->{'verbosity'} = $verbosity;
-    $self->{'outhandle'} = STDERR;
-    $self->{'outhandle'} = $outhandle if defined $outhandle;
-    
-    &util::rm("$phindex_dir/clauses") if (-e "$phindex_dir/clauses");
-    open(TEXT, ">$phindex_dir/clauses") 
-	|| die "Cannot open $phindex_dir/clauses: $!";
-    $self->{'txthandle'} = TEXT;
-
-    my $docfile = &util::filename_cat("$phindex_dir", "docs.txt");
-    &util::rm($docfile) if (-e $docfile);
-    open(DOCS, ">$docfile")
-	|| die "Cannot open $docfile: $!";
-    $self->{'dochandle'} = DOCS;
-
-   return bless $self, $class;
-
-}
-
-sub process {
-    my $self = shift (@_);
-    my ($doc_obj) = @_;
-
-    my $verbosity = $self->{'verbosity'};
-    my $top_section = $doc_obj->get_top_section();
-
-    my $title = $doc_obj->get_metadata_element ($top_section, "Title");
-    print "process: $title\n" if ($verbosity > 2);
-
-
-    # only consider english-language files
-    my $doclanguage = $doc_obj->get_metadata_element ($top_section, "Language");
-    my $phrlanguage = $self->{'language_exp'};
-    return if ($doclanguage && ($doclanguage !~ /$phrlanguage/i));
-    
-    # record this file
-    my $total++;
-    print "file $total: $file\n" if ($self->{'$verbosity'});
-    
-    # store object ID & title in document index file
-    my $OID = $doc_obj->get_OID();
-    $OID = "NULL" unless defined $OID;
-
-    my $dochandle = $self->{'dochandle'};
-    # print "dochandle: =$dochandle=\n";
-    print $dochandle "<Document>\t$OID\t$title\n";
-    
-    # Store the text of this object
-    my $indexlist = $self->{'indexes'};
-    my @parts;
-    my ($index, $part, $level, $field, $section, $data, $text);
-
-    # Output the document delimiter
-    my $txthandle = $self->{'txthandle'};
-    print $txthandle $self->{'delimiter'}, "\n";
-
-    # Iterarate over all the indexes specified in collect.cfg and
-    # add their text to the clauses file.
-    foreach $index (@$indexlist) {
-	$text = "";
-
-	# Iterate over all the feilds in each index
-	@parts = split(/,/, $index);
-	foreach $part (@parts) {
-
-	    # Each field has a level and a data element ((e.g. document:Title)
-	    ($level, $field) = split(/:/, $part);
-	    die unless ($level && $field);
-
-	    # Extract the text from every section
-	    # (In phind, document:text and section:text are equivalent)
-	    if ($field eq "text") {
-		$data = "";
-		$section = $doc_obj->get_top_section();
-		while (defined($section)) {
-		    $data .= $doc_obj->get_text($section) . "\n";
-		    $section = $doc_obj->get_next_section($section);
-		}
-		$text .= convert_gml_to_tokens($data) . "\n";
-	    }
-
-	    # Extract a metadata field from a document
-	    elsif ($level eq "document") {
-		$data = $doc_obj->get_metadata_element($doc_obj->get_top_section(), $field);
-		$text .= convert_gml_to_tokens($data) . "\n";
-	    } 
-	    
-	    # Extract metadata from every section in a document
-	    elsif ($level eq "section") {
-		$data = "";
-		$section = $doc_obj->get_top_section();
-		while (defined($section)) {
-		    $data .= $doc_obj->get_metadata_element($section, $field) . "\n";
-		    $section = $doc_obj->get_next_section($section);
-		}
-		$text .= convert_gml_to_tokens($data) . "\n";
-	    } 
-	    
-	    # Some sort of specification which I don't understand
-	    else {
-		die "Unknown level ($level) in phind key ($part) in phind index ($index)\n";
-	    }
-
-	}
-
-	# print the text
-	print $txthandle "$text";
-
-    }
-}
-
-
-
-sub convert_gml_to_tokens {
-    
-    $_ = shift @_;
-
-    # FIRST, remove GML tags
-
-    # Replace all whitespace with a simple space
-    s/\s+/ /gs;
-
-    # Remove everything that is in a tag
-    s/\s*<p>\s*/ PARAGRAPHBREAK /isg;
-    s/\s*<br>\s*/ LINEBREAK /isg;
-    s/<[^>]*>/ /sg;
-
-    # Now we have the text, but it may contain HTML 
-    # elements coded as &gt; etc.  Remove these tags. 
-    s/&lt;/</sg;
-    s/&gt;/>/sg;
-
-    s/\s+/ /sg;
-    s/\s*<p>\s*/ PARAGRAPHBREAK /isg;
-    s/\s*<br>\s*/ LINEBREAK /isg;
-    s/<[^>]*>/ /sg;
-
-    # remove &amp; and other miscellaneous markup tags
-    s/&amp;/&/sg;
-    s/&lt;/</sg;
-    s/&gt;/>/sg;
-    s/&amp;/&/sg;
-
-    # replace<p> and <br> placeholders with carriage returns
-    s/PARAGRAPHBREAK/\n/sg;
-    s/LINEBREAK/\n/sg;
-
-
-    # Exceptional punctuation
-    # 
-    # We make special cases of some punctuation
-
-    # remove any apostrophe that indicates omitted letters 
-    s/(\w+)\'(\w*\s)/ $1$2 /g;
-
-    # remove period that appears in a person's initals
-    s/\s([A-Z])\./ $1 /g;
-
-    # replace hyphens in hypheanted words and names with a space
-    s/([A-Za-z])-\s*([A-Za-z])/$1 $2/g;
-
-
-    # Convert the remaining text to "clause format",
-    # This means removing all excess punctuation and garbage text,
-    # normalising valid punctuation to fullstops and commas,
-    # then putting one cluse on each line.
-
-    # Insert newline when the end of a sentence is detected
-    # (delimter is:  "[\.\?\!]\s")
-    s/\s*[\.\?\!]\s+/\n/g; 
-
-    # split numbers after four digits
-    s/(\d\d\d\d)/$1 /g;
-
-    # split words after 32 characters
-
-    # squash repeated punctuation 
-    tr/A-Za-z0-9 //cs;
-
-    # save email addresses
-    # s/\w+@\w+\.[\w\.]+/EMAIL/g;
-
-    # normalise clause breaks (mostly punctuation symbols) to commas
-    s/[^A-Za-z0-9 \n]+/ , /g;
-
-    # Remove repeated commas, and replace with newline
-    s/\s*,[, ]+/\n/g;
-
-    # remove extra whitespace
-    s/ +/ /sg;
-    s/^\s+//mg;
-    s/\s*$/\n/mg;
-
-    # remove lines that contain one word or less
-    s/^\w*$//mg;
-    s/^\s*$//mg;
-    tr/\n//s;
-
-    return $_;
-}
-
-
-
-1;
-
Index: trunk/gsdl/src/recpt/Makefile.in
===================================================================
--- trunk/gsdl/src/recpt/Makefile.in	(revision 1642)
+++ trunk/gsdl/src/recpt/Makefile.in	(revision 1643)
@@ -78,5 +78,6 @@
                 userdb.h usersaction.h vlistbrowserclass.h hlistbrowserclass.h \
 		datelistbrowserclass.h invbrowserclass.h pagedbrowserclass.h \
-		htmlbrowserclass.h delhistoryaction.h historydb.h collectoraction.h
+		htmlbrowserclass.h phindbrowserclass.h \
+		delhistoryaction.h historydb.h collectoraction.h
 
 RECEPTHEADERS = 
@@ -95,5 +96,5 @@
 		vlistbrowserclass.cpp hlistbrowserclass.cpp datelistbrowserclass.cpp \
 		invbrowserclass.cpp pagedbrowserclass.cpp htmlbrowserclass.cpp \
-		delhistoryaction.cpp historydb.cpp collectoraction.cpp
+		phindbrowserclass.cpp delhistoryaction.cpp historydb.cpp collectoraction.cpp
 
 
@@ -114,6 +115,6 @@
                 infodbclass.o userdb.o usersaction.o vlistbrowserclass.o \
 		hlistbrowserclass.o datelistbrowserclass.o invbrowserclass.o \
-		pagedbrowserclass.o htmlbrowserclass.o delhistoryaction.o \
-		historydb.o parse.yy.o collectoraction.o
+		pagedbrowserclass.o htmlbrowserclass.o phindbrowserclass.o \
+		delhistoryaction.o historydb.o parse.yy.o collectoraction.o
 
 RECPTOBJECTS = recptmain.o
Index: trunk/gsdl/src/recpt/librarymain.cpp
===================================================================
--- trunk/gsdl/src/recpt/librarymain.cpp	(revision 1642)
+++ trunk/gsdl/src/recpt/librarymain.cpp	(revision 1643)
@@ -61,4 +61,5 @@
 #include "pagedbrowserclass.h"
 #include "htmlbrowserclass.h"
+#include "phindbrowserclass.h"
 
 #include "recptconfig.h"
@@ -297,4 +298,7 @@
   recpt.add_browser (&ahtmlbrowserclass);
 
+  phindbrowserclass aphindbrowserclass;
+  recpt.add_browser (&aphindbrowserclass);
+
   cgiwrapper (recpt, "");
   return 0;
Index: trunk/gsdl/src/recpt/phindbrowserclass.cpp
===================================================================
--- trunk/gsdl/src/recpt/phindbrowserclass.cpp	(revision 1643)
+++ trunk/gsdl/src/recpt/phindbrowserclass.cpp	(revision 1643)
@@ -0,0 +1,66 @@
+/**********************************************************************
+ *
+ * phindbrowserclass.cpp -- 
+ *
+ * Copyright (C) 2000 The New Zealand Digital Library Project
+ *
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+#include "phindbrowserclass.h"
+#include <assert.h>
+#include "OIDtools.h"
+
+phindbrowserclass::phindbrowserclass () {
+}
+
+phindbrowserclass::~phindbrowserclass () {
+}
+
+// returns the name that specifies the browserclass type
+text_t phindbrowserclass::get_browser_name () {
+  return "Phind";
+}
+
+// redirect to Phind web page
+// I don't really understand what I'm doing here.  Oh well.
+void phindbrowserclass::processOID (cgiargsclass &args, recptproto *collectproto,
+				   ostream &logout) {
+
+  // should only reach here from within a classification
+  if (!args["d"].empty()) return;
+  if (args["cl"].empty()) return;
+
+  FilterRequest_t request;
+  FilterResponse_t response;
+  comerror_t err;
+  request.filterName = "NullFilter";
+  request.filterResultOptions = FROID;
+  request.docSet.push_back (args["cl"] + ".fc");
+  collectproto->filter (args["c"], request, response, err, logout);
+
+  if (err != noError || response.docInfo[0].OID.empty()) return;
+
+  // redirect to html page
+  args["a"] = "p";
+  args["p"] = "phind";
+  args["hp"] = response.docInfo[0].OID;
+}
Index: trunk/gsdl/src/recpt/phindbrowserclass.h
===================================================================
--- trunk/gsdl/src/recpt/phindbrowserclass.h	(revision 1643)
+++ trunk/gsdl/src/recpt/phindbrowserclass.h	(revision 1643)
@@ -0,0 +1,51 @@
+/**********************************************************************
+ *
+ * phindbrowserclass.h -- 
+ *
+ * Copyright (C) 2000 The New Zealand Digital Library Project
+ *
+ *
+ * A component of the Greenstone digital library software
+ * from the New Zealand Digital Library Project at the
+ * University of Waikato, New Zealand.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ *********************************************************************/
+
+
+#ifndef PHINDBROWSERCLASS_H
+#define PHINDBROWSERCLASS_H
+
+#include "browserclass.h"
+
+class phindbrowserclass : public browserclass {
+protected:
+
+public:
+
+  phindbrowserclass ();
+  virtual ~phindbrowserclass ();
+
+  // returns the name that specifies the browserclass type
+  virtual text_t get_browser_name ();
+
+  void processOID (cgiargsclass &args, recptproto *collectproto,
+		   ostream &logout);
+
+
+};
+
+#endif
