Index: unk/gsdl/bin/script/gs2html.pl
===================================================================
--- /trunk/gsdl/bin/script/gs2html.pl	(revision 1444)
+++ 	(revision )
@@ -1,130 +1,0 @@
-#!/usr/bin/perl -w
-
-###########################################################################
-#
-# gs2html.pl -- convert various documents to HTML
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 1999 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-# Convert Microsoft Word, and Adobe PDF to HTML using the
-# appropriate specialist convert util
-
-
-BEGIN {
-    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
-    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
-
-}
-
-use parsargv;
-use util;
-use Cwd;
-use File::Basename;
-
-
-sub print_usage
-{
-
-    print STDERR "Usage: $0 [-t doc|pdf] filename\n";
-    exit(1);
-}
-
-
-sub main
-{
-    my (@ARGV) = @_;
-
-    my ($input_type,$verbose);
-
-    if (!parsargv::parse(\@ARGV,
-			 'type/(doc|pdf)/', \$input_type,
-			 'verbose/\d+/0',   \$verbose))
-    {
-	print_usage();
-    }
-
-    if (scalar(@ARGV!=1))
-    {
-	print_usage();
-    }
-    
-    my $input_filename = $ARGV[0];
-    if (!-r $input_filename)
-    {
-	print STDERR "Error: unable to open $input_filename for reading\n";
-	exit(1);
-    }
-
-    my ($tailname,$dirname,$suffix) 
-	= File::Basename::fileparse($input_filename,'\..+');
-    my $output_filename = &util::filename_cat($dirname,"$tailname.html");
-
-    if ($input_type eq "")
-    {
-	$input_type = substr($suffix,1,length($suffix)-1);
-    }
-    
-    # Change to temporary working directory
-    my $stored_dir = cwd();
-    chdir ($dirname) || die "Unable to change to directory $dirname";
-
-    
-    # Select convert utility
-    my $cmd = "";
-    if (!defined $input_type)
-    {
-	print STDERR "Error: No filename extension or input type defined\n";
-	exit(1);
-    }
-    elsif ($input_type eq "doc")
-    {
-	my $wv_cfgfile = &util::filename_cat($ENV{'GSDLHOME'},"etc","wvhtml.xml");
-	$cmd = "( echo Processing ; wvHtml --config $wv_cfgfile \"$input_filename\" 2>&1 > \"$output_filename\" )";
-	$cmd .= " | fgrep -v wvWarning" if ($verbose<3);
-	$cmd .= " | fgrep -v wvError" if ($verbose<5);
-    }
-    elsif ($input_type eq "pdf")
-    {
-	$cmd = "pdftohtml  -F -d $dirname -o \"$tailname.html\" \"$input_filename\"";
-	$cmd .= " >/dev/null" if ($verbose<3);
-    }
-    else
-    {
-	print STDERR "Error: Unable to convert to type '$input_type'\n";
-	exit(1);
-    }
-
-    if (system($cmd)>0)
-    {
-	print STDERR "Error: failed to execute $cmd: $!\n";
-	exit(1);
-    }
-
-    # restore to original working directory
-    chdir ($stored_dir) || die "Unable to return to directory $stored_dir";
-
-}
-
-
-
-&main(@ARGV)
-
Index: unk/gsdl/bin/script/gs2txt.pl
===================================================================
--- /trunk/gsdl/bin/script/gs2txt.pl	(revision 1444)
+++ 	(revision )
@@ -1,132 +1,0 @@
-#!/usr/bin/perl -w
-
-###########################################################################
-#
-# gs2txt.pl -- convert various documents to TEXT
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 1999 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-# Convert Microsoft Word, Adobe PDF, and Adobe Postscript to text 
-# using the appropriate specialist convert util
-
-
-BEGIN {
-    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
-    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
-
-}
-
-use parsargv;
-use util;
-use Cwd;
-use File::Basename;
-
-
-sub print_usage
-{
-
-    print STDERR "Usage: $0 [-t doc|pdf|ps] filename\n";
-    exit(1);
-}
-
-
-sub main
-{
-    my (@ARGV) = @_;
-
-    my ($input_type,$verbose);
-
-    if (!parsargv::parse(\@ARGV,
-			 'type/(doc|pdf|ps)/', \$input_type,
-			 'verbose/\d+/0',   \$verbose))
-    {
-	print_usage();
-    }
-
-    if (scalar(@ARGV!=1))
-    {
-	print_usage();
-    }
-    
-    my $input_filename = $ARGV[0];
-    if (!-r $input_filename)
-    {
-	print STDERR "Error: unable to open $input_filename for reading\n";
-	exit(1);
-    }
-
-    my ($tailname,$dirname,$suffix) 
-	= File::Basename::fileparse($input_filename,'\..+');
-    my $output_filename = &util::filename_cat($dirname,"$tailname.txt");
-
-    if ($input_type eq "")
-    {
-	$input_type = substr($suffix,1,length($suffix)-1);
-    }
-    
-    # Change to temporary working directory
-    my $stored_dir = cwd();
-    chdir ($dirname) || die "Unable to change to directory $dirname";
-
-    
-    # Select convert utility
-    my $cmd = "";
-    if (!defined $input_type)
-    {
-	print STDERR "Error: No filename extension or input type defined\n";
-	exit(1);
-    }
-    elsif ($input_type eq "doc")
-    {
-	my $wv_cfgfile = &util::filename_cat($ENV{'GSDLHOME'},"etc","wvtext.xml");
-	$cmd = "( echo Processing ; wvHtml --config $wv_cfgfile \"$input_filename\" 2>&1 > \"$output_filename\" )";
-	$cmd .= " | fgrep -v wvWarning" if ($verbose<3);
-	$cmd .= " | fgrep -v wvError" if ($verbose<5);
-    }
-    elsif ($input_type eq "pdf")
-    {
-	$cmd = "pdftotext \"$input_filename\"";
-    }
-    elsif ($input_type eq "ps")
-    {
-	$cmd = "ps2ascii \"$input_filename\" > \"$output_filename\"";
-    }
-    else
-    {
-	print STDERR "Error: Unable to convert to type '$input_type'\n";
-	exit(1);
-    }
-
-    if (system($cmd)>0)
-    {
-	print STDERR "Error: failed to execute $cmd: $!\n";
-	exit(1);
-    }
-
-    # restore to original working directory
-    chdir ($stored_dir) || die "Unable to return to directory $stored_dir";
-
-}
-
-
-
-&main(@ARGV)
Index: /trunk/gsdl/bin/script/gsConvert.pl
===================================================================
--- /trunk/gsdl/bin/script/gsConvert.pl	(revision 1445)
+++ /trunk/gsdl/bin/script/gsConvert.pl	(revision 1445)
@@ -0,0 +1,389 @@
+#!/usr/bin/perl -w
+
+###########################################################################
+#
+# gsConvert.pl -- convert documents to HTML ot TEXT format
+#
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+# gsConvert.pl converts documents in a range of formats to HTML or TEXT
+# by exploiting third-party programs.  These are usually found in the
+# $GSDLHOME/packages directory.
+#
+# Currently, we can convert Microsoft Word and Adobe PDF using specialised
+# conversion utilities.   We can convery any file to text with a perl 
+# implementation of the UNIX strings command.
+
+
+BEGIN {
+    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
+    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
+}
+
+use parsargv;
+use util;
+use Cwd;
+use File::Basename;
+
+
+sub print_usage
+{
+    print STDERR "Usage: $0 [-type doc|pdf] [-output html|text] filename\n";
+    exit(1);
+}
+
+
+sub main
+{
+    my (@ARGV) = @_;
+    my ($input_type,$output_type,$verbose);
+
+    # read command-line arguments
+    if (!parsargv::parse(\@ARGV,
+			 'type/(doc|pdf)/', \$input_type,
+			 'output/(html|text)/', \$output_type,
+			 'verbose/\d+/0',   \$verbose))
+    {
+	print_usage();
+    }
+
+    # Make sure the input file exists and can be opened for reading
+    if (scalar(@ARGV!=1)) {
+	print_usage();
+    }
+    my $input_filename = $ARGV[0];
+    if (!-r $input_filename) {
+	print STDERR "Error: unable to open $input_filename for reading\n";
+	exit(1);
+    }
+
+    # Deduce filenames
+    my ($tailname,$dirname,$suffix) 
+	= File::Basename::fileparse($input_filename,'\..+');
+    my $output_filestem = &util::filename_cat($dirname,"$tailname");
+
+    if ($input_type eq "")
+    {
+	$input_type = substr($suffix,1,length($suffix)-1);
+    }
+    
+    # Change to temporary working directory
+    my $stored_dir = cwd();
+    chdir ($dirname) || die "Unable to change to directory $dirname";
+    
+    # Select convert utility
+    if (!defined $input_type) {
+	print STDERR "Error: No filename extension or input type defined\n";
+	exit(1);
+    } 
+    elsif ($input_type eq "doc") {
+	print &convertDOC($input_filename, $output_filestem, $output_type);
+	print "\n";
+    } 
+    elsif ($input_type eq "pdf") {
+	print &convertPDF($dirname, $input_filename, $output_filestem, $output_type);
+	print "\n";
+    } 
+    elsif ($input_type eq "ps") {
+	print &convertPS($input_filename, $output_filestem, $output_type);
+	print "\n";
+    } 
+    else {
+	print STDERR "Error: Unable to convert type '$input_type'\n";
+	exit(1);
+    }
+    
+    # restore to original working directory
+    chdir ($stored_dir) || die "Unable to return to directory $stored_dir";
+
+}
+
+&main(@ARGV);
+
+
+
+# Document-type conversion fucntions
+#
+# The following functions attempt to convert documents from their
+# input type to the specified output type.  If no output type was
+# given, then they first attempt HTML, and then TEXT.
+#
+# Each returns the output type ("html" or "text") or "fail" if no 
+# conversion is possible.
+
+# Convert a Microsoft word document
+
+sub convertDOC {
+    ($input_filename, $output_filestem, $output_type) = @_;
+
+    my $success = 0;
+
+    # Attempt specialised conversion to HTML
+    if (!$output_type || ($output_type =~ /html/i)) {
+	$success = &doc_to_html($input_filename, $output_filestem);
+	if ($success) {
+	    return "html";
+	}
+    }
+
+    # Attempt simple conversion to HTML
+    if (!$output_type || ($output_type =~ /html/i)) {
+	$success = &any_to_html($input_filename, $output_filestem);
+	if ($success) {
+	    return "html";
+	}
+    }
+
+    # Convert to text
+    if (!$output_type || ($output_type =~ /text/i)) {
+	$success = any_to_text($input_filename, $output_filestem);
+	if ($success) {
+	    return "text";
+	}
+    }
+
+    return "fail";
+
+}
+
+
+# Convert an Adobe PDF document
+
+sub convertPDF {
+    ($dirname, $input_filename, $output_filestem, $output_type) = @_;
+
+    my $success = 0;
+
+    # Attempt conversion to HTML
+    if (!$output_type || ($output_type =~ /html/i)) {
+	$success = &pdf_to_html($dirname, $input_filename, $output_filestem);
+	if ($success) {
+	    return "html";
+	}
+    }
+
+    # Attempt conversion to TEXT
+    if (!$output_type || ($output_type =~ /text/i)) {
+	$success = &pdf_to_text($input_filename, $output_filestem);
+	if ($success) {
+	    return "text";
+	}
+    }
+
+    return "fail";
+
+}
+
+
+# Convert an Adobe PostScript document
+
+sub convertPS {
+    ($input_filename, $output_filestem, $output_type) = @_;
+
+    my $success = 0;
+
+    # Attempt conversion to TEXT
+    if (!$output_type || ($output_type =~ /text/i)) {
+	$success = &ps_to_text($input_filename, $output_filestem);
+	if ($success) {
+	    return "text";
+	}
+    }
+
+    return "fail";
+
+}
+
+
+
+# Specific type-to-type cponversions
+#
+# Each of the following functions attempts to convert a document from 
+# a specific format to another.  If they succeed yhey return 1 and leave 
+# the output document(s) in the appropriate place; if they fail they 
+# return 0 and delete any working files.
+
+
+# Attempt to convert a word document to html with the wv program
+
+sub doc_to_html {
+    ($input_filename, $output_filestem) = @_;
+
+    # print "Processing...\n";
+
+    # formulate the command
+    my $wv_home = &util::filename_cat($ENV{'GSDLHOME'},"packages","wv-0.5.44-gs");
+    my $wvHtml = &util::filename_cat($wv_home, "bin", "wvHtml");
+    return 0 unless (-e "$wvHtml");
+    $cmd = "$wvHtml \"$input_filename\" >\"$output_filestem.html\" 2>\"$output_filestem.err\"";
+    
+    # execute the command
+    if (system($cmd)>0)
+    {
+	print STDERR "Error executing $cmd: $!. Continuing...\n";
+    }
+	
+    # Was the conversion successful?
+    if (-e "$output_filestem.html") {
+	open(TMP, "$output_filestem.html");
+	$line = <TMP>;
+	close(TMP);
+	if ($line =~ /DOCTYPE HTML/) {
+	    &util::rm("$output_filestem.err");
+	    return 1;
+	} else {
+	    # An error of some sort occurred
+	    &util::rm("$output_filestem.html");
+	    &util::rm("$output_filestem.err");
+	}
+    }
+    return 0;
+}
+
+
+# Convert a pdf file to html with the pdftohtml command
+
+sub pdf_to_html {
+    ($dirname, $input_filename, $output_filestem) = @_;
+
+    $cmd = "pdftohtml -F -d $dirname -o \"$output_filestem.html\" \"$input_filename\"";
+    $cmd .= " > $output_filestem.out";
+    
+    if (system($cmd)>0)
+    {
+	print STDERR "Error executing $cmd: $!\n";
+	&util::rm("$output_filestem.html") if (-e "$output_filestem.html");
+	&util::rm("$output_filestem.out") if (-e "$output_filestem.out");
+	return 0;
+    }
+
+    &util::rm("$output_filestem.out") if (-e "$output_filestem.out");
+    return 1;
+}
+
+
+# Convert a PDF file to text with the pdftotext command
+
+sub pdf_to_text {
+    ($dirname, $input_filename, $output_filestem) = @_;
+
+    $cmd = "pdftotext \"$input_filename\" > \"$output_filestem.text\"";
+    $cmd .= " 2> $output_filestem.err";
+    
+    if (system($cmd)>0)
+    {
+	print STDERR "Error executing $cmd: $!\n";
+	&util::rm("$output_filestem.text") if (-e "$output_filestem.text");
+	&util::rm("$output_filestem.err") if (-e "$output_filestem.err");
+	return 0;
+    }
+
+    &util::rm("$output_filestem.err") if (-e "$output_filestem.err");
+    return 1;
+}
+
+
+# Convert a PostScript document to text with ps2ascii
+
+sub ps_to_text {
+    ($input_filename, $output_filestem) = @_;
+
+    my $cmd = "ps2ascii \"$input_filename\" > \"$output_filestem.text\"";
+    $cmd .= " 2> $output_filestem.err";
+    
+    if (system($cmd)>0)
+    {
+	print STDERR "Error executing $cmd: $!\n";
+	&util::rm("$output_filestem.text") if (-e "$output_filestem.text");
+	&util::rm("$output_filestem.err") if (-e "$output_filestem.err");
+	return 0;
+    }
+
+    &util::rm("$output_filestem.err") if (-e "$output_filestem.err");
+    return 1;
+}
+
+
+# Convert any file to HTML with a crude perl implementation of the
+# UNIX strings command.
+
+sub any_to_html {
+    ($input_filename, $output_filestem) = @_;
+
+    # First generate a text file
+    return 0 unless (&any_to_text($input_filename, $output_filestem));
+
+    # create an HTML file from the text file
+    open(TEXT, "<$output_filestem.text");
+    open(HTML, ">$output_filestem.html");
+
+    print HTML '<html><head>
+<META HTTP-EQUIV="Content-Type" CONTENT="text/html">
+<META NAME="GENERATOR" CONTENT="Greenstone any-to-html">
+</head><body>\n\n';
+    while (<TEXT>) { 
+	print HTML "<p> ", $_;
+	
+    }
+    print HTML "\n</body></html>]\n";
+
+    &util::rm("$output_filestem.text") if (-e "$output_filestem.text");
+    return 1;
+}
+
+# Convert any file to TEXT with a crude perl implementation of the
+# UNIX strings command.
+
+sub any_to_text {
+    ($input_filename, $output_filestem) = @_;
+
+    open(IN, "<$input_filename");
+    open(OUT, ">$output_filestem.text");
+
+    my ($line);
+    while (<IN>) { 
+	$line = $_;
+
+	# delete anything that isn't a printable character
+	$line =~ s/[^\040-\176]+/\n/sg;
+
+	# delete any string less than 10 characters long
+	while ($line =~ /^[^\n]{1,9}$/m) {
+	    $line =~ s/^[^\n]{0,9}$/\n/mg;
+	    $line =~ s/\n+/\n/sg;
+	}
+
+	# remove extraneous whitespace
+	$line =~ s/\n+/\n/gs;
+	$line =~ s/^\n//gs;
+	
+	# output whatever is left
+	if ($line =~ /[^\n]/) {
+	    print OUT $line;
+	}
+    }
+    return 1;
+}
+
+
+
