source: trunk/gsdl/bin/script/gsConvert.pl@ 1654

#!/usr/bin/perl -w

###########################################################################
#
# gsConvert.pl -- convert documents to HTML ot TEXT format
#
# A component of the Greenstone digital library software
# from the New Zealand Digital Library Project at the 
# University of Waikato, New Zealand.
#
# Copyright (C) 1999 New Zealand Digital Library Project
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
#
###########################################################################

# gsConvert.pl converts documents in a range of formats to HTML or TEXT
# by exploiting third-party programs.  These are usually found in the
# $GSDLHOME/packages directory.
#
# Currently, we can convert Microsoft Word and Adobe PDF using specialised
# conversion utilities.   We can convery any file to text with a perl 
# implementation of the UNIX strings command.


BEGIN {
    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
}

use parsargv;
use util;
use Cwd;
use File::Basename;


sub print_usage
{
    print STDERR "Usage: $0 [-type doc|pdf] [-output html|text] filename\n";
    exit(1);
}


sub main
{
    my (@ARGV) = @_;
    my ($input_type,$output_type,$verbose);

    # read command-line arguments
    if (!parsargv::parse(\@ARGV,
             'type/(doc|pdf)/', \$input_type,
             'output/(html|text)/', \$output_type,
             'verbose/\d+/0',   \$verbose))
    {
    print_usage();
    }

    # Make sure the input file exists and can be opened for reading
    if (scalar(@ARGV!=1)) {
    print_usage();
    }
    my $input_filename = $ARGV[0];
    if (!-r $input_filename) {
    print STDERR "Error: unable to open $input_filename for reading\n";
    exit(1);
    }

    # Deduce filenames
    my ($tailname,$dirname,$suffix) 
    = File::Basename::fileparse($input_filename,'\..+');
    my $output_filestem = &util::filename_cat($dirname,"$tailname");

    if ($input_type eq "")
    {
    $input_type = substr($suffix,1,length($suffix)-1);
    }
    
    # Change to temporary working directory
    my $stored_dir = cwd();
    chdir ($dirname) || die "Unable to change to directory $dirname";
    
    # Select convert utility
    if (!defined $input_type) {
    print STDERR "Error: No filename extension or input type defined\n";
    exit(1);
    } 
    elsif ($input_type eq "doc") {
    print &convertDOC($input_filename, $output_filestem, $output_type);
    print "\n";
    } 
    elsif ($input_type eq "pdf") {
    print &convertPDF($dirname, $input_filename, $output_filestem, $output_type);
    print "\n";
    } 
    elsif ($input_type eq "ps") {
    print &convertPS($input_filename, $output_filestem, $output_type);
    print "\n";
    } 
    else {
    print STDERR "Error: Unable to convert type '$input_type'\n";
    exit(1);
    }
    
    # restore to original working directory
    chdir ($stored_dir) || die "Unable to return to directory $stored_dir";

}

&main(@ARGV);



# Document-type conversion fucntions
#
# The following functions attempt to convert documents from their
# input type to the specified output type.  If no output type was
# given, then they first attempt HTML, and then TEXT.
#
# Each returns the output type ("html" or "text") or "fail" if no 
# conversion is possible.

# Convert a Microsoft word document

sub convertDOC {
    ($input_filename, $output_filestem, $output_type) = @_;

    # Many .doc files are not in fact word documents!
    my $realtype = &find_docfile_type($input_filename);

    if ($realtype eq "word678") {
    return &convertWord678($input_filename, $output_filestem, $output_type);
    } elsif ($realtype eq "rtf") {
    return &convertRTF($input_filename, $output_filestem, $output_type);
    } else {
    return &convertAnything($input_filename, $output_filestem, $output_type);
    }
}

# Convert a Microsoft word 6/7/8 document

sub convertWord678 {
    ($input_filename, $output_filestem, $output_type) = @_;

    my $success = 0;

    # Attempt specialised conversion to HTML
    if (!$output_type || ($output_type =~ /html/i)) {
    $success = &doc_to_html($input_filename, $output_filestem);
    if ($success) {
        return "html";
    }
    }

    return &convertAnything($input_filename, $output_filestem, $output_type);
}


# Convert a Rich Text Format (RTF) file

sub convertRTF {
    ($input_filename, $output_filestem, $output_type) = @_;

    my $success = 0;

    # Attempt specialised conversion to HTML
    if (!$output_type || ($output_type =~ /html/i)) {
    $success = &rtf_to_html($input_filename, $output_filestem);
    if ($success) {
        return "html";
    }
    }

    return &convertAnything($input_filename, $output_filestem, $output_type);
}


# Convert an unidentified file

sub convertAnything {
    ($input_filename, $output_filestem, $output_type) = @_;
    
    my $success = 0;
    
    # Attempt simple conversion to HTML
    if (!$output_type || ($output_type =~ /html/i)) {
    $success = &any_to_html($input_filename, $output_filestem);
    if ($success) {
        return "html";
    }
    }

    # Convert to text
    if (!$output_type || ($output_type =~ /text/i)) {
    $success = any_to_text($input_filename, $output_filestem);
    if ($success) {
        return "text";
    }
    }
    return "fail";
}



# Convert an Adobe PDF document

sub convertPDF {
    ($dirname, $input_filename, $output_filestem, $output_type) = @_;

    my $success = 0;

    # Attempt conversion to HTML
    if (!$output_type || ($output_type =~ /html/i)) {
    $success = &pdf_to_html($dirname, $input_filename, $output_filestem);
    if ($success) {
        return "html";
    }
    }

    # Attempt conversion to TEXT
    if (!$output_type || ($output_type =~ /text/i)) {
    $success = &pdf_to_text($input_filename, $output_filestem);
    if ($success) {
        return "text";
    }
    }

    return "fail";

}


# Convert an Adobe PostScript document

sub convertPS {
    ($input_filename, $output_filestem, $output_type) = @_;

    my $success = 0;

    # Attempt conversion to TEXT
    if (!$output_type || ($output_type =~ /text/i)) {
    $success = &ps_to_text($input_filename, $output_filestem);
    if ($success) {
        return "text";
    }
    }

    return "fail";

}


# Find the real type of a .doc file
#
# We seem to have alot of files with a .dco extension that are .rtf 
# files or Word 5 files.  This function attempts to tell the difference.

sub find_docfile_type {
    ($input_filename) = @_;
    
    open(CHK, "<$input_filename");
    my $line = "";
    my $first = 1;

    while (<CHK>) {
    
    $line = $_;

    if ($first) {
        # check to see if this is an rtf file
        if ($line =~ /^\{\\rtf/) {
        close(CHK);
        return "rtf";
        }
    }
    
    # is theis a word 6/7/8 document?
    if ($line =~ /Word\.Document\.[678]/) {
        close(CHK);
        return "word678";
    }

    $first = 0;

    }

    return "unknown";
}



# Specific type-to-type cponversions
#
# Each of the following functions attempts to convert a document from 
# a specific format to another.  If they succeed yhey return 1 and leave 
# the output document(s) in the appropriate place; if they fail they 
# return 0 and delete any working files.


# Attempt to convert a word document to html with the wv program

sub doc_to_html {
    ($input_filename, $output_filestem) = @_;

    # formulate the command
    my $wv_home = &util::filename_cat($ENV{'GSDLHOME'}, "packages", "unix", "wv");
    my $wv_conf = &util::filename_cat($wv_home, "lib", "wv", "wvHtml.xml");
    my $wvWare = &util::filename_cat($wv_home, "bin", "wvWare");
    return 0 unless (-e "$wvWare");
    $cmd = "ulimit -t 20;";
    $cmd .= "$wvWare --charset utf-8 --config $wv_conf";
    $cmd .= " \"$input_filename\" > \"$output_filestem.html\" 2>\"$output_filestem.err\"";

    # execute the command
    if (system($cmd)>0)
    {
    print STDERR "Error executing wv converter: $!. Continuing...\n";
    }

    # Was the conversion successful?
    if (-e "$output_filestem.html") {
    open(TMP, "$output_filestem.html");
    $line = <TMP>;
    close(TMP);
    if ($line && $line =~ /DOCTYPE HTML/) {
        &util::rm("$output_filestem.err");
        return 1;
    } else {
        # An error of some sort occurred
        &util::rm("$output_filestem.html");
        &util::rm("$output_filestem.err");
    }
    }
    return 0;
}


# Attempt to convert an RTF document to html with rtftohtml
#
# rtf2html isn't distributed with Greenstone because it is not 
# distributed under teh GPL.  If you know of a better solution,
# please let me know.

sub rtf_to_html {
    ($input_filename, $output_filestem) = @_;

    # formulate the command
    my $r_cmd = &util::filename_cat($ENV{'GSDLHOME'}, "packages", "unix", 
                    "rtf2html", "rtf2html", "rtf2html");
    $r_cmd = "rtf2html" unless (-e "$r_cmd");
    return 0 unless (-e "$r_cmd");
    $cmd = "ulimit -t 20;";
    $cmd .= "$r_cmd";
    $cmd .= " \"$input_filename\" > \"$output_filestem.html\" 2>\"$output_filestem.err\"";

    # execute the command
    if (system($cmd)>0)
    {
    print STDERR "Error executing rtf converter: $!. Continuing...\n";
    }

    # Was the conversion successful?
    if (-e "$output_filestem.html") {
    open(TMP, "$output_filestem.html");
    $line = <TMP>;
    close(TMP);
    if ($line && $line =~ /DOCTYPE HTML/) {
        &util::rm("$output_filestem.err");
        return 1;
    } else {
        # An error of some sort occurred
        &util::rm("$output_filestem.html");
        &util::rm("$output_filestem.err");
    }
    }
    return 0;
}


# Convert a pdf file to html with the pdftohtml command

sub pdf_to_html {
    ($dirname, $input_filename, $output_filestem) = @_;

    $cmd = "pdftohtml -F -d $dirname -o \"$output_filestem.html\" \"$input_filename\"";
    $cmd .= " > $output_filestem.out";
    
    if (system($cmd)>0)
    {
    print STDERR "Error executing $cmd: $!\n";
    &util::rm("$output_filestem.html") if (-e "$output_filestem.html");
    &util::rm("$output_filestem.out") if (-e "$output_filestem.out");
    return 0;
    }

    &util::rm("$output_filestem.out") if (-e "$output_filestem.out");
    return 1;
}


# Convert a PDF file to text with the pdftotext command

sub pdf_to_text {
    ($dirname, $input_filename, $output_filestem) = @_;

    $cmd = "pdftotext \"$input_filename\" > \"$output_filestem.text\"";
    $cmd .= " 2> $output_filestem.err";
    
    if (system($cmd)>0)
    {
    print STDERR "Error executing $cmd: $!\n";
    &util::rm("$output_filestem.text") if (-e "$output_filestem.text");
    &util::rm("$output_filestem.err") if (-e "$output_filestem.err");
    return 0;
    }

    &util::rm("$output_filestem.err") if (-e "$output_filestem.err");
    return 1;
}


# Convert a PostScript document to text with ps2ascii

sub ps_to_text {
    ($input_filename, $output_filestem) = @_;

    my $cmd = "ps2ascii \"$input_filename\" > \"$output_filestem.text\"";
    $cmd .= " 2> $output_filestem.err";
    
    if (system($cmd)>0)
    {
    print STDERR "Error executing $cmd: $!\n";
    &util::rm("$output_filestem.text") if (-e "$output_filestem.text");
    &util::rm("$output_filestem.err") if (-e "$output_filestem.err");
    return 0;
    }

    &util::rm("$output_filestem.err") if (-e "$output_filestem.err");
    return 1;
}


# Convert any file to HTML with a crude perl implementation of the
# UNIX strings command.

sub any_to_html {
    ($input_filename, $output_filestem) = @_;

    # First generate a text file
    return 0 unless (&any_to_text($input_filename, $output_filestem));

    # create an HTML file from the text file
    open(TEXT, "<$output_filestem.text");
    open(HTML, ">$output_filestem.html");

    print HTML '<html><head>
<META HTTP-EQUIV="Content-Type" CONTENT="text/html">
<META NAME="GENERATOR" CONTENT="Greenstone any_to_html">
</head><body>\n\n';
    while (<TEXT>) { 
    print HTML "<p> ", $_;
    
    }
    print HTML "\n</body></html>]\n";

    &util::rm("$output_filestem.text") if (-e "$output_filestem.text");
    return 1;
}

# Convert any file to TEXT with a crude perl implementation of the
# UNIX strings command.

sub any_to_text {
    ($input_filename, $output_filestem) = @_;

    open(IN, "<$input_filename");
    open(OUT, ">$output_filestem.text");

    my ($line);
    while (<IN>) { 
    $line = $_;
    
    # delete anything that isn't a printable character
    $line =~ s/[^\040-\176]+/\n/sg;

    # delete any string less than 10 characters long
    $line =~ s/^[^\n]{0,9}$/\n/mg;
    while ($line =~ /^[^\n]{1,9}$/m) {
        $line =~ s/^[^\n]{0,9}$/\n/mg;
        $line =~ s/\n+/\n/sg;
    }

    # remove extraneous whitespace
    $line =~ s/\n+/\n/gs;
    $line =~ s/^\n//gs;

    # output whatever is left
    if ($line =~ /[^\n ]/) {
        print OUT $line;
    }
    }
    return 1;
}