Changeset 2755

Timestamp:

2001-09-26T10:43:44+12:00 (23 years ago)

Author:

jrm21

Message:

import.pl now takes an option for saving file conversion failures to a log.
By default, import.pl will use <collectdir>/etc/fail.log. Currently only
the plugins based on ConvertToPlug will do this. Not yet tested on Win9X.

Location:

trunk/gsdl

Files:

• bin/script/gsConvert.pl (modified) (27 diffs)
• bin/script/import.pl (modified) (6 diffs)
• bin/script/pdftohtml.pl (modified) (9 diffs)
• perllib/plugin.pm (modified) (2 diffs)
• perllib/plugins/BasPlug.pm (modified) (2 diffs)
• perllib/plugins/ConvertToPlug.pm (modified) (2 diffs)

trunk/gsdl/bin/script/gsConvert.pl

@@ -28 +28 @@
 
 # gsConvert.pl converts documents in a range of formats to HTML or TEXT
-# by exploiting third-party programs.  These are usually found in the
-# $GSDLHOME/packages directory.
-#
-# Currently, we can convert Microsoft Word and Adobe PDF using specialised
-# conversion utilities.   We can convery any file to text with a perl 
-# implementation of the UNIX strings command.
+# by exploiting third-party programs.  The sources of these are usually found
+# in the $GSDLHOME/packages directory, and the executables should live in
+# $GSDLHOME/bin/$GSDLOS (which is on the search path).
+#
+# Currently, we can convert Microsoft Word, RTF, Adobe PDF and PostScript
+# using specialised conversion utilities.   We can try to convert any file to
+# text with a perl implementation of the UNIX strings command.
 #
 # We try to convert Postscript files to text using "gs" which is often on
-# *nix machines. If it isn't (or we're running on Windoze), we do some feeble
-# text extraction on it using regexps.
+# *nix machines. We fall back to performing weak text extraction by using
+# regular expressions.
 
 BEGIN {
@@ -49 +50 @@
 use File::Basename;
 
+# Are we running on WinNT or Win2000 (or later)?
+my $is_winnt_2000=eval {require Win32; return (Win32::IsWinNT()); return 0;};
+if (!defined($is_winnt_2000)) {$is_winnt_2000=0;}
 
 sub print_usage
@@ -56 +60 @@
     print STDERR "              or text using third-party programs.\n\n";
     print STDERR "  usage: $0 [options] filename\n";
-    print STDERR "  options:\n\t-type\tdoc|pdf|ps|rtf\n\t-output\thtml|text\n";
-    print STDERR "\t-timeout\t<max cpu seconds>\n";
+    print STDERR "  options:\n\t-type\tdoc|pdf|ps|rtf\t(input file type)\n";
+    print STDERR "\t-errlog\t<filename>\t(append err messages)\n";
+    print STDERR "\t-output\thtml|text\n";
+    print STDERR "\t-timeout\t<max cpu seconds>\t(ulimit on unix systems)\n";
     exit(1);
 }
 
+my $faillogfile="";
 
 sub main
@@ -71 +78 @@
     if (!parsargv::parse(\@ARGV,
              'type/(doc|pdf|ps|rtf)/', \$input_type,
+             '/errlog/.*/', \$faillogfile,
              'output/(html|text)/', \$output_type,
              'timeout/\d+/0',\$timeout,
@@ -198 +206 @@
     }
 
-    return &convertAnything($input_filename, $output_filestem, $output_type);
+# rtf is so ugly that's it's not worth running strings over.
+# One day I'll write some quick'n'dirty regexps to try to extract text - jrm21
+#    return &convertAnything($input_filename, $output_filestem, $output_type);
+    return "fail";
 }
 
@@ -232 +243 @@
 
 sub convertPDF {
-    ($dirname, $input_filename, $output_filestem, $output_type) = @_;
+    my ($dirname, $input_filename, $output_filestem, $output_type) = @_;
 
     my $success = 0;
@@ -300 +311 @@
         return "rtf";
         }
+        $first = 0;
     }
     
@@ -308 +320 @@
     }
 
-    $first = 0;
-
     }
 
@@ -320 +330 @@
 #
 # Each of the following functions attempts to convert a document from 
-# a specific format to another.  If they succeed yhey return 1 and leave 
+# a specific format to another.  If they succeed they return 1 and leave 
 # the output document(s) in the appropriate place; if they fail they 
 # return 0 and delete any working files.
@@ -348 +358 @@
     # redirecting STDERR is a bad idea on windows 95/98
     $cmd .= " 2> \"$output_filestem.err\""
-    if $ENV{'GSDLOS'} !~ /^windows$/i;
+    if ($ENV{'GSDLOS'} !~ /^windows$/i || $is_winnt_2000);
 
     # execute the command
+    $!=0;
     if (system($cmd)!=0)
     {
-    print STDERR "Error executing wv converter: $!. Continuing...\n";
+    print STDERR "Error executing wv converter:$!\n";
+    if (-s "$output_filestem.err") {
+        open (ERRFILE, "<$output_filestem.err");
+
+        my $write_to_fail_log=0;
+        if ($faillogfile ne "" && defined(open(FAILLOG,">>$faillogfile")))
+        {$write_to_fail_log=1;}
+
+        my $line;
+        while ($line=<ERRFILE>) {
+        if ($line =~ /\w/) {
+            print STDERR "$line";
+            print FAILLOG "$line" if ($write_to_fail_log);
+        }
+        if ($line !~ m/startup error/) {next;}
+        print STDERR " (given an invalid .DOC file?)\n";
+        print FAILLOG " (given an invalid .DOC file?)\n"
+        if ($write_to_fail_log);
+        
+        } # while ERRFILE
+        close FAILLOG if ($write_to_fail_log);
+    }
+    print STDERR "Continuing...\n";
+    return 0; # we can try any_to_text
     }
 
@@ -365 +399 @@
         &util::rm("$output_filestem.err") if -e "$output_filestem.err";
         return 1;
-    } else {
-        # An error of some sort occurred
-        &util::rm("$output_filestem.html") if -e "$output_filestem.html";
-        &util::rm("$output_filestem.err") if -e "$output_filestem.err";
-    }
-    }
-
+    }
+    }
+    
+    # If here, an error of some sort occurred
+    &util::rm("$output_filestem.html") if -e "$output_filestem.html";
+    if (-e "$output_filestem.err") {
+    if ($faillogfile ne "" && defined(open(FAILLOG,">>$faillogfile"))) {
+        open (ERRLOG,"$output_filestem.err");
+        while (<ERRLOG>) {print FAILLOG $_;}
+        close FAILLOG;
+        close ERRLOG;
+    }
+    &util::rm("$output_filestem.err");
+    }
+    
     return 0;
 }
@@ -390 +432 @@
 
     $cmd .= " 2>\"$output_filestem.err\""
-        unless $ENV{'GSDLOS'} =~ /^windows$/i;
+        if ($ENV{'GSDLOS'} !~ /^windows$/i || $is_winnt_2000);
 
 
     # execute the command
+    $!=0;
     if (system($cmd)!=0)
     {
-    print STDERR "Error executing rtf converter: $!.\n";
+    print STDERR "Error executing rtf converter $!\n";
     # don't currently bother printing out error log...
     # keep going, in case it still created an HTML file...
     }
 
-    &util::rm("$output_filestem.err") if (-e "$output_filestem.err");
-
     # Was the conversion successful?
+    my $was_successful=0;
     if (-s "$output_filestem.html") {
-    return 1;
+    # make sure we have some content other than header
+    open (HTML, "$output_filestem.html"); # what to do if fail?
+    my $line;
+    my $past_header=0;
+    while ($line=<HTML>) {
+
+        if ($past_header == 0) {
+        if ($line =~ /<body>/) {$past_header=1;}
+        next;
+        }
+
+        $line =~ s/<[^>]+>//g;
+        if ($line =~ /\w/ && $past_header) {  # we found some content...
+        $was_successful=1;
+        last;
+        }
+    }
+    close HTML;
+    }
+
+    if ($was_successful) {
+    &util::rm("$output_filestem.err")
+        if (-e "$output_filestem.err");
+    # insert the (modified) table of contents, if it exists.
+    if (-e "${output_filestem}_ToC.html") {
+        &util::mv("$output_filestem.html","$output_filestem.src");
+        my $open_failed=0;
+        open HTMLSRC, "$output_filestem.src" || ++$open_failed;
+        open TOC, "${output_filestem}_ToC.html" || ++$open_failed;
+        open HTML, ">$output_filestem.html" || ++$open_failed;
+        
+        if ($open_failed) {
+        close HTMLSRC;
+        close TOC;
+        close HTML;
+        &util::mv("$output_filestem.src","$output_filestem.html");
+        return 1;
+        }
+
+        # print out header info from src html.
+        while (($_ = <HTMLSRC>) =~ /\w/) {
+        print HTML "$_";
+        }
+
+        # print out table of contents, making links relative
+        <TOC>; <TOC>; # ignore first 2 lines
+        print HTML scalar(<TOC>); # line 3 = "<ol>\n"
+        my $line;
+        while ($line=<TOC>) {
+        $line =~ s@</body></html>$@@ ; # only last line has this
+        # make link relative
+        $line =~ s@href=\"[^\#]+@href=\"@;
+        print HTML $line;
+        }
+        close TOC;
+
+        # rest of html src
+        while (<HTMLSRC>) {
+        print HTML $_;
+        }
+        close HTMLSRC;
+        close HTML;
+
+        &util::rm("${output_filestem}_ToC.html");
+        &util::rm("${output_filestem}.src");
+    }
+    # we don't yet do anything with footnotes ($output_filestem_fn.html) :(
+    return 1; # success
+    }
+
+    if (-e "$output_filestem.err") {
+    if ($faillogfile ne "" && defined(open(FAILLOG,">>$faillogfile")))
+    {
+        print FAILLOG "Error - rtftohtml - couldn't extract text\n";
+        print FAILLOG " (rtf file might be too recent):\n";
+        open (ERRLOG, "$output_filestem.err");
+        while (<ERRLOG>) {print FAILLOG $_;}
+        close ERRLOG;
+        close FAILLOG;
+    }
+    &util::rm("$output_filestem.err");
     }
 
@@ -417 +539 @@
 
 sub pdf_to_html {
-    ($dirname, $input_filename, $output_filestem) = @_;
+    my ($dirname, $input_filename, $output_filestem) = @_;
 
     $cmd = "";
     if ($timeout) {$cmd = "ulimit -t $timeout;";}
-    $cmd .= "perl -S pdftohtml.pl -F ";
+    $cmd .= "perl -S pdftohtml.pl ";
     $cmd .= " \"$input_filename\" \"$output_filestem\"";
+    
+    if ($ENV{'GSDLOS'} !~ /^windows$/i || $is_winnt_2000) {
+    $cmd .= " > \"$output_filestem.out\" 2> \"$output_filestem.err\"";
+    } else {
+    $cmd .= " > \"$output_filestem.err\"";
+    }
+
     $!=0;
 
@@ -428 +557 @@
     if ($retval!=0)
     {
-    print STDERR "Error executing $cmd";
+    print STDERR "Error executing pdftohtml.pl";
     if ($!) {print STDERR ": $!";} 
     print STDERR "\n";
@@ -440 +569 @@
     if (-s "$output_filestem.err") {
         open (ERRLOG, "$output_filestem.err") || die "$!";
-        print STDERR "pdftohtml:\n";
+        print STDERR "pdftohtml error log:\n";
         while (<ERRLOG>) {
         print STDERR "$_";
@@ -447 +576 @@
     }
     &util::rm("$output_filestem.html") if (-e "$output_filestem.html");
-    &util::rm("$output_filestem.err") if (-e "$output_filestem.err");
+    if (-e "$output_filestem.err") {
+        if ($faillogfile ne "" && defined(open(FAILLOG,">>$faillogfile")))
+        {
+        open (ERRLOG, "$output_filestem.err");
+        while (<ERRLOG>) {print FAILLOG $_;}
+        close ERRLOG;
+        close FAILLOG;
+        }   
+    &util::rm("$output_filestem.err");
+    }
     return 0;
     }
@@ -459 +597 @@
 
 sub pdf_to_text {
-    ($dirname, $input_filename, $output_filestem) = @_;
+    my ($dirname, $input_filename, $output_filestem) = @_;
 
     my $cmd = "pdftotext \"$input_filename\" \"$output_filestem.text\"";
-    $cmd .= " 2> \"$output_filestem.err\"";
+
+    if ($ENV{'GSDLOS'} !~ /^windows$/i) {
+    $cmd .= " > \"$output_filestem.out\" 2> \"$output_filestem.err\"";
+    } else {
+    $cmd .= " > \"$output_filestem.err\"";
+    }
     
     if (system($cmd)!=0)
@@ -468 +611 @@
     print STDERR "Error executing $cmd: $!\n";
     &util::rm("$output_filestem.text") if (-e "$output_filestem.text");
-    &util::rm("$output_filestem.err") if (-e "$output_filestem.err");
-    return 0;
+    }
+
+    # make sure there is some extracted text.
+    if (-e "$output_filestem.text") {
+    open (EXTR_TEXT, "$output_filestem.text") || warn "open: $!";
+    binmode(EXTR_TEXT); # just in case...
+    my $line="";
+    my $seen_text=0;
+    while (($seen_text==0) && ($line=<EXTR_TEXT>)) {
+        if ($line=~ /\w/) {$seen_text=1;}
+    }
+    close EXTR_TEXT;
+    if ($seen_text==0) { # no text was extracted
+        print STDERR "Error: pdftotext found no text\n";
+        &util::rm("$output_filestem.text");
+    }
     }
 
@@ -478 +635 @@
     if (-s "$output_filestem.err") {
         open (ERRLOG, "$output_filestem.err") || die "$!";
-        print STDERR "pdftotext:\n";
+        print STDERR "pdftotext error log:\n";
         while (<ERRLOG>) {
         print STDERR "$_";
@@ -487 +644 @@
     &util::rm("$output_filestem.out") if (-e "$output_filestem.out");
     &util::rm("$output_filestem.text") if (-e "$output_filestem.text");
-    &util::rm("$output_filestem.err") if (-e "$output_filestem.err");
-
+    if (-e "$output_filestem.err") {
+        if ($faillogfile ne "" && defined(open(FAILLOG,">>$faillogfile")))
+        {
+        open (ERRLOG,"$output_filestem.err");
+        while (<ERRLOG>) {print FAILLOG $_;}
+        close ERRLOG;
+        close FAILLOG;
+        }
+        &util::rm("$output_filestem.err");
+    }
     return 0;
     }
@@ -537 +702 @@
     if ($error ne "")
     {
-    print STDERR "PSPlug: WARNING: Error executing gs: $error\n";
+    print STDERR "Warning: Error executing gs: $error\n";
     &util::rm("$output_filestem.text") if (-e "$output_filestem.text");
+
+    if ("$faillogfile" ne "" && defined(open (FAILLOG, ">>$faillogfile")))
+    {
+        print FAILLOG "gs - $error\n";
+        if (-e "$output_filestem.err") {
+        open(ERRLOG, "$output_filestem.err");
+        while (<ERRLOG>) {print FAILLOG $_;}
+        close ERRLOG;
+        }
+        close FAILLOG;
+    }
     &util::rm("$output_filestem.err") if (-e "$output_filestem.err");
+
 
     # Fine then. We'll just do a lousy job by ourselves...
@@ -545 +722 @@
     # http://snark.ptc.spbu.ru/mail-archives/lout/brown/msg00003.html
     # 
-    print STDERR "PSPlug: Stripping text from postscript\n";
+    print STDERR "Stripping text from postscript\n";
     my $errorcode=0;
     open (IN, "$input_filename") 
@@ -554 +731 @@
     
     my $text="";  # this is for whole .ps file...
-    while (<IN>) {
-        $text.=$_;
-    }
+    $text = join('', <IN>); # see man perlport, under "System Resources"
     close IN;
 
     # Make sure this is a ps file...
     if ($text !~ /^%!/) {
-        print STDERR "Bad postscript header: not %!\n";
+        print STDERR "Bad postscript header: not '%!'\n";
+        if ($faillogfile ne "" && defined(open(FAILLOG, ">>$faillogfile")))
+        {
+        print FAILLOG "Bad postscript header: not '%!'\n";
+        close FAILLOG;
+        }
         return 0;
     }
@@ -666 +846 @@
     print HTML "</head><body>\n\n";
 
-    while (<TEXT>) { 
-    print HTML "<p> ", $_;
+    my $line;
+    while ($line=<TEXT>) { 
+    $line =~ s/</&lt;/g;
+    $line =~ s/>/&gt;/g;
+    if ($line =~ /^\s*$/) {
+        print HTML "<p>";
+    } else {
+        print HTML "<br> ", $line;
+    }
     }
     print HTML "\n</body></html>\n";
@@ -680 +867 @@
 # Convert any file to TEXT with a crude perl implementation of the
 # UNIX strings command.
+# Note - this assumes ascii charsets :(     (jrm21)
 
 sub any_to_text {
     ($input_filename, $output_filestem) = @_;
 
-    open(IN, "<$input_filename");
+    open(IN, "<$input_filename") || return 0;
     binmode(IN);
-    open(OUT, ">$output_filestem.text");
+    open(OUT, ">$output_filestem.text") || return 0;
 
     my ($line);
-    my $dgcount = 0;
+    my $output_line_count = 0;
     while (<IN>) { 
     $line = $_;
@@ -710 +898 @@
     if ($line =~ /[^\n ]/) {
         print OUT $line;
+        ++$output_line_count;
     }
     }
@@ -716 +905 @@
     close IN;
 
-    return 1;
-}
+    if ($output_line_count) { # try to protect against binary only formats
+    return 1;
+    }
+
+    &util::rm("$output_filestem.text");
+    return 0;
+
+}

trunk/gsdl/bin/script/import.pl

@@ -80 +80 @@
     print STDOUT "   -collectdir directory  Collection directory (defaults to " .
     &util::filename_cat ($ENV{'GSDLHOME'}, "collect") . ")\n";
-    print STDOUT "   -out                   Filename or handle to print output status to.\n";
-    print STDOUT "                          The default is STDERR\n\n";
+    print STDOUT "   -out name              Filename or handle to print output status to.\n";
+    print STDOUT "   -faillog name          Filename to log import failures to.\n";
+    print STDOUT "                          The default is <collectdir>/colname/etc/fail.log\n\n";
     print STDOUT "  [Type \"perl -S import.pl | more\" if this help text scrolled off your screen]";
     print STDOUT "\n" unless $ENV{'GSDLOS'} =~ /^windows$/i;
@@ -93 +94 @@
     $maxdocs, $collection, $configfilename, $collectcfg, 
     $pluginfo, $sortmeta, $archive_info_filename, 
-    $archive_info, $processor, $out, $collectdir);
+    $archive_info, $processor, $out, $faillogname, $collectdir);
 
     # note that no defaults are passed for most options as they're set
@@ -110 +111 @@
              'maxdocs/^\-?\d+/', \$maxdocs,
              'collectdir/.*/', \$collectdir,
-             'out/.*/STDERR', \$out)) {
+             'out/.*/STDERR', \$out,
+             'faillog/.*/', \$faillogname)) {
     &print_usage();
     die "\n";
@@ -131 +133 @@
     die "\n";
     }
+    
+    # check and/or set fail log file
+    if ($faillogname eq "") { 
+    $faillogname="$ENV{GSDLCOLLECTDIR}/etc/fail.log";
+    } else {
+    my $can_open=1;
+    open (TESTOPEN, ">$faillogname") || ($can_open=0);
+    close (TESTOPEN);
+    if ($can_open==0) {
+        warn "fail.log - cannot write to \"$faillogname\", using default\n \"$ENV{GSDLCOLLECTDIR}/etc/fail.log\" instead.\n";
+        $faillogname="$ENV{GSDLCOLLECTDIR}/etc/fail.log";
+    }
+    }
+    # test that default is writable...
+    if ($faillogname eq "$ENV{GSDLCOLLECTDIR}/etc/fail.log") {
+    my $can_open=1;
+    open (TESTOPEN, ">$faillogname") || ($can_open=0);
+    close (TESTOPEN);
+    if ($can_open==0) {
+        warn "warning - cannot write to \"$faillogname\".\n";
+        $faillogname="";
+    }
+    }
+
 
     # check sortmeta
@@ -149 +175 @@
     # get the list of plugins for this collection and set any options that
     # were specified in the collect.cfg (all import.pl options except
-    # -collectdir and -out may be specified in the collect.cfg (these
+    # -collectdir, -out and -faillog may be specified in the collect.cfg (these
     # options must be known before we read the collect.cfg))
     my $plugins = [];
@@ -232 +258 @@
 
     # load all the plugins
-    $pluginfo = &plugin::load_plugins ($plugins, $verbosity, $out);
+    $pluginfo = &plugin::load_plugins ($plugins, $verbosity, $out, $faillogname);
     if (scalar(@$pluginfo) == 0) {
     print $out "No plugins were loaded.\n";

trunk/gsdl/bin/script/pdftohtml.pl

@@ -45 +45 @@
 # note - we don't actually ever use most of these options...
 print STDERR  
-    ("pdftohtml version 0.22 - modified for NZDL use\n",
+    ("pdftohtml.pl wrapper for pdftohtml version 0.22, modified for GSDL use.\n",
      "Usage: pdftohtml [options] <PDF-file> <html-file>\n",
-     "  -f <int>      : first page to convert\n",
-     "  -l <int>      : last page to convert\n",
-     "  -d <dir>      : target directory (default: basename of pdf-file)\n",
-     "  -o <file>     : name of output file; - means stdout (default index.html)\n",
-     "  -q            : don't print any messages or errors\n",
-     "  -h            : print this usage information\n",
-     "  -p            : exchange .pdf links by .html\n",
-# these options now have no effect in gs-custom pdftohtml
-#     "  -c            : generate complex HTML document\n",
-#     "  -F            : don't use frames in HTML document\n",
-     "  -i            : ignore images\n",
-     "  -e <string>   : set extension for images (in the Html-file) (default png)\n"
+     "Options:\n",
+     "\t-i\tignore images (don't extract)\n",
+     "\t-a\tallow images only (continue even if no text is present)\n"
      );
 exit (1);
@@ -65 +56 @@
 sub main {
     my (@ARGV) = @_;
-    my ($first,$last,$target_dir,$out_file,$img_ext,
-    $optq,$opth,$optp,$optF,$opti);
+    my ($allow_no_text,$ignore_images);
     
     # read command-line arguments so that
     # you can change the command in this script
     if (!parsargv::parse(\@ARGV,
-             'f/\d+/1', \$first,
-             'l/\d+/1', \$last,
-             'd/[\S]*/', \$target_dir,
-             'o/[\S]*/', \$out_file,
-             'e/[\S]*/', \$img_ext,
-             'q', \$optq,
-             'h', \$opth,
-             'p', \$optp,
-#            'c', \$optc,
-             'F', \$optF,
-             'i', \$opti
+             'a', \$allow_no_text,
+             'i', \$ignore_images
              ))
     {
@@ -119 +100 @@
     $cmd = "pdftohtml" if ($ENV{'GSDLOS'} =~ /^windows$/);
 
+    $cmd .= " -i" if ($ignore_images);
     $cmd .= " -noframes \"$input_filename\" \"$output_filestem.html\"";
-    $cmd .= " > \"$output_filestem.out\"";
-
-    # attempting to redirect STDERR on windows 95/98 is a bad idea
-    $cmd .= " 2> \"$output_filestem.err\"" 
-    if $ENV{'GSDLOS'} !~ /^windows$/i;
 
 # system() returns -1 if it can't run, otherwise it's $cmds ret val.
     # note we return 0 if the file is "encrypted"
+    $!=0;
     if (system($cmd)!=0) {
-    print STDERR "Error executing $cmd: $!\n";
+    print STDERR "pdftohtml error for $input_filename $!\n";
     # leave these for gsConvert.pl...
     #&util::rm("$output_filestem.text") if (-e "$output_filestem.text");
     #&util::rm("$output_filestem.err") if (-e "$output_filestem.err");
-    return 0;
+    return 1;
     }
 
     if (! -e "$output_filestem.html") {
-    return 0;
+    return 1;
     }
 
 # post-process to remove </b><b> and </i><i>, as these break up
 # words, screwing up indexing and searching.
+# At the same time, check that our .html file has some textual content.
     &util::mv("$output_filestem.html","$output_filestem.html.tmp");
+    $!=0;
     open INFILE, "$output_filestem.html.tmp" ||
     die "Couldn't open file: $!";
@@ -148 +128 @@
     die "Couldn't open file for writing: $!";
     my $line;
+    my $seen_textual_content=$allow_no_text;
     while ($line=<INFILE>) {
     $line =~ s#</b><b>##g;
     $line =~ s#</i><i>##g;
     $line =~ s#\\#\\\\#g; # until macro language parsing is fixed...
+# check for any extracted text
+    if ($seen_textual_content == 0) {
+        my $tmp_line=$line;
+        $tmp_line =~ s/<[^>]*>//g;
+        $tmp_line =~ s/Page\s\d+//;
+        $tmp_line =~ s/\s*//g;
+        if ($tmp_line ne "") {
+        $seen_textual_content=1;
+        }
+    }
+
 # escape underscores, but not if they're inside tags (eg img/href names)
     my $inatag = 0; # allow multi-line tags
@@ -178 +170 @@
     &util::rm("$output_filestem.html.tmp");
 
-
     # Need to convert images from PPM format to PNG format
     my @images;
@@ -192 +183 @@
     }
     close IMAGES;
+    &util::rm("${directory}image.log") if (-e "${directory}image.log");
+
+    # no need to go any further if there is no text extracted from pdf.
+    if ($seen_textual_content == 0) {
+    print STDERR "Error: PDF contains no extractable text\n";
+    # remove images...
+    for $image (@images) {
+        chomp($image);
+        &util::rm("${directory}$image");
+    }
+    return 1;
+    }
+
+
 
     for $image (@images) {
@@ -200 +205 @@
         if (system($cmd)!=0) {
         print STDERR "Error executing $cmd\n";
-        #return 0; # not sure about whether to leave this one in or take it out
+        #return 1; # not sure about whether to leave this one in or take it out
         next;
         }
@@ -211 +216 @@
         if (system($cmd)!=0) {
             print STDERR "Cannot convert $image into PNG format (tried `pnmtopng' and `convert')...\n";
-            #return 0; # not sure about whether to leave this one in or take it out
+            #return 1; # not sure about whether to leave this one in or take it out
             next;
         }
@@ -219 +224 @@
     }
 
-    return 1;
+    return 0;
 }
 
-# indicate our error status
-if (&main(@ARGV)) {exit 0;}
-exit 1;
+# indicate our error status, 0 = success
+exit (&main(@ARGV));
+

trunk/gsdl/perllib/plugin.pm

@@ -30 +30 @@
 sub load_plugins {
     my ($plugin_list) = shift @_;
-    ($verbosity, $outhandle) = @_; # globals
+    ($verbosity, $outhandle, $faillogname) = @_; # globals
     my @plugin_objects = ();
 
@@ -58 +58 @@
     
     # initialize plugin
-    $plugobj->init($verbosity, $outhandle);
+    $plugobj->init($verbosity, $outhandle, $faillogname);
 
     # add this object to the list

trunk/gsdl/perllib/plugins/BasPlug.pm

@@ -175 +175 @@
 sub init {
     my $self = shift (@_);
-    my ($verbosity, $outhandle) = @_;
+    my ($verbosity, $outhandle, $faillogname) = @_;
 
     # verbosity is passed through from the processor
@@ -182 +182 @@
     # as is the outhandle ...
     $self->{'outhandle'} = $outhandle if defined $outhandle;
+    $self->{'faillogname'} = $faillogname;
 
     # set process_exp and block_exp to defaults unless they were

trunk/gsdl/perllib/plugins/ConvertToPlug.pm

@@ -168 +168 @@
     print $outhandle "Converting $tailname$suffix to $convert_to format\n";
     }
+
+    my $errlog = &util::filename_cat($tmp_dirname, "err.log");
     
     # Execute the conversion command and get the type of the result,
     # making sure the converter gives us the appropriate output type
     my $output_type = lc($convert_to);
-    my $cmd = "perl -S gsConvert.pl -verbose $verbosity -output $output_type \"$tmp_filename\"";
+    my $cmd = "perl -S gsConvert.pl -verbose $verbosity -errlog \"$errlog\" -output $output_type \"$tmp_filename\"";
     $output_type = `$cmd`;
 
@@ -182 +184 @@
     if ($output_type eq "fail") {
     print $outhandle "Could not convert $tailname$suffix to $convert_to format\n";
+    if ($self->{'faillogname'} ne "" && -s "$errlog") {
+        open(SAVELOG, ">>$self->{'faillogname'}");
+        open(ERRLOG, "$errlog");
+        print SAVELOG "$tailname$suffix (converting to $convert_to) failed:\n";
+        while (<ERRLOG>) {
+        print SAVELOG "$_";
+        }
+        close ERRLOG;
+        print SAVELOG "\n";
+        close SAVELOG;
+    }
+    &util::rm("$errlog") if (-e "$errlog");
     return "";
     }