root/main/trunk/greenstone2/bin/script/gsConvert.pl @ 24362

Revision 24362, 46.6 KB (checked in by ak19, 8 years ago)

The method of locating perl has changed once more: util now defines the fuction get_perl_exec which is used by other scripts to obtain the path to the perl executable they should use.

  • Property svn:executable set to *
  • Property svn:keywords set to Author Date Id Revision
Line 
1#!/usr/bin/perl -w
2
3###########################################################################
4#
5# gsConvert.pl -- convert documents to HTML or TEXT format
6#
7# A component of the Greenstone digital library software
8# from the New Zealand Digital Library Project at the
9# University of Waikato, New Zealand.
10#
11# Copyright (C) 1999-2002 New Zealand Digital Library Project
12#
13# This program is free software; you can redistribute it and/or modify
14# it under the terms of the GNU General Public License as published by
15# the Free Software Foundation; either version 2 of the License, or
16# (at your option) any later version.
17#
18# This program is distributed in the hope that it will be useful,
19# but WITHOUT ANY WARRANTY; without even the implied warranty of
20# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
21# GNU General Public License for more details.
22#
23# You should have received a copy of the GNU General Public License
24# along with this program; if not, write to the Free Software
25# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
26#
27###########################################################################
28
29# gsConvert.pl converts documents in a range of formats to HTML or TEXT
30# by exploiting third-party programs.  The sources of these are usually found
31# in the $GSDLHOME/packages directory, and the executables should live in
32# $GSDLHOME/bin/$GSDLOS (which is on the search path).
33#
34# Currently, we can convert the following formats by using external
35# conversion utilities:
36# Microsoft Word (versions 2,6,7 [==95?], 8[==97?], 9[==2000?]), RTF,
37# Adobe PDF, PostScript, MS PowerPoint (95 and 97), and MS Excel (95 and 97).
38#
39# We can try to convert any file to text with a perl implementation of the
40# UNIX strings command.
41#
42# We try to convert Postscript files to text using "gs" which is often on
43# *nix machines. We fall back to performing weak text extraction by using
44# regular expressions.
45
46BEGIN {
47    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
48    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
49}
50
51use strict;
52
53use parsargv;
54use util;
55use Cwd;
56
57# Are we running on WinNT or Win2000 (or later)?
58my $is_winnt_2000=eval {require Win32; return (Win32::IsWinNT()); return 0;};
59if (!defined($is_winnt_2000)) {$is_winnt_2000=0;}
60
61my $use_strings;
62my $pdf_complex;
63my $pdf_nohidden;
64my $pdf_zoom;
65my $pdf_ignore_images;
66my $pdf_allow_images_only;
67my $windows_scripting;
68
69sub print_usage
70{
71    print STDERR "\n";
72    print STDERR "gsConvert.pl: Converts documents in a range of formats to html\n";
73    print STDERR "              or text using third-party programs.\n\n";
74    print STDERR "  usage: $0 [options] filename\n";
75    print STDERR "  options:\n\t-type\tdoc|dot|pdf|ps|ppt|rtf|xls\t(input file type)\n";
76    print STDERR "\t-errlog\t<filename>\t(append err messages)\n";
77    print STDERR "\t-output\tauto|html|text|pagedimg_jpg|pagedimg_gif|pagedimg_png\t(output file type)\n";
78    print STDERR "\t-timeout\t<max cpu seconds>\t(ulimit on unix systems)\n";
79    print STDERR "\t-use_strings\tuse strings to extract text if conversion fails\n";
80    print STDERR "\t-windows_scripting\tuse windows VB script (if available) to convert Microsoft Word and PPT documents\n";
81    print STDERR "\t-pdf_complex\tuse complex output when converting PDF to HTML\n";
82    print STDERR "\t-pdf_nohidden\tDon't attempt to extract hidden text from PDF files\n";
83    print STDERR "\t-pdf_ignore_images\tdon't attempt to extract images when\n";
84    print STDERR "\t\tconverting PDF to HTML\n";
85    print STDERR "\t-pdf_allow_images_only\tallow images only (continue even if no text is present when converting to HTML)\n";
86    print STDERR "\t-pdf_zoom\tfactor by which to zoom PDF (only useful if\n";
87    print STDERR "\t\t-pdf_complex is set\n";
88    exit(1);
89}
90
91my $faillogfile="";
92my $timeout=0;
93
94sub main
95{
96    my (@ARGV) = @_;
97    my ($input_type,$output_type,$verbose);
98
99    # Dynamically figure out what the --type option can support, based on whether -windows_scripting
100    # is in use or not
101    my $default_type_re = "(doc|dot|pdf|ps|ppt|rtf|xls)";
102    #my $enhanced_type_re = "(docx?|dot|pdf|ps|pptx?|rtf|xlsx?)";
103    #my $enhanced_type_re = "(docx?|dot|pdf|ps|pptx?|rtf|xlsx?)";
104    # Currently only have VBA for Word and PPT(but no XLS)
105    my $enhanced_type_re = "(docx?|dot|pdf|ps|pptx?|rtf|xls)";
106
107    my $type_re = $default_type_re;
108   
109    foreach my $a (@ARGV) {
110        if ($a =~ m/^windows_scripting$/i) {
111            $type_re = $enhanced_type_re;
112        }
113    }
114   
115    # read command-line arguments
116    if (!parsargv::parse(\@ARGV,
117             "type/$type_re/", \$input_type,
118             '/errlog/.*/', \$faillogfile,
119             'output/(auto|html|text|pagedimg).*/', \$output_type,
120             'timeout/\d+/0',\$timeout,
121             'verbose/\d+/0', \$verbose,
122             'windows_scripting',\$windows_scripting,
123             'use_strings', \$use_strings,
124             'pdf_complex', \$pdf_complex,
125             'pdf_ignore_images', \$pdf_ignore_images,
126             'pdf_allow_images_only', \$pdf_allow_images_only,
127             'pdf_nohidden', \$pdf_nohidden,
128             'pdf_zoom/\d+/2', \$pdf_zoom
129             ))
130    {
131    print_usage();
132    }
133     
134    # Make sure the input file exists and can be opened for reading
135    if (scalar(@ARGV!=1)) {
136    print_usage();
137    }
138
139    my $input_filename = $ARGV[0];
140    if (!-r $input_filename) {
141    print STDERR "Error: unable to open $input_filename for reading\n";
142    exit(1);
143    }
144
145    # Deduce filenames
146    my ($tailname,$dirname,$suffix)
147    = File::Basename::fileparse($input_filename, "\\.[^\\.]+\$");
148    my $output_filestem = &util::filename_cat($dirname, "$tailname");
149
150    if ($input_type eq "")
151    {
152    $input_type = lc (substr($suffix,1,length($suffix)-1));
153    }
154   
155    # Change to temporary working directory
156    my $stored_dir = cwd();
157    chdir ($dirname) || die "Unable to change to directory $dirname";
158
159    # Select convert utility
160    if (!defined $input_type) {
161    print STDERR "Error: No filename extension or input type defined\n";
162    exit(1);
163    }
164    elsif ($input_type =~ m/^docx?$/ || $input_type eq "dot") {
165    print &convertDOC($input_filename, $output_filestem, $output_type);
166    print "\n";
167    }
168    elsif ($input_type eq "rtf") {
169    print &convertRTF($input_filename, $output_filestem, $output_type);
170    print "\n";
171    }
172    elsif ($input_type eq "pdf") {
173    print &convertPDF($dirname, $input_filename, $output_filestem, $output_type);
174    print "\n";
175    }
176    elsif ($input_type eq "ps") {
177    print &convertPS($dirname, $input_filename, $output_filestem, $output_type);
178    print "\n";
179    }
180    elsif ($input_type =~ m/pptx?$/) {
181    print &convertPPT($input_filename, $output_filestem, $output_type);
182    print "\n";
183    }
184    elsif ($input_type =~ m/xlsx?$/) {
185    print &convertXLS($input_filename, $output_filestem, $output_type);
186    print "\n";
187    }
188    else {
189    print STDERR "Error: Unable to convert type '$input_type'\n";
190    exit(1);
191    }
192   
193    # restore to original working directory
194    chdir ($stored_dir) || die "Unable to return to directory $stored_dir";
195
196}
197
198&main(@ARGV);
199
200
201
202# Document-type conversion functions
203#
204# The following functions attempt to convert documents from their
205# input type to the specified output type.  If no output type was
206# given, then they first attempt HTML, and then TEXT.
207#
208# Each returns the output type ("html" or "text") or "fail" if no
209# conversion is possible.
210
211# Convert a Microsoft word document
212
213sub convertDOC {
214    my ($input_filename, $output_filestem, $output_type) = @_;
215
216    # Many .doc files are not in fact word documents!
217    my $realtype = &find_docfile_type($input_filename);
218
219    if ($realtype eq "word6" || $realtype eq "word7"
220        || $realtype eq "word8" || $realtype eq "docx") {
221    return &convertWord678($input_filename, $output_filestem, $output_type);
222    } elsif ($realtype eq "rtf") {
223    return &convertRTF($input_filename, $output_filestem, $output_type);
224    } else {
225    return &convertAnything($input_filename, $output_filestem, $output_type);
226    }
227}
228
229# Convert a Microsoft word 6/7/8 document
230
231sub convertWord678 {
232    my ($input_filename, $output_filestem, $output_type) = @_;
233
234    my $success = 0;
235    if (!$output_type || ($output_type =~ m/html/i)){
236    if ($windows_scripting) {
237        $success = &native_doc_to_html($input_filename, $output_filestem);
238    }
239    else {
240        $success = &doc_to_html($input_filename, $output_filestem);   
241    }
242    if ($success) {
243       return "html";
244    }
245    }
246    return &convertAnything($input_filename, $output_filestem, $output_type);
247}
248
249
250# Convert a Rich Text Format (RTF) file
251
252sub convertRTF {
253    my ($input_filename, $output_filestem, $output_type) = @_;
254
255    my $success = 0;
256
257    # Attempt specialised conversion to HTML
258    if (!$output_type || ($output_type =~ m/html/i)) {
259
260    if ($windows_scripting) {
261        $success = &native_doc_to_html($input_filename, $output_filestem);
262    }
263    else {
264        $success = &rtf_to_html($input_filename, $output_filestem);
265    }
266    if ($success) {
267        return "html";
268    }
269    }
270
271# rtf is so ugly that's it's not worth running strings over.
272# One day I'll write some quick'n'dirty regexps to try to extract text - jrm21
273#    return &convertAnything($input_filename, $output_filestem, $output_type);
274    return "fail";
275}
276
277
278# Convert an unidentified file
279
280sub convertAnything {
281    my ($input_filename, $output_filestem, $output_type) = @_;
282   
283    my $success = 0;
284 
285    # Attempt simple conversion to HTML
286    if (!$output_type || ($output_type =~ m/html/i)) {
287    $success = &any_to_html($input_filename, $output_filestem);
288    if ($success) {
289        return "html";
290    }
291    }
292
293    # Convert to text
294    if (!$output_type || ($output_type =~ m/text/i)) {
295    $success = &any_to_text($input_filename, $output_filestem);
296    if ($success) {
297        return "text";
298    }
299    }
300    return "fail";
301}
302
303
304
305# Convert an Adobe PDF document
306
307sub convertPDF {
308    my ($dirname, $input_filename, $output_filestem, $output_type) = @_;
309
310    my $success = 0;
311    $output_type =~ s/.*\-(.*)/$1/i;
312    # Attempt coversion to Image
313    if ($output_type =~ m/jp?g|gif|png/i) {
314    $success = &pdfps_to_img($dirname, $input_filename, $output_filestem, $output_type);
315    if ($success){
316        return "item";
317    }
318    }
319
320    # Attempt conversion to HTML
321    if (!$output_type || ($output_type =~ m/html/i)) {
322    $success = &pdf_to_html($dirname, $input_filename, $output_filestem);
323    if ($success) {
324        return "html";
325    }
326    }
327
328    # Attempt conversion to TEXT
329    if (!$output_type || ($output_type =~ m/text/i)) {
330    $success = &pdf_to_text($dirname, $input_filename, $output_filestem);
331    if ($success) {
332        return "text";
333    }
334    }
335
336    return "fail";
337
338}
339
340
341# Convert an Adobe PostScript document
342
343sub convertPS {
344    my ($dirname,$input_filename, $output_filestem, $output_type) = @_;
345
346    my $success = 0;
347    $output_type =~ s/.*\-(.*)/$1/i;
348    # Attempt coversion to Image
349    if ($output_type =~ m/jp?g|gif|png/i) {
350    $success = &pdfps_to_img($dirname, $input_filename, $output_filestem, $output_type);
351    if ($success){
352        return "item";
353    }
354    }
355
356    # Attempt conversion to TEXT
357    if (!$output_type || ($output_type =~ m/text/i)) {
358    $success = &ps_to_text($input_filename, $output_filestem);
359    if ($success) {
360        return "text";
361    }
362    }
363    return "fail";
364}
365
366
367sub convertPPT {
368    my ($input_filename, $output_filestem, $output_type) = @_;
369    my $success = 0;
370
371    my $ppt_convert_type = "";
372
373    #if (!$output_type || $windows_scripting || ($output_type !~ m/html/i) || ($output_type !~ m/text/i)){
374    if ($windows_scripting && ($output_type !~ m/html/i) && ($output_type !~ m/text/i)){
375    if ($output_type =~ m/gif/i) {
376        $ppt_convert_type = "-g";
377    } elsif ($output_type =~ m/jp?g/i){
378        $ppt_convert_type = "-j";
379    } elsif ($output_type =~ m/png/i){
380        $ppt_convert_type = "-p";
381    }
382    my $vbScript = &util::filename_cat($ENV{'GSDLHOME'}, "bin",
383                       $ENV{'GSDLOS'}, "pptextract");
384    $vbScript = "pptextract" if ($ENV{'GSDLOS'} =~ m/^windows$/i);
385           
386    my $cmd = "";
387    if ($timeout) {$cmd = "ulimit -t $timeout;";}
388    # if the converting directory already exists
389    if (-d $output_filestem) {
390        print STDERR "**The conversion directory already exists\n";
391        return "item";
392    } else {
393        $cmd .=  "$vbScript $ppt_convert_type \"$input_filename\" \"$output_filestem\"";
394        $cmd .= " 2>\"$output_filestem.err\""
395        if ($ENV{'GSDLOS'} !~ m/^windows$/i || $is_winnt_2000);
396        if (system($cmd) !=0) {
397        print STDERR "Powerpoint VB Scripting convert failed\n";
398        } else {
399        return "item";
400        }
401    }
402    } elsif (!$output_type || ($output_type =~ m/html/i)) {
403    # Attempt conversion to HTML
404    #if (!$output_type || ($output_type =~ m/html/i)) {
405    # formulate the command
406    my $cmd = "";
407    my $full_perl_path = &util::get_perl_exec();
408    $cmd .= "\"$full_perl_path\" -S ppttohtml.pl ";
409    $cmd .= " \"$input_filename\" \"$output_filestem.html\"";
410    $cmd .= " 2>\"$output_filestem.err\""
411        if ($ENV{'GSDLOS'} !~ m/^windows$/i || $is_winnt_2000);
412
413    # execute the command
414    $!=0;
415    if (system($cmd)!=0)
416    {
417        print STDERR "Powerpoint 95/97 converter failed $!\n";
418    } else {
419        return "html";
420    }
421    }
422
423    $success = &any_to_text($input_filename, $output_filestem);
424    if ($success) {
425    return "text";
426    }
427   
428    return "fail";
429}
430
431
432sub convertXLS {
433    my ($input_filename, $output_filestem, $output_type) = @_;
434
435    my $success = 0;
436
437    # Attempt conversion to HTML
438    if (!$output_type || ($output_type =~ m/html/i)) {
439    # formulate the command
440    my $cmd = "";
441    my $full_perl_path = &util::get_perl_exec();
442    $cmd .= "\"$full_perl_path\" -S xlstohtml.pl ";
443    $cmd .= " \"$input_filename\" \"$output_filestem.html\"";
444    $cmd .= " 2>\"$output_filestem.err\""
445        if ($ENV{'GSDLOS'} !~ m/^windows$/i || $is_winnt_2000);
446   
447   
448    # execute the command
449    $!=0;
450    if (system($cmd)!=0)
451    {
452        print STDERR "Excel 95/97 converter failed $!\n";
453    } else {
454        return "html";
455    }
456    }
457
458    $success = &any_to_text($input_filename, $output_filestem);
459    if ($success) {
460    return "text";
461    }
462
463    return "fail";
464}
465
466
467
468# Find the real type of a .doc file
469#
470# We seem to have a lot of files with a .doc extension that are .rtf
471# files or Word 5 files.  This function attempts to tell the difference.
472sub find_docfile_type {
473    my ($input_filename) = @_;
474   
475    if (($windows_scripting) && ($input_filename =~ m/\.docx$/)) {
476        return "docx";
477    }
478   
479    open(CHK, "<$input_filename");
480    binmode(CHK);
481    my $line = "";
482    my $first = 1;
483
484    while (<CHK>) {
485   
486    $line = $_;
487
488    if ($first) {
489        # check to see if this is an rtf file
490        if ($line =~ m/^\{\\rtf/) {
491        close(CHK);
492        return "rtf";
493        }
494        $first = 0;
495    }
496   
497    # is this is a word 6/7/8 document?
498    if ($line =~ m/Word\.Document\.([678])/) {
499        close(CHK);
500
501        return "word$1";
502    }
503
504    }
505
506    return "unknown";
507}
508
509
510# Specific type-to-type conversions
511#
512# Each of the following functions attempts to convert a document from
513# a specific format to another.  If they succeed they return 1 and leave
514# the output document(s) in the appropriate place; if they fail they
515# return 0 and delete any working files.
516
517
518# Attempt to convert a word document to html with the wv program
519sub doc_to_html {
520    my ($input_filename, $output_filestem) = @_;
521
522    my $wvWare = &util::filename_cat($ENV{'GSDLHOME'}, "bin", $ENV{'GSDLOS'}, "wvWare");
523
524    if ( -d "$ENV{'GSDLHOME'}/bin/$ENV{'GSDLOS'}/wv" && $ENV{'GSDLOS'} eq "linux" ) {
525        $ENV{'PATH'} = "$ENV{'GSDLHOME'}/bin/$ENV{'GSDLOS'}/wv/bin:$ENV{'PATH'}";
526        $ENV{'LD_LIBRARY_PATH'} = "$ENV{'GSDLHOME'}/bin/$ENV{'GSDLOS'}/wv/lib:$ENV{'LD_LIBRARY_PATH'}";
527        $wvWare = &util::filename_cat($ENV{'GSDLHOME'}, "bin", $ENV{'GSDLOS'}, "wv", "bin", "wvWare");
528    }
529
530    # don't include path on windows (to avoid having to play about
531    # with quoting when GSDLHOME might contain spaces) but assume
532    # that the PATH is set up correctly
533    $wvWare = "wvWare" if ($ENV{'GSDLOS'} =~ m/^windows$/i);
534
535    my $wv_conf = &util::filename_cat($ENV{'GSDLHOME'}, "etc",
536                      "packages", "wv", "wvHtml.xml");
537   
538    # Added the following to work with replace_srcdoc_with_html.pl:
539    # Make wvWare put any associated (image) files of the word doc into
540    # folder docname-without-extention_files. This folder should be at
541    # the same level as the html file generated from the doc.
542    # wvWare will take care of proper interlinking.
543
544    # This step is necessary for replace_srcdoc_with_html.pl which will
545    # move the html and associated files into the import folder. We
546    # want to ensure that the associated files won't overwrite similarly
547    # named items already in import. Hence we put them in a folder first
548    # (to which the html links properly) and that will allow
549    # replace_srcdoc_with_html.pl to move them safely to /import.
550
551    # To do all this, we need to use wvWare's --dir and --basename options
552    # where dir is the full path to the image folder directory and
553    # basename is the full path to the image folder appended to the name
554    # which is to be prepended to every image file:
555    # eg. if the images were to have names like sample0.jpg to sampleN.jpg,
556    # then the basename is "/full/path/to/imgdir/sample".
557    # In this case, basename is the full path to and name of the document.
558    # HOWEVER: basename always takes full path, not relative url, so
559    # the greenstone browser is unable to display the images (absolute paths
560    # cause it to give an "external link" message)
561    # See http://osdir.com/ml/lib.wvware.devel/2002-11/msg00014.html
562    # and http://rpmfind.net/linux/RPM/freshmeat/rpms/wv/wv-0.5.44-1.i386.html
563    # "added --dir option to wvHtml so that pictures can be placed in
564    # a seperate directory"
565    # "running wvWare through IMP to view word documents as html. It gets
566    # invoked like this:
567    # wvWare --dir=/tmp-wvWare --basename=/tmp-wvWare/img$$- $tmp_word >$tmp_output"
568   
569    # toppath is the folder where html is generated
570    # docname is the name (without extension) of the html to be generated
571    # suffix (extension) is thrown away
572    my ($docname, $toppath)
573    = &File::Basename::fileparse($input_filename, "\\.[^\\.]+\$");
574
575    # We want the image folder generated to have the same name as windows
576    # would generate ($windows_scripting) when it converts from word to html.
577    # That is, foldername=docname_files
578    my $assoc_dir = &util::filename_cat($toppath, $docname."_files");
579    #print "assoc_dir: ".$assoc_dir."\n";  # same as "$output_filestem._files"
580   
581    # ensure this image directory exists
582    # if it exists already, just delete and recreate
583    if(-e $assoc_dir) {
584    &util::rm_r($assoc_dir);
585    } 
586    &util::mk_dir($assoc_dir);
587
588    # the images are all going to be called image0, image1,..., imageN
589    my $img_basenames = &util::filename_cat($assoc_dir, $docname);
590   
591    #print STDERR "****toppath: $toppath\n****docname: $docname\n;
592    #print STDERR "****img_basenames: $img_basenames\n" if($img_basenames);
593    #print STDERR "****assoc_dir: $assoc_dir\n" if($assoc_dir);
594
595    my $cmd = "";
596    if ($timeout) {$cmd = "ulimit -t $timeout;";}
597    # wvWare's --dir and --basename options for image directory.
598    # Replaced the next line with the *2 lines* following it:
599               # $cmd .= "$wvWare --charset utf-8 --config \"$wv_conf\"";
600    $cmd .= "$wvWare --dir \"$assoc_dir\" --basename \"$img_basenames\"";
601    $cmd .= " --charset utf-8 --config \"$wv_conf\"";
602    $cmd .= " \"$input_filename\" > \"$output_filestem.html\"";
603
604    # redirecting STDERR is a bad idea on windows 95/98
605    $cmd .= " 2> \"$output_filestem.err\""
606    if ($ENV{'GSDLOS'} !~ m/^windows$/i || $is_winnt_2000);
607    # execute the command
608    $!=0;
609    if (system($cmd)!=0)
610    {
611    print STDERR "Error executing wv converter:$!\n";
612    if (-s "$output_filestem.err") {
613        open (ERRFILE, "<$output_filestem.err");
614
615        my $write_to_fail_log=0;
616        if ($faillogfile ne "" && defined(open(FAILLOG,">>$faillogfile")))
617        {$write_to_fail_log=1;}
618
619        my $line;
620        while ($line=<ERRFILE>) {
621        if ($line =~ m/\w/) {
622            print STDERR "$line";
623            print FAILLOG "$line" if ($write_to_fail_log);
624        }
625        if ($line !~ m/startup error/) {next;}
626        print STDERR " (given an invalid .DOC file?)\n";
627        print FAILLOG " (given an invalid .DOC file?)\n"
628        if ($write_to_fail_log);
629       
630        } # while ERRFILE
631        close FAILLOG if ($write_to_fail_log);
632    }
633    return 0; # we can try any_to_text
634    }
635
636    # Was the conversion successful?
637
638    if (-s "$output_filestem.html") { # if file has non-zero size (i.e. it has contents)
639    open(TMP, "$output_filestem.html");
640    my $line = <TMP>;
641    close(TMP);
642    if ($line && $line =~ m/DOCTYPE HTML/) {
643        &util::rm("$output_filestem.err") if -e "$output_filestem.err";   
644
645        # Inserted this code to remove the images directory if it was still empty after
646        # the html was generated (in case there were no images in the word document)
647        if (&util::is_dir_empty($assoc_dir)) {
648        #print STDERR "***gsConvert.pl: Image dir $assoc_dir is empty, removing***\n";
649        &util::rm_r($assoc_dir);
650        } else { # there was an image folder (it was generated)
651        # Therefore, the html file generated contains absolute links to the images
652        # Replace them with relative links instead, so the folder can be moved elsewhere
653        &make_links_to_assocdir_relative($toppath, $docname, "$output_filestem.html", $assoc_dir, $docname."_files");   
654        }
655        return 1;
656    }
657    }
658   
659    # If here, an error of some sort occurred
660    &util::rm("$output_filestem.html") if -e "$output_filestem.html";
661    if (-e "$output_filestem.err") {
662    if ($faillogfile ne "" && defined(open(FAILLOG,">>$faillogfile"))) {
663        open (ERRLOG,"$output_filestem.err");
664        while (<ERRLOG>) {print FAILLOG $_;}
665        close FAILLOG;
666        close ERRLOG;
667    }
668    &util::rm("$output_filestem.err");
669    }
670   
671    return 0;
672}
673
674# Method to work with doc_to_html - Word docs might contain images.
675# When such word docs are converted with wvWare, we make it generate a
676# <filename>_files folder with the associated images, while the html file
677# <filename> refers to the images using absolute paths to <filename>_files.
678# This method reads in that html file and replaces all the absolute paths to
679# the images in <filename>_files with the relative paths to the images from
680# that folder. (I.e. with <filename>_files/<imagename.ext>).
681sub make_links_to_assocdir_relative{
682    # toppath is the top-level folder in which the html file we're going to be fixing resides
683    # docname is just the name (without extension) of the html file
684    # html_file is the full path to the html file: /full/path/docname.html
685    # assoc_dir_path is toppath/docname_files
686    # assoc_dirname is the directory name of the folder with associated imgs: docname_files
687    my ($toppath, $docname, $html_file, $assoc_dir_path, $assoc_dirname) = @_;
688
689    # 1. Read all the contents of the html into a string
690    # open the original file for reading
691    unless(open(FIN, "<$html_file")) {
692    print STDERR "gsConvert.pl: Unable to open $html_file for reading absolute urls...ERROR: $!\n";
693    return 0;
694    }
695    # From http://perl.plover.com/local.html
696    # "It's cheaper to read the file all at once, without all the splitting and reassembling.
697    # (Some people call this slurping the file.) Perl has a special feature to support this:
698    # If the $/ variable is undefined, the <...> operator will read the entire file all at once"
699    my $html_contents;
700    {
701    local $/ = undef;        # Read entire file at once
702    $html_contents = <FIN>;  # Now file is read in as one single 'line'
703    }
704    close(FIN); # close the file
705    #print STDERR $html_contents;
706   
707    # 2. Replace (substitute) *all* ocurrences of the assoc_dir_path in a hrefs and img src
708    # values with assoc_dirname
709    # At the end: g means substitute all occurrences (global), while s at the end means treat
710    # all new lines as a regular space. This interacts with g to consider all the lines
711    # together as a single line so that multi-occurrences can be replaced.
712
713    # we can't just replace $assoc_dir_path with $assoc_dir
714    # $assoc_dir_path represents a regular expression that needs to be replaced
715    # if it contains ., -, [, ], or Windows style backslashes in paths  -- which all have special
716    # meaning in Perl regular expressions -- we need to escape these first
717    my $safe_reg_expression = $assoc_dir_path;
718    $safe_reg_expression =~ s/\\/\\\\/g;
719    $safe_reg_expression =~ s/\./\\./g;
720    $safe_reg_expression =~ s/\-/\\-/g;
721    $safe_reg_expression =~ s/\[/\\[/g;
722    $safe_reg_expression =~ s/\]/\\]/g;
723    $safe_reg_expression =~ s/ /%20/g; # wvWare put %20 in place of space, so we need to change our prefix to match
724
725    # The following regular expression substitution looks for <a or <image, followed by any other
726    # attributes and values until it comes to the FIRST (indicated by ?) href= or src=
727    # followed by " or ' no quotes at all around path, followed by the associated folder's pathname
728    # followed by characters (for the img filename), then finally the optional closing quotes
729    # in " or ' form, followed by any other attributes and values until the first > to end the tag.
730    # The substitution: all the parts preceding associated folder's pathname are retained,
731    # the associated folder path name is replaced by associated folder directory name
732    # and the rest upto and including the closing > tag is retained.
733    # The sg at the end of the pattern match treats all of html_contents as a single line (s)
734    # and performs a global replace (g) meaning that all occurrences that match in that single line
735    # are substituted.
736    $html_contents =~ s/(<(a|img).*?(href|src)=(\"|\')?)$safe_reg_expression(.*?(\"|\')?.*?>)/$1$assoc_dirname$5/sg;
737               #$html_contents =~ s/$safe_reg_expression/$assoc_dirname/gs; # this works, used as fall-back
738    # now replace any %20 chars in filenames of href or src attributes to use literal space ' '. Calls a function for this
739    $html_contents =~ s/(<(a|img).*?(href|src)=(\"|\')?)(.*)(.*?(\"|\')?.*?>)/&post_process_assocfile_urls($1, $5, $6)/sge;
740
741    #print STDERR "****assoc_dirname: $assoc_dirname***\n";
742    #print STDERR "****safe_reg_expression: $safe_reg_expression***\n";
743   
744    # delete the original file and recreate it
745    my $copy_of_filename = $html_file;
746    &util::rm($copy_of_filename); # deleted the file
747
748    # Recreate the original file for writing the updated contents
749    unless(open(FOUT, ">$html_file")) {  # open it as a new file for writing
750    print STDERR "gsConvert.pl: Unable to open $html_file for writing relative links...ERROR: $!\n";
751    return 0;
752    }
753
754    # write out the updated contents and close the file
755    print FOUT $html_contents;
756    close(FOUT);
757    return 1;
758}
759
760# Utility routine to make sure HTML plugin gets img src/href link pathnames that contain
761# url slashes (/) instead of windows-style backwards slashes, and to convert all %20
762# introduced in link pathnames by wvWare into space again. Converts all percent signs
763# introduced by URL encoding filenames generated into %25 in these url links referencing them
764sub post_process_assocfile_urls
765{
766    my ($pre, $text, $post) = @_;
767
768    $text =~ s/%20/ /g; # Convert %20s to space and not underscore since underscores mess with incremental rebuild
769    # $text =~ s/%20/_/g; # reinstated this line, since we no longer replace spaces with %20. We replace them with underscores
770    $text =~ s/\\/\//g;
771    $text =~ s/%/%25/g;
772
773    return "$pre$text$post";
774}
775
776# Attempt to convert a word document to html with the word2html scripting program
777sub native_doc_to_html {
778    my ($input_filename, $output_filestem) = @_;
779
780    # build up the path to the doc-to-html conversion tool we're going to use
781    my $vbScript = &util::filename_cat($ENV{'GSDLHOME'}, "bin", $ENV{'GSDLOS'});
782
783    if ($ENV{'GSDLOS'} =~ m/^windows$/i) {
784        # if windows scripting with docx input, use new VBscript to get the local Word install (if
785        # any) to do the conversion, since docX can't be processed by word2html's windows_scripting
786       
787        if($input_filename =~ m/docx$/i) {  # need to use full path to docx2html script,
788                                            # else script launch fails when there are error msgs
789            $vbScript = &util::filename_cat($vbScript, "docx2html.vbs");
790            $vbScript = "CScript //Nologo \"$vbScript\"";   # launche with CScript for error output in STDERR
791                                    # //Nologo flag avoids Microsoft's opening/logo msgs
792            print STDERR "About to use windows scripting to process docx file $input_filename.\n";
793            print STDERR "   This may take some time. Please wait...\n";
794        }
795        else {  # old doc versions. use the usual VB executable word2html for the
796                # conversion. Doesn't need full path, since bin\windows is on PATH         
797            $vbScript = "word2html"; #$vbScript = "\"".&util::filename_cat($vbScript, "word2html")."\"";
798        }
799    }
800    else { # not windows
801        $vbScript = "\"".&util::filename_cat($vbScript, "word2html")."\"";
802    }
803
804    if (-e "$output_filestem.html") {
805    print STDERR "    The conversion file:\n";
806    print STDERR "      $output_filestem.html\n";
807    print STDERR "    ... already exists.  Skipping\n";
808    return 1;
809    }
810
811    my $cmd = "";
812    if ($timeout) {$cmd = "ulimit -t $timeout;";}
813    #$cmd .= "$vbScript \"$input_filename\" \"$output_filestem.html\"";
814    #$cmd .=  "$vbScript $input_filename $output_filestem.html";
815    $cmd .=  "$vbScript \"$input_filename\" \"$output_filestem.html\"";
816
817    # redirecting STDERR
818   
819    $cmd .= " 2> \"$output_filestem.err\""
820        if ($ENV {'GSDLOS'} !~ m/^windows$/i || $is_winnt_2000);   
821    #print STDERR "@@@@@@@@@ cmd=$cmd\n";
822   
823    # execute the command
824    $!=0;
825    if (system($cmd)!=0)
826    {
827    print STDERR "Error executing $vbScript converter:$!\n";
828    if (-s "$output_filestem.err") {
829        open (ERRFILE, "<$output_filestem.err");
830       
831        my $write_to_fail_log=0;
832        if ($faillogfile ne "" && defined(open(FAILLOG,">>$faillogfile")))
833        {$write_to_fail_log=1;}
834
835        my $line;
836        while ($line=<ERRFILE>) {
837        if ($line =~ m/\w/) {
838            print STDERR "$line";
839            print FAILLOG "$line" if ($write_to_fail_log);
840        }
841        if ($line !~ m/startup error/) {next;}
842        print STDERR " (given an invalid .DOC file?)\n";
843        print FAILLOG " (given an invalid .DOC file?)\n"
844        if ($write_to_fail_log);
845       
846        } # while ERRFILE
847        close FAILLOG if ($write_to_fail_log);
848    }
849    return 0; # we can try any_to_text
850    }
851
852    # Was the conversion successful?
853    if (-s "$output_filestem.html") {
854    open(TMP, "$output_filestem.html");
855    my $line = <TMP>;
856    close(TMP);
857    if ($line && $line =~ m/html/i) {
858        &util::rm("$output_filestem.err") if -e "$output_filestem.err";
859        return 1;
860    }
861    }
862   
863    # If here, an error of some sort occurred
864    &util::rm("$output_filestem.html") if -e "$output_filestem.html";
865    if (-e "$output_filestem.err") {
866    if ($faillogfile ne "" && defined(open(FAILLOG,">>$faillogfile"))) {
867        open (ERRLOG,"$output_filestem.err");
868        while (<ERRLOG>) {print FAILLOG $_;}
869        close FAILLOG;
870        close ERRLOG;
871    }
872    &util::rm("$output_filestem.err");
873    }
874    return 0;
875}
876
877# Attempt to convert an RTF document to html with rtftohtml
878sub rtf_to_html {
879    my ($input_filename, $output_filestem) = @_;
880
881    # formulate the command
882    my $cmd = "";
883    if ($timeout) {$cmd = "ulimit -t $timeout;";}
884    $cmd .= "rtftohtml";
885    #$cmd .= "rtf-converter";
886
887    $cmd .= " -o \"$output_filestem.html\" \"$input_filename\"";
888
889    $cmd .= " 2>\"$output_filestem.err\""
890        if ($ENV{'GSDLOS'} !~ m/^windows$/i || $is_winnt_2000);
891
892
893    # execute the command
894    $!=0;
895    if (system($cmd)!=0)
896    {
897    print STDERR "Error executing rtf converter $!\n";
898    # don't currently bother printing out error log...
899    # keep going, in case it still created an HTML file...
900    }
901
902    # Was the conversion successful?
903    my $was_successful=0;
904    if (-s "$output_filestem.html") {
905    # make sure we have some content other than header
906    open (HTML, "$output_filestem.html"); # what to do if fail?
907    my $line;
908    my $past_header=0;
909    while ($line=<HTML>) {
910
911        if ($past_header == 0) {
912        if ($line =~ m/<body>/) {$past_header=1;}
913        next;
914        }
915
916        $line =~ s/<[^>]+>//g;
917        if ($line =~ m/\w/ && $past_header) {  # we found some content...
918        $was_successful=1;
919        last;
920        }
921    }
922    close HTML;
923    }
924
925    if ($was_successful) {
926    &util::rm("$output_filestem.err")
927        if (-e "$output_filestem.err");
928    # insert the (modified) table of contents, if it exists.
929    if (-e "${output_filestem}_ToC.html") {
930        &util::mv("$output_filestem.html","$output_filestem.src");
931        my $open_failed=0;
932        open HTMLSRC, "$output_filestem.src" || ++$open_failed;
933        open TOC, "${output_filestem}_ToC.html" || ++$open_failed;
934        open HTML, ">$output_filestem.html" || ++$open_failed;
935       
936        if ($open_failed) {
937        close HTMLSRC;
938        close TOC;
939        close HTML;
940        &util::mv("$output_filestem.src","$output_filestem.html");
941        return 1;
942        }
943
944        # print out header info from src html.
945        while (defined($_ = <HTMLSRC>) && $_ =~ m/\w/) {
946        print HTML "$_";
947        }
948
949        # print out table of contents, making links relative
950        <TOC>; <TOC>; # ignore first 2 lines
951        print HTML scalar(<TOC>); # line 3 = "<ol>\n"
952        my $line;
953        while ($line=<TOC>) {
954        $line =~ s@</body></html>$@@i ; # only last line has this
955        # make link relative
956        $line =~ s@href=\"[^\#]+@href=\"@i;
957        print HTML $line;
958        }
959        close TOC;
960
961        # rest of html src
962        while (<HTMLSRC>) {
963        print HTML $_;
964        }
965        close HTMLSRC;
966        close HTML;
967
968        &util::rm("${output_filestem}_ToC.html");
969        &util::rm("${output_filestem}.src");
970    }
971    # we don't yet do anything with footnotes ($output_filestem_fn.html) :(
972    return 1; # success
973    }
974
975    if (-e "$output_filestem.err") {
976    if ($faillogfile ne "" && defined(open(FAILLOG,">>$faillogfile")))
977    {
978        print FAILLOG "Error - rtftohtml - couldn't extract text\n";
979        #print FAILLOG "Error - rtf-converter - couldn't extract text\n";
980        print FAILLOG " (rtf file might be too recent):\n";
981        open (ERRLOG, "$output_filestem.err");
982        while (<ERRLOG>) {print FAILLOG $_;}
983        close ERRLOG;
984        close FAILLOG;
985    }
986    &util::rm("$output_filestem.err");
987    }
988
989    &util::rm("$output_filestem.html") if (-e "$output_filestem.html");
990
991    return 0;
992}
993
994
995# Convert a pdf file to html with the pdftohtml command
996
997sub pdf_to_html {
998    my ($dirname, $input_filename, $output_filestem) = @_;
999
1000    my $cmd = "";
1001    if ($timeout) {$cmd = "ulimit -t $timeout;";}
1002    my $full_perl_path = &util::get_perl_exec();
1003    $cmd .= "\"$full_perl_path\" -S pdftohtml.pl -zoom $pdf_zoom";
1004    $cmd .= " -c" if ($pdf_complex);
1005    $cmd .= " -i" if ($pdf_ignore_images);
1006    $cmd .= " -a" if ($pdf_allow_images_only);
1007    $cmd .= " -hidden" unless ($pdf_nohidden);
1008    $cmd .= " \"$input_filename\" \"$output_filestem\"";
1009   
1010    if ($ENV{'GSDLOS'} !~ m/^windows$/i || $is_winnt_2000) {
1011    $cmd .= " > \"$output_filestem.out\" 2> \"$output_filestem.err\"";
1012    } else {
1013    $cmd .= " > \"$output_filestem.err\"";
1014    }
1015
1016    $!=0;
1017
1018    my $retval=system($cmd);
1019    if ($retval!=0)
1020    {
1021    print STDERR "Error executing pdftohtml.pl";
1022    if ($!) {print STDERR ": $!";}
1023    print STDERR "\n";
1024    }
1025
1026    # make sure the converter made something
1027    if ($retval!=0 || ! -s "$output_filestem.html")
1028    {
1029    &util::rm("$output_filestem.out") if (-e "$output_filestem.out");
1030    # print out the converter's std err, if any
1031    if (-s "$output_filestem.err") {
1032        open (ERRLOG, "$output_filestem.err") || die "$!";
1033        print STDERR "pdftohtml error log:\n";
1034        while (<ERRLOG>) {
1035        print STDERR "$_";
1036        }
1037        close ERRLOG;
1038    }
1039    print STDERR "***********output filestem $output_filestem.html\n";
1040    &util::rm("$output_filestem.html") if (-e "$output_filestem.html");
1041    if (-e "$output_filestem.err") {
1042        if ($faillogfile ne "" && defined(open(FAILLOG,">>$faillogfile")))
1043        {
1044        open (ERRLOG, "$output_filestem.err");
1045        while (<ERRLOG>) {print FAILLOG $_;}
1046        close ERRLOG;
1047        close FAILLOG;
1048        }   
1049        &util::rm("$output_filestem.err");
1050    }
1051    return 0;
1052    }
1053
1054    &util::rm("$output_filestem.err") if (-e "$output_filestem.err");
1055    &util::rm("$output_filestem.out") if (-e "$output_filestem.out");
1056    return 1;
1057}
1058
1059# Convert a pdf file to various types of image with the convert command
1060
1061sub pdfps_to_img {
1062    my ($dirname, $input_filename, $output_filestem, $output_type) = @_;
1063
1064    # Check that ImageMagick is installed and available on the path (except for Windows 95/98)
1065    if (!($ENV{'GSDLOS'} eq "windows" && !Win32::IsWinNT())) {
1066    my $result = `identify 2>&1`;
1067    if ($? == -1 || $? == 256) {  # Linux and Windows return different values for "program not found"
1068        #ImageMagick is not installed, thus the convert utility is not available.
1069        print STDERR "*** ImageMagick is not installed, the convert utility is not available. Unable to convert PDF/PS to images\n";
1070        return 0;
1071    }
1072    }
1073
1074    my $cmd = "";
1075    if ($timeout) {$cmd = "ulimit -t $timeout;";}
1076    $output_type =~ s/.*\_(.*)/$1/i;
1077    my $full_perl_path = &util::get_perl_exec();
1078    $cmd .= "\"$full_perl_path\" -S pdfpstoimg.pl -convert_to $output_type \"$input_filename\" \"$output_filestem\"";
1079    if ($ENV{'GSDLOS'} !~ m/^windows$/i || $is_winnt_2000) {
1080    $cmd .= " > \"$output_filestem.out\" 2> \"$output_filestem.err\"";
1081    } else {
1082    $cmd .= " > \"$output_filestem.err\"";
1083    }
1084
1085    # don't include path on windows (to avoid having to play about
1086    # with quoting when GSDLHOME might contain spaces) but assume
1087    # that the PATH is set up correctly
1088    $!=0;
1089    my $retval=system($cmd);
1090    if ($retval!=0)
1091    {
1092    print STDERR "Error executing pdftoimg.pl";
1093    if ($!) {print STDERR ": $!";}
1094    print STDERR "\n";
1095    }
1096
1097    #make sure the converter made something
1098    #if ($retval !=0) || ! -s "$output_filestem")
1099    if ($retval !=0)
1100    {
1101    &util::rm("$output_filestem.out") if (-e "$output_filestem.out");
1102    #print out the converter's std err, if any
1103    if (-s "$output_filestem.err") {
1104        open (ERRLOG, "$output_filestem.err") || die "$!";
1105        print STDERR "pdfpstoimg error log:\n";
1106        while (<ERRLOG>) {
1107        print STDERR "$_";
1108        }
1109        close ERRLOG;
1110    }
1111    #&util::rm("$output_filestem.html") if (-e "$output_filestem.html");
1112    if (-e "$output_filestem.err") {
1113        if ($faillogfile ne "" && defined(open(FAILLOG,">>$faillogfile")))
1114        {
1115        open (ERRLOG, "$output_filestem.err");
1116        while (<ERRLOG>) {print FAILLOG $_;}
1117        close ERRLOG;
1118        close FAILLOG;
1119       }   
1120        &util::rm("$output_filestem.err");
1121    }
1122    return 0;
1123    }
1124    &util::rm("$output_filestem.err") if (-e "$output_filestem.err");
1125    &util::rm("$output_filestem.out") if (-e "$output_filestem.out");
1126    return 1;
1127}
1128
1129# Convert a PDF file to text with the pdftotext command
1130
1131sub pdf_to_text {
1132    my ($dirname, $input_filename, $output_filestem) = @_;
1133
1134    my $cmd = "pdftotext \"$input_filename\" \"$output_filestem.text\"";
1135
1136    if ($ENV{'GSDLOS'} !~ m/^windows$/i) {
1137    $cmd .= " > \"$output_filestem.out\" 2> \"$output_filestem.err\"";
1138    } else {
1139    $cmd .= " > \"$output_filestem.err\"";
1140    }
1141   
1142    if (system($cmd)!=0)
1143    {
1144    print STDERR "Error executing $cmd: $!\n";
1145    &util::rm("$output_filestem.text") if (-e "$output_filestem.text");
1146    }
1147
1148    # make sure there is some extracted text.
1149    if (-e "$output_filestem.text") {
1150    open (EXTR_TEXT, "$output_filestem.text") || warn "open: $!";
1151    binmode(EXTR_TEXT); # just in case...
1152    my $line="";
1153    my $seen_text=0;
1154    while (($seen_text==0) && ($line=<EXTR_TEXT>)) {
1155        if ($line=~ m/\w/) {$seen_text=1;}
1156    }
1157    close EXTR_TEXT;
1158    if ($seen_text==0) { # no text was extracted
1159        print STDERR "Error: pdftotext found no text\n";
1160        &util::rm("$output_filestem.text");
1161    }
1162    }
1163
1164    # make sure the converter made something
1165    if (! -s "$output_filestem.text")
1166    {
1167    # print out the converters std err, if any
1168    if (-s "$output_filestem.err") {
1169        open (ERRLOG, "$output_filestem.err") || die "$!";
1170        print STDERR "pdftotext error log:\n";
1171        while (<ERRLOG>) {
1172        print STDERR "$_";
1173        }
1174        close ERRLOG;
1175    }
1176    # does this converter create a .out file?
1177    &util::rm("$output_filestem.out") if (-e "$output_filestem.out");
1178    &util::rm("$output_filestem.text") if (-e "$output_filestem.text");
1179    if (-e "$output_filestem.err") {
1180        if ($faillogfile ne "" && defined(open(FAILLOG,">>$faillogfile")))
1181        {
1182        open (ERRLOG,"$output_filestem.err");
1183        while (<ERRLOG>) {print FAILLOG $_;}
1184        close ERRLOG;
1185        close FAILLOG;
1186        }
1187        &util::rm("$output_filestem.err");
1188    }
1189    return 0;
1190    }
1191    &util::rm("$output_filestem.err") if (-e "$output_filestem.err");
1192    return 1;
1193}
1194
1195# Convert a PostScript document to text
1196# note - just using "ps2ascii" isn't good enough, as it
1197# returns 0 for a postscript interpreter error. ps2ascii is just
1198# a wrapper to "gs" anyway, so we use that cmd here.
1199
1200sub ps_to_text {
1201    my ($input_filename, $output_filestem) = @_;
1202
1203    my $error = "";
1204
1205    # if we're on windows we'll fall straight through without attempting
1206    # to use gs
1207    if ($ENV{'GSDLOS'} =~ m/^windows$/i) {
1208    $error = "Windows does not support gs";
1209
1210    } else {
1211    my $cmd = "";
1212    if ($timeout) {$cmd = "ulimit -t $timeout; ";}
1213    $cmd .= "gs -q -dNODISPLAY -dNOBIND -dWRITESYSTEMDICT -dSIMPLE -c save ";
1214    $cmd .= "-f ps2ascii.ps \"$input_filename\" -c quit > \"$output_filestem.text\"";
1215    #$cmd .= "pstotext -output \"$output_filestem.text\" $input_filename\"";
1216    $cmd .= " 2> $output_filestem.err";
1217    $!=0;
1218
1219    my $retcode=system($cmd);
1220    $retcode = $? >> 8;  # see man perlfunc - system for this...
1221    # if system returns -1 | 127 (couldn't start program), look at $! for message
1222
1223    if ($retcode!=0) {if ($!) {$error=$!;} else {$error="couldn't run.\n";}}
1224    elsif (! -e "$output_filestem.text") {
1225        $error="did not create output file.\n";
1226    }
1227    else
1228    {   # make sure the interpreter didn't get an error. It is technically
1229        # possible for the actual text to start with this, but....
1230        open PSOUT, "$output_filestem.text";
1231        if (<PSOUT> =~ m/^Error: (.*)/) {
1232        $error="interpreter error - \"$1\"";
1233        }
1234        close PSOUT;
1235    }
1236    }
1237
1238    if ($error ne "")
1239    {
1240    print STDERR "Warning: Error executing gs: $error\n";
1241    &util::rm("$output_filestem.text") if (-e "$output_filestem.text");
1242
1243    if ("$faillogfile" ne "" && defined(open (FAILLOG, ">>$faillogfile")))
1244    {
1245        print FAILLOG "gs - $error\n";
1246        if (-e "$output_filestem.err") {
1247        open(ERRLOG, "$output_filestem.err");
1248        while (<ERRLOG>) {print FAILLOG $_;}
1249        close ERRLOG;
1250        }
1251        close FAILLOG;
1252    }
1253    &util::rm("$output_filestem.err") if (-e "$output_filestem.err");
1254
1255
1256    # Fine then. We'll just do a lousy job by ourselves...
1257    # Based on 5-line regexp sed script found at:
1258    # http://snark.ptc.spbu.ru/mail-archives/lout/brown/msg00003.html
1259    #
1260    print STDERR "Stripping text from postscript\n";
1261    my $errorcode=0;
1262    open (IN, "$input_filename")
1263        ||  ($errorcode=1, warn "Couldn't read file: $!");
1264    open (OUT, ">$output_filestem.text")
1265        ||  ($errorcode=1, warn "Couldn't write file: $!");
1266    if ($errorcode) {print STDERR "errors\n";return 0;}
1267   
1268    my $text="";  # this is for whole .ps file...
1269    $text = join('', <IN>); # see man perlport, under "System Resources"
1270    close IN;
1271
1272    # Make sure this is a ps file...
1273    if ($text !~ m/^%!/) {
1274        print STDERR "Bad postscript header: not '%!'\n";
1275        if ($faillogfile ne "" && defined(open(FAILLOG, ">>$faillogfile")))
1276        {
1277        print FAILLOG "Bad postscript header: not '%!'\n";
1278        close FAILLOG;
1279        }
1280        return 0;
1281    }
1282
1283    # if ps has Page data, then use it to delete all stuff before it.
1284    $text =~ s/^.*?%%Page:.*?\n//s; # treat string as single line
1285   
1286    # remove all leading non-data stuff
1287    $text =~ s/^.*?\(//s;
1288
1289    # remove all newline chars for easier processing
1290    $text =~ s/\n//g;
1291   
1292    # Big assumption here - assume that if any co-ordinates are
1293    # given, then we are at the end of a sentence.
1294    $text =~ s/\)-?\d+\ -?\d+/\) \(\n\)/g;
1295
1296    # special characters--
1297    $text =~ s/\(\|\)/\(\ - \)/g; # j -> em-dash?
1298
1299    # ? ps text formatting (eg italics?) ?
1300    $text =~ s/Fn\(f\)/\(\{\)/g; # f -> {
1301    $text =~ s/Fn\(g\)/\(\}\)/g; # g -> }
1302    $text =~ s/Fn\(j\)/\(\|\)/g; # j -> |
1303    # default - remove the rest
1304    $text =~ s/\ ?F.\((.+?)\)/\($1\)/g;
1305
1306    # attempt to add whitespace between words...
1307    # this is based purely on observation, and may be completely wrong...
1308    $text =~ s/([^F])[defghijkuy]\(/$1 \( /g;
1309    # eg I notice "b(" is sometimes NOT a space if preceded by a
1310    # negative number.
1311    $text =~ s/\)\d+ ?b\(/\) \( /g;
1312
1313    # change quoted braces to brackets
1314    $text =~ s/([^\\])\\\(/$1\{/g;
1315    $text =~ s/([^\\])\\\)/$1\}/g ;
1316
1317    # remove everything that is not between braces
1318    $text =~ s/\)([^\(\)])+?\(//sg ;
1319   
1320    # remove any Trailer eof stuff.
1321    $text =~ s/\)[^\)]*$//sg;
1322
1323    ### ligatures have special characters...
1324    $text =~ s/\\013/ff/g;
1325    $text =~ s/\\014/fi/g;
1326    $text =~ s/\\015/fl/g;
1327    $text =~ s/\\016/ffi/g;
1328    $text =~ s/\\214/fi/g;
1329    $text =~ s/\\215/fl/g;
1330    $text =~ s/\\017/\n\* /g; # asterisk?
1331    $text =~ s/\\023/\023/g;  # e acute ('e)
1332    $text =~ s/\\177/\252/g;  # u"
1333#   $text =~ s/ ?? /\344/g;  # a"
1334
1335    print OUT "$text";
1336    close OUT;
1337    }
1338    # wrap the text - use a minimum length. ie, first space after this length.
1339    my $wrap_length=72;
1340    &util::mv("$output_filestem.text", "$output_filestem.text.tmp");
1341    open INFILE, "$output_filestem.text.tmp" ||
1342    die "Couldn't open file: $!";
1343    open OUTFILE, ">$output_filestem.text" ||
1344    die "Couldn't open file for writing: $!";
1345    my $line="";
1346    while ($line=<INFILE>) {
1347    while (length($line)>0) {
1348        if (length($line)>$wrap_length) {
1349        $line =~ s/^(.{$wrap_length}[^\s]*)\s*//;
1350        print OUTFILE "$1\n";
1351        } else {
1352        print OUTFILE "$line";
1353        $line="";
1354        }
1355    }
1356    }
1357    close INFILE;
1358    close OUTFILE;
1359    &util::rm("$output_filestem.text.tmp");
1360
1361    &util::rm("$output_filestem.err") if (-e "$output_filestem.err");
1362    return 1;
1363}
1364
1365
1366# Convert any file to HTML with a crude perl implementation of the
1367# UNIX strings command.
1368
1369sub any_to_html {
1370    my ($input_filename, $output_filestem) = @_;
1371
1372    # First generate a text file
1373    return 0 unless (&any_to_text($input_filename, $output_filestem));
1374
1375    # create an HTML file from the text file
1376    open(TEXT, "<$output_filestem.text");
1377    open(HTML, ">$output_filestem.html");
1378
1379    print HTML "<html><head>\n";
1380    print HTML "<META HTTP-EQUIV=\"Content-Type\" CONTENT=\"text/html\">\n";
1381    print HTML "<META NAME=\"GENERATOR\" CONTENT=\"Greenstone any_to_html\">\n";
1382    print HTML "</head><body>\n\n";
1383
1384    my $line;
1385    while ($line=<TEXT>) {
1386    $line =~ s/</&lt;/g;
1387    $line =~ s/>/&gt;/g;
1388    if ($line =~ m/^\s*$/) {
1389        print HTML "<p>";
1390    } else {
1391        print HTML "<br> ", $line;
1392    }
1393    }
1394    print HTML "\n</body></html>\n";
1395
1396    close HTML;
1397    close TEXT;
1398
1399    &util::rm("$output_filestem.text") if (-e "$output_filestem.text");
1400    return 1;
1401}
1402
1403# Convert any file to TEXT with a crude perl implementation of the
1404# UNIX strings command.
1405# Note - this assumes ascii charsets :(     (jrm21)
1406
1407sub any_to_text {
1408    my ($input_filename, $output_filestem) = @_;
1409
1410    if (!$use_strings) {
1411      return 0;
1412    }
1413
1414    print STDERR "\n**** In any to text****\n\n";
1415    open(IN, "<$input_filename") || return 0;
1416    binmode(IN);
1417    open(OUT, ">$output_filestem.text") || return 0;
1418
1419    my ($line);
1420    my $output_line_count = 0;
1421    while (<IN>) {
1422    $line = $_;
1423
1424    # delete anything that isn't a printable character
1425    $line =~ s/[^\040-\176]+/\n/sg;
1426
1427    # delete any string less than 10 characters long
1428    $line =~ s/^.{0,9}$/\n/mg;
1429    while ($line =~ m/^.{1,9}$/m) {
1430        $line =~ s/^.{0,9}$/\n/mg;
1431        $line =~ s/\n+/\n/sg;
1432    }
1433
1434    # remove extraneous whitespace
1435    $line =~ s/\n+/\n/gs;
1436    $line =~ s/^\n//gs;
1437
1438    # output whatever is left
1439    if ($line =~ m/[^\n ]/) {
1440        print OUT $line;
1441        ++$output_line_count;
1442    }
1443    }
1444
1445    close OUT;
1446    close IN;
1447
1448    if ($output_line_count) { # try to protect against binary only formats
1449    return 1;
1450    }
1451
1452    &util::rm("$output_filestem.text");
1453    return 0;
1454
1455}
Note: See TracBrowser for help on using the browser.