source: trunk/gsdl/bin/script/gsConvert.pl@ 2600

Last change on this file since 2600 was 2600, checked in by jrm21, 23 years ago

when extracting text from postscript, we now wrap the lines, so that a
large paragraph doesn't keep going east.

  • Property svn:executable set to *
  • Property svn:keywords set to Author Date Id Revision
File size: 18.4 KB
Line 
1#!/usr/bin/perl -w
2
3###########################################################################
4#
5# gsConvert.pl -- convert documents to HTML or TEXT format
6#
7# A component of the Greenstone digital library software
8# from the New Zealand Digital Library Project at the
9# University of Waikato, New Zealand.
10#
11# Copyright (C) 1999 New Zealand Digital Library Project
12#
13# This program is free software; you can redistribute it and/or modify
14# it under the terms of the GNU General Public License as published by
15# the Free Software Foundation; either version 2 of the License, or
16# (at your option) any later version.
17#
18# This program is distributed in the hope that it will be useful,
19# but WITHOUT ANY WARRANTY; without even the implied warranty of
20# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21# GNU General Public License for more details.
22#
23# You should have received a copy of the GNU General Public License
24# along with this program; if not, write to the Free Software
25# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
26#
27###########################################################################
28
29# gsConvert.pl converts documents in a range of formats to HTML or TEXT
30# by exploiting third-party programs. These are usually found in the
31# $GSDLHOME/packages directory.
32#
33# Currently, we can convert Microsoft Word and Adobe PDF using specialised
34# conversion utilities. We can convery any file to text with a perl
35# implementation of the UNIX strings command.
36#
37# We try to convert Postscript files to text using "gs" which is often on
38# *nix machines. If it isn't (or we're running on Windoze), we do some feeble
39# text extraction on it using regexps.
40
41BEGIN {
42 die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
43 unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
44}
45
46use parsargv;
47use util;
48use Cwd;
49use File::Basename;
50
51
52sub print_usage
53{
54 print STDERR "\n";
55 print STDERR "gsConvert.pl: Converts documents in a range of formats to html\n";
56 print STDERR " or text using third-party programs.\n\n";
57 print STDERR " usage: $0 [options] filename\n";
58 print STDERR " options:\n\t-type\tdoc|pdf|ps|rtf\n\t-output\thtml|text\n";
59 print STDERR "\t-timeout\t<max cpu seconds>\n";
60 exit(1);
61}
62
63
64sub main
65{
66 my (@ARGV) = @_;
67 my ($input_type,$output_type,$verbose,$timeout);
68
69 $timeout = 0;
70 # read command-line arguments
71 if (!parsargv::parse(\@ARGV,
72 'type/(doc|pdf|ps|rtf)/', \$input_type,
73 'output/(html|text)/', \$output_type,
74 'timeout/\d+/0',\$timeout,
75 'verbose/\d+/0', \$verbose))
76 {
77 print_usage();
78 }
79
80 # Make sure the input file exists and can be opened for reading
81 if (scalar(@ARGV!=1)) {
82 print_usage();
83 }
84
85 my $input_filename = $ARGV[0];
86 if (!-r $input_filename) {
87 print STDERR "Error: unable to open $input_filename for reading\n";
88 exit(1);
89 }
90
91 # Deduce filenames
92 my ($tailname,$dirname,$suffix)
93 = File::Basename::fileparse($input_filename, "\\.[^\\.]+\$");
94 my $output_filestem = &util::filename_cat($dirname, "$tailname");
95
96 if ($input_type eq "")
97 {
98 $input_type = lc (substr($suffix,1,length($suffix)-1));
99 }
100
101 # Change to temporary working directory
102 my $stored_dir = cwd();
103 chdir ($dirname) || die "Unable to change to directory $dirname";
104
105 # Select convert utility
106 if (!defined $input_type) {
107 print STDERR "Error: No filename extension or input type defined\n";
108 exit(1);
109 }
110 elsif ($input_type eq "doc") {
111 print &convertDOC($input_filename, $output_filestem, $output_type);
112 print "\n";
113 }
114 elsif ($input_type eq "rtf") {
115 print &convertRTF($input_filename, $output_filestem, $output_type);
116 print "\n";
117 }
118 elsif ($input_type eq "pdf") {
119 print &convertPDF($dirname, $input_filename, $output_filestem, $output_type);
120 print "\n";
121 }
122 elsif ($input_type eq "ps") {
123 print &convertPS($input_filename, $output_filestem, $output_type);
124 print "\n";
125 }
126 else {
127 print STDERR "Error: Unable to convert type '$input_type'\n";
128 exit(1);
129 }
130
131 # restore to original working directory
132 chdir ($stored_dir) || die "Unable to return to directory $stored_dir";
133
134}
135
136&main(@ARGV);
137
138
139
140# Document-type conversion functions
141#
142# The following functions attempt to convert documents from their
143# input type to the specified output type. If no output type was
144# given, then they first attempt HTML, and then TEXT.
145#
146# Each returns the output type ("html" or "text") or "fail" if no
147# conversion is possible.
148
149# Convert a Microsoft word document
150
151sub convertDOC {
152 ($input_filename, $output_filestem, $output_type) = @_;
153
154 # Many .doc files are not in fact word documents!
155 my $realtype = &find_docfile_type($input_filename);
156
157 if ($realtype eq "word6" || $realtype eq "word7" || $realtype eq "word8") {
158 return &convertWord678($input_filename, $output_filestem, $output_type);
159 } elsif ($realtype eq "rtf") {
160 return &convertRTF($input_filename, $output_filestem, $output_type);
161 } else {
162 return &convertAnything($input_filename, $output_filestem, $output_type);
163 }
164}
165
166# Convert a Microsoft word 6/7/8 document
167
168sub convertWord678 {
169 ($input_filename, $output_filestem, $output_type) = @_;
170
171 my $success = 0;
172
173 # Attempt specialised conversion to HTML
174 if (!$output_type || ($output_type =~ /html/i)) {
175 $success = &doc_to_html($input_filename, $output_filestem);
176 if ($success) {
177 return "html";
178 }
179 }
180
181 return &convertAnything($input_filename, $output_filestem, $output_type);
182}
183
184
185# Convert a Rich Text Format (RTF) file
186
187sub convertRTF {
188 ($input_filename, $output_filestem, $output_type) = @_;
189
190 my $success = 0;
191
192 # Attempt specialised conversion to HTML
193 if (!$output_type || ($output_type =~ /html/i)) {
194 $success = &rtf_to_html($input_filename, $output_filestem);
195 if ($success) {
196 return "html";
197 }
198 }
199
200 return &convertAnything($input_filename, $output_filestem, $output_type);
201}
202
203
204# Convert an unidentified file
205
206sub convertAnything {
207 ($input_filename, $output_filestem, $output_type) = @_;
208
209 my $success = 0;
210
211 # Attempt simple conversion to HTML
212 if (!$output_type || ($output_type =~ /html/i)) {
213 $success = &any_to_html($input_filename, $output_filestem);
214 if ($success) {
215 return "html";
216 }
217 }
218
219 # Convert to text
220 if (!$output_type || ($output_type =~ /text/i)) {
221 $success = &any_to_text($input_filename, $output_filestem);
222 if ($success) {
223 return "text";
224 }
225 }
226 return "fail";
227}
228
229
230
231# Convert an Adobe PDF document
232
233sub convertPDF {
234 ($dirname, $input_filename, $output_filestem, $output_type) = @_;
235
236 my $success = 0;
237
238 # Attempt conversion to HTML
239 if (!$output_type || ($output_type =~ /html/i)) {
240 $success = &pdf_to_html($dirname, $input_filename, $output_filestem);
241 if ($success) {
242 return "html";
243 }
244 }
245
246 # Attempt conversion to TEXT
247 if (!$output_type || ($output_type =~ /text/i)) {
248 $success = &pdf_to_text($dirname, $input_filename, $output_filestem);
249 if ($success) {
250 return "text";
251 }
252 }
253
254 return "fail";
255
256}
257
258
259# Convert an Adobe PostScript document
260
261sub convertPS {
262 ($input_filename, $output_filestem, $output_type) = @_;
263
264 my $success = 0;
265
266 # Attempt conversion to TEXT
267 if (!$output_type || ($output_type =~ /text/i)) {
268 $success = &ps_to_text($input_filename, $output_filestem);
269 if ($success) {
270 return "text";
271 }
272 }
273
274 return "fail";
275
276}
277
278
279# Find the real type of a .doc file
280#
281# We seem to have a lot of files with a .doc extension that are .rtf
282# files or Word 5 files. This function attempts to tell the difference.
283
284sub find_docfile_type {
285 ($input_filename) = @_;
286
287 open(CHK, "<$input_filename");
288 binmode(CHK);
289 my $line = "";
290 my $first = 1;
291
292 while (<CHK>) {
293
294 $line = $_;
295
296 if ($first) {
297 # check to see if this is an rtf file
298 if ($line =~ /^\{\\rtf/) {
299 close(CHK);
300 return "rtf";
301 }
302 }
303
304 # is this is a word 6/7/8 document?
305 if ($line =~ /Word\.Document\.([678])/) {
306 close(CHK);
307 return "word$1";
308 }
309
310 $first = 0;
311
312 }
313
314 return "unknown";
315}
316
317
318
319# Specific type-to-type conversions
320#
321# Each of the following functions attempts to convert a document from
322# a specific format to another. If they succeed yhey return 1 and leave
323# the output document(s) in the appropriate place; if they fail they
324# return 0 and delete any working files.
325
326
327# Attempt to convert a word document to html with the wv program
328
329sub doc_to_html {
330 ($input_filename, $output_filestem) = @_;
331
332 my $wvWare = &util::filename_cat($ENV{'GSDLHOME'}, "bin",
333 $ENV{'GSDLOS'}, "wvWare");
334
335 # don't include path on windows (to avoid having to play about
336 # with quoting when GSDLHOME might contain spaces) but assume
337 # that the PATH is set up correctly
338 $wvWare = "wvWare" if ($ENV{'GSDLOS'} =~ /^windows$/i);
339
340 my $wv_conf = &util::filename_cat($ENV{'GSDLHOME'}, "etc",
341 "packages", "wv", "wvHtml.xml");
342
343 my $cmd = "";
344 if ($timeout) {$cmd = "ulimit -t $timeout;";}
345 $cmd .= "$wvWare --charset utf-8 --config \"$wv_conf\"";
346 $cmd .= " \"$input_filename\" > \"$output_filestem.html\"";
347
348 # redirecting STDERR is a bad idea on windows 95/98
349 $cmd .= " 2> \"$output_filestem.err\""
350 if $ENV{'GSDLOS'} !~ /^windows$/i;
351
352 # execute the command
353 if (system($cmd)!=0)
354 {
355 print STDERR "Error executing wv converter: $!. Continuing...\n";
356 }
357
358 # Was the conversion successful?
359
360 if (-e "$output_filestem.html") {
361 open(TMP, "$output_filestem.html");
362 $line = <TMP>;
363 close(TMP);
364 if ($line && $line =~ /DOCTYPE HTML/) {
365 &util::rm("$output_filestem.err") if -e "$output_filestem.err";
366 return 1;
367 } else {
368 # An error of some sort occurred
369 &util::rm("$output_filestem.html");
370 &util::rm("$output_filestem.err") if -e "$output_filestem.err";
371 }
372 }
373
374 return 0;
375}
376
377
378# Attempt to convert an RTF document to html with rtftohtml
379
380sub rtf_to_html {
381 my ($input_filename, $output_filestem) = @_;
382
383 # formulate the command
384 $cmd = "";
385 if ($timeout) {$cmd = "ulimit -t $timeout;";}
386 $cmd .= "rtftohtml";
387
388 # it automatically uses $output_filestem.html
389 $cmd .= " \"$input_filename\"";
390
391 $cmd .= " 2>\"$output_filestem.err\""
392 unless $ENV{'GSDLOS'} =~ /^windows$/i;
393
394
395 # execute the command
396 if (system($cmd)!=0)
397 {
398 print STDERR "Error executing rtf converter: $!. Continuing...\n";
399 }
400
401 &util::rm("$output_filestem.err") if (-e "$output_filestem.err");
402
403 # Was the conversion successful?
404 if (-e "$output_filestem.html") {
405 return 1;
406 }
407
408 return 0;
409}
410
411
412# Convert a pdf file to html with the pdftohtml command
413
414sub pdf_to_html {
415 ($dirname, $input_filename, $output_filestem) = @_;
416
417 $cmd = "";
418 if ($timeout) {$cmd = "ulimit -t $timeout;";}
419 $cmd .= "perl -S pdftohtml.pl -F ";
420 $cmd .= " \"$input_filename\" \"$output_filestem\"";
421 $!=0;
422
423 if (system($cmd)!=0)
424 {
425 print STDERR "Error executing $cmd";
426 if ($!) {print STDERR ": $!";}
427 print STDERR "\n";
428 return 0;
429 }
430
431 # make sure the converter made something
432 if (! -e "$output_filestem.html")
433 {
434 &util::rm("$output_filestem.out") if (-e "$output_filestem.out");
435 # print out the converters std err, if any
436 if (-e "$output_filestem.err") {
437 open (ERRLOG, "$output_filestem.err") || die "$!";
438 print STDERR "pdftohtml:\n";
439 while (<ERRLOG>) {
440 print STDERR "$_";
441 }
442 close ERRLOG;
443 }
444 return 0;
445 }
446
447 &util::rm("$output_filestem.out") if (-e "$output_filestem.out");
448 return 1;
449}
450
451# Convert a PDF file to text with the pdftotext command
452
453sub pdf_to_text {
454 ($dirname, $input_filename, $output_filestem) = @_;
455
456 my $cmd = "pdftotext \"$input_filename\" \"$output_filestem.text\"";
457 $cmd .= " 2> \"$output_filestem.err\"";
458
459 if (system($cmd)!=0)
460 {
461 print STDERR "Error executing $cmd: $!\n";
462 &util::rm("$output_filestem.text") if (-e "$output_filestem.text");
463 &util::rm("$output_filestem.err") if (-e "$output_filestem.err");
464 return 0;
465 }
466
467 # make sure the converter made something
468 if (! -e "$output_filestem.text")
469 {
470 &util::rm("$output_filestem.out") if (-e "$output_filestem.out");
471 # print out the converters std err, if any
472 if (-e "$output_filestem.err") {
473 open (ERRLOG, "$output_filestem.err") || die "$!";
474 print STDERR "pdftotext:\n";
475 while (<ERRLOG>) {
476 print STDERR "$_";
477 }
478 close ERRLOG;
479 }
480 return 0;
481 }
482
483 &util::rm("$output_filestem.err") if (-e "$output_filestem.err");
484 return 1;
485}
486
487# Convert a PostScript document to text
488# note - just using "ps2ascii" isn't good enough, as it
489# returns 0 for a postscript interpreter error. ps2ascii is just
490# a wrapper to "gs" anyway, so we use that cmd here.
491
492sub ps_to_text {
493 my ($input_filename, $output_filestem) = @_;
494
495 my $error = "";
496
497 # if we're on windows we'll fall straight through without attempting
498 # to use gs
499 if ($ENV{'GSDLOS'} =~ /^windows$/i) {
500 $error = "Windows does not support gs";
501
502 } else {
503 my $cmd = "gs -q -dNODISPLAY -dNOBIND -dWRITESYSTEMDICT -dSIMPLE -c save ";
504 $cmd .= "-f ps2ascii.ps \"$input_filename\" -c quit > \"$output_filestem.text\"";
505 $cmd .= " 2> $output_filestem.err";
506 $!=0;
507
508 my $retcode=system($cmd);
509 $retcode = $? >> 8; # see man perlfunc - system for this...
510 # if system returns -1 | 127 (couldn't start program), look at $! for message
511
512 if ($retcode!=0) {if ($!) {$error=$!;} else {$error="couldn't run.\n";}}
513 elsif (! -e "$output_filestem.text") {
514 $error="did not create output file.\n";
515 }
516 else
517 { # make sure the interpreter didn't get an error. It is technically
518 # possible for the actual text to start with this, but....
519 open PSOUT, "$output_filestem.text";
520 if (<PSOUT> =~ /^Error: (.*)/) {
521 $error="interpreter error - \"$1\"";
522 }
523 close PSOUT;
524 }
525 }
526
527 if ($error ne "")
528 {
529 print STDERR "PSPlug: WARNING: Error executing gs: $error\n";
530 &util::rm("$output_filestem.text") if (-e "$output_filestem.text");
531 &util::rm("$output_filestem.err") if (-e "$output_filestem.err");
532
533 # Fine then. We'll just do a lousy job by ourselves...
534 # Based on 5-line regexp sed script found at:
535 # http://snark.ptc.spbu.ru/mail-archives/lout/brown/msg00003.html
536 #
537 print STDERR "PSPlug: Stripping text from postscript\n";
538 my $errorcode=0;
539 open (IN, "$input_filename")
540 || ($errorcode=1, warn "Couldn't read file: $!");
541 open (OUT, ">$output_filestem.text")
542 || ($errorcode=1, warn "Couldn't write file: $!");
543 if ($errorcode) {print STDERR "errors\n";return 0;}
544
545 my $text=""; # this is for whole .ps file...
546 while (<IN>) {
547 $text.=$_;
548 }
549 close IN;
550
551 # Make sure this is a ps file...
552 if ($text !~ /^%!/) {
553 print STDERR "Bad postscript header: not %!\n";
554 return 0;
555 }
556
557 # if ps has Page data, then use it to delete all stuff before it.
558 $text =~ s/^.*?%%Page:.*?\n//s; # treat string as single line
559
560 # remove all leading non-data stuff
561 $text =~ s/^.*?\(//s;
562
563 # remove all newline chars for easier processing
564 $text =~ s/\n//g;
565
566 # Big assumption here - assume that if any co-ordinates are
567 # given, then we are at the end of a sentence.
568 $text =~ s/\)-?\d+\ -?\d+/\) \(\n\)/g;
569
570 # special characters--
571 $text =~ s/\(\|\)/\(\ - \)/g; # j -> em-dash?
572
573 # ? ps text formatting (eg italics?) ?
574 $text =~ s/Fn\(f\)/\(\{\)/g; # f -> {
575 $text =~ s/Fn\(g\)/\(\}\)/g; # g -> }
576 $text =~ s/Fn\(j\)/\(\|\)/g; # j -> |
577 # default - remove the rest
578 $text =~ s/\ ?F.\((.+?)\)/\($1\)/g;
579
580 # attempt to add whitespace between words...
581 # this is based purely on observation, and may be completely wrong...
582 $text =~ s/([^F])[defghijkuy]\(/$1 \( /g;
583 # eg I notice "b(" is sometimes NOT a space if preceded by a
584 # negative number.
585 $text =~ s/\)\d+ ?b\(/\) \( /g;
586
587 # change quoted braces to brackets
588 $text =~ s/([^\\])\\\(/$1\{/g;
589 $text =~ s/([^\\])\\\)/$1\}/g ;
590
591 # remove everything that is not between braces
592 $text =~ s/\)([^\(\)])+?\(//sg ;
593
594 # remove any Trailer eof stuff.
595 $text =~ s/\)[^\)]*$//sg;
596
597 ### ligatures have special characters...
598 $text =~ s/\\013/ff/g;
599 $text =~ s/\\014/fi/g;
600 $text =~ s/\\015/fl/g;
601 $text =~ s/\\016/ffi/g;
602 $text =~ s/\\214/fi/g;
603 $text =~ s/\\215/fl/g;
604 $text =~ s/\\017/\n\* /g; # asterisk?
605 $text =~ s/\\023/\023/g; # e acute ('e)
606 $text =~ s/\\177/\252/g; # u"
607# $text =~ s/ ?? /\344/g; # a"
608
609 print OUT "$text";
610 close OUT;
611 }
612 # wrap the text - use a minimum length. ie, first space after this length.
613 my $wrap_length=72;
614 &util::mv("$output_filestem.text", "$output_filestem.text.tmp");
615 open INFILE, "$output_filestem.text.tmp" ||
616 die "Couldn't open file: $!";
617 open OUTFILE, ">$output_filestem.text" ||
618 die "Couldn't open file for writing: $!";
619 my $line="";
620 while ($line=<INFILE>) {
621 while (length($line)>0) {
622 if (length($line)>$wrap_length) {
623 $line =~ s/^(.{$wrap_length}[^\s]*)\s*//;
624 print OUTFILE "$1\n";
625 } else {
626 print OUTFILE "$line";
627 $line="";
628 }
629 }
630 }
631 close INFILE;
632 close OUTFILE;
633 &util::rm("$output_filestem.text.tmp");
634
635 &util::rm("$output_filestem.err") if (-e "$output_filestem.err");
636 return 1;
637}
638
639
640# Convert any file to HTML with a crude perl implementation of the
641# UNIX strings command.
642
643sub any_to_html {
644 ($input_filename, $output_filestem) = @_;
645
646 # First generate a text file
647 return 0 unless (&any_to_text($input_filename, $output_filestem));
648
649 # create an HTML file from the text file
650 open(TEXT, "<$output_filestem.text");
651 open(HTML, ">$output_filestem.html");
652
653 print HTML "<html><head>\n";
654 print HTML "<META HTTP-EQUIV=\"Content-Type\" CONTENT=\"text/html\">\n";
655 print HTML "<META NAME=\"GENERATOR\" CONTENT=\"Greenstone any_to_html\">\n";
656 print HTML "</head><body>\n\n";
657
658 while (<TEXT>) {
659 print HTML "<p> ", $_;
660 }
661 print HTML "\n</body></html>\n";
662
663 close HTML;
664 close TEXT;
665
666 &util::rm("$output_filestem.text") if (-e "$output_filestem.text");
667 return 1;
668}
669
670# Convert any file to TEXT with a crude perl implementation of the
671# UNIX strings command.
672
673sub any_to_text {
674 ($input_filename, $output_filestem) = @_;
675
676 open(IN, "<$input_filename");
677 binmode(IN);
678 open(OUT, ">$output_filestem.text");
679
680 my ($line);
681 my $dgcount = 0;
682 while (<IN>) {
683 $line = $_;
684
685 # delete anything that isn't a printable character
686 $line =~ s/[^\040-\176]+/\n/sg;
687
688 # delete any string less than 10 characters long
689 $line =~ s/^.{0,9}$/\n/mg;
690 while ($line =~ /^.{1,9}$/m) {
691 $line =~ s/^.{0,9}$/\n/mg;
692 $line =~ s/\n+/\n/sg;
693 }
694
695 # remove extraneous whitespace
696 $line =~ s/\n+/\n/gs;
697 $line =~ s/^\n//gs;
698
699 # output whatever is left
700 if ($line =~ /[^\n ]/) {
701 print OUT $line;
702 }
703 }
704
705 close OUT;
706 close IN;
707
708 return 1;
709}
Note: See TracBrowser for help on using the repository browser.