source: main/trunk/greenstone2/bin/script/gsConvert.pl@ 32277

Last change on this file since 32277 was 32277, checked in by ak19, 6 years ago

First attempt at PDFv2Plugin.pm.
  • Property svn:executable set to *
  • Property svn:keywords set to Author Date Id Revision
File size: 43.9 KB
Line 
1#!/usr/bin/perl -w
2
3###########################################################################
4#
5# gsConvert.pl -- convert documents to HTML or TEXT format
6#
7# A component of the Greenstone digital library software
8# from the New Zealand Digital Library Project at the
9# University of Waikato, New Zealand.
10#
11# Copyright (C) 1999-2002 New Zealand Digital Library Project
12#
13# This program is free software; you can redistribute it and/or modify
14# it under the terms of the GNU General Public License as published by
15# the Free Software Foundation; either version 2 of the License, or
16# (at your option) any later version.
17#
18# This program is distributed in the hope that it will be useful,
19# but WITHOUT ANY WARRANTY; without even the implied warranty of
20# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21# GNU General Public License for more details.
22#
23# You should have received a copy of the GNU General Public License
24# along with this program; if not, write to the Free Software
25# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
26#
27###########################################################################
28
29# gsConvert.pl converts documents in a range of formats to HTML or TEXT
30# by exploiting third-party programs. The sources of these are usually found
31# in the $GSDLHOME/packages directory, and the executables should live in
32# $GSDLHOME/bin/$GSDLOS (which is on the search path).
33#
34# Currently, we can convert the following formats by using external
35# conversion utilities:
36# Microsoft Word (versions 2,6,7 [==95?], 8[==97?], 9[==2000?]), RTF,
37# Adobe PDF, PostScript, MS PowerPoint (95 and 97), and MS Excel (95 and 97).
38#
39# We can try to convert any file to text with a perl implementation of the
40# UNIX strings command.
41#
42# We try to convert Postscript files to text using "gs" which is often on
43# *nix machines. We fall back to performing weak text extraction by using
44# regular expressions.
45
46BEGIN {
47 die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
48 unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
49}
50
51use strict;
52
53use parsargv;
54use util;
55use FileUtils;
56use Cwd;
57
58# Are we running on WinNT or Win2000 (or later)?
59my $is_winnt_2000=eval {require Win32; return (Win32::IsWinNT()); return 0;};
60if (!defined($is_winnt_2000)) {$is_winnt_2000=0;}
61
62my $use_strings;
63my $pdf_tool;
64my $pdf_complex;
65my $pdf_nohidden;
66my $pdf_zoom;
67my $pdf_ignore_images;
68my $pdf_allow_images_only;
69my $windows_scripting;
70my $enc;
71
72sub print_usage
73{
74 print STDERR "\n";
75 print STDERR "gsConvert.pl: Converts documents in a range of formats to html\n";
76 print STDERR " or text using third-party programs.\n\n";
77 print STDERR " usage: $0 [options] filename\n";
78 print STDERR " options:\n\t-type\tdoc|dot|pdf|ps|ppt|rtf|xls\t(input file type)\n";
79 print STDERR "\t-errlog\t<filename>\t(append err messages)\n";
80 print STDERR "\t-output\tauto|html|pretty_html|paged_pretty_html|paged_html|text|paged_text|pagedimg_jpg|pagedimg_gif|pagedimg_png|pagedimgtxt_jpg|pagedimgtxt_png\t(output file type)\n";
81 print STDERR "\t-timeout\t<max cpu seconds>\t(ulimit on unix systems)\n";
82 print STDERR "\t-use_strings\tuse strings to extract text if conversion fails\n";
83 print STDERR "\t-windows_scripting\tuse windows VB script (if available) to convert Microsoft Word and PPT documents\n";
84 print STDERR "\t-pdf_tool\tpdftohtml|xpdftools|pdfbox (not all output types are supported by every pdf_tool)\n";
85 print STDERR "\t-pdf_complex\tuse complex output when converting PDF to HTML\n";
86 print STDERR "\t-pdf_nohidden\tDon't attempt to extract hidden text from PDF files\n";
87 print STDERR "\t-pdf_ignore_images\tdon't attempt to extract images when\n";
88 print STDERR "\t\tconverting PDF to HTML\n";
89 print STDERR "\t-pdf_allow_images_only\tallow images only (continue even if no text is present when converting to HTML)\n";
90 print STDERR "\t-pdf_zoom\tfactor by which to zoom PDF (only useful if\n";
91 print STDERR "\t\t-pdf_complex is set\n";
92 exit(1);
93}
94
95my $faillogfile="";
96my $timeout=0;
97my $verbosity=0;
98
99sub main
100{
101 my (@ARGV) = @_;
102 my ($input_type,$output_type,$verbose);
103
104 # Dynamically figure out what the --type option can support, based on whether -windows_scripting
105 # is in use or not
106 my $default_type_re = "(doc|dot|pdf|ps|ppt|rtf|xls)";
107 #my $enhanced_type_re = "(docx?|dot|pdf|ps|pptx?|rtf|xlsx?)";
108 #my $enhanced_type_re = "(docx?|dot|pdf|ps|pptx?|rtf|xlsx?)";
109 # Currently only have VBA for Word and PPT(but no XLS)
110 my $enhanced_type_re = "(docx?|dot|pdf|ps|pptx?|rtf|xls)";
111
112 my $type_re = $default_type_re;
113
114 foreach my $a (@ARGV) {
115 if ($a =~ m/^windows_scripting$/i) {
116 $type_re = $enhanced_type_re;
117 }
118 }
119
120 # read command-line arguments
121 if (!parsargv::parse(\@ARGV,
122 "type/$type_re/", \$input_type,
123 '/errlog/.*/', \$faillogfile,
124 'output/(auto|html|text|pagedimg).*/', \$output_type, # regex includes html_multi and paged_html besides html
125 'timeout/\d+/0',\$timeout,
126 'verbose/\d+/0', \$verbose,
127 'windows_scripting',\$windows_scripting,
128 'use_strings', \$use_strings,
129 'pdf_tool/(pdftohtml|pdfbox|xpdftools)/', \$pdf_tool, # the old pdftohtml tool, pdfbox extensions or the newer xpdf-tools
130 'pdf_complex', \$pdf_complex, # options for pdf_tool = pdftohtml (the old pdftohtml tool)
131 'pdf_ignore_images', \$pdf_ignore_images,
132 'pdf_allow_images_only', \$pdf_allow_images_only,
133 'pdf_nohidden', \$pdf_nohidden,
134 'pdf_zoom/\d+/2', \$pdf_zoom
135 ))
136 {
137 print_usage();
138 }
139
140 $verbosity=$verbose if defined $verbose;
141
142 # Make sure the input file exists and can be opened for reading
143 if (scalar(@ARGV!=1)) {
144 print_usage();
145 }
146
147 my $input_filename = $ARGV[0];
148 if (!-r $input_filename) {
149 print STDERR "Error: unable to open $input_filename for reading\n";
150 exit(1);
151 }
152
153 # Deduce filenames
154 my ($tailname,$dirname,$suffix)
155 = File::Basename::fileparse($input_filename, "\\.[^\\.]+\$");
156 my $output_filestem = &FileUtils::filenameConcatenate($dirname, "$tailname");
157
158 if ($input_type eq "")
159 {
160 $input_type = lc (substr($suffix,1,length($suffix)-1));
161 }
162
163 # Change to temporary working directory
164 my $stored_dir = cwd();
165 chdir ($dirname) || die "Unable to change to directory $dirname";
166
167 # Select convert utility
168 if (!defined $input_type) {
169 print STDERR "Error: No filename extension or input type defined\n";
170 exit(1);
171 }
172 elsif ($input_type =~ m/^docx?$/ || $input_type eq "dot") {
173 print &convertDOC($input_filename, $output_filestem, $output_type);
174 print "\n";
175 }
176 elsif ($input_type eq "rtf") {
177 print &convertRTF($input_filename, $output_filestem, $output_type);
178 print "\n";
179 }
180 elsif ($input_type eq "pdf") {
181 print &convertPDF($dirname, $input_filename, $output_filestem, $output_type);
182 print "\n";
183 }
184 elsif ($input_type eq "ps") {
185 print &convertPS($dirname, $input_filename, $output_filestem, $output_type);
186 print "\n";
187 }
188 elsif ($input_type =~ m/pptx?$/) {
189 print &convertPPT($input_filename, $output_filestem, $output_type);
190 print "\n";
191 }
192 elsif ($input_type =~ m/xlsx?$/) {
193 print &convertXLS($input_filename, $output_filestem, $output_type);
194 print "\n";
195 }
196 else {
197 print STDERR "Error: Unable to convert type '$input_type'\n";
198 exit(1);
199 }
200
201 # restore to original working directory
202 chdir ($stored_dir) || die "Unable to return to directory $stored_dir";
203
204}
205
206&main(@ARGV);
207
208
209
210# Document-type conversion functions
211#
212# The following functions attempt to convert documents from their
213# input type to the specified output type. If no output type was
214# given, then they first attempt HTML, and then TEXT.
215#
216# Each returns the output type ("html" or "text") or "fail" if no
217# conversion is possible.
218
219# Convert a Microsoft word document
220
221sub convertDOC {
222 my ($input_filename, $output_filestem, $output_type) = @_;
223
224 # Many .doc files are not in fact word documents!
225 my $realtype = &find_docfile_type($input_filename);
226
227 if ($realtype eq "word6" || $realtype eq "word7"
228 || $realtype eq "word8" || $realtype eq "docx") {
229 return &convertWord678($input_filename, $output_filestem, $output_type);
230 } elsif ($realtype eq "rtf") {
231 return &convertRTF($input_filename, $output_filestem, $output_type);
232 } else {
233 return &convertAnything($input_filename, $output_filestem, $output_type);
234 }
235}
236
237# Convert a Microsoft word 6/7/8 document
238
239sub convertWord678 {
240 my ($input_filename, $output_filestem, $output_type) = @_;
241
242 my $success = 0;
243 if (!$output_type || ($output_type =~ m/html/i)){
244 if ($windows_scripting) {
245 $success = &native_doc_to_html($input_filename, $output_filestem);
246 }
247 else {
248 $success = &doc_to_html($input_filename, $output_filestem);
249 }
250 if ($success) {
251 return "html";
252 }
253 }
254 return &convertAnything($input_filename, $output_filestem, $output_type);
255}
256
257
258# Convert a Rich Text Format (RTF) file
259
260sub convertRTF {
261 my ($input_filename, $output_filestem, $output_type) = @_;
262
263 my $success = 0;
264
265 # Attempt specialised conversion to HTML
266 if (!$output_type || ($output_type =~ m/html/i)) {
267
268 if ($windows_scripting) {
269 $success = &native_doc_to_html($input_filename, $output_filestem);
270 }
271 else {
272 $success = &rtf_to_html($input_filename, $output_filestem);
273 }
274 if ($success) {
275 return "html";
276 }
277 }
278
279# rtf is so ugly that's it's not worth running strings over.
280# One day I'll write some quick'n'dirty regexps to try to extract text - jrm21
281# return &convertAnything($input_filename, $output_filestem, $output_type);
282 return "fail";
283}
284
285
286# Convert an unidentified file
287
288sub convertAnything {
289 my ($input_filename, $output_filestem, $output_type) = @_;
290
291 my $success = 0;
292
293 # Attempt simple conversion to HTML
294 if (!$output_type || ($output_type =~ m/html/i)) {
295 $success = &any_to_html($input_filename, $output_filestem);
296 if ($success) {
297 return "html";
298 }
299 }
300
301 # Convert to text
302 if (!$output_type || ($output_type =~ m/text/i)) {
303 $success = &any_to_text($input_filename, $output_filestem);
304 if ($success) {
305 return "text";
306 }
307 }
308 return "fail";
309}
310
311
312
313# Convert an Adobe PDF document
314
315sub convertPDF {
316 my ($dirname, $input_filename, $output_filestem, $output_type) = @_;
317
318 my $success = 0;
319 $output_type =~ s/.*\-(.*)/$1/i;
320
321 print STDERR "@@@@@@@@ Using $pdf_tool for the conversion\n";
322
323 # First determine which pdf conversion tool we're using among pdftohtml/pdfbox/xpdftools
324 # and then decide which conversion command to run based on the output type
325 # (pdfbox does not currently go through gsConvert.pl
326 # as PDFBoxConverter inherits from AutoLoadConverters)
327
328 if ($pdf_tool eq "pdftohtml" ) { # old pdftohtml tool
329 # Attempt coversion to Image
330 if ($output_type =~ m/jp?g|gif|png/i) {
331 $success = &pdfps_to_img($dirname, $input_filename, $output_filestem, $output_type);
332 if ($success){
333 return "item";
334 }
335 }
336
337 # Attempt conversion to HTML
338 # Uses the old pdftohtml that doesn't work for newer PDF versions
339 if ($output_type =~ m/^html/i) {
340 #if (!$output_type || ($output_type =~ m/^html/i)) {
341 $success = &pdf_to_html($dirname, $input_filename, $output_filestem);
342 if ($success) {
343 return "html";
344 }
345 }
346
347 # Attempt conversion to TEXT (not for Windows, but PDFPlugin/PDFv1Plugin takes care of that
348 if (!$output_type || ($output_type =~ m/text/i)) {
349 $success = &pdf_to_text($dirname, $input_filename, $output_filestem);
350
351 if ($success) {
352 return "text";
353 }
354 }
355 }
356
357 elsif ($pdf_tool eq "xpdftools" ) {
358
359 # default to pretty html output
360 if (!$output_type) {
361 $output_type = "pretty_html";
362 }
363
364 # Attempt coversion to Image
365 #if ($output_type =~ m/jp?g|gif|png/i) {
366 # $success = &pdfps_to_img($dirname, $input_filename, $output_filestem, $output_type);
367 # if ($success){
368 # return "item";
369 # }
370 #}
371
372 # Attempt conversion to (paged) pretty HTML using the newer pdftohtml of Xpdftools.
373 if ($output_type =~ m/pretty_html$/i) {
374 $success = &xpdf_to_html($dirname, $input_filename, $output_filestem);
375 if ($success) {
376 return $output_type;
377 }
378 }
379
380 # Attempt conversion to TEXT
381 # Proper paged_text processing not yet implemented with xpdf
382 if ($output_type =~ m/text/i) {
383 $success = &xpdf_to_text($dirname, $input_filename, $output_filestem, $output_type);
384
385 if ($success) {
386 return "text";
387 }
388 }
389 }
390
391 return "fail";
392
393}
394
395
396# Convert an Adobe PostScript document
397
398sub convertPS {
399 my ($dirname,$input_filename, $output_filestem, $output_type) = @_;
400
401 my $success = 0;
402 $output_type =~ s/.*\-(.*)/$1/i;
403 # Attempt coversion to Image
404 if ($output_type =~ m/jp?g|gif|png/i) {
405 $success = &pdfps_to_img($dirname, $input_filename, $output_filestem, $output_type);
406 if ($success){
407 return "item";
408 }
409 }
410
411 # Attempt conversion to TEXT
412 if (!$output_type || ($output_type =~ m/text/i)) {
413 $success = &ps_to_text($input_filename, $output_filestem);
414 if ($success) {
415 return "text";
416 }
417 }
418 return "fail";
419}
420
421
422sub convertPPT {
423 my ($input_filename, $output_filestem, $output_type) = @_;
424 my $success = 0;
425
426 my $ppt_convert_type = "";
427
428 #if (!$output_type || $windows_scripting || ($output_type !~ m/html/i) || ($output_type !~ m/text/i)){
429 if ($windows_scripting && ($output_type !~ m/html/i) && ($output_type !~ m/text/i)){
430 if ($output_type =~ m/gif/i) {
431 $ppt_convert_type = "-g";
432 } elsif ($output_type =~ m/jp?g/i){
433 $ppt_convert_type = "-j";
434 } elsif ($output_type =~ m/png/i){
435 $ppt_convert_type = "-p";
436 }
437 my $vbScript = &FileUtils::filenameConcatenate($ENV{'GSDLHOME'}, "bin",
438 $ENV{'GSDLOS'}, "pptextract");
439 $vbScript = "CScript //Nologo \"".$vbScript.".vbs\"" if ($ENV{'GSDLOS'} =~ m/^windows$/i); # now we use the .vbs VBScript
440 # $vbScript = "pptextract" if ($ENV{'GSDLOS'} =~ m/^windows$/i); # back when the pptextract.exe VB executable was used
441
442 my $cmd = "";
443 if ($timeout) {$cmd = "ulimit -t $timeout;";}
444 # if the converting directory already exists
445 if (-d $output_filestem) {
446 print STDERR "**The conversion directory already exists\n";
447 return "item";
448 } else {
449 $cmd .= "$vbScript $ppt_convert_type \"$input_filename\" \"$output_filestem\"";
450 $cmd .= " 2>\"$output_filestem.err\""
451 if ($ENV{'GSDLOS'} !~ m/^windows$/i || $is_winnt_2000);
452
453 if (system($cmd) !=0) {
454 print STDERR "Powerpoint VB Scripting convert failed\n";
455 } else {
456 return "item";
457 }
458 }
459 } elsif (!$output_type || ($output_type =~ m/html/i)) {
460 # Attempt conversion to HTML
461 #if (!$output_type || ($output_type =~ m/html/i)) {
462 # formulate the command
463 my $cmd = "";
464 my $full_perl_path = &util::get_perl_exec();
465 $cmd .= "\"$full_perl_path\" -S ppttohtml.pl ";
466 $cmd .= " \"$input_filename\" \"$output_filestem.html\"";
467 $cmd .= " 2>\"$output_filestem.err\""
468 if ($ENV{'GSDLOS'} !~ m/^windows$/i || $is_winnt_2000);
469
470 # execute the command
471 $!=0;
472 if (system($cmd)!=0)
473 {
474 print STDERR "Powerpoint 95/97 converter failed $!\n";
475 } else {
476 return "html";
477 }
478 }
479
480 $success = &any_to_text($input_filename, $output_filestem);
481 if ($success) {
482 return "text";
483 }
484
485 return "fail";
486}
487
488
489sub convertXLS {
490 my ($input_filename, $output_filestem, $output_type) = @_;
491
492 my $success = 0;
493
494 # Attempt conversion to HTML
495 if (!$output_type || ($output_type =~ m/html/i)) {
496 # formulate the command
497 my $cmd = "";
498 my $full_perl_path = &util::get_perl_exec();
499 $cmd .= "\"$full_perl_path\" -S xlstohtml.pl ";
500 $cmd .= " \"$input_filename\" \"$output_filestem.html\"";
501 $cmd .= " 2>\"$output_filestem.err\""
502 if ($ENV{'GSDLOS'} !~ m/^windows$/i || $is_winnt_2000);
503
504
505 # execute the command
506 $!=0;
507 if (system($cmd)!=0)
508 {
509 print STDERR "Excel 95/97 converter failed $!\n";
510 } else {
511 return "html";
512 }
513 }
514
515 $success = &any_to_text($input_filename, $output_filestem);
516 if ($success) {
517 return "text";
518 }
519
520 return "fail";
521}
522
523
524
525# Find the real type of a .doc file
526#
527# We seem to have a lot of files with a .doc extension that are .rtf
528# files or Word 5 files. This function attempts to tell the difference.
529sub find_docfile_type {
530 my ($input_filename) = @_;
531
532 if (($windows_scripting) && ($input_filename =~ m/\.docx$/)) {
533 return "docx";
534 }
535
536 open(CHK, "<$input_filename");
537 binmode(CHK);
538 my $line = "";
539 my $first = 1;
540
541 while (<CHK>) {
542
543 $line = $_;
544
545 if ($first) {
546 # check to see if this is an rtf file
547 if ($line =~ m/^\{\\rtf/) {
548 close(CHK);
549 return "rtf";
550 }
551 $first = 0;
552 }
553
554 # is this is a word 6/7/8 document?
555 if ($line =~ m/Word\.Document\.([678])/) {
556 close(CHK);
557
558 return "word$1";
559 }
560
561 }
562
563 return "unknown";
564}
565
566
567# Specific type-to-type conversions
568#
569# Each of the following functions attempts to convert a document from
570# a specific format to another. If they succeed they return 1 and leave
571# the output document(s) in the appropriate place; if they fail they
572# return 0 and delete any working files.
573
574
575# Attempt to convert a word document to html with the wv program
576sub doc_to_html {
577 my ($input_filename, $output_filestem) = @_;
578
579 my $wvware_status = 0;
580
581 # need to ensure that the path to perl is quoted (in case there's spaces in it)
582 my $launch_cmd = "\"".&util::get_perl_exec()."\" -S wvware.pl \"$input_filename\" \"$output_filestem\" \"$faillogfile\" $verbosity $timeout";
583
584# print STDERR "***** wvware launch cmd = $launch_cmd\n";
585
586 $wvware_status = system($launch_cmd)/256;
587 return $wvware_status;
588}
589
590# Attempt to convert a word document to html with the word2html scripting program
591sub native_doc_to_html {
592 my ($input_filename, $output_filestem) = @_;
593
594 # build up the path to the doc-to-html conversion tool we're going to use
595 my $vbScript = &FileUtils::filenameConcatenate($ENV{'GSDLHOME'}, "bin", $ENV{'GSDLOS'});
596
597 if ($ENV{'GSDLOS'} =~ m/^windows$/i) {
598 # if windows scripting with docx input, use new VBscript to get the local Word install (if
599 # any) to do the conversion, since docX can't be processed by word2html's windows_scripting
600
601 if($input_filename =~ m/docx$/i) { # need to use full path to docx2html script,
602 # else script launch fails when there are error msgs
603 $vbScript = &FileUtils::filenameConcatenate($vbScript, "docx2html.vbs");
604 $vbScript = "CScript //Nologo \"$vbScript\""; # launch with CScript for error output in STDERR
605 # //Nologo flag avoids Microsoft's opening/logo msgs
606 print STDERR "About to use windows scripting to process docx file $input_filename.\n";
607 print STDERR " This may take some time. Please wait...\n";
608 }
609 else { # old doc versions. use the usual VB executable word2html for the
610 # conversion. Doesn't need full path, since bin\windows is on PATH
611 $vbScript = "word2html"; #$vbScript = "\"".&FileUtils::filenameConcatenate($vbScript, "word2html")."\"";
612 }
613 }
614 else { # not windows
615 $vbScript = "\"".&FileUtils::filenameConcatenate($vbScript, "word2html")."\"";
616 }
617
618 if (-e "$output_filestem.html") {
619 print STDERR " The conversion file:\n";
620 print STDERR " $output_filestem.html\n";
621 print STDERR " ... already exists. Skipping\n";
622 return 1;
623 }
624
625 my $cmd = "";
626 if ($timeout) {$cmd = "ulimit -t $timeout;";}
627 #$cmd .= "$vbScript \"$input_filename\" \"$output_filestem.html\"";
628 #$cmd .= "$vbScript $input_filename $output_filestem.html";
629 $cmd .= "$vbScript \"$input_filename\" \"$output_filestem.html\"";
630
631 # redirecting STDERR
632
633 $cmd .= " 2> \"$output_filestem.err\""
634 if ($ENV {'GSDLOS'} !~ m/^windows$/i || $is_winnt_2000);
635 #print STDERR "@@@@@@@@@ cmd=$cmd\n";
636
637 # execute the command
638 $!=0;
639 if (system($cmd)!=0)
640 {
641 print STDERR "Error executing $vbScript converter:$!\n";
642 if (-s "$output_filestem.err") {
643 open (ERRFILE, "<$output_filestem.err");
644
645 my $write_to_fail_log=0;
646 if ($faillogfile ne "" && defined(open(FAILLOG,">>$faillogfile")))
647 {$write_to_fail_log=1;}
648
649 my $line;
650 while ($line=<ERRFILE>) {
651 if ($line =~ m/\w/) {
652 print STDERR "$line";
653 print FAILLOG "$line" if ($write_to_fail_log);
654 }
655 if ($line !~ m/startup error/) {next;}
656 print STDERR " (given an invalid .DOC file?)\n";
657 print FAILLOG " (given an invalid .DOC file?)\n"
658 if ($write_to_fail_log);
659
660 } # while ERRFILE
661 close FAILLOG if ($write_to_fail_log);
662 }
663 return 0; # we can try any_to_text
664 }
665
666 # Was the conversion successful?
667 if (-s "$output_filestem.html") {
668 open(TMP, "$output_filestem.html");
669 my $line = <TMP>;
670 close(TMP);
671 if ($line && $line =~ m/html/i) {
672 &FileUtils::removeFiles("$output_filestem.err") if -e "$output_filestem.err";
673 return 1;
674 }
675 }
676
677 # If here, an error of some sort occurred
678 &FileUtils::removeFiles("$output_filestem.html") if -e "$output_filestem.html";
679 if (-e "$output_filestem.err") {
680 if ($faillogfile ne "" && defined(open(FAILLOG,">>$faillogfile"))) {
681 open (ERRLOG,"$output_filestem.err");
682 while (<ERRLOG>) {print FAILLOG $_;}
683 close FAILLOG;
684 close ERRLOG;
685 }
686 &FileUtils::removeFiles("$output_filestem.err");
687 }
688 return 0;
689}
690
691# Attempt to convert an RTF document to html with rtftohtml
692sub rtf_to_html {
693 my ($input_filename, $output_filestem) = @_;
694
695 # formulate the command
696 my $cmd = "";
697 if ($timeout) {$cmd = "ulimit -t $timeout;";}
698 $cmd .= "rtftohtml";
699 #$cmd .= "rtf-converter";
700
701 $cmd .= " -o \"$output_filestem.html\" \"$input_filename\"";
702
703 $cmd .= " 2>\"$output_filestem.err\""
704 if ($ENV{'GSDLOS'} !~ m/^windows$/i || $is_winnt_2000);
705
706
707 # execute the command
708 $!=0;
709 if (system($cmd)!=0)
710 {
711 print STDERR "Error executing rtf converter $!\n";
712 # don't currently bother printing out error log...
713 # keep going, in case it still created an HTML file...
714 }
715
716 # Was the conversion successful?
717 my $was_successful=0;
718 if (-s "$output_filestem.html") {
719 # make sure we have some content other than header
720 open (HTML, "$output_filestem.html"); # what to do if fail?
721 my $line;
722 my $past_header=0;
723 while ($line=<HTML>) {
724
725 if ($past_header == 0) {
726 if ($line =~ m/<body>/) {$past_header=1;}
727 next;
728 }
729
730 $line =~ s/<[^>]+>//g;
731 if ($line =~ m/\w/ && $past_header) { # we found some content...
732 $was_successful=1;
733 last;
734 }
735 }
736 close HTML;
737 }
738
739 if ($was_successful) {
740 &FileUtils::removeFiles("$output_filestem.err")
741 if (-e "$output_filestem.err");
742 # insert the (modified) table of contents, if it exists.
743 if (-e "${output_filestem}_ToC.html") {
744 &FileUtils::moveFiles("$output_filestem.html","$output_filestem.src");
745 my $open_failed=0;
746 open HTMLSRC, "$output_filestem.src" || ++$open_failed;
747 open TOC, "${output_filestem}_ToC.html" || ++$open_failed;
748 open HTML, ">$output_filestem.html" || ++$open_failed;
749
750 if ($open_failed) {
751 close HTMLSRC;
752 close TOC;
753 close HTML;
754 &FileUtils::moveFiles("$output_filestem.src","$output_filestem.html");
755 return 1;
756 }
757
758 # print out header info from src html.
759 while (defined($_ = <HTMLSRC>) && $_ =~ m/\w/) {
760 print HTML "$_";
761 }
762
763 # print out table of contents, making links relative
764 <TOC>; <TOC>; # ignore first 2 lines
765 print HTML scalar(<TOC>); # line 3 = "<ol>\n"
766 my $line;
767 while ($line=<TOC>) {
768 $line =~ s@</body></html>$@@i ; # only last line has this
769 # make link relative
770 $line =~ s@href=\"[^\#]+@href=\"@i;
771 print HTML $line;
772 }
773 close TOC;
774
775 # rest of html src
776 while (<HTMLSRC>) {
777 print HTML $_;
778 }
779 close HTMLSRC;
780 close HTML;
781
782 &FileUtils::removeFiles("${output_filestem}_ToC.html");
783 &FileUtils::removeFiles("${output_filestem}.src");
784 }
785 # we don't yet do anything with footnotes ($output_filestem_fn.html) :(
786 return 1; # success
787 }
788
789 if (-e "$output_filestem.err") {
790 if ($faillogfile ne "" && defined(open(FAILLOG,">>$faillogfile")))
791 {
792 print FAILLOG "Error - rtftohtml - couldn't extract text\n";
793 #print FAILLOG "Error - rtf-converter - couldn't extract text\n";
794 print FAILLOG " (rtf file might be too recent):\n";
795 open (ERRLOG, "$output_filestem.err");
796 while (<ERRLOG>) {print FAILLOG $_;}
797 close ERRLOG;
798 close FAILLOG;
799 }
800 &FileUtils::removeFiles("$output_filestem.err");
801 }
802
803 &FileUtils::removeFiles("$output_filestem.html") if (-e "$output_filestem.html");
804
805 return 0;
806}
807
808
809# Convert a pdf file to html with the old pdftohtml command
810# which only works for older PDF versions
811sub pdf_to_html {
812 my ($dirname, $input_filename, $output_filestem) = @_;
813
814 my $cmd = "";
815 if ($timeout) {$cmd = "ulimit -t $timeout;";}
816 my $full_perl_path = &util::get_perl_exec();
817 $cmd .= "\"$full_perl_path\" -S pdftohtml.pl -zoom $pdf_zoom";
818 $cmd .= " -c" if ($pdf_complex);
819 $cmd .= " -i" if ($pdf_ignore_images);
820 $cmd .= " -a" if ($pdf_allow_images_only);
821 $cmd .= " -hidden" unless ($pdf_nohidden);
822 $cmd .= " \"$input_filename\" \"$output_filestem\"";
823
824 if ($ENV{'GSDLOS'} !~ m/^windows$/i || $is_winnt_2000) {
825 $cmd .= " > \"$output_filestem.out\" 2> \"$output_filestem.err\"";
826 } else {
827 $cmd .= " > \"$output_filestem.err\"";
828 }
829
830 $!=0;
831
832 my $retval=system($cmd);
833 if ($retval!=0)
834 {
835 print STDERR "Error executing pdftohtml.pl";
836 if ($!) {print STDERR ": $!";}
837 print STDERR "\n";
838 }
839
840 # make sure the converter made something
841 if ($retval!=0 || ! -s "$output_filestem.html")
842 {
843 &FileUtils::removeFiles("$output_filestem.out") if (-e "$output_filestem.out");
844 # print out the converter's std err, if any
845 if (-s "$output_filestem.err") {
846 open (ERRLOG, "$output_filestem.err") || die "$!";
847 print STDERR "pdftohtml error log:\n";
848 while (<ERRLOG>) {
849 print STDERR "$_";
850 }
851 close ERRLOG;
852 }
853 #print STDERR "***********output filestem $output_filestem.html\n";
854 &FileUtils::removeFiles("$output_filestem.html") if (-e "$output_filestem.html");
855 if (-e "$output_filestem.err") {
856 if ($faillogfile ne "" && defined(open(FAILLOG,">>$faillogfile")))
857 {
858 open (ERRLOG, "$output_filestem.err");
859 while (<ERRLOG>) {print FAILLOG $_;}
860 close ERRLOG;
861 close FAILLOG;
862 }
863 &FileUtils::removeFiles("$output_filestem.err");
864 }
865 return 0;
866 }
867
868 &FileUtils::removeFiles("$output_filestem.err") if (-e "$output_filestem.err");
869 &FileUtils::removeFiles("$output_filestem.out") if (-e "$output_filestem.out");
870 return 1;
871}
872
873
874# Convert a pdf file to html with the newer Xpdftools' pdftohtml
875# This generates "paged HTML" where extracted, selectable text is positioned
876# over screenshots of each page.
877# Since xpdf's pdftohtml fails if the output dir already exists and for easier
878# naming, the output files are created in a "pages" subdirectory of the tmp
879# location parent of $output_filestem instead
880sub xpdf_to_html {
881 my ($dirname, $input_filename, $output_filestem) = @_;
882
883 my $cmd = "";
884
885 # build up the path to the doc-to-html conversion tool we're going to use
886 my $xpdf_pdftohtml = &FileUtils::filenameConcatenate(_get_xpdftools_bindir(), "pdftohtml");
887
888 # We'll create the file by name $output_filestem during post-conversion processing.
889 # Note that Xpdf tools will only create its conversion products in a dir that does
890 # not yet exist. So we'll create this location as a subdir of the output_filestem's
891 # parent directory. The parent dir is the already generated tmp area for conversion. So:
892 # - tmpdir gs2build/tmp/<random-num> already exists at this stage
893 # - We'll create gs2build/tmp/<rand>/output_filestem.html later, during post-processing
894 # - For now, XPdftools will create gs2build/tmp/<rand>/pages and put its products in there.
895 my ($tailname, $tmp_dirname, $suffix)
896 = &File::Basename::fileparse($output_filestem, "\\.[^\\.]+\$");
897 $tmp_dirname = &FileUtils::filenameConcatenate($tmp_dirname, "pages");
898
899 # xpdf's pdftohtml tool also takes a zoom factor, where a zoom of 1 is 100%
900 $cmd .= "\"$xpdf_pdftohtml\"";
901 $cmd .= " -z $pdf_zoom" if ($pdf_zoom);
902# $cmd .= " -c" if ($pdf_complex);
903# $cmd .= " -i" if ($pdf_ignore_images);
904# $cmd .= " -a" if ($pdf_allow_images_only);
905# $cmd .= " -hidden" unless ($pdf_nohidden);
906 $cmd .= " \"$input_filename\" \"$tmp_dirname\"";
907 #$cmd .= " \"$input_filename\" \"$output_filestem\"";
908
909 if ($ENV{'GSDLOS'} !~ m/^windows$/i || $is_winnt_2000) {
910 $cmd .= " > \"$output_filestem.out\" 2> \"$output_filestem.err\"";
911 } else {
912 $cmd .= " > \"$output_filestem.err\"";
913 }
914
915 #print STDERR "@@@@ Running command: $cmd\n";
916
917 $!=0;
918 my $retval=system($cmd);
919 if ($retval!=0)
920 {
921 print STDERR "Error executing xpdf's pdftohtml tool";
922 if ($!) {print STDERR ": $!";}
923 print STDERR "\n";
924 }
925
926 # make sure the converter made something
927 if ($retval!=0 || ! -s &FileUtils::filenameConcatenate($tmp_dirname,"index.html"))
928 {
929 &FileUtils::removeFiles("$output_filestem.out") if (-e "$output_filestem.out");
930 # print out the converter's std err, if any
931 if (-s "$output_filestem.err") {
932 open (ERRLOG, "$output_filestem.err") || die "$!";
933 print STDERR "pdftohtml error log:\n";
934 while (<ERRLOG>) {
935 print STDERR "$_";
936 }
937 close ERRLOG;
938 }
939 #print STDERR "***********output filestem $output_filestem.html\n";
940 &FileUtils::removeFiles("$tmp_dirname") if (-d "$tmp_dirname");
941 if (-e "$output_filestem.err") {
942 if ($faillogfile ne "" && defined(open(FAILLOG,">>$faillogfile")))
943 {
944 open (ERRLOG, "$output_filestem.err");
945 while (<ERRLOG>) {print FAILLOG $_;}
946 close ERRLOG;
947 close FAILLOG;
948 }
949 &FileUtils::removeFiles("$output_filestem.err");
950 }
951 return 0;
952 }
953
954 &FileUtils::removeFiles("$output_filestem.err") if (-e "$output_filestem.err");
955 &FileUtils::removeFiles("$output_filestem.out") if (-e "$output_filestem.out");
956 return 1;
957}
958
959# Returns the path to xpdf-tools's containing bin dir appropriate for this machine's OS and bitness
960sub _get_xpdftools_bindir {
961
962 # build up the path to the containing bin dir of the xpdf conversion tool we're going to use
963 my $xpdf_tools_bin = &FileUtils::filenameConcatenate($ENV{'GSDLHOME'}, "bin", $ENV{'GSDLOS'}, "xpdf-tools", "bin");
964 return $xpdf_tools_bin;
965}
966
967# Convert a pdf file to various types of image with the convert command
968
969sub pdfps_to_img {
970 my ($dirname, $input_filename, $output_filestem, $output_type) = @_;
971
972 # Check that ImageMagick is installed and available on the path (except for Windows 95/98)
973 if (!($ENV{'GSDLOS'} eq "windows" && !Win32::IsWinNT())) {
974 my $imagick_cmd = "\"".&util::get_perl_exec()."\" -S gs-magick.pl";
975 $imagick_cmd = $imagick_cmd." --verbosity=$verbosity" if defined $verbosity;
976 my $result = `$imagick_cmd identify 2>&1`;
977
978 # Linux and Windows return different values for "program not found".
979 # Linux returns -1 and Windows 256 for "program not found". But once they're
980 # converted to signed values, it will be -1 for Linux and 1 for Windows.
981 # Whenever we test for return values other than 0, shift by 8 and perform
982 # unsigned to signed status conversion on $? to get expected range of return vals
983 # Although gs-magick.pl already shifts its $? by 8, converts it to a signed value
984 # and then exits on that, by the time we get here, we need to do it again
985 my $status = $?;
986 $status >>= 8;
987 $status = (($status & 0x80) ? -(0x100 - ($status & 0xFF)) : $status);
988 if (($ENV{'GSDLOS'} ne "windows" && $status == -1) || ($ENV{'GSDLOS'} eq "windows" && $status == 1)) {
989 # if ($status == -1 || $status == 1) #if ($status == -1 || $status == 256) {
990 #ImageMagick is not installed, thus the convert utility is not available.
991 print STDERR "*** ImageMagick is not installed, the convert utility is not available. Unable to convert PDF/PS to images. Status: $status\n";
992 return 0;
993 }
994 }
995
996 my $cmd = "";
997 if ($timeout) {$cmd = "ulimit -t $timeout;";}
998 $output_type =~ s/.*\_(.*)/$1/i;
999 my $full_perl_path = &util::get_perl_exec();
1000 $cmd .= "\"$full_perl_path\" -S pdfpstoimg.pl -convert_to $output_type \"$input_filename\" \"$output_filestem\"";
1001 if ($ENV{'GSDLOS'} !~ m/^windows$/i || $is_winnt_2000) {
1002 $cmd .= " > \"$output_filestem.out\" 2> \"$output_filestem.err\"";
1003 } else {
1004 $cmd .= " > \"$output_filestem.err\"";
1005 }
1006
1007 # don't include path on windows (to avoid having to play about
1008 # with quoting when GSDLHOME might contain spaces) but assume
1009 # that the PATH is set up correctly
1010 $!=0;
1011 my $retval=system($cmd);
1012 if ($retval!=0)
1013 {
1014 print STDERR "Error executing pdfpstoimg.pl";
1015 if ($!) {print STDERR ": $!";}
1016 print STDERR "\n";
1017 }
1018
1019 #make sure the converter made something
1020 #if ($retval !=0) || ! -s "$output_filestem")
1021 if ($retval !=0)
1022 {
1023 &FileUtils::removeFiles("$output_filestem.out") if (-e "$output_filestem.out");
1024 #print out the converter's std err, if any
1025 if (-s "$output_filestem.err") {
1026 open (ERRLOG, "$output_filestem.err") || die "$!";
1027 print STDERR "pdfpstoimg error log:\n";
1028 while (<ERRLOG>) {
1029 print STDERR "$_";
1030 }
1031 close ERRLOG;
1032 }
1033 #&FileUtils::removeFiles("$output_filestem.html") if (-e "$output_filestem.html");
1034 if (-e "$output_filestem.err") {
1035 if ($faillogfile ne "" && defined(open(FAILLOG,">>$faillogfile")))
1036 {
1037 open (ERRLOG, "$output_filestem.err");
1038 while (<ERRLOG>) {print FAILLOG $_;}
1039 close ERRLOG;
1040 close FAILLOG;
1041 }
1042 &FileUtils::removeFiles("$output_filestem.err");
1043 }
1044 return 0;
1045 }
1046 &FileUtils::removeFiles("$output_filestem.err") if (-e "$output_filestem.err");
1047 &FileUtils::removeFiles("$output_filestem.out") if (-e "$output_filestem.out");
1048 return 1;
1049}
1050
1051# Convert a PDF file to text with xpdftools' pdftotext command
1052# Works for Windows too, whereas the old pdftotxt didn't
1053sub xpdf_to_text {
1054 my ($dirname, $input_filename, $output_filestem, $output_type) = @_;
1055
1056 my $cmd = "";
1057
1058 # build up the path to the doc-to-txt conversion tool we're going to use
1059 my $xpdf_pdftotxt = &FileUtils::filenameConcatenate(_get_xpdftools_bindir(), "pdftotext");
1060
1061 # For xpdf's pdftotxt options, see https://www.xpdfreader.com/pdftotext-man.html
1062 $cmd .= "\"$xpdf_pdftotxt\"";
1063 if($enc) {
1064 $cmd .= " -enc $enc"; # decode the bytes in the file using the designated encoding scheme
1065 } else {
1066 # as per https://www.xpdfreader.com/pdftotext-man.html
1067 # xpdf's pdftotxt defaults to using Latin-1 encoding, should we default to UTF-8?
1068 $cmd .= " -enc UTF-8"; # see https://www.xpdfreader.com/xpdfrc-man.html
1069 }
1070
1071 if ($output_type ne "paged_text") { # output_type eq "text", don't bother about page break markers
1072 $cmd .= " -nopgbrk";
1073 }
1074 # Avoid the silly solitary carriage returns (CR in Notepad) at the end
1075 # of lines that ends up as \n appended to the doc title
1076 # by setting the end of line marker to unix style solitary newline (LF or \n),
1077 # which doesn't end up in the doc title
1078 $cmd .= " -eol unix";
1079 $cmd .= " \"$input_filename\" \"$output_filestem.text\"";
1080
1081 print STDERR "@@@@ Running command: $cmd\n";
1082
1083 return _run_pdf_to_text_cmd($cmd, $output_filestem);
1084}
1085
1086# Convert a PDF file to text with the pdftotext command
1087
1088sub pdf_to_text {
1089 my ($dirname, $input_filename, $output_filestem) = @_;
1090
1091 my $cmd = "pdftotext \"$input_filename\" \"$output_filestem.text\"";
1092
1093 return _run_pdf_to_text_cmd($cmd, $output_filestem);
1094}
1095
1096sub _run_pdf_to_text_cmd {
1097 my ($cmd, $output_filestem) = @_;
1098
1099 if ($ENV{'GSDLOS'} !~ m/^windows$/i) {
1100 $cmd .= " > \"$output_filestem.out\" 2> \"$output_filestem.err\"";
1101 } else {
1102 $cmd .= " > \"$output_filestem.err\"";
1103 }
1104
1105 if (system($cmd)!=0)
1106 {
1107 print STDERR "Error executing $cmd: $!\n";
1108 &FileUtils::removeFiles("$output_filestem.text") if (-e "$output_filestem.text");
1109 }
1110
1111 # make sure there is some extracted text.
1112 if (-e "$output_filestem.text") {
1113 open (EXTR_TEXT, "$output_filestem.text") || warn "open: $!";
1114 binmode(EXTR_TEXT); # just in case...
1115 my $line="";
1116 my $seen_text=0;
1117 while (($seen_text==0) && ($line=<EXTR_TEXT>)) {
1118 if ($line=~ m/\w/) {$seen_text=1;}
1119 }
1120 close EXTR_TEXT;
1121 if ($seen_text==0) { # no text was extracted
1122 print STDERR "Error: pdftotext found no text\n";
1123 &FileUtils::removeFiles("$output_filestem.text");
1124 }
1125 }
1126
1127 # make sure the converter made something
1128 if (! -s "$output_filestem.text")
1129 {
1130 # print out the converters std err, if any
1131 if (-s "$output_filestem.err") {
1132 open (ERRLOG, "$output_filestem.err") || die "$!";
1133 print STDERR "pdftotext error log:\n";
1134 while (<ERRLOG>) {
1135 print STDERR "$_";
1136 }
1137 close ERRLOG;
1138 }
1139 # does this converter create a .out file?
1140 &FileUtils::removeFiles("$output_filestem.out") if (-e "$output_filestem.out");
1141 &FileUtils::removeFiles("$output_filestem.text") if (-e "$output_filestem.text");
1142 if (-e "$output_filestem.err") {
1143 if ($faillogfile ne "" && defined(open(FAILLOG,">>$faillogfile")))
1144 {
1145 open (ERRLOG,"$output_filestem.err");
1146 while (<ERRLOG>) {print FAILLOG $_;}
1147 close ERRLOG;
1148 close FAILLOG;
1149 }
1150 &FileUtils::removeFiles("$output_filestem.err");
1151 }
1152 return 0;
1153 }
1154 &FileUtils::removeFiles("$output_filestem.err") if (-e "$output_filestem.err");
1155 return 1;
1156}
1157
1158# Convert a PostScript document to text
1159# note - just using "ps2ascii" isn't good enough, as it
1160# returns 0 for a postscript interpreter error. ps2ascii is just
1161# a wrapper to "gs" anyway, so we use that cmd here.
1162
1163sub ps_to_text {
1164 my ($input_filename, $output_filestem) = @_;
1165
1166 my $error = "";
1167
1168 # if we're on windows we'll fall straight through without attempting
1169 # to use gs
1170 if ($ENV{'GSDLOS'} =~ m/^windows$/i) {
1171 $error = "Windows does not support gs";
1172
1173 } else {
1174 my $cmd = "";
1175 if ($timeout) {$cmd = "ulimit -t $timeout; ";}
1176 $cmd .= "gs -q -dNODISPLAY -dNOBIND -dWRITESYSTEMDICT -dSIMPLE -c save ";
1177 $cmd .= "-f ps2ascii.ps \"$input_filename\" -c quit > \"$output_filestem.text\"";
1178 #$cmd .= "pstotext -output \"$output_filestem.text\" $input_filename\"";
1179 $cmd .= " 2> $output_filestem.err";
1180 $!=0;
1181
1182 my $retcode=system($cmd);
1183 $retcode = $? >> 8; # see man perlfunc - system for this...
1184 # if system returns -1 | 127 (couldn't start program), look at $! for message
1185
1186 if ($retcode!=0) {if ($!) {$error=$!;} else {$error="couldn't run.\n";}}
1187 elsif (! -e "$output_filestem.text") {
1188 $error="did not create output file.\n";
1189 }
1190 else
1191 { # make sure the interpreter didn't get an error. It is technically
1192 # possible for the actual text to start with this, but....
1193 open PSOUT, "$output_filestem.text";
1194 if (<PSOUT> =~ m/^Error: (.*)/) {
1195 $error="interpreter error - \"$1\"";
1196 }
1197 close PSOUT;
1198 }
1199 }
1200
1201 if ($error ne "")
1202 {
1203 print STDERR "Warning: Error executing gs: $error\n";
1204 print STDERR "Resorting to Perl regular expressions to extract text from PostScript...\n";
1205 &FileUtils::removeFiles("$output_filestem.text") if (-e "$output_filestem.text");
1206
1207 if ("$faillogfile" ne "" && defined(open (FAILLOG, ">>$faillogfile")))
1208 {
1209 print FAILLOG "gs - $error\n";
1210 if (-e "$output_filestem.err") {
1211 open(ERRLOG, "$output_filestem.err");
1212 while (<ERRLOG>) {print FAILLOG $_;}
1213 close ERRLOG;
1214 }
1215 close FAILLOG;
1216 }
1217 &FileUtils::removeFiles("$output_filestem.err") if (-e "$output_filestem.err");
1218
1219
1220 # Fine then. We'll just do a lousy job by ourselves...
1221 # Based on 5-line regexp sed script found at:
1222 # http://snark.ptc.spbu.ru/mail-archives/lout/brown/msg00003.html
1223 #
1224 print STDERR "Stripping text from postscript\n";
1225 my $errorcode=0;
1226 open (IN, "$input_filename")
1227 || ($errorcode=1, warn "Couldn't read file: $!");
1228 open (OUT, ">$output_filestem.text")
1229 || ($errorcode=1, warn "Couldn't write file: $!");
1230 if ($errorcode) {print STDERR "errors\n";return 0;}
1231
1232 my $text=""; # this is for whole .ps file...
1233 $text = join('', <IN>); # see man perlport, under "System Resources"
1234 close IN;
1235
1236 # Make sure this is a ps file...
1237 if ($text !~ m/^%!/) {
1238 print STDERR "Bad postscript header: not '%!'\n";
1239 if ($faillogfile ne "" && defined(open(FAILLOG, ">>$faillogfile")))
1240 {
1241 print FAILLOG "Bad postscript header: not '%!'\n";
1242 close FAILLOG;
1243 }
1244 return 0;
1245 }
1246
1247 # if ps has Page data, then use it to delete all stuff before it.
1248 $text =~ s/^.*?%%Page:.*?\n//s; # treat string as single line
1249
1250 # remove all leading non-data stuff
1251 $text =~ s/^.*?\(//s;
1252
1253 # remove all newline chars for easier processing
1254 $text =~ s/\n//g;
1255
1256 # Big assumption here - assume that if any co-ordinates are
1257 # given, then we are at the end of a sentence.
1258 $text =~ s/\)-?\d+\ -?\d+/\) \(\n\)/g;
1259
1260 # special characters--
1261 $text =~ s/\(\|\)/\(\ - \)/g; # j -> em-dash?
1262
1263 # ? ps text formatting (eg italics?) ?
1264 $text =~ s/Fn\(f\)/\(\{\)/g; # f -> {
1265 $text =~ s/Fn\(g\)/\(\}\)/g; # g -> }
1266 $text =~ s/Fn\(j\)/\(\|\)/g; # j -> |
1267 # default - remove the rest
1268 $text =~ s/\ ?F.\((.+?)\)/\($1\)/g;
1269
1270 # attempt to add whitespace between words...
1271 # this is based purely on observation, and may be completely wrong...
1272 $text =~ s/([^F])[defghijkuy]\(/$1 \( /g;
1273 # eg I notice "b(" is sometimes NOT a space if preceded by a
1274 # negative number.
1275 $text =~ s/\)\d+ ?b\(/\) \( /g;
1276
1277 # change quoted braces to brackets
1278 $text =~ s/([^\\])\\\(/$1\{/g;
1279 $text =~ s/([^\\])\\\)/$1\}/g ;
1280
1281 # remove everything that is not between braces
1282 $text =~ s/\)([^\(\)])+?\(//sg ;
1283
1284 # remove any Trailer eof stuff.
1285 $text =~ s/\)[^\)]*$//sg;
1286
1287 ### ligatures have special characters...
1288 $text =~ s/\\013/ff/g;
1289 $text =~ s/\\014/fi/g;
1290 $text =~ s/\\015/fl/g;
1291 $text =~ s/\\016/ffi/g;
1292 $text =~ s/\\214/fi/g;
1293 $text =~ s/\\215/fl/g;
1294 $text =~ s/\\017/\n\* /g; # asterisk?
1295 $text =~ s/\\023/\023/g; # e acute ('e)
1296 $text =~ s/\\177/\252/g; # u"
1297# $text =~ s/ ?? /\344/g; # a"
1298
1299 print OUT "$text";
1300 close OUT;
1301 }
1302 # wrap the text - use a minimum length. ie, first space after this length.
1303 my $wrap_length=72;
1304 &FileUtils::moveFiles("$output_filestem.text", "$output_filestem.text.tmp");
1305 open INFILE, "$output_filestem.text.tmp" ||
1306 die "Couldn't open file: $!";
1307 open OUTFILE, ">$output_filestem.text" ||
1308 die "Couldn't open file for writing: $!";
1309 my $line="";
1310 while ($line=<INFILE>) {
1311 while (length($line)>0) {
1312 if (length($line)>$wrap_length) {
1313 $line =~ s/^(.{$wrap_length}[^\s]*)\s*//;
1314 print OUTFILE "$1\n";
1315 } else {
1316 print OUTFILE "$line";
1317 $line="";
1318 }
1319 }
1320 }
1321 close INFILE;
1322 close OUTFILE;
1323 &FileUtils::removeFiles("$output_filestem.text.tmp");
1324
1325 &FileUtils::removeFiles("$output_filestem.err") if (-e "$output_filestem.err");
1326 return 1;
1327}
1328
1329
1330# Convert any file to HTML with a crude perl implementation of the
1331# UNIX strings command.
1332
1333sub any_to_html {
1334 my ($input_filename, $output_filestem) = @_;
1335
1336 # First generate a text file
1337 return 0 unless (&any_to_text($input_filename, $output_filestem));
1338
1339 # create an HTML file from the text file
1340 open(TEXT, "<$output_filestem.text");
1341 open(HTML, ">$output_filestem.html");
1342
1343 print HTML "<html><head>\n";
1344 print HTML "<META HTTP-EQUIV=\"Content-Type\" CONTENT=\"text/html\">\n";
1345 print HTML "<META NAME=\"GENERATOR\" CONTENT=\"Greenstone any_to_html\">\n";
1346 print HTML "</head><body>\n\n";
1347
1348 my $line;
1349 while ($line=<TEXT>) {
1350 $line =~ s/</&lt;/g;
1351 $line =~ s/>/&gt;/g;
1352 if ($line =~ m/^\s*$/) {
1353 print HTML "<p>";
1354 } else {
1355 print HTML "<br> ", $line;
1356 }
1357 }
1358 print HTML "\n</body></html>\n";
1359
1360 close HTML;
1361 close TEXT;
1362
1363 &FileUtils::removeFiles("$output_filestem.text") if (-e "$output_filestem.text");
1364 return 1;
1365}
1366
1367# Convert any file to TEXT with a crude perl implementation of the
1368# UNIX strings command.
1369# Note - this assumes ascii charsets :( (jrm21)
1370
1371sub any_to_text {
1372 my ($input_filename, $output_filestem) = @_;
1373
1374 if (!$use_strings) {
1375 return 0;
1376 }
1377
1378 print STDERR "\n**** In any to text****\n\n";
1379 open(IN, "<$input_filename") || return 0;
1380 binmode(IN);
1381 open(OUT, ">$output_filestem.text") || return 0;
1382
1383 my ($line);
1384 my $output_line_count = 0;
1385 while (<IN>) {
1386 $line = $_;
1387
1388 # delete anything that isn't a printable character
1389 $line =~ s/[^\040-\176]+/\n/sg;
1390
1391 # delete any string less than 10 characters long
1392 $line =~ s/^.{0,9}$/\n/mg;
1393 while ($line =~ m/^.{1,9}$/m) {
1394 $line =~ s/^.{0,9}$/\n/mg;
1395 $line =~ s/\n+/\n/sg;
1396 }
1397
1398 # remove extraneous whitespace
1399 $line =~ s/\n+/\n/gs;
1400 $line =~ s/^\n//gs;
1401
1402 # output whatever is left
1403 if ($line =~ m/[^\n ]/) {
1404 print OUT $line;
1405 ++$output_line_count;
1406 }
1407 }
1408
1409 close OUT;
1410 close IN;
1411
1412 if ($output_line_count) { # try to protect against binary only formats
1413 return 1;
1414 }
1415
1416 &FileUtils::removeFiles("$output_filestem.text");
1417 return 0;
1418
1419}
Note: See TracBrowser for help on using the repository browser.