Context Navigation

source: main/trunk/greenstone2/bin/script/gsConvert.pl@ 32277

Last change on this file since 32277 was 32277, checked in by ak19, 6 years ago
First attempt at PDFv2Plugin.pm.
Property svn:executable set to ``* Property svn:keywords set to `Author Date Id Revision`
File size: 43.9 KB

Rev	Line
[1445]	1	#!/usr/bin/perl -w
	2
	3	###########################################################################
	4	#
[2032]	5	# gsConvert.pl -- convert documents to HTML or TEXT format
[1445]	6	#
	7	# A component of the Greenstone digital library software
	8	# from the New Zealand Digital Library Project at the
	9	# University of Waikato, New Zealand.
	10	#
[3013]	11	# Copyright (C) 1999-2002 New Zealand Digital Library Project
[1445]	12	#
	13	# This program is free software; you can redistribute it and/or modify
	14	# it under the terms of the GNU General Public License as published by
	15	# the Free Software Foundation; either version 2 of the License, or
	16	# (at your option) any later version.
	17	#
	18	# This program is distributed in the hope that it will be useful,
	19	# but WITHOUT ANY WARRANTY; without even the implied warranty of
	20	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	21	# GNU General Public License for more details.
	22	#
	23	# You should have received a copy of the GNU General Public License
	24	# along with this program; if not, write to the Free Software
	25	# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
	26	#
	27	###########################################################################
	28
	29	# gsConvert.pl converts documents in a range of formats to HTML or TEXT
[2755]	30	# by exploiting third-party programs. The sources of these are usually found
	31	# in the $GSDLHOME/packages directory, and the executables should live in
	32	# $GSDLHOME/bin/$GSDLOS (which is on the search path).
[1445]	33	#
[3013]	34	# Currently, we can convert the following formats by using external
	35	# conversion utilities:
	36	# Microsoft Word (versions 2,6,7 [==95?], 8[==97?], 9[==2000?]), RTF,
	37	# Adobe PDF, PostScript, MS PowerPoint (95 and 97), and MS Excel (95 and 97).
[2032]	38	#
[3013]	39	# We can try to convert any file to text with a perl implementation of the
	40	# UNIX strings command.
	41	#
[2032]	42	# We try to convert Postscript files to text using "gs" which is often on
[2755]	43	# *nix machines. We fall back to performing weak text extraction by using
	44	# regular expressions.
[1445]	45
	46	BEGIN {
	47	die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
	48	unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
	49	}
	50
[22429]	51	use strict;
	52
[1445]	53	use parsargv;
	54	use util;
[27509]	55	use FileUtils;
[1445]	56	use Cwd;
	57
[2755]	58	# Are we running on WinNT or Win2000 (or later)?
	59	my $is_winnt_2000=eval {require Win32; return (Win32::IsWinNT()); return 0;};
	60	if (!defined($is_winnt_2000)) {$is_winnt_2000=0;}
[1445]	61
[3350]	62	my $use_strings;
[32273]	63	my $pdf_tool;
[3720]	64	my $pdf_complex;
[4103]	65	my $pdf_nohidden;
[3720]	66	my $pdf_zoom;
	67	my $pdf_ignore_images;
[10451]	68	my $pdf_allow_images_only;
[10282]	69	my $windows_scripting;
[32224]	70	my $enc;
[3350]	71
[1445]	72	sub print_usage
	73	{
[1970]	74	print STDERR "\n";
	75	print STDERR "gsConvert.pl: Converts documents in a range of formats to html\n";
	76	print STDERR " or text using third-party programs.\n\n";
	77	print STDERR " usage: $0 [options] filename\n";
[22642]	78	print STDERR " options:\n\t-type\tdoc\|dot\|pdf\|ps\|ppt\|rtf\|xls\t(input file type)\n";
[2755]	79	print STDERR "\t-errlog\t<filename>\t(append err messages)\n";
[32277]	80	print STDERR "\t-output\tauto\|html\|pretty_html\|paged_pretty_html\|paged_html\|text\|paged_text\|pagedimg_jpg\|pagedimg_gif\|pagedimg_png\|pagedimgtxt_jpg\|pagedimgtxt_png\t(output file type)\n";
[2755]	81	print STDERR "\t-timeout\t<max cpu seconds>\t(ulimit on unix systems)\n";
[3720]	82	print STDERR "\t-use_strings\tuse strings to extract text if conversion fails\n";
[22568]	83	print STDERR "\t-windows_scripting\tuse windows VB script (if available) to convert Microsoft Word and PPT documents\n";
[32273]	84	print STDERR "\t-pdf_tool\tpdftohtml\|xpdftools\|pdfbox (not all output types are supported by every pdf_tool)\n";
[3720]	85	print STDERR "\t-pdf_complex\tuse complex output when converting PDF to HTML\n";
[4103]	86	print STDERR "\t-pdf_nohidden\tDon't attempt to extract hidden text from PDF files\n";
[3720]	87	print STDERR "\t-pdf_ignore_images\tdon't attempt to extract images when\n";
	88	print STDERR "\t\tconverting PDF to HTML\n";
[10451]	89	print STDERR "\t-pdf_allow_images_only\tallow images only (continue even if no text is present when converting to HTML)\n";
[3720]	90	print STDERR "\t-pdf_zoom\tfactor by which to zoom PDF (only useful if\n";
	91	print STDERR "\t\t-pdf_complex is set\n";
[1445]	92	exit(1);
	93	}
	94
[2755]	95	my $faillogfile="";
[3538]	96	my $timeout=0;
[24375]	97	my $verbosity=0;
[1445]	98
	99	sub main
	100	{
	101	my (@ARGV) = @_;
[3538]	102	my ($input_type,$output_type,$verbose);
[1960]	103
[23473]	104	# Dynamically figure out what the --type option can support, based on whether -windows_scripting
	105	# is in use or not
	106	my $default_type_re = "(doc\|dot\|pdf\|ps\|ppt\|rtf\|xls)";
	107	#my $enhanced_type_re = "(docx?\|dot\|pdf\|ps\|pptx?\|rtf\|xlsx?)";
	108	#my $enhanced_type_re = "(docx?\|dot\|pdf\|ps\|pptx?\|rtf\|xlsx?)";
	109	# Currently only have VBA for Word and PPT(but no XLS)
	110	my $enhanced_type_re = "(docx?\|dot\|pdf\|ps\|pptx?\|rtf\|xls)";
	111
	112	my $type_re = $default_type_re;
	113
	114	foreach my $a (@ARGV) {
	115	if ($a =~ m/^windows_scripting$/i) {
	116	$type_re = $enhanced_type_re;
	117	}
	118	}
	119
[1445]	120	# read command-line arguments
	121	if (!parsargv::parse(\@ARGV,
[23473]	122	"type/$type_re/", \$input_type,
[2755]	123	'/errlog/.*/', \$faillogfile,
[32273]	124	'output/(auto\|html\|text\|pagedimg).*/', \$output_type, # regex includes html_multi and paged_html besides html
[1692]	125	'timeout/\d+/0',\$timeout,
[10282]	126	'verbose/\d+/0', \$verbose,
[22429]	127	'windows_scripting',\$windows_scripting,
[3720]	128	'use_strings', \$use_strings,
[32273]	129	'pdf_tool/(pdftohtml\|pdfbox\|xpdftools)/', \$pdf_tool, # the old pdftohtml tool, pdfbox extensions or the newer xpdf-tools
	130	'pdf_complex', \$pdf_complex, # options for pdf_tool = pdftohtml (the old pdftohtml tool)
[9482]	131	'pdf_ignore_images', \$pdf_ignore_images,
[10451]	132	'pdf_allow_images_only', \$pdf_allow_images_only,
[4103]	133	'pdf_nohidden', \$pdf_nohidden,
[3720]	134	'pdf_zoom/\d+/2', \$pdf_zoom
	135	))
[1445]	136	{
	137	print_usage();
	138	}
[24375]	139
	140	$verbosity=$verbose if defined $verbose;
	141
[1445]	142	# Make sure the input file exists and can be opened for reading
	143	if (scalar(@ARGV!=1)) {
	144	print_usage();
	145	}
[1928]	146
[1445]	147	my $input_filename = $ARGV[0];
	148	if (!-r $input_filename) {
	149	print STDERR "Error: unable to open $input_filename for reading\n";
	150	exit(1);
	151	}
	152
	153	# Deduce filenames
	154	my ($tailname,$dirname,$suffix)
[2241]	155	= File::Basename::fileparse($input_filename, "\\.[^\\.]+\$");
[27509]	156	my $output_filestem = &FileUtils::filenameConcatenate($dirname, "$tailname");
[1445]	157
	158	if ($input_type eq "")
	159	{
[2241]	160	$input_type = lc (substr($suffix,1,length($suffix)-1));
[1445]	161	}
	162
	163	# Change to temporary working directory
	164	my $stored_dir = cwd();
	165	chdir ($dirname) \|\| die "Unable to change to directory $dirname";
[10357]	166
[1445]	167	# Select convert utility
	168	if (!defined $input_type) {
	169	print STDERR "Error: No filename extension or input type defined\n";
	170	exit(1);
	171	}
[23473]	172	elsif ($input_type =~ m/^docx?$/ \|\| $input_type eq "dot") {
[1445]	173	print &convertDOC($input_filename, $output_filestem, $output_type);
	174	print "\n";
	175	}
[1684]	176	elsif ($input_type eq "rtf") {
	177	print &convertRTF($input_filename, $output_filestem, $output_type);
	178	print "\n";
	179	}
[1445]	180	elsif ($input_type eq "pdf") {
	181	print &convertPDF($dirname, $input_filename, $output_filestem, $output_type);
	182	print "\n";
	183	}
	184	elsif ($input_type eq "ps") {
[22429]	185	print &convertPS($dirname, $input_filename, $output_filestem, $output_type);
[1445]	186	print "\n";
	187	}
[23473]	188	elsif ($input_type =~ m/pptx?$/) {
[2977]	189	print &convertPPT($input_filename, $output_filestem, $output_type);
	190	print "\n";
	191	}
[23473]	192	elsif ($input_type =~ m/xlsx?$/) {
[2991]	193	print &convertXLS($input_filename, $output_filestem, $output_type);
	194	print "\n";
	195	}
[1445]	196	else {
	197	print STDERR "Error: Unable to convert type '$input_type'\n";
	198	exit(1);
	199	}
	200
	201	# restore to original working directory
	202	chdir ($stored_dir) \|\| die "Unable to return to directory $stored_dir";
	203
	204	}
	205
	206	&main(@ARGV);
	207
	208
	209
[2241]	210	# Document-type conversion functions
[1445]	211	#
	212	# The following functions attempt to convert documents from their
	213	# input type to the specified output type. If no output type was
	214	# given, then they first attempt HTML, and then TEXT.
	215	#
	216	# Each returns the output type ("html" or "text") or "fail" if no
	217	# conversion is possible.
	218
	219	# Convert a Microsoft word document
	220
	221	sub convertDOC {
[22429]	222	my ($input_filename, $output_filestem, $output_type) = @_;
[1445]	223
[1654]	224	# Many .doc files are not in fact word documents!
	225	my $realtype = &find_docfile_type($input_filename);
	226
[23473]	227	if ($realtype eq "word6" \|\| $realtype eq "word7"
	228	\|\| $realtype eq "word8" \|\| $realtype eq "docx") {
[1654]	229	return &convertWord678($input_filename, $output_filestem, $output_type);
	230	} elsif ($realtype eq "rtf") {
	231	return &convertRTF($input_filename, $output_filestem, $output_type);
	232	} else {
	233	return &convertAnything($input_filename, $output_filestem, $output_type);
	234	}
	235	}
	236
	237	# Convert a Microsoft word 6/7/8 document
	238
	239	sub convertWord678 {
[22429]	240	my ($input_filename, $output_filestem, $output_type) = @_;
[1654]	241
[1445]	242	my $success = 0;
[16435]	243	if (!$output_type \|\| ($output_type =~ m/html/i)){
[10282]	244	if ($windows_scripting) {
	245	$success = &native_doc_to_html($input_filename, $output_filestem);
	246	}
	247	else {
	248	$success = &doc_to_html($input_filename, $output_filestem);
	249	}
[1445]	250	if ($success) {
[10282]	251	return "html";
[1445]	252	}
	253	}
[1654]	254	return &convertAnything($input_filename, $output_filestem, $output_type);
	255	}
	256
	257
	258	# Convert a Rich Text Format (RTF) file
	259
	260	sub convertRTF {
[22429]	261	my ($input_filename, $output_filestem, $output_type) = @_;
[1654]	262
	263	my $success = 0;
	264
	265	# Attempt specialised conversion to HTML
[16435]	266	if (!$output_type \|\| ($output_type =~ m/html/i)) {
[12704]	267
	268	if ($windows_scripting) {
	269	$success = &native_doc_to_html($input_filename, $output_filestem);
	270	}
	271	else {
	272	$success = &rtf_to_html($input_filename, $output_filestem);
	273	}
[1654]	274	if ($success) {
	275	return "html";
	276	}
	277	}
	278
[2755]	279	# rtf is so ugly that's it's not worth running strings over.
	280	# One day I'll write some quick'n'dirty regexps to try to extract text - jrm21
	281	# return &convertAnything($input_filename, $output_filestem, $output_type);
	282	return "fail";
[1654]	283	}
	284
	285
	286	# Convert an unidentified file
	287
	288	sub convertAnything {
[22429]	289	my ($input_filename, $output_filestem, $output_type) = @_;
[1654]	290
	291	my $success = 0;
[10464]	292
[1445]	293	# Attempt simple conversion to HTML
[16435]	294	if (!$output_type \|\| ($output_type =~ m/html/i)) {
[1445]	295	$success = &any_to_html($input_filename, $output_filestem);
	296	if ($success) {
	297	return "html";
	298	}
	299	}
	300
	301	# Convert to text
[16435]	302	if (!$output_type \|\| ($output_type =~ m/text/i)) {
[2241]	303	$success = &any_to_text($input_filename, $output_filestem);
[1445]	304	if ($success) {
	305	return "text";
	306	}
	307	}
	308	return "fail";
	309	}
	310
	311
[1654]	312
[1445]	313	# Convert an Adobe PDF document
	314
	315	sub convertPDF {
[2755]	316	my ($dirname, $input_filename, $output_filestem, $output_type) = @_;
[1445]	317
	318	my $success = 0;
[10357]	319	$output_type =~ s/.\-(.)/$1/i;
[32277]	320
	321	print STDERR "@@@@@@@@ Using $pdf_tool for the conversion\n";
	322
[32273]	323	# First determine which pdf conversion tool we're using among pdftohtml/pdfbox/xpdftools
	324	# and then decide which conversion command to run based on the output type
	325	# (pdfbox does not currently go through gsConvert.pl
	326	# as PDFBoxConverter inherits from AutoLoadConverters)
	327
	328	if ($pdf_tool eq "pdftohtml" ) { # old pdftohtml tool
[10357]	329	# Attempt coversion to Image
[16435]	330	if ($output_type =~ m/jp?g\|gif\|png/i) {
[17329]	331	$success = &pdfps_to_img($dirname, $input_filename, $output_filestem, $output_type);
[10357]	332	if ($success){
	333	return "item";
	334	}
	335	}
[1445]	336
	337	# Attempt conversion to HTML
[32205]	338	# Uses the old pdftohtml that doesn't work for newer PDF versions
[32223]	339	if ($output_type =~ m/^html/i) {
	340	#if (!$output_type \|\| ($output_type =~ m/^html/i)) {
[1445]	341	$success = &pdf_to_html($dirname, $input_filename, $output_filestem);
	342	if ($success) {
	343	return "html";
	344	}
	345	}
	346
[32273]	347	# Attempt conversion to TEXT (not for Windows, but PDFPlugin/PDFv1Plugin takes care of that
	348	if (!$output_type \|\| ($output_type =~ m/text/i)) {
	349	$success = &pdf_to_text($dirname, $input_filename, $output_filestem);
[32205]	350
[1445]	351	if ($success) {
	352	return "text";
	353	}
	354	}
[32273]	355	}
	356
[32277]	357	elsif ($pdf_tool eq "xpdftools" ) {
	358
	359	# default to pretty html output
[32273]	360	if (!$output_type) {
[32277]	361	$output_type = "pretty_html";
[32273]	362	}
	363
	364	# Attempt coversion to Image
	365	#if ($output_type =~ m/jp?g\|gif\|png/i) {
	366	# $success = &pdfps_to_img($dirname, $input_filename, $output_filestem, $output_type);
	367	# if ($success){
	368	# return "item";
	369	# }
	370	#}
	371
[32277]	372	# Attempt conversion to (paged) pretty HTML using the newer pdftohtml of Xpdftools.
	373	if ($output_type =~ m/pretty_html$/i) {
[32273]	374	$success = &xpdf_to_html($dirname, $input_filename, $output_filestem);
	375	if ($success) {
	376	return $output_type;
	377	}
	378	}
	379
	380	# Attempt conversion to TEXT
[32277]	381	# Proper paged_text processing not yet implemented with xpdf
	382	if ($output_type =~ m/text/i) {
	383	$success = &xpdf_to_text($dirname, $input_filename, $output_filestem, $output_type);
[32273]	384
	385	if ($success) {
	386	return "text";
	387	}
	388	}
[32277]	389	}
	390
[1445]	391	return "fail";
	392
	393	}
	394
	395
	396	# Convert an Adobe PostScript document
	397
	398	sub convertPS {
[22429]	399	my ($dirname,$input_filename, $output_filestem, $output_type) = @_;
[1445]	400
	401	my $success = 0;
[10534]	402	$output_type =~ s/.\-(.)/$1/i;
	403	# Attempt coversion to Image
[16435]	404	if ($output_type =~ m/jp?g\|gif\|png/i) {
[17329]	405	$success = &pdfps_to_img($dirname, $input_filename, $output_filestem, $output_type);
[10534]	406	if ($success){
	407	return "item";
	408	}
	409	}
[1445]	410
	411	# Attempt conversion to TEXT
[16435]	412	if (!$output_type \|\| ($output_type =~ m/text/i)) {
[1445]	413	$success = &ps_to_text($input_filename, $output_filestem);
	414	if ($success) {
	415	return "text";
	416	}
	417	}
	418	return "fail";
	419	}
	420
	421
[2977]	422	sub convertPPT {
	423	my ($input_filename, $output_filestem, $output_type) = @_;
[10357]	424	my $success = 0;
[2977]	425
[10282]	426	my $ppt_convert_type = "";
[22513]	427
[16435]	428	#if (!$output_type \|\| $windows_scripting \|\| ($output_type !~ m/html/i) \|\| ($output_type !~ m/text/i)){
	429	if ($windows_scripting && ($output_type !~ m/html/i) && ($output_type !~ m/text/i)){
	430	if ($output_type =~ m/gif/i) {
[10282]	431	$ppt_convert_type = "-g";
[16435]	432	} elsif ($output_type =~ m/jp?g/i){
[10282]	433	$ppt_convert_type = "-j";
[16435]	434	} elsif ($output_type =~ m/png/i){
[10282]	435	$ppt_convert_type = "-p";
	436	}
[27509]	437	my $vbScript = &FileUtils::filenameConcatenate($ENV{'GSDLHOME'}, "bin",
[10282]	438	$ENV{'GSDLOS'}, "pptextract");
[28355]	439	$vbScript = "CScript //Nologo \"".$vbScript.".vbs\"" if ($ENV{'GSDLOS'} =~ m/^windows$/i); # now we use the .vbs VBScript
	440	# $vbScript = "pptextract" if ($ENV{'GSDLOS'} =~ m/^windows$/i); # back when the pptextract.exe VB executable was used
[10282]	441
[22429]	442	my $cmd = "";
[10357]	443	if ($timeout) {$cmd = "ulimit -t $timeout;";}
[22429]	444	# if the converting directory already exists
[10282]	445	if (-d $output_filestem) {
[22429]	446	print STDERR "**The conversion directory already exists\n";
[10282]	447	return "item";
	448	} else {
[10521]	449	$cmd .= "$vbScript $ppt_convert_type \"$input_filename\" \"$output_filestem\"";
[10282]	450	$cmd .= " 2>\"$output_filestem.err\""
[16435]	451	if ($ENV{'GSDLOS'} !~ m/^windows$/i \|\| $is_winnt_2000);
[28355]	452
[10282]	453	if (system($cmd) !=0) {
	454	print STDERR "Powerpoint VB Scripting convert failed\n";
	455	} else {
	456	return "item";
	457	}
	458	}
[16435]	459	} elsif (!$output_type \|\| ($output_type =~ m/html/i)) {
[10282]	460	# Attempt conversion to HTML
[16435]	461	#if (!$output_type \|\| ($output_type =~ m/html/i)) {
[2977]	462	# formulate the command
[22429]	463	my $cmd = "";
[24362]	464	my $full_perl_path = &util::get_perl_exec();
[24124]	465	$cmd .= "\"$full_perl_path\" -S ppttohtml.pl ";
[2977]	466	$cmd .= " \"$input_filename\" \"$output_filestem.html\"";
	467	$cmd .= " 2>\"$output_filestem.err\""
[16435]	468	if ($ENV{'GSDLOS'} !~ m/^windows$/i \|\| $is_winnt_2000);
[10357]	469
[2977]	470	# execute the command
	471	$!=0;
	472	if (system($cmd)!=0)
	473	{
[2991]	474	print STDERR "Powerpoint 95/97 converter failed $!\n";
[2977]	475	} else {
	476	return "html";
	477	}
[10464]	478	}
[2977]	479
	480	$success = &any_to_text($input_filename, $output_filestem);
	481	if ($success) {
	482	return "text";
	483	}
[10464]	484
[2977]	485	return "fail";
	486	}
	487
	488
[2991]	489	sub convertXLS {
	490	my ($input_filename, $output_filestem, $output_type) = @_;
[2977]	491
[2991]	492	my $success = 0;
[2977]	493
[2991]	494	# Attempt conversion to HTML
[16435]	495	if (!$output_type \|\| ($output_type =~ m/html/i)) {
[2991]	496	# formulate the command
[22429]	497	my $cmd = "";
[24362]	498	my $full_perl_path = &util::get_perl_exec();
[24124]	499	$cmd .= "\"$full_perl_path\" -S xlstohtml.pl ";
[2991]	500	$cmd .= " \"$input_filename\" \"$output_filestem.html\"";
	501	$cmd .= " 2>\"$output_filestem.err\""
[16435]	502	if ($ENV{'GSDLOS'} !~ m/^windows$/i \|\| $is_winnt_2000);
[2991]	503
	504
	505	# execute the command
	506	$!=0;
	507	if (system($cmd)!=0)
	508	{
	509	print STDERR "Excel 95/97 converter failed $!\n";
	510	} else {
	511	return "html";
	512	}
	513	}
[2977]	514
[2991]	515	$success = &any_to_text($input_filename, $output_filestem);
	516	if ($success) {
	517	return "text";
	518	}
	519
	520	return "fail";
	521	}
	522
	523
	524
[1654]	525	# Find the real type of a .doc file
	526	#
[2012]	527	# We seem to have a lot of files with a .doc extension that are .rtf
[1654]	528	# files or Word 5 files. This function attempts to tell the difference.
	529	sub find_docfile_type {
[22429]	530	my ($input_filename) = @_;
[23473]	531
	532	if (($windows_scripting) && ($input_filename =~ m/\.docx$/)) {
	533	return "docx";
	534	}
	535
[1654]	536	open(CHK, "<$input_filename");
[1734]	537	binmode(CHK);
[1654]	538	my $line = "";
	539	my $first = 1;
	540
	541	while (<CHK>) {
	542
	543	$line = $_;
[1960]	544
[1654]	545	if ($first) {
	546	# check to see if this is an rtf file
[16435]	547	if ($line =~ m/^\{\\rtf/) {
[1654]	548	close(CHK);
	549	return "rtf";
	550	}
[2755]	551	$first = 0;
[1654]	552	}
	553
[1734]	554	# is this is a word 6/7/8 document?
[16435]	555	if ($line =~ m/Word\.Document\.([678])/) {
[1654]	556	close(CHK);
[23473]	557
[1734]	558	return "word$1";
[1654]	559	}
	560
	561	}
	562
	563	return "unknown";
	564	}
	565
	566
[1734]	567	# Specific type-to-type conversions
[1445]	568	#
	569	# Each of the following functions attempts to convert a document from
[2755]	570	# a specific format to another. If they succeed they return 1 and leave
[1445]	571	# the output document(s) in the appropriate place; if they fail they
	572	# return 0 and delete any working files.
	573
	574
	575	# Attempt to convert a word document to html with the wv program
	576	sub doc_to_html {
[22429]	577	my ($input_filename, $output_filestem) = @_;
[1445]	578
[24371]	579	my $wvware_status = 0;
[24375]	580
[24371]	581	# need to ensure that the path to perl is quoted (in case there's spaces in it)
[24513]	582	my $launch_cmd = "\"".&util::get_perl_exec()."\" -S wvware.pl \"$input_filename\" \"$output_filestem\" \"$faillogfile\" $verbosity $timeout";
[15120]	583
[30683]	584	# print STDERR "***** wvware launch cmd = $launch_cmd\n";
[15120]	585
[24371]	586	$wvware_status = system($launch_cmd)/256;
	587	return $wvware_status;
[1445]	588	}
	589
[10282]	590	# Attempt to convert a word document to html with the word2html scripting program
	591	sub native_doc_to_html {
[22429]	592	my ($input_filename, $output_filestem) = @_;
[1445]	593
[24166]	594	# build up the path to the doc-to-html conversion tool we're going to use
[27509]	595	my $vbScript = &FileUtils::filenameConcatenate($ENV{'GSDLHOME'}, "bin", $ENV{'GSDLOS'});
[10282]	596
[24164]	597	if ($ENV{'GSDLOS'} =~ m/^windows$/i) {
[24166]	598	# if windows scripting with docx input, use new VBscript to get the local Word install (if
	599	# any) to do the conversion, since docX can't be processed by word2html's windows_scripting
	600
	601	if($input_filename =~ m/docx$/i) { # need to use full path to docx2html script,
	602	# else script launch fails when there are error msgs
[27509]	603	$vbScript = &FileUtils::filenameConcatenate($vbScript, "docx2html.vbs");
[28355]	604	$vbScript = "CScript //Nologo \"$vbScript\""; # launch with CScript for error output in STDERR
[24169]	605	# //Nologo flag avoids Microsoft's opening/logo msgs
	606	print STDERR "About to use windows scripting to process docx file $input_filename.\n";
	607	print STDERR " This may take some time. Please wait...\n";
[24166]	608	}
	609	else { # old doc versions. use the usual VB executable word2html for the
	610	# conversion. Doesn't need full path, since bin\windows is on PATH
[27509]	611	$vbScript = "word2html"; #$vbScript = "\"".&FileUtils::filenameConcatenate($vbScript, "word2html")."\"";
[24166]	612	}
	613	}
	614	else { # not windows
[27509]	615	$vbScript = "\"".&FileUtils::filenameConcatenate($vbScript, "word2html")."\"";
[24164]	616	}
	617
[10445]	618	if (-e "$output_filestem.html") {
[22429]	619	print STDERR " The conversion file:\n";
	620	print STDERR " $output_filestem.html\n";
	621	print STDERR " ... already exists. Skipping\n";
[10445]	622	return 1;
	623	}
[10282]	624
	625	my $cmd = "";
	626	if ($timeout) {$cmd = "ulimit -t $timeout;";}
	627	#$cmd .= "$vbScript \"$input_filename\" \"$output_filestem.html\"";
[10445]	628	#$cmd .= "$vbScript $input_filename $output_filestem.html";
[10521]	629	$cmd .= "$vbScript \"$input_filename\" \"$output_filestem.html\"";
[10445]	630
[10282]	631	# redirecting STDERR
[24166]	632
	633	$cmd .= " 2> \"$output_filestem.err\""
	634	if ($ENV {'GSDLOS'} !~ m/^windows$/i \|\| $is_winnt_2000);
	635	#print STDERR "@@@@@@@@@ cmd=$cmd\n";
	636
[10282]	637	# execute the command
	638	$!=0;
	639	if (system($cmd)!=0)
	640	{
[24164]	641	print STDERR "Error executing $vbScript converter:$!\n";
[10282]	642	if (-s "$output_filestem.err") {
	643	open (ERRFILE, "<$output_filestem.err");
[24166]	644
[10282]	645	my $write_to_fail_log=0;
	646	if ($faillogfile ne "" && defined(open(FAILLOG,">>$faillogfile")))
	647	{$write_to_fail_log=1;}
	648
	649	my $line;
	650	while ($line=<ERRFILE>) {
[16435]	651	if ($line =~ m/\w/) {
[10282]	652	print STDERR "$line";
	653	print FAILLOG "$line" if ($write_to_fail_log);
	654	}
	655	if ($line !~ m/startup error/) {next;}
	656	print STDERR " (given an invalid .DOC file?)\n";
	657	print FAILLOG " (given an invalid .DOC file?)\n"
	658	if ($write_to_fail_log);
	659
	660	} # while ERRFILE
	661	close FAILLOG if ($write_to_fail_log);
	662	}
	663	return 0; # we can try any_to_text
	664	}
	665
	666	# Was the conversion successful?
	667	if (-s "$output_filestem.html") {
	668	open(TMP, "$output_filestem.html");
[22429]	669	my $line = <TMP>;
[10282]	670	close(TMP);
[22429]	671	if ($line && $line =~ m/html/i) {
[27509]	672	&FileUtils::removeFiles("$output_filestem.err") if -e "$output_filestem.err";
[10282]	673	return 1;
	674	}
	675	}
	676
	677	# If here, an error of some sort occurred
[27509]	678	&FileUtils::removeFiles("$output_filestem.html") if -e "$output_filestem.html";
[10282]	679	if (-e "$output_filestem.err") {
	680	if ($faillogfile ne "" && defined(open(FAILLOG,">>$faillogfile"))) {
	681	open (ERRLOG,"$output_filestem.err");
	682	while (<ERRLOG>) {print FAILLOG $_;}
	683	close FAILLOG;
	684	close ERRLOG;
	685	}
[27509]	686	&FileUtils::removeFiles("$output_filestem.err");
[10282]	687	}
	688	return 0;
	689	}
	690
[1654]	691	# Attempt to convert an RTF document to html with rtftohtml
	692	sub rtf_to_html {
[2241]	693	my ($input_filename, $output_filestem) = @_;
[1654]	694
	695	# formulate the command
[22429]	696	my $cmd = "";
[1692]	697	if ($timeout) {$cmd = "ulimit -t $timeout;";}
[2574]	698	$cmd .= "rtftohtml";
[10282]	699	#$cmd .= "rtf-converter";
[1654]	700
[3246]	701	$cmd .= " -o \"$output_filestem.html\" \"$input_filename\"";
[2574]	702
	703	$cmd .= " 2>\"$output_filestem.err\""
[16435]	704	if ($ENV{'GSDLOS'} !~ m/^windows$/i \|\| $is_winnt_2000);
[2574]	705
	706
[1654]	707	# execute the command
[2755]	708	$!=0;
[2060]	709	if (system($cmd)!=0)
[1654]	710	{
[2755]	711	print STDERR "Error executing rtf converter $!\n";
[2656]	712	# don't currently bother printing out error log...
	713	# keep going, in case it still created an HTML file...
[1654]	714	}
	715
	716	# Was the conversion successful?
[2755]	717	my $was_successful=0;
[2656]	718	if (-s "$output_filestem.html") {
[2755]	719	# make sure we have some content other than header
	720	open (HTML, "$output_filestem.html"); # what to do if fail?
	721	my $line;
	722	my $past_header=0;
	723	while ($line=<HTML>) {
	724
	725	if ($past_header == 0) {
[16435]	726	if ($line =~ m/<body>/) {$past_header=1;}
[2755]	727	next;
	728	}
	729
	730	$line =~ s/<[^>]+>//g;
[16435]	731	if ($line =~ m/\w/ && $past_header) { # we found some content...
[2755]	732	$was_successful=1;
	733	last;
	734	}
	735	}
	736	close HTML;
[1654]	737	}
[2574]	738
[2755]	739	if ($was_successful) {
[27509]	740	&FileUtils::removeFiles("$output_filestem.err")
[2755]	741	if (-e "$output_filestem.err");
	742	# insert the (modified) table of contents, if it exists.
	743	if (-e "${output_filestem}_ToC.html") {
[27509]	744	&FileUtils::moveFiles("$output_filestem.html","$output_filestem.src");
[2755]	745	my $open_failed=0;
	746	open HTMLSRC, "$output_filestem.src" \|\| ++$open_failed;
	747	open TOC, "${output_filestem}_ToC.html" \|\| ++$open_failed;
	748	open HTML, ">$output_filestem.html" \|\| ++$open_failed;
	749
	750	if ($open_failed) {
	751	close HTMLSRC;
	752	close TOC;
	753	close HTML;
[27509]	754	&FileUtils::moveFiles("$output_filestem.src","$output_filestem.html");
[2755]	755	return 1;
	756	}
	757
	758	# print out header info from src html.
[16435]	759	while (defined($_ = <HTMLSRC>) && $_ =~ m/\w/) {
[2755]	760	print HTML "$_";
	761	}
	762
	763	# print out table of contents, making links relative
	764	<TOC>; <TOC>; # ignore first 2 lines
	765	print HTML scalar(<TOC>); # line 3 = "<ol>\n"
	766	my $line;
	767	while ($line=<TOC>) {
[22429]	768	$line =~ s@</body></html>$@@i ; # only last line has this
[2755]	769	# make link relative
[22429]	770	$line =~ s@href=\"[^\#]+@href=\"@i;
[2755]	771	print HTML $line;
	772	}
	773	close TOC;
	774
	775	# rest of html src
	776	while (<HTMLSRC>) {
	777	print HTML $_;
	778	}
	779	close HTMLSRC;
	780	close HTML;
	781
[27509]	782	&FileUtils::removeFiles("${output_filestem}_ToC.html");
	783	&FileUtils::removeFiles("${output_filestem}.src");
[2755]	784	}
	785	# we don't yet do anything with footnotes ($output_filestem_fn.html) :(
	786	return 1; # success
	787	}
	788
	789	if (-e "$output_filestem.err") {
	790	if ($faillogfile ne "" && defined(open(FAILLOG,">>$faillogfile")))
	791	{
	792	print FAILLOG "Error - rtftohtml - couldn't extract text\n";
[10282]	793	#print FAILLOG "Error - rtf-converter - couldn't extract text\n";
[2755]	794	print FAILLOG " (rtf file might be too recent):\n";
	795	open (ERRLOG, "$output_filestem.err");
	796	while (<ERRLOG>) {print FAILLOG $_;}
	797	close ERRLOG;
	798	close FAILLOG;
	799	}
[27509]	800	&FileUtils::removeFiles("$output_filestem.err");
[2755]	801	}
	802
[27509]	803	&FileUtils::removeFiles("$output_filestem.html") if (-e "$output_filestem.html");
[2656]	804
[1654]	805	return 0;
	806	}
	807
	808
[32205]	809	# Convert a pdf file to html with the old pdftohtml command
	810	# which only works for older PDF versions
[1445]	811	sub pdf_to_html {
[2755]	812	my ($dirname, $input_filename, $output_filestem) = @_;
[1445]	813
[22429]	814	my $cmd = "";
[1692]	815	if ($timeout) {$cmd = "ulimit -t $timeout;";}
[24362]	816	my $full_perl_path = &util::get_perl_exec();
[24124]	817	$cmd .= "\"$full_perl_path\" -S pdftohtml.pl -zoom $pdf_zoom";
[3720]	818	$cmd .= " -c" if ($pdf_complex);
	819	$cmd .= " -i" if ($pdf_ignore_images);
[10451]	820	$cmd .= " -a" if ($pdf_allow_images_only);
[4103]	821	$cmd .= " -hidden" unless ($pdf_nohidden);
[1928]	822	$cmd .= " \"$input_filename\" \"$output_filestem\"";
[2755]	823
[16435]	824	if ($ENV{'GSDLOS'} !~ m/^windows$/i \|\| $is_winnt_2000) {
[2755]	825	$cmd .= " > \"$output_filestem.out\" 2> \"$output_filestem.err\"";
	826	} else {
	827	$cmd .= " > \"$output_filestem.err\"";
	828	}
	829
[2117]	830	$!=0;
[2241]	831
[2656]	832	my $retval=system($cmd);
	833	if ($retval!=0)
[1445]	834	{
[2755]	835	print STDERR "Error executing pdftohtml.pl";
[2117]	836	if ($!) {print STDERR ": $!";}
	837	print STDERR "\n";
[1445]	838	}
	839
[1692]	840	# make sure the converter made something
[2656]	841	if ($retval!=0 \|\| ! -s "$output_filestem.html")
[1692]	842	{
[27509]	843	&FileUtils::removeFiles("$output_filestem.out") if (-e "$output_filestem.out");
[2656]	844	# print out the converter's std err, if any
	845	if (-s "$output_filestem.err") {
[1692]	846	open (ERRLOG, "$output_filestem.err") \|\| die "$!";
[2755]	847	print STDERR "pdftohtml error log:\n";
[1692]	848	while (<ERRLOG>) {
	849	print STDERR "$_";
	850	}
	851	close ERRLOG;
	852	}
[24608]	853	#print STDERR "***********output filestem $output_filestem.html\n";
[27509]	854	&FileUtils::removeFiles("$output_filestem.html") if (-e "$output_filestem.html");
[2755]	855	if (-e "$output_filestem.err") {
	856	if ($faillogfile ne "" && defined(open(FAILLOG,">>$faillogfile")))
	857	{
	858	open (ERRLOG, "$output_filestem.err");
	859	while (<ERRLOG>) {print FAILLOG $_;}
	860	close ERRLOG;
	861	close FAILLOG;
	862	}
[27509]	863	&FileUtils::removeFiles("$output_filestem.err");
[2755]	864	}
[1692]	865	return 0;
	866	}
[10357]	867
[27509]	868	&FileUtils::removeFiles("$output_filestem.err") if (-e "$output_filestem.err");
	869	&FileUtils::removeFiles("$output_filestem.out") if (-e "$output_filestem.out");
[10357]	870	return 1;
	871	}
	872
[32205]	873
	874	# Convert a pdf file to html with the newer Xpdftools' pdftohtml
	875	# This generates "paged HTML" where extracted, selectable text is positioned
	876	# over screenshots of each page.
	877	# Since xpdf's pdftohtml fails if the output dir already exists and for easier
	878	# naming, the output files are created in a "pages" subdirectory of the tmp
	879	# location parent of $output_filestem instead
	880	sub xpdf_to_html {
	881	my ($dirname, $input_filename, $output_filestem) = @_;
	882
	883	my $cmd = "";
	884
	885	# build up the path to the doc-to-html conversion tool we're going to use
[32224]	886	my $xpdf_pdftohtml = &FileUtils::filenameConcatenate(_get_xpdftools_bindir(), "pdftohtml");
[32209]	887
[32205]	888	# We'll create the file by name $output_filestem during post-conversion processing.
	889	# Note that Xpdf tools will only create its conversion products in a dir that does
	890	# not yet exist. So we'll create this location as a subdir of the output_filestem's
	891	# parent directory. The parent dir is the already generated tmp area for conversion. So:
	892	# - tmpdir gs2build/tmp/<random-num> already exists at this stage
	893	# - We'll create gs2build/tmp/<rand>/output_filestem.html later, during post-processing
	894	# - For now, XPdftools will create gs2build/tmp/<rand>/pages and put its products in there.
	895	my ($tailname, $tmp_dirname, $suffix)
	896	= &File::Basename::fileparse($output_filestem, "\\.[^\\.]+\$");
	897	$tmp_dirname = &FileUtils::filenameConcatenate($tmp_dirname, "pages");
[32224]	898
[32205]	899	# xpdf's pdftohtml tool also takes a zoom factor, where a zoom of 1 is 100%
	900	$cmd .= "\"$xpdf_pdftohtml\"";
	901	$cmd .= " -z $pdf_zoom" if ($pdf_zoom);
	902	# $cmd .= " -c" if ($pdf_complex);
	903	# $cmd .= " -i" if ($pdf_ignore_images);
	904	# $cmd .= " -a" if ($pdf_allow_images_only);
	905	# $cmd .= " -hidden" unless ($pdf_nohidden);
	906	$cmd .= " \"$input_filename\" \"$tmp_dirname\"";
	907	#$cmd .= " \"$input_filename\" \"$output_filestem\"";
	908
	909	if ($ENV{'GSDLOS'} !~ m/^windows$/i \|\| $is_winnt_2000) {
	910	$cmd .= " > \"$output_filestem.out\" 2> \"$output_filestem.err\"";
	911	} else {
	912	$cmd .= " > \"$output_filestem.err\"";
	913	}
	914
	915	#print STDERR "@@@@ Running command: $cmd\n";
	916
	917	$!=0;
	918	my $retval=system($cmd);
	919	if ($retval!=0)
	920	{
	921	print STDERR "Error executing xpdf's pdftohtml tool";
	922	if ($!) {print STDERR ": $!";}
	923	print STDERR "\n";
	924	}
	925
	926	# make sure the converter made something
	927	if ($retval!=0 \|\| ! -s &FileUtils::filenameConcatenate($tmp_dirname,"index.html"))
	928	{
	929	&FileUtils::removeFiles("$output_filestem.out") if (-e "$output_filestem.out");
	930	# print out the converter's std err, if any
	931	if (-s "$output_filestem.err") {
	932	open (ERRLOG, "$output_filestem.err") \|\| die "$!";
	933	print STDERR "pdftohtml error log:\n";
	934	while (<ERRLOG>) {
	935	print STDERR "$_";
	936	}
	937	close ERRLOG;
	938	}
	939	#print STDERR "***********output filestem $output_filestem.html\n";
	940	&FileUtils::removeFiles("$tmp_dirname") if (-d "$tmp_dirname");
	941	if (-e "$output_filestem.err") {
	942	if ($faillogfile ne "" && defined(open(FAILLOG,">>$faillogfile")))
	943	{
	944	open (ERRLOG, "$output_filestem.err");
	945	while (<ERRLOG>) {print FAILLOG $_;}
	946	close ERRLOG;
	947	close FAILLOG;
	948	}
	949	&FileUtils::removeFiles("$output_filestem.err");
	950	}
	951	return 0;
	952	}
	953
	954	&FileUtils::removeFiles("$output_filestem.err") if (-e "$output_filestem.err");
	955	&FileUtils::removeFiles("$output_filestem.out") if (-e "$output_filestem.out");
	956	return 1;
	957	}
	958
[32224]	959	# Returns the path to xpdf-tools's containing bin dir appropriate for this machine's OS and bitness
	960	sub _get_xpdftools_bindir {
[32205]	961
[32263]	962	# build up the path to the containing bin dir of the xpdf conversion tool we're going to use
	963	my $xpdf_tools_bin = &FileUtils::filenameConcatenate($ENV{'GSDLHOME'}, "bin", $ENV{'GSDLOS'}, "xpdf-tools", "bin");
	964	return $xpdf_tools_bin;
[32224]	965	}
[32205]	966
[10357]	967	# Convert a pdf file to various types of image with the convert command
	968
[17329]	969	sub pdfps_to_img {
[10357]	970	my ($dirname, $input_filename, $output_filestem, $output_type) = @_;
[10401]	971
	972	# Check that ImageMagick is installed and available on the path (except for Windows 95/98)
	973	if (!($ENV{'GSDLOS'} eq "windows" && !Win32::IsWinNT())) {
[24600]	974	my $imagick_cmd = "\"".&util::get_perl_exec()."\" -S gs-magick.pl";
[24763]	975	$imagick_cmd = $imagick_cmd." --verbosity=$verbosity" if defined $verbosity;
[24600]	976	my $result = `$imagick_cmd identify 2>&1`;
	977
	978	# Linux and Windows return different values for "program not found".
	979	# Linux returns -1 and Windows 256 for "program not found". But once they're
	980	# converted to signed values, it will be -1 for Linux and 1 for Windows.
	981	# Whenever we test for return values other than 0, shift by 8 and perform
	982	# unsigned to signed status conversion on $? to get expected range of return vals
	983	# Although gs-magick.pl already shifts its $? by 8, converts it to a signed value
	984	# and then exits on that, by the time we get here, we need to do it again
	985	my $status = $?;
	986	$status >>= 8;
	987	$status = (($status & 0x80) ? -(0x100 - ($status & 0xFF)) : $status);
[25798]	988	if (($ENV{'GSDLOS'} ne "windows" && $status == -1) \|\| ($ENV{'GSDLOS'} eq "windows" && $status == 1)) {
	989	# if ($status == -1 \|\| $status == 1) #if ($status == -1 \|\| $status == 256) {
[10401]	990	#ImageMagick is not installed, thus the convert utility is not available.
[25798]	991	print STDERR "*** ImageMagick is not installed, the convert utility is not available. Unable to convert PDF/PS to images. Status: $status\n";
[10401]	992	return 0;
	993	}
	994	}
	995
[22429]	996	my $cmd = "";
[10357]	997	if ($timeout) {$cmd = "ulimit -t $timeout;";}
	998	$output_type =~ s/.\_(.)/$1/i;
[24362]	999	my $full_perl_path = &util::get_perl_exec();
[24124]	1000	$cmd .= "\"$full_perl_path\" -S pdfpstoimg.pl -convert_to $output_type \"$input_filename\" \"$output_filestem\"";
[16435]	1001	if ($ENV{'GSDLOS'} !~ m/^windows$/i \|\| $is_winnt_2000) {
[10357]	1002	$cmd .= " > \"$output_filestem.out\" 2> \"$output_filestem.err\"";
	1003	} else {
	1004	$cmd .= " > \"$output_filestem.err\"";
	1005	}
	1006
	1007	# don't include path on windows (to avoid having to play about
	1008	# with quoting when GSDLHOME might contain spaces) but assume
	1009	# that the PATH is set up correctly
	1010	$!=0;
	1011	my $retval=system($cmd);
	1012	if ($retval!=0)
	1013	{
[28166]	1014	print STDERR "Error executing pdfpstoimg.pl";
[10357]	1015	if ($!) {print STDERR ": $!";}
	1016	print STDERR "\n";
	1017	}
	1018
	1019	#make sure the converter made something
	1020	#if ($retval !=0) \|\| ! -s "$output_filestem")
	1021	if ($retval !=0)
	1022	{
[27509]	1023	&FileUtils::removeFiles("$output_filestem.out") if (-e "$output_filestem.out");
[10357]	1024	#print out the converter's std err, if any
	1025	if (-s "$output_filestem.err") {
	1026	open (ERRLOG, "$output_filestem.err") \|\| die "$!";
[17329]	1027	print STDERR "pdfpstoimg error log:\n";
[10357]	1028	while (<ERRLOG>) {
	1029	print STDERR "$_";
	1030	}
	1031	close ERRLOG;
	1032	}
[27509]	1033	#&FileUtils::removeFiles("$output_filestem.html") if (-e "$output_filestem.html");
[10357]	1034	if (-e "$output_filestem.err") {
	1035	if ($faillogfile ne "" && defined(open(FAILLOG,">>$faillogfile")))
	1036	{
	1037	open (ERRLOG, "$output_filestem.err");
	1038	while (<ERRLOG>) {print FAILLOG $_;}
	1039	close ERRLOG;
	1040	close FAILLOG;
	1041	}
[27509]	1042	&FileUtils::removeFiles("$output_filestem.err");
[10357]	1043	}
	1044	return 0;
	1045	}
[27509]	1046	&FileUtils::removeFiles("$output_filestem.err") if (-e "$output_filestem.err");
	1047	&FileUtils::removeFiles("$output_filestem.out") if (-e "$output_filestem.out");
[1445]	1048	return 1;
	1049	}
	1050
[32224]	1051	# Convert a PDF file to text with xpdftools' pdftotext command
	1052	# Works for Windows too, whereas the old pdftotxt didn't
	1053	sub xpdf_to_text {
[32277]	1054	my ($dirname, $input_filename, $output_filestem, $output_type) = @_;
[32224]	1055
	1056	my $cmd = "";
	1057
	1058	# build up the path to the doc-to-txt conversion tool we're going to use
	1059	my $xpdf_pdftotxt = &FileUtils::filenameConcatenate(_get_xpdftools_bindir(), "pdftotext");
	1060
	1061	# For xpdf's pdftotxt options, see https://www.xpdfreader.com/pdftotext-man.html
	1062	$cmd .= "\"$xpdf_pdftotxt\"";
	1063	if($enc) {
	1064	$cmd .= " -enc $enc"; # decode the bytes in the file using the designated encoding scheme
	1065	} else {
	1066	# as per https://www.xpdfreader.com/pdftotext-man.html
	1067	# xpdf's pdftotxt defaults to using Latin-1 encoding, should we default to UTF-8?
	1068	$cmd .= " -enc UTF-8"; # see https://www.xpdfreader.com/xpdfrc-man.html
	1069	}
[32277]	1070
	1071	if ($output_type ne "paged_text") { # output_type eq "text", don't bother about page break markers
[32224]	1072	$cmd .= " -nopgbrk";
[32277]	1073	}
[32224]	1074	# Avoid the silly solitary carriage returns (CR in Notepad) at the end
	1075	# of lines that ends up as \n appended to the doc title
	1076	# by setting the end of line marker to unix style solitary newline (LF or \n),
	1077	# which doesn't end up in the doc title
	1078	$cmd .= " -eol unix";
	1079	$cmd .= " \"$input_filename\" \"$output_filestem.text\"";
	1080
	1081	print STDERR "@@@@ Running command: $cmd\n";
	1082
	1083	return _run_pdf_to_text_cmd($cmd, $output_filestem);
	1084	}
	1085
[1445]	1086	# Convert a PDF file to text with the pdftotext command
	1087
	1088	sub pdf_to_text {
[2755]	1089	my ($dirname, $input_filename, $output_filestem) = @_;
[1445]	1090
[2248]	1091	my $cmd = "pdftotext \"$input_filename\" \"$output_filestem.text\"";
[32224]	1092
	1093	return _run_pdf_to_text_cmd($cmd, $output_filestem);
	1094	}
[2755]	1095
[32224]	1096	sub _run_pdf_to_text_cmd {
	1097	my ($cmd, $output_filestem) = @_;
	1098
[16435]	1099	if ($ENV{'GSDLOS'} !~ m/^windows$/i) {
[2755]	1100	$cmd .= " > \"$output_filestem.out\" 2> \"$output_filestem.err\"";
	1101	} else {
	1102	$cmd .= " > \"$output_filestem.err\"";
	1103	}
[1445]	1104
[2060]	1105	if (system($cmd)!=0)
[1445]	1106	{
	1107	print STDERR "Error executing $cmd: $!\n";
[27509]	1108	&FileUtils::removeFiles("$output_filestem.text") if (-e "$output_filestem.text");
[1445]	1109	}
	1110
[2755]	1111	# make sure there is some extracted text.
	1112	if (-e "$output_filestem.text") {
	1113	open (EXTR_TEXT, "$output_filestem.text") \|\| warn "open: $!";
	1114	binmode(EXTR_TEXT); # just in case...
	1115	my $line="";
	1116	my $seen_text=0;
	1117	while (($seen_text==0) && ($line=<EXTR_TEXT>)) {
[16435]	1118	if ($line=~ m/\w/) {$seen_text=1;}
[2755]	1119	}
	1120	close EXTR_TEXT;
	1121	if ($seen_text==0) { # no text was extracted
	1122	print STDERR "Error: pdftotext found no text\n";
[27509]	1123	&FileUtils::removeFiles("$output_filestem.text");
[2755]	1124	}
	1125	}
	1126
[1692]	1127	# make sure the converter made something
[2656]	1128	if (! -s "$output_filestem.text")
[1692]	1129	{
	1130	# print out the converters std err, if any
[2656]	1131	if (-s "$output_filestem.err") {
[1692]	1132	open (ERRLOG, "$output_filestem.err") \|\| die "$!";
[2755]	1133	print STDERR "pdftotext error log:\n";
[1692]	1134	while (<ERRLOG>) {
	1135	print STDERR "$_";
	1136	}
	1137	close ERRLOG;
	1138	}
[2656]	1139	# does this converter create a .out file?
[27509]	1140	&FileUtils::removeFiles("$output_filestem.out") if (-e "$output_filestem.out");
	1141	&FileUtils::removeFiles("$output_filestem.text") if (-e "$output_filestem.text");
[2755]	1142	if (-e "$output_filestem.err") {
	1143	if ($faillogfile ne "" && defined(open(FAILLOG,">>$faillogfile")))
	1144	{
	1145	open (ERRLOG,"$output_filestem.err");
	1146	while (<ERRLOG>) {print FAILLOG $_;}
	1147	close ERRLOG;
	1148	close FAILLOG;
	1149	}
[27509]	1150	&FileUtils::removeFiles("$output_filestem.err");
[2755]	1151	}
[1692]	1152	return 0;
	1153	}
[27509]	1154	&FileUtils::removeFiles("$output_filestem.err") if (-e "$output_filestem.err");
[1445]	1155	return 1;
	1156	}
	1157
[2012]	1158	# Convert a PostScript document to text
	1159	# note - just using "ps2ascii" isn't good enough, as it
	1160	# returns 0 for a postscript interpreter error. ps2ascii is just
	1161	# a wrapper to "gs" anyway, so we use that cmd here.
[1445]	1162
	1163	sub ps_to_text {
[2241]	1164	my ($input_filename, $output_filestem) = @_;
[1445]	1165
[2241]	1166	my $error = "";
	1167
	1168	# if we're on windows we'll fall straight through without attempting
	1169	# to use gs
[16435]	1170	if ($ENV{'GSDLOS'} =~ m/^windows$/i) {
[2241]	1171	$error = "Windows does not support gs";
	1172
	1173	} else {
[3538]	1174	my $cmd = "";
	1175	if ($timeout) {$cmd = "ulimit -t $timeout; ";}
	1176	$cmd .= "gs -q -dNODISPLAY -dNOBIND -dWRITESYSTEMDICT -dSIMPLE -c save ";
[2241]	1177	$cmd .= "-f ps2ascii.ps \"$input_filename\" -c quit > \"$output_filestem.text\"";
[10357]	1178	#$cmd .= "pstotext -output \"$output_filestem.text\" $input_filename\"";
[2241]	1179	$cmd .= " 2> $output_filestem.err";
	1180	$!=0;
[10357]	1181
[2241]	1182	my $retcode=system($cmd);
	1183	$retcode = $? >> 8; # see man perlfunc - system for this...
	1184	# if system returns -1 \| 127 (couldn't start program), look at $! for message
	1185
	1186	if ($retcode!=0) {if ($!) {$error=$!;} else {$error="couldn't run.\n";}}
	1187	elsif (! -e "$output_filestem.text") {
	1188	$error="did not create output file.\n";
[2012]	1189	}
[2241]	1190	else
	1191	{ # make sure the interpreter didn't get an error. It is technically
	1192	# possible for the actual text to start with this, but....
	1193	open PSOUT, "$output_filestem.text";
[16435]	1194	if (<PSOUT> =~ m/^Error: (.*)/) {
[2241]	1195	$error="interpreter error - \"$1\"";
	1196	}
	1197	close PSOUT;
	1198	}
[2012]	1199	}
[2241]	1200
[2012]	1201	if ($error ne "")
[1445]	1202	{
[2755]	1203	print STDERR "Warning: Error executing gs: $error\n";
[30724]	1204	print STDERR "Resorting to Perl regular expressions to extract text from PostScript...\n";
[27509]	1205	&FileUtils::removeFiles("$output_filestem.text") if (-e "$output_filestem.text");
[2755]	1206
	1207	if ("$faillogfile" ne "" && defined(open (FAILLOG, ">>$faillogfile")))
	1208	{
	1209	print FAILLOG "gs - $error\n";
	1210	if (-e "$output_filestem.err") {
	1211	open(ERRLOG, "$output_filestem.err");
	1212	while (<ERRLOG>) {print FAILLOG $_;}
	1213	close ERRLOG;
	1214	}
	1215	close FAILLOG;
	1216	}
[27509]	1217	&FileUtils::removeFiles("$output_filestem.err") if (-e "$output_filestem.err");
[2012]	1218
[2755]	1219
[2012]	1220	# Fine then. We'll just do a lousy job by ourselves...
[2031]	1221	# Based on 5-line regexp sed script found at:
[2012]	1222	# http://snark.ptc.spbu.ru/mail-archives/lout/brown/msg00003.html
	1223	#
[2755]	1224	print STDERR "Stripping text from postscript\n";
[2012]	1225	my $errorcode=0;
	1226	open (IN, "$input_filename")
	1227	\|\| ($errorcode=1, warn "Couldn't read file: $!");
	1228	open (OUT, ">$output_filestem.text")
	1229	\|\| ($errorcode=1, warn "Couldn't write file: $!");
	1230	if ($errorcode) {print STDERR "errors\n";return 0;}
	1231
[2031]	1232	my $text=""; # this is for whole .ps file...
[2755]	1233	$text = join('', <IN>); # see man perlport, under "System Resources"
[2031]	1234	close IN;
	1235
[2447]	1236	# Make sure this is a ps file...
[16435]	1237	if ($text !~ m/^%!/) {
[2755]	1238	print STDERR "Bad postscript header: not '%!'\n";
	1239	if ($faillogfile ne "" && defined(open(FAILLOG, ">>$faillogfile")))
	1240	{
	1241	print FAILLOG "Bad postscript header: not '%!'\n";
	1242	close FAILLOG;
	1243	}
[2447]	1244	return 0;
	1245	}
	1246
[2031]	1247	# if ps has Page data, then use it to delete all stuff before it.
	1248	$text =~ s/^.?%%Page:.?\n//s; # treat string as single line
	1249
	1250	# remove all leading non-data stuff
	1251	$text =~ s/^.*?\(//s;
	1252
	1253	# remove all newline chars for easier processing
	1254	$text =~ s/\n//g;
	1255
	1256	# Big assumption here - assume that if any co-ordinates are
	1257	# given, then we are at the end of a sentence.
	1258	$text =~ s/\)-?\d+\ -?\d+/\) \(\n\)/g;
	1259
	1260	# special characters--
	1261	$text =~ s/\(\\|\)/\(\ - \)/g; # j -> em-dash?
	1262
	1263	# ? ps text formatting (eg italics?) ?
	1264	$text =~ s/Fn\(f\)/\(\{\)/g; # f -> {
	1265	$text =~ s/Fn\(g\)/\(\}\)/g; # g -> }
	1266	$text =~ s/Fn\(j\)/\(\\|\)/g; # j -> \|
	1267	# default - remove the rest
	1268	$text =~ s/\ ?F.\((.+?)\)/\($1\)/g;
	1269
	1270	# attempt to add whitespace between words...
	1271	# this is based purely on observation, and may be completely wrong...
	1272	$text =~ s/([^F])[defghijkuy]\(/$1 \( /g;
	1273	# eg I notice "b(" is sometimes NOT a space if preceded by a
	1274	# negative number.
	1275	$text =~ s/\)\d+ ?b\(/\) \( /g;
	1276
	1277	# change quoted braces to brackets
	1278	$text =~ s/([^\\])\\\(/$1\{/g;
	1279	$text =~ s/([^\\])\\\)/$1\}/g ;
	1280
	1281	# remove everything that is not between braces
	1282	$text =~ s/\)([^\(\)])+?\(//sg ;
	1283
	1284	# remove any Trailer eof stuff.
	1285	$text =~ s/\)[^\)]*$//sg;
	1286
	1287	### ligatures have special characters...
	1288	$text =~ s/\\013/ff/g;
	1289	$text =~ s/\\014/fi/g;
	1290	$text =~ s/\\015/fl/g;
	1291	$text =~ s/\\016/ffi/g;
	1292	$text =~ s/\\214/fi/g;
	1293	$text =~ s/\\215/fl/g;
	1294	$text =~ s/\\017/\n\* /g; # asterisk?
	1295	$text =~ s/\\023/\023/g; # e acute ('e)
	1296	$text =~ s/\\177/\252/g; # u"
	1297	# $text =~ s/ ?? /\344/g; # a"
	1298
	1299	print OUT "$text";
	1300	close OUT;
[1960]	1301	}
[2600]	1302	# wrap the text - use a minimum length. ie, first space after this length.
	1303	my $wrap_length=72;
[27509]	1304	&FileUtils::moveFiles("$output_filestem.text", "$output_filestem.text.tmp");
[2600]	1305	open INFILE, "$output_filestem.text.tmp" \|\|
	1306	die "Couldn't open file: $!";
	1307	open OUTFILE, ">$output_filestem.text" \|\|
	1308	die "Couldn't open file for writing: $!";
	1309	my $line="";
	1310	while ($line=<INFILE>) {
	1311	while (length($line)>0) {
	1312	if (length($line)>$wrap_length) {
	1313	$line =~ s/^(.{$wrap_length}[^\s])\s//;
	1314	print OUTFILE "$1\n";
	1315	} else {
	1316	print OUTFILE "$line";
	1317	$line="";
	1318	}
	1319	}
	1320	}
	1321	close INFILE;
	1322	close OUTFILE;
[27509]	1323	&FileUtils::removeFiles("$output_filestem.text.tmp");
[2600]	1324
[27509]	1325	&FileUtils::removeFiles("$output_filestem.err") if (-e "$output_filestem.err");
[1445]	1326	return 1;
	1327	}
	1328
	1329
	1330	# Convert any file to HTML with a crude perl implementation of the
	1331	# UNIX strings command.
	1332
	1333	sub any_to_html {
[22429]	1334	my ($input_filename, $output_filestem) = @_;
[1445]	1335
	1336	# First generate a text file
	1337	return 0 unless (&any_to_text($input_filename, $output_filestem));
	1338
	1339	# create an HTML file from the text file
	1340	open(TEXT, "<$output_filestem.text");
	1341	open(HTML, ">$output_filestem.html");
	1342
[2241]	1343	print HTML "<html><head>\n";
	1344	print HTML "<META HTTP-EQUIV=\"Content-Type\" CONTENT=\"text/html\">\n";
	1345	print HTML "<META NAME=\"GENERATOR\" CONTENT=\"Greenstone any_to_html\">\n";
	1346	print HTML "</head><body>\n\n";
[1734]	1347
[2755]	1348	my $line;
	1349	while ($line=<TEXT>) {
	1350	$line =~ s/</</g;
	1351	$line =~ s/>/>/g;
[16435]	1352	if ($line =~ m/^\s*$/) {
[2755]	1353	print HTML "<p>";
	1354	} else {
	1355	print HTML "<br> ", $line;
	1356	}
[1445]	1357	}
[1734]	1358	print HTML "\n</body></html>\n";
[1445]	1359
[2241]	1360	close HTML;
	1361	close TEXT;
	1362
[27509]	1363	&FileUtils::removeFiles("$output_filestem.text") if (-e "$output_filestem.text");
[1445]	1364	return 1;
	1365	}
	1366
	1367	# Convert any file to TEXT with a crude perl implementation of the
	1368	# UNIX strings command.
[2755]	1369	# Note - this assumes ascii charsets :( (jrm21)
[1445]	1370
	1371	sub any_to_text {
[22429]	1372	my ($input_filename, $output_filestem) = @_;
[1445]	1373
[3350]	1374	if (!$use_strings) {
	1375	return 0;
	1376	}
[15120]	1377
	1378	print STDERR "\n** In any to text**\n\n";
[2755]	1379	open(IN, "<$input_filename") \|\| return 0;
[1734]	1380	binmode(IN);
[2755]	1381	open(OUT, ">$output_filestem.text") \|\| return 0;
[1445]	1382
	1383	my ($line);
[2755]	1384	my $output_line_count = 0;
[1445]	1385	while (<IN>) {
	1386	$line = $_;
[1734]	1387
[1445]	1388	# delete anything that isn't a printable character
	1389	$line =~ s/[^\040-\176]+/\n/sg;
	1390
	1391	# delete any string less than 10 characters long
[1734]	1392	$line =~ s/^.{0,9}$/\n/mg;
[16435]	1393	while ($line =~ m/^.{1,9}$/m) {
[1734]	1394	$line =~ s/^.{0,9}$/\n/mg;
[1445]	1395	$line =~ s/\n+/\n/sg;
	1396	}
	1397
	1398	# remove extraneous whitespace
	1399	$line =~ s/\n+/\n/gs;
	1400	$line =~ s/^\n//gs;
[1578]	1401
[1445]	1402	# output whatever is left
[16435]	1403	if ($line =~ m/[^\n ]/) {
[1445]	1404	print OUT $line;
[2755]	1405	++$output_line_count;
[1445]	1406	}
	1407	}
[2241]	1408
	1409	close OUT;
	1410	close IN;
	1411
[2755]	1412	if ($output_line_count) { # try to protect against binary only formats
	1413	return 1;
	1414	}
	1415
[27509]	1416	&FileUtils::removeFiles("$output_filestem.text");
[2755]	1417	return 0;
	1418
[1445]	1419	}

Note: See TracBrowser for help on using the repository browser.

Download in other formats: