root/gs2-extensions/pdf-box/trunk/java/perllib/plugins/PDFBoxConverter.pm @ 32282

Revision 32282, 13.1 KB (checked in by ak19, 15 months ago)

This was meant to be oart of commit 32278, where I forgot to commit the updated PDFBoxConvert.pm. The commit message for 32278 was: Our custom pdf-box class PDFToImagesAndText.java now takes two additional flags, textOnly and imagesOnly, which can be used to support paged_text and the original pagedimg_ output formats, besides pagedimgtxt_

Line 
1###########################################################################
2#
3# PDFBoxConverter - helper plugin that does pdf document conversion with PDFBox
4#
5# A component of the Greenstone digital library software
6# from the New Zealand Digital Library Project at the
7# University of Waikato, New Zealand.
8#
9# Copyright (C) 2010 New Zealand Digital Library Project
10#
11# This program is free software; you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation; either version 2 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program; if not, write to the Free Software
23# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
24#
25###########################################################################
26package PDFBoxConverter;
27
28use BaseMediaConverter;
29
30use strict;
31no strict 'refs'; # allow filehandles to be variables and viceversa
32no strict 'subs'; # allow barewords (eg STDERR) as function arguments
33
34#use HTML::Entities; # for encoding characters into their HTML entities when PDFBox converts to text
35
36use gsprintf 'gsprintf';
37use FileUtils;
38
39# these two variables mustn't be initialised here or they will get stuck
40# at those values.
41our $pdfbox_conversion_available;
42our $no_pdfbox_conversion_reason;
43
44BEGIN {
45    @PDFBoxConverter::ISA = ('BaseMediaConverter');
46
47    # Check that PDFBox is installed and available on the path
48    $pdfbox_conversion_available = 1;
49    $no_pdfbox_conversion_reason = "";
50   
51    if (!defined $ENV{'GEXT_PDFBOX'}) {
52    $pdfbox_conversion_available = 0;
53    $no_pdfbox_conversion_reason = "gextpdfboxnotinstalled";
54    }
55    else {
56    my $gextpb_home = $ENV{'GEXT_PDFBOX'};
57    my $pbajar = &FileUtils::filenameConcatenate($gextpb_home,"lib","java","pdfbox-app.jar");
58
59    if (!-e $pbajar) {
60        &gsprintf(STDERR,"**** Failed to find $pbajar\n");
61        $pdfbox_conversion_available = 0;
62        $no_pdfbox_conversion_reason = "gextpdfboxjarnotinstalled";
63    }
64    else {
65        # test to see if java is in path
66        # Need to run java -version instead of just java, since the %ERRORLEVEL% returned
67        # for `java` (which is checked below for failure of the command) is 0 for JDK 1.6*
68        # while %ERRORLEVEL% is 1 for JDK 1.7*
69        # If `java -version` is run however, %ERRORLEVEL% returned is 0 if java is
70        # installed, regardless of whether the JDK version is 1.6* or 1.7*.
71        my $java = &util::get_java_command();
72           
73        my $cmd = "$java -version";
74        if ($ENV{'GSDLOS'} =~ /^windows/i) {
75        $cmd .= " >nul 2>&1"; # java 2>&1 >null or java >null 2>&1 both work (%ERRORLEVEL% is 0)
76        }
77        else {
78        # On Ubuntu, java >/dev/null 2>&1 works,
79        # but java 2>&1 >/dev/null doesn't work: output goes to screen anyway
80        $cmd .= " >/dev/null 2>&1"; # " >/dev/null 2>&1 &" - don't need & at end for Linux Centos anymore (Ubuntu was already fine without it)
81        }
82
83        my $status = system($cmd);
84
85        if ($status != 0) {
86           
87        my $error_message =  "**** Testing for java\n";
88        $error_message .= "Failed to run: $cmd\n";
89        $error_message .=  "Error variable: |$!| and status: $status\n";
90
91        &gsprintf(STDERR, "PDFBoxConverter: $error_message");
92
93        $pdfbox_conversion_available = 0;
94        $no_pdfbox_conversion_reason = "couldnotrunjava";
95        }
96    }
97    }
98
99}
100
101my $arguments = [ ];
102
103my $options = { 'name' => "PDFBoxConverter",
104        'desc' => "{PDFBoxConverter.desc}",
105        'abstract' => "yes",
106        'inherits' => "yes",
107        'args' => $arguments };
108
109sub new {
110    my ($class) = shift (@_);
111    my ($pluginlist,$inputargs,$hashArgOptLists,$auxilary) = @_;
112    push(@$pluginlist, $class);
113
114    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
115    push(@{$hashArgOptLists->{"OptList"}},$options);
116
117
118    my $self = new BaseMediaConverter($pluginlist, $inputargs,
119                      $hashArgOptLists, $auxilary);
120
121    if ($self->{'info_only'}) {
122    # don't worry about any options etc
123    return bless $self, $class;
124    }
125    if ($pdfbox_conversion_available) {
126    my $gextpb_home = $ENV{'GEXT_PDFBOX'};
127    my $pbajar = &FileUtils::filenameConcatenate($gextpb_home,"lib","java","pdfbox-app.jar");
128    my $java = &util::get_java_command();
129    $self->{'pdfbox_txt_launch_cmd'} = "$java -cp \"$pbajar\" org.apache.pdfbox.tools.ExtractText";
130    $self->{'pdfbox_html_launch_cmd'} = "$java -cp \"$pbajar\" -Dline.separator=\"<br />\" org.apache.pdfbox.tools.ExtractText";
131#   $self->{'pdfbox_img_launch_cmd'} = "java -cp \"$pbajar\" org.apache.pdfbox.tools.PDFToImage"; # pdfbox 2.09 cmd for converting each PDF page to an image (gif, jpg, png)
132
133    # We use this next cmd to launch our new custom PDFBox class (PDFBoxToImagesAndText.java) to convert each PDF page into an image (gif, jpg, png)
134    # AND its extracted text. Or just each page's extracted text. An item file is still generated,
135    # but this time referring to txtfiles too, not just the images. Result: searchable paged output.
136    # Our new custom class PDFBoxToImagesAndText.java lives in the new build folder, so add that to the classpath for the launch cmd
137    my $pdfbox_build = &FileUtils::filenameConcatenate($gextpb_home,"build");
138    my $classpath = &util::pathname_cat($pbajar,$pdfbox_build);
139    $self->{'pdfbox_imgtxt_launch_cmd'} = "java -cp \"$classpath\" org.greenstone.pdfbox.PDFBoxToImagesAndText";
140    }
141    else {       
142    $self->{'no_pdfbox_conversion_reason'} = $no_pdfbox_conversion_reason;
143
144    my $outhandle = $self->{'outhandle'};
145    &gsprintf($outhandle, "PDFBoxConverter: {PDFBoxConverter.noconversionavailable} ({PDFBoxConverter.$no_pdfbox_conversion_reason})\n");
146    } 
147
148    $self->{'pdfbox_conversion_available'} = $pdfbox_conversion_available;
149   
150    return bless $self, $class;
151
152}
153
154sub init {
155    my $self = shift(@_);
156    my ($verbosity, $outhandle, $failhandle) = @_;
157
158    $self->{'pbtmp_file_paths'} = ();
159}
160
161sub deinit {
162    my $self = shift(@_);
163
164    $self->clean_up_temporary_files();
165}
166
167
168sub convert {
169    my $self = shift(@_);
170    my ($source_file_full_path, $target_file_type) = @_;
171
172    return 0 unless $pdfbox_conversion_available;
173    # check the filename
174    return 0 if ( !-f $source_file_full_path);
175
176    # Although PDFBoxConverter inherits from AutoLoadConverters and therefore
177    # doesn't go through gsConvert.pl, still set the -pdf_tool flag in convert_options
178    # in case in future PDFBoxConverter no longer inherits from AutoLoadConverters
179    # and ends up going through gsConvert.pl
180    $self->{'convert_options'} .= " -pdf_tool pdfbox";
181   
182    my $img_output_mode = 0;
183   
184    my $convert_to = $self->{'convert_to'};
185    my $paged_txt_output_mode = ($convert_to =~ /(pagedimgtxt|paged_text)/) ? 1 : 0;
186   
187    # the following line is necessary to avoid 'uninitialised variable' error
188    # messages concerning the converted_to member variable when PDFPlugin's
189    # use_sections option is checked.
190    # PDFBox plugin now processes use_sections option, when working with v1.5.0
191    # of the PDFBox jar file (which embeds each page in special <div> tags).
192    if ($target_file_type eq "html") {
193    $self->{'converted_to'} = "HTML";
194    } elsif ($target_file_type eq "jpg" || $target_file_type eq "gif" || $target_file_type eq "png") {
195    # GIF not supported by PDFBox at present, see https://pdfbox.apache.org/1.8/commandline.html#pdftoimage
196    $self->{'converted_to'} = $target_file_type;   
197    $img_output_mode = 1;
198    } else {
199    $self->{'converted_to'} = "text";
200    }
201
202    my $outhandle = $self->{'outhandle'};
203    my $verbosity = $self->{'verbosity'};
204
205    my $source_file_no_path = &File::Basename::basename($source_file_full_path);
206    # Determine the full name and path of the output file
207    my $target_file_path;
208    if ($self->{'enable_cache'}) {
209    $self->init_cache_for_file($source_file_full_path);
210    my $cache_dir = $self->{'cached_dir'};
211    my $file_root = $self->{'cached_file_root'};
212    #$file_root .= "_$convert_id" if ($convert_id ne "");
213
214    # append the output filetype suffix only for non-image output formats, since for
215    # images we can be outputting multiple image files per single PDF input file
216    my $target_file = ($img_output_mode || $paged_txt_output_mode) ? "$file_root" : "$file_root.$target_file_type";
217
218    $target_file_path = &FileUtils::filenameConcatenate($cache_dir,$target_file);
219    }
220    else {
221    # this is in gsdl/tmp. get a tmp filename in collection instead???
222    $target_file_path = &util::get_tmp_filename($target_file_type);
223
224    # for image files, remove the suffix, since we can have many output image files
225    # per input PDF (one img for each page of the PDF, for example)
226    if($img_output_mode || $paged_txt_output_mode) {
227        $target_file_path =~ s/\.[^.]*$//g;
228        if(!&FileUtils::directoryExists($target_file_path)) {       
229        mkdir($target_file_path);
230        }
231       
232        # once the item file for the imgs has been created, need to adjust target_file_path
233
234        # below, we'll store the dir just created to pbtmp_file_paths, so all imgs and the
235        # item file generated in it can be deleted in one go on clean_up
236    }
237   
238    push(@{$self->{'pbtmp_file_paths'}}, $target_file_path);
239    }
240
241    # Generate and run the convert command
242    my $convert_cmd = "";
243
244    # want the filename without extension, because any images
245    # are to be generated with the same filename as the PDF
246    my ($tailname, $dirname, $suffix) = &File::Basename::fileparse($source_file_full_path, "\\.[^\\.]+\$");
247
248    if($img_output_mode || $paged_txt_output_mode) { # converting each page to image and/or text
249    my $output_prefix = &FileUtils::filenameConcatenate($target_file_path, $tailname);
250   
251    #$convert_cmd = $paged_txt_output_mode ? $self->{'pdfbox_imgtxt_launch_cmd'} : $self->{'pdfbox_img_launch_cmd'};
252    $convert_cmd = $self->{'pdfbox_imgtxt_launch_cmd'};
253    $convert_cmd .= " -textOnly" unless($img_output_mode); # if paged txt only and no images
254    $convert_cmd .= " -imagesOnly" unless($paged_txt_output_mode); # set to images only unless there's text too
255    $convert_cmd .= " -imageType $target_file_type" if($img_output_mode);
256    $convert_cmd .= " -outputPrefix \"$output_prefix\"";
257    $convert_cmd .= " \"$source_file_full_path\"";
258   
259    } else { # single stream of text or html
260   
261    if ($target_file_type eq "html") {
262        $convert_cmd = $self->{'pdfbox_html_launch_cmd'};
263        $convert_cmd .= " -html" if ($target_file_type eq "html");
264    } else {
265        $convert_cmd = $self->{'pdfbox_txt_launch_cmd'};
266    }
267    $convert_cmd .= " \"$source_file_full_path\" \"$target_file_path\"";
268    }
269
270    if ($verbosity>2) {
271    &gsprintf($outhandle,"Convert command: $convert_cmd\n");
272    }
273
274    my $print_info = { 'message_prefix' => "PDFBox Conversion",
275               'message' => "Converting $source_file_no_path to: $target_file_type" };
276    # $print_info->{'cache_mode'} = $cache_mode if ($cache_mode ne "");
277
278    my ($regenerated,$result,$had_error)
279    = $self->autorun_general_cmd($convert_cmd,$source_file_full_path, $target_file_path,$print_info);
280
281    if($img_output_mode || $paged_txt_output_mode) {
282    # now the images have been generated, generate the "$target_file_path/tailname.item"
283    # item file for them, which is also the target_file_path that needs to be returned
284    $target_file_path = &util::create_itemfile($target_file_path, $tailname, $target_file_type);
285    #print STDERR "**** item file: $target_file_path\n";
286    }
287    elsif ($self->{'converted_to'} eq "text") {
288    # ensure html entities are doubly escaped for pdfbox to text conversion: &amp; -> &amp;amp;
289    # conversion to html does it automatically, but conversion to text doesn't
290    # and this results in illegal characters in doc.xml
291
292    my $fulltext = &FileUtils::readUTF8File($target_file_path);
293    if(defined $fulltext) {
294        #$fulltext = &HTML::Entities::encode($fulltext); # doesn't seem to help
295        $fulltext =~ s@&@&amp;@sg; # Kathy's fix to ensure doc contents don't break XML
296        &FileUtils::writeUTF8File($target_file_path, \$fulltext);
297    } else {
298        print STDERR "PDFBoxConverter::convert(): Unable to read from converted file\n";
299        $had_error = 1;
300    }
301    }
302
303    if ($had_error) {
304    return (0, $result,$target_file_path);
305    }
306    return (1, $result,$target_file_path);
307}
308
309sub convert_without_result {
310    my $self = shift(@_);
311
312    my $source_file_path = shift(@_);
313    my $target_file_type = shift(@_);
314    my $convert_options  = shift(@_) || "";
315    my $convert_id       = shift(@_) || "";
316
317    return $self->convert($source_file_path,$target_file_type,
318              $convert_options,$convert_id,"without_result");
319}
320
321sub clean_up_temporary_files {
322    my $self = shift(@_);
323
324    foreach my $pbtmp_file_path (@{$self->{'pbtmp_file_paths'}}) {
325    if (-d $pbtmp_file_path) {
326        #print STDERR "@@@@@@ cleanup called on $pbtmp_file_path\n";
327        &FileUtils::removeFilesRecursive($pbtmp_file_path);
328    }
329    elsif (-e $pbtmp_file_path) {
330        &FileUtils::removeFiles($pbtmp_file_path);
331    }
332    }
333
334    $self->{'pbtmp_file_paths'} = ();
335}
336
337
3381;
Note: See TracBrowser for help on using the browser.