root/gs2-extensions/pdf-box/trunk/java/perllib/plugins/PDFBoxConverter.pm @ 32748

Revision 32748, 14.1 KB (checked in by davidb, 8 months ago)

Change to help address an issue with the cached-folder name being generated

Line 
1###########################################################################
2#
3# PDFBoxConverter - helper plugin that does pdf document conversion with PDFBox
4#
5# A component of the Greenstone digital library software
6# from the New Zealand Digital Library Project at the
7# University of Waikato, New Zealand.
8#
9# Copyright (C) 2010 New Zealand Digital Library Project
10#
11# This program is free software; you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation; either version 2 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program; if not, write to the Free Software
23# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
24#
25###########################################################################
26package PDFBoxConverter;
27
28use BaseMediaConverter;
29
30use strict;
31no strict 'refs'; # allow filehandles to be variables and viceversa
32no strict 'subs'; # allow barewords (eg STDERR) as function arguments
33
34#use HTML::Entities; # for encoding characters into their HTML entities when PDFBox converts to text
35
36use gsprintf 'gsprintf';
37use FileUtils;
38
39# these two variables mustn't be initialised here or they will get stuck
40# at those values.
41our $pdfbox_conversion_available;
42our $no_pdfbox_conversion_reason;
43
44BEGIN {
45    @PDFBoxConverter::ISA = ('BaseMediaConverter');
46
47    # Check that PDFBox is installed and available on the path
48    $pdfbox_conversion_available = 1;
49    $no_pdfbox_conversion_reason = "";
50   
51    if (!defined $ENV{'GEXT_PDFBOX'}) {
52    $pdfbox_conversion_available = 0;
53    $no_pdfbox_conversion_reason = "gextpdfboxnotinstalled";
54    }
55    else {
56    my $gextpb_home = $ENV{'GEXT_PDFBOX'};
57    my $pbajar = &FileUtils::filenameConcatenate($gextpb_home,"lib","java","pdfbox-app.jar");
58
59    if (!-e $pbajar) {
60        &gsprintf(STDERR,"**** Failed to find $pbajar\n");
61        $pdfbox_conversion_available = 0;
62        $no_pdfbox_conversion_reason = "gextpdfboxjarnotinstalled";
63    }
64    else {
65        # test to see if java is in path
66        # Need to run java -version instead of just java, since the %ERRORLEVEL% returned
67        # for `java` (which is checked below for failure of the command) is 0 for JDK 1.6*
68        # while %ERRORLEVEL% is 1 for JDK 1.7*
69        # If `java -version` is run however, %ERRORLEVEL% returned is 0 if java is
70        # installed, regardless of whether the JDK version is 1.6* or 1.7*.
71        my $java = &util::get_java_command();
72           
73        my $cmd = "$java -version";
74        if ($ENV{'GSDLOS'} =~ /^windows/i) {
75        $cmd .= " >nul 2>&1"; # java 2>&1 >null or java >null 2>&1 both work (%ERRORLEVEL% is 0)
76        }
77        else {
78        # On Ubuntu, java >/dev/null 2>&1 works,
79        # but java 2>&1 >/dev/null doesn't work: output goes to screen anyway
80        $cmd .= " >/dev/null 2>&1"; # " >/dev/null 2>&1 &" - don't need & at end for Linux Centos anymore (Ubuntu was already fine without it)
81        }
82
83        my $status = system($cmd);
84
85        if ($status != 0) {
86           
87        my $error_message =  "**** Testing for java\n";
88        $error_message .= "Failed to run: $cmd\n";
89        $error_message .=  "Error variable: |$!| and status: $status\n";
90
91        &gsprintf(STDERR, "PDFBoxConverter: $error_message");
92
93        $pdfbox_conversion_available = 0;
94        $no_pdfbox_conversion_reason = "couldnotrunjava";
95        }
96    }
97    }
98
99}
100
101my $arguments = [ ];
102
103my $options = { 'name' => "PDFBoxConverter",
104        'desc' => "{PDFBoxConverter.desc}",
105        'abstract' => "yes",
106        'inherits' => "yes",
107        'args' => $arguments };
108
109sub new {
110    my ($class) = shift (@_);
111    my ($pluginlist,$inputargs,$hashArgOptLists,$auxilary) = @_;
112    push(@$pluginlist, $class);
113
114    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
115    push(@{$hashArgOptLists->{"OptList"}},$options);
116
117
118    my $self = new BaseMediaConverter($pluginlist, $inputargs,
119                      $hashArgOptLists, $auxilary);
120
121    if ($self->{'info_only'}) {
122    # don't worry about any options etc
123    return bless $self, $class;
124    }
125    if ($pdfbox_conversion_available) {
126    my $gextpb_home = $ENV{'GEXT_PDFBOX'};
127    my $pbajar = &FileUtils::filenameConcatenate($gextpb_home,"lib","java","pdfbox-app.jar");
128    my $pbjbigjar = &FileUtils::filenameConcatenate($gextpb_home,"lib","java","jbig2-imageio-3.0.1.jar");
129    # Not including the following JPEG2000 jar, as it is under commercial license:
130    # https://github.com/jai-imageio/jai-imageio-jpeg2000 leading to https://bintray.com/jai-imageio/maven/jai-imageio-jpeg2000# (Files tab)
131    # my $pbjp2jar = &FileUtils::filenameConcatenate($gextpb_home,"lib","java","jai-imageio-jpeg2000-1.3.0.jar"); # jpeg2000
132    my $java = &util::get_java_command();
133    $self->{'pdfbox_txt_launch_cmd'} = "$java -cp \"$pbajar\" org.apache.pdfbox.tools.ExtractText";
134    $self->{'pdfbox_html_launch_cmd'} = "$java -cp \"$pbajar\" -Dline.separator=\"<br />\" org.apache.pdfbox.tools.ExtractText";
135
136    # We use this next cmd to launch our new custom PDFBox class (PDFBoxToImagesAndText.java) to convert each PDF page into an image (gif, jpg, png)
137    # AND its extracted text. Or just each page's extracted text. An item file is still generated,
138    # but this time referring to txtfiles too, not just the images. Result: searchable paged output.
139    # Our new custom class PDFBoxToImagesAndText.java lives in the new build folder, so add that to the classpath for the launch cmd
140    my $pdfbox_build = &FileUtils::filenameConcatenate($gextpb_home,"build");
141    # put the pdfbox jar, the jbig2-imageio library (Apache Software License 2.0)
142    # and our build folder containing our custom PDFBox class on the classpath
143    my $classpath = &util::pathname_cat($pbajar, $pbjbigjar, $pdfbox_build);
144#   $self->{'pdfbox_img_launch_cmd'} = "java -cp \"$classpath\" org.apache.pdfbox.tools.PDFToImage"; # pdfbox 2.09 cmd for converting each PDF page to an image (jpg, png) 
145    $self->{'pdfbox_imgtxt_launch_cmd'} = "java -cp \"$classpath\" org.greenstone.pdfbox.PDFBoxToImagesAndText";
146    }
147    else {       
148    $self->{'no_pdfbox_conversion_reason'} = $no_pdfbox_conversion_reason;
149
150    my $outhandle = $self->{'outhandle'};
151    &gsprintf($outhandle, "PDFBoxConverter: {PDFBoxConverter.noconversionavailable} ({PDFBoxConverter.$no_pdfbox_conversion_reason})\n");
152    } 
153
154    $self->{'pdfbox_conversion_available'} = $pdfbox_conversion_available;
155   
156    return bless $self, $class;
157
158}
159
160sub init {
161    my $self = shift(@_);
162    my ($verbosity, $outhandle, $failhandle) = @_;
163
164    $self->{'pbtmp_file_paths'} = ();
165}
166
167sub deinit {
168    my $self = shift(@_);
169
170    $self->clean_up_temporary_files();
171}
172
173
174sub convert {
175    my $self = shift(@_);
176    my ($source_file_full_path, $target_file_type) = @_;
177
178    return 0 unless $pdfbox_conversion_available;
179    # check the filename
180    return 0 if ( !-f $source_file_full_path);
181
182    # Although PDFBoxConverter inherits from AutoLoadConverters and therefore
183    # doesn't go through gsConvert.pl, still set the -pdf_tool flag in convert_options
184    # in case in future PDFBoxConverter no longer inherits from AutoLoadConverters
185    # and ends up going through gsConvert.pl
186    $self->{'convert_options'} .= " -pdf_tool pdfbox";
187
188    my $img_output_mode = 0;
189   
190    my $convert_to = $self->{'convert_to'};
191    my $paged_txt_output_mode = ($convert_to =~ /(pagedimgtxt|paged_text)/) ? 1 : 0;
192   
193    # the following line is necessary to avoid 'uninitialised variable' error
194    # messages concerning the converted_to member variable when PDFPlugin's
195    # use_sections option is checked.
196    # PDFBox plugin now processes use_sections option, when working with v1.5.0
197    # of the PDFBox jar file (which embeds each page in special <div> tags).
198    if ($target_file_type eq "html") {
199    $self->{'converted_to'} = "HTML";
200    } elsif ($target_file_type eq "jpg" || $target_file_type eq "png") { # || $target_file_type eq "gif"
201    # GIF not supported by PDFBox at present, see https://pdfbox.apache.org/1.8/commandline.html#pdftoimage
202    $self->{'converted_to'} = $target_file_type;   
203    $img_output_mode = 1;
204    } else {
205    $self->{'converted_to'} = "text";
206    }
207
208    my $outhandle = $self->{'outhandle'};
209    my $verbosity = $self->{'verbosity'};
210
211    my $source_file_no_path = &File::Basename::basename($source_file_full_path);
212    # Determine the full name and path of the output file
213    my $target_file_path;
214    if ($self->{'enable_cache'}) {
215    $self->init_cache_for_file($source_file_full_path);
216    my $cache_dir = $self->{'cached_dir'};
217    my $file_root = $self->{'cached_file_root'};
218    #$file_root .= "_$convert_id" if ($convert_id ne "");
219
220    # append the output filetype suffix only for non-image output formats, since for
221    # images we can be outputting multiple image files per single PDF input file
222    #my $target_file = ($img_output_mode || $paged_txt_output_mode) ? "$file_root" : "$file_root.$target_file_type";
223    my $target_file = ($img_output_mode || $paged_txt_output_mode) ? "" : "$file_root.$target_file_type";
224   
225
226    $target_file_path = &FileUtils::filenameConcatenate($cache_dir,$target_file);
227    }
228    else {
229    # this is in gsdl/tmp. get a tmp filename in collection instead???
230    $target_file_path = &util::get_tmp_filename($target_file_type);
231
232    # for image files, remove the suffix, since we can have many output image files
233    # per input PDF (one img for each page of the PDF, for example)
234    if($img_output_mode || $paged_txt_output_mode) {
235        $target_file_path =~ s/\.[^.]*$//g;
236        if(!&FileUtils::directoryExists($target_file_path)) {       
237        mkdir($target_file_path);
238        }
239       
240        # once the item file for the imgs has been created, need to adjust target_file_path
241
242        # below, we'll store the dir just created to pbtmp_file_paths, so all imgs and the
243        # item file generated in it can be deleted in one go on clean_up
244    }
245   
246    push(@{$self->{'pbtmp_file_paths'}}, $target_file_path);
247    }
248
249    # Generate and run the convert command
250    my $convert_cmd = "";
251
252    # want the filename without extension, because any images
253    # are to be generated with the same filename as the PDF
254    my ($tailname, $dirname, $suffix) = &File::Basename::fileparse($source_file_full_path, "\\.[^\\.]+\$");
255
256    if($img_output_mode || $paged_txt_output_mode) { # converting each page to image and/or text
257    my $output_prefix = &FileUtils::filenameConcatenate($target_file_path, $tailname);
258
259    # Our custom class does renaming of the pages (simplified to just numbers) for PagedImagePlugin
260    #$convert_cmd = $paged_txt_output_mode ? $self->{'pdfbox_imgtxt_launch_cmd'} : $self->{'pdfbox_img_launch_cmd'};
261    $convert_cmd = $self->{'pdfbox_imgtxt_launch_cmd'};
262    $convert_cmd .= " -imagesOnly" unless($paged_txt_output_mode); # set to images only unless there's text too
263    if($img_output_mode) { # whether images-only or images-and-text mode
264        $convert_cmd .= " -imageType $target_file_type";
265        $convert_cmd .= " -dpi ". $self->{"dpi"} if defined $self->{"dpi"};
266    } else { # img_output_mode off, so paged txt only and no images
267        $convert_cmd .= " -textOnly";       
268    }
269    $convert_cmd .= " -outputPrefix \"$output_prefix\"";
270    $convert_cmd .= " \"$source_file_full_path\"";
271   
272    } else { # single stream of text or html
273   
274    if ($target_file_type eq "html") {
275        $convert_cmd = $self->{'pdfbox_html_launch_cmd'};
276        $convert_cmd .= " -html" if ($target_file_type eq "html");
277    } else {
278        $convert_cmd = $self->{'pdfbox_txt_launch_cmd'};
279    }
280    $convert_cmd .= " \"$source_file_full_path\" \"$target_file_path\"";
281    }
282
283    if ($verbosity>2) {
284    &gsprintf($outhandle,"Convert command: $convert_cmd\n");
285    }
286
287    my $print_info = { 'message_prefix' => "PDFBox Conversion",
288               'message' => "Converting $source_file_no_path to: $target_file_type" };
289    # $print_info->{'cache_mode'} = $cache_mode if ($cache_mode ne "");
290
291    my ($regenerated,$result,$had_error)
292    = $self->autorun_general_cmd($convert_cmd,$source_file_full_path, $target_file_path,$print_info);
293
294    if($img_output_mode || $paged_txt_output_mode) {
295    # now the images have been generated, generate the "$target_file_path/tailname.item"
296    # item file for them, which is also the target_file_path that needs to be returned
297    $target_file_path = &util::create_itemfile($target_file_path, $tailname, $target_file_type);
298    #print STDERR "**** item file: $target_file_path\n";
299    }
300    elsif ($self->{'converted_to'} eq "text") {
301    # ensure html entities are doubly escaped for pdfbox to text conversion: &amp; -> &amp;amp;
302    # conversion to html does it automatically, but conversion to text doesn't
303    # and this results in illegal characters in doc.xml
304
305    my $fulltext = &FileUtils::readUTF8File($target_file_path);
306    if(defined $fulltext) {
307        #$fulltext = &HTML::Entities::encode($fulltext); # doesn't seem to help
308        $fulltext =~ s@&@&amp;@sg; # Kathy's fix to ensure doc contents don't break XML
309        &FileUtils::writeUTF8File($target_file_path, \$fulltext);
310    } else {
311        print STDERR "PDFBoxConverter::convert(): Unable to read from converted file\n";
312        $had_error = 1;
313    }
314    }
315
316    if ($had_error) {
317    return (0, $result,$target_file_path);
318    }
319    return (1, $result,$target_file_path);
320}
321
322sub convert_without_result {
323    my $self = shift(@_);
324
325    my $source_file_path = shift(@_);
326    my $target_file_type = shift(@_);
327    my $convert_options  = shift(@_) || "";
328    my $convert_id       = shift(@_) || "";
329
330    return $self->convert($source_file_path,$target_file_type,
331              $convert_options,$convert_id,"without_result");
332}
333
334sub clean_up_temporary_files {
335    my $self = shift(@_);
336
337    foreach my $pbtmp_file_path (@{$self->{'pbtmp_file_paths'}}) {
338    if (-d $pbtmp_file_path) {
339        #print STDERR "@@@@@@ cleanup called on $pbtmp_file_path\n";
340        &FileUtils::removeFilesRecursive($pbtmp_file_path);
341    }
342    elsif (-e $pbtmp_file_path) {
343        &FileUtils::removeFiles($pbtmp_file_path);
344    }
345    }
346
347    $self->{'pbtmp_file_paths'} = ();
348}
349
350
3511;
Note: See TracBrowser for help on using the browser.