Changeset 32224

Timestamp:

2018-06-27T18:30:21+12:00 (6 years ago)

Author:

ak19

Message:

Adding PDF to text support for Windows using Xpdf's pdftotext tool. Previously PDFPlugin would override the output mode to HTML on Windows if txt output mode was selected.

Location:

main/trunk/greenstone2

Files:

• bin/script/gsConvert.pl (modified) (7 diffs)
• perllib/plugins/PDFPlugin.pm (modified) (1 diff)

main/trunk/greenstone2/bin/script/gsConvert.pl

@@ -67 +67 @@
 my $pdf_allow_images_only;
 my $windows_scripting;
+my $enc;
 
 sub print_usage
@@ -345 +346 @@
     # Attempt conversion to TEXT
     if (!$output_type || ($output_type =~ m/text/i)) {
-    $success = &pdf_to_text($dirname, $input_filename, $output_filestem);
+        if ($ENV{'GSDLOS'} =~ m/^windows$/i) { # we now have pdf to text support for windows by using xpdf tools
+            $success = &xpdf_to_text($dirname, $input_filename, $output_filestem);
+        } else {
+            $success = &pdf_to_text($dirname, $input_filename, $output_filestem);
+        }
     if ($success) {
         return "text";
@@ -846 +851 @@
 
     # build up the path to the doc-to-html conversion tool we're going to use
-    my $xpdf_pdftohtml = &FileUtils::filenameConcatenate($ENV{'GSDLHOME'}, "bin", $ENV{'GSDLOS'}, "xpdf-tools");
-    
-    if($ENV{'GSDLOS'} =~ m/^windows$/i) { # For Windows, just use the 32 bit xpdf's pdftohtml as it works the same as the 64 bit
-    $xpdf_pdftohtml = &FileUtils::filenameConcatenate($xpdf_pdftohtml, "bin32");
-    } else { # unix (linux|darwin), use the bin32/bin64 folder depending on the BITNESS env var
-    
-    # Don't use $ENV{'GSDLARCH'}, use the new $ENV{'BITNESS'}, since
-    # $ENV{'GSDLARCH'} is only (meant to be) set when many other 32-bit or 64-bit
-    # specific subdirectories exist in a greenstone installation.
-    # None of those locations need exist when xpdf-tools is installed with GS.
-    # So don't depend on GSDLARCH as forcing that to be exported has side-effects
-    if($ENV{'BITNESS'}) {
-        $xpdf_pdftohtml = &FileUtils::filenameConcatenate($xpdf_pdftohtml, "bin".$ENV{'BITNESS'});
-    } else { # what if $ENV{'BITNESS'} undefined, fallback on bin32? or 64?
-        $xpdf_pdftohtml = &FileUtils::filenameConcatenate($xpdf_pdftohtml, "bin32");
-    }
-    }
+    my $xpdf_pdftohtml = &FileUtils::filenameConcatenate(_get_xpdftools_bindir(), "pdftohtml");
     
     # We'll create the file by name $output_filestem during post-conversion processing.
@@ -874 +863 @@
     = &File::Basename::fileparse($output_filestem, "\\.[^\\.]+\$");
     $tmp_dirname = &FileUtils::filenameConcatenate($tmp_dirname, "pages");
-
-    $xpdf_pdftohtml = &FileUtils::filenameConcatenate($xpdf_pdftohtml, "pdftohtml");
+    
     # xpdf's pdftohtml tool also takes a zoom factor, where a zoom of 1 is 100%
     $cmd .= "\"$xpdf_pdftohtml\"";
@@ -936 +924 @@
 }
 
-
+# Returns the path to xpdf-tools's containing bin dir appropriate for this machine's OS and bitness
+sub _get_xpdftools_bindir {
+
+    # build up the path to the containing bin dir of the xpdf conversion tool we're going to use
+    my $xpdf_tools_bin = &FileUtils::filenameConcatenate($ENV{'GSDLHOME'}, "bin", $ENV{'GSDLOS'}, "xpdf-tools");
+    
+    if($ENV{'GSDLOS'} =~ m/^windows$/i) { # For Windows, just use the 32 bit xpdf's pdftohtml as it works the same as the 64 bit
+        $xpdf_tools_bin = &FileUtils::filenameConcatenate($xpdf_tools_bin, "bin32");
+    } else { # unix (linux|darwin), use the bin32/bin64 folder depending on the BITNESS env var
+        
+        # Don't use $ENV{'GSDLARCH'}, use the new $ENV{'BITNESS'}, since $ENV{'GSDLARCH'}
+        # isn't always set and has side-effects when it is set:
+        # $ENV{'GSDLARCH'} is only (meant to be) set when many other 32-bit or 64-bit
+        # specific subdirectories exist in a greenstone installation.
+        # None of those locations need exist when xpdf-tools is installed with GS.
+        # So don't depend on GSDLARCH as forcing that to be exported has side-effects
+        if($ENV{'BITNESS'}) {
+            $xpdf_tools_bin = &FileUtils::filenameConcatenate($xpdf_tools_bin, "bin".$ENV{'BITNESS'});
+        } else { # what if $ENV{'BITNESS'} undefined, fallback on bin32? or 64?
+            $xpdf_tools_bin = &FileUtils::filenameConcatenate($xpdf_tools_bin, "bin32");
+        }
+    }
+    
+    return $xpdf_tools_bin;
+}
 
 # Convert a pdf file to various types of image with the convert command
@@ -1022 +1034 @@
 }
 
+# Convert a PDF file to text with xpdftools' pdftotext command
+# Works for Windows too, whereas the old pdftotxt didn't
+sub xpdf_to_text {
+    my ($dirname, $input_filename, $output_filestem) = @_;
+
+    my $cmd = "";
+
+    # build up the path to the doc-to-txt conversion tool we're going to use
+    my $xpdf_pdftotxt = &FileUtils::filenameConcatenate(_get_xpdftools_bindir(), "pdftotext");
+    
+    # For xpdf's pdftotxt options, see https://www.xpdfreader.com/pdftotext-man.html
+    $cmd .= "\"$xpdf_pdftotxt\"";
+    if($enc) {
+        $cmd .= " -enc $enc"; # decode the bytes in the file using the designated encoding scheme
+    } else {
+        # as per https://www.xpdfreader.com/pdftotext-man.html
+        # xpdf's pdftotxt defaults to using Latin-1 encoding, should we default to UTF-8?
+        $cmd .= " -enc UTF-8"; # see https://www.xpdfreader.com/xpdfrc-man.html
+    }
+    $cmd .= " -nopgbrk";
+    # Avoid the silly solitary carriage returns (CR in Notepad) at the end
+    # of lines that ends up as \n appended to the doc title
+    # by setting the end of line marker to unix style solitary newline (LF or \n),
+    # which doesn't end up in the doc title
+    $cmd .= " -eol unix";
+    $cmd .= " \"$input_filename\" \"$output_filestem.text\"";    
+
+    print STDERR "@@@@ Running command: $cmd\n";
+    
+    return _run_pdf_to_text_cmd($cmd, $output_filestem);
+}
+
 # Convert a PDF file to text with the pdftotext command
 
@@ -1028 +1072 @@
 
     my $cmd = "pdftotext \"$input_filename\" \"$output_filestem.text\"";
+    
+    return _run_pdf_to_text_cmd($cmd, $output_filestem);
+}
+
+sub _run_pdf_to_text_cmd {
+    my ($cmd, $output_filestem) = @_;
 
     if ($ENV{'GSDLOS'} !~ m/^windows$/i) {

main/trunk/greenstone2/perllib/plugins/PDFPlugin.pm

@@ -151 +151 @@
     # TODO: Start supporting PDF to txt on Windows if we're going to be using XPDF Tools (incl pdftotext) on Windows/Linux/Mac
     if ($self->{'convert_to'} eq "text" && $ENV{'GSDLOS'} =~ /^windows$/i) {
-    print STDERR "Windows does not support pdf to text. PDFs will be converted to HTML instead\n";
-    $self->{'convert_to'} = "html";
+        print STDERR "On Windows, Greenstone now uses Xpdf tools to support pdf to text conversion.\n";
+    #print STDERR "Windows does not support pdf to text. PDFs will be converted to HTML instead\n";
+    #$self->{'convert_to'} = "html";
     }
     elsif ($self->{'convert_to'} eq "auto") {