Index: main/trunk/greenstone2/bin/script/gsConvert.pl
===================================================================
--- main/trunk/greenstone2/bin/script/gsConvert.pl	(revision 32276)
+++ main/trunk/greenstone2/bin/script/gsConvert.pl	(revision 32277)
@@ -78,5 +78,5 @@
     print STDERR "  options:\n\t-type\tdoc|dot|pdf|ps|ppt|rtf|xls\t(input file type)\n";
     print STDERR "\t-errlog\t<filename>\t(append err messages)\n";
-    print STDERR "\t-output\tauto|html|paged_html|text|pagedimg_jpg|pagedimg_gif|pagedimg_png\t(output file type)\n";
+    print STDERR "\t-output\tauto|html|pretty_html|paged_pretty_html|paged_html|text|paged_text|pagedimg_jpg|pagedimg_gif|pagedimg_png|pagedimgtxt_jpg|pagedimgtxt_png\t(output file type)\n";
     print STDERR "\t-timeout\t<max cpu seconds>\t(ulimit on unix systems)\n";
     print STDERR "\t-use_strings\tuse strings to extract text if conversion fails\n";
@@ -318,5 +318,7 @@
     my $success = 0;
     $output_type =~ s/.*\-(.*)/$1/i;
-
+    
+    print STDERR "@@@@@@@@ Using $pdf_tool for the conversion\n";
+    
     # First determine which pdf conversion tool we're using among pdftohtml/pdfbox/xpdftools
     # and then decide which conversion command to run based on the output type 
@@ -353,8 +355,9 @@
   }
     
-  elsif ($pdf_tool eq "xpdftools" ) {
-	# default to html output
+    elsif ($pdf_tool eq "xpdftools" ) {
+	
+	# default to pretty html output
 	if (!$output_type) {
-	    $output_type = "html";
+	    $output_type = "pretty_html";
 	}
 	
@@ -367,6 +370,6 @@
 	#}
 	
-	# Attempt conversion to (paged) HTML using the newer pdftohtml of Xpdftools.
-	if ($output_type =~ m/^(paged_html|html)$/i) {
+	# Attempt conversion to (paged) pretty HTML using the newer pdftohtml of Xpdftools.
+	if ($output_type =~ m/pretty_html$/i) {
 	    $success = &xpdf_to_html($dirname, $input_filename, $output_filestem);
 	    if ($success) {
@@ -376,6 +379,7 @@
 	
 	# Attempt conversion to TEXT
-	if (!$output_type || ($output_type =~ m/text/i)) {	    
-	    $success = &xpdf_to_text($dirname, $input_filename, $output_filestem);
+	# Proper paged_text processing not yet implemented with xpdf
+	if ($output_type =~ m/text/i) {
+	    $success = &xpdf_to_text($dirname, $input_filename, $output_filestem, $output_type);
 	    
 	    if ($success) {
@@ -383,6 +387,6 @@
 	    }
 	}
-  }
-	
+    }
+    
     return "fail";
 
@@ -1048,5 +1052,5 @@
 # Works for Windows too, whereas the old pdftotxt didn't
 sub xpdf_to_text {
-    my ($dirname, $input_filename, $output_filestem) = @_;
+    my ($dirname, $input_filename, $output_filestem, $output_type) = @_;
 
     my $cmd = "";
@@ -1064,5 +1068,8 @@
 		$cmd .= " -enc UTF-8"; # see https://www.xpdfreader.com/xpdfrc-man.html
 	}
+
+    if ($output_type ne "paged_text") { # output_type eq "text", don't bother about page break markers
 	$cmd .= " -nopgbrk";
+    }
 	# Avoid the silly solitary carriage returns (CR in Notepad) at the end
 	# of lines that ends up as \n appended to the doc title
Index: main/trunk/greenstone2/perllib/plugins/ConvertBinaryFile.pm
===================================================================
--- main/trunk/greenstone2/perllib/plugins/ConvertBinaryFile.pm	(revision 32276)
+++ main/trunk/greenstone2/perllib/plugins/ConvertBinaryFile.pm	(revision 32277)
@@ -161,5 +161,5 @@
     }
 
-    if ($convert_to =~ /^html/ || $convert_to eq "paged_html") { # may be html or html_multi, or paged_html with the new Xpdf's own pdftohtml
+    if ($convert_to =~ /^html/ || $convert_to =~ /pretty_html$/) { # may be html or html_multi, or )paged_)pretty_html with the new Xpdf's own pdftohtml
 	$self->{'convert_to_plugin'} = "HTMLPlugin";
 	$self->{'convert_to_ext'} = "html";
Index: main/trunk/greenstone2/perllib/plugins/PDFPlugin.pm
===================================================================
--- main/trunk/greenstone2/perllib/plugins/PDFPlugin.pm	(revision 32276)
+++ main/trunk/greenstone2/perllib/plugins/PDFPlugin.pm	(revision 32277)
@@ -159,5 +159,5 @@
     # check convert_to
     if ($self->{'convert_to'} eq "text" && $ENV{'GSDLOS'} =~ /^windows$/i) {
-	&gsprintf::gsprintf(STDERR, "{PDFPlugin.win_old_pdftotext_unsupported}\n", "PDFPlugin");
+	&gsprintf::gsprintf(STDERR, "{PDFPlugin.win_old_pdftotext_unsupported}\n");
 	$self->{'convert_to'} = "html";
     }
Index: main/trunk/greenstone2/perllib/plugins/PDFv1Plugin.pm
===================================================================
--- main/trunk/greenstone2/perllib/plugins/PDFv1Plugin.pm	(revision 32276)
+++ main/trunk/greenstone2/perllib/plugins/PDFv1Plugin.pm	(revision 32277)
@@ -93,5 +93,5 @@
        'type' => "flag" },
      { 'name' => "zoom",
-       'desc' => "{PDFPlugin.zoom}",
+       'desc' => "{PDFv1Plugin.zoom}",
        'deft' => "2",
        'range' => "1,3", # actually the range is 0.5-3 
@@ -147,5 +147,5 @@
     # check convert_to
     if ($self->{'convert_to'} eq "text" && $ENV{'GSDLOS'} =~ /^windows$/i) {
-	&gsprintf::gsprintf(STDERR, "{PDFPlugin.win_old_pdftotext_unsupported}\n", "PDFv1Plugin");
+	&gsprintf::gsprintf(STDERR, "{PDFv1Plugin.win_old_pdftotext_unsupported}\n");
 	$self->{'convert_to'} = "html";
     }
Index: main/trunk/greenstone2/perllib/plugins/PDFv2Plugin.pm
===================================================================
--- main/trunk/greenstone2/perllib/plugins/PDFv2Plugin.pm	(revision 32277)
+++ main/trunk/greenstone2/perllib/plugins/PDFv2Plugin.pm	(revision 32277)
@@ -0,0 +1,779 @@
+###########################################################################
+#
+# PDFv2Plugin.pm -- pdf plugin that uses xpdftools or, if switched on,
+# pdfbox, to process PDFs.
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999-2001 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+package PDFv2Plugin;
+
+use strict;
+no strict 'refs'; # so we can use a var for filehandles (e.g. STDERR)
+no strict 'subs'; # allow filehandles to be variables and viceversa
+
+use ReadTextFile;
+use unicode;
+use Mojo::DOM; # for HTML parsing
+
+use AutoLoadConverters;
+use ConvertBinaryFile;
+
+@PDFv2Plugin::ISA = ('ConvertBinaryFile', 'AutoLoadConverters', 'ReadTextFile');
+
+
+my $convert_to_list =
+    [ {	'name' => "auto",
+	'desc' => "{ConvertBinaryFile.convert_to.auto}" },
+      {	'name' => "text", # xpdftools
+	'desc' => "{ConvertBinaryFile.convert_to.text}" },
+      {	'name' => "paged_text",	# xpdftools
+	'desc' => "{ConvertBinaryFile.convert_to.paged_text}" },
+      
+      {	'name' => "html", # pdfbox ## TODO: rename this to html_without_imgs
+	'desc' => "{PDFPlugin.convert_to.html}" },
+      {	'name' => "pretty_html", # xpdftools
+	'desc' => "{PDFPlugin.convert_to.pretty_html}" },
+      { 'name' => "paged_pretty_html", # xpdftools
+	'desc' => "{PDFPlugin.convert_to.paged_pretty_html}"},
+
+      #pdfbox
+      { 'name' => "pagedimg_jpg",
+	'desc' => "{ConvertBinaryFile.convert_to.pagedimg_jpg}"},
+      { 'name' => "pagedimg_png",
+	'desc' => "{ConvertBinaryFile.convert_to.pagedimg_png}"},
+      
+      { 'name' => "pagedimgtxt_jpg",
+	'desc' => "{ConvertBinaryFile.convert_to.pagedimgtxt_jpg}"},
+      { 'name' => "pagedimgtxt_png",
+	'desc' => "{ConvertBinaryFile.convert_to.pagedimgtxt_png}"},
+      ];
+
+
+my $arguments = 
+    [
+     { 'name' => "convert_to",
+       'desc' => "{ConvertBinaryFile.convert_to}",
+       'type' => "enum",
+       'reqd' => "yes",
+       'list' => $convert_to_list, 
+       'deft' => "pretty_html" },	 
+     { 'name' => "process_exp",
+       'desc' => "{BaseImporter.process_exp}",
+       'type' => "regexp",
+       'deft' => &get_default_process_exp(),
+       'reqd' => "no" },
+     { 'name' => "block_exp",
+       'desc' => "{CommonUtil.block_exp}",
+       'type' => "regexp",
+       'deft' => &get_default_block_exp() },
+     { 'name' => "metadata_fields",
+       'desc' => "{HTMLPlugin.metadata_fields}",
+       'type' => "string",
+       'deft' => "Title,Author,Subject,Keywords" },
+      { 'name' => "metadata_field_separator",
+	'desc' => "{HTMLPlugin.metadata_field_separator}",
+	'type' => "string",
+	'deft' => "" },
+#     { 'name' => "noimages",
+#       'desc' => "{PDFPlugin.noimages}",
+#       'type' => "flag" },
+#     { 'name' => "allowimagesonly",
+#       'desc' => "{PDFPlugin.allowimagesonly}",
+#       'type' => "flag" },
+#     { 'name' => "complex",
+#       'desc' => "{PDFPlugin.complex}",
+#       'type' => "flag" },
+#     { 'name' => "nohidden",
+#       'desc' => "{PDFPlugin.nohidden}",
+#       'type' => "flag" },
+     { 'name' => "zoom",
+       'desc' => "{PDFv2Plugin.zoom}",
+       'deft' => "1",
+       'type' => "string" }, # xpdftools' zoom takes fractions
+     { 'name' => "use_sections",
+       'desc' => "{PDFPlugin.use_sections}",
+       'type' => "flag" },
+     { 'name' => "description_tags",
+       'desc' => "{HTMLPlugin.description_tags}",
+       'type' => "flag" },
+      { 'name' => "use_realistic_book",
+        'desc' => "{PDFPlugin.use_realistic_book}",
+	'type' => "flag"}
+     ];
+
+my $options = { 'name'     => "PDFv2Plugin",
+		'desc'     => "{PDFPlugin.desc}",
+		'abstract' => "no",
+		'inherits' => "yes",
+		'srcreplaceable' => "yes", # Source docs in PDF can be replaced with GS-generated html		
+		'args'     => $arguments };
+
+sub new {
+    my ($class) = shift (@_);
+    my ($pluginlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$pluginlist, $class);
+
+    push(@$inputargs,"-title_sub");
+    push(@$inputargs,'^(Page\s+\d+)?(\s*1\s+)?');
+
+    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
+    push(@{$hashArgOptLists->{"OptList"}},$options);
+
+    my $auto_converter_self = new AutoLoadConverters($pluginlist,$inputargs,$hashArgOptLists,["PDFBoxConverter"],1);
+    my $cbf_self = new ConvertBinaryFile($pluginlist, $inputargs, $hashArgOptLists);
+    my $self = BaseImporter::merge_inheritance($auto_converter_self, $cbf_self);
+    
+    if ($self->{'info_only'}) {
+	# don't worry about any options etc
+	return bless $self, $class;
+    }
+    
+    $self = bless $self, $class;
+    $self->{'file_type'} = "PDF";
+
+    # convert_options are passed through to gsConvert.pl by ConvertBinaryFile.pm
+    
+    # the most important option is the tool that's used to do the conversion
+    $self->{'convert_options'} = "-pdf_tool xpdftools"; # default for PDFv2Plugin. If pdfbox_conversion is on, the pdfbpox GS extension sets pdf_tool to pdfbox
+
+    # pdf_zoom is supported by xpdftools' pdftohtml. So for pretty_html and paged_pretty_html
+    my $zoom = $self->{"zoom"};
+    $self->{'convert_options'} .= " -pdf_zoom $zoom";
+
+    # check convert_to
+    
+    # Not all available conversion output options are possible with xpdftools, as some are
+    # only handled by pdfbox. If a format is unavailable with xpdftools, default to pretty_html
+    if ($self->{'convert_to'} =~ /^html$/) {
+	&gsprintf::gsprintf(STDERR, "{PDFv2Plugin.conversion_needs_pdfbox}\n", ($self->{'convert_to'}, "pretty_html"));
+#	$self->{'convert_to'} = "pretty_html";
+    }
+    elsif ($self->{'convert_to'} =~ /^pagedimg/) {
+	&gsprintf::gsprintf(STDERR, "{PDFv2Plugin.conversion_needs_pdfbox}\n", ($self->{'convert_to'}, "paged_pretty_html"));
+#	$self->{'convert_to'} = "paged_pretty_html";
+    }
+    elsif ($self->{'convert_to'} =~ /^paged_text$/) {
+	# TODO
+	print STDERR "@@@ Conversion to " . $self->{'convert_to'} , " with Xpdf Tools is not yet implemented.\n";
+	print STDERR "@@@ Converting to a text instead.\n";
+	$self->{'convert_to'} = "text";
+    }
+    
+    # PDFv2Plugin now supports PDF to txt conversion on Windows too:
+    # using XPDF Tools (incl pdftotext) on Windows/Linux/Mac
+    elsif ($self->{'convert_to'} eq "text" && $ENV{'GSDLOS'} =~ /^windows$/i) {
+	&gsprintf::gsprintf(STDERR, "{PDFv2Plugin.win_pdftotext_info}\n");
+    }
+    elsif ($self->{'convert_to'} eq "auto") {
+	# choose pretty_html is the best default option when using xpdftools
+	$self->{'convert_to'} = "pretty_html";
+    }
+    if ($self->{'use_realistic_book'}) {
+	if ($self->{'convert_to'} ne "html") {
+	    print STDERR "PDFs will be converted to HTML for realistic book functionality\n";
+	    $self->{'convert_to'} = "html";
+	}
+    }
+    # set convert_to_plugin and convert_to_ext
+    $self->set_standard_convert_settings();
+
+    my $secondary_plugin_name = $self->{'convert_to_plugin'};
+    my $secondary_plugin_options = $self->{'secondary_plugin_options'};
+
+    if (!defined $secondary_plugin_options->{$secondary_plugin_name}) {
+	$secondary_plugin_options->{$secondary_plugin_name} = [];
+    }
+    my $specific_options = $secondary_plugin_options->{$secondary_plugin_name};
+
+    # following title_sub removes "Page 1" added by pdftohtml, and a leading
+    # "1", which is often the page number at the top of the page. Bad Luck
+    # if your document title actually starts with "1 " - is there a better way?
+    push(@$specific_options , "-title_sub", '^(Page\s+\d+)?(\s*1\s+)?');
+    my $associate_tail_re = $self->{'associate_tail_re'};
+    if ((defined $associate_tail_re) && ($associate_tail_re ne "")) {
+	push(@$specific_options, "-associate_tail_re", $associate_tail_re);
+    }
+    push(@$specific_options, "-file_rename_method", "none");
+    
+    if ($secondary_plugin_name eq "HTMLPlugin") {
+	# pdftohtml always produces utf8 - What about pdfbox???
+	# push(@$specific_options, "-input_encoding", "utf8");
+	push(@$specific_options, "-extract_language") if $self->{'extract_language'};
+	push(@$specific_options, "-processing_tmp_files");
+	# Instruct HTMLPlug (when eventually accessed through read_into_doc_obj) 
+	# to extract these metadata fields from the HEAD META fields
+	if (defined $self->{'metadata_fields'} && $self->{'metadata_fields'} =~ /\S/) {
+	    push(@$specific_options,"-metadata_fields",$self->{'metadata_fields'});
+	} else {
+	    push(@$specific_options,"-metadata_fields","Title,GENERATOR,date,author<Creator>");
+	}
+	if (defined $self->{'metadata_field_separator'} && $self->{'metadata_field_separator'} =~ /\S/) {
+	    push(@$specific_options,"-metadata_field_separator",$self->{'metadata_field_separator'});
+	}
+	if ($self->{'use_sections'} || $self->{'description_tags'}) {
+	    $self->{'description_tags'} = 1;
+	    push(@$specific_options, "-description_tags");
+	}
+	if ($self->{'use_realistic_book'}) {
+	    push(@$specific_options, "-use_realistic_book");
+	}
+	if($self->{'convert_to'} eq "paged_pretty_html") { # for paged pretty html, the default should be to sectionalise
+	    # the single superpage, the one containing divs representing individual pages as sections, on headings
+	    push(@$specific_options, "sectionalise_using_h_tags");
+	}
+    }
+    elsif ($secondary_plugin_name eq "PagedImagePlugin") {
+	push(@$specific_options, "-screenviewsize", "1000");
+	push(@$specific_options, "-enable_cache");
+	push(@$specific_options, "-processing_tmp_files");
+    }
+
+    $self = bless $self, $class;
+    $self->load_secondary_plugins($class,$secondary_plugin_options,$hashArgOptLists);
+    return $self;
+}
+
+sub get_default_process_exp {
+    my $self = shift (@_);
+
+    return q^(?i)\.pdf$^;
+}
+
+# so we don't inherit HTMLPlug's block exp...
+sub get_default_block_exp {
+    return "";
+}
+
+sub init {
+    my $self = shift (@_);
+
+    # ConvertBinaryFile init
+    $self->SUPER::init(@_);
+    $self->AutoLoadConverters::init(@_);
+
+}
+
+sub begin {
+    my $self = shift (@_);
+
+    $self->AutoLoadConverters::begin(@_);
+    $self->SUPER::begin(@_);
+
+}
+
+sub deinit {
+    my $self = shift (@_);
+    
+    $self->AutoLoadConverters::deinit(@_);
+    $self->SUPER::deinit(@_);
+
+}
+  
+# By setting hashing to be on ga xml this ensures that two 
+# PDF files that are identical except for the metadata
+# to hash to different values. Without this, when each PDF
+# file is converted to HTML there is a chance that they 
+# will both be *identical* if the conversion utility does
+# not embed the metadata in the generated HTML. This is
+# certainly the case when PDFBOX is being used. 
+
+# This change makes this convert to based plugin more 
+# consistent with the original vision that the same document 
+# with different metadata should
+# be seen as different.
+
+sub get_oid_hash_type {
+    my $self = shift (@_);
+    return "hash_on_ga_xml";
+}
+  
+  
+sub tmp_area_convert_file {
+
+    my $self = shift (@_);
+    return $self->AutoLoadConverters::tmp_area_convert_file(@_);
+
+}
+
+# Overriding to do some extra handling for pretty_html/paged_pretty_html output mode
+sub run_conversion_command {
+    my $self = shift (@_);
+    my ($tmp_dirname, $tmp_inputPDFname, $utf8_tailname, $lc_suffix, $tailname, $suffix) = @_;    
+
+    if($self->{'convert_to'} !~ /pretty_html/) {
+	return $self->ConvertBinaryFile::run_conversion_command(@_);
+    }
+
+    # else, paged_pretty_html or pretty_html
+
+    # if output mode is (paged_)pretty_html, we use Xpdf tools' pdftohtml and tell it
+    # to create a subdir called "pages" in the tmp area to puts its products
+    # in there. (Xpdf's pdftohtml needs to be passed a *non-existent* directory
+    # parameter, the "pages" subdir). If Xpdf's pdftohtml has successfully run,
+    # the intermediary output file tmp/<random-num>/pages/index.html should
+    # exist (besides other output products there)
+
+    # We let ConvertBinaryFile proceed normally, but the return value should reflect
+    # that on success it should expect the intermediary product tmpdir/pages/index.html
+    # (which is the product of xpdftohtml conversion).
+    my $output_filename = $self->ConvertBinaryFile::run_conversion_command(@_);
+    $output_filename = &FileUtils::filenameConcatenate($tmp_dirname, "pages", "index.html");
+
+    # However, when convert_post_process() is done, it should have output the final
+    # product of the (paged_)pretty_html conversion: an html file of the same name and in the
+    # same tmp location as the input PDF file.
+
+    my ($name_prefix, $output_dir, $ext)
+	= &File::Basename::fileparse($tmp_inputPDFname, "\\.[^\\.]+\$");
+    $self->{'conv_filename_after_post_process'} = &FileUtils::filenameConcatenate($output_dir, $name_prefix.".html");
+#    print STDERR "@@@@@ final paged html file will be: " . $self->{'conv_filename_after_post_process'} . "\n";
+
+    return $output_filename;
+}
+
+sub convert_post_process
+{
+    my $self = shift (@_);
+    my ($conv_filename) = @_;
+
+    my $outhandle=$self->{'outhandle'};
+
+    if($self->{'convert_to'} =~ /pretty_html/) { # (paged_)pretty_html
+	# special post-processing for (paged_)pretty_html mode, as HTML pages generated
+	# by xpdf's pdftohtml need to be massaged into the form we want 
+	$self->xpdftohtml_convert_post_process($conv_filename);
+    }
+    else { # use PDFPlugin's usual post processing
+	$self->default_convert_post_process($conv_filename);
+    }
+}
+
+# Called after gsConvert.pl has been run to convert a PDF to (paged_)pretty_html
+# using Xpdftools' pdftohtml
+# This method will do some cleanup of the HTML files produced after XPDF has produced
+# an HTML doc for each PDF page: it first gets rid of the default index.html.
+# Instead, it constructs a single html page containing each original HTML page
+# <body> nested as divs instead, with simple section information inserted at the top
+# of each 'page' <div> and some further styling customisation. This HTML manipulation
+# is to be done with the Mojo::DOM perl package.
+# Note that since xpdf's pdftohtml would have failed if the output dir already
+# existed and for simpler naming, the output files are created in a new "pages"
+# subdirectory of the tmp location parent of $conv_filename instead
+sub xpdftohtml_convert_post_process
+{
+    my $self = shift (@_);
+    my ($pages_index_html) = @_; # = tmp/<rand>/pages/index.html for (paged_)pretty_html output mode
+    my $output_filename = $self->{'conv_filename_after_post_process'};
+
+    # Read in all the html files in tmp's "pages" subdir, except for index.html.
+    # and use it to create a new html file called $self->{'conv_filename_after_post_process'}
+    # which will consist of a slightly modified version of
+    # each of the other html files concatenated together.
+
+    my $outhandle=$self->{'outhandle'};
+
+    my ($tailname, $pages_subdir, $suffix)
+	= &File::Basename::fileparse($pages_index_html, "\\.[^\\.]+\$");
+
+    # Code from util::create_itemfile()
+    # Read in all the files
+    opendir(DIR, $pages_subdir) || die "can't opendir $pages_subdir: $!";
+    my @page_files = grep {-f "$pages_subdir/$_"} readdir(DIR);
+    closedir DIR;
+    # Sort files in the directory by page_num
+    # files are named index.html, page1.html, page2.html, ..., pagen.html
+    sub page_number {
+	my ($dir) = @_;
+	my ($pagenum) =($dir =~ m/^page(\d+)\.html?$/i);
+	$pagenum = 0 unless defined $pagenum; # index.html will be given pagenum=0
+	return $pagenum;
+    }
+    # sort the files in the directory in the order of page_num rather than lexically.
+    @page_files = sort { page_number($a) <=> page_number($b) } @page_files;
+
+    #my $num_html_pages = (scalar(@page_files) - 1)/2; # skip index file.
+              # For every html file there's an img file, so halve the total num.
+              # What about other file types that may potentially be there too???
+    my $num_html_pages = 0;
+    foreach my $pagefile (@page_files) {
+	$num_html_pages++ if $pagefile =~ m/\.html?$/ && $pagefile !~ /^index\.html?/i;	
+    }
+
+    # Prepare to create our new html page that will contain all the individual
+    # htmls generated by xpdf's pdftohtml in sequence.
+    # First write the opening html tags out to the output file. These are the
+    # same tags and their contents, including <meta>, as is generated by 
+    # Xpdf's pdftohtml for each of its individual html pages.
+    my $start_text = "<html>\n<head>\n";
+    my ($output_tailname, $tmp_subdir, $html_suffix)
+	= &File::Basename::fileparse($output_filename, "\\.[^\\.]+\$");
+    $start_text .= "<title>$output_tailname</title>\n";
+    $start_text .= "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\">\n";
+    $start_text .= "</head>\n<body>\n\n";
+    
+    if($self->{'convert_to'} =~ /paged_pretty_html/) { # then add the <h>tags for sectionalising
+	$start_text .= "<h1>$output_tailname</h1>\n\n";
+    }
+
+    #handle content encodings the same way that default_convert_post_process does
+    # $self->utf8_write_file ($start_text, $conv_filename); # will close file after write    
+    # Don't want to build a giant string in memory of all the pages concatenated
+    # and then write it out in one go. Instead, build up the final single page
+    # by writing each modified (paged_)pretty_html file out to it as this is processed.
+    # Copying file open/close code from CommonUtil::utf8_write_file()
+    if (!open (OUTFILE, ">:utf8", $output_filename)) {
+	gsprintf(STDERR, "PDFv2Plugin::xpdftohtml_convert_post_process {CommonUtil.could_not_open_for_writing} ($!)\n", $output_filename);
+	die "\n";
+    }
+    print OUTFILE $start_text;
+
+    # Get the contents of each individual HTML page generated by Xpdf, after first
+    # modifying each, and write each out into our single all-encompassing html
+    foreach my $pagefile (@page_files) {
+	if ($pagefile =~ m/\.html?$/ && $pagefile !~ /^index\.html?/i) {
+	    my $page_num = page_number($pagefile);    
+	    # get full path to pagefile
+	    $pagefile = &FileUtils::filenameConcatenate($pages_subdir, $pagefile);
+#	    print STDERR "@@@ About to process html file $pagefile (num $page_num)\n";
+	    my $modified_page_contents = $self->_process_pretty_html_page($pagefile, $page_num, $num_html_pages);
+	    print OUTFILE "$modified_page_contents\n\n";
+	}
+    }
+
+    # we've now created a single HTML file by concatenating (a modified version)
+    # of each paged html file
+    print OUTFILE "</body>\n</html>\n"; # write out closing tags
+    close OUTFILE; # done
+
+    # Get rid of all the htm(l) files incl index.html in the associated "pages"
+    # subdir, since we've now processed them all into a single html file
+    # one folder level up and we don't want HTMLPlugin to process all of them next.
+    &FileUtils::removeFilesFiltered($pages_subdir, "\.html?\$"); #  no specific whitelist, but blacklist htm(l)
+
+    # now the tmp area should contain a single html file contain all the html pages'
+    # contents in sequence, and a "pages" subdir containing the screenshot images
+    # of each page.    
+    # HTMLPlugin will process these further in the plugin pipeline
+}
+
+# For whatever reason, most html <tags> don't get printed out in GLI
+# So when debugging, use this function to print them out as [tags] instead.
+sub _debug_print_html
+{
+    my $self = shift (@_);
+    my ($string_or_dom) = @_;
+
+    # can't seem to determine type of string with ref/reftype
+    # https://stackoverflow.com/questions/1731333/how-do-i-tell-what-type-of-value-is-in-a-perl-variable
+    # Not needed, as $dom objects seem to get correctly stringified in string contexts
+    # $dom.to_string/$dom.stringify seem to get called, no need to call them
+    # https://stackoverflow.com/questions/5214543/what-is-stringification-in-perl
+    my $escapedTxt = $string_or_dom; 
+    $escapedTxt =~ s@\<@[@sg;
+    $escapedTxt =~ s@\>@]@sg;
+
+    print STDERR "#### $escapedTxt\n";
+}
+
+# Helper function for (paged_)pretty_html
+# to read in each page of pretty_html generated by Xpdf's pdftohtml
+# then modify the html suitably using the HTML parsing functions offered by 
+# Mojo::DOM, then return the modified HTML content as a string.
+# For paged_pretty_html, some additional modification is done to sectionalise the final html
+# See https://mojolicious.org/perldoc/Mojo/DOM
+sub _process_pretty_html_page
+{
+    my $self = shift (@_);
+    my ($pagefile, $page_num, $num_html_pages) = @_;
+
+    my $text = "";
+
+    # handling content encoding the same way default_convert_post_process does
+    $self->read_file ($pagefile, "utf8", "", \$text);
+
+    my $dom = Mojo::DOM->new($text);
+
+#    $self->_debug_print_html($dom);
+
+    # there's a <style> element on the <html>, we need to shift it into the <div>
+    # tag that we'll be creating. We'll first slightly modify the <style> element
+    # store the first style element, which is the only one and in the <body>
+    # we'll later insert it as child of an all-encompassing div that we'll create
+    my $page_style_tag_str = $dom->at('html')->at('style')->to_string;
+    # In the style tag, convert id style references to class style references
+    my $css_class = ".p".$page_num."f";
+    $page_style_tag_str =~ s@\#f@$css_class@sg;
+    my $style_element = Mojo::DOM->new($page_style_tag_str)->at('style'); # modified    
+#$self->_debug_print_html($style_element);
+
+    # need to know the image's height to set the height of the surrounding
+    # div that's to replace this page's <body>:
+    my $img_height = $dom->find('img')->[0]{height};
+
+    # 2. Adjust the img#background src attribute to point to the pages subdir for imgs
+    # 3. Set that img tag's class=background, and change its id to background+$page_num
+    my $bg_img_tag=$dom->find('img#background')->[0];
+    my $img_src_str = $bg_img_tag->{src};
+    $img_src_str = "pages/$img_src_str";
+    $bg_img_tag->attr(src => $img_src_str); # reset
+#$self->_debug_print_html($bg_img_tag);
+    # set both class and modified id attributes in one step:
+    $bg_img_tag->attr({class => "background", id => "background".$page_num});
+#$self->_debug_print_html($bg_img_tag);
+
+    # get all the <span> nested inside <div class="txt"> elements and
+    # 1. set their class attr to be "p + page_num + id-of-the-span",
+    # 2. then delete the id, because the span ids have been reused when element
+    # ids ought to be unique. Which is why we set the modified ids to be the
+    # value of the class attribute instead
+    $dom->find('div.txt span')->each(sub { 
+	$_->attr(class => "p". $page_num. $_->{id});
+	delete $_->{id};
+				     }); # both changes done in one find() operation
+#$self->_debug_print_html($dom->find('div.txt span')->last);
+
+    # Finally can create our new dom, starting with a div tag for the current page
+    # Must be: <div id="$page_num" style="position:relative; height:$img_height;"/>
+#    my $new_dom = Mojo::DOM->new_tag('div', id => "page".$page_num, style => "position: relative; height: ".$img_height."px;" )
+    my $new_dom = Mojo::DOM->new_tag('div', style => "position: relative; height: ".$img_height."px;" );
+#$self->_debug_print_html($new_dom);
+    $new_dom->at('div')->append_content($style_element)->root;
+
+
+#$self->_debug_print_html($new_dom);
+    # Copy across all the old html's body tag's child nodes into the new dom's new div tag
+    $dom->at('body')->child_nodes->each(sub { $new_dom->at('div')->append_content($_)}); #$_->to_string
+#$self->_debug_print_html($new_dom);
+
+    # build up the outer div
+    my $inner_div_str = $new_dom->to_string;
+    my $page_div = "<div id=\"page".$page_num."\">\n";
+    
+    if($self->{'convert_to'} =~ /paged_pretty_html/) { # then add the <h>tags for sectionalising
+	
+	# Append a page range bucket heading if applicable: if we have more than 10 pages
+	# to display in the current bucket AND we're on the first page of each bucket of 10 pages.
+	# Dr Bainbridge thinks for now we need only consider PDFs where the
+	# total number of pages < 1000 and create buckets of size 10 (e.g. 1-10, ... 51-60, ...)
+	# If number of remaining pages >= 10, then create new bucket heading
+	# e.g. "Pages 30-40"
+	if(($page_num % 10) == 1 && ($num_html_pages - $page_num) > 10) {
+	    # Double-digit page numbers that start with 2
+	    # i.e. 21 to 29 (and 30) should be in 21 to 30 range
+	    my $start_range = $page_num - ($page_num % 10) + 1;
+	    my $end_range = $page_num + 10 - ($page_num % 10);
+	    $page_div .= "<h2 style=\"font-size:1em;font-weight:normal;\">Pages ".$start_range . "-" . $end_range."</h2>\n";
+	}
+
+	# No sectionalising for 10 pages or under. Otherwise, every page is a section too, not just buckets
+	if($num_html_pages > 10) {
+		# Whether we're starting a new bucket or not, add a simpler heading: just the pagenumber, "Page #"	
+		$page_div .= "<h3 style=\"font-size:1em;font-weight:normal;\">Page ".$page_num."</h3>\n";		
+	}
+    }
+
+    $page_div .= $inner_div_str;
+    $page_div .= "\n</div>";
+
+    # Finished processing a single html page of the (paged_)pretty_html output generated by
+    # Xpdf's pdftohtml: finished massaging that single html page into the right form
+    return $page_div;
+}
+
+# This subroutine is called to do the PDFv2Plugin post-processing for all cases
+# except the "pretty_html" or "paged_pretty_html" conversion modes.
+# This is what PDFPlugin always used to do:
+sub default_convert_post_process
+{
+    my $self = shift (@_);
+    my ($conv_filename) = @_;
+    my $outhandle=$self->{'outhandle'};
+
+    #$self->{'input_encoding'} = "utf8"; # TODO: The output is always in utf8 (is it?? it is for html, but what about other types?)
+    #my ($language, $encoding) = $self->textcat_get_language_encoding ($conv_filename);
+
+    # read in file ($text will be in utf8)
+    my $text = "";
+    # encoding will be utf8 for html files - what about other types? will we do this step for them anyway?
+    $self->read_file ($conv_filename, "utf8", "", \$text);
+
+    # To support the use_sections option with PDFBox: Greenstone splits PDFs into pages for 
+    # sections. The PDFPlugin code wants each new page to be prefixed with <a name=pagenum></a>,
+    # which it then splits on to generate page-based sections. However, that's not what PDFBox 
+    # generates in its HTML output. Fortunately, PDFBox does have its own page-separator: it
+    # embeds each page in an extra div. The div opener is: 
+    # <div style=\"page-break-before:always; page-break-after:always\">
+    # The PDFPlugin now looks for this and prefixes <a name=0></a> to each such div. (The 
+    # pagenumber is fixed at 0 since I'm unable to work out how to increment the pagenum during 
+    # a regex substitution even with regex extensions on.) Later, when we process each section 
+    # to get the pagenum, PDFBox's output for this is pre-processed by having a loopcounter 
+    # that increments the pagenum for each subsequent section.
+
+    #$pdfbox_pageheader="\<div style=\"page-break-before:always; page-break-after:always\">";
+    my $loopcounter = 0; # used later on!
+    $text =~ s@\<div style=\"page-break-before:always; page-break-after:always\">@<a name=$loopcounter></a><div style=\"page-break-before:always; page-break-after:always\">@g;
+
+
+    # Calculate number of pages based on <a ...> tags (we have a <a name=1> etc
+    # for each page).  Metadata based on this calculation not set until process()
+    # 
+    # Note: this is done even if we are not breaking the document into pages as it might
+    # be useful to give an indication of document length in browser through setting
+    # num_pages as metadata.
+    # Clean html from low and hight surrogates D800âDFFF
+    $text =~ s@[\N{U+D800}-\N{U+DFFF}]@\ @g;
+    my @pages = ($text =~ m/\<[Aa] name=\"?\w+\"?>/ig); #<div style=\"?page-break-before:always; page-break-after:always\"?>
+    my $num_pages = scalar(@pages);
+    $self->{'num_pages'} = $num_pages;
+
+    if ($self->{'use_sections'}
+	&& $self->{'converted_to'} eq "HTML") {
+
+	print $outhandle "PDFv2Plugin: Calculating sections...\n";
+
+	# we have "<a name=1></a>" etc for each page
+	# it may be <A name=
+	my @sections = split('<[Aa] name=', $text);
+
+	my $top_section = "";
+
+	if (scalar (@sections) == 1) { #only one section - no split!
+	    print $outhandle "PDFv2Plugin: warning - no sections found\n";
+	} else {
+	    $top_section .= shift @sections; # keep HTML header etc as top_section
+	}
+
+	# handle first section specially for title? Or all use first 100...
+	
+	my $title = $sections[0];
+	$title =~ s/^\"?\w+\"?>//; # specific for pdftohtml...
+	$title =~ s/<\/([^>]+)><\1>//g; # (eg) </b><b> - no space
+	$title =~ s/<[^>]*>/ /g;
+	$title =~ s/(?:&nbsp;|\xc2\xa0)/ /g; # utf-8 for nbsp...
+	$title =~ s/^\s+//s;
+	$title =~ s/\s+$//;
+	$title =~ s/\s+/ /gs;
+	$title =~ s/^$self->{'title_sub'}// if ($self->{'title_sub'});
+	$title =~ s/^\s+//s; # in case title_sub introduced any...
+	$title = substr ($title, 0, 100);
+	$title =~ s/\s\S*$/.../;
+
+
+	if (scalar (@sections) == 1) { # no sections found
+	    $top_section .= $sections[0];
+	    @sections=();
+	} else {
+	    $top_section .= "<!--<Section>\n<Metadata name=\"Title\">$title</Metadata>\n-->\n <!--</Section>-->\n";
+	}
+
+	# add metadata per section...
+	foreach my $section (@sections) {
+	    # section names are not always just digits, may be like "outline"
+	    $section =~ s@^\"?(\w+)\"?></a>@@; # leftover from split expression...
+
+	    $title = $1; # Greenstone does magic if sections are titled digits
+
+	    # A title of pagenum=0 means use_sections is being applied on output from PDFBox,
+	    # which didn't originally have a <a name=incremented pagenumber></a> to split each page. 
+	    # Our Perl code then prefixed <a name=0></a> to it. Now need to increment the pagenum here:
+	    if($loopcounter > 0 || ($title eq 0 && $loopcounter == 0)) { # implies use_sections with PDFBox
+		$title = ++$loopcounter;
+	    }
+
+	    if (! defined($title) ) {
+		print STDERR "no title: $section\n";
+		$title = " "; # get rid of the undefined warning in next line
+	    }
+	    my $newsection = "<!-- from PDFv2Plugin -->\n<!-- <Section>\n";
+	    $newsection .= "<Metadata name=\"Title\">" . $title
+		. "</Metadata>\n--><br />\n";
+	    $newsection .= $section;
+	    $newsection .= "<!--</Section>-->\n";
+	    $section = $newsection;
+	}
+
+	$text=join('', ($top_section, @sections));
+    }
+
+    if ($self->{'use_sections'}
+	&& $self->{'converted_to'} eq "text") {
+	print STDERR "**** When converting PDF to text, cannot apply use_sections\n";
+    }
+
+
+    # The following should no longer be needed, now that strings
+    # read in are Unicode aware (in the Perl sense) rather than
+    # raw binary strings that just happen to be UTF-8 compliant
+
+    # turn any high bytes that aren't valid utf-8 into utf-8.
+##    unicode::ensure_utf8(\$text);
+
+    # Write it out again!
+    $self->utf8_write_file (\$text, $conv_filename);
+}
+
+
+# do plugin specific processing of doc_obj for HTML type
+sub process {
+    my $self = shift (@_);
+    my ($pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
+
+    my $result = $self->process_type($base_dir,$file,$doc_obj);
+
+    # fix up the extracted date metadata to be in Greenstone date format,
+    # and fix the capitalisation of 'date'
+    my $cursection = $doc_obj->get_top_section();
+    foreach my $datemeta (@{$doc_obj->get_metadata($cursection, "date")}) {
+	$doc_obj->delete_metadata($cursection, "date", $datemeta);
+
+	# We're just interested in the date bit, not the time
+	# some pdf creators (eg "Acrobat 5.0 Scan Plug-in for Windows")
+	# set a /CreationDate, and set /ModDate to 000000000. pdftohtml
+	# extracts the ModDate, so it is 0...
+	$datemeta =~ /(\d+)-(\d+)-(\d+)/;
+	my ($year, $month, $day) = ($1,$2,$3);
+	if (defined($year) && defined($month) && defined($day)) {
+	    if ($year == 0) {next}
+	    if ($year < 100) {$year += 1900} # just to be safe
+	    if ($month =~ /^\d$/) {$month="0$month"} # single digit
+	    if ($day =~ /^\d$/) {$day="0$day"} # single digit
+	    my $date="$year$month$day";
+	    $doc_obj->add_utf8_metadata($cursection, "Date", $date);
+	}
+    }
+
+    $doc_obj->add_utf8_metadata($cursection, "NumPages", $self->{'num_pages'}) if defined $self->{'num_pages'};
+    
+    if ($self->{'use_sections'} && $self->{'converted_to'} eq "HTML") {
+	# For gs2 we explicitly make it a paged document, cos greenstone won't get it
+	# right if any section has an empty title, or one with letters in it
+	if (&util::is_gs3()) {
+	    # but for gs3, paged docs currently use image slider which is ugly if there are no images
+	    $doc_obj->set_utf8_metadata_element ($cursection, "gsdlthistype", "Hierarchy");
+	} else {
+	    $doc_obj->set_utf8_metadata_element ($cursection, "gsdlthistype", "Paged");
+	}
+    }
+
+    return $result;
+}
+
+1;
Index: main/trunk/greenstone2/perllib/strings.properties
===================================================================
--- main/trunk/greenstone2/perllib/strings.properties	(revision 32276)
+++ main/trunk/greenstone2/perllib/strings.properties	(revision 32277)
@@ -845,4 +845,6 @@
 ConvertBinaryFile.convert_to.text:Plain text format.
 
+ConvertBinaryFile.convert_to.paged_text:Text separately extracted for each individual page.
+
 ConvertBinaryFile.convert_to.pagedimg:A series of images.
 
@@ -852,4 +854,8 @@
 
 ConvertBinaryFile.convert_to.pagedimg_png:A series of images in PNG format. 
+
+ConvertBinaryFile.convert_to.pagedimgtxt_jpg:A series of images in JPEG format with any extracted text, one for each page.
+
+ConvertBinaryFile.convert_to.pagedimgtxt_png:A series of images in PNG format with any extracted text, one for each page.
 
 ConvertBinaryFile.desc:This plugin is inherited by such plugins as WordPlugin, PowerPointPlugin, PostScriptPlugin, RTFPlugin and PDFPlugin. It facilitates the conversion of these document types to either HTML, TEXT or a series of images. It works by dynamically loading an appropriate secondary plugin (HTMLPlugin, StructuredHTMLPlugin, PagedImagePlugin or TextPlugin) based on the plugin argument 'convert_to'. 
@@ -1165,5 +1171,9 @@
 PDFPlugin.complex:Create more complex output. With this option set the output html will look much more like the original PDF file. For this to function properly you Ghostscript installed (for *nix gs should be on your path while for windows you must have gswin32c.exe on your path).
 
-PDFPlugin.convert_to.paged_html:A series of HTML pages, one for each page. Each HTML page contains selectable text positionally overlaid on top of a screenshot of the PDF page background comprising any images, tables and drawings.
+PDFPlugin.convert_to.html:HTML. Text only, no images.
+
+PDFPlugin.convert_to.pretty_html:A series of HTML pages, one for each page. Each HTML page contains selectable text positionally overlaid on top of a screenshot of the PDF page background comprising any images, tables and drawings.
+
+PDFPlugin.convert_to.paged_pretty_html:Sectionalised variant of pretty_html to allow jumping to individual pages.
 
 PDFPlugin.deprecated_plugin:*************IMPORTANT******************\nPDFPlugin is being deprecated.\nConsider upgrading to the recommended PDFv2Plugin, which supports newer versions of PDFs.\nAlternatively, if you wish to retain the old style of conversion and are NOT relying on PDFBox,\nchange to PDFv1Plugin.\nIf you are using PDFBox then upgrade to PDFv2Plugin.\n*****************************************\n
@@ -1179,7 +1189,15 @@
 PDFPlugin.use_sections:Create a separate section for each page of the PDF file.
 
-PDFPlugin.win_old_pdftotext_unsupported:*** On Windows, %s does not support pdf to text. PDFs will be converted to HTML instead.\n*** Use PDFv2Plugin if you really want pdf to text conversion.
-
-PDFPlugin.zoom:The factor by which to zoom the PDF for output. If not outputting as paged_html, then zoom is only useful if -complex is set.
+PDFPlugin.win_old_pdftotext_unsupported:*** On Windows, PDFPlugin pdfbox_conversion must be turned on for text output. PDFs will be converted to HTML instead.\n*** Use PDFv2Plugin for additional pdf to text conversion options.
+
+PDFv1Plugin.win_old_pdftotext_unsupported:*** On Windows, PDFv1Plugin does not support pdf to text. PDFs will be converted to HTML instead.\n*** Use PDFv2Plugin if you want pdf to actual text conversion.
+
+PDFv1Plugin.zoom:The factor by which to zoom the PDF for output. Only useful if -complex is set.
+
+PDFv2Plugin.zoom:The factor by which to zoom the PDF for (paged_)pretty_html output. Can be fractional.
+
+PDFv2Plugin.win_pdftotext_info:PDFv2Plugin uses Xpdf Tools to support pdf to text conversion, including on Windows.
+
+PDFv2Plugin.conversion_needs_pdfbox:*** Conversion to %s not supported with Xpdf Tools, defaulting to %s. Turn on pdfbox_conversion if you wish to enable output to selected format.
 
 PostScriptPlugin.desc:This is a \"poor man's\" ps to text converter. If you are serious, consider using the PRESCRIPT package, which is available for download at http://www.nzdl.org/html/software.html
