Changeset 35164

Timestamp:

2021-05-17T09:46:29+12:00 (3 years ago)

Author:

kjdon

Message:

xpdf seems to output surrogate pairs into the html - these end up being invalid in utf8. Detect these and convert to proper characters, which can then be written out as valid utf8.

File:

• main/trunk/greenstone2/perllib/plugins/PDFv2Plugin.pm (modified) (2 diffs)

main/trunk/greenstone2/perllib/plugins/PDFv2Plugin.pm

@@ -443 +443 @@
 #       print STDERR "@@@ About to process html file $pagefile (num $page_num)\n";
         my $modified_page_contents = $self->_process_pretty_html_page($pagefile, $page_num, $num_html_pages);
+        # check for surrogates which are illegal in utf8 - need converting to proper codepoint
+        if ($modified_page_contents =~ /[\x{D800}-\x{DFFF}]/) {
+        $modified_page_contents =~ s/([\x{D800}-\x{DBFF}])([\x{DC00}-\x{DFFF}])/desurrogate($1, $2)/ge;
+        }
         print OUTFILE "$modified_page_contents\n\n";
     }
@@ -461 +465 @@
     # of each page.    
     # HTMLPlugin will process these further in the plugin pipeline
+}
+
+# Turn a surrogate pair into a character
+sub desurrogate
+{
+    my ($hi, $lo) = @_;
+    my $codepoint = 0x10000 + (ord($hi) - 0xD800) * 0x400 + (ord($lo) - 0xDC00);
+    return chr($codepoint);
 }