Changeset 35166 for main/trunk/greenstone2/perllib/plugins/HTMLPlugin.pm

Timestamp:

2021-05-17T12:28:53+12:00 (3 years ago)

Author:

kjdon

Message:

added code that handles utf16 surrogate pair entities.

File:

• main/trunk/greenstone2/perllib/plugins/HTMLPlugin.pm (modified) (2 diffs)

main/trunk/greenstone2/perllib/plugins/HTMLPlugin.pm

@@ -200 +200 @@
     # Convert entities to their UTF8 equivalents
     $$textref =~ s/&(lt|gt|amp|quot|nbsp);/&z$1;/go;
-    $$textref =~ s/&([^;]+);/&ghtml::getcharequiv($1,1,0)/gseo; # on this occassion, want it left as utf8
+    $$textref =~ s/&([^;]+);/&ghtml::getcharequiv($1,1,0,0)/gseo; # on this occassion, want it left as utf8 and we throw away utf16 surrogates (is that the right thing to do here?)
     $$textref =~ s/&z(lt|gt|amp|quot|nbsp);/&$1;/go;
 
@@ -1626 +1626 @@
     # Convert entities to their Unicode code-point equivalents
     $$textref =~ s/&(lt|gt|amp|quot|nbsp);/&z$1;/go;
-    $$textref =~ s/&([^;]+);/&ghtml::getcharequiv($1,1,1)/gseo;
+    $$textref =~ s/&([^;]+);/&ghtml::getcharequiv($1,1,1,1)/gseo; # leaves surrogate pairs as entities
+    $$textref =~ s/&\#([^;]+);&\#([^;]+);/&ghtml::desurrogate($1, $2)/gseo; # convert surrogate pairs
     $$textref =~ s/&z(lt|gt|amp|quot|nbsp);/&$1;/go;