Changeset 3181 for trunk

Timestamp:

2002-06-25T20:15:26+12:00 (22 years ago)

Author:

sjboddie

Message:

Altered the getcharequiv() function so it now converts entities to raw
utf-8 characters. This should now work for entities like ∏ (i.e.
characters outside of the latin 1 character set).

Location:

trunk/gsdl/perllib

Files:

• classify/phind.pm (modified) (1 diff)
• ghtml.pm (modified) (2 diffs)
• plugins/HTMLPlug.pm (modified) (1 diff)

trunk/gsdl/perllib/classify/phind.pm

@@ -461 +461 @@
     # 2. Split the remaining text into space-delimited tokens
 
-    # Convert any HTML special characters (like ") to their UTF8 equivalent
-    s/&([^;]+);/&unicode::ascii2utf8(\&ghtml::getcharequiv($1,1))/gse;
+    # Convert entities to their UTF8 equivalents
+    s/&([^;]+);/&ghtml::getcharequiv($1,1)/gse;
 
     # Split text at word boundaries

trunk/gsdl/perllib/ghtml.pm

@@ -27 +27 @@
 
 package ghtml;
+
+use unicode;
 
 # htmlsafe(TEXT)
@@ -142 +144 @@
 
 
-# returns the character in the standard html font. It assumes that the
+# returns the character as a raw utf-8 character. It assumes that the
 # & and ; have been stripped off the string.
 sub getcharequiv {
     my ($entity, $convertsymbols) = @_;
 
-    # replace &#8218 with comma (",")
-    $entity =~ s/8218/044/;
-
     # a numeric entity
     if ($entity =~ /^\#0*(\d+)/) {
-    return pack("c", $1);
+    return &unicode::unicode2utf8([$1]);
     }
     
     # a named character entity
     if (defined $charnetosf{$entity}) {
-    return pack("c", $charnetosf{$entity});
+    return &unicode::unicode2utf8([$charnetosf{$entity}]);
     }
 
     # a named symbol entity
     if ($convertsymbols && defined $symnetosf{$entity}) {
-    return pack("c", $symnetosf{$entity});
+    return &unicode::unicode2utf8([$symnetosf{$entity}]);
     }
 

trunk/gsdl/perllib/plugins/HTMLPlug.pm

@@ -676 +676 @@
     &BasPlug::read_file($self, $filename, $encoding, $language, $textref);
 
-
-    # Convert things like é to their UTF8 equivalents
+    # Convert entities to their UTF8 equivalents
     $$textref =~ s/&(lt|gt|amp|quot);/&z$1;/go;
-    $$textref =~ s/&([^;]+);/&unicode::ascii2utf8(\&ghtml::getcharequiv($1,1))/gseo;
+    $$textref =~ s/&([^;]+);/&ghtml::getcharequiv($1,1)/gseo;
     $$textref =~ s/&z(lt|gt|amp|quot);/&$1;/go;
 }