Changeset 3542

Timestamp:

2002-11-21T10:33:13+13:00 (21 years ago)

Author:

jrm21

Message:

ghtml returns utf8, not iso-8859-1, so any html entities were being converted
twice. We now make the text utf8 first.

File:

• trunk/gsdl/perllib/plugins/HBPlug.pm (modified) (6 diffs)

trunk/gsdl/perllib/plugins/HBPlug.pm

@@ -44 +44 @@
 use ghtml;
 use BasPlug;
+use unicode;
 use util;
 use doc;
@@ -111 +112 @@
     close FILE;
     }
-}
-
+    # text is in utf8
+}
+
+# converts the text to utf8, as ghtml does that for é etc.
 sub HB_gettext {
     my $self = shift (@_);
@@ -139 +142 @@
     $line =~ s/<\/?(body|html|font)\b[^>]*>//ig; # remove any unwanted tags
 
-    # convert any alphanumeric character entities to their extended
-    # ascii equivalent for indexing purposes
-    &ghtml::convertcharentities ($line);
-
     $$text .= $line;
     }
+    #
+    if ($self->{'input_encoding'} eq "iso_8859_1") {
+    # convert to utf-8
+    $$text=&unicode::unicode2utf8(&unicode::convert2unicode("iso_8859_1", $text));
+    }
+    # convert any alphanumeric character entities to their utf-8
+    # equivalent for indexing purposes
+    &ghtml::convertcharentities ($$text);
+
     $$text =~ s/\s+/ /g; # remove \n's
 }
@@ -205 +213 @@
     my ($doc_obj, $cursection, $field, $value) = @_;
 
-    if ($self->{'input_encoding'} eq "ascii") {
-    $doc_obj->add_utf8_metadata ($cursection, $field, $value);
-    } else {
-    $doc_obj->add_metadata ($cursection, $field, $value);
-    }
+# All text should now be in utf-8
+#    if ($self->{'input_encoding'} eq "ascii") {
+    $doc_obj->add_utf8_metadata ($cursection, $field, $value);
+#    } else {
+#   $doc_obj->add_metadata ($cursection, $field, $value);
+#    }
 }
 
@@ -234 +243 @@
     my $html = "";
     $self->HB_read_html_file ($htmlfile, \$html);
+    # html is in utf8
 
     # create a new document
@@ -305 +315 @@
 
         # add the text for this section
-        if ($self->{'input_encoding'} eq "ascii") {
-        $doc_obj->add_utf8_text ($cursection, $sectiontext);
-        } else {
-        $doc_obj->add_text ($cursection, $sectiontext);
-        }
+# All read text should now be in utf-8
+#       if ($self->{'input_encoding'} eq "ascii") {
+        $doc_obj->add_utf8_text ($cursection, $sectiontext);
+#       } else {
+#       $doc_obj->add_text ($cursection, $sectiontext);
+#       }
     } else {
         print $outhandle "WARNING - leftover text\n" , $self->shorten($html),