Changeset 22841

Timestamp:

2010-09-02T14:20:00+12:00 (14 years ago)

Author:

davidb

Message:

More explicit use of utf8 for input and output file handling. Relies on strings in Perl being Unicode aware (and not merely binary bytes) otherwise binary bytes will then be incorrectly re-incoded as UTF-8 (which is not what you want as they already are in UTF-8 form). In the case of this plugin, both text and metadata (read from doc.xml using XML::Parser) is is binary byte format, and so needs to be decoded before being added into $doc_obj->add_utf8_...l

File:

• main/trunk/greenstone2/perllib/plugins/GreenstoneXMLPlugin.pm (modified) (5 diffs)

main/trunk/greenstone2/perllib/plugins/GreenstoneXMLPlugin.pm

@@ +1 @@
+
 ###########################################################################
 #
@@ -31 +32 @@
 package GreenstoneXMLPlugin;
 
+use Encode;
+
 use ReadXMLFile;
 
@@ -39 +42 @@
     @GreenstoneXMLPlugin::ISA = ('ReadXMLFile');
 }
+
+
 
 
@@ -143 +148 @@
     }
     elsif ($element eq "Metadata") {
-    $self->{'doc_obj'}->add_utf8_metadata($self->{'section'}, $self->{'metadata_name'},$self->{'metadata_value'});
+    # text read in by XML::Parser is in Perl's binary byte value
+    # form ... need to explicitly make it UTF-8
+
+    my $metadata_name = decode("utf-8",$self->{'metadata_name'});
+    my $metadata_value = decode("utf-8",$self->{'metadata_value'});
+
+    $self->{'doc_obj'}->add_utf8_metadata($self->{'section'}, 
+                          $metadata_name,$metadata_value);
+
         # Ensure this value is added to the allvalues database in gseditor.
         # Note that the database constraints prevent multiple occurances of the
@@ -182 +195 @@
     }
     elsif ($element eq "Content" && $self->{'content'} ne "") {
-    $self->{'doc_obj'}->add_utf8_text($self->{'section'}, $self->{'content'});
+
+    # text read in by XML::Parser is in Perl's binary byte value
+    # form ... need to explicitly make it UTF-8
+    my $content = decode("utf-8",$self->{'content'});
+
+    $self->{'doc_obj'}->add_utf8_text($self->{'section'}, $content);
     $self->{'content'} = "";
     }