Changeset 22857

Timestamp:

2010-09-06T19:08:11+12:00 (14 years ago)

Author:

davidb

Message:

Further adjustments to our reading in of text files/data to be Unicode aware. These changes address the issue of when the data being read in is in a different encoding. The key 'problem' was that the (now) Unicode aware was being erroneously being re-encoded within HTMLPlugin

Location:

main/trunk/greenstone2/perllib/plugins

Files:

• HTMLPlugin.pm (modified) (1 diff)
• MetadataXMLPlugin.pm (modified) (2 diffs)
• ReadTextFile.pm (modified) (2 diffs)

main/trunk/greenstone2/perllib/plugins/HTMLPlugin.pm

@@ -1296 +1296 @@
 
     $$text =~ s/\s+/ /g; # remove \n's
+
+    # At this point $$text is a binary byte string
+    # => turn it into a Unicode aware string, so full
+    # Unicode aware pattern matching can be used.
+    # For instance: 's/\x{0101}//g' or '[[:upper:]]'
+    # 
+
+    $$text = decode("utf8",$$text);
 }
 

main/trunk/greenstone2/perllib/plugins/MetadataXMLPlugin.pm

@@ -89 +89 @@
 use strict;
 no strict 'refs';
+
+use Encode;
+
 use BasePlugin;
 use util;
@@ -339 +342 @@
     }
     elsif ($element eq "Metadata") {
-    &metadatautil::store_saved_metadata($self,$self->{'metadata_name'}, $self->{'metadata_value'}, $self->{'metadata_accumulate'});
+    my $metadata_name = decode("utf8",$self->{'metadata_name'});
+    my $metadata_value = decode("utf8",$self->{'metadata_value'});
+    
+    &metadatautil::store_saved_metadata($self,
+                        $metadata_name, $metadata_value, 
+                        $self->{'metadata_accumulate'});
     $self->{'metadata_name'} = "";
     }

main/trunk/greenstone2/perllib/plugins/ReadTextFile.pm

@@ -222 +222 @@
     }
 
-    # At this point $$testref is a binary byte string
+    # At this point $$textref is a binary byte string
     # => turn it into a Unicode aware string, so full
     # Unicode aware pattern matching can be used.
@@ -235 +235 @@
 
 # Not currently used
-sub read_file_usingPerlsEncodeModule {
+sub UNUSED_read_file_usingPerlsEncodeModule {
+##sub read_file {
     my $self = shift (@_);
     my ($filename, $encoding, $language, $textref) = @_;