Changeset 27106 for main/trunk

Timestamp:

2013-03-21T12:39:49+13:00 (11 years ago)

Author:

kjdon

Message:

need to do the same utf8 decode step that is used in ReadTextFile on the text and metadata so that we get proper utf8 strings

File:

• main/trunk/greenstone2/perllib/plugins/MARCPlugin.pm (modified) (3 diffs)

main/trunk/greenstone2/perllib/plugins/MARCPlugin.pm

@@ -29 +29 @@
 use SplitTextFile;
 use MetadataRead;
+
+use Encode;
 
 use unicode;
@@ -179 +181 @@
     if ($encoding eq "utf8") {
     # nothing needs to be done
-    return $line;
-    }
-
-    if ($encoding eq "iso_8859_1") {
+    #return $line;
+    } elsif ($encoding eq "iso_8859_1") {
     # we'll use ascii2utf8() for this as it's faster than going
     # through convert2unicode()
-    return &unicode::ascii2utf8 (\$line);
-    }
+    #return &unicode::ascii2utf8 (\$line);
+    $line = &unicode::ascii2utf8 (\$line);
+    } else {
 
     # everything else uses unicode::convert2unicode
-    return &unicode::unicode2utf8 (&unicode::convert2unicode ($encoding, \$line));
+    $line = &unicode::unicode2utf8 (&unicode::convert2unicode ($encoding, \$line));
+    }
+    # At this point $line is a binary byte string
+    # => turn it into a Unicode aware string, so full
+    # Unicode aware pattern matching can be used.
+    # For instance: 's/\x{0101}//g' or '[[:upper:]]'
+
+    return decode ("utf8", $line);
 }
 
@@ -266 +274 @@
 
     my $encoding = $self->{'readfile_encoding'}->{$filename};
-
     if (defined ($self->{'metadata_mapping'}) ) {
     if ($self->{'type'} ne "ascii" ){