Changeset 22844

Timestamp:

2010-09-02T14:32:53+12:00 (14 years ago)

Author:

davidb

Message:

More explicit use of utf8 for input and output file handling. Relies on strings in Perl being Unicode aware (and not merely binary bytes) otherwise binary bytes will then be incorrectly re-incoded as UTF-8 (which is not what you want as they already are in UTF-8 form). Decoding the text read in at this point to UTF-8 means that (subsequent) regular expression processing of the data can make us Unicode aware operations, such as what Unicode classes as punctuation

File:

• main/trunk/greenstone2/perllib/plugins/ReadTextFile.pm (modified) (2 diffs)

main/trunk/greenstone2/perllib/plugins/ReadTextFile.pm

@@ -30 +30 @@
 no strict 'refs'; # allow filehandles to be variables and viceversa
 
+use Encode;
 
 use multiread;
@@ -220 +221 @@
     $reader->read_file ($textref);
     }
+
+    # At this point $$testref is a binary byte string
+    # => turn it into a Unicode aware string, so full
+    # Unicode aware pattern matching can be used.
+    # For instance: 's/\x{0101}//g' or '[[:upper:]]'
+    # 
+
+    $$textref = decode("utf8",$$textref);
+
+    close FILE;
+}
+
+
+# Not currently used
+sub read_file_usingPerlsEncodeModule {
+    my $self = shift (@_);
+    my ($filename, $encoding, $language, $textref) = @_;
+
+    if (!-r $filename)
+    {
+        my $outhandle = $self->{'outhandle'};
+        gsprintf($outhandle, "{ReadTextFile.read_denied}\n", $filename) if $self->{'verbosity'};
+        # print $outhandle "Read permission denied for $filename\n" if $self->{'verbosity'};
+        return;
+    }
+    $$textref = "";
+    if (!open (FILE, $filename)) {
+        gsprintf(STDERR, "ReadTextFile::read_file {ReadTextFile.could_not_open_f
+or_reading} ($!)\n", $filename);
+        die "\n";
+    }
+
+    my $store_slash = $/;
+    undef $/;
+    my $text = <FILE>;
+    $/ = $store_slash;
+
+    $$textref = decode($encoding,$text);
+
     close FILE;
 }