Changeset 22840

Timestamp:

2010-09-02T14:18:47+12:00 (14 years ago)

Author:

davidb

Message:

More explicit use of utf8 for input and output file handling. Relies on strings in Perl being Unicode aware (and not merely binary bytes) otherwise binary bytes will then be incorrectly re-incoded as UTF-8 (which is not what you want as they already are in UTF-8 form). In the case of this plugin, text that comes in from doctxt.xml (read with XPATH) is Unicode aware (can be added to $doc_obj directly with add_utf8_....). In the case of the metadata (read from docmets.xml using XML::Parser) is is binary byte format, and so needs to be decoded before being added into $doc_obj->add_utf8_...l

File:

• main/trunk/greenstone2/perllib/plugins/GreenstoneMETSPlugin.pm (modified) (6 diffs)

main/trunk/greenstone2/perllib/plugins/GreenstoneMETSPlugin.pm

@@ -32 +32 @@
 package GreenstoneMETSPlugin;
 
+use Encode;
 use ghtml;
 
@@ -45 +46 @@
 }
 
-my $arguments = [
-         ];
+my $arguments = [ { 'name' => "process_exp",
+            'desc' => "{BasePlugin.process_exp}",
+            'type' => "regexp",
+            'reqd' => "no",
+            'deft' => &get_default_process_exp()
+            }
+          ];
+
 my $options = { 'name'     => "GreenstoneMETSPlugin",
         'desc'     => "{GreenstoneMETSPlugin.desc}",
         'abstract' => "no",
-        'inherits' => "yes" };
+        'inherits' => "yes",
+        'args'     => $arguments };
+ 
 
 
@@ -100 +109 @@
     $filename =~ s/docmets.xml$/doctxt.xml/;
    
-    if (!open (FILEIN,"<$filename")){
+    if (!open (FILEIN, "<:utf8", $filename)) {
     print STDERR "Warning: unable to open the $filename\n";
     $self->{'xmltxt'} = "";
@@ -111 +120 @@
         }
     }
+
         my $xml_parser = XML::XPath->new (xml=> $xml_text);
     #my $xml_tree = $xml_parser->parse ($xml_text);
@@ -222 +232 @@
     
     foreach my $md_pair (@$md_list){
-        my $metadata_name = $md_pair->{'metadata_name'};
-        my $metadata_value = $md_pair->{'metadata_value'};
-        $self->{'doc_obj'}->add_utf8_metadata($self->{'section'}, $metadata_name, $metadata_value);
+        # text read in by XML::Parser is in Perl's binary byte value
+        # form ... need to explicitly make it UTF-8
+
+        my $metadata_name = decode("utf8",$md_pair->{'metadata_name'});
+        my $metadata_value = decode("utf8",$md_pair->{'metadata_value'});
+
+        $self->{'doc_obj'}->add_utf8_metadata($self->{'section'}, 
+                          $metadata_name, $metadata_value);
     }
     
@@ -231 +246 @@
     
     foreach my $section_content (@$content_list){
+        # Don't need to decode $content as this has been readin in
+        # through XPath which (unlike XML::Parser) correctly sets
+        # the string to be UTF8 rather than a 'binary' string of bytes
         my $content = $section_content->{'section_content'};
+
         $self->{'doc_obj'}->add_utf8_text($self->{'section'},$content);
     }