Changeset 22950

Timestamp:

2010-09-23T16:07:18+12:00 (13 years ago)

Author:

davidb

Message:

Old routine used to work on raw binary strings that just happened to be UTF-8 compliant. Now strings are (in the Perl sense) Unicode aware, we *don't* want to check for them being valid utf8. In fact, things can start to go wrong if we do

File:

• main/trunk/greenstone2/perllib/doc.pm (modified) (2 diffs)

main/trunk/greenstone2/perllib/doc.pm

@@ -857 +857 @@
     my ($section, $field, $value) = @_;
 
+#    my ($cpackage,$cfilename,$cline,$csubr,$chas_args,$cwantarray) = caller(1);
+#    my ($lcfilename) = ($cfilename =~ m/([^\\\/]*)$/);
+#    print STDERR "** Calling method: $lcfilename:$cline $cpackage->$csubr\n";
+
     my $section_ptr = $self->_lookup_section($section);
     if (!defined $section_ptr) {
@@ -872 +876 @@
     
     #print STDERR "###$field=$value\n";
+
+    # For now, supress this check.  Given that text data read in is now 
+    # Unicode aware, then the following block of code can (ironically enough) 
+    # cause our unicode compliant string to be re-encoded (leading to
+    # a double-encoded UTF-8 string, which we definitely don't want!).
+    
+
     # double check that the value is utf-8
-    if (!&unicode::check_is_utf8($value)) {
-    print STDERR "doc::add_utf8_metadata - warning: '$field''s value $value wasn't utf8.";
-    &unicode::ensure_utf8(\$value);
-    print STDERR " Tried converting to utf8: $value\n";
-    }
+#    if (!&unicode::check_is_utf8($value)) {
+#   print STDERR "doc::add_utf8_metadata - warning: '$field''s value $value wasn't utf8.";
+#   &unicode::ensure_utf8(\$value);
+#   print STDERR " Tried converting to utf8: $value\n";
+#    }
 
     push (@{$section_ptr->{'metadata'}}, [$field, $value]);