Changeset 7798 for trunk/gsdl

Timestamp:

2004-07-22T12:13:45+12:00 (20 years ago)

Author:

jrm21

Message:

added a function, unicode::ensure_utf8(), that will test that the given
string ref is encoded with utf-8, and will modify it if necessary.

add_utf8_metadata() makes use of this function. (Eg previously source filenames
could cause problems).

Location:

trunk/gsdl/perllib

Files:

• doc.pm (modified) (2 diffs)
• unicode.pm (modified) (2 diffs)

trunk/gsdl/perllib/doc.pm

@@ -788 +788 @@
 }
 
-# add_utf8_metadata assumes the text has already been converted
-# to the UTF-8 encoding.
 sub add_utf8_metadata {
     my $self = shift (@_);
@@ -809 +807 @@
     }
     
+    # double check that the value is utf-8
+    if (unicode::ensure_utf8(\$value)) {
+    print STDERR "doc::add_utf8_metadata: warning: '$field' wasn't utf8\n";
+    }
+
     push (@{$section_ptr->{'metadata'}}, [$field, $value]);
 }

trunk/gsdl/perllib/unicode.pm

@@ -6 +6 @@
 # University of Waikato, New Zealand.
 #
-# Copyright (C) 1999 New Zealand Digital Library Project
+# Copyright (C) 1999-2004 New Zealand Digital Library Project
 #
 # This program is free software; you can redistribute it and/or modify
@@ -468 +468 @@
 }
 
+
+# this makes sure that the referenced input string is utf8 encoded, and
+# will change/remove bytes that aren't.
+# returns 0 if the text was already utf8, or 1 if text modified to become utf8
+sub ensure_utf8 {
+    my $stringref=shift;
+
+    my $value=$$stringref;
+
+    my $non_utf8_found = 0;
+    $value =~ m/^/g; # to set \G
+    while ($value =~ m!\G.*?([\x80-\xff]+)!sg) {
+    my $highbytes=$1;
+    my $highbyteslength=length($highbytes);
+    # make sure this block of high bytes is utf-8
+    $highbytes =~ /^/g; # set pos()
+    my $byte_replaced = 0;
+    while ($highbytes =~
+        m!\G (?: [\xc0-\xdf][\x80-\xbf]    | # 2 byte utf-8
+            [\xe0-\xef][\x80-\xbf]{2} | # 3 byte
+            [\xf0-\xf7][\x80-\xbf]{3}   # 4 byte
+            [\xf8-\xfb][\x80-\xbf]{4}   # 5 byte
+            [\xfc-\xfd][\x80-\xbf]{5}   # 6 byte
+            )*([\x80-\xff])? !xg
+        ) {
+        # this highbyte is "out-of-place" for valid utf-8
+        my $badbyte=$1;
+        if (!defined $badbyte) {next} # hit end of string
+        my $pos=pos($highbytes);
+        # replace bad byte. assume iso-8859-1 -> utf-8
+        # ascii2utf8 does "extended ascii"... ie iso-8859-1
+        my $replacement=&unicode::ascii2utf8(\$badbyte);
+        substr($highbytes, $pos-1, 1, $replacement);
+        # update the position to continue searching (for \G)
+        pos($highbytes) = $pos+length($replacement)-1;
+        $byte_replaced = 1;
+    }
+    if ($byte_replaced) {
+        # replace this block of high bytes in the $value
+        $non_utf8_found = 1;
+        my $replength=length($highbytes); # we've changed the length
+        my $textpos=pos($value); # pos at end of last match
+        # replace bad bytes with good bytes
+        substr($value, $textpos-$highbyteslength,
+                $highbyteslength, $highbytes);
+        # update the position to continue searching (for \G)
+        pos($value)=$textpos+($replength-$highbyteslength)+1;
+    }
+    }
+
+    $stringref = \$value;
+    return $non_utf8_found;
+}
+
 1;