Changeset 23387 for main/trunk/greenstone2/perllib/unicode.pm

Timestamp:

2010-12-06T13:15:10+13:00 (13 years ago)

Author:

davidb

Message:

Further changes to deal with documents that use different filename encodings on the file-system. Now sets UTF8URL metadata to perform the cross-document look up. Files stored in doc.pm as associated files are now always raw filenames (rather than potentially UTF8 encoded). Storing of filenames seen by HTMLPlug when scanning for files to block on is now done in Unicode aware strings rather than utf8 but unware strings.

File:

• main/trunk/greenstone2/perllib/unicode.pm (modified) (3 diffs)

main/trunk/greenstone2/perllib/unicode.pm

@@ -627 +627 @@
 
 sub url_decode {
-    my ($text) = @_;
+    my ($text,$and_numeric_entities) = @_;
 
     $text =~ s/\%([0-9A-F]{2})/pack('C', hex($1))/ige;
-    $text =~ s/\&\#x([0-9A-F]+);/pack('C', hex($1))/ige;
-    $text =~ s/\&\#([0-9]+);/pack('C', $1)/ige;
+
+    if ((defined $and_numeric_entities) && ($and_numeric_entities)) {
+    $text =~ s/\&\#x([0-9A-F]+);/pack('C', hex($1))/ige;
+    $text =~ s/\&\#([0-9]+);/pack('C', $1)/ige;
+    }
 
     return $text;
@@ -773 +776 @@
 }
 
-
 sub url_encoded_to_raw_filename
 {
@@ -787 +789 @@
 }
 
+
+sub raw_filename_to_utf8_url_encoded
+{
+    my ($str_in) = @_;
+
+    $str_in = Encode::encode("utf8",$str_in) if !check_is_utf8($str_in);
+
+    my @url_encoded_chars
+    = map { $_ > 128 ?                  # Representable in %XX form
+            sprintf("%%%2X", $_) :  
+            chr($_)                 # otherwise, Ascii char
+        } unpack("U*", $str_in); # Unpack utf8 characters
+
+    
+    my $str_out = join("", @url_encoded_chars);
+
+    return $str_out;
+
+}
+
+sub utf8_url_encoded_to_raw_filename
+{
+    my ($str_in) = @_;
+
+    my $utf8_str_out = $str_in;
+
+    $utf8_str_out =~ s/%([0-9A-F]{2})/chr(hex($1))/eig;
+
+    my $unicode_str_out = decode("utf8",$utf8_str_out);
+    my $raw_str_out = utf8::downgrade($unicode_str_out);
+    
+    return $raw_str_out;
+}
+
+sub analyze_raw_string
+{
+    my ($str_in) = @_;
+
+    my $uses_bytecodes = 0;
+    my $exceeds_bytecodes = 0;
+
+    map { $exceeds_bytecodes = 1 if ($_ >= 256);
+      $uses_bytecodes    = 1 if (($_ >= 128) && ($_ < 256));
+    } unpack("U*", $str_in); # Unpack Unicode characters
+
+    return ($uses_bytecodes,$exceeds_bytecodes);
+}
+
+
 1;