Changeset 1279 for branches/New_Config_Format-branch/gsdl/perllib/unicode.pm

Timestamp:

2000-07-13T10:21:53+12:00 (24 years ago)

Author:

sjboddie

Message:

merged changes to trunk into New_Config_Format branch

File:

• branches/New_Config_Format-branch/gsdl/perllib/unicode.pm (modified) (3 diffs)

branches/New_Config_Format-branch/gsdl/perllib/unicode.pm

@@ -31 +31 @@
 package unicode;
 
-
-
-# ascii2unicode takes a (extended) ascii string and
-# returns a unicode array.
+%translations = ();
+
+# ascii2unicode takes an (extended) ascii string (ISO-8859-1)
+# and returns a unicode array.
 sub ascii2unicode {
     my ($in) = @_;
@@ -43 +43 @@
     while ($i < $len) {
     push (@$out, ord(substr ($in, $i, 1))); 
+    $i++;
+    }
+
+    return $out;
+}
+
+# arabic2unicode takes an 8 bit Arabic string (ISO-8859-6)
+# and returns a unicode array
+sub arabic2unicode {
+    my ($in) = @_;
+    my $out = [];
+
+    my $i = 0;
+    my $len = length($in);
+    while ($i < $len) {
+    my $c = ord(substr ($in, $i, 1));
+    $c += (1567-191) if ($c >= 0x80);
+    push (@$out, $c);
+    $i++;
+    }
+
+    return $out;
+}
+
+# windows2unicode takes a windows encoding (e.g. Windows 1256 (Arabic))
+# and returns a unicode array. These encodings are similar to but not
+# identical to the corresponding ISO-8859 encodings.
+#
+# The map files for these encodings should be in unicode/MAPPINGS/WINDOWS
+sub windows2unicode {
+    my ($encoding, $in) = @_;
+    my $out = [];
+
+    my $mapfile = &util::filename_cat($ENV{'GSDLHOME'}, "unicode", "MAPPINGS", 
+                      "WINDOWS", "$encoding.TXT"); 
+    return $out unless &loadmapping ($encoding, $mapfile);
+
+    my $i = 0;
+    my $len = length($in);
+    while ($i < $len) {
+    my $c = ord(substr ($in, $i, 1));
+    $c = $translations{"$encoding-unicode"}->{$c} if ($c >= 0x80);
+    push (@$out, $c);
     $i++;
     }
@@ -193 +236 @@
 }
 
+# loadmapping expects the mapfile to contain (at least) two
+# tab-separated fields. The first field is the mapped value
+# and the second field is the unicode value.
+#
+# It returns 1 if successful, 0 if unsuccessful
+sub loadmapping {
+    my ($encoding, $mapfile) = @_;
+    
+    my $to = "$encoding-unicode";
+    my $from = "unicode-$encoding";
+
+    # check to see if the encoding has already been loaded
+    if (defined $translations{$to} && defined $translations{$from}) {
+    return 1;
+    }
+
+    if (!open (MAPFILE, $mapfile)) {
+    print STDERR "ERROR: unable to load mapfile $mapfile\n";
+    return 0;
+    }
+
+    my ($line, @line);
+    $translations{$to} = {};
+    $translations{$from} = {};
+    while (defined ($line = <MAPFILE>)) {
+    # remove comments
+    $line =~ s/\#.*$//;
+    next unless $line =~ /\S/;
+
+    # split the line into fields and do a few
+    # simple sanity checks
+    @line = split (/\t/, $line);
+    next unless (scalar(@line) >= 2 &&
+             $line[0] =~ /^0x/ &&
+             $line[1] =~ /^0x/);
+
+    my $a = hex($line[0]);
+    my $b = hex($line[1]);
+
+    $translations{$to}->{$a} = $b;
+    $translations{$from}->{$b} = $a;
+    }
+
+    close (MAPFILE);
+
+    return 1;
+}
+
 
 1;
-