Index: trunk/gsdl/perllib/multiread.pm
===================================================================
--- trunk/gsdl/perllib/multiread.pm	(revision 1225)
+++ trunk/gsdl/perllib/multiread.pm	(revision 1227)
@@ -26,9 +26,10 @@
 # encodings currently supported are
 #
-# utf8     - either utf8 or unicode (automatically detected)
-# unicode  - just unicode (doesn't currently do endian detection)
-# gb       - GB
-# latin1   - extended ascii (iso-8859-1)
-# arabic   - 8 bit arabic (iso-8859-6)
+# utf8         - either utf8 or unicode (automatically detected)
+# unicode      - just unicode (doesn't currently do endian detection)
+# gb           - GB
+# iso_8859_1   - extended ascii (iso-8859-1)
+# iso_8859_6   - 8 bit arabic (iso-8859-6)
+# windows_1256 - Windows codepage 1256 (Arabic)
 
 package multiread;
@@ -169,5 +170,5 @@
     }
 
-    if ($self->{'encoding'} eq "latin1") {
+    if ($self->{'encoding'} eq "iso_8859_1") {
 	# Latin 1 extended ascii (ISO-8859-1)
 	return undef if (eof ($handle));
@@ -175,8 +176,14 @@
     }
 
-    if ($self->{'encoding'} eq "arabic") {
+    if ($self->{'encoding'} eq "iso_8859_6") {
 	# 8 bit Arabic (IOS-8859-6)
 	return undef if (eof ($handle));
 	return &unicode::unicode2utf8(&unicode::arabic2unicode (getc ($handle)));
+    }
+
+    if ($self->{'encoding'} eq "windows_1256") {
+	# Windows 1256 (Arabic)
+	return undef if (eof ($handle));
+	return &unicode::unicode2utf8(&unicode::windows2unicode ("1256", getc ($handle)));
     }
 
@@ -242,5 +249,5 @@
     }
     
-    if ($self->{'encoding'} eq "latin1") {
+    if ($self->{'encoding'} eq "iso_8859_1") {
 	# extended ascii (ISO-8859-1)
 	my $line = "";
@@ -251,5 +258,5 @@
     }
     
-    if ($self->{'encoding'} eq "arabic") {
+    if ($self->{'encoding'} eq "iso_8859_6") {
 	# 8 bit arabic (ISO-8859-6)
 	my $line = "";
@@ -259,4 +266,13 @@
 	return undef;
     }
+    
+    if ($self->{'encoding'} eq "windows_1256") {
+	# Windows 1256 (Arabic)
+	my $line = "";
+	if (defined ($line = <$handle>)) {
+	    return &unicode::unicode2utf8(&unicode::windows2unicode ("1256", $line));
+	}
+	return undef;
+    }
 
     # unknown encoding
@@ -266,4 +282,6 @@
 
 # will convert entire contents of file to utf8 and append result to $outputref
+# this may be a slightly faster way to get the contents of a file than by 
+# recursively calling read_line()
 sub read_file {
     my $self = shift (@_);
@@ -298,20 +316,31 @@
     if ($self->{'encoding'} eq "gb") {
 	undef $/;
-	$$outputref .= &unicode::unicode2utf8 (&gb::gb2unicode (<$handle>));
-	$/ = "\n";
-	return;
-    }
-    
-    if ($self->{'encoding'} eq "latin1") {
-	undef $/;
-	$$outputref .= &unicode::ascii2utf8 (<$handle>);
-	$/ = "\n";
-	return;
-    }
-    
-    if ($self->{'encoding'} eq "arabic") {
-	undef $/;
-	$$outputref .= &unicode::unicode2utf8(&unicode::arabic2unicode (<$handle>));
-	$/ = "\n";
+	my $text = <$handle>;
+	$/ = "\n";
+	$$outputref .= &unicode::unicode2utf8 (&gb::gb2unicode ($text));
+	return;
+    }
+    
+    if ($self->{'encoding'} eq "iso_8859_1") {
+	undef $/;
+	my $text = <$handle>;
+	$/ = "\n";
+	$$outputref .= &unicode::ascii2utf8 ($text);
+	return;
+    }
+    
+    if ($self->{'encoding'} eq "iso_8859_6") {
+	my $text = <$handle>;
+	undef $/;
+	$/ = "\n";
+	$$outputref .= &unicode::unicode2utf8(&unicode::arabic2unicode ($text));
+	return;
+    }
+
+    if ($self->{'encoding'} eq "windows_1256") {
+	undef $/;
+	my $text = <$handle>;
+	$/ = "\n";
+	$$outputref .= &unicode::unicode2utf8(&unicode::windows2unicode ("1256", $text));
 	return;
     }
Index: trunk/gsdl/perllib/plugins/BasPlug.pm
===================================================================
--- trunk/gsdl/perllib/plugins/BasPlug.pm	(revision 1225)
+++ trunk/gsdl/perllib/plugins/BasPlug.pm	(revision 1227)
@@ -47,5 +47,6 @@
     print STDERR "                      gb (GB or GBK simplified Chinese)\n";
     print STDERR "                      iso_8859_6 (8 bit Arabic)\n";
-    print STDERR "                      Arabic (the same as iso-8859-6)\n";
+    print STDERR "                      windows_1256 (Windows codepage 1256 (Arabic))\n";
+    print STDERR "                      Arabic (the same as windows_1256)\n";
     print STDERR "                      utf8 (either utf8 or unicode -- automatically detected)\n";
     print STDERR "                      unicode (just unicode -- doesn't currently do endian\n";
@@ -57,5 +58,5 @@
 
     my $self = {};
-    my $encodings = "^(iso_8859_1|Latin1|ascii|gb|iso_8859_6|Arabic|utf8|unicode)\$";
+    my $encodings = "^(iso_8859_1|Latin1|ascii|gb|iso_8859_6|windows_1256|Arabic|utf8|unicode)\$";
 
     # general options available to all plugins
@@ -109,6 +110,6 @@
     if ($self->{'input_encoding'} =~ /^(Latin1|iso_8859_1)$/) {
 	$encoding = "latin1";
-    } elsif ($self->{'input_encoding'} =~ /^(Arabic|iso_8859_6)$/) {
-	$encoding = "arabic";
+    } elsif ($self->{'input_encoding'} =~ /^(Arabic|windows_1256)$/) {
+	$encoding = "windows_1256";
     } else {
 	$encoding = $self->{'input_encoding'};
Index: trunk/gsdl/perllib/unicode.pm
===================================================================
--- trunk/gsdl/perllib/unicode.pm	(revision 1225)
+++ trunk/gsdl/perllib/unicode.pm	(revision 1227)
@@ -31,5 +31,5 @@
 package unicode;
 
-
+%translations = ();
 
 # ascii2unicode takes an (extended) ascii string (ISO-8859-1)
@@ -66,4 +66,30 @@
     return $out;
 }
+
+# windows2unicode takes a windows encoding (e.g. Windows 1256 (Arabic))
+# and returns a unicode array. These encodings are similar to but not
+# identical to the corresponding ISO-8859 encodings.
+#
+# The map files for these encodings should be in unicode/MAPPINGS/WINDOWS
+sub windows2unicode {
+    my ($encoding, $in) = @_;
+    my $out = [];
+
+    my $mapfile = &util::filename_cat($ENV{'GSDLHOME'}, "unicode", "MAPPINGS", 
+				      "WINDOWS", "$encoding.TXT"); 
+    return $out unless &loadmapping ($encoding, $mapfile);
+
+    my $i = 0;
+    my $len = length($in);
+    while ($i < $len) {
+	my $c = ord(substr ($in, $i, 1));
+	$c = $translations{"$encoding-unicode"}->{$c} if ($c >= 0x80);
+	push (@$out, $c);
+	$i++;
+    }
+
+    return $out;
+}
+
 
 # ascii2utf8 takes a (extended) ascii string and
@@ -210,5 +236,52 @@
 }
 
+# loadmapping expects the mapfile to contain (at least) two
+# tab-separated fields. The first field is the mapped value
+# and the second field is the unicode value.
+#
+# It returns 1 if successful, 0 if unsuccessful
+sub loadmapping {
+    my ($encoding, $mapfile) = @_;
+    
+    my $to = "$encoding-unicode";
+    my $from = "unicode-$encoding";
+
+    # check to see if the encoding has already been loaded
+    if (defined $translations{$to} && defined $translations{$from}) {
+	return 1;
+    }
+
+    if (!open (MAPFILE, $mapfile)) {
+	print STDERR "ERROR: unable to load mapfile $mapfile\n";
+	return 0;
+    }
+
+    my ($line, @line);
+    $translations{$to} = {};
+    $translations{$from} = {};
+    while (defined ($line = <MAPFILE>)) {
+	# remove comments
+	$line =~ s/\#.*$//;
+	next unless $line =~ /\S/;
+
+	# split the line into fields and do a few
+	# simple sanity checks
+	@line = split (/\t/, $line);
+	next unless (scalar(@line) >= 2 &&
+		     $line[0] =~ /^0x/ &&
+		     $line[1] =~ /^0x/);
+
+	my $a = hex($line[0]);
+	my $b = hex($line[1]);
+
+	$translations{$to}->{$a} = $b;
+	$translations{$from}->{$b} = $a;
+    }
+
+    close (MAPFILE);
+
+    return 1;
+}
+
 
 1;
-
