Index: gsdl/trunk/perllib/plugins/BasePlugin.pm
===================================================================
--- gsdl/trunk/perllib/plugins/BasePlugin.pm	(revision 16520)
+++ gsdl/trunk/perllib/plugins/BasePlugin.pm	(revision 16557)
@@ -35,4 +35,5 @@
 use encodings;
 use unicode;
+use textcat;
 use doc;
 eval "require diagnostics"; # some perl distros (eg mac) don't have this
@@ -53,4 +54,5 @@
       { 'name' => "unicode",
 	'desc' => "{BasePlugin.encoding.unicode}" } ];
+      
 
 my $e = $encodings::encodings;
@@ -66,5 +68,13 @@
 our $encoding_plus_auto_list = 
     [ { 'name' => "auto",
-	'desc' => "{BasePlugin.filename_encoding.auto}" } ];
+	'desc' => "{BasePlugin.filename_encoding.auto}" },
+       { 'name' => "auto-language-analysis",
+	'desc' => "{BasePlugin.filename_encoding.auto_language_analysis}" }, # textcat
+      { 'name' => "auto-filesystem-encoding",
+	'desc' => "{BasePlugin.filename_encoding.auto_filesystem_encoding}" }, # locale
+      { 'name' => "auto-fl",
+	'desc' => "{BasePlugin.filename_encoding.auto_fl}" }, # locale followed by textcat
+      { 'name' => "auto-lf",
+	'desc' => "{BasePlugin.filename_encoding.auto_lf}" } ]; # texcat followed by locale 
 
 push(@{$encoding_plus_auto_list},@{$encoding_list});
@@ -376,30 +386,108 @@
     my $filemeta = $file;
 
-    my $filename_encoding = $self->{'filename_encoding'};
-    if ($filename_encoding eq "auto") {
-	# we check the locale first
-	if (!defined $self->{'filesystem_encoding'}) {
-	    $self->{'filesystem_encoding'} = $self->get_filesystem_encoding();
-	    $self->{'filesystem_encoding'} = "undefined" if !defined $self->{'filesystem_encoding'};
-	}
-	if ($self->{'filesystem_encoding'} ne "undefined") {
-	    $filename_encoding = $self->{'filesystem_encoding'};
-	} else {
-	    # try the encoding of the document, if available
-	    if (defined $file_encoding) {
-		$filename_encoding = $file_encoding;
-	    } else {
-		# use utf8
-		$filename_encoding = "utf8";
-	    }
-	}
+    my $filename_encoding = $self->{'filename_encoding'}; # filename encoding setting
+
+##  print STDERR "**** User chose filename encoding setting: $filename_encoding\n";
+    
+    # Whenever filename-encoding is set to any of the auto settings, we
+    # check if the filename is already in UTF8. If it is, then we're done.
+    if($filename_encoding =~ m/auto/) {
+	if(&unicode::check_is_utf8($filemeta)) 
+	{
+##	    print STDERR "**** It is already UTF8\n";
+	    $filename_encoding = "utf8";
+	    return $filemeta;
+	} 
+    }
+    
+    # Auto setting, but filename is not utf8
+    if ($filename_encoding eq "auto") 
+    {
+	# try textcat
+	$filename_encoding = $self->textcat_encoding($filemeta);
 	
-    }
-
-    if ($filename_encoding !~ /(?:ascii|utf8|unicode)/) {
-	$filemeta = unicode::unicode2utf8(
-	  unicode::convert2unicode($filename_encoding, \$filemeta)
+	# check the locale next
+	$filename_encoding = $self->locale_encoding() if $filename_encoding eq "undefined";
+	
+
+	# now try the encoding of the document, if available
+	if ($filename_encoding eq "undefined" && defined $file_encoding) {
+	    $filename_encoding = $file_encoding;
+	}
+
+    }
+
+    elsif ($filename_encoding eq "auto-language-analysis") 
+    {	
+	$filename_encoding = $self->textcat_encoding($filemeta);
+
+	# now try the encoding of the document, if available
+	if ($filename_encoding eq "undefined" && defined $file_encoding) {
+	    $filename_encoding = $file_encoding;
+	} 
+    }
+
+    elsif ($filename_encoding eq "auto-filesystem-encoding") 
+    {
+	# try locale
+	$filename_encoding = $self->locale_encoding();
+    }
+
+    elsif ($filename_encoding eq "auto-fl") 
+    {
+	# filesystem-encoding (locale) then language-analysis (textcat)
+	$filename_encoding = $self->locale_encoding();
+	
+	# try textcat
+	$filename_encoding = $self->textcat_encoding($filemeta) if $filename_encoding eq "undefined";
+	    
+	# else assume filename encoding is encoding of file content, if that's available
+	if ($filename_encoding eq "undefined" && defined $file_encoding) {
+	    $filename_encoding = $file_encoding;
+	}
+    }
+	
+    elsif ($filename_encoding eq "auto-lf") 
+    {
+	# language-analysis (textcat) then filesystem-encoding (locale)
+	$filename_encoding = $self->textcat_encoding($filemeta);
+	
+	# guess filename encoding from encoding of file content, if available
+	if ($filename_encoding eq "undefined" && defined $file_encoding) {
+	    $filename_encoding = $file_encoding;
+	}
+
+	# try locale
+	$filename_encoding = $self->locale_encoding() if $filename_encoding eq "undefined";
+    }
+
+##    print STDERR "**** filename_encoding selected: $filename_encoding \n";
+    	
+    # if still undefined, use utf8 as fallback
+    if ($filename_encoding eq "undefined") {
+	$filename_encoding = "utf8";
+    }
+
+    # if the filename encoding is set to utf8 but it isn't utf8 already--such as when
+    # 1. the utf8 fallback is used, or 2. if the system locale is used and happens to
+    # be always utf8 (in which case the filename's encoding is also set as utf8 even 
+    # though the filename need not be if it originates from another system)--in such
+    # cases attempt to make the filename utf8 to match.
+    if($filename_encoding eq "utf8" && !&unicode::check_is_utf8($filemeta)) {
+##	print STDERR "**** BEFORE utf8 conversion: $filemeta\n";
+	&unicode::ensure_utf8(\$filemeta);
+##	print STDERR "**** AFTER utf8 conversion: $filemeta\n";
+    }
+
+
+    # convert non-unicode encodings to utf8
+    if ($filename_encoding !~ m/(?:ascii|utf8|unicode)/) {
+	$filemeta = &unicode::unicode2utf8(
+	  &unicode::convert2unicode($filename_encoding, \$filemeta)
 	);
     }
+
+    print "*** filename encoding found: $filename_encoding\n";
+    print "*** utf8 encoded filename: $filemeta\n";
 
     return $filemeta;
@@ -424,5 +512,114 @@
 }
 
-
+sub locale_encoding {
+    my $self = shift(@_);
+    
+    if (!defined $self->{'filesystem_encoding'}) {
+	$self->{'filesystem_encoding'} = $self->get_filesystem_encoding();
+	$self->{'filesystem_encoding'} = "undefined" if !defined $self->{'filesystem_encoding'};
+    }
+
+    print "filename encoding determined based on locale: " . $self->{'filesystem_encoding'} . "\n";
+    return $self->{'filesystem_encoding'}; # can be the string "undefined"
+}
+
+sub textcat_encoding {
+    my $self = shift(@_);
+    my ($filemeta) = @_;
+
+    # analyse filenames without extensions and digits (and trimmed of surrounding 
+    # whitespace), so that irrelevant chars don't confuse textcat
+    my $strictfilemeta = $filemeta;
+    $strictfilemeta =~ s/\.[^\.]+$//g;
+    $strictfilemeta =~ s/\d//g;
+    $strictfilemeta =~ s/^\s*//g;
+    $strictfilemeta =~ s/\s*$//g;
+    
+##    print STDERR "**** strict filename is |$strictfilemeta|\n";
+    my $filename_encoding = $self->encoding_from_language_analysis($strictfilemeta);
+    if(!defined $filename_encoding) {
+	$filename_encoding = "undefined";
+    }
+
+##    print STDERR "**** textcat found filename encoding: " . $file_textcat_encoding_map{$strictfilemeta} . "\n";
+    return $filename_encoding; # can be the string "undefined"
+}
+
+# performs textcat
+sub encoding_from_language_analysis {
+    my $self = shift(@_);
+    my ($text) = @_;
+
+    my $outhandle = $self->{'outhandle'};
+    my $best_encoding = undef;
+   
+    # get the language/encoding of the file using textcat
+    $self->{'textcat'} = new textcat() unless defined($self->{'textcat'});
+    #my $results = $self->{'textcat'}->classify(\$text);
+    my $results = $self->{'textcat'}->classify_cached(\$text);
+
+
+    if (scalar @$results < 0) { 
+	print STDERR "**** Textcat returned 0 results\n";
+	return undef;
+    }
+    
+    print STDERR "**** TEXTCAT RESULTS for $text: ";
+    print STDERR join(",", @$results);
+    print STDERR "\n";
+
+    # We have some results, we choose the first
+    my ($language, $encoding) = $results->[0] =~ /^([^-]*)(?:-(.*))?$/;
+    
+    $best_encoding = $encoding;
+    if (!defined $best_encoding) {
+##	print STDERR "**** Textcat cannot determine encoding of filename: it's undefined.\n";
+	return undef;
+    } 
+        
+    if (defined $best_encoding && $best_encoding =~ m/^iso_8859/ && &unicode::check_is_utf8($text)) {
+	# the text is valid utf8, so assume that's the real encoding (since textcat is based on probabilities)
+##	print STDERR "*** Filename turns out to be UTF8\n";
+	$best_encoding = 'utf8';
+    }
+    
+    
+    # check for equivalents where textcat doesn't have some encodings...
+    # eg MS versions of standard encodings
+    if (defined $best_encoding && $best_encoding =~ /^iso_8859_(\d+)/) {
+##	print STDERR "**** best_encoding is ISO_8859: $best_encoding\n";
+
+	my $iso = $1; # which variant of the iso standard?
+	# iso-8859 sets don't use chars 0x80-0x9f, windows codepages do
+	if ($text =~ /[\x80-\x9f]/) {
+##	    print STDERR "**** best_encoding is some windows value: $best_encoding\n";
+	    # Western Europe
+	    if ($iso == 1 or $iso == 15) { $best_encoding = 'windows_1252' }
+	    elsif ($iso == 2) {$best_encoding = 'windows_1250'} # Central Europe
+	    elsif ($iso == 5) {$best_encoding = 'windows_1251'} # Cyrillic
+	    elsif ($iso == 6) {$best_encoding = 'windows_1256'} # Arabic
+	    elsif ($iso == 7) {$best_encoding = 'windows_1253'} # Greek
+	    elsif ($iso == 8) {$best_encoding = 'windows_1255'} # Hebrew
+	    elsif ($iso == 9) {$best_encoding = 'windows_1254'} # Turkish
+##	    print STDERR "**** best_encoding windows value: $best_encoding\n";
+	}
+    }
+    
+    if (defined $best_encoding && $best_encoding !~ /^(ascii|utf8|unicode)$/ &&
+	!defined $encodings::encodings->{$best_encoding}) 
+    {
+	if ($self->{'verbosity'}) { 
+	    gsprintf($outhandle, "BasePlugin: {ReadTextFile.unsupported_encoding}\n", $text, $best_encoding, "undef");
+	}
+##	print STDERR "***** unsupported encoding: $best_encoding. Setting it to undefined.\n";	
+	$best_encoding = undef;
+    }
+##    print STDERR "**** language: $language\n" if defined $language;
+##    print STDERR "**** encoding: $best_encoding\n" if defined $encoding;
+    
+    return $best_encoding;
+}
+
+# uses locale
 sub get_filesystem_encoding {
 
