Changeset 16557

Timestamp:

2008-07-25T18:31:50+12:00 (16 years ago)

Author:

ak19

Message:

Auto filename encoding has several additional settings now, these are handled by subroutine filepath_to_utf8 which has changed accordingly. Some additional helper subroutines added. This file BasePlugin.pm is an intermediate but working version (still has many debug output statements even when most are commented out, but as I want to test the changes out on Windows first, I want to retain the debug statements).

File:

• gsdl/trunk/perllib/plugins/BasePlugin.pm (modified) (5 diffs)

gsdl/trunk/perllib/plugins/BasePlugin.pm

@@ -35 +35 @@
 use encodings;
 use unicode;
+use textcat;
 use doc;
 eval "require diagnostics"; # some perl distros (eg mac) don't have this
@@ -53 +54 @@
       { 'name' => "unicode",
     'desc' => "{BasePlugin.encoding.unicode}" } ];
+      
 
 my $e = $encodings::encodings;
@@ -66 +68 @@
 our $encoding_plus_auto_list = 
     [ { 'name' => "auto",
-    'desc' => "{BasePlugin.filename_encoding.auto}" } ];
+    'desc' => "{BasePlugin.filename_encoding.auto}" },
+       { 'name' => "auto-language-analysis",
+    'desc' => "{BasePlugin.filename_encoding.auto_language_analysis}" }, # textcat
+      { 'name' => "auto-filesystem-encoding",
+    'desc' => "{BasePlugin.filename_encoding.auto_filesystem_encoding}" }, # locale
+      { 'name' => "auto-fl",
+    'desc' => "{BasePlugin.filename_encoding.auto_fl}" }, # locale followed by textcat
+      { 'name' => "auto-lf",
+    'desc' => "{BasePlugin.filename_encoding.auto_lf}" } ]; # texcat followed by locale 
 
 push(@{$encoding_plus_auto_list},@{$encoding_list});
@@ -376 +386 @@
     my $filemeta = $file;
 
-    my $filename_encoding = $self->{'filename_encoding'};
-    if ($filename_encoding eq "auto") {
-    # we check the locale first
-    if (!defined $self->{'filesystem_encoding'}) {
-        $self->{'filesystem_encoding'} = $self->get_filesystem_encoding();
-        $self->{'filesystem_encoding'} = "undefined" if !defined $self->{'filesystem_encoding'};
-    }
-    if ($self->{'filesystem_encoding'} ne "undefined") {
-        $filename_encoding = $self->{'filesystem_encoding'};
-    } else {
-        # try the encoding of the document, if available
-        if (defined $file_encoding) {
-        $filename_encoding = $file_encoding;
-        } else {
-        # use utf8
-        $filename_encoding = "utf8";
-        }
-    }
+    my $filename_encoding = $self->{'filename_encoding'}; # filename encoding setting
+
+##  print STDERR "**** User chose filename encoding setting: $filename_encoding\n";
+    
+    # Whenever filename-encoding is set to any of the auto settings, we
+    # check if the filename is already in UTF8. If it is, then we're done.
+    if($filename_encoding =~ m/auto/) {
+    if(&unicode::check_is_utf8($filemeta)) 
+    {
+##      print STDERR "**** It is already UTF8\n";
+        $filename_encoding = "utf8";
+        return $filemeta;
+    } 
+    }
+    
+    # Auto setting, but filename is not utf8
+    if ($filename_encoding eq "auto") 
+    {
+    # try textcat
+    $filename_encoding = $self->textcat_encoding($filemeta);
     
-    }
-
-    if ($filename_encoding !~ /(?:ascii|utf8|unicode)/) {
-    $filemeta = unicode::unicode2utf8(
-      unicode::convert2unicode($filename_encoding, \$filemeta)
+    # check the locale next
+    $filename_encoding = $self->locale_encoding() if $filename_encoding eq "undefined";
+    
+
+    # now try the encoding of the document, if available
+    if ($filename_encoding eq "undefined" && defined $file_encoding) {
+        $filename_encoding = $file_encoding;
+    }
+
+    }
+
+    elsif ($filename_encoding eq "auto-language-analysis") 
+    {   
+    $filename_encoding = $self->textcat_encoding($filemeta);
+
+    # now try the encoding of the document, if available
+    if ($filename_encoding eq "undefined" && defined $file_encoding) {
+        $filename_encoding = $file_encoding;
+    } 
+    }
+
+    elsif ($filename_encoding eq "auto-filesystem-encoding") 
+    {
+    # try locale
+    $filename_encoding = $self->locale_encoding();
+    }
+
+    elsif ($filename_encoding eq "auto-fl") 
+    {
+    # filesystem-encoding (locale) then language-analysis (textcat)
+    $filename_encoding = $self->locale_encoding();
+    
+    # try textcat
+    $filename_encoding = $self->textcat_encoding($filemeta) if $filename_encoding eq "undefined";
+        
+    # else assume filename encoding is encoding of file content, if that's available
+    if ($filename_encoding eq "undefined" && defined $file_encoding) {
+        $filename_encoding = $file_encoding;
+    }
+    }
+    
+    elsif ($filename_encoding eq "auto-lf") 
+    {
+    # language-analysis (textcat) then filesystem-encoding (locale)
+    $filename_encoding = $self->textcat_encoding($filemeta);
+    
+    # guess filename encoding from encoding of file content, if available
+    if ($filename_encoding eq "undefined" && defined $file_encoding) {
+        $filename_encoding = $file_encoding;
+    }
+
+    # try locale
+    $filename_encoding = $self->locale_encoding() if $filename_encoding eq "undefined";
+    }
+
+##    print STDERR "**** filename_encoding selected: $filename_encoding \n";
+        
+    # if still undefined, use utf8 as fallback
+    if ($filename_encoding eq "undefined") {
+    $filename_encoding = "utf8";
+    }
+
+    # if the filename encoding is set to utf8 but it isn't utf8 already--such as when
+    # 1. the utf8 fallback is used, or 2. if the system locale is used and happens to
+    # be always utf8 (in which case the filename's encoding is also set as utf8 even 
+    # though the filename need not be if it originates from another system)--in such
+    # cases attempt to make the filename utf8 to match.
+    if($filename_encoding eq "utf8" && !&unicode::check_is_utf8($filemeta)) {
+##  print STDERR "**** BEFORE utf8 conversion: $filemeta\n";
+    &unicode::ensure_utf8(\$filemeta);
+##  print STDERR "**** AFTER utf8 conversion: $filemeta\n";
+    }
+
+
+    # convert non-unicode encodings to utf8
+    if ($filename_encoding !~ m/(?:ascii|utf8|unicode)/) {
+    $filemeta = &unicode::unicode2utf8(
+      &unicode::convert2unicode($filename_encoding, \$filemeta)
     );
     }
+
+    print "*** filename encoding found: $filename_encoding\n";
+    print "*** utf8 encoded filename: $filemeta\n";
 
     return $filemeta;
@@ -424 +512 @@
 }
 
-
+sub locale_encoding {
+    my $self = shift(@_);
+    
+    if (!defined $self->{'filesystem_encoding'}) {
+    $self->{'filesystem_encoding'} = $self->get_filesystem_encoding();
+    $self->{'filesystem_encoding'} = "undefined" if !defined $self->{'filesystem_encoding'};
+    }
+
+    print "filename encoding determined based on locale: " . $self->{'filesystem_encoding'} . "\n";
+    return $self->{'filesystem_encoding'}; # can be the string "undefined"
+}
+
+sub textcat_encoding {
+    my $self = shift(@_);
+    my ($filemeta) = @_;
+
+    # analyse filenames without extensions and digits (and trimmed of surrounding 
+    # whitespace), so that irrelevant chars don't confuse textcat
+    my $strictfilemeta = $filemeta;
+    $strictfilemeta =~ s/\.[^\.]+$//g;
+    $strictfilemeta =~ s/\d//g;
+    $strictfilemeta =~ s/^\s*//g;
+    $strictfilemeta =~ s/\s*$//g;
+    
+##    print STDERR "**** strict filename is |$strictfilemeta|\n";
+    my $filename_encoding = $self->encoding_from_language_analysis($strictfilemeta);
+    if(!defined $filename_encoding) {
+    $filename_encoding = "undefined";
+    }
+
+##    print STDERR "**** textcat found filename encoding: " . $file_textcat_encoding_map{$strictfilemeta} . "\n";
+    return $filename_encoding; # can be the string "undefined"
+}
+
+# performs textcat
+sub encoding_from_language_analysis {
+    my $self = shift(@_);
+    my ($text) = @_;
+
+    my $outhandle = $self->{'outhandle'};
+    my $best_encoding = undef;
+   
+    # get the language/encoding of the file using textcat
+    $self->{'textcat'} = new textcat() unless defined($self->{'textcat'});
+    #my $results = $self->{'textcat'}->classify(\$text);
+    my $results = $self->{'textcat'}->classify_cached(\$text);
+
+
+    if (scalar @$results < 0) { 
+    print STDERR "**** Textcat returned 0 results\n";
+    return undef;
+    }
+    
+    print STDERR "**** TEXTCAT RESULTS for $text: ";
+    print STDERR join(",", @$results);
+    print STDERR "\n";
+
+    # We have some results, we choose the first
+    my ($language, $encoding) = $results->[0] =~ /^([^-]*)(?:-(.*))?$/;
+    
+    $best_encoding = $encoding;
+    if (!defined $best_encoding) {
+##  print STDERR "**** Textcat cannot determine encoding of filename: it's undefined.\n";
+    return undef;
+    } 
+        
+    if (defined $best_encoding && $best_encoding =~ m/^iso_8859/ && &unicode::check_is_utf8($text)) {
+    # the text is valid utf8, so assume that's the real encoding (since textcat is based on probabilities)
+##  print STDERR "*** Filename turns out to be UTF8\n";
+    $best_encoding = 'utf8';
+    }
+    
+    
+    # check for equivalents where textcat doesn't have some encodings...
+    # eg MS versions of standard encodings
+    if (defined $best_encoding && $best_encoding =~ /^iso_8859_(\d+)/) {
+##  print STDERR "**** best_encoding is ISO_8859: $best_encoding\n";
+
+    my $iso = $1; # which variant of the iso standard?
+    # iso-8859 sets don't use chars 0x80-0x9f, windows codepages do
+    if ($text =~ /[\x80-\x9f]/) {
+##      print STDERR "**** best_encoding is some windows value: $best_encoding\n";
+        # Western Europe
+        if ($iso == 1 or $iso == 15) { $best_encoding = 'windows_1252' }
+        elsif ($iso == 2) {$best_encoding = 'windows_1250'} # Central Europe
+        elsif ($iso == 5) {$best_encoding = 'windows_1251'} # Cyrillic
+        elsif ($iso == 6) {$best_encoding = 'windows_1256'} # Arabic
+        elsif ($iso == 7) {$best_encoding = 'windows_1253'} # Greek
+        elsif ($iso == 8) {$best_encoding = 'windows_1255'} # Hebrew
+        elsif ($iso == 9) {$best_encoding = 'windows_1254'} # Turkish
+##      print STDERR "**** best_encoding windows value: $best_encoding\n";
+    }
+    }
+    
+    if (defined $best_encoding && $best_encoding !~ /^(ascii|utf8|unicode)$/ &&
+    !defined $encodings::encodings->{$best_encoding}) 
+    {
+    if ($self->{'verbosity'}) { 
+        gsprintf($outhandle, "BasePlugin: {ReadTextFile.unsupported_encoding}\n", $text, $best_encoding, "undef");
+    }
+##  print STDERR "***** unsupported encoding: $best_encoding. Setting it to undefined.\n";  
+    $best_encoding = undef;
+    }
+##    print STDERR "**** language: $language\n" if defined $language;
+##    print STDERR "**** encoding: $best_encoding\n" if defined $encoding;
+    
+    return $best_encoding;
+}
+
+# uses locale
 sub get_filesystem_encoding {