Changeset 16767

Timestamp:

2008-08-13T16:25:49+12:00 (16 years ago)

Author:

ak19

Message:

In progress: Filename encoding after working with it on Windows. Still need to get interlinking html files (with multilingual filenames) working

File:

• gsdl/trunk/perllib/plugins/BasePlugin.pm (modified) (18 diffs)

gsdl/trunk/perllib/plugins/BasePlugin.pm

@@ -411 +411 @@
     my $filename_encoding = $self->{'filename_encoding'}; # filename encoding setting
 
-##  print STDERR "**** User chose filename encoding setting: $filename_encoding\n";
-    
     # Whenever filename-encoding is set to any of the auto settings, we
     # check if the filename is already in UTF8. If it is, then we're done.
@@ -418 +416 @@
     if(&unicode::check_is_utf8($filemeta)) 
     {
-##      print STDERR "**** It is already UTF8\n";
         $filename_encoding = "utf8";
         return $filemeta;
@@ -484 +481 @@
     $filename_encoding = $self->locale_encoding() if $filename_encoding eq "undefined";
     }
-
-##    print STDERR "**** filename_encoding selected: $filename_encoding \n";
         
     # if still undefined, use utf8 as fallback
@@ -492 +487 @@
     }
 
+    print STDERR "**** UTF8 encoding the filename $filemeta ";
+    
     # if the filename encoding is set to utf8 but it isn't utf8 already--such as when
     # 1. the utf8 fallback is used, or 2. if the system locale is used and happens to
@@ -498 +495 @@
     # cases attempt to make the filename utf8 to match.
     if($filename_encoding eq "utf8" && !&unicode::check_is_utf8($filemeta)) {
-##  print STDERR "**** BEFORE utf8 conversion: $filemeta\n";
     &unicode::ensure_utf8(\$filemeta);
-##  print STDERR "**** AFTER utf8 conversion: $filemeta\n";
-    }
-
+    }
 
     # convert non-unicode encodings to utf8
@@ -511 +505 @@
     }
 
-    print "*** filename encoding found: $filename_encoding\n";
-    print "*** utf8 encoded filename: $filemeta\n";
-
+    print STDERR " from encoding $filename_encoding -> $filemeta\n";
     return $filemeta;
 }
@@ -528 +520 @@
     my ($filemeta) = $file =~ /([^\\\/]+)$/; # getting the tail of the filepath (skips all string parts containing slashes upto the end)
     $filemeta = $self->filepath_to_utf8($filemeta, $file_encoding);
-
+    
     my $dmsafe_filemeta = &ghtml::dmsafe($filemeta);
 
@@ -543 +535 @@
     }
 
-    print "filename encoding determined based on locale: " . $self->{'filesystem_encoding'} . "\n";
+    print STDERR "*** filename encoding determined based on locale: " . $self->{'filesystem_encoding'} . "\n";
     return $self->{'filesystem_encoding'}; # can be the string "undefined"
 }
@@ -559 +551 @@
     $strictfilemeta =~ s/\s*$//g;
     
-##    print STDERR "**** strict filename is |$strictfilemeta|\n";
     my $filename_encoding = $self->encoding_from_language_analysis($strictfilemeta);
     if(!defined $filename_encoding) {
@@ -565 +556 @@
     }
 
-##    print STDERR "**** textcat found filename encoding: " . $file_textcat_encoding_map{$strictfilemeta} . "\n";
     return $filename_encoding; # can be the string "undefined"
 }
@@ -580 +570 @@
     $self->{'textcat'} = new textcat() unless defined($self->{'textcat'});
     #my $results = $self->{'textcat'}->classify(\$text);
-    my $results = $self->{'textcat'}->classify_cached(\$text);
+    my $results = $self->{'textcat'}->classify_cached_filename(\$text);
 
 
     if (scalar @$results < 0) { 
-    print STDERR "**** Textcat returned 0 results\n";
     return undef;
     }
     
-    print STDERR "**** TEXTCAT RESULTS for $text: ";
-    print STDERR join(",", @$results);
-    print STDERR "\n";
+##    print STDERR "**** TEXTCAT RESULTS for $text: ";
+##    print STDERR join(",", @$results);
+##    print STDERR "\n";
 
     # We have some results, we choose the first
@@ -597 +586 @@
     $best_encoding = $encoding;
     if (!defined $best_encoding) {
-##  print STDERR "**** Textcat cannot determine encoding of filename: it's undefined.\n";
     return undef;
     } 
@@ -603 +591 @@
     if (defined $best_encoding && $best_encoding =~ m/^iso_8859/ && &unicode::check_is_utf8($text)) {
     # the text is valid utf8, so assume that's the real encoding (since textcat is based on probabilities)
-##  print STDERR "*** Filename turns out to be UTF8\n";
     $best_encoding = 'utf8';
     }
@@ -611 +598 @@
     # eg MS versions of standard encodings
     if (defined $best_encoding && $best_encoding =~ /^iso_8859_(\d+)/) {
-##  print STDERR "**** best_encoding is ISO_8859: $best_encoding\n";
-
     my $iso = $1; # which variant of the iso standard?
     # iso-8859 sets don't use chars 0x80-0x9f, windows codepages do
     if ($text =~ /[\x80-\x9f]/) {
-##      print STDERR "**** best_encoding is some windows value: $best_encoding\n";
         # Western Europe
         if ($iso == 1 or $iso == 15) { $best_encoding = 'windows_1252' }
@@ -625 +609 @@
         elsif ($iso == 8) {$best_encoding = 'windows_1255'} # Hebrew
         elsif ($iso == 9) {$best_encoding = 'windows_1254'} # Turkish
-##      print STDERR "**** best_encoding windows value: $best_encoding\n";
     }
     }
@@ -635 +618 @@
         gsprintf($outhandle, "BasePlugin: {ReadTextFile.unsupported_encoding}\n", $text, $best_encoding, "undef");
     }
-##  print STDERR "***** unsupported encoding: $best_encoding. Setting it to undefined.\n";  
     $best_encoding = undef;
     }
-##    print STDERR "**** language: $language\n" if defined $language;
-##    print STDERR "**** encoding: $best_encoding\n" if defined $encoding;
     
     return $best_encoding;
@@ -706 +686 @@
     my $filemeta = $self->filename_to_utf8_metadata($filename_no_path, $file_encoding);
     $doc_obj->set_utf8_metadata_element($top_section, "Source", $filemeta);
-
 }
      
@@ -724 +703 @@
     my $top_section = $doc_obj->get_top_section();
     my $oid = $doc_obj->get_metadata_element($top_section,$metadata_doc_id);
-##  print STDERR "**** oid = $oid\n";
         $doc_obj->set_OID($oid);
     }