Changeset 11389

Timestamp:

2006-03-17T13:42:39+13:00 (18 years ago)

Author:

jrm21

Message:

try to get the encoding from a '<meta http-equiv' tag if HTML.

make sure we add the filename/Source metadata as utf-8, so that we
won't create invalid xml files if the filename is in some other encoding.

If the file contents aren't in utf8, assume the filename is in the same
encoding as the contents.

print out the encoding we used to read this file if verbosity >= 3.

File:

• trunk/gsdl/perllib/plugins/BasPlug.pm (modified) (9 diffs)

trunk/gsdl/perllib/plugins/BasPlug.pm

@@ -45 +45 @@
 use multiread;
 use encodings;
+use unicode;
 use cnseg;
 use acronym;
@@ -808 +809 @@
     # Do encoding stuff
     my ($language, $encoding) = $self->textcat_get_language_encoding ($filename);
+    if ($self->{'verbosity'} > 2) {
+    print $outhandle "BasPlug: reading $file as ($encoding,$language)\n";
+    }
 
     # create a new document
@@ -819 +823 @@
     my ($filemeta) = $file =~ /([^\\\/]+)$/;
     # how do we know what encoding the filename is in?
-    $doc_obj->add_metadata($doc_obj->get_top_section(), "Source", &ghtml::dmsafe($filemeta));
+    # assume it is in the same encoding as its contents
+    if ($encoding !~ /(?:ascii|utf8|unicode)/) {
+    $filemeta = unicode::unicode2utf8(
+        unicode::convert2unicode($encoding, \$filemeta)
+    );
+    }
+    $doc_obj->add_utf8_metadata($doc_obj->get_top_section(), "Source", &ghtml::dmsafe($filemeta));
     if ($self->{'cover_image'}) {
     $self->associate_cover_image($doc_obj, $filename);
@@ -1028 +1038 @@
         my $outhandle = $self->{'outhandle'};
         gsprintf($outhandle, "$plugin_name: {BasPlug.wrong_encoding}\n", $filename, $encoding, $extracted_encoding);
-            # print $outhandle "$plugin_name: WARNING: $filename was read using $encoding encoding but ";
-            # print $outhandle "appears to be encoded as $extracted_encoding.\n";
         }
     } else {
@@ -1047 +1055 @@
     my $outhandle = $self->{'outhandle'};
     my $unicode_format = "";
+    my $best_language = "";
+    my $best_encoding = "";
+    
     # read in file
-    open (FILE, $filename) || (gsprintf(STDERR, "BasPlug::get_language_encoding {BasPlug.could_not_open_for_reading} ($!)\n", $filename) && die "\n"); # die "BasPlug::get_language_encoding could not open $filename for reading ($!)\n";
+    if (!open (FILE, $filename)) {
+    gsprintf(STDERR, "BasPlug::get_language_encoding {BasPlug.could_not_open_for_reading} ($!)\n", $filename);
+    # this is a pretty bad error, but try to continue anyway
+    return ($self->{'default_language'}, $self->{'input_encoding'});
+    }
     undef $/;
     my $text = <FILE>;
@@ -1069 +1084 @@
     }
     
-    # VB scripting generated Word to HTML file
-    if ($text =~ /charset=(windows.*)[\"]/ig){
-    my $vbhtml_encoding = $1;
-    $vbhtml_encoding =~ s/-+/_/g;
-    $self->{'input_encoding'} = $vbhtml_encoding;
-    }
-    
-    # remove <title>stuff</title> -- as titles tend often to be in English
-    # for foreign language documents
-    $text =~ s/<title>(.|\n)*?<\/title>//i;
-
-    # remove all HTML tags
+
+    # handle html files specially
     # XXX this doesn't match plugins derived from HTMLPlug (except ConvertTo)
     if (ref($self) eq 'HTMLPlug' ||
     (exists $self->{'converted_to'} && $self->{'converted_to'} eq 'HTML')){
+
+    # remove <title>stuff</title> -- as titles tend often to be in English
+    # for foreign language documents
+    $text =~ s!<title>.*?</title>!!si;
+
+    # see if this html file specifies its encoding
+    if ($text =~ /^<\?xml.*encoding="(.+?)"/) {
+        $best_encoding = $1;
+    } elsif ($text =~ /<meta http-equiv.*content-type.*charset=(.+?)"/i) {
+        $best_encoding = $1;
+    }
+    if ($best_encoding) { # we extracted an encoding
+        $best_encoding =~ s/-+/_/g;
+        $best_encoding = lc($best_encoding); # lowercase
+        if ($best_encoding eq "utf_8") { $best_encoding = "utf8" }
+        $self->{'input_encoding'} = $best_encoding;
+    }
+        
+    # remove all HTML tags
     $text =~ s/<[^>]*>//sg;
     }
@@ -1093 +1117 @@
     # first one in the list - otherwise use the defaults
     if (scalar @$results > 3) {
-    my $best_encoding="";
     if ($unicode_format) { # in case the first had a BOM
         $best_encoding=$unicode_format;
@@ -1122 +1145 @@
              $filename, $self->{'default_language'});
         }       
-        return ($self->{'default_language'}, $self->{'input_encoding'});
+        $best_language = $self->{'default_language'};
+        $best_encoding = $self->{'input_encoding'};
 
     } else {
@@ -1130 +1154 @@
              $filename, $self->{'default_language'});
         }
-        return ($self->{'default_language'}, $best_encoding);
-    }
-    }
-
-    # format language/encoding
-    my ($language, $encoding) = $results->[0] =~ /^([^-]*)(?:-(.*))?$/;
-    if (!defined $language) {
-    if ($self->{'verbosity'}>2) {
-        gsprintf($outhandle,
-             "BasPlug: {BasPlug.could_not_extract_language}\n",
-             $filename, $self->{'default_language'});
-    }
-    $language = $self->{'default_language'};
-    }
-    if (!defined $encoding) {
-    if ($self->{'verbosity'}>2) {
-        gsprintf($outhandle,
-             "BasPlug: {BasPlug.could_not_extract_encoding}\n",
-             $filename, $self->{'default_encoding'});
-    }
-    $encoding = $self->{'default_encoding'};
-    }
-
+        $best_language = $self->{'default_language'};
+    }
+    } else { # <= 3 suggestions
+    my ($language, $encoding) = $results->[0] =~ /^([^-]*)(?:-(.*))?$/;
+    if (!defined $language) {
+        if ($self->{'verbosity'}>2) {
+        gsprintf($outhandle,
+            "BasPlug: {BasPlug.could_not_extract_language}\n",
+            $filename, $self->{'default_language'});
+        }
+        $language = $self->{'default_language'};
+    }
+    if (!defined $encoding) {
+        if ($self->{'verbosity'}>2) {
+        gsprintf($outhandle,
+            "BasPlug: {BasPlug.could_not_extract_encoding}\n",
+            $filename, $self->{'default_encoding'});
+        }
+        $encoding = $self->{'default_encoding'};
+    }
+    $best_language = $language;
+    if (! $best_encoding ) { # may already be set... eg from html meta tag
+        $best_encoding = $encoding;
+    }
+    }
+
+    my $text_copy = $text;
+    if ($best_encoding =~ /^iso_8859/ && unicode::ensure_utf8(\$text_copy)==0) {
+    # the text is valid utf8, so assume that's the real encoding
+    # (since textcat is based on probabilities)
+    $best_encoding = 'utf8';
+    }
 
     # check for equivalents where textcat doesn't have some encodings...
     # eg MS versions of standard encodings
-    if ($encoding =~ /^iso_8859_(\d+)/) {
+    if ($best_encoding =~ /^iso_8859_(\d+)/) {
     my $iso = $1; # which variant of the iso standard?
     # iso-8859 sets don't use chars 0x80-0x9f, windows codepages do
     if ($text =~ /[\x80-\x9f]/) {
         # Western Europe
-        if ($iso == 1 or $iso == 15) { $encoding = 'windows_1252' }
-        elsif ($iso == 2) { $encoding = 'windows_1250' } # Central Europe
-        elsif ($iso == 5) { $encoding = 'windows_1251' } # Cyrillic
-        elsif ($iso == 6) { $encoding = 'windows_1256' } # Arabic
-        elsif ($iso == 7) { $encoding = 'windows_1253' } # Greek
-        elsif ($iso == 8) { $encoding = 'windows_1255' } # Hebrew
-        elsif ($iso == 9) { $encoding = 'windows_1254' } # Turkish
-    }
-    }
-
-    if ($encoding !~ /^(ascii|utf8|unicode)$/ &&
-    !defined $encodings::encodings->{$encoding}) {
+        if ($iso == 1 or $iso == 15) { $best_encoding = 'windows_1252' }
+        elsif ($iso == 2) {$best_encoding = 'windows_1250'} # Central Europe
+        elsif ($iso == 5) {$best_encoding = 'windows_1251'} # Cyrillic
+        elsif ($iso == 6) {$best_encoding = 'windows_1256'} # Arabic
+        elsif ($iso == 7) {$best_encoding = 'windows_1253'} # Greek
+        elsif ($iso == 8) {$best_encoding = 'windows_1255'} # Hebrew
+        elsif ($iso == 9) {$best_encoding = 'windows_1254'} # Turkish
+    }
+    }
+
+    if ($best_encoding !~ /^(ascii|utf8|unicode)$/ &&
+    !defined $encodings::encodings->{$best_encoding}) {
     if ($self->{'verbosity'}) {
         gsprintf($outhandle, "BasPlug: {BasPlug.unsupported_encoding}\n",
-             $filename, $encoding, $self->{'default_encoding'});
-    }
-    $encoding = $self->{'default_encoding'};
-    }
-
-    return ($language, $encoding);
+             $filename, $best_encoding, $self->{'default_encoding'});
+    }
+    $best_encoding = $self->{'default_encoding'};
+    }
+
+    return ($best_language, $best_encoding);
 }