Changeset 16724 for gsdl/trunk/perllib/plugins/ReadTextFile.pm

Timestamp:

2008-08-12T14:54:31+12:00 (16 years ago)

Author:

ak19

Message:

1. Dr Bainbridge added some language-encoding related methods that work with multiread.pm; 2. When checking for encoding specified in head tag (meta http-equiv) it checks whether this meta tag is the first item nested in a comment in which case it is ignored. What about containing comments that do not contain meta http-equiv as the first element? 3. textcat calls classify_cached_filecontents which stores the fileencoding of the file

File:

• gsdl/trunk/perllib/plugins/ReadTextFile.pm (modified) (7 diffs)

gsdl/trunk/perllib/plugins/ReadTextFile.pm

@@ -139 +139 @@
     $doc_obj->set_OIDtype ($processor->{'OIDtype'}, $processor->{'OIDmetadata'});    
     $doc_obj->add_utf8_metadata($top_section, "Plugin", "$self->{'plugin_type'}");
-   $doc_obj->add_utf8_metadata($top_section, "FileSize", (-s $filename_full_path));
+    $doc_obj->add_utf8_metadata($top_section, "FileSize", (-s $filename_full_path));
     $self->set_Source_metadata($doc_obj, $filename_no_path, $encoding);
 
@@ -225 +225 @@
 
 
+sub read_file_no_decoding {
+    my $self = shift (@_);
+    my ($filename, $textref) = @_;
+
+    if (!-r $filename)
+    {
+    my $outhandle = $self->{'outhandle'};
+    gsprintf($outhandle, "{ReadTextFile.read_denied}\n", $filename) if $self->{'verbosity'};
+    # print $outhandle "Read permission denied for $filename\n" if $self->{'verbosity'};
+    return;
+    }
+    $$textref = "";
+    if (!open (FILE, $filename)) {
+    gsprintf(STDERR, "ReadTextFile::read_file {ReadTextFile.could_not_open_for_reading} ($!)\n", $filename);
+    die "\n";
+    }
+     
+    my $reader = new multiread();
+    $reader->set_handle ('ReadTextFile::FILE');
+    $reader->read_file_no_decoding ($textref);
+    
+    $self->{'reader'} = $reader;
+
+    close FILE;
+}
+
+
+sub decode_text {
+    my $self = shift (@_);
+    my ($raw_text, $encoding, $language, $textref) = @_;
+
+    my $reader = $self->{'reader'};
+    if (!defined $reader) {
+    gsprintf(STDERR, "ReadTextFile::decode_text needs to call ReadTextFile::read_file_no_decoding first\n");
+    }
+    else {
+    $reader->set_encoding($encoding);
+    $reader->decode_text($raw_text,$textref);
+    }
+}
+
+
 sub textcat_get_language_encoding {
     my $self = shift (@_);
     my ($filename) = @_;
 
-    
     my ($language, $encoding, $extracted_encoding);
     if ($self->{'input_encoding'} eq "auto") {
@@ -235 +276 @@
         ($language, $encoding) = $self->get_language_encoding ($filename);
     } elsif ($self->{'extract_language'}) {
-        # use textcat to get language metadata
+    # use textcat to get language metadata
         ($language, $extracted_encoding) = $self->get_language_encoding ($filename);
         $encoding = $self->{'input_encoding'};
@@ -241 +282 @@
     # to english in iso-8859-1 (except for some punctuation). We don't have
     # a language model for en_utf8, so textcat always says iso-8859-1!
-        if ($extracted_encoding ne $encoding && $language ne "en"
-        && $self->{'verbosity'}) {
+        if ($extracted_encoding ne $encoding && $language ne "en" && $self->{'verbosity'}) {
         my $plugin_name = ref ($self);
         my $outhandle = $self->{'outhandle'};
@@ -249 +289 @@
     } else {
         $language = $self->{'default_language'};
-        $encoding = $self->{'input_encoding'};
-    }
+        $encoding = $self->{'input_encoding'}; 
+    }
+    
+#    print STDERR "**** language encoding of contents of file $filename:\n\t****$language $encoding\n";
 
     return ($language, $encoding);
@@ -305 +347 @@
     if ($text =~ /^<\?xml.*encoding="(.+?)"/) {
         $best_encoding = $1;
-    } elsif ($text =~ /<meta http-equiv.*content-type.*charset=(.+?)"/i) {#"
+    }
+    # check the meta http-equiv charset tag unless it is commented out
+    elsif (($text !~ /<!--[^<>]?<meta http-equiv/i) && ($text =~ /<meta http-equiv.*content-type.*charset=(.+?)\"/i)) {            
         $best_encoding = $1;
+#       print STDERR "**** meta tag found, encoding is: $best_encoding\n";
     }
     if ($best_encoding) { # we extracted an encoding
@@ -323 +368 @@
     # get the language/encoding
     $self->{'textcat'} = new textcat() if (!defined($self->{'textcat'}));
-    my $results = $self->{'textcat'}->classify(\$text);
+#    my $results = $self->{'textcat'}->classify(\$text);
+    my $results = $self->{'textcat'}->classify_cached_filecontents(\$text, $filename);
 
     # if textcat returns 3 or less possibilities we'll use the