Changeset 23335

Timestamp:

2010-11-19T13:29:29+13:00 (13 years ago)

Author:

davidb

Message:

Work done on improving handing of filenames when the actualy filename encoding used is not necesarrily known. Tested for Linux. Work currently includes some debug statements that will be removed once testing for Windows and Mac is done.

Location:

main/trunk/greenstone2/perllib/plugins

Files:

• BasePlugin.pm (modified) (5 diffs)
• DirectoryPlugin.pm (modified) (9 diffs)
• HTMLPlugin.pm (modified) (10 diffs)
• ImageConverter.pm (modified) (3 diffs)
• ImagePlugin.pm (modified) (2 diffs)
• ReadTextFile.pm (modified) (2 diffs)

main/trunk/greenstone2/perllib/plugins/BasePlugin.pm

@@ -31 +31 @@
 
 use File::Basename;
+use Encode;
 
 use encodings;
@@ -785 +786 @@
 }
 
-# is there ever only one Source? Sometimes this will be called twice, for images etc that are converted.
+
+
+sub deduce_filename_encoding
+{
+    my $self = shift (@_);  
+    my ($file,$metadata) = @_;
+
+    my $gs_filename_encoding = $metadata->{"gs.filename_encoding"};
+    my $deduced_filename_encoding = undef;
+    
+    # Start by looking for manually assigned metadata
+    if (defined $gs_filename_encoding) {
+    if (ref ($gs_filename_encoding) eq "ARRAY") {
+        my $outhandle = $self->{'outhandle'};
+        
+        $deduced_filename_encoding = $gs_filename_encoding->[0];
+        
+        my $num_vals = scalar(@$gs_filename_encoding);
+        if ($num_vals>1) {
+        print $outhandle "Warning: gs.filename_encoding multiply defined for $file\n";
+        print $outhandle "         Selecting first value: $deduced_filename_encoding\n";
+        }
+    } 
+    else {
+        $deduced_filename_encoding = $gs_filename_encoding;
+    }
+    }
+    
+#   binmode(STDERR,":utf8");
+    
+#   print STDERR "**** file = $file\n";
+#   print STDERR "**** debug file = ", &unicode::debug_unicode_string($file),"\n";;
+    
+#   print STDERR "******* dfe = $deduced_filename_encoding\n";
+    
+    if (!defined $deduced_filename_encoding || ($deduced_filename_encoding =~ m/^\s*$/)) {
+    # Look to see if plugin specifies this value
+    }
+    
+    if (!defined $deduced_filename_encoding || ($deduced_filename_encoding =~ m/^\s*$/)) {
+    # See if we can determine the file system encoding through locale
+    # Unix only ?
+    
+    # if locale shows us filesystem is utf8, check to see filename is consistent
+    # => if not, then we have an "alien" filename on our hands
+    }
+    
+    
+    if (!defined $deduced_filename_encoding || ($deduced_filename_encoding =~ m/^\s*$/)) {
+    # Last chance, apply textcat to deduce filename encoding
+    }
+    
+    return $deduced_filename_encoding;
+}
+
+
+
+
+# Notionally written to be called once for each document, it is however safe to
+# call multiple times (as in the case of ImagePlugin) which calls this later on
+# after the original image has potentially been converted to a *new* source image
+# format (e.g. TIFF to PNG)
+
 sub set_Source_metadata {
     my $self = shift (@_);  
-    my ($doc_obj, $filename_no_path, $file_encoding) = @_;
+    my ($doc_obj, $raw_file, $filename_encoding) = @_;
+
+    # 1. Sets the filename (Source) for display encoded as Unicode if possible,
+    #    and (as a fallback) using %xx if not for non-ascii chars
+    # 2. Sets the url ref (SourceFile) to the URL encoded version
+    #    of filename for generated files
 
     my $top_section = $doc_obj->get_top_section();
     
     # UTF-8 version of filename
-    my $filemeta = $self->filename_to_utf8_metadata($filename_no_path, $file_encoding);
+    print STDERR "**** setting Source Metadata given: $raw_file\n";
+
+##    my $filemeta = $self->filename_to_utf8_metadata($raw_file, $filename_encoding);
+
+    my $url_encoded_filename;
+    if (defined $filename_encoding) {
+    # => Generate a pretty print version of filename that is mapped to Unicode
+
+    # Use filename_encoding to map raw filename to a Perl unicode-aware string 
+    $url_encoded_filename = decode($filename_encoding,$raw_file);
+
+    print STDERR "@@@@ pretty print using $filename_encoding: ", encode("utf8",$url_encoded_filename),"\n";
+    }
+    else {
+    # otherwise generate %xx encoded version of filename for char > 127
+    $url_encoded_filename = &unicode::raw_filename_to_url_encoded($raw_file);
+    }
+
+    print STDERR "***** saving Source as:             $url_encoded_filename\n";
+
     
     # Source is the UTF8 display name - not necessarily the name of the file on the system
-    $doc_obj->set_utf8_metadata_element($top_section, "Source", $filemeta); 
-
-    $filemeta = &util::rename_file($filemeta, $self->{'file_rename_method'});
+    $doc_obj->set_utf8_metadata_element($top_section, "Source", $url_encoded_filename); 
+
+    my $renamed_raw_file = &util::rename_file($raw_file, $self->{'file_rename_method'});
     # If using URL encoding, then SourceFile is the url-reference to url-encoded
-    # filemeta: it's a url that refers to the actual file on the system
-    $filemeta = &unicode::filename_to_url($filemeta);
-
-    $doc_obj->set_utf8_metadata_element($top_section, "SourceFile", $filemeta);
+    # renamed_raw_url: it's a url that refers to the actual file on the system
+    my $renamed_raw_url = &unicode::filename_to_url($renamed_raw_file);
+
+    $doc_obj->set_utf8_metadata_element($top_section, "SourceFile",
+                    $renamed_raw_url);
+
+    print STDERR "***** saving SourceFile as:         $renamed_raw_url\n";
 }
    
@@ -863 +953 @@
  
 
-    # sets the UTF8 filename (Source) for display and sets the url ref to URL encoded version
-    # of the UTF8 filename (SourceFile) for generated files
-    $self->set_Source_metadata($doc_obj, $filename_no_path);
-
+    my $filename_encoding = $self->deduce_filename_encoding($file,$metadata);
+    $self->set_Source_metadata($doc_obj,$filename_no_path,$filename_encoding);
 
     # plugin specific stuff - what args do we need here??
@@ -1032 +1120 @@
     my $file_derived_title = $file;
     $file_derived_title =~ s/_/ /g;
-    $file_derived_title =~ s/\.[^.]$//;
+    $file_derived_title =~ s/\.[^.]+$//;
 
     return $file_derived_title;
@@ -1044 +1132 @@
 
     if (!defined $doc_obj->get_metadata_element ($section, "Title") 
-    or $doc_obj->get_metadata_element($section, "Title") eq "") {
-
-    my $file_derived_title = $self->filename_to_utf8_metadata($self->filename_based_title($file));
+    || $doc_obj->get_metadata_element($section, "Title") eq "") {
+
+    my $source_file = $doc_obj->get_metadata_element($section, "Source");
+    my $file_derived_title;
+    if (defined $source_file) {
+        $file_derived_title =  $self->filename_based_title($source_file);
+    }
+    else {
+        # pp = pretty print
+        my $pp_file = (defined $source_file) ? $source_file : $file;
+
+        my $raw_title = $self->filename_based_title($file);
+        my $file_derived_title = &unicode::raw_filename_to_url_encoded($raw_title);
+    }
+
+
     if (!defined $doc_obj->get_metadata_element ($section, "Title")) {
         $doc_obj->add_utf8_metadata ($section, "Title", $file_derived_title);

main/trunk/greenstone2/perllib/plugins/DirectoryPlugin.pm

@@ -312 +312 @@
     
     for (my $i = 0; $i < scalar(@dir); $i++) {
-    my $subfile = $dir[$i];
+    my $raw_subfile = $dir[$i];
+    next if ($raw_subfile =~ m/^\.\.?$/);
+
     my $this_file_base_dir = $base_dir;
-    next if ($subfile =~ m/^\.\.?$/);
-
-    # Recursively read each $subfile
-    print $outhandle "DirectoryPlugin block recurring: $subfile\n" if ($verbosity > 2);
+    my $raw_file_subfile = &util::filename_cat($file, $raw_subfile);
+
+    # Recursively read each $raw_subfile
+    print $outhandle "DirectoryPlugin block recurring: $raw_file_subfile\n" if ($verbosity > 2);
     
     #$count += &plugin::file_block_read ($pluginfo, $this_file_base_dir,
+
     &plugin::file_block_read ($pluginfo, $this_file_base_dir,
-                  &util::filename_cat($file, $subfile),
+                  $raw_file_subfile,
                   $block_hash, $metadata, $gli);
     
@@ -390 +393 @@
     }
     @dir = readdir (DIR);
+    map { $_ = &unicode::raw_filename_to_url_encoded($_) } @dir;
     closedir (DIR);
 
@@ -436 +440 @@
     for (my $i = 0; $i < scalar(@dir); $i++) {
     my $subfile = $dir[$i];
+    next if ($subfile =~ m/^\.\.?$/);
+
     my $this_file_base_dir = $base_dir;
-    next if ($subfile =~ m/^\.\.?$/);
-    my $file_subfile = &util::filename_cat($file, $subfile);
-    my $full_filename = &util::filename_cat($this_file_base_dir, $file_subfile);
-    if ($self->file_is_blocked($block_hash,$full_filename)) {
-        print STDERR "DirectoryPlugin: file $full_filename was blocked for metadata_read\n" if ($verbosity > 2);
+    my $raw_subfile = &unicode::url_encoded_to_raw_filename($subfile);
+
+    my $raw_file_subfile = &util::filename_cat($file, $raw_subfile);
+    my $raw_full_filename = &util::filename_cat($this_file_base_dir, $raw_file_subfile);
+
+    if ($self->file_is_blocked($block_hash,$raw_full_filename)) {
+        print STDERR "DirectoryPlugin: file $raw_full_filename was blocked for metadata_read\n" if ($verbosity > 2);
         next;
     }
     
-    # Recursively read each $subfile
-    print $outhandle "DirectoryPlugin metadata recurring: $subfile\n" if ($verbosity > 2);
+    # Recursively read each $raw_subfile
+    print $outhandle "DirectoryPlugin metadata recurring: $raw_subfile\n" if ($verbosity > 2);
     
     &plugin::metadata_read ($pluginfo, $this_file_base_dir,
-                $file_subfile,$block_hash,
+                $raw_file_subfile,$block_hash,
                 \@extrametakeys, \%extrametadata,
                 \%extrametafile,
@@ -497 +505 @@
         last if (!opendir (DIR, $dirname));
         my @dirnow = readdir (DIR);
+        map { $_ = &unicode::raw_filename_to_url_encoded($_) } @dirnow;
         closedir (DIR);
 
@@ -518 +527 @@
 
     my $subfile = $dir[$i];
-    my $this_file_base_dir = $base_dir;
     last if ($maxdocs != -1 && ($count + $total_count) >= $maxdocs);
     next if ($subfile =~ /^\.\.?$/);
 
-    my $file_subfile = &util::filename_cat($file, $subfile);
-    my $full_filename 
-        = &util::filename_cat($this_file_base_dir,$file_subfile);
-
-    if ($self->file_is_blocked($block_hash,$full_filename)) {
-        print STDERR "DirectoryPlugin: file $full_filename was blocked for read\n" if ($verbosity > 2);
+    my $this_file_base_dir = $base_dir;
+    my $raw_subfile = &unicode::url_encoded_to_raw_filename($subfile);
+
+    my $raw_file_subfile = &util::filename_cat($file, $raw_subfile);
+    my $raw_full_filename 
+        = &util::filename_cat($this_file_base_dir,$raw_file_subfile);
+
+    if ($self->file_is_blocked($block_hash,$raw_full_filename)) {
+        print STDERR "DirectoryPlugin: file $raw_full_filename was blocked for read\n" if ($verbosity > 2);
         next;
     }
-    #print STDERR "processing $full_filename\n";
+    #print STDERR "processing $raw_full_filename\n";
     # Follow Windows shortcuts
-    if ($subfile =~ /(?i)\.lnk$/ && $ENV{'GSDLOS'} =~ /^windows$/i) {
+    if ($raw_subfile =~ /(?i)\.lnk$/ && $ENV{'GSDLOS'} =~ /^windows$/i) {
         require Win32::Shortcut;
-        my $shortcut = new Win32::Shortcut(&util::filename_cat($dirname, $subfile));
+        my $shortcut = new Win32::Shortcut(&util::filename_cat($dirname, $raw_subfile));
         if ($shortcut) {
         # The file to be processed is now the target of the shortcut
         $this_file_base_dir = "";
         $file = "";
-        $subfile = $shortcut->Path;
+        $raw_subfile = $shortcut->Path;
         }
     }
 
     # check for a symlink pointing back to a leading directory
-    if (-d "$dirname/$subfile" && -l "$dirname/$subfile") {
+    if (-d "$dirname/$raw_subfile" && -l "$dirname/$raw_subfile") {
         # readlink gives a "fatal error" on systems that don't implement
         # symlinks. This assumes the the -l test above would fail on those.
-        my $linkdest=readlink "$dirname/$subfile";
+        my $linkdest=readlink "$dirname/$raw_subfile";
         if (!defined ($linkdest)) {
         # system error - file not found?
@@ -555 +566 @@
         if ($linkdest =~ m@^[\./\\]+$@ ||
             index($dirname, $linkdest) != -1) {
-            warn "DirectoryPlugin: Ignoring recursive symlink ($dirname/$subfile -> $linkdest)\n";
+            warn "DirectoryPlugin: Ignoring recursive symlink ($dirname/$raw_subfile -> $linkdest)\n";
             next;
             ;
@@ -562 +573 @@
     }
 
-    print $outhandle "DirectoryPlugin: preparing metadata for $subfile\n" if ($verbosity > 2);
-
-    # Make a copy of $in_metadata to pass to $subfile
+    print $outhandle "DirectoryPlugin: preparing metadata for $raw_subfile\n" if ($verbosity > 2);
+
+    # Make a copy of $in_metadata to pass to $raw_subfile
     my $out_metadata = {};
     &metadatautil::combine_metadata_structures($out_metadata, $in_metadata);
 
     # check the assocfile_info
-    if (defined $self->{'assocfile_info'}->{$full_filename}) {
-        &metadatautil::combine_metadata_structures($out_metadata, $self->{'assocfile_info'}->{$full_filename});
-    }
-        ## encode the filename as perl5 doesn't handle unicode filenames    
-       
-        my $tmpfile = Encode::encode_utf8($subfile); 
+    if (defined $self->{'assocfile_info'}->{$raw_full_filename}) {
+        &metadatautil::combine_metadata_structures($out_metadata, $self->{'assocfile_info'}->{$raw_full_filename});
+    }
+
+    # $subfile by this point is url-encoded => all ASCII chars => no need to encode as UTF8
+
     # Next add metadata read in XML files (if it is supplied)
     if ($additionalmetadata == 1) {
         foreach my $filespec (@extrametakeys) {
-        ## use the utf8 encoded filename to do the filename comparison
-        if ($tmpfile =~ /^$filespec$/) {
+        ## use the url-encoded filename to do the filename comparison
+
+        if ($subfile =~ /^$filespec$/) {
             print $outhandle "File \"$subfile\" matches filespec \"$filespec\"\n" 
             if ($verbosity > 2);
@@ -605 +617 @@
         # Look to see if it's a completely new file
 
-        if (!$block_hash->{'new_files'}->{$full_filename}) {
+        if (!$block_hash->{'new_files'}->{$raw_full_filename}) {
         # Not a new file, must be an existing file
         # Let' see if it's newer than the last import.pl
 
 
-        if (! -d $full_filename) {
-            if (!$block_hash->{'reindex_files'}->{$full_filename}) {
+        if (! -d $raw_full_filename) {
+            if (!$block_hash->{'reindex_files'}->{$raw_full_filename}) {
             # filename has been around for longer than inf_timestamp
             print $outhandle "**** Skipping $subfile\n" if ($verbosity >3);
@@ -634 +646 @@
     
     $count += &plugin::read ($pluginfo, $this_file_base_dir,
-                 $file_subfile, $block_hash,
+                 $raw_file_subfile, $block_hash,
                  $out_metadata, $processor, $maxdocs, ($total_count + $count), $gli);
     }

main/trunk/greenstone2/perllib/plugins/HTMLPlugin.pm

@@ -181 +181 @@
     my @file_blocks;
     
-    my ($language, $encoding) = $self->textcat_get_language_encoding ($filename_full_path);
+    my ($language, $content_encoding) = $self->textcat_get_language_encoding ($filename_full_path);
+    $self->{'store_content_encoding'}->{$filename_full_path} = $content_encoding;
 
     # read in file ($text will be in utf8)
@@ -229 +230 @@
     # Convert the url_original_filename into its utf8 version. Store the utf8 link along with the url_original_filename
     my $utf8_link = "";
-    $self->decode_text($link,$encoding,$language,\$utf8_link);
+    $self->decode_text($link,$content_encoding,$language,\$utf8_link);
 
     $self->{'utf8_to_original_filename'}->{$utf8_link} = $url_original_filename;
@@ -272 +273 @@
         
     my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
+
+    # Lookup content_encoding worked out in file_block pass for this file
+    # Store it under the local name 'content_encoding' so its nice and 
+    # easy to access
+    $self->{'content_encoding'} = $self->{'store_content_encoding'}->{$filename_full_path};
+
     # get the input file
     my $input_filename = $file;
@@ -311 +318 @@
     $self->set_Source_metadata($doc_obj, $filename_no_path);
     }
+
+    delete $self->{'store_content_encoding'}->{$filename_full_path};
+    $self->{'content_encoding'} = undef;
+
     return ($process_status,$doc_obj);
 }
@@ -367 +378 @@
     # links, so even if 'file_is_url' is off, still need to store info
 
-    my ($tailname,$dirname,$suffix) = &File::Basename::fileparse($file, "\\.[^\\.]+\$");
-    my $utf8_file = $self->filename_to_utf8_metadata($file);
-    $utf8_file =~ s/&\#095;/_/g;
+    my ($tailname,$dirname) = &File::Basename::fileparse($file);
+    print STDERR "***!! file = $file\n";
+#    my $utf8_file = $self->filename_to_utf8_metadata($file);
+#    $utf8_file =~ s/&\#095;/_/g;
+    my $utf8_file = &unicode::raw_filename_to_url_encoded($tailname);
+    print STDERR "***!! utf8_file = $utf8_file\n";
+
     my $web_url = "http://";
     if(defined $dirname) { # local directory
@@ -535 +550 @@
     $self->process_section($textref, $base_dir, $file, $doc_obj, $cursection);
     }
+
     return 1;
 }
@@ -737 +753 @@
     return $front . $link . $back if $href =~ m/^(mailto|news|gopher|nntp|telnet|javascript):/is;
 
-
     if (($rl == 0) || ($filename =~ m/$self->{'process_exp'}/) || 
     ($href =~ m/\/$/) || ($href =~ m/^(mailto|news|gopher|nntp|telnet|javascript):/i)) {
+
+
+    # If web page didn't give encoding, then default to utf8
+    print "*************** looking up $file\n";
+
+    my $content_encoding= $self->{'content_encoding'} || "utf8";
+    $href = encode($content_encoding,$href);
+
+    $href = &unicode::raw_filename_to_url_encoded($href);
+    $href = &unicode::filename_to_url($href);
+
     &ghtml::urlsafe ($href);
+    print STDERR "***!!! href=$href\n";    
+
     return $front . "_httpextlink_&rl=" . $rl . "&href=" . $href . $hash_part . $back;
     } else {
-    # link is to some other type of file (eg image) so we'll
+    # link is to some other type of file (e.g., an image) so we'll
     # need to associate that file
     return $front . $self->add_file ($href, $rl, $hash_part, $base_dir, $doc_obj, $section) . $back;
@@ -769 +797 @@
     $filename =~ s/([\\\/])tidytmp([\\\/])/$1import$2/;
     }
-    # Replace %XX's in URL with decoded value if required. Note that the filename may include the %XX in some
-    # situations. If the *original* file's name was in URL encoding, the following method will not decode it.
+
+    # Replace %XX's in URL with decoded value if required. Note that the
+    # filename may include the %XX in some situations. If the *original*
+    # file's name was in URL encoding, the following method will not decode
+    # it.
     my $utf8_filename = $filename;
-    $filename = $self->opt_url_decode($utf8_filename);
+    my $opt_decode_utf8_filename = $self->opt_url_decode($utf8_filename);
+
+    my $content_encoding= $self->{'content_encoding'} || "utf8";
+
+    # The filenames that come through the HTML file have been decoded
+    # into Unicode aware Perl strings.  Need to convert them back
+    # to their initial raw-byte encoding to match the file that
+    # exists on the file system
+    $filename = encode($content_encoding, $opt_decode_utf8_filename);
+
 
     # some special processing if the intended filename was converted to utf8, but
@@ -778 +818 @@
     if (!-e $filename) {
     # try the original filename stored in map
-    my $original_filename = $self->{'utf8_to_original_filename'}->{$filename};
+    print STDERR "***###!! orig filename did not exist: $filename\n";
+
+    my $original_filename = $self->{'utf8_to_original_filename'}->{$utf8_filename};
+
+    print STDERR "**** Trying for $original_filename\n";
+
     if (defined $original_filename && -e $original_filename) {
+        print STDERR "*** found match\n";
         $filename = $original_filename;
     }
@@ -1118 +1164 @@
     $title =~ s/^$self->{'title_sub'}// if ($self->{'title_sub'});
     $title =~ s/^\s+//s; # in case title_sub introduced any...
-    $doc_obj->add_utf8_metadata ($section, 'Title', $title);
+    print STDERR "**** adding Title: ", Encode::encode("utf8",$title), "\n";
+    $doc_obj->add_utf8_metadata ($section, "Title", $title);
     print $outhandle " extracted Title metadata \"$title\" from $from\n" 
         if ($self->{'verbosity'} > 2);

main/trunk/greenstone2/perllib/plugins/ImageConverter.pm

@@ -152 +152 @@
 sub generate_images {
     my $self = shift(@_);
-    my ($filename_full_path, $filename_no_path, $doc_obj, $section) = @_;
+    my ($filename_full_path, $filename_no_path, $doc_obj, $section, $filename_encoding) = @_;
 
     # check image magick status
@@ -196 +196 @@
 
     if ($converttotype ne "" && $filename_full_path !~ m/$converttotype$/) {
-    #    $doc_obj->add_utf8_metadata($section, "Image", $utf8_filename_meta);
+#   #    $doc_obj->add_utf8_metadata($section, "Image", $utf8_filename_meta);
 
     my ($result, $converted_filename_full_path)
@@ -214 +214 @@
 
     # here we overwrite the original with the potentially converted one
-    $doc_obj->set_utf8_metadata_element($section, "Source", &unicode::url_decode($filename_no_path)); # displayname of generated image
-    $doc_obj->set_utf8_metadata_element($section, "SourceFile", $url_to_filename_no_path); # displayname of generated image
+#    $doc_obj->set_utf8_metadata_element($section, "Source", &unicode::url_decode($filename_no_path)); # displayname of generated image
+#    $doc_obj->set_utf8_metadata_element($section, "SourceFile", $url_to_filename_no_path); # displayname of generated image
+
+#    $self->set_Source_metadata($doc_obj,$url_to_filename_no_path,undef);
+
+    $self->set_Source_metadata($doc_obj,&unicode::url_decode($filename_no_path),
+                   $filename_encoding);
+
 
     # use identify to get info about the (possibly converted) image

main/trunk/greenstone2/perllib/plugins/ImagePlugin.pm

@@ -110 +110 @@
     my ($pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
 
-
     my $outhandle = $self->{'outhandle'};
     my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
@@ -116 +115 @@
     if ($self->{'image_conversion_available'} == 1)
     {
-    my $utf8_filename_no_path = $self->filepath_to_utf8($filename_no_path);
-    my $url_encoded_filename = &util::rename_file($utf8_filename_no_path, $self->{'file_rename_method'});
-    $self->generate_images($filename_full_path, $url_encoded_filename, $doc_obj, $doc_obj->get_top_section()); # should we check the return value?
+    my $filename_encoding = $self->deduce_filename_encoding($file,$metadata);
+
+#   my $utf8_filename_no_path = $self->filepath_to_utf8($filename_no_path);
+#   my $url_encoded_filename = &util::rename_file($utf8_filename_no_path, $self->{'file_rename_method'});
+
+#   $self->generate_images($filename_full_path, $url_encoded_filename, 
+#                  $doc_obj, $doc_obj->get_top_section()); # should we check the return value?
+
+    $filename_no_path = &unicode::raw_filename_to_url_encoded($filename_no_path);
+
+    # should we check the return value?
+    $self->generate_images($filename_full_path, $filename_no_path, 
+                   $doc_obj, $doc_obj->get_top_section(),$filename_encoding); 
+
     }
     else

main/trunk/greenstone2/perllib/plugins/ReadTextFile.pm

@@ -128 +128 @@
 
     # Do encoding stuff
-    my ($language, $encoding) = $self->textcat_get_language_encoding ($filename_full_path);
+    my ($language, $content_encoding) = $self->textcat_get_language_encoding ($filename_full_path);
     if ($self->{'verbosity'} > 2) {
-    print $outhandle "ReadTextFile: reading $file as ($encoding,$language)\n";
+    print $outhandle "ReadTextFile: reading $file as ($content_encoding,$language)\n";
     }
 
@@ -140 +140 @@
     $doc_obj->add_utf8_metadata($top_section, "Plugin", "$self->{'plugin_type'}");
     $doc_obj->add_utf8_metadata($top_section, "FileSize", (-s $filename_full_path));
-    $self->set_Source_metadata($doc_obj, $filename_no_path, $encoding);
+
+    my $filename_encoding = $self->deduce_filename_encoding($file,$metadata);
+    $self->set_Source_metadata($doc_obj, $filename_no_path, $filename_encoding);
 
     $doc_obj->add_utf8_metadata($top_section, "Language", $language);
-    $doc_obj->add_utf8_metadata($top_section, "Encoding", $encoding);
+    $doc_obj->add_utf8_metadata($top_section, "Encoding", $content_encoding);
     
     # read in file ($text will be in utf8)
     my $text = "";
-    $self->read_file ($filename_full_path, $encoding, $language, \$text);
+    $self->read_file ($filename_full_path, $content_encoding, $language, \$text);
 
     if (!length ($text)) {