Index: /main/trunk/greenstone2/perllib/plugins/BasePlugin.pm
===================================================================
--- /main/trunk/greenstone2/perllib/plugins/BasePlugin.pm	(revision 23334)
+++ /main/trunk/greenstone2/perllib/plugins/BasePlugin.pm	(revision 23335)
@@ -31,4 +31,5 @@
 
 use File::Basename;
+use Encode;
 
 use encodings;
@@ -785,23 +786,112 @@
 }
 
-# is there ever only one Source? Sometimes this will be called twice, for images etc that are converted.
+
+
+sub deduce_filename_encoding
+{
+    my $self = shift (@_);  
+    my ($file,$metadata) = @_;
+
+    my $gs_filename_encoding = $metadata->{"gs.filename_encoding"};
+    my $deduced_filename_encoding = undef;
+    
+    # Start by looking for manually assigned metadata
+    if (defined $gs_filename_encoding) {
+	if (ref ($gs_filename_encoding) eq "ARRAY") {
+	    my $outhandle = $self->{'outhandle'};
+	    
+	    $deduced_filename_encoding = $gs_filename_encoding->[0];
+	    
+	    my $num_vals = scalar(@$gs_filename_encoding);
+	    if ($num_vals>1) {
+		print $outhandle "Warning: gs.filename_encoding multiply defined for $file\n";
+		print $outhandle "         Selecting first value: $deduced_filename_encoding\n";
+	    }
+	} 
+	else {
+	    $deduced_filename_encoding = $gs_filename_encoding;
+	}
+    }
+    
+#	binmode(STDERR,":utf8");
+    
+#	print STDERR "**** file = $file\n";
+#	print STDERR "**** debug file = ", &unicode::debug_unicode_string($file),"\n";;
+    
+#	print STDERR "******* dfe = $deduced_filename_encoding\n";
+    
+    if (!defined $deduced_filename_encoding || ($deduced_filename_encoding =~ m/^\s*$/)) {
+	# Look to see if plugin specifies this value
+    }
+    
+    if (!defined $deduced_filename_encoding || ($deduced_filename_encoding =~ m/^\s*$/)) {
+	# See if we can determine the file system encoding through locale
+	# Unix only ?
+	
+	# if locale shows us filesystem is utf8, check to see filename is consistent
+	# => if not, then we have an "alien" filename on our hands
+    }
+    
+    
+    if (!defined $deduced_filename_encoding || ($deduced_filename_encoding =~ m/^\s*$/)) {
+	# Last chance, apply textcat to deduce filename encoding
+    }
+    
+    return $deduced_filename_encoding;
+}
+
+
+
+
+# Notionally written to be called once for each document, it is however safe to
+# call multiple times (as in the case of ImagePlugin) which calls this later on
+# after the original image has potentially been converted to a *new* source image
+# format (e.g. TIFF to PNG)
+
 sub set_Source_metadata {
     my $self = shift (@_);  
-    my ($doc_obj, $filename_no_path, $file_encoding) = @_;
+    my ($doc_obj, $raw_file, $filename_encoding) = @_;
+
+    # 1. Sets the filename (Source) for display encoded as Unicode if possible,
+    #    and (as a fallback) using %xx if not for non-ascii chars
+    # 2. Sets the url ref (SourceFile) to the URL encoded version
+    #    of filename for generated files
 
     my $top_section = $doc_obj->get_top_section();
     
     # UTF-8 version of filename
-    my $filemeta = $self->filename_to_utf8_metadata($filename_no_path, $file_encoding);
+    print STDERR "**** setting Source Metadata given: $raw_file\n";
+
+##    my $filemeta = $self->filename_to_utf8_metadata($raw_file, $filename_encoding);
+
+    my $url_encoded_filename;
+    if (defined $filename_encoding) {
+	# => Generate a pretty print version of filename that is mapped to Unicode
+
+	# Use filename_encoding to map raw filename to a Perl unicode-aware string 
+	$url_encoded_filename = decode($filename_encoding,$raw_file);
+
+	print STDERR "@@@@ pretty print using $filename_encoding: ", encode("utf8",$url_encoded_filename),"\n";
+    }
+    else {
+	# otherwise generate %xx encoded version of filename for char > 127
+	$url_encoded_filename = &unicode::raw_filename_to_url_encoded($raw_file);
+    }
+
+    print STDERR "***** saving Source as:             $url_encoded_filename\n";
+
     
     # Source is the UTF8 display name - not necessarily the name of the file on the system
-    $doc_obj->set_utf8_metadata_element($top_section, "Source", $filemeta); 
-
-    $filemeta = &util::rename_file($filemeta, $self->{'file_rename_method'});
+    $doc_obj->set_utf8_metadata_element($top_section, "Source", $url_encoded_filename); 
+
+    my $renamed_raw_file = &util::rename_file($raw_file, $self->{'file_rename_method'});
     # If using URL encoding, then SourceFile is the url-reference to url-encoded
-    # filemeta: it's a url that refers to the actual file on the system
-    $filemeta = &unicode::filename_to_url($filemeta);
-
-    $doc_obj->set_utf8_metadata_element($top_section, "SourceFile", $filemeta);
+    # renamed_raw_url: it's a url that refers to the actual file on the system
+    my $renamed_raw_url = &unicode::filename_to_url($renamed_raw_file);
+
+    $doc_obj->set_utf8_metadata_element($top_section, "SourceFile",
+					$renamed_raw_url);
+
+    print STDERR "***** saving SourceFile as:         $renamed_raw_url\n";
 }
    
@@ -863,8 +953,6 @@
  
 
-    # sets the UTF8 filename (Source) for display and sets the url ref to URL encoded version
-    # of the UTF8 filename (SourceFile) for generated files
-    $self->set_Source_metadata($doc_obj, $filename_no_path);
-
+    my $filename_encoding = $self->deduce_filename_encoding($file,$metadata);
+    $self->set_Source_metadata($doc_obj,$filename_no_path,$filename_encoding);
 
     # plugin specific stuff - what args do we need here??
@@ -1032,5 +1120,5 @@
     my $file_derived_title = $file;
     $file_derived_title =~ s/_/ /g;
-    $file_derived_title =~ s/\.[^.]$//;
+    $file_derived_title =~ s/\.[^.]+$//;
 
     return $file_derived_title;
@@ -1044,7 +1132,20 @@
 
     if (!defined $doc_obj->get_metadata_element ($section, "Title") 
-	or $doc_obj->get_metadata_element($section, "Title") eq "") {
-
-	my $file_derived_title = $self->filename_to_utf8_metadata($self->filename_based_title($file));
+	|| $doc_obj->get_metadata_element($section, "Title") eq "") {
+
+	my $source_file = $doc_obj->get_metadata_element($section, "Source");
+	my $file_derived_title;
+	if (defined $source_file) {
+	    $file_derived_title =  $self->filename_based_title($source_file);
+	}
+	else {
+	    # pp = pretty print
+	    my $pp_file = (defined $source_file) ? $source_file : $file;
+
+	    my $raw_title = $self->filename_based_title($file);
+	    my $file_derived_title = &unicode::raw_filename_to_url_encoded($raw_title);
+	}
+
+
 	if (!defined $doc_obj->get_metadata_element ($section, "Title")) {
 	    $doc_obj->add_utf8_metadata ($section, "Title", $file_derived_title);
Index: /main/trunk/greenstone2/perllib/plugins/DirectoryPlugin.pm
===================================================================
--- /main/trunk/greenstone2/perllib/plugins/DirectoryPlugin.pm	(revision 23334)
+++ /main/trunk/greenstone2/perllib/plugins/DirectoryPlugin.pm	(revision 23335)
@@ -312,14 +312,17 @@
     
     for (my $i = 0; $i < scalar(@dir); $i++) {
-	my $subfile = $dir[$i];
+	my $raw_subfile = $dir[$i];
+	next if ($raw_subfile =~ m/^\.\.?$/);
+
 	my $this_file_base_dir = $base_dir;
-	next if ($subfile =~ m/^\.\.?$/);
-
-	# Recursively read each $subfile
-	print $outhandle "DirectoryPlugin block recurring: $subfile\n" if ($verbosity > 2);
+	my $raw_file_subfile = &util::filename_cat($file, $raw_subfile);
+
+	# Recursively read each $raw_subfile
+	print $outhandle "DirectoryPlugin block recurring: $raw_file_subfile\n" if ($verbosity > 2);
 	
 	#$count += &plugin::file_block_read ($pluginfo, $this_file_base_dir,
+
 	&plugin::file_block_read ($pluginfo, $this_file_base_dir,
-				  &util::filename_cat($file, $subfile),
+				  $raw_file_subfile,
 				  $block_hash, $metadata, $gli);
 	
@@ -390,4 +393,5 @@
     }
     @dir = readdir (DIR);
+    map { $_ = &unicode::raw_filename_to_url_encoded($_) } @dir;
     closedir (DIR);
 
@@ -436,18 +440,22 @@
     for (my $i = 0; $i < scalar(@dir); $i++) {
 	my $subfile = $dir[$i];
+	next if ($subfile =~ m/^\.\.?$/);
+
 	my $this_file_base_dir = $base_dir;
-	next if ($subfile =~ m/^\.\.?$/);
-	my $file_subfile = &util::filename_cat($file, $subfile);
-	my $full_filename = &util::filename_cat($this_file_base_dir, $file_subfile);
-	if ($self->file_is_blocked($block_hash,$full_filename)) {
-	    print STDERR "DirectoryPlugin: file $full_filename was blocked for metadata_read\n" if ($verbosity > 2);
+	my $raw_subfile = &unicode::url_encoded_to_raw_filename($subfile);
+
+	my $raw_file_subfile = &util::filename_cat($file, $raw_subfile);
+	my $raw_full_filename = &util::filename_cat($this_file_base_dir, $raw_file_subfile);
+
+	if ($self->file_is_blocked($block_hash,$raw_full_filename)) {
+	    print STDERR "DirectoryPlugin: file $raw_full_filename was blocked for metadata_read\n" if ($verbosity > 2);
 	    next;
 	}
 	
-	# Recursively read each $subfile
-	print $outhandle "DirectoryPlugin metadata recurring: $subfile\n" if ($verbosity > 2);
+	# Recursively read each $raw_subfile
+	print $outhandle "DirectoryPlugin metadata recurring: $raw_subfile\n" if ($verbosity > 2);
 	
 	&plugin::metadata_read ($pluginfo, $this_file_base_dir,
-				$file_subfile,$block_hash,
+				$raw_file_subfile,$block_hash,
 				\@extrametakeys, \%extrametadata,
 				\%extrametafile,
@@ -497,4 +505,5 @@
 	    last if (!opendir (DIR, $dirname));
 	    my @dirnow = readdir (DIR);
+	    map { $_ = &unicode::raw_filename_to_url_encoded($_) } @dirnow;
 	    closedir (DIR);
 
@@ -518,34 +527,36 @@
 
 	my $subfile = $dir[$i];
-	my $this_file_base_dir = $base_dir;
 	last if ($maxdocs != -1 && ($count + $total_count) >= $maxdocs);
 	next if ($subfile =~ /^\.\.?$/);
 
-	my $file_subfile = &util::filename_cat($file, $subfile);
-	my $full_filename 
-	    = &util::filename_cat($this_file_base_dir,$file_subfile);
-
-	if ($self->file_is_blocked($block_hash,$full_filename)) {
-	    print STDERR "DirectoryPlugin: file $full_filename was blocked for read\n" if ($verbosity > 2);
+	my $this_file_base_dir = $base_dir;
+	my $raw_subfile = &unicode::url_encoded_to_raw_filename($subfile);
+
+	my $raw_file_subfile = &util::filename_cat($file, $raw_subfile);
+	my $raw_full_filename 
+	    = &util::filename_cat($this_file_base_dir,$raw_file_subfile);
+
+	if ($self->file_is_blocked($block_hash,$raw_full_filename)) {
+	    print STDERR "DirectoryPlugin: file $raw_full_filename was blocked for read\n" if ($verbosity > 2);
 	    next;
 	}
-	#print STDERR "processing $full_filename\n";
+	#print STDERR "processing $raw_full_filename\n";
 	# Follow Windows shortcuts
-	if ($subfile =~ /(?i)\.lnk$/ && $ENV{'GSDLOS'} =~ /^windows$/i) {
+	if ($raw_subfile =~ /(?i)\.lnk$/ && $ENV{'GSDLOS'} =~ /^windows$/i) {
 	    require Win32::Shortcut;
-	    my $shortcut = new Win32::Shortcut(&util::filename_cat($dirname, $subfile));
+	    my $shortcut = new Win32::Shortcut(&util::filename_cat($dirname, $raw_subfile));
 	    if ($shortcut) {
 		# The file to be processed is now the target of the shortcut
 		$this_file_base_dir = "";
 		$file = "";
-		$subfile = $shortcut->Path;
+		$raw_subfile = $shortcut->Path;
 	    }
 	}
 
 	# check for a symlink pointing back to a leading directory
-	if (-d "$dirname/$subfile" && -l "$dirname/$subfile") {
+	if (-d "$dirname/$raw_subfile" && -l "$dirname/$raw_subfile") {
 	    # readlink gives a "fatal error" on systems that don't implement
 	    # symlinks. This assumes the the -l test above would fail on those.
-	    my $linkdest=readlink "$dirname/$subfile";
+	    my $linkdest=readlink "$dirname/$raw_subfile";
 	    if (!defined ($linkdest)) {
 		# system error - file not found?
@@ -555,5 +566,5 @@
 		if ($linkdest =~ m@^[\./\\]+$@ ||
 		    index($dirname, $linkdest) != -1) {
-		    warn "DirectoryPlugin: Ignoring recursive symlink ($dirname/$subfile -> $linkdest)\n";
+		    warn "DirectoryPlugin: Ignoring recursive symlink ($dirname/$raw_subfile -> $linkdest)\n";
 		    next;
 		    ;
@@ -562,22 +573,23 @@
 	}
 
-	print $outhandle "DirectoryPlugin: preparing metadata for $subfile\n" if ($verbosity > 2);
-
-	# Make a copy of $in_metadata to pass to $subfile
+	print $outhandle "DirectoryPlugin: preparing metadata for $raw_subfile\n" if ($verbosity > 2);
+
+	# Make a copy of $in_metadata to pass to $raw_subfile
 	my $out_metadata = {};
 	&metadatautil::combine_metadata_structures($out_metadata, $in_metadata);
 
 	# check the assocfile_info
-	if (defined $self->{'assocfile_info'}->{$full_filename}) {
-	    &metadatautil::combine_metadata_structures($out_metadata, $self->{'assocfile_info'}->{$full_filename});
-	}
-        ## encode the filename as perl5 doesn't handle unicode filenames    
-       
-        my $tmpfile = Encode::encode_utf8($subfile); 
+	if (defined $self->{'assocfile_info'}->{$raw_full_filename}) {
+	    &metadatautil::combine_metadata_structures($out_metadata, $self->{'assocfile_info'}->{$raw_full_filename});
+	}
+
+	# $subfile by this point is url-encoded => all ASCII chars => no need to encode as UTF8
+
 	# Next add metadata read in XML files (if it is supplied)
 	if ($additionalmetadata == 1) {
 	    foreach my $filespec (@extrametakeys) {
-		## use the utf8 encoded filename to do the filename comparison
-		if ($tmpfile =~ /^$filespec$/) {
+		## use the url-encoded filename to do the filename comparison
+
+		if ($subfile =~ /^$filespec$/) {
 		    print $outhandle "File \"$subfile\" matches filespec \"$filespec\"\n" 
 			if ($verbosity > 2);
@@ -605,11 +617,11 @@
 	    # Look to see if it's a completely new file
 
-	    if (!$block_hash->{'new_files'}->{$full_filename}) {
+	    if (!$block_hash->{'new_files'}->{$raw_full_filename}) {
 		# Not a new file, must be an existing file
 		# Let' see if it's newer than the last import.pl
 
 
-		if (! -d $full_filename) {
-		    if (!$block_hash->{'reindex_files'}->{$full_filename}) {
+		if (! -d $raw_full_filename) {
+		    if (!$block_hash->{'reindex_files'}->{$raw_full_filename}) {
 			# filename has been around for longer than inf_timestamp
 			print $outhandle "**** Skipping $subfile\n" if ($verbosity >3);
@@ -634,5 +646,5 @@
 	
 	$count += &plugin::read ($pluginfo, $this_file_base_dir,
-				 $file_subfile, $block_hash,
+				 $raw_file_subfile, $block_hash,
 				 $out_metadata, $processor, $maxdocs, ($total_count + $count), $gli);
     }
Index: /main/trunk/greenstone2/perllib/plugins/HTMLPlugin.pm
===================================================================
--- /main/trunk/greenstone2/perllib/plugins/HTMLPlugin.pm	(revision 23334)
+++ /main/trunk/greenstone2/perllib/plugins/HTMLPlugin.pm	(revision 23335)
@@ -181,5 +181,6 @@
     my @file_blocks;
     
-    my ($language, $encoding) = $self->textcat_get_language_encoding ($filename_full_path);
+    my ($language, $content_encoding) = $self->textcat_get_language_encoding ($filename_full_path);
+    $self->{'store_content_encoding'}->{$filename_full_path} = $content_encoding;
 
     # read in file ($text will be in utf8)
@@ -229,5 +230,5 @@
 	# Convert the url_original_filename into its utf8 version. Store the utf8 link along with the url_original_filename
 	my $utf8_link = "";
-	$self->decode_text($link,$encoding,$language,\$utf8_link);
+	$self->decode_text($link,$content_encoding,$language,\$utf8_link);
 
 	$self->{'utf8_to_original_filename'}->{$utf8_link} = $url_original_filename;
@@ -272,4 +273,10 @@
         
     my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
+
+    # Lookup content_encoding worked out in file_block pass for this file
+    # Store it under the local name 'content_encoding' so its nice and 
+    # easy to access
+    $self->{'content_encoding'} = $self->{'store_content_encoding'}->{$filename_full_path};
+
     # get the input file
     my $input_filename = $file;
@@ -311,4 +318,8 @@
 	$self->set_Source_metadata($doc_obj, $filename_no_path);
     }
+
+    delete $self->{'store_content_encoding'}->{$filename_full_path};
+    $self->{'content_encoding'} = undef;
+
     return ($process_status,$doc_obj);
 }
@@ -367,7 +378,11 @@
     # links, so even if 'file_is_url' is off, still need to store info
 
-    my ($tailname,$dirname,$suffix) = &File::Basename::fileparse($file, "\\.[^\\.]+\$");
-    my $utf8_file = $self->filename_to_utf8_metadata($file);
-    $utf8_file =~ s/&\#095;/_/g;
+    my ($tailname,$dirname) = &File::Basename::fileparse($file);
+    print STDERR "***!! file = $file\n";
+#    my $utf8_file = $self->filename_to_utf8_metadata($file);
+#    $utf8_file =~ s/&\#095;/_/g;
+    my $utf8_file = &unicode::raw_filename_to_url_encoded($tailname);
+    print STDERR "***!! utf8_file = $utf8_file\n";
+
     my $web_url = "http://";
     if(defined $dirname) { # local directory
@@ -535,4 +550,5 @@
 	$self->process_section($textref, $base_dir, $file, $doc_obj, $cursection);
     }
+
     return 1;
 }
@@ -737,11 +753,23 @@
     return $front . $link . $back if $href =~ m/^(mailto|news|gopher|nntp|telnet|javascript):/is;
 
-
     if (($rl == 0) || ($filename =~ m/$self->{'process_exp'}/) || 
 	($href =~ m/\/$/) || ($href =~ m/^(mailto|news|gopher|nntp|telnet|javascript):/i)) {
+
+
+	# If web page didn't give encoding, then default to utf8
+	print "*************** looking up $file\n";
+
+	my $content_encoding= $self->{'content_encoding'} || "utf8";
+	$href = encode($content_encoding,$href);
+
+	$href = &unicode::raw_filename_to_url_encoded($href);
+	$href = &unicode::filename_to_url($href);
+
 	&ghtml::urlsafe ($href);
+	print STDERR "***!!! href=$href\n";    
+
 	return $front . "_httpextlink_&amp;rl=" . $rl . "&amp;href=" . $href . $hash_part . $back;
     } else {
-	# link is to some other type of file (eg image) so we'll
+	# link is to some other type of file (e.g., an image) so we'll
 	# need to associate that file
 	return $front . $self->add_file ($href, $rl, $hash_part, $base_dir, $doc_obj, $section) . $back;
@@ -769,8 +797,20 @@
 	$filename =~ s/([\\\/])tidytmp([\\\/])/$1import$2/;
     }
-    # Replace %XX's in URL with decoded value if required. Note that the filename may include the %XX in some
-    # situations. If the *original* file's name was in URL encoding, the following method will not decode it.
+
+    # Replace %XX's in URL with decoded value if required. Note that the
+    # filename may include the %XX in some situations. If the *original*
+    # file's name was in URL encoding, the following method will not decode
+    # it.
     my $utf8_filename = $filename;
-    $filename = $self->opt_url_decode($utf8_filename);
+    my $opt_decode_utf8_filename = $self->opt_url_decode($utf8_filename);
+
+    my $content_encoding= $self->{'content_encoding'} || "utf8";
+
+    # The filenames that come through the HTML file have been decoded
+    # into Unicode aware Perl strings.  Need to convert them back
+    # to their initial raw-byte encoding to match the file that
+    # exists on the file system
+    $filename = encode($content_encoding, $opt_decode_utf8_filename);
+
 
     # some special processing if the intended filename was converted to utf8, but
@@ -778,6 +818,12 @@
     if (!-e $filename) {
 	# try the original filename stored in map
-	my $original_filename = $self->{'utf8_to_original_filename'}->{$filename};
+	print STDERR "***###!! orig filename did not exist: $filename\n";
+
+	my $original_filename = $self->{'utf8_to_original_filename'}->{$utf8_filename};
+
+	print STDERR "**** Trying for $original_filename\n";
+
 	if (defined $original_filename && -e $original_filename) {
+	    print STDERR "*** found match\n";
 	    $filename = $original_filename;
 	}
@@ -1118,5 +1164,6 @@
 	$title =~ s/^$self->{'title_sub'}// if ($self->{'title_sub'});
 	$title =~ s/^\s+//s; # in case title_sub introduced any...
-	$doc_obj->add_utf8_metadata ($section, 'Title', $title);
+	print STDERR "**** adding Title: ", Encode::encode("utf8",$title), "\n";
+	$doc_obj->add_utf8_metadata ($section, "Title", $title);
 	print $outhandle " extracted Title metadata \"$title\" from $from\n" 
 	    if ($self->{'verbosity'} > 2);
Index: /main/trunk/greenstone2/perllib/plugins/ImageConverter.pm
===================================================================
--- /main/trunk/greenstone2/perllib/plugins/ImageConverter.pm	(revision 23334)
+++ /main/trunk/greenstone2/perllib/plugins/ImageConverter.pm	(revision 23335)
@@ -152,5 +152,5 @@
 sub generate_images {
     my $self = shift(@_);
-    my ($filename_full_path, $filename_no_path, $doc_obj, $section) = @_;
+    my ($filename_full_path, $filename_no_path, $doc_obj, $section, $filename_encoding) = @_;
 
     # check image magick status
@@ -196,5 +196,5 @@
 
     if ($converttotype ne "" && $filename_full_path !~ m/$converttotype$/) {
-	#    $doc_obj->add_utf8_metadata($section, "Image", $utf8_filename_meta);
+#	#    $doc_obj->add_utf8_metadata($section, "Image", $utf8_filename_meta);
 
 	my ($result, $converted_filename_full_path)
@@ -214,6 +214,12 @@
 
     # here we overwrite the original with the potentially converted one
-    $doc_obj->set_utf8_metadata_element($section, "Source", &unicode::url_decode($filename_no_path)); # displayname of generated image
-    $doc_obj->set_utf8_metadata_element($section, "SourceFile", $url_to_filename_no_path); # displayname of generated image
+#    $doc_obj->set_utf8_metadata_element($section, "Source", &unicode::url_decode($filename_no_path)); # displayname of generated image
+#    $doc_obj->set_utf8_metadata_element($section, "SourceFile", $url_to_filename_no_path); # displayname of generated image
+
+#    $self->set_Source_metadata($doc_obj,$url_to_filename_no_path,undef);
+
+    $self->set_Source_metadata($doc_obj,&unicode::url_decode($filename_no_path),
+			       $filename_encoding);
+
 
     # use identify to get info about the (possibly converted) image
Index: /main/trunk/greenstone2/perllib/plugins/ImagePlugin.pm
===================================================================
--- /main/trunk/greenstone2/perllib/plugins/ImagePlugin.pm	(revision 23334)
+++ /main/trunk/greenstone2/perllib/plugins/ImagePlugin.pm	(revision 23335)
@@ -110,5 +110,4 @@
     my ($pluginfo, $base_dir, $file, $metadata, $doc_obj, $gli) = @_;
 
-
     my $outhandle = $self->{'outhandle'};
     my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
@@ -116,7 +115,18 @@
     if ($self->{'image_conversion_available'} == 1)
     {
-	my $utf8_filename_no_path = $self->filepath_to_utf8($filename_no_path);
-	my $url_encoded_filename = &util::rename_file($utf8_filename_no_path, $self->{'file_rename_method'});
-	$self->generate_images($filename_full_path, $url_encoded_filename, $doc_obj, $doc_obj->get_top_section()); # should we check the return value?
+	my $filename_encoding = $self->deduce_filename_encoding($file,$metadata);
+
+#	my $utf8_filename_no_path = $self->filepath_to_utf8($filename_no_path);
+#	my $url_encoded_filename = &util::rename_file($utf8_filename_no_path, $self->{'file_rename_method'});
+
+#	$self->generate_images($filename_full_path, $url_encoded_filename, 
+#			       $doc_obj, $doc_obj->get_top_section()); # should we check the return value?
+
+	$filename_no_path = &unicode::raw_filename_to_url_encoded($filename_no_path);
+
+	# should we check the return value?
+	$self->generate_images($filename_full_path, $filename_no_path, 
+			       $doc_obj, $doc_obj->get_top_section(),$filename_encoding); 
+
     }
     else
Index: /main/trunk/greenstone2/perllib/plugins/ReadTextFile.pm
===================================================================
--- /main/trunk/greenstone2/perllib/plugins/ReadTextFile.pm	(revision 23334)
+++ /main/trunk/greenstone2/perllib/plugins/ReadTextFile.pm	(revision 23335)
@@ -128,7 +128,7 @@
 
     # Do encoding stuff
-    my ($language, $encoding) = $self->textcat_get_language_encoding ($filename_full_path);
+    my ($language, $content_encoding) = $self->textcat_get_language_encoding ($filename_full_path);
     if ($self->{'verbosity'} > 2) {
-	print $outhandle "ReadTextFile: reading $file as ($encoding,$language)\n";
+	print $outhandle "ReadTextFile: reading $file as ($content_encoding,$language)\n";
     }
 
@@ -140,12 +140,14 @@
     $doc_obj->add_utf8_metadata($top_section, "Plugin", "$self->{'plugin_type'}");
     $doc_obj->add_utf8_metadata($top_section, "FileSize", (-s $filename_full_path));
-    $self->set_Source_metadata($doc_obj, $filename_no_path, $encoding);
+
+    my $filename_encoding = $self->deduce_filename_encoding($file,$metadata);
+    $self->set_Source_metadata($doc_obj, $filename_no_path, $filename_encoding);
 
     $doc_obj->add_utf8_metadata($top_section, "Language", $language);
-    $doc_obj->add_utf8_metadata($top_section, "Encoding", $encoding);
+    $doc_obj->add_utf8_metadata($top_section, "Encoding", $content_encoding);
     
     # read in file ($text will be in utf8)
     my $text = "";
-    $self->read_file ($filename_full_path, $encoding, $language, \$text);
+    $self->read_file ($filename_full_path, $content_encoding, $language, \$text);
 
     if (!length ($text)) {
