Index: /main/trunk/greenstone2/perllib/plugins/HTMLPlugin.pm
===================================================================
--- /main/trunk/greenstone2/perllib/plugins/HTMLPlugin.pm	(revision 31439)
+++ /main/trunk/greenstone2/perllib/plugins/HTMLPlugin.pm	(revision 31440)
@@ -186,7 +186,8 @@
     
     my ($language, $content_encoding) = $self->textcat_get_language_encoding ($filename_full_path);
-    $self->{'store_content_encoding'}->{$filename_full_path} = $content_encoding;
-
-    # read in file ($text will be in utf8)
+    $self->{'store_content_encoding'}->{$filename_full_path} = [$content_encoding, $language];
+    
+
+    # read in file ($text will be in the filesystem encoding)
     my $raw_text = "";
     $self->read_file_no_decoding($filename_full_path, \$raw_text);
@@ -296,5 +297,5 @@
 	&util::block_filename($block_hash,$url_original_filename) if $url_original_filename ne $html_fname;
 
-			# but only add the linked file to the blocklist if the current html file does not link to itself
+	# but only add the linked file to the blocklist if the current html file does not link to itself
 		
     }
@@ -332,8 +333,8 @@
     my ($filename_full_path, $filename_no_path) = &util::get_full_filenames($base_dir, $file);
 
-    # Lookup content_encoding worked out in file_block pass for this file
-    # Store it under the local name 'content_encoding' so its nice and 
-    # easy to access
-    $self->{'content_encoding'} = $self->{'store_content_encoding'}->{$filename_full_path};
+    # Lookup content_encoding and language worked out in file_block pass for this file
+    # Store them under the local names they are nice and easy to access
+    $self->{'content_encoding'} = $self->{'store_content_encoding'}->{$filename_full_path}[0];
+    $self->{'language'} = $self->{'store_content_encoding'}->{$filename_full_path}[1];
 
     # get the input file
@@ -463,8 +464,10 @@
     my $url_encoded_file = &unicode::raw_filename_to_url_encoded($tailname);
     my $utf8_url_encoded_file = &unicode::raw_filename_to_utf8_url_encoded($tailname);
-
+    
     my $web_url = "http://";
     my $utf8_web_url = "http://";
+    
     if(defined $dirname) { # local directory
+
         # Check for "ftp" in the domain name of the directory
         #  structure to determine if this URL should be a ftp:// URL
@@ -481,6 +484,11 @@
 	$dirname .= &util::get_dirsep() if $dirname ne ""; # if there's a directory, it should end on "/"
 
-	$web_url = $web_url.$dirname.$url_encoded_file; 
-	$utf8_web_url = $utf8_web_url.$dirname.$utf8_url_encoded_file; 
+	# this local directory in import may need to be URL encoded like the file
+	my $url_encoded_dir = &unicode::raw_filename_to_url_encoded($dirname);
+	my $utf8_url_encoded_dir =  &unicode::raw_filename_to_utf8_url_encoded($dirname);
+	
+	# changed here
+	$web_url = $web_url.$url_encoded_dir.$url_encoded_file; 
+	$utf8_web_url = $utf8_web_url.$utf8_url_encoded_dir.$utf8_url_encoded_file; 
     } else {
 	$web_url = $web_url.$url_encoded_file;
@@ -493,4 +501,6 @@
 	print STDERR "*******DEBUG: upgraded_file:       $upgraded_file\n";
 	print STDERR "*******DEBUG: adding URL metadata: $utf8_url_encoded_file\n";
+	print STDERR "*******DEBUG: web url:             $web_url\n";
+	print STDERR "*******DEBUG: utf8 web url:        $utf8_web_url\n";
     }
 
@@ -847,4 +857,10 @@
     }
 
+    # can't remember adding this :-( must have had a reason though...
+    if ($link =~ /^\_http/ || $link =~ /^\_libraryname\_/) {
+	# assume it is a greenstone one and leave alone
+	return $front . $link . $back;
+    }
+
     # attempt to sort out targets - frames are not handled 
     # well in this plugin and some cases will screw things
@@ -912,14 +928,23 @@
 	    $href = encode($content_encoding,$href);
 	}
-
-	$href = &unicode::raw_filename_to_utf8_url_encoded($href);	
+	
+	$href = &unicode::raw_filename_to_utf8_url_encoded($href); 
 	$href = &unicode::filename_to_url($href);
-
+	
 	&ghtml::urlsafe ($href);
-
+	
 	if ((defined $ENV{"DEBUG_UNICODE"}) && ($ENV{"DEBUG_UNICODE"})) {
 		print STDERR "******DEBUG: href=$href\n";    
 	}
-
+	
+	#TODO here
+#	if ($rl ==1) {
+	    # have a relative link, we need to do URL encoding etc so it matches what has happened for that file
+	    #$href = &util::rename_file($href, $self->{'file_rename_method'});
+#	    $href = &unicode::raw_filename_to_url_encoded($href);
+	    # then, this might be url encoded, so we replace % with %25
+#	    $href = &unicode::filename_to_url($href);
+#	    print STDERR "DEBUG: url encoded href = $href\n";
+#	}
 
 	return $front . "_httpextlink_&amp;rl=" . $rl . "&amp;href=" . $href . $hash_part . $back;
@@ -1022,4 +1047,5 @@
 	# exists on the file system
 	$filename = encode($content_encoding, $opt_decode_unicode_filename);
+	
     }
     elsif ($ENV{'GSDLOS'} =~ /^darwin$/i) {
@@ -1051,5 +1077,4 @@
     # some special processing if the intended filename was converted to utf8, but
     # the actual file still needs to be renamed
-    #if (!&util::fd_exists($filename)) {
     if (!&FileUtils::fileExists($filename)) {
 	# try the original filename stored in map
@@ -1071,5 +1096,5 @@
 		}
 	    $filename = $original_filename;
-	}
+	} 
     }
     
@@ -1160,5 +1185,5 @@
     my $self = shift (@_);
     my ($link, $base_dir, $file) = @_;
-
+ 
     # strip off hash part, e.g. #foo, but watch out for any entities, e.g. &#x3B1;
     my ($before_hash, $hash_part) = $link =~ m/^(.*?[^&])(\#.*)?$/;
@@ -1168,11 +1193,12 @@
 		my $outhandle = $self->{'outhandle'};
 		print $outhandle "HTMLPlugin: ERROR - badly formatted tag ignored ($link)\n"
-			if $self->{'verbosity'};
+		    if $self->{'verbosity'};
 		return ($link, "", 0);
     }
 	
+#    my $dirname;
     if ($before_hash =~ s@^((?:http|https|ftp|file|mms)://)@@i) {
-		my $type = $1;
-		my $before_hash_file = $before_hash;
+	my $type = $1;
+	my $before_hash_file = $before_hash;
 		
 	if ($link =~ m/^(http|ftp):/i) {
@@ -1187,12 +1213,18 @@
 	my $before_hash_url = $before_hash_file;
 	if ($ENV{'GSDLOS'} =~ /^windows$/i) {
-		$before_hash_url =~ s@\\@\/@g;
-	}
-	
+	    $before_hash_url =~ s@\\@\/@g;
+	}
+	
+	######## TODO need to check this for encoding stufff
 	my $linkfilename = &FileUtils::filenameConcatenate($base_dir, $before_hash_file);
-	
+	print STDERR "chekcing for existence whether relative link or not $linkfilename\n";
 	my $rl = 0;
 	$rl = 1 if (-e $linkfilename);
-
+	if (-e $linkfilename) {
+	    
+	    print STDERR "DOES exist $linkfilename\n";
+	} else {
+	    print STDERR "DOESN'T exist $linkfilename\n";
+	}
 	# make sure there's a slash on the end if it's a directory
 	if ($before_hash_url !~ m/\/$/) {
@@ -1203,4 +1235,5 @@
     } elsif ($link !~ m/^(mailto|news|gopher|nntp|telnet|javascript):/i && $link !~ m/^\//) {
 
+	#### TODO whst is this test doing???
 	if ($before_hash =~ s@^/@@ || $before_hash =~ m/\\/) {
 
@@ -1240,19 +1273,38 @@
 	    }
 	} else {
-	    # Turn relative file path into full path
+
+	    # Turn relative file path into full path (inside import dir)
 	    my $dirname = &File::Basename::dirname($file);
-	    $before_hash = &FileUtils::filenameConcatenate($dirname, $before_hash);
+
+	    # we want to add dirname (which is raw filesystem path) onto $before_hash, (which is perl unicode aware string). Convert dirname to perl string
+
+	    my $unicode_dirname ="";
+	    #my $content_encoding = $self->{'content_encoding'};
+	    #my $language = $self->{'language'};
+
+	    # actually I think this is wrong. why should we use content encoding?
+	    #$self->decode_text($dirname, $content_encoding, $language, \$unicode_dirname);
+	    #my $filename_encoding = $self->{'filename_encoding'};
+	    # filename_encoding might be auto...
+
+	    # TODO what is the best thing to do here?????
+	    # try and guess default filesystem encoding, similar to deduce_filename_encoding, but without a file?
+	    my $filename_encoding = "utf8";
+	    # copied this from set_Source_metadata in BasePlugin
+	    if ((defined $filename_encoding) && ($filename_encoding ne "ascii")) {
+		# Use filename_encoding to map raw filename to a Perl unicode-aware string 
+		$unicode_dirname = decode($filename_encoding,$dirname);		
+	    }
+	    else {
+		# otherwise generate %xx encoded version of filename for char > 127
+		$unicode_dirname = &unicode::raw_filename_to_url_encoded($dirname);
+	    }
+ 
+	    $before_hash = &FileUtils::filenameConcatenate($unicode_dirname, $before_hash);
 	    $before_hash = $self->eval_dir_dots($before_hash);	 
-		$before_hash =~ s@\\@/@g; # for windows   	    
-	}
-
-	my $linkfilename = &FileUtils::filenameConcatenate($base_dir, $before_hash); 
-
-
-#	print STDERR "**** linkfilename = $linkfilename\n";
-#	if (!&util::fd_exists($linkfilename)) {
-#	    print STDERR "***** Warning: Could not find $linkfilename\n";
-#	}
-
+	    $before_hash =~ s@\\@/@g; # for windows   	    
+	}
+
+	my $linkfilename = &FileUtils::filenameConcatenate($base_dir, $before_hash); 	
 
 	# make sure there's a slash on the end if it's a directory
@@ -1260,7 +1312,4 @@
 	    $before_hash .= "/" if (-d $linkfilename);
 	}
-
-#	print STDERR "*** returning: $before_hash\n";
-
 	return ("http://" . $before_hash, $hash_part, 1);
     } else {
