Index: /main/trunk/gli/src/org/greenstone/gatherer/metadata/DocXMLFile.java
===================================================================
--- /main/trunk/gli/src/org/greenstone/gatherer/metadata/DocXMLFile.java	(revision 34393)
+++ /main/trunk/gli/src/org/greenstone/gatherer/metadata/DocXMLFile.java	(revision 34394)
@@ -41,6 +41,11 @@
 public abstract class DocXMLFile extends File
 {
+    static boolean isWin = Utility.isWindows();
+    // For Linux, we continue using gsdlsourcefilename as key to the metadata mapping
+    // For Windows, we use the hex encoded long file paths as key
+    static String GSDL_SOURCE_FILE_METANAME = isWin ? "gsdlfullsourcepath" : "gsdlsourcefilename";
+    
     protected HashMap source_file_name_to_description_elements_mapping = new HashMap();
-
+    
     protected final String MetadataWrap;
     protected final String MetadataItem;
@@ -57,104 +62,11 @@
     }
 
-	/**
-	 * Checks if various versions of the file object's filename, denoted relatively by file_relative_path,
-	 * occur in the source_file_name_to_description_elements_mapping map
-	*/
-	private ArrayList findSourceFileMapKeyMatch(File file, String file_relative_path) {
-		ArrayList description_elements_list = null;
-				
-		///System.err.println("Looking for key " + file_relative_path);
-		description_elements_list = (ArrayList) source_file_name_to_description_elements_mapping.get(file_relative_path);
-		if(description_elements_list != null) {
-			///System.err.println("   Found key matching REGULAR filepath: " + file_relative_path);
-			return description_elements_list;			
-		}
-		else if(!Utility.isWindows()) { // couldn't find a matching key, we're done
-			///System.err.println("Unable to find ex.meta for regular file path form " + file_relative_path);
-			return null;
-		}
-		
-		// Now we can try windows short filename as map key
-		
-		String win_short_file_relative_path = "";
-		try{
-			win_short_file_relative_path = Utility.getWindowsShortFileName(file.getAbsolutePath());				
-			//System.err.println("@@@ Searching for short file name: " + win_short_file_relative_path);
-		} catch(Exception e) { // we're done trying to find a matching key
-			System.err.println("Failed to convert to windows short file name: " + win_short_file_relative_path);			
-			return null;
-		}
-		
-		// Got a windows short file name, lop off import folder again
-		int import_index = win_short_file_relative_path.indexOf("import");
-		if (import_index != -1) {
-			win_short_file_relative_path = win_short_file_relative_path.substring(import_index + "import".length() + 1);
-		}
-			
-		///System.err.println("### Looking for Windows short file name |" + win_short_file_relative_path +  "| in map of sourcefilenames to doc.xml's ex meta.");
-		description_elements_list = (ArrayList) source_file_name_to_description_elements_mapping.get(win_short_file_relative_path);
-		if (description_elements_list != null) {
-			///System.err.println("   Found key matching FULL win shortfile path: " + win_short_file_relative_path);
-			return description_elements_list; // found
-		}
-		
-		// else, check whether a map key is matched by any REMAINING combination of windows shortfile path and regular path:
-		// - windows shortfilename's rel-dir-path with regular tailname
-		// - and regular rel-dir-path with windows shortfilename's tailname
-				
-		String shortFileTailName = win_short_file_relative_path;
-		String shortFileRelDirPath = "";
-		int lastSep = win_short_file_relative_path.lastIndexOf(File.separator);
-		if(lastSep != -1) {			
-			shortFileTailName = win_short_file_relative_path.substring(lastSep+1);
-			shortFileRelDirPath = win_short_file_relative_path.substring(0, lastSep+1); // include the slash
-		}
-		
-		String fileTailName = file_relative_path;
-		String fileRelDirPath = "";
-		lastSep = file_relative_path.lastIndexOf(File.separator);
-		if(lastSep != -1) {			
-			fileTailName = file_relative_path.substring(lastSep+1);
-			fileRelDirPath = file_relative_path.substring(0, lastSep+1); // include the slash
-		}
-		
-		String path = shortFileRelDirPath + fileTailName;
-		///System.err.println("### Looking for Windows short file name |" + path +  "| in map of sourcefilenames to doc.xml's ex meta.");
-		description_elements_list = (ArrayList) source_file_name_to_description_elements_mapping.get(path);
-		
-		if(description_elements_list != null) {
-			///System.err.println("   Found key matching MIX of win shortfile path and regular path: " + path);
-			return description_elements_list; // found
-		}
-
-		// try the other combination
-		path = fileRelDirPath + shortFileTailName;
-		///System.err.println("### Looking for Windows short file name |" + path +  "| in map of sourcefilenames to doc.xml's ex meta.");
-		description_elements_list = (ArrayList) source_file_name_to_description_elements_mapping.get(path);
-		
-		if(description_elements_list != null) {
-			///System.err.println("   Found key matching MIX of regular path and win shortfile path: " + path);
-			return description_elements_list; // found
-		}		
-		
-		// could not find gsdlsourcefilename in map
-		///System.err.println("Unable to find ex.meta for regular file path form " + file_relative_path);
-		///System.err.println("    Or for windows shortFile path form, or for combinations with regular file path form");		
-		
-		return description_elements_list; // returns null at this point
-	}
-	
-
-    public ArrayList getMetadataExtractedFromFile(File file)
+    /** On Windows, file_relative_path will be hex-encoded for codepts beyond ASCII.
+     * But keys into the source_file_name_to_description_elements_mapping will then also match on Windows */
+    public ArrayList getMetadataExtractedFromFile(File file, String file_relative_path)												
     {
 	// Build up a list of metadata extracted from this file
 	ArrayList metadata_values = new ArrayList();
-
-	String file_relative_path = file.getAbsolutePath();
-	int import_index = file_relative_path.indexOf("import");
-	if (import_index != -1) {
-	    file_relative_path = file_relative_path.substring(import_index + "import".length() + 1);
-	}
-
+	
 	///for (Object relFilename : source_file_name_to_description_elements_mapping.keySet()) {
 	///    System.err.println("\n@@@ relFilename: " + relFilename);
@@ -162,8 +74,8 @@
 	
 	// Check whether this file (i.e. doc.xml or docmets.xml on inheritance) file contains extracted metadata for the specified file
-	//ArrayList description_elements_list = (ArrayList) source_file_name_to_description_elements_mapping.get(file_relative_path);
-	ArrayList description_elements_list = findSourceFileMapKeyMatch(file, file_relative_path);
+	ArrayList description_elements_list = (ArrayList) source_file_name_to_description_elements_mapping.get(file_relative_path);
 	if (description_elements_list == null) {
 			// ...it doesn't
+			///System.err.println("Unable to find meta for file path form " + file_relative_path);
 			return metadata_values; // we're done
 	}
@@ -337,5 +249,11 @@
 		
 		// Note which file this is for
-		else if (metadata_element_name.equals("gsdlsourcefilename")) {
+		//else if (metadata_element_name.equals("gsdlsourcefilename")) {	
+		else if (metadata_element_name.equals(GSDL_SOURCE_FILE_METANAME)) {
+		    // On Unix, GSDL_SOURCE_FILE_METANAME is the gsdlsourcefilename metadata field
+		    // which may be encoded by the encoding denoted in fileRenameMethod (and will need decoding)
+		    // On Windows, GSDL_SOURCE_FILE_METANAME is a different metadata field that
+		    // will be hex encoded for non-ASCII chars
+		    
 		    // Extract the gsdlsourcefilename element value
 		    int value_index = line.indexOf(">", name_index) + ">".length();
@@ -354,9 +272,9 @@
 
 			// Make sure the path matches the OS that is running
-			if (is_unix_path && Utility.isWindows()) {
+			if (is_unix_path && isWin) {
 			    // Convert path from Unix to Windows
 			    gsdlsourcefilename_value = gsdlsourcefilename_value.replaceAll("\\/", "\\\\");
 			}
-			else if (!is_unix_path && !Utility.isWindows()) {
+			else if (!is_unix_path && !isWin) {
 			    // Convert path from Windows to Unix
 			    gsdlsourcefilename_value = gsdlsourcefilename_value.replaceAll("\\\\", "/");
@@ -369,4 +287,8 @@
 			}
 
+			// Would be better to store hex src file name decoded? But how do we know what encoding the filename is in
+			// https://stackoverflow.com/questions/13990941/how-to-convert-hex-string-to-java-string
+			
+			
 			((ArrayList) source_file_name_to_description_elements_mapping.get(gsdlsourcefilename_value)).add(new Integer(description_element_start));
 		    }
@@ -383,5 +305,5 @@
 		    }
 		}
-
+		
 		// Ignore metadata starting with gsdl (used to be lowercase metadata and /metadata)
 		if (metadata_element_name.startsWith("gsdl")) {
@@ -399,20 +321,27 @@
 	    buffered_reader.close();
 
-	    // Now that we're done skimming, we actually need to decode gsdlsourcefilename
-	    // based on whatever fileRenameMethod was used to encode it, so that we can
-	    // at last properly compare properly against filenames on the file system
-	    // in order to load the correct ex.meta for the file.
-	    // Now that we should have both gsdlsourcefilename AND fileRenameMethod set,
-	    // we can finally perform the decoding of gsdlsourcefilename.	    
-	    if(fileRenameMethod == null) {
-		fileRenameMethod = FILE_RENAME_METHOD_URL; // default for building
+	    // ON WINDOWS, we're working with hex encoded full file path instead of with gsdlsourcefilename,
+	    // so needn't bother decoding gsdlsourcefilename as it's unused.
+	    // On UNIX, continue decoding gsdlsourcefilename as before
+	    if(!isWin) {
+		// Now that we're done skimming, we actually need to decode gsdlsourcefilename
+		// based on whatever fileRenameMethod was used to encode it, so that we can
+		// at last properly compare properly against filenames on the file system
+		// in order to load the correct ex.meta for the file.
+		// Now that we should have both gsdlsourcefilename AND fileRenameMethod set,
+		// we can finally perform the decoding of gsdlsourcefilename.	    
+		if(fileRenameMethod == null) {
+		    fileRenameMethod = FILE_RENAME_METHOD_URL; // default for building
+		}
+		
+		// If gsdlsourcefilename was encoded, we remove it from the map under its encoded
+		// filename, decode it and add it back into map using its decoded filename.
+		if(!fileRenameMethod.equals(FILE_RENAME_METHOD_NONE)) {
+		    ArrayList value_list = (ArrayList) source_file_name_to_description_elements_mapping.remove(gsdlsourcefilename_value);
+		    gsdlsourcefilename_value = decodeSourceFilename(gsdlsourcefilename_value, fileRenameMethod, is_unix_path);
+		    source_file_name_to_description_elements_mapping.put(gsdlsourcefilename_value, value_list);
+		}
 	    }
-	    // If gsdlsourcefilename was encoded, we remove it from the map under its encoded
-	    // filename, decode it and add it back into map using its decoded filename.
-	    if(!fileRenameMethod.equals(FILE_RENAME_METHOD_NONE)) {		
-		ArrayList value_list = (ArrayList) source_file_name_to_description_elements_mapping.remove(gsdlsourcefilename_value);		
-		gsdlsourcefilename_value = decodeSourceFilename(gsdlsourcefilename_value, fileRenameMethod, is_unix_path);
-		source_file_name_to_description_elements_mapping.put(gsdlsourcefilename_value, value_list);
-	    }		
+	    
 	}
 	catch (FileNotFoundException exception) {
@@ -708,9 +637,9 @@
 
 			// Make sure the path matches the OS that is running
-			if (is_unix_path && Utility.isWindows()) {
+			if (is_unix_path && isWin) {
 			    // Convert path from Unix to Windows
 			    gsdlsourcefilename_value = gsdlsourcefilename_value.replaceAll("\\/", "\\\\");
 			}
-			else if (!is_unix_path && !Utility.isWindows()) {
+			else if (!is_unix_path && !isWin) {
 			    // Convert path from Windows to Unix
 			    gsdlsourcefilename_value = gsdlsourcefilename_value.replaceAll("\\\\", "/");
Index: /main/trunk/gli/src/org/greenstone/gatherer/metadata/DocXMLFileManager.java
===================================================================
--- /main/trunk/gli/src/org/greenstone/gatherer/metadata/DocXMLFileManager.java	(revision 34393)
+++ /main/trunk/gli/src/org/greenstone/gatherer/metadata/DocXMLFileManager.java	(revision 34394)
@@ -31,5 +31,5 @@
 import java.util.*;
 import org.greenstone.gatherer.DebugStream;
-
+import org.greenstone.gatherer.util.Utility;
 
 /** This class is a static class that manages the doc.xml files */
@@ -37,5 +37,4 @@
 {
     static private ArrayList doc_xml_files = new ArrayList();
-
 
     static public void clearDocXMLFiles()
@@ -47,4 +46,13 @@
     static public ArrayList getMetadataExtractedFromFile(File file)
     {
+	// Work out relative file path and its hex encoded value here,
+	// avoids making DocXMLFile.java recalculate these each time
+	String file_relative_path = file.getAbsolutePath();
+	int import_index = file_relative_path.indexOf("import");
+	if (import_index != -1) {
+	    file_relative_path = file_relative_path.substring(import_index + "import".length() + 1);
+	}
+	String searchFileName = DocXMLFile.isWin ? Utility.stringToHex(file_relative_path) : file_relative_path;
+		
 	// Build up a list of metadata values extracted from this file
 	ArrayList metadata_values = new ArrayList();
@@ -53,5 +61,6 @@
 	for (int i = 0; i < doc_xml_files.size(); i++) {
 	    DocXMLFile doc_xml_file = (DocXMLFile) doc_xml_files.get(i);
-	    metadata_values.addAll(doc_xml_file.getMetadataExtractedFromFile(file));
+	    ///System.err.println("@@@@ Looking at doc.xml file: " + doc_xml_files.get(i));
+	    metadata_values.addAll(doc_xml_file.getMetadataExtractedFromFile(file, searchFileName));
 	}
 
Index: /main/trunk/gli/src/org/greenstone/gatherer/util/Utility.java
===================================================================
--- /main/trunk/gli/src/org/greenstone/gatherer/util/Utility.java	(revision 34393)
+++ /main/trunk/gli/src/org/greenstone/gatherer/util/Utility.java	(revision 34394)
@@ -94,27 +94,44 @@
 	}	
     }
-
-	// Copied from GS3 main java code at GSDL3SRCHOME\src\java\org\greenstone/util\Misc.java
-	// Debugging function to print a string's non-basic chars in hex, so stringToHex on all non-basic and non-printable ASCII
-	// Dr Bainbridge said that printing anything with charCode over 128 in hex is okay, but I'd already made extra allowances for non-printable ASCII
-	// Based on https://stackoverflow.com/questions/923863/converting-a-string-to-hexadecimal-in-java
-	public static String debugUnicodeString(String str) {
-	  String result = "";
-	  for(int i = 0; i < str.length(); i++) {
-			int charCode = str.codePointAt(i); // unicode codepoint / ASCII code
-			
-			// ASCII table: https://cdn.sparkfun.com/assets/home_page_posts/2/1/2/1/ascii_table_black.png
-			// If the unicode character code pt is less than the ASCII code for space and greater than for tilda, let's display the char in hex (x0000 format)
-			if((charCode >= 20 && charCode <= 126) || charCode == 9 || charCode == 10 || charCode == 13) { // space, tilda, TAB, LF, CR are printable, leave them in for XML element printing
-				result += str.charAt(i);
-			} else {
-				result += "x{" + String.format("%04x", charCode) + "}"; // looks like: x{4-char-codepoint}
-			}
-	  }
-	  
-	  return result;
-	}
-	
-	/**
+    
+    // Copied from GS3 main java code at GSDL3SRCHOME\src\java\org\greenstone/util\Misc.java
+    // Debugging function to print a string's non-basic chars in hex, so stringToHex on all non-basic and non-printable ASCII
+    // Dr Bainbridge said that printing anything with charCode over 128 in hex is okay, but I'd already made extra allowances for non-printable ASCII
+    // Based on https://stackoverflow.com/questions/923863/converting-a-string-to-hexadecimal-in-java
+    public static String debugUnicodeString(String str) {
+	String result = "";
+	for(int i = 0; i < str.length(); i++) {
+	    int charCode = str.codePointAt(i); // unicode codepoint / ASCII code
+	    
+	    // ASCII table: https://cdn.sparkfun.com/assets/home_page_posts/2/1/2/1/ascii_table_black.png
+	    // If the unicode character code pt is less than the ASCII code for space and greater than for tilda, let's display the char in hex (x0000 format)
+	    if((charCode >= 20 && charCode <= 126) || charCode == 9 || charCode == 10 || charCode == 13) { // space, tilda, TAB, LF, CR are printable, leave them in for XML element printing
+		result += str.charAt(i);
+	    } else {
+		result += "x{" + String.format("%04x", charCode) + "}"; // looks like: x{4-char-codepoint}				
+	    }
+	}
+	
+	return result;
+    }
+    
+    // Version of debugUnicodeString that, on Windows, mimics perl unicode::debug_unicode_string
+    // exactly by producing hex/unicode codepoints for ALL codepoints beyond ASCII
+    public static String stringToHex(String str) {
+	String result = "";
+	for(int i = 0; i < str.length(); i++) {
+	    int charCode = str.codePointAt(i); // unicode codepoint / ASCII code
+	    
+	    if(charCode <=127) { // ASCII
+		result += str.charAt(i);
+	    } else { // non-ASCII
+		result += "\\x{" + String.format("%04x", charCode) + "}"; // looks like: \x{4-char-codepoint}
+	    }
+	}    
+	
+	return result;
+    }
+	
+    /**
      * returns the short filename (8.3) for a file in Windows
      * 
Index: /main/trunk/greenstone2/perllib/doc.pm
===================================================================
--- /main/trunk/greenstone2/perllib/doc.pm	(revision 34393)
+++ /main/trunk/greenstone2/perllib/doc.pm	(revision 34394)
@@ -125,5 +125,15 @@
 		# For Unix-based systems, there is no difference between the two
 		$self->{'source_path'} = $source_filename;
-	    }
+	    }	    
+
+	    # On Windows, the code above has ensured source_path is the Win long (full) path name.
+	    # To help GLI associate metadata with an easily calculated and accurate representation of
+	    # filenames, we now store the Win long path name, hex encoded.
+	    # We're not using this field on Linux, as I can't get the hex encodings generated to match
+	    # what GLI Java code generates. But for symmetry we store this field on Unix too, but we need
+	    # to hex-encode source_path on Unix too, or it may not be UTF-8 and doc.xml will be invalid
+	    my $hexencodedlongsourcepath = &unicode::debug_unicode_string($self->{'source_path'});	    
+	    $self->set_utf8_metadata_element ($self->get_top_section(), "gsdlfullsourcepath", $hexencodedlongsourcepath);
+	   
 	}
 	else {
