Index: /main/trunk/gli/src/org/greenstone/gatherer/metadata/FilenameEncoding.java
===================================================================
--- /main/trunk/gli/src/org/greenstone/gatherer/metadata/FilenameEncoding.java	(revision 34414)
+++ /main/trunk/gli/src/org/greenstone/gatherer/metadata/FilenameEncoding.java	(revision 34415)
@@ -77,9 +77,9 @@
 	public static final Pattern HEX_PATTERN = Pattern.compile("(&#x[0-9a-zA-Z]{1,4}+;)");
 
-    /** The hex entity version of the ampersand character.
+    /** The hex version of the ampersand character: previously hex entity (&#x26) now hex url encoded (%26).
      * We use this in place of the ampersand character in filenames in metadata.xml files to
      * preserve the reference to the literal ampersand in the real file name on the file system.
      */
-    public static final String HEX_ENTITY_AMPERSAND = FilenameEncoding.hexEntityForChar("&"); //"&#x26;";
+    public static final String HEX_AMPERSAND = "%26"; //= FilenameEncoding.hexEntityForChar("&"); //"&#x26;";
     
 
@@ -257,12 +257,6 @@
 	
     /** URL encoded version of the byte codes of the given file's name */
-    public static String calcURLEncodedFilePath(File file) {	
-		if(!MULTIPLE_FILENAME_ENCODINGS_SUPPORTED) {
-		    return file.getAbsolutePath();
-		}
-		else {
-		    String filename = fileToURLEncoding(file);
-		    return filename;
-		}
+    public static String calcURLEncodedFilePath(File file) {
+		return fileToURLEncoding(file);
     }
 
@@ -380,7 +374,5 @@
 	// just return input filename param, but with any & in the filename replaced with its hex entity
 		if(!MULTIPLE_FILENAME_ENCODINGS_SUPPORTED) {
-		    // protect ampersands in filenames by converting it to its hex entity
 		    String filepath = file.getAbsolutePath();
-		    filepath = filepath.replace("&", HEX_ENTITY_AMPERSAND);
 		    return filepath;
 		}
@@ -430,5 +422,5 @@
 			
 			// Before proceeding, protect & in the filename too.
-			// &'s ASCII code is 36 in decimal, and 26 in hex, so replace with &#x26; (HEX_ENTITY_AMPERSAND)
+			// &'s ASCII code is 36 in decimal, and 26 in hex, so replace with &#x26; (HEX_AMPERSAND)
 			// But dangerous to do simple replace if there are &#x...; entities in the filename already!
 			// That is, we'll want to protect & by replacing with &'s hex value, but we don't want to replace the & in "&#x....;" with the same!
@@ -445,5 +437,5 @@
 			//filename_url_encoded = filename_url_encoded.replace("%2B", "&#x2B;"); // Don't do this, won't get regex escaped when converted back to a + by caller
 			filename_url_encoded = filename_url_encoded.replace("%2B", "+"); // + signs are special, as they will need to be escaped since the caller wants the filename representing a regex
-			filename_url_encoded = filename_url_encoded.replace("%26", HEX_ENTITY_AMPERSAND); // convert URL encoding for ampersand into hex entity for ampersand
+			filename_url_encoded = filename_url_encoded.replace("%26", "&"); // now putting back ampersands too, instead of replacing with HEX_ENTITY_AMPERSAND (&#x26;)
 		}
 		catch (Exception e) {
@@ -544,5 +536,5 @@
 	    // just return input filename param, but with any & in the filename replaced with its hex entity
 		if(!MULTIPLE_FILENAME_ENCODINGS_SUPPORTED) {
-		    return filename.replace("&", HEX_ENTITY_AMPERSAND);
+		    return filename; //return filename.replace("&", HEX_AMPERSAND);
 		}
 		
@@ -567,5 +559,5 @@
 	public static String relativeFilenameToURLEncoding(String filename) {
 		if(!MULTIPLE_FILENAME_ENCODINGS_SUPPORTED) { // on a UTF-8 file system, DO NOT do the stuff below, just return input param
-		    return filename.replace("&", HEX_ENTITY_AMPERSAND);
+		    return filename; // return filename.replace("&", HEX_AMPERSAND);
 		}
 		
@@ -580,5 +572,5 @@
 	public static String filenameToURLEncodingWithPrefixRemoved(String filename, String removeFilePathPrefix) {
 		if(!MULTIPLE_FILENAME_ENCODINGS_SUPPORTED) { // on a UTF-8 file system, DO NOT do the stuff below, just return input param
-		    return filename.replace("&", HEX_ENTITY_AMPERSAND);
+		    return filename; //return filename.replace("&", HEX_AMPERSAND);
 		}
 		
Index: /main/trunk/gli/src/org/greenstone/gatherer/metadata/MetadataXMLFile.java
===================================================================
--- /main/trunk/gli/src/org/greenstone/gatherer/metadata/MetadataXMLFile.java	(revision 34414)
+++ /main/trunk/gli/src/org/greenstone/gatherer/metadata/MetadataXMLFile.java	(revision 34415)
@@ -79,6 +79,4 @@
 	    loaded_file = this;
 	    loaded_file_document = document;
-		
-		reEncodeFilenamesInMetadataXML(loaded_file_document);
 	}
 
@@ -108,15 +106,9 @@
 	    loaded_file = this;
 	    loaded_file_document = document;
-		reEncodeFilenamesInMetadataXML(loaded_file_document);
-		
 	}
 
 	// Determine the file's path relative to the location of the metadata.xml file
 	String metadata_xml_file_directory_path = FilenameEncoding.fileToURLEncoding(getParentFile());
-
 	String file_relative_path = file_node.getURLEncodedFilePath();
-	if(!FilenameEncoding.MULTIPLE_FILENAME_ENCODINGS_SUPPORTED) {
-	    file_relative_path = file_relative_path.replace("&", FilenameEncoding.HEX_ENTITY_AMPERSAND);
-	}
 	file_relative_path = file_relative_path.substring(metadata_xml_file_directory_path.length());
 	
@@ -132,4 +124,14 @@
 	}
 	else {
+		// When XML files are parsed, predefined XML entities get resolved, which includes & in &amp; and &#x...;
+		// see https://en.wikipedia.org/wiki/List_of_XML_and_HTML_character_entity_references#Predefined_entities_in_XML
+		// (and https://stackoverflow.com/questions/1777878/is-there-a-java-xml-api-that-can-parse-a-document-without-resolving-character-en)
+		// We don't want &/entities in FileName elements stored in metadata.xml, as we'd have to put the entities
+		// back (undo the xml entity resolution) after each XML parse operation, which is costly and slows GLI down
+		// when assigning meta to multiple docs.
+		// Instead, when writing out or comparing against FileName elements in metadata.xml, we ensure all
+		// ampersands are replaced by their hex URL encoded value of %26.		
+		file_relative_path = file_relative_path.replace("&", FilenameEncoding.HEX_AMPERSAND);
+		
 	    // Convert the file path into a regular expression that will match it
 	    file_path_regexp = MetadataTools.getRegularExpressionThatMatchesFilePath(file_relative_path);
@@ -153,5 +155,5 @@
 		Element current_filename_element = (Element) filename_elements_nodelist.item(j);
 		String current_filename_element_value = XMLTools.getElementTextValue(current_filename_element);
-
+		
 		// Only exact matches can be extended with new metadata
 		if (current_filename_element_value.equals(file_path_regexp)) {
@@ -200,5 +202,5 @@
 	    // changed, it must be applied on the file(name) whose metadata has been adjusted
 	    if(metadata_element_name_full.equals(FILENAME_ENCODING_METADATA)) {
-			metadata_value_string = processFilenameEncoding(file_path_regexp, 
+			metadata_value_string = processFilenameEncoding(file_path_regexp, // file_path_regexp has & replaced by HEX_AMPERSAND but processFilenameEncoding doesn't use param
 										file_node, metadata_value_string, false);
 		                      // true only if removing meta
@@ -252,8 +254,25 @@
 	loaded_file_changed = true;
     }
-
-
-    // DO NOT REMOVE THE System.err DEBUGGING STATEMENTS FROM THIS METHOD: HELPS WITH TESTING/DEBUGGING
+	
+	// DO NOT REMOVE THE System.err DEBUGGING STATEMENTS FROM THIS METHOD: HELPS WITH TESTING/DEBUGGING
     // WHEN FILE-LEVEL META IS ASSIGNED TO NON-ASCII ENCODED FILENAMES OR WITH FILENAMES CONTAINING +/ampersand
+	
+	
+	// By default, XML parsing automatically resolves certain predefined XML entities including the ampersand.
+	// https://en.wikipedia.org/wiki/List_of_XML_and_HTML_character_entity_references#Predefined_entities_in_XML
+	// "The XML specification defines five "predefined entities" representing special characters, and requires that all XML processors honor them. The entities can be explicitly declared in a DTD, as well, but if this is done, the replacement text must be the same as the built-in definitions. XML also allows other named entities of any size to be defined on a per-document basis."
+	// Also https://stackoverflow.com/questions/1777878/is-there-a-java-xml-api-that-can-parse-a-document-without-resolving-character-en
+	// which suggests using StAX instead of SAX or DOM parsers allows us to bypass automatic entity resolution.
+	// However, https://docs.oracle.com/javase/tutorial/jaxp/stax/why.html and
+	// https://docs.oracle.com/javase/tutorial/jaxp/stax/api.html show that StAX works like SAX rather than DOM parser
+	// while the XMLTools.parseXML() that we use throughout this file relies on DOMParser behaviour to get access to the
+	// XML DOM Document, so that it's not straightforward to replace DOMParser's use in Document XMLTools.parseXML() with
+	// an equivalent using a streambased StAX parser.
+	// Instead, method reEncodeFilenamesInMetadataXML(Doc doc) has been removed, as the solution is to no longer store
+	// ampersands: no longer encoding ampersands to entities but to %26, and all hex entities in filenames are further
+	// protected from XML's entity resolution because their ampersand prefixes are encoded as %26 (i.e. &#xDDDD; is
+	// stored as %26#xDDDD;) and therefore we no longer need to go over the XML Doc reinstating entities after parseXML
+	// either, entities being now preserved though with %26 prefixed in place of the & prefix.	
+	
     public ArrayList getMetadataAssignedToFile(File file, boolean fileEncodingOnly)
     {
@@ -272,6 +291,4 @@
 	    loaded_file = this;
 	    loaded_file_document = document;
-		
-		reEncodeFilenamesInMetadataXML(loaded_file_document);
 	}
 
@@ -285,6 +302,9 @@
 	    file_relative_path = file_relative_path.substring(FilenameEncoding.URL_FILE_SEPARATOR.length());
 	}
-	String hexdecoded_regexed_file_relative_path = FilenameEncoding.decodeStringContainingHexEntities(file_relative_path);
-
+	
+	// Ampersands and entities in FileName elements are protected from being resolved on parsing back the XML.
+	// To compare apples with apples convert any & to its hex url encoded value of %26	
+	file_relative_path = file_relative_path.replace("&", FilenameEncoding.HEX_AMPERSAND);
+	
 	// Build up a list of metadata assigned to this file
 	ArrayList metadata_values = new ArrayList();
@@ -305,6 +325,6 @@
 
 		//System.err.println("\n  Original TAIL filename was: " + Utility.debugUnicodeString(file.getName()));
-		//System.err.println("Looking in meta.xml for hexdecoded_regexed_file_RELATIVE_path: " + hexdecoded_regexed_file_relative_path
-			//+ " - debug version: " + Utility.debugUnicodeString(hexdecoded_regexed_file_relative_path));		
+		//System.err.println("Looking in meta.xml for file_relative_path: " + file_relative_path);
+			//+ " - debug version: " + Utility.debugUnicodeString(file_relative_path));		
 	
 		// Does this fileset specify metadata for one file only?
@@ -315,13 +335,11 @@
 		}
 
-		String hexdecoded_current_filename_element_value = FilenameEncoding.decodeStringContainingHexEntities(current_filename_element_value);
-		//System.err.println("   Checking to see if it matches " + hexdecoded_current_filename_element_value + " - debug: " + Utility.debugUnicodeString(hexdecoded_current_filename_element_value));
-			//System.err.println("   Checking to see if it matches " + current_filename_element_value + " - debug: " + Utility.debugUnicodeString(current_filename_element_value));
+		//System.err.println("   Checking to see if it matches " + current_filename_element_value);// + " - debug: " + Utility.debugUnicodeString(current_filename_element_value));
 		
 		// This fileset specifies metadata for the file
 		// MetadataXMLFile.addMetadata(CollectionTreeNode, ArrayList) stored filename in uppercase hex
 		// so need to make sure everything hex has been decoded (no more hex) to compare apples with apples		
-		if (hexdecoded_regexed_file_relative_path.matches(hexdecoded_current_filename_element_value)) {	//if (file_relative_path.matches(current_filename_element_value)) {
-		    	//System.err.println("   @@@ Found a match in meta.xml for hexdecoded_regexed_file_relative_path: " + hexdecoded_regexed_file_relative_path + "\n");
+		if (file_relative_path.matches(current_filename_element_value)) {
+		    	//System.err.println("   @@@ Found a match in meta.xml for file_relative_path: " + file_relative_path + "\n");
 		    current_fileset_matches = true;
 		    if (!file_relative_path.equals("") && current_filename_element_value.equals(DIRECTORY_FILENAME)) {
@@ -330,10 +348,10 @@
 		    break;
 		} //else {
-			//System.err.println( hexdecoded_regexed_file_relative_path + " does not match " + hexdecoded_current_filename_element_value);
-			//System.err.println( Utility.debugUnicodeString(hexdecoded_regexed_file_relative_path) + " does not match " + Utility.debugUnicodeString(hexdecoded_current_filename_element_value));
+			//System.err.println("    ###" + file_relative_path + " does not match " + current_filename_element_value);
+			//System.err.println( Utility.debugUnicodeString(file_relative_path) + " does not match " + Utility.debugUnicodeString(current_filename_element_value));
 		//}
 		
 		// This fileset specifies metadata for the folder the file is in
-		if (hexdecoded_regexed_file_relative_path.startsWith(hexdecoded_current_filename_element_value + FilenameEncoding.URL_FILE_SEPARATOR)) {
+		if (file_relative_path.startsWith(current_filename_element_value + FilenameEncoding.URL_FILE_SEPARATOR)) {
 		    current_fileset_matches = true;
 		    folder_metadata_inherited_from = new File(metadata_xml_file_directory, current_filename_element_value);
@@ -435,6 +453,4 @@
 	    loaded_file = this;
 	    loaded_file_document = document;
-		
-		reEncodeFilenamesInMetadataXML(loaded_file_document);
 	}
 
@@ -442,7 +458,4 @@
 	String metadata_xml_file_directory_path = FilenameEncoding.fileToURLEncoding(getParentFile());
 	String file_relative_path = file_node.getURLEncodedFilePath();
-	if(!FilenameEncoding.MULTIPLE_FILENAME_ENCODINGS_SUPPORTED) {
-	    file_relative_path = file_relative_path.replace("&", FilenameEncoding.HEX_ENTITY_AMPERSAND);
-	}
 	file_relative_path = file_relative_path.substring(metadata_xml_file_directory_path.length());
 	if (file_relative_path.startsWith(FilenameEncoding.URL_FILE_SEPARATOR)) {
@@ -457,4 +470,8 @@
 	}
 	else {
+		// Ampersands and entities in FileName elements are protected from being resolved on parsing back the XML.
+		// To compare apples with apples convert any & to its hex url encoded value of %26	
+		file_relative_path = file_relative_path.replace("&", FilenameEncoding.HEX_AMPERSAND);
+		
 	    // Convert the file path into a regular expression that will match it
 	    file_path_regexp = MetadataTools.getRegularExpressionThatMatchesFilePath(file_relative_path);
@@ -558,6 +575,4 @@
 	    loaded_file = this;
 	    loaded_file_document = document;
-		
-		reEncodeFilenamesInMetadataXML(loaded_file_document);
 	}
 
@@ -565,7 +580,4 @@
 	String metadata_xml_file_directory_path = FilenameEncoding.fileToURLEncoding(getParentFile());
 	String file_relative_path = file_node.getURLEncodedFilePath();
-	if(!FilenameEncoding.MULTIPLE_FILENAME_ENCODINGS_SUPPORTED) {
-	    file_relative_path = file_relative_path.replace("&", FilenameEncoding.HEX_ENTITY_AMPERSAND);
-	}
 	file_relative_path = file_relative_path.substring(metadata_xml_file_directory_path.length());	
 	if (file_relative_path.startsWith(FilenameEncoding.URL_FILE_SEPARATOR)) {
@@ -580,4 +592,8 @@
 	}
 	else {
+		// Ampersands and entities in FileName elements are protected from being resolved on parsing back the XML.
+		// To compare apples with apples convert any & to its hex url encoded value of %26	
+		file_relative_path = file_relative_path.replace("&", FilenameEncoding.HEX_AMPERSAND);
+		
 	    // Convert the file path into a regular expression that will match it
 	    file_path_regexp = MetadataTools.getRegularExpressionThatMatchesFilePath(file_relative_path);
@@ -684,64 +700,4 @@
 		}
     }
-
-	/** 
-	 * parseXML(metadata.xml) has the side-effect of resolving html entities.
-	 * Although this is not done by the GLIEntityResolver usage in parseXML(), something
-	 * in parseXML() is resolving the html entities, including those used in carefully
-	 * html-entity-escaped filenames.
-	 * We need to get the filenames in the DOM correct after parsing a metadata.xml file
-	 * into memory, so that we have the correct filenames and so that we'll write it out correctly.
-	 * Therefore, always call this method after a successful parseXML() call on a metadata.xml.
-	 * @param doc is the Document where the FILENAME_ELEMENTs need to be re-encoded.
-	 * At the end of this function, the doc will be modified with the re-encoded filenames.
-	 * 
-	 * DO NOT REMOVE THE DEBUGGING STATEMENTS IN THIS FUNCTION, AS THEY'RE USEFUL
-	 * FOR DEBUGGING ENCODING ISSUES TO DO WITH FILE LEVEL META ASSIGNED TO FILENAMES
-	 * THAT ARE NON-ASCII OR CONTAIN +/ampersands IN THEM.
-	*/
-	static private void reEncodeFilenamesInMetadataXML(Document doc) {
-		
-		String curr_directory_path = FilenameEncoding.fullFilepathToURLEncoding("."); // returns the curr dir path after removing the /./ at end		
-		//System.err.println("@@@ curr_directory_path: " + curr_directory_path);
-		
-		//System.err.println("PARSED loaded_file contains:\n" +  XMLTools.elementToString(doc.getDocumentElement(), true));
-		
-		// Read all the FileSet elements in the file
-		NodeList fileset_elements_nodelist = doc.getElementsByTagName(FILESET_ELEMENT);
-		for (int i = 0; i < fileset_elements_nodelist.getLength(); i++) {
-			Element current_fileset_element = (Element) fileset_elements_nodelist.item(i);
-
-			// get the value of all FileName elements
-			NodeList filename_elements_nodelist = current_fileset_element.getElementsByTagName(FILENAME_ELEMENT);
-			for (int j = 0; j < filename_elements_nodelist.getLength(); j++) {
-				Element filename_element = (Element) filename_elements_nodelist.item(j);
-				String filename = XMLTools.getElementTextValue(filename_element);
-				if(!filename.equals(DIRECTORY_FILENAME)) {
-					// Reencode filename after parseXML() had the side-effect of decoding entities in filename elements
-					
-					//System.err.println("Filename before reencoding was: " + filename);					
-
-				    String encoded_filename = filename;
-				    if(!FilenameEncoding.MULTIPLE_FILENAME_ENCODINGS_SUPPORTED) {
-					encoded_filename = encoded_filename.replace("&", FilenameEncoding.HEX_ENTITY_AMPERSAND);
-				    } else {
-					// Can't convert to URI with backslash-escaped chars (backslash used in regexed filename are illegal in URI object
-					// created by filenameToURLEncoding). So replace backslashes in regex with url-encoded hex-value of backslash, %5C.
-					encoded_filename = filename.replace("\\", "%5C");
-					
-					// get the URL encoded filename preserving special encodings, with any curr_directory_path prefix removed
-					encoded_filename = FilenameEncoding.filenameToURLEncodingWithPrefixRemoved(encoded_filename, curr_directory_path);
-					
-					// Reintrodudce the backslash characters in place of their %5C hex placeholders
-					encoded_filename = encoded_filename.replace("%5C", "\\");
-				    }
-				    // Update filename element in DOM
-				    XMLTools.setElementTextValue(filename_element, encoded_filename);
-				    //System.err.println("Filename after reencoding was: " + encoded_filename);
-				}
-			}
-		}		
-		//System.err.println("RE-ENCODED loaded_file contains:\n" +  XMLTools.elementToString(doc.getDocumentElement(), true));
-	}
 	
     /**
@@ -763,6 +719,4 @@
 	    return;
 	}
-	// Always call this method after calling parseXMLFile
-	reEncodeFilenamesInMetadataXML(document);
 	
 	// Read all the Metadata elements in the file
