Changeset 33748 for main/trunk

Timestamp:

2019-12-03T21:06:44+13:00 (4 years ago)

Author:

ak19

Message:

Linux bugfixes to recent commits to do with getting file-level meta assigned to non-ascii filenames or filenames containing plus/ampersand signs to work. Cumulative past commits were sufficient for fixing these issues on Windows. All those changes plus the current ones get it all working on Linux too.

Location:

main/trunk/gli/src/org/greenstone/gatherer/metadata

Files:

• FilenameEncoding.java (modified) (7 diffs)
• MetadataXMLFile.java (modified) (6 diffs)

main/trunk/gli/src/org/greenstone/gatherer/metadata/FilenameEncoding.java

@@ -76 +76 @@
     /** Compiled pattern for hex entities of characters. These are of the forn "&#x....;" with 1 to 4 digits */
     public static final Pattern HEX_PATTERN = Pattern.compile("(&#x[0-9a-zA-Z]{1,4}+;)");
-    
+
+    /** The hex entity version of the ampersand character.
+     * We use this in place of the ampersand character in filenames in metadata.xml files to
+     * preserve the reference to the literal ampersand in the real file name on the file system.
+     */
+    public static final String HEX_ENTITY_AMPERSAND = FilenameEncoding.hexEntityForChar("&"); //"&";
+    
 
 //*********************** BUSY REFRESHING / REQUIRING  REFRESH *********************
@@ -371 +377 @@
     
     public static String fileToURLEncoding(File file) {
+    // on a UTF-8 file system, DO NOT do the stuff further below,
+    // just return input filename param, but with any & in the filename replaced with its hex entity
         if(!MULTIPLE_FILENAME_ENCODINGS_SUPPORTED) {
-            return file.getAbsolutePath();
+            // protect ampersands in filenames by converting it to its hex entity
+            String filepath = file.getAbsolutePath();
+            filepath = filepath.replace("&", HEX_ENTITY_AMPERSAND);
+            return filepath;
         }
         
@@ -419 +430 @@
             
             // Before proceeding, protect & in the filename too.
-            // &'s ASCII code is 36 in decimal, and 26 in hex, so replace with &
+            // &'s ASCII code is 36 in decimal, and 26 in hex, so replace with & (HEX_ENTITY_AMPERSAND)
             // But dangerous to do simple replace if there are &#x...; entities in the filename already!
             // That is, we'll want to protect & by replacing with &'s hex value, but we don't want to replace the & in "&#x....;" with the same!
@@ -434 +445 @@
             //filename_url_encoded = filename_url_encoded.replace("%2B", "+"); // Don't do this, won't get regex escaped when converted back to a + by caller
             filename_url_encoded = filename_url_encoded.replace("%2B", "+"); // + signs are special, as they will need to be escaped since the caller wants the filename representing a regex
-            filename_url_encoded = filename_url_encoded.replace("%26", "&"); // convert URL encoding for ampersand into hex entity for ampersand
+            filename_url_encoded = filename_url_encoded.replace("%26", HEX_ENTITY_AMPERSAND); // convert URL encoding for ampersand into hex entity for ampersand
         }
         catch (Exception e) {
@@ -530 +541 @@
     */
     public static String fullFilepathToURLEncoding(String filename) {
-        if(!MULTIPLE_FILENAME_ENCODINGS_SUPPORTED) { // on a UTF-8 file system, DO NOT do the stuff below, just return input param
-            return filename;
+        // on a UTF-8 file system, DO NOT do the stuff further below,
+        // just return input filename param, but with any & in the filename replaced with its hex entity
+        if(!MULTIPLE_FILENAME_ENCODINGS_SUPPORTED) {
+            return filename.replace("&", HEX_ENTITY_AMPERSAND);
         }
         
@@ -554 +567 @@
     public static String relativeFilenameToURLEncoding(String filename) {
         if(!MULTIPLE_FILENAME_ENCODINGS_SUPPORTED) { // on a UTF-8 file system, DO NOT do the stuff below, just return input param
-            return filename;
+            return filename.replace("&", HEX_ENTITY_AMPERSAND);
         }
         
@@ -567 +580 @@
     public static String filenameToURLEncodingWithPrefixRemoved(String filename, String removeFilePathPrefix) {
         if(!MULTIPLE_FILENAME_ENCODINGS_SUPPORTED) { // on a UTF-8 file system, DO NOT do the stuff below, just return input param
-            return filename;
+            return filename.replace("&", HEX_ENTITY_AMPERSAND);
         }
         

main/trunk/gli/src/org/greenstone/gatherer/metadata/MetadataXMLFile.java

@@ -87 +87 @@
     // Determine the file's path relative to the location of the metadata.xml file
     String metadata_xml_file_directory_path = FilenameEncoding.fileToURLEncoding(getParentFile());
-    String file_relative_path = file_node.getURLEncodedFilePath().substring(metadata_xml_file_directory_path.length());
+
+    String file_relative_path = file_node.getURLEncodedFilePath();
+    if(!FilenameEncoding.MULTIPLE_FILENAME_ENCODINGS_SUPPORTED) {
+        file_relative_path = file_relative_path.replace("&", FilenameEncoding.HEX_ENTITY_AMPERSAND);
+    }
+    file_relative_path = file_relative_path.substring(metadata_xml_file_directory_path.length());
+    
     if (file_relative_path.startsWith(FilenameEncoding.URL_FILE_SEPARATOR)) {
         file_relative_path = file_relative_path.substring(FilenameEncoding.URL_FILE_SEPARATOR.length());
@@ -103 +109 @@
     }
 
+    // LEAVE THIS DEBUGGING STATEMENT IN - USEFUL TO DEBUG FILENAME ENCODING ISSUES WHEN META ASSIGNED
     //System.err.println("MetadataXMLFile.addMetadata() Adding meta for file regexp: "
     //  + file_path_regexp + " - " + org.greenstone.gatherer.util.Utility.debugUnicodeString(file_path_regexp));
@@ -220 +227 @@
 
 
+    // DO NOT REMOVE THE System.err DEBUGGING STATEMENTS FROM THIS METHOD: HELPS WITH TESTING/DEBUGGING
+    // WHEN FILE-LEVEL META IS ASSIGNED TO NON-ASCII ENCODED FILENAMES OR WITH FILENAMES CONTAINING +/ampersand
     public ArrayList getMetadataAssignedToFile(File file, boolean fileEncodingOnly)
     {
@@ -652 +661 @@
      * At the end of this function, the doc will be modified with the re-encoded filenames.
      * 
+     * DO NOT REMOVE THE DEBUGGING STATEMENTS IN THIS FUNCTION, AS THEY'RE USEFUL
+     * FOR DEBUGGING ENCODING ISSUES TO DO WITH FILE LEVEL META ASSIGNED TO FILENAMES
+     * THAT ARE NON-ASCII OR CONTAIN +/ampersands IN THEM.
     */
     static private void reEncodeFilenamesInMetadataXML(Document doc) {
-        if(!FilenameEncoding.MULTIPLE_FILENAME_ENCODINGS_SUPPORTED) {
-            return;
-        }
         
         String curr_directory_path = FilenameEncoding.fullFilepathToURLEncoding("."); // returns the curr dir path after removing the /./ at end        
@@ -677 +686 @@
                     
                     //System.err.println("Filename before reencoding was: " + filename);                    
-                    
+
+                    String encoded_filename = filename;
+                    if(!FilenameEncoding.MULTIPLE_FILENAME_ENCODINGS_SUPPORTED) {
+                    encoded_filename = encoded_filename.replace("&", FilenameEncoding.HEX_ENTITY_AMPERSAND);
+                    } else {
                     // Can't convert to URI with backslash-escaped chars (backslash used in regexed filename are illegal in URI object
                     // created by filenameToURLEncoding). So replace backslashes in regex with url-encoded hex-value of backslash, %5C.
-                    String encoded_filename = filename.replace("\\", "%5C");
+                    encoded_filename = filename.replace("\\", "%5C");
                     
                     // get the URL encoded filename preserving special encodings, with any curr_directory_path prefix removed
@@ -687 +700 @@
                     // Reintrodudce the backslash characters in place of their %5C hex placeholders
                     encoded_filename = encoded_filename.replace("%5C", "\\");
-                    
-                    // Update filename element in DOM
-                    XMLTools.setElementTextValue(filename_element, encoded_filename);
-                    //System.err.println("Filename after reencoding was: " + encoded_filename);
+                    }
+                    // Update filename element in DOM
+                    XMLTools.setElementTextValue(filename_element, encoded_filename);
+                    //System.err.println("Filename after reencoding was: " + encoded_filename);
                 }
             }