Changeset 33739

Timestamp:

2019-12-02T23:15:50+13:00 (4 years ago)

Author:

ak19

Message:

Bugfix to GLI to being able to parse metadata.xml files containing & chars. Basically, after FilenameEncoding.filtToURLEncoding() calls URI.toASCIIString(), I replace all non-hex-entity ampersands with their hex entity. This preserves it correctly in the metadata.xml files. Some details: Because we need to use URI.toASCIIString(), which only converts char values > 127 to in the URI to its URL encoded hex value (and not hex entity &#x....; as I now find, despite having coded carefully around hex entities with extra unnecessary effort), the code for encoding values less than 127 as their URL hex code has to be manually done. And escaping & to its hex entity becomes complicated by the fact that we don't want to modify any existing hex entities in the ASCIIstring produced by toASCIIString. (Not a complication when we're dealing with strings containing URL/% encoded hex values.) If more chars in metadata.xml pose a problem for GLI parsing them, then a better solution should be invented to replace URI.toASCIIString(). If only stringToHex() would suffice. I think however that toASCIIString() properly URL encodes unicode codepoints, so that one can have 3 or 4 %XX in sequence, like %xx%xx%xx. Work to be done: plus signs in filenames still need to be handled. GLI handles them okay so far as loading the correct file level meta attached to a filename containing + signs, but when built metadata for such a file does not appear in doc.xml. This may be a perl issue if it happens on build?

Location:

main/trunk/gli/src/org/greenstone/gatherer/metadata

Files:

• FilenameEncoding.java (modified) (4 diffs)
• MetadataXMLFile.java (modified) (2 diffs)

main/trunk/gli/src/org/greenstone/gatherer/metadata/FilenameEncoding.java

@@ -355 +355 @@
     }   
 
-    /** Attempting to produce the equivalent method fileToURLEncoding() above, but taking a String as input parameter */
-     public static String fileNameToHex(String filename) {
-
-        String hexFilename = "";
-        for(int i = 0; i < filename.length(); i++) {
-            int charCode = filename.codePointAt(i); // unicode codepoint / ASCII code
+    /**
+     * Attempting to produce the equivalent method fileToURLEncoding(), but taking a String as input parameter
+     * UNUSED - REPLACED by filenameToURLEncoding(String str) which reuses existing fileToURLEncoding(File) method.
+    */
+    public static String stringToHex(String str) {
+
+        String hex_str = "";
+        for(int i = 0; i < str.length(); i++) {
+            int charCode = str.codePointAt(i); // unicode codepoint / ASCII code
             
             // ASCII table: https://cdn.sparkfun.com/assets/home_page_posts/2/1/2/1/ascii_table_black.png
             // If the unicode character code pt is less than the ASCII code for space and greater than for tilda, let's display the char in hex (x0000 format)
-            if((charCode >= 20 && charCode <= 126) || charCode == 9 || charCode == 10 || charCode == 13 || charCode == 36 || charCode == 43) { // space, tilda, TAB, LF, CR are printable, leave them in for XML element printing. And spaces and plus signs (ASCII codes 36 and 43) need to be converted to hex too 
-                hexFilename += filename.charAt(i);
+            if((charCode >= 20 && charCode <= 126) || charCode == 9 || charCode == 10 || charCode == 13 /*|| charCode == 36 || charCode == 43*/) { // space, tilda, TAB, LF, CR are printable, leave them in for XML element printing. And spaces and plus signs (ASCII codes 36 and 43) need to be converted to hex too 
+                hex_str += str.charAt(i);
             } else {
-                hexFilename += "&#x" + String.format("%x", charCode).toUpperCase() + ";"; // looks like: "&#x[up-to-4-hexdigits-in-UPPERCASE];"
+                hex_str += "&#x" + String.format("%x", charCode).toUpperCase() + ";"; // looks like: "&#x[up-to-4-hexdigits-in-UPPERCASE];"
             }
         }
           
-        return hexFilename;
+        return hex_str;
      }
-
+     
+    
+    /** Takes a String containing a single char and returns the hex entity for it */
+    public static String hexEntityForChar(String char_as_string) {
+        int charCode = char_as_string.codePointAt(0); // unicode codepoint / ASCII code     
+        String hexCodeStr = "&#x" + String.format("%x", charCode).toUpperCase() + ";";
+        return hexCodeStr;
+     } 
+    
+    /**
+     * Given a String containing 0 or more occurrences of CHARACTER,
+     * this method will replace all occurrences of that CHARACTER with its hex entity variant, "&x....;"
+     * Special care is taken where the CHARACTER to be replaced is &,
+     * as in that case, we don't want to replace any existing hex entities already present in the String.
+    */
+    public static String escapeAllCharWithHexEntity(String str, char CHARACTER/*, String hexCodeString*/) {
+        String char_as_string = Character.toString(CHARACTER);
+        String hexCodeString = hexEntityForChar(char_as_string);
+        
+        //System.err.println("@@@ hexCodeString for: " + char_as_string + " is: " + hexCodeString);
+        
+        Pattern hexPattern = Pattern.compile("(&#x[0-9a-zA-Z]{1,4}+;)");
+        Matcher hexPatternMatch = hexPattern.matcher(str);      
+        
+        // want to replace all & with &x26; (the hex for ampsersand) IFF the & is not already a hexcode/doesn't already match hexPattern
+        int searchIndex = 0;
+        
+        boolean finished = false;
+        while(!finished) {          
+            
+            searchIndex = str.indexOf(CHARACTER, searchIndex);
+            
+            if(searchIndex == -1) {
+                finished = true;
+            }   
+            else {              
+                
+                // replacing ampersands, &, is a special case: don't want to replace the & of (hex) entities in the string:
+                if(hexPatternMatch.find(searchIndex) && searchIndex == hexPatternMatch.start()) {
+                    searchIndex = hexPatternMatch.end();
+                } else {
+                    
+                    String tmp = str.substring(0, searchIndex) + hexCodeString;
+                    searchIndex++;
+                    if(str.length() > searchIndex) {
+                        tmp += str.substring(searchIndex);
+                    }
+                    str = tmp;
+                    searchIndex = searchIndex+ hexCodeString.length() - 1;
+                    
+                    // String has been modified, so have to update Matcher
+                    hexPatternMatch = hexPattern.matcher(str);
+                    
+                    if(searchIndex >= str.length()) {
+                        finished = true;
+                    }
+                }
+            }
+        }
+        
+        return str;
+    }
+    
     
     // follows Dr Bainbridge's method below, but with a String parameter instead of a file parameter
@@ -428 +493 @@
             return filename;
         }
-        
         File file = new File (filename);
         return fileToURLEncoding(file);
@@ -454 +518 @@
         }
 
+        // we'll want to protect & by replacing with &'s hex value
+        // but we don't want to replace &#x....; with the same!
+        Pattern plain_ampersand_not_hex_prefix_Pattern = Pattern.compile("&[^#]");
+        
+        
+        int containsAmp = 0;
+        if(file.getName().contains("&amp;")) {
+            System.err.println("@@@ 1 to encode " + file.getName());
+            containsAmp = 1;
+        } else if(file.getName().contains("&")) {
+            System.err.println("@@@ 2 to encode " + file.getName());
+            containsAmp = 2;
+        } else {
+            System.err.println("@@@ 0 to encode " + file.getName());
+        }
+        
+        
         String filename_url_encoded = "";
         
@@ -489 +570 @@
             // then used to determine how to URL encode it
             
-            String filename_ascii = filename_uri.toASCIIString();
-            String filename_raw_bytes = URLDecoder.decode(filename_ascii,"ISO-8859-1");
+            String filename_ascii = filename_uri.toASCIIString();
+            // protect & and + in the filename too
+            filename_ascii = escapeAllCharWithHexEntity(filename_ascii, '&');
+            
+            if(containsAmp > 0) System.err.println("@@@ filename_ascii: " + filename_ascii);
+            
+            
+            //if(containsAmp > 0) System.err.println("@@@ filename_ascii with hexed &: " + filename_ascii);
+            String filename_raw_bytes = URLDecoder.decode(filename_ascii,"ISO-8859-1");         
             filename_url_encoded = iso_8859_1_filename_to_url_encoded(filename_raw_bytes);
+            
+            
+            //filename_url_encoded = filename_url_encoded.replace("&", "%26"); // &'s ASCII code is 36 in decimal, and 26 in hex
+            //filename_ascii = filename_ascii.replace("+", "%2B"); // +'s ASCII code is 43 decimal, 2b in hex, 2B when uppercased
+            //if(containsAmp > 0) System.err.println("@@@ filename_url_encoded: " + filename_url_encoded);
             
         }

main/trunk/gli/src/org/greenstone/gatherer/metadata/MetadataXMLFile.java

@@ -668 +668 @@
         }
         
-        String metadata_xml_file_directory_path = FilenameEncoding.filenameToURLEncoding(".");
-        metadata_xml_file_directory_path = metadata_xml_file_directory_path.substring(0, metadata_xml_file_directory_path.length()-2); // cut off /. at end
-        System.err.println("@@@ metadata_xml_file_directory_path: " + metadata_xml_file_directory_path);
+        String curr_directory_path = FilenameEncoding.filenameToURLEncoding(".");
+        curr_directory_path = curr_directory_path.substring(0, curr_directory_path.length()-2); // cut off /. at end
+        //System.err.println("@@@ curr_directory_path: " + curr_directory_path);
         
         //System.err.println("PARSED loaded_file contains:\n" +  XMLTools.elementToString(doc.getDocumentElement(), true));
@@ -695 +695 @@
 
                     // now lop off the metadataxml dir prefix the FilenameEncoding.filenameToURLEncoding(STRING) variant would have added
-                    encoded_filename = encoded_filename.substring(metadata_xml_file_directory_path.length());
+                    encoded_filename = encoded_filename.substring(curr_directory_path.length());
                     if (encoded_filename.startsWith(FilenameEncoding.URL_FILE_SEPARATOR)) {
                         encoded_filename = encoded_filename.substring(FilenameEncoding.URL_FILE_SEPARATOR.length());