Changeset 33746 for main/trunk

Timestamp:

2019-12-03T17:31:17+13:00 (4 years ago)

Author:

ak19

Message:

1. Bugfix for dealing with + in filenames: file-level metadata now sticks and also ends up in doc.xml on build, as should happen. 2. Better (more optimal) bugfix for & in filenames, to get metadata to still stick after yesterday's first bugfix for this. Sadly, the improved code no longer needs the new function I introduced yesterday (escapeAllCharWithHexEntity). Leaving the function in, in case it ever comes in handy or as an idea. 3. Refactoring some code. 4. Removed some debugging statements. But some things are still largely commented out. Will remove hereafter.

Location:

main/trunk/gli/src/org/greenstone/gatherer/metadata

Files:

• FilenameEncoding.java (modified) (11 diffs)
• MetadataXMLFile.java (modified) (3 diffs)

main/trunk/gli/src/org/greenstone/gatherer/metadata/FilenameEncoding.java

@@ -73 +73 @@
      * a file, it means this still needs to be retrieved. */
     public static Map map = new HashMap();
+    
+    /** Compiled pattern for hex entities of characters. These are of the forn "&#x....;" with 1 to 4 digits */
+    public static final Pattern HEX_PATTERN = Pattern.compile("(&#x[0-9a-zA-Z]{1,4}+;)");
+    
 
 //*********************** BUSY REFRESHING / REQUIRING  REFRESH *********************
@@ -310 +314 @@
     */
     public static String decodeStringContainingHexEntities(String str) {
-        String result = "";
-        Pattern hexPattern = Pattern.compile("(&#x[0-9a-zA-Z]{1,4}+;)");
-        Matcher matcher = hexPattern.matcher(str);
+        String result = "";     
+        Matcher matcher = HEX_PATTERN.matcher(str);
         
         int searchFromIndex = 0;
@@ -327 +330 @@
             String hexNumberStr = hexPart.substring(3, hexPart.length()-1); // lose the "&#x" prefix and the ";" suffix to get just the hex number portion of the match
             // https://stackoverflow.com/questions/16625865/java-unicode-to-hex-string
-            // https://stackoverflow.com/questions/11194513/convert-hex-string-to-int
-            
-            //System.err.println("hexNumberStr so far: " + hexNumberStr);           
+            // https://stackoverflow.com/questions/11194513/convert-hex-string-to-int           
+            
+            //System.err.println("hexNumberStr so far: " + hexNumberStr);
             hexNumberStr = "0x" + hexNumberStr; // e.g "0xDDDD"
             //int hexNumber = Integer.parseInt(hexNumberStr);
@@ -388 +391 @@
     */
     public static String escapeAllCharWithHexEntity(String str, char CHARACTER/*, String hexCodeString*/) {
+        
+        if(str.indexOf(CHARACTER) == -1) { // nothing to replace, we're done
+            return str;         
+        }
+        
         String char_as_string = Character.toString(CHARACTER);
         String hexCodeString = hexEntityForChar(char_as_string);
@@ -393 +401 @@
         //System.err.println("@@@ hexCodeString for: " + char_as_string + " is: " + hexCodeString);
         
-        Pattern hexPattern = Pattern.compile("(&#x[0-9a-zA-Z]{1,4}+;)");
-        Matcher hexPatternMatch = hexPattern.matcher(str);      
-        
-        // want to replace all & with &x26; (the hex for ampsersand) IFF the & is not already a hexcode/doesn't already match hexPattern
+        Matcher hexPatternMatch = HEX_PATTERN.matcher(str); // looks for a hex entity, which has the pattern "&#x....;"
+        
+        // want to replace all & with &x26; (the hex for ampsersand) IFF the & is not already a hexcode/doesn't already match HEX_PATTERN
         int searchIndex = 0;
         
@@ -423 +430 @@
                     
                     // String has been modified, so have to update Matcher
-                    hexPatternMatch = hexPattern.matcher(str);
+                    hexPatternMatch = HEX_PATTERN.matcher(str);
                     
                     if(searchIndex >= str.length()) {
@@ -493 +500 @@
             String filename_ascii = filename_uri.toASCIIString();
             
+            // The URI.toASCIIString() call above only encodes values > 127.
+            // But we also need to protect + and & signs in filenames
+            filename_ascii = filename_ascii.replace("+", "%252B"); // +'s ASCII code is 43 decimal, 2b in hex, 2B when uppercased
+            filename_ascii = filename_ascii.replace("&", "%2526"); // &'s ASCII code is 36 in decimal, and 26 in hex            
+            
             // Before proceeding, protect & in the filename too.
             // &'s ASCII code is 36 in decimal, and 26 in hex, so replace with &
@@ -499 +511 @@
             //filename_url_encoded = filename_url_encoded.replace("&", "&x26;");// SO THIS IS BAD
             //filename_url_encoded = filename_url_encoded.replace("&", hexEntityForChar("&"));// SAME, STILL BAD
-            filename_ascii = escapeAllCharWithHexEntity(filename_ascii, '&'); // Good: CAREFULLY replaces & that are not part of hex entities
-            
+            ///filename_ascii = escapeAllCharWithHexEntity(filename_ascii, '&'); // Good: CAREFULLY replaces & that are not part of hex entities            
             
             String filename_raw_bytes = URLDecoder.decode(filename_ascii,"ISO-8859-1");         
             filename_url_encoded = iso_8859_1_filename_to_url_encoded(filename_raw_bytes);
             
+            //filename_url_encoded = filename_url_encoded.replace("%2B", "+"); // Don't do this, won't get regex escaped when converted back to a + by caller
+            filename_url_encoded = filename_url_encoded.replace("%2B", "+"); // + signs are special, as they will need to be escaped since the caller wants the filename representing a regex
+            filename_url_encoded = filename_url_encoded.replace("%26", "&");           
         }
         catch (Exception e) {
@@ -605 +619 @@
         
         File file = new File (filename);
-        //return fileToURLEncoding(file);
         String filename_url_encoded = fileToURLEncoding(file);
         
@@ -630 +643 @@
         String curr_directory_path = FilenameEncoding.fullFilepathToURLEncoding(".");
         return filenameToURLEncodingWithPrefixRemoved(filename, curr_directory_path);
-        
-        /*
-        String curr_directory_path = FilenameEncoding.fullFilepathToURLEncoding(".");       
-        if (curr_directory_path.endsWith("."+FilenameEncoding.URL_FILE_SEPARATOR)) {
-            curr_directory_path = curr_directory_path.substring(0, curr_directory_path.length()-2); // cut off /. at end            
-        }
-        
-        File file = new File (filename);        
-        String filename_url_encoded = fileToURLEncoding(file); // returns a full filepath
-        
-        // now lop off the current dir prefix that FilenameEncoding.filenameToURLEncoding(STRING) variant would have added
-        filename_url_encoded = filename_url_encoded.substring(curr_directory_path.length());
-        // remove any remaining slash prefix
-        if (filename_url_encoded.startsWith(FilenameEncoding.URL_FILE_SEPARATOR)) {
-            filename_url_encoded = filename_url_encoded.substring(FilenameEncoding.URL_FILE_SEPARATOR.length());
-        }
-        
-        return filename_url_encoded;
-        */
     }
     
@@ -660 +654 @@
         }
         
-        /*if (removeFilePathPrefix.endsWith("."+FilenameEncoding.URL_FILE_SEPARATOR)) {
-            removeFilePathPrefix = removeFilePathPrefix.substring(0, removeFilePathPrefix.length()-2); // cut off /. at end     
-        }
-        */
-        
         File file = new File (filename);        
         String filename_url_encoded = fileToURLEncoding(file); // returns a full filepath
-        
-        System.err.println("@@@ full url encoded filename: " + filename_url_encoded);
         
         // now lop off the given removeFilePathPrefix that FilenameEncoding.filenameToURLEncoding(STRING) variant would have added

main/trunk/gli/src/org/greenstone/gatherer/metadata/MetadataXMLFile.java

@@ -668 +668 @@
         }
         
-        String curr_directory_path = FilenameEncoding.fullFilepathToURLEncoding(".");
-        
-        //String curr_directory_path = FilenameEncoding.filenameToURLEncoding(".");
-        //curr_directory_path = curr_directory_path.substring(0, curr_directory_path.length()-2); // cut off /. at end
-        System.err.println("@@@ curr_directory_path: " + curr_directory_path);
+        String curr_directory_path = FilenameEncoding.fullFilepathToURLEncoding("."); // returns the curr dir path after removing the /./ at end        
+        //System.err.println("@@@ curr_directory_path: " + curr_directory_path);
         
         //System.err.println("PARSED loaded_file contains:\n" +  XMLTools.elementToString(doc.getDocumentElement(), true));
@@ -689 +686 @@
                     // Reencode filename after parseXML() had the side-effect of decoding entities in filename elements
                     
-                    System.err.println("Filename before reencoding was: " + filename);                  
+                    //System.err.println("Filename before reencoding was: " + filename);                    
                     
                     // Can't convert to URI with backslash-escaped chars (backslash used in regexed filename are illegal in URI object
                     // created by filenameToURLEncoding). So replace backslashes in regex with url-encoded hex-value of backslash, %5C.
-                    String encoded_filename = filename.replace("\\", "%5C"); 
-                    
-                    /*
-                    encoded_filename = FilenameEncoding.filenameToURLEncoding(encoded_filename);
-                    // now lop off the metadataxml dir prefix the FilenameEncoding.filenameToURLEncoding(STRING) variant would have added
-                    encoded_filename = encoded_filename.substring(curr_directory_path.length());
-                    if (encoded_filename.startsWith(FilenameEncoding.URL_FILE_SEPARATOR)) {
-                        encoded_filename = encoded_filename.substring(FilenameEncoding.URL_FILE_SEPARATOR.length());
-                    }
-                    */
+                    String encoded_filename = filename.replace("\\", "%5C");
                     
                     // get the URL encoded filename preserving special encodings, with any curr_directory_path prefix removed
@@ -708 +696 @@
                     
                     // Reintrodudce the backslash characters in place of their %5C hex placeholders
-                    encoded_filename = encoded_filename.replace("%5C", "\\");               
+                    encoded_filename = encoded_filename.replace("%5C", "\\");
                     
                     // Update filename element in DOM
                     XMLTools.setElementTextValue(filename_element, encoded_filename);
-                    System.err.println("Filename after reencoding was: " + encoded_filename);
+                    //System.err.println("Filename after reencoding was: " + encoded_filename);
                 }
             }