Changeset 33747 for main

Timestamp:

2019-12-03T17:50:30+13:00 (4 years ago)

Author:

ak19

Message:

Tidying up code some more and moving unused (but reusable and possibly useful) FilenameEncoding.java functions to end of file.

Location:

main/trunk/gli/src/org/greenstone/gatherer/metadata

Files:

• FilenameEncoding.java (modified) (5 diffs)
• MetadataXMLFile.java (modified) (1 diff)

main/trunk/gli/src/org/greenstone/gatherer/metadata/FilenameEncoding.java

@@ -354 +354 @@
     }   
 
+    
+    // Dr Bainbridge's methods
+    /* On Linux machines that are set to using an ISO-8859 (Latin) type encoding, 
+    * we can work with URL-encoded filenames in Java. Java works with whatever 
+    * encoding the filesystem uses. Unlike systems working with UTF-8, where Java 
+    * interprets filenames as UTF-8 (a destructive process since characters invalid
+    * for UTF-8 are replaced with the invalid character, which means the original
+    * character's byte codes can not be regained), working with an ISO-8859-1 
+    * system means the original byte codes of the characters are preserved, 
+    * regardless of whether the characters represent ISO-8859-1 or not. Such byte
+    * codes are converted by the following method to the correct URL versions of 
+    * the strings that the filenames represent (that is, the correct URL representations
+    * of the filenames in their original encodings). This is useful for interactions with
+    * Perl as Java and Perl can use URL-encoded filenames to talk about the same files
+    * on the file system, instead of having to work out what encoding they are in. */
+    
+    public static String fileToURLEncoding(File file) {
+        if(!MULTIPLE_FILENAME_ENCODINGS_SUPPORTED) {
+            return file.getAbsolutePath();
+        }
+        
+        String filename_url_encoded = "";
+        
+        // The following test for whether the file exists or not is a problem
+        // when a File object--whose actual file is in the process of being moved 
+        // and therefore temporarily does not 'exist' on the actual system--can't
+        // be URL encoded: the following would return "" when a file doesn't exist.
+        // So commenting out the test.
+        /* 
+        if(!file.getName().equals("recycle")) {
+            if(!file.isFile() && !file.isDirectory()) {
+            System.err.println("*** ERROR. Java can't see file: " + file.getAbsolutePath());
+            return "";
+            }
+            
+            if(!file.exists()) {
+            System.err.println("*** NOTE: File doesn't exist: " + file.getAbsolutePath());
+            return ""; //file.getName();
+            }
+        }
+        */
+
+        URI filename_uri = file.toURI();
+        try {
+            // The trick:
+            //  1. toASCIIString() will %xx encode values > 127
+            //  2. Decode the result to "ISO-8859-1" 
+            //  3. URL encode the bytes to string
+            
+            // Step 2 forces the string to be 8-bit values.  It
+            // doesn't matter if the starting raw filename was *not*
+            // in the ISO-8859-1 encoding, the effect is to ensure
+            // we have an 8-bit byte string that (numerically)
+            // captures the right value.  These numerical values are
+            // then used to determine how to URL encode it
+            
+            String filename_ascii = filename_uri.toASCIIString();
+            
+            // The URI.toASCIIString() call above only encodes values > 127.
+            // But we also need to protect + and & signs in filenames. Do this by URL encoding.
+            // But need to double URL encode, else it will get decoded too early, in methods called shortly hereafter.
+            filename_ascii = filename_ascii.replace("+", "%252B"); // +'s ASCII code is 43 decimal, 2b in hex, 2B when uppercased
+            filename_ascii = filename_ascii.replace("&", "%2526"); // &'s ASCII code is 36 in decimal, and 26 in hex            
+            
+            // Before proceeding, protect & in the filename too.
+            // &'s ASCII code is 36 in decimal, and 26 in hex, so replace with &
+            // But dangerous to do simple replace if there are &#x...; entities in the filename already!
+            // That is, we'll want to protect & by replacing with &'s hex value, but we don't want to replace the & in "&#x....;" with the same!
+            //filename_url_encoded = filename_url_encoded.replace("&", "&x26;");// SO THIS IS BAD
+            //filename_url_encoded = filename_url_encoded.replace("&", hexEntityForChar("&"));// SAME, STILL BAD
+            ///filename_ascii = escapeAllCharWithHexEntity(filename_ascii, '&'); // Good: CAREFULLY replaces & that are not part of hex entities            
+            
+            String filename_raw_bytes = URLDecoder.decode(filename_ascii,"ISO-8859-1");         
+            filename_url_encoded = iso_8859_1_filename_to_url_encoded(filename_raw_bytes);
+            
+            // For chars that were protected by being URL encoded, now convert them to the correct version we want them in.
+            // For +: this char is special in regex, so it needs to be converted from URL encoding back to + so it will get properly escaped for regex
+            // For &: this char is special in XML, so since the call to iso_8859_1_filename_to_url_encoded() is over, we can finally convert & to hex entity now.
+            //filename_url_encoded = filename_url_encoded.replace("%2B", "+"); // Don't do this, won't get regex escaped when converted back to a + by caller
+            filename_url_encoded = filename_url_encoded.replace("%2B", "+"); // + signs are special, as they will need to be escaped since the caller wants the filename representing a regex
+            filename_url_encoded = filename_url_encoded.replace("%26", "&"); // convert URL encoding for ampersand into hex entity for ampersand
+        }
+        catch (Exception e) {
+            e.printStackTrace();
+            // Give up trying to convert
+            filename_url_encoded = file.getAbsolutePath(); 
+        }
+        return filename_url_encoded;
+    }
+    
+    // For unicode codepoints see:
+    // http://unicode.org/Public/MAPPINGS/ISO8859/8859-1.TXT for ISO8859-1 (Latin-1)
+    // where 0xE2 maps to codepoint 0x00E2 and is defined as "Latin small letter a with circumflex"
+    // http://unicode.org/Public/MAPPINGS/ISO8859/8859-7.TXT for ISO8859-7 (Greek)
+    // where 0xE2 maps to codepoint 0x03B2 and is defined as "Greek small letter beta"    
+    public static String iso_8859_1_filename_to_url_encoded(String raw_bytes_filename) 
+        throws Exception
+    {
+        String urlEncoded = "";
+
+        try {
+            // By this point we have a UTF-8 encoded string that captures
+            // what the ISO-8859-1 (Latin-1) character is that corresponded to the
+            // 8-bit numeric value for that character in the filename
+            // on the file system
+            
+            // For example:
+            //   File system char:             <lower-case beta char in Latin-7> = %E2
+            //   Equivalent Latin 1 char:      <lower-case a with circumflex>    = %E2
+            //   Mapped to UTF-8:              <lower-case a with circumflex>    = <C3><A2>
+
+            // Our task is to take the string the contains <C3><A2> and ensure that
+            // we "see" it as <E2>
+
+            byte [] raw_bytes = raw_bytes_filename.getBytes("ISO-8859-1");
+            String unicode_filename = new String(raw_bytes,"UTF-8");
+            
+            for(int i = 0; i < unicode_filename.length(); i++) {
+            char charVal = unicode_filename.charAt(i);
+            if ((int)charVal > 255) {
+                urlEncoded += String.format("&#x%02X;", (int)charVal);
+            }
+            else if((int)charVal > 127) {
+                urlEncoded += String.format("%%%02X", (int)charVal);
+            } else {
+                urlEncoded += String.format("%c", (char)charVal);
+            }
+            }
+        }
+        catch (Exception e) {
+            //e.printStackTrace();
+            throw(e);
+        }
+
+        return urlEncoded;
+    }
+
+    // unused for now
+    public static String raw_filename_to_url_encoded(String fileName)
+        throws Exception
+    {
+        String urlEncoded = "";
+        try {
+            byte[] bytes = fileName.getBytes();
+            
+            for(int i = 0; i < bytes.length; i++) {
+            // mask each byte (by applying & 0xFF) to make the signed
+            // byte (in the range -128 to 127) unsigned (in the range
+            // 0 to 255).
+
+            int byteVal = (int)(bytes[i] & 0xFF);
+            
+            if(byteVal > 127) {
+                urlEncoded += String.format("%%%02X", (int)byteVal);
+            } else {
+                urlEncoded += String.format("%c",(char)byteVal);
+            }
+            }
+        }
+        catch (Exception e) {
+            //e.printStackTrace();
+            throw(e);
+        }
+
+        return urlEncoded;
+    }
+   
+ // FURTHER HELPER METHODS 
+ 
+    /**
+     * Produce the equivalent of method fileToURLEncoding(), but taking a String as input parameter.
+     * If filename is relative, then the current directory (gli?) will be prefixed to what is returned
+     * and should be removed manually by the caller. Alternatively, for relative paths, call the variant
+     * relativeFilenameToURLEncoding(String), which will remove any added filepath prefix.
+    */
+    public static String fullFilepathToURLEncoding(String filename) {
+        if(!MULTIPLE_FILENAME_ENCODINGS_SUPPORTED) { // on a UTF-8 file system, DO NOT do the stuff below, just return input param
+            return filename;
+        }
+        
+        File file = new File (filename);
+        String filename_url_encoded = fileToURLEncoding(file);
+        
+        // if the current directory (".") was passed in as filename,
+        // then the filename_url_encoded looks like /full/path/./
+        // In that case, remove the ./ at the end
+        if (filename_url_encoded.endsWith(FilenameEncoding.URL_FILE_SEPARATOR+"."+FilenameEncoding.URL_FILE_SEPARATOR)) {
+            filename_url_encoded = filename_url_encoded.substring(0, filename_url_encoded.length()-2); // cut off /. at end
+        }
+        
+        return filename_url_encoded;
+    }
+ 
+    /**
+     * Produce the equivalent of method fileToURLEncoding(), but taking a String as input parameter
+     * If filename is a relative path, call this method to get it specially URL encoded.
+     * This method will remove the current directory that is prefixed as an intermediary step.
+    */
+    public static String relativeFilenameToURLEncoding(String filename) {
+        if(!MULTIPLE_FILENAME_ENCODINGS_SUPPORTED) { // on a UTF-8 file system, DO NOT do the stuff below, just return input param
+            return filename;
+        }
+        
+        String curr_directory_path = FilenameEncoding.fullFilepathToURLEncoding(".");
+        return filenameToURLEncodingWithPrefixRemoved(filename, curr_directory_path);
+    }
+    
+    /**
+     * Produce the equivalent of method fileToURLEncoding(), but taking a String as input parameter
+     * Convenience method that will return the specially URL encoded version of filename
+     * with the provided removeFilePathPrefix removed */
+    public static String filenameToURLEncodingWithPrefixRemoved(String filename, String removeFilePathPrefix) {
+        if(!MULTIPLE_FILENAME_ENCODINGS_SUPPORTED) { // on a UTF-8 file system, DO NOT do the stuff below, just return input param
+            return filename;
+        }
+        
+        File file = new File (filename);        
+        String filename_url_encoded = fileToURLEncoding(file); // returns a full filepath
+        
+        // now lop off the given removeFilePathPrefix that FilenameEncoding.filenameToURLEncoding(STRING) variant would have added
+        filename_url_encoded = filename_url_encoded.substring(removeFilePathPrefix.length());
+        // remove any remaining slash prefix
+        if (filename_url_encoded.startsWith(FilenameEncoding.URL_FILE_SEPARATOR)) {
+            filename_url_encoded = filename_url_encoded.substring(FilenameEncoding.URL_FILE_SEPARATOR.length());
+        }
+        
+        return filename_url_encoded;
+    }
+    
+// UNUSED now, but useful functions and escapeAllCharWithHexEntity() took effort to write.
+    
     /**
      * Attempting to produce the equivalent method fileToURLEncoding(), but taking a String as input parameter
@@ -375 +606 @@
         return hex_str;
      }
-     
     
     /** Takes a String containing a single char and returns the hex entity for it */
@@ -390 +620 @@
      * as in that case, we don't want to replace any existing hex entities already present in the String.
     */
-    public static String escapeAllCharWithHexEntity(String str, char CHARACTER/*, String hexCodeString*/) {
+    public static String escapeAllCharWithHexEntity(String str, char CHARACTER) {
         
         if(str.indexOf(CHARACTER) == -1) { // nothing to replace, we're done
@@ -398 +628 @@
         String char_as_string = Character.toString(CHARACTER);
         String hexCodeString = hexEntityForChar(char_as_string);
-        
-        //System.err.println("@@@ hexCodeString for: " + char_as_string + " is: " + hexCodeString);
         
         Matcher hexPatternMatch = HEX_PATTERN.matcher(str); // looks for a hex entity, which has the pattern "&#x....;"
@@ -441 +669 @@
         return str;
     }
-
-    
-    // Dr Bainbridge's methods
-    /* On Linux machines that are set to using an ISO-8859 (Latin) type encoding, 
-    * we can work with URL-encoded filenames in Java. Java works with whatever 
-    * encoding the filesystem uses. Unlike systems working with UTF-8, where Java 
-    * interprets filenames as UTF-8 (a destructive process since characters invalid
-    * for UTF-8 are replaced with the invalid character, which means the original
-    * character's byte codes can not be regained), working with an ISO-8859-1 
-    * system means the original byte codes of the characters are preserved, 
-    * regardless of whether the characters represent ISO-8859-1 or not. Such byte
-    * codes are converted by the following method to the correct URL versions of 
-    * the strings that the filenames represent (that is, the correct URL representations
-    * of the filenames in their original encodings). This is useful for interactions with
-    * Perl as Java and Perl can use URL-encoded filenames to talk about the same files
-    * on the file system, instead of having to work out what encoding they are in. */
-    
-    public static String fileToURLEncoding(File file) {
-        if(!MULTIPLE_FILENAME_ENCODINGS_SUPPORTED) {
-            return file.getAbsolutePath();
-        }
-        
-        String filename_url_encoded = "";
-        
-        // The following test for whether the file exists or not is a problem
-        // when a File object--whose actual file is in the process of being moved 
-        // and therefore temporarily does not 'exist' on the actual system--can't
-        // be URL encoded: the following would return "" when a file doesn't exist.
-        // So commenting out the test.
-        /* 
-        if(!file.getName().equals("recycle")) {
-            if(!file.isFile() && !file.isDirectory()) {
-            System.err.println("*** ERROR. Java can't see file: " + file.getAbsolutePath());
-            return "";
-            }
-            
-            if(!file.exists()) {
-            System.err.println("*** NOTE: File doesn't exist: " + file.getAbsolutePath());
-            return ""; //file.getName();
-            }
-        }
-        */
-
-        URI filename_uri = file.toURI();
-        try {
-            // The trick:
-            //  1. toASCIIString() will %xx encode values > 127
-            //  2. Decode the result to "ISO-8859-1" 
-            //  3. URL encode the bytes to string
-            
-            // Step 2 forces the string to be 8-bit values.  It
-            // doesn't matter if the starting raw filename was *not*
-            // in the ISO-8859-1 encoding, the effect is to ensure
-            // we have an 8-bit byte string that (numerically)
-            // captures the right value.  These numerical values are
-            // then used to determine how to URL encode it
-            
-            String filename_ascii = filename_uri.toASCIIString();
-            
-            // The URI.toASCIIString() call above only encodes values > 127.
-            // But we also need to protect + and & signs in filenames
-            filename_ascii = filename_ascii.replace("+", "%252B"); // +'s ASCII code is 43 decimal, 2b in hex, 2B when uppercased
-            filename_ascii = filename_ascii.replace("&", "%2526"); // &'s ASCII code is 36 in decimal, and 26 in hex            
-            
-            // Before proceeding, protect & in the filename too.
-            // &'s ASCII code is 36 in decimal, and 26 in hex, so replace with &#x26;
-            // But dangerous to do simple replace if there are &#x...; entities in the filename already!
-            // That is, we'll want to protect & by replacing with &'s hex value, but we don't want to replace the & in "&#x....;" with the same!
-            //filename_url_encoded = filename_url_encoded.replace("&", "&x26;");// SO THIS IS BAD
-            //filename_url_encoded = filename_url_encoded.replace("&", hexEntityForChar("&"));// SAME, STILL BAD
-            ///filename_ascii = escapeAllCharWithHexEntity(filename_ascii, '&'); // Good: CAREFULLY replaces & that are not part of hex entities            
-            
-            String filename_raw_bytes = URLDecoder.decode(filename_ascii,"ISO-8859-1");         
-            filename_url_encoded = iso_8859_1_filename_to_url_encoded(filename_raw_bytes);
-            
-            //filename_url_encoded = filename_url_encoded.replace("%2B", "&#x2B;"); // Don't do this, won't get regex escaped when converted back to a + by caller
-            filename_url_encoded = filename_url_encoded.replace("%2B", "+"); // + signs are special, as they will need to be escaped since the caller wants the filename representing a regex
-            filename_url_encoded = filename_url_encoded.replace("%26", "&#x26;");           
-        }
-        catch (Exception e) {
-            e.printStackTrace();
-            // Give up trying to convert
-            filename_url_encoded = file.getAbsolutePath(); 
-        }
-        return filename_url_encoded;
-    }
-    
-    // For unicode codepoints see:
-    // http://unicode.org/Public/MAPPINGS/ISO8859/8859-1.TXT for ISO8859-1 (Latin-1)
-    // where 0xE2 maps to codepoint 0x00E2 and is defined as "Latin small letter a with circumflex"
-    // http://unicode.org/Public/MAPPINGS/ISO8859/8859-7.TXT for ISO8859-7 (Greek)
-    // where 0xE2 maps to codepoint 0x03B2 and is defined as "Greek small letter beta"    
-    public static String iso_8859_1_filename_to_url_encoded(String raw_bytes_filename) 
-        throws Exception
-    {
-        String urlEncoded = "";
-
-        try {
-            // By this point we have a UTF-8 encoded string that captures
-            // what the ISO-8859-1 (Latin-1) character is that corresponded to the
-            // 8-bit numeric value for that character in the filename
-            // on the file system
-            
-            // For example:
-            //   File system char:             <lower-case beta char in Latin-7> = %E2
-            //   Equivalent Latin 1 char:      <lower-case a with circumflex>    = %E2
-            //   Mapped to UTF-8:              <lower-case a with circumflex>    = <C3><A2>
-
-            // Our task is to take the string the contains <C3><A2> and ensure that
-            // we "see" it as <E2>
-
-            byte [] raw_bytes = raw_bytes_filename.getBytes("ISO-8859-1");
-            String unicode_filename = new String(raw_bytes,"UTF-8");
-            
-            for(int i = 0; i < unicode_filename.length(); i++) {
-            char charVal = unicode_filename.charAt(i);
-            if ((int)charVal > 255) {
-                urlEncoded += String.format("&#x%02X;", (int)charVal);
-            }
-            else if((int)charVal > 127) {
-                urlEncoded += String.format("%%%02X", (int)charVal);
-            } else {
-                urlEncoded += String.format("%c", (char)charVal);
-            }
-            }
-        }
-        catch (Exception e) {
-            //e.printStackTrace();
-            throw(e);
-        }
-
-        return urlEncoded;
-    }
-
-    // unused for now
-    public static String raw_filename_to_url_encoded(String fileName)
-        throws Exception
-    {
-        String urlEncoded = "";
-        try {
-            byte[] bytes = fileName.getBytes();
-            
-            for(int i = 0; i < bytes.length; i++) {
-            // mask each byte (by applying & 0xFF) to make the signed
-            // byte (in the range -128 to 127) unsigned (in the range
-            // 0 to 255).
-
-            int byteVal = (int)(bytes[i] & 0xFF);
-            
-            if(byteVal > 127) {
-                urlEncoded += String.format("%%%02X", (int)byteVal);
-            } else {
-                urlEncoded += String.format("%c",(char)byteVal);
-            }
-            }
-        }
-        catch (Exception e) {
-            //e.printStackTrace();
-            throw(e);
-        }
-
-        return urlEncoded;
-    }
-   
- // FURTHER HELPER METHODS 
- 
-    /**
-     * Produce the equivalent of method fileToURLEncoding(), but taking a String as input parameter.
-     * If filename is relative, then the current directory (gli?) will be prefixed to what is returned
-     * and should be removed manually by the caller. Alternatively, for relative paths, call the variant
-     * relativeFilenameToURLEncoding(String), which will remove any added filepath prefix.
-    */
-    public static String fullFilepathToURLEncoding(String filename) {
-        if(!MULTIPLE_FILENAME_ENCODINGS_SUPPORTED) { // on a UTF-8 file system, DO NOT do the stuff below, just return input param
-            return filename;
-        }
-        
-        File file = new File (filename);
-        String filename_url_encoded = fileToURLEncoding(file);
-        
-        // if the current directory (".") was passed in as filename,
-        // then the filename_url_encoded looks like /full/path/./
-        // In that case, remove the ./ at the end
-        if (filename_url_encoded.endsWith(FilenameEncoding.URL_FILE_SEPARATOR+"."+FilenameEncoding.URL_FILE_SEPARATOR)) {
-            filename_url_encoded = filename_url_encoded.substring(0, filename_url_encoded.length()-2); // cut off /. at end
-        }
-        
-        return filename_url_encoded;
-    }
- 
-    /**
-     * Produce the equivalent of method fileToURLEncoding(), but taking a String as input parameter
-     * If filename is a relative path, call this method to get it specially URL encoded.
-     * This method will remove the current directory that is prefixed as an intermediary step.
-    */
-    public static String relativeFilenameToURLEncoding(String filename) {
-        if(!MULTIPLE_FILENAME_ENCODINGS_SUPPORTED) { // on a UTF-8 file system, DO NOT do the stuff below, just return input param
-            return filename;
-        }
-        
-        String curr_directory_path = FilenameEncoding.fullFilepathToURLEncoding(".");
-        return filenameToURLEncodingWithPrefixRemoved(filename, curr_directory_path);
-    }
-    
-    /**
-     * Produce the equivalent of method fileToURLEncoding(), but taking a String as input parameter
-     * Convenience method that will return the specially URL encoded version of filename
-     * with the provided removeFilePathPrefix removed */
-    public static String filenameToURLEncodingWithPrefixRemoved(String filename, String removeFilePathPrefix) {
-        if(!MULTIPLE_FILENAME_ENCODINGS_SUPPORTED) { // on a UTF-8 file system, DO NOT do the stuff below, just return input param
-            return filename;
-        }
-        
-        File file = new File (filename);        
-        String filename_url_encoded = fileToURLEncoding(file); // returns a full filepath
-        
-        // now lop off the given removeFilePathPrefix that FilenameEncoding.filenameToURLEncoding(STRING) variant would have added
-        filename_url_encoded = filename_url_encoded.substring(removeFilePathPrefix.length());
-        // remove any remaining slash prefix
-        if (filename_url_encoded.startsWith(FilenameEncoding.URL_FILE_SEPARATOR)) {
-            filename_url_encoded = filename_url_encoded.substring(FilenameEncoding.URL_FILE_SEPARATOR.length());
-        }
-        
-        return filename_url_encoded;
-    }
 }

main/trunk/gli/src/org/greenstone/gatherer/metadata/MetadataXMLFile.java

@@ -631 +631 @@
     static public void saveLoadedFile()
     {
-    // If we have a file loaded into memory and it has been modified, save it now
-    if (loaded_file != null && loaded_file_changed == true) {
-        //System.err.println("START saveLoadedFile(), loaded_file_document:\n" + XMLTools.elementToString(loaded_file_document.getDocumentElement(), true));
-        
-        XMLTools.writeXMLFile(loaded_file, loaded_file_document, nonEscapingElements);
-    
-    /*  // DEBUGGING:
-        Document doc = XMLTools.parseXMLFile(loaded_file);
-        System.err.println("AT END saveLoadedFile(), PARSED loaded_file contains:\n" +  XMLTools.elementToString(doc.getDocumentElement(), true));
-        
-        reEncodeFilenamesInMetadataXML(doc);
-        System.err.println("AT END saveLoadedFile(), RE-ENCODED loaded_file contains:\n" +  XMLTools.elementToString(doc.getDocumentElement(), true));
-    */  
-        loaded_file_changed = false;
-    }
-    
-    //System.err.println("@@@@ END of saveLoadedFile()");
-    //Utility.printCaller();
+        // If we have a file loaded into memory and it has been modified, save it now
+        if (loaded_file != null && loaded_file_changed == true) {
+            //System.err.println("START saveLoadedFile(), loaded_file_document:\n" + XMLTools.elementToString(loaded_file_document.getDocumentElement(), true));
+            
+            XMLTools.writeXMLFile(loaded_file, loaded_file_document, nonEscapingElements);
+
+            loaded_file_changed = false;
+        }
     }