Changeset 33043

Timestamp:

2019-04-30T18:33:42+12:00 (5 years ago)

Author:

ak19

Message:

Not a bugfix, but to help with encoding issues, including to help with current, still unresolved encoding issue. 1. Introduction of string2hex functions in JavaScript and Java. 2. Overriding GSXML.elemtToString() with introduction of additional debugEncoding parameter that will turn on string2hex use when printing request and response XMLs. Now non-basic ASCII characters in the XML will be printed in hex if debugEncoding parameter passed in is true. 3. The inactive Connector elements in server8.xml.svn now also have the attribute URIEncoding set to UTF-8. Some of these inactive Connectors get turned on at times, such as for https. In which case we will need tomcact to also interpret get/post data coming in through those connectors to be sent on as utf-8.

Location:

main/trunk/greenstone3

Files:

• resources/tomcat/server_tomcat8.xml.svn (modified) (4 diffs)
• src/java/org/greenstone/gsdl3/util/GSXML.java (modified) (4 diffs)
• src/java/org/greenstone/util/Misc.java (modified) (1 diff)
• web/interfaces/default/js/javascript-global-functions.js (modified) (1 diff)

main/trunk/greenstone3/resources/tomcat/server_tomcat8.xml.svn

@@ -70 +70 @@
            port="@localhost.port.http@"  protocol="HTTP/1.1"
                connectionTimeout="20000"
-               redirectPort="@https.redirect.port@" />
+               redirectPort="@https.redirect.port@"
+               URIEncoding="UTF-8" />
     -->
     <!-- A "Connector" using the shared thread pool-->    
@@ -96 +97 @@
     <!--
     <Connector port="@https.redirect.port@" protocol="org.apache.coyote.http11.Http11NioProtocol"
-               maxThreads="150" SSLEnabled="true">
+               maxThreads="150" SSLEnabled="true" URIEncoding="UTF-8">
         <SSLHostConfig>
             <Certificate certificateKeystoreFile="conf/localhost-rsa.jks"
@@ -111 +112 @@
     <!--
     <Connector port="@https.redirect.port@" protocol="org.apache.coyote.http11.Http11AprProtocol"
-               maxThreads="150" SSLEnabled="true" >
+               maxThreads="150" SSLEnabled="true" URIEncoding="UTF-8" >
         <UpgradeProtocol className="org.apache.coyote.http2.Http2Protocol" />
         <SSLHostConfig>
@@ -127 +128 @@
             keystorePass="@keystore.pass@"
             clientAuth="false" sslProtocol="TLS"
-        keystoreType="@keystore.type@" />
+        keystoreType="@keystore.type@"
+        URIEncoding="UTF-8" />
     @https.comment.out.end@
 

main/trunk/greenstone3/src/java/org/greenstone/gsdl3/util/GSXML.java

@@ -43 +43 @@
 
 import org.greenstone.gsdl3.util.MyNodeList;
+import org.greenstone.util.Misc;
+
 /** various functions for extracting info out of GS XML */
 public class GSXML
@@ -1629 +1631 @@
     }
 
-    public static String elementToString(Element e, boolean indent)
+    // pass in debugEncoding=true to investigate encoding issues. This function will then return non-basic ASCII characters in hex
+    public static String elementToString(Element e, boolean indent, boolean debugEncoding)
     {
         String str = "";
@@ -1647 +1650 @@
             trans.transform(new DOMSource(e), new StreamResult(sw));
             str += sw.toString();
+            
+            // if debugging encoding issues, then encode unicode code pts as hex for all but non-alphanumeric and space/tab/newline chars
+            if(debugEncoding) str = Misc.stringToHex(str);
         }
         catch (Exception ex)
@@ -1658 +1664 @@
     }
 
+    public static String elementToString(Element e, boolean indent)
+    {
+        return elementToString(e, indent, false);
+    }
+    
     public static ArrayList<String> getGroupsFromSecurityResponse(Element securityResponse)
     {

main/trunk/greenstone3/src/java/org/greenstone/util/Misc.java

@@ -56 +56 @@
     }   
     }
-
+    
+    
+    // Debugging function to print a string's non-basic chars in hex
+    // Based on https://stackoverflow.com/questions/923863/converting-a-string-to-hexadecimal-in-java
+    public static String stringToHex(String str) {
+      String result = "";
+      for(int i = 0; i < str.length(); i++) {
+            int charCode = str.codePointAt(i); // unicode codepoint / ASCII code
+            
+            // ASCII table: https://cdn.sparkfun.com/assets/home_page_posts/2/1/2/1/ascii_table_black.png
+            // If the unicode character code pt is less than the ASCII code for space and greater than for tilda, let's display the char in hex (x0000 format)
+            if((charCode >= 20 && charCode <= 126) || charCode == 9 || charCode == 10 || charCode == 13) { // space to tilda, TAB, LF, CR are printable
+                result += str.charAt(i);
+            } else {
+                result += "x" + String.format("%04x", charCode);
+            }
+      }
+      
+      return result;
+    }
+    
+    
     public static void printHash(HashMap map) {
     Set entries = map.entrySet();

main/trunk/greenstone3/web/interfaces/default/js/javascript-global-functions.js

@@ -11 +11 @@
 {
     return $("#" + id.replace(/\./g, "\\.").replace(/:/g,"\\:"));
+}
+
+// Debugging function to print a string's non-basic chars in hex
+// Based on https://stackoverflow.com/questions/36637146/javascript-encode-string-to-hex/36637293
+// https://stackoverflow.com/questions/21647928/javascript-unicode-string-to-hex
+gs.functions.string2hex = function(str) {
+    var hex, i;
+
+    var result = "";
+    for (i=0; i<str.length; i++) {
+        charcode = str.charCodeAt(i);
+        // ASCII table: https://cdn.sparkfun.com/assets/home_page_posts/2/1/2/1/ascii_table_black.png
+        // if the unicode character code pt is less than the ASCII code for space and greater than for tilda, let's display the char in hex (x0000 format)
+        if(charcode < 20 || charcode > 126) { //doesn't work: if(str.charAt(i) < ' ' || str.charAt(i) > '~') {
+            hex = charcode.toString(16);
+            result += "x" + ("000"+hex).slice(-4);
+        }
+        else {
+            result += str.charAt(i);
+        }
+    }
+
+    return result;
 }