Changeset 31880

Timestamp:

2017-08-14T22:23:04+12:00 (6 years ago)

Author:

ak19

Message:

All the changes that were required to set up multiple proxy servers, one for HTTP, one for HTTPS, one for FTP. Still need to test on Windows

Location:

main/trunk

Files:

• gli/classes/dictionary.properties (modified) (1 diff)
• gli/src/org/greenstone/gatherer/Gatherer.java (modified) (1 diff)
• gli/src/org/greenstone/gatherer/download/DownloadJob.java (modified) (3 diffs)
• gli/src/org/greenstone/gatherer/download/DownloadScrollPane.java (modified) (1 diff)
• gli/src/org/greenstone/gatherer/download/ServerInfoDialog.java (modified) (3 diffs)
• gli/src/org/greenstone/gatherer/gui/DownloadPane.java (modified) (4 diffs)
• gli/src/org/greenstone/gatherer/gui/Preferences.java (modified) (11 diffs)
• greenstone2/perllib/downloaders/WebDownload.pm (modified) (2 diffs)
• greenstone2/perllib/downloaders/WgetDownload.pm (modified) (3 diffs)
• greenstone2/perllib/strings.properties (modified) (2 diffs)

main/trunk/gli/classes/dictionary.properties

@@ -1004 +1004 @@
 Preferences.Connection.No_Check_Certificate: No certificate checking (effective on 'https' URLs)
 Preferences.Connection.No_Check_Certificate_Tooltip:Tick to download even from 'https' URLs that don't have a (valid) security certificate.
-Preferences.Connection.Proxy_Host:Proxy Host:
-Preferences.Connection.Proxy_Host_Tooltip:The address of your proxy server
-Preferences.Connection.Proxy_Port:Proxy Port:
-Preferences.Connection.Proxy_Port_Tooltip:The port number of your proxy server
-Preferences.Connection.Proxy_Host_Missing:Enter a value for the proxy host or turn off Use Proxy Connection in the Connection tab.
+Preferences.Connection.HTTP_Proxy_Host:HTTP Proxy Host:
+Preferences.Connection.HTTP_Proxy_Host_Tooltip:The address of your HTTP proxy server
+Preferences.Connection.HTTP_Proxy_Port_Tooltip:The port number of your HTTP proxy server
+Preferences.Connection.HTTPS_Proxy_Host:HTTPS Proxy Host:
+Preferences.Connection.HTTPS_Proxy_Host_Tooltip:The address of your HTTPS proxy server
+Preferences.Connection.HTTPS_Proxy_Port_Tooltip:The port number of your HTTPS proxy server
+Preferences.Connection.FTP_Proxy_Host:FTP Proxy Host:
+Preferences.Connection.FTP_Proxy_Host_Tooltip:The address of your FTP proxy server
+Preferences.Connection.FTP_Proxy_Port_Tooltip:The port number of your FTP proxy server
+Preferences.Connection.Proxy_Host_Missing:Enter a value for at least one proxy host or turn off Use Proxy Connection in the Connection tab.
+Preferences.Connection.Proxy_Port:Port:
 Preferences.Connection.Servlet:Servlet:
 Preferences.Connection.Servlet_Tooltip:The servlet to use to view your site

main/trunk/gli/src/org/greenstone/gatherer/Gatherer.java

@@ -1453 +1453 @@
         try {// Can throw several exceptions
             if(Configuration.get("general.use_proxy", true)) {
+                // These are Java properties, therefore see
+                // https://docs.oracle.com/javase/7/docs/api/java/net/doc-files/net-properties.html
+                // https://docs.oracle.com/javase/8/docs/technotes/guides/net/proxies.html
+                // https://stackoverflow.com/questions/14243590/proxy-settings-in-java
+
+                // Not sure what proxyType is. And proxySet ceased to exist since JDK 6 or before
                 System.setProperty("http.proxyType", "4");
-                System.setProperty("http.proxyHost", Configuration.getString("general.proxy_host", true));
-                System.setProperty("http.proxyPort", Configuration.getString("general.proxy_port", true));
-                System.setProperty("http.proxySet", "true");
+                System.setProperty("http.proxyHost", Configuration.getString("general.HTTP_proxy_host", true));
+                System.setProperty("http.proxyPort", Configuration.getString("general.HTTP_proxy_port", true));
+                //System.setProperty("http.proxySet", "true");
+                
+                System.setProperty("https.proxyHost", Configuration.getString("general.HTTPS_proxy_host", true));
+                System.setProperty("https.proxyPort", Configuration.getString("general.HTTPS_proxy_port", true));
+
+                System.setProperty("ftp.proxyHost", Configuration.getString("general.FTP_proxy_host", true));
+                System.setProperty("ftp.proxyPort", Configuration.getString("general.FTP_proxy_port", true));
+                
             } else {
                 System.setProperty("http.proxyHost", "");
                 System.setProperty("http.proxyPort", "");
-                System.setProperty("http.proxySet", "false");
+                //System.setProperty("http.proxySet", "false");
+                System.setProperty("https.proxyHost", "");
+                System.setProperty("https.proxyPort", "");
+                System.setProperty("ftp.proxyHost", "");
+                System.setProperty("ftp.proxyPort", "");
             }
         } catch (Exception error) {

main/trunk/gli/src/org/greenstone/gatherer/download/DownloadJob.java

@@ -107 +107 @@
     private final String mode;
    
-    private String proxy_url; // only the main thread (where DownloadJob runs) modifies this, so no synching needed
+    private Properties proxy_urls; // only the main thread (where DownloadJob runs) modifies this, so no synching needed
     
     /**
      */
-    public DownloadJob(Download download, String proxy_pass, String proxy_user, DownloadScrollPane mummy, String mode, String proxy_url) {
+    public DownloadJob(Download download, String proxy_pass, String proxy_user, DownloadScrollPane mummy, String mode, Properties proxy_urls) {
     URL url = null;
     int folder_hash;
 
-        this.proxy_url =  proxy_url;
+        this.proxy_urls =  proxy_urls;
     
         download_option = downloadToHashMap(download);
@@ -310 +310 @@
         Process prcs = null; 
 
-        if (proxy_url != null && !proxy_url.equals("")) {           
+        if (proxy_urls.size() != 0) {
         // Specify proxies as environment variables
         // Need to manually specify GSDLHOME and GSDLOS also
+        env = new String[4+proxy_urls.size()];
         
-        proxy_url = proxy_url.replaceAll("http://","");
-        
-        env = new String[7];
-        
-        env[0] = "http_proxy=http://"+proxy_url;
-        env[1] = "https_proxy=http://"+proxy_url; // yes, HTTP protocol for https:// too
-        // see also https://wiki.archlinux.org/index.php/proxy_settings
-        env[2] = "ftp_proxy=ftp://"+proxy_url;
-        env[3] = "GSDLHOME=" + Configuration.gsdl_path;
-        env[4] = "GSDLOS=" + Gatherer.client_operating_system;
+        env[0] = "GSDLHOME=" + Configuration.gsdl_path;
+        env[1] = "GSDLOS=" + Gatherer.client_operating_system;
         // teach it where the wgetrc file lives, in gs2build/bin/<os>:
-        env[5] = "WGETRC=" + LocalGreenstone.getBinOSDirectoryPath(Gatherer.client_operating_system)+"wgetrc";
+        env[2] = "WGETRC=" + LocalGreenstone.getBinOSDirectoryPath(Gatherer.client_operating_system)+"wgetrc";
         // Issue discovered on Windows: If PATH is not made available to perl, so that wget or something else needed by
         // WgetDownload.pm's open3()  call is not on the PATH, then the perl open() call to run wget fails.
         // So make PATH available to get open3() working:
-        env[6] = "PATH="+System.getenv("PATH");     
+        env[3] = "PATH="+System.getenv("PATH");
+
+        int next_index = 4;
+        String proxy_url = proxy_urls.getProperty("HTTP");
+        if(proxy_url != null) {
+            env[next_index] = "http_proxy="+proxy_url;
+            next_index++;
+        }
+        proxy_url = proxy_urls.getProperty("HTTPS");
+        if(proxy_url != null) {
+            env[next_index] = "https_proxy="+proxy_url;
+            next_index++;
+        }
+        proxy_url = proxy_urls.getProperty("FTP");
+        if(proxy_url != null) {
+            env[next_index] = "ftp_proxy="+proxy_url;
+            next_index++;
+        }               
         
         prcs = rt.exec(cmd, env);
@@ -623 +633 @@
         String [] env = null;
 
-        if (proxy_url != null && !proxy_url.equals("")) {           
+        if (proxy_urls.size() != 0) {
         // Specify proxies as environment variables
         // Need to manually specify GSDLHOME and GSDLOS also
+        env = new String[4+proxy_urls.size()];
         
-        proxy_url = proxy_url.replaceAll("http://","");
-        
-        env = new String[7];
-        
-        env[0] = "http_proxy=http://"+proxy_url;
-        env[1] = "https_proxy=http://"+proxy_url; // yes, HTTP protocol for https:// too
-        // see also https://wiki.archlinux.org/index.php/proxy_settings
-        env[2] = "ftp_proxy=ftp://"+proxy_url;
-        env[3] = "GSDLHOME=" + Configuration.gsdl_path;
-        env[4] = "GSDLOS=" + Gatherer.client_operating_system;
+        env[0] = "GSDLHOME=" + Configuration.gsdl_path;
+        env[1] = "GSDLOS=" + Gatherer.client_operating_system;
         // teach it where the wgetrc file lives, in gs2build/bin/<os>:
-        env[5] = "WGETRC=" + LocalGreenstone.getBinOSDirectoryPath(Gatherer.client_operating_system)+"wgetrc";
+        env[2] = "WGETRC=" + LocalGreenstone.getBinOSDirectoryPath(Gatherer.client_operating_system)+"wgetrc";
         // Issue discovered on Windows: If PATH is not made available to perl, so that wget or something else needed by
         // WgetDownload.pm's open3()  call is not on the PATH, then the perl open() call to run wget fails.
         // So make PATH available to get open3() working:
-        env[6] = "PATH="+System.getenv("PATH");     
+        env[3] = "PATH="+System.getenv("PATH");
+
+        int next_index = 4;
+        String proxy_url = proxy_urls.getProperty("HTTP");
+        if(proxy_url != null) {
+            env[next_index] = "http_proxy="+proxy_url;          
+            next_index++;
+        }
+        proxy_url = proxy_urls.getProperty("HTTPS");
+        if(proxy_url != null) {
+            env[next_index] = "https_proxy="+proxy_url;
+            next_index++;
+        }
+        proxy_url = proxy_urls.getProperty("FTP");
+        if(proxy_url != null) {
+            env[next_index] = "ftp_proxy="+proxy_url;
+            next_index++;
+        }               
         
         prcs = new SafeProcess(cmd, env, null);
-        } 
+        }
         else if(Gatherer.isGsdlRemote && Gatherer.isDownloadEnabled && !Utility.isWindows()) {
         // Not Windows, but running client with download panel

main/trunk/gli/src/org/greenstone/gatherer/download/DownloadScrollPane.java

@@ -183 +183 @@
     }
 
-    public void newDownloadJob(Download download, String mode, String proxy_url) {
+    public void newDownloadJob(Download download, String mode, Properties proxy_urls) {
     // Create the job and fill in the details from gatherer.config.
 
     DebugStream.println("About to create a new job");
     
-        DownloadJob new_job = new DownloadJob(download, Configuration.proxy_pass, Configuration.proxy_user, this, mode, proxy_url);
+        DownloadJob new_job = new DownloadJob(download, Configuration.proxy_pass, Configuration.proxy_user, this, mode, proxy_urls);
     // Tell it to run as soon as possible
 

main/trunk/gli/src/org/greenstone/gatherer/download/ServerInfoDialog.java

@@ -8 +8 @@
 import java.io.*;
 import java.util.ArrayList;
+import java.util.Properties;
 import org.greenstone.gatherer.cdm.Argument;
 import org.greenstone.gatherer.Configuration;
@@ -27 +28 @@
     private JDialog info_dialog;
     private Download download;
-    private String proxy_url;
+    private Properties proxy_urls;
     
-    public ServerInfoDialog(String url, String proxy_url, String mode, Download download) {
+    public ServerInfoDialog(String url, Properties proxy_urls, String mode, Download download) {
     super();
     this.url = url;
     this.mode = mode;
         this.download = download;
-        this.proxy_url = proxy_url;
+        this.proxy_urls = proxy_urls;
     this.info_dialog = this;
     
@@ -106 +107 @@
     try {       
         SafeProcess prcs = null;
-        
-        if (proxy_url != null && !proxy_url.equals("")) {
-            // Specify proxies as environment variables
-            // Need to manually specify GSDLHOME and GSDLOS also
-            
-            proxy_url = proxy_url.replaceAll("http://","");
-            
-            String [] env = new String[7];
-            
-            env[0] = "http_proxy=http://"+proxy_url;
-            env[1] = "https_proxy=http://"+proxy_url; // yes, HTTP protocol for https:// too
-            // see also https://wiki.archlinux.org/index.php/proxy_settings
-            env[2] = "ftp_proxy=ftp://"+proxy_url;
-            env[3] = "GSDLHOME=" + Configuration.gsdl_path;
-            env[4] = "GSDLOS=" + Gatherer.client_operating_system;
-            // teach it where the wgetrc file lives, in gs2build/bin/<os>:
-            env[5] = "WGETRC=" + LocalGreenstone.getBinOSDirectoryPath(Gatherer.client_operating_system)+"wgetrc";
-            // Issue discovered on Windows: If PATH is not made available to perl, so that wget or something else needed by
-            // WgetDownload.pm's open3()  call is not on the PATH, then the perl open() call to run wget fails.
-            // So make PATH available to get open3() working:
-            env[6] = "PATH="+System.getenv("PATH");
-            
-            prcs = new SafeProcess(command, env, null);
-        } 
-        else {
-            // Will inherit the GLI's environment, with GSDLHOME and GSDLOS set
-            prcs = new SafeProcess(command);
+
+        if (proxy_urls.size() != 0) {
+        // Specify proxies as environment variables
+        // Need to manually specify GSDLHOME and GSDLOS also
+        String[] env = new String[4+proxy_urls.size()];
+
+        env[0] = "GSDLHOME=" + Configuration.gsdl_path;
+        env[1] = "GSDLOS=" + Gatherer.client_operating_system;
+        // teach it where the wgetrc file lives, in gs2build/bin/<os>:
+        env[2] = "WGETRC=" + LocalGreenstone.getBinOSDirectoryPath(Gatherer.client_operating_system)+"wgetrc";
+        // Issue discovered on Windows: If PATH is not made available to perl, so that wget or something else needed by
+        // WgetDownload.pm's open3()  call is not on the PATH, then the perl open() call to run wget fails.
+        // So make PATH available to get open3() working:
+        env[3] = "PATH="+System.getenv("PATH");     
+
+        int next_index = 4;
+        String proxy_url = proxy_urls.getProperty("HTTP");
+        if(proxy_url != null) {
+            env[next_index] = "http_proxy="+proxy_url;
+            next_index++;
         }
+        proxy_url = proxy_urls.getProperty("HTTPS");
+        if(proxy_url != null) {
+            env[next_index] = "https_proxy="+proxy_url;
+            next_index++;
+        }
+        proxy_url = proxy_urls.getProperty("FTP");
+        if(proxy_url != null) {
+            env[next_index] = "ftp_proxy="+proxy_url;
+            next_index++;
+        }               
+        
+        prcs = new SafeProcess(command, env, null);
+        } else {
+        // Will inherit the GLI's environment, with GSDLHOME and GSDLOS set
+        prcs = new SafeProcess(command);
+        }
         
         // special processing of Process' stderr stream: 

main/trunk/gli/src/org/greenstone/gatherer/gui/DownloadPane.java

@@ -91 +91 @@
     private String mode = null;
     private TreePath previous_path;
-    private String proxy_url = "";
+    private Properties proxy_urls = new Properties(); // proxy_urls for each of HTTP, HTTPS, FTP
     private Proxy proxyObject = null;
         
@@ -601 +601 @@
         }
         
-        getter.newDownloadJob((Download)download_map.get(mode) ,mode,proxy_url);
+        getter.newDownloadJob((Download)download_map.get(mode) ,mode,proxy_urls);
         }
     }
@@ -723 +723 @@
     }
 
-    // TODO: Still need to read up on and test how to set wget proxy on windows
-    // Need to be on a windows machine that requires proxy, and get wget to work on cmdline
-    // If that works, then need to check https URLS also work.
-    // See https://superuser.com/questions/526710/how-to-set-http-proxy-address-for-wget-under-windows
-    private boolean checkProxy(){
+    
+    // SOME READING:
+    // https://superuser.com/questions/526710/how-to-set-http-proxy-address-for-wget-under-windows
+    // https://arstechnica.com/civis/viewtopic.php?f=10&t=1281165
+    // "http or https to an https proxy?"
+    // About "CONNECT": https://daniel.haxx.se/docs/sshproxy.html
+    // "You need an SSH client that can issue CONNECT requests through the company HTTP proxy." Seems to imply http? Also uses it for ftp.
+    // https://forum.ivorde.com/set-up-ftp-proxy-via-command-line-in-linux-freebsd-t19733.html sets ftp_proxy to a http url.
+    // So it's seems to be just whatever protocol the proxy server has. When the proxy supports all three protocols
+    // (apparently the common case as per page below describing firefox prefs), then wiki.archlinux sets all three ftp_proxy/http_proxy/https_proxy to the same.
+    // See https://forums.freebsd.org/threads/57378/
+    // "I believe that ftp(1) only uses HTTP-type proxies for fetching URLs. I.e. you can't do traditional open->cd->get interactive style of FTP with it via a HTTP proxy. If you do something like ftp ftp://ftp.example.com/path/file, it should work with your proxy setup. For traditional interactive FTP, you need to be directly talking to the remote server or using the less common FTP proxy/gate functionality."    
+    // https://wiki.archlinux.org/index.php/Proxy_settings
+    // This does https_proxy = http_proxy, and explicitly prefixes "https://" to http_proxy.
+    // https://www.howtogeek.com/293213/how-to-configure-a-proxy-server-in-firefox/
+    /* You’ll usually want to click the “Use the proxy server for all protocols” option. Firefox will also use your HTTP proxy server for SSL-encrypted HTTPS connections and File Transfer Protocol (FTP) connections.
+Uncheck this box if you want to enter separate proxy servers for HTTP, HTTPS, and FTP connections. This isn’t common.
+If you’re configuring a SOCKS proxy, leave the HTTP Proxy, SSL Proxy, and FTP Proxy boxes empty. Enter the address of the SOCKS proxy into the “SOCKS Host” and its port into the “Port” box.
+    */    
+    private boolean checkProxy(){    
+    
+    proxy_urls.clear();
+    
+    Download current_download = (Download)download_map.get(mode);
+    
+        Argument arg = current_download.getArgument("proxy_on");
+  
+    if (arg == null) return true;
+    
+    // Determine if we have to use a proxy.
+    if(Configuration.get("general.use_proxy", true)) {
+        
+        boolean http_proxy_set = setProxyURLFor("HTTP");
+        boolean https_proxy_set = setProxyURLFor("HTTPS");
+        boolean ftp_proxy_set = setProxyURLFor("FTP");
+        
+        if(proxy_urls.size() == 0 ||
+           (!http_proxy_set && !https_proxy_set && !ftp_proxy_set)) {
+        
+        // if checkProxy() failed for all protocols
+        // OR if none of the proxies were setup by user, then turn off proxying
+        arg = current_download.getArgument("proxy_on");
+        arg.setValue("false"); 
+        arg.setAssigned(false);
+        proxy_urls.clear();
+        return false;
+        } else { // proxy details have been successfully set for at least one proxy protocol
+        return true;
+        }   
+
+    } else { // if proxy_on was off
+        // unset proxy_on argument too
+        arg = current_download.getArgument("proxy_on");
+        arg.setValue("false"); 
+        arg.setAssigned(false);
+    }
+    
+    return true;
+    }
+
+    private boolean setProxyURLFor(String protocol) {
+    String proxy_host = Configuration.getString("general."+protocol+"_proxy_host", true);
+    if(proxy_host.equals("")) { // no proxy details for this protocol
+        ///System.err.println("### general."+protocol+"proxy_host was empty");
+        return true;
+    }
+    
+    String proxy_port = Configuration.getString("general."+protocol+"_proxy_port", true);
+    // Find out whether the user has already authenticated themselves
+    
+    // remove the protocol prefix from proxy_host, and store it
+    String proxy_protocol = "";
+    int proxy_protocol_index = proxy_host.indexOf("://");
+    if(proxy_protocol_index != -1) {
+        proxy_protocol_index += "://".length();
+        proxy_protocol = proxy_host.substring(0, proxy_protocol_index);
+        proxy_host = proxy_host.substring(proxy_protocol_index);
+    } else { // no explicit protocol for proxy host specified,
+        // then set explicit protocol to be the same as the protocol param: http|https|ftp
+        proxy_protocol = protocol.toLowerCase()+"://";
+    }
+    
+    String user_pass = "";
+    String address = proxy_host + ":" + proxy_port;
+    
+    int count = 0;
+    // Only for wget, need to avoid a second automatic authentication popup (first asks
+    // the proxy authentication for wget, and the second will ask the same for the realm)
+    // Once the authentication has been reused, it will set the GAuthenticator state back to REGULAR
+    GAuthenticator.setMode(GAuthenticator.DOWNLOAD);
+    while(count < 3 && (user_pass = (String) GAuthenticator.authentications.get(address)) == null) {
+        Authenticator.requestPasswordAuthentication(proxy_host, null, Integer.parseInt(proxy_port), "http://", Dictionary.get("WGet.Prompt"), "HTTP");
+        count++;
+    }
+    if(count >= 3) {
+        return false;
+    }
+    
+    // https://askubuntu.com/questions/664777/systemwide-proxy-settings-in-ubuntu
+    // http://www.rgagnon.com/javadetails/java-0085.html
+    // how-do-i-make-httpurlconnection-use-a-proxy
+    // https://stackoverflow.com/questions/8030908/how-to-check-if-proxy-is-working-in-java
+    
+    //proxyObject = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxy_host, Integer.parseInt(proxy_port))); // proxyObject only used by getRedirectURL(), which we're no longer calling
+    
+    Download current_download = (Download)download_map.get(mode);
+    Argument arg = current_download.getArgument("proxy_on");
+    
+    if(user_pass.indexOf("@") != -1) {
+        
+        arg.setValue("true"); // proxy_on argument
+        arg.setAssigned(true);
+        
+        arg = current_download.getArgument(protocol.toLowerCase()+"_proxy_host");
+        arg.setValue(proxy_host);
+        arg.setAssigned(true);
+        
+        arg = current_download.getArgument(protocol.toLowerCase()+"_proxy_port");
+        arg.setValue(proxy_port);
+        arg.setAssigned(true);
+
+        // Write the use proxy command - we don't do this anymore, instead we set environment
+        // variables as these do work for windows after all (Aug 2017). Hopefully these can't
+        // be spied on like the following can (using ps/task manager. If going back to the
+        // following, be aware: can't add protocol+user_name and protocol+user_password as
+        // accepted parameters in WgetDownload.pm for HTTP, HTTPS and FTP protocols. Only
+        // one set of username and password can be set for wget's --proxy-user and
+        // --proxy-password flags. No separate flags for each protocol.
+        /*if (Utility.isWindows()) {
+          
+          arg = current_download.getArgument("user_name");
+          arg.setValue(user_pass.substring(0, user_pass.indexOf("@")));
+          arg.setAssigned(true);
+          
+          arg = current_download.getArgument("user_password");
+          arg.setValue(user_pass.substring(user_pass.indexOf("@") + 1));
+          arg.setAssigned(true);
+          }
+          
+          else{*/
+        String user_name = user_pass.substring(0, user_pass.indexOf("@"));
+        String user_pwd = user_pass.substring(user_pass.indexOf("@") + 1);
+        /*}*/
+        
+        // construct proxy_url and prefix the stored proxy protocol to it
+        String proxy_url = proxy_protocol+user_name+":"+user_pwd+"@"+proxy_host+":"+proxy_port+"/";     
+        proxy_urls.setProperty(protocol, proxy_url);
+        return true;
+    }
+    else{
+        return false;
+    }   
+    }
+    
+    
+    private boolean old_checkProxy(){
      
-    proxy_url = null;
-
+    String proxy_url = null; // ORIGINALLY A MEMBER VAR
+    
     Download current_download = (Download)download_map.get(mode);
     
@@ -860 +1011 @@
         }
 
-        server_info = new ServerInfoDialog(str_url ,proxy_url, mode,(Download)download_map.get(mode));
+        server_info = new ServerInfoDialog(str_url ,proxy_urls, mode,(Download)download_map.get(mode));
         
     }

main/trunk/gli/src/org/greenstone/gatherer/gui/Preferences.java

@@ -56 +56 @@
     extends ModalDialog
 {
+    static final public String HTTP = "HTTP";
+    static final public String HTTPS = "HTTPS";
+    static final public String FTP = "FTP";
+    
     static final public String CONNECTION_PREFS = "connection";
     static final public String GENERAL_PREFS = "general";
@@ -86 +90 @@
     private JRadioButton expert_mode_radio_button;
     private JRadioButton librarian_mode_radio_button;
-    private JSpinner proxy_port_field;
+    private JSpinner http_proxy_port_field;
+    private JSpinner https_proxy_port_field;
+    private JSpinner ftp_proxy_port_field;
     private JTabbedPane tab_pane;
     private JTextArea mode_description_textarea;
@@ -93 +99 @@
     private JTextField library_path_field;
     private JTextField program_field;
-    private JTextField proxy_host_field;
+    private JTextField http_proxy_host_field;
+    private JTextField https_proxy_host_field;
+    private JTextField ftp_proxy_host_field;
     private JTextField collect_dir_field;
 
@@ -337 +345 @@
     use_proxy_checkbox.setPreferredSize(ROW_SIZE);
     
-        JPanel proxy_host_pane = new JPanel();
-        proxy_host_pane.setComponentOrientation(Dictionary.getOrientation());
-    proxy_host_pane.setPreferredSize(ROW_SIZE);
-        
-    JLabel proxy_host_label = new JLabel(Dictionary.get("Preferences.Connection.Proxy_Host"));
-        proxy_host_label.setComponentOrientation(Dictionary.getOrientation());
-    proxy_host_label.setPreferredSize(LABEL_SIZE);
-    
-    proxy_host_field = new JTextField(Configuration.getString("general.proxy_host", true));
-    proxy_host_field.setEnabled(currently_enabled);
-    proxy_host_field.setToolTipText(Dictionary.get("Preferences.Connection.Proxy_Host_Tooltip"));
-    proxy_host_field.setComponentOrientation(Dictionary.getOrientation());
-        
-    JPanel proxy_port_pane = new JPanel();
-        proxy_port_pane.setComponentOrientation(Dictionary.getOrientation());
-    proxy_port_pane.setPreferredSize(ROW_SIZE);
-        
-    JLabel proxy_port_label = new JLabel(Dictionary.get("Preferences.Connection.Proxy_Port"));
-    proxy_port_label.setPreferredSize(LABEL_SIZE);
-        proxy_port_label.setComponentOrientation(Dictionary.getOrientation());
-    
-    String port_value = Configuration.getString("general.proxy_port", true);
-    if(port_value.length() > 0) {
-       proxy_port_field = new JSpinner(new SpinnerNumberModel((new Integer(port_value)).intValue(), 0, 65535, 1));
-    }
-    else {
-       proxy_port_field = new JSpinner(new SpinnerNumberModel(0, 0, 65535, 1));
-    }
-    proxy_port_field.setEnabled(currently_enabled);
-    proxy_port_field.setToolTipText(Dictionary.get("Preferences.Connection.Proxy_Port_Tooltip"));
     
     // Connection
@@ -405 +383 @@
     collect_dir_pane.add(chdir_button, BorderLayout.LINE_END);
 
-    proxy_host_pane.setLayout(new BorderLayout());
-    proxy_host_pane.add(proxy_host_label, BorderLayout.LINE_START);
-    proxy_host_pane.add(proxy_host_field, BorderLayout.CENTER);
-
-    proxy_port_pane.setLayout(new BorderLayout());
-    proxy_port_pane.add(proxy_port_label, BorderLayout.LINE_START);
-    proxy_port_pane.add(proxy_port_field, BorderLayout.CENTER);
 
     connection_pane.setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
-    connection_pane.setLayout(new GridLayout(9,1,0,2));
+    connection_pane.setLayout(new GridLayout(10,1,0,2));
     connection_pane.add(program_pane);
     connection_pane.add(library_path_pane);
@@ -428 +399 @@
     connection_pane.add(no_check_certificate_checkbox);
     connection_pane.add(use_proxy_checkbox);
-    connection_pane.add(proxy_host_pane);
-    connection_pane.add(proxy_port_pane);
+    
+    // set up the proxy host and port fields for each of http, https and ftp    
+        setupProxyHostPane(HTTP, connection_pane, currently_enabled, http_proxy_host_field, http_proxy_port_field);
+    setupProxyHostPane(HTTPS, connection_pane, currently_enabled, https_proxy_host_field, https_proxy_port_field);
+    setupProxyHostPane(FTP, connection_pane, currently_enabled, ftp_proxy_host_field, ftp_proxy_port_field);
 
     return connection_pane;
@@ -570 +544 @@
     return general_pane;
     }
+
+    private void setupProxyHostPane(String protocol, JPanel connection_pane, boolean isEnabled,
+                    JTextField proxy_host_field, JSpinner proxy_port_field)
+    {
+    final Dimension WIDER_LABEL_SIZE = new Dimension(180, 25);
+    final Dimension NARROWER_LABEL_SIZE = new Dimension(60, 25);
+    final Dimension WIDER_ROW_SIZE = new Dimension(480, 25);
+    final Dimension NARROWER_ROW_SIZE = new Dimension(120, 25);
+    
+    JPanel proxy_host_pane = new JPanel();
+        proxy_host_pane.setComponentOrientation(Dictionary.getOrientation());
+    proxy_host_pane.setPreferredSize(WIDER_ROW_SIZE);
+        
+    JLabel proxy_host_label = new JLabel(Dictionary.get("Preferences.Connection."+protocol+"_Proxy_Host"));
+        proxy_host_label.setComponentOrientation(Dictionary.getOrientation());
+    proxy_host_label.setPreferredSize(WIDER_LABEL_SIZE);
+    
+    proxy_host_field = new JTextField(Configuration.getString("general."+protocol+"_proxy_host", true));
+    proxy_host_field.setEnabled(isEnabled);
+    proxy_host_field.setToolTipText(Dictionary.get("Preferences.Connection."+protocol+"_Proxy_Host_Tooltip"));
+    proxy_host_field.setComponentOrientation(Dictionary.getOrientation());
+        
+    JPanel proxy_port_pane = new JPanel();
+        proxy_port_pane.setComponentOrientation(Dictionary.getOrientation());
+    proxy_port_pane.setPreferredSize(NARROWER_ROW_SIZE);
+        
+    JLabel proxy_port_label = new JLabel(Dictionary.get("Preferences.Connection.Proxy_Port"));
+    proxy_port_label.setPreferredSize(NARROWER_LABEL_SIZE);
+        proxy_port_label.setComponentOrientation(Dictionary.getOrientation());
+    
+    String port_value = Configuration.getString("general."+protocol+"_proxy_port", true);
+    if(port_value.length() > 0) {
+       proxy_port_field = new JSpinner(new SpinnerNumberModel((new Integer(port_value)).intValue(), 0, 65535, 1));
+    }
+    else {
+       proxy_port_field = new JSpinner(new SpinnerNumberModel(0, 0, 65535, 1));
+    }
+    proxy_port_field.setEnabled(isEnabled);
+    proxy_port_field.setToolTipText(Dictionary.get("Preferences.Connection."+protocol+"_Proxy_Port_Tooltip"));
+
+
+    // Since the actual object member vars were null when this method is called
+    // need to ensure they are assigned the correct value now, else those refs will remain null
+    if(protocol.equals(HTTP)) {
+        http_proxy_host_field = proxy_host_field;
+        http_proxy_port_field = proxy_port_field;
+    } else if (protocol.equals(HTTPS)) {
+        https_proxy_host_field = proxy_host_field;
+        https_proxy_port_field = proxy_port_field;
+    } else if(protocol.equals(FTP)) {
+        ftp_proxy_host_field = proxy_host_field;
+        ftp_proxy_port_field = proxy_port_field;
+    } 
+    
+    proxy_host_pane.setLayout(new BorderLayout());
+    proxy_host_pane.add(proxy_host_label, BorderLayout.LINE_START);
+    proxy_host_pane.add(proxy_host_field, BorderLayout.CENTER);
+
+    proxy_port_pane.setLayout(new BorderLayout());
+    proxy_port_pane.add(proxy_port_label, BorderLayout.LINE_START);
+    proxy_port_pane.add(proxy_port_field, BorderLayout.CENTER);
+
+    //connection_pane.add(proxy_host_pane);
+    //connection_pane.add(proxy_port_pane);
+
+    JPanel proxy_panel = new JPanel();
+    proxy_panel.setLayout(new BorderLayout());  
+    proxy_panel.add(proxy_host_pane, BorderLayout.WEST);
+    proxy_panel.add(proxy_port_pane, BorderLayout.CENTER);
+
+    connection_pane.add(proxy_panel);
+    }
+    
 
     /** Generate the controls for altering the detail mode.
@@ -786 +833 @@
         this.close = close;
     }
+
+    private void setProxyHostForProtocol(String protocol,
+                         JTextField proxy_host_field,
+                         JSpinner proxy_port_field)
+    {       
+        Configuration.setString("general."+protocol+"_proxy_host", true, proxy_host_field.getText());
+        Configuration.setString("general."+protocol+"_proxy_port", true, proxy_port_field.getValue() + "");
+    }
+    
     public void actionPerformed(ActionEvent event)
     {
@@ -876 +932 @@
         
         Configuration.set("general.use_proxy", true, use_proxy_checkbox.isSelected());
-        Configuration.setString("general.proxy_host", true, proxy_host_field.getText());
-        Configuration.setString("general.proxy_port", true, proxy_port_field.getValue() + "");
+        setProxyHostForProtocol(HTTP, http_proxy_host_field, http_proxy_port_field);
+        setProxyHostForProtocol(HTTPS, https_proxy_host_field, https_proxy_port_field);
+        setProxyHostForProtocol(FTP, ftp_proxy_host_field, ftp_proxy_port_field);
         Gatherer.setProxy();
 
@@ -975 +1032 @@
 
         // If proxy is on but proxy details are incomplete, then can't continue 
-        if (use_proxy_checkbox.isSelected() && proxy_host_field.getText().equals("")) {
+        if (use_proxy_checkbox.isSelected()
+        && http_proxy_host_field.getText().equals("")
+        && https_proxy_host_field.getText().equals("")
+        && ftp_proxy_host_field.getText().equals("")) {
         JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("Preferences.Connection.Proxy_Host_Missing"),
                           Dictionary.get("General.Error"), JOptionPane.ERROR_MESSAGE);
@@ -1144 +1204 @@
         Configuration.set("general.use_proxy", true, enabled);
         // Fortunately this is already driven by the event thread.
-        proxy_host_field.setEnabled(enabled);
-        proxy_port_field.setEnabled(enabled);
+        http_proxy_host_field.setEnabled(enabled);
+        http_proxy_port_field.setEnabled(enabled);
+        https_proxy_host_field.setEnabled(enabled);
+        https_proxy_port_field.setEnabled(enabled);
+        ftp_proxy_host_field.setEnabled(enabled);
+        ftp_proxy_port_field.setEnabled(enabled);
     }
     }

main/trunk/greenstone2/perllib/downloaders/WebDownload.pm

@@ -193 +193 @@
     my $strBaseCMD = $strOptions." --tries=2 -q -O - \"$self->{'url'}\"";
 
-    #&util::print_env(STDERR, "https_proxy", "http_proxy", "HTTPS_PROXY", "HTTP_PROXY", "ftp_proxy", "FTP_PROXY");
+    #&util::print_env(STDERR, "https_proxy", "http_proxy", "ftp_proxy");
     #&util::print_env(STDERR);  
     
@@ -201 +201 @@
     
     print STDERR "Server information is unavailable.\n";
-
-    #&util::print_env(STDERR, "https_proxy", "http_proxy");
-    
-    if ($self->{'proxy_on'} && $self->{'proxy_host'} && $self->{'proxy_port'}) { # if proxying set, the settings may be wrong
-        &gsprintf::gsprintf_multiline(STDERR, "{WebDownload.proxied_connect_failed_info}\n", $self->{'proxy_host'}, $self->{'proxy_port'});
+    
+    if ($self->{'proxy_on'}) { # if proxying set, the settings may be wrong
+        &gsprintf::gsprintf_multiline(STDERR, "{WebDownload.proxied_connect_failed_info}\n");
+        
+        if($self->{'http_proxy_host'} && defined $self->{'http_proxy_port'}) {
+        &gsprintf::gsprintf_multiline(STDERR, "{WebDownload.http_proxy_settings}\n", $self->{'http_proxy_host'}, $self->{'http_proxy_port'});
+        }
+        if($self->{'https_proxy_host'} && defined $self->{'https_proxy_port'}) {
+        &gsprintf::gsprintf_multiline(STDERR, "{WebDownload.https_proxy_settings}\n", $self->{'https_proxy_host'}, $self->{'https_proxy_port'});
+        }
+        if($self->{'ftp_proxy_host'} && defined $self->{'ftp_proxy_port'}) {
+        &gsprintf::gsprintf_multiline(STDERR, "{WebDownload.ftp_proxy_settings}\n", $self->{'ftp_proxy_host'}, $self->{'ftp_proxy_port'});
+        }
     } else { # else no proxy set, the user may need proxy settings
         &gsprintf::gsprintf_multiline(STDERR, "{WebDownload.proxyless_connect_failed_info}\n");

main/trunk/greenstone2/perllib/downloaders/WgetDownload.pm

@@ -51 +51 @@
     'reqd' => "no",
     'hiddengli' => "yes"},
-      { 'name' => "proxy_host",  
-    'desc' => "{WgetDownload.proxy_host}",         
-    'type' => "string",
-    'reqd' => "no",
-    'hiddengli' => "yes"},
-      { 'name' => "proxy_port", 
-    'desc' => "{WgetDownload.proxy_port}",         
+      { 'name' => "http_proxy_host",
+    'desc' => "{WgetDownload.http_proxy_host}",
+    'type' => "string",
+    'reqd' => "no",
+    'hiddengli' => "yes"},
+      { 'name' => "http_proxy_port",
+    'desc' => "{WgetDownload.http_proxy_port}",
+    'type' => "string",
+    'reqd' => "no",
+    'hiddengli' => "yes"},
+      { 'name' => "https_proxy_host",  
+    'desc' => "{WgetDownload.https_proxy_host}",         
+    'type' => "string",
+    'reqd' => "no",
+    'hiddengli' => "yes"},
+      { 'name' => "https_proxy_port", 
+    'desc' => "{WgetDownload.https_proxy_port}",         
+    'type' => "string",
+    'reqd' => "no",
+    'hiddengli' => "yes"},
+      { 'name' => "ftp_proxy_host",  
+    'desc' => "{WgetDownload.ftp_proxy_host}",         
+    'type' => "string",
+    'reqd' => "no",
+    'hiddengli' => "yes"},
+      { 'name' => "ftp_proxy_port", 
+    'desc' => "{WgetDownload.ftp_proxy_port}",         
     'type' => "string",
     'reqd' => "no",
     'hiddengli' => "yes"},
       { 'name' => "user_name",  
-    'desc' => "{WgetDownload.user_name}",         
+    'desc' => "{WgetDownload.user_name}",
     'type' => "string",
     'reqd' => "no",
     'hiddengli' => "yes"},
       { 'name' => "user_password", 
-    'desc' => "{WgetDownload.user_password}",         
+    'desc' => "{WgetDownload.user_password}",
     'type' => "string",
     'reqd' => "no",
@@ -171 +191 @@
     my ($self) = @_;
     my $strOptions = "";
-    
-    if ($self->{'proxy_on'} && $self->{'proxy_host'} && $self->{'proxy_port'})
+
+    if($self->{'http_proxy_host'} && $self->{'http_proxy_port'}) {
+    $strOptions .= " -e http_proxy=$self->{'http_proxy_host'}:$self->{'http_proxy_port'} ";
+    }
+    if($self->{'https_proxy_host'} && $self->{'https_proxy_port'}) {
+    $strOptions .= " -e https_proxy=$self->{'https_proxy_host'}:$self->{'https_proxy_port'} ";
+    }
+    if($self->{'ftp_proxy_host'} && $self->{'ftp_proxy_port'}) {
+    $strOptions .= " -e ftp_proxy=$self->{'ftp_proxy_host'}:$self->{'ftp_proxy_port'} ";
+    }
+    
+    # For wget, there is only one set pair of proxy-user and proxy-passwd, so wget seems to assume
+    # that all 3 proxy protocols (http|https|ftp) will use the same username and pwd combination?
+    # Note that this only matters when passing the proxying details as flags to wget, not when
+    # the proxies are setup as environment variables.
+    if ($self->{'user_name'} && $self->{'user_password'})
     {
-    $strOptions .= " -e https_proxy=$self->{'proxy_host'}:$self->{'proxy_port'} ";  
-    $strOptions .= " -e http_proxy=$self->{'proxy_host'}:$self->{'proxy_port'} ";       
-    
-    if ($self->{'user_name'} && $self->{'user_password'})
-    {
-        $strOptions .= "--proxy-user=$self->{'user_name'}"." --proxy-passwd=$self->{'user_password'}";
-    }
-    }
-    
-    if ($self->{'proxy_on'}) {
-    $strOptions .= " --proxy ";
-    }
+    $strOptions .= "--proxy-user=$self->{'user_name'}"." --proxy-passwd=$self->{'user_password'}";      
+    # how is "--proxy-passwd" instead of "--proxy-password" even working????        
+    # see https://www.gnu.org/software/wget/manual/html_node/Proxies.html
+    # and https://www.gnu.org/software/wget/manual/wget.html
+    # Not touching this, in case the manual is simply wrong. Since our code works in 
+    # practice (when we were still using wget proxy username/pwd flags for windows).
+    }   
     
     return $strOptions;
@@ -202 +231 @@
     # http://www.perlmonks.org/?node=what%20is%20true%20and%20false%20in%20Perl%3F
     
-    if (!$ENV{'http_proxy'} && !$ENV{'https_proxy'}) {
-    $strOptions .= $self->addProxySettingsAsWgetFlags();
-    } # else wget will use proxy settings in environment, assume enough settings have been provided
+    if ($self->{'proxy_on'}) {
+    if(!$ENV{'http_proxy'} && !$ENV{'https_proxy'} && !$ENV{'ftp_proxy'}) {
+        $strOptions .= $self->addProxySettingsAsWgetFlags();
+    } # else wget will use proxy settings in environment, assume enough settings have been provided
+    # either way, we're using the proxy
+    $strOptions .= " --proxy ";
+    }
     
     if($self->{'no_check_certificate'}) { # URL may be http that gets redirected to https, so if no_check_certificate is on, turn it on even if URL is http

main/trunk/greenstone2/perllib/strings.properties

@@ -1336 +1336 @@
 WebDownload.html_only:Download only HTML files, and ignore associated files e.g images and stylesheets
 WebDownload.html_only_disp:Only HTML files
-WebDownload.proxied_connect_failed_info:Current proxy settings are:\n- host=%s\n- port=%s
+WebDownload.proxied_connect_failed_info:Current proxy settings are:
+WebDownload.http_proxy_settings:- HTTP host=%s : port=%s
+WebDownload.https_proxy_settings:- HTTPS host=%s : port=%s
+WebDownload.ftp_proxy_settings:- FTP host=%s : port=%s
 WebDownload.proxyless_connect_failed_info:- The external server might not be responding\n- or you might need to switch on proxy settings
 WebDownload.connect_failed_info:- or try ticking No Certificate Checking (affects 'https' URLs)\nin File > Preferences > Connection
@@ -1342 +1345 @@
 WgetDownload.desc: Base class that handles calls to wget
 WgetDownload.proxy_on:Proxy on
-WgetDownload.proxy_host:Proxy host
-WgetDownload.proxy_port:Proxy port
+WgetDownload.http_proxy_host:HTTP proxy host
+WgetDownload.http_proxy_port:HTTP proxy port
+WgetDownload.https_proxy_host:HTTPS proxy host
+WgetDownload.https_proxy_port:HTTPS proxy port
+WgetDownload.ftp_proxy_host:FTP proxy host
+WgetDownload.ftp_proxy_port:FTP proxy port
 WgetDownload.user_name:User name
 WgetDownload.user_password:User password