source: main/trunk/gli/src/org/greenstone/gatherer/gui/DownloadPane.java@ 34521

/**
 *#########################################################################
 *
 * A component of the Gatherer application, part of the Greenstone digital
 * library suite from the New Zealand Digital Library Project at the
 * University of Waikato, New Zealand.
 *
 * <BR><BR>
 *
 * Author: John Thompson, Greenstone Digital Library, University of Waikato
 *
 * <BR><BR>
 *
 * Copyright (C) 1999 New Zealand Digital Library Project
 *
 * <BR><BR>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * <BR><BR>
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * <BR><BR>
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 *########################################################################
 */
package org.greenstone.gatherer.gui;

import java.awt.*;
import java.awt.event.*;
import java.io.*;
import java.net.*;
import java.util.*;
import javax.swing.*;
import javax.swing.border.*;
import javax.swing.event.*;
import javax.swing.tree.*;
import org.greenstone.gatherer.Configuration;
import org.greenstone.gatherer.DebugStream;
import org.greenstone.gatherer.Dictionary;
import org.greenstone.gatherer.Gatherer;
import org.greenstone.gatherer.file.WorkspaceTree;
import org.greenstone.gatherer.greenstone.LocalGreenstone;
import org.greenstone.gatherer.util.SafeProcess;
import org.greenstone.gatherer.util.StaticStrings;
import org.greenstone.gatherer.util.Utility;
import org.greenstone.gatherer.download.URLConnectionManager;
import org.greenstone.gatherer.download.Download;
import org.greenstone.gatherer.download.DownloadScrollPane;
import org.greenstone.gatherer.download.ServerInfoDialog;
import org.greenstone.gatherer.util.XMLTools;
import org.greenstone.gatherer.cdm.*;
import org.greenstone.gatherer.gui.*;
import org.w3c.dom.*;
import org.xml.sax.*;
import org.greenstone.gatherer.GAuthenticator;

/**
 * @author John Thompson, Greenstone Digital Library, University of Waikato
 * @version 2.1
 */
public class DownloadPane
    extends JPanel {

    static final private Dimension LABEL_SIZE = new Dimension(225, 25);
    static final private Dimension TREE_SIZE = new Dimension(150, 500);
    //static final private String CONTENTS[] = { "DOWNLOAD.MODE.WebDownload", "DOWNLOAD.MODE.MediaWikiDownload", "DOWNLOAD.MODE.OAIDownload", "DOWNLOAD.MODE.ZDownload" , "DOWNLOAD.MODE.SRWDownload"};
    private String CONTENTS[] = null;

    private boolean download_button_enabled = false;
    private boolean ready = false;

    private JPanel options_pane;
    // TODO should use Vector to store all loaded downloads!!

    private DesignTree tree;
    private HashMap download_map;
    private ServerInfoDialog server_info;
    private JScrollPane list_scroll;
    private DownloadScrollPane getter;
    private String mode = null;
    private TreePath previous_path;
    private Properties proxy_urls = new Properties(); // proxy_urls for each of HTTP, HTTPS, FTP
    private Proxy proxyObject = null;
        
    /** Main System code */
    public DownloadPane() {
    super();
        JScrollPane scrol_tmp;
        this.setComponentOrientation(Dictionary.getOrientation());
    // TODO: Download the WDownload and the download panel fixed!!
    getter = new DownloadScrollPane();
        getter.start();
    list_scroll = getter.getDownloadJobList();      
        list_scroll.setComponentOrientation(Dictionary.getOrientation());

    // TODO should use Vector to store all loaded downloads!!
    String lang = Configuration.getLanguage();
    download_map = new HashMap();

    // run downloadinfo.pl -describeall, load the downloaders into the download_map, 
    // and get back their list of names, which are of the form "<downloader>Download". 
    // Store these names in the CONTENTS[] array as "DOWNLOAD.MODE.<downloader>Download", 
    // with z3950 as a minor exception: DOWNLOAD.MODE.ZDownload.
    ArrayList<String> downloaderNamesList = loadDownloadersInfo(lang);  
    int size = downloaderNamesList.size();
    CONTENTS = new String[size];
    for(int i = 0; i < size; i++) {
        String downloadName = downloaderNamesList.get(i); // e.g. "WebDownload"     
        CONTENTS[i] = "DOWNLOAD.MODE."+downloadName.replace("3950", ""); // A special case is Z3950Download,
                                     // which has to be stored in CONTENTS array as DOWNLOAD.MODE.ZDownload
    }

    // Creation
    tree = new DesignTree();
        tree.setComponentOrientation(Dictionary.getOrientation());
    options_pane = new JPanel();
        options_pane.setComponentOrientation(Dictionary.getOrientation());
    
    JButton clear_cache_button = new GLIButton(Dictionary.get("Mirroring.ClearCache"), Dictionary.get("Mirroring.ClearCache_Tooltip"));
    clear_cache_button.setEnabled(true);
    clear_cache_button.setMnemonic(KeyEvent.VK_C);

    JButton download_button = new GLIButton(Dictionary.get("Mirroring.Download"), Dictionary.get("Mirroring.Download_Tooltip"));
    download_button.setEnabled(true);
    download_button.setMnemonic(KeyEvent.VK_D);

    JButton information_button =  new GLIButton(Dictionary.get("Download.ServerInformation"), Dictionary.get("Download.ServerInformation_Tooltip"));
    information_button.setEnabled(true);
    information_button.setMnemonic(KeyEvent.VK_S);


        JButton preferences_button =  new GLIButton(Dictionary.get("Mirroring.Preferences"), Dictionary.get("Mirroring.Preferences_Tooltip"));
    preferences_button.setEnabled(true);
    preferences_button.setMnemonic(KeyEvent.VK_P);

    // Connect
    clear_cache_button.addActionListener(new ClearCacheListener());
    download_button.addActionListener(new DownloadButtonListener());
    preferences_button.addActionListener(new PreferencesButtonActionListener());
    information_button.addActionListener(new InformationButtonActionListener());
    tree.addTreeSelectionListener(new TreeListener());

    // Add to Panel
    JPanel button_pane = new JPanel();
        button_pane.setComponentOrientation(Dictionary.getOrientation());
    button_pane.setLayout(new GridLayout(1,4)); // GridLayout so button pane resizes with window-width
    button_pane.setBorder(BorderFactory.createEtchedBorder());
    button_pane.add(clear_cache_button);
    button_pane.add(download_button);
    button_pane.add(information_button);
    button_pane.add(preferences_button);

    JPanel tree_pane = new JPanel();
        tree_pane.setComponentOrientation(Dictionary.getOrientation());
    tree_pane.setLayout(new BorderLayout());
        scrol_tmp = new JScrollPane(tree);
        scrol_tmp.setComponentOrientation(Dictionary.getOrientation());
        tree_pane.add(scrol_tmp, BorderLayout.CENTER);
    tree_pane.setPreferredSize(TREE_SIZE);
 

    Color colour_two = Configuration.getColor("coloring.collection_tree_background", false);
    options_pane.setBackground(colour_two);
    options_pane.setBorder(BorderFactory.createEtchedBorder());


    JScrollPane options_scroll_pane = new JScrollPane(options_pane);
        options_scroll_pane.setComponentOrientation(Dictionary.getOrientation());
    JSplitPane mode_pane = new JSplitPane();
        mode_pane.setComponentOrientation(Dictionary.getOrientation());
    mode_pane.setBorder(BorderFactory.createEmptyBorder(0,0,0,0));
        if (Dictionary.getOrientation().isLeftToRight()){
            mode_pane.add(tree_pane,JSplitPane.LEFT);
            mode_pane.add(options_scroll_pane,JSplitPane.RIGHT); 
            mode_pane.setDividerLocation(TREE_SIZE.width);
        }else{
            mode_pane.add(tree_pane,JSplitPane.RIGHT);
            mode_pane.add(options_scroll_pane,JSplitPane.LEFT);
            mode_pane.setDividerLocation(1-TREE_SIZE.width);
        }
        
    
    JPanel edit_pane = new JPanel();
        edit_pane.setComponentOrientation(Dictionary.getOrientation());
    edit_pane.setBorder(BorderFactory.createCompoundBorder(BorderFactory.createEmptyBorder(2,0,0,0), BorderFactory.createCompoundBorder(BorderFactory.createTitledBorder("Download Setting"), BorderFactory.createEmptyBorder(2,2,2,2))));
    edit_pane.setLayout(new BorderLayout());
    edit_pane.add(mode_pane,BorderLayout.CENTER);
    edit_pane.add(button_pane,BorderLayout.PAGE_END);
    
    // Add to "this"
    setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
    setLayout(new GridLayout(2,1));
    add(edit_pane);
    add(list_scroll);

    //set the mode to the first downloader in the list
    mode = convertCONTENTStoMode(CONTENTS[0]); // e.g. Web
    generateOptions(options_pane,(Download)download_map.get(mode));
    previous_path = tree.getSelectionPath();
    }

    /** System Utilities */
    public void modeChanged(int gli_mode) {
    // do nothing at this stage - should we be renewing download options??
    }

    private void addHeader(String name, Color color, JPanel target_pane) {
    JPanel header = new JPanel();
        header.setComponentOrientation(Dictionary.getOrientation());
    header.setBackground(color);
    JPanel inner_pane = new JPanel();
        inner_pane.setComponentOrientation(Dictionary.getOrientation());
    inner_pane.setBorder(BorderFactory.createCompoundBorder(BorderFactory.createEmptyBorder(5,5,5,5), BorderFactory.createRaisedBevelBorder()));
    inner_pane.setBackground(color);
    JLabel header_label = new JLabel("<html><strong>" + name + "</strong></html>");
        header_label.setComponentOrientation(Dictionary.getOrientation());
    header_label.setBackground(Configuration.getColor("coloring.collection_heading_background", false));
    header_label.setHorizontalAlignment(JLabel.CENTER);
    header_label.setOpaque(true);

    // Layout
    inner_pane.setLayout(new BorderLayout());
    inner_pane.add(header_label, BorderLayout.CENTER);

    header.setLayout(new BorderLayout());
    header.add(inner_pane, BorderLayout.CENTER);
    target_pane.add(header);
    }

    /** Supporting Functions */
    private ArrayList<String> loadDownloadersInfo(String lang) {
    Document document = null;
    
    try {
        if (Gatherer.isGsdlRemote) {
        String output = Gatherer.remoteGreenstoneServer.getScriptOptions("downloadinfo.pl", "&describeall");
        Reader reader = new StringReader(output);
        document = XMLTools.parseXML(reader);
        }
        else {
        ArrayList args_list = new ArrayList();
        String args[] = null;
        if(Configuration.perl_path != null) {
            args_list.add(Configuration.perl_path);
        } else if(Utility.isWindows()) {
            args_list.add("Perl.exe");
        } else {
            args_list.add("perl");
        }
        args_list.add("-S");
        args_list.add(LocalGreenstone.getBinScriptDirectoryPath()+"downloadinfo.pl");
        args_list.add("-describeall");
        args_list.add("-xml");
        args_list.add("-language");
        args_list.add(lang);        
        
        // Create the process.
        args = (String []) args_list.toArray(new String[0]);

        SafeProcess process = new SafeProcess(args);
        DebugStream.println("Getting Download Info: "+args_list);

        // run the SafeProcess
        int exitVal = process.runProcess();
        if(exitVal != 0) {
            throw new Exception("*** Error running Download Info process, process exited with: "
                    + exitVal);
        }       
        
        // get the result and process it.
        // This time we expect XML to have come out of the process std error stream.
        String errStreamOutput = process.getStdError();
        ///System.err.println("*********\nDownload Pane data, got:\n" + errStreamOutput + "\n**********\n");
        StringReader xmlStrReader = new StringReader(errStreamOutput);
        document = XMLTools.parseXML(xmlStrReader);
        xmlStrReader.close();
        
        }

            
    }
    catch (Exception error) {
        System.err.println("Failed when trying to parse downloadinfo.pl -describeall");
        error.printStackTrace();
    }

    if(document != null) {
        return parseXML(document.getDocumentElement());
    }

    return null;
    }

    private ArrayList<String> parseXML(Node root) {
    ArrayList<String> downloaders = null;
    Element downloadList = (Element)root;
    int length = -1;
    if(downloadList.hasAttribute("length")) {
        length = Integer.parseInt(downloadList.getAttribute("length"));
        downloaders = new ArrayList<String>(length);        
        
        for (Node node = downloadList.getFirstChild(); node != null; node = node.getNextSibling()) {
        // goes through each <DownloadInfo> of describeAll
        
        String download_name = null;
        
        for(Node infoNode = node.getFirstChild(); 
            infoNode != null; infoNode = infoNode.getNextSibling()) {
            
            String node_name = infoNode.getNodeName();
            if(node_name.equalsIgnoreCase("Name")) { // <Name>WebDownload</Name>
            download_name = XMLTools.getValue(infoNode); // e.g. WebDownload        
            }
            
            // At this top level of <DownloadInfo> elements,
            // skip all the downloaders that are Abstract, as these are pure superclasses       
            else if(node_name.equalsIgnoreCase("Abstract")) {
            String isAbstract = XMLTools.getValue(infoNode);
            
            if(isAbstract.equalsIgnoreCase("no") && download_name != null) {
                downloaders.add(download_name);
                Download downloader = parseDownloadInfoXML(node); // parse the <DownloadInfo> node properly
                // now embedded references to abstract superclasses (embedded <DownloadInfo> nodes)
                // will be handled
                
                String shortName = download_name.replace("Download", ""); // e.g. "Web"
                download_map.put(shortName, downloader);
            }
            }
        }
        }
    }

    return downloaders;
    }

    private Download parseDownloadInfoXML(Node root) {

    Download download = new Download();
    String node_name = null;
    for (Node node = root.getFirstChild(); node != null; node = node.getNextSibling()) {
        node_name = node.getNodeName();
        if(node_name.equalsIgnoreCase("Name")) {
        String name = XMLTools.getValue(node);
        download.setName(name);
        }
        else if (node_name.equalsIgnoreCase("Desc")) {
        download.setDescription(XMLTools.getValue(node));
        }
        else if (node_name.equalsIgnoreCase("Abstract")) {
        download.setIsAbstract(XMLTools.getValue(node).equalsIgnoreCase(StaticStrings.YES_STR));
        }
        else if(node_name.equalsIgnoreCase("Arguments")) {
        for(Node arg = node.getFirstChild(); arg != null; arg = arg.getNextSibling()) {
            node_name = arg.getNodeName();
                     if(node_name.equalsIgnoreCase("Option")) {
            Argument argument = new Argument((Element)arg);
            argument.parseXML((Element)arg);
            argument.setValue(argument.getDefaultValue());
            download.addArgument(argument);
             }
           
        }
        }
            else if(node_name.equalsIgnoreCase("DownloadInfo")) {
                Download super_download = parseDownloadInfoXML(node);
        download.setSuper(super_download);
        }
    }

    if(download.getName() != null) {
        return download;
    }
    return null;
    }

    /** Update the previous setup */
    private boolean updateArguments(boolean checkRequired)
    {
    boolean cont = true;
    for(int i = 0; i < options_pane.getComponentCount(); i++) {
       
        Component component = options_pane.getComponent(i);
        if(component instanceof ArgumentControl) {
        cont = cont && ((ArgumentControl)component).updateArgument(checkRequired);
        }
    }

    if(cont){return true; }

    return false;
    }
    
    /** Generate Controls for Options */
    /* at some stage we should think about which options should be shown for
     * different modes. Currently, always show all options (unless hidden)*/
    private void generateOptions(JPanel options_pane, ArgumentContainer data) {
    options_pane.removeAll();
    /** Create the current option panel */

    ArrayList arguments = data.getArguments(true, false);
        int mode = Configuration.getMode();
        ArrayList added_arguments = new ArrayList();
     
    for(int i = 0; i < arguments.size(); i++) {
        Argument argument = (Argument) arguments.get(i);

        if (argument.isHiddenGLI()) continue;
        ArgumentControl argument_control = new ArgumentControl(argument,false,null);
        added_arguments.add(argument_control);
    }
    

        options_pane.setLayout(new GridLayout(added_arguments.size(),1));
        for(int i = 0; i <  added_arguments.size(); i++) { 
        options_pane.add((ArgumentControl)added_arguments.get(i));
        
    }
    }

    /** Behaviour Functions */
    public void afterDisplay() {
    ready = true;        
    }


    public void gainFocus() {
    if(!ready) {
        return;
    }

    // It is also a good time to determine if the download should be enabled - ie if its allowed to be enabled and a valid URL is present in the field.
    download_button_enabled = true;
    //download_button.setEnabled(download_button_enabled);
    }



    public void refresh(int refresh_reason, boolean ready)
    {
    }

    /** Private classes */
    /** This tree provides a 'table of contents' for the various components of the design process (collection configuration in more technical terms). */
    private class DesignTree extends JTree {

    private DesignNode root = null;
    /** Constructor. Automatically generates all of the nodes, in the order of CONTENTS. */
    public DesignTree() {
        super();
            this.setComponentOrientation(Dictionary.getOrientation());
        resetModel(Configuration.getMode());
        expandRow(0);
        setRootVisible(false);
        setSelectionRow(0);
    }

    /** Reset the model used by the design page contents tree. This is necessary to hide the partitions entry when in lower detail modes
     * @param mode the current detail mode as an int
     */
    public void resetModel(int mode) {
        root = new DesignNode("DOWNLOAD.MODE.Root");
        // Now add the design categories.
        for(int i = 0; i < CONTENTS.length; i++) {
        root.add(new DesignNode(CONTENTS[i]));
        }
        this.setModel(new DefaultTreeModel(root));
        updateUI();
    }
    /** Set the current view to the one specified.
     * @param type the name of the desired view as a String
     */
    public void setSelectedView(String type) {
        type = Dictionary.get(type);
        for(int i = 0; i < root.getChildCount(); i++) {
        DesignNode child = (DesignNode) root.getChildAt(i);
        if(child.toString().equals(type)) {
            TreePath path = new TreePath(child.getPath());
            setSelectionPath(path);
        }
        }
    }
    }

    /** A tree node that retains a reference to one of the possible design sub-views relating to the different sub-managers. */
    private class DesignNode extends DefaultMutableTreeNode {
    /** Constructor.
     * @param object The <strong>Object</strong> assigned to this node.
     */
    public DesignNode(String object) {
        super(object);
    }
    /** Retrieve a textual representation of the object.
     * @return a String
     */
    public String toString() {
        // return Dictionary.get("CDM.GUI." + (String)getUserObject());
        return Dictionary.get((String) getUserObject());
    }
    }

    /** Listens for selection changes in the 'contents' tree, and switches to the appropriate view. */
    private class TreeListener
    implements TreeSelectionListener {
    /** Called whenever the selection changes, we must update the view so it matches the node selected.
     * @param event A <strong>TreeSelectionEvent</strong> containing more information about the tree selection.
     * @see org.greenstone.gatherer.cdm.ClassifierManager
     * @see org.greenstone.gatherer.cdm.CollectionDesignManager
     * @see org.greenstone.gatherer.cdm.CollectionMetaManager
     * @see org.greenstone.gatherer.cdm.FormatManager
     * @see org.greenstone.gatherer.cdm.LanguageManager
     * @see org.greenstone.gatherer.cdm.MetadataSetView
     * @see org.greenstone.gatherer.cdm.SubcollectionManager
     * @see org.greenstone.gatherer.cdm.TranslationView
     * @see org.greenstone.gatherer.cdm.PlugInManager
     */
    public void valueChanged(TreeSelectionEvent event) {
        if(!tree.isSelectionEmpty()) {
        TreePath path = tree.getSelectionPath();

        DesignNode node = (DesignNode)path.getLastPathComponent();
        String type = (String)node.getUserObject();
        Gatherer.g_man.wait(true);

        // type has the value DOWNLOAD.MODE.<downloader>Download,
        // mode should then be of the form <downloader>
        mode = convertCONTENTStoMode(type);
        generateOptions(options_pane,(Download)download_map.get(mode));

        tree.setSelectionPath(path);
        previous_path = path;
        repaint();

        Gatherer.g_man.wait(false);
        }
    }
    }

    private String convertCONTENTStoMode(String content) {
    return content.replace("DOWNLOAD.MODE.", "").replace("ZDownload", "Z3950").replace("Download", "");
    }

    private class ClearCacheListener
    implements ActionListener {
    public void actionPerformed(ActionEvent event) {
        // Retrieve the cache folder and delete it.
        Utility.delete(Utility.getCacheDir());
        // ...and refresh the node in the workspace tree to show it's all gone
        Gatherer.g_man.refreshWorkspaceTree(WorkspaceTree.DOWNLOADED_FILES_CHANGED);
    }
    }

    private class DownloadButtonListener
    implements ActionListener {
    public void actionPerformed(ActionEvent event) {

        if(checkURL(true) && checkProxy() == true) {
        
        // Proxy settings are now set. Check that the url is not a redirect, else get
        // redirect url (we do this step in order to avoid some unintuitive behaviour from wget)
        Download current_download = (Download)download_map.get(mode);
        
        boolean noCheckCertificate = Configuration.get("general.no_check_certificate", true);       
        Argument no_check_cert_arg = current_download.getArgument("no_check_certificate");
        if (no_check_cert_arg != null) { // e.g. z3950 doesn't use this
            if(noCheckCertificate) {
            no_check_cert_arg.setValue("true");
            no_check_cert_arg.setAssigned(true);
            } else {
            no_check_cert_arg.setValue("false");
            no_check_cert_arg.setAssigned(false); // only assigned Arguments have values
            }
        }
        Argument arg_url = current_download.getArgument("url");
        
        if(arg_url != null) { // it's null for z3950 and possibly for other downloaders
            String url_str = arg_url.getValue();    
            
            // No longer following URL redirects, since some of this has been taken care of by wget
            // For the rest, assume the user will get the URL right that they want to download from
            /*
              String redirect_url_str = getRedirectURL(url_str);
              
              // only update the Argument and its GUI ArgumentControl if the URL
              // has in fact changed
              if(!url_str.equals(redirect_url_str)) {
              arg_url.setValue(redirect_url_str);
              updateArgument(arg_url, redirect_url_str);
              }
            */
        }
        
        getter.newDownloadJob((Download)download_map.get(mode) ,mode,proxy_urls);
        }
    }
    }

    /**
     * The Java code here will retrieve the page at the given url. If the response code is 
     * a redirect, it will get the redirect url so that wget may be called with the proper url. 
     * This preprocessing of the URL is necessary because:
     * Wget does not behave the way the browser does when faced with urls of the form
     * http://www.englishhistory.net/tudor/citizens and if that page does not exist.
     * The directory listing with a slash at the end (http://www.englishhistory.net/tudor/citizens/)
     * does exist, however. In order to prevent wget from assuming that the root URL
     * to traverse is http://www.englishhistory.net/tudor/ instead of the intended
     * http://www.englishhistory.net/tudor/citizens/, we need give wget the redirect location
     * that's returned when we initially make a request for http://www.englishhistory.net/tudor/citizens
     * The proper url is sent back in the Location header, allowing us to bypass wget's 
     * unexpected behaviour.
     * This method ensures that urls like http://www.nzdl.org/niupepa also continue to work:
     * there is no http://www.nzdl.org/niupepa/ page, because this url actually redirects to an 
     * entirely different URL.
     * @return the redirect url for the given url if any redirection is involved, or the
     * url_str.
     *
     * Adding another useful URL on setting Java System Properties:
     * https://stackoverflow.com/questions/12181843/using-java-to-download-files-from-a-https-url
     */
    private String getRedirectURL(String url_str) {
    boolean noMoreRedirects = false;
    boolean gotException = false;
    final int TIMEOUT = 2 * 1000; // ms
    
    HttpURLConnection connection = null;
    if(url_str.startsWith("http:") || url_str.startsWith("https:")) { // only test http urls
        try {
        // URLConnectionManager class has special handling for https URLs,
        // so you can control whether you want it to check an HTTPS site's security certificates for you or not
        boolean noCheckCertificates = true;
        connection = (HttpURLConnection)URLConnectionManager.getConnection(url_str, this.proxyObject, noCheckCertificates);
        
        // don't let it automatically follow redirects, since we want to
        // find out whether we are dealing with redirects in the first place
        connection.setInstanceFollowRedirects(false);
        // Connection timeout: if we can't connect, like if the proxy is wrong, don't wait forever
        // Read timeout: *idle time* when retrieving a link. Don't wait forever to retrieve a page (e.g. if page doesn't exist)
        // https://stackoverflow.com/questions/6829801/httpurlconnection-setconnecttimeout-has-no-effect
        connection.setConnectTimeout(TIMEOUT); 
        connection.setReadTimeout(TIMEOUT); 
        
        // now check for whether we get a redirect response
        // HTTP Codes 3xx are redirects, http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html
        int responseCode = connection.getResponseCode();
        if(responseCode >= 300 && responseCode < 400) { 
            //String responseMsg = connection.getResponseMessage();

            // Get the Location header since this specifies the new location of the resource
            String location = connection.getHeaderField("Location");

            // this becomes the url that wget should download from
            url_str = location.trim();
        } else {
           noMoreRedirects = true; 
        }       
        connection.disconnect();
        } catch(Exception e) {
        gotException = true;
        if(connection != null) {
            connection.disconnect();
        }
        System.err.println("Checking redirection. Tried to connect to "
                   + url_str + ",\nbut got exception: " + e);
        }       
    } 

    if(noMoreRedirects || gotException) {
        return url_str;
    }
    else { // continue checking for whether the new URL redirects elsewhere again
        return getRedirectURL(url_str);
    }
    }

    
    /** For a string-based Argument whose value has changed, this method
     * updates the GUI ArgumentControl's value correspondingly. */
    private void updateArgument(Argument arg, String value) {
    for(int i = 0; i < options_pane.getComponentCount(); i++) {
        Component component = options_pane.getComponent(i);
        if(component instanceof ArgumentControl) {
        ArgumentControl control = (ArgumentControl)component;
        if(control.getArgument() == arg) {
            control.setValue(value);
            control.repaint();
        }
        }
    }
    }

    private boolean checkURL(boolean checkRequired){
    
    if (!updateArguments(checkRequired)){
        return false;
    }

    Download current_download = (Download)download_map.get(mode);
    Argument arg_url = current_download.getArgument("url");
    
        if (arg_url == null) return true;
 
    String url_str = arg_url.getValue();
    URL url = null;
    try {
        url = new URL(url_str);
    }
    catch(MalformedURLException error) {
        JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("Mirroring.Invalid_URL"), Dictionary.get("Mirroring.Invalid_URL_Title"), JOptionPane.ERROR_MESSAGE);
        return false;
    }

    return true;
    }

    
    // SOME READING:
    // https://superuser.com/questions/526710/how-to-set-http-proxy-address-for-wget-under-windows
    // https://arstechnica.com/civis/viewtopic.php?f=10&t=1281165
    // "http or https to an https proxy?"
    // About "CONNECT": https://daniel.haxx.se/docs/sshproxy.html
    // "You need an SSH client that can issue CONNECT requests through the company HTTP proxy." Seems to imply http? Also uses it for ftp.
    // https://forum.ivorde.com/set-up-ftp-proxy-via-command-line-in-linux-freebsd-t19733.html sets ftp_proxy to a http url.
    // So it's seems to be just whatever protocol the proxy server has. When the proxy supports all three protocols
    // (apparently the common case as per page below describing firefox prefs), then wiki.archlinux sets all three ftp_proxy/http_proxy/https_proxy to the same.
    // See https://forums.freebsd.org/threads/57378/
    // "I believe that ftp(1) only uses HTTP-type proxies for fetching URLs. I.e. you can't do traditional open->cd->get interactive style of FTP with it via a HTTP proxy. If you do something like ftp ftp://ftp.example.com/path/file, it should work with your proxy setup. For traditional interactive FTP, you need to be directly talking to the remote server or using the less common FTP proxy/gate functionality."    
    // https://wiki.archlinux.org/index.php/Proxy_settings
    // This does https_proxy = http_proxy, and explicitly prefixes "https://" to http_proxy.
    // https://www.howtogeek.com/293213/how-to-configure-a-proxy-server-in-firefox/
    /* You'll usually want to click the "Use the proxy server for all protocols" option. Firefox will also use your HTTP proxy server for SSL-encrypted HTTPS connections and File Transfer Protocol (FTP) connections.
Uncheck this box if you want to enter separate proxy servers for HTTP, HTTPS, and FTP connections. This isn't common.
If you're configuring a SOCKS proxy, leave the HTTP Proxy, SSL Proxy, and FTP Proxy boxes empty. Enter the address of the SOCKS proxy into the "SOCKS Host" and its port into the "Port" box.
    */    
    private boolean checkProxy(){    
    
    proxy_urls.clear();
    
    Download current_download = (Download)download_map.get(mode);
    
        Argument arg = current_download.getArgument("proxy_on");
  
    if (arg == null) return true;
    
    // Determine if we have to use a proxy.
    if(Configuration.get("general.use_proxy", true)) {
        
        boolean http_proxy_set = setProxyURLFor("HTTP");
        boolean https_proxy_set = setProxyURLFor("HTTPS");
        boolean ftp_proxy_set = setProxyURLFor("FTP");
        
        if(proxy_urls.size() == 0 ||
           (!http_proxy_set && !https_proxy_set && !ftp_proxy_set)) {
        
        // if checkProxy() failed for all protocols
        // OR if none of the proxies were setup by user, then turn off proxying
        arg = current_download.getArgument("proxy_on");
        arg.setValue("false"); 
        arg.setAssigned(false);
        proxy_urls.clear();
        return false;
        } else { // proxy details have been successfully set for at least one proxy protocol
        return true;
        }   

    } else { // if proxy_on was off
        // unset proxy_on argument too
        arg = current_download.getArgument("proxy_on");
        arg.setValue("false"); 
        arg.setAssigned(false);
        proxy_urls.clear();
    }
    
    return true;
    }

    private boolean setProxyURLFor(String protocol) {
    String proxy_host = Configuration.getString("general."+protocol+"_proxy_host", true);
    if(proxy_host.equals("")) { // no proxy details for this protocol
        ///System.err.println("### general."+protocol+"proxy_host was empty");
        return true;
    }
    
    String proxy_port = Configuration.getString("general."+protocol+"_proxy_port", true);
    // Find out whether the user has already authenticated themselves
    
    // remove the protocol prefix from proxy_host, and store it
    String proxy_protocol = "";
    int proxy_protocol_index = proxy_host.indexOf("://");
    if(proxy_protocol_index != -1) {
        proxy_protocol_index += "://".length();
        proxy_protocol = proxy_host.substring(0, proxy_protocol_index);
        proxy_host = proxy_host.substring(proxy_protocol_index);
    } else { // no explicit protocol for proxy host specified,
        // then set explicit protocol to be the same as the protocol param: http|https|ftp
        proxy_protocol = protocol.toLowerCase()+"://";
    }
    
    String user_pass = "";
    String address = proxy_host + ":" + proxy_port;
    
    int count = 0;
    // Only for wget, need to avoid a second automatic authentication popup (first asks
    // the proxy authentication for wget, and the second will ask the same for the realm)
    // Once the authentication has been reused, it will set the GAuthenticator state back to REGULAR
    GAuthenticator.setMode(GAuthenticator.DOWNLOAD);
    while(count < 3 && (user_pass = (String) GAuthenticator.authentications.get(address)) == null) {
        Authenticator.requestPasswordAuthentication(proxy_host, null, Integer.parseInt(proxy_port), "http://", Dictionary.get("WGet.Prompt"), "HTTP");
        count++;
    }
    if(count >= 3) {
        return false;
    }
    
    // https://askubuntu.com/questions/664777/systemwide-proxy-settings-in-ubuntu
    // http://www.rgagnon.com/javadetails/java-0085.html
    // how-do-i-make-httpurlconnection-use-a-proxy
    // https://stackoverflow.com/questions/8030908/how-to-check-if-proxy-is-working-in-java
    
    //proxyObject = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxy_host, Integer.parseInt(proxy_port))); // proxyObject only used by getRedirectURL(), which we're no longer calling
    
    Download current_download = (Download)download_map.get(mode);
    Argument arg = current_download.getArgument("proxy_on");
    
    if(user_pass.indexOf("@") != -1) {
        
        arg.setValue("true"); // proxy_on argument
        arg.setAssigned(true);
        
        arg = current_download.getArgument(protocol.toLowerCase()+"_proxy_host");
        arg.setValue(proxy_host);
        arg.setAssigned(true);
        
        arg = current_download.getArgument(protocol.toLowerCase()+"_proxy_port");
        arg.setValue(proxy_port);
        arg.setAssigned(true);

        // Write the use proxy command - we don't do this anymore, instead we set environment
        // variables as these do work for windows after all (Aug 2017). Hopefully these can't
        // be spied on like the following can (using ps/task manager. If going back to the
        // following, be aware: can't add protocol+user_name and protocol+user_password as
        // accepted parameters in WgetDownload.pm for HTTP, HTTPS and FTP protocols. Only
        // one set of username and password can be set for wget's --proxy-user and
        // --proxy-password flags. No separate flags for each protocol.
        /*if (Utility.isWindows()) {
          
          arg = current_download.getArgument("user_name");
          arg.setValue(user_pass.substring(0, user_pass.indexOf("@")));
          arg.setAssigned(true);
          
          arg = current_download.getArgument("user_password");
          arg.setValue(user_pass.substring(user_pass.indexOf("@") + 1));
          arg.setAssigned(true);
          }
          
          else{*/
        String user_name = user_pass.substring(0, user_pass.indexOf("@"));
        String user_pwd = user_pass.substring(user_pass.indexOf("@") + 1);
        /*}*/
        
        // construct proxy_url and prefix the stored proxy protocol to it
        String proxy_url = proxy_protocol+user_name+":"+user_pwd+"@"+proxy_host+":"+proxy_port+"/";     
        proxy_urls.setProperty(protocol, proxy_url);
        return true;
    }
    else{
        return false;
    }   
    }
    
    
    private boolean old_checkProxy(){
     
    String proxy_url = null; // ORIGINALLY A MEMBER VAR
    
    Download current_download = (Download)download_map.get(mode);
    
        Argument arg = current_download.getArgument("proxy_on");
  
    if (arg == null) return true;
    
    // Determine if we have to use a proxy.
    if(Configuration.get("general.use_proxy", true)) {
          
        String proxy_host = Configuration.getString("general.proxy_host", true);
        String proxy_port = Configuration.getString("general.proxy_port", true);
        // Find out whether the user has already authenticated themselves
        String user_pass = "";
        String address = proxy_host + ":" + proxy_port;
        
            int count = 0;
        // Only for wget, need to avoid a second automatic authentication popup (first asks
        // the proxy authentication for wget, and the second will ask the same for the realm)
        // Once the authentication has been reused, it will set the GAuthenticator state back to REGULAR
        GAuthenticator.setMode(GAuthenticator.DOWNLOAD);
        while(count < 3 && (user_pass = (String) GAuthenticator.authentications.get(address)) == null) {
        Authenticator.requestPasswordAuthentication(proxy_host, null, Integer.parseInt(proxy_port), "http://", Dictionary.get("WGet.Prompt"), "HTTP");
        count++;
        }
        if(count >= 3) {
        return false;
        }

        // https://askubuntu.com/questions/664777/systemwide-proxy-settings-in-ubuntu
        // http://www.rgagnon.com/javadetails/java-0085.html
        // how-do-i-make-httpurlconnection-use-a-proxy
        // https://stackoverflow.com/questions/8030908/how-to-check-if-proxy-is-working-in-java
        proxyObject = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxy_host, Integer.parseInt(proxy_port)));

        if(user_pass.indexOf("@") != -1) {
       
           arg.setValue("true"); // proxy_on argument
           arg.setAssigned(true);
           
           arg = current_download.getArgument("proxy_host");
           arg.setValue(proxy_host);
           arg.setAssigned(true);
           
           arg = current_download.getArgument("proxy_port");
           arg.setValue(proxy_port);
           arg.setAssigned(true);

        
        // Write the use proxy command - we don't do this anymore, instead we set environment variables - hopefully these can't be spied on like the following can (using ps) - actually the environment stuff didn't work for windows, so lets go back to this
        /*if (Utility.isWindows()) {
        
           arg = current_download.getArgument("user_name");
           arg.setValue(user_pass.substring(0, user_pass.indexOf("@")));
           arg.setAssigned(true);

           arg = current_download.getArgument("user_password");
           arg.setValue(user_pass.substring(user_pass.indexOf("@") + 1));
           arg.setAssigned(true);
          }
         
           else{*/
            String user_name = user_pass.substring(0, user_pass.indexOf("@"));
            String user_pwd = user_pass.substring(user_pass.indexOf("@") + 1);
            proxy_url = user_name+":"+user_pwd+"@"+proxy_host+":"+proxy_port+"/";
           
           /*}*/
        
           return true;
        }
        else{
        // unset proxy_on argument
        arg = current_download.getArgument("proxy_on");
        arg.setValue("false"); 
        arg.setAssigned(false);
        return false;
        }

    } else {
        // unset proxy_on argument
        arg = current_download.getArgument("proxy_on");
        arg.setValue("false"); 
        arg.setAssigned(false);
    }
    
    return true;
    }

    /*
    private class PreferencesButtonActionListener
    implements ActionListener {
    public void actionPerformed(ActionEvent event) {
    new Preferences(Preferences.CONNECTION_PREFS);
    }
    }*/
    
    private class InformationButtonActionListener
    implements ActionListener {
    public void actionPerformed(ActionEvent event) {
        //turn off the check for find argument
        Download current_download = (Download)download_map.get(mode);

            if (!checkProxy() || !checkURL(false) )return;

        if(server_info != null) {
        server_info.dispose();
        }
        
        Argument arg_url = current_download.getArgument("url");
        String str_url = "";
        if( arg_url!= null && arg_url.isAssigned()) {
        str_url = arg_url.getValue();       
        /*
        String redirected_url = getRedirectURL(str_url); // work out the real URL
        if(!str_url.equals(redirected_url)) {
            arg_url.setValue(redirected_url);
        }
        */
        }

        boolean noCheckCertificate = Configuration.get("general.no_check_certificate", true);
        Argument no_check_cert_arg = current_download.getArgument("no_check_certificate");
        if (no_check_cert_arg !=null) { //eg for z3950
        if(noCheckCertificate) {        
            no_check_cert_arg.setValue("true");
            no_check_cert_arg.setAssigned(true);
        } else {
            no_check_cert_arg.setValue("false");
            no_check_cert_arg.setAssigned(false); // only assigned Arguments have values
        }
        }

        server_info = new ServerInfoDialog(str_url ,proxy_urls, mode,(Download)download_map.get(mode));
        
    }
    }

     private class PreferencesButtonActionListener
    implements ActionListener {
    public void actionPerformed(ActionEvent event) {
        new Preferences(Preferences.CONNECTION_PREFS);
    }
    }
}