source: main/trunk/gli/src/org/greenstone/gatherer/gui/DownloadPane.java@ 31877

/**
 *#########################################################################
 *
 * A component of the Gatherer application, part of the Greenstone digital
 * library suite from the New Zealand Digital Library Project at the
 * University of Waikato, New Zealand.
 *
 * <BR><BR>
 *
 * Author: John Thompson, Greenstone Digital Library, University of Waikato
 *
 * <BR><BR>
 *
 * Copyright (C) 1999 New Zealand Digital Library Project
 *
 * <BR><BR>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * <BR><BR>
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * <BR><BR>
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 *########################################################################
 */
package org.greenstone.gatherer.gui;

import java.awt.*;
import java.awt.event.*;
import java.io.*;
import java.net.*;
import java.util.*;
import javax.swing.*;
import javax.swing.border.*;
import javax.swing.event.*;
import javax.swing.tree.*;
import org.greenstone.gatherer.Configuration;
import org.greenstone.gatherer.DebugStream;
import org.greenstone.gatherer.Dictionary;
import org.greenstone.gatherer.Gatherer;
import org.greenstone.gatherer.file.WorkspaceTree;
import org.greenstone.gatherer.greenstone.LocalGreenstone;
import org.greenstone.gatherer.util.SafeProcess;
import org.greenstone.gatherer.util.StaticStrings;
import org.greenstone.gatherer.util.Utility;
import org.greenstone.gatherer.download.URLConnectionManager;
import org.greenstone.gatherer.download.Download;
import org.greenstone.gatherer.download.DownloadScrollPane;
import org.greenstone.gatherer.download.ServerInfoDialog;
import org.greenstone.gatherer.util.XMLTools;
import org.greenstone.gatherer.cdm.*;
import org.greenstone.gatherer.gui.*;
import org.w3c.dom.*;
import org.xml.sax.*;
import org.greenstone.gatherer.GAuthenticator;

/**
 * @author John Thompson, Greenstone Digital Library, University of Waikato
 * @version 2.1
 */
public class DownloadPane
    extends JPanel {

    static final private Dimension LABEL_SIZE = new Dimension(225, 25);
    static final private Dimension TREE_SIZE = new Dimension(150, 500);
    //static final private String CONTENTS[] = { "DOWNLOAD.MODE.WebDownload", "DOWNLOAD.MODE.MediaWikiDownload", "DOWNLOAD.MODE.OAIDownload", "DOWNLOAD.MODE.ZDownload" , "DOWNLOAD.MODE.SRWDownload"};
    private String CONTENTS[] = null;

    private boolean download_button_enabled = false;
    private boolean ready = false;

    private JPanel options_pane;
    // TODO should use Vector to store all loaded downloads!!

    private DesignTree tree;
    private HashMap download_map;
    private ServerInfoDialog server_info;
    private JScrollPane list_scroll;
    private DownloadScrollPane getter;
    private String mode = null;
    private TreePath previous_path;
    private String proxy_url = "";
    private Proxy proxyObject = null;
        
    /** Main System code */
    public DownloadPane() {
    super();
        JScrollPane scrol_tmp;
        this.setComponentOrientation(Dictionary.getOrientation());
    // TODO: Download the WDownload and the download panel fixed!!
    getter = new DownloadScrollPane();
        getter.start();
    list_scroll = getter.getDownloadJobList();      
        list_scroll.setComponentOrientation(Dictionary.getOrientation());

    // TODO should use Vector to store all loaded downloads!!
    String lang = Configuration.getLanguage();
    download_map = new HashMap();

    // run downloadinfo.pl -describeall, load the downloaders into the download_map, 
    // and get back their list of names, which are of the form "<downloader>Download". 
    // Store these names in the CONTENTS[] array as "DOWNLOAD.MODE.<downloader>Download", 
    // with z3950 as a minor exception: DOWNLOAD.MODE.ZDownload.
    ArrayList<String> downloaderNamesList = loadDownloadersInfo(lang);  
    int size = downloaderNamesList.size();
    CONTENTS = new String[size];
    for(int i = 0; i < size; i++) {
        String downloadName = downloaderNamesList.get(i); // e.g. "WebDownload"     
        CONTENTS[i] = "DOWNLOAD.MODE."+downloadName.replace("3950", ""); // A special case is Z3950Download,
                                     // which has to be stored in CONTENTS array as DOWNLOAD.MODE.ZDownload
    }

    // Creation
    tree = new DesignTree();
        tree.setComponentOrientation(Dictionary.getOrientation());
    options_pane = new JPanel();
        options_pane.setComponentOrientation(Dictionary.getOrientation());
    
    JButton clear_cache_button = new GLIButton(Dictionary.get("Mirroring.ClearCache"), Dictionary.get("Mirroring.ClearCache_Tooltip"));
    clear_cache_button.setEnabled(true);
    clear_cache_button.setMnemonic(KeyEvent.VK_C);

    JButton download_button = new GLIButton(Dictionary.get("Mirroring.Download"), Dictionary.get("Mirroring.Download_Tooltip"));
    download_button.setEnabled(true);
    download_button.setMnemonic(KeyEvent.VK_D);

    JButton information_button =  new GLIButton(Dictionary.get("Download.ServerInformation"), Dictionary.get("Download.ServerInformation_Tooltip"));
    information_button.setEnabled(true);
    information_button.setMnemonic(KeyEvent.VK_S);


        JButton preferences_button =  new GLIButton(Dictionary.get("Mirroring.Preferences"), Dictionary.get("Mirroring.Preferences_Tooltip"));
    preferences_button.setEnabled(true);
    preferences_button.setMnemonic(KeyEvent.VK_P);

    // Connect
    clear_cache_button.addActionListener(new ClearCacheListener());
    download_button.addActionListener(new DownloadButtonListener());
    preferences_button.addActionListener(new PreferencesButtonActionListener());
    information_button.addActionListener(new InformationButtonActionListener());
    tree.addTreeSelectionListener(new TreeListener());

    // Add to Panel
    JPanel button_pane = new JPanel();
        button_pane.setComponentOrientation(Dictionary.getOrientation());
    button_pane.setLayout(new GridLayout(1,4)); // GridLayout so button pane resizes with window-width
    button_pane.setBorder(BorderFactory.createEtchedBorder());
    button_pane.add(clear_cache_button);
    button_pane.add(download_button);
    button_pane.add(information_button);
    button_pane.add(preferences_button);

    JPanel tree_pane = new JPanel();
        tree_pane.setComponentOrientation(Dictionary.getOrientation());
    tree_pane.setLayout(new BorderLayout());
        scrol_tmp = new JScrollPane(tree);
        scrol_tmp.setComponentOrientation(Dictionary.getOrientation());
        tree_pane.add(scrol_tmp, BorderLayout.CENTER);
    tree_pane.setPreferredSize(TREE_SIZE);
 

    Color colour_two = Configuration.getColor("coloring.collection_tree_background", false);
    options_pane.setBackground(colour_two);
    options_pane.setBorder(BorderFactory.createEtchedBorder());


    JScrollPane options_scroll_pane = new JScrollPane(options_pane);
        options_scroll_pane.setComponentOrientation(Dictionary.getOrientation());
    JSplitPane mode_pane = new JSplitPane();
        mode_pane.setComponentOrientation(Dictionary.getOrientation());
    mode_pane.setBorder(BorderFactory.createEmptyBorder(0,0,0,0));
        if (Dictionary.getOrientation().isLeftToRight()){
            mode_pane.add(tree_pane,JSplitPane.LEFT);
            mode_pane.add(options_scroll_pane,JSplitPane.RIGHT); 
            mode_pane.setDividerLocation(TREE_SIZE.width);
        }else{
            mode_pane.add(tree_pane,JSplitPane.RIGHT);
            mode_pane.add(options_scroll_pane,JSplitPane.LEFT);
            mode_pane.setDividerLocation(1-TREE_SIZE.width);
        }
        
    
    JPanel edit_pane = new JPanel();
        edit_pane.setComponentOrientation(Dictionary.getOrientation());
    edit_pane.setBorder(BorderFactory.createCompoundBorder(BorderFactory.createEmptyBorder(2,0,0,0), BorderFactory.createCompoundBorder(BorderFactory.createTitledBorder("Download Setting"), BorderFactory.createEmptyBorder(2,2,2,2))));
    edit_pane.setLayout(new BorderLayout());
    edit_pane.add(mode_pane,BorderLayout.CENTER);
    edit_pane.add(button_pane,BorderLayout.PAGE_END);
    
    // Add to "this"
    setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
    setLayout(new GridLayout(2,1));
    add(edit_pane);
    add(list_scroll);

    //set the mode to the first downloader in the list
    mode = convertCONTENTStoMode(CONTENTS[0]); // e.g. Web
    generateOptions(options_pane,(Download)download_map.get(mode));
    previous_path = tree.getSelectionPath();
    }

    /** System Utilities */
    public void modeChanged(int gli_mode) {
    // do nothing at this stage - should we be renewing download options??
    }

    private void addHeader(String name, Color color, JPanel target_pane) {
    JPanel header = new JPanel();
        header.setComponentOrientation(Dictionary.getOrientation());
    header.setBackground(color);
    JPanel inner_pane = new JPanel();
        inner_pane.setComponentOrientation(Dictionary.getOrientation());
    inner_pane.setBorder(BorderFactory.createCompoundBorder(BorderFactory.createEmptyBorder(5,5,5,5), BorderFactory.createRaisedBevelBorder()));
    inner_pane.setBackground(color);
    JLabel header_label = new JLabel("<html><strong>" + name + "</strong></html>");
        header_label.setComponentOrientation(Dictionary.getOrientation());
    header_label.setBackground(Configuration.getColor("coloring.collection_heading_background", false));
    header_label.setHorizontalAlignment(JLabel.CENTER);
    header_label.setOpaque(true);

    // Layout
    inner_pane.setLayout(new BorderLayout());
    inner_pane.add(header_label, BorderLayout.CENTER);

    header.setLayout(new BorderLayout());
    header.add(inner_pane, BorderLayout.CENTER);
    target_pane.add(header);
    }

    /** Supporting Functions */
    private ArrayList<String> loadDownloadersInfo(String lang) {
    Document document = null;
    
    try {
        if (Gatherer.isGsdlRemote) {
        String output = Gatherer.remoteGreenstoneServer.getScriptOptions("downloadinfo.pl", "&describeall");
        Reader reader = new StringReader(output);
        document = XMLTools.parseXML(reader);
        }
        else {
        ArrayList args_list = new ArrayList();
        String args[] = null;
        if(Configuration.perl_path != null) {
            args_list.add(Configuration.perl_path);
        } else if(Utility.isWindows()) {
            args_list.add("Perl.exe");
        } else {
            args_list.add("perl");
        }
        args_list.add("-S");
        args_list.add(LocalGreenstone.getBinScriptDirectoryPath()+"downloadinfo.pl");
        args_list.add("-describeall");
        args_list.add("-xml");
        args_list.add("-language");
        args_list.add(lang);        
        
        // Create the process.
        args = (String []) args_list.toArray(new String[0]);

        SafeProcess process = new SafeProcess(args);
        DebugStream.println("Getting Download Info: "+args_list);

        // run the SafeProcess
        int exitVal = process.runProcess();
        if(exitVal != 0) {
            throw new Exception("*** Error running Download Info process, process exited with: "
                    + exitVal);
        }       
        
        // get the result and process it.
        // This time we expect XML to have come out of the process std error stream.
        String errStreamOutput = process.getStdError();
        ///System.err.println("*********\nDownload Pane data, got:\n" + errStreamOutput + "\n**********\n");
        StringReader xmlStrReader = new StringReader(errStreamOutput);
        document = XMLTools.parseXML(xmlStrReader);
        xmlStrReader.close();
        
        }

            
    }
    catch (Exception error) {
        System.err.println("Failed when trying to parse downloadinfo.pl -describeall");
        error.printStackTrace();
    }

    if(document != null) {
        return parseXML(document.getDocumentElement());
    }

    return null;
    }

    private ArrayList<String> parseXML(Node root) {
    ArrayList<String> downloaders = null;
    Element downloadList = (Element)root;
    int length = -1;
    if(downloadList.hasAttribute("length")) {
        length = Integer.parseInt(downloadList.getAttribute("length"));
        downloaders = new ArrayList<String>(length);        
        
        for (Node node = downloadList.getFirstChild(); node != null; node = node.getNextSibling()) {
        // goes through each <DownloadInfo> of describeAll
        
        String download_name = null;
        
        for(Node infoNode = node.getFirstChild(); 
            infoNode != null; infoNode = infoNode.getNextSibling()) {
            
            String node_name = infoNode.getNodeName();
            if(node_name.equalsIgnoreCase("Name")) { // <Name>WebDownload</Name>
            download_name = XMLTools.getValue(infoNode); // e.g. WebDownload        
            }
            
            // At this top level of <DownloadInfo> elements,
            // skip all the downloaders that are Abstract, as these are pure superclasses       
            else if(node_name.equalsIgnoreCase("Abstract")) {
            String isAbstract = XMLTools.getValue(infoNode);
            
            if(isAbstract.equalsIgnoreCase("no") && download_name != null) {
                downloaders.add(download_name);
                Download downloader = parseDownloadInfoXML(node); // parse the <DownloadInfo> node properly
                // now embedded references to abstract superclasses (embedded <DownloadInfo> nodes)
                // will be handled
                
                String shortName = download_name.replace("Download", ""); // e.g. "Web"
                download_map.put(shortName, downloader);
            }
            }
        }
        }
    }

    return downloaders;
    }

    private Download parseDownloadInfoXML(Node root) {

    Download download = new Download();
    String node_name = null;
    for (Node node = root.getFirstChild(); node != null; node = node.getNextSibling()) {
        node_name = node.getNodeName();
        if(node_name.equalsIgnoreCase("Name")) {
        String name = XMLTools.getValue(node);
        download.setName(name);
        }
        else if (node_name.equalsIgnoreCase("Desc")) {
        download.setDescription(XMLTools.getValue(node));
        }
        else if (node_name.equalsIgnoreCase("Abstract")) {
        download.setIsAbstract(XMLTools.getValue(node).equalsIgnoreCase(StaticStrings.YES_STR));
        }
        else if(node_name.equalsIgnoreCase("Arguments")) {
        for(Node arg = node.getFirstChild(); arg != null; arg = arg.getNextSibling()) {
            node_name = arg.getNodeName();
                     if(node_name.equalsIgnoreCase("Option")) {
            Argument argument = new Argument((Element)arg);
            argument.parseXML((Element)arg);
            argument.setValue(argument.getDefaultValue());
            download.addArgument(argument);
             }
           
        }
        }
            else if(node_name.equalsIgnoreCase("DownloadInfo")) {
                Download super_download = parseDownloadInfoXML(node);
        download.setSuper(super_download);
        }
    }

    if(download.getName() != null) {
        return download;
    }
    return null;
    }

    /** Update the previous setup */
    private boolean updateArguments(boolean checkRequired)
    {
    boolean cont = true;
    for(int i = 0; i < options_pane.getComponentCount(); i++) {
       
        Component component = options_pane.getComponent(i);
        if(component instanceof ArgumentControl) {
        cont = cont && ((ArgumentControl)component).updateArgument(checkRequired);
        }
    }

    if(cont){return true; }

    return false;
    }
    
    /** Generate Controls for Options */
    /* at some stage we should think about which options should be shown for
     * different modes. Currently, always show all options (unless hidden)*/
    private void generateOptions(JPanel options_pane, ArgumentContainer data) {
    options_pane.removeAll();
    /** Create the current option panel */

    ArrayList arguments = data.getArguments(true, false);
        int mode = Configuration.getMode();
        ArrayList added_arguments = new ArrayList();
     
    for(int i = 0; i < arguments.size(); i++) {
        Argument argument = (Argument) arguments.get(i);

        if (argument.isHiddenGLI()) continue;
        ArgumentControl argument_control = new ArgumentControl(argument,false,null);
        added_arguments.add(argument_control);
    }
    

        options_pane.setLayout(new GridLayout(added_arguments.size(),1));
        for(int i = 0; i <  added_arguments.size(); i++) { 
        options_pane.add((ArgumentControl)added_arguments.get(i));
        
    }
    }

    /** Behaviour Functions */
    public void afterDisplay() {
    ready = true;        
    }


    public void gainFocus() {
    if(!ready) {
        return;
    }

    // It is also a good time to determine if the download should be enabled - ie if its allowed to be enabled and a valid URL is present in the field.
    download_button_enabled = true;
    //download_button.setEnabled(download_button_enabled);
    }



    public void refresh(int refresh_reason, boolean ready)
    {
    }

    /** Private classes */
    /** This tree provides a 'table of contents' for the various components of the design process (collection configuration in more technical terms). */
    private class DesignTree extends JTree {

    private DesignNode root = null;
    /** Constructor. Automatically generates all of the nodes, in the order of CONTENTS. */
    public DesignTree() {
        super();
            this.setComponentOrientation(Dictionary.getOrientation());
        resetModel(Configuration.getMode());
        expandRow(0);
        setRootVisible(false);
        setSelectionRow(0);
    }

    /** Reset the model used by the design page contents tree. This is necessary to hide the partitions entry when in lower detail modes
     * @param mode the current detail mode as an int
     */
    public void resetModel(int mode) {
        root = new DesignNode("DOWNLOAD.MODE.Root");
        // Now add the design categories.
        for(int i = 0; i < CONTENTS.length; i++) {
        root.add(new DesignNode(CONTENTS[i]));
        }
        this.setModel(new DefaultTreeModel(root));
        updateUI();
    }
    /** Set the current view to the one specified.
     * @param type the name of the desired view as a String
     */
    public void setSelectedView(String type) {
        type = Dictionary.get(type);
        for(int i = 0; i < root.getChildCount(); i++) {
        DesignNode child = (DesignNode) root.getChildAt(i);
        if(child.toString().equals(type)) {
            TreePath path = new TreePath(child.getPath());
            setSelectionPath(path);
        }
        }
    }
    }

    /** A tree node that retains a reference to one of the possible design sub-views relating to the different sub-managers. */
    private class DesignNode extends DefaultMutableTreeNode {
    /** Constructor.
     * @param object The <strong>Object</strong> assigned to this node.
     */
    public DesignNode(String object) {
        super(object);
    }
    /** Retrieve a textual representation of the object.
     * @return a String
     */
    public String toString() {
        // return Dictionary.get("CDM.GUI." + (String)getUserObject());
        return Dictionary.get((String) getUserObject());
    }
    }

    /** Listens for selection changes in the 'contents' tree, and switches to the appropriate view. */
    private class TreeListener
    implements TreeSelectionListener {
    /** Called whenever the selection changes, we must update the view so it matches the node selected.
     * @param event A <strong>TreeSelectionEvent</strong> containing more information about the tree selection.
     * @see org.greenstone.gatherer.cdm.ClassifierManager
     * @see org.greenstone.gatherer.cdm.CollectionDesignManager
     * @see org.greenstone.gatherer.cdm.CollectionMetaManager
     * @see org.greenstone.gatherer.cdm.FormatManager
     * @see org.greenstone.gatherer.cdm.LanguageManager
     * @see org.greenstone.gatherer.cdm.MetadataSetView
     * @see org.greenstone.gatherer.cdm.SubcollectionManager
     * @see org.greenstone.gatherer.cdm.TranslationView
     * @see org.greenstone.gatherer.cdm.PlugInManager
     */
    public void valueChanged(TreeSelectionEvent event) {
        if(!tree.isSelectionEmpty()) {
        TreePath path = tree.getSelectionPath();

        DesignNode node = (DesignNode)path.getLastPathComponent();
        String type = (String)node.getUserObject();
        Gatherer.g_man.wait(true);

        // type has the value DOWNLOAD.MODE.<downloader>Download,
        // mode should then be of the form <downloader>
        mode = convertCONTENTStoMode(type);
        generateOptions(options_pane,(Download)download_map.get(mode));

        tree.setSelectionPath(path);
        previous_path = path;
        repaint();

        Gatherer.g_man.wait(false);
        }
    }
    }

    private String convertCONTENTStoMode(String content) {
    return content.replace("DOWNLOAD.MODE.", "").replace("ZDownload", "Z3950").replace("Download", "");
    }

    private class ClearCacheListener
    implements ActionListener {
    public void actionPerformed(ActionEvent event) {
        // Retrieve the cache folder and delete it.
        Utility.delete(Utility.getCacheDir());
        // ...and refresh the node in the workspace tree to show it's all gone
        Gatherer.g_man.refreshWorkspaceTree(WorkspaceTree.DOWNLOADED_FILES_CHANGED);
    }
    }

    private class DownloadButtonListener
    implements ActionListener {
    public void actionPerformed(ActionEvent event) {

        if(checkURL(true) && checkProxy() == true) {
        
        // Proxy settings are now set. Check that the url is not a redirect, else get
        // redirect url (we do this step in order to avoid some unintuitive behaviour from wget)
        Download current_download = (Download)download_map.get(mode);
        
        boolean noCheckCertificate = Configuration.get("general.no_check_certificate", true);       
        Argument no_check_cert_arg = current_download.getArgument("no_check_certificate");
        if(noCheckCertificate) {
            no_check_cert_arg.setValue("true");
            no_check_cert_arg.setAssigned(true);
        } else {
            no_check_cert_arg.setValue("false");
            no_check_cert_arg.setAssigned(false); // only assigned Arguments have values
        }

        Argument arg_url = current_download.getArgument("url");
        
        if(arg_url != null) { // it's null for z3950 and possibly for other downloaders
            String url_str = arg_url.getValue();    
            
            // No longer following URL redirects, since some of this has been taken care of by wget
            // For the rest, assume the user will get the URL right that they want to download from
            /*
              String redirect_url_str = getRedirectURL(url_str);
              
              // only update the Argument and its GUI ArgumentControl if the URL
              // has in fact changed
              if(!url_str.equals(redirect_url_str)) {
              arg_url.setValue(redirect_url_str);
              updateArgument(arg_url, redirect_url_str);
              }
            */
        }
        
        getter.newDownloadJob((Download)download_map.get(mode) ,mode,proxy_url);
        }
    }
    }

    /**
     * The Java code here will retrieve the page at the given url. If the response code is 
     * a redirect, it will get the redirect url so that wget may be called with the proper url. 
     * This preprocessing of the URL is necessary because:
     * Wget does not behave the way the browser does when faced with urls of the form
     * http://www.englishhistory.net/tudor/citizens and if that page does not exist.
     * The directory listing with a slash at the end (http://www.englishhistory.net/tudor/citizens/)
     * does exist, however. In order to prevent wget from assuming that the root URL
     * to traverse is http://www.englishhistory.net/tudor/ instead of the intended
     * http://www.englishhistory.net/tudor/citizens/, we need give wget the redirect location
     * that's returned when we initially make a request for http://www.englishhistory.net/tudor/citizens
     * The proper url is sent back in the Location header, allowing us to bypass wget's 
     * unexpected behaviour.
     * This method ensures that urls like http://www.nzdl.org/niupepa also continue to work:
     * there is no http://www.nzdl.org/niupepa/ page, because this url actually redirects to an 
     * entirely different URL.
     * @return the redirect url for the given url if any redirection is involved, or the
     * url_str.
     *
     * Adding another useful URL on setting Java System Properties:
     * https://stackoverflow.com/questions/12181843/using-java-to-download-files-from-a-https-url
     */
    private String getRedirectURL(String url_str) {
    boolean noMoreRedirects = false;
    boolean gotException = false;
    final int TIMEOUT = 2 * 1000; // ms
    
    HttpURLConnection connection = null;
    if(url_str.startsWith("http:") || url_str.startsWith("https:")) { // only test http urls
        try {
        // URLConnectionManager class has special handling for https URLs,
        // so you can control whether you want it to check an HTTPS site's security certificates for you or not
        boolean noCheckCertificates = true;
        connection = (HttpURLConnection)URLConnectionManager.getConnection(url_str, this.proxyObject, noCheckCertificates);
        
        // don't let it automatically follow redirects, since we want to
        // find out whether we are dealing with redirects in the first place
        connection.setInstanceFollowRedirects(false);
        // Connection timeout: if we can't connect, like if the proxy is wrong, don't wait forever
        // Read timeout: *idle time* when retrieving a link. Don't wait forever to retrieve a page (e.g. if page doesn't exist)
        // https://stackoverflow.com/questions/6829801/httpurlconnection-setconnecttimeout-has-no-effect
        connection.setConnectTimeout(TIMEOUT); 
        connection.setReadTimeout(TIMEOUT); 
        
        // now check for whether we get a redirect response
        // HTTP Codes 3xx are redirects, http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html
        int responseCode = connection.getResponseCode();
        if(responseCode >= 300 && responseCode < 400) { 
            //String responseMsg = connection.getResponseMessage();

            // Get the Location header since this specifies the new location of the resource
            String location = connection.getHeaderField("Location");

            // this becomes the url that wget should download from
            url_str = location.trim();
        } else {
           noMoreRedirects = true; 
        }       
        connection.disconnect();
        } catch(Exception e) {
        gotException = true;
        if(connection != null) {
            connection.disconnect();
        }
        System.err.println("Checking redirection. Tried to connect to "
                   + url_str + ",\nbut got exception: " + e);
        }       
    } 

    if(noMoreRedirects || gotException) {
        return url_str;
    }
    else { // continue checking for whether the new URL redirects elsewhere again
        return getRedirectURL(url_str);
    }
    }

    
    /** For a string-based Argument whose value has changed, this method
     * updates the GUI ArgumentControl's value correspondingly. */
    private void updateArgument(Argument arg, String value) {
    for(int i = 0; i < options_pane.getComponentCount(); i++) {
        Component component = options_pane.getComponent(i);
        if(component instanceof ArgumentControl) {
        ArgumentControl control = (ArgumentControl)component;
        if(control.getArgument() == arg) {
            control.setValue(value);
            control.repaint();
        }
        }
    }
    }

    private boolean checkURL(boolean checkRequired){
    
    if (!updateArguments(checkRequired)){
        return false;
    }

    Download current_download = (Download)download_map.get(mode);
    Argument arg_url = current_download.getArgument("url");
    
        if (arg_url == null) return true;
 
    String url_str = arg_url.getValue();
    URL url = null;
    try {
        url = new URL(url_str);
    }
    catch(MalformedURLException error) {
        JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("Mirroring.Invalid_URL"), Dictionary.get("Mirroring.Invalid_URL_Title"), JOptionPane.ERROR_MESSAGE);
        return false;
    }

    return true;
    }

    // TODO: Still need to read up on and test how to set wget proxy on windows
    // Need to be on a windows machine that requires proxy, and get wget to work on cmdline
    // If that works, then need to check https URLS also work.
    // See https://superuser.com/questions/526710/how-to-set-http-proxy-address-for-wget-under-windows
    private boolean checkProxy(){
     
    proxy_url = null;

    Download current_download = (Download)download_map.get(mode);
    
        Argument arg = current_download.getArgument("proxy_on");
  
    if (arg == null) return true;
    
    // Determine if we have to use a proxy.
    if(Configuration.get("general.use_proxy", true)) {
          
        String proxy_host = Configuration.getString("general.proxy_host", true);
        String proxy_port = Configuration.getString("general.proxy_port", true);
        // Find out whether the user has already authenticated themselves
        String user_pass = "";
        String address = proxy_host + ":" + proxy_port;
        
            int count = 0;
        // Only for wget, need to avoid a second automatic authentication popup (first asks
        // the proxy authentication for wget, and the second will ask the same for the realm)
        // Once the authentication has been reused, it will set the GAuthenticator state back to REGULAR
        GAuthenticator.setMode(GAuthenticator.DOWNLOAD);
        while(count < 3 && (user_pass = (String) GAuthenticator.authentications.get(address)) == null) {
        Authenticator.requestPasswordAuthentication(proxy_host, null, Integer.parseInt(proxy_port), "http://", Dictionary.get("WGet.Prompt"), "HTTP");
        count++;
        }
        if(count >= 3) {
        return false;
        }

        // https://askubuntu.com/questions/664777/systemwide-proxy-settings-in-ubuntu
        // http://www.rgagnon.com/javadetails/java-0085.html
        // how-do-i-make-httpurlconnection-use-a-proxy
        // https://stackoverflow.com/questions/8030908/how-to-check-if-proxy-is-working-in-java
        proxyObject = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxy_host, Integer.parseInt(proxy_port)));

        if(user_pass.indexOf("@") != -1) {
       
           arg.setValue("true"); // proxy_on argument
           arg.setAssigned(true);
           
           arg = current_download.getArgument("proxy_host");
           arg.setValue(proxy_host);
           arg.setAssigned(true);
           
           arg = current_download.getArgument("proxy_port");
           arg.setValue(proxy_port);
           arg.setAssigned(true);

        
        // Write the use proxy command - we don't do this anymore, instead we set environment variables - hopefully these can't be spied on like the following can (using ps) - actually the environment stuff didn't work for windows, so lets go back to this
        /*if (Utility.isWindows()) {
        
           arg = current_download.getArgument("user_name");
           arg.setValue(user_pass.substring(0, user_pass.indexOf("@")));
           arg.setAssigned(true);

           arg = current_download.getArgument("user_password");
           arg.setValue(user_pass.substring(user_pass.indexOf("@") + 1));
           arg.setAssigned(true);
          }
         
           else{*/
            String user_name = user_pass.substring(0, user_pass.indexOf("@"));
            String user_pwd = user_pass.substring(user_pass.indexOf("@") + 1);
            proxy_url = user_name+":"+user_pwd+"@"+proxy_host+":"+proxy_port+"/";
           
           /*}*/
        
           return true;
        }
        else{
        // unset proxy_on argument
        arg = current_download.getArgument("proxy_on");
        arg.setValue("false"); 
        arg.setAssigned(false);
        return false;
        }

    } else {
        // unset proxy_on argument
        arg = current_download.getArgument("proxy_on");
        arg.setValue("false"); 
        arg.setAssigned(false);
    }
    
    return true;
    }

    /*
    private class PreferencesButtonActionListener
    implements ActionListener {
    public void actionPerformed(ActionEvent event) {
    new Preferences(Preferences.CONNECTION_PREFS);
    }
    }*/
    
    private class InformationButtonActionListener
    implements ActionListener {
    public void actionPerformed(ActionEvent event) {
        //turn off the check for find argument
        Download current_download = (Download)download_map.get(mode);

            if (!checkProxy() || !checkURL(false) )return;

        if(server_info != null) {
        server_info.dispose();
        }
        
        Argument arg_url = current_download.getArgument("url");
        String str_url = "";
        if( arg_url!= null && arg_url.isAssigned()) {
        str_url = arg_url.getValue();       
        /*
        String redirected_url = getRedirectURL(str_url); // work out the real URL
        if(!str_url.equals(redirected_url)) {
            arg_url.setValue(redirected_url);
        }
        */
        }

        boolean noCheckCertificate = Configuration.get("general.no_check_certificate", true);
        Argument no_check_cert_arg = current_download.getArgument("no_check_certificate");
        if(noCheckCertificate) {        
        no_check_cert_arg.setValue("true");
        no_check_cert_arg.setAssigned(true);
        } else {
        no_check_cert_arg.setValue("false");
        no_check_cert_arg.setAssigned(false); // only assigned Arguments have values
        }

        server_info = new ServerInfoDialog(str_url ,proxy_url, mode,(Download)download_map.get(mode));
        
    }
    }

     private class PreferencesButtonActionListener
    implements ActionListener {
    public void actionPerformed(ActionEvent event) {
        new Preferences(Preferences.CONNECTION_PREFS);
    }
    }
}