source: main/trunk/gli/src/org/greenstone/gatherer/gui/DownloadPane.java@ 31860

Last change on this file since 31860 was 31860, checked in by ak19, 7 years ago

The changes necessary for getting the new no_check_certificate checkbox to appear and work in GLI and get propagated to the perl code that launches wget. This checkbox controls whether wget is launched with the no-check-certificate flag to retrieve Https URLs despite lack of (valid) certificates.
  • Property svn:keywords set to Author Date Id Revision
File size: 31.7 KB
Line 
1/**
2 *#########################################################################
3 *
4 * A component of the Gatherer application, part of the Greenstone digital
5 * library suite from the New Zealand Digital Library Project at the
6 * University of Waikato, New Zealand.
7 *
8 * <BR><BR>
9 *
10 * Author: John Thompson, Greenstone Digital Library, University of Waikato
11 *
12 * <BR><BR>
13 *
14 * Copyright (C) 1999 New Zealand Digital Library Project
15 *
16 * <BR><BR>
17 *
18 * This program is free software; you can redistribute it and/or modify
19 * it under the terms of the GNU General Public License as published by
20 * the Free Software Foundation; either version 2 of the License, or
21 * (at your option) any later version.
22 *
23 * <BR><BR>
24 *
25 * This program is distributed in the hope that it will be useful,
26 * but WITHOUT ANY WARRANTY; without even the implied warranty of
27 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
28 * GNU General Public License for more details.
29 *
30 * <BR><BR>
31 *
32 * You should have received a copy of the GNU General Public License
33 * along with this program; if not, write to the Free Software
34 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
35 *########################################################################
36 */
37package org.greenstone.gatherer.gui;
38
39import java.awt.*;
40import java.awt.event.*;
41import java.io.*;
42import java.net.*;
43import java.util.*;
44import javax.swing.*;
45import javax.swing.border.*;
46import javax.swing.event.*;
47import javax.swing.tree.*;
48import org.greenstone.gatherer.Configuration;
49import org.greenstone.gatherer.DebugStream;
50import org.greenstone.gatherer.Dictionary;
51import org.greenstone.gatherer.Gatherer;
52import org.greenstone.gatherer.file.WorkspaceTree;
53import org.greenstone.gatherer.greenstone.LocalGreenstone;
54import org.greenstone.gatherer.util.SafeProcess;
55import org.greenstone.gatherer.util.StaticStrings;
56import org.greenstone.gatherer.util.Utility;
57import org.greenstone.gatherer.download.URLConnectionManager;
58import org.greenstone.gatherer.download.Download;
59import org.greenstone.gatherer.download.DownloadScrollPane;
60import org.greenstone.gatherer.download.ServerInfoDialog;
61import org.greenstone.gatherer.util.XMLTools;
62import org.greenstone.gatherer.cdm.*;
63import org.greenstone.gatherer.gui.*;
64import org.w3c.dom.*;
65import org.xml.sax.*;
66import org.greenstone.gatherer.GAuthenticator;
67
68/**
69 * @author John Thompson, Greenstone Digital Library, University of Waikato
70 * @version 2.1
71 */
72public class DownloadPane
73 extends JPanel {
74
75 static final private Dimension LABEL_SIZE = new Dimension(225, 25);
76 static final private Dimension TREE_SIZE = new Dimension(150, 500);
77 //static final private String CONTENTS[] = { "DOWNLOAD.MODE.WebDownload", "DOWNLOAD.MODE.MediaWikiDownload", "DOWNLOAD.MODE.OAIDownload", "DOWNLOAD.MODE.ZDownload" , "DOWNLOAD.MODE.SRWDownload"};
78 private String CONTENTS[] = null;
79
80 private boolean download_button_enabled = false;
81 private boolean ready = false;
82
83 private JPanel options_pane;
84 // TODO should use Vector to store all loaded downloads!!
85
86 private DesignTree tree;
87 private HashMap download_map;
88 private ServerInfoDialog server_info;
89 private JScrollPane list_scroll;
90 private DownloadScrollPane getter;
91 private String mode = null;
92 private TreePath previous_path;
93 private String proxy_url = "";
94 private Proxy proxyObject = null;
95
96 /** Main System code */
97 public DownloadPane() {
98 super();
99 JScrollPane scrol_tmp;
100 this.setComponentOrientation(Dictionary.getOrientation());
101 // TODO: Download the WDownload and the download panel fixed!!
102 getter = new DownloadScrollPane();
103 getter.start();
104 list_scroll = getter.getDownloadJobList();
105 list_scroll.setComponentOrientation(Dictionary.getOrientation());
106
107 // TODO should use Vector to store all loaded downloads!!
108 String lang = Configuration.getLanguage();
109 download_map = new HashMap();
110
111 // run downloadinfo.pl -describeall, load the downloaders into the download_map,
112 // and get back their list of names, which are of the form "<downloader>Download".
113 // Store these names in the CONTENTS[] array as "DOWNLOAD.MODE.<downloader>Download",
114 // with z3950 as a minor exception: DOWNLOAD.MODE.ZDownload.
115 ArrayList<String> downloaderNamesList = loadDownloadersInfo(lang);
116 int size = downloaderNamesList.size();
117 CONTENTS = new String[size];
118 for(int i = 0; i < size; i++) {
119 String downloadName = downloaderNamesList.get(i); // e.g. "WebDownload"
120 CONTENTS[i] = "DOWNLOAD.MODE."+downloadName.replace("3950", ""); // A special case is Z3950Download,
121 // which has to be stored in CONTENTS array as DOWNLOAD.MODE.ZDownload
122 }
123
124 // Creation
125 tree = new DesignTree();
126 tree.setComponentOrientation(Dictionary.getOrientation());
127 options_pane = new JPanel();
128 options_pane.setComponentOrientation(Dictionary.getOrientation());
129
130 JButton clear_cache_button = new GLIButton(Dictionary.get("Mirroring.ClearCache"), Dictionary.get("Mirroring.ClearCache_Tooltip"));
131 clear_cache_button.setEnabled(true);
132 clear_cache_button.setMnemonic(KeyEvent.VK_C);
133
134 JButton download_button = new GLIButton(Dictionary.get("Mirroring.Download"), Dictionary.get("Mirroring.Download_Tooltip"));
135 download_button.setEnabled(true);
136 download_button.setMnemonic(KeyEvent.VK_D);
137
138 JButton information_button = new GLIButton(Dictionary.get("Download.ServerInformation"), Dictionary.get("Download.ServerInformation_Tooltip"));
139 information_button.setEnabled(true);
140 information_button.setMnemonic(KeyEvent.VK_S);
141
142
143 JButton preferences_button = new GLIButton(Dictionary.get("Mirroring.Preferences"), Dictionary.get("Mirroring.Preferences_Tooltip"));
144 preferences_button.setEnabled(true);
145 preferences_button.setMnemonic(KeyEvent.VK_P);
146
147 // Connect
148 clear_cache_button.addActionListener(new ClearCacheListener());
149 download_button.addActionListener(new DownloadButtonListener());
150 preferences_button.addActionListener(new PreferencesButtonActionListener());
151 information_button.addActionListener(new InformationButtonActionListener());
152 tree.addTreeSelectionListener(new TreeListener());
153
154 // Add to Panel
155 JPanel button_pane = new JPanel();
156 button_pane.setComponentOrientation(Dictionary.getOrientation());
157 button_pane.setLayout(new GridLayout(1,4)); // GridLayout so button pane resizes with window-width
158 button_pane.setBorder(BorderFactory.createEtchedBorder());
159 button_pane.add(clear_cache_button);
160 button_pane.add(download_button);
161 button_pane.add(information_button);
162 button_pane.add(preferences_button);
163
164 JPanel tree_pane = new JPanel();
165 tree_pane.setComponentOrientation(Dictionary.getOrientation());
166 tree_pane.setLayout(new BorderLayout());
167 scrol_tmp = new JScrollPane(tree);
168 scrol_tmp.setComponentOrientation(Dictionary.getOrientation());
169 tree_pane.add(scrol_tmp, BorderLayout.CENTER);
170 tree_pane.setPreferredSize(TREE_SIZE);
171
172
173 Color colour_two = Configuration.getColor("coloring.collection_tree_background", false);
174 options_pane.setBackground(colour_two);
175 options_pane.setBorder(BorderFactory.createEtchedBorder());
176
177
178 JScrollPane options_scroll_pane = new JScrollPane(options_pane);
179 options_scroll_pane.setComponentOrientation(Dictionary.getOrientation());
180 JSplitPane mode_pane = new JSplitPane();
181 mode_pane.setComponentOrientation(Dictionary.getOrientation());
182 mode_pane.setBorder(BorderFactory.createEmptyBorder(0,0,0,0));
183 if (Dictionary.getOrientation().isLeftToRight()){
184 mode_pane.add(tree_pane,JSplitPane.LEFT);
185 mode_pane.add(options_scroll_pane,JSplitPane.RIGHT);
186 mode_pane.setDividerLocation(TREE_SIZE.width);
187 }else{
188 mode_pane.add(tree_pane,JSplitPane.RIGHT);
189 mode_pane.add(options_scroll_pane,JSplitPane.LEFT);
190 mode_pane.setDividerLocation(1-TREE_SIZE.width);
191 }
192
193
194 JPanel edit_pane = new JPanel();
195 edit_pane.setComponentOrientation(Dictionary.getOrientation());
196 edit_pane.setBorder(BorderFactory.createCompoundBorder(BorderFactory.createEmptyBorder(2,0,0,0), BorderFactory.createCompoundBorder(BorderFactory.createTitledBorder("Download Setting"), BorderFactory.createEmptyBorder(2,2,2,2))));
197 edit_pane.setLayout(new BorderLayout());
198 edit_pane.add(mode_pane,BorderLayout.CENTER);
199 edit_pane.add(button_pane,BorderLayout.PAGE_END);
200
201 // Add to "this"
202 setBorder(BorderFactory.createEmptyBorder(5,5,5,5));
203 setLayout(new GridLayout(2,1));
204 add(edit_pane);
205 add(list_scroll);
206
207 //set the mode to the first downloader in the list
208 mode = convertCONTENTStoMode(CONTENTS[0]); // e.g. Web
209 generateOptions(options_pane,(Download)download_map.get(mode));
210 previous_path = tree.getSelectionPath();
211 }
212
213 /** System Utilities */
214 public void modeChanged(int gli_mode) {
215 // do nothing at this stage - should we be renewing download options??
216 }
217
218 private void addHeader(String name, Color color, JPanel target_pane) {
219 JPanel header = new JPanel();
220 header.setComponentOrientation(Dictionary.getOrientation());
221 header.setBackground(color);
222 JPanel inner_pane = new JPanel();
223 inner_pane.setComponentOrientation(Dictionary.getOrientation());
224 inner_pane.setBorder(BorderFactory.createCompoundBorder(BorderFactory.createEmptyBorder(5,5,5,5), BorderFactory.createRaisedBevelBorder()));
225 inner_pane.setBackground(color);
226 JLabel header_label = new JLabel("<html><strong>" + name + "</strong></html>");
227 header_label.setComponentOrientation(Dictionary.getOrientation());
228 header_label.setBackground(Configuration.getColor("coloring.collection_heading_background", false));
229 header_label.setHorizontalAlignment(JLabel.CENTER);
230 header_label.setOpaque(true);
231
232 // Layout
233 inner_pane.setLayout(new BorderLayout());
234 inner_pane.add(header_label, BorderLayout.CENTER);
235
236 header.setLayout(new BorderLayout());
237 header.add(inner_pane, BorderLayout.CENTER);
238 target_pane.add(header);
239 }
240
241 /** Supporting Functions */
242 private ArrayList<String> loadDownloadersInfo(String lang) {
243 Document document = null;
244
245 try {
246 if (Gatherer.isGsdlRemote) {
247 String output = Gatherer.remoteGreenstoneServer.getScriptOptions("downloadinfo.pl", "&describeall");
248 Reader reader = new StringReader(output);
249 document = XMLTools.parseXML(reader);
250 }
251 else {
252 ArrayList args_list = new ArrayList();
253 String args[] = null;
254 if(Configuration.perl_path != null) {
255 args_list.add(Configuration.perl_path);
256 } else if(Utility.isWindows()) {
257 args_list.add("Perl.exe");
258 } else {
259 args_list.add("perl");
260 }
261 args_list.add("-S");
262 args_list.add(LocalGreenstone.getBinScriptDirectoryPath()+"downloadinfo.pl");
263 args_list.add("-describeall");
264 args_list.add("-xml");
265 args_list.add("-language");
266 args_list.add(lang);
267
268 // Create the process.
269 args = (String []) args_list.toArray(new String[0]);
270
271 SafeProcess process = new SafeProcess(args);
272 DebugStream.println("Getting Download Info: "+args_list);
273
274 // run the SafeProcess
275 int exitVal = process.runProcess();
276 if(exitVal != 0) {
277 throw new Exception("*** Error running Download Info process, process exited with: "
278 + exitVal);
279 }
280
281 // get the result and process it.
282 // This time we expect XML to have come out of the process std error stream.
283 String errStreamOutput = process.getStdError();
284 ///System.err.println("*********\nDownload Pane data, got:\n" + errStreamOutput + "\n**********\n");
285 StringReader xmlStrReader = new StringReader(errStreamOutput);
286 document = XMLTools.parseXML(xmlStrReader);
287 xmlStrReader.close();
288
289 }
290
291
292 }
293 catch (Exception error) {
294 System.err.println("Failed when trying to parse downloadinfo.pl -describeall");
295 error.printStackTrace();
296 }
297
298 if(document != null) {
299 return parseXML(document.getDocumentElement());
300 }
301
302 return null;
303 }
304
305 private ArrayList<String> parseXML(Node root) {
306 ArrayList<String> downloaders = null;
307 Element downloadList = (Element)root;
308 int length = -1;
309 if(downloadList.hasAttribute("length")) {
310 length = Integer.parseInt(downloadList.getAttribute("length"));
311 downloaders = new ArrayList<String>(length);
312
313 for (Node node = downloadList.getFirstChild(); node != null; node = node.getNextSibling()) {
314 // goes through each <DownloadInfo> of describeAll
315
316 String download_name = null;
317
318 for(Node infoNode = node.getFirstChild();
319 infoNode != null; infoNode = infoNode.getNextSibling()) {
320
321 String node_name = infoNode.getNodeName();
322 if(node_name.equalsIgnoreCase("Name")) { // <Name>WebDownload</Name>
323 download_name = XMLTools.getValue(infoNode); // e.g. WebDownload
324 }
325
326 // At this top level of <DownloadInfo> elements,
327 // skip all the downloaders that are Abstract, as these are pure superclasses
328 else if(node_name.equalsIgnoreCase("Abstract")) {
329 String isAbstract = XMLTools.getValue(infoNode);
330
331 if(isAbstract.equalsIgnoreCase("no") && download_name != null) {
332 downloaders.add(download_name);
333 Download downloader = parseDownloadInfoXML(node); // parse the <DownloadInfo> node properly
334 // now embedded references to abstract superclasses (embedded <DownloadInfo> nodes)
335 // will be handled
336
337 String shortName = download_name.replace("Download", ""); // e.g. "Web"
338 download_map.put(shortName, downloader);
339 }
340 }
341 }
342 }
343 }
344
345 return downloaders;
346 }
347
348 private Download parseDownloadInfoXML(Node root) {
349
350 Download download = new Download();
351 String node_name = null;
352 for (Node node = root.getFirstChild(); node != null; node = node.getNextSibling()) {
353 node_name = node.getNodeName();
354 if(node_name.equalsIgnoreCase("Name")) {
355 String name = XMLTools.getValue(node);
356 download.setName(name);
357 }
358 else if (node_name.equalsIgnoreCase("Desc")) {
359 download.setDescription(XMLTools.getValue(node));
360 }
361 else if (node_name.equalsIgnoreCase("Abstract")) {
362 download.setIsAbstract(XMLTools.getValue(node).equalsIgnoreCase(StaticStrings.YES_STR));
363 }
364 else if(node_name.equalsIgnoreCase("Arguments")) {
365 for(Node arg = node.getFirstChild(); arg != null; arg = arg.getNextSibling()) {
366 node_name = arg.getNodeName();
367 if(node_name.equalsIgnoreCase("Option")) {
368 Argument argument = new Argument((Element)arg);
369 argument.parseXML((Element)arg);
370 argument.setValue(argument.getDefaultValue());
371 download.addArgument(argument);
372 }
373
374 }
375 }
376 else if(node_name.equalsIgnoreCase("DownloadInfo")) {
377 Download super_download = parseDownloadInfoXML(node);
378 download.setSuper(super_download);
379 }
380 }
381
382 if(download.getName() != null) {
383 return download;
384 }
385 return null;
386 }
387
388 /** Update the previous setup */
389 private boolean updateArguments(boolean checkRequired)
390 {
391 boolean cont = true;
392 for(int i = 0; i < options_pane.getComponentCount(); i++) {
393
394 Component component = options_pane.getComponent(i);
395 if(component instanceof ArgumentControl) {
396 cont = cont && ((ArgumentControl)component).updateArgument(checkRequired);
397 }
398 }
399
400 if(cont){return true; }
401
402 return false;
403 }
404
405 /** Generate Controls for Options */
406 /* at some stage we should think about which options should be shown for
407 * different modes. Currently, always show all options (unless hidden)*/
408 private void generateOptions(JPanel options_pane, ArgumentContainer data) {
409 options_pane.removeAll();
410 /** Create the current option panel */
411
412 ArrayList arguments = data.getArguments(true, false);
413 int mode = Configuration.getMode();
414 ArrayList added_arguments = new ArrayList();
415
416 for(int i = 0; i < arguments.size(); i++) {
417 Argument argument = (Argument) arguments.get(i);
418
419 if (argument.isHiddenGLI()) continue;
420 ArgumentControl argument_control = new ArgumentControl(argument,false,null);
421 added_arguments.add(argument_control);
422 }
423
424
425 options_pane.setLayout(new GridLayout(added_arguments.size(),1));
426 for(int i = 0; i < added_arguments.size(); i++) {
427 options_pane.add((ArgumentControl)added_arguments.get(i));
428
429 }
430 }
431
432 /** Behaviour Functions */
433 public void afterDisplay() {
434 ready = true;
435 }
436
437
438 public void gainFocus() {
439 if(!ready) {
440 return;
441 }
442
443 // It is also a good time to determine if the download should be enabled - ie if its allowed to be enabled and a valid URL is present in the field.
444 download_button_enabled = true;
445 //download_button.setEnabled(download_button_enabled);
446 }
447
448
449
450 public void refresh(int refresh_reason, boolean ready)
451 {
452 }
453
454 /** Private classes */
455 /** This tree provides a 'table of contents' for the various components of the design process (collection configuration in more technical terms). */
456 private class DesignTree extends JTree {
457
458 private DesignNode root = null;
459 /** Constructor. Automatically generates all of the nodes, in the order of CONTENTS. */
460 public DesignTree() {
461 super();
462 this.setComponentOrientation(Dictionary.getOrientation());
463 resetModel(Configuration.getMode());
464 expandRow(0);
465 setRootVisible(false);
466 setSelectionRow(0);
467 }
468
469 /** Reset the model used by the design page contents tree. This is necessary to hide the partitions entry when in lower detail modes
470 * @param mode the current detail mode as an int
471 */
472 public void resetModel(int mode) {
473 root = new DesignNode("DOWNLOAD.MODE.Root");
474 // Now add the design categories.
475 for(int i = 0; i < CONTENTS.length; i++) {
476 root.add(new DesignNode(CONTENTS[i]));
477 }
478 this.setModel(new DefaultTreeModel(root));
479 updateUI();
480 }
481 /** Set the current view to the one specified.
482 * @param type the name of the desired view as a String
483 */
484 public void setSelectedView(String type) {
485 type = Dictionary.get(type);
486 for(int i = 0; i < root.getChildCount(); i++) {
487 DesignNode child = (DesignNode) root.getChildAt(i);
488 if(child.toString().equals(type)) {
489 TreePath path = new TreePath(child.getPath());
490 setSelectionPath(path);
491 }
492 }
493 }
494 }
495
496 /** A tree node that retains a reference to one of the possible design sub-views relating to the different sub-managers. */
497 private class DesignNode extends DefaultMutableTreeNode {
498 /** Constructor.
499 * @param object The <strong>Object</strong> assigned to this node.
500 */
501 public DesignNode(String object) {
502 super(object);
503 }
504 /** Retrieve a textual representation of the object.
505 * @return a String
506 */
507 public String toString() {
508 // return Dictionary.get("CDM.GUI." + (String)getUserObject());
509 return Dictionary.get((String) getUserObject());
510 }
511 }
512
513 /** Listens for selection changes in the 'contents' tree, and switches to the appropriate view. */
514 private class TreeListener
515 implements TreeSelectionListener {
516 /** Called whenever the selection changes, we must update the view so it matches the node selected.
517 * @param event A <strong>TreeSelectionEvent</strong> containing more information about the tree selection.
518 * @see org.greenstone.gatherer.cdm.ClassifierManager
519 * @see org.greenstone.gatherer.cdm.CollectionDesignManager
520 * @see org.greenstone.gatherer.cdm.CollectionMetaManager
521 * @see org.greenstone.gatherer.cdm.FormatManager
522 * @see org.greenstone.gatherer.cdm.LanguageManager
523 * @see org.greenstone.gatherer.cdm.MetadataSetView
524 * @see org.greenstone.gatherer.cdm.SubcollectionManager
525 * @see org.greenstone.gatherer.cdm.TranslationView
526 * @see org.greenstone.gatherer.cdm.PlugInManager
527 */
528 public void valueChanged(TreeSelectionEvent event) {
529 if(!tree.isSelectionEmpty()) {
530 TreePath path = tree.getSelectionPath();
531
532 DesignNode node = (DesignNode)path.getLastPathComponent();
533 String type = (String)node.getUserObject();
534 Gatherer.g_man.wait(true);
535
536 // type has the value DOWNLOAD.MODE.<downloader>Download,
537 // mode should then be of the form <downloader>
538 mode = convertCONTENTStoMode(type);
539 generateOptions(options_pane,(Download)download_map.get(mode));
540
541 tree.setSelectionPath(path);
542 previous_path = path;
543 repaint();
544
545 Gatherer.g_man.wait(false);
546 }
547 }
548 }
549
550 private String convertCONTENTStoMode(String content) {
551 return content.replace("DOWNLOAD.MODE.", "").replace("ZDownload", "Z3950").replace("Download", "");
552 }
553
554 private class ClearCacheListener
555 implements ActionListener {
556 public void actionPerformed(ActionEvent event) {
557 // Retrieve the cache folder and delete it.
558 Utility.delete(Utility.getCacheDir());
559 // ...and refresh the node in the workspace tree to show it's all gone
560 Gatherer.g_man.refreshWorkspaceTree(WorkspaceTree.DOWNLOADED_FILES_CHANGED);
561 }
562 }
563
564 private class DownloadButtonListener
565 implements ActionListener {
566 public void actionPerformed(ActionEvent event) {
567
568 if(checkURL(true) && checkProxy() == true) {
569
570 // Proxy settings are now set. Check that the url is not a redirect, else get
571 // redirect url (we do this step in order to avoid some unintuitive behaviour from wget)
572 Download current_download = (Download)download_map.get(mode);
573
574 boolean noCheckCertificate = Configuration.get("general.no_check_certificate", true);
575 Argument no_check_cert_arg = current_download.getArgument("no_check_certificate");
576 if(noCheckCertificate) {
577 no_check_cert_arg.setValue("true");
578 no_check_cert_arg.setAssigned(true);
579 } else {
580 no_check_cert_arg.setValue("false");
581 no_check_cert_arg.setAssigned(false); // only assigned Arguments have values
582 }
583
584 Argument arg_url = current_download.getArgument("url");
585
586 if(arg_url != null) { // it's null for z3950 and possibly for other downloaders
587 String url_str = arg_url.getValue();
588
589 // No longer following URL redirects, since some of this has been taken care of by wget
590 // For the rest, assume the user will get the URL right that they want to download from
591 /*
592 String redirect_url_str = getRedirectURL(url_str);
593
594 // only update the Argument and its GUI ArgumentControl if the URL
595 // has in fact changed
596 if(!url_str.equals(redirect_url_str)) {
597 arg_url.setValue(redirect_url_str);
598 updateArgument(arg_url, redirect_url_str);
599 }
600 */
601 }
602
603 getter.newDownloadJob((Download)download_map.get(mode) ,mode,proxy_url);
604 }
605 }
606 }
607
608 /**
609 * The Java code here will retrieve the page at the given url. If the response code is
610 * a redirect, it will get the redirect url so that wget may be called with the proper url.
611 * This preprocessing of the URL is necessary because:
612 * Wget does not behave the way the browser does when faced with urls of the form
613 * http://www.englishhistory.net/tudor/citizens and if that page does not exist.
614 * The directory listing with a slash at the end (http://www.englishhistory.net/tudor/citizens/)
615 * does exist, however. In order to prevent wget from assuming that the root URL
616 * to traverse is http://www.englishhistory.net/tudor/ instead of the intended
617 * http://www.englishhistory.net/tudor/citizens/, we need give wget the redirect location
618 * that's returned when we initially make a request for http://www.englishhistory.net/tudor/citizens
619 * The proper url is sent back in the Location header, allowing us to bypass wget's
620 * unexpected behaviour.
621 * This method ensures that urls like http://www.nzdl.org/niupepa also continue to work:
622 * there is no http://www.nzdl.org/niupepa/ page, because this url actually redirects to an
623 * entirely different URL.
624 * @return the redirect url for the given url if any redirection is involved, or the
625 * url_str.
626 *
627 * Adding another useful URL on setting Java System Properties:
628 * https://stackoverflow.com/questions/12181843/using-java-to-download-files-from-a-https-url
629 */
630 private String getRedirectURL(String url_str) {
631 boolean noMoreRedirects = false;
632 boolean gotException = false;
633 final int TIMEOUT = 2 * 1000; // ms
634
635 HttpURLConnection connection = null;
636 if(url_str.startsWith("http:") || url_str.startsWith("https:")) { // only test http urls
637 try {
638 // URLConnectionManager class has special handling for https URLs,
639 // so you can control whether you want it to check an HTTPS site's security certificates for you or not
640 boolean noCheckCertificates = true;
641 connection = (HttpURLConnection)URLConnectionManager.getConnection(url_str, this.proxyObject, noCheckCertificates);
642
643 // don't let it automatically follow redirects, since we want to
644 // find out whether we are dealing with redirects in the first place
645 connection.setInstanceFollowRedirects(false);
646 // Connection timeout: if we can't connect, like if the proxy is wrong, don't wait forever
647 // Read timeout: *idle time* when retrieving a link. Don't wait forever to retrieve a page (e.g. if page doesn't exist)
648 // https://stackoverflow.com/questions/6829801/httpurlconnection-setconnecttimeout-has-no-effect
649 connection.setConnectTimeout(TIMEOUT);
650 connection.setReadTimeout(TIMEOUT);
651
652 // now check for whether we get a redirect response
653 // HTTP Codes 3xx are redirects, http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html
654 int responseCode = connection.getResponseCode();
655 if(responseCode >= 300 && responseCode < 400) {
656 //String responseMsg = connection.getResponseMessage();
657
658 // Get the Location header since this specifies the new location of the resource
659 String location = connection.getHeaderField("Location");
660
661 // this becomes the url that wget should download from
662 url_str = location.trim();
663 } else {
664 noMoreRedirects = true;
665 }
666 connection.disconnect();
667 } catch(Exception e) {
668 gotException = true;
669 if(connection != null) {
670 connection.disconnect();
671 }
672 System.err.println("Checking redirection. Tried to connect to "
673 + url_str + ",\nbut got exception: " + e);
674 }
675 }
676
677 if(noMoreRedirects || gotException) {
678 return url_str;
679 }
680 else { // continue checking for whether the new URL redirects elsewhere again
681 return getRedirectURL(url_str);
682 }
683 }
684
685
686 /** For a string-based Argument whose value has changed, this method
687 * updates the GUI ArgumentControl's value correspondingly. */
688 private void updateArgument(Argument arg, String value) {
689 for(int i = 0; i < options_pane.getComponentCount(); i++) {
690 Component component = options_pane.getComponent(i);
691 if(component instanceof ArgumentControl) {
692 ArgumentControl control = (ArgumentControl)component;
693 if(control.getArgument() == arg) {
694 control.setValue(value);
695 control.repaint();
696 }
697 }
698 }
699 }
700
701 private boolean checkURL(boolean checkRequired){
702
703 if (!updateArguments(checkRequired)){
704 return false;
705 }
706
707 Download current_download = (Download)download_map.get(mode);
708 Argument arg_url = current_download.getArgument("url");
709
710 if (arg_url == null) return true;
711
712 String url_str = arg_url.getValue();
713 URL url = null;
714 try {
715 url = new URL(url_str);
716 }
717 catch(MalformedURLException error) {
718 JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("Mirroring.Invalid_URL"), Dictionary.get("Mirroring.Invalid_URL_Title"), JOptionPane.ERROR_MESSAGE);
719 return false;
720 }
721
722 return true;
723 }
724
725 // TODO: Still need to read up on and test how to set wget proxy on windows
726 // Need to be on a windows machine that requires proxy, and get wget to work on cmdline
727 // If that works, then need to check https URLS also work.
728 // See https://superuser.com/questions/526710/how-to-set-http-proxy-address-for-wget-under-windows
729 private boolean checkProxy(){
730
731 proxy_url = null;
732
733 Download current_download = (Download)download_map.get(mode);
734
735 Argument arg = current_download.getArgument("proxy_on");
736
737 if (arg == null) return true;
738
739 // Determine if we have to use a proxy.
740 if(Configuration.get("general.use_proxy", true)) {
741
742 String proxy_host = Configuration.getString("general.proxy_host", true);
743 String proxy_port = Configuration.getString("general.proxy_port", true);
744 // Find out whether the user has already authenticated themselves
745 String user_pass = "";
746 String address = proxy_host + ":" + proxy_port;
747
748 int count = 0;
749 // Only for wget, need to avoid a second automatic authentication popup (first asks
750 // the proxy authentication for wget, and the second will ask the same for the realm)
751 // Once the authentication has been reused, it will set the GAuthenticator state back to REGULAR
752 GAuthenticator.setMode(GAuthenticator.DOWNLOAD);
753 while(count < 3 && (user_pass = (String) GAuthenticator.authentications.get(address)) == null) {
754 Authenticator.requestPasswordAuthentication(proxy_host, null, Integer.parseInt(proxy_port), "http://", Dictionary.get("WGet.Prompt"), "HTTP");
755 count++;
756 }
757 if(count >= 3) {
758 return false;
759 }
760
761 // https://askubuntu.com/questions/664777/systemwide-proxy-settings-in-ubuntu
762 // http://www.rgagnon.com/javadetails/java-0085.html
763 // how-do-i-make-httpurlconnection-use-a-proxy
764 // https://stackoverflow.com/questions/8030908/how-to-check-if-proxy-is-working-in-java
765 proxyObject = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxy_host, Integer.parseInt(proxy_port)));
766
767 if(user_pass.indexOf("@") != -1) {
768
769 // Write the use proxy command - we don't do this anymore, instead we set environment variables - hopefully these can't be spied on like the following can (using ps) - actually the environment stuff didn't work for windows, so lets go back to this
770 if (Utility.isWindows()) {
771
772 arg.setValue("true"); // proxy_on argument
773 arg.setAssigned(true);
774
775 arg = current_download.getArgument("proxy_host");
776 arg.setValue(proxy_host);
777 arg.setAssigned(true);
778
779 arg = current_download.getArgument("proxy_port");
780 arg.setValue(proxy_port);
781 arg.setAssigned(true);
782
783 arg = current_download.getArgument("user_name");
784 arg.setValue(user_pass.substring(0, user_pass.indexOf("@")));
785 arg.setAssigned(true);
786
787 arg = current_download.getArgument("user_password");
788 arg.setValue(user_pass.substring(user_pass.indexOf("@") + 1));
789 arg.setAssigned(true);
790 }
791 else{
792 String user_name = user_pass.substring(0, user_pass.indexOf("@"));
793 String user_pwd = user_pass.substring(user_pass.indexOf("@") + 1);
794 proxy_url = user_name+":"+user_pwd+"@"+proxy_host+":"+proxy_port+"/";
795
796 }
797
798 return true;
799 }
800 else{
801 return false;
802 }
803
804 }
805
806 return true;
807 }
808
809 /*
810 private class PreferencesButtonActionListener
811 implements ActionListener {
812 public void actionPerformed(ActionEvent event) {
813 new Preferences(Preferences.CONNECTION_PREFS);
814 }
815 }*/
816
817 private class InformationButtonActionListener
818 implements ActionListener {
819 public void actionPerformed(ActionEvent event) {
820 //turn off the check for find argument
821 Download current_download = (Download)download_map.get(mode);
822
823 if (!checkProxy() || !checkURL(false) )return;
824
825 if(server_info != null) {
826 server_info.dispose();
827 }
828
829 Argument arg_url = current_download.getArgument("url");
830 String str_url = "";
831 if( arg_url!= null && arg_url.isAssigned()) {
832 str_url = arg_url.getValue();
833 }
834
835 boolean noCheckCertificate = Configuration.get("general.no_check_certificate", true);
836 Argument no_check_cert_arg = current_download.getArgument("no_check_certificate");
837 if(noCheckCertificate) {
838 no_check_cert_arg.setValue("true");
839 no_check_cert_arg.setAssigned(true);
840 } else {
841 no_check_cert_arg.setValue("false");
842 no_check_cert_arg.setAssigned(false); // only assigned Arguments have values
843 }
844
845
846 //str_url = getRedirectURL(str_url); // work out the real URL
847
848 server_info = new ServerInfoDialog(str_url ,proxy_url, mode,(Download)download_map.get(mode));
849
850 }
851 }
852
853 private class PreferencesButtonActionListener
854 implements ActionListener {
855 public void actionPerformed(ActionEvent event) {
856 new Preferences(Preferences.CONNECTION_PREFS);
857 }
858 }
859}
Note: See TracBrowser for help on using the repository browser.