Changeset 31851

Timestamp:

2017-08-03T19:28:08+12:00 (6 years ago)

Author:

ak19

Message:

1. Fixes to get proxying to work on Windows. 2. Fixes to timeout if a page doesn't exist and it takes forever to read. Both for downloading from a URL and getting server info (perl code), and also in Java code, when doing a getRedirectURL(). Generally, a URL is correct and when wget is launched, a cancel operation in the Java GUI successfully causes and interrupt which then terminates wget. However, if the URL doesn't exist, either when getting serer info or when downloading, the wget launched by the perl seems to block or something, and the interrupt is not noticed until the wget is manually terminated through the task manager. Then the interrupt is finally noticed. If pages would indicate they don't exist, then it wouldn't have been a problem. This issue is now circumvented through setting a read-timeout, to stop retrieving pages that don't exist but that take forever to access anyway as they don't indicate that they don't exist. A connect timeout is for if you get proxy details wrong or something like that and it takes forever to connect.

Location:

main/trunk

Files:

• gli/src/org/greenstone/gatherer/gui/DownloadPane.java (modified) (2 diffs)
• greenstone2/perllib/downloaders/WebDownload.pm (modified) (2 diffs)
• greenstone2/perllib/downloaders/WgetDownload.pm (modified) (2 diffs)

main/trunk/gli/src/org/greenstone/gatherer/gui/DownloadPane.java

@@ -613 +613 @@
     boolean noMoreRedirects = false;
     boolean gotException = false;
-
+    final int TIMEOUT = 2 * 1000; // ms
+    
     HttpURLConnection connection = null;
     if(url_str.startsWith("http:") || url_str.startsWith("https:")) { // only test http urls
@@ -625 +626 @@
         // find out whether we are dealing with redirects in the first place
         connection.setInstanceFollowRedirects(false);
-        connection.setConnectTimeout(5 * 1000); // ms
+        // Connection timeout: if we can't connect, like if the proxy is wrong, don't wait forever
+        // Read timeout: *idle time* when retrieving a link. Don't wait forever to retrieve a page (e.g. if page doesn't exist)
+        // https://stackoverflow.com/questions/6829801/httpurlconnection-setconnecttimeout-has-no-effect
+        connection.setConnectTimeout(TIMEOUT); 
+        connection.setReadTimeout(TIMEOUT); 
         
         // now check for whether we get a redirect response

main/trunk/greenstone2/perllib/downloaders/WebDownload.pm

@@ -115 +115 @@
     }
     #my $cmdWget = "-N -k -x -t 2 -P \"".$hashGeneralOptions->{"cache_dir"}."\" $strWgetOptions $strOptions ".$self->{'url'};
-    my $cmdWget = "-N -k -x -t 2  $strWgetOptions $strOptions $cache_dir " .$self->{'url'};   
-
+    my $cmdWget = "-N -k -x -t 2 --read-timeout=2 --connect-timeout=2 $strWgetOptions $strOptions $cache_dir " .$self->{'url'};   
+
+    #print STDOUT "\n@@@@ RUNNING WGET CMD: $cmdWget\n\n";
+    
     # Download the web pages
     # print "Start download from $self->{'url'}...\n";
@@ -186 +188 @@
     my $strOptions = $self->getWgetOptions();
 
-    my $strBaseCMD = $strOptions." -q -O - \"$self->{'url'}\"";
+    my $strBaseCMD = $strOptions." --timeout=4 --tries=1 -q -O - \"$self->{'url'}\"";
 
   

main/trunk/greenstone2/perllib/downloaders/WgetDownload.pm

@@ -167 +167 @@
     {
 
-    $strOptions .= " -e httpproxy=$self->{'proxy_host'}:$self->{'proxy_port'} ";
+    if($self->{'url'} =~ m/^https\:/) {
+        $strOptions .= " -e https_proxy=$self->{'proxy_host'}:$self->{'proxy_port'} ";  
+    } else {
+        $strOptions .= " -e http_proxy=$self->{'proxy_host'}:$self->{'proxy_port'} ";
+    }   
 
     if ($self->{'user_name'} && $self->{'user_password'})
@@ -179 +183 @@
     }
 
+    if($self->{'no_check_certificate'} && $self->{'url'} =~ m/^https\:/) {
+        $strOptions .= " --no-check-certificate ";
+    }
+    
     return $strOptions;
 }