Show
Ignore:
Timestamp:
14.08.2017 22:23:04 (2 years ago)
Author:
ak19
Message:

All the changes that were required to set up multiple proxy servers, one for HTTP, one for HTTPS, one for FTP. Still need to test on Windows

Location:
main/trunk/greenstone2/perllib
Files:
3 modified

Legend:

Unmodified
Added
Removed
  • main/trunk/greenstone2/perllib/downloaders/WebDownload.pm

    r31878 r31880  
    193193    my $strBaseCMD = $strOptions." --tries=2 -q -O - \"$self->{'url'}\""; 
    194194 
    195     #&util::print_env(STDERR, "https_proxy", "http_proxy", "HTTPS_PROXY", "HTTP_PROXY", "ftp_proxy", "FTP_PROXY"); 
     195    #&util::print_env(STDERR, "https_proxy", "http_proxy", "ftp_proxy"); 
    196196    #&util::print_env(STDERR);   
    197197     
     
    201201     
    202202    print STDERR "Server information is unavailable.\n"; 
    203  
    204     #&util::print_env(STDERR, "https_proxy", "http_proxy"); 
    205      
    206     if ($self->{'proxy_on'} && $self->{'proxy_host'} && $self->{'proxy_port'}) { # if proxying set, the settings may be wrong 
    207         &gsprintf::gsprintf_multiline(STDERR, "{WebDownload.proxied_connect_failed_info}\n", $self->{'proxy_host'}, $self->{'proxy_port'}); 
     203     
     204    if ($self->{'proxy_on'}) { # if proxying set, the settings may be wrong 
     205        &gsprintf::gsprintf_multiline(STDERR, "{WebDownload.proxied_connect_failed_info}\n"); 
     206         
     207        if($self->{'http_proxy_host'} && defined $self->{'http_proxy_port'}) { 
     208        &gsprintf::gsprintf_multiline(STDERR, "{WebDownload.http_proxy_settings}\n", $self->{'http_proxy_host'}, $self->{'http_proxy_port'}); 
     209        } 
     210        if($self->{'https_proxy_host'} && defined $self->{'https_proxy_port'}) { 
     211        &gsprintf::gsprintf_multiline(STDERR, "{WebDownload.https_proxy_settings}\n", $self->{'https_proxy_host'}, $self->{'https_proxy_port'}); 
     212        } 
     213        if($self->{'ftp_proxy_host'} && defined $self->{'ftp_proxy_port'}) { 
     214        &gsprintf::gsprintf_multiline(STDERR, "{WebDownload.ftp_proxy_settings}\n", $self->{'ftp_proxy_host'}, $self->{'ftp_proxy_port'}); 
     215        } 
    208216    } else { # else no proxy set, the user may need proxy settings 
    209217        &gsprintf::gsprintf_multiline(STDERR, "{WebDownload.proxyless_connect_failed_info}\n"); 
  • main/trunk/greenstone2/perllib/downloaders/WgetDownload.pm

    r31878 r31880  
    5151    'reqd' => "no", 
    5252    'hiddengli' => "yes"}, 
    53       { 'name' => "proxy_host",   
    54     'desc' => "{WgetDownload.proxy_host}",          
    55     'type' => "string", 
    56     'reqd' => "no", 
    57     'hiddengli' => "yes"}, 
    58       { 'name' => "proxy_port",  
    59     'desc' => "{WgetDownload.proxy_port}",          
     53      { 'name' => "http_proxy_host", 
     54    'desc' => "{WgetDownload.http_proxy_host}", 
     55    'type' => "string", 
     56    'reqd' => "no", 
     57    'hiddengli' => "yes"}, 
     58      { 'name' => "http_proxy_port", 
     59    'desc' => "{WgetDownload.http_proxy_port}", 
     60    'type' => "string", 
     61    'reqd' => "no", 
     62    'hiddengli' => "yes"}, 
     63      { 'name' => "https_proxy_host",   
     64    'desc' => "{WgetDownload.https_proxy_host}",          
     65    'type' => "string", 
     66    'reqd' => "no", 
     67    'hiddengli' => "yes"}, 
     68      { 'name' => "https_proxy_port",  
     69    'desc' => "{WgetDownload.https_proxy_port}",          
     70    'type' => "string", 
     71    'reqd' => "no", 
     72    'hiddengli' => "yes"}, 
     73      { 'name' => "ftp_proxy_host",   
     74    'desc' => "{WgetDownload.ftp_proxy_host}",          
     75    'type' => "string", 
     76    'reqd' => "no", 
     77    'hiddengli' => "yes"}, 
     78      { 'name' => "ftp_proxy_port",  
     79    'desc' => "{WgetDownload.ftp_proxy_port}",          
    6080    'type' => "string", 
    6181    'reqd' => "no", 
    6282    'hiddengli' => "yes"}, 
    6383      { 'name' => "user_name",   
    64     'desc' => "{WgetDownload.user_name}",          
     84    'desc' => "{WgetDownload.user_name}", 
    6585    'type' => "string", 
    6686    'reqd' => "no", 
    6787    'hiddengli' => "yes"}, 
    6888      { 'name' => "user_password",  
    69     'desc' => "{WgetDownload.user_password}",          
     89    'desc' => "{WgetDownload.user_password}", 
    7090    'type' => "string", 
    7191    'reqd' => "no", 
     
    171191    my ($self) = @_; 
    172192    my $strOptions = ""; 
    173      
    174     if ($self->{'proxy_on'} && $self->{'proxy_host'} && $self->{'proxy_port'}) 
     193 
     194    if($self->{'http_proxy_host'} && $self->{'http_proxy_port'}) { 
     195    $strOptions .= " -e http_proxy=$self->{'http_proxy_host'}:$self->{'http_proxy_port'} "; 
     196    } 
     197    if($self->{'https_proxy_host'} && $self->{'https_proxy_port'}) { 
     198    $strOptions .= " -e https_proxy=$self->{'https_proxy_host'}:$self->{'https_proxy_port'} "; 
     199    } 
     200    if($self->{'ftp_proxy_host'} && $self->{'ftp_proxy_port'}) { 
     201    $strOptions .= " -e ftp_proxy=$self->{'ftp_proxy_host'}:$self->{'ftp_proxy_port'} "; 
     202    } 
     203     
     204    # For wget, there is only one set pair of proxy-user and proxy-passwd, so wget seems to assume 
     205    # that all 3 proxy protocols (http|https|ftp) will use the same username and pwd combination? 
     206    # Note that this only matters when passing the proxying details as flags to wget, not when 
     207    # the proxies are setup as environment variables. 
     208    if ($self->{'user_name'} && $self->{'user_password'}) 
    175209    { 
    176     $strOptions .= " -e https_proxy=$self->{'proxy_host'}:$self->{'proxy_port'} ";   
    177     $strOptions .= " -e http_proxy=$self->{'proxy_host'}:$self->{'proxy_port'} ";        
    178      
    179     if ($self->{'user_name'} && $self->{'user_password'}) 
    180     { 
    181         $strOptions .= "--proxy-user=$self->{'user_name'}"." --proxy-passwd=$self->{'user_password'}"; 
    182     } 
    183     } 
    184      
    185     if ($self->{'proxy_on'}) { 
    186     $strOptions .= " --proxy "; 
    187     } 
     210    $strOptions .= "--proxy-user=$self->{'user_name'}"." --proxy-passwd=$self->{'user_password'}";       
     211    # how is "--proxy-passwd" instead of "--proxy-password" even working????         
     212    # see https://www.gnu.org/software/wget/manual/html_node/Proxies.html 
     213    # and https://www.gnu.org/software/wget/manual/wget.html 
     214    # Not touching this, in case the manual is simply wrong. Since our code works in  
     215    # practice (when we were still using wget proxy username/pwd flags for windows). 
     216    }    
    188217     
    189218    return $strOptions; 
     
    202231    # http://www.perlmonks.org/?node=what%20is%20true%20and%20false%20in%20Perl%3F 
    203232     
    204     if (!$ENV{'http_proxy'} && !$ENV{'https_proxy'}) { 
    205     $strOptions .= $self->addProxySettingsAsWgetFlags(); 
    206     } # else wget will use proxy settings in environment, assume enough settings have been provided 
     233    if ($self->{'proxy_on'}) { 
     234    if(!$ENV{'http_proxy'} && !$ENV{'https_proxy'} && !$ENV{'ftp_proxy'}) { 
     235        $strOptions .= $self->addProxySettingsAsWgetFlags(); 
     236    } # else wget will use proxy settings in environment, assume enough settings have been provided 
     237    # either way, we're using the proxy 
     238    $strOptions .= " --proxy "; 
     239    } 
    207240     
    208241    if($self->{'no_check_certificate'}) { # URL may be http that gets redirected to https, so if no_check_certificate is on, turn it on even if URL is http 
  • main/trunk/greenstone2/perllib/strings.properties

    r31875 r31880  
    13361336WebDownload.html_only:Download only HTML files, and ignore associated files e.g images and stylesheets 
    13371337WebDownload.html_only_disp:Only HTML files 
    1338 WebDownload.proxied_connect_failed_info:Current proxy settings are:\n- host=%s\n- port=%s 
     1338WebDownload.proxied_connect_failed_info:Current proxy settings are: 
     1339WebDownload.http_proxy_settings:- HTTP host=%s : port=%s 
     1340WebDownload.https_proxy_settings:- HTTPS host=%s : port=%s 
     1341WebDownload.ftp_proxy_settings:- FTP host=%s : port=%s 
    13391342WebDownload.proxyless_connect_failed_info:- The external server might not be responding\n- or you might need to switch on proxy settings 
    13401343WebDownload.connect_failed_info:- or try ticking No Certificate Checking (affects 'https' URLs)\nin File > Preferences > Connection 
     
    13421345WgetDownload.desc: Base class that handles calls to wget 
    13431346WgetDownload.proxy_on:Proxy on 
    1344 WgetDownload.proxy_host:Proxy host 
    1345 WgetDownload.proxy_port:Proxy port 
     1347WgetDownload.http_proxy_host:HTTP proxy host 
     1348WgetDownload.http_proxy_port:HTTP proxy port 
     1349WgetDownload.https_proxy_host:HTTPS proxy host 
     1350WgetDownload.https_proxy_port:HTTPS proxy port 
     1351WgetDownload.ftp_proxy_host:FTP proxy host 
     1352WgetDownload.ftp_proxy_port:FTP proxy port 
    13461353WgetDownload.user_name:User name 
    13471354WgetDownload.user_password:User password