root/main/trunk/greenstone2/perllib/downloaders/WgetDownload.pm @ 31874

Revision 31874, 19.5 KB (checked in by ak19, 2 years ago)

Fix ahead of switching over to proxy env vars on Windows. This fix sets proxy settings for Windows downloads, so that http_proxy and https_proxy flags can both be set.

  • Property svn:keywords set to Author Date Id Revision
Line 
1###########################################################################
2#
3# WgetDownload.pm -- Download base module that handles calling Wget
4# A component of the Greenstone digital library software
5# from the New Zealand Digital Library Project at the
6# University of Waikato, New Zealand.
7#
8# Copyright (C) 2006 New Zealand Digital Library Project
9#
10# This program is free software; you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation; either version 2 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License
21# along with this program; if not, write to the Free Software
22# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
23#
24###########################################################################
25
26package WgetDownload;
27
28eval {require bytes};
29
30# suppress the annoying "subroutine redefined" warning that various
31# plugins cause under perl 5.6
32$SIG{__WARN__} = sub {warn($_[0]) unless ($_[0] =~ /Subroutine\s+\S+\sredefined/)};
33
34use BaseDownload;
35use strict;
36use Cwd;
37use util;
38use IPC::Open3;
39use IO::Select;
40use IO::Socket;
41
42
43sub BEGIN {
44    @WgetDownload::ISA = ('BaseDownload');
45}
46
47my $arguments =
48     [ { 'name' => "proxy_on",
49    'desc' => "{WgetDownload.proxy_on}",
50    'type' => "flag",
51    'reqd' => "no",
52    'hiddengli' => "yes"},
53      { 'name' => "proxy_host", 
54    'desc' => "{WgetDownload.proxy_host}",         
55    'type' => "string",
56    'reqd' => "no",
57    'hiddengli' => "yes"},
58      { 'name' => "proxy_port",
59    'desc' => "{WgetDownload.proxy_port}",         
60    'type' => "string",
61    'reqd' => "no",
62    'hiddengli' => "yes"},
63      { 'name' => "user_name", 
64    'desc' => "{WgetDownload.user_name}",         
65    'type' => "string",
66    'reqd' => "no",
67    'hiddengli' => "yes"},
68      { 'name' => "user_password",
69    'desc' => "{WgetDownload.user_password}",         
70    'type' => "string",
71    'reqd' => "no",
72    'hiddengli' => "yes"},
73      { 'name' => "no_check_certificate",
74    'desc' => "{WgetDownload.no_check_certificate}",
75    'type' => "flag",
76    'reqd' => "no",
77    'hiddengli' => "yes"}
78     ];
79
80my $options = { 'name'     => "WgetDownload",
81        'desc'     => "{WgetDownload.desc}",
82        'abstract' => "yes",
83        'inherits' => "yes",
84        'args'     => $arguments };
85
86
87# Declaring file global variables related to the wget child process so that
88# the termination signal handler for SIGTERM can close the streams and tidy
89# up before ending the child process.
90my $childpid;
91my ($chld_out, $chld_in);
92my ($serverSocket, $read_set);
93
94# The port this script's server socket will be listening on, to handle
95# incoming signals from GLI to terminate wget. This is also file global,
96# since OAIDownload.pm will make several calls on wget using the same
97# instance of this script and we want to reuse whatever port GLI gave us.
98my $port;
99
100# When this script is called from the command line, this handler will be called
101# if this process is killed or abruptly ends due to receiving one of the
102# terminating signals that this handler is registered to deal with.
103sub abrupt_end_handler {
104    my $termination_signal = shift (@_);
105
106    if(defined $childpid) {
107    close($chld_out);
108    close($chld_in);
109   
110    print STDOUT "Received termination signal: $termination_signal\n";
111
112    # Send TERM signal to child process to terminate it. Sending the INT signal doesn't work
113    # See http://perldoc.perl.org/perlipc.html#Signals
114    # Warning on using kill at http://perldoc.perl.org/perlfork.html
115    kill("TERM", $childpid);
116
117    # If the SIGTERM sent on Linux calls this handler, we want to make
118    # sure any socket connection is closed.
119    # Otherwise sockets are only used when this script is run from GLI
120    # in which case the handlers don't really get called.
121    if(defined $serverSocket) {
122        $read_set->remove($serverSocket) if defined $read_set;
123        close($serverSocket);
124    }
125    }
126
127    exit(0);
128}
129
130# Registering a handler for when termination signals SIGINT and SIGTERM are received to stop
131# the wget child process. SIGTERM--generated by Java's Process.destroy()--is the default kill
132# signal (kill -15) on Linux, while SIGINT is generated upon Ctrl-C (also on Windows).
133# Note that SIGKILL can't be handled as the handler won't get called for it. More information:
134# http://affy.blogspot.com/p5be/ch13.htm
135# http://perldoc.perl.org/perlipc.html#Signals
136$SIG{'INT'} = \&abrupt_end_handler;
137$SIG{'TERM'} = \&abrupt_end_handler;
138
139sub new {
140    my ($class) = shift (@_);
141    my ($getlist,$inputargs,$hashArgOptLists) = @_;
142    push(@$getlist, $class);
143
144    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
145    push(@{$hashArgOptLists->{"OptList"}},$options);
146
147    my $self = new BaseDownload($getlist,$inputargs,$hashArgOptLists);
148
149    # the wget binary is dependent on the gnomelib_env (particularly lib/libiconv2.dylib) being set, particularly on Mac Lions (android too?)
150    &util::set_gnomelib_env(); # this will set the gnomelib env once for each subshell launched, by first checking if GEXTGNOME is not already set
151
152    return bless $self, $class;
153}
154
155sub checkWgetSetup
156{
157    my ($self,$blnGliCall) = @_;
158    #TODO: proxy detection??
159   
160    if((!$blnGliCall) && $self->{'proxy_on'})
161    {
162    &checkProxySetup($self);
163    }
164    &checkURL($self);
165}
166
167sub getWgetOptions
168{
169    my ($self) = @_;
170    my $strOptions = "";
171
172    # If http_proxy ENV VARS are not set, but proxy Perl vars are set, then we're on Windows
173    # and need to use the proxy vars as flags to wget
174    # If http_proxy Env Vars are set, then we're on Linux, wget will use the http(s) proxy
175    # env vars and we shouldn't be passing any proxy perl vars as flags
176   
177    # Truth in Perl: https://home.ubalt.edu/abento/452/perl/perltruth.html
178    # http://www.perlmonks.org/?node=what%20is%20true%20and%20false%20in%20Perl%3F
179   
180    if (!$ENV{'http_proxy'} && !$ENV{'https_proxy'}) {
181    if ($self->{'proxy_on'} && $self->{'proxy_host'} && $self->{'proxy_port'})
182    {
183        $strOptions .= " -e https_proxy=$self->{'proxy_host'}:$self->{'proxy_port'} "; 
184        $strOptions .= " -e http_proxy=$self->{'proxy_host'}:$self->{'proxy_port'} ";       
185       
186        if ($self->{'user_name'} && $self->{'user_password'})
187        {
188        $strOptions .= "--proxy-user=$self->{'user_name'}"." --proxy-passwd=$self->{'user_password'}";
189        }
190    }
191
192    if ($self->{'proxy_on'}) {
193        $strOptions .= " --proxy ";
194    }
195    }
196
197    if($self->{'no_check_certificate'}) { #&& $self->{'url'} =~ m/^https\:/) { # URL may be http that gets redirected to https, so if no_check_certificate is on, turn it on even if URL is http
198
199    $strOptions .= " --no-check-certificate ";
200    }
201   
202    return $strOptions;
203}
204
205# Checking for proxy setup: proxy server, proxy port, proxy username and password.
206sub checkProxySetup
207{
208    my ($self) = @_;
209    ($self->{'proxy_on'}) || &error("checkProxySetup","The proxy is not on? How could that be happening?");
210    # Setup .wgetrc by using $self->{'proxy_host'} and $self->{'proxy_port'}
211    # Test if the connection is successful. If the connection wasn't successful then ask user to supply username and password.
212
213}
214
215# Returns true if the wget status needs to be monitored through sockets
216# (if a socket is used to communicate with the Java program on when to
217# terminate wget). True if we are running gli, or if the particular type
218# of WgetDownload is *not* OAIDownload (in that case, the original way of
219# terminating the perl script from Java terminated wget as well).
220sub dealingWithSockets() {
221    my ($self) = @_;
222    return (defined $self->{'gli'} && $self->{'gli'} && !defined $port && ref($self) ne "OAIDownload");
223                       # use ref($self) to find the classname of an object
224}
225
226
227sub useWget
228{
229    #local $| = 1; # autoflush stdout buffer
230    #print STDOUT "*** Start of subroutine useWget in $0\n";
231
232    my ($self, $cmdWget,$blnShow, $working_dir) = @_;
233
234    my ($strReadIn,$strLine,$command);
235    $strReadIn = "" unless defined $strReadIn;
236
237    my $current_dir = cwd();
238    my $changed_dir = 0;
239    if (defined $working_dir && -e $working_dir) {
240    chdir "$working_dir";
241    $changed_dir = 1;
242    }
243
244    # When we are running this script through GLI, the SIGTERM signal handler
245    # won't get called on Windows when wget is to be prematurely terminated.
246    # Instead, when wget has to be terminated in the middle of execution, GLI will
247    # connect to a serverSocket here to communicate when it's time to stop wget.
248    if($self->dealingWithSockets()) {
249
250    $port = <STDIN>; # gets a port on localhost that's not yet in use
251    chomp($port);
252   
253    $serverSocket = IO::Socket::INET->new( Proto     => 'tcp',
254                           LocalPort => $port,
255                           Listen    => 1,
256                           Reuse     => 1);
257   
258    die "can't setup server" unless $serverSocket;
259    #print STDOUT "[Serversocket $0 accepting clients at port $port]\n";
260
261    $read_set = new IO::Select();         # create handle set for reading
262    $read_set->add($serverSocket);        # add the main socket to the set
263    }
264
265    my $wget_file_path = &FileUtils::filenameConcatenate($ENV{'GSDLHOME'}, "bin", $ENV{'GSDLOS'}, "wget");
266    $command = "\"$wget_file_path\" $cmdWget";
267    #print STDOUT "Command is: $command\n"; # displayed in GLI output
268    #print STDERR "Command is: $command\n"; # goes into ServerInfoDialog
269   
270    # Wget's output needs to be monitored to find out when it has naturally terminated.
271    # Wget's output is sent to its STDERR so we can't use open2 without doing 2>&1.
272    # On linux, 2>&1 launches a subshell which then launches wget, meaning that killing
273    # the childpid does not kill wget on Linux but the subshell that launched it instead.
274    # Therefore, we use open3. Though the child process wget sends output only to its stdout,
275    # using open3 says chld_err is undefined and the output of wget only comes in chld_out(!)
276    # However that may be, it works with open3. But to avoid the confusion of managing and
277    # closing an extra unused handle, a single handle is used instead for both the child's
278    # stderr and stdout.
279    # See http://blog.0x1fff.com/2009/09/howto-execute-system-commands-in-perl.html
280    # for why this is the right thing to do.
281
282    # Both open2 and open3 don't return on failure, but raise an exception. The handling
283    # of the exception is described on p.568 of the Perl Cookbook
284    eval {
285    $childpid = open3($chld_in, $chld_out, $chld_out, $command);
286    };
287    if ($@) {
288    if($@ =~ m/^open3/) {
289        die "open3 failed in $0: $!\n$@\n";     
290    }
291    die "Tried to launch open3 in $0, got unexpected exception: $@";
292    }
293
294    my $loop = 1;
295    while($loop)
296    {
297    if (defined(my $strLine=<$chld_out>)) { # we're reading in from child process' stdout
298        if($blnShow) {
299        print STDERR "$strLine\n";
300        }
301        $strReadIn .= $strLine;
302    }
303    else { # wget finished, terminate naturally
304        #print STDOUT "\nPerl: open2 command, input stream closed. Wget terminated naturally.\n";
305        close($chld_in);
306        close($chld_out);
307        waitpid $childpid, 0;
308        $loop = 0;
309       
310        $childpid = undef;
311        if(defined $port) {
312        $read_set->remove($serverSocket);
313        close($serverSocket);
314        }
315    }
316
317    # if we run this script from the command-line (as opposed to from GLI),
318    # then we're not working with sockets and can therefore can skip the next bits
319    next unless(defined $port);
320
321    # http://www.perlfect.com/articles/select.shtml
322    # "multiplex between several filehandles within a single thread of control,
323    # thus creating the effect of parallelism in the handling of I/O."
324    my @rh_set = $read_set->can_read(0.002); # every 2 ms check if there's a client socket connecting
325
326    # take all readable handles in turn
327    foreach my $rh (@rh_set) {
328        if($rh == $serverSocket) {
329        my $client = $rh->accept();
330        #$client->autoflush(1); # autoflush output buffer - don't put this back in: output split irregularly over lines
331        print $client "Talked to ServerSocket (port $port). Connection accepted\n";
332       
333        # Read from the client (getting rid of the trailing newline)
334        # Has the client sent the <<STOP>> signal?
335        my $signal = <$client>;
336        chomp($signal);
337        if($signal eq "<<STOP>>") {
338            print $client "Perl received STOP signal (on port $port): stopping wget\n";
339            $loop = 0;                          # out of outer while loop
340            $self->{'forced_quit'} = 1;         # subclasses need to know we're quitting
341           
342            # Sometimes the wget process takes some time to start up. If the STOP signal
343            # was sent, don't try to terminate the process until we know it is running.
344            # Otherwise wget may start running after we tried to kill it. Wait 5 seconds
345            # for it to start up, checking for whether it is running in order to kill it.
346            for(my $seconds = 1; $seconds <= 5; $seconds++) {
347            if(kill(0, $childpid)) {
348                # If kill(0, $childpid) returns true, then the process is running
349                # and we need to kill it.
350                close($chld_in);
351                close($chld_out);
352                kill("TERM", $childpid);
353               
354                $childpid = undef;
355               
356                # Stop monitoring the read_handle and close the serverSocket
357                # (the Java end will close the client socket that Java opened)
358                $read_set->remove($rh);     #$read_set->remove($serverSocket);
359                close($rh);             #close($serverSocket);
360                print $client "Perl terminated wget and is about to exit\n";
361                last;                           # out of inner for loop
362            }
363            else { # the process may just be starting up, wait
364                sleep(1);
365            }
366            }
367            last;                               # out of foreach loop
368        }
369        }
370    }
371    }
372
373    if ($changed_dir) {
374    chdir $current_dir;
375    }
376   
377    return $strReadIn;
378}
379
380
381sub useWgetMonitored
382{
383    #local $| = 1; # autoflush stdout buffer
384    #print STDOUT "*** Start of subroutine useWgetMonitored in $0\n";
385
386    my ($self, $cmdWget,$blnShow, $working_dir) = @_;
387
388
389    my $current_dir = cwd();
390    my $changed_dir = 0;
391    if (defined $working_dir && -e $working_dir) {
392    chdir "$working_dir";
393    $changed_dir = 1;
394    }
395
396    # When we are running this script through GLI, the SIGTERM signal handler
397    # won't get called on Windows when wget is to be prematurely terminated.
398    # Instead, when wget has to be terminated in the middle of execution, GLI will
399    # connect to a serverSocket here to communicate when it's time to stop wget.
400    if($self->dealingWithSockets()) {
401
402    $port = <STDIN>; # gets a port on localhost that's not yet in use
403    chomp($port);
404   
405    $serverSocket = IO::Socket::INET->new( Proto     => 'tcp',
406                           LocalPort => $port,
407                           Listen    => 1,
408                           Reuse     => 1);
409   
410    die "can't setup server" unless $serverSocket;
411    #print STDOUT "[Serversocket $0 accepting clients at port $port]\n";
412
413    $read_set = new IO::Select();         # create handle set for reading
414    $read_set->add($serverSocket);        # add the main socket to the set
415    }
416
417    my $wget_file_path = &FileUtils::filenameConcatenate($ENV{'GSDLHOME'}, "bin", $ENV{'GSDLOS'}, "wget");
418    my $command = "\"$wget_file_path\" $cmdWget";
419    #print STDOUT "Command is: $command\n";
420
421    eval {     # see p.568 of Perl Cookbook
422    $childpid = open3($chld_in, $chld_out, $chld_out, $command);
423    };
424    if ($@) {
425    if($@ =~ m/^open3/) {
426        die "open3 failed in $0: $!\n$@\n";     
427    }
428    die "Tried to launch open3 in $0, got unexpected exception: $@";
429    }
430
431    my $full_text = "";
432    my $error_text = "";
433    my @follow_list = ();
434    my $line;
435
436    my $loop = 1;
437    while($loop)
438    {
439    if (defined($line=<$chld_out>)) { # we're reading in from child process' stdout
440        if((defined $blnShow) && $blnShow)
441        {
442        print STDERR "$line";
443        }
444       
445        if ($line =~ m/^Location:\s*(.*?)\s*\[following\]\s*$/i) {
446        my $follow_url = $1;
447        push(@follow_list,$follow_url);
448        }
449       
450        if ($line =~ m/ERROR\s+\d+/) {
451        $error_text .= $line;
452        }
453       
454        $full_text .= $line;
455    }
456    else { # wget finished, terminate naturally
457        #print STDERR "\nPerl: open2 command, input stream closed. Wget terminated naturally.\n";
458        close($chld_in);
459        close($chld_out);
460        # Program terminates only when the following line is included
461        # http://perldoc.perl.org/IPC/Open2.html explains why this is necessary
462        # it prevents the child from turning into a "zombie process".
463        # While the wget process terminates without it, this perl script does not:
464        # the DOS prompt is not returned without it.
465        waitpid $childpid, 0;
466        $loop = 0;
467       
468        $childpid = undef;
469        if(defined $port) {
470        $read_set->remove($serverSocket);
471        close($serverSocket);
472        }
473    }
474
475    # if we run this script from the command-line (as opposed to from GLI),
476    # then we're not working with sockets and can therefore can skip the next bits
477    next unless(defined $port);
478
479    # http://www.perlfect.com/articles/select.shtml
480    # "multiplex between several filehandles within a single thread of control,
481    # thus creating the effect of parallelism in the handling of I/O."
482    my @rh_set = $read_set->can_read(0.002); # every 2 ms check if there's a client socket connecting
483
484    # take all readable handles in turn
485    foreach my $rh (@rh_set) {
486        if($rh == $serverSocket) {
487        my $client = $rh->accept();
488        #$client->autoflush(1); # autoflush output buffer - don't put this back in: splits output irregularly over multilines
489        print $client "Talked to ServerSocket (port $port). Connection accepted\n";
490       
491        # Read from the client (getting rid of trailing newline)
492        # Has the client sent the <<STOP>> signal?
493        my $signal = <$client>;
494        chomp($signal);
495        if($signal eq "<<STOP>>") {
496            print $client "Perl received STOP signal (on port $port): stopping wget\n";
497            $loop = 0;                          # out of outer while loop
498            $self->{'forced_quit'} = 1;         # subclasses need to know we're quitting
499           
500            # Sometimes the wget process takes some time to start up. If the STOP signal
501            # was sent, don't try to terminate the process until we know it is running.
502            # Otherwise wget may start running after we tried to kill it. Wait 5 seconds
503            # for it to start up, checking for whether it is running in order to kill it.
504            for(my $seconds = 1; $seconds <= 5; $seconds++) {
505            if(kill(0, $childpid)) {
506                # If kill(0, $childpid) returns true, then the process is running
507                # and we need to kill it.
508                close($chld_in);
509                close($chld_out);
510                kill("TERM", $childpid);
511               
512                $childpid = undef;
513               
514                # Stop monitoring the read_handle and close the serverSocket
515                # (the Java end will close the client socket that Java opened)
516                $read_set->remove($rh);     #$read_set->remove($serverSocket);
517                close($rh);             #close($serverSocket);
518                print $client "Perl terminated wget and is about to exit\n";
519                last;                           # out of inner for loop
520            }
521            else { # the process may just be starting up, wait
522                sleep(1);
523            }
524            }
525            last;                               # out of foreach loop
526        }
527        }
528    }
529    }
530
531    my $command_status = $?;
532    if ($command_status != 0) {
533    $error_text .= "Exit error: $command_status";
534    }
535
536    if ($changed_dir) {
537    chdir $current_dir;
538    }
539   
540    my $final_follow = pop(@follow_list); # might be undefined, but that's OK
541   
542    return ($full_text,$error_text,$final_follow);
543}
544
545
546# TODO: Check if the URL is valid?? Not sure what should be in this function yet!!
547sub checkURL
548{
549    my ($self) = @_;
550    if ($self->{'url'} eq "")
551    {
552    &error("checkURL","no URL is specified!! Please specifies the URL for downloading.");
553    }
554}
555
556sub error
557{
558    my ($strFunctionName,$strError) = @_;
559    {
560    print "Error occoured in WgetDownload.pm\n".
561        "In Function:".$strFunctionName."\n".
562        "Error Message:".$strError."\n";
563    exit(-1);
564    }
565}
566
5671;
568
Note: See TracBrowser for help on using the browser.