Changeset 16791 for gsdl/trunk/perllib/downloaders/WgetDownload.pm

Timestamp:

2008-08-14T16:42:19+12:00 (15 years ago)

Author:

davidb

Message:

Improvement to downloading capabilities for WGet and OAI

File:

• gsdl/trunk/perllib/downloaders/WgetDownload.pm (modified) (3 diffs)

gsdl/trunk/perllib/downloaders/WgetDownload.pm

@@ -105 +105 @@
     my $strOptions = "";
     
-    if($self->{'proxy_on'} && $self->{'proxy_host'} && $self->{'proxy_port'})
+    if ($self->{'proxy_on'} && $self->{'proxy_host'} && $self->{'proxy_port'})
     {
 
@@ -116 +116 @@
     }
 
-    $strOptions .= " -Y on ";
+    if ($self->{'proxy_on'}) {
+    $strOptions .= " --proxy ";
+    }
 
     return $strOptions;
@@ -167 +169 @@
 }
 
+
+sub useWgetMonitored
+{
+    my ($self, $cmdWget,$blnShow, $working_dir) = @_;
+
+
+    my $current_dir = cwd();
+    my $changed_dir = 0;
+    if (defined $working_dir && -e $working_dir) {
+    chdir "$working_dir";
+    $changed_dir = 1;
+    }
+    my $wget_file_path = &util::filename_cat($ENV{'GSDLHOME'}, "bin", $ENV{'GSDLOS'}, "wget");
+    my $command = "\"$wget_file_path\" $cmdWget 2>&1 |";
+
+###    print STDERR "**** wget cmd = $command\n";
+
+    open(*WIN,$command) || die "wget request failed: $!\n";
+
+    my $full_text = "";
+    my $error_text = "";
+    my @follow_list = ();
+    my $line;
+
+    while (defined($line=<WIN>))
+    {
+    if((defined $blnShow) && $blnShow)
+    {
+        print STDERR "$line";
+    }
+
+    if ($line =~ m/^Location:\s*(.*?)\s*\[following\]\s*$/i) {
+        my $follow_url = $1;
+        push(@follow_list,$follow_url);
+    }
+
+    if ($line =~ m/ERROR\s+\d+/) {
+        $error_text .= $line;
+    }
+
+    $full_text .= $line;
+    }
+
+    close(WIN);
+
+    my $command_status = $?;
+    if ($command_status != 0) {
+    $error_text .= "Exit error: $command_status";
+    }
+
+    if ($changed_dir) {
+    chdir $current_dir;
+    }
+    
+    my $final_follow = pop(@follow_list); # might be undefined, but that's OK
+    
+    return ($full_text,$error_text,$final_follow);
+}
+
+
 # TODO: Check if the URL is valid?? Not sure what should be in this function yet!!
 sub checkURL