Changeset 16791

Timestamp:

2008-08-14T16:42:19+12:00 (16 years ago)

Author:

davidb

Message:

Improvement to downloading capabilities for WGet and OAI

Location:

gsdl/trunk/perllib/downloaders

Files:

• OAIDownload.pm (modified) (22 diffs)
• WgetDownload.pm (modified) (3 diffs)

gsdl/trunk/perllib/downloaders/OAIDownload.pm

@@ -66 +66 @@
     'type' => "flag",
     'reqd' => "no"},
+      { 'name' => "get_doc_exts",
+    'disp' => "{OAIDownload.get_doc_exts_disp}",
+    'desc' => "{OAIDownload.get_doc_exts}",
+    'type' => "string",
+    'deft' => "doc,pdf,ppt",
+    'reqd' => "no"},
       { 'name' => "max_records", 
     'disp' => "{OAIDownload.max_records_disp}",
@@ -80 +86 @@
         'args'     => $arguments };
 
-my $self;
-
-my $strWgetOptions="";
+##my $self;
+
+#### my $strWgetOptions="";
 
 sub new 
@@ -93 +99 @@
     if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
 
-    $self = (defined $hashArgOptLists)? new WgetDownload($getlist,$inputargs,$hashArgOptLists): new WgetDownload($getlist,$inputargs);
+    my $self = (defined $hashArgOptLists)? new WgetDownload($getlist,$inputargs,$hashArgOptLists): new WgetDownload($getlist,$inputargs);
 
     if ($self->{'info_only'}) {
@@ -101 +107 @@
 
     my $parser = new XML::Parser('Style' => 'Stream',
+                                 'PluginObj' => $self,
                  'Handlers' => {'Char' => \&Char,
                         'Start' => \&OAI_StartTag,
@@ -113 +120 @@
     }
     
+
+    # set up hashmap for individual items in get_doc_exts
+    # to make testing for matches easier
+
+    $self->{'lookup_exts'} = {};
+    my $get_doc_exts = $self->{'get_doc_exts'};
+
+    if ((defined $get_doc_exts) && ($get_doc_exts ne "")) {
+    my @exts = split(/,\s*/,$get_doc_exts);
+    foreach my $e (@exts) {
+        $self->{'lookup_exts'}->{lc($e)} = 1;
+    }
+    }
+
+
     return bless $self, $class;
 }
@@ -121 +143 @@
     my ($hashGeneralOptions) = @_;
 
-##    print STDERR "here2";
-   
-    $strWgetOptions = $self->getWgetOptions();
-    my $cmdWget = $strWgetOptions;
+##    my $cmdWget = $strWgetOptions;
   
     my $strOutputDir ="";
@@ -136 +155 @@
     my $strIDs = $self->getOAIIDs($strBasURL);
  
-   if($strIDs eq "")
-    {
-    print STDERR "Error: No ID being found\n";
+    if($strIDs eq "")
+    {
+    print STDERR "Error: No IDs found\n";
     return 0;
     }
+
     my $aryIDs = $self->parseOAIIDs($strIDs);
     my $intIDs = 0;
@@ -155 +175 @@
     $self->getOAIRecords($aryIDs, $strOutputDir, $strBasURL, $intMaxRecords, $blnDownloadDoc);
 
-    my $tmp_file = "$ENV{GSDLHOME}/tmp/oai.tmp";
-    &util::rm($tmp_file); 
+#    my $tmp_file = &util::filename_cat($ENV{'GSDLHOME'},"tmp","oai.tmp");
+#    &util::rm($tmp_file); 
 
     return 1;
@@ -164 +184 @@
 {
     my ($self,$strBasURL) = @_;
-    my ($cmdWget);
+##    my ($cmdWget);
      
     my $wgetOptions = $self->getWgetOptions();
 
-    $cmdWget = $wgetOptions;
+    my $cmdWget = $wgetOptions;
   
     print STDERR  "Gathering OAI identifiers.....\n";
@@ -256 +276 @@
 sub getOAIDoc
 {
-    my ($self,$strRecord, $strSubDirPath) = @_;
+    my ($self,$strRecord, $oai_rec_filename) = @_;
  
     print  STDERR "Gathering source documents.....\n";
@@ -264 +284 @@
     {
     my $strMetaTag = $1;
-    
-    if ($strMetaTag =~ m/<(dc:)?identifier>(.*?)<\/(dc:)?identifier>/s)
+    my $had_valid_url = 0;
+
+    while ($strMetaTag =~ s/<(dc:)?identifier>(.*?)<\/(dc:)?identifier>//is)
     {
-        my $strDocURL = $2;
-
-        my ($unused,$strDocFile) = $self->dirFileSplit($strDocURL);
-
-            my $strSoureDirPath ="";
-
-        $strSoureDirPath = &util::filename_cat($strSubDirPath,"srcdocs");
-
-        &util::mk_dir($strSoureDirPath)  if (!-e "$strSoureDirPath");
-        
-        my $strFullDocFilePath = &util::filename_cat($strSoureDirPath,$strDocFile);
-        
-        my $wget_cmd = $strWgetOptions." -q -O \"$strFullDocFilePath\" \"$strDocURL\"";
-
-        my $strResponse =  $self->useWget($wget_cmd,1);
-
-        if($strResponse ne "")
+        my $doc_id_url = $2;
+
+        next if ($doc_id_url !~ m/^(https?|ftp):\/\//);
+
+        my $orig_doc_id_url = $doc_id_url;
+        $had_valid_url = 1;
+
+        my ($doc_dir_url_prefix,$doc_id_tail) = ($doc_id_url =~ m/^(.*)\/(.*?)$/);
+        my $faked_ext = 0;
+        my $primary_doc_match = 0;
+
+        my ($id_file_ext) = ($doc_id_tail =~ m/\.([^\.]+)$/);
+
+        if (defined $id_file_ext) {
+        # cross-check this filename extension with get_doc_exts option
+        # if provided
+        my $lookup_exts = $self->{'lookup_exts'};
+
+        if (defined $lookup_exts->{lc($id_file_ext)}) {
+            # this initial URL matches requirement
+            $primary_doc_match = 1;
+        }
+        }
+        else {
+        $faked_ext = 1;
+        $id_file_ext = "html";
+        }
+
+        
+        if ((!$primary_doc_match) && ($id_file_ext =~ m/^html?$/i)) {
+        # Download this doc if HTML, scan through it looking for a link
+        # that does match get_doc_exts
+        
+
+        # 1. Generate a tmp name
+        my $tmp_filename = &util::get_tmp_filename();
+
+        # 2. Download it
+        my $wget_opts2 = $self->getWgetOptions();
+        my $wget_cmd2 = "$wget_opts2 --convert-links -O \"$tmp_filename\" \"$doc_id_url\"";
+
+        my ($stdout_and_err2,$error2,$follow2) =  $self->useWgetMonitored($wget_cmd2);
+
+        if($error2 ne "")
+        {
+            print STDERR "Error occured while retrieving OAI source documents: $error2\n";
+            exit(-1);
+        }
+
+        if (defined $follow2) {
+            # src url was "redirected" to another place
+            # => pick up on this and make it the new doc_id_url
+            $doc_id_url = $follow2;
+        }
+
+        my $primary_doc_html = "";
+        if (open(HIN,"<$tmp_filename")) {
+            my $line;
+            while (defined ($line = <HIN>)) {
+            $primary_doc_html .= $line;
+            }
+            close(HIN);
+
+            # 3. Scan through it looking for match
+            # 
+            # if got match, change $doc_id_url to this new URL and
+            # $id_file_ext to 'match' 
+            
+            my @href_links = ($primary_doc_html =~ m/href="(.*?)"/gsi);
+
+            my $lookup_exts = $self->{'lookup_exts'};
+
+            foreach my $href (@href_links) {
+            my ($ext) = ($href =~ m/\.([^\.]+)$/);
+
+            if ((defined $ext) && (defined $lookup_exts->{$ext})) {
+
+                if ($href !~ m/^(https?|ftp):\/\//) {
+                # link is within current site
+                my ($site_domain) = ($doc_id_url =~ m/^((?:https?|ftp):\/\/.*?)\//);
+
+                $href = "$site_domain$href";
+                }
+
+                $doc_id_url = $href;
+                $id_file_ext = $ext;
+                last;
+            }
+            }
+        }
+        else {
+            print STDERR "Error occurred while retrieving OAI source documents:\n";
+            print STDERR "$!\n";
+        }
+
+        if (-e $tmp_filename) {
+            &util::rm($tmp_filename); 
+        }
+        }
+
+        my $download_doc_filename = $oai_rec_filename;
+        $download_doc_filename =~ s/\.oai$/\.$id_file_ext/;
+
+        my ($unused,$download_doc_file) = $self->dirFileSplit($download_doc_filename);
+
+        my $wget_opts = $self->getWgetOptions();
+        my $wget_cmd = "$wget_opts --convert-links -O \"$download_doc_filename\" \"$doc_id_url\"";
+
+        my ($stdout_and_err,$errors,$follow) =  $self->useWgetMonitored($wget_cmd);
+
+        if($errors ne "")
         {
-        print STDERR "Error occured while retriving OAI souce documents: $strResponse\n";
+        print STDERR "Error occured while retriving OAI souce documents:\n";
+        print STDERR "$errors\n";
         exit(-1);
         }
 
-        $strRecord =~ s/<metadata>(.*?)<(dc:)?identifier>$strDocURL<\/(dc:)?identifier>(.*?)<\/metadata>/<metadata>$1<OrigURL>$strDocURL<\/OrigURL>\n   <identifier>srcdocs\/$strDocFile<\/identifier>$4<\/metadata>/s;
+        
+        $strRecord =~ s/<metadata>(.*?)<(dc:)?identifier>$orig_doc_id_url<\/(dc:)?identifier>(.*?)<\/metadata>/<metadata>$1<${2}identifier>$orig_doc_id_url<\/${2}identifier>\n   <gi.Sourcedoc>$download_doc_file<\/gi.Sourcedoc>$4<\/metadata>/s;
     }
-    else
+
+    if (!$had_valid_url)
     {
         print  STDERR "\tNo souce document URL is specified in the OAI record (No (dc:)?identifier is provided)\n";
@@ -300 +418 @@
     print  STDERR "\tNo souce document URL is specified in the OAI record (No metadata field is provided)\n";
     }
-    
+
+    return $strRecord;
 }
 
@@ -313 +432 @@
     foreach my $strID ( @$aryIDs)
     {
-    print  STDERR "Gathering OAI record with ID:$strID.....\n";
+    print  STDERR "Gathering OAI record with ID $strID.....\n";
        
-    my $cmdWget= $strWgetOptions." -q -O - \"$strBasURL?verb=GetRecord&metadataPrefix=$metadata_prefix&identifier=$strID\"";
+    my $wget_opts = $self->getWgetOptions();
+    my $cmdWget= "$wget_opts -q -O - \"$strBasURL?verb=GetRecord&metadataPrefix=$metadata_prefix&identifier=$strID\"";
 
     my $strRecord =  $self->useWget($cmdWget);
 
-        
-        my @fileDirs = split(":",$strID);  
+    my @fileDirs = split(":",$strID);  
+    my $local_id = pop @fileDirs;
 
     # setup directories
@@ -328 +448 @@
         my $host =$self->{'url'}; 
   
-        $host =~ s/http:\/\///g;
+        $host =~ s/https?:\/\///g;
 
         $host =~ s/:.*//g; 
 
-    my $midDir = join ("/",@fileDirs);
-    my $strFileURL = "$strOutputDir/$host/".$midDir.".oai";
+    my $strFileURL = "$strOutputDir/$host/$local_id.oai";
+
 
     # prepare subdirectory for record (if needed)
@@ -346 +466 @@
     if($blnDownloadDoc)
     {
-        $self->getOAIDoc($strRecord,$strSubDirPath);
+        $strRecord = $self->getOAIDoc($strRecord,$strFileURL);
     }
 
@@ -378 +498 @@
     my $strIdentify = "verb=Identify";
     my $strListSets = "verb=ListSets";
+    my $strListMdFormats = "verb=ListMetadataFormats";
 
     my $strIdentifyCMD = $strBaseCMD;
@@ -392 +513 @@
     print STDERR "General information:\n";
     $self->parse_xml($strIdentifyText);
+    print STDERR "\n";
+
+    print STDERR "=" x 10, "\n";
+    print STDERR "Metadata Format Information (metadataPrefix):\n";
+    print STDERR "=" x 10, "\n";
+
+    my $strListMdFormatsCMD = $strBaseCMD;
+    $strListMdFormatsCMD =~ s/_OPTS_/$strListMdFormats/;    
+    my $strListMdFormatsText = $self->useWget($strListMdFormatsCMD);
+
+    $self->parse_xml($strListMdFormatsText);
+    print STDERR "\n";
+
+    print STDERR "=" x 10, "\n";
+    print STDERR "List Information:\n";
+    print STDERR "=" x 10, "\n";
 
     my $strListSetCMD = $strBaseCMD;
@@ -397 +534 @@
     my $strListSetsText = $self->useWget($strListSetCMD);
 
-
-    print STDERR "List Information:\n";
     $self->parse_xml($strListSetsText);
 }
@@ -405 +540 @@
 {    
     my ($self) = shift (@_);
-    my ($strOutputText) = @_;
+    my ($xml_text) = @_;
    
+    #### change this to work directly from $xml_text
+
     #Open a temporary file to store OAI information, and store the information to the temp file
-    my $name = "$ENV{GSDLHOME}/tmp/oai.tmp"; 
+    my $name = &util::filename_cat($ENV{GSDLHOME},"tmp","oai.tmp"); 
 
     open(*OAIOUT,"> $name");
     
-    print OAIOUT $strOutputText;
+    print OAIOUT $xml_text;
     close(OAIOUT);
 
     $self->{'temp_file_name'} = $name;
+
+##    print STDERR "**** xml text = $xml_text\n";
 
     eval {
     $self->{'parser'}->parsefile("$name");
+##  $self->{'parser'}->parse($xml_text);
     };
     
     if ($@) {
-    die "OAI: $name is not a well formed XML file ($@)\n";
-    }
-}
-
-END{
-    if($self->{'info'})
-    {
-    unlink($self->{'temp_file_name'}) or die "Could not unlink $self->{'temp_file_name'}: $!";
-    }
-}
+    die "OAI: Parsed file $name is not a well formed XML file ($@)\n";
+##  die "OAI: Parsed text is not a well formed XML file ($@)\n";
+    }
+
+    unlink($self->{'temp_file_name'}) or die "Could not unlink $self->{'temp_file_name'}: $!";
+}
+
+####END 
+#{
+#    if($self->{'info'})
+#    {
+#   unlink($self->{'temp_file_name'}) or die "Could not unlink $self->{'temp_file_name'}: $!";
+#    }
+#}
 
 # This Char function overrides the one in XML::Parser::Stream to overcome a
@@ -439 +583 @@
     use bytes;  # Necessary to prevent encoding issues with XML::Parser 2.31+
     $_[0]->{'Text'} .= $_[1];
+
+    my $self = $_[0]->{'PluginObj'};
     if ((defined $self->{'subfield'} && ($self->{'subfield'} ne ""))) {
     $self->{'text'} .= $_[1];
@@ -454 +600 @@
     my ($expat, $element, %attr) = @_;
 
+    my $self = $expat->{'PluginObj'};
     $self->{'subfield'} = $element;
     
@@ -461 +608 @@
 {
     my ($expat, $element) = @_;
+
+    my $self = $expat->{'PluginObj'};
     $self->{'text'} = "";
     $self->{'subfield'} = "";

gsdl/trunk/perllib/downloaders/WgetDownload.pm

@@ -105 +105 @@
     my $strOptions = "";
     
-    if($self->{'proxy_on'} && $self->{'proxy_host'} && $self->{'proxy_port'})
+    if ($self->{'proxy_on'} && $self->{'proxy_host'} && $self->{'proxy_port'})
     {
 
@@ -116 +116 @@
     }
 
-    $strOptions .= " -Y on ";
+    if ($self->{'proxy_on'}) {
+    $strOptions .= " --proxy ";
+    }
 
     return $strOptions;
@@ -167 +169 @@
 }
 
+
+sub useWgetMonitored
+{
+    my ($self, $cmdWget,$blnShow, $working_dir) = @_;
+
+
+    my $current_dir = cwd();
+    my $changed_dir = 0;
+    if (defined $working_dir && -e $working_dir) {
+    chdir "$working_dir";
+    $changed_dir = 1;
+    }
+    my $wget_file_path = &util::filename_cat($ENV{'GSDLHOME'}, "bin", $ENV{'GSDLOS'}, "wget");
+    my $command = "\"$wget_file_path\" $cmdWget 2>&1 |";
+
+###    print STDERR "**** wget cmd = $command\n";
+
+    open(*WIN,$command) || die "wget request failed: $!\n";
+
+    my $full_text = "";
+    my $error_text = "";
+    my @follow_list = ();
+    my $line;
+
+    while (defined($line=<WIN>))
+    {
+    if((defined $blnShow) && $blnShow)
+    {
+        print STDERR "$line";
+    }
+
+    if ($line =~ m/^Location:\s*(.*?)\s*\[following\]\s*$/i) {
+        my $follow_url = $1;
+        push(@follow_list,$follow_url);
+    }
+
+    if ($line =~ m/ERROR\s+\d+/) {
+        $error_text .= $line;
+    }
+
+    $full_text .= $line;
+    }
+
+    close(WIN);
+
+    my $command_status = $?;
+    if ($command_status != 0) {
+    $error_text .= "Exit error: $command_status";
+    }
+
+    if ($changed_dir) {
+    chdir $current_dir;
+    }
+    
+    my $final_follow = pop(@follow_list); # might be undefined, but that's OK
+    
+    return ($full_text,$error_text,$final_follow);
+}
+
+
 # TODO: Check if the URL is valid?? Not sure what should be in this function yet!!
 sub checkURL