Changeset 20926 for gsdl

Timestamp:

2009-11-11T11:28:33+13:00 (14 years ago)

Author:

kjdon

Message:

if can't download a source doc for some reason, move on to the next one. Don't quit, otherwise won't get any more of the records downloaded. This way, at least one gets the records even if can't get the source docs. doc urls may have ' in them, convert to ' before trying to download

File:

• gsdl/trunk/perllib/downloaders/OAIDownload.pm (modified) (7 diffs)

gsdl/trunk/perllib/downloaders/OAIDownload.pm

@@ -295 +295 @@
     {
         my $doc_id_url = $2;
-
+        print STDERR "Found doc url: $doc_id_url\n";
         next if ($doc_id_url !~ m/^(https?|ftp):\/\//);
 
@@ -334 +334 @@
         my $wget_opts2 = $self->getWgetOptions();
         my $wget_cmd2 = "$wget_opts2 --convert-links -O \"$tmp_filename\" \"$doc_id_url\"";
-
         my ($stdout_and_err2,$error2,$follow2) =  $self->useWgetMonitored($wget_cmd2);
         return $strRecord if $self->{'forced_quit'};
@@ -340 +339 @@
         if($error2 ne "")
         {
-            print STDERR "Error occured while retrieving OAI source documents: $error2\n";
-            exit(-1);
+            print STDERR "Error occured while retrieving OAI source documents (1): $error2\n";
+            #exit(-1);
+            next; 
         }
 
@@ -386 +386 @@
         }
         else {
-            print STDERR "Error occurred while retrieving OAI source documents:\n";
+            print STDERR "Error occurred while retrieving OAI source documents (2):\n";
             print STDERR "$!\n";
         }
@@ -402 +402 @@
         my ($unused,$download_doc_file) = $self->dirFileSplit($download_doc_filename);
 
+        # may have ' in url - others??
+        my $safe_doc_id_url = $doc_id_url;
+        $safe_doc_id_url =~ s/'/\'/g;
+
         my $wget_opts = $self->getWgetOptions();
-        my $wget_cmd = "$wget_opts --convert-links -O \"$download_doc_filename\" \"$doc_id_url\"";
-
+        my $wget_cmd = "$wget_opts --convert-links -O \"$download_doc_filename\" \"$safe_doc_id_url\"";
+        
         my ($stdout_and_err,$errors,$follow) =  $self->useWgetMonitored($wget_cmd);
         return $strRecord if $self->{'forced_quit'};
@@ -410 +414 @@
         if($errors ne "")
         {
-        print STDERR "Error occured while retriving OAI souce documents:\n";
+        print STDERR "Error occured while retriving OAI souce documents (3):\n";
         print STDERR "$errors\n";
-        exit(-1);
+        #exit(-1);
+        next;
         }
 
         
-        $strRecord =~ s/<metadata>(.*?)<(dc:)?identifier>$orig_doc_id_url<\/(dc:)?identifier>(.*?)<\/metadata>/<metadata>$1<${2}identifier>$orig_doc_id_url<\/${2}identifier>\n   <gi.Sourcedoc>$download_doc_file<\/gi.Sourcedoc>$4<\/metadata>/s;
+        $strRecord =~ s/<metadata>(.*?)<((?:dc:)?identifier)>$orig_doc_id_url<\/((?:dc:)?identifier)>(.*?)<\/metadata>/<metadata>$1<${2}>$orig_doc_id_url<\/${2}>\n   <gi.Sourcedoc>$download_doc_file<\/gi.Sourcedoc>$4<\/metadata>/s;
     }
 
@@ -446 +451 @@
     my $wget_opts = $self->getWgetOptions();
     my $cmdWget= "$wget_opts -q -O - \"$strBasURL?verb=GetRecord&metadataPrefix=$metadata_prefix&identifier=$strID\"";
-
+    
     my $strRecord =  $self->useWget($cmdWget);