Changeset 14948 for gsdl/trunk/perllib/downloaders/OAIDownload.pm

Timestamp:

2008-01-24T22:07:55+13:00 (16 years ago)

Author:

davidb

Message:

Added support for resumptionToken

File:

• gsdl/trunk/perllib/downloaders/OAIDownload.pm (modified) (4 diffs)

gsdl/trunk/perllib/downloaders/OAIDownload.pm

@@ -121 +121 @@
     my ($hashGeneralOptions) = @_;
 
-    print STDERR "here2";
+##    print STDERR "here2";
    
     $strWgetOptions = $self->getWgetOptions();
@@ -175 +175 @@
     $cmdWget .= " -q -O - \"$strBasURL?verb=ListIdentifiers&metadataPrefix=$metadata_prefix";
 
-
     # if $set specified, add it in to URL
     my $set = $self->{'set'};
@@ -182 +181 @@
     $cmdWget .= "\" ";
 
- 
+    my $accumulated_strIDs = "";
     my $strIDs =  $self->useWget($cmdWget);
 
@@ -192 +191 @@
 
     print STDERR "<<Download Information>>\n";
-
+    
     $self->parse_xml($strIDs);
 
-    return $strIDs;
+    $accumulated_strIDs = $strIDs;
+
+    while ($strIDs =~ m/<resumptionToken.*?>(.*?)<\/resumptionToken>/s) {
+    # top up list with further requests for IDs
+
+    my $resumption_token = $1;
+
+    $cmdWget = $wgetOptions;
+
+    $cmdWget .= " -q -O - \"$strBasURL?verb=ListIdentifiers&resumptionToken=$resumption_token\"";
+
+    $strIDs =  $self->useWget($cmdWget);
+
+    $self->parse_xml($strIDs);
+
+    $accumulated_strIDs .= $strIDs;
+
+    my @accumulated_identifiers 
+        = ($accumulated_strIDs =~ m/<identifier>(.*?)<\/identifier>/sg);
+
+    my $num_acc_identifiers = scalar(@accumulated_identifiers);
+    if ($num_acc_identifiers > $self->{'max_records'}) {
+        last;
+    }
+    }
+
+    return $accumulated_strIDs;
 }