Changeset 14948

Show
Ignore:
Timestamp:
24.01.2008 22:07:55 (12 years ago)
Author:
davidb
Message:

Added support for resumptionToken

Files:
1 modified

Legend:

Unmodified
Added
Removed
  • gsdl/trunk/perllib/downloaders/OAIDownload.pm

    r14941 r14948  
    121121    my ($hashGeneralOptions) = @_; 
    122122 
    123     print STDERR "here2"; 
     123##    print STDERR "here2"; 
    124124    
    125125    $strWgetOptions = $self->getWgetOptions(); 
     
    175175    $cmdWget .= " -q -O - \"$strBasURL?verb=ListIdentifiers&metadataPrefix=$metadata_prefix"; 
    176176 
    177  
    178177    # if $set specified, add it in to URL 
    179178    my $set = $self->{'set'}; 
     
    182181    $cmdWget .= "\" "; 
    183182 
    184   
     183    my $accumulated_strIDs = ""; 
    185184    my $strIDs =  $self->useWget($cmdWget); 
    186185 
     
    192191 
    193192    print STDERR "<<Download Information>>\n"; 
    194  
     193     
    195194    $self->parse_xml($strIDs); 
    196195 
    197     return $strIDs; 
     196    $accumulated_strIDs = $strIDs; 
     197 
     198    while ($strIDs =~ m/<resumptionToken.*?>(.*?)<\/resumptionToken>/s) { 
     199    # top up list with further requests for IDs 
     200 
     201    my $resumption_token = $1; 
     202 
     203    $cmdWget = $wgetOptions; 
     204 
     205    $cmdWget .= " -q -O - \"$strBasURL?verb=ListIdentifiers&resumptionToken=$resumption_token\""; 
     206 
     207    $strIDs =  $self->useWget($cmdWget); 
     208 
     209    $self->parse_xml($strIDs); 
     210 
     211    $accumulated_strIDs .= $strIDs; 
     212 
     213    my @accumulated_identifiers  
     214        = ($accumulated_strIDs =~ m/<identifier>(.*?)<\/identifier>/sg); 
     215 
     216    my $num_acc_identifiers = scalar(@accumulated_identifiers); 
     217    if ($num_acc_identifiers > $self->{'max_records'}) { 
     218        last; 
     219    } 
     220    } 
     221 
     222    return $accumulated_strIDs; 
    198223} 
    199224