Changeset 25224

Timestamp:

2012-03-15T12:26:14+13:00 (12 years ago)

Author:

kjdon

Message:

removed default value for max_records option. If not specified, now this will download all records. Previously the only way to download all records was to set max_records to a bigger number than the number of records available. Also fixed a bug where if you didn't specify a place to download into, it would try to download into /path-based-on-url.

File:

• main/trunk/greenstone2/perllib/downloaders/OAIDownload.pm (modified) (14 diffs)

main/trunk/greenstone2/perllib/downloaders/OAIDownload.pm

@@ -76 +76 @@
     'desc' => "{OAIDownload.max_records}",
     'type' => "int",
-    'deft' => "500",
     'range' => "1,",
     'reqd' => "no"} ];
@@ -85 +84 @@
         'inherits' => "yes",
         'args'     => $arguments };
-
-##my $self;
-
-#### my $strWgetOptions="";
 
 sub new 
@@ -119 +114 @@
     &util::mk_dir($tmp_dir);
     }
-    
+
+    # if max_records not specified, parsing will have set it to ""
+    undef $self->{'max_records'} if $self->{'max_records'} eq "";
 
     # set up hashmap for individual items in get_doc_exts
@@ -126 +123 @@
     $self->{'lookup_exts'} = {};
     my $get_doc_exts = $self->{'get_doc_exts'};
-
+    
     if ((defined $get_doc_exts) && ($get_doc_exts ne "")) {
     my @exts = split(/,\s*/,$get_doc_exts);
@@ -148 +145 @@
     $strOutputDir = $hashGeneralOptions->{"cache_dir"};
     my $strBasURL = $self->{'url'};
-    my $intMaxRecords = $self->{'max_records'};
     my $blnDownloadDoc = $self->{'get_doc'};
 
@@ -163 +159 @@
     my $aryIDs = $self->parseOAIIDs($strIDs);
     my $intIDs = 0;
-    if($self->{'max_records'} < scalar(@$aryIDs))
+    if(defined $self->{'max_records'} && $self->{'max_records'} < scalar(@$aryIDs))
     {
     $intIDs = $self->{'max_records'};
@@ -173 +169 @@
     print STDERR "<<Total number of record(s):$intIDs>>\n";
 
-    $self->getOAIRecords($aryIDs, $strOutputDir, $strBasURL, $intMaxRecords, $blnDownloadDoc);
+    $self->getOAIRecords($aryIDs, $strOutputDir, $strBasURL, $self->{'max_records'}, $blnDownloadDoc);
 
 #    my $tmp_file = &util::filename_cat($ENV{'GSDLHOME'},"tmp","oai.tmp");
@@ -191 +187 @@
   
     print STDERR  "Gathering OAI identifiers.....\n";
-
+    
     my $metadata_prefix = $self->{'metadata_prefix'};
     $cmdWget .= " -q -O - \"$strBasURL?verb=ListIdentifiers&metadataPrefix=$metadata_prefix";
@@ -218 +214 @@
 
     $accumulated_strIDs = $strIDs;
-
+    my $max_recs = $self->{'max_records'};
     while ($strIDs =~ m/<resumptionToken.*?>\s*(.*?)\s*<\/resumptionToken>/s) { 
     # top up list with further requests for IDs
@@ -241 +237 @@
 
     my $num_acc_identifiers = scalar(@accumulated_identifiers);
-    if ($num_acc_identifiers > $self->{'max_records'}) {
+    if (defined  $max_recs && $num_acc_identifiers > $max_recs ) {
         last;
     }
@@ -440 +436 @@
 {
     my ($self,$aryIDs, $strOutputDir, $strBasURL, $intMaxRecords, $blnDownloadDoc) = @_;
-
     my $intDocCounter = 0;
 
@@ -463 +458 @@
         my $host =$self->{'url'}; 
   
-        $host =~ s/https?:\/\///g;
+        $host =~ s@https?:\/\/@@g;
 
         $host =~ s/:.*//g; 
 
-    my $strFileURL = "$strOutputDir/$host/$local_id.oai";
-
+    my $strFileURL = "";
+    if ($strOutputDir ne "") {
+        $strFileURL = "$strOutputDir/";
+    }
+    $strFileURL .= "$host/$local_id.oai";
 
     # prepare subdirectory for record (if needed)
@@ -490 +488 @@
     close(OAIOUT);
 
-        print STDERR "Saving records to $strFileURL\n";
+        print STDERR "Saving record to $strFileURL\n";
         print STDERR "<<Done>>\n";
     $intDocCounter ++;  
-    last if ($intDocCounter >= $intMaxRecords);
-    }
-
-    ($intDocCounter >= $intMaxRecords) ? 
+    last if (defined $intMaxRecords && $intDocCounter >= $intMaxRecords);
+    }
+
+    (defined $intMaxRecords && $intDocCounter >= $intMaxRecords) ? 
     print  STDERR "Reached maximum download records, use -max_records to set the maximum.\n": 
     print  STDERR "Complete download meta record from $strBasURL\n";
@@ -575 +573 @@
 ##  $self->{'parser'}->parse($xml_text);
     };
-    
+
     if ($@) {
     die "OAI: Parsed file $name is not a well formed XML file ($@)\n";