Changeset 12465 for trunk/gsdl/perllib/downloaders/OAIDownload.pm

Timestamp:

2006-08-18T09:24:29+12:00 (18 years ago)

Author:

shaoqun

Message:

fixed th bugs on windows

File:

• trunk/gsdl/perllib/downloaders/OAIDownload.pm (modified) (20 diffs)

trunk/gsdl/perllib/downloaders/OAIDownload.pm

@@ -37 +37 @@
 use XMLParser;
 
-use IO::File;
 use POSIX qw(tmpnam);
+use util;
 
 sub BEGIN {
@@ -56 +56 @@
     'reqd' => "no"},
       { 'name' => "get_doc",
-    'disp' => "{OAIDownload.qet_doc_disp}",
+    'disp' => "{OAIDownload.get_doc_disp}",
     'desc' => "{OAIDownload.get_doc}",
     'type' => "flag",
@@ -76 +76 @@
 my $self;
 
+my $strWgetOptions="";
+
 sub new 
 {
@@ -107 +109 @@
     my ($hashGeneralOptions) = @_;
 
-    # Checking if the wget has been well setup
-    # &WgetDownload::checkWgetSetup($self,$hashGeneralOptions->{'gli_call'});
-    
-    my $strOutputDir = $hashGeneralOptions->{"cache_dir"};
+    print STDERR "here2";
+   
+    $strWgetOptions = $self->getWgetOptions();
+    my $cmdWget = $strWgetOptions;
+  
+    my $strOutputDir ="";
+    $strOutputDir = $hashGeneralOptions->{"cache_dir"};
     my $strBasURL = $self->{'url'};
     my $intMaxRecords = $self->{'max_records'};
@@ -116 +121 @@
 
     print STDERR "<<Defined Maximum>>\n";
-    my $strIDs = &getOAIIDs($self,$strBasURL);
-    if($strIDs eq "")
+
+    my $strIDs = $self->getOAIIDs($strBasURL);
+ 
+   if($strIDs eq "")
     {
     print STDERR "Error: No ID being found\n";
     return 0;
     }
-    my $aryIDs = &parseOAIIDs($strIDs);
+    my $aryIDs = $self->parseOAIIDs($strIDs);
     my $intIDs = 0;
     if($self->{'max_records'} < scalar(@$aryIDs))
@@ -134 +141 @@
     print STDERR "<<Total number of record(s):$intIDs>>\n";
 
-    &getOAIRecords($aryIDs, $strOutputDir, $strBasURL, $intMaxRecords, $blnDownloadDoc);
+    $self->getOAIRecords($aryIDs, $strOutputDir, $strBasURL, $intMaxRecords, $blnDownloadDoc);
+
+    my $tmp_file = "$ENV{GSDLHOME}/tmp/oai.tmp";
+    &util::rm($tmp_file); 
 
     return 1;
@@ -143 +153 @@
     my ($self,$strBasURL) = @_;
     my ($cmdWget);
+     
+    my $wgetOptions = $self->getWgetOptions();
+
+    $cmdWget = $wgetOptions;
+  
     print STDERR  "Gathering OAI identifiers.....\n";
+
     if($self->{'set'} ne "")
     {
-    $cmdWget = "-q -O - \"$strBasURL?verb=ListIdentifiers&metadataPrefix=oai_dc&set=$self->{'set'}\" ";
+    $cmdWget .= " -q -O - \"$strBasURL?verb=ListIdentifiers&metadataPrefix=oai_dc&set=$self->{'set'}\" ";
     }
     else
     {
-    $cmdWget = "-q -O - \"$strBasURL?verb=ListIdentifiers&metadataPrefix=oai_dc\" ";
-    }
-    my $strIDs =  &WgetDownload::useWget($cmdWget);
+    $cmdWget .= " -q -O - \"$strBasURL?verb=ListIdentifiers&metadataPrefix=oai_dc\" ";
+    }
+
+ 
+    my $strIDs =  $self->useWget($cmdWget);
+
+    if (!defined $strIDs or $strIDs eq ""  ){
+    print STDERR "Server information is unavailable.\n";
+    print STDERR "<<Finished>>\n";
+        return;  
+    }
+
+    print STDERR "<<Download Information>>\n";
+
+    $self->parse_xml($strIDs);
+
     return $strIDs;
 }
@@ -158 +187 @@
 sub parseOAIIDs
 {   
-    my ($strIDs) = @_;
+    my ($self,$strIDs) = @_;
 
     print STDERR "Parsing OAI identifiers.....\n";
@@ -177 +206 @@
 sub dirFileSplit
 {
-    my ($strFile) = @_;
-
-    my @aryDirs = split("/",$strFile);
+    my ($self,$strFile) = @_;
+
+    my @aryDirs = split("[/\]",$strFile);
+   
     my $strLocalFile = pop(@aryDirs);
     my $strSubDirs = join("/",@aryDirs);
@@ -188 +218 @@
 sub getOAIDoc
 {
-    my ($strRecord, $strSubDirPath) = @_;
-    
+    my ($self,$strRecord, $strSubDirPath) = @_;
+ 
     print  STDERR "Gathering source documents.....\n";
     # look out for identifier tag in metadata section
+   
     if ($strRecord =~ m/<metadata>(.*)<\/metadata>/s)
     {
@@ -200 +231 @@
         my $strDocURL = $2;
 
-        my ($unused,$strDocFile) = dirFileSplit($strDocURL);
-
-        my $strSoureDirPath = &util::filename_cat($strSubDirPath,"srcdocs");
+        my ($unused,$strDocFile) = $self->dirFileSplit($strDocURL);
+
+            my $strSoureDirPath ="";
+
+        $strSoureDirPath = &util::filename_cat($strSubDirPath,"srcdocs");
+
         &util::mk_dir($strSoureDirPath)  if (!-e "$strSoureDirPath");
         
         my $strFullDocFilePath = &util::filename_cat($strSoureDirPath,$strDocFile);
         
-        my $wget_cmd = "-q -O $strFullDocFilePath \"$strDocURL\"";
-
-        my $strResponse =  &WgetDownload::useWget($wget_cmd,1);
+        my $wget_cmd = $strWgetOptions." -q -O $strFullDocFilePath \"$strDocURL\"";
+
+        my $strResponse =  $self->useWget($wget_cmd,1);
 
         if($strResponse ne "")
@@ -233 +267 @@
 sub getOAIRecords
 {
-    my ($aryIDs, $strOutputDir, $strBasURL, $intMaxRecords, $blnDownloadDoc) = @_;
+    my ($self,$aryIDs, $strOutputDir, $strBasURL, $intMaxRecords, $blnDownloadDoc) = @_;
 
     my $intDocCounter = 0;
@@ -240 +274 @@
     {
     print  STDERR "Gathering OAI record with ID:$strID.....\n";
-    # wget it;
-    my $cmdWget= "-q -O - \"$strBasURL?verb=GetRecord&metadataPrefix=oai_dc&identifier=$strID\"";
-    my $strRecord =  &WgetDownload::useWget($cmdWget);
+       
+    my $cmdWget= $strWgetOptions." -q -O - \"$strBasURL?verb=GetRecord&metadataPrefix=oai_dc&identifier=$strID\"";
+
+    my $strRecord =  $self->useWget($cmdWget);
+
+        
+        my @fileDirs = split(":",$strID);  
 
     # setup directories
-    my $strFileURL = "$strOutputDir/$strID.oai";
-    $strFileURL =~ s/:/\//g;
-
+
+        $strOutputDir  =~ s/"//g;
+  
+    my $strFileURL = "$strOutputDir/$fileDirs[0]/$fileDirs[1].oai";
+    
     # prepare subdirectory for record (if needed)
-    my ($strSubDirPath,$unused) = dirFileSplit($strFileURL);
+    my ($strSubDirPath,$unused) = ("", "");
+
+        ($strSubDirPath,$unused) = $self->dirFileSplit($strFileURL);
+   
     &util::mk_all_dir($strSubDirPath);
 
     my $ds = &util::get_dirsep();
-    my $strOutputFile = &util::filename_cat($strOutputDir,"$strID.oai");
-    $strOutputFile =~ s/:/$ds/g;
-
+    
     if($blnDownloadDoc)
     {
-        &getOAIDoc($strRecord,$strSubDirPath);
+        $self->getOAIDoc($strRecord,$strSubDirPath);
     }
 
     # save record 
-    open (OAIOUT,">$strOutputFile")
+    open (OAIOUT,">$strFileURL")
         || die "Unable to save oai metadata record: $!\n";
     print OAIOUT $strRecord;
     close(OAIOUT);
 
-    $intDocCounter ++;
-    print STDERR "<<Done>>\n";
+        print STDERR "Saving records to $strFileURL\n";
+        print STDERR "<<Done>>\n";
+    $intDocCounter ++;  
     last if ($intDocCounter >= $intMaxRecords);
     }
+
     ($intDocCounter >= $intMaxRecords) ? 
     print  STDERR "Reach maximum download records, use -max_records to set the maximum.\n": 
     print  STDERR "Complete download meta record from $strBasURL\n";
 
+       print STDERR "<<Finished>>\n";
 }
 
@@ -282 +326 @@
     if(!defined $self){ die "System Error: No \$self defined for url_information in OAIDownload\n";}
     
-    my $strBaseCMD = "-q -O - \"$self->{'url'}?_OPTS_\"";
+    my $wgetOptions = $self->getWgetOptions();
+    my $strBaseCMD = $wgetOptions." -q -O - \"$self->{'url'}?_OPTS_\"";
  
     my $strIdentify = "verb=Identify";
@@ -290 +335 @@
     $strIdentifyCMD =~ s/_OPTS_/$strIdentify/;  
 
-    my $strIdentifyText = &WgetDownload::useWget($strIdentifyCMD);
+    my $strIdentifyText = $self->useWget($strIdentifyCMD);
+
+     if (!defined $strIdentifyText or $strIdentifyText eq ""  ){
+    print STDERR "Server information is unavailable.\n";
+    print STDERR "<<Finished>>\n";
+        return;  
+    }
 
     print STDERR "General information:\n";
@@ -297 +348 @@
     my $strListSetCMD = $strBaseCMD;
     $strListSetCMD =~ s/_OPTS_/$strListSets/;   
-    my $strListSetsText = &WgetDownload::useWget($strListSetCMD);
+    my $strListSetsText = $self->useWget($strListSetCMD);
+
+
     print STDERR "List Information:\n";
     $self->parse_xml($strListSetsText);
@@ -306 +359 @@
     my ($self) = shift (@_);
     my ($strOutputText) = @_;
-    my ($name,$fh);
-    
+   
     #Open a temporary file to store OAI information, and store the information to the temp file
-    do {$name = tmpnam()}
-    until $fh = IO::File->new($name, O_RDWR|O_CREAT|O_EXCL);
-    print $fh $strOutputText;
-    close($fh);
+    my $name = "$ENV{GSDLHOME}/tmp/oai.tmp"; 
+
+    open(*OAIOUT,"> $name");
+    
+    print OAIOUT $strOutputText;
+    close(OAIOUT);
 
     $self->{'temp_file_name'} = $name;
@@ -340 +394 @@
     if ((defined $self->{'subfield'} && ($self->{'subfield'} ne ""))) {
     $self->{'text'} .= $_[1];
-    $self->{'text'} =~ s/[\n]|[" "]//g;
+    $self->{'text'} =~ s/[\n]|([ ]{2,})//g;
     if($self->{'text'} ne "")
     {       
@@ -352 +406 @@
 {
     my ($expat, $element, %attr) = @_;
+
     $self->{'subfield'} = $element;
+    
 }
 
@@ -364 +420 @@
 sub error
 {
-    my ($strFunctionName,$strError) = @_;
+    my ($self,$strFunctionName,$strError) = @_;
     {
     print "Error occoured in OAIDownload.pm\n".