Changeset 12465

Timestamp:

2006-08-18T09:24:29+12:00 (18 years ago)

Author:

shaoqun

Message:

fixed th bugs on windows

Location:

trunk/gsdl/perllib

Files:

• download.pm (modified) (1 diff)
• downloaders/OAIDownload.pm (modified) (20 diffs)
• downloaders/SRWDownload.pm (modified) (5 diffs)
• downloaders/WebDownload.pm (modified) (6 diffs)
• downloaders/WgetDownload.pm (modified) (3 diffs)
• downloaders/Z3950Download.pm (modified) (6 diffs)

trunk/gsdl/perllib/download.pm

@@ -36 +36 @@
 sub load_download {
     my ($download_name,$download_options) = @_;
+
     my ($download_obj);
 
-    my $coldownloadname = &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, 
+    my $coldownloadname ="";
+
+    if ($ENV{'GSDLCOLLECTDIR'}){
+   
+    $coldownloadname = &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, 
                           "perllib/downloaders", 
                           "${download_name}.pm");
+
+   }
+   
     my $maindownloadname = &util::filename_cat($ENV{'GSDLHOME'},
                            "perllib/downloaders", 
                            "${download_name}.pm");
+
     if (-e $coldownloadname) { require $coldownloadname;}
     elsif (-e $maindownloadname ) { require $maindownloadname; }

trunk/gsdl/perllib/downloaders/OAIDownload.pm

@@ -37 +37 @@
 use XMLParser;
 
-use IO::File;
 use POSIX qw(tmpnam);
+use util;
 
 sub BEGIN {
@@ -56 +56 @@
     'reqd' => "no"},
       { 'name' => "get_doc",
-    'disp' => "{OAIDownload.qet_doc_disp}",
+    'disp' => "{OAIDownload.get_doc_disp}",
     'desc' => "{OAIDownload.get_doc}",
     'type' => "flag",
@@ -76 +76 @@
 my $self;
 
+my $strWgetOptions="";
+
 sub new 
 {
@@ -107 +109 @@
     my ($hashGeneralOptions) = @_;
 
-    # Checking if the wget has been well setup
-    # &WgetDownload::checkWgetSetup($self,$hashGeneralOptions->{'gli_call'});
-    
-    my $strOutputDir = $hashGeneralOptions->{"cache_dir"};
+    print STDERR "here2";
+   
+    $strWgetOptions = $self->getWgetOptions();
+    my $cmdWget = $strWgetOptions;
+  
+    my $strOutputDir ="";
+    $strOutputDir = $hashGeneralOptions->{"cache_dir"};
     my $strBasURL = $self->{'url'};
     my $intMaxRecords = $self->{'max_records'};
@@ -116 +121 @@
 
     print STDERR "<<Defined Maximum>>\n";
-    my $strIDs = &getOAIIDs($self,$strBasURL);
-    if($strIDs eq "")
+
+    my $strIDs = $self->getOAIIDs($strBasURL);
+ 
+   if($strIDs eq "")
     {
     print STDERR "Error: No ID being found\n";
     return 0;
     }
-    my $aryIDs = &parseOAIIDs($strIDs);
+    my $aryIDs = $self->parseOAIIDs($strIDs);
     my $intIDs = 0;
     if($self->{'max_records'} < scalar(@$aryIDs))
@@ -134 +141 @@
     print STDERR "<<Total number of record(s):$intIDs>>\n";
 
-    &getOAIRecords($aryIDs, $strOutputDir, $strBasURL, $intMaxRecords, $blnDownloadDoc);
+    $self->getOAIRecords($aryIDs, $strOutputDir, $strBasURL, $intMaxRecords, $blnDownloadDoc);
+
+    my $tmp_file = "$ENV{GSDLHOME}/tmp/oai.tmp";
+    &util::rm($tmp_file); 
 
     return 1;
@@ -143 +153 @@
     my ($self,$strBasURL) = @_;
     my ($cmdWget);
+     
+    my $wgetOptions = $self->getWgetOptions();
+
+    $cmdWget = $wgetOptions;
+  
     print STDERR  "Gathering OAI identifiers.....\n";
+
     if($self->{'set'} ne "")
     {
-    $cmdWget = "-q -O - \"$strBasURL?verb=ListIdentifiers&metadataPrefix=oai_dc&set=$self->{'set'}\" ";
+    $cmdWget .= " -q -O - \"$strBasURL?verb=ListIdentifiers&metadataPrefix=oai_dc&set=$self->{'set'}\" ";
     }
     else
     {
-    $cmdWget = "-q -O - \"$strBasURL?verb=ListIdentifiers&metadataPrefix=oai_dc\" ";
-    }
-    my $strIDs =  &WgetDownload::useWget($cmdWget);
+    $cmdWget .= " -q -O - \"$strBasURL?verb=ListIdentifiers&metadataPrefix=oai_dc\" ";
+    }
+
+ 
+    my $strIDs =  $self->useWget($cmdWget);
+
+    if (!defined $strIDs or $strIDs eq ""  ){
+    print STDERR "Server information is unavailable.\n";
+    print STDERR "<<Finished>>\n";
+        return;  
+    }
+
+    print STDERR "<<Download Information>>\n";
+
+    $self->parse_xml($strIDs);
+
     return $strIDs;
 }
@@ -158 +187 @@
 sub parseOAIIDs
 {   
-    my ($strIDs) = @_;
+    my ($self,$strIDs) = @_;
 
     print STDERR "Parsing OAI identifiers.....\n";
@@ -177 +206 @@
 sub dirFileSplit
 {
-    my ($strFile) = @_;
-
-    my @aryDirs = split("/",$strFile);
+    my ($self,$strFile) = @_;
+
+    my @aryDirs = split("[/\]",$strFile);
+   
     my $strLocalFile = pop(@aryDirs);
     my $strSubDirs = join("/",@aryDirs);
@@ -188 +218 @@
 sub getOAIDoc
 {
-    my ($strRecord, $strSubDirPath) = @_;
-    
+    my ($self,$strRecord, $strSubDirPath) = @_;
+ 
     print  STDERR "Gathering source documents.....\n";
     # look out for identifier tag in metadata section
+   
     if ($strRecord =~ m/<metadata>(.*)<\/metadata>/s)
     {
@@ -200 +231 @@
         my $strDocURL = $2;
 
-        my ($unused,$strDocFile) = dirFileSplit($strDocURL);
-
-        my $strSoureDirPath = &util::filename_cat($strSubDirPath,"srcdocs");
+        my ($unused,$strDocFile) = $self->dirFileSplit($strDocURL);
+
+            my $strSoureDirPath ="";
+
+        $strSoureDirPath = &util::filename_cat($strSubDirPath,"srcdocs");
+
         &util::mk_dir($strSoureDirPath)  if (!-e "$strSoureDirPath");
         
         my $strFullDocFilePath = &util::filename_cat($strSoureDirPath,$strDocFile);
         
-        my $wget_cmd = "-q -O $strFullDocFilePath \"$strDocURL\"";
-
-        my $strResponse =  &WgetDownload::useWget($wget_cmd,1);
+        my $wget_cmd = $strWgetOptions." -q -O $strFullDocFilePath \"$strDocURL\"";
+
+        my $strResponse =  $self->useWget($wget_cmd,1);
 
         if($strResponse ne "")
@@ -233 +267 @@
 sub getOAIRecords
 {
-    my ($aryIDs, $strOutputDir, $strBasURL, $intMaxRecords, $blnDownloadDoc) = @_;
+    my ($self,$aryIDs, $strOutputDir, $strBasURL, $intMaxRecords, $blnDownloadDoc) = @_;
 
     my $intDocCounter = 0;
@@ -240 +274 @@
     {
     print  STDERR "Gathering OAI record with ID:$strID.....\n";
-    # wget it;
-    my $cmdWget= "-q -O - \"$strBasURL?verb=GetRecord&metadataPrefix=oai_dc&identifier=$strID\"";
-    my $strRecord =  &WgetDownload::useWget($cmdWget);
+       
+    my $cmdWget= $strWgetOptions." -q -O - \"$strBasURL?verb=GetRecord&metadataPrefix=oai_dc&identifier=$strID\"";
+
+    my $strRecord =  $self->useWget($cmdWget);
+
+        
+        my @fileDirs = split(":",$strID);  
 
     # setup directories
-    my $strFileURL = "$strOutputDir/$strID.oai";
-    $strFileURL =~ s/:/\//g;
-
+
+        $strOutputDir  =~ s/"//g;
+  
+    my $strFileURL = "$strOutputDir/$fileDirs[0]/$fileDirs[1].oai";
+    
     # prepare subdirectory for record (if needed)
-    my ($strSubDirPath,$unused) = dirFileSplit($strFileURL);
+    my ($strSubDirPath,$unused) = ("", "");
+
+        ($strSubDirPath,$unused) = $self->dirFileSplit($strFileURL);
+   
     &util::mk_all_dir($strSubDirPath);
 
     my $ds = &util::get_dirsep();
-    my $strOutputFile = &util::filename_cat($strOutputDir,"$strID.oai");
-    $strOutputFile =~ s/:/$ds/g;
-
+    
     if($blnDownloadDoc)
     {
-        &getOAIDoc($strRecord,$strSubDirPath);
+        $self->getOAIDoc($strRecord,$strSubDirPath);
     }
 
     # save record 
-    open (OAIOUT,">$strOutputFile")
+    open (OAIOUT,">$strFileURL")
         || die "Unable to save oai metadata record: $!\n";
     print OAIOUT $strRecord;
     close(OAIOUT);
 
-    $intDocCounter ++;
-    print STDERR "<<Done>>\n";
+        print STDERR "Saving records to $strFileURL\n";
+        print STDERR "<<Done>>\n";
+    $intDocCounter ++;  
     last if ($intDocCounter >= $intMaxRecords);
     }
+
     ($intDocCounter >= $intMaxRecords) ? 
     print  STDERR "Reach maximum download records, use -max_records to set the maximum.\n": 
     print  STDERR "Complete download meta record from $strBasURL\n";
 
+       print STDERR "<<Finished>>\n";
 }
 
@@ -282 +326 @@
     if(!defined $self){ die "System Error: No \$self defined for url_information in OAIDownload\n";}
     
-    my $strBaseCMD = "-q -O - \"$self->{'url'}?_OPTS_\"";
+    my $wgetOptions = $self->getWgetOptions();
+    my $strBaseCMD = $wgetOptions." -q -O - \"$self->{'url'}?_OPTS_\"";
  
     my $strIdentify = "verb=Identify";
@@ -290 +335 @@
     $strIdentifyCMD =~ s/_OPTS_/$strIdentify/;  
 
-    my $strIdentifyText = &WgetDownload::useWget($strIdentifyCMD);
+    my $strIdentifyText = $self->useWget($strIdentifyCMD);
+
+     if (!defined $strIdentifyText or $strIdentifyText eq ""  ){
+    print STDERR "Server information is unavailable.\n";
+    print STDERR "<<Finished>>\n";
+        return;  
+    }
 
     print STDERR "General information:\n";
@@ -297 +348 @@
     my $strListSetCMD = $strBaseCMD;
     $strListSetCMD =~ s/_OPTS_/$strListSets/;   
-    my $strListSetsText = &WgetDownload::useWget($strListSetCMD);
+    my $strListSetsText = $self->useWget($strListSetCMD);
+
+
     print STDERR "List Information:\n";
     $self->parse_xml($strListSetsText);
@@ -306 +359 @@
     my ($self) = shift (@_);
     my ($strOutputText) = @_;
-    my ($name,$fh);
-    
+   
     #Open a temporary file to store OAI information, and store the information to the temp file
-    do {$name = tmpnam()}
-    until $fh = IO::File->new($name, O_RDWR|O_CREAT|O_EXCL);
-    print $fh $strOutputText;
-    close($fh);
+    my $name = "$ENV{GSDLHOME}/tmp/oai.tmp"; 
+
+    open(*OAIOUT,"> $name");
+    
+    print OAIOUT $strOutputText;
+    close(OAIOUT);
 
     $self->{'temp_file_name'} = $name;
@@ -340 +394 @@
     if ((defined $self->{'subfield'} && ($self->{'subfield'} ne ""))) {
     $self->{'text'} .= $_[1];
-    $self->{'text'} =~ s/[\n]|[" "]//g;
+    $self->{'text'} =~ s/[\n]|([ ]{2,})//g;
     if($self->{'text'} ne "")
     {       
@@ -352 +406 @@
 {
     my ($expat, $element, %attr) = @_;
+
     $self->{'subfield'} = $element;
+    
 }
 
@@ -364 +420 @@
 sub error
 {
-    my ($strFunctionName,$strError) = @_;
+    my ($self,$strFunctionName,$strError) = @_;
     {
     print "Error occoured in OAIDownload.pm\n".

trunk/gsdl/perllib/downloaders/SRWDownload.pm

@@ -34 +34 @@
 use strict;
 
-use BasDownload;
+use Z3950Download;
 use IPC::Open2;
 
 sub BEGIN {
-    @SRWDownload::ISA = ('BasDownload');
-}
-
-local (*YAZOUT, *YAZIN); 
-
-# args same as Z3950Download at the moment - should it be based on that??
-my $arguments = 
-    [  { 'name' => "host", 
-    'disp' => "{Z3950Download.host_disp}",
-    'desc' => "{Z3950Download.host}",
-    'type' => "string",
-    'reqd' => "yes"},
-      { 'name' => "port", 
-    'disp' => "{Z3950Download.port_disp}",
-    'desc' => "{Z3950Download.port}",
-    'type' => "string",
-    'reqd' => "yes"},
-      { 'name' => "database", 
-    'disp' => "{Z3950Download.database_disp}",
-    'desc' => "{Z3950Download.database}",
-        'type' => "string",
-    'reqd' => "yes"},
-      { 'name' => "find", 
-    'disp' => "{Z3950Download.find_disp}",
-    'desc' => "{Z3950Download.find}",
-    'type' => "string",
-    'deft' => "",
-    'reqd' => "yes"},
-      { 'name' => "max_records", 
-    'disp' => "{Z3950Download.max_records_disp}",
-    'desc' => "{Z3950Download.max_records}",
-    'type' => "int",
-    'deft' => "500",
-    'reqd' => "no"}];
+    @SRWDownload::ISA = ('Z3950Download');
+}
+
+my $arguments;
 
 my $options = { 'name'     => "SRWDownload",
         'desc'     => "{SRWDownload.desc}",
         'abstract' => "no",
-        'inherits' => "yes",
-        'args'     => $arguments };
+        'inherits' => "yes"
+        };
 
 
@@ -89 +59 @@
     if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
 
-    my $self = (defined $hashArgOptLists)? new BasDownload($getlist,$inputargs,$hashArgOptLists): new BasDownload($getlist,$inputargs);
+    my $self = (defined $hashArgOptLists)? new Z3950Download($getlist,$inputargs,$hashArgOptLists): new Z3950Download($getlist,$inputargs);
 
     if ($self->{'info_only'}) {
@@ -106 +76 @@
     my ($hashGeneralOptions) = @_;
     my ($strOpen,$strBase,$strFind,$strResponse,$intAmount,$intMaxRecords,$strRecords);
+
+    my $url = $self->{'url'};
+
     print STDERR "<<Defined Maximum>>\n";
 
-    my $url = $self->{'url'};
-
-    open2(*YAZOUT, *YAZIN, "yaz-client $url")
-    or die "can't open pipe to yaz-client: $!";
+    my  $yaz =  $self->{'yaz'}; 
+  
+    my $childpid = open2(*YAZOUT, *YAZIN, $yaz)
+    or (print STDERR "<<Finished>>\n" and die "can't open pipe to yaz-client: $!");
+
+    $self->{'YAZOUT'} = *YAZOUT;
+    $self->{'YAZIN'} = *YAZIN;
+
+    $strOpen = $self->open_connection("open $url"); 
+
+    if (!$strOpen) {
+        print STDERR "Cannot connect to $url\n"; 
+        print STDERR "<<Finished>>\n";  
+    return 0;
+    }
 
     print STDERR "Opening connection to \"$self->{'url'}\"\n";
-    #$strOpen = &run_command_with_output("open $self->{'url'}");
     print STDERR "Access database: \"$self->{'database'}\"\n";
-    &run_command_without_output("base $self->{'database'}");
-    &run_command_without_output("querytype prefix");
+    $self->run_command_without_output("base $self->{'database'}");
+    $self->run_command_without_output("querytype prefix");
     print STDERR "Searching for keyword: \"$self->{'find'}\"\n";
-    $intAmount = &findAmount($self->{'find'});
+    $intAmount =$self->findAmount($self->{'find'});
 
     if($intAmount <= 0)
     {
     ($intAmount == -1)? 
-        print STDERR "Unexpected format, Parsing operation can not be performed\n" : 
+        print STDERR "Something wrong with the arguments,downloading can not be performed\n" : 
         print STDERR "No Record is found\n";
+        print STDERR "<<Finished>>\n";
     return 0;
     }
     $intMaxRecords = ($self->{'max_records'} > $intAmount)? $intAmount : $self->{'max_records'};
     print STDERR "<<Total number of record(s):$intMaxRecords>>\n";
-    $strRecords = &getRecords($intMaxRecords);
-    print STDERR $strRecords;
-    &saveRecords($self,$strRecords,$hashGeneralOptions->{'cache_dir'},$intMaxRecords);
-    print STDERR "Closing connection\n";
+  
+    $strRecords = $self->getRecords($intMaxRecords);
+  
+    $self->saveRecords($strRecords,$hashGeneralOptions->{'cache_dir'},$intMaxRecords);
+    print STDERR "Closing connection...\n";
+    print STDERR "<<Finished>>\n";
     close(YAZOUT);
     close(YAZIN);
-    return 1;
-}
-
-sub findAmount
-{
-    my($strFindTarget) = @_;
-    my $strResponse = &run_command_with_output("find $strFindTarget");
-
-    return ($strResponse =~ m/^Number of hits: (\d+)/m)? $1:-1;  
-}
-
-sub getRecords
-{
-    my ($intMaxRecords) = @_;
-    my ($strShow,$intStartNumber,$strResponse,$strRecords,$intRecordsLeft);
-
-    $intStartNumber = 1;
-    $intRecordsLeft = $intMaxRecords;
-    while ($intRecordsLeft > 0)
-    {
-    if($intRecordsLeft > 50)
-    {
-        print STDERR "<<Done:50>>\n";
-        print STDERR "Yaz is Gathering records: $intStartNumber - ".($intStartNumber+49)."\n";
-        
-        $strShow = "show $intStartNumber+50";
-        $intStartNumber = $intStartNumber + 50;
-        $intRecordsLeft = $intRecordsLeft - 50;
-    }
-    else
-    {
-        print STDERR "<<Done:".($intRecordsLeft).">>\n";
-        print STDERR "Yaz is Gathering records: $intStartNumber - ".($intStartNumber+$intRecordsLeft-1)."\n";
-        $strShow = "show $intStartNumber+$intRecordsLeft";
-        $intRecordsLeft = 0;
-    }
-    
-    $strResponse = &run_command_with_output($strShow);
-
-        ## need to change this
-               
-        print STDERR $strResponse;
-    
-        if($strResponse =~  m/pos=[\d]*(.*)>\n/s)
-    {
-           $strRecords .= "$1>\n";
-           
-           $strRecords =~ s/pos=[\d]*(.*)?\n//g;            
-    }
-    }
-    return $strRecords;
-}
+   return 1;
+}
+
 
 sub saveRecords
@@ -195 +132 @@
     # setup directories
     # Currently only gather the MARC format
-    my $strFileName = &generateFileName($self,$intMaxRecords);
-    my $strFileURL = "$strOutputDir/$self->{'host'}/$strFileName.marc";
-    $strFileURL =~ s/:/\//g;
-
+    $strRecords ="<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<collection>$strRecords</collection>";  
+    my $strFileName =  $self->generateFileName($intMaxRecords);
+    my $host = $self->{'host'};
+    $host =~ s/http:\/\//srw\//;
+    $strOutputDir  =~ s/"//g; 
+    my $strFileURL = "$strOutputDir/$host/$strFileName.xml";
+  
     # prepare subdirectory for record (if needed)
-    my ($strSubDirPath,$unused) = dirFileSplit($strFileURL);
+
+    my ($strSubDirPath,$unused) = $self->dirFileSplit($strFileURL);
     &util::mk_all_dir($strSubDirPath);
 
     my $ds = &util::get_dirsep();
-    my $strOutputFile = &util::filename_cat($strOutputDir,$self->{'host'},"$strFileName.marc");
-    $strOutputFile =~ s/:/$ds/g;
-
-    print STDERR "Saving records to \"$strOutputFile\"\n";
+  
+    print STDERR "Saving records to \"$strFileURL\"\n";
 
     # save record 
-    open (ZOUT,">$strOutputFile")
+    open (ZOUT,">$strFileURL")
     || die "Unable to save oai metadata record: $!\n";
     print ZOUT $strRecords;
@@ -216 +155 @@
 }
 
-sub run_command_with_output
-{
-    my ($strCMD) = @_;
-    return &run_command($strCMD,"^Elapsed:.*\$");
-}
-
-sub run_command_without_output
-{
-    my ($strCMD) = @_;
-    &run_command($strCMD);
-}
-
-sub run_command
-{
-    my ($strCMD,$strStopRE) = @_;
+sub get{
+   my ($self,$strShow,$numRecord) = @_;  
+
+   $self->run_command($strShow);  
+   
+   my $strFullOutput="";
+   my $count=0;
+   my $readRecord = 0;
+   
+   while (my $strLine = <YAZOUT>)
+   {
     
-    print YAZIN "$strCMD\n";
-    if (!defined $strStopRE){return "";}
-    else
-    {
-    my $strFullOutput;
-        while (my $strLine = <YAZOUT>)
-    {
-            $strFullOutput .= $strLine;
-        if($strLine =~ m/$strStopRE/){return $strFullOutput;}
-    }
-    }
-}
-
-sub url_information
-{
-   my ($self) = shift (@_);
+       return $strFullOutput if ($count >= $numRecord);
+
+       return $strFullOutput if($strLine =~ m/^HTTP ERROR/i);
+
+       if ($strLine =~ m/pos=[\d]*/i ){ 
+           $count++;
+       $readRecord = 1;
+       next;  
+       }
+      
+       next if(!$readRecord);
+
+       $strFullOutput .= $strLine;     
+   }
+ 
+}
+
+sub url_information{
+   my ($self) = @_;
 
    my $url = $self->{'url'};
 
-  open2(*YAZOUT, *YAZIN, "yaz-client $url") or die "can't open pipe to yaz-client: $!";
-
-   my $strFullOutput="";
-
-   while (my $strLine = <YAZOUT>)
-    {
-          $strFullOutput .= $strLine;  
-      }
-
-   return  $strFullOutput;
-}
-
-sub generateFileName
-{
-    my ($self,$intMaxRecords) = @_;
-    my $strFileName = ($self->{'database'})."_".($self->{'find'})."_".($intMaxRecords);
-}
-
-sub dirFileSplit
-{
-    my ($strFile) = @_;
-
-    my @aryDirs = split("/",$strFile);
-    my $strLocalFile = pop(@aryDirs);
-    my $strSubDirs = join("/",@aryDirs);
-
-    return ($strSubDirs,$strLocalFile);
+   $url =~ s#http://##; 
+
+  return $self->SUPER::url_information($url);  
+
 }
 
 sub error
 {
-    my ($strFunctionName,$strError) = @_;
+    my ($self, $strFunctionName,$strError) = @_;
     {
     print STDERR "Error occoured in SRWDownload.pm\n".

trunk/gsdl/perllib/downloaders/WebDownload.pm

@@ -99 +99 @@
     my ($hashGeneralOptions) = @_;
 
-    # TODO: the checking for Wget is still not complete, we need to 
-    #       check if the proxy has been set or not, and whether the 
-    #       connection has been established.
-    # Checking if the wget has been well setup
-    # &WgetDownload::checkWgetSetup($self,$hashGeneralOptions->{'gli_call'});
-
+  
     # Download options
-    my $strOptions = &generateOptionsString($self);
-    my $strWgetOptions = &WgetDownload::getWgetOptions($self);
-    
-   
+    my $strOptions = $self->generateOptionsString();
+    my $strWgetOptions = $self->getWgetOptions();
+      
     # Setup the command for using wget
     my $cmdWget = "-N -k -x -t 2 -P ".$hashGeneralOptions->{"cache_dir"}." $strWgetOptions $strOptions ".$self->{'url'};
     
-    print "**************".$cmdWget."\n";
-
     # Download the web pages
     # print "Strat download from $self->{'url'}...\n";
-    
     print STDERR "<<Undefined Maximum>>\n";
-    my $strResponse = &WgetDownload::useWget($cmdWget,1);
-    #if ($strResponse ne ""){print "$strResponse\n";}
-    
-    # print "Finish download from $self->{'url'}...\n";
-
+
+    my $strResponse = $self->useWget($cmdWget,1);
+ 
+    # if ($strResponse ne ""){print "$strResponse\n";}
+    
+    print STDERR "Finish download from $self->{'url'}\n";
+
+    print STDERR "<<Finished>>\n";
+  
     return 1;
 }
@@ -146 +141 @@
     else
     {
-    &error("setupOptions","Incorrect Depth is defined!!\n");
+    $self->error("setupOptions","Incorrect Depth is defined!!\n");
     }
 
@@ -174 +169 @@
 {
     my ($self) = shift (@_);
-    if(!defined $self){ die "System Error: No \$self defined for url_information in WebDownload\n";}
-    
-    my $strBaseCMD = "-q -O - \"$self->{'url'}\"";
-
-    my $strIdentifyText = &WgetDownload::useWget($strBaseCMD);
-    
+
+    my $strOptions = $self->getWgetOptions();
+
+    my $strBaseCMD = $strOptions." -q -O - \"$self->{'url'}\"";
+
+  
+    my $strIdentifyText = $self->useWget($strBaseCMD);
+    
+    if (!defined $strIdentifyText or $strIdentifyText eq ""  ){
+    print STDERR "Server information is unavailable.\n";
+    print STDERR "<<Finished>>\n";
+         return;  
+    }
+
     while ($strIdentifyText =~ m/^(.*)<title>(.*?)<\/title>(.*)$/s)
     {
@@ -185 +188 @@
     print STDERR "Page Title: $2\n";
     }
-
+  
     while ($strIdentifyText =~ m/^(.*)<meta (.*?)>(.*)$/s)
     {
     $strIdentifyText = $1.$3;
     my $strTempString = $2;
-    print STDERR "Meta Information:\n";
+    print STDERR "\n";
+
     while($strTempString =~ m/(.*?)=[\"|\'](.*?)[\"|\'](.*?)$/s)
     {
@@ -202 +206 @@
         $strMetaName =~ s/^([" "])+//m;
         $strMetaContain =~ s/^([" "])+//m;
-        print STDERR "\t$strMetaName: $strMetaContain\n";
-        
+              
+        print STDERR "$strMetaName: $strMetaContain\n\n";
+           
     }
-    print STDERR "\n";
-    }
+
+    }
+
+    print STDERR "<<Finished>>\n";
+
 }
 
@@ -222 +230 @@
 
 1;
+

trunk/gsdl/perllib/downloaders/WgetDownload.pm

@@ -34 +34 @@
 use BasDownload;
 use strict;
+use IPC::Open2;
 
 sub BEGIN {
@@ -40 +41 @@
 
 my $arguments = 
-#    [ { 'name' => "url", 
-#   'desc' => "{WgetDownload.url}",         
-#   'type' => "string",
-#   'deft' => "",
-#   'reqd' => "yes"},
      [ { 'name' => "proxy_on", 
     'desc' => "{WgetDownload.proxy_on}",
@@ -125 +121 @@
     # Setup .wgetrc by using $self->{'proxy_host'} and $self->{'proxy_port'}
     # Test if the connection is succeful. If the connection wasn't succeful then ask user to supply username and password.
-    
 
-    # TODO: How to test run if the proxy setup is working correctly?? 
-    # Use -spider to test whether the connection is working correctly.
-    # TODO: Ask user to supply username and password.
-    # Try to use the .wgetrc to setup the user name and password
-    
 }
 
 sub useWget
 {
-    my ($cmdWget,$blnShow) = @_;
+    my ($self, $cmdWget,$blnShow) = @_;
 
-    my $strReadIn = "";
-    my $strLine;
+    my ($os,$strReadIn,$strLine,$command);
+   
+    $os = $ENV{'GSDLOS'};
+ 
+  
+    if ($os =~ /windows/i){
+    $command = "\"$ENV{'GSDLHOME'}\\bin\\windows\\wget\" $cmdWget |";
+    }
+    else{
+        $command = "$ENV{'GSDLHOME'}/packages/wget/wget-1.9/src/wget $cmdWget |";  
+    }
 
-    open (WIN,"$ENV{'GSDLHOME'}/packages/wget/wget-1.9/src/wget $cmdWget|") || die "wget request failed: $!\n";
+    
+
+   open(*WIN,$command) || die "wget request failed: $!\n";
+
+    
+
     while (defined($strLine=<WIN>))
     {
+
+
     if($blnShow)
     {
-        print "$strReadIn\n";
+        print STDERR "$strReadIn\n";
     }
+
     $strReadIn .= $strLine;
     }
+
     close(WIN);
 

trunk/gsdl/perllib/downloaders/Z3950Download.pm

@@ -40 +40 @@
     @Z3950Download::ISA = ('BasDownload');
 }
-
-local (*YAZOUT, *YAZIN); 
 
 my $arguments = 
@@ -97 +95 @@
     # Must set $self->{'url'}, since GLI use $self->{'url'} to calculate the log file name!
     $self->{'url'} = $self->{'host'}.":".$self->{'port'};
+
+     my $os = $ENV{'GSDLOS'};
+
+    if ($os !~ /windows/) {
+    $self->{'yaz'} = "$ENV{'GSDLHOME'}/packages/yaz/yaz-2.1.4/client/yaz-client"; 
+    }
+    else{
+    $self->{'yaz'} = "$ENV{'GSDLHOME'}/bin/windows/yaz-client"; 
+    }
+    
     return bless $self, $class;
 
@@ -106 +114 @@
     my ($hashGeneralOptions) = @_;
     my ($strOpen,$strBase,$strFind,$strResponse,$intAmount,$intMaxRecords,$strRecords);
+
+    my $url = $self->{'url'};
+ 
     print STDERR "<<Defined Maximum>>\n";
  
-    my $url = $self->{'url'};
     print STDERR "Opening connection to $url\n";
      
-    my $childpid = open2(*YAZOUT, *YAZIN, "yaz-client")
-    or die "can't open pipe to yaz-client: $!";
- 
-    $strOpen = &run_command_with_output("open $url");  
+    my  $yaz = $self->{'yaz'};
+ 
+    my $childpid = open2(*YAZOUT, *YAZIN, $yaz)
+    or (print STDERR "<<Finished>>\n" and die "can't open pipe to yaz-client: $!");
+    $self->{'YAZOUT'} = *YAZOUT;
+    $self->{'YAZIN'} = *YAZIN;
+
+    $strOpen = $self->open_connection("open $url");  
+
+    if (!$strOpen) {
+        print STDERR "Cannot connect to $url\n"; 
+        print STDERR "<<Finished>>\n";  
+    return 0;
+    }
+
     print STDERR "Access database: \"$self->{'database'}\"\n";
-    &run_command_without_output("base $self->{'database'}");
+    $self->run_command_without_output("base $self->{'database'}");
     print STDERR "Searching for keyword: \"$self->{'find'}\"\n";
-    $intAmount = &findAmount($self->{'find'});
+    $intAmount = $self->findAmount($self->{'find'});
 
     if($intAmount <= 0)
     {
     ($intAmount == -1)? 
-        print STDERR "Unexpected format, Parsing operation can not be performed\n" : 
-        print STDERR "No Record is found\n";
+        print STDERR "Something wrong with the arguments,downloading can not be performed\n": 
+        print STDERR "No Record is found\n";
+    print STDERR "<<Finished>>\n";
     return 0;
     }
     $intMaxRecords = ($self->{'max_records'} > $intAmount)? $intAmount : $self->{'max_records'};
     print STDERR "<<Total number of record(s):$intMaxRecords>>\n";
-    $strRecords = &getRecords($intMaxRecords);
-    print STDERR $strRecords;
-    &saveRecords($self,$strRecords,$hashGeneralOptions->{'cache_dir'},$intMaxRecords);
+    $strRecords = "Records: $intMaxRecords\n".$self->getRecords($intMaxRecords);
+   
+    $self->saveRecords($strRecords,$hashGeneralOptions->{'cache_dir'},$intMaxRecords);
     print STDERR "Closing connection...\n";
+    print STDERR "<<Finished>>\n";
+
     close(YAZOUT);
     close(YAZIN);
-    waitpid($childpid, 0);
     return 1;
 }
 
+sub open_connection{
+  my ($self,$strCommand) =  (@_);
+  
+  $self->run_command($strCommand);  
+
+  my $out = $self->{'YAZOUT'}; 
+
+  $_ = <$out>;
+  
+  return (/Connecting...OK/i)? 1: 0; 
+ 
+}
+
 sub findAmount
 {
+    my ($self) = shift (@_);
     my($strFindTarget) = @_;
-    my $strResponse = &run_command_with_output("find $strFindTarget");
-    return ($strResponse =~ m/^Number of hits: (\d+)/m)? $1:-1;    
+    my $strResponse = $self->run_command_with_output("find $strFindTarget","^Number of hits:");
+   return ($strResponse =~ m/^Number of hits: (\d+)/m)? $1:-1;    
 }
 
 sub getRecords
 {
+    my ($self) = shift (@_);
     my ($intMaxRecords) = @_;
-    my ($strShow,$intStartNumber,$strResponse,$strRecords,$intRecordsLeft);
+    my ($strShow,$intStartNumber,$numRecords,$strResponse,$strRecords,$intRecordsLeft);
 
     $intStartNumber = 1;
     $intRecordsLeft = $intMaxRecords;
+    $numRecords = 0;
+    $strResponse ="";
+
     while ($intRecordsLeft > 0)
     {
     if($intRecordsLeft > 50)
     {
-        print STDERR "<<Done:50>>\n";
+       
         print STDERR "Yaz is Gathering records: $intStartNumber - ".($intStartNumber+49)."\n";
-        
+        $numRecords = 50;
         $strShow = "show $intStartNumber+50";
         $intStartNumber = $intStartNumber + 50;
         $intRecordsLeft = $intRecordsLeft - 50;
+             
     }
     else
     {
-        print STDERR "<<Done:".($intRecordsLeft).">>\n";
+        $numRecords = $intRecordsLeft;
         print STDERR "Yaz is Gathering records: $intStartNumber - ".($intStartNumber+$intRecordsLeft-1)."\n";
         $strShow = "show $intStartNumber+$intRecordsLeft";
         $intRecordsLeft = 0;
+       
+           }
+    
+    $strResponse .= $self->get($strShow,$numRecords);
+         
+    if ($strResponse eq ""){
+        print STDERR "<<ERROR: failed to get $numRecords records>>\n";
     }
+    else{
+        print STDERR "<<Done:$numRecords>>\n";
+    }
+    }
+
+    return  "$strResponse\n";
     
-    $strResponse = &run_command_with_output($strShow);
-       
-    if($strResponse =~ m/Records: (\d*?)\n(.*?)nextResultSetPosition = (\d*?)\n/s)
-    {
-        $strRecords .= $2;
-    }
-    }
-    return $strRecords;
 }
 
@@ -188 +235 @@
     # setup directories
     # Currently only gather the MARC format
-    my $strFileName = &generateFileName($self,$intMaxRecords);
+    my $strFileName = $self->generateFileName($intMaxRecords);
+
+    $strOutputDir  =~ s/"//g;
+
     my $strFileURL = "$strOutputDir/$self->{'host'}/$strFileName.marc";
-    $strFileURL =~ s/:/\//g;
-
+  
     # prepare subdirectory for record (if needed)
-    my ($strSubDirPath,$unused) = dirFileSplit($strFileURL);
+    my ($strSubDirPath,$unused) = $self->dirFileSplit($strFileURL);
+ 
     &util::mk_all_dir($strSubDirPath);
 
     my $ds = &util::get_dirsep();
     my $strOutputFile = &util::filename_cat($strOutputDir,$self->{'host'},"$strFileName.marc");
-    $strOutputFile =~ s/:/$ds/g;
-
+   
     print STDERR "Saving records to \"$strOutputFile\"\n";
 
     # save record 
     open (ZOUT,">$strOutputFile")
-    || die "Unable to save oai metadata record: $!\n";
+    || die "Unable to save Z3950 record: $!\n";
     print ZOUT $strRecords;
     close(ZOUT);
 }
 
+
 sub run_command_with_output
 {
+    my ($self,$strCMD,$strStopRE) =@_;
+   
+    $self->run_command($strCMD); 
+   
+    return $self->get_output($strStopRE);
+ 
+}
+
+sub get{
+   my ($self,$strShow,$numRecord) = @_;  
+
+   $self->run_command($strShow);  
+   
+   my $strFullOutput="";
+   my $count=0;
+   my $readRecord = 0;
+   
+   while (my $strLine = <YAZOUT>)
+   {
+   
+       if ($strLine =~ m/Records: ([\d]*)/i ){ 
+       $readRecord = 1;
+       next;  
+       }
+      
+      return $strFullOutput if ($strLine =~ m/nextResultSetPosition|Not connected/i);
+        
+      next if(!$readRecord);
+      
+      $strFullOutput .= $strLine;     
+  }
+   
+}
+
+sub run_command_without_output
+{
+     my ($self) = shift (@_);
     my ($strCMD) = @_;
-    
-    return &run_command($strCMD,"^Elapsed:.*\$");
-}
-
-sub run_command_without_output
-{
-    my ($strCMD) = @_;
-
-    &run_command($strCMD);
+
+    $self->run_command($strCMD);
 }
 
 sub run_command
 {
-    my ($strCMD,$strStopRE) = @_;
-    
-  
-    print YAZIN "$strCMD\n";
+    my ($self,$strCMD) = @_; 
+ 
+    my $input = $self->{'YAZIN'};
+
+    print $input "$strCMD\n";  
+}
+
+sub get_output{
+    my ($self,$strStopRE) = @_;  
+
     if (!defined $strStopRE){return "";}
     else
     {
     my $strFullOutput;
-    while (my $strLine = <YAZOUT>)
+        my $output = $self->{'YAZOUT'};
+    while (my $strLine = <$output>)
     {
-        $strFullOutput .= $strLine;
-        if($strLine =~ m/$strStopRE/){return $strFullOutput;}
+           $strFullOutput .= $strLine;   
+       if($strLine =~ m/^$strStopRE|Not connected/i){return $strFullOutput;}
     }
     }
@@ -245 +332 @@
     my ($self,$intMaxRecords) = @_;
     my $strFileName = ($self->{'database'})."_".($self->{'find'})."_".($intMaxRecords);
+ 
 }
 
 sub dirFileSplit
 {
-    my ($strFile) = @_;
-
-    my @aryDirs = split("/",$strFile);
+    my ($self,$strFile) = @_;
+
+    my @aryDirs = split("[/\]",$strFile);
+   
     my $strLocalFile = pop(@aryDirs);
     my $strSubDirs = join("/",@aryDirs);
@@ -258 +347 @@
 }
 
+sub url_information
+{
+   my ($self,$url) = @_;
+
+   $url = $self->{'url'} unless defined $url;
+
+   my  $yaz =  $self->{'yaz'};
+   
+   my $childpid = open2(*YAZOUT, *YAZIN, $yaz)
+       or die "can't open pipe to yaz-client: $!";
+  
+   $self->{'YAZOUT'} = *YAZOUT;
+   $self->{'YAZIN'} = *YAZIN;
+
+   my $strOpen = $self->open_connection("open $url"); 
+   
+    if (!$strOpen) {
+        print STDERR "Cannot connect to $url\n"; 
+        print STDERR "<<Finished>>\n";  
+    return 0;
+    }
+
+   
+   $strOpen = $self->run_command_with_output("open $url","^Options");  
+
+
+   $strOpen =~ s/Z> //g;
+   $strOpen =~ s/Elapsed:.*//g;
+
+   print STDERR $strOpen; 
+
+   print STDERR "<<Finished>>\n"; 
+
+   close(YAZOUT);
+   close(YAZIN);
+
+   return 0;
+
+}
+
 sub error
 {
-    my ($strFunctionName,$strError) = @_;
+    my ($self,$strFunctionName,$strError) = @_;
     {
     print STDERR "Error occoured in Z3950Download.pm\n".