Index: /trunk/gsdl/perllib/download.pm
===================================================================
--- /trunk/gsdl/perllib/download.pm	(revision 12464)
+++ /trunk/gsdl/perllib/download.pm	(revision 12465)
@@ -36,12 +36,21 @@
 sub load_download {
     my ($download_name,$download_options) = @_;
+
     my ($download_obj);
 
-    my $coldownloadname = &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, 
+    my $coldownloadname ="";
+
+    if ($ENV{'GSDLCOLLECTDIR'}){
+   
+    $coldownloadname = &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, 
 					      "perllib/downloaders", 
 					      "${download_name}.pm");
+
+   }
+   
     my $maindownloadname = &util::filename_cat($ENV{'GSDLHOME'},
 					       "perllib/downloaders", 
 					       "${download_name}.pm");
+
     if (-e $coldownloadname) { require $coldownloadname;}
     elsif (-e $maindownloadname ) { require $maindownloadname; }
Index: /trunk/gsdl/perllib/downloaders/OAIDownload.pm
===================================================================
--- /trunk/gsdl/perllib/downloaders/OAIDownload.pm	(revision 12464)
+++ /trunk/gsdl/perllib/downloaders/OAIDownload.pm	(revision 12465)
@@ -37,6 +37,6 @@
 use XMLParser;
 
-use IO::File;
 use POSIX qw(tmpnam);
+use util;
 
 sub BEGIN {
@@ -56,5 +56,5 @@
 	'reqd' => "no"},
       { 'name' => "get_doc",
-	'disp' => "{OAIDownload.qet_doc_disp}",
+	'disp' => "{OAIDownload.get_doc_disp}",
 	'desc' => "{OAIDownload.get_doc}",
 	'type' => "flag",
@@ -76,4 +76,6 @@
 my $self;
 
+my $strWgetOptions="";
+
 sub new 
 {
@@ -107,8 +109,11 @@
     my ($hashGeneralOptions) = @_;
 
-    # Checking if the wget has been well setup
-    # &WgetDownload::checkWgetSetup($self,$hashGeneralOptions->{'gli_call'});
-    
-    my $strOutputDir = $hashGeneralOptions->{"cache_dir"};
+    print STDERR "here2";
+   
+    $strWgetOptions = $self->getWgetOptions();
+    my $cmdWget = $strWgetOptions;
+  
+    my $strOutputDir ="";
+    $strOutputDir = $hashGeneralOptions->{"cache_dir"};
     my $strBasURL = $self->{'url'};
     my $intMaxRecords = $self->{'max_records'};
@@ -116,11 +121,13 @@
 
     print STDERR "<<Defined Maximum>>\n";
-    my $strIDs = &getOAIIDs($self,$strBasURL);
-    if($strIDs eq "")
+
+    my $strIDs = $self->getOAIIDs($strBasURL);
+ 
+   if($strIDs eq "")
     {
 	print STDERR "Error: No ID being found\n";
 	return 0;
     }
-    my $aryIDs = &parseOAIIDs($strIDs);
+    my $aryIDs = $self->parseOAIIDs($strIDs);
     my $intIDs = 0;
     if($self->{'max_records'} < scalar(@$aryIDs))
@@ -134,5 +141,8 @@
     print STDERR "<<Total number of record(s):$intIDs>>\n";
 
-    &getOAIRecords($aryIDs, $strOutputDir, $strBasURL, $intMaxRecords, $blnDownloadDoc);
+    $self->getOAIRecords($aryIDs, $strOutputDir, $strBasURL, $intMaxRecords, $blnDownloadDoc);
+
+    my $tmp_file = "$ENV{GSDLHOME}/tmp/oai.tmp";
+    &util::rm($tmp_file); 
 
     return 1;
@@ -143,14 +153,33 @@
     my ($self,$strBasURL) = @_;
     my ($cmdWget);
+     
+    my $wgetOptions = $self->getWgetOptions();
+
+    $cmdWget = $wgetOptions;
+  
     print STDERR  "Gathering OAI identifiers.....\n";
+
     if($self->{'set'} ne "")
     {
-	$cmdWget = "-q -O - \"$strBasURL?verb=ListIdentifiers&metadataPrefix=oai_dc&set=$self->{'set'}\" ";
+	$cmdWget .= " -q -O - \"$strBasURL?verb=ListIdentifiers&metadataPrefix=oai_dc&set=$self->{'set'}\" ";
     }
     else
     {
-	$cmdWget = "-q -O - \"$strBasURL?verb=ListIdentifiers&metadataPrefix=oai_dc\" ";
-    }
-    my $strIDs =  &WgetDownload::useWget($cmdWget);
+	$cmdWget .= " -q -O - \"$strBasURL?verb=ListIdentifiers&metadataPrefix=oai_dc\" ";
+    }
+
+ 
+    my $strIDs =  $self->useWget($cmdWget);
+
+    if (!defined $strIDs or $strIDs eq ""  ){
+	print STDERR "Server information is unavailable.\n";
+	print STDERR "<<Finished>>\n";
+        return;  
+    }
+
+    print STDERR "<<Download Information>>\n";
+
+    $self->parse_xml($strIDs);
+
     return $strIDs;
 }
@@ -158,5 +187,5 @@
 sub parseOAIIDs
 {   
-    my ($strIDs) = @_;
+    my ($self,$strIDs) = @_;
 
     print STDERR "Parsing OAI identifiers.....\n";
@@ -177,7 +206,8 @@
 sub dirFileSplit
 {
-    my ($strFile) = @_;
-
-    my @aryDirs = split("/",$strFile);
+    my ($self,$strFile) = @_;
+
+    my @aryDirs = split("[/\]",$strFile);
+   
     my $strLocalFile = pop(@aryDirs);
     my $strSubDirs = join("/",@aryDirs);
@@ -188,8 +218,9 @@
 sub getOAIDoc
 {
-    my ($strRecord, $strSubDirPath) = @_;
-    
+    my ($self,$strRecord, $strSubDirPath) = @_;
+ 
     print  STDERR "Gathering source documents.....\n";
     # look out for identifier tag in metadata section
+   
     if ($strRecord =~ m/<metadata>(.*)<\/metadata>/s)
     {
@@ -200,14 +231,17 @@
 	    my $strDocURL = $2;
 
-	    my ($unused,$strDocFile) = dirFileSplit($strDocURL);
-
-	    my $strSoureDirPath = &util::filename_cat($strSubDirPath,"srcdocs");
+	    my ($unused,$strDocFile) = $self->dirFileSplit($strDocURL);
+
+            my $strSoureDirPath ="";
+
+	    $strSoureDirPath = &util::filename_cat($strSubDirPath,"srcdocs");
+
 	    &util::mk_dir($strSoureDirPath)  if (!-e "$strSoureDirPath");
 	    
 	    my $strFullDocFilePath = &util::filename_cat($strSoureDirPath,$strDocFile);
 	    
-	    my $wget_cmd = "-q -O $strFullDocFilePath \"$strDocURL\"";
-
-	    my $strResponse =  &WgetDownload::useWget($wget_cmd,1);
+	    my $wget_cmd = $strWgetOptions." -q -O $strFullDocFilePath \"$strDocURL\"";
+
+	    my $strResponse =  $self->useWget($wget_cmd,1);
 
 	    if($strResponse ne "")
@@ -233,5 +267,5 @@
 sub getOAIRecords
 {
-    my ($aryIDs, $strOutputDir, $strBasURL, $intMaxRecords, $blnDownloadDoc) = @_;
+    my ($self,$aryIDs, $strOutputDir, $strBasURL, $intMaxRecords, $blnDownloadDoc) = @_;
 
     my $intDocCounter = 0;
@@ -240,39 +274,49 @@
     {
 	print  STDERR "Gathering OAI record with ID:$strID.....\n";
-	# wget it;
-	my $cmdWget= "-q -O - \"$strBasURL?verb=GetRecord&metadataPrefix=oai_dc&identifier=$strID\"";
-	my $strRecord =  &WgetDownload::useWget($cmdWget);
+	   
+	my $cmdWget= $strWgetOptions." -q -O - \"$strBasURL?verb=GetRecord&metadataPrefix=oai_dc&identifier=$strID\"";
+
+	my $strRecord =  $self->useWget($cmdWget);
+
+        
+        my @fileDirs = split(":",$strID);  
 
 	# setup directories
-	my $strFileURL = "$strOutputDir/$strID.oai";
-	$strFileURL =~ s/:/\//g;
-
+
+        $strOutputDir  =~ s/"//g;
+  
+	my $strFileURL = "$strOutputDir/$fileDirs[0]/$fileDirs[1].oai";
+	
 	# prepare subdirectory for record (if needed)
-	my ($strSubDirPath,$unused) = dirFileSplit($strFileURL);
+	my ($strSubDirPath,$unused) = ("", "");
+
+       	($strSubDirPath,$unused) = $self->dirFileSplit($strFileURL);
+   
 	&util::mk_all_dir($strSubDirPath);
 
 	my $ds = &util::get_dirsep();
-	my $strOutputFile = &util::filename_cat($strOutputDir,"$strID.oai");
-	$strOutputFile =~ s/:/$ds/g;
-
+	
 	if($blnDownloadDoc)
 	{
-	    &getOAIDoc($strRecord,$strSubDirPath);
+	    $self->getOAIDoc($strRecord,$strSubDirPath);
 	}
 
 	# save record 
-	open (OAIOUT,">$strOutputFile")
+	open (OAIOUT,">$strFileURL")
 	    || die "Unable to save oai metadata record: $!\n";
 	print OAIOUT $strRecord;
 	close(OAIOUT);
 
-	$intDocCounter ++;
-	print STDERR "<<Done>>\n";
+        print STDERR "Saving records to $strFileURL\n";
+        print STDERR "<<Done>>\n";
+	$intDocCounter ++;	
 	last if ($intDocCounter >= $intMaxRecords);
     }
+
     ($intDocCounter >= $intMaxRecords) ? 
 	print  STDERR "Reach maximum download records, use -max_records to set the maximum.\n": 
 	print  STDERR "Complete download meta record from $strBasURL\n";
 
+       print STDERR "<<Finished>>\n";
 }
 
@@ -282,5 +326,6 @@
     if(!defined $self){ die "System Error: No \$self defined for url_information in OAIDownload\n";}
     
-    my $strBaseCMD = "-q -O - \"$self->{'url'}?_OPTS_\"";
+    my $wgetOptions = $self->getWgetOptions();
+    my $strBaseCMD = $wgetOptions." -q -O - \"$self->{'url'}?_OPTS_\"";
  
     my $strIdentify = "verb=Identify";
@@ -290,5 +335,11 @@
     $strIdentifyCMD =~ s/_OPTS_/$strIdentify/;	
 
-    my $strIdentifyText = &WgetDownload::useWget($strIdentifyCMD);
+    my $strIdentifyText = $self->useWget($strIdentifyCMD);
+
+     if (!defined $strIdentifyText or $strIdentifyText eq ""  ){
+	print STDERR "Server information is unavailable.\n";
+	print STDERR "<<Finished>>\n";
+        return;  
+    }
 
     print STDERR "General information:\n";
@@ -297,5 +348,7 @@
     my $strListSetCMD = $strBaseCMD;
     $strListSetCMD =~ s/_OPTS_/$strListSets/;	
-    my $strListSetsText = &WgetDownload::useWget($strListSetCMD);
+    my $strListSetsText = $self->useWget($strListSetCMD);
+
+
     print STDERR "List Information:\n";
     $self->parse_xml($strListSetsText);
@@ -306,11 +359,12 @@
     my ($self) = shift (@_);
     my ($strOutputText) = @_;
-    my ($name,$fh);
-    
+   
     #Open a temporary file to store OAI information, and store the information to the temp file
-    do {$name = tmpnam()}
-	until $fh = IO::File->new($name, O_RDWR|O_CREAT|O_EXCL);
-    print $fh $strOutputText;
-    close($fh);
+    my $name = "$ENV{GSDLHOME}/tmp/oai.tmp"; 
+
+    open(*OAIOUT,"> $name");
+	
+    print OAIOUT $strOutputText;
+    close(OAIOUT);
 
     $self->{'temp_file_name'} = $name;
@@ -340,5 +394,5 @@
     if ((defined $self->{'subfield'} && ($self->{'subfield'} ne ""))) {
 	$self->{'text'} .= $_[1];
-	$self->{'text'} =~ s/[\n]|[" "]//g;
+	$self->{'text'} =~ s/[\n]|([ ]{2,})//g;
 	if($self->{'text'} ne "")
 	{	    
@@ -352,5 +406,7 @@
 {
     my ($expat, $element, %attr) = @_;
+
     $self->{'subfield'} = $element;
+    
 }
 
@@ -364,5 +420,5 @@
 sub error
 {
-    my ($strFunctionName,$strError) = @_;
+    my ($self,$strFunctionName,$strError) = @_;
     {
 	print "Error occoured in OAIDownload.pm\n".
Index: /trunk/gsdl/perllib/downloaders/SRWDownload.pm
===================================================================
--- /trunk/gsdl/perllib/downloaders/SRWDownload.pm	(revision 12464)
+++ /trunk/gsdl/perllib/downloaders/SRWDownload.pm	(revision 12465)
@@ -34,48 +34,18 @@
 use strict;
 
-use BasDownload;
+use Z3950Download;
 use IPC::Open2;
 
 sub BEGIN {
-    @SRWDownload::ISA = ('BasDownload');
-}
-
-local (*YAZOUT, *YAZIN); 
-
-# args same as Z3950Download at the moment - should it be based on that??
-my $arguments = 
-    [  { 'name' => "host", 
-	'disp' => "{Z3950Download.host_disp}",
-	'desc' => "{Z3950Download.host}",
-	'type' => "string",
-	'reqd' => "yes"},
-      { 'name' => "port", 
-	'disp' => "{Z3950Download.port_disp}",
-	'desc' => "{Z3950Download.port}",
-	'type' => "string",
-	'reqd' => "yes"},
-      { 'name' => "database", 
-	'disp' => "{Z3950Download.database_disp}",
-	'desc' => "{Z3950Download.database}",
-      	'type' => "string",
-	'reqd' => "yes"},
-      { 'name' => "find", 
-	'disp' => "{Z3950Download.find_disp}",
-	'desc' => "{Z3950Download.find}",
-	'type' => "string",
-	'deft' => "",
-	'reqd' => "yes"},
-      { 'name' => "max_records", 
-	'disp' => "{Z3950Download.max_records_disp}",
-	'desc' => "{Z3950Download.max_records}",
-	'type' => "int",
-	'deft' => "500",
-	'reqd' => "no"}];
+    @SRWDownload::ISA = ('Z3950Download');
+}
+
+my $arguments;
 
 my $options = { 'name'     => "SRWDownload",
 		'desc'     => "{SRWDownload.desc}",
 		'abstract' => "no",
-		'inherits' => "yes",
-		'args'     => $arguments };
+		'inherits' => "yes"
+		};
 
 
@@ -89,5 +59,5 @@
     if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
 
-    my $self = (defined $hashArgOptLists)? new BasDownload($getlist,$inputargs,$hashArgOptLists): new BasDownload($getlist,$inputargs);
+    my $self = (defined $hashArgOptLists)? new Z3950Download($getlist,$inputargs,$hashArgOptLists): new Z3950Download($getlist,$inputargs);
 
     if ($self->{'info_only'}) {
@@ -106,86 +76,53 @@
     my ($hashGeneralOptions) = @_;
     my ($strOpen,$strBase,$strFind,$strResponse,$intAmount,$intMaxRecords,$strRecords);
+
+    my $url = $self->{'url'};
+
     print STDERR "<<Defined Maximum>>\n";
 
-    my $url = $self->{'url'};
-
-    open2(*YAZOUT, *YAZIN, "yaz-client $url")
-	or die "can't open pipe to yaz-client: $!";
+    my  $yaz =  $self->{'yaz'}; 
+  
+    my $childpid = open2(*YAZOUT, *YAZIN, $yaz)
+  	or (print STDERR "<<Finished>>\n" and die "can't open pipe to yaz-client: $!");
+
+    $self->{'YAZOUT'} = *YAZOUT;
+    $self->{'YAZIN'} = *YAZIN;
+
+    $strOpen = $self->open_connection("open $url"); 
+
+    if (!$strOpen) {
+        print STDERR "Cannot connect to $url\n"; 
+        print STDERR "<<Finished>>\n";  
+	return 0;
+    }
 
     print STDERR "Opening connection to \"$self->{'url'}\"\n";
-    #$strOpen = &run_command_with_output("open $self->{'url'}");
     print STDERR "Access database: \"$self->{'database'}\"\n";
-    &run_command_without_output("base $self->{'database'}");
-    &run_command_without_output("querytype prefix");
+    $self->run_command_without_output("base $self->{'database'}");
+    $self->run_command_without_output("querytype prefix");
     print STDERR "Searching for keyword: \"$self->{'find'}\"\n";
-    $intAmount = &findAmount($self->{'find'});
+    $intAmount =$self->findAmount($self->{'find'});
 
     if($intAmount <= 0)
     {
 	($intAmount == -1)? 
-	    print STDERR "Unexpected format, Parsing operation can not be performed\n" : 
+	    print STDERR "Something wrong with the arguments,downloading can not be performed\n" : 
 	    print STDERR "No Record is found\n";
+	    print STDERR "<<Finished>>\n";
 	return 0;
     }
     $intMaxRecords = ($self->{'max_records'} > $intAmount)? $intAmount : $self->{'max_records'};
     print STDERR "<<Total number of record(s):$intMaxRecords>>\n";
-    $strRecords = &getRecords($intMaxRecords);
-    print STDERR $strRecords;
-    &saveRecords($self,$strRecords,$hashGeneralOptions->{'cache_dir'},$intMaxRecords);
-    print STDERR "Closing connection\n";
+  
+    $strRecords = $self->getRecords($intMaxRecords);
+  
+    $self->saveRecords($strRecords,$hashGeneralOptions->{'cache_dir'},$intMaxRecords);
+    print STDERR "Closing connection...\n";
+    print STDERR "<<Finished>>\n";
     close(YAZOUT);
     close(YAZIN);
-    return 1;
-}
-
-sub findAmount
-{
-    my($strFindTarget) = @_;
-    my $strResponse = &run_command_with_output("find $strFindTarget");
-
-    return ($strResponse =~ m/^Number of hits: (\d+)/m)? $1:-1;  
-}
-
-sub getRecords
-{
-    my ($intMaxRecords) = @_;
-    my ($strShow,$intStartNumber,$strResponse,$strRecords,$intRecordsLeft);
-
-    $intStartNumber = 1;
-    $intRecordsLeft = $intMaxRecords;
-    while ($intRecordsLeft > 0)
-    {
-	if($intRecordsLeft > 50)
-	{
-	    print STDERR "<<Done:50>>\n";
-	    print STDERR "Yaz is Gathering records: $intStartNumber - ".($intStartNumber+49)."\n";
-	    
-	    $strShow = "show $intStartNumber+50";
-	    $intStartNumber = $intStartNumber + 50;
-	    $intRecordsLeft = $intRecordsLeft - 50;
-	}
-	else
-	{
-	    print STDERR "<<Done:".($intRecordsLeft).">>\n";
-	    print STDERR "Yaz is Gathering records: $intStartNumber - ".($intStartNumber+$intRecordsLeft-1)."\n";
-	    $strShow = "show $intStartNumber+$intRecordsLeft";
-	    $intRecordsLeft = 0;
-	}
-	
-	$strResponse = &run_command_with_output($strShow);
-
-        ## need to change this
-               
-        print STDERR $strResponse;
-	
-        if($strResponse =~  m/pos=[\d]*(.*)>\n/s)
-	{
-           $strRecords .= "$1>\n";
-           
-           $strRecords =~ s/pos=[\d]*(.*)?\n//g;            
-	}
-    }
-    return $strRecords;
-}
+   return 1;
+}
+
 
 sub saveRecords
@@ -195,20 +132,22 @@
     # setup directories
     # Currently only gather the MARC format
-    my $strFileName = &generateFileName($self,$intMaxRecords);
-    my $strFileURL = "$strOutputDir/$self->{'host'}/$strFileName.marc";
-    $strFileURL =~ s/:/\//g;
-
+    $strRecords ="<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<collection>$strRecords</collection>";  
+    my $strFileName =  $self->generateFileName($intMaxRecords);
+    my $host = $self->{'host'};
+    $host =~ s/http:\/\//srw\//;
+    $strOutputDir  =~ s/"//g; 
+    my $strFileURL = "$strOutputDir/$host/$strFileName.xml";
+  
     # prepare subdirectory for record (if needed)
-    my ($strSubDirPath,$unused) = dirFileSplit($strFileURL);
+
+    my ($strSubDirPath,$unused) = $self->dirFileSplit($strFileURL);
     &util::mk_all_dir($strSubDirPath);
 
     my $ds = &util::get_dirsep();
-    my $strOutputFile = &util::filename_cat($strOutputDir,$self->{'host'},"$strFileName.marc");
-    $strOutputFile =~ s/:/$ds/g;
-
-    print STDERR "Saving records to \"$strOutputFile\"\n";
+  
+    print STDERR "Saving records to \"$strFileURL\"\n";
 
     # save record 
-    open (ZOUT,">$strOutputFile")
+    open (ZOUT,">$strFileURL")
 	|| die "Unable to save oai metadata record: $!\n";
     print ZOUT $strRecords;
@@ -216,71 +155,47 @@
 }
 
-sub run_command_with_output
-{
-    my ($strCMD) = @_;
-    return &run_command($strCMD,"^Elapsed:.*\$");
-}
-
-sub run_command_without_output
-{
-    my ($strCMD) = @_;
-    &run_command($strCMD);
-}
-
-sub run_command
-{
-    my ($strCMD,$strStopRE) = @_;
+sub get{
+   my ($self,$strShow,$numRecord) = @_;  
+
+   $self->run_command($strShow);  
+   
+   my $strFullOutput="";
+   my $count=0;
+   my $readRecord = 0;
+   
+   while (my $strLine = <YAZOUT>)
+   {
     
-    print YAZIN "$strCMD\n";
-    if (!defined $strStopRE){return "";}
-    else
-    {
-	my $strFullOutput;
-    	while (my $strLine = <YAZOUT>)
-	{
-            $strFullOutput .= $strLine;
-	    if($strLine =~ m/$strStopRE/){return $strFullOutput;}
-	}
-    }
-}
-
-sub url_information
-{
-   my ($self) = shift (@_);
+       return $strFullOutput if ($count >= $numRecord);
+
+       return $strFullOutput if($strLine =~ m/^HTTP ERROR/i);
+
+       if ($strLine =~ m/pos=[\d]*/i ){ 
+           $count++;
+	   $readRecord = 1;
+	   next;  
+       }
+      
+       next if(!$readRecord);
+
+       $strFullOutput .= $strLine;     
+   }
+ 
+}
+
+sub url_information{
+   my ($self) = @_;
 
    my $url = $self->{'url'};
 
-  open2(*YAZOUT, *YAZIN, "yaz-client $url") or die "can't open pipe to yaz-client: $!";
-
-   my $strFullOutput="";
-
-   while (my $strLine = <YAZOUT>)
-	{
-          $strFullOutput .= $strLine;  
-      }
-
-   return  $strFullOutput;
-}
-
-sub generateFileName
-{
-    my ($self,$intMaxRecords) = @_;
-    my $strFileName = ($self->{'database'})."_".($self->{'find'})."_".($intMaxRecords);
-}
-
-sub dirFileSplit
-{
-    my ($strFile) = @_;
-
-    my @aryDirs = split("/",$strFile);
-    my $strLocalFile = pop(@aryDirs);
-    my $strSubDirs = join("/",@aryDirs);
-
-    return ($strSubDirs,$strLocalFile);
+   $url =~ s#http://##; 
+
+  return $self->SUPER::url_information($url);  
+
 }
 
 sub error
 {
-    my ($strFunctionName,$strError) = @_;
+    my ($self, $strFunctionName,$strError) = @_;
     {
 	print STDERR "Error occoured in SRWDownload.pm\n".
Index: /trunk/gsdl/perllib/downloaders/WebDownload.pm
===================================================================
--- /trunk/gsdl/perllib/downloaders/WebDownload.pm	(revision 12464)
+++ /trunk/gsdl/perllib/downloaders/WebDownload.pm	(revision 12465)
@@ -99,29 +99,24 @@
     my ($hashGeneralOptions) = @_;
 
-    # TODO: the checking for Wget is still not complete, we need to 
-    #       check if the proxy has been set or not, and whether the 
-    #       connection has been established.
-    # Checking if the wget has been well setup
-    # &WgetDownload::checkWgetSetup($self,$hashGeneralOptions->{'gli_call'});
-
+  
     # Download options
-    my $strOptions = &generateOptionsString($self);
-    my $strWgetOptions = &WgetDownload::getWgetOptions($self);
-    
-   
+    my $strOptions = $self->generateOptionsString();
+    my $strWgetOptions = $self->getWgetOptions();
+      
     # Setup the command for using wget
     my $cmdWget = "-N -k -x -t 2 -P ".$hashGeneralOptions->{"cache_dir"}." $strWgetOptions $strOptions ".$self->{'url'};
     
-    print "**************".$cmdWget."\n";
-
     # Download the web pages
     # print "Strat download from $self->{'url'}...\n";
-    
     print STDERR "<<Undefined Maximum>>\n";
-    my $strResponse = &WgetDownload::useWget($cmdWget,1);
-    #if ($strResponse ne ""){print "$strResponse\n";}
-    
-    # print "Finish download from $self->{'url'}...\n";
-
+
+    my $strResponse = $self->useWget($cmdWget,1);
+ 
+    # if ($strResponse ne ""){print "$strResponse\n";}
+    
+    print STDERR "Finish download from $self->{'url'}\n";
+
+    print STDERR "<<Finished>>\n";
+  
     return 1;
 }
@@ -146,5 +141,5 @@
     else
     {
-	&error("setupOptions","Incorrect Depth is defined!!\n");
+	$self->error("setupOptions","Incorrect Depth is defined!!\n");
     }
 
@@ -174,10 +169,18 @@
 {
     my ($self) = shift (@_);
-    if(!defined $self){ die "System Error: No \$self defined for url_information in WebDownload\n";}
-    
-    my $strBaseCMD = "-q -O - \"$self->{'url'}\"";
-
-    my $strIdentifyText = &WgetDownload::useWget($strBaseCMD);
-    
+
+    my $strOptions = $self->getWgetOptions();
+
+    my $strBaseCMD = $strOptions." -q -O - \"$self->{'url'}\"";
+
+  
+    my $strIdentifyText = $self->useWget($strBaseCMD);
+    
+    if (!defined $strIdentifyText or $strIdentifyText eq ""  ){
+	print STDERR "Server information is unavailable.\n";
+	print STDERR "<<Finished>>\n";
+         return;  
+    }
+
     while ($strIdentifyText =~ m/^(.*)<title>(.*?)<\/title>(.*)$/s)
     {
@@ -185,10 +188,11 @@
 	print STDERR "Page Title: $2\n";
     }
-
+  
     while ($strIdentifyText =~ m/^(.*)<meta (.*?)>(.*)$/s)
     {
 	$strIdentifyText = $1.$3;
 	my $strTempString = $2;
-	print STDERR "Meta Information:\n";
+	print STDERR "\n";
+
 	while($strTempString =~ m/(.*?)=[\"|\'](.*?)[\"|\'](.*?)$/s)
 	{
@@ -202,9 +206,13 @@
 	    $strMetaName =~ s/^([" "])+//m;
 	    $strMetaContain =~ s/^([" "])+//m;
-	    print STDERR "\t$strMetaName: $strMetaContain\n";
-	    
+              
+	    print STDERR "$strMetaName: $strMetaContain\n\n";
+           
 	}
-	print STDERR "\n";
-    }
+
+    }
+
+    print STDERR "<<Finished>>\n";
+
 }
 
@@ -222,2 +230,3 @@
 
 1;
+
Index: /trunk/gsdl/perllib/downloaders/WgetDownload.pm
===================================================================
--- /trunk/gsdl/perllib/downloaders/WgetDownload.pm	(revision 12464)
+++ /trunk/gsdl/perllib/downloaders/WgetDownload.pm	(revision 12465)
@@ -34,4 +34,5 @@
 use BasDownload;
 use strict;
+use IPC::Open2;
 
 sub BEGIN {
@@ -40,9 +41,4 @@
 
 my $arguments = 
-#    [ { 'name' => "url", 
-#	'desc' => "{WgetDownload.url}",         
-#	'type' => "string",
-#	'deft' => "",
-#	'reqd' => "yes"},
      [ { 'name' => "proxy_on", 
 	'desc' => "{WgetDownload.proxy_on}",
@@ -125,29 +121,41 @@
     # Setup .wgetrc by using $self->{'proxy_host'} and $self->{'proxy_port'}
     # Test if the connection is succeful. If the connection wasn't succeful then ask user to supply username and password.
-    
 
-    # TODO: How to test run if the proxy setup is working correctly?? 
-    # Use -spider to test whether the connection is working correctly.
-    # TODO: Ask user to supply username and password.
-    # Try to use the .wgetrc to setup the user name and password
-    
 }
 
 sub useWget
 {
-    my ($cmdWget,$blnShow) = @_;
+    my ($self, $cmdWget,$blnShow) = @_;
 
-    my $strReadIn = "";
-    my $strLine;
+    my ($os,$strReadIn,$strLine,$command);
+   
+    $os = $ENV{'GSDLOS'};
+ 
+  
+    if ($os =~ /windows/i){
+	$command = "\"$ENV{'GSDLHOME'}\\bin\\windows\\wget\" $cmdWget |";
+    }
+    else{
+        $command = "$ENV{'GSDLHOME'}/packages/wget/wget-1.9/src/wget $cmdWget |";  
+    }
 
-    open (WIN,"$ENV{'GSDLHOME'}/packages/wget/wget-1.9/src/wget $cmdWget|") || die "wget request failed: $!\n";
+    
+
+   open(*WIN,$command) || die "wget request failed: $!\n";
+
+    
+
     while (defined($strLine=<WIN>))
     {
+
+
 	if($blnShow)
 	{
-	    print "$strReadIn\n";
+	    print STDERR "$strReadIn\n";
 	}
+
 	$strReadIn .= $strLine;
     }
+
     close(WIN);
 
Index: /trunk/gsdl/perllib/downloaders/Z3950Download.pm
===================================================================
--- /trunk/gsdl/perllib/downloaders/Z3950Download.pm	(revision 12464)
+++ /trunk/gsdl/perllib/downloaders/Z3950Download.pm	(revision 12465)
@@ -40,6 +40,4 @@
     @Z3950Download::ISA = ('BasDownload');
 }
-
-local (*YAZOUT, *YAZIN); 
 
 my $arguments = 
@@ -97,4 +95,14 @@
     # Must set $self->{'url'}, since GLI use $self->{'url'} to calculate the log file name!
     $self->{'url'} = $self->{'host'}.":".$self->{'port'};
+
+     my $os = $ENV{'GSDLOS'};
+
+    if ($os !~ /windows/) {
+	$self->{'yaz'} = "$ENV{'GSDLHOME'}/packages/yaz/yaz-2.1.4/client/yaz-client"; 
+    }
+    else{
+	$self->{'yaz'} = "$ENV{'GSDLHOME'}/bin/windows/yaz-client"; 
+    }
+    
     return bless $self, $class;
 
@@ -106,78 +114,117 @@
     my ($hashGeneralOptions) = @_;
     my ($strOpen,$strBase,$strFind,$strResponse,$intAmount,$intMaxRecords,$strRecords);
+
+    my $url = $self->{'url'};
+ 
     print STDERR "<<Defined Maximum>>\n";
  
-    my $url = $self->{'url'};
     print STDERR "Opening connection to $url\n";
      
-    my $childpid = open2(*YAZOUT, *YAZIN, "yaz-client")
-	or die "can't open pipe to yaz-client: $!";
- 
-    $strOpen = &run_command_with_output("open $url");  
+    my  $yaz = $self->{'yaz'};
+ 
+    my $childpid = open2(*YAZOUT, *YAZIN, $yaz)
+	or (print STDERR "<<Finished>>\n" and die "can't open pipe to yaz-client: $!");
+    $self->{'YAZOUT'} = *YAZOUT;
+    $self->{'YAZIN'} = *YAZIN;
+
+    $strOpen = $self->open_connection("open $url");  
+
+    if (!$strOpen) {
+        print STDERR "Cannot connect to $url\n"; 
+        print STDERR "<<Finished>>\n";  
+	return 0;
+    }
+
     print STDERR "Access database: \"$self->{'database'}\"\n";
-    &run_command_without_output("base $self->{'database'}");
+    $self->run_command_without_output("base $self->{'database'}");
     print STDERR "Searching for keyword: \"$self->{'find'}\"\n";
-    $intAmount = &findAmount($self->{'find'});
+    $intAmount = $self->findAmount($self->{'find'});
 
     if($intAmount <= 0)
     {
 	($intAmount == -1)? 
-	    print STDERR "Unexpected format, Parsing operation can not be performed\n" : 
-	    print STDERR "No Record is found\n";
+	    print STDERR "Something wrong with the arguments,downloading can not be performed\n": 
+		print STDERR "No Record is found\n";
+	print STDERR "<<Finished>>\n";
 	return 0;
     }
     $intMaxRecords = ($self->{'max_records'} > $intAmount)? $intAmount : $self->{'max_records'};
     print STDERR "<<Total number of record(s):$intMaxRecords>>\n";
-    $strRecords = &getRecords($intMaxRecords);
-    print STDERR $strRecords;
-    &saveRecords($self,$strRecords,$hashGeneralOptions->{'cache_dir'},$intMaxRecords);
+    $strRecords = "Records: $intMaxRecords\n".$self->getRecords($intMaxRecords);
+   
+    $self->saveRecords($strRecords,$hashGeneralOptions->{'cache_dir'},$intMaxRecords);
     print STDERR "Closing connection...\n";
+    print STDERR "<<Finished>>\n";
+
     close(YAZOUT);
     close(YAZIN);
-    waitpid($childpid, 0);
     return 1;
 }
 
+sub open_connection{
+  my ($self,$strCommand) =  (@_);
+  
+  $self->run_command($strCommand);  
+
+  my $out = $self->{'YAZOUT'}; 
+
+  $_ = <$out>;
+  
+  return (/Connecting...OK/i)? 1: 0; 
+ 
+}
+
 sub findAmount
 {
+    my ($self) = shift (@_);
     my($strFindTarget) = @_;
-    my $strResponse = &run_command_with_output("find $strFindTarget");
-    return ($strResponse =~ m/^Number of hits: (\d+)/m)? $1:-1;    
+    my $strResponse = $self->run_command_with_output("find $strFindTarget","^Number of hits:");
+   return ($strResponse =~ m/^Number of hits: (\d+)/m)? $1:-1;    
 }
 
 sub getRecords
 {
+    my ($self) = shift (@_);
     my ($intMaxRecords) = @_;
-    my ($strShow,$intStartNumber,$strResponse,$strRecords,$intRecordsLeft);
+    my ($strShow,$intStartNumber,$numRecords,$strResponse,$strRecords,$intRecordsLeft);
 
     $intStartNumber = 1;
     $intRecordsLeft = $intMaxRecords;
+    $numRecords = 0;
+    $strResponse ="";
+
     while ($intRecordsLeft > 0)
     {
 	if($intRecordsLeft > 50)
 	{
-	    print STDERR "<<Done:50>>\n";
+	   
 	    print STDERR "Yaz is Gathering records: $intStartNumber - ".($intStartNumber+49)."\n";
-	    
+	    $numRecords = 50;
 	    $strShow = "show $intStartNumber+50";
 	    $intStartNumber = $intStartNumber + 50;
 	    $intRecordsLeft = $intRecordsLeft - 50;
+             
 	}
 	else
 	{
-	    print STDERR "<<Done:".($intRecordsLeft).">>\n";
+	    $numRecords = $intRecordsLeft;
 	    print STDERR "Yaz is Gathering records: $intStartNumber - ".($intStartNumber+$intRecordsLeft-1)."\n";
 	    $strShow = "show $intStartNumber+$intRecordsLeft";
 	    $intRecordsLeft = 0;
+	   
+           }
+	
+	$strResponse .= $self->get($strShow,$numRecords);
+         
+	if ($strResponse eq ""){
+	    print STDERR "<<ERROR: failed to get $numRecords records>>\n";
 	}
+	else{
+	    print STDERR "<<Done:$numRecords>>\n";
+	}
+    }
+
+    return  "$strResponse\n";
 	
-	$strResponse = &run_command_with_output($strShow);
-       
-	if($strResponse =~ m/Records: (\d*?)\n(.*?)nextResultSetPosition = (\d*?)\n/s)
-	{
-	    $strRecords .= $2;
-	}
-    }
-    return $strRecords;
 }
 
@@ -188,53 +235,93 @@
     # setup directories
     # Currently only gather the MARC format
-    my $strFileName = &generateFileName($self,$intMaxRecords);
+    my $strFileName = $self->generateFileName($intMaxRecords);
+
+    $strOutputDir  =~ s/"//g;
+
     my $strFileURL = "$strOutputDir/$self->{'host'}/$strFileName.marc";
-    $strFileURL =~ s/:/\//g;
-
+  
     # prepare subdirectory for record (if needed)
-    my ($strSubDirPath,$unused) = dirFileSplit($strFileURL);
+    my ($strSubDirPath,$unused) = $self->dirFileSplit($strFileURL);
+ 
     &util::mk_all_dir($strSubDirPath);
 
     my $ds = &util::get_dirsep();
     my $strOutputFile = &util::filename_cat($strOutputDir,$self->{'host'},"$strFileName.marc");
-    $strOutputFile =~ s/:/$ds/g;
-
+   
     print STDERR "Saving records to \"$strOutputFile\"\n";
 
     # save record 
     open (ZOUT,">$strOutputFile")
-	|| die "Unable to save oai metadata record: $!\n";
+	|| die "Unable to save Z3950 record: $!\n";
     print ZOUT $strRecords;
     close(ZOUT);
 }
 
+
 sub run_command_with_output
 {
+    my ($self,$strCMD,$strStopRE) =@_;
+   
+    $self->run_command($strCMD); 
+   
+    return $self->get_output($strStopRE);
+ 
+}
+
+sub get{
+   my ($self,$strShow,$numRecord) = @_;  
+
+   $self->run_command($strShow);  
+   
+   my $strFullOutput="";
+   my $count=0;
+   my $readRecord = 0;
+   
+   while (my $strLine = <YAZOUT>)
+   {
+   
+       if ($strLine =~ m/Records: ([\d]*)/i ){ 
+	   $readRecord = 1;
+	   next;  
+       }
+      
+      return $strFullOutput if ($strLine =~ m/nextResultSetPosition|Not connected/i);
+        
+      next if(!$readRecord);
+      
+      $strFullOutput .= $strLine;     
+  }
+   
+}
+
+sub run_command_without_output
+{
+     my ($self) = shift (@_);
     my ($strCMD) = @_;
-    
-    return &run_command($strCMD,"^Elapsed:.*\$");
-}
-
-sub run_command_without_output
-{
-    my ($strCMD) = @_;
-
-    &run_command($strCMD);
+
+    $self->run_command($strCMD);
 }
 
 sub run_command
 {
-    my ($strCMD,$strStopRE) = @_;
-    
-  
-    print YAZIN "$strCMD\n";
+    my ($self,$strCMD) = @_; 
+ 
+    my $input = $self->{'YAZIN'};
+
+    print $input "$strCMD\n";  
+}
+
+sub get_output{
+    my ($self,$strStopRE) = @_;  
+
     if (!defined $strStopRE){return "";}
     else
     {
 	my $strFullOutput;
-	while (my $strLine = <YAZOUT>)
+        my $output = $self->{'YAZOUT'};
+	while (my $strLine = <$output>)
 	{
-	    $strFullOutput .= $strLine;
-	    if($strLine =~ m/$strStopRE/){return $strFullOutput;}
+           $strFullOutput .= $strLine;   
+	   if($strLine =~ m/^$strStopRE|Not connected/i){return $strFullOutput;}
 	}
     }
@@ -245,11 +332,13 @@
     my ($self,$intMaxRecords) = @_;
     my $strFileName = ($self->{'database'})."_".($self->{'find'})."_".($intMaxRecords);
+ 
 }
 
 sub dirFileSplit
 {
-    my ($strFile) = @_;
-
-    my @aryDirs = split("/",$strFile);
+    my ($self,$strFile) = @_;
+
+    my @aryDirs = split("[/\]",$strFile);
+   
     my $strLocalFile = pop(@aryDirs);
     my $strSubDirs = join("/",@aryDirs);
@@ -258,7 +347,47 @@
 }
 
+sub url_information
+{
+   my ($self,$url) = @_;
+
+   $url = $self->{'url'} unless defined $url;
+
+   my  $yaz =  $self->{'yaz'};
+   
+   my $childpid = open2(*YAZOUT, *YAZIN, $yaz)
+       or die "can't open pipe to yaz-client: $!";
+  
+   $self->{'YAZOUT'} = *YAZOUT;
+   $self->{'YAZIN'} = *YAZIN;
+
+   my $strOpen = $self->open_connection("open $url"); 
+   
+    if (!$strOpen) {
+        print STDERR "Cannot connect to $url\n"; 
+        print STDERR "<<Finished>>\n";  
+	return 0;
+    }
+
+   
+   $strOpen = $self->run_command_with_output("open $url","^Options");  
+
+
+   $strOpen =~ s/Z> //g;
+   $strOpen =~ s/Elapsed:.*//g;
+
+   print STDERR $strOpen; 
+
+   print STDERR "<<Finished>>\n"; 
+
+   close(YAZOUT);
+   close(YAZIN);
+
+   return 0;
+
+}
+
 sub error
 {
-    my ($strFunctionName,$strError) = @_;
+    my ($self,$strFunctionName,$strError) = @_;
     {
 	print STDERR "Error occoured in Z3950Download.pm\n".
