Index: trunk/gsdl/perllib/downloaders/BasDownload.pm
===================================================================
--- trunk/gsdl/perllib/downloaders/BasDownload.pm	(revision 11783)
+++ trunk/gsdl/perllib/downloaders/BasDownload.pm	(revision 11783)
@@ -0,0 +1,235 @@
+###########################################################################
+#
+# BasDownload.pm -- base class for all the Download modules
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 2006 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package BasDownload;
+
+eval {require bytes};
+
+use strict;
+no strict 'subs';
+
+use gsprintf 'gsprintf';
+use printusage;
+
+# suppress the annoying "subroutine redefined" warning that various
+# gets cause under perl 5.6
+$SIG{__WARN__} = sub {warn($_[0]) unless ($_[0] =~ /Subroutine\s+\S+\sredefined/)};
+
+my $arguments = [];
+
+my $options = { 'name'     => "BasDownload",
+		'desc'     => "{BasDownload.desc}",
+		'abstract' => "yes",
+		'inherits' => "no" };
+
+sub new 
+{
+    my $class = shift (@_);
+    my ($downloadlist,$args,$hashArgOptLists) = @_;
+    push(@$downloadlist, $class);
+    my $strDownloadName = (defined $downloadlist->[0]) ? $downloadlist->[0] : $class;
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my $self = {};
+    $self->{'download_type'} = $strDownloadName;
+    $self->{'option_list'} = $hashArgOptLists->{"OptList"};
+    $self->{"info_only"} = 0;
+
+    # Check if gsdlinfo is in the argument list or not - if it is, don't parse 
+    # the args, just return the object.  
+    foreach my $strArg (@{$args})
+    {
+	if($strArg eq "-gsdlinfo")
+	{
+	    $self->{"info_only"} = 1;
+	    return bless $self, $class;
+	}
+    }
+    
+    delete $self->{"info_only"};
+    
+    if(!parse2::parse($args,$hashArgOptLists->{"ArgList"},$self))
+    {
+	my $classTempClass = bless $self, $class;
+	print STDERR "<BadDownload d=$self->{'download_name'}>\n";
+	&gsprintf(STDERR, "\n{BasDownload.bad_general_option}\n", $self->{'download_name'});
+	$classTempClass->print_txt_usage("");  # Use default resource bundle
+	die "\n";
+    }
+
+    return bless $self, $class;
+
+}
+
+sub download
+{
+    my ($self) = shift (@_);
+    my ($hashGeneralOptions) = @_;
+    &error("download","No download specified for $hashGeneralOptions->{download_mode}.\n");
+}
+
+
+sub print_xml_usage
+{
+    my $self = shift(@_);
+    my $header = shift(@_);
+    
+    # XML output is always in UTF-8
+    gsprintf::output_strings_in_UTF8;
+
+    if ($header) {
+	&PrintUsage::print_xml_header("download");
+    }
+    $self->print_xml();
+}
+
+
+sub print_xml
+{
+    my $self = shift(@_);
+    my $optionlistref = $self->{'option_list'};
+    my @optionlist = @$optionlistref;
+    my $downloadoptions = shift(@$optionlistref);
+    return if (!defined($downloadoptions));
+
+    gsprintf(STDERR, "<DownloadInfo>\n");
+    gsprintf(STDERR, "  <Name>$downloadoptions->{'name'}</Name>\n");
+    my $desc = gsprintf::lookup_string($downloadoptions->{'desc'});
+    $desc =~ s/</&amp;lt;/g; # doubly escaped
+    $desc =~ s/>/&amp;gt;/g;
+    gsprintf(STDERR, "  <Desc>$desc</Desc>\n");
+    gsprintf(STDERR, "  <Abstract>$downloadoptions->{'abstract'}</Abstract>\n");
+    gsprintf(STDERR, "  <Inherits>$downloadoptions->{'inherits'}</Inherits>\n");
+    gsprintf(STDERR, "  <Arguments>\n");
+    if (defined($downloadoptions->{'args'})) {
+	&PrintUsage::print_options_xml($downloadoptions->{'args'});
+    }
+    gsprintf(STDERR, "  </Arguments>\n");
+    # Recurse up the download hierarchy
+    $self->print_xml();
+    gsprintf(STDERR, "</DownloadInfo>\n");
+}
+
+
+sub print_txt_usage
+{
+    my $self = shift(@_);
+
+    # Print the usage message for a download (recursively)
+    my $descoffset = $self->determine_description_offset(0);
+    $self->print_download_usage($descoffset, 1);
+}
+
+sub determine_description_offset
+{
+    my $self = shift(@_);
+    my $maxoffset = shift(@_);
+
+    my $optionlistref = $self->{'option_list'};
+    my @optionlist = @$optionlistref;
+    my $downloadoptions = pop(@$optionlistref);
+    return $maxoffset if (!defined($downloadoptions));
+
+    # Find the length of the longest option string of this download
+    my $downloadargs = $downloadoptions->{'args'};
+    if (defined($downloadargs)) {
+	my $longest = &PrintUsage::find_longest_option_string($downloadargs);
+	if ($longest > $maxoffset) {
+	    $maxoffset = $longest;
+	}
+    }
+
+    # Recurse up the download hierarchy
+    $maxoffset = $self->determine_description_offset($maxoffset);
+    $self->{'option_list'} = \@optionlist;
+    return $maxoffset;
+}
+
+
+sub print_download_usage
+{
+    my $self = shift(@_);
+    my $descoffset = shift(@_);
+    my $isleafclass = shift(@_);
+
+    my $optionlistref = $self->{'option_list'};
+    my @optionlist = @$optionlistref;
+    my $downloadoptions = shift(@$optionlistref);
+    return if (!defined($downloadoptions));
+
+    my $downloadname = $downloadoptions->{'name'};
+    my $downloadargs = $downloadoptions->{'args'};
+    my $downloaddesc = $downloadoptions->{'desc'};
+
+    # Produce the usage information using the data structure above
+    if ($isleafclass) {
+	if (defined($downloaddesc)) {
+	    gsprintf(STDERR, "$downloaddesc\n\n");
+	}
+	gsprintf(STDERR, " {common.usage}: download $downloadname [{common.options}]\n\n");
+    }
+
+    # Display the download options, if there are some
+    if (defined($downloadargs)) {
+	# Calculate the column offset of the option descriptions
+	my $optiondescoffset = $descoffset + 2;  # 2 spaces between options & descriptions
+
+	if ($isleafclass) {
+	    gsprintf(STDERR, " {common.specific_options}:\n");
+	}
+	else {
+	    gsprintf(STDERR, " {common.general_options}:\n", $downloadname);
+	}
+
+	# Display the download options
+	&PrintUsage::print_options_txt($downloadargs, $optiondescoffset);
+    }
+
+    # Recurse up the download hierarchy
+    $self->print_download_usage($descoffset, 0);
+    $self->{'option_list'} = \@optionlist;
+}
+
+sub url_information
+{
+    my ($self) = @_;
+    print STDERR "There is no extra information provided for this Download.\n";
+    return "";
+}
+
+sub error
+{
+      my ($strFunctionName,$strError) = @_;
+    {
+	print "Error occoured in BasDownload.pm\n".
+	    "In Function: ".$strFunctionName."\n".
+	    "Error Message: ".$strError."\n";
+	exit(-1);
+    }  
+}
+
+1;
Index: trunk/gsdl/perllib/downloaders/OAIDownload.pm
===================================================================
--- trunk/gsdl/perllib/downloaders/OAIDownload.pm	(revision 11783)
+++ trunk/gsdl/perllib/downloaders/OAIDownload.pm	(revision 11783)
@@ -0,0 +1,377 @@
+###########################################################################
+#
+# WebDownload.pm -- base class for all the import plugins
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package OAIDownload;
+
+eval {require bytes};
+
+# suppress the annoying "subroutine redefined" warning that various
+# plugins cause under perl 5.6
+$SIG{__WARN__} = sub {warn($_[0]) unless ($_[0] =~ /Subroutine\s+\S+\sredefined/)};
+
+use strict;
+
+use WgetDownload;
+use XMLParser;
+
+use IO::File;
+use POSIX qw(tmpnam);
+
+sub BEGIN {
+    @OAIDownload::ISA = ('WgetDownload');
+}
+
+my $arguments = 
+    [ { 'name' => "url", 
+	'disp' => "{OAIDownload.url_disp}",
+	'desc' => "{OAIDownload.url}",
+	'type' => "string",
+	'reqd' => "yes"},
+      { 'name' => "set", 
+	'disp' => "{OAIDownload.set_disp}",
+	'desc' => "{OAIDownload.set}",
+	'type' => "string",
+	'reqd' => "no"},
+      { 'name' => "get_doc",
+	'disp' => "{OAIDownload.qet_doc_disp}",
+	'desc' => "{OAIDownload.get_doc}",
+	'type' => "flag",
+	'reqd' => "no"},
+      { 'name' => "max_records", 
+	'disp' => "{OAIDownload.max_records_disp}",
+	'desc' => "{OAIDownload.max_records}",
+	'type' => "int",
+	'deft' => "500",
+	'range' => "1,",
+	'reqd' => "no"} ];
+
+my $options = { 'name'     => "OAIDownload",
+		'desc'     => "{OAIDownload.desc}",
+		'abstract' => "no",
+		'inherits' => "yes",
+		'args'     => $arguments };
+
+my $self;
+
+sub new 
+{
+    my ($class) = shift (@_);
+    my ($getlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$getlist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    $self = (defined $hashArgOptLists)? new WgetDownload($getlist,$inputargs,$hashArgOptLists): new WgetDownload($getlist,$inputargs);
+
+    if ($self->{'info_only'}) {
+	# don't worry about any options etc
+	return bless $self, $class;
+    }
+
+    my $parser = new XML::Parser('Style' => 'Stream',
+				 'Handlers' => {'Char' => \&Char,
+						'Start' => \&OAI_StartTag,
+						'End' => \&OAI_EndTag
+						});
+    $self->{'parser'} = $parser;
+    
+    return bless $self, $class;
+}
+
+sub download
+{
+    my ($self) = shift (@_);
+    my ($hashGeneralOptions) = @_;
+
+    # Checking if the wget has been well setup
+    # &WgetDownload::checkWgetSetup($self,$hashGeneralOptions->{'gli_call'});
+    
+    my $strOutputDir = $hashGeneralOptions->{"cache_dir"};
+    my $strBasURL = $self->{'url'};
+    my $intMaxRecords = $self->{'max_records'};
+    my $blnDownloadDoc = $self->{'get_doc'};
+
+    print STDERR "<<Defined Maximum>>\n";
+    my $strIDs = &getOAIIDs($self,$strBasURL);
+    if($strIDs eq "")
+    {
+	print STDERR "Error: No ID being found\n";
+	return 0;
+    }
+    my $aryIDs = &parseOAIIDs($strIDs);
+    my $intIDs = 0;
+    if($self->{'max_records'} < scalar(@$aryIDs))
+    {
+	$intIDs = $self->{'max_records'};
+    }
+    else
+    {
+	$intIDs = scalar(@$aryIDs);
+    }
+    print STDERR "<<Total number of record(s):$intIDs>>\n";
+
+    &getOAIRecords($aryIDs, $strOutputDir, $strBasURL, $intMaxRecords, $blnDownloadDoc);
+
+    return 1;
+}
+
+sub getOAIIDs
+{
+    my ($self,$strBasURL) = @_;
+    my ($cmdWget);
+    print STDERR  "Gathering OAI identifiers.....\n";
+    if($self->{'set'} ne "")
+    {
+	$cmdWget = "-q -O - \"$strBasURL?verb=ListIdentifiers&metadataPrefix=oai_dc&set=$self->{'set'}\" ";
+    }
+    else
+    {
+	$cmdWget = "-q -O - \"$strBasURL?verb=ListIdentifiers&metadataPrefix=oai_dc\" ";
+    }
+    my $strIDs =  &WgetDownload::useWget($cmdWget);
+    return $strIDs;
+}
+
+sub parseOAIIDs
+{   
+    my ($strIDs) = @_;
+
+    print STDERR "Parsing OAI identifiers.....\n";
+    $strIDs =~ s/^.*?<identifier>/<identifier>/s;
+    $strIDs =~ s/^(.*<\/identifier>).*$/$1/s;
+
+    my @aryIDs = ();
+
+    while ($strIDs =~ m/<identifier>(.*?)<\/identifier>(.*)$/s)
+    {
+	$strIDs = $2;
+	push(@aryIDs,$1);
+    }
+    
+    return \@aryIDs;
+} 
+
+sub dirFileSplit
+{
+    my ($strFile) = @_;
+
+    my @aryDirs = split("/",$strFile);
+    my $strLocalFile = pop(@aryDirs);
+    my $strSubDirs = join("/",@aryDirs);
+
+    return ($strSubDirs,$strLocalFile);
+}
+
+sub getOAIDoc
+{
+    my ($strRecord, $strSubDirPath) = @_;
+    
+    print  STDERR "Gathering source documents.....\n";
+    # look out for identifier tag in metadata section
+    if ($strRecord =~ m/<metadata>(.*)<\/metadata>/s)
+    {
+	my $strMetaTag = $1;
+	
+	if ($strMetaTag =~ m/<(dc:)?identifier>(.*?)<\/(dc:)?identifier>/s)
+	{
+	    my $strDocURL = $2;
+
+	    my ($unused,$strDocFile) = dirFileSplit($strDocURL);
+
+	    my $strSoureDirPath = &util::filename_cat($strSubDirPath,"srcdocs");
+	    &util::mk_dir($strSoureDirPath)  if (!-e "$strSoureDirPath");
+	    
+	    my $strFullDocFilePath = &util::filename_cat($strSoureDirPath,$strDocFile);
+	    
+	    my $wget_cmd = "-q -O $strFullDocFilePath \"$strDocURL\"";
+
+	    my $strResponse =  &WgetDownload::useWget($wget_cmd,1);
+
+	    if($strResponse ne "")
+	    {
+		print STDERR "Error occured while retriving OAI souce documents: $strResponse\n";
+		exit(-1);
+	    }
+
+	    $strRecord =~ s/<metadata>(.*?)<(dc:)?identifier>$strDocURL<\/(dc:)?identifier>(.*?)<\/metadata>/<metadata>$1<OrigURL>$strDocURL<\/OrigURL>\n   <identifier>srcdocs\/$strDocFile<\/identifier>$4<\/metadata>/s;
+	}
+	else
+	{
+	    print  STDERR "\tNo souce document URL is specified in the OAI record (No (dc:)?identifier is provided)\n";
+	}
+    }
+    else
+    {
+	print  STDERR "\tNo souce document URL is specified in the OAI record (No metadata field is provided)\n";
+    }
+    
+}
+
+sub getOAIRecords
+{
+    my ($aryIDs, $strOutputDir, $strBasURL, $intMaxRecords, $blnDownloadDoc) = @_;
+
+    my $intDocCounter = 0;
+
+    foreach my $strID ( @$aryIDs)
+    {
+	print  STDERR "Gathering OAI record with ID:$strID.....\n";
+	# wget it;
+	my $cmdWget= "-q -O - \"$strBasURL?verb=GetRecord&metadataPrefix=oai_dc&identifier=$strID\"";
+	my $strRecord =  &WgetDownload::useWget($cmdWget);
+
+	# setup directories
+	my $strFileURL = "$strOutputDir/$strID.oai";
+	$strFileURL =~ s/:/\//g;
+
+	# prepare subdirectory for record (if needed)
+	my ($strSubDirPath,$unused) = dirFileSplit($strFileURL);
+	&util::mk_all_dir($strSubDirPath);
+
+	my $ds = &util::get_dirsep();
+	my $strOutputFile = &util::filename_cat($strOutputDir,"$strID.oai");
+	$strOutputFile =~ s/:/$ds/g;
+
+	if($blnDownloadDoc)
+	{
+	    &getOAIDoc($strRecord,$strSubDirPath);
+	}
+
+	# save record 
+	open (OAIOUT,">$strOutputFile")
+	    || die "Unable to save oai metadata record: $!\n";
+	print OAIOUT $strRecord;
+	close(OAIOUT);
+
+	$intDocCounter ++;
+	print STDERR "<<Done>>\n";
+	last if ($intDocCounter >= $intMaxRecords);
+    }
+    ($intDocCounter >= $intMaxRecords) ? 
+	print  STDERR "Reach maximum download records, use -max_records to set the maximum.\n": 
+	print  STDERR "Complete download meta record from $strBasURL\n";
+
+}
+
+sub url_information
+{
+    my ($self) = shift (@_);
+    if(!defined $self){ die "System Error: No \$self defined for url_information in OAIDownload\n";}
+    
+    my $strBaseCMD = "-q -O - \"$self->{'url'}?_OPTS_\"";
+ 
+    my $strIdentify = "verb=Identify";
+    my $strListSets = "verb=ListSets";
+
+    my $strIdentifyCMD = $strBaseCMD;
+    $strIdentifyCMD =~ s/_OPTS_/$strIdentify/;	
+
+    my $strIdentifyText = &WgetDownload::useWget($strIdentifyCMD);
+
+    print STDERR "General information:\n";
+    $self->parse_xml($strIdentifyText);
+
+    my $strListSetCMD = $strBaseCMD;
+    $strListSetCMD =~ s/_OPTS_/$strListSets/;	
+    my $strListSetsText = &WgetDownload::useWget($strListSetCMD);
+    print STDERR "List Information:\n";
+    $self->parse_xml($strListSetsText);
+}
+
+sub parse_xml
+{    
+    my ($self) = shift (@_);
+    my ($strOutputText) = @_;
+    my ($name,$fh);
+    
+    #Open a temporary file to store OAI information, and store the information to the temp file
+    do {$name = tmpnam()}
+	until $fh = IO::File->new($name, O_RDWR|O_CREAT|O_EXCL);
+    print $fh $strOutputText;
+    close($fh);
+
+    $self->{'temp_file_name'} = $name;
+
+    eval {
+	$self->{'parser'}->parsefile("$name");
+    };
+    
+    if ($@) {
+	die "OAI: $name is not a well formed XML file ($@)\n";
+    }
+}
+
+END{
+    if($self->{'info'})
+    {
+	unlink($self->{'temp_file_name'}) or die "Could not unlink $self->{'temp_file_name'}: $!";
+    }
+}
+
+# This Char function overrides the one in XML::Parser::Stream to overcome a
+# problem where $expat->{Text} is treated as the return value, slowing
+# things down significantly in some cases.
+sub Char {   
+    use bytes;  # Necessary to prevent encoding issues with XML::Parser 2.31+
+    $_[0]->{'Text'} .= $_[1];
+    if ((defined $self->{'subfield'} && ($self->{'subfield'} ne ""))) {
+	$self->{'text'} .= $_[1];
+	$self->{'text'} =~ s/[\n]|[" "]//g;
+	if($self->{'text'} ne "")
+	{	    
+	    print STDERR " $self->{'subfield'}:($self->{'text'})\n";
+	}
+    }
+    return undef;
+}
+
+sub OAI_StartTag
+{
+    my ($expat, $element, %attr) = @_;
+    $self->{'subfield'} = $element;
+}
+
+sub OAI_EndTag
+{
+    my ($expat, $element) = @_;
+    $self->{'text'} = "";
+    $self->{'subfield'} = "";
+}
+
+sub error
+{
+    my ($strFunctionName,$strError) = @_;
+    {
+	print "Error occoured in OAIDownload.pm\n".
+	    "In Function:".$strFunctionName."\n".
+	    "Error Message:".$strError."\n";
+	exit(-1);
+    }
+}
+
+
+
+1;
Index: trunk/gsdl/perllib/downloaders/SRWDownload.pm
===================================================================
--- trunk/gsdl/perllib/downloaders/SRWDownload.pm	(revision 11783)
+++ trunk/gsdl/perllib/downloaders/SRWDownload.pm	(revision 11783)
@@ -0,0 +1,293 @@
+###########################################################################
+#
+# SRWDownload.pm -- base class for all the import plugins
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package SRWDownload;
+
+eval {require bytes};
+
+# suppress the annoying "subroutine redefined" warning that various
+# plugins cause under perl 5.6
+$SIG{__WARN__} = sub {warn($_[0]) unless ($_[0] =~ /Subroutine\s+\S+\sredefined/)};
+
+use strict;
+
+use BasDownload;
+use IPC::Open2;
+
+sub BEGIN {
+    @SRWDownload::ISA = ('BasDownload');
+}
+
+local (*YAZOUT, *YAZIN); 
+
+# args same as Z3950Download at the moment - should it be based on that??
+my $arguments = 
+    [  { 'name' => "host", 
+	'disp' => "{Z3950Download.host_disp}",
+	'desc' => "{Z3950Download.host}",
+	'type' => "string",
+	'reqd' => "yes"},
+      { 'name' => "port", 
+	'disp' => "{Z3950Download.port_disp}",
+	'desc' => "{Z3950Download.port}",
+	'type' => "string",
+	'reqd' => "yes"},
+      { 'name' => "database", 
+	'disp' => "{Z3950Download.database_disp}",
+	'desc' => "{Z3950Download.database}",
+      	'type' => "string",
+	'reqd' => "yes"},
+      { 'name' => "find", 
+	'disp' => "{Z3950Download.find_disp}",
+	'desc' => "{Z3950Download.find}",
+	'type' => "string",
+	'deft' => "",
+	'reqd' => "yes"},
+      { 'name' => "max_records", 
+	'disp' => "{Z3950Download.max_records_disp}",
+	'desc' => "{Z3950Download.max_records}",
+	'type' => "int",
+	'deft' => "500",
+	'reqd' => "no"}];
+
+my $options = { 'name'     => "SRWDownload",
+		'desc'     => "{SRWDownload.desc}",
+		'abstract' => "no",
+		'inherits' => "yes",
+		'args'     => $arguments };
+
+
+sub new 
+{
+    my ($class) = shift (@_);
+    my ($getlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$getlist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my $self = (defined $hashArgOptLists)? new BasDownload($getlist,$inputargs,$hashArgOptLists): new BasDownload($getlist,$inputargs);
+
+    if ($self->{'info_only'}) {
+	# don't worry about any options etc
+	return bless $self, $class;
+    }
+
+    # Must set $self->{'url'}, since GLI use $self->{'url'} to calculate the log file name!
+    $self->{'url'} = $self->{'host'}.":".$self->{'port'};
+    return bless $self, $class;
+}
+
+sub download
+{
+    my ($self) = shift (@_);
+    my ($hashGeneralOptions) = @_;
+    my ($strOpen,$strBase,$strFind,$strResponse,$intAmount,$intMaxRecords,$strRecords);
+    print STDERR "<<Defined Maximum>>\n";
+
+    my $url = $self->{'url'};
+
+    open2(*YAZOUT, *YAZIN, "yaz-client $url")
+	or die "can't open pipe to yaz-client: $!";
+
+    print STDERR "Opening connection to \"$self->{'url'}\"\n";
+    #$strOpen = &run_command_with_output("open $self->{'url'}");
+    print STDERR "Access database: \"$self->{'database'}\"\n";
+    &run_command_without_output("base $self->{'database'}");
+    &run_command_without_output("querytype prefix");
+    print STDERR "Searching for keyword: \"$self->{'find'}\"\n";
+    $intAmount = &findAmount($self->{'find'});
+
+    if($intAmount <= 0)
+    {
+	($intAmount == -1)? 
+	    print STDERR "Unexpected format, Parsing operation can not be performed\n" : 
+	    print STDERR "No Record is found\n";
+	return 0;
+    }
+    $intMaxRecords = ($self->{'max_records'} > $intAmount)? $intAmount : $self->{'max_records'};
+    print STDERR "<<Total number of record(s):$intMaxRecords>>\n";
+    $strRecords = &getRecords($intMaxRecords);
+    print STDERR $strRecords;
+    &saveRecords($self,$strRecords,$hashGeneralOptions->{'cache_dir'},$intMaxRecords);
+    print STDERR "Closing connection\n";
+    close(YAZOUT);
+    close(YAZIN);
+    return 1;
+}
+
+sub findAmount
+{
+    my($strFindTarget) = @_;
+    my $strResponse = &run_command_with_output("find $strFindTarget");
+
+    return ($strResponse =~ m/^Number of hits: (\d+)/m)? $1:-1;  
+}
+
+sub getRecords
+{
+    my ($intMaxRecords) = @_;
+    my ($strShow,$intStartNumber,$strResponse,$strRecords,$intRecordsLeft);
+
+    $intStartNumber = 1;
+    $intRecordsLeft = $intMaxRecords;
+    while ($intRecordsLeft > 0)
+    {
+	if($intRecordsLeft > 50)
+	{
+	    print STDERR "<<Done:50>>\n";
+	    print STDERR "Yaz is Gathering records: $intStartNumber - ".($intStartNumber+49)."\n";
+	    
+	    $strShow = "show $intStartNumber+50";
+	    $intStartNumber = $intStartNumber + 50;
+	    $intRecordsLeft = $intRecordsLeft - 50;
+	}
+	else
+	{
+	    print STDERR "<<Done:".($intRecordsLeft).">>\n";
+	    print STDERR "Yaz is Gathering records: $intStartNumber - ".($intStartNumber+$intRecordsLeft-1)."\n";
+	    $strShow = "show $intStartNumber+$intRecordsLeft";
+	    $intRecordsLeft = 0;
+	}
+	
+	$strResponse = &run_command_with_output($strShow);
+
+        ## need to change this
+               
+        print STDERR $strResponse;
+	
+        if($strResponse =~  m/pos=[\d]*(.*)>\n/s)
+	{
+           $strRecords .= "$1>\n";
+           
+           $strRecords =~ s/pos=[\d]*(.*)?\n//g;            
+	}
+    }
+    return $strRecords;
+}
+
+sub saveRecords
+{
+    my ($self,$strRecords,$strOutputDir,$intMaxRecords) = @_;
+
+    # setup directories
+    # Currently only gather the MARC format
+    my $strFileName = &generateFileName($self,$intMaxRecords);
+    my $strFileURL = "$strOutputDir/$self->{'host'}/$strFileName.marc";
+    $strFileURL =~ s/:/\//g;
+
+    # prepare subdirectory for record (if needed)
+    my ($strSubDirPath,$unused) = dirFileSplit($strFileURL);
+    &util::mk_all_dir($strSubDirPath);
+
+    my $ds = &util::get_dirsep();
+    my $strOutputFile = &util::filename_cat($strOutputDir,$self->{'host'},"$strFileName.marc");
+    $strOutputFile =~ s/:/$ds/g;
+
+    print STDERR "Saving records to \"$strOutputFile\"\n";
+
+    # save record 
+    open (ZOUT,">$strOutputFile")
+	|| die "Unable to save oai metadata record: $!\n";
+    print ZOUT $strRecords;
+    close(ZOUT);
+}
+
+sub run_command_with_output
+{
+    my ($strCMD) = @_;
+    return &run_command($strCMD,"^Elapsed:.*\$");
+}
+
+sub run_command_without_output
+{
+    my ($strCMD) = @_;
+    &run_command($strCMD);
+}
+
+sub run_command
+{
+    my ($strCMD,$strStopRE) = @_;
+    
+    print YAZIN "$strCMD\n";
+    if (!defined $strStopRE){return "";}
+    else
+    {
+	my $strFullOutput;
+    	while (my $strLine = <YAZOUT>)
+	{
+            $strFullOutput .= $strLine;
+	    if($strLine =~ m/$strStopRE/){return $strFullOutput;}
+	}
+    }
+}
+
+sub url_information
+{
+   my ($self) = shift (@_);
+
+   my $url = $self->{'url'};
+
+  open2(*YAZOUT, *YAZIN, "yaz-client $url") or die "can't open pipe to yaz-client: $!";
+
+   my $strFullOutput="";
+
+   while (my $strLine = <YAZOUT>)
+	{
+          $strFullOutput .= $strLine;  
+      }
+
+   return  $strFullOutput;
+}
+
+sub generateFileName
+{
+    my ($self,$intMaxRecords) = @_;
+    my $strFileName = ($self->{'database'})."_".($self->{'find'})."_".($intMaxRecords);
+}
+
+sub dirFileSplit
+{
+    my ($strFile) = @_;
+
+    my @aryDirs = split("/",$strFile);
+    my $strLocalFile = pop(@aryDirs);
+    my $strSubDirs = join("/",@aryDirs);
+
+    return ($strSubDirs,$strLocalFile);
+}
+
+sub error
+{
+    my ($strFunctionName,$strError) = @_;
+    {
+	print STDERR "Error occoured in SRWDownload.pm\n".
+	    "In Function:".$strFunctionName."\n".
+	    "Error Message:".$strError."\n";
+	exit(-1);
+    }
+}
+
+1;
Index: trunk/gsdl/perllib/downloaders/WebDownload.pm
===================================================================
--- trunk/gsdl/perllib/downloaders/WebDownload.pm	(revision 11783)
+++ trunk/gsdl/perllib/downloaders/WebDownload.pm	(revision 11783)
@@ -0,0 +1,223 @@
+###########################################################################
+#
+# WebDownload.pm -- base class for all the import plugins
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package WebDownload;
+
+eval {require bytes};
+
+# suppress the annoying "subroutine redefined" warning that various
+# plugins cause under perl 5.6
+$SIG{__WARN__} = sub {warn($_[0]) unless ($_[0] =~ /Subroutine\s+\S+\sredefined/)};
+
+use WgetDownload;
+
+sub BEGIN {
+    @WebDownload::ISA = ('WgetDownload');
+}
+
+use strict; # every perl program should have this!
+no strict 'refs'; # make an exception so we can use variables as filehandles
+
+my $arguments = 
+    [ { 'name' => "url", 
+	'disp' => "{WebDownload.url_disp}",
+	'desc' => "{WebDownload.url}",
+	'type' => "string",
+	'reqd' => "yes"},
+      { 'name' => "depth", 
+	'disp' => "{WebDownload.depth_disp}",
+	'desc' => "{WebDownload.depth}",
+	'type' => "int",
+	'deft' => "0",
+	"range" => "0,",
+	'reqd' => "no"},
+      { 'name' => "below", 
+	'disp' => "{WebDownload.below_disp}",
+	'desc' => "{WebDownload.below}",
+	'type' => "flag",
+	'reqd' => "no"},
+      { 'name' => "within", 
+	'disp' => "{WebDownload.within_disp}",
+	'desc' => "{WebDownload.within}",
+	'type' => "flag",
+	'reqd' => "no"},
+      { 'name' => "html_only", 
+	'disp' => "{WebDownload.html_only_disp}",
+	'desc' => "{WebDownload.html_only}",
+	'type' => "flag",
+	'reqd' => "no"}
+      ];
+
+my $options = { 'name'     => "WebDownload",
+		'desc'     => "{WebDownload.desc}",
+		'abstract' => "no",
+		'inherits' => "yes",
+		'args'     => $arguments };
+
+
+my $self;
+
+sub new 
+{
+    my ($class) = shift (@_);
+    my ($getlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$getlist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+     my $self = (defined $hashArgOptLists)? new WgetDownload($getlist,$inputargs,$hashArgOptLists): new WgetDownload($getlist,$inputargs);
+    
+    return bless $self, $class;
+}
+
+sub download
+{
+    my ($self) = shift (@_);
+    my ($hashGeneralOptions) = @_;
+
+    # TODO: the checking for Wget is still not complete, we need to 
+    #       check if the proxy has been set or not, and whether the 
+    #       connection has been established.
+    # Checking if the wget has been well setup
+    # &WgetDownload::checkWgetSetup($self,$hashGeneralOptions->{'gli_call'});
+
+    # Download options
+    my $strOptions = &generateOptionsString($self);
+    my $strWgetOptions = &WgetDownload::getWgetOptions($self);
+    
+   
+    # Setup the command for using wget
+    my $cmdWget = "-N -k -x -t 2 -P ".$hashGeneralOptions->{"cache_dir"}." $strWgetOptions $strOptions ".$self->{'url'};
+    
+    print "**************".$cmdWget."\n";
+
+    # Download the web pages
+    # print "Strat download from $self->{'url'}...\n";
+    
+    print STDERR "<<Undefined Maximum>>\n";
+    my $strResponse = &WgetDownload::useWget($cmdWget,1);
+    #if ($strResponse ne ""){print "$strResponse\n";}
+    
+    # print "Finish download from $self->{'url'}...\n";
+
+    return 1;
+}
+
+sub generateOptionsString
+{
+    my ($self) = @_;
+    my $strOptions;
+
+    (defined $self) || &error("generateOptionsString","No \$self is defined!!\n");
+    (defined $self->{'depth'})|| &error("generateOptionsString","No depth is defined!!\n");
+    
+   
+    if($self->{'depth'} == 0)
+    {
+	$strOptions .= " ";
+    }
+    elsif($self->{'depth'} > 0)
+    {
+	$strOptions .= "-r -l ".$self->{'depth'}." ";
+    }
+    else
+    {
+	&error("setupOptions","Incorrect Depth is defined!!\n");
+    }
+
+    if($self->{'below'})
+    {
+	$strOptions .="-np ";
+    }
+
+     if($self->{'html_only'})
+    {
+	$strOptions .="-A .html,.htm,.shm,.shtml,.asp,.php,.cgi,*?*=* ";
+    }
+    else{
+
+	$strOptions .="-p ";
+    }
+
+    if (!$self->{'within'}){
+	$strOptions .="-H ";
+    }
+
+    return $strOptions;
+  
+}
+
+sub url_information
+{
+    my ($self) = shift (@_);
+    if(!defined $self){ die "System Error: No \$self defined for url_information in WebDownload\n";}
+    
+    my $strBaseCMD = "-q -O - \"$self->{'url'}\"";
+
+    my $strIdentifyText = &WgetDownload::useWget($strBaseCMD);
+    
+    while ($strIdentifyText =~ m/^(.*)<title>(.*?)<\/title>(.*)$/s)
+    {
+	$strIdentifyText = $1.$3;
+	print STDERR "Page Title: $2\n";
+    }
+
+    while ($strIdentifyText =~ m/^(.*)<meta (.*?)>(.*)$/s)
+    {
+	$strIdentifyText = $1.$3;
+	my $strTempString = $2;
+	print STDERR "Meta Information:\n";
+	while($strTempString =~ m/(.*?)=[\"|\'](.*?)[\"|\'](.*?)$/s)
+	{
+	    # Store the infromation in to variable, since next time when we do 
+	    # regular expression, we will lost all the $1, $2, $X....
+	    $strTempString = $3;
+	    my $strMetaName = $1;
+	    my $strMetaContain = $2;
+	    
+	    # Take out the extra space in the beginning of the string.
+	    $strMetaName =~ s/^([" "])+//m;
+	    $strMetaContain =~ s/^([" "])+//m;
+	    print STDERR "\t$strMetaName: $strMetaContain\n";
+	    
+	}
+	print STDERR "\n";
+    }
+}
+
+
+sub error
+{
+    my ($strFunctionName,$strError) = @_;
+    {
+	print "Error occoured in WebDownload.pm\n".
+	    "In Function:".$strFunctionName."\n".
+	    "Error Message:".$strError."\n";
+	exit(-1);
+    }
+}
+
+1;
Index: trunk/gsdl/perllib/downloaders/WgetDownload.pm
===================================================================
--- trunk/gsdl/perllib/downloaders/WgetDownload.pm	(revision 11783)
+++ trunk/gsdl/perllib/downloaders/WgetDownload.pm	(revision 11783)
@@ -0,0 +1,178 @@
+###########################################################################
+#
+# WgetDownload.pm -- Download base module that handles calling Wget
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 2006 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package WgetDownload;
+
+eval {require bytes};
+
+# suppress the annoying "subroutine redefined" warning that various
+# plugins cause under perl 5.6
+$SIG{__WARN__} = sub {warn($_[0]) unless ($_[0] =~ /Subroutine\s+\S+\sredefined/)};
+
+use BasDownload;
+use strict;
+
+sub BEGIN {
+    @WgetDownload::ISA = ('BasDownload');
+}
+
+my $arguments = 
+#    [ { 'name' => "url", 
+#	'desc' => "{WgetDownload.url}",         
+#	'type' => "string",
+#	'deft' => "",
+#	'reqd' => "yes"},
+     [ { 'name' => "proxy_on", 
+	'desc' => "{WgetDownload.proxy_on}",
+	'type' => "flag",
+	'reqd' => "no",
+	'hiddengli' => "yes"},
+      { 'name' => "proxy_host",  
+	'desc' => "{WgetDownload.proxy_host}",         
+	'type' => "string",
+	'reqd' => "no",
+	'hiddengli' => "yes"},
+      { 'name' => "proxy_port", 
+	'desc' => "{WgetDownload.proxy_port}",         
+	'type' => "string",
+	'reqd' => "no",
+	'hiddengli' => "yes"},
+      { 'name' => "user_name",  
+	'desc' => "{WgetDownload.user_name}",         
+	'type' => "string",
+	'reqd' => "no",
+	'hiddengli' => "yes"},
+      { 'name' => "user_password", 
+	'desc' => "{WgetDownload.user_password}",         
+	'type' => "string",
+	'reqd' => "no",
+	'hiddengli' => "yes"}];
+
+my $options = { 'name'     => "WgetDownload",
+		'desc'     => "{WgetDownload.desc}",
+		'abstract' => "yes",
+		'inherits' => "yes",
+		'args'     => $arguments };
+
+
+sub new {
+    my ($class) = shift (@_);
+    my ($getlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$getlist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my $self = (defined $hashArgOptLists)? new BasDownload($getlist,$inputargs,$hashArgOptLists): new BasDownload($getlist,$inputargs);
+
+    return bless $self, $class;
+}
+
+sub checkWgetSetup
+{
+    my ($self,$blnGliCall) = @_;
+    #TODO: proxy detection??
+    
+    if((!$blnGliCall) && $self->{'proxy_on'})
+    {
+	&checkProxySetup($self);
+    }
+    &checkURL($self); 
+}
+
+sub getWgetOptions
+{
+    my ($self) = @_;
+    my $strOptions = "";
+    
+    if($self->{'proxy_on'} && $self->{'proxy_host'} && $self->{'proxy_port'} 
+       && $self->{'user_name'} && $self->{'user_password'} 
+       )
+    {
+	$strOptions = " -e httpproxy=$self->{'proxy_host'}:$self->{'proxy_port'} --proxy-user=$self->{'user_name'}"." --proxy-passwd=$self->{'user_password'} -Y on ";
+    }
+
+    return $strOptions;
+}
+
+# Checking for proxy setup: proxy server, proxy port, proxy username and password.
+sub checkProxySetup
+{
+    my ($self) = @_;
+    ($self->{'proxy_on'}) || &error("checkProxySetup","The proxy is not on? How could that be happening?");
+    # Setup .wgetrc by using $self->{'proxy_host'} and $self->{'proxy_port'}
+    # Test if the connection is succeful. If the connection wasn't succeful then ask user to supply username and password.
+    
+
+    # TODO: How to test run if the proxy setup is working correctly?? 
+    # Use -spider to test whether the connection is working correctly.
+    # TODO: Ask user to supply username and password.
+    # Try to use the .wgetrc to setup the user name and password
+    
+}
+
+sub useWget
+{
+    my ($cmdWget,$blnShow) = @_;
+
+    my $strReadIn = "";
+    my $strLine;
+
+    open (WIN,"$ENV{'GSDLHOME'}/packages/wget/wget-1.9/src/wget $cmdWget|") || die "wget request failed: $!\n";
+    while (defined($strLine=<WIN>))
+    {
+	if($blnShow)
+	{
+	    print "$strReadIn\n";
+	}
+	$strReadIn .= $strLine;
+    }
+    close(WIN);
+
+    return $strReadIn;
+}
+
+# TODO: Check if the URL is valid?? Not sure what should be in this function yet!!
+sub checkURL
+{
+    my ($self) = @_;
+    if ($self->{'url'} eq "")
+    {
+	&error("checkURL","no URL is specified!! Please specifies the URL for downloading.");
+    }
+}
+
+sub error
+{
+    my ($strFunctionName,$strError) = @_;
+    {
+	print "Error occoured in WgetDownload.pm\n".
+	    "In Function:".$strFunctionName."\n".
+	    "Error Message:".$strError."\n";
+	exit(-1);
+    }
+}
+
+1;
Index: trunk/gsdl/perllib/downloaders/Z3950Download.pm
===================================================================
--- trunk/gsdl/perllib/downloaders/Z3950Download.pm	(revision 11783)
+++ trunk/gsdl/perllib/downloaders/Z3950Download.pm	(revision 11783)
@@ -0,0 +1,271 @@
+###########################################################################
+#
+# WebDownload.pm -- base class for all the import plugins
+# A component of the Greenstone digital library software
+# from the New Zealand Digital Library Project at the 
+# University of Waikato, New Zealand.
+#
+# Copyright (C) 1999 New Zealand Digital Library Project
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+###########################################################################
+
+package Z3950Download;
+
+eval {require bytes};
+
+# suppress the annoying "subroutine redefined" warning that various
+# plugins cause under perl 5.6
+$SIG{__WARN__} = sub {warn($_[0]) unless ($_[0] =~ /Subroutine\s+\S+\sredefined/)};
+
+use strict;
+
+use BasDownload;
+use IPC::Open2;
+
+sub BEGIN {
+    @Z3950Download::ISA = ('BasDownload');
+}
+
+local (*YAZOUT, *YAZIN); 
+
+my $arguments = 
+    [  { 'name' => "host", 
+	'disp' => "{Z3950Download.host_disp}",
+	'desc' => "{Z3950Download.host}",
+	'type' => "string",
+	'reqd' => "yes"},
+      { 'name' => "port", 
+	'disp' => "{Z3950Download.port_disp}",
+	'desc' => "{Z3950Download.port}",
+	'type' => "string",
+	'reqd' => "yes"},
+      { 'name' => "database", 
+	'disp' => "{Z3950Download.database_disp}",
+	'desc' => "{Z3950Download.database}",
+      	'type' => "string",
+	'reqd' => "yes"},
+      { 'name' => "find", 
+	'disp' => "{Z3950Download.find_disp}",
+	'desc' => "{Z3950Download.find}",
+	'type' => "string",
+	'deft' => "",
+	'reqd' => "yes"},
+      { 'name' => "max_records", 
+	'disp' => "{Z3950Download.max_records_disp}",
+	'desc' => "{Z3950Download.max_records}",
+	'type' => "int",
+	'deft' => "500",
+	'reqd' => "no"}];
+
+my $options = { 'name'     => "Z3950Download",
+		'desc'     => "{Z3950Download.desc}",
+		'abstract' => "no",
+		'inherits' => "yes",
+		'args'     => $arguments };
+
+
+sub new 
+{
+    my ($class) = shift (@_);
+    my ($getlist,$inputargs,$hashArgOptLists) = @_;
+    push(@$getlist, $class);
+
+    if(defined $arguments){ push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});}
+    if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
+
+    my $self = (defined $hashArgOptLists)? new BasDownload($getlist,$inputargs,$hashArgOptLists): new BasDownload($getlist,$inputargs);
+
+    if ($self->{'info_only'}) {
+	# don't worry about any options etc
+	return bless $self, $class;
+    }
+
+    # Must set $self->{'url'}, since GLI use $self->{'url'} to calculate the log file name!
+    $self->{'url'} = $self->{'host'}.":".$self->{'port'};
+    return bless $self, $class;
+
+}
+
+sub download
+{
+    my ($self) = shift (@_);
+    my ($hashGeneralOptions) = @_;
+    my ($strOpen,$strBase,$strFind,$strResponse,$intAmount,$intMaxRecords,$strRecords);
+    print STDERR "<<Defined Maximum>>\n";
+ 
+    my $url = $self->{'url'};
+    print STDERR "Opening connection to $url\n";
+     
+    my $childpid = open2(*YAZOUT, *YAZIN, "yaz-client")
+	or die "can't open pipe to yaz-client: $!";
+ 
+    $strOpen = &run_command_with_output("open $url");  
+    print STDERR "Access database: \"$self->{'database'}\"\n";
+    &run_command_without_output("base $self->{'database'}");
+    print STDERR "Searching for keyword: \"$self->{'find'}\"\n";
+    $intAmount = &findAmount($self->{'find'});
+
+    if($intAmount <= 0)
+    {
+	($intAmount == -1)? 
+	    print STDERR "Unexpected format, Parsing operation can not be performed\n" : 
+	    print STDERR "No Record is found\n";
+	return 0;
+    }
+    $intMaxRecords = ($self->{'max_records'} > $intAmount)? $intAmount : $self->{'max_records'};
+    print STDERR "<<Total number of record(s):$intMaxRecords>>\n";
+    $strRecords = &getRecords($intMaxRecords);
+    print STDERR $strRecords;
+    &saveRecords($self,$strRecords,$hashGeneralOptions->{'cache_dir'},$intMaxRecords);
+    print STDERR "Closing connection...\n";
+    close(YAZOUT);
+    close(YAZIN);
+    waitpid($childpid, 0);
+    return 1;
+}
+
+sub findAmount
+{
+    my($strFindTarget) = @_;
+    my $strResponse = &run_command_with_output("find $strFindTarget");
+    return ($strResponse =~ m/^Number of hits: (\d+)/m)? $1:-1;    
+}
+
+sub getRecords
+{
+    my ($intMaxRecords) = @_;
+    my ($strShow,$intStartNumber,$strResponse,$strRecords,$intRecordsLeft);
+
+    $intStartNumber = 1;
+    $intRecordsLeft = $intMaxRecords;
+    while ($intRecordsLeft > 0)
+    {
+	if($intRecordsLeft > 50)
+	{
+	    print STDERR "<<Done:50>>\n";
+	    print STDERR "Yaz is Gathering records: $intStartNumber - ".($intStartNumber+49)."\n";
+	    
+	    $strShow = "show $intStartNumber+50";
+	    $intStartNumber = $intStartNumber + 50;
+	    $intRecordsLeft = $intRecordsLeft - 50;
+	}
+	else
+	{
+	    print STDERR "<<Done:".($intRecordsLeft).">>\n";
+	    print STDERR "Yaz is Gathering records: $intStartNumber - ".($intStartNumber+$intRecordsLeft-1)."\n";
+	    $strShow = "show $intStartNumber+$intRecordsLeft";
+	    $intRecordsLeft = 0;
+	}
+	
+	$strResponse = &run_command_with_output($strShow);
+       
+	if($strResponse =~ m/Records: (\d*?)\n(.*?)nextResultSetPosition = (\d*?)\n/s)
+	{
+	    $strRecords .= $2;
+	}
+    }
+    return $strRecords;
+}
+
+sub saveRecords
+{
+    my ($self,$strRecords,$strOutputDir,$intMaxRecords) = @_;
+
+    # setup directories
+    # Currently only gather the MARC format
+    my $strFileName = &generateFileName($self,$intMaxRecords);
+    my $strFileURL = "$strOutputDir/$self->{'host'}/$strFileName.marc";
+    $strFileURL =~ s/:/\//g;
+
+    # prepare subdirectory for record (if needed)
+    my ($strSubDirPath,$unused) = dirFileSplit($strFileURL);
+    &util::mk_all_dir($strSubDirPath);
+
+    my $ds = &util::get_dirsep();
+    my $strOutputFile = &util::filename_cat($strOutputDir,$self->{'host'},"$strFileName.marc");
+    $strOutputFile =~ s/:/$ds/g;
+
+    print STDERR "Saving records to \"$strOutputFile\"\n";
+
+    # save record 
+    open (ZOUT,">$strOutputFile")
+	|| die "Unable to save oai metadata record: $!\n";
+    print ZOUT $strRecords;
+    close(ZOUT);
+}
+
+sub run_command_with_output
+{
+    my ($strCMD) = @_;
+    
+    return &run_command($strCMD,"^Elapsed:.*\$");
+}
+
+sub run_command_without_output
+{
+    my ($strCMD) = @_;
+
+    &run_command($strCMD);
+}
+
+sub run_command
+{
+    my ($strCMD,$strStopRE) = @_;
+    
+  
+    print YAZIN "$strCMD\n";
+    if (!defined $strStopRE){return "";}
+    else
+    {
+	my $strFullOutput;
+	while (my $strLine = <YAZOUT>)
+	{
+	    $strFullOutput .= $strLine;
+	    if($strLine =~ m/$strStopRE/){return $strFullOutput;}
+	}
+    }
+}
+
+sub generateFileName
+{
+    my ($self,$intMaxRecords) = @_;
+    my $strFileName = ($self->{'database'})."_".($self->{'find'})."_".($intMaxRecords);
+}
+
+sub dirFileSplit
+{
+    my ($strFile) = @_;
+
+    my @aryDirs = split("/",$strFile);
+    my $strLocalFile = pop(@aryDirs);
+    my $strSubDirs = join("/",@aryDirs);
+
+    return ($strSubDirs,$strLocalFile);
+}
+
+sub error
+{
+    my ($strFunctionName,$strError) = @_;
+    {
+	print STDERR "Error occoured in Z3950Download.pm\n".
+	    "In Function:".$strFunctionName."\n".
+	    "Error Message:".$strError."\n";
+	exit(-1);
+    }
+}
+
+1;
