Index: /gsdl/trunk/perllib/downloaders/OAIDownload.pm
===================================================================
--- /gsdl/trunk/perllib/downloaders/OAIDownload.pm	(revision 16790)
+++ /gsdl/trunk/perllib/downloaders/OAIDownload.pm	(revision 16791)
@@ -66,4 +66,10 @@
 	'type' => "flag",
 	'reqd' => "no"},
+      { 'name' => "get_doc_exts",
+	'disp' => "{OAIDownload.get_doc_exts_disp}",
+	'desc' => "{OAIDownload.get_doc_exts}",
+	'type' => "string",
+	'deft' => "doc,pdf,ppt",
+	'reqd' => "no"},
       { 'name' => "max_records", 
 	'disp' => "{OAIDownload.max_records_disp}",
@@ -80,7 +86,7 @@
 		'args'     => $arguments };
 
-my $self;
-
-my $strWgetOptions="";
+##my $self;
+
+#### my $strWgetOptions="";
 
 sub new 
@@ -93,5 +99,5 @@
     if(defined $options) { push(@{$hashArgOptLists->{"OptList"}},$options)};
 
-    $self = (defined $hashArgOptLists)? new WgetDownload($getlist,$inputargs,$hashArgOptLists): new WgetDownload($getlist,$inputargs);
+    my $self = (defined $hashArgOptLists)? new WgetDownload($getlist,$inputargs,$hashArgOptLists): new WgetDownload($getlist,$inputargs);
 
     if ($self->{'info_only'}) {
@@ -101,4 +107,5 @@
 
     my $parser = new XML::Parser('Style' => 'Stream',
+                                 'PluginObj' => $self,
 				 'Handlers' => {'Char' => \&Char,
 						'Start' => \&OAI_StartTag,
@@ -113,4 +120,19 @@
     }
     
+
+    # set up hashmap for individual items in get_doc_exts
+    # to make testing for matches easier
+
+    $self->{'lookup_exts'} = {};
+    my $get_doc_exts = $self->{'get_doc_exts'};
+
+    if ((defined $get_doc_exts) && ($get_doc_exts ne "")) {
+	my @exts = split(/,\s*/,$get_doc_exts);
+	foreach my $e (@exts) {
+	    $self->{'lookup_exts'}->{lc($e)} = 1;
+	}
+    }
+
+
     return bless $self, $class;
 }
@@ -121,8 +143,5 @@
     my ($hashGeneralOptions) = @_;
 
-##    print STDERR "here2";
-   
-    $strWgetOptions = $self->getWgetOptions();
-    my $cmdWget = $strWgetOptions;
+##    my $cmdWget = $strWgetOptions;
   
     my $strOutputDir ="";
@@ -136,9 +155,10 @@
     my $strIDs = $self->getOAIIDs($strBasURL);
  
-   if($strIDs eq "")
-    {
-	print STDERR "Error: No ID being found\n";
+    if($strIDs eq "")
+    {
+	print STDERR "Error: No IDs found\n";
 	return 0;
     }
+
     my $aryIDs = $self->parseOAIIDs($strIDs);
     my $intIDs = 0;
@@ -155,6 +175,6 @@
     $self->getOAIRecords($aryIDs, $strOutputDir, $strBasURL, $intMaxRecords, $blnDownloadDoc);
 
-    my $tmp_file = "$ENV{GSDLHOME}/tmp/oai.tmp";
-    &util::rm($tmp_file); 
+#    my $tmp_file = &util::filename_cat($ENV{'GSDLHOME'},"tmp","oai.tmp");
+#    &util::rm($tmp_file); 
 
     return 1;
@@ -164,9 +184,9 @@
 {
     my ($self,$strBasURL) = @_;
-    my ($cmdWget);
+##    my ($cmdWget);
      
     my $wgetOptions = $self->getWgetOptions();
 
-    $cmdWget = $wgetOptions;
+    my $cmdWget = $wgetOptions;
   
     print STDERR  "Gathering OAI identifiers.....\n";
@@ -256,5 +276,5 @@
 sub getOAIDoc
 {
-    my ($self,$strRecord, $strSubDirPath) = @_;
+    my ($self,$strRecord, $oai_rec_filename) = @_;
  
     print  STDERR "Gathering source documents.....\n";
@@ -264,32 +284,130 @@
     {
 	my $strMetaTag = $1;
-	
-	if ($strMetaTag =~ m/<(dc:)?identifier>(.*?)<\/(dc:)?identifier>/s)
+	my $had_valid_url = 0;
+
+	while ($strMetaTag =~ s/<(dc:)?identifier>(.*?)<\/(dc:)?identifier>//is)
 	{
-	    my $strDocURL = $2;
-
-	    my ($unused,$strDocFile) = $self->dirFileSplit($strDocURL);
-
-            my $strSoureDirPath ="";
-
-	    $strSoureDirPath = &util::filename_cat($strSubDirPath,"srcdocs");
-
-	    &util::mk_dir($strSoureDirPath)  if (!-e "$strSoureDirPath");
-	    
-	    my $strFullDocFilePath = &util::filename_cat($strSoureDirPath,$strDocFile);
-	    
-	    my $wget_cmd = $strWgetOptions." -q -O \"$strFullDocFilePath\" \"$strDocURL\"";
-
-	    my $strResponse =  $self->useWget($wget_cmd,1);
-
-	    if($strResponse ne "")
+	    my $doc_id_url = $2;
+
+	    next if ($doc_id_url !~ m/^(https?|ftp):\/\//);
+
+	    my $orig_doc_id_url = $doc_id_url;
+	    $had_valid_url = 1;
+
+	    my ($doc_dir_url_prefix,$doc_id_tail) = ($doc_id_url =~ m/^(.*)\/(.*?)$/);
+	    my $faked_ext = 0;
+	    my $primary_doc_match = 0;
+
+	    my ($id_file_ext) = ($doc_id_tail =~ m/\.([^\.]+)$/);
+
+	    if (defined $id_file_ext) {
+		# cross-check this filename extension with get_doc_exts option
+		# if provided
+		my $lookup_exts = $self->{'lookup_exts'};
+
+		if (defined $lookup_exts->{lc($id_file_ext)}) {
+		    # this initial URL matches requirement
+		    $primary_doc_match = 1;
+		}
+	    }
+	    else {
+		$faked_ext = 1;
+		$id_file_ext = "html";
+	    }
+
+		
+	    if ((!$primary_doc_match) && ($id_file_ext =~ m/^html?$/i)) {
+		# Download this doc if HTML, scan through it looking for a link
+		# that does match get_doc_exts
+		
+
+		# 1. Generate a tmp name
+		my $tmp_filename = &util::get_tmp_filename();
+
+		# 2. Download it
+		my $wget_opts2 = $self->getWgetOptions();
+		my $wget_cmd2 = "$wget_opts2 --convert-links -O \"$tmp_filename\" \"$doc_id_url\"";
+
+		my ($stdout_and_err2,$error2,$follow2) =  $self->useWgetMonitored($wget_cmd2);
+
+		if($error2 ne "")
+		{
+		    print STDERR "Error occured while retrieving OAI source documents: $error2\n";
+		    exit(-1);
+		}
+
+		if (defined $follow2) {
+		    # src url was "redirected" to another place
+		    # => pick up on this and make it the new doc_id_url
+		    $doc_id_url = $follow2;
+		}
+
+		my $primary_doc_html = "";
+		if (open(HIN,"<$tmp_filename")) {
+		    my $line;
+		    while (defined ($line = <HIN>)) {
+			$primary_doc_html .= $line;
+		    }
+		    close(HIN);
+
+		    # 3. Scan through it looking for match
+		    # 
+		    # if got match, change $doc_id_url to this new URL and
+		    # $id_file_ext to 'match' 
+		    
+		    my @href_links = ($primary_doc_html =~ m/href="(.*?)"/gsi);
+
+		    my $lookup_exts = $self->{'lookup_exts'};
+
+		    foreach my $href (@href_links) {
+			my ($ext) = ($href =~ m/\.([^\.]+)$/);
+
+			if ((defined $ext) && (defined $lookup_exts->{$ext})) {
+
+			    if ($href !~ m/^(https?|ftp):\/\//) {
+				# link is within current site
+				my ($site_domain) = ($doc_id_url =~ m/^((?:https?|ftp):\/\/.*?)\//);
+
+				$href = "$site_domain$href";
+			    }
+
+			    $doc_id_url = $href;
+			    $id_file_ext = $ext;
+			    last;
+			}
+		    }
+		}
+		else {
+		    print STDERR "Error occurred while retrieving OAI source documents:\n";
+		    print STDERR "$!\n";
+		}
+
+		if (-e $tmp_filename) {
+		    &util::rm($tmp_filename); 
+		}
+	    }
+
+	    my $download_doc_filename = $oai_rec_filename;
+	    $download_doc_filename =~ s/\.oai$/\.$id_file_ext/;
+
+	    my ($unused,$download_doc_file) = $self->dirFileSplit($download_doc_filename);
+
+	    my $wget_opts = $self->getWgetOptions();
+	    my $wget_cmd = "$wget_opts --convert-links -O \"$download_doc_filename\" \"$doc_id_url\"";
+
+	    my ($stdout_and_err,$errors,$follow) =  $self->useWgetMonitored($wget_cmd);
+
+	    if($errors ne "")
 	    {
-		print STDERR "Error occured while retriving OAI souce documents: $strResponse\n";
+		print STDERR "Error occured while retriving OAI souce documents:\n";
+		print STDERR "$errors\n";
 		exit(-1);
 	    }
 
-	    $strRecord =~ s/<metadata>(.*?)<(dc:)?identifier>$strDocURL<\/(dc:)?identifier>(.*?)<\/metadata>/<metadata>$1<OrigURL>$strDocURL<\/OrigURL>\n   <identifier>srcdocs\/$strDocFile<\/identifier>$4<\/metadata>/s;
+	    
+	    $strRecord =~ s/<metadata>(.*?)<(dc:)?identifier>$orig_doc_id_url<\/(dc:)?identifier>(.*?)<\/metadata>/<metadata>$1<${2}identifier>$orig_doc_id_url<\/${2}identifier>\n   <gi.Sourcedoc>$download_doc_file<\/gi.Sourcedoc>$4<\/metadata>/s;
 	}
-	else
+
+	if (!$had_valid_url)
 	{
 	    print  STDERR "\tNo souce document URL is specified in the OAI record (No (dc:)?identifier is provided)\n";
@@ -300,5 +418,6 @@
 	print  STDERR "\tNo souce document URL is specified in the OAI record (No metadata field is provided)\n";
     }
-    
+
+    return $strRecord;
 }
 
@@ -313,12 +432,13 @@
     foreach my $strID ( @$aryIDs)
     {
-	print  STDERR "Gathering OAI record with ID:$strID.....\n";
+	print  STDERR "Gathering OAI record with ID $strID.....\n";
 	   
-	my $cmdWget= $strWgetOptions." -q -O - \"$strBasURL?verb=GetRecord&metadataPrefix=$metadata_prefix&identifier=$strID\"";
+	my $wget_opts = $self->getWgetOptions();
+	my $cmdWget= "$wget_opts -q -O - \"$strBasURL?verb=GetRecord&metadataPrefix=$metadata_prefix&identifier=$strID\"";
 
 	my $strRecord =  $self->useWget($cmdWget);
 
-        
-        my @fileDirs = split(":",$strID);  
+	my @fileDirs = split(":",$strID);  
+	my $local_id = pop @fileDirs;
 
 	# setup directories
@@ -328,10 +448,10 @@
         my $host =$self->{'url'}; 
   
-        $host =~ s/http:\/\///g;
+        $host =~ s/https?:\/\///g;
 
         $host =~ s/:.*//g; 
 
-	my $midDir = join ("/",@fileDirs);
-	my $strFileURL = "$strOutputDir/$host/".$midDir.".oai";
+	my $strFileURL = "$strOutputDir/$host/$local_id.oai";
+
 
 	# prepare subdirectory for record (if needed)
@@ -346,5 +466,5 @@
 	if($blnDownloadDoc)
 	{
-	    $self->getOAIDoc($strRecord,$strSubDirPath);
+	    $strRecord = $self->getOAIDoc($strRecord,$strFileURL);
 	}
 
@@ -378,4 +498,5 @@
     my $strIdentify = "verb=Identify";
     my $strListSets = "verb=ListSets";
+    my $strListMdFormats = "verb=ListMetadataFormats";
 
     my $strIdentifyCMD = $strBaseCMD;
@@ -392,4 +513,20 @@
     print STDERR "General information:\n";
     $self->parse_xml($strIdentifyText);
+    print STDERR "\n";
+
+    print STDERR "=" x 10, "\n";
+    print STDERR "Metadata Format Information (metadataPrefix):\n";
+    print STDERR "=" x 10, "\n";
+
+    my $strListMdFormatsCMD = $strBaseCMD;
+    $strListMdFormatsCMD =~ s/_OPTS_/$strListMdFormats/;	
+    my $strListMdFormatsText = $self->useWget($strListMdFormatsCMD);
+
+    $self->parse_xml($strListMdFormatsText);
+    print STDERR "\n";
+
+    print STDERR "=" x 10, "\n";
+    print STDERR "List Information:\n";
+    print STDERR "=" x 10, "\n";
 
     my $strListSetCMD = $strBaseCMD;
@@ -397,6 +534,4 @@
     my $strListSetsText = $self->useWget($strListSetCMD);
 
-
-    print STDERR "List Information:\n";
     $self->parse_xml($strListSetsText);
 }
@@ -405,31 +540,40 @@
 {    
     my ($self) = shift (@_);
-    my ($strOutputText) = @_;
+    my ($xml_text) = @_;
    
+    #### change this to work directly from $xml_text
+
     #Open a temporary file to store OAI information, and store the information to the temp file
-    my $name = "$ENV{GSDLHOME}/tmp/oai.tmp"; 
+    my $name = &util::filename_cat($ENV{GSDLHOME},"tmp","oai.tmp"); 
 
     open(*OAIOUT,"> $name");
 	
-    print OAIOUT $strOutputText;
+    print OAIOUT $xml_text;
     close(OAIOUT);
 
     $self->{'temp_file_name'} = $name;
+
+##    print STDERR "**** xml text = $xml_text\n";
 
     eval {
 	$self->{'parser'}->parsefile("$name");
+##	$self->{'parser'}->parse($xml_text);
     };
     
     if ($@) {
-	die "OAI: $name is not a well formed XML file ($@)\n";
-    }
-}
-
-END{
-    if($self->{'info'})
-    {
-	unlink($self->{'temp_file_name'}) or die "Could not unlink $self->{'temp_file_name'}: $!";
-    }
-}
+	die "OAI: Parsed file $name is not a well formed XML file ($@)\n";
+##	die "OAI: Parsed text is not a well formed XML file ($@)\n";
+    }
+
+    unlink($self->{'temp_file_name'}) or die "Could not unlink $self->{'temp_file_name'}: $!";
+}
+
+####END 
+#{
+#    if($self->{'info'})
+#    {
+#	unlink($self->{'temp_file_name'}) or die "Could not unlink $self->{'temp_file_name'}: $!";
+#    }
+#}
 
 # This Char function overrides the one in XML::Parser::Stream to overcome a
@@ -439,4 +583,6 @@
     use bytes;  # Necessary to prevent encoding issues with XML::Parser 2.31+
     $_[0]->{'Text'} .= $_[1];
+
+    my $self = $_[0]->{'PluginObj'};
     if ((defined $self->{'subfield'} && ($self->{'subfield'} ne ""))) {
 	$self->{'text'} .= $_[1];
@@ -454,4 +600,5 @@
     my ($expat, $element, %attr) = @_;
 
+    my $self = $expat->{'PluginObj'};
     $self->{'subfield'} = $element;
     
@@ -461,4 +608,6 @@
 {
     my ($expat, $element) = @_;
+
+    my $self = $expat->{'PluginObj'};
     $self->{'text'} = "";
     $self->{'subfield'} = "";
Index: /gsdl/trunk/perllib/downloaders/WgetDownload.pm
===================================================================
--- /gsdl/trunk/perllib/downloaders/WgetDownload.pm	(revision 16790)
+++ /gsdl/trunk/perllib/downloaders/WgetDownload.pm	(revision 16791)
@@ -105,5 +105,5 @@
     my $strOptions = "";
     
-    if($self->{'proxy_on'} && $self->{'proxy_host'} && $self->{'proxy_port'})
+    if ($self->{'proxy_on'} && $self->{'proxy_host'} && $self->{'proxy_port'})
     {
 
@@ -116,5 +116,7 @@
     }
 
-    $strOptions .= " -Y on ";
+    if ($self->{'proxy_on'}) {
+	$strOptions .= " --proxy ";
+    }
 
     return $strOptions;
@@ -167,4 +169,64 @@
 }
 
+
+sub useWgetMonitored
+{
+    my ($self, $cmdWget,$blnShow, $working_dir) = @_;
+
+
+    my $current_dir = cwd();
+    my $changed_dir = 0;
+    if (defined $working_dir && -e $working_dir) {
+	chdir "$working_dir";
+	$changed_dir = 1;
+    }
+    my $wget_file_path = &util::filename_cat($ENV{'GSDLHOME'}, "bin", $ENV{'GSDLOS'}, "wget");
+    my $command = "\"$wget_file_path\" $cmdWget 2>&1 |";
+
+###    print STDERR "**** wget cmd = $command\n";
+
+    open(*WIN,$command) || die "wget request failed: $!\n";
+
+    my $full_text = "";
+    my $error_text = "";
+    my @follow_list = ();
+    my $line;
+
+    while (defined($line=<WIN>))
+    {
+	if((defined $blnShow) && $blnShow)
+	{
+	    print STDERR "$line";
+	}
+
+	if ($line =~ m/^Location:\s*(.*?)\s*\[following\]\s*$/i) {
+	    my $follow_url = $1;
+	    push(@follow_list,$follow_url);
+	}
+
+	if ($line =~ m/ERROR\s+\d+/) {
+	    $error_text .= $line;
+	}
+
+	$full_text .= $line;
+    }
+
+    close(WIN);
+
+    my $command_status = $?;
+    if ($command_status != 0) {
+	$error_text .= "Exit error: $command_status";
+    }
+
+    if ($changed_dir) {
+	chdir $current_dir;
+    }
+    
+    my $final_follow = pop(@follow_list); # might be undefined, but that's OK
+    
+    return ($full_text,$error_text,$final_follow);
+}
+
+
 # TODO: Check if the URL is valid?? Not sure what should be in this function yet!!
 sub checkURL
