Changeset 28250 for main/trunk/greenstone2

Timestamp:

2013-09-10T15:37:45+12:00 (11 years ago)

Author:

ak19

Message:

1. Jenny and I fixed an oversight in OAIDownload, thanks to Kathy's suggestion, where an html page's contents used to be inspected for file types specified to be downloaded with GetDocument ONLY if the file extension was htm(l). Now the test is whether the header specifies Content-Type text/html. 2. Deprecated utils functions replaced with their FileUtils equivalents.

Location:

main/trunk/greenstone2/perllib

Files:

• download.pm (modified) (2 diffs)
• downloaders/MediaWikiDownload.pm (modified) (1 diff)
• downloaders/OAIDownload.pm (modified) (6 diffs)
• downloaders/SRWDownload.pm (modified) (1 diff)
• downloaders/WgetDownload.pm (modified) (3 diffs)
• downloaders/Z3950Download.pm (modified) (2 diffs)

main/trunk/greenstone2/perllib/download.pm

@@ -44 +44 @@
     if ($ENV{'GSDLCOLLECTDIR'}){
    
-    $coldownloadname = &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, 
+    $coldownloadname = &FileUtils::filenameConcatenate($ENV{'GSDLCOLLECTDIR'}, 
                           "perllib","downloaders", 
                           "${download_name}.pm");
@@ -50 +50 @@
     }
    
-    my $maindownloadname = &util::filename_cat($ENV{'GSDLHOME'},
+    my $maindownloadname = &FileUtils::filenameConcatenate($ENV{'GSDLHOME'},
                            "perllib","downloaders", 
                            "${download_name}.pm");

main/trunk/greenstone2/perllib/downloaders/MediaWikiDownload.pm

@@ -128 +128 @@
     # check css files for HTML pages are downloaded as well
     $self->{'url'} =~ /http:\/\/([^\/]*)\//;        
-    my $base_url = &util::filename_cat($hashGeneralOptions->{"cache_dir"}, $1);    
+    my $base_url = &FileUtils::filenameConcatenate($hashGeneralOptions->{"cache_dir"}, $1);    
     &check_file($base_url, $self);
         

main/trunk/greenstone2/perllib/downloaders/OAIDownload.pm

@@ -112 +112 @@
     my $tmp_dir = "$ENV{GSDLHOME}/tmp";
     if (! -e $tmp_dir) {
-    &util::mk_dir($tmp_dir);
+    &FileUtils::makeDirectory($tmp_dir);
     }
 
@@ -171 +171 @@
     $self->getOAIRecords($aryIDs, $strOutputDir, $strBasURL, $self->{'max_records'}, $blnDownloadDoc);
 
-#    my $tmp_file = &util::filename_cat($ENV{'GSDLHOME'},"tmp","oai.tmp");
-#    &util::rm($tmp_file); 
+#    my $tmp_file = &FileUtils::filenameConcatenate($ENV{'GSDLHOME'},"tmp","oai.tmp");
+#    &FileUtils::removeFiles($tmp_file); 
 
     return 1;
@@ -318 +318 @@
         }
 
+
+        my $is_page_html = 0;
+        if($id_file_ext =~ m/^html?$/i) {
+            $is_page_html = 1;
+        } elsif ($doc_id_url) { # important: if no doc id has currently been typed into the url field, skip this block
         
-        if ((!$primary_doc_match) && ($id_file_ext =~ m/^html?$/i)) {
+            # get the page and check the header's content-type and see if this is text/html
+            # if so, $is_page_html is true
+            # See http://superuser.com/questions/197009/wget-head-request
+            
+            my $wget_opts3 = $self->getWgetOptions();
+            my $wget_header_cmd = "$wget_opts3 -S --spider \"$doc_id_url\"";
+            my $page_content = $self->useWget($wget_header_cmd);            
+            
+            if($page_content && $page_content =~ m@Content-Type:\s*text/html@i) {               
+                $is_page_html = 1;
+            }
+        }
+        
+        if (!$primary_doc_match && $is_page_html) {     
+        
         # Download this doc if HTML, scan through it looking for a link
         # that does match get_doc_exts
@@ -387 +406 @@
 
         if (-e $tmp_filename) {
-            &util::rm($tmp_filename); 
+            &FileUtils::removeFiles($tmp_filename); 
         }
         }
@@ -474 +493 @@
         ($strSubDirPath,$unused) = $self->dirFileSplit($strFileURL);
    
-    &util::mk_all_dir($strSubDirPath);
+    &FileUtils::makeAllDirectories($strSubDirPath);
 
     my $ds = &util::get_dirsep();
@@ -559 +578 @@
 
     #Open a temporary file to store OAI information, and store the information to the temp file
-    my $name = &util::filename_cat($ENV{GSDLHOME},"tmp","oai.tmp"); 
+    my $name = &FileUtils::filenameConcatenate($ENV{GSDLHOME},"tmp","oai.tmp"); 
 
     open(*OAIOUT,"> $name");

main/trunk/greenstone2/perllib/downloaders/SRWDownload.pm

@@ -127 +127 @@
     $host =~ s/http:\/\///;
     $strOutputDir  =~ s/"//g; #"
-    my $strOutputFile = &util::filename_cat($strOutputDir,$host,"$strFileName.xml");
+    my $strOutputFile = &FileUtils::filenameConcatenate($strOutputDir,$host,"$strFileName.xml");
  
     # prepare subdirectory for record (if needed)
 
     my ($strSubDirPath,$unused) = $self->dirFileSplit($strOutputFile);
-    &util::mk_all_dir($strSubDirPath);
+    &FileUtils::makeAllDirectories($strSubDirPath);
 
     print STDERR "Saving records to \"$strOutputFile\"\n";

main/trunk/greenstone2/perllib/downloaders/WgetDownload.pm

@@ -238 +238 @@
     }
 
-    my $wget_file_path = &util::filename_cat($ENV{'GSDLHOME'}, "bin", $ENV{'GSDLOS'}, "wget");
+    my $wget_file_path = &FileUtils::filenameConcatenate($ENV{'GSDLHOME'}, "bin", $ENV{'GSDLOS'}, "wget");
     $command = "\"$wget_file_path\" $cmdWget";
     #print STDOUT "Command is: $command\n";
@@ -387 +387 @@
     }
 
-    my $wget_file_path = &util::filename_cat($ENV{'GSDLHOME'}, "bin", $ENV{'GSDLOS'}, "wget");
+    my $wget_file_path = &FileUtils::filenameConcatenate($ENV{'GSDLHOME'}, "bin", $ENV{'GSDLOS'}, "wget");
     my $command = "\"$wget_file_path\" $cmdWget";
     #print STDOUT "Command is: $command\n";
@@ -538 +538 @@
 
 1;
+

main/trunk/greenstone2/perllib/downloaders/Z3950Download.pm

@@ -97 +97 @@
     $self->{'url'} = $self->{'host'}.":".$self->{'port'};
 
-    $self->{'yaz'} = &util::filename_cat($ENV{'GSDLHOME'}, "bin", $ENV{'GSDLOS'}, "yaz-client");
+    $self->{'yaz'} = &FileUtils::filenameConcatenate($ENV{'GSDLHOME'}, "bin", $ENV{'GSDLOS'}, "yaz-client");
     
     return bless $self, $class;
@@ -278 +278 @@
     $strOutputDir  =~ s/"//g; #"
 
-    my $strOutputFile = &util::filename_cat($strOutputDir,$self->{'host'},"$strFileName.marc");
+    my $strOutputFile = &FileUtils::filenameConcatenate($strOutputDir,$self->{'host'},"$strFileName.marc");
      # prepare subdirectory for record (if needed)
     my ($strSubDirPath,$unused) = $self->dirFileSplit($strOutputFile);
  
-    &util::mk_all_dir($strSubDirPath);
+    &FileUtils::makeAllDirectories($strSubDirPath);
   
     print STDERR "Saving records to \"$strOutputFile\"\n";