Changeset 1533

Timestamp:

2000-09-11T10:59:28+12:00 (24 years ago)

Author:

paynter

Message:

Added support for GNU Wget to web site mirroring tools (perviously, you had to use w3mirror, which is very hard to install).

Location:

trunk/gsdl/bin/script

Files:

• gsWget.pl (added)
• mirror.pl (modified) (5 diffs)
• update.pl (modified) (4 diffs)

trunk/gsdl/bin/script/mirror.pl

@@ -29 +29 @@
 
 # This program uses w3mirror to mirror a web site.  It looks for a
-# w3mirror configuration file in etc/w3mir.cfg, and if it finds one then
-# it runs w3mirror (using this configuration file) in the import
-# directory.  Afterwards, a mirror of the web site (as dictated by the 
-# configuration file) will be in the import directory.
+# mirror program configuration files in etc, and if it finds them then
+# it runs the mirroring software using this configuration file, and placing
+# the mirror in the import directory.
+#
+# mirror.pl can use the w3mirror program or the wget program if they are
+# installed.
+#
+# To use w3mirror, the configuration file must be in etc/w3mir.cfg.
+# To use GNU wget, the configuartion file (i.e. a wgetrc file) must
+# be in etc/wget.cfg and a file of the URLs to read in etc/wget.url
 
 BEGIN {
@@ -77 +83 @@
     }
 
-    # check the configuration file for options
+    # get the etc directory
+    $etcdir = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, "etc");
+    
+    # check the collection configuration file for options
     my $interval = 0;
-    $configfilename = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, "etc/collect.cfg");
+    $configfilename = &util::filename_cat ($ENV{'GSDLCOLLECTDIR'}, 
+                       "etc", "collect.cfg");
     if (-e $configfilename) {
     $collectcfg = &colcfg::read_collect_cfg ($configfilename);
@@ -95 +105 @@
     $importdir =~ s/\/$//;
 
-    $etcdir = "$ENV{'GSDLCOLLECTDIR'}/etc";
-
-    print "import directory: $importdir\n";
-    print "   w3config file: $etcdir/w3mir.cfg\n\n";
-
     # make sure there is an import directory
     if (! -e "$importdir") {
@@ -105 +110 @@
     }
 
-    # make sure there is a w3mir file
-    if (! -e "$etcdir/w3mir.cfg") {
-    die "Couldn't find the w3mir configuration file $etcdir/w3mir.cfg\n";
+    # if w3mir.cfg exists, 
+    # then we are using w3mirror to mirror the remote site
+    if (-e "$etcdir/w3mir.cfg") {
+
+    # run the mirror program from the import directory
+    my  $cmd = "cd $importdir; ";
+    $cmd .= "gsw3mir.pl -cfgfile $etcdir/w3mir.cfg";
+    # print "\n$cmd\n";
+    `$cmd`;
+
+    } 
+
+    # if wget.cfg and wget.url both exist, 
+    # then we are using GNU wget to mirror the remote site
+    elsif ((-e "$etcdir/wget.cfg") && (-e "$etcdir/wget.url")) {
+    $ENV{WGETRC} = "$etcdir/wget.cfg";
+    my $cmd = "gsWget.pl --input-file=$etcdir/wget.url --directory-prefix=$importdir";
+    system($cmd);
     }
 
-    # run the mirror program from the import directory
-    my  $command = "cd $importdir; gsw3mir.pl -cfgfile $etcdir/w3mir.cfg\n";
-    # print "\n$command\n";
-    `$command`;
+    # otherwise, there are no mirror copnfiguration files
+    else {
+    die "Couldn't find the mirror configuration files in $etcdir\n";
+    }
+
 
 }
@@ -121 +142 @@
 
 
+

trunk/gsdl/bin/script/update.pl

@@ -117 +117 @@
     # if there is no mirror information, we're all done
     if (!defined($mirror)) {
-    print "No mirror information\n";
+    print "No mirror command in $configfilename\n";
     exit;
     }
@@ -129 +129 @@
     }
 
-    # make sure there is a w3mir file
-    if (! -e "$etcdir/w3mir.cfg") {
-    die "Couldn't find the w3mir configuration file $etcdir/w3mir.cfg\n";
-    }
-
     # make sure there is an import directory
     if (! -e "$importdir") {
@@ -139 +134 @@
     }
 
-    # make sure there is an empty archives directory
-    if (-e "$archivedir") {
-    &util::rm_r ($archivedir);
-    }
-    &util::mk_dir($archivedir);
-    
     print "archives directory: $archivedir\n";
     print "  import directory: $importdir\n";
     print "     etc directory: $etcdir\n";
-    print "     w3config file: $etcdir/w3mir.cfg\n";
-
     print "          interval: $interval days\n";
 
@@ -179 +166 @@
     # Import the collection
     print WLOG "\n\nStarting import at " . time . "\n\n";
-    $command = "import.pl $collection";
+    $command = "import.pl -removeold $collection";
     print WLOG "Executing: $command\n";
     print WLOG `$command`;