Changeset 2671

Timestamp:

2001-07-27T09:17:04+12:00 (23 years ago)

Author:

sjboddie

Message:

* empty log message *

File:

• trunk/gsdl/bin/script/grab_collection.pl (modified) (4 diffs)

trunk/gsdl/bin/script/grab_collection.pl

@@ -33 +33 @@
 
 # This is where we start our mirroring
-$address = 'http://ginkgo.cisti.nrc.ca:8080/cgi-bin/library?a=p&p=about&c=envl&u=1';
+$address = 'http://nowhere.com/cgi-bin/library?a=p&p=about&c=demo&u=1';
 
 
@@ -125 +125 @@
 $linknumber    = 0;                    # used to name/number the dl-ed html files
 
-#my $failed     = 0;
-#while ($failed == 0)
-#{
-#    if ($linknumber % $dir_entries == 0) 
-#   {
-#       if (!((-e $outputdir.$linknumber)&&(-d $outputdir.$linknumber)))
-#       {
-#       $failed++;
-#       mkdir($outputdir.$linknumber, 0777) or print " ** Cannot create ",$outputdir.$linknumber, "!: $!\n";
-#       }
-#       $numberdir = $linknumber;
-#   }
-
-#    $check_file = $outputdir.$numberdir."/".$linknumber.".html";
-#    if ((-e $check_file)&&($failed == 0)) 
-#    { 
-#   $linknumber++;
-#    } 
-#    else
-#    {
-#   $failed++;
+my $failed     = 0;
+while ($failed == 0)
+{
+    if ($linknumber % $dir_entries == 0) 
+    {
+        if (!((-e $outputdir.$linknumber)&&(-d $outputdir.$linknumber)))
+        {
+        $failed++;
+        mkdir($outputdir.$linknumber, 0777) or print " ** Cannot create ",$outputdir.$linknumber, "!: $!\n";
+        }
+        $numberdir = $linknumber;
+    }
+
+    $check_file = $outputdir.$numberdir."/".$linknumber.".html";
+    if ((-e $check_file)&&($failed == 0)) 
+    { 
+    $linknumber++;
+    } 
+    else
+    {
+    $failed++;
     # I'm subtracting 1 from the starting link, 
     # just in case it only loaded half the page ;^)
-#   if($linknumber>0) 
-#   { 
-#       $linknumber--; 
-#   }
-#   print " Will start downloading at number $linknumber \n";
-#    }
-#}
+    if($linknumber>0) 
+    { 
+        $linknumber--; 
+    }
+    print " Will start downloading at number $linknumber \n";
+    }
+}
 
 # if we're starting from scratch, then we might as well nuke the links file
@@ -179 +179 @@
 # if we're NOT starting from scratch, then read in old links from links text file
 # and grab the old image-links as well...
-#if ($linknumber != 0)
-#{
+if ($linknumber != 0)
+{
     # load the old links from links.txt, if it doesn't exist, then give up :(
-#    my $this = "";
-#    my $that = "";
-#    open (CHECK, "links.txt") or die " ** Cannot find/open links.txt file!: $! **\n";
-#    while(eof CHECK == 0)
-#    {
-#   while($this ne "\n")
-#   {
-#       read CHECK, $this ,1;
-#       $that = $that.$this;   
-#   }
-#   $linkz_list[$linkz_pointer] = $that;
+    my $this = "";
+    my $that = "";
+    open (CHECK, "links.txt") or die " ** Cannot find/open links.txt file!: $! **\n";
+    while(eof CHECK == 0)
+    {
+    while($this ne "\n")
+    {
+        read CHECK, $this ,1;
+        $that = $that.$this;   
+    }
+    $linkz_list[$linkz_pointer] = $that;
     
-#   for my $search(0 .. (length($that) - 3))
-#   {
-#       if((substr($that, $search, 3) eq '?a=')||(substr($that, $search, 3) eq '&a='))
-#       {
-#       $short_linkz_list[$linkz_pointer] = substr($that, $search);
-#       last;
-#       }
-#   }
-#   $linkz_pointer++;
-#   $that = ""; $this = "";
-#    }
-#    close(CHECK);
-#    print "- I found ",($#linkz_list + 1)," links in links.txt -\n";
+    for my $search(0 .. (length($that) - 3))
+    {
+        if((substr($that, $search, 3) eq '?a=')||(substr($that, $search, 3) eq '&a='))
+        {
+        $short_linkz_list[$linkz_pointer] = substr($that, $search);
+        last;
+        }
+    }
+    $linkz_pointer++;
+    $that = ""; $this = "";
+    }
+    close(CHECK);
+    print "- I found ",($#linkz_list + 1)," links in links.txt -\n";
     
     #make sure that we start dl-ing the correct first page 
-#    $address = $linkz_list[$linknumber];
+    $address = $linkz_list[$linknumber];
 
     # load the old image links from image.txt (if it doesn't exist, no big deal ;)
-#    my $im_this = "";
-#    my $im_that = "";
-#    open (IMAGES, "images.txt") || print " ** Cannot find/open images.txt file! : $! **\n";
-#    while(eof IMAGES == 0)
-#    {
-#   while($im_this ne "\n")
-#   {
-#       read IMAGES, $im_this ,1;
-#       $im_that = $im_that.$im_this;   
-#   }
-#   $image_list[$image_pointer] = $im_that;
-#   $image_pointer++;
-#   $im_that = ""; $im_this = "";
-#    }
-#    close(IMAGES);
-#    print "- I found ",($#image_list + 1)," picture-links in images.txt -\n";
+    my $im_this = "";
+    my $im_that = "";
+    open (IMAGES, "images.txt") || print " ** Cannot find/open images.txt file! : $! **\n";
+    while(eof IMAGES == 0)
+    {
+    while($im_this ne "\n")
+    {
+        read IMAGES, $im_this ,1;
+        $im_that = $im_that.$im_this;   
+    }
+    $image_list[$image_pointer] = $im_that;
+    $image_pointer++;
+    $im_that = ""; $im_this = "";
+    }
+    close(IMAGES);
+    print "- I found ",($#image_list + 1)," picture-links in images.txt -\n";
 
     #..and last but not least, load any image_dirs from image_dirs.txt
     # again, if its not there, no big deal :)
-#    my $imd_this = "";
-#    my $imd_that = "";
-#    open (IMAGE_DIR, "image_dirs.txt") || print " ** Cannot find/open image_dirs.txt file!: $! **\n";
-#    while(eof IMAGE_DIR == 0)
-#    {
-#   while($imd_this ne "\n")
-#   {
-#       read IMAGE_DIR, $imd_this ,1;
-#       $imd_that = $imd_that.$imd_this;   
-#   }
-#   $image_dirs_list[$image_dirs_pointer] = $imd_that;
-#   $image_dirs_pointer++;
-#   $imd_that = ""; $imd_this = "";
-#    }
-#    close(IMAGE_DIR);
-#    print "- I found ",($#image_dirs_list + 1)," picture directories in image_dirs.txt -\n";
-#}
+    my $imd_this = "";
+    my $imd_that = "";
+    open (IMAGE_DIR, "image_dirs.txt") || print " ** Cannot find/open image_dirs.txt file!: $! **\n";
+    while(eof IMAGE_DIR == 0)
+    {
+    while($imd_this ne "\n")
+    {
+        read IMAGE_DIR, $imd_this ,1;
+        $imd_that = $imd_that.$imd_this;   
+    }
+    $image_dirs_list[$image_dirs_pointer] = $imd_that;
+    $image_dirs_pointer++;
+    $imd_that = ""; $imd_this = "";
+    }
+    close(IMAGE_DIR);
+    print "- I found ",($#image_dirs_list + 1)," picture directories in image_dirs.txt -\n";
+}
 
 #  Just keep going till we can find no more new links
@@ -291 +291 @@
     }
 
+    # ignore mailto urls
+    if ($data[$i] !~ /mailto:/i) {
     
-        #----------- the link is NOT an image ----------------      
-    if ($its_an_image == 0)
-    { 
-#       &its_a_link($temp[1], $outputdir);
-        &its_a_link($data[$i], $outputdir);
-    }
-
-        #----------- the link IS an image ----------------
-    if ($its_an_image != 0)
-    { 
-#       &its_an_image($temp[1], $finaldir);
-        &its_an_image($data[$i], $finaldir); 
+        #----------- the link is NOT an image ----------------      
+        if ($its_an_image == 0)
+        { 
+        &its_a_link($data[$i], $outputdir);
+        }
+
+        #----------- the link IS an image ----------------
+        if ($its_an_image != 0)
+        { 
+        &its_an_image($data[$i], $finaldir); 
+        }
     }
     }