source: trunk/gsdl/src/java/org/nzdl/gsdl/GsdlCollageApplet/DownloadUrls.java@ 6816

package org.nzdl.gsdl.GsdlCollageApplet;

import java.awt.*;
import java.io.*;
import java.net.*;
import java.util.*;

/** 
 *  @author Katrina Edgar
 *  @author David Bainbridge
 *
 *  Controls retrieval of images from the specified starting url. Follows appropriate
 *  links from this starting point, traversing in a tree-like state through several other
 *  pages. Filters images and links based on specified parameters. Also controls the quantity
 *  of downloading that occurs by restricting the number of downloaded images that are yet to
 *  be displayed to 10, and the total number of downloads allowed is also restricted by
 *  the applet application (to prevent downloading occur infinitely). */
public class DownloadUrls extends Thread {

    /** Refers to applet */
    GsdlCollageApplet app_          = null;
    /** Refers to download thread */
    DownloadImages download_images_ = null;

    /** The address from which the application should start looking for images */
    String starting_url_  = null;
    /** Restricts links followed from the starting url to links that contain this string */
    String href_musthave_ = null;
    /** Restricts links followed from the starting url to links that do not contain this string. 
     *  Also prevents image names from containing this string */
    String image_mustnothave_ = null;
    /** Ignore images whose names begin with this string */
    String image_ignore_  = null;
    /** Restricts the types of images included in the collage, for example jpg, gif, etc. */
    String image_type_ = null;

    /** A static delay used when attempting to download more images into a full downloading buffer */
    final int delay_       = 1000;
    /** The maximum number of images to have downloaded and not yet displayed */ 
    final int buffer_size_ = 10;

    /** Used in cases where the image maps to a url outside of it's original location.
     *  When used with Greenstone the collage images will refer to documents in the collections
     *  from which the images are sourced. When used individually, the images may be saved into
     *  a user directory and the pages they reference may be external hyperlinks. */
    Hashtable external_links_ = null;

    /** Records all urls which have already been examined */
    Hashtable visited_url_  = null;
    /** Determines whether there are still pages to examine and images to download */
    boolean thread_running_ = true;

    /** Constructor to initialise a download thread from which images are found, 
     *  saves parameters into local variables for use within the class.
     *
     *  @param app reference to the applet
     *  @param download_images class which stores the images retrieved in triplets
     *  @param starting_url the url from which the search for images should begin
     *  @param href_musthave restricts links to only those containing this string
     *  @param image_mustnothave restricts links and image names to only those that don't contain this string
     *  @param image_ignore restricts the beginning of image names
     *  @param image_type restricts the type of images included in the collage to those named */
    public DownloadUrls(GsdlCollageApplet app, 
            DownloadImages download_images, String starting_url, 
            String href_musthave, String image_mustnothave, 
            String image_ignore, String image_type)
    {
    super("DownloadUrls");
    app_             = app;
    download_images_ = download_images;

    starting_url_  = starting_url;
    href_musthave_ = href_musthave;
    image_mustnothave_ = image_mustnothave;
    image_ignore_  = image_ignore;
    image_type_    = image_type;
    }

    /** Determines whether or not a url has already been examined
     *
     *  @param url_string the url to check
     *  @return true if the url has been visited, false if not */
    public boolean already_visited(String url_string)
    {
    int hash_pos = url_string.indexOf("#");
    if (hash_pos>0)
     {
         // strip off #anchor reference
         url_string = url_string.substring(0,hash_pos);
     }

    // if the url has been visited before, return true
    if (visited_url_.containsKey(url_string))
    {
        if (app_.verbosity() > 3)
        {
            System.err.println("Visited " + url_string + " Before!");
        }
        return true;
    }
    
    visited_url_.put(url_string,"visited");

    return false;
    }

    /** Restricts the type of images that can be included in the collage
     *
     * @param url_string the url to check
     * @return true if the image is of a specified type, false if not */
    public boolean image_file_extension(String url_string)
    {
    // lower case comparisons
    String url_lstring = url_string.toLowerCase();

    if (image_type_ == null)
        return true;

    String tmp = image_type_;
    String original_image_type_ = image_type_;

    while (image_type_ != null && image_type_.indexOf("%") >= 0) {
        
        tmp = image_type_.substring(0, image_type_.indexOf("%"));
        if (image_type_.length() > image_type_.indexOf("%") + 1)
        image_type_ = image_type_.substring(image_type_.indexOf("%") + 1, image_type_.length());
        else
        image_type_ = null;

        if (url_lstring.endsWith(tmp)) {
        image_type_ = original_image_type_;
        return true;
        }
    }

    if (image_type_ != null && url_lstring.endsWith(image_type_)) {
        image_type_ = original_image_type_;
        return true;
    }

    image_type_ = original_image_type_;
    return false;
    }

    /** Restricts images to only those that satisfy several specified conditions
     *  regarding the content of the image name and url.
     *
     *  @param url_string the url to check
     *  @return true if the image is satisfactory, false if not */
    public boolean filter_image(String url_string)
    {
    if (image_ignore_==null || !url_string.startsWith(image_ignore_))
    {
        if (!already_visited(url_string))
        {
        if (image_mustnothave_ != null) {
            
            String tmp = image_mustnothave_;
            String original_image_mustnothave_ = image_mustnothave_;

            while (image_mustnothave_ != null && image_mustnothave_.indexOf("%") >= 0) {
        
            tmp = image_mustnothave_.substring(0, image_mustnothave_.indexOf("%"));
            if (image_mustnothave_.length() > image_mustnothave_.indexOf("%") + 1)
                image_mustnothave_ = image_mustnothave_.substring(image_mustnothave_.indexOf("%") + 1, 
                                          image_mustnothave_.length());
            else
                image_mustnothave_ = null;
            
            if (url_string.indexOf(tmp) >= 0) {
                image_mustnothave_ = original_image_mustnothave_;
                return false;
            }
            }

            if (image_mustnothave_ != null && url_string.indexOf(image_mustnothave_) >= 0) {
            image_mustnothave_ = original_image_mustnothave_;
            return false;
            }

            if (app_.verbosity() > 2) {
            System.err.println("src url = "+ url_string);
            }
            
            image_mustnothave_ = original_image_mustnothave_;
            return true;
        }
        }
    }

    return false;
    }

    /** Restricts links to only those that satisfy several specified conditions
     *  regarding the address of the link.
     *
     *  @param url_string the url to check
     *  @param new_url_string the url from which this link was found
     *  @param depth the number of links followed on this path
     *  @return true if the image is satisfactory, false if not */
    public boolean filter_href(String url_string, String new_url_string, int depth)
    {
    boolean has_href = false;
    String tmp = href_musthave_;
    String original_href_musthave_ = href_musthave_;

    // checks the it does contain this content
    if (href_musthave_ !=  null) {

        while (href_musthave_ != null && href_musthave_.indexOf("%") >= 0) {
        
        tmp = href_musthave_.substring(0, href_musthave_.indexOf("%"));
        if (href_musthave_.length() > href_musthave_.indexOf("%") + 1)
            href_musthave_ = href_musthave_.substring(href_musthave_.indexOf("%") + 1, href_musthave_.length());
        else
            href_musthave_ = null;
        
        if (url_string.indexOf(tmp) >= 0)
            has_href = true;
        }
        
        if (href_musthave_ != null && url_string.indexOf(href_musthave_) >= 0)
        has_href = true;

        href_musthave_ = original_href_musthave_;
    }

    tmp = image_mustnothave_;
    String original_image_mustnothave_ = image_mustnothave_;

    // checks that it doesn't contain this content
    if (image_mustnothave_ != null) {

        while (image_mustnothave_ != null && image_mustnothave_.indexOf("%") >= 0) {
        
        tmp = image_mustnothave_.substring(0, image_mustnothave_.indexOf("%"));
        if (image_mustnothave_.length() > image_mustnothave_.indexOf("%") + 1)
            image_mustnothave_ = image_mustnothave_.substring(image_mustnothave_.indexOf("%") + 1, image_mustnothave_.length());
        else
            image_mustnothave_ = null;
        
        if (url_string.indexOf(tmp) >= 0)
            has_href = false;
        }
        if (image_mustnothave_ != null && url_string.indexOf(image_mustnothave_) >= 0)
        has_href = false;
      
        image_mustnothave_ = original_image_mustnothave_;
    }

    // return true if the link is valid and false if not
    if (href_musthave_==null || has_href)
    {
        // might be another URL
        if (depth < app_.maxDepth())
        {
        if (!new_url_string.startsWith(url_string))
        {
            return true;
        }
        }
    }
    return false;
    }

    /** Adds an image to the stored downloaded images as a triplet.
     *  Ensures that the number of images downloaded but not displayed at
     *  anyone time is controlled by using a buffer. If the buffer is
     *  full this function will wait until space becomes available before
     *  continuing. It also restricts the
     *  total number of images to download as specified by the applet.
     *
     *  @param url the image to download
     *  @param from_url the url that this image was sourced from
     *  @param img_name the name of the image */
    public void add_image(URL url, String from_url, String img_name)
    {
    try {

        boolean had_to_wait = false;

        // ensure that we don't download too many images
        while (download_images_.size() >= buffer_size_)
        {
        had_to_wait = true;
        Thread.sleep(delay_);
        }
        
        // get the image from the url
        Image image = Toolkit.getDefaultToolkit().getImage(url);

        // push image onto the downloaded images
        download_images_.push(image,from_url, img_name);
        
        // if have completed the maximum number of downloads for the
        // application then stop
        if (download_images_.size() == app_.maxDownloads())
        {
            stop();
        }
        
    }
    catch (Exception e) {
        thread_running_ = false;
        stop();
        e.printStackTrace();
    }
    }

    /** Connects to the starting url and looks for all images and links from this
     *  original page. Image links are processed first, so that any images found can be
     *  downloaded immediately and placed on the applet. Secondly, the links to other
     *  pages are recursively processed by this function and treated as a starting url
     *
     *  @param new_url the url from which to start searching for images and links
     *  @param depth the number of links that have been followed on this path */
    public void rec_add_images(String new_url, int depth)
    {
    System.err.println("Parsing url = " + new_url);
    
    if (already_visited(new_url)) return;

    // check if there is a scenario where external hyperlinks are being used
    externalLinks();
    String img_name = new String();

    // connect to the url
    CURL curl = new CURL(new_url);
    if (curl.connected_ok())
    {
        // read the page
        curl.readAll();

        // get all the <code><img src=</code> links into a vector
        Vector src_links = curl.getSrcLinks();
        
        // process each of the image links according to the parameters given.
        for (int i = 0; i < src_links.size(); i++) 
        {
        URL url = (URL)src_links.elementAt(i);
        String url_string  = url.toString();

        if (image_file_extension(url_string))
            {
            if (filter_image(url_string))
                {
                img_name = url_string.substring(url_string.lastIndexOf("/") + 1, url_string.length());

                if (external_links_ != null && ! external_links_.isEmpty()) {
                    String ext = (String) external_links_.get(img_name);
                    
                    if (ext != null) 
                    add_image(url, ext, img_name);
                    else
                    add_image(url, new_url, img_name);
                }
                else {
                    add_image(url, new_url, img_name);
                }
                }
            }
        
        }

        // get all the <code><a href=</code> links into a vector
        Vector href_links = curl.getHrefLinks();
        
        // process each of the href links according to the parameters given.
        for (int i = 0; i < href_links.size(); i++) 
        {
            URL url = (URL)href_links.elementAt(i);
            String url_string  = url.toString();
            
            if (image_file_extension(url_string))
            {
                if (filter_image(url_string))
                {
                    img_name = url_string.substring(url_string.lastIndexOf("/") + 1, url_string.length());

                    if (external_links_ != null && ! external_links_.isEmpty()) {
                    String ext = (String) external_links_.get(img_name);

                    if (ext != null) 
                        add_image(url, ext, img_name);
                    else
                        add_image(url, new_url, img_name);
                    }
                    else {
                    add_image(url, url_string, img_name);
                    }
                }
            }
            else
            {
                if (filter_href(url_string,new_url,depth))
                {
                    rec_add_images(url_string,depth+1);
                    
                }
            }
        }
    }
   
    else {
        System.err.println("Unable able to download "+new_url);
    }
    }

    
    /** Used in cases where the image maps to a url outside of it's original location.
     *  When used with Greenstone the collage images will refer to documents in the collections
     *  from which the images are sourced. When used individually, the images may be saved into
     *  a user directory and the pages they reference may be external hyperlinks. 
     *  This function reads that external links file and creates a hash map of the image to
     *  its external hyperlink. If the file does not exist the download thread will continue
     *  and assume the first case, that links are internal. */
    public void externalLinks() {

    try {

        if (starting_url_.indexOf("gsdl") >= 0) {
        external_links_ = null;
        return;
        }
        
        // open a url to the file written
        URL u = new URL(starting_url_ + "externallinks");

        BufferedReader r = new BufferedReader(new InputStreamReader(u.openStream()));
        
        external_links_ = new Hashtable();

        String l = r.readLine();
        // split the line of the space, first part is the image, second part the link
        while (l != null) {

        String tmp1 = new String();
        String tmp2 = new String();

        if (l.indexOf(" ") >= 0) {
            
            tmp1 = l.substring(0, l.indexOf(" "));
            if (l.length() > l.indexOf(" ") + 1)
            tmp2 = l.substring(l.indexOf(" ") + 1, l.length());
            else
            tmp2 = null;
            
            if (tmp2 != null) {
            external_links_.put(tmp1, tmp2);
            //System.err.println(tmp1 + " " + tmp2);
            }
        }
        l = r.readLine();
        }
        
        r.close();

    } catch (Exception e) {
        e.printStackTrace();
        return;
    }
    }

    /** Controls the download thread */
    public void run () 
    {
    System.err.println("Starting download thread.");
    visited_url_ = new Hashtable();
    rec_add_images(starting_url_,1);

    System.err.println("Download thread finished.");
    }
}