source: trunk/gli/src/org/greenstone/gatherer/download/DownloadJob.java@ 13466

/**
 *#########################################################################
 *
 * A component of the Gatherer application, part of the Greenstone digital
 * library suite from the New Zealand Digital Library Project at the
 * University of Waikato, New Zealand.
 *
 * <BR><BR>
 *
 * Author: John Thompson, Greenstone Digital Library, University of Waikato
 *
 * <BR><BR>
 *
 * Copyright (C) 1999 New Zealand Digital Library Project
 *
 * <BR><BR>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * <BR><BR>
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * <BR><BR>
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 *########################################################################
 */
package org.greenstone.gatherer.download;

import java.awt.event.*;
import java.io.*;
import java.net.*;
import java.util.*;
import javax.swing.tree.*;
import org.greenstone.gatherer.Configuration;
import org.greenstone.gatherer.DebugStream;
import org.greenstone.gatherer.Dictionary;
import org.greenstone.gatherer.Gatherer;
import org.greenstone.gatherer.GAuthenticator;
import org.greenstone.gatherer.LocalGreenstone;
import org.greenstone.gatherer.file.WorkspaceTree;
import org.greenstone.gatherer.util.AppendLineOnlyFileDocument;
import org.greenstone.gatherer.util.GURL;
import org.greenstone.gatherer.util.Utility;
import org.greenstone.gatherer.cdm.Argument;
import org.greenstone.gatherer.collection.*;
/**
 * @author John Thompson, Greenstone Digital Library, University of Waikato
 * @version 2.0
 */
public class DownloadJob
    implements ActionListener {
     
    private boolean debug;
    private boolean higher_directories;
    private boolean no_parents;
    private boolean other_hosts; 
    private boolean page_requisites; 
    private boolean quiet;

    private AppendLineOnlyFileDocument download_log;

    private DownloadProgressBar progress;

    private GURL initial = null;
    private GURL url = null;

   
    // private TreeModel model;

    private int depth;
    private int previous_state;
    private int state;

    private String download_url = "";

    //    private String current_url;
    //    private String destination;
    private String proxy_pass;
    private String proxy_user;

    private Vector encountered_urls;
    private Vector failed_urls;
    private Download download;
    private DownloadScrollPane mummy;
    private HashMap download_option;

    public static int COMPLETE = 0;
    public static int PAUSED   = 1;
    public static int RUNNING  = 2;
    public static int STOPPED  = 3;

    public static int UNKNOWN_MAX  = 0;
    public static int DEFINED_MAX  = 1;
    public static int UNDEFINED_MAX  = 2;

    private String mode = null;
   
    private String proxy_url;

    /**
     */
    public DownloadJob(Download download, String proxy_pass, String proxy_user, DownloadScrollPane mummy, String mode, String proxy_url) {
    URL url = null;
    int folder_hash;

        this.proxy_url =  proxy_url;
    
        download_option = downloadToHashMap(download);
    if (!mode.equals("Z3950") && !mode.equals("SRW")) {
        Argument url_arg = (Argument)download_option.get((String)"url");
        download_url =  url_arg.getValue();
        
    }
    else {
        Argument host_arg = (Argument)download_option.get((String)"host");
        Argument port_arg = (Argument)download_option.get((String)"port");
        download_url = host_arg.getValue() + ":" +port_arg.getValue();
    }
       
    folder_hash = download_url.hashCode();
    String log_filename = Utility.getLogDir(null) + "download-"+ mode + folder_hash + ".log";
    File log_file = new File(log_filename);
    if(log_file.exists()) {
        log_file.delete();
    }

    File parent_log_file = log_file.getParentFile();
    parent_log_file.mkdirs();
    parent_log_file = null;
    log_file = null;

    this.download_log = new AppendLineOnlyFileDocument(log_filename, false);
   
    this.proxy_pass = proxy_pass;
    this.proxy_user = proxy_user;
    this.mummy = mummy;
    this.mode = mode;
    this.download = download;

    progress = new DownloadProgressBar(this,download_url, true);
    encountered_urls = new Vector();
    failed_urls = new Vector();

    previous_state = STOPPED;
    state = STOPPED;
    }

    private HashMap downloadToHashMap(Download download)
    {
    HashMap download_option = new HashMap();
    ArrayList arguments = download.getArguments(true, false);
    for(int i = 0; i < arguments.size(); i++) {
        Argument argument = (Argument) arguments.get(i);
        download_option.put(argument.getName(), argument);
    }
    return download_option;
    }

    /** Depending on which button on the progress bar was pushed,
     * this method will affect the state of the DownloadJob and perhaps make
     * calls to wget.class if necessary.
     * @param event The ActionEvent fired from within the DownloadProgressBar
     * which we must respond to.
     */
    public void actionPerformed(ActionEvent event) {
    // The stop_start_button is used to alternately start or stop the
    // job. If the current state of the job is paused then this
    // restart is logically equivelent to a resume.
    if(event.getSource() == progress.stop_start_button) {
        previous_state = state;
        if (state == RUNNING) {
        state = STOPPED;
        } else {
        //previous_state = state;
        state = RUNNING;
        mummy.resumeThread();
        }
    }
    else if (event.getSource() == progress.close_button) {
        if(state == RUNNING) {
        previous_state = state;
        state = STOPPED; // do we need to do anything else to stop this?
        } 
        mummy.deleteDownloadJob(this);
    }
    }


    public void callDownload() {
    
    ArrayList command_list = new ArrayList();
        if (Utility.isWindows()) {
        command_list.add(Configuration.perl_path);
        command_list.add("-S");
    }
    command_list.add(LocalGreenstone.getBinScriptDirectoryPath()+"downloadfrom.pl");
    command_list.add("-download_mode");
    command_list.add(mode);
    command_list.add("-cache_dir");
    command_list.add(Gatherer.getGLIUserCacheDirectoryPath());
    
    ArrayList all_arg = download.getArguments(true,false);
    for(int i = 0; i < all_arg.size(); i++) {
        Argument argument = (Argument) all_arg.get(i);
        if(argument.isAssigned()) {
        command_list.add("-" + argument.getName());
        if(argument.getType() != Argument.FLAG) {
            command_list.add(argument.getValue());
        }
        }   
    }

    String [] cmd = (String []) command_list.toArray(new String[0]);
    DebugStream.println("Download job, "+command_list);

    if (previous_state == DownloadJob.COMPLETE) {
        progress.mirrorBegun(true, true);
    }
    else {
        progress.mirrorBegun(false, true);
    }

    try {
        Runtime rt = Runtime.getRuntime();

        String [] env = null;
     
        Process prcs = null; 

         
        if (Utility.isWindows()) {
                prcs = rt.exec(cmd);   
        }
        else {
        if (proxy_url != null && !proxy_url.equals("")) {           
            // Specify proxies as environment variables
            // Need to manually specify GSDLHOME and GSDLOS also
            env = new String[4];
                    proxy_url = proxy_url.replaceAll("http://","");
            env[0] = "http_proxy=http://"+proxy_url;
            env[1] = "ftp_proxy=ftp://"+proxy_url;
            env[2] = "GSDLHOME=" + Configuration.gsdl_path;
            env[3] = "GSDLOS=" + Gatherer.client_operating_system;
            prcs = rt.exec(cmd, env); 
        } 
        else {
            // Will inherit the GLI's environment, with GSDLHOME and GSDLOS set
            prcs = rt.exec(cmd);
        }
        }
 
        //System.out.println(newcmd);       

        InputStreamReader isr = new InputStreamReader(prcs.getErrorStream());
        BufferedReader br = new BufferedReader(isr);
        // Capture the standard error stream and seach for two particular occurances.
        String line="";
        boolean ignore_for_robots = false;
        int max_download = DownloadJob.UNKNOWN_MAX;


        while ((line = br.readLine()) != null && !line.trim().equals("<<Finished>>") && state != STOPPED) {

        if ( max_download == DownloadJob.UNKNOWN_MAX) {
            if(line.lastIndexOf("<<Defined Maximum>>") != -1) {
            max_download = DownloadJob.DEFINED_MAX;
            }
            else if (line.lastIndexOf("<<Undefined Maximum>>") != -1) {
            max_download = DownloadJob.UNDEFINED_MAX;
            }
        }
        else if(max_download == DownloadJob.UNDEFINED_MAX) {
            DebugStream.println(line);
            download_log.appendLine(line);
            // The first magic special test is to see if we've just
            // asked for the robots.txt file. If so we ignore
            // the next add and then the next complete/error.
            if(line.lastIndexOf("robots.txt;") != -1) {
            DebugStream.println("***** Requesting robot.txt");
            ignore_for_robots = true;
            }
            // If line contains "=> `" display text as the 
            // currently downloading url. Unique to add download.
            else if(line.lastIndexOf("=> `") != -1) {
            if(!ignore_for_robots) {
                // Add download
                String new_url = line.substring(line.indexOf("`") + 1, line.lastIndexOf("'"));
                //addDownload("http:/" + new_url);
            }
            }
            // If line contains "/s) - `" set currently 
            // downloading url to "Download Complete".
            else if(line.lastIndexOf("/s) - `") != -1) {
            String current_file_downloading = line.substring(line.indexOf("`") + 1, line.lastIndexOf("'"));
            if(!ignore_for_robots) {
                DebugStream.println("Not ignore for robots");
                // Download complete
                downloadComplete(current_file_downloading);
            }
            else {
                DebugStream.println("Ignore for robots");
                ignore_for_robots = false;
            }
            }
            // The already there line begins "File `..." However this
            // is only true in english, so instead I looked and there
            // are few (if any at all) other messages than those above
            // and not overwriting messages that use " `" so we'll 
            // look for that. Note this method is not guarenteed to be
            // unique like the previous two.
            else if(line.lastIndexOf(" `") != -1) {
            // Not Overwriting
            DebugStream.println("Already there.");
            String new_url = 
                line.substring(line.indexOf("`") + 1, line.lastIndexOf("'"));
            //addDownload("http:/" + new_url);
            downloadWarning();
            }
            // Any other important message starts with the time in the form hh:mm:ss
            else if(line.length() > 7) {
            if(line.charAt(2) == ':' && line.charAt(5) == ':') {
                if(!ignore_for_robots) {
                DebugStream.println("Error.");
                downloadFailed();
                }
                else {
                ignore_for_robots = false;
                }
            }
            }
        }
        else if (max_download == DownloadJob.DEFINED_MAX) {
            if (line.lastIndexOf("<<Total number of record(s):") != -1) {
            String total_ID = line.substring(line.indexOf(":") + 1, line.indexOf(">"));
            progress.setTotalDownload((Integer.valueOf(total_ID)).intValue());
            progress.resetFileCount();
            }
            else if (line.lastIndexOf("<<Done>>") != -1) {
            progress.increaseFileCount();
            }
            else if(line.lastIndexOf("<<Done:") != -1) {
            String completed_amount = line.substring(line.indexOf(":") + 1, line.indexOf(">"));
            progress.increaseFileCount((Integer.valueOf(completed_amount)).intValue());
            }
            
            DebugStream.println(line);
            download_log.appendLine(line);
        }
        else {
            System.out.println("Error!!");
            System.exit(-1);
        }
        }
        if(state == STOPPED) {
        isr.close();
        prcs.destroy(); // This doesn't always work, but it's worth a try
        }
       
    }
    catch (Exception ioe) {
        //message(Utility.ERROR, ioe.toString());
        //JTest
        DebugStream.printStackTrace(ioe);
    }
    // If we've got to here and the state isn't STOPPED then the
    // job is complete.
    if(state == DownloadJob.RUNNING) {
        progress.mirrorComplete();
        previous_state = state;
        state = DownloadJob.COMPLETE;
        
    }
    // refresh the workspace tree
    Gatherer.g_man.refreshWorkspaceTree(WorkspaceTree.DOWNLOADED_FILES_CHANGED);
       
    }


    /** Called by the WGet native code when the current download is 
     * completed. In turn all download listeners are informed.
     */
    public void downloadComplete() {
    progress.downloadComplete();
    }


    public void downloadComplete(String current_file_downloading)
    {
    progress.downloadComplete();
    DebugStream.println("Download complete: " + current_file_downloading);
    }


    /** Called by the WGet native code when the requested download returns
     * a status code other than 200.
     */
    public void downloadFailed() {
    // TODO!!
    //failed_urls.add(current_url); // It is the current url that failed
    progress.downloadFailed();
    //DebugStream.println("Download failed: " + current_url);
    } 
     
    /**
     */
    public void downloadWarning() {
    progress.downloadWarning();
    }


    /** 
     * @return A String representing the initial urls host (root node
     * of tree that we are mirroring).
     */
    public String getHost() {
    return url.getHost();
    }

    public AppendLineOnlyFileDocument getLogDocument() {
    return download_log;
    }

    /**
     * @return Returns the progress bar associated with this job.
     */
    public DownloadProgressBar getProgressBar() {
    return progress;
    }

    /** Called to discover if the user wanted this thread to run or if
     * it is paused.
     * @return An int representing the current DownloadJob state.
     */
    public int getState() {
    return state;
    }

    /** Returns the current state of the stop flag for this job.
     * @return A boolean representing whether the user has requested to
     * stop.
     */
    public boolean hasSignalledStop() {
    if(state == DownloadJob.STOPPED || state == DownloadJob.PAUSED || 
       state == DownloadJob.COMPLETE) {
        return true;
    }
    return false;
    }

    public void setState(int state) {
    previous_state = this.state;
    this.state = state;
    }

    /** A convenience call.
     * @return A String representing the url of the initial url (root node of the mirrored tree).
     */
    public String toString() {
    return download_url;
    }

    /** Called by the WGet native code to signal the current progress of 
     * downloading.
     * @param current A long representing the number of bytes that have 
     * been downloaded since the last update. 
     * @param expected A long representing the total number of bytes
     * expected for this download.
     */
    public void updateProgress(long current, long expected) {
    progress.updateProgress(current, expected);
    }
}