source: main/trunk/gli/src/org/greenstone/gatherer/download/DownloadJob.java@ 22103

/**
 *#########################################################################
 *
 * A component of the Gatherer application, part of the Greenstone digital
 * library suite from the New Zealand Digital Library Project at the
 * University of Waikato, New Zealand.
 *
 * <BR><BR>
 *
 * Author: John Thompson, Greenstone Digital Library, University of Waikato
 *
 * <BR><BR>
 *
 * Copyright (C) 1999 New Zealand Digital Library Project
 *
 * <BR><BR>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * <BR><BR>
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * <BR><BR>
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 *########################################################################
 */
package org.greenstone.gatherer.download;

import java.awt.event.*;
import java.io.*;
import java.net.*;
import java.util.*;
import javax.swing.tree.*;
import org.greenstone.gatherer.Configuration;
import org.greenstone.gatherer.DebugStream;
import org.greenstone.gatherer.Dictionary;
import org.greenstone.gatherer.Gatherer;
import org.greenstone.gatherer.GAuthenticator;
import org.greenstone.gatherer.greenstone.LocalGreenstone;
import org.greenstone.gatherer.file.WorkspaceTree;
import org.greenstone.gatherer.util.AppendLineOnlyFileDocument;
import org.greenstone.gatherer.util.Utility;
import org.greenstone.gatherer.cdm.Argument;
import org.greenstone.gatherer.collection.*;
/**
 * @author John Thompson, Greenstone Digital Library, University of Waikato
 * @version 2.0
 */
public class DownloadJob
    implements ActionListener {
     
    private boolean debug;
    private boolean higher_directories;
    private boolean no_parents;
    private boolean other_hosts; 
    private boolean page_requisites; 
    private boolean quiet;

    private AppendLineOnlyFileDocument download_log;

    private DownloadProgressBar progress;

    private int depth;
    private int previous_state;
    private int state;

    private String download_url = "";
    
    //    private String current_url;
    //    private String destination;
    private String proxy_pass;
    private String proxy_user;

    private Vector encountered_urls;
    private Vector failed_urls;
    private Download download;
    private DownloadScrollPane mummy;
    private HashMap download_option;

    public static int COMPLETE = 0;
    public static int PAUSED   = 1;
    public static int RUNNING  = 2;
    public static int STOPPED  = 3;

    public static int UNKNOWN_MAX  = 0;
    public static int DEFINED_MAX  = 1;
    public static int UNDEFINED_MAX  = 2;
    
    // To prematurely terminate wget, we will need to use sockets and find a free port.
    // We will look at a limited range of ports. This range will be reused (circular buffer)
    private static final int PORT_BASE = 50000;
    private static final int PORT_BLOCK_SIZE = 100;   
    private static int nextFreePort = PORT_BASE;  // Keep track what port numbers we have checked for availability
    int port;                                     // package access. The socket port number this instance of DownloadJob will use
    
    private String mode = null;
   
    private String proxy_url;
    
    /**
     */
    public DownloadJob(Download download, String proxy_pass, String proxy_user, DownloadScrollPane mummy, String mode, String proxy_url) {
    URL url = null;
    int folder_hash;

        this.proxy_url =  proxy_url;
    
        download_option = downloadToHashMap(download);
    if (!mode.equals("Z3950") && !mode.equals("SRW")) {
        Argument url_arg = (Argument)download_option.get((String)"url");
        download_url =  url_arg.getValue();
        
    }
    else {
        Argument host_arg = (Argument)download_option.get((String)"host");
        Argument port_arg = (Argument)download_option.get((String)"port");
        download_url = host_arg.getValue() + ":" +port_arg.getValue();
    }
       
    folder_hash = download_url.hashCode();
    String log_filename = Utility.getLogDir(null) + "download-"+ mode + folder_hash + ".log";
    File log_file = new File(log_filename);
    if(log_file.exists()) {
        log_file.delete();
    }

    File parent_log_file = log_file.getParentFile();
    parent_log_file.mkdirs();
    parent_log_file = null;
    log_file = null;

    this.download_log = new AppendLineOnlyFileDocument(log_filename, false);
   
    this.proxy_pass = proxy_pass;
    this.proxy_user = proxy_user;
    this.mummy = mummy;
    this.mode = mode;
    this.download = download;

    progress = new DownloadProgressBar(this,download_url, true);
    encountered_urls = new Vector();
    failed_urls = new Vector();

    previous_state = STOPPED;
    state = STOPPED;
    }

    private HashMap downloadToHashMap(Download download)
    {
    HashMap download_option = new HashMap();
    ArrayList arguments = download.getArguments(true, false);
    for(int i = 0; i < arguments.size(); i++) {
        Argument argument = (Argument) arguments.get(i);
        download_option.put(argument.getName(), argument);
    }
    return download_option;
    }

    /** Depending on which button on the progress bar was pushed,
     * this method will affect the state of the DownloadJob and perhaps make
     * calls to wget.class if necessary.
     * @param event The ActionEvent fired from within the DownloadProgressBar
     * which we must respond to.
     */
    public void actionPerformed(ActionEvent event) {
    // The stop_start_button is used to alternately start or stop the
    // job. If the current state of the job is paused then this
    // restart is logically equivalent to a resume.
    if(event.getSource() == progress.stop_start_button) {
        previous_state = state;
        if (state == RUNNING) {
        state = STOPPED;
        } else {
        //previous_state = state;
        state = RUNNING;
        mummy.resumeThread();
        }
    }
    else if (event.getSource() == progress.close_button) {
        if(state == RUNNING) {
        previous_state = state;
        state = STOPPED; // do we need to do anything else to stop this?
        } 
        mummy.deleteDownloadJob(this);
    }
    }

    /** Given a portnumber to check, returns true if it is available
     * (if nothing's listening there already). */
    public static boolean isPortAvailable(int portnum) {
    Socket tmpSocket = null;
    try {
        tmpSocket = new Socket("localhost", portnum);
        tmpSocket.close();
        return false;

    } catch(ConnectException ex){
        // "Signals that an error occurred while attempting to connect a socket 
        // to a remote address and port. Typically, the connection was refused
        // remotely (e.g., no process is listening on the remote address/port)."
        System.err.println("Port " + portnum + " not yet in use.");
        tmpSocket = null;
        return true;

    } catch(Exception ex) { 
        // includes BindException "Signals that an error occurred while attempting
        // to bind a socket to a local address and port. Typically, the port is in
        // use, or the requested local address could not be assigned."
        tmpSocket = null;
        return false;
    }
    }

    /** Circular buffer. Modifies the value of nextFreePort (the buffer index). */
    private void incrementNextFreePort() {
    int offset = nextFreePort - PORT_BASE;
    offset = (offset + 1) % PORT_BLOCK_SIZE;
    nextFreePort = PORT_BASE + offset;
    } 

    public void callDownload() {

    ArrayList command_list = new ArrayList();
    
    // the following also works for client-gli if downloading is enabled (when there's a gs2build directory inside gli)
    command_list.add(Configuration.perl_path);
    command_list.add("-S");
    command_list.add(LocalGreenstone.getBinScriptDirectoryPath()+"downloadfrom.pl");
    command_list.add("-download_mode");
    command_list.add(mode);
    command_list.add("-cache_dir");
    command_list.add(Gatherer.getGLIUserCacheDirectoryPath());
    // For the purposes of prematurely terminating wget from GLI (which creates a socket
    // as a communication channel between GLI and Perl), it is important to tell the script
    // that we're running as GLI. Because when running from the command prompt, it should
    // not create this socket and do the related processing.
    command_list.add("-gli");
    
    ArrayList all_arg = download.getArguments(true,false);
    for(int i = 0; i < all_arg.size(); i++) {
        Argument argument = (Argument) all_arg.get(i);
        if(argument.isAssigned()) {
        command_list.add("-" + argument.getName());
        if(argument.getType() != Argument.FLAG) {
            command_list.add(argument.getValue());
        }
        }   
    }

    String [] cmd = (String []) command_list.toArray(new String[0]);
    DebugStream.println("Download job, "+command_list);

    if (previous_state == DownloadJob.COMPLETE) {
        progress.mirrorBegun(true, true);
    }
    else {
        progress.mirrorBegun(false, true);
    }

    try {
        Runtime rt = Runtime.getRuntime();

        String [] env = null;
     
        Process prcs = null; 

         
        if (Utility.isWindows()) {
                prcs = rt.exec(cmd);   
        }
        else {
        if (proxy_url != null && !proxy_url.equals("")) {           
            // Specify proxies as environment variables
            // Need to manually specify GSDLHOME and GSDLOS also
            env = new String[4];
                    proxy_url = proxy_url.replaceAll("http://","");
            env[0] = "http_proxy=http://"+proxy_url;
            env[1] = "ftp_proxy=ftp://"+proxy_url;
            env[2] = "GSDLHOME=" + Configuration.gsdl_path;
            env[3] = "GSDLOS=" + Gatherer.client_operating_system;
            prcs = rt.exec(cmd, env); 
        } 
        else if(Gatherer.isGsdlRemote && Gatherer.isDownloadEnabled) {
            // Not Windows, but running client with download panel
            // Need to manually specify GSDLHOME and GSDLOS
            env = new String[2];
            env[0] = "GSDLHOME=" + Configuration.gsdl_path;
            env[1] = "GSDLOS=" + Gatherer.client_operating_system;
            prcs = rt.exec(cmd, env); 
        }
        else {
            // Will inherit the GLI's environment, with GSDLHOME and GSDLOS set
            prcs = rt.exec(cmd);
        }
        }
        //System.out.println(newcmd);       

        // Can use the following if debugging WgetDownload.pm - Reads debug stmts from the perl process' STDIN stream
        //(new PerlReaderThread(prcs)).start();

        InputStream is = prcs.getInputStream();
        BufferedReader reader = new BufferedReader(new InputStreamReader(is));

        // To be able to stop Wget, we use sockets to communicate with the perl process that launched wget
        if (mode.equals("Web") || mode.equals("MediaWiki")) { // wget download modes other than OAI
        
        // Need to find an available (unused) port within the range we're looking for to pass it
        // the Perl child process, so that it may set up a listening ServerSocket at that port number
        try {
            boolean foundFreePort = false;
            for(int i = 0; i < PORT_BLOCK_SIZE; i++) {

            if(isPortAvailable(nextFreePort)) {
                foundFreePort = true;
                break;
                
            } else {
                incrementNextFreePort();
            }
            }
            
            if(foundFreePort) {
            // Free port number currently found becomes the port number of the socket that this
            // DownloadJob instance will be connecting to when the user wants to prematurely stop Wget.
            this.port = nextFreePort;
            incrementNextFreePort();
            
            } else {
            throw new Exception("Cannot find an available port in the range " 
                        + PORT_BASE + "-" + (PORT_BASE+PORT_BLOCK_SIZE)
                        + "\nwhich is necessary for forcibly terminating wget.");
            }

            // Communicate the chosen port for this DownloadJob instance to the perl process, so
            // that it can set up a ServerSocket at that port to listen for any signal to terminate wget
            OutputStream os = prcs.getOutputStream();
            String p = ""+this.port+"\n";
            System.err.println("Portnumber found: " + p);

            os.write(p.getBytes());
            os.close();
          
        } catch(Exception ex) {
            System.err.println("Sent available portnumber " + this.port + " to process' outputstream.\nBut got exception: " + ex);
        }
        }
        
        BufferedReader br = new BufferedReader(new InputStreamReader(prcs.getErrorStream()));
        // Capture the standard error stream and search for two particular occurrences.
        String line="";
        boolean ignore_for_robots = false;
        int max_download = DownloadJob.UNKNOWN_MAX;
       
        while ((line = br.readLine()) != null && !line.trim().equals("<<Finished>>") && state != STOPPED) {
        if ( max_download == DownloadJob.UNKNOWN_MAX) {
            if(line.lastIndexOf("<<Defined Maximum>>") != -1) {
            max_download = DownloadJob.DEFINED_MAX;
            }
            else if (line.lastIndexOf("<<Undefined Maximum>>") != -1) {
            max_download = DownloadJob.UNDEFINED_MAX;
            }
        }
        else if(max_download == DownloadJob.UNDEFINED_MAX) {
            DebugStream.println(line);
            download_log.appendLine(line);
            // The first magic special test is to see if we've just
            // asked for the robots.txt file. If so we ignore
            // the next add and then the next complete/error.
            if(line.lastIndexOf("robots.txt;") != -1) {
            DebugStream.println("***** Requesting robot.txt");
            ignore_for_robots = true;
            }
            // If line contains "=> `" display text as the 
            // currently downloading url. Unique to add download.
            else if(line.lastIndexOf("=> `") != -1) {
            if(!ignore_for_robots) {
                // Add download
                String new_url = line.substring(line.indexOf("`") + 1, line.lastIndexOf("'"));
                progress.addDownload("file"); //addDownload("http:/" + new_url.substring(cachedir_prefix_length()-1));              
            }
            }
            // If line contains "/s) - `" set currently 
            // downloading url to "Download Complete".
            else if(line.lastIndexOf("/s) - `") != -1) {
            String current_file_downloading = line.substring(line.indexOf("`") + 1, line.lastIndexOf("'"));
            if(!ignore_for_robots) {
                DebugStream.println("Not ignore for robots");
                // Download complete
                downloadComplete(current_file_downloading);
            }
            else {
                DebugStream.println("Ignore for robots");
                ignore_for_robots = false;
            }
            }
            // The already there line begins "File `..." However this
            // is only true in english, so instead I looked and there
            // are few (if any at all) other messages than those above
            // and not overwriting messages that use " `" so we'll 
            // look for that. Note this method is not guarenteed to be
            // unique like the previous two.
            else if(line.lastIndexOf(" `") != -1) {
            // Not Overwriting
            DebugStream.println("Already there.");
            String new_url = 
                line.substring(line.indexOf("`") + 1, line.lastIndexOf("'"));
            progress.addDownload("file"); //addDownload("http:/" + new_url.substring(cachedir_prefix_length()-1));
            downloadWarning();
            }
            // Any other important message starts with the time in the form hh:mm:ss
            else if(line.length() > 7) {
            if(line.charAt(2) == ':' && line.charAt(5) == ':') {
                if(!ignore_for_robots) {
                DebugStream.println("Error.");
                downloadFailed();
                }
                else {
                ignore_for_robots = false;
                }
            }
            }
        }
        else if (max_download == DownloadJob.DEFINED_MAX) {
            if (line.lastIndexOf("<<Total number of record(s):") != -1) {
            String total_ID = line.substring(line.indexOf(":") + 1, line.indexOf(">"));
            progress.setTotalDownload((Integer.valueOf(total_ID)).intValue());
            progress.resetFileCount();  
            progress.addDownload("files"); // for display: "Downloading files"
            }
            else if (line.lastIndexOf("<<Done>>") != -1) {
            progress.increaseFileCount();
            }
            else if(line.lastIndexOf("<<Done:") != -1) {
            String completed_amount = line.substring(line.indexOf(":") + 1, line.indexOf(">"));
            progress.increaseFileCount((Integer.valueOf(completed_amount)).intValue());
            }
            
            DebugStream.println(line);
            download_log.appendLine(line);
        }
        else {
            System.out.println("Error!!");
            System.exit(-1);
        }
        }

        if(state == STOPPED) {
        boolean terminatePerlScript = true;
            
        // When GLI is working with wget-based download modes other than OAI (MediaWiki and Web 
        // download) and the STOP button has been pressed, wget needs to be prematurely terminated.
        // Only wget download modes Web and MediaWiki require the use of sockets to communicate
        // with the perl script in order to get wget to terminate. Other download modes, including
        // wgetdownload mode OAI, can terminate in the traditional manner: close process inputstream 
        // and kill perl process. OAI launches many wgets. So that when the perl process is terminated,
        // the currently running wget will finish off but other wgets are no longer launched. 
        if(prcs != null && (mode.equals("Web") || mode.equals("MediaWiki"))) { 

            // create a socket to the perl child process and communicate the STOP message
            Socket clientSocket = null;
            if(clientSocket == null) {
            try {
                clientSocket = new Socket("localhost", this.port); // connect to the port chosen for this DownloadJob instance
                
                BufferedReader clientReader = new BufferedReader(new InputStreamReader(clientSocket.getInputStream()));
                String response = clientReader.readLine(); // see if we've been connected
                System.err.println("Communicating with perl download script on port " + this.port 
                           + "\nGot response from perl: " + response); 
                
                // Send the STOP signal
                OutputStream os = clientSocket.getOutputStream();
                String message = "<<STOP>>\n";
                os.write(message.getBytes());
                response = clientReader.readLine(); // see whether the stop signal has been received
                System.err.println("GLI sent STOP signal to perl to terminate wget." 
                           + "\nGot response from perl: " + response);

                response = clientReader.readLine(); // see whether the perl script is ready to be terminated
                System.err.println("Got another response from perl: " + response);
                os.close();
                
                clientReader.close();
                clientSocket.close(); // close the clientSocket (the Perl end will close the server socket that Perl opened)
                clientReader = null;
                clientSocket = null;
                
                if(response == null) {
                terminatePerlScript = false;
                }
            } catch(IOException ex) {
                System.err.println("Tried to communicate through client socket - port " + this.port + ", but got exception: " + ex);
            } catch(Exception ex) {
                System.err.println("Tried to open client socket, but got exception: " + ex);
            }
            }
        }
        
        //prcs.getInputStream().close();
        prcs.getErrorStream().close();
        br.close();
        br = null;
        if(terminatePerlScript) {
            prcs.destroy(); // This doesn't always work, but it's worth a try       
            prcs = null;
        }

        // Notify the DownloadScrollPane which is waiting on this job to complete that we are ready 
        synchronized(this) {
            this.notify();
        }
        }
    }
    catch (Exception ioe) {
        //message(Utility.ERROR, ioe.toString());
        //JTest
        DebugStream.printStackTrace(ioe);
    }
    // If we've got to here and the state isn't STOPPED then the
    // job is complete.
    if(state == DownloadJob.RUNNING) {
        progress.mirrorComplete();
        previous_state = state;
        state = DownloadJob.COMPLETE;
    }
    // refresh the workspace tree
    Gatherer.g_man.refreshWorkspaceTree(WorkspaceTree.DOWNLOADED_FILES_CHANGED);
    }


    /** Called by the WGet native code when the current download is 
     * completed. In turn all download listeners are informed.
     */
    public void downloadComplete() {
    progress.downloadComplete();
    }


    public void downloadComplete(String current_file_downloading)
    {
    progress.downloadComplete();
    DebugStream.println("Download complete: " + current_file_downloading);
    }


    /** Called by the WGet native code when the requested download returns
     * a status code other than 200.
     */
    public void downloadFailed() {
    // TODO!!
    //failed_urls.add(current_url); // It is the current url that failed
    progress.downloadFailed();
    //DebugStream.println("Download failed: " + current_url);
    } 
     
    /**
     */
    public void downloadWarning() {
    progress.downloadWarning();
    }

    public AppendLineOnlyFileDocument getLogDocument() {
    return download_log;
    }

    /**
     * @return Returns the progress bar associated with this job.
     */
    public DownloadProgressBar getProgressBar() {
    return progress;
    }

    /** Called to discover if the user wanted this thread to run or if
     * it is paused.
     * @return An int representing the current DownloadJob state.
     */
    public int getState() {
    return state;
    }

    /** Returns the current state of the stop flag for this job.
     * @return A boolean representing whether the user has requested to
     * stop.
     */
    public boolean hasSignalledStop() {
    if(state == DownloadJob.STOPPED || state == DownloadJob.PAUSED || 
       state == DownloadJob.COMPLETE) {
        return true;
    }
    return false;
    }

    public void setState(int state) {
    previous_state = this.state;
    this.state = state;
    }

    /** A convenience call.
     * @return A String representing the url of the initial url (root node of the mirrored tree).
     */
    public String toString() {
    return download_url;
    }

    /** Called by the WGet native code to signal the current progress of 
     * downloading.
     * @param current A long representing the number of bytes that have 
     * been downloaded since the last update. 
     * @param expected A long representing the total number of bytes
     * expected for this download.
     */
    public void updateProgress(long current, long expected) {
    progress.updateProgress(current, expected);
    }

    
    // Inner thread class that reads from process downloadfrom.pl's errorstream
    private class PerlReaderThread extends Thread {
    Process prcs = null;

    public PerlReaderThread(Process proc) {
        this.prcs = proc;
    }

    public void run() {
        try {
        if(prcs != null) {
            String message = null;
            BufferedReader eReader = new BufferedReader(new InputStreamReader(prcs.getInputStream()));
            while(prcs != null && (message = eReader.readLine()) != null) {
            if(!message.equals("\n")) {
                System.err.println("**** Perl STDOUT: " + message);
            }
            }
            
            if(prcs != null && eReader != null) {
            eReader.close();
            eReader = null;
            System.err.println("**** Perl ENDed.");
            }
        }
        } catch(Exception e) {
        System.err.println("Thread - caught exception: " + e);
        }
    }
    }
}