source: trunk/gli/src/org/greenstone/gatherer/shell/GShell.java@ 9095

/**
 *#########################################################################
 *
 * A component of the Gatherer application, part of the Greenstone digital
 * library suite from the New Zealand Digital Library Project at the
 * University of Waikato, New Zealand.
 *
 * <BR><BR>
 *
 * Author: John Thompson, Greenstone Digital Library, University of Waikato
 *
 * <BR><BR>
 *
 * Copyright (C) 1999 New Zealand Digital Library Project
 *
 * <BR><BR>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * <BR><BR>
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * <BR><BR>
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 *########################################################################
 */
package org.greenstone.gatherer.shell;

import java.io.*;
import java.net.*;
import java.util.ArrayList;
import java.util.Enumeration;
import javax.swing.*;
import javax.swing.event.*;
import javax.swing.tree.*;
import org.greenstone.gatherer.Configuration;
import org.greenstone.gatherer.DebugStream;
import org.greenstone.gatherer.Dictionary;
import org.greenstone.gatherer.Gatherer;
import org.greenstone.gatherer.GathererApplet;
import org.greenstone.gatherer.cdm.CollectionConfiguration;
import org.greenstone.gatherer.cdm.CollectionDesignManager;
import org.greenstone.gatherer.cdm.CollectionMetaManager;
import org.greenstone.gatherer.cdm.CollectionMeta;
import org.greenstone.gatherer.metadata.DocXMLFileManager;
import org.greenstone.gatherer.util.StaticStrings; 
import org.greenstone.gatherer.util.Utility;

/** The <strong>GShell</strong> is reponsible for running a separately threaded process in the command shell. This is necessary for executing the Perl Scripts and also for other system related funcitonality.
 */
public class GShell
    extends Thread {
    /** A flag used to determine if this process has been asked to cancel. */
    private boolean cancel = false;
    /** The list of listeners associated with this class. */
    private EventListenerList listeners = null;
    /** The current status of this shell process. */
    private int status = -1;
    /** The type of message being sent. */
    private int msg_type = -1;
    /** The type of shell process. */
    private int type = -1;
    /** The caller of this process, and thus the class most interested in messages. */
    private GShellListener caller = null;
    /** The progress monitor associated with this process. */
    private GShellProgressMonitor progress = null;
    /** Arguments to be given to the process (including the executable you are calling. */
    private String args[] = null;
    /** Element in process type enumeration. */
    static final public int BUILD = 0;
    /** Element in process type enumeration. */
    static final public int IMPORT = 1;
    /** Element in process type enumeration. */
    static final public int NEW = 2;
    /** Element in process type enumeration. */
    static final public int EXPORT = 3;
    /** Element in process type enumeration. */
    static final public int CONVERT = 4;
    /** Element in process type enumeration. */
    static final public int OTHER = 5;
    /** Element in status type enumeration. */
    static final public int ERROR = 0;
    /** Element in status type enumeration. */
    static final public int OK = 1;
    static final public int CANCELLED = 2;
    /** Element in process type name enumeration. */
    static public String GSHELL_BUILD = "gshell_build";
    /** Element in process type name enumeration. */
    static public String GSHELL_IMPORT = "gshell_import";
    /** Element in process type name enumeration. */
    static public String GSHELL_NEW = "gshell_new";
    /** Element in process type name enumeration */
    static public String GSHELL_EXPORT = "gshell_export";
    /** Element in process type name enumeration */
    static public String GSHELL_CONVERT = "gshell_convert";

    /** Determine if the given process is still executing. It does this by attempting to throw an exception - not the most efficient way, but the only one as far as I know
     * @param process the Process to test
     * @return true if it is still executing, false otherwise
     */
    static public boolean processRunning(Process process) {
    boolean process_running = false;

    try {
        process.exitValue(); // This will throw an exception if the process hasn't ended yet.
    }
    catch(IllegalThreadStateException itse) {
        process_running = true;
    }
    catch(Exception exception) {
        DebugStream.printStackTrace(exception);
    }
    return process_running;
    }

    /** Constructor gatherer all the data required to create a new process, and emit meaningfull messages.
     * @param args A <strong>String[]</strong> containing the arguments to the process thread, including the name of the executable.
     * @param type An <strong>int</strong> that indicates what group of processes this process belongs to, as some are treated slightly differently (i.e an IMPORT type process is always followed by a BUILD one).
     * @param msg_type As process threads may be background (like a makecol.pl call) or important processes in their own right (such as an IMPORT-BUILD) we must be able to set what level messages posted by this class will have by usings this <strong>int</strong>.
     * @param caller The default <i>GShellListener</i> that is interested in the progress of this process.
     * @param progress The <i>GShellProgressMonitor</i> associated with this process.
     * @param name A <strong>String</strong> identifier given to the process, for convience and debug reasons.
     */
    public GShell(String args[], int type, int msg_type, GShellListener caller, GShellProgressMonitor progress, String name) {
    super(name);
    this.args = args;
    this.msg_type = msg_type;
    this.type = type;
    this.caller = caller;
    this.progress = progress;
    this.status = 0;
    // Lower this jobs priority
    this.setPriority(Thread.MIN_PRIORITY);
    listeners = new EventListenerList();
    listeners.add(GShellListener.class, caller);
    }
    /** This method adds another shell listener to this process.
     * @param listener The new <i>GShellListener</i>.
     */
    public void addGShellListener(GShellListener listener) {
    listeners.add(GShellListener.class, listener);
    }
    /** This method removes a certain shell listener from this process.
     * @param listener The <i>GShellListener</i> to be removed.
     */
    /* private void removeGShellListener(GShellListener listener) {
       listeners.remove(GShellListener.class, listener);
       } */



    protected boolean got_stream_char(InputStreamReader isr, StringBuffer line_buffer,
                      BufferedOutputStream bos) throws IOException 
    {
    // Hopefully this doesn't block if the process is trying to write to STDOUT/STDERR.

    boolean input_status = false;

    if(isr.ready()) {
        input_status = true;
        int c = isr.read();
        if(c == '\n' || c == '\r') {
        if(line_buffer.length() > 0) {
            String line = line_buffer.toString();
            DebugStream.println("* " + line + " *");
            fireMessage(type, typeAsString(type) + "> " + line, status, bos);
            line = null;
            line_buffer = new StringBuffer();
        }
        }
        else {
        line_buffer.append((char)c);
        }
    }

    return input_status;
    }


    protected StringBuffer get_stream_char(InputStreamReader isr, StringBuffer line_buffer,
                   BufferedOutputStream bos) throws IOException 
    {
    int c = isr.read();
    ///atherer.println("isr: '" + (char) c + "'");
    if(c == '\n' || c == '\r') {
        if(line_buffer.length() > 0) {
        String line = line_buffer.toString();
        // DebugStream.println("* " + line + " *");
        fireMessage(type, typeAsString(type) + "> " + line, status, bos);
        line_buffer = new StringBuffer();
        }
    }
    else {
        line_buffer.append((char)c);
    }

    return line_buffer;
    }


    protected int check_for_error(String line, StringBuffer error_list, 
                  int error_count)
    {
    if (line.startsWith("ERROR:")) {
        error_count++;
        if (error_count<10) {
        error_list.append(line);
        }
        else if (error_count==10) {
        error_list.append("...\n");
        }
    }
    return error_count;
    }
        

    protected void runRemote(String[] args, BufferedOutputStream bos) 
    {
    if(hasSignalledStop()) { return; }
    int error_count = 0;
    StringBuffer error_list = new StringBuffer();
    try {
        if (type == IMPORT) {
        String col_name = args[args.length-1];
        if (progress!=null) {
            progress.messageOnProgressBar("Uploading data to server");
        }

        String collect_directory_path = Gatherer.getCollectDirectoryPath();
        
        // zip up import folder
        Utility.zipup(collect_directory_path, col_name, "import", this);

        if(hasSignalledStop()) { return; }
        // upload it to gsdl server
        GathererApplet.upload_url_zip(col_name, "import", this);

        if(hasSignalledStop()) { return; }
        // upload etc folder to server (need collect.cfg and any hfiles)
        Utility.zipup(collect_directory_path, col_name, "etc", this);

        if(hasSignalledStop()) { return; }
        GathererApplet.upload_url_zip(col_name, "etc", this);

        String collection_directory_path = Gatherer.c_man.getCollectionDirectoryPath(col_name);
        File img_dir = new File(Gatherer.c_man.getCollectionImagesDirectoryPath());
        if (img_dir.exists() && img_dir.list().length > 0) {
            // upload images/ directory to server
            if(hasSignalledStop()) { return; }
            Utility.zipup(collect_directory_path, col_name, "images", this);

            if(hasSignalledStop()) { return; }
            GathererApplet.upload_url_zip(col_name, "images", this);            
        }

        if(hasSignalledStop()) { return; }
        // see if collection specific image needs uploading
        CollectionMetaManager cmm 
            = CollectionDesignManager.collectionmeta_manager;
        
        CollectionMeta icon_collection_collectionmeta 
            = cmm.getMetadatum(CollectionConfiguration.COLLECTIONMETADATA_ICONCOLLECTION_STR);
        CollectionMeta icon_collection_small_collectionmeta 
            = cmm.getMetadatum(CollectionConfiguration.COLLECTIONMETADATA_ICONCOLLECTIONSMALL_STR);
        
        String ics_text = icon_collection_collectionmeta.getValue(CollectionMeta.TEXT);
        String icsc_text = icon_collection_small_collectionmeta.getValue(CollectionMeta.TEXT);
        if ((ics_text != null) && (ics_text != "")) {
            // Stub code for detecting when collectoin image changed
            // => need to upload images
            // System.err.println("**** ics_text = " + ics_text);
        }
        
        if (progress!=null) {
            progress.messageOnProgressBar("");
        }
        }
        
        if(hasSignalledStop()) { return; }
        String perl_cmd = args[0];
        int perl_cmd_root = perl_cmd.lastIndexOf(File.separator);
        
        if (perl_cmd_root > 0) {
        String perl_cmd_cut = perl_cmd.substring(perl_cmd_root+1);
        perl_cmd = perl_cmd_cut;
        }
        
        String launch  = Gatherer.cgiBase + "launch";
        launch = launch + "?cmd=" + perl_cmd;
        
        for(int i = 1; i<args.length; i++) {
        
        String arg = args[i];
        
        if (arg.equals("-collectdir") || arg.equals("-importdir")
            || arg.equals("-builddir")) {
            // skip it
            i++;
            continue;
        }
        
        launch += "&";
        
        if(arg.startsWith(StaticStrings.MINUS_CHARACTER)) {
            String name = arg.substring(1);
            launch = launch + name + "=";
            if (i+1<args.length-1) {
            if (!args[i+1].startsWith(StaticStrings.MINUS_CHARACTER)) {
                i++;
                String val = URLEncoder.encode(args[i],"UTF-8");
                launch = launch + val;
            }
            }
        }
        else {
            launch = launch + "c=" + arg;
        }
        }
        
        System.err.println("**** launch url = " + launch);
        // fireMessage(type, Dictionary.get("GShell.Command") + ": " + launch, status, null); // ****
        
        URL launch_url = new URL(launch);
        URLConnection launch_connection = launch_url.openConnection();
        InputStream stdis = launch_connection.getInputStream();
        InputStreamReader stdisr = new InputStreamReader(stdis, "UTF-8");
        
        BufferedReader stdbr = new BufferedReader(stdisr);
        
        if (type == GShell.NEW) {
        while(true) {
            String line = stdbr.readLine();
            if (line == null) { break; }
            error_count = check_for_error(line,error_list,error_count);
        }
        }
        else {
        while(!hasSignalledStop()) {
            String line = stdbr.readLine();
            if (line == null) { break; }
            error_count = check_for_error(line,error_list,error_count);
            fireMessage(type, typeAsString(type) + "> " + line, status, bos);
        }
        System.err.println("Wants to stop (1)."); 
        }
        stdbr.close();
        
        if (error_count>0) {
        status = ERROR;
        System.err.println(error_list);
        if (type != GShell.NEW) {
            fireMessage(type, typeAsString(type) + "> " + error_list, status, null);
        }
        }
        else if(hasSignalledStop()) {
        //User pressed the cancel button.
        //Status already set in hasSignalledStop method.
        }
        else {
        status = OK;
        fireMessage(type, typeAsString(type) + "> " + Dictionary.get("GShell.Success"), status, null);
        }


    }
    // Exception
    catch (Exception exception) {
        System.err.println("Exception in GShell.runRemote() - unexpected");
        DebugStream.printStackTrace(exception);
        status = ERROR;
    }
    }


    protected void runLocal(String[] args, BufferedOutputStream bos) 
    {
    try {
        String command = "";
        for(int i = 0; i < args.length; i++) {
        command = command + args[i] + " ";
        }
        
        DebugStream.println("Command: "+command);
        ///ystem.err.println("Command: " + command);
        fireMessage(type, Dictionary.get("GShell.Command") + ": " + command, status, null);
        
        Runtime rt = Runtime.getRuntime();
        Process prcs = rt.exec(args);
        
        InputStreamReader eisr = new InputStreamReader( prcs.getErrorStream(), "UTF-8" );
        InputStreamReader stdisr = new InputStreamReader( prcs.getInputStream(), "UTF-8"  );
    
        StringBuffer eline_buffer = new StringBuffer();
        StringBuffer stdline_buffer = new StringBuffer();
    
        while(type != GShell.NEW && processRunning(prcs) && !hasSignalledStop()) {
        // Hopefully this doesn't block if the process is trying to write to STDOUT.
        if((eisr!=null) && eisr.ready()) { 
            eline_buffer = get_stream_char(eisr,eline_buffer,bos);
        }
        // Hopefully this won't block if the process is trying to write to STDERR
        else if(stdisr.ready()) {
            stdline_buffer = get_stream_char(stdisr,stdline_buffer,bos);
        }
        else {
            try {
            sleep(100);
            }
            catch(Exception exception) {
            }
        }
        }
    
        if(!hasSignalledStop()) {
        // Of course, just because the process is finished doesn't
        // mean the incoming streams are empty. Unfortunately I've
        // got no chance of preserving order, so I'll process the
        // error stream first, then the out stream
        while(eisr.ready()) {
            eline_buffer = get_stream_char(eisr,eline_buffer,bos);
        }
        
        while(stdisr.ready()) {
            stdline_buffer = get_stream_char(stdisr,stdline_buffer,bos);
        }
        
        // Ensure that any messages still remaining in the string buffers are fired off.
        if(eline_buffer.length() > 0) {
            String eline = eline_buffer.toString();
            //DebugStream.println("Last bit of eline: " + eline);
            fireMessage(type, typeAsString(type) + "> " + eline, status, bos);
            eline = null;
        }
        
        if(stdline_buffer.length() > 0) {
            String stdline = stdline_buffer.toString();
            //DebugStream.println("Last bit of stdline: " + stdline);
            fireMessage(type, typeAsString(type) + "> " + stdline, status, null);
            stdline = null;
        }
        }
        else {
        System.err.println("We've been asked to stop.");
        }
    
    
        if(!hasSignalledStop()) {
        // Now display final message based on exit value
        
        prcs.waitFor(); 

        if(prcs.exitValue() == 0) {
            status = OK;
            fireMessage(type, typeAsString(type) + "> " + Dictionary.get("GShell.Success"), status, null);
        }
        else {
            status = ERROR;
            fireMessage(type, typeAsString(type) + "> " + Dictionary.get("GShell.Failure"), status, null);
        }
        
        eisr.close();
        stdisr.close();
        }
        else {
        // I need to somehow kill the child process. Unfortunately
        // Thread.stop() and Process.destroy() both fail to do
        // this. But now, thankx to the magic of Michaels 'close the
        // stream suggestion', it works fine (no it doesn't!)
        prcs.getInputStream().close();
        prcs.getErrorStream().close();
        prcs.getOutputStream().close();
        prcs.destroy();
        status = CANCELLED;
        }
    }
    // Exception
    catch (Exception exception) {
        DebugStream.println("Exception in GShell.runLocal() - unexpected");
        DebugStream.printStackTrace(exception);
        status = ERROR;
    }
    }
    
    

    /** Any threaded class must include this method to allow the thread body to be run. */
    public void run() {

    String col_name = args[args.length-1];      

    // Determine if the user has asked for an outfile.
    String out_name = null;
    BufferedOutputStream bos = null;
    if(type == IMPORT || type == BUILD) {
        if(type == IMPORT) {
        out_name = (String) Gatherer.c_man.getCollection().build_options.getImportValue("out");
        }
        else {
        out_name = (String) Gatherer.c_man.getCollection().build_options.getBuildValue("out");
        }
        if(out_name != null && out_name.length() > 0) {
        try {
            bos = new BufferedOutputStream(new FileOutputStream(new File(out_name), true));
        }
        catch (Exception error) {
            DebugStream.printStackTrace(error);
        }
        }
    }

    // Issue a processBegun event
    fireProcessBegun(type, status);
    if (Gatherer.isGsdlRemote) {
        runRemote(args,bos);
    }
    else {
        runLocal(args,bos);
    }

    if(status == OK) {
        if (type == NEW) {
        if (Gatherer.isGsdlRemote) {
            GathererApplet.download_url_zip(col_name, ".", this);
            if (!hasSignalledStop()) {
            Utility.unzip(Gatherer.getCollectDirectoryPath(), col_name);
            }
        }
        }
        else if(type == IMPORT) {
        
        // download the archives directory (if gsdl server is remote)           
        if (Gatherer.isGsdlRemote) {

            if (progress!=null) {
            progress.messageOnProgressBar("Downloading archive data from server");
            }

            Utility.delete(Gatherer.c_man.getCollectionArchivesDirectoryPath()); // remove current archives
            GathererApplet.download_url_zip(col_name, "archives", this);
               if (!hasSignalledStop()) {
               Utility.unzip(Gatherer.getCollectDirectoryPath(), col_name);
               }
               if (progress!=null) {
               progress.messageOnProgressBar("");
               }      
        }

        // Refresh the DocXMLFileManager
        fireMessage(type, typeAsString(type) + "> " + Dictionary.get("GShell.Parsing_Metadata_Start"), status, null);
        DocXMLFileManager.clearDocXMLFiles();
        DocXMLFileManager.loadDocXMLFiles(new File(Gatherer.c_man.getCollectionArchivesDirectoryPath()));
        fireMessage(type, typeAsString(type) + "> " + Dictionary.get("GShell.Parsing_Metadata_Complete"), status, null);
        }

        else if(type == BUILD) {
        // download the building directory (if gsdl server is remote)
        if (Gatherer.isGsdlRemote) {
            if (progress!=null) {
            progress.messageOnProgressBar("Downloading index data from server");
            }

            File buildDir = new File(Gatherer.c_man.getCollectionBuildingDirectoryPath());
            Utility.delete(buildDir); // remove current build dir
            buildDir.mkdir(); //Make a clean dir
            GathererApplet.download_url_zip(col_name, "building/build.cfg", this); //Only need build.cfg
            if (!hasSignalledStop()) {
            Utility.unzip(Gatherer.getCollectDirectoryPath(), col_name);
            }
            
            if (progress!=null) {
            progress.messageOnProgressBar("");
            }
        }
        }
    }

    // We're done.
    fireProcessComplete(type, status);
    // Close bos
    if(bos != null) {
        try {
        bos.close();
        bos = null;
        }
        catch(Exception error) {
        DebugStream.printStackTrace(error);
        }
    }
    }
    /** Method for firing a message to all interested listeners.
     * @param type An <strong>int</strong> indicating the process type.
     * @param message The message as a <strong>String</strong>.
     * @param status An <strong>int</strong> specifying the current status of the process.
     */
    public void fireMessage(int type, String message, int status, BufferedOutputStream bos) {
    GShellEvent event = new GShellEvent(this, 0, type, message, status);
    // If there is a progress monitor attached, pass the event to it first. Note that we pass a queue of messages as the processing may cause one message to be split into several.
    ArrayList message_queue = new ArrayList();
    message_queue.add(event);
    if(progress != null) {
        progress.process(message_queue);
    }
    for(int j = 0; j < message_queue.size(); j++) {
        GShellEvent current_event = (GShellEvent) message_queue.get(j);
        // If the event hasn't been vetoed, pass it on to other listeners
        if(!current_event.isVetoed()) {
        Object[] concerned = listeners.getListenerList();
        for(int i = 0; i < concerned.length ; i++) {
            if(concerned[i] == GShellListener.class) {
            ((GShellListener)concerned[i+1]).message(current_event);
            }
        }
        concerned = null;
        }
    }
    // And if we have a buffered output stream from error messages, send the message there
    if(bos != null) {
        try {
        bos.write(message.getBytes(), 0, message.length());
        }
        catch(Exception exception) {
        DebugStream.println("Exception in GShell.fireMessage() - unexpected");
        DebugStream.printStackTrace(exception);
        }
    }
    message_queue = null;
    event = null;
    }

    /** Method for firing a process begun event which is called, strangly enough, when the process begins.
     * @param type An <strong>int</strong> indicating the process type.
     * @param status An <strong>int</strong> specifying the current status of the process.
     */
    protected void fireProcessBegun(int type, int status) {
    // Start the progres monitor if available
    if(progress != null) {
        progress.start();
    }
    // Fire an event
    GShellEvent event = new GShellEvent(this, 0, type, "", status);
    Object[] concerned = listeners.getListenerList();
    for(int i = 0; i < concerned.length ; i++) {
        if(concerned[i] == GShellListener.class) {
        ((GShellListener)concerned[i+1]).processBegun(event);
        }
    }
    }
    /** Method for firing a process complete event which is called, no surprise here, when the process ends.
     * @param type An <strong>int</strong> indicating the process type.
     * @param status An <strong>int</strong> specifying the current status of the process.
     */
    protected void fireProcessComplete(int type, int status) {
    // Tidy up by stopping the progress bar. If it was cancelled then the cancel command has arrived via the progress bars and they don't need to be told again (it actually causes problems).
    if(progress != null && status != CANCELLED) {
        progress.stop();
    }
    
    // If we were cancelled, and we are lower details modes, fire off one last message.
    if(status == CANCELLED && Configuration.getMode() <= Configuration.SYSTEMS_MODE) {
        GShellEvent current_event = new GShellEvent(this, 0, type, Dictionary.get("GShell.Build.BuildCancelled"), status);
        Object[] concerned = listeners.getListenerList();
        for(int i = 0; i < concerned.length ; i++) {
        if(concerned[i] == GShellListener.class) {
            ((GShellListener)concerned[i+1]).message(current_event);
        }
        }
        concerned = null;
    }
    // And firing off an event
    GShellEvent event = new GShellEvent(this, 0, type, "", status);
    Object[] concerned = listeners.getListenerList();
    for(int i = 0; i < concerned.length ; i++) {
        if(concerned[i] == GShellListener.class) {
        ((GShellListener)concerned[i+1]).processComplete(event);
        }
    }
    }

    /** Method to determine if the user, via the progress monitor, has signalled stop.
     * @return A <strong>boolean</strong> indicating if the user wanted to stop.
     */
    public boolean hasSignalledStop() {
    boolean has_signalled_stop = false;
    if(progress != null) {
        has_signalled_stop = progress.hasSignalledStop();
    }
    if(has_signalled_stop) {
        status = CANCELLED;
    }
    return has_signalled_stop;
    }

    /** Converts a type into a text representation.
     * @param type An <strong>int</strong> which maps to a shell process type.
     * @return A <strong>String</strong> which is the thread process's text name.
     */
    public String typeAsString(int type) {
    String name = null;
    switch(type) {
    case BUILD:
        name = Dictionary.get("GShell.Build");
        break;
    case IMPORT:
        name = Dictionary.get("GShell.Import");
        break;
    case NEW:
        name = Dictionary.get("GShell.New");
        break;
    case EXPORT:
        name= Dictionary.get("GShell.Export");
        break;
    case CONVERT:
        name = Dictionary.get("GShell.Convert");
        break;
    default:
        name = Dictionary.get("GShell.Other");
    }
    return name;
    }
}