source: main/trunk/gli/src/org/greenstone/gatherer/shell/GShell.java@ 26333

/**
 *#########################################################################
 *
 * A component of the Gatherer application, part of the Greenstone digital
 * library suite from the New Zealand Digital Library Project at the
 * University of Waikato, New Zealand.
 *
 * <BR><BR>
 *
 * Author: John Thompson, Greenstone Digital Library, University of Waikato
 *
 * <BR><BR>
 *
 * Copyright (C) 1999 New Zealand Digital Library Project
 *
 * <BR><BR>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * <BR><BR>
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * <BR><BR>
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 *########################################################################
 */
package org.greenstone.gatherer.shell;

import java.io.*;
import java.net.*;
import java.util.ArrayList;
import java.util.Enumeration;
import java.util.regex.*;
import javax.swing.*;
import javax.swing.event.*;
import javax.swing.tree.*;
import org.greenstone.gatherer.Configuration;
import org.greenstone.gatherer.DebugStream;
import org.greenstone.gatherer.Dictionary;
import org.greenstone.gatherer.Gatherer;
import org.greenstone.gatherer.cdm.CollectionConfiguration;
import org.greenstone.gatherer.cdm.CollectionDesignManager;
import org.greenstone.gatherer.cdm.CollectionMetaManager;
import org.greenstone.gatherer.cdm.CollectionMeta;
import org.greenstone.gatherer.collection.CollectionManager;
import org.greenstone.gatherer.metadata.DocXMLFileManager;
import org.greenstone.gatherer.remote.RemoteGreenstoneServer;
import org.greenstone.gatherer.util.StaticStrings; 
import org.greenstone.gatherer.util.Utility;


/** The <strong>GShell</strong> is reponsible for running a separately threaded process in the command shell. This is necessary for executing the Perl Scripts and also for other system related funcitonality.
 */
public class GShell
    extends Thread {
    /** A flag used to determine if this process has been asked to cancel. */
    private boolean cancel = false;
    private BufferedOutputStream buffered_output_stream = null;
    /** The list of listeners associated with this class. */
    private EventListenerList listeners = null;
    /** The current status of this shell process. */
    private int status = -1;
    /** The type of message being sent. */
    private int msg_type = -1;
    /** The type of shell process. */
    private int type = -1;
    /** The caller of this process, and thus the class most interested in messages. */
    private GShellListener caller = null;
    /** The progress monitor associated with this process. */
    private GShellProgressMonitor progress = null;
    /** Arguments to be given to the process (including the executable you are calling. */
    private String args[] = null;
    /** The command_output returned from executing a process */
    private String commandOutput = null;

    /** Elements in process type enumeration. */
    static final public int BUILD = 0;
    static final public int IMPORT = 1;
    static final public int NEW = 2;
    static final public int EXPORTAS = 3;
    static final public int CDIMAGE = 4;
    static final public int CONVERT = 5;
    static final public int EXPLODE = 6;
    static final public int SRCREPLACE = 7; // for replacing source docs with their html
    static final public int SCHEDULE = 8;
    static final public int DELETE = 9;

    /** Elements in status type enumeration. */
    static final public int ERROR = 0;
    static final public int OK = 1;
    static final public int CANCELLED = 2;

    /** Elements in process type name enumeration. */
    static public String GSHELL_BUILD = "gshell_build";
    static public String GSHELL_IMPORT = "gshell_import";
    static public String GSHELL_NEW = "gshell_new";
    static public String GSHELL_EXPORTAS = "gshell_exportas";
    static public String GSHELL_CDIMAGE = "gshell_cdimage";
    static public String GSHELL_CONVERT = "gshell_convert";
    static public String GSHELL_EXPLODE = "gshell_explode";
    static public String GSHELL_SRCREPLACE = "gshell_srcreplace"; // for replacing source docs with their html versions
    static public String GSHELL_SCHEDULE = "gshell_schedule"; 
    static public String GSHELL_FEDORA_COLDELETE = "gshell_fedora_col_delete";

    /** Determine if the given process is still executing. It does this by attempting to throw an exception - not the most efficient way, but the only one as far as I know
     * @param process the Process to test
     * @return true if it is still executing, false otherwise
     */
    static public boolean processRunning(Process process) {
    boolean process_running = false;

    try {
        process.exitValue(); // This will throw an exception if the process hasn't ended yet.
    }
    catch(IllegalThreadStateException itse) {
        process_running = true;
    }
    catch(Exception exception) {
        DebugStream.printStackTrace(exception);
    }
    return process_running;
    }

    /** Constructor gatherer all the data required to create a new process, and emit meaningfull messages.
     * @param args A <strong>String[]</strong> containing the arguments to the process thread, including the name of the executable.
     * @param type An <strong>int</strong> that indicates what group of processes this process belongs to, as some are treated slightly differently (i.e an IMPORT type process is always followed by a BUILD one).
     * @param msg_type As process threads may be background (like a makecol.pl call) or important processes in their own right (such as an IMPORT-BUILD) we must be able to set what level messages posted by this class will have by usings this <strong>int</strong>.
     * @param caller The default <i>GShellListener</i> that is interested in the progress of this process.
     * @param progress The <i>GShellProgressMonitor</i> associated with this process.
     * @param name A <strong>String</strong> identifier given to the process, for convience and debug reasons.
     */
    public GShell(String args[], int type, int msg_type, GShellListener caller, GShellProgressMonitor progress, String name) {
    super(name);
    this.args = args;
    this.msg_type = msg_type;
    this.type = type;
    this.caller = caller;
    this.progress = progress;
    this.status = 0;
    // Lower this jobs priority
    this.setPriority(Thread.MIN_PRIORITY);
    listeners = new EventListenerList();
    listeners.add(GShellListener.class, caller);
    }
    /** This method adds another shell listener to this process.
     * @param listener The new <i>GShellListener</i>.
     */
    public void addGShellListener(GShellListener listener) {
    listeners.add(GShellListener.class, listener);
    }
    /** This method removes a certain shell listener from this process.
     * @param listener The <i>GShellListener</i> to be removed.
     */
    /* private void removeGShellListener(GShellListener listener) {
       listeners.remove(GShellListener.class, listener);
       } */

    protected StringBuffer get_stream_char(InputStreamReader isr, StringBuffer line_buffer,
                   BufferedOutputStream bos) throws IOException 
    {
    int c = isr.read();
    ///atherer.println("isr: '" + (char) c + "'");
    if(c == '\n' || c == '\r') {
        if(line_buffer.length() > 0) {
        String line = line_buffer.toString();
        // DebugStream.println("* " + line + " *");     
        fireMessage(type, typeAsString(type) + "> " + line, status, bos);
        line_buffer = new StringBuffer();
        }
    }
    else {
        line_buffer.append((char)c);
    }

    return line_buffer;
    }

    public String getCommandOutput() { return commandOutput; }
    public void resetCommandOutput() { commandOutput = null; }

    private void runRemote(String[] args, BufferedOutputStream bos)
    {
    // Make sure the process hasn't been cancelled
    if (hasSignalledStop()) {
        return;
    }

    try {
        int directory_name_end = args[0].lastIndexOf(File.separator);
        String script_name = ((directory_name_end != -1) ? args[0].substring(directory_name_end + 1) : args[0]);
        System.err.println("Script name: " + script_name);
        String collection_name = args[args.length - 1];
        System.err.println("Collection name: " + collection_name);


        String script_args = "";
        for (int i = 1; i < (args.length - 1); i++) {
        // Skip arguments that won't make sense on the server
        if (args[i].equals("-collectdir") || args[i].equals("-importdir") || args[i].equals("-builddir")) {
            i++;
            continue;
        }

        // Script arguments get changed to CGI arguments
        if (args[i].startsWith("-")) {
            script_args += "&" + args[i].substring(1) + "=";
            if ((i + 1) < (args.length - 1) && !args[i + 1].startsWith("-")) {
            script_args += URLEncoder.encode(args[i + 1], "UTF-8");
            i++;
            }
        }
        }

        System.err.println("Script args: " + script_args);
        buffered_output_stream = bos;
        String command_output = Gatherer.remoteGreenstoneServer.runScript(collection_name, script_name, script_args, this);
        this.commandOutput = command_output;
        status = (command_output.equals("") ? CANCELLED : OK);
    }
    catch (Exception exception) {
        DebugStream.printStackTrace(exception);
        status = ERROR;
    }
    }


    private void runLocal(String[] args, BufferedOutputStream bos) 
    {
    try {
        String command = "";
        for(int i = 0; i < args.length; i++) {
        command = command + args[i] + " ";
        }
        
        ///ystem.err.println("Command: " + command);
        fireMessage(type, Dictionary.get("GShell.Command") + ": " + command, status, null);
        
        Runtime rt = Runtime.getRuntime();
        Process prcs = null;
      
        prcs = rt.exec(args);
        // If we used single argument exec, spaces in filenames or paths cause problems
        // I.e. better to avoid: prcs = rt.exec(command);
        
        InputStreamReader eisr = new InputStreamReader( prcs.getErrorStream(), "UTF-8" );
        InputStreamReader stdisr = new InputStreamReader( prcs.getInputStream(), "UTF-8"  );        
    
        StringBuffer eline_buffer = new StringBuffer();
        StringBuffer stdline_buffer = new StringBuffer();
    
        while(/*type != GShell.NEW &&*/ processRunning(prcs) && !hasSignalledStop()) {
        // Hopefully this doesn't block if the process is trying to write to STDOUT.
        if((eisr!=null) && eisr.ready()) { 
            eline_buffer = get_stream_char(eisr,eline_buffer,bos);
        }
        // Hopefully this won't block if the process is trying to write to STDERR
        else if(stdisr.ready()) {
            stdline_buffer = get_stream_char(stdisr,stdline_buffer,bos);
        }
        else {
            try {
            sleep(100);
            }
            catch(Exception exception) {         
            }
        }
        }
    
        if(!hasSignalledStop()) {
        // Of course, just because the process is finished doesn't
        // mean the incoming streams are empty. Unfortunately I've
        // got no chance of preserving order, so I'll process the
        // error stream first, then the out stream
        while(eisr.ready()) {
            eline_buffer = get_stream_char(eisr,eline_buffer,bos);
        }
        
        while(stdisr.ready()) {
            stdline_buffer = get_stream_char(stdisr,stdline_buffer,bos);
        }
        
        // Ensure that any messages still remaining in the string buffers are fired off.
        if(eline_buffer.length() > 0) {
            String eline = eline_buffer.toString();
            //DebugStream.println("Last bit of eline: " + eline);
            fireMessage(type, typeAsString(type) + "> " + eline, status, bos);
            eline = null;
        }
        
        if(stdline_buffer.length() > 0) {
            String stdline = stdline_buffer.toString();
            //DebugStream.println("Last bit of stdline: " + stdline);
            fireMessage(type, typeAsString(type) + "> " + stdline, status, null);
            stdline = null;
        }
        }
        else {
        DebugStream.println("We've been asked to stop.");
        }
    
    
        if(!hasSignalledStop()) {
        // Now display final message based on exit value
        
        prcs.waitFor(); 

        if(prcs.exitValue() == 0) {
            status = OK;
            fireMessage(type, typeAsString(type) + "> " + Dictionary.get("GShell.Success"), status, null);
        }
        else {
            status = ERROR;
            fireMessage(type, typeAsString(type) + "> " + Dictionary.get("GShell.Failure"), status, null);
        }
        
        eisr.close();
        stdisr.close();
        }
        else {
        // I need to somehow kill the child process. Unfortunately
        // Thread.stop() and Process.destroy() both fail to do
        // this. But now, thankx to the magic of Michaels 'close the
        // stream suggestion', it works fine (no it doesn't!)
        prcs.getInputStream().close();
        prcs.getErrorStream().close();
        prcs.getOutputStream().close();
        prcs.destroy();
        status = CANCELLED;
        }
    }
    // Exception
    catch (Exception exception) {
        DebugStream.println("Exception in GShell.runLocal() - unexpected");
        DebugStream.printStackTrace(exception);
        status = ERROR;
    }
    }
    
    

    /** Any threaded class must include this method to allow the thread body to be run. */
    public void run() {
    String col_name = args[args.length-1];      

    // Determine if the user has asked for an outfile.
    String out_name = null;
    BufferedOutputStream bos = null;
    if(type == IMPORT || type == BUILD || type == SCHEDULE) {
        if(type == IMPORT) {
        out_name = (String) Gatherer.c_man.getCollection().import_options.getValue("out");
        }
        else if(type == BUILD) {
        out_name = (String) Gatherer.c_man.getCollection().build_options.getValue("out");
        }
        else { // SCHEDULE
        out_name = (String) Gatherer.c_man.getCollection().schedule_options.getValue("out"); 
        }
        if(out_name != null && out_name.length() > 0) {
        try {
            bos = new BufferedOutputStream(new FileOutputStream(new File(out_name), true));
        }
        catch (Exception error) {
            DebugStream.printStackTrace(error);
        }
        }
    }

    // Issue a processBegun event
    //ystem.err.println("\nFiring process begun for " + type + "...");
    fireProcessBegun(type, status);
    //ystem.err.println("Done process begun.");
    if (Gatherer.isGsdlRemote) {
        runRemote(args,bos);
    }
    else {
        runLocal(args,bos);
    }
    //ystem.err.println("Done runLocal().");

    if(status == OK) {
        if (type == NEW) {
        if (Gatherer.isGsdlRemote) {
            Gatherer.remoteGreenstoneServer.downloadCollection(col_name);
        }
        }
        else if(type == IMPORT) {
        
        // download the archives directory (if gsdl server is remote)           
        if (Gatherer.isGsdlRemote) {

            if (progress!=null) {
            progress.messageOnProgressBar("Downloading archive data from server");
            }

            Gatherer.remoteGreenstoneServer.downloadCollectionArchives(col_name);

            if (progress!=null) {
            progress.messageOnProgressBar("");
            }      
        }

        // Refresh the DocXMLFileManager
        fireMessage(type, typeAsString(type) + "> " + Dictionary.get("GShell.Parsing_Metadata_Start"), status, null);
        DocXMLFileManager.clearDocXMLFiles();
        if (Configuration.fedora_info.isActive()) { // FLI case
            File collection_export_directory = new File(CollectionManager.getLoadedCollectionExportDirectoryPath());
            DocXMLFileManager.loadDocXMLFiles(collection_export_directory,"docmets.xml");
        }
        else {
            File collection_archives_directory = new File(CollectionManager.getLoadedCollectionArchivesDirectoryPath());
            DocXMLFileManager.loadDocXMLFiles(collection_archives_directory,"doc.xml");
        }


        fireMessage(type, typeAsString(type) + "> " + Dictionary.get("GShell.Parsing_Metadata_Complete"), status, null);
        }

        else if(type == BUILD) {
        // download the building directory (if gsdl server is remote)
        if ((Gatherer.isGsdlRemote) && (!Configuration.fedora_info.isActive())) {
            if (progress!=null) {
            progress.messageOnProgressBar("Downloading index data from server");
            }
                        
            if (!Gatherer.GS3){
            // Only need to download build.cfg file
            File build_cfg_file = new File(CollectionManager.getLoadedCollectionBuildingDirectoryPath(), "build.cfg");
            Gatherer.remoteGreenstoneServer.downloadCollectionFile(col_name, build_cfg_file);
            }else{
            // Only need to download buildConfig.xml file
            File buildConfig_xml_file = new File(CollectionManager.getLoadedCollectionBuildingDirectoryPath(), "buildConfig.xml");
            Gatherer.remoteGreenstoneServer.downloadCollectionFile(col_name, buildConfig_xml_file);
            }

            if (progress!=null) {
            progress.messageOnProgressBar("");
            }
        }
        }
        else if(type == CDIMAGE) {
        // download exported files from tmp folder (if gsdl server is remote)           
        if (Gatherer.isGsdlRemote) {
            if (progress!=null) {
            progress.messageOnProgressBar("Downloading CD-ROM data from server");
            }

            // !! TO DO
            
            if (progress!=null) {
            progress.messageOnProgressBar("");
            }
        }
        }
    }

    // We're done.  
    //ystem.err.println("Firing process complete for " + type + "...");
    fireProcessComplete(type, status);
    // Close bos
    if(bos != null) {
        try {
        bos.close();
        bos = null;
        }
        catch(Exception error) {
        DebugStream.printStackTrace(error);
        }
    }
    }


    public void fireMessage(String message)
    {
    fireMessage(type, typeAsString(type) + "> " + message, status, buffered_output_stream);
    }


    /** Method for firing a message to all interested listeners.
     * @param type An <strong>int</strong> indicating the process type.
     * @param message The message as a <strong>String</strong>.
     * @param status An <strong>int</strong> specifying the current status of the process.
     */
    public void fireMessage(int type, String message, int status, BufferedOutputStream bos) {
    GShellEvent event = new GShellEvent(this, 0, type, message, status);
    // If there is a progress monitor attached, pass the event to it first. Note that we pass a queue of messages as the processing may cause one message to be split into several.
    ArrayList message_queue = new ArrayList();
    message_queue.add(event);
    if(progress != null) {
        progress.process(message_queue);
    }
    for(int j = 0; j < message_queue.size(); j++) {
        GShellEvent current_event = (GShellEvent) message_queue.get(j);
        // If the event hasn't been vetoed, pass it on to other listeners
        if(!current_event.isVetoed()) {
        Object[] concerned = listeners.getListenerList();
        for(int i = 0; i < concerned.length ; i++) {
            if(concerned[i] == GShellListener.class) {
            ((GShellListener)concerned[i+1]).message(current_event);
            }
        }
        concerned = null;
        }
    }
    // And if we have a buffered output stream from error messages, send the message there
    if(bos != null) {
        try {
        bos.write(message.getBytes(), 0, message.length());
        }
        catch(Exception exception) {
        DebugStream.println("Exception in GShell.fireMessage() - unexpected");
        DebugStream.printStackTrace(exception);
        }
    }
    message_queue = null;
    event = null;
    }

    /** Method for firing a process begun event which is called, strangely enough, when the process begins.
     * @param type An <strong>int</strong> indicating the process type.
     * @param status An <strong>int</strong> specifying the current status of the process.
     */
    protected void fireProcessBegun(int type, int status) {
    // Start the progres monitor if available
    if(progress != null) {
        //ystem.err.println("About to call progress.start().");
        progress.start();
        //ystem.err.println("Called progress.start().");
    }
    // Fire an event
    GShellEvent event = new GShellEvent(this, 0, type, "", status);
    Object[] concerned = listeners.getListenerList();
    for(int i = 0; i < concerned.length ; i++) {
        if(concerned[i] == GShellListener.class) {
        ((GShellListener)concerned[i+1]).processBegun(event);
        }
    }
    }


    /** Method for firing a process complete event which is called, no surprise here, when the process ends.
     * @param type An <strong>int</strong> indicating the process type.
     * @param status An <strong>int</strong> specifying the current status of the process.
     */
    protected void fireProcessComplete(int type, int status) {
    // Tidy up by stopping the progress bar. If it was cancelled then the cancel command has arrived via the progress bars and they don't need to be told again (it actually causes problems).
    if(progress != null && status != CANCELLED) {
        progress.stop();
    }
    
    // If we were cancelled, and we are lower details modes, fire off one last message.
    if(status == CANCELLED && Configuration.getMode() <= Configuration.LIBRARIAN_MODE) {
        GShellEvent current_event = new GShellEvent(this, 0, type, Dictionary.get("GShell.Build.BuildCancelled"), status);
        Object[] concerned = listeners.getListenerList();
        for(int i = 0; i < concerned.length ; i++) {
        if(concerned[i] == GShellListener.class) {
            ((GShellListener)concerned[i+1]).message(current_event);
        }
        }
        concerned = null;
    }
    
    String msg = "";
    // If we are creating collection and have trouble with permissions, we need more messages
    if(status == ERROR && type == GShell.NEW){
      msg = args[args.length-1];
    }
    // And firing off an event
    GShellEvent event = new GShellEvent(this, 0, type, msg, status);
    Object[] concerned = listeners.getListenerList();
    for(int i = 0; i < concerned.length ; i++) {
        if(concerned[i] == GShellListener.class) {
        ((GShellListener)concerned[i+1]).processComplete(event);
        }
    }
    }

    /** Method to determine if the user, via the progress monitor, has signalled stop.
     * @return A <strong>boolean</strong> indicating if the user wanted to stop.
     */
    public boolean hasSignalledStop() {
    boolean has_signalled_stop = false;
    if(progress != null) {
        has_signalled_stop = progress.hasSignalledStop();
    }
    if(has_signalled_stop) {
        status = CANCELLED;
    }
    return has_signalled_stop;
    }

    /** Converts a type into a text representation.
     * @param type An <strong>int</strong> which maps to a shell process type.
     * @return A <strong>String</strong> which is the thread process's text name.
     */
    public String typeAsString(int type) {
    String name = null;
    switch(type) {
    case BUILD:
        name = "buildcol.pl";
        break;
    case IMPORT:
        name = "import.pl";
        break;
    case NEW:
        name = "mkcol.pl";
        break;
    case EXPORTAS:
        name = "export.pl";
        break;
    case CDIMAGE:
        name = "exportcol.pl";
        break;
    case CONVERT:
        name = "convert_coll_from_gs2.pl";
        break;
    case EXPLODE:
        name = "explode_metadata_database.pl";
        break;
    case SRCREPLACE: // To work with replace_srcdoc_with_html.pl
        name = "replace_srcdoc_with_html.pl";
        break;
    case SCHEDULE:
        name = "schedule.pl";
        break; 
    default:
        name = "";
    }
    return name;
    }
}