source: main/trunk/gli/src/org/greenstone/gatherer/remote/RemoteGreenstoneServer.java@ 34160

/**
 *#########################################################################
 *
 * A component of the Gatherer application, part of the Greenstone digital
 * library suite from the New Zealand Digital Library Project at the
 * University of Waikato, New Zealand.
 *
 * Author: Michael Dewsnip, NZDL Project, University of Waikato
 *
 * Copyright (C) 2005 New Zealand Digital Library Project
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 *########################################################################
 */

package org.greenstone.gatherer.remote;

import java.io.*;
import java.net.*;
import java.util.*;
import java.util.zip.*;
import javax.swing.*;
import java.io.ByteArrayOutputStream;
import org.greenstone.gatherer.Configuration;
import org.greenstone.gatherer.DebugStream;
import org.greenstone.gatherer.Dictionary;
import org.greenstone.gatherer.FedoraInfo;
import org.greenstone.gatherer.GAuthenticator;
import org.greenstone.gatherer.Gatherer;
import org.greenstone.gatherer.collection.CollectionManager;
import org.greenstone.gatherer.shell.GShell;
import org.greenstone.gatherer.util.UnzipTools;
import org.greenstone.gatherer.util.Utility;
import org.greenstone.gatherer.util.XMLTools;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.methods.multipart.FilePart;
import org.apache.commons.httpclient.methods.multipart.MultipartRequestEntity;
import org.apache.commons.httpclient.methods.multipart.Part;
import org.apache.commons.httpclient.methods.multipart.*;
import org.apache.commons.httpclient.params.*;
import org.apache.commons.httpclient.HttpStatus;


public class RemoteGreenstoneServer
{
    // A PasswordAuthentication object is created whenever it is required
    static private PasswordAuthentication remote_greenstone_server_authentication = null;
    // static private PasswordAuthentication remote_greenstone_server_authentication = new PasswordAuthentication(System.getProperty("user.name"), new char[] { });

    // the language and region environment variables (in "lang_REGION" form)
    // this is necessary in order for the client and server sides to zip and unzip
    // using the same settings
    public final String lang_region;

    private ActionQueue remote_greenstone_server_action_queue;
    private RemoteGreenstoneServer.ProgressBar progress_bar;

    public RemoteGreenstoneServer() {
    // Create the progress_bar first since ActionQueue uses it in its thread 
    // (the thread will start immediately).
    progress_bar = new RemoteGreenstoneServer.ProgressBar();
    remote_greenstone_server_action_queue = new ActionQueue();

    String langReg = System.getenv("LANG");
    lang_region = (langReg == null) ? "" : langReg; 
    }

    // ----------------------------------------------------------------------------------------------------
    //   PUBLIC LAYER
    // ----------------------------------------------------------------------------------------------------
    public String deleteCollection(String collection_name)
    {
    return performAction(new RemoteGreenstoneServerAction.DeleteCollectionAction(collection_name));
    }


    public String deleteCollectionFile(String collection_name, File collection_file)
    {
    return performAction(new RemoteGreenstoneServerAction.DeleteCollectionFileAction(collection_name, collection_file));
    }


    public String downloadCollection(String collection_name)
    {
    return performAction(new RemoteGreenstoneServerAction.DownloadCollectionAction(collection_name));
    }


    public String downloadCollectionArchives(String collection_name)
    {
    return performAction(new RemoteGreenstoneServerAction.DownloadCollectionArchivesAction(collection_name));
    }


    public String downloadCollectionConfigurations()
    {
    return performAction(new RemoteGreenstoneServerAction.DownloadCollectionConfigurationsAction());
    }


    public String downloadCollectionFile(String collection_name, File collection_file)
    {
    return performAction(new RemoteGreenstoneServerAction.DownloadCollectionFileAction(collection_name, collection_file));
    }
    
    // get web.xml from the server -- for a remote gli of GS3
    public String downloadWebXMLFile()
    {
    return performAction(new RemoteGreenstoneServerAction.DownloadWebXMLFileAction());
    }

    public String getScriptOptions(String script_name, String script_arguments)
    {
    return performAction(new RemoteGreenstoneServerAction.GetScriptOptionsAction(script_name, script_arguments));
    }
    
    //get all available site names from the server -- for a remote gli of GS3
    public String getSiteNames()
    {
    return performAction(new RemoteGreenstoneServerAction.GetSiteNamesAction());
    }

    public String moveCollectionFile(String collection_name, File source_collection_file, File target_collection_file)
    {
    return performAction(new RemoteGreenstoneServerAction.MoveCollectionFileAction(
                             collection_name, source_collection_file, target_collection_file));
    }


    public String newCollectionDirectory(String collection_name, File new_collection_directory)
    {
    return performAction(new RemoteGreenstoneServerAction.NewCollectionDirectoryAction(
                             collection_name, new_collection_directory));
    }


    public String runScript(String collection_name, String script_name, String script_arguments, GShell shell)
    {
    return performAction(new RemoteGreenstoneServerAction.RunScriptAction(
                             collection_name, script_name, script_arguments, shell));
    }


    public String uploadCollectionFile(String collection_name, File collection_file)
    {
    return performAction(new RemoteGreenstoneServerAction.UploadCollectionFilesAction(
                             collection_name, new File[] { collection_file }));
    }


    public String uploadCollectionFiles(String collection_name, File[] collection_files)
    {
    return performAction(new RemoteGreenstoneServerAction.UploadCollectionFilesAction(collection_name, collection_files));
    }


    public String uploadFilesIntoCollection(String collection_name, File[] source_files, File target_collection_directory)
    {
    return performAction(new RemoteGreenstoneServerAction.UploadFilesIntoCollectionAction(
                             collection_name, source_files, target_collection_directory));
    }

    public boolean exists(String collection_name, File collection_file)
    {
    String result = performAction(new RemoteGreenstoneServerAction.ExistsAction(collection_name, collection_file));
    if(result.indexOf("exists") != -1) {
        return true;
    }
    else if(result.indexOf("does not exist") != -1) {
        return false;
    }
    return false;
    }

    public int getGreenstoneVersion()
    {
    // returns message "Greenstone version is: <version number of the Greenstone remote server>"
    String result = performAction(new RemoteGreenstoneServerAction.VersionAction());
    int index = result.indexOf(":"); 
    if(index != -1) {
        result = result.substring(index+1).trim(); // skip the space after colon, must remove surrounding spaces
    }
    // if space is returned, then the request failed (the server may not have been running)
    int greenstoneVersion = result.equals("") ? -1 : Integer.parseInt(result);
    return greenstoneVersion;
    }

    /** For constructing the preview command (the library URL) with */
    public String getLibraryURL(String serverHomeURL)
    {
    // returns message "Greenstone library URL suffix is: <e.g. /greenstone3/library or /gsdl/cgi-bin/library>"
    String libSuffix = performAction(new RemoteGreenstoneServerAction.LibraryURLSuffixAction());
    int index = libSuffix.indexOf(":"); 
    if(index != -1) {
        libSuffix = libSuffix.substring(index+1).trim(); // skip the space after colon and remove surrounding spaces
    }
    
    // serverHomeURL is of the form, http://domain/other/stuff. We want the prefix upto & including domain
    // and prepend that to the libraryURLSuffix
    index = -1;
    for(int i = 0; i < 3; i++) {
        index = serverHomeURL.indexOf("/", index+1);
        if(index == -1) { // shouldn't happen, but if it does, we'll be in an infinite loop
        break; 
        }
    }
    serverHomeURL = serverHomeURL.substring(0, index);
    return serverHomeURL + libSuffix;
    }

    // ----------------------------------------------------------------------------------------------------


    public void exit()
    {
    System.err.println("Exiting, number of jobs on queue: " + remote_greenstone_server_action_queue.size());

    // If there are still jobs on the queue we must wait for the jobs to finish
    while (remote_greenstone_server_action_queue.size() > 0) {
        synchronized (remote_greenstone_server_action_queue) {
        try {
            DebugStream.println("Waiting for queue to become empty...");
            remote_greenstone_server_action_queue.wait(500);
        }
        catch (InterruptedException exception) {}
        }
    }
    }
    

    // ----------------------------------------------------------------------------------------------------
    //   QUEUE LAYER
    // ----------------------------------------------------------------------------------------------------


    /** Returns null if we cannot wait for the action to finish, "" if the action failed, or the action output. */
    private String performAction(RemoteGreenstoneServerAction remote_greenstone_server_action)
    {
    // Check for whether the queue thread stopped running because 
    // the user cancelled out. If so, exit GLI.
    if(remote_greenstone_server_action_queue.hasExited()) {
        remote_greenstone_server_action_queue.clear();
        //remote_greenstone_server_action_queue = null;
        Gatherer.exit();
        return "";
    }

    // Add the action to the queue
    remote_greenstone_server_action_queue.addAction(remote_greenstone_server_action);
    String threadName = Thread.currentThread().getName();

    /*
    // If we're running in the GUI thread we must return immediately
    // We cannot wait for the action to complete because this will block any GUI updates
    if (SwingUtilities.isEventDispatchThread()) {
        System.err.println(threadName 
                   + "\tACTION QUEUED: In event dispatch thread, returning immediately...\n\t" 
                   + remote_greenstone_server_action);
        return null;
    }
    */

    // Otherwise wait until the action is processed
    try {
        synchronized (remote_greenstone_server_action) {
        while(!remote_greenstone_server_action.processed) {
            //System.err.println("Waiting for action to complete...: " + remote_greenstone_server_action);
            DebugStream.println("Waiting for action to complete...");
            remote_greenstone_server_action.wait(); // wait to be notified when the action has been processed 
        }
        }
    } catch (Exception e) {
        System.err.println("RemoteGreenstoneServer.performAction() - exception: " + e);
        e.printStackTrace();
    }

    // Return "" if the action failed
    if (!remote_greenstone_server_action.processed_successfully) {
        return "";
    }
    // Otherwise return the action output
    return remote_greenstone_server_action.action_output;
    }


    // ----------------------------------------------------------------------------------------------------
    //   PROGRESS BAR
    // ----------------------------------------------------------------------------------------------------

    static class ProgressBar
    extends JProgressBar
    {
    public ProgressBar()
    {
        setBackground(Configuration.getColor("coloring.collection_tree_background", false));
        setForeground(Configuration.getColor("coloring.collection_tree_foreground", false));
        setString(Dictionary.get("FileActions.No_Activity"));
        setStringPainted(true);
    }

    /** synchronized to avoid conflicts since several threads access this */
    synchronized public void setAction(String action)
    {
        if (action != null) {
        DebugStream.println(action);
        }

        // We cannot call this from the GUI thread otherwise the progress bar won't start
        if (SwingUtilities.isEventDispatchThread()) {
        System.err.println("ERROR: RemoteGreenstoneServerProgressBar.setAction() called from event dispatch thread!");
        return;
        }

        // Set the string on the progress bar, and start or stop it
        if (action == null) {
        setString(Dictionary.get("FileActions.No_Activity"));
        setIndeterminate(false);
        }
        else {
        setString(action);
        setIndeterminate(true);
        }
    }
    }
    
    /** synchronized to avoid conflicts since several threads access this */
    synchronized public RemoteGreenstoneServer.ProgressBar getProgressBar()
    {
    return progress_bar;
    }


    // ----------------------------------------------------------------------------------------------------
    //   AUTHENTICATION LAYER
    // ----------------------------------------------------------------------------------------------------

    static private class RemoteGreenstoneServerAuthenticateTask
    extends Thread
    {
    public void run()
    {
        remote_greenstone_server_authentication = new RemoteGreenstoneServerAuthenticator().getAuthentication();
    }
    

    static private class RemoteGreenstoneServerAuthenticator
        extends GAuthenticator
    {
        public PasswordAuthentication getAuthentication(String username, String password)
        {
        return getPasswordAuthentication(username,password);
        }

        public PasswordAuthentication getAuthentication()
        {
        return getPasswordAuthentication();
        }

        protected String getMessageString()
        {
        if (Gatherer.GS3){
            return (Dictionary.get("RemoteGreenstoneServer.Authentication_Message_gs3") + " " + Configuration.site_name);
        }
        return Dictionary.get("RemoteGreenstoneServer.Authentication_Message");
        }
    }
    }
    
    public void set_remote_greenstone_server_authentication_to_null() {
    remote_greenstone_server_authentication = null;
    }

    private void authenticateUser()
    throws RemoteGreenstoneServerAction.ActionCancelledException
    {
    // If we don't have any authentication information then ask for it now
    if (remote_greenstone_server_authentication == null) {
        try {
        // We have to do this on the GUI thread
        SwingUtilities.invokeAndWait(new RemoteGreenstoneServerAuthenticateTask());
        }
        catch (Exception exception) {
        System.err.println("Exception occurred when authenticating the user: " + exception);
        DebugStream.printStackTrace(exception);
        }

        // If it is still null then the user has cancelled the authentication, so the action is cancelled
        if (remote_greenstone_server_authentication == null) {
        throw new RemoteGreenstoneServerAction.ActionCancelledException();
        }
    }
    }


    public String getUsername()
    {
    if (remote_greenstone_server_authentication != null) {
        return remote_greenstone_server_authentication.getUserName();
    }

    return null;
    }


    // ----------------------------------------------------------------------------------------------------
    //   REQUEST LAYER
    // ----------------------------------------------------------------------------------------------------


    /** Returns the command output if the action completed, throws some kind of exception otherwise. */
    String downloadFile(String gliserver_args, String file_path)
    throws Exception
    {
    while (true) {
        // Check that Configuration.gliserver_url is set
        if (Configuration.gliserver_url == null) {
        throw new Exception("Empty gliserver URL: please set this in Preferences before continuing.");
        }

        // Ask for authentication information (if necessary), then perform the action
        authenticateUser();
        String gliserver_url_string = Configuration.gliserver_url.toString();
        String command_output = downloadFileInternal(gliserver_url_string, gliserver_args, file_path);

        // Debugging - print any ok messages to stderr
        //System.err.println("**** RECEIVED (sendCommandToServer()): " + command_output);

        // Check the first line to see if authentication has failed; if so, go around the loop again
        if (command_output.startsWith("ERROR: Authentication failed:")) {
        JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("RemoteGreenstoneServer.Error", command_output.substring("ERROR: ".length())), Dictionary.get("RemoteGreenstoneServer.Error_Title"), JOptionPane.ERROR_MESSAGE);
        remote_greenstone_server_authentication = null;
        continue;
        }
        // Check if the collection is locked by someone else; if so, see if the user wants to steal the lock
        else if (command_output.startsWith("ERROR: Collection is locked by: ")) {
        if (JOptionPane.showConfirmDialog(Gatherer.g_man, Dictionary.get("RemoteGreenstoneServer.Steal_Lock_Message", command_output.substring("ERROR: Collection is locked by: ".length())), Dictionary.get("RemoteGreenstoneServer.Error_Title"), JOptionPane.YES_NO_OPTION) == JOptionPane.NO_OPTION) {
            // The user has decided to cancel the action
            throw new RemoteGreenstoneServerAction.ActionCancelledException();
        }

        // The user has decided to steal the lock... rerun the command with "&steal_lock="
        gliserver_args += "&steal_lock=";
        continue;
        }
        // Handle other types of errors by throwing an exception
        else if (command_output.startsWith("ERROR: ")) {
        throw new Exception(command_output.substring("ERROR: ".length()));
        }

        // There were no exceptions thrown so the action must have succeeded
        return command_output;
    }
    }

    /** Returns true or false depending on whether authentication is required for the cmd 
     * string embedded in the given gliserver_args. No authentication is required for either 
     * of the commands greenstone-server-version and get-library-url-suffix. */
    private boolean isAuthenticationRequired(String gliserver_args) {
    return ((gliserver_args.indexOf("greenstone-server-version") == -1) 
        && (gliserver_args.indexOf("get-library-url-suffix") == -1));
    }


    /** Returns the command output if the action completed, throws some kind of exception otherwise. 
     *  Package access, so that RemoteGreenstoneServerAction.java can call this.
     */
    String sendCommandToServer(String gliserver_args, GShell shell)
    throws Exception
    {
    while (true) {
        
        // Check that Configuration.gliserver_url is set
        if (Configuration.gliserver_url == null) {
        throw new Exception("Empty gliserver URL: please set this in Preferences before continuing.");
        }

        // Ask for authentication information (if necessary), then perform the action
        if(isAuthenticationRequired(gliserver_args)) {
        try {
            authenticateUser();
        } catch (RemoteGreenstoneServerAction.ActionCancelledException e) {
            // Authentication popup only appears at the start. If the user cancelled
            // out of it, then another remote action always remains on the queue,
            // preventing a clean exit. Need to clear queue before exit.
            synchronized (remote_greenstone_server_action_queue) {          
            remote_greenstone_server_action_queue.clear();          
            }
            Gatherer.exit();
        }
        }
        String gliserver_url_string = Configuration.gliserver_url.toString();
        String command_output = sendCommandToServerInternal(gliserver_url_string, gliserver_args, shell);
                
        // Debugging - print any ok messages to stderr
        //if(!(command_output.trim().startsWith("<"))) {
        //System.err.println("**** RECEIVED (sendCommandToServer()): " + command_output);
        //}
        
        // Check the first line to see if authentication has failed; if so, go around the loop again
        if (command_output.startsWith("ERROR: Authentication failed:")) {
        JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("RemoteGreenstoneServer.Error", command_output.substring("ERROR: ".length())), Dictionary.get("RemoteGreenstoneServer.Error_Title"), JOptionPane.ERROR_MESSAGE);
        remote_greenstone_server_authentication = null;
        continue;
        }
        // Check if the collection is locked by someone else; if so, see if the user wants to steal the lock
        else if (command_output.startsWith("ERROR: Collection is locked by: ")) {
        if (JOptionPane.showConfirmDialog(Gatherer.g_man, Dictionary.get("RemoteGreenstoneServer.Steal_Lock_Message", command_output.substring("ERROR: Collection is locked by: ".length())), Dictionary.get("RemoteGreenstoneServer.Error_Title"), JOptionPane.YES_NO_OPTION) == JOptionPane.NO_OPTION) {
            // The user has decided to cancel the action
            throw new RemoteGreenstoneServerAction.ActionCancelledException();
        }

        // The user has decided to steal the lock... rerun the command with "&steal_lock="
        gliserver_args += "&steal_lock=";
        continue;
        }
        // Handle other types of errors by throwing an exception
        else if (command_output.startsWith("ERROR: ")) { 
        throw new Exception(command_output.substring("ERROR: ".length()));
        } else if (command_output.indexOf("ERROR: ") != -1) { // check if ERROR occurs anywhere else in the output
        throw new Exception(command_output);
        }

        
        // There were no exceptions thrown so the action must have succeeded
        return command_output;
    }
    }


    /** Returns the command output if the action completed, throws some kind of exception otherwise. */
    String uploadFile(String gliserver_args, String file_path)
    throws Exception
    {
    while (true) {
        // Check that Configuration.gliserver_url is set
        if (Configuration.gliserver_url == null) {
        throw new Exception("Empty gliserver URL: please set this in Preferences before continuing.");
        }

        // Ask for authentication information (if necessary), then perform the action
        authenticateUser();
        String gliserver_url_string = Configuration.gliserver_url.toString();
        String command_output = uploadFileInternal(gliserver_url_string, gliserver_args, file_path);
        // System.err.println("Command output: " + command_output);

        // Check the first line to see if authentication has failed; if so, go around the loop again
        if (command_output.startsWith("ERROR: Authentication failed:")) {
        JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("RemoteGreenstoneServer.Error", command_output.substring("ERROR: ".length())), Dictionary.get("RemoteGreenstoneServer.Error_Title"), JOptionPane.ERROR_MESSAGE);
        remote_greenstone_server_authentication = null;
        continue;
        }
        // Check if the collection is locked by someone else; if so, see if the user wants to steal the lock
        else if (command_output.startsWith("ERROR: Collection is locked by: ")) {
        if (JOptionPane.showConfirmDialog(Gatherer.g_man, Dictionary.get("RemoteGreenstoneServer.Steal_Lock_Message", command_output.substring("ERROR: Collection is locked by: ".length())), Dictionary.get("RemoteGreenstoneServer.Error_Title"), JOptionPane.YES_NO_OPTION) == JOptionPane.NO_OPTION) {
            // The user has decided to cancel the action
            throw new RemoteGreenstoneServerAction.ActionCancelledException();
        }

        // The user has decided to steal the lock... rerun the command with "&steal_lock="
        gliserver_args += "&steal_lock=";
        continue;
        }
        // Handle other types of errors by throwing an exception
        else if (command_output.startsWith("ERROR: ")) {
        throw new Exception(command_output.substring("ERROR: ".length()));
        }

        // There were no exceptions thrown so the action must have succeeded
        return command_output;
    }
    }


    // ----------------------------------------------------------------------------------------------------
    //   NETWORK LAYER
    // ----------------------------------------------------------------------------------------------------


    /** Returns the command output if the action completed, throws some kind of exception otherwise. */
    private String downloadFileInternal(String download_cgi, String cgi_args, String file_path)
    throws Exception
    {
    DebugStream.println("gliserver URL: " + download_cgi);
    System.err.println("gliserver args: " + cgi_args);

    // Add username and password, and a timestamp
    cgi_args += "&un=" + remote_greenstone_server_authentication.getUserName();
    cgi_args += "&pw=" + new String(remote_greenstone_server_authentication.getPassword());
    cgi_args += "&ts=" + System.currentTimeMillis();
    if (Gatherer.GS3){
        cgi_args += "&site=" + Configuration.site_name;
        String library_servlet_name = Configuration.servlet_path;
        if (library_servlet_name.charAt(0) == '/') {
        library_servlet_name = library_servlet_name.substring(1);
        }
        cgi_args += "&servlet=" + library_servlet_name;
    }

    URL download_url = new URL(download_cgi);
    URLConnection dl_connection = download_url.openConnection();
    dl_connection.setDoOutput(true);
    OutputStream dl_os = dl_connection.getOutputStream();

    PrintWriter dl_out = new PrintWriter(dl_os);
    dl_out.println(cgi_args);
    dl_out.close();

    // Download result from running cgi script
    InputStream dl_is = dl_connection.getInputStream();
    BufferedInputStream dl_bis = new BufferedInputStream(dl_is);
    DataInputStream dl_dbis = new DataInputStream(dl_bis);

    String first_line = "";
    byte[] buf = new byte[1024];
    int len = dl_dbis.read(buf);
    if (len >= 0) {
        String first_chunk = new String(buf, 0, len);
        // first_line = first_chunk.substring(0, ((first_chunk.indexOf("\n") != -1) ? first_chunk.indexOf("\n") : len));
        first_line = first_chunk.substring(0, ((first_chunk.indexOf("\n") != -1) ? first_chunk.indexOf("\n") : ((first_chunk.length()<len) ? first_chunk.length():len)));
        // Save the data to file
        FileOutputStream zip_fos = new FileOutputStream(file_path);
        BufferedOutputStream zip_bfos = new BufferedOutputStream(zip_fos);

        while (len >= 0) {
        zip_bfos.write(buf, 0, len);
        len = dl_dbis.read(buf);
        }

        zip_bfos.close();
        zip_fos.close();
    }

    dl_dbis.close();
    dl_bis.close();
    dl_is.close();
    return first_line;
    }


    /** Returns the command output if the action completed, throws some kind of exception otherwise. */
    private String sendCommandToServerInternal(String gliserver_url_string, String cgi_args, GShell shell)
    throws Exception
    {
    DebugStream.println("gliserver URL: " + gliserver_url_string);
    System.err.println("gliserver args: " + cgi_args);

    // Add username and password, and a timestamp
    if(isAuthenticationRequired(cgi_args)) {
        cgi_args += "&un=" + remote_greenstone_server_authentication.getUserName();
        cgi_args += "&pw=" + new String(remote_greenstone_server_authentication.getPassword());
    }
    cgi_args += "&ts=" + System.currentTimeMillis();
    if (Gatherer.GS3){
        cgi_args += "&site=" + Configuration.site_name;
    }

    URL gliserver_url = new URL(gliserver_url_string + "?" + cgi_args);
    URLConnection gliserver_connection = gliserver_url.openConnection();

    // Read the output of the command from the server, and return it
    StringBuffer command_output_buffer = new StringBuffer(2048);
    InputStream gliserver_is = gliserver_connection.getInputStream();
    BufferedReader gliserver_in = new BufferedReader(new InputStreamReader(gliserver_is, "UTF-8"));
    String gliserver_output_line = gliserver_in.readLine();
    while (gliserver_output_line != null) {
        if (shell != null) {
        shell.fireMessage(gliserver_output_line);
        if (shell.hasSignalledStop()) {
            throw new RemoteGreenstoneServerAction.ActionCancelledException();
        }
        }
        command_output_buffer.append(gliserver_output_line + "\n");
        gliserver_output_line = gliserver_in.readLine();
    }
    gliserver_in.close();

    
    int startIndex = command_output_buffer.indexOf("<?xml");
    if(startIndex > 0) { // not -1, so "<?xml" is present, and not 0, so not at start

        // iff dealing with XML, make sure to parse out any lines preceding the XML content
        command_output_buffer = XMLTools.readXMLStream(command_output_buffer.toString());
    }
    
    return command_output_buffer.toString();
    }

           
    /** Returns the command output if the action completed, throws some kind of exception otherwise. */
    private String uploadFileInternal(String upload_cgi, String cgi_args, String file_path)
    throws Exception
    {
    System.err.println("gliserver URL: " + upload_cgi);
    System.err.println("gliserver args: " + cgi_args);

    //For a remote GS3
    //GS3 is running on Tomcat, and Tomcat requires a connection timeout to be set up at the client
    //side while uploading files. As HttpURLConnection couldn't set the connection timeout, HttpClient.jar 
    //from Jakarta is applied to solve this problem only for uploading files. 
    if (Gatherer.GS3) {

        // Setup the POST method
        PostMethod httppost = new PostMethod(upload_cgi+"?"+cgi_args); // cgi_args: QUERY_STRING on perl server side
                    
        // construct the multipartrequest form
        String[] cgi_array=cgi_args.split("&");// get parameter-value pairs from cgi_args string
        Part[] parts=new Part[cgi_array.length+5];
        
        // The FilePart: consisting of the (cgi-arg) name and (optional) filename in the Content-Disposition 
        // of the POST request Header (see CGI.pm), and the actual zip file itself. It uses the defaults:
        // Content-Type: application/octet-stream; charset=ISO-8859-1,Content-Transfer-Encoding: binary
        parts[0]= new FilePart("uploaded_file", "zipFile", new File(file_path));

        parts[1]= new StringPart("un", remote_greenstone_server_authentication.getUserName());
        parts[2]= new StringPart("pw", new String(remote_greenstone_server_authentication.getPassword()));
        parts[3]= new StringPart("ts", String.valueOf(System.currentTimeMillis()));
        parts[4]= new StringPart("site", Configuration.site_name);
        // find all parameters of cgi-args and add them into Part[]
        for (int i=0; i<cgi_array.length;i++){
        parts[5+i]=new StringPart(cgi_array[i].substring(0,cgi_array[i].indexOf("=")),cgi_array[i].substring(cgi_array[i].indexOf("=")+1,cgi_array[i].length()));
        }

        // set MultipartRequestEntity on the POST method
        httppost.setRequestEntity(new MultipartRequestEntity(parts, httppost.getParams()));

        // See file gli/request.txt for the multipart request that's been generated:
        //httppost.getRequestEntity().writeRequest(new FileOutputStream("request.txt", true)); // true: appends
        
        //set up the HttpClient connection
        HttpClient client=new HttpClient(); 
        client.getParams().setBooleanParameter(HttpMethodParams.USE_EXPECT_CONTINUE, false);
        client.getParams().setConnectionManagerTimeout(200000); //set the connection timeout
        // get the output of the command from the server
        String command_output = ""; 
        try{
        client.executeMethod(httppost);
        if (httppost.getStatusCode() == HttpStatus.SC_OK) {
            command_output = httppost.getStatusLine().toString();
        } else {
            command_output = httppost.getStatusLine().toString();
            System.out.println("Unexpected failure: " + httppost.getStatusLine().toString());
        }
        }catch(IOException e){
        e.printStackTrace();
        }finally{
        httppost.releaseConnection();
        }
        return command_output;
    }

    //For a remote GS2
    // Add username and password, and a timestamp
    cgi_args += "&un=" + remote_greenstone_server_authentication.getUserName();
    cgi_args += "&pw=" + new String(remote_greenstone_server_authentication.getPassword());
    cgi_args += "&ts=" + System.currentTimeMillis();

    // Open a HTTP connection to the URL
    URL url = new URL(upload_cgi);
    HttpURLConnection gliserver_connection = (HttpURLConnection) url.openConnection();

    gliserver_connection.setDoInput(true);         // Allow Inputs
    gliserver_connection.setDoOutput(true);        // Allow Outputs
    gliserver_connection.setUseCaches(false);      // Don't use a cached copy.

    gliserver_connection.setRequestProperty("Connection", "Keep-Alive");        

    // Send zip file to server
    File file = new File(file_path);
    FileInputStream fileInputStream = new FileInputStream(file);

    // Add file size argument, because IIS 6 needs a lot of help
    int file_size = fileInputStream.available();
    cgi_args += "&fs=" + file_size;

    DataOutputStream dos = new DataOutputStream(gliserver_connection.getOutputStream());
    dos.writeBytes(cgi_args + "\n");

    // create a buffer of maximum size
    final int maxBufferSize = 1024;
    int bytesAvailable = file_size;
    int bufferSize = Math.min(bytesAvailable, maxBufferSize);
    byte[] buffer = new byte[bufferSize];

    // read file and write it into form...
    // !! This uses a lot of memory when the file being uploaded is big -- Java seems to need to keep
    //   the entire file in the DataOutputStream? (Use Runtime.getRuntime().totalMemory() to see)
    int bytesRead = fileInputStream.read(buffer, 0, bufferSize);
    while (bytesRead > 0) {
        dos.write(buffer, 0, bytesRead);
        bytesAvailable = fileInputStream.available();
        bufferSize = Math.min(bytesAvailable, maxBufferSize);
        bytesRead = fileInputStream.read(buffer, 0, bufferSize);
    }

    // close streams
    fileInputStream.close();
    dos.flush();
    dos.close();

    // Read the output of the command from the server, and return it
    String command_output = "";
    InputStream gliserver_is = gliserver_connection.getInputStream();
    BufferedReader gliserver_in = new BufferedReader(new InputStreamReader(gliserver_is, "UTF-8"));
    String gliserver_output_line = gliserver_in.readLine();
    while (gliserver_output_line != null) {
        command_output += gliserver_output_line + "\n";
        gliserver_output_line = gliserver_in.readLine();
    }
    gliserver_in.close();

    return command_output;
    }


    // ----------------------------------------------------------------------------------------------------
    //   UTILITIES
    // ----------------------------------------------------------------------------------------------------


    public String getPathRelativeToDirectory(File file, String directory_path)
    {
    String file_path = file.getAbsolutePath();
    if (!file_path.startsWith(directory_path)) {
        System.err.println("ERROR: File path " + file_path + " is not a child of " + directory_path);
        return file_path;
    }

    String relative_file_path = file_path.substring(directory_path.length());
    if (relative_file_path.startsWith(File.separator)) {
        relative_file_path = relative_file_path.substring(File.separator.length());
    }
    return relative_file_path;
    }
}