source: gli/branches/rtl-gli/src/org/greenstone/gatherer/shell/GImportProgressMonitor.java@ 18360

/**
 *#########################################################################
 *
 * A component of the Gatherer application, part of the Greenstone digital
 * library suite from the New Zealand Digital Library Project at the
 * University of Waikato, New Zealand.
 *
 * <BR><BR>
 *
 * Author: John Thompson, Greenstone Digital Library, University of Waikato
 *
 * <BR><BR>
 *
 * Copyright (C) 1999 New Zealand Digital Library Project
 *
 * <BR><BR>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * <BR><BR>
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * <BR><BR>
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 *########################################################################
 */
package org.greenstone.gatherer.shell;

import java.awt.Component;
import java.util.*;
import org.greenstone.gatherer.Configuration;
import org.greenstone.gatherer.DebugStream;
import org.greenstone.gatherer.Dictionary;
import org.greenstone.gatherer.Gatherer;
import org.greenstone.gatherer.gui.GProgressBar;
import org.greenstone.gatherer.util.StaticStrings;

/** This implementation of <i>GShellProgressMonitor</i> is designed to parse and translate the progress of a import.pl call.
 * @author John Thompson, Greenstone Digital Library, University of Waikato
 * @version 2.1
 */
public class GImportProgressMonitor
    implements GShellProgressMonitor {
    /** Indicates if the progress bar is currently showing a string. */
    private boolean showing_string = false;
    /** Indicates if the GUI has asked the process this object monitors to stop. */
    private boolean stop = false;
    /** A count of the extracted files processed so far. */
    private int extracted_file_count;
    /** The number of documents processed (or rejected) so far. */
    private int file_count;
    /** The next value to be set for the progress bar - I use this rather than a compounding progress measure to try to limit rounding inaccuracies (nothing looks worse than the progress bar having to jump the last 10-15%) */
    private int next_progress_value;
    /** The number of files expected to be scanned by this import process. */
    private int num_expected_docs;
    /** This holds the number of documents actually processed by the import command, as garnered from the final block of text output. */
    private int num_docs;

    private int threshold = Configuration.SYSTEMS_MODE;
    /** The progress bar this monitor updates. */
    private GProgressBar progress_bar;

    /** The progress bar that is shared with build monitor. */
    private GProgressBar shared_progress_bar;

    /** */
    static final private String BLOCKED_ATTRIBUTE = "blocked";
    static final private String CONSIDERED_ATTRIBUTE = "considered";
    static final private String FAILED_ATTRIBUTE = "failed";
    static final private String IGNORED_ATTRIBUTE = "ignored";
    static final private String IMPORT_ELEMENT = "Import";
    static final private String PROCESSED_ATTRIBUTE = "processed";  

    static final private String ARGUMENT_ATTRIBUTE = "a";
    /** */
    static final private String NAME_ATTRIBUTE = "n";
    /** */
    static final private String PLUGIN_ATTRIBUTE = "p";
    static final private String REASON_ATTRIBUTE = "r";
    /** The fixed portion of the progress bar used for the calculating of file size and other pre-import functions. */
    static final private int CALCULATION = 50000;
    /** The fixed portion of the progress bar used for extracted metadata. */
    static final private int EXTRACTED = 200000;
    /** The fixed portion of the progress bar used for processed documents. */
    static final private int PROCESSED = 750000;
    /** The maximum value for this progress bar. */
    static final private int MAXIMUM = CALCULATION + EXTRACTED + PROCESSED;
    /** The minimum value for this progress bar. */
    static final private int MINIMUM = 0;
    /** The element name of a file detected message. */
    static final private String FILE_ELEMENT = "File";
    /** The element name of a file processing message. */
    static final private String PROCESSING_ELEMENT = "Processing";
    /** The element name of a processing error message. */
    static final private String PROCESSINGERROR_ELEMENT = "ProcessingError";
    /** The element name of an import complete message. */
    static final private String IMPORTCOMPLETE_ELEMENT = "ImportComplete";
    /** The element name of a file not processed. */
    static final private String NONPROCESSEDFILE_ELEMENT = "NonProcessedFile";
    /** The element name of a file not recognised. */
    static final private String NONRECOGNISEDFILE_ELEMENT = "NonRecognisedFile";
    /** The element name of a Warning message */
    static final private String WARNING_ELEMENT = "Warning";
    /** The element name of a BadArgument message */
    static final private String BADARGUMENT_ELEMENT = "BadArgument";
    static final private String BADARGUMENTVALUE_ELEMENT = "BadArgumentValue";
    static final private String BADPLUGIN_ELEMENT = "BadPlugin";

    public GImportProgressMonitor() {
    progress_bar = new GProgressBar();
    progress_bar.setIndeterminate(false);
    progress_bar.setMaximum(MAXIMUM);
    progress_bar.setMinimum(MINIMUM);
    progress_bar.setString(null);
    progress_bar.setStringPainted(true);
    next_progress_value = CALCULATION;
    shared_progress_bar = new GProgressBar();
    shared_progress_bar.setIndeterminate(false);
    shared_progress_bar.setMaximum(MAXIMUM * 2);
    shared_progress_bar.setMinimum(MINIMUM);
    shared_progress_bar.setString(null); // Default string
    shared_progress_bar.setStringPainted(true);
    setValue(MINIMUM);
    }

    /** Method to register a new progress bar with this monitor.
     * @param progress_bar The new <strong>GProgressBar</strong>.
     */
    public void addProgressBar(GProgressBar progress_bar) {
    this.progress_bar = progress_bar;
    progress_bar.setMaximum(MAXIMUM);
    progress_bar.setMinimum(MINIMUM);
    setValue(MINIMUM);
    next_progress_value = CALCULATION;
    }

    /** Determine the script exit value according to the progress monitor. This gets around a problem where several script failures actually result with a successful exit value.
     * @return A <i>int</i> with a value of zero if and only if the script was successful.
     */
    public int exitValue() {
    if(num_docs > 0) {
        return 0;
    }
    return 1;
    }

    /** Retrieve the number of documents recorded by the import monitor during the execution of the import scripts.
     * @return An <i>int</i> giving the document number.
     */
    public int getNumberOfDocuments() {
    return num_docs;
    }

    /** Method to retrieve whatever control is being used as the progress indicator. Usually a <strong>GProgressBar</strong> but there may be others implemented later.
     * @return A <strong>Component</strong> on which the progress of the process is being displayed.
     */
    public Component getProgress() {
    return progress_bar;
    }

    public GProgressBar getSharedProgress() {
    return shared_progress_bar;
    }

    public void messageOnProgressBar(String message)
    {
    if (message!=null && !message.equals("")) {
        progress_bar.setString(message);
        shared_progress_bar.setString(message);
        showing_string = true;
    }
    else {
        progress_bar.setString(null);
        shared_progress_bar.setString(null);
        showing_string = false;
    }
    }

    /** Method to determine the state of the stop flag, which may be set by the visual component that created this monitor.
     * @return A <strong>boolean</strong> indicating if the process has been asked to stop.
     */
    public synchronized boolean hasSignalledStop() {
    return stop;
    }

    /** Inform the progress bar that it should programatically increment progress by one step. This is only called during the metadata archive extraction so each step should be (1000000 / 5) / num_docs. */
    public void increment() {
    extracted_file_count++;
    // The current progress is calculated to be:
    // The fixed calculation value plus the fixed processed value plus some portion of the fixed extracted value. This portion is the extracted_file_count over the total number of documents available. Note that this breaks badly for bibliographical files (for now).
    setValue(CALCULATION + PROCESSED + ((EXTRACTED * extracted_file_count) / num_docs));
    }

    /** This method is used to 'feed in' a line of text captured from the process.
     * @param queue a queue which at the moment should contain a single GShellEvent
     */
    public void process(ArrayList queue) {
    // Retrieve the first event.
    GShellEvent event = (GShellEvent) queue.get(0);
    // Remove 'import.pl> ' bit
    String line = event.getMessage();
    line = line.substring(line.indexOf(StaticStrings.GREATER_THAN_CHARACTER) + 1);
    line = line.trim();
    // System.err.println("**** line = " + line);

    ///ystem.err.println("Process: " + line);
    if(line.startsWith(StaticStrings.LESS_THAN_CHARACTER) && line.endsWith(StaticStrings.GREATER_THAN_CHARACTER)) {
        // No original event is passed on, even in the lower modes
        event.veto();
        // Create a new element from it
        GShellElement element = new GShellElement(line);
        String name = element.getElementName();
        ///ystem.err.println("Gatherer tag: " + name);
        // We may be reading a file. Remember we have to offset process as we recieve this message 'before' a document is processed. Hence the use of 'next_progress_value'
        if(name.equals(IMPORT_ELEMENT)) {
        ///ystem.err.println("#Import");
        // We're into parsing output, so we don't need the 'calculating file size' etc string.
        if(showing_string) {
            progress_bar.setString(null);
            showing_string = false;
        }
        if(Configuration.getMode() <= threshold) {
            queue.add(new GShellEvent(event.getSource(), 0, event.getType(), Dictionary.get("GShell.Import.ImportBegun1"), event.getStatus()));
        }
        }
        else if(name.equals(FILE_ELEMENT)) {
        ///ystem.err.println("#File");
        file_count++;
        // Set the next progress
        if(next_progress_value > 0) {
            setValue(next_progress_value);
        }
        // Now we calculate the next progress value
        next_progress_value = CALCULATION + ((PROCESSED * file_count) / num_expected_docs);
        event.veto(); // Unconditional veto
        }
        // Or we're being told what plugin is actually processing the file
        else if(name.equals(PROCESSING_ELEMENT)) {
        ///ystem.err.println("#FileProcessing");
        // If we are at lower mode settings fire a new 'dumbed down' event
        if(Configuration.getMode() <= threshold) {
            String args[] = new String[2];
            args[0] = element.getAttribute(NAME_ATTRIBUTE);
            args[1] = element.getAttribute(PLUGIN_ATTRIBUTE);
            queue.add(new GShellEvent(event.getSource(), 0, event.getType(), Dictionary.get("GShell.Import.FileProcessing", args), event.getStatus()));
            args = null;
        }
        }
        // processing error
        else if (name.equals(PROCESSINGERROR_ELEMENT)) {
        if(Configuration.getMode() <= threshold) {
            String args[] = new String[1];
            args[0] = element.getAttribute(NAME_ATTRIBUTE);
            String reason = element.getAttribute(REASON_ATTRIBUTE);
            if (reason == null || reason.equals("")) {
            queue.add(new GShellEvent(event.getSource(), 0, event.getType(), Dictionary.get("GShell.Import.FileProcessingError", args), event.getStatus()));
            }
            else {
            queue.add(new GShellEvent(event.getSource(), 0, event.getType(), Dictionary.get("GShell.Import.FileProcessingError", args) + " (" + reason + ")", event.getStatus()));
            }
            args = null;
        }
        }
        // unrecognised file
        else if (name.equals(NONRECOGNISEDFILE_ELEMENT)) {
        if(Configuration.getMode() <= threshold) {
            String args[] = new String[1];
            args[0] =element.getAttribute(NAME_ATTRIBUTE);
            queue.add(new GShellEvent(event.getSource(), 0, event.getType(), Dictionary.get("GShell.Import.FileNotRecognised", args), event.getStatus()));
            args = null;
        }
        }
        // unprocessed file
        else if (name.equals(NONPROCESSEDFILE_ELEMENT)) {
        if(Configuration.getMode() <= threshold) {
            String args[] = new String[1];
            args[0] =element.getAttribute(NAME_ATTRIBUTE);          
            queue.add(new GShellEvent(event.getSource(), 0, event.getType(), Dictionary.get("GShell.Import.FileNotProcessed", args), event.getStatus()));
            args = null;
        }
        }
        // We had a bad argument to a plugin
        else if (name.equals(BADARGUMENT_ELEMENT)) {
        if(Configuration.getMode() <= threshold) {
            String args[] = new String[1];
            args[0] = element.getAttribute(ARGUMENT_ATTRIBUTE);    
            queue.add(new GShellEvent(event.getSource(), 0, event.getType(), Dictionary.get("GShell.BadArgument", args), event.getStatus()));
            args = null;
        }
        }
        // We had a bad argument value to a plugin
        else if (name.equals(BADARGUMENTVALUE_ELEMENT)) {
        if(Configuration.getMode() <= threshold) {
            String args[] = new String[1];
            args[0] = element.getAttribute(ARGUMENT_ATTRIBUTE);    
            queue.add(new GShellEvent(event.getSource(), 0, event.getType(), Dictionary.get("GShell.BadArgumentValue", args), event.getStatus()));
            args = null;
        }
        }
        // And this one tellsa us the plugin
        else if (name.equals(BADPLUGIN_ELEMENT)) {
        if(Configuration.getMode() <= threshold) {
            String args[] = new String[1];
            args[0] = element.getAttribute(PLUGIN_ATTRIBUTE);    
            queue.add(new GShellEvent(event.getSource(), 0, event.getType(), Dictionary.get("GShell.BadPluginOptions", args), event.getStatus()));
            args = null;
        }
        }
        else if (name.equals(WARNING_ELEMENT)) {
        if(Configuration.getMode() <= threshold) {
            String args[] = new String[1];
            args[0] = element.getAttribute(PLUGIN_ATTRIBUTE);
            String reason = element.getAttribute(REASON_ATTRIBUTE);
            queue.add(new GShellEvent(event.getSource(), 0, event.getType(), Dictionary.get("GShell.Import.Warning", args) + " " + reason, event.getStatus()));
            args = null;
        }
        }
        
        
        // Or the import complete element
        else if(name.equals(IMPORTCOMPLETE_ELEMENT)) {
        // Set the next progress
        setValue(next_progress_value);
        // If we are at lower mode settings fire a new 'dumbed down' event
        if(Configuration.getMode() <= threshold) {
            queue.add(new GShellEvent(event.getSource(), 0, event.getType(), Dictionary.get("GShell.Import.ImportComplete1"), event.getStatus()));
            String considered_str = element.getAttribute(CONSIDERED_ATTRIBUTE);

            if (considered_str.equals(StaticStrings.ONE_CHARACTER)) {
            queue.add(new GShellEvent(event.getSource(), 0, event.getType(), Dictionary.get("GShell.Import.ImportComplete2.1"), event.getStatus()));    
            } else {
            queue.add(new GShellEvent(event.getSource(), 0, event.getType(), Dictionary.get("GShell.Import.ImportComplete2", considered_str), event.getStatus()));
            }
            considered_str = null;
            // The number of documents processed
            String processed_str = element.getAttribute(PROCESSED_ATTRIBUTE);
            if(!processed_str.equals(StaticStrings.ZERO_CHARACTER)) {
            if(processed_str.equals(StaticStrings.ONE_CHARACTER)) {
                queue.add(new GShellEvent(event.getSource(), 0, event.getType(), "    "+Dictionary.get("GShell.Import.ImportComplete.Processed.1"), event.getStatus()));
            }
            else {
                queue.add(new GShellEvent(event.getSource(), 0, event.getType(), "    "+Dictionary.get("GShell.Import.ImportComplete.Processed", processed_str), event.getStatus()));
            }
            }
            // Try to set num docs
            try {
            num_docs = Integer.parseInt(processed_str);
            }
            catch(Exception exception) {
            num_docs = 0;
            }  
            processed_str = null;

            // The number of documents blocked
            String blocked_str = element.getAttribute(BLOCKED_ATTRIBUTE);
            if(!blocked_str.equals(StaticStrings.ZERO_CHARACTER)) {
            if (blocked_str.equals(StaticStrings.ONE_CHARACTER)) {
                queue.add(new GShellEvent(event.getSource(), 0, event.getType(), "    "+Dictionary.get("GShell.Import.ImportComplete.Blocked.1"), event.getStatus()));
            } else {
                queue.add(new GShellEvent(event.getSource(), 0, event.getType(), "    "+Dictionary.get("GShell.Import.ImportComplete.Blocked", blocked_str), event.getStatus()));
            }
            }
            blocked_str = null;
            
            // The number of documents ignored
            String ignored_str = element.getAttribute(IGNORED_ATTRIBUTE);
            if(!ignored_str.equals(StaticStrings.ZERO_CHARACTER)) {
            if(ignored_str.equals(StaticStrings.ONE_CHARACTER)) {
                queue.add(new GShellEvent(event.getSource(), 0, event.getType(), "    "+Dictionary.get("GShell.Import.ImportComplete.Ignored.1"), event.getStatus()));
            } else {
                queue.add(new GShellEvent(event.getSource(), 0, event.getType(), "    "+Dictionary.get("GShell.Import.ImportComplete.Ignored", ignored_str), event.getStatus()));
            }
            }
            ignored_str = null;
            
            // The number of documents failed
            String failed_str = element.getAttribute(FAILED_ATTRIBUTE);
            if(!failed_str.equals(StaticStrings.ZERO_CHARACTER)) {
            if(failed_str.equals(StaticStrings.ONE_CHARACTER)) {
                queue.add(new GShellEvent(event.getSource(), 0, event.getType(), "    "+Dictionary.get("GShell.Import.ImportComplete.Failed.1"), event.getStatus()));
            } else {
                queue.add(new GShellEvent(event.getSource(), 0, event.getType(), "    "+Dictionary.get("GShell.Import.ImportComplete.Failed", failed_str), event.getStatus()));
            }
            }
            failed_str = null;
            // Message is finally ready
            queue.add(new GShellEvent(event.getSource(), 0, event.getType(), Dictionary.get("GShell.Import.ImportComplete3"), event.getStatus()));
        } // if mode below threshhold
        else {
            // Try to set num docs
            String processed_str = element.getAttribute(PROCESSED_ATTRIBUTE);
            try {
            num_docs = Integer.parseInt(processed_str);
            }
            catch(Exception exception) {
            num_docs = 0;
            }
        }
        }
        else {
        // Veto it
        event.veto();
        }
    } // GLI output
    else if(Configuration.getMode() <= threshold) {
        event.veto();
    }
    }

    public void reset() {
    setValue(MINIMUM);
    progress_bar.setString(null);
    progress_bar.updateUI();
    }

    public void saving() {
    progress_bar.setString(Dictionary.get("SaveProgressDialog.Title"));
    showing_string = true;
    }

    /** Since the creator of this process monitor is actually in the GUI, this class provides the simpliest way to send a cancel process message between the two.
     * @param state The desired state of the stop flag as a <strong>boolean</strong>.
     */
    public synchronized void setStop(boolean state) {
    this.stop = state;
    }

    /** This method resets this monitor to the start, reseting the process parsing and progress bar.
     */
    public void start() {
    stop = false;
    setValue(MINIMUM);
    progress_bar.setString(Dictionary.get("FileActions.Calculating_Size"));
    showing_string = true;
    extracted_file_count = 0;
    file_count = 0;
    next_progress_value = -1;
    num_docs = -1;
    // !! HACK: This is multiplied by 2 because the metadata_read now outputs a <File> tag also
    // Ideally the metadata read pass would be completely separate and the GLI could work out the number
    //   of documents accurately and provide a much better progress bar
    num_expected_docs = Gatherer.c_man.getCollection().getCount() * 2;
    }

    /** This method indicates the process is complete. 
     */
    public void stop() {
    setValue(MAXIMUM);
    }

    int previous_value = -1;

    private void setValue(int value)
    {
    // Don't let the value go higher than the maximum
    if (value > MAXIMUM) {
        DebugStream.println("Error: Tried to set progress bar higher than maximum!\n");
        return;
    }
    // Don't let the value go backwards
    if (value != 0 && value < previous_value) {
        DebugStream.println("Error: Tried to set progress bar smaller than previous!\n");
        return;
    }

    progress_bar.setValue(value);
    shared_progress_bar.setValue(value);
    previous_value = value;
    }
}